From 25985e901bf5020f0fb6d9e7857d6c081d47e1fe Mon Sep 17 00:00:00 2001 From: Taus Brock-Nannestad Date: Thu, 26 Sep 2019 17:53:10 +0200 Subject: [PATCH 001/148] Python: Remove a few false positives from `py/unused-import`. --- python/ql/src/Imports/UnusedImport.ql | 25 ++++++++++++++++--- .../Imports/unused/imports_test.py | 14 +++++++++++ 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/python/ql/src/Imports/UnusedImport.ql b/python/ql/src/Imports/UnusedImport.ql index 6d927550c0c..d87b0efd254 100644 --- a/python/ql/src/Imports/UnusedImport.ql +++ b/python/ql/src/Imports/UnusedImport.ql @@ -58,16 +58,28 @@ predicate imported_module_used_in_doctest(Import imp) { } predicate imported_module_used_in_typehint(Import imp) { - exists(string modname | - imp.getAName().getAsname().(Name).getId() = modname - and + exists(string modname, Location loc | + imp.getAName().getAsname().(Name).getId() = modname and + loc.getFile() = imp.getScope().(Module).getFile() + | /* Look for typehints containing the patterns: * # type: …name… */ exists(Comment typehint | - typehint.getLocation().getFile() = imp.getScope().(Module).getFile() and + loc = typehint.getLocation() and typehint.getText().regexpMatch("# type:.*" + modname + ".*") ) + or + // Type hint is inside a string annotation, as needed for forward references + exists(string typehint, Expr annotation | + annotation = any(Arguments a).getAnAnnotation() + or + annotation = any(AnnAssign a).getAnnotation() + | + annotation.pointsTo(Value::forString(typehint)) and + loc = annotation.getLocation() and + typehint.regexpMatch(".*\\b" + modname + "\\b.*") + ) ) } @@ -100,6 +112,11 @@ predicate unused_import(Import imp, Variable name) { not imported_module_used_in_doctest(imp) and not imported_module_used_in_typehint(imp) + and + /* Only consider import statements that actually point-to something (possibly an unknown module). + * If this is not the case, it's likely that the import statement never gets executed. + */ + imp.getAName().getValue().pointsTo(_) } diff --git a/python/ql/test/query-tests/Imports/unused/imports_test.py b/python/ql/test/query-tests/Imports/unused/imports_test.py index a1b457f0422..cf2024ac183 100644 --- a/python/ql/test/query-tests/Imports/unused/imports_test.py +++ b/python/ql/test/query-tests/Imports/unused/imports_test.py @@ -76,3 +76,17 @@ import typing foo = None # type: typing.Optional[int] + +# OK since the import statement is never executed +if False: + import never_imported + +# OK since the imported module is used in a forward-referenced type annotation. +import only_used_in_parameter_annotation + +def func(x: 'Optional[only_used_in_parameter_annotation]'): + pass + +import only_used_in_annotated_assignment + +var : 'Optional[only_used_in_annotated_assignment]' = 5 From 4341e88fc4dbeeb24aae204c0288693d3e87bd04 Mon Sep 17 00:00:00 2001 From: Taus Brock-Nannestad Date: Fri, 27 Sep 2019 13:03:27 +0200 Subject: [PATCH 002/148] Python: Clean up comments in preparation for autoformat. --- python/ql/src/Imports/UnusedImport.ql | 29 ++++++++++++--------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/python/ql/src/Imports/UnusedImport.ql b/python/ql/src/Imports/UnusedImport.ql index d87b0efd254..951e7f87513 100644 --- a/python/ql/src/Imports/UnusedImport.ql +++ b/python/ql/src/Imports/UnusedImport.ql @@ -20,7 +20,7 @@ predicate global_name_used(Module m, Variable name) { u.getEnclosingModule() = m ) or - /* A use of an undefined class local variable, will use the global variable */ + // A use of an undefined class local variable, will use the global variable exists(Name u, LocalVariable v | u.uses(v) and v.getId() = name.getId() and @@ -33,10 +33,10 @@ predicate global_name_used(Module m, Variable name) { predicate all_not_understood(Module m) { exists(GlobalVariable a | a.getId() = "__all__" and a.getScope() = m | - /* __all__ is not defined as a simple list */ + // `__all__` is not defined as a simple list not m.declaredInAll(_) or - /* __all__ is modified */ + // `__all__` is modified exists(Call c | c.getFunc().(Attribute).getObject() = a.getALoad()) ) } @@ -45,10 +45,9 @@ predicate imported_module_used_in_doctest(Import imp) { exists(string modname | imp.getAName().getAsname().(Name).getId() = modname and - /* Look for doctests containing the patterns: - * >>> …name… - * ... …name… - */ + // Look for doctests containing the patterns: + // >>> …name… + // ... …name… exists(StrConst doc | doc.getEnclosingModule() = imp.getScope() and doc.isDocString() and @@ -62,9 +61,8 @@ predicate imported_module_used_in_typehint(Import imp) { imp.getAName().getAsname().(Name).getId() = modname and loc.getFile() = imp.getScope().(Module).getFile() | - /* Look for typehints containing the patterns: - * # type: …name… - */ + // Look for type hints containing the patterns: + // # type: …name… exists(Comment typehint | loc = typehint.getLocation() and typehint.getText().regexpMatch("# type:.*" + modname + ".*") @@ -95,10 +93,10 @@ predicate unused_import(Import imp, Variable name) { and not global_name_used(imp.getScope(), name) and - /* Imports in __init__.py are used to force module loading */ + // Imports in `__init__.py` are used to force module loading not imp.getEnclosingModule().isPackageInit() and - /* Name may be imported for use in epytext documentation */ + // Name may be imported for use in epytext documentation not exists(Comment cmt | cmt.getText().matches("%L{" + name.getId() + "}%") | cmt.getLocation().getFile() = imp.getLocation().getFile() @@ -106,16 +104,15 @@ predicate unused_import(Import imp, Variable name) { and not name_acceptable_for_unused_variable(name) and - /* Assume that opaque `__all__` includes imported module */ + // Assume that opaque `__all__` includes imported module not all_not_understood(imp.getEnclosingModule()) and not imported_module_used_in_doctest(imp) and not imported_module_used_in_typehint(imp) and - /* Only consider import statements that actually point-to something (possibly an unknown module). - * If this is not the case, it's likely that the import statement never gets executed. - */ + // Only consider import statements that actually point-to something (possibly an unknown module). + // If this is not the case, it's likely that the import statement never gets executed. imp.getAName().getValue().pointsTo(_) } From 9878e4fe263f54120fe282810274377a06f9cc01 Mon Sep 17 00:00:00 2001 From: Taus Brock-Nannestad Date: Fri, 27 Sep 2019 13:04:17 +0200 Subject: [PATCH 003/148] Python: Apply four-space autoformat. --- python/ql/src/Imports/UnusedImport.ql | 53 +++++++++------------------ 1 file changed, 18 insertions(+), 35 deletions(-) diff --git a/python/ql/src/Imports/UnusedImport.ql b/python/ql/src/Imports/UnusedImport.ql index 951e7f87513..3ac04e2e4d2 100644 --- a/python/ql/src/Imports/UnusedImport.ql +++ b/python/ql/src/Imports/UnusedImport.ql @@ -14,7 +14,7 @@ import python import Variables.Definition predicate global_name_used(Module m, Variable name) { - exists (Name u, GlobalVariable v | + exists(Name u, GlobalVariable v | u.uses(v) and v.getId() = name.getId() and u.getEnclosingModule() = m @@ -31,8 +31,7 @@ predicate global_name_used(Module m, Variable name) { /** Holds if a module has `__all__` but we don't understand it */ predicate all_not_understood(Module m) { - exists(GlobalVariable a | - a.getId() = "__all__" and a.getScope() = m | + exists(GlobalVariable a | a.getId() = "__all__" and a.getScope() = m | // `__all__` is not defined as a simple list not m.declaredInAll(_) or @@ -43,8 +42,7 @@ predicate all_not_understood(Module m) { predicate imported_module_used_in_doctest(Import imp) { exists(string modname | - imp.getAName().getAsname().(Name).getId() = modname - and + imp.getAName().getAsname().(Name).getId() = modname and // Look for doctests containing the patterns: // >>> …name… // ... …name… @@ -60,7 +58,7 @@ predicate imported_module_used_in_typehint(Import imp) { exists(string modname, Location loc | imp.getAName().getAsname().(Name).getId() = modname and loc.getFile() = imp.getScope().(Module).getFile() - | + | // Look for type hints containing the patterns: // # type: …name… exists(Comment typehint | @@ -73,7 +71,7 @@ predicate imported_module_used_in_typehint(Import imp) { annotation = any(Arguments a).getAnAnnotation() or annotation = any(AnnAssign a).getAnnotation() - | + | annotation.pointsTo(Value::forString(typehint)) and loc = annotation.getLocation() and typehint.regexpMatch(".*\\b" + modname + "\\b.*") @@ -81,43 +79,28 @@ predicate imported_module_used_in_typehint(Import imp) { ) } - predicate unused_import(Import imp, Variable name) { - ((Name)imp.getAName().getAsname()).getVariable() = name - and - not imp.getAnImportedModuleName() = "__future__" - and - not imp.getEnclosingModule().declaredInAll(name.getId()) - and - imp.getScope() = imp.getEnclosingModule() - and - not global_name_used(imp.getScope(), name) - and + imp.getAName().getAsname().(Name).getVariable() = name and + not imp.getAnImportedModuleName() = "__future__" and + not imp.getEnclosingModule().declaredInAll(name.getId()) and + imp.getScope() = imp.getEnclosingModule() and + not global_name_used(imp.getScope(), name) and // Imports in `__init__.py` are used to force module loading - not imp.getEnclosingModule().isPackageInit() - and + not imp.getEnclosingModule().isPackageInit() and // Name may be imported for use in epytext documentation - not exists(Comment cmt | - cmt.getText().matches("%L{" + name.getId() + "}%") | + not exists(Comment cmt | cmt.getText().matches("%L{" + name.getId() + "}%") | cmt.getLocation().getFile() = imp.getLocation().getFile() - ) - and - not name_acceptable_for_unused_variable(name) - and + ) and + not name_acceptable_for_unused_variable(name) and // Assume that opaque `__all__` includes imported module - not all_not_understood(imp.getEnclosingModule()) - and - not imported_module_used_in_doctest(imp) - and - not imported_module_used_in_typehint(imp) - and - // Only consider import statements that actually point-to something (possibly an unknown module). + not all_not_understood(imp.getEnclosingModule()) and + not imported_module_used_in_doctest(imp) and + not imported_module_used_in_typehint(imp) and + // Only consider import statements that actually point-to something (possibly an unknown module). // If this is not the case, it's likely that the import statement never gets executed. imp.getAName().getValue().pointsTo(_) } - from Stmt s, Variable name where unused_import(s, name) select s, "Import of '" + name.getId() + "' is not used." - From 0ccc0057f95314fa179a80216558bb6704eb1a94 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Tue, 8 Oct 2019 15:04:11 +0200 Subject: [PATCH 004/148] add Deferred model to Promises.qll --- change-notes/1.23/analysis-javascript.md | 1 + .../ql/src/semmle/javascript/Promises.qll | 33 +++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/change-notes/1.23/analysis-javascript.md b/change-notes/1.23/analysis-javascript.md index 04b5e6cb541..87b12c19230 100644 --- a/change-notes/1.23/analysis-javascript.md +++ b/change-notes/1.23/analysis-javascript.md @@ -38,3 +38,4 @@ ## Changes to QL libraries * `Expr.getDocumentation()` now handles chain assignments. +* Added `Deferred` as a promise library in Promises.qll diff --git a/javascript/ql/src/semmle/javascript/Promises.qll b/javascript/ql/src/semmle/javascript/Promises.qll index 858064e85a0..794a866aee2 100644 --- a/javascript/ql/src/semmle/javascript/Promises.qll +++ b/javascript/ql/src/semmle/javascript/Promises.qll @@ -32,6 +32,39 @@ module Bluebird { } } +/** + * Provides classes for working with various Deferred implementations + */ +module Deferred { + private DataFlow::SourceNode deferred() { + exists(VarAccess var, DataFlow::NewNode instantiation | + var.getName() = "Deferred" and + result = DataFlow::exprNode(var).getALocalSource() and + // Sanity check that result really is a Deferred implementation + instantiation = result.getAnInstantiation() and + exists(instantiation.getAMemberCall("resolve")) + ) + } + + /** + * A promise object created by a Deferred constructor + */ + private class DeferredPromiseDefinition extends PromiseDefinition, DataFlow::NewNode { + DeferredPromiseDefinition() { this = deferred().getAnInstantiation() } + + override DataFlow::FunctionNode getExecutor() { result = getCallback(0) } + } + + /** + * A resolved promise created by a `new Deferred().resolve()` call. + */ + class ResolvedDeferredPromiseDefinition extends ResolvedPromiseDefinition { + ResolvedDeferredPromiseDefinition() { this = any(DeferredPromiseDefinition def).getAMemberCall("resolve") } + + override DataFlow::Node getValue() { result = getArgument(0) } + } +} + /** * Provides classes for working with the `q` library (https://github.com/kriskowal/q). */ From 411ed702fb1eabc9f308a34abe6105b688d12c36 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Wed, 9 Oct 2019 13:50:12 +0200 Subject: [PATCH 005/148] change change-notes --- change-notes/1.23/analysis-javascript.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/change-notes/1.23/analysis-javascript.md b/change-notes/1.23/analysis-javascript.md index 87b12c19230..70d903cb944 100644 --- a/change-notes/1.23/analysis-javascript.md +++ b/change-notes/1.23/analysis-javascript.md @@ -9,6 +9,7 @@ - [rate-limiter-flexible](https://www.npmjs.com/package/rate-limiter-flexible) * The call graph has been improved to resolve method calls in more cases. This may produce more security alerts. +* Promises derived from a Deferred object are now recognized. ## New queries @@ -38,4 +39,3 @@ ## Changes to QL libraries * `Expr.getDocumentation()` now handles chain assignments. -* Added `Deferred` as a promise library in Promises.qll From c7eb0f17a9e35a0d9e0be951373df7bcad9c4f87 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Wed, 9 Oct 2019 13:59:00 +0200 Subject: [PATCH 006/148] add TaintTracking test for new Deferred model --- .../TaintTracking/BasicTaintTracking.expected | 1 + .../ql/test/library-tests/TaintTracking/promise.js | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/javascript/ql/test/library-tests/TaintTracking/BasicTaintTracking.expected b/javascript/ql/test/library-tests/TaintTracking/BasicTaintTracking.expected index 2722f67d6fd..77592a2e855 100644 --- a/javascript/ql/test/library-tests/TaintTracking/BasicTaintTracking.expected +++ b/javascript/ql/test/library-tests/TaintTracking/BasicTaintTracking.expected @@ -69,6 +69,7 @@ typeInferenceMismatch | promise.js:5:25:5:32 | source() | promise.js:5:8:5:33 | bluebir ... urce()) | | promise.js:10:24:10:31 | source() | promise.js:10:8:10:32 | Promise ... urce()) | | promise.js:12:20:12:27 | source() | promise.js:13:8:13:23 | resolver.promise | +| promise.js:22:23:22:30 | source() | promise.js:22:7:22:31 | promise ... urce()) | | sanitizer-guards.js:2:11:2:18 | source() | sanitizer-guards.js:4:8:4:8 | x | | sanitizer-guards.js:13:14:13:21 | source() | sanitizer-guards.js:15:10:15:15 | this.x | | sanitizer-guards.js:13:14:13:21 | source() | sanitizer-guards.js:21:14:21:19 | this.x | diff --git a/javascript/ql/test/library-tests/TaintTracking/promise.js b/javascript/ql/test/library-tests/TaintTracking/promise.js index cd5351720ba..87e89451d99 100644 --- a/javascript/ql/test/library-tests/TaintTracking/promise.js +++ b/javascript/ql/test/library-tests/TaintTracking/promise.js @@ -12,3 +12,12 @@ function closure() { resolver.resolve(source()); sink(resolver.promise); // NOT OK } + +class Deferred { + +} + +function deferred() { + var promise = new Deferred(); + sink(promise.resolve(source())); // NOT OK +} \ No newline at end of file From 4ec825b5b6f3dbb1ba8c5e20bf70963772c9bda0 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Wed, 9 Oct 2019 16:18:04 +0200 Subject: [PATCH 007/148] made model of Deferred more precise --- .../ql/src/semmle/javascript/Promises.qll | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/javascript/ql/src/semmle/javascript/Promises.qll b/javascript/ql/src/semmle/javascript/Promises.qll index 794a866aee2..362bb2f5063 100644 --- a/javascript/ql/src/semmle/javascript/Promises.qll +++ b/javascript/ql/src/semmle/javascript/Promises.qll @@ -37,10 +37,23 @@ module Bluebird { */ module Deferred { private DataFlow::SourceNode deferred() { - exists(VarAccess var, DataFlow::NewNode instantiation | - var.getName() = "Deferred" and - result = DataFlow::exprNode(var).getALocalSource() and - // Sanity check that result really is a Deferred implementation + ( + exists(Variable var | + var.getName() = "Deferred" and + (var.getADeclaration() instanceof LocalNamespaceDecl or var.getScope() instanceof GlobalScope) and + result = DataFlow::valueNode(var.getADefinition()) + ) + or + result.(DataFlow::ParameterNode).getName() = "Deferred" + or + exists(Function f | + f.getName() = "Deferred" and + result = DataFlow::valueNode(f) + ) + ) + and + // Sanity check that it is a Deferred implementation + exists(DataFlow::NewNode instantiation | instantiation = result.getAnInstantiation() and exists(instantiation.getAMemberCall("resolve")) ) From 0a6b343820e43fd47efa61cf8acec0deab53c458 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Thu, 10 Oct 2019 11:50:34 +0200 Subject: [PATCH 008/148] add "class Deferred{...}" as potential Deferred implementation to fix the tests --- javascript/ql/src/semmle/javascript/Promises.qll | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/javascript/ql/src/semmle/javascript/Promises.qll b/javascript/ql/src/semmle/javascript/Promises.qll index 362bb2f5063..488adebab6e 100644 --- a/javascript/ql/src/semmle/javascript/Promises.qll +++ b/javascript/ql/src/semmle/javascript/Promises.qll @@ -46,10 +46,15 @@ module Deferred { or result.(DataFlow::ParameterNode).getName() = "Deferred" or - exists(Function f | + exists(Function f | f.getName() = "Deferred" and result = DataFlow::valueNode(f) ) + or + exists(ClassDefinition c | + c.getName() = "Deferred" and + result = DataFlow::valueNode(c) + ) ) and // Sanity check that it is a Deferred implementation From 31009d979d323eb496530ae99c9f719ebed67272 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Fri, 11 Oct 2019 12:04:34 +0200 Subject: [PATCH 009/148] add type tracking to detect instances --- .../ql/src/semmle/javascript/Promises.qll | 73 ++++++++++++------- .../library-tests/TaintTracking/promise.js | 2 + 2 files changed, 50 insertions(+), 25 deletions(-) diff --git a/javascript/ql/src/semmle/javascript/Promises.qll b/javascript/ql/src/semmle/javascript/Promises.qll index 488adebab6e..13eb1e8114e 100644 --- a/javascript/ql/src/semmle/javascript/Promises.qll +++ b/javascript/ql/src/semmle/javascript/Promises.qll @@ -36,39 +36,60 @@ module Bluebird { * Provides classes for working with various Deferred implementations */ module Deferred { - private DataFlow::SourceNode deferred() { - ( - exists(Variable var | + class DeferredClass extends DataFlow::SourceNode { + DeferredClass() { + exists(Variable var | var.getName() = "Deferred" and - (var.getADeclaration() instanceof LocalNamespaceDecl or var.getScope() instanceof GlobalScope) and - result = DataFlow::valueNode(var.getADefinition()) - ) - or - result.(DataFlow::ParameterNode).getName() = "Deferred" - or - exists(Function f | - f.getName() = "Deferred" and - result = DataFlow::valueNode(f) + ( + var.getADeclaration() instanceof LocalNamespaceDecl or + var.getScope() instanceof GlobalScope + ) and + this = DataFlow::valueNode(var.getADefinition()) ) or - exists(ClassDefinition c | - c.getName() = "Deferred" and - result = DataFlow::valueNode(c) + this.(DataFlow::ParameterNode).getName() = "Deferred" + or + exists(Function f | + f.getName() = "Deferred" and + this = DataFlow::valueNode(f) ) - ) - and - // Sanity check that it is a Deferred implementation - exists(DataFlow::NewNode instantiation | - instantiation = result.getAnInstantiation() and - exists(instantiation.getAMemberCall("resolve")) - ) + or + exists(ClassDefinition c | + c.getName() = "Deferred" and + this = DataFlow::valueNode(c) + ) + } + } + + class DeferredInstance extends DataFlow::NewNode { + DeferredClass deferredClass; + + DeferredInstance() { this = deferredClass.getAnInstantiation() } + + private DataFlow::SourceNode ref(DataFlow::TypeTracker t) { + t.start() and + result = this + or + exists(DataFlow::TypeTracker t2 | result = ref(t2).track(t2, t)) + } + + DeferredClass getDeferredClass() { result = deferredClass } + + DataFlow::CallNode getPromiseMemberCall(string methodName) { + result = ref(DataFlow::TypeTracker::end()).getAMemberCall(methodName) + } } /** * A promise object created by a Deferred constructor */ - private class DeferredPromiseDefinition extends PromiseDefinition, DataFlow::NewNode { - DeferredPromiseDefinition() { this = deferred().getAnInstantiation() } + private class DeferredPromiseDefinition extends PromiseDefinition, DeferredInstance { + DeferredPromiseDefinition() { + this = any(DeferredClass c | + exists(any(DeferredInstance i | i.getDeferredClass() = c).getPromiseMemberCall("resolve")) and + exists(any(DeferredInstance i | i.getDeferredClass() = c).getPromiseMemberCall("reject")) + ).getAnInstantiation() + } override DataFlow::FunctionNode getExecutor() { result = getCallback(0) } } @@ -77,7 +98,9 @@ module Deferred { * A resolved promise created by a `new Deferred().resolve()` call. */ class ResolvedDeferredPromiseDefinition extends ResolvedPromiseDefinition { - ResolvedDeferredPromiseDefinition() { this = any(DeferredPromiseDefinition def).getAMemberCall("resolve") } + ResolvedDeferredPromiseDefinition() { + this = any(DeferredPromiseDefinition def).getPromiseMemberCall("resolve") + } override DataFlow::Node getValue() { result = getArgument(0) } } diff --git a/javascript/ql/test/library-tests/TaintTracking/promise.js b/javascript/ql/test/library-tests/TaintTracking/promise.js index 87e89451d99..d16e57241e8 100644 --- a/javascript/ql/test/library-tests/TaintTracking/promise.js +++ b/javascript/ql/test/library-tests/TaintTracking/promise.js @@ -20,4 +20,6 @@ class Deferred { function deferred() { var promise = new Deferred(); sink(promise.resolve(source())); // NOT OK + + new Deferred().reject("foo") // <- a reject has to exist. } \ No newline at end of file From 592cb18bf486ba6f8b37abf291516e365d4f8e25 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Fri, 4 Oct 2019 16:28:38 +0200 Subject: [PATCH 010/148] add array callbacks to useOfReturnlessFunction query --- .../ql/src/Expressions/ExprHasNoEffect.ql | 115 +---------------- .../ql/src/Expressions/ExprHasNoEffect.qll | 121 ++++++++++++++++++ .../src/Statements/UseOfReturnlessFunction.ql | 63 ++++++++- 3 files changed, 178 insertions(+), 121 deletions(-) diff --git a/javascript/ql/src/Expressions/ExprHasNoEffect.ql b/javascript/ql/src/Expressions/ExprHasNoEffect.ql index 14024ec8f38..929e84616f0 100644 --- a/javascript/ql/src/Expressions/ExprHasNoEffect.ql +++ b/javascript/ql/src/Expressions/ExprHasNoEffect.ql @@ -13,122 +13,9 @@ */ import javascript -import DOMProperties -import semmle.javascript.frameworks.xUnit -import semmle.javascript.RestrictedLocations import ExprHasNoEffect -/** - * Holds if `e` is of the form `x;` or `e.p;` and has a JSDoc comment containing a tag. - * In that case, it is probably meant as a declaration and shouldn't be flagged by this query. - * - * This will still flag cases where the JSDoc comment contains no tag at all (and hence carries - * no semantic information), and expression statements with an ordinary (non-JSDoc) comment - * attached to them. - */ -predicate isDeclaration(Expr e) { - (e instanceof VarAccess or e instanceof PropAccess) and - exists(e.getParent().(ExprStmt).getDocumentation().getATag()) -} - -/** - * Holds if there exists a getter for a property called `name` anywhere in the program. - */ -predicate isGetterProperty(string name) { - // there is a call of the form `Object.defineProperty(..., name, descriptor)` ... - exists(CallToObjectDefineProperty defProp | name = defProp.getPropertyName() | - // ... where `descriptor` defines a getter - defProp.hasPropertyAttributeWrite("get", _) - or - // ... where `descriptor` may define a getter - exists(DataFlow::SourceNode descriptor | descriptor.flowsTo(defProp.getPropertyDescriptor()) | - descriptor.isIncomplete(_) - or - // minimal escape analysis for the descriptor - exists(DataFlow::InvokeNode invk | - not invk = defProp and - descriptor.flowsTo(invk.getAnArgument()) - ) - ) - ) - or - // there is an object expression with a getter property `name` - exists(ObjectExpr obj | obj.getPropertyByName(name) instanceof PropertyGetter) -} - -/** - * A property access that may invoke a getter. - */ -class GetterPropertyAccess extends PropAccess { - override predicate isImpure() { isGetterProperty(getPropertyName()) } -} - -/** - * Holds if `c` is an indirect eval call of the form `(dummy, eval)(...)`, where - * `dummy` is some expression whose value is discarded, and which simply - * exists to prevent the call from being interpreted as a direct eval. - */ -predicate isIndirectEval(CallExpr c, Expr dummy) { - exists(SeqExpr seq | seq = c.getCallee().stripParens() | - dummy = seq.getOperand(0) and - seq.getOperand(1).(GlobalVarAccess).getName() = "eval" and - seq.getNumOperands() = 2 - ) -} - -/** - * Holds if `c` is a call of the form `(dummy, e[p])(...)`, where `dummy` is - * some expression whose value is discarded, and which simply exists - * to prevent the call from being interpreted as a method call. - */ -predicate isReceiverSuppressingCall(CallExpr c, Expr dummy, PropAccess callee) { - exists(SeqExpr seq | seq = c.getCallee().stripParens() | - dummy = seq.getOperand(0) and - seq.getOperand(1) = callee and - seq.getNumOperands() = 2 - ) -} - -/** - * Holds if evaluating `e` has no side effects (except potentially allocating - * and initializing a new object). - * - * For calls, we do not check whether their arguments have any side effects: - * even if they do, the call itself is useless and should be flagged by this - * query. - */ -predicate noSideEffects(Expr e) { - e.isPure() - or - // `new Error(...)`, `new SyntaxError(...)`, etc. - forex(Function f | f = e.flow().(DataFlow::NewNode).getACallee() | - f.(ExternalType).getASupertype*().getName() = "Error" - ) -} from Expr e -where - noSideEffects(e) and - inVoidContext(e) and - // disregard pure expressions wrapped in a void(...) - not e instanceof VoidExpr and - // filter out directives (unknown directives are handled by UnknownDirective.ql) - not exists(Directive d | e = d.getExpr()) and - // or about externs - not e.inExternsFile() and - // don't complain about declarations - not isDeclaration(e) and - // exclude DOM properties, which sometimes have magical auto-update properties - not isDOMProperty(e.(PropAccess).getPropertyName()) and - // exclude xUnit.js annotations - not e instanceof XUnitAnnotation and - // exclude common patterns that are most likely intentional - not isIndirectEval(_, e) and - not isReceiverSuppressingCall(_, e, _) and - // exclude anonymous function expressions as statements; these can only arise - // from a syntax error we already flag - not exists(FunctionExpr fe, ExprStmt es | fe = e | - fe = es.getExpr() and - not exists(fe.getName()) - ) +where hasNoEffect(e) select e.(FirstLineOf), "This expression has no effect." diff --git a/javascript/ql/src/Expressions/ExprHasNoEffect.qll b/javascript/ql/src/Expressions/ExprHasNoEffect.qll index 858f719ba0a..822bcbb26fa 100644 --- a/javascript/ql/src/Expressions/ExprHasNoEffect.qll +++ b/javascript/ql/src/Expressions/ExprHasNoEffect.qll @@ -3,6 +3,9 @@ */ import javascript +import DOMProperties +import semmle.javascript.frameworks.xUnit +import semmle.javascript.RestrictedLocations /** * Holds if `e` appears in a syntactic context where its value is discarded. @@ -37,3 +40,121 @@ predicate inVoidContext(Expr e) { or exists(LogicalBinaryExpr logical | e = logical.getRightOperand() and inVoidContext(logical)) } + + +/** + * Holds if `e` is of the form `x;` or `e.p;` and has a JSDoc comment containing a tag. + * In that case, it is probably meant as a declaration and shouldn't be flagged by this query. + * + * This will still flag cases where the JSDoc comment contains no tag at all (and hence carries + * no semantic information), and expression statements with an ordinary (non-JSDoc) comment + * attached to them. + */ +predicate isDeclaration(Expr e) { + (e instanceof VarAccess or e instanceof PropAccess) and + exists(e.getParent().(ExprStmt).getDocumentation().getATag()) +} + +/** + * Holds if there exists a getter for a property called `name` anywhere in the program. + */ +predicate isGetterProperty(string name) { + // there is a call of the form `Object.defineProperty(..., name, descriptor)` ... + exists(CallToObjectDefineProperty defProp | name = defProp.getPropertyName() | + // ... where `descriptor` defines a getter + defProp.hasPropertyAttributeWrite("get", _) + or + // ... where `descriptor` may define a getter + exists(DataFlow::SourceNode descriptor | descriptor.flowsTo(defProp.getPropertyDescriptor()) | + descriptor.isIncomplete(_) + or + // minimal escape analysis for the descriptor + exists(DataFlow::InvokeNode invk | + not invk = defProp and + descriptor.flowsTo(invk.getAnArgument()) + ) + ) + ) + or + // there is an object expression with a getter property `name` + exists(ObjectExpr obj | obj.getPropertyByName(name) instanceof PropertyGetter) +} + +/** + * A property access that may invoke a getter. + */ +class GetterPropertyAccess extends PropAccess { + override predicate isImpure() { isGetterProperty(getPropertyName()) } +} + +/** + * Holds if `c` is an indirect eval call of the form `(dummy, eval)(...)`, where + * `dummy` is some expression whose value is discarded, and which simply + * exists to prevent the call from being interpreted as a direct eval. + */ +predicate isIndirectEval(CallExpr c, Expr dummy) { + exists(SeqExpr seq | seq = c.getCallee().stripParens() | + dummy = seq.getOperand(0) and + seq.getOperand(1).(GlobalVarAccess).getName() = "eval" and + seq.getNumOperands() = 2 + ) +} + +/** + * Holds if `c` is a call of the form `(dummy, e[p])(...)`, where `dummy` is + * some expression whose value is discarded, and which simply exists + * to prevent the call from being interpreted as a method call. + */ +predicate isReceiverSuppressingCall(CallExpr c, Expr dummy, PropAccess callee) { + exists(SeqExpr seq | seq = c.getCallee().stripParens() | + dummy = seq.getOperand(0) and + seq.getOperand(1) = callee and + seq.getNumOperands() = 2 + ) +} + +/** + * Holds if evaluating `e` has no side effects (except potentially allocating + * and initializing a new object). + * + * For calls, we do not check whether their arguments have any side effects: + * even if they do, the call itself is useless and should be flagged by this + * query. + */ +predicate noSideEffects(Expr e) { + e.isPure() + or + // `new Error(...)`, `new SyntaxError(...)`, etc. + forex(Function f | f = e.flow().(DataFlow::NewNode).getACallee() | + f.(ExternalType).getASupertype*().getName() = "Error" + ) +} + +/** + * Holds if the expression `e` should be reported as having no effect. + */ +predicate hasNoEffect(Expr e) { + noSideEffects(e) and + inVoidContext(e) and + // disregard pure expressions wrapped in a void(...) + not e instanceof VoidExpr and + // filter out directives (unknown directives are handled by UnknownDirective.ql) + not exists(Directive d | e = d.getExpr()) and + // or about externs + not e.inExternsFile() and + // don't complain about declarations + not isDeclaration(e) and + // exclude DOM properties, which sometimes have magical auto-update properties + not isDOMProperty(e.(PropAccess).getPropertyName()) and + // exclude xUnit.js annotations + not e instanceof XUnitAnnotation and + // exclude common patterns that are most likely intentional + not isIndirectEval(_, e) and + not isReceiverSuppressingCall(_, e, _) and + // exclude anonymous function expressions as statements; these can only arise + // from a syntax error we already flag + not exists(FunctionExpr fe, ExprStmt es | fe = e | + fe = es.getExpr() and + not exists(fe.getName()) + ) +} \ No newline at end of file diff --git a/javascript/ql/src/Statements/UseOfReturnlessFunction.ql b/javascript/ql/src/Statements/UseOfReturnlessFunction.ql index c63127b83a3..61a485313e4 100644 --- a/javascript/ql/src/Statements/UseOfReturnlessFunction.ql +++ b/javascript/ql/src/Statements/UseOfReturnlessFunction.ql @@ -82,17 +82,66 @@ predicate alwaysThrows(Function f) { ) } -from DataFlow::CallNode call -where +predicate callToVoidFunction(DataFlow::CallNode call, Function func) { not call.isIndefinite(_) and - forex(Function f | f = call.getACallee() | + func = call.getACallee() and + forall(Function f | f = call.getACallee() | returnsVoid(f) and not isStub(f) and not alwaysThrows(f) + ) +} + +predicate hasNonVoidCallbackMethod(string name) { + name = "every" or + name = "filter" or + name = "find" or + name = "findIndex" or + name = "flatMap" or + name = "map" or + name = "reduce" or + name = "reduceRight" or + name = "some" or + name = "sort" +} + +DataFlow::SourceNode array(DataFlow::TypeTracker t) { + t.start() and result instanceof DataFlow::ArrayCreationNode + or + exists (DataFlow::TypeTracker t2 | + result = array(t2).track(t2, t) + ) +} + +DataFlow::SourceNode array() { result = array(DataFlow::TypeTracker::end()) } + +predicate voidArrayCallback(DataFlow::MethodCallNode call, Function func) { + hasNonVoidCallbackMethod(call.getMethodName()) and + func = call.getAnArgument().getALocalSource().asExpr() and + 1 = count(DataFlow::Node arg | arg = call.getAnArgument() and arg.getALocalSource().asExpr() instanceof Function) and + returnsVoid(func) and + not isStub(func) and + not alwaysThrows(func) and + ( + call.getReceiver().getALocalSource() = array() + or + call.getCalleeNode() instanceof LodashUnderscore::Member + ) +} + +from DataFlow::CallNode call, Function func, string name, string msg +where + ( + callToVoidFunction(call, func) and + msg = "the $@ does not return anything, yet the return value is used." and + name = "function " + call.getCalleeName() + or + voidArrayCallback(call, func) and + msg = "the $@ does not return anything, yet the return value from the call to " + call.getCalleeName() + "() is used." and + name = "callback function" ) and - not benignContext(call.asExpr()) and - + // Avoid double reporting from js/useless-expression + not hasNoEffect(func.getBodyStmt(func.getNumBodyStmt() - 1).(ExprStmt).getExpr()) and // anonymous one-shot closure. Those are used in weird ways and we ignore them. not oneshotClosure(call.asExpr()) select - call, "the function $@ does not return anything, yet the return value is used.", call.getACallee(), call.getCalleeName() - + call, msg, func, name From a7c1c34e1e48a311057ae223d545ed59a54439ee Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Fri, 11 Oct 2019 17:14:58 +0200 Subject: [PATCH 011/148] fix test output, and add new test for array callbacks --- .../UseOfReturnlessFunction.expected | 11 ++++++----- .../Statements/UseOfReturnlessFunction/tst.js | 7 +++++++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/javascript/ql/test/query-tests/Statements/UseOfReturnlessFunction/UseOfReturnlessFunction.expected b/javascript/ql/test/query-tests/Statements/UseOfReturnlessFunction/UseOfReturnlessFunction.expected index 500d900ff9e..0dc283d5edc 100644 --- a/javascript/ql/test/query-tests/Statements/UseOfReturnlessFunction/UseOfReturnlessFunction.expected +++ b/javascript/ql/test/query-tests/Statements/UseOfReturnlessFunction/UseOfReturnlessFunction.expected @@ -1,5 +1,6 @@ -| tst.js:20:17:20:33 | onlySideEffects() | the function $@ does not return anything, yet the return value is used. | tst.js:11:5:13:5 | functio ... )\\n } | onlySideEffects | -| tst.js:24:13:24:29 | onlySideEffects() | the function $@ does not return anything, yet the return value is used. | tst.js:11:5:13:5 | functio ... )\\n } | onlySideEffects | -| tst.js:30:20:30:36 | onlySideEffects() | the function $@ does not return anything, yet the return value is used. | tst.js:11:5:13:5 | functio ... )\\n } | onlySideEffects | -| tst.js:53:10:53:34 | bothOnl ... fects() | the function $@ does not return anything, yet the return value is used. | tst.js:11:5:13:5 | functio ... )\\n } | bothOnlyHaveSideEffects | -| tst.js:53:10:53:34 | bothOnl ... fects() | the function $@ does not return anything, yet the return value is used. | tst.js:48:2:50:5 | functio ... )\\n } | bothOnlyHaveSideEffects | +| tst.js:20:17:20:33 | onlySideEffects() | the $@ does not return anything, yet the return value is used. | tst.js:11:5:13:5 | functio ... )\\n } | function onlySideEffects | +| tst.js:24:13:24:29 | onlySideEffects() | the $@ does not return anything, yet the return value is used. | tst.js:11:5:13:5 | functio ... )\\n } | function onlySideEffects | +| tst.js:30:20:30:36 | onlySideEffects() | the $@ does not return anything, yet the return value is used. | tst.js:11:5:13:5 | functio ... )\\n } | function onlySideEffects | +| tst.js:53:10:53:34 | bothOnl ... fects() | the $@ does not return anything, yet the return value is used. | tst.js:11:5:13:5 | functio ... )\\n } | function bothOnlyHaveSideEffects | +| tst.js:53:10:53:34 | bothOnl ... fects() | the $@ does not return anything, yet the return value is used. | tst.js:48:2:50:5 | functio ... )\\n } | function bothOnlyHaveSideEffects | +| tst.js:76:12:76:46 | [1,2,3] ... n, 3)}) | the $@ does not return anything, yet the return value from the call to filter() is used. | tst.js:76:27:76:45 | n => {equals(n, 3)} | callback function | diff --git a/javascript/ql/test/query-tests/Statements/UseOfReturnlessFunction/tst.js b/javascript/ql/test/query-tests/Statements/UseOfReturnlessFunction/tst.js index 8bb49f7e761..1306c6e7f65 100644 --- a/javascript/ql/test/query-tests/Statements/UseOfReturnlessFunction/tst.js +++ b/javascript/ql/test/query-tests/Statements/UseOfReturnlessFunction/tst.js @@ -68,4 +68,11 @@ var h = returnsValue() || alwaysThrows(); // OK! console.log(h); + + function equals(x, y) { + return x === y; + } + + var foo = [1,2,3].filter(n => {equals(n, 3)}) // NOT OK! + console.log(foo); })(); \ No newline at end of file From 28056791a57d53a265cf6348d71e93a4aa60a4f8 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Mon, 14 Oct 2019 14:14:26 +0200 Subject: [PATCH 012/148] add .getALocalSource() when testing for lodash-members --- javascript/ql/src/Statements/UseOfReturnlessFunction.ql | 6 +++--- .../UseOfReturnlessFunction.expected | 1 + .../query-tests/Statements/UseOfReturnlessFunction/tst.js | 4 ++++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/javascript/ql/src/Statements/UseOfReturnlessFunction.ql b/javascript/ql/src/Statements/UseOfReturnlessFunction.ql index 61a485313e4..2d70cecb0d3 100644 --- a/javascript/ql/src/Statements/UseOfReturnlessFunction.ql +++ b/javascript/ql/src/Statements/UseOfReturnlessFunction.ql @@ -113,8 +113,8 @@ DataFlow::SourceNode array(DataFlow::TypeTracker t) { DataFlow::SourceNode array() { result = array(DataFlow::TypeTracker::end()) } -predicate voidArrayCallback(DataFlow::MethodCallNode call, Function func) { - hasNonVoidCallbackMethod(call.getMethodName()) and +predicate voidArrayCallback(DataFlow::CallNode call, Function func) { + hasNonVoidCallbackMethod(call.getCalleeName()) and func = call.getAnArgument().getALocalSource().asExpr() and 1 = count(DataFlow::Node arg | arg = call.getAnArgument() and arg.getALocalSource().asExpr() instanceof Function) and returnsVoid(func) and @@ -123,7 +123,7 @@ predicate voidArrayCallback(DataFlow::MethodCallNode call, Function func) { ( call.getReceiver().getALocalSource() = array() or - call.getCalleeNode() instanceof LodashUnderscore::Member + call.getCalleeNode().getALocalSource() instanceof LodashUnderscore::Member ) } diff --git a/javascript/ql/test/query-tests/Statements/UseOfReturnlessFunction/UseOfReturnlessFunction.expected b/javascript/ql/test/query-tests/Statements/UseOfReturnlessFunction/UseOfReturnlessFunction.expected index 0dc283d5edc..98ae4e3696c 100644 --- a/javascript/ql/test/query-tests/Statements/UseOfReturnlessFunction/UseOfReturnlessFunction.expected +++ b/javascript/ql/test/query-tests/Statements/UseOfReturnlessFunction/UseOfReturnlessFunction.expected @@ -4,3 +4,4 @@ | tst.js:53:10:53:34 | bothOnl ... fects() | the $@ does not return anything, yet the return value is used. | tst.js:11:5:13:5 | functio ... )\\n } | function bothOnlyHaveSideEffects | | tst.js:53:10:53:34 | bothOnl ... fects() | the $@ does not return anything, yet the return value is used. | tst.js:48:2:50:5 | functio ... )\\n } | function bothOnlyHaveSideEffects | | tst.js:76:12:76:46 | [1,2,3] ... n, 3)}) | the $@ does not return anything, yet the return value from the call to filter() is used. | tst.js:76:27:76:45 | n => {equals(n, 3)} | callback function | +| tst.js:80:12:80:50 | filter( ... 3) } ) | the $@ does not return anything, yet the return value from the call to filter() is used. | tst.js:80:28:80:48 | x => { ... x, 3) } | callback function | diff --git a/javascript/ql/test/query-tests/Statements/UseOfReturnlessFunction/tst.js b/javascript/ql/test/query-tests/Statements/UseOfReturnlessFunction/tst.js index 1306c6e7f65..2b523223e91 100644 --- a/javascript/ql/test/query-tests/Statements/UseOfReturnlessFunction/tst.js +++ b/javascript/ql/test/query-tests/Statements/UseOfReturnlessFunction/tst.js @@ -75,4 +75,8 @@ var foo = [1,2,3].filter(n => {equals(n, 3)}) // NOT OK! console.log(foo); + + import { filter } from 'lodash' + var bar = filter([1,2,4], x => { equals(x, 3) } ) // NOT OK! + console.log(bar); })(); \ No newline at end of file From b29f88450b7dc062a6ca77572f54a95050020668 Mon Sep 17 00:00:00 2001 From: Robert Marsh Date: Thu, 17 Oct 2019 12:10:23 -0700 Subject: [PATCH 013/148] C++: buffer read side effects on unmodeled funcs --- .../raw/internal/TranslatedCall.qll | 22 ++- .../test/library-tests/ir/ir/raw_ir.expected | 154 +++++++++--------- .../ir/ssa/aliased_ssa_ir.expected | 6 +- .../ir/ssa/unaliased_ssa_ir.expected | 6 +- 4 files changed, 96 insertions(+), 92 deletions(-) diff --git a/cpp/ql/src/semmle/code/cpp/ir/implementation/raw/internal/TranslatedCall.qll b/cpp/ql/src/semmle/code/cpp/ir/implementation/raw/internal/TranslatedCall.qll index 4a7aa607f57..fbe4910a0dd 100644 --- a/cpp/ql/src/semmle/code/cpp/ir/implementation/raw/internal/TranslatedCall.qll +++ b/cpp/ql/src/semmle/code/cpp/ir/implementation/raw/internal/TranslatedCall.qll @@ -460,14 +460,18 @@ class TranslatedSideEffect extends TranslatedElement, TTranslatedArgumentSideEff } override Type getInstructionOperandType(InstructionTag tag, TypedOperandTag operandTag) { - tag instanceof OnlyInstructionTag and - result = arg.getType().getUnspecifiedType().(DerivedType).getBaseType() and - operandTag instanceof SideEffectOperandTag - or - tag instanceof OnlyInstructionTag and - result = arg.getType().getUnspecifiedType() and - not result instanceof DerivedType and - operandTag instanceof SideEffectOperandTag + if hasSpecificReadSideEffect(any(Opcode::BufferReadSideEffect op)) + then result instanceof UnknownType + else ( + tag instanceof OnlyInstructionTag and + result = arg.getType().getUnspecifiedType().(DerivedType).getBaseType() and + operandTag instanceof SideEffectOperandTag + or + tag instanceof OnlyInstructionTag and + result = arg.getType().getUnspecifiedType() and + not result instanceof DerivedType and + operandTag instanceof SideEffectOperandTag + ) } predicate hasSpecificWriteSideEffect(Opcode op) { @@ -517,7 +521,7 @@ class TranslatedSideEffect extends TranslatedElement, TTranslatedArgumentSideEff ) or not call.getTarget() instanceof SideEffectFunction and - op instanceof Opcode::IndirectReadSideEffect + op instanceof Opcode::BufferReadSideEffect } final override int getInstructionIndex(InstructionTag tag) { diff --git a/cpp/ql/test/library-tests/ir/ir/raw_ir.expected b/cpp/ql/test/library-tests/ir/ir/raw_ir.expected index 3bff2bff690..b2556c8616b 100644 --- a/cpp/ql/test/library-tests/ir/ir/raw_ir.expected +++ b/cpp/ql/test/library-tests/ir/ir/raw_ir.expected @@ -14,7 +14,7 @@ bad_asts.cpp: # 16| r0_10(int) = Constant[1] : # 16| r0_11(int) = Call : func:r0_9, this:r0_8, 0:r0_10 # 16| mu0_12(unknown) = ^CallSideEffect : ~mu0_2 -# 16| v0_13(void) = ^IndirectReadSideEffect[-1] : &:r0_8, ~mu0_2 +# 16| v0_13(void) = ^BufferReadSideEffect[-1] : &:r0_8, ~mu0_2 # 16| mu0_14(S) = ^IndirectMayWriteSideEffect[-1] : &:r0_8 # 17| v0_15(void) = NoOp : # 14| v0_16(void) = ReturnVoid : @@ -2675,8 +2675,8 @@ ir.cpp: # 585| r0_8(char *) = Convert : r0_7 # 585| v0_9(void) = Call : func:r0_3, 0:r0_5, 1:r0_6, 2:r0_8 # 585| mu0_10(unknown) = ^CallSideEffect : ~mu0_2 -# 585| v0_11(void) = ^IndirectReadSideEffect[0] : &:r0_5, ~mu0_2 -# 585| v0_12(void) = ^IndirectReadSideEffect[2] : &:r0_8, ~mu0_2 +# 585| v0_11(void) = ^BufferReadSideEffect[0] : &:r0_5, ~mu0_2 +# 585| v0_12(void) = ^BufferReadSideEffect[2] : &:r0_8, ~mu0_2 # 585| mu0_13(unknown) = ^BufferMayWriteSideEffect[0] : &:r0_5 # 585| mu0_14(unknown) = ^BufferMayWriteSideEffect[2] : &:r0_8 # 586| v0_15(void) = NoOp : @@ -2721,7 +2721,7 @@ ir.cpp: # 617| r0_10(char *) = Convert : r0_9 # 617| v0_11(void) = Call : func:r0_8, this:r0_7, 0:r0_10 # 617| mu0_12(unknown) = ^CallSideEffect : ~mu0_2 -# 617| v0_13(void) = ^IndirectReadSideEffect[0] : &:r0_10, ~mu0_2 +# 617| v0_13(void) = ^BufferReadSideEffect[0] : &:r0_10, ~mu0_2 # 617| mu0_14(unknown) = ^BufferMayWriteSideEffect[0] : &:r0_10 # 618| r0_15(glval) = VariableAddress[s3] : # 618| r0_16(glval) = FunctionAddress[ReturnObject] : @@ -2734,7 +2734,7 @@ ir.cpp: # 619| r0_23(char *) = Convert : r0_22 # 619| v0_24(void) = Call : func:r0_21, this:r0_20, 0:r0_23 # 619| mu0_25(unknown) = ^CallSideEffect : ~mu0_2 -# 619| v0_26(void) = ^IndirectReadSideEffect[0] : &:r0_23, ~mu0_2 +# 619| v0_26(void) = ^BufferReadSideEffect[0] : &:r0_23, ~mu0_2 # 619| mu0_27(unknown) = ^BufferMayWriteSideEffect[0] : &:r0_23 # 620| v0_28(void) = NoOp : # 615| v0_29(void) = ReturnVoid : @@ -2758,7 +2758,7 @@ ir.cpp: # 623| r0_12(glval) = FunctionAddress[c_str] : # 623| r0_13(char *) = Call : func:r0_12, this:r0_11 # 623| mu0_14(unknown) = ^CallSideEffect : ~mu0_2 -# 623| v0_15(void) = ^IndirectReadSideEffect[-1] : &:r0_11, ~mu0_2 +# 623| v0_15(void) = ^BufferReadSideEffect[-1] : &:r0_11, ~mu0_2 # 623| mu0_16(String) = ^IndirectMayWriteSideEffect[-1] : &:r0_11 # 624| r0_17(glval) = VariableAddress[p] : # 624| r0_18(String *) = Load : &:r0_17, ~mu0_2 @@ -2766,14 +2766,14 @@ ir.cpp: # 624| r0_20(glval) = FunctionAddress[c_str] : # 624| r0_21(char *) = Call : func:r0_20, this:r0_19 # 624| mu0_22(unknown) = ^CallSideEffect : ~mu0_2 -# 624| v0_23(void) = ^IndirectReadSideEffect[-1] : &:r0_19, ~mu0_2 +# 624| v0_23(void) = ^BufferReadSideEffect[-1] : &:r0_19, ~mu0_2 # 624| mu0_24(String) = ^IndirectMayWriteSideEffect[-1] : &:r0_19 # 625| r0_25(glval) = VariableAddress[s] : # 625| r0_26(glval) = Convert : r0_25 # 625| r0_27(glval) = FunctionAddress[c_str] : # 625| r0_28(char *) = Call : func:r0_27, this:r0_26 # 625| mu0_29(unknown) = ^CallSideEffect : ~mu0_2 -# 625| v0_30(void) = ^IndirectReadSideEffect[-1] : &:r0_26, ~mu0_2 +# 625| v0_30(void) = ^BufferReadSideEffect[-1] : &:r0_26, ~mu0_2 # 625| mu0_31(String) = ^IndirectMayWriteSideEffect[-1] : &:r0_26 # 626| v0_32(void) = NoOp : # 622| v0_33(void) = ReturnVoid : @@ -2881,21 +2881,21 @@ ir.cpp: # 653| r0_6(int) = Constant[0] : # 653| r0_7(int) = Call : func:r0_5, this:r0_4, 0:r0_6 # 653| mu0_8(unknown) = ^CallSideEffect : ~mu0_2 -# 653| v0_9(void) = ^IndirectReadSideEffect[-1] : &:r0_4, ~mu0_2 +# 653| v0_9(void) = ^BufferReadSideEffect[-1] : &:r0_4, ~mu0_2 # 653| mu0_10(C) = ^IndirectMayWriteSideEffect[-1] : &:r0_4 # 654| r0_11(C *) = CopyValue : r0_3 # 654| r0_12(glval) = FunctionAddress[InstanceMemberFunction] : # 654| r0_13(int) = Constant[1] : # 654| r0_14(int) = Call : func:r0_12, this:r0_11, 0:r0_13 # 654| mu0_15(unknown) = ^CallSideEffect : ~mu0_2 -# 654| v0_16(void) = ^IndirectReadSideEffect[-1] : &:r0_11, ~mu0_2 +# 654| v0_16(void) = ^BufferReadSideEffect[-1] : &:r0_11, ~mu0_2 # 654| mu0_17(C) = ^IndirectMayWriteSideEffect[-1] : &:r0_11 #-----| r0_18(C *) = CopyValue : r0_3 # 655| r0_19(glval) = FunctionAddress[InstanceMemberFunction] : # 655| r0_20(int) = Constant[2] : # 655| r0_21(int) = Call : func:r0_19, this:r0_18, 0:r0_20 # 655| mu0_22(unknown) = ^CallSideEffect : ~mu0_2 -#-----| v0_23(void) = ^IndirectReadSideEffect[-1] : &:r0_18, ~mu0_2 +#-----| v0_23(void) = ^BufferReadSideEffect[-1] : &:r0_18, ~mu0_2 #-----| mu0_24(C) = ^IndirectMayWriteSideEffect[-1] : &:r0_18 # 656| v0_25(void) = NoOp : # 652| v0_26(void) = ReturnVoid : @@ -2927,7 +2927,7 @@ ir.cpp: # 662| r0_20(char *) = Convert : r0_19 # 662| v0_21(void) = Call : func:r0_18, this:r0_17, 0:r0_20 # 662| mu0_22(unknown) = ^CallSideEffect : ~mu0_2 -# 662| v0_23(void) = ^IndirectReadSideEffect[0] : &:r0_20, ~mu0_2 +# 662| v0_23(void) = ^BufferReadSideEffect[0] : &:r0_20, ~mu0_2 # 662| mu0_24(unknown) = ^BufferMayWriteSideEffect[0] : &:r0_20 # 664| v0_25(void) = NoOp : # 658| v0_26(void) = ReturnVoid : @@ -3127,7 +3127,7 @@ ir.cpp: # 721| r0_6(char) = Constant[111] : # 721| r0_7(long) = Call : func:r0_4, 0:r0_5, 1:r0_6 # 721| mu0_8(unknown) = ^CallSideEffect : ~mu0_2 -# 721| v0_9(void) = ^IndirectReadSideEffect[0] : &:r0_5, ~mu0_2 +# 721| v0_9(void) = ^BufferReadSideEffect[0] : &:r0_5, ~mu0_2 # 721| mu0_10(unknown) = ^BufferMayWriteSideEffect[0] : &:r0_5 # 721| r0_11(double) = Convert : r0_7 # 721| mu0_12(double) = Store : &:r0_3, r0_11 @@ -3201,7 +3201,7 @@ ir.cpp: # 731| r7_3(char *) = Convert : r7_2 # 731| v7_4(void) = Call : func:r7_1, this:r7_0, 0:r7_3 # 731| mu7_5(unknown) = ^CallSideEffect : ~mu0_2 -# 731| v7_6(void) = ^IndirectReadSideEffect[0] : &:r7_3, ~mu0_2 +# 731| v7_6(void) = ^BufferReadSideEffect[0] : &:r7_3, ~mu0_2 # 731| mu7_7(unknown) = ^BufferMayWriteSideEffect[0] : &:r7_3 # 731| v7_8(void) = ThrowValue : &:r7_0, ~mu0_2 #-----| Exception -> Block 9 @@ -3226,7 +3226,7 @@ ir.cpp: # 736| r10_5(char *) = Load : &:r10_4, ~mu0_2 # 736| v10_6(void) = Call : func:r10_3, this:r10_2, 0:r10_5 # 736| mu10_7(unknown) = ^CallSideEffect : ~mu0_2 -# 736| v10_8(void) = ^IndirectReadSideEffect[0] : &:r10_5, ~mu0_2 +# 736| v10_8(void) = ^BufferReadSideEffect[0] : &:r10_5, ~mu0_2 # 736| mu10_9(unknown) = ^BufferMayWriteSideEffect[0] : &:r10_5 # 736| v10_10(void) = ThrowValue : &:r10_2, ~mu0_2 #-----| Exception -> Block 2 @@ -3268,8 +3268,8 @@ ir.cpp: #-----| r0_11(glval) = FieldAddress[base_s] : r0_10 # 745| r0_12(String &) = Call : func:r0_8, this:r0_7, 0:r0_11 # 745| mu0_13(unknown) = ^CallSideEffect : ~mu0_2 -#-----| v0_14(void) = ^IndirectReadSideEffect[-1] : &:r0_7, ~mu0_2 -#-----| v0_15(void) = ^IndirectReadSideEffect[0] : &:r0_11, ~mu0_2 +#-----| v0_14(void) = ^BufferReadSideEffect[-1] : &:r0_7, ~mu0_2 +#-----| v0_15(void) = ^BufferReadSideEffect[0] : &:r0_11, ~mu0_2 #-----| mu0_16(String) = ^IndirectMayWriteSideEffect[-1] : &:r0_7 #-----| mu0_17(unknown) = ^BufferMayWriteSideEffect[0] : &:r0_11 #-----| r0_18(glval) = VariableAddress[#return] : @@ -3343,8 +3343,8 @@ ir.cpp: #-----| r0_11(Base *) = ConvertToBase[Middle : Base] : r0_10 # 754| r0_12(Base &) = Call : func:r0_8, this:r0_7, 0:r0_11 # 754| mu0_13(unknown) = ^CallSideEffect : ~mu0_2 -#-----| v0_14(void) = ^IndirectReadSideEffect[-1] : &:r0_7, ~mu0_2 -#-----| v0_15(void) = ^IndirectReadSideEffect[0] : &:r0_11, ~mu0_2 +#-----| v0_14(void) = ^BufferReadSideEffect[-1] : &:r0_7, ~mu0_2 +#-----| v0_15(void) = ^BufferReadSideEffect[0] : &:r0_11, ~mu0_2 #-----| mu0_16(Base) = ^IndirectMayWriteSideEffect[-1] : &:r0_7 #-----| mu0_17(unknown) = ^BufferMayWriteSideEffect[0] : &:r0_11 #-----| r0_18(Middle *) = CopyValue : r0_3 @@ -3355,8 +3355,8 @@ ir.cpp: #-----| r0_23(glval) = FieldAddress[middle_s] : r0_22 # 754| r0_24(String &) = Call : func:r0_20, this:r0_19, 0:r0_23 # 754| mu0_25(unknown) = ^CallSideEffect : ~mu0_2 -#-----| v0_26(void) = ^IndirectReadSideEffect[-1] : &:r0_19, ~mu0_2 -#-----| v0_27(void) = ^IndirectReadSideEffect[0] : &:r0_23, ~mu0_2 +#-----| v0_26(void) = ^BufferReadSideEffect[-1] : &:r0_19, ~mu0_2 +#-----| v0_27(void) = ^BufferReadSideEffect[0] : &:r0_23, ~mu0_2 #-----| mu0_28(String) = ^IndirectMayWriteSideEffect[-1] : &:r0_19 #-----| mu0_29(unknown) = ^BufferMayWriteSideEffect[0] : &:r0_23 #-----| r0_30(glval) = VariableAddress[#return] : @@ -3421,8 +3421,8 @@ ir.cpp: #-----| r0_11(Middle *) = ConvertToBase[Derived : Middle] : r0_10 # 763| r0_12(Middle &) = Call : func:r0_8, this:r0_7, 0:r0_11 # 763| mu0_13(unknown) = ^CallSideEffect : ~mu0_2 -#-----| v0_14(void) = ^IndirectReadSideEffect[-1] : &:r0_7, ~mu0_2 -#-----| v0_15(void) = ^IndirectReadSideEffect[0] : &:r0_11, ~mu0_2 +#-----| v0_14(void) = ^BufferReadSideEffect[-1] : &:r0_7, ~mu0_2 +#-----| v0_15(void) = ^BufferReadSideEffect[0] : &:r0_11, ~mu0_2 #-----| mu0_16(Middle) = ^IndirectMayWriteSideEffect[-1] : &:r0_7 #-----| mu0_17(unknown) = ^BufferMayWriteSideEffect[0] : &:r0_11 #-----| r0_18(Derived *) = CopyValue : r0_3 @@ -3433,8 +3433,8 @@ ir.cpp: #-----| r0_23(glval) = FieldAddress[derived_s] : r0_22 # 763| r0_24(String &) = Call : func:r0_20, this:r0_19, 0:r0_23 # 763| mu0_25(unknown) = ^CallSideEffect : ~mu0_2 -#-----| v0_26(void) = ^IndirectReadSideEffect[-1] : &:r0_19, ~mu0_2 -#-----| v0_27(void) = ^IndirectReadSideEffect[0] : &:r0_23, ~mu0_2 +#-----| v0_26(void) = ^BufferReadSideEffect[-1] : &:r0_19, ~mu0_2 +#-----| v0_27(void) = ^BufferReadSideEffect[0] : &:r0_23, ~mu0_2 #-----| mu0_28(String) = ^IndirectMayWriteSideEffect[-1] : &:r0_19 #-----| mu0_29(unknown) = ^BufferMayWriteSideEffect[0] : &:r0_23 #-----| r0_30(glval) = VariableAddress[#return] : @@ -3645,8 +3645,8 @@ ir.cpp: # 808| r0_27(glval) = ConvertToBase[Middle : Base] : r0_26 # 808| r0_28(Base &) = Call : func:r0_25, this:r0_24, 0:r0_27 # 808| mu0_29(unknown) = ^CallSideEffect : ~mu0_2 -# 808| v0_30(void) = ^IndirectReadSideEffect[-1] : &:r0_24, ~mu0_2 -# 808| v0_31(void) = ^IndirectReadSideEffect[0] : &:r0_27, ~mu0_2 +# 808| v0_30(void) = ^BufferReadSideEffect[-1] : &:r0_24, ~mu0_2 +# 808| v0_31(void) = ^BufferReadSideEffect[0] : &:r0_27, ~mu0_2 # 808| mu0_32(Base) = ^IndirectMayWriteSideEffect[-1] : &:r0_24 # 808| mu0_33(unknown) = ^BufferMayWriteSideEffect[0] : &:r0_27 # 809| r0_34(glval) = VariableAddress[b] : @@ -3656,13 +3656,13 @@ ir.cpp: # 809| r0_38(glval) = ConvertToBase[Middle : Base] : r0_37 # 809| v0_39(void) = Call : func:r0_36, 0:r0_38 # 809| mu0_40(unknown) = ^CallSideEffect : ~mu0_2 -# 809| v0_41(void) = ^IndirectReadSideEffect[0] : &:r0_38, ~mu0_2 +# 809| v0_41(void) = ^BufferReadSideEffect[0] : &:r0_38, ~mu0_2 # 809| mu0_42(unknown) = ^BufferMayWriteSideEffect[0] : &:r0_38 # 809| r0_43(glval) = Convert : v0_39 # 809| r0_44(Base &) = Call : func:r0_35, this:r0_34, 0:r0_43 # 809| mu0_45(unknown) = ^CallSideEffect : ~mu0_2 -# 809| v0_46(void) = ^IndirectReadSideEffect[-1] : &:r0_34, ~mu0_2 -# 809| v0_47(void) = ^IndirectReadSideEffect[0] : &:r0_43, ~mu0_2 +# 809| v0_46(void) = ^BufferReadSideEffect[-1] : &:r0_34, ~mu0_2 +# 809| v0_47(void) = ^BufferReadSideEffect[0] : &:r0_43, ~mu0_2 # 809| mu0_48(Base) = ^IndirectMayWriteSideEffect[-1] : &:r0_34 # 809| mu0_49(unknown) = ^BufferMayWriteSideEffect[0] : &:r0_43 # 810| r0_50(glval) = VariableAddress[b] : @@ -3672,13 +3672,13 @@ ir.cpp: # 810| r0_54(glval) = ConvertToBase[Middle : Base] : r0_53 # 810| v0_55(void) = Call : func:r0_52, 0:r0_54 # 810| mu0_56(unknown) = ^CallSideEffect : ~mu0_2 -# 810| v0_57(void) = ^IndirectReadSideEffect[0] : &:r0_54, ~mu0_2 +# 810| v0_57(void) = ^BufferReadSideEffect[0] : &:r0_54, ~mu0_2 # 810| mu0_58(unknown) = ^BufferMayWriteSideEffect[0] : &:r0_54 # 810| r0_59(glval) = Convert : v0_55 # 810| r0_60(Base &) = Call : func:r0_51, this:r0_50, 0:r0_59 # 810| mu0_61(unknown) = ^CallSideEffect : ~mu0_2 -# 810| v0_62(void) = ^IndirectReadSideEffect[-1] : &:r0_50, ~mu0_2 -# 810| v0_63(void) = ^IndirectReadSideEffect[0] : &:r0_59, ~mu0_2 +# 810| v0_62(void) = ^BufferReadSideEffect[-1] : &:r0_50, ~mu0_2 +# 810| v0_63(void) = ^BufferReadSideEffect[0] : &:r0_59, ~mu0_2 # 810| mu0_64(Base) = ^IndirectMayWriteSideEffect[-1] : &:r0_50 # 810| mu0_65(unknown) = ^BufferMayWriteSideEffect[0] : &:r0_59 # 811| r0_66(glval) = VariableAddress[pm] : @@ -3708,8 +3708,8 @@ ir.cpp: # 816| r0_90(glval) = Convert : r0_89 # 816| r0_91(Middle &) = Call : func:r0_87, this:r0_86, 0:r0_90 # 816| mu0_92(unknown) = ^CallSideEffect : ~mu0_2 -# 816| v0_93(void) = ^IndirectReadSideEffect[-1] : &:r0_86, ~mu0_2 -# 816| v0_94(void) = ^IndirectReadSideEffect[0] : &:r0_90, ~mu0_2 +# 816| v0_93(void) = ^BufferReadSideEffect[-1] : &:r0_86, ~mu0_2 +# 816| v0_94(void) = ^BufferReadSideEffect[0] : &:r0_90, ~mu0_2 # 816| mu0_95(Middle) = ^IndirectMayWriteSideEffect[-1] : &:r0_86 # 816| mu0_96(unknown) = ^BufferMayWriteSideEffect[0] : &:r0_90 # 817| r0_97(glval) = VariableAddress[m] : @@ -3719,8 +3719,8 @@ ir.cpp: # 817| r0_101(glval) = Convert : r0_100 # 817| r0_102(Middle &) = Call : func:r0_98, this:r0_97, 0:r0_101 # 817| mu0_103(unknown) = ^CallSideEffect : ~mu0_2 -# 817| v0_104(void) = ^IndirectReadSideEffect[-1] : &:r0_97, ~mu0_2 -# 817| v0_105(void) = ^IndirectReadSideEffect[0] : &:r0_101, ~mu0_2 +# 817| v0_104(void) = ^BufferReadSideEffect[-1] : &:r0_97, ~mu0_2 +# 817| v0_105(void) = ^BufferReadSideEffect[0] : &:r0_101, ~mu0_2 # 817| mu0_106(Middle) = ^IndirectMayWriteSideEffect[-1] : &:r0_97 # 817| mu0_107(unknown) = ^BufferMayWriteSideEffect[0] : &:r0_101 # 818| r0_108(glval) = VariableAddress[pb] : @@ -3745,8 +3745,8 @@ ir.cpp: # 822| r0_127(glval) = ConvertToBase[Middle : Base] : r0_126 # 822| r0_128(Base &) = Call : func:r0_124, this:r0_123, 0:r0_127 # 822| mu0_129(unknown) = ^CallSideEffect : ~mu0_2 -# 822| v0_130(void) = ^IndirectReadSideEffect[-1] : &:r0_123, ~mu0_2 -# 822| v0_131(void) = ^IndirectReadSideEffect[0] : &:r0_127, ~mu0_2 +# 822| v0_130(void) = ^BufferReadSideEffect[-1] : &:r0_123, ~mu0_2 +# 822| v0_131(void) = ^BufferReadSideEffect[0] : &:r0_127, ~mu0_2 # 822| mu0_132(Base) = ^IndirectMayWriteSideEffect[-1] : &:r0_123 # 822| mu0_133(unknown) = ^BufferMayWriteSideEffect[0] : &:r0_127 # 823| r0_134(glval) = VariableAddress[b] : @@ -3757,13 +3757,13 @@ ir.cpp: # 823| r0_139(glval) = ConvertToBase[Middle : Base] : r0_138 # 823| v0_140(void) = Call : func:r0_136, 0:r0_139 # 823| mu0_141(unknown) = ^CallSideEffect : ~mu0_2 -# 823| v0_142(void) = ^IndirectReadSideEffect[0] : &:r0_139, ~mu0_2 +# 823| v0_142(void) = ^BufferReadSideEffect[0] : &:r0_139, ~mu0_2 # 823| mu0_143(unknown) = ^BufferMayWriteSideEffect[0] : &:r0_139 # 823| r0_144(glval) = Convert : v0_140 # 823| r0_145(Base &) = Call : func:r0_135, this:r0_134, 0:r0_144 # 823| mu0_146(unknown) = ^CallSideEffect : ~mu0_2 -# 823| v0_147(void) = ^IndirectReadSideEffect[-1] : &:r0_134, ~mu0_2 -# 823| v0_148(void) = ^IndirectReadSideEffect[0] : &:r0_144, ~mu0_2 +# 823| v0_147(void) = ^BufferReadSideEffect[-1] : &:r0_134, ~mu0_2 +# 823| v0_148(void) = ^BufferReadSideEffect[0] : &:r0_144, ~mu0_2 # 823| mu0_149(Base) = ^IndirectMayWriteSideEffect[-1] : &:r0_134 # 823| mu0_150(unknown) = ^BufferMayWriteSideEffect[0] : &:r0_144 # 824| r0_151(glval) = VariableAddress[b] : @@ -3774,13 +3774,13 @@ ir.cpp: # 824| r0_156(glval) = ConvertToBase[Middle : Base] : r0_155 # 824| v0_157(void) = Call : func:r0_153, 0:r0_156 # 824| mu0_158(unknown) = ^CallSideEffect : ~mu0_2 -# 824| v0_159(void) = ^IndirectReadSideEffect[0] : &:r0_156, ~mu0_2 +# 824| v0_159(void) = ^BufferReadSideEffect[0] : &:r0_156, ~mu0_2 # 824| mu0_160(unknown) = ^BufferMayWriteSideEffect[0] : &:r0_156 # 824| r0_161(glval) = Convert : v0_157 # 824| r0_162(Base &) = Call : func:r0_152, this:r0_151, 0:r0_161 # 824| mu0_163(unknown) = ^CallSideEffect : ~mu0_2 -# 824| v0_164(void) = ^IndirectReadSideEffect[-1] : &:r0_151, ~mu0_2 -# 824| v0_165(void) = ^IndirectReadSideEffect[0] : &:r0_161, ~mu0_2 +# 824| v0_164(void) = ^BufferReadSideEffect[-1] : &:r0_151, ~mu0_2 +# 824| v0_165(void) = ^BufferReadSideEffect[0] : &:r0_161, ~mu0_2 # 824| mu0_166(Base) = ^IndirectMayWriteSideEffect[-1] : &:r0_151 # 824| mu0_167(unknown) = ^BufferMayWriteSideEffect[0] : &:r0_161 # 825| r0_168(glval) = VariableAddress[pd] : @@ -3814,8 +3814,8 @@ ir.cpp: # 830| r0_196(glval) = Convert : r0_195 # 830| r0_197(Derived &) = Call : func:r0_192, this:r0_191, 0:r0_196 # 830| mu0_198(unknown) = ^CallSideEffect : ~mu0_2 -# 830| v0_199(void) = ^IndirectReadSideEffect[-1] : &:r0_191, ~mu0_2 -# 830| v0_200(void) = ^IndirectReadSideEffect[0] : &:r0_196, ~mu0_2 +# 830| v0_199(void) = ^BufferReadSideEffect[-1] : &:r0_191, ~mu0_2 +# 830| v0_200(void) = ^BufferReadSideEffect[0] : &:r0_196, ~mu0_2 # 830| mu0_201(Derived) = ^IndirectMayWriteSideEffect[-1] : &:r0_191 # 830| mu0_202(unknown) = ^BufferMayWriteSideEffect[0] : &:r0_196 # 831| r0_203(glval) = VariableAddress[d] : @@ -3826,8 +3826,8 @@ ir.cpp: # 831| r0_208(glval) = Convert : r0_207 # 831| r0_209(Derived &) = Call : func:r0_204, this:r0_203, 0:r0_208 # 831| mu0_210(unknown) = ^CallSideEffect : ~mu0_2 -# 831| v0_211(void) = ^IndirectReadSideEffect[-1] : &:r0_203, ~mu0_2 -# 831| v0_212(void) = ^IndirectReadSideEffect[0] : &:r0_208, ~mu0_2 +# 831| v0_211(void) = ^BufferReadSideEffect[-1] : &:r0_203, ~mu0_2 +# 831| v0_212(void) = ^BufferReadSideEffect[0] : &:r0_208, ~mu0_2 # 831| mu0_213(Derived) = ^IndirectMayWriteSideEffect[-1] : &:r0_203 # 831| mu0_214(unknown) = ^BufferMayWriteSideEffect[0] : &:r0_208 # 832| r0_215(glval) = VariableAddress[pb] : @@ -3972,7 +3972,7 @@ ir.cpp: # 868| r0_6(char *) = Convert : r0_5 # 868| v0_7(void) = Call : func:r0_4, this:r0_3, 0:r0_6 # 868| mu0_8(unknown) = ^CallSideEffect : ~mu0_2 -# 868| v0_9(void) = ^IndirectReadSideEffect[0] : &:r0_6, ~mu0_2 +# 868| v0_9(void) = ^BufferReadSideEffect[0] : &:r0_6, ~mu0_2 # 868| mu0_10(unknown) = ^BufferMayWriteSideEffect[0] : &:r0_6 # 869| v0_11(void) = NoOp : # 867| v0_12(void) = ReturnVoid : @@ -4189,7 +4189,7 @@ ir.cpp: # 945| r0_37(char *) = Convert : r0_36 # 945| v0_38(void) = Call : func:r0_35, this:r0_34, 0:r0_37 # 945| mu0_39(unknown) = ^CallSideEffect : ~mu0_2 -# 945| v0_40(void) = ^IndirectReadSideEffect[0] : &:r0_37, ~mu0_2 +# 945| v0_40(void) = ^BufferReadSideEffect[0] : &:r0_37, ~mu0_2 # 945| mu0_41(unknown) = ^BufferMayWriteSideEffect[0] : &:r0_37 # 946| r0_42(glval) = FunctionAddress[operator new] : # 946| r0_43(unsigned long) = Constant[256] : @@ -4677,7 +4677,7 @@ ir.cpp: # 1035| r0_28(float) = Constant[1.0] : # 1035| r0_29(char) = Call : func:r0_27, this:r0_26, 0:r0_28 # 1035| mu0_30(unknown) = ^CallSideEffect : ~mu0_2 -# 1035| v0_31(void) = ^IndirectReadSideEffect[-1] : &:r0_26, ~mu0_2 +# 1035| v0_31(void) = ^BufferReadSideEffect[-1] : &:r0_26, ~mu0_2 # 1035| mu0_32(decltype([...](...){...})) = ^IndirectMayWriteSideEffect[-1] : &:r0_26 # 1036| r0_33(glval) = VariableAddress[lambda_val] : # 1036| r0_34(glval) = FunctionAddress[(constructor)] : @@ -4694,7 +4694,7 @@ ir.cpp: # 1036| r0_45(decltype([...](...){...})) = Load : &:r0_35, ~mu0_2 # 1036| v0_46(void) = Call : func:r0_34, this:r0_33, 0:r0_45 # 1036| mu0_47(unknown) = ^CallSideEffect : ~mu0_2 -# 1036| v0_48(void) = ^IndirectReadSideEffect[0] : &:r0_45, ~mu0_2 +# 1036| v0_48(void) = ^BufferReadSideEffect[0] : &:r0_45, ~mu0_2 # 1036| mu0_49(unknown) = ^BufferMayWriteSideEffect[0] : &:r0_45 # 1037| r0_50(glval) = VariableAddress[lambda_val] : # 1037| r0_51(glval) = Convert : r0_50 @@ -4702,7 +4702,7 @@ ir.cpp: # 1037| r0_53(float) = Constant[2.0] : # 1037| r0_54(char) = Call : func:r0_52, this:r0_51, 0:r0_53 # 1037| mu0_55(unknown) = ^CallSideEffect : ~mu0_2 -# 1037| v0_56(void) = ^IndirectReadSideEffect[-1] : &:r0_51, ~mu0_2 +# 1037| v0_56(void) = ^BufferReadSideEffect[-1] : &:r0_51, ~mu0_2 # 1037| mu0_57(decltype([...](...){...})) = ^IndirectMayWriteSideEffect[-1] : &:r0_51 # 1038| r0_58(glval) = VariableAddress[lambda_ref_explicit] : # 1038| r0_59(glval) = VariableAddress[#temp1038:30] : @@ -4719,7 +4719,7 @@ ir.cpp: # 1039| r0_70(float) = Constant[3.0] : # 1039| r0_71(char) = Call : func:r0_69, this:r0_68, 0:r0_70 # 1039| mu0_72(unknown) = ^CallSideEffect : ~mu0_2 -# 1039| v0_73(void) = ^IndirectReadSideEffect[-1] : &:r0_68, ~mu0_2 +# 1039| v0_73(void) = ^BufferReadSideEffect[-1] : &:r0_68, ~mu0_2 # 1039| mu0_74(decltype([...](...){...})) = ^IndirectMayWriteSideEffect[-1] : &:r0_68 # 1040| r0_75(glval) = VariableAddress[lambda_val_explicit] : # 1040| r0_76(glval) = FunctionAddress[(constructor)] : @@ -4732,7 +4732,7 @@ ir.cpp: # 1040| r0_83(decltype([...](...){...})) = Load : &:r0_77, ~mu0_2 # 1040| v0_84(void) = Call : func:r0_76, this:r0_75, 0:r0_83 # 1040| mu0_85(unknown) = ^CallSideEffect : ~mu0_2 -# 1040| v0_86(void) = ^IndirectReadSideEffect[0] : &:r0_83, ~mu0_2 +# 1040| v0_86(void) = ^BufferReadSideEffect[0] : &:r0_83, ~mu0_2 # 1040| mu0_87(unknown) = ^BufferMayWriteSideEffect[0] : &:r0_83 # 1041| r0_88(glval) = VariableAddress[lambda_val_explicit] : # 1041| r0_89(glval) = Convert : r0_88 @@ -4740,7 +4740,7 @@ ir.cpp: # 1041| r0_91(float) = Constant[4.0] : # 1041| r0_92(char) = Call : func:r0_90, this:r0_89, 0:r0_91 # 1041| mu0_93(unknown) = ^CallSideEffect : ~mu0_2 -# 1041| v0_94(void) = ^IndirectReadSideEffect[-1] : &:r0_89, ~mu0_2 +# 1041| v0_94(void) = ^BufferReadSideEffect[-1] : &:r0_89, ~mu0_2 # 1041| mu0_95(decltype([...](...){...})) = ^IndirectMayWriteSideEffect[-1] : &:r0_89 # 1042| r0_96(glval) = VariableAddress[lambda_mixed_explicit] : # 1042| r0_97(glval) = VariableAddress[#temp1042:32] : @@ -4761,7 +4761,7 @@ ir.cpp: # 1043| r0_112(float) = Constant[5.0] : # 1043| r0_113(char) = Call : func:r0_111, this:r0_110, 0:r0_112 # 1043| mu0_114(unknown) = ^CallSideEffect : ~mu0_2 -# 1043| v0_115(void) = ^IndirectReadSideEffect[-1] : &:r0_110, ~mu0_2 +# 1043| v0_115(void) = ^BufferReadSideEffect[-1] : &:r0_110, ~mu0_2 # 1043| mu0_116(decltype([...](...){...})) = ^IndirectMayWriteSideEffect[-1] : &:r0_110 # 1044| r0_117(glval) = VariableAddress[r] : # 1044| r0_118(glval) = VariableAddress[x] : @@ -4797,7 +4797,7 @@ ir.cpp: # 1046| r0_148(float) = Constant[6.0] : # 1046| r0_149(char) = Call : func:r0_147, this:r0_146, 0:r0_148 # 1046| mu0_150(unknown) = ^CallSideEffect : ~mu0_2 -# 1046| v0_151(void) = ^IndirectReadSideEffect[-1] : &:r0_146, ~mu0_2 +# 1046| v0_151(void) = ^BufferReadSideEffect[-1] : &:r0_146, ~mu0_2 # 1046| mu0_152(decltype([...](...){...})) = ^IndirectMayWriteSideEffect[-1] : &:r0_146 # 1047| v0_153(void) = NoOp : # 1031| v0_154(void) = ReturnVoid : @@ -4862,7 +4862,7 @@ ir.cpp: # 1034| r0_10(glval) = FunctionAddress[c_str] : # 1034| r0_11(char *) = Call : func:r0_10, this:r0_9 # 1034| mu0_12(unknown) = ^CallSideEffect : ~mu0_2 -# 1034| v0_13(void) = ^IndirectReadSideEffect[-1] : &:r0_9, ~mu0_2 +# 1034| v0_13(void) = ^BufferReadSideEffect[-1] : &:r0_9, ~mu0_2 # 1034| mu0_14(String) = ^IndirectMayWriteSideEffect[-1] : &:r0_9 #-----| r0_15(lambda [] type at line 1034, col. 21 *) = CopyValue : r0_3 #-----| r0_16(glval) = FieldAddress[x] : r0_15 @@ -4905,7 +4905,7 @@ ir.cpp: # 1036| r0_9(glval) = FunctionAddress[c_str] : # 1036| r0_10(char *) = Call : func:r0_9, this:r0_8 # 1036| mu0_11(unknown) = ^CallSideEffect : ~mu0_2 -#-----| v0_12(void) = ^IndirectReadSideEffect[-1] : &:r0_8, ~mu0_2 +#-----| v0_12(void) = ^BufferReadSideEffect[-1] : &:r0_8, ~mu0_2 #-----| mu0_13(String) = ^IndirectMayWriteSideEffect[-1] : &:r0_8 #-----| r0_14(lambda [] type at line 1036, col. 21 *) = CopyValue : r0_3 #-----| r0_15(glval) = FieldAddress[x] : r0_14 @@ -4933,7 +4933,7 @@ ir.cpp: # 1038| r0_10(glval) = FunctionAddress[c_str] : # 1038| r0_11(char *) = Call : func:r0_10, this:r0_9 # 1038| mu0_12(unknown) = ^CallSideEffect : ~mu0_2 -# 1038| v0_13(void) = ^IndirectReadSideEffect[-1] : &:r0_9, ~mu0_2 +# 1038| v0_13(void) = ^BufferReadSideEffect[-1] : &:r0_9, ~mu0_2 # 1038| mu0_14(String) = ^IndirectMayWriteSideEffect[-1] : &:r0_9 # 1038| r0_15(int) = Constant[0] : # 1038| r0_16(glval) = PointerAdd[1] : r0_11, r0_15 @@ -4990,7 +4990,7 @@ ir.cpp: # 1040| r0_9(glval) = FunctionAddress[c_str] : # 1040| r0_10(char *) = Call : func:r0_9, this:r0_8 # 1040| mu0_11(unknown) = ^CallSideEffect : ~mu0_2 -#-----| v0_12(void) = ^IndirectReadSideEffect[-1] : &:r0_8, ~mu0_2 +#-----| v0_12(void) = ^BufferReadSideEffect[-1] : &:r0_8, ~mu0_2 #-----| mu0_13(String) = ^IndirectMayWriteSideEffect[-1] : &:r0_8 # 1040| r0_14(int) = Constant[0] : # 1040| r0_15(glval) = PointerAdd[1] : r0_10, r0_14 @@ -5016,7 +5016,7 @@ ir.cpp: # 1042| r0_10(glval) = FunctionAddress[c_str] : # 1042| r0_11(char *) = Call : func:r0_10, this:r0_9 # 1042| mu0_12(unknown) = ^CallSideEffect : ~mu0_2 -# 1042| v0_13(void) = ^IndirectReadSideEffect[-1] : &:r0_9, ~mu0_2 +# 1042| v0_13(void) = ^BufferReadSideEffect[-1] : &:r0_9, ~mu0_2 # 1042| mu0_14(String) = ^IndirectMayWriteSideEffect[-1] : &:r0_9 #-----| r0_15(lambda [] type at line 1042, col. 32 *) = CopyValue : r0_3 #-----| r0_16(glval) = FieldAddress[x] : r0_15 @@ -5044,7 +5044,7 @@ ir.cpp: # 1045| r0_10(glval) = FunctionAddress[c_str] : # 1045| r0_11(char *) = Call : func:r0_10, this:r0_9 # 1045| mu0_12(unknown) = ^CallSideEffect : ~mu0_2 -# 1045| v0_13(void) = ^IndirectReadSideEffect[-1] : &:r0_9, ~mu0_2 +# 1045| v0_13(void) = ^BufferReadSideEffect[-1] : &:r0_9, ~mu0_2 # 1045| mu0_14(String) = ^IndirectMayWriteSideEffect[-1] : &:r0_9 #-----| r0_15(lambda [] type at line 1045, col. 23 *) = CopyValue : r0_3 #-----| r0_16(glval) = FieldAddress[x] : r0_15 @@ -5083,7 +5083,7 @@ ir.cpp: # 1069| r0_12(glval) = FunctionAddress[begin] : # 1069| r0_13(iterator) = Call : func:r0_12, this:r0_11 # 1069| mu0_14(unknown) = ^CallSideEffect : ~mu0_2 -#-----| v0_15(void) = ^IndirectReadSideEffect[-1] : &:r0_11, ~mu0_2 +#-----| v0_15(void) = ^BufferReadSideEffect[-1] : &:r0_11, ~mu0_2 #-----| mu0_16(vector) = ^IndirectMayWriteSideEffect[-1] : &:r0_11 # 1069| mu0_17(iterator) = Store : &:r0_9, r0_13 # 1069| r0_18(glval) = VariableAddress[(__end)] : @@ -5092,7 +5092,7 @@ ir.cpp: # 1069| r0_21(glval) = FunctionAddress[end] : # 1069| r0_22(iterator) = Call : func:r0_21, this:r0_20 # 1069| mu0_23(unknown) = ^CallSideEffect : ~mu0_2 -#-----| v0_24(void) = ^IndirectReadSideEffect[-1] : &:r0_20, ~mu0_2 +#-----| v0_24(void) = ^BufferReadSideEffect[-1] : &:r0_20, ~mu0_2 #-----| mu0_25(vector) = ^IndirectMayWriteSideEffect[-1] : &:r0_20 # 1069| mu0_26(iterator) = Store : &:r0_18, r0_22 #-----| Goto -> Block 4 @@ -5104,7 +5104,7 @@ ir.cpp: # 1075| r1_3(glval) = FunctionAddress[operator*] : # 1075| r1_4(int &) = Call : func:r1_3, this:r1_2 # 1075| mu1_5(unknown) = ^CallSideEffect : ~mu0_2 -#-----| v1_6(void) = ^IndirectReadSideEffect[-1] : &:r1_2, ~mu0_2 +#-----| v1_6(void) = ^BufferReadSideEffect[-1] : &:r1_2, ~mu0_2 #-----| mu1_7(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r1_2 # 1075| r1_8(glval) = Convert : r1_4 # 1075| mu1_9(int &) = Store : &:r1_0, r1_8 @@ -5136,7 +5136,7 @@ ir.cpp: #-----| r4_4(iterator) = Load : &:r4_3, ~mu0_2 # 1069| r4_5(bool) = Call : func:r4_2, this:r4_1, 0:r4_4 # 1069| mu4_6(unknown) = ^CallSideEffect : ~mu0_2 -#-----| v4_7(void) = ^IndirectReadSideEffect[-1] : &:r4_1, ~mu0_2 +#-----| v4_7(void) = ^BufferReadSideEffect[-1] : &:r4_1, ~mu0_2 #-----| mu4_8(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r4_1 # 1069| v4_9(void) = ConditionalBranch : r4_5 #-----| False -> Block 8 @@ -5149,7 +5149,7 @@ ir.cpp: # 1069| r5_3(glval) = FunctionAddress[operator*] : # 1069| r5_4(int &) = Call : func:r5_3, this:r5_2 # 1069| mu5_5(unknown) = ^CallSideEffect : ~mu0_2 -#-----| v5_6(void) = ^IndirectReadSideEffect[-1] : &:r5_2, ~mu0_2 +#-----| v5_6(void) = ^BufferReadSideEffect[-1] : &:r5_2, ~mu0_2 #-----| mu5_7(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r5_2 # 1069| r5_8(int) = Load : &:r5_4, ~mu0_2 # 1069| mu5_9(int) = Store : &:r5_0, r5_8 @@ -5171,7 +5171,7 @@ ir.cpp: # 1069| r7_2(glval) = FunctionAddress[operator++] : # 1069| r7_3(iterator &) = Call : func:r7_2, this:r7_1 # 1069| mu7_4(unknown) = ^CallSideEffect : ~mu0_2 -#-----| v7_5(void) = ^IndirectReadSideEffect[-1] : &:r7_1, ~mu0_2 +#-----| v7_5(void) = ^BufferReadSideEffect[-1] : &:r7_1, ~mu0_2 #-----| mu7_6(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r7_1 #-----| Goto (back edge) -> Block 4 @@ -5186,7 +5186,7 @@ ir.cpp: # 1075| r8_7(glval) = FunctionAddress[begin] : # 1075| r8_8(iterator) = Call : func:r8_7, this:r8_6 # 1075| mu8_9(unknown) = ^CallSideEffect : ~mu0_2 -#-----| v8_10(void) = ^IndirectReadSideEffect[-1] : &:r8_6, ~mu0_2 +#-----| v8_10(void) = ^BufferReadSideEffect[-1] : &:r8_6, ~mu0_2 #-----| mu8_11(vector) = ^IndirectMayWriteSideEffect[-1] : &:r8_6 # 1075| mu8_12(iterator) = Store : &:r8_4, r8_8 # 1075| r8_13(glval) = VariableAddress[(__end)] : @@ -5195,7 +5195,7 @@ ir.cpp: # 1075| r8_16(glval) = FunctionAddress[end] : # 1075| r8_17(iterator) = Call : func:r8_16, this:r8_15 # 1075| mu8_18(unknown) = ^CallSideEffect : ~mu0_2 -#-----| v8_19(void) = ^IndirectReadSideEffect[-1] : &:r8_15, ~mu0_2 +#-----| v8_19(void) = ^BufferReadSideEffect[-1] : &:r8_15, ~mu0_2 #-----| mu8_20(vector) = ^IndirectMayWriteSideEffect[-1] : &:r8_15 # 1075| mu8_21(iterator) = Store : &:r8_13, r8_17 #-----| Goto -> Block 9 @@ -5208,7 +5208,7 @@ ir.cpp: #-----| r9_4(iterator) = Load : &:r9_3, ~mu0_2 # 1075| r9_5(bool) = Call : func:r9_2, this:r9_1, 0:r9_4 # 1075| mu9_6(unknown) = ^CallSideEffect : ~mu0_2 -#-----| v9_7(void) = ^IndirectReadSideEffect[-1] : &:r9_1, ~mu0_2 +#-----| v9_7(void) = ^BufferReadSideEffect[-1] : &:r9_1, ~mu0_2 #-----| mu9_8(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r9_1 # 1075| v9_9(void) = ConditionalBranch : r9_5 #-----| False -> Block 3 @@ -5219,7 +5219,7 @@ ir.cpp: # 1075| r10_1(glval) = FunctionAddress[operator++] : # 1075| r10_2(iterator &) = Call : func:r10_1, this:r10_0 # 1075| mu10_3(unknown) = ^CallSideEffect : ~mu0_2 -#-----| v10_4(void) = ^IndirectReadSideEffect[-1] : &:r10_0, ~mu0_2 +#-----| v10_4(void) = ^BufferReadSideEffect[-1] : &:r10_0, ~mu0_2 #-----| mu10_5(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r10_0 #-----| Goto (back edge) -> Block 9 @@ -5378,7 +5378,7 @@ ir.cpp: # 1140| r7_3(char *) = Convert : r7_2 # 1140| v7_4(void) = Call : func:r7_1, this:r7_0, 0:r7_3 # 1140| mu7_5(unknown) = ^CallSideEffect : ~mu0_2 -# 1140| v7_6(void) = ^IndirectReadSideEffect[0] : &:r7_3, ~mu0_2 +# 1140| v7_6(void) = ^BufferReadSideEffect[0] : &:r7_3, ~mu0_2 # 1140| mu7_7(unknown) = ^BufferMayWriteSideEffect[0] : &:r7_3 # 1140| v7_8(void) = ThrowValue : &:r7_0, ~mu0_2 #-----| Exception -> Block 9 @@ -5403,7 +5403,7 @@ ir.cpp: # 1145| r10_5(char *) = Load : &:r10_4, ~mu0_2 # 1145| v10_6(void) = Call : func:r10_3, this:r10_2, 0:r10_5 # 1145| mu10_7(unknown) = ^CallSideEffect : ~mu0_2 -# 1145| v10_8(void) = ^IndirectReadSideEffect[0] : &:r10_5, ~mu0_2 +# 1145| v10_8(void) = ^BufferReadSideEffect[0] : &:r10_5, ~mu0_2 # 1145| mu10_9(unknown) = ^BufferMayWriteSideEffect[0] : &:r10_5 # 1145| v10_10(void) = ThrowValue : &:r10_2, ~mu0_2 #-----| Exception -> Block 2 diff --git a/cpp/ql/test/library-tests/ir/ssa/aliased_ssa_ir.expected b/cpp/ql/test/library-tests/ir/ssa/aliased_ssa_ir.expected index 9d005c57ca8..9e32c962fb7 100644 --- a/cpp/ql/test/library-tests/ir/ssa/aliased_ssa_ir.expected +++ b/cpp/ql/test/library-tests/ir/ssa/aliased_ssa_ir.expected @@ -327,7 +327,7 @@ ssa.cpp: # 97| v0_13(void) = Call : func:r0_10, 0:r0_12 # 97| m0_14(unknown) = ^CallSideEffect : ~m0_5 # 97| m0_15(unknown) = Chi : total:m0_5, partial:m0_14 -# 97| v0_16(void) = ^IndirectReadSideEffect[0] : &:r0_12, ~m0_15 +# 97| v0_16(void) = ^BufferReadSideEffect[0] : &:r0_12, ~m0_15 # 97| m0_17(unknown) = ^BufferMayWriteSideEffect[0] : &:r0_12 # 97| m0_18(unknown) = Chi : total:m0_15, partial:m0_17 # 98| v0_19(void) = NoOp : @@ -381,7 +381,7 @@ ssa.cpp: # 108| v0_19(void) = Call : func:r0_16, 0:r0_18 # 108| m0_20(unknown) = ^CallSideEffect : ~m0_5 # 108| m0_21(unknown) = Chi : total:m0_5, partial:m0_20 -# 108| v0_22(void) = ^IndirectReadSideEffect[0] : &:r0_18, ~m0_21 +# 108| v0_22(void) = ^BufferReadSideEffect[0] : &:r0_18, ~m0_21 # 108| m0_23(unknown) = ^BufferMayWriteSideEffect[0] : &:r0_18 # 108| m0_24(unknown) = Chi : total:m0_21, partial:m0_23 # 109| v0_25(void) = NoOp : @@ -451,7 +451,7 @@ ssa.cpp: # 119| v0_27(void) = Call : func:r0_24, 0:r0_26 # 119| m0_28(unknown) = ^CallSideEffect : ~m0_19 # 119| m0_29(unknown) = Chi : total:m0_19, partial:m0_28 -# 119| v0_30(void) = ^IndirectReadSideEffect[0] : &:r0_26, ~m0_29 +# 119| v0_30(void) = ^BufferReadSideEffect[0] : &:r0_26, ~m0_29 # 119| m0_31(unknown) = ^BufferMayWriteSideEffect[0] : &:r0_26 # 119| m0_32(unknown) = Chi : total:m0_29, partial:m0_31 # 120| v0_33(void) = NoOp : diff --git a/cpp/ql/test/library-tests/ir/ssa/unaliased_ssa_ir.expected b/cpp/ql/test/library-tests/ir/ssa/unaliased_ssa_ir.expected index a68690c7055..0a203453a72 100644 --- a/cpp/ql/test/library-tests/ir/ssa/unaliased_ssa_ir.expected +++ b/cpp/ql/test/library-tests/ir/ssa/unaliased_ssa_ir.expected @@ -326,7 +326,7 @@ ssa.cpp: # 97| r0_11(void *) = Convert : r0_10 # 97| v0_12(void) = Call : func:r0_9, 0:r0_11 # 97| mu0_13(unknown) = ^CallSideEffect : ~mu0_2 -# 97| v0_14(void) = ^IndirectReadSideEffect[0] : &:r0_11, ~mu0_2 +# 97| v0_14(void) = ^BufferReadSideEffect[0] : &:r0_11, ~mu0_2 # 97| mu0_15(unknown) = ^BufferMayWriteSideEffect[0] : &:r0_11 # 98| v0_16(void) = NoOp : # 95| v0_17(void) = ReturnVoid : @@ -377,7 +377,7 @@ ssa.cpp: # 108| r0_17(void *) = Convert : r0_16 # 108| v0_18(void) = Call : func:r0_15, 0:r0_17 # 108| mu0_19(unknown) = ^CallSideEffect : ~mu0_2 -# 108| v0_20(void) = ^IndirectReadSideEffect[0] : &:r0_17, ~mu0_2 +# 108| v0_20(void) = ^BufferReadSideEffect[0] : &:r0_17, ~mu0_2 # 108| mu0_21(unknown) = ^BufferMayWriteSideEffect[0] : &:r0_17 # 109| v0_22(void) = NoOp : # 105| v0_23(void) = ReturnVoid : @@ -440,7 +440,7 @@ ssa.cpp: # 119| r0_23(void *) = Convert : r0_22 # 119| v0_24(void) = Call : func:r0_21, 0:r0_23 # 119| mu0_25(unknown) = ^CallSideEffect : ~mu0_2 -# 119| v0_26(void) = ^IndirectReadSideEffect[0] : &:r0_23, ~mu0_2 +# 119| v0_26(void) = ^BufferReadSideEffect[0] : &:r0_23, ~mu0_2 # 119| mu0_27(unknown) = ^BufferMayWriteSideEffect[0] : &:r0_23 # 120| v0_28(void) = NoOp : # 116| v0_29(void) = ReturnVoid : From e57fef093b6f4d7a6202a031270bbf2e2361dad2 Mon Sep 17 00:00:00 2001 From: Robert Marsh Date: Fri, 18 Oct 2019 10:08:53 -0700 Subject: [PATCH 014/148] C++: accept syntax-zoo changes --- .../library-tests/syntax-zoo/aliased_ssa_sanity.expected | 8 ++++---- .../syntax-zoo/unaliased_ssa_sanity.expected | 4 ---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/cpp/ql/test/library-tests/syntax-zoo/aliased_ssa_sanity.expected b/cpp/ql/test/library-tests/syntax-zoo/aliased_ssa_sanity.expected index 5fb356f5701..8512c3dc732 100644 --- a/cpp/ql/test/library-tests/syntax-zoo/aliased_ssa_sanity.expected +++ b/cpp/ql/test/library-tests/syntax-zoo/aliased_ssa_sanity.expected @@ -1,9 +1,9 @@ missingOperand | misc.c:125:5:125:11 | CopyValue: (statement expression) | Instruction 'CopyValue' is missing an expected operand with tag 'Unary' in function '$@'. | misc.c:97:6:97:10 | IR: misc3 | void misc3() | -| parameterinitializer.cpp:27:3:27:6 | IndirectReadSideEffect: my_c | Instruction 'IndirectReadSideEffect' is missing an expected operand with tag 'SideEffect' in function '$@'. | allocators.cpp:14:5:14:8 | IR: main | int main() | -| parameterinitializer.cpp:27:3:27:6 | IndirectReadSideEffect: my_c | Instruction 'IndirectReadSideEffect' is missing an expected operand with tag 'SideEffect' in function '$@'. | no_dynamic_init.cpp:9:5:9:8 | IR: main | int main() | -| parameterinitializer.cpp:27:3:27:6 | IndirectReadSideEffect: my_c | Instruction 'IndirectReadSideEffect' is missing an expected operand with tag 'SideEffect' in function '$@'. | parameterinitializer.cpp:18:5:18:8 | IR: main | int main() | -| parameterinitializer.cpp:27:3:27:6 | IndirectReadSideEffect: my_c | Instruction 'IndirectReadSideEffect' is missing an expected operand with tag 'SideEffect' in function '$@'. | stream_it.cpp:16:5:16:8 | IR: main | int main() | +| parameterinitializer.cpp:27:3:27:6 | BufferReadSideEffect: my_c | Instruction 'BufferReadSideEffect' is missing an expected operand with tag 'SideEffect' in function '$@'. | allocators.cpp:14:5:14:8 | IR: main | int main() | +| parameterinitializer.cpp:27:3:27:6 | BufferReadSideEffect: my_c | Instruction 'BufferReadSideEffect' is missing an expected operand with tag 'SideEffect' in function '$@'. | no_dynamic_init.cpp:9:5:9:8 | IR: main | int main() | +| parameterinitializer.cpp:27:3:27:6 | BufferReadSideEffect: my_c | Instruction 'BufferReadSideEffect' is missing an expected operand with tag 'SideEffect' in function '$@'. | parameterinitializer.cpp:18:5:18:8 | IR: main | int main() | +| parameterinitializer.cpp:27:3:27:6 | BufferReadSideEffect: my_c | Instruction 'BufferReadSideEffect' is missing an expected operand with tag 'SideEffect' in function '$@'. | stream_it.cpp:16:5:16:8 | IR: main | int main() | | try_catch.cpp:13:5:13:16 | ThrowValue: throw ... | Instruction 'ThrowValue' is missing an expected operand with tag 'Load' in function '$@'. | try_catch.cpp:11:6:11:17 | IR: bypass_catch | void bypass_catch() | unexpectedOperand duplicateOperand diff --git a/cpp/ql/test/library-tests/syntax-zoo/unaliased_ssa_sanity.expected b/cpp/ql/test/library-tests/syntax-zoo/unaliased_ssa_sanity.expected index 22a395b3f89..d68395fe136 100644 --- a/cpp/ql/test/library-tests/syntax-zoo/unaliased_ssa_sanity.expected +++ b/cpp/ql/test/library-tests/syntax-zoo/unaliased_ssa_sanity.expected @@ -1,9 +1,5 @@ missingOperand | misc.c:125:5:125:11 | CopyValue: (statement expression) | Instruction 'CopyValue' is missing an expected operand with tag 'Unary' in function '$@'. | misc.c:97:6:97:10 | IR: misc3 | void misc3() | -| parameterinitializer.cpp:27:3:27:6 | IndirectReadSideEffect: my_c | Instruction 'IndirectReadSideEffect' is missing an expected operand with tag 'SideEffect' in function '$@'. | allocators.cpp:14:5:14:8 | IR: main | int main() | -| parameterinitializer.cpp:27:3:27:6 | IndirectReadSideEffect: my_c | Instruction 'IndirectReadSideEffect' is missing an expected operand with tag 'SideEffect' in function '$@'. | no_dynamic_init.cpp:9:5:9:8 | IR: main | int main() | -| parameterinitializer.cpp:27:3:27:6 | IndirectReadSideEffect: my_c | Instruction 'IndirectReadSideEffect' is missing an expected operand with tag 'SideEffect' in function '$@'. | parameterinitializer.cpp:18:5:18:8 | IR: main | int main() | -| parameterinitializer.cpp:27:3:27:6 | IndirectReadSideEffect: my_c | Instruction 'IndirectReadSideEffect' is missing an expected operand with tag 'SideEffect' in function '$@'. | stream_it.cpp:16:5:16:8 | IR: main | int main() | | try_catch.cpp:13:5:13:16 | ThrowValue: throw ... | Instruction 'ThrowValue' is missing an expected operand with tag 'Load' in function '$@'. | try_catch.cpp:11:6:11:17 | IR: bypass_catch | void bypass_catch() | unexpectedOperand duplicateOperand From 8a1d1e7b7a119f1b39db0d424ff34f7a7140e88d Mon Sep 17 00:00:00 2001 From: Taus Brock-Nannestad Date: Mon, 21 Oct 2019 16:07:48 +0200 Subject: [PATCH 015/148] Python: Modernise and false positive in `py/undefined-export`. --- python/ql/src/Variables/UndefinedExport.ql | 14 +++++++------- python/ql/src/semmle/python/Module.qll | 7 +++++++ python/ql/src/semmle/python/objects/ObjectAPI.qll | 6 ++++++ 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/python/ql/src/Variables/UndefinedExport.ql b/python/ql/src/Variables/UndefinedExport.ql index 6bf128def79..cbe43475afa 100644 --- a/python/ql/src/Variables/UndefinedExport.ql +++ b/python/ql/src/Variables/UndefinedExport.ql @@ -22,10 +22,10 @@ predicate declaredInAll(Module m, StrConst name) ) } -predicate mutates_globals(PythonModuleObject m) { +predicate mutates_globals(ModuleValue m) { exists(CallNode globals | globals = Object::builtin("globals").(FunctionObject).getACall() and - globals.getScope() = m.getModule() | + globals.getScope() = m.getScope() | exists(AttrNode attr | attr.getObject() = globals) or exists(SubscriptNode sub | sub.getValue() = globals and sub.isStore()) @@ -34,7 +34,7 @@ predicate mutates_globals(PythonModuleObject m) { exists(Object enum_convert | enum_convert.hasLongName("enum.Enum._convert") and exists(CallNode call | - call.getScope() = m.getModule() + call.getScope() = m.getScope() | enum_convert.(FunctionObject).getACall() = call or call.getFunction().refersTo(enum_convert) @@ -42,11 +42,11 @@ predicate mutates_globals(PythonModuleObject m) { ) } -from PythonModuleObject m, StrConst name, string exported_name -where declaredInAll(m.getModule(), name) and +from ModuleValue m, StrConst name, string exported_name +where declaredInAll(m.getScope(), name) and exported_name = name.strValue() and not m.hasAttribute(exported_name) and -not (m.getShortName() = "__init__" and exists(m.getPackage().getModule().getSubModule(exported_name))) and -not exists(ImportStarNode imp | imp.getEnclosingModule() = m.getModule() | not imp.getModule().refersTo(_)) and +not (m.getScope().getShortName() = "__init__" and exists(m.getScope().getPackage().getSubModule(exported_name))) and +not exists(ImportStarNode imp | imp.getEnclosingModule() = m.getScope() | imp.getModule().pointsTo().isAbsent()) and not mutates_globals(m) select name, "The name '" + exported_name + "' is exported by __all__ but is not defined." \ No newline at end of file diff --git a/python/ql/src/semmle/python/Module.qll b/python/ql/src/semmle/python/Module.qll index a3bf174897f..68361d1b12c 100644 --- a/python/ql/src/semmle/python/Module.qll +++ b/python/ql/src/semmle/python/Module.qll @@ -52,6 +52,13 @@ class Module extends Module_, Scope, AstNode { result = moduleNameFromFile(this.getPath()) } + /** Gets the short name of the module. For example the short name of module x.y.z is 'z' */ + string getShortName() { + result = this.getName().suffix(this.getPackage().getName().length()+1) + or + result = this.getName() and not exists(this.getPackage()) + } + /** Gets this module */ override Module getEnclosingModule() { result = this diff --git a/python/ql/src/semmle/python/objects/ObjectAPI.qll b/python/ql/src/semmle/python/objects/ObjectAPI.qll index dd6b856875a..3fc003b86e5 100644 --- a/python/ql/src/semmle/python/objects/ObjectAPI.qll +++ b/python/ql/src/semmle/python/objects/ObjectAPI.qll @@ -99,6 +99,12 @@ class Value extends TObject { this.(ObjectInternal).hasAttribute(name) } + /** Whether this value is absent from the database, but has been inferred to likely exist */ + predicate isAbsent() { + this instanceof AbsentModuleObjectInternal + or + this instanceof AbsentModuleAttributeObjectInternal + } } /** Class representing modules in the Python program From 4fe1ba0ea401477995f1c1ab1b494461cf576ac1 Mon Sep 17 00:00:00 2001 From: Taus Brock-Nannestad Date: Mon, 21 Oct 2019 16:15:13 +0200 Subject: [PATCH 016/148] Python: Refactor `py/undefined-export` for more clarity. --- python/ql/src/Variables/UndefinedExport.ql | 28 ++++++++++++++++------ 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/python/ql/src/Variables/UndefinedExport.ql b/python/ql/src/Variables/UndefinedExport.ql index cbe43475afa..5e739030f2a 100644 --- a/python/ql/src/Variables/UndefinedExport.ql +++ b/python/ql/src/Variables/UndefinedExport.ql @@ -42,11 +42,25 @@ predicate mutates_globals(ModuleValue m) { ) } +predicate is_exported_submodule_name(ModuleValue m, string exported_name) { + m.getScope().getShortName() = "__init__" and + exists(m.getScope().getPackage().getSubModule(exported_name)) +} + +predicate contains_unknown_import_star(ModuleValue m) { + exists(ImportStarNode imp | imp.getEnclosingModule() = m.getScope() | + imp.getModule().pointsTo().isAbsent() + or + not exists(imp.getModule().pointsTo()) + ) +} + from ModuleValue m, StrConst name, string exported_name -where declaredInAll(m.getScope(), name) and -exported_name = name.strValue() and -not m.hasAttribute(exported_name) and -not (m.getScope().getShortName() = "__init__" and exists(m.getScope().getPackage().getSubModule(exported_name))) and -not exists(ImportStarNode imp | imp.getEnclosingModule() = m.getScope() | imp.getModule().pointsTo().isAbsent()) and -not mutates_globals(m) -select name, "The name '" + exported_name + "' is exported by __all__ but is not defined." \ No newline at end of file +where + declaredInAll(m.getScope(), name) and + exported_name = name.strValue() and + not m.hasAttribute(exported_name) and + not is_exported_submodule_name(m, exported_name) and + not contains_unknown_import_star(m) and + not mutates_globals(m) +select name, "The name '" + exported_name + "' is exported by __all__ but is not defined." From ab2c8f312cc67ceeb28bd3a4e3e6132e9617726a Mon Sep 17 00:00:00 2001 From: Taus Brock-Nannestad Date: Mon, 21 Oct 2019 16:15:48 +0200 Subject: [PATCH 017/148] Python: Apply autoformat. --- python/ql/src/Variables/UndefinedExport.ql | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/python/ql/src/Variables/UndefinedExport.ql b/python/ql/src/Variables/UndefinedExport.ql index 5e739030f2a..b7a835b42d6 100644 --- a/python/ql/src/Variables/UndefinedExport.ql +++ b/python/ql/src/Variables/UndefinedExport.ql @@ -14,18 +14,20 @@ import python /** Whether name is declared in the __all__ list of this module */ -predicate declaredInAll(Module m, StrConst name) -{ - exists(Assign a, GlobalVariable all | - a.defines(all) and a.getScope() = m and - all.getId() = "__all__" and ((List)a.getValue()).getAnElt() = name +predicate declaredInAll(Module m, StrConst name) { + exists(Assign a, GlobalVariable all | + a.defines(all) and + a.getScope() = m and + all.getId() = "__all__" and + a.getValue().(List).getAnElt() = name ) } predicate mutates_globals(ModuleValue m) { exists(CallNode globals | globals = Object::builtin("globals").(FunctionObject).getACall() and - globals.getScope() = m.getScope() | + globals.getScope() = m.getScope() + | exists(AttrNode attr | attr.getObject() = globals) or exists(SubscriptNode sub | sub.getValue() = globals and sub.isStore()) @@ -33,9 +35,7 @@ predicate mutates_globals(ModuleValue m) { or exists(Object enum_convert | enum_convert.hasLongName("enum.Enum._convert") and - exists(CallNode call | - call.getScope() = m.getScope() - | + exists(CallNode call | call.getScope() = m.getScope() | enum_convert.(FunctionObject).getACall() = call or call.getFunction().refersTo(enum_convert) ) From 2e0244cda62deea17afe438c30e289d5c97913ca Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Mon, 21 Oct 2019 20:21:41 +0200 Subject: [PATCH 018/148] address review feedback --- .../ql/src/Expressions/ExprHasNoEffect.ql | 1 + .../ql/src/Expressions/ExprHasNoEffect.qll | 1 - .../src/Statements/UseOfReturnlessFunction.ql | 25 +++++++++++++++---- .../UseOfReturnlessFunction.expected | 8 +++--- .../Statements/UseOfReturnlessFunction/tst.js | 3 +++ 5 files changed, 28 insertions(+), 10 deletions(-) diff --git a/javascript/ql/src/Expressions/ExprHasNoEffect.ql b/javascript/ql/src/Expressions/ExprHasNoEffect.ql index 929e84616f0..917ab81a3e7 100644 --- a/javascript/ql/src/Expressions/ExprHasNoEffect.ql +++ b/javascript/ql/src/Expressions/ExprHasNoEffect.ql @@ -14,6 +14,7 @@ import javascript import ExprHasNoEffect +import semmle.javascript.RestrictedLocations from Expr e diff --git a/javascript/ql/src/Expressions/ExprHasNoEffect.qll b/javascript/ql/src/Expressions/ExprHasNoEffect.qll index 822bcbb26fa..bee980e5fd8 100644 --- a/javascript/ql/src/Expressions/ExprHasNoEffect.qll +++ b/javascript/ql/src/Expressions/ExprHasNoEffect.qll @@ -5,7 +5,6 @@ import javascript import DOMProperties import semmle.javascript.frameworks.xUnit -import semmle.javascript.RestrictedLocations /** * Holds if `e` appears in a syntactic context where its value is discarded. diff --git a/javascript/ql/src/Statements/UseOfReturnlessFunction.ql b/javascript/ql/src/Statements/UseOfReturnlessFunction.ql index 2d70cecb0d3..fe03c66cf4a 100644 --- a/javascript/ql/src/Statements/UseOfReturnlessFunction.ql +++ b/javascript/ql/src/Statements/UseOfReturnlessFunction.ql @@ -82,8 +82,19 @@ predicate alwaysThrows(Function f) { ) } +/** + * Holds if the last statement in the function is flagged by the js/useless-expression query. + */ +predicate alwaysHasNoEffect(Function f) { + exists(ReachableBasicBlock entry, DataFlow::Node noEffect | + entry = f.getEntryBB() and + hasNoEffect(noEffect.asExpr()) and + entry.dominates(noEffect.getBasicBlock()) + ) +} + predicate callToVoidFunction(DataFlow::CallNode call, Function func) { - not call.isIndefinite(_) and + not call.isIncomplete() and func = call.getACallee() and forall(Function f | f = call.getACallee() | returnsVoid(f) and not isStub(f) and not alwaysThrows(f) @@ -113,6 +124,11 @@ DataFlow::SourceNode array(DataFlow::TypeTracker t) { DataFlow::SourceNode array() { result = array(DataFlow::TypeTracker::end()) } +/** + * Holds if `call` is an Array or Lodash method accepting a callback `func`, + * where the `call` expects a callback that returns an expression, + * but `func` does return a value. + */ predicate voidArrayCallback(DataFlow::CallNode call, Function func) { hasNonVoidCallbackMethod(call.getCalleeName()) and func = call.getAnArgument().getALocalSource().asExpr() and @@ -132,15 +148,14 @@ where ( callToVoidFunction(call, func) and msg = "the $@ does not return anything, yet the return value is used." and - name = "function " + call.getCalleeName() + name = func.describe() or voidArrayCallback(call, func) and - msg = "the $@ does not return anything, yet the return value from the call to " + call.getCalleeName() + "() is used." and + msg = "the $@ does not return anything, yet the return value from the call to " + call.getCalleeName() + " is used." and name = "callback function" ) and not benignContext(call.asExpr()) and - // Avoid double reporting from js/useless-expression - not hasNoEffect(func.getBodyStmt(func.getNumBodyStmt() - 1).(ExprStmt).getExpr()) and + not alwaysHasNoEffect(func) and // anonymous one-shot closure. Those are used in weird ways and we ignore them. not oneshotClosure(call.asExpr()) select diff --git a/javascript/ql/test/query-tests/Statements/UseOfReturnlessFunction/UseOfReturnlessFunction.expected b/javascript/ql/test/query-tests/Statements/UseOfReturnlessFunction/UseOfReturnlessFunction.expected index 98ae4e3696c..f23b702bf20 100644 --- a/javascript/ql/test/query-tests/Statements/UseOfReturnlessFunction/UseOfReturnlessFunction.expected +++ b/javascript/ql/test/query-tests/Statements/UseOfReturnlessFunction/UseOfReturnlessFunction.expected @@ -1,7 +1,7 @@ | tst.js:20:17:20:33 | onlySideEffects() | the $@ does not return anything, yet the return value is used. | tst.js:11:5:13:5 | functio ... )\\n } | function onlySideEffects | | tst.js:24:13:24:29 | onlySideEffects() | the $@ does not return anything, yet the return value is used. | tst.js:11:5:13:5 | functio ... )\\n } | function onlySideEffects | | tst.js:30:20:30:36 | onlySideEffects() | the $@ does not return anything, yet the return value is used. | tst.js:11:5:13:5 | functio ... )\\n } | function onlySideEffects | -| tst.js:53:10:53:34 | bothOnl ... fects() | the $@ does not return anything, yet the return value is used. | tst.js:11:5:13:5 | functio ... )\\n } | function bothOnlyHaveSideEffects | -| tst.js:53:10:53:34 | bothOnl ... fects() | the $@ does not return anything, yet the return value is used. | tst.js:48:2:50:5 | functio ... )\\n } | function bothOnlyHaveSideEffects | -| tst.js:76:12:76:46 | [1,2,3] ... n, 3)}) | the $@ does not return anything, yet the return value from the call to filter() is used. | tst.js:76:27:76:45 | n => {equals(n, 3)} | callback function | -| tst.js:80:12:80:50 | filter( ... 3) } ) | the $@ does not return anything, yet the return value from the call to filter() is used. | tst.js:80:28:80:48 | x => { ... x, 3) } | callback function | +| tst.js:53:10:53:34 | bothOnl ... fects() | the $@ does not return anything, yet the return value is used. | tst.js:11:5:13:5 | functio ... )\\n } | function onlySideEffects | +| tst.js:53:10:53:34 | bothOnl ... fects() | the $@ does not return anything, yet the return value is used. | tst.js:48:2:50:5 | functio ... )\\n } | function onlySideEffects2 | +| tst.js:76:12:76:46 | [1,2,3] ... n, 3)}) | the $@ does not return anything, yet the return value from the call to filter is used. | tst.js:76:27:76:45 | n => {equals(n, 3)} | callback function | +| tst.js:80:12:80:50 | filter( ... 3) } ) | the $@ does not return anything, yet the return value from the call to filter is used. | tst.js:80:28:80:48 | x => { ... x, 3) } | callback function | diff --git a/javascript/ql/test/query-tests/Statements/UseOfReturnlessFunction/tst.js b/javascript/ql/test/query-tests/Statements/UseOfReturnlessFunction/tst.js index 2b523223e91..8a6a44707ed 100644 --- a/javascript/ql/test/query-tests/Statements/UseOfReturnlessFunction/tst.js +++ b/javascript/ql/test/query-tests/Statements/UseOfReturnlessFunction/tst.js @@ -79,4 +79,7 @@ import { filter } from 'lodash' var bar = filter([1,2,4], x => { equals(x, 3) } ) // NOT OK! console.log(bar); + + var baz = [1,2,3].filter(n => {n === 3}) // OK + console.log(baz); })(); \ No newline at end of file From db22916850202d1e159fc21e651670f501d29bc2 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Tue, 22 Oct 2019 09:37:19 +0200 Subject: [PATCH 019/148] fix the alwaysHasNoEffect predicate, and rename it to lastStatementHasNoEffect --- .../ql/src/Statements/UseOfReturnlessFunction.ql | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/javascript/ql/src/Statements/UseOfReturnlessFunction.ql b/javascript/ql/src/Statements/UseOfReturnlessFunction.ql index fe03c66cf4a..0f168e263dd 100644 --- a/javascript/ql/src/Statements/UseOfReturnlessFunction.ql +++ b/javascript/ql/src/Statements/UseOfReturnlessFunction.ql @@ -85,11 +85,11 @@ predicate alwaysThrows(Function f) { /** * Holds if the last statement in the function is flagged by the js/useless-expression query. */ -predicate alwaysHasNoEffect(Function f) { - exists(ReachableBasicBlock entry, DataFlow::Node noEffect | - entry = f.getEntryBB() and - hasNoEffect(noEffect.asExpr()) and - entry.dominates(noEffect.getBasicBlock()) +predicate lastStatementHasNoEffect(Function f) { + exists(DataFlow::Node noEffect | + noEffect.getContainer() = f and + hasNoEffect(noEffect.asExpr()) and + not exists(noEffect.getASuccessor()) ) } @@ -155,7 +155,7 @@ where name = "callback function" ) and not benignContext(call.asExpr()) and - not alwaysHasNoEffect(func) and + not lastStatementHasNoEffect(func) and // anonymous one-shot closure. Those are used in weird ways and we ignore them. not oneshotClosure(call.asExpr()) select From ad3185c5586efc622cea59956868e729a545ae4a Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Tue, 22 Oct 2019 10:33:05 +0200 Subject: [PATCH 020/148] simplify lastStatementHasNoEffect and use the control-flow to determine which statement is the last --- javascript/ql/src/Statements/UseOfReturnlessFunction.ql | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/javascript/ql/src/Statements/UseOfReturnlessFunction.ql b/javascript/ql/src/Statements/UseOfReturnlessFunction.ql index 0f168e263dd..2fc3f01a560 100644 --- a/javascript/ql/src/Statements/UseOfReturnlessFunction.ql +++ b/javascript/ql/src/Statements/UseOfReturnlessFunction.ql @@ -86,11 +86,7 @@ predicate alwaysThrows(Function f) { * Holds if the last statement in the function is flagged by the js/useless-expression query. */ predicate lastStatementHasNoEffect(Function f) { - exists(DataFlow::Node noEffect | - noEffect.getContainer() = f and - hasNoEffect(noEffect.asExpr()) and - not exists(noEffect.getASuccessor()) - ) + hasNoEffect(f.getExit().getAPredecessor()) } predicate callToVoidFunction(DataFlow::CallNode call, Function func) { From ee7cf17b1567028914db80b26ba33aec54290251 Mon Sep 17 00:00:00 2001 From: Calum Grant Date: Wed, 23 Oct 2019 11:22:52 +0100 Subject: [PATCH 021/148] C#: Add test case for local disposal. --- .../NoDisposeCallOnLocalIDisposable.cs | 5 +++++ .../NoDisposeCallOnLocalIDisposable.expected | 1 + 2 files changed, 6 insertions(+) diff --git a/csharp/ql/test/query-tests/API Abuse/NoDisposeCallOnLocalIDisposable/NoDisposeCallOnLocalIDisposable.cs b/csharp/ql/test/query-tests/API Abuse/NoDisposeCallOnLocalIDisposable/NoDisposeCallOnLocalIDisposable.cs index 45caec0022d..c5336b72ab3 100644 --- a/csharp/ql/test/query-tests/API Abuse/NoDisposeCallOnLocalIDisposable/NoDisposeCallOnLocalIDisposable.cs +++ b/csharp/ql/test/query-tests/API Abuse/NoDisposeCallOnLocalIDisposable/NoDisposeCallOnLocalIDisposable.cs @@ -83,6 +83,11 @@ class Test // GOOD: Disposed automatically. using var c2 = new Timer(TimerProc); + // GOOD: ownership taken via ?? (false positive) + StringReader source = null; + using(XmlReader.Create(source ?? new StringReader("xml"), null)) + ; + return null; } diff --git a/csharp/ql/test/query-tests/API Abuse/NoDisposeCallOnLocalIDisposable/NoDisposeCallOnLocalIDisposable.expected b/csharp/ql/test/query-tests/API Abuse/NoDisposeCallOnLocalIDisposable/NoDisposeCallOnLocalIDisposable.expected index aa70ad5a1cd..128686a7b63 100644 --- a/csharp/ql/test/query-tests/API Abuse/NoDisposeCallOnLocalIDisposable/NoDisposeCallOnLocalIDisposable.expected +++ b/csharp/ql/test/query-tests/API Abuse/NoDisposeCallOnLocalIDisposable/NoDisposeCallOnLocalIDisposable.expected @@ -2,4 +2,5 @@ | NoDisposeCallOnLocalIDisposable.cs:53:18:53:73 | object creation of type FileStream | Disposable 'FileStream' is created here but is not disposed. | | NoDisposeCallOnLocalIDisposable.cs:54:9:54:64 | object creation of type FileStream | Disposable 'FileStream' is created here but is not disposed. | | NoDisposeCallOnLocalIDisposable.cs:76:25:76:71 | call to method Create | Disposable 'XmlReader' is created here but is not disposed. | +| NoDisposeCallOnLocalIDisposable.cs:88:42:88:64 | object creation of type StringReader | Disposable 'StringReader' is created here but is not disposed. | | NoDisposeCallOnLocalIDisposableBad.cs:8:22:8:56 | object creation of type FileStream | Disposable 'FileStream' is created here but is not disposed. | From 01ad93d199d8d5bdfb3aebd5e3d0bc251e28eabb Mon Sep 17 00:00:00 2001 From: Calum Grant Date: Wed, 23 Oct 2019 12:26:01 +0100 Subject: [PATCH 022/148] C#: Fix for false positive. --- .../src/API Abuse/NoDisposeCallOnLocalIDisposable.ql | 11 ++++++++++- .../NoDisposeCallOnLocalIDisposable.expected | 1 - 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/csharp/ql/src/API Abuse/NoDisposeCallOnLocalIDisposable.ql b/csharp/ql/src/API Abuse/NoDisposeCallOnLocalIDisposable.ql index 646ab04ac1f..d8de5242fcc 100644 --- a/csharp/ql/src/API Abuse/NoDisposeCallOnLocalIDisposable.ql +++ b/csharp/ql/src/API Abuse/NoDisposeCallOnLocalIDisposable.ql @@ -54,7 +54,16 @@ private class Conf extends DataFlow::Configuration { ) or // A disposing method - exists(Call c, Parameter p | e = c.getArgumentForParameter(p) | mayBeDisposed(p)) + exists(Call c, Parameter p | + e = c.getArgumentForParameter(p) + or + // e.g `Stream.Create(input ?? new TextReader())` + exists(NullCoalescingExpr nce | + nce = c.getArgumentForParameter(p) and e = nce.getRightOperand() + ) + | + mayBeDisposed(p) + ) or // Things that are assigned to fields, properties, or indexers may be disposed exists(AssignableDefinition def, Assignable a | diff --git a/csharp/ql/test/query-tests/API Abuse/NoDisposeCallOnLocalIDisposable/NoDisposeCallOnLocalIDisposable.expected b/csharp/ql/test/query-tests/API Abuse/NoDisposeCallOnLocalIDisposable/NoDisposeCallOnLocalIDisposable.expected index 128686a7b63..aa70ad5a1cd 100644 --- a/csharp/ql/test/query-tests/API Abuse/NoDisposeCallOnLocalIDisposable/NoDisposeCallOnLocalIDisposable.expected +++ b/csharp/ql/test/query-tests/API Abuse/NoDisposeCallOnLocalIDisposable/NoDisposeCallOnLocalIDisposable.expected @@ -2,5 +2,4 @@ | NoDisposeCallOnLocalIDisposable.cs:53:18:53:73 | object creation of type FileStream | Disposable 'FileStream' is created here but is not disposed. | | NoDisposeCallOnLocalIDisposable.cs:54:9:54:64 | object creation of type FileStream | Disposable 'FileStream' is created here but is not disposed. | | NoDisposeCallOnLocalIDisposable.cs:76:25:76:71 | call to method Create | Disposable 'XmlReader' is created here but is not disposed. | -| NoDisposeCallOnLocalIDisposable.cs:88:42:88:64 | object creation of type StringReader | Disposable 'StringReader' is created here but is not disposed. | | NoDisposeCallOnLocalIDisposableBad.cs:8:22:8:56 | object creation of type FileStream | Disposable 'FileStream' is created here but is not disposed. | From 6b15bf62fd65bc63448365c0aabe045adb50fa75 Mon Sep 17 00:00:00 2001 From: Calum Grant Date: Wed, 23 Oct 2019 13:49:22 +0100 Subject: [PATCH 023/148] C#: Rewrite null-coalsecing logic --- .../API Abuse/NoDisposeCallOnLocalIDisposable.ql | 13 +++---------- .../NoDisposeCallOnLocalIDisposable.cs | 2 +- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/csharp/ql/src/API Abuse/NoDisposeCallOnLocalIDisposable.ql b/csharp/ql/src/API Abuse/NoDisposeCallOnLocalIDisposable.ql index d8de5242fcc..3a50d474577 100644 --- a/csharp/ql/src/API Abuse/NoDisposeCallOnLocalIDisposable.ql +++ b/csharp/ql/src/API Abuse/NoDisposeCallOnLocalIDisposable.ql @@ -54,16 +54,7 @@ private class Conf extends DataFlow::Configuration { ) or // A disposing method - exists(Call c, Parameter p | - e = c.getArgumentForParameter(p) - or - // e.g `Stream.Create(input ?? new TextReader())` - exists(NullCoalescingExpr nce | - nce = c.getArgumentForParameter(p) and e = nce.getRightOperand() - ) - | - mayBeDisposed(p) - ) + exists(Call c, Parameter p | e = c.getArgumentForParameter(p) | mayBeDisposed(p)) or // Things that are assigned to fields, properties, or indexers may be disposed exists(AssignableDefinition def, Assignable a | @@ -90,6 +81,8 @@ private class Conf extends DataFlow::Configuration { override predicate isAdditionalFlowStep(DataFlow::Node node1, DataFlow::Node node2) { node2.asExpr() = any(LocalScopeDisposableCreation other | other.getAnArgument() = node1.asExpr()) + or + exists(NullCoalescingExpr nce | node1.asExpr() = nce.getRightOperand() and node2.asExpr() = nce) } override predicate isBarrierOut(DataFlow::Node node) { diff --git a/csharp/ql/test/query-tests/API Abuse/NoDisposeCallOnLocalIDisposable/NoDisposeCallOnLocalIDisposable.cs b/csharp/ql/test/query-tests/API Abuse/NoDisposeCallOnLocalIDisposable/NoDisposeCallOnLocalIDisposable.cs index c5336b72ab3..82a33086144 100644 --- a/csharp/ql/test/query-tests/API Abuse/NoDisposeCallOnLocalIDisposable/NoDisposeCallOnLocalIDisposable.cs +++ b/csharp/ql/test/query-tests/API Abuse/NoDisposeCallOnLocalIDisposable/NoDisposeCallOnLocalIDisposable.cs @@ -83,7 +83,7 @@ class Test // GOOD: Disposed automatically. using var c2 = new Timer(TimerProc); - // GOOD: ownership taken via ?? (false positive) + // GOOD: ownership taken via ?? StringReader source = null; using(XmlReader.Create(source ?? new StringReader("xml"), null)) ; From 60226801aa2f92e14c7299a3e6dd1a7b78047a7d Mon Sep 17 00:00:00 2001 From: Shati Patel Date: Wed, 23 Oct 2019 14:54:02 +0100 Subject: [PATCH 024/148] Docs: Update terminology A more in-depth attempt at changing terminology for GHU. I've only updated the non-language specific topics so far. --- docs/language/learn-ql/about-ql.rst | 4 +- .../learn-ql/advanced/abstract-classes.rst | 4 +- .../learn-ql/advanced/advanced-ql.rst | 2 +- .../learn-ql/advanced/constraining-types.rst | 2 +- .../determining-specific-types-variables.rst | 8 ++-- .../learn-ql/advanced/equivalence.rst | 2 +- .../advanced/monotonic-aggregates.rst | 2 +- .../learn-ql/beginner/find-thief-3.rst | 2 +- docs/language/learn-ql/beginner/fire-2.rst | 2 +- docs/language/learn-ql/beginner/heir.rst | 2 +- .../learn-ql/beginner/ql-tutorials.rst | 13 +++--- docs/language/learn-ql/database.rst | 33 +++++++++++++++ docs/language/learn-ql/index.rst | 35 ++++++++-------- docs/language/learn-ql/intro-to-data-flow.rst | 24 +++++------ docs/language/learn-ql/introduction-to-ql.rst | 33 +++++++++------ docs/language/learn-ql/locations.rst | 10 +++-- .../learn-ql/python/pointsto-type-infer.rst | 4 +- .../learn-ql/ql-etudes/river-crossing.rst | 4 +- docs/language/learn-ql/snapshot.rst | 33 --------------- docs/language/learn-ql/technical-info.rst | 4 +- .../introduction-to-queries.rst | 42 ++++++++++--------- .../learn-ql/writing-queries/path-queries.rst | 28 ++++++------- .../learn-ql/writing-queries/query-help.rst | 12 +++--- .../writing-queries/select-statement.rst | 2 +- .../writing-queries/writing-queries.rst | 23 +++++----- docs/language/ql-handbook/annotations.rst | 2 +- docs/language/ql-handbook/index.rst | 2 +- docs/language/ql-handbook/types.rst | 6 +-- .../ql-training/slide-snippets/info.rst | 2 +- 29 files changed, 179 insertions(+), 163 deletions(-) create mode 100644 docs/language/learn-ql/database.rst delete mode 100644 docs/language/learn-ql/snapshot.rst diff --git a/docs/language/learn-ql/about-ql.rst b/docs/language/learn-ql/about-ql.rst index 145f98ad600..8f2f26018a4 100644 --- a/docs/language/learn-ql/about-ql.rst +++ b/docs/language/learn-ql/about-ql.rst @@ -1,11 +1,11 @@ About QL ======== -This section is aimed at users with a background in general purpose programming as well as in databases. For a basic introduction and information on how to get started, see :doc:`Introduction to the QL language ` and :doc:`Learning QL <../index>`. +This section is aimed at users with a background in general purpose programming as well as in databases. For a basic introduction and information on how to get started, see :doc:`Introduction to QL ` and :doc:`Learning CodeQL <../index>`. QL is a declarative, object-oriented query language that is optimized to enable efficient analysis of hierarchical data structures, in particular, databases representing software artifacts. -The queries and metrics used in LGTM are implemented using QL. This ensures that they can be extended or revised easily to keep up with changes in definitions of best coding practice. We continually improve existing queries as we work towards the ultimate goal of 100% precision. +The queries and metrics used in LGTM are implemented using CodeQL, which uses QL to analyze code. This ensures that they can be extended or revised easily to keep up with changes in definitions of best coding practice. We continually improve existing queries as we work towards the ultimate goal of 100% precision. You can write queries to identify security vulnerabilities, find coding errors and bugs, or find code that breaks your team's guidelines for best practice. You can also create customized versions of the default queries to accommodate a new framework. diff --git a/docs/language/learn-ql/advanced/abstract-classes.rst b/docs/language/learn-ql/advanced/abstract-classes.rst index d616a31395b..32441b35996 100644 --- a/docs/language/learn-ql/advanced/abstract-classes.rst +++ b/docs/language/learn-ql/advanced/abstract-classes.rst @@ -4,7 +4,7 @@ Semantics of abstract classes Concrete classes ---------------- -Concrete QL classes, as described in the QL language handbook topic on `Classes `__, lend themselves well to top-down modeling. We start from general superclasses representing large sets of values, and carve out individual subclasses representing more restricted sets of values. +Concrete classes, as described in the QL language handbook topic on `Classes `__, lend themselves well to top-down modeling. We start from general superclasses representing large sets of values, and carve out individual subclasses representing more restricted sets of values. A classic example where this approach is useful is when modeling ASTs (Abstract Syntax Trees): the node types of an AST form a natural inheritance hierarchy, where, for example, there is a class ``Expr`` representing all expression nodes, with many different subclasses for different categories of expressions. There might be a class ``ArithmeticExpr`` representing arithmetic expressions, which in turn could have subclasses ``AddExpr`` and ``SubExpr``. @@ -57,7 +57,7 @@ Like a concrete class, an abstract class has one or more superclasses and a char Example ~~~~~~~ -The following example is taken from the standard QL library for Java: +The following example is taken from the CodeQL library for Java: .. code-block:: ql diff --git a/docs/language/learn-ql/advanced/advanced-ql.rst b/docs/language/learn-ql/advanced/advanced-ql.rst index e9397bd5ab2..67af5a6fb26 100644 --- a/docs/language/learn-ql/advanced/advanced-ql.rst +++ b/docs/language/learn-ql/advanced/advanced-ql.rst @@ -8,7 +8,7 @@ Advanced QL ./* -Topics on advanced uses of QL. These topics assume that you are familiar with the QL language and the basics of query writing. +Topics on advanced uses of QL. These topics assume that you are familiar with QL and the basics of query writing. - :doc:`Semantics of abstract classes ` - :doc:`Choosing appropriate ways to constrain types ` diff --git a/docs/language/learn-ql/advanced/constraining-types.rst b/docs/language/learn-ql/advanced/constraining-types.rst index e150c8ba380..8c00f86fe50 100644 --- a/docs/language/learn-ql/advanced/constraining-types.rst +++ b/docs/language/learn-ql/advanced/constraining-types.rst @@ -8,7 +8,7 @@ Type constraint methods Note - The examples below use the Java QL library. All QL libraries support using these methods to constrain variables, the only difference is in the names of the classes used. + The examples below use the CodeQL library for Java. All CodeQL libraries support using these methods to constrain variables, the only difference is in the names of the classes used. There are several ways of imposing type constraints on variables: diff --git a/docs/language/learn-ql/advanced/determining-specific-types-variables.rst b/docs/language/learn-ql/advanced/determining-specific-types-variables.rst index ce3c5a9e211..7deadde39c9 100644 --- a/docs/language/learn-ql/advanced/determining-specific-types-variables.rst +++ b/docs/language/learn-ql/advanced/determining-specific-types-variables.rst @@ -1,14 +1,14 @@ Determining the most specific types of a variable ================================================= -It is sometimes useful to be able to determine what QL types an entity has -- especially when you are unfamiliar with the QL library used by a query. To help with this, QL provides a predicate called ``getAQlClass()``, which returns the most specific QL types of the entity that it is called on. +It is sometimes useful to be able to determine what types an entity has -- especially when you are unfamiliar with the library used by a query. To help with this, there is a predicate called ``getAQlClass()``, which returns the most specific QL types of the entity that it is called on. This can be useful when you are not sure of the most precise class of a value. Discovering a more precise class can allow you to cast to it and use predicates that are not available on the more general class. Example ------- -If you were working with a Java snapshot database, you might use ``getAQlClass()`` on every ``Expr`` in a callable called ``c``: +If you were working with a Java database, you might use ``getAQlClass()`` on every ``Expr`` in a callable called ``c``: **Java example** @@ -23,6 +23,6 @@ If you were working with a Java snapshot database, you might use ``getAQlClass() and e.getEnclosingCallable() = c select e, e.getAQlClass() -The result of this query is a list of the most specific types of every ``Expr`` in that function. You will see multiple results for some expressions because that expression is represented by more than one QL type. +The result of this query is a list of the most specific types of every ``Expr`` in that function. You will see multiple results for some expressions because that expression is represented by more than one type. -For example, ``StringLiteral``\ s like ``"Hello"`` in Java belong to both the ``StringLiteral`` QL class (a specialization of the ``Literal`` QL class) and the ``CompileTimeConstantExpr`` QL class. So any instances of ``StringLiteral``\ s in the results will produce more than one result, one for each of the QL classes to which they belong. +For example, ``StringLiteral``\ s like ``"Hello"`` in Java belong to both the ``StringLiteral`` class (a specialization of the ``Literal`` class) and the ``CompileTimeConstantExpr`` class. So any instances of ``StringLiteral``\ s in the results will produce more than one result, one for each of the classes to which they belong. diff --git a/docs/language/learn-ql/advanced/equivalence.rst b/docs/language/learn-ql/advanced/equivalence.rst index 40f00cde00f..bf5efb595d0 100644 --- a/docs/language/learn-ql/advanced/equivalence.rst +++ b/docs/language/learn-ql/advanced/equivalence.rst @@ -6,7 +6,7 @@ The two expressions: #. ``a() != b()`` #. ``not(a() = b())`` -look equivalent - so much so that inexperienced (and even experienced) QL programmers have been known to rewrite one as the other. However, they are not equivalent due to the quantifiers involved. +look equivalent - so much so that inexperienced (and even experienced) programmers have been known to rewrite one as the other. However, they are not equivalent due to the quantifiers involved. Thinking of ``a()`` and ``b()`` as sets of values, the first expression says that there is a pair of values (one from each side of the inequality) which are different. diff --git a/docs/language/learn-ql/advanced/monotonic-aggregates.rst b/docs/language/learn-ql/advanced/monotonic-aggregates.rst index 515dec6dff0..161776dbcf6 100644 --- a/docs/language/learn-ql/advanced/monotonic-aggregates.rst +++ b/docs/language/learn-ql/advanced/monotonic-aggregates.rst @@ -1,7 +1,7 @@ Monotonic aggregates in QL ========================== -In addition to standard QL aggregates, QL also supports *monotonic* aggregates. These are a slightly different way of computing aggregates which have some advantages, notably the ability to be used recursively, which normal aggregates do not have. You can enable them in a scope by adding the \ ``language[monotonicAggregates]`` pragma on a predicate, class, or module. +In addition to standard aggregates, QL also supports *monotonic* aggregates. These are a slightly different way of computing aggregates which have some advantages, notably the ability to be used recursively, which normal aggregates do not have. You can enable them in a scope by adding the \ ``language[monotonicAggregates]`` pragma on a predicate, class, or module. Syntax ------ diff --git a/docs/language/learn-ql/beginner/find-thief-3.rst b/docs/language/learn-ql/beginner/find-thief-3.rst index 03f7c866230..f8323147cf1 100644 --- a/docs/language/learn-ql/beginner/find-thief-3.rst +++ b/docs/language/learn-ql/beginner/find-thief-3.rst @@ -77,4 +77,4 @@ What next? - Help the villagers track down another criminal in the :doc:`next tutorial `. - Find out more about the concepts you discovered in this tutorial in the `QL language handbook `__. -- Explore the libraries that help you get data about code in :doc:`Learning QL <../../index>`. +- Explore the libraries that help you get data about code in :doc:`Learning CodeQL <../../index>`. diff --git a/docs/language/learn-ql/beginner/fire-2.rst b/docs/language/learn-ql/beginner/fire-2.rst index c5380fb0b6b..0c7e4852ae7 100644 --- a/docs/language/learn-ql/beginner/fire-2.rst +++ b/docs/language/learn-ql/beginner/fire-2.rst @@ -40,4 +40,4 @@ What next? - Find out who will be the new ruler of the village in the :doc:`next tutorial `. - Learn more about predicates and classes in the `QL language handbook `__. -- Explore the libraries that help you get data about code in :doc:`Learning QL <../../index>`. +- Explore the libraries that help you get data about code in :doc:`Learning CodeQL <../../index>`. diff --git a/docs/language/learn-ql/beginner/heir.rst b/docs/language/learn-ql/beginner/heir.rst index b5efd73fc40..f0f0c315025 100644 --- a/docs/language/learn-ql/beginner/heir.rst +++ b/docs/language/learn-ql/beginner/heir.rst @@ -161,4 +161,4 @@ What next? - Learn more about recursion in the `QL language handbook `__. - Put your QL skills to the test and solve the :doc:`River crossing puzzle <../ql-etudes/river-crossing>`. -- Start using QL to analyze projects. See :doc:`Learning QL <../../index>` for a summary of the available languages and resources. +- Start using QL to analyze projects. See :doc:`Learning CodeQL <../../index>` for a summary of the available languages and resources. diff --git a/docs/language/learn-ql/beginner/ql-tutorials.rst b/docs/language/learn-ql/beginner/ql-tutorials.rst index 2ba6e449b86..d97c4d6688e 100644 --- a/docs/language/learn-ql/beginner/ql-tutorials.rst +++ b/docs/language/learn-ql/beginner/ql-tutorials.rst @@ -7,18 +7,21 @@ QL detective tutorials ./* -Welcome to QL! These tutorials are aimed at complete beginners. They teach you how to write QL queries and introduce you to key logic concepts along the way. +Welcome to the detective tutorials! These are aimed at complete beginners who would like to learn the basics of QL, +before analyzing code with CodeQL. +The tutorials teach you how to write queries and introduce you to key logic concepts along the way. -We recommend you first read the :doc:`Introduction to the QL language <../introduction-to-ql>` page for a basic description of QL. +We recommend you first read the :doc:`Introduction to QL <../introduction-to-ql>` page for a description of the language and +some simple examples. -Currently the following tutorials are available: +Currently the following detective tutorials are available: -- :doc:`Find the thief ` - a three part mystery that introduces logical connectives, quantifiers and aggregates +- :doc:`Find the thief ` - a three part mystery that introduces logical connectives, quantifiers, and aggregates - :doc:`Catch the fire starter ` - an intriguing search that introduces predicates and classes - :doc:`Crown the rightful heir ` - a detective puzzle that introduces recursion Further resources ----------------- -- For a summary of available learning resources, see :doc:`Learning QL <../../index>`. +- For a summary of available learning resources, see :doc:`Learning CodeQL <../../index>`. - For an overview of the important concepts in QL, see the `QL language handbook `__. diff --git a/docs/language/learn-ql/database.rst b/docs/language/learn-ql/database.rst new file mode 100644 index 00000000000..70ba0091853 --- /dev/null +++ b/docs/language/learn-ql/database.rst @@ -0,0 +1,33 @@ +What's in a CodeQL database? +============================ + +A CodeQL database contains a variety of data related to a particular code base at a particular point in time. For details of how the database is generated see `Database generation `__. + +The database contains a full, hierarchical representation of the program defined by the code base. The database schema varies according to the language analyzed. The schema provides an interface between the initial lexical analysis during the extraction process, and the actual complex analysis using CodeQL. When the source code languages being analyzed change (such as Java 7 evolving into Java 8), this interface between the analysis phases can also change. + +For each language, a CodeQL library defines classes to provide a layer of abstraction over the database tables. This provides an object-oriented view of the data which makes it easier to write queries. This is easiest to explain using an example. + +Example +------- + +For a Java program, two key tables are: + +- The ``expressions`` table containing a row for every single expression in the source code that was analyzed during the build process. +- The ``statements`` table containing a row for every single statement in the source code that was analyzed during the build process. + +The CodeQL library defines classes to provide a layer of abstraction over each of these tables (and the related auxiliary tables): ``Expr`` and ``Stmt``. + +Most classes in the CodeQL library are similar: they are abstractions over one or more database tables. Looking at one of the CodeQL libraries illustrates this: + +.. code-block:: ql + + class Expr extends StmtParent, @expr { + ... + + /** the location of this expression */ + Location getLocation() { exprs(this,_,_,result) } + + ... + } + +The ``Expr`` class, shown here, extends from the database type ``@expr``. Member predicates of the ``Expr`` class are implemented in terms of the database-provided ``exprs`` table. diff --git a/docs/language/learn-ql/index.rst b/docs/language/learn-ql/index.rst index 3fc348be096..76649890131 100644 --- a/docs/language/learn-ql/index.rst +++ b/docs/language/learn-ql/index.rst @@ -1,14 +1,13 @@ -Learning QL -########### +Learning CodeQL +############### +CodeQL is the code analysis platform used by security researchers to automate `variant analysis `__. +You can use CodeQL queries to explore code and quickly find variants of security vulnerabilities and bugs. +These queries are easy to write and share–visit the topics below and `our open source repository on GitHub `__ to learn more. +You can also try out CodeQL in the `query console `__ on `LGTM.com `__. +Here, you can query open source projects directly, without having to download CodeQL databases and libraries. -`QL `__ is the query language used in Semmle's `variant analysis `__ engine. -You can use queries written in QL to explore code and quickly find variants of security vulnerabilities and bugs. -The QL language is also part of the technology behind `LGTM `__, Semmle's analysis platform that combines deep semantic code search with data science insights to help developers ship secure code. - -QL queries are easy to write and share–visit the topics below and `our open source repository on GitHub `__ to learn more. -You can also try out QL in the `query console `__ on `LGTM.com `__. -Here, you can write QL code to query open source projects directly, without having to download snapshots and libraries. +CodeQL is based on a powerful query language called QL. The following topics help you understand QL in general, as well as how to use it when analyzing code with CodeQL. .. _getting-started: @@ -25,10 +24,10 @@ If you are new to QL, start by looking at the following topics: beginner/ql-tutorials ql-etudes/river-crossing -QL training and variant analysis examples -****************************************** +CodeQL training and variant analysis examples +********************************************* -To start learning how to use QL in variant analysis for a specific language, see: +To start learning how to use CodeQL for variant analysis for code written in a specific language, see: .. toctree:: :maxdepth: -1 @@ -37,8 +36,8 @@ To start learning how to use QL in variant analysis for a specific language, see .. _writing-ql-queries: -Writing QL queries -****************** +Writing CodeQL queries +********************** To learn more about writing your own queries, see: @@ -48,7 +47,7 @@ To learn more about writing your own queries, see: writing-queries/writing-queries -For more information on writing QL to query code written in a specific language see: +For more information on using CodeQL to query code written in a specific language, see: .. toctree:: :maxdepth: 2 @@ -77,10 +76,10 @@ For more technical information see: Reference topics **************** -For a more comprehensive guide to QL see the following reference topics: +For a more comprehensive guide to the query language itself, see the following reference topics: -- `QL language handbook `__—a description of important concepts in QL -- `QL language specification `__—a formal specification of the QL language. +- `QL language handbook `__—a description of important concepts in QL. +- `QL language specification `__—a formal specification of QL. Search ****** diff --git a/docs/language/learn-ql/intro-to-data-flow.rst b/docs/language/learn-ql/intro-to-data-flow.rst index 024df3525e8..4868c3e952d 100644 --- a/docs/language/learn-ql/intro-to-data-flow.rst +++ b/docs/language/learn-ql/intro-to-data-flow.rst @@ -1,15 +1,15 @@ -Introduction to data flow analysis in QL -######################################## +Introduction to data flow analysis with CodeQL +############################################## Overview ******** Data flow analysis computes the possible values that a variable can hold at various points in a program, determining how those values propagate through the program and where they are used. -Many of Semmle's built-in security queries implement data flow analysis, which can highlight the fate of potentially malicious or insecure data that can cause vulnerabilities in your code base. +Many CodeQL security queries implement data flow analysis, which can highlight the fate of potentially malicious or insecure data that can cause vulnerabilities in your code base. These queries help you understand if data is used in an insecure way, whether dangerous arguments are passed to functions, or whether sensitive data can leak. -As well as highlighting potential security issues, you can also use data flow analysis to understand other aspects of how a program behaves, by finding, for example, uses of unititialized variables and resource leaks. +As well as highlighting potential security issues, you can also use data flow analysis to understand other aspects of how a program behaves, by finding, for example, uses of uninitialized variables and resource leaks. -The following sections provide a brief introduction to data flow analysis in QL. +The following sections provide a brief introduction to data flow analysis with CodeQL. See the following tutorials for more information about analyzing data flow in specific languages: @@ -30,7 +30,7 @@ See the following tutorials for more information about analyzing data flow in sp Data flow graph *************** -The QL data flow libraries implement data flow analysis on a program or function by modeling its data flow graph. +The CodeQL data flow libraries implement data flow analysis on a program or function by modeling its data flow graph. Unlike the `abstract syntax tree `__, the data flow graph does not reflect the syntactic structure of the program, but models the way data flows through the program at runtime. Nodes in the abstract syntax tree represent syntactic elements such as statements or expressions. Nodes in the data flow graph, on the other hand, represent semantic elements that carry values at runtime. @@ -58,18 +58,18 @@ Computing an accurate and complete data flow graph presents several challenges: - Aliasing between variables can result in a single write changing the value that multiple pointers point to. - The data flow graph can be very large and slow to compute. -To overcome these potential problems, two kinds of data flow are modeled in the QL libraries: +To overcome these potential problems, two kinds of data flow are modeled in the CodeQL libraries: -- Local data flow, concerning the data flow within a single function. When reasoning about local, you only considers edges between data flow nodes belonging to the same function.It is generally sufficiently fast, efficient and precise for many queries, and it is usually possible to compute the local data flow for all functions in a snapshot. +- Local data flow, concerning the data flow within a single function. When reasoning about local, you only considers edges between data flow nodes belonging to the same function.It is generally sufficiently fast, efficient and precise for many queries, and it is usually possible to compute the local data flow for all functions in a CodeQL database. - Global data flow, effectively considers the data flow within an entire program, by calculating data flow between functions and through object properties. Computing global data flow is typically more time and energy intensive than local data flow, therefore queries should be refined to look for more specific sources and sinks. -Many of the built-in queries included in the latest Semmle release contain examples of both local and global data flow analysis. See `the built-in queries `__ for details. +Many CodeQL queries contain examples of both local and global data flow analysis. See `the built-in queries `__ for details. Normal data flow vs taint tracking ********************************** -In the QL standard libraries, we make a distinction between 'normal' data flow and taint tracking. +In the standard CodeQL libraries, we make a distinction between 'normal' data flow and taint tracking. The normal data flow libraries are used to analyze the information flow in which data values are preserved at each step. For example, if you are tracking an insecure object ``x`` (which might be some untrusted or potentially malicious data), a step in the program may 'change' its value. So, in a simple process such as ``y = x + 1``, a normal data flow analysis will highlight the use of ``x``, but not ``y``. @@ -81,5 +81,5 @@ These flow steps are modeled in the taint-tracking library using predicates that What next? ********** -- Search for ``DataFlow`` and ``TaintTracking`` in the `QL standard libraries `__ to learn more about the technical implementation of data flow analysis in QL for specific programming languages. -- Visit `Learning QL `__ to find language-specific QL tutorials on data flow and other topics. +- Search for ``DataFlow`` and ``TaintTracking`` in the `standard CodeQL libraries `__ to learn more about the technical implementation of data flow analysis for specific programming languages. +- Visit `Learning CodeQL `__ to find language-specific tutorials on data flow and other topics. diff --git a/docs/language/learn-ql/introduction-to-ql.rst b/docs/language/learn-ql/introduction-to-ql.rst index 737c593fb4a..a06b80839ac 100644 --- a/docs/language/learn-ql/introduction-to-ql.rst +++ b/docs/language/learn-ql/introduction-to-ql.rst @@ -1,18 +1,22 @@ -Introduction to the QL language -=============================== +Introduction to QL +================== -QL is a powerful query language that is used to analyze code. Queries written in QL can be used to find errors and uncover variants of important security vulnerabilities. Visit Semmle's `security research page `__ to read about examples of vulnerabilities that we have recently found in open source projects using QL queries. +QL is a powerful query language that underlies CodeQL, which is used to analyze code. +Queries written with CodeQL can find errors and uncover variants of important security vulnerabilities. +Visit Semmle's `security research page `__ to read about examples of vulnerabilities that we have recently found in open source projects. + +Before diving into code analysis with CodeQL, it can be helpful to learn about the underlying language more generally. QL is a logic programming language, so it is built up of logical formulas. QL uses common logical connectives (such as ``and``, ``or``, and ``not``), quantifiers (such as ``forall`` and ``exists``), and other important logical concepts such as predicates. -QL also supports recursion and aggregates. This allows you to write complex recursive queries using simple QL syntax and directly use aggregates such as ``count``, ``sum`` and ``average``. +QL also supports recursion and aggregates. This allows you to write complex recursive queries using simple QL syntax and directly use aggregates such as ``count``, ``sum``, and ``average``. Basic syntax ------------ -The basic syntax will look familiar to anyone who has used SQL, but it is used somewhat differently. +The basic syntax of QL will look familiar to anyone who has used SQL, but it is used somewhat differently. -A QL query is defined by a **select** clause, which specifies what the result of the query should be. You can try out the examples and exercises in this topic directly in LGTM. Open the `query console `__. Before you can run a query, you need to select a language and project to query (for these logic examples, any language and project will do). +A query is defined by a **select** clause, which specifies what the result of the query should be. You can try out the examples and exercises in this topic directly in LGTM. Open the `query console `__. Before you can run a query, you need to select a language and project to query (for these logic examples, any language and project will do). Once you have selected a language, the query console is populated with the query: @@ -110,9 +114,12 @@ To simplify the query, we can introduce a class ``SmallInt`` representing the in ➤ `See this in the query console `__ -Now that you've seen some general examples, let's use QL queries to analyze projects. In particular, LGTM generates a database representing the code and then QL is used to query this database. See `Database generation `__ for more details on how the database is built. +Now that you've seen some general examples, let's use the CodeQL libraries to analyze projects. +In particular, LGTM generates a database representing the code and then CodeQL is used to query this database. See `Database generation `__ for more details on how the database is built. -The previous exercises just used the primitive types built in to QL. Although we chose a project to query, they did not use the project-specific database. The following example queries *do* use these databases and give you an idea of what QL can be used for. There are more details about how to write QL `below <#learning-ql>`__, so don't worry if you don't fully understand these examples yet! +.. XX: Perhaps a link to the "CodeQL libraries for X"? + +The previous exercises just used the primitive types built in to QL. Although we chose a project to query, they did not use the project-specific database. The following example queries *do* use these databases and give you an idea of what CodeQL can be used for. There are more details about how to use CodeQL `below <#learning-ql>`__, so don't worry if you don't fully understand these examples yet! Python ~~~~~~ @@ -153,9 +160,9 @@ Java ➤ `See this in the query console `__. The ``from`` clause defines a variable ``p`` representing a parameter. The ``where`` clause finds unused parameters by limiting the parameters ``p`` to those which are not accessed. Finally, the ``select`` clause lists these parameters. -Learning QL ------------ +Learning CodeQL +--------------- -- To find out more about how to write your own QL queries, try working through the :doc:`QL detective tutorials `. -- For an overview of the other available resources, see :doc:`Learning QL <../index>`. -- For a more technical description of QL, see :doc:`About QL `. +- To find out more about how to write your own queries, try working through the :doc:`QL detective tutorials `. +- For an overview of the other available resources, see :doc:`Learning CodeQL <../index>`. +- For a more technical description of the underlying language, see :doc:`About QL `. diff --git a/docs/language/learn-ql/locations.rst b/docs/language/learn-ql/locations.rst index eebb6fec68e..df210819ef2 100644 --- a/docs/language/learn-ql/locations.rst +++ b/docs/language/learn-ql/locations.rst @@ -1,6 +1,8 @@ Locations and strings for QL entities ===================================== +.. Not sure how much of this topic needs to change, and what the title should be + Providing locations ------------------- @@ -55,18 +57,18 @@ By convention, the location of an entire file may also be denoted by a ``file:// Other types of URL ^^^^^^^^^^^^^^^^^^ -The following, less-common types of URL are valid QL but are not supported by LGTM and will be omitted from any results: +The following, less-common types of URL are valid but are not supported by LGTM and will be omitted from any results: - **HTTP URLs** are supported in some client applications. For an example, see the code snippet above. - **Folder URLs** can be useful, for example to provide folder-level metrics. They may use a file URL, for example ``file:///opt/src:0:0:0:0``, but they may also start with a scheme of ``folder://``, and no trailing numbers, for example ``folder:///opt/src``. -- **Relative file URLs** are like normal file URLs, but start with the scheme ``relative://``. They are typically only meaningful in the context of a particular snapshot, and are taken to be implicitly prefixed by the snapshot's source location. Note that, in particular, the relative URL of a file will stay constant regardless of where the snapshot is analyzed. It is often most convenient to produce these URLs as input when importing external information; selecting one from a QL class would be unusual, and client applications may not handle it appropriately. +- **Relative file URLs** are like normal file URLs, but start with the scheme ``relative://``. They are typically only meaningful in the context of a particular database, and are taken to be implicitly prefixed by the database's source location. Note that, in particular, the relative URL of a file will stay constant regardless of where the database is analyzed. It is often most convenient to produce these URLs as input when importing external information; selecting one from a QL class would be unusual, and client applications may not handle it appropriately. Providing location information ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If no ``getURL()`` member predicate is defined, a QL class is checked for the presence of a member predicate called ``hasLocationInfo(..)``. This can be understood as a convenient way of providing file URLs (see above) without constructing the long URL string in QL. ``hasLocationInfo(..)`` should be a predicate, its first column must be ``string``-typed (it corresponds to the "path" portion of a file URL), and it must have an additional 3 or 4 ``int``-typed columns, which are interpreted like a trailing group of three or four numbers on a file URL. -For example, let us imagine that the locations for methods provided by the extractor extend from the first character of the method name to the closing curly brace of the method body, and we want to "fix" them in QL to ensure that only the method name is selected. The following code shows two ways of achieving this: +For example, let us imagine that the locations for methods provided by the extractor extend from the first character of the method name to the closing curly brace of the method body, and we want to "fix" them to ensure that only the method name is selected. The following code shows two ways of achieving this: .. code-block:: ql @@ -104,7 +106,7 @@ Using extracted location information Finally, if the above two predicates fail, client applications will attempt to call a predicate called ``getLocation()`` with no parameters, and try to apply one of the above two predicates to the result. This allows certain locations to be put into the database, assigned identifiers, and picked up. -By convention, the return value of the ``getLocation()`` predicate should be a QL class called ``Location``, and it should define a version of ``hasLocationInfo(..)`` (or ``getURL()``, though the former is preferable). If the ``Location`` class does not provide either of these member predicates, then no location information will be available. +By convention, the return value of the ``getLocation()`` predicate should be a class called ``Location``, and it should define a version of ``hasLocationInfo(..)`` (or ``getURL()``, though the former is preferable). If the ``Location`` class does not provide either of these member predicates, then no location information will be available. The ``toString()`` predicate ---------------------------- diff --git a/docs/language/learn-ql/python/pointsto-type-infer.rst b/docs/language/learn-ql/python/pointsto-type-infer.rst index bcbab520477..a50e0776a83 100644 --- a/docs/language/learn-ql/python/pointsto-type-infer.rst +++ b/docs/language/learn-ql/python/pointsto-type-infer.rst @@ -231,5 +231,5 @@ For more information on writing QL, see: - `QL language handbook `__ - an introduction to the concepts of QL. - :doc:`Learning QL <../../index>` - an overview of the resources for learning how to write your own QL queries. -- `Database generation `__ - an overview of the process that creates a snapshot from source code. -- :doc:`What's in a snapshot? <../snapshot>` - a description of the snapshot database. +- `Database generation `__ - an overview of the process that creates a database from source code. +- :doc:`What's in a CodeQL database? <../database>` - a description of the CodeQL database. diff --git a/docs/language/learn-ql/ql-etudes/river-crossing.rst b/docs/language/learn-ql/ql-etudes/river-crossing.rst index e6fb0024612..fef3363a08b 100644 --- a/docs/language/learn-ql/ql-etudes/river-crossing.rst +++ b/docs/language/learn-ql/ql-etudes/river-crossing.rst @@ -1,7 +1,7 @@ River crossing puzzle ##################### -The aim of this tutorial is to write a QL query that finds a solution to the following classical logic puzzle: +The aim of this tutorial is to write a query that finds a solution to the following classical logic puzzle: .. pull-quote:: @@ -243,7 +243,7 @@ You could tweak the predicate and the select clause to make the solution clearer Alternative solutions --------------------- -Here are some more example QL queries that solve the river crossing puzzle: +Here are some more example queries that solve the river crossing puzzle: #. This query uses a modified ``path`` variable to describe the resulting path in more detail. diff --git a/docs/language/learn-ql/snapshot.rst b/docs/language/learn-ql/snapshot.rst deleted file mode 100644 index f89229b8b27..00000000000 --- a/docs/language/learn-ql/snapshot.rst +++ /dev/null @@ -1,33 +0,0 @@ -What's in a snapshot database? -=============================== - -A snapshot database contains a variety of data related to a particular code base at a particular point in time. For details of how the database is generated see `Database generation `__. - -The database contains a full, hierarchical representation of the program defined by the code base. The database schema varies according to the language analyzed. The schema provides an interface between the initial lexical analysis during the extraction process, and the actual complex analysis using QL. When the source code languages being analyzed change (such as Java 7 evolving into Java 8), this interface between the analysis phases can also change. - -For each language, a QL library defines classes to provide a layer of abstraction over the database tables. This provides an object-oriented view of the data which makes it easier to write queries. This is easiest to explain using an example. - -Example -------- - -For a Java program, two key tables are: - -- The ``expressions`` table containing a row for every single expression in the source code that was analyzed during the build process. -- The ``statements`` table containing a row for every single statement in the source code that was analyzed during the build process. - -The QL library defines classes to provide a layer of abstraction over each of these tables (and the related auxiliary tables): ``Expr`` and ``Stmt``. - -Most classes in the QL library are similar: they are abstractions over one or more database tables. Looking at one of the QL libraries illustrates this: - -.. code-block:: ql - - class Expr extends StmtParent, @expr { - ... - - /** the location of this expression */ - Location getLocation() { exprs(this,_,_,result) } - - ... - } - -The ``Expr`` class, shown here, extends from the database type ``@expr``. Member predicates of the ``Expr`` class are implemented in terms of the database-provided ``exprs`` table. diff --git a/docs/language/learn-ql/technical-info.rst b/docs/language/learn-ql/technical-info.rst index 777cc2f1041..7494011f7c6 100644 --- a/docs/language/learn-ql/technical-info.rst +++ b/docs/language/learn-ql/technical-info.rst @@ -4,6 +4,6 @@ Technical information .. toctree:: :hidden: - snapshot + database -- :doc:`What's in a snapshot database ` \ No newline at end of file +- :doc:`What's in a CodeQL database? ` \ No newline at end of file diff --git a/docs/language/learn-ql/writing-queries/introduction-to-queries.rst b/docs/language/learn-ql/writing-queries/introduction-to-queries.rst index c36c8c47de7..de75db30742 100644 --- a/docs/language/learn-ql/writing-queries/introduction-to-queries.rst +++ b/docs/language/learn-ql/writing-queries/introduction-to-queries.rst @@ -4,13 +4,15 @@ Introduction to query files Overview ******** -Queries are programs written in QL. The QL queries included in the Semmle tools are designed to highlight issues related to the security, correctness, maintainability, and readability of a code base. You can also write custom queries to find specific issues relevant to your own project. Three important types of query are: +Queries are programs written with CodeQL. They are designed to highlight issues related to the security, correctness, maintainability, and readability of a code base. You can also write custom queries to find specific issues relevant to your own project. Three important types of query are: - **Alert queries**: queries that highlight issues in specific locations in your code. - **Path queries**: queries that describe the flow of information between a source and a sink in your code. - **Metric queries**: queries that compute statistics for your code. -You can add custom queries to `custom query packs `__ to analyze your projects in `LGTM `__, use them to analyze a project using the `QL command-line tools `__, or you can contribute to Semmle's built-in queries in our `open source repository on GitHub `__. +You can add custom queries to `custom query packs `__ to analyze your projects in `LGTM `__, use them to analyze a project using the `command-line tools `__, or you can contribute to the standard CodeQL queries in our `open source repository on GitHub `__. + +.. TODO: Change "command-line tools" to a link to the CodeQL CLI? Similarly, change "QL for Eclipse". .. pull-quote:: @@ -22,13 +24,13 @@ You can add custom queries to `custom query packs `__, and detailed technical information about QL in the `QL language handbook `__ and the `QL language specification `__. -For information on how to format QL code when contributing queries to the GitHub repository, see the `QL style guide `__. +For information on how to format your code when contributing queries to the GitHub repository, see the `QL style guide `__. Basic query structure ********************* -`Queries `__ written in QL have the file extension ``.ql``, and contain a ``select`` clause. Many of the built-in Semmle queries include additional optional information, and have the following structure:: +`Queries `__ written with CodeQL have the file extension ``.ql``, and contain a ``select`` clause. Many of the existing CodeQL queries include additional optional information, and have the following structure:: /** * @@ -36,9 +38,9 @@ Basic query structure * */ - import /* ... QL libraries or modules ... */ + import /* ... CodeQL libraries or modules ... */ - /* ... Optional, define QL classes and predicates ... */ + /* ... Optional, define CodeQL classes and predicates ... */ from /* ... variable declarations ... */ where /* ... logical formula ... */ @@ -49,9 +51,9 @@ The following sections describe the information that is typically included in a Query metadata ============== -Query metadata is used to identify your custom queries when they are added to the Semmle repository or used in your analysis. Metadata provides information about the query's purpose, and also specifies how to interpret and display the query results. For a full list of metadata properties, see the :doc:`query metadata reference `. The exact metadata requirement depends on how you are going to run your query: +Query metadata is used to identify your custom queries when they are added to the GitHub repository or used in your analysis. Metadata provides information about the query's purpose, and also specifies how to interpret and display the query results. For a full list of metadata properties, see the :doc:`query metadata reference `. The exact metadata requirement depends on how you are going to run your query: -- If you are contributing a query to the Semmle open source repository for inclusion in the standard queries, please read the `query metadata style guide `__. +- If you are contributing a query to the GitHub repository, please read the `query metadata style guide `__. - If you are adding a custom query to a query pack for analysis using LGTM , see `Writing custom queries to include in LGTM analysis `__. - If you are analyzing a project using the `QL command-line tools `__, see `Preparing custom queries `__. - If you are running a query in the query console on LGTM or in the Quick query window in QL for Eclipse, metadata is not mandatory. However, if you want your results to be displayed as either an 'alert' or a 'path', you must specify the correct `@kind` property, as explained below. See `Using the query console `__ and `Running a quick query `__ for further information. @@ -60,7 +62,7 @@ Query metadata is used to identify your custom queries when they are added to th Note - Queries that are contributed to the Semmle open source repository, added to a query pack in LGTM, or used to analyze a project with the QL command-line tools must have a query type (``@kind``) specified. The ``@kind`` property indicates how to interpret and display the results of the query analysis: + Queries that are contributed to the open source repository, added to a query pack in LGTM, or used to analyze a project with the QL command-line tools must have a query type (``@kind``) specified. The ``@kind`` property indicates how to interpret and display the results of the query analysis: - Alert query metadata must contain ``@kind problem``. - Path query metadata must contain ``@kind path-problem``. @@ -71,8 +73,8 @@ Query metadata is used to identify your custom queries when they are added to th Import statements ================= -Each query generally contains one or more ``import`` statements, which define the `QL libraries `__ or `modules `__ to import into the query. QL libraries and modules provide a way of grouping together related `types `__, `predicates `__, and other modules. The contents of each library or module that you import can then be accessed by the query. -`Semmle's open source repository on GitHub `__ contains all of the standard QL libraries for each supported language. +Each query generally contains one or more ``import`` statements, which define the `libraries `__ or `modules `__ to import into the query. Libraries and modules provide a way of grouping together related `types `__, `predicates `__, and other modules. The contents of each library or module that you import can then be accessed by the query. +Our `open source repository on GitHub `__ contains the standard CodeQL libraries for each supported language. When writing your own alert queries, you would typically import the standard library for the language of the project that you are querying, using ``import`` followed by a language: @@ -83,13 +85,13 @@ When writing your own alert queries, you would typically import the standard lib - JavaScript/TypeScript: ``javascript`` - Python: ``python`` -There are also QL libraries containing commonly used predicates, types, and other modules associated with different analyses, including data flow, control flow, and taint-tracking. In order to calculate path graphs, path queries require you to import a data flow library into the query file. See :doc:`Constructing path queries ` for further information. +There are also CodeQL libraries containing commonly used predicates, types, and other modules associated with different analyses, including data flow, control flow, and taint-tracking. In order to calculate path graphs, path queries require you to import a data flow library into the query file. See :doc:`Constructing path queries ` for further information. -You can explore the contents of all the standard QL libraries in the `QL library reference documentation `__, using `QL for Eclipse `__, or in the `GitHub repository `__. +You can explore the contents of all the standard CodeQL libraries in the `CodeQL library reference documentation `__, using `QL for Eclipse `__, or in the `GitHub repository `__. -Optional QL classes and predicates ----------------------------------- +Optional CodeQL classes and predicates +-------------------------------------- You can customize your analysis by defining your own predicates and classes in the query. See `Defining a predicate `__ and `Defining a class `__ for further details. @@ -97,13 +99,13 @@ From clause =========== The ``from`` clause declares the variables that are used in the query. Each declaration must be of the form `` ``. -For more information on the `types `__ available in QL, and to learn how to define your own types using `classes `__, see the `QL language handbook `__. +For more information on the available `types `__, and to learn how to define your own types using `classes `__, see the `QL language handbook `__. Where clause ============ The ``where`` clause defines the logical conditions to apply to the variables declared in the ``from`` clause to generate your results. This clause uses `aggregations `__, `predicates `__, and logical `formulas `_ to limit the variables of interest to a smaller set, which meet the defined conditions. -The QL libraries group commonly used predicates for specific languages and frameworks. You can also define your own predicates in the body of the query file or in your own custom modules, as described above. +The CodeQL libraries group commonly used predicates for specific languages and frameworks. You can also define your own predicates in the body of the query file or in your own custom modules, as described above. Select clause ============= @@ -138,6 +140,6 @@ What next? - See the queries used in real-life variant analysis on the `Semmle blog `__. - To learn more about writing path queries, see :doc:`Constructing path queries `. -- Take a look at the `built-in queries `__ to see examples of the queries included in the Semmle tools. -- Explore the `query cookbooks `__ to see how to access the basic language elements contained in the QL libraries. -- For a full list of resources to help you learn QL, including beginner tutorials and language-specific examples, visit `Learning QL `__. \ No newline at end of file +- Take a look at the `built-in queries `__ to see examples of the queries included in CodeQL. +- Explore the `query cookbooks `__ to see how to access the basic language elements contained in the CodeQL libraries. +- For a full list of resources to help you learn CodeQL, including beginner tutorials and language-specific examples, visit `Learning CodeQL `__. \ No newline at end of file diff --git a/docs/language/learn-ql/writing-queries/path-queries.rst b/docs/language/learn-ql/writing-queries/path-queries.rst index 50de68dbe55..8b793a63181 100644 --- a/docs/language/learn-ql/writing-queries/path-queries.rst +++ b/docs/language/learn-ql/writing-queries/path-queries.rst @@ -5,8 +5,8 @@ Overview ======== Security researchers are particularly interested in the way that information flows in a program. Many vulnerabilities are caused by seemingly benign data flowing to unexpected locations, and being used in a malicious way. -Path queries written in QL are particularly useful for analyzing data flow as they can be used to track the path taken by a variable from its possible starting points (``source``) to its possible end points (``sink``). -To model paths in QL, your query must provide information about the ``source`` and the ``sink``, as well as the data flow steps that link them. +Path queries written with CodeQL are particularly useful for analyzing data flow as they can be used to track the path taken by a variable from its possible starting points (``source``) to its possible end points (``sink``). +To model paths with CodeQL, your query must provide information about the ``source`` and the ``sink``, as well as the data flow steps that link them. This topic provides information on how to structure a path query file so you can explore the paths associated with the results of data flow analysis. @@ -17,8 +17,8 @@ This topic provides information on how to structure a path query file so you can The alerts generated by path queries are displayed by default in `LGTM `__ and included in the results generated using the `QL command-line tools `__. You can also view the paths explanations generated by your path query `directly in LGTM `__, or using the `Path explorer view `__ in `QL for Eclipse `__. -To learn more about modeling data flow in QL, see :doc:`Introduction to data flow <../intro-to-data-flow>`. -For more language-specific information on analyzing data flow with QL see: +To learn more about modeling data flow with CodeQL, see :doc:`Introduction to data flow <../intro-to-data-flow>`. +For more language-specific information on analyzing data flow see: - :doc:`Analyzing data flow in C/C++ <../cpp/dataflow>` - :doc:`Analyzing data flow in C# <../csharp/dataflow>` @@ -29,7 +29,7 @@ For more language-specific information on analyzing data flow with QL see: Path query examples ******************* -The easiest way to get started writing your own path query is to modify one of the existing queries. Visit the links below to see all of the built-in path queries: +The easiest way to get started writing your own path query is to modify one of the existing queries. Visit the links below to see all the built-in path queries: - `C/C++ path queries `__ - `C# path queries `__ @@ -43,7 +43,7 @@ Constructing a path query ========================= Path queries require certain metadata, query predicates, and ``select`` statement structures. -Many of the built-in path queries included in the Semmle tools follow a simple structure, which depends on how the language you are analyzing is modeled in QL. +Many of the built-in path queries included in CodeQL follow a simple structure, which depends on how the language you are analyzing is modeled with CodeQL. For C/C++, C#, Java, and JavaScript you should use the following template:: @@ -63,7 +63,7 @@ For C/C++, C#, Java, and JavaScript you should use the following template:: Where: -- ``DataFlow::Pathgraph`` is the path graph module you need to import from the standard QL libraries. +- ``DataFlow::Pathgraph`` is the path graph module you need to import from the standard CodeQL libraries. - ``source`` and ``sink`` are nodes on the `path graph `__, and ``DataFlow::PathNode`` is their type. - ``Configuration`` is a class containing the predicates which define how data may flow between the ``source`` and the ``sink``. @@ -85,7 +85,7 @@ For Python you should use a slightly different template:: Where: -- ``semmle.python.security.Paths`` is the path graph module imported from the standard QL libraries. +- ``semmle.python.security.Paths`` is the path graph module imported from the standard CodeQL libraries. - ``source`` and ``sink`` are nodes on the path graph, ``TaintedPathSource source`` and ``TaintedPathSink`` are their respective types. Note, you do not need to declare a configuration class to define the data flow from the ``source`` to the ``sink`` in a Python path query. @@ -103,7 +103,7 @@ Generating path explanations In order to generate path explanations, your query needs to compute a `path graph `__. To do this you need to define a `query predicate `__ called ``edges`` in your query. This predicate defines the edge relations of the graph you are computing, and it is used to compute the paths related to each result that your query generates. -You can import a predefined ``edges`` predicate from a path graph module in one of the standard QL data flow libraries. In addition to the path graph module, the data flow libraries contain the other ``classes``, ``predicates``, and ``modules`` that are commonly used in data flow analysis. The import statement to use depends on the language that you are analyzing. +You can import a predefined ``edges`` predicate from a path graph module in one of the standard CodeQL data flow libraries. In addition to the path graph module, the data flow libraries contain the other ``classes``, ``predicates``, and ``modules`` that are commonly used in data flow analysis. The import statement to use depends on the language that you are analyzing. For C/C++, C#. Java, and JavaScript you would use:: @@ -115,7 +115,7 @@ For Python, the ``Paths`` module contains the ``edges`` predicate:: import semmle.python.security.Paths -You can also import libraries specifically designed to implement data flow analysis in various common frameworks and environments, and many additional libraries are included with the Semmle tools. To see examples of the different libraries used in data flow analysis, see the links to the built-in queries above or browse the `standard QL libraries `__. +You can also import libraries specifically designed to implement data flow analysis in various common frameworks and environments, and many additional libraries are included with CodeQL. To see examples of the different libraries used in data flow analysis, see the links to the built-in queries above or browse the `standard CodeQL libraries `__. For all languages, you can also optionally define a ``nodes`` query predicate, which specifies the nodes of the path graph that you are interested in. If ``nodes`` is defined, only edges with endpoints defined by these nodes are selected. If ``nodes`` is not defined, you select all possible endpoints of ``edges``. @@ -128,7 +128,7 @@ You can also define your own ``edges`` predicate in the body of your query. It s /** Logical conditions which hold if `(a,b)` is an edge in the data flow graph */ } -For more examples of how to define an ``edges`` predicate, visit the `standard QL libraries `__ and search for ``edges``. +For more examples of how to define an ``edges`` predicate, visit the `standard CodeQL libraries `__ and search for ``edges``. Declaring sources and sinks *************************** @@ -136,7 +136,7 @@ Declaring sources and sinks You must provide information about the ``source`` and ``sink`` in your path query. These are objects that correspond to the nodes of the paths that you are exploring. The name and the type of the ``source`` and the ``sink`` must be declared in the ``from`` statement of the query, and the types must be compatible with the nodes of the graph computed by the ``edges`` predicate. -If you are querying C/C++, C#, Java or JavaScript code (and you have used ``import DataFlow::PathGraph`` in your query), the definitions of the ``source`` and ``sink`` are accessed via the ``Configuration`` class in the data flow library. You should declare all three of these objects in the ``from`` statement. +If you are querying C/C++, C#, Java, or JavaScript code (and you have used ``import DataFlow::PathGraph`` in your query), the definitions of the ``source`` and ``sink`` are accessed via the ``Configuration`` class in the data flow library. You should declare all three of these objects in the ``from`` statement. For example:: from Configuration config, DataFlow::PathNode source, DataFlow::PathNode sink @@ -191,5 +191,5 @@ What next? ********** - Take a look at the path queries for `C/C++ `__, `C# `__, `Java `__, `JavaScript `__, and `Python `__ to see examples of the queries included in the Semmle tools. -- Explore the `query cookbooks `__ to see how to access the basic language elements contained in the QL libraries. -- For a full list of resources to help you learn QL, including beginner tutorials and language-specific examples, visit `Learning QL `__. +- Explore the `query cookbooks `__ to see how to access the basic language elements contained in the CodeQL libraries. +- For a full list of resources to help you learn CodeQL, including beginner tutorials and language-specific examples, visit `Learning CodeQL `__. diff --git a/docs/language/learn-ql/writing-queries/query-help.rst b/docs/language/learn-ql/writing-queries/query-help.rst index 9e295df7620..c3692b7fac0 100644 --- a/docs/language/learn-ql/writing-queries/query-help.rst +++ b/docs/language/learn-ql/writing-queries/query-help.rst @@ -2,16 +2,16 @@ Query help reference ******************** This topic provides detailed information on the structure of query help files. -For more information about how to write useful query help in a style that is consistent with Semmle's built-in queries, see the `Query help style guide `__ on GitHub. +For more information about how to write useful query help in a style that is consistent with the standard CodeQL queries, see the `Query help style guide `__ on GitHub. .. pull-quote:: Note - You can access the query help for Semmle's built-in queries by visiting the `Built-in query pages `__. - You can also access the raw query help files in the `Semmle/ql GitHub repository `__. - For example, the `JavaScript security queries `__ and `C/C++ critical queries `__. + You can access the query help for CodeQL queries by visiting the `Built-in query pages `__. + You can also access the raw query help files in the `GitHub repository `__. + For example, see the `JavaScript security queries `__ and `C/C++ critical queries `__. For queries run by default on LGTM, there are several different ways to access the query help. For further information, see `Where do I see the query help for a query on LGTM? `__ in the LGTM user help. @@ -207,5 +207,5 @@ The included file, `ThreadUnsafeICryptoTransformOverview.qhelp `__ on GitHub. -- To learn more about writing custom queries, and how to format your QL for clarity and consistency, see `Writing QL queries `__. +- To learn more about contributing to the standard CodeQL queries and libraries, see our `Contributing guidelines `__ on GitHub. +- To learn more about writing custom queries, and how to format your code for clarity and consistency, see `Writing CodeQL queries `__. diff --git a/docs/language/learn-ql/writing-queries/select-statement.rst b/docs/language/learn-ql/writing-queries/select-statement.rst index ea2b256e421..86e0770d426 100644 --- a/docs/language/learn-ql/writing-queries/select-statement.rst +++ b/docs/language/learn-ql/writing-queries/select-statement.rst @@ -27,7 +27,7 @@ If you look at some of the LGTM queries, you'll see that they can select extra e Developing a select statement ----------------------------- -Here's a simple query that uses the standard QL ``CodeDuplication.qll`` library to identify similar files. +Here's a simple query that uses the standard CodeQL ``CodeDuplication.qll`` library to identify similar files. Basic select statement ~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/language/learn-ql/writing-queries/writing-queries.rst b/docs/language/learn-ql/writing-queries/writing-queries.rst index 1c3877e1a77..caedde2b0a0 100644 --- a/docs/language/learn-ql/writing-queries/writing-queries.rst +++ b/docs/language/learn-ql/writing-queries/writing-queries.rst @@ -1,8 +1,7 @@ -Writing QL queries -################## +Writing CodeQL queries +###################### - -If you are familiar with QL, you can modify the existing Semmle queries or write custom queries to analyze, improve, and secure your own projects. Get started by reading the information for query writers and viewing the examples provided below. +If you are familiar with CodeQL, you can modify the existing queries or write custom queries to analyze, improve, and secure your own projects. Get started by reading the information for query writers and viewing the examples provided below. Information for query writers ***************************** @@ -18,19 +17,21 @@ Information for query writers ../locations -Visit `Learning QL `__ to find basic information about QL, as well as help and advice on writing QL for specific programming languages. To learn more about the structure of query files, the key information to include when writing your own QL queries, and how to format your QL for clarity and consistency, see the following topics: +Visit `Learning CodeQL `__ to find basic information about CodeQL. This includes information about the underlying query language QL, as well as help and advice on writing queries for specific programming languages. +To learn more about the structure of query files, the key information to include when writing your own queries, and how to format them for clarity and consistency, see the following topics: - :doc:`Introduction to query files `–an introduction to the information contained in a basic query file. - :doc:`Constructing path queries `–a quick guide to structuring path queries to use in security research. -- :doc:`Introduction to data flow analysis in QL <../intro-to-data-flow>`–a brief introduction to modeling data flow using QL. +- :doc:`Introduction to data flow analysis with CodeQL <../intro-to-data-flow>`–a brief introduction to modeling data flow using CodeQL. - :doc:`Defining 'select' statements `–further detail on developing query alert messages to provide extra information in your query results. -- :doc:`Locations and strings for QL entities <../locations>`–further detail on providing location information in query results. +- :doc:`Locations and strings for CodeQL entities <../locations>`–further detail on providing location information in query results. - `QL style guide on GitHub `__–a guide to formatting QL for consistency and clarity. -Viewing the built-in QL queries +Viewing existing CodeQL queries ******************************* -The easiest way to get started writing your own queries is to modify an existing query. To view examples of the queries included in the latest release of the Semmle tools, or to try out the QL query cookbooks, visit `Exploring QL queries `__. You can also find all of the Semmle queries in our `open source repository on GitHub `__. +The easiest way to get started writing your own queries is to modify an existing query. To see these queries, or to try out the CodeQL query cookbooks, visit `Exploring CodeQL queries `__. +SYou can also find all the CodeQL queries in our `open source repository on GitHub `__. You can also find examples of queries developed to find security vulnerabilities and bugs in open-source software projects in the `Semmle demos GitHub repository `__ and the `Semmle blog `__. @@ -45,7 +46,9 @@ Contributing queries query-help Contributions to the standard queries and libraries are very welcome–see our `contributing guidelines `__ for further information. -If you are contributing a query to the open source GitHub repository, writing a custom query for LGTM, or using a custom query in an analysis with the QL command-line tools, then you need to include extra metadata in your query to ensure that the query results are interpreted and displayed correctly. See the following topics for more information on query metadata: +If you are contributing a query to the open source GitHub repository, writing a custom query for LGTM, or using a custom query in an analysis with our command-line tools, then you need to include extra metadata in your query to ensure that the query results are interpreted and displayed correctly. See the following topics for more information on query metadata: + +.. TODO: Change "command-line tools" to a link to the CodeQL CLI? - :doc:`Query metadata reference ` - `Query metadata style guide on GitHub `__ diff --git a/docs/language/ql-handbook/annotations.rst b/docs/language/ql-handbook/annotations.rst index e4f4b0c40cd..8673b8194f6 100644 --- a/docs/language/ql-handbook/annotations.rst +++ b/docs/language/ql-handbook/annotations.rst @@ -245,7 +245,7 @@ Compiler pragmas **Available for**: |characteristic predicates|, |member predicates|, |non-member predicates| -The following compiler pragmas affect the compilation and optimization of QL queries. You +The following compiler pragmas affect the compilation and optimization of queries. You should avoid using these annotations unless you experience significant performance issues. Before adding pragmas to your code, contact Semmle to describe the performance problems. diff --git a/docs/language/ql-handbook/index.rst b/docs/language/ql-handbook/index.rst index 2bd22a3a628..701d7d02237 100644 --- a/docs/language/ql-handbook/index.rst +++ b/docs/language/ql-handbook/index.rst @@ -9,7 +9,7 @@ help you understand how the language works, and how to write more advanced QL co Each section describes an important concept or syntactic construct of QL. For an overview, see the table of contents below. -If you are just getting started with QL, see `Learning QL `_ +If you are just getting started with QL, see `Learning CodeQL `_ for a list of the available resources. .. index:: specification diff --git a/docs/language/ql-handbook/types.rst b/docs/language/ql-handbook/types.rst index 999ec5ca51f..6aa9043bd62 100644 --- a/docs/language/ql-handbook/types.rst +++ b/docs/language/ql-handbook/types.rst @@ -230,7 +230,7 @@ abstract class, you can add more subclasses to it. **Example** -If you are writing a security query in QL, you may be interested in identifying +If you are writing a security query, you may be interested in identifying all expressions that can be interpreted as SQL queries. You can use the following abstract class to describe these expressions:: @@ -438,7 +438,7 @@ In the standard QL language libraries, this is usually done as follows: and provide a definition for any abstract predicates from ``A``. - Annotate the algebraic datatype with :ref:`private`, and leave the classes public. -For example, the following code snippet from the QL for C# data flow library defines classes +For example, the following code snippet from the CodeQL data-flow library for C# defines classes for dealing with tainted or untainted values. In this case, it doesn't make sense for ``TaintType`` to extend a database type. It is part of the taint analysis, not the underlying program, so it's helpful to extend a new type (namely ``TTaintType``):: @@ -473,7 +473,7 @@ Database types Database types are defined in the database schema. This means that they depend on the database that you are querying, and vary according to the data you are analyzing. -For example, if you are querying a QL database for a Java project, the database types may +For example, if you are querying a CodeQL database for a Java project, the database types may include ``@ifstmt``, representing an if statement in the Java code, and ``@variable``, representing a variable. diff --git a/docs/language/ql-training/slide-snippets/info.rst b/docs/language/ql-training/slide-snippets/info.rst index d093559eaba..583a456207b 100644 --- a/docs/language/ql-training/slide-snippets/info.rst +++ b/docs/language/ql-training/slide-snippets/info.rst @@ -5,7 +5,7 @@ To try the examples in this presentation we recommend you download `QL for Eclip **QL language resources** -- If you are new to QL, try the QL language tutorials at `Learning QL `__. +- If you are new to QL, try the QL language tutorials at `Learning CodeQL `__. - To learn more about the main features of QL, try looking at the `QL language handbook `__. - For further information about writing queries in QL, see `Writing QL queries `__. From 48c0d9ecca4e3ec3592611a7284adb3df1de4c09 Mon Sep 17 00:00:00 2001 From: Calum Grant Date: Wed, 23 Oct 2019 15:17:26 +0100 Subject: [PATCH 025/148] C#: Add qltests for ?? dataflow. --- .../dataflow/local/DataFlow.expected | 2 +- .../dataflow/local/DataFlowStep.expected | 20 +++++++++---- .../dataflow/local/LocalDataFlow.cs | 6 ++++ .../dataflow/local/TaintTracking.expected | 4 ++- .../dataflow/local/TaintTrackingStep.expected | 30 ++++++++++++++----- 5 files changed, 47 insertions(+), 15 deletions(-) diff --git a/csharp/ql/test/library-tests/dataflow/local/DataFlow.expected b/csharp/ql/test/library-tests/dataflow/local/DataFlow.expected index 63462bbad86..d1801f1833a 100644 --- a/csharp/ql/test/library-tests/dataflow/local/DataFlow.expected +++ b/csharp/ql/test/library-tests/dataflow/local/DataFlow.expected @@ -8,7 +8,7 @@ | LocalDataFlow.cs:412:15:412:20 | access to local variable sink70 | | LocalDataFlow.cs:420:19:420:24 | access to local variable sink71 | | LocalDataFlow.cs:430:23:430:28 | access to local variable sink72 | -| LocalDataFlow.cs:466:15:466:21 | access to parameter tainted | +| LocalDataFlow.cs:472:15:472:21 | access to parameter tainted | | SSA.cs:9:15:9:22 | access to local variable ssaSink0 | | SSA.cs:25:15:25:22 | access to local variable ssaSink1 | | SSA.cs:43:15:43:22 | access to local variable ssaSink2 | diff --git a/csharp/ql/test/library-tests/dataflow/local/DataFlowStep.expected b/csharp/ql/test/library-tests/dataflow/local/DataFlowStep.expected index d38e1f02215..ba8398610a1 100644 --- a/csharp/ql/test/library-tests/dataflow/local/DataFlowStep.expected +++ b/csharp/ql/test/library-tests/dataflow/local/DataFlowStep.expected @@ -545,7 +545,10 @@ | LocalDataFlow.cs:408:15:408:22 | access to local variable nonSink0 | LocalDataFlow.cs:415:31:415:38 | access to local variable nonSink0 | | LocalDataFlow.cs:411:13:411:34 | SSA def(sink70) | LocalDataFlow.cs:412:15:412:20 | access to local variable sink70 | | LocalDataFlow.cs:411:22:411:34 | ... = ... | LocalDataFlow.cs:411:13:411:34 | SSA def(sink70) | +| LocalDataFlow.cs:411:22:411:34 | SSA def(sink0) | LocalDataFlow.cs:443:34:443:38 | access to local variable sink0 | +| LocalDataFlow.cs:411:22:411:34 | SSA def(sink0) | LocalDataFlow.cs:444:22:444:26 | access to local variable sink0 | | LocalDataFlow.cs:411:30:411:34 | access to local variable sink0 | LocalDataFlow.cs:411:22:411:34 | ... = ... | +| LocalDataFlow.cs:411:30:411:34 | access to local variable sink0 | LocalDataFlow.cs:411:22:411:34 | SSA def(sink0) | | LocalDataFlow.cs:412:15:412:20 | [post] access to local variable sink70 | LocalDataFlow.cs:419:13:419:18 | access to local variable sink70 | | LocalDataFlow.cs:412:15:412:20 | access to local variable sink70 | LocalDataFlow.cs:419:13:419:18 | access to local variable sink70 | | LocalDataFlow.cs:415:9:415:38 | SSA def(nonSink0) | LocalDataFlow.cs:416:15:416:22 | access to local variable nonSink0 | @@ -562,12 +565,19 @@ | LocalDataFlow.cs:427:17:427:22 | access to local variable sink70 | LocalDataFlow.cs:429:18:429:30 | SSA def(sink72) | | LocalDataFlow.cs:429:18:429:30 | SSA def(sink72) | LocalDataFlow.cs:430:23:430:28 | access to local variable sink72 | | LocalDataFlow.cs:435:17:435:24 | access to local variable nonSink0 | LocalDataFlow.cs:437:18:437:33 | SSA def(nonSink17) | +| LocalDataFlow.cs:435:17:435:24 | access to local variable nonSink0 | LocalDataFlow.cs:443:22:443:29 | access to local variable nonSink0 | | LocalDataFlow.cs:437:18:437:33 | SSA def(nonSink17) | LocalDataFlow.cs:438:23:438:31 | access to local variable nonSink17 | -| LocalDataFlow.cs:458:28:458:30 | this | LocalDataFlow.cs:458:41:458:45 | this access | -| LocalDataFlow.cs:458:50:458:52 | this | LocalDataFlow.cs:458:56:458:60 | this access | -| LocalDataFlow.cs:458:50:458:52 | value | LocalDataFlow.cs:458:64:458:68 | access to parameter value | -| LocalDataFlow.cs:464:41:464:47 | tainted | LocalDataFlow.cs:466:15:466:21 | access to parameter tainted | -| LocalDataFlow.cs:469:44:469:53 | nonTainted | LocalDataFlow.cs:471:15:471:24 | access to parameter nonTainted | +| LocalDataFlow.cs:443:13:443:38 | SSA def(sink73) | LocalDataFlow.cs:445:15:445:20 | access to local variable sink73 | +| LocalDataFlow.cs:443:22:443:29 | access to local variable nonSink0 | LocalDataFlow.cs:444:31:444:38 | access to local variable nonSink0 | +| LocalDataFlow.cs:443:22:443:38 | ... ?? ... | LocalDataFlow.cs:443:13:443:38 | SSA def(sink73) | +| LocalDataFlow.cs:443:34:443:38 | access to local variable sink0 | LocalDataFlow.cs:444:22:444:26 | access to local variable sink0 | +| LocalDataFlow.cs:444:13:444:38 | SSA def(sink74) | LocalDataFlow.cs:446:15:446:20 | access to local variable sink74 | +| LocalDataFlow.cs:444:22:444:38 | ... ?? ... | LocalDataFlow.cs:444:13:444:38 | SSA def(sink74) | +| LocalDataFlow.cs:464:28:464:30 | this | LocalDataFlow.cs:464:41:464:45 | this access | +| LocalDataFlow.cs:464:50:464:52 | this | LocalDataFlow.cs:464:56:464:60 | this access | +| LocalDataFlow.cs:464:50:464:52 | value | LocalDataFlow.cs:464:64:464:68 | access to parameter value | +| LocalDataFlow.cs:470:41:470:47 | tainted | LocalDataFlow.cs:472:15:472:21 | access to parameter tainted | +| LocalDataFlow.cs:475:44:475:53 | nonTainted | LocalDataFlow.cs:477:15:477:24 | access to parameter nonTainted | | SSA.cs:5:17:5:17 | SSA entry def(this.S) | SSA.cs:67:9:67:14 | access to field S | | SSA.cs:5:17:5:17 | this | SSA.cs:67:9:67:12 | this access | | SSA.cs:5:26:5:32 | tainted | SSA.cs:8:24:8:30 | access to parameter tainted | diff --git a/csharp/ql/test/library-tests/dataflow/local/LocalDataFlow.cs b/csharp/ql/test/library-tests/dataflow/local/LocalDataFlow.cs index 154b4d5fcb9..65de9fc65fe 100644 --- a/csharp/ql/test/library-tests/dataflow/local/LocalDataFlow.cs +++ b/csharp/ql/test/library-tests/dataflow/local/LocalDataFlow.cs @@ -438,6 +438,12 @@ public class LocalDataFlow Check(nonSink17); break; } + + // Null-coalescing expressions + var sink73 = nonSink0 ?? sink0; + var sink74 = sink0 ?? nonSink0; + Check(sink73); + Check(sink74); } static void Check(T x) { } diff --git a/csharp/ql/test/library-tests/dataflow/local/TaintTracking.expected b/csharp/ql/test/library-tests/dataflow/local/TaintTracking.expected index ba3d91130f2..756d79333c1 100644 --- a/csharp/ql/test/library-tests/dataflow/local/TaintTracking.expected +++ b/csharp/ql/test/library-tests/dataflow/local/TaintTracking.expected @@ -62,7 +62,9 @@ | LocalDataFlow.cs:412:15:412:20 | access to local variable sink70 | | LocalDataFlow.cs:420:19:420:24 | access to local variable sink71 | | LocalDataFlow.cs:430:23:430:28 | access to local variable sink72 | -| LocalDataFlow.cs:466:15:466:21 | access to parameter tainted | +| LocalDataFlow.cs:445:15:445:20 | access to local variable sink73 | +| LocalDataFlow.cs:446:15:446:20 | access to local variable sink74 | +| LocalDataFlow.cs:472:15:472:21 | access to parameter tainted | | SSA.cs:9:15:9:22 | access to local variable ssaSink0 | | SSA.cs:25:15:25:22 | access to local variable ssaSink1 | | SSA.cs:43:15:43:22 | access to local variable ssaSink2 | diff --git a/csharp/ql/test/library-tests/dataflow/local/TaintTrackingStep.expected b/csharp/ql/test/library-tests/dataflow/local/TaintTrackingStep.expected index f9511969108..f320c87cb8b 100644 --- a/csharp/ql/test/library-tests/dataflow/local/TaintTrackingStep.expected +++ b/csharp/ql/test/library-tests/dataflow/local/TaintTrackingStep.expected @@ -690,7 +690,10 @@ | LocalDataFlow.cs:408:15:408:22 | access to local variable nonSink0 | LocalDataFlow.cs:415:31:415:38 | access to local variable nonSink0 | | LocalDataFlow.cs:411:13:411:34 | SSA def(sink70) | LocalDataFlow.cs:412:15:412:20 | access to local variable sink70 | | LocalDataFlow.cs:411:22:411:34 | ... = ... | LocalDataFlow.cs:411:13:411:34 | SSA def(sink70) | +| LocalDataFlow.cs:411:22:411:34 | SSA def(sink0) | LocalDataFlow.cs:443:34:443:38 | access to local variable sink0 | +| LocalDataFlow.cs:411:22:411:34 | SSA def(sink0) | LocalDataFlow.cs:444:22:444:26 | access to local variable sink0 | | LocalDataFlow.cs:411:30:411:34 | access to local variable sink0 | LocalDataFlow.cs:411:22:411:34 | ... = ... | +| LocalDataFlow.cs:411:30:411:34 | access to local variable sink0 | LocalDataFlow.cs:411:22:411:34 | SSA def(sink0) | | LocalDataFlow.cs:412:15:412:20 | [post] access to local variable sink70 | LocalDataFlow.cs:419:13:419:18 | access to local variable sink70 | | LocalDataFlow.cs:412:15:412:20 | access to local variable sink70 | LocalDataFlow.cs:419:13:419:18 | access to local variable sink70 | | LocalDataFlow.cs:415:9:415:38 | SSA def(nonSink0) | LocalDataFlow.cs:416:15:416:22 | access to local variable nonSink0 | @@ -707,15 +710,26 @@ | LocalDataFlow.cs:427:17:427:22 | access to local variable sink70 | LocalDataFlow.cs:429:18:429:30 | SSA def(sink72) | | LocalDataFlow.cs:429:18:429:30 | SSA def(sink72) | LocalDataFlow.cs:430:23:430:28 | access to local variable sink72 | | LocalDataFlow.cs:435:17:435:24 | access to local variable nonSink0 | LocalDataFlow.cs:437:18:437:33 | SSA def(nonSink17) | +| LocalDataFlow.cs:435:17:435:24 | access to local variable nonSink0 | LocalDataFlow.cs:443:22:443:29 | access to local variable nonSink0 | | LocalDataFlow.cs:437:18:437:33 | SSA def(nonSink17) | LocalDataFlow.cs:438:23:438:31 | access to local variable nonSink17 | -| LocalDataFlow.cs:458:28:458:30 | this | LocalDataFlow.cs:458:41:458:45 | this access | -| LocalDataFlow.cs:458:50:458:52 | this | LocalDataFlow.cs:458:56:458:60 | this access | -| LocalDataFlow.cs:458:50:458:52 | value | LocalDataFlow.cs:458:50:458:52 | value | -| LocalDataFlow.cs:458:50:458:52 | value | LocalDataFlow.cs:458:64:458:68 | access to parameter value | -| LocalDataFlow.cs:464:41:464:47 | tainted | LocalDataFlow.cs:464:41:464:47 | tainted | -| LocalDataFlow.cs:464:41:464:47 | tainted | LocalDataFlow.cs:466:15:466:21 | access to parameter tainted | -| LocalDataFlow.cs:469:44:469:53 | nonTainted | LocalDataFlow.cs:469:44:469:53 | nonTainted | -| LocalDataFlow.cs:469:44:469:53 | nonTainted | LocalDataFlow.cs:471:15:471:24 | access to parameter nonTainted | +| LocalDataFlow.cs:443:13:443:38 | SSA def(sink73) | LocalDataFlow.cs:445:15:445:20 | access to local variable sink73 | +| LocalDataFlow.cs:443:22:443:29 | access to local variable nonSink0 | LocalDataFlow.cs:443:22:443:38 | ... ?? ... | +| LocalDataFlow.cs:443:22:443:29 | access to local variable nonSink0 | LocalDataFlow.cs:444:31:444:38 | access to local variable nonSink0 | +| LocalDataFlow.cs:443:22:443:38 | ... ?? ... | LocalDataFlow.cs:443:13:443:38 | SSA def(sink73) | +| LocalDataFlow.cs:443:34:443:38 | access to local variable sink0 | LocalDataFlow.cs:443:22:443:38 | ... ?? ... | +| LocalDataFlow.cs:443:34:443:38 | access to local variable sink0 | LocalDataFlow.cs:444:22:444:26 | access to local variable sink0 | +| LocalDataFlow.cs:444:13:444:38 | SSA def(sink74) | LocalDataFlow.cs:446:15:446:20 | access to local variable sink74 | +| LocalDataFlow.cs:444:22:444:26 | access to local variable sink0 | LocalDataFlow.cs:444:22:444:38 | ... ?? ... | +| LocalDataFlow.cs:444:22:444:38 | ... ?? ... | LocalDataFlow.cs:444:13:444:38 | SSA def(sink74) | +| LocalDataFlow.cs:444:31:444:38 | access to local variable nonSink0 | LocalDataFlow.cs:444:22:444:38 | ... ?? ... | +| LocalDataFlow.cs:464:28:464:30 | this | LocalDataFlow.cs:464:41:464:45 | this access | +| LocalDataFlow.cs:464:50:464:52 | this | LocalDataFlow.cs:464:56:464:60 | this access | +| LocalDataFlow.cs:464:50:464:52 | value | LocalDataFlow.cs:464:50:464:52 | value | +| LocalDataFlow.cs:464:50:464:52 | value | LocalDataFlow.cs:464:64:464:68 | access to parameter value | +| LocalDataFlow.cs:470:41:470:47 | tainted | LocalDataFlow.cs:470:41:470:47 | tainted | +| LocalDataFlow.cs:470:41:470:47 | tainted | LocalDataFlow.cs:472:15:472:21 | access to parameter tainted | +| LocalDataFlow.cs:475:44:475:53 | nonTainted | LocalDataFlow.cs:475:44:475:53 | nonTainted | +| LocalDataFlow.cs:475:44:475:53 | nonTainted | LocalDataFlow.cs:477:15:477:24 | access to parameter nonTainted | | SSA.cs:5:17:5:17 | SSA entry def(this.S) | SSA.cs:67:9:67:14 | access to field S | | SSA.cs:5:17:5:17 | this | SSA.cs:67:9:67:12 | this access | | SSA.cs:5:26:5:32 | tainted | SSA.cs:5:26:5:32 | tainted | From 9b8516cbd67083c82e197abbde4dfd2267936593 Mon Sep 17 00:00:00 2001 From: Shati Patel Date: Wed, 23 Oct 2019 17:40:48 +0100 Subject: [PATCH 026/148] Remove some mentions of "CodeQL" and fix typos --- docs/language/learn-ql/advanced/constraining-types.rst | 2 +- docs/language/learn-ql/database.rst | 2 +- docs/language/learn-ql/intro-to-data-flow.rst | 6 +++--- docs/language/learn-ql/introduction-to-ql.rst | 2 +- .../learn-ql/writing-queries/introduction-to-queries.rst | 6 +++--- docs/language/learn-ql/writing-queries/path-queries.rst | 8 ++++---- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/language/learn-ql/advanced/constraining-types.rst b/docs/language/learn-ql/advanced/constraining-types.rst index 8c00f86fe50..3a8d2e76637 100644 --- a/docs/language/learn-ql/advanced/constraining-types.rst +++ b/docs/language/learn-ql/advanced/constraining-types.rst @@ -8,7 +8,7 @@ Type constraint methods Note - The examples below use the CodeQL library for Java. All CodeQL libraries support using these methods to constrain variables, the only difference is in the names of the classes used. + The examples below use the CodeQL library for Java. All libraries support using these methods to constrain variables, the only difference is in the names of the classes used. There are several ways of imposing type constraints on variables: diff --git a/docs/language/learn-ql/database.rst b/docs/language/learn-ql/database.rst index 70ba0091853..fe8e2e273ef 100644 --- a/docs/language/learn-ql/database.rst +++ b/docs/language/learn-ql/database.rst @@ -17,7 +17,7 @@ For a Java program, two key tables are: The CodeQL library defines classes to provide a layer of abstraction over each of these tables (and the related auxiliary tables): ``Expr`` and ``Stmt``. -Most classes in the CodeQL library are similar: they are abstractions over one or more database tables. Looking at one of the CodeQL libraries illustrates this: +Most classes in the library are similar: they are abstractions over one or more database tables. Looking at one of the libraries illustrates this: .. code-block:: ql diff --git a/docs/language/learn-ql/intro-to-data-flow.rst b/docs/language/learn-ql/intro-to-data-flow.rst index 4868c3e952d..fcd907af947 100644 --- a/docs/language/learn-ql/intro-to-data-flow.rst +++ b/docs/language/learn-ql/intro-to-data-flow.rst @@ -58,9 +58,9 @@ Computing an accurate and complete data flow graph presents several challenges: - Aliasing between variables can result in a single write changing the value that multiple pointers point to. - The data flow graph can be very large and slow to compute. -To overcome these potential problems, two kinds of data flow are modeled in the CodeQL libraries: +To overcome these potential problems, two kinds of data flow are modeled in the libraries: -- Local data flow, concerning the data flow within a single function. When reasoning about local, you only considers edges between data flow nodes belonging to the same function.It is generally sufficiently fast, efficient and precise for many queries, and it is usually possible to compute the local data flow for all functions in a CodeQL database. +- Local data flow, concerning the data flow within a single function. When reasoning about local data flow, you only consider edges between data flow nodes belonging to the same function. It is generally sufficiently fast, efficient and precise for many queries, and it is usually possible to compute the local data flow for all functions in a CodeQL database. - Global data flow, effectively considers the data flow within an entire program, by calculating data flow between functions and through object properties. Computing global data flow is typically more time and energy intensive than local data flow, therefore queries should be refined to look for more specific sources and sinks. @@ -69,7 +69,7 @@ Many CodeQL queries contain examples of both local and global data flow analysis Normal data flow vs taint tracking ********************************** -In the standard CodeQL libraries, we make a distinction between 'normal' data flow and taint tracking. +In the standard libraries, we make a distinction between 'normal' data flow and taint tracking. The normal data flow libraries are used to analyze the information flow in which data values are preserved at each step. For example, if you are tracking an insecure object ``x`` (which might be some untrusted or potentially malicious data), a step in the program may 'change' its value. So, in a simple process such as ``y = x + 1``, a normal data flow analysis will highlight the use of ``x``, but not ``y``. diff --git a/docs/language/learn-ql/introduction-to-ql.rst b/docs/language/learn-ql/introduction-to-ql.rst index a06b80839ac..e54fd9b3c69 100644 --- a/docs/language/learn-ql/introduction-to-ql.rst +++ b/docs/language/learn-ql/introduction-to-ql.rst @@ -1,7 +1,7 @@ Introduction to QL ================== -QL is a powerful query language that underlies CodeQL, which is used to analyze code. +QL is the powerful query language that underlies CodeQL, which is used to analyze code. Queries written with CodeQL can find errors and uncover variants of important security vulnerabilities. Visit Semmle's `security research page `__ to read about examples of vulnerabilities that we have recently found in open source projects. diff --git a/docs/language/learn-ql/writing-queries/introduction-to-queries.rst b/docs/language/learn-ql/writing-queries/introduction-to-queries.rst index de75db30742..4dca736817c 100644 --- a/docs/language/learn-ql/writing-queries/introduction-to-queries.rst +++ b/docs/language/learn-ql/writing-queries/introduction-to-queries.rst @@ -30,7 +30,7 @@ For information on how to format your code when contributing queries to the GitH Basic query structure ********************* -`Queries `__ written with CodeQL have the file extension ``.ql``, and contain a ``select`` clause. Many of the existing CodeQL queries include additional optional information, and have the following structure:: +`Queries `__ written with CodeQL have the file extension ``.ql``, and contain a ``select`` clause. Many of the existing queries include additional optional information, and have the following structure:: /** * @@ -85,9 +85,9 @@ When writing your own alert queries, you would typically import the standard lib - JavaScript/TypeScript: ``javascript`` - Python: ``python`` -There are also CodeQL libraries containing commonly used predicates, types, and other modules associated with different analyses, including data flow, control flow, and taint-tracking. In order to calculate path graphs, path queries require you to import a data flow library into the query file. See :doc:`Constructing path queries ` for further information. +There are also libraries containing commonly used predicates, types, and other modules associated with different analyses, including data flow, control flow, and taint-tracking. In order to calculate path graphs, path queries require you to import a data flow library into the query file. See :doc:`Constructing path queries ` for further information. -You can explore the contents of all the standard CodeQL libraries in the `CodeQL library reference documentation `__, using `QL for Eclipse `__, or in the `GitHub repository `__. +You can explore the contents of all the standard libraries in the `CodeQL library reference documentation `__, using `QL for Eclipse `__, or in the `GitHub repository `__. Optional CodeQL classes and predicates diff --git a/docs/language/learn-ql/writing-queries/path-queries.rst b/docs/language/learn-ql/writing-queries/path-queries.rst index 8b793a63181..73708b17eed 100644 --- a/docs/language/learn-ql/writing-queries/path-queries.rst +++ b/docs/language/learn-ql/writing-queries/path-queries.rst @@ -6,7 +6,7 @@ Overview Security researchers are particularly interested in the way that information flows in a program. Many vulnerabilities are caused by seemingly benign data flowing to unexpected locations, and being used in a malicious way. Path queries written with CodeQL are particularly useful for analyzing data flow as they can be used to track the path taken by a variable from its possible starting points (``source``) to its possible end points (``sink``). -To model paths with CodeQL, your query must provide information about the ``source`` and the ``sink``, as well as the data flow steps that link them. +To model paths, your query must provide information about the ``source`` and the ``sink``, as well as the data flow steps that link them. This topic provides information on how to structure a path query file so you can explore the paths associated with the results of data flow analysis. @@ -18,7 +18,7 @@ This topic provides information on how to structure a path query file so you can To learn more about modeling data flow with CodeQL, see :doc:`Introduction to data flow <../intro-to-data-flow>`. -For more language-specific information on analyzing data flow see: +For more language-specific information on analyzing data flow, see: - :doc:`Analyzing data flow in C/C++ <../cpp/dataflow>` - :doc:`Analyzing data flow in C# <../csharp/dataflow>` @@ -103,7 +103,7 @@ Generating path explanations In order to generate path explanations, your query needs to compute a `path graph `__. To do this you need to define a `query predicate `__ called ``edges`` in your query. This predicate defines the edge relations of the graph you are computing, and it is used to compute the paths related to each result that your query generates. -You can import a predefined ``edges`` predicate from a path graph module in one of the standard CodeQL data flow libraries. In addition to the path graph module, the data flow libraries contain the other ``classes``, ``predicates``, and ``modules`` that are commonly used in data flow analysis. The import statement to use depends on the language that you are analyzing. +You can import a predefined ``edges`` predicate from a path graph module in one of the standard data flow libraries. In addition to the path graph module, the data flow libraries contain the other ``classes``, ``predicates``, and ``modules`` that are commonly used in data flow analysis. The import statement to use depends on the language that you are analyzing. For C/C++, C#. Java, and JavaScript you would use:: @@ -115,7 +115,7 @@ For Python, the ``Paths`` module contains the ``edges`` predicate:: import semmle.python.security.Paths -You can also import libraries specifically designed to implement data flow analysis in various common frameworks and environments, and many additional libraries are included with CodeQL. To see examples of the different libraries used in data flow analysis, see the links to the built-in queries above or browse the `standard CodeQL libraries `__. +You can also import libraries specifically designed to implement data flow analysis in various common frameworks and environments, and many additional libraries are included with CodeQL. To see examples of the different libraries used in data flow analysis, see the links to the built-in queries above or browse the `standard libraries `__. For all languages, you can also optionally define a ``nodes`` query predicate, which specifies the nodes of the path graph that you are interested in. If ``nodes`` is defined, only edges with endpoints defined by these nodes are selected. If ``nodes`` is not defined, you select all possible endpoints of ``edges``. From 6cf8f06191b0288da7c6f7beca4dc2e1d1258f3b Mon Sep 17 00:00:00 2001 From: Shati Patel Date: Wed, 23 Oct 2019 17:48:35 +0100 Subject: [PATCH 027/148] Docs: Update COBOL --- .../cobol/introduce-libraries-cobol.rst | 24 +++++++++---------- docs/language/learn-ql/cobol/ql-for-cobol.rst | 12 +++++----- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/docs/language/learn-ql/cobol/introduce-libraries-cobol.rst b/docs/language/learn-ql/cobol/introduce-libraries-cobol.rst index 8ab24df1edd..2d01200484d 100644 --- a/docs/language/learn-ql/cobol/introduce-libraries-cobol.rst +++ b/docs/language/learn-ql/cobol/introduce-libraries-cobol.rst @@ -1,10 +1,10 @@ -Introducing the QL libraries for COBOL -====================================== +Introducing the CodeQL libraries for COBOL +========================================== Overview -------- -There is an extensive QL library for analyzing COBOL code. The classes in this library present the data from a snapshot database in an object-oriented form and provide abstractions and predicates to help you with common analysis tasks. +There is an extensive library for analyzing COBOL code. The classes in this library present the data from a CodeQL database in an object-oriented form and provide abstractions and predicates to help you with common analysis tasks. The library is implemented as a set of QL modules–that is, files with the extension ``.qll``. The module ``cobol.qll`` imports most other standard library modules, so you can include the complete library by beginning your query with: @@ -12,12 +12,12 @@ The library is implemented as a set of QL modules–that is, files with the exte import cobol -The rest of this tutorial briefly summarizes the most important QL classes and predicates provided by this library, including references to the `detailed API documentation `__ where applicable. +The rest of this tutorial briefly summarizes the most important classes and predicates provided by this library, including references to the `detailed API documentation `__ where applicable. Introducing the library ----------------------- -The QL COBOL library presents information about COBOL source code at different levels: +The CodeQL library for COBOL presents information about COBOL source code at different levels: - **Textual** — classes that represent source code as unstructured text files - **Lexical** — classes that represent comments and other tokens of interest @@ -36,7 +36,7 @@ At its most basic level, a COBOL code base can simply be viewed as a collection Files and folders ^^^^^^^^^^^^^^^^^ -In QL, files are represented as entities of class `File `__, and folders as entities of class `Folder `__, both of which are subclasses of class `Container `__. +Files are represented as entities of class `File `__, and folders as entities of class `Folder `__, both of which are subclasses of class `Container `__. Class `Container `__ provides the following member predicates: @@ -48,7 +48,7 @@ Note that while ``getAFile`` and ``getAFolder`` are declared on class `Container Both files and folders have paths, which can be accessed by the predicate ``Container.getAbsolutePath()``. For example, if ``f`` represents a file with the path ``/home/user/project/src/main.cbl``, then ``f.getAbsolutePath()`` evaluates to the string ``"/home/user/project/src/main.cbl"``, while ``f.getParentContainer().getAbsolutePath()`` returns ``"/home/user/project/src"``. -These paths are absolute file system paths. If you want to obtain the path of a file relative to the snapshot source location, use ``Container.getRelativePath()`` instead. Note, however, that a snapshot may contain files that are not located underneath the snapshot source location; for such files, ``getRelativePath()`` will not return anything. +These paths are absolute file system paths. If you want to obtain the path of a file relative to the source location in the CodeQL database, use ``Container.getRelativePath()`` instead. Note, however, that a database may contain files that are not located underneath the source location; for such files, ``getRelativePath()`` will not return anything. The following member predicates of class `Container `__ provide more information about the name of a file or folder: @@ -68,9 +68,9 @@ For example, the following query computes, for each folder, the number of COBOL Locations ^^^^^^^^^ -Most entities in a snapshot database have an associated source location. Locations are identified by four pieces of information: a file, a start line, a start column, an end line, and an end column. Line and column counts are 1-based (so the first character of a file is at line 1, column 1), and the end position is inclusive. +Most entities in a CodeQL database have an associated source location. Locations are identified by four pieces of information: a file, a start line, a start column, an end line, and an end column. Line and column counts are 1-based (so the first character of a file is at line 1, column 1), and the end position is inclusive. -All entities associated with a source location belong to the QL class `Locatable `__. The location itself is modeled by the QL class `Location `__ and can be accessed through the member predicate ``Locatable.getLocation()``. The `Location `__ class provides the following member predicates: +All entities associated with a source location belong to the class `Locatable `__. The location itself is modeled by the class `Location `__ and can be accessed through the member predicate ``Locatable.getLocation()``. The `Location `__ class provides the following member predicates: - ``Location.getFile()``, ``Location.getStartLine()``, ``Location.getStartColumn()``, ``Location.getEndLine()``, ``Location.getEndColumn()`` return detailed information about the location. - ``Location.getNumLines()`` returns the number of (whole or partial) lines covered by the location. @@ -104,13 +104,13 @@ The most important member predicates are as follows: Syntactic level ~~~~~~~~~~~~~~~ -The majority of classes in the QL COBOL library is concerned with representing a COBOL program as a collection of `abstract syntax trees `__ (ASTs). +The majority of classes in the CodeQL library for COBOL are concerned with representing a COBOL program as a collection of `abstract syntax trees `__ (ASTs). -The QL class `ASTNode `__ contains all entities representing nodes in the abstract syntax trees and defines generic tree traversal predicates: +The class `ASTNode `__ contains all entities representing nodes in the abstract syntax trees and defines generic tree traversal predicates: - ``ASTNode.getParent()``: returns the parent node of this AST node, if any. -Please note that the QL libraries for COBOL do not currently represent all possible parts of a COBOL program. Due to the complexity of the language, and its many dialects, this is an ongoing task. We prioritize elements that are of interest to queries, and expand this selection over time. Please check the `detailed API documentation `__ to see what is currently available. +Please note that the libraries for COBOL do not currently represent all possible parts of a COBOL program. Due to the complexity of the language, and its many dialects, this is an ongoing task. We prioritize elements that are of interest to queries, and expand this selection over time. Please check the `detailed API documentation `__ to see what is currently available. The main structure of any COBOL program is represented by the `Unit `__ class and its subclasses. For example, each program definition has a `ProgramDefinition `__ counterpart. For each ``PROCEDURE DIVISION`` in the program, there will be a `ProcedureDivision `__ class. diff --git a/docs/language/learn-ql/cobol/ql-for-cobol.rst b/docs/language/learn-ql/cobol/ql-for-cobol.rst index 12ba25e69f4..f85ff62649e 100644 --- a/docs/language/learn-ql/cobol/ql-for-cobol.rst +++ b/docs/language/learn-ql/cobol/ql-for-cobol.rst @@ -1,5 +1,5 @@ -QL for COBOL -============ +CodeQL for COBOL +================ .. toctree:: :glob: @@ -7,14 +7,14 @@ QL for COBOL introduce-libraries-cobol -This page provides an overview of the QL for COBOL documentation that is currently available. +This page provides an overview of the CodeQL for COBOL documentation that is currently available. - `Basic COBOL query `__ describes how to write and run queries using LGTM. -- :doc:`Introducing the QL libraries for COBOL ` introduces the standard libraries used to write queries for COBOL code. +- :doc:`Introducing the CodeQL libraries for COBOL ` introduces the standard libraries used to write queries for COBOL code. Other resources --------------- -- For the queries used in LGTM, display a `COBOL query `__ and click **Open in query console** to see the QL code used to find alerts. -- For more information about the COBOL QL library see the `QL library for COBOL `__. +- For the queries used in LGTM, display a `COBOL query `__ and click **Open in query console** to see the code used to find alerts. +- For more information about the library for COBOL see the `CodeQL library for COBOL `__. From 2aefcbd42c69a55e29813f592b87c64bd2375dd5 Mon Sep 17 00:00:00 2001 From: Shati Patel Date: Wed, 23 Oct 2019 18:16:20 +0100 Subject: [PATCH 028/148] Docs: Update C/C++ --- .../learn-ql/cpp/conversions-classes.rst | 18 ++++++------- docs/language/learn-ql/cpp/dataflow.rst | 6 ++--- .../learn-ql/cpp/expressions-types.rst | 8 +++--- .../learn-ql/cpp/function-classes.rst | 10 +++---- .../learn-ql/cpp/introduce-libraries-cpp.rst | 24 ++++++++--------- .../cpp/private-field-initialization.rst | 6 ++--- docs/language/learn-ql/cpp/ql-for-cpp.rst | 26 ++++++++++--------- .../cpp/value-numbering-hash-cons.rst | 2 +- .../learn-ql/cpp/zero-space-terminator.rst | 8 +++--- 9 files changed, 55 insertions(+), 53 deletions(-) diff --git a/docs/language/learn-ql/cpp/conversions-classes.rst b/docs/language/learn-ql/cpp/conversions-classes.rst index 716991f96b2..016efa50d67 100644 --- a/docs/language/learn-ql/cpp/conversions-classes.rst +++ b/docs/language/learn-ql/cpp/conversions-classes.rst @@ -4,12 +4,12 @@ Tutorial: Conversions and classes Overview -------- -This topic contains worked examples of how to write queries using the standard QL library classes for C/C++ conversions and classes. +This topic contains worked examples of how to write queries using the CodeQL library classes for C/C++ conversions and classes. Conversions ----------- -Let us take a look at the QL ``Conversion`` class in the standard library: +Let us take a look at the ``Conversion`` class in the standard library: - ``Expr`` @@ -128,26 +128,26 @@ Unlike the earlier versions of the query, this query would return each side of t Note - In general, QL predicates named ``getAXxx`` exploit the ability to return multiple results (multiple instances of ``Xxx``) whereas plain ``getXxx`` predicates usually return at most one specific instance of ``Xxx``. + In general, predicates named ``getAXxx`` exploit the ability to return multiple results (multiple instances of ``Xxx``) whereas plain ``getXxx`` predicates usually return at most one specific instance of ``Xxx``. Classes ------- -Next we're going to look at C++ classes, using the following QL classes: +Next we're going to look at C++ classes, using the following CodeQL classes: - ``Type`` - - ``UserType``—includes classes, typedefs and enums + - ``UserType``—includes classes, typedefs, and enums - ``Class``—a class or struct - - ``Struct``—a struct, which is treated as a subtype of Class in QL. + - ``Struct``—a struct, which is treated as a subtype of ``Class`` - ``TemplateClass``—a C++ class template Finding derived classes ~~~~~~~~~~~~~~~~~~~~~~~ -We want to create a query that checks for destructors that should be ``virtual``. Specifically, when a class and a class derived from it both have destructors, the base class destructor should generally be virtual. This ensures that the derived class destructor is always invoked. A ``Destructor`` in QL is a subtype of ``MemberFunction``: +We want to create a query that checks for destructors that should be ``virtual``. Specifically, when a class and a class derived from it both have destructors, the base class destructor should generally be virtual. This ensures that the derived class destructor is always invoked. In the CodeQL library, ``Destructor`` is a subtype of ``MemberFunction``: - ``Function`` @@ -221,13 +221,13 @@ Our last change is to use ``Function.isVirtual()`` to find cases where the base That completes the query. -There is a similar built-in LGTM `query `__ that finds classes in a C/C++ project with virtual functions but no virtual destructor. You can take a look at the QL code for this query by clicking **Open in query console** at the top of that page. +There is a similar built-in LGTM `query `__ that finds classes in a C/C++ project with virtual functions but no virtual destructor. You can take a look at the code for this query by clicking **Open in query console** at the top of that page. What next? ---------- - Explore other ways of querying classes using examples from the `C/C++ cookbook `__. - Take a look at the :doc:`Analyzing data flow in C/C++ ` tutorial. -- Try the worked examples in the following topics: :doc:`Example: Checking that constructors initialize all private fields ` and :doc:`Example: Checking for allocations equal to 'strlen(string)' without space for a null terminator `. +- Try the worked examples in the following topics: :doc:`Example: Checking that constructors initialize all private fields `, and :doc:`Example: Checking for allocations equal to 'strlen(string)' without space for a null terminator `. - Find out more about QL in the `QL language handbook `__ and `QL language specification `__. - Learn more about the query console in `Using the query console `__. diff --git a/docs/language/learn-ql/cpp/dataflow.rst b/docs/language/learn-ql/cpp/dataflow.rst index 4867f3daf47..0b28c71445f 100644 --- a/docs/language/learn-ql/cpp/dataflow.rst +++ b/docs/language/learn-ql/cpp/dataflow.rst @@ -4,10 +4,10 @@ Analyzing data flow in C/C++ Overview -------- -This topic describes how data flow analysis is implemented in the QL for C/C++ library and includes examples to help you write your own data flow queries. -The following sections describe how to utilize the QL libraries for local data flow, global data flow and taint tracking. +This topic describes how data flow analysis is implemented in the CodeQL libraries for C/C++ and includes examples to help you write your own data flow queries. +The following sections describe how to utilize the libraries for local data flow, global data flow, and taint tracking. -For a more general introduction to modeling data flow in QL, see :doc:`Introduction to data flow analysis in QL <../intro-to-data-flow>`. +For a more general introduction to modeling data flow, see :doc:`Introduction to data flow analysis with CodeQL <../intro-to-data-flow>`. Local data flow --------------- diff --git a/docs/language/learn-ql/cpp/expressions-types.rst b/docs/language/learn-ql/cpp/expressions-types.rst index d6bd0b2c740..1595d594683 100644 --- a/docs/language/learn-ql/cpp/expressions-types.rst +++ b/docs/language/learn-ql/cpp/expressions-types.rst @@ -4,12 +4,12 @@ Tutorial: Expressions, types and statements Overview -------- -This topic contains worked examples of how to write queries using the standard QL library classes for C/C++ expressions, types, and statements. +This topic contains worked examples of how to write queries using the standard CodeQL library classes for C/C++ expressions, types, and statements. Expressions and types --------------------- -Each part of an expression in C becomes an instance of the QL ``Expr`` class. For example, the C code ``x = x + 1`` becomes an ``AssignExpr``, an ``AddExpr``, two instances of ``VariableAccess`` and a ``Literal``. All of these QL classes extend ``Expr``. +Each part of an expression in C becomes an instance of the ``Expr`` class. For example, the C code ``x = x + 1`` becomes an ``AssignExpr``, an ``AddExpr``, two instances of ``VariableAccess`` and a ``Literal``. All of these CodeQL classes extend ``Expr``. Finding assignments to zero ~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -26,7 +26,7 @@ In the following example we find instances of ``AssignExpr`` which assign the co ➤ `See this in the query console `__ -The ``where`` clause in this example gets the expression on the right side of the assignment, ``getRValue()``, and compares it with zero. Notice that there are no checks to make sure that the right side of the assignment is an integer or that it has a value (that is, it is compile-time constant, rather than a variable). For expressions where either of these assumptions is wrong, the associated QL predicate simply does not return anything and the ``where`` clause will not produce a result. You could think of it as if there is an implicit ``exists(e.getRValue().getValue().toInt())`` at the beginning of this line. +The ``where`` clause in this example gets the expression on the right side of the assignment, ``getRValue()``, and compares it with zero. Notice that there are no checks to make sure that the right side of the assignment is an integer or that it has a value (that is, it is compile-time constant, rather than a variable). For expressions where either of these assumptions is wrong, the associated predicate simply does not return anything and the ``where`` clause will not produce a result. You could think of it as if there is an implicit ``exists(e.getRValue().getValue().toInt())`` at the beginning of this line. It is also worth noting that the query above would find this C code: @@ -34,7 +34,7 @@ It is also worth noting that the query above would find this C code: yPtr = NULL; -This is because the snapshot contains a representation of the code base after the preprocessor transforms have run (for more information, see `Database generation `__). This means that any macro invocations, such as the ``NULL`` define used here, are expanded during the creation of the snapshot. If you want to write queries about macros then there are some special library classes that have been designed specifically for this purpose (for example, the ``Macro``, ``MacroInvocation`` classes and predicates like ``Element.isInMacroExpansion()``). In this case, it is good that macros are expanded, but we do not want to find assignments to pointers. +This is because the database contains a representation of the code base after the preprocessor transforms have run (for more information, see `Database generation `__). This means that any macro invocations, such as the ``NULL`` define used here, are expanded during the creation of the database. If you want to write queries about macros then there are some special library classes that have been designed specifically for this purpose (for example, the ``Macro``, ``MacroInvocation`` classes and predicates like ``Element.isInMacroExpansion()``). In this case, it is good that macros are expanded, but we do not want to find assignments to pointers. Finding assignments of 0 to an integer ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/language/learn-ql/cpp/function-classes.rst b/docs/language/learn-ql/cpp/function-classes.rst index e8f0f743722..51f07a8d0f0 100644 --- a/docs/language/learn-ql/cpp/function-classes.rst +++ b/docs/language/learn-ql/cpp/function-classes.rst @@ -4,14 +4,14 @@ Tutorial: Function classes Overview -------- -The standard QL library for C and C++ represents functions using the ``Function`` class (see :doc:`Introducing the C/C++ libraries `). +The standard CodeQL library for C and C++ represents functions using the ``Function`` class (see :doc:`Introducing the C/C++ libraries `). The example queries in this topic explore some of the most useful library predicates for querying functions. Finding all static functions ---------------------------- -Using the member predicate ``Function.isStatic()`` we can list all of the static functions in a snapshot: +Using the member predicate ``Function.isStatic()`` we can list all the static functions in a database: .. code-block:: ql @@ -26,7 +26,7 @@ This query is very general, so there are probably too many results to be interes Finding functions that are not called ------------------------------------- -It might be more interesting to find functions that are not called, using the standard QL ``FunctionCall`` class from the **abstract syntax tree** category (see :doc:`Introducing the C/C++ libraries `). The ``FunctionCall`` class can be used to identify places where a function is actually used, and it is related to ``Function`` through the ``FunctionCall.getTarget()`` predicate. +It might be more interesting to find functions that are not called, using the standard CodeQL ``FunctionCall`` class from the **abstract syntax tree** category (see :doc:`Introducing the C/C++ libraries `). The ``FunctionCall`` class can be used to identify places where a function is actually used, and it is related to ``Function`` through the ``FunctionCall.getTarget()`` predicate. .. code-block:: ql @@ -58,9 +58,9 @@ You can modify the query to remove functions where a function pointer is used to This query returns fewer results. However, if you examine the results then you can probably still find potential refinements. -For example, there is a more complicated LGTM `query `__ that finds unused static functions. To see the QL code for this query, click **Open in query console** at the top of the page. +For example, there is a more complicated LGTM `query `__ that finds unused static functions. To see the code for this query, click **Open in query console** at the top of the page. - You can explore the definition of an element in the standard QL libraries and see what predicates are available. Use the keyboard **F3** button to open the definition of any element. Alternatively, hover over the element and click **Jump to definition** in the tooltip displayed. The library file is opened in a new tab with the definition highlighted. + You can explore the definition of an element in the standard libraries and see what predicates are available. Use the keyboard **F3** button to open the definition of any element. Alternatively, hover over the element and click **Jump to definition** in the tooltip displayed. The library file is opened in a new tab with the definition highlighted. Finding a specific function --------------------------- diff --git a/docs/language/learn-ql/cpp/introduce-libraries-cpp.rst b/docs/language/learn-ql/cpp/introduce-libraries-cpp.rst index 1ab7cdd1498..6bea8d40c2c 100644 --- a/docs/language/learn-ql/cpp/introduce-libraries-cpp.rst +++ b/docs/language/learn-ql/cpp/introduce-libraries-cpp.rst @@ -1,23 +1,23 @@ -Introducing the C/C++ libraries -=============================== +Introducing the CodeQL libraries for C/C++ +========================================== Overview -------- -There is an extensive QL library for analyzing snapshots extracted from C/C++ projects. The QL classes in this library present the data from a snapshot database in an object-oriented form and provide abstractions and predicates to help you with common analysis tasks. The library is implemented as a set of QL modules, that is, files with the extension ``.qll``. The module ``cpp.qll`` imports all of the core C/C++ library modules, so you can include the complete library by beginning your query with: +There is an extensive library for analyzing CodeQL databases extracted from C/C++ projects. The classes in this library present the data from a database in an object-oriented form and provide abstractions and predicates to help you with common analysis tasks. The library is implemented as a set of QL modules, that is, files with the extension ``.qll``. The module ``cpp.qll`` imports all the core C/C++ library modules, so you can include the complete library by beginning your query with: .. code-block:: ql import cpp -The rest of this topic summarizes available QL classes and corresponding C/C++ constructs. +The rest of this topic summarizes the available CodeQL classes and corresponding C/C++ constructs. -NOTE: You can find related classes and features using the query console's auto-complete feature. You can also press *F3* to jump to the definition of any element; QL library files are opened in new tabs in the console. +NOTE: You can find related classes and features using the query console's auto-complete feature. You can also press *F3* to jump to the definition of any element; library files are opened in new tabs in the console. Summary of the library classes ------------------------------ -The most commonly used standard QL library classes are listed below. The listing is broken down by functionality. Each QL library class is annotated with a C/C++ construct it corresponds to. +The most commonly used standard library classes are listed below. The listing is broken down by functionality. Each library class is annotated with a C/C++ construct it corresponds to. Declaration classes ~~~~~~~~~~~~~~~~~~~ @@ -25,7 +25,7 @@ Declaration classes This table lists `Declaration `__ classes representing C/C++ declarations. +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Example syntax | QL class | Remarks | +| Example syntax | CodeQL class | Remarks | +=================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================+=======================================================================================================================================================================+====================================================================================================================================================================================================+ | ``int`` *var* ``;`` | `GlobalVariable `__ | | +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -154,7 +154,7 @@ Statement classes This table lists subclasses of `Stmt `__ representing C/C++ statements. +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Example syntax | QL class | Remarks | +| Example syntax | CodeQL class | Remarks | +===============================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================+==================================================================================================================================================================+===================================================================================================================================================================================================================================================================================================+ | ``__asm__ ("`` *movb %bh, (%eax)* ``");`` | `AsmStmt `__ | Specific to a given CPU instruction set | +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -215,7 +215,7 @@ Expression classes This table lists subclasses of `Expr `__ representing C/C++ expressions. +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Example syntax | QL class(es) | Remarks | +| Example syntax | CodeQL class(es) | Remarks | +========================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================+==========================================================================================================================================================================================================+=============================================================================================================================================================================================================================================================================================================+ | ``{`` `Expr `__... ``}`` | | `ArrayAggregateLiteral `__ | | | | | `ClassAggregateLiteral `__ | | @@ -417,7 +417,7 @@ Type classes This table lists subclasses of `Type `__ representing C/C++ types. +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Example syntax | QL class | Remarks | +| Example syntax | CodeQL class | Remarks | +=============================================================================================================================================================================================================================================================================================================================================================================================================================================+==================================================================================================================================================================+==================================================================================================================================================================================================================================================================================+ | ``void`` | `VoidType `__ | | +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -485,7 +485,7 @@ Preprocessor classes This table lists `Preprocessor `__ classes representing C/C++ preprocessing directives. +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Example syntax | QL class | Remarks | +| Example syntax | CodeQL class | Remarks | +=============================================================================================================================================================================================================================================================================================================================================================================================================================================+==================================================================================================================================================================+===================================================================================================================================================================================================================================================================================+ | ``#elif`` *condition* | `PreprocessorElif `__ | | +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -522,6 +522,6 @@ This table lists `Preprocessor `, :doc:`Expressions, types and statements `, :doc:`Conversions and classes `, and :doc:`Analyzing data flow in C/C++ `. +- Experiment with the worked examples in the CodeQL for C/C++ topics: :doc:`Function classes `, :doc:`Expressions, types and statements `, :doc:`Conversions and classes `, and :doc:`Analyzing data flow in C/C++ `. - Find out more about QL in the `QL language handbook `__ and `QL language specification `__. - Learn more about the query console in `Using the query console `__. diff --git a/docs/language/learn-ql/cpp/private-field-initialization.rst b/docs/language/learn-ql/cpp/private-field-initialization.rst index 4aa155adeb0..26b57660142 100644 --- a/docs/language/learn-ql/cpp/private-field-initialization.rst +++ b/docs/language/learn-ql/cpp/private-field-initialization.rst @@ -4,7 +4,7 @@ Example: Checking that constructors initialize all private fields Overview -------- -This topic describes how a C++ query was developed. The example introduces recursive predicates and demonstrates the typical workflow used to refine a query. For a full overview of the topics available for learning to write QL queries for C/C++ code, see :doc:`QL for C/C++ `. +This topic describes how a C++ query was developed. The example introduces recursive predicates and demonstrates the typical workflow used to refine a query. For a full overview of the topics available for learning to write queries for C/C++ code, see :doc:`CodeQL for C/C++ `. Problem—finding every private field and checking for initialization ------------------------------------------------------------------- @@ -29,7 +29,7 @@ We can start by looking at every private field in a class and checking that ever #. ``f.isPrivate()`` checks if the field is private. #. ``not exists(Assignment a | a = f.getAnAssignment() and a.getEnclosingFunction() = c)`` checks that there is no assignment to the field in the constructor. -This QL code looks fairly complete, but when you test it on a project, there are several results that contain examples that we have overlooked. +This code looks fairly complete, but when you test it on a project, there are several results that contain examples that we have overlooked. Refinement 1—excluding fields initialized by lists -------------------------------------------------- @@ -62,7 +62,7 @@ These can be excluded by adding an extra condition to check for this special con Refinement 2—excluding fields initialized by external libraries --------------------------------------------------------------- -When you test the revised query, you may discover that fields from classes in external libraries are over-reported. This is often because a header file declares a constructor that is defined in a source file that is not analyzed (external libraries are often excluded from analysis). When the source code is analyzed, the snapshot is populated with a ``Constructor`` entry with no body. This ``constructor`` therefore contains no assignments and consequently the query reports that any fields initialized by the constructor are "uninitialized." There is no particular reason to be suspicious of these cases, and we can exclude them from the results by defining a condition to exclude constructors that have no body: +When you test the revised query, you may discover that fields from classes in external libraries are over-reported. This is often because a header file declares a constructor that is defined in a source file that is not analyzed (external libraries are often excluded from analysis). When the source code is analyzed, the CodeQL database is populated with a ``Constructor`` entry with no body. This ``constructor`` therefore contains no assignments and consequently the query reports that any fields initialized by the constructor are "uninitialized." There is no particular reason to be suspicious of these cases, and we can exclude them from the results by defining a condition to exclude constructors that have no body: .. code-block:: ql diff --git a/docs/language/learn-ql/cpp/ql-for-cpp.rst b/docs/language/learn-ql/cpp/ql-for-cpp.rst index 909d0e2e323..b1596c7de92 100644 --- a/docs/language/learn-ql/cpp/ql-for-cpp.rst +++ b/docs/language/learn-ql/cpp/ql-for-cpp.rst @@ -1,5 +1,5 @@ -QL for C/C++ -============ +CodeQL for C/C++ +================ .. toctree:: :glob: @@ -13,19 +13,19 @@ QL for C/C++ private-field-initialization zero-space-terminator -These topics provide an overview of the QL C/C++ standard libraries and show examples of how to write queries that use them. +These topics provide an overview of the CodeQL libraries for C/C++ and show examples of how to write queries that use them. -- `Basic C/C++ QL query `__ describes how to write and run queries using LGTM. +- `Basic C/C++ query `__ describes how to write and run queries using LGTM. -- :doc:`Introducing the QL libraries for C/C++ ` introduces the standard libraries used to write queries for C and C++ code. +- :doc:`Introducing the CodeQL libraries for C/C++ ` introduces the standard libraries used to write queries for C and C++ code. -- :doc:`Tutorial: Function classes ` demonstrates how to write queries using the standard QL library classes for C/C++ functions. +- :doc:`Tutorial: Function classes ` demonstrates how to write queries using the standard CodeQL library classes for C/C++ functions. -- :doc:`Tutorial: Expressions, types and statements ` demonstrates how to write queries using the standard QL library classes for C/C++ expressions, types and statements. +- :doc:`Tutorial: Expressions, types and statements ` demonstrates how to write queries using the standard CodeQL library classes for C/C++ expressions, types and statements. -- :doc:`Tutorial: Conversions and classes ` demonstrates how to write queries using the standard QL library classes for C/C++ conversions and classes. +- :doc:`Tutorial: Conversions and classes ` demonstrates how to write queries using the standard CodeQL library classes for C/C++ conversions and classes. -- :doc:`Tutorial: Analyzing data flow in C/C++ ` demonstrates how to write queries using the standard QL for C/C++ data flow and taint tracking libraries. +- :doc:`Tutorial: Analyzing data flow in C/C++ ` demonstrates how to write queries using the standard data flow and taint tracking libraries for C/C++. - :doc:`Example: Checking that constructors initialize all private fields ` works through the development of a query. It introduces recursive predicates and shows the typical workflow used to refine a query. @@ -51,6 +51,8 @@ Advanced libraries Other resources --------------- -- For examples of how to query common C/C++ elements, see the `C/C++ QL cookbook `__. -- For the queries used in LGTM, display a `C/C++ query `__ and click **Open in query console** to see the QL code used to find alerts. -- For more information about the C/C++ QL library see the `QL library for C/C++ `__. +.. TODO: Rename the cookbooks: C/C++ cookbook, or C/C++ CodeQL cookbook, or CodeQL cookbook for C/C++, or...? + +- For examples of how to query common C/C++ elements, see the `C/C++ cookbook `__. +- For the queries used in LGTM, display a `C/C++ query `__ and click **Open in query console** to see the code used to find alerts. +- For more information about the library for C/C++ see the `CodeQL library for C/C++ `__. diff --git a/docs/language/learn-ql/cpp/value-numbering-hash-cons.rst b/docs/language/learn-ql/cpp/value-numbering-hash-cons.rst index 7a1bbd2888b..e05f93dbae1 100644 --- a/docs/language/learn-ql/cpp/value-numbering-hash-cons.rst +++ b/docs/language/learn-ql/cpp/value-numbering-hash-cons.rst @@ -4,7 +4,7 @@ Hash consing and value numbering Overview -------- -In C and C++ QL databases, each node in the abstract syntax tree is represented by a separate object. This allows both analysis and results display to refer to specific appearances of a piece of syntax. However, it is frequently useful to determine whether two expressions are equivalent, either syntactically or semantically. +In C and C++ databases, each node in the abstract syntax tree is represented by a separate object. This allows both analysis and results display to refer to specific appearances of a piece of syntax. However, it is frequently useful to determine whether two expressions are equivalent, either syntactically or semantically. The `hash consing `__ library (defined in ``semmle.code.cpp.valuenumbering.HashCons``) provides a mechanism for identifying expressions that have the same syntactic structure. The `global value numbering `__ library (defined in ``semmle.code.cpp.valuenumbering.GlobalValueNumbering``) provides a mechanism for identifying expressions that compute the same value at runtime. diff --git a/docs/language/learn-ql/cpp/zero-space-terminator.rst b/docs/language/learn-ql/cpp/zero-space-terminator.rst index e977470ea6f..7026d9fa9ae 100644 --- a/docs/language/learn-ql/cpp/zero-space-terminator.rst +++ b/docs/language/learn-ql/cpp/zero-space-terminator.rst @@ -4,7 +4,7 @@ Example: Checking for allocations equal to ``strlen(string)`` without space for Overview -------- -This topic describes how a C/C++ query for detecting a potential buffer overflow was developed. For a full overview of the topics available for learning to write QL queries for C/C++ code, see :doc:`QL for C/C++ `. +This topic describes how a C/C++ query for detecting a potential buffer overflow was developed. For a full overview of the topics available for learning to write queries for C/C++ code, see :doc:`CodeQL for C/C++ `. Problem—detecting memory allocation that omits space for a null termination character ------------------------------------------------------------------------------------- @@ -32,7 +32,7 @@ Defining the entities of interest You could approach this problem either by searching for code similar to the call to ``malloc`` in line 3 or the call to ``strcpy`` in line 5 (see example above). For our basic query, we start with a simple assumption: any call to ``malloc`` with only a ``strlen`` to define the memory size is likely to cause an error when the memory is populated. -Calls to ``strlen`` can be identified using the library `StrlenCall `__ class, but we need to define a new class to identify calls to ``malloc``. Both the library class and the new class need to extend the standard QL class ``FunctionCall``, with the added restriction of the function name that they apply to: +Calls to ``strlen`` can be identified using the library `StrlenCall `__ class, but we need to define a new class to identify calls to ``malloc``. Both the library class and the new class need to extend the standard class ``FunctionCall``, with the added restriction of the function name that they apply to: .. code-block:: ql @@ -52,7 +52,7 @@ Calls to ``strlen`` can be identified using the library `StrlenCall Date: Wed, 23 Oct 2019 21:23:00 +0100 Subject: [PATCH 029/148] C#: Add ?? as a local dataflow step. --- csharp/ql/src/API Abuse/NoDisposeCallOnLocalIDisposable.ql | 2 -- .../semmle/code/csharp/dataflow/internal/DataFlowPrivate.qll | 4 ++++ csharp/ql/test/library-tests/dataflow/local/DataFlow.expected | 2 ++ .../test/library-tests/dataflow/local/DataFlowStep.expected | 4 ++++ 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/csharp/ql/src/API Abuse/NoDisposeCallOnLocalIDisposable.ql b/csharp/ql/src/API Abuse/NoDisposeCallOnLocalIDisposable.ql index 3a50d474577..646ab04ac1f 100644 --- a/csharp/ql/src/API Abuse/NoDisposeCallOnLocalIDisposable.ql +++ b/csharp/ql/src/API Abuse/NoDisposeCallOnLocalIDisposable.ql @@ -81,8 +81,6 @@ private class Conf extends DataFlow::Configuration { override predicate isAdditionalFlowStep(DataFlow::Node node1, DataFlow::Node node2) { node2.asExpr() = any(LocalScopeDisposableCreation other | other.getAnArgument() = node1.asExpr()) - or - exists(NullCoalescingExpr nce | node1.asExpr() = nce.getRightOperand() and node2.asExpr() = nce) } override predicate isBarrierOut(DataFlow::Node node) { diff --git a/csharp/ql/src/semmle/code/csharp/dataflow/internal/DataFlowPrivate.qll b/csharp/ql/src/semmle/code/csharp/dataflow/internal/DataFlowPrivate.qll index d1ca7596f30..e69d5d93bf5 100644 --- a/csharp/ql/src/semmle/code/csharp/dataflow/internal/DataFlowPrivate.qll +++ b/csharp/ql/src/semmle/code/csharp/dataflow/internal/DataFlowPrivate.qll @@ -86,6 +86,10 @@ module LocalFlow { scope = e2 and isSuccessor = true or + e1 = e2.(NullCoalescingExpr).getAnOperand() and + scope = e2 and + isSuccessor = false + or e1 = e2.(SuppressNullableWarningExpr).getExpr() and scope = e2 and isSuccessor = true diff --git a/csharp/ql/test/library-tests/dataflow/local/DataFlow.expected b/csharp/ql/test/library-tests/dataflow/local/DataFlow.expected index d1801f1833a..78c0204ed20 100644 --- a/csharp/ql/test/library-tests/dataflow/local/DataFlow.expected +++ b/csharp/ql/test/library-tests/dataflow/local/DataFlow.expected @@ -8,6 +8,8 @@ | LocalDataFlow.cs:412:15:412:20 | access to local variable sink70 | | LocalDataFlow.cs:420:19:420:24 | access to local variable sink71 | | LocalDataFlow.cs:430:23:430:28 | access to local variable sink72 | +| LocalDataFlow.cs:445:15:445:20 | access to local variable sink73 | +| LocalDataFlow.cs:446:15:446:20 | access to local variable sink74 | | LocalDataFlow.cs:472:15:472:21 | access to parameter tainted | | SSA.cs:9:15:9:22 | access to local variable ssaSink0 | | SSA.cs:25:15:25:22 | access to local variable ssaSink1 | diff --git a/csharp/ql/test/library-tests/dataflow/local/DataFlowStep.expected b/csharp/ql/test/library-tests/dataflow/local/DataFlowStep.expected index ba8398610a1..1b93aded791 100644 --- a/csharp/ql/test/library-tests/dataflow/local/DataFlowStep.expected +++ b/csharp/ql/test/library-tests/dataflow/local/DataFlowStep.expected @@ -568,11 +568,15 @@ | LocalDataFlow.cs:435:17:435:24 | access to local variable nonSink0 | LocalDataFlow.cs:443:22:443:29 | access to local variable nonSink0 | | LocalDataFlow.cs:437:18:437:33 | SSA def(nonSink17) | LocalDataFlow.cs:438:23:438:31 | access to local variable nonSink17 | | LocalDataFlow.cs:443:13:443:38 | SSA def(sink73) | LocalDataFlow.cs:445:15:445:20 | access to local variable sink73 | +| LocalDataFlow.cs:443:22:443:29 | access to local variable nonSink0 | LocalDataFlow.cs:443:22:443:38 | ... ?? ... | | LocalDataFlow.cs:443:22:443:29 | access to local variable nonSink0 | LocalDataFlow.cs:444:31:444:38 | access to local variable nonSink0 | | LocalDataFlow.cs:443:22:443:38 | ... ?? ... | LocalDataFlow.cs:443:13:443:38 | SSA def(sink73) | +| LocalDataFlow.cs:443:34:443:38 | access to local variable sink0 | LocalDataFlow.cs:443:22:443:38 | ... ?? ... | | LocalDataFlow.cs:443:34:443:38 | access to local variable sink0 | LocalDataFlow.cs:444:22:444:26 | access to local variable sink0 | | LocalDataFlow.cs:444:13:444:38 | SSA def(sink74) | LocalDataFlow.cs:446:15:446:20 | access to local variable sink74 | +| LocalDataFlow.cs:444:22:444:26 | access to local variable sink0 | LocalDataFlow.cs:444:22:444:38 | ... ?? ... | | LocalDataFlow.cs:444:22:444:38 | ... ?? ... | LocalDataFlow.cs:444:13:444:38 | SSA def(sink74) | +| LocalDataFlow.cs:444:31:444:38 | access to local variable nonSink0 | LocalDataFlow.cs:444:22:444:38 | ... ?? ... | | LocalDataFlow.cs:464:28:464:30 | this | LocalDataFlow.cs:464:41:464:45 | this access | | LocalDataFlow.cs:464:50:464:52 | this | LocalDataFlow.cs:464:56:464:60 | this access | | LocalDataFlow.cs:464:50:464:52 | value | LocalDataFlow.cs:464:64:464:68 | access to parameter value | From 6ac163abace5021bfae1deb3e1b018e5ed0e831a Mon Sep 17 00:00:00 2001 From: Calum Grant Date: Wed, 23 Oct 2019 21:54:49 +0100 Subject: [PATCH 030/148] C#: Add change note --- change-notes/1.23/analysis-csharp.md | 1 + 1 file changed, 1 insertion(+) diff --git a/change-notes/1.23/analysis-csharp.md b/change-notes/1.23/analysis-csharp.md index 7ec412a0eb2..59d9903fffb 100644 --- a/change-notes/1.23/analysis-csharp.md +++ b/change-notes/1.23/analysis-csharp.md @@ -43,5 +43,6 @@ The following changes in version 1.23 affect C# analysis in all applications. * There is now a `DataFlow::localExprFlow` predicate and a `TaintTracking::localExprTaint` predicate to make it easy to use the most common case of local data flow and taint: from one `Expr` to another. +* Data is now tracked through null-coalescing expressions (`??`). ## Changes to autobuilder From b9ba534bcbe071405a94c0b192c3b1aa3472beb5 Mon Sep 17 00:00:00 2001 From: Calum Grant Date: Thu, 24 Oct 2019 11:06:34 +0100 Subject: [PATCH 031/148] C#: Update qltest output. --- csharp/ql/test/library-tests/cil/dataflow/DataFlow.expected | 2 ++ 1 file changed, 2 insertions(+) diff --git a/csharp/ql/test/library-tests/cil/dataflow/DataFlow.expected b/csharp/ql/test/library-tests/cil/dataflow/DataFlow.expected index 65e04f8d48c..d82ff134b35 100644 --- a/csharp/ql/test/library-tests/cil/dataflow/DataFlow.expected +++ b/csharp/ql/test/library-tests/cil/dataflow/DataFlow.expected @@ -12,6 +12,8 @@ | dataflow.cs:46:35:46:39 | "t1b" | dataflow.cs:46:18:46:40 | call to method Taint1 | | dataflow.cs:49:35:49:38 | "t6" | dataflow.cs:49:18:49:45 | call to method TaintIndirect | | dataflow.cs:49:41:49:44 | "t6" | dataflow.cs:49:18:49:45 | call to method TaintIndirect | +| dataflow.cs:102:30:102:33 | null | dataflow.cs:74:21:74:52 | ... ?? ... | | dataflow.cs:102:30:102:33 | null | dataflow.cs:89:24:89:51 | ... ? ... : ... | | dataflow.cs:102:30:102:33 | null | dataflow.cs:108:20:108:33 | call to method IndirectNull | +| dataflow.cs:109:23:109:26 | null | dataflow.cs:74:21:74:52 | ... ?? ... | | dataflow.cs:109:23:109:26 | null | dataflow.cs:89:24:89:51 | ... ? ... : ... | From 8e31b8167afe6c19c3381ea439b959286ce10a11 Mon Sep 17 00:00:00 2001 From: Jonas Jensen Date: Thu, 24 Oct 2019 14:46:10 +0200 Subject: [PATCH 032/148] C++: Add a sample class in PrintAST.ql I've found myself typing out this class whenever I want to print the AST of one function. I hope it will be useful to others too. --- cpp/ql/src/semmle/code/cpp/PrintAST.ql | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cpp/ql/src/semmle/code/cpp/PrintAST.ql b/cpp/ql/src/semmle/code/cpp/PrintAST.ql index 6fc40dd0525..23f249585b9 100644 --- a/cpp/ql/src/semmle/code/cpp/PrintAST.ql +++ b/cpp/ql/src/semmle/code/cpp/PrintAST.ql @@ -7,3 +7,12 @@ import cpp import PrintAST + +/** + * Temporarily tweak this class or make a copy to control which functions are + * printed. + */ +class Cfg extends PrintASTConfiguration { + /** Holds if the AST for `func` should be printed. */ + override predicate shouldPrintFunction(Function func) { any() } +} From 5c07750286a2bfa2c4bdffc35f4267e488d1f7f6 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Thu, 24 Oct 2019 11:28:42 +0200 Subject: [PATCH 033/148] simplify the heuristic for Deferred promises --- .../ql/src/semmle/javascript/Promises.qll | 48 ++++--------------- 1 file changed, 8 insertions(+), 40 deletions(-) diff --git a/javascript/ql/src/semmle/javascript/Promises.qll b/javascript/ql/src/semmle/javascript/Promises.qll index 13eb1e8114e..ae8554c1a0a 100644 --- a/javascript/ql/src/semmle/javascript/Promises.qll +++ b/javascript/ql/src/semmle/javascript/Promises.qll @@ -36,35 +36,9 @@ module Bluebird { * Provides classes for working with various Deferred implementations */ module Deferred { - class DeferredClass extends DataFlow::SourceNode { - DeferredClass() { - exists(Variable var | - var.getName() = "Deferred" and - ( - var.getADeclaration() instanceof LocalNamespaceDecl or - var.getScope() instanceof GlobalScope - ) and - this = DataFlow::valueNode(var.getADefinition()) - ) - or - this.(DataFlow::ParameterNode).getName() = "Deferred" - or - exists(Function f | - f.getName() = "Deferred" and - this = DataFlow::valueNode(f) - ) - or - exists(ClassDefinition c | - c.getName() = "Deferred" and - this = DataFlow::valueNode(c) - ) - } - } - class DeferredInstance extends DataFlow::NewNode { - DeferredClass deferredClass; - - DeferredInstance() { this = deferredClass.getAnInstantiation() } + // Describes both `new Deferred()`, `new $.Deferred` and other variants. + DeferredInstance() { this.getCalleeName() = "Deferred" } private DataFlow::SourceNode ref(DataFlow::TypeTracker t) { t.start() and @@ -72,23 +46,17 @@ module Deferred { or exists(DataFlow::TypeTracker t2 | result = ref(t2).track(t2, t)) } - - DeferredClass getDeferredClass() { result = deferredClass } - - DataFlow::CallNode getPromiseMemberCall(string methodName) { - result = ref(DataFlow::TypeTracker::end()).getAMemberCall(methodName) - } + + DataFlow::SourceNode ref() { result = ref(DataFlow::TypeTracker::end()) } } /** * A promise object created by a Deferred constructor */ private class DeferredPromiseDefinition extends PromiseDefinition, DeferredInstance { - DeferredPromiseDefinition() { - this = any(DeferredClass c | - exists(any(DeferredInstance i | i.getDeferredClass() = c).getPromiseMemberCall("resolve")) and - exists(any(DeferredInstance i | i.getDeferredClass() = c).getPromiseMemberCall("reject")) - ).getAnInstantiation() + DeferredPromiseDefinition() { + // hardening of the "Deferred" heuristic: a method call to `resolve`. + exists(ref().getAMethodCall("resolve")) } override DataFlow::FunctionNode getExecutor() { result = getCallback(0) } @@ -99,7 +67,7 @@ module Deferred { */ class ResolvedDeferredPromiseDefinition extends ResolvedPromiseDefinition { ResolvedDeferredPromiseDefinition() { - this = any(DeferredPromiseDefinition def).getPromiseMemberCall("resolve") + this = any(DeferredPromiseDefinition def).ref().getAMethodCall("resolve") } override DataFlow::Node getValue() { result = getArgument(0) } From f9e76b27f5def0fbe4f210e0b548d7236bf4f5db Mon Sep 17 00:00:00 2001 From: Shati Patel Date: Thu, 24 Oct 2019 14:26:51 +0100 Subject: [PATCH 034/148] Docs: Update C# --- docs/language/learn-ql/csharp/dataflow.rst | 8 ++--- .../csharp/introduce-libraries-csharp.rst | 34 +++++++++---------- .../learn-ql/csharp/ql-for-csharp.rst | 18 +++++----- 3 files changed, 29 insertions(+), 31 deletions(-) diff --git a/docs/language/learn-ql/csharp/dataflow.rst b/docs/language/learn-ql/csharp/dataflow.rst index e9d8a63be76..3b9bb69d7da 100644 --- a/docs/language/learn-ql/csharp/dataflow.rst +++ b/docs/language/learn-ql/csharp/dataflow.rst @@ -4,10 +4,10 @@ Analyzing data flow in C# Overview -------- -This topic describes how data flow analysis is implemented in the QL for C# library and includes examples to help you write your own data flow queries. -The following sections describe how to utilize the QL libraries for local data flow, global data flow and taint tracking. +This topic describes how data flow analysis is implemented in the CodeQL libraries for C# and includes examples to help you write your own data flow queries. +The following sections describe how to utilize the libraries for local data flow, global data flow, and taint tracking. -For a more general introduction to modeling data flow in QL, see :doc:`Introduction to data flow analysis in QL <../intro-to-data-flow>`. +For a more general introduction to modeling data flow, see :doc:`Introduction to data flow analysis with CodeQL <../intro-to-data-flow>`. Local data flow --------------- @@ -548,6 +548,6 @@ This can be adapted from the ``SystemUriFlow`` class: What next? ---------- -- Learn about the QL standard libraries used to write queries for C# in :doc:`Introducing the C# libraries `. +- Learn about the standard libraries used to write queries for C# in :doc:`Introducing the C# libraries `. - Find out more about QL in the `QL language handbook `__ and `QL language specification `__. - Learn more about the query console in `Using the query console `__. diff --git a/docs/language/learn-ql/csharp/introduce-libraries-csharp.rst b/docs/language/learn-ql/csharp/introduce-libraries-csharp.rst index 43e6c11327c..32a782d5034 100644 --- a/docs/language/learn-ql/csharp/introduce-libraries-csharp.rst +++ b/docs/language/learn-ql/csharp/introduce-libraries-csharp.rst @@ -1,25 +1,23 @@ -Introducing the C# libraries -============================ +Introducing the CodeQL libraries for C# +======================================= Overview -------- -The C# QL libraries are a data model for analysis of C# code. QL is an object-oriented language, so the data model is represented as *QL classes*, which are organized into *QL libraries*. The QL classes are a layer of logic built on top of an underlying database. - -The core library is imported at the top of each query using: +There is an extensive library for analyzing CodeQL databases extracted from C# projects. The classes in this library present the data from a database in an object-oriented form and provide abstractions and predicates to help you with common analysis tasks. The library is implemented as a set of QL modules, that is, files with the extension ``.qll``. The module ``csharp.qll`` imports all the core C# library modules, so you can include the complete library by beginning your query with: .. code-block:: ql import csharp -Since this is required for all C# queries, it is omitted from QL snippets below. +Since this is required for all C# queries, it is omitted from code snippets below. -The core library contains all the program elements, including `files <#files>`__, `types <#types>`__, methods, `variables <#variables>`__, `statements <#statements>`__, and `expressions <#expressions>`__. This is sufficient for most queries, however additional libraries can be imported for bespoke functionality such as control flow and data flow. See :doc:`QL for C# ` for information about these additional libraries. +The core library contains all the program elements, including `files <#files>`__, `types <#types>`__, methods, `variables <#variables>`__, `statements <#statements>`__, and `expressions <#expressions>`__. This is sufficient for most queries, however additional libraries can be imported for bespoke functionality such as control flow and data flow. See :doc:`CodeQL for C# ` for information about these additional libraries. Class hierarchies ~~~~~~~~~~~~~~~~~ -Each section contains a QL class hierarchy, showing the inheritance structure between QL classes. For example: +Each section contains a class hierarchy, showing the inheritance structure between CodeQL classes. For example: - ``Expr`` @@ -46,13 +44,13 @@ Each section contains a QL class hierarchy, showing the inheritance structure be This means that the class ``AddExpr`` extends class ``BinaryArithmeticOperation``, which in turn extends class ``ArithmeticOperation`` and so on. If you want to query any arithmetic operation, then use the class ``ArithmeticOperation``, but if you specifically want to limit the query to addition operations, then use the class ``AddExpr``. -QL classes can also be considered to be *sets*, and the ``extends`` relation between classes defines a subset. Every member of class ``AddExpr`` is also in the class ``BinaryArithmeticOperation``. In general, QL classes overlap and an entity can be a member of several classes. +Classes can also be considered to be *sets*, and the ``extends`` relation between classes defines a subset. Every member of class ``AddExpr`` is also in the class ``BinaryArithmeticOperation``. In general, classes overlap and an entity can be a member of several classes. This overview omits some of the less important or intermediate classes from the class hierarchy. Each class has predicates, which are logical propositions about that class. They also define navigable relationships between classes. Predicates are inherited, so for example the ``AddExpr`` class inherits the predicates ``getLeftOperand()`` and ``getRightOperand()`` from ``BinaryArithmeticOperation``, and ``getType()`` from class ``Expr``. This is similar to how methods are inherited in object-oriented programming languages. -In this overview, we present the most common and useful predicates. Consult the reference, QL source code, and autocomplete in the editor for the complete list of predicates available on each class. +In this overview, we present the most common and useful predicates. Consult the `reference `__, the CodeQL source code, and autocomplete in the editor for the complete list of predicates available on each class. Exercises ~~~~~~~~~ @@ -72,7 +70,7 @@ Exercise 1: Simplify the following query: Files ----- -Files are represented by the QL class `File `__, and directories by the QL class `Folder `__. The database contains all of the source files and assemblies used during the compilation. +Files are represented by the class `File `__, and directories by the class `Folder `__. The database contains all of the source files and assemblies used during the compilation. Class hierarchy ~~~~~~~~~~~~~~~ @@ -143,7 +141,7 @@ To list all elements in ``Main.cs``, their QL class and location: where e.getFile().getShortName() = "Main" select e, e.getAQlClass(), e.getLocation() -Note that ``getAQlClass()`` is available on all QL classes and is a useful way to figure out the QL class of something. Often the same element will have several QL classes which are all returned by ``getAQlClass()``. +Note that ``getAQlClass()`` is available on all entities and is a useful way to figure out the QL class of something. Often the same element will have several classes which are all returned by ``getAQlClass()``. Locations --------- @@ -234,7 +232,7 @@ Find declarations containing a username: Variables --------- -The QL class `Variable `__ represents C# variables, such as fields, parameters and local variables. The database contains all variables from the source code, as well as all fields and parameters from assemblies referenced by the program. +The class `Variable `__ represents C# variables, such as fields, parameters and local variables. The database contains all variables from the source code, as well as all fields and parameters from assemblies referenced by the program. Class hierarchy ~~~~~~~~~~~~~~~ @@ -283,7 +281,7 @@ Find all unused local variables: Types ----- -Types are represented by the QL class `Type `__ and consist of builtin types, interfaces, classes, structs, enums, and type parameters. The database contains types from the program and all referenced assemblies including mscorlib and the .NET framework. +Types are represented by the CodeQL class `Type `__ and consist of builtin types, interfaces, classes, structs, enums, and type parameters. The database contains types from the program and all referenced assemblies including mscorlib and the .NET framework. The builtin types (``object``, ``int``, ``double`` etc.) have corresponding types (``System.Object``, ``System.Int32`` etc.) in mscorlib. @@ -438,7 +436,7 @@ Exercise 5: Write a query to find all classes starting with the letter ``A``. (` Callables --------- -Callables are represented by the QL class `Callable `__ and are anything that can be called independently, such as methods, constructors, destructors, operators, anonymous functions, indexers, and property accessors. +Callables are represented by the class `Callable `__ and are anything that can be called independently, such as methods, constructors, destructors, operators, anonymous functions, indexers, and property accessors. The database contains all of the callables in your program and in all referenced assemblies. @@ -564,7 +562,7 @@ Find ``Main`` methods which are not ``private``: Statements ---------- -Statements are represented by the QL class `Stmt `__ and make up the body of methods (and other callables). The database contains all statements in the source code, but does not contain any statements from referenced assemblies where the source code is not available. +Statements are represented by the class `Stmt `__ and make up the body of methods (and other callables). The database contains all statements in the source code, but does not contain any statements from referenced assemblies where the source code is not available. Class hierarchy ~~~~~~~~~~~~~~~ @@ -922,7 +920,7 @@ Exercise 9: Limit the previous query to string types. Exclude empty passwords or Attributes ---------- -C# attributes are represented by the QL class `Attribute `__. They can be present on many C# elements, such as classes, methods, fields, and parameters. The database contains attributes from the source code and all assembly references. +C# attributes are represented by the class `Attribute `__. They can be present on many C# elements, such as classes, methods, fields, and parameters. The database contains attributes from the source code and all assembly references. The attribute of any ``Element`` can be obtained via ``getAnAttribute()``, whereas if you have an attribute, you can find its element via ``getTarget()``. The following two query fragments are identical: @@ -1122,6 +1120,6 @@ Here is the fixed version: What next? ---------- -- Visit :doc:`Tutorial: Analyzing data flow in C# ` to learn more about writing queries using the standard QL for C# data flow and taint tracking libraries. +- Visit :doc:`Tutorial: Analyzing data flow in C# ` to learn more about writing queries using the standard data flow and taint tracking libraries. - Find out more about QL in the `QL language handbook `__ and `QL language specification `__. - Learn more about the query console in `Using the query console `__. \ No newline at end of file diff --git a/docs/language/learn-ql/csharp/ql-for-csharp.rst b/docs/language/learn-ql/csharp/ql-for-csharp.rst index 56d8f7ef017..013464ed036 100644 --- a/docs/language/learn-ql/csharp/ql-for-csharp.rst +++ b/docs/language/learn-ql/csharp/ql-for-csharp.rst @@ -1,5 +1,5 @@ -QL for C# -========= +CodeQL for C# +============= .. toctree:: :glob: @@ -8,17 +8,17 @@ QL for C# introduce-libraries-csharp dataflow -These topics provide an overview of the QL C# libraries and show examples of how to use them. +These topics provide an overview of the CodeQL libraries for C# and show examples of how to use them. -- `Basic C# QL query `__ describes how to write and run queries using LGTM. +- `Basic C# query `__ describes how to write and run queries using LGTM. -- :doc:`Introducing the C# libraries ` introduces the standard libraries used to write queries for C# code. +- :doc:`Introducing the CodeQL libraries for C# ` introduces the standard libraries used to write queries for C# code. .. raw:: html -- :doc:`Tutorial: Analyzing data flow in C# ` demonstrates how to write queries using the standard QL for C# data flow and taint tracking libraries. +- :doc:`Tutorial: Analyzing data flow in C# ` demonstrates how to write queries using the standard data flow and taint tracking libraries for C#. .. raw:: html @@ -35,6 +35,6 @@ These topics provide an overview of the QL C# libraries and show examples of how Other resources --------------- -- For examples of how to query common C# elements, see the `C# QL cookbook `__. -- For the queries used in LGTM, display a `C# query `__ and click **Open in query console** to see the QL code used to find alerts. -- For more information about the C/C++ QL library see the `QL library for C# `__. +- For examples of how to query common C# elements, see the `C# cookbook `__. +- For the queries used in LGTM, display a `C# query `__ and click **Open in query console** to see the code used to find alerts. +- For more information about the library for C# see the `CodeQL library for C# `__. From fbc11e505f6d779444fae33ce0abd12ffae79dc9 Mon Sep 17 00:00:00 2001 From: Shati Patel Date: Thu, 24 Oct 2019 14:27:27 +0100 Subject: [PATCH 035/148] Docs: Update Go --- docs/language/learn-ql/go/ql-for-go.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/language/learn-ql/go/ql-for-go.rst b/docs/language/learn-ql/go/ql-for-go.rst index 47080ebfe89..baa390dc7c6 100644 --- a/docs/language/learn-ql/go/ql-for-go.rst +++ b/docs/language/learn-ql/go/ql-for-go.rst @@ -1,7 +1,7 @@ -QL for Go -========= +CodeQL for Go +============= -This page provides an overview of the QL for Go documentation that is currently available. +This page provides an overview of the CodeQL for Go documentation that is currently available. - `Basic Go query `__ describes how to write and run queries using LGTM. @@ -9,5 +9,5 @@ This page provides an overview of the QL for Go documentation that is currently Other resources --------------- -- For the queries used in LGTM, display a `Go query `__ and click **Open in query console** to see the QL code used to find alerts. -- For more information about the Go QL library see the `QL library for Go `__. \ No newline at end of file +- For the queries used in LGTM, display a `Go query `__ and click **Open in query console** to see the code used to find alerts. +- For more information about the library for Go see the `CodeQL library for Go `__. \ No newline at end of file From 6090867542f945fd18f2fd18dd25b6508c2d681f Mon Sep 17 00:00:00 2001 From: Shati Patel Date: Thu, 24 Oct 2019 14:58:10 +0100 Subject: [PATCH 036/148] Docs: Update Java --- docs/language/learn-ql/java/annotations.rst | 18 +++++------ .../learn-ql/java/ast-class-reference.rst | 14 ++++---- docs/language/learn-ql/java/call-graph.rst | 6 ++-- docs/language/learn-ql/java/dataflow.rst | 6 ++-- .../learn-ql/java/expressions-statements.rst | 12 +++---- .../java/introduce-libraries-java.rst | 22 ++++++------- docs/language/learn-ql/java/javadoc.rst | 8 ++--- docs/language/learn-ql/java/ql-for-java.rst | 32 +++++++++---------- .../learn-ql/java/source-locations.rst | 10 +++--- .../learn-ql/java/types-class-hierarchy.rst | 24 +++++++------- 10 files changed, 76 insertions(+), 76 deletions(-) diff --git a/docs/language/learn-ql/java/annotations.rst b/docs/language/learn-ql/java/annotations.rst index 2808633dcf1..9fbb70776c4 100644 --- a/docs/language/learn-ql/java/annotations.rst +++ b/docs/language/learn-ql/java/annotations.rst @@ -4,9 +4,9 @@ Tutorial: Annotations Overview -------- -Snapshots of Java projects contain information about all annotations attached to program elements. +CodeQL databases of Java projects contain information about all annotations attached to program elements. -In QL, annotations are represented as follows: +Annotations are represented by the following CodeQL classes: - The class ``Annotatable`` represents all entities that may have an annotation attached to them (that is, packages, reference types, fields, methods, and local variables). - The class ``AnnotationType`` represents a Java annotation type, such as ``java.lang.Override``; annotation types are interfaces. @@ -23,7 +23,7 @@ As an example, recall that the Java standard library defines an annotation ``Sup String[] value; } -In QL, ``SuppressWarnings`` is represented as an ``AnnotationType``, with ``value`` as its only ``AnnotationElement``. +``SuppressWarnings`` is represented as an ``AnnotationType``, with ``value`` as its only ``AnnotationElement``. A typical usage of ``SuppressWarnings`` would be the following annotation to prevent a warning about using raw types: @@ -35,7 +35,7 @@ A typical usage of ``SuppressWarnings`` would be the following annotation to pre } } -In QL, the expression ``@SuppressWarnings("rawtypes")`` is represented as an ``Annotation``. The string literal ``"rawtypes"`` is used to initialize the annotation element ``value``, and its value can be extracted from the annotation by means of the ``getValue`` predicate. +The expression ``@SuppressWarnings("rawtypes")`` is represented as an ``Annotation``. The string literal ``"rawtypes"`` is used to initialize the annotation element ``value``, and its value can be extracted from the annotation by means of the ``getValue`` predicate. We could then write the following query to find all ``@SuppressWarnings`` annotations attached to constructors, and return both the annotation itself and the value of its ``value`` element: @@ -101,7 +101,7 @@ As a first step, let us write a query that finds all ``@Override`` annotations. where ann.getType().hasQualifiedName("java.lang", "Override") select ann -As always, it is a good idea to try this query on a Java snapshot to make sure it actually produces some results. On the earlier example, it should find the annotation on ``Sub1.m``. Next, we encapsulate the concept of an ``@Override`` annotation as a QL class: +As always, it is a good idea to try this query on a CodeQL database for a Java project to make sure it actually produces some results. On the earlier example, it should find the annotation on ``Sub1.m``. Next, we encapsulate the concept of an ``@Override`` annotation as a CodeQL class: :: @@ -147,7 +147,7 @@ For example, consider the following example program: Here, both ``A.m`` and ``A.n`` are marked as deprecated. Methods ``n`` and ``r`` both call ``m``, but note that ``n`` itself is deprecated, so we probably should not warn about this call. -Like in the previous example, we start by defining a QL class for representing ``@Deprecated`` annotations: +Like in the previous example, we start by defining a class for representing ``@Deprecated`` annotations: .. code-block:: ql @@ -204,7 +204,7 @@ For instance, consider this slightly updated example: Here, the programmer has explicitly suppressed warnings about deprecated calls in ``A.r``, so our query should not flag the call to ``A.m`` any more. -To do so, we first introduce a QL class for representing all ``@SuppressWarnings`` annotations where the string ``deprecated`` occurs among the list of warnings to suppress: +To do so, we first introduce a class for representing all ``@SuppressWarnings`` annotations where the string ``deprecated`` occurs among the list of warnings to suppress: .. code-block:: ql @@ -238,6 +238,6 @@ Now we can extend our query to filter out calls in methods carrying a ``Suppress What next? ---------- -- Take a look at some of the other tutorials: :doc:`Tutorial: Javadoc `, :doc:`Tutorial: Working with source locations `. -- Find out how specific classes in the AST are represented in the QL standard library for Java: :doc:`AST class reference `. +- Take a look at some of the other tutorials: :doc:`Tutorial: Javadoc ` and :doc:`Tutorial: Working with source locations `. +- Find out how specific classes in the AST are represented in the standard library for Java: :doc:`AST class reference `. - Find out more about QL in the `QL language handbook `__ and `QL language specification `__. diff --git a/docs/language/learn-ql/java/ast-class-reference.rst b/docs/language/learn-ql/java/ast-class-reference.rst index e9fcc4b1d6a..268b95acac5 100644 --- a/docs/language/learn-ql/java/ast-class-reference.rst +++ b/docs/language/learn-ql/java/ast-class-reference.rst @@ -15,7 +15,7 @@ Statement classes This table lists all subclasses of `Stmt`_. +------------------------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------+---------------------------------------------+ -| Statement syntax | QL class | Superclasses | Remarks | +| Statement syntax | CodeQL class | Superclasses | Remarks | +========================================================================+===========================================================================================================================================================+===================================+=============================================+ | ``;`` | `EmptyStmt `__ | | | +------------------------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------+---------------------------------------------+ @@ -87,7 +87,7 @@ Literals All classes in this subsection are subclasses of `Literal `__. +---------------------------+--------------------------+ -| Expression syntax example | QL class | +| Expression syntax example | CodeQL class | +===========================+==========================+ | ``true`` | ``BooleanLiteral`` | +---------------------------+--------------------------+ @@ -112,7 +112,7 @@ Unary expressions All classes in this subsection are subclasses of `UnaryExpr `__. +---------------------------+-----------------+---------------------+---------------------------------------------------+ -| Expression syntax example | QL class | Superclasses | Remarks | +| Expression syntax example | CodeQL class | Superclasses | Remarks | +===========================+=================+=====================+===================================================+ | ``x++`` | ``PostIncExpr`` | ``UnaryAssignExpr`` | | +---------------------------+-----------------+---------------------+---------------------------------------------------+ @@ -137,7 +137,7 @@ Binary expressions All classes in this subsection are subclasses of `BinaryExpr `__. +---------------------------+--------------------+--------------------+ -| Expression syntax example | QL class | Superclasses | +| Expression syntax example | CodeQL class | Superclasses | +===========================+====================+====================+ | ``x * y`` | ``MulExpr`` | | +---------------------------+--------------------+--------------------+ @@ -184,7 +184,7 @@ Assignment expressions All classes in this table are subclasses of `Assignment `__. +---------------------------+-----------------------+--------------+ -| Expression syntax example | QL class | Superclasses | +| Expression syntax example | CodeQL class | Superclasses | +===========================+=======================+==============+ | ``x = y`` | ``AssignExpr`` | | +---------------------------+-----------------------+--------------+ @@ -215,7 +215,7 @@ Accesses ~~~~~~~~ +--------------------------------------+-------------------------------------------------------------------------------------------------------------------------+ -| Expression syntax examples | QL class | +| Expression syntax examples | CodeQL class | +======================================+=========================================================================================================================+ | ``this`` | `ThisAccess `__ | +--------------------------------------+ + @@ -250,7 +250,7 @@ Miscellaneous ~~~~~~~~~~~~~ +------------------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------------------------+ -| Expression syntax examples | QL class | Remarks | +| Expression syntax examples | CodeQL class | Remarks | +==================================================================+=======================================================================================================================+=============================================================================+ | ``(int) f`` | `CastExpr `__ | | +------------------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------------------------+ diff --git a/docs/language/learn-ql/java/call-graph.rst b/docs/language/learn-ql/java/call-graph.rst index 4be0153fca4..bb24c2d8550 100644 --- a/docs/language/learn-ql/java/call-graph.rst +++ b/docs/language/learn-ql/java/call-graph.rst @@ -4,7 +4,7 @@ Tutorial: Navigating the call graph Call graph API -------------- -The QL Java library provides two abstract classes for representing a program's call graph: ``Callable`` and ``Call``. The former is simply the common superclass of ``Method`` and ``Constructor``, the latter is a common superclass of ``MethodAccess``, ``ClassInstanceExpression``, ``ThisConstructorInvocationStmt`` and ``SuperConstructorInvocationStmt``. Simply put, a ``Callable`` is something that can be invoked, and a ``Call`` is something that invokes a ``Callable``. +The CodeQL library for Java provides two abstract classes for representing a program's call graph: ``Callable`` and ``Call``. The former is simply the common superclass of ``Method`` and ``Constructor``, the latter is a common superclass of ``MethodAccess``, ``ClassInstanceExpression``, ``ThisConstructorInvocationStmt`` and ``SuperConstructorInvocationStmt``. Simply put, a ``Callable`` is something that can be invoked, and a ``Call`` is something that invokes a ``Callable``. For example, in the following program all callables and calls have been annotated with comments: @@ -160,6 +160,6 @@ Finally, on many Java projects there are methods that are invoked indirectly by What next? ---------- -- Find out how to query metadata and white space: :doc:`Tutorial: Annotations `, :doc:`Tutorial: Javadoc `, :doc:`Tutorial: Working with source locations `. -- Find out how specific classes in the AST are represented in the QL standard library for Java: :doc:`AST class reference `. +- Find out how to query metadata and white space: :doc:`Tutorial: Annotations `, :doc:`Tutorial: Javadoc `, and :doc:`Tutorial: Working with source locations `. +- Find out how specific classes in the AST are represented in the standard library for Java: :doc:`AST class reference `. - Find out more about QL in the `QL language handbook `__ and `QL language specification `__. diff --git a/docs/language/learn-ql/java/dataflow.rst b/docs/language/learn-ql/java/dataflow.rst index 39a2b6d5f50..974579e1bcc 100644 --- a/docs/language/learn-ql/java/dataflow.rst +++ b/docs/language/learn-ql/java/dataflow.rst @@ -4,10 +4,10 @@ Analyzing data flow in Java Overview -------- -This topic describes how data flow analysis is implemented in the QL for Java library and includes examples to help you write your own data flow queries. -The following sections describe how to utilize the QL libraries for local data flow, global data flow and taint tracking. +This topic describes how data flow analysis is implemented in the CodeQL libraries for Java and includes examples to help you write your own data flow queries. +The following sections describe how to utilize the libraries for local data flow, global data flow, and taint tracking. -For a more general introduction to modeling data flow in QL, see :doc:`Introduction to data flow analysis in QL <../intro-to-data-flow>`. +For a more general introduction to modeling data flow, see :doc:`Introduction to data flow analysis with CodeQL <../intro-to-data-flow>`. Local data flow --------------- diff --git a/docs/language/learn-ql/java/expressions-statements.rst b/docs/language/learn-ql/java/expressions-statements.rst index 0dfaa9f18ed..b08f2cf0b8b 100644 --- a/docs/language/learn-ql/java/expressions-statements.rst +++ b/docs/language/learn-ql/java/expressions-statements.rst @@ -22,12 +22,12 @@ If ``l`` is bigger than 2\ :sup:`31`\ - 1 (the largest positive value of type `` All primitive numeric types have a maximum value, beyond which they will wrap around to their lowest possible value (called an "overflow"). For ``int``, this maximum value is 2\ :sup:`31`\ - 1. Type ``long`` can accommodate larger values up to a maximum of 2\ :sup:`63`\ - 1. In this example, this means that ``l`` can take on a value that is higher than the maximum for type ``int``; ``i`` will never be able to reach this value, instead overflowing and returning to a low value. -We will develop a query that finds code that looks like it might exhibit this kind of behavior. We will be using several of the standard QL library classes for representing statements and functions, a full list of which can be found in the :doc:`AST class reference `. +We will develop a query that finds code that looks like it might exhibit this kind of behavior. We will be using several of the standard library classes for representing statements and functions, a full list of which can be found in the :doc:`AST class reference `. Initial query ------------- -We start out by writing a query that finds less-than expressions (QL class ``LTExpr``) where the left operand is of type ``int`` and the right operand is of type ``long``: +We start out by writing a query that finds less-than expressions (CodeQL class ``LTExpr``) where the left operand is of type ``int`` and the right operand is of type ``long``: .. code-block:: ql @@ -57,7 +57,7 @@ Notice that we use the predicate ``getType`` (available on all subclasses of ``E The class ``LoopStmt`` is a common superclass of all loops, including, in particular, ``for`` loops as in our example above. While different kinds of loops have different syntax, they all have a loop condition, which can be accessed through predicate ``getCondition``. We use the reflexive transitive closure operator ``*`` applied to the ``getAChildExpr`` predicate to express the requirement that ``expr`` should be nested inside the loop condition. In particular, it can be the loop condition itself. -The final conjunct in the ``where`` clause takes advantage of the fact that QL predicates can return more than one value (they are really relations). In particular, ``getAnOperand`` may return *either* operand of ``expr``, so ``expr.getAnOperand().isCompileTimeConstant()`` holds if at least one of the operands is constant. Negating this condition means that the query will only find expressions where *neither* of the operands is constant. +The final conjunct in the ``where`` clause takes advantage of the fact that `predicates `__ can return more than one value (they are really relations). In particular, ``getAnOperand`` may return *either* operand of ``expr``, so ``expr.getAnOperand().isCompileTimeConstant()`` holds if at least one of the operands is constant. Negating this condition means that the query will only find expressions where *neither* of the operands is constant. Generalizing the query ---------------------- @@ -76,7 +76,7 @@ In order to compare the ranges of types, we define a predicate that returns the (pt.hasName("long") and result=64) } -We now want to generalize our query to apply to any comparison where the width of the type on the smaller end of the comparison is less than the width of the type on the greater end. Let us call such a comparison *overflow prone*, and introduce an abstract QL class to model it: +We now want to generalize our query to apply to any comparison where the width of the type on the smaller end of the comparison is less than the width of the type on the greater end. Let us call such a comparison *overflow prone*, and introduce an abstract class to model it: .. code-block:: ql @@ -121,6 +121,6 @@ Now we rewrite our query to make use of these new classes: What next? ---------- -- Have a look at some of the other tutorials: :doc:`Tutorial: Types and the class hierarchy `, :doc:`Tutorial: Navigating the call graph `, :doc:`Tutorial: Annotations `, :doc:`Tutorial: Javadoc `, :doc:`Tutorial: Working with source locations `. -- Find out how specific classes in the AST are represented in the QL standard library for Java: :doc:`AST class reference `. +- Have a look at some of the other tutorials: :doc:`Tutorial: Types and the class hierarchy `, :doc:`Tutorial: Navigating the call graph `, :doc:`Tutorial: Annotations `, :doc:`Tutorial: Javadoc `, and :doc:`Tutorial: Working with source locations `. +- Find out how specific classes in the AST are represented in the standard library for Java: :doc:`AST class reference `. - Find out more about QL in the `QL language handbook `__ and `QL language specification `__. diff --git a/docs/language/learn-ql/java/introduce-libraries-java.rst b/docs/language/learn-ql/java/introduce-libraries-java.rst index 9f37957e5c6..377823c6f70 100644 --- a/docs/language/learn-ql/java/introduce-libraries-java.rst +++ b/docs/language/learn-ql/java/introduce-libraries-java.rst @@ -1,25 +1,25 @@ -Introducing the QL libraries for Java -===================================== +Introducing the CodeQL libraries for Java +========================================= Overview -------- -There is an extensive QL library for analyzing Java code. The QL classes in this library present the data from a snapshot database in an object-oriented form and provide abstractions and predicates to help you with common analysis tasks. +There is an extensive library for analyzing CodeQL databases extracted from Java projects. The classes in this library present the data from a database in an object-oriented form and provide abstractions and predicates to help you with common analysis tasks. -The library is implemented as a set of QL modules, that is, files with the extension ``.qll``. The module ``java.qll`` imports all other standard library modules, so you can include the complete library by beginning your query with: +The library is implemented as a set of QL modules, that is, files with the extension ``.qll``. The module ``java.qll`` imports all the core C/C++ library modules, so you can include the complete library by beginning your query with: .. code-block:: ql import java -The rest of this topic briefly summarizes the most important QL classes and predicates provided by this library. +The rest of this topic briefly summarizes the most important classes and predicates provided by this library. The example queries in this topic illustrate the types of results returned by different library classes. The results themselves are not interesting but can be used as the basis for developing a more complex query. The tutorial topics show how you can take a simple query and fine-tune it to find precisely the results you're interested in. Summary of the library classes ------------------------------ -The most important classes in the standard QL Java library can be grouped into five main categories: +The most important classes in the standard Java library can be grouped into five main categories: #. Classes for representing program elements (such as classes and methods) #. Classes for representing AST nodes (such as statements and expressions) @@ -89,7 +89,7 @@ Several more specialized classes are available as well: - A ``LocalClass``, which is `a class declared inside a method or constructor `__. - An ``AnonymousClass``, which is an `anonymous class `__. -Finally, the QL library also has a number of singleton classes that wrap frequently used Java standard library classes: ``TypeObject``, ``TypeCloneable``, ``TypeRuntime``, ``TypeSerializable``, ``TypeString``, ``TypeSystem`` and ``TypeClass``. Each QL class represents the standard Java class suggested by its name. +Finally, the library also has a number of singleton classes that wrap frequently used Java standard library classes: ``TypeObject``, ``TypeCloneable``, ``TypeRuntime``, ``TypeSerializable``, ``TypeString``, ``TypeSystem`` and ``TypeClass``. Each CodeQL class represents the standard Java class suggested by its name. As an example, we can write a query that finds all nested classes that directly extend ``Object``: @@ -160,7 +160,7 @@ As an example, the following query finds all type variables with type bound ``Nu ➤ `See this in the query console `__. When we ran it on the LGTM.com demo projects, the *neo4j/neo4j*, *gradle/gradle* and *hibernate/hibernate-orm* projects all contained examples of this pattern. -For dealing with legacy code that is unaware of generics, every generic type has a "raw" version without any type parameters. In QL, raw types are represented using class ``RawType``, which has the expected subclasses ``RawClass`` and ``RawInterface``. Again, there is a predicate ``getSourceDeclaration`` for obtaining the corresponding generic type. As an example, we can find variables of (raw) type ``Map``: +For dealing with legacy code that is unaware of generics, every generic type has a "raw" version without any type parameters. In the CodeQL libraries, raw types are represented using class ``RawType``, which has the expected subclasses ``RawClass`` and ``RawInterface``. Again, there is a predicate ``getSourceDeclaration`` for obtaining the corresponding generic type. As an example, we can find variables of (raw) type ``Map``: .. code-block:: ql @@ -342,7 +342,7 @@ For example, the following query finds methods with a `cyclomatic complexity `, :doc:`Expressions and statements `, :doc:`Navigating the call graph `, :doc:`Annotations `, :doc:`Javadoc ` and :doc:`Working with source locations `. -- Find out how specific classes in the AST are represented in the QL standard library for Java: :doc:`AST class reference `. +- Experiment with the worked examples in the CodeQL for Java tutorial topics: :doc:`Types and the class hierarchy `, :doc:`Expressions and statements `, :doc:`Navigating the call graph `, :doc:`Annotations `, :doc:`Javadoc ` and :doc:`Working with source locations `. +- Find out how specific classes in the AST are represented in the standard library for Java: :doc:`AST class reference `. - Find out more about QL in the `QL language handbook `__ and `QL language specification `__. diff --git a/docs/language/learn-ql/java/javadoc.rst b/docs/language/learn-ql/java/javadoc.rst index 4b76da9c1fa..bd0a11b9132 100644 --- a/docs/language/learn-ql/java/javadoc.rst +++ b/docs/language/learn-ql/java/javadoc.rst @@ -49,7 +49,7 @@ The ``JavadocTag`` has several subclasses representing specific kinds of Javadoc Example: Finding spurious @param tags ------------------------------------- -As an example of using the QL Javadoc API, let us write a query that finds ``@param`` tags that refer to a non-existent parameter. +As an example of using the CodeQL Javadoc API, let us write a query that finds ``@param`` tags that refer to a non-existent parameter. For example, consider the following program: @@ -110,7 +110,7 @@ For example, consider the following Java program: Notice that the Javadoc comment of ``A.foo`` documents two thrown exceptions: ``IOException`` and ``RuntimeException``. The former is clearly spurious: ``A.foo`` does not have a ``throws IOException`` clause, and thus cannot throw this kind of exception. On the other hand, ``RuntimeException`` is an unchecked exception, so it can be thrown even if there is no explicit ``throws`` clause listing it. Therefore, our query should flag the ``@throws`` tag for ``IOException``, but not the one for ``RuntimeException.`` -Recall from above that QL represents ``@throws`` tags using class ``ThrowsTag``. This class does not provide a member predicate for determining the exception type that is being documented, so we first need to implement our own version. A simple version might look as follows: +Recall from above that the CodeQL library represents ``@throws`` tags using class ``ThrowsTag``. This class does not provide a member predicate for determining the exception type that is being documented, so we first need to implement our own version. A simple version might look as follows: .. code-block:: ql @@ -170,7 +170,7 @@ This program defines its own class ``IOException``, which is unrelated to the cl As an example of the second problem, method ``A.foo`` from our previous example was annotated with a ``@throws RuntimeException`` tag. Our current version of ``mayThrow``, however, would think that ``A.foo`` cannot throw a ``RuntimeException``, and thus flag the tag as spurious. -We can make ``mayThrow`` less restrictive by introducing a new QL class to represent unchecked exceptions, which are just the subtypes of ``java.lang.RuntimeException`` and ``java.lang.Error``: +We can make ``mayThrow`` less restrictive by introducing a new class to represent unchecked exceptions, which are just the subtypes of ``java.lang.RuntimeException`` and ``java.lang.Error``: .. code-block:: ql @@ -220,5 +220,5 @@ What next? ---------- - Find out how you can use the location API to define queries on whitespace: :doc:`Tutorial: Working with source locations `. -- Find out how specific classes in the AST are represented in the QL standard library for Java: :doc:`AST class reference `. +- Find out how specific classes in the AST are represented in the standard library for Java: :doc:`AST class reference `. - Find out more about QL in the `QL language handbook `__ and `QL language specification `__. diff --git a/docs/language/learn-ql/java/ql-for-java.rst b/docs/language/learn-ql/java/ql-for-java.rst index f09a5ae990d..aa26c5ba6bb 100644 --- a/docs/language/learn-ql/java/ql-for-java.rst +++ b/docs/language/learn-ql/java/ql-for-java.rst @@ -1,5 +1,5 @@ -QL for Java -=========== +CodeQL for Java +=============== .. toctree:: :glob: @@ -15,31 +15,31 @@ QL for Java source-locations ast-class-reference -These topics provide an overview of the QL Java standard library and show examples of how to use them. +These topics provide an overview of the CodeQL libraries for Java and show examples of how to use them. -- `Basic Java QL query `__ describes how to write and run queries using LGTM. +- `Basic Java query `__ describes how to write and run queries using LGTM. -- :doc:`Introducing the QL libraries for Java ` an introduction to the organization of the standard libraries used to write queries for Java code. +- :doc:`Introducing the CodeQL libraries for Java ` introduces the standard libraries used to write queries for Java code. -- :doc:`Tutorial: Analyzing data flow in Java ` demonstrates how to write queries using the standard QL for Java data flow and taint tracking libraries. +- :doc:`Tutorial: Analyzing data flow in Java ` demonstrates how to write queries using the standard data flow and taint tracking libraries for Java. -- :doc:`Tutorial: Types and the class hierarchy ` introduces the QL classes for representing a program's class hierarchy by means of examples. +- :doc:`Tutorial: Types and the class hierarchy ` introduces the classes for representing a program's class hierarchy by means of examples. -- :doc:`Tutorial: Expressions and statements ` introduces the QL classes for representing a program's syntactic structure by means of examples. +- :doc:`Tutorial: Expressions and statements ` introduces the classes for representing a program's syntactic structure by means of examples. -- :doc:`Tutorial: Navigating the call graph ` a worked example of how to write a query that navigates a program's call graph to find unused methods. +- :doc:`Tutorial: Navigating the call graph ` is a worked example of how to write a query that navigates a program's call graph to find unused methods. -- :doc:`Tutorial: Annotations ` introduces the QL classes for representing annotations by means of examples. +- :doc:`Tutorial: Annotations ` introduces the classes for representing annotations by means of examples. -- :doc:`Tutorial: Javadoc ` introduces the QL classes for representing Javadoc comments by means of examples. +- :doc:`Tutorial: Javadoc ` introduces the classes for representing Javadoc comments by means of examples. -- :doc:`Tutorial: Working with source locations ` a worked example of how to write a query that uses the location information provided in the snapshot for finding likely bugs. +- :doc:`Tutorial: Working with source locations ` is a worked example of how to write a query that uses the location information provided in the database for finding likely bugs. -- :doc:`AST class reference ` an overview of all AST classes in the QL standard library for Java. +- :doc:`AST class reference ` gives an overview of all AST classes in the standard CodeQL library for Java. Other resources --------------- -- For examples of how to query common Java elements, see the `Java QL cookbook `__. -- For the queries used in LGTM, display a `Java query `__ and click **Open in query console** to see the QL code used to find alerts. -- For more information about the Java QL library see the `QL library for Java `__. +- For examples of how to query common Java elements, see the `Java cookbook `__. +- For the queries used in LGTM, display a `Java query `__ and click **Open in query console** to see the code used to find alerts. +- For more information about the library for Java see the `CodeQL library for Java `__. diff --git a/docs/language/learn-ql/java/source-locations.rst b/docs/language/learn-ql/java/source-locations.rst index ebb40916d04..6e29777fab9 100644 --- a/docs/language/learn-ql/java/source-locations.rst +++ b/docs/language/learn-ql/java/source-locations.rst @@ -16,12 +16,12 @@ Note that the source layout gives a fairly clear indication of the intended mean We will now develop a query that finds this kind of suspicious nesting, where the operator of the inner expression has more white space around it than the operator of the outer expression. This pattern may not necessarily indicate a bug, but at the very least it makes the code hard to read and prone to misinterpretation. -White space is not directly represented in the snapshot database, but we can deduce its presence from the location information associated with program elements and AST nodes. So we will start by providing an overview of source location management in the QL standard library. +White space is not directly represented in the CodeQL database, but we can deduce its presence from the location information associated with program elements and AST nodes. So we will start by providing an overview of source location management in the standard library for Java. Location API ------------ -For every entity that has a representation in Java source code (including, in particular, program elements and AST nodes), the QL standard library provides the following predicates for accessing source location information: +For every entity that has a representation in Java source code (including, in particular, program elements and AST nodes), the standard CodeQL library provides the following predicates for accessing source location information: - ``getLocation`` returns a ``Location`` object describing the start and end position of the entity. - ``getFile`` returns a ``File`` object representing the file containing the entity. @@ -123,7 +123,7 @@ If we run this initial query, we might notice some false positives arising from i< start + 100 -Note that our predicate ``operatorWS`` computes the **total** amount of white space around the operator, which, in this case, is one for the ``<`` and two for the ``+``. Ideally, we would like to exclude cases where the amount of white space before and after the operator are not the same. Currently, snapshot databases do not record enough information to figure this out, but as an approximation we could require that the total number of white space characters is even: +Note that our predicate ``operatorWS`` computes the **total** amount of white space around the operator, which, in this case, is one for the ``<`` and two for the ``+``. Ideally, we would like to exclude cases where the amount of white space before and after the operator are not the same. Currently, CodeQL databases do not record enough information to figure this out, but as an approximation we could require that the total number of white space characters is even: .. code-block:: ql @@ -141,7 +141,7 @@ Note that our predicate ``operatorWS`` computes the **total** amount of white sp ➤ `See this in the query console `__. Any results will be refined by our changes to the query. -Another source of false positives are associative operators: in an expression of the form ``x + y+z``, the first plus is syntactically nested inside the second, since + in Java associates to the left; hence the expression is flagged as suspicious. But since + is associative to begin with, it does not matter which way around the operators are nested, so this is a false positive.To exclude these cases, let us define a new QL class identifying binary expressions with an associative operator: +Another source of false positives are associative operators: in an expression of the form ``x + y+z``, the first plus is syntactically nested inside the second, since + in Java associates to the left; hence the expression is flagged as suspicious. But since + is associative to begin with, it does not matter which way around the operators are nested, so this is a false positive.To exclude these cases, let us define a new class identifying binary expressions with an associative operator: .. code-block:: ql @@ -184,5 +184,5 @@ Whitespace suggests that the programmer meant to toggle ``i`` between zero and o What next? ---------- -- Find out how specific classes in the AST are represented in the QL standard library for Java: :doc:`AST class reference `. +- Find out how specific classes in the AST are represented in the standard library for Java: :doc:`AST class reference `. - Find out more about QL in the `QL language handbook `__ and `QL language specification `__. diff --git a/docs/language/learn-ql/java/types-class-hierarchy.rst b/docs/language/learn-ql/java/types-class-hierarchy.rst index cc7e6da4afd..7677c64965d 100644 --- a/docs/language/learn-ql/java/types-class-hierarchy.rst +++ b/docs/language/learn-ql/java/types-class-hierarchy.rst @@ -4,7 +4,7 @@ Tutorial: Types and the class hierarchy Overview -------- -The standard QL library for Java represents Java types by means of the ``Type`` class and its various subclasses. +The standard CodeQL library represents Java types by means of the ``Type`` class and its various subclasses. In particular, class ``PrimitiveType`` represents primitive types that are built into the Java language (such as ``boolean`` and ``int``), whereas ``RefType`` and its subclasses represent reference types, that is classes, interfaces, array types, and so on. This includes both types from the Java standard library (like ``java.lang.Object``) and types defined by non-library code. @@ -62,7 +62,7 @@ In this tutorial, we do not try to distinguish these two cases. Our query should - Both ``source`` and ``target`` are array types. - The element type of ``source`` is a transitive super type of the element type of ``target``. -This recipe is not too difficult to translate into a QL query: +This recipe is not too difficult to translate into a query: .. code-block:: ql @@ -93,7 +93,7 @@ In code that does not use generics, this method is often used in the following w Here, ``l`` has the raw type ``List``, so ``l.toArray`` has return type ``Object[]``, independent of the type of its argument array. Hence the cast goes from ``Object[]`` to ``A[]`` and will be flagged as problematic by our query, although at runtime this cast can never go wrong. -To identify these cases, we can create two QL classes that represent, respectively, the ``Collection.toArray`` class, and calls to this method or any method that overrides it: +To identify these cases, we can create two CodeQL classes that represent, respectively, the ``Collection.toArray`` class, and calls to this method or any method that overrides it: .. code-block:: ql @@ -160,7 +160,7 @@ Since ``zkProp`` is a map from ``Object`` to ``Object``, ``zkProp.entrySet`` ret In general, we want to find calls to ``Collection.contains`` (or any of its overriding methods in any parameterized instance of ``Collection``), such that the type ``E`` of collection elements and the type ``A`` of the argument to ``contains`` are unrelated, that is, they have no common subtype. -We start by creating a QL class that describes ``java.util.Collection``: +We start by creating a class that describes ``java.util.Collection``: .. code-block:: ql @@ -179,7 +179,7 @@ To make sure we have not mistyped anything, we can run a simple test query: This query should return precisely one result. -Next, we can create a QL class that describes ``java.util.Collection.contains``: +Next, we can create a class that describes ``java.util.Collection.contains``: .. code-block:: ql @@ -198,9 +198,9 @@ Notice that we use ``hasStringSignature`` to check that: Alternatively, we could have implemented these three checks more verbosely using ``hasName``, ``getNumberOfParameters``, and ``getParameter(0).getType() instanceof TypeObject``. -As before, it is a good idea to test the new QL class by running a simple query to select all instances of ``JavaUtilCollectionContains``; again there should only be a single result. +As before, it is a good idea to test the new class by running a simple query to select all instances of ``JavaUtilCollectionContains``; again there should only be a single result. -Now we want to identify all calls to ``Collection.contains``, including any methods that override it, and considering all parameterized instances of ``Collection`` and its subclasses. That is, we are looking for method accesses where the source declaration of the invoked method (reflexively or transitively) overrides ``Collection.contains``. We encode this in a QL class ``JavaUtilCollectionContainsCall``: +Now we want to identify all calls to ``Collection.contains``, including any methods that override it, and considering all parameterized instances of ``Collection`` and its subclasses. That is, we are looking for method accesses where the source declaration of the invoked method (reflexively or transitively) overrides ``Collection.contains``. We encode this in a CodeQL class ``JavaUtilCollectionContainsCall``: .. code-block:: ql @@ -230,7 +230,7 @@ For the latter, we proceed as follows: - Find a (reflexive or transitive) super type ``S`` of ``D`` that is a parameterized instance of ``java.util.Collection``. - Return the (only) type argument of ``S``. -We encode this in QL as follows: +We encode this as follows: .. code-block:: ql @@ -268,11 +268,11 @@ Now we are ready to write a first version of our query: Improvements ~~~~~~~~~~~~ -For many programs, this query yields a large number of false positive results due to type variables and wild cards: if the collection element type is some type variable ``E`` and the argument type is ``String``, for example, QL will consider that the two have no common subtype, and our query will flag the call. An easy way to exclude such false positive results is to simply require that neither ``collEltType`` nor ``argType`` are instances of ``TypeVariable``. +For many programs, this query yields a large number of false positive results due to type variables and wild cards: if the collection element type is some type variable ``E`` and the argument type is ``String``, for example, CodeQL will consider that the two have no common subtype, and our query will flag the call. An easy way to exclude such false positive results is to simply require that neither ``collEltType`` nor ``argType`` are instances of ``TypeVariable``. Another source of false positives is autoboxing of primitive types: if, for example, the collection's element type is ``Integer`` and the argument is of type ``int``, predicate ``haveCommonDescendant`` will fail, since ``int`` is not a ``RefType``. Thus, our query should check that ``collEltType`` is not the boxed type of ``argType``. -Finally, ``null`` is special because its type (known as ```` in QL) is compatible with every reference type, so we should exclude it from consideration. +Finally, ``null`` is special because its type (known as ```` in the CodeQL library) is compatible with every reference type, so we should exclude it from consideration. Adding these three improvements, our final query becomes: @@ -295,6 +295,6 @@ Adding these three improvements, our final query becomes: What next? ---------- -- Take a look at some of the other tutorials: :doc:`Tutorial: Expressions and statements `, :doc:`Tutorial: Navigating the call graph `, :doc:`Tutorial: Annotations `, :doc:`Tutorial: Javadoc `, :doc:`Tutorial: Working with source locations `. -- Find out how specific classes in the AST are represented in the QL standard library for Java: :doc:`AST class reference `. +- Take a look at some of the other tutorials: :doc:`Tutorial: Expressions and statements `, :doc:`Tutorial: Navigating the call graph `, :doc:`Tutorial: Annotations `, :doc:`Tutorial: Javadoc `, and :doc:`Tutorial: Working with source locations `. +- Find out how specific classes in the AST are represented in the standard library for Java: :doc:`AST class reference `. - Find out more about QL in the `QL language handbook `__ and `QL language specification `__. From 5489a80372ab2fdbf1c30f7d3a5c0625a4ed9bf1 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Thu, 10 Oct 2019 16:20:27 +0200 Subject: [PATCH 037/148] add query for detecting ignored calls to Array.prototype.concat --- change-notes/1.23/analysis-javascript.md | 1 + .../src/Statements/IgnoreConcatReturn.qhelp | 44 +++++++++++++++++++ .../ql/src/Statements/IgnoreConcatReturn.ql | 42 ++++++++++++++++++ .../src/Statements/examples/IgnoreConcat.js | 5 +++ .../Statements/examples/IgnoreConcatFixed.js | 5 +++ .../IgnoreConcatReturn.expected | 2 + .../IgnoreConcatReturn.qlref | 1 + .../Statements/IgnoreConcatReturn/tst.js | 13 ++++++ 8 files changed, 113 insertions(+) create mode 100644 javascript/ql/src/Statements/IgnoreConcatReturn.qhelp create mode 100644 javascript/ql/src/Statements/IgnoreConcatReturn.ql create mode 100644 javascript/ql/src/Statements/examples/IgnoreConcat.js create mode 100644 javascript/ql/src/Statements/examples/IgnoreConcatFixed.js create mode 100644 javascript/ql/test/query-tests/Statements/IgnoreConcatReturn/IgnoreConcatReturn.expected create mode 100644 javascript/ql/test/query-tests/Statements/IgnoreConcatReturn/IgnoreConcatReturn.qlref create mode 100644 javascript/ql/test/query-tests/Statements/IgnoreConcatReturn/tst.js diff --git a/change-notes/1.23/analysis-javascript.md b/change-notes/1.23/analysis-javascript.md index 73bbf1247f7..90f46403f26 100644 --- a/change-notes/1.23/analysis-javascript.md +++ b/change-notes/1.23/analysis-javascript.md @@ -23,6 +23,7 @@ | Use of returnless function (`js/use-of-returnless-function`) | maintainability, correctness | Highlights calls where the return value is used, but the callee never returns a value. Results are shown on LGTM by default. | | Useless regular expression character escape (`js/useless-regexp-character-escape`) | correctness, security, external/cwe/cwe-20 | Highlights regular expression strings with useless character escapes, indicating a possible violation of [CWE-20](https://cwe.mitre.org/data/definitions/20.html). Results are shown on LGTM by default. | | Unreachable method overloads (`js/unreachable-method-overloads`) | correctness, typescript | Highlights method overloads that are impossible to use from client code. Results are shown on LGTM by default. | +| Ignoring return from concat (`js/ignore-return-from-concat`) | maintainability, correctness | Highlights calls to the concat method on array where the return value is ignored. Results are shown on LGTM by default. | ## Changes to existing queries diff --git a/javascript/ql/src/Statements/IgnoreConcatReturn.qhelp b/javascript/ql/src/Statements/IgnoreConcatReturn.qhelp new file mode 100644 index 00000000000..1b69a61b3a9 --- /dev/null +++ b/javascript/ql/src/Statements/IgnoreConcatReturn.qhelp @@ -0,0 +1,44 @@ + + + +

+The concat method on is pure and does not modify any of the input +arrays. It is therefore generally an error to ignore the return value from a +call to concat. +

+ + + + +

+Use the returned value from the call to concat. +

+ + + + +

+In the following example a function extend is defined. The +function uses the concat method to add elements to the +arr array. However, the extend function has no +effect as the return value from concat is ignored. +

+ + + +

+Assigning the returned value from the call to concat to the +arr variable fixes the error. +

+ + + + + + +
  • Mozilla Developer Network: Array concat.
    • + +     +     diff --git a/javascript/ql/src/Statements/IgnoreConcatReturn.ql b/javascript/ql/src/Statements/IgnoreConcatReturn.ql new file mode 100644 index 00000000000..65c379b3437 --- /dev/null +++ b/javascript/ql/src/Statements/IgnoreConcatReturn.ql @@ -0,0 +1,42 @@ +/** + * @name Ignoring return from concat + * @description The concat method does not modify an array, ignoring the result of a call to concat is therefore generally an error. + * @kind problem + * @problem.severity warning + * @id js/ignore-result-from-concat + * @tags maintainability, + * correctness + * @precision high + */ + +import javascript +import Expressions.ExprHasNoEffect + +DataFlow::SourceNode array(DataFlow::TypeTracker t) { + t.start() and + result instanceof DataFlow::ArrayCreationNode + or + exists (DataFlow::TypeTracker t2 | + result = array(t2).track(t2, t) + ) +} + +DataFlow::SourceNode array() { result = array(DataFlow::TypeTracker::end()) } + +predicate isArrayMethod(DataFlow::MethodCallNode call) { + call.getReceiver().getALocalSource() = array() +} + +predicate isIncomplete(DataFlow::Node node) { + any(DataFlow::Incompleteness cause | node.analyze().getAValue().isIndefinite(cause)) != "global" +} + +from DataFlow::CallNode call +where + isArrayMethod(call) and + call.getCalleeName() = "concat" and + call.getNumArgument() = 1 and + (call.getArgument(0).getALocalSource() = array() or isIncomplete(call.getArgument(0))) and + not call.getArgument(0).asExpr().(ArrayExpr).getSize() = 0 and + inVoidContext(call.asExpr()) +select call, "Return value from call to concat ignored." diff --git a/javascript/ql/src/Statements/examples/IgnoreConcat.js b/javascript/ql/src/Statements/examples/IgnoreConcat.js new file mode 100644 index 00000000000..7137d5261a6 --- /dev/null +++ b/javascript/ql/src/Statements/examples/IgnoreConcat.js @@ -0,0 +1,5 @@ +var arr = [1,2,3]; + +function extend(others) { + arr.concat(others); +} \ No newline at end of file diff --git a/javascript/ql/src/Statements/examples/IgnoreConcatFixed.js b/javascript/ql/src/Statements/examples/IgnoreConcatFixed.js new file mode 100644 index 00000000000..1c7977e9ef0 --- /dev/null +++ b/javascript/ql/src/Statements/examples/IgnoreConcatFixed.js @@ -0,0 +1,5 @@ +var arr = [1,2,3]; + +function extend(others) { + arr = arr.concat(others); +} \ No newline at end of file diff --git a/javascript/ql/test/query-tests/Statements/IgnoreConcatReturn/IgnoreConcatReturn.expected b/javascript/ql/test/query-tests/Statements/IgnoreConcatReturn/IgnoreConcatReturn.expected new file mode 100644 index 00000000000..1ae8b59226a --- /dev/null +++ b/javascript/ql/test/query-tests/Statements/IgnoreConcatReturn/IgnoreConcatReturn.expected @@ -0,0 +1,2 @@ +| tst.js:3:1:3:19 | arr.concat([1,2,3]) | Return value from call to concat ignored. | +| tst.js:5:1:5:15 | arr.concat(arr) | Return value from call to concat ignored. | diff --git a/javascript/ql/test/query-tests/Statements/IgnoreConcatReturn/IgnoreConcatReturn.qlref b/javascript/ql/test/query-tests/Statements/IgnoreConcatReturn/IgnoreConcatReturn.qlref new file mode 100644 index 00000000000..485692fff07 --- /dev/null +++ b/javascript/ql/test/query-tests/Statements/IgnoreConcatReturn/IgnoreConcatReturn.qlref @@ -0,0 +1 @@ +Statements/IgnoreConcatReturn.ql \ No newline at end of file diff --git a/javascript/ql/test/query-tests/Statements/IgnoreConcatReturn/tst.js b/javascript/ql/test/query-tests/Statements/IgnoreConcatReturn/tst.js new file mode 100644 index 00000000000..f2c5f34cc60 --- /dev/null +++ b/javascript/ql/test/query-tests/Statements/IgnoreConcatReturn/tst.js @@ -0,0 +1,13 @@ +var arr = [1,2,3]; + +arr.concat([1,2,3]); // NOT OK! + +arr.concat(arr); // NOT OK! + +console.log(arr.concat([1,2,3])); + +arr.concat(null); +arr.concat(); +arr.concat([]); + +({concat: Array.prototype.concat}.concat(arr)); \ No newline at end of file From 8f58e7e6c950ca6033a7b29803e32876b9c4b869 Mon Sep 17 00:00:00 2001 From: Jonas Jensen Date: Thu, 24 Oct 2019 17:34:01 +0200 Subject: [PATCH 038/148] C++: Clarify qldoc --- cpp/ql/src/semmle/code/cpp/PrintAST.ql | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cpp/ql/src/semmle/code/cpp/PrintAST.ql b/cpp/ql/src/semmle/code/cpp/PrintAST.ql index 23f249585b9..e4c53030da5 100644 --- a/cpp/ql/src/semmle/code/cpp/PrintAST.ql +++ b/cpp/ql/src/semmle/code/cpp/PrintAST.ql @@ -13,6 +13,9 @@ import PrintAST * printed. */ class Cfg extends PrintASTConfiguration { - /** Holds if the AST for `func` should be printed. */ + /** + * TWEAK THIS PREDICATE AS NEEDED. + * Holds if the AST for `func` should be printed. + */ override predicate shouldPrintFunction(Function func) { any() } } From 5b26d03f1ca2dba1a91e86f363dd855e09529230 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Fri, 25 Oct 2019 15:45:35 +0200 Subject: [PATCH 039/148] introduce backtracking, and also marking join/slice calls --- ...atReturn.qhelp => IgnoreArrayResult.qhelp} | 16 ++++--- .../ql/src/Statements/IgnoreArrayResult.ql | 45 +++++++++++++++++++ .../ql/src/Statements/IgnoreConcatReturn.ql | 42 ----------------- .../{IgnoreConcat.js => IgnoreArrayResult.js} | 0 ...ncatFixed.js => IgnoreArrayResultFixed.js} | 0 .../IgnoreArrayResult.expected} | 0 .../IgnoreArrayResult.qlref} | 0 .../tst.js | 0 8 files changed, 55 insertions(+), 48 deletions(-) rename javascript/ql/src/Statements/{IgnoreConcatReturn.qhelp => IgnoreArrayResult.qhelp} (50%) create mode 100644 javascript/ql/src/Statements/IgnoreArrayResult.ql delete mode 100644 javascript/ql/src/Statements/IgnoreConcatReturn.ql rename javascript/ql/src/Statements/examples/{IgnoreConcat.js => IgnoreArrayResult.js} (100%) rename javascript/ql/src/Statements/examples/{IgnoreConcatFixed.js => IgnoreArrayResultFixed.js} (100%) rename javascript/ql/test/query-tests/Statements/{IgnoreConcatReturn/IgnoreConcatReturn.expected => IgnoreArrayResult/IgnoreArrayResult.expected} (100%) rename javascript/ql/test/query-tests/Statements/{IgnoreConcatReturn/IgnoreConcatReturn.qlref => IgnoreArrayResult/IgnoreArrayResult.qlref} (100%) rename javascript/ql/test/query-tests/Statements/{IgnoreConcatReturn => IgnoreArrayResult}/tst.js (100%) diff --git a/javascript/ql/src/Statements/IgnoreConcatReturn.qhelp b/javascript/ql/src/Statements/IgnoreArrayResult.qhelp similarity index 50% rename from javascript/ql/src/Statements/IgnoreConcatReturn.qhelp rename to javascript/ql/src/Statements/IgnoreArrayResult.qhelp index 1b69a61b3a9..b70ff1f1b32 100644 --- a/javascript/ql/src/Statements/IgnoreConcatReturn.qhelp +++ b/javascript/ql/src/Statements/IgnoreArrayResult.qhelp @@ -4,16 +4,18 @@

    -The concat method on is pure and does not modify any of the input -arrays. It is therefore generally an error to ignore the return value from a -call to concat. +The concat, join and slice methods are +pure and does not modify any of the inputs or the array the method was called +on. It is therefore generally an error to ignore the return value from a call +to one of these methods.

    -Use the returned value from the call to concat. +Use the returned value from the calls to concat, join +or slice.

    @@ -26,19 +28,21 @@ function uses the concat method to add elements to the effect as the return value from concat is ignored.

    - +

    Assigning the returned value from the call to concat to the arr variable fixes the error.

    - +
  • Mozilla Developer Network: Array concat.
    • +
  • Mozilla Developer Network: Array slice.
    • +
  • Mozilla Developer Network: Array join.
    •      diff --git a/javascript/ql/src/Statements/IgnoreArrayResult.ql b/javascript/ql/src/Statements/IgnoreArrayResult.ql new file mode 100644 index 00000000000..b3703c6992d --- /dev/null +++ b/javascript/ql/src/Statements/IgnoreArrayResult.ql @@ -0,0 +1,45 @@ +/** + * @name Ignoring result from pure array method + * @description The array methods do not modify the array, ignoring the result of such a call is therefore generally an error. + * @kind problem + * @problem.severity warning + * @id js/ignore-array-result + * @tags maintainability, + * correctness + * @precision high + */ + +import javascript +import Expressions.ExprHasNoEffect + +DataFlow::SourceNode callsArray(DataFlow::TypeBackTracker t, DataFlow::MethodCallNode call) { + isIgnoredPureArrayCall(call) and + ( + t.start() and + result = call.getReceiver() + or + exists(DataFlow::TypeBackTracker t2 | result = callsArray(t2, call).backtrack(t2, t)) + ) +} + +DataFlow::SourceNode callsArray(DataFlow::MethodCallNode call) { + result = callsArray(DataFlow::TypeBackTracker::end(), call) +} + +predicate isIgnoredPureArrayCall(DataFlow::MethodCallNode call) { + inVoidContext(call.asExpr()) and + ( + call.getMethodName() = "concat" and + call.getNumArgument() = 1 + or + call.getMethodName() = "join" and + call.getNumArgument() < 2 + or + call.getMethodName() = "slice" and + call.getNumArgument() < 3 + ) +} + +from DataFlow::MethodCallNode call +where callsArray(call) instanceof DataFlow::ArrayCreationNode +select call, "Result from call to " + call.getMethodName() + " ignored." diff --git a/javascript/ql/src/Statements/IgnoreConcatReturn.ql b/javascript/ql/src/Statements/IgnoreConcatReturn.ql deleted file mode 100644 index 65c379b3437..00000000000 --- a/javascript/ql/src/Statements/IgnoreConcatReturn.ql +++ /dev/null @@ -1,42 +0,0 @@ -/** - * @name Ignoring return from concat - * @description The concat method does not modify an array, ignoring the result of a call to concat is therefore generally an error. - * @kind problem - * @problem.severity warning - * @id js/ignore-result-from-concat - * @tags maintainability, - * correctness - * @precision high - */ - -import javascript -import Expressions.ExprHasNoEffect - -DataFlow::SourceNode array(DataFlow::TypeTracker t) { - t.start() and - result instanceof DataFlow::ArrayCreationNode - or - exists (DataFlow::TypeTracker t2 | - result = array(t2).track(t2, t) - ) -} - -DataFlow::SourceNode array() { result = array(DataFlow::TypeTracker::end()) } - -predicate isArrayMethod(DataFlow::MethodCallNode call) { - call.getReceiver().getALocalSource() = array() -} - -predicate isIncomplete(DataFlow::Node node) { - any(DataFlow::Incompleteness cause | node.analyze().getAValue().isIndefinite(cause)) != "global" -} - -from DataFlow::CallNode call -where - isArrayMethod(call) and - call.getCalleeName() = "concat" and - call.getNumArgument() = 1 and - (call.getArgument(0).getALocalSource() = array() or isIncomplete(call.getArgument(0))) and - not call.getArgument(0).asExpr().(ArrayExpr).getSize() = 0 and - inVoidContext(call.asExpr()) -select call, "Return value from call to concat ignored." diff --git a/javascript/ql/src/Statements/examples/IgnoreConcat.js b/javascript/ql/src/Statements/examples/IgnoreArrayResult.js similarity index 100% rename from javascript/ql/src/Statements/examples/IgnoreConcat.js rename to javascript/ql/src/Statements/examples/IgnoreArrayResult.js diff --git a/javascript/ql/src/Statements/examples/IgnoreConcatFixed.js b/javascript/ql/src/Statements/examples/IgnoreArrayResultFixed.js similarity index 100% rename from javascript/ql/src/Statements/examples/IgnoreConcatFixed.js rename to javascript/ql/src/Statements/examples/IgnoreArrayResultFixed.js diff --git a/javascript/ql/test/query-tests/Statements/IgnoreConcatReturn/IgnoreConcatReturn.expected b/javascript/ql/test/query-tests/Statements/IgnoreArrayResult/IgnoreArrayResult.expected similarity index 100% rename from javascript/ql/test/query-tests/Statements/IgnoreConcatReturn/IgnoreConcatReturn.expected rename to javascript/ql/test/query-tests/Statements/IgnoreArrayResult/IgnoreArrayResult.expected diff --git a/javascript/ql/test/query-tests/Statements/IgnoreConcatReturn/IgnoreConcatReturn.qlref b/javascript/ql/test/query-tests/Statements/IgnoreArrayResult/IgnoreArrayResult.qlref similarity index 100% rename from javascript/ql/test/query-tests/Statements/IgnoreConcatReturn/IgnoreConcatReturn.qlref rename to javascript/ql/test/query-tests/Statements/IgnoreArrayResult/IgnoreArrayResult.qlref diff --git a/javascript/ql/test/query-tests/Statements/IgnoreConcatReturn/tst.js b/javascript/ql/test/query-tests/Statements/IgnoreArrayResult/tst.js similarity index 100% rename from javascript/ql/test/query-tests/Statements/IgnoreConcatReturn/tst.js rename to javascript/ql/test/query-tests/Statements/IgnoreArrayResult/tst.js From 841dac1abad86183a0c590fbe45eb486efd81039 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Fri, 25 Oct 2019 17:46:55 +0200 Subject: [PATCH 040/148] address review feedback --- javascript/ql/src/Statements/UseOfReturnlessFunction.ql | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/javascript/ql/src/Statements/UseOfReturnlessFunction.ql b/javascript/ql/src/Statements/UseOfReturnlessFunction.ql index 2fc3f01a560..c01c6b5e3eb 100644 --- a/javascript/ql/src/Statements/UseOfReturnlessFunction.ql +++ b/javascript/ql/src/Statements/UseOfReturnlessFunction.ql @@ -127,8 +127,10 @@ DataFlow::SourceNode array() { result = array(DataFlow::TypeTracker::end()) } */ predicate voidArrayCallback(DataFlow::CallNode call, Function func) { hasNonVoidCallbackMethod(call.getCalleeName()) and - func = call.getAnArgument().getALocalSource().asExpr() and - 1 = count(DataFlow::Node arg | arg = call.getAnArgument() and arg.getALocalSource().asExpr() instanceof Function) and + exists(int index | + index = min(int i | exists(call.getCallback(i))) and + func = call.getCallback(index).getFunction() + ) and returnsVoid(func) and not isStub(func) and not alwaysThrows(func) and From da23898ebae5e0701ba1aa40783401e232619c16 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Sat, 26 Oct 2019 23:26:45 +0200 Subject: [PATCH 041/148] update tests --- .../Statements/IgnoreArrayResult/IgnoreArrayResult.expected | 4 ++-- .../Statements/IgnoreArrayResult/IgnoreArrayResult.qlref | 2 +- .../ql/test/query-tests/Statements/IgnoreArrayResult/tst.js | 4 ---- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/javascript/ql/test/query-tests/Statements/IgnoreArrayResult/IgnoreArrayResult.expected b/javascript/ql/test/query-tests/Statements/IgnoreArrayResult/IgnoreArrayResult.expected index 1ae8b59226a..9f5c5a0b082 100644 --- a/javascript/ql/test/query-tests/Statements/IgnoreArrayResult/IgnoreArrayResult.expected +++ b/javascript/ql/test/query-tests/Statements/IgnoreArrayResult/IgnoreArrayResult.expected @@ -1,2 +1,2 @@ -| tst.js:3:1:3:19 | arr.concat([1,2,3]) | Return value from call to concat ignored. | -| tst.js:5:1:5:15 | arr.concat(arr) | Return value from call to concat ignored. | +| tst.js:3:1:3:19 | arr.concat([1,2,3]) | Result from call to concat ignored. | +| tst.js:5:1:5:15 | arr.concat(arr) | Result from call to concat ignored. | diff --git a/javascript/ql/test/query-tests/Statements/IgnoreArrayResult/IgnoreArrayResult.qlref b/javascript/ql/test/query-tests/Statements/IgnoreArrayResult/IgnoreArrayResult.qlref index 485692fff07..2cbc7e722a5 100644 --- a/javascript/ql/test/query-tests/Statements/IgnoreArrayResult/IgnoreArrayResult.qlref +++ b/javascript/ql/test/query-tests/Statements/IgnoreArrayResult/IgnoreArrayResult.qlref @@ -1 +1 @@ -Statements/IgnoreConcatReturn.ql \ No newline at end of file +Statements/IgnoreArrayResult.ql \ No newline at end of file diff --git a/javascript/ql/test/query-tests/Statements/IgnoreArrayResult/tst.js b/javascript/ql/test/query-tests/Statements/IgnoreArrayResult/tst.js index f2c5f34cc60..fc00470c8e1 100644 --- a/javascript/ql/test/query-tests/Statements/IgnoreArrayResult/tst.js +++ b/javascript/ql/test/query-tests/Statements/IgnoreArrayResult/tst.js @@ -6,8 +6,4 @@ arr.concat(arr); // NOT OK! console.log(arr.concat([1,2,3])); -arr.concat(null); -arr.concat(); -arr.concat([]); - ({concat: Array.prototype.concat}.concat(arr)); \ No newline at end of file From c6f53199d400c11c4ad71f75e44484bef81a6e18 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Sun, 27 Oct 2019 00:24:38 +0200 Subject: [PATCH 042/148] ignore when the reciever is the empty array --- javascript/ql/src/Statements/IgnoreArrayResult.ql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/javascript/ql/src/Statements/IgnoreArrayResult.ql b/javascript/ql/src/Statements/IgnoreArrayResult.ql index b3703c6992d..b6137e664d7 100644 --- a/javascript/ql/src/Statements/IgnoreArrayResult.ql +++ b/javascript/ql/src/Statements/IgnoreArrayResult.ql @@ -41,5 +41,7 @@ predicate isIgnoredPureArrayCall(DataFlow::MethodCallNode call) { } from DataFlow::MethodCallNode call -where callsArray(call) instanceof DataFlow::ArrayCreationNode +where + callsArray(call) instanceof DataFlow::ArrayCreationNode and + not call.getReceiver().asExpr().(ArrayExpr).getSize() = 0 select call, "Result from call to " + call.getMethodName() + " ignored." From 92cebea235d6906a18e818c5234696fae2e58965 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Sun, 27 Oct 2019 00:25:59 +0200 Subject: [PATCH 043/148] update tests to include empty reciever case --- .../ql/test/query-tests/Statements/IgnoreArrayResult/tst.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/javascript/ql/test/query-tests/Statements/IgnoreArrayResult/tst.js b/javascript/ql/test/query-tests/Statements/IgnoreArrayResult/tst.js index fc00470c8e1..47efe8c1cb6 100644 --- a/javascript/ql/test/query-tests/Statements/IgnoreArrayResult/tst.js +++ b/javascript/ql/test/query-tests/Statements/IgnoreArrayResult/tst.js @@ -6,4 +6,6 @@ arr.concat(arr); // NOT OK! console.log(arr.concat([1,2,3])); -({concat: Array.prototype.concat}.concat(arr)); \ No newline at end of file +({concat: Array.prototype.concat}.concat(arr)); + +[].concat([1,2,3]); \ No newline at end of file From b2c31701f3cf6dfdfb900fdaeba38b5e0ced2444 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Sun, 27 Oct 2019 09:07:45 +0100 Subject: [PATCH 044/148] add documentation to two predicates --- javascript/ql/src/Statements/UseOfReturnlessFunction.ql | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/javascript/ql/src/Statements/UseOfReturnlessFunction.ql b/javascript/ql/src/Statements/UseOfReturnlessFunction.ql index c01c6b5e3eb..77ec29ba9ee 100644 --- a/javascript/ql/src/Statements/UseOfReturnlessFunction.ql +++ b/javascript/ql/src/Statements/UseOfReturnlessFunction.ql @@ -89,6 +89,9 @@ predicate lastStatementHasNoEffect(Function f) { hasNoEffect(f.getExit().getAPredecessor()) } +/** + * Holds if `func` is a callee of `call`, and all possible callees of `call` never return a value. + */ predicate callToVoidFunction(DataFlow::CallNode call, Function func) { not call.isIncomplete() and func = call.getACallee() and @@ -97,6 +100,11 @@ predicate callToVoidFunction(DataFlow::CallNode call, Function func) { ) } +/** + * Holds if `name` is the name of a method from `Array.prototype` or Lodash, + * where that method takes a callback as parameter, + * and the callback is expected to return a value. + */ predicate hasNonVoidCallbackMethod(string name) { name = "every" or name = "filter" or From d94b0cab29b83be4759c58b4fa3ddf935127e964 Mon Sep 17 00:00:00 2001 From: shati-patel <42641846+shati-patel@users.noreply.github.com> Date: Mon, 28 Oct 2019 12:05:51 +0000 Subject: [PATCH 045/148] Update docs/language/learn-ql/java/introduce-libraries-java.rst Co-Authored-By: Felicity Chapman --- docs/language/learn-ql/java/introduce-libraries-java.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/language/learn-ql/java/introduce-libraries-java.rst b/docs/language/learn-ql/java/introduce-libraries-java.rst index 377823c6f70..b3d5987ab41 100644 --- a/docs/language/learn-ql/java/introduce-libraries-java.rst +++ b/docs/language/learn-ql/java/introduce-libraries-java.rst @@ -6,7 +6,7 @@ Overview There is an extensive library for analyzing CodeQL databases extracted from Java projects. The classes in this library present the data from a database in an object-oriented form and provide abstractions and predicates to help you with common analysis tasks. -The library is implemented as a set of QL modules, that is, files with the extension ``.qll``. The module ``java.qll`` imports all the core C/C++ library modules, so you can include the complete library by beginning your query with: +The library is implemented as a set of QL modules, that is, files with the extension ``.qll``. The module ``java.qll`` imports all the core Java library modules, so you can include the complete library by beginning your query with: .. code-block:: ql From 1fc786bea7f6b3a05f7ba91ef1c4cba75483650d Mon Sep 17 00:00:00 2001 From: Tom Hvitved Date: Mon, 28 Oct 2019 13:11:10 +0100 Subject: [PATCH 046/148] C#: Add `precision` tag to `cs/deserialized-delegate` --- .../ql/src/Security Features/CWE-502/DeserializedDelegate.ql | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/csharp/ql/src/Security Features/CWE-502/DeserializedDelegate.ql b/csharp/ql/src/Security Features/CWE-502/DeserializedDelegate.ql index 1c68f01e78b..31d28311908 100644 --- a/csharp/ql/src/Security Features/CWE-502/DeserializedDelegate.ql +++ b/csharp/ql/src/Security Features/CWE-502/DeserializedDelegate.ql @@ -5,14 +5,11 @@ * @kind problem * @id cs/deserialized-delegate * @problem.severity warning + * @precision high * @tags security * external/cwe/cwe-502 */ -/* - * consider: @precision high - */ - import csharp import semmle.code.csharp.frameworks.system.linq.Expressions import semmle.code.csharp.serialization.Deserializers From c3f23f542ae4108f1773a6da76fc484d47fa1c1d Mon Sep 17 00:00:00 2001 From: Tom Hvitved Date: Mon, 28 Oct 2019 13:15:20 +0100 Subject: [PATCH 047/148] C#: Add change note --- change-notes/1.23/analysis-csharp.md | 1 + 1 file changed, 1 insertion(+) diff --git a/change-notes/1.23/analysis-csharp.md b/change-notes/1.23/analysis-csharp.md index 7ec412a0eb2..4f1cb268468 100644 --- a/change-notes/1.23/analysis-csharp.md +++ b/change-notes/1.23/analysis-csharp.md @@ -8,6 +8,7 @@ The following changes in version 1.23 affect C# analysis in all applications. | **Query** | **Tags** | **Purpose** | |-----------------------------|-----------|--------------------------------------------------------------------| +| Deserialized delegate (`cs/deserialized-delegate`) | security | Finds unsafe deserialization of delegate types. | | Unsafe year argument for 'DateTime' constructor (`cs/unsafe-year-construction`) | reliability, date-time | Finds incorrect manipulation of `DateTime` values, which could lead to invalid dates. | | Mishandling the Japanese era start date (`cs/mishandling-japanese-era`) | reliability, date-time | Finds hard-coded Japanese era start dates that could be invalid. | From b13535ac7d2c673f0aa7f941e20e82b283edfeca Mon Sep 17 00:00:00 2001 From: Jonas Jensen Date: Mon, 28 Oct 2019 15:58:20 +0100 Subject: [PATCH 048/148] C++: Implement DataFlow::BarrierGuard for AST+IR The change note is copied from the Java change note. --- change-notes/1.23/analysis-cpp.md | 4 ++ .../cpp/dataflow/internal/DataFlowUtil.qll | 16 +++-- .../cpp/ir/dataflow/internal/DataFlowUtil.qll | 13 ++-- .../dataflow/dataflow-tests/BarrierGuard.cpp | 68 +++++++++++++++++++ .../dataflow-tests/DataflowTestCommon.qll | 16 +++++ .../dataflow-tests/IRDataflowTestCommon.qll | 17 +++++ .../dataflow/dataflow-tests/test.expected | 10 +++ .../dataflow-tests/test_diff.expected | 3 + .../dataflow/dataflow-tests/test_ir.expected | 7 ++ docs/language/learn-ql/cpp/dataflow.rst | 2 + 10 files changed, 147 insertions(+), 9 deletions(-) create mode 100644 cpp/ql/test/library-tests/dataflow/dataflow-tests/BarrierGuard.cpp diff --git a/change-notes/1.23/analysis-cpp.md b/change-notes/1.23/analysis-cpp.md index 64e9b98fcfb..f22436f55cb 100644 --- a/change-notes/1.23/analysis-cpp.md +++ b/change-notes/1.23/analysis-cpp.md @@ -39,6 +39,10 @@ The following changes in version 1.23 affect C/C++ analysis in all applications. definition of `x` when `x` is a variable of pointer type. It no longer considers deep paths such as `f(&x.myField)` to be definitions of `x`. These changes are in line with the user expectations we've observed. +* The data-flow library now makes it easier to specify barriers/sanitizers + arising from guards by overriding the predicate + `isBarrierGuard`/`isSanitizerGuard` on data-flow and taint-tracking + configurations respectively. * There is now a `DataFlow::localExprFlow` predicate and a `TaintTracking::localExprTaint` predicate to make it easy to use the most common case of local data flow and taint: from one `Expr` to another. diff --git a/cpp/ql/src/semmle/code/cpp/dataflow/internal/DataFlowUtil.qll b/cpp/ql/src/semmle/code/cpp/dataflow/internal/DataFlowUtil.qll index 0caa2ab05df..cd1fbd487e0 100644 --- a/cpp/ql/src/semmle/code/cpp/dataflow/internal/DataFlowUtil.qll +++ b/cpp/ql/src/semmle/code/cpp/dataflow/internal/DataFlowUtil.qll @@ -5,6 +5,8 @@ private import cpp private import semmle.code.cpp.dataflow.internal.FlowVar private import semmle.code.cpp.models.interfaces.DataFlow +private import semmle.code.cpp.controlflow.Guards +private import semmle.code.cpp.valuenumbering.GlobalValueNumbering cached private newtype TNode = @@ -680,12 +682,16 @@ VariableAccess getAnAccessToAssignedVariable(Expr assign) { * * It is important that all extending classes in scope are disjoint. */ -class BarrierGuard extends Expr { - /** NOT YET SUPPORTED. Holds if this guard validates `e` upon evaluating to `branch`. */ - abstract deprecated predicate checks(Expr e, boolean branch); +class BarrierGuard extends GuardCondition { + /** Override this predicate to hold if this guard validates `e` upon evaluating to `b`. */ + abstract predicate checks(Expr e, boolean branch); /** Gets a node guarded by this guard. */ - final Node getAGuardedNode() { - none() // stub + final ExprNode getAGuardedNode() { + exists(GVN value, boolean branch | + result.getExpr() = value.getAnExpr() and + this.checks(value.getAnExpr(), branch) and + this.controls(result.getExpr().getBasicBlock(), branch) + ) } } diff --git a/cpp/ql/src/semmle/code/cpp/ir/dataflow/internal/DataFlowUtil.qll b/cpp/ql/src/semmle/code/cpp/ir/dataflow/internal/DataFlowUtil.qll index 0476ea3c30a..f824e0b6bf2 100644 --- a/cpp/ql/src/semmle/code/cpp/ir/dataflow/internal/DataFlowUtil.qll +++ b/cpp/ql/src/semmle/code/cpp/ir/dataflow/internal/DataFlowUtil.qll @@ -5,6 +5,7 @@ private import cpp private import semmle.code.cpp.ir.IR private import semmle.code.cpp.controlflow.IRGuards +private import semmle.code.cpp.ir.ValueNumbering /** * A newtype wrapper to prevent accidental casts between `Node` and @@ -220,7 +221,7 @@ predicate localFlow(Node source, Node sink) { localFlowStep*(source, sink) } predicate localExprFlow(Expr e1, Expr e2) { localFlow(exprNode(e1), exprNode(e2)) } /** - * A guard that validates some expression. + * A guard that validates some instruction. * * To use this in a configuration, extend the class and provide a * characteristic predicate precisely specifying the guard, and override @@ -229,11 +230,15 @@ predicate localExprFlow(Expr e1, Expr e2) { localFlow(exprNode(e1), exprNode(e2) * It is important that all extending classes in scope are disjoint. */ class BarrierGuard extends IRGuardCondition { - /** NOT YET SUPPORTED. Holds if this guard validates `e` upon evaluating to `b`. */ - abstract deprecated predicate checks(Instruction e, boolean b); + /** Override this predicate to hold if this guard validates `instr` upon evaluating to `b`. */ + abstract predicate checks(Instruction instr, boolean b); /** Gets a node guarded by this guard. */ final Node getAGuardedNode() { - none() // stub + exists(ValueNumber value, boolean edge | + result.asInstruction() = value.getAnInstruction() and + this.checks(value.getAnInstruction(), edge) and + this.controls(result.asInstruction().getBlock(), edge) + ) } } diff --git a/cpp/ql/test/library-tests/dataflow/dataflow-tests/BarrierGuard.cpp b/cpp/ql/test/library-tests/dataflow/dataflow-tests/BarrierGuard.cpp new file mode 100644 index 00000000000..9c3c8bc4569 --- /dev/null +++ b/cpp/ql/test/library-tests/dataflow/dataflow-tests/BarrierGuard.cpp @@ -0,0 +1,68 @@ +int source(); +void sink(int); +bool guarded(int); + +void bg_basic(int source) { + if (guarded(source)) { + sink(source); // no flow + } else { + sink(source); // flow + } +} + +void bg_not(int source) { + if (!guarded(source)) { + sink(source); // flow + } else { + sink(source); // no flow + } +} + +void bg_and(int source, bool arbitrary) { + if (guarded(source) && arbitrary) { + sink(source); // no flow + } else { + sink(source); // flow + } +} + +void bg_or(int source, bool arbitrary) { + if (guarded(source) || arbitrary) { + sink(source); // flow + } else { + sink(source); // flow + } +} + +void bg_return(int source) { + if (!guarded(source)) { + return; + } + sink(source); // no flow +} + +struct XY { + int x, y; +}; + +void bg_stackstruct(XY s1, XY s2) { + s1.x = source(); + if (guarded(s1.x)) { + sink(s1.x); // no flow + } else if (guarded(s1.y)) { + sink(s1.x); // flow + } else if (guarded(s2.y)) { + sink(s1.x); // flow + } +} + +void bg_structptr(XY *p1, XY *p2) { + p1->x = source(); + if (guarded(p1->x)) { + sink(p1->x); // no flow [FALSE POSITIVE in AST] + } else if (guarded(p1->y)) { + sink(p1->x); // flow [NOT DETECTED in IR] + } else if (guarded(p2->x)) { + sink(p1->x); // flow [NOT DETECTED in IR] + } +} diff --git a/cpp/ql/test/library-tests/dataflow/dataflow-tests/DataflowTestCommon.qll b/cpp/ql/test/library-tests/dataflow/dataflow-tests/DataflowTestCommon.qll index 02ee4d45380..a81394640ee 100644 --- a/cpp/ql/test/library-tests/dataflow/dataflow-tests/DataflowTestCommon.qll +++ b/cpp/ql/test/library-tests/dataflow/dataflow-tests/DataflowTestCommon.qll @@ -1,6 +1,20 @@ import cpp import semmle.code.cpp.dataflow.DataFlow +/** + * A `BarrierGuard` that stops flow to all occurrences of `x` within statement + * S in `if (guarded(x)) S`. + */ +// This is tested in `BarrierGuard.cpp`. +class TestBarrierGuard extends DataFlow::BarrierGuard { + TestBarrierGuard() { this.(FunctionCall).getTarget().getName() = "guarded" } + + override predicate checks(Expr checked, boolean isTrue) { + checked = this.(FunctionCall).getArgument(0) and + isTrue = true + } +} + /** Common data flow configuration to be used by tests. */ class TestAllocationConfig extends DataFlow::Configuration { TestAllocationConfig() { this = "TestAllocationConfig" } @@ -26,4 +40,6 @@ class TestAllocationConfig extends DataFlow::Configuration { override predicate isBarrier(DataFlow::Node barrier) { barrier.asExpr().(VariableAccess).getTarget().hasName("barrier") } + + override predicate isBarrierGuard(DataFlow::BarrierGuard bg) { bg instanceof TestBarrierGuard } } diff --git a/cpp/ql/test/library-tests/dataflow/dataflow-tests/IRDataflowTestCommon.qll b/cpp/ql/test/library-tests/dataflow/dataflow-tests/IRDataflowTestCommon.qll index eb5fa14e2e0..490f7e4290a 100644 --- a/cpp/ql/test/library-tests/dataflow/dataflow-tests/IRDataflowTestCommon.qll +++ b/cpp/ql/test/library-tests/dataflow/dataflow-tests/IRDataflowTestCommon.qll @@ -1,5 +1,20 @@ import cpp import semmle.code.cpp.ir.dataflow.DataFlow +import semmle.code.cpp.ir.IR + +/** + * A `BarrierGuard` that stops flow to all occurrences of `x` within statement + * S in `if (guarded(x)) S`. + */ +// This is tested in `BarrierGuard.cpp`. +class TestBarrierGuard extends DataFlow::BarrierGuard { + TestBarrierGuard() { this.(CallInstruction).getStaticCallTarget().getName() = "guarded" } + + override predicate checks(Instruction checked, boolean isTrue) { + checked = this.(CallInstruction).getPositionalArgument(0) and + isTrue = true + } +} /** Common data flow configuration to be used by tests. */ class TestAllocationConfig extends DataFlow::Configuration { @@ -24,4 +39,6 @@ class TestAllocationConfig extends DataFlow::Configuration { override predicate isBarrier(DataFlow::Node barrier) { barrier.asExpr().(VariableAccess).getTarget().hasName("barrier") } + + override predicate isBarrierGuard(DataFlow::BarrierGuard bg) { bg instanceof TestBarrierGuard } } diff --git a/cpp/ql/test/library-tests/dataflow/dataflow-tests/test.expected b/cpp/ql/test/library-tests/dataflow/dataflow-tests/test.expected index b9fca1f678f..6527db4b77e 100644 --- a/cpp/ql/test/library-tests/dataflow/dataflow-tests/test.expected +++ b/cpp/ql/test/library-tests/dataflow/dataflow-tests/test.expected @@ -1,3 +1,13 @@ +| BarrierGuard.cpp:9:10:9:15 | source | BarrierGuard.cpp:5:19:5:24 | source | +| BarrierGuard.cpp:15:10:15:15 | source | BarrierGuard.cpp:13:17:13:22 | source | +| BarrierGuard.cpp:25:10:25:15 | source | BarrierGuard.cpp:21:17:21:22 | source | +| BarrierGuard.cpp:31:10:31:15 | source | BarrierGuard.cpp:29:16:29:21 | source | +| BarrierGuard.cpp:33:10:33:15 | source | BarrierGuard.cpp:29:16:29:21 | source | +| BarrierGuard.cpp:53:13:53:13 | x | BarrierGuard.cpp:49:10:49:15 | call to source | +| BarrierGuard.cpp:55:13:55:13 | x | BarrierGuard.cpp:49:10:49:15 | call to source | +| BarrierGuard.cpp:62:14:62:14 | x | BarrierGuard.cpp:60:11:60:16 | call to source | +| BarrierGuard.cpp:64:14:64:14 | x | BarrierGuard.cpp:60:11:60:16 | call to source | +| BarrierGuard.cpp:66:14:66:14 | x | BarrierGuard.cpp:60:11:60:16 | call to source | | acrossLinkTargets.cpp:12:8:12:8 | x | acrossLinkTargets.cpp:19:27:19:32 | call to source | | clang.cpp:18:8:18:19 | sourceArray1 | clang.cpp:12:9:12:20 | sourceArray1 | | clang.cpp:22:8:22:20 | & ... | clang.cpp:12:9:12:20 | sourceArray1 | diff --git a/cpp/ql/test/library-tests/dataflow/dataflow-tests/test_diff.expected b/cpp/ql/test/library-tests/dataflow/dataflow-tests/test_diff.expected index cd7e3dac785..7d0d4e7d72e 100644 --- a/cpp/ql/test/library-tests/dataflow/dataflow-tests/test_diff.expected +++ b/cpp/ql/test/library-tests/dataflow/dataflow-tests/test_diff.expected @@ -1,3 +1,6 @@ +| BarrierGuard.cpp:60:11:60:16 | BarrierGuard.cpp:62:14:62:14 | AST only | +| BarrierGuard.cpp:60:11:60:16 | BarrierGuard.cpp:64:14:64:14 | AST only | +| BarrierGuard.cpp:60:11:60:16 | BarrierGuard.cpp:66:14:66:14 | AST only | | clang.cpp:12:9:12:20 | clang.cpp:22:8:22:20 | AST only | | clang.cpp:28:27:28:32 | clang.cpp:29:27:29:28 | AST only | | clang.cpp:28:27:28:32 | clang.cpp:30:27:30:34 | AST only | diff --git a/cpp/ql/test/library-tests/dataflow/dataflow-tests/test_ir.expected b/cpp/ql/test/library-tests/dataflow/dataflow-tests/test_ir.expected index 83ba546480f..9b67a013a58 100644 --- a/cpp/ql/test/library-tests/dataflow/dataflow-tests/test_ir.expected +++ b/cpp/ql/test/library-tests/dataflow/dataflow-tests/test_ir.expected @@ -1,3 +1,10 @@ +| BarrierGuard.cpp:9:10:9:15 | Load: source | BarrierGuard.cpp:5:19:5:24 | InitializeParameter: source | +| BarrierGuard.cpp:15:10:15:15 | Load: source | BarrierGuard.cpp:13:17:13:22 | InitializeParameter: source | +| BarrierGuard.cpp:25:10:25:15 | Load: source | BarrierGuard.cpp:21:17:21:22 | InitializeParameter: source | +| BarrierGuard.cpp:31:10:31:15 | Load: source | BarrierGuard.cpp:29:16:29:21 | InitializeParameter: source | +| BarrierGuard.cpp:33:10:33:15 | Load: source | BarrierGuard.cpp:29:16:29:21 | InitializeParameter: source | +| BarrierGuard.cpp:53:13:53:13 | Load: x | BarrierGuard.cpp:49:10:49:15 | Call: call to source | +| BarrierGuard.cpp:55:13:55:13 | Load: x | BarrierGuard.cpp:49:10:49:15 | Call: call to source | | acrossLinkTargets.cpp:12:8:12:8 | Convert: (int)... | acrossLinkTargets.cpp:19:27:19:32 | Call: call to source | | acrossLinkTargets.cpp:12:8:12:8 | Load: x | acrossLinkTargets.cpp:19:27:19:32 | Call: call to source | | clang.cpp:18:8:18:19 | Convert: (const int *)... | clang.cpp:12:9:12:20 | InitializeParameter: sourceArray1 | diff --git a/docs/language/learn-ql/cpp/dataflow.rst b/docs/language/learn-ql/cpp/dataflow.rst index 4867f3daf47..50fccdc5cc4 100644 --- a/docs/language/learn-ql/cpp/dataflow.rst +++ b/docs/language/learn-ql/cpp/dataflow.rst @@ -166,6 +166,7 @@ The following predicates are defined in the configuration: - ``isSource``—defines where data may flow from - ``isSink``—defines where data may flow to - ``isBarrier``—optional, restricts the data flow +- ``isBarrierGuard``—optional, restricts the data flow - ``isAdditionalFlowStep``—optional, adds additional flow steps The characteristic predicate ``MyDataFlowConfiguration()`` defines the name of the configuration, so ``"MyDataFlowConfiguration"`` should be replaced by the name of your class. @@ -204,6 +205,7 @@ The following predicates are defined in the configuration: - ``isSource``—defines where taint may flow from - ``isSink``—defines where taint may flow to - ``isSanitizer``—optional, restricts the taint flow +- ``isSanitizerGuard``—optional, restricts the taint flow - ``isAdditionalTaintStep``—optional, adds additional taint steps Similar to global data flow, the characteristic predicate ``MyTaintTrackingConfiguration()`` defines the unique name of the configuration, so ``"MyTaintTrackingConfiguration"`` should be replaced by the name of your class. From d693eb8c20a147a927ae2e5dda179d9233cde6b9 Mon Sep 17 00:00:00 2001 From: Geoffrey White <40627776+geoffw0@users.noreply.github.com> Date: Mon, 28 Oct 2019 17:39:45 +0000 Subject: [PATCH 049/148] CPP: Correct the ConditionallyUninitializedVariable examples. --- .../CWE/CWE-457/ConditionallyUninitializedVariableBad.c | 6 +++--- .../CWE/CWE-457/ConditionallyUninitializedVariableGood.c | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cpp/ql/src/Security/CWE/CWE-457/ConditionallyUninitializedVariableBad.c b/cpp/ql/src/Security/CWE/CWE-457/ConditionallyUninitializedVariableBad.c index 1f281f3cfd9..73a01c2d900 100644 --- a/cpp/ql/src/Security/CWE/CWE-457/ConditionallyUninitializedVariableBad.c +++ b/cpp/ql/src/Security/CWE/CWE-457/ConditionallyUninitializedVariableBad.c @@ -19,7 +19,7 @@ int notify(int deviceNumber) { DeviceConfig config; initDeviceConfig(&config, deviceNumber); // BAD: Using config without checking the status code that is returned - if (config->isEnabled) { - notifyChannel(config->channel); + if (config.isEnabled) { + notifyChannel(config.channel); } -} \ No newline at end of file +} diff --git a/cpp/ql/src/Security/CWE/CWE-457/ConditionallyUninitializedVariableGood.c b/cpp/ql/src/Security/CWE/CWE-457/ConditionallyUninitializedVariableGood.c index a9dcc06c9a5..ced43a66cfc 100644 --- a/cpp/ql/src/Security/CWE/CWE-457/ConditionallyUninitializedVariableGood.c +++ b/cpp/ql/src/Security/CWE/CWE-457/ConditionallyUninitializedVariableGood.c @@ -20,8 +20,8 @@ void notify(int deviceNumber) { int statusCode = initDeviceConfig(&config, deviceNumber); if (statusCode == 0) { // GOOD: Status code returned by initialization function is checked, so this is safe - if (config->isEnabled) { - notifyChannel(config->channel); + if (config.isEnabled) { + notifyChannel(config.channel); } } -} \ No newline at end of file +} From c40c88ec4bfd2ddd5e79ed48fb1c955e23bf2a7b Mon Sep 17 00:00:00 2001 From: Geoffrey White <40627776+geoffw0@users.noreply.github.com> Date: Mon, 28 Oct 2019 18:43:00 +0000 Subject: [PATCH 050/148] CPP: Add test cases for ConditionallyUninitializedVariables.ql. --- ...onditionallyUninitializedVariable.expected | 3 + .../ConditionallyUninitializedVariable.qlref | 1 + .../examples.cpp | 43 +++++++++ .../test.cpp | 96 +++++++++++++++++++ 4 files changed, 143 insertions(+) create mode 100644 cpp/ql/test/query-tests/Security/CWE/CWE-457/semmle/ConditionallyUninitializedVariable/ConditionallyUninitializedVariable.expected create mode 100644 cpp/ql/test/query-tests/Security/CWE/CWE-457/semmle/ConditionallyUninitializedVariable/ConditionallyUninitializedVariable.qlref create mode 100644 cpp/ql/test/query-tests/Security/CWE/CWE-457/semmle/ConditionallyUninitializedVariable/examples.cpp create mode 100644 cpp/ql/test/query-tests/Security/CWE/CWE-457/semmle/ConditionallyUninitializedVariable/test.cpp diff --git a/cpp/ql/test/query-tests/Security/CWE/CWE-457/semmle/ConditionallyUninitializedVariable/ConditionallyUninitializedVariable.expected b/cpp/ql/test/query-tests/Security/CWE/CWE-457/semmle/ConditionallyUninitializedVariable/ConditionallyUninitializedVariable.expected new file mode 100644 index 00000000000..60964c8f178 --- /dev/null +++ b/cpp/ql/test/query-tests/Security/CWE/CWE-457/semmle/ConditionallyUninitializedVariable/ConditionallyUninitializedVariable.expected @@ -0,0 +1,3 @@ +| examples.cpp:38:3:38:18 | call to initDeviceConfig | The status of this call to $@ is not checked, potentially leaving $@ uninitialized. | examples.cpp:13:5:13:20 | initDeviceConfig | initDeviceConfig | examples.cpp:37:16:37:21 | config | config | +| test.cpp:22:2:22:17 | call to maybeInitialize1 | The status of this call to $@ is not checked, potentially leaving $@ uninitialized. | test.cpp:4:5:4:20 | maybeInitialize1 | maybeInitialize1 | test.cpp:19:6:19:6 | a | a | +| test.cpp:68:2:68:17 | call to maybeInitialize2 | The status of this call to $@ is not checked, potentially leaving $@ uninitialized. | test.cpp:51:6:51:21 | maybeInitialize2 | maybeInitialize2 | test.cpp:66:6:66:6 | a | a | diff --git a/cpp/ql/test/query-tests/Security/CWE/CWE-457/semmle/ConditionallyUninitializedVariable/ConditionallyUninitializedVariable.qlref b/cpp/ql/test/query-tests/Security/CWE/CWE-457/semmle/ConditionallyUninitializedVariable/ConditionallyUninitializedVariable.qlref new file mode 100644 index 00000000000..5150d627257 --- /dev/null +++ b/cpp/ql/test/query-tests/Security/CWE/CWE-457/semmle/ConditionallyUninitializedVariable/ConditionallyUninitializedVariable.qlref @@ -0,0 +1 @@ +Security/CWE/CWE-457/ConditionallyUninitializedVariable.ql diff --git a/cpp/ql/test/query-tests/Security/CWE/CWE-457/semmle/ConditionallyUninitializedVariable/examples.cpp b/cpp/ql/test/query-tests/Security/CWE/CWE-457/semmle/ConditionallyUninitializedVariable/examples.cpp new file mode 100644 index 00000000000..ccb15904d02 --- /dev/null +++ b/cpp/ql/test/query-tests/Security/CWE/CWE-457/semmle/ConditionallyUninitializedVariable/examples.cpp @@ -0,0 +1,43 @@ +// based on the qhelp + +int getMaxDevices(); +bool fetchIsDeviceEnabled(int deviceNumber); +int fetchDeviceChannel(int deviceNumber); +void notifyChannel(int channel); + +struct DeviceConfig { + bool isEnabled; + int channel; +}; + +int initDeviceConfig(DeviceConfig *ref, int deviceNumber) { + if (deviceNumber >= getMaxDevices()) { + // No device with that number, return -1 to indicate failure + return -1; + } + // Device with that number, fetch parameters and initialize struct + ref->isEnabled = fetchIsDeviceEnabled(deviceNumber); + ref->channel = fetchDeviceChannel(deviceNumber); + // Return 0 to indicate success + return 0; +} + +void notifyGood(int deviceNumber) { + DeviceConfig config; + int statusCode = initDeviceConfig(&config, deviceNumber); + if (statusCode == 0) { + // GOOD: Status code returned by initialization function is checked, so this is safe + if (config.isEnabled) { + notifyChannel(config.channel); + } + } +} + +int notifyBad(int deviceNumber) { + DeviceConfig config; + initDeviceConfig(&config, deviceNumber); + // BAD: Using config without checking the status code that is returned + if (config.isEnabled) { + notifyChannel(config.channel); + } +} \ No newline at end of file diff --git a/cpp/ql/test/query-tests/Security/CWE/CWE-457/semmle/ConditionallyUninitializedVariable/test.cpp b/cpp/ql/test/query-tests/Security/CWE/CWE-457/semmle/ConditionallyUninitializedVariable/test.cpp new file mode 100644 index 00000000000..a3c9b0a24aa --- /dev/null +++ b/cpp/ql/test/query-tests/Security/CWE/CWE-457/semmle/ConditionallyUninitializedVariable/test.cpp @@ -0,0 +1,96 @@ + +void use(int i); + +int maybeInitialize1(int *v) +{ + static int resources = 100; + + if (resources == 0) + { + return 0; // FAIL + } + + *v = resources--; + return 1; // SUCCESS +} + +void test1() +{ + int a, b, c, d, e, f; + int result1, result2; + + maybeInitialize1(&a); // BAD (initialization not checked) + use(a); + + if (maybeInitialize1(&b) == 1) // GOOD + { + use(b); + } + + if (maybeInitialize1(&c) == 0) // BAD (initialization check is wrong) [NOT DETECTED] + { + use(c); + } + + result1 = maybeInitialize1(&d); // BAD (initialization stored but not checked) [NOT DETECTED] + use(d); + + result2 = maybeInitialize1(&e); // GOOD + if (result2 == 1) + { + use(e); + } + + if (maybeInitialize1(&f) == 0) // GOOD + { + return; + } + use(f); +} + +bool maybeInitialize2(int *v) +{ + static int resources = 100; + + if (resources > 0) + { + *v = resources--; + return true; // SUCCESS + } + + return false; // FAIL +} + +void test2() +{ + int a, b; + + maybeInitialize2(&a); // BAD (initialization not checked) + use(a); + + if (maybeInitialize2(&b)) // GOOD + { + use(b); + } +} + +int alwaysInitialize(int *v) +{ + static int resources = 0; + + *v = resources++; + return 1; // SUCCESS +} + +void test3() +{ + int a, b; + + alwaysInitialize(&a); // GOOD (initialization never fails) + use(a); + + if (alwaysInitialize(&b) == 1) // GOOD + { + use(b); + } +} From 2d64fedeb021db7fa0a919bad6f189ae4805907f Mon Sep 17 00:00:00 2001 From: Geoffrey White <40627776+geoffw0@users.noreply.github.com> Date: Fri, 25 Oct 2019 11:19:31 +0100 Subject: [PATCH 051/148] CPP: Speed up VirtualDispatch.qll's getAViableTarget. --- .../code/cpp/dispatch/VirtualDispatch.qll | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/cpp/ql/src/semmle/code/cpp/dispatch/VirtualDispatch.qll b/cpp/ql/src/semmle/code/cpp/dispatch/VirtualDispatch.qll index 2fa9322843e..1113fc430e7 100644 --- a/cpp/ql/src/semmle/code/cpp/dispatch/VirtualDispatch.qll +++ b/cpp/ql/src/semmle/code/cpp/dispatch/VirtualDispatch.qll @@ -28,6 +28,19 @@ module VirtualDispatch { not result.hasName("IUnknown") } + /** + * Helper predicate for `getAViableTarget`, which computes the viable targets for + * virtual calls based on the qualifier type. + */ + private Function getAViableVirtualCallTarget(Class qualifierType, MemberFunction staticTarget) { + exists(Class qualifierSubType | + result = getAPossibleImplementation(staticTarget) and + qualifierType = qualifierSubType.getABaseClass*() and + mayInherit(qualifierSubType, result) and + not cannotInherit(qualifierSubType, result) + ) + } + /** * Gets a viable target for the given function call. * @@ -42,18 +55,9 @@ module VirtualDispatch { * If `c` is not a virtual call, the result will be `c.getTarget()`. */ Function getAViableTarget(Call c) { - exists(Function staticTarget | staticTarget = c.getTarget() | - if c.(FunctionCall).isVirtual() and staticTarget instanceof MemberFunction - then - exists(Class qualifierType, Class qualifierSubType | - result = getAPossibleImplementation(staticTarget) and - qualifierType = getCallQualifierType(c) and - qualifierType = qualifierSubType.getABaseClass*() and - mayInherit(qualifierSubType, result) and - not cannotInherit(qualifierSubType, result) - ) - else result = staticTarget - ) + if c.(FunctionCall).isVirtual() and c.getTarget() instanceof MemberFunction + then result = getAViableVirtualCallTarget(getCallQualifierType(c), c.getTarget()) + else result = c.getTarget() } /** Holds if `f` is declared in `c` or a transitive base class of `c`. */ From 3584c0b2e5ae646b0e4c53d4d80810e84ed5c780 Mon Sep 17 00:00:00 2001 From: Geoffrey White <40627776+geoffw0@users.noreply.github.com> Date: Fri, 25 Oct 2019 11:22:42 +0100 Subject: [PATCH 052/148] CPP: Speed up InitializationFunctions.qll's getTarget. --- .../CWE/CWE-457/InitializationFunctions.qll | 51 +++++++++++-------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/cpp/ql/src/Security/CWE/CWE-457/InitializationFunctions.qll b/cpp/ql/src/Security/CWE/CWE-457/InitializationFunctions.qll index 240bd7aa25e..720cf11950b 100644 --- a/cpp/ql/src/Security/CWE/CWE-457/InitializationFunctions.qll +++ b/cpp/ql/src/Security/CWE/CWE-457/InitializationFunctions.qll @@ -618,6 +618,33 @@ Function getAPossibleDefinition(Function undefinedFunction) { ) and result.isDefined() } + +private Function getTarget1(Call c) { + /* + * If there is at least one defined target after performing some simple virtual dispatch + * resolution, then the result is all the defined targets. + */ + + result = VirtualDispatch::getAViableTarget(c) and + result.isDefined() +} + +private Function getTarget2(Call c) { + /* + * If we can use the heuristic matching of functions to find definitions for some of the viable + * targets, return those. + */ + + not exists(getTarget1(c)) and + result = getAPossibleDefinition(VirtualDispatch::getAViableTarget(c)) +} + +private Function getTarget3(Call c) { + not exists(getTarget1(c)) and + not exists(getTarget2(c)) and + // Otherwise, the result is the undefined `Function` instances. + result = VirtualDispatch::getAViableTarget(c) +} /** * Gets a possible target for the Call, using the name and parameter matching if we did not associate @@ -625,27 +652,9 @@ Function getAPossibleDefinition(Function undefinedFunction) { * dispatch resolution. */ Function getTarget(Call c) { - if VirtualDispatch::getAViableTarget(c).isDefined() - then - /* - * If there is at least one defined target after performing some simple virtual dispatch - * resolution, then the result is all the defined targets. - */ - - result = VirtualDispatch::getAViableTarget(c) and - result.isDefined() - else - if exists(getAPossibleDefinition(VirtualDispatch::getAViableTarget(c))) - then - /* - * If we can use the heuristic matching of functions to find definitions for some of the viable - * targets, return those. - */ - - result = getAPossibleDefinition(VirtualDispatch::getAViableTarget(c)) - else - // Otherwise, the result is the undefined `Function` instances - result = VirtualDispatch::getAViableTarget(c) + result = getTarget1(c) or + result = getTarget2(c) or + result = getTarget3(c) } /** From ff62afb575e242cdc01586491d40908e18a65f23 Mon Sep 17 00:00:00 2001 From: Jonas Jensen Date: Tue, 29 Oct 2019 10:38:23 +0100 Subject: [PATCH 053/148] C++: Rename parameter to `b` to match QLDoc --- cpp/ql/src/semmle/code/cpp/dataflow/internal/DataFlowUtil.qll | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/ql/src/semmle/code/cpp/dataflow/internal/DataFlowUtil.qll b/cpp/ql/src/semmle/code/cpp/dataflow/internal/DataFlowUtil.qll index cd1fbd487e0..fc0b5a90882 100644 --- a/cpp/ql/src/semmle/code/cpp/dataflow/internal/DataFlowUtil.qll +++ b/cpp/ql/src/semmle/code/cpp/dataflow/internal/DataFlowUtil.qll @@ -684,7 +684,7 @@ VariableAccess getAnAccessToAssignedVariable(Expr assign) { */ class BarrierGuard extends GuardCondition { /** Override this predicate to hold if this guard validates `e` upon evaluating to `b`. */ - abstract predicate checks(Expr e, boolean branch); + abstract predicate checks(Expr e, boolean b); /** Gets a node guarded by this guard. */ final ExprNode getAGuardedNode() { From 563f32193cfbb747c19c0b28884be70264a6296c Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Tue, 29 Oct 2019 12:10:12 +0100 Subject: [PATCH 054/148] suggestions from @max-schaefer Co-Authored-By: Max Schaefer <54907921+max-schaefer@users.noreply.github.com> --- javascript/ql/src/Statements/IgnoreArrayResult.qhelp | 2 +- javascript/ql/src/Statements/IgnoreArrayResult.ql | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/javascript/ql/src/Statements/IgnoreArrayResult.qhelp b/javascript/ql/src/Statements/IgnoreArrayResult.qhelp index b70ff1f1b32..2232784d881 100644 --- a/javascript/ql/src/Statements/IgnoreArrayResult.qhelp +++ b/javascript/ql/src/Statements/IgnoreArrayResult.qhelp @@ -5,7 +5,7 @@

    The concat, join and slice methods are -pure and does not modify any of the inputs or the array the method was called +pure and do not modify any of the inputs or the array the method was called on. It is therefore generally an error to ignore the return value from a call to one of these methods.

    diff --git a/javascript/ql/src/Statements/IgnoreArrayResult.ql b/javascript/ql/src/Statements/IgnoreArrayResult.ql index b6137e664d7..5ab778ed95b 100644 --- a/javascript/ql/src/Statements/IgnoreArrayResult.ql +++ b/javascript/ql/src/Statements/IgnoreArrayResult.ql @@ -1,10 +1,10 @@ /** * @name Ignoring result from pure array method - * @description The array methods do not modify the array, ignoring the result of such a call is therefore generally an error. + * @description Ignoring the result of an array method that does not modify its receiver is generally an error. * @kind problem * @problem.severity warning * @id js/ignore-array-result - * @tags maintainability, + * @tags maintainability * correctness * @precision high */ @@ -16,7 +16,7 @@ DataFlow::SourceNode callsArray(DataFlow::TypeBackTracker t, DataFlow::MethodCal isIgnoredPureArrayCall(call) and ( t.start() and - result = call.getReceiver() + result = call.getReceiver().getALocalSource() or exists(DataFlow::TypeBackTracker t2 | result = callsArray(t2, call).backtrack(t2, t)) ) From 2d01e7c5ed97392e55db948755d445dc8337a68a Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Tue, 29 Oct 2019 12:13:01 +0100 Subject: [PATCH 055/148] simplify the callsArray predicate --- javascript/ql/src/Statements/IgnoreArrayResult.ql | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/javascript/ql/src/Statements/IgnoreArrayResult.ql b/javascript/ql/src/Statements/IgnoreArrayResult.ql index 5ab778ed95b..6f6698797f1 100644 --- a/javascript/ql/src/Statements/IgnoreArrayResult.ql +++ b/javascript/ql/src/Statements/IgnoreArrayResult.ql @@ -13,13 +13,11 @@ import javascript import Expressions.ExprHasNoEffect DataFlow::SourceNode callsArray(DataFlow::TypeBackTracker t, DataFlow::MethodCallNode call) { - isIgnoredPureArrayCall(call) and - ( - t.start() and - result = call.getReceiver().getALocalSource() - or - exists(DataFlow::TypeBackTracker t2 | result = callsArray(t2, call).backtrack(t2, t)) - ) + isIgnoredPureArrayCall(call) and + t.start() and + result = call.getReceiver().getALocalSource() + or + exists(DataFlow::TypeBackTracker t2 | result = callsArray(t2, call).backtrack(t2, t)) } DataFlow::SourceNode callsArray(DataFlow::MethodCallNode call) { From 3337eaf0f9ec2b9de5bf1f58bebd8e5550209509 Mon Sep 17 00:00:00 2001 From: Shati Patel Date: Tue, 29 Oct 2019 12:35:03 +0000 Subject: [PATCH 056/148] Docs: Update JavaScript/TypeScript --- .../javascript/ast-class-reference.rst | 24 +++--- .../javascript/dataflow-cheat-sheet.rst | 8 +- .../language/learn-ql/javascript/dataflow.rst | 8 +- .../learn-ql/javascript/flow-labels.rst | 2 +- .../javascript/introduce-libraries-js.rst | 78 +++++++++---------- .../javascript/introduce-libraries-ts.rst | 24 +++--- .../learn-ql/javascript/ql-for-javascript.rst | 26 +++---- .../learn-ql/javascript/type-tracking.rst | 6 +- 8 files changed, 89 insertions(+), 87 deletions(-) diff --git a/docs/language/learn-ql/javascript/ast-class-reference.rst b/docs/language/learn-ql/javascript/ast-class-reference.rst index 1231f83c4cf..1b513b2d4ec 100644 --- a/docs/language/learn-ql/javascript/ast-class-reference.rst +++ b/docs/language/learn-ql/javascript/ast-class-reference.rst @@ -7,7 +7,7 @@ Statement classes This table lists subclasses of `Stmt `__ representing ECMAScript and TypeScript statements. +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Statement syntax | QL class | Superclasses | Remarks | +| Statement syntax | CodeQL class | Superclasses | Remarks | +=============================================================================================================================================================================================================================================================================================================================================================================================================================================+==================================================================================================================================================================+==========================================================================================================================================================================================================================================================================================================================================================================================================================+===================================================================================================================================================================================================+ | `Expr `__ ``;`` | `ExprStmt `__ | | | +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -105,7 +105,7 @@ Literals All classes in this subsection are subclasses of `Literal `__. +-------------------+------------------------------------------------------------------------------------------------------------------------+ -| Expression syntax | QL class | +| Expression syntax | CodeQL class | +===================+========================================================================================================================+ | ``true`` | `BooleanLiteral `__ | +-------------------+------------------------------------------------------------------------------------------------------------------------+ @@ -123,7 +123,7 @@ All classes in this subsection are subclasses of `Literal `__, which has subclasses to represent specific kinds of identifiers: +All identifiers are represented by the class `Identifier `__, which has subclasses to represent specific kinds of identifiers: - `VarAccess `__: an identifier that refers to a variable - `VarDecl `__: an identifier that declares a variable, for example ``x`` in ``var x = "hi"`` or in ``function(x) { }`` @@ -142,7 +142,7 @@ Primary expressions All classes in this subsection are subclasses of `Expr `__. +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Expression syntax | QL class | Superclasses | Remarks | +| Expression syntax | CodeQL class | Superclasses | Remarks | +======================================================================================================================================================================================================================================================================+==========================================================================================================================================+======================================================================================================================+================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================+ | ``this`` | `ThisExpr `__ | | | +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -167,7 +167,7 @@ Properties All classes in this subsection are subclasses of `Property `__. Note that `Property `__ is not a subclass of `Expr `__. +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------+ -| Property syntax | QL class | Superclasses | +| Property syntax | CodeQL class | Superclasses | +=====================================================================================================================================================================================================================================================================================================================================================================+========================================================================================================================+============================================================================================================================+ | `Identifier `__ ``:`` `Expr `__ | `ValueProperty `__ | | +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------+ @@ -182,7 +182,7 @@ Property accesses All classes in this subsection are subclasses of `PropAccess `__. +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------+ -| Expression syntax | QL class | +| Expression syntax | CodeQL class | +=========================================================================================================================================================================================================================+==============================================================================================================+ | `Expr `__ ``.`` `Identifier `__ | `DotExpr `__ | +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------+ @@ -195,7 +195,7 @@ Function calls and ``new`` All classes in this subsection are subclasses of `InvokeExpr `__. +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Expression syntax | QL class | Remarks | +| Expression syntax | CodeQL class | Remarks | +============================================================================================================================================================================================================================================================================================================================================+========================================================================================================================+==========================================================================================================================================================================================================================================================================================================================================================================+ | `Expr `__ ``(`` `Expr `__... ``)`` | `CallExpr `__ | | +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -210,7 +210,7 @@ Unary expressions All classes in this subsection are subclasses of `UnaryExpr `__. +---------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------+ -| Expression syntax | QL class | +| Expression syntax | CodeQL class | +===============================================================================================================+======================================================================================================================+ | ``~`` `Expr `__ | `BitNotExpr `__ | +---------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------+ @@ -235,7 +235,7 @@ Binary expressions All classes in this subsection are subclasses of `BinaryExpr `__. +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Expression syntax | QL class | Superclasses | +| Expression syntax | CodeQL class | Superclasses | +======================================================================================================================================================================================================================+========================================================================================================================+====================================================================================================================================================================================================================================+ | `Expr `__ ``*`` `Expr `__ | `MulExpr `__ | | +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -292,7 +292,7 @@ Assignment expressions All classes in this table are subclasses of `Assignment `__. +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------+ -| Expression syntax | QL class | Superclasses | +| Expression syntax | CodeQL class | Superclasses | +================================================================================================================================================================================================================+==============================================================================================================================+================================================================================================================================+ | `Expr `__ ``=`` `Expr `__ | `AssignExpr `__ | | +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------+ @@ -327,7 +327,7 @@ Update expressions All classes in this table are subclasses of `UpdateExpr `__. +-----------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------+ -| Expression syntax | QL class | +| Expression syntax | CodeQL class | +===========================================================================================================+==================================================================================================================+ | `Expr `__ ``++`` | `PostIncExpr `__ | +-----------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------+ @@ -344,7 +344,7 @@ Miscellaneous All classes in this table are subclasses of `Expr `__. +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------+ -| Expression syntax | QL class | +| Expression syntax | CodeQL class | +======================================================================================================================================================================================================================================================================================================================+==========================================================================================================================+ | `Expr `__ ``?`` `Expr `__ ``:`` `Expr `__ | `ConditionalExpr `__ | +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------+ diff --git a/docs/language/learn-ql/javascript/dataflow-cheat-sheet.rst b/docs/language/learn-ql/javascript/dataflow-cheat-sheet.rst index 5159c26bf19..a2ff4309ee0 100644 --- a/docs/language/learn-ql/javascript/dataflow-cheat-sheet.rst +++ b/docs/language/learn-ql/javascript/dataflow-cheat-sheet.rst @@ -1,14 +1,14 @@ Data flow cheat sheet ===================== -This page describes parts of the JavaScript QL libraries commonly used for variant analysis and in data flow queries. +This page describes parts of the JavaScript libraries commonly used for variant analysis and in data flow queries. Taint tracking path queries --------------------------- Use the following template to create a taint tracking path query: -.. code-block:: ql +.. code-block:: ql /** * @kind path-problem @@ -134,10 +134,10 @@ Files - `File `__, `Folder `__ extends - `Container `__ -- file or folder in the snapshot + `Container `__ -- file or folder in the database - `getBaseName `__ -- the name of the file or folder - - `getRelativePath `__ -- path relative to the snapshot root + - `getRelativePath `__ -- path relative to the database root AST nodes --------- diff --git a/docs/language/learn-ql/javascript/dataflow.rst b/docs/language/learn-ql/javascript/dataflow.rst index 3cd41de6a84..15cd1ad7e46 100644 --- a/docs/language/learn-ql/javascript/dataflow.rst +++ b/docs/language/learn-ql/javascript/dataflow.rst @@ -4,13 +4,13 @@ Analyzing data flow in JavaScript and TypeScript Overview -------- -This topic describes how data flow analysis is implemented in the QL libraries for JavaScript/TypeScript and includes examples to help you write your own data flow queries. -The following sections describe how to utilize the QL libraries for local data flow, global data flow, and taint tracking. +This topic describes how data flow analysis is implemented in the CodeQL libraries for JavaScript/TypeScript and includes examples to help you write your own data flow queries. +The following sections describe how to utilize the libraries for local data flow, global data flow, and taint tracking. As our running example, we will develop a query that identifies command-line arguments that are passed as a file path to the standard Node.js ``readFile`` function. While this is not a problematic pattern as such, it is typical of the kind of reasoning that is frequently used in security queries. -For a more general introduction to modeling data flow in QL, see :doc:`Introduction to data flow analysis in QL <../intro-to-data-flow>`. +For a more general introduction to modeling data flow, see :doc:`Introduction to data flow analysis with CodeQL <../intro-to-data-flow>`. Data flow nodes --------------- @@ -174,7 +174,7 @@ There are two points worth making about the source node API: 2. Strings are not source nodes and cannot be tracked using this API. You can, however, use the ``mayHaveStringValue`` predicate on class ``DataFlow::Node`` to reason about the possible string values flowing into a data flow node. -For a full description of the ``DataFlow::SourceNode`` API, see the `QL JavaScript standard library `__. +For a full description of the ``DataFlow::SourceNode`` API, see the `JavaScript standard library `__. Exercises ~~~~~~~~~ diff --git a/docs/language/learn-ql/javascript/flow-labels.rst b/docs/language/learn-ql/javascript/flow-labels.rst index aa6ded8e7bc..c693319576d 100644 --- a/docs/language/learn-ql/javascript/flow-labels.rst +++ b/docs/language/learn-ql/javascript/flow-labels.rst @@ -393,6 +393,6 @@ string may be an absolute path and whether it may contain ``..`` components. What next? ---------- -- Learn about the QL standard libraries used to write queries for JavaScript in :doc:`Introducing the JavaScript libraries `. +- Learn about the standard CodeQL libraries used to write queries for JavaScript in :doc:`Introducing the JavaScript libraries `. - Find out more about QL in the `QL language handbook `__ and `QL language specification `__. - Learn more about the query console in `Using the query console `__. diff --git a/docs/language/learn-ql/javascript/introduce-libraries-js.rst b/docs/language/learn-ql/javascript/introduce-libraries-js.rst index 0e3a7e5acdd..edac4a8c21b 100644 --- a/docs/language/learn-ql/javascript/introduce-libraries-js.rst +++ b/docs/language/learn-ql/javascript/introduce-libraries-js.rst @@ -1,10 +1,10 @@ -Introducing the QL libraries for JavaScript -=========================================== +Introducing the CodeQL libraries for JavaScript +=============================================== Overview -------- -There is an extensive QL library for analyzing JavaScript code. The classes in this library present the data from a snapshot database in an object-oriented form and provide abstractions and predicates to help you with common analysis tasks. +There is an extensive CodeQL library for analyzing JavaScript code. The classes in this library present the data from a CodeQL database in an object-oriented form and provide abstractions and predicates to help you with common analysis tasks. The library is implemented as a set of QL modules, that is, files with the extension ``.qll``. The module ``javascript.qll`` imports most other standard library modules, so you can include the complete library by beginning your query with: @@ -12,12 +12,12 @@ The library is implemented as a set of QL modules, that is, files with the exten import javascript -The rest of this tutorial briefly summarizes the most important QL classes and predicates provided by this library, including references to the `detailed API documentation `__ where applicable. +The rest of this tutorial briefly summarizes the most important classes and predicates provided by this library, including references to the `detailed API documentation `__ where applicable. Introducing the library ----------------------- -The QL JavaScript library presents information about JavaScript source code at different levels: +The CodeQL library for JavaScript presents information about JavaScript source code at different levels: - **Textual** — classes that represent source code as unstructured text files - **Lexical** — classes that represent source code as a series of tokens and comments @@ -39,12 +39,12 @@ Textual level At its most basic level, a JavaScript code base can simply be viewed as a collection of files organized into folders, where each file is composed of zero or more lines of text. -Note that the textual content of a program is not included in the snapshot database unless you specifically request it during extraction. In particular, snapshots on LGTM do not normally include textual information. +Note that the textual content of a program is not included in the CodeQL database unless you specifically request it during extraction. In particular, databases on LGTM (also known as "snapshots") do not normally include textual information. Files and folders ^^^^^^^^^^^^^^^^^ -In QL, files are represented as entities of class `File `__, and folders as entities of class `Folder `__, both of which are subclasses of class `Container `__. +In the CodeQL libraries, files are represented as entities of class `File `__, and folders as entities of class `Folder `__, both of which are subclasses of class `Container `__. Class `Container `__ provides the following member predicates: @@ -56,7 +56,7 @@ Note that while ``getAFile`` and ``getAFolder`` are declared on class `Container Both files and folders have paths, which can be accessed by the predicate ``Container.getAbsolutePath()``. For example, if ``f`` represents a file with the path ``/home/user/project/src/index.js``, then ``f.getAbsolutePath()`` evaluates to the string ``"/home/user/project/src/index.js"``, while ``f.getParentContainer().getAbsolutePath()`` returns ``"/home/user/project/src"``. -These paths are absolute file system paths. If you want to obtain the path of a file relative to the snapshot source location, use ``Container.getRelativePath()`` instead. Note, however, that a snapshot may contain files that are not located underneath the snapshot source location; for such files, ``getRelativePath()`` will not return anything. +These paths are absolute file system paths. If you want to obtain the path of a file relative to the source location in the CodeQL database, use ``Container.getRelativePath()`` instead. Note, however, that a database may contain files that are not located underneath the source location; for such files, ``getRelativePath()`` will not return anything. The following member predicates of class `Container `__ provide more information about the name of a file or folder: @@ -78,9 +78,9 @@ For example, the following query computes, for each folder, the number of JavaSc Locations ^^^^^^^^^ -Most entities in a snapshot database have an associated source location. Locations are identified by four pieces of information: a file, a start line, a start column, an end line, and an end column. Line and column counts are 1-based (so the first character of a file is at line 1, column 1), and the end position is inclusive. +Most entities in a CodeQL database have an associated source location. Locations are identified by four pieces of information: a file, a start line, a start column, an end line, and an end column. Line and column counts are 1-based (so the first character of a file is at line 1, column 1), and the end position is inclusive. -All entities associated with a source location belong to the QL class `Locatable `__. The location itself is modeled by the QL class `Location `__ and can be accessed through the member predicate ``Locatable.getLocation()``. The `Location `__ class provides the following member predicates: +All entities associated with a source location belong to the class `Locatable `__. The location itself is modeled by the class `Location `__ and can be accessed through the member predicate ``Locatable.getLocation()``. The `Location `__ class provides the following member predicates: - ``Location.getFile()``, ``Location.getStartLine()``, ``Location.getStartColumn()``, ``Location.getEndLine()``, ``Location.getEndColumn()`` return detailed information about the location. - ``Location.getNumLines()`` returns the number of (whole or partial) lines covered by the location. @@ -90,12 +90,12 @@ All entities associated with a source location belong to the QL class `Locatable Lines ^^^^^ -Lines of text in files are represented by the QL class `Line `__. This class offers the following member predicates: +Lines of text in files are represented by the class `Line `__. This class offers the following member predicates: - ``Line.getText()`` returns the text of the line, excluding any terminating newline characters. - ``Line.getTerminator()`` returns the terminator character(s) of the line. The last line in a file may not have any terminator characters, in which case this predicate does not return anything; otherwise it returns either the two-character string ``"\r\n"`` (carriage-return followed by newline), or one of the one-character strings ``"\n"`` (newline), ``"\r"`` (carriage-return), ``"\u2028"`` (Unicode character LINE SEPARATOR), ``"\u2029"`` (Unicode character PARAGRAPH SEPARATOR). -Note that, as mentioned above, the textual representation of the program is not included in the snapshot database by default. +Note that, as mentioned above, the textual representation of the program is not included in the CodeQL database by default. Lexical level ~~~~~~~~~~~~~ @@ -180,9 +180,9 @@ As an example of a query using only lexical information, consider the following Syntactic level ~~~~~~~~~~~~~~~ -The majority of classes in the QL JavaScript library is concerned with representing a JavaScript program as a collection of `abstract syntax trees `__ (ASTs). +The majority of classes in the JavaScript library is concerned with representing a JavaScript program as a collection of `abstract syntax trees `__ (ASTs). -The QL class `ASTNode `__ contains all entities representing nodes in the abstract syntax trees and defines generic tree traversal predicates: +The class `ASTNode `__ contains all entities representing nodes in the abstract syntax trees and defines generic tree traversal predicates: - ``ASTNode.getChild(i)``: returns the ``i``\ th child of this AST node. - ``ASTNode.getAChild()``: returns any child of this AST node. @@ -352,7 +352,7 @@ As an example of how to use expression AST nodes, here is a query that finds exp Functions ^^^^^^^^^ -JavaScript provides several ways of defining functions: in ECMAScript 5, there are function declaration statements and function expressions, and ECMAScript 2015 adds arrow function expressions. These different syntactic forms are represented by the QL classes `FunctionDeclStmt `__ (a subclass of `Stmt `__), `FunctionExpr `__ (a subclass of `Expr `__) and `ArrowFunctionExpr `__ (also a subclass of +JavaScript provides several ways of defining functions: in ECMAScript 5, there are function declaration statements and function expressions, and ECMAScript 2015 adds arrow function expressions. These different syntactic forms are represented by the classes `FunctionDeclStmt `__ (a subclass of `Stmt `__), `FunctionExpr `__ (a subclass of `Expr `__) and `ArrowFunctionExpr `__ (also a subclass of `Expr `__), respectively. All three are subclasses of `Function `__, which provides common member predicates for accessing function parameters or the function body: - ``Function.getId()`` returns the `Identifier `__ naming the function, which may not be defined for function expressions. @@ -389,7 +389,7 @@ As another example, this query finds functions that have two parameters that bin Classes ^^^^^^^ -Classes can be defined either by class declaration statements, represented by the QL class `ClassDeclStmt `__ (which is a subclass of `Stmt `__), or by class expressions, represented by the QL class `ClassExpr `__ (which is a subclass of `Expr `__). Both of these classes are also subclasses of `ClassDefinition `__, which provides common member predicates for accessing the name of a class, its superclass, and its body: +Classes can be defined either by class declaration statements, represented by the CodeQL class `ClassDeclStmt `__ (which is a subclass of `Stmt `__), or by class expressions, represented by the CodeQL class `ClassExpr `__ (which is a subclass of `Expr `__). Both of these classes are also subclasses of `ClassDefinition `__, which provides common member predicates for accessing the name of a class, its superclass, and its body: - ``ClassDefinition.getIdentifier()`` returns the `Identifier `__ naming the function, which may not be defined for class expressions. - ``ClassDefinition.getSuperClass()`` returns the `Expr `__ specifying the superclass, which may not be defined. @@ -400,14 +400,14 @@ Classes can be defined either by class declaration statements, represented by th Note that class fields are not a standard language feature yet, so details of their representation may change. -Method definitions are represented by the QL class `MethodDefinition `__, which (like its counterpart `FieldDefinition `__ for fields) is a subclass of `MemberDefinition `__. That class provides the following important member predicates: +Method definitions are represented by the class `MethodDefinition `__, which (like its counterpart `FieldDefinition `__ for fields) is a subclass of `MemberDefinition `__. That class provides the following important member predicates: - ``MemberDefinition.isStatic()``: holds if this is a static member. - ``MemberDefinition.isComputed()``: holds if the name of this member is computed at runtime. - ``MemberDefinition.getName()``: gets the name of this member if it can be determined statically. - ``MemberDefinition.getInit()``: gets the initializer of this field; for methods, the initializer is a function expressions, for fields it may be an arbitrary expression, and may be undefined. -There are three QL classes for modeling special methods: `ConstructorDefinition `__ models constructors, while `GetterMethodDefinition `__ and `SetterMethodDefinition `__ model getter and setter methods, respectively. +There are three classes for modeling special methods: `ConstructorDefinition `__ models constructors, while `GetterMethodDefinition `__ and `SetterMethodDefinition `__ model getter and setter methods, respectively. Declarations and binding patterns ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -472,7 +472,7 @@ As an example of a query involving properties, consider the following query that Modules ^^^^^^^ -The JavaScript library has support for working with ECMAScript 2015 modules, as well as legacy CommonJS modules (still commonly employed by Node.js code bases) and AMD-style modules. The QL classes `ES2015Module `__, `NodeModule `__, and `AMDModule `__ represent these three types of modules, and all three extend the common superclass `Module `__. +The JavaScript library has support for working with ECMAScript 2015 modules, as well as legacy CommonJS modules (still commonly employed by Node.js code bases) and AMD-style modules. The classes `ES2015Module `__, `NodeModule `__, and `AMDModule `__ represent these three types of modules, and all three extend the common superclass `Module `__. The most important member predicates defined by `Module `__ are: @@ -485,12 +485,12 @@ Moreover, there is a class `Import `__, `Variable `__, `VarDecl `__ and `VarAccess `__, respectively. +Name binding is modeled in the JavaScript libraries using four concepts: *scopes*, *variables*, *variable declarations*, and *variable accesses*, represented by the classes `Scope `__, `Variable `__, `VarDecl `__ and `VarAccess `__, respectively. Scopes ^^^^^^ -In ECMAScript 5, there are three kinds of scopes: the global scope (one per program), function scopes (one per function), and catch clause scopes (one per ``catch`` clause). These three kinds of scopes are represented by the QL classes `GlobalScope `__, `FunctionScope `__ and `CatchScope `__. ECMAScript 2015 adds block scopes for ``let``-bound variables, which are also represented by QL class `Scope `__, class expression scopes (`ClassExprScope `__), +In ECMAScript 5, there are three kinds of scopes: the global scope (one per program), function scopes (one per function), and catch clause scopes (one per ``catch`` clause). These three kinds of scopes are represented by the classes `GlobalScope `__, `FunctionScope `__ and `CatchScope `__. ECMAScript 2015 adds block scopes for ``let``-bound variables, which are also represented by class `Scope `__, class expression scopes (`ClassExprScope `__), and module scopes (`ModuleScope `__). Class `Scope `__ provides the following API: @@ -510,7 +510,7 @@ It is important not to confuse variables and their declarations: local variables Variable declarations and accesses ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Variables may be declared by variable declarators, by function declaration statements and expressions, by class declaration statements or expressions, or by parameters of functions and ``catch`` clauses. While these declarations differ in their syntactic form, in each case there is an identifier naming the declared variable. We consider that identifier to be the declaration proper, and assign it the QL class `VarDecl `__. Identifiers that reference a variable, on the other hand, are given the QL class `VarAccess `__. +Variables may be declared by variable declarators, by function declaration statements and expressions, by class declaration statements or expressions, or by parameters of functions and ``catch`` clauses. While these declarations differ in their syntactic form, in each case there is an identifier naming the declared variable. We consider that identifier to be the declaration proper, and assign it the class `VarDecl `__. Identifiers that reference a variable, on the other hand, are given the class `VarAccess `__. The most important predicates involving variables, their declarations, and their accesses are as follows: @@ -538,9 +538,9 @@ As an example, consider the following query which finds distinct function declar Control flow ~~~~~~~~~~~~ -A different program representation in terms of intraprocedural control flow graphs (CFGs) is provided by the QL classes in library `CFG.qll `__. +A different program representation in terms of intraprocedural control flow graphs (CFGs) is provided by the classes in library `CFG.qll `__. -Class `ControlFlowNode `__ represents a single node in the control flow graph, which is either an expression, a statement, or a synthetic control flow node. Note that `Expr `__ and `Stmt `__ do not inherit from `ControlFlowNode `__ at the QL level, although their entity types are compatible, so you can explicitly cast from one to the other if you need to map between the AST-based and the CFG-based program representations. +Class `ControlFlowNode `__ represents a single node in the control flow graph, which is either an expression, a statement, or a synthetic control flow node. Note that `Expr `__ and `Stmt `__ do not inherit from `ControlFlowNode `__ at the CodeQL level, although their entity types are compatible, so you can explicitly cast from one to the other if you need to map between the AST-based and the CFG-based program representations. There are two kinds of synthetic control flow nodes: entry nodes (class `ControlFlowEntryNode `__), which represent the beginning of a top-level or function, and exit nodes (class `ControlFlowExitNode `__), which represent their end. They do not correspond to any AST nodes, but simply serve as the unique entry point and exit point of a control flow graph. Entry and exit nodes can be accessed through the predicates ``StmtContainer.getEntry()`` and ``StmtContainer.getExit()``. @@ -578,7 +578,7 @@ Data flow Definitions and uses ^^^^^^^^^^^^^^^^^^^^ -Library `DefUse.qll `__ provides QL classes and predicates to determine `def-use `__ relationships between definitions and uses of variables. +Library `DefUse.qll `__ provides classes and predicates to determine `def-use `__ relationships between definitions and uses of variables. Classes `VarDef `__ and `VarUse `__ contain all expressions that define and use a variable, respectively. For the former, you can use predicate ``VarDef.getAVariable()`` to find out which variables are defined by a given variable definition (recall that destructuring assignments in ECMAScript 2015 define several variables at the same time). Similarly, predicate ``VarUse.getVariable()`` returns the (single) variable being accessed by a variable use. @@ -645,7 +645,7 @@ Note that the data flow modeling in this library is intraprocedural, that is, fl Type inference ~~~~~~~~~~~~~~ -The library ``semmle.javascript.dataflow.TypeInference`` implements a simple type inference for JavaScript based on intraprocedural, heap-insensitive flow analysis. Basically, the inference algorithm approximates the possible concrete runtime values of variables and expressions as sets of abstract values (represented by QL class `AbstractValue `__), each of which stands for a set of concrete values. +The library ``semmle.javascript.dataflow.TypeInference`` implements a simple type inference for JavaScript based on intraprocedural, heap-insensitive flow analysis. Basically, the inference algorithm approximates the possible concrete runtime values of variables and expressions as sets of abstract values (represented by the class `AbstractValue `__), each of which stands for a set of concrete values. For example, there is an abstract value representing all non-zero numbers, and another representing all non-empty strings except for those that can be converted to a number. Both of these abstract values are fairly coarse approximations that represent very large sets of concrete values. @@ -657,7 +657,7 @@ Each indefinite abstract value is associated with a string value describing the To check whether an abstract value is indefinite, you can use the ``isIndefinite`` member predicate. Its single argument describes the cause of imprecision. -Each abstract value has one or more associated types (QL class `InferredType `__ corresponding roughly to the type tags computed by the ``typeof`` operator. The types are ``null``, ``undefined``, ``boolean``, ``number``, ``string``, ``function``, ``class``, ``date`` and ``object``. +Each abstract value has one or more associated types (CodeQL class `InferredType `__ corresponding roughly to the type tags computed by the ``typeof`` operator. The types are ``null``, ``undefined``, ``boolean``, ``number``, ``string``, ``function``, ``class``, ``date`` and ``object``. To access the results of the type inference, use class `DataFlow::AnalyzedNode `__: any `DataFlow::Node `__ can be cast to this class, and additionally there is a convenience predicate ``Expr::analyze`` that maps expressions directly to their corresponding ``AnalyzedNode``\ s. @@ -820,12 +820,12 @@ The most important classes include (all in module ``HTTP``): - ``CookieDefinition``: an expression that sets a cookie in an HTTP response. - ``RequestInputAccess``: an expression that accesses user-controlled request data. -For each framework library, there is a corresponding QL library (for example ``semmle.javacript.frameworks.Express``) that instantiates the above classes for that framework and adds framework-specific classes. +For each framework library, there is a corresponding CodeQL library (for example ``semmle.javacript.frameworks.Express``) that instantiates the above classes for that framework and adds framework-specific classes. Node.js ^^^^^^^ -The ``semmle.javascript.NodeJS`` library provides support for working with `Node.js `__ modules through the following QL classes: +The ``semmle.javascript.NodeJS`` library provides support for working with `Node.js `__ modules through the following classes: - `NodeModule `__: a top-level that defines a Node.js module; see the section on `Modules <#modules>`__ for more information. - `Require `__: a call to the special ``require`` function that imports a module. @@ -844,7 +844,7 @@ As an example of the use of these classes, here is a query that counts for every NPM ^^^ -The ``semmle.javascript.NPM`` library provides support for working with `NPM `__ packages through the following QL classes: +The ``semmle.javascript.NPM`` library provides support for working with `NPM `__ packages through the following classes: - `PackageJSON `__: a ``package.json`` file describing an NPM package; various getter predicates are available for accessing detailed information about the package, which are described in the `online API documentation `__. - `BugTrackerInfo `__, `ContributorInfo `__, `RepositoryInfo `__: these classes model parts of the ``package.json`` file providing information on bug tracking systems, contributors and repositories. @@ -903,7 +903,7 @@ Miscellaneous Externs ^^^^^^^ -The ``semmle.javascript.Externs`` library provides support for working with `externs `__ through the following QL classes: +The ``semmle.javascript.Externs`` library provides support for working with `externs `__ through the following classes: - `ExternalDecl `__: common superclass modeling all different kinds of externs declarations; it defines two member predicates: @@ -938,7 +938,7 @@ JSDoc The ``semmle.javascript.JSDoc`` library provides support for working with `JSDoc comments `__. Documentation comments are parsed into an abstract syntax tree representation closely following the format employed by the `Doctrine `__ JSDoc parser. -A JSDoc comment as a whole is represented by an entity of QL class `JSDoc `__, while individual tags are represented by QL class `JSDocTag `__. Important member predicates of these two classes include: +A JSDoc comment as a whole is represented by an entity of class `JSDoc `__, while individual tags are represented by class `JSDocTag `__. Important member predicates of these two classes include: - ``JSDoc.getDescription()`` returns the descriptive header of the JSDoc comment, if any. - ``JSDoc.getComment()`` maps the `JSDoc `__ entity to its underlying `Comment `__ entity. @@ -948,7 +948,7 @@ A JSDoc comment as a whole is represented by an entity of QL class `JSDoc `__ and its subclasses, which again represent type expressions as abstract syntax trees. Examples of type expressions are `JSDocAnyTypeExpr `__, representing the "any" type ``*``, or `JSDocNullTypeExpr `__, representing the null type. +Types in JSDoc comments are represented by the class `JSDocTypeExpr `__ and its subclasses, which again represent type expressions as abstract syntax trees. Examples of type expressions are `JSDocAnyTypeExpr `__, representing the "any" type ``*``, or `JSDocNullTypeExpr `__, representing the null type. As an example, here is a query that finds ``@param`` tags that do not specify the name of the documented parameter: @@ -979,9 +979,9 @@ However, unlike HTML, JSX is interleaved with JavaScript, hence `JSXElement `__ files that were processed by the JavaScript extractor when building the snapshot database. +The ``semmle.javascript.JSON`` library provides support for working with `JSON `__ files that were processed by the JavaScript extractor when building the CodeQL database. -JSON files are modeled as trees of JSON values. Each JSON value is represented by an entity of QL class `JSONValue `__, which provides the following member predicates: +JSON files are modeled as trees of JSON values. Each JSON value is represented by an entity of class `JSONValue `__, which provides the following member predicates: - ``JSONValue.getParent()`` returns the JSON object or array in which this value occurs. - ``JSONValue.getChild(i)`` returns the ``i``\ th child of this JSON object or array. @@ -1000,16 +1000,16 @@ Class `JSONValue `__. Similar to `ASTNode `__, class `RegExpTerm `__ provides member predicates ``getParent()`` and ``getChild(i)`` to navigate the structure of the syntax tree. +The ``semmle.javascript.Regexp`` library provides support for working with regular expression literals. The syntactic structure of regular expression literals is represented as an abstract syntax tree of regular expression terms, modeled by the class `RegExpTerm `__. Similar to `ASTNode `__, class `RegExpTerm `__ provides member predicates ``getParent()`` and ``getChild(i)`` to navigate the structure of the syntax tree. Various subclasses of `RegExpTerm `__ model different kinds of regular expression constructs and operators; see `the API documentation `__ for details. YAML ^^^^ -The ``semmle.javascript.YAML`` library provides support for working with `YAML `__ files that were processed by the JavaScript extractor when building the snapshot database. +The ``semmle.javascript.YAML`` library provides support for working with `YAML `__ files that were processed by the JavaScript extractor when building the CodeQL database. -YAML files are modeled as trees of YAML nodes. Each YAML node is represented by an entity of QL class `YAMLNode `__, which provides, among others, the following member predicates: +YAML files are modeled as trees of YAML nodes. Each YAML node is represented by an entity of class `YAMLNode `__, which provides, among others, the following member predicates: - ``YAMLNode.getParentNode()`` returns the YAML collection in which this node is syntactically nested. - ``YAMLNode.getChildNode(i)`` returns the ``i``\ th child node of this node, ``YAMLNode.getAChildNode()`` returns any child node of this node. @@ -1029,6 +1029,6 @@ Predicate ``YAMLMapping.maps(key, value)`` models the key-value relation represe What next? ---------- -- Learn about the QL standard libraries used to write queries for TypeScript in :doc:`Introducing the TypeScript libraries `. +- Learn about the standard CodeQL libraries used to write queries for TypeScript in :doc:`Introducing the TypeScript libraries `. - Find out more about QL in the `QL language handbook `__ and `QL language specification `__. - Learn more about the query console in `Using the query console `__. diff --git a/docs/language/learn-ql/javascript/introduce-libraries-ts.rst b/docs/language/learn-ql/javascript/introduce-libraries-ts.rst index a31bf9ffad6..4884cc8ce09 100644 --- a/docs/language/learn-ql/javascript/introduce-libraries-ts.rst +++ b/docs/language/learn-ql/javascript/introduce-libraries-ts.rst @@ -1,16 +1,16 @@ -Introducing the QL libraries for TypeScript -=========================================== +Introducing the CodeQL libraries for TypeScript +=============================================== Overview -------- -Support for analyzing TypeScript code is bundled with the JavaScript QL libraries, so you can include the full TypeScript library by importing the ``javascript.qll`` module: +Support for analyzing TypeScript code is bundled with the CodeQL libraries for JavaScript, so you can include the full TypeScript library by importing the ``javascript.qll`` module: .. code-block:: ql import javascript -The :doc:`QL library introduction for JavaScript ` covers most of this library, and is also relevant for TypeScript analysis. This document supplements the JavaScript QL documentation with the TypeScript-specific classes and predicates. +The :doc:`CodeQL library introduction for JavaScript ` covers most of this library, and is also relevant for TypeScript analysis. This document supplements the JavaScript documentation with the TypeScript-specific classes and predicates. Syntax ------ @@ -124,7 +124,7 @@ Select expressions that cast a value to a type parameter: Classes and interfaces ~~~~~~~~~~~~~~~~~~~~~~ -The QL class `ClassOrInterface `__ is a common supertype of classes and interfaces, and provides some TypeScript-specific member predicates: +The CodeQL class `ClassOrInterface `__ is a common supertype of classes and interfaces, and provides some TypeScript-specific member predicates: - ``ClassOrInterface.isAbstract()`` holds if this is an interface or a class with the ``abstract`` modifier. - ``ClassOrInterface.getASuperInterface()`` gets a type from the ``implements`` clause of a class or from the ``extends`` clause of an interface. @@ -134,7 +134,7 @@ The QL class `ClassOrInterface `__. -Also see the documentation for classes in the `Introduction to the QL libraries for JavaScript `__. +Also see the documentation for classes in the `Introduction to the CodeQL libraries for JavaScript `__. To select the type references to a class or an interface, use ``getTypeName()``. @@ -175,6 +175,8 @@ Ambient nodes are mostly ignored by control flow and data flow analysis. The out Static type information ----------------------- +.. TODO: Remove link to QL command-line tools below? + Static type information and global name binding is available for projects with "full" TypeScript extraction enabled. This option is enabled by default for projects on LGTM.com. If you are using the `QL command-line tools `__, you must enable it by passing ``--typescript-full`` to the JavaScript extractor. For further information on customizing calls to the extractor, see `Customizing JavaScript extraction `__. **Note:** Without full extraction, the classes and predicates described in this section are empty. @@ -262,7 +264,7 @@ Additionally, ``Type`` has the following subclasses which overlap partially with Canonical names and named types ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``CanonicalName`` is a QL class representing a qualified name relative to a root scope, such as a module or the global scope. It typically represents an entity such as a type, namespace, variable, or function. ``TypeName`` and ``Namespace`` are subclasses of this class. +``CanonicalName`` is a CodeQL class representing a qualified name relative to a root scope, such as a module or the global scope. It typically represents an entity such as a type, namespace, variable, or function. ``TypeName`` and ``Namespace`` are subclasses of this class. Canonical names can be recognized using the ``hasQualifiedName`` predicate: @@ -274,7 +276,7 @@ For convenience, this predicate is also available on other classes, such as ``Ty Function types ~~~~~~~~~~~~~~ -There is no QL class for function types, as any type with a call or construct signature is usable as a function. The type ``CallSignatureType`` represents such a signature (with or without the ``new`` keyword). +There is no CodeQL class for function types, as any type with a call or construct signature is usable as a function. The type ``CallSignatureType`` represents such a signature (with or without the ``new`` keyword). Signatures can be obtained in several ways: @@ -353,7 +355,7 @@ TypeScript also allows you to import types and namespaces, and give them local n The local name ``B`` is represented as a `LocalTypeName `__ named ``B``, restricted to just the file containing the import. An import statement can also introduce a `Variable `__ and a `LocalNamespaceName `__. -The following table shows the relevant QL classes for working with each kind of name. The classes are described in more detail below. +The following table shows the relevant classes for working with each kind of name. The classes are described in more detail below. +-----------+------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------+ | Kind | Local alias | Canonical name | Definition | Access | @@ -419,7 +421,7 @@ Find imported names that are used as both a type and a value: Namespace names ~~~~~~~~~~~~~~~ -Namespaces are represented by the QL classes `Namespace `__ and `LocalNamespaceName `__. The `NamespaceDefinition `__ class represents a syntactic definition of a namespace, which includes ordinary namespace declarations as well as enum declarations. +Namespaces are represented by the classes `Namespace `__ and `LocalNamespaceName `__. The `NamespaceDefinition `__ class represents a syntactic definition of a namespace, which includes ordinary namespace declarations as well as enum declarations. Note that these classes deal exclusively with namespaces referenced from inside type annotations, not through expressions. @@ -443,6 +445,6 @@ A `LocalNamespaceName `. +- Learn about the standard CodeQL libraries used to write queries for JavaScript in :doc:`Introducing the JavaScript libraries `. - Find out more about QL in the `QL language handbook `__ and `QL language specification `__. - Learn more about the query console in `Using the query console `__. \ No newline at end of file diff --git a/docs/language/learn-ql/javascript/ql-for-javascript.rst b/docs/language/learn-ql/javascript/ql-for-javascript.rst index 5df6e2581e5..d2d864cc072 100644 --- a/docs/language/learn-ql/javascript/ql-for-javascript.rst +++ b/docs/language/learn-ql/javascript/ql-for-javascript.rst @@ -1,5 +1,5 @@ -QL for JavaScript -================= +CodeQL for JavaScript +===================== .. toctree:: :glob: @@ -13,25 +13,25 @@ QL for JavaScript ast-class-reference dataflow-cheat-sheet -These documents provide an overview of the QL JavaScript and TypeScript standard libraries and show examples of how to use them. +These documents provide an overview of the CodeQL libraries for JavaScript and TypeScript and show examples of how to use them. -- `Basic JavaScript QL query `__ describes how to write and run queries using LGTM. +- `Basic JavaScript query `__ describes how to write and run queries using LGTM. -- :doc:`Introducing the QL libraries for JavaScript `: an overview of the standard libraries used to write queries for JavaScript code. There is an extensive QL library for analyzing JavaScript code. This tutorial briefly summarizes the most important QL classes and predicates provided by this library. +- :doc:`Introducing the CodeQL libraries for JavaScript ` introduces the standard libraries used to write queries for JavaScript code. There is an extensive CodeQL library for analyzing JavaScript code. This tutorial briefly summarizes the most important classes and predicates provided by this library. -- :doc:`Introducing the QL libraries for TypeScript `: an overview of the standard libraries used to write queries for TypeScript code. +- :doc:`Introducing the CodeQL libraries for TypeScript ` introduces the standard libraries used to write queries for TypeScript code. -- :doc:`Analyzing data flow in JavaScript/TypeScript `: demonstrates how to write queries using the standard QL for JavaScript/TypeScript data flow and taint tracking libraries. +- :doc:`Analyzing data flow in JavaScript/TypeScript ` demonstrates how to write queries using the standard data flow and taint tracking libraries for JavaScript/TypeScript. -- :doc:`Advanced data-flow analysis using flow labels `: shows a more advanced example of data flow analysis using flow labels. +- :doc:`Advanced data-flow analysis using flow labels ` shows a more advanced example of data flow analysis using flow labels. -- :doc:`AST class reference `: an overview of all AST classes in the QL standard library for JavaScript. +- :doc:`AST class reference ` gives an overview of all AST classes in the standard CodeQL library for JavaScript. -- :doc:`Data flow cheat sheet `: bits of QL commonly used for variant analysis and in data flow queries. +- :doc:`Data flow cheat sheet ` lists parts of the CodeQL libraries that are commonly used for variant analysis and in data flow queries. Other resources --------------- -- For examples of how to query common JavaScript elements, see the `JavaScript QL cookbook `__ -- For the queries used in LGTM, display a `JavaScript query `__ and click **Open in query console** to see the QL code used to find alerts. -- For more information about the JavaScript QL library see the `QL library for JavaScript `__. +- For examples of how to query common JavaScript elements, see the `JavaScript cookbook `__. +- For the queries used in LGTM, display a `JavaScript query `__ and click **Open in query console** to see the code used to find alerts. +- For more information about the library for JavaScript see the `CodeQL library for JavaScript `__. diff --git a/docs/language/learn-ql/javascript/type-tracking.rst b/docs/language/learn-ql/javascript/type-tracking.rst index d6bcb9e7a7c..f0dadd9ef70 100644 --- a/docs/language/learn-ql/javascript/type-tracking.rst +++ b/docs/language/learn-ql/javascript/type-tracking.rst @@ -1,8 +1,8 @@ Tutorial: API modelling using type tracking =========================================== -This tutorial demonstrates how to build a simple model of the Firebase API in QL -using the JavaScript type-tracking library. +This tutorial demonstrates how to build a simple model of the Firebase API +using the CodeQL type-tracking library for JavaScript. The type-tracking library makes it possible to track values through properties and function calls, usually to recognize method calls and properties accessed on a specific type of object. @@ -89,7 +89,7 @@ For instance, ``firebaseSetterCall()`` fails to find anything in this example: var ref = getDatabase().ref("forecast"); ref.set("Rain"); -Notice that the QL predicate ``firebaseDatabase()`` still finds the call to ``firebase.database()``, +Notice that the predicate ``firebaseDatabase()`` still finds the call to ``firebase.database()``, but not the ``getDatabase()`` call. This means ``firebaseRef()`` has no result, which in turn means ``firebaseSetterCall()`` has no result. From e2b446db199d66105ec71a4c5fcb4276bdcac99a Mon Sep 17 00:00:00 2001 From: Shati Patel Date: Mon, 28 Oct 2019 17:23:31 +0000 Subject: [PATCH 057/148] Docs: Update Python --- .../language/learn-ql/python/control-flow.rst | 8 ++-- docs/language/learn-ql/python/functions.rst | 6 +-- .../python/introduce-libraries-python.rst | 40 +++++++++---------- .../learn-ql/python/pointsto-type-infer.rst | 14 +++---- .../learn-ql/python/ql-for-python.rst | 26 ++++++------ .../python/statements-expressions.rst | 6 +-- .../learn-ql/python/taint-tracking.rst | 8 ++-- 7 files changed, 50 insertions(+), 58 deletions(-) diff --git a/docs/language/learn-ql/python/control-flow.rst b/docs/language/learn-ql/python/control-flow.rst index 45b30f51f21..fc41f59c933 100644 --- a/docs/language/learn-ql/python/control-flow.rst +++ b/docs/language/learn-ql/python/control-flow.rst @@ -1,12 +1,12 @@ Tutorial: Control flow analysis =============================== -In order to analyze the `Control-flow graph `__ of a ``Scope`` we can use the two QL classes ``ControlFlowNode`` and ``BasicBlock``. These classes allow you to ask such questions as "can you reach point A from point B?" or "Is it possible to reach point B *without* going through point A?". To report results we use the class ``AstNode``, which represents a syntactic element and corresponds to the source code - allowing the results of the query to be more easily understood. +To analyze the `Control-flow graph `__ of a ``Scope`` we can use the two CodeQL classes ``ControlFlowNode`` and ``BasicBlock``. These classes allow you to ask such questions as "can you reach point A from point B?" or "Is it possible to reach point B *without* going through point A?". To report results we use the class ``AstNode``, which represents a syntactic element and corresponds to the source code - allowing the results of the query to be more easily understood. The ``ControlFlowNode`` class ----------------------------- -The ``ControlFlowNode`` class represents nodes in the control flow graph. There is a one-to-many relation between AST nodes and control flow nodes. Each syntactic element, the ``AstNode,`` maps to zero, one or many ``ControlFlowNode`` classes, but each ControlFlowNode maps to exactly one ``AstNode``. +The ``ControlFlowNode`` class represents nodes in the control flow graph. There is a one-to-many relation between AST nodes and control flow nodes. Each syntactic element, the ``AstNode,`` maps to zero, one, or many ``ControlFlowNode`` classes, but each ``ControlFlowNode`` maps to exactly one ``AstNode``. To show why this complex relation is required consider the following Python code: @@ -21,7 +21,7 @@ To show why this complex relation is required consider the following Python code There are many paths through the above code. There are three different paths through the call to ``close_resource();`` one normal path, one path that breaks out of the loop, and one path where an exception is raised by ``might_raise()``. (An annotated flow graph can be seen :doc:`here `.) -The simplest use of the ``ControlFlowNode`` and ``AstNode`` classes is to find unreachable code. There is one ``ControlFlowNode`` per path through any ``AstNode`` and any ``AstNode`` that is unreachable has no paths flowing through it; therefore any ``AstNode`` without a corresponding ``ControlFlowNode`` is unreachable. +The simplest use of the ``ControlFlowNode`` and ``AstNode`` classes is to find unreachable code. There is one ``ControlFlowNode`` per path through any ``AstNode`` and any ``AstNode`` that is unreachable has no paths flowing through it. Therefore, any ``AstNode`` without a corresponding ``ControlFlowNode`` is unreachable. **Unreachable AST nodes** @@ -103,5 +103,5 @@ Combining these conditions we get: What next? ---------- -- Experiment with the worked examples in the QL for Python tutorial topic: :doc:`Taint tracking and data flow analysis in Python `. +- Experiment with the worked examples in the tutorial topic :doc:`Taint tracking and data flow analysis in Python `. - Find out more about QL in the `QL language handbook `__ and `QL language specification `__. diff --git a/docs/language/learn-ql/python/functions.rst b/docs/language/learn-ql/python/functions.rst index 7e33b77f5fc..79fcfa270d9 100644 --- a/docs/language/learn-ql/python/functions.rst +++ b/docs/language/learn-ql/python/functions.rst @@ -1,14 +1,14 @@ Tutorial: Functions =================== -This example uses the standard QL class ``Function`` (see :doc:`Introducing the Python libraries `). +This example uses the standard CodeQL class ``Function`` (see :doc:`Introducing the Python libraries `). Finding all functions called "get..." ------------------------------------- In this example we look for all the "getters" in a program. Programmers moving to Python from Java are often tempted to write lots of getter and setter methods, rather than use properties. We might want to find those methods. -Using the member predicate ``Function.getName()``, we can list all of the getter functions in a snapshot: +Using the member predicate ``Function.getName()``, we can list all of the getter functions in a database: Tip @@ -77,5 +77,5 @@ In a later tutorial we will see how to use the type-inference library to find ca What next? ---------- -- Experiment with the worked examples in the QL for Python tutorial topics: :doc:`Statements and expressions `, :doc:`Control flow `, :doc:`Points-to analysis and type inference `. +- Experiment with the worked examples in the following tutorial topics: :doc:`Statements and expressions `, :doc:`Control flow `, and :doc:`Points-to analysis and type inference `. - Find out more about QL in the `QL language handbook `__ and `QL language specification `__. diff --git a/docs/language/learn-ql/python/introduce-libraries-python.rst b/docs/language/learn-ql/python/introduce-libraries-python.rst index 4314cf36913..60d965a461b 100644 --- a/docs/language/learn-ql/python/introduce-libraries-python.rst +++ b/docs/language/learn-ql/python/introduce-libraries-python.rst @@ -1,39 +1,35 @@ -Introducing the QL libraries for Python -======================================= +Introducing the CodeQL libraries for Python +=========================================== -These libraries have been created to help you analyze Python code, providing an object-oriented layer on top of the raw data in the snapshot database. They are written in standard QL. - -The QL libraries all have a ``.qll`` extension, to signify that they contain QL library code but no actual queries. Each file contains a QL class or hierarchy of classes. - -You can include all of the standard libraries by beginning each query with this statement: +There is an extensive library for analyzing CodeQL databases extracted from Python projects. The classes in this library present the data from a database in an object-oriented form and provide abstractions and predicates to help you with common analysis tasks. The library is implemented as a set of QL modules, that is, files with the extension ``.qll``. The module ``python.qll`` imports all the core Python library modules, so you can include the complete library by beginning your query with: .. code-block:: ql import python -The rest of this tutorial summarizes the contents of the standard QL libraries. We recommend that you read this and then work through the practical examples in the Python tutorials shown at the end of the page. +The rest of this tutorial summarizes the contents of the standard libraries for Python. We recommend that you read this and then work through the practical examples in the tutorials shown at the end of the page. Overview of the library ----------------------- -The QL Python library incorporates a large number of classes, each class corresponding either to one kind of entity in Python source code or to an entity that can be derived form the source code using static analysis. These classes can be divided into four categories: +The CodeQL library for Python incorporates a large number of classes. Each class corresponds either to one kind of entity in Python source code or to an entity that can be derived form the source code using static analysis. These classes can be divided into four categories: - **Syntactic** - classes that represent entities in the Python source code. - **Control flow** - classes that represent entities from the control flow graphs. - **Type inference** - classes that represent the inferred values and types of entities in the Python source code. -- **Taint tracking** - classes that represent the source, sinks and kinds of taint used to implement taint-tracking queries. +- **Taint tracking** - classes that represent the source, sinks and kinds of taint used to implement taint-tracking queries. Syntactic classes ~~~~~~~~~~~~~~~~~ -This part of the library represents the Python source code. The ``Module``, ``Class`` and ``Function`` classes correspond to Python modules, classes and functions respectively, collectively these are known as ``Scope`` classes. Each ``Scope`` contains a list of statements each of which is represented by a subclass of the class ``Stmt``. Statements themselves can contain other statements or expressions which are represented by subclasses of ``Expr``. Finally, there are a few additional classes for the parts of more complex expressions such as list comprehensions. Collectively these classes are subclasses of ``AstNode`` and form an `Abstract syntax tree `__ (AST). The root of each AST is a ``Module``. +This part of the library represents the Python source code. The ``Module``, ``Class``, and ``Function`` classes correspond to Python modules, classes, and functions respectively, collectively these are known as ``Scope`` classes. Each ``Scope`` contains a list of statements each of which is represented by a subclass of the class ``Stmt``. Statements themselves can contain other statements or expressions which are represented by subclasses of ``Expr``. Finally, there are a few additional classes for the parts of more complex expressions such as list comprehensions. Collectively these classes are subclasses of ``AstNode`` and form an `Abstract syntax tree `__ (AST). The root of each AST is a ``Module``. `Symbolic information `__ is attached to the AST in the form of variables (represented by the class ``Variable``). Scope ^^^^^ -A Python program is a group of modules. Technically a module is just a list of statements, but we often think of it as composed of classes and functions. These top-level entities, the module, class and function are represented by the three classes (`Module `__, `Class `__ and `Function `__ which are all subclasses of ``Scope``. +A Python program is a group of modules. Technically a module is just a list of statements, but we often think of it as composed of classes and functions. These top-level entities, the module, class, and function are represented by the three CodeQL classes (`Module `__, `Class `__ and `Function `__ which are all subclasses of ``Scope``. - ``Scope`` @@ -65,7 +61,7 @@ A statement is represented by the `Stmt `__ class representing the ``for`` statement has a number of member predicates to access its parts: +The `For `__ class representing the ``for`` statement has a number of member predicates to access its parts: - ``getTarget()`` returns the ``Expr`` representing the variable ``var``. - ``getIter()`` returns the ``Expr`` resenting the variable ``seq``. @@ -98,7 +94,7 @@ As an example, to find expressions of the form ``a+2`` where the left is a simpl Variable ^^^^^^^^ -Variables are represented by the `Variable `__ class in the Python QL library. There are two subclasses, ``LocalVariable`` for function-level and class-level variables and ``GlobalVariable`` for module-level variables. +Variables are represented by the `Variable `__ class in the CodeQL library. There are two subclasses, ``LocalVariable`` for function-level and class-level variables and ``GlobalVariable`` for module-level variables. Other source code elements ^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -108,7 +104,7 @@ Although the meaning of the program is encoded by the syntactic elements, ``Scop Examples ^^^^^^^^ -Each syntactic element in Python source is recorded in the snapshot. These can be queried via the corresponding class. Let us start with a couple of simple examples. +Each syntactic element in Python source is recorded in the CodeQL database. These can be queried via the corresponding class. Let us start with a couple of simple examples. 1. Finding all finally blocks ''''''''''''''''''''''''''''' @@ -137,13 +133,13 @@ A block that does nothing is one that contains no statements except ``pass`` sta not exists(Stmt s | s = ex.getAStmt() | not s instanceof Pass) -where ``ex`` is an ``ExceptStmt`` and ``Pass`` is the class representing ``pass`` statements. Instead of using the double negative, **"no**\ *statements that are*\ **not**\ *pass statements"*, this can also be expressed positively, "all statements must be pass statements." The positive form is expressed in QL using the ``forall`` quantifier: +where ``ex`` is an ``ExceptStmt`` and ``Pass`` is the class representing ``pass`` statements. Instead of using the double negative, "**no** \ *statements that are* \ **not** \ *pass statements"*, this can also be expressed positively, *"all statements must be pass statements."* The positive form is expressed using the ``forall`` quantifier: .. code-block:: ql forall(Stmt s | s = ex.getAStmt() | s instanceof Pass) -Both forms are equivalent. Using the positive QL expression, the whole query looks like this: +Both forms are equivalent. Using the positive expression, the whole query looks like this: **Find pass-only ``except`` blocks** @@ -160,9 +156,9 @@ Both forms are equivalent. Using the positive QL expression, the whole query loo Summary ^^^^^^^ -The most commonly used standard QL library classes in the syntactic part of the library are organized as follows: +The most commonly used standard classes in the syntactic part of the library are organized as follows: -``Module``, ``Class``, ``Function``, ``Stmt`` and ``Expr`` - they are all subclasses of `AstNode `__. +``Module``, ``Class``, ``Function``, ``Stmt``, and ``Expr`` - they are all subclasses of `AstNode `__. Abstract syntax tree '''''''''''''''''''' @@ -293,7 +289,7 @@ The classes in the control-flow part of the library are: Type-inference classes ---------------------- -The QL library for Python also supplies some classes for accessing the inferred types of values. The classes ``Value`` and ``ClassValue`` allow you to query the possible classes that an expression may have at runtime. For example, which ``ClassValue``\ s are iterable can be determined using the query: +The CodeQL library for Python also supplies some classes for accessing the inferred types of values. The classes ``Value`` and ``ClassValue`` allow you to query the possible classes that an expression may have at runtime. For example, which ``ClassValue``\ s are iterable can be determined using the query: **Find iterable "ClassValue"s** @@ -321,7 +317,7 @@ These classes are explained in more detail in :doc:`Tutorial: Points-to analysis Taint-tracking classes ---------------------- -The QL library for Python also supplies classes to specify taint-tracking analyses. The ``Configuration`` class can be overrridden to specify a taint-tracking analysis, by specifying source, sinks, sanitizers and additional flow steps. For those analyses that require additional types of taint to be tracked the ``TaintKind`` class can be overridden. +The CodeQL library for Python also supplies classes to specify taint-tracking analyses. The ``Configuration`` class can be overridden to specify a taint-tracking analysis, by specifying source, sinks, sanitizers and additional flow steps. For those analyses that require additional types of taint to be tracked the ``TaintKind`` class can be overridden. Summary @@ -336,5 +332,5 @@ These classes are explained in more detail in :doc:`Tutorial: Taint tracking and What next? ---------- -- Experiment with the worked examples in the QL for Python tutorial topics: :doc:`Functions `, :doc:`Statements and expressions `, :doc:`Control flow `, :doc:`Points-to analysis and type inference ` and :doc:`Taint tracking and data flow analysis in Python `. +- Experiment with the worked examples in the following tutorial topics: :doc:`Functions `, :doc:`Statements and expressions `, :doc:`Control flow `, :doc:`Points-to analysis and type inference `, and :doc:`Taint tracking and data flow analysis in Python `. - Find out more about QL in the `QL language handbook `__ and `QL language specification `__. diff --git a/docs/language/learn-ql/python/pointsto-type-infer.rst b/docs/language/learn-ql/python/pointsto-type-infer.rst index a50e0776a83..eb80efe815f 100644 --- a/docs/language/learn-ql/python/pointsto-type-infer.rst +++ b/docs/language/learn-ql/python/pointsto-type-infer.rst @@ -1,12 +1,12 @@ Tutorial: Points-to analysis and type inference =============================================== -This topic contains worked examples of how to write queries using the standard QL library classes for Python type inference. +This topic contains worked examples of how to write queries using the standard CodeQL library classes for Python type inference. The ``Value`` class -------------------- -The ``Value`` class and its subclasses ``FunctionValue``, ``ClassValue`` and ``ModuleValue`` represent the values an expression may hold at runtime. +The ``Value`` class and its subclasses ``FunctionValue``, ``ClassValue``, and ``ModuleValue`` represent the values an expression may hold at runtime. Summary ~~~~~~~ @@ -201,7 +201,7 @@ There are two problems with this query: - It assumes that any call to something named "eval" is a call to the builtin ``eval`` function, which may result in some false positive results. - It assumes that ``eval`` cannot be referred to by any other name, which may result in some false negative results. -We can get much more accurate results using call-graph analysis. First, we can precisely identify the ``FunctionValue`` for the ``eval`` function, by using the ``Value::named`` QL predicate as follows: +We can get much more accurate results using call-graph analysis. First, we can precisely identify the ``FunctionValue`` for the ``eval`` function, by using the ``Value::named`` predicate as follows: .. code-block:: ql @@ -227,9 +227,5 @@ Then we can use ``Value.getACall()`` to identify calls to the ``eval`` function, What next? ---------- -For more information on writing QL, see: - -- `QL language handbook `__ - an introduction to the concepts of QL. -- :doc:`Learning QL <../../index>` - an overview of the resources for learning how to write your own QL queries. -- `Database generation `__ - an overview of the process that creates a database from source code. -- :doc:`What's in a CodeQL database? <../database>` - a description of the CodeQL database. +- Find out more about QL in the `QL language handbook `__ and `QL language specification `__. +- Read a description of the CodeQL database in :doc:`What's in a CodeQL database? <../database>` diff --git a/docs/language/learn-ql/python/ql-for-python.rst b/docs/language/learn-ql/python/ql-for-python.rst index 479a30dccb5..680c0c374b5 100644 --- a/docs/language/learn-ql/python/ql-for-python.rst +++ b/docs/language/learn-ql/python/ql-for-python.rst @@ -1,5 +1,5 @@ -QL for Python -============= +CodeQL for Python +================= .. toctree:: :glob: @@ -13,25 +13,25 @@ QL for Python taint-tracking pointsto-type-infer -The following tutorials and worked examples are designed to help you learn how to write effective and efficient QL queries for Python projects. You should work through these topics in the order displayed. +The following tutorials and worked examples are designed to help you learn how to write effective and efficient queries for Python projects. You should work through these topics in the order displayed. -- `Basic Python QL query `__ describes how to write and run queries using LGTM. +- `Basic Python query `__ describes how to write and run queries using LGTM. -- :doc:`Introducing the QL libraries for Python ` an introduction to the standard QL libraries used to write queries for Python code. +- :doc:`Introducing the CodeQL libraries for Python ` introduces the standard libraries used to write queries for Python code. -- :doc:`Tutorial: Functions ` worked examples of how to write queries using the standard QL library classes for Python functions. +- :doc:`Tutorial: Functions ` demonstrates how to write queries using the standard CodeQL library classes for Python functions. -- :doc:`Tutorial: Statements and expressions ` worked examples of how to write queries using the standard QL library classes for Python statements and expressions. +- :doc:`Tutorial: Statements and expressions ` demonstrates how to write queries using the standard CodeQL library classes for Python statements and expressions. -- :doc:`Tutorial: Control flow ` worked examples of how to write queries using the standard QL library classes for Python control flow. +- :doc:`Tutorial: Control flow ` demonstrates how to write queries using the standard CodeQL library classes for Python control flow. -- :doc:`Tutorial: Points-to analysis and type inference ` worked examples of how to write queries using the standard QL library classes for Python type inference. +- :doc:`Tutorial: Points-to analysis and type inference ` demonstrates how to write queries using the standard CodeQL library classes for Python type inference. -- :doc:`Taint tracking and data flow analysis in Python ` worked examples of how to write queries using the standard taint tracking and data flow QL libraries for Python. +- :doc:`Taint tracking and data flow analysis in Python ` demonstrates how to write queries using the standard taint tracking and data flow libraries for Python. Other resources --------------- -- For examples of how to query common Python elements, see the `Python QL cookbook `__ -- For the queries used in LGTM, display a `Python query `__ and click **Open in query console** to see the QL code used to find alerts -- For more information about the Python QL library see the `QL library for Python `__. +- For examples of how to query common Python elements, see the `Python cookbook `__. +- For the queries used in LGTM, display a `Python query `__ and click **Open in query console** to see the code used to find alerts. +- For more information about the library for Python see the `CodeQL library for Python `__. diff --git a/docs/language/learn-ql/python/statements-expressions.rst b/docs/language/learn-ql/python/statements-expressions.rst index 8aa1bea96f4..9a0a2484fc7 100644 --- a/docs/language/learn-ql/python/statements-expressions.rst +++ b/docs/language/learn-ql/python/statements-expressions.rst @@ -4,7 +4,7 @@ Tutorial: Statements and expressions Statements ---------- -The bulk of Python code takes the form of statements. Each different type of statement in Python is represented by a separate class in QL. +The bulk of Python code takes the form of statements. Each different type of statement in Python is represented by a separate CodeQL class. Here is the full class hierarchy: @@ -165,7 +165,7 @@ The clause ``cmp.getOp(0) instanceof Is and cmp.getComparator(0) = literal`` che Example: Duplicates in dictionary literals ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -If there are duplicate keys in a Python dictionary, then the second key will overwrite the first, which is almost certainly a mistake. We can find these duplicates in QL, but the query is more complex than previous examples and will require us to write a ``predicate`` as a helper. +If there are duplicate keys in a Python dictionary, then the second key will overwrite the first, which is almost certainly a mistake. We can find these duplicates with CodeQL, but the query is more complex than previous examples and will require us to write a ``predicate`` as a helper. Here is the query: @@ -274,5 +274,5 @@ Here is the relevant part of the class hierarchy: What next? ---------- -- Experiment with the worked examples in the QL for Python tutorial topics: :doc:`Control flow `, :doc:`Points-to analysis and type inference `. +- Experiment with the worked examples in the following tutorial topics: :doc:`Control flow ` and :doc:`Points-to analysis and type inference `. - Find out more about QL in the `QL language handbook `__ and `QL language specification `__. diff --git a/docs/language/learn-ql/python/taint-tracking.rst b/docs/language/learn-ql/python/taint-tracking.rst index 71cd000c716..7335df5af11 100644 --- a/docs/language/learn-ql/python/taint-tracking.rst +++ b/docs/language/learn-ql/python/taint-tracking.rst @@ -13,10 +13,10 @@ Taint tracking differs from basic data flow in that it considers non-value-prese For example, in the assignment ``dir = path + "/"``, if ``path`` is tainted then ``dir`` is also tainted, even though there is no data flow from ``path`` to ``path + "/"``. -Separate QL libraries have been written to handle 'normal' data flow and taint tracking in :doc:`C/C++ <../cpp/dataflow>`, :doc:`C# <../csharp/dataflow>`, :doc:`Java <../java/dataflow>`, and :doc:`JavaScript <../javascript/dataflow>`. You can access the appropriate classes and predicates that reason about these different modes of data flow by importing the appropriate QL library in your query. +Separate CodeQL libraries have been written to handle 'normal' data flow and taint tracking in :doc:`C/C++ <../cpp/dataflow>`, :doc:`C# <../csharp/dataflow>`, :doc:`Java <../java/dataflow>`, and :doc:`JavaScript <../javascript/dataflow>`. You can access the appropriate classes and predicates that reason about these different modes of data flow by importing the appropriate library in your query. In Python analysis, we can use the same taint tracking library to model both 'normal' data flow and taint flow, but we are still able make the distinction between steps that preserve value and those that don't by defining additional data flow properties. -For further information on data flow and taint tracking in QL, see :doc:`Introduction to data flow <../intro-to-data-flow>`. +For further information on data flow and taint tracking with CodeQL, see :doc:`Introduction to data flow <../intro-to-data-flow>`. Fundamentals of taint tracking and data flow analysis ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -222,7 +222,7 @@ The ``TaintTracking::Source`` and ``TaintTracking::Sink`` classes have predicate ... } -The ``TaintKind`` itself is just a string (a QL string, not a QL entity representing a Python string), +The ``TaintKind`` itself is just a string (a QL string, not a CodeQL entity representing a Python string), which provides methods to extend flow and allow the kind of taint to change along the path. The ``TaintKind`` class has many predicates allowing flow to be modified. This simplest ``TaintKind`` does not override any predicates, meaning that it only flows as opaque data. @@ -254,5 +254,5 @@ which defines the simplest possible taint kind class, ``HardcodedValue``, and cu What next? ---------- -- Experiment with the worked examples in the QL for Python tutorial topics: :doc:`Control flow `, and :doc:`Points-to analysis and type inference `. +- Experiment with the worked examples in the following tutorial topics: :doc:`Control flow ` and :doc:`Points-to analysis and type inference `. - Find out more about QL in the `QL language handbook `__ and `QL language specification `__. From 49dd2216a65e55824bb0371d8d476eebae1c241b Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Thu, 10 Oct 2019 16:14:12 +0200 Subject: [PATCH 058/148] Python: Refactor django library Use General.qll for routing, like in other web libraries --- .../src/semmle/python/web/django/General.qll | 28 +++++++++++++ .../src/semmle/python/web/django/Request.qll | 41 ++++--------------- 2 files changed, 35 insertions(+), 34 deletions(-) create mode 100644 python/ql/src/semmle/python/web/django/General.qll diff --git a/python/ql/src/semmle/python/web/django/General.qll b/python/ql/src/semmle/python/web/django/General.qll new file mode 100644 index 00000000000..954df1892e8 --- /dev/null +++ b/python/ql/src/semmle/python/web/django/General.qll @@ -0,0 +1,28 @@ +import python +import semmle.python.regex +import semmle.python.web.Http + +predicate django_route(CallNode call, ControlFlowNode regex, FunctionValue view) { + exists(FunctionValue url | + Value::named("django.conf.urls.url") = url and + url.getArgumentForCall(call, 0) = regex and + url.getArgumentForCall(call, 1).pointsTo(view) + ) +} + +class DjangoRouteRegex extends RegexString { + DjangoRouteRegex() { django_route(_, this.getAFlowNode(), _) } +} + +class DjangoRoute extends CallNode { + DjangoRoute() { django_route(this, _, _) } + + FunctionValue getViewFunction() { django_route(this, _, result) } + + string getNamedArgument() { + exists(DjangoRouteRegex regex | + django_route(this, regex.getAFlowNode(), _) and + regex.getGroupName(_, _) = result + ) + } +} diff --git a/python/ql/src/semmle/python/web/django/Request.qll b/python/ql/src/semmle/python/web/django/Request.qll index d2ad2ff30d3..71da78f9b43 100644 --- a/python/ql/src/semmle/python/web/django/Request.qll +++ b/python/ql/src/semmle/python/web/django/Request.qll @@ -1,7 +1,7 @@ import python -import semmle.python.regex import semmle.python.security.TaintTracking import semmle.python.web.Http +import semmle.python.web.django.General /** A django.request.HttpRequest object */ class DjangoRequest extends TaintKind { @@ -52,7 +52,7 @@ abstract class DjangoRequestSource extends HttpRequestTaintSource { private class DjangoFunctionBasedViewRequestArgument extends DjangoRequestSource { DjangoFunctionBasedViewRequestArgument() { exists(FunctionValue view | - url_dispatch(_, _, view) and + django_route(_, _, view) and this = view.getScope().getArg(0).asName().getAFlowNode() ) } @@ -76,41 +76,14 @@ class DjangoClassBasedViewRequestArgument extends DjangoRequestSource { } } -/* *********** Routing ********* */ -/* Function based views */ -predicate url_dispatch(CallNode call, ControlFlowNode regex, FunctionValue view) { - exists(FunctionValue url | - Value::named("django.conf.urls.url") = url and - url.getArgumentForCall(call, 0) = regex and - url.getArgumentForCall(call, 1).pointsTo(view) - ) -} - -class UrlRegex extends RegexString { - UrlRegex() { url_dispatch(_, this.getAFlowNode(), _) } -} - -class UrlRouting extends CallNode { - UrlRouting() { url_dispatch(this, _, _) } - - FunctionValue getViewFunction() { url_dispatch(this, _, result) } - - string getNamedArgument() { - exists(UrlRegex regex | - url_dispatch(this, regex.getAFlowNode(), _) and - regex.getGroupName(_, _) = result - ) - } -} - /** An argument specified in a url routing table */ -class HttpRequestParameter extends HttpRequestTaintSource { - HttpRequestParameter() { - exists(UrlRouting url | - this.(ControlFlowNode).getNode() = url +class DjangoRequestParameter extends HttpRequestTaintSource { + DjangoRequestParameter() { + exists(DjangoRoute route | + this.(ControlFlowNode).getNode() = route .getViewFunction() .getScope() - .getArgByName(url.getNamedArgument()) + .getArgByName(route.getNamedArgument()) ) } From afe7a0536cc0ad3cc93b396c0d3ed40bb1c5502c Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Thu, 10 Oct 2019 17:25:09 +0200 Subject: [PATCH 059/148] Python: Support positional arguments in Django routes --- change-notes/1.23/analysis-python.md | 5 +++ .../src/semmle/python/web/django/General.qll | 12 +++++++ .../src/semmle/python/web/django/Request.qll | 14 +++++--- .../library-tests/web/django/Sinks.expected | 4 +++ .../library-tests/web/django/Sources.expected | 9 +++++ .../ql/test/library-tests/web/django/test.py | 33 ++++++++++++++++++- 6 files changed, 71 insertions(+), 6 deletions(-) diff --git a/change-notes/1.23/analysis-python.md b/change-notes/1.23/analysis-python.md index 00fab13e098..6cea1745284 100644 --- a/change-notes/1.23/analysis-python.md +++ b/change-notes/1.23/analysis-python.md @@ -20,3 +20,8 @@ |----------------------------|------------------------|------------| | Unreachable code | Fewer false positives | Analysis now accounts for uses of `contextlib.suppress` to suppress exceptions. | | `__iter__` method returns a non-iterator | Better alert message | Alert now highlights which class is expected to be an iterator. | + + +## Changes to QL libraries + +* Django library now recognizes positional arguments from a `django.conf.urls.url` regex (Django version 1.x) diff --git a/python/ql/src/semmle/python/web/django/General.qll b/python/ql/src/semmle/python/web/django/General.qll index 954df1892e8..423d853cdad 100644 --- a/python/ql/src/semmle/python/web/django/General.qll +++ b/python/ql/src/semmle/python/web/django/General.qll @@ -25,4 +25,16 @@ class DjangoRoute extends CallNode { regex.getGroupName(_, _) = result ) } + + /** + * Get the number of positional arguments that will be passed to the view. + * Will only return a result if there are no named arguments. + */ + int getNumPositionalArguments() { + exists(DjangoRouteRegex regex | + django_route(this, regex.getAFlowNode(), _) and + not exists(string s | s = regex.getGroupName(_, _)) and + result = count(regex.getGroupNumber(_, _)) + ) + } } diff --git a/python/ql/src/semmle/python/web/django/Request.qll b/python/ql/src/semmle/python/web/django/Request.qll index 71da78f9b43..15ea032e6d7 100644 --- a/python/ql/src/semmle/python/web/django/Request.qll +++ b/python/ql/src/semmle/python/web/django/Request.qll @@ -79,11 +79,15 @@ class DjangoClassBasedViewRequestArgument extends DjangoRequestSource { /** An argument specified in a url routing table */ class DjangoRequestParameter extends HttpRequestTaintSource { DjangoRequestParameter() { - exists(DjangoRoute route | - this.(ControlFlowNode).getNode() = route - .getViewFunction() - .getScope() - .getArgByName(route.getNamedArgument()) + exists(DjangoRoute route, Function f | + f = route.getViewFunction().getScope() | + this.(ControlFlowNode).getNode() = f.getArgByName(route.getNamedArgument()) + or + exists(int i | i >= 0 | + i < route.getNumPositionalArguments() and + // +1 because first argument is always the request + this.(ControlFlowNode).getNode() = f.getArg(i+1) + ) ) } diff --git a/python/ql/test/library-tests/web/django/Sinks.expected b/python/ql/test/library-tests/web/django/Sinks.expected index 82d1b71e53a..7201a7a20a9 100644 --- a/python/ql/test/library-tests/web/django/Sinks.expected +++ b/python/ql/test/library-tests/web/django/Sinks.expected @@ -4,3 +4,7 @@ | test.py:25 | BinaryExpr | externally controlled string | | test.py:26 | BinaryExpr | externally controlled string | | test.py:34 | BinaryExpr | externally controlled string | +| test.py:46 | Attribute() | externally controlled string | +| test.py:57 | Attribute() | externally controlled string | +| test.py:60 | Attribute() | externally controlled string | +| test.py:63 | Attribute() | externally controlled string | diff --git a/python/ql/test/library-tests/web/django/Sources.expected b/python/ql/test/library-tests/web/django/Sources.expected index 8f421a2d169..7482f21cdda 100644 --- a/python/ql/test/library-tests/web/django/Sources.expected +++ b/python/ql/test/library-tests/web/django/Sources.expected @@ -1,2 +1,11 @@ | test.py:11 | request | django.request.HttpRequest | | test.py:31 | request | django.request.HttpRequest | +| test.py:56 | arg0 | externally controlled string | +| test.py:56 | request | django.request.HttpRequest | +| test.py:59 | arg0 | externally controlled string | +| test.py:59 | arg1 | externally controlled string | +| test.py:59 | arg2 | externally controlled string | +| test.py:59 | request | django.request.HttpRequest | +| test.py:62 | arg0 | externally controlled string | +| test.py:62 | arg1 | externally controlled string | +| test.py:62 | request | django.request.HttpRequest | diff --git a/python/ql/test/library-tests/web/django/test.py b/python/ql/test/library-tests/web/django/test.py index e6c39c30a32..3cabac0e5c5 100644 --- a/python/ql/test/library-tests/web/django/test.py +++ b/python/ql/test/library-tests/web/django/test.py @@ -34,7 +34,38 @@ def maybe_xss(request): resp.write("first name is " + first_name) return resp -urlpatterns2 = [ +urlpatterns = [ # Route to code_execution url(r'^maybe_xss$', maybe_xss, name='maybe_xss') ] + + +# Non capturing group (we correctly identify page_number as a request parameter) + +def show_articles(request, page_number=1): + return HttpResponse('articles page: {}'.format(page_number)) + +urlpatterns = [ + # one pattern to support `articles/page-` and ensuring that articles/ goes to page-1 + url(r'articles/^(?:page-(?P\d+)/)?$', show_articles), +] + + +# Positional arguments + +def xxs_positional_arg1(request, arg0): + return HttpResponse('xxs_positional_arg1: {}'.format(arg0)) + +def xxs_positional_arg2(request, arg0, arg1, arg2): + return HttpResponse('xxs_positional_arg2: {} {} {}'.format(arg0, arg1, arg2)) + +def xxs_positional_arg3(request, arg0, arg1): + return HttpResponse('xxs_positional_arg3: {} {}'.format(arg0, arg1)) + +urlpatterns = [ + # passing as positional argument is not the recommended way of doing things, + # but it is certainly possible + url(r'^(.+)$', xxs_positional_arg1, name='xxs_positional_arg1'), + url(r'^([^/]+)/([^/]+)/([^/]+)$', xxs_positional_arg2, name='xxs_positional_arg2'), + url(r'^([^/]+)/(?:foo|bar)/([^/]+)$', xxs_positional_arg3, name='xxs_positional_arg3'), +] From 471318369b4fee5c66239a5f33c91d9410df8f76 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Mon, 14 Oct 2019 16:51:50 +0200 Subject: [PATCH 060/148] Python: Don't quote %s in django example This is vulnerable to SQL injection because of the quotes around %s -- added some code that highlights this in test.py Since our examples did this in the safe query, I ended up rewriting them completely, causing a lot of trouble for myself :D --- .../src/Security/CWE-089/SqlInjection.qhelp | 17 +++++-- .../CWE-089/examples/sql_injection.py | 28 +++++----- .../library-tests/web/django/Sinks.expected | 21 ++++---- .../library-tests/web/django/Sources.expected | 26 ++++++---- .../ql/test/library-tests/web/django/test.py | 51 ++++++++++++------- .../Security/CWE-089/SqlInjection.expected | 27 +++++----- .../Security/CWE-089/sql_injection.py | 48 ++++++++++------- 7 files changed, 128 insertions(+), 90 deletions(-) diff --git a/python/ql/src/Security/CWE-089/SqlInjection.qhelp b/python/ql/src/Security/CWE-089/SqlInjection.qhelp index e976401a6b5..286b71a6047 100644 --- a/python/ql/src/Security/CWE-089/SqlInjection.qhelp +++ b/python/ql/src/Security/CWE-089/SqlInjection.qhelp @@ -21,20 +21,29 @@ or prepared statements.

    -In the following snippet, from an example django app, -a name is stored in the database using two different queries. +In the following snippet, a user is fetched from the database using three +different queries.

    In the first case, the query string is built by -directly using string formatting from a user-supplied request attribute. +directly using string formatting from a user-supplied request parameter. The parameter may include quote characters, so this code is vulnerable to a SQL injection attack.

    In the second case, the user-supplied request attribute is passed -to the database using query parameters. +to the database using query parameters. The database connector library will +take care of escaping and inserting quotes as needed. +

    + +

    +In the third case, the placeholder in the SQL string has been manually quoted. Since most +databaseconnector libraries will insert their own quotes, doing so yourself will make the code +vulnerable to SQL injection attacks. In this example, if username was +; DROP ALL TABLES -- , the final SQL query would be +SELECT * FROM users WHERE username = ''; DROP ALL TABLES -- ''

    diff --git a/python/ql/src/Security/CWE-089/examples/sql_injection.py b/python/ql/src/Security/CWE-089/examples/sql_injection.py index 541c580f712..fe6dbc50c01 100644 --- a/python/ql/src/Security/CWE-089/examples/sql_injection.py +++ b/python/ql/src/Security/CWE-089/examples/sql_injection.py @@ -1,21 +1,19 @@ - -from django.conf.urls import patterns, url +from django.conf.urls import url from django.db import connection -def save_name(request): +def show_user(request, username): + with connection.cursor() as cursor: + # BAD -- Using string formatting + cursor.execute("SELECT * FROM users WHERE username = '%s'" % username) + user = cursor.fetchone() - if request.method == 'POST': - name = request.POST.get('name') - curs = connection.cursor() - #BAD -- Using string formatting - curs.execute( - "insert into names_file ('name') values ('%s')" % name) - #GOOD -- Using parameters - curs.execute( - "insert into names_file ('name') values ('%s')", name) + # GOOD -- Using parameters + cursor.execute("SELECT * FROM users WHERE username = %s", username) + user = cursor.fetchone() + # BAD -- Manually quoting placeholder (%s) + cursor.execute("SELECT * FROM users WHERE username = '%s'", username) + user = cursor.fetchone() -urlpatterns = patterns(url(r'^save_name/$', - upload, name='save_name')) - +urlpatterns = [url(r'^users/(?P[^/]+)$', show_user)] diff --git a/python/ql/test/library-tests/web/django/Sinks.expected b/python/ql/test/library-tests/web/django/Sinks.expected index 7201a7a20a9..31a42364762 100644 --- a/python/ql/test/library-tests/web/django/Sinks.expected +++ b/python/ql/test/library-tests/web/django/Sinks.expected @@ -1,10 +1,13 @@ -| test.py:18 | Str | externally controlled string | +| test.py:14 | Str | externally controlled string | +| test.py:15 | Str | externally controlled string | +| test.py:18 | BinaryExpr | externally controlled string | | test.py:21 | BinaryExpr | externally controlled string | -| test.py:24 | BinaryExpr | externally controlled string | -| test.py:25 | BinaryExpr | externally controlled string | -| test.py:26 | BinaryExpr | externally controlled string | -| test.py:34 | BinaryExpr | externally controlled string | -| test.py:46 | Attribute() | externally controlled string | -| test.py:57 | Attribute() | externally controlled string | -| test.py:60 | Attribute() | externally controlled string | -| test.py:63 | Attribute() | externally controlled string | +| test.py:22 | BinaryExpr | externally controlled string | +| test.py:23 | BinaryExpr | externally controlled string | +| test.py:37 | Str | externally controlled string | +| test.py:44 | BinaryExpr | externally controlled string | +| test.py:48 | Attribute() | externally controlled string | +| test.py:59 | Attribute() | externally controlled string | +| test.py:70 | Attribute() | externally controlled string | +| test.py:73 | Attribute() | externally controlled string | +| test.py:76 | Attribute() | externally controlled string | diff --git a/python/ql/test/library-tests/web/django/Sources.expected b/python/ql/test/library-tests/web/django/Sources.expected index 7482f21cdda..371b1e7cb6f 100644 --- a/python/ql/test/library-tests/web/django/Sources.expected +++ b/python/ql/test/library-tests/web/django/Sources.expected @@ -1,11 +1,17 @@ | test.py:11 | request | django.request.HttpRequest | -| test.py:31 | request | django.request.HttpRequest | -| test.py:56 | arg0 | externally controlled string | -| test.py:56 | request | django.request.HttpRequest | -| test.py:59 | arg0 | externally controlled string | -| test.py:59 | arg1 | externally controlled string | -| test.py:59 | arg2 | externally controlled string | -| test.py:59 | request | django.request.HttpRequest | -| test.py:62 | arg0 | externally controlled string | -| test.py:62 | arg1 | externally controlled string | -| test.py:62 | request | django.request.HttpRequest | +| test.py:11 | username | externally controlled string | +| test.py:41 | request | django.request.HttpRequest | +| test.py:47 | bar | externally controlled string | +| test.py:47 | foo | externally controlled string | +| test.py:47 | request | django.request.HttpRequest | +| test.py:58 | page_number | externally controlled string | +| test.py:58 | request | django.request.HttpRequest | +| test.py:69 | arg0 | externally controlled string | +| test.py:69 | request | django.request.HttpRequest | +| test.py:72 | arg0 | externally controlled string | +| test.py:72 | arg1 | externally controlled string | +| test.py:72 | arg2 | externally controlled string | +| test.py:72 | request | django.request.HttpRequest | +| test.py:75 | arg0 | externally controlled string | +| test.py:75 | arg1 | externally controlled string | +| test.py:75 | request | django.request.HttpRequest | diff --git a/python/ql/test/library-tests/web/django/test.py b/python/ql/test/library-tests/web/django/test.py index 3cabac0e5c5..1240b9ae5a1 100644 --- a/python/ql/test/library-tests/web/django/test.py +++ b/python/ql/test/library-tests/web/django/test.py @@ -5,28 +5,38 @@ from django.db.models.expressions import RawSQL from django.http.response import HttpResponse import base64 -class Name(models.Model): +class User(models.Model): pass -def save_name(request): +def show_user(request, username): + with connection.cursor() as cursor: + # GOOD -- Using parameters + cursor.execute("SELECT * FROM users WHERE username = %s", username) + User.objects.raw("SELECT * FROM users WHERE username = %s", (username,)) - if request.method == 'POST': - name = request.POST.get('name') - curs = connection.cursor() - #GOOD -- Using parameters - curs.execute( - "insert into names_file ('name') values ('%s')", name) - #BAD -- Using string formatting - curs.execute( - "insert into names_file ('name') values ('%s')" % name) + # BAD -- Using string formatting + cursor.execute("SELECT * FROM users WHERE username = '%s'" % username) - #BAD -- other ways of executing raw SQL code with string interpolation - Name.objects.annotate(RawSQL("insert into names_file ('name') values ('%s')" % name)) - Name.objects.raw("insert into names_file ('name') values ('%s')" % name) - Name.objects.extra("insert into names_file ('name') values ('%s')" % name) + # BAD -- other ways of executing raw SQL code with string interpolation + User.objects.annotate(RawSQL("insert into names_file ('name') values ('%s')" % username)) + User.objects.raw("insert into names_file ('name') values ('%s')" % username) + User.objects.extra("insert into names_file ('name') values ('%s')" % username) -urlpatterns1 = patterns(url(r'^save_name/$', - save_name, name='save_name')) + # BAD (but currently no custom query to find this) + # + # It is exposed to SQL injection (https://docs.djangoproject.com/en/2.2/ref/models/querysets/#extra) + # For example, using name = "; DROP ALL TABLES -- " + # will result in SQL: SELECT * FROM name WHERE name = ''; DROP ALL TABLES -- '' + # + # This shouldn't be very widespread, since using a normal string will result in invalid SQL + # Using name = "example", will result in SQL: SELECT * FROM name WHERE name = ''example'' + # which in MySQL will give a syntax error + # + # When testing this out locally, none of the queries worked against SQLite3, but I could use + # the SQL injection against MySQL. + User.objects.raw("SELECT * FROM users WHERE username = '%s'", (username,)) + +urlpatterns = patterns(url(r'^users/(?P[^/]+)$', show_user)) def maybe_xss(request): first_name = request.POST.get('first_name', '') @@ -34,9 +44,12 @@ def maybe_xss(request): resp.write("first name is " + first_name) return resp +def xss_kwargs(request, foo, bar, baz=None): + return HttpResponse('xss_kwargs: {} {}'.format(foo, bar)) + urlpatterns = [ - # Route to code_execution - url(r'^maybe_xss$', maybe_xss, name='maybe_xss') + url(r'^maybe_xss$', maybe_xss, name='maybe_xss'), + url(r'^bar/(?P[^/]+)/foo/(?P[^/]+)', xss_kwargs, name='xss_kwargs'), ] diff --git a/python/ql/test/query-tests/Security/CWE-089/SqlInjection.expected b/python/ql/test/query-tests/Security/CWE-089/SqlInjection.expected index 9c00ffdb2db..461d42fccf1 100644 --- a/python/ql/test/query-tests/Security/CWE-089/SqlInjection.expected +++ b/python/ql/test/query-tests/Security/CWE-089/SqlInjection.expected @@ -1,17 +1,14 @@ edges -| sql_injection.py:9:15:9:21 | django.request.HttpRequest | sql_injection.py:12:16:12:22 | django.request.HttpRequest | -| sql_injection.py:12:16:12:22 | django.request.HttpRequest | sql_injection.py:12:16:12:27 | django.http.request.QueryDict | -| sql_injection.py:12:16:12:27 | django.http.request.QueryDict | sql_injection.py:12:16:12:39 | externally controlled string | -| sql_injection.py:12:16:12:39 | externally controlled string | sql_injection.py:19:63:19:66 | externally controlled string | -| sql_injection.py:12:16:12:39 | externally controlled string | sql_injection.py:22:88:22:91 | externally controlled string | -| sql_injection.py:12:16:12:39 | externally controlled string | sql_injection.py:23:76:23:79 | externally controlled string | -| sql_injection.py:12:16:12:39 | externally controlled string | sql_injection.py:24:78:24:81 | externally controlled string | -| sql_injection.py:19:63:19:66 | externally controlled string | sql_injection.py:19:13:19:66 | externally controlled string | -| sql_injection.py:22:88:22:91 | externally controlled string | sql_injection.py:22:38:22:91 | externally controlled string | -| sql_injection.py:23:76:23:79 | externally controlled string | sql_injection.py:23:26:23:79 | externally controlled string | -| sql_injection.py:24:78:24:81 | externally controlled string | sql_injection.py:24:28:24:81 | externally controlled string | +| sql_injection.py:12:24:12:31 | externally controlled string | sql_injection.py:19:70:19:77 | externally controlled string | +| sql_injection.py:12:24:12:31 | externally controlled string | sql_injection.py:22:88:22:95 | externally controlled string | +| sql_injection.py:12:24:12:31 | externally controlled string | sql_injection.py:23:76:23:83 | externally controlled string | +| sql_injection.py:12:24:12:31 | externally controlled string | sql_injection.py:24:78:24:85 | externally controlled string | +| sql_injection.py:19:70:19:77 | externally controlled string | sql_injection.py:19:24:19:77 | externally controlled string | +| sql_injection.py:22:88:22:95 | externally controlled string | sql_injection.py:22:38:22:95 | externally controlled string | +| sql_injection.py:23:76:23:83 | externally controlled string | sql_injection.py:23:26:23:83 | externally controlled string | +| sql_injection.py:24:78:24:85 | externally controlled string | sql_injection.py:24:28:24:85 | externally controlled string | #select -| sql_injection.py:19:13:19:66 | BinaryExpr | sql_injection.py:9:15:9:21 | django.request.HttpRequest | sql_injection.py:19:13:19:66 | externally controlled string | This SQL query depends on $@. | sql_injection.py:9:15:9:21 | request | a user-provided value | -| sql_injection.py:22:38:22:91 | BinaryExpr | sql_injection.py:9:15:9:21 | django.request.HttpRequest | sql_injection.py:22:38:22:91 | externally controlled string | This SQL query depends on $@. | sql_injection.py:9:15:9:21 | request | a user-provided value | -| sql_injection.py:23:26:23:79 | BinaryExpr | sql_injection.py:9:15:9:21 | django.request.HttpRequest | sql_injection.py:23:26:23:79 | externally controlled string | This SQL query depends on $@. | sql_injection.py:9:15:9:21 | request | a user-provided value | -| sql_injection.py:24:28:24:81 | BinaryExpr | sql_injection.py:9:15:9:21 | django.request.HttpRequest | sql_injection.py:24:28:24:81 | externally controlled string | This SQL query depends on $@. | sql_injection.py:9:15:9:21 | request | a user-provided value | +| sql_injection.py:19:24:19:77 | BinaryExpr | sql_injection.py:12:24:12:31 | externally controlled string | sql_injection.py:19:24:19:77 | externally controlled string | This SQL query depends on $@. | sql_injection.py:12:24:12:31 | username | a user-provided value | +| sql_injection.py:22:38:22:95 | BinaryExpr | sql_injection.py:12:24:12:31 | externally controlled string | sql_injection.py:22:38:22:95 | externally controlled string | This SQL query depends on $@. | sql_injection.py:12:24:12:31 | username | a user-provided value | +| sql_injection.py:23:26:23:83 | BinaryExpr | sql_injection.py:12:24:12:31 | externally controlled string | sql_injection.py:23:26:23:83 | externally controlled string | This SQL query depends on $@. | sql_injection.py:12:24:12:31 | username | a user-provided value | +| sql_injection.py:24:28:24:85 | BinaryExpr | sql_injection.py:12:24:12:31 | externally controlled string | sql_injection.py:24:28:24:85 | externally controlled string | This SQL query depends on $@. | sql_injection.py:12:24:12:31 | username | a user-provided value | diff --git a/python/ql/test/query-tests/Security/CWE-089/sql_injection.py b/python/ql/test/query-tests/Security/CWE-089/sql_injection.py index b312241b809..9ccca7503bd 100644 --- a/python/ql/test/query-tests/Security/CWE-089/sql_injection.py +++ b/python/ql/test/query-tests/Security/CWE-089/sql_injection.py @@ -1,28 +1,40 @@ +"""This is copied from ql/python/ql/test/library-tests/web/django/test.py +and a only a slight extension of ql/python/ql/src/Security/CWE-089/examples/sql_injection.py +""" -from django.conf.urls import patterns, url +from django.conf.urls import url from django.db import connection, models from django.db.models.expressions import RawSQL -class Name(models.Model): +class User(models.Model): pass -def save_name(request): +def show_user(request, username): + with connection.cursor() as cursor: + # GOOD -- Using parameters + cursor.execute("SELECT * FROM users WHERE username = %s", username) + User.objects.raw("SELECT * FROM users WHERE username = %s", (username,)) - if request.method == 'POST': - name = request.POST.get('name') - curs = connection.cursor() - #GOOD -- Using parameters - curs.execute( - "insert into names_file ('name') values ('%s')", name) - #BAD -- Using string formatting - curs.execute( - "insert into names_file ('name') values ('%s')" % name) + # BAD -- Using string formatting + cursor.execute("SELECT * FROM users WHERE username = '%s'" % username) - #BAD -- other ways of executing raw SQL code with string interpolation - Name.objects.annotate(RawSQL("insert into names_file ('name') values ('%s')" % name)) - Name.objects.raw("insert into names_file ('name') values ('%s')" % name) - Name.objects.extra("insert into names_file ('name') values ('%s')" % name) + # BAD -- other ways of executing raw SQL code with string interpolation + User.objects.annotate(RawSQL("insert into names_file ('name') values ('%s')" % username)) + User.objects.raw("insert into names_file ('name') values ('%s')" % username) + User.objects.extra("insert into names_file ('name') values ('%s')" % username) -urlpatterns = patterns(url(r'^save_name/$', - save_name, name='save_name')) + # BAD (but currently no custom query to find this) + # + # It is exposed to SQL injection (https://docs.djangoproject.com/en/2.2/ref/models/querysets/#extra) + # For example, using name = "; DROP ALL TABLES -- " + # will result in SQL: SELECT * FROM name WHERE name = ''; DROP ALL TABLES -- '' + # + # This shouldn't be very widespread, since using a normal string will result in invalid SQL + # Using name = "example", will result in SQL: SELECT * FROM name WHERE name = ''example'' + # which in MySQL will give a syntax error + # + # When testing this out locally, none of the queries worked against SQLite3, but I could use + # the SQL injection against MySQL. + User.objects.raw("SELECT * FROM users WHERE username = '%s'", (username,)) +urlpatterns = [url(r'^users/(?P[^/]+)$', show_user)] From 91f269ed7b873cc7fc329f562a99552297467a0e Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Tue, 15 Oct 2019 16:44:26 +0200 Subject: [PATCH 061/148] Python: Remove unused django sinks This would find instances of `thing = MyThing.objects.get(field=userinput)`, and what seems to be a query that wants to match on `thing = MyThing(); thing.field=userinput`. Both are not vulnerable to user-input, due to the build-in escaping by django. The DjangoModelFieldWrite actually matches on `MyThing.field=userinput` and not `thing.field=userinput`. I suspect this to be a mistake. Matching on `thing.field=userinput`, would require this CodeQL: attr.getObject(_).pointsTo().getClass() = model --- .../ql/src/semmle/python/web/django/Model.qll | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/python/ql/src/semmle/python/web/django/Model.qll b/python/ql/src/semmle/python/web/django/Model.qll index 34cf5856802..b8f47b64bdf 100644 --- a/python/ql/src/semmle/python/web/django/Model.qll +++ b/python/ql/src/semmle/python/web/django/Model.qll @@ -54,33 +54,6 @@ class DjangoModelObjects extends TaintSource { override string toString() { result = "django.db.models.Model.objects" } } -/** A write to a field of a django model, which is a vulnerable to external data. */ -class DjangoModelFieldWrite extends SqlInjectionSink { - DjangoModelFieldWrite() { - exists(AttrNode attr, DjangoModel model | - this = attr and attr.isStore() and attr.getObject(_).pointsTo(model) - ) - } - - override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind } - - override string toString() { result = "django model field write" } -} - -/** A direct reference to a django model object, which is vulnerable to external data. */ -class DjangoModelDirectObjectReference extends TaintSink { - DjangoModelDirectObjectReference() { - exists(CallNode objects_get_call, ControlFlowNode objects | this = objects_get_call.getAnArg() | - objects_get_call.getFunction().(AttrNode).getObject("get") = objects and - any(DjangoDbTableObjects objs).taints(objects) - ) - } - - override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind } - - override string toString() { result = "django model object reference" } -} - /** * A call to the `raw` method on a django model. This allows a raw SQL query * to be sent to the database, which is a security risk. From fb864b7262ec63ed695edf8fd084f6eb1b124da1 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Mon, 21 Oct 2019 16:39:55 +0200 Subject: [PATCH 062/148] Python: Consolidate tests for django The tests in 3/ was not Python 3 specific anymore --- .../3/library-tests/web/django/Sinks.expected | 6 -- .../test/3/library-tests/web/django/Sinks.ql | 13 --- .../library-tests/web/django/Sources.expected | 8 -- .../3/library-tests/web/django/Sources.ql | 12 --- .../3/library-tests/web/django/Taint.expected | 24 ------ .../test/3/library-tests/web/django/Taint.ql | 14 --- .../web/django/django/__init__.py | 1 - .../web/django/django/conf/__init__.py | 1 - .../web/django/django/conf/urls.py | 3 - .../web/django/django/db/__init__.py | 1 - .../web/django/django/db/models/__init__.py | 2 - .../django/django/db/models/expressions.py | 2 - .../web/django/django/http/__init__.py | 2 - .../web/django/django/http/response.py | 5 -- .../test/3/library-tests/web/django/models.py | 10 --- .../test/3/library-tests/web/django/rawsql.py | 23 ----- .../test/3/library-tests/web/django/urls.py | 9 -- .../test/3/library-tests/web/django/views.py | 19 ---- python/ql/test/3/library-tests/web/options | 1 - .../library-tests/web/django/Sinks.expected | 28 +++--- .../library-tests/web/django/Sources.expected | 32 +++---- .../ql/test/library-tests/web/django/sql.py | 53 ++++++++++++ .../ql/test/library-tests/web/django/test.py | 86 +++---------------- .../ql/test/library-tests/web/django/views.py | 65 ++++++++++++++ .../Security/lib/django/conf/urls.py | 5 +- 25 files changed, 161 insertions(+), 264 deletions(-) delete mode 100644 python/ql/test/3/library-tests/web/django/Sinks.expected delete mode 100644 python/ql/test/3/library-tests/web/django/Sinks.ql delete mode 100644 python/ql/test/3/library-tests/web/django/Sources.expected delete mode 100644 python/ql/test/3/library-tests/web/django/Sources.ql delete mode 100644 python/ql/test/3/library-tests/web/django/Taint.expected delete mode 100644 python/ql/test/3/library-tests/web/django/Taint.ql delete mode 100644 python/ql/test/3/library-tests/web/django/django/__init__.py delete mode 100644 python/ql/test/3/library-tests/web/django/django/conf/__init__.py delete mode 100644 python/ql/test/3/library-tests/web/django/django/conf/urls.py delete mode 100644 python/ql/test/3/library-tests/web/django/django/db/__init__.py delete mode 100644 python/ql/test/3/library-tests/web/django/django/db/models/__init__.py delete mode 100644 python/ql/test/3/library-tests/web/django/django/db/models/expressions.py delete mode 100644 python/ql/test/3/library-tests/web/django/django/http/__init__.py delete mode 100644 python/ql/test/3/library-tests/web/django/django/http/response.py delete mode 100644 python/ql/test/3/library-tests/web/django/models.py delete mode 100644 python/ql/test/3/library-tests/web/django/rawsql.py delete mode 100644 python/ql/test/3/library-tests/web/django/urls.py delete mode 100644 python/ql/test/3/library-tests/web/django/views.py delete mode 100644 python/ql/test/3/library-tests/web/options create mode 100644 python/ql/test/library-tests/web/django/sql.py create mode 100644 python/ql/test/library-tests/web/django/views.py diff --git a/python/ql/test/3/library-tests/web/django/Sinks.expected b/python/ql/test/3/library-tests/web/django/Sinks.expected deleted file mode 100644 index b4142a2d82a..00000000000 --- a/python/ql/test/3/library-tests/web/django/Sinks.expected +++ /dev/null @@ -1,6 +0,0 @@ -| models.py:9 | key | externally controlled string | -| rawsql.py:4 | BinaryExpr | externally controlled string | -| rawsql.py:13 | BinaryExpr | externally controlled string | -| rawsql.py:18 | BinaryExpr | externally controlled string | -| rawsql.py:22 | BinaryExpr | externally controlled string | -| views.py:8 | Attribute() | externally controlled string | diff --git a/python/ql/test/3/library-tests/web/django/Sinks.ql b/python/ql/test/3/library-tests/web/django/Sinks.ql deleted file mode 100644 index db9b6a89b93..00000000000 --- a/python/ql/test/3/library-tests/web/django/Sinks.ql +++ /dev/null @@ -1,13 +0,0 @@ - -import python - - -import semmle.python.web.django.Request -import semmle.python.web.django.Model -import semmle.python.web.django.Db -import semmle.python.web.django.Response -import semmle.python.security.strings.Untrusted - -from TaintSink sink, TaintKind kind -where sink.sinks(kind) -select sink.getLocation().toString(), sink.(ControlFlowNode).getNode().toString(), kind.toString() diff --git a/python/ql/test/3/library-tests/web/django/Sources.expected b/python/ql/test/3/library-tests/web/django/Sources.expected deleted file mode 100644 index 965142343dc..00000000000 --- a/python/ql/test/3/library-tests/web/django/Sources.expected +++ /dev/null @@ -1,8 +0,0 @@ -| models.py:9 | Attribute | django.db.models.Model.objects | -| rawsql.py:13 | Attribute | django.db.models.Model.objects | -| rawsql.py:16 | Attribute | django.db.models.Model.objects | -| rawsql.py:21 | Attribute | django.db.models.Model.objects | -| views.py:6 | request | django.request.HttpRequest | -| views.py:8 | HttpResponse() | django.response.HttpResponse | -| views.py:11 | path | externally controlled string | -| views.py:11 | request | django.request.HttpRequest | diff --git a/python/ql/test/3/library-tests/web/django/Sources.ql b/python/ql/test/3/library-tests/web/django/Sources.ql deleted file mode 100644 index 60d42cb2ac1..00000000000 --- a/python/ql/test/3/library-tests/web/django/Sources.ql +++ /dev/null @@ -1,12 +0,0 @@ - -import python - - -import semmle.python.web.django.Request -import semmle.python.web.django.Model -import semmle.python.web.django.Response -import semmle.python.security.strings.Untrusted - -from TaintSource src, TaintKind kind -where src.isSourceOf(kind) -select src.getLocation().toString(), src.(ControlFlowNode).getNode().toString(), kind.toString() diff --git a/python/ql/test/3/library-tests/web/django/Taint.expected b/python/ql/test/3/library-tests/web/django/Taint.expected deleted file mode 100644 index 3f40979c8a2..00000000000 --- a/python/ql/test/3/library-tests/web/django/Taint.expected +++ /dev/null @@ -1,24 +0,0 @@ -| models.py:9 | Attribute | django.db.models.Model.objects | -| rawsql.py:13 | Attribute | django.db.models.Model.objects | -| rawsql.py:13 | Attribute() | django.db.models.Model.objects | -| rawsql.py:16 | Attribute | django.db.models.Model.objects | -| rawsql.py:16 | Attribute() | django.db.models.Model.objects | -| rawsql.py:17 | Attribute() | django.db.models.Model.objects | -| rawsql.py:17 | m | django.db.models.Model.objects | -| rawsql.py:18 | Attribute() | django.db.models.Model.objects | -| rawsql.py:18 | m | django.db.models.Model.objects | -| rawsql.py:21 | Attribute | django.db.models.Model.objects | -| rawsql.py:21 | Attribute() | django.db.models.Model.objects | -| rawsql.py:22 | Attribute() | django.db.models.Model.objects | -| rawsql.py:22 | m | django.db.models.Model.objects | -| views.py:6 | request | django.request.HttpRequest | -| views.py:8 | Attribute | django.http.request.QueryDict | -| views.py:8 | Attribute() | externally controlled string | -| views.py:8 | HttpResponse() | django.response.HttpResponse | -| views.py:8 | request | django.request.HttpRequest | -| views.py:11 | path | externally controlled string | -| views.py:11 | request | django.request.HttpRequest | -| views.py:12 | Dict | {externally controlled string} | -| views.py:12 | path | externally controlled string | -| views.py:13 | env | {externally controlled string} | -| views.py:13 | request | django.request.HttpRequest | diff --git a/python/ql/test/3/library-tests/web/django/Taint.ql b/python/ql/test/3/library-tests/web/django/Taint.ql deleted file mode 100644 index b12a6560125..00000000000 --- a/python/ql/test/3/library-tests/web/django/Taint.ql +++ /dev/null @@ -1,14 +0,0 @@ - -import python - - -import semmle.python.web.django.Request -import semmle.python.web.django.Model -import semmle.python.web.django.Response -import semmle.python.security.strings.Untrusted - - -from TaintedNode node - -select node.getLocation().toString(), node.getAstNode().toString(), node.getTaintKind().toString() - diff --git a/python/ql/test/3/library-tests/web/django/django/__init__.py b/python/ql/test/3/library-tests/web/django/django/__init__.py deleted file mode 100644 index fc8d5b8c09b..00000000000 --- a/python/ql/test/3/library-tests/web/django/django/__init__.py +++ /dev/null @@ -1 +0,0 @@ -#Fake django package \ No newline at end of file diff --git a/python/ql/test/3/library-tests/web/django/django/conf/__init__.py b/python/ql/test/3/library-tests/web/django/django/conf/__init__.py deleted file mode 100644 index fc8d5b8c09b..00000000000 --- a/python/ql/test/3/library-tests/web/django/django/conf/__init__.py +++ /dev/null @@ -1 +0,0 @@ -#Fake django package \ No newline at end of file diff --git a/python/ql/test/3/library-tests/web/django/django/conf/urls.py b/python/ql/test/3/library-tests/web/django/django/conf/urls.py deleted file mode 100644 index 580cc063d99..00000000000 --- a/python/ql/test/3/library-tests/web/django/django/conf/urls.py +++ /dev/null @@ -1,3 +0,0 @@ - -def url(regex, view): - pass \ No newline at end of file diff --git a/python/ql/test/3/library-tests/web/django/django/db/__init__.py b/python/ql/test/3/library-tests/web/django/django/db/__init__.py deleted file mode 100644 index fc8d5b8c09b..00000000000 --- a/python/ql/test/3/library-tests/web/django/django/db/__init__.py +++ /dev/null @@ -1 +0,0 @@ -#Fake django package \ No newline at end of file diff --git a/python/ql/test/3/library-tests/web/django/django/db/models/__init__.py b/python/ql/test/3/library-tests/web/django/django/db/models/__init__.py deleted file mode 100644 index eb9c72adc45..00000000000 --- a/python/ql/test/3/library-tests/web/django/django/db/models/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -class Model: - pass diff --git a/python/ql/test/3/library-tests/web/django/django/db/models/expressions.py b/python/ql/test/3/library-tests/web/django/django/db/models/expressions.py deleted file mode 100644 index d7e0d1c27b6..00000000000 --- a/python/ql/test/3/library-tests/web/django/django/db/models/expressions.py +++ /dev/null @@ -1,2 +0,0 @@ -class RawSQL: - pass diff --git a/python/ql/test/3/library-tests/web/django/django/http/__init__.py b/python/ql/test/3/library-tests/web/django/django/http/__init__.py deleted file mode 100644 index e1a02f873cb..00000000000 --- a/python/ql/test/3/library-tests/web/django/django/http/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ - -from .response import HttpResponse diff --git a/python/ql/test/3/library-tests/web/django/django/http/response.py b/python/ql/test/3/library-tests/web/django/django/http/response.py deleted file mode 100644 index f3dd53d15b0..00000000000 --- a/python/ql/test/3/library-tests/web/django/django/http/response.py +++ /dev/null @@ -1,5 +0,0 @@ - -class HttpResponse: - - def __init__(self, *args): - pass diff --git a/python/ql/test/3/library-tests/web/django/models.py b/python/ql/test/3/library-tests/web/django/models.py deleted file mode 100644 index 571c594d055..00000000000 --- a/python/ql/test/3/library-tests/web/django/models.py +++ /dev/null @@ -1,10 +0,0 @@ - -from django.db import models - -class MyModel(models.Model): - title = models.CharField(max_length=500) - summary = models.TextField(blank=True) - -def update_my_model(key, title): - item = MyModel.objects.get(pk=key) - item.title = title diff --git a/python/ql/test/3/library-tests/web/django/rawsql.py b/python/ql/test/3/library-tests/web/django/rawsql.py deleted file mode 100644 index c6c82909c2d..00000000000 --- a/python/ql/test/3/library-tests/web/django/rawsql.py +++ /dev/null @@ -1,23 +0,0 @@ -from django.db.models.expressions import RawSQL - -def raw1(arg): - return RawSQL("select foo from bar where baz = %s" % arg, "") - - -from django.db import models - -class MyModel(models.Model): - pass - -def raw2(arg): - MyModel.objects.raw("select foo from bar where baz = %s" % arg) - -def raw3(arg): - m = MyModel.objects.filter('foo') - m = m.filter('bar') - m.raw("select foo from bar where baz = %s" % arg) - -def raw4(arg): - m = MyModel.objects.filter('foo') - m.extra("select foo from bar where baz = %s" % arg) - diff --git a/python/ql/test/3/library-tests/web/django/urls.py b/python/ql/test/3/library-tests/web/django/urls.py deleted file mode 100644 index d5a941ae519..00000000000 --- a/python/ql/test/3/library-tests/web/django/urls.py +++ /dev/null @@ -1,9 +0,0 @@ -from django.conf.urls import url -import views - -urlpatterns = [ - - url(r'^route1$', views.view_func1), - url(r'^(?P.*)$', views.view_func2), - url(r'^route2$', views.ClassView.as_view()) -] diff --git a/python/ql/test/3/library-tests/web/django/views.py b/python/ql/test/3/library-tests/web/django/views.py deleted file mode 100644 index f7de60a23f6..00000000000 --- a/python/ql/test/3/library-tests/web/django/views.py +++ /dev/null @@ -1,19 +0,0 @@ - -from django.http import HttpResponse -from django.shortcuts import redirect, render -from django.views.generic import View - -def view_func1(request): - # Whether this is safe depends on template.html -- annoyingly - return HttpResponse(request.GET.get("untrusted")) - - -def view_func2(request, path='default'): - env = {'path': path} - return render(request, 'vulnerable-path.html', env) - - -class ClassView(View): - - def get(self, request): - pass diff --git a/python/ql/test/3/library-tests/web/options b/python/ql/test/3/library-tests/web/options deleted file mode 100644 index f2d60a4bc3c..00000000000 --- a/python/ql/test/3/library-tests/web/options +++ /dev/null @@ -1 +0,0 @@ -semmle-extractor-options: --max-import-depth=3 --lang=3 diff --git a/python/ql/test/library-tests/web/django/Sinks.expected b/python/ql/test/library-tests/web/django/Sinks.expected index 31a42364762..86e05bca6dc 100644 --- a/python/ql/test/library-tests/web/django/Sinks.expected +++ b/python/ql/test/library-tests/web/django/Sinks.expected @@ -1,13 +1,15 @@ -| test.py:14 | Str | externally controlled string | -| test.py:15 | Str | externally controlled string | -| test.py:18 | BinaryExpr | externally controlled string | -| test.py:21 | BinaryExpr | externally controlled string | -| test.py:22 | BinaryExpr | externally controlled string | -| test.py:23 | BinaryExpr | externally controlled string | -| test.py:37 | Str | externally controlled string | -| test.py:44 | BinaryExpr | externally controlled string | -| test.py:48 | Attribute() | externally controlled string | -| test.py:59 | Attribute() | externally controlled string | -| test.py:70 | Attribute() | externally controlled string | -| test.py:73 | Attribute() | externally controlled string | -| test.py:76 | Attribute() | externally controlled string | +| sql.py:13 | Str | externally controlled string | +| sql.py:14 | Str | externally controlled string | +| sql.py:17 | BinaryExpr | externally controlled string | +| sql.py:20 | BinaryExpr | externally controlled string | +| sql.py:21 | BinaryExpr | externally controlled string | +| sql.py:22 | BinaryExpr | externally controlled string | +| sql.py:36 | Str | externally controlled string | +| sql.py:42 | BinaryExpr | externally controlled string | +| sql.py:47 | BinaryExpr | externally controlled string | +| views.py:7 | Attribute() | externally controlled string | +| views.py:11 | Attribute() | externally controlled string | +| views.py:15 | Attribute() | externally controlled string | +| views.py:22 | Attribute() | externally controlled string | +| views.py:27 | Attribute() | externally controlled string | +| views.py:31 | Attribute() | externally controlled string | diff --git a/python/ql/test/library-tests/web/django/Sources.expected b/python/ql/test/library-tests/web/django/Sources.expected index 371b1e7cb6f..a0c253b27c5 100644 --- a/python/ql/test/library-tests/web/django/Sources.expected +++ b/python/ql/test/library-tests/web/django/Sources.expected @@ -1,17 +1,17 @@ +| test.py:5 | path | externally controlled string | +| test.py:5 | request | django.request.HttpRequest | +| test.py:11 | path | externally controlled string | | test.py:11 | request | django.request.HttpRequest | -| test.py:11 | username | externally controlled string | -| test.py:41 | request | django.request.HttpRequest | -| test.py:47 | bar | externally controlled string | -| test.py:47 | foo | externally controlled string | -| test.py:47 | request | django.request.HttpRequest | -| test.py:58 | page_number | externally controlled string | -| test.py:58 | request | django.request.HttpRequest | -| test.py:69 | arg0 | externally controlled string | -| test.py:69 | request | django.request.HttpRequest | -| test.py:72 | arg0 | externally controlled string | -| test.py:72 | arg1 | externally controlled string | -| test.py:72 | arg2 | externally controlled string | -| test.py:72 | request | django.request.HttpRequest | -| test.py:75 | arg0 | externally controlled string | -| test.py:75 | arg1 | externally controlled string | -| test.py:75 | request | django.request.HttpRequest | +| views.py:6 | bar | externally controlled string | +| views.py:6 | foo | externally controlled string | +| views.py:6 | request | django.request.HttpRequest | +| views.py:10 | request | django.request.HttpRequest | +| views.py:14 | request | django.request.HttpRequest | +| views.py:25 | page_number | externally controlled string | +| views.py:25 | request | django.request.HttpRequest | +| views.py:30 | arg0 | externally controlled string | +| views.py:30 | arg1 | externally controlled string | +| views.py:30 | request | django.request.HttpRequest | +| views.py:50 | request | django.request.HttpRequest | +| views.py:50 | username | externally controlled string | +| views.py:59 | request | django.request.HttpRequest | diff --git a/python/ql/test/library-tests/web/django/sql.py b/python/ql/test/library-tests/web/django/sql.py new file mode 100644 index 00000000000..7809c24edc1 --- /dev/null +++ b/python/ql/test/library-tests/web/django/sql.py @@ -0,0 +1,53 @@ +from django.db import connection, models +from django.db.models.expressions import RawSQL + + +class User(models.Model): + username = models.CharField(max_length=100) + description = models.TextField(blank=True) + + +def show_user(username): + with connection.cursor() as cursor: + # GOOD -- Using parameters + cursor.execute("SELECT * FROM users WHERE username = %s", username) + User.objects.raw("SELECT * FROM users WHERE username = %s", (username,)) + + # BAD -- Using string formatting + cursor.execute("SELECT * FROM users WHERE username = '%s'" % username) + + # BAD -- other ways of executing raw SQL code with string interpolation + User.objects.annotate(RawSQL("insert into names_file ('name') values ('%s')" % username)) + User.objects.raw("insert into names_file ('name') values ('%s')" % username) + User.objects.extra("insert into names_file ('name') values ('%s')" % username) + + # BAD (but currently no custom query to find this) + # + # It is exposed to SQL injection (https://docs.djangoproject.com/en/2.2/ref/models/querysets/#extra) + # For example, using name = "; DROP ALL TABLES -- " + # will result in SQL: SELECT * FROM name WHERE name = ''; DROP ALL TABLES -- '' + # + # This shouldn't be very widespread, since using a normal string will result in invalid SQL + # Using name = "example", will result in SQL: SELECT * FROM name WHERE name = ''example'' + # which in MySQL will give a syntax error + # + # When testing this out locally, none of the queries worked against SQLite3, but I could use + # the SQL injection against MySQL. + User.objects.raw("SELECT * FROM users WHERE username = '%s'", (username,)) + + +def raw3(arg): + m = User.objects.filter('foo') + m = m.filter('bar') + m.raw("select foo from bar where baz = %s" % arg) + + +def raw4(arg): + m = User.objects.filter('foo') + m.extra("select foo from bar where baz = %s" % arg) + + +def update_user(key, description1): + # Neither of these are exposed to sql-injections + user = User.objects.get(pk=key) + item.description = description diff --git a/python/ql/test/library-tests/web/django/test.py b/python/ql/test/library-tests/web/django/test.py index 1240b9ae5a1..5664647c3a7 100644 --- a/python/ql/test/library-tests/web/django/test.py +++ b/python/ql/test/library-tests/web/django/test.py @@ -1,84 +1,18 @@ +from django.conf.urls import url +from django.shortcuts import redirect, render -from django.conf.urls import patterns, url -from django.db import connection, models -from django.db.models.expressions import RawSQL -from django.http.response import HttpResponse -import base64 -class User(models.Model): - pass +def with_template(request, path='default'): + env = {'path': path} + # We would need to understand django templates to know if this is safe or not + return render(request, 'possibly-vulnerable-template.html', env) -def show_user(request, username): - with connection.cursor() as cursor: - # GOOD -- Using parameters - cursor.execute("SELECT * FROM users WHERE username = %s", username) - User.objects.raw("SELECT * FROM users WHERE username = %s", (username,)) - # BAD -- Using string formatting - cursor.execute("SELECT * FROM users WHERE username = '%s'" % username) +def vuln_redirect(request, path): + return redirect(path) - # BAD -- other ways of executing raw SQL code with string interpolation - User.objects.annotate(RawSQL("insert into names_file ('name') values ('%s')" % username)) - User.objects.raw("insert into names_file ('name') values ('%s')" % username) - User.objects.extra("insert into names_file ('name') values ('%s')" % username) - - # BAD (but currently no custom query to find this) - # - # It is exposed to SQL injection (https://docs.djangoproject.com/en/2.2/ref/models/querysets/#extra) - # For example, using name = "; DROP ALL TABLES -- " - # will result in SQL: SELECT * FROM name WHERE name = ''; DROP ALL TABLES -- '' - # - # This shouldn't be very widespread, since using a normal string will result in invalid SQL - # Using name = "example", will result in SQL: SELECT * FROM name WHERE name = ''example'' - # which in MySQL will give a syntax error - # - # When testing this out locally, none of the queries worked against SQLite3, but I could use - # the SQL injection against MySQL. - User.objects.raw("SELECT * FROM users WHERE username = '%s'", (username,)) - -urlpatterns = patterns(url(r'^users/(?P[^/]+)$', show_user)) - -def maybe_xss(request): - first_name = request.POST.get('first_name', '') - resp = HttpResponse() - resp.write("first name is " + first_name) - return resp - -def xss_kwargs(request, foo, bar, baz=None): - return HttpResponse('xss_kwargs: {} {}'.format(foo, bar)) urlpatterns = [ - url(r'^maybe_xss$', maybe_xss, name='maybe_xss'), - url(r'^bar/(?P[^/]+)/foo/(?P[^/]+)', xss_kwargs, name='xss_kwargs'), -] - - -# Non capturing group (we correctly identify page_number as a request parameter) - -def show_articles(request, page_number=1): - return HttpResponse('articles page: {}'.format(page_number)) - -urlpatterns = [ - # one pattern to support `articles/page-` and ensuring that articles/ goes to page-1 - url(r'articles/^(?:page-(?P\d+)/)?$', show_articles), -] - - -# Positional arguments - -def xxs_positional_arg1(request, arg0): - return HttpResponse('xxs_positional_arg1: {}'.format(arg0)) - -def xxs_positional_arg2(request, arg0, arg1, arg2): - return HttpResponse('xxs_positional_arg2: {} {} {}'.format(arg0, arg1, arg2)) - -def xxs_positional_arg3(request, arg0, arg1): - return HttpResponse('xxs_positional_arg3: {} {}'.format(arg0, arg1)) - -urlpatterns = [ - # passing as positional argument is not the recommended way of doing things, - # but it is certainly possible - url(r'^(.+)$', xxs_positional_arg1, name='xxs_positional_arg1'), - url(r'^([^/]+)/([^/]+)/([^/]+)$', xxs_positional_arg2, name='xxs_positional_arg2'), - url(r'^([^/]+)/(?:foo|bar)/([^/]+)$', xxs_positional_arg3, name='xxs_positional_arg3'), + url(r'^(?P.*)$', with_template), + url(r'^redirect/(?P.*)$', vuln_redirect), ] diff --git a/python/ql/test/library-tests/web/django/views.py b/python/ql/test/library-tests/web/django/views.py new file mode 100644 index 00000000000..ffa49118785 --- /dev/null +++ b/python/ql/test/library-tests/web/django/views.py @@ -0,0 +1,65 @@ +from django.conf.urls import patterns, url +from django.http.response import HttpResponse +from django.views.generic import View + + +def url_match_xss(request, foo, bar, no_taint=None): + return HttpResponse('url_match_xss: {} {}'.format(foo, bar)) + + +def get_params_xss(request): + return HttpResponse(request.GET.get("untrusted")) + + +def post_params_xss(request): + return HttpResponse(request.POST.get("untrusted")) + + +class ClassView(View): + + # TODO: Currently we don't flag `untrusted` as a DjangoRequestParameter + def get(self, request, untrusted): + return HttpResponse('ClassView: {}'.format(untrusted)) + + +def show_articles(request, page_number=1): + page_number = int(page_number) + return HttpResponse('articles page: {}'.format(page_number)) + + +def xxs_positional_arg(request, arg0, arg1, no_taint=None): + return HttpResponse('xxs_positional_arg: {} {}'.format(arg0, arg1)) + + +urlpatterns = [ + url(r'^url_match/(?P[^/]+)/(?P[^/]+)$', url_match_xss), + url(r'^get_params$', get_params_xss), + url(r'^post_params$', post_params_xss), + url(r'^class_view/(?P.+)$', ClassView.as_view()), + + # one pattern to support `articles/page-` and ensuring that articles/ goes to page-1 + url(r'articles/^(?:page-(?P\d+)/)?$', show_articles), + # passing as positional argument is not the recommended way of doing things, but it is certainly + # possible + url(r'^([^/]+)/(?:foo|bar)/([^/]+)$', xxs_positional_arg, name='xxs_positional_arg'), +] + + +# Using patterns() for routing + +def show_user(request, username): + pass + + +urlpatterns = patterns(url(r'^users/(?P[^/]+)$', show_user)) + + +# Show we understand the keyword arguments to django.conf.urls.url + +def we_understand_url_kwargs(request): + pass + + +urlpatterns = [ + url(view=we_understand_url_kwargs, regex=r'^specifying-as-kwargs-is-not-a-problem$') +] diff --git a/python/ql/test/query-tests/Security/lib/django/conf/urls.py b/python/ql/test/query-tests/Security/lib/django/conf/urls.py index cf65525e893..6c1a83baf0b 100644 --- a/python/ql/test/query-tests/Security/lib/django/conf/urls.py +++ b/python/ql/test/query-tests/Security/lib/django/conf/urls.py @@ -1,7 +1,6 @@ - -def url(pattern, *args): +# https://docs.djangoproject.com/en/1.11/_modules/django/conf/urls/#url +def url(regex, view, kwargs=None, name=None): pass def patterns(*urls): pass - From fc851b46c3d41cda88ecfecd8c35821d96ac7222 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Mon, 21 Oct 2019 16:44:35 +0200 Subject: [PATCH 063/148] Python: Fix Django class-based views --- .../src/semmle/python/web/django/Request.qll | 2 +- .../library-tests/web/django/Sinks.expected | 7 ++++--- .../library-tests/web/django/Sources.expected | 18 ++++++++++-------- .../ql/test/library-tests/web/django/views.py | 11 +++++++++-- .../Security/lib/django/views/__init__.py | 0 .../Security/lib/django/views/generic.py | 2 ++ 6 files changed, 26 insertions(+), 14 deletions(-) create mode 100644 python/ql/test/query-tests/Security/lib/django/views/__init__.py create mode 100644 python/ql/test/query-tests/Security/lib/django/views/generic.py diff --git a/python/ql/src/semmle/python/web/django/Request.qll b/python/ql/src/semmle/python/web/django/Request.qll index 15ea032e6d7..24a29ea4542 100644 --- a/python/ql/src/semmle/python/web/django/Request.qll +++ b/python/ql/src/semmle/python/web/django/Request.qll @@ -67,7 +67,7 @@ private class DjangoView extends ClassValue { } private FunctionValue djangoViewHttpMethod() { - exists(DjangoView view | view.attr(httpVerbLower()) = result) + exists(DjangoView view | view.lookup(httpVerbLower()) = result) } class DjangoClassBasedViewRequestArgument extends DjangoRequestSource { diff --git a/python/ql/test/library-tests/web/django/Sinks.expected b/python/ql/test/library-tests/web/django/Sinks.expected index 86e05bca6dc..e3d233c9990 100644 --- a/python/ql/test/library-tests/web/django/Sinks.expected +++ b/python/ql/test/library-tests/web/django/Sinks.expected @@ -10,6 +10,7 @@ | views.py:7 | Attribute() | externally controlled string | | views.py:11 | Attribute() | externally controlled string | | views.py:15 | Attribute() | externally controlled string | -| views.py:22 | Attribute() | externally controlled string | -| views.py:27 | Attribute() | externally controlled string | -| views.py:31 | Attribute() | externally controlled string | +| views.py:23 | Attribute() | externally controlled string | +| views.py:29 | Attribute() | externally controlled string | +| views.py:34 | Attribute() | externally controlled string | +| views.py:38 | Attribute() | externally controlled string | diff --git a/python/ql/test/library-tests/web/django/Sources.expected b/python/ql/test/library-tests/web/django/Sources.expected index a0c253b27c5..35e0d3d8ced 100644 --- a/python/ql/test/library-tests/web/django/Sources.expected +++ b/python/ql/test/library-tests/web/django/Sources.expected @@ -7,11 +7,13 @@ | views.py:6 | request | django.request.HttpRequest | | views.py:10 | request | django.request.HttpRequest | | views.py:14 | request | django.request.HttpRequest | -| views.py:25 | page_number | externally controlled string | -| views.py:25 | request | django.request.HttpRequest | -| views.py:30 | arg0 | externally controlled string | -| views.py:30 | arg1 | externally controlled string | -| views.py:30 | request | django.request.HttpRequest | -| views.py:50 | request | django.request.HttpRequest | -| views.py:50 | username | externally controlled string | -| views.py:59 | request | django.request.HttpRequest | +| views.py:22 | request | django.request.HttpRequest | +| views.py:28 | request | django.request.HttpRequest | +| views.py:32 | page_number | externally controlled string | +| views.py:32 | request | django.request.HttpRequest | +| views.py:37 | arg0 | externally controlled string | +| views.py:37 | arg1 | externally controlled string | +| views.py:37 | request | django.request.HttpRequest | +| views.py:57 | request | django.request.HttpRequest | +| views.py:57 | username | externally controlled string | +| views.py:66 | request | django.request.HttpRequest | diff --git a/python/ql/test/library-tests/web/django/views.py b/python/ql/test/library-tests/web/django/views.py index ffa49118785..57d1ea5460c 100644 --- a/python/ql/test/library-tests/web/django/views.py +++ b/python/ql/test/library-tests/web/django/views.py @@ -15,11 +15,18 @@ def post_params_xss(request): return HttpResponse(request.POST.get("untrusted")) -class ClassView(View): +class Foo(object): + # Note: since Foo is used as the super type in a class view, it will be able to handle requests. + # TODO: Currently we don't flag `untrusted` as a DjangoRequestParameter + def post(self, request, untrusted): + return HttpResponse('Foo post: {}'.format(untrusted)) + + +class ClassView(View, Foo): # TODO: Currently we don't flag `untrusted` as a DjangoRequestParameter def get(self, request, untrusted): - return HttpResponse('ClassView: {}'.format(untrusted)) + return HttpResponse('ClassView get: {}'.format(untrusted)) def show_articles(request, page_number=1): diff --git a/python/ql/test/query-tests/Security/lib/django/views/__init__.py b/python/ql/test/query-tests/Security/lib/django/views/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/ql/test/query-tests/Security/lib/django/views/generic.py b/python/ql/test/query-tests/Security/lib/django/views/generic.py new file mode 100644 index 00000000000..d924b03d062 --- /dev/null +++ b/python/ql/test/query-tests/Security/lib/django/views/generic.py @@ -0,0 +1,2 @@ +class View: + pass From 63f24476e96771b0d557d3b7c2c01c547068d511 Mon Sep 17 00:00:00 2001 From: Max Schaefer Date: Wed, 30 Oct 2019 08:29:10 +0000 Subject: [PATCH 064/148] JavaScript: Refactor `DoubleEscaping.ql`. --- .../ql/src/Security/CWE-116/DoubleEscaping.ql | 62 ++++++++++++------- 1 file changed, 41 insertions(+), 21 deletions(-) diff --git a/javascript/ql/src/Security/CWE-116/DoubleEscaping.ql b/javascript/ql/src/Security/CWE-116/DoubleEscaping.ql index a150a2d7a7a..124dcf304aa 100644 --- a/javascript/ql/src/Security/CWE-116/DoubleEscaping.ql +++ b/javascript/ql/src/Security/CWE-116/DoubleEscaping.ql @@ -62,30 +62,23 @@ predicate escapingScheme(string metachar, string regex) { } /** - * A call to `String.prototype.replace` that replaces all instances of a pattern. + * A method call that performs string replacement. */ -class Replacement extends DataFlow::Node { - RegExpLiteral pattern; - - Replacement() { - exists(DataFlow::MethodCallNode mcn | this = mcn | - mcn.getMethodName() = "replace" and - pattern.flow().(DataFlow::SourceNode).flowsTo(mcn.getArgument(0)) and - mcn.getNumArgument() = 2 and - pattern.isGlobal() - ) - } - +abstract class Replacement extends DataFlow::Node { /** * Holds if this replacement replaces the string `input` with `output`. */ - predicate replaces(string input, string output) { - exists(DataFlow::MethodCallNode mcn | - mcn = this and - input = getStringValue(pattern) and - output = mcn.getArgument(1).getStringValue() - ) - } + abstract predicate replaces(string input, string output); + + /** + * Gets the input of this replacement. + */ + abstract DataFlow::Node getInput(); + + /** + * Gets the output of this replacement. + */ + abstract DataFlow::SourceNode getOutput(); /** * Holds if this replacement escapes `char` using `metachar`. @@ -119,7 +112,7 @@ class Replacement extends DataFlow::Node { * Gets the previous replacement in this chain of replacements. */ Replacement getPreviousReplacement() { - result = getASimplePredecessor*(this.(DataFlow::MethodCallNode).getReceiver()) + result.getOutput() = getASimplePredecessor*(getInput()) } /** @@ -147,6 +140,33 @@ class Replacement extends DataFlow::Node { } } +/** + * A call to `String.prototype.replace` that replaces all instances of a pattern. + */ +class GlobalStringReplacement extends Replacement, DataFlow::MethodCallNode { + RegExpLiteral pattern; + + GlobalStringReplacement() { + this.getMethodName() = "replace" and + pattern.flow().(DataFlow::SourceNode).flowsTo(this.getArgument(0)) and + this.getNumArgument() = 2 and + pattern.isGlobal() + } + + override predicate replaces(string input, string output) { + input = getStringValue(pattern) and + output = this.getArgument(1).getStringValue() + } + + override DataFlow::Node getInput() { + result = this.getReceiver() + } + + override DataFlow::SourceNode getOutput() { + result = this + } +} + from Replacement primary, Replacement supplementary, string message, string metachar where primary.escapes(metachar, _) and From bd1c99d8a428770a12c90450ce3518854427b352 Mon Sep 17 00:00:00 2001 From: Max Schaefer Date: Wed, 30 Oct 2019 09:35:49 +0000 Subject: [PATCH 065/148] JavaScript: Recognise `JSON.stringify` and `JSON.parse` as escaper/unescaper. --- .../ql/src/Security/CWE-116/DoubleEscaping.ql | 46 +++++++++++++++++++ .../DoubleEscaping/DoubleEscaping.expected | 2 + .../Security/CWE-116/DoubleEscaping/tst.js | 20 ++++++++ 3 files changed, 68 insertions(+) diff --git a/javascript/ql/src/Security/CWE-116/DoubleEscaping.ql b/javascript/ql/src/Security/CWE-116/DoubleEscaping.ql index 124dcf304aa..589a935d5fe 100644 --- a/javascript/ql/src/Security/CWE-116/DoubleEscaping.ql +++ b/javascript/ql/src/Security/CWE-116/DoubleEscaping.ql @@ -167,6 +167,52 @@ class GlobalStringReplacement extends Replacement, DataFlow::MethodCallNode { } } +/** + * A call to `JSON.stringify`, viewed as a string replacement. + */ +class JsonStringifyReplacement extends Replacement, DataFlow::CallNode { + JsonStringifyReplacement() { + this = DataFlow::globalVarRef("JSON").getAMemberCall("stringify") + } + + override predicate replaces(string input, string output) { + input = "\\" and output = "\\\\" + // the other replacements are not relevant for this query + } + + override DataFlow::Node getInput() { + result = this.getArgument(0) + } + + override DataFlow::SourceNode getOutput() { + result = this + } +} + +/** + * A call to `JSON.parse`, viewed as a string replacement. + */ +class JsonParseReplacement extends Replacement { + JsonParserCall self; + + JsonParseReplacement() { + this = self + } + + override predicate replaces(string input, string output) { + input = "\\\\" and output = "\\" + // the other replacements are not relevant for this query + } + + override DataFlow::Node getInput() { + result = self.getInput() + } + + override DataFlow::SourceNode getOutput() { + result = self.getOutput() + } +} + from Replacement primary, Replacement supplementary, string message, string metachar where primary.escapes(metachar, _) and diff --git a/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/DoubleEscaping.expected b/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/DoubleEscaping.expected index d4930ad07cf..e3d5a9a4252 100644 --- a/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/DoubleEscaping.expected +++ b/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/DoubleEscaping.expected @@ -5,3 +5,5 @@ | tst.js:53:10:53:33 | s.repla ... , '\\\\') | This replacement may produce '\\' characters that are double-unescaped $@. | tst.js:53:10:54:33 | s.repla ... , '\\'') | here | | tst.js:60:7:60:28 | s.repla ... '%25') | This replacement may double-escape '%' characters from $@. | tst.js:59:7:59:28 | s.repla ... '%26') | here | | tst.js:68:10:70:38 | s.repla ... &") | This replacement may double-escape '&' characters from $@. | tst.js:68:10:69:39 | s.repla ... apos;") | here | +| tst.js:74:10:77:10 | JSON.st ... ) | This replacement may double-escape '\\' characters from $@. | tst.js:75:12:76:37 | s.repla ... u003E") | here | +| tst.js:86:10:86:22 | JSON.parse(s) | This replacement may produce '\\' characters that are double-unescaped $@. | tst.js:86:10:86:47 | JSON.pa ... g, "<") | here | diff --git a/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/tst.js b/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/tst.js index 19ab94d2691..ee9cd930b6f 100644 --- a/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/tst.js +++ b/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/tst.js @@ -69,3 +69,23 @@ function badEncode(s) { .replace(indirect2, "'") .replace(indirect3, "&"); } + +function badEscape1(s) { + return JSON.stringify( + s.replace(//g, "\\u003E") + ); +} + +function goodEscape1(s) { + return JSON.stringify(s) + .replace(//g, "\\u003E"); +} + +function badUnescape2(s) { + return JSON.parse(s).replace(/\\u003C/g, "<").replace(/\\u003E/g, ">"); +} + +function goodUnescape2(s) { + return JSON.parse(s.replace(/\\u003C/g, "<").replace(/\\u003E/g, ">")); +} From 1e6c983d62ea4d96730841d5672a5277cdc378cd Mon Sep 17 00:00:00 2001 From: Jonas Jensen Date: Wed, 30 Oct 2019 13:42:17 +0100 Subject: [PATCH 066/148] C++: Use getASTVariable in DefaultTaintTracking This library is not yet used in a query or test, so it broke silently when `VariableAddressInstruction.getVariable` was removed. --- cpp/ql/src/semmle/code/cpp/ir/dataflow/DefaultTaintTracking.qll | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/ql/src/semmle/code/cpp/ir/dataflow/DefaultTaintTracking.qll b/cpp/ql/src/semmle/code/cpp/ir/dataflow/DefaultTaintTracking.qll index 08556a4af39..0753dfd266e 100644 --- a/cpp/ql/src/semmle/code/cpp/ir/dataflow/DefaultTaintTracking.qll +++ b/cpp/ql/src/semmle/code/cpp/ir/dataflow/DefaultTaintTracking.qll @@ -37,7 +37,7 @@ private class DefaultTaintTrackingCfg extends DataFlow::Configuration { } private predicate accessesVariable(CopyInstruction copy, Variable var) { - exists(VariableAddressInstruction va | va.getVariable().getAST() = var | + exists(VariableAddressInstruction va | va.getASTVariable() = var | copy.(StoreInstruction).getDestinationAddress() = va or copy.(LoadInstruction).getSourceAddress() = va From aaeca325193deedc7ac8ea163cbe574514d44b94 Mon Sep 17 00:00:00 2001 From: Max Schaefer Date: Wed, 30 Oct 2019 09:55:28 +0000 Subject: [PATCH 067/148] JavaScript: Recognize string escaping using `.replace` with a callback. --- javascript/ql/src/Security/CWE-116/DoubleEscaping.ql | 8 ++++++++ .../CWE-116/DoubleEscaping/DoubleEscaping.expected | 1 + .../query-tests/Security/CWE-116/DoubleEscaping/tst.js | 9 +++++++++ 3 files changed, 18 insertions(+) diff --git a/javascript/ql/src/Security/CWE-116/DoubleEscaping.ql b/javascript/ql/src/Security/CWE-116/DoubleEscaping.ql index 589a935d5fe..4e6969e54c3 100644 --- a/javascript/ql/src/Security/CWE-116/DoubleEscaping.ql +++ b/javascript/ql/src/Security/CWE-116/DoubleEscaping.ql @@ -156,6 +156,14 @@ class GlobalStringReplacement extends Replacement, DataFlow::MethodCallNode { override predicate replaces(string input, string output) { input = getStringValue(pattern) and output = this.getArgument(1).getStringValue() + or + exists(DataFlow::FunctionNode replacer, DataFlow::PropRead pr, DataFlow::ObjectLiteralNode map | + replacer = getCallback(1) and + replacer.getParameter(0).flowsToExpr(pr.getPropertyNameExpr()) and + pr = map.getAPropertyRead() and + pr.flowsTo(replacer.getAReturn()) and + map.asExpr().(ObjectExpr).getPropertyByName(input).getInit().getStringValue() = output + ) } override DataFlow::Node getInput() { diff --git a/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/DoubleEscaping.expected b/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/DoubleEscaping.expected index e3d5a9a4252..d7e2b558a06 100644 --- a/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/DoubleEscaping.expected +++ b/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/DoubleEscaping.expected @@ -7,3 +7,4 @@ | tst.js:68:10:70:38 | s.repla ... &") | This replacement may double-escape '&' characters from $@. | tst.js:68:10:69:39 | s.repla ... apos;") | here | | tst.js:74:10:77:10 | JSON.st ... ) | This replacement may double-escape '\\' characters from $@. | tst.js:75:12:76:37 | s.repla ... u003E") | here | | tst.js:86:10:86:22 | JSON.parse(s) | This replacement may produce '\\' characters that are double-unescaped $@. | tst.js:86:10:86:47 | JSON.pa ... g, "<") | here | +| tst.js:99:10:99:66 | s.repla ... &") | This replacement may double-escape '&' characters from $@. | tst.js:99:10:99:43 | s.repla ... epl[c]) | here | diff --git a/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/tst.js b/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/tst.js index ee9cd930b6f..78c73ce9584 100644 --- a/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/tst.js +++ b/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/tst.js @@ -89,3 +89,12 @@ function badUnescape2(s) { function goodUnescape2(s) { return JSON.parse(s.replace(/\\u003C/g, "<").replace(/\\u003E/g, ">")); } + +function badEncodeWithReplacer(s) { + var repl = { + '"': """, + "'": "'", + "&": "&" + }; + return s.replace(/["']/g, (c) => repl[c]).replace(/&/g, "&"); +} From 02d16b1dc99ddbcb3b2c8148b0cb53731214b039 Mon Sep 17 00:00:00 2001 From: Max Schaefer Date: Wed, 30 Oct 2019 10:59:04 +0000 Subject: [PATCH 068/148] JavaScript: Recognise wrapped string replacement functions. --- .../ql/src/Security/CWE-116/DoubleEscaping.ql | 28 +++++++++++++++++++ .../DoubleEscaping/DoubleEscaping.expected | 1 + .../Security/CWE-116/DoubleEscaping/tst.js | 8 ++++++ 3 files changed, 37 insertions(+) diff --git a/javascript/ql/src/Security/CWE-116/DoubleEscaping.ql b/javascript/ql/src/Security/CWE-116/DoubleEscaping.ql index 4e6969e54c3..733afad0b06 100644 --- a/javascript/ql/src/Security/CWE-116/DoubleEscaping.ql +++ b/javascript/ql/src/Security/CWE-116/DoubleEscaping.ql @@ -221,6 +221,34 @@ class JsonParseReplacement extends Replacement { } } +/** + * A string replacement wrapped in a utility function. + */ +class WrappedReplacement extends Replacement, DataFlow::CallNode { + int i; + + Replacement inner; + + WrappedReplacement() { + exists(DataFlow::FunctionNode wrapped | wrapped.getFunction() = getACallee() | + wrapped.getParameter(i).flowsTo(inner.getInput()) and + inner.getOutput().flowsTo(wrapped.getAReturn()) + ) + } + + override predicate replaces(string input, string output) { + inner.replaces(input, output) + } + + override DataFlow::Node getInput() { + result = getArgument(i) + } + + override DataFlow::SourceNode getOutput() { + result = this + } +} + from Replacement primary, Replacement supplementary, string message, string metachar where primary.escapes(metachar, _) and diff --git a/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/DoubleEscaping.expected b/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/DoubleEscaping.expected index d7e2b558a06..d0747087d12 100644 --- a/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/DoubleEscaping.expected +++ b/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/DoubleEscaping.expected @@ -8,3 +8,4 @@ | tst.js:74:10:77:10 | JSON.st ... ) | This replacement may double-escape '\\' characters from $@. | tst.js:75:12:76:37 | s.repla ... u003E") | here | | tst.js:86:10:86:22 | JSON.parse(s) | This replacement may produce '\\' characters that are double-unescaped $@. | tst.js:86:10:86:47 | JSON.pa ... g, "<") | here | | tst.js:99:10:99:66 | s.repla ... &") | This replacement may double-escape '&' characters from $@. | tst.js:99:10:99:43 | s.repla ... epl[c]) | here | +| tst.js:107:10:107:53 | encodeD ... &") | This replacement may double-escape '&' characters from $@. | tst.js:107:10:107:30 | encodeD ... otes(s) | here | diff --git a/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/tst.js b/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/tst.js index 78c73ce9584..d88422e74a4 100644 --- a/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/tst.js +++ b/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/tst.js @@ -98,3 +98,11 @@ function badEncodeWithReplacer(s) { }; return s.replace(/["']/g, (c) => repl[c]).replace(/&/g, "&"); } + +function encodeDoubleQuotes(s) { + return s.replace(/"/g, """); +} + +function badWrappedEncode(s) { + return encodeDoubleQuotes(s).replace(/&/g, "&"); +} From 5349e0f88142ae4e7a63038fa3238b5001e0972d Mon Sep 17 00:00:00 2001 From: Max Schaefer Date: Wed, 30 Oct 2019 13:02:59 +0000 Subject: [PATCH 069/148] JavaScript: Recognise wrapped chains of replacements. --- javascript/ql/src/Security/CWE-116/DoubleEscaping.ql | 11 +++++++++-- .../CWE-116/DoubleEscaping/DoubleEscaping.expected | 1 + .../Security/CWE-116/DoubleEscaping/tst.js | 8 ++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/javascript/ql/src/Security/CWE-116/DoubleEscaping.ql b/javascript/ql/src/Security/CWE-116/DoubleEscaping.ql index 733afad0b06..2fc91a9854d 100644 --- a/javascript/ql/src/Security/CWE-116/DoubleEscaping.ql +++ b/javascript/ql/src/Security/CWE-116/DoubleEscaping.ql @@ -115,6 +115,13 @@ abstract class Replacement extends DataFlow::Node { result.getOutput() = getASimplePredecessor*(getInput()) } + /** + * Gets the next replacement in this chain of replacements. + */ + Replacement getNextReplacement() { + this = result.getPreviousReplacement() + } + /** * Gets an earlier replacement in this chain of replacements that * performs an escaping. @@ -231,8 +238,8 @@ class WrappedReplacement extends Replacement, DataFlow::CallNode { WrappedReplacement() { exists(DataFlow::FunctionNode wrapped | wrapped.getFunction() = getACallee() | - wrapped.getParameter(i).flowsTo(inner.getInput()) and - inner.getOutput().flowsTo(wrapped.getAReturn()) + wrapped.getParameter(i).flowsTo(inner.getPreviousReplacement*().getInput()) and + inner.getNextReplacement*().getOutput().flowsTo(wrapped.getAReturn()) ) } diff --git a/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/DoubleEscaping.expected b/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/DoubleEscaping.expected index d0747087d12..23e5a4a6b05 100644 --- a/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/DoubleEscaping.expected +++ b/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/DoubleEscaping.expected @@ -9,3 +9,4 @@ | tst.js:86:10:86:22 | JSON.parse(s) | This replacement may produce '\\' characters that are double-unescaped $@. | tst.js:86:10:86:47 | JSON.pa ... g, "<") | here | | tst.js:99:10:99:66 | s.repla ... &") | This replacement may double-escape '&' characters from $@. | tst.js:99:10:99:43 | s.repla ... epl[c]) | here | | tst.js:107:10:107:53 | encodeD ... &") | This replacement may double-escape '&' characters from $@. | tst.js:107:10:107:30 | encodeD ... otes(s) | here | +| tst.js:115:10:115:47 | encodeQ ... &") | This replacement may double-escape '&' characters from $@. | tst.js:115:10:115:24 | encodeQuotes(s) | here | diff --git a/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/tst.js b/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/tst.js index d88422e74a4..628166ed04a 100644 --- a/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/tst.js +++ b/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/tst.js @@ -106,3 +106,11 @@ function encodeDoubleQuotes(s) { function badWrappedEncode(s) { return encodeDoubleQuotes(s).replace(/&/g, "&"); } + +function encodeQuotes(s) { + return s.replace(/"/g, """).replace(/'/g, "'"); +} + +function badWrappedEncode2(s) { + return encodeQuotes(s).replace(/&/g, "&"); +} From a8214ce7ee070ad082430a6ea0140548ea1b0799 Mon Sep 17 00:00:00 2001 From: Max Schaefer Date: Wed, 30 Oct 2019 14:15:59 +0000 Subject: [PATCH 070/148] JavaScript: Fix regexes for escaping schemes. --- javascript/ql/src/Security/CWE-116/DoubleEscaping.ql | 6 +++--- .../query-tests/Security/CWE-116/DoubleEscaping/tst.js | 9 +++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/javascript/ql/src/Security/CWE-116/DoubleEscaping.ql b/javascript/ql/src/Security/CWE-116/DoubleEscaping.ql index 2fc91a9854d..5d5157bafb3 100644 --- a/javascript/ql/src/Security/CWE-116/DoubleEscaping.ql +++ b/javascript/ql/src/Security/CWE-116/DoubleEscaping.ql @@ -54,11 +54,11 @@ DataFlow::Node getASimplePredecessor(DataFlow::Node nd) { * into a form described by regular expression `regex`. */ predicate escapingScheme(string metachar, string regex) { - metachar = "&" and regex = "&.*;" + metachar = "&" and regex = "&.+;" or - metachar = "%" and regex = "%.*" + metachar = "%" and regex = "%.+" or - metachar = "\\" and regex = "\\\\.*" + metachar = "\\" and regex = "\\\\.+" } /** diff --git a/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/tst.js b/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/tst.js index 628166ed04a..e05779eeb7a 100644 --- a/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/tst.js +++ b/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/tst.js @@ -114,3 +114,12 @@ function encodeQuotes(s) { function badWrappedEncode2(s) { return encodeQuotes(s).replace(/&/g, "&"); } + +function roundtrip(s) { + return JSON.parse(JSON.stringify(s)); +} + +// dubious, but out of scope for this query +function badRoundtrip(s) { + return s.replace(/\\\\/g, "\\").replace(/\\/g, "\\\\"); +} From 8c133ff61d873b574ff0fef778f5248f8c4b11d9 Mon Sep 17 00:00:00 2001 From: Max Schaefer Date: Wed, 30 Oct 2019 14:46:50 +0000 Subject: [PATCH 071/148] JavaScript: Deal with (un-)escaping on captured variables. --- javascript/ql/src/Security/CWE-116/DoubleEscaping.ql | 7 ++++++- .../query-tests/Security/CWE-116/DoubleEscaping/tst.js | 7 +++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/javascript/ql/src/Security/CWE-116/DoubleEscaping.ql b/javascript/ql/src/Security/CWE-116/DoubleEscaping.ql index 5d5157bafb3..b6d76956630 100644 --- a/javascript/ql/src/Security/CWE-116/DoubleEscaping.ql +++ b/javascript/ql/src/Security/CWE-116/DoubleEscaping.ql @@ -46,7 +46,12 @@ string getStringValue(RegExpLiteral rl) { */ DataFlow::Node getASimplePredecessor(DataFlow::Node nd) { result = nd.getAPredecessor() and - not nd.(DataFlow::SsaDefinitionNode).getSsaVariable().getDefinition() instanceof SsaPhiNode + not exists(SsaDefinition ssa | + ssa = nd.(DataFlow::SsaDefinitionNode).getSsaVariable().getDefinition() + | + ssa instanceof SsaPhiNode or + ssa instanceof SsaVariableCapture + ) } /** diff --git a/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/tst.js b/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/tst.js index e05779eeb7a..7b840e2be08 100644 --- a/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/tst.js +++ b/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/tst.js @@ -123,3 +123,10 @@ function roundtrip(s) { function badRoundtrip(s) { return s.replace(/\\\\/g, "\\").replace(/\\/g, "\\\\"); } + +function testWithCapturedVar(x) { + var captured = x; + (function() { + captured = captured.replace(/\\/g, "\\\\"); + })(); +} From bb0771b36c3065c60dd4aa3f04a481b241ff9c86 Mon Sep 17 00:00:00 2001 From: Max Schaefer Date: Wed, 30 Oct 2019 14:49:01 +0000 Subject: [PATCH 072/148] JavaScript: Deal with escape-unescape-escape (and similar) chains. --- javascript/ql/src/Security/CWE-116/DoubleEscaping.ql | 8 ++++++-- .../query-tests/Security/CWE-116/DoubleEscaping/tst.js | 4 ++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/javascript/ql/src/Security/CWE-116/DoubleEscaping.ql b/javascript/ql/src/Security/CWE-116/DoubleEscaping.ql index b6d76956630..db8624458b7 100644 --- a/javascript/ql/src/Security/CWE-116/DoubleEscaping.ql +++ b/javascript/ql/src/Security/CWE-116/DoubleEscaping.ql @@ -135,7 +135,9 @@ abstract class Replacement extends DataFlow::Node { exists(Replacement pred | pred = this.getPreviousReplacement() | if pred.escapes(_, metachar) then result = pred - else result = pred.getAnEarlierEscaping(metachar) + else ( + not pred.unescapes(metachar, _) and result = pred.getAnEarlierEscaping(metachar) + ) ) } @@ -147,7 +149,9 @@ abstract class Replacement extends DataFlow::Node { exists(Replacement succ | this = succ.getPreviousReplacement() | if succ.unescapes(metachar, _) then result = succ - else result = succ.getALaterUnescaping(metachar) + else ( + not succ.escapes(_, metachar) and result = succ.getALaterUnescaping(metachar) + ) ) } } diff --git a/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/tst.js b/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/tst.js index 7b840e2be08..9c69c837f6b 100644 --- a/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/tst.js +++ b/javascript/ql/test/query-tests/Security/CWE-116/DoubleEscaping/tst.js @@ -130,3 +130,7 @@ function testWithCapturedVar(x) { captured = captured.replace(/\\/g, "\\\\"); })(); } + +function cloneAndStringify(s) { + return JSON.stringify(JSON.parse(JSON.stringify(s))); +} From 3bbded57d3c45c1adc1056d23c3c74f0ffb38568 Mon Sep 17 00:00:00 2001 From: Max Schaefer Date: Wed, 30 Oct 2019 14:49:18 +0000 Subject: [PATCH 073/148] JavaScript: Autoformat. --- .../ql/src/Security/CWE-116/DoubleEscaping.ql | 52 +++++-------------- 1 file changed, 13 insertions(+), 39 deletions(-) diff --git a/javascript/ql/src/Security/CWE-116/DoubleEscaping.ql b/javascript/ql/src/Security/CWE-116/DoubleEscaping.ql index db8624458b7..6340a2fcedc 100644 --- a/javascript/ql/src/Security/CWE-116/DoubleEscaping.ql +++ b/javascript/ql/src/Security/CWE-116/DoubleEscaping.ql @@ -116,16 +116,12 @@ abstract class Replacement extends DataFlow::Node { /** * Gets the previous replacement in this chain of replacements. */ - Replacement getPreviousReplacement() { - result.getOutput() = getASimplePredecessor*(getInput()) - } + Replacement getPreviousReplacement() { result.getOutput() = getASimplePredecessor*(getInput()) } /** * Gets the next replacement in this chain of replacements. */ - Replacement getNextReplacement() { - this = result.getPreviousReplacement() - } + Replacement getNextReplacement() { this = result.getPreviousReplacement() } /** * Gets an earlier replacement in this chain of replacements that @@ -182,35 +178,25 @@ class GlobalStringReplacement extends Replacement, DataFlow::MethodCallNode { ) } - override DataFlow::Node getInput() { - result = this.getReceiver() - } + override DataFlow::Node getInput() { result = this.getReceiver() } - override DataFlow::SourceNode getOutput() { - result = this - } + override DataFlow::SourceNode getOutput() { result = this } } /** * A call to `JSON.stringify`, viewed as a string replacement. */ class JsonStringifyReplacement extends Replacement, DataFlow::CallNode { - JsonStringifyReplacement() { - this = DataFlow::globalVarRef("JSON").getAMemberCall("stringify") - } + JsonStringifyReplacement() { this = DataFlow::globalVarRef("JSON").getAMemberCall("stringify") } override predicate replaces(string input, string output) { input = "\\" and output = "\\\\" // the other replacements are not relevant for this query } - override DataFlow::Node getInput() { - result = this.getArgument(0) - } + override DataFlow::Node getInput() { result = this.getArgument(0) } - override DataFlow::SourceNode getOutput() { - result = this - } + override DataFlow::SourceNode getOutput() { result = this } } /** @@ -219,22 +205,16 @@ class JsonStringifyReplacement extends Replacement, DataFlow::CallNode { class JsonParseReplacement extends Replacement { JsonParserCall self; - JsonParseReplacement() { - this = self - } + JsonParseReplacement() { this = self } override predicate replaces(string input, string output) { input = "\\\\" and output = "\\" // the other replacements are not relevant for this query } - override DataFlow::Node getInput() { - result = self.getInput() - } + override DataFlow::Node getInput() { result = self.getInput() } - override DataFlow::SourceNode getOutput() { - result = self.getOutput() - } + override DataFlow::SourceNode getOutput() { result = self.getOutput() } } /** @@ -252,17 +232,11 @@ class WrappedReplacement extends Replacement, DataFlow::CallNode { ) } - override predicate replaces(string input, string output) { - inner.replaces(input, output) - } + override predicate replaces(string input, string output) { inner.replaces(input, output) } - override DataFlow::Node getInput() { - result = getArgument(i) - } + override DataFlow::Node getInput() { result = getArgument(i) } - override DataFlow::SourceNode getOutput() { - result = this - } + override DataFlow::SourceNode getOutput() { result = this } } from Replacement primary, Replacement supplementary, string message, string metachar From 27d0b51c6b4eabc891c71eb8952874c07932c7e3 Mon Sep 17 00:00:00 2001 From: alistair Date: Wed, 30 Oct 2019 16:10:03 +0000 Subject: [PATCH 074/148] CPP & C#: Review of qhelp PR #2151 got merged without a review of the qhelp by a technical writer. The current PR makes changes I would have suggested on that PR. --- .../NtohlArrayNoBoundOpenSource.qhelp | 2 +- .../ConditionallyUninitializedVariable.qhelp | 2 +- .../ConditionallyUninitializedVariable.ql | 2 +- .../CWE-091/XMLInjection.qhelp | 4 ++-- .../CWE-114/AssemblyPathInjection.qhelp | 6 ++--- .../CWE-321/HardcodedEncryptionKey.ql | 4 ++-- .../CWE-327/InsecureSQLConnection.qhelp | 22 +++++++++---------- .../CWE-327/InsecureSQLConnection.ql | 2 +- 8 files changed, 21 insertions(+), 23 deletions(-) diff --git a/cpp/ql/src/Likely Bugs/Memory Management/Buffer Overflow/NtohlArrayNoBoundOpenSource.qhelp b/cpp/ql/src/Likely Bugs/Memory Management/Buffer Overflow/NtohlArrayNoBoundOpenSource.qhelp index 522d6cde74c..fc8f309f73a 100644 --- a/cpp/ql/src/Likely Bugs/Memory Management/Buffer Overflow/NtohlArrayNoBoundOpenSource.qhelp +++ b/cpp/ql/src/Likely Bugs/Memory Management/Buffer Overflow/NtohlArrayNoBoundOpenSource.qhelp @@ -14,7 +14,7 @@ byte order function, such as ntohl. The use of a network-to-host byte order function is therefore a good indicator that the returned value is unvalidated data retrieved from the network, and should not be used without further validation. In particular, the returned value should not be used as an array index or array length -value without validation, which may result in a buffer overflow vulnerability. +value without validation, as this could result in a buffer overflow vulnerability.

    diff --git a/cpp/ql/src/Security/CWE/CWE-457/ConditionallyUninitializedVariable.qhelp b/cpp/ql/src/Security/CWE/CWE-457/ConditionallyUninitializedVariable.qhelp index b9118edc736..8e6a8903483 100644 --- a/cpp/ql/src/Security/CWE/CWE-457/ConditionallyUninitializedVariable.qhelp +++ b/cpp/ql/src/Security/CWE/CWE-457/ConditionallyUninitializedVariable.qhelp @@ -37,7 +37,7 @@ which is then subsequently accessed to fetch properties of the device. However, check the return value from the function call to initDeviceConfig. If the device number passed to the notify function was invalid, the initDeviceConfig function will leave the config variable uninitialized, -which would result in the notify function accessing uninitialized memory.

    +which will result in the notify function accessing uninitialized memory.

    diff --git a/cpp/ql/src/Security/CWE/CWE-457/ConditionallyUninitializedVariable.ql b/cpp/ql/src/Security/CWE/CWE-457/ConditionallyUninitializedVariable.ql index eb00fb9ea10..f9eb2fe5400 100644 --- a/cpp/ql/src/Security/CWE/CWE-457/ConditionallyUninitializedVariable.ql +++ b/cpp/ql/src/Security/CWE/CWE-457/ConditionallyUninitializedVariable.ql @@ -2,7 +2,7 @@ * @name Conditionally uninitialized variable * @description When an initialization function is used to initialize a local variable, but the * returned status code is not checked, the variable may be left in an uninitialized - * state, and reading the variable may result in undefined behaviour. + * state, and reading the variable may result in undefined behavior. * @kind problem * @problem.severity warning * @opaque-id SM02313 diff --git a/csharp/ql/src/Security Features/CWE-091/XMLInjection.qhelp b/csharp/ql/src/Security Features/CWE-091/XMLInjection.qhelp index 4e70b06531a..3aff9901bfc 100644 --- a/csharp/ql/src/Security Features/CWE-091/XMLInjection.qhelp +++ b/csharp/ql/src/Security Features/CWE-091/XMLInjection.qhelp @@ -36,10 +36,10 @@ which ensures the content is appropriately escaped.

  • - XML Injection (The Web Application Security Consortium). + Web Application Security Consortium: XML Injection.
  • - WriteRaw (Microsoft documentation). + Microsoft Docs: WriteRaw.
    • diff --git a/csharp/ql/src/Security Features/CWE-114/AssemblyPathInjection.qhelp b/csharp/ql/src/Security Features/CWE-114/AssemblyPathInjection.qhelp index 5f95181c092..e1dbe9c1bd0 100644 --- a/csharp/ql/src/Security Features/CWE-114/AssemblyPathInjection.qhelp +++ b/csharp/ql/src/Security Features/CWE-114/AssemblyPathInjection.qhelp @@ -14,7 +14,7 @@ was not intended to be loaded, and executing arbitrary code.

    Avoid loading assemblies based on user provided input. If this is not possible, ensure that the path is validated before being used with Assembly. For example, compare the provided input -against a whitelist of known safe assemblies, or confirm that path is restricted to a single +against a whitelist of known safe assemblies, or confirm that the path is restricted to a single directory which only contains safe assemblies.

    @@ -30,8 +30,8 @@ is only loaded if the user input matches one of those options.

    -
  • - System.Reflection.Assembly (Microsoft documentation). +
  • Microsoft: + System.Reflection.Assembly.
    • diff --git a/csharp/ql/src/Security Features/CWE-321/HardcodedEncryptionKey.ql b/csharp/ql/src/Security Features/CWE-321/HardcodedEncryptionKey.ql index c09a67d756c..cce122ffa62 100644 --- a/csharp/ql/src/Security Features/CWE-321/HardcodedEncryptionKey.ql +++ b/csharp/ql/src/Security Features/CWE-321/HardcodedEncryptionKey.ql @@ -1,6 +1,6 @@ /** - * @name Do not use hard-coded encryption keys. - * @description The .Key property or rgbKey parameter of a SymmetricAlgorithm should never be a hardcoded value. + * @name Hard-coded encryption key + * @description The .Key property or rgbKey parameter of a SymmetricAlgorithm should never be a hard-coded value. * @kind problem * @id cs/hardcoded-key * @problem.severity error diff --git a/csharp/ql/src/Security Features/CWE-327/InsecureSQLConnection.qhelp b/csharp/ql/src/Security Features/CWE-327/InsecureSQLConnection.qhelp index ae69402eb7e..c59feeed61c 100644 --- a/csharp/ql/src/Security Features/CWE-327/InsecureSQLConnection.qhelp +++ b/csharp/ql/src/Security Features/CWE-327/InsecureSQLConnection.qhelp @@ -3,7 +3,6 @@ "qhelp.dtd"> -

    Finds uses of insecure SQL Connections string by not enabling the Encrypt option.

    SQL Server connections where the client is not enforcing the encryption in transit are susceptible to multiple attacks, including a man-in-the-middle, that would potentially compromise the user credentials and/or the TDS session. @@ -29,18 +28,17 @@ - -

  • - Selectively using secure connection to SQL Server +
  • Microsoft, SQL Protocols blog: + Selectively using secure connection to SQL Server.
    • -
  • - Net SqlClient (ADO .Net) +
  • Microsoft: + SqlConnection.ConnectionString Property. +
    • +
  • Microsoft: + Using Connection String Keywords with SQL Server Native Client. +
    • +
  • Microsoft: + Setting the connection properties.
    • -
  • SQL native driver (SNAC) -
    • -
  • - JDBC driver -
    • -     diff --git a/csharp/ql/src/Security Features/CWE-327/InsecureSQLConnection.ql b/csharp/ql/src/Security Features/CWE-327/InsecureSQLConnection.ql index 0f855150aa9..78bcc1c19e5 100644 --- a/csharp/ql/src/Security Features/CWE-327/InsecureSQLConnection.ql +++ b/csharp/ql/src/Security Features/CWE-327/InsecureSQLConnection.ql @@ -1,6 +1,6 @@ /** * @name Insecure SQL connection - * @description TODO. + * @description Using an SQL Server connection without enforcing encryption is a security vulnerability. * @kind path-problem * @id cs/insecure-sql-connection * @problem.severity error From 24c9b8b9b14dba8b2b348790a6edb5754af31d70 Mon Sep 17 00:00:00 2001 From: Robert Marsh Date: Wed, 30 Oct 2019 14:06:19 -0700 Subject: [PATCH 075/148] C++: fix unbound variables --- .../cpp/ir/implementation/raw/internal/TranslatedCall.qll | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cpp/ql/src/semmle/code/cpp/ir/implementation/raw/internal/TranslatedCall.qll b/cpp/ql/src/semmle/code/cpp/ir/implementation/raw/internal/TranslatedCall.qll index fbe4910a0dd..75e0d463f2a 100644 --- a/cpp/ql/src/semmle/code/cpp/ir/implementation/raw/internal/TranslatedCall.qll +++ b/cpp/ql/src/semmle/code/cpp/ir/implementation/raw/internal/TranslatedCall.qll @@ -461,7 +461,10 @@ class TranslatedSideEffect extends TranslatedElement, TTranslatedArgumentSideEff override Type getInstructionOperandType(InstructionTag tag, TypedOperandTag operandTag) { if hasSpecificReadSideEffect(any(Opcode::BufferReadSideEffect op)) - then result instanceof UnknownType + then + result instanceof UnknownType and + tag instanceof OnlyInstructionTag and + operandTag instanceof SideEffectOperandTag else ( tag instanceof OnlyInstructionTag and result = arg.getType().getUnspecifiedType().(DerivedType).getBaseType() and From ceea96e03f0649c324ee4095f0c1033679cf2052 Mon Sep 17 00:00:00 2001 From: Tom Hvitved Date: Thu, 31 Oct 2019 12:00:16 +0100 Subject: [PATCH 076/148] C#: Update change note --- change-notes/1.23/analysis-csharp.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/change-notes/1.23/analysis-csharp.md b/change-notes/1.23/analysis-csharp.md index 4f1cb268468..cde0b4db71e 100644 --- a/change-notes/1.23/analysis-csharp.md +++ b/change-notes/1.23/analysis-csharp.md @@ -8,7 +8,7 @@ The following changes in version 1.23 affect C# analysis in all applications. | **Query** | **Tags** | **Purpose** | |-----------------------------|-----------|--------------------------------------------------------------------| -| Deserialized delegate (`cs/deserialized-delegate`) | security | Finds unsafe deserialization of delegate types. | +| Deserialized delegate (`cs/deserialized-delegate`) | security, external/cwe/cwe-502 | Finds unsafe deserialization of delegate types. | | Unsafe year argument for 'DateTime' constructor (`cs/unsafe-year-construction`) | reliability, date-time | Finds incorrect manipulation of `DateTime` values, which could lead to invalid dates. | | Mishandling the Japanese era start date (`cs/mishandling-japanese-era`) | reliability, date-time | Finds hard-coded Japanese era start dates that could be invalid. | From 8aae1f443f7242c39c1a56974197525b9166dc59 Mon Sep 17 00:00:00 2001 From: Max Schaefer Date: Tue, 22 Oct 2019 09:18:34 +0100 Subject: [PATCH 077/148] JavaScript: Use type tracking instead of auxiliary data-flow configuration to track indirect command arguments. --- .../dataflow/IndirectCommandArgument.qll | 48 +++++++------ .../CWE-078/CommandInjection.expected | 67 ------------------- .../CWE-078/IndirectCommandInjection.expected | 67 ------------------- ...llCommandInjectionFromEnvironment.expected | 67 ------------------- 4 files changed, 26 insertions(+), 223 deletions(-) diff --git a/javascript/ql/src/semmle/javascript/security/dataflow/IndirectCommandArgument.qll b/javascript/ql/src/semmle/javascript/security/dataflow/IndirectCommandArgument.qll index f509d9f8c6c..9f62ad7ccc8 100644 --- a/javascript/ql/src/semmle/javascript/security/dataflow/IndirectCommandArgument.qll +++ b/javascript/ql/src/semmle/javascript/security/dataflow/IndirectCommandArgument.qll @@ -10,7 +10,7 @@ import javascript * That is, either `shell` is a Unix shell (`sh` or similar) and * `arg` is `"-c"`, or `shell` is `cmd.exe` and `arg` is `"/c"`. */ -private predicate shellCmd(ConstantString shell, string arg) { +private predicate shellCmd(Expr shell, string arg) { exists(string s | s = shell.getStringValue() | (s = "sh" or s = "bash" or s = "/bin/sh" or s = "/bin/bash") and arg = "-c" @@ -23,25 +23,29 @@ private predicate shellCmd(ConstantString shell, string arg) { } /** - * Data flow configuration for tracking string literals that look like they - * may refer to an operating-system shell, and array literals that may end up being - * interpreted as argument lists for system commands. + * Gets a data-flow node whose value ends up being interpreted as the command argument in `sys` + * after a flow summarized by `t`. */ -private class ArgumentListTracking extends DataFlow::Configuration { - ArgumentListTracking() { this = "ArgumentListTracking" } +private DataFlow::Node commandArgument(SystemCommandExecution sys, DataFlow::TypeBackTracker t) { + t.start() and + result = sys.getACommandArgument() + or + exists(DataFlow::TypeBackTracker t2 | + t = t2.smallstep(result, commandArgument(sys, t2)) + ) +} - override predicate isSource(DataFlow::Node nd) { - nd instanceof DataFlow::ArrayCreationNode - or - exists(ConstantString shell | shellCmd(shell, _) | nd = DataFlow::valueNode(shell)) - } - - override predicate isSink(DataFlow::Node nd) { - exists(SystemCommandExecution sys | - nd = sys.getACommandArgument() or - nd = sys.getArgumentList() - ) - } +/** + * Gets a data-flow node whose value ends up being interpreted as the argument list in `sys` + * after a flow summarized by `t`. + */ +private DataFlow::SourceNode argumentList(SystemCommandExecution sys, DataFlow::TypeBackTracker t) { + t.start() and + result = sys.getArgumentList().getALocalSource() + or + exists(DataFlow::TypeBackTracker t2 | + result = argumentList(sys, t2).backtrack(t2, t) + ) } /** @@ -60,11 +64,11 @@ private class ArgumentListTracking extends DataFlow::Configuration { */ predicate isIndirectCommandArgument(DataFlow::Node source, SystemCommandExecution sys) { exists( - ArgumentListTracking cfg, DataFlow::ArrayCreationNode args, ConstantString shell, string dashC + DataFlow::ArrayCreationNode args, DataFlow::Node shell, string dashC | - shellCmd(shell, dashC) and - cfg.hasFlow(DataFlow::valueNode(shell), sys.getACommandArgument()) and - cfg.hasFlow(args, sys.getArgumentList()) and + shellCmd(shell.asExpr(), dashC) and + shell = commandArgument(sys, DataFlow::TypeBackTracker::end()) and + args = argumentList(sys, DataFlow::TypeBackTracker::end()) and args.getAPropertyWrite().getRhs().mayHaveStringValue(dashC) and source = args.getAPropertyWrite().getRhs() ) diff --git a/javascript/ql/test/query-tests/Security/CWE-078/CommandInjection.expected b/javascript/ql/test/query-tests/Security/CWE-078/CommandInjection.expected index 115db66ab5a..abfab9e262d 100644 --- a/javascript/ql/test/query-tests/Security/CWE-078/CommandInjection.expected +++ b/javascript/ql/test/query-tests/Security/CWE-078/CommandInjection.expected @@ -22,48 +22,12 @@ nodes | child_process-test.js:25:13:25:31 | "foo" + cmd + "bar" | | child_process-test.js:25:13:25:31 | "foo" + cmd + "bar" | | child_process-test.js:25:21:25:23 | cmd | -| child_process-test.js:36:7:36:20 | sh | -| child_process-test.js:36:12:36:20 | 'cmd.exe' | -| child_process-test.js:36:12:36:20 | 'cmd.exe' | -| child_process-test.js:38:7:38:20 | sh | -| child_process-test.js:38:12:38:20 | '/bin/sh' | -| child_process-test.js:38:12:38:20 | '/bin/sh' | -| child_process-test.js:39:14:39:15 | sh | -| child_process-test.js:39:14:39:15 | sh | -| child_process-test.js:39:18:39:30 | [ flag, cmd ] | -| child_process-test.js:39:18:39:30 | [ flag, cmd ] | -| child_process-test.js:39:18:39:30 | [ flag, cmd ] | | child_process-test.js:39:26:39:28 | cmd | | child_process-test.js:39:26:39:28 | cmd | -| child_process-test.js:41:9:41:17 | args | -| child_process-test.js:41:16:41:17 | [] | -| child_process-test.js:41:16:41:17 | [] | | child_process-test.js:43:15:43:17 | cmd | | child_process-test.js:43:15:43:17 | cmd | -| child_process-test.js:44:17:44:27 | "/bin/bash" | -| child_process-test.js:44:17:44:27 | "/bin/bash" | -| child_process-test.js:44:17:44:27 | "/bin/bash" | -| child_process-test.js:44:30:44:33 | args | -| child_process-test.js:44:30:44:33 | args | -| child_process-test.js:46:9:46:12 | "sh" | -| child_process-test.js:46:9:46:12 | "sh" | -| child_process-test.js:46:15:46:18 | args | -| child_process-test.js:48:9:48:17 | args | -| child_process-test.js:48:16:48:17 | [] | -| child_process-test.js:48:16:48:17 | [] | | child_process-test.js:50:15:50:17 | cmd | | child_process-test.js:50:15:50:17 | cmd | -| child_process-test.js:51:17:51:32 | `/bin` + "/bash" | -| child_process-test.js:51:17:51:32 | `/bin` + "/bash" | -| child_process-test.js:51:17:51:32 | `/bin` + "/bash" | -| child_process-test.js:51:35:51:38 | args | -| child_process-test.js:51:35:51:38 | args | -| child_process-test.js:55:14:55:16 | cmd | -| child_process-test.js:55:19:55:22 | args | -| child_process-test.js:56:12:56:14 | cmd | -| child_process-test.js:56:12:56:14 | cmd | -| child_process-test.js:56:17:56:20 | args | -| child_process-test.js:56:17:56:20 | args | | execSeries.js:3:20:3:22 | arr | | execSeries.js:6:14:6:16 | arr | | execSeries.js:6:14:6:21 | arr[i++] | @@ -114,9 +78,6 @@ nodes | third-party-command-injection.js:5:20:5:26 | command | | third-party-command-injection.js:6:21:6:27 | command | | third-party-command-injection.js:6:21:6:27 | command | -| tst_shell-command-injection-from-environment.js:4:25:4:61 | ['-rf', ... temp")] | -| tst_shell-command-injection-from-environment.js:4:25:4:61 | ['-rf', ... temp")] | -| tst_shell-command-injection-from-environment.js:4:25:4:61 | ['-rf', ... temp")] | edges | child_process-test.js:6:9:6:49 | cmd | child_process-test.js:17:13:17:15 | cmd | | child_process-test.js:6:9:6:49 | cmd | child_process-test.js:17:13:17:15 | cmd | @@ -146,33 +107,6 @@ edges | child_process-test.js:6:25:6:31 | req.url | child_process-test.js:6:15:6:38 | url.par ... , true) | | child_process-test.js:25:21:25:23 | cmd | child_process-test.js:25:13:25:31 | "foo" + cmd + "bar" | | child_process-test.js:25:21:25:23 | cmd | child_process-test.js:25:13:25:31 | "foo" + cmd + "bar" | -| child_process-test.js:36:7:36:20 | sh | child_process-test.js:39:14:39:15 | sh | -| child_process-test.js:36:7:36:20 | sh | child_process-test.js:39:14:39:15 | sh | -| child_process-test.js:36:12:36:20 | 'cmd.exe' | child_process-test.js:36:7:36:20 | sh | -| child_process-test.js:36:12:36:20 | 'cmd.exe' | child_process-test.js:36:7:36:20 | sh | -| child_process-test.js:38:7:38:20 | sh | child_process-test.js:39:14:39:15 | sh | -| child_process-test.js:38:7:38:20 | sh | child_process-test.js:39:14:39:15 | sh | -| child_process-test.js:38:12:38:20 | '/bin/sh' | child_process-test.js:38:7:38:20 | sh | -| child_process-test.js:38:12:38:20 | '/bin/sh' | child_process-test.js:38:7:38:20 | sh | -| child_process-test.js:39:18:39:30 | [ flag, cmd ] | child_process-test.js:39:18:39:30 | [ flag, cmd ] | -| child_process-test.js:41:9:41:17 | args | child_process-test.js:44:30:44:33 | args | -| child_process-test.js:41:9:41:17 | args | child_process-test.js:44:30:44:33 | args | -| child_process-test.js:41:9:41:17 | args | child_process-test.js:46:15:46:18 | args | -| child_process-test.js:41:16:41:17 | [] | child_process-test.js:41:9:41:17 | args | -| child_process-test.js:41:16:41:17 | [] | child_process-test.js:41:9:41:17 | args | -| child_process-test.js:44:17:44:27 | "/bin/bash" | child_process-test.js:44:17:44:27 | "/bin/bash" | -| child_process-test.js:46:9:46:12 | "sh" | child_process-test.js:55:14:55:16 | cmd | -| child_process-test.js:46:9:46:12 | "sh" | child_process-test.js:55:14:55:16 | cmd | -| child_process-test.js:46:15:46:18 | args | child_process-test.js:55:19:55:22 | args | -| child_process-test.js:48:9:48:17 | args | child_process-test.js:51:35:51:38 | args | -| child_process-test.js:48:9:48:17 | args | child_process-test.js:51:35:51:38 | args | -| child_process-test.js:48:16:48:17 | [] | child_process-test.js:48:9:48:17 | args | -| child_process-test.js:48:16:48:17 | [] | child_process-test.js:48:9:48:17 | args | -| child_process-test.js:51:17:51:32 | `/bin` + "/bash" | child_process-test.js:51:17:51:32 | `/bin` + "/bash" | -| child_process-test.js:55:14:55:16 | cmd | child_process-test.js:56:12:56:14 | cmd | -| child_process-test.js:55:14:55:16 | cmd | child_process-test.js:56:12:56:14 | cmd | -| child_process-test.js:55:19:55:22 | args | child_process-test.js:56:17:56:20 | args | -| child_process-test.js:55:19:55:22 | args | child_process-test.js:56:17:56:20 | args | | execSeries.js:3:20:3:22 | arr | execSeries.js:6:14:6:16 | arr | | execSeries.js:6:14:6:16 | arr | execSeries.js:6:14:6:21 | arr[i++] | | execSeries.js:6:14:6:21 | arr[i++] | execSeries.js:14:24:14:30 | command | @@ -222,7 +156,6 @@ edges | third-party-command-injection.js:5:20:5:26 | command | third-party-command-injection.js:6:21:6:27 | command | | third-party-command-injection.js:5:20:5:26 | command | third-party-command-injection.js:6:21:6:27 | command | | third-party-command-injection.js:5:20:5:26 | command | third-party-command-injection.js:6:21:6:27 | command | -| tst_shell-command-injection-from-environment.js:4:25:4:61 | ['-rf', ... temp")] | tst_shell-command-injection-from-environment.js:4:25:4:61 | ['-rf', ... temp")] | #select | child_process-test.js:17:13:17:15 | cmd | child_process-test.js:6:25:6:31 | req.url | child_process-test.js:17:13:17:15 | cmd | This command depends on $@. | child_process-test.js:6:25:6:31 | req.url | a user-provided value | | child_process-test.js:18:17:18:19 | cmd | child_process-test.js:6:25:6:31 | req.url | child_process-test.js:18:17:18:19 | cmd | This command depends on $@. | child_process-test.js:6:25:6:31 | req.url | a user-provided value | diff --git a/javascript/ql/test/query-tests/Security/CWE-078/IndirectCommandInjection.expected b/javascript/ql/test/query-tests/Security/CWE-078/IndirectCommandInjection.expected index a16ba531410..9377069360e 100644 --- a/javascript/ql/test/query-tests/Security/CWE-078/IndirectCommandInjection.expected +++ b/javascript/ql/test/query-tests/Security/CWE-078/IndirectCommandInjection.expected @@ -1,40 +1,4 @@ nodes -| child_process-test.js:36:7:36:20 | sh | -| child_process-test.js:36:12:36:20 | 'cmd.exe' | -| child_process-test.js:36:12:36:20 | 'cmd.exe' | -| child_process-test.js:38:7:38:20 | sh | -| child_process-test.js:38:12:38:20 | '/bin/sh' | -| child_process-test.js:38:12:38:20 | '/bin/sh' | -| child_process-test.js:39:14:39:15 | sh | -| child_process-test.js:39:14:39:15 | sh | -| child_process-test.js:39:18:39:30 | [ flag, cmd ] | -| child_process-test.js:39:18:39:30 | [ flag, cmd ] | -| child_process-test.js:39:18:39:30 | [ flag, cmd ] | -| child_process-test.js:41:9:41:17 | args | -| child_process-test.js:41:16:41:17 | [] | -| child_process-test.js:41:16:41:17 | [] | -| child_process-test.js:44:17:44:27 | "/bin/bash" | -| child_process-test.js:44:17:44:27 | "/bin/bash" | -| child_process-test.js:44:17:44:27 | "/bin/bash" | -| child_process-test.js:44:30:44:33 | args | -| child_process-test.js:44:30:44:33 | args | -| child_process-test.js:46:9:46:12 | "sh" | -| child_process-test.js:46:9:46:12 | "sh" | -| child_process-test.js:46:15:46:18 | args | -| child_process-test.js:48:9:48:17 | args | -| child_process-test.js:48:16:48:17 | [] | -| child_process-test.js:48:16:48:17 | [] | -| child_process-test.js:51:17:51:32 | `/bin` + "/bash" | -| child_process-test.js:51:17:51:32 | `/bin` + "/bash" | -| child_process-test.js:51:17:51:32 | `/bin` + "/bash" | -| child_process-test.js:51:35:51:38 | args | -| child_process-test.js:51:35:51:38 | args | -| child_process-test.js:55:14:55:16 | cmd | -| child_process-test.js:55:19:55:22 | args | -| child_process-test.js:56:12:56:14 | cmd | -| child_process-test.js:56:12:56:14 | cmd | -| child_process-test.js:56:17:56:20 | args | -| child_process-test.js:56:17:56:20 | args | | command-line-parameter-command-injection.js:4:10:4:21 | process.argv | | command-line-parameter-command-injection.js:4:10:4:21 | process.argv | | command-line-parameter-command-injection.js:4:10:4:21 | process.argv | @@ -84,37 +48,7 @@ nodes | command-line-parameter-command-injection.js:27:14:27:57 | `node $ ... ption"` | | command-line-parameter-command-injection.js:27:32:27:35 | args | | command-line-parameter-command-injection.js:27:32:27:45 | args.join(' ') | -| tst_shell-command-injection-from-environment.js:4:25:4:61 | ['-rf', ... temp")] | -| tst_shell-command-injection-from-environment.js:4:25:4:61 | ['-rf', ... temp")] | -| tst_shell-command-injection-from-environment.js:4:25:4:61 | ['-rf', ... temp")] | edges -| child_process-test.js:36:7:36:20 | sh | child_process-test.js:39:14:39:15 | sh | -| child_process-test.js:36:7:36:20 | sh | child_process-test.js:39:14:39:15 | sh | -| child_process-test.js:36:12:36:20 | 'cmd.exe' | child_process-test.js:36:7:36:20 | sh | -| child_process-test.js:36:12:36:20 | 'cmd.exe' | child_process-test.js:36:7:36:20 | sh | -| child_process-test.js:38:7:38:20 | sh | child_process-test.js:39:14:39:15 | sh | -| child_process-test.js:38:7:38:20 | sh | child_process-test.js:39:14:39:15 | sh | -| child_process-test.js:38:12:38:20 | '/bin/sh' | child_process-test.js:38:7:38:20 | sh | -| child_process-test.js:38:12:38:20 | '/bin/sh' | child_process-test.js:38:7:38:20 | sh | -| child_process-test.js:39:18:39:30 | [ flag, cmd ] | child_process-test.js:39:18:39:30 | [ flag, cmd ] | -| child_process-test.js:41:9:41:17 | args | child_process-test.js:44:30:44:33 | args | -| child_process-test.js:41:9:41:17 | args | child_process-test.js:44:30:44:33 | args | -| child_process-test.js:41:9:41:17 | args | child_process-test.js:46:15:46:18 | args | -| child_process-test.js:41:16:41:17 | [] | child_process-test.js:41:9:41:17 | args | -| child_process-test.js:41:16:41:17 | [] | child_process-test.js:41:9:41:17 | args | -| child_process-test.js:44:17:44:27 | "/bin/bash" | child_process-test.js:44:17:44:27 | "/bin/bash" | -| child_process-test.js:46:9:46:12 | "sh" | child_process-test.js:55:14:55:16 | cmd | -| child_process-test.js:46:9:46:12 | "sh" | child_process-test.js:55:14:55:16 | cmd | -| child_process-test.js:46:15:46:18 | args | child_process-test.js:55:19:55:22 | args | -| child_process-test.js:48:9:48:17 | args | child_process-test.js:51:35:51:38 | args | -| child_process-test.js:48:9:48:17 | args | child_process-test.js:51:35:51:38 | args | -| child_process-test.js:48:16:48:17 | [] | child_process-test.js:48:9:48:17 | args | -| child_process-test.js:48:16:48:17 | [] | child_process-test.js:48:9:48:17 | args | -| child_process-test.js:51:17:51:32 | `/bin` + "/bash" | child_process-test.js:51:17:51:32 | `/bin` + "/bash" | -| child_process-test.js:55:14:55:16 | cmd | child_process-test.js:56:12:56:14 | cmd | -| child_process-test.js:55:14:55:16 | cmd | child_process-test.js:56:12:56:14 | cmd | -| child_process-test.js:55:19:55:22 | args | child_process-test.js:56:17:56:20 | args | -| child_process-test.js:55:19:55:22 | args | child_process-test.js:56:17:56:20 | args | | command-line-parameter-command-injection.js:4:10:4:21 | process.argv | command-line-parameter-command-injection.js:4:10:4:21 | process.argv | | command-line-parameter-command-injection.js:8:22:8:33 | process.argv | command-line-parameter-command-injection.js:8:22:8:36 | process.argv[2] | | command-line-parameter-command-injection.js:8:22:8:33 | process.argv | command-line-parameter-command-injection.js:8:22:8:36 | process.argv[2] | @@ -159,7 +93,6 @@ edges | command-line-parameter-command-injection.js:27:32:27:35 | args | command-line-parameter-command-injection.js:27:32:27:45 | args.join(' ') | | command-line-parameter-command-injection.js:27:32:27:45 | args.join(' ') | command-line-parameter-command-injection.js:27:14:27:57 | `node $ ... ption"` | | command-line-parameter-command-injection.js:27:32:27:45 | args.join(' ') | command-line-parameter-command-injection.js:27:14:27:57 | `node $ ... ption"` | -| tst_shell-command-injection-from-environment.js:4:25:4:61 | ['-rf', ... temp")] | tst_shell-command-injection-from-environment.js:4:25:4:61 | ['-rf', ... temp")] | #select | command-line-parameter-command-injection.js:4:10:4:21 | process.argv | command-line-parameter-command-injection.js:4:10:4:21 | process.argv | command-line-parameter-command-injection.js:4:10:4:21 | process.argv | This command depends on an unsanitized $@. | command-line-parameter-command-injection.js:4:10:4:21 | process.argv | command-line argument | | command-line-parameter-command-injection.js:8:10:8:36 | "cmd.sh ... argv[2] | command-line-parameter-command-injection.js:8:22:8:33 | process.argv | command-line-parameter-command-injection.js:8:10:8:36 | "cmd.sh ... argv[2] | This command depends on an unsanitized $@. | command-line-parameter-command-injection.js:8:22:8:33 | process.argv | command-line argument | diff --git a/javascript/ql/test/query-tests/Security/CWE-078/ShellCommandInjectionFromEnvironment.expected b/javascript/ql/test/query-tests/Security/CWE-078/ShellCommandInjectionFromEnvironment.expected index bc0e994749e..ec3e7c99ecb 100644 --- a/javascript/ql/test/query-tests/Security/CWE-078/ShellCommandInjectionFromEnvironment.expected +++ b/javascript/ql/test/query-tests/Security/CWE-078/ShellCommandInjectionFromEnvironment.expected @@ -1,77 +1,10 @@ nodes -| child_process-test.js:36:7:36:20 | sh | -| child_process-test.js:36:12:36:20 | 'cmd.exe' | -| child_process-test.js:36:12:36:20 | 'cmd.exe' | -| child_process-test.js:38:7:38:20 | sh | -| child_process-test.js:38:12:38:20 | '/bin/sh' | -| child_process-test.js:38:12:38:20 | '/bin/sh' | -| child_process-test.js:39:14:39:15 | sh | -| child_process-test.js:39:14:39:15 | sh | -| child_process-test.js:39:18:39:30 | [ flag, cmd ] | -| child_process-test.js:39:18:39:30 | [ flag, cmd ] | -| child_process-test.js:39:18:39:30 | [ flag, cmd ] | -| child_process-test.js:41:9:41:17 | args | -| child_process-test.js:41:16:41:17 | [] | -| child_process-test.js:41:16:41:17 | [] | -| child_process-test.js:44:17:44:27 | "/bin/bash" | -| child_process-test.js:44:17:44:27 | "/bin/bash" | -| child_process-test.js:44:17:44:27 | "/bin/bash" | -| child_process-test.js:44:30:44:33 | args | -| child_process-test.js:44:30:44:33 | args | -| child_process-test.js:46:9:46:12 | "sh" | -| child_process-test.js:46:9:46:12 | "sh" | -| child_process-test.js:46:15:46:18 | args | -| child_process-test.js:48:9:48:17 | args | -| child_process-test.js:48:16:48:17 | [] | -| child_process-test.js:48:16:48:17 | [] | -| child_process-test.js:51:17:51:32 | `/bin` + "/bash" | -| child_process-test.js:51:17:51:32 | `/bin` + "/bash" | -| child_process-test.js:51:17:51:32 | `/bin` + "/bash" | -| child_process-test.js:51:35:51:38 | args | -| child_process-test.js:51:35:51:38 | args | -| child_process-test.js:55:14:55:16 | cmd | -| child_process-test.js:55:19:55:22 | args | -| child_process-test.js:56:12:56:14 | cmd | -| child_process-test.js:56:12:56:14 | cmd | -| child_process-test.js:56:17:56:20 | args | -| child_process-test.js:56:17:56:20 | args | -| tst_shell-command-injection-from-environment.js:4:25:4:61 | ['-rf', ... temp")] | -| tst_shell-command-injection-from-environment.js:4:25:4:61 | ['-rf', ... temp")] | -| tst_shell-command-injection-from-environment.js:4:25:4:61 | ['-rf', ... temp")] | | tst_shell-command-injection-from-environment.js:5:14:5:53 | 'rm -rf ... "temp") | | tst_shell-command-injection-from-environment.js:5:14:5:53 | 'rm -rf ... "temp") | | tst_shell-command-injection-from-environment.js:5:26:5:53 | path.jo ... "temp") | | tst_shell-command-injection-from-environment.js:5:36:5:44 | __dirname | | tst_shell-command-injection-from-environment.js:5:36:5:44 | __dirname | edges -| child_process-test.js:36:7:36:20 | sh | child_process-test.js:39:14:39:15 | sh | -| child_process-test.js:36:7:36:20 | sh | child_process-test.js:39:14:39:15 | sh | -| child_process-test.js:36:12:36:20 | 'cmd.exe' | child_process-test.js:36:7:36:20 | sh | -| child_process-test.js:36:12:36:20 | 'cmd.exe' | child_process-test.js:36:7:36:20 | sh | -| child_process-test.js:38:7:38:20 | sh | child_process-test.js:39:14:39:15 | sh | -| child_process-test.js:38:7:38:20 | sh | child_process-test.js:39:14:39:15 | sh | -| child_process-test.js:38:12:38:20 | '/bin/sh' | child_process-test.js:38:7:38:20 | sh | -| child_process-test.js:38:12:38:20 | '/bin/sh' | child_process-test.js:38:7:38:20 | sh | -| child_process-test.js:39:18:39:30 | [ flag, cmd ] | child_process-test.js:39:18:39:30 | [ flag, cmd ] | -| child_process-test.js:41:9:41:17 | args | child_process-test.js:44:30:44:33 | args | -| child_process-test.js:41:9:41:17 | args | child_process-test.js:44:30:44:33 | args | -| child_process-test.js:41:9:41:17 | args | child_process-test.js:46:15:46:18 | args | -| child_process-test.js:41:16:41:17 | [] | child_process-test.js:41:9:41:17 | args | -| child_process-test.js:41:16:41:17 | [] | child_process-test.js:41:9:41:17 | args | -| child_process-test.js:44:17:44:27 | "/bin/bash" | child_process-test.js:44:17:44:27 | "/bin/bash" | -| child_process-test.js:46:9:46:12 | "sh" | child_process-test.js:55:14:55:16 | cmd | -| child_process-test.js:46:9:46:12 | "sh" | child_process-test.js:55:14:55:16 | cmd | -| child_process-test.js:46:15:46:18 | args | child_process-test.js:55:19:55:22 | args | -| child_process-test.js:48:9:48:17 | args | child_process-test.js:51:35:51:38 | args | -| child_process-test.js:48:9:48:17 | args | child_process-test.js:51:35:51:38 | args | -| child_process-test.js:48:16:48:17 | [] | child_process-test.js:48:9:48:17 | args | -| child_process-test.js:48:16:48:17 | [] | child_process-test.js:48:9:48:17 | args | -| child_process-test.js:51:17:51:32 | `/bin` + "/bash" | child_process-test.js:51:17:51:32 | `/bin` + "/bash" | -| child_process-test.js:55:14:55:16 | cmd | child_process-test.js:56:12:56:14 | cmd | -| child_process-test.js:55:14:55:16 | cmd | child_process-test.js:56:12:56:14 | cmd | -| child_process-test.js:55:19:55:22 | args | child_process-test.js:56:17:56:20 | args | -| child_process-test.js:55:19:55:22 | args | child_process-test.js:56:17:56:20 | args | -| tst_shell-command-injection-from-environment.js:4:25:4:61 | ['-rf', ... temp")] | tst_shell-command-injection-from-environment.js:4:25:4:61 | ['-rf', ... temp")] | | tst_shell-command-injection-from-environment.js:5:26:5:53 | path.jo ... "temp") | tst_shell-command-injection-from-environment.js:5:14:5:53 | 'rm -rf ... "temp") | | tst_shell-command-injection-from-environment.js:5:26:5:53 | path.jo ... "temp") | tst_shell-command-injection-from-environment.js:5:14:5:53 | 'rm -rf ... "temp") | | tst_shell-command-injection-from-environment.js:5:36:5:44 | __dirname | tst_shell-command-injection-from-environment.js:5:26:5:53 | path.jo ... "temp") | From 03c9a40ba35db5be5e635703db8b2ab82beca909 Mon Sep 17 00:00:00 2001 From: Max Schaefer Date: Tue, 22 Oct 2019 09:32:51 +0100 Subject: [PATCH 078/148] JavaScript: Add libraries for forward and backward data-flow exploration. --- .../dataflow/BackwardExploration.qll | 44 +++++++++++++++++++ .../dataflow/ForwardExploration.qll | 42 ++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 javascript/ql/src/semmle/javascript/dataflow/BackwardExploration.qll create mode 100644 javascript/ql/src/semmle/javascript/dataflow/ForwardExploration.qll diff --git a/javascript/ql/src/semmle/javascript/dataflow/BackwardExploration.qll b/javascript/ql/src/semmle/javascript/dataflow/BackwardExploration.qll new file mode 100644 index 00000000000..caf67e6db7f --- /dev/null +++ b/javascript/ql/src/semmle/javascript/dataflow/BackwardExploration.qll @@ -0,0 +1,44 @@ +/** + * Provides machinery for performing backward data-flow exploration. + * + * Importing this module effectively makes all data-flow and taint-tracking configurations + * ignore their `isSource` predicate. Instead, flow is tracked from any _initial node_ (that is, + * a node without incoming flow) to a sink node. All initial nodes are then treated as source + * nodes. + * + * Data-flow exploration cannot be used with configurations depending on other configurations. + * + * NOTE: This library should only be used for debugging, not in production code. Backward + * exploration in particular does not scale on non-trivial code bases and hence is of limited + * usefulness as it stands. + */ + +import javascript + +private class BackwardExploringConfiguration extends DataFlow::Configuration { + DataFlow::Configuration cfg; + + BackwardExploringConfiguration() { + this = cfg + } + + override predicate isSource(DataFlow::Node node) { any() } + + override predicate isSource(DataFlow::Node node, DataFlow::FlowLabel lbl) { any() } + + override predicate hasFlow(DataFlow::Node source, DataFlow::Node sink) { + exists(DataFlow::PathNode src, DataFlow::PathNode snk | hasFlowPath(src, snk) | + source = src.getNode() and + sink = snk.getNode() + ) + } + + override predicate hasFlowPath(DataFlow::SourcePathNode source, DataFlow::SinkPathNode sink) { + exists(DataFlow::MidPathNode first | + source.getConfiguration() = this and + source.getASuccessor() = first and + not exists(DataFlow::MidPathNode mid | mid.getASuccessor() = first) and + first.getASuccessor*() = sink + ) + } +} diff --git a/javascript/ql/src/semmle/javascript/dataflow/ForwardExploration.qll b/javascript/ql/src/semmle/javascript/dataflow/ForwardExploration.qll new file mode 100644 index 00000000000..0e3a7ae91c2 --- /dev/null +++ b/javascript/ql/src/semmle/javascript/dataflow/ForwardExploration.qll @@ -0,0 +1,42 @@ +/** + * Provides machinery for performing forward data-flow exploration. + * + * Importing this module effectively makes all data-flow and taint-tracking configurations + * ignore their `isSink` predicate. Instead, flow is tracked from source nodes as far as + * possible, until a _terminal node_ (that is, a node without any outgoing flow) is reached. + * All terminal nodes are then treated as sink nodes. + * + * Data-flow exploration cannot be used with configurations depending on other configurations. + * + * NOTE: This library should only be used for debugging, not in production code. + */ + +import javascript + +private class ForwardExploringConfiguration extends DataFlow::Configuration { + DataFlow::Configuration cfg; + + ForwardExploringConfiguration() { + this = cfg + } + + override predicate isSink(DataFlow::Node node) { any() } + + override predicate isSink(DataFlow::Node node, DataFlow::FlowLabel lbl) { any() } + + override predicate hasFlow(DataFlow::Node source, DataFlow::Node sink) { + exists(DataFlow::PathNode src, DataFlow::PathNode snk | hasFlowPath(src, snk) | + source = src.getNode() and + sink = snk.getNode() + ) + } + + override predicate hasFlowPath(DataFlow::SourcePathNode source, DataFlow::SinkPathNode sink) { + exists(DataFlow::MidPathNode last | + source.getConfiguration() = this and + source.getASuccessor*() = last and + not last.getASuccessor() instanceof DataFlow::MidPathNode and + last.getASuccessor() = sink + ) + } +} From 413f49bba546436a9a9f49df1fd004368f87efe5 Mon Sep 17 00:00:00 2001 From: Rachel Mant Date: Thu, 31 Oct 2019 22:50:44 +0000 Subject: [PATCH 079/148] Query cpp/unused-static-variable was producing incorrect results for constexpr variables --- .../src/Best Practices/Unused Entities/UnusedStaticVariables.ql | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/ql/src/Best Practices/Unused Entities/UnusedStaticVariables.ql b/cpp/ql/src/Best Practices/Unused Entities/UnusedStaticVariables.ql index 26cf42521e5..3ad43998d18 100644 --- a/cpp/ql/src/Best Practices/Unused Entities/UnusedStaticVariables.ql +++ b/cpp/ql/src/Best Practices/Unused Entities/UnusedStaticVariables.ql @@ -21,6 +21,7 @@ from Variable v where v.isStatic() and v.hasDefinition() and + not v.isConstexpr() and not exists(VariableAccess a | a.getTarget() = v) and not v instanceof MemberVariable and not declarationHasSideEffects(v) and From d94e91b39ba6149e6d4232140375bdb0b340d0cb Mon Sep 17 00:00:00 2001 From: shati-patel <42641846+shati-patel@users.noreply.github.com> Date: Fri, 1 Nov 2019 11:03:12 +0000 Subject: [PATCH 080/148] Apply suggestions from code review Co-Authored-By: Felicity Chapman --- docs/language/learn-ql/python/introduce-libraries-python.rst | 2 +- docs/language/learn-ql/writing-queries/path-queries.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/language/learn-ql/python/introduce-libraries-python.rst b/docs/language/learn-ql/python/introduce-libraries-python.rst index 60d965a461b..1594afc9679 100644 --- a/docs/language/learn-ql/python/introduce-libraries-python.rst +++ b/docs/language/learn-ql/python/introduce-libraries-python.rst @@ -12,7 +12,7 @@ The rest of this tutorial summarizes the contents of the standard libraries for Overview of the library ----------------------- -The CodeQL library for Python incorporates a large number of classes. Each class corresponds either to one kind of entity in Python source code or to an entity that can be derived form the source code using static analysis. These classes can be divided into four categories: +The CodeQL library for Python incorporates a large number of classes. Each class corresponds either to one kind of entity in Python source code or to an entity that can be derived from the source code using static analysis. These classes can be divided into four categories: - **Syntactic** - classes that represent entities in the Python source code. - **Control flow** - classes that represent entities from the control flow graphs. diff --git a/docs/language/learn-ql/writing-queries/path-queries.rst b/docs/language/learn-ql/writing-queries/path-queries.rst index 73708b17eed..01a6e0d97ea 100644 --- a/docs/language/learn-ql/writing-queries/path-queries.rst +++ b/docs/language/learn-ql/writing-queries/path-queries.rst @@ -105,7 +105,7 @@ To do this you need to define a `query predicate Date: Fri, 1 Nov 2019 11:21:12 +0000 Subject: [PATCH 081/148] Docs: Rename Sphinx project to "Learning CodeQL" --- docs/language/global-sphinx-files/_templates/layout.html | 2 +- docs/language/learn-ql/conf.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/language/global-sphinx-files/_templates/layout.html b/docs/language/global-sphinx-files/_templates/layout.html index 6aa09b5fb38..de3b189e1ee 100644 --- a/docs/language/global-sphinx-files/_templates/layout.html +++ b/docs/language/global-sphinx-files/_templates/layout.html @@ -61,7 +61,7 @@
    - Learn QL + Learn CodeQL QL for variant analysis QL tools Queries diff --git a/docs/language/learn-ql/conf.py b/docs/language/learn-ql/conf.py index d9b1de5e9fc..94d3bf2205f 100644 --- a/docs/language/learn-ql/conf.py +++ b/docs/language/learn-ql/conf.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Learn QL documentation build configuration file, created by +# Learn CodeQL documentation build configuration file, created by # on Tuesday Nov 13 2018. # # This file is execfile()d with the current directory set to its @@ -41,16 +41,16 @@ highlight_language = 'ql' master_doc = 'index' # General information about the project. -project = u'Learning QL' +project = u'Learning CodeQL' # -- Project-specifc options for HTML output ---------------------------------------------- # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -html_title = 'Learn QL' +html_title = 'Learn CodeQL' # Output file base name for HTML help builder. -htmlhelp_basename = 'Learn QL' +htmlhelp_basename = 'Learn CodeQL' # The version info for this project, if different from version and release in main conf.py file. # The short X.Y version. From 9f37237b4a3fbdd340402f23825e4fab29a3099d Mon Sep 17 00:00:00 2001 From: yh-semmle Date: Sat, 20 Jul 2019 23:25:34 -0400 Subject: [PATCH 082/148] Java 13: add stmt kind `@yieldstmt` to dbscheme --- java/ql/src/config/semmlecode.dbscheme | 1 + 1 file changed, 1 insertion(+) diff --git a/java/ql/src/config/semmlecode.dbscheme b/java/ql/src/config/semmlecode.dbscheme index 886aec63fb9..e7706df98aa 100755 --- a/java/ql/src/config/semmlecode.dbscheme +++ b/java/ql/src/config/semmlecode.dbscheme @@ -494,6 +494,7 @@ case @stmt.kind of | 20 = @superconstructorinvocationstmt | 21 = @case | 22 = @catchclause +| 23 = @yieldstmt ; #keyset[parent,idx] From 8fb4dbe09280cd0aba67b1d4b313f3409ab6b318 Mon Sep 17 00:00:00 2001 From: yh-semmle Date: Sat, 7 Sep 2019 11:51:48 -0400 Subject: [PATCH 083/148] Java 13: account for changes to switch expressions --- java/ql/src/semmle/code/java/Completion.qll | 6 +- .../src/semmle/code/java/ControlFlowGraph.qll | 26 ++++---- java/ql/src/semmle/code/java/Expr.qll | 4 +- java/ql/src/semmle/code/java/Statement.qll | 59 +++++++++---------- 4 files changed, 46 insertions(+), 49 deletions(-) diff --git a/java/ql/src/semmle/code/java/Completion.qll b/java/ql/src/semmle/code/java/Completion.qll index a6195bcf600..c8b4f966a3b 100644 --- a/java/ql/src/semmle/code/java/Completion.qll +++ b/java/ql/src/semmle/code/java/Completion.qll @@ -52,13 +52,13 @@ newtype Completion = (innerValue = true or innerValue = false) } or /** - * The expression or statement completes via a `break` statement without a value. + * The expression or statement completes via a `break` statement. */ BreakCompletion(MaybeLabel l) or /** - * The expression or statement completes via a value `break` statement. + * The expression or statement completes via a `yield` statement. */ - ValueBreakCompletion(NormalOrBooleanCompletion c) or + YieldCompletion(NormalOrBooleanCompletion c) or /** * The expression or statement completes via a `continue` statement. */ diff --git a/java/ql/src/semmle/code/java/ControlFlowGraph.qll b/java/ql/src/semmle/code/java/ControlFlowGraph.qll index 45f96a36c9d..9c7cc976d46 100644 --- a/java/ql/src/semmle/code/java/ControlFlowGraph.qll +++ b/java/ql/src/semmle/code/java/ControlFlowGraph.qll @@ -804,23 +804,23 @@ private module ControlFlowGraphImpl { or // handle `switch` expression exists(SwitchExpr switch | switch = n | - // value `break` terminates the `switch` - last(switch.getAStmt(), last, ValueBreakCompletion(completion)) + // `yield` terminates the `switch` + last(switch.getAStmt(), last, YieldCompletion(completion)) or // any other abnormal completion is propagated last(switch.getAStmt(), last, completion) and - not completion instanceof ValueBreakCompletion and + not completion instanceof YieldCompletion and completion != NormalCompletion() ) or // the last node in a case rule is the last node in the right-hand side last(n.(SwitchCase).getRuleStatement(), last, completion) or - // ...and if the rhs is an expression we wrap the completion as a value break + // ...and if the rhs is an expression we wrap the completion as a yield exists(Completion caseCompletion | last(n.(SwitchCase).getRuleExpression(), last, caseCompletion) and if caseCompletion instanceof NormalOrBooleanCompletion - then completion = ValueBreakCompletion(caseCompletion) + then completion = YieldCompletion(caseCompletion) else completion = caseCompletion ) or @@ -833,18 +833,18 @@ private module ControlFlowGraphImpl { // `throw` statements or throwing calls give rise to ` Throw` completion exists(ThrowableType tt | mayThrow(n, tt) | last = n and completion = ThrowCompletion(tt)) or - // `break` statements without value give rise to a `Break` completion - exists(BreakStmt break | break = n and last = n and not break.hasValue() | + // `break` statements give rise to a `Break` completion + exists(BreakStmt break | break = n and last = n | completion = labelledBreakCompletion(MkLabel(break.getLabel())) or not exists(break.getLabel()) and completion = anonymousBreakCompletion() ) or - // value break statements get their completion wrapped as a value break + // yield statements get their completion wrapped as a yield exists(Completion caseCompletion | - last(n.(BreakStmt).getValue(), last, caseCompletion) and + last(n.(YieldStmt).getValue(), last, caseCompletion) and if caseCompletion instanceof NormalOrBooleanCompletion - then completion = ValueBreakCompletion(caseCompletion) + then completion = YieldCompletion(caseCompletion) else completion = caseCompletion ) or @@ -1112,9 +1112,9 @@ private module ControlFlowGraphImpl { n = case and result = first(case.getRuleStatement()) ) or - // Value break - exists(BreakStmt break | completion = NormalCompletion() | - n = break and result = first(break.getValue()) + // Yield + exists(YieldStmt yield | completion = NormalCompletion() | + n = yield and result = first(yield.getValue()) ) or // Synchronized statements execute their expression _before_ synchronization, so the CFG reflects that. diff --git a/java/ql/src/semmle/code/java/Expr.qll b/java/ql/src/semmle/code/java/Expr.qll index 0ab25628e47..313a8eb6aa8 100755 --- a/java/ql/src/semmle/code/java/Expr.qll +++ b/java/ql/src/semmle/code/java/Expr.qll @@ -1084,7 +1084,7 @@ class ConditionalExpr extends Expr, @conditionalexpr { } /** - * PREVIEW FEATURE in Java 12. Subject to removal in a future release. + * PREVIEW FEATURE in Java 13. Subject to removal in a future release. * * A `switch` expression. */ @@ -1117,7 +1117,7 @@ class SwitchExpr extends Expr, @switchexpr { Expr getAResult() { result = getACase().getRuleExpression() or - exists(BreakStmt break | break.(JumpStmt).getTarget() = this and result = break.getValue()) + exists(YieldStmt yield | yield.(JumpStmt).getTarget() = this and result = yield.getValue()) } /** Gets a printable representation of this expression. */ diff --git a/java/ql/src/semmle/code/java/Statement.qll b/java/ql/src/semmle/code/java/Statement.qll index 2f98615b243..6bcaa73a5da 100755 --- a/java/ql/src/semmle/code/java/Statement.qll +++ b/java/ql/src/semmle/code/java/Statement.qll @@ -417,7 +417,7 @@ class SwitchCase extends Stmt, @case { SwitchStmt getSwitch() { result.getACase() = this } /** - * PREVIEW FEATURE in Java 12. Subject to removal in a future release. + * PREVIEW FEATURE in Java 13. Subject to removal in a future release. * * Gets the switch expression to which this case belongs, if any. */ @@ -432,7 +432,7 @@ class SwitchCase extends Stmt, @case { } /** - * PREVIEW FEATURE in Java 12. Subject to removal in a future release. + * PREVIEW FEATURE in Java 13. Subject to removal in a future release. * * Holds if this `case` is a switch labeled rule of the form `... -> ...`. */ @@ -443,14 +443,14 @@ class SwitchCase extends Stmt, @case { } /** - * PREVIEW FEATURE in Java 12. Subject to removal in a future release. + * PREVIEW FEATURE in Java 13. Subject to removal in a future release. * * Gets the expression on the right-hand side of the arrow, if any. */ Expr getRuleExpression() { result.getParent() = this and result.getIndex() = -1 } /** - * PREVIEW FEATURE in Java 12. Subject to removal in a future release. + * PREVIEW FEATURE in Java 13. Subject to removal in a future release. * * Gets the statement on the right-hand side of the arrow, if any. */ @@ -465,7 +465,7 @@ class ConstCase extends SwitchCase { Expr getValue() { result.getParent() = this and result.getIndex() = 0 } /** - * PREVIEW FEATURE in Java 12. Subject to removal in a future release. + * PREVIEW FEATURE in Java 13. Subject to removal in a future release. * * Gets the `case` constant at the specified index. */ @@ -558,10 +558,11 @@ class ThrowStmt extends Stmt, @throwstmt { } } -/** A `break` or `continue` statement. */ +/** A `break`, `yield` or `continue` statement. */ class JumpStmt extends Stmt { JumpStmt() { this instanceof BreakStmt or + this instanceof YieldStmt or this instanceof ContinueStmt } @@ -581,13 +582,12 @@ class JumpStmt extends Stmt { ( result instanceof LoopStmt or - this instanceof BreakStmt and result instanceof SwitchStmt + (this instanceof BreakStmt or this instanceof YieldStmt) and + result instanceof SwitchStmt ) } - private SwitchExpr getSwitchExprTarget() { - this.(BreakStmt).hasValue() and result = this.getParent+() - } + private SwitchExpr getSwitchExprTarget() { result = this.(YieldStmt).getParent+() } private StmtParent getEnclosingTarget() { result = getSwitchExprTarget() @@ -598,7 +598,7 @@ class JumpStmt extends Stmt { } /** - * Gets the statement or `switch` expression that this `break` or `continue` jumps to. + * Gets the statement or `switch` expression that this `break`, `yield` or `continue` jumps to. */ StmtParent getTarget() { result = getLabelTarget() @@ -615,34 +615,31 @@ class BreakStmt extends Stmt, @breakstmt { /** Holds if this `break` statement has an explicit label. */ predicate hasLabel() { exists(string s | s = this.getLabel()) } - /** - * PREVIEW FEATURE in Java 12. Subject to removal in a future release. - * - * Gets the value of this `break` statement, if any. - */ - Expr getValue() { result.getParent() = this } - - /** - * PREVIEW FEATURE in Java 12. Subject to removal in a future release. - * - * Holds if this `break` statement has a value. - */ - predicate hasValue() { exists(Expr e | e.getParent() = this) } - /** Gets a printable representation of this statement. May include more detail than `toString()`. */ override string pp() { - if this.hasLabel() - then result = "break " + this.getLabel() - else - if this.hasValue() - then result = "break ..." - else result = "break" + if this.hasLabel() then result = "break " + this.getLabel() else result = "break" } /** This statement's Halstead ID (used to compute Halstead metrics). */ override string getHalsteadID() { result = "BreakStmt" } } +/** + * PREVIEW FEATURE in Java 13. Subject to removal in a future release. + * + * A `yield` statement. + */ +class YieldStmt extends Stmt, @yieldstmt { + /** + * Gets the value of this `yield` statement. + */ + Expr getValue() { result.getParent() = this } + + override string pp() { result = "yield ..." } + + override string getHalsteadID() { result = "YieldStmt" } +} + /** A `continue` statement. */ class ContinueStmt extends Stmt, @continuestmt { /** Gets the label targeted by this `continue` statement, if any. */ From de0869c216139626919c4287699d8d36b881f07c Mon Sep 17 00:00:00 2001 From: yh-semmle Date: Fri, 11 Oct 2019 09:52:25 -0400 Subject: [PATCH 084/148] Java 13: remove superfluous disjunct in `JumpStmt.getAPotentialTarget()` --- java/ql/src/semmle/code/java/Statement.qll | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/java/ql/src/semmle/code/java/Statement.qll b/java/ql/src/semmle/code/java/Statement.qll index 6bcaa73a5da..2a5f94ef781 100755 --- a/java/ql/src/semmle/code/java/Statement.qll +++ b/java/ql/src/semmle/code/java/Statement.qll @@ -582,8 +582,7 @@ class JumpStmt extends Stmt { ( result instanceof LoopStmt or - (this instanceof BreakStmt or this instanceof YieldStmt) and - result instanceof SwitchStmt + this instanceof BreakStmt and result instanceof SwitchStmt ) } From e8a65101bc41624cbddc4e5d8036c2debb0a430b Mon Sep 17 00:00:00 2001 From: yh-semmle Date: Sun, 27 Oct 2019 15:57:51 -0400 Subject: [PATCH 085/148] Java 13: add db stats for `@yieldstmt` --- java/ql/src/config/semmlecode.dbscheme.stats | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/java/ql/src/config/semmlecode.dbscheme.stats b/java/ql/src/config/semmlecode.dbscheme.stats index 621bfff8b15..22e7ff7e830 100644 --- a/java/ql/src/config/semmlecode.dbscheme.stats +++ b/java/ql/src/config/semmlecode.dbscheme.stats @@ -189,6 +189,10 @@ 28672 +@yieldstmt +100 + + @arrayaccess 53561 From e232f538e97ed4dca2ed204ed3415ba93d094f60 Mon Sep 17 00:00:00 2001 From: yh-semmle Date: Sun, 27 Oct 2019 19:55:09 -0400 Subject: [PATCH 086/148] Java 13: update test options --- java/ql/test/library-tests/guards12/options | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/ql/test/library-tests/guards12/options b/java/ql/test/library-tests/guards12/options index 33d712034f9..99827347b32 100644 --- a/java/ql/test/library-tests/guards12/options +++ b/java/ql/test/library-tests/guards12/options @@ -1 +1 @@ -//semmle-extractor-options: --javac-args --enable-preview -source 12 -target 12 +//semmle-extractor-options: --javac-args --enable-preview -source 13 -target 13 From 637394fb62ed028486731297ba7c2ad1e402fd8c Mon Sep 17 00:00:00 2001 From: Asger F Date: Wed, 30 Oct 2019 12:04:11 +0000 Subject: [PATCH 087/148] TS: Add TypeScript-3.7.1-rc --- .../extractor/lib/typescript/package.json | 2 +- javascript/extractor/lib/typescript/yarn.lock | 89 ++++++++++--------- 2 files changed, 46 insertions(+), 45 deletions(-) diff --git a/javascript/extractor/lib/typescript/package.json b/javascript/extractor/lib/typescript/package.json index fd440415096..966210ec8b2 100644 --- a/javascript/extractor/lib/typescript/package.json +++ b/javascript/extractor/lib/typescript/package.json @@ -2,7 +2,7 @@ "name": "typescript-parser-wrapper", "private": true, "dependencies": { - "typescript": "3.6.3" + "typescript": "^3.7.1-rc" }, "scripts": { "build": "tsc --project tsconfig.json", diff --git a/javascript/extractor/lib/typescript/yarn.lock b/javascript/extractor/lib/typescript/yarn.lock index 62205cc17f4..9b539464cec 100644 --- a/javascript/extractor/lib/typescript/yarn.lock +++ b/javascript/extractor/lib/typescript/yarn.lock @@ -4,31 +4,31 @@ "@types/node@12.7.11": version "12.7.11" - resolved "node-12.7.11.tgz#be879b52031cfb5d295b047f5462d8ef1a716446" + resolved node-12.7.11.tgz#be879b52031cfb5d295b047f5462d8ef1a716446 ansi-regex@^2.0.0: version "2.1.1" - resolved "ansi-regex-2.1.1.tgz#c3b33ab5ee360d86e0e628f0468ae7ef27d654df" + resolved ansi-regex-2.1.1.tgz#c3b33ab5ee360d86e0e628f0468ae7ef27d654df ansi-styles@^2.2.1: version "2.2.1" - resolved "ansi-styles-2.2.1.tgz#b432dd3358b634cf75e1e4664368240533c1ddbe" + resolved ansi-styles-2.2.1.tgz#b432dd3358b634cf75e1e4664368240533c1ddbe ansi-styles@^3.1.0: version "3.2.0" - resolved "ansi-styles-3.2.0.tgz#c159b8d5be0f9e5a6f346dab94f16ce022161b88" + resolved ansi-styles-3.2.0.tgz#c159b8d5be0f9e5a6f346dab94f16ce022161b88 dependencies: color-convert "^1.9.0" argparse@^1.0.7: version "1.0.9" - resolved "argparse-1.0.9.tgz#73d83bc263f86e97f8cc4f6bae1b0e90a7d22c86" + resolved argparse-1.0.9.tgz#73d83bc263f86e97f8cc4f6bae1b0e90a7d22c86 dependencies: sprintf-js "~1.0.2" babel-code-frame@^6.22.0: version "6.26.0" - resolved "babel-code-frame-6.26.0.tgz#63fd43f7dc1e3bb7ce35947db8fe369a3f58c74b" + resolved babel-code-frame-6.26.0.tgz#63fd43f7dc1e3bb7ce35947db8fe369a3f58c74b dependencies: chalk "^1.1.3" esutils "^2.0.2" @@ -36,22 +36,22 @@ babel-code-frame@^6.22.0: balanced-match@^1.0.0: version "1.0.0" - resolved "balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" + resolved balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767 brace-expansion@^1.1.7: version "1.1.8" - resolved "brace-expansion-1.1.8.tgz#c07b211c7c952ec1f8efd51a77ef0d1d3990a292" + resolved brace-expansion-1.1.8.tgz#c07b211c7c952ec1f8efd51a77ef0d1d3990a292 dependencies: balanced-match "^1.0.0" concat-map "0.0.1" builtin-modules@^1.1.1: version "1.1.1" - resolved "builtin-modules-1.1.1.tgz#270f076c5a72c02f5b65a47df94c5fe3a278892f" + resolved builtin-modules-1.1.1.tgz#270f076c5a72c02f5b65a47df94c5fe3a278892f chalk@^1.1.3: version "1.1.3" - resolved "chalk-1.1.3.tgz#a8115c55e4a702fe4d150abd3872822a7e09fc98" + resolved chalk-1.1.3.tgz#a8115c55e4a702fe4d150abd3872822a7e09fc98 dependencies: ansi-styles "^2.2.1" escape-string-regexp "^1.0.2" @@ -61,7 +61,7 @@ chalk@^1.1.3: chalk@^2.3.0: version "2.3.0" - resolved "chalk-2.3.0.tgz#b5ea48efc9c1793dccc9b4767c93914d3f2d52ba" + resolved chalk-2.3.0.tgz#b5ea48efc9c1793dccc9b4767c93914d3f2d52ba dependencies: ansi-styles "^3.1.0" escape-string-regexp "^1.0.5" @@ -69,45 +69,45 @@ chalk@^2.3.0: color-convert@^1.9.0: version "1.9.1" - resolved "color-convert-1.9.1.tgz#c1261107aeb2f294ebffec9ed9ecad529a6097ed" + resolved color-convert-1.9.1.tgz#c1261107aeb2f294ebffec9ed9ecad529a6097ed dependencies: color-name "^1.1.1" color-name@^1.1.1: version "1.1.3" - resolved "color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25" + resolved color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25 commander@^2.12.1: version "2.13.0" - resolved "commander-2.13.0.tgz#6964bca67685df7c1f1430c584f07d7597885b9c" + resolved commander-2.13.0.tgz#6964bca67685df7c1f1430c584f07d7597885b9c concat-map@0.0.1: version "0.0.1" - resolved "concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" + resolved concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b diff@^3.2.0: version "3.4.0" - resolved "diff-3.4.0.tgz#b1d85507daf3964828de54b37d0d73ba67dda56c" + resolved diff-3.4.0.tgz#b1d85507daf3964828de54b37d0d73ba67dda56c escape-string-regexp@^1.0.2, escape-string-regexp@^1.0.5: version "1.0.5" - resolved "escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4" + resolved escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4 esprima@^4.0.0: version "4.0.0" - resolved "esprima-4.0.0.tgz#4499eddcd1110e0b218bacf2fa7f7f59f55ca804" + resolved esprima-4.0.0.tgz#4499eddcd1110e0b218bacf2fa7f7f59f55ca804 esutils@^2.0.2: version "2.0.2" - resolved "esutils-2.0.2.tgz#0abf4f1caa5bcb1f7a9d8acc6dea4faaa04bac9b" + resolved esutils-2.0.2.tgz#0abf4f1caa5bcb1f7a9d8acc6dea4faaa04bac9b fs.realpath@^1.0.0: version "1.0.0" - resolved "fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" + resolved fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f glob@^7.1.1: version "7.1.2" - resolved "glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" + resolved glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15 dependencies: fs.realpath "^1.0.0" inflight "^1.0.4" @@ -118,93 +118,93 @@ glob@^7.1.1: has-ansi@^2.0.0: version "2.0.0" - resolved "has-ansi-2.0.0.tgz#34f5049ce1ecdf2b0649af3ef24e45ed35416d91" + resolved has-ansi-2.0.0.tgz#34f5049ce1ecdf2b0649af3ef24e45ed35416d91 dependencies: ansi-regex "^2.0.0" has-flag@^2.0.0: version "2.0.0" - resolved "has-flag-2.0.0.tgz#e8207af1cc7b30d446cc70b734b5e8be18f88d51" + resolved has-flag-2.0.0.tgz#e8207af1cc7b30d446cc70b734b5e8be18f88d51 inflight@^1.0.4: version "1.0.6" - resolved "inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9" + resolved inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9 dependencies: once "^1.3.0" wrappy "1" inherits@2: version "2.0.3" - resolved "inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" + resolved inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de js-tokens@^3.0.2: version "3.0.2" - resolved "js-tokens-3.0.2.tgz#9866df395102130e38f7f996bceb65443209c25b" + resolved js-tokens-3.0.2.tgz#9866df395102130e38f7f996bceb65443209c25b js-yaml@^3.7.0: version "3.10.0" - resolved "js-yaml-3.10.0.tgz#2e78441646bd4682e963f22b6e92823c309c62dc" + resolved js-yaml-3.10.0.tgz#2e78441646bd4682e963f22b6e92823c309c62dc dependencies: argparse "^1.0.7" esprima "^4.0.0" minimatch@^3.0.4: version "3.0.4" - resolved "minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" + resolved minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083 dependencies: brace-expansion "^1.1.7" once@^1.3.0: version "1.4.0" - resolved "once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" + resolved once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1 dependencies: wrappy "1" path-is-absolute@^1.0.0: version "1.0.1" - resolved "path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" + resolved path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f path-parse@^1.0.5: version "1.0.5" - resolved "path-parse-1.0.5.tgz#3c1adf871ea9cd6c9431b6ea2bd74a0ff055c4c1" + resolved path-parse-1.0.5.tgz#3c1adf871ea9cd6c9431b6ea2bd74a0ff055c4c1 resolve@^1.3.2: version "1.5.0" - resolved "resolve-1.5.0.tgz#1f09acce796c9a762579f31b2c1cc4c3cddf9f36" + resolved resolve-1.5.0.tgz#1f09acce796c9a762579f31b2c1cc4c3cddf9f36 dependencies: path-parse "^1.0.5" semver@^5.3.0: version "5.5.0" - resolved "semver-5.5.0.tgz#dc4bbc7a6ca9d916dee5d43516f0092b58f7b8ab" + resolved semver-5.5.0.tgz#dc4bbc7a6ca9d916dee5d43516f0092b58f7b8ab sprintf-js@~1.0.2: version "1.0.3" - resolved "sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" + resolved sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c strip-ansi@^3.0.0: version "3.0.1" - resolved "strip-ansi-3.0.1.tgz#6a385fb8853d952d5ff05d0e8aaf94278dc63dcf" + resolved strip-ansi-3.0.1.tgz#6a385fb8853d952d5ff05d0e8aaf94278dc63dcf dependencies: ansi-regex "^2.0.0" supports-color@^2.0.0: version "2.0.0" - resolved "supports-color-2.0.0.tgz#535d045ce6b6363fa40117084629995e9df324c7" + resolved supports-color-2.0.0.tgz#535d045ce6b6363fa40117084629995e9df324c7 supports-color@^4.0.0: version "4.5.0" - resolved "supports-color-4.5.0.tgz#be7a0de484dec5c5cddf8b3d59125044912f635b" + resolved supports-color-4.5.0.tgz#be7a0de484dec5c5cddf8b3d59125044912f635b dependencies: has-flag "^2.0.0" tslib@^1.8.0, tslib@^1.8.1: version "1.9.0" - resolved "tslib-1.9.0.tgz#e37a86fda8cbbaf23a057f473c9f4dc64e5fc2e8" + resolved tslib-1.9.0.tgz#e37a86fda8cbbaf23a057f473c9f4dc64e5fc2e8 tslint@^5.9.1: version "5.9.1" - resolved "tslint-5.9.1.tgz#1255f87a3ff57eb0b0e1f0e610a8b4748046c9ae" + resolved tslint-5.9.1.tgz#1255f87a3ff57eb0b0e1f0e610a8b4748046c9ae dependencies: babel-code-frame "^6.22.0" builtin-modules "^1.1.1" @@ -221,14 +221,15 @@ tslint@^5.9.1: tsutils@^2.12.1: version "2.19.1" - resolved "tsutils-2.19.1.tgz#76d7ebdea9d7a7bf4a05f50ead3701b0168708d7" + resolved tsutils-2.19.1.tgz#76d7ebdea9d7a7bf4a05f50ead3701b0168708d7 dependencies: tslib "^1.8.1" -typescript@3.6.3: - version "3.6.3" - resolved "typescript-3.6.3.tgz#fea942fabb20f7e1ca7164ff626f1a9f3f70b4da" +typescript@^3.7.1-rc: + version "3.7.1-rc" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.7.1-rc.tgz#2054b872d67f8dc732e36c1df397f9327f37ada0" + integrity sha512-2rMtWppLsaPvmpXsoIAXWDBQVnJMw1ITGGSnidMuayLj9iCmMRT69ncKZw/Mk5rXfJkilApKucWQZxproALoRw== wrappy@1: version "1.0.2" - resolved "wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" + resolved wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f From 36b6c32f4fa177aa1f4021b162801346aac69bfe Mon Sep 17 00:00:00 2001 From: Asger F Date: Wed, 30 Oct 2019 12:14:07 +0000 Subject: [PATCH 088/148] TS: Update expected output --- .../TypeScript/PromiseType/PromiseType.expected | 2 -- .../TypeScript/Types/GetExprType.expected | 2 +- .../TypeScript/Types/GetTypeDefinitionType.expected | 2 +- .../TypeScript/Types/GetTypeExprType.expected | 12 ++++++------ .../TypeScript/Types/ReferenceDefinition.expected | 2 ++ 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/javascript/ql/test/library-tests/TypeScript/PromiseType/PromiseType.expected b/javascript/ql/test/library-tests/TypeScript/PromiseType/PromiseType.expected index bd0c467ad14..40c1f30e72a 100644 --- a/javascript/ql/test/library-tests/TypeScript/PromiseType/PromiseType.expected +++ b/javascript/ql/test/library-tests/TypeScript/PromiseType/PromiseType.expected @@ -1,10 +1,8 @@ | p1 | MyPromise | string | | p2 | MyPromise | any | | p3 | Promise | string | -| p4 | PromiseLike | string | | p5 | PromiseLike | string | | p6 | Thenable | string | -| p7 | Thenable | string | | p8 | ThenPromise | string | | p9 | JQueryPromise | string | | p10 | JQueryGenericPromise | string | diff --git a/javascript/ql/test/library-tests/TypeScript/Types/GetExprType.expected b/javascript/ql/test/library-tests/TypeScript/Types/GetExprType.expected index 371cdb05b21..429f056d2df 100644 --- a/javascript/ql/test/library-tests/TypeScript/Types/GetExprType.expected +++ b/javascript/ql/test/library-tests/TypeScript/Types/GetExprType.expected @@ -115,4 +115,4 @@ | type_definitions.ts:16:5:16:9 | color | Color | | type_definitions.ts:18:6:18:22 | EnumWithOneMember | EnumWithOneMember | | type_definitions.ts:19:5:19:5 | e | EnumWithOneMember | -| type_definitions.ts:22:5:22:23 | aliasForNumberArray | number[] | +| type_definitions.ts:22:5:22:23 | aliasForNumberArray | Alias | diff --git a/javascript/ql/test/library-tests/TypeScript/Types/GetTypeDefinitionType.expected b/javascript/ql/test/library-tests/TypeScript/Types/GetTypeDefinitionType.expected index 00d2813750e..670cb08a713 100644 --- a/javascript/ql/test/library-tests/TypeScript/Types/GetTypeDefinitionType.expected +++ b/javascript/ql/test/library-tests/TypeScript/Types/GetTypeDefinitionType.expected @@ -5,4 +5,4 @@ | type_definitions.ts:8:1:10:1 | class C ... x: T\\n} | C | | type_definitions.ts:13:1:15:1 | enum Co ... blue\\n} | Color | | type_definitions.ts:18:1:18:33 | enum En ... ember } | EnumWithOneMember | -| type_definitions.ts:21:1:21:20 | type Alias = T[]; | T[] | +| type_definitions.ts:21:1:21:20 | type Alias = T[]; | Alias | diff --git a/javascript/ql/test/library-tests/TypeScript/Types/GetTypeExprType.expected b/javascript/ql/test/library-tests/TypeScript/Types/GetTypeExprType.expected index 5bd82703bd8..3efd9dee572 100644 --- a/javascript/ql/test/library-tests/TypeScript/Types/GetTypeExprType.expected +++ b/javascript/ql/test/library-tests/TypeScript/Types/GetTypeExprType.expected @@ -57,14 +57,14 @@ | tst.ts:37:17:37:18 | [] | [] | | tst.ts:38:27:38:47 | [number ... ring[]] | [number, ...string[]] | | tst.ts:38:28:38:33 | number | number | -| tst.ts:38:36:38:46 | ...string[] | string[] | +| tst.ts:38:36:38:46 | ...string[] | string | | tst.ts:38:39:38:44 | string | string | | tst.ts:38:39:38:46 | string[] | string[] | | tst.ts:39:39:39:68 | [number ... mber[]] | [number, string?, ...number[]] | | tst.ts:39:40:39:45 | number | number | | tst.ts:39:48:39:53 | string | string | | tst.ts:39:48:39:54 | string? | string | -| tst.ts:39:57:39:67 | ...number[] | number[] | +| tst.ts:39:57:39:67 | ...number[] | number | | tst.ts:39:60:39:65 | number | number | | tst.ts:39:60:39:67 | number[] | number[] | | tst.ts:40:18:40:24 | unknown | unknown | @@ -84,10 +84,10 @@ | type_definitions.ts:11:10:11:15 | number | number | | type_definitions.ts:16:12:16:16 | Color | Color | | type_definitions.ts:19:8:19:24 | EnumWithOneMember | EnumWithOneMember | -| type_definitions.ts:21:6:21:10 | Alias | T[] | +| type_definitions.ts:21:6:21:10 | Alias | Alias | | type_definitions.ts:21:12:21:12 | T | T | | type_definitions.ts:21:17:21:17 | T | T | -| type_definitions.ts:21:17:21:19 | T[] | T[] | -| type_definitions.ts:22:26:22:30 | Alias | T[] | -| type_definitions.ts:22:26:22:38 | Alias | number[] | +| type_definitions.ts:21:17:21:19 | T[] | Alias | +| type_definitions.ts:22:26:22:30 | Alias | Alias | +| type_definitions.ts:22:26:22:38 | Alias | Alias | | type_definitions.ts:22:32:22:37 | number | number | diff --git a/javascript/ql/test/library-tests/TypeScript/Types/ReferenceDefinition.expected b/javascript/ql/test/library-tests/TypeScript/Types/ReferenceDefinition.expected index 9704ac96d53..12c9ebd4a10 100644 --- a/javascript/ql/test/library-tests/TypeScript/Types/ReferenceDefinition.expected +++ b/javascript/ql/test/library-tests/TypeScript/Types/ReferenceDefinition.expected @@ -1,3 +1,5 @@ +| Alias | type_definitions.ts:21:1:21:20 | type Alias = T[]; | +| Alias | type_definitions.ts:21:1:21:20 | type Alias = T[]; | | C | type_definition_objects.ts:3:8:3:17 | class C {} | | C | type_definitions.ts:8:1:10:1 | class C ... x: T\\n} | | C | type_definitions.ts:8:1:10:1 | class C ... x: T\\n} | From f76006e490de30d8fe9a6fb2e47bb93b86074438 Mon Sep 17 00:00:00 2001 From: Asger F Date: Wed, 30 Oct 2019 13:05:41 +0000 Subject: [PATCH 089/148] JS: Delete duplicate test case (typo) --- .../OptionalChaining/ShortCirtuiting.expected | 8 -------- .../library-tests/OptionalChaining/ShortCirtuiting.ql | 7 ------- 2 files changed, 15 deletions(-) delete mode 100644 javascript/ql/test/library-tests/OptionalChaining/ShortCirtuiting.expected delete mode 100644 javascript/ql/test/library-tests/OptionalChaining/ShortCirtuiting.ql diff --git a/javascript/ql/test/library-tests/OptionalChaining/ShortCirtuiting.expected b/javascript/ql/test/library-tests/OptionalChaining/ShortCirtuiting.expected deleted file mode 100644 index d61dccc8c5a..00000000000 --- a/javascript/ql/test/library-tests/OptionalChaining/ShortCirtuiting.expected +++ /dev/null @@ -1,8 +0,0 @@ -| short-circuiting.js:4:10:4:11 | o1 | file://:0:0:0:0 | null | -| short-circuiting.js:4:10:4:11 | o1 | short-circuiting.js:2:14:2:15 | object literal | -| short-circuiting.js:8:10:8:11 | o2 | file://:0:0:0:0 | null | -| short-circuiting.js:8:10:8:11 | o2 | short-circuiting.js:6:14:6:15 | object literal | -| short-circuiting.js:13:10:13:11 | o3 | file://:0:0:0:0 | null | -| short-circuiting.js:13:10:13:11 | o3 | short-circuiting.js:10:14:10:15 | object literal | -| short-circuiting.js:14:10:14:11 | o4 | file://:0:0:0:0 | null | -| short-circuiting.js:14:10:14:11 | o4 | short-circuiting.js:11:14:11:15 | object literal | diff --git a/javascript/ql/test/library-tests/OptionalChaining/ShortCirtuiting.ql b/javascript/ql/test/library-tests/OptionalChaining/ShortCirtuiting.ql deleted file mode 100644 index e14838bd3ed..00000000000 --- a/javascript/ql/test/library-tests/OptionalChaining/ShortCirtuiting.ql +++ /dev/null @@ -1,7 +0,0 @@ -import javascript - -from CallExpr c, Expr arg -where - c.getCalleeName() = "DUMP" and - arg = c.getArgument(0) -select arg, arg.analyze().getAValue() From 869fe4558f3039b9d02cd01040c732c5fcddc35e Mon Sep 17 00:00:00 2001 From: Asger F Date: Wed, 30 Oct 2019 13:21:07 +0000 Subject: [PATCH 090/148] TS: Support optional chaining --- .../src/com/semmle/jcorn/CustomParser.java | 4 +- .../src/com/semmle/jcorn/Parser.java | 16 +- .../src/com/semmle/js/ast/Chainable.java | 10 + .../js/parser/TypeScriptASTConverter.java | 23 +- .../tests/ts/input/optionalChaining.ts | 3 + .../ts/output/trap/optionalChaining.ts.trap | 303 ++++++++++++++++++ .../OptionalChainRoot.expected | 18 ++ .../OptionalChaining/OptionalUse.expected | 18 ++ .../OptionalChaining/ShortCircuiting.expected | 8 + .../short-circuiting-typescript.ts | 15 + .../OptionalChaining/tst-typescript.ts | 15 + 11 files changed, 417 insertions(+), 16 deletions(-) create mode 100644 javascript/extractor/tests/ts/input/optionalChaining.ts create mode 100644 javascript/extractor/tests/ts/output/trap/optionalChaining.ts.trap create mode 100644 javascript/ql/test/library-tests/OptionalChaining/short-circuiting-typescript.ts create mode 100644 javascript/ql/test/library-tests/OptionalChaining/tst-typescript.ts diff --git a/javascript/extractor/src/com/semmle/jcorn/CustomParser.java b/javascript/extractor/src/com/semmle/jcorn/CustomParser.java index a77548ed21f..888a0aec639 100644 --- a/javascript/extractor/src/com/semmle/jcorn/CustomParser.java +++ b/javascript/extractor/src/com/semmle/jcorn/CustomParser.java @@ -7,6 +7,7 @@ import com.semmle.js.ast.AssignmentExpression; import com.semmle.js.ast.BlockStatement; import com.semmle.js.ast.CallExpression; import com.semmle.js.ast.CatchClause; +import com.semmle.js.ast.Chainable; import com.semmle.js.ast.ClassExpression; import com.semmle.js.ast.ComprehensionBlock; import com.semmle.js.ast.ComprehensionExpression; @@ -470,7 +471,8 @@ public class CustomParser extends FlowParser { Expression property = this.parsePropertyIdentifierOrIdentifier(); MemberExpression node = - new MemberExpression(start, base, property, false, false, isOnOptionalChain(false, base)); + new MemberExpression( + start, base, property, false, false, Chainable.isOnOptionalChain(false, base)); return Pair.make(this.finishNode(node), true); } else if (this.eat(doubleDot)) { SourceLocation start = new SourceLocation(startLoc); diff --git a/javascript/extractor/src/com/semmle/jcorn/Parser.java b/javascript/extractor/src/com/semmle/jcorn/Parser.java index 013f8e57181..f0e3c0d004f 100644 --- a/javascript/extractor/src/com/semmle/jcorn/Parser.java +++ b/javascript/extractor/src/com/semmle/jcorn/Parser.java @@ -1520,10 +1520,6 @@ public class Parser { } } - protected boolean isOnOptionalChain(boolean optional, Expression base) { - return optional || base instanceof Chainable && ((Chainable) base).isOnOptionalChain(); - } - /** * Parse a single subscript {@code s}; if more subscripts could follow, return {@code Pair.make(s, * true}, otherwise return {@code Pair.make(s, false)}. @@ -1544,7 +1540,7 @@ public class Parser { this.parseExpression(false, null), true, optional, - isOnOptionalChain(optional, base)); + Chainable.isOnOptionalChain(optional, base)); this.expect(TokenType.bracketR); return Pair.make(this.finishNode(node), true); } else if (!noCalls && this.eat(TokenType.parenL)) { @@ -1572,10 +1568,10 @@ public class Parser { new ArrayList<>(), exprList, optional, - isOnOptionalChain(optional, base)); + Chainable.isOnOptionalChain(optional, base)); return Pair.make(this.finishNode(node), true); } else if (this.type == TokenType.backQuote) { - if (isOnOptionalChain(optional, base)) { + if (Chainable.isOnOptionalChain(optional, base)) { this.raise(base, "An optional chain may not be used in a tagged template expression."); } TaggedTemplateExpression node = @@ -1590,7 +1586,7 @@ public class Parser { this.parseIdent(true), false, optional, - isOnOptionalChain(optional, base)); + Chainable.isOnOptionalChain(optional, base)); return Pair.make(this.finishNode(node), true); } else { return Pair.make(base, false); @@ -1832,7 +1828,7 @@ public class Parser { Expression callee = this.parseSubscripts(this.parseExprAtom(null), innerStartPos, innerStartLoc, true); - if (isOnOptionalChain(false, callee)) + if (Chainable.isOnOptionalChain(false, callee)) this.raise(callee, "An optional chain may not be used in a `new` expression."); return parseNewArguments(startLoc, callee); @@ -2314,7 +2310,7 @@ public class Parser { } if (node instanceof MemberExpression) { - if (isOnOptionalChain(false, (MemberExpression) node)) + if (Chainable.isOnOptionalChain(false, (MemberExpression) node)) this.raise(node, "Invalid left-hand side in assignment"); if (!isBinding) return node; } diff --git a/javascript/extractor/src/com/semmle/js/ast/Chainable.java b/javascript/extractor/src/com/semmle/js/ast/Chainable.java index a6efe38d1eb..7d6e17d360b 100644 --- a/javascript/extractor/src/com/semmle/js/ast/Chainable.java +++ b/javascript/extractor/src/com/semmle/js/ast/Chainable.java @@ -7,4 +7,14 @@ public interface Chainable { /** Is this on an optional chain? */ abstract boolean isOnOptionalChain(); + + /** + * Returns true if a chainable node is on an optional chain. + * + * @param optional true if the node in question is itself optional (has the ?. token) + * @param base the calle or base of the optional access + */ + public static boolean isOnOptionalChain(boolean optional, Expression base) { + return optional || base instanceof Chainable && ((Chainable) base).isOnOptionalChain(); + } } diff --git a/javascript/extractor/src/com/semmle/js/parser/TypeScriptASTConverter.java b/javascript/extractor/src/com/semmle/js/parser/TypeScriptASTConverter.java index 33d131cbf00..1c39417e02e 100644 --- a/javascript/extractor/src/com/semmle/js/parser/TypeScriptASTConverter.java +++ b/javascript/extractor/src/com/semmle/js/parser/TypeScriptASTConverter.java @@ -17,6 +17,7 @@ import com.semmle.js.ast.BlockStatement; import com.semmle.js.ast.BreakStatement; import com.semmle.js.ast.CallExpression; import com.semmle.js.ast.CatchClause; +import com.semmle.js.ast.Chainable; import com.semmle.js.ast.ClassBody; import com.semmle.js.ast.ClassDeclaration; import com.semmle.js.ast.ClassExpression; @@ -877,7 +878,10 @@ public class TypeScriptASTConverter { } Expression callee = convertChild(node, "expression"); List typeArguments = convertChildrenAsTypes(node, "typeArguments"); - CallExpression call = new CallExpression(loc, callee, typeArguments, arguments, false, false); + boolean optional = node.has("questionDotToken"); + boolean onOptionalChain = Chainable.isOnOptionalChain(optional, callee); + CallExpression call = + new CallExpression(loc, callee, typeArguments, arguments, optional, onOptionalChain); attachResolvedSignature(call, node); return call; } @@ -1108,7 +1112,9 @@ public class TypeScriptASTConverter { throws ParseError { Expression object = convertChild(node, "expression"); Expression property = convertChild(node, "argumentExpression"); - return new MemberExpression(loc, object, property, true, false, false); + boolean optional = node.has("questionDotToken"); + boolean onOptionalChain = Chainable.isOnOptionalChain(optional, object); + return new MemberExpression(loc, object, property, true, optional, onOptionalChain); } private Node convertEmptyStatement(SourceLocation loc) { @@ -1584,10 +1590,14 @@ public class TypeScriptASTConverter { private Node convertMetaProperty(JsonObject node, SourceLocation loc) throws ParseError { Position metaStart = loc.getStart(); - String keywordKind = syntaxKinds.get(node.getAsJsonPrimitive("keywordToken").getAsInt() + "").getAsString(); + String keywordKind = + syntaxKinds.get(node.getAsJsonPrimitive("keywordToken").getAsInt() + "").getAsString(); String identifier = keywordKind.equals("ImportKeyword") ? "import" : "new"; Position metaEnd = - new Position(metaStart.getLine(), metaStart.getColumn() + identifier.length(), metaStart.getOffset() + identifier.length()); + new Position( + metaStart.getLine(), + metaStart.getColumn() + identifier.length(), + metaStart.getOffset() + identifier.length()); SourceLocation metaLoc = new SourceLocation(identifier, metaStart, metaEnd); Identifier meta = new Identifier(metaLoc, identifier); return new MetaProperty(loc, meta, convertChild(node, "name")); @@ -1967,8 +1977,11 @@ public class TypeScriptASTConverter { private Node convertPropertyAccessExpression(JsonObject node, SourceLocation loc) throws ParseError { + Expression base = convertChild(node, "expression"); + boolean optional = node.has("questionDotToken"); + boolean onOptionalChain = Chainable.isOnOptionalChain(optional, base); return new MemberExpression( - loc, convertChild(node, "expression"), convertChild(node, "name"), false, false, false); + loc, base, convertChild(node, "name"), false, optional, onOptionalChain); } private Node convertPropertyAssignment(JsonObject node, SourceLocation loc) throws ParseError { diff --git a/javascript/extractor/tests/ts/input/optionalChaining.ts b/javascript/extractor/tests/ts/input/optionalChaining.ts new file mode 100644 index 00000000000..5070e38ab24 --- /dev/null +++ b/javascript/extractor/tests/ts/input/optionalChaining.ts @@ -0,0 +1,3 @@ +base?.x.y; +base?.(x).y; +base?.[z].y; diff --git a/javascript/extractor/tests/ts/output/trap/optionalChaining.ts.trap b/javascript/extractor/tests/ts/output/trap/optionalChaining.ts.trap new file mode 100644 index 00000000000..0a414c9da57 --- /dev/null +++ b/javascript/extractor/tests/ts/output/trap/optionalChaining.ts.trap @@ -0,0 +1,303 @@ +#10000=@"/optionalChaining.ts;sourcefile" +files(#10000,"/optionalChaining.ts","optionalChaining","ts",0) +#10001=@"/;folder" +folders(#10001,"/","") +containerparent(#10001,#10000) +#10002=@"loc,{#10000},0,0,0,0" +locations_default(#10002,#10000,0,0,0,0) +hasLocation(#10000,#10002) +#20000=@"global_scope" +scopes(#20000,0) +#20001=@"script;{#10000},1,1" +#20002=* +lines(#20002,#20001,"base?.x.y;"," +") +#20003=@"loc,{#10000},1,1,1,10" +locations_default(#20003,#10000,1,1,1,10) +hasLocation(#20002,#20003) +#20004=* +lines(#20004,#20001,"base?.(x).y;"," +") +#20005=@"loc,{#10000},2,1,2,12" +locations_default(#20005,#10000,2,1,2,12) +hasLocation(#20004,#20005) +#20006=* +lines(#20006,#20001,"base?.[z].y;"," +") +#20007=@"loc,{#10000},3,1,3,12" +locations_default(#20007,#10000,3,1,3,12) +hasLocation(#20006,#20007) +numlines(#20001,3,3,0) +#20008=* +tokeninfo(#20008,6,#20001,0,"base") +#20009=@"loc,{#10000},1,1,1,4" +locations_default(#20009,#10000,1,1,1,4) +hasLocation(#20008,#20009) +#20010=* +tokeninfo(#20010,8,#20001,1,"?.") +#20011=@"loc,{#10000},1,5,1,6" +locations_default(#20011,#10000,1,5,1,6) +hasLocation(#20010,#20011) +#20012=* +tokeninfo(#20012,6,#20001,2,"x") +#20013=@"loc,{#10000},1,7,1,7" +locations_default(#20013,#10000,1,7,1,7) +hasLocation(#20012,#20013) +#20014=* +tokeninfo(#20014,8,#20001,3,".") +#20015=@"loc,{#10000},1,8,1,8" +locations_default(#20015,#10000,1,8,1,8) +hasLocation(#20014,#20015) +#20016=* +tokeninfo(#20016,6,#20001,4,"y") +#20017=@"loc,{#10000},1,9,1,9" +locations_default(#20017,#10000,1,9,1,9) +hasLocation(#20016,#20017) +#20018=* +tokeninfo(#20018,8,#20001,5,";") +#20019=@"loc,{#10000},1,10,1,10" +locations_default(#20019,#10000,1,10,1,10) +hasLocation(#20018,#20019) +#20020=* +tokeninfo(#20020,6,#20001,6,"base") +#20021=@"loc,{#10000},2,1,2,4" +locations_default(#20021,#10000,2,1,2,4) +hasLocation(#20020,#20021) +#20022=* +tokeninfo(#20022,8,#20001,7,"?.") +#20023=@"loc,{#10000},2,5,2,6" +locations_default(#20023,#10000,2,5,2,6) +hasLocation(#20022,#20023) +#20024=* +tokeninfo(#20024,8,#20001,8,"(") +#20025=@"loc,{#10000},2,7,2,7" +locations_default(#20025,#10000,2,7,2,7) +hasLocation(#20024,#20025) +#20026=* +tokeninfo(#20026,6,#20001,9,"x") +#20027=@"loc,{#10000},2,8,2,8" +locations_default(#20027,#10000,2,8,2,8) +hasLocation(#20026,#20027) +#20028=* +tokeninfo(#20028,8,#20001,10,")") +#20029=@"loc,{#10000},2,9,2,9" +locations_default(#20029,#10000,2,9,2,9) +hasLocation(#20028,#20029) +#20030=* +tokeninfo(#20030,8,#20001,11,".") +#20031=@"loc,{#10000},2,10,2,10" +locations_default(#20031,#10000,2,10,2,10) +hasLocation(#20030,#20031) +#20032=* +tokeninfo(#20032,6,#20001,12,"y") +#20033=@"loc,{#10000},2,11,2,11" +locations_default(#20033,#10000,2,11,2,11) +hasLocation(#20032,#20033) +#20034=* +tokeninfo(#20034,8,#20001,13,";") +#20035=@"loc,{#10000},2,12,2,12" +locations_default(#20035,#10000,2,12,2,12) +hasLocation(#20034,#20035) +#20036=* +tokeninfo(#20036,6,#20001,14,"base") +#20037=@"loc,{#10000},3,1,3,4" +locations_default(#20037,#10000,3,1,3,4) +hasLocation(#20036,#20037) +#20038=* +tokeninfo(#20038,8,#20001,15,"?.") +#20039=@"loc,{#10000},3,5,3,6" +locations_default(#20039,#10000,3,5,3,6) +hasLocation(#20038,#20039) +#20040=* +tokeninfo(#20040,8,#20001,16,"[") +#20041=@"loc,{#10000},3,7,3,7" +locations_default(#20041,#10000,3,7,3,7) +hasLocation(#20040,#20041) +#20042=* +tokeninfo(#20042,6,#20001,17,"z") +#20043=@"loc,{#10000},3,8,3,8" +locations_default(#20043,#10000,3,8,3,8) +hasLocation(#20042,#20043) +#20044=* +tokeninfo(#20044,8,#20001,18,"]") +#20045=@"loc,{#10000},3,9,3,9" +locations_default(#20045,#10000,3,9,3,9) +hasLocation(#20044,#20045) +#20046=* +tokeninfo(#20046,8,#20001,19,".") +#20047=@"loc,{#10000},3,10,3,10" +locations_default(#20047,#10000,3,10,3,10) +hasLocation(#20046,#20047) +#20048=* +tokeninfo(#20048,6,#20001,20,"y") +#20049=@"loc,{#10000},3,11,3,11" +locations_default(#20049,#10000,3,11,3,11) +hasLocation(#20048,#20049) +#20050=* +tokeninfo(#20050,8,#20001,21,";") +#20051=@"loc,{#10000},3,12,3,12" +locations_default(#20051,#10000,3,12,3,12) +hasLocation(#20050,#20051) +#20052=* +tokeninfo(#20052,0,#20001,22,"") +#20053=@"loc,{#10000},4,1,4,0" +locations_default(#20053,#10000,4,1,4,0) +hasLocation(#20052,#20053) +toplevels(#20001,0) +#20054=@"loc,{#10000},1,1,4,0" +locations_default(#20054,#10000,1,1,4,0) +hasLocation(#20001,#20054) +#20055=* +stmts(#20055,2,#20001,0,"base?.x.y;") +hasLocation(#20055,#20003) +stmtContainers(#20055,#20001) +#20056=* +exprs(#20056,14,#20055,0,"base?.x.y") +#20057=@"loc,{#10000},1,1,1,9" +locations_default(#20057,#10000,1,1,1,9) +hasLocation(#20056,#20057) +enclosingStmt(#20056,#20055) +exprContainers(#20056,#20001) +#20058=* +exprs(#20058,14,#20056,0,"base?.x") +#20059=@"loc,{#10000},1,1,1,7" +locations_default(#20059,#10000,1,1,1,7) +hasLocation(#20058,#20059) +enclosingStmt(#20058,#20055) +exprContainers(#20058,#20001) +#20060=* +exprs(#20060,79,#20058,0,"base") +hasLocation(#20060,#20009) +enclosingStmt(#20060,#20055) +exprContainers(#20060,#20001) +literals("base","base",#20060) +#20061=@"var;{base};{#20000}" +variables(#20061,"base",#20000) +bind(#20060,#20061) +#20062=* +exprs(#20062,0,#20058,1,"x") +hasLocation(#20062,#20013) +enclosingStmt(#20062,#20055) +exprContainers(#20062,#20001) +literals("x","x",#20062) +isOptionalChaining(#20058) +#20063=* +exprs(#20063,0,#20056,1,"y") +hasLocation(#20063,#20017) +enclosingStmt(#20063,#20055) +exprContainers(#20063,#20001) +literals("y","y",#20063) +#20064=* +stmts(#20064,2,#20001,1,"base?.(x).y;") +hasLocation(#20064,#20005) +stmtContainers(#20064,#20001) +#20065=* +exprs(#20065,14,#20064,0,"base?.(x).y") +#20066=@"loc,{#10000},2,1,2,11" +locations_default(#20066,#10000,2,1,2,11) +hasLocation(#20065,#20066) +enclosingStmt(#20065,#20064) +exprContainers(#20065,#20001) +#20067=* +exprs(#20067,13,#20065,0,"base?.(x)") +#20068=@"loc,{#10000},2,1,2,9" +locations_default(#20068,#10000,2,1,2,9) +hasLocation(#20067,#20068) +enclosingStmt(#20067,#20064) +exprContainers(#20067,#20001) +#20069=* +exprs(#20069,79,#20067,-1,"base") +hasLocation(#20069,#20021) +enclosingStmt(#20069,#20064) +exprContainers(#20069,#20001) +literals("base","base",#20069) +bind(#20069,#20061) +#20070=* +exprs(#20070,79,#20067,0,"x") +hasLocation(#20070,#20027) +enclosingStmt(#20070,#20064) +exprContainers(#20070,#20001) +literals("x","x",#20070) +#20071=@"var;{x};{#20000}" +variables(#20071,"x",#20000) +bind(#20070,#20071) +isOptionalChaining(#20067) +#20072=* +exprs(#20072,0,#20065,1,"y") +hasLocation(#20072,#20033) +enclosingStmt(#20072,#20064) +exprContainers(#20072,#20001) +literals("y","y",#20072) +#20073=* +stmts(#20073,2,#20001,2,"base?.[z].y;") +hasLocation(#20073,#20007) +stmtContainers(#20073,#20001) +#20074=* +exprs(#20074,14,#20073,0,"base?.[z].y") +#20075=@"loc,{#10000},3,1,3,11" +locations_default(#20075,#10000,3,1,3,11) +hasLocation(#20074,#20075) +enclosingStmt(#20074,#20073) +exprContainers(#20074,#20001) +#20076=* +exprs(#20076,15,#20074,0,"base?.[z]") +#20077=@"loc,{#10000},3,1,3,9" +locations_default(#20077,#10000,3,1,3,9) +hasLocation(#20076,#20077) +enclosingStmt(#20076,#20073) +exprContainers(#20076,#20001) +#20078=* +exprs(#20078,79,#20076,0,"base") +hasLocation(#20078,#20037) +enclosingStmt(#20078,#20073) +exprContainers(#20078,#20001) +literals("base","base",#20078) +bind(#20078,#20061) +#20079=* +exprs(#20079,79,#20076,1,"z") +hasLocation(#20079,#20043) +enclosingStmt(#20079,#20073) +exprContainers(#20079,#20001) +literals("z","z",#20079) +#20080=@"var;{z};{#20000}" +variables(#20080,"z",#20000) +bind(#20079,#20080) +isOptionalChaining(#20076) +#20081=* +exprs(#20081,0,#20074,1,"y") +hasLocation(#20081,#20049) +enclosingStmt(#20081,#20073) +exprContainers(#20081,#20001) +literals("y","y",#20081) +#20082=* +entry_cfg_node(#20082,#20001) +#20083=@"loc,{#10000},1,1,1,0" +locations_default(#20083,#10000,1,1,1,0) +hasLocation(#20082,#20083) +#20084=* +exit_cfg_node(#20084,#20001) +hasLocation(#20084,#20053) +successor(#20073,#20078) +successor(#20081,#20074) +successor(#20079,#20076) +successor(#20078,#20079) +successor(#20076,#20081) +successor(#20078,#20084) +successor(#20074,#20084) +successor(#20064,#20069) +successor(#20072,#20065) +successor(#20070,#20067) +successor(#20069,#20070) +successor(#20067,#20072) +successor(#20069,#20073) +successor(#20065,#20073) +successor(#20055,#20060) +successor(#20063,#20056) +successor(#20062,#20058) +successor(#20060,#20062) +successor(#20058,#20063) +successor(#20060,#20064) +successor(#20056,#20064) +successor(#20082,#20055) +numlines(#10000,3,3,0) +filetype(#10000,"typescript") diff --git a/javascript/ql/test/library-tests/OptionalChaining/OptionalChainRoot.expected b/javascript/ql/test/library-tests/OptionalChaining/OptionalChainRoot.expected index 5ec86ee7462..1ff8c71d6db 100644 --- a/javascript/ql/test/library-tests/OptionalChaining/OptionalChainRoot.expected +++ b/javascript/ql/test/library-tests/OptionalChaining/OptionalChainRoot.expected @@ -1,7 +1,25 @@ +| short-circuiting-typescript.ts:3:5:3:18 | x?.(o1 = null) | short-circuiting-typescript.ts:3:5:3:18 | x?.(o1 = null) | +| short-circuiting-typescript.ts:7:5:7:18 | x?.[o2 = null] | short-circuiting-typescript.ts:7:5:7:18 | x?.[o2 = null] | +| short-circuiting-typescript.ts:12:5:12:31 | x?.[o3 ... = null) | short-circuiting-typescript.ts:12:5:12:18 | x?.[o3 = null] | +| short-circuiting-typescript.ts:12:5:12:31 | x?.[o3 ... = null) | short-circuiting-typescript.ts:12:5:12:31 | x?.[o3 ... = null) | | short-circuiting.js:3:5:3:18 | x?.(o1 = null) | short-circuiting.js:3:5:3:18 | x?.(o1 = null) | | short-circuiting.js:7:5:7:18 | x?.[o2 = null] | short-circuiting.js:7:5:7:18 | x?.[o2 = null] | | short-circuiting.js:12:5:12:31 | x?.[o3 ... = null) | short-circuiting.js:12:5:12:18 | x?.[o3 = null] | | short-circuiting.js:12:5:12:31 | x?.[o3 ... = null) | short-circuiting.js:12:5:12:31 | x?.[o3 ... = null) | +| tst-typescript.ts:2:1:2:6 | a?.b.c | tst-typescript.ts:2:1:2:4 | a?.b | +| tst-typescript.ts:3:1:3:6 | a.b?.c | tst-typescript.ts:3:1:3:6 | a.b?.c | +| tst-typescript.ts:4:1:4:7 | a?.b?.c | tst-typescript.ts:4:1:4:4 | a?.b | +| tst-typescript.ts:4:1:4:7 | a?.b?.c | tst-typescript.ts:4:1:4:7 | a?.b?.c | +| tst-typescript.ts:7:1:7:7 | f?.()() | tst-typescript.ts:7:1:7:5 | f?.() | +| tst-typescript.ts:8:1:8:7 | f()?.() | tst-typescript.ts:8:1:8:7 | f()?.() | +| tst-typescript.ts:9:1:9:9 | f?.()?.() | tst-typescript.ts:9:1:9:5 | f?.() | +| tst-typescript.ts:9:1:9:9 | f?.()?.() | tst-typescript.ts:9:1:9:9 | f?.()?.() | +| tst-typescript.ts:12:1:12:8 | a?.m().b | tst-typescript.ts:12:1:12:4 | a?.m | +| tst-typescript.ts:13:1:13:9 | a.m?.().b | tst-typescript.ts:13:1:13:7 | a.m?.() | +| tst-typescript.ts:14:1:14:8 | a.m()?.b | tst-typescript.ts:14:1:14:8 | a.m()?.b | +| tst-typescript.ts:15:1:15:11 | a?.m?.()?.b | tst-typescript.ts:15:1:15:4 | a?.m | +| tst-typescript.ts:15:1:15:11 | a?.m?.()?.b | tst-typescript.ts:15:1:15:8 | a?.m?.() | +| tst-typescript.ts:15:1:15:11 | a?.m?.()?.b | tst-typescript.ts:15:1:15:11 | a?.m?.()?.b | | tst.js:2:1:2:6 | a?.b.c | tst.js:2:1:2:4 | a?.b | | tst.js:3:1:3:6 | a.b?.c | tst.js:3:1:3:6 | a.b?.c | | tst.js:4:1:4:7 | a?.b?.c | tst.js:4:1:4:4 | a?.b | diff --git a/javascript/ql/test/library-tests/OptionalChaining/OptionalUse.expected b/javascript/ql/test/library-tests/OptionalChaining/OptionalUse.expected index 430d4d52299..8358bb0ecde 100644 --- a/javascript/ql/test/library-tests/OptionalChaining/OptionalUse.expected +++ b/javascript/ql/test/library-tests/OptionalChaining/OptionalUse.expected @@ -1,7 +1,25 @@ +| short-circuiting-typescript.ts:3:5:3:18 | x?.(o1 = null) | +| short-circuiting-typescript.ts:7:5:7:18 | x?.[o2 = null] | +| short-circuiting-typescript.ts:12:5:12:18 | x?.[o3 = null] | +| short-circuiting-typescript.ts:12:5:12:31 | x?.[o3 ... = null) | | short-circuiting.js:3:5:3:18 | x?.(o1 = null) | | short-circuiting.js:7:5:7:18 | x?.[o2 = null] | | short-circuiting.js:12:5:12:18 | x?.[o3 = null] | | short-circuiting.js:12:5:12:31 | x?.[o3 ... = null) | +| tst-typescript.ts:2:1:2:4 | a?.b | +| tst-typescript.ts:3:1:3:6 | a.b?.c | +| tst-typescript.ts:4:1:4:4 | a?.b | +| tst-typescript.ts:4:1:4:7 | a?.b?.c | +| tst-typescript.ts:7:1:7:5 | f?.() | +| tst-typescript.ts:8:1:8:7 | f()?.() | +| tst-typescript.ts:9:1:9:5 | f?.() | +| tst-typescript.ts:9:1:9:9 | f?.()?.() | +| tst-typescript.ts:12:1:12:4 | a?.m | +| tst-typescript.ts:13:1:13:7 | a.m?.() | +| tst-typescript.ts:14:1:14:8 | a.m()?.b | +| tst-typescript.ts:15:1:15:4 | a?.m | +| tst-typescript.ts:15:1:15:8 | a?.m?.() | +| tst-typescript.ts:15:1:15:11 | a?.m?.()?.b | | tst.js:2:1:2:4 | a?.b | | tst.js:3:1:3:6 | a.b?.c | | tst.js:4:1:4:4 | a?.b | diff --git a/javascript/ql/test/library-tests/OptionalChaining/ShortCircuiting.expected b/javascript/ql/test/library-tests/OptionalChaining/ShortCircuiting.expected index d61dccc8c5a..6711c704828 100644 --- a/javascript/ql/test/library-tests/OptionalChaining/ShortCircuiting.expected +++ b/javascript/ql/test/library-tests/OptionalChaining/ShortCircuiting.expected @@ -1,3 +1,11 @@ +| short-circuiting-typescript.ts:4:10:4:11 | o1 | file://:0:0:0:0 | null | +| short-circuiting-typescript.ts:4:10:4:11 | o1 | short-circuiting-typescript.ts:2:14:2:15 | object literal | +| short-circuiting-typescript.ts:8:10:8:11 | o2 | file://:0:0:0:0 | null | +| short-circuiting-typescript.ts:8:10:8:11 | o2 | short-circuiting-typescript.ts:6:14:6:15 | object literal | +| short-circuiting-typescript.ts:13:10:13:11 | o3 | file://:0:0:0:0 | null | +| short-circuiting-typescript.ts:13:10:13:11 | o3 | short-circuiting-typescript.ts:10:14:10:15 | object literal | +| short-circuiting-typescript.ts:14:10:14:11 | o4 | file://:0:0:0:0 | null | +| short-circuiting-typescript.ts:14:10:14:11 | o4 | short-circuiting-typescript.ts:11:14:11:15 | object literal | | short-circuiting.js:4:10:4:11 | o1 | file://:0:0:0:0 | null | | short-circuiting.js:4:10:4:11 | o1 | short-circuiting.js:2:14:2:15 | object literal | | short-circuiting.js:8:10:8:11 | o2 | file://:0:0:0:0 | null | diff --git a/javascript/ql/test/library-tests/OptionalChaining/short-circuiting-typescript.ts b/javascript/ql/test/library-tests/OptionalChaining/short-circuiting-typescript.ts new file mode 100644 index 00000000000..7367e64dd4c --- /dev/null +++ b/javascript/ql/test/library-tests/OptionalChaining/short-circuiting-typescript.ts @@ -0,0 +1,15 @@ +(function() { + var o1 = {}; + x?.(o1 = null); + DUMP(o1); + + var o2 = {}; + x?.[o2 = null]; + DUMP(o2); + + var o3 = {}, + o4 = {}; + x?.[o3 = null]?.(o4 = null); + DUMP(o3); + DUMP(o4); +}); diff --git a/javascript/ql/test/library-tests/OptionalChaining/tst-typescript.ts b/javascript/ql/test/library-tests/OptionalChaining/tst-typescript.ts new file mode 100644 index 00000000000..39389024d6d --- /dev/null +++ b/javascript/ql/test/library-tests/OptionalChaining/tst-typescript.ts @@ -0,0 +1,15 @@ +a.b.c; +a?.b.c; +a.b?.c; +a?.b?.c; + +f()(); +f?.()(); +f()?.(); +f?.()?.(); + +a.m().b; +a?.m().b; +a.m?.().b; +a.m()?.b; +a?.m?.()?.b; From f50f3b48c4f09920cfec667461054e8028fefdbd Mon Sep 17 00:00:00 2001 From: Asger F Date: Wed, 30 Oct 2019 13:35:50 +0000 Subject: [PATCH 091/148] TS: Add test for ?? operator (already works) --- javascript/ql/test/library-tests/Flow/AbstractValues.expected | 2 ++ javascript/ql/test/library-tests/Flow/abseval.expected | 2 ++ javascript/ql/test/library-tests/Flow/getAPrototype.expected | 1 + javascript/ql/test/library-tests/Flow/nullish-coalescing-op.ts | 3 +++ javascript/ql/test/library-tests/Flow/types.expected | 1 + 5 files changed, 9 insertions(+) create mode 100644 javascript/ql/test/library-tests/Flow/nullish-coalescing-op.ts diff --git a/javascript/ql/test/library-tests/Flow/AbstractValues.expected b/javascript/ql/test/library-tests/Flow/AbstractValues.expected index f172a33c000..42117cac98a 100644 --- a/javascript/ql/test/library-tests/Flow/AbstractValues.expected +++ b/javascript/ql/test/library-tests/Flow/AbstractValues.expected @@ -208,6 +208,8 @@ | nodeJsLib.js:1:18:1:43 | instance of function nodeJsModule | | nodeJsLib.js:3:15:3:37 | function nodeJsFoo | | nodeJsLib.js:3:15:3:37 | instance of function nodeJsFoo | +| nullish-coalescing-op.ts:1:1:3:1 | function test | +| nullish-coalescing-op.ts:1:1:3:1 | instance of function test | | o.js:1:1:2:0 | exports object of module o | | o.js:1:1:2:0 | module object of module o | | objlit.js:1:14:1:15 | object literal | diff --git a/javascript/ql/test/library-tests/Flow/abseval.expected b/javascript/ql/test/library-tests/Flow/abseval.expected index 30169b47e8b..00d7f27da89 100644 --- a/javascript/ql/test/library-tests/Flow/abseval.expected +++ b/javascript/ql/test/library-tests/Flow/abseval.expected @@ -204,6 +204,8 @@ | nodeJsClient.js:5:5:5:6 | x2 | nodeJsClient.js:5:10:5:15 | es.foo | esLib.js:3:8:3:24 | function foo | | nodeJsClient.js:5:5:5:6 | x2 | nodeJsClient.js:5:10:5:15 | es.foo | file://:0:0:0:0 | indefinite value (call) | | nodeJsClient.js:5:5:5:6 | x2 | nodeJsClient.js:5:10:5:15 | es.foo | file://:0:0:0:0 | indefinite value (heap) | +| nullish-coalescing-op.ts:2:7:2:7 | y | nullish-coalescing-op.ts:2:11:2:18 | x ?? 0.5 | file://:0:0:0:0 | indefinite value (call) | +| nullish-coalescing-op.ts:2:7:2:7 | y | nullish-coalescing-op.ts:2:11:2:18 | x ?? 0.5 | file://:0:0:0:0 | non-zero value | | objlit.js:1:5:1:5 | A | objlit.js:1:9:1:15 | A \|\| {} | file://:0:0:0:0 | indefinite value (global) | | objlit.js:1:5:1:5 | A | objlit.js:1:9:1:15 | A \|\| {} | objlit.js:1:14:1:15 | object literal | | objlit.js:7:5:7:5 | B | objlit.js:7:9:7:10 | {} | objlit.js:7:9:7:10 | object literal | diff --git a/javascript/ql/test/library-tests/Flow/getAPrototype.expected b/javascript/ql/test/library-tests/Flow/getAPrototype.expected index e8825515715..07ff1f76b24 100644 --- a/javascript/ql/test/library-tests/Flow/getAPrototype.expected +++ b/javascript/ql/test/library-tests/Flow/getAPrototype.expected @@ -40,6 +40,7 @@ | nestedImport.js:9:1:12:1 | instance of function tst | nestedImport.js:9:1:12:1 | instance of function tst | | nodeJsLib.js:1:18:1:43 | instance of function nodeJsModule | nodeJsLib.js:1:18:1:43 | instance of function nodeJsModule | | nodeJsLib.js:3:15:3:37 | instance of function nodeJsFoo | nodeJsLib.js:3:15:3:37 | instance of function nodeJsFoo | +| nullish-coalescing-op.ts:1:1:3:1 | instance of function test | nullish-coalescing-op.ts:1:1:3:1 | instance of function test | | objlit.js:2:9:2:21 | instance of anonymous function | objlit.js:2:9:2:21 | instance of anonymous function | | objlit.js:4:8:4:20 | instance of method baz | objlit.js:4:8:4:20 | instance of method baz | | objlit.js:10:2:12:1 | instance of anonymous function | objlit.js:10:2:12:1 | instance of anonymous function | diff --git a/javascript/ql/test/library-tests/Flow/nullish-coalescing-op.ts b/javascript/ql/test/library-tests/Flow/nullish-coalescing-op.ts new file mode 100644 index 00000000000..9fb2e4c1312 --- /dev/null +++ b/javascript/ql/test/library-tests/Flow/nullish-coalescing-op.ts @@ -0,0 +1,3 @@ +function test(x) { + let y = x ?? 0.5; +} diff --git a/javascript/ql/test/library-tests/Flow/types.expected b/javascript/ql/test/library-tests/Flow/types.expected index ba7aca62b08..9b13b469bed 100644 --- a/javascript/ql/test/library-tests/Flow/types.expected +++ b/javascript/ql/test/library-tests/Flow/types.expected @@ -113,6 +113,7 @@ | nodeJsClient.js:2:5:2:6 | es | nodeJsClient.js:2:10:2:27 | require('./esLib') | boolean, class, date, function, null, number, object, regular expression,string or undefined | | nodeJsClient.js:4:5:4:6 | x1 | nodeJsClient.js:4:10:4:15 | nj.foo | boolean, class, date, function, null, number, object, regular expression,string or undefined | | nodeJsClient.js:5:5:5:6 | x2 | nodeJsClient.js:5:10:5:15 | es.foo | boolean, class, date, function, null, number, object, regular expression,string or undefined | +| nullish-coalescing-op.ts:2:7:2:7 | y | nullish-coalescing-op.ts:2:11:2:18 | x ?? 0.5 | boolean, class, date, function, null, number, object, regular expression,string or undefined | | objlit.js:1:5:1:5 | A | objlit.js:1:9:1:15 | A \|\| {} | boolean, class, date, function, null, number, object, regular expression,string or undefined | | objlit.js:7:5:7:5 | B | objlit.js:7:9:7:10 | {} | object | | objlit.js:14:5:14:6 | x1 | objlit.js:14:10:14:10 | A | boolean, class, date, function, null, number, object, regular expression,string or undefined | From 4e7b987fa37fb6988d2659884974bd9581f8da79 Mon Sep 17 00:00:00 2001 From: Asger F Date: Wed, 30 Oct 2019 14:16:57 +0000 Subject: [PATCH 092/148] TS: Rename IsTypeExpr -> PredicateTypeExpr --- .../src/com/semmle/js/ast/DefaultVisitor.java | 4 +-- .../src/com/semmle/js/ast/NodeCopier.java | 7 ++-- .../src/com/semmle/js/ast/Visitor.java | 4 +-- .../com/semmle/js/extractor/ASTExtractor.java | 8 ++--- .../com/semmle/js/extractor/ScopeManager.java | 6 ++-- .../semmle/js/extractor/TypeExprKinds.java | 4 +-- .../js/parser/TypeScriptASTConverter.java | 4 +-- .../src/com/semmle/ts/ast/IsTypeExpr.java | 32 ----------------- .../com/semmle/ts/ast/PredicateTypeExpr.java | 34 +++++++++++++++++++ .../ql/src/semmle/javascript/TypeScript.qll | 2 +- .../ql/src/semmlecode.javascript.dbscheme | 2 +- .../src/semmlecode.javascript.dbscheme.stats | 2 +- 12 files changed, 56 insertions(+), 53 deletions(-) delete mode 100644 javascript/extractor/src/com/semmle/ts/ast/IsTypeExpr.java create mode 100644 javascript/extractor/src/com/semmle/ts/ast/PredicateTypeExpr.java diff --git a/javascript/extractor/src/com/semmle/js/ast/DefaultVisitor.java b/javascript/extractor/src/com/semmle/js/ast/DefaultVisitor.java index d6779fdaa3b..cb11b9376ea 100644 --- a/javascript/extractor/src/com/semmle/js/ast/DefaultVisitor.java +++ b/javascript/extractor/src/com/semmle/js/ast/DefaultVisitor.java @@ -34,13 +34,13 @@ import com.semmle.ts.ast.InferTypeExpr; import com.semmle.ts.ast.InterfaceDeclaration; import com.semmle.ts.ast.InterfaceTypeExpr; import com.semmle.ts.ast.IntersectionTypeExpr; -import com.semmle.ts.ast.IsTypeExpr; import com.semmle.ts.ast.KeywordTypeExpr; import com.semmle.ts.ast.MappedTypeExpr; import com.semmle.ts.ast.NamespaceDeclaration; import com.semmle.ts.ast.NonNullAssertion; import com.semmle.ts.ast.OptionalTypeExpr; import com.semmle.ts.ast.ParenthesizedTypeExpr; +import com.semmle.ts.ast.PredicateTypeExpr; import com.semmle.ts.ast.RestTypeExpr; import com.semmle.ts.ast.TupleTypeExpr; import com.semmle.ts.ast.TypeAliasDeclaration; @@ -634,7 +634,7 @@ public class DefaultVisitor implements Visitor { } @Override - public R visit(IsTypeExpr nd, C c) { + public R visit(PredicateTypeExpr nd, C c) { return visit((TypeExpression) nd, c); } diff --git a/javascript/extractor/src/com/semmle/js/ast/NodeCopier.java b/javascript/extractor/src/com/semmle/js/ast/NodeCopier.java index cdfc0441a57..0a8716ed732 100644 --- a/javascript/extractor/src/com/semmle/js/ast/NodeCopier.java +++ b/javascript/extractor/src/com/semmle/js/ast/NodeCopier.java @@ -30,13 +30,13 @@ import com.semmle.ts.ast.InferTypeExpr; import com.semmle.ts.ast.InterfaceDeclaration; import com.semmle.ts.ast.InterfaceTypeExpr; import com.semmle.ts.ast.IntersectionTypeExpr; -import com.semmle.ts.ast.IsTypeExpr; import com.semmle.ts.ast.KeywordTypeExpr; import com.semmle.ts.ast.MappedTypeExpr; import com.semmle.ts.ast.NamespaceDeclaration; import com.semmle.ts.ast.NonNullAssertion; import com.semmle.ts.ast.OptionalTypeExpr; import com.semmle.ts.ast.ParenthesizedTypeExpr; +import com.semmle.ts.ast.PredicateTypeExpr; import com.semmle.ts.ast.RestTypeExpr; import com.semmle.ts.ast.TupleTypeExpr; import com.semmle.ts.ast.TypeAliasDeclaration; @@ -717,8 +717,9 @@ public class NodeCopier implements Visitor { } @Override - public INode visit(IsTypeExpr nd, Void c) { - return new IsTypeExpr(visit(nd.getLoc()), copy(nd.getLeft()), copy(nd.getRight())); + public INode visit(PredicateTypeExpr nd, Void c) { + return new PredicateTypeExpr( + visit(nd.getLoc()), copy(nd.getExpression()), copy(nd.getTypeExpr())); } @Override diff --git a/javascript/extractor/src/com/semmle/js/ast/Visitor.java b/javascript/extractor/src/com/semmle/js/ast/Visitor.java index f825d448719..97a0dd73118 100644 --- a/javascript/extractor/src/com/semmle/js/ast/Visitor.java +++ b/javascript/extractor/src/com/semmle/js/ast/Visitor.java @@ -30,13 +30,13 @@ import com.semmle.ts.ast.InferTypeExpr; import com.semmle.ts.ast.InterfaceDeclaration; import com.semmle.ts.ast.InterfaceTypeExpr; import com.semmle.ts.ast.IntersectionTypeExpr; -import com.semmle.ts.ast.IsTypeExpr; import com.semmle.ts.ast.KeywordTypeExpr; import com.semmle.ts.ast.MappedTypeExpr; import com.semmle.ts.ast.NamespaceDeclaration; import com.semmle.ts.ast.NonNullAssertion; import com.semmle.ts.ast.OptionalTypeExpr; import com.semmle.ts.ast.ParenthesizedTypeExpr; +import com.semmle.ts.ast.PredicateTypeExpr; import com.semmle.ts.ast.RestTypeExpr; import com.semmle.ts.ast.TupleTypeExpr; import com.semmle.ts.ast.TypeAliasDeclaration; @@ -256,7 +256,7 @@ public interface Visitor { public R visit(TypeofTypeExpr nd, C c); - public R visit(IsTypeExpr nd, C c); + public R visit(PredicateTypeExpr nd, C c); public R visit(InterfaceTypeExpr nd, C c); diff --git a/javascript/extractor/src/com/semmle/js/extractor/ASTExtractor.java b/javascript/extractor/src/com/semmle/js/extractor/ASTExtractor.java index c2a3ee70cb8..cc29551b391 100644 --- a/javascript/extractor/src/com/semmle/js/extractor/ASTExtractor.java +++ b/javascript/extractor/src/com/semmle/js/extractor/ASTExtractor.java @@ -125,13 +125,13 @@ import com.semmle.ts.ast.InferTypeExpr; import com.semmle.ts.ast.InterfaceDeclaration; import com.semmle.ts.ast.InterfaceTypeExpr; import com.semmle.ts.ast.IntersectionTypeExpr; -import com.semmle.ts.ast.IsTypeExpr; import com.semmle.ts.ast.KeywordTypeExpr; import com.semmle.ts.ast.MappedTypeExpr; import com.semmle.ts.ast.NamespaceDeclaration; import com.semmle.ts.ast.NonNullAssertion; import com.semmle.ts.ast.OptionalTypeExpr; import com.semmle.ts.ast.ParenthesizedTypeExpr; +import com.semmle.ts.ast.PredicateTypeExpr; import com.semmle.ts.ast.RestTypeExpr; import com.semmle.ts.ast.TupleTypeExpr; import com.semmle.ts.ast.TypeAliasDeclaration; @@ -1707,10 +1707,10 @@ public class ASTExtractor { } @Override - public Label visit(IsTypeExpr nd, Context c) { + public Label visit(PredicateTypeExpr nd, Context c) { Label key = super.visit(nd, c); - visit(nd.getLeft(), key, 0, IdContext.varInTypeBind); - visit(nd.getRight(), key, 1, IdContext.typeBind); + visit(nd.getExpression(), key, 0, IdContext.varInTypeBind); + visit(nd.getTypeExpr(), key, 1, IdContext.typeBind); return key; } diff --git a/javascript/extractor/src/com/semmle/js/extractor/ScopeManager.java b/javascript/extractor/src/com/semmle/js/extractor/ScopeManager.java index 1270b0a9409..3bffa4ad3ae 100644 --- a/javascript/extractor/src/com/semmle/js/extractor/ScopeManager.java +++ b/javascript/extractor/src/com/semmle/js/extractor/ScopeManager.java @@ -46,9 +46,9 @@ import com.semmle.ts.ast.InferTypeExpr; import com.semmle.ts.ast.InterfaceDeclaration; import com.semmle.ts.ast.InterfaceTypeExpr; import com.semmle.ts.ast.IntersectionTypeExpr; -import com.semmle.ts.ast.IsTypeExpr; import com.semmle.ts.ast.NamespaceDeclaration; import com.semmle.ts.ast.ParenthesizedTypeExpr; +import com.semmle.ts.ast.PredicateTypeExpr; import com.semmle.ts.ast.TupleTypeExpr; import com.semmle.ts.ast.TypeAliasDeclaration; import com.semmle.ts.ast.UnionTypeExpr; @@ -660,8 +660,8 @@ public class ScopeManager { } @Override - public Void visit(IsTypeExpr nd, Void c) { - return nd.getRight().accept(this, c); + public Void visit(PredicateTypeExpr nd, Void c) { + return nd.getTypeExpr().accept(this, c); } @Override diff --git a/javascript/extractor/src/com/semmle/js/extractor/TypeExprKinds.java b/javascript/extractor/src/com/semmle/js/extractor/TypeExprKinds.java index 78b901da475..6c9263a5360 100644 --- a/javascript/extractor/src/com/semmle/js/extractor/TypeExprKinds.java +++ b/javascript/extractor/src/com/semmle/js/extractor/TypeExprKinds.java @@ -16,11 +16,11 @@ import com.semmle.ts.ast.IndexedAccessTypeExpr; import com.semmle.ts.ast.InferTypeExpr; import com.semmle.ts.ast.InterfaceTypeExpr; import com.semmle.ts.ast.IntersectionTypeExpr; -import com.semmle.ts.ast.IsTypeExpr; import com.semmle.ts.ast.KeywordTypeExpr; import com.semmle.ts.ast.MappedTypeExpr; import com.semmle.ts.ast.OptionalTypeExpr; import com.semmle.ts.ast.ParenthesizedTypeExpr; +import com.semmle.ts.ast.PredicateTypeExpr; import com.semmle.ts.ast.RestTypeExpr; import com.semmle.ts.ast.TupleTypeExpr; import com.semmle.ts.ast.TypeParameter; @@ -159,7 +159,7 @@ public class TypeExprKinds { } @Override - public Integer visit(IsTypeExpr nd, Void c) { + public Integer visit(PredicateTypeExpr nd, Void c) { return isTypeExpr; } diff --git a/javascript/extractor/src/com/semmle/js/parser/TypeScriptASTConverter.java b/javascript/extractor/src/com/semmle/js/parser/TypeScriptASTConverter.java index 1c39417e02e..af0cf755a0e 100644 --- a/javascript/extractor/src/com/semmle/js/parser/TypeScriptASTConverter.java +++ b/javascript/extractor/src/com/semmle/js/parser/TypeScriptASTConverter.java @@ -127,13 +127,13 @@ import com.semmle.ts.ast.InferTypeExpr; import com.semmle.ts.ast.InterfaceDeclaration; import com.semmle.ts.ast.InterfaceTypeExpr; import com.semmle.ts.ast.IntersectionTypeExpr; -import com.semmle.ts.ast.IsTypeExpr; import com.semmle.ts.ast.KeywordTypeExpr; import com.semmle.ts.ast.MappedTypeExpr; import com.semmle.ts.ast.NamespaceDeclaration; import com.semmle.ts.ast.NonNullAssertion; import com.semmle.ts.ast.OptionalTypeExpr; import com.semmle.ts.ast.ParenthesizedTypeExpr; +import com.semmle.ts.ast.PredicateTypeExpr; import com.semmle.ts.ast.RestTypeExpr; import com.semmle.ts.ast.TupleTypeExpr; import com.semmle.ts.ast.TypeAliasDeclaration; @@ -2199,7 +2199,7 @@ public class TypeScriptASTConverter { } private Node convertTypePredicate(JsonObject node, SourceLocation loc) throws ParseError { - return new IsTypeExpr( + return new PredicateTypeExpr( loc, convertChildAsType(node, "parameterName"), convertChildAsType(node, "type")); } diff --git a/javascript/extractor/src/com/semmle/ts/ast/IsTypeExpr.java b/javascript/extractor/src/com/semmle/ts/ast/IsTypeExpr.java deleted file mode 100644 index a64fdb83650..00000000000 --- a/javascript/extractor/src/com/semmle/ts/ast/IsTypeExpr.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.semmle.ts.ast; - -import com.semmle.js.ast.SourceLocation; -import com.semmle.js.ast.Visitor; - -/** - * A type of form E is T where E is a parameter name or this and - * T is a type. - */ -public class IsTypeExpr extends TypeExpression { - private final ITypeExpression left; // Always Identifier or KeywordTypeExpr (in case of 'this') - private final ITypeExpression right; - - public IsTypeExpr(SourceLocation loc, ITypeExpression left, ITypeExpression right) { - super("IsTypeExpr", loc); - this.left = left; - this.right = right; - } - - public ITypeExpression getLeft() { - return left; - } - - public ITypeExpression getRight() { - return right; - } - - @Override - public R accept(Visitor v, C c) { - return v.visit(this, c); - } -} diff --git a/javascript/extractor/src/com/semmle/ts/ast/PredicateTypeExpr.java b/javascript/extractor/src/com/semmle/ts/ast/PredicateTypeExpr.java new file mode 100644 index 00000000000..215a345646a --- /dev/null +++ b/javascript/extractor/src/com/semmle/ts/ast/PredicateTypeExpr.java @@ -0,0 +1,34 @@ +package com.semmle.ts.ast; + +import com.semmle.js.ast.SourceLocation; +import com.semmle.js.ast.Visitor; + +/** + * A type of form E is T where E is a parameter name or this and + * T is a type. + */ +public class PredicateTypeExpr extends TypeExpression { + private final ITypeExpression expression; + private final ITypeExpression type; + + public PredicateTypeExpr(SourceLocation loc, ITypeExpression expression, ITypeExpression type) { + super("PredicateTypeExpr", loc); + this.expression = expression; + this.type = type; + } + + /** Returns the E in E is T. */ + public ITypeExpression getExpression() { + return expression; + } + + /** Returns the T in E is T. */ + public ITypeExpression getTypeExpr() { + return type; + } + + @Override + public R accept(Visitor v, C c) { + return v.visit(this, c); + } +} diff --git a/javascript/ql/src/semmle/javascript/TypeScript.qll b/javascript/ql/src/semmle/javascript/TypeScript.qll index 005f9a5ee12..e85ea681ece 100644 --- a/javascript/ql/src/semmle/javascript/TypeScript.qll +++ b/javascript/ql/src/semmle/javascript/TypeScript.qll @@ -918,7 +918,7 @@ class TypeofTypeExpr extends @typeoftypeexpr, TypeExpr { * * This can only occur as the return type of a function type. */ -class IsTypeExpr extends @istypeexpr, TypeExpr { +class IsTypeExpr extends @predicatetypeexpr, TypeExpr { /** * Gets the parameter name or `this` token `E` in `E is T`. */ diff --git a/javascript/ql/src/semmlecode.javascript.dbscheme b/javascript/ql/src/semmlecode.javascript.dbscheme index 874ebfcd4d2..735e288e580 100644 --- a/javascript/ql/src/semmlecode.javascript.dbscheme +++ b/javascript/ql/src/semmlecode.javascript.dbscheme @@ -559,7 +559,7 @@ case @typeexpr.kind of | 17 = @localvartypeaccess | 18 = @qualifiedvartypeaccess | 19 = @thisvartypeaccess -| 20 = @istypeexpr +| 20 = @predicatetypeexpr | 21 = @interfacetypeexpr | 22 = @typeparameter | 23 = @plainfunctiontypeexpr diff --git a/javascript/ql/src/semmlecode.javascript.dbscheme.stats b/javascript/ql/src/semmlecode.javascript.dbscheme.stats index 146950b5be7..f9ac7439548 100644 --- a/javascript/ql/src/semmlecode.javascript.dbscheme.stats +++ b/javascript/ql/src/semmlecode.javascript.dbscheme.stats @@ -850,7 +850,7 @@ 20 -@istypeexpr +@predicatetypeexpr 86 From b81931e402c87659bf98e279770aa18b48f92e24 Mon Sep 17 00:00:00 2001 From: Asger F Date: Wed, 30 Oct 2019 14:51:00 +0000 Subject: [PATCH 093/148] TS: Support assertion types --- .../src/com/semmle/js/ast/NodeCopier.java | 5 ++- .../com/semmle/js/extractor/ASTExtractor.java | 3 ++ .../js/parser/TypeScriptASTConverter.java | 5 ++- .../com/semmle/ts/ast/PredicateTypeExpr.java | 12 +++++- .../ql/src/semmle/javascript/TypeScript.qll | 42 ++++++++++++++++--- .../ql/src/semmlecode.javascript.dbscheme | 2 + .../src/semmlecode.javascript.dbscheme.stats | 11 +++++ .../TypeScript/TypeAnnotations/IsTypeExpr.qll | 5 --- .../TypeAnnotations/PredicateTypeExpr.qll | 13 ++++++ .../TypeScript/TypeAnnotations/tests.expected | 16 +++++++ .../TypeScript/TypeAnnotations/tests.ql | 2 +- .../TypeScript/TypeAnnotations/tst.ts | 12 ++++++ 12 files changed, 113 insertions(+), 15 deletions(-) delete mode 100644 javascript/ql/test/library-tests/TypeScript/TypeAnnotations/IsTypeExpr.qll create mode 100644 javascript/ql/test/library-tests/TypeScript/TypeAnnotations/PredicateTypeExpr.qll diff --git a/javascript/extractor/src/com/semmle/js/ast/NodeCopier.java b/javascript/extractor/src/com/semmle/js/ast/NodeCopier.java index 0a8716ed732..9c61ca081df 100644 --- a/javascript/extractor/src/com/semmle/js/ast/NodeCopier.java +++ b/javascript/extractor/src/com/semmle/js/ast/NodeCopier.java @@ -719,7 +719,10 @@ public class NodeCopier implements Visitor { @Override public INode visit(PredicateTypeExpr nd, Void c) { return new PredicateTypeExpr( - visit(nd.getLoc()), copy(nd.getExpression()), copy(nd.getTypeExpr())); + visit(nd.getLoc()), + copy(nd.getExpression()), + copy(nd.getTypeExpr()), + nd.hasAssertsKeyword()); } @Override diff --git a/javascript/extractor/src/com/semmle/js/extractor/ASTExtractor.java b/javascript/extractor/src/com/semmle/js/extractor/ASTExtractor.java index cc29551b391..d3571fc5a57 100644 --- a/javascript/extractor/src/com/semmle/js/extractor/ASTExtractor.java +++ b/javascript/extractor/src/com/semmle/js/extractor/ASTExtractor.java @@ -1711,6 +1711,9 @@ public class ASTExtractor { Label key = super.visit(nd, c); visit(nd.getExpression(), key, 0, IdContext.varInTypeBind); visit(nd.getTypeExpr(), key, 1, IdContext.typeBind); + if (nd.hasAssertsKeyword()) { + trapwriter.addTuple("hasAssertsKeyword", key); + } return key; } diff --git a/javascript/extractor/src/com/semmle/js/parser/TypeScriptASTConverter.java b/javascript/extractor/src/com/semmle/js/parser/TypeScriptASTConverter.java index af0cf755a0e..7f826a7dc1d 100644 --- a/javascript/extractor/src/com/semmle/js/parser/TypeScriptASTConverter.java +++ b/javascript/extractor/src/com/semmle/js/parser/TypeScriptASTConverter.java @@ -2200,7 +2200,10 @@ public class TypeScriptASTConverter { private Node convertTypePredicate(JsonObject node, SourceLocation loc) throws ParseError { return new PredicateTypeExpr( - loc, convertChildAsType(node, "parameterName"), convertChildAsType(node, "type")); + loc, + convertChildAsType(node, "parameterName"), + convertChildAsType(node, "type"), + node.has("assertsModifier")); } private Node convertTypeReference(JsonObject node, SourceLocation loc) throws ParseError { diff --git a/javascript/extractor/src/com/semmle/ts/ast/PredicateTypeExpr.java b/javascript/extractor/src/com/semmle/ts/ast/PredicateTypeExpr.java index 215a345646a..474cb888c0b 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/PredicateTypeExpr.java +++ b/javascript/extractor/src/com/semmle/ts/ast/PredicateTypeExpr.java @@ -10,11 +10,17 @@ import com.semmle.js.ast.Visitor; public class PredicateTypeExpr extends TypeExpression { private final ITypeExpression expression; private final ITypeExpression type; + private final boolean hasAssertsKeyword; - public PredicateTypeExpr(SourceLocation loc, ITypeExpression expression, ITypeExpression type) { + public PredicateTypeExpr( + SourceLocation loc, + ITypeExpression expression, + ITypeExpression type, + boolean hasAssertsKeyword) { super("PredicateTypeExpr", loc); this.expression = expression; this.type = type; + this.hasAssertsKeyword = hasAssertsKeyword; } /** Returns the E in E is T. */ @@ -27,6 +33,10 @@ public class PredicateTypeExpr extends TypeExpression { return type; } + public boolean hasAssertsKeyword() { + return hasAssertsKeyword; + } + @Override public R accept(Visitor v, C c) { return v.visit(this, c); diff --git a/javascript/ql/src/semmle/javascript/TypeScript.qll b/javascript/ql/src/semmle/javascript/TypeScript.qll index e85ea681ece..597c059eb05 100644 --- a/javascript/ql/src/semmle/javascript/TypeScript.qll +++ b/javascript/ql/src/semmle/javascript/TypeScript.qll @@ -914,20 +914,49 @@ class TypeofTypeExpr extends @typeoftypeexpr, TypeExpr { } /** - * A type of form `E is T` where `E` is a parameter name or `this`, and `T` is a type. + * A function return type that refines the type of one of its parameters or `this`. * - * This can only occur as the return type of a function type. + * Examples: + * ```js + * function f(x): x is string {} + * function f(x): asserts x is string {} + * function f(x): asserts x {} + * ``` */ -class IsTypeExpr extends @predicatetypeexpr, TypeExpr { +class PredicateTypeExpr extends @predicatetypeexpr, TypeExpr { /** * Gets the parameter name or `this` token `E` in `E is T`. */ VarTypeAccess getParameterName() { result = this.getChildTypeExpr(0) } /** - * Gets the type `T` in `E is T`. + * Gets the type `T` in `E is T` or `asserts E is T`. + * + * Has no results for types of form `asserts E`. */ TypeExpr getPredicateType() { result = this.getChildTypeExpr(1) } + + /** + * Holds if this is a type of form `asserts E is T` or `asserts E`. + */ + predicate hasAssertsKeyword() { + hasAssertsKeyword(this) + } +} + +/** + * A function return type of form `x is T` or `asserts x is T`. + * + * Examples: + * ```js + * function f(x): x is string {} + * function f(x): asserts x is string {} + * ``` + */ +class IsTypeExpr extends PredicateTypeExpr { + IsTypeExpr() { + exists(getPredicateType()) + } } /** @@ -966,15 +995,16 @@ class ReadonlyTypeExpr extends @readonlytypeexpr, TypeExpr { * * This can occur as * - part of the operand to a `typeof` type, or - * - as the first operand to an `is` type. + * - as the first operand to a predicate type * * For example, it may occur as the `E` in these examples: * ``` * var x : typeof E * function f(...) : E is T {} + * function f(...) : asserts E {} * ``` * - * In the latter case, this may also refer to the pseudo-variable `this`. + * In the latter two cases, this may also refer to the pseudo-variable `this`. */ class VarTypeAccess extends @vartypeaccess, TypeExpr { } diff --git a/javascript/ql/src/semmlecode.javascript.dbscheme b/javascript/ql/src/semmlecode.javascript.dbscheme index 735e288e580..e0bd730ef98 100644 --- a/javascript/ql/src/semmlecode.javascript.dbscheme +++ b/javascript/ql/src/semmlecode.javascript.dbscheme @@ -637,6 +637,8 @@ case @type.kind of @unionorintersectiontype = @uniontype | @intersectiontype; @typevariabletype = @canonicaltypevariabletype | @lexicaltypevariabletype; +hasAssertsKeyword(int node: @predicatetypeexpr ref); + @typed_ast_node = @expr | @typeexpr | @function; ast_node_type( unique int node: @typed_ast_node ref, diff --git a/javascript/ql/src/semmlecode.javascript.dbscheme.stats b/javascript/ql/src/semmlecode.javascript.dbscheme.stats index f9ac7439548..0c1660bc7c8 100644 --- a/javascript/ql/src/semmlecode.javascript.dbscheme.stats +++ b/javascript/ql/src/semmlecode.javascript.dbscheme.stats @@ -7918,6 +7918,17 @@ +hasAssertsKeyword +66 + + +stmt +66 + + + + + isAbstractMember 66 diff --git a/javascript/ql/test/library-tests/TypeScript/TypeAnnotations/IsTypeExpr.qll b/javascript/ql/test/library-tests/TypeScript/TypeAnnotations/IsTypeExpr.qll deleted file mode 100644 index a0ff44c2bdd..00000000000 --- a/javascript/ql/test/library-tests/TypeScript/TypeAnnotations/IsTypeExpr.qll +++ /dev/null @@ -1,5 +0,0 @@ -import javascript - -query predicate test_IsTypeExpr(IsTypeExpr type, VarTypeAccess res0, TypeExpr res1) { - res0 = type.getParameterName() and res1 = type.getPredicateType() -} diff --git a/javascript/ql/test/library-tests/TypeScript/TypeAnnotations/PredicateTypeExpr.qll b/javascript/ql/test/library-tests/TypeScript/TypeAnnotations/PredicateTypeExpr.qll new file mode 100644 index 00000000000..c38bf54ad1f --- /dev/null +++ b/javascript/ql/test/library-tests/TypeScript/TypeAnnotations/PredicateTypeExpr.qll @@ -0,0 +1,13 @@ +import javascript + +query predicate test_IsTypeExpr(IsTypeExpr type, VarTypeAccess res0, TypeExpr res1) { + res0 = type.getParameterName() and res1 = type.getPredicateType() +} + +query predicate test_PredicateTypeExpr(PredicateTypeExpr type, VarTypeAccess res0) { + res0 = type.getParameterName() +} + +query predicate test_hasAssertsKeyword(PredicateTypeExpr type) { + type.hasAssertsKeyword() +} diff --git a/javascript/ql/test/library-tests/TypeScript/TypeAnnotations/tests.expected b/javascript/ql/test/library-tests/TypeScript/TypeAnnotations/tests.expected index eb0c717b2db..5db52c2afcd 100644 --- a/javascript/ql/test/library-tests/TypeScript/TypeAnnotations/tests.expected +++ b/javascript/ql/test/library-tests/TypeScript/TypeAnnotations/tests.expected @@ -105,6 +105,9 @@ test_VariableTypes | tst.ts:135:5:135:24 | tupleWithRestElement | tupleWithRestElement | tst.ts:135:27:135:47 | [number ... ring[]] | | tst.ts:136:5:136:36 | tupleWi ... lements | tupleWithOptionalAndRestElements | tst.ts:136:39:136:68 | [number ... mber[]] | | tst.ts:137:5:137:15 | unknownType | unknownType | tst.ts:137:18:137:24 | unknown | +| tst.ts:142:17:142:25 | condition | condition | tst.ts:142:28:142:30 | any | +| tst.ts:142:33:142:35 | msg | msg | tst.ts:142:39:142:44 | string | +| tst.ts:148:25:148:27 | val | val | tst.ts:148:30:148:32 | any | test_QualifiedTypeAccess | tst.ts:63:19:63:21 | N.I | tst.ts:63:19:63:19 | N | tst.ts:63:21:63:21 | I | | tst.ts:64:20:64:24 | N.M.I | tst.ts:64:20:64:22 | N.M | tst.ts:64:24:64:24 | I | @@ -148,6 +151,17 @@ test_IsTypeExpr | tst.ts:76:21:76:32 | that is Leaf | tst.ts:76:21:76:24 | that | tst.ts:76:29:76:32 | Leaf | | tst.ts:80:36:80:55 | x is Generic | tst.ts:80:36:80:36 | x | tst.ts:80:41:80:55 | Generic | | tst.ts:81:38:81:50 | x is typeof x | tst.ts:81:38:81:38 | x | tst.ts:81:43:81:50 | typeof x | +| tst.ts:148:36:148:56 | asserts ... string | tst.ts:148:44:148:46 | val | tst.ts:148:51:148:56 | string | +test_PredicateTypeExpr +| tst.ts:75:17:75:28 | this is Leaf | tst.ts:75:17:75:20 | this | +| tst.ts:76:21:76:32 | that is Leaf | tst.ts:76:21:76:24 | that | +| tst.ts:80:36:80:55 | x is Generic | tst.ts:80:36:80:36 | x | +| tst.ts:81:38:81:50 | x is typeof x | tst.ts:81:38:81:38 | x | +| tst.ts:142:48:142:64 | asserts condition | tst.ts:142:56:142:64 | condition | +| tst.ts:148:36:148:56 | asserts ... string | tst.ts:148:44:148:46 | val | +test_hasAssertsKeyword +| tst.ts:142:48:142:64 | asserts condition | +| tst.ts:148:36:148:56 | asserts ... string | test_ThisParameterTypes | function hasThisParam | tst.ts:116:29:116:32 | void | | method hasThisParam of interface InterfaceWithThisParam | tst.ts:119:22:119:43 | Interfa ... isParam | @@ -254,6 +268,8 @@ test_ReturnTypes | tst.ts:94:1:94:37 | functio ... rn x; } | function f1 | tst.ts:94:23:94:23 | S | | tst.ts:95:1:95:53 | functio ... x,y]; } | function f2 | tst.ts:95:31:95:35 | [S,T] | | tst.ts:96:1:96:52 | functio ... rn x; } | function f3 | tst.ts:96:38:96:38 | S | +| tst.ts:142:1:146:1 | functio ... )\\n }\\n} | function assert | tst.ts:142:48:142:64 | asserts condition | +| tst.ts:148:1:152:1 | functio ... ;\\n }\\n} | function assertIsString | tst.ts:148:36:148:56 | asserts ... string | test_KeyofTypeExpr | tst.ts:49:16:49:30 | keyof Interface | tst.ts:49:22:49:30 | Interface | | tst.ts:113:26:113:35 | keyof Node | tst.ts:113:32:113:35 | Node | diff --git a/javascript/ql/test/library-tests/TypeScript/TypeAnnotations/tests.ql b/javascript/ql/test/library-tests/TypeScript/TypeAnnotations/tests.ql index ff894fb06a5..bdabbc5f533 100644 --- a/javascript/ql/test/library-tests/TypeScript/TypeAnnotations/tests.ql +++ b/javascript/ql/test/library-tests/TypeScript/TypeAnnotations/tests.ql @@ -11,7 +11,7 @@ import GenericTypeExpr import IntersectionTypeExpr import FunctionTypeReturns import InterfaceTypeExpr -import IsTypeExpr +import PredicateTypeExpr import ThisParameterTypes import ChildIndex import TypeArguments diff --git a/javascript/ql/test/library-tests/TypeScript/TypeAnnotations/tst.ts b/javascript/ql/test/library-tests/TypeScript/TypeAnnotations/tst.ts index a9985b42878..faeb9e9c833 100644 --- a/javascript/ql/test/library-tests/TypeScript/TypeAnnotations/tst.ts +++ b/javascript/ql/test/library-tests/TypeScript/TypeAnnotations/tst.ts @@ -138,3 +138,15 @@ let unknownType: unknown; let taggedTemplateLiteralTypeArg1 = someTag`Hello`; let taggedTemplateLiteralTypeArg2 = someTag`Hello`; + +function assert(condition: any, msg?: string): asserts condition { + if (!condition) { + throw new AssertionError(msg) + } +} + +function assertIsString(val: any): asserts val is string { + if (typeof val !== "string") { + throw new AssertionError("Not a string!"); + } +} From 341c11523c806fa2739b4aae6bbe5a24ca349a3a Mon Sep 17 00:00:00 2001 From: Asger F Date: Wed, 30 Oct 2019 15:02:08 +0000 Subject: [PATCH 094/148] TS: Add recursive type alias tests (already works) --- .../TypeScript/Types/GetExprType.expected | 24 +++++++++++++ .../Types/GetTypeDefinitionType.expected | 3 ++ .../TypeScript/Types/GetTypeExprType.expected | 36 +++++++++++++++++++ .../Types/ReferenceDefinition.expected | 4 +++ .../TypeScript/Types/type_alias.ts | 24 +++++++++++++ 5 files changed, 91 insertions(+) diff --git a/javascript/ql/test/library-tests/TypeScript/Types/GetExprType.expected b/javascript/ql/test/library-tests/TypeScript/Types/GetExprType.expected index 429f056d2df..62832265649 100644 --- a/javascript/ql/test/library-tests/TypeScript/Types/GetExprType.expected +++ b/javascript/ql/test/library-tests/TypeScript/Types/GetExprType.expected @@ -93,6 +93,30 @@ | tst.ts:43:28:43:30 | foo | "foo" | | tst.ts:43:33:43:37 | "foo" | "foo" | | type_alias.ts:3:5:3:5 | b | boolean | +| type_alias.ts:7:5:7:5 | c | ValueOrArray | +| type_alias.ts:14:9:14:32 | [proper ... ]: Json | any | +| type_alias.ts:14:10:14:17 | property | string | +| type_alias.ts:17:5:17:8 | json | Json | +| type_alias.ts:21:18:21:35 | [key: string]: any | any | +| type_alias.ts:21:19:21:21 | key | string | +| type_alias.ts:23:7:23:12 | myNode | VirtualNode | +| type_alias.ts:24:5:27:5 | ["div", ... ]\\n ] | VirtualNode | +| type_alias.ts:24:6:24:10 | "div" | "div" | +| type_alias.ts:24:13:24:28 | { id: "parent" } | string \| { [key: string]: any; } | +| type_alias.ts:24:15:24:16 | id | string | +| type_alias.ts:24:19:24:26 | "parent" | "parent" | +| type_alias.ts:25:9:25:61 | ["div", ... child"] | VirtualNode | +| type_alias.ts:25:10:25:14 | "div" | "div" | +| type_alias.ts:25:17:25:37 | { id: " ... hild" } | string \| { [key: string]: any; } | +| type_alias.ts:25:19:25:20 | id | string | +| type_alias.ts:25:23:25:35 | "first-child" | "first-child" | +| type_alias.ts:25:40:25:60 | "I'm th ... child" | "I'm the first child" | +| type_alias.ts:26:9:26:63 | ["div", ... child"] | VirtualNode | +| type_alias.ts:26:10:26:14 | "div" | "div" | +| type_alias.ts:26:17:26:38 | { id: " ... hild" } | string \| { [key: string]: any; } | +| type_alias.ts:26:19:26:20 | id | string | +| type_alias.ts:26:23:26:36 | "second-child" | "second-child" | +| type_alias.ts:26:41:26:62 | "I'm th ... child" | "I'm the second child" | | type_definition_objects.ts:1:13:1:17 | dummy | typeof dummy.ts | | type_definition_objects.ts:1:24:1:32 | "./dummy" | any | | type_definition_objects.ts:3:14:3:14 | C | C | diff --git a/javascript/ql/test/library-tests/TypeScript/Types/GetTypeDefinitionType.expected b/javascript/ql/test/library-tests/TypeScript/Types/GetTypeDefinitionType.expected index 670cb08a713..66c343fd601 100644 --- a/javascript/ql/test/library-tests/TypeScript/Types/GetTypeDefinitionType.expected +++ b/javascript/ql/test/library-tests/TypeScript/Types/GetTypeDefinitionType.expected @@ -1,4 +1,7 @@ | type_alias.ts:1:1:1:17 | type B = boolean; | boolean | +| type_alias.ts:5:1:5:50 | type Va ... ay>; | ValueOrArray | +| type_alias.ts:9:1:15:13 | type Js ... Json[]; | Json | +| type_alias.ts:19:1:21:57 | type Vi ... ode[]]; | VirtualNode | | type_definition_objects.ts:3:8:3:17 | class C {} | C | | type_definition_objects.ts:6:8:6:16 | enum E {} | E | | type_definitions.ts:3:1:5:1 | interfa ... x: S;\\n} | I | diff --git a/javascript/ql/test/library-tests/TypeScript/Types/GetTypeExprType.expected b/javascript/ql/test/library-tests/TypeScript/Types/GetTypeExprType.expected index 3efd9dee572..0b9af780186 100644 --- a/javascript/ql/test/library-tests/TypeScript/Types/GetTypeExprType.expected +++ b/javascript/ql/test/library-tests/TypeScript/Types/GetTypeExprType.expected @@ -71,6 +71,42 @@ | type_alias.ts:1:6:1:6 | B | boolean | | type_alias.ts:1:10:1:16 | boolean | boolean | | type_alias.ts:3:8:3:8 | B | boolean | +| type_alias.ts:5:6:5:17 | ValueOrArray | ValueOrArray | +| type_alias.ts:5:19:5:19 | T | T | +| type_alias.ts:5:24:5:24 | T | T | +| type_alias.ts:5:24:5:49 | T \| Arr ... ray> | ValueOrArray | +| type_alias.ts:5:28:5:32 | Array | T[] | +| type_alias.ts:5:28:5:49 | Array> | ValueOrArray[] | +| type_alias.ts:5:34:5:45 | ValueOrArray | ValueOrArray | +| type_alias.ts:5:34:5:48 | ValueOrArray | ValueOrArray | +| type_alias.ts:5:47:5:47 | T | T | +| type_alias.ts:7:8:7:19 | ValueOrArray | ValueOrArray | +| type_alias.ts:7:8:7:27 | ValueOrArray | ValueOrArray | +| type_alias.ts:7:21:7:26 | number | number | +| type_alias.ts:9:6:9:9 | Json | Json | +| type_alias.ts:10:5:15:12 | \| strin ... Json[] | Json | +| type_alias.ts:10:7:10:12 | string | string | +| type_alias.ts:11:7:11:12 | number | number | +| type_alias.ts:12:7:12:13 | boolean | boolean | +| type_alias.ts:13:7:13:10 | null | null | +| type_alias.ts:14:7:14:34 | { [prop ... Json } | { [property: string]: Json; } | +| type_alias.ts:14:20:14:25 | string | string | +| type_alias.ts:14:29:14:32 | Json | Json | +| type_alias.ts:15:7:15:10 | Json | Json | +| type_alias.ts:15:7:15:12 | Json[] | Json[] | +| type_alias.ts:17:11:17:14 | Json | Json | +| type_alias.ts:19:6:19:16 | VirtualNode | VirtualNode | +| type_alias.ts:20:5:21:56 | \| strin ... Node[]] | VirtualNode | +| type_alias.ts:20:7:20:12 | string | string | +| type_alias.ts:21:7:21:56 | [string ... Node[]] | [string, { [key: string]: any; }, ...VirtualNod... | +| type_alias.ts:21:8:21:13 | string | string | +| type_alias.ts:21:16:21:37 | { [key: ... : any } | { [key: string]: any; } | +| type_alias.ts:21:24:21:29 | string | string | +| type_alias.ts:21:33:21:35 | any | any | +| type_alias.ts:21:40:21:55 | ...VirtualNode[] | VirtualNode | +| type_alias.ts:21:43:21:53 | VirtualNode | VirtualNode | +| type_alias.ts:21:43:21:55 | VirtualNode[] | VirtualNode[] | +| type_alias.ts:23:15:23:25 | VirtualNode | VirtualNode | | type_definitions.ts:3:11:3:11 | I | I | | type_definitions.ts:3:13:3:13 | S | S | | type_definitions.ts:4:6:4:6 | S | S | diff --git a/javascript/ql/test/library-tests/TypeScript/Types/ReferenceDefinition.expected b/javascript/ql/test/library-tests/TypeScript/Types/ReferenceDefinition.expected index 12c9ebd4a10..3426a936189 100644 --- a/javascript/ql/test/library-tests/TypeScript/Types/ReferenceDefinition.expected +++ b/javascript/ql/test/library-tests/TypeScript/Types/ReferenceDefinition.expected @@ -11,3 +11,7 @@ | EnumWithOneMember | type_definitions.ts:18:26:18:31 | member | | I | type_definitions.ts:3:1:5:1 | interfa ... x: S;\\n} | | I | type_definitions.ts:3:1:5:1 | interfa ... x: S;\\n} | +| Json | type_alias.ts:9:1:15:13 | type Js ... Json[]; | +| ValueOrArray | type_alias.ts:5:1:5:50 | type Va ... ay>; | +| ValueOrArray | type_alias.ts:5:1:5:50 | type Va ... ay>; | +| VirtualNode | type_alias.ts:19:1:21:57 | type Vi ... ode[]]; | diff --git a/javascript/ql/test/library-tests/TypeScript/Types/type_alias.ts b/javascript/ql/test/library-tests/TypeScript/Types/type_alias.ts index 3a831766991..d13eb159754 100644 --- a/javascript/ql/test/library-tests/TypeScript/Types/type_alias.ts +++ b/javascript/ql/test/library-tests/TypeScript/Types/type_alias.ts @@ -1,3 +1,27 @@ type B = boolean; var b: B; + +type ValueOrArray = T | Array>; + +var c: ValueOrArray; + +type Json = + | string + | number + | boolean + | null + | { [property: string]: Json } + | Json[]; + +var json: Json; + +type VirtualNode = + | string + | [string, { [key: string]: any }, ...VirtualNode[]]; + +const myNode: VirtualNode = + ["div", { id: "parent" }, + ["div", { id: "first-child" }, "I'm the first child"], + ["div", { id: "second-child" }, "I'm the second child"] + ]; From 7dfd4e06875d690be8e3e5a78409773c3a7efd2e Mon Sep 17 00:00:00 2001 From: Asger F Date: Wed, 30 Oct 2019 15:11:05 +0000 Subject: [PATCH 095/148] TS: Stop using the deprecated TypeReference.typeArguments --- .../extractor/lib/typescript/src/type_table.ts | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/javascript/extractor/lib/typescript/src/type_table.ts b/javascript/extractor/lib/typescript/src/type_table.ts index 553afcf2e41..19eb64a8f10 100644 --- a/javascript/extractor/lib/typescript/src/type_table.ts +++ b/javascript/extractor/lib/typescript/src/type_table.ts @@ -597,7 +597,7 @@ export class TypeTable { let tupleType = tupleReference.target; let minLength = tupleType.minLength != null ? tupleType.minLength - : tupleReference.typeArguments.length; + : this.typeChecker.getTypeArguments(tupleReference).length; let hasRestElement = tupleType.hasRestElement ? 't' : 'f'; let prefix = `tuple;${minLength};${hasRestElement}`; return this.makeTypeStringVectorFromTypeReferenceArguments(prefix, type); @@ -714,11 +714,12 @@ export class TypeTable { // There can be an extra type argument at the end, denoting an explicit 'this' type argument. // We discard the extra argument in our model. let target = type.target; - if (type.typeArguments == null) return tag; + let typeArguments = this.typeChecker.getTypeArguments(type); + if (typeArguments == null) return tag; if (target.typeParameters != null) { - return this.makeTypeStringVector(tag, type.typeArguments, target.typeParameters.length); + return this.makeTypeStringVector(tag, typeArguments, target.typeParameters.length); } else { - return this.makeTypeStringVector(tag, type.typeArguments); + return this.makeTypeStringVector(tag, typeArguments); } } @@ -992,7 +993,7 @@ export class TypeTable { * `T` is the type parameter declared on the `Promise` interface. */ private getSelfType(type: ts.Type): ts.TypeReference { - if (isTypeReference(type) && type.typeArguments != null && type.typeArguments.length > 0) { + if (isTypeReference(type) && this.typeChecker.getTypeArguments(type).length > 0) { return type.target; } return null; @@ -1181,8 +1182,9 @@ export class TypeTable { if (isTypeReference(type)) { // Note that this case also handles tuple types, since a tuple type is represented as // a reference to a synthetic generic interface. - if (type.typeArguments != null) { - type.typeArguments.forEach(callback); + let typeArguments = this.typeChecker.getTypeArguments(type); + if (typeArguments != null) { + typeArguments.forEach(callback); } } else if (type.flags & ts.TypeFlags.UnionOrIntersection) { (type as ts.UnionOrIntersectionType).types.forEach(callback); From 4846e53a101c7ba2895abba4c57fa1aa34b2e5dc Mon Sep 17 00:00:00 2001 From: Asger F Date: Wed, 30 Oct 2019 15:45:06 +0000 Subject: [PATCH 096/148] TS: Blacklist another cyclic property --- javascript/extractor/lib/typescript/src/main.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/javascript/extractor/lib/typescript/src/main.ts b/javascript/extractor/lib/typescript/src/main.ts index f0f847fd29a..bce59fc79d3 100644 --- a/javascript/extractor/lib/typescript/src/main.ts +++ b/javascript/extractor/lib/typescript/src/main.ts @@ -126,7 +126,7 @@ function checkCycle(root: any) { function isBlacklistedProperty(k: string) { return k === "parent" || k === "pos" || k === "end" || k === "symbol" || k === "localSymbol" - || k === "flowNode" || k === "returnFlowNode" + || k === "flowNode" || k === "returnFlowNode" || k === "endFlowNode" || k === "nextContainer" || k === "locals" || k === "bindDiagnostics" || k === "bindSuggestionDiagnostics"; } From 1b8335a4e912a9e0cbb7ac3099a20fda0a0ac5c4 Mon Sep 17 00:00:00 2001 From: Asger F Date: Thu, 31 Oct 2019 13:30:39 +0000 Subject: [PATCH 097/148] JS: Update change note --- change-notes/1.23/analysis-javascript.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/change-notes/1.23/analysis-javascript.md b/change-notes/1.23/analysis-javascript.md index 20f9ad1df87..b8f9fca3b3f 100644 --- a/change-notes/1.23/analysis-javascript.md +++ b/change-notes/1.23/analysis-javascript.md @@ -12,7 +12,7 @@ * The call graph has been improved to resolve method calls in more cases. This may produce more security alerts. -* TypeScript 3.6 features are supported. +* TypeScript 3.6 and 3.7 features are now supported. ## New queries From 09a25424773d148a84a7d0657081d59bf758414b Mon Sep 17 00:00:00 2001 From: Asger F Date: Fri, 1 Nov 2019 11:09:25 +0000 Subject: [PATCH 098/148] TS: Update a javadoc comment --- .../extractor/src/com/semmle/ts/ast/PredicateTypeExpr.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/javascript/extractor/src/com/semmle/ts/ast/PredicateTypeExpr.java b/javascript/extractor/src/com/semmle/ts/ast/PredicateTypeExpr.java index 474cb888c0b..27bd3c1718c 100644 --- a/javascript/extractor/src/com/semmle/ts/ast/PredicateTypeExpr.java +++ b/javascript/extractor/src/com/semmle/ts/ast/PredicateTypeExpr.java @@ -4,8 +4,8 @@ import com.semmle.js.ast.SourceLocation; import com.semmle.js.ast.Visitor; /** - * A type of form E is T where E is a parameter name or this and - * T is a type. + * A type of form E is T, asserts E is T or asserts E where E is + * a parameter name or this and T is a type. */ public class PredicateTypeExpr extends TypeExpression { private final ITypeExpression expression; From 9bc45f351ccb66652f436a7ce669e93196c4083c Mon Sep 17 00:00:00 2001 From: Asger F Date: Fri, 1 Nov 2019 16:16:58 +0000 Subject: [PATCH 099/148] TS: Fix typo in stats file --- javascript/ql/src/semmlecode.javascript.dbscheme.stats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/javascript/ql/src/semmlecode.javascript.dbscheme.stats b/javascript/ql/src/semmlecode.javascript.dbscheme.stats index 0c1660bc7c8..f7d36078a5a 100644 --- a/javascript/ql/src/semmlecode.javascript.dbscheme.stats +++ b/javascript/ql/src/semmlecode.javascript.dbscheme.stats @@ -7922,7 +7922,7 @@ 66 -stmt +node 66 From 79dbdac8fa5d6dcc05ff5c82af0003171141be0a Mon Sep 17 00:00:00 2001 From: Asger F Date: Fri, 1 Nov 2019 16:44:30 +0000 Subject: [PATCH 100/148] TS: Support declare modifier for fields --- .../com/semmle/js/ast/DeclarationFlags.java | 18 +++++++++++++++--- .../com/semmle/js/ast/MemberDefinition.java | 5 +++++ .../com/semmle/js/extractor/ASTExtractor.java | 4 ++++ .../js/parser/TypeScriptASTConverter.java | 3 +++ .../ql/src/semmle/javascript/Classes.qll | 6 ++++++ .../ql/src/semmlecode.javascript.dbscheme | 4 ++-- .../TypeScript/DeclareInClass/test.expected | 1 + .../TypeScript/DeclareInClass/test.ql | 5 +++++ .../TypeScript/DeclareInClass/tst.ts | 3 +++ 9 files changed, 44 insertions(+), 5 deletions(-) create mode 100644 javascript/ql/test/library-tests/TypeScript/DeclareInClass/test.expected create mode 100644 javascript/ql/test/library-tests/TypeScript/DeclareInClass/test.ql create mode 100644 javascript/ql/test/library-tests/TypeScript/DeclareInClass/tst.ts diff --git a/javascript/extractor/src/com/semmle/js/ast/DeclarationFlags.java b/javascript/extractor/src/com/semmle/js/ast/DeclarationFlags.java index 2a936bcc3b7..8a6921cb20b 100644 --- a/javascript/extractor/src/com/semmle/js/ast/DeclarationFlags.java +++ b/javascript/extractor/src/com/semmle/js/ast/DeclarationFlags.java @@ -17,9 +17,10 @@ public class DeclarationFlags { public static final int protected_ = 1 << 6; public static final int optional = 1 << 7; public static final int definiteAssignmentAssertion = 1 << 8; + public static final int declareKeyword = 1 << 9; public static final int none = 0; - public static final int numberOfFlags = 9; + public static final int numberOfFlags = 10; public static final List names = Arrays.asList( @@ -31,7 +32,8 @@ public class DeclarationFlags { "private", "protected", "optional", - "definiteAssignmentAssertion"); + "definiteAssignmentAssertion", + "declare"); public static final List relationNames = Arrays.asList( @@ -43,7 +45,8 @@ public class DeclarationFlags { "hasPrivateKeyword", "hasProtectedKeyword", "isOptionalMember", - "hasDefiniteAssignmentAssertion"); + "hasDefiniteAssignmentAssertion", + "hasDeclareKeyword"); public static boolean isComputed(int flags) { return (flags & computed) != 0; @@ -81,6 +84,10 @@ public class DeclarationFlags { return (flags & definiteAssignmentAssertion) != 0; } + public static boolean hasDeclareKeyword(int flags) { + return (flags & declareKeyword) != 0; + } + /** Returns a mask with the computed bit set to the value of enable. */ public static int getComputed(boolean enable) { return enable ? computed : 0; @@ -128,6 +135,11 @@ public class DeclarationFlags { return enable ? definiteAssignmentAssertion : 0; } + /** Returns a mask with the declare keyword bit set to the value of enable. */ + public static int getDeclareKeyword(boolean enable) { + return enable ? declareKeyword : 0; + } + /** Returns true if the nth bit is set in flags. */ public static boolean hasNthFlag(int flags, int n) { return (flags & (1 << n)) != 0; diff --git a/javascript/extractor/src/com/semmle/js/ast/MemberDefinition.java b/javascript/extractor/src/com/semmle/js/ast/MemberDefinition.java index bfd0f62462f..c04b79fa366 100644 --- a/javascript/extractor/src/com/semmle/js/ast/MemberDefinition.java +++ b/javascript/extractor/src/com/semmle/js/ast/MemberDefinition.java @@ -75,6 +75,11 @@ public abstract class MemberDefinition extends Node { return DeclarationFlags.isReadonly(flags); } + /** Returns true if this has the declare modifier. */ + public boolean hasDeclareKeyword() { + return DeclarationFlags.hasDeclareKeyword(flags); + } + /** * Returns the expression denoting the name of the member, or {@code null} if this is a * call/construct signature. diff --git a/javascript/extractor/src/com/semmle/js/extractor/ASTExtractor.java b/javascript/extractor/src/com/semmle/js/extractor/ASTExtractor.java index d3571fc5a57..af15c7ad51c 100644 --- a/javascript/extractor/src/com/semmle/js/extractor/ASTExtractor.java +++ b/javascript/extractor/src/com/semmle/js/extractor/ASTExtractor.java @@ -1410,6 +1410,10 @@ public class ASTExtractor { } } + if (nd.hasDeclareKeyword()) { + trapwriter.addTuple("hasDeclareKeyword", methkey); + } + return methkey; } diff --git a/javascript/extractor/src/com/semmle/js/parser/TypeScriptASTConverter.java b/javascript/extractor/src/com/semmle/js/parser/TypeScriptASTConverter.java index 7f826a7dc1d..4912b5577f8 100644 --- a/javascript/extractor/src/com/semmle/js/parser/TypeScriptASTConverter.java +++ b/javascript/extractor/src/com/semmle/js/parser/TypeScriptASTConverter.java @@ -2009,6 +2009,9 @@ public class TypeScriptASTConverter { if (node.get("exclamationToken") != null) { flags |= DeclarationFlags.definiteAssignmentAssertion; } + if (hasModifier(node, "DeclareKeyword")) { + flags |= DeclarationFlags.declareKeyword; + } FieldDefinition fieldDefinition = new FieldDefinition( loc, diff --git a/javascript/ql/src/semmle/javascript/Classes.qll b/javascript/ql/src/semmle/javascript/Classes.qll index 5cf94d409f5..793ff5a92e4 100644 --- a/javascript/ql/src/semmle/javascript/Classes.qll +++ b/javascript/ql/src/semmle/javascript/Classes.qll @@ -1059,6 +1059,12 @@ class FieldDeclaration extends MemberDeclaration, @field { /** Holds if this is a TypeScript field marked as definitely assigned with the `!` operator. */ predicate hasDefiniteAssignmentAssertion() { hasDefiniteAssignmentAssertion(this) } + + override predicate isAmbient() { + hasDeclareKeyword(this) + or + getParent().isAmbient() + } } /** diff --git a/javascript/ql/src/semmlecode.javascript.dbscheme b/javascript/ql/src/semmlecode.javascript.dbscheme index e0bd730ef98..5a5adbf98dc 100644 --- a/javascript/ql/src/semmlecode.javascript.dbscheme +++ b/javascript/ql/src/semmlecode.javascript.dbscheme @@ -200,8 +200,8 @@ case @stmt.kind of isInstantiated(unique int decl: @namespacedeclaration ref); -@declarablestmt = @declstmt | @namespacedeclaration | @classdeclstmt | @functiondeclstmt | @enumdeclaration | @externalmoduledeclaration | @globalaugmentationdeclaration; -hasDeclareKeyword(unique int stmt: @declarablestmt ref); +@declarablenode = @declstmt | @namespacedeclaration | @classdeclstmt | @functiondeclstmt | @enumdeclaration | @externalmoduledeclaration | @globalaugmentationdeclaration | @field; +hasDeclareKeyword(unique int stmt: @declarablenode ref); isForAwaitOf(unique int forof: @forofstmt ref); diff --git a/javascript/ql/test/library-tests/TypeScript/DeclareInClass/test.expected b/javascript/ql/test/library-tests/TypeScript/DeclareInClass/test.expected new file mode 100644 index 00000000000..17658f9c17d --- /dev/null +++ b/javascript/ql/test/library-tests/TypeScript/DeclareInClass/test.expected @@ -0,0 +1 @@ +| tst.ts:2:5:2:22 | declare x: number; | true | diff --git a/javascript/ql/test/library-tests/TypeScript/DeclareInClass/test.ql b/javascript/ql/test/library-tests/TypeScript/DeclareInClass/test.ql new file mode 100644 index 00000000000..b4a528dc9b1 --- /dev/null +++ b/javascript/ql/test/library-tests/TypeScript/DeclareInClass/test.ql @@ -0,0 +1,5 @@ +import javascript + +from FieldDeclaration f, boolean ambient +where if f.isAmbient() then ambient = true else ambient = false +select f, ambient diff --git a/javascript/ql/test/library-tests/TypeScript/DeclareInClass/tst.ts b/javascript/ql/test/library-tests/TypeScript/DeclareInClass/tst.ts new file mode 100644 index 00000000000..0952c23532d --- /dev/null +++ b/javascript/ql/test/library-tests/TypeScript/DeclareInClass/tst.ts @@ -0,0 +1,3 @@ +class C { + declare x: number; +} From 2d7443ef986c66d354c7d4ed13d9fd842922abb6 Mon Sep 17 00:00:00 2001 From: Asger F Date: Mon, 4 Nov 2019 07:55:20 +0000 Subject: [PATCH 101/148] TS: Add upgrade script --- .../old.dbscheme | 1173 ++++++++++++++++ .../semmlecode.javascript.dbscheme | 1175 +++++++++++++++++ .../upgrade.properties | 2 + 3 files changed, 2350 insertions(+) create mode 100644 javascript/upgrades/874ebfcd4d2fd346c10ebec145466d9071d78606/old.dbscheme create mode 100644 javascript/upgrades/874ebfcd4d2fd346c10ebec145466d9071d78606/semmlecode.javascript.dbscheme create mode 100644 javascript/upgrades/874ebfcd4d2fd346c10ebec145466d9071d78606/upgrade.properties diff --git a/javascript/upgrades/874ebfcd4d2fd346c10ebec145466d9071d78606/old.dbscheme b/javascript/upgrades/874ebfcd4d2fd346c10ebec145466d9071d78606/old.dbscheme new file mode 100644 index 00000000000..874ebfcd4d2 --- /dev/null +++ b/javascript/upgrades/874ebfcd4d2fd346c10ebec145466d9071d78606/old.dbscheme @@ -0,0 +1,1173 @@ +/*** Standard fragments ***/ + +/** Files and folders **/ + +@location = @location_default; + +locations_default(unique int id: @location_default, + int file: @file ref, + int beginLine: int ref, + int beginColumn: int ref, + int endLine: int ref, + int endColumn: int ref + ); + +@sourceline = @locatable; + +numlines(int element_id: @sourceline ref, + int num_lines: int ref, + int num_code: int ref, + int num_comment: int ref + ); + + +/* + fromSource(0) = unknown, + fromSource(1) = from source, + fromSource(2) = from library +*/ +files(unique int id: @file, + varchar(900) name: string ref, + varchar(900) simple: string ref, + varchar(900) ext: string ref, + int fromSource: int ref); + +folders(unique int id: @folder, + varchar(900) name: string ref, + varchar(900) simple: string ref); + + +@container = @folder | @file ; + + +containerparent(int parent: @container ref, + unique int child: @container ref); + +/** Duplicate code **/ + +duplicateCode( + unique int id : @duplication, + varchar(900) relativePath : string ref, + int equivClass : int ref); + +similarCode( + unique int id : @similarity, + varchar(900) relativePath : string ref, + int equivClass : int ref); + +@duplication_or_similarity = @duplication | @similarity; + +tokens( + int id : @duplication_or_similarity ref, + int offset : int ref, + int beginLine : int ref, + int beginColumn : int ref, + int endLine : int ref, + int endColumn : int ref); + +/** External data **/ + +externalData( + int id : @externalDataElement, + varchar(900) path : string ref, + int column: int ref, + varchar(900) value : string ref +); + +snapshotDate(unique date snapshotDate : date ref); + +sourceLocationPrefix(varchar(900) prefix : string ref); + +/** Version control data **/ + +svnentries( + int id : @svnentry, + varchar(500) revision : string ref, + varchar(500) author : string ref, + date revisionDate : date ref, + int changeSize : int ref +); + +svnaffectedfiles( + int id : @svnentry ref, + int file : @file ref, + varchar(500) action : string ref +); + +svnentrymsg( + int id : @svnentry ref, + varchar(500) message : string ref +); + +svnchurn( + int commit : @svnentry ref, + int file : @file ref, + int addedLines : int ref, + int deletedLines : int ref +); + + +/*** JavaScript-specific part ***/ + +filetype( + int file: @file ref, + string filetype: string ref +) + +// top-level code fragments +toplevels (unique int id: @toplevel, + int kind: int ref); + +isExterns (int toplevel: @toplevel ref); + +case @toplevel.kind of + 0 = @script +| 1 = @inline_script +| 2 = @event_handler +| 3 = @javascript_url; + +isModule (int tl: @toplevel ref); +isNodejs (int tl: @toplevel ref); +isES2015Module (int tl: @toplevel ref); +isClosureModule (int tl: @toplevel ref); + +// statements +#keyset[parent, idx] +stmts (unique int id: @stmt, + int kind: int ref, + int parent: @stmtparent ref, + int idx: int ref, + varchar(900) tostring: string ref); + +stmtContainers (unique int stmt: @stmt ref, + int container: @stmt_container ref); + +jumpTargets (unique int jump: @stmt ref, + int target: @stmt ref); + +@stmtparent = @stmt | @toplevel | @functionexpr | @arrowfunctionexpr; +@stmt_container = @toplevel | @function | @namespacedeclaration | @externalmoduledeclaration | @globalaugmentationdeclaration; + +case @stmt.kind of + 0 = @emptystmt +| 1 = @blockstmt +| 2 = @exprstmt +| 3 = @ifstmt +| 4 = @labeledstmt +| 5 = @breakstmt +| 6 = @continuestmt +| 7 = @withstmt +| 8 = @switchstmt +| 9 = @returnstmt +| 10 = @throwstmt +| 11 = @trystmt +| 12 = @whilestmt +| 13 = @dowhilestmt +| 14 = @forstmt +| 15 = @forinstmt +| 16 = @debuggerstmt +| 17 = @functiondeclstmt +| 18 = @vardeclstmt +| 19 = @case +| 20 = @catchclause +| 21 = @forofstmt +| 22 = @constdeclstmt +| 23 = @letstmt +| 24 = @legacy_letstmt +| 25 = @foreachstmt +| 26 = @classdeclstmt +| 27 = @importdeclaration +| 28 = @exportalldeclaration +| 29 = @exportdefaultdeclaration +| 30 = @exportnameddeclaration +| 31 = @namespacedeclaration +| 32 = @importequalsdeclaration +| 33 = @exportassigndeclaration +| 34 = @interfacedeclaration +| 35 = @typealiasdeclaration +| 36 = @enumdeclaration +| 37 = @externalmoduledeclaration +| 38 = @exportasnamespacedeclaration +| 39 = @globalaugmentationdeclaration +; + +@declstmt = @vardeclstmt | @constdeclstmt | @letstmt | @legacy_letstmt; + +@exportdeclaration = @exportalldeclaration | @exportdefaultdeclaration | @exportnameddeclaration; + +@namespacedefinition = @namespacedeclaration | @enumdeclaration; +@typedefinition = @classdefinition | @interfacedeclaration | @enumdeclaration | @typealiasdeclaration | @enum_member; + +isInstantiated(unique int decl: @namespacedeclaration ref); + +@declarablestmt = @declstmt | @namespacedeclaration | @classdeclstmt | @functiondeclstmt | @enumdeclaration | @externalmoduledeclaration | @globalaugmentationdeclaration; +hasDeclareKeyword(unique int stmt: @declarablestmt ref); + +isForAwaitOf(unique int forof: @forofstmt ref); + +// expressions +#keyset[parent, idx] +exprs (unique int id: @expr, + int kind: int ref, + int parent: @exprparent ref, + int idx: int ref, + varchar(900) tostring: string ref); + +literals (varchar(900) value: string ref, + varchar(900) raw: string ref, + unique int expr: @exprortype ref); + +enclosingStmt (unique int expr: @exprortype ref, + int stmt: @stmt ref); + +exprContainers (unique int expr: @exprortype ref, + int container: @stmt_container ref); + +arraySize (unique int ae: @arraylike ref, + int sz: int ref); + +isDelegating (int yield: @yieldexpr ref); + +@exprorstmt = @expr | @stmt; +@exprortype = @expr | @typeexpr; +@exprparent = @exprorstmt | @property | @functiontypeexpr; +@arraylike = @arrayexpr | @arraypattern; +@type_annotation = @typeexpr | @jsdoc_type_expr; + +case @expr.kind of + 0 = @label +| 1 = @nullliteral +| 2 = @booleanliteral +| 3 = @numberliteral +| 4 = @stringliteral +| 5 = @regexpliteral +| 6 = @thisexpr +| 7 = @arrayexpr +| 8 = @objexpr +| 9 = @functionexpr +| 10 = @seqexpr +| 11 = @conditionalexpr +| 12 = @newexpr +| 13 = @callexpr +| 14 = @dotexpr +| 15 = @indexexpr +| 16 = @negexpr +| 17 = @plusexpr +| 18 = @lognotexpr +| 19 = @bitnotexpr +| 20 = @typeofexpr +| 21 = @voidexpr +| 22 = @deleteexpr +| 23 = @eqexpr +| 24 = @neqexpr +| 25 = @eqqexpr +| 26 = @neqqexpr +| 27 = @ltexpr +| 28 = @leexpr +| 29 = @gtexpr +| 30 = @geexpr +| 31 = @lshiftexpr +| 32 = @rshiftexpr +| 33 = @urshiftexpr +| 34 = @addexpr +| 35 = @subexpr +| 36 = @mulexpr +| 37 = @divexpr +| 38 = @modexpr +| 39 = @bitorexpr +| 40 = @xorexpr +| 41 = @bitandexpr +| 42 = @inexpr +| 43 = @instanceofexpr +| 44 = @logandexpr +| 45 = @logorexpr +| 47 = @assignexpr +| 48 = @assignaddexpr +| 49 = @assignsubexpr +| 50 = @assignmulexpr +| 51 = @assigndivexpr +| 52 = @assignmodexpr +| 53 = @assignlshiftexpr +| 54 = @assignrshiftexpr +| 55 = @assignurshiftexpr +| 56 = @assignorexpr +| 57 = @assignxorexpr +| 58 = @assignandexpr +| 59 = @preincexpr +| 60 = @postincexpr +| 61 = @predecexpr +| 62 = @postdecexpr +| 63 = @parexpr +| 64 = @vardeclarator +| 65 = @arrowfunctionexpr +| 66 = @spreadelement +| 67 = @arraypattern +| 68 = @objectpattern +| 69 = @yieldexpr +| 70 = @taggedtemplateexpr +| 71 = @templateliteral +| 72 = @templateelement +| 73 = @arraycomprehensionexpr +| 74 = @generatorexpr +| 75 = @forincomprehensionblock +| 76 = @forofcomprehensionblock +| 77 = @legacy_letexpr +| 78 = @vardecl +| 79 = @proper_varaccess +| 80 = @classexpr +| 81 = @superexpr +| 82 = @newtargetexpr +| 83 = @namedimportspecifier +| 84 = @importdefaultspecifier +| 85 = @importnamespacespecifier +| 86 = @namedexportspecifier +| 87 = @expexpr +| 88 = @assignexpexpr +| 89 = @jsxelement +| 90 = @jsxqualifiedname +| 91 = @jsxemptyexpr +| 92 = @awaitexpr +| 93 = @functionsentexpr +| 94 = @decorator +| 95 = @exportdefaultspecifier +| 96 = @exportnamespacespecifier +| 97 = @bindexpr +| 98 = @externalmodulereference +| 99 = @dynamicimport +| 100 = @expressionwithtypearguments +| 101 = @prefixtypeassertion +| 102 = @astypeassertion +| 103 = @export_varaccess +| 104 = @decorator_list +| 105 = @non_null_assertion +| 106 = @bigintliteral +| 107 = @nullishcoalescingexpr +| 108 = @e4x_xml_anyname +| 109 = @e4x_xml_static_attribute_selector +| 110 = @e4x_xml_dynamic_attribute_selector +| 111 = @e4x_xml_filter_expression +| 112 = @e4x_xml_static_qualident +| 113 = @e4x_xml_dynamic_qualident +| 114 = @e4x_xml_dotdotexpr +| 115 = @importmetaexpr +; + +@varaccess = @proper_varaccess | @export_varaccess; +@varref = @vardecl | @varaccess; + +@identifier = @label | @varref | @typeidentifier; + +@literal = @nullliteral | @booleanliteral | @numberliteral | @stringliteral | @regexpliteral | @bigintliteral; + +@propaccess = @dotexpr | @indexexpr; + +@invokeexpr = @newexpr | @callexpr; + +@unaryexpr = @negexpr | @plusexpr | @lognotexpr | @bitnotexpr | @typeofexpr | @voidexpr | @deleteexpr | @spreadelement; + +@equalitytest = @eqexpr | @neqexpr | @eqqexpr | @neqqexpr; + +@comparison = @equalitytest | @ltexpr | @leexpr | @gtexpr | @geexpr; + +@binaryexpr = @comparison | @lshiftexpr | @rshiftexpr | @urshiftexpr | @addexpr | @subexpr | @mulexpr | @divexpr | @modexpr | @expexpr | @bitorexpr | @xorexpr | @bitandexpr | @inexpr | @instanceofexpr | @logandexpr | @logorexpr | @nullishcoalescingexpr; + +@assignment = @assignexpr | @assignaddexpr | @assignsubexpr | @assignmulexpr | @assigndivexpr | @assignmodexpr | @assignexpexpr | @assignlshiftexpr | @assignrshiftexpr | @assignurshiftexpr | @assignorexpr | @assignxorexpr | @assignandexpr; + +@updateexpr = @preincexpr | @postincexpr | @predecexpr | @postdecexpr; + +@pattern = @varref | @arraypattern | @objectpattern; + +@comprehensionexpr = @arraycomprehensionexpr | @generatorexpr; + +@comprehensionblock = @forincomprehensionblock | @forofcomprehensionblock; + +@importspecifier = @namedimportspecifier | @importdefaultspecifier | @importnamespacespecifier; + +@exportspecifier = @namedexportspecifier | @exportdefaultspecifier | @exportnamespacespecifier; + +@typeassertion = @astypeassertion | @prefixtypeassertion; + +@classdefinition = @classdeclstmt | @classexpr; +@interfacedefinition = @interfacedeclaration | @interfacetypeexpr; +@classorinterface = @classdefinition | @interfacedefinition; + +@lexical_decl = @vardecl | @typedecl; +@lexical_access = @varaccess | @localtypeaccess | @localvartypeaccess | @localnamespaceaccess; +@lexical_ref = @lexical_decl | @lexical_access; + +@e4x_xml_attribute_selector = @e4x_xml_static_attribute_selector | @e4x_xml_dynamic_attribute_selector; +@e4x_xml_qualident = @e4x_xml_static_qualident | @e4x_xml_dynamic_qualident; + +// scopes +scopes (unique int id: @scope, + int kind: int ref); + +case @scope.kind of + 0 = @globalscope +| 1 = @functionscope +| 2 = @catchscope +| 3 = @modulescope +| 4 = @blockscope +| 5 = @forscope +| 6 = @forinscope // for-of scopes work the same as for-in scopes +| 7 = @comprehensionblockscope +| 8 = @classexprscope +| 9 = @namespacescope +| 10 = @classdeclscope +| 11 = @interfacescope +| 12 = @typealiasscope +| 13 = @mappedtypescope +| 14 = @enumscope +| 15 = @externalmodulescope +| 16 = @conditionaltypescope; + +scopenodes (unique int node: @ast_node ref, + int scope: @scope ref); + +scopenesting (unique int inner: @scope ref, + int outer: @scope ref); + +// functions +@function = @functiondeclstmt | @functionexpr | @arrowfunctionexpr; + +@parameterized = @function | @catchclause; +@type_parameterized = @function | @classorinterface | @typealiasdeclaration | @mappedtypeexpr | @infertypeexpr; + +isGenerator (int fun: @function ref); +hasRestParameter (int fun: @function ref); +isAsync (int fun: @function ref); + +// variables and lexically scoped type names +#keyset[scope, name] +variables (unique int id: @variable, + varchar(900) name: string ref, + int scope: @scope ref); + +#keyset[scope, name] +local_type_names (unique int id: @local_type_name, + varchar(900) name: string ref, + int scope: @scope ref); + +#keyset[scope, name] +local_namespace_names (unique int id: @local_namespace_name, + varchar(900) name: string ref, + int scope: @scope ref); + +isArgumentsObject (int id: @variable ref); + +@lexical_name = @variable | @local_type_name | @local_namespace_name; + +@bind_id = @varaccess | @localvartypeaccess; +bind (unique int id: @bind_id ref, + int decl: @variable ref); + +decl (unique int id: @vardecl ref, + int decl: @variable ref); + +@typebind_id = @localtypeaccess | @export_varaccess; +typebind (unique int id: @typebind_id ref, + int decl: @local_type_name ref); + +@typedecl_id = @typedecl | @vardecl; +typedecl (unique int id: @typedecl_id ref, + int decl: @local_type_name ref); + +namespacedecl (unique int id: @vardecl ref, + int decl: @local_namespace_name ref); + +@namespacebind_id = @localnamespaceaccess | @export_varaccess; +namespacebind (unique int id: @namespacebind_id ref, + int decl: @local_namespace_name ref); + + +// properties in object literals, property patterns in object patterns, and method declarations in classes +#keyset[parent, index] +properties (unique int id: @property, + int parent: @property_parent ref, + int index: int ref, + int kind: int ref, + varchar(900) tostring: string ref); + +case @property.kind of + 0 = @value_property +| 1 = @property_getter +| 2 = @property_setter +| 3 = @jsx_attribute +| 4 = @function_call_signature +| 5 = @constructor_call_signature +| 6 = @index_signature +| 7 = @enum_member +| 8 = @proper_field +| 9 = @parameter_field +; + +@property_parent = @objexpr | @objectpattern | @classdefinition | @jsxelement | @interfacedefinition | @enumdeclaration; +@property_accessor = @property_getter | @property_setter; +@call_signature = @function_call_signature | @constructor_call_signature; +@field = @proper_field | @parameter_field; +@field_or_vardeclarator = @field | @vardeclarator; + +isComputed (int id: @property ref); +isMethod (int id: @property ref); +isStatic (int id: @property ref); +isAbstractMember (int id: @property ref); +isConstEnum (int id: @enumdeclaration ref); +isAbstractClass (int id: @classdeclstmt ref); + +hasPublicKeyword (int id: @property ref); +hasPrivateKeyword (int id: @property ref); +hasProtectedKeyword (int id: @property ref); +hasReadonlyKeyword (int id: @property ref); +isOptionalMember (int id: @property ref); +hasDefiniteAssignmentAssertion (int id: @field_or_vardeclarator ref); + +#keyset[constructor, param_index] +parameter_fields( + unique int field: @parameter_field ref, + int constructor: @functionexpr ref, + int param_index: int ref +); + +// types +#keyset[parent, idx] +typeexprs ( + unique int id: @typeexpr, + int kind: int ref, + int parent: @typeexpr_parent ref, + int idx: int ref, + varchar(900) tostring: string ref +); + +case @typeexpr.kind of + 0 = @localtypeaccess +| 1 = @typedecl +| 2 = @keywordtypeexpr +| 3 = @stringliteraltypeexpr +| 4 = @numberliteraltypeexpr +| 5 = @booleanliteraltypeexpr +| 6 = @arraytypeexpr +| 7 = @uniontypeexpr +| 8 = @indexedaccesstypeexpr +| 9 = @intersectiontypeexpr +| 10 = @parenthesizedtypeexpr +| 11 = @tupletypeexpr +| 12 = @keyoftypeexpr +| 13 = @qualifiedtypeaccess +| 14 = @generictypeexpr +| 15 = @typelabel +| 16 = @typeoftypeexpr +| 17 = @localvartypeaccess +| 18 = @qualifiedvartypeaccess +| 19 = @thisvartypeaccess +| 20 = @istypeexpr +| 21 = @interfacetypeexpr +| 22 = @typeparameter +| 23 = @plainfunctiontypeexpr +| 24 = @constructortypeexpr +| 25 = @localnamespaceaccess +| 26 = @qualifiednamespaceaccess +| 27 = @mappedtypeexpr +| 28 = @conditionaltypeexpr +| 29 = @infertypeexpr +| 30 = @importtypeaccess +| 31 = @importnamespaceaccess +| 32 = @importvartypeaccess +| 33 = @optionaltypeexpr +| 34 = @resttypeexpr +| 35 = @bigintliteraltypeexpr +| 36 = @readonlytypeexpr +; + +@typeref = @typeaccess | @typedecl; +@typeidentifier = @typedecl | @localtypeaccess | @typelabel | @localvartypeaccess | @localnamespaceaccess; +@typeexpr_parent = @expr | @stmt | @property | @typeexpr; +@literaltypeexpr = @stringliteraltypeexpr | @numberliteraltypeexpr | @booleanliteraltypeexpr | @bigintliteraltypeexpr; +@typeaccess = @localtypeaccess | @qualifiedtypeaccess | @importtypeaccess; +@vartypeaccess = @localvartypeaccess | @qualifiedvartypeaccess | @thisvartypeaccess | @importvartypeaccess; +@namespaceaccess = @localnamespaceaccess | @qualifiednamespaceaccess | @importnamespaceaccess; +@importtypeexpr = @importtypeaccess | @importnamespaceaccess | @importvartypeaccess; + +@functiontypeexpr = @plainfunctiontypeexpr | @constructortypeexpr; + +// types +types ( + unique int id: @type, + int kind: int ref, + varchar(900) tostring: string ref +); + +#keyset[parent, idx] +type_child ( + int child: @type ref, + int parent: @type ref, + int idx: int ref +); + +case @type.kind of + 0 = @anytype +| 1 = @stringtype +| 2 = @numbertype +| 3 = @uniontype +| 4 = @truetype +| 5 = @falsetype +| 6 = @typereference +| 7 = @objecttype +| 8 = @canonicaltypevariabletype +| 9 = @typeoftype +| 10 = @voidtype +| 11 = @undefinedtype +| 12 = @nulltype +| 13 = @nevertype +| 14 = @plainsymboltype +| 15 = @uniquesymboltype +| 16 = @objectkeywordtype +| 17 = @intersectiontype +| 18 = @tupletype +| 19 = @lexicaltypevariabletype +| 20 = @thistype +| 21 = @numberliteraltype +| 22 = @stringliteraltype +| 23 = @unknowntype +| 24 = @biginttype +| 25 = @bigintliteraltype +; + +@booleanliteraltype = @truetype | @falsetype; +@symboltype = @plainsymboltype | @uniquesymboltype; +@unionorintersectiontype = @uniontype | @intersectiontype; +@typevariabletype = @canonicaltypevariabletype | @lexicaltypevariabletype; + +@typed_ast_node = @expr | @typeexpr | @function; +ast_node_type( + unique int node: @typed_ast_node ref, + int typ: @type ref); + +declared_function_signature( + unique int node: @function ref, + int sig: @signature_type ref +); + +invoke_expr_signature( + unique int node: @invokeexpr ref, + int sig: @signature_type ref +); + +invoke_expr_overload_index( + unique int node: @invokeexpr ref, + int index: int ref +); + +symbols ( + unique int id: @symbol, + int kind: int ref, + varchar(900) name: string ref +); + +symbol_parent ( + unique int symbol: @symbol ref, + int parent: @symbol ref +); + +symbol_module ( + int symbol: @symbol ref, + varchar(900) moduleName: string ref +); + +symbol_global ( + int symbol: @symbol ref, + varchar(900) globalName: string ref +); + +case @symbol.kind of + 0 = @root_symbol +| 1 = @member_symbol +| 2 = @other_symbol +; + +@type_with_symbol = @typereference | @typevariabletype | @typeoftype | @uniquesymboltype; +@ast_node_with_symbol = @typedefinition | @namespacedefinition | @toplevel | @typeaccess | @namespaceaccess | @vardecl | @function | @invokeexpr; + +ast_node_symbol( + unique int node: @ast_node_with_symbol ref, + int symbol: @symbol ref); + +type_symbol( + unique int typ: @type_with_symbol ref, + int symbol: @symbol ref); + +#keyset[typ, name] +type_property( + int typ: @type ref, + varchar(900) name: string ref, + int propertyType: @type ref); + +@literaltype = @stringliteraltype | @numberliteraltype | @booleanliteraltype | @bigintliteraltype; +@type_with_literal_value = @stringliteraltype | @numberliteraltype | @bigintliteraltype; +type_literal_value( + unique int typ: @type_with_literal_value ref, + varchar(900) value: string ref); + +signature_types ( + unique int id: @signature_type, + int kind: int ref, + varchar(900) tostring: string ref, + int type_parameters: int ref, + int required_params: int ref +); + +case @signature_type.kind of + 0 = @function_signature_type +| 1 = @constructor_signature_type +; + +#keyset[typ, kind, index] +type_contains_signature ( + int typ: @type ref, + int kind: int ref, // constructor/call/index + int index: int ref, // ordering of overloaded signatures + int sig: @signature_type ref +); + +#keyset[parent, index] +signature_contains_type ( + int child: @type ref, + int parent: @signature_type ref, + int index: int ref +); + +#keyset[sig, index] +signature_parameter_name ( + int sig: @signature_type ref, + int index: int ref, + varchar(900) name: string ref +); + +number_index_type ( + unique int baseType: @type ref, + int propertyType: @type ref +); + +string_index_type ( + unique int baseType: @type ref, + int propertyType: @type ref +); + +base_type_names( + int typeName: @symbol ref, + int baseTypeName: @symbol ref +); + +self_types( + int typeName: @symbol ref, + int selfType: @typereference ref +); + +tuple_type_min_length( + unique int typ: @type ref, + int minLength: int ref +); + +tuple_type_rest( + unique int typ: @type ref +); + +// comments +comments (unique int id: @comment, + int kind: int ref, + int toplevel: @toplevel ref, + varchar(900) text: string ref, + varchar(900) tostring: string ref); + +case @comment.kind of + 0 = @slashslashcomment +| 1 = @slashstarcomment +| 2 = @doccomment +| 3 = @htmlcommentstart +| 4 = @htmlcommentend; + +@htmlcomment = @htmlcommentstart | @htmlcommentend; +@linecomment = @slashslashcomment | @htmlcomment; +@blockcomment = @slashstarcomment | @doccomment; + +// source lines +lines (unique int id: @line, + int toplevel: @toplevel ref, + varchar(900) text: string ref, + varchar(2) terminator: string ref); +indentation (int file: @file ref, + int lineno: int ref, + varchar(1) indentChar: string ref, + int indentDepth: int ref); + +// JavaScript parse errors +jsParseErrors (unique int id: @js_parse_error, + int toplevel: @toplevel ref, + varchar(900) message: string ref, + varchar(900) line: string ref); + +// regular expressions +#keyset[parent, idx] +regexpterm (unique int id: @regexpterm, + int kind: int ref, + int parent: @regexpparent ref, + int idx: int ref, + varchar(900) tostring: string ref); + +@regexpparent = @regexpterm | @regexpliteral; + +case @regexpterm.kind of + 0 = @regexp_alt +| 1 = @regexp_seq +| 2 = @regexp_caret +| 3 = @regexp_dollar +| 4 = @regexp_wordboundary +| 5 = @regexp_nonwordboundary +| 6 = @regexp_positive_lookahead +| 7 = @regexp_negative_lookahead +| 8 = @regexp_star +| 9 = @regexp_plus +| 10 = @regexp_opt +| 11 = @regexp_range +| 12 = @regexp_dot +| 13 = @regexp_group +| 14 = @regexp_normal_char +| 15 = @regexp_hex_escape +| 16 = @regexp_unicode_escape +| 17 = @regexp_dec_escape +| 18 = @regexp_oct_escape +| 19 = @regexp_ctrl_escape +| 20 = @regexp_char_class_escape +| 21 = @regexp_id_escape +| 22 = @regexp_backref +| 23 = @regexp_char_class +| 24 = @regexp_char_range +| 25 = @regexp_positive_lookbehind +| 26 = @regexp_negative_lookbehind +| 27 = @regexp_unicode_property_escape; + +regexpParseErrors (unique int id: @regexp_parse_error, + int regexp: @regexpterm ref, + varchar(900) message: string ref); + +@regexp_quantifier = @regexp_star | @regexp_plus | @regexp_opt | @regexp_range; +@regexp_escape = @regexp_char_escape | @regexp_char_class_escape | @regexp_unicode_property_escape; +@regexp_char_escape = @regexp_hex_escape | @regexp_unicode_escape | @regexp_dec_escape | @regexp_oct_escape | @regexp_ctrl_escape | @regexp_id_escape; +@regexp_constant = @regexp_normal_char | @regexp_char_escape; +@regexp_lookahead = @regexp_positive_lookahead | @regexp_negative_lookahead; +@regexp_lookbehind = @regexp_positive_lookbehind | @regexp_negative_lookbehind; +@regexp_subpattern = @regexp_lookahead | @regexp_lookbehind; + +isGreedy (int id: @regexp_quantifier ref); +rangeQuantifierLowerBound (unique int id: @regexp_range ref, int lo: int ref); +rangeQuantifierUpperBound (unique int id: @regexp_range ref, int hi: int ref); +isCapture (unique int id: @regexp_group ref, int number: int ref); +isNamedCapture (unique int id: @regexp_group ref, string name: string ref); +isInverted (int id: @regexp_char_class ref); +regexpConstValue (unique int id: @regexp_constant ref, varchar(1) value: string ref); +charClassEscape (unique int id: @regexp_char_class_escape ref, varchar(1) value: string ref); +backref (unique int id: @regexp_backref ref, int value: int ref); +namedBackref (unique int id: @regexp_backref ref, string name: string ref); +unicodePropertyEscapeName (unique int id: @regexp_unicode_property_escape ref, string name: string ref); +unicodePropertyEscapeValue (unique int id: @regexp_unicode_property_escape ref, string value: string ref); + +// tokens +#keyset[toplevel, idx] +tokeninfo (unique int id: @token, + int kind: int ref, + int toplevel: @toplevel ref, + int idx: int ref, + varchar(900) value: string ref); + +case @token.kind of + 0 = @token_eof +| 1 = @token_null_literal +| 2 = @token_boolean_literal +| 3 = @token_numeric_literal +| 4 = @token_string_literal +| 5 = @token_regular_expression +| 6 = @token_identifier +| 7 = @token_keyword +| 8 = @token_punctuator; + +// associate comments with the token immediately following them (which may be EOF) +next_token (int comment: @comment ref, int token: @token ref); + +// JSON +#keyset[parent, idx] +json (unique int id: @json_value, + int kind: int ref, + int parent: @json_parent ref, + int idx: int ref, + varchar(900) tostring: string ref); + +json_literals (varchar(900) value: string ref, + varchar(900) raw: string ref, + unique int expr: @json_value ref); + +json_properties (int obj: @json_object ref, + varchar(900) property: string ref, + int value: @json_value ref); + +json_errors (unique int id: @json_parse_error, + varchar(900) message: string ref); + +json_locations(unique int locatable: @json_locatable ref, + int location: @location_default ref); + +case @json_value.kind of + 0 = @json_null +| 1 = @json_boolean +| 2 = @json_number +| 3 = @json_string +| 4 = @json_array +| 5 = @json_object; + +@json_parent = @json_object | @json_array | @file; + +@json_locatable = @json_value | @json_parse_error; + +// locations +@ast_node = @toplevel | @stmt | @expr | @property | @typeexpr; + +@locatable = @file + | @ast_node + | @comment + | @line + | @js_parse_error | @regexp_parse_error + | @regexpterm + | @json_locatable + | @token + | @cfg_node + | @jsdoc | @jsdoc_type_expr | @jsdoc_tag + | @yaml_locatable + | @xmllocatable + | @configLocatable; + +hasLocation (unique int locatable: @locatable ref, + int location: @location ref); + +// CFG +entry_cfg_node (unique int id: @entry_node, int container: @stmt_container ref); +exit_cfg_node (unique int id: @exit_node, int container: @stmt_container ref); +guard_node (unique int id: @guard_node, int kind: int ref, int test: @expr ref); +case @guard_node.kind of + 0 = @falsy_guard +| 1 = @truthy_guard; +@condition_guard = @falsy_guard | @truthy_guard; + +@synthetic_cfg_node = @entry_node | @exit_node | @guard_node; +@cfg_node = @synthetic_cfg_node | @exprparent; + +successor (int pred: @cfg_node ref, int succ: @cfg_node ref); + +// JSDoc comments +jsdoc (unique int id: @jsdoc, varchar(900) description: string ref, int comment: @comment ref); +#keyset[parent, idx] +jsdoc_tags (unique int id: @jsdoc_tag, varchar(900) title: string ref, + int parent: @jsdoc ref, int idx: int ref, varchar(900) tostring: string ref); +jsdoc_tag_descriptions (unique int tag: @jsdoc_tag ref, varchar(900) text: string ref); +jsdoc_tag_names (unique int tag: @jsdoc_tag ref, varchar(900) text: string ref); + +#keyset[parent, idx] +jsdoc_type_exprs (unique int id: @jsdoc_type_expr, + int kind: int ref, + int parent: @jsdoc_type_expr_parent ref, + int idx: int ref, + varchar(900) tostring: string ref); +case @jsdoc_type_expr.kind of + 0 = @jsdoc_any_type_expr +| 1 = @jsdoc_null_type_expr +| 2 = @jsdoc_undefined_type_expr +| 3 = @jsdoc_unknown_type_expr +| 4 = @jsdoc_void_type_expr +| 5 = @jsdoc_named_type_expr +| 6 = @jsdoc_applied_type_expr +| 7 = @jsdoc_nullable_type_expr +| 8 = @jsdoc_non_nullable_type_expr +| 9 = @jsdoc_record_type_expr +| 10 = @jsdoc_array_type_expr +| 11 = @jsdoc_union_type_expr +| 12 = @jsdoc_function_type_expr +| 13 = @jsdoc_optional_type_expr +| 14 = @jsdoc_rest_type_expr +; + +#keyset[id, idx] +jsdoc_record_field_name (int id: @jsdoc_record_type_expr ref, int idx: int ref, varchar(900) name: string ref); +jsdoc_prefix_qualifier (int id: @jsdoc_type_expr ref); +jsdoc_has_new_parameter (int fn: @jsdoc_function_type_expr ref); + +@jsdoc_type_expr_parent = @jsdoc_type_expr | @jsdoc_tag; + +jsdoc_errors (unique int id: @jsdoc_error, int tag: @jsdoc_tag ref, varchar(900) message: string ref, varchar(900) tostring: string ref); + +// YAML +#keyset[parent, idx] +yaml (unique int id: @yaml_node, + int kind: int ref, + int parent: @yaml_node_parent ref, + int idx: int ref, + varchar(900) tag: string ref, + varchar(900) tostring: string ref); + +case @yaml_node.kind of + 0 = @yaml_scalar_node +| 1 = @yaml_mapping_node +| 2 = @yaml_sequence_node +| 3 = @yaml_alias_node +; + +@yaml_collection_node = @yaml_mapping_node | @yaml_sequence_node; + +@yaml_node_parent = @yaml_collection_node | @file; + +yaml_anchors (unique int node: @yaml_node ref, + varchar(900) anchor: string ref); + +yaml_aliases (unique int alias: @yaml_alias_node ref, + varchar(900) target: string ref); + +yaml_scalars (unique int scalar: @yaml_scalar_node ref, + int style: int ref, + varchar(900) value: string ref); + +yaml_errors (unique int id: @yaml_error, + varchar(900) message: string ref); + +yaml_locations(unique int locatable: @yaml_locatable ref, + int location: @location_default ref); + +@yaml_locatable = @yaml_node | @yaml_error; + +/* XML Files */ + +xmlEncoding( + unique int id: @file ref, + varchar(900) encoding: string ref +); + +xmlDTDs( + unique int id: @xmldtd, + varchar(900) root: string ref, + varchar(900) publicId: string ref, + varchar(900) systemId: string ref, + int fileid: @file ref +); + +xmlElements( + unique int id: @xmlelement, + varchar(900) name: string ref, + int parentid: @xmlparent ref, + int idx: int ref, + int fileid: @file ref +); + +xmlAttrs( + unique int id: @xmlattribute, + int elementid: @xmlelement ref, + varchar(900) name: string ref, + varchar(3600) value: string ref, + int idx: int ref, + int fileid: @file ref +); + +xmlNs( + int id: @xmlnamespace, + varchar(900) prefixName: string ref, + varchar(900) URI: string ref, + int fileid: @file ref +); + +xmlHasNs( + int elementId: @xmlnamespaceable ref, + int nsId: @xmlnamespace ref, + int fileid: @file ref +); + +xmlComments( + unique int id: @xmlcomment, + varchar(3600) text: string ref, + int parentid: @xmlparent ref, + int fileid: @file ref +); + +xmlChars( + unique int id: @xmlcharacters, + varchar(3600) text: string ref, + int parentid: @xmlparent ref, + int idx: int ref, + int isCDATA: int ref, + int fileid: @file ref +); + +@xmlparent = @file | @xmlelement; +@xmlnamespaceable = @xmlelement | @xmlattribute; + +xmllocations( + int xmlElement: @xmllocatable ref, + int location: @location_default ref +); + +@xmllocatable = @xmlcharacters | @xmlelement | @xmlcomment | @xmlattribute | @xmldtd | @file | @xmlnamespace; + +@dataflownode = @expr | @functiondeclstmt | @classdeclstmt | @namespacedeclaration | @enumdeclaration | @property; + +@optionalchainable = @callexpr | @propaccess; + +isOptionalChaining(int id: @optionalchainable ref); + +/* + * configuration files with key value pairs + */ + +configs( + unique int id: @config +); + +configNames( + unique int id: @configName, + int config: @config ref, + string name: string ref +); + +configValues( + unique int id: @configValue, + int config: @config ref, + string value: string ref +); + +configLocations( + int locatable: @configLocatable ref, + int location: @location_default ref +); + +@configLocatable = @config | @configName | @configValue; + +/** + * The time taken for the extraction of a file. + * This table contains non-deterministic content. + * + * The sum of the `time` column for each (`file`, `timerKind`) pair + * is the total time taken for extraction of `file`. The `extractionPhase` + * column provides a granular view of the extraction time of the file. + */ +extraction_time( + int file : @file ref, + // see `com.semmle.js.extractor.ExtractionMetrics.ExtractionPhase`. + int extractionPhase: int ref, + // 0 for the elapsed CPU time in nanoseconds, 1 for the elapsed wallclock time in nanoseconds + int timerKind: int ref, + float time: float ref +) + +/** + * Non-timing related data for the extraction of a single file. + * This table contains non-deterministic content. + */ +extraction_data( + int file : @file ref, + // the absolute path to the cache file + varchar(900) cacheFile: string ref, + boolean fromCache: boolean ref, + int length: int ref +) diff --git a/javascript/upgrades/874ebfcd4d2fd346c10ebec145466d9071d78606/semmlecode.javascript.dbscheme b/javascript/upgrades/874ebfcd4d2fd346c10ebec145466d9071d78606/semmlecode.javascript.dbscheme new file mode 100644 index 00000000000..5a5adbf98dc --- /dev/null +++ b/javascript/upgrades/874ebfcd4d2fd346c10ebec145466d9071d78606/semmlecode.javascript.dbscheme @@ -0,0 +1,1175 @@ +/*** Standard fragments ***/ + +/** Files and folders **/ + +@location = @location_default; + +locations_default(unique int id: @location_default, + int file: @file ref, + int beginLine: int ref, + int beginColumn: int ref, + int endLine: int ref, + int endColumn: int ref + ); + +@sourceline = @locatable; + +numlines(int element_id: @sourceline ref, + int num_lines: int ref, + int num_code: int ref, + int num_comment: int ref + ); + + +/* + fromSource(0) = unknown, + fromSource(1) = from source, + fromSource(2) = from library +*/ +files(unique int id: @file, + varchar(900) name: string ref, + varchar(900) simple: string ref, + varchar(900) ext: string ref, + int fromSource: int ref); + +folders(unique int id: @folder, + varchar(900) name: string ref, + varchar(900) simple: string ref); + + +@container = @folder | @file ; + + +containerparent(int parent: @container ref, + unique int child: @container ref); + +/** Duplicate code **/ + +duplicateCode( + unique int id : @duplication, + varchar(900) relativePath : string ref, + int equivClass : int ref); + +similarCode( + unique int id : @similarity, + varchar(900) relativePath : string ref, + int equivClass : int ref); + +@duplication_or_similarity = @duplication | @similarity; + +tokens( + int id : @duplication_or_similarity ref, + int offset : int ref, + int beginLine : int ref, + int beginColumn : int ref, + int endLine : int ref, + int endColumn : int ref); + +/** External data **/ + +externalData( + int id : @externalDataElement, + varchar(900) path : string ref, + int column: int ref, + varchar(900) value : string ref +); + +snapshotDate(unique date snapshotDate : date ref); + +sourceLocationPrefix(varchar(900) prefix : string ref); + +/** Version control data **/ + +svnentries( + int id : @svnentry, + varchar(500) revision : string ref, + varchar(500) author : string ref, + date revisionDate : date ref, + int changeSize : int ref +); + +svnaffectedfiles( + int id : @svnentry ref, + int file : @file ref, + varchar(500) action : string ref +); + +svnentrymsg( + int id : @svnentry ref, + varchar(500) message : string ref +); + +svnchurn( + int commit : @svnentry ref, + int file : @file ref, + int addedLines : int ref, + int deletedLines : int ref +); + + +/*** JavaScript-specific part ***/ + +filetype( + int file: @file ref, + string filetype: string ref +) + +// top-level code fragments +toplevels (unique int id: @toplevel, + int kind: int ref); + +isExterns (int toplevel: @toplevel ref); + +case @toplevel.kind of + 0 = @script +| 1 = @inline_script +| 2 = @event_handler +| 3 = @javascript_url; + +isModule (int tl: @toplevel ref); +isNodejs (int tl: @toplevel ref); +isES2015Module (int tl: @toplevel ref); +isClosureModule (int tl: @toplevel ref); + +// statements +#keyset[parent, idx] +stmts (unique int id: @stmt, + int kind: int ref, + int parent: @stmtparent ref, + int idx: int ref, + varchar(900) tostring: string ref); + +stmtContainers (unique int stmt: @stmt ref, + int container: @stmt_container ref); + +jumpTargets (unique int jump: @stmt ref, + int target: @stmt ref); + +@stmtparent = @stmt | @toplevel | @functionexpr | @arrowfunctionexpr; +@stmt_container = @toplevel | @function | @namespacedeclaration | @externalmoduledeclaration | @globalaugmentationdeclaration; + +case @stmt.kind of + 0 = @emptystmt +| 1 = @blockstmt +| 2 = @exprstmt +| 3 = @ifstmt +| 4 = @labeledstmt +| 5 = @breakstmt +| 6 = @continuestmt +| 7 = @withstmt +| 8 = @switchstmt +| 9 = @returnstmt +| 10 = @throwstmt +| 11 = @trystmt +| 12 = @whilestmt +| 13 = @dowhilestmt +| 14 = @forstmt +| 15 = @forinstmt +| 16 = @debuggerstmt +| 17 = @functiondeclstmt +| 18 = @vardeclstmt +| 19 = @case +| 20 = @catchclause +| 21 = @forofstmt +| 22 = @constdeclstmt +| 23 = @letstmt +| 24 = @legacy_letstmt +| 25 = @foreachstmt +| 26 = @classdeclstmt +| 27 = @importdeclaration +| 28 = @exportalldeclaration +| 29 = @exportdefaultdeclaration +| 30 = @exportnameddeclaration +| 31 = @namespacedeclaration +| 32 = @importequalsdeclaration +| 33 = @exportassigndeclaration +| 34 = @interfacedeclaration +| 35 = @typealiasdeclaration +| 36 = @enumdeclaration +| 37 = @externalmoduledeclaration +| 38 = @exportasnamespacedeclaration +| 39 = @globalaugmentationdeclaration +; + +@declstmt = @vardeclstmt | @constdeclstmt | @letstmt | @legacy_letstmt; + +@exportdeclaration = @exportalldeclaration | @exportdefaultdeclaration | @exportnameddeclaration; + +@namespacedefinition = @namespacedeclaration | @enumdeclaration; +@typedefinition = @classdefinition | @interfacedeclaration | @enumdeclaration | @typealiasdeclaration | @enum_member; + +isInstantiated(unique int decl: @namespacedeclaration ref); + +@declarablenode = @declstmt | @namespacedeclaration | @classdeclstmt | @functiondeclstmt | @enumdeclaration | @externalmoduledeclaration | @globalaugmentationdeclaration | @field; +hasDeclareKeyword(unique int stmt: @declarablenode ref); + +isForAwaitOf(unique int forof: @forofstmt ref); + +// expressions +#keyset[parent, idx] +exprs (unique int id: @expr, + int kind: int ref, + int parent: @exprparent ref, + int idx: int ref, + varchar(900) tostring: string ref); + +literals (varchar(900) value: string ref, + varchar(900) raw: string ref, + unique int expr: @exprortype ref); + +enclosingStmt (unique int expr: @exprortype ref, + int stmt: @stmt ref); + +exprContainers (unique int expr: @exprortype ref, + int container: @stmt_container ref); + +arraySize (unique int ae: @arraylike ref, + int sz: int ref); + +isDelegating (int yield: @yieldexpr ref); + +@exprorstmt = @expr | @stmt; +@exprortype = @expr | @typeexpr; +@exprparent = @exprorstmt | @property | @functiontypeexpr; +@arraylike = @arrayexpr | @arraypattern; +@type_annotation = @typeexpr | @jsdoc_type_expr; + +case @expr.kind of + 0 = @label +| 1 = @nullliteral +| 2 = @booleanliteral +| 3 = @numberliteral +| 4 = @stringliteral +| 5 = @regexpliteral +| 6 = @thisexpr +| 7 = @arrayexpr +| 8 = @objexpr +| 9 = @functionexpr +| 10 = @seqexpr +| 11 = @conditionalexpr +| 12 = @newexpr +| 13 = @callexpr +| 14 = @dotexpr +| 15 = @indexexpr +| 16 = @negexpr +| 17 = @plusexpr +| 18 = @lognotexpr +| 19 = @bitnotexpr +| 20 = @typeofexpr +| 21 = @voidexpr +| 22 = @deleteexpr +| 23 = @eqexpr +| 24 = @neqexpr +| 25 = @eqqexpr +| 26 = @neqqexpr +| 27 = @ltexpr +| 28 = @leexpr +| 29 = @gtexpr +| 30 = @geexpr +| 31 = @lshiftexpr +| 32 = @rshiftexpr +| 33 = @urshiftexpr +| 34 = @addexpr +| 35 = @subexpr +| 36 = @mulexpr +| 37 = @divexpr +| 38 = @modexpr +| 39 = @bitorexpr +| 40 = @xorexpr +| 41 = @bitandexpr +| 42 = @inexpr +| 43 = @instanceofexpr +| 44 = @logandexpr +| 45 = @logorexpr +| 47 = @assignexpr +| 48 = @assignaddexpr +| 49 = @assignsubexpr +| 50 = @assignmulexpr +| 51 = @assigndivexpr +| 52 = @assignmodexpr +| 53 = @assignlshiftexpr +| 54 = @assignrshiftexpr +| 55 = @assignurshiftexpr +| 56 = @assignorexpr +| 57 = @assignxorexpr +| 58 = @assignandexpr +| 59 = @preincexpr +| 60 = @postincexpr +| 61 = @predecexpr +| 62 = @postdecexpr +| 63 = @parexpr +| 64 = @vardeclarator +| 65 = @arrowfunctionexpr +| 66 = @spreadelement +| 67 = @arraypattern +| 68 = @objectpattern +| 69 = @yieldexpr +| 70 = @taggedtemplateexpr +| 71 = @templateliteral +| 72 = @templateelement +| 73 = @arraycomprehensionexpr +| 74 = @generatorexpr +| 75 = @forincomprehensionblock +| 76 = @forofcomprehensionblock +| 77 = @legacy_letexpr +| 78 = @vardecl +| 79 = @proper_varaccess +| 80 = @classexpr +| 81 = @superexpr +| 82 = @newtargetexpr +| 83 = @namedimportspecifier +| 84 = @importdefaultspecifier +| 85 = @importnamespacespecifier +| 86 = @namedexportspecifier +| 87 = @expexpr +| 88 = @assignexpexpr +| 89 = @jsxelement +| 90 = @jsxqualifiedname +| 91 = @jsxemptyexpr +| 92 = @awaitexpr +| 93 = @functionsentexpr +| 94 = @decorator +| 95 = @exportdefaultspecifier +| 96 = @exportnamespacespecifier +| 97 = @bindexpr +| 98 = @externalmodulereference +| 99 = @dynamicimport +| 100 = @expressionwithtypearguments +| 101 = @prefixtypeassertion +| 102 = @astypeassertion +| 103 = @export_varaccess +| 104 = @decorator_list +| 105 = @non_null_assertion +| 106 = @bigintliteral +| 107 = @nullishcoalescingexpr +| 108 = @e4x_xml_anyname +| 109 = @e4x_xml_static_attribute_selector +| 110 = @e4x_xml_dynamic_attribute_selector +| 111 = @e4x_xml_filter_expression +| 112 = @e4x_xml_static_qualident +| 113 = @e4x_xml_dynamic_qualident +| 114 = @e4x_xml_dotdotexpr +| 115 = @importmetaexpr +; + +@varaccess = @proper_varaccess | @export_varaccess; +@varref = @vardecl | @varaccess; + +@identifier = @label | @varref | @typeidentifier; + +@literal = @nullliteral | @booleanliteral | @numberliteral | @stringliteral | @regexpliteral | @bigintliteral; + +@propaccess = @dotexpr | @indexexpr; + +@invokeexpr = @newexpr | @callexpr; + +@unaryexpr = @negexpr | @plusexpr | @lognotexpr | @bitnotexpr | @typeofexpr | @voidexpr | @deleteexpr | @spreadelement; + +@equalitytest = @eqexpr | @neqexpr | @eqqexpr | @neqqexpr; + +@comparison = @equalitytest | @ltexpr | @leexpr | @gtexpr | @geexpr; + +@binaryexpr = @comparison | @lshiftexpr | @rshiftexpr | @urshiftexpr | @addexpr | @subexpr | @mulexpr | @divexpr | @modexpr | @expexpr | @bitorexpr | @xorexpr | @bitandexpr | @inexpr | @instanceofexpr | @logandexpr | @logorexpr | @nullishcoalescingexpr; + +@assignment = @assignexpr | @assignaddexpr | @assignsubexpr | @assignmulexpr | @assigndivexpr | @assignmodexpr | @assignexpexpr | @assignlshiftexpr | @assignrshiftexpr | @assignurshiftexpr | @assignorexpr | @assignxorexpr | @assignandexpr; + +@updateexpr = @preincexpr | @postincexpr | @predecexpr | @postdecexpr; + +@pattern = @varref | @arraypattern | @objectpattern; + +@comprehensionexpr = @arraycomprehensionexpr | @generatorexpr; + +@comprehensionblock = @forincomprehensionblock | @forofcomprehensionblock; + +@importspecifier = @namedimportspecifier | @importdefaultspecifier | @importnamespacespecifier; + +@exportspecifier = @namedexportspecifier | @exportdefaultspecifier | @exportnamespacespecifier; + +@typeassertion = @astypeassertion | @prefixtypeassertion; + +@classdefinition = @classdeclstmt | @classexpr; +@interfacedefinition = @interfacedeclaration | @interfacetypeexpr; +@classorinterface = @classdefinition | @interfacedefinition; + +@lexical_decl = @vardecl | @typedecl; +@lexical_access = @varaccess | @localtypeaccess | @localvartypeaccess | @localnamespaceaccess; +@lexical_ref = @lexical_decl | @lexical_access; + +@e4x_xml_attribute_selector = @e4x_xml_static_attribute_selector | @e4x_xml_dynamic_attribute_selector; +@e4x_xml_qualident = @e4x_xml_static_qualident | @e4x_xml_dynamic_qualident; + +// scopes +scopes (unique int id: @scope, + int kind: int ref); + +case @scope.kind of + 0 = @globalscope +| 1 = @functionscope +| 2 = @catchscope +| 3 = @modulescope +| 4 = @blockscope +| 5 = @forscope +| 6 = @forinscope // for-of scopes work the same as for-in scopes +| 7 = @comprehensionblockscope +| 8 = @classexprscope +| 9 = @namespacescope +| 10 = @classdeclscope +| 11 = @interfacescope +| 12 = @typealiasscope +| 13 = @mappedtypescope +| 14 = @enumscope +| 15 = @externalmodulescope +| 16 = @conditionaltypescope; + +scopenodes (unique int node: @ast_node ref, + int scope: @scope ref); + +scopenesting (unique int inner: @scope ref, + int outer: @scope ref); + +// functions +@function = @functiondeclstmt | @functionexpr | @arrowfunctionexpr; + +@parameterized = @function | @catchclause; +@type_parameterized = @function | @classorinterface | @typealiasdeclaration | @mappedtypeexpr | @infertypeexpr; + +isGenerator (int fun: @function ref); +hasRestParameter (int fun: @function ref); +isAsync (int fun: @function ref); + +// variables and lexically scoped type names +#keyset[scope, name] +variables (unique int id: @variable, + varchar(900) name: string ref, + int scope: @scope ref); + +#keyset[scope, name] +local_type_names (unique int id: @local_type_name, + varchar(900) name: string ref, + int scope: @scope ref); + +#keyset[scope, name] +local_namespace_names (unique int id: @local_namespace_name, + varchar(900) name: string ref, + int scope: @scope ref); + +isArgumentsObject (int id: @variable ref); + +@lexical_name = @variable | @local_type_name | @local_namespace_name; + +@bind_id = @varaccess | @localvartypeaccess; +bind (unique int id: @bind_id ref, + int decl: @variable ref); + +decl (unique int id: @vardecl ref, + int decl: @variable ref); + +@typebind_id = @localtypeaccess | @export_varaccess; +typebind (unique int id: @typebind_id ref, + int decl: @local_type_name ref); + +@typedecl_id = @typedecl | @vardecl; +typedecl (unique int id: @typedecl_id ref, + int decl: @local_type_name ref); + +namespacedecl (unique int id: @vardecl ref, + int decl: @local_namespace_name ref); + +@namespacebind_id = @localnamespaceaccess | @export_varaccess; +namespacebind (unique int id: @namespacebind_id ref, + int decl: @local_namespace_name ref); + + +// properties in object literals, property patterns in object patterns, and method declarations in classes +#keyset[parent, index] +properties (unique int id: @property, + int parent: @property_parent ref, + int index: int ref, + int kind: int ref, + varchar(900) tostring: string ref); + +case @property.kind of + 0 = @value_property +| 1 = @property_getter +| 2 = @property_setter +| 3 = @jsx_attribute +| 4 = @function_call_signature +| 5 = @constructor_call_signature +| 6 = @index_signature +| 7 = @enum_member +| 8 = @proper_field +| 9 = @parameter_field +; + +@property_parent = @objexpr | @objectpattern | @classdefinition | @jsxelement | @interfacedefinition | @enumdeclaration; +@property_accessor = @property_getter | @property_setter; +@call_signature = @function_call_signature | @constructor_call_signature; +@field = @proper_field | @parameter_field; +@field_or_vardeclarator = @field | @vardeclarator; + +isComputed (int id: @property ref); +isMethod (int id: @property ref); +isStatic (int id: @property ref); +isAbstractMember (int id: @property ref); +isConstEnum (int id: @enumdeclaration ref); +isAbstractClass (int id: @classdeclstmt ref); + +hasPublicKeyword (int id: @property ref); +hasPrivateKeyword (int id: @property ref); +hasProtectedKeyword (int id: @property ref); +hasReadonlyKeyword (int id: @property ref); +isOptionalMember (int id: @property ref); +hasDefiniteAssignmentAssertion (int id: @field_or_vardeclarator ref); + +#keyset[constructor, param_index] +parameter_fields( + unique int field: @parameter_field ref, + int constructor: @functionexpr ref, + int param_index: int ref +); + +// types +#keyset[parent, idx] +typeexprs ( + unique int id: @typeexpr, + int kind: int ref, + int parent: @typeexpr_parent ref, + int idx: int ref, + varchar(900) tostring: string ref +); + +case @typeexpr.kind of + 0 = @localtypeaccess +| 1 = @typedecl +| 2 = @keywordtypeexpr +| 3 = @stringliteraltypeexpr +| 4 = @numberliteraltypeexpr +| 5 = @booleanliteraltypeexpr +| 6 = @arraytypeexpr +| 7 = @uniontypeexpr +| 8 = @indexedaccesstypeexpr +| 9 = @intersectiontypeexpr +| 10 = @parenthesizedtypeexpr +| 11 = @tupletypeexpr +| 12 = @keyoftypeexpr +| 13 = @qualifiedtypeaccess +| 14 = @generictypeexpr +| 15 = @typelabel +| 16 = @typeoftypeexpr +| 17 = @localvartypeaccess +| 18 = @qualifiedvartypeaccess +| 19 = @thisvartypeaccess +| 20 = @predicatetypeexpr +| 21 = @interfacetypeexpr +| 22 = @typeparameter +| 23 = @plainfunctiontypeexpr +| 24 = @constructortypeexpr +| 25 = @localnamespaceaccess +| 26 = @qualifiednamespaceaccess +| 27 = @mappedtypeexpr +| 28 = @conditionaltypeexpr +| 29 = @infertypeexpr +| 30 = @importtypeaccess +| 31 = @importnamespaceaccess +| 32 = @importvartypeaccess +| 33 = @optionaltypeexpr +| 34 = @resttypeexpr +| 35 = @bigintliteraltypeexpr +| 36 = @readonlytypeexpr +; + +@typeref = @typeaccess | @typedecl; +@typeidentifier = @typedecl | @localtypeaccess | @typelabel | @localvartypeaccess | @localnamespaceaccess; +@typeexpr_parent = @expr | @stmt | @property | @typeexpr; +@literaltypeexpr = @stringliteraltypeexpr | @numberliteraltypeexpr | @booleanliteraltypeexpr | @bigintliteraltypeexpr; +@typeaccess = @localtypeaccess | @qualifiedtypeaccess | @importtypeaccess; +@vartypeaccess = @localvartypeaccess | @qualifiedvartypeaccess | @thisvartypeaccess | @importvartypeaccess; +@namespaceaccess = @localnamespaceaccess | @qualifiednamespaceaccess | @importnamespaceaccess; +@importtypeexpr = @importtypeaccess | @importnamespaceaccess | @importvartypeaccess; + +@functiontypeexpr = @plainfunctiontypeexpr | @constructortypeexpr; + +// types +types ( + unique int id: @type, + int kind: int ref, + varchar(900) tostring: string ref +); + +#keyset[parent, idx] +type_child ( + int child: @type ref, + int parent: @type ref, + int idx: int ref +); + +case @type.kind of + 0 = @anytype +| 1 = @stringtype +| 2 = @numbertype +| 3 = @uniontype +| 4 = @truetype +| 5 = @falsetype +| 6 = @typereference +| 7 = @objecttype +| 8 = @canonicaltypevariabletype +| 9 = @typeoftype +| 10 = @voidtype +| 11 = @undefinedtype +| 12 = @nulltype +| 13 = @nevertype +| 14 = @plainsymboltype +| 15 = @uniquesymboltype +| 16 = @objectkeywordtype +| 17 = @intersectiontype +| 18 = @tupletype +| 19 = @lexicaltypevariabletype +| 20 = @thistype +| 21 = @numberliteraltype +| 22 = @stringliteraltype +| 23 = @unknowntype +| 24 = @biginttype +| 25 = @bigintliteraltype +; + +@booleanliteraltype = @truetype | @falsetype; +@symboltype = @plainsymboltype | @uniquesymboltype; +@unionorintersectiontype = @uniontype | @intersectiontype; +@typevariabletype = @canonicaltypevariabletype | @lexicaltypevariabletype; + +hasAssertsKeyword(int node: @predicatetypeexpr ref); + +@typed_ast_node = @expr | @typeexpr | @function; +ast_node_type( + unique int node: @typed_ast_node ref, + int typ: @type ref); + +declared_function_signature( + unique int node: @function ref, + int sig: @signature_type ref +); + +invoke_expr_signature( + unique int node: @invokeexpr ref, + int sig: @signature_type ref +); + +invoke_expr_overload_index( + unique int node: @invokeexpr ref, + int index: int ref +); + +symbols ( + unique int id: @symbol, + int kind: int ref, + varchar(900) name: string ref +); + +symbol_parent ( + unique int symbol: @symbol ref, + int parent: @symbol ref +); + +symbol_module ( + int symbol: @symbol ref, + varchar(900) moduleName: string ref +); + +symbol_global ( + int symbol: @symbol ref, + varchar(900) globalName: string ref +); + +case @symbol.kind of + 0 = @root_symbol +| 1 = @member_symbol +| 2 = @other_symbol +; + +@type_with_symbol = @typereference | @typevariabletype | @typeoftype | @uniquesymboltype; +@ast_node_with_symbol = @typedefinition | @namespacedefinition | @toplevel | @typeaccess | @namespaceaccess | @vardecl | @function | @invokeexpr; + +ast_node_symbol( + unique int node: @ast_node_with_symbol ref, + int symbol: @symbol ref); + +type_symbol( + unique int typ: @type_with_symbol ref, + int symbol: @symbol ref); + +#keyset[typ, name] +type_property( + int typ: @type ref, + varchar(900) name: string ref, + int propertyType: @type ref); + +@literaltype = @stringliteraltype | @numberliteraltype | @booleanliteraltype | @bigintliteraltype; +@type_with_literal_value = @stringliteraltype | @numberliteraltype | @bigintliteraltype; +type_literal_value( + unique int typ: @type_with_literal_value ref, + varchar(900) value: string ref); + +signature_types ( + unique int id: @signature_type, + int kind: int ref, + varchar(900) tostring: string ref, + int type_parameters: int ref, + int required_params: int ref +); + +case @signature_type.kind of + 0 = @function_signature_type +| 1 = @constructor_signature_type +; + +#keyset[typ, kind, index] +type_contains_signature ( + int typ: @type ref, + int kind: int ref, // constructor/call/index + int index: int ref, // ordering of overloaded signatures + int sig: @signature_type ref +); + +#keyset[parent, index] +signature_contains_type ( + int child: @type ref, + int parent: @signature_type ref, + int index: int ref +); + +#keyset[sig, index] +signature_parameter_name ( + int sig: @signature_type ref, + int index: int ref, + varchar(900) name: string ref +); + +number_index_type ( + unique int baseType: @type ref, + int propertyType: @type ref +); + +string_index_type ( + unique int baseType: @type ref, + int propertyType: @type ref +); + +base_type_names( + int typeName: @symbol ref, + int baseTypeName: @symbol ref +); + +self_types( + int typeName: @symbol ref, + int selfType: @typereference ref +); + +tuple_type_min_length( + unique int typ: @type ref, + int minLength: int ref +); + +tuple_type_rest( + unique int typ: @type ref +); + +// comments +comments (unique int id: @comment, + int kind: int ref, + int toplevel: @toplevel ref, + varchar(900) text: string ref, + varchar(900) tostring: string ref); + +case @comment.kind of + 0 = @slashslashcomment +| 1 = @slashstarcomment +| 2 = @doccomment +| 3 = @htmlcommentstart +| 4 = @htmlcommentend; + +@htmlcomment = @htmlcommentstart | @htmlcommentend; +@linecomment = @slashslashcomment | @htmlcomment; +@blockcomment = @slashstarcomment | @doccomment; + +// source lines +lines (unique int id: @line, + int toplevel: @toplevel ref, + varchar(900) text: string ref, + varchar(2) terminator: string ref); +indentation (int file: @file ref, + int lineno: int ref, + varchar(1) indentChar: string ref, + int indentDepth: int ref); + +// JavaScript parse errors +jsParseErrors (unique int id: @js_parse_error, + int toplevel: @toplevel ref, + varchar(900) message: string ref, + varchar(900) line: string ref); + +// regular expressions +#keyset[parent, idx] +regexpterm (unique int id: @regexpterm, + int kind: int ref, + int parent: @regexpparent ref, + int idx: int ref, + varchar(900) tostring: string ref); + +@regexpparent = @regexpterm | @regexpliteral; + +case @regexpterm.kind of + 0 = @regexp_alt +| 1 = @regexp_seq +| 2 = @regexp_caret +| 3 = @regexp_dollar +| 4 = @regexp_wordboundary +| 5 = @regexp_nonwordboundary +| 6 = @regexp_positive_lookahead +| 7 = @regexp_negative_lookahead +| 8 = @regexp_star +| 9 = @regexp_plus +| 10 = @regexp_opt +| 11 = @regexp_range +| 12 = @regexp_dot +| 13 = @regexp_group +| 14 = @regexp_normal_char +| 15 = @regexp_hex_escape +| 16 = @regexp_unicode_escape +| 17 = @regexp_dec_escape +| 18 = @regexp_oct_escape +| 19 = @regexp_ctrl_escape +| 20 = @regexp_char_class_escape +| 21 = @regexp_id_escape +| 22 = @regexp_backref +| 23 = @regexp_char_class +| 24 = @regexp_char_range +| 25 = @regexp_positive_lookbehind +| 26 = @regexp_negative_lookbehind +| 27 = @regexp_unicode_property_escape; + +regexpParseErrors (unique int id: @regexp_parse_error, + int regexp: @regexpterm ref, + varchar(900) message: string ref); + +@regexp_quantifier = @regexp_star | @regexp_plus | @regexp_opt | @regexp_range; +@regexp_escape = @regexp_char_escape | @regexp_char_class_escape | @regexp_unicode_property_escape; +@regexp_char_escape = @regexp_hex_escape | @regexp_unicode_escape | @regexp_dec_escape | @regexp_oct_escape | @regexp_ctrl_escape | @regexp_id_escape; +@regexp_constant = @regexp_normal_char | @regexp_char_escape; +@regexp_lookahead = @regexp_positive_lookahead | @regexp_negative_lookahead; +@regexp_lookbehind = @regexp_positive_lookbehind | @regexp_negative_lookbehind; +@regexp_subpattern = @regexp_lookahead | @regexp_lookbehind; + +isGreedy (int id: @regexp_quantifier ref); +rangeQuantifierLowerBound (unique int id: @regexp_range ref, int lo: int ref); +rangeQuantifierUpperBound (unique int id: @regexp_range ref, int hi: int ref); +isCapture (unique int id: @regexp_group ref, int number: int ref); +isNamedCapture (unique int id: @regexp_group ref, string name: string ref); +isInverted (int id: @regexp_char_class ref); +regexpConstValue (unique int id: @regexp_constant ref, varchar(1) value: string ref); +charClassEscape (unique int id: @regexp_char_class_escape ref, varchar(1) value: string ref); +backref (unique int id: @regexp_backref ref, int value: int ref); +namedBackref (unique int id: @regexp_backref ref, string name: string ref); +unicodePropertyEscapeName (unique int id: @regexp_unicode_property_escape ref, string name: string ref); +unicodePropertyEscapeValue (unique int id: @regexp_unicode_property_escape ref, string value: string ref); + +// tokens +#keyset[toplevel, idx] +tokeninfo (unique int id: @token, + int kind: int ref, + int toplevel: @toplevel ref, + int idx: int ref, + varchar(900) value: string ref); + +case @token.kind of + 0 = @token_eof +| 1 = @token_null_literal +| 2 = @token_boolean_literal +| 3 = @token_numeric_literal +| 4 = @token_string_literal +| 5 = @token_regular_expression +| 6 = @token_identifier +| 7 = @token_keyword +| 8 = @token_punctuator; + +// associate comments with the token immediately following them (which may be EOF) +next_token (int comment: @comment ref, int token: @token ref); + +// JSON +#keyset[parent, idx] +json (unique int id: @json_value, + int kind: int ref, + int parent: @json_parent ref, + int idx: int ref, + varchar(900) tostring: string ref); + +json_literals (varchar(900) value: string ref, + varchar(900) raw: string ref, + unique int expr: @json_value ref); + +json_properties (int obj: @json_object ref, + varchar(900) property: string ref, + int value: @json_value ref); + +json_errors (unique int id: @json_parse_error, + varchar(900) message: string ref); + +json_locations(unique int locatable: @json_locatable ref, + int location: @location_default ref); + +case @json_value.kind of + 0 = @json_null +| 1 = @json_boolean +| 2 = @json_number +| 3 = @json_string +| 4 = @json_array +| 5 = @json_object; + +@json_parent = @json_object | @json_array | @file; + +@json_locatable = @json_value | @json_parse_error; + +// locations +@ast_node = @toplevel | @stmt | @expr | @property | @typeexpr; + +@locatable = @file + | @ast_node + | @comment + | @line + | @js_parse_error | @regexp_parse_error + | @regexpterm + | @json_locatable + | @token + | @cfg_node + | @jsdoc | @jsdoc_type_expr | @jsdoc_tag + | @yaml_locatable + | @xmllocatable + | @configLocatable; + +hasLocation (unique int locatable: @locatable ref, + int location: @location ref); + +// CFG +entry_cfg_node (unique int id: @entry_node, int container: @stmt_container ref); +exit_cfg_node (unique int id: @exit_node, int container: @stmt_container ref); +guard_node (unique int id: @guard_node, int kind: int ref, int test: @expr ref); +case @guard_node.kind of + 0 = @falsy_guard +| 1 = @truthy_guard; +@condition_guard = @falsy_guard | @truthy_guard; + +@synthetic_cfg_node = @entry_node | @exit_node | @guard_node; +@cfg_node = @synthetic_cfg_node | @exprparent; + +successor (int pred: @cfg_node ref, int succ: @cfg_node ref); + +// JSDoc comments +jsdoc (unique int id: @jsdoc, varchar(900) description: string ref, int comment: @comment ref); +#keyset[parent, idx] +jsdoc_tags (unique int id: @jsdoc_tag, varchar(900) title: string ref, + int parent: @jsdoc ref, int idx: int ref, varchar(900) tostring: string ref); +jsdoc_tag_descriptions (unique int tag: @jsdoc_tag ref, varchar(900) text: string ref); +jsdoc_tag_names (unique int tag: @jsdoc_tag ref, varchar(900) text: string ref); + +#keyset[parent, idx] +jsdoc_type_exprs (unique int id: @jsdoc_type_expr, + int kind: int ref, + int parent: @jsdoc_type_expr_parent ref, + int idx: int ref, + varchar(900) tostring: string ref); +case @jsdoc_type_expr.kind of + 0 = @jsdoc_any_type_expr +| 1 = @jsdoc_null_type_expr +| 2 = @jsdoc_undefined_type_expr +| 3 = @jsdoc_unknown_type_expr +| 4 = @jsdoc_void_type_expr +| 5 = @jsdoc_named_type_expr +| 6 = @jsdoc_applied_type_expr +| 7 = @jsdoc_nullable_type_expr +| 8 = @jsdoc_non_nullable_type_expr +| 9 = @jsdoc_record_type_expr +| 10 = @jsdoc_array_type_expr +| 11 = @jsdoc_union_type_expr +| 12 = @jsdoc_function_type_expr +| 13 = @jsdoc_optional_type_expr +| 14 = @jsdoc_rest_type_expr +; + +#keyset[id, idx] +jsdoc_record_field_name (int id: @jsdoc_record_type_expr ref, int idx: int ref, varchar(900) name: string ref); +jsdoc_prefix_qualifier (int id: @jsdoc_type_expr ref); +jsdoc_has_new_parameter (int fn: @jsdoc_function_type_expr ref); + +@jsdoc_type_expr_parent = @jsdoc_type_expr | @jsdoc_tag; + +jsdoc_errors (unique int id: @jsdoc_error, int tag: @jsdoc_tag ref, varchar(900) message: string ref, varchar(900) tostring: string ref); + +// YAML +#keyset[parent, idx] +yaml (unique int id: @yaml_node, + int kind: int ref, + int parent: @yaml_node_parent ref, + int idx: int ref, + varchar(900) tag: string ref, + varchar(900) tostring: string ref); + +case @yaml_node.kind of + 0 = @yaml_scalar_node +| 1 = @yaml_mapping_node +| 2 = @yaml_sequence_node +| 3 = @yaml_alias_node +; + +@yaml_collection_node = @yaml_mapping_node | @yaml_sequence_node; + +@yaml_node_parent = @yaml_collection_node | @file; + +yaml_anchors (unique int node: @yaml_node ref, + varchar(900) anchor: string ref); + +yaml_aliases (unique int alias: @yaml_alias_node ref, + varchar(900) target: string ref); + +yaml_scalars (unique int scalar: @yaml_scalar_node ref, + int style: int ref, + varchar(900) value: string ref); + +yaml_errors (unique int id: @yaml_error, + varchar(900) message: string ref); + +yaml_locations(unique int locatable: @yaml_locatable ref, + int location: @location_default ref); + +@yaml_locatable = @yaml_node | @yaml_error; + +/* XML Files */ + +xmlEncoding( + unique int id: @file ref, + varchar(900) encoding: string ref +); + +xmlDTDs( + unique int id: @xmldtd, + varchar(900) root: string ref, + varchar(900) publicId: string ref, + varchar(900) systemId: string ref, + int fileid: @file ref +); + +xmlElements( + unique int id: @xmlelement, + varchar(900) name: string ref, + int parentid: @xmlparent ref, + int idx: int ref, + int fileid: @file ref +); + +xmlAttrs( + unique int id: @xmlattribute, + int elementid: @xmlelement ref, + varchar(900) name: string ref, + varchar(3600) value: string ref, + int idx: int ref, + int fileid: @file ref +); + +xmlNs( + int id: @xmlnamespace, + varchar(900) prefixName: string ref, + varchar(900) URI: string ref, + int fileid: @file ref +); + +xmlHasNs( + int elementId: @xmlnamespaceable ref, + int nsId: @xmlnamespace ref, + int fileid: @file ref +); + +xmlComments( + unique int id: @xmlcomment, + varchar(3600) text: string ref, + int parentid: @xmlparent ref, + int fileid: @file ref +); + +xmlChars( + unique int id: @xmlcharacters, + varchar(3600) text: string ref, + int parentid: @xmlparent ref, + int idx: int ref, + int isCDATA: int ref, + int fileid: @file ref +); + +@xmlparent = @file | @xmlelement; +@xmlnamespaceable = @xmlelement | @xmlattribute; + +xmllocations( + int xmlElement: @xmllocatable ref, + int location: @location_default ref +); + +@xmllocatable = @xmlcharacters | @xmlelement | @xmlcomment | @xmlattribute | @xmldtd | @file | @xmlnamespace; + +@dataflownode = @expr | @functiondeclstmt | @classdeclstmt | @namespacedeclaration | @enumdeclaration | @property; + +@optionalchainable = @callexpr | @propaccess; + +isOptionalChaining(int id: @optionalchainable ref); + +/* + * configuration files with key value pairs + */ + +configs( + unique int id: @config +); + +configNames( + unique int id: @configName, + int config: @config ref, + string name: string ref +); + +configValues( + unique int id: @configValue, + int config: @config ref, + string value: string ref +); + +configLocations( + int locatable: @configLocatable ref, + int location: @location_default ref +); + +@configLocatable = @config | @configName | @configValue; + +/** + * The time taken for the extraction of a file. + * This table contains non-deterministic content. + * + * The sum of the `time` column for each (`file`, `timerKind`) pair + * is the total time taken for extraction of `file`. The `extractionPhase` + * column provides a granular view of the extraction time of the file. + */ +extraction_time( + int file : @file ref, + // see `com.semmle.js.extractor.ExtractionMetrics.ExtractionPhase`. + int extractionPhase: int ref, + // 0 for the elapsed CPU time in nanoseconds, 1 for the elapsed wallclock time in nanoseconds + int timerKind: int ref, + float time: float ref +) + +/** + * Non-timing related data for the extraction of a single file. + * This table contains non-deterministic content. + */ +extraction_data( + int file : @file ref, + // the absolute path to the cache file + varchar(900) cacheFile: string ref, + boolean fromCache: boolean ref, + int length: int ref +) diff --git a/javascript/upgrades/874ebfcd4d2fd346c10ebec145466d9071d78606/upgrade.properties b/javascript/upgrades/874ebfcd4d2fd346c10ebec145466d9071d78606/upgrade.properties new file mode 100644 index 00000000000..bf1a8cb0c61 --- /dev/null +++ b/javascript/upgrades/874ebfcd4d2fd346c10ebec145466d9071d78606/upgrade.properties @@ -0,0 +1,2 @@ +description: add support for TypeScript 3.7 features +compatibility: backwards From 7f55e3f336ffc7f838aea2c0b8c468826a83d5af Mon Sep 17 00:00:00 2001 From: Esben Sparre Andreasen Date: Mon, 4 Nov 2019 09:05:50 +0100 Subject: [PATCH 102/148] JS: classify Doxygen-generated files as "generated" --- change-notes/1.23/analysis-javascript.md | 1 + javascript/ql/src/semmle/javascript/GeneratedCode.qll | 5 +++++ .../filters/ClassifyFiles/ClassifyFiles.expected | 1 + .../filters/ClassifyFiles/doxygen-generated.html | 8 ++++++++ 4 files changed, 15 insertions(+) create mode 100644 javascript/ql/test/query-tests/filters/ClassifyFiles/doxygen-generated.html diff --git a/change-notes/1.23/analysis-javascript.md b/change-notes/1.23/analysis-javascript.md index 20f9ad1df87..554827ee9e4 100644 --- a/change-notes/1.23/analysis-javascript.md +++ b/change-notes/1.23/analysis-javascript.md @@ -14,6 +14,7 @@ * TypeScript 3.6 features are supported. +* Automatic classification of generated files has been improved, in particular files generated by Doxygen are now recognized. ## New queries diff --git a/javascript/ql/src/semmle/javascript/GeneratedCode.qll b/javascript/ql/src/semmle/javascript/GeneratedCode.qll index b4877f8a8fa..0f8b161e7ae 100644 --- a/javascript/ql/src/semmle/javascript/GeneratedCode.qll +++ b/javascript/ql/src/semmle/javascript/GeneratedCode.qll @@ -137,6 +137,11 @@ private predicate isGeneratedHtml(File f) { e.getAttributeByName("name").getValue() = "generator" ) or + exists(HTML::CommentNode comment | + comment.getText().regexpMatch("\\s*Generated by [\\w-]+ \\d+\\.\\d+\\.\\d+\\s*") and + comment.getFile() = f + ) + or 20 < countStartingHtmlElements(f, _) } diff --git a/javascript/ql/test/query-tests/filters/ClassifyFiles/ClassifyFiles.expected b/javascript/ql/test/query-tests/filters/ClassifyFiles/ClassifyFiles.expected index 045fccd225e..7c583b1bf39 100644 --- a/javascript/ql/test/query-tests/filters/ClassifyFiles/ClassifyFiles.expected +++ b/javascript/ql/test/query-tests/filters/ClassifyFiles/ClassifyFiles.expected @@ -6,6 +6,7 @@ | ai.1.2.3-build0123.js:0:0:0:0 | ai.1.2.3-build0123.js | library | | bundle-directive.js:0:0:0:0 | bundle-directive.js | generated | | data.js:0:0:0:0 | data.js | generated | +| doxygen-generated.html:0:0:0:0 | doxygen-generated.html | generated | | etherpad.html:0:0:0:0 | etherpad.html | generated | | exported-data.js:0:0:0:0 | exported-data.js | generated | | htmltidy.html:0:0:0:0 | htmltidy.html | generated | diff --git a/javascript/ql/test/query-tests/filters/ClassifyFiles/doxygen-generated.html b/javascript/ql/test/query-tests/filters/ClassifyFiles/doxygen-generated.html new file mode 100644 index 00000000000..b659cc3fcb0 --- /dev/null +++ b/javascript/ql/test/query-tests/filters/ClassifyFiles/doxygen-generated.html @@ -0,0 +1,8 @@ + + + + + + 1 + + From d2f985038c9f989f6a158e9d269f2a2b53c018dc Mon Sep 17 00:00:00 2001 From: Taus Brock-Nannestad Date: Mon, 4 Nov 2019 10:48:42 +0100 Subject: [PATCH 103/148] Python: Fix missing modernisation. --- python/ql/src/Variables/UndefinedExport.ql | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/python/ql/src/Variables/UndefinedExport.ql b/python/ql/src/Variables/UndefinedExport.ql index b7a835b42d6..edd2e375c0f 100644 --- a/python/ql/src/Variables/UndefinedExport.ql +++ b/python/ql/src/Variables/UndefinedExport.ql @@ -25,7 +25,7 @@ predicate declaredInAll(Module m, StrConst name) { predicate mutates_globals(ModuleValue m) { exists(CallNode globals | - globals = Object::builtin("globals").(FunctionObject).getACall() and + globals = Value::named("globals").(FunctionValue).getACall() and globals.getScope() = m.getScope() | exists(AttrNode attr | attr.getObject() = globals) @@ -33,11 +33,12 @@ predicate mutates_globals(ModuleValue m) { exists(SubscriptNode sub | sub.getValue() = globals and sub.isStore()) ) or - exists(Object enum_convert | - enum_convert.hasLongName("enum.Enum._convert") and + exists(Value enum_convert, ClassValue enum_class | + enum_class.getASuperType() = Value::named("enum.Enum") and + enum_convert = enum_class.attr("_convert") and exists(CallNode call | call.getScope() = m.getScope() | - enum_convert.(FunctionObject).getACall() = call or - call.getFunction().refersTo(enum_convert) + enum_convert.getACall() = call or + call.getFunction().pointsTo(enum_convert) ) ) } From 6593477d0ba3c0e19473e3e847e963a9f3646793 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Mon, 4 Nov 2019 11:32:21 +0100 Subject: [PATCH 104/148] Python: Limit what functions we treat as returning sensitive data Before this change, any function that has a parameter that was called password/credentials would be treated as returning sensitive data of that kind. `py/clear-text-logging-sensitive-data` would alert if one of these are logged, which has a LOT of false-positives. --- python/ql/src/semmle/python/security/SensitiveData.qll | 6 ------ 1 file changed, 6 deletions(-) diff --git a/python/ql/src/semmle/python/security/SensitiveData.qll b/python/ql/src/semmle/python/security/SensitiveData.qll index 9b253da6937..5dddd9b0d30 100644 --- a/python/ql/src/semmle/python/security/SensitiveData.qll +++ b/python/ql/src/semmle/python/security/SensitiveData.qll @@ -112,12 +112,6 @@ module SensitiveData { private SensitiveData fromFunction(Value func) { result = HeuristicNames::getSensitiveDataForName(func.getName()) - or - // This is particularly to pick up methods with an argument like "password", which - // may indicate a lookup. - exists(string name | name = func.(PythonFunctionValue).getScope().getAnArg().asName().getId() | - result = HeuristicNames::getSensitiveDataForName(name) - ) } abstract class Source extends TaintSource { From b6f4ce024400ad1e7cb6fc33793c4bdbfe82667f Mon Sep 17 00:00:00 2001 From: Asger F Date: Mon, 4 Nov 2019 14:58:38 +0000 Subject: [PATCH 105/148] TS: Depend on typescript 3.7.2 --- javascript/extractor/lib/typescript/package.json | 2 +- javascript/extractor/lib/typescript/yarn.lock | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/javascript/extractor/lib/typescript/package.json b/javascript/extractor/lib/typescript/package.json index 966210ec8b2..3b3360158d3 100644 --- a/javascript/extractor/lib/typescript/package.json +++ b/javascript/extractor/lib/typescript/package.json @@ -2,7 +2,7 @@ "name": "typescript-parser-wrapper", "private": true, "dependencies": { - "typescript": "^3.7.1-rc" + "typescript": "^3.7.2" }, "scripts": { "build": "tsc --project tsconfig.json", diff --git a/javascript/extractor/lib/typescript/yarn.lock b/javascript/extractor/lib/typescript/yarn.lock index 9b539464cec..f2236efb429 100644 --- a/javascript/extractor/lib/typescript/yarn.lock +++ b/javascript/extractor/lib/typescript/yarn.lock @@ -225,10 +225,9 @@ tsutils@^2.12.1: dependencies: tslib "^1.8.1" -typescript@^3.7.1-rc: - version "3.7.1-rc" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.7.1-rc.tgz#2054b872d67f8dc732e36c1df397f9327f37ada0" - integrity sha512-2rMtWppLsaPvmpXsoIAXWDBQVnJMw1ITGGSnidMuayLj9iCmMRT69ncKZw/Mk5rXfJkilApKucWQZxproALoRw== +typescript@^3.7.2: + version "3.7.2" + resolved "typescript-3.7.2.tgz" wrappy@1: version "1.0.2" From 57aa166bffd57703bbaf6d767d5fc89dfb2afcdd Mon Sep 17 00:00:00 2001 From: Asger F Date: Mon, 4 Nov 2019 14:59:31 +0000 Subject: [PATCH 106/148] TS: Clean up yarn.lock --- javascript/extractor/lib/typescript/yarn.lock | 82 +++++++++---------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/javascript/extractor/lib/typescript/yarn.lock b/javascript/extractor/lib/typescript/yarn.lock index f2236efb429..5116c7eb5c3 100644 --- a/javascript/extractor/lib/typescript/yarn.lock +++ b/javascript/extractor/lib/typescript/yarn.lock @@ -4,31 +4,31 @@ "@types/node@12.7.11": version "12.7.11" - resolved node-12.7.11.tgz#be879b52031cfb5d295b047f5462d8ef1a716446 + resolved "node-12.7.11.tgz#be879b52031cfb5d295b047f5462d8ef1a716446" ansi-regex@^2.0.0: version "2.1.1" - resolved ansi-regex-2.1.1.tgz#c3b33ab5ee360d86e0e628f0468ae7ef27d654df + resolved "ansi-regex-2.1.1.tgz#c3b33ab5ee360d86e0e628f0468ae7ef27d654df" ansi-styles@^2.2.1: version "2.2.1" - resolved ansi-styles-2.2.1.tgz#b432dd3358b634cf75e1e4664368240533c1ddbe + resolved "ansi-styles-2.2.1.tgz#b432dd3358b634cf75e1e4664368240533c1ddbe" ansi-styles@^3.1.0: version "3.2.0" - resolved ansi-styles-3.2.0.tgz#c159b8d5be0f9e5a6f346dab94f16ce022161b88 + resolved "ansi-styles-3.2.0.tgz#c159b8d5be0f9e5a6f346dab94f16ce022161b88" dependencies: color-convert "^1.9.0" argparse@^1.0.7: version "1.0.9" - resolved argparse-1.0.9.tgz#73d83bc263f86e97f8cc4f6bae1b0e90a7d22c86 + resolved "argparse-1.0.9.tgz#73d83bc263f86e97f8cc4f6bae1b0e90a7d22c86" dependencies: sprintf-js "~1.0.2" babel-code-frame@^6.22.0: version "6.26.0" - resolved babel-code-frame-6.26.0.tgz#63fd43f7dc1e3bb7ce35947db8fe369a3f58c74b + resolved "babel-code-frame-6.26.0.tgz#63fd43f7dc1e3bb7ce35947db8fe369a3f58c74b" dependencies: chalk "^1.1.3" esutils "^2.0.2" @@ -36,22 +36,22 @@ babel-code-frame@^6.22.0: balanced-match@^1.0.0: version "1.0.0" - resolved balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767 + resolved "balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" brace-expansion@^1.1.7: version "1.1.8" - resolved brace-expansion-1.1.8.tgz#c07b211c7c952ec1f8efd51a77ef0d1d3990a292 + resolved "brace-expansion-1.1.8.tgz#c07b211c7c952ec1f8efd51a77ef0d1d3990a292" dependencies: balanced-match "^1.0.0" concat-map "0.0.1" builtin-modules@^1.1.1: version "1.1.1" - resolved builtin-modules-1.1.1.tgz#270f076c5a72c02f5b65a47df94c5fe3a278892f + resolved "builtin-modules-1.1.1.tgz#270f076c5a72c02f5b65a47df94c5fe3a278892f" chalk@^1.1.3: version "1.1.3" - resolved chalk-1.1.3.tgz#a8115c55e4a702fe4d150abd3872822a7e09fc98 + resolved "chalk-1.1.3.tgz#a8115c55e4a702fe4d150abd3872822a7e09fc98" dependencies: ansi-styles "^2.2.1" escape-string-regexp "^1.0.2" @@ -61,7 +61,7 @@ chalk@^1.1.3: chalk@^2.3.0: version "2.3.0" - resolved chalk-2.3.0.tgz#b5ea48efc9c1793dccc9b4767c93914d3f2d52ba + resolved "chalk-2.3.0.tgz#b5ea48efc9c1793dccc9b4767c93914d3f2d52ba" dependencies: ansi-styles "^3.1.0" escape-string-regexp "^1.0.5" @@ -69,45 +69,45 @@ chalk@^2.3.0: color-convert@^1.9.0: version "1.9.1" - resolved color-convert-1.9.1.tgz#c1261107aeb2f294ebffec9ed9ecad529a6097ed + resolved "color-convert-1.9.1.tgz#c1261107aeb2f294ebffec9ed9ecad529a6097ed" dependencies: color-name "^1.1.1" color-name@^1.1.1: version "1.1.3" - resolved color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25 + resolved "color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25" commander@^2.12.1: version "2.13.0" - resolved commander-2.13.0.tgz#6964bca67685df7c1f1430c584f07d7597885b9c + resolved "commander-2.13.0.tgz#6964bca67685df7c1f1430c584f07d7597885b9c" concat-map@0.0.1: version "0.0.1" - resolved concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b + resolved "concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" diff@^3.2.0: version "3.4.0" - resolved diff-3.4.0.tgz#b1d85507daf3964828de54b37d0d73ba67dda56c + resolved "diff-3.4.0.tgz#b1d85507daf3964828de54b37d0d73ba67dda56c" escape-string-regexp@^1.0.2, escape-string-regexp@^1.0.5: version "1.0.5" - resolved escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4 + resolved "escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4" esprima@^4.0.0: version "4.0.0" - resolved esprima-4.0.0.tgz#4499eddcd1110e0b218bacf2fa7f7f59f55ca804 + resolved "esprima-4.0.0.tgz#4499eddcd1110e0b218bacf2fa7f7f59f55ca804" esutils@^2.0.2: version "2.0.2" - resolved esutils-2.0.2.tgz#0abf4f1caa5bcb1f7a9d8acc6dea4faaa04bac9b + resolved "esutils-2.0.2.tgz#0abf4f1caa5bcb1f7a9d8acc6dea4faaa04bac9b" fs.realpath@^1.0.0: version "1.0.0" - resolved fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f + resolved "fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" glob@^7.1.1: version "7.1.2" - resolved glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15 + resolved "glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" dependencies: fs.realpath "^1.0.0" inflight "^1.0.4" @@ -118,93 +118,93 @@ glob@^7.1.1: has-ansi@^2.0.0: version "2.0.0" - resolved has-ansi-2.0.0.tgz#34f5049ce1ecdf2b0649af3ef24e45ed35416d91 + resolved "has-ansi-2.0.0.tgz#34f5049ce1ecdf2b0649af3ef24e45ed35416d91" dependencies: ansi-regex "^2.0.0" has-flag@^2.0.0: version "2.0.0" - resolved has-flag-2.0.0.tgz#e8207af1cc7b30d446cc70b734b5e8be18f88d51 + resolved "has-flag-2.0.0.tgz#e8207af1cc7b30d446cc70b734b5e8be18f88d51" inflight@^1.0.4: version "1.0.6" - resolved inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9 + resolved "inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9" dependencies: once "^1.3.0" wrappy "1" inherits@2: version "2.0.3" - resolved inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de + resolved "inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" js-tokens@^3.0.2: version "3.0.2" - resolved js-tokens-3.0.2.tgz#9866df395102130e38f7f996bceb65443209c25b + resolved "js-tokens-3.0.2.tgz#9866df395102130e38f7f996bceb65443209c25b" js-yaml@^3.7.0: version "3.10.0" - resolved js-yaml-3.10.0.tgz#2e78441646bd4682e963f22b6e92823c309c62dc + resolved "js-yaml-3.10.0.tgz#2e78441646bd4682e963f22b6e92823c309c62dc" dependencies: argparse "^1.0.7" esprima "^4.0.0" minimatch@^3.0.4: version "3.0.4" - resolved minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083 + resolved "minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" dependencies: brace-expansion "^1.1.7" once@^1.3.0: version "1.4.0" - resolved once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1 + resolved "once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" dependencies: wrappy "1" path-is-absolute@^1.0.0: version "1.0.1" - resolved path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f + resolved "path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" path-parse@^1.0.5: version "1.0.5" - resolved path-parse-1.0.5.tgz#3c1adf871ea9cd6c9431b6ea2bd74a0ff055c4c1 + resolved "path-parse-1.0.5.tgz#3c1adf871ea9cd6c9431b6ea2bd74a0ff055c4c1" resolve@^1.3.2: version "1.5.0" - resolved resolve-1.5.0.tgz#1f09acce796c9a762579f31b2c1cc4c3cddf9f36 + resolved "resolve-1.5.0.tgz#1f09acce796c9a762579f31b2c1cc4c3cddf9f36" dependencies: path-parse "^1.0.5" semver@^5.3.0: version "5.5.0" - resolved semver-5.5.0.tgz#dc4bbc7a6ca9d916dee5d43516f0092b58f7b8ab + resolved "semver-5.5.0.tgz#dc4bbc7a6ca9d916dee5d43516f0092b58f7b8ab" sprintf-js@~1.0.2: version "1.0.3" - resolved sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c + resolved "sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" strip-ansi@^3.0.0: version "3.0.1" - resolved strip-ansi-3.0.1.tgz#6a385fb8853d952d5ff05d0e8aaf94278dc63dcf + resolved "strip-ansi-3.0.1.tgz#6a385fb8853d952d5ff05d0e8aaf94278dc63dcf" dependencies: ansi-regex "^2.0.0" supports-color@^2.0.0: version "2.0.0" - resolved supports-color-2.0.0.tgz#535d045ce6b6363fa40117084629995e9df324c7 + resolved "supports-color-2.0.0.tgz#535d045ce6b6363fa40117084629995e9df324c7" supports-color@^4.0.0: version "4.5.0" - resolved supports-color-4.5.0.tgz#be7a0de484dec5c5cddf8b3d59125044912f635b + resolved "supports-color-4.5.0.tgz#be7a0de484dec5c5cddf8b3d59125044912f635b" dependencies: has-flag "^2.0.0" tslib@^1.8.0, tslib@^1.8.1: version "1.9.0" - resolved tslib-1.9.0.tgz#e37a86fda8cbbaf23a057f473c9f4dc64e5fc2e8 + resolved "tslib-1.9.0.tgz#e37a86fda8cbbaf23a057f473c9f4dc64e5fc2e8" tslint@^5.9.1: version "5.9.1" - resolved tslint-5.9.1.tgz#1255f87a3ff57eb0b0e1f0e610a8b4748046c9ae + resolved "tslint-5.9.1.tgz#1255f87a3ff57eb0b0e1f0e610a8b4748046c9ae" dependencies: babel-code-frame "^6.22.0" builtin-modules "^1.1.1" @@ -221,7 +221,7 @@ tslint@^5.9.1: tsutils@^2.12.1: version "2.19.1" - resolved tsutils-2.19.1.tgz#76d7ebdea9d7a7bf4a05f50ead3701b0168708d7 + resolved "tsutils-2.19.1.tgz#76d7ebdea9d7a7bf4a05f50ead3701b0168708d7" dependencies: tslib "^1.8.1" @@ -231,4 +231,4 @@ typescript@^3.7.2: wrappy@1: version "1.0.2" - resolved wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f + resolved "wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" From 016808b92ea17a62abe3a29688cf6be02a97ec68 Mon Sep 17 00:00:00 2001 From: Max Schaefer Date: Mon, 4 Nov 2019 17:00:12 +0000 Subject: [PATCH 107/148] JavaScript: Address review comments. --- change-notes/1.23/analysis-javascript.md | 1 + javascript/ql/src/Security/CWE-116/DoubleEscaping.qhelp | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/change-notes/1.23/analysis-javascript.md b/change-notes/1.23/analysis-javascript.md index aa9d2086d14..c87e16ce569 100644 --- a/change-notes/1.23/analysis-javascript.md +++ b/change-notes/1.23/analysis-javascript.md @@ -31,6 +31,7 @@ | **Query** | **Expected impact** | **Change** | |--------------------------------|------------------------------|---------------------------------------------------------------------------| +| Double escaping or unescaping (`js/double-escaping`) | More results | This rule now detects additional escaping and unescaping functions. | | Incomplete string escaping or encoding (`js/incomplete-sanitization`) | Fewer false-positive results | This rule now recognizes additional ways delimiters can be stripped away. | | Client-side cross-site scripting (`js/xss`) | More results, fewer false-positive results | More potential vulnerabilities involving functions that manipulate DOM attributes are now recognized, and more sanitizers are detected. | | Code injection (`js/code-injection`) | More results | More potential vulnerabilities involving functions that manipulate DOM event handler attributes are now recognized. | diff --git a/javascript/ql/src/Security/CWE-116/DoubleEscaping.qhelp b/javascript/ql/src/Security/CWE-116/DoubleEscaping.qhelp index 02cce7c5911..6c36fde4b28 100644 --- a/javascript/ql/src/Security/CWE-116/DoubleEscaping.qhelp +++ b/javascript/ql/src/Security/CWE-116/DoubleEscaping.qhelp @@ -10,8 +10,8 @@ attacks such as cross-site scripting. One particular example of this is HTML ent where HTML special characters are replaced by HTML character entities to prevent them from being interpreted as HTML markup. For example, the less-than character is encoded as &lt; and the double-quote character as &quot;. -Other examples include backslash-escaping for including untrusted data in string literals and -percent-encoding for URI components. +Other examples include backslash escaping or JSON encoding for including untrusted data in string +literals, and percent-encoding for URI components.

    The reverse process of replacing escape sequences with the characters they represent is known as From 6cac9619d3faeaa47b1db702425731058517dcec Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Mon, 4 Nov 2019 18:44:13 +0100 Subject: [PATCH 108/148] add missing not Co-Authored-By: Max Schaefer <54907921+max-schaefer@users.noreply.github.com> --- javascript/ql/src/Statements/UseOfReturnlessFunction.ql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/javascript/ql/src/Statements/UseOfReturnlessFunction.ql b/javascript/ql/src/Statements/UseOfReturnlessFunction.ql index 77ec29ba9ee..59d88bc51a0 100644 --- a/javascript/ql/src/Statements/UseOfReturnlessFunction.ql +++ b/javascript/ql/src/Statements/UseOfReturnlessFunction.ql @@ -131,7 +131,7 @@ DataFlow::SourceNode array() { result = array(DataFlow::TypeTracker::end()) } /** * Holds if `call` is an Array or Lodash method accepting a callback `func`, * where the `call` expects a callback that returns an expression, - * but `func` does return a value. + * but `func` does not return a value. */ predicate voidArrayCallback(DataFlow::CallNode call, Function func) { hasNonVoidCallbackMethod(call.getCalleeName()) and From 8ebfe15f0d286139b370a0cc2f3016459c9b1520 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Mon, 4 Nov 2019 18:54:43 +0100 Subject: [PATCH 109/148] apply doc feedback from mchammer01 Co-Authored-By: mc <42146119+mchammer01@users.noreply.github.com> --- javascript/ql/src/Statements/IgnoreArrayResult.qhelp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/javascript/ql/src/Statements/IgnoreArrayResult.qhelp b/javascript/ql/src/Statements/IgnoreArrayResult.qhelp index 2232784d881..048ca6680fc 100644 --- a/javascript/ql/src/Statements/IgnoreArrayResult.qhelp +++ b/javascript/ql/src/Statements/IgnoreArrayResult.qhelp @@ -22,17 +22,17 @@ or slice.

    -In the following example a function extend is defined. The +A function extend is defined in the following example. The function uses the concat method to add elements to the arr array. However, the extend function has no -effect as the return value from concat is ignored. +effect as the return value from concat is ignored:

    Assigning the returned value from the call to concat to the -arr variable fixes the error. +arr variable fixes the error:

    From aa47e3f6d2ff7d45dac3d7b3a5c00b83d4da3931 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Mon, 4 Nov 2019 18:52:29 +0100 Subject: [PATCH 110/148] update change-note to reflect changed query --- change-notes/1.23/analysis-javascript.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/change-notes/1.23/analysis-javascript.md b/change-notes/1.23/analysis-javascript.md index 90f46403f26..e3421a3de45 100644 --- a/change-notes/1.23/analysis-javascript.md +++ b/change-notes/1.23/analysis-javascript.md @@ -23,7 +23,7 @@ | Use of returnless function (`js/use-of-returnless-function`) | maintainability, correctness | Highlights calls where the return value is used, but the callee never returns a value. Results are shown on LGTM by default. | | Useless regular expression character escape (`js/useless-regexp-character-escape`) | correctness, security, external/cwe/cwe-20 | Highlights regular expression strings with useless character escapes, indicating a possible violation of [CWE-20](https://cwe.mitre.org/data/definitions/20.html). Results are shown on LGTM by default. | | Unreachable method overloads (`js/unreachable-method-overloads`) | correctness, typescript | Highlights method overloads that are impossible to use from client code. Results are shown on LGTM by default. | -| Ignoring return from concat (`js/ignore-return-from-concat`) | maintainability, correctness | Highlights calls to the concat method on array where the return value is ignored. Results are shown on LGTM by default. | +| Ignoring result from pure array method (`js/ignore-array-result`) | maintainability, correctness | Highlights calls to array methods without side effects where the return value is ignored. Results are shown on LGTM by default. | ## Changes to existing queries From bdb81c268c22a92758c60cdea795459fab268f07 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Mon, 4 Nov 2019 18:56:03 +0100 Subject: [PATCH 111/148] change tense --- javascript/ql/src/Statements/IgnoreArrayResult.qhelp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/javascript/ql/src/Statements/IgnoreArrayResult.qhelp b/javascript/ql/src/Statements/IgnoreArrayResult.qhelp index 048ca6680fc..70e3cb4e5cc 100644 --- a/javascript/ql/src/Statements/IgnoreArrayResult.qhelp +++ b/javascript/ql/src/Statements/IgnoreArrayResult.qhelp @@ -5,7 +5,7 @@

    The concat, join and slice methods are -pure and do not modify any of the inputs or the array the method was called +pure and do not modify any of the inputs or the array the method is called on. It is therefore generally an error to ignore the return value from a call to one of these methods.

    From df3c70e57e1613c6ad1096e3028924634c612196 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Tue, 5 Nov 2019 10:40:14 +0100 Subject: [PATCH 112/148] add js/ignore-array-result to correctness-core suite --- javascript/config/suites/javascript/correctness-core | 1 + 1 file changed, 1 insertion(+) diff --git a/javascript/config/suites/javascript/correctness-core b/javascript/config/suites/javascript/correctness-core index 6a53405e1c3..d6b91e83a07 100644 --- a/javascript/config/suites/javascript/correctness-core +++ b/javascript/config/suites/javascript/correctness-core @@ -37,5 +37,6 @@ + semmlecode-javascript-queries/RegExp/UnboundBackref.ql: /Correctness/Regular Expressions + semmlecode-javascript-queries/RegExp/UnmatchableCaret.ql: /Correctness/Regular Expressions + semmlecode-javascript-queries/RegExp/UnmatchableDollar.ql: /Correctness/Regular Expressions ++ semmlecode-javascript-queries/Statements/IgnoreArrayResult.ql: /Correctness/Statements + semmlecode-javascript-queries/Statements/InconsistentLoopOrientation.ql: /Correctness/Statements + semmlecode-javascript-queries/Statements/UnreachableStatement.ql: /Correctness/Statements From d8f3a2c55037421e30a5d02cb7273af53e3cf9e7 Mon Sep 17 00:00:00 2001 From: Asger F Date: Thu, 31 Oct 2019 12:58:39 +0000 Subject: [PATCH 113/148] JS: Add lvalue of for..of loop as a PropRead --- .../semmle/javascript/dataflow/DataFlow.qll | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/javascript/ql/src/semmle/javascript/dataflow/DataFlow.qll b/javascript/ql/src/semmle/javascript/dataflow/DataFlow.qll index 8798df6cc37..07920cba3c9 100644 --- a/javascript/ql/src/semmle/javascript/dataflow/DataFlow.qll +++ b/javascript/ql/src/semmle/javascript/dataflow/DataFlow.qll @@ -741,7 +741,8 @@ module DataFlow { PropReadAsSourceNode() { this = TPropNode(any(PropertyPattern p)) or this instanceof RestPatternNode or - this instanceof ElementPatternNode + this instanceof ElementPatternNode or + this = lvalueNode(any(ForOfStmt stmt).getLValue()) } } @@ -826,6 +827,24 @@ module DataFlow { override string getPropertyName() { result = astNode.getImportedName() } } + /** + * The left-hand side of a `for..of` statement, seen as a property read + * on the object being iterated over. + */ + private class ForOfLvalueAsPropRead extends PropRead { + ForOfStmt stmt; + + ForOfLvalueAsPropRead() { + this = lvalueNode(stmt.getLValue()) + } + + override Node getBase() { result = stmt.getIterationDomain().flow() } + + override Expr getPropertyNameExpr() { none() } + + override string getPropertyName() { none() } + } + /** * A data flow node representing an unused parameter. * From d8ac0abb7f26879e59d7f6addd8cbe995aeedf5b Mon Sep 17 00:00:00 2001 From: Asger F Date: Tue, 5 Nov 2019 10:06:21 +0000 Subject: [PATCH 114/148] JS: Add test --- .../ql/test/library-tests/DataFlow/flowStep.expected | 5 +++++ .../ql/test/library-tests/DataFlow/incomplete.expected | 5 +++++ .../ql/test/library-tests/DataFlow/parameters.expected | 1 + javascript/ql/test/library-tests/DataFlow/sources.expected | 7 +++++++ javascript/ql/test/library-tests/DataFlow/sources.js | 5 +++++ 5 files changed, 23 insertions(+) diff --git a/javascript/ql/test/library-tests/DataFlow/flowStep.expected b/javascript/ql/test/library-tests/DataFlow/flowStep.expected index a6446732fbe..107dcfb98c7 100644 --- a/javascript/ql/test/library-tests/DataFlow/flowStep.expected +++ b/javascript/ql/test/library-tests/DataFlow/flowStep.expected @@ -7,6 +7,11 @@ | sources.js:3:11:3:11 | x | sources.js:4:10:4:10 | x | | sources.js:4:10:4:13 | x+19 | sources.js:3:1:5:6 | (functi ... \\n})(23) | | sources.js:5:4:5:5 | 23 | sources.js:3:11:3:11 | x | +| sources.js:9:14:9:18 | array | sources.js:10:19:10:23 | array | +| sources.js:9:14:9:18 | array | sources.js:11:23:11:27 | array | +| sources.js:10:12:10:14 | key | sources.js:10:28:10:30 | key | +| sources.js:11:12:11:18 | key | sources.js:11:32:11:34 | key | +| sources.js:11:14:11:16 | key | sources.js:11:12:11:18 | key | | tst.js:1:1:1:1 | x | tst.js:28:2:28:1 | x | | tst.js:1:1:1:1 | x | tst.js:32:1:32:0 | x | | tst.js:1:10:1:11 | fs | tst.js:1:10:1:11 | fs | diff --git a/javascript/ql/test/library-tests/DataFlow/incomplete.expected b/javascript/ql/test/library-tests/DataFlow/incomplete.expected index aee8fd44421..35413033fe7 100644 --- a/javascript/ql/test/library-tests/DataFlow/incomplete.expected +++ b/javascript/ql/test/library-tests/DataFlow/incomplete.expected @@ -8,6 +8,11 @@ | sources.js:1:6:1:11 | exceptional return of anonymous function | call | | sources.js:3:1:5:6 | exceptional return of (functi ... \\n})(23) | call | | sources.js:3:2:5:1 | exceptional return of anonymous function | call | +| sources.js:9:1:12:1 | exceptional return of function foo | call | +| sources.js:9:14:9:18 | array | call | +| sources.js:10:12:10:14 | key | heap | +| sources.js:11:12:11:18 | key | heap | +| sources.js:11:14:11:16 | key | heap | | tst.js:1:10:1:11 | fs | import | | tst.js:16:1:20:9 | exceptional return of (functi ... ("arg") | call | | tst.js:16:2:20:1 | exceptional return of function f | call | diff --git a/javascript/ql/test/library-tests/DataFlow/parameters.expected b/javascript/ql/test/library-tests/DataFlow/parameters.expected index b7746f41c8a..31bab07c4e2 100644 --- a/javascript/ql/test/library-tests/DataFlow/parameters.expected +++ b/javascript/ql/test/library-tests/DataFlow/parameters.expected @@ -1,5 +1,6 @@ | sources.js:1:6:1:6 | x | | sources.js:3:11:3:11 | x | +| sources.js:9:14:9:18 | array | | tst.js:16:13:16:13 | a | | tst.js:32:12:32:12 | b | | tst.js:87:11:87:24 | { p: x, ...o } | diff --git a/javascript/ql/test/library-tests/DataFlow/sources.expected b/javascript/ql/test/library-tests/DataFlow/sources.expected index 185cc89e4ad..37a54027371 100644 --- a/javascript/ql/test/library-tests/DataFlow/sources.expected +++ b/javascript/ql/test/library-tests/DataFlow/sources.expected @@ -13,6 +13,12 @@ | sources.js:3:2:5:1 | functio ... x+19;\\n} | | sources.js:3:11:3:11 | x | | sources.js:7:1:7:3 | /x/ | +| sources.js:9:1:9:0 | this | +| sources.js:9:1:12:1 | functio ... ey; }\\n} | +| sources.js:9:14:9:18 | array | +| sources.js:10:12:10:14 | key | +| sources.js:11:12:11:18 | { key } | +| sources.js:11:14:11:16 | key | | tst.js:1:1:1:0 | this | | tst.js:1:1:1:24 | import ... m 'fs'; | | tst.js:1:10:1:11 | fs | @@ -60,6 +66,7 @@ | tst.js:72:9:72:9 | p | | tst.js:72:9:72:11 | p() | | tst.js:75:9:75:21 | import('foo') | +| tst.js:80:10:80:10 | v | | tst.js:83:11:83:28 | [ for (v of o) v ] | | tst.js:85:11:85:28 | ( for (v of o) v ) | | tst.js:87:1:96:2 | (functi ... r: 0\\n}) | diff --git a/javascript/ql/test/library-tests/DataFlow/sources.js b/javascript/ql/test/library-tests/DataFlow/sources.js index 653c5fc129c..7a5bb869efe 100644 --- a/javascript/ql/test/library-tests/DataFlow/sources.js +++ b/javascript/ql/test/library-tests/DataFlow/sources.js @@ -5,3 +5,8 @@ new (x => x); })(23); /x/; + +function foo(array) { + for (let key of array) { key; } + for (let { key } of array) { key; } +} From 48c7d1d7c1218ad0b022bab561e69c74905876ce Mon Sep 17 00:00:00 2001 From: Matthew Gretton-Dann Date: Fri, 18 Oct 2019 11:20:16 +0100 Subject: [PATCH 115/148] C++: add *_template_parameter_value() tuples --- cpp/ql/src/semmlecode.cpp.dbscheme | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/cpp/ql/src/semmlecode.cpp.dbscheme b/cpp/ql/src/semmlecode.cpp.dbscheme index 98a075d5495..bd182f697bf 100644 --- a/cpp/ql/src/semmlecode.cpp.dbscheme +++ b/cpp/ql/src/semmlecode.cpp.dbscheme @@ -731,6 +731,11 @@ class_template_argument( int index: int ref, int arg_type: @type ref ); +class_template_argument_value( + int type_id: @usertype ref, + int index: int ref, + int arg_value: @expr ref +); is_proxy_class_for( unique int id: @usertype ref, @@ -755,6 +760,11 @@ function_template_argument( int index: int ref, int arg_type: @type ref ); +function_template_argument_value( + int function_id: @function ref, + int index: int ref, + int arg_value: @expr ref +); is_variable_template(unique int id: @variable ref); variable_instantiation( @@ -766,6 +776,11 @@ variable_template_argument( int index: int ref, int arg_type: @type ref ); +variable_template_argument_value( + int variable_id: @variable ref, + int index: int ref, + int arg_value: @expr ref +); /* Fixed point types From df7d21220be83026d76636877c98693fffc08ef0 Mon Sep 17 00:00:00 2001 From: Matthew Gretton-Dann Date: Fri, 18 Oct 2019 11:20:48 +0100 Subject: [PATCH 116/148] C++: Basic lib support for template param values --- cpp/ql/src/semmle/code/cpp/Class.qll | 9 +++++++++ cpp/ql/src/semmle/code/cpp/Declaration.qll | 7 +++++++ cpp/ql/src/semmle/code/cpp/Function.qll | 10 ++++++++++ cpp/ql/src/semmle/code/cpp/Variable.qll | 9 +++++++++ 4 files changed, 35 insertions(+) diff --git a/cpp/ql/src/semmle/code/cpp/Class.qll b/cpp/ql/src/semmle/code/cpp/Class.qll index 5ac4f52392e..44e8b4e65ef 100644 --- a/cpp/ql/src/semmle/code/cpp/Class.qll +++ b/cpp/ql/src/semmle/code/cpp/Class.qll @@ -614,6 +614,15 @@ class Class extends UserType { class_template_argument(underlyingElement(this), i, unresolveElement(result)) } + /** + * Gets the `i`th template argument value used to instantiate this class from a + * class template. When called on a class template, this will return the + * `i`th template parameter value. + */ + override Expr getTemplateArgumentValue(int i) { + class_template_argument_value(underlyingElement(this), i, unresolveElement(result)) + } + /** * Holds if this class/struct is polymorphic (has a virtual function, or * inherits one). diff --git a/cpp/ql/src/semmle/code/cpp/Declaration.qll b/cpp/ql/src/semmle/code/cpp/Declaration.qll index 5d13bf4883a..a3a93aaca96 100644 --- a/cpp/ql/src/semmle/code/cpp/Declaration.qll +++ b/cpp/ql/src/semmle/code/cpp/Declaration.qll @@ -203,6 +203,13 @@ abstract class Declaration extends Locatable, @declaration { */ Type getTemplateArgument(int index) { none() } + /** + * Gets the `i`th template argument value used to instantiate this declaration + * from a template. When called on a template, this will return the `i`th template + * parameter value if it exists. + */ + Expr getTemplateArgumentValue(int index) { none() } + /** Gets the number of template arguments for this declaration. */ final int getNumberOfTemplateArguments() { result = count(int i | exists(getTemplateArgument(i))) diff --git a/cpp/ql/src/semmle/code/cpp/Function.qll b/cpp/ql/src/semmle/code/cpp/Function.qll index 94d11d63575..be031f2b915 100644 --- a/cpp/ql/src/semmle/code/cpp/Function.qll +++ b/cpp/ql/src/semmle/code/cpp/Function.qll @@ -352,6 +352,16 @@ class Function extends Declaration, ControlFlowNode, AccessHolder, @function { function_template_argument(underlyingElement(this), index, unresolveElement(result)) } + /** + * Gets the value of the `i`th template argument used to instantiate this + * function from a function template if that argument was a 'non-type' + * argument. When called on a function template, this with return the value + * of the `i`th template parameter. + */ + override Expr getTemplateArgumentValue(int index) { + function_template_argument_value(underlyingElement(this), index, unresolveElement(result)) + } + /** * Holds if this function is defined in several files. This is illegal in * C (though possible in some C++ compilers), and likely indicates that diff --git a/cpp/ql/src/semmle/code/cpp/Variable.qll b/cpp/ql/src/semmle/code/cpp/Variable.qll index 6fd33d9a037..deae1069aab 100644 --- a/cpp/ql/src/semmle/code/cpp/Variable.qll +++ b/cpp/ql/src/semmle/code/cpp/Variable.qll @@ -164,6 +164,15 @@ class Variable extends Declaration, @variable { variable_template_argument(underlyingElement(this), index, unresolveElement(result)) } + /** + * Gets the `i`th template argument value used to instantiate this variable from a + * variable template. When called on a variable template, this will return the + * `i`th template parameter value. + */ + override Expr getTemplateArgumentValue(int index) { + variable_template_argument_value(underlyingElement(this), index, unresolveElement(result)) + } + /** * Holds if this is a compiler-generated variable. For example, a * [range-based for loop](http://en.cppreference.com/w/cpp/language/range-for) From faf5ba432b2306d09d77965c48a84edc1c245d94 Mon Sep 17 00:00:00 2001 From: Matthew Gretton-Dann Date: Fri, 18 Oct 2019 11:22:03 +0100 Subject: [PATCH 117/148] C++: Update expected test results --- .../library-tests/ir/ir/PrintAST.expected | 50 +++++++++---------- .../test/library-tests/ir/ir/raw_ir.expected | 23 +++++++++ .../ptr_to_member/segfault/exprs.expected | 4 ++ .../templates/CPP-202/template_args.expected | 4 +- .../templates/CPP-204/element.expected | 6 +++ 5 files changed, 60 insertions(+), 27 deletions(-) diff --git a/cpp/ql/test/library-tests/ir/ir/PrintAST.expected b/cpp/ql/test/library-tests/ir/ir/PrintAST.expected index 19a4288a4c4..e61fe2b440f 100644 --- a/cpp/ql/test/library-tests/ir/ir/PrintAST.expected +++ b/cpp/ql/test/library-tests/ir/ir/PrintAST.expected @@ -67,31 +67,7 @@ bad_asts.cpp: # 5| params: #-----| 0: [Parameter] p#0 #-----| Type = [RValueReferenceType] S && -# 9| [MemberFunction] int Bad::S::MemberFunction(int) -# 9| params: -# 9| 0: [Parameter] y -# 9| Type = [IntType] int -# 9| body: [Block] { ... } -# 10| 0: [ReturnStmt] return ... -# 10| 0: [AddExpr] ... + ... -# 10| Type = [IntType] int -# 10| ValueCategory = prvalue -# 10| 0: [AddExpr] ... + ... -# 10| Type = [IntType] int -# 10| ValueCategory = prvalue -# 10| 0: [Literal] Unknown literal -# 10| Type = [IntType] int -# 10| ValueCategory = prvalue -# 10| 1: [PointerFieldAccess] x -# 10| Type = [IntType] int -# 10| ValueCategory = prvalue(load) -#-----| -1: [ThisExpr] this -#-----| Type = [PointerType] S * -#-----| ValueCategory = prvalue(load) -# 10| 1: [VariableAccess] y -# 10| Type = [IntType] int -# 10| ValueCategory = prvalue(load) -# 9| [TopLevelFunction] int MemberFunction(int) +# 9| [FunctionTemplateInstantiation,MemberFunction] int Bad::S::MemberFunction(int) # 9| params: # 9| 0: [Parameter] y # 9| Type = [IntType] int @@ -116,6 +92,30 @@ bad_asts.cpp: # 10| 1: [VariableAccess] y # 10| Type = [IntType] int # 10| ValueCategory = prvalue(load) +# 9| [MemberFunction,TemplateFunction] int Bad::S::MemberFunction(int) +# 9| params: +# 9| 0: [Parameter] y +# 9| Type = [IntType] int +# 9| body: [Block] { ... } +# 10| 0: [ReturnStmt] return ... +# 10| 0: [AddExpr] ... + ... +# 10| Type = [IntType] int +# 10| ValueCategory = prvalue +# 10| 0: [AddExpr] ... + ... +# 10| Type = [IntType] int +# 10| ValueCategory = prvalue +# 10| 0: [Literal] Unknown literal +# 10| Type = [IntType] int +# 10| ValueCategory = prvalue +# 10| 1: [PointerFieldAccess] x +# 10| Type = [IntType] int +# 10| ValueCategory = prvalue(load) +#-----| -1: [ThisExpr] this +#-----| Type = [PointerType] S * +#-----| ValueCategory = prvalue(load) +# 10| 1: [VariableAccess] y +# 10| Type = [IntType] int +# 10| ValueCategory = prvalue(load) # 14| [TopLevelFunction] void Bad::CallBadMemberFunction() # 14| params: # 14| body: [Block] { ... } diff --git a/cpp/ql/test/library-tests/ir/ir/raw_ir.expected b/cpp/ql/test/library-tests/ir/ir/raw_ir.expected index 90c3061114e..f0c96d2bfa0 100644 --- a/cpp/ql/test/library-tests/ir/ir/raw_ir.expected +++ b/cpp/ql/test/library-tests/ir/ir/raw_ir.expected @@ -1,4 +1,27 @@ bad_asts.cpp: +# 9| int Bad::S::MemberFunction(int) +# 9| Block 0 +# 9| v0_0(void) = EnterFunction : +# 9| mu0_1(unknown) = AliasedDefinition : +# 9| mu0_2(unknown) = UnmodeledDefinition : +# 9| r0_3(glval) = InitializeThis : +# 9| r0_4(glval) = VariableAddress[y] : +# 9| mu0_5(int) = InitializeParameter[y] : &:r0_4 +# 10| r0_6(glval) = VariableAddress[#return] : +# 10| r0_7(int) = Constant[6] : +#-----| r0_8(S *) = CopyValue : r0_3 +# 10| r0_9(glval) = FieldAddress[x] : r0_8 +# 10| r0_10(int) = Load : &:r0_9, ~mu0_2 +# 10| r0_11(int) = Add : r0_7, r0_10 +# 10| r0_12(glval) = VariableAddress[y] : +# 10| r0_13(int) = Load : &:r0_12, ~mu0_2 +# 10| r0_14(int) = Add : r0_11, r0_13 +# 10| mu0_15(int) = Store : &:r0_6, r0_14 +# 9| r0_16(glval) = VariableAddress[#return] : +# 9| v0_17(void) = ReturnValue : &:r0_16, ~mu0_2 +# 9| v0_18(void) = UnmodeledUse : mu* +# 9| v0_19(void) = ExitFunction : + # 14| void Bad::CallBadMemberFunction() # 14| Block 0 # 14| v0_0(void) = EnterFunction : diff --git a/cpp/ql/test/library-tests/ptr_to_member/segfault/exprs.expected b/cpp/ql/test/library-tests/ptr_to_member/segfault/exprs.expected index 7af2f7c8c17..e745ca0142d 100644 --- a/cpp/ql/test/library-tests/ptr_to_member/segfault/exprs.expected +++ b/cpp/ql/test/library-tests/ptr_to_member/segfault/exprs.expected @@ -1,3 +1,7 @@ +| file://:0:0:0:0 | Unknown literal | file://:0:0:0:0 | unsigned long | +| file://:0:0:0:0 | Unknown literal | file://:0:0:0:0 | unsigned long | +| file://:0:0:0:0 | Unknown literal | file://:0:0:0:0 | unsigned long | +| file://:0:0:0:0 | Unknown literal | file://:0:0:0:0 | unsigned long | | segfault.cpp:25:46:25:65 | call to S | file://:0:0:0:0 | void | | segfault.cpp:25:46:25:65 | call to S | file://:0:0:0:0 | void | | segfault.cpp:25:48:25:55 | __second | segfault.cpp:15:7:15:11 | tuple | diff --git a/cpp/ql/test/library-tests/templates/CPP-202/template_args.expected b/cpp/ql/test/library-tests/templates/CPP-202/template_args.expected index 0f20692d443..ae10f9ddf63 100644 --- a/cpp/ql/test/library-tests/templates/CPP-202/template_args.expected +++ b/cpp/ql/test/library-tests/templates/CPP-202/template_args.expected @@ -1,6 +1,6 @@ | file://:0:0:0:0 | __va_list_tag | | -| test.cpp:3:8:3:9 | s1<> | | -| test.cpp:3:8:3:9 | s1<> | | +| test.cpp:3:8:3:9 | s1<> | bool | +| test.cpp:3:8:3:9 | s1<> | bool | | test.cpp:5:8:5:9 | s2 | T | | test.cpp:5:8:5:9 | s2 | T | | test.cpp:7:8:7:9 | s3> | (unnamed) | diff --git a/cpp/ql/test/library-tests/templates/CPP-204/element.expected b/cpp/ql/test/library-tests/templates/CPP-204/element.expected index 6cb84a117d5..b1662c2a3b3 100644 --- a/cpp/ql/test/library-tests/templates/CPP-204/element.expected +++ b/cpp/ql/test/library-tests/templates/CPP-204/element.expected @@ -1,6 +1,12 @@ | file://:0:0:0:0 | | | file://:0:0:0:0 | 0 | | file://:0:0:0:0 | (global namespace) | +| file://:0:0:0:0 | Unknown literal | +| file://:0:0:0:0 | Unknown literal | +| file://:0:0:0:0 | Unknown literal | +| file://:0:0:0:0 | Unknown literal | +| file://:0:0:0:0 | Unknown literal | +| file://:0:0:0:0 | Unknown literal | | file://:0:0:0:0 | __va_list_tag | | file://:0:0:0:0 | __va_list_tag & | | file://:0:0:0:0 | __va_list_tag && | From 6334ad92c5de06fa362f51e4727d3da5397f11ba Mon Sep 17 00:00:00 2001 From: Matthew Gretton-Dann Date: Fri, 18 Oct 2019 11:25:15 +0100 Subject: [PATCH 118/148] C++: Add DB Upgrade script. --- .../old.dbscheme | 1918 ++++++++++++++++ .../semmlecode.cpp.dbscheme | 1933 +++++++++++++++++ .../upgrade.properties | 2 + 3 files changed, 3853 insertions(+) create mode 100644 cpp/upgrades/98a075d5495d7be7ede26557708cf22cfa3964ef/old.dbscheme create mode 100644 cpp/upgrades/98a075d5495d7be7ede26557708cf22cfa3964ef/semmlecode.cpp.dbscheme create mode 100644 cpp/upgrades/98a075d5495d7be7ede26557708cf22cfa3964ef/upgrade.properties diff --git a/cpp/upgrades/98a075d5495d7be7ede26557708cf22cfa3964ef/old.dbscheme b/cpp/upgrades/98a075d5495d7be7ede26557708cf22cfa3964ef/old.dbscheme new file mode 100644 index 00000000000..98a075d5495 --- /dev/null +++ b/cpp/upgrades/98a075d5495d7be7ede26557708cf22cfa3964ef/old.dbscheme @@ -0,0 +1,1918 @@ + +/** + * An invocation of the compiler. Note that more than one file may be + * compiled per invocation. For example, this command compiles three + * source files: + * + * gcc -c f1.c f2.c f3.c + * + * The `id` simply identifies the invocation, while `cwd` is the working + * directory from which the compiler was invoked. + */ +compilations( + /** + * An invocation of the compiler. Note that more than one file may + * be compiled per invocation. For example, this command compiles + * three source files: + * + * gcc -c f1.c f2.c f3.c + */ + unique int id : @compilation, + string cwd : string ref +); + +/** + * The arguments that were passed to the extractor for a compiler + * invocation. If `id` is for the compiler invocation + * + * gcc -c f1.c f2.c f3.c + * + * then typically there will be rows for + * + * num | arg + * --- | --- + * 0 | *path to extractor* + * 1 | `--mimic` + * 2 | `/usr/bin/gcc` + * 3 | `-c` + * 4 | f1.c + * 5 | f2.c + * 6 | f3.c + */ +#keyset[id, num] +compilation_args( + int id : @compilation ref, + int num : int ref, + string arg : string ref +); + +/** + * The source files that are compiled by a compiler invocation. + * If `id` is for the compiler invocation + * + * gcc -c f1.c f2.c f3.c + * + * then there will be rows for + * + * num | arg + * --- | --- + * 0 | f1.c + * 1 | f2.c + * 2 | f3.c + * + * Note that even if those files `#include` headers, those headers + * do not appear as rows. + */ +#keyset[id, num] +compilation_compiling_files( + int id : @compilation ref, + int num : int ref, + int file : @file ref +); + +/** + * The time taken by the extractor for a compiler invocation. + * + * For each file `num`, there will be rows for + * + * kind | seconds + * ---- | --- + * 1 | CPU seconds used by the extractor frontend + * 2 | Elapsed seconds during the extractor frontend + * 3 | CPU seconds used by the extractor backend + * 4 | Elapsed seconds during the extractor backend + */ +#keyset[id, num, kind] +compilation_time( + int id : @compilation ref, + int num : int ref, + /* kind: + 1 = frontend_cpu_seconds + 2 = frontend_elapsed_seconds + 3 = extractor_cpu_seconds + 4 = extractor_elapsed_seconds + */ + int kind : int ref, + float seconds : float ref +); + +/** + * An error or warning generated by the extractor. + * The diagnostic message `diagnostic` was generated during compiler + * invocation `compilation`, and is the `file_number_diagnostic_number`th + * message generated while extracting the `file_number`th file of that + * invocation. + */ +#keyset[compilation, file_number, file_number_diagnostic_number] +diagnostic_for( + int diagnostic : @diagnostic ref, + int compilation : @compilation ref, + int file_number : int ref, + int file_number_diagnostic_number : int ref +); + +/** + * If extraction was successful, then `cpu_seconds` and + * `elapsed_seconds` are the CPU time and elapsed time (respectively) + * that extraction took for compiler invocation `id`. + */ +compilation_finished( + unique int id : @compilation ref, + float cpu_seconds : float ref, + float elapsed_seconds : float ref +); + + +/** + * External data, loaded from CSV files during snapshot creation. See + * [Tutorial: Incorporating external data](https://help.semmle.com/wiki/display/SD/Tutorial%3A+Incorporating+external+data) + * for more information. + */ +externalData( + int id : @externalDataElement, + string path : string ref, + int column: int ref, + string value : string ref +); + +/** + * The date of the snapshot. + */ +snapshotDate(unique date snapshotDate : date ref); + +/** + * The source location of the snapshot. + */ +sourceLocationPrefix(string prefix : string ref); + +/** + * Data used by the 'duplicate code' detection. + */ +duplicateCode( + unique int id : @duplication, + string relativePath : string ref, + int equivClass : int ref +); + +/** + * Data used by the 'similar code' detection. + */ +similarCode( + unique int id : @similarity, + string relativePath : string ref, + int equivClass : int ref +); + +/** + * Data used by the 'duplicate code' and 'similar code' detection. + */ +@duplication_or_similarity = @duplication | @similarity + +/** + * Data used by the 'duplicate code' and 'similar code' detection. + */ +#keyset[id, offset] +tokens( + int id : @duplication_or_similarity ref, + int offset : int ref, + int beginLine : int ref, + int beginColumn : int ref, + int endLine : int ref, + int endColumn : int ref +); + +/** + * Information about packages that provide code used during compilation. + * The `id` is just a unique identifier. + * The `namespace` is typically the name of the package manager that + * provided the package (e.g. "dpkg" or "yum"). + * The `package_name` is the name of the package, and `version` is its + * version (as a string). + */ +external_packages( + unique int id: @external_package, + string namespace : string ref, + string package_name : string ref, + string version : string ref +); + +/** + * Holds if File `fileid` was provided by package `package`. + */ +header_to_external_package( + int fileid : @file ref, + int package : @external_package ref +); + +/* + * Version history + */ + +svnentries( + unique int id : @svnentry, + string revision : string ref, + string author : string ref, + date revisionDate : date ref, + int changeSize : int ref +) + +svnaffectedfiles( + int id : @svnentry ref, + int file : @file ref, + string action : string ref +) + +svnentrymsg( + unique int id : @svnentry ref, + string message : string ref +) + +svnchurn( + int commit : @svnentry ref, + int file : @file ref, + int addedLines : int ref, + int deletedLines : int ref +) + +/* + * C++ dbscheme + */ + +@location = @location_stmt | @location_expr | @location_default ; + +/** + * The location of an element that is not an expression or a statement. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `file`. + * For more information, see + * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html). + */ +locations_default( + /** The location of an element that is not an expression or a statement. */ + unique int id: @location_default, + int container: @container ref, + int startLine: int ref, + int startColumn: int ref, + int endLine: int ref, + int endColumn: int ref +); + +/** + * The location of a statement. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `file`. + * For more information, see + * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html). + */ +locations_stmt( + /** The location of a statement. */ + unique int id: @location_stmt, + int container: @container ref, + int startLine: int ref, + int startColumn: int ref, + int endLine: int ref, + int endColumn: int ref +); + +/** + * The location of an expression. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `file`. + * For more information, see + * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html). + */ +locations_expr( + /** The location of an expression. */ + unique int id: @location_expr, + int container: @container ref, + int startLine: int ref, + int startColumn: int ref, + int endLine: int ref, + int endColumn: int ref +); + +/** An element for which line-count information is available. */ +@sourceline = @file | @function | @variable | @enumconstant | @xmllocatable; + +numlines( + int element_id: @sourceline ref, + int num_lines: int ref, + int num_code: int ref, + int num_comment: int ref +); + +diagnostics( + unique int id: @diagnostic, + int severity: int ref, + string error_tag: string ref, + string error_message: string ref, + string full_error_message: string ref, + int location: @location_default ref +); + +/* + fromSource(0) = unknown, + fromSource(1) = from source, + fromSource(2) = from library +*/ +files( + unique int id: @file, + string name: string ref, + string simple: string ref, + string ext: string ref, + int fromSource: int ref +); + +folders( + unique int id: @folder, + string name: string ref, + string simple: string ref +); + +@container = @folder | @file + +containerparent( + int parent: @container ref, + unique int child: @container ref +); + +fileannotations( + int id: @file ref, + int kind: int ref, + string name: string ref, + string value: string ref +); + +inmacroexpansion( + int id: @element ref, + int inv: @macroinvocation ref +); + +affectedbymacroexpansion( + int id: @element ref, + int inv: @macroinvocation ref +); + +/* + case @macroinvocations.kind of + 1 = macro expansion + | 2 = other macro reference + ; +*/ +macroinvocations( + unique int id: @macroinvocation, + int macro_id: @ppd_define ref, + int location: @location_default ref, + int kind: int ref +); + +macroparent( + unique int id: @macroinvocation ref, + int parent_id: @macroinvocation ref +); + +// a macroinvocation may be part of another location +// the way to find a constant expression that uses a macro +// is thus to find a constant expression that has a location +// to which a macro invocation is bound +macrolocationbind( + int id: @macroinvocation ref, + int location: @location ref +); + +#keyset[invocation, argument_index] +macro_argument_unexpanded( + int invocation: @macroinvocation ref, + int argument_index: int ref, + string text: string ref +); + +#keyset[invocation, argument_index] +macro_argument_expanded( + int invocation: @macroinvocation ref, + int argument_index: int ref, + string text: string ref +); + +/* + case @function.kind of + 1 = normal + | 2 = constructor + | 3 = destructor + | 4 = conversion + | 5 = operator + | 6 = builtin // GCC built-in functions, e.g. __builtin___memcpy_chk + ; +*/ +functions( + unique int id: @function, + string name: string ref, + int kind: int ref +); + +function_entry_point(int id: @function ref, unique int entry_point: @stmt ref); + +function_return_type(int id: @function ref, int return_type: @type ref); + +purefunctions(unique int id: @function ref); + +function_deleted(unique int id: @function ref); + +function_defaulted(unique int id: @function ref); + + + +#keyset[id, type_id] +fun_decls( + int id: @fun_decl, + int function: @function ref, + int type_id: @type ref, + string name: string ref, + int location: @location_default ref +); +fun_def(unique int id: @fun_decl ref); +fun_specialized(unique int id: @fun_decl ref); +fun_implicit(unique int id: @fun_decl ref); +fun_decl_specifiers( + int id: @fun_decl ref, + string name: string ref +) +#keyset[fun_decl, index] +fun_decl_throws( + int fun_decl: @fun_decl ref, + int index: int ref, + int type_id: @type ref +); +/* an empty throw specification is different from none */ +fun_decl_empty_throws(unique int fun_decl: @fun_decl ref); +fun_decl_noexcept( + int fun_decl: @fun_decl ref, + int constant: @expr ref +); +fun_decl_empty_noexcept(int fun_decl: @fun_decl ref); +fun_decl_typedef_type( + unique int fun_decl: @fun_decl ref, + int typedeftype_id: @usertype ref +); + +param_decl_bind( + unique int id: @var_decl ref, + int index: int ref, + int fun_decl: @fun_decl ref +); + +#keyset[id, type_id] +var_decls( + int id: @var_decl, + int variable: @variable ref, + int type_id: @type ref, + string name: string ref, + int location: @location_default ref +); +var_def(unique int id: @var_decl ref); +var_decl_specifiers( + int id: @var_decl ref, + string name: string ref +) + +type_decls( + unique int id: @type_decl, + int type_id: @type ref, + int location: @location_default ref +); +type_def(unique int id: @type_decl ref); +type_decl_top( + unique int type_decl: @type_decl ref +); + +namespace_decls( + unique int id: @namespace_decl, + int namespace_id: @namespace ref, + int location: @location_default ref, + int bodylocation: @location_default ref +); + +usings( + unique int id: @using, + int element_id: @element ref, + int location: @location_default ref +); + +/** The element which contains the `using` declaration. */ +using_container( + int parent: @element ref, + int child: @using ref +); + +static_asserts( + unique int id: @static_assert, + int condition : @expr ref, + string message : string ref, + int location: @location_default ref +); + +// each function has an ordered list of parameters +#keyset[id, type_id] +#keyset[function, index, type_id] +params( + int id: @parameter, + int function: @functionorblock ref, + int index: int ref, + int type_id: @type ref +); + +overrides(int new: @function ref, int old: @function ref); + +#keyset[id, type_id] +membervariables( + int id: @membervariable, + int type_id: @type ref, + string name: string ref +); + +#keyset[id, type_id] +globalvariables( + int id: @globalvariable, + int type_id: @type ref, + string name: string ref +); + +#keyset[id, type_id] +localvariables( + int id: @localvariable, + int type_id: @type ref, + string name: string ref +); + +autoderivation( + unique int var: @variable ref, + int derivation_type: @type ref +); + +enumconstants( + unique int id: @enumconstant, + int parent: @usertype ref, + int index: int ref, + int type_id: @type ref, + string name: string ref, + int location: @location_default ref +); + +@variable = @localscopevariable | @globalvariable | @membervariable; + +@localscopevariable = @localvariable | @parameter; + +/* + Built-in types are the fundamental types, e.g., integral, floating, and void. + + case @builtintype.kind of + 1 = error + | 2 = unknown + | 3 = void + | 4 = boolean + | 5 = char + | 6 = unsigned_char + | 7 = signed_char + | 8 = short + | 9 = unsigned_short + | 10 = signed_short + | 11 = int + | 12 = unsigned_int + | 13 = signed_int + | 14 = long + | 15 = unsigned_long + | 16 = signed_long + | 17 = long_long + | 18 = unsigned_long_long + | 19 = signed_long_long + | 20 = __int8 // Microsoft-specific + | 21 = __int16 // Microsoft-specific + | 22 = __int32 // Microsoft-specific + | 23 = __int64 // Microsoft-specific + | 24 = float + | 25 = double + | 26 = long_double + | 27 = _Complex_float // C99-specific + | 28 = _Complex_double // C99-specific + | 29 = _Complex_long double // C99-specific + | 30 = _Imaginary_float // C99-specific + | 31 = _Imaginary_double // C99-specific + | 32 = _Imaginary_long_double // C99-specific + | 33 = wchar_t // Microsoft-specific + | 34 = decltype_nullptr // C++11 + | 35 = __int128 + | 36 = unsigned___int128 + | 37 = signed___int128 + | 38 = __float128 + | 39 = _Complex___float128 + | 40 = _Decimal32 + | 41 = _Decimal64 + | 42 = _Decimal128 + | 43 = char16_t + | 44 = char32_t + | 45 = _Float32 + | 46 = _Float32x + | 47 = _Float64 + | 48 = _Float64x + | 49 = _Float128 + | 50 = _Float128x + ; +*/ +builtintypes( + unique int id: @builtintype, + string name: string ref, + int kind: int ref, + int size: int ref, + int sign: int ref, + int alignment: int ref +); + +/* + Derived types are types that are directly derived from existing types and + point to, refer to, transform type data to return a new type. + + case @derivedtype.kind of + 1 = pointer + | 2 = reference + | 3 = type_with_specifiers + | 4 = array + | 5 = gnu_vector + | 6 = routineptr + | 7 = routinereference + | 8 = rvalue_reference // C++11 +// ... 9 type_conforming_to_protocols deprecated + | 10 = block + ; +*/ +derivedtypes( + unique int id: @derivedtype, + string name: string ref, + int kind: int ref, + int type_id: @type ref +); + +pointerishsize(unique int id: @derivedtype ref, + int size: int ref, + int alignment: int ref); + +arraysizes( + unique int id: @derivedtype ref, + int num_elements: int ref, + int bytesize: int ref, + int alignment: int ref +); + +typedefbase( + unique int id: @usertype ref, + int type_id: @type ref +); + +decltypes( + unique int id: @decltype, + int expr: @expr ref, + int base_type: @type ref, + boolean parentheses_would_change_meaning: boolean ref +); + +/* + case @usertype.kind of + 1 = struct + | 2 = class + | 3 = union + | 4 = enum + | 5 = typedef // classic C: typedef typedef type name + | 6 = template + | 7 = template_parameter + | 8 = template_template_parameter + | 9 = proxy_class // a proxy class associated with a template parameter +// ... 10 objc_class deprecated +// ... 11 objc_protocol deprecated +// ... 12 objc_category deprecated + | 13 = scoped_enum + | 14 = using_alias // a using name = type style typedef + ; +*/ +usertypes( + unique int id: @usertype, + string name: string ref, + int kind: int ref +); + +usertypesize( + unique int id: @usertype ref, + int size: int ref, + int alignment: int ref +); + +usertype_final(unique int id: @usertype ref); + +usertype_uuid( + unique int id: @usertype ref, + unique string uuid: string ref +); + +mangled_name( + unique int id: @declaration ref, + int mangled_name : @mangledname +); + +is_pod_class(unique int id: @usertype ref); +is_standard_layout_class(unique int id: @usertype ref); + +is_complete(unique int id: @usertype ref); + +is_class_template(unique int id: @usertype ref); +class_instantiation( + int to: @usertype ref, + int from: @usertype ref +); +class_template_argument( + int type_id: @usertype ref, + int index: int ref, + int arg_type: @type ref +); + +is_proxy_class_for( + unique int id: @usertype ref, + unique int templ_param_id: @usertype ref +); + +type_mentions( + unique int id: @type_mention, + int type_id: @type ref, + int location: @location ref, + // a_symbol_reference_kind from the EDG frontend. See symbol_ref.h there. + int kind: int ref +); + +is_function_template(unique int id: @function ref); +function_instantiation( + unique int to: @function ref, + int from: @function ref +); +function_template_argument( + int function_id: @function ref, + int index: int ref, + int arg_type: @type ref +); + +is_variable_template(unique int id: @variable ref); +variable_instantiation( + unique int to: @variable ref, + int from: @variable ref +); +variable_template_argument( + int variable_id: @variable ref, + int index: int ref, + int arg_type: @type ref +); + +/* + Fixed point types + precision(1) = short, precision(2) = default, precision(3) = long + is_unsigned(1) = unsigned is_unsigned(2) = signed + is_fract_type(1) = declared with _Fract + saturating(1) = declared with _Sat +*/ +/* TODO +fixedpointtypes( + unique int id: @fixedpointtype, + int precision: int ref, + int is_unsigned: int ref, + int is_fract_type: int ref, + int saturating: int ref); +*/ + +routinetypes( + unique int id: @routinetype, + int return_type: @type ref +); + +routinetypeargs( + int routine: @routinetype ref, + int index: int ref, + int type_id: @type ref +); + +ptrtomembers( + unique int id: @ptrtomember, + int type_id: @type ref, + int class_id: @type ref +); + +/* + specifiers for types, functions, and variables + + "public", + "protected", + "private", + + "const", + "volatile", + "static", + + "pure", + "virtual", + "sealed", // Microsoft + "__interface", // Microsoft + "inline", + "explicit", + + "near", // near far extension + "far", // near far extension + "__ptr32", // Microsoft + "__ptr64", // Microsoft + "__sptr", // Microsoft + "__uptr", // Microsoft + "dllimport", // Microsoft + "dllexport", // Microsoft + "thread", // Microsoft + "naked", // Microsoft + "microsoft_inline", // Microsoft + "forceinline", // Microsoft + "selectany", // Microsoft + "nothrow", // Microsoft + "novtable", // Microsoft + "noreturn", // Microsoft + "noinline", // Microsoft + "noalias", // Microsoft + "restrict", // Microsoft +*/ + +specifiers( + unique int id: @specifier, + unique string str: string ref +); + +typespecifiers( + int type_id: @type ref, + int spec_id: @specifier ref +); + +funspecifiers( + int func_id: @function ref, + int spec_id: @specifier ref +); + +varspecifiers( + int var_id: @accessible ref, + int spec_id: @specifier ref +); + +attributes( + unique int id: @attribute, + int kind: int ref, + string name: string ref, + string name_space: string ref, + int location: @location_default ref +); + +case @attribute.kind of + 0 = @gnuattribute +| 1 = @stdattribute +| 2 = @declspec +| 3 = @msattribute +| 4 = @alignas +// ... 5 @objc_propertyattribute deprecated +; + +attribute_args( + unique int id: @attribute_arg, + int kind: int ref, + int attribute: @attribute ref, + int index: int ref, + int location: @location_default ref +); + +case @attribute_arg.kind of + 0 = @attribute_arg_empty +| 1 = @attribute_arg_token +| 2 = @attribute_arg_constant +| 3 = @attribute_arg_type +; + +attribute_arg_value( + unique int arg: @attribute_arg ref, + string value: string ref +); +attribute_arg_type( + unique int arg: @attribute_arg ref, + int type_id: @type ref +); +attribute_arg_name( + unique int arg: @attribute_arg ref, + string name: string ref +); + +typeattributes( + int type_id: @type ref, + int spec_id: @attribute ref +); + +funcattributes( + int func_id: @function ref, + int spec_id: @attribute ref +); + +varattributes( + int var_id: @accessible ref, + int spec_id: @attribute ref +); + +stmtattributes( + int stmt_id: @stmt ref, + int spec_id: @attribute ref +); + +@type = @builtintype + | @derivedtype + | @usertype + /* TODO | @fixedpointtype */ + | @routinetype + | @ptrtomember + | @decltype; + +unspecifiedtype( + unique int type_id: @type ref, + int unspecified_type_id: @type ref +); + +member( + int parent: @type ref, + int index: int ref, + int child: @member ref +); + +@enclosingfunction_child = @usertype | @variable | @namespace + +enclosingfunction( + unique int child: @enclosingfunction_child ref, + int parent: @function ref +); + +derivations( + unique int derivation: @derivation, + int sub: @type ref, + int index: int ref, + int super: @type ref, + int location: @location_default ref +); + +derspecifiers( + int der_id: @derivation ref, + int spec_id: @specifier ref +); + +/** + * Contains the byte offset of the base class subobject within the derived + * class. Only holds for non-virtual base classes, but see table + * `virtual_base_offsets` for offsets of virtual base class subobjects. + */ +direct_base_offsets( + unique int der_id: @derivation ref, + int offset: int ref +); + +/** + * Contains the byte offset of the virtual base class subobject for class + * `super` within a most-derived object of class `sub`. `super` can be either a + * direct or indirect base class. + */ +#keyset[sub, super] +virtual_base_offsets( + int sub: @usertype ref, + int super: @usertype ref, + int offset: int ref +); + +frienddecls( + unique int id: @frienddecl, + int type_id: @type ref, + int decl_id: @declaration ref, + int location: @location_default ref +); + +@declaredtype = @usertype ; + +@declaration = @function + | @declaredtype + | @variable + | @enumconstant + | @frienddecl; + +@member = @membervariable + | @function + | @declaredtype + | @enumconstant; + +@locatable = @diagnostic + | @declaration + | @ppd_include + | @ppd_define + | @macroinvocation + /*| @funcall*/ + | @xmllocatable + | @attribute + | @attribute_arg; + +@namedscope = @namespace | @usertype; + +@element = @locatable + | @file + | @folder + | @specifier + | @type + | @expr + | @namespace + | @initialiser + | @stmt + | @derivation + | @comment + | @preprocdirect + | @fun_decl + | @var_decl + | @type_decl + | @namespace_decl + | @using + | @namequalifier + | @specialnamequalifyingelement + | @static_assert + | @type_mention + | @lambdacapture; + +@exprparent = @element; + +comments( + unique int id: @comment, + string contents: string ref, + int location: @location_default ref +); + +commentbinding( + int id: @comment ref, + int element: @element ref +); + +exprconv( + int converted: @expr ref, + unique int conversion: @expr ref +); + +compgenerated(unique int id: @element ref); + +/** + * `destructor_call` destructs the `i`'th entity that should be + * destructed following `element`. Note that entities should be + * destructed in reverse construction order, so for a given `element` + * these should be called from highest to lowest `i`. + */ +#keyset[element, destructor_call] +#keyset[element, i] +synthetic_destructor_call( + int element: @element ref, + int i: int ref, + int destructor_call: @routineexpr ref +); + +namespaces( + unique int id: @namespace, + string name: string ref +); + +namespace_inline( + unique int id: @namespace ref +); + +namespacembrs( + int parentid: @namespace ref, + unique int memberid: @namespacembr ref +); + +@namespacembr = @declaration | @namespace; + +exprparents( + int expr_id: @expr ref, + int child_index: int ref, + int parent_id: @exprparent ref +); + +expr_isload(unique int expr_id: @expr ref); + +@cast = @c_style_cast + | @const_cast + | @dynamic_cast + | @reinterpret_cast + | @static_cast + ; + +/* +case @conversion.kind of + 0 = @simple_conversion // a numeric conversion, qualification conversion, or a reinterpret_cast +| 1 = @bool_conversion // conversion to 'bool' +| 2 = @base_class_conversion // a derived-to-base conversion +| 3 = @derived_class_conversion // a base-to-derived conversion +| 4 = @pm_base_class_conversion // a derived-to-base conversion of a pointer to member +| 5 = @pm_derived_class_conversion // a base-to-derived conversion of a pointer to member +| 6 = @glvalue_adjust // an adjustment of the type of a glvalue +| 7 = @prvalue_adjust // an adjustment of the type of a prvalue +; +*/ +/** + * Describes the semantics represented by a cast expression. This is largely + * independent of the source syntax of the cast, so it is separate from the + * regular expression kind. + */ +conversionkinds( + unique int expr_id: @cast ref, + int kind: int ref +); + +/* +case @funbindexpr.kind of + 0 = @normal_call // a normal call +| 1 = @virtual_call // a virtual call +| 2 = @adl_call // a call whose target is only found by ADL +; +*/ +iscall(unique int caller: @funbindexpr ref, int kind: int ref); + +numtemplatearguments( + unique int expr_id: @expr ref, + int num: int ref +); + +specialnamequalifyingelements( + unique int id: @specialnamequalifyingelement, + unique string name: string ref +); + +@namequalifiableelement = @expr | @namequalifier; +@namequalifyingelement = @namespace + | @specialnamequalifyingelement + | @usertype; + +namequalifiers( + unique int id: @namequalifier, + unique int qualifiableelement: @namequalifiableelement ref, + int qualifyingelement: @namequalifyingelement ref, + int location: @location_default ref +); + +varbind( + int expr: @varbindexpr ref, + int var: @accessible ref +); + +funbind( + int expr: @funbindexpr ref, + int fun: @function ref +); + +@any_new_expr = @new_expr + | @new_array_expr; + +@new_or_delete_expr = @any_new_expr + | @delete_expr + | @delete_array_expr; + +/* + case @allocator.form of + 0 = plain + | 1 = alignment + ; +*/ + +/** + * The allocator function associated with a `new` or `new[]` expression. + * The `form` column specified whether the allocation call contains an alignment + * argument. + */ +expr_allocator( + unique int expr: @any_new_expr ref, + int func: @function ref, + int form: int ref +); + +/* + case @deallocator.form of + 0 = plain + | 1 = size + | 2 = alignment + | 3 = size_and_alignment + ; +*/ + +/** + * The deallocator function associated with a `delete`, `delete[]`, `new`, or + * `new[]` expression. For a `new` or `new[]` expression, the deallocator is the + * one used to free memory if the initialization throws an exception. + * The `form` column specifies whether the deallocation call contains a size + * argument, and alignment argument, or both. + */ +expr_deallocator( + unique int expr: @new_or_delete_expr ref, + int func: @function ref, + int form: int ref +); + +/** + * Holds if the `@conditionalexpr` is of the two operand form + * `guard ? : false`. + */ +expr_cond_two_operand( + unique int cond: @conditionalexpr ref +); + +/** + * The guard of `@conditionalexpr` `guard ? true : false` + */ +expr_cond_guard( + unique int cond: @conditionalexpr ref, + int guard: @expr ref +); + +/** + * The expression used when the guard of `@conditionalexpr` + * `guard ? true : false` holds. For the two operand form + * `guard ?: false` consider using `expr_cond_guard` instead. + */ +expr_cond_true( + unique int cond: @conditionalexpr ref, + int true: @expr ref +); + +/** + * The expression used when the guard of `@conditionalexpr` + * `guard ? true : false` does not hold. + */ +expr_cond_false( + unique int cond: @conditionalexpr ref, + int false: @expr ref +); + +/** A string representation of the value. */ +values( + unique int id: @value, + string str: string ref +); + +/** The actual text in the source code for the value, if any. */ +valuetext( + unique int id: @value ref, + string text: string ref +); + +valuebind( + int val: @value ref, + unique int expr: @expr ref +); + +fieldoffsets( + unique int id: @variable ref, + int byteoffset: int ref, + int bitoffset: int ref +); + +bitfield( + unique int id: @variable ref, + int bits: int ref, + int declared_bits: int ref +); + +/* TODO +memberprefix( + int member: @expr ref, + int prefix: @expr ref +); +*/ + +/* + kind(1) = mbrcallexpr + kind(2) = mbrptrcallexpr + kind(3) = mbrptrmbrcallexpr + kind(4) = ptrmbrptrmbrcallexpr + kind(5) = mbrreadexpr // x.y + kind(6) = mbrptrreadexpr // p->y + kind(7) = mbrptrmbrreadexpr // x.*pm + kind(8) = mbrptrmbrptrreadexpr // x->*pm + kind(9) = staticmbrreadexpr // static x.y + kind(10) = staticmbrptrreadexpr // static p->y +*/ +/* TODO +memberaccess( + int member: @expr ref, + int kind: int ref +); +*/ + +initialisers( + unique int init: @initialiser, + int var: @accessible ref, + unique int expr: @expr ref, + int location: @location_expr ref +); + +/** + * An ancestor for the expression, for cases in which we cannot + * otherwise find the expression's parent. + */ +expr_ancestor( + int exp: @expr ref, + int ancestor: @element ref +); + +exprs( + unique int id: @expr, + int kind: int ref, + int location: @location_expr ref +); + +/* + case @value.category of + 1 = prval + | 2 = xval + | 3 = lval + ; +*/ +expr_types( + int id: @expr ref, + int typeid: @type ref, + int value_category: int ref +); + +case @expr.kind of + 1 = @errorexpr +| 2 = @address_of // & AddressOfExpr +| 3 = @reference_to // ReferenceToExpr (implicit?) +| 4 = @indirect // * PointerDereferenceExpr +| 5 = @ref_indirect // ReferenceDereferenceExpr (implicit?) +// ... +| 8 = @array_to_pointer // (???) +| 9 = @vacuous_destructor_call // VacuousDestructorCall +// ... +| 11 = @assume // Microsoft +| 12 = @parexpr +| 13 = @arithnegexpr +| 14 = @unaryplusexpr +| 15 = @complementexpr +| 16 = @notexpr +| 17 = @conjugation // GNU ~ operator +| 18 = @realpartexpr // GNU __real +| 19 = @imagpartexpr // GNU __imag +| 20 = @postincrexpr +| 21 = @postdecrexpr +| 22 = @preincrexpr +| 23 = @predecrexpr +| 24 = @conditionalexpr +| 25 = @addexpr +| 26 = @subexpr +| 27 = @mulexpr +| 28 = @divexpr +| 29 = @remexpr +| 30 = @jmulexpr // C99 mul imaginary +| 31 = @jdivexpr // C99 div imaginary +| 32 = @fjaddexpr // C99 add real + imaginary +| 33 = @jfaddexpr // C99 add imaginary + real +| 34 = @fjsubexpr // C99 sub real - imaginary +| 35 = @jfsubexpr // C99 sub imaginary - real +| 36 = @paddexpr // pointer add (pointer + int or int + pointer) +| 37 = @psubexpr // pointer sub (pointer - integer) +| 38 = @pdiffexpr // difference between two pointers +| 39 = @lshiftexpr +| 40 = @rshiftexpr +| 41 = @andexpr +| 42 = @orexpr +| 43 = @xorexpr +| 44 = @eqexpr +| 45 = @neexpr +| 46 = @gtexpr +| 47 = @ltexpr +| 48 = @geexpr +| 49 = @leexpr +| 50 = @minexpr // GNU minimum +| 51 = @maxexpr // GNU maximum +| 52 = @assignexpr +| 53 = @assignaddexpr +| 54 = @assignsubexpr +| 55 = @assignmulexpr +| 56 = @assigndivexpr +| 57 = @assignremexpr +| 58 = @assignlshiftexpr +| 59 = @assignrshiftexpr +| 60 = @assignandexpr +| 61 = @assignorexpr +| 62 = @assignxorexpr +| 63 = @assignpaddexpr // assign pointer add +| 64 = @assignpsubexpr // assign pointer sub +| 65 = @andlogicalexpr +| 66 = @orlogicalexpr +| 67 = @commaexpr +| 68 = @subscriptexpr // access to member of an array, e.g., a[5] +// ... 69 @objc_subscriptexpr deprecated +// ... 70 @cmdaccess deprecated +// ... +| 73 = @virtfunptrexpr +| 74 = @callexpr +// ... 75 @msgexpr_normal deprecated +// ... 76 @msgexpr_super deprecated +// ... 77 @atselectorexpr deprecated +// ... 78 @atprotocolexpr deprecated +| 79 = @vastartexpr +| 80 = @vaargexpr +| 81 = @vaendexpr +| 82 = @vacopyexpr +// ... 83 @atencodeexpr deprecated +| 84 = @varaccess +| 85 = @thisaccess +// ... 86 @objc_box_expr deprecated +| 87 = @new_expr +| 88 = @delete_expr +| 89 = @throw_expr +| 90 = @condition_decl // a variable declared in a condition, e.g., if(int x = y > 2) +| 91 = @braced_init_list +| 92 = @type_id +| 93 = @runtime_sizeof +| 94 = @runtime_alignof +| 95 = @sizeof_pack +| 96 = @expr_stmt // GNU extension +| 97 = @routineexpr +| 98 = @type_operand // used to access a type in certain contexts (haven't found any examples yet....) +| 99 = @offsetofexpr // offsetof ::= type and field +| 100 = @hasassignexpr // __has_assign ::= type +| 101 = @hascopyexpr // __has_copy ::= type +| 102 = @hasnothrowassign // __has_nothrow_assign ::= type +| 103 = @hasnothrowconstr // __has_nothrow_constructor ::= type +| 104 = @hasnothrowcopy // __has_nothrow_copy ::= type +| 105 = @hastrivialassign // __has_trivial_assign ::= type +| 106 = @hastrivialconstr // __has_trivial_constructor ::= type +| 107 = @hastrivialcopy // __has_trivial_copy ::= type +| 108 = @hasuserdestr // __has_user_destructor ::= type +| 109 = @hasvirtualdestr // __has_virtual_destructor ::= type +| 110 = @isabstractexpr // __is_abstract ::= type +| 111 = @isbaseofexpr // __is_base_of ::= type type +| 112 = @isclassexpr // __is_class ::= type +| 113 = @isconvtoexpr // __is_convertible_to ::= type type +| 114 = @isemptyexpr // __is_empty ::= type +| 115 = @isenumexpr // __is_enum ::= type +| 116 = @ispodexpr // __is_pod ::= type +| 117 = @ispolyexpr // __is_polymorphic ::= type +| 118 = @isunionexpr // __is_union ::= type +| 119 = @typescompexpr // GNU __builtin_types_compatible ::= type type +| 120 = @intaddrexpr // EDG internal builtin, used to implement offsetof +// ... +| 122 = @hastrivialdestructor // __has_trivial_destructor ::= type +| 123 = @literal +| 124 = @uuidof +| 127 = @aggregateliteral +| 128 = @delete_array_expr +| 129 = @new_array_expr +// ... 130 @objc_array_literal deprecated +// ... 131 @objc_dictionary_literal deprecated +| 132 = @foldexpr +// ... +| 200 = @ctordirectinit +| 201 = @ctorvirtualinit +| 202 = @ctorfieldinit +| 203 = @ctordelegatinginit +| 204 = @dtordirectdestruct +| 205 = @dtorvirtualdestruct +| 206 = @dtorfielddestruct +// ... +| 210 = @static_cast +| 211 = @reinterpret_cast +| 212 = @const_cast +| 213 = @dynamic_cast +| 214 = @c_style_cast +| 215 = @lambdaexpr +| 216 = @param_ref +| 217 = @noopexpr +// ... +| 294 = @istriviallyconstructibleexpr +| 295 = @isdestructibleexpr +| 296 = @isnothrowdestructibleexpr +| 297 = @istriviallydestructibleexpr +| 298 = @istriviallyassignableexpr +| 299 = @isnothrowassignableexpr +| 300 = @istrivialexpr +| 301 = @isstandardlayoutexpr +| 302 = @istriviallycopyableexpr +| 303 = @isliteraltypeexpr +| 304 = @hastrivialmoveconstructorexpr +| 305 = @hastrivialmoveassignexpr +| 306 = @hasnothrowmoveassignexpr +| 307 = @isconstructibleexpr +| 308 = @isnothrowconstructibleexpr +| 309 = @hasfinalizerexpr +| 310 = @isdelegateexpr +| 311 = @isinterfaceclassexpr +| 312 = @isrefarrayexpr +| 313 = @isrefclassexpr +| 314 = @issealedexpr +| 315 = @issimplevalueclassexpr +| 316 = @isvalueclassexpr +| 317 = @isfinalexpr +| 319 = @noexceptexpr +| 320 = @builtinshufflevector +| 321 = @builtinchooseexpr +| 322 = @builtinaddressof +| 323 = @vec_fill +| 324 = @builtinconvertvector +| 325 = @builtincomplex +; + +new_allocated_type( + unique int expr: @new_expr ref, + int type_id: @type ref +); + +new_array_allocated_type( + unique int expr: @new_array_expr ref, + int type_id: @type ref +); + +/** + * The field being initialized by an initializer expression within an aggregate + * initializer for a class/struct/union. + */ +#keyset[aggregate, field] +aggregate_field_init( + int aggregate: @aggregateliteral ref, + int initializer: @expr ref, + int field: @membervariable ref +); + +/** + * The index of the element being initialized by an initializer expression + * within an aggregate initializer for an array. + */ +#keyset[aggregate, element_index] +aggregate_array_init( + int aggregate: @aggregateliteral ref, + int initializer: @expr ref, + int element_index: int ref +); + +@ctorinit = @ctordirectinit + | @ctorvirtualinit + | @ctorfieldinit + | @ctordelegatinginit; +@dtordestruct = @dtordirectdestruct + | @dtorvirtualdestruct + | @dtorfielddestruct; + + +condition_decl_bind( + unique int expr: @condition_decl ref, + unique int decl: @declaration ref +); + +typeid_bind( + unique int expr: @type_id ref, + int type_id: @type ref +); + +uuidof_bind( + unique int expr: @uuidof ref, + int type_id: @type ref +); + +@runtime_sizeof_or_alignof = @runtime_sizeof | @runtime_alignof; + +sizeof_bind( + unique int expr: @runtime_sizeof_or_alignof ref, + int type_id: @type ref +); + +code_block( + unique int block: @literal ref, + unique int routine: @function ref +); + +lambdas( + unique int expr: @lambdaexpr ref, + string default_capture: string ref, + boolean has_explicit_return_type: boolean ref +); + +lambda_capture( + unique int id: @lambdacapture, + int lambda: @lambdaexpr ref, + int index: int ref, + int field: @membervariable ref, + boolean captured_by_reference: boolean ref, + boolean is_implicit: boolean ref, + int location: @location_default ref +); + +@funbindexpr = @routineexpr + | @new_expr + | @delete_expr + | @delete_array_expr + | @ctordirectinit + | @ctorvirtualinit + | @ctordelegatinginit + | @dtordirectdestruct + | @dtorvirtualdestruct; + +@varbindexpr = @varaccess | @ctorfieldinit | @dtorfielddestruct; +@addressable = @function | @variable ; +@accessible = @addressable | @enumconstant ; + +fold( + int expr: @foldexpr ref, + string operator: string ref, + boolean is_left_fold: boolean ref +); + +stmts( + unique int id: @stmt, + int kind: int ref, + int location: @location_stmt ref +); + +case @stmt.kind of + 1 = @stmt_expr +| 2 = @stmt_if +| 3 = @stmt_while +| 4 = @stmt_goto +| 5 = @stmt_label +| 6 = @stmt_return +| 7 = @stmt_block +| 8 = @stmt_end_test_while // do { ... } while ( ... ) +| 9 = @stmt_for +| 10 = @stmt_switch_case +| 11 = @stmt_switch +| 13 = @stmt_asm // "asm" statement or the body of an asm function +| 15 = @stmt_try_block +| 16 = @stmt_microsoft_try // Microsoft +| 17 = @stmt_decl +| 18 = @stmt_set_vla_size // C99 +| 19 = @stmt_vla_decl // C99 +| 25 = @stmt_assigned_goto // GNU +| 26 = @stmt_empty +| 27 = @stmt_continue +| 28 = @stmt_break +| 29 = @stmt_range_based_for // C++11 +// ... 30 @stmt_at_autoreleasepool_block deprecated +// ... 31 @stmt_objc_for_in deprecated +// ... 32 @stmt_at_synchronized deprecated +| 33 = @stmt_handler +// ... 34 @stmt_finally_end deprecated +| 35 = @stmt_constexpr_if +; + +type_vla( + int type_id: @type ref, + int decl: @stmt_vla_decl ref +); + +variable_vla( + int var: @variable ref, + int decl: @stmt_vla_decl ref +); + +if_then( + unique int if_stmt: @stmt_if ref, + int then_id: @stmt ref +); + +if_else( + unique int if_stmt: @stmt_if ref, + int else_id: @stmt ref +); + +constexpr_if_then( + unique int constexpr_if_stmt: @stmt_constexpr_if ref, + int then_id: @stmt ref +); + +constexpr_if_else( + unique int constexpr_if_stmt: @stmt_constexpr_if ref, + int else_id: @stmt ref +); + +while_body( + unique int while_stmt: @stmt_while ref, + int body_id: @stmt ref +); + +do_body( + unique int do_stmt: @stmt_end_test_while ref, + int body_id: @stmt ref +); + +#keyset[switch_stmt, index] +switch_case( + int switch_stmt: @stmt_switch ref, + int index: int ref, + int case_id: @stmt_switch_case ref +); + +switch_body( + unique int switch_stmt: @stmt_switch ref, + int body_id: @stmt ref +); + +for_initialization( + unique int for_stmt: @stmt_for ref, + int init_id: @stmt ref +); + +for_condition( + unique int for_stmt: @stmt_for ref, + int condition_id: @expr ref +); + +for_update( + unique int for_stmt: @stmt_for ref, + int update_id: @expr ref +); + +for_body( + unique int for_stmt: @stmt_for ref, + int body_id: @stmt ref +); + +@stmtparent = @stmt | @expr_stmt ; +stmtparents( + unique int id: @stmt ref, + int index: int ref, + int parent: @stmtparent ref +); + +ishandler(unique int block: @stmt_block ref); + +@cfgnode = @stmt | @expr | @function | @initialiser ; +successors( + int from: @cfgnode ref, + int to: @cfgnode ref +); + +truecond( + unique int from: @cfgnode ref, + int to: @cfgnode ref +); + +falsecond( + unique int from: @cfgnode ref, + int to: @cfgnode ref +); + +stmt_decl_bind( + int stmt: @stmt_decl ref, + int num: int ref, + int decl: @declaration ref +); + +stmt_decl_entry_bind( + int stmt: @stmt_decl ref, + int num: int ref, + int decl_entry: @element ref +); + +@functionorblock = @function | @stmt_block; + +blockscope( + unique int block: @stmt_block ref, + int enclosing: @functionorblock ref +); + +@jump = @stmt_goto | @stmt_break | @stmt_continue; + +@jumporlabel = @jump | @stmt_label | @literal; + +jumpinfo( + unique int id: @jumporlabel ref, + string str: string ref, + int target: @stmt ref +); + +preprocdirects( + unique int id: @preprocdirect, + int kind: int ref, + int location: @location_default ref +); +case @preprocdirect.kind of + 0 = @ppd_if +| 1 = @ppd_ifdef +| 2 = @ppd_ifndef +| 3 = @ppd_elif +| 4 = @ppd_else +| 5 = @ppd_endif +| 6 = @ppd_plain_include +| 7 = @ppd_define +| 8 = @ppd_undef +| 9 = @ppd_line +| 10 = @ppd_error +| 11 = @ppd_pragma +| 12 = @ppd_objc_import +| 13 = @ppd_include_next +| 18 = @ppd_warning +; + +@ppd_include = @ppd_plain_include | @ppd_objc_import | @ppd_include_next; + +@ppd_branch = @ppd_if | @ppd_ifdef | @ppd_ifndef | @ppd_elif; + +preprocpair( + int begin : @ppd_branch ref, + int elseelifend : @preprocdirect ref +); + +preproctrue(int branch : @ppd_branch ref); +preprocfalse(int branch : @ppd_branch ref); + +preproctext( + unique int id: @preprocdirect ref, + string head: string ref, + string body: string ref +); + +includes( + unique int id: @ppd_include ref, + int included: @file ref +); + +link_targets( + unique int id: @link_target, + int binary: @file ref +); + +link_parent( + int element : @element ref, + int link_target : @link_target ref +); + +/* XML Files */ + +xmlEncoding(unique int id: @file ref, string encoding: string ref); + +xmlDTDs( + unique int id: @xmldtd, + string root: string ref, + string publicId: string ref, + string systemId: string ref, + int fileid: @file ref +); + +xmlElements( + unique int id: @xmlelement, + string name: string ref, + int parentid: @xmlparent ref, + int idx: int ref, + int fileid: @file ref +); + +xmlAttrs( + unique int id: @xmlattribute, + int elementid: @xmlelement ref, + string name: string ref, + string value: string ref, + int idx: int ref, + int fileid: @file ref +); + +xmlNs( + int id: @xmlnamespace, + string prefixName: string ref, + string URI: string ref, + int fileid: @file ref +); + +xmlHasNs( + int elementId: @xmlnamespaceable ref, + int nsId: @xmlnamespace ref, + int fileid: @file ref +); + +xmlComments( + unique int id: @xmlcomment, + string text: string ref, + int parentid: @xmlparent ref, + int fileid: @file ref +); + +xmlChars( + unique int id: @xmlcharacters, + string text: string ref, + int parentid: @xmlparent ref, + int idx: int ref, + int isCDATA: int ref, + int fileid: @file ref +); + +@xmlparent = @file | @xmlelement; +@xmlnamespaceable = @xmlelement | @xmlattribute; + +xmllocations( + int xmlElement: @xmllocatable ref, + int location: @location_default ref +); + +@xmllocatable = @xmlcharacters + | @xmlelement + | @xmlcomment + | @xmlattribute + | @xmldtd + | @file + | @xmlnamespace; diff --git a/cpp/upgrades/98a075d5495d7be7ede26557708cf22cfa3964ef/semmlecode.cpp.dbscheme b/cpp/upgrades/98a075d5495d7be7ede26557708cf22cfa3964ef/semmlecode.cpp.dbscheme new file mode 100644 index 00000000000..bd182f697bf --- /dev/null +++ b/cpp/upgrades/98a075d5495d7be7ede26557708cf22cfa3964ef/semmlecode.cpp.dbscheme @@ -0,0 +1,1933 @@ + +/** + * An invocation of the compiler. Note that more than one file may be + * compiled per invocation. For example, this command compiles three + * source files: + * + * gcc -c f1.c f2.c f3.c + * + * The `id` simply identifies the invocation, while `cwd` is the working + * directory from which the compiler was invoked. + */ +compilations( + /** + * An invocation of the compiler. Note that more than one file may + * be compiled per invocation. For example, this command compiles + * three source files: + * + * gcc -c f1.c f2.c f3.c + */ + unique int id : @compilation, + string cwd : string ref +); + +/** + * The arguments that were passed to the extractor for a compiler + * invocation. If `id` is for the compiler invocation + * + * gcc -c f1.c f2.c f3.c + * + * then typically there will be rows for + * + * num | arg + * --- | --- + * 0 | *path to extractor* + * 1 | `--mimic` + * 2 | `/usr/bin/gcc` + * 3 | `-c` + * 4 | f1.c + * 5 | f2.c + * 6 | f3.c + */ +#keyset[id, num] +compilation_args( + int id : @compilation ref, + int num : int ref, + string arg : string ref +); + +/** + * The source files that are compiled by a compiler invocation. + * If `id` is for the compiler invocation + * + * gcc -c f1.c f2.c f3.c + * + * then there will be rows for + * + * num | arg + * --- | --- + * 0 | f1.c + * 1 | f2.c + * 2 | f3.c + * + * Note that even if those files `#include` headers, those headers + * do not appear as rows. + */ +#keyset[id, num] +compilation_compiling_files( + int id : @compilation ref, + int num : int ref, + int file : @file ref +); + +/** + * The time taken by the extractor for a compiler invocation. + * + * For each file `num`, there will be rows for + * + * kind | seconds + * ---- | --- + * 1 | CPU seconds used by the extractor frontend + * 2 | Elapsed seconds during the extractor frontend + * 3 | CPU seconds used by the extractor backend + * 4 | Elapsed seconds during the extractor backend + */ +#keyset[id, num, kind] +compilation_time( + int id : @compilation ref, + int num : int ref, + /* kind: + 1 = frontend_cpu_seconds + 2 = frontend_elapsed_seconds + 3 = extractor_cpu_seconds + 4 = extractor_elapsed_seconds + */ + int kind : int ref, + float seconds : float ref +); + +/** + * An error or warning generated by the extractor. + * The diagnostic message `diagnostic` was generated during compiler + * invocation `compilation`, and is the `file_number_diagnostic_number`th + * message generated while extracting the `file_number`th file of that + * invocation. + */ +#keyset[compilation, file_number, file_number_diagnostic_number] +diagnostic_for( + int diagnostic : @diagnostic ref, + int compilation : @compilation ref, + int file_number : int ref, + int file_number_diagnostic_number : int ref +); + +/** + * If extraction was successful, then `cpu_seconds` and + * `elapsed_seconds` are the CPU time and elapsed time (respectively) + * that extraction took for compiler invocation `id`. + */ +compilation_finished( + unique int id : @compilation ref, + float cpu_seconds : float ref, + float elapsed_seconds : float ref +); + + +/** + * External data, loaded from CSV files during snapshot creation. See + * [Tutorial: Incorporating external data](https://help.semmle.com/wiki/display/SD/Tutorial%3A+Incorporating+external+data) + * for more information. + */ +externalData( + int id : @externalDataElement, + string path : string ref, + int column: int ref, + string value : string ref +); + +/** + * The date of the snapshot. + */ +snapshotDate(unique date snapshotDate : date ref); + +/** + * The source location of the snapshot. + */ +sourceLocationPrefix(string prefix : string ref); + +/** + * Data used by the 'duplicate code' detection. + */ +duplicateCode( + unique int id : @duplication, + string relativePath : string ref, + int equivClass : int ref +); + +/** + * Data used by the 'similar code' detection. + */ +similarCode( + unique int id : @similarity, + string relativePath : string ref, + int equivClass : int ref +); + +/** + * Data used by the 'duplicate code' and 'similar code' detection. + */ +@duplication_or_similarity = @duplication | @similarity + +/** + * Data used by the 'duplicate code' and 'similar code' detection. + */ +#keyset[id, offset] +tokens( + int id : @duplication_or_similarity ref, + int offset : int ref, + int beginLine : int ref, + int beginColumn : int ref, + int endLine : int ref, + int endColumn : int ref +); + +/** + * Information about packages that provide code used during compilation. + * The `id` is just a unique identifier. + * The `namespace` is typically the name of the package manager that + * provided the package (e.g. "dpkg" or "yum"). + * The `package_name` is the name of the package, and `version` is its + * version (as a string). + */ +external_packages( + unique int id: @external_package, + string namespace : string ref, + string package_name : string ref, + string version : string ref +); + +/** + * Holds if File `fileid` was provided by package `package`. + */ +header_to_external_package( + int fileid : @file ref, + int package : @external_package ref +); + +/* + * Version history + */ + +svnentries( + unique int id : @svnentry, + string revision : string ref, + string author : string ref, + date revisionDate : date ref, + int changeSize : int ref +) + +svnaffectedfiles( + int id : @svnentry ref, + int file : @file ref, + string action : string ref +) + +svnentrymsg( + unique int id : @svnentry ref, + string message : string ref +) + +svnchurn( + int commit : @svnentry ref, + int file : @file ref, + int addedLines : int ref, + int deletedLines : int ref +) + +/* + * C++ dbscheme + */ + +@location = @location_stmt | @location_expr | @location_default ; + +/** + * The location of an element that is not an expression or a statement. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `file`. + * For more information, see + * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html). + */ +locations_default( + /** The location of an element that is not an expression or a statement. */ + unique int id: @location_default, + int container: @container ref, + int startLine: int ref, + int startColumn: int ref, + int endLine: int ref, + int endColumn: int ref +); + +/** + * The location of a statement. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `file`. + * For more information, see + * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html). + */ +locations_stmt( + /** The location of a statement. */ + unique int id: @location_stmt, + int container: @container ref, + int startLine: int ref, + int startColumn: int ref, + int endLine: int ref, + int endColumn: int ref +); + +/** + * The location of an expression. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `file`. + * For more information, see + * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html). + */ +locations_expr( + /** The location of an expression. */ + unique int id: @location_expr, + int container: @container ref, + int startLine: int ref, + int startColumn: int ref, + int endLine: int ref, + int endColumn: int ref +); + +/** An element for which line-count information is available. */ +@sourceline = @file | @function | @variable | @enumconstant | @xmllocatable; + +numlines( + int element_id: @sourceline ref, + int num_lines: int ref, + int num_code: int ref, + int num_comment: int ref +); + +diagnostics( + unique int id: @diagnostic, + int severity: int ref, + string error_tag: string ref, + string error_message: string ref, + string full_error_message: string ref, + int location: @location_default ref +); + +/* + fromSource(0) = unknown, + fromSource(1) = from source, + fromSource(2) = from library +*/ +files( + unique int id: @file, + string name: string ref, + string simple: string ref, + string ext: string ref, + int fromSource: int ref +); + +folders( + unique int id: @folder, + string name: string ref, + string simple: string ref +); + +@container = @folder | @file + +containerparent( + int parent: @container ref, + unique int child: @container ref +); + +fileannotations( + int id: @file ref, + int kind: int ref, + string name: string ref, + string value: string ref +); + +inmacroexpansion( + int id: @element ref, + int inv: @macroinvocation ref +); + +affectedbymacroexpansion( + int id: @element ref, + int inv: @macroinvocation ref +); + +/* + case @macroinvocations.kind of + 1 = macro expansion + | 2 = other macro reference + ; +*/ +macroinvocations( + unique int id: @macroinvocation, + int macro_id: @ppd_define ref, + int location: @location_default ref, + int kind: int ref +); + +macroparent( + unique int id: @macroinvocation ref, + int parent_id: @macroinvocation ref +); + +// a macroinvocation may be part of another location +// the way to find a constant expression that uses a macro +// is thus to find a constant expression that has a location +// to which a macro invocation is bound +macrolocationbind( + int id: @macroinvocation ref, + int location: @location ref +); + +#keyset[invocation, argument_index] +macro_argument_unexpanded( + int invocation: @macroinvocation ref, + int argument_index: int ref, + string text: string ref +); + +#keyset[invocation, argument_index] +macro_argument_expanded( + int invocation: @macroinvocation ref, + int argument_index: int ref, + string text: string ref +); + +/* + case @function.kind of + 1 = normal + | 2 = constructor + | 3 = destructor + | 4 = conversion + | 5 = operator + | 6 = builtin // GCC built-in functions, e.g. __builtin___memcpy_chk + ; +*/ +functions( + unique int id: @function, + string name: string ref, + int kind: int ref +); + +function_entry_point(int id: @function ref, unique int entry_point: @stmt ref); + +function_return_type(int id: @function ref, int return_type: @type ref); + +purefunctions(unique int id: @function ref); + +function_deleted(unique int id: @function ref); + +function_defaulted(unique int id: @function ref); + + + +#keyset[id, type_id] +fun_decls( + int id: @fun_decl, + int function: @function ref, + int type_id: @type ref, + string name: string ref, + int location: @location_default ref +); +fun_def(unique int id: @fun_decl ref); +fun_specialized(unique int id: @fun_decl ref); +fun_implicit(unique int id: @fun_decl ref); +fun_decl_specifiers( + int id: @fun_decl ref, + string name: string ref +) +#keyset[fun_decl, index] +fun_decl_throws( + int fun_decl: @fun_decl ref, + int index: int ref, + int type_id: @type ref +); +/* an empty throw specification is different from none */ +fun_decl_empty_throws(unique int fun_decl: @fun_decl ref); +fun_decl_noexcept( + int fun_decl: @fun_decl ref, + int constant: @expr ref +); +fun_decl_empty_noexcept(int fun_decl: @fun_decl ref); +fun_decl_typedef_type( + unique int fun_decl: @fun_decl ref, + int typedeftype_id: @usertype ref +); + +param_decl_bind( + unique int id: @var_decl ref, + int index: int ref, + int fun_decl: @fun_decl ref +); + +#keyset[id, type_id] +var_decls( + int id: @var_decl, + int variable: @variable ref, + int type_id: @type ref, + string name: string ref, + int location: @location_default ref +); +var_def(unique int id: @var_decl ref); +var_decl_specifiers( + int id: @var_decl ref, + string name: string ref +) + +type_decls( + unique int id: @type_decl, + int type_id: @type ref, + int location: @location_default ref +); +type_def(unique int id: @type_decl ref); +type_decl_top( + unique int type_decl: @type_decl ref +); + +namespace_decls( + unique int id: @namespace_decl, + int namespace_id: @namespace ref, + int location: @location_default ref, + int bodylocation: @location_default ref +); + +usings( + unique int id: @using, + int element_id: @element ref, + int location: @location_default ref +); + +/** The element which contains the `using` declaration. */ +using_container( + int parent: @element ref, + int child: @using ref +); + +static_asserts( + unique int id: @static_assert, + int condition : @expr ref, + string message : string ref, + int location: @location_default ref +); + +// each function has an ordered list of parameters +#keyset[id, type_id] +#keyset[function, index, type_id] +params( + int id: @parameter, + int function: @functionorblock ref, + int index: int ref, + int type_id: @type ref +); + +overrides(int new: @function ref, int old: @function ref); + +#keyset[id, type_id] +membervariables( + int id: @membervariable, + int type_id: @type ref, + string name: string ref +); + +#keyset[id, type_id] +globalvariables( + int id: @globalvariable, + int type_id: @type ref, + string name: string ref +); + +#keyset[id, type_id] +localvariables( + int id: @localvariable, + int type_id: @type ref, + string name: string ref +); + +autoderivation( + unique int var: @variable ref, + int derivation_type: @type ref +); + +enumconstants( + unique int id: @enumconstant, + int parent: @usertype ref, + int index: int ref, + int type_id: @type ref, + string name: string ref, + int location: @location_default ref +); + +@variable = @localscopevariable | @globalvariable | @membervariable; + +@localscopevariable = @localvariable | @parameter; + +/* + Built-in types are the fundamental types, e.g., integral, floating, and void. + + case @builtintype.kind of + 1 = error + | 2 = unknown + | 3 = void + | 4 = boolean + | 5 = char + | 6 = unsigned_char + | 7 = signed_char + | 8 = short + | 9 = unsigned_short + | 10 = signed_short + | 11 = int + | 12 = unsigned_int + | 13 = signed_int + | 14 = long + | 15 = unsigned_long + | 16 = signed_long + | 17 = long_long + | 18 = unsigned_long_long + | 19 = signed_long_long + | 20 = __int8 // Microsoft-specific + | 21 = __int16 // Microsoft-specific + | 22 = __int32 // Microsoft-specific + | 23 = __int64 // Microsoft-specific + | 24 = float + | 25 = double + | 26 = long_double + | 27 = _Complex_float // C99-specific + | 28 = _Complex_double // C99-specific + | 29 = _Complex_long double // C99-specific + | 30 = _Imaginary_float // C99-specific + | 31 = _Imaginary_double // C99-specific + | 32 = _Imaginary_long_double // C99-specific + | 33 = wchar_t // Microsoft-specific + | 34 = decltype_nullptr // C++11 + | 35 = __int128 + | 36 = unsigned___int128 + | 37 = signed___int128 + | 38 = __float128 + | 39 = _Complex___float128 + | 40 = _Decimal32 + | 41 = _Decimal64 + | 42 = _Decimal128 + | 43 = char16_t + | 44 = char32_t + | 45 = _Float32 + | 46 = _Float32x + | 47 = _Float64 + | 48 = _Float64x + | 49 = _Float128 + | 50 = _Float128x + ; +*/ +builtintypes( + unique int id: @builtintype, + string name: string ref, + int kind: int ref, + int size: int ref, + int sign: int ref, + int alignment: int ref +); + +/* + Derived types are types that are directly derived from existing types and + point to, refer to, transform type data to return a new type. + + case @derivedtype.kind of + 1 = pointer + | 2 = reference + | 3 = type_with_specifiers + | 4 = array + | 5 = gnu_vector + | 6 = routineptr + | 7 = routinereference + | 8 = rvalue_reference // C++11 +// ... 9 type_conforming_to_protocols deprecated + | 10 = block + ; +*/ +derivedtypes( + unique int id: @derivedtype, + string name: string ref, + int kind: int ref, + int type_id: @type ref +); + +pointerishsize(unique int id: @derivedtype ref, + int size: int ref, + int alignment: int ref); + +arraysizes( + unique int id: @derivedtype ref, + int num_elements: int ref, + int bytesize: int ref, + int alignment: int ref +); + +typedefbase( + unique int id: @usertype ref, + int type_id: @type ref +); + +decltypes( + unique int id: @decltype, + int expr: @expr ref, + int base_type: @type ref, + boolean parentheses_would_change_meaning: boolean ref +); + +/* + case @usertype.kind of + 1 = struct + | 2 = class + | 3 = union + | 4 = enum + | 5 = typedef // classic C: typedef typedef type name + | 6 = template + | 7 = template_parameter + | 8 = template_template_parameter + | 9 = proxy_class // a proxy class associated with a template parameter +// ... 10 objc_class deprecated +// ... 11 objc_protocol deprecated +// ... 12 objc_category deprecated + | 13 = scoped_enum + | 14 = using_alias // a using name = type style typedef + ; +*/ +usertypes( + unique int id: @usertype, + string name: string ref, + int kind: int ref +); + +usertypesize( + unique int id: @usertype ref, + int size: int ref, + int alignment: int ref +); + +usertype_final(unique int id: @usertype ref); + +usertype_uuid( + unique int id: @usertype ref, + unique string uuid: string ref +); + +mangled_name( + unique int id: @declaration ref, + int mangled_name : @mangledname +); + +is_pod_class(unique int id: @usertype ref); +is_standard_layout_class(unique int id: @usertype ref); + +is_complete(unique int id: @usertype ref); + +is_class_template(unique int id: @usertype ref); +class_instantiation( + int to: @usertype ref, + int from: @usertype ref +); +class_template_argument( + int type_id: @usertype ref, + int index: int ref, + int arg_type: @type ref +); +class_template_argument_value( + int type_id: @usertype ref, + int index: int ref, + int arg_value: @expr ref +); + +is_proxy_class_for( + unique int id: @usertype ref, + unique int templ_param_id: @usertype ref +); + +type_mentions( + unique int id: @type_mention, + int type_id: @type ref, + int location: @location ref, + // a_symbol_reference_kind from the EDG frontend. See symbol_ref.h there. + int kind: int ref +); + +is_function_template(unique int id: @function ref); +function_instantiation( + unique int to: @function ref, + int from: @function ref +); +function_template_argument( + int function_id: @function ref, + int index: int ref, + int arg_type: @type ref +); +function_template_argument_value( + int function_id: @function ref, + int index: int ref, + int arg_value: @expr ref +); + +is_variable_template(unique int id: @variable ref); +variable_instantiation( + unique int to: @variable ref, + int from: @variable ref +); +variable_template_argument( + int variable_id: @variable ref, + int index: int ref, + int arg_type: @type ref +); +variable_template_argument_value( + int variable_id: @variable ref, + int index: int ref, + int arg_value: @expr ref +); + +/* + Fixed point types + precision(1) = short, precision(2) = default, precision(3) = long + is_unsigned(1) = unsigned is_unsigned(2) = signed + is_fract_type(1) = declared with _Fract + saturating(1) = declared with _Sat +*/ +/* TODO +fixedpointtypes( + unique int id: @fixedpointtype, + int precision: int ref, + int is_unsigned: int ref, + int is_fract_type: int ref, + int saturating: int ref); +*/ + +routinetypes( + unique int id: @routinetype, + int return_type: @type ref +); + +routinetypeargs( + int routine: @routinetype ref, + int index: int ref, + int type_id: @type ref +); + +ptrtomembers( + unique int id: @ptrtomember, + int type_id: @type ref, + int class_id: @type ref +); + +/* + specifiers for types, functions, and variables + + "public", + "protected", + "private", + + "const", + "volatile", + "static", + + "pure", + "virtual", + "sealed", // Microsoft + "__interface", // Microsoft + "inline", + "explicit", + + "near", // near far extension + "far", // near far extension + "__ptr32", // Microsoft + "__ptr64", // Microsoft + "__sptr", // Microsoft + "__uptr", // Microsoft + "dllimport", // Microsoft + "dllexport", // Microsoft + "thread", // Microsoft + "naked", // Microsoft + "microsoft_inline", // Microsoft + "forceinline", // Microsoft + "selectany", // Microsoft + "nothrow", // Microsoft + "novtable", // Microsoft + "noreturn", // Microsoft + "noinline", // Microsoft + "noalias", // Microsoft + "restrict", // Microsoft +*/ + +specifiers( + unique int id: @specifier, + unique string str: string ref +); + +typespecifiers( + int type_id: @type ref, + int spec_id: @specifier ref +); + +funspecifiers( + int func_id: @function ref, + int spec_id: @specifier ref +); + +varspecifiers( + int var_id: @accessible ref, + int spec_id: @specifier ref +); + +attributes( + unique int id: @attribute, + int kind: int ref, + string name: string ref, + string name_space: string ref, + int location: @location_default ref +); + +case @attribute.kind of + 0 = @gnuattribute +| 1 = @stdattribute +| 2 = @declspec +| 3 = @msattribute +| 4 = @alignas +// ... 5 @objc_propertyattribute deprecated +; + +attribute_args( + unique int id: @attribute_arg, + int kind: int ref, + int attribute: @attribute ref, + int index: int ref, + int location: @location_default ref +); + +case @attribute_arg.kind of + 0 = @attribute_arg_empty +| 1 = @attribute_arg_token +| 2 = @attribute_arg_constant +| 3 = @attribute_arg_type +; + +attribute_arg_value( + unique int arg: @attribute_arg ref, + string value: string ref +); +attribute_arg_type( + unique int arg: @attribute_arg ref, + int type_id: @type ref +); +attribute_arg_name( + unique int arg: @attribute_arg ref, + string name: string ref +); + +typeattributes( + int type_id: @type ref, + int spec_id: @attribute ref +); + +funcattributes( + int func_id: @function ref, + int spec_id: @attribute ref +); + +varattributes( + int var_id: @accessible ref, + int spec_id: @attribute ref +); + +stmtattributes( + int stmt_id: @stmt ref, + int spec_id: @attribute ref +); + +@type = @builtintype + | @derivedtype + | @usertype + /* TODO | @fixedpointtype */ + | @routinetype + | @ptrtomember + | @decltype; + +unspecifiedtype( + unique int type_id: @type ref, + int unspecified_type_id: @type ref +); + +member( + int parent: @type ref, + int index: int ref, + int child: @member ref +); + +@enclosingfunction_child = @usertype | @variable | @namespace + +enclosingfunction( + unique int child: @enclosingfunction_child ref, + int parent: @function ref +); + +derivations( + unique int derivation: @derivation, + int sub: @type ref, + int index: int ref, + int super: @type ref, + int location: @location_default ref +); + +derspecifiers( + int der_id: @derivation ref, + int spec_id: @specifier ref +); + +/** + * Contains the byte offset of the base class subobject within the derived + * class. Only holds for non-virtual base classes, but see table + * `virtual_base_offsets` for offsets of virtual base class subobjects. + */ +direct_base_offsets( + unique int der_id: @derivation ref, + int offset: int ref +); + +/** + * Contains the byte offset of the virtual base class subobject for class + * `super` within a most-derived object of class `sub`. `super` can be either a + * direct or indirect base class. + */ +#keyset[sub, super] +virtual_base_offsets( + int sub: @usertype ref, + int super: @usertype ref, + int offset: int ref +); + +frienddecls( + unique int id: @frienddecl, + int type_id: @type ref, + int decl_id: @declaration ref, + int location: @location_default ref +); + +@declaredtype = @usertype ; + +@declaration = @function + | @declaredtype + | @variable + | @enumconstant + | @frienddecl; + +@member = @membervariable + | @function + | @declaredtype + | @enumconstant; + +@locatable = @diagnostic + | @declaration + | @ppd_include + | @ppd_define + | @macroinvocation + /*| @funcall*/ + | @xmllocatable + | @attribute + | @attribute_arg; + +@namedscope = @namespace | @usertype; + +@element = @locatable + | @file + | @folder + | @specifier + | @type + | @expr + | @namespace + | @initialiser + | @stmt + | @derivation + | @comment + | @preprocdirect + | @fun_decl + | @var_decl + | @type_decl + | @namespace_decl + | @using + | @namequalifier + | @specialnamequalifyingelement + | @static_assert + | @type_mention + | @lambdacapture; + +@exprparent = @element; + +comments( + unique int id: @comment, + string contents: string ref, + int location: @location_default ref +); + +commentbinding( + int id: @comment ref, + int element: @element ref +); + +exprconv( + int converted: @expr ref, + unique int conversion: @expr ref +); + +compgenerated(unique int id: @element ref); + +/** + * `destructor_call` destructs the `i`'th entity that should be + * destructed following `element`. Note that entities should be + * destructed in reverse construction order, so for a given `element` + * these should be called from highest to lowest `i`. + */ +#keyset[element, destructor_call] +#keyset[element, i] +synthetic_destructor_call( + int element: @element ref, + int i: int ref, + int destructor_call: @routineexpr ref +); + +namespaces( + unique int id: @namespace, + string name: string ref +); + +namespace_inline( + unique int id: @namespace ref +); + +namespacembrs( + int parentid: @namespace ref, + unique int memberid: @namespacembr ref +); + +@namespacembr = @declaration | @namespace; + +exprparents( + int expr_id: @expr ref, + int child_index: int ref, + int parent_id: @exprparent ref +); + +expr_isload(unique int expr_id: @expr ref); + +@cast = @c_style_cast + | @const_cast + | @dynamic_cast + | @reinterpret_cast + | @static_cast + ; + +/* +case @conversion.kind of + 0 = @simple_conversion // a numeric conversion, qualification conversion, or a reinterpret_cast +| 1 = @bool_conversion // conversion to 'bool' +| 2 = @base_class_conversion // a derived-to-base conversion +| 3 = @derived_class_conversion // a base-to-derived conversion +| 4 = @pm_base_class_conversion // a derived-to-base conversion of a pointer to member +| 5 = @pm_derived_class_conversion // a base-to-derived conversion of a pointer to member +| 6 = @glvalue_adjust // an adjustment of the type of a glvalue +| 7 = @prvalue_adjust // an adjustment of the type of a prvalue +; +*/ +/** + * Describes the semantics represented by a cast expression. This is largely + * independent of the source syntax of the cast, so it is separate from the + * regular expression kind. + */ +conversionkinds( + unique int expr_id: @cast ref, + int kind: int ref +); + +/* +case @funbindexpr.kind of + 0 = @normal_call // a normal call +| 1 = @virtual_call // a virtual call +| 2 = @adl_call // a call whose target is only found by ADL +; +*/ +iscall(unique int caller: @funbindexpr ref, int kind: int ref); + +numtemplatearguments( + unique int expr_id: @expr ref, + int num: int ref +); + +specialnamequalifyingelements( + unique int id: @specialnamequalifyingelement, + unique string name: string ref +); + +@namequalifiableelement = @expr | @namequalifier; +@namequalifyingelement = @namespace + | @specialnamequalifyingelement + | @usertype; + +namequalifiers( + unique int id: @namequalifier, + unique int qualifiableelement: @namequalifiableelement ref, + int qualifyingelement: @namequalifyingelement ref, + int location: @location_default ref +); + +varbind( + int expr: @varbindexpr ref, + int var: @accessible ref +); + +funbind( + int expr: @funbindexpr ref, + int fun: @function ref +); + +@any_new_expr = @new_expr + | @new_array_expr; + +@new_or_delete_expr = @any_new_expr + | @delete_expr + | @delete_array_expr; + +/* + case @allocator.form of + 0 = plain + | 1 = alignment + ; +*/ + +/** + * The allocator function associated with a `new` or `new[]` expression. + * The `form` column specified whether the allocation call contains an alignment + * argument. + */ +expr_allocator( + unique int expr: @any_new_expr ref, + int func: @function ref, + int form: int ref +); + +/* + case @deallocator.form of + 0 = plain + | 1 = size + | 2 = alignment + | 3 = size_and_alignment + ; +*/ + +/** + * The deallocator function associated with a `delete`, `delete[]`, `new`, or + * `new[]` expression. For a `new` or `new[]` expression, the deallocator is the + * one used to free memory if the initialization throws an exception. + * The `form` column specifies whether the deallocation call contains a size + * argument, and alignment argument, or both. + */ +expr_deallocator( + unique int expr: @new_or_delete_expr ref, + int func: @function ref, + int form: int ref +); + +/** + * Holds if the `@conditionalexpr` is of the two operand form + * `guard ? : false`. + */ +expr_cond_two_operand( + unique int cond: @conditionalexpr ref +); + +/** + * The guard of `@conditionalexpr` `guard ? true : false` + */ +expr_cond_guard( + unique int cond: @conditionalexpr ref, + int guard: @expr ref +); + +/** + * The expression used when the guard of `@conditionalexpr` + * `guard ? true : false` holds. For the two operand form + * `guard ?: false` consider using `expr_cond_guard` instead. + */ +expr_cond_true( + unique int cond: @conditionalexpr ref, + int true: @expr ref +); + +/** + * The expression used when the guard of `@conditionalexpr` + * `guard ? true : false` does not hold. + */ +expr_cond_false( + unique int cond: @conditionalexpr ref, + int false: @expr ref +); + +/** A string representation of the value. */ +values( + unique int id: @value, + string str: string ref +); + +/** The actual text in the source code for the value, if any. */ +valuetext( + unique int id: @value ref, + string text: string ref +); + +valuebind( + int val: @value ref, + unique int expr: @expr ref +); + +fieldoffsets( + unique int id: @variable ref, + int byteoffset: int ref, + int bitoffset: int ref +); + +bitfield( + unique int id: @variable ref, + int bits: int ref, + int declared_bits: int ref +); + +/* TODO +memberprefix( + int member: @expr ref, + int prefix: @expr ref +); +*/ + +/* + kind(1) = mbrcallexpr + kind(2) = mbrptrcallexpr + kind(3) = mbrptrmbrcallexpr + kind(4) = ptrmbrptrmbrcallexpr + kind(5) = mbrreadexpr // x.y + kind(6) = mbrptrreadexpr // p->y + kind(7) = mbrptrmbrreadexpr // x.*pm + kind(8) = mbrptrmbrptrreadexpr // x->*pm + kind(9) = staticmbrreadexpr // static x.y + kind(10) = staticmbrptrreadexpr // static p->y +*/ +/* TODO +memberaccess( + int member: @expr ref, + int kind: int ref +); +*/ + +initialisers( + unique int init: @initialiser, + int var: @accessible ref, + unique int expr: @expr ref, + int location: @location_expr ref +); + +/** + * An ancestor for the expression, for cases in which we cannot + * otherwise find the expression's parent. + */ +expr_ancestor( + int exp: @expr ref, + int ancestor: @element ref +); + +exprs( + unique int id: @expr, + int kind: int ref, + int location: @location_expr ref +); + +/* + case @value.category of + 1 = prval + | 2 = xval + | 3 = lval + ; +*/ +expr_types( + int id: @expr ref, + int typeid: @type ref, + int value_category: int ref +); + +case @expr.kind of + 1 = @errorexpr +| 2 = @address_of // & AddressOfExpr +| 3 = @reference_to // ReferenceToExpr (implicit?) +| 4 = @indirect // * PointerDereferenceExpr +| 5 = @ref_indirect // ReferenceDereferenceExpr (implicit?) +// ... +| 8 = @array_to_pointer // (???) +| 9 = @vacuous_destructor_call // VacuousDestructorCall +// ... +| 11 = @assume // Microsoft +| 12 = @parexpr +| 13 = @arithnegexpr +| 14 = @unaryplusexpr +| 15 = @complementexpr +| 16 = @notexpr +| 17 = @conjugation // GNU ~ operator +| 18 = @realpartexpr // GNU __real +| 19 = @imagpartexpr // GNU __imag +| 20 = @postincrexpr +| 21 = @postdecrexpr +| 22 = @preincrexpr +| 23 = @predecrexpr +| 24 = @conditionalexpr +| 25 = @addexpr +| 26 = @subexpr +| 27 = @mulexpr +| 28 = @divexpr +| 29 = @remexpr +| 30 = @jmulexpr // C99 mul imaginary +| 31 = @jdivexpr // C99 div imaginary +| 32 = @fjaddexpr // C99 add real + imaginary +| 33 = @jfaddexpr // C99 add imaginary + real +| 34 = @fjsubexpr // C99 sub real - imaginary +| 35 = @jfsubexpr // C99 sub imaginary - real +| 36 = @paddexpr // pointer add (pointer + int or int + pointer) +| 37 = @psubexpr // pointer sub (pointer - integer) +| 38 = @pdiffexpr // difference between two pointers +| 39 = @lshiftexpr +| 40 = @rshiftexpr +| 41 = @andexpr +| 42 = @orexpr +| 43 = @xorexpr +| 44 = @eqexpr +| 45 = @neexpr +| 46 = @gtexpr +| 47 = @ltexpr +| 48 = @geexpr +| 49 = @leexpr +| 50 = @minexpr // GNU minimum +| 51 = @maxexpr // GNU maximum +| 52 = @assignexpr +| 53 = @assignaddexpr +| 54 = @assignsubexpr +| 55 = @assignmulexpr +| 56 = @assigndivexpr +| 57 = @assignremexpr +| 58 = @assignlshiftexpr +| 59 = @assignrshiftexpr +| 60 = @assignandexpr +| 61 = @assignorexpr +| 62 = @assignxorexpr +| 63 = @assignpaddexpr // assign pointer add +| 64 = @assignpsubexpr // assign pointer sub +| 65 = @andlogicalexpr +| 66 = @orlogicalexpr +| 67 = @commaexpr +| 68 = @subscriptexpr // access to member of an array, e.g., a[5] +// ... 69 @objc_subscriptexpr deprecated +// ... 70 @cmdaccess deprecated +// ... +| 73 = @virtfunptrexpr +| 74 = @callexpr +// ... 75 @msgexpr_normal deprecated +// ... 76 @msgexpr_super deprecated +// ... 77 @atselectorexpr deprecated +// ... 78 @atprotocolexpr deprecated +| 79 = @vastartexpr +| 80 = @vaargexpr +| 81 = @vaendexpr +| 82 = @vacopyexpr +// ... 83 @atencodeexpr deprecated +| 84 = @varaccess +| 85 = @thisaccess +// ... 86 @objc_box_expr deprecated +| 87 = @new_expr +| 88 = @delete_expr +| 89 = @throw_expr +| 90 = @condition_decl // a variable declared in a condition, e.g., if(int x = y > 2) +| 91 = @braced_init_list +| 92 = @type_id +| 93 = @runtime_sizeof +| 94 = @runtime_alignof +| 95 = @sizeof_pack +| 96 = @expr_stmt // GNU extension +| 97 = @routineexpr +| 98 = @type_operand // used to access a type in certain contexts (haven't found any examples yet....) +| 99 = @offsetofexpr // offsetof ::= type and field +| 100 = @hasassignexpr // __has_assign ::= type +| 101 = @hascopyexpr // __has_copy ::= type +| 102 = @hasnothrowassign // __has_nothrow_assign ::= type +| 103 = @hasnothrowconstr // __has_nothrow_constructor ::= type +| 104 = @hasnothrowcopy // __has_nothrow_copy ::= type +| 105 = @hastrivialassign // __has_trivial_assign ::= type +| 106 = @hastrivialconstr // __has_trivial_constructor ::= type +| 107 = @hastrivialcopy // __has_trivial_copy ::= type +| 108 = @hasuserdestr // __has_user_destructor ::= type +| 109 = @hasvirtualdestr // __has_virtual_destructor ::= type +| 110 = @isabstractexpr // __is_abstract ::= type +| 111 = @isbaseofexpr // __is_base_of ::= type type +| 112 = @isclassexpr // __is_class ::= type +| 113 = @isconvtoexpr // __is_convertible_to ::= type type +| 114 = @isemptyexpr // __is_empty ::= type +| 115 = @isenumexpr // __is_enum ::= type +| 116 = @ispodexpr // __is_pod ::= type +| 117 = @ispolyexpr // __is_polymorphic ::= type +| 118 = @isunionexpr // __is_union ::= type +| 119 = @typescompexpr // GNU __builtin_types_compatible ::= type type +| 120 = @intaddrexpr // EDG internal builtin, used to implement offsetof +// ... +| 122 = @hastrivialdestructor // __has_trivial_destructor ::= type +| 123 = @literal +| 124 = @uuidof +| 127 = @aggregateliteral +| 128 = @delete_array_expr +| 129 = @new_array_expr +// ... 130 @objc_array_literal deprecated +// ... 131 @objc_dictionary_literal deprecated +| 132 = @foldexpr +// ... +| 200 = @ctordirectinit +| 201 = @ctorvirtualinit +| 202 = @ctorfieldinit +| 203 = @ctordelegatinginit +| 204 = @dtordirectdestruct +| 205 = @dtorvirtualdestruct +| 206 = @dtorfielddestruct +// ... +| 210 = @static_cast +| 211 = @reinterpret_cast +| 212 = @const_cast +| 213 = @dynamic_cast +| 214 = @c_style_cast +| 215 = @lambdaexpr +| 216 = @param_ref +| 217 = @noopexpr +// ... +| 294 = @istriviallyconstructibleexpr +| 295 = @isdestructibleexpr +| 296 = @isnothrowdestructibleexpr +| 297 = @istriviallydestructibleexpr +| 298 = @istriviallyassignableexpr +| 299 = @isnothrowassignableexpr +| 300 = @istrivialexpr +| 301 = @isstandardlayoutexpr +| 302 = @istriviallycopyableexpr +| 303 = @isliteraltypeexpr +| 304 = @hastrivialmoveconstructorexpr +| 305 = @hastrivialmoveassignexpr +| 306 = @hasnothrowmoveassignexpr +| 307 = @isconstructibleexpr +| 308 = @isnothrowconstructibleexpr +| 309 = @hasfinalizerexpr +| 310 = @isdelegateexpr +| 311 = @isinterfaceclassexpr +| 312 = @isrefarrayexpr +| 313 = @isrefclassexpr +| 314 = @issealedexpr +| 315 = @issimplevalueclassexpr +| 316 = @isvalueclassexpr +| 317 = @isfinalexpr +| 319 = @noexceptexpr +| 320 = @builtinshufflevector +| 321 = @builtinchooseexpr +| 322 = @builtinaddressof +| 323 = @vec_fill +| 324 = @builtinconvertvector +| 325 = @builtincomplex +; + +new_allocated_type( + unique int expr: @new_expr ref, + int type_id: @type ref +); + +new_array_allocated_type( + unique int expr: @new_array_expr ref, + int type_id: @type ref +); + +/** + * The field being initialized by an initializer expression within an aggregate + * initializer for a class/struct/union. + */ +#keyset[aggregate, field] +aggregate_field_init( + int aggregate: @aggregateliteral ref, + int initializer: @expr ref, + int field: @membervariable ref +); + +/** + * The index of the element being initialized by an initializer expression + * within an aggregate initializer for an array. + */ +#keyset[aggregate, element_index] +aggregate_array_init( + int aggregate: @aggregateliteral ref, + int initializer: @expr ref, + int element_index: int ref +); + +@ctorinit = @ctordirectinit + | @ctorvirtualinit + | @ctorfieldinit + | @ctordelegatinginit; +@dtordestruct = @dtordirectdestruct + | @dtorvirtualdestruct + | @dtorfielddestruct; + + +condition_decl_bind( + unique int expr: @condition_decl ref, + unique int decl: @declaration ref +); + +typeid_bind( + unique int expr: @type_id ref, + int type_id: @type ref +); + +uuidof_bind( + unique int expr: @uuidof ref, + int type_id: @type ref +); + +@runtime_sizeof_or_alignof = @runtime_sizeof | @runtime_alignof; + +sizeof_bind( + unique int expr: @runtime_sizeof_or_alignof ref, + int type_id: @type ref +); + +code_block( + unique int block: @literal ref, + unique int routine: @function ref +); + +lambdas( + unique int expr: @lambdaexpr ref, + string default_capture: string ref, + boolean has_explicit_return_type: boolean ref +); + +lambda_capture( + unique int id: @lambdacapture, + int lambda: @lambdaexpr ref, + int index: int ref, + int field: @membervariable ref, + boolean captured_by_reference: boolean ref, + boolean is_implicit: boolean ref, + int location: @location_default ref +); + +@funbindexpr = @routineexpr + | @new_expr + | @delete_expr + | @delete_array_expr + | @ctordirectinit + | @ctorvirtualinit + | @ctordelegatinginit + | @dtordirectdestruct + | @dtorvirtualdestruct; + +@varbindexpr = @varaccess | @ctorfieldinit | @dtorfielddestruct; +@addressable = @function | @variable ; +@accessible = @addressable | @enumconstant ; + +fold( + int expr: @foldexpr ref, + string operator: string ref, + boolean is_left_fold: boolean ref +); + +stmts( + unique int id: @stmt, + int kind: int ref, + int location: @location_stmt ref +); + +case @stmt.kind of + 1 = @stmt_expr +| 2 = @stmt_if +| 3 = @stmt_while +| 4 = @stmt_goto +| 5 = @stmt_label +| 6 = @stmt_return +| 7 = @stmt_block +| 8 = @stmt_end_test_while // do { ... } while ( ... ) +| 9 = @stmt_for +| 10 = @stmt_switch_case +| 11 = @stmt_switch +| 13 = @stmt_asm // "asm" statement or the body of an asm function +| 15 = @stmt_try_block +| 16 = @stmt_microsoft_try // Microsoft +| 17 = @stmt_decl +| 18 = @stmt_set_vla_size // C99 +| 19 = @stmt_vla_decl // C99 +| 25 = @stmt_assigned_goto // GNU +| 26 = @stmt_empty +| 27 = @stmt_continue +| 28 = @stmt_break +| 29 = @stmt_range_based_for // C++11 +// ... 30 @stmt_at_autoreleasepool_block deprecated +// ... 31 @stmt_objc_for_in deprecated +// ... 32 @stmt_at_synchronized deprecated +| 33 = @stmt_handler +// ... 34 @stmt_finally_end deprecated +| 35 = @stmt_constexpr_if +; + +type_vla( + int type_id: @type ref, + int decl: @stmt_vla_decl ref +); + +variable_vla( + int var: @variable ref, + int decl: @stmt_vla_decl ref +); + +if_then( + unique int if_stmt: @stmt_if ref, + int then_id: @stmt ref +); + +if_else( + unique int if_stmt: @stmt_if ref, + int else_id: @stmt ref +); + +constexpr_if_then( + unique int constexpr_if_stmt: @stmt_constexpr_if ref, + int then_id: @stmt ref +); + +constexpr_if_else( + unique int constexpr_if_stmt: @stmt_constexpr_if ref, + int else_id: @stmt ref +); + +while_body( + unique int while_stmt: @stmt_while ref, + int body_id: @stmt ref +); + +do_body( + unique int do_stmt: @stmt_end_test_while ref, + int body_id: @stmt ref +); + +#keyset[switch_stmt, index] +switch_case( + int switch_stmt: @stmt_switch ref, + int index: int ref, + int case_id: @stmt_switch_case ref +); + +switch_body( + unique int switch_stmt: @stmt_switch ref, + int body_id: @stmt ref +); + +for_initialization( + unique int for_stmt: @stmt_for ref, + int init_id: @stmt ref +); + +for_condition( + unique int for_stmt: @stmt_for ref, + int condition_id: @expr ref +); + +for_update( + unique int for_stmt: @stmt_for ref, + int update_id: @expr ref +); + +for_body( + unique int for_stmt: @stmt_for ref, + int body_id: @stmt ref +); + +@stmtparent = @stmt | @expr_stmt ; +stmtparents( + unique int id: @stmt ref, + int index: int ref, + int parent: @stmtparent ref +); + +ishandler(unique int block: @stmt_block ref); + +@cfgnode = @stmt | @expr | @function | @initialiser ; +successors( + int from: @cfgnode ref, + int to: @cfgnode ref +); + +truecond( + unique int from: @cfgnode ref, + int to: @cfgnode ref +); + +falsecond( + unique int from: @cfgnode ref, + int to: @cfgnode ref +); + +stmt_decl_bind( + int stmt: @stmt_decl ref, + int num: int ref, + int decl: @declaration ref +); + +stmt_decl_entry_bind( + int stmt: @stmt_decl ref, + int num: int ref, + int decl_entry: @element ref +); + +@functionorblock = @function | @stmt_block; + +blockscope( + unique int block: @stmt_block ref, + int enclosing: @functionorblock ref +); + +@jump = @stmt_goto | @stmt_break | @stmt_continue; + +@jumporlabel = @jump | @stmt_label | @literal; + +jumpinfo( + unique int id: @jumporlabel ref, + string str: string ref, + int target: @stmt ref +); + +preprocdirects( + unique int id: @preprocdirect, + int kind: int ref, + int location: @location_default ref +); +case @preprocdirect.kind of + 0 = @ppd_if +| 1 = @ppd_ifdef +| 2 = @ppd_ifndef +| 3 = @ppd_elif +| 4 = @ppd_else +| 5 = @ppd_endif +| 6 = @ppd_plain_include +| 7 = @ppd_define +| 8 = @ppd_undef +| 9 = @ppd_line +| 10 = @ppd_error +| 11 = @ppd_pragma +| 12 = @ppd_objc_import +| 13 = @ppd_include_next +| 18 = @ppd_warning +; + +@ppd_include = @ppd_plain_include | @ppd_objc_import | @ppd_include_next; + +@ppd_branch = @ppd_if | @ppd_ifdef | @ppd_ifndef | @ppd_elif; + +preprocpair( + int begin : @ppd_branch ref, + int elseelifend : @preprocdirect ref +); + +preproctrue(int branch : @ppd_branch ref); +preprocfalse(int branch : @ppd_branch ref); + +preproctext( + unique int id: @preprocdirect ref, + string head: string ref, + string body: string ref +); + +includes( + unique int id: @ppd_include ref, + int included: @file ref +); + +link_targets( + unique int id: @link_target, + int binary: @file ref +); + +link_parent( + int element : @element ref, + int link_target : @link_target ref +); + +/* XML Files */ + +xmlEncoding(unique int id: @file ref, string encoding: string ref); + +xmlDTDs( + unique int id: @xmldtd, + string root: string ref, + string publicId: string ref, + string systemId: string ref, + int fileid: @file ref +); + +xmlElements( + unique int id: @xmlelement, + string name: string ref, + int parentid: @xmlparent ref, + int idx: int ref, + int fileid: @file ref +); + +xmlAttrs( + unique int id: @xmlattribute, + int elementid: @xmlelement ref, + string name: string ref, + string value: string ref, + int idx: int ref, + int fileid: @file ref +); + +xmlNs( + int id: @xmlnamespace, + string prefixName: string ref, + string URI: string ref, + int fileid: @file ref +); + +xmlHasNs( + int elementId: @xmlnamespaceable ref, + int nsId: @xmlnamespace ref, + int fileid: @file ref +); + +xmlComments( + unique int id: @xmlcomment, + string text: string ref, + int parentid: @xmlparent ref, + int fileid: @file ref +); + +xmlChars( + unique int id: @xmlcharacters, + string text: string ref, + int parentid: @xmlparent ref, + int idx: int ref, + int isCDATA: int ref, + int fileid: @file ref +); + +@xmlparent = @file | @xmlelement; +@xmlnamespaceable = @xmlelement | @xmlattribute; + +xmllocations( + int xmlElement: @xmllocatable ref, + int location: @location_default ref +); + +@xmllocatable = @xmlcharacters + | @xmlelement + | @xmlcomment + | @xmlattribute + | @xmldtd + | @file + | @xmlnamespace; diff --git a/cpp/upgrades/98a075d5495d7be7ede26557708cf22cfa3964ef/upgrade.properties b/cpp/upgrades/98a075d5495d7be7ede26557708cf22cfa3964ef/upgrade.properties new file mode 100644 index 00000000000..3388d1abe28 --- /dev/null +++ b/cpp/upgrades/98a075d5495d7be7ede26557708cf22cfa3964ef/upgrade.properties @@ -0,0 +1,2 @@ +description: Add support for value template paramters. +compatibility: partial From 4ab87291f35bb831dae94d314f91308bd4c0c2d1 Mon Sep 17 00:00:00 2001 From: Matthew Gretton-Dann Date: Fri, 18 Oct 2019 15:29:16 +0100 Subject: [PATCH 119/148] C++: Further use of TemplateArgumentValue --- cpp/ql/src/semmle/code/cpp/Declaration.qll | 6 ++++++ cpp/ql/src/semmle/code/cpp/exprs/Call.qll | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/cpp/ql/src/semmle/code/cpp/Declaration.qll b/cpp/ql/src/semmle/code/cpp/Declaration.qll index a3a93aaca96..a10faf0f918 100644 --- a/cpp/ql/src/semmle/code/cpp/Declaration.qll +++ b/cpp/ql/src/semmle/code/cpp/Declaration.qll @@ -197,6 +197,12 @@ abstract class Declaration extends Locatable, @declaration { */ final Type getATemplateArgument() { result = getTemplateArgument(_) } + /** + * Gets a template argument used to instantiate this declaration from a template. + * When called on a template, this will return a template parameter value. + */ + final Expr getATemplateArgumentValue() { result = getTemplateArgumentValue(_) } + /** * Gets the `i`th template argument used to instantiate this declaration from a * template. When called on a template, this will return the `i`th template parameter. diff --git a/cpp/ql/src/semmle/code/cpp/exprs/Call.qll b/cpp/ql/src/semmle/code/cpp/exprs/Call.qll index 882918316f6..85807cf0d14 100644 --- a/cpp/ql/src/semmle/code/cpp/exprs/Call.qll +++ b/cpp/ql/src/semmle/code/cpp/exprs/Call.qll @@ -141,15 +141,27 @@ class FunctionCall extends Call, @funbindexpr { /** Gets an explicit template argument for this call. */ Type getAnExplicitTemplateArgument() { result = getExplicitTemplateArgument(_) } + /** Gets an explicit template argument value for this call. */ + Expr getAnExplicitTemplateArgumentValue() { result = getExplicitTemplateArgumentValue(_) } + /** Gets a template argument for this call. */ Type getATemplateArgument() { result = getTarget().getATemplateArgument() } + /** Gets a template argument value for this call. */ + Expr getATemplateArgumentValue() { result = getTarget().getATemplateArgumentValue() } + /** Gets the nth explicit template argument for this call. */ Type getExplicitTemplateArgument(int n) { n < getNumberOfExplicitTemplateArguments() and result = getTemplateArgument(n) } + /** Gets the nth explicit template argument value for this call. */ + Expr getExplicitTemplateArgumentValue(int n) { + n < getNumberOfExplicitTemplateArguments() and + result = getTemplateArgumentValue(n) + } + /** Gets the number of explicit template arguments for this call. */ int getNumberOfExplicitTemplateArguments() { if numtemplatearguments(underlyingElement(this), _) @@ -160,9 +172,15 @@ class FunctionCall extends Call, @funbindexpr { /** Gets the number of template arguments for this call. */ int getNumberOfTemplateArguments() { result = count(int i | exists(getTemplateArgument(i))) } + /** Gets the number of template argument which are values for this call. */ + int getNumberOfTemplateArgumentValues() { result = count(int i | exists(getTemplateArgumentValue(i))) } + /** Gets the nth template argument for this call (indexed from 0). */ Type getTemplateArgument(int n) { result = getTarget().getTemplateArgument(n) } + /** Gets the nth template argument value for this call (indexed from 0). */ + Expr getTemplateArgumentValue(int n) { result = getTarget().getTemplateArgumentValue(n) } + /** Holds if any template arguments for this call are implicit / deduced. */ predicate hasImplicitTemplateArguments() { exists(int i | From 469832668fae1aa5202bf8881af4b7796949f14c Mon Sep 17 00:00:00 2001 From: Matthew Gretton-Dann Date: Fri, 18 Oct 2019 15:30:41 +0100 Subject: [PATCH 120/148] C++: Add some simple non-type template tests --- .../templates/nontype_instantiations/classes/test.cpp | 4 ++++ .../templates/nontype_instantiations/classes/test.expected | 2 ++ .../templates/nontype_instantiations/classes/test.ql | 4 ++++ .../templates/nontype_instantiations/functions/test.cpp | 5 +++++ .../templates/nontype_instantiations/functions/test.expected | 2 ++ .../templates/nontype_instantiations/functions/test.ql | 4 ++++ 6 files changed, 21 insertions(+) create mode 100644 cpp/ql/test/library-tests/templates/nontype_instantiations/classes/test.cpp create mode 100644 cpp/ql/test/library-tests/templates/nontype_instantiations/classes/test.expected create mode 100644 cpp/ql/test/library-tests/templates/nontype_instantiations/classes/test.ql create mode 100644 cpp/ql/test/library-tests/templates/nontype_instantiations/functions/test.cpp create mode 100644 cpp/ql/test/library-tests/templates/nontype_instantiations/functions/test.expected create mode 100644 cpp/ql/test/library-tests/templates/nontype_instantiations/functions/test.ql diff --git a/cpp/ql/test/library-tests/templates/nontype_instantiations/classes/test.cpp b/cpp/ql/test/library-tests/templates/nontype_instantiations/classes/test.cpp new file mode 100644 index 00000000000..4f16f623daf --- /dev/null +++ b/cpp/ql/test/library-tests/templates/nontype_instantiations/classes/test.cpp @@ -0,0 +1,4 @@ +template +class Int { }; + +Int<10> i; diff --git a/cpp/ql/test/library-tests/templates/nontype_instantiations/classes/test.expected b/cpp/ql/test/library-tests/templates/nontype_instantiations/classes/test.expected new file mode 100644 index 00000000000..524097b88d6 --- /dev/null +++ b/cpp/ql/test/library-tests/templates/nontype_instantiations/classes/test.expected @@ -0,0 +1,2 @@ +| test.cpp:2:7:2:9 | Int<10> | file://:0:0:0:0 | int | test.cpp:4:5:4:6 | 10 | +| test.cpp:2:7:2:9 | Int | file://:0:0:0:0 | int | file://:0:0:0:0 | Unknown literal | diff --git a/cpp/ql/test/library-tests/templates/nontype_instantiations/classes/test.ql b/cpp/ql/test/library-tests/templates/nontype_instantiations/classes/test.ql new file mode 100644 index 00000000000..347bb2dba00 --- /dev/null +++ b/cpp/ql/test/library-tests/templates/nontype_instantiations/classes/test.ql @@ -0,0 +1,4 @@ +import cpp + +from Class c +select c, c.getATemplateArgument(), c.getATemplateArgumentValue() diff --git a/cpp/ql/test/library-tests/templates/nontype_instantiations/functions/test.cpp b/cpp/ql/test/library-tests/templates/nontype_instantiations/functions/test.cpp new file mode 100644 index 00000000000..625258906db --- /dev/null +++ b/cpp/ql/test/library-tests/templates/nontype_instantiations/functions/test.cpp @@ -0,0 +1,5 @@ +// semmle-extractor-options: --edg --trap_container=folder --edg --trap-compression=none +template +int addToSelf() { return i + i; }; + +int bar() { return addToSelf<10>(); } diff --git a/cpp/ql/test/library-tests/templates/nontype_instantiations/functions/test.expected b/cpp/ql/test/library-tests/templates/nontype_instantiations/functions/test.expected new file mode 100644 index 00000000000..70e845e7b59 --- /dev/null +++ b/cpp/ql/test/library-tests/templates/nontype_instantiations/functions/test.expected @@ -0,0 +1,2 @@ +| test.cpp:3:5:3:5 | addToSelf | file://:0:0:0:0 | int | test.cpp:5:30:5:31 | 10 | +| test.cpp:3:5:3:13 | addToSelf | file://:0:0:0:0 | int | file://:0:0:0:0 | Unknown literal | diff --git a/cpp/ql/test/library-tests/templates/nontype_instantiations/functions/test.ql b/cpp/ql/test/library-tests/templates/nontype_instantiations/functions/test.ql new file mode 100644 index 00000000000..4dd6e754cd6 --- /dev/null +++ b/cpp/ql/test/library-tests/templates/nontype_instantiations/functions/test.ql @@ -0,0 +1,4 @@ +import cpp + +from Function f +select f, f.getATemplateArgument(), f.getATemplateArgumentValue() From 57cd9b3990f2396e36c3d38c53db8ca4e6f70e79 Mon Sep 17 00:00:00 2001 From: Matthew Gretton-Dann Date: Mon, 21 Oct 2019 17:12:20 +0100 Subject: [PATCH 121/148] C++: Update test results We now support getting the name used for non-type template parameters --- .../ptr_to_member/segfault/exprs.expected | 8 ++++---- .../library-tests/templates/CPP-204/element.expected | 12 ++++++------ .../nontype_instantiations/classes/test.expected | 2 +- .../nontype_instantiations/functions/test.expected | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/cpp/ql/test/library-tests/ptr_to_member/segfault/exprs.expected b/cpp/ql/test/library-tests/ptr_to_member/segfault/exprs.expected index e745ca0142d..e5539a762ab 100644 --- a/cpp/ql/test/library-tests/ptr_to_member/segfault/exprs.expected +++ b/cpp/ql/test/library-tests/ptr_to_member/segfault/exprs.expected @@ -1,7 +1,7 @@ -| file://:0:0:0:0 | Unknown literal | file://:0:0:0:0 | unsigned long | -| file://:0:0:0:0 | Unknown literal | file://:0:0:0:0 | unsigned long | -| file://:0:0:0:0 | Unknown literal | file://:0:0:0:0 | unsigned long | -| file://:0:0:0:0 | Unknown literal | file://:0:0:0:0 | unsigned long | +| file://:0:0:0:0 | __i | file://:0:0:0:0 | unsigned long | +| file://:0:0:0:0 | uls | file://:0:0:0:0 | unsigned long | +| file://:0:0:0:0 | uls | file://:0:0:0:0 | unsigned long | +| file://:0:0:0:0 | uls | file://:0:0:0:0 | unsigned long | | segfault.cpp:25:46:25:65 | call to S | file://:0:0:0:0 | void | | segfault.cpp:25:46:25:65 | call to S | file://:0:0:0:0 | void | | segfault.cpp:25:48:25:55 | __second | segfault.cpp:15:7:15:11 | tuple | diff --git a/cpp/ql/test/library-tests/templates/CPP-204/element.expected b/cpp/ql/test/library-tests/templates/CPP-204/element.expected index b1662c2a3b3..44384471af9 100644 --- a/cpp/ql/test/library-tests/templates/CPP-204/element.expected +++ b/cpp/ql/test/library-tests/templates/CPP-204/element.expected @@ -1,12 +1,12 @@ | file://:0:0:0:0 | | | file://:0:0:0:0 | 0 | | file://:0:0:0:0 | (global namespace) | -| file://:0:0:0:0 | Unknown literal | -| file://:0:0:0:0 | Unknown literal | -| file://:0:0:0:0 | Unknown literal | -| file://:0:0:0:0 | Unknown literal | -| file://:0:0:0:0 | Unknown literal | -| file://:0:0:0:0 | Unknown literal | +| file://:0:0:0:0 | B | +| file://:0:0:0:0 | X | +| file://:0:0:0:0 | X | +| file://:0:0:0:0 | X | +| file://:0:0:0:0 | X | +| file://:0:0:0:0 | Y | | file://:0:0:0:0 | __va_list_tag | | file://:0:0:0:0 | __va_list_tag & | | file://:0:0:0:0 | __va_list_tag && | diff --git a/cpp/ql/test/library-tests/templates/nontype_instantiations/classes/test.expected b/cpp/ql/test/library-tests/templates/nontype_instantiations/classes/test.expected index 524097b88d6..ffa9bfb2cf4 100644 --- a/cpp/ql/test/library-tests/templates/nontype_instantiations/classes/test.expected +++ b/cpp/ql/test/library-tests/templates/nontype_instantiations/classes/test.expected @@ -1,2 +1,2 @@ | test.cpp:2:7:2:9 | Int<10> | file://:0:0:0:0 | int | test.cpp:4:5:4:6 | 10 | -| test.cpp:2:7:2:9 | Int | file://:0:0:0:0 | int | file://:0:0:0:0 | Unknown literal | +| test.cpp:2:7:2:9 | Int | file://:0:0:0:0 | int | file://:0:0:0:0 | i | diff --git a/cpp/ql/test/library-tests/templates/nontype_instantiations/functions/test.expected b/cpp/ql/test/library-tests/templates/nontype_instantiations/functions/test.expected index 70e845e7b59..493e9f03b20 100644 --- a/cpp/ql/test/library-tests/templates/nontype_instantiations/functions/test.expected +++ b/cpp/ql/test/library-tests/templates/nontype_instantiations/functions/test.expected @@ -1,2 +1,2 @@ | test.cpp:3:5:3:5 | addToSelf | file://:0:0:0:0 | int | test.cpp:5:30:5:31 | 10 | -| test.cpp:3:5:3:13 | addToSelf | file://:0:0:0:0 | int | file://:0:0:0:0 | Unknown literal | +| test.cpp:3:5:3:13 | addToSelf | file://:0:0:0:0 | int | file://:0:0:0:0 | i | From ca898d4be04ec3d899637e2e9e1627c5f5fd5541 Mon Sep 17 00:00:00 2001 From: Matthew Gretton-Dann Date: Tue, 22 Oct 2019 10:54:14 +0100 Subject: [PATCH 122/148] C++: Further nontype template testcases. --- .../nontype_instantiations/general/test.cpp | 19 +++++++++++++++++++ .../general/test.expected | 10 ++++++++++ .../nontype_instantiations/general/test.ql | 4 ++++ 3 files changed, 33 insertions(+) create mode 100644 cpp/ql/test/library-tests/templates/nontype_instantiations/general/test.cpp create mode 100644 cpp/ql/test/library-tests/templates/nontype_instantiations/general/test.expected create mode 100644 cpp/ql/test/library-tests/templates/nontype_instantiations/general/test.ql diff --git a/cpp/ql/test/library-tests/templates/nontype_instantiations/general/test.cpp b/cpp/ql/test/library-tests/templates/nontype_instantiations/general/test.cpp new file mode 100644 index 00000000000..1baabe3ca30 --- /dev/null +++ b/cpp/ql/test/library-tests/templates/nontype_instantiations/general/test.cpp @@ -0,0 +1,19 @@ +// semmle-extractor-options: --edg --trap_container=folder --edg --trap-compression=none +template +struct C { }; + +static const int one1 = 1, one2 = 1; +C c = C(); +C e; + +template +struct D { }; + +D a; +D b; + +template +struct E { }; + +E z; + diff --git a/cpp/ql/test/library-tests/templates/nontype_instantiations/general/test.expected b/cpp/ql/test/library-tests/templates/nontype_instantiations/general/test.expected new file mode 100644 index 00000000000..70dfdbc3edf --- /dev/null +++ b/cpp/ql/test/library-tests/templates/nontype_instantiations/general/test.expected @@ -0,0 +1,10 @@ +| test.cpp:3:8:3:8 | C<1> | file://:0:0:0:0 | int | test.cpp:5:25:5:25 | 1 | +| test.cpp:3:8:3:8 | C<2> | file://:0:0:0:0 | int | file://:0:0:0:0 | 2 | +| test.cpp:3:8:3:8 | C | file://:0:0:0:0 | int | file://:0:0:0:0 | x | +| test.cpp:10:8:10:8 | D | test.cpp:9:19:9:19 | T | file://:0:0:0:0 | X | +| test.cpp:10:8:10:8 | D | file://:0:0:0:0 | int | test.cpp:12:8:12:8 | 2 | +| test.cpp:10:8:10:8 | D | file://:0:0:0:0 | long | file://:0:0:0:0 | 2 | +| test.cpp:16:8:16:8 | E | file://:0:0:0:0 | T * | file://:0:0:0:0 | X | +| test.cpp:16:8:16:8 | E | test.cpp:15:19:15:19 | T | file://:0:0:0:0 | X | +| test.cpp:16:8:16:8 | E | file://:0:0:0:0 | int | file://:0:0:0:0 | 0 | +| test.cpp:16:8:16:8 | E | file://:0:0:0:0 | int * | file://:0:0:0:0 | 0 | diff --git a/cpp/ql/test/library-tests/templates/nontype_instantiations/general/test.ql b/cpp/ql/test/library-tests/templates/nontype_instantiations/general/test.ql new file mode 100644 index 00000000000..8915d6523b6 --- /dev/null +++ b/cpp/ql/test/library-tests/templates/nontype_instantiations/general/test.ql @@ -0,0 +1,4 @@ +import cpp + +from Declaration d +select d, d.getATemplateArgument(), d.getATemplateArgumentValue() From f1c3ce04d1ae8a17a37dcea8010872ebe3377760 Mon Sep 17 00:00:00 2001 From: Matthew Gretton-Dann Date: Tue, 22 Oct 2019 10:54:58 +0100 Subject: [PATCH 123/148] C++: Correct spelling mistake --- .../98a075d5495d7be7ede26557708cf22cfa3964ef/upgrade.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/upgrades/98a075d5495d7be7ede26557708cf22cfa3964ef/upgrade.properties b/cpp/upgrades/98a075d5495d7be7ede26557708cf22cfa3964ef/upgrade.properties index 3388d1abe28..495bdeebdac 100644 --- a/cpp/upgrades/98a075d5495d7be7ede26557708cf22cfa3964ef/upgrade.properties +++ b/cpp/upgrades/98a075d5495d7be7ede26557708cf22cfa3964ef/upgrade.properties @@ -1,2 +1,2 @@ -description: Add support for value template paramters. +description: Add support for value template parameters. compatibility: partial From 809d97de02f362e10381d2fc23b0c989705d7c40 Mon Sep 17 00:00:00 2001 From: Matthew Gretton-Dann Date: Wed, 23 Oct 2019 16:29:32 +0100 Subject: [PATCH 124/148] C++: Print print nontype template params --- cpp/ql/src/semmle/code/cpp/Print.qll | 10 +++++++++- cpp/ql/test/library-tests/ir/ir/PrintAST.expected | 7 ++++--- cpp/ql/test/library-tests/ir/ir/raw_ir.expected | 2 +- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/cpp/ql/src/semmle/code/cpp/Print.qll b/cpp/ql/src/semmle/code/cpp/Print.qll index ab979d89b03..7be1537e83b 100644 --- a/cpp/ql/src/semmle/code/cpp/Print.qll +++ b/cpp/ql/src/semmle/code/cpp/Print.qll @@ -35,6 +35,14 @@ private string getParameterTypeString(Type parameterType) { else result = parameterType.(DumpType).getTypeIdentityString() } +private string getTemplateArgumentString(Declaration d, int i) { + if exists(d.getTemplateArgumentValue(i)) + then + result = d.getTemplateArgument(i).(DumpType).getTypeIdentityString() + " " + d.getTemplateArgumentValue(i) + else + result = d.getTemplateArgument(i).(DumpType).getTypeIdentityString() +} + /** * A `Declaration` extended to add methods for generating strings useful only for dumps and debugging. */ @@ -56,7 +64,7 @@ abstract private class DumpDeclaration extends Declaration { strictconcat(int i | exists(this.getTemplateArgument(i)) | - this.getTemplateArgument(i).(DumpType).getTypeIdentityString(), ", " order by i + getTemplateArgumentString(this, i), ", " order by i ) + ">" else result = "" } diff --git a/cpp/ql/test/library-tests/ir/ir/PrintAST.expected b/cpp/ql/test/library-tests/ir/ir/PrintAST.expected index e61fe2b440f..dab6a40fae6 100644 --- a/cpp/ql/test/library-tests/ir/ir/PrintAST.expected +++ b/cpp/ql/test/library-tests/ir/ir/PrintAST.expected @@ -67,7 +67,7 @@ bad_asts.cpp: # 5| params: #-----| 0: [Parameter] p#0 #-----| Type = [RValueReferenceType] S && -# 9| [FunctionTemplateInstantiation,MemberFunction] int Bad::S::MemberFunction(int) +# 9| [FunctionTemplateInstantiation,MemberFunction] int Bad::S::MemberFunction(int) # 9| params: # 9| 0: [Parameter] y # 9| Type = [IntType] int @@ -92,7 +92,7 @@ bad_asts.cpp: # 10| 1: [VariableAccess] y # 10| Type = [IntType] int # 10| ValueCategory = prvalue(load) -# 9| [MemberFunction,TemplateFunction] int Bad::S::MemberFunction(int) +# 9| [MemberFunction,TemplateFunction] int Bad::S::MemberFunction(int) # 9| params: # 9| 0: [Parameter] y # 9| Type = [IntType] int @@ -104,8 +104,9 @@ bad_asts.cpp: # 10| 0: [AddExpr] ... + ... # 10| Type = [IntType] int # 10| ValueCategory = prvalue -# 10| 0: [Literal] Unknown literal +# 10| 0: [Literal] t # 10| Type = [IntType] int +# 10| Value = [Literal] t # 10| ValueCategory = prvalue # 10| 1: [PointerFieldAccess] x # 10| Type = [IntType] int diff --git a/cpp/ql/test/library-tests/ir/ir/raw_ir.expected b/cpp/ql/test/library-tests/ir/ir/raw_ir.expected index f0c96d2bfa0..b38e63e6e76 100644 --- a/cpp/ql/test/library-tests/ir/ir/raw_ir.expected +++ b/cpp/ql/test/library-tests/ir/ir/raw_ir.expected @@ -1,5 +1,5 @@ bad_asts.cpp: -# 9| int Bad::S::MemberFunction(int) +# 9| int Bad::S::MemberFunction(int) # 9| Block 0 # 9| v0_0(void) = EnterFunction : # 9| mu0_1(unknown) = AliasedDefinition : From afeaa6254d4fb367c02bad6e0edb12963402bbd4 Mon Sep 17 00:00:00 2001 From: Matthew Gretton-Dann Date: Tue, 29 Oct 2019 14:06:15 +0000 Subject: [PATCH 125/148] C++: Improve Template Value docs. --- cpp/ql/src/semmle/code/cpp/Declaration.qll | 33 ++++++++++++++++++++-- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/cpp/ql/src/semmle/code/cpp/Declaration.qll b/cpp/ql/src/semmle/code/cpp/Declaration.qll index a10faf0f918..b3d12efe1e9 100644 --- a/cpp/ql/src/semmle/code/cpp/Declaration.qll +++ b/cpp/ql/src/semmle/code/cpp/Declaration.qll @@ -193,19 +193,34 @@ abstract class Declaration extends Locatable, @declaration { /** * Gets a template argument used to instantiate this declaration from a template. - * When called on a template, this will return a template parameter. + * When called on a template, this will return a template parameter type for + * both typed and non-typed parameters. */ final Type getATemplateArgument() { result = getTemplateArgument(_) } /** * Gets a template argument used to instantiate this declaration from a template. - * When called on a template, this will return a template parameter value. + * When called on a template, this will return a non-typed template + * parameter value. */ final Expr getATemplateArgumentValue() { result = getTemplateArgumentValue(_) } /** * Gets the `i`th template argument used to instantiate this declaration from a - * template. When called on a template, this will return the `i`th template parameter. + * template. When called on a template, this will return the `i`th template + * parameter's type. + * + * For example: + * + * `template class Foo;` + * + * Will have `getTemplateArgument(0)` return `T`, and + * `getTemplateArgument(1)` return `T`. + * + * `Foo bar; + * + * Will have `getTemplateArgument())` return `int`, and + * `getTemplateArgument(1)` return `int`. */ Type getTemplateArgument(int index) { none() } @@ -213,6 +228,18 @@ abstract class Declaration extends Locatable, @declaration { * Gets the `i`th template argument value used to instantiate this declaration * from a template. When called on a template, this will return the `i`th template * parameter value if it exists. + * + * For example: + * + * `template class Foo;` + * + * Will have `getTemplateArgumentValue(1)` return `X`, and no result for + * `getTemplateArgumentValue(0)`. + * + * `Foo bar; + * + * Will have `getTemplateArgumentValue(1)` return `10`, and no result for + * `getTemplateArgumentValue(0)`. */ Expr getTemplateArgumentValue(int index) { none() } From 45ec8527c3c03812e2d02bfd8abb4de4a91a945a Mon Sep 17 00:00:00 2001 From: Matthew Gretton-Dann Date: Tue, 29 Oct 2019 16:21:41 +0000 Subject: [PATCH 126/148] C++: Update expected test output. --- cpp/ql/test/library-tests/ir/ir/raw_ir.expected | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/ql/test/library-tests/ir/ir/raw_ir.expected b/cpp/ql/test/library-tests/ir/ir/raw_ir.expected index b38e63e6e76..a188c1b3629 100644 --- a/cpp/ql/test/library-tests/ir/ir/raw_ir.expected +++ b/cpp/ql/test/library-tests/ir/ir/raw_ir.expected @@ -20,7 +20,8 @@ bad_asts.cpp: # 9| r0_16(glval) = VariableAddress[#return] : # 9| v0_17(void) = ReturnValue : &:r0_16, ~mu0_2 # 9| v0_18(void) = UnmodeledUse : mu* -# 9| v0_19(void) = ExitFunction : +# 9| v0_19(void) = AliasedUse : ~mu0_2 +# 9| v0_20(void) = ExitFunction : # 14| void Bad::CallBadMemberFunction() # 14| Block 0 From 6fe22a76da7dd14eda5b002df5d3788c2f366652 Mon Sep 17 00:00:00 2001 From: Matthew Gretton-Dann Date: Fri, 1 Nov 2019 14:31:51 +0000 Subject: [PATCH 127/148] C++: Change API for exposing template parameters. Note that Declaration::getTemplateArgumentType() and Declaration::getTemplateArgumentValue() need to be public so that they can be overriden in derived classes. --- cpp/ql/src/semmle/code/cpp/Class.qll | 4 +- cpp/ql/src/semmle/code/cpp/Declaration.qll | 51 ++++++++++++++----- cpp/ql/src/semmle/code/cpp/Function.qll | 2 +- cpp/ql/src/semmle/code/cpp/Print.qll | 8 +-- cpp/ql/src/semmle/code/cpp/Type.qll | 2 +- cpp/ql/src/semmle/code/cpp/Variable.qll | 2 +- cpp/ql/src/semmle/code/cpp/exprs/Call.qll | 21 ++++---- .../templates/CPP-202/template_args.expected | 4 +- .../nontype_instantiations/classes/test.ql | 2 +- .../nontype_instantiations/functions/test.ql | 2 +- .../general/test.expected | 23 +++++---- .../nontype_instantiations/general/test.ql | 14 ++++- 12 files changed, 86 insertions(+), 49 deletions(-) diff --git a/cpp/ql/src/semmle/code/cpp/Class.qll b/cpp/ql/src/semmle/code/cpp/Class.qll index 44e8b4e65ef..5ccf6193c4f 100644 --- a/cpp/ql/src/semmle/code/cpp/Class.qll +++ b/cpp/ql/src/semmle/code/cpp/Class.qll @@ -610,7 +610,7 @@ class Class extends UserType { * class template. When called on a class template, this will return the * `i`th template parameter. */ - override Type getTemplateArgument(int i) { + override Type getTemplateArgumentType(int i) { class_template_argument(underlyingElement(this), i, unresolveElement(result)) } @@ -632,7 +632,7 @@ class Class extends UserType { } override predicate involvesTemplateParameter() { - getATemplateArgument().involvesTemplateParameter() + getATemplateArgument().(Type).involvesTemplateParameter() } /** Holds if this class, struct or union was declared 'final'. */ diff --git a/cpp/ql/src/semmle/code/cpp/Declaration.qll b/cpp/ql/src/semmle/code/cpp/Declaration.qll index b3d12efe1e9..9c8ce250d8d 100644 --- a/cpp/ql/src/semmle/code/cpp/Declaration.qll +++ b/cpp/ql/src/semmle/code/cpp/Declaration.qll @@ -196,33 +196,36 @@ abstract class Declaration extends Locatable, @declaration { * When called on a template, this will return a template parameter type for * both typed and non-typed parameters. */ - final Type getATemplateArgument() { result = getTemplateArgument(_) } + final Locatable getATemplateArgument() { result = getTemplateArgument(_) } /** * Gets a template argument used to instantiate this declaration from a template. * When called on a template, this will return a non-typed template * parameter value. */ - final Expr getATemplateArgumentValue() { result = getTemplateArgumentValue(_) } + final Locatable getATemplateArgumentKind() { result = getTemplateArgumentKind(_) } /** * Gets the `i`th template argument used to instantiate this declaration from a - * template. When called on a template, this will return the `i`th template - * parameter's type. + * template. * * For example: * * `template class Foo;` * * Will have `getTemplateArgument(0)` return `T`, and - * `getTemplateArgument(1)` return `T`. + * `getTemplateArgument(1)` return `X`. * * `Foo bar; * * Will have `getTemplateArgument())` return `int`, and - * `getTemplateArgument(1)` return `int`. + * `getTemplateArgument(1)` return `1`. */ - Type getTemplateArgument(int index) { none() } + final Locatable getTemplateArgument(int index) { + if exists(getTemplateArgumentValue(index)) + then result = getTemplateArgumentValue(index) + else result = getTemplateArgumentType(index) + } /** * Gets the `i`th template argument value used to instantiate this declaration @@ -233,20 +236,44 @@ abstract class Declaration extends Locatable, @declaration { * * `template class Foo;` * - * Will have `getTemplateArgumentValue(1)` return `X`, and no result for - * `getTemplateArgumentValue(0)`. + * Will have `getTemplateArgumentKind(1)` return `T`, and no result for + * `getTemplateArgumentKind(0)`. * * `Foo bar; * - * Will have `getTemplateArgumentValue(1)` return `10`, and no result for - * `getTemplateArgumentValue(0)`. + * Will have `getTemplateArgumentKind(1)` return `int`, and no result for + * `getTemplateArgumentKind(0)`. */ - Expr getTemplateArgumentValue(int index) { none() } + final Locatable getTemplateArgumentKind(int index) { + if exists(getTemplateArgumentValue(index)) + then result = getTemplateArgumentType(index) + else none() + } /** Gets the number of template arguments for this declaration. */ final int getNumberOfTemplateArguments() { result = count(int i | exists(getTemplateArgument(i))) } + + /** + * INTERNAL: Do not use. + * + * Gets a Type for a template argument. May be the template argument itself + * or the type of a non-type template argument. + * + * Use `getTemplateArgument` or `getTemplateKind` instead. + */ + Type getTemplateArgumentType(int index) { none() } + + /** + * INTERNAL: Do not use. + * + * Gets an Expression representing the value of a non-type template + * argument. + * + * Use `getTemplateArgument` or `getTemplateKind` instead. + */ + Expr getTemplateArgumentValue(int index) { none() } } /** diff --git a/cpp/ql/src/semmle/code/cpp/Function.qll b/cpp/ql/src/semmle/code/cpp/Function.qll index be031f2b915..5436c3ba218 100644 --- a/cpp/ql/src/semmle/code/cpp/Function.qll +++ b/cpp/ql/src/semmle/code/cpp/Function.qll @@ -348,7 +348,7 @@ class Function extends Declaration, ControlFlowNode, AccessHolder, @function { * function template. When called on a function template, this will return the * `i`th template parameter. */ - override Type getTemplateArgument(int index) { + override Type getTemplateArgumentType(int index) { function_template_argument(underlyingElement(this), index, unresolveElement(result)) } diff --git a/cpp/ql/src/semmle/code/cpp/Print.qll b/cpp/ql/src/semmle/code/cpp/Print.qll index 7be1537e83b..e3ad19f6028 100644 --- a/cpp/ql/src/semmle/code/cpp/Print.qll +++ b/cpp/ql/src/semmle/code/cpp/Print.qll @@ -36,11 +36,11 @@ private string getParameterTypeString(Type parameterType) { } private string getTemplateArgumentString(Declaration d, int i) { - if exists(d.getTemplateArgumentValue(i)) + if exists(d.getTemplateArgumentKind(i)) then - result = d.getTemplateArgument(i).(DumpType).getTypeIdentityString() + " " + d.getTemplateArgumentValue(i) - else - result = d.getTemplateArgument(i).(DumpType).getTypeIdentityString() + result = d.getTemplateArgumentKind(i).(DumpType).getTypeIdentityString() + " " + + d.getTemplateArgument(i) + else result = d.getTemplateArgument(i).(DumpType).getTypeIdentityString() } /** diff --git a/cpp/ql/src/semmle/code/cpp/Type.qll b/cpp/ql/src/semmle/code/cpp/Type.qll index 10a1b6b1724..ea1e7fd5026 100644 --- a/cpp/ql/src/semmle/code/cpp/Type.qll +++ b/cpp/ql/src/semmle/code/cpp/Type.qll @@ -210,7 +210,7 @@ class Type extends Locatable, @type { // A function call that provides an explicit template argument that refers to T uses T. // We exclude calls within instantiations, since they do not appear directly in the source. exists(FunctionCall c | - c.getAnExplicitTemplateArgument().refersTo(this) and + c.getAnExplicitTemplateArgument().(Type).refersTo(this) and result = c and not c.getEnclosingFunction().isConstructedFrom(_) ) diff --git a/cpp/ql/src/semmle/code/cpp/Variable.qll b/cpp/ql/src/semmle/code/cpp/Variable.qll index deae1069aab..689f1c3bfa5 100644 --- a/cpp/ql/src/semmle/code/cpp/Variable.qll +++ b/cpp/ql/src/semmle/code/cpp/Variable.qll @@ -160,7 +160,7 @@ class Variable extends Declaration, @variable { * variable template. When called on a variable template, this will return the * `i`th template parameter. */ - override Type getTemplateArgument(int index) { + override Type getTemplateArgumentType(int index) { variable_template_argument(underlyingElement(this), index, unresolveElement(result)) } diff --git a/cpp/ql/src/semmle/code/cpp/exprs/Call.qll b/cpp/ql/src/semmle/code/cpp/exprs/Call.qll index 85807cf0d14..22a25969a8f 100644 --- a/cpp/ql/src/semmle/code/cpp/exprs/Call.qll +++ b/cpp/ql/src/semmle/code/cpp/exprs/Call.qll @@ -139,27 +139,27 @@ class FunctionCall extends Call, @funbindexpr { override string getCanonicalQLClass() { result = "FunctionCall" } /** Gets an explicit template argument for this call. */ - Type getAnExplicitTemplateArgument() { result = getExplicitTemplateArgument(_) } + Locatable getAnExplicitTemplateArgument() { result = getExplicitTemplateArgument(_) } /** Gets an explicit template argument value for this call. */ - Expr getAnExplicitTemplateArgumentValue() { result = getExplicitTemplateArgumentValue(_) } + Locatable getAnExplicitTemplateArgumentKind() { result = getExplicitTemplateArgumentKind(_) } /** Gets a template argument for this call. */ - Type getATemplateArgument() { result = getTarget().getATemplateArgument() } + Locatable getATemplateArgument() { result = getTarget().getATemplateArgument() } /** Gets a template argument value for this call. */ - Expr getATemplateArgumentValue() { result = getTarget().getATemplateArgumentValue() } + Locatable getATemplateArgumentKind() { result = getTarget().getATemplateArgumentKind() } /** Gets the nth explicit template argument for this call. */ - Type getExplicitTemplateArgument(int n) { + Locatable getExplicitTemplateArgument(int n) { n < getNumberOfExplicitTemplateArguments() and result = getTemplateArgument(n) } /** Gets the nth explicit template argument value for this call. */ - Expr getExplicitTemplateArgumentValue(int n) { + Locatable getExplicitTemplateArgumentKind(int n) { n < getNumberOfExplicitTemplateArguments() and - result = getTemplateArgumentValue(n) + result = getTemplateArgumentKind(n) } /** Gets the number of explicit template arguments for this call. */ @@ -172,14 +172,11 @@ class FunctionCall extends Call, @funbindexpr { /** Gets the number of template arguments for this call. */ int getNumberOfTemplateArguments() { result = count(int i | exists(getTemplateArgument(i))) } - /** Gets the number of template argument which are values for this call. */ - int getNumberOfTemplateArgumentValues() { result = count(int i | exists(getTemplateArgumentValue(i))) } - /** Gets the nth template argument for this call (indexed from 0). */ - Type getTemplateArgument(int n) { result = getTarget().getTemplateArgument(n) } + Locatable getTemplateArgument(int n) { result = getTarget().getTemplateArgument(n) } /** Gets the nth template argument value for this call (indexed from 0). */ - Expr getTemplateArgumentValue(int n) { result = getTarget().getTemplateArgumentValue(n) } + Locatable getTemplateArgumentKind(int n) { result = getTarget().getTemplateArgumentKind(n) } /** Holds if any template arguments for this call are implicit / deduced. */ predicate hasImplicitTemplateArguments() { diff --git a/cpp/ql/test/library-tests/templates/CPP-202/template_args.expected b/cpp/ql/test/library-tests/templates/CPP-202/template_args.expected index ae10f9ddf63..f38eef610e4 100644 --- a/cpp/ql/test/library-tests/templates/CPP-202/template_args.expected +++ b/cpp/ql/test/library-tests/templates/CPP-202/template_args.expected @@ -1,6 +1,6 @@ | file://:0:0:0:0 | __va_list_tag | | -| test.cpp:3:8:3:9 | s1<> | bool | -| test.cpp:3:8:3:9 | s1<> | bool | +| test.cpp:3:8:3:9 | s1<> | {...} | +| test.cpp:3:8:3:9 | s1<> | (null) | | test.cpp:5:8:5:9 | s2 | T | | test.cpp:5:8:5:9 | s2 | T | | test.cpp:7:8:7:9 | s3> | (unnamed) | diff --git a/cpp/ql/test/library-tests/templates/nontype_instantiations/classes/test.ql b/cpp/ql/test/library-tests/templates/nontype_instantiations/classes/test.ql index 347bb2dba00..900424eb501 100644 --- a/cpp/ql/test/library-tests/templates/nontype_instantiations/classes/test.ql +++ b/cpp/ql/test/library-tests/templates/nontype_instantiations/classes/test.ql @@ -1,4 +1,4 @@ import cpp from Class c -select c, c.getATemplateArgument(), c.getATemplateArgumentValue() +select c, c.getATemplateArgumentKind(), c.getATemplateArgument() diff --git a/cpp/ql/test/library-tests/templates/nontype_instantiations/functions/test.ql b/cpp/ql/test/library-tests/templates/nontype_instantiations/functions/test.ql index 4dd6e754cd6..98f8a4112d4 100644 --- a/cpp/ql/test/library-tests/templates/nontype_instantiations/functions/test.ql +++ b/cpp/ql/test/library-tests/templates/nontype_instantiations/functions/test.ql @@ -1,4 +1,4 @@ import cpp from Function f -select f, f.getATemplateArgument(), f.getATemplateArgumentValue() +select f, f.getATemplateArgumentKind(), f.getATemplateArgument() diff --git a/cpp/ql/test/library-tests/templates/nontype_instantiations/general/test.expected b/cpp/ql/test/library-tests/templates/nontype_instantiations/general/test.expected index 70dfdbc3edf..fa5df86f68f 100644 --- a/cpp/ql/test/library-tests/templates/nontype_instantiations/general/test.expected +++ b/cpp/ql/test/library-tests/templates/nontype_instantiations/general/test.expected @@ -1,10 +1,13 @@ -| test.cpp:3:8:3:8 | C<1> | file://:0:0:0:0 | int | test.cpp:5:25:5:25 | 1 | -| test.cpp:3:8:3:8 | C<2> | file://:0:0:0:0 | int | file://:0:0:0:0 | 2 | -| test.cpp:3:8:3:8 | C | file://:0:0:0:0 | int | file://:0:0:0:0 | x | -| test.cpp:10:8:10:8 | D | test.cpp:9:19:9:19 | T | file://:0:0:0:0 | X | -| test.cpp:10:8:10:8 | D | file://:0:0:0:0 | int | test.cpp:12:8:12:8 | 2 | -| test.cpp:10:8:10:8 | D | file://:0:0:0:0 | long | file://:0:0:0:0 | 2 | -| test.cpp:16:8:16:8 | E | file://:0:0:0:0 | T * | file://:0:0:0:0 | X | -| test.cpp:16:8:16:8 | E | test.cpp:15:19:15:19 | T | file://:0:0:0:0 | X | -| test.cpp:16:8:16:8 | E | file://:0:0:0:0 | int | file://:0:0:0:0 | 0 | -| test.cpp:16:8:16:8 | E | file://:0:0:0:0 | int * | file://:0:0:0:0 | 0 | +| test.cpp:3:8:3:8 | C<1> | 0 | int | test.cpp:5:25:5:25 | 1 | +| test.cpp:3:8:3:8 | C<2> | 0 | int | file://:0:0:0:0 | 2 | +| test.cpp:3:8:3:8 | C | 0 | int | file://:0:0:0:0 | x | +| test.cpp:10:8:10:8 | D | 0 | | test.cpp:9:19:9:19 | T | +| test.cpp:10:8:10:8 | D | 1 | T | file://:0:0:0:0 | X | +| test.cpp:10:8:10:8 | D | 0 | | file://:0:0:0:0 | int | +| test.cpp:10:8:10:8 | D | 1 | int | test.cpp:12:8:12:8 | 2 | +| test.cpp:10:8:10:8 | D | 0 | | file://:0:0:0:0 | long | +| test.cpp:10:8:10:8 | D | 1 | long | file://:0:0:0:0 | 2 | +| test.cpp:16:8:16:8 | E | 0 | | test.cpp:15:19:15:19 | T | +| test.cpp:16:8:16:8 | E | 1 | T * | file://:0:0:0:0 | X | +| test.cpp:16:8:16:8 | E | 0 | | file://:0:0:0:0 | int | +| test.cpp:16:8:16:8 | E | 1 | int * | file://:0:0:0:0 | 0 | diff --git a/cpp/ql/test/library-tests/templates/nontype_instantiations/general/test.ql b/cpp/ql/test/library-tests/templates/nontype_instantiations/general/test.ql index 8915d6523b6..68d597e0f98 100644 --- a/cpp/ql/test/library-tests/templates/nontype_instantiations/general/test.ql +++ b/cpp/ql/test/library-tests/templates/nontype_instantiations/general/test.ql @@ -1,4 +1,14 @@ import cpp -from Declaration d -select d, d.getATemplateArgument(), d.getATemplateArgumentValue() +string maybeGetTemplateArgumentKind(Declaration d, int i) { + ( + if exists(d.getTemplateArgumentKind(i)) + then result = d.getTemplateArgumentKind(i).toString() + else result = "" + ) and + i = [0 .. d.getNumberOfTemplateArguments()] +} + +from Declaration d, int i +where i >= 0 and i < d.getNumberOfTemplateArguments() +select d, i, maybeGetTemplateArgumentKind(d, i), d.getTemplateArgument(i) From 6b4506dbea2edc9113c39238041d5a337087142c Mon Sep 17 00:00:00 2001 From: Matthew Gretton-Dann Date: Fri, 1 Nov 2019 14:35:48 +0000 Subject: [PATCH 128/148] C++: Update schema stats --- cpp/ql/src/semmlecode.cpp.dbscheme.stats | 10940 +++++++++++---------- 1 file changed, 5769 insertions(+), 5171 deletions(-) diff --git a/cpp/ql/src/semmlecode.cpp.dbscheme.stats b/cpp/ql/src/semmlecode.cpp.dbscheme.stats index bd1964b0d4f..3a73724456a 100644 --- a/cpp/ql/src/semmlecode.cpp.dbscheme.stats +++ b/cpp/ql/src/semmlecode.cpp.dbscheme.stats @@ -1,15 +1,15 @@ @compilation -10588 +10177 @externalDataElement -72 +70 @duplication -185332 +185440 @similarity @@ -17,7 +17,7 @@ @external_package -133 +128 @svnentry @@ -25,67 +25,67 @@ @location_default -9664446 +9335283 @location_stmt -2176932 +2178211 @location_expr -9745314 +9746017 @diagnostic -76160 +73204 @file -66167 +63599 @folder -12107 +11637 @macroinvocation -40814994 +39320374 @function -3565918 +3552236 @fun_decl -3657006 +3635139 @var_decl -5579181 +5495954 @type_decl -1469165 +1413769 @namespace_decl -151627 +145743 @using -320601 +308161 @static_assert -132358 +131017 @parameter -4779409 +4724692 @membervariable -325099 +321201 @globalvariable @@ -93,47 +93,47 @@ @localvariable -527101 +521758 @enumconstant -94487 +94097 @builtintype -559 +537 @derivedtype -4572495 +4651908 @decltype -45841 +44063 @usertype -4363319 +4337028 @mangledname -532986 +514256 @type_mention -1699359 +1682135 @routinetype -444572 +436132 @ptrtomember -12180 +13227 @specifier -547 +525 @gnuattribute @@ -141,11 +141,11 @@ @stdattribute -369 +365 @declspec -58441 +57849 @msattribute @@ -153,7 +153,7 @@ @alignas -267 +257 @attribute_arg_empty @@ -161,55 +161,55 @@ @attribute_arg_token -15930 +15768 @attribute_arg_constant -111500 +111565 @attribute_arg_type -60 +58 @derivation -370309 +392220 @frienddecl -231178 +222208 @comment -1743633 +1675974 @namespace -8497 +8167 @specialnamequalifyingelement -12 +11 @namequalifier -1125621 +1116919 @value -8776796 +8788031 @initialiser -1685017 +1668506 @literal -4374193 +4381545 @errorexpr -48066 +46201 @address_of @@ -217,15 +217,15 @@ @reference_to -1077837 +1039133 @indirect -294638 +294811 @ref_indirect -1267903 +1225212 @array_to_pointer @@ -233,31 +233,31 @@ @vacuous_destructor_call -4449 +4288 @assume -3328 +3294 @parexpr -3001187 +3002950 @arithnegexpr -654789 +654792 @unaryplusexpr -206 +198 @complementexpr -28027 +28043 @notexpr -357915 +358125 @conjugation @@ -265,51 +265,51 @@ @realpartexpr -72 +70 @imagpartexpr -72 +70 @postincrexpr -43039 +43064 @postdecrexpr -5401 +5404 @preincrexpr -63954 +61508 @predecrexpr -26586 +25554 @conditionalexpr -154429 +154519 @addexpr -209503 +209626 @subexpr -134974 +135053 @mulexpr -91616 +91670 @divexpr -63973 +64010 @remexpr -4657 +4660 @jmulexpr @@ -337,59 +337,59 @@ @paddexpr -87252 +87303 @psubexpr -23151 +23165 @pdiffexpr -25868 +24865 @lshiftexpr -347564 +347768 @rshiftexpr -59286 +59321 @andexpr -236938 +237077 @orexpr -145217 +143745 @xorexpr -37281 +37303 @eqexpr -212621 +212746 @neexpr -88456 +88508 @gtexpr -43936 +43962 @ltexpr -54831 +54863 @geexpr -22361 +22374 @leexpr -213875 +214001 @minexpr @@ -401,7 +401,7 @@ @assignexpr -551713 +552037 @assignaddexpr @@ -413,15 +413,15 @@ @assignmulexpr -7597 +7302 @assigndivexpr -2297 +2208 @assignremexpr -303 +292 @assignlshiftexpr @@ -433,39 +433,39 @@ @assignandexpr -7740 +7744 @assignorexpr -18663 +18674 @assignxorexpr -21992 +22005 @assignpaddexpr -13214 +13080 @assignpsubexpr -583 +577 @andlogicalexpr -116840 +116908 @orlogicalexpr -57827 +57861 @commaexpr -10721 +10329 @subscriptexpr -165754 +165851 @virtfunptrexpr @@ -473,15 +473,15 @@ @callexpr -248513 +239197 @vastartexpr -3710 +3673 @vaargexpr -1094 +1051 @vaendexpr @@ -493,63 +493,63 @@ @varaccess -5376400 +5325591 @thisaccess -1181340 +1169367 @new_expr -29515 +28370 @delete_expr -6564 +6309 @throw_expr -23777 +22855 @condition_decl -7646 +7349 @braced_init_list -133 +128 @type_id -4899 +4708 @runtime_sizeof -281622 +281787 @runtime_alignof -1560 +1561 @sizeof_pack -267 +502 @expr_stmt -160546 +160640 @routineexpr -2309113 +2232038 @type_operand -127928 +128003 @offsetofexpr -35356 +35377 @hasassignexpr @@ -597,11 +597,11 @@ @isbaseofexpr -19 +39 @isclassexpr -6 +215 @isconvtoexpr @@ -609,15 +609,15 @@ @isemptyexpr -5 +175 @isenumexpr -3 +23 @ispodexpr -3 +11 @ispolyexpr @@ -629,7 +629,7 @@ @typescompexpr -46285 +46312 @intaddrexpr @@ -637,11 +637,11 @@ @hastrivialdestructor -97 +105 @uuidof -856 +848 @aggregateliteral @@ -649,11 +649,11 @@ @delete_array_expr -1495 +1437 @new_array_expr -5364 +5310 @foldexpr @@ -661,59 +661,59 @@ @ctordirectinit -93264 +89645 @ctorvirtualinit -6904 +6636 @ctorfieldinit -209661 +201538 @ctordelegatinginit -741 +712 @dtordirectdestruct -30828 +30111 @dtorvirtualdestruct -2698 +2594 @dtorfielddestruct -32542 +31279 @static_cast -224735 +226064 @reinterpret_cast -31207 +30891 @const_cast -5847 +5643 @dynamic_cast -1094 +1051 @c_style_cast -4242732 +4244118 @lambdaexpr -12641 +12513 @param_ref -53208 +51144 @noopexpr @@ -745,7 +745,7 @@ @istrivialexpr -2565 +2512 @isstandardlayoutexpr @@ -753,7 +753,7 @@ @istriviallycopyableexpr -48 +46 @isliteraltypeexpr @@ -813,11 +813,11 @@ @isfinalexpr -2 +175 @noexceptexpr -449 +432 @builtinshufflevector @@ -825,11 +825,11 @@ @builtinchooseexpr -7028 +7032 @builtinaddressof -4388 +4218 @vec_fill @@ -845,67 +845,67 @@ @lambdacapture -21730 +21509 @stmt_expr -1284462 +1271443 @stmt_if -524558 +524866 @stmt_while -32907 +31630 @stmt_goto -111432 +111498 @stmt_label -85508 +85558 @stmt_return -1197201 +1154730 @stmt_block -1398281 +1348007 @stmt_end_test_while -149900 +149988 @stmt_for -32387 +32059 @stmt_switch_case -281530 +281695 @stmt_switch -55225 +55258 @stmt_asm -251715 +251863 @stmt_try_block -18964 +18228 @stmt_microsoft_try -171 +169 @stmt_decl -647378 +623601 @stmt_set_vla_size @@ -917,27 +917,27 @@ @stmt_assigned_goto -9137 +9142 @stmt_empty -103681 +102630 @stmt_continue -8603 +8608 @stmt_break -232406 +232543 @stmt_range_based_for -24 +23 @stmt_handler -21888 +21666 @stmt_constexpr_if @@ -945,51 +945,51 @@ @ppd_plain_include -321707 +309224 @ppd_define -349947 +336367 @ppd_if -171077 +164439 @ppd_ifdef -66823 +64230 @ppd_ifndef -91793 +88231 @ppd_elif -22768 +21885 @ppd_else -63638 +61169 @ppd_endif -329694 +316901 @ppd_undef -21152 +20331 @ppd_line -12523 +12530 @ppd_error -48 +46 @ppd_pragma -37040 +36665 @ppd_objc_import @@ -997,7 +997,7 @@ @ppd_include_next -97 +93 @ppd_warning @@ -1005,7 +1005,7 @@ @link_target -644 +619 @xmldtd @@ -1034,15 +1034,15 @@ compilations -10588 +10177 id -10588 +10177 cwd -12 +11 @@ -1056,7 +1056,7 @@ 1 2 -10588 +10177 @@ -1072,7 +1072,7 @@ 871 872 -12 +11 @@ -1082,11 +1082,11 @@ compilation_args -394124 +394355 id -4637 +4640 num @@ -1094,7 +1094,7 @@ arg -20097 +20109 @@ -1113,7 +1113,7 @@ 87 88 -3527 +3529 88 @@ -1149,7 +1149,7 @@ 86 87 -3529 +3531 87 @@ -1322,12 +1322,12 @@ 1 2 -19123 +19134 2 2575 -974 +975 @@ -1343,7 +1343,7 @@ 1 2 -19748 +19760 2 @@ -1358,19 +1358,19 @@ compilation_compiling_files -10588 +10177 id -10588 +10177 num -12 +11 file -5373 +5164 @@ -1384,7 +1384,7 @@ 1 2 -10588 +10177 @@ -1400,7 +1400,7 @@ 1 2 -10588 +10177 @@ -1416,7 +1416,7 @@ 871 872 -12 +11 @@ -1432,7 +1432,7 @@ 442 443 -12 +11 @@ -1448,17 +1448,17 @@ 1 2 -303 +292 2 3 -5044 +4849 3 14 -24 +23 @@ -1474,7 +1474,7 @@ 1 2 -5373 +5164 @@ -1484,23 +1484,23 @@ compilation_time -42255 +40616 id -10563 +10154 num -12 +11 kind -48 +46 seconds -12375 +11965 @@ -1514,7 +1514,7 @@ 1 2 -10563 +10154 @@ -1530,7 +1530,7 @@ 4 5 -10563 +10154 @@ -1544,14 +1544,19 @@ 12 +2 +3 +58 + + 3 4 -3160 +2792 4 5 -7403 +7302 @@ -1567,7 +1572,7 @@ 869 870 -12 +11 @@ -1583,7 +1588,7 @@ 4 5 -12 +11 @@ -1597,9 +1602,9 @@ 12 -1018 -1019 -12 +1024 +1025 +11 @@ -1615,7 +1620,7 @@ 869 870 -48 +46 @@ -1631,7 +1636,7 @@ 1 2 -48 +46 @@ -1645,24 +1650,24 @@ 12 -6 -7 -12 +7 +8 +11 -9 -10 -12 +10 +11 +11 -561 -562 -12 +560 +561 +11 -597 -598 -12 +591 +592 +11 @@ -1678,27 +1683,27 @@ 1 2 -7755 +7501 2 3 -2613 +2617 3 4 -1045 +759 4 -324 -936 +8 +899 -417 -614 -24 +8 +632 +186 @@ -1714,7 +1719,7 @@ 1 2 -12375 +11965 @@ -1730,12 +1735,17 @@ 1 2 -10490 +10305 2 3 -1884 +1635 + + +3 +4 +23 @@ -1745,23 +1755,23 @@ diagnostic_for -940397 +903906 diagnostic -940397 +73204 compilation -10223 +9826 file_number -12 +11 file_number_diagnostic_number -7208 +6929 @@ -1775,7 +1785,17 @@ 1 2 -940397 +9861 + + +2 +3 +60456 + + +118 +840 +2886 @@ -1791,7 +1811,7 @@ 1 2 -940397 +73204 @@ -1807,7 +1827,12 @@ 1 2 -940397 +73181 + + +2 +3 +23 @@ -1823,37 +1848,37 @@ 2 3 -24 +23 7 8 -5227 +5024 8 9 -1665 +1600 9 10 -218 +210 247 248 -2066 +1986 263 444 -802 +771 446 594 -218 +210 @@ -1869,7 +1894,7 @@ 1 2 -10223 +9826 @@ -1885,37 +1910,37 @@ 2 3 -24 +23 7 8 -5227 +5024 8 9 -1665 +1600 9 10 -218 +210 247 248 -2066 +1986 263 444 -802 +771 446 594 -218 +210 @@ -1931,7 +1956,7 @@ 6265 6266 -12 +11 @@ -1947,7 +1972,7 @@ 841 842 -12 +11 @@ -1963,7 +1988,7 @@ 593 594 -12 +11 @@ -1979,47 +2004,47 @@ 1 2 -2954 +2839 2 5 -656 +630 5 6 -1057 +1016 7 14 -571 +549 15 16 -60 +58 17 18 -632 +607 18 23 -486 +467 26 40 -583 +560 42 842 -206 +198 @@ -2035,52 +2060,52 @@ 4 9 -619 +595 10 11 -1057 +1016 14 27 -571 +549 30 31 -60 +58 34 35 -632 +607 36 45 -486 +467 52 79 -583 +560 84 85 -194 +186 254 255 -2893 +2780 272 842 -109 +105 @@ -2096,7 +2121,7 @@ 1 2 -7208 +6929 @@ -2106,19 +2131,19 @@ compilation_finished -10588 +10177 id -10588 +10177 cpu_seconds -8436 +7957 elapsed_seconds -182 +186 @@ -2132,7 +2157,7 @@ 1 2 -10588 +10177 @@ -2148,7 +2173,7 @@ 1 2 -10588 +10177 @@ -2164,17 +2189,17 @@ 1 2 -7184 +6730 2 3 -850 +712 3 -12 -401 +9 +514 @@ -2190,12 +2215,12 @@ 1 2 -8023 +7560 2 3 -413 +397 @@ -2211,67 +2236,62 @@ 1 2 -36 +35 2 3 -12 +35 3 4 -12 +11 -4 -5 -12 - - -8 -9 -12 +7 +8 +11 10 11 -12 +11 -20 -21 -12 +21 +22 +11 -22 -23 -12 +27 +28 +11 53 54 -12 +11 -124 -125 -12 +120 +121 +11 -134 -135 -12 +125 +126 +11 -230 -231 -12 +233 +234 +11 -258 -259 -12 +263 +264 +11 @@ -2287,67 +2307,62 @@ 1 2 -36 +35 2 3 -12 +35 3 4 -12 +11 -4 -5 -12 - - -8 -9 -12 +7 +8 +11 10 11 -12 +11 -20 -21 -12 +21 +22 +11 -22 -23 -12 +26 +27 +11 -49 -50 -12 +51 +52 +11 -100 -101 -12 +92 +93 +11 118 119 -12 +11 -172 -173 -12 +163 +164 +11 -217 -218 -12 +215 +216 +11 @@ -2357,23 +2372,23 @@ externalData -145 +140 id -72 +70 path -12 +11 column -24 +23 value -145 +140 @@ -2387,7 +2402,7 @@ 1 2 -72 +70 @@ -2403,7 +2418,7 @@ 2 3 -72 +70 @@ -2419,7 +2434,7 @@ 2 3 -72 +70 @@ -2435,7 +2450,7 @@ 6 7 -12 +11 @@ -2451,7 +2466,7 @@ 2 3 -12 +11 @@ -2467,7 +2482,7 @@ 12 13 -12 +11 @@ -2483,7 +2498,7 @@ 6 7 -24 +23 @@ -2499,7 +2514,7 @@ 1 2 -24 +23 @@ -2515,7 +2530,7 @@ 6 7 -24 +23 @@ -2531,7 +2546,7 @@ 1 2 -145 +140 @@ -2547,7 +2562,7 @@ 1 2 -145 +140 @@ -2563,7 +2578,7 @@ 1 2 -145 +140 @@ -2573,41 +2588,41 @@ snapshotDate -12 +11 snapshotDate -12 +11 sourceLocationPrefix -12 +11 prefix -12 +11 duplicateCode -185332 +185440 id -185332 +185440 relativePath -761 +762 equivClass -76596 +76641 @@ -2621,7 +2636,7 @@ 1 2 -185332 +185440 @@ -2637,7 +2652,7 @@ 1 2 -185332 +185440 @@ -2765,27 +2780,27 @@ 1 2 -20462 +20474 2 3 -32635 +32655 3 4 -10716 +10723 4 5 -6015 +6019 5 9 -5954 +5957 9 @@ -2806,7 +2821,7 @@ 1 2 -75730 +75775 2 @@ -3122,27 +3137,27 @@ tokens -39620249 +39643528 id -279171 +279335 offset -21169 +21181 beginLine -785436 +785898 beginColumn -1302 +1303 endLine -785436 +785898 endColumn @@ -3160,72 +3175,72 @@ 100 101 -8596 +8601 101 102 -27640 +27656 102 105 -22613 +22626 105 108 -24506 +24520 108 111 -13844 +13852 111 114 -23787 +23801 114 116 -22681 +22694 116 124 -23658 +23672 124 132 -21660 +21673 132 154 -21347 +21360 154 186 -21323 +21335 186 202 -23418 +23432 202 416 -20978 +20991 416 3446 -3115 +3117 @@ -3246,47 +3261,47 @@ 5 6 -109755 +109819 6 7 -15399 +15408 7 8 -28395 +28412 8 12 -23645 +23659 12 17 -22797 +22811 17 19 -18969 +18980 19 22 -22791 +22805 22 28 -21470 +21483 28 151 -14545 +14553 @@ -3302,42 +3317,42 @@ 2 26 -22933 +22946 26 31 -22318 +22331 31 32 -2642 +2643 32 33 -163572 +163668 33 51 -21310 +21323 51 61 -22472 +22485 61 80 -21058 +21071 80 132 -2863 +2865 @@ -3358,47 +3373,47 @@ 5 6 -109755 +109819 6 7 -15399 +15408 7 8 -28395 +28412 8 12 -23645 +23659 12 17 -22797 +22811 17 19 -18969 +18980 19 22 -22791 +22805 22 28 -21470 +21483 28 151 -14545 +14553 @@ -3414,42 +3429,42 @@ 2 26 -21667 +21679 26 31 -24352 +24366 31 32 -1622 +1623 32 33 -163861 +163957 33 54 -22103 +22116 54 63 -20942 +20954 63 82 -21488 +21501 82 133 -3133 +3135 @@ -3465,17 +3480,17 @@ 2 3 -4559 +4562 4 5 -1339 +1340 6 7 -2593 +2594 8 @@ -3485,37 +3500,37 @@ 11 12 -1609 +1610 13 23 -1855 +1856 24 62 -1622 +1623 64 130 -1646 +1647 141 250 -1597 +1598 251 982 -1591 +1592 986 45432 -1929 +1930 @@ -3531,17 +3546,17 @@ 2 3 -4559 +4562 4 5 -1339 +1340 6 7 -2593 +2594 8 @@ -3551,42 +3566,42 @@ 11 12 -1609 +1610 13 23 -1855 +1856 24 62 -1622 +1623 64 130 -1646 +1647 141 246 -1597 +1598 247 964 -1591 +1592 969 32541 -1591 +1592 32546 33841 -337 +338 @@ -3602,47 +3617,47 @@ 1 2 -5899 +5902 2 3 -3416 +3418 3 4 -1609 +1610 4 7 -1886 +1887 7 12 -1837 +1838 12 15 -1640 +1641 15 23 -1591 +1592 23 68 -1609 +1610 68 161 -1591 +1592 161 @@ -3663,17 +3678,17 @@ 2 3 -4559 +4562 4 5 -1339 +1340 6 7 -2593 +2594 8 @@ -3683,42 +3698,42 @@ 11 12 -1609 +1610 13 23 -1855 +1856 24 62 -1622 +1623 64 130 -1646 +1647 141 246 -1597 +1598 247 964 -1591 +1592 969 32541 -1591 +1592 32546 33841 -337 +338 @@ -3734,47 +3749,47 @@ 1 2 -5899 +5902 2 3 -3416 +3418 3 4 -1609 +1610 4 7 -1911 +1912 7 12 -1825 +1826 12 15 -1609 +1610 15 24 -1677 +1678 24 74 -1616 +1617 74 169 -1591 +1592 170 @@ -3795,37 +3810,37 @@ 1 2 -403939 +404176 2 3 -103118 +103179 3 4 -44274 +44300 4 6 -70568 +70610 6 8 -58457 +58491 8 13 -65861 +65900 13 138 -39217 +39240 @@ -3841,52 +3856,52 @@ 1 7 -64810 +64849 7 12 -64350 +64387 12 23 -60079 +60114 23 32 -36740 +36762 32 33 -267244 +267401 33 41 -62672 +62709 41 55 -61271 +61307 55 69 -61541 +61577 69 94 -59243 +59278 94 248 -47482 +47510 @@ -3902,47 +3917,47 @@ 1 5 -64429 +64467 5 9 -62611 +62647 9 15 -63016 +63053 15 29 -59870 +59905 29 32 -9205 +9210 32 33 -349310 +349515 33 37 -61959 +61996 37 42 -61400 +61436 42 122 -53633 +53664 @@ -3958,7 +3973,7 @@ 1 2 -785436 +785898 @@ -3974,47 +3989,47 @@ 1 5 -64294 +64332 5 9 -62611 +62647 9 15 -63133 +63170 15 29 -59735 +59770 29 32 -9254 +9259 32 33 -350355 +350560 33 37 -63588 +63625 37 43 -65241 +65279 43 123 -47224 +47251 @@ -4430,37 +4445,37 @@ 1 2 -403939 +404176 2 3 -103118 +103179 3 4 -44274 +44300 4 6 -70568 +70610 6 8 -58457 +58491 8 13 -65861 +65900 13 138 -39217 +39240 @@ -4476,52 +4491,52 @@ 1 7 -64810 +64849 7 12 -64350 +64387 12 23 -60079 +60114 23 32 -36740 +36762 32 33 -267244 +267401 33 41 -62672 +62709 41 55 -61271 +61307 55 69 -61541 +61577 69 94 -59243 +59278 94 248 -47482 +47510 @@ -4537,7 +4552,7 @@ 1 2 -785436 +785898 @@ -4553,47 +4568,47 @@ 1 5 -64429 +64467 5 9 -62611 +62647 9 15 -63016 +63053 15 29 -59870 +59905 29 32 -9205 +9210 32 33 -349310 +349515 33 37 -61959 +61996 37 42 -61400 +61436 42 122 -53633 +53664 @@ -4609,47 +4624,47 @@ 1 5 -64294 +64332 5 9 -62611 +62647 9 15 -63133 +63170 15 29 -59735 +59770 29 32 -9254 +9259 32 33 -350355 +350560 33 37 -63588 +63625 37 43 -65241 +65279 43 123 -47224 +47251 @@ -5039,23 +5054,23 @@ external_packages -133 +128 id -133 +128 namespace -12 +11 package_name -133 +128 version -133 +128 @@ -5069,7 +5084,7 @@ 1 2 -133 +128 @@ -5085,7 +5100,7 @@ 1 2 -133 +128 @@ -5101,7 +5116,7 @@ 1 2 -133 +128 @@ -5117,7 +5132,7 @@ 11 12 -12 +11 @@ -5133,7 +5148,7 @@ 11 12 -12 +11 @@ -5149,7 +5164,7 @@ 11 12 -12 +11 @@ -5165,7 +5180,7 @@ 1 2 -133 +128 @@ -5181,7 +5196,7 @@ 1 2 -133 +128 @@ -5197,7 +5212,7 @@ 1 2 -133 +128 @@ -5213,7 +5228,7 @@ 1 2 -133 +128 @@ -5229,7 +5244,7 @@ 1 2 -133 +128 @@ -5245,7 +5260,7 @@ 1 2 -133 +128 @@ -5255,15 +5270,15 @@ header_to_external_package -9093 +8740 fileid -9093 +8740 package -133 +128 @@ -5277,7 +5292,7 @@ 1 2 -9093 +8740 @@ -5293,42 +5308,42 @@ 1 2 -36 +35 2 3 -12 +11 15 16 -24 +23 63 64 -12 +11 71 72 -12 +11 85 86 -12 +11 243 244 -12 +11 251 252 -12 +11 @@ -6546,31 +6561,31 @@ locations_default -9664446 +9335283 id -9664446 +9335283 container -78275 +75237 startLine -161084 +154834 startColumn -6017 +5818 endLine -160902 +154658 endColumn -8169 +11182 @@ -6584,7 +6599,7 @@ 1 2 -9664446 +9335283 @@ -6600,7 +6615,7 @@ 1 2 -9664446 +9335283 @@ -6616,7 +6631,7 @@ 1 2 -9664446 +9335283 @@ -6632,7 +6647,7 @@ 1 2 -9664446 +9335283 @@ -6648,7 +6663,7 @@ 1 2 -9664446 +9335283 @@ -6664,62 +6679,62 @@ 1 2 -12752 +12257 2 19 -6625 +6368 19 25 -6090 +5818 25 31 -6066 +5830 31 41 -6491 +6204 41 54 -6138 +5854 54 72 -6175 +5877 72 -99 -5968 +98 +5690 -99 -137 -5944 +98 +136 +5702 -137 -221 -5895 +136 +216 +5690 -221 -431 -5871 +216 +417 +5655 -431 +417 19703 -4254 +4288 @@ -6735,62 +6750,62 @@ 1 2 -12752 +12257 2 15 -6503 +6251 15 20 -6758 +6496 20 25 -6102 +5842 25 32 -6710 +6461 32 41 -6284 +6052 41 53 -6321 +6064 53 70 -5932 +5690 70 96 -5908 +5678 96 152 -5883 +5667 152 -325 -5871 +322 +5643 -325 +324 8863 -3245 +3131 @@ -6806,62 +6821,62 @@ 1 2 -12752 +12257 2 4 -6722 +6461 4 8 -7087 +6812 8 11 -5895 +5632 11 14 -6418 +6099 14 18 -6880 +6601 18 23 -6394 +6134 23 29 -6455 +6227 29 37 -6260 +6029 37 50 -6224 +6052 50 78 -5944 +5737 78 -167 -1239 +168 +1191 @@ -6877,62 +6892,62 @@ 1 2 -12752 +12257 2 15 -6479 +6227 15 20 -6795 +6531 20 25 -6029 +5772 25 32 -6710 +6473 32 41 -6297 +6052 41 53 -6309 +6052 53 70 -6017 +5772 70 96 -5920 +5690 96 153 -5932 +5725 153 332 -5883 +5643 332 8863 -3148 +3038 @@ -6948,62 +6963,62 @@ 1 2 -12752 +12257 2 14 -6236 +5994 14 19 -6710 +6414 19 23 -6224 +5994 23 27 -6005 +5690 27 32 -6187 +5994 32 38 -6005 +5690 38 45 -6138 +5830 45 54 -5920 +5713 54 65 -5993 +5807 65 -79 -5883 +80 +5842 -79 -184 -4218 +80 +336 +4007 @@ -7019,62 +7034,62 @@ 1 2 -31193 +29982 2 3 -19802 +19034 3 4 -18234 +17527 4 5 -10260 +9861 5 6 -9129 +8775 6 7 -6734 +6473 7 9 -14344 +13787 9 13 -13943 +13390 13 30 -12350 +11871 30 112 -12119 +11637 112 -2168 -12083 +2116 +11614 -2193 +2135 6440 -887 +876 @@ -7090,42 +7105,42 @@ 1 2 -57268 +55046 2 3 -35496 +34119 3 4 -11341 +10901 4 5 -8983 +8635 5 8 -14794 +14220 8 26 -12363 +11871 26 115 -12095 +11637 115 6440 -8740 +8401 @@ -7141,57 +7156,57 @@ 1 2 -32287 +31034 2 3 -19693 +18929 3 4 -20544 +19747 4 5 -9737 +9359 5 6 -9384 +9020 6 7 -6856 +6590 7 9 -14612 +14033 9 13 -14028 +13495 13 27 -12436 +11953 27 60 -12132 +11637 60 153 -9372 +9032 @@ -7207,22 +7222,22 @@ 1 2 -119911 +115246 2 3 -18976 +18239 3 7 -13530 +13016 7 184 -8667 +8331 @@ -7238,57 +7253,57 @@ 1 2 -32311 +31057 2 3 -19584 +18824 3 4 -19024 +18286 4 5 -10247 +9850 5 6 -9409 +9043 6 7 -6637 +6379 7 9 -14636 +14068 9 13 -13700 +13168 13 28 -12144 +11673 28 -68 -12119 +69 +11813 -68 -190 -11269 +69 +258 +10668 @@ -7304,57 +7319,57 @@ 1 2 -948 +934 2 3 -1142 +1074 3 5 -534 +537 5 8 -547 +514 8 16 -474 +467 16 -38 -461 +37 +444 -39 -126 -461 +37 +119 +444 -135 -517 -461 +125 +511 +444 -553 -5009 -461 +518 +4987 +444 -5423 -19562 -461 +5021 +19113 +444 -23702 -199709 -60 +19590 +199744 +70 @@ -7370,47 +7385,47 @@ 1 2 -2589 +2500 2 3 -522 +525 3 6 -474 +455 6 15 -486 +467 15 55 -461 +444 59 -223 -461 +226 +444 229 -1196 -461 +1198 +444 -1266 +1275 2344 -461 +444 -2536 +2542 6440 -97 +93 @@ -7426,57 +7441,57 @@ 1 2 -984 +969 2 3 -1130 +1063 3 4 -291 +280 4 6 -498 +502 6 10 -486 +455 10 -21 -474 +20 +444 -21 -60 -461 +20 +55 +444 -60 -208 -461 +56 +195 +444 -209 -929 -461 +196 +829 +444 -957 -1965 -461 +851 +1958 +444 -1969 +1958 6490 -303 +327 @@ -7492,55 +7507,55 @@ 1 2 -984 +969 2 3 -1130 +1063 3 4 -291 +280 4 6 -498 +502 6 10 -486 +455 10 -21 -474 +20 +444 -21 -60 -461 +20 +55 +444 -60 -208 -461 +56 +195 +444 -209 -931 -461 +205 +828 +444 -958 -1966 -461 +851 +1962 +467 -1969 +1965 6462 303 @@ -7558,42 +7573,42 @@ 1 2 -2735 +2640 2 3 -522 +525 3 7 -547 +525 7 13 -474 +455 13 29 -498 +479 29 -51 -474 +54 +455 -51 -90 -461 +54 +109 +444 -91 +110 428 -303 +292 @@ -7609,62 +7624,62 @@ 1 2 -30670 +29480 2 3 -20240 +19455 3 4 -17979 +17281 4 5 -10430 +10025 5 6 -8898 +8553 6 7 -6868 +6601 7 9 -14368 +13811 9 13 -13992 +13437 13 30 -12302 +11824 30 111 -12144 +11649 111 -2100 -12071 +2058 +11602 -2100 +2083 6440 -936 +934 @@ -7680,42 +7695,42 @@ 1 2 -56758 +54555 2 3 -35654 +34271 3 4 -11341 +10901 4 5 -9105 +8751 5 7 -11949 +11474 7 19 -12338 +11859 19 76 -12083 +11614 76 6440 -11670 +11229 @@ -7731,22 +7746,22 @@ 1 2 -118792 +114171 2 3 -19267 +18508 3 7 -12983 +12502 7 46 -9858 +9476 @@ -7762,57 +7777,57 @@ 1 2 -31752 +30520 2 3 -20082 +19303 3 4 -20252 +19466 4 5 -9992 +9604 5 6 -9263 +8903 6 7 -6977 +6707 7 9 -14599 +14021 9 13 -14101 +13565 13 27 -12350 +11871 27 60 -12107 +11626 60 153 -9421 +9067 @@ -7828,57 +7843,57 @@ 1 2 -31801 +30567 2 3 -19973 +19197 3 4 -18745 +18017 4 5 -10490 +10083 5 6 -9263 +8903 6 7 -6710 +6449 7 9 -14526 +13963 9 13 -14174 +13612 13 29 -12350 +11871 29 71 -12132 +11602 71 -190 -10734 +258 +10387 @@ -7894,52 +7909,47 @@ 1 2 -2309 +4428 2 3 -1094 +1180 3 4 -607 +888 4 -7 -753 +6 +1004 -7 -17 -656 +6 +13 +841 -17 -49 -619 +13 +61 +841 -55 -236 -619 +61 +698 +841 -241 -4440 -619 +705 +10765 +841 -4617 -11969 -619 - - -12066 -25444 -267 +11014 +25447 +315 @@ -7955,47 +7965,42 @@ 1 2 -3257 +5106 2 3 -1033 +1285 3 4 -583 +1028 4 -9 -644 +7 +969 -9 -29 -619 +7 +33 +841 -29 -166 -619 +33 +376 +841 -172 -1667 -619 +387 +2684 +841 -1667 -2846 -619 - - -2870 +2688 6440 -170 +268 @@ -8011,52 +8016,47 @@ 1 2 -2334 +4463 2 3 -1106 +1180 3 4 -632 +899 4 -7 -741 +6 +1004 -7 -17 -632 +6 +14 +852 -17 -45 -619 +14 +57 +852 -45 -189 -619 +58 +510 +841 -192 -1191 -619 +518 +2401 +841 -1218 -2414 -619 - - -2469 +2415 4808 -243 +245 @@ -8072,42 +8072,42 @@ 1 2 -3367 +5141 2 3 -1130 +1343 3 4 -595 +1028 4 -11 -668 +7 +969 -11 +7 23 -632 +852 23 -40 -632 +47 +864 -40 -68 -632 +47 +81 +864 -68 -82 -510 +81 +83 +116 @@ -8123,52 +8123,47 @@ 1 2 -2334 +4463 2 3 -1106 +1180 3 4 -632 +911 4 -7 -741 +6 +993 -7 -17 -644 +6 +13 +841 -17 -46 -619 +13 +56 +841 -46 -194 -619 +56 +501 +841 -202 -1220 -619 +505 +2374 +841 -1252 -2470 -619 - - -2481 +2398 4808 -230 +268 @@ -8178,31 +8173,31 @@ locations_stmt -2176932 +2178211 id -2176932 +2178211 container -3078 +3080 startLine -296727 +296902 startColumn -1228 +1229 endLine -294872 +295045 endColumn -1493 +1494 @@ -8216,7 +8211,7 @@ 1 2 -2176932 +2178211 @@ -8232,7 +8227,7 @@ 1 2 -2176932 +2178211 @@ -8248,7 +8243,7 @@ 1 2 -2176932 +2178211 @@ -8264,7 +8259,7 @@ 1 2 -2176932 +2178211 @@ -8280,7 +8275,7 @@ 1 2 -2176932 +2178211 @@ -8493,7 +8488,7 @@ 38 48 -251 +252 48 @@ -8534,7 +8529,7 @@ 6 15 -251 +252 15 @@ -8681,37 +8676,37 @@ 1 2 -113847 +113914 2 3 -78993 +79039 3 4 -31327 +31345 4 6 -24942 +24957 6 16 -22619 +22632 16 129 -22275 +22288 129 222 -2722 +2723 @@ -8727,32 +8722,32 @@ 1 2 -175967 +176070 2 3 -51906 +51936 3 5 -22552 +22565 5 13 -22373 +22386 13 125 -22257 +22270 125 176 -1671 +1672 @@ -8768,32 +8763,32 @@ 1 2 -130943 +131019 2 3 -73506 +73549 3 4 -30767 +30785 4 7 -26583 +26598 7 16 -22312 +22325 16 45 -12615 +12623 @@ -8809,27 +8804,27 @@ 1 2 -192251 +192364 2 3 -47310 +47337 3 4 -21765 +21778 4 9 -23264 +23278 9 30 -12136 +12143 @@ -8845,32 +8840,32 @@ 1 2 -156444 +156536 2 3 -62371 +62408 3 4 -20782 +20794 4 9 -24659 +24674 9 30 -22398 +22411 30 73 -10071 +10077 @@ -9266,37 +9261,37 @@ 1 2 -116305 +116374 2 3 -72848 +72891 3 4 -31996 +32015 4 6 -25618 +25633 6 16 -23031 +23044 16 126 -22121 +22134 126 228 -2949 +2951 @@ -9312,27 +9307,27 @@ 1 2 -175604 +175707 2 3 -52066 +52096 3 5 -22207 +22220 5 14 -23375 +23389 14 173 -21617 +21630 @@ -9348,27 +9343,27 @@ 1 2 -194592 +194706 2 3 -42314 +42339 3 4 -20677 +20689 4 8 -22760 +22774 8 32 -14526 +14535 @@ -9384,32 +9379,32 @@ 1 2 -133173 +133251 2 3 -67871 +67910 3 4 -32611 +32630 4 7 -26503 +26518 7 16 -22183 +22196 16 46 -12529 +12536 @@ -9425,32 +9420,32 @@ 1 2 -156272 +156364 2 3 -62230 +62266 3 4 -21132 +21144 4 9 -23756 +23770 9 33 -22318 +22331 33 73 -9162 +9167 @@ -9830,15 +9825,15 @@ locations_expr -9745314 +9746017 id -9745314 +9746017 container -3559 +3563 startLine @@ -9868,7 +9863,7 @@ 1 2 -9745314 +9746017 @@ -9884,7 +9879,7 @@ 1 2 -9745314 +9746017 @@ -9900,7 +9895,7 @@ 1 2 -9745314 +9746017 @@ -9916,7 +9911,7 @@ 1 2 -9745314 +9746017 @@ -9932,7 +9927,7 @@ 1 2 -9745314 +9746017 @@ -9953,67 +9948,67 @@ 4 11 -280 +284 11 -36 -267 +37 +272 -36 -112 -270 +37 +114 +269 -112 -207 +114 +210 +269 + + +210 +328 268 -207 -326 +328 +498 268 -326 -493 -267 +499 +771 +269 -493 -760 -267 +772 +1223 +268 -760 -1208 -267 +1223 +1918 +268 -1210 -1899 -267 +1918 +3307 +268 -1900 -3251 -267 +3322 +7569 +268 -3258 -7330 -267 +7607 +160039 +268 -7359 -51115 -267 - - -51126 +170837 491727 -16 +3 @@ -10029,17 +10024,17 @@ 1 3 -300 +296 3 7 -315 +319 7 17 -271 +272 17 @@ -10049,47 +10044,47 @@ 35 59 -279 +278 59 88 -270 +271 88 138 -268 +271 138 -206 -267 +207 +271 -206 -336 +207 +339 268 -336 -525 +339 +532 268 -525 -946 -267 +532 +949 +268 -947 -2013 -267 +951 +2069 +268 -2019 +2076 142752 -248 +242 @@ -10105,67 +10100,67 @@ 1 3 -308 +307 3 7 -293 +296 7 17 -273 +274 17 -35 -267 +36 +283 -35 -51 -278 +36 +52 +287 -51 -63 -289 +52 +64 +297 -63 -73 -290 +64 +74 +279 -73 -83 +74 +84 +296 + + +84 +93 +270 + + +93 +106 282 -83 -92 -276 +106 +122 +271 -92 -104 -270 +122 +150 +268 -104 -120 -276 - - -120 -145 -270 - - -145 +150 330 -187 +153 @@ -10181,17 +10176,17 @@ 1 3 -300 +296 3 7 -312 +316 7 17 -274 +275 17 @@ -10201,17 +10196,17 @@ 35 59 -278 +277 59 89 -275 +277 89 139 -271 +273 139 @@ -10230,18 +10225,18 @@ 547 -971 -267 +973 +269 -972 -2152 -267 +975 +2167 +268 -2152 +2170 143560 -242 +239 @@ -10257,7 +10252,7 @@ 1 3 -278 +279 3 @@ -10267,57 +10262,57 @@ 7 19 -278 +281 19 -41 -268 +42 +275 -41 +42 60 -279 +272 60 -72 -267 +73 +294 -72 -83 -280 +73 +84 +277 -83 -94 -283 +84 +95 +291 -94 -105 -292 - - -105 -119 -278 - - -119 -136 +95 +106 279 -136 -162 -269 +106 +120 +282 -162 +120 +138 +275 + + +138 +166 +270 + + +166 416 -210 +190 @@ -10667,22 +10662,22 @@ 16130 -46279 +46280 29 -48722 -86671 +48729 +86681 29 -87010 -224371 +87033 +224378 29 -237409 -710014 +237411 +710028 6 @@ -10743,17 +10738,17 @@ 1424 -2061 +2063 29 -2064 -2425 +2066 +2429 29 -2427 -2796 +2429 +2797 22 @@ -11336,18 +11331,18 @@ 35 -20304 -51564 +20305 +51570 35 -54934 +54937 117747 35 -117846 -185890 +117863 +185908 35 @@ -11408,12 +11403,12 @@ 1983 -2458 +2459 35 -2468 -2637 +2470 +2640 32 @@ -11469,7 +11464,7 @@ 2070 -5989 +5990 35 @@ -11616,7 +11611,7 @@ 2067 -5989 +5990 35 @@ -11647,23 +11642,23 @@ numlines -544741 +523603 element_id -537168 +516324 num_lines -10272 +9873 num_code -7938 +7630 num_comment -4364 +4194 @@ -11677,12 +11672,12 @@ 1 2 -529679 +509126 2 7 -7488 +7197 @@ -11698,12 +11693,12 @@ 1 2 -529740 +509184 2 7 -7427 +7139 @@ -11719,12 +11714,12 @@ 1 2 -537083 +516242 2 3 -85 +81 @@ -11740,393 +11735,41 @@ 1 2 -4655 +4475 2 3 -1325 +1273 3 4 -765 +736 4 6 -838 +806 6 12 -814 +782 12 24 -778 +747 24 118 -778 +747 118 7658 -316 - - - - - - -num_lines -num_code - - -12 - - -1 -2 -4728 - - -2 -3 -1337 - - -3 -4 -802 - - -4 -6 -850 - - -6 -11 -923 - - -11 -18 -790 - - -18 -29 -790 - - -29 -32 -48 - - - - - - -num_lines -num_comment - - -12 - - -1 -2 -4716 - - -2 -3 -1337 - - -3 -4 -826 - - -4 -6 -838 - - -6 -10 -802 - - -10 -16 -899 - - -16 -26 -814 - - -26 -27 -36 - - - - - - -num_code -element_id - - -12 - - -1 -2 -3184 - - -2 -3 -1239 - - -3 -4 -632 - - -4 -6 -607 - - -6 -10 -619 - - -10 -21 -619 - - -21 -67 -595 - - -68 -7861 -437 - - - - - - -num_code -num_lines - - -12 - - -1 -2 -3209 - - -2 -3 -1264 - - -3 -4 -644 - - -4 -6 -607 - - -6 -10 -692 - - -10 -21 -644 - - -21 -34 -607 - - -34 -42 -267 - - - - - - -num_code -num_comment - - -12 - - -1 -2 -3233 - - -2 -3 -1227 - - -3 -4 -680 - - -4 -6 -619 - - -6 -10 -680 - - -10 -19 -668 - - -19 -28 -595 - - -28 -33 -230 - - - - - - -num_comment -element_id - - -12 - - -1 -2 -2030 - - -2 -3 -559 - - -3 -4 -328 - - -4 -7 -340 - - -7 -14 -340 - - -14 -38 -328 - - -39 -280 -328 - - -283 -35679 -109 - - - - - - -num_comment -num_lines - - -12 - - -1 -2 -2042 - - -2 -3 -571 - - -3 -5 -389 - - -5 -8 -364 - - -8 -18 -364 - - -18 -58 -328 - - -61 -118 303 @@ -12134,6 +11777,358 @@ +num_lines +num_code + + +12 + + +1 +2 +4545 + + +2 +3 +1285 + + +3 +4 +771 + + +4 +6 +817 + + +6 +11 +888 + + +11 +18 +759 + + +18 +29 +759 + + +29 +32 +46 + + + + + + +num_lines +num_comment + + +12 + + +1 +2 +4533 + + +2 +3 +1285 + + +3 +4 +794 + + +4 +6 +806 + + +6 +10 +771 + + +10 +16 +864 + + +16 +26 +782 + + +26 +27 +35 + + + + + + +num_code +element_id + + +12 + + +1 +2 +3061 + + +2 +3 +1191 + + +3 +4 +607 + + +4 +6 +584 + + +6 +10 +595 + + +10 +21 +595 + + +21 +67 +572 + + +68 +7861 +420 + + + + + + +num_code +num_lines + + +12 + + +1 +2 +3084 + + +2 +3 +1215 + + +3 +4 +619 + + +4 +6 +584 + + +6 +10 +666 + + +10 +21 +619 + + +21 +34 +584 + + +34 +42 +257 + + + + + + +num_code +num_comment + + +12 + + +1 +2 +3108 + + +2 +3 +1180 + + +3 +4 +654 + + +4 +6 +595 + + +6 +10 +654 + + +10 +19 +642 + + +19 +28 +572 + + +28 +33 +222 + + + + + + +num_comment +element_id + + +12 + + +1 +2 +1951 + + +2 +3 +537 + + +3 +4 +315 + + +4 +7 +327 + + +7 +14 +327 + + +14 +38 +315 + + +39 +280 +315 + + +283 +35679 +105 + + + + + + +num_comment +num_lines + + +12 + + +1 +2 +1963 + + +2 +3 +549 + + +3 +5 +373 + + +5 +8 +350 + + +8 +18 +350 + + +18 +58 +315 + + +61 +118 +292 + + + + + + num_comment num_code @@ -12143,37 +12138,37 @@ 1 2 -2042 +1963 2 3 -571 +549 3 5 -389 +373 5 8 -364 +350 8 17 -340 +327 17 52 -328 +315 56 108 -328 +315 @@ -12183,31 +12178,31 @@ diagnostics -76160 +73204 id -76160 +73204 severity -36 +35 error_tag -97 +93 error_message -121 +116 full_error_message -65948 +63389 location -145 +140 @@ -12221,7 +12216,7 @@ 1 2 -76160 +73204 @@ -12237,7 +12232,7 @@ 1 2 -76160 +73204 @@ -12253,7 +12248,7 @@ 1 2 -76160 +73204 @@ -12269,7 +12264,7 @@ 1 2 -76160 +73204 @@ -12285,7 +12280,7 @@ 1 2 -76160 +73204 @@ -12301,17 +12296,17 @@ 1 2 -12 +11 2 3 -12 +11 6262 6263 -12 +11 @@ -12327,12 +12322,12 @@ 1 2 -24 +23 6 7 -12 +11 @@ -12348,12 +12343,12 @@ 1 2 -24 +23 8 9 -12 +11 @@ -12369,17 +12364,17 @@ 1 2 -12 +11 2 3 -12 +11 5422 5423 -12 +11 @@ -12395,17 +12390,17 @@ 1 2 -12 +11 2 3 -12 +11 9 10 -12 +11 @@ -12421,32 +12416,32 @@ 1 2 -24 +23 2 3 -24 +23 5 6 -12 +11 417 418 -12 +11 841 842 -12 +11 4996 4997 -12 +11 @@ -12462,7 +12457,7 @@ 1 2 -97 +93 @@ -12478,12 +12473,12 @@ 1 2 -85 +81 3 4 -12 +11 @@ -12499,27 +12494,27 @@ 1 2 -36 +35 2 3 -24 +23 5 6 -12 +11 417 418 -12 +11 4996 4997 -12 +11 @@ -12535,17 +12530,17 @@ 1 2 -60 +58 2 3 -24 +23 5 6 -12 +11 @@ -12561,42 +12556,42 @@ 1 2 -24 +23 2 3 -24 +23 5 6 -12 +11 10 11 -12 +11 75 76 -12 +11 332 333 -12 +11 841 842 -12 +11 4996 4997 -12 +11 @@ -12612,7 +12607,7 @@ 1 2 -121 +116 @@ -12628,7 +12623,7 @@ 1 2 -121 +116 @@ -12644,37 +12639,37 @@ 1 2 -36 +35 2 3 -24 +23 5 6 -12 +11 10 11 -12 +11 75 76 -12 +11 332 333 -12 +11 4996 4997 -12 +11 @@ -12690,17 +12685,17 @@ 1 2 -85 +81 2 3 -24 +23 5 6 -12 +11 @@ -12716,12 +12711,12 @@ 1 2 -65936 +63377 841 842 -12 +11 @@ -12737,7 +12732,7 @@ 1 2 -65948 +63389 @@ -12753,7 +12748,7 @@ 1 2 -65948 +63389 @@ -12769,7 +12764,7 @@ 1 2 -65948 +63389 @@ -12785,7 +12780,7 @@ 1 2 -65948 +63389 @@ -12801,12 +12796,12 @@ 1 2 -133 +128 6254 6255 -12 +11 @@ -12822,7 +12817,7 @@ 1 2 -145 +140 @@ -12838,12 +12833,12 @@ 1 2 -133 +128 3 4 -12 +11 @@ -12859,12 +12854,12 @@ 1 2 -133 +128 5 6 -12 +11 @@ -12880,12 +12875,12 @@ 1 2 -133 +128 5414 5415 -12 +11 @@ -12895,27 +12890,27 @@ files -66167 +63599 id -66167 +63599 name -66167 +63599 simple -45148 +43397 ext -109 +105 fromSource -12 +11 @@ -12929,7 +12924,7 @@ 1 2 -66167 +63599 @@ -12945,7 +12940,7 @@ 1 2 -66167 +63599 @@ -12961,7 +12956,7 @@ 1 2 -66167 +63599 @@ -12977,7 +12972,7 @@ 1 2 -66167 +63599 @@ -12993,7 +12988,7 @@ 1 2 -66167 +63599 @@ -13009,7 +13004,7 @@ 1 2 -66167 +63599 @@ -13025,7 +13020,7 @@ 1 2 -66167 +63599 @@ -13041,7 +13036,7 @@ 1 2 -66167 +63599 @@ -13057,22 +13052,22 @@ 1 2 -34196 +32869 2 3 -6868 +6601 3 7 -3586 +3446 7 42 -498 +479 @@ -13088,22 +13083,22 @@ 1 2 -34196 +32869 2 3 -6868 +6601 3 7 -3586 +3446 7 42 -498 +479 @@ -13119,17 +13114,17 @@ 1 2 -40322 +38758 2 3 -4193 +4031 3 6 -632 +607 @@ -13145,7 +13140,7 @@ 1 2 -45148 +43397 @@ -13161,47 +13156,47 @@ 1 2 -12 +11 3 4 -12 +11 15 16 -12 +11 38 39 -12 +11 80 81 -12 +11 114 115 -12 +11 441 442 -12 +11 738 739 -12 +11 4013 4014 -12 +11 @@ -13217,47 +13212,47 @@ 1 2 -12 +11 3 4 -12 +11 15 16 -12 +11 38 39 -12 +11 80 81 -12 +11 114 115 -12 +11 441 442 -12 +11 738 739 -12 +11 4013 4014 -12 +11 @@ -13273,47 +13268,47 @@ 1 2 -12 +11 3 4 -12 +11 15 16 -12 +11 38 39 -12 +11 75 76 -12 +11 112 113 -12 +11 428 429 -12 +11 658 659 -12 +11 2838 2839 -12 +11 @@ -13329,7 +13324,7 @@ 1 2 -109 +105 @@ -13345,7 +13340,7 @@ 5443 5444 -12 +11 @@ -13361,7 +13356,7 @@ 5443 5444 -12 +11 @@ -13377,7 +13372,7 @@ 3714 3715 -12 +11 @@ -13393,7 +13388,7 @@ 9 10 -12 +11 @@ -13403,19 +13398,19 @@ folders -12107 +11637 id -12107 +11637 name -12107 +11637 simple -3476 +3341 @@ -13429,7 +13424,7 @@ 1 2 -12107 +11637 @@ -13445,7 +13440,7 @@ 1 2 -12107 +11637 @@ -13461,7 +13456,7 @@ 1 2 -12107 +11637 @@ -13477,7 +13472,7 @@ 1 2 -12107 +11637 @@ -13493,27 +13488,27 @@ 1 2 -1872 +1799 2 3 -741 +712 3 4 -498 +479 4 28 -279 +268 28 121 -85 +81 @@ -13529,27 +13524,27 @@ 1 2 -1872 +1799 2 3 -741 +712 3 4 -498 +479 4 28 -279 +268 28 121 -85 +81 @@ -13559,15 +13554,15 @@ containerparent -78250 +75214 parent -12107 +11637 child -78250 +75214 @@ -13581,42 +13576,42 @@ 1 2 -5506 +5293 2 3 -1568 +1507 3 4 -668 +642 4 6 -1118 +1074 6 10 -899 +864 10 14 -960 +923 14 30 -923 +888 30 153 -461 +444 @@ -13632,7 +13627,7 @@ 1 2 -78250 +75214 @@ -13642,23 +13637,23 @@ fileannotations -5688625 +5467887 id -5348 +5141 kind -24 +23 name -60186 +57851 value -49318 +47404 @@ -13672,12 +13667,12 @@ 1 2 -145 +140 2 3 -5202 +5001 @@ -13693,62 +13688,62 @@ 1 96 -401 +385 96 219 -401 +385 221 285 -401 +385 285 444 -401 +385 445 521 -401 +385 526 619 -425 +408 619 708 -401 +385 708 895 -401 +385 896 928 -97 +93 930 931 -1531 +1472 1076 1667 -401 +385 1689 2272 -85 +81 @@ -13764,57 +13759,57 @@ 1 108 -401 +385 108 269 -401 +385 269 356 -401 +385 371 630 -401 +385 630 729 -401 +385 733 948 -401 +385 948 1062 -401 +385 1079 1494 -279 +268 1495 1496 -1531 +1472 1496 1865 -401 +385 1963 4058 -328 +315 @@ -13830,12 +13825,12 @@ 428 429 -12 +11 440 441 -12 +11 @@ -13851,12 +13846,12 @@ 2 3 -12 +11 4949 4950 -12 +11 @@ -13872,12 +13867,12 @@ 1 2 -12 +11 4057 4058 -12 +11 @@ -13893,62 +13888,62 @@ 1 2 -9871 +9487 2 3 -6491 +6239 3 6 -4996 +4802 6 8 -4935 +4743 8 14 -4850 +4662 14 18 -4424 +4253 18 21 -4619 +4440 21 34 -4862 +4673 34 128 -4813 +4627 129 236 -4595 +4416 236 395 -4558 +4381 395 440 -1167 +1121 @@ -13964,7 +13959,7 @@ 1 2 -60186 +57851 @@ -13980,62 +13975,62 @@ 1 2 -11038 +10609 2 3 -8728 +8389 3 4 -2808 +2699 4 6 -4352 +4183 6 10 -5470 +5258 10 14 -3586 +3446 14 17 -4643 +4463 17 22 -4643 +4463 22 40 -4558 +4381 40 83 -4582 +4405 83 163 -4534 +4358 164 1933 -1239 +1191 @@ -14051,67 +14046,67 @@ 1 2 -7524 +7232 2 5 -2419 +2325 5 8 -3573 +3435 8 21 -3804 +3657 21 23 -2735 +2629 23 25 -4461 +4288 25 40 -3586 +3446 40 195 -3890 +3739 195 207 -3902 +3750 207 273 -4048 +3891 273 327 -3744 +3598 328 407 -4315 +4148 407 441 -1312 +1261 @@ -14127,12 +14122,12 @@ 1 2 -49306 +47393 2 3 -12 +11 @@ -14148,67 +14143,67 @@ 1 2 -7549 +7256 2 5 -2795 +2687 5 8 -3610 +3470 8 16 -4060 +3902 16 18 -3343 +3213 18 21 -4072 +3914 21 31 -4461 +4288 31 42 -4023 +3867 42 54 -3768 +3622 54 81 -3780 +3633 81 109 -3829 +3680 109 133 -3756 +3610 133 149 -267 +257 @@ -14218,15 +14213,15 @@ inmacroexpansion -57126596 +57160160 id -15544516 +15553649 inv -2493763 +2495229 @@ -14240,42 +14235,42 @@ 1 2 -3895411 +3897700 2 3 -2466113 +2467562 3 4 -2018102 +2019287 4 5 -1769908 +1770948 5 6 -1845270 +1846354 6 7 -884925 +885445 7 8 -1535759 +1536661 8 6265 -1129025 +1129688 @@ -14291,52 +14286,52 @@ 1 2 -651228 +651611 2 3 -350884 +351091 3 4 -145822 +145908 4 5 -276335 +276498 5 8 -230677 +230812 8 11 -202111 +202230 11 19 -210471 +210594 19 34 -192538 +192651 34 167 -188020 +188130 167 153127 -45673 +45699 @@ -14346,15 +14341,15 @@ affectedbymacroexpansion -35527526 +35548400 id -4083475 +4085874 inv -3282712 +3284641 @@ -14368,42 +14363,42 @@ 1 2 -1341531 +1342319 2 3 -719304 +719727 3 4 -682963 +683364 4 6 -320945 +321133 6 10 -313534 +313718 10 24 -308059 +308240 24 64 -309945 +310127 64 9803 -87190 +87242 @@ -14419,72 +14414,72 @@ 1 2 -253184 +253333 2 3 -209979 +210102 3 4 -202347 +202466 4 5 -258125 +258276 5 6 -301422 +301599 6 7 -247881 +248027 7 8 -231148 +231284 8 9 -228002 +228136 9 10 -182499 +182606 10 12 -285679 +285847 12 16 -297410 +297584 16 23 -275398 +275560 23 60 -248963 +249109 60 526 -60669 +60704 @@ -14494,23 +14489,23 @@ macroinvocations -40814994 +39320374 id -40814994 +39320374 macro_id -89374 +85906 location -836970 +804492 kind -24 +23 @@ -14524,7 +14519,7 @@ 1 2 -40814994 +39320374 @@ -14540,7 +14535,7 @@ 1 2 -40814994 +39320374 @@ -14556,7 +14551,7 @@ 1 2 -40814994 +39320374 @@ -14572,52 +14567,52 @@ 1 2 -19267 +18286 2 3 -16946 +16475 3 4 -3744 +3598 4 6 -7925 +7630 6 11 -7658 +7326 11 21 -7160 +6823 21 41 -6758 +6555 41 -104 -6710 +107 +6485 -104 -454 -6710 +107 +450 +6449 -454 -201153 -6491 +453 +201240 +6274 @@ -14633,37 +14628,37 @@ 1 2 -47483 +45640 2 3 -11803 +11345 3 4 -5895 +5667 4 6 -7682 +7384 6 13 -7403 +7115 13 65 -6734 +6473 65 3614 -2370 +2278 @@ -14679,12 +14674,12 @@ 1 2 -82566 +79362 2 3 -6807 +6543 @@ -14700,42 +14695,42 @@ 1 2 -314219 +299713 2 3 -186394 +180341 3 4 -44431 +42999 4 5 -58533 +56378 5 8 -67687 +65247 8 16 -64113 +61777 16 46 -64830 +62536 46 -262302 -36761 +262546 +35498 @@ -14751,12 +14746,12 @@ 1 2 -783044 +752659 2 353 -53925 +51833 @@ -14772,7 +14767,7 @@ 1 2 -836970 +804492 @@ -14788,12 +14783,12 @@ 37566 37567 -12 +11 -3319916 -3319917 -12 +3327545 +3327546 +11 @@ -14809,12 +14804,12 @@ 2199 2200 -12 +11 5713 5714 -12 +11 @@ -14830,12 +14825,12 @@ 6537 6538 -12 +11 62313 62314 -12 +11 @@ -14845,15 +14840,15 @@ macroparent -35822215 +34476404 id -35822215 +34476404 parent_id -27895299 +26856682 @@ -14867,7 +14862,7 @@ 1 2 -35822215 +34476404 @@ -14883,17 +14878,17 @@ 1 2 -21477676 +20687700 2 3 -5474478 +5262423 3 88 -943144 +906558 @@ -14903,15 +14898,15 @@ macrolocationbind -4016679 +4019039 id -2801506 +2803152 location -2004363 +2005541 @@ -14925,22 +14920,22 @@ 1 2 -2200726 +2202019 2 3 -339324 +339524 3 7 -231800 +231936 7 57 -29655 +29673 @@ -14956,22 +14951,22 @@ 1 2 -1602305 +1603246 2 3 -171008 +171108 3 8 -155504 +155595 8 723 -75546 +75590 @@ -14981,19 +14976,19 @@ macro_argument_unexpanded -104803767 +100941827 invocation -31131960 +30008413 argument_index -802 +771 text -345935 +332512 @@ -15007,22 +15002,22 @@ 1 2 -8804221 +8477672 2 3 -12965574 +12498009 3 4 -7092535 +6840222 4 67 -2269629 +2192508 @@ -15038,22 +15033,22 @@ 1 2 -8880284 +8550795 2 3 -13152746 +12678071 3 4 -6885304 +6640869 4 67 -2213625 +2138677 @@ -15069,17 +15064,17 @@ 51815 51816 -705 +677 52017 -186703 -60 +187640 +58 -770141 -2560947 -36 +773038 +2568177 +35 @@ -15095,17 +15090,17 @@ 2 3 -705 +677 13 1002 -60 +58 6617 19688 -36 +35 @@ -15121,62 +15116,62 @@ 1 2 -42535 +39868 2 3 -69291 +64826 3 4 -15621 +15435 4 5 -46376 +44296 5 8 -26513 +26980 8 12 -15560 +15447 12 16 -24531 +23275 16 22 -26403 +25414 22 -42 -26257 +41 +25075 -42 -113 -25978 +41 +101 +24970 -113 -51881 -25953 +101 +4363 +24958 -51881 -580841 -911 +4608 +581891 +1963 @@ -15192,17 +15187,17 @@ 1 2 -250082 +240378 2 3 -84450 +81173 3 9 -11402 +10960 @@ -15212,19 +15207,19 @@ macro_argument_expanded -104803767 +100941827 invocation -31131960 +30008413 argument_index -802 +771 text -209491 +201362 @@ -15238,22 +15233,22 @@ 1 2 -8804221 +8477672 2 3 -12965574 +12498009 3 4 -7092535 +6840222 4 67 -2269629 +2192508 @@ -15269,22 +15264,22 @@ 1 2 -12795323 +12315365 2 3 -11169279 +10770085 3 4 -5961987 +5753415 4 9 -1205370 +1169546 @@ -15300,17 +15295,17 @@ 51815 51816 -705 +677 52017 -186703 -60 +187640 +58 -770141 -2560947 -36 +773038 +2568177 +35 @@ -15326,17 +15321,17 @@ 1 2 -692 +666 2 75 -60 +58 867 13964 -48 +46 @@ -15352,62 +15347,57 @@ 1 2 -25820 +23871 2 3 -42583 +40791 3 4 -6758 +6636 4 -5 -15791 - - -5 6 -3367 +18450 6 7 -24555 +23743 7 9 -16702 +16475 9 15 -19146 +17597 15 31 -15851 +15984 31 -87 -15912 +86 +15108 -87 -393 -15742 +86 +365 +15225 -396 -1164221 -7257 +365 +1165688 +7478 @@ -15423,22 +15413,22 @@ 1 2 -105651 +101551 2 3 -87976 +84562 3 6 -15779 +15166 6 66 -85 +81 @@ -15448,19 +15438,19 @@ functions -3565918 +3552236 id -3565918 +3552236 name -315848 +303954 kind -85 +81 @@ -15474,7 +15464,7 @@ 1 2 -3565918 +3552236 @@ -15490,7 +15480,7 @@ 1 2 -3565918 +3552236 @@ -15506,27 +15496,27 @@ 1 2 -214682 +206387 2 3 -31266 +29924 3 5 -28360 +27435 5 13 -23826 +23100 13 -109484 -17711 +118759 +17106 @@ -15542,12 +15532,12 @@ 1 2 -314231 +302400 2 3 -1616 +1554 @@ -15563,37 +15553,37 @@ 32 33 -12 +11 -477 -478 -12 +478 +479 +11 2735 2736 -12 +11 -5928 -5929 -12 +6052 +6053 +11 -43779 -43780 -12 +43879 +43880 +11 -108979 -108980 -12 +109948 +109949 +11 -131406 -131407 -12 +140883 +140884 +11 @@ -15609,37 +15599,37 @@ 11 12 -12 +11 42 43 -12 +11 -228 -229 -12 +229 +230 +11 -1485 -1486 -12 +1494 +1495 +11 2735 2736 -12 +11 -2858 -2859 -12 +2879 +2880 +11 18756 18757 -12 +11 @@ -15649,15 +15639,15 @@ function_entry_point -1068501 +1030977 id -1065255 +1027857 entry_point -1068501 +1030977 @@ -15671,12 +15661,12 @@ 1 2 -1062313 +1025030 2 9 -2941 +2827 @@ -15692,7 +15682,7 @@ 1 2 -1068501 +1030977 @@ -15702,15 +15692,15 @@ function_return_type -3577065 +3562951 id -3565347 +3551686 return_type -1033588 +1048715 @@ -15724,12 +15714,12 @@ 1 2 -3554126 +3540901 2 6 -11220 +10784 @@ -15745,17 +15735,17 @@ 1 2 -319945 +309177 2 3 -649566 +677865 3 -81742 -64076 +82133 +61671 @@ -15765,60 +15755,60 @@ purefunctions -21018 +20805 id -21018 +20805 function_deleted -51251 +49379 id -51251 +49379 function_defaulted -9554 +9184 id -9554 +9184 fun_decls -3668324 +3646017 id -3657006 +3635139 function -3484737 +3469555 type_id -1026294 +1041634 name -281968 +271389 location -875943 +843835 @@ -15832,7 +15822,7 @@ 1 2 -3657006 +3635139 @@ -15848,12 +15838,12 @@ 1 2 -3646053 +3624611 2 6 -10952 +10527 @@ -15869,7 +15859,7 @@ 1 2 -3657006 +3635139 @@ -15885,7 +15875,7 @@ 1 2 -3657006 +3635139 @@ -15901,12 +15891,12 @@ 1 2 -3343699 +3333988 2 9 -141038 +135566 @@ -15922,12 +15912,12 @@ 1 2 -3466345 +3451876 2 6 -18392 +17678 @@ -15943,7 +15933,7 @@ 1 2 -3484737 +3469555 @@ -15959,12 +15949,12 @@ 1 2 -3389832 +3378332 2 9 -94905 +91222 @@ -15980,17 +15970,17 @@ 1 2 -305953 +295716 2 3 -648059 +676381 3 -87017 -72282 +87408 +69535 @@ -16006,17 +15996,17 @@ 1 2 -318972 +308231 2 3 -644096 +672572 3 -80918 -63225 +81309 +60830 @@ -16032,12 +16022,12 @@ 1 2 -956638 +974669 2 -7429 -69656 +7459 +66965 @@ -16053,17 +16043,17 @@ 1 2 -921129 +940538 2 5 -81642 +78474 5 -22296 -23522 +22349 +22621 @@ -16079,32 +16069,32 @@ 1 2 -169351 +162826 2 3 -32554 +31420 3 4 -18866 +17994 4 7 -25650 +24841 7 19 -21261 +20518 19 -109747 -14283 +119022 +13787 @@ -16120,32 +16110,32 @@ 1 2 -181155 +174172 2 3 -31412 +30099 3 4 -16654 +16171 4 7 -22623 +21862 7 -26 -21383 +25 +20366 -26 -109468 -8740 +25 +118743 +8716 @@ -16161,17 +16151,17 @@ 1 2 -246046 +236860 2 5 -23170 +22271 5 -56245 -12752 +60936 +12257 @@ -16187,27 +16177,27 @@ 1 2 -180486 +173599 2 3 -47823 +46213 3 4 -18040 +17340 4 8 -22696 +21815 8 -8793 -12922 +8910 +12420 @@ -16223,27 +16213,27 @@ 1 2 -589769 +567012 2 3 -147895 +142319 3 5 -69790 +67350 5 -179 -65766 +141 +63307 -179 -3027 -2723 +142 +3039 +3844 @@ -16259,22 +16249,22 @@ 1 2 -602727 +579468 2 3 -164743 +158514 3 9 -69753 +67747 9 -3027 -38718 +3039 +38103 @@ -16290,17 +16280,17 @@ 1 2 -773854 +744445 2 4 -68209 +65890 4 -1514 -33879 +1520 +33500 @@ -16316,12 +16306,12 @@ 1 2 -848299 +817018 2 134 -27643 +26816 @@ -16331,22 +16321,22 @@ fun_def -1296860 +1250592 id -1296860 +1250592 fun_specialized -6505 +6439 id -6505 +6439 @@ -16364,11 +16354,11 @@ fun_decl_specifiers -512313 +512614 id -274687 +274848 name @@ -16386,17 +16376,17 @@ 1 2 -72048 +72090 2 3 -167651 +167750 3 4 -34987 +35007 @@ -16583,26 +16573,26 @@ fun_decl_empty_throws -1416734 +1471304 fun_decl -1416734 +1471304 fun_decl_noexcept -16569 +15937 fun_decl -15693 +15096 constant -16435 +15809 @@ -16616,12 +16606,12 @@ 1 2 -14818 +14255 2 3 -875 +841 @@ -16637,12 +16627,12 @@ 1 2 -16301 +15680 2 3 -133 +128 @@ -16652,26 +16642,26 @@ fun_decl_empty_noexcept -364559 +350681 fun_decl -364559 +350681 fun_decl_typedef_type -194 +186 fun_decl -194 +186 typedeftype_id -97 +93 @@ -16685,7 +16675,7 @@ 1 2 -194 +186 @@ -16701,7 +16691,7 @@ 2 3 -97 +93 @@ -16711,19 +16701,19 @@ param_decl_bind -4830661 +4766932 id -4830661 +4766932 index -389 +373 fun_decl -3158896 +3153051 @@ -16737,7 +16727,7 @@ 1 2 -4830661 +4766932 @@ -16753,7 +16743,7 @@ 1 2 -4830661 +4766932 @@ -16769,57 +16759,57 @@ 2 3 -158 +151 4 10 -24 +23 15 44 -24 +23 115 191 -24 +23 263 337 -24 +23 422 615 -24 +23 885 1215 -24 +23 1695 2656 -24 +23 4907 -13240 -24 +13252 +23 -34828 -76062 -24 +34902 +76574 +23 -259854 -259855 -12 +269844 +269845 +11 @@ -16835,57 +16825,57 @@ 2 3 -158 +151 4 10 -24 +23 15 44 -24 +23 115 191 -24 +23 263 337 -24 +23 422 615 -24 +23 885 1215 -24 +23 1695 2656 -24 +23 4907 -13240 -24 +13252 +23 -34828 -76062 -24 +34902 +76574 +23 -259854 -259855 -12 +269844 +269845 +11 @@ -16901,22 +16891,22 @@ 1 2 -2234266 +2258317 2 3 -501246 +486913 3 4 -262445 +252985 4 33 -160938 +154834 @@ -16932,22 +16922,22 @@ 1 2 -2234266 +2258317 2 3 -501246 +486913 3 4 -262445 +252985 4 33 -160938 +154834 @@ -16957,27 +16947,27 @@ var_decls -5602108 +5518014 id -5579181 +5495954 variable -5340611 +5266618 type_id -2038268 +2068405 name -139592 +134175 location -1358468 +1305755 @@ -16991,7 +16981,7 @@ 1 2 -5579181 +5495954 @@ -17007,12 +16997,12 @@ 1 2 -5556716 +5474337 2 7 -22465 +21616 @@ -17028,7 +17018,7 @@ 1 2 -5579181 +5495954 @@ -17044,12 +17034,12 @@ 1 2 -5579132 +5495907 2 3 -48 +46 @@ -17065,12 +17055,12 @@ 1 2 -5146145 +5079674 2 9 -194466 +186943 @@ -17086,12 +17076,12 @@ 1 2 -5290709 +5218605 2 7 -49902 +48012 @@ -17107,12 +17097,12 @@ 1 2 -5321866 +5248600 2 3 -18745 +18017 @@ -17128,12 +17118,12 @@ 1 2 -5225490 +5155963 2 9 -115121 +110654 @@ -17149,22 +17139,22 @@ 1 2 -1580700 +1627085 2 3 -244952 +236522 3 -9 -154629 +10 +155430 -9 -5708 -57986 +10 +5721 +49367 @@ -17180,22 +17170,22 @@ 1 2 -1607080 +1652441 2 3 -234777 +226741 3 -11 -155359 +13 +156353 -11 -5254 -41052 +13 +5267 +32869 @@ -17211,17 +17201,17 @@ 1 2 -1850925 +1888086 2 5 -162458 +156341 5 770 -24884 +23977 @@ -17237,17 +17227,17 @@ 1 2 -1771751 +1811867 2 4 -165132 +159052 4 3604 -101384 +97485 @@ -17263,42 +17253,42 @@ 1 2 -58861 +56530 2 3 -20860 +20039 3 4 -12290 +11801 4 6 -12715 +12268 6 10 -11524 +11088 10 22 -10673 +10270 22 255 -10490 +10083 263 -149390 -2176 +159500 +2091 @@ -17314,37 +17304,37 @@ 1 2 -62277 +59814 2 3 -20118 +19314 3 4 -13359 +12841 4 6 -11949 +11532 6 11 -11499 +11077 11 28 -10515 +10095 28 -148426 -9871 +158536 +9499 @@ -17360,32 +17350,32 @@ 1 2 -84973 +81606 2 3 -18149 +17445 3 4 -9676 +9347 4 7 -11706 +11275 7 28 -10576 +10165 28 -111448 -4510 +120730 +4335 @@ -17401,32 +17391,32 @@ 1 2 -80329 +77212 2 3 -21273 +20448 3 4 -7780 +7478 4 7 -12423 +11941 7 21 -10563 +10154 21 9993 -7220 +6940 @@ -17442,22 +17432,22 @@ 1 2 -1008108 +967927 2 3 -144734 +139422 3 6 -124044 +119546 6 -114124 -81581 +123459 +78860 @@ -17473,22 +17463,22 @@ 1 2 -1060320 +1018112 2 3 -107961 +103970 3 6 -111863 +107943 6 -113877 -78323 +123212 +75728 @@ -17504,17 +17494,17 @@ 1 2 -1166129 +1119863 2 3 -93045 +89797 3 -104379 -99293 +113652 +96095 @@ -17530,12 +17520,12 @@ 1 2 -1348524 +1296197 2 52 -9943 +9558 @@ -17545,11 +17535,11 @@ var_def -2573394 +2477557 id -2573394 +2477557 @@ -17619,19 +17609,19 @@ type_decls -1469165 +1413769 id -1469165 +1413769 type_id -1433996 +1379965 location -1203571 +1156868 @@ -17645,7 +17635,7 @@ 1 2 -1469165 +1413769 @@ -17661,7 +17651,7 @@ 1 2 -1469165 +1413769 @@ -17677,12 +17667,12 @@ 1 2 -1408431 +1355392 2 24 -25564 +24572 @@ -17698,12 +17688,12 @@ 1 2 -1409866 +1356771 2 24 -24130 +23194 @@ -17719,12 +17709,12 @@ 1 2 -1142582 +1098234 2 469 -60988 +58633 @@ -17740,12 +17730,12 @@ 1 2 -1144223 +1099812 2 469 -59347 +57056 @@ -17755,45 +17745,45 @@ type_def -1035654 +995467 id -1035654 +995467 type_decl_top -297552 +286006 type_decl -297552 +286006 namespace_decls -151627 +145743 id -151627 +145743 namespace_id -8485 +8155 location -135422 +130167 bodylocation -135787 +130518 @@ -17807,7 +17797,7 @@ 1 2 -151627 +145743 @@ -17823,7 +17813,7 @@ 1 2 -151627 +145743 @@ -17839,7 +17829,7 @@ 1 2 -151627 +145743 @@ -17855,42 +17845,42 @@ 1 2 -4011 +3855 2 3 -1191 +1145 3 4 -449 +432 4 7 -717 +689 7 13 -656 +630 13 27 -656 +630 28 163 -644 +619 172 3743 -158 +151 @@ -17906,42 +17896,42 @@ 1 2 -4011 +3855 2 3 -1191 +1145 3 4 -449 +432 4 7 -717 +689 7 13 -656 +630 13 27 -656 +630 28 163 -644 +619 172 3743 -158 +151 @@ -17957,42 +17947,42 @@ 1 2 -4011 +3855 2 3 -1191 +1145 3 4 -449 +432 4 7 -717 +689 7 13 -656 +630 13 27 -656 +630 28 163 -644 +619 172 3742 -158 +151 @@ -18008,12 +17998,12 @@ 1 2 -126135 +121240 2 8 -9287 +8927 @@ -18029,12 +18019,12 @@ 1 2 -126135 +121240 2 8 -9287 +8927 @@ -18050,12 +18040,12 @@ 1 2 -134595 +129373 2 3 -826 +794 @@ -18071,12 +18061,12 @@ 1 2 -126900 +121976 2 11 -8886 +8541 @@ -18092,12 +18082,12 @@ 1 2 -126900 +121976 2 9 -8886 +8541 @@ -18113,12 +18103,12 @@ 1 2 -135349 +130097 2 5 -437 +420 @@ -18128,19 +18118,19 @@ usings -320601 +308161 id -320601 +308161 element_id -49002 +47101 location -26464 +25437 @@ -18154,7 +18144,7 @@ 1 2 -320601 +308161 @@ -18170,7 +18160,7 @@ 1 2 -320601 +308161 @@ -18186,17 +18176,17 @@ 1 2 -41234 +39634 2 4 -4169 +4007 4 127 -3598 +3458 @@ -18212,17 +18202,17 @@ 1 2 -41234 +39634 2 4 -4169 +4007 4 127 -3598 +3458 @@ -18238,22 +18228,22 @@ 1 2 -20033 +19256 2 3 -2540 +2442 3 21 -2054 +1974 21 347 -1835 +1764 @@ -18269,22 +18259,22 @@ 1 2 -20033 +19256 2 3 -2540 +2442 3 21 -2054 +1974 21 347 -1835 +1764 @@ -18294,15 +18284,15 @@ using_container -485187 +466360 parent -8667 +8331 child -301029 +289348 @@ -18316,47 +18306,47 @@ 1 2 -3379 +3248 2 3 -559 +537 3 6 -668 +642 6 15 -656 +630 16 31 -705 +677 31 143 -291 +280 178 179 -1397 +1343 179 182 -765 +736 182 501 -243 +233 @@ -18372,22 +18362,22 @@ 1 2 -217478 +209039 2 3 -55773 +53609 3 6 -22659 +21780 6 47 -5117 +4919 @@ -18397,23 +18387,23 @@ static_asserts -132358 +131017 id -132358 +131017 condition -132358 +131017 message -30094 +29789 location -16938 +16766 @@ -18427,7 +18417,7 @@ 1 2 -132358 +131017 @@ -18443,7 +18433,7 @@ 1 2 -132358 +131017 @@ -18459,7 +18449,7 @@ 1 2 -132358 +131017 @@ -18475,7 +18465,7 @@ 1 2 -132358 +131017 @@ -18491,7 +18481,7 @@ 1 2 -132358 +131017 @@ -18507,7 +18497,7 @@ 1 2 -132358 +131017 @@ -18523,32 +18513,32 @@ 1 2 -22395 +22168 2 3 -428 +424 3 4 -2867 +2837 4 11 -1463 +1448 12 17 -2379 +2355 17 513 -560 +554 @@ -18564,32 +18554,32 @@ 1 2 -22395 +22168 2 3 -428 +424 3 4 -2867 +2837 4 11 -1463 +1448 12 17 -2379 +2355 17 513 -560 +554 @@ -18605,12 +18595,12 @@ 1 2 -27912 +27629 2 33 -2181 +2159 @@ -18626,32 +18616,32 @@ 1 2 -2939 +2909 2 3 -2801 +2772 3 4 -1344 +1330 5 6 -3776 +3738 6 7 -177 +176 14 15 -2062 +2042 16 @@ -18661,12 +18651,12 @@ 17 18 -3447 +3412 19 52 -349 +345 @@ -18682,32 +18672,32 @@ 1 2 -2939 +2909 2 3 -2801 +2772 3 4 -1344 +1330 5 6 -3776 +3738 6 7 -177 +176 14 15 -2062 +2042 16 @@ -18717,12 +18707,12 @@ 17 18 -3447 +3412 19 52 -349 +345 @@ -18738,22 +18728,22 @@ 1 2 -4475 +4429 2 3 -6063 +6002 3 4 -6208 +6145 4 7 -191 +189 @@ -18763,23 +18753,23 @@ params -4803552 +4749557 id -4779409 +4724692 function -3118318 +3117564 index -389 +373 type_id -1865866 +1906163 @@ -18793,12 +18783,12 @@ 1 2 -4778668 +4723979 2 69 -741 +712 @@ -18814,7 +18804,7 @@ 1 2 -4779409 +4724692 @@ -18830,12 +18820,12 @@ 1 2 -4757747 +4702211 2 7 -21662 +22481 @@ -18851,22 +18841,22 @@ 1 2 -2207692 +2232786 2 3 -482853 +472740 3 4 -264220 +254691 4 33 -163552 +157346 @@ -18882,22 +18872,22 @@ 1 2 -2207692 +2232786 2 3 -482853 +472740 3 4 -264220 +254691 4 33 -163552 +157346 @@ -18913,22 +18903,22 @@ 1 2 -2314438 +2335471 2 3 -493113 +480884 3 5 -270978 +262941 5 20 -39787 +38267 @@ -18944,57 +18934,57 @@ 2 3 -158 +151 4 10 -24 +23 15 44 -24 +23 115 191 -24 +23 263 337 -24 +23 422 615 -24 +23 883 1209 -24 +23 1659 2596 -24 +23 4880 -13455 -24 +13467 +23 -35189 -74910 -24 +35263 +75722 +23 -256345 -256346 -12 +266636 +266637 +11 @@ -19010,57 +19000,57 @@ 2 3 -158 +151 4 10 -24 +23 15 44 -24 +23 115 191 -24 +23 263 337 -24 +23 422 615 -24 +23 883 1209 -24 +23 1659 2596 -24 +23 4880 -13455 -24 +13467 +23 -35189 -74910 -24 +35263 +75722 +23 -256516 -256517 -12 +266807 +266808 +11 @@ -19076,57 +19066,57 @@ 1 2 -158 +151 3 7 -24 +23 8 20 -24 +23 36 40 -24 +23 42 45 -24 +23 61 105 -24 +23 173 257 -24 +23 406 661 -24 +23 1178 2710 -24 +23 -7491 -16702 -24 +7492 +16795 +23 -135733 -135734 -12 +145286 +145287 +11 @@ -19142,22 +19132,22 @@ 1 2 -1495095 +1547921 2 3 -199608 +193171 3 -13 -141671 +18 +144002 -13 -5275 -29491 +18 +5288 +21067 @@ -19173,22 +19163,22 @@ 1 2 -1517195 +1569164 2 3 -187245 +181276 3 -17 -140893 +25 +143324 -17 -4798 -20532 +25 +4811 +12397 @@ -19204,12 +19194,12 @@ 1 2 -1751170 +1795906 2 33 -114695 +110257 @@ -19219,15 +19209,15 @@ overrides -161127 +159494 new -126169 +124890 old -15666 +15507 @@ -19241,12 +19231,12 @@ 1 2 -91218 +90293 2 3 -34944 +34590 3 @@ -19267,37 +19257,37 @@ 1 2 -8232 +8148 2 3 -2043 +2022 3 4 -883 +874 4 5 -1278 +1265 5 10 -1245 +1233 10 43 -1179 +1167 44 218 -804 +795 @@ -19307,19 +19297,19 @@ membervariables -329658 +325582 id -325099 +321201 type_id -145086 +139772 name -59068 +56776 @@ -19333,12 +19323,12 @@ 1 2 -320637 +316912 2 7 -4461 +4288 @@ -19354,7 +19344,7 @@ 1 2 -325099 +321201 @@ -19370,22 +19360,22 @@ 1 2 -118391 +113949 2 3 -13396 +12934 3 11 -10989 +10656 11 -1212 -2309 +1715 +2231 @@ -19401,17 +19391,17 @@ 1 2 -125916 +121310 2 3 -9883 +9511 3 266 -9287 +8950 @@ -19427,32 +19417,32 @@ 1 2 -31217 +29971 2 3 -9056 +8705 3 4 -5944 +5725 4 6 -4485 +4311 6 15 -4534 +4358 15 -1317 -3829 +1953 +3704 @@ -19468,32 +19458,32 @@ 1 2 -38256 +36736 2 3 -7524 +7232 3 4 -4035 +3891 4 7 -4765 +4592 7 -319 -4437 +291 +4264 -323 +318 605 -48 +58 @@ -19674,19 +19664,19 @@ localvariables -527180 +521836 id -527101 +521758 type_id -48087 +47599 name -76217 +75444 @@ -19700,12 +19690,12 @@ 1 2 -527022 +521680 2 3 -79 +78 @@ -19721,7 +19711,7 @@ 1 2 -527101 +521758 @@ -19737,32 +19727,32 @@ 1 2 -26759 +26487 2 3 -6841 +6772 3 4 -2893 +2864 4 6 -4119 +4077 6 13 -3862 +3823 13 4908 -3611 +3575 @@ -19778,22 +19768,22 @@ 1 2 -36072 +35706 2 3 -4969 +4919 3 5 -4323 +4279 5 1158 -2722 +2694 @@ -19809,32 +19799,32 @@ 1 2 -43526 +43085 2 3 -13096 +12963 3 4 -5140 +5088 4 7 -6821 +6752 7 31 -5767 +5708 31 6112 -1865 +1846 @@ -19850,17 +19840,17 @@ 1 2 -64564 +63910 2 3 -6472 +6406 3 819 -5180 +5127 @@ -19870,11 +19860,11 @@ autoderivation -19759 +19559 var -19759 +19559 derivation_type @@ -19892,7 +19882,7 @@ 1 2 -19759 +19559 @@ -19938,31 +19928,31 @@ enumconstants -94487 +94097 id -94487 +94097 parent -7539 +8031 index -7678 +7600 type_id -7394 +7887 name -73851 +73102 location -76856 +76077 @@ -19976,7 +19966,7 @@ 1 2 -94487 +94097 @@ -19992,7 +19982,7 @@ 1 2 -94487 +94097 @@ -20008,7 +19998,7 @@ 1 2 -94487 +94097 @@ -20024,7 +20014,7 @@ 1 2 -94487 +94097 @@ -20040,7 +20030,7 @@ 1 2 -94487 +94097 @@ -20056,47 +20046,47 @@ 1 2 -1041 +1598 2 3 -830 +822 3 4 -2484 +2459 4 5 -586 +580 5 6 -586 +580 6 -8 -540 +9 +730 -8 -13 -659 +9 +16 +613 -13 -46 -573 +16 +392 +606 -46 +426 1166 -237 +39 @@ -20112,47 +20102,42 @@ 1 2 -1041 +1598 2 3 -843 +835 3 4 -2530 +2505 4 5 -579 +574 5 6 -593 +587 6 -8 -527 +9 +704 -8 -13 -632 +9 +17 +626 -13 -49 -566 - - -50 +17 1166 -224 +600 @@ -20168,12 +20153,12 @@ 1 2 -6788 +7287 2 3 -751 +743 @@ -20189,47 +20174,42 @@ 1 2 -1041 +1598 2 3 -843 +835 3 4 -2530 +2505 4 5 -579 +574 5 6 -593 +587 6 -8 -527 +9 +704 -8 -13 -632 +9 +17 +626 -13 -49 -566 - - -50 +17 1166 -224 +600 @@ -20245,47 +20225,47 @@ 1 2 -1080 +1637 2 3 -843 +835 3 4 -2484 +2459 4 5 -586 +580 5 6 -566 +561 6 -8 -514 +9 +704 -8 -13 -659 +9 +16 +613 -13 -46 -566 +16 +427 +606 -46 +466 1166 -237 +32 @@ -20301,47 +20281,47 @@ 1 2 -2642 +2616 2 3 -942 +932 3 4 -935 +926 4 7 -579 +574 7 10 -698 +691 10 11 -355 +352 11 12 -560 +554 12 30 -606 +600 30 -1166 -355 +1253 +352 @@ -20357,47 +20337,47 @@ 1 2 -2642 +2616 2 3 -942 +932 3 4 -935 +926 4 7 -579 +574 7 9 -685 +678 9 11 -369 +365 11 12 -560 +554 12 29 -606 +600 29 -1145 -355 +1232 +352 @@ -20413,47 +20393,47 @@ 1 2 -2642 +2616 2 3 -942 +932 3 4 -935 +926 4 7 -579 +574 7 9 -685 +678 9 11 -369 +365 11 12 -560 +554 12 28 -606 +600 28 -1123 -355 +1210 +352 @@ -20469,47 +20449,47 @@ 1 2 -2642 +2616 2 3 -942 +932 3 4 -935 +926 4 7 -579 +574 7 10 -698 +691 10 11 -355 +352 11 12 -560 +554 12 28 -606 +600 28 674 -355 +352 @@ -20525,47 +20505,47 @@ 1 2 -2642 +2616 2 3 -942 +932 3 4 -935 +926 4 7 -579 +574 7 10 -698 +691 10 11 -355 +352 11 12 -560 +554 12 28 -606 +600 28 774 -355 +352 @@ -20581,47 +20561,47 @@ 1 2 -1041 +1598 2 3 -810 +802 3 4 -2458 +2433 4 5 -566 +561 5 6 -566 +561 6 -8 -520 +9 +704 -8 -13 -639 +9 +16 +600 -13 -46 -560 +16 +470 +593 -46 +479 1166 -230 +32 @@ -20637,7 +20617,7 @@ 1 2 -7388 +7881 137 @@ -20658,47 +20638,42 @@ 1 2 -1041 +1598 2 3 -823 +815 3 4 -2504 +2479 4 5 -560 +554 5 6 -573 +567 6 -8 -507 +9 +678 -8 -13 -612 +9 +17 +613 -13 -51 -560 - - -52 +17 1166 -210 +580 @@ -20714,47 +20689,42 @@ 1 2 -1041 +1598 2 3 -823 +815 3 4 -2504 +2479 4 5 -560 +554 5 6 -573 +567 6 -8 -507 +9 +678 -8 -13 -612 +9 +17 +613 -13 -51 -560 - - -52 +17 1166 -210 +580 @@ -20770,47 +20740,47 @@ 1 2 -1080 +1637 2 3 -823 +815 3 4 -2458 +2433 4 5 -566 +561 5 6 -547 +541 6 -8 -494 +9 +678 -8 -13 -639 +9 +16 +600 -13 -47 -560 +16 +470 +593 -48 +621 1166 -224 +26 @@ -20826,17 +20796,17 @@ 1 2 -66396 +65723 2 3 -6927 +6856 3 239 -527 +521 @@ -20852,17 +20822,17 @@ 1 2 -67049 +66369 2 3 -6274 +6210 3 239 -527 +521 @@ -20878,12 +20848,12 @@ 1 2 -68762 +68065 2 13 -5088 +5036 @@ -20899,17 +20869,17 @@ 1 2 -61941 +61313 2 3 -11382 +11267 3 239 -527 +521 @@ -20925,12 +20895,12 @@ 1 2 -71346 +70623 2 23 -2504 +2479 @@ -20946,12 +20916,12 @@ 1 2 -71531 +70773 2 229 -5325 +5304 @@ -20967,12 +20937,12 @@ 1 2 -71643 +70884 2 229 -5213 +5193 @@ -20988,12 +20958,12 @@ 1 2 -73211 +72469 2 17 -3644 +3607 @@ -21009,17 +20979,17 @@ 1 2 -66535 +65828 2 3 -10090 +9994 3 229 -230 +254 @@ -21035,12 +21005,12 @@ 1 2 -76685 +75907 2 27 -171 +169 @@ -21050,31 +21020,31 @@ builtintypes -559 +537 id -559 +537 name -559 +537 kind -559 +537 size -85 +81 sign -36 +35 alignment -60 +58 @@ -21088,7 +21058,7 @@ 1 2 -559 +537 @@ -21104,7 +21074,7 @@ 1 2 -559 +537 @@ -21120,7 +21090,7 @@ 1 2 -559 +537 @@ -21136,7 +21106,7 @@ 1 2 -559 +537 @@ -21152,7 +21122,7 @@ 1 2 -559 +537 @@ -21168,7 +21138,7 @@ 1 2 -559 +537 @@ -21184,7 +21154,7 @@ 1 2 -559 +537 @@ -21200,7 +21170,7 @@ 1 2 -559 +537 @@ -21216,7 +21186,7 @@ 1 2 -559 +537 @@ -21232,7 +21202,7 @@ 1 2 -559 +537 @@ -21248,7 +21218,7 @@ 1 2 -559 +537 @@ -21264,7 +21234,7 @@ 1 2 -559 +537 @@ -21280,7 +21250,7 @@ 1 2 -559 +537 @@ -21296,7 +21266,7 @@ 1 2 -559 +537 @@ -21312,7 +21282,7 @@ 1 2 -559 +537 @@ -21328,37 +21298,37 @@ 1 2 -12 +11 2 3 -12 +11 4 5 -12 +11 6 7 -12 +11 9 10 -12 +11 11 12 -12 +11 13 14 -12 +11 @@ -21374,37 +21344,37 @@ 1 2 -12 +11 2 3 -12 +11 4 5 -12 +11 6 7 -12 +11 9 10 -12 +11 11 12 -12 +11 13 14 -12 +11 @@ -21420,37 +21390,37 @@ 1 2 -12 +11 2 3 -12 +11 4 5 -12 +11 6 7 -12 +11 9 10 -12 +11 11 12 -12 +11 13 14 -12 +11 @@ -21466,12 +21436,12 @@ 1 2 -24 +23 3 4 -60 +58 @@ -21487,12 +21457,12 @@ 1 2 -60 +58 2 3 -24 +23 @@ -21508,17 +21478,17 @@ 6 7 -12 +11 12 13 -12 +11 28 29 -12 +11 @@ -21534,17 +21504,17 @@ 6 7 -12 +11 12 13 -12 +11 28 29 -12 +11 @@ -21560,17 +21530,17 @@ 6 7 -12 +11 12 13 -12 +11 28 29 -12 +11 @@ -21586,12 +21556,12 @@ 5 6 -24 +23 7 8 -12 +11 @@ -21607,7 +21577,7 @@ 5 6 -36 +35 @@ -21623,27 +21593,27 @@ 4 5 -12 +11 7 8 -12 +11 10 11 -12 +11 12 13 -12 +11 13 14 -12 +11 @@ -21659,27 +21629,27 @@ 4 5 -12 +11 7 8 -12 +11 10 11 -12 +11 12 13 -12 +11 13 14 -12 +11 @@ -21695,27 +21665,27 @@ 4 5 -12 +11 7 8 -12 +11 10 11 -12 +11 12 13 -12 +11 13 14 -12 +11 @@ -21731,12 +21701,12 @@ 1 2 -12 +11 2 3 -48 +46 @@ -21752,7 +21722,7 @@ 3 4 -60 +58 @@ -21762,23 +21732,23 @@ derivedtypes -4572495 +4651908 id -4572495 +4651908 name -2240442 +2328554 kind -97 +93 type_id -2743147 +2780308 @@ -21792,7 +21762,7 @@ 1 2 -4572495 +4651908 @@ -21808,7 +21778,7 @@ 1 2 -4572495 +4651908 @@ -21824,7 +21794,7 @@ 1 2 -4572495 +4651908 @@ -21840,17 +21810,17 @@ 1 2 -1640327 +1709696 2 3 -479376 +500222 3 -42612 -120737 +43371 +118634 @@ -21866,12 +21836,12 @@ 1 2 -2240417 +2328530 2 3 -24 +23 @@ -21887,17 +21857,17 @@ 1 2 -1640607 +1709965 2 3 -479097 +499954 3 -42594 -120737 +43353 +118634 @@ -21913,42 +21883,42 @@ 21 22 -12 +11 61 62 -12 +11 2051 2052 -12 +11 -25122 -25123 -12 +27346 +27347 +11 -42664 -42665 -12 +43423 +43424 +11 -53936 -53937 -12 +58521 +58522 +11 -97759 -97760 -12 +102765 +102766 +11 -154522 -154523 -12 +163929 +163930 +11 @@ -21964,42 +21934,42 @@ 1 2 -12 +11 13 14 -12 +11 38 39 -12 +11 1152 1153 -12 +11 -13367 -13368 -12 +15027 +15028 +11 -36098 -36099 -12 +39331 +39332 +11 -50248 -50249 -12 +53789 +53790 +11 -83386 -83387 -12 +89933 +89934 +11 @@ -22015,42 +21985,42 @@ 12 13 -12 +11 21 22 -12 +11 969 970 -12 +11 -25122 -25123 -12 +27346 +27347 +11 -42664 -42665 -12 +43423 +43424 +11 -53936 -53937 -12 +58521 +58522 +11 -97431 -97432 -12 +102434 +102435 +11 -154522 -154523 -12 +163929 +163930 +11 @@ -22066,22 +22036,22 @@ 1 2 -1661856 +1683160 2 3 -405550 +392781 3 4 -615674 +646083 4 202 -60064 +58283 @@ -22097,22 +22067,22 @@ 1 2 -1663206 +1684457 2 3 -405380 +392618 3 4 -614495 +644949 4 198 -60064 +58283 @@ -22128,22 +22098,22 @@ 1 2 -1663534 +1684784 2 3 -406814 +393985 3 4 -614446 +644902 4 7 -58350 +56635 @@ -22153,19 +22123,19 @@ pointerishsize -3358396 +3426426 id -3358396 +3426426 size -24 +23 alignment -12 +11 @@ -22179,7 +22149,7 @@ 1 2 -3358396 +3426426 @@ -22195,7 +22165,7 @@ 1 2 -3358396 +3426426 @@ -22211,12 +22181,12 @@ 21 22 -12 +11 -276244 -276245 -12 +293219 +293220 +11 @@ -22232,7 +22202,7 @@ 1 2 -24 +23 @@ -22246,9 +22216,9 @@ 12 -276265 -276266 -12 +293240 +293241 +11 @@ -22264,7 +22234,7 @@ 2 3 -12 +11 @@ -22274,23 +22244,23 @@ arraysizes -18830 +18099 id -18830 +18099 num_elements -2394 +2301 bytesize -2783 +2675 alignment -85 +81 @@ -22304,7 +22274,7 @@ 1 2 -18830 +18099 @@ -22320,7 +22290,7 @@ 1 2 -18830 +18099 @@ -22336,7 +22306,7 @@ 1 2 -18830 +18099 @@ -22352,37 +22322,37 @@ 1 2 -182 +175 2 3 -1422 +1367 3 4 -85 +81 4 5 -182 +175 5 13 -182 +175 13 25 -182 +175 38 116 -158 +151 @@ -22398,27 +22368,27 @@ 1 2 -1762 +1694 2 3 -243 +233 3 4 -133 +128 4 6 -182 +175 6 11 -72 +70 @@ -22434,27 +22404,27 @@ 1 2 -1774 +1705 2 3 -255 +245 3 4 -158 +151 4 7 -182 +175 7 8 -24 +23 @@ -22470,42 +22440,42 @@ 1 2 -170 +163 2 3 -1556 +1495 3 4 -133 +128 4 6 -218 +210 6 9 -218 +210 9 17 -218 +210 17 53 -218 +210 55 75 -48 +46 @@ -22521,22 +22491,22 @@ 1 2 -2115 +2033 2 3 -389 +373 3 6 -243 +233 6 7 -36 +35 @@ -22552,22 +22522,22 @@ 1 2 -2176 +2091 2 3 -303 +292 3 5 -255 +245 5 7 -48 +46 @@ -22583,37 +22553,37 @@ 17 18 -12 +11 18 19 -12 +11 33 34 -12 +11 50 51 -12 +11 181 182 -12 +11 389 390 -12 +11 861 862 -12 +11 @@ -22629,37 +22599,37 @@ 4 5 -12 +11 5 6 -12 +11 13 14 -12 +11 16 17 -12 +11 39 40 -12 +11 40 41 -12 +11 190 191 -12 +11 @@ -22675,37 +22645,37 @@ 1 2 -12 +11 2 3 -12 +11 14 15 -12 +11 17 18 -12 +11 41 42 -12 +11 53 54 -12 +11 191 192 -12 +11 @@ -22715,15 +22685,15 @@ typedefbase -1919913 +1867884 id -1919913 +1867884 type_id -899600 +871937 @@ -22737,7 +22707,7 @@ 1 2 -1919913 +1867884 @@ -22753,22 +22723,22 @@ 1 2 -698168 +676919 2 3 -93312 +90264 3 6 -72975 +70856 6 -5423 -35144 +5485 +33897 @@ -22778,23 +22748,23 @@ decltypes -45841 +44063 id -45841 +44063 expr -43325 +41644 base_type -6479 +6227 parentheses_would_change_meaning -24 +23 @@ -22808,7 +22778,7 @@ 1 2 -45841 +44063 @@ -22824,7 +22794,7 @@ 1 2 -45841 +44063 @@ -22840,7 +22810,7 @@ 1 2 -45841 +44063 @@ -22856,12 +22826,12 @@ 1 2 -40809 +39225 2 3 -2516 +2418 @@ -22877,12 +22847,12 @@ 1 2 -40809 +39225 2 3 -2516 +2418 @@ -22898,7 +22868,7 @@ 1 2 -43325 +41644 @@ -22914,17 +22884,17 @@ 1 2 -3622 +3482 2 3 -2552 +2453 3 277 -303 +292 @@ -22940,22 +22910,22 @@ 1 2 -2540 +2442 2 3 -3282 +3154 3 12 -522 +502 13 2445 -133 +128 @@ -22971,7 +22941,7 @@ 1 2 -6479 +6227 @@ -22987,12 +22957,12 @@ 8 9 -12 +11 1082 1083 -12 +11 @@ -23008,12 +22978,12 @@ 8 9 -12 +11 3556 3557 -12 +11 @@ -23029,12 +22999,12 @@ 8 9 -12 +11 525 526 -12 +11 @@ -23044,19 +23014,19 @@ usertypes -4363319 +4337028 id -4363319 +4337028 name -899429 +917472 kind -133 +128 @@ -23070,7 +23040,7 @@ 1 2 -4363319 +4337028 @@ -23086,7 +23056,7 @@ 1 2 -4363319 +4337028 @@ -23102,22 +23072,22 @@ 1 2 -596127 +606962 2 3 -199888 +203676 3 8 -67589 +69886 8 -31201 -35824 +32070 +36947 @@ -23133,12 +23103,12 @@ 1 2 -842355 +861619 2 10 -57074 +55852 @@ -23154,57 +23124,57 @@ 23 24 -12 +11 372 373 -12 +11 773 774 -12 +11 -1526 -1527 -12 +1856 +1857 +11 -4123 -4124 -12 +4195 +4196 +11 14604 14605 -12 +11 -17528 -17529 -12 +17544 +17545 +11 -20287 -20288 -12 +20425 +20426 +11 -75247 -75248 -12 +75551 +75552 +11 -81118 -81119 -12 +90575 +90576 +11 -143330 -143331 -12 +145253 +145254 +11 @@ -23220,52 +23190,52 @@ 16 17 -12 +11 38 39 -24 +23 404 405 -12 +11 548 549 -12 +11 -776 -777 -12 +777 +778 +11 -2846 -2847 -12 +2848 +2849 +11 4545 4546 -12 +11 -9121 -9122 -12 +9129 +9130 +11 11659 11660 -12 +11 -49218 -49219 -12 +53824 +53825 +11 @@ -23275,19 +23245,19 @@ usertypesize -1364814 +1420908 id -1364814 +1420908 size -1641 +1577 alignment -97 +93 @@ -23301,7 +23271,7 @@ 1 2 -1364814 +1420908 @@ -23317,7 +23287,7 @@ 1 2 -1364814 +1420908 @@ -23333,52 +23303,52 @@ 1 2 -474 +455 2 3 -218 +210 3 4 -72 +70 4 5 -109 +105 5 8 -121 +116 8 12 -109 +105 12 16 -133 +128 16 36 -133 +128 37 170 -133 +128 260 -86086 -133 +95028 +128 @@ -23394,17 +23364,17 @@ 1 2 -1349 +1297 2 3 -194 +186 3 6 -97 +93 @@ -23420,42 +23390,42 @@ 1 2 -12 +11 3 4 -12 +11 7 8 -12 +11 50 51 -12 +11 54 55 -12 +11 -1916 -1917 -12 +2246 +2247 +11 -9656 -9657 -12 +9717 +9718 +11 -100584 -100585 -12 +109526 +109527 +11 @@ -23471,37 +23441,37 @@ 1 2 -24 +23 3 4 -12 +11 8 9 -12 +11 12 13 -12 +11 17 18 -12 +11 25 26 -12 +11 106 107 -12 +11 @@ -23511,26 +23481,26 @@ usertype_final -1331 +1317 id -1331 +1317 usertype_uuid -4897 +4847 id -4897 +4847 uuid -4897 +4847 @@ -23544,7 +23514,7 @@ 1 2 -4897 +4847 @@ -23560,7 +23530,7 @@ 1 2 -4897 +4847 @@ -23570,15 +23540,15 @@ mangled_name -4359684 +4333558 id -4359684 +4333558 mangled_name -532986 +514256 @@ -23592,7 +23562,7 @@ 1 2 -4359684 +4333558 @@ -23608,32 +23578,32 @@ 1 2 -325269 +310895 2 3 -68246 +66497 3 4 -35958 +35217 4 -8 -45307 +7 +39225 -8 -34 -40237 +7 +23 +38664 -34 -8470 -17967 +23 +8589 +23755 @@ -23643,59 +23613,59 @@ is_pod_class -1041477 +1104649 id -1041477 +1104649 is_standard_layout_class -1118962 +1180366 id -1118962 +1180366 is_complete -1345984 +1398952 id -1345984 +1398952 is_class_template -246617 +238660 id -246617 +238660 class_instantiation -1108921 +1176522 to -1107195 +1174863 from -72513 +72363 @@ -23709,12 +23679,12 @@ 1 2 -1105566 +1173297 2 4 -1628 +1565 @@ -23730,47 +23700,47 @@ 1 2 -21601 +21289 2 3 -12897 +12771 3 4 -7488 +7314 4 5 -4789 +4942 5 7 -5871 +5994 7 11 -6503 +6438 11 20 -5555 +5585 20 -89 -5470 +87 +5456 -89 -2079 -2334 +87 +3629 +2570 @@ -23780,19 +23750,19 @@ class_template_argument -2800063 +3135757 type_id -1339675 +1428725 index -1361 +1308 arg_type -921287 +904899 @@ -23806,27 +23776,27 @@ 1 2 -593732 +586000 2 3 -438105 +428957 3 4 -175599 +258501 4 7 -108629 +129501 7 113 -23607 +25764 @@ -23842,22 +23812,22 @@ 1 2 -619710 +613658 2 3 -445691 +442173 3 4 -187999 +272382 4 113 -86274 +100511 @@ -23873,37 +23843,37 @@ 1 2 -12 +11 2 3 -863 +829 3 26 -109 +105 29 64 -109 +105 69 -427 -109 +435 +105 -594 -8438 -109 +616 +9263 +105 -11951 -101100 -48 +13601 +119714 +46 @@ -23919,37 +23889,37 @@ 1 2 -12 +11 2 3 -863 +829 3 14 -121 +116 14 26 -109 +105 29 -147 -109 +148 +105 -191 -3560 -109 +198 +3591 +105 -11321 -39550 -36 +11582 +41130 +35 @@ -23965,22 +23935,27 @@ 1 2 -571096 +550198 2 3 -205784 +200100 3 -5 -75369 +4 +54918 -5 -5259 -69036 +4 +10 +70727 + + +10 +11421 +28954 @@ -23996,17 +23971,253 @@ 1 2 -805922 +786767 2 3 -96011 +98069 3 22 -19353 +20062 + + + + + + + + +class_template_argument_value +362635 + + +type_id +233565 + + +index +163 + + +arg_value +345447 + + + + +type_id +index + + +12 + + +1 +2 +209845 + + +2 +3 +14512 + + +3 +14 +9207 + + + + + + +type_id +arg_value + + +12 + + +1 +2 +197436 + + +2 +3 +17971 + + +3 +22 +17527 + + +24 +173 +630 + + + + + + +index +type_id + + +12 + + +8 +9 +58 + + +20 +21 +11 + + +25 +26 +11 + + +206 +207 +11 + + +389 +390 +11 + + +552 +553 +11 + + +1312 +1313 +11 + + +5393 +5394 +11 + + +5423 +5424 +11 + + +9668 +9669 +11 + + + + + + +index +arg_value + + +12 + + +8 +9 +58 + + +20 +21 +11 + + +42 +43 +11 + + +311 +312 +11 + + +514 +515 +11 + + +715 +716 +11 + + +1585 +1586 +11 + + +6302 +6303 +11 + + +8160 +8161 +11 + + +11875 +11876 +11 + + + + + + +arg_value +type_id + + +12 + + +1 +2 +328387 + + +2 +4 +17059 + + + + + + +arg_value +index + + +12 + + +1 +2 +345447 @@ -24016,15 +24227,15 @@ is_proxy_class_for -50120 +49017 id -50120 +49017 templ_param_id -50120 +49017 @@ -24038,7 +24249,7 @@ 1 2 -50120 +49017 @@ -24054,7 +24265,7 @@ 1 2 -50120 +49017 @@ -24064,19 +24275,19 @@ type_mentions -1699359 +1682135 id -1699359 +1682135 type_id -68077 +67387 location -1668455 +1651543 kind @@ -24094,7 +24305,7 @@ 1 2 -1699359 +1682135 @@ -24110,7 +24321,7 @@ 1 2 -1699359 +1682135 @@ -24126,7 +24337,7 @@ 1 2 -1699359 +1682135 @@ -24142,37 +24353,37 @@ 1 2 -30463 +30154 2 3 -12463 +12337 3 4 -3684 +3646 4 7 -6142 +6080 7 13 -5213 +5160 13 35 -5259 +5206 35 9490 -4850 +4801 @@ -24188,37 +24399,37 @@ 1 2 -30463 +30154 2 3 -12463 +12337 3 4 -3684 +3646 4 7 -6142 +6080 7 13 -5213 +5160 13 35 -5259 +5206 35 9490 -4850 +4801 @@ -24234,12 +24445,12 @@ 1 2 -66818 +66141 2 3 -1258 +1246 @@ -24255,12 +24466,12 @@ 1 2 -1637892 +1621291 2 5 -30562 +30252 @@ -24276,12 +24487,12 @@ 1 2 -1637892 +1621291 2 5 -30562 +30252 @@ -24297,7 +24508,7 @@ 1 2 -1668455 +1651543 @@ -24370,26 +24581,26 @@ is_function_template -1058083 +1023581 id -1058083 +1023581 function_instantiation -739463 +719194 to -739463 +719194 from -136103 +134923 @@ -24403,7 +24614,7 @@ 1 2 -739463 +719194 @@ -24419,37 +24630,32 @@ 1 2 -63018 +64125 2 3 -32396 +31385 3 4 -8193 +7898 4 -5 -9299 +6 +12467 -5 -10 -10940 +6 +15 +10434 -10 -57 -10223 - - -58 +15 645 -2030 +8611 @@ -24459,19 +24665,19 @@ function_template_argument -1910796 +1981225 function_id -1112422 +1088560 index -243 +233 arg_type -369847 +359386 @@ -24485,22 +24691,22 @@ 1 2 -634796 +600045 2 3 -361033 +302330 3 -5 -84645 +4 +131242 -5 +4 21 -31947 +54941 @@ -24516,22 +24722,22 @@ 1 2 -657966 +618530 2 3 -332393 +293520 3 -5 -90905 +4 +116683 -5 +4 21 -31156 +59825 @@ -24547,102 +24753,102 @@ 4 5 -12 +11 7 8 -12 +11 17 18 -12 +11 39 40 -12 +11 65 66 -12 +11 152 153 -12 +11 -238 -239 -12 +241 +242 +11 -325 -326 -12 +328 +329 +11 -422 -423 -12 +425 +426 +11 -520 -521 -12 +523 +524 +11 -754 -755 -12 +758 +759 +11 -1007 -1008 -12 +1011 +1012 +11 -1259 -1260 -12 +1520 +1521 +11 -1514 -1515 -12 +1701 +1702 +11 -2559 -2560 -12 +2566 +2567 +11 -2804 -2805 -12 +2917 +2918 +11 -4818 -4819 -12 +5686 +5687 +11 -13309 -13310 -12 +17602 +17603 +11 -40569 -40570 -12 +42858 +42859 +11 -84931 -84932 -12 +88286 +88287 +11 @@ -24658,102 +24864,102 @@ 4 5 -12 +11 7 8 -12 +11 14 15 -12 +11 22 23 -12 +11 32 33 -12 +11 54 55 -12 +11 -57 -58 -12 +60 +61 +11 -61 -62 -12 +62 +63 +11 -75 -76 -12 +78 +79 +11 -90 -91 -12 +91 +92 +11 -137 -138 -12 +140 +141 +11 214 215 -12 +11 -238 -239 -12 +242 +243 +11 -316 -317 -12 +317 +318 +11 -517 -518 -12 +521 +522 +11 -695 -696 -12 +700 +701 +11 -1468 -1469 -12 +1491 +1492 +11 -3652 -3653 -12 +3698 +3699 +11 -8773 -8774 -12 +8798 +8799 +11 -17388 -17389 -12 +17653 +17654 +11 @@ -24769,27 +24975,27 @@ 1 2 -249547 +242738 2 3 -48698 +46902 3 6 -29783 +28756 6 -19 -27996 +20 +27704 -19 -906 -13821 +20 +1989 +13285 @@ -24805,17 +25011,238 @@ 1 2 -341230 +331740 2 5 -28129 +27131 5 -10 -486 +12 +514 + + + + + + + + +function_template_argument_value +209892 + + +function_id +113037 + + +index +163 + + +arg_value +181755 + + + + +function_id +index + + +12 + + +1 +2 +107207 + + +2 +14 +5830 + + + + + + +function_id +arg_value + + +12 + + +1 +2 +89610 + + +2 +3 +17106 + + +3 +113 +6321 + + + + + + +index +function_id + + +12 + + +3 +4 +70 + + +5 +6 +11 + + +6 +7 +11 + + +111 +112 +11 + + +441 +442 +11 + + +857 +858 +11 + + +2136 +2137 +11 + + +2428 +2429 +11 + + +4217 +4218 +11 + + + + + + +index +arg_value + + +12 + + +5 +6 +70 + + +7 +8 +11 + + +8 +9 +11 + + +157 +158 +11 + + +405 +406 +11 + + +973 +974 +11 + + +2523 +2524 +11 + + +4848 +4849 +11 + + +6604 +6605 +11 + + + + + + +arg_value +function_id + + +12 + + +1 +2 +153712 + + +2 +3 +27949 + + +3 +4 +93 + + + + + + +arg_value +index + + +12 + + +1 +2 +181755 @@ -24825,26 +25252,26 @@ is_variable_template -19778 +19011 id -19778 +19011 variable_instantiation -31837 +38057 to -31837 +38057 from -6734 +6929 @@ -24858,7 +25285,7 @@ 1 2 -31837 +38057 @@ -24874,37 +25301,37 @@ 1 2 -2382 +2383 2 3 -2030 +2033 3 4 -449 +432 4 5 -692 +782 5 9 -534 +560 9 -29 -534 +21 +525 -42 -214 -109 +24 +286 +210 @@ -24914,11 +25341,11 @@ variable_template_argument -5529 +5499 variable_id -843 +848 index @@ -24926,7 +25353,7 @@ arg_type -4257 +4214 @@ -24940,12 +25367,12 @@ 1 2 -692 +685 2 3 -112 +123 3 @@ -24966,12 +25393,12 @@ 1 2 -369 +371 2 3 -125 +130 3 @@ -24981,12 +25408,12 @@ 4 6 -72 +71 6 8 -72 +71 8 @@ -25025,13 +25452,13 @@ 6 -9 -10 +10 +11 6 -17 -18 +20 +21 6 @@ -25066,8 +25493,8 @@ 6 -137 -138 +139 +140 6 @@ -25089,17 +25516,17 @@ 1 2 -3532 +3490 2 3 -481 +476 3 11 -243 +247 @@ -25115,12 +25542,148 @@ 1 2 -4046 +3999 + + +2 +4 +215 + + + + + + + + +variable_template_argument_value +137 + + +variable_id +19 + + +index +13 + + +arg_value +137 + + + + +variable_id +index + + +12 + + +1 +2 +13 2 3 -210 +6 + + + + + + +variable_id +arg_value + + +12 + + +3 +4 +13 + + +15 +16 +6 + + + + + + +index +variable_id + + +12 + + +1 +2 +6 + + +3 +4 +6 + + + + + + +index +arg_value + + +12 + + +6 +7 +6 + + +15 +16 +6 + + + + + + +arg_value +variable_id + + +12 + + +1 +2 +137 + + + + + + +arg_value +index + + +12 + + +1 +2 +137 @@ -25130,15 +25693,15 @@ routinetypes -444572 +436132 id -444572 +436132 return_type -184814 +178086 @@ -25152,7 +25715,7 @@ 1 2 -444572 +436132 @@ -25168,22 +25731,22 @@ 1 2 -148344 +142985 2 3 -21030 +20191 3 -18 -13943 +17 +13367 -18 -7709 -1495 +17 +8001 +1542 @@ -25193,19 +25756,19 @@ routinetypeargs -741201 +726544 routine -362760 +357377 index -389 +373 type_id -209394 +208724 @@ -25219,27 +25782,27 @@ 1 2 -169412 +166448 2 3 -95355 +96574 3 4 -58144 +55888 4 6 -31351 +30298 6 33 -8497 +8167 @@ -25255,22 +25818,22 @@ 1 2 -194831 +190916 2 3 -95853 +97018 3 4 -49707 +47778 4 22 -22367 +21663 @@ -25286,62 +25849,62 @@ 1 2 -133 +128 2 7 -24 +23 10 16 -24 +23 19 33 -24 +23 52 74 -24 +23 93 115 -24 +23 139 199 -24 +23 269 364 -24 +23 473 700 -24 +23 1319 -3279 -24 +3293 +23 -8061 -15906 -24 +8075 +16341 +23 -29841 -29842 -12 +30585 +30586 +11 @@ -25357,62 +25920,62 @@ 1 2 -133 +128 2 5 -24 +23 6 10 -24 +23 11 22 -24 +23 33 37 -24 +23 38 42 -24 +23 44 83 -24 +23 124 183 -24 +23 248 346 -24 +23 463 950 -24 +23 -2463 -5546 -24 +2464 +5642 +23 -12306 -12307 -12 +12850 +12851 +11 @@ -25428,27 +25991,27 @@ 1 2 -123691 +124231 2 3 -40869 +41258 3 4 -13505 +13016 4 7 -17942 +17293 7 -1097 -13384 +1099 +12923 @@ -25464,17 +26027,17 @@ 1 2 -156830 +158176 2 3 -41161 +39576 3 33 -11402 +10971 @@ -25484,19 +26047,19 @@ ptrtomembers -12180 +13227 id -12180 +13227 type_id -10247 +10013 class_id -5519 +6601 @@ -25510,7 +26073,7 @@ 1 2 -12180 +13227 @@ -25526,7 +26089,7 @@ 1 2 -12180 +13227 @@ -25542,12 +26105,12 @@ 1 2 -9895 +9628 2 -65 -352 +100 +385 @@ -25563,12 +26126,12 @@ 1 2 -9895 +9628 2 -65 -352 +100 +385 @@ -25584,22 +26147,22 @@ 1 2 -4619 +5573 2 -7 -352 +3 +490 8 -9 -461 +17 +502 -16 +64 65 -85 +35 @@ -25615,22 +26178,22 @@ 1 2 -4619 +5573 2 -7 -352 +3 +490 8 -9 -461 +17 +502 -16 +64 65 -85 +35 @@ -25640,15 +26203,15 @@ specifiers -547 +525 id -547 +525 str -547 +525 @@ -25662,7 +26225,7 @@ 1 2 -547 +525 @@ -25678,7 +26241,7 @@ 1 2 -547 +525 @@ -25688,15 +26251,15 @@ typespecifiers -1464874 +1479986 type_id -1460193 +1472695 spec_id -85 +81 @@ -25710,12 +26273,12 @@ 1 2 -1455513 +1465403 2 3 -4680 +7291 @@ -25731,37 +26294,37 @@ 102 103 -12 +11 222 223 -12 +11 518 519 -12 +11 -718 -719 -12 +957 +958 +11 2323 2324 -12 +11 -19328 -19329 -12 +20241 +20242 +11 -97291 -97292 -12 +102297 +102298 +11 @@ -25771,15 +26334,15 @@ funspecifiers -11370456 +11412148 func_id -3511992 +3500391 spec_id -194 +186 @@ -25793,27 +26356,27 @@ 1 2 -342616 +329999 2 3 -464083 +448167 3 4 -909155 +883458 4 5 -1670402 +1717910 5 8 -125733 +120855 @@ -25829,82 +26392,82 @@ 22 23 -12 +11 169 170 -12 +11 491 492 -12 +11 -506 -507 -12 +598 +599 +11 632 633 -12 +11 5650 5651 -12 +11 9530 9531 -12 +11 10451 10452 -12 +11 -11821 -11822 -12 +11834 +11835 +11 14848 14849 -12 +11 -25330 -25331 -12 +26196 +26197 +11 -33532 -33533 -12 +33724 +33725 +11 -122462 -122463 -12 +131801 +131802 +11 -204591 -204592 -12 +214925 +214926 +11 -232301 -232302 -12 +242884 +242885 +11 -263009 -263010 -12 +272918 +272919 +11 @@ -25914,11 +26477,11 @@ varspecifiers -1124158 +1113938 var_id -935091 +926194 spec_id @@ -25936,17 +26499,17 @@ 1 2 -795258 +787198 2 3 -92279 +91911 3 5 -47553 +47084 @@ -25980,13 +26543,13 @@ 6 -8409 -8410 +8411 +8412 6 -8751 -8752 +8753 +8754 6 @@ -25995,13 +26558,13 @@ 6 -13352 -13353 +13439 +13440 6 -42323 -42324 +42412 +42413 6 @@ -26590,11 +27153,11 @@     attribute_args -118319 +118389 id -118319 +118389 kind @@ -26602,7 +27165,7 @@ attribute -116952 +117020 index @@ -26610,7 +27173,7 @@ location -58256 +58290 @@ -26624,7 +27187,7 @@ 1 2 -118319 +118389 @@ -26640,7 +27203,7 @@ 1 2 -118319 +118389 @@ -26656,7 +27219,7 @@ 1 2 -118319 +118389 @@ -26672,7 +27235,7 @@ 1 2 -118319 +118389 @@ -26772,12 +27335,12 @@ 1 2 -116132 +116200 2 4 -819 +820 @@ -26793,7 +27356,7 @@ 1 2 -116431 +116499 2 @@ -26814,12 +27377,12 @@ 1 2 -116132 +116200 2 4 -819 +820 @@ -26835,7 +27398,7 @@ 1 2 -116137 +116205 2 @@ -26955,22 +27518,22 @@ 1 2 -28468 +28485 2 3 -7516 +7521 3 4 -19458 +19469 4 25 -2812 +2814 @@ -26986,12 +27549,12 @@ 1 2 -52020 +52051 2 3 -6235 +6239 @@ -27007,22 +27570,22 @@ 1 2 -28452 +28469 2 3 -7534 +7539 3 4 -19454 +19466 4 25 -2814 +2815 @@ -27038,7 +27601,7 @@ 1 2 -58250 +58285 3 @@ -27053,15 +27616,15 @@ attribute_arg_value -118292 +118362 arg -118292 +118362 value -2257 +2258 @@ -27075,7 +27638,7 @@ 1 2 -118292 +118362 @@ -27091,7 +27654,7 @@ 1 2 -1808 +1809 2 @@ -27116,15 +27679,15 @@ attribute_arg_type -60 +58 arg -60 +58 type_id -36 +35 @@ -27138,7 +27701,7 @@ 1 2 -60 +58 @@ -27154,12 +27717,12 @@ 1 2 -12 +11 2 3 -24 +23 @@ -27222,15 +27785,15 @@ typeattributes -12119 +12701 type_id -10831 +11462 spec_id -12119 +12701 @@ -27244,12 +27807,12 @@ 1 2 -10296 +10948 2 34 -534 +514 @@ -27265,7 +27828,7 @@ 1 2 -12119 +12701 @@ -27275,15 +27838,15 @@ funcattributes -327044 +314529 func_id -174943 +168330 spec_id -327044 +314529 @@ -27297,22 +27860,22 @@ 1 2 -94078 +90603 2 3 -13554 +13028 3 4 -65134 +62606 4 14 -2176 +2091 @@ -27328,7 +27891,7 @@ 1 2 -327044 +314529 @@ -27444,15 +28007,15 @@ unspecifiedtype -9406353 +9451546 type_id -9406353 +9451546 unspecified_type_id -5062897 +5133540 @@ -27466,7 +28029,7 @@ 1 2 -9406353 +9451546 @@ -27482,22 +28045,17 @@ 1 2 -2733810 +2776289 2 3 -1943594 +1978655 3 -30 -379779 - - -30 -7873 -5713 +7936 +378596 @@ -27507,19 +28065,19 @@ member -5112192 +5066844 parent -809800 +833190 index -2966 +2851 child -5095258 +5050544 @@ -27533,47 +28091,47 @@ 1 2 -46948 +45126 2 3 -191658 +219789 3 4 -208336 +209109 4 5 -90285 +90638 5 7 -65073 +67210 7 9 -67212 +65539 9 15 -68221 +66497 15 -39 -62471 +45 +62758 -39 +45 245 -9591 +6520 @@ -27589,47 +28147,47 @@ 1 2 -46218 +44425 2 3 -191719 +219824 3 4 -202769 +203723 4 5 -93154 +93454 5 7 -65766 +67876 7 9 -67346 +65668 9 15 -68999 +67245 15 -38 -61462 +41 +62641 -38 +41 281 -12363 +8331 @@ -27645,62 +28203,62 @@ 1 2 -547 +525 2 5 -267 +257 5 9 -255 +245 9 18 -230 +222 23 100 -230 +222 122 186 -230 +222 188 293 -230 +222 296 374 -230 +222 375 542 -230 +222 572 3257 -230 +222 3427 -21556 -230 +22111 +222 -29162 -65993 -48 +30337 +70680 +46 @@ -27716,62 +28274,62 @@ 1 2 -534 +514 2 5 -218 +210 5 7 -230 +222 7 13 -230 +222 13 70 -230 +222 71 160 -230 +222 160 256 -230 +222 263 345 -230 +222 346 476 -243 +233 492 1685 -230 +222 1926 -8765 -230 +8775 +222 -10213 -66838 -121 +10226 +71530 +116 @@ -27787,7 +28345,7 @@ 1 2 -5095258 +5050544 @@ -27803,12 +28361,12 @@ 1 2 -5078555 +5034466 2 7 -16702 +16078 @@ -27818,15 +28376,15 @@ enclosingfunction -131751 +127164 child -131751 +127164 parent -74239 +71639 @@ -27840,7 +28398,7 @@ 1 2 -131751 +127164 @@ -27856,27 +28414,27 @@ 1 2 -39240 +37811 2 3 -21930 +21231 3 4 -7208 +6952 4 6 -5652 +5445 6 45 -206 +198 @@ -27886,27 +28444,27 @@ derivations -370309 +392220 derivation -370309 +392220 sub -346555 +367484 index -72 +70 super -237840 +236475 location -96351 +92613 @@ -27920,7 +28478,7 @@ 1 2 -370309 +392220 @@ -27936,7 +28494,7 @@ 1 2 -370309 +392220 @@ -27952,7 +28510,7 @@ 1 2 -370309 +392220 @@ -27968,7 +28526,7 @@ 1 2 -370309 +392220 @@ -27984,12 +28542,12 @@ 1 2 -326108 +345926 2 7 -20447 +21558 @@ -28005,12 +28563,12 @@ 1 2 -334933 +356161 2 7 -11621 +11322 @@ -28026,12 +28584,12 @@ 1 2 -326120 +345937 2 7 -20434 +21546 @@ -28047,12 +28605,12 @@ 1 2 -334921 +356150 2 7 -11633 +11334 @@ -28068,32 +28626,32 @@ 1 2 -12 +11 4 5 -12 +11 44 45 -12 +11 220 221 -12 +11 -957 -958 -12 +970 +971 +11 -29236 -29237 -12 +32328 +32329 +11 @@ -28109,32 +28667,32 @@ 1 2 -12 +11 4 5 -12 +11 44 45 -12 +11 220 221 -12 +11 -956 -957 -12 +969 +970 +11 -28508 -28509 -12 +31450 +31451 +11 @@ -28150,32 +28708,32 @@ 1 2 -12 +11 3 4 -12 +11 29 30 -12 +11 84 85 -12 +11 -441 -442 -12 +452 +453 +11 -19051 -19052 -12 +19713 +19714 +11 @@ -28191,32 +28749,32 @@ 1 2 -12 +11 4 5 -12 +11 17 18 -12 +11 51 52 -12 +11 254 255 -12 +11 7602 7603 -12 +11 @@ -28232,12 +28790,12 @@ 1 2 -222438 +221343 2 -766 -15402 +1134 +15131 @@ -28253,12 +28811,12 @@ 1 2 -222450 +221355 2 -766 -15390 +1134 +15120 @@ -28274,12 +28832,12 @@ 1 2 -237342 +235996 2 4 -498 +479 @@ -28295,12 +28853,12 @@ 1 2 -229780 +228634 2 439 -8059 +7840 @@ -28316,22 +28874,22 @@ 1 2 -74652 +70786 2 3 -9336 +8985 3 8 -7306 +7606 8 -682 -5057 +698 +5234 @@ -28347,22 +28905,22 @@ 1 2 -77193 +73146 2 3 -7002 +6858 3 -9 -7609 +8 +7443 -9 -682 -4546 +8 +698 +5164 @@ -28378,12 +28936,12 @@ 1 2 -96327 +92589 2 4 -24 +23 @@ -28399,22 +28957,22 @@ 1 2 -77655 +73882 2 3 -9056 +8833 3 -11 -7269 +10 +7326 -11 -682 -2370 +10 +698 +2570 @@ -28424,15 +28982,15 @@ derspecifiers -372947 +394756 der_id -370272 +392185 spec_id -48 +46 @@ -28446,12 +29004,12 @@ 1 2 -367598 +389615 2 3 -2674 +2570 @@ -28467,22 +29025,22 @@ 220 221 -12 +11 242 243 -12 +11 979 980 -12 +11 -29238 -29239 -12 +32343 +32344 +11 @@ -28492,15 +29050,15 @@ direct_base_offsets -282029 +307366 der_id -282029 +307366 offset -206 +210 @@ -28514,7 +29072,7 @@ 1 2 -282029 +307366 @@ -28530,62 +29088,67 @@ 1 2 -36 +46 2 3 -12 +11 4 5 -24 +23 5 6 -12 +11 6 7 -24 +11 7 8 -12 +23 8 9 -24 +11 + + +10 +11 +11 11 12 -12 +11 21 22 -12 +11 -78 -79 -12 +85 +86 +11 200 201 -12 +11 -22837 -22838 -12 +25931 +25932 +11 @@ -28595,19 +29158,19 @@ virtual_base_offsets -7026 +6753 sub -3890 +3739 super -534 +514 offset -267 +257 @@ -28621,22 +29184,22 @@ 1 2 -3063 +2944 2 4 -340 +327 4 7 -279 +268 7 11 -206 +198 @@ -28652,17 +29215,17 @@ 1 2 -3282 +3154 2 4 -328 +315 4 8 -279 +268 @@ -28678,52 +29241,52 @@ 1 2 -85 +81 2 3 -48 +46 3 4 -60 +58 4 5 -97 +93 5 7 -36 +35 8 13 -48 +46 13 15 -48 +46 15 23 -48 +46 24 60 -48 +46 196 197 -12 +11 @@ -28739,32 +29302,32 @@ 1 2 -303 +292 2 3 -85 +81 4 6 -36 +35 6 8 -48 +46 8 10 -48 +46 14 15 -12 +11 @@ -28780,57 +29343,57 @@ 2 3 -36 +35 4 5 -12 +11 5 6 -24 +23 6 8 -24 +23 8 9 -36 +35 10 12 -24 +23 14 19 -24 +23 20 27 -24 +23 28 31 -24 +23 36 97 -24 +23 97 98 -12 +11 @@ -28846,37 +29409,37 @@ 1 2 -85 +81 2 3 -36 +35 3 4 -48 +46 5 7 -24 +23 7 10 -24 +23 12 14 -24 +23 21 29 -24 +23 @@ -28886,23 +29449,23 @@ frienddecls -231178 +222208 id -231178 +222208 type_id -19511 +18753 decl_id -28810 +27692 location -8096 +7782 @@ -28916,7 +29479,7 @@ 1 2 -231178 +222208 @@ -28932,7 +29495,7 @@ 1 2 -231178 +222208 @@ -28948,7 +29511,7 @@ 1 2 -231178 +222208 @@ -28964,47 +29527,47 @@ 1 2 -6856 +6590 2 3 -2467 +2371 3 4 -1191 +1145 4 6 -1714 +1647 6 10 -1556 +1495 10 16 -1470 +1413 16 34 -1786 +1717 36 59 -1470 +1413 59 129 -996 +958 @@ -29020,47 +29583,47 @@ 1 2 -6856 +6590 2 3 -2467 +2371 3 4 -1191 +1145 4 6 -1714 +1647 6 10 -1556 +1495 10 16 -1470 +1413 16 34 -1786 +1717 36 59 -1470 +1413 59 129 -996 +958 @@ -29076,17 +29639,17 @@ 1 2 -17748 +17059 2 5 -1543 +1483 5 31 -218 +210 @@ -29102,37 +29665,37 @@ 1 2 -15815 +15201 2 3 -2261 +2173 3 5 -2479 +2383 5 9 -2188 +2103 9 20 -2030 +1951 20 28 -2467 +2371 28 390 -1568 +1507 @@ -29148,37 +29711,37 @@ 1 2 -15815 +15201 2 3 -2261 +2173 3 5 -2479 +2383 5 9 -2188 +2103 9 20 -2030 +1951 20 28 -2467 +2371 28 390 -1568 +1507 @@ -29194,12 +29757,12 @@ 1 2 -28215 +27120 2 46 -595 +572 @@ -29215,17 +29778,17 @@ 1 2 -6917 +6648 2 3 -1045 +1004 3 18239 -133 +128 @@ -29241,12 +29804,12 @@ 1 2 -7609 +7314 2 1215 -486 +467 @@ -29262,17 +29825,17 @@ 1 2 -6929 +6660 2 3 -1033 +993 3 1734 -133 +128 @@ -29282,19 +29845,19 @@ comments -1743633 +1675974 id -1743633 +1675974 contents -864492 +830947 location -1743633 +1675974 @@ -29308,7 +29871,7 @@ 1 2 -1743633 +1675974 @@ -29324,7 +29887,7 @@ 1 2 -1743633 +1675974 @@ -29340,17 +29903,17 @@ 1 2 -731525 +703139 2 3 -83149 +79923 3 10736 -49817 +47883 @@ -29366,17 +29929,17 @@ 1 2 -731525 +703139 2 3 -83149 +79923 3 10736 -49817 +47883 @@ -29392,7 +29955,7 @@ 1 2 -1743633 +1675974 @@ -29408,7 +29971,7 @@ 1 2 -1743633 +1675974 @@ -29418,15 +29981,15 @@ commentbinding -778436 +748230 id -679021 +652673 element -747438 +718434 @@ -29440,17 +30003,17 @@ 1 2 -616002 +592099 2 4 -58545 +56273 4 97 -4473 +4299 @@ -29466,12 +30029,12 @@ 1 2 -716439 +688638 2 3 -30998 +29796 @@ -29481,15 +30044,15 @@ exprconv -6445331 +6446816 converted -6445038 +6446523 conversion -6445331 +6446816 @@ -29503,7 +30066,7 @@ 1 2 -6444746 +6446231 2 @@ -29524,7 +30087,7 @@ 1 2 -6445331 +6446816 @@ -29534,30 +30097,30 @@ compgenerated -6720341 +6652484 id -6720341 +6652484 synthetic_destructor_call -59493 +57185 element -47422 +45582 i -340 +327 destructor_call -49573 +47650 @@ -29571,17 +30134,17 @@ 1 2 -38791 +37285 2 3 -6321 +6076 3 29 -2309 +2220 @@ -29597,17 +30160,17 @@ 1 2 -38791 +37285 2 3 -6321 +6076 3 29 -2309 +2220 @@ -29623,27 +30186,27 @@ 1 2 -255 +245 2 7 -24 +23 22 43 -24 +23 190 711 -24 +23 3901 3902 -12 +11 @@ -29659,27 +30222,27 @@ 1 2 -255 +245 2 7 -24 +23 21 37 -24 +23 146 550 -24 +23 3297 3298 -12 +11 @@ -29695,17 +30258,17 @@ 1 2 -43823 +42123 2 3 -3719 +3575 3 26 -2030 +1951 @@ -29721,7 +30284,7 @@ 1 2 -49573 +47650 @@ -29731,15 +30294,15 @@ namespaces -8497 +8167 id -8497 +8167 name -4570 +4393 @@ -29753,7 +30316,7 @@ 1 2 -8497 +8167 @@ -29769,17 +30332,17 @@ 1 2 -3841 +3692 2 3 -461 +444 3 139 -267 +257 @@ -29789,26 +30352,26 @@ namespace_inline -170 +163 id -170 +163 namespacembrs -1570027 +1616300 parentid -7913 +7606 memberid -1570027 +1616300 @@ -29822,62 +30385,67 @@ 1 2 -875 +829 2 3 -826 +794 3 4 -644 +385 4 -6 -680 - - -6 -9 +5 607 -9 -16 -680 +5 +8 +666 -16 -27 -607 +8 +14 +677 -27 -46 -619 - - -46 -85 -607 - - -85 -169 -607 - - -169 -444 +14 +22 595 -475 -29073 -559 +22 +42 +595 + + +42 +63 +572 + + +63 +127 +572 + + +127 +304 +572 + + +321 +1041 +572 + + +1101 +32701 +163 @@ -29893,7 +30461,7 @@ 1 2 -1570027 +1616300 @@ -29903,19 +30471,19 @@ exprparents -13615303 +13481241 expr_id -13615112 +13481052 child_index -3163 +3131 parent_id -9644209 +9549204 @@ -29929,7 +30497,7 @@ 1 2 -13615105 +13481045 2 @@ -29950,12 +30518,12 @@ 1 2 -13614927 +13480869 2 4 -184 +182 @@ -29971,37 +30539,37 @@ 1 2 -72 +71 2 3 -652 +645 3 4 -1674 +1657 4 46 -243 +241 46 56 -237 +234 56 3660 -237 +234 6434 -1187727 -46 +1188148 +45 @@ -30017,37 +30585,37 @@ 1 2 -72 +71 2 3 -652 +645 3 4 -1674 +1657 4 31 -243 +241 31 41 -237 +234 41 3645 -237 +234 6419 -1187740 -46 +1188161 +45 @@ -30063,17 +30631,17 @@ 1 2 -6840662 +6772879 2 3 -2212086 +2190858 3 1681 -591461 +585466 @@ -30089,17 +30657,17 @@ 1 2 -6840682 +6772898 2 3 -2212066 +2190839 3 480 -591461 +585466 @@ -30109,22 +30677,22 @@ expr_isload -5063114 +5014574 expr_id -5063114 +5014574 conversionkinds -4254113 +4255501 expr_id -4254113 +4255501 kind @@ -30142,7 +30710,7 @@ 1 2 -4254113 +4255501 @@ -30166,8 +30734,8 @@ 1 -13109 -13110 +13672 +13673 1 @@ -30181,8 +30749,8 @@ 1 -4161592 -4161593 +4162417 +4162418 1 @@ -30193,15 +30761,15 @@ iscall -2365240 +2284876 caller -2365240 +2284876 kind -36 +35 @@ -30215,7 +30783,7 @@ 1 2 -2365240 +2284876 @@ -30231,17 +30799,17 @@ 1319 1320 -12 +11 -5604 -5605 -12 +5643 +5644 +11 -187644 -187645 -12 +188582 +188583 +11 @@ -30251,15 +30819,15 @@ numtemplatearguments -157948 +152181 expr_id -157948 +152181 num -48 +46 @@ -30273,7 +30841,7 @@ 1 2 -157948 +152181 @@ -30289,22 +30857,22 @@ 3 4 -12 +11 41 42 -12 +11 651 652 -12 +11 -12298 -12299 -12 +12329 +12330 +11 @@ -30314,15 +30882,15 @@ specialnamequalifyingelements -12 +11 id -12 +11 name -12 +11 @@ -30336,7 +30904,7 @@ 1 2 -12 +11 @@ -30352,7 +30920,7 @@ 1 2 -12 +11 @@ -30362,23 +30930,23 @@ namequalifiers -1125621 +1116919 id -1125621 +1116919 qualifiableelement -1125621 +1116919 qualifyingelement -37001 +38961 location -509589 +506087 @@ -30392,7 +30960,7 @@ 1 2 -1125621 +1116919 @@ -30408,7 +30976,7 @@ 1 2 -1125621 +1116919 @@ -30424,7 +30992,7 @@ 1 2 -1125621 +1116919 @@ -30440,7 +31008,7 @@ 1 2 -1125621 +1116919 @@ -30456,7 +31024,7 @@ 1 2 -1125621 +1116919 @@ -30472,7 +31040,7 @@ 1 2 -1125621 +1116919 @@ -30488,109 +31056,109 @@ 1 2 -17109 +18965 2 3 -7935 +8142 3 4 -5081 - - -4 -10 -2860 - - -10 -86 -2781 - - -86 -24926 -1232 - - - - - - -qualifyingelement -qualifiableelement - - -12 - - -1 -2 -17109 - - -2 -3 -7935 - - -3 -4 -5081 - - -4 -10 -2860 - - -10 -86 -2781 - - -86 -24926 -1232 - - - - - - -qualifyingelement -location - - -12 - - -1 -2 -22745 - - -2 -3 -4817 - - -3 -4 -3802 +5043 4 11 -3018 +3027 + + +11 +129 +2935 + + +132 +24926 +848 + + + + + + +qualifyingelement +qualifiableelement + + +12 + + +1 +2 +18965 + + +2 +3 +8142 + + +3 +4 +5043 + + +4 +11 +3027 + + +11 +129 +2935 + + +132 +24926 +848 + + + + + + +qualifyingelement +location + + +12 + + +1 +2 +24850 + + +2 +3 +4762 + + +3 +4 +3770 + + +4 +11 +2974 11 16728 -2616 +2603 @@ -30606,22 +31174,22 @@ 1 2 -387650 +384719 2 3 -57004 +56877 3 7 -39037 +38851 7 381 -25895 +25639 @@ -30637,22 +31205,22 @@ 1 2 -387650 +384719 2 3 -57004 +56877 3 7 -39037 +38851 7 381 -25895 +25639 @@ -30668,17 +31236,17 @@ 1 2 -452478 +449080 2 3 -44785 +44650 3 190 -12324 +12356 @@ -30688,15 +31256,15 @@ varbind -5497554 +5445518 expr -5497428 +5445394 var -1549133 +1534266 @@ -30710,12 +31278,12 @@ 1 2 -5497303 +5445270 2 3 -125 +123 @@ -30731,32 +31299,32 @@ 1 2 -685882 +679661 2 3 -311295 +308179 3 4 -236831 +234430 4 5 -93953 +93020 5 9 -134777 +133430 9 6150 -86393 +85544 @@ -30766,15 +31334,15 @@ funbind -2442767 +2419854 expr -2142538 +2122577 fun -438611 +434205 @@ -30788,17 +31356,17 @@ 1 2 -1842553 +1825541 2 3 -299807 +296859 3 5 -177 +176 @@ -30814,32 +31382,32 @@ 1 2 -255153 +252065 2 3 -76151 +75516 3 4 -31642 +31302 4 7 -33910 +33957 7 37 -33831 +33520 37 6664 -7922 +7841 @@ -30849,19 +31417,19 @@ expr_allocator -27692 +26617 expr -27692 +26617 func -133 +128 form -12 +11 @@ -30875,7 +31443,7 @@ 1 2 -27692 +26617 @@ -30891,7 +31459,7 @@ 1 2 -27692 +26617 @@ -30907,42 +31475,42 @@ 1 2 -36 +35 3 4 -12 +11 4 5 -12 +11 5 6 -12 +11 7 8 -24 +23 39 40 -12 +11 973 974 -12 +11 1237 1238 -12 +11 @@ -30958,7 +31526,7 @@ 1 2 -133 +128 @@ -30974,7 +31542,7 @@ 2278 2279 -12 +11 @@ -30990,7 +31558,7 @@ 11 12 -12 +11 @@ -31000,19 +31568,19 @@ expr_deallocator -30998 +29796 expr -30998 +29796 func -145 +140 form -24 +23 @@ -31026,7 +31594,7 @@ 1 2 -30998 +29796 @@ -31042,7 +31610,7 @@ 1 2 -30998 +29796 @@ -31058,42 +31626,42 @@ 1 2 -36 +35 2 3 -24 +23 3 4 -12 +11 4 5 -12 +11 7 8 -24 +23 118 119 -12 +11 883 884 -12 +11 1521 1522 -12 +11 @@ -31109,7 +31677,7 @@ 1 2 -145 +140 @@ -31125,12 +31693,12 @@ 891 892 -12 +11 1659 1660 -12 +11 @@ -31146,12 +31714,12 @@ 4 5 -12 +11 8 9 -12 +11 @@ -31161,26 +31729,26 @@ expr_cond_two_operand -610 +611 cond -610 +611 expr_cond_guard -154429 +154519 cond -154429 +154519 guard -154429 +154519 @@ -31194,7 +31762,7 @@ 1 2 -154429 +154519 @@ -31210,7 +31778,7 @@ 1 2 -154429 +154519 @@ -31220,15 +31788,15 @@ expr_cond_true -154429 +154519 cond -154429 +154519 true -154429 +154519 @@ -31242,7 +31810,7 @@ 1 2 -154429 +154519 @@ -31258,7 +31826,7 @@ 1 2 -154429 +154519 @@ -31268,15 +31836,15 @@ expr_cond_false -154429 +154519 cond -154429 +154519 false -154429 +154519 @@ -31290,7 +31858,7 @@ 1 2 -154429 +154519 @@ -31306,7 +31874,7 @@ 1 2 -154429 +154519 @@ -31316,15 +31884,15 @@ values -8776796 +8788031 id -8776796 +8788031 str -651390 +651491 @@ -31338,7 +31906,7 @@ 1 2 -8776796 +8788031 @@ -31354,17 +31922,17 @@ 1 2 -540103 +540120 2 3 -65463 +65472 3 -4044535 -45824 +4046841 +45899 @@ -31374,15 +31942,15 @@ valuetext -4759381 +4766569 id -4759381 +4766569 text -704971 +705085 @@ -31396,7 +31964,7 @@ 1 2 -4759381 +4766569 @@ -31412,22 +31980,22 @@ 1 2 -528095 +528108 2 3 -102815 +102830 3 7 -56849 +56879 7 427516 -17212 +17268 @@ -31437,15 +32005,15 @@ valuebind -9499071 +9509366 val -8769631 +8779926 expr -9499071 +9509366 @@ -31459,7 +32027,7 @@ 1 2 -8040923 +8051218 2 @@ -31485,7 +32053,7 @@ 1 2 -9499071 +9509366 @@ -31495,19 +32063,19 @@ fieldoffsets -259539 +251575 id -259539 +251575 byteoffset -3914 +9022 bitoffset -85 +52 @@ -31521,7 +32089,7 @@ 1 2 -259539 +251575 @@ -31537,7 +32105,7 @@ 1 2 -259539 +251575 @@ -31553,42 +32121,27 @@ 1 2 -1689 +6132 2 3 -534 +782 3 -4 -194 - - -4 6 -340 +750 6 -11 -328 +17 +678 -11 -23 -328 - - -23 -84 -303 - - -90 -12369 -194 +17 +11685 +678 @@ -31604,12 +32157,12 @@ 1 2 -3853 +8579 2 -8 -60 +9 +443 @@ -31623,29 +32176,44 @@ 12 -1 -2 -36 +83 +84 +6 -2 -3 -12 +87 +88 +6 -3 -4 -12 +102 +103 +6 -4 -5 -12 +122 +123 +6 -21338 -21339 -12 +127 +128 +6 + + +153 +154 +6 + + +195 +196 +6 + + +37692 +37693 +6 @@ -31659,24 +32227,44 @@ 12 -1 -2 -36 +40 +41 +6 -2 -3 -24 +44 +45 +6 -3 -4 -12 +45 +46 +6 -322 -323 -12 +52 +53 +6 + + +54 +55 +6 + + +58 +59 +6 + + +66 +67 +6 + + +1383 +1384 +6 @@ -31686,11 +32274,11 @@ bitfield -13357 +13365 id -13357 +13365 bits @@ -31712,7 +32300,7 @@ 1 2 -13357 +13365 @@ -31728,7 +32316,7 @@ 1 2 -13357 +13365 @@ -31892,23 +32480,23 @@ initialisers -1685017 +1668506 init -1685017 +1668506 var -649685 +643667 expr -1685017 +1668506 location -321906 +318643 @@ -31922,7 +32510,7 @@ 1 2 -1685017 +1668506 @@ -31938,7 +32526,7 @@ 1 2 -1685017 +1668506 @@ -31954,7 +32542,7 @@ 1 2 -1685017 +1668506 @@ -31970,22 +32558,22 @@ 1 2 -562000 +556871 2 16 -29079 +28784 16 17 -49688 +49185 17 53 -8917 +8827 @@ -32001,22 +32589,22 @@ 1 2 -562000 +556871 2 16 -29079 +28784 16 17 -49688 +49185 17 53 -8917 +8827 @@ -32032,7 +32620,7 @@ 1 2 -649672 +643654 2 @@ -32053,7 +32641,7 @@ 1 2 -1685017 +1668506 @@ -32069,7 +32657,7 @@ 1 2 -1685017 +1668506 @@ -32085,7 +32673,7 @@ 1 2 -1685017 +1668506 @@ -32101,27 +32689,27 @@ 1 2 -248450 +245906 2 3 -24096 +23845 3 7 -24485 +24243 7 65 -24346 +24126 67 109238 -527 +521 @@ -32137,22 +32725,22 @@ 1 2 -271334 +268551 2 3 -25342 +25091 3 -24 -24155 +23 +23904 -24 +23 12632 -1074 +1096 @@ -32168,27 +32756,27 @@ 1 2 -248450 +245906 2 3 -24096 +23845 3 7 -24485 +24243 7 65 -24346 +24126 67 109238 -527 +521 @@ -32198,15 +32786,15 @@ expr_ancestor -67772 +68706 exp -67176 +68133 ancestor -49695 +51330 @@ -32220,12 +32808,12 @@ 1 2 -66605 +67584 2 4 -571 +549 @@ -32241,17 +32829,17 @@ 1 2 -37405 +39517 2 3 -10150 +9756 3 29 -2139 +2056 @@ -32261,19 +32849,19 @@ exprs -18449753 +18460593 id -18449753 +18460593 kind -393 +1168 location -6150603 +3836046 @@ -32287,7 +32875,7 @@ 1 2 -18449753 +18460593 @@ -32303,7 +32891,7 @@ 1 2 -18449753 +18460593 @@ -32317,69 +32905,69 @@ 12 -5 -28 -30 +1 +14 +105 -71 -82 -30 +15 +44 +93 -94 -255 -30 +47 +86 +93 -271 -627 -30 +90 +223 +93 -858 -1879 -30 +296 +463 +93 -2191 -3716 -30 +483 +715 +93 -4305 -6068 -30 +788 +2129 +93 -7004 -11661 -30 +2165 +2950 +93 -12107 -20202 -30 +3030 +4378 +93 -21965 -29561 -30 +4477 +6077 +93 -32830 -41032 -30 +6490 +17249 +93 -44670 -145995 -30 +19347 +188353 +93 -447805 -725766 -24 +191022 +403956 +35 @@ -32394,68 +32982,68 @@ 1 +4 +105 + + +6 +13 +105 + + +13 24 -30 +93 -27 -72 -30 +24 +38 +93 -77 -157 -30 +38 +134 +93 -171 -402 -30 +144 +259 +93 -422 -1083 -30 +269 +480 +93 -1179 -1862 -30 +481 +1074 +93 -2201 -4268 -30 +1095 +1411 +93 -4679 -6584 -30 +1422 +2045 +93 -6624 -11083 -30 +2059 +4557 +93 -11359 -12983 -30 +5888 +59687 +93 -17158 -28621 -30 - - -29980 -88945 -30 - - -128315 -425042 -24 +72153 +117893 +23 @@ -32471,22 +33059,37 @@ 1 2 -4450587 +1838158 2 3 -874643 +753582 3 -6 -496205 +4 +356208 -6 -22445 -329167 +4 +5 +276250 + + +5 +9 +301489 + + +9 +72 +287759 + + +72 +136887 +22598 @@ -32502,22 +33105,17 @@ 1 2 -4599301 +2734142 2 3 -812812 +859259 3 -5 -498909 - - -5 -33 -239579 +30 +242644 @@ -32527,19 +33125,19 @@ expr_types -18617706 +18629051 id -18444987 +18456012 typeid -1372691 +1345343 value_category -36 +35 @@ -32553,12 +33151,12 @@ 1 2 -18272500 +18283254 2 4 -172487 +172758 @@ -32574,7 +33172,7 @@ 1 2 -18444987 +18456012 @@ -32590,42 +33188,42 @@ 1 2 -530591 +526443 2 3 -259102 +254107 3 4 -113553 +110864 4 5 -95427 +92110 5 8 -122925 +118483 8 14 -105736 +102381 14 -46 -103864 +47 +101797 -46 -111456 -41489 +47 +123210 +39155 @@ -32641,17 +33239,17 @@ 1 2 -1207109 +1185928 2 3 -157753 +151877 3 4 -7828 +7536 @@ -32667,17 +33265,17 @@ 4533 4534 -12 +11 -340895 -340896 -12 +342547 +342548 +11 -1171875 -1171876 -12 +1232420 +1232421 +11 @@ -32691,19 +33289,19 @@ 12 -1340 -1341 -12 +1341 +1342 +11 -29733 -29734 -12 +29841 +29842 +11 -96111 -96112 -12 +98243 +98244 +11 @@ -32713,15 +33311,15 @@ new_allocated_type -29527 +28382 expr -29527 +28382 type_id -18234 +17527 @@ -32735,7 +33333,7 @@ 1 2 -29527 +28382 @@ -32751,22 +33349,22 @@ 1 2 -12667 +12175 2 3 -4035 +3879 3 9 -1446 +1390 10 92 -85 +81 @@ -32776,15 +33374,15 @@ new_array_allocated_type -5364 +5310 expr -5364 +5310 type_id -2306 +2283 @@ -32798,7 +33396,7 @@ 1 2 -5364 +5310 @@ -32819,12 +33417,12 @@ 2 3 -2043 +2022 3 7 -177 +176 8 @@ -33376,15 +33974,15 @@ condition_decl_bind -7646 +7349 expr -7646 +7349 decl -7646 +7349 @@ -33398,7 +33996,7 @@ 1 2 -7646 +7349 @@ -33414,7 +34012,7 @@ 1 2 -7646 +7349 @@ -33424,15 +34022,15 @@ typeid_bind -4899 +4708 expr -4899 +4708 type_id -2601 +2500 @@ -33446,7 +34044,7 @@ 1 2 -4899 +4708 @@ -33462,22 +34060,22 @@ 1 2 -1300 +1250 2 3 -996 +958 3 6 -230 +222 6 17 -72 +70 @@ -33487,15 +34085,15 @@ uuidof_bind -856 +848 expr -856 +848 type_id -652 +645 @@ -33509,7 +34107,7 @@ 1 2 -856 +848 @@ -33525,12 +34123,12 @@ 1 2 -481 +476 2 3 -138 +137 3 @@ -33545,15 +34143,15 @@ sizeof_bind -156997 +157089 expr -156997 +157089 type_id -2673 +2674 @@ -33567,7 +34165,7 @@ 1 2 -156997 +157089 @@ -33583,7 +34181,7 @@ 1 2 -1044 +1045 2 @@ -33681,11 +34279,11 @@ lambdas -12641 +12513 expr -12641 +12513 default_capture @@ -33707,7 +34305,7 @@ 1 2 -12641 +12513 @@ -33723,7 +34321,7 @@ 1 2 -12641 +12513 @@ -33812,23 +34410,23 @@ lambda_capture -21730 +21509 id -21730 +21509 lambda -10097 +9994 index -112 +110 field -21730 +21509 captured_by_reference @@ -33840,7 +34438,7 @@ location -14051 +13909 @@ -33854,7 +34452,7 @@ 1 2 -21730 +21509 @@ -33870,7 +34468,7 @@ 1 2 -21730 +21509 @@ -33886,7 +34484,7 @@ 1 2 -21730 +21509 @@ -33902,7 +34500,7 @@ 1 2 -21730 +21509 @@ -33918,7 +34516,7 @@ 1 2 -21730 +21509 @@ -33934,7 +34532,7 @@ 1 2 -21730 +21509 @@ -33950,27 +34548,27 @@ 1 2 -5002 +4951 2 3 -2379 +2355 3 4 -1265 +1252 4 6 -922 +913 6 18 -527 +521 @@ -33986,27 +34584,27 @@ 1 2 -5002 +4951 2 3 -2379 +2355 3 4 -1265 +1252 4 6 -922 +913 6 18 -527 +521 @@ -34022,27 +34620,27 @@ 1 2 -5002 +4951 2 3 -2379 +2355 3 4 -1265 +1252 4 6 -922 +913 6 18 -527 +521 @@ -34058,12 +34656,12 @@ 1 2 -9629 +9531 2 3 -467 +463 @@ -34079,7 +34677,7 @@ 1 2 -10077 +9975 2 @@ -34100,27 +34698,27 @@ 1 2 -5483 +5428 2 3 -2504 +2479 3 4 -1041 +1030 4 7 -823 +815 7 18 -243 +241 @@ -34429,7 +35027,7 @@ 2 3 -85 +84 @@ -34450,7 +35048,7 @@ 2 3 -46 +45 @@ -34562,7 +35160,7 @@ 1 2 -21730 +21509 @@ -34578,7 +35176,7 @@ 1 2 -21730 +21509 @@ -34594,7 +35192,7 @@ 1 2 -21730 +21509 @@ -34610,7 +35208,7 @@ 1 2 -21730 +21509 @@ -34626,7 +35224,7 @@ 1 2 -21730 +21509 @@ -34642,7 +35240,7 @@ 1 2 -21730 +21509 @@ -34900,17 +35498,17 @@ 1 2 -12667 +12539 2 6 -1061 +1050 6 68 -322 +319 @@ -34926,12 +35524,12 @@ 1 2 -13115 +12982 2 68 -935 +926 @@ -34947,12 +35545,12 @@ 1 2 -13511 +13374 2 8 -540 +534 @@ -34968,17 +35566,17 @@ 1 2 -12667 +12539 2 6 -1061 +1050 6 68 -322 +319 @@ -34994,7 +35592,7 @@ 1 2 -14032 +13889 2 @@ -35015,7 +35613,7 @@ 1 2 -14051 +13909 @@ -35141,19 +35739,19 @@ stmts -4943740 +4765273 id -4943740 +4765273 kind -230 +222 location -1317574 +1266448 @@ -35167,7 +35765,7 @@ 1 2 -4943740 +4765273 @@ -35183,7 +35781,7 @@ 1 2 -4943740 +4765273 @@ -35199,97 +35797,97 @@ 2 3 -12 +11 28 29 -12 +11 338 339 -12 +11 473 474 -12 +11 690 691 -12 +11 1560 1561 -12 +11 1757 1758 -12 +11 -2076 -2077 -12 +2080 +2081 +11 -2191 -2192 -12 +2193 +2194 +11 2707 2708 -12 +11 2748 2749 -12 +11 -2944 -2945 -12 +2946 +2947 +11 3296 3297 -12 +11 4540 4541 -12 +11 -28688 -28689 -12 +28694 +28695 +11 -53254 -53255 -12 +53369 +53370 +11 -85878 -85879 -12 +86211 +86212 +11 -98483 -98484 -12 +98824 +98825 +11 -115024 -115025 -12 +115365 +115366 +11 @@ -35305,97 +35903,97 @@ 2 3 -12 +11 23 24 -12 +11 106 107 -12 +11 112 113 -12 +11 178 179 -12 +11 252 253 -12 +11 296 297 -12 +11 661 662 -12 +11 663 664 -12 +11 991 992 -12 +11 1030 1031 -12 +11 1408 1409 -12 +11 1918 1919 -12 +11 2676 2677 -12 +11 10172 10173 -12 +11 10231 10232 -12 +11 22268 22269 -12 +11 26534 26535 -12 +11 31957 31958 -12 +11 @@ -35411,32 +36009,32 @@ 1 2 -779628 +747015 2 3 -179173 +173050 3 4 -120275 +116169 4 6 -106891 +103199 6 19 -102794 +99168 19 -4895 -28810 +4923 +27844 @@ -35452,12 +36050,12 @@ 1 2 -1291693 +1241571 2 9 -25881 +24876 @@ -35563,15 +36161,15 @@ if_then -524558 +524866 if_stmt -524558 +524866 then_id -524558 +524866 @@ -35585,7 +36183,7 @@ 1 2 -524558 +524866 @@ -35601,7 +36199,7 @@ 1 2 -524558 +524866 @@ -35611,15 +36209,15 @@ if_else -148234 +148322 if_stmt -148234 +148322 else_id -148234 +148322 @@ -35633,7 +36231,7 @@ 1 2 -148234 +148322 @@ -35649,7 +36247,7 @@ 1 2 -148234 +148322 @@ -35755,15 +36353,15 @@ while_body -32907 +31630 while_stmt -32907 +31630 body_id -32907 +31630 @@ -35777,7 +36375,7 @@ 1 2 -32907 +31630 @@ -35793,7 +36391,7 @@ 1 2 -32907 +31630 @@ -35803,15 +36401,15 @@ do_body -149900 +149988 do_stmt -149900 +149988 body_id -149900 +149988 @@ -35825,7 +36423,7 @@ 1 2 -149900 +149988 @@ -35841,7 +36439,7 @@ 1 2 -149900 +149988 @@ -35851,11 +36449,11 @@ switch_case -281530 +281695 switch_stmt -55225 +55258 index @@ -35863,7 +36461,7 @@ case_id -281530 +281695 @@ -35877,17 +36475,17 @@ 1 5 -4288 +4290 5 6 -48725 +48753 6 156 -2212 +2213 @@ -35903,17 +36501,17 @@ 1 5 -4288 +4290 5 6 -48725 +48753 6 156 -2212 +2213 @@ -36031,7 +36629,7 @@ 1 2 -281530 +281695 @@ -36047,7 +36645,7 @@ 1 2 -281530 +281695 @@ -36057,15 +36655,15 @@ switch_body -55225 +55258 switch_stmt -55225 +55258 body_id -55225 +55258 @@ -36079,7 +36677,7 @@ 1 2 -55225 +55258 @@ -36095,7 +36693,7 @@ 1 2 -55225 +55258 @@ -36105,15 +36703,15 @@ for_initialization -29922 +29619 for_stmt -29922 +29619 init_id -29922 +29619 @@ -36127,7 +36725,7 @@ 1 2 -29922 +29619 @@ -36143,7 +36741,7 @@ 1 2 -29922 +29619 @@ -36153,15 +36751,15 @@ for_condition -31781 +31459 for_stmt -31781 +31459 condition_id -31781 +31459 @@ -36175,7 +36773,7 @@ 1 2 -31781 +31459 @@ -36191,7 +36789,7 @@ 1 2 -31781 +31459 @@ -36201,15 +36799,15 @@ for_update -29659 +29358 for_stmt -29659 +29358 update_id -29659 +29358 @@ -36223,7 +36821,7 @@ 1 2 -29659 +29358 @@ -36239,7 +36837,7 @@ 1 2 -29659 +29358 @@ -36249,15 +36847,15 @@ for_body -32387 +32059 for_stmt -32387 +32059 body_id -32387 +32059 @@ -36271,7 +36869,7 @@ 1 2 -32387 +32059 @@ -36287,7 +36885,7 @@ 1 2 -32387 +32059 @@ -36297,19 +36895,19 @@ stmtparents -4168478 +4131960 id -4168478 +4131960 index -12799 +836 parent -1762006 +1583254 @@ -36323,7 +36921,7 @@ 1 2 -4168478 +4131960 @@ -36339,7 +36937,7 @@ 1 2 -4168478 +4131960 @@ -36355,52 +36953,57 @@ 1 2 -4020 +182 2 3 -1094 +149 3 -4 -520 +5 +41 -4 -5 -1496 +5 +7 +72 7 -8 -1047 +9 +68 -8 -12 -830 +9 +18 +63 -12 -29 -1127 +18 +39 +64 -29 -37 -909 +39 +111 +63 -37 -74 -975 +112 +836 +63 -74 -191996 -777 +920 +136298 +63 + + +254134 +706946 +5 @@ -36416,52 +37019,57 @@ 1 2 -4020 +182 2 3 -1094 +149 3 -4 -520 +5 +41 -4 -5 -1496 +5 +7 +72 7 -8 -1047 +9 +68 -8 -12 -830 +9 +18 +63 -12 -29 -1127 +18 +39 +64 -29 -37 -909 +39 +111 +63 -37 -74 -975 +112 +836 +63 -74 -191996 -777 +920 +136298 +63 + + +254134 +706946 +5 @@ -36477,32 +37085,27 @@ 1 2 -1004414 +841847 2 3 -386602 +376317 3 4 -109343 +145318 4 -6 -115439 +9 +119501 -6 -17 -133386 - - -17 -1943 -12819 +9 +465 +100269 @@ -36518,32 +37121,27 @@ 1 2 -1004414 +841847 2 3 -386602 +376317 3 4 -109343 +145318 4 -6 -115439 +9 +119501 -6 -17 -133386 - - -17 -1943 -12819 +9 +465 +100269 @@ -36553,26 +37151,26 @@ ishandler -21888 +21666 block -21888 +21666 successors -17214104 +17224218 from -16042874 +16052300 to -16040997 +16050421 @@ -36586,12 +37184,12 @@ 1 2 -15044821 +15053661 2 156 -998052 +998638 @@ -36607,12 +37205,12 @@ 1 2 -15328885 +15337891 2 419 -712111 +712530 @@ -36622,15 +37220,15 @@ truecond -966805 +967373 from -966805 +967373 to -937112 +937663 @@ -36644,7 +37242,7 @@ 1 2 -966805 +967373 @@ -36660,12 +37258,12 @@ 1 2 -913540 +914077 2 21 -23572 +23585 @@ -36675,15 +37273,15 @@ falsecond -966805 +967373 from -966805 +967373 to -812105 +812583 @@ -36697,7 +37295,7 @@ 1 2 -966805 +967373 @@ -36713,17 +37311,17 @@ 1 2 -698749 +699160 2 3 -87006 +87057 3 25 -26349 +26365 @@ -36733,11 +37331,11 @@ stmt_decl_bind -538233 +532777 stmt -531279 +525894 num @@ -36745,7 +37343,7 @@ decl -538233 +532777 @@ -36759,12 +37357,12 @@ 1 2 -525776 +520447 2 9 -5503 +5447 @@ -36780,12 +37378,12 @@ 1 2 -525756 +520427 2 9 -5523 +5467 @@ -36903,7 +37501,7 @@ 1 2 -538233 +532777 @@ -36919,7 +37517,7 @@ 1 2 -538233 +532777 @@ -36929,19 +37527,19 @@ stmt_decl_entry_bind -519726 +520031 stmt -474524 +474803 num -491 +492 decl_entry -495657 +495948 @@ -36955,12 +37553,12 @@ 1 2 -441842 +442101 2 274 -32682 +32702 @@ -36976,12 +37574,12 @@ 1 2 -441842 +442101 2 15 -32682 +32702 @@ -37064,12 +37662,12 @@ 1 2 -483634 +483918 2 85 -12022 +12029 @@ -37085,7 +37683,7 @@ 1 2 -495588 +495879 2 @@ -37100,15 +37698,15 @@ blockscope -1398256 +1347984 block -1398256 +1347984 enclosing -1254288 +1209602 @@ -37122,7 +37720,7 @@ 1 2 -1398256 +1347984 @@ -37138,12 +37736,12 @@ 1 2 -1171636 +1130157 2 509 -82651 +79444 @@ -37153,19 +37751,19 @@ jumpinfo -366503 +366719 id -366503 +366719 str -6361 +6365 target -85508 +85558 @@ -37179,7 +37777,7 @@ 1 2 -366503 +366719 @@ -37195,7 +37793,7 @@ 1 2 -366503 +366719 @@ -37216,12 +37814,12 @@ 2 3 -3507 +3509 3 4 -835 +836 4 @@ -37231,12 +37829,12 @@ 5 7 -491 +492 7 15 -500 +501 15 @@ -37257,12 +37855,12 @@ 1 2 -5095 +5098 2 3 -700 +701 3 @@ -37293,32 +37891,32 @@ 2 3 -21251 +21263 3 4 -7559 +7564 4 5 -3816 +3818 5 6 -39695 +39718 6 7 -11019 +11025 7 162 -1880 +1882 @@ -37334,7 +37932,7 @@ 1 2 -85508 +85558 @@ -37344,19 +37942,19 @@ preprocdirects -1456121 +1399618 id -1456121 +1399618 kind -158 +151 location -1449131 +1392900 @@ -37370,7 +37968,7 @@ 1 2 -1456121 +1399618 @@ -37386,7 +37984,7 @@ 1 2 -1456121 +1399618 @@ -37402,67 +38000,67 @@ 4 5 -12 +11 8 9 -12 +11 500 501 -12 +11 929 930 -12 +11 1740 1741 -12 +11 1873 1874 -12 +11 5235 5236 -12 +11 5497 5498 -12 +11 7551 7552 -12 +11 14073 14074 -12 +11 26464 26465 -12 +11 27121 27122 -12 +11 28787 28788 -12 +11 @@ -37478,67 +38076,67 @@ 4 5 -12 +11 7 8 -12 +11 500 501 -12 +11 929 930 -12 +11 1740 1741 -12 +11 1873 1874 -12 +11 5235 5236 -12 +11 5497 5498 -12 +11 7551 7552 -12 +11 14073 14074 -12 +11 26122 26123 -12 +11 27121 27122 -12 +11 28555 28556 -12 +11 @@ -37554,12 +38152,12 @@ 1 2 -1448766 +1392549 2 234 -364 +350 @@ -37575,7 +38173,7 @@ 1 2 -1449131 +1392900 @@ -37585,15 +38183,15 @@ preprocpair -416102 +399956 begin -329694 +316901 elseelifend -416102 +399956 @@ -37607,17 +38205,17 @@ 1 2 -261399 +251256 2 3 -60247 +57909 3 53 -8047 +7735 @@ -37633,7 +38231,7 @@ 1 2 -416102 +399956 @@ -37643,41 +38241,41 @@ preproctrue -183112 +176006 branch -183112 +176006 preprocfalse -130000 +124956 branch -130000 +124956 preproctext -1062775 +1021536 id -1062775 +1021536 head -510083 +490290 body -192959 +185471 @@ -37691,7 +38289,7 @@ 1 2 -1062775 +1021536 @@ -37707,7 +38305,7 @@ 1 2 -1062775 +1021536 @@ -37723,22 +38321,22 @@ 1 2 -380459 +365696 2 3 -86116 +82774 3 19 -38414 +36923 19 752 -5093 +4895 @@ -37754,12 +38352,12 @@ 1 2 -486208 +467341 2 38 -23875 +22948 @@ -37775,12 +38373,12 @@ 1 2 -181252 +174219 2 64395 -11706 +11252 @@ -37796,12 +38394,12 @@ 1 2 -182893 +175796 2 21671 -10065 +9674 @@ -37811,15 +38409,15 @@ includes -321805 +309317 id -321805 +309317 included -60162 +57827 @@ -37833,7 +38431,7 @@ 1 2 -321805 +309317 @@ -37849,37 +38447,37 @@ 1 2 -29503 +28358 2 3 -9895 +9511 3 4 -5117 +4919 4 6 -5348 +5141 6 11 -4643 +4463 11 41 -4558 +4381 41 763 -1094 +1051 @@ -37889,15 +38487,15 @@ link_targets -644 +619 id -644 +619 binary -644 +619 @@ -37911,7 +38509,7 @@ 1 2 -644 +619 @@ -37927,7 +38525,7 @@ 1 2 -644 +619 @@ -37937,15 +38535,15 @@ link_parent -19143690 +19072521 element -5076549 +5117392 link_target -644 +619 @@ -37959,32 +38557,32 @@ 1 2 -1439430 +1455425 2 3 -1904730 +1924496 3 4 -772395 +779113 4 6 -426167 +434215 6 -27 -383109 +28 +394382 -27 +28 45 -150715 +129758 @@ -38000,67 +38598,67 @@ 2 3 -97 +93 5 557 -48 +46 -2555 -5912 -48 +2662 +6059 +46 -6322 -8240 -48 +6479 +8402 +46 -9873 -12705 -48 +10060 +12929 +46 -12726 -19568 -48 +13044 +20082 +46 -24613 -26319 -48 +24978 +26985 +46 -26410 -31881 -48 +27088 +32537 +46 -32502 -37729 -48 +33104 +38465 +46 -39858 -40615 -48 +41123 +42511 +46 -43096 -52498 -48 +44410 +55889 +46 -53361 -126529 -48 +55937 +132605 +46 -345660 -345661 -12 +362472 +362473 +11 From afe666500f54db4dc7aa7c688bcab1c1a12b7264 Mon Sep 17 00:00:00 2001 From: Matthew Gretton-Dann Date: Tue, 5 Nov 2019 11:34:37 +0000 Subject: [PATCH 129/148] C++: Simplify getTemplateArgument*() impl. --- cpp/ql/src/semmle/code/cpp/Class.qll | 18 ------------ cpp/ql/src/semmle/code/cpp/Declaration.qll | 33 ++++++++++------------ cpp/ql/src/semmle/code/cpp/Function.qll | 19 ------------- cpp/ql/src/semmle/code/cpp/Variable.qll | 18 ------------ 4 files changed, 15 insertions(+), 73 deletions(-) diff --git a/cpp/ql/src/semmle/code/cpp/Class.qll b/cpp/ql/src/semmle/code/cpp/Class.qll index 5ccf6193c4f..297654a1afa 100644 --- a/cpp/ql/src/semmle/code/cpp/Class.qll +++ b/cpp/ql/src/semmle/code/cpp/Class.qll @@ -605,24 +605,6 @@ class Class extends UserType { class_instantiation(underlyingElement(this), unresolveElement(c)) } - /** - * Gets the `i`th template argument used to instantiate this class from a - * class template. When called on a class template, this will return the - * `i`th template parameter. - */ - override Type getTemplateArgumentType(int i) { - class_template_argument(underlyingElement(this), i, unresolveElement(result)) - } - - /** - * Gets the `i`th template argument value used to instantiate this class from a - * class template. When called on a class template, this will return the - * `i`th template parameter value. - */ - override Expr getTemplateArgumentValue(int i) { - class_template_argument_value(underlyingElement(this), i, unresolveElement(result)) - } - /** * Holds if this class/struct is polymorphic (has a virtual function, or * inherits one). diff --git a/cpp/ql/src/semmle/code/cpp/Declaration.qll b/cpp/ql/src/semmle/code/cpp/Declaration.qll index 9c8ce250d8d..89f52f174de 100644 --- a/cpp/ql/src/semmle/code/cpp/Declaration.qll +++ b/cpp/ql/src/semmle/code/cpp/Declaration.qll @@ -255,25 +255,22 @@ abstract class Declaration extends Locatable, @declaration { result = count(int i | exists(getTemplateArgument(i))) } - /** - * INTERNAL: Do not use. - * - * Gets a Type for a template argument. May be the template argument itself - * or the type of a non-type template argument. - * - * Use `getTemplateArgument` or `getTemplateKind` instead. - */ - Type getTemplateArgumentType(int index) { none() } + private Type getTemplateArgumentType(int index) { + class_template_argument(underlyingElement(this), index, unresolveElement(result)) + or + function_template_argument(underlyingElement(this), index, unresolveElement(result)) + or + variable_template_argument(underlyingElement(this), index, unresolveElement(result)) + } - /** - * INTERNAL: Do not use. - * - * Gets an Expression representing the value of a non-type template - * argument. - * - * Use `getTemplateArgument` or `getTemplateKind` instead. - */ - Expr getTemplateArgumentValue(int index) { none() } + + private Expr getTemplateArgumentValue(int index) { + class_template_argument_value(underlyingElement(this), index, unresolveElement(result)) + or + function_template_argument_value(underlyingElement(this), index, unresolveElement(result)) + or + variable_template_argument_value(underlyingElement(this), index, unresolveElement(result)) + } } /** diff --git a/cpp/ql/src/semmle/code/cpp/Function.qll b/cpp/ql/src/semmle/code/cpp/Function.qll index 5436c3ba218..14edf014b21 100644 --- a/cpp/ql/src/semmle/code/cpp/Function.qll +++ b/cpp/ql/src/semmle/code/cpp/Function.qll @@ -343,25 +343,6 @@ class Function extends Declaration, ControlFlowNode, AccessHolder, @function { function_instantiation(underlyingElement(this), unresolveElement(f)) } - /** - * Gets the `i`th template argument used to instantiate this function from a - * function template. When called on a function template, this will return the - * `i`th template parameter. - */ - override Type getTemplateArgumentType(int index) { - function_template_argument(underlyingElement(this), index, unresolveElement(result)) - } - - /** - * Gets the value of the `i`th template argument used to instantiate this - * function from a function template if that argument was a 'non-type' - * argument. When called on a function template, this with return the value - * of the `i`th template parameter. - */ - override Expr getTemplateArgumentValue(int index) { - function_template_argument_value(underlyingElement(this), index, unresolveElement(result)) - } - /** * Holds if this function is defined in several files. This is illegal in * C (though possible in some C++ compilers), and likely indicates that diff --git a/cpp/ql/src/semmle/code/cpp/Variable.qll b/cpp/ql/src/semmle/code/cpp/Variable.qll index 689f1c3bfa5..194cc5333c7 100644 --- a/cpp/ql/src/semmle/code/cpp/Variable.qll +++ b/cpp/ql/src/semmle/code/cpp/Variable.qll @@ -155,24 +155,6 @@ class Variable extends Declaration, @variable { variable_instantiation(underlyingElement(this), unresolveElement(v)) } - /** - * Gets the `i`th template argument used to instantiate this variable from a - * variable template. When called on a variable template, this will return the - * `i`th template parameter. - */ - override Type getTemplateArgumentType(int index) { - variable_template_argument(underlyingElement(this), index, unresolveElement(result)) - } - - /** - * Gets the `i`th template argument value used to instantiate this variable from a - * variable template. When called on a variable template, this will return the - * `i`th template parameter value. - */ - override Expr getTemplateArgumentValue(int index) { - variable_template_argument_value(underlyingElement(this), index, unresolveElement(result)) - } - /** * Holds if this is a compiler-generated variable. For example, a * [range-based for loop](http://en.cppreference.com/w/cpp/language/range-for) From 8eef953cd75ee10c1292a467fff3297202e5cbc1 Mon Sep 17 00:00:00 2001 From: Matthew Gretton-Dann Date: Tue, 5 Nov 2019 11:35:35 +0000 Subject: [PATCH 130/148] C++: Update 1.23 change notes --- change-notes/1.23/analysis-cpp.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/change-notes/1.23/analysis-cpp.md b/change-notes/1.23/analysis-cpp.md index f22436f55cb..a92f78ca2f7 100644 --- a/change-notes/1.23/analysis-cpp.md +++ b/change-notes/1.23/analysis-cpp.md @@ -54,3 +54,8 @@ The following changes in version 1.23 affect C/C++ analysis in all applications. lead to regressions (or improvements) in how queries are optimized because optimization in QL relies on static size estimates, and the control-flow edge relations will now have different size estimates than before. +* Support has been added for non-type template arguments. This means that the + return type of `Declaration::getTemplateArgument()` and + `Declaration::getATemplateArgument` have changed to `Locatable`. See the + documentation for `Declaration::getTemplateArgument()` and + `Declaration::getTemplateArgumentKind()` for details. From 5106626bd07d613f8e2f08193921b14f8539581d Mon Sep 17 00:00:00 2001 From: Geoffrey White <40627776+geoffw0@users.noreply.github.com> Date: Tue, 5 Nov 2019 13:06:43 +0000 Subject: [PATCH 131/148] CPP: QLDoc helper predicates. --- .../CWE/CWE-457/InitializationFunctions.qll | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/cpp/ql/src/Security/CWE/CWE-457/InitializationFunctions.qll b/cpp/ql/src/Security/CWE/CWE-457/InitializationFunctions.qll index 720cf11950b..52b251bad00 100644 --- a/cpp/ql/src/Security/CWE/CWE-457/InitializationFunctions.qll +++ b/cpp/ql/src/Security/CWE/CWE-457/InitializationFunctions.qll @@ -619,35 +619,41 @@ Function getAPossibleDefinition(Function undefinedFunction) { result.isDefined() } +/** + * Helper predicate for `getTarget`, that computes possible targets of a `Call`. + * + * If there is at least one defined target after performing some simple virtual dispatch + * resolution, then the result is all the defined targets. + */ private Function getTarget1(Call c) { - /* - * If there is at least one defined target after performing some simple virtual dispatch - * resolution, then the result is all the defined targets. - */ - result = VirtualDispatch::getAViableTarget(c) and result.isDefined() } +/** + * Helper predicate for `getTarget`, that computes possible targets of a `Call`. + * + * If we can use the heuristic matching of functions to find definitions for some of the viable + * targets, return those. + */ private Function getTarget2(Call c) { - /* - * If we can use the heuristic matching of functions to find definitions for some of the viable - * targets, return those. - */ - not exists(getTarget1(c)) and result = getAPossibleDefinition(VirtualDispatch::getAViableTarget(c)) } +/** + * Helper predicate for `getTarget`, that computes possible targets of a `Call`. + * + * Otherwise, the result is the undefined `Function` instances. + */ private Function getTarget3(Call c) { not exists(getTarget1(c)) and not exists(getTarget2(c)) and - // Otherwise, the result is the undefined `Function` instances. result = VirtualDispatch::getAViableTarget(c) } /** - * Gets a possible target for the Call, using the name and parameter matching if we did not associate + * Gets a possible target for the `Call`, using the name and parameter matching if we did not associate * this call with a specific definition at link or compile time, and performing simple virtual * dispatch resolution. */ From 7456a92d6dc5e45f449bf485faba572dfc5a1324 Mon Sep 17 00:00:00 2001 From: Geoffrey White <40627776+geoffw0@users.noreply.github.com> Date: Tue, 5 Nov 2019 13:10:19 +0000 Subject: [PATCH 132/148] CPP: Autoformat. --- cpp/ql/src/Security/CWE/CWE-457/InitializationFunctions.qll | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/ql/src/Security/CWE/CWE-457/InitializationFunctions.qll b/cpp/ql/src/Security/CWE/CWE-457/InitializationFunctions.qll index 52b251bad00..86102008e2b 100644 --- a/cpp/ql/src/Security/CWE/CWE-457/InitializationFunctions.qll +++ b/cpp/ql/src/Security/CWE/CWE-457/InitializationFunctions.qll @@ -618,7 +618,7 @@ Function getAPossibleDefinition(Function undefinedFunction) { ) and result.isDefined() } - + /** * Helper predicate for `getTarget`, that computes possible targets of a `Call`. * From df2fbfb3d02485b5a568769ff78d49c6841282b5 Mon Sep 17 00:00:00 2001 From: Jonas Jensen Date: Fri, 1 Nov 2019 13:16:45 +0100 Subject: [PATCH 133/148] C++: localInstruction{Flow,Taint} helpers These are analogous to the existing `localExpr{Flow,Taint}` predicates. --- .../semmle/code/cpp/ir/dataflow/internal/DataFlowUtil.qll | 8 ++++++++ .../code/cpp/ir/dataflow/internal/TaintTrackingUtil.qll | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/cpp/ql/src/semmle/code/cpp/ir/dataflow/internal/DataFlowUtil.qll b/cpp/ql/src/semmle/code/cpp/ir/dataflow/internal/DataFlowUtil.qll index f824e0b6bf2..cd989c94710 100644 --- a/cpp/ql/src/semmle/code/cpp/ir/dataflow/internal/DataFlowUtil.qll +++ b/cpp/ql/src/semmle/code/cpp/ir/dataflow/internal/DataFlowUtil.qll @@ -214,6 +214,14 @@ private predicate simpleInstructionLocalFlowStep(Instruction iFrom, Instruction */ predicate localFlow(Node source, Node sink) { localFlowStep*(source, sink) } +/** + * Holds if data can flow from `i1` to `i2` in zero or more + * local (intra-procedural) steps. + */ +predicate localInstructionFlow(Instruction e1, Instruction e2) { + localFlow(instructionNode(e1), instructionNode(e2)) +} + /** * Holds if data can flow from `e1` to `e2` in zero or more * local (intra-procedural) steps. diff --git a/cpp/ql/src/semmle/code/cpp/ir/dataflow/internal/TaintTrackingUtil.qll b/cpp/ql/src/semmle/code/cpp/ir/dataflow/internal/TaintTrackingUtil.qll index e34709e94ec..8d7c9194f4f 100644 --- a/cpp/ql/src/semmle/code/cpp/ir/dataflow/internal/TaintTrackingUtil.qll +++ b/cpp/ql/src/semmle/code/cpp/ir/dataflow/internal/TaintTrackingUtil.qll @@ -53,6 +53,14 @@ private predicate localInstructionTaintStep(Instruction nodeFrom, Instruction no */ predicate localTaint(DataFlow::Node source, DataFlow::Node sink) { localTaintStep*(source, sink) } +/** + * Holds if taint can flow from `i1` to `i2` in zero or more + * local (intra-procedural) steps. + */ +predicate localInstructionTaint(Instruction i1, Instruction i2) { + localTaint(DataFlow::instructionNode(i1), DataFlow::instructionNode(i2)) +} + /** * Holds if taint can flow from `e1` to `e2` in zero or more * local (intra-procedural) steps. From 20ae183c16437c57ebdc651078caa39375103277 Mon Sep 17 00:00:00 2001 From: Matthew Gretton-Dann Date: Tue, 5 Nov 2019 14:18:29 +0000 Subject: [PATCH 134/148] C++: Tidy up formatting --- cpp/ql/src/semmle/code/cpp/Declaration.qll | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/cpp/ql/src/semmle/code/cpp/Declaration.qll b/cpp/ql/src/semmle/code/cpp/Declaration.qll index 89f52f174de..04135880129 100644 --- a/cpp/ql/src/semmle/code/cpp/Declaration.qll +++ b/cpp/ql/src/semmle/code/cpp/Declaration.qll @@ -257,18 +257,17 @@ abstract class Declaration extends Locatable, @declaration { private Type getTemplateArgumentType(int index) { class_template_argument(underlyingElement(this), index, unresolveElement(result)) - or + or function_template_argument(underlyingElement(this), index, unresolveElement(result)) - or + or variable_template_argument(underlyingElement(this), index, unresolveElement(result)) } - private Expr getTemplateArgumentValue(int index) { class_template_argument_value(underlyingElement(this), index, unresolveElement(result)) - or + or function_template_argument_value(underlyingElement(this), index, unresolveElement(result)) - or + or variable_template_argument_value(underlyingElement(this), index, unresolveElement(result)) } } From 16b63b3d01a68b8065f463002f43254aa9cdb369 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Tue, 5 Nov 2019 15:45:17 +0100 Subject: [PATCH 135/148] move deferred model to the query where it is used --- .../src/Statements/UseOfReturnlessFunction.ql | 51 +++++++++++++++++++ .../ql/src/semmle/javascript/Promises.qll | 42 --------------- .../TaintTracking/BasicTaintTracking.expected | 1 - .../library-tests/TaintTracking/promise.js | 11 ---- .../Statements/UseOfReturnlessFunction/tst.js | 6 +++ 5 files changed, 57 insertions(+), 54 deletions(-) diff --git a/javascript/ql/src/Statements/UseOfReturnlessFunction.ql b/javascript/ql/src/Statements/UseOfReturnlessFunction.ql index f8f459cc84d..48af33b8e43 100644 --- a/javascript/ql/src/Statements/UseOfReturnlessFunction.ql +++ b/javascript/ql/src/Statements/UseOfReturnlessFunction.ql @@ -149,6 +149,57 @@ predicate voidArrayCallback(DataFlow::CallNode call, Function func) { ) } + +/** + * Provides classes for working with various Deferred implementations. + * It is a heuristic. The heuristic assume that a class is a promise defintion + * if the class is called "Deferred" and the method `resolve` is called on an instance. + * + * Removes some false positives in the js/use-of-returnless-function query. + */ +module Deferred { + /** + * An instance of a `Deferred` class. + * E.g. the result from `new Deferred()` or `new $.Deferred()`. + */ + class DeferredInstance extends DataFlow::NewNode { + // Describes both `new Deferred()`, `new $.Deferred` and other variants. + DeferredInstance() { this.getCalleeName() = "Deferred" } + + private DataFlow::SourceNode ref(DataFlow::TypeTracker t) { + t.start() and + result = this + or + exists(DataFlow::TypeTracker t2 | result = ref(t2).track(t2, t)) + } + + DataFlow::SourceNode ref() { result = ref(DataFlow::TypeTracker::end()) } + } + + /** + * A promise object created by a Deferred constructor + */ + private class DeferredPromiseDefinition extends PromiseDefinition, DeferredInstance { + DeferredPromiseDefinition() { + // hardening of the "Deferred" heuristic: a method call to `resolve`. + exists(ref().getAMethodCall("resolve")) + } + + override DataFlow::FunctionNode getExecutor() { result = getCallback(0) } + } + + /** + * A resolved promise created by a `new Deferred().resolve()` call. + */ + class ResolvedDeferredPromiseDefinition extends ResolvedPromiseDefinition { + ResolvedDeferredPromiseDefinition() { + this = any(DeferredPromiseDefinition def).ref().getAMethodCall("resolve") + } + + override DataFlow::Node getValue() { result = getArgument(0) } + } +} + from DataFlow::CallNode call, Function func, string name, string msg where ( diff --git a/javascript/ql/src/semmle/javascript/Promises.qll b/javascript/ql/src/semmle/javascript/Promises.qll index ae8554c1a0a..858064e85a0 100644 --- a/javascript/ql/src/semmle/javascript/Promises.qll +++ b/javascript/ql/src/semmle/javascript/Promises.qll @@ -32,48 +32,6 @@ module Bluebird { } } -/** - * Provides classes for working with various Deferred implementations - */ -module Deferred { - class DeferredInstance extends DataFlow::NewNode { - // Describes both `new Deferred()`, `new $.Deferred` and other variants. - DeferredInstance() { this.getCalleeName() = "Deferred" } - - private DataFlow::SourceNode ref(DataFlow::TypeTracker t) { - t.start() and - result = this - or - exists(DataFlow::TypeTracker t2 | result = ref(t2).track(t2, t)) - } - - DataFlow::SourceNode ref() { result = ref(DataFlow::TypeTracker::end()) } - } - - /** - * A promise object created by a Deferred constructor - */ - private class DeferredPromiseDefinition extends PromiseDefinition, DeferredInstance { - DeferredPromiseDefinition() { - // hardening of the "Deferred" heuristic: a method call to `resolve`. - exists(ref().getAMethodCall("resolve")) - } - - override DataFlow::FunctionNode getExecutor() { result = getCallback(0) } - } - - /** - * A resolved promise created by a `new Deferred().resolve()` call. - */ - class ResolvedDeferredPromiseDefinition extends ResolvedPromiseDefinition { - ResolvedDeferredPromiseDefinition() { - this = any(DeferredPromiseDefinition def).ref().getAMethodCall("resolve") - } - - override DataFlow::Node getValue() { result = getArgument(0) } - } -} - /** * Provides classes for working with the `q` library (https://github.com/kriskowal/q). */ diff --git a/javascript/ql/test/library-tests/TaintTracking/BasicTaintTracking.expected b/javascript/ql/test/library-tests/TaintTracking/BasicTaintTracking.expected index 77592a2e855..2722f67d6fd 100644 --- a/javascript/ql/test/library-tests/TaintTracking/BasicTaintTracking.expected +++ b/javascript/ql/test/library-tests/TaintTracking/BasicTaintTracking.expected @@ -69,7 +69,6 @@ typeInferenceMismatch | promise.js:5:25:5:32 | source() | promise.js:5:8:5:33 | bluebir ... urce()) | | promise.js:10:24:10:31 | source() | promise.js:10:8:10:32 | Promise ... urce()) | | promise.js:12:20:12:27 | source() | promise.js:13:8:13:23 | resolver.promise | -| promise.js:22:23:22:30 | source() | promise.js:22:7:22:31 | promise ... urce()) | | sanitizer-guards.js:2:11:2:18 | source() | sanitizer-guards.js:4:8:4:8 | x | | sanitizer-guards.js:13:14:13:21 | source() | sanitizer-guards.js:15:10:15:15 | this.x | | sanitizer-guards.js:13:14:13:21 | source() | sanitizer-guards.js:21:14:21:19 | this.x | diff --git a/javascript/ql/test/library-tests/TaintTracking/promise.js b/javascript/ql/test/library-tests/TaintTracking/promise.js index d16e57241e8..9714d258df5 100644 --- a/javascript/ql/test/library-tests/TaintTracking/promise.js +++ b/javascript/ql/test/library-tests/TaintTracking/promise.js @@ -11,15 +11,4 @@ function closure() { let resolver = Promise.withResolver(); resolver.resolve(source()); sink(resolver.promise); // NOT OK -} - -class Deferred { - -} - -function deferred() { - var promise = new Deferred(); - sink(promise.resolve(source())); // NOT OK - - new Deferred().reject("foo") // <- a reject has to exist. } \ No newline at end of file diff --git a/javascript/ql/test/query-tests/Statements/UseOfReturnlessFunction/tst.js b/javascript/ql/test/query-tests/Statements/UseOfReturnlessFunction/tst.js index 8a6a44707ed..90a52153dea 100644 --- a/javascript/ql/test/query-tests/Statements/UseOfReturnlessFunction/tst.js +++ b/javascript/ql/test/query-tests/Statements/UseOfReturnlessFunction/tst.js @@ -82,4 +82,10 @@ var baz = [1,2,3].filter(n => {n === 3}) // OK console.log(baz); + + class Deferred { + + } + + new Deferred().resolve(onlySideEffects()); // OK })(); \ No newline at end of file From 8661de11f2ff9c5651899095b3465f11faaf5216 Mon Sep 17 00:00:00 2001 From: james Date: Tue, 5 Nov 2019 14:50:28 +0000 Subject: [PATCH 136/148] docs: update ql terminology --- docs/language/learn-ql/ql-training.rst | 52 +++++++++---------- .../ql-training/cpp/bad-overflow-guard.rst | 24 ++++----- .../ql-training/cpp/control-flow-cpp.rst | 4 +- .../ql-training/cpp/data-flow-cpp.rst | 4 +- .../ql-training/cpp/global-data-flow-cpp.rst | 4 +- .../language/ql-training/cpp/intro-ql-cpp.rst | 38 +++++++------- .../cpp/program-representation-cpp.rst | 10 ++-- docs/language/ql-training/cpp/snprintf.rst | 2 +- docs/language/ql-training/index.rst | 4 +- .../ql-training/java/apache-struts-java.rst | 2 +- .../java/global-data-flow-java.rst | 2 +- .../ql-training/java/intro-ql-java.rst | 36 ++++++------- .../java/program-representation-java.rst | 2 +- .../ql-training/java/query-injection-java.rst | 20 +++---- .../slide-snippets/abstract-syntax-tree.rst | 10 ++-- .../ql-training/slide-snippets/info.rst | 33 ------------ .../slide-snippets/intro-ql-general.rst | 16 +++--- 17 files changed, 115 insertions(+), 148 deletions(-) delete mode 100644 docs/language/ql-training/slide-snippets/info.rst diff --git a/docs/language/learn-ql/ql-training.rst b/docs/language/learn-ql/ql-training.rst index 66afa06d754..757de7daed6 100644 --- a/docs/language/learn-ql/ql-training.rst +++ b/docs/language/learn-ql/ql-training.rst @@ -1,19 +1,19 @@ -QL training and variant analysis examples -######################################### +CodeQL training and variant analysis examples +============================================= QL and variant analysis -======================= +----------------------- `Variant analysis `__ is the process of using a known vulnerability as a seed to find similar problems in your code. Security engineers typically perform variant analysis to identify possible vulnerabilities and to ensure that these threats are properly fixed across multiple code bases. -`QL `__ is Semmle's variant analysis engine, and it is also the technology that underpins LGTM, Semmle's community driven security analysis platform. Together, QL and LGTM provide continuous monitoring and scalable variant analysis for your projects, even if you don’t have your own team of dedicated security engineers. You can read more about using QL and LGTM in variant analysis in the `Semmle blog `__. +`CodeQL `__ is the code analysis engine that underpins LGTM, Semmle's community driven security analysis platform. Together, CodeQL and LGTM provide continuous monitoring and scalable variant analysis for your projects, even if you don’t have your own team of dedicated security engineers. You can read more about using CodeQL and LGTM in variant analysis in the `Semmle blog `__. -The QL language is easy to learn, and exploring code using QL is the most efficient way to perform variant analysis. +CodeQL is easy to learn, and exploring code using CodeQL is the most efficient way to perform variant analysis. Learning QL for variant analysis -================================ +-------------------------------- -Start learning how to use QL in variant analysis for a specific language by looking at the topics below. Each topic links to a short presentation on the QL language, QL libraries, or an example variant discovered using QL. +Start learning how to use CodeQL in variant analysis for a specific language by looking at the topics below. Each topic links to a short presentation on CodeQL, its libraries, or an example variant discovered using CodeQL. .. |arrow-l| unicode:: U+2190 @@ -24,7 +24,7 @@ Start learning how to use QL in variant analysis for a specific language by look When you have selected a presentation, use |arrow-r| and |arrow-l| to navigate between slides. Press **p** to view the additional notes on slides that have an information icon |info| in the top right corner, and press **f** to enter full-screen mode. -The presentations contain a number of QL query examples. +The presentations contain a number of query examples. We recommend that you download `QL for Eclipse `__ and import the example snapshot for each presentation so that you can find the bugs mentioned in the slides. @@ -32,35 +32,35 @@ We recommend that you download `QL for Eclipse `__–an introduction to variant analysis and QL for C/C++ programmers. +- `Introduction to variant analysis: CodeQL for C/C++ <../ql-training/cpp/intro-ql-cpp.html>`__–an introduction to variant analysis and CodeQL for C/C++ programmers. - `Example: Bad overflow guard <../ql-training/cpp/bad-overflow-guard.html>`__–an example of iterative query development to find bad overflow guards in a C++ project. -- `Program representation: QL for C/C++ <../ql-training/cpp/program-representation-cpp.html>`__–information on how QL analysis represents C/C++ programs. -- `Introduction to local data flow <../ql-training/cpp/data-flow-cpp.html>`__–an introduction to analyzing local data flow in C/C++ using QL, including an example demonstrating how to develop a query to find a real CVE. +- `Program representation: CodeQL for C/C++ <../ql-training/cpp/program-representation-cpp.html>`__–information on how CodeQL analysis represents C/C++ programs. +- `Introduction to local data flow <../ql-training/cpp/data-flow-cpp.html>`__–an introduction to analyzing local data flow in C/C++ using CodeQL, including an example demonstrating how to develop a query to find a real CVE. - `Exercise: snprintf overflow <../ql-training/cpp/snprintf.html>`__–an example demonstrating how to develop a data flow query. -- `Introduction to global data flow <../ql-training/cpp/global-data-flow-cpp.html>`__–an introduction to analyzing global data flow in C/C++ using QL. -- `Analyzing control flow: QL for C/C++ <../ql-training/cpp/control-flow-cpp.html>`__–an introduction to analyzing control flow in C/C++ using QL. +- `Introduction to global data flow <../ql-training/cpp/global-data-flow-cpp.html>`__–an introduction to analyzing global data flow in C/C++ using CodeQL. +- `Analyzing control flow: CodeQL for C/C++ <../ql-training/cpp/control-flow-cpp.html>`__–an introduction to analyzing control flow in C/C++ using QL. -QL and variant analysis for Java --------------------------------- +CodeQL and variant analysis for Java +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- `Introduction to variant analysis: QL for Java <../ql-training/java/intro-ql-java.html>`__–an introduction to variant analysis and QL for Java programmers. +- `Introduction to variant analysis: CodeQL for Java <../ql-training/java/intro-ql-java.html>`__–an introduction to variant analysis and CodeQL for Java programmers. - `Example: Query injection <../ql-training/java/query-injection-java.html>`__–an example of iterative query development to find unsanitized SPARQL injections in a Java project. -- `Program representation: QL for Java <../ql-training/java/program-representation-java.html>`__–information on how QL analysis represents Java programs. -- `Introduction to local data flow <../ql-training/java/data-flow-java.html>`__–an introduction to analyzing local data flow in Java using QL, including an example demonstrating how to develop a query to find a real CVE. +- `Program representation: CodeQL for Java <../ql-training/java/program-representation-java.html>`__–information on how CodeQL analysis represents Java programs. +- `Introduction to local data flow <../ql-training/java/data-flow-java.html>`__–an introduction to analyzing local data flow in Java using CodeQL, including an example demonstrating how to develop a query to find a real CVE. - `Exercise: Apache Struts <../ql-training/java/apache-struts-java.html>`__–an example demonstrating how to develop a data flow query. -- `Introduction to global data flow <../ql-training/java/global-data-flow-java.html>`__–an introduction to analyzing global data flow in Java using QL. +- `Introduction to global data flow <../ql-training/java/global-data-flow-java.html>`__–an introduction to analyzing global data flow in Java using CodeQL. More resources --------------- +~~~~~~~~~~~~~~ -- If you are completely new to QL, look at our introductory topics in :ref:`Getting started `. -- To find more detailed information about how to write QL queries for specific languages, visit the links in :ref:`Writing QL queries `. -- To read more about how QL queries have been used in Semmle's security research, and to read about new QL developments, visit the `Semmle blog `__. +- If you are completely new to CodeQL, look at our introductory topics in :doc:`Learning CodeQL `. +- To find more detailed information about how to write CodeQL queries for specific languages, visit the links in :ref:`Writing CodeQL queries `. +- To read more about how CodeQL queries have been used in Semmle's security research, and to read about new CodeQL developments, visit the `Semmle blog `__. - Find more examples of queries written by Semmle's own security researchers in the `Semmle Demos repository `__ on GitHub. diff --git a/docs/language/ql-training/cpp/bad-overflow-guard.rst b/docs/language/ql-training/cpp/bad-overflow-guard.rst index 12f35440906..5e9fbcfc700 100644 --- a/docs/language/ql-training/cpp/bad-overflow-guard.rst +++ b/docs/language/ql-training/cpp/bad-overflow-guard.rst @@ -2,7 +2,7 @@ Example: Bad overflow guard =========================== -QL for C/C++ +CodeQL for C/C++ .. container:: semmle-logo @@ -127,13 +127,13 @@ This happens even though the overflow check passed! .. rst-class:: background2 -Developing a QL query -===================== +Developing a CodeQL query +========================= Finding bad overflow guards -QL query: bad overflow guards -============================= +CodeQL query: bad overflow guards +================================== Let’s look for overflow guards of the form ``v + b < v``, using the classes ``AddExpr``, ``Variable`` and ``RelationalOperation`` from the ``cpp`` library. @@ -153,10 +153,10 @@ Let’s look for overflow guards of the form ``v + b < v``, using the classes - a ``RelationalOperation``: the overflow comparison check. - a ``Variable``: used as an argument to both the addition and comparison. - - The ``where`` part of the query ties these three QL variables together using `predicates `__ defined in the `standard QL for C/C++ library `__. + - The ``where`` part of the query ties these three variables together using `predicates `__ defined in the `standard CodeQL for C/C++ library `__. -QL query: bad overflow guards -============================= +CodeQL query: bad overflow guards +================================= We want to ensure the operands being added have size less than 4 bytes. @@ -180,8 +180,8 @@ We can get the size (in bytes) of a type using the ``getSize()`` method. - We therefore write a helper predicate for small expressions. - This predicate effectively represents the set of all expressions in the database where the size of the type of the expression is less than 4 bytes, that is, less than 32-bits. -QL query: bad overflow guards -============================= +CodeQL query: bad overflow guards +================================== We can ensure the operands being added have size less than 4 bytes, using our new predicate. @@ -216,8 +216,8 @@ Now our query becomes: - The “range” part, ``op = a.getAnOperand()``, restricts ``op`` to being one of the two operands to the addition. - The “condition” part, ``isSmall(op)``, says that the ``forall`` holds only if the condition (that the ``op`` is small) holds for everything in the range–that is, both the arguments to the addition. -QL query: bad overflow guards -============================= +CodeQL query: bad overflow guards +================================= Sometimes the result of the addition is cast to a small type of size less than 4 bytes, preventing automatic widening. We don’t want our query to flag these instances. diff --git a/docs/language/ql-training/cpp/control-flow-cpp.rst b/docs/language/ql-training/cpp/control-flow-cpp.rst index ea8e2c1b158..a8171a44b5d 100644 --- a/docs/language/ql-training/cpp/control-flow-cpp.rst +++ b/docs/language/ql-training/cpp/control-flow-cpp.rst @@ -2,7 +2,7 @@ Analyzing control flow ====================== -QL for C/C++ +CodeQL for C/C++ .. container:: semmle-logo @@ -226,7 +226,7 @@ A ``GuardCondition`` is a ``Boolean`` condition that controls one or more basic Further materials ================= -- QL for C/C++: https://help.semmle.com/QL/learn-ql/ql/cpp/ql-for-cpp.html +- CodeQL for C/C++: https://help.semmle.com/QL/learn-ql/ql/cpp/ql-for-cpp.html - API reference: https://help.semmle.com/qldoc/cpp .. rst-class:: end-slide diff --git a/docs/language/ql-training/cpp/data-flow-cpp.rst b/docs/language/ql-training/cpp/data-flow-cpp.rst index 2b020499eb0..5377d8d52d1 100644 --- a/docs/language/ql-training/cpp/data-flow-cpp.rst +++ b/docs/language/ql-training/cpp/data-flow-cpp.rst @@ -86,9 +86,9 @@ Write a query that flags ``printf`` calls where the format argument is not a ``S .. note:: - This first query is about finding places where the format specifier is not a constant string. In QL for C/C++, constant strings are modeled as ``StringLiteral`` nodes, so we are looking for calls to format functions where the format specifier argument is not a string literal. + This first query is about finding places where the format specifier is not a constant string. In CodeQL for C/C++, constant strings are modeled as ``StringLiteral`` nodes, so we are looking for calls to format functions where the format specifier argument is not a string literal. - The `C/C++ standard libraries `__ include many different formatting functions that may be vulnerable to this particular attack–including ``printf``, ``snprintf``, and others. Furthermore, each of these different formatting functions may include the format string in a different position in the argument list. Instead of laboriously listing all these different variants, we can make use of the QL for C/C++ standard library class ``FormattingFunction``, which provides an interface that models common formatting functions in C/C++. + The `C/C++ standard libraries `__ include many different formatting functions that may be vulnerable to this particular attack–including ``printf``, ``snprintf``, and others. Furthermore, each of these different formatting functions may include the format string in a different position in the argument list. Instead of laboriously listing all these different variants, we can make use of the CodeQL for C/C++ standard library class ``FormattingFunction``, which provides an interface that models common formatting functions in C/C++. Meh... ====== diff --git a/docs/language/ql-training/cpp/global-data-flow-cpp.rst b/docs/language/ql-training/cpp/global-data-flow-cpp.rst index 6033581ffc3..bc048ac59f4 100644 --- a/docs/language/ql-training/cpp/global-data-flow-cpp.rst +++ b/docs/language/ql-training/cpp/global-data-flow-cpp.rst @@ -2,7 +2,7 @@ Introduction to global data flow ================================ -QL for C/C++ +CodeQL for C/C++ .. container:: semmle-logo @@ -77,7 +77,7 @@ The library class ``SecurityOptions`` provides a (configurable) model of what co .. note:: - We first define what it means to be a *source* of tainted data for this particular problem. In this case, what we care about is whether the format string can be provided by an external user to our application or service. As there are many such ways external data could be introduced into the system, the standard QL libraries for C/C++ include an extensible API for modeling user input. In this case, we will simply use the predefined set of *user inputs*, which includes arguments provided to command line applications. + We first define what it means to be a *source* of tainted data for this particular problem. In this case, what we care about is whether the format string can be provided by an external user to our application or service. As there are many such ways external data could be introduced into the system, the standard CodeQL libraries for C/C++ include an extensible API for modeling user input. In this case, we will simply use the predefined set of *user inputs*, which includes arguments provided to command line applications. Defining sinks (exercise) diff --git a/docs/language/ql-training/cpp/intro-ql-cpp.rst b/docs/language/ql-training/cpp/intro-ql-cpp.rst index 82eb62a3ba8..dddc86ca77f 100644 --- a/docs/language/ql-training/cpp/intro-ql-cpp.rst +++ b/docs/language/ql-training/cpp/intro-ql-cpp.rst @@ -2,7 +2,7 @@ Introduction to variant analysis ================================ -QL for C/C++ +CodeQL for C/C++ .. container:: semmle-logo @@ -56,14 +56,14 @@ Oops .. note:: - Here’s a simple (artificial) bug, which we’ll develop a QL query to catch. + Here’s a simple (artificial) bug, which we’ll develop a query to catch. This function writes a value to a given location in an array, first trying to do a bounds check to validate that the location is within bounds. However, the return statement has been commented out, leaving a redundant if statement and no bounds checking. This case can act as our “patient zero” in the variant analysis game. -A simple QL query -================= +A simple CodeQL query +===================== .. literalinclude:: ../query-examples/cpp/empty-if-cpp.ql :language: ql @@ -72,9 +72,9 @@ A simple QL query We are going to write a simple query which finds “if statements” with empty “then” blocks, so we can highlight the results like those on the previous slide. The query can be run in the `query console on LGTM `__, or in your `IDE `__. - A `QL query `__ consists of a “select” clause that indicates what results should be returned. Typically it will also provide a “from” clause to declare some variables, and a “where” clause to state conditions over those variables. For more information on the structure of query files (including links to useful topics in the `QL language handbook `__), see `Introduction to query files `__. + A `query `__ consists of a “select” clause that indicates what results should be returned. Typically it will also provide a “from” clause to declare some variables, and a “where” clause to state conditions over those variables. For more information on the structure of query files (including links to useful topics in the `QL language handbook `__), see `Introduction to query files `__. - In our example here, the first line of the query imports the `C/C++ standard QL library `__, which defines concepts like ``IfStmt`` and ``Block``. + In our example here, the first line of the query imports the `CodeQL for C/C++ standard library `__, which defines concepts like ``IfStmt`` and ``Block``. The query proper starts by declaring two variables–ifStmt and block. These variables represent sets of values in the database, according to the type of each of the variables. For example, ifStmt has the type IfStmt, which means it represents the set of all if statements in the program. If we simply selected these two variables:: @@ -97,8 +97,8 @@ A simple QL query -Structure of a QL query -======================= +Structure of a query +==================== A **query file** has the extension ``.ql`` and contains a **query clause**, and optionally **predicates**, **classes**, and **modules**. @@ -110,14 +110,14 @@ Each query library also implicitly defines a module. .. note:: - QL queries are always contained in query files with the file extension ``.ql``. `Quick queries `__, run in `QL for Eclipse `__, are no exception: the quick query window maintains a temporary QL file in the background. + Queries are always contained in query files with the file extension ``.ql``. `Quick queries `__, run in `QL for Eclipse `__, are no exception: the quick query window maintains a temporary query file in the background. - Parts of queries can be lifted into `QL library files `__ with the extension ``qll``. Definitions within such libraries can be brought into scope using ``import`` statements, and similarly QLL files can import each other’s definitions using “import” statements. + Parts of queries can be lifted into `library files `__ with the extension ``qll``. Definitions within such libraries can be brought into scope using ``import`` statements, and similarly QLL files can import each other’s definitions using “import” statements. Logic can be encapsulated as user-defined `predicates `__ and `classes `__, and organized into `modules `__. Each QLL file implicitly defines a module, but QL and QLL files can also contain explicit module definitions, as we will see later. -Predicates in QL -================ +Predicates +========== A predicate allows you to pull out and name parts of a query. @@ -135,14 +135,14 @@ A predicate allows you to pull out and name parts of a query. .. note:: - A QL predicate takes zero or more parameters, and its body is a condition on those parameters. The predicate may (or may not) hold. Predicates may also be recursive, simply by referring to themselves (directly or indirectly). + A `predicate `__ takes zero or more parameters, and its body is a condition on those parameters. The predicate may (or may not) hold. Predicates may also be `recursive `__, simply by referring to themselves (directly or indirectly). You can imagine a predicate to be a self-contained from-where-select statement, that produces an intermediate relation, or table. In this case, the ``isEmpty`` predicate will be the set of all blocks which are empty. -Classes in QL -============= +Classes +======= -A QL class allows you to name a set of values and define (member) predicates on them. +A class allows you to name a set of values and define (member) predicates on them. A class has at least one supertype and optionally a **characteristic predicate**; it contains the values that belong to *all* supertypes *and* satisfy the characteristic predicate, if provided. @@ -162,8 +162,8 @@ Member predicates are inherited and can be overridden. In the example, declaring a variable “EmptyBlock e” will allow it to range over only those blocks that have zero statements. -Classes in QL continued -======================= +Classes continued +================= .. container:: column-left @@ -198,7 +198,7 @@ Iterative query refinement .. note:: - QL makes it very easy to experiment with analysis ideas. A common workflow is to start with a simple query (like our “redundant if-statement” example), examine a few results, refine the query based on any patterns that emerge and repeat. + CodeQL makes it very easy to experiment with analysis ideas. A common workflow is to start with a simple query (like our “redundant if-statement” example), examine a few results, refine the query based on any patterns that emerge and repeat. As an exercise, refine the redundant-if query based on the observation that if the if-statement has an “else” clause, then even if the body of the “then” clause is empty, it’s not actually redundant. diff --git a/docs/language/ql-training/cpp/program-representation-cpp.rst b/docs/language/ql-training/cpp/program-representation-cpp.rst index 1850e3e5671..b21677fccab 100644 --- a/docs/language/ql-training/cpp/program-representation-cpp.rst +++ b/docs/language/ql-training/cpp/program-representation-cpp.rst @@ -2,7 +2,7 @@ Program representation ====================== -QL for C/C++ +CodeQL for C/C++ .. container:: semmle-logo @@ -25,8 +25,8 @@ Agenda .. resume slides -AST QL classes -============== +AST classes +=========== Important AST classes include: @@ -66,9 +66,9 @@ Working with variables Working with functions ====================== -Functions are represented by the Function QL class. Each declaration or definition of a function is represented by a ``FunctionDeclarationEntry``. +Functions are represented by the Function class. Each declaration or definition of a function is represented by a ``FunctionDeclarationEntry``. -Calls to functions are modeled by QL class Call and its subclasses: +Calls to functions are modeled by class Call and its subclasses: - ``Call.getTarget()`` gets the declared target of the call; undefined for calls through function pointers - ``Function.getACallToThisFunction()`` gets a call to this function diff --git a/docs/language/ql-training/cpp/snprintf.rst b/docs/language/ql-training/cpp/snprintf.rst index 77e46933fcb..dafebc4a720 100644 --- a/docs/language/ql-training/cpp/snprintf.rst +++ b/docs/language/ql-training/cpp/snprintf.rst @@ -2,7 +2,7 @@ Exercise: ``snprintf`` overflow =============================== -QL for C/C++ +CodeQL for C/C++ .. container:: semmle-logo diff --git a/docs/language/ql-training/index.rst b/docs/language/ql-training/index.rst index 0cc4ca9b024..309fe92c052 100644 --- a/docs/language/ql-training/index.rst +++ b/docs/language/ql-training/index.rst @@ -1,5 +1,5 @@ -QL training and variant analysis examples -========================================= +CodeQL training and variant analysis examples +============================================= .. container:: semmle-logo diff --git a/docs/language/ql-training/java/apache-struts-java.rst b/docs/language/ql-training/java/apache-struts-java.rst index 7fcd2e003cd..5c27ec73979 100644 --- a/docs/language/ql-training/java/apache-struts-java.rst +++ b/docs/language/ql-training/java/apache-struts-java.rst @@ -67,7 +67,7 @@ RCE in Apache Struts Finding the RCE yourself ======================== -#. Create a QL class to find the interface ``org.apache.struts2.rest.handler.ContentTypeHandler`` +#. Create a class to find the interface ``org.apache.struts2.rest.handler.ContentTypeHandler`` **Hint**: Use predicate ``hasQualifiedName(...)`` diff --git a/docs/language/ql-training/java/global-data-flow-java.rst b/docs/language/ql-training/java/global-data-flow-java.rst index 665899f8459..f43c31e0f4f 100644 --- a/docs/language/ql-training/java/global-data-flow-java.rst +++ b/docs/language/ql-training/java/global-data-flow-java.rst @@ -2,7 +2,7 @@ Introduction to global data flow ================================ -QL for Java +CodeQL for Java .. container:: semmle-logo diff --git a/docs/language/ql-training/java/intro-ql-java.rst b/docs/language/ql-training/java/intro-ql-java.rst index 392c18309cb..f80f878492d 100644 --- a/docs/language/ql-training/java/intro-ql-java.rst +++ b/docs/language/ql-training/java/intro-ql-java.rst @@ -2,7 +2,7 @@ Introduction to variant analysis ================================ -QL for Java +CodeQL for Java .. container:: semmle-logo @@ -56,14 +56,14 @@ Oops .. note:: - Here’s a simple (artificial) bug, which we’ll develop a QL query to catch. + Here’s a simple (artificial) bug, which we’ll develop a query to catch. This function writes a value to a given location in an array, first trying to do a bounds check to validate that the location is within bounds. However, the return statement has been commented out, leaving a redundant if statement and no bounds checking. This case can act as our “patient zero” in the variant analysis game. -A simple QL query -================= +A simple CodeQL query +===================== .. literalinclude:: ../query-examples/java/empty-if-java.ql :language: ql @@ -72,9 +72,9 @@ A simple QL query We are going to write a simple query which finds “if statements” with empty “then” blocks, so we can highlight the results like those on the previous slide. The query can be run in the `query console on LGTM `__, or in your `IDE `__. - A `QL query `__ consists of a “select” clause that indicates what results should be returned. Typically it will also provide a “from” clause to declare some variables, and a “where” clause to state conditions over those variables. For more information on the structure of query files (including links to useful topics in the `QL language handbook `__), see `Introduction to query files `__. + A `query `__ consists of a “select” clause that indicates what results should be returned. Typically it will also provide a “from” clause to declare some variables, and a “where” clause to state conditions over those variables. For more information on the structure of query files (including links to useful topics in the `QL language handbook `__), see `Introduction to query files `__. - In our example here, the first line of the query imports the `Java standard QL library `__, which defines concepts like ``IfStmt`` and ``Block``. + In our example here, the first line of the query imports the `CodeQL for Java library `__, which defines concepts like ``IfStmt`` and ``Block``. The query proper starts by declaring two variables–ifStmt and block. These variables represent sets of values in the database, according to the type of each of the variables. For example, ``ifStmt`` has the type ``IfStmt``, which means it represents the set of all if statements in the program. If we simply selected these two variables:: @@ -96,8 +96,8 @@ A simple QL query Finally, we select a location, at which to report the problem, and a message, to explain what the problem is. -Structure of a QL query -======================= +Structure of a query +===================== A **query file** has the extension ``.ql`` and contains a **query clause**, and optionally **predicates**, **classes**, and **modules**. @@ -109,14 +109,14 @@ Each query library also implicitly defines a module. .. note:: - QL queries are always contained in query files with the file extension ``.ql``. `Quick queries `__, run in `QL for Eclipse `__, are no exception: the quick query window maintains a temporary QL file in the background. + Queries are always contained in query files with the file extension ``.ql``. `Quick queries `__, run in `QL for Eclipse `__, are no exception: the quick query window maintains a temporary QL file in the background. - Parts of queries can be lifted into `QL library files `__ with the extension ``.qll``. Definitions within such libraries can be brought into scope using “import” statements, and similarly QLL files can import each other’s definitions using “import” statements. + Parts of queries can be lifted into `library files `__ with the extension ``.qll``. Definitions within such libraries can be brought into scope using “import” statements, and similarly QLL files can import each other’s definitions using “import” statements. Logic can be encapsulated as user-defined `predicates `__ and `classes `__, and organized into `modules `__. Each QLL file implicitly defines a module, but QL and QLL files can also contain explicit module definitions, as we will see later. -Predicates in QL -================ +Predicates +========== A predicate allows you to pull out and name parts of a query. @@ -134,15 +134,15 @@ A predicate allows you to pull out and name parts of a query. .. note:: - A `QL predicate `__ takes zero or more parameters, and its body is a condition on those parameters. The predicate may (or may not) hold. Predicates may also be `recursive `__, simply by referring to themselves (directly or indirectly). + A `predicate `__ takes zero or more parameters, and its body is a condition on those parameters. The predicate may (or may not) hold. Predicates may also be `recursive `__, simply by referring to themselves (directly or indirectly). You can imagine a predicate to be a self-contained from-where-select statement, that produces an intermediate relation, or table. In this case, the ``isEmpty`` predicate will be the set of all blocks which are empty. -Classes in QL -============= +Classes +======= -A QL class allows you to name a set of values and define (member) predicates on them. +A class allows you to name a set of values and define (member) predicates on them. A class has at least one supertype and optionally a **characteristic predicate**; it contains the values that belong to *all* supertypes *and* satisfy the characteristic predicate, if provided. @@ -162,7 +162,7 @@ Member predicates are inherited and can be overridden. In the example, declaring a variable “EmptyBlock e” will allow it to range over only those blocks that have zero statements. -Classes in QL continued +Classes in continued ======================= .. container:: column-left @@ -197,7 +197,7 @@ Iterative query refinement .. note:: - QL makes it very easy to experiment with analysis ideas. A common workflow is to start with a simple query (like our “redundant if-statement” example), examine a few results, refine the query based on any patterns that emerge and repeat. + CodeQL makes it very easy to experiment with analysis ideas. A common workflow is to start with a simple query (like our “redundant if-statement” example), examine a few results, refine the query based on any patterns that emerge and repeat. As an exercise, refine the redundant-if query based on the observation that if the if-statement has an “else” clause, then even if the body of the “then” clause is empty, it’s not actually redundant. diff --git a/docs/language/ql-training/java/program-representation-java.rst b/docs/language/ql-training/java/program-representation-java.rst index d090c30aebe..ef078b53562 100644 --- a/docs/language/ql-training/java/program-representation-java.rst +++ b/docs/language/ql-training/java/program-representation-java.rst @@ -2,7 +2,7 @@ Program representation ====================== -QL for Java +CodeQL for Java .. container:: semmle-logo diff --git a/docs/language/ql-training/java/query-injection-java.rst b/docs/language/ql-training/java/query-injection-java.rst index 67f7fe21a76..b3ef7d7e182 100644 --- a/docs/language/ql-training/java/query-injection-java.rst +++ b/docs/language/ql-training/java/query-injection-java.rst @@ -2,7 +2,7 @@ Example: Query injection ======================== -QL for Java +CodeQL for Java .. container:: semmle-logo @@ -65,13 +65,13 @@ SPARQL injection .. rst-class:: background2 -Developing a QL query -====================== +Developing a query +=================== Finding a query concatenation -QL query: find SPARQL methods -============================= +CodeQL query: find SPARQL methods +================================= Let’s start by looking for calls to methods with names of the form ``sparql*Query``, using the classes ``Method`` and ``MethodAccess`` from the Java library. @@ -88,10 +88,10 @@ Let’s start by looking for calls to methods with names of the form ``sparql*Qu - a ``MethodAccess``: the call to a SPARQL query method - a ``Method``: the SPARQL query method. - - The ``where`` part of the query ties these three QL variables together using `predicates `__ defined in the `standard QL for Java library `__. + - The ``where`` part of the query ties these variables together using `predicates `__ defined in the `standard CodeQL for Java library `__. -QL query: find string concatenation -=================================== +CodeQL query: find string concatenation +======================================= - We now need to define what would make these API calls unsafe. - A simple heuristic would be to look for string concatenation used in the query argument. @@ -113,8 +113,8 @@ Looking at autocomplete suggestions, we see that we can get the type of an expre - We therefore write a helper predicate for finding string concatenation. - This predicate effectively represents the set of all ``add`` expressions in the database where the type of the expression is ``TypeString`` - that is, the addition produces a ``String`` value. -QL query: SPARQL injection -========================== +CodeQL query: SPARQL injection +============================== We can now combine our predicate with the existing query. Note that we do not need to specify that the argument of the method access is an ``AddExpr`` - this is implied by the ``isStringConcat`` requirement. diff --git a/docs/language/ql-training/slide-snippets/abstract-syntax-tree.rst b/docs/language/ql-training/slide-snippets/abstract-syntax-tree.rst index c640aa7a16a..6a26b5c74a4 100644 --- a/docs/language/ql-training/slide-snippets/abstract-syntax-tree.rst +++ b/docs/language/ql-training/slide-snippets/abstract-syntax-tree.rst @@ -35,9 +35,9 @@ The basic representation of an analyzed program is an *abstract syntax tree (AST .. note:: - When writing queries in QL it is important to have in mind the underlying representation of the program which is stored in the database. Typically queries make use of the “AST” representation of the program - a tree structure where program elements are nested within other program elements. + When writing queries it is important to have in mind the underlying representation of the program which is stored in the database. Typically queries make use of the “AST” representation of the program - a tree structure where program elements are nested within other program elements. - The following topics contain overviews of the important AST classes and QL libraries for C/C++, C#, and Java: + The following topics contain overviews of the important AST classes and CodeQL libraries for C/C++, C#, and Java: - `Introducing the C/C++ libraries `__ - `Introducing the C# libraries `__ @@ -47,19 +47,19 @@ The basic representation of an analyzed program is an *abstract syntax tree (AST Database representations of ASTs ================================ -AST nodes and other program elements are encoded in the database as *entity values*. Entities are implemented as integers, but in QL they are opaque - all one can do with them is to check their equality. +AST nodes and other program elements are encoded in the database as *entity values*. Entities are implemented as integers, but in QL they are opaque---all one can do with them is to check their equality. Each entity belongs to an entity type. Entity types have names starting with “@” and are defined in the database schema (not in QL). Properties of AST nodes and their relationships to each other are encoded by database relations, which are predicates defined in the database (not in QL). -Entity types are rarely used directly, the usual pattern is to define a QL class that extends the type and exposes properties of its entities through member predicates. +Entity types are rarely used directly, the usual pattern is to define a class that extends the type and exposes properties of its entities through member predicates. .. note:: ASTs are a typical example of the kind of data representation one finds in object-oriented programming, with data-carrying nodes that reference each other. At first glance, QL, which can only work with atomic values, does not seem to be well suited for working with this kind of data. However, ultimately all that we require of the nodes in an AST is that they have an identity. The relationships among nodes, usually implemented by reference-valued object fields in other languages, can just as well (and arguably more naturally) be represented as relations over nodes. Attaching data (such as strings or numbers) to nodes can also be represented with relations over nodes and primitive values. All we need is a way for relations to reference nodes. This is achieved in QL (as in other database languages) by means of *entity values* (or entities, for short), which are opaque atomic values, implemented as integers under the hood. - It is the job of the extractor to create entity values for all AST nodes and populate database relations that encode the relationship between AST nodes and any values associated with them. These relations are *extensional*, that is, explicitly stored in the database, unlike the relations described by QL predicates, which we also refer to as *intensional* relations. Entity values belong to *entity types*, whose name starts with “@” to set them apart from primitive types and classes. + It is the job of the extractor to create entity values for all AST nodes and populate database relations that encode the relationship between AST nodes and any values associated with them. These relations are *extensional*, that is, explicitly stored in the database, unlike the relations described by predicates, which we also refer to as *intensional* relations. Entity values belong to *entity types*, whose name starts with “@” to set them apart from primitive types and classes. The interface between entity types and extensional relations on the one hand and QL predicates and classes on the other hand is provided by the *database schema*, which defines the available entity types and the schema of each extensional relation, that is, how many columns the relation has, and which entity type or primitive type the values in each column come from. QL programs can refer to entity types and extensional relations just as they would refer to QL classes and predicates, with the restriction that entity types cannot be directly selected in a ``select`` clause, since they do not have a well-defined string representation. diff --git a/docs/language/ql-training/slide-snippets/info.rst b/docs/language/ql-training/slide-snippets/info.rst deleted file mode 100644 index 583a456207b..00000000000 --- a/docs/language/ql-training/slide-snippets/info.rst +++ /dev/null @@ -1,33 +0,0 @@ -Information -=========== - -To try the examples in this presentation we recommend you download `QL for Eclipse `__. - -**QL language resources** - -- If you are new to QL, try the QL language tutorials at `Learning CodeQL `__. -- To learn more about the main features of QL, try looking at the `QL language handbook `__. -- For further information about writing queries in QL, see `Writing QL queries `__. - -**QL queries** - -The QL queries included in the latest Semmle release are open source. View them in the `semmle/ql repository `__. - -**Extra information** - -.. |arrow-l| unicode:: U+2190 - -.. |arrow-r| unicode:: U+2192 - -- Press |arrow-l| and |arrow-r| to navigate between slides -- Pressing **p** toggles between the slide and any extra notes (where they're available) -- Pressing **f** toggles full screen viewing on/off - -.. note:: - - To run the queries featured in this training presentation, we recommend you download the free-to-use `QL for Eclipse plugin `__. - - This plugin allows you to locally access the latest features of QL, including the standard QL libraries and queries. It also provides standard IDE features such as syntax highlighting, jump-to-definition, and tab completion. - - When you have setup QL for Eclipse we recommend increasing the “Memory for running queries” from the default setting of 4096MB to 8192MB, to ensure that all the queries complete quickly. - \ No newline at end of file diff --git a/docs/language/ql-training/slide-snippets/intro-ql-general.rst b/docs/language/ql-training/slide-snippets/intro-ql-general.rst index f0c00c131fc..6fa68c7243e 100644 --- a/docs/language/ql-training/slide-snippets/intro-ql-general.rst +++ b/docs/language/ql-training/slide-snippets/intro-ql-general.rst @@ -67,9 +67,9 @@ Complete text of the analysis (nothing left out!): .. note:: - Once the mission critical bug was discovered on Curiosity, JPL contacted Semmle for help discovering whether variants of the problem might exist elsewhere in the Curiosity control software. In 20 minutes, research engineers from Semmle produced a QL query and shared it with the JPL team. It finds all functions that are passed an array as an argument whose size is smaller than expected. + Once the mission critical bug was discovered on Curiosity, JPL contacted Semmle for help discovering whether variants of the problem might exist elsewhere in the Curiosity control software. In 20 minutes, research engineers from Semmle produced a CodeQL query and shared it with the JPL team. It finds all functions that are passed an array as an argument whose size is smaller than expected. - (The goal here is not to fully understand the QL, but to illustrate the power of the language and its standard libraries.) + (The goal here is not to fully understand the query, but to illustrate the power of the language and its standard libraries.) Find all instances! @@ -105,16 +105,16 @@ Analysis overview Semmle’s analysis works by extracting a queryable database from your project. For compiled languages, Semmle’s tools observe an ordinary build of the source code. Each time a compiler is invoked to process a source file, a copy of that file is made, and all relevant information about the source code (syntactic data about the abstract syntax tree, semantic data like name binding and type information, data on the operation of the C preprocessor, etc.) is collected. For interpreted languages, the extractor gathers similar information by running directly on the source code. Multi-language code bases are analyzed one language at a time. - Once the extraction finishes, all this information is collected into a single `snapshot database `__, which is then ready to query, possibly on a different machine. A copy of the source files, made at the time the database was created, is also included in the snapshot so analysis results can be displayed at the correct location in the code. The database schema is (source) language specific. + Once the extraction finishes, all this information is collected into a single `CodeQL database `__, which is then ready to query, possibly on a different machine. A copy of the source files, made at the time the database was created, is also included in the snapshot so analysis results can be displayed at the correct location in the code. The database schema is (source) language specific. - Queries are written in `QL `__ and usually depend on one or more of the `standard QL libraries `__ (and of course you can write your own custom libraries). They are compiled into an efficiently executable format by the QL compiler and then run on a snapshot database by the QL evaluator, either on a remote worker machine or locally on a developer’s machine. + Queries are written in `QL `__ and usually depend on one or more of the `standard CodeQL libraries `__ (and of course you can write your own custom libraries). They are compiled into an efficiently executable format by the QL compiler and then run on a CodeQL database by the QL evaluator, either on a remote worker machine or locally on a developer’s machine. Query results can be interpreted and presented in a variety of ways, including displaying them in an `IDE plugin `__ such as QL for Eclipse, or in a web dashboard as on `LGTM `__. Introducing QL ============== -QL is the query language running all Semmle analysis. +QL is the query language running all CodeQL analysis. QL is: @@ -126,12 +126,12 @@ QL is: .. note:: - QL is the high-level, object-oriented logic language that underpins all of Semmle’s libraries and analyses. You can learn lots more about QL by visiting `Introduction to the QL language `__ and `About QL `__. + QL is the high-level, object-oriented logic language that underpins all CodeQL libraries and analyses. You can learn lots more about QL by visiting `Introduction to the QL language `__ and `About QL `__. The key features of QL are: - All common logic connectives are available, including quantifiers like ``exist``, which can also introduce new variables. - The language is declarative–the user focuses on stating what they would like to find, and leaves the details of how to evaluate the query to the engine. - - The object-oriented layer allows Semmle to distribute rich standard libraries for program analysis. These model the common AST node types, control flow and name lookup, and define further layers on top–for example control flow or data flow analysis. The `standard QL libraries and queries `__ ship as source and can be inspected by the user, and new abstractions are readily defined. + - The object-oriented layer allows Semmle to distribute rich standard libraries for program analysis. These model the common AST node types, control flow and name lookup, and define further layers on top–for example control flow or data flow analysis. The `standard CodeQL libraries and queries `__ ship as source and can be inspected by the user, and new abstractions are readily defined. - The database generated by Semmle’s tools is treated as read-only; queries cannot insert new data into it, though they can inspect its contents in various ways. - You can start writing QL and running QL queries on open source projects in the `query console `__ on LGTM.com. You can also download snapshots from LGTM.com to query locally, by `running queries in your IDE `__ equipped with a QL plugin or extension. + You can start writing running queries on open source projects in the `query console `__ on LGTM.com. You can also download snapshots from LGTM.com to query locally, by `running queries in your IDE `__. From 69d7baa9bcfc4ac4ce72459e92bc66436197859c Mon Sep 17 00:00:00 2001 From: james Date: Tue, 5 Nov 2019 15:35:18 +0000 Subject: [PATCH 137/148] docs: update snapshot terminology --- docs/language/ql-training/cpp/bad-overflow-guard.rst | 8 ++++---- docs/language/ql-training/cpp/control-flow-cpp.rst | 6 +++--- docs/language/ql-training/cpp/data-flow-cpp.rst | 6 +++--- docs/language/ql-training/cpp/global-data-flow-cpp.rst | 6 +++--- docs/language/ql-training/cpp/intro-ql-cpp.rst | 6 +++--- .../ql-training/cpp/program-representation-cpp.rst | 4 ++-- docs/language/ql-training/cpp/snprintf.rst | 6 +++--- docs/language/ql-training/java/apache-struts-java.rst | 6 +++--- docs/language/ql-training/java/data-flow-java.rst | 6 +++--- docs/language/ql-training/java/global-data-flow-java.rst | 6 +++--- docs/language/ql-training/java/intro-ql-java.rst | 6 +++--- docs/language/ql-training/java/query-injection-java.rst | 6 +++--- .../ql-training/slide-snippets/abstract-syntax-tree.rst | 2 +- .../{snapshot-note.rst => database-note.rst} | 2 +- .../ql-training/slide-snippets/global-data-flow.rst | 6 +++--- .../ql-training/slide-snippets/intro-ql-general.rst | 6 +++--- .../ql-training/slide-snippets/local-data-flow.rst | 4 ++-- 17 files changed, 46 insertions(+), 46 deletions(-) rename docs/language/ql-training/slide-snippets/{snapshot-note.rst => database-note.rst} (66%) diff --git a/docs/language/ql-training/cpp/bad-overflow-guard.rst b/docs/language/ql-training/cpp/bad-overflow-guard.rst index 5e9fbcfc700..6937e9a1a4f 100644 --- a/docs/language/ql-training/cpp/bad-overflow-guard.rst +++ b/docs/language/ql-training/cpp/bad-overflow-guard.rst @@ -16,7 +16,7 @@ Setup For this example you should download: - `QL for Eclipse `__ -- `ChakraCore snapshot `__ +- `ChakraCore database `__ .. note:: @@ -24,9 +24,9 @@ For this example you should download: You can query the project in `the query console `__ on LGTM.com. - .. insert snapshot-note.rst to explain differences between snapshot available to download and the version available in the query console. + .. insert database-note.rst to explain differences between database available to download and the version available in the query console. - .. include:: ../slide-snippets/snapshot-note.rst + .. include:: ../slide-snippets/database-note.rst .. resume slides @@ -233,4 +233,4 @@ The final query .. literalinclude:: ../query-examples/cpp/bad-overflow-guard-3.ql :language: ql -This query finds a single result in our historic snapshot, which was `a genuine bug in ChakraCore `__. +This query finds a single result in our historic database, which was `a genuine bug in ChakraCore `__. diff --git a/docs/language/ql-training/cpp/control-flow-cpp.rst b/docs/language/ql-training/cpp/control-flow-cpp.rst index a8171a44b5d..4e6ebbf4292 100644 --- a/docs/language/ql-training/cpp/control-flow-cpp.rst +++ b/docs/language/ql-training/cpp/control-flow-cpp.rst @@ -18,7 +18,7 @@ Setup For this example you should download: - `QL for Eclipse `__ -- `ChakraCore snapshot `__ +- `ChakraCore database `__ .. note:: @@ -26,9 +26,9 @@ For this example you should download: You can query the project in `the query console `__ on LGTM.com. - .. insert snapshot-note.rst to explain differences between snapshot available to download and the version available in the query console. + .. insert database-note.rst to explain differences between database available to download and the version available in the query console. - .. include:: ../slide-snippets/snapshot-note.rst + .. include:: ../slide-snippets/database-note.rst .. resume slides diff --git a/docs/language/ql-training/cpp/data-flow-cpp.rst b/docs/language/ql-training/cpp/data-flow-cpp.rst index 5377d8d52d1..cb7ed7b5133 100644 --- a/docs/language/ql-training/cpp/data-flow-cpp.rst +++ b/docs/language/ql-training/cpp/data-flow-cpp.rst @@ -16,7 +16,7 @@ Setup For this example you should download: - `QL for Eclipse `__ -- `dotnet/coreclr snapshot `__ +- `dotnet/coreclr database `__ .. note:: @@ -24,9 +24,9 @@ For this example you should download: You can query the project in `the query console `__ on LGTM.com. - .. insert snapshot-note.rst to explain differences between snapshot available to download and the version available in the query console. + .. insert database-note.rst to explain differences between database available to download and the version available in the query console. - .. include:: ../slide-snippets/snapshot-note.rst + .. include:: ../slide-snippets/database-note.rst .. resume slides diff --git a/docs/language/ql-training/cpp/global-data-flow-cpp.rst b/docs/language/ql-training/cpp/global-data-flow-cpp.rst index bc048ac59f4..5a5e93aafda 100644 --- a/docs/language/ql-training/cpp/global-data-flow-cpp.rst +++ b/docs/language/ql-training/cpp/global-data-flow-cpp.rst @@ -16,7 +16,7 @@ Setup For this example you should download: - `QL for Eclipse `__ -- `dotnet/coreclr snapshot `__ +- `dotnet/coreclr database `__ .. note:: @@ -24,9 +24,9 @@ For this example you should download: You can query the project in `the query console `__ on LGTM.com. - .. insert snapshot-note.rst to explain differences between snapshot available to download and the version available in the query console. + .. insert database-note.rst to explain differences between database available to download and the version available in the query console. - .. include:: ../slide-snippets/snapshot-note.rst + .. include:: ../slide-snippets/database-note.rst .. resume slides diff --git a/docs/language/ql-training/cpp/intro-ql-cpp.rst b/docs/language/ql-training/cpp/intro-ql-cpp.rst index dddc86ca77f..e7d4b318b25 100644 --- a/docs/language/ql-training/cpp/intro-ql-cpp.rst +++ b/docs/language/ql-training/cpp/intro-ql-cpp.rst @@ -16,7 +16,7 @@ Setup For this example you should download: - `QL for Eclipse `__ -- `exiv2 snapshot `__ +- `exiv2 database `__ .. note:: @@ -24,9 +24,9 @@ For this example you should download: You can also query the project in `the query console `__ on LGTM.com. - .. insert snapshot-note.rst to explain differences between snapshot available to download and the version available in the query console. + .. insert database-note.rst to explain differences between database available to download and the version available in the query console. - .. include:: ../slide-snippets/snapshot-note.rst + .. include:: ../slide-snippets/database-note.rst .. resume slides diff --git a/docs/language/ql-training/cpp/program-representation-cpp.rst b/docs/language/ql-training/cpp/program-representation-cpp.rst index b21677fccab..077b36520f6 100644 --- a/docs/language/ql-training/cpp/program-representation-cpp.rst +++ b/docs/language/ql-training/cpp/program-representation-cpp.rst @@ -107,7 +107,7 @@ Working with macros #define square(x) x*x y = square(y0), z = square(z0) -is represented in the snapshot database as: +is represented in the CodeQL database database as: - A Macro entity representing the text of the *head* and *body* of the macro - Assignment nodes, representing the two assignments after preprocessing @@ -121,4 +121,4 @@ Useful predicates on ``Element``: ``isInMacroExpansion()``, ``isAffectedByMacro( .. note:: - The snapshot also contains information about macro definitions, which are represented by class ``Macro``. These macro definitions are related to the AST nodes resulting from their uses by the class ``MacroAccess``. \ No newline at end of file + The CodeQL database also contains information about macro definitions, which are represented by class ``Macro``. These macro definitions are related to the AST nodes resulting from their uses by the class ``MacroAccess``. \ No newline at end of file diff --git a/docs/language/ql-training/cpp/snprintf.rst b/docs/language/ql-training/cpp/snprintf.rst index dafebc4a720..26c54708c0a 100644 --- a/docs/language/ql-training/cpp/snprintf.rst +++ b/docs/language/ql-training/cpp/snprintf.rst @@ -16,7 +16,7 @@ Setup For this example you should download: - `QL for Eclipse `__ -- `rsyslog snapshot `__ +- `rsyslog database `__ .. note:: @@ -24,9 +24,9 @@ For this example you should download: You can also query the project in `the query console `__ on LGTM.com. - .. insert snapshot-note.rst to explain differences between snapshot available to download and the version available in the query console. + .. insert database-note.rst to explain differences between database available to download and the version available in the query console. - .. include:: ../slide-snippets/snapshot-note.rst + .. include:: ../slide-snippets/database-note.rst .. resume slides diff --git a/docs/language/ql-training/java/apache-struts-java.rst b/docs/language/ql-training/java/apache-struts-java.rst index 5c27ec73979..73d0122b7ea 100644 --- a/docs/language/ql-training/java/apache-struts-java.rst +++ b/docs/language/ql-training/java/apache-struts-java.rst @@ -20,7 +20,7 @@ Setup For this example you should download: - `QL for Eclipse `__ -- `Apache Struts snapshot `__ +- `Apache Struts database `__ .. note:: @@ -28,9 +28,9 @@ For this example you should download: You can also query the project in `the query console `__ on LGTM.com. - .. insert snapshot-note.rst to explain differences between snapshot available to download and the version available in the query console. + .. insert database-note.rst to explain differences between database available to download and the version available in the query console. - .. include:: ../slide-snippets/snapshot-note.rst + .. include:: ../slide-snippets/database-note.rst .. resume slides diff --git a/docs/language/ql-training/java/data-flow-java.rst b/docs/language/ql-training/java/data-flow-java.rst index be9ba98456e..0b9026aadfb 100644 --- a/docs/language/ql-training/java/data-flow-java.rst +++ b/docs/language/ql-training/java/data-flow-java.rst @@ -16,7 +16,7 @@ Setup For this example you should download: - `QL for Eclipse `__ -- `VIVO Vitro snapshot `__ +- `VIVO Vitro database `__ .. note:: @@ -24,9 +24,9 @@ For this example you should download: You can also query the project in `the query console `__ on LGTM.com. - .. insert snapshot-note.rst to explain differences between snapshot available to download and the version available in the query console. + .. insert database-note.rst to explain differences between database available to download and the version available in the query console. - .. include:: ../slide-snippets/snapshot-note.rst + .. include:: ../slide-snippets/database-note.rst .. resume slides diff --git a/docs/language/ql-training/java/global-data-flow-java.rst b/docs/language/ql-training/java/global-data-flow-java.rst index f43c31e0f4f..062ce30b04c 100644 --- a/docs/language/ql-training/java/global-data-flow-java.rst +++ b/docs/language/ql-training/java/global-data-flow-java.rst @@ -16,7 +16,7 @@ Setup For this example you should download: - `QL for Eclipse `__ -- `Apache Struts snapshot `__ +- `Apache Struts database `__ .. note:: @@ -24,9 +24,9 @@ For this example you should download: You can also query the project in `the query console `__ on LGTM.com. - .. insert snapshot-note.rst to explain differences between snapshot available to download and the version available in the query console. + .. insert database-note.rst to explain differences between database available to download and the version available in the query console. - .. include:: ../slide-snippets/snapshot-note.rst + .. include:: ../slide-snippets/database-note.rst .. resume slides diff --git a/docs/language/ql-training/java/intro-ql-java.rst b/docs/language/ql-training/java/intro-ql-java.rst index f80f878492d..393e35cf5a8 100644 --- a/docs/language/ql-training/java/intro-ql-java.rst +++ b/docs/language/ql-training/java/intro-ql-java.rst @@ -16,7 +16,7 @@ Setup For this example you should download: - `QL for Eclipse `__ -- `Apache Struts snapshot `__ +- `Apache Struts database `__ .. note:: @@ -24,9 +24,9 @@ For this example you should download: You can also query the project in `the query console `__ on LGTM.com. - .. insert snapshot-note.rst to explain differences between snapshot available to download and the version available in the query console. + .. insert database-note.rst to explain differences between database available to download and the version available in the query console. - .. include:: ../slide-snippets/snapshot-note.rst + .. include:: ../slide-snippets/database-note.rst .. resume slides diff --git a/docs/language/ql-training/java/query-injection-java.rst b/docs/language/ql-training/java/query-injection-java.rst index b3ef7d7e182..dc345d3b39c 100644 --- a/docs/language/ql-training/java/query-injection-java.rst +++ b/docs/language/ql-training/java/query-injection-java.rst @@ -16,7 +16,7 @@ Setup For this example you should download: - `QL for Eclipse `__ -- `VIVO Vitro snapshot `__ +- `VIVO Vitro database `__ .. note:: @@ -24,9 +24,9 @@ For this example you should download: You can also query the project in `the query console `__ on LGTM.com. - .. insert snapshot-note.rst to explain differences between snapshot available to download and the version available in the query console. + .. insert database-note.rst to explain differences between database available to download and the version available in the query console. - .. include:: ../slide-snippets/snapshot-note.rst + .. include:: ../slide-snippets/database-note.rst .. resume slides diff --git a/docs/language/ql-training/slide-snippets/abstract-syntax-tree.rst b/docs/language/ql-training/slide-snippets/abstract-syntax-tree.rst index 6a26b5c74a4..aaa2cd23d71 100644 --- a/docs/language/ql-training/slide-snippets/abstract-syntax-tree.rst +++ b/docs/language/ql-training/slide-snippets/abstract-syntax-tree.rst @@ -63,7 +63,7 @@ Entity types are rarely used directly, the usual pattern is to define a class th The interface between entity types and extensional relations on the one hand and QL predicates and classes on the other hand is provided by the *database schema*, which defines the available entity types and the schema of each extensional relation, that is, how many columns the relation has, and which entity type or primitive type the values in each column come from. QL programs can refer to entity types and extensional relations just as they would refer to QL classes and predicates, with the restriction that entity types cannot be directly selected in a ``select`` clause, since they do not have a well-defined string representation. - For example, the database schemas for C/++, C#, and Java snapshot databases are here: + For example, the database schemas for C/++, C#, and Java CodeQL databases are here: - https://github.com/Semmle/ql/blob/master/cpp/ql/src/semmlecode.cpp.dbscheme - https://github.com/Semmle/ql/blob/master/csharp/ql/src/semmlecode.csharp.dbscheme diff --git a/docs/language/ql-training/slide-snippets/snapshot-note.rst b/docs/language/ql-training/slide-snippets/database-note.rst similarity index 66% rename from docs/language/ql-training/slide-snippets/snapshot-note.rst rename to docs/language/ql-training/slide-snippets/database-note.rst index 4a32243211b..af6dce23728 100644 --- a/docs/language/ql-training/slide-snippets/snapshot-note.rst +++ b/docs/language/ql-training/slide-snippets/database-note.rst @@ -1 +1 @@ -Note that results generated in the query console are likely to differ to those generated in the QL plugin as LGTM.com analyzes the most recent revisions of each project that has been added–the snapshot available to download above is based on an historical version of the codebase. \ No newline at end of file +Note that results generated in the query console are likely to differ to those generated in the QL plugin as LGTM.com analyzes the most recent revisions of each project that has been added–the CodeQL database available to download above is based on an historical version of the codebase. \ No newline at end of file diff --git a/docs/language/ql-training/slide-snippets/global-data-flow.rst b/docs/language/ql-training/slide-snippets/global-data-flow.rst index f18dc22d178..6cb52af0f84 100644 --- a/docs/language/ql-training/slide-snippets/global-data-flow.rst +++ b/docs/language/ql-training/slide-snippets/global-data-flow.rst @@ -17,15 +17,15 @@ Global data flow and taint tracking - Recap: - - Local (“intra-procedural”) data flow models flow within one function; feasible to compute for all functions in a snapshot - - Global (“inter-procedural”) data flow models flow across function calls; not feasible to compute for all functions in a snapshot + - Local (“intra-procedural”) data flow models flow within one function; feasible to compute for all functions in a CodeQL database + - Global (“inter-procedural”) data flow models flow across function calls; not feasible to compute for all functions in a CodeQL database - For global data flow (and taint tracking), we must therefore provide restrictions to ensure the problem is tractable. - Typically, this involves specifying the *source* and *sink*. .. note:: - As we mentioned in the previous slide deck, while local data flow is feasible to compute for all functions in a snapshot, global data flow is not. This is because the number of paths becomes exponentially larger for global data flow. + As we mentioned in the previous slide deck, while local data flow is feasible to compute for all functions in a CodeQL database, global data flow is not. This is because the number of paths becomes exponentially larger for global data flow. The global data flow (and taint tracking) avoids this problem by requiring that the query author specifies which ``sources`` and ``sinks`` are applicable. This allows the implementation to compute paths between the restricted set of nodes, rather than the full graph. diff --git a/docs/language/ql-training/slide-snippets/intro-ql-general.rst b/docs/language/ql-training/slide-snippets/intro-ql-general.rst index 6fa68c7243e..d559c5bbeb7 100644 --- a/docs/language/ql-training/slide-snippets/intro-ql-general.rst +++ b/docs/language/ql-training/slide-snippets/intro-ql-general.rst @@ -105,7 +105,7 @@ Analysis overview Semmle’s analysis works by extracting a queryable database from your project. For compiled languages, Semmle’s tools observe an ordinary build of the source code. Each time a compiler is invoked to process a source file, a copy of that file is made, and all relevant information about the source code (syntactic data about the abstract syntax tree, semantic data like name binding and type information, data on the operation of the C preprocessor, etc.) is collected. For interpreted languages, the extractor gathers similar information by running directly on the source code. Multi-language code bases are analyzed one language at a time. - Once the extraction finishes, all this information is collected into a single `CodeQL database `__, which is then ready to query, possibly on a different machine. A copy of the source files, made at the time the database was created, is also included in the snapshot so analysis results can be displayed at the correct location in the code. The database schema is (source) language specific. + Once the extraction finishes, all this information is collected into a single `CodeQL database `__, which is then ready to query, possibly on a different machine. A copy of the source files, made at the time the database was created, is also included in the CodeQL database so analysis results can be displayed at the correct location in the code. The database schema is (source) language specific. Queries are written in `QL `__ and usually depend on one or more of the `standard CodeQL libraries `__ (and of course you can write your own custom libraries). They are compiled into an efficiently executable format by the QL compiler and then run on a CodeQL database by the QL evaluator, either on a remote worker machine or locally on a developer’s machine. @@ -121,7 +121,7 @@ QL is: - a **logic** language based on first-order logic - a **declarative** language without side effects - an **object-oriented** language -- a **query** language working on a read-only snapshot database +- a **query** language working on a read-only CodeQL database database - equipped with rich standard libraries **for program analysis** .. note:: @@ -134,4 +134,4 @@ QL is: - The object-oriented layer allows Semmle to distribute rich standard libraries for program analysis. These model the common AST node types, control flow and name lookup, and define further layers on top–for example control flow or data flow analysis. The `standard CodeQL libraries and queries `__ ship as source and can be inspected by the user, and new abstractions are readily defined. - The database generated by Semmle’s tools is treated as read-only; queries cannot insert new data into it, though they can inspect its contents in various ways. - You can start writing running queries on open source projects in the `query console `__ on LGTM.com. You can also download snapshots from LGTM.com to query locally, by `running queries in your IDE `__. + You can start writing running queries on open source projects in the `query console `__ on LGTM.com. You can also download CodeQL databases from LGTM.com to query locally, by `running queries in your IDE `__. diff --git a/docs/language/ql-training/slide-snippets/local-data-flow.rst b/docs/language/ql-training/slide-snippets/local-data-flow.rst index 8a0eeac6374..068dfc9170a 100644 --- a/docs/language/ql-training/slide-snippets/local-data-flow.rst +++ b/docs/language/ql-training/slide-snippets/local-data-flow.rst @@ -61,8 +61,8 @@ Data flow graphs Local vs global data flow ========================= -- Local (“intra-procedural”) data flow models flow within one function; feasible to compute for all functions in a snapshot -- Global (“inter-procedural”) data flow models flow across function calls; not feasible to compute for all functions in a snapshot +- Local (“intra-procedural”) data flow models flow within one function; feasible to compute for all functions in a CodeQL database +- Global (“inter-procedural”) data flow models flow across function calls; not feasible to compute for all functions in a CodeQL database - Different APIs, so discussed separately - This slide deck focuses on the former From 2d00ca5773b7773184b3beabbd3799876d3fc06b Mon Sep 17 00:00:00 2001 From: james Date: Fri, 4 Oct 2019 09:18:19 +0100 Subject: [PATCH 138/148] docs: semmle logo (cherry picked from commit 4a8e8fa0de4470aaeed94357f70b54b68c0b84a7) --- .../static/theme/css/default.css | 18 +-- .../_static-training/title-slide.svg | 149 +----------------- .../ql-training/cpp/bad-overflow-guard.rst | 4 - .../ql-training/cpp/control-flow-cpp.rst | 4 - .../ql-training/cpp/data-flow-cpp.rst | 4 - .../ql-training/cpp/global-data-flow-cpp.rst | 4 - .../language/ql-training/cpp/intro-ql-cpp.rst | 4 - .../cpp/program-representation-cpp.rst | 4 - docs/language/ql-training/cpp/snprintf.rst | 4 - docs/language/ql-training/index.rst | 4 - .../ql-training/java/apache-struts-java.rst | 4 - .../ql-training/java/data-flow-java.rst | 4 - .../java/global-data-flow-java.rst | 4 - .../ql-training/java/intro-ql-java.rst | 4 - .../java/program-representation-java.rst | 4 - .../ql-training/java/query-injection-java.rst | 4 - docs/language/ql-training/template.rst | 4 - 17 files changed, 4 insertions(+), 223 deletions(-) diff --git a/docs/language/ql-training/_static-training/slides-semmle-2/static/theme/css/default.css b/docs/language/ql-training/_static-training/slides-semmle-2/static/theme/css/default.css index 2c05ea5e93b..4f706120747 100644 --- a/docs/language/ql-training/_static-training/slides-semmle-2/static/theme/css/default.css +++ b/docs/language/ql-training/_static-training/slides-semmle-2/static/theme/css/default.css @@ -1300,13 +1300,13 @@ aside.gdbar img { .title-slide hgroup h1 { font-size: 2em; line-height: 1.4; - /*letter-spacing: -3px;*/ color: white; margin: auto; display: block; position: absolute; top: 0; bottom: 10%; + left: 1.25em; height: 0; } /* line 898, ../scss/default.scss */ @@ -1430,31 +1430,19 @@ hgroup .pre { color: #5c31ff; } -/* title slide (deck title, subtitle, semmle logo)*/ +/* title slide (deck title, subtitle)*/ .title-slide { background-image: url("../../title-slide.svg"); background-size: cover; } -.semmle-logo sup { - vertical-align: super; - font-size: 0.3em; - font-weight: 100; -} - -.title-slide .semmle-logo { - color: white; - font-size: 1.2em; - position: absolute; - top: 10%; -} - .title-slide p { color: white; font-size: 1em; position: absolute; bottom: 30%; + left: 2.6em; } .title-slide hgroup .pre { diff --git a/docs/language/ql-training/_static-training/title-slide.svg b/docs/language/ql-training/_static-training/title-slide.svg index 6f9a19f4a1b..13eb2d34fef 100644 --- a/docs/language/ql-training/_static-training/title-slide.svg +++ b/docs/language/ql-training/_static-training/title-slide.svg @@ -1,148 +1 @@ - - - - - - image/svg+xml - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + \ No newline at end of file diff --git a/docs/language/ql-training/cpp/bad-overflow-guard.rst b/docs/language/ql-training/cpp/bad-overflow-guard.rst index 6937e9a1a4f..dceeec3320f 100644 --- a/docs/language/ql-training/cpp/bad-overflow-guard.rst +++ b/docs/language/ql-training/cpp/bad-overflow-guard.rst @@ -4,10 +4,6 @@ Example: Bad overflow guard CodeQL for C/C++ -.. container:: semmle-logo - - Semmle :sup:`TM` - .. rst-class:: setup Setup diff --git a/docs/language/ql-training/cpp/control-flow-cpp.rst b/docs/language/ql-training/cpp/control-flow-cpp.rst index 4e6ebbf4292..3fddc3f8072 100644 --- a/docs/language/ql-training/cpp/control-flow-cpp.rst +++ b/docs/language/ql-training/cpp/control-flow-cpp.rst @@ -4,10 +4,6 @@ Analyzing control flow CodeQL for C/C++ -.. container:: semmle-logo - - Semmle :sup:`TM` - .. Include information slides here .. rst-class:: setup diff --git a/docs/language/ql-training/cpp/data-flow-cpp.rst b/docs/language/ql-training/cpp/data-flow-cpp.rst index cb7ed7b5133..e78e8e86220 100644 --- a/docs/language/ql-training/cpp/data-flow-cpp.rst +++ b/docs/language/ql-training/cpp/data-flow-cpp.rst @@ -4,10 +4,6 @@ Introduction to data flow Finding string formatting vulnerabilities in C/C++ -.. container:: semmle-logo - - Semmle :sup:`TM` - .. rst-class:: setup Setup diff --git a/docs/language/ql-training/cpp/global-data-flow-cpp.rst b/docs/language/ql-training/cpp/global-data-flow-cpp.rst index 5a5e93aafda..aaa567e7a8e 100644 --- a/docs/language/ql-training/cpp/global-data-flow-cpp.rst +++ b/docs/language/ql-training/cpp/global-data-flow-cpp.rst @@ -3,10 +3,6 @@ Introduction to global data flow ================================ CodeQL for C/C++ - -.. container:: semmle-logo - - Semmle :sup:`TM` .. rst-class:: setup diff --git a/docs/language/ql-training/cpp/intro-ql-cpp.rst b/docs/language/ql-training/cpp/intro-ql-cpp.rst index e7d4b318b25..cbe29e6c520 100644 --- a/docs/language/ql-training/cpp/intro-ql-cpp.rst +++ b/docs/language/ql-training/cpp/intro-ql-cpp.rst @@ -4,10 +4,6 @@ Introduction to variant analysis CodeQL for C/C++ -.. container:: semmle-logo - - Semmle :sup:`TM` - .. rst-class:: setup Setup diff --git a/docs/language/ql-training/cpp/program-representation-cpp.rst b/docs/language/ql-training/cpp/program-representation-cpp.rst index 077b36520f6..3c062089428 100644 --- a/docs/language/ql-training/cpp/program-representation-cpp.rst +++ b/docs/language/ql-training/cpp/program-representation-cpp.rst @@ -4,10 +4,6 @@ Program representation CodeQL for C/C++ -.. container:: semmle-logo - - Semmle :sup:`TM` - .. rst-class:: agenda Agenda diff --git a/docs/language/ql-training/cpp/snprintf.rst b/docs/language/ql-training/cpp/snprintf.rst index 26c54708c0a..c15bf74d286 100644 --- a/docs/language/ql-training/cpp/snprintf.rst +++ b/docs/language/ql-training/cpp/snprintf.rst @@ -4,10 +4,6 @@ Exercise: ``snprintf`` overflow CodeQL for C/C++ -.. container:: semmle-logo - - Semmle :sup:`TM` - .. rst-class:: setup Setup diff --git a/docs/language/ql-training/index.rst b/docs/language/ql-training/index.rst index 309fe92c052..021bb58d3c4 100644 --- a/docs/language/ql-training/index.rst +++ b/docs/language/ql-training/index.rst @@ -1,10 +1,6 @@ CodeQL training and variant analysis examples ============================================= -.. container:: semmle-logo - - Semmle :sup:`TM` - .. toctree:: :glob: :maxdepth: 1 diff --git a/docs/language/ql-training/java/apache-struts-java.rst b/docs/language/ql-training/java/apache-struts-java.rst index 73d0122b7ea..be071d41562 100644 --- a/docs/language/ql-training/java/apache-struts-java.rst +++ b/docs/language/ql-training/java/apache-struts-java.rst @@ -8,10 +8,6 @@ Exercise: Apache Struts CVE-2017-9805 -.. container:: semmle-logo - - Semmle :sup:`TM` - .. rst-class:: setup Setup diff --git a/docs/language/ql-training/java/data-flow-java.rst b/docs/language/ql-training/java/data-flow-java.rst index 0b9026aadfb..93ccc87b7d1 100644 --- a/docs/language/ql-training/java/data-flow-java.rst +++ b/docs/language/ql-training/java/data-flow-java.rst @@ -2,10 +2,6 @@ Introduction to data flow ========================= -.. container:: semmle-logo - - Semmle :sup:`TM` - Finding SPARQL injection vulnerabilities in Java .. rst-class:: setup diff --git a/docs/language/ql-training/java/global-data-flow-java.rst b/docs/language/ql-training/java/global-data-flow-java.rst index 062ce30b04c..9edacde93c3 100644 --- a/docs/language/ql-training/java/global-data-flow-java.rst +++ b/docs/language/ql-training/java/global-data-flow-java.rst @@ -4,10 +4,6 @@ Introduction to global data flow CodeQL for Java -.. container:: semmle-logo - - Semmle :sup:`TM` - .. rst-class:: setup Setup diff --git a/docs/language/ql-training/java/intro-ql-java.rst b/docs/language/ql-training/java/intro-ql-java.rst index 393e35cf5a8..f7afe713c4a 100644 --- a/docs/language/ql-training/java/intro-ql-java.rst +++ b/docs/language/ql-training/java/intro-ql-java.rst @@ -4,10 +4,6 @@ Introduction to variant analysis CodeQL for Java -.. container:: semmle-logo - - Semmle :sup:`TM` - .. rst-class:: setup Setup diff --git a/docs/language/ql-training/java/program-representation-java.rst b/docs/language/ql-training/java/program-representation-java.rst index ef078b53562..ba9b1a9d80d 100644 --- a/docs/language/ql-training/java/program-representation-java.rst +++ b/docs/language/ql-training/java/program-representation-java.rst @@ -4,10 +4,6 @@ Program representation CodeQL for Java -.. container:: semmle-logo - - Semmle :sup:`TM` - .. rst-class:: agenda Agenda diff --git a/docs/language/ql-training/java/query-injection-java.rst b/docs/language/ql-training/java/query-injection-java.rst index dc345d3b39c..a5b46eabda5 100644 --- a/docs/language/ql-training/java/query-injection-java.rst +++ b/docs/language/ql-training/java/query-injection-java.rst @@ -4,10 +4,6 @@ Example: Query injection CodeQL for Java -.. container:: semmle-logo - - Semmle :sup:`TM` - .. rst-class:: setup Setup diff --git a/docs/language/ql-training/template.rst b/docs/language/ql-training/template.rst index 0cce4a11435..21c5abef144 100644 --- a/docs/language/ql-training/template.rst +++ b/docs/language/ql-training/template.rst @@ -27,10 +27,6 @@ Template slide deck Second subheading -.. container:: semmle-logo - - Semmle :sup:`TM` - .. Set up slide. Include link to QL4E snapshots required for examples .. rst-class:: setup From 0b6592f650421c5b5ff001674042577e038367bb Mon Sep 17 00:00:00 2001 From: james Date: Mon, 30 Sep 2019 11:51:17 +0100 Subject: [PATCH 139/148] docs: version number and small css tweaks (cherry picked from commit ff78feeeeaec99bdf779242c33e25f57c7bcab29) --- .../slides-semmle-2/static/theme/css/default.css | 6 ++++-- docs/language/ql-training/conf.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/language/ql-training/_static-training/slides-semmle-2/static/theme/css/default.css b/docs/language/ql-training/_static-training/slides-semmle-2/static/theme/css/default.css index 4f706120747..3f8a89c6555 100644 --- a/docs/language/ql-training/_static-training/slides-semmle-2/static/theme/css/default.css +++ b/docs/language/ql-training/_static-training/slides-semmle-2/static/theme/css/default.css @@ -485,6 +485,7 @@ ul { margin-left: 2.2em; margin-bottom: 1em; position: relative; + width: 90%; } /* line 300, ../scss/default.scss */ ul li { @@ -1452,6 +1453,7 @@ hgroup .pre { .subheading { position: absolute; top: 62.5%; + left: 0; } .subheading p { @@ -1557,7 +1559,7 @@ p.first.admonition-title { text-align: left; font-size: 0.8em; width: 100%; - overflow: scroll; + overflow: auto; border: 1px solid black; } @@ -1596,7 +1598,7 @@ p.first.admonition-title { display: block; position: fixed; top: 0; - right: -1%; + right: 0; font-size: 1.2em; } diff --git a/docs/language/ql-training/conf.py b/docs/language/ql-training/conf.py index de712cd06a4..8f8a13569b5 100644 --- a/docs/language/ql-training/conf.py +++ b/docs/language/ql-training/conf.py @@ -86,9 +86,9 @@ htmlhelp_basename = 'QL training' # built documents. # # The short X.Y version. -version = u'1.21' +version = u'1.22' # The full version, including alpha/beta/rc tags. -release = u'1.21' +release = u'1.22' copyright = u'2019 Semmle Ltd' author = u'Semmle Ltd' From 488ce158887ed174a854730d81dd353c962a5c1d Mon Sep 17 00:00:00 2001 From: James Fletcher <42464962+jf205@users.noreply.github.com> Date: Tue, 5 Nov 2019 19:57:24 +0000 Subject: [PATCH 140/148] Apply suggestions from code review Co-Authored-By: shati-patel <42641846+shati-patel@users.noreply.github.com> Co-Authored-By: Felicity Chapman --- docs/language/learn-ql/ql-training.rst | 12 ++++++------ docs/language/ql-training/cpp/data-flow-cpp.rst | 4 ++-- docs/language/ql-training/cpp/intro-ql-cpp.rst | 2 +- .../ql-training/cpp/program-representation-cpp.rst | 6 +++--- docs/language/ql-training/java/intro-ql-java.rst | 2 +- .../ql-training/slide-snippets/intro-ql-general.rst | 2 +- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/language/learn-ql/ql-training.rst b/docs/language/learn-ql/ql-training.rst index 757de7daed6..4187ae621b5 100644 --- a/docs/language/learn-ql/ql-training.rst +++ b/docs/language/learn-ql/ql-training.rst @@ -1,8 +1,8 @@ CodeQL training and variant analysis examples ============================================= -QL and variant analysis ------------------------ +CodeQL and variant analysis +--------------------------- `Variant analysis `__ is the process of using a known vulnerability as a seed to find similar problems in your code. Security engineers typically perform variant analysis to identify possible vulnerabilities and to ensure that these threats are properly fixed across multiple code bases. @@ -10,8 +10,8 @@ QL and variant analysis CodeQL is easy to learn, and exploring code using CodeQL is the most efficient way to perform variant analysis. -Learning QL for variant analysis --------------------------------- +Learning CodeQL for variant analysis +------------------------------------ Start learning how to use CodeQL in variant analysis for a specific language by looking at the topics below. Each topic links to a short presentation on CodeQL, its libraries, or an example variant discovered using CodeQL. @@ -45,7 +45,7 @@ CodeQL and variant analysis for C/C++ - `Introduction to local data flow <../ql-training/cpp/data-flow-cpp.html>`__–an introduction to analyzing local data flow in C/C++ using CodeQL, including an example demonstrating how to develop a query to find a real CVE. - `Exercise: snprintf overflow <../ql-training/cpp/snprintf.html>`__–an example demonstrating how to develop a data flow query. - `Introduction to global data flow <../ql-training/cpp/global-data-flow-cpp.html>`__–an introduction to analyzing global data flow in C/C++ using CodeQL. -- `Analyzing control flow: CodeQL for C/C++ <../ql-training/cpp/control-flow-cpp.html>`__–an introduction to analyzing control flow in C/C++ using QL. +- `Analyzing control flow: CodeQL for C/C++ <../ql-training/cpp/control-flow-cpp.html>`__–an introduction to analyzing control flow in C/C++ using CodeQL. CodeQL and variant analysis for Java ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -61,6 +61,6 @@ More resources ~~~~~~~~~~~~~~ - If you are completely new to CodeQL, look at our introductory topics in :doc:`Learning CodeQL `. -- To find more detailed information about how to write CodeQL queries for specific languages, visit the links in :ref:`Writing CodeQL queries `. +- To find more detailed information about how to write queries for specific languages, visit the links in :ref:`Writing CodeQL queries `. - To read more about how CodeQL queries have been used in Semmle's security research, and to read about new CodeQL developments, visit the `Semmle blog `__. - Find more examples of queries written by Semmle's own security researchers in the `Semmle Demos repository `__ on GitHub. diff --git a/docs/language/ql-training/cpp/data-flow-cpp.rst b/docs/language/ql-training/cpp/data-flow-cpp.rst index e78e8e86220..32a3dfa233b 100644 --- a/docs/language/ql-training/cpp/data-flow-cpp.rst +++ b/docs/language/ql-training/cpp/data-flow-cpp.rst @@ -82,9 +82,9 @@ Write a query that flags ``printf`` calls where the format argument is not a ``S .. note:: - This first query is about finding places where the format specifier is not a constant string. In CodeQL for C/C++, constant strings are modeled as ``StringLiteral`` nodes, so we are looking for calls to format functions where the format specifier argument is not a string literal. + This first query is about finding places where the format specifier is not a constant string. In the CodeQL libraries for C/C++, constant strings are modeled as ``StringLiteral`` nodes, so we are looking for calls to format functions where the format specifier argument is not a string literal. - The `C/C++ standard libraries `__ include many different formatting functions that may be vulnerable to this particular attack–including ``printf``, ``snprintf``, and others. Furthermore, each of these different formatting functions may include the format string in a different position in the argument list. Instead of laboriously listing all these different variants, we can make use of the CodeQL for C/C++ standard library class ``FormattingFunction``, which provides an interface that models common formatting functions in C/C++. + The `C/C++ standard libraries `__ include many different formatting functions that may be vulnerable to this particular attack–including ``printf``, ``snprintf``, and others. Furthermore, each of these different formatting functions may include the format string in a different position in the argument list. Instead of laboriously listing all these different variants, we can make use of the standard CodeQL class ``FormattingFunction``, which provides an interface that models common formatting functions in C/C++. Meh... ====== diff --git a/docs/language/ql-training/cpp/intro-ql-cpp.rst b/docs/language/ql-training/cpp/intro-ql-cpp.rst index cbe29e6c520..9cfb1d3b442 100644 --- a/docs/language/ql-training/cpp/intro-ql-cpp.rst +++ b/docs/language/ql-training/cpp/intro-ql-cpp.rst @@ -108,7 +108,7 @@ Each query library also implicitly defines a module. Queries are always contained in query files with the file extension ``.ql``. `Quick queries `__, run in `QL for Eclipse `__, are no exception: the quick query window maintains a temporary query file in the background. - Parts of queries can be lifted into `library files `__ with the extension ``qll``. Definitions within such libraries can be brought into scope using ``import`` statements, and similarly QLL files can import each other’s definitions using “import” statements. + Parts of queries can be lifted into `library files `__ with the extension ``.qll``. Definitions within such libraries can be brought into scope using ``import`` statements, and similarly QLL files can import each other’s definitions using “import” statements. Logic can be encapsulated as user-defined `predicates `__ and `classes `__, and organized into `modules `__. Each QLL file implicitly defines a module, but QL and QLL files can also contain explicit module definitions, as we will see later. diff --git a/docs/language/ql-training/cpp/program-representation-cpp.rst b/docs/language/ql-training/cpp/program-representation-cpp.rst index 3c062089428..156cb0a4a68 100644 --- a/docs/language/ql-training/cpp/program-representation-cpp.rst +++ b/docs/language/ql-training/cpp/program-representation-cpp.rst @@ -64,7 +64,7 @@ Working with functions Functions are represented by the Function class. Each declaration or definition of a function is represented by a ``FunctionDeclarationEntry``. -Calls to functions are modeled by class Call and its subclasses: +Calls to functions are modeled by class ``Call`` and its subclasses: - ``Call.getTarget()`` gets the declared target of the call; undefined for calls through function pointers - ``Function.getACallToThisFunction()`` gets a call to this function @@ -103,7 +103,7 @@ Working with macros #define square(x) x*x y = square(y0), z = square(z0) -is represented in the CodeQL database database as: +is represented in the CodeQL database as: - A Macro entity representing the text of the *head* and *body* of the macro - Assignment nodes, representing the two assignments after preprocessing @@ -117,4 +117,4 @@ Useful predicates on ``Element``: ``isInMacroExpansion()``, ``isAffectedByMacro( .. note:: - The CodeQL database also contains information about macro definitions, which are represented by class ``Macro``. These macro definitions are related to the AST nodes resulting from their uses by the class ``MacroAccess``. \ No newline at end of file + The CodeQL database also contains information about macro definitions, which are represented by class ``Macro``. These macro definitions are related to the AST nodes resulting from their uses by the class ``MacroAccess``. diff --git a/docs/language/ql-training/java/intro-ql-java.rst b/docs/language/ql-training/java/intro-ql-java.rst index f7afe713c4a..46106204ed4 100644 --- a/docs/language/ql-training/java/intro-ql-java.rst +++ b/docs/language/ql-training/java/intro-ql-java.rst @@ -158,7 +158,7 @@ Member predicates are inherited and can be overridden. In the example, declaring a variable “EmptyBlock e” will allow it to range over only those blocks that have zero statements. -Classes in continued +Classes continued ======================= .. container:: column-left diff --git a/docs/language/ql-training/slide-snippets/intro-ql-general.rst b/docs/language/ql-training/slide-snippets/intro-ql-general.rst index d559c5bbeb7..ae0c0cdabb5 100644 --- a/docs/language/ql-training/slide-snippets/intro-ql-general.rst +++ b/docs/language/ql-training/slide-snippets/intro-ql-general.rst @@ -121,7 +121,7 @@ QL is: - a **logic** language based on first-order logic - a **declarative** language without side effects - an **object-oriented** language -- a **query** language working on a read-only CodeQL database database +- a **query** language working on a read-only CodeQL database - equipped with rich standard libraries **for program analysis** .. note:: From 8d02a740dda1b8ed961249b4e40d7e3dfd0c7409 Mon Sep 17 00:00:00 2001 From: james Date: Tue, 5 Nov 2019 20:03:47 +0000 Subject: [PATCH 141/148] docs: address remaining comments --- docs/language/ql-training/conf.py | 8 ++++---- .../ql-training/slide-snippets/local-data-flow.rst | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/language/ql-training/conf.py b/docs/language/ql-training/conf.py index 8f8a13569b5..117f515a54f 100644 --- a/docs/language/ql-training/conf.py +++ b/docs/language/ql-training/conf.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# QL training slides build configuration file +# CodeQL training slides build configuration file # # This file is execfile()d with the current directory set to its # containing dir. @@ -59,7 +59,7 @@ highlight_language = 'ql' master_doc = 'index' # General information about the project. -project = u'QL training and variant analysis examples' +project = u'CodeQL training and variant analysis examples' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, @@ -77,10 +77,10 @@ slide_theme_path = ["_static-training/"] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -html_title = 'QL training and variant analysis examples' +html_title = 'CodeQL training and variant analysis examples' # Output file base name for HTML help builder. -htmlhelp_basename = 'QL training' +htmlhelp_basename = 'CodeQL training' # The Semmle version info for the current release you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. diff --git a/docs/language/ql-training/slide-snippets/local-data-flow.rst b/docs/language/ql-training/slide-snippets/local-data-flow.rst index 068dfc9170a..85b851a9ce0 100644 --- a/docs/language/ql-training/slide-snippets/local-data-flow.rst +++ b/docs/language/ql-training/slide-snippets/local-data-flow.rst @@ -70,7 +70,7 @@ Local vs global data flow For further information, see: - - `Introduction to data flow analysis in QL `__ + - `Introduction to data flow analysis in CodeQL `__ .. rst-class:: background2 From 21d4e5f18624397b811c9fba594924de5be640f4 Mon Sep 17 00:00:00 2001 From: Esben Sparre Andreasen Date: Wed, 6 Nov 2019 10:16:43 +0100 Subject: [PATCH 142/148] Doc: Add missing `t` in `support`. --- change-notes/1.23/analysis-javascript.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/change-notes/1.23/analysis-javascript.md b/change-notes/1.23/analysis-javascript.md index 812e1e67748..49f527af424 100644 --- a/change-notes/1.23/analysis-javascript.md +++ b/change-notes/1.23/analysis-javascript.md @@ -2,7 +2,7 @@ ## General improvements -* Suppor for `globalThis` has been added. +* Support for `globalThis` has been added. * Support for the following frameworks and libraries has been improved: - [firebase](https://www.npmjs.com/package/firebase) From dc923ef694ca8cec80f8b25c7285ee13c6231753 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Wed, 6 Nov 2019 13:28:46 +0100 Subject: [PATCH 143/148] remove change note Co-Authored-By: Esben Sparre Andreasen --- change-notes/1.23/analysis-javascript.md | 1 - 1 file changed, 1 deletion(-) diff --git a/change-notes/1.23/analysis-javascript.md b/change-notes/1.23/analysis-javascript.md index 8b89369fcff..bb135758904 100644 --- a/change-notes/1.23/analysis-javascript.md +++ b/change-notes/1.23/analysis-javascript.md @@ -11,7 +11,6 @@ - [rate-limiter-flexible](https://www.npmjs.com/package/rate-limiter-flexible) * The call graph has been improved to resolve method calls in more cases. This may produce more security alerts. -* Promises derived from a Deferred object are now recognized. * TypeScript 3.6 and 3.7 features are now supported. From 19554ff6e7a8584e63b76bbde90ff88379e26781 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Wed, 6 Nov 2019 13:37:54 +0100 Subject: [PATCH 144/148] change "e.g." to "for example" in qldoc --- javascript/ql/src/Statements/UseOfReturnlessFunction.ql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/javascript/ql/src/Statements/UseOfReturnlessFunction.ql b/javascript/ql/src/Statements/UseOfReturnlessFunction.ql index 48af33b8e43..123515a7cbd 100644 --- a/javascript/ql/src/Statements/UseOfReturnlessFunction.ql +++ b/javascript/ql/src/Statements/UseOfReturnlessFunction.ql @@ -160,7 +160,7 @@ predicate voidArrayCallback(DataFlow::CallNode call, Function func) { module Deferred { /** * An instance of a `Deferred` class. - * E.g. the result from `new Deferred()` or `new $.Deferred()`. + * For example the result from `new Deferred()` or `new $.Deferred()`. */ class DeferredInstance extends DataFlow::NewNode { // Describes both `new Deferred()`, `new $.Deferred` and other variants. From 24615f2bdbb73398e129cc28fa8a95326c5e53c8 Mon Sep 17 00:00:00 2001 From: james Date: Wed, 6 Nov 2019 12:37:20 +0000 Subject: [PATCH 145/148] docs: further review comments --- docs/language/ql-training/cpp/control-flow-cpp.rst | 2 +- docs/language/ql-training/cpp/intro-ql-cpp.rst | 12 ++++++------ .../ql-training/cpp/program-representation-cpp.rst | 10 +++++----- docs/language/ql-training/java/intro-ql-java.rst | 10 +++++----- .../ql-training/java/program-representation-java.rst | 12 ++++++------ .../ql-training/java/query-injection-java.rst | 6 +++--- .../ql-training/slide-snippets/local-data-flow.rst | 2 +- 7 files changed, 27 insertions(+), 27 deletions(-) diff --git a/docs/language/ql-training/cpp/control-flow-cpp.rst b/docs/language/ql-training/cpp/control-flow-cpp.rst index 3fddc3f8072..f72633b714f 100644 --- a/docs/language/ql-training/cpp/control-flow-cpp.rst +++ b/docs/language/ql-training/cpp/control-flow-cpp.rst @@ -89,7 +89,7 @@ Control flow graphs Modeling control flow ===================== -The control flow is modeled with a QL class, ``ControlFlowNode``. Examples of control flow nodes include statements and expressions. +The control flow is modeled with a CodeQL class, ``ControlFlowNode``. Examples of control flow nodes include statements and expressions. - ``ControlFlowNode`` provides API for traversing the control flow graph: diff --git a/docs/language/ql-training/cpp/intro-ql-cpp.rst b/docs/language/ql-training/cpp/intro-ql-cpp.rst index 9cfb1d3b442..4e2cd4ca0b4 100644 --- a/docs/language/ql-training/cpp/intro-ql-cpp.rst +++ b/docs/language/ql-training/cpp/intro-ql-cpp.rst @@ -70,7 +70,7 @@ A simple CodeQL query A `query `__ consists of a “select” clause that indicates what results should be returned. Typically it will also provide a “from” clause to declare some variables, and a “where” clause to state conditions over those variables. For more information on the structure of query files (including links to useful topics in the `QL language handbook `__), see `Introduction to query files `__. - In our example here, the first line of the query imports the `CodeQL for C/C++ standard library `__, which defines concepts like ``IfStmt`` and ``Block``. + In our example here, the first line of the query imports the `CodeQL library for C/C++ `__, which defines concepts like ``IfStmt`` and ``Block``. The query proper starts by declaring two variables–ifStmt and block. These variables represent sets of values in the database, according to the type of each of the variables. For example, ifStmt has the type IfStmt, which means it represents the set of all if statements in the program. If we simply selected these two variables:: @@ -135,10 +135,10 @@ A predicate allows you to pull out and name parts of a query. You can imagine a predicate to be a self-contained from-where-select statement, that produces an intermediate relation, or table. In this case, the ``isEmpty`` predicate will be the set of all blocks which are empty. -Classes -======= +Classes in QL +============= -A class allows you to name a set of values and define (member) predicates on them. +A QL class allows you to name a set of values and define (member) predicates on them. A class has at least one supertype and optionally a **characteristic predicate**; it contains the values that belong to *all* supertypes *and* satisfy the characteristic predicate, if provided. @@ -158,8 +158,8 @@ Member predicates are inherited and can be overridden. In the example, declaring a variable “EmptyBlock e” will allow it to range over only those blocks that have zero statements. -Classes continued -================= +Classes in QL continued +======================= .. container:: column-left diff --git a/docs/language/ql-training/cpp/program-representation-cpp.rst b/docs/language/ql-training/cpp/program-representation-cpp.rst index 156cb0a4a68..8f7a4d43de9 100644 --- a/docs/language/ql-training/cpp/program-representation-cpp.rst +++ b/docs/language/ql-training/cpp/program-representation-cpp.rst @@ -21,16 +21,16 @@ Agenda .. resume slides -AST classes -=========== +AST CodeQL classes +================== -Important AST classes include: +Important AST CodeQL classes include: - ``Expr``: expressions such as assignments, variable references, function calls, ... - ``Stmt``: statements such as conditionals, loops, try statements, ... - ``DeclarationEntry``: places where functions, variables or types are declared and/or defined -These three (and all other AST classes) are subclasses of ``Element``. +These three (and all other AST CodeQL classes) are subclasses of ``Element``. Symbol table ============ @@ -64,7 +64,7 @@ Working with functions Functions are represented by the Function class. Each declaration or definition of a function is represented by a ``FunctionDeclarationEntry``. -Calls to functions are modeled by class ``Call`` and its subclasses: +Calls to functions are modeled by CodeQL class ``Call`` and its subclasses: - ``Call.getTarget()`` gets the declared target of the call; undefined for calls through function pointers - ``Function.getACallToThisFunction()`` gets a call to this function diff --git a/docs/language/ql-training/java/intro-ql-java.rst b/docs/language/ql-training/java/intro-ql-java.rst index 46106204ed4..1666d9caca8 100644 --- a/docs/language/ql-training/java/intro-ql-java.rst +++ b/docs/language/ql-training/java/intro-ql-java.rst @@ -70,7 +70,7 @@ A simple CodeQL query A `query `__ consists of a “select” clause that indicates what results should be returned. Typically it will also provide a “from” clause to declare some variables, and a “where” clause to state conditions over those variables. For more information on the structure of query files (including links to useful topics in the `QL language handbook `__), see `Introduction to query files `__. - In our example here, the first line of the query imports the `CodeQL for Java library `__, which defines concepts like ``IfStmt`` and ``Block``. + In our example here, the first line of the query imports the `CodeQL library for Java `__, which defines concepts like ``IfStmt`` and ``Block``. The query proper starts by declaring two variables–ifStmt and block. These variables represent sets of values in the database, according to the type of each of the variables. For example, ``ifStmt`` has the type ``IfStmt``, which means it represents the set of all if statements in the program. If we simply selected these two variables:: @@ -135,10 +135,10 @@ A predicate allows you to pull out and name parts of a query. You can imagine a predicate to be a self-contained from-where-select statement, that produces an intermediate relation, or table. In this case, the ``isEmpty`` predicate will be the set of all blocks which are empty. -Classes -======= +Classes in QL +============= -A class allows you to name a set of values and define (member) predicates on them. +A QL class allows you to name a set of values and define (member) predicates on them. A class has at least one supertype and optionally a **characteristic predicate**; it contains the values that belong to *all* supertypes *and* satisfy the characteristic predicate, if provided. @@ -158,7 +158,7 @@ Member predicates are inherited and can be overridden. In the example, declaring a variable “EmptyBlock e” will allow it to range over only those blocks that have zero statements. -Classes continued +Classes in QL continued ======================= .. container:: column-left diff --git a/docs/language/ql-training/java/program-representation-java.rst b/docs/language/ql-training/java/program-representation-java.rst index ba9b1a9d80d..a66a81d6083 100644 --- a/docs/language/ql-training/java/program-representation-java.rst +++ b/docs/language/ql-training/java/program-representation-java.rst @@ -12,7 +12,7 @@ Agenda - Abstract syntax trees - Database representation - Program elements -- AST classes +- AST CodeQL classes .. insert abstract-syntax-tree.rst @@ -23,7 +23,7 @@ Agenda Program elements ================ -- The QL class ``Element`` represents program elements with a name. +- The CodeQL class ``Element`` represents program elements with a name. - This includes: packages (``Package``), compilation units (``CompilationUnit``), types (``Type``), methods (``Method``), constructors (``Constructor``), and variables (``Variable``). - It is often convenient to refer to an element that might either be a method or a constructor; the class ``Callable``, which is a common superclass of ``Method`` and ``Constructor``, can be used for this purpose. @@ -31,7 +31,7 @@ Program elements AST === -There are two primary AST classes, used within ``Callables``: +There are two primary AST CodeQL classes, used within ``Callables``: - ``Expr``: expressions such as assignments, variable references, function calls, ... - ``Stmt``: statements such as conditionals, loops, try statements, ... @@ -47,7 +47,7 @@ Types The database also includes information about the types used in a program: -- ``PrimitiveType`` represents a `primitive type `__, that is, one of ``boolean``, ``byte``, ``char``, ``double``, ``float``, ``int``, ``long``, ``short``. QL also classifies ``void`` and ```` (the type of the ``null`` literal) as primitive types. +- ``PrimitiveType`` represents a `primitive type `__, that is, one of ``boolean``, ``byte``, ``char``, ``double``, ``float``, ``int``, ``long``, ``short``. CodeQL also classifies ``void`` and ```` (the type of the ``null`` literal) as primitive types. - ``RefType`` represents a reference type; it has several subclasses: - ``Class`` represents a Java class. @@ -74,9 +74,9 @@ Working with variables Working with callables ====================== -Callables are represented by the ``Callable`` QL class. +Callables are represented by the ``Callable`` CodeQL class. -Calls to callables are modeled by the QL class ``Call`` and its subclasses: +Calls to callables are modeled by the CodeQL class ``Call`` and its subclasses: - ``Call.getCallee()`` gets the declared target of the call - ``Call.getAReference()`` gets a call to this function diff --git a/docs/language/ql-training/java/query-injection-java.rst b/docs/language/ql-training/java/query-injection-java.rst index a5b46eabda5..799c99e5209 100644 --- a/docs/language/ql-training/java/query-injection-java.rst +++ b/docs/language/ql-training/java/query-injection-java.rst @@ -77,14 +77,14 @@ Let’s start by looking for calls to methods with names of the form ``sparql*Qu .. note:: - - When performing `variant analysis `__, it is usually helpful to write a simple query that finds the simple syntactic pattern, before trying to go on to describe the cases where it goes wrong. - - In this case, we start by looking for all the method calls which appear to run, before trying to refine the query to find cases which are vulnerable to query injection. + - When performing `variant analysis `__, it is usually helpful to write a simple query that finds the simple syntactic pattern, before trying to go on to describe the cases where it goes wrong. + - In this case, we start by looking for all the method calls that appear to run, before trying to refine the query to find cases which are vulnerable to query injection. - The ``select`` clause defines what this query is looking for: - a ``MethodAccess``: the call to a SPARQL query method - a ``Method``: the SPARQL query method. - - The ``where`` part of the query ties these variables together using `predicates `__ defined in the `standard CodeQL for Java library `__. + - The ``where`` part of the query ties these variables together using `predicates `__ defined in the `standard CodeQL library for Java `__. CodeQL query: find string concatenation ======================================= diff --git a/docs/language/ql-training/slide-snippets/local-data-flow.rst b/docs/language/ql-training/slide-snippets/local-data-flow.rst index 85b851a9ce0..ed681b0398a 100644 --- a/docs/language/ql-training/slide-snippets/local-data-flow.rst +++ b/docs/language/ql-training/slide-snippets/local-data-flow.rst @@ -70,7 +70,7 @@ Local vs global data flow For further information, see: - - `Introduction to data flow analysis in CodeQL `__ + - `Introduction to data flow analysis with CodeQL `__ .. rst-class:: background2 From 2e7bd4db6f163b3d19eda68fab2b460dc8f43196 Mon Sep 17 00:00:00 2001 From: James Fletcher <42464962+jf205@users.noreply.github.com> Date: Wed, 6 Nov 2019 12:59:45 +0000 Subject: [PATCH 146/148] Update docs/language/ql-training/cpp/program-representation-cpp.rst Co-Authored-By: shati-patel <42641846+shati-patel@users.noreply.github.com> --- docs/language/ql-training/cpp/program-representation-cpp.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/language/ql-training/cpp/program-representation-cpp.rst b/docs/language/ql-training/cpp/program-representation-cpp.rst index 8f7a4d43de9..7048251aa02 100644 --- a/docs/language/ql-training/cpp/program-representation-cpp.rst +++ b/docs/language/ql-training/cpp/program-representation-cpp.rst @@ -64,7 +64,7 @@ Working with functions Functions are represented by the Function class. Each declaration or definition of a function is represented by a ``FunctionDeclarationEntry``. -Calls to functions are modeled by CodeQL class ``Call`` and its subclasses: +Calls to functions are modeled by the CodeQL class ``Call`` and its subclasses: - ``Call.getTarget()`` gets the declared target of the call; undefined for calls through function pointers - ``Function.getACallToThisFunction()`` gets a call to this function From b6f16dee81a97f6f50d047beb9a90c0e9f9ee1c5 Mon Sep 17 00:00:00 2001 From: Taus Brock-Nannestad Date: Wed, 6 Nov 2019 15:14:48 +0100 Subject: [PATCH 147/148] Python: Fix bad join order in `py/unused-import` --- python/ql/src/Imports/UnusedImport.ql | 59 +++++++++++++++++---------- 1 file changed, 38 insertions(+), 21 deletions(-) diff --git a/python/ql/src/Imports/UnusedImport.ql b/python/ql/src/Imports/UnusedImport.ql index 3ac04e2e4d2..4049f3d5e41 100644 --- a/python/ql/src/Imports/UnusedImport.ql +++ b/python/ql/src/Imports/UnusedImport.ql @@ -41,41 +41,58 @@ predicate all_not_understood(Module m) { } predicate imported_module_used_in_doctest(Import imp) { - exists(string modname | + exists(string modname, string docstring | imp.getAName().getAsname().(Name).getId() = modname and // Look for doctests containing the patterns: // >>> …name… // ... …name… - exists(StrConst doc | - doc.getEnclosingModule() = imp.getScope() and - doc.isDocString() and - doc.getText().regexpMatch("[\\s\\S]*(>>>|\\.\\.\\.).*" + modname + "[\\s\\S]*") - ) + docstring = doctest_in_scope(imp.getScope()) and + docstring.regexpMatch("[\\s\\S]*(>>>|\\.\\.\\.).*" + modname + "[\\s\\S]*") + ) +} + +pragma[noinline] +private string doctest_in_scope(Scope scope) { + exists(StrConst doc | + doc.getEnclosingModule() = scope and + doc.isDocString() and + result = doc.getText() and + result.regexpMatch("[\\s\\S]*(>>>|\\.\\.\\.)[\\s\\S]*") + ) +} + +pragma[noinline] +private string typehint_annotation_in_file(File file) { + exists(StrConst annotation | + annotation = any(Arguments a).getAnAnnotation() + or + annotation = any(AnnAssign a).getAnnotation() + | + annotation.pointsTo(Value::forString(result)) and + file = annotation.getLocation().getFile() + ) +} + +pragma[noinline] +private string typehint_comment_in_file(File file) { + exists(Comment typehint | + file = typehint.getLocation().getFile() and + result = typehint.getText() and + result.matches("# type:%") ) } predicate imported_module_used_in_typehint(Import imp) { - exists(string modname, Location loc | + exists(string modname, File file | imp.getAName().getAsname().(Name).getId() = modname and - loc.getFile() = imp.getScope().(Module).getFile() + file = imp.getScope().(Module).getFile() | // Look for type hints containing the patterns: // # type: …name… - exists(Comment typehint | - loc = typehint.getLocation() and - typehint.getText().regexpMatch("# type:.*" + modname + ".*") - ) + typehint_comment_in_file(file).regexpMatch("# type:.*" + modname + ".*") or // Type hint is inside a string annotation, as needed for forward references - exists(string typehint, Expr annotation | - annotation = any(Arguments a).getAnAnnotation() - or - annotation = any(AnnAssign a).getAnnotation() - | - annotation.pointsTo(Value::forString(typehint)) and - loc = annotation.getLocation() and - typehint.regexpMatch(".*\\b" + modname + "\\b.*") - ) + typehint_annotation_in_file(file).regexpMatch(".*\\b" + modname + "\\b.*") ) } From 43148083ebf7ca19832a5017b88f0853f5559ab9 Mon Sep 17 00:00:00 2001 From: Taus Brock-Nannestad Date: Wed, 6 Nov 2019 16:36:28 +0100 Subject: [PATCH 148/148] Python: Fix bad join order for `global_name_used`. As it turns out, there was a further bad join-order in the `global_name_used` predicate. In this case, there was a common subexpression in the RA that was being factored out and evaluated separately, producing a large number of tuples. --- python/ql/src/Imports/UnusedImport.ql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/ql/src/Imports/UnusedImport.ql b/python/ql/src/Imports/UnusedImport.ql index 3ac04e2e4d2..3d9163f9320 100644 --- a/python/ql/src/Imports/UnusedImport.ql +++ b/python/ql/src/Imports/UnusedImport.ql @@ -13,17 +13,17 @@ import python import Variables.Definition -predicate global_name_used(Module m, Variable name) { +predicate global_name_used(Module m, string name) { exists(Name u, GlobalVariable v | u.uses(v) and - v.getId() = name.getId() and + v.getId() = name and u.getEnclosingModule() = m ) or // A use of an undefined class local variable, will use the global variable exists(Name u, LocalVariable v | u.uses(v) and - v.getId() = name.getId() and + v.getId() = name and u.getEnclosingModule() = m and not v.getScope().getEnclosingScope*() instanceof Function ) @@ -84,7 +84,7 @@ predicate unused_import(Import imp, Variable name) { not imp.getAnImportedModuleName() = "__future__" and not imp.getEnclosingModule().declaredInAll(name.getId()) and imp.getScope() = imp.getEnclosingModule() and - not global_name_used(imp.getScope(), name) and + not global_name_used(imp.getScope(), name.getId()) and // Imports in `__init__.py` are used to force module loading not imp.getEnclosingModule().isPackageInit() and // Name may be imported for use in epytext documentation