From 9db67d498840569fae6f6321c2a6de34c9e5a321 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Tue, 3 May 2022 12:09:10 +0200 Subject: [PATCH 01/91] move the Actions API out of experimental --- .../ql/{src/experimental => lib}/semmle/javascript/Actions.qll | 0 .../ql/src/experimental/Security/CWE-094/ExpressionInjection.ql | 2 +- .../ql/src/experimental/Security/CWE-094/UntrustedCheckout.ql | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename javascript/ql/{src/experimental => lib}/semmle/javascript/Actions.qll (100%) diff --git a/javascript/ql/src/experimental/semmle/javascript/Actions.qll b/javascript/ql/lib/semmle/javascript/Actions.qll similarity index 100% rename from javascript/ql/src/experimental/semmle/javascript/Actions.qll rename to javascript/ql/lib/semmle/javascript/Actions.qll diff --git a/javascript/ql/src/experimental/Security/CWE-094/ExpressionInjection.ql b/javascript/ql/src/experimental/Security/CWE-094/ExpressionInjection.ql index 29a34897880..29ffa9e27e2 100644 --- a/javascript/ql/src/experimental/Security/CWE-094/ExpressionInjection.ql +++ b/javascript/ql/src/experimental/Security/CWE-094/ExpressionInjection.ql @@ -12,7 +12,7 @@ */ import javascript -import experimental.semmle.javascript.Actions +import semmle.javascript.Actions bindingset[context] private predicate isExternalUserControlledIssue(string context) { diff --git a/javascript/ql/src/experimental/Security/CWE-094/UntrustedCheckout.ql b/javascript/ql/src/experimental/Security/CWE-094/UntrustedCheckout.ql index 2715f788e5c..cf19d07015d 100644 --- a/javascript/ql/src/experimental/Security/CWE-094/UntrustedCheckout.ql +++ b/javascript/ql/src/experimental/Security/CWE-094/UntrustedCheckout.ql @@ -13,7 +13,7 @@ */ import javascript -import experimental.semmle.javascript.Actions +import semmle.javascript.Actions /** * An action step that doesn't contain `actor` or `label` check in `if:` or From bc470b89f1dd1e2abd0eb168f748a6eedde4425f Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Tue, 3 May 2022 12:14:30 +0200 Subject: [PATCH 02/91] leave a deprecated alias for Actions.qll --- javascript/ql/src/experimental/semmle/javascript/Actions.qll | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 javascript/ql/src/experimental/semmle/javascript/Actions.qll diff --git a/javascript/ql/src/experimental/semmle/javascript/Actions.qll b/javascript/ql/src/experimental/semmle/javascript/Actions.qll new file mode 100644 index 00000000000..2938cc14692 --- /dev/null +++ b/javascript/ql/src/experimental/semmle/javascript/Actions.qll @@ -0,0 +1,4 @@ +/** DEPRECATED: Use `semmle.javascript.Actions` instead. */ +deprecated module Actions { + import semmle.javascript.Actions::Actions +} From fc6eedd07a341c54f1d8b36280807106779b993f Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Tue, 3 May 2022 12:23:13 +0200 Subject: [PATCH 03/91] generalize the file pattern for github/actions related YAML --- javascript/ql/lib/semmle/javascript/Actions.qll | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/javascript/ql/lib/semmle/javascript/Actions.qll b/javascript/ql/lib/semmle/javascript/Actions.qll index 17c2691c947..a387d26bce4 100644 --- a/javascript/ql/lib/semmle/javascript/Actions.qll +++ b/javascript/ql/lib/semmle/javascript/Actions.qll @@ -16,7 +16,7 @@ module Actions { this.getLocation() .getFile() .getRelativePath() - .matches(["experimental/Security/CWE-094/.github/workflows/%", ".github/workflows/%"]) + .regexpMatch("(^|.*/)\\.github/workflows/.*\\.yml$") } } From 2a65d1d3ec663bc3ab28a18cd8592ce747fc6953 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Tue, 3 May 2022 13:12:47 +0200 Subject: [PATCH 04/91] move js/actions/injection out of experimental --- .../Security/CWE-094/ExpressionInjection.qhelp | 0 .../{experimental => }/Security/CWE-094/ExpressionInjection.ql | 0 .../Security/CWE-094/examples/comment_issue_bad.yml | 0 .../Security/CWE-094/examples/comment_issue_good.yml | 0 .../test/experimental/Security/CWE-094/ExpressionInjection.qlref | 1 - .../ExpressionInjection}/.github/workflows/comment_issue.yml | 0 .../CWE-094/ExpressionInjection}/ExpressionInjection.expected | 0 .../CWE-094/ExpressionInjection/ExpressionInjection.qlref | 1 + .../query-tests/Security/CWE-094/ExpressionInjection/empty.js | 1 + 9 files changed, 2 insertions(+), 1 deletion(-) rename javascript/ql/src/{experimental => }/Security/CWE-094/ExpressionInjection.qhelp (100%) rename javascript/ql/src/{experimental => }/Security/CWE-094/ExpressionInjection.ql (100%) rename javascript/ql/src/{experimental => }/Security/CWE-094/examples/comment_issue_bad.yml (100%) rename javascript/ql/src/{experimental => }/Security/CWE-094/examples/comment_issue_good.yml (100%) delete mode 100644 javascript/ql/test/experimental/Security/CWE-094/ExpressionInjection.qlref rename javascript/ql/test/{experimental/Security/CWE-094 => query-tests/Security/CWE-094/ExpressionInjection}/.github/workflows/comment_issue.yml (100%) rename javascript/ql/test/{experimental/Security/CWE-094 => query-tests/Security/CWE-094/ExpressionInjection}/ExpressionInjection.expected (100%) create mode 100644 javascript/ql/test/query-tests/Security/CWE-094/ExpressionInjection/ExpressionInjection.qlref create mode 100644 javascript/ql/test/query-tests/Security/CWE-094/ExpressionInjection/empty.js diff --git a/javascript/ql/src/experimental/Security/CWE-094/ExpressionInjection.qhelp b/javascript/ql/src/Security/CWE-094/ExpressionInjection.qhelp similarity index 100% rename from javascript/ql/src/experimental/Security/CWE-094/ExpressionInjection.qhelp rename to javascript/ql/src/Security/CWE-094/ExpressionInjection.qhelp diff --git a/javascript/ql/src/experimental/Security/CWE-094/ExpressionInjection.ql b/javascript/ql/src/Security/CWE-094/ExpressionInjection.ql similarity index 100% rename from javascript/ql/src/experimental/Security/CWE-094/ExpressionInjection.ql rename to javascript/ql/src/Security/CWE-094/ExpressionInjection.ql diff --git a/javascript/ql/src/experimental/Security/CWE-094/examples/comment_issue_bad.yml b/javascript/ql/src/Security/CWE-094/examples/comment_issue_bad.yml similarity index 100% rename from javascript/ql/src/experimental/Security/CWE-094/examples/comment_issue_bad.yml rename to javascript/ql/src/Security/CWE-094/examples/comment_issue_bad.yml diff --git a/javascript/ql/src/experimental/Security/CWE-094/examples/comment_issue_good.yml b/javascript/ql/src/Security/CWE-094/examples/comment_issue_good.yml similarity index 100% rename from javascript/ql/src/experimental/Security/CWE-094/examples/comment_issue_good.yml rename to javascript/ql/src/Security/CWE-094/examples/comment_issue_good.yml diff --git a/javascript/ql/test/experimental/Security/CWE-094/ExpressionInjection.qlref b/javascript/ql/test/experimental/Security/CWE-094/ExpressionInjection.qlref deleted file mode 100644 index e3d73f1c726..00000000000 --- a/javascript/ql/test/experimental/Security/CWE-094/ExpressionInjection.qlref +++ /dev/null @@ -1 +0,0 @@ -experimental/Security/CWE-094/ExpressionInjection.ql diff --git a/javascript/ql/test/experimental/Security/CWE-094/.github/workflows/comment_issue.yml b/javascript/ql/test/query-tests/Security/CWE-094/ExpressionInjection/.github/workflows/comment_issue.yml similarity index 100% rename from javascript/ql/test/experimental/Security/CWE-094/.github/workflows/comment_issue.yml rename to javascript/ql/test/query-tests/Security/CWE-094/ExpressionInjection/.github/workflows/comment_issue.yml diff --git a/javascript/ql/test/experimental/Security/CWE-094/ExpressionInjection.expected b/javascript/ql/test/query-tests/Security/CWE-094/ExpressionInjection/ExpressionInjection.expected similarity index 100% rename from javascript/ql/test/experimental/Security/CWE-094/ExpressionInjection.expected rename to javascript/ql/test/query-tests/Security/CWE-094/ExpressionInjection/ExpressionInjection.expected diff --git a/javascript/ql/test/query-tests/Security/CWE-094/ExpressionInjection/ExpressionInjection.qlref b/javascript/ql/test/query-tests/Security/CWE-094/ExpressionInjection/ExpressionInjection.qlref new file mode 100644 index 00000000000..edaea6fbb21 --- /dev/null +++ b/javascript/ql/test/query-tests/Security/CWE-094/ExpressionInjection/ExpressionInjection.qlref @@ -0,0 +1 @@ +Security/CWE-094/ExpressionInjection.ql diff --git a/javascript/ql/test/query-tests/Security/CWE-094/ExpressionInjection/empty.js b/javascript/ql/test/query-tests/Security/CWE-094/ExpressionInjection/empty.js new file mode 100644 index 00000000000..a243684db7f --- /dev/null +++ b/javascript/ql/test/query-tests/Security/CWE-094/ExpressionInjection/empty.js @@ -0,0 +1 @@ +console.log('test') \ No newline at end of file From 48fb01f9f733cadc749bce248aa02d76c05417ca Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Tue, 3 May 2022 13:13:54 +0200 Subject: [PATCH 05/91] set js/actions/injection as a high precision warning query --- javascript/ql/src/Security/CWE-094/ExpressionInjection.ql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/javascript/ql/src/Security/CWE-094/ExpressionInjection.ql b/javascript/ql/src/Security/CWE-094/ExpressionInjection.ql index 29ffa9e27e2..e6022456439 100644 --- a/javascript/ql/src/Security/CWE-094/ExpressionInjection.ql +++ b/javascript/ql/src/Security/CWE-094/ExpressionInjection.ql @@ -3,7 +3,7 @@ * @description Using user-controlled GitHub Actions contexts like `run:` or `script:` may allow a malicious * user to inject code into the GitHub action. * @kind problem - * @problem.severity error + * @problem.severity warning * @precision high * @id js/actions/injection * @tags actions From df4bfef8c7da8412295631407669bb5d41709a61 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Tue, 3 May 2022 13:25:12 +0200 Subject: [PATCH 06/91] expand the qhelp for js/actions/injection --- .../ql/src/Security/CWE-094/ExpressionInjection.qhelp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/javascript/ql/src/Security/CWE-094/ExpressionInjection.qhelp b/javascript/ql/src/Security/CWE-094/ExpressionInjection.qhelp index b7c678f6e39..b9a3248408f 100644 --- a/javascript/ql/src/Security/CWE-094/ExpressionInjection.qhelp +++ b/javascript/ql/src/Security/CWE-094/ExpressionInjection.qhelp @@ -12,16 +12,21 @@

+

+ Code injection in GitHub actions may allow an attacker to + exfiltrate the temporary GitHub repository authorization token. + The token has write access to the repository, and thus an attacker + can use it to modify the repository. +

+

- The best practice to avoid code injection vulnerabilities in GitHub workflows is to set the untrusted input value of the expression to an intermediate environment variable. -

@@ -49,6 +54,7 @@
  • GitHub Security Lab Research: Keeping your GitHub Actions and workflows secure: Untrusted input.
    • +
  • GitHub Docs: Security hardening for GitHub Actions.
    •  From d8cc82bdb1f745622367f28fbcda30e63efec0a1 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Tue, 3 May 2022 13:32:22 +0200 Subject: [PATCH 07/91] add change-note --- .../ql/src/change-notes/2022-05-03-actions-injection.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 javascript/ql/src/change-notes/2022-05-03-actions-injection.md diff --git a/javascript/ql/src/change-notes/2022-05-03-actions-injection.md b/javascript/ql/src/change-notes/2022-05-03-actions-injection.md new file mode 100644 index 00000000000..57eda2fc21b --- /dev/null +++ b/javascript/ql/src/change-notes/2022-05-03-actions-injection.md @@ -0,0 +1,6 @@ +--- +category: newQuery +--- +* The `js/actions/injection` query has been added. It highlights GitHub Actions workflows that may allow an + attacker to execute arbitrary code in the workflow. + The query previously existed an experimental query. From 7530923af3d6810aa1ebc30f8b186d053f775d68 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Tue, 3 May 2022 14:17:37 +0200 Subject: [PATCH 08/91] add missing qldoc --- javascript/ql/lib/semmle/javascript/Actions.qll | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/javascript/ql/lib/semmle/javascript/Actions.qll b/javascript/ql/lib/semmle/javascript/Actions.qll index a387d26bce4..c74c2576483 100644 --- a/javascript/ql/lib/semmle/javascript/Actions.qll +++ b/javascript/ql/lib/semmle/javascript/Actions.qll @@ -21,6 +21,8 @@ module Actions { } /** + * A YAML node that may contain sub-nodes. + * * Actions are quite flexible in parsing YAML. * * For example: @@ -48,6 +50,7 @@ module Actions { this instanceof YAMLScalar } + /** Gets sub-name identified by `name`. */ YAMLNode getNode(string name) { exists(YAMLMapping mapping | mapping = this and @@ -68,6 +71,7 @@ module Actions { ) } + /** Gets the number of elements in this mapping or sequence. */ int getElementCount() { exists(YAMLMapping mapping | mapping = this and @@ -113,6 +117,7 @@ module Actions { On() { workflow.lookup("on") = this } + /** Gets the workflow that this trigger is in. */ Workflow getWorkflow() { result = workflow } } @@ -283,6 +288,7 @@ module Actions { Ref() { with.lookup("ref") = this } + /** Gets the `with` field this field belongs to. */ With getWith() { result = with } } From 31a4de902e354a5d1990305e5ef4c7220de0bf23 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Tue, 3 May 2022 14:19:36 +0200 Subject: [PATCH 09/91] add missing security severity --- javascript/ql/src/Security/CWE-094/ExpressionInjection.ql | 1 + 1 file changed, 1 insertion(+) diff --git a/javascript/ql/src/Security/CWE-094/ExpressionInjection.ql b/javascript/ql/src/Security/CWE-094/ExpressionInjection.ql index e6022456439..403686ebd7f 100644 --- a/javascript/ql/src/Security/CWE-094/ExpressionInjection.ql +++ b/javascript/ql/src/Security/CWE-094/ExpressionInjection.ql @@ -4,6 +4,7 @@ * user to inject code into the GitHub action. * @kind problem * @problem.severity warning + * @security-severity 9.3 * @precision high * @id js/actions/injection * @tags actions From 8e2b00d209d3213c585d3da445c41bb6ebdbb16a Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Tue, 3 May 2022 15:51:38 +0200 Subject: [PATCH 10/91] make the big disjunctions more readable by using a set literal --- .../Security/CWE-094/ExpressionInjection.ql | 46 +++++++++++-------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/javascript/ql/src/Security/CWE-094/ExpressionInjection.ql b/javascript/ql/src/Security/CWE-094/ExpressionInjection.ql index 403686ebd7f..80af8dae82a 100644 --- a/javascript/ql/src/Security/CWE-094/ExpressionInjection.ql +++ b/javascript/ql/src/Security/CWE-094/ExpressionInjection.ql @@ -23,14 +23,18 @@ private predicate isExternalUserControlledIssue(string context) { bindingset[context] private predicate isExternalUserControlledPullRequest(string context) { - context.regexpMatch("\\bgithub\\s*\\.\\s*event\\s*\\.\\s*pull_request\\s*\\.\\s*title\\b") or - context.regexpMatch("\\bgithub\\s*\\.\\s*event\\s*\\.\\s*pull_request\\s*\\.\\s*body\\b") or - context - .regexpMatch("\\bgithub\\s*\\.\\s*event\\s*\\.\\s*pull_request\\s*\\.\\s*head\\s*\\.\\s*label\\b") or - context - .regexpMatch("\\bgithub\\s*\\.\\s*event\\s*\\.\\s*pull_request\\s*\\.\\s*head\\s*\\.\\s*repo\\s*\\.\\s*default_branch\\b") or - context - .regexpMatch("\\bgithub\\s*\\.\\s*event\\s*\\.\\s*pull_request\\s*\\.\\s*head\\s*\\.\\s*ref\\b") + exists(string reg | + reg = + [ + "\\bgithub\\s*\\.\\s*event\\s*\\.\\s*pull_request\\s*\\.\\s*title\\b", + "\\bgithub\\s*\\.\\s*event\\s*\\.\\s*pull_request\\s*\\.\\s*body\\b", + "\\bgithub\\s*\\.\\s*event\\s*\\.\\s*pull_request\\s*\\.\\s*head\\s*\\.\\s*label\\b", + "\\bgithub\\s*\\.\\s*event\\s*\\.\\s*pull_request\\s*\\.\\s*head\\s*\\.\\s*repo\\s*\\.\\s*default_branch\\b", + "\\bgithub\\s*\\.\\s*event\\s*\\.\\s*pull_request\\s*\\.\\s*head\\s*\\.\\s*ref\\b", + ] + | + context.regexpMatch(reg) + ) } bindingset[context] @@ -52,18 +56,20 @@ private predicate isExternalUserControlledGollum(string context) { bindingset[context] private predicate isExternalUserControlledCommit(string context) { - context - .regexpMatch("\\bgithub\\s*\\.\\s*event\\s*\\.\\s*commits(?:\\[[0-9]\\]|\\s*\\.\\s*\\*)+\\s*\\.\\s*message\\b") or - context.regexpMatch("\\bgithub\\s*\\.\\s*event\\s*\\.\\s*head_commit\\s*\\.\\s*message\\b") or - context - .regexpMatch("\\bgithub\\s*\\.\\s*event\\s*\\.\\s*head_commit\\s*\\.\\s*author\\s*\\.\\s*email\\b") or - context - .regexpMatch("\\bgithub\\s*\\.\\s*event\\s*\\.\\s*head_commit\\s*\\.\\s*author\\s*\\.\\s*name\\b") or - context - .regexpMatch("\\bgithub\\s*\\.\\s*event\\s*\\.\\s*commits(?:\\[[0-9]\\]|\\s*\\.\\s*\\*)+\\s*\\.\\s*author\\s*\\.\\s*email\\b") or - context - .regexpMatch("\\bgithub\\s*\\.\\s*event\\s*\\.\\s*commits(?:\\[[0-9]\\]|\\s*\\.\\s*\\*)+\\s*\\.\\s*author\\s*\\.\\s*name\\b") or - context.regexpMatch("\\bgithub\\s*\\.\\s*head_ref\\b") + exists(string reg | + reg = + [ + "\\bgithub\\s*\\.\\s*event\\s*\\.\\s*commits(?:\\[[0-9]\\]|\\s*\\.\\s*\\*)+\\s*\\.\\s*message\\b", + "\\bgithub\\s*\\.\\s*event\\s*\\.\\s*head_commit\\s*\\.\\s*message\\b", + "\\bgithub\\s*\\.\\s*event\\s*\\.\\s*head_commit\\s*\\.\\s*author\\s*\\.\\s*email\\b", + "\\bgithub\\s*\\.\\s*event\\s*\\.\\s*head_commit\\s*\\.\\s*author\\s*\\.\\s*name\\b", + "\\bgithub\\s*\\.\\s*event\\s*\\.\\s*commits(?:\\[[0-9]\\]|\\s*\\.\\s*\\*)+\\s*\\.\\s*author\\s*\\.\\s*email\\b", + "\\bgithub\\s*\\.\\s*event\\s*\\.\\s*commits(?:\\[[0-9]\\]|\\s*\\.\\s*\\*)+\\s*\\.\\s*author\\s*\\.\\s*name\\b", + "\\bgithub\\s*\\.\\s*head_ref\\b" + ] + | + context.regexpMatch(reg) + ) } bindingset[context] From a8f7a4459e45a9922483762111e4e4d08e160aa0 Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Tue, 16 Nov 2021 13:16:03 +0000 Subject: [PATCH 11/91] Port redos libraries from Python --- .../code/java/dataflow/ExternalFlow.qll | 1 + .../lib/semmle/code/java/regex/RegexFlow.qll | 18 + .../semmle/code/java/regex/RegexTreeView.qll | 998 +++++++++++++++ java/ql/lib/semmle/code/java/regex/regex.qll | 907 +++++++++++++ .../performance/ExponentialBackTracking.qll | 342 +++++ .../java/security/performance/ReDoSUtil.qll | 1135 +++++++++++++++++ .../security/performance/RegExpTreeView.qll | 49 + .../performance/SuperlinearBackTracking.qll | 420 ++++++ 8 files changed, 3870 insertions(+) create mode 100644 java/ql/lib/semmle/code/java/regex/RegexFlow.qll create mode 100644 java/ql/lib/semmle/code/java/regex/RegexTreeView.qll create mode 100644 java/ql/lib/semmle/code/java/regex/regex.qll create mode 100644 java/ql/lib/semmle/code/java/security/performance/ExponentialBackTracking.qll create mode 100644 java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll create mode 100644 java/ql/lib/semmle/code/java/security/performance/RegExpTreeView.qll create mode 100644 java/ql/lib/semmle/code/java/security/performance/SuperlinearBackTracking.qll diff --git a/java/ql/lib/semmle/code/java/dataflow/ExternalFlow.qll b/java/ql/lib/semmle/code/java/dataflow/ExternalFlow.qll index a6a31559260..6d14dc5f95c 100644 --- a/java/ql/lib/semmle/code/java/dataflow/ExternalFlow.qll +++ b/java/ql/lib/semmle/code/java/dataflow/ExternalFlow.qll @@ -140,6 +140,7 @@ private module Frameworks { private import semmle.code.java.frameworks.jOOQ private import semmle.code.java.frameworks.JMS private import semmle.code.java.frameworks.RabbitMQ + private import semmle.code.java.regex.RegexFlow } private predicate sourceModelCsv(string row) { diff --git a/java/ql/lib/semmle/code/java/regex/RegexFlow.qll b/java/ql/lib/semmle/code/java/regex/RegexFlow.qll new file mode 100644 index 00000000000..54b16ae8a4b --- /dev/null +++ b/java/ql/lib/semmle/code/java/regex/RegexFlow.qll @@ -0,0 +1,18 @@ +import java +import semmle.code.java.dataflow.ExternalFlow + +private class RegexSinkCsv extends SinkModelCsv { + override predicate row(string row) { + row = + [ + //"namespace;type;subtypes;name;signature;ext;input;kind" + "java.util.regex;Pattern;false;compile;(String);;Argument[0];regex-use", + "java.util.regex;Pattern;false;compile;(String,int);;Argument[0];regex-use", + "java.util.regex;Pattern;false;matches;(String,CharSequence);;Argument[0];regex-use", + "java.util;String;false;matches;(String);;Argument[0];regex-use", + "java.util;String;false;split;(String);;Argument[0];regex-use", + "java.util;String;false;split;(String,int);;Argument[0];regex-use", + "com.google.common.base;Splitter;false;onPattern;(String);;Argument[0];regex-use" + ] + } +} diff --git a/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll b/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll new file mode 100644 index 00000000000..1b6013b26a0 --- /dev/null +++ b/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll @@ -0,0 +1,998 @@ +/** Provides a class hierarchy corresponding to a parse tree of regular expressions. */ + +import java +private import semmle.code.java.regex.regex + +/** + * An element containing a regular expression term, that is, either + * a string literal (parsed as a regular expression) + * or another regular expression term. + * + * For sequences and alternations, we require at least one child. + * Otherwise, we wish to represent the term differently. + * This avoids multiple representations of the same term. + */ +newtype TRegExpParent = + /** A string literal used as a regular expression */ + TRegExpLiteral(Regex re) or + /** A quantified term */ + TRegExpQuantifier(Regex re, int start, int end) { re.qualifiedItem(start, end, _, _) } or + /** A sequence term */ + TRegExpSequence(Regex re, int start, int end) { + re.sequence(start, end) and + exists(seqChild(re, start, end, 1)) // if a sequence does not have more than one element, it should be treated as that element instead. + } or + /** An alternation term */ + TRegExpAlt(Regex re, int start, int end) { + re.alternation(start, end) and + exists(int part_end | + re.alternationOption(start, end, start, part_end) and + part_end < end + ) // if an alternation does not have more than one element, it should be treated as that element instead. + } or + /** A character class term */ + TRegExpCharacterClass(Regex re, int start, int end) { re.charSet(start, end) } or + /** A character range term */ + TRegExpCharacterRange(Regex re, int start, int end) { re.charRange(_, start, _, _, end) } or + /** A group term */ + TRegExpGroup(Regex re, int start, int end) { re.group(start, end) } or + /** A special character */ + TRegExpSpecialChar(Regex re, int start, int end) { re.specialCharacter(start, end, _) } or + /** A normal character */ + TRegExpNormalChar(Regex re, int start, int end) { re.normalCharacter(start, end) } or + /** A back reference */ + TRegExpBackRef(Regex re, int start, int end) { re.backreference(start, end) } + +/** + * An element containing a regular expression term, that is, either + * a string literal (parsed as a regular expression) + * or another regular expression term. + */ +class RegExpParent extends TRegExpParent { + /** Gets a textual representation of this element. */ + string toString() { result = "RegExpParent" } + + /** Gets the `i`th child term. */ + abstract RegExpTerm getChild(int i); + + /** Gets a child term . */ + RegExpTerm getAChild() { result = this.getChild(_) } + + /** Gets the number of child terms. */ + int getNumChild() { result = count(this.getAChild()) } + + /** Gets the associated regex. */ + abstract Regex getRegex(); +} + +/** A string literal used as a regular expression */ +class RegExpLiteral extends TRegExpLiteral, RegExpParent { + Regex re; + + RegExpLiteral() { this = TRegExpLiteral(re) } + + override RegExpTerm getChild(int i) { i = 0 and result.getRegex() = re and result.isRootTerm() } + + /** Holds if dot, `.`, matches all characters, including newlines. */ + predicate isDotAll() { re.getAMode() = "DOTALL" } + + /** Holds if this regex matching is case-insensitive for this regex. */ + predicate isIgnoreCase() { re.getAMode() = "IGNORECASE" } + + /** Get a string representing all modes for this regex. */ + string getFlags() { result = concat(string mode | mode = re.getAMode() | mode, " | ") } + + override Regex getRegex() { result = re } + + /** Gets the primary QL class for this regex. */ + string getPrimaryQLClass() { result = "RegExpLiteral" } +} + +/** + * A regular expression term, that is, a syntactic part of a regular expression. + */ +class RegExpTerm extends RegExpParent { + Regex re; + int start; + int end; + + RegExpTerm() { + this = TRegExpAlt(re, start, end) + or + this = TRegExpBackRef(re, start, end) + or + this = TRegExpCharacterClass(re, start, end) + or + this = TRegExpCharacterRange(re, start, end) + or + this = TRegExpNormalChar(re, start, end) + or + this = TRegExpGroup(re, start, end) + or + this = TRegExpQuantifier(re, start, end) + or + this = TRegExpSequence(re, start, end) + or + this = TRegExpSpecialChar(re, start, end) + } + + /** + * Gets the outermost term of this regular expression. + */ + RegExpTerm getRootTerm() { + this.isRootTerm() and result = this + or + result = this.getParent().(RegExpTerm).getRootTerm() + } + + /** + * Holds if this term is part of a string literal + * that is interpreted as a regular expression. + */ + predicate isUsedAsRegExp() { any() } + + /** + * Holds if this is the root term of a regular expression. + */ + predicate isRootTerm() { start = 0 and end = re.getText().length() } + + override RegExpTerm getChild(int i) { + result = this.(RegExpAlt).getChild(i) + or + result = this.(RegExpBackRef).getChild(i) + or + result = this.(RegExpCharacterClass).getChild(i) + or + result = this.(RegExpCharacterRange).getChild(i) + or + result = this.(RegExpNormalChar).getChild(i) + or + result = this.(RegExpGroup).getChild(i) + or + result = this.(RegExpQuantifier).getChild(i) + or + result = this.(RegExpSequence).getChild(i) + or + result = this.(RegExpSpecialChar).getChild(i) + } + + /** + * Gets the parent term of this regular expression term, or the + * regular expression literal if this is the root term. + */ + RegExpParent getParent() { result.getAChild() = this } + + override Regex getRegex() { result = re } + + /** Gets the offset at which this term starts. */ + int getStart() { result = start } + + /** Gets the offset at which this term ends. */ + int getEnd() { result = end } + + override string toString() { result = re.getText().substring(start, end) } + + /** + * Gets the location of the surrounding regex, as locations inside the regex do not exist. + * To get location information corresponding to the term inside the regex, + * use `hasLocationInfo`. + */ + Location getLocation() { result = re.getLocation() } + + /** Holds if this term is found at the specified location offsets. */ + predicate hasLocationInfo( + string filepath, int startline, int startcolumn, int endline, int endcolumn + ) { + exists(int re_start, int re_end | + re.getLocation().hasLocationInfo(filepath, startline, re_start, endline, re_end) and + startcolumn = re_start + start + 4 and + endcolumn = re_start + end + 3 + ) + } + + /** Gets the file in which this term is found. */ + File getFile() { result = this.getLocation().getFile() } + + /** Gets the raw source text of this term. */ + string getRawValue() { result = this.toString() } + + /** Gets the string literal in which this term is found. */ + RegExpLiteral getLiteral() { result = TRegExpLiteral(re) } + + /** Gets the regular expression term that is matched (textually) before this one, if any. */ + RegExpTerm getPredecessor() { + exists(RegExpTerm parent | parent = this.getParent() | + result = parent.(RegExpSequence).previousElement(this) + or + not exists(parent.(RegExpSequence).previousElement(this)) and + not parent instanceof RegExpSubPattern and + result = parent.getPredecessor() + ) + } + + /** Gets the regular expression term that is matched (textually) after this one, if any. */ + RegExpTerm getSuccessor() { + exists(RegExpTerm parent | parent = this.getParent() | + result = parent.(RegExpSequence).nextElement(this) + or + not exists(parent.(RegExpSequence).nextElement(this)) and + not parent instanceof RegExpSubPattern and + result = parent.getSuccessor() + ) + } + + /** Gets the primary QL class for this term. */ + string getPrimaryQLClass() { result = "RegExpTerm" } +} + +/** + * A quantified regular expression term. + * + * Example: + * + * ``` + * ((ECMA|Java)[sS]cript)* + * ``` + */ +class RegExpQuantifier extends RegExpTerm, TRegExpQuantifier { + int part_end; + boolean maybe_empty; + boolean may_repeat_forever; + + RegExpQuantifier() { + this = TRegExpQuantifier(re, start, end) and + re.qualifiedPart(start, part_end, end, maybe_empty, may_repeat_forever) + } + + override RegExpTerm getChild(int i) { + i = 0 and + result.getRegex() = re and + result.getStart() = start and + result.getEnd() = part_end + } + + /** Hols if this term may match an unlimited number of times. */ + predicate mayRepeatForever() { may_repeat_forever = true } + + /** Gets the qualifier for this term. That is e.g "?" for "a?". */ + string getQualifier() { result = re.getText().substring(part_end, end) } + + override string getPrimaryQLClass() { result = "RegExpQuantifier" } +} + +/** + * A regular expression term that permits unlimited repetitions. + */ +class InfiniteRepetitionQuantifier extends RegExpQuantifier { + InfiniteRepetitionQuantifier() { this.mayRepeatForever() } +} + +/** + * A star-quantified term. + * + * Example: + * + * ``` + * \w* + * ``` + */ +class RegExpStar extends InfiniteRepetitionQuantifier { + RegExpStar() { this.getQualifier().charAt(0) = "*" } + + override string getPrimaryQLClass() { result = "RegExpStar" } +} + +/** + * A plus-quantified term. + * + * Example: + * + * ``` + * \w+ + * ``` + */ +class RegExpPlus extends InfiniteRepetitionQuantifier { + RegExpPlus() { this.getQualifier().charAt(0) = "+" } + + override string getPrimaryQLClass() { result = "RegExpPlus" } +} + +/** + * An optional term. + * + * Example: + * + * ``` + * ;? + * ``` + */ +class RegExpOpt extends RegExpQuantifier { + RegExpOpt() { this.getQualifier().charAt(0) = "?" } + + override string getPrimaryQLClass() { result = "RegExpOpt" } +} + +/** + * A range-quantified term + * + * Examples: + * + * ``` + * \w{2,4} + * \w{2,} + * \w{2} + * ``` + */ +class RegExpRange extends RegExpQuantifier { + string upper; + string lower; + + RegExpRange() { re.multiples(part_end, end, lower, upper) } + + /** Gets the string defining the upper bound of this range, if any. */ + string getUpper() { result = upper } + + /** Gets the string defining the lower bound of this range, if any. */ + string getLower() { result = lower } + + /** + * Gets the upper bound of the range, if any. + * + * If there is no upper bound, any number of repetitions is allowed. + * For a term of the form `r{lo}`, both the lower and the upper bound + * are `lo`. + */ + int getUpperBound() { result = this.getUpper().toInt() } + + /** Gets the lower bound of the range. */ + int getLowerBound() { result = this.getLower().toInt() } + + override string getPrimaryQLClass() { result = "RegExpRange" } +} + +/** + * A sequence term. + * + * Example: + * + * ``` + * (ECMA|Java)Script + * ``` + * + * This is a sequence with the elements `(ECMA|Java)` and `Script`. + */ +class RegExpSequence extends RegExpTerm, TRegExpSequence { + RegExpSequence() { this = TRegExpSequence(re, start, end) } + + override RegExpTerm getChild(int i) { result = seqChild(re, start, end, i) } + + /** Gets the element preceding `element` in this sequence. */ + RegExpTerm previousElement(RegExpTerm element) { element = this.nextElement(result) } + + /** Gets the element following `element` in this sequence. */ + RegExpTerm nextElement(RegExpTerm element) { + exists(int i | + element = this.getChild(i) and + result = this.getChild(i + 1) + ) + } + + override string getPrimaryQLClass() { result = "RegExpSequence" } +} + +pragma[nomagic] +private int seqChildEnd(Regex re, int start, int end, int i) { + result = seqChild(re, start, end, i).getEnd() +} + +// moved out so we can use it in the charpred +private RegExpTerm seqChild(Regex re, int start, int end, int i) { + re.sequence(start, end) and + ( + i = 0 and + result.getRegex() = re and + result.getStart() = start and + exists(int itemEnd | + re.item(start, itemEnd) and + result.getEnd() = itemEnd + ) + or + i > 0 and + result.getRegex() = re and + exists(int itemStart | itemStart = seqChildEnd(re, start, end, i - 1) | + result.getStart() = itemStart and + re.item(itemStart, result.getEnd()) + ) + ) +} + +/** + * An alternative term, that is, a term of the form `a|b`. + * + * Example: + * + * ``` + * ECMA|Java + * ``` + */ +class RegExpAlt extends RegExpTerm, TRegExpAlt { + RegExpAlt() { this = TRegExpAlt(re, start, end) } + + override RegExpTerm getChild(int i) { + i = 0 and + result.getRegex() = re and + result.getStart() = start and + exists(int part_end | + re.alternationOption(start, end, start, part_end) and + result.getEnd() = part_end + ) + or + i > 0 and + result.getRegex() = re and + exists(int part_start | + part_start = this.getChild(i - 1).getEnd() + 1 // allow for the | + | + result.getStart() = part_start and + re.alternationOption(start, end, part_start, result.getEnd()) + ) + } + + override string getPrimaryQLClass() { result = "RegExpAlt" } +} + +/** + * An escaped regular expression term, that is, a regular expression + * term starting with a backslash, which is not a backreference. + * + * Example: + * + * ``` + * \. + * \w + * ``` + */ +class RegExpEscape extends RegExpNormalChar { + RegExpEscape() { re.escapedCharacter(start, end) } + + /** + * Gets the name of the escaped; for example, `w` for `\w`. + * TODO: Handle named escapes. + */ + override string getValue() { + this.isIdentityEscape() and result = this.getUnescaped() + or + this.getUnescaped() = "n" and result = "\n" + or + this.getUnescaped() = "r" and result = "\r" + or + this.getUnescaped() = "t" and result = "\t" + or + // TODO: Find a way to include a formfeed character + // this.getUnescaped() = "f" and result = " " + // or + this.isUnicode() and + result = this.getUnicode() + } + + /** Holds if this terms name is given by the part following the escape character. */ + predicate isIdentityEscape() { not this.getUnescaped() in ["n", "r", "t", "f"] } + + override string getPrimaryQLClass() { result = "RegExpEscape" } + + /** Gets the part of the term following the escape character. That is e.g. "w" if the term is "\w". */ + private string getUnescaped() { result = this.getText().suffix(1) } + + /** + * Gets the text for this escape. That is e.g. "\w". + */ + private string getText() { result = re.getText().substring(start, end) } + + /** + * Holds if this is a unicode escape. + */ + private predicate isUnicode() { this.getText().prefix(2) = ["\\u", "\\U"] } + + /** + * Gets the unicode char for this escape. + * E.g. for `\u0061` this returns "a". + */ + private string getUnicode() { + exists(int codepoint | codepoint = sum(this.getHexValueFromUnicode(_)) | + result = codepoint.toUnicode() + ) + } + + /** + * Gets int value for the `index`th char in the hex number of the unicode escape. + * E.g. for `\u0061` and `index = 2` this returns 96 (the number `6` interpreted as hex). + */ + private int getHexValueFromUnicode(int index) { + this.isUnicode() and + exists(string hex, string char | hex = this.getText().suffix(2) | + char = hex.charAt(index) and + result = 16.pow(hex.length() - index - 1) * toHex(char) + ) + } +} + +/** + * Gets the hex number for the `hex` char. + */ +private int toHex(string hex) { + hex = [0 .. 9].toString() and + result = hex.toInt() + or + result = 10 and hex = ["a", "A"] + or + result = 11 and hex = ["b", "B"] + or + result = 12 and hex = ["c", "C"] + or + result = 13 and hex = ["d", "D"] + or + result = 14 and hex = ["e", "E"] + or + result = 15 and hex = ["f", "F"] +} + +/** + * A character class escape in a regular expression. + * That is, an escaped charachter that denotes multiple characters. + * + * Examples: + * + * ``` + * \w + * \S + * ``` + */ +class RegExpCharacterClassEscape extends RegExpEscape { + RegExpCharacterClassEscape() { this.getValue() in ["d", "D", "s", "S", "w", "W"] } + + override RegExpTerm getChild(int i) { none() } + + override string getPrimaryQLClass() { result = "RegExpCharacterClassEscape" } +} + +/** + * A character class in a regular expression. + * + * Examples: + * + * ``` + * [a-z_] + * [^<>&] + * ``` + */ +class RegExpCharacterClass extends RegExpTerm, TRegExpCharacterClass { + RegExpCharacterClass() { this = TRegExpCharacterClass(re, start, end) } + + /** Holds if this character class is inverted, matching the opposite of its content. */ + predicate isInverted() { re.getChar(start + 1) = "^" } + + /** Gets the `i`th char inside this charater class. */ + string getCharThing(int i) { result = re.getChar(i + start) } + + /** Holds if this character class can match anything. */ + predicate isUniversalClass() { + // [^] + this.isInverted() and not exists(this.getAChild()) + or + // [\w\W] and similar + not this.isInverted() and + exists(string cce1, string cce2 | + cce1 = this.getAChild().(RegExpCharacterClassEscape).getValue() and + cce2 = this.getAChild().(RegExpCharacterClassEscape).getValue() + | + cce1 != cce2 and cce1.toLowerCase() = cce2.toLowerCase() + ) + } + + override RegExpTerm getChild(int i) { + i = 0 and + result.getRegex() = re and + exists(int itemStart, int itemEnd | + result.getStart() = itemStart and + re.char_set_start(start, itemStart) and + re.char_set_child(start, itemStart, itemEnd) and + result.getEnd() = itemEnd + ) + or + i > 0 and + result.getRegex() = re and + exists(int itemStart | itemStart = this.getChild(i - 1).getEnd() | + result.getStart() = itemStart and + re.char_set_child(start, itemStart, result.getEnd()) + ) + } + + override string getPrimaryQLClass() { result = "RegExpCharacterClass" } +} + +/** + * A character range in a character class in a regular expression. + * + * Example: + * + * ``` + * a-z + * ``` + */ +class RegExpCharacterRange extends RegExpTerm, TRegExpCharacterRange { + int lower_end; + int upper_start; + + RegExpCharacterRange() { + this = TRegExpCharacterRange(re, start, end) and + re.charRange(_, start, lower_end, upper_start, end) + } + + /** Holds if this range goes from `lo` to `hi`, in effect is `lo-hi`. */ + predicate isRange(string lo, string hi) { + lo = re.getText().substring(start, lower_end) and + hi = re.getText().substring(upper_start, end) + } + + override RegExpTerm getChild(int i) { + i = 0 and + result.getRegex() = re and + result.getStart() = start and + result.getEnd() = lower_end + or + i = 1 and + result.getRegex() = re and + result.getStart() = upper_start and + result.getEnd() = end + } + + override string getPrimaryQLClass() { result = "RegExpCharacterRange" } +} + +/** + * A normal character in a regular expression, that is, a character + * without special meaning. This includes escaped characters. + * + * Examples: + * ``` + * t + * \t + * ``` + */ +class RegExpNormalChar extends RegExpTerm, TRegExpNormalChar { + RegExpNormalChar() { this = TRegExpNormalChar(re, start, end) } + + /** + * Holds if this constant represents a valid Unicode character (as opposed + * to a surrogate code point that does not correspond to a character by itself.) + */ + predicate isCharacter() { any() } + + /** Gets the string representation of the char matched by this term. */ + string getValue() { result = re.getText().substring(start, end) } + + override RegExpTerm getChild(int i) { none() } + + override string getPrimaryQLClass() { result = "RegExpNormalChar" } +} + +/** + * A constant regular expression term, that is, a regular expression + * term matching a single string. Currently, this will always be a single character. + * + * Example: + * + * ``` + * a + * ``` + */ +class RegExpConstant extends RegExpTerm { + string value; + + RegExpConstant() { + this = TRegExpNormalChar(re, start, end) and + not this instanceof RegExpCharacterClassEscape and + // exclude chars in qualifiers + // TODO: push this into regex library + not exists(int qstart, int qend | re.qualifiedPart(_, qstart, qend, _, _) | + qstart <= start and end <= qend + ) and + value = this.(RegExpNormalChar).getValue() + } + + /** + * Holds if this constant represents a valid Unicode character (as opposed + * to a surrogate code point that does not correspond to a character by itself.) + */ + predicate isCharacter() { any() } + + /** Gets the string matched by this constant term. */ + string getValue() { result = value } + + override RegExpTerm getChild(int i) { none() } + + override string getPrimaryQLClass() { result = "RegExpConstant" } +} + +/** + * A grouped regular expression. + * + * Examples: + * + * ``` + * (ECMA|Java) + * (?:ECMA|Java) + * (?['"]) + * ``` + */ +class RegExpGroup extends RegExpTerm, TRegExpGroup { + RegExpGroup() { this = TRegExpGroup(re, start, end) } + + /** + * Gets the index of this capture group within the enclosing regular + * expression literal. + * + * For example, in the regular expression `/((a?).)(?:b)/`, the + * group `((a?).)` has index 1, the group `(a?)` nested inside it + * has index 2, and the group `(?:b)` has no index, since it is + * not a capture group. + */ + int getNumber() { result = re.getGroupNumber(start, end) } + + /** Holds if this is a named capture group. */ + predicate isNamed() { exists(this.getName()) } + + /** Gets the name of this capture group, if any. */ + string getName() { result = re.getGroupName(start, end) } + + override RegExpTerm getChild(int i) { + result.getRegex() = re and + i = 0 and + re.groupContents(start, end, result.getStart(), result.getEnd()) + } + + override string getPrimaryQLClass() { result = "RegExpGroup" } +} + +/** + * A special character in a regular expression. + * + * Examples: + * ``` + * ^ + * $ + * . + * ``` + */ +class RegExpSpecialChar extends RegExpTerm, TRegExpSpecialChar { + string char; + + RegExpSpecialChar() { + this = TRegExpSpecialChar(re, start, end) and + re.specialCharacter(start, end, char) + } + + /** + * Holds if this constant represents a valid Unicode character (as opposed + * to a surrogate code point that does not correspond to a character by itself.) + */ + predicate isCharacter() { any() } + + /** Gets the char for this term. */ + string getChar() { result = char } + + override RegExpTerm getChild(int i) { none() } + + override string getPrimaryQLClass() { result = "RegExpSpecialChar" } +} + +/** + * A dot regular expression. + * + * Example: + * + * ``` + * . + * ``` + */ +class RegExpDot extends RegExpSpecialChar { + RegExpDot() { this.getChar() = "." } + + override string getPrimaryQLClass() { result = "RegExpDot" } +} + +/** + * A dollar assertion `$` matching the end of a line. + * + * Example: + * + * ``` + * $ + * ``` + */ +class RegExpDollar extends RegExpSpecialChar { + RegExpDollar() { this.getChar() = "$" } + + override string getPrimaryQLClass() { result = "RegExpDollar" } +} + +/** + * A caret assertion `^` matching the beginning of a line. + * + * Example: + * + * ``` + * ^ + * ``` + */ +class RegExpCaret extends RegExpSpecialChar { + RegExpCaret() { this.getChar() = "^" } + + override string getPrimaryQLClass() { result = "RegExpCaret" } +} + +/** + * A zero-width match, that is, either an empty group or an assertion. + * + * Examples: + * ``` + * () + * (?=\w) + * ``` + */ +class RegExpZeroWidthMatch extends RegExpGroup { + RegExpZeroWidthMatch() { re.zeroWidthMatch(start, end) } + + override RegExpTerm getChild(int i) { none() } + + override string getPrimaryQLClass() { result = "RegExpZeroWidthMatch" } +} + +/** + * A zero-width lookahead or lookbehind assertion. + * + * Examples: + * + * ``` + * (?=\w) + * (?!\n) + * (?<=\.) + * (?` + * in a regular expression. + * + * Examples: + * + * ``` + * \1 + * (?P=quote) + * ``` + */ +class RegExpBackRef extends RegExpTerm, TRegExpBackRef { + RegExpBackRef() { this = TRegExpBackRef(re, start, end) } + + /** + * Gets the number of the capture group this back reference refers to, if any. + */ + int getNumber() { result = re.getBackrefNumber(start, end) } + + /** + * Gets the name of the capture group this back reference refers to, if any. + */ + string getName() { result = re.getBackrefName(start, end) } + + /** Gets the capture group this back reference refers to. */ + RegExpGroup getGroup() { + result.getLiteral() = this.getLiteral() and + ( + result.getNumber() = this.getNumber() or + result.getName() = this.getName() + ) + } + + override RegExpTerm getChild(int i) { none() } + + override string getPrimaryQLClass() { result = "RegExpBackRef" } +} + +/** Gets the parse tree resulting from parsing `re`, if such has been constructed. */ +RegExpTerm getParsedRegExp(StringLiteral re) { result.getRegex() = re and result.isRootTerm() } diff --git a/java/ql/lib/semmle/code/java/regex/regex.qll b/java/ql/lib/semmle/code/java/regex/regex.qll new file mode 100644 index 00000000000..5dae7020fd9 --- /dev/null +++ b/java/ql/lib/semmle/code/java/regex/regex.qll @@ -0,0 +1,907 @@ +import java +import semmle.code.java.dataflow.DataFlow2 +import semmle.code.java.dataflow.ExternalFlow + +class RegexFlowConf extends DataFlow2::Configuration { + RegexFlowConf() { this = "RegexFlowConf" } + + override predicate isSource(DataFlow2::Node node) { node.asExpr() instanceof StringLiteral } + + override predicate isSink(DataFlow2::Node node) { sinkNode(node, "regex-use") } +} + +/** + * Holds if `s` is used as a regex, with the mode `mode` (if known). + * If regex mode is not known, `mode` will be `"None"`. + */ +predicate used_as_regex(Expr s, string mode) { + any(RegexFlowConf c).hasFlow(DataFlow2::exprNode(s), _) and + mode = "None" // TODO: proper mode detection +} + +/** + * A string literal that is used as a regular exprssion. + * TODO: adjust parser for java regex syntax + */ +abstract class RegexString extends Expr { + RegexString() { this instanceof StringLiteral } + + /** + * Helper predicate for `char_set_start(int start, int end)`. + * + * In order to identify left brackets ('[') which actually start a character class, + * we perform a left to right scan of the string. + * + * To avoid negative recursion we return a boolean. See `escaping`, + * the helper for `escapingChar`, for a clean use of this pattern. + * + * result is true for those start chars that actually mark a start of a char set. + */ + boolean char_set_start(int pos) { + exists(int index | + // is opening bracket + this.char_set_delimiter(index, pos) = true and + ( + // if this is the first bracket, `pos` starts a char set + index = 1 and result = true + or + // if the previous char set delimiter was not a closing bracket, `pos` does + // not start a char set. This is needed to handle cases such as `[[]` (a + // char set that matches the `[` char) + index > 1 and + not this.char_set_delimiter(index - 1, _) = false and + result = false + or + // special handling of cases such as `[][]` (the character-set of the characters `]` and `[`). + exists(int prev_closing_bracket_pos | + // previous bracket is a closing bracket + this.char_set_delimiter(index - 1, prev_closing_bracket_pos) = false and + if + // check if the character that comes before the previous closing bracket + // is an opening bracket (taking `^` into account) + exists(int pos_before_prev_closing_bracket | + if this.getChar(prev_closing_bracket_pos - 1) = "^" + then pos_before_prev_closing_bracket = prev_closing_bracket_pos - 2 + else pos_before_prev_closing_bracket = prev_closing_bracket_pos - 1 + | + this.char_set_delimiter(index - 2, pos_before_prev_closing_bracket) = true + ) + then + // brackets without anything in between is not valid character ranges, so + // the first closing bracket in `[]]` and `[^]]` does not count, + // + // and we should _not_ mark the second opening bracket in `[][]` and `[^][]` + // as starting a new char set. ^ ^ + exists(int pos_before_prev_closing_bracket | + this.char_set_delimiter(index - 2, pos_before_prev_closing_bracket) = true + | + result = this.char_set_start(pos_before_prev_closing_bracket).booleanNot() + ) + else + // if not, `pos` does in fact mark a real start of a character range + result = true + ) + ) + ) + } + + /** + * Helper predicate for chars that could be character-set delimiters. + * Holds if the (non-escaped) char at `pos` in the string, is the (one-based) `index` occurrence of a bracket (`[` or `]`) in the string. + * Result if `true` is the char is `[`, and `false` if the char is `]`. + */ + boolean char_set_delimiter(int index, int pos) { + pos = rank[index](int p | this.nonEscapedCharAt(p) = "[" or this.nonEscapedCharAt(p) = "]") and + ( + this.nonEscapedCharAt(pos) = "[" and result = true + or + this.nonEscapedCharAt(pos) = "]" and result = false + ) + } + + /** Hold is a character set starts between `start` and `end`. */ + predicate char_set_start(int start, int end) { + this.char_set_start(start) = true and + ( + this.getChar(start + 1) = "^" and end = start + 2 + or + not this.getChar(start + 1) = "^" and end = start + 1 + ) + } + + /** Whether there is a character class, between start (inclusive) and end (exclusive) */ + predicate charSet(int start, int end) { + exists(int inner_start, int inner_end | + this.char_set_start(start, inner_start) and + not this.char_set_start(_, start) + | + end = inner_end + 1 and + inner_end > inner_start and + this.nonEscapedCharAt(inner_end) = "]" and + not exists(int mid | this.nonEscapedCharAt(mid) = "]" | mid > inner_start and mid < inner_end) + ) + } + + /** An indexed version of `char_set_token/3` */ + private predicate char_set_token(int charset_start, int index, int token_start, int token_end) { + token_start = + rank[index](int start, int end | this.char_set_token(charset_start, start, end) | start) and + this.char_set_token(charset_start, token_start, token_end) + } + + /** Either a char or a - */ + private predicate char_set_token(int charset_start, int start, int end) { + this.char_set_start(charset_start, start) and + ( + this.escapedCharacter(start, end) + or + exists(this.nonEscapedCharAt(start)) and end = start + 1 + ) + or + this.char_set_token(charset_start, _, start) and + ( + this.escapedCharacter(start, end) + or + exists(this.nonEscapedCharAt(start)) and + end = start + 1 and + not this.getChar(start) = "]" + ) + } + + /** + * Holds if the character set starting at `charset_start` contains either + * a character or a range found between `start` and `end`. + */ + predicate char_set_child(int charset_start, int start, int end) { + this.char_set_token(charset_start, start, end) and + not exists(int range_start, int range_end | + this.charRange(charset_start, range_start, _, _, range_end) and + range_start <= start and + range_end >= end + ) + or + this.charRange(charset_start, start, _, _, end) + } + + /** + * Holds if the character set starting at `charset_start` contains a character range + * with lower bound found between `start` and `lower_end` + * and upper bound found between `upper_start` and `end`. + */ + predicate charRange(int charset_start, int start, int lower_end, int upper_start, int end) { + exists(int index | + this.charRangeEnd(charset_start, index) = true and + this.char_set_token(charset_start, index - 2, start, lower_end) and + this.char_set_token(charset_start, index, upper_start, end) + ) + } + + /** + * Helper predicate for `charRange`. + * We can determine where character ranges end by a left to right sweep. + * + * To avoid negative recursion we return a boolean. See `escaping`, + * the helper for `escapingChar`, for a clean use of this pattern. + */ + private boolean charRangeEnd(int charset_start, int index) { + this.char_set_token(charset_start, index, _, _) and + ( + index in [1, 2] and result = false + or + index > 2 and + exists(int connector_start | + this.char_set_token(charset_start, index - 1, connector_start, _) and + this.nonEscapedCharAt(connector_start) = "-" and + result = + this.charRangeEnd(charset_start, index - 2) + .booleanNot() + .booleanAnd(this.charRangeEnd(charset_start, index - 1).booleanNot()) + ) + or + not exists(int connector_start | + this.char_set_token(charset_start, index - 1, connector_start, _) and + this.nonEscapedCharAt(connector_start) = "-" + ) and + result = false + ) + } + + /** Holds if the character at `pos` is a "\" that is actually escaping what comes after. */ + predicate escapingChar(int pos) { this.escaping(pos) = true } + + /** + * Helper predicate for `escapingChar`. + * In order to avoid negative recusrion, we return a boolean. + * This way, we can refer to `escaping(pos - 1).booleanNot()` + * rather than to a negated version of `escaping(pos)`. + */ + private boolean escaping(int pos) { + pos = -1 and result = false + or + this.getChar(pos) = "\\" and result = this.escaping(pos - 1).booleanNot() + or + this.getChar(pos) != "\\" and result = false + } + + /** Gets the text of this regex */ + string getText() { result = this.(StringLiteral).getValue() } + + string getChar(int i) { result = this.getText().charAt(i) } + + string nonEscapedCharAt(int i) { + result = this.getText().charAt(i) and + not exists(int x, int y | this.escapedCharacter(x, y) and i in [x .. y - 1]) + } + + private predicate isOptionDivider(int i) { this.nonEscapedCharAt(i) = "|" } + + private predicate isGroupEnd(int i) { this.nonEscapedCharAt(i) = ")" and not this.inCharSet(i) } + + private predicate isGroupStart(int i) { this.nonEscapedCharAt(i) = "(" and not this.inCharSet(i) } + + predicate failedToParse(int i) { + exists(this.getChar(i)) and + not exists(int start, int end | + this.top_level(start, end) and + start <= i and + end > i + ) + } + + /** Named unicode characters, eg \N{degree sign} */ + private predicate escapedName(int start, int end) { + this.escapingChar(start) and + this.getChar(start + 1) = "N" and + this.getChar(start + 2) = "{" and + this.getChar(end - 1) = "}" and + end > start and + not exists(int i | start + 2 < i and i < end - 1 | this.getChar(i) = "}") + } + + /** + * Holds if an escaped character is found between `start` and `end`. + * Escaped characters include hex values, octal values and named escapes, + * but excludes backreferences. + */ + predicate escapedCharacter(int start, int end) { + this.escapingChar(start) and + not this.numbered_backreference(start, _, _) and + ( + // hex value \xhh + this.getChar(start + 1) = "x" and end = start + 4 + or + // octal value \o, \oo, or \ooo + end in [start + 2 .. start + 4] and + forall(int i | i in [start + 1 .. end - 1] | this.isOctal(i)) and + not ( + end < start + 4 and + this.isOctal(end) + ) + or + // 16-bit hex value \uhhhh + this.getChar(start + 1) = "u" and end = start + 6 + or + // 32-bit hex value \Uhhhhhhhh + this.getChar(start + 1) = "U" and end = start + 10 + or + escapedName(start, end) + or + // escape not handled above, update when adding a new case + not this.getChar(start + 1) in ["x", "u", "U", "N"] and + not exists(this.getChar(start + 1).toInt()) and + end = start + 2 + ) + } + + pragma[inline] + private predicate isOctal(int index) { this.getChar(index) = [0 .. 7].toString() } + + /** Holds if `index` is inside a character set. */ + predicate inCharSet(int index) { + exists(int x, int y | this.charSet(x, y) and index in [x + 1 .. y - 2]) + } + + /** + * 'simple' characters are any that don't alter the parsing of the regex. + */ + private predicate simpleCharacter(int start, int end) { + end = start + 1 and + not this.charSet(start, _) and + not this.charSet(_, start + 1) and + exists(string c | c = this.getChar(start) | + exists(int x, int y, int z | + this.charSet(x, z) and + this.char_set_start(x, y) + | + start = y + or + start = z - 2 + or + start > y and start < z - 2 and not this.charRange(_, _, start, end, _) + ) + or + not this.inCharSet(start) and + not c = "(" and + not c = "[" and + not c = ")" and + not c = "|" and + not this.qualifier(start, _, _, _) + ) + } + + predicate character(int start, int end) { + ( + this.simpleCharacter(start, end) and + not exists(int x, int y | this.escapedCharacter(x, y) and x <= start and y >= end) + or + this.escapedCharacter(start, end) + ) and + not exists(int x, int y | this.group_start(x, y) and x <= start and y >= end) and + not exists(int x, int y | this.backreference(x, y) and x <= start and y >= end) + } + + predicate normalCharacter(int start, int end) { + this.character(start, end) and + not this.specialCharacter(start, end, _) + } + + predicate specialCharacter(int start, int end, string char) { + this.character(start, end) and + end = start + 1 and + char = this.getChar(start) and + (char = "$" or char = "^" or char = ".") and + not this.inCharSet(start) + } + + /** Whether the text in the range start,end is a group */ + predicate group(int start, int end) { + this.groupContents(start, end, _, _) + or + this.emptyGroup(start, end) + } + + /** Gets the number of the group in start,end */ + int getGroupNumber(int start, int end) { + this.group(start, end) and + result = + count(int i | this.group(i, _) and i < start and not this.non_capturing_group_start(i, _)) + 1 + } + + /** Gets the name, if it has one, of the group in start,end */ + string getGroupName(int start, int end) { + this.group(start, end) and + exists(int name_end | + this.named_group_start(start, name_end) and + result = this.getText().substring(start + 4, name_end - 1) + ) + } + + /** Whether the text in the range start, end is a group and can match the empty string. */ + predicate zeroWidthMatch(int start, int end) { + this.emptyGroup(start, end) + or + this.negativeAssertionGroup(start, end) + or + this.positiveLookaheadAssertionGroup(start, end) + or + this.positiveLookbehindAssertionGroup(start, end) + } + + /** Holds if an empty group is found between `start` and `end`. */ + predicate emptyGroup(int start, int end) { + exists(int endm1 | end = endm1 + 1 | + this.group_start(start, endm1) and + this.isGroupEnd(endm1) + ) + } + + private predicate emptyMatchAtStartGroup(int start, int end) { + this.emptyGroup(start, end) + or + this.negativeAssertionGroup(start, end) + or + this.positiveLookaheadAssertionGroup(start, end) + } + + private predicate emptyMatchAtEndGroup(int start, int end) { + this.emptyGroup(start, end) + or + this.negativeAssertionGroup(start, end) + or + this.positiveLookbehindAssertionGroup(start, end) + } + + private predicate negativeAssertionGroup(int start, int end) { + exists(int in_start | + this.negative_lookahead_assertion_start(start, in_start) + or + this.negative_lookbehind_assertion_start(start, in_start) + | + this.groupContents(start, end, in_start, _) + ) + } + + /** Holds if a negative lookahead is found between `start` and `end` */ + predicate negativeLookaheadAssertionGroup(int start, int end) { + exists(int in_start | this.negative_lookahead_assertion_start(start, in_start) | + this.groupContents(start, end, in_start, _) + ) + } + + /** Holds if a negative lookbehind is found between `start` and `end` */ + predicate negativeLookbehindAssertionGroup(int start, int end) { + exists(int in_start | this.negative_lookbehind_assertion_start(start, in_start) | + this.groupContents(start, end, in_start, _) + ) + } + + /** Holds if a positive lookahead is found between `start` and `end` */ + predicate positiveLookaheadAssertionGroup(int start, int end) { + exists(int in_start | this.lookahead_assertion_start(start, in_start) | + this.groupContents(start, end, in_start, _) + ) + } + + /** Holds if a positive lookbehind is found between `start` and `end` */ + predicate positiveLookbehindAssertionGroup(int start, int end) { + exists(int in_start | this.lookbehind_assertion_start(start, in_start) | + this.groupContents(start, end, in_start, _) + ) + } + + private predicate group_start(int start, int end) { + this.non_capturing_group_start(start, end) + or + this.flag_group_start(start, end, _) + or + this.named_group_start(start, end) + or + this.named_backreference_start(start, end) + or + this.lookahead_assertion_start(start, end) + or + this.negative_lookahead_assertion_start(start, end) + or + this.lookbehind_assertion_start(start, end) + or + this.negative_lookbehind_assertion_start(start, end) + or + this.comment_group_start(start, end) + or + this.simple_group_start(start, end) + } + + private predicate non_capturing_group_start(int start, int end) { + this.isGroupStart(start) and + this.getChar(start + 1) = "?" and + this.getChar(start + 2) = ":" and + end = start + 3 + } + + private predicate simple_group_start(int start, int end) { + this.isGroupStart(start) and + this.getChar(start + 1) != "?" and + end = start + 1 + } + + private predicate named_group_start(int start, int end) { + this.isGroupStart(start) and + this.getChar(start + 1) = "?" and + this.getChar(start + 2) = "P" and + this.getChar(start + 3) = "<" and + not this.getChar(start + 4) = "=" and + not this.getChar(start + 4) = "!" and + exists(int name_end | + name_end = min(int i | i > start + 4 and this.getChar(i) = ">") and + end = name_end + 1 + ) + } + + private predicate named_backreference_start(int start, int end) { + this.isGroupStart(start) and + this.getChar(start + 1) = "?" and + this.getChar(start + 2) = "P" and + this.getChar(start + 3) = "=" and + // Should this be looking for unescaped ")"? + // TODO: test this + end = min(int i | i > start + 4 and this.getChar(i) = "?") + } + + private predicate flag_group_start(int start, int end, string c) { + this.isGroupStart(start) and + this.getChar(start + 1) = "?" and + end = start + 3 and + c = this.getChar(start + 2) and + c in ["i", "L", "m", "s", "u", "x"] + } + + /** + * Gets the mode of this regular expression string if + * it is defined by a prefix. + */ + string getModeFromPrefix() { + exists(string c | this.flag_group_start(_, _, c) | + c = "i" and result = "IGNORECASE" + or + c = "L" and result = "LOCALE" + or + c = "m" and result = "MULTILINE" + or + c = "s" and result = "DOTALL" + or + c = "u" and result = "UNICODE" + or + c = "x" and result = "VERBOSE" + ) + } + + private predicate lookahead_assertion_start(int start, int end) { + this.isGroupStart(start) and + this.getChar(start + 1) = "?" and + this.getChar(start + 2) = "=" and + end = start + 3 + } + + private predicate negative_lookahead_assertion_start(int start, int end) { + this.isGroupStart(start) and + this.getChar(start + 1) = "?" and + this.getChar(start + 2) = "!" and + end = start + 3 + } + + private predicate lookbehind_assertion_start(int start, int end) { + this.isGroupStart(start) and + this.getChar(start + 1) = "?" and + this.getChar(start + 2) = "<" and + this.getChar(start + 3) = "=" and + end = start + 4 + } + + private predicate negative_lookbehind_assertion_start(int start, int end) { + this.isGroupStart(start) and + this.getChar(start + 1) = "?" and + this.getChar(start + 2) = "<" and + this.getChar(start + 3) = "!" and + end = start + 4 + } + + private predicate comment_group_start(int start, int end) { + this.isGroupStart(start) and + this.getChar(start + 1) = "?" and + this.getChar(start + 2) = "#" and + end = start + 3 + } + + predicate groupContents(int start, int end, int in_start, int in_end) { + this.group_start(start, in_start) and + end = in_end + 1 and + this.top_level(in_start, in_end) and + this.isGroupEnd(in_end) + } + + private predicate named_backreference(int start, int end, string name) { + this.named_backreference_start(start, start + 4) and + end = min(int i | i > start + 4 and this.getChar(i) = ")") + 1 and + name = this.getText().substring(start + 4, end - 2) + } + + private predicate numbered_backreference(int start, int end, int value) { + this.escapingChar(start) and + // starting with 0 makes it an octal escape + not this.getChar(start + 1) = "0" and + exists(string text, string svalue, int len | + end = start + len and + text = this.getText() and + len in [2 .. 3] + | + svalue = text.substring(start + 1, start + len) and + value = svalue.toInt() and + // value is composed of digits + forall(int i | i in [start + 1 .. start + len - 1] | this.getChar(i) = [0 .. 9].toString()) and + // a longer reference is not possible + not ( + len = 2 and + exists(text.substring(start + 1, start + len + 1).toInt()) + ) and + // 3 octal digits makes it an octal escape + not forall(int i | i in [start + 1 .. start + 4] | this.isOctal(i)) + // TODO: Inside a character set, all numeric escapes are treated as characters. + ) + } + + /** Whether the text in the range start,end is a back reference */ + predicate backreference(int start, int end) { + this.numbered_backreference(start, end, _) + or + this.named_backreference(start, end, _) + } + + /** Gets the number of the back reference in start,end */ + int getBackrefNumber(int start, int end) { this.numbered_backreference(start, end, result) } + + /** Gets the name, if it has one, of the back reference in start,end */ + string getBackrefName(int start, int end) { this.named_backreference(start, end, result) } + + private predicate baseItem(int start, int end) { + this.character(start, end) and + not exists(int x, int y | this.charSet(x, y) and x <= start and y >= end) + or + this.group(start, end) + or + this.charSet(start, end) + or + this.backreference(start, end) + } + + private predicate qualifier(int start, int end, boolean maybe_empty, boolean may_repeat_forever) { + this.short_qualifier(start, end, maybe_empty, may_repeat_forever) and + not this.getChar(end) = "?" + or + exists(int short_end | this.short_qualifier(start, short_end, maybe_empty, may_repeat_forever) | + if this.getChar(short_end) = "?" then end = short_end + 1 else end = short_end + ) + } + + private predicate short_qualifier( + int start, int end, boolean maybe_empty, boolean may_repeat_forever + ) { + ( + this.getChar(start) = "+" and maybe_empty = false and may_repeat_forever = true + or + this.getChar(start) = "*" and maybe_empty = true and may_repeat_forever = true + or + this.getChar(start) = "?" and maybe_empty = true and may_repeat_forever = false + ) and + end = start + 1 + or + exists(string lower, string upper | + this.multiples(start, end, lower, upper) and + (if lower = "" or lower.toInt() = 0 then maybe_empty = true else maybe_empty = false) and + if upper = "" then may_repeat_forever = true else may_repeat_forever = false + ) + } + + /** + * Holds if a repetition quantifier is found between `start` and `end`, + * with the given lower and upper bounds. If a bound is omitted, the corresponding + * string is empty. + */ + predicate multiples(int start, int end, string lower, string upper) { + exists(string text, string match, string inner | + text = this.getText() and + end = start + match.length() and + inner = match.substring(1, match.length() - 1) + | + match = text.regexpFind("\\{[0-9]+\\}", _, start) and + lower = inner and + upper = lower + or + match = text.regexpFind("\\{[0-9]*,[0-9]*\\}", _, start) and + exists(int commaIndex | + commaIndex = inner.indexOf(",") and + lower = inner.prefix(commaIndex) and + upper = inner.suffix(commaIndex + 1) + ) + ) + } + + /** + * Whether the text in the range start,end is a qualified item, where item is a character, + * a character set or a group. + */ + predicate qualifiedItem(int start, int end, boolean maybe_empty, boolean may_repeat_forever) { + this.qualifiedPart(start, _, end, maybe_empty, may_repeat_forever) + } + + /** + * Holds if a qualified part is found between `start` and `part_end` and the qualifier is + * found between `part_end` and `end`. + * + * `maybe_empty` is true if the part is optional. + * `may_repeat_forever` is true if the part may be repeated unboundedly. + */ + predicate qualifiedPart( + int start, int part_end, int end, boolean maybe_empty, boolean may_repeat_forever + ) { + this.baseItem(start, part_end) and + this.qualifier(part_end, end, maybe_empty, may_repeat_forever) + } + + /** Holds if the range `start`, `end` contains a character, a quantifier, a character set or a group. */ + predicate item(int start, int end) { + this.qualifiedItem(start, end, _, _) + or + this.baseItem(start, end) and not this.qualifier(end, _, _, _) + } + + private predicate subsequence(int start, int end) { + ( + start = 0 or + this.group_start(_, start) or + this.isOptionDivider(start - 1) + ) and + this.item(start, end) + or + exists(int mid | + this.subsequence(start, mid) and + this.item(mid, end) + ) + } + + /** + * Whether the text in the range start,end is a sequence of 1 or more items, where an item is a character, + * a character set or a group. + */ + predicate sequence(int start, int end) { + this.sequenceOrQualified(start, end) and + not this.qualifiedItem(start, end, _, _) + } + + private predicate sequenceOrQualified(int start, int end) { + this.subsequence(start, end) and + not this.item_start(end) + } + + private predicate item_start(int start) { + this.character(start, _) or + this.isGroupStart(start) or + this.charSet(start, _) or + this.backreference(start, _) + } + + private predicate item_end(int end) { + this.character(_, end) + or + exists(int endm1 | this.isGroupEnd(endm1) and end = endm1 + 1) + or + this.charSet(_, end) + or + this.qualifier(_, end, _, _) + } + + private predicate top_level(int start, int end) { + this.subalternation(start, end, _) and + not this.isOptionDivider(end) + } + + private predicate subalternation(int start, int end, int item_start) { + this.sequenceOrQualified(start, end) and + not this.isOptionDivider(start - 1) and + item_start = start + or + start = end and + not this.item_end(start) and + this.isOptionDivider(end) and + item_start = start + or + exists(int mid | + this.subalternation(start, mid, _) and + this.isOptionDivider(mid) and + item_start = mid + 1 + | + this.sequenceOrQualified(item_start, end) + or + not this.item_start(end) and end = item_start + ) + } + + /** + * Whether the text in the range start,end is an alternation + */ + predicate alternation(int start, int end) { + this.top_level(start, end) and + exists(int less | this.subalternation(start, less, _) and less < end) + } + + /** + * Whether the text in the range start,end is an alternation and the text in part_start, part_end is one of the + * options in that alternation. + */ + predicate alternationOption(int start, int end, int part_start, int part_end) { + this.alternation(start, end) and + this.subalternation(start, part_end, part_start) + } + + /** A part of the regex that may match the start of the string. */ + private predicate firstPart(int start, int end) { + start = 0 and end = this.getText().length() + or + exists(int x | this.firstPart(x, end) | + this.emptyMatchAtStartGroup(x, start) or + this.qualifiedItem(x, start, true, _) or + this.specialCharacter(x, start, "^") + ) + or + exists(int y | this.firstPart(start, y) | + this.item(start, end) + or + this.qualifiedPart(start, end, y, _, _) + ) + or + exists(int x, int y | this.firstPart(x, y) | + this.groupContents(x, y, start, end) + or + this.alternationOption(x, y, start, end) + ) + } + + /** A part of the regex that may match the end of the string. */ + private predicate lastPart(int start, int end) { + start = 0 and end = this.getText().length() + or + exists(int y | this.lastPart(start, y) | + this.emptyMatchAtEndGroup(end, y) + or + this.qualifiedItem(end, y, true, _) + or + this.specialCharacter(end, y, "$") + or + y = end + 2 and this.escapingChar(end) and this.getChar(end + 1) = "Z" + ) + or + exists(int x | + this.lastPart(x, end) and + this.item(start, end) + ) + or + exists(int y | this.lastPart(start, y) | this.qualifiedPart(start, end, y, _, _)) + or + exists(int x, int y | this.lastPart(x, y) | + this.groupContents(x, y, start, end) + or + this.alternationOption(x, y, start, end) + ) + } + + /** + * Whether the item at [start, end) is one of the first items + * to be matched. + */ + predicate firstItem(int start, int end) { + ( + this.character(start, end) + or + this.qualifiedItem(start, end, _, _) + or + this.charSet(start, end) + ) and + this.firstPart(start, end) + } + + /** + * Whether the item at [start, end) is one of the last items + * to be matched. + */ + predicate lastItem(int start, int end) { + ( + this.character(start, end) + or + this.qualifiedItem(start, end, _, _) + or + this.charSet(start, end) + ) and + this.lastPart(start, end) + } +} + +/** A string literal used as a regular expression */ +class Regex extends RegexString { + Regex() { used_as_regex(this, _) } + + /** + * Gets a mode (if any) of this regular expression. Can be any of: + * DEBUG + * IGNORECASE + * LOCALE + * MULTILINE + * DOTALL + * UNICODE + * VERBOSE + */ + string getAMode() { + result != "None" and + used_as_regex(this, result) + or + result = this.getModeFromPrefix() + } +} diff --git a/java/ql/lib/semmle/code/java/security/performance/ExponentialBackTracking.qll b/java/ql/lib/semmle/code/java/security/performance/ExponentialBackTracking.qll new file mode 100644 index 00000000000..8d308a93104 --- /dev/null +++ b/java/ql/lib/semmle/code/java/security/performance/ExponentialBackTracking.qll @@ -0,0 +1,342 @@ +/** + * This library implements the analysis described in the following two papers: + * + * James Kirrage, Asiri Rathnayake, Hayo Thielecke: Static Analysis for + * Regular Expression Denial-of-Service Attacks. NSS 2013. + * (http://www.cs.bham.ac.uk/~hxt/research/reg-exp-sec.pdf) + * Asiri Rathnayake, Hayo Thielecke: Static Analysis for Regular Expression + * Exponential Runtime via Substructural Logics. 2014. + * (https://www.cs.bham.ac.uk/~hxt/research/redos_full.pdf) + * + * The basic idea is to search for overlapping cycles in the NFA, that is, + * states `q` such that there are two distinct paths from `q` to itself + * that consume the same word `w`. + * + * For any such state `q`, an attack string can be constructed as follows: + * concatenate a prefix `v` that takes the NFA to `q` with `n` copies of + * the word `w` that leads back to `q` along two different paths, followed + * by a suffix `x` that is _not_ accepted in state `q`. A backtracking + * implementation will need to explore at least 2^n different ways of going + * from `q` back to itself while trying to match the `n` copies of `w` + * before finally giving up. + * + * Now in order to identify overlapping cycles, all we have to do is find + * pumpable forks, that is, states `q` that can transition to two different + * states `r1` and `r2` on the same input symbol `c`, such that there are + * paths from both `r1` and `r2` to `q` that consume the same word. The latter + * condition is equivalent to saying that `(q, q)` is reachable from `(r1, r2)` + * in the product NFA. + * + * This is what the library does. It makes a simple attempt to construct a + * prefix `v` leading into `q`, but only to improve the alert message. + * And the library tries to prove the existence of a suffix that ensures + * rejection. This check might fail, which can cause false positives. + * + * Finally, sometimes it depends on the translation whether the NFA generated + * for a regular expression has a pumpable fork or not. We implement one + * particular translation, which may result in false positives or negatives + * relative to some particular JavaScript engine. + * + * More precisely, the library constructs an NFA from a regular expression `r` + * as follows: + * + * * Every sub-term `t` gives rise to an NFA state `Match(t,i)`, representing + * the state of the automaton before attempting to match the `i`th character in `t`. + * * There is one accepting state `Accept(r)`. + * * There is a special `AcceptAnySuffix(r)` state, which accepts any suffix string + * by using an epsilon transition to `Accept(r)` and an any transition to itself. + * * Transitions between states may be labelled with epsilon, or an abstract + * input symbol. + * * Each abstract input symbol represents a set of concrete input characters: + * either a single character, a set of characters represented by a + * character class, or the set of all characters. + * * The product automaton is constructed lazily, starting with pair states + * `(q, q)` where `q` is a fork, and proceding along an over-approximate + * step relation. + * * The over-approximate step relation allows transitions along pairs of + * abstract input symbols where the symbols have overlap in the characters they accept. + * * Once a trace of pairs of abstract input symbols that leads from a fork + * back to itself has been identified, we attempt to construct a concrete + * string corresponding to it, which may fail. + * * Lastly we ensure that any state reached by repeating `n` copies of `w` has + * a suffix `x` (possible empty) that is most likely __not__ accepted. + */ + +import ReDoSUtil + +/** + * Holds if state `s` might be inside a backtracking repetition. + */ +pragma[noinline] +private predicate stateInsideBacktracking(State s) { + s.getRepr().getParent*() instanceof MaybeBacktrackingRepetition +} + +/** + * A infinitely repeating quantifier that might backtrack. + */ +private class MaybeBacktrackingRepetition extends InfiniteRepetitionQuantifier { + MaybeBacktrackingRepetition() { + exists(RegExpTerm child | + child instanceof RegExpAlt or + child instanceof RegExpQuantifier + | + child.getParent+() = this + ) + } +} + +/** + * A state in the product automaton. + */ +private newtype TStatePair = + /** + * We lazily only construct those states that we are actually + * going to need: `(q, q)` for every fork state `q`, and any + * pair of states that can be reached from a pair that we have + * already constructed. To cut down on the number of states, + * we only represent states `(q1, q2)` where `q1` is lexicographically + * no bigger than `q2`. + * + * States are only constructed if both states in the pair are + * inside a repetition that might backtrack. + */ + MkStatePair(State q1, State q2) { + isFork(q1, _, _, _, _) and q2 = q1 + or + (step(_, _, _, q1, q2) or step(_, _, _, q2, q1)) and + rankState(q1) <= rankState(q2) + } + +/** + * Gets a unique number for a `state`. + * Is used to create an ordering of states, where states with the same `toString()` will be ordered differently. + */ +private int rankState(State state) { + state = + rank[result](State s, Location l | + l = s.getRepr().getLocation() + | + s order by l.getStartLine(), l.getStartColumn(), s.toString() + ) +} + +/** + * A state in the product automaton. + */ +private class StatePair extends TStatePair { + State q1; + State q2; + + StatePair() { this = MkStatePair(q1, q2) } + + /** Gets a textual representation of this element. */ + string toString() { result = "(" + q1 + ", " + q2 + ")" } + + /** Gets the first component of the state pair. */ + State getLeft() { result = q1 } + + /** Gets the second component of the state pair. */ + State getRight() { result = q2 } +} + +/** + * Holds for all constructed state pairs. + * + * Used in `statePairDist` + */ +private predicate isStatePair(StatePair p) { any() } + +/** + * Holds if there are transitions from the components of `q` to the corresponding + * components of `r`. + * + * Used in `statePairDist` + */ +private predicate delta2(StatePair q, StatePair r) { step(q, _, _, r) } + +/** + * Gets the minimum length of a path from `q` to `r` in the + * product automaton. + */ +private int statePairDist(StatePair q, StatePair r) = + shortestDistances(isStatePair/1, delta2/2)(q, r, result) + +/** + * Holds if there are transitions from `q` to `r1` and from `q` to `r2` + * labelled with `s1` and `s2`, respectively, where `s1` and `s2` do not + * trivially have an empty intersection. + * + * This predicate only holds for states associated with regular expressions + * that have at least one repetition quantifier in them (otherwise the + * expression cannot be vulnerable to ReDoS attacks anyway). + */ +pragma[noopt] +private predicate isFork(State q, InputSymbol s1, InputSymbol s2, State r1, State r2) { + stateInsideBacktracking(q) and + exists(State q1, State q2 | + q1 = epsilonSucc*(q) and + delta(q1, s1, r1) and + q2 = epsilonSucc*(q) and + delta(q2, s2, r2) and + // Use pragma[noopt] to prevent intersect(s1,s2) from being the starting point of the join. + // From (s1,s2) it would find a huge number of intermediate state pairs (q1,q2) originating from different literals, + // and discover at the end that no `q` can reach both `q1` and `q2` by epsilon transitions. + exists(intersect(s1, s2)) + | + s1 != s2 + or + r1 != r2 + or + r1 = r2 and q1 != q2 + or + // If q can reach itself by epsilon transitions, then there are two distinct paths to the q1/q2 state: + // one that uses the loop and one that doesn't. The engine will separately attempt to match with each path, + // despite ending in the same state. The "fork" thus arises from the choice of whether to use the loop or not. + // To avoid every state in the loop becoming a fork state, + // we arbitrarily pick the InfiniteRepetitionQuantifier state as the canonical fork state for the loop + // (every epsilon-loop must contain such a state). + // + // We additionally require that the there exists another InfiniteRepetitionQuantifier `mid` on the path from `q` to itself. + // This is done to avoid flagging regular expressions such as `/(a?)*b/` - that only has polynomial runtime, and is detected by `js/polynomial-redos`. + // The below code is therefore a heuritic, that only flags regular expressions such as `/(a*)*b/`, + // and does not flag regular expressions such as `/(a?b?)c/`, but the latter pattern is not used frequently. + r1 = r2 and + q1 = q2 and + epsilonSucc+(q) = q and + exists(RegExpTerm term | term = q.getRepr() | term instanceof InfiniteRepetitionQuantifier) and + // One of the mid states is an infinite quantifier itself + exists(State mid, RegExpTerm term | + mid = epsilonSucc+(q) and + term = mid.getRepr() and + term instanceof InfiniteRepetitionQuantifier and + q = epsilonSucc+(mid) and + not mid = q + ) + ) and + stateInsideBacktracking(r1) and + stateInsideBacktracking(r2) +} + +/** + * Gets the state pair `(q1, q2)` or `(q2, q1)`; note that only + * one or the other is defined. + */ +private StatePair mkStatePair(State q1, State q2) { + result = MkStatePair(q1, q2) or result = MkStatePair(q2, q1) +} + +/** + * Holds if there are transitions from the components of `q` to the corresponding + * components of `r` labelled with `s1` and `s2`, respectively. + */ +private predicate step(StatePair q, InputSymbol s1, InputSymbol s2, StatePair r) { + exists(State r1, State r2 | step(q, s1, s2, r1, r2) and r = mkStatePair(r1, r2)) +} + +/** + * Holds if there are transitions from the components of `q` to `r1` and `r2` + * labelled with `s1` and `s2`, respectively. + * + * We only consider transitions where the resulting states `(r1, r2)` are both + * inside a repetition that might backtrack. + */ +pragma[noopt] +private predicate step(StatePair q, InputSymbol s1, InputSymbol s2, State r1, State r2) { + exists(State q1, State q2 | q.getLeft() = q1 and q.getRight() = q2 | + deltaClosed(q1, s1, r1) and + deltaClosed(q2, s2, r2) and + // use noopt to force the join on `intersect` to happen last. + exists(intersect(s1, s2)) + ) and + stateInsideBacktracking(r1) and + stateInsideBacktracking(r2) +} + +private newtype TTrace = + Nil() or + Step(InputSymbol s1, InputSymbol s2, TTrace t) { + exists(StatePair p | + isReachableFromFork(_, p, t, _) and + step(p, s1, s2, _) + ) + or + t = Nil() and isFork(_, s1, s2, _, _) + } + +/** + * A list of pairs of input symbols that describe a path in the product automaton + * starting from some fork state. + */ +private class Trace extends TTrace { + /** Gets a textual representation of this element. */ + string toString() { + this = Nil() and result = "Nil()" + or + exists(InputSymbol s1, InputSymbol s2, Trace t | this = Step(s1, s2, t) | + result = "Step(" + s1 + ", " + s2 + ", " + t + ")" + ) + } +} + +/** + * Gets a string corresponding to the trace `t`. + */ +private string concretise(Trace t) { + t = Nil() and result = "" + or + exists(InputSymbol s1, InputSymbol s2, Trace rest | t = Step(s1, s2, rest) | + result = concretise(rest) + intersect(s1, s2) + ) +} + +/** + * Holds if `r` is reachable from `(fork, fork)` under input `w`, and there is + * a path from `r` back to `(fork, fork)` with `rem` steps. + */ +private predicate isReachableFromFork(State fork, StatePair r, Trace w, int rem) { + // base case + exists(InputSymbol s1, InputSymbol s2, State q1, State q2 | + isFork(fork, s1, s2, q1, q2) and + r = MkStatePair(q1, q2) and + w = Step(s1, s2, Nil()) and + rem = statePairDist(r, MkStatePair(fork, fork)) + ) + or + // recursive case + exists(StatePair p, Trace v, InputSymbol s1, InputSymbol s2 | + isReachableFromFork(fork, p, v, rem + 1) and + step(p, s1, s2, r) and + w = Step(s1, s2, v) and + rem >= statePairDist(r, MkStatePair(fork, fork)) + ) +} + +/** + * Gets a state in the product automaton from which `(fork, fork)` is + * reachable in zero or more epsilon transitions. + */ +private StatePair getAForkPair(State fork) { + isFork(fork, _, _, _, _) and + result = MkStatePair(epsilonPred*(fork), epsilonPred*(fork)) +} + +/** + * Holds if `fork` is a pumpable fork with word `w`. + */ +private predicate isPumpable(State fork, string w) { + exists(StatePair q, Trace t | + isReachableFromFork(fork, q, t, _) and + q = getAForkPair(fork) and + w = concretise(t) + ) +} + +/** + * An instantiation of `ReDoSConfiguration` for exponential backtracking. + */ +class ExponentialReDoSConfiguration extends ReDoSConfiguration { + ExponentialReDoSConfiguration() { this = "ExponentialReDoSConfiguration" } + + override predicate isReDoSCandidate(State state, string pump) { isPumpable(state, pump) } +} diff --git a/java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll b/java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll new file mode 100644 index 00000000000..2cd324ed8f7 --- /dev/null +++ b/java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll @@ -0,0 +1,1135 @@ +/** + * Provides classes for working with regular expressions that can + * perform backtracking in superlinear/exponential time. + * + * This module contains a number of utility predicates for compiling a regular expression into a NFA and reasoning about this NFA. + * + * The `ReDoSConfiguration` contains a `isReDoSCandidate` predicate that is used to + * to determine which states the prefix/suffix search should happen on. + * There is only meant to exist one `ReDoSConfiguration` at a time. + * + * The predicate `hasReDoSResult` outputs a de-duplicated set of + * states that will cause backtracking (a rejecting suffix exists). + */ + +import RegExpTreeView + +/** + * A configuration for which parts of a regular expression should be considered relevant for + * the different predicates in `ReDoS.qll`. + * Used to adjust the computations for either superlinear or exponential backtracking. + */ +abstract class ReDoSConfiguration extends string { + bindingset[this] + ReDoSConfiguration() { any() } + + /** + * Holds if `state` with the pump string `pump` is a candidate for a + * ReDoS vulnerable state. + * This is used to determine which states are considered for the prefix/suffix construction. + */ + abstract predicate isReDoSCandidate(State state, string pump); +} + +/** + * Holds if repeating `pump' starting at `state` is a candidate for causing backtracking. + * No check whether a rejected suffix exists has been made. + */ +private predicate isReDoSCandidate(State state, string pump) { + any(ReDoSConfiguration conf).isReDoSCandidate(state, pump) and + ( + not any(ReDoSConfiguration conf).isReDoSCandidate(epsilonSucc+(state), _) + or + epsilonSucc+(state) = state and + state = + max(State s, Location l | + s = epsilonSucc+(state) and + l = s.getRepr().getLocation() and + any(ReDoSConfiguration conf).isReDoSCandidate(s, _) and + s.getRepr() instanceof InfiniteRepetitionQuantifier + | + s order by l.getStartLine(), l.getStartColumn(), l.getEndColumn(), l.getEndLine() + ) + ) +} + +/** + * Gets the char after `c` (from a simplified ASCII table). + */ +private string nextChar(string c) { exists(int code | code = ascii(c) | code + 1 = ascii(result)) } + +/** + * Gets an approximation for the ASCII code for `char`. + * Only the easily printable chars are included (so no newline, tab, null, etc). + */ +private int ascii(string char) { + char = + rank[result](string c | + c = + "! \"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~" + .charAt(_) + ) +} + +/** + * Holds if `t` matches at least an epsilon symbol. + * + * That is, this term does not restrict the language of the enclosing regular expression. + * + * This is implemented as an under-approximation, and this predicate does not hold for sub-patterns in particular. + */ +predicate matchesEpsilon(RegExpTerm t) { + t instanceof RegExpStar + or + t instanceof RegExpOpt + or + t.(RegExpRange).getLowerBound() = 0 + or + exists(RegExpTerm child | + child = t.getAChild() and + matchesEpsilon(child) + | + t instanceof RegExpAlt or + t instanceof RegExpGroup or + t instanceof RegExpPlus or + t instanceof RegExpRange + ) + or + matchesEpsilon(t.(RegExpBackRef).getGroup()) + or + forex(RegExpTerm child | child = t.(RegExpSequence).getAChild() | matchesEpsilon(child)) +} + +/** + * A lookahead/lookbehind that matches the empty string. + */ +class EmptyPositiveSubPatttern extends RegExpSubPattern { + EmptyPositiveSubPatttern() { + ( + this instanceof RegExpPositiveLookahead + or + this instanceof RegExpPositiveLookbehind + ) and + matchesEpsilon(this.getOperand()) + } +} + +/** + * A branch in a disjunction that is the root node in a literal, or a literal + * whose root node is not a disjunction. + */ +class RegExpRoot extends RegExpTerm { + RegExpParent parent; + + RegExpRoot() { + exists(RegExpAlt alt | + alt.isRootTerm() and + this = alt.getAChild() and + parent = alt.getParent() + ) + or + this.isRootTerm() and + not this instanceof RegExpAlt and + parent = this.getParent() + } + + /** + * Holds if this root term is relevant to the ReDoS analysis. + */ + predicate isRelevant() { + // there is at least one repetition + getRoot(any(InfiniteRepetitionQuantifier q)) = this and + // is actually used as a RegExp + isUsedAsRegExp() and + // not excluded for library specific reasons + not isExcluded(getRootTerm().getParent()) + } +} + +/** + * A constant in a regular expression that represents valid Unicode character(s). + */ +private class RegexpCharacterConstant extends RegExpConstant { + RegexpCharacterConstant() { this.isCharacter() } +} + +/** + * A regexp term that is relevant for this ReDoS analysis. + */ +class RelevantRegExpTerm extends RegExpTerm { + RelevantRegExpTerm() { getRoot(this).isRelevant() } +} + +/** + * Holds if `term` is the chosen canonical representative for all terms with string representation `str`. + * The string representation includes which flags are used with the regular expression. + * + * Using canonical representatives gives a huge performance boost when working with tuples containing multiple `InputSymbol`s. + * The number of `InputSymbol`s is decreased by 3 orders of magnitude or more in some larger benchmarks. + */ +private predicate isCanonicalTerm(RelevantRegExpTerm term, string str) { + term = + min(RelevantRegExpTerm t, Location loc, File file | + loc = t.getLocation() and + file = t.getFile() and + str = t.getRawValue() + "|" + getCanonicalizationFlags(t.getRootTerm()) + | + t order by t.getFile().getRelativePath(), loc.getStartLine(), loc.getStartColumn() + ) +} + +/** + * Gets a string reperesentation of the flags used with the regular expression. + * Only the flags that are relevant for the canonicalization are included. + */ +string getCanonicalizationFlags(RegExpTerm root) { + root.isRootTerm() and + (if RegExpFlags::isIgnoreCase(root) then result = "i" else result = "") +} + +/** + * An abstract input symbol, representing a set of concrete characters. + */ +private newtype TInputSymbol = + /** An input symbol corresponding to character `c`. */ + Char(string c) { + c = + any(RegexpCharacterConstant cc | + cc instanceof RelevantRegExpTerm and + not RegExpFlags::isIgnoreCase(cc.getRootTerm()) + ).getValue().charAt(_) + or + // normalize everything to lower case if the regexp is case insensitive + c = + any(RegexpCharacterConstant cc, string char | + cc instanceof RelevantRegExpTerm and + RegExpFlags::isIgnoreCase(cc.getRootTerm()) and + char = cc.getValue().charAt(_) + | + char.toLowerCase() + ) + } or + /** + * An input symbol representing all characters matched by + * a (non-universal) character class that has string representation `charClassString`. + */ + CharClass(string charClassString) { + exists(RelevantRegExpTerm recc | isCanonicalTerm(recc, charClassString) | + recc instanceof RegExpCharacterClass and + not recc.(RegExpCharacterClass).isUniversalClass() + or + recc instanceof RegExpCharacterClassEscape + ) + } or + /** An input symbol representing all characters matched by `.`. */ + Dot() or + /** An input symbol representing all characters. */ + Any() or + /** An epsilon transition in the automaton. */ + Epsilon() + +/** + * Gets the canonical CharClass for `term`. + */ +CharClass getCanonicalCharClass(RegExpTerm term) { + exists(string str | isCanonicalTerm(term, str) | result = CharClass(str)) +} + +/** + * Holds if `a` and `b` are input symbols from the same regexp. + */ +private predicate sharesRoot(TInputSymbol a, TInputSymbol b) { + exists(RegExpRoot root | + belongsTo(a, root) and + belongsTo(b, root) + ) +} + +/** + * Holds if the `a` is an input symbol from a regexp that has root `root`. + */ +private predicate belongsTo(TInputSymbol a, RegExpRoot root) { + exists(State s | getRoot(s.getRepr()) = root | + delta(s, a, _) + or + delta(_, a, s) + ) +} + +/** + * An abstract input symbol, representing a set of concrete characters. + */ +class InputSymbol extends TInputSymbol { + InputSymbol() { not this instanceof Epsilon } + + /** + * Gets a string representation of this input symbol. + */ + string toString() { + this = Char(result) + or + this = CharClass(result) + or + this = Dot() and result = "." + or + this = Any() and result = "[^]" + } +} + +/** + * An abstract input symbol that represents a character class. + */ +abstract class CharacterClass extends InputSymbol { + /** + * Gets a character that is relevant for intersection-tests involving this + * character class. + * + * Specifically, this is any of the characters mentioned explicitly in the + * character class, offset by one if it is inverted. For character class escapes, + * the result is as if the class had been written out as a series of intervals. + * + * This set is large enough to ensure that for any two intersecting character + * classes, one contains a relevant character from the other. + */ + abstract string getARelevantChar(); + + /** + * Holds if this character class matches `char`. + */ + bindingset[char] + abstract predicate matches(string char); + + /** + * Gets a character matched by this character class. + */ + string choose() { result = getARelevantChar() and matches(result) } +} + +/** + * Provides implementations for `CharacterClass`. + */ +private module CharacterClasses { + /** + * Holds if the character class `cc` has a child (constant or range) that matches `char`. + */ + pragma[noinline] + predicate hasChildThatMatches(RegExpCharacterClass cc, string char) { + if RegExpFlags::isIgnoreCase(cc.getRootTerm()) + then + // normalize everything to lower case if the regexp is case insensitive + exists(string c | hasChildThatMatchesIgnoringCasingFlags(cc, c) | char = c.toLowerCase()) + else hasChildThatMatchesIgnoringCasingFlags(cc, char) + } + + /** + * Holds if the character class `cc` has a child (constant or range) that matches `char`. + * Ignores whether the character class is inside a regular expression that has the ignore case flag. + */ + pragma[noinline] + predicate hasChildThatMatchesIgnoringCasingFlags(RegExpCharacterClass cc, string char) { + exists(getCanonicalCharClass(cc)) and + exists(RegExpTerm child | child = cc.getAChild() | + char = child.(RegexpCharacterConstant).getValue() + or + rangeMatchesOnLetterOrDigits(child, char) + or + not rangeMatchesOnLetterOrDigits(child, _) and + char = getARelevantChar() and + exists(string lo, string hi | child.(RegExpCharacterRange).isRange(lo, hi) | + lo <= char and + char <= hi + ) + or + exists(RegExpCharacterClassEscape escape | escape = child | + escape.getValue() = escape.getValue().toLowerCase() and + classEscapeMatches(escape.getValue(), char) + or + char = getARelevantChar() and + escape.getValue() = escape.getValue().toUpperCase() and + not classEscapeMatches(escape.getValue().toLowerCase(), char) + ) + ) + } + + /** + * Holds if `range` is a range on lower-case, upper-case, or digits, and matches `char`. + * This predicate is used to restrict the searchspace for ranges by only joining `getAnyPossiblyMatchedChar` + * on a few ranges. + */ + private predicate rangeMatchesOnLetterOrDigits(RegExpCharacterRange range, string char) { + exists(string lo, string hi | + range.isRange(lo, hi) and lo = lowercaseLetter() and hi = lowercaseLetter() + | + lo <= char and + char <= hi and + char = lowercaseLetter() + ) + or + exists(string lo, string hi | + range.isRange(lo, hi) and lo = upperCaseLetter() and hi = upperCaseLetter() + | + lo <= char and + char <= hi and + char = upperCaseLetter() + ) + or + exists(string lo, string hi | range.isRange(lo, hi) and lo = digit() and hi = digit() | + lo <= char and + char <= hi and + char = digit() + ) + } + + private string lowercaseLetter() { result = "abdcefghijklmnopqrstuvwxyz".charAt(_) } + + private string upperCaseLetter() { result = "ABCDEFGHIJKLMNOPQRSTUVWXYZ".charAt(_) } + + private string digit() { result = [0 .. 9].toString() } + + /** + * Gets a char that could be matched by a regular expression. + * Includes all printable ascii chars, all constants mentioned in a regexp, and all chars matches by the regexp `/\s|\d|\w/`. + */ + string getARelevantChar() { + exists(ascii(result)) + or + exists(RegexpCharacterConstant c | result = c.getValue().charAt(_)) + or + classEscapeMatches(_, result) + } + + /** + * Gets a char that is mentioned in the character class `c`. + */ + private string getAMentionedChar(RegExpCharacterClass c) { + exists(RegExpTerm child | child = c.getAChild() | + result = child.(RegexpCharacterConstant).getValue() + or + child.(RegExpCharacterRange).isRange(result, _) + or + child.(RegExpCharacterRange).isRange(_, result) + or + exists(RegExpCharacterClassEscape escape | child = escape | + result = min(string s | classEscapeMatches(escape.getValue().toLowerCase(), s)) + or + result = max(string s | classEscapeMatches(escape.getValue().toLowerCase(), s)) + ) + ) + } + + /** + * An implementation of `CharacterClass` for positive (non inverted) character classes. + */ + private class PositiveCharacterClass extends CharacterClass { + RegExpCharacterClass cc; + + PositiveCharacterClass() { this = getCanonicalCharClass(cc) and not cc.isInverted() } + + override string getARelevantChar() { result = getAMentionedChar(cc) } + + override predicate matches(string char) { hasChildThatMatches(cc, char) } + } + + /** + * An implementation of `CharacterClass` for inverted character classes. + */ + private class InvertedCharacterClass extends CharacterClass { + RegExpCharacterClass cc; + + InvertedCharacterClass() { this = getCanonicalCharClass(cc) and cc.isInverted() } + + override string getARelevantChar() { + result = nextChar(getAMentionedChar(cc)) or + nextChar(result) = getAMentionedChar(cc) + } + + bindingset[char] + override predicate matches(string char) { not hasChildThatMatches(cc, char) } + } + + /** + * Holds if the character class escape `clazz` (\d, \s, or \w) matches `char`. + */ + pragma[noinline] + private predicate classEscapeMatches(string clazz, string char) { + clazz = "d" and + char = "0123456789".charAt(_) + or + clazz = "s" and + char = [" ", "\t", "\r", "\n", 11.toUnicode(), 12.toUnicode()] // 11.toUnicode() = \v, 12.toUnicode() = \f + or + clazz = "w" and + char = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_".charAt(_) + } + + /** + * An implementation of `CharacterClass` for \d, \s, and \w. + */ + private class PositiveCharacterClassEscape extends CharacterClass { + RegExpCharacterClassEscape cc; + + PositiveCharacterClassEscape() { + this = getCanonicalCharClass(cc) and cc.getValue() = ["d", "s", "w"] + } + + override string getARelevantChar() { + cc.getValue() = "d" and + result = ["0", "9"] + or + cc.getValue() = "s" and + result = " " + or + cc.getValue() = "w" and + result = ["a", "Z", "_", "0", "9"] + } + + override predicate matches(string char) { classEscapeMatches(cc.getValue(), char) } + + override string choose() { + cc.getValue() = "d" and + result = "9" + or + cc.getValue() = "s" and + result = " " + or + cc.getValue() = "w" and + result = "a" + } + } + + /** + * An implementation of `CharacterClass` for \D, \S, and \W. + */ + private class NegativeCharacterClassEscape extends CharacterClass { + RegExpCharacterClassEscape cc; + + NegativeCharacterClassEscape() { + this = getCanonicalCharClass(cc) and cc.getValue() = ["D", "S", "W"] + } + + override string getARelevantChar() { + cc.getValue() = "D" and + result = ["a", "Z", "!"] + or + cc.getValue() = "S" and + result = ["a", "9", "!"] + or + cc.getValue() = "W" and + result = [" ", "!"] + } + + bindingset[char] + override predicate matches(string char) { + not classEscapeMatches(cc.getValue().toLowerCase(), char) + } + } +} + +private class EdgeLabel extends TInputSymbol { + string toString() { + this = Epsilon() and result = "" + or + exists(InputSymbol s | this = s and result = s.toString()) + } +} + +/** + * Gets the state before matching `t`. + */ +pragma[inline] +private State before(RegExpTerm t) { result = Match(t, 0) } + +/** + * Gets a state the NFA may be in after matching `t`. + */ +private State after(RegExpTerm t) { + exists(RegExpAlt alt | t = alt.getAChild() | result = after(alt)) + or + exists(RegExpSequence seq, int i | t = seq.getChild(i) | + result = before(seq.getChild(i + 1)) + or + i + 1 = seq.getNumChild() and result = after(seq) + ) + or + exists(RegExpGroup grp | t = grp.getAChild() | result = after(grp)) + or + exists(RegExpStar star | t = star.getAChild() | result = before(star)) + or + exists(RegExpPlus plus | t = plus.getAChild() | + result = before(plus) or + result = after(plus) + ) + or + exists(RegExpOpt opt | t = opt.getAChild() | result = after(opt)) + or + exists(RegExpRoot root | t = root | result = AcceptAnySuffix(root)) +} + +/** + * Holds if the NFA has a transition from `q1` to `q2` labelled with `lbl`. + */ +predicate delta(State q1, EdgeLabel lbl, State q2) { + exists(RegexpCharacterConstant s, int i | + q1 = Match(s, i) and + ( + not RegExpFlags::isIgnoreCase(s.getRootTerm()) and + lbl = Char(s.getValue().charAt(i)) + or + // normalize everything to lower case if the regexp is case insensitive + RegExpFlags::isIgnoreCase(s.getRootTerm()) and + exists(string c | c = s.getValue().charAt(i) | lbl = Char(c.toLowerCase())) + ) and + ( + q2 = Match(s, i + 1) + or + s.getValue().length() = i + 1 and + q2 = after(s) + ) + ) + or + exists(RegExpDot dot | q1 = before(dot) and q2 = after(dot) | + if RegExpFlags::isDotAll(dot.getRootTerm()) then lbl = Any() else lbl = Dot() + ) + or + exists(RegExpCharacterClass cc | + cc.isUniversalClass() and q1 = before(cc) and lbl = Any() and q2 = after(cc) + or + q1 = before(cc) and + lbl = CharClass(cc.getRawValue() + "|" + getCanonicalizationFlags(cc.getRootTerm())) and + q2 = after(cc) + ) + or + exists(RegExpCharacterClassEscape cc | + q1 = before(cc) and + lbl = CharClass(cc.getRawValue() + "|" + getCanonicalizationFlags(cc.getRootTerm())) and + q2 = after(cc) + ) + or + exists(RegExpAlt alt | lbl = Epsilon() | q1 = before(alt) and q2 = before(alt.getAChild())) + or + exists(RegExpSequence seq | lbl = Epsilon() | q1 = before(seq) and q2 = before(seq.getChild(0))) + or + exists(RegExpGroup grp | lbl = Epsilon() | q1 = before(grp) and q2 = before(grp.getChild(0))) + or + exists(RegExpStar star | lbl = Epsilon() | + q1 = before(star) and q2 = before(star.getChild(0)) + or + q1 = before(star) and q2 = after(star) + ) + or + exists(RegExpPlus plus | lbl = Epsilon() | q1 = before(plus) and q2 = before(plus.getChild(0))) + or + exists(RegExpOpt opt | lbl = Epsilon() | + q1 = before(opt) and q2 = before(opt.getChild(0)) + or + q1 = before(opt) and q2 = after(opt) + ) + or + exists(RegExpRoot root | q1 = AcceptAnySuffix(root) | + lbl = Any() and q2 = q1 + or + lbl = Epsilon() and q2 = Accept(root) + ) + or + exists(RegExpRoot root | q1 = Match(root, 0) | lbl = Any() and q2 = q1) + or + exists(RegExpDollar dollar | q1 = before(dollar) | + lbl = Epsilon() and q2 = Accept(getRoot(dollar)) + ) + or + exists(EmptyPositiveSubPatttern empty | q1 = before(empty) | + lbl = Epsilon() and q2 = after(empty) + ) +} + +/** + * Gets a state that `q` has an epsilon transition to. + */ +State epsilonSucc(State q) { delta(q, Epsilon(), result) } + +/** + * Gets a state that has an epsilon transition to `q`. + */ +State epsilonPred(State q) { q = epsilonSucc(result) } + +/** + * Holds if there is a state `q` that can be reached from `q1` + * along epsilon edges, such that there is a transition from + * `q` to `q2` that consumes symbol `s`. + */ +predicate deltaClosed(State q1, InputSymbol s, State q2) { delta(epsilonSucc*(q1), s, q2) } + +/** + * Gets the root containing the given term, that is, the root of the literal, + * or a branch of the root disjunction. + */ +RegExpRoot getRoot(RegExpTerm term) { + result = term or + result = getRoot(term.getParent()) +} + +/** + * A state in the NFA. + */ +private newtype TState = + /** + * A state representing that the NFA is about to match a term. + * `i` is used to index into multi-char literals. + */ + Match(RelevantRegExpTerm t, int i) { + i = 0 + or + exists(t.(RegexpCharacterConstant).getValue().charAt(i)) + } or + /** + * An accept state, where exactly the given input string is accepted. + */ + Accept(RegExpRoot l) { l.isRelevant() } or + /** + * An accept state, where the given input string, or any string that has this + * string as a prefix, is accepted. + */ + AcceptAnySuffix(RegExpRoot l) { l.isRelevant() } + +/** + * Gets a state that is about to match the regular expression `t`. + */ +State mkMatch(RegExpTerm t) { result = Match(t, 0) } + +/** + * A state in the NFA corresponding to a regular expression. + * + * Each regular expression literal `l` has one accepting state + * `Accept(l)`, one state that accepts all suffixes `AcceptAnySuffix(l)`, + * and a state `Match(t, i)` for every subterm `t`, + * which represents the state of the NFA before starting to + * match `t`, or the `i`th character in `t` if `t` is a constant. + */ +class State extends TState { + RegExpTerm repr; + + State() { + this = Match(repr, _) or + this = Accept(repr) or + this = AcceptAnySuffix(repr) + } + + /** + * Gets a string representation for this state in a regular expression. + */ + string toString() { + exists(int i | this = Match(repr, i) | result = "Match(" + repr + "," + i + ")") + or + this instanceof Accept and + result = "Accept(" + repr + ")" + or + this instanceof AcceptAnySuffix and + result = "AcceptAny(" + repr + ")" + } + + /** + * Gets the location for this state. + */ + Location getLocation() { result = repr.getLocation() } + + /** + * Gets the term represented by this state. + */ + RegExpTerm getRepr() { result = repr } +} + +/** + * Gets the minimum char that is matched by both the character classes `c` and `d`. + */ +private string getMinOverlapBetweenCharacterClasses(CharacterClass c, CharacterClass d) { + result = min(getAOverlapBetweenCharacterClasses(c, d)) +} + +/** + * Gets a char that is matched by both the character classes `c` and `d`. + * And `c` and `d` is not the same character class. + */ +private string getAOverlapBetweenCharacterClasses(CharacterClass c, CharacterClass d) { + sharesRoot(c, d) and + result = [c.getARelevantChar(), d.getARelevantChar()] and + c.matches(result) and + d.matches(result) and + not c = d +} + +/** + * Gets a character that is represented by both `c` and `d`. + */ +string intersect(InputSymbol c, InputSymbol d) { + (sharesRoot(c, d) or [c, d] = Any()) and + ( + c = Char(result) and + d = getAnInputSymbolMatching(result) + or + result = getMinOverlapBetweenCharacterClasses(c, d) + or + result = c.(CharacterClass).choose() and + ( + d = c + or + d = Dot() and + not (result = "\n" or result = "\r") + or + d = Any() + ) + or + (c = Dot() or c = Any()) and + (d = Dot() or d = Any()) and + result = "a" + ) + or + result = intersect(d, c) +} + +/** + * Gets a symbol that matches `char`. + */ +bindingset[char] +InputSymbol getAnInputSymbolMatching(string char) { + result = Char(char) + or + result.(CharacterClass).matches(char) + or + result = Dot() and + not (char = "\n" or char = "\r") + or + result = Any() +} + +/** + * Predicates for constructing a prefix string that leads to a given state. + */ +private module PrefixConstruction { + /** + * Holds if `state` starts the string matched by the regular expression. + */ + private predicate isStartState(State state) { + state instanceof StateInPumpableRegexp and + ( + state = Match(any(RegExpRoot r), _) + or + exists(RegExpCaret car | state = after(car)) + ) + } + + /** + * Holds if `state` is the textually last start state for the regular expression. + */ + private predicate lastStartState(State state) { + exists(RegExpRoot root | + state = + max(State s, Location l | + isStartState(s) and getRoot(s.getRepr()) = root and l = s.getRepr().getLocation() + | + s + order by + l.getStartLine(), l.getStartColumn(), s.getRepr().toString(), l.getEndColumn(), + l.getEndLine() + ) + ) + } + + /** + * Holds if there exists any transition (Epsilon() or other) from `a` to `b`. + */ + private predicate existsTransition(State a, State b) { delta(a, _, b) } + + /** + * Gets the minimum number of transitions it takes to reach `state` from the `start` state. + */ + int prefixLength(State start, State state) = + shortestDistances(lastStartState/1, existsTransition/2)(start, state, result) + + /** + * Gets the minimum number of transitions it takes to reach `state` from the start state. + */ + private int lengthFromStart(State state) { result = prefixLength(_, state) } + + /** + * Gets a string for which the regular expression will reach `state`. + * + * Has at most one result for any given `state`. + * This predicate will not always have a result even if there is a ReDoS issue in + * the regular expression. + */ + string prefix(State state) { + lastStartState(state) and + result = "" + or + // the search stops past the last redos candidate state. + lengthFromStart(state) <= max(lengthFromStart(any(State s | isReDoSCandidate(s, _)))) and + exists(State prev | + // select a unique predecessor (by an arbitrary measure) + prev = + min(State s, Location loc | + lengthFromStart(s) = lengthFromStart(state) - 1 and + loc = s.getRepr().getLocation() and + delta(s, _, state) + | + s + order by + loc.getStartLine(), loc.getStartColumn(), loc.getEndLine(), loc.getEndColumn(), + s.getRepr().toString() + ) + | + // greedy search for the shortest prefix + result = prefix(prev) and delta(prev, Epsilon(), state) + or + not delta(prev, Epsilon(), state) and + result = prefix(prev) + getCanonicalEdgeChar(prev, state) + ) + } + + /** + * Gets a canonical char for which there exists a transition from `prev` to `next` in the NFA. + */ + private string getCanonicalEdgeChar(State prev, State next) { + result = + min(string c | delta(prev, any(InputSymbol symbol | c = intersect(Any(), symbol)), next)) + } + + /** + * A state within a regular expression that has a pumpable state. + */ + class StateInPumpableRegexp extends State { + pragma[noinline] + StateInPumpableRegexp() { + exists(State s | isReDoSCandidate(s, _) | getRoot(s.getRepr()) = getRoot(this.getRepr())) + } + } +} + +/** + * Predicates for testing the presence of a rejecting suffix. + * + * These predicates are used to ensure that the all states reached from the fork + * by repeating `w` have a rejecting suffix. + * + * For example, a regexp like `/^(a+)+/` will accept any string as long the prefix is + * some number of `"a"`s, and it is therefore not possible to construct a rejecting suffix. + * + * A regexp like `/(a+)+$/` or `/(a+)+b/` trivially has a rejecting suffix, + * as the suffix "X" will cause both the regular expressions to be rejected. + * + * The string `w` is repeated any number of times because it needs to be + * infinitely repeatedable for the attack to work. + * For the regular expression `/((ab)+)*abab/` the accepting state is not reachable from the fork + * using epsilon transitions. But any attempt at repeating `w` will end in a state that accepts all suffixes. + */ +private module SuffixConstruction { + import PrefixConstruction + + /** + * Holds if all states reachable from `fork` by repeating `w` + * are likely rejectable by appending some suffix. + */ + predicate reachesOnlyRejectableSuffixes(State fork, string w) { + isReDoSCandidate(fork, w) and + forex(State next | next = process(fork, w, w.length() - 1) | isLikelyRejectable(next)) + } + + /** + * Holds if there likely exists a suffix starting from `s` that leads to the regular expression being rejected. + * This predicate might find impossible suffixes when searching for suffixes of length > 1, which can cause FPs. + */ + pragma[noinline] + private predicate isLikelyRejectable(StateInPumpableRegexp s) { + // exists a reject edge with some char. + hasRejectEdge(s) + or + hasEdgeToLikelyRejectable(s) + or + // stopping here is rejection + isRejectState(s) + } + + /** + * Holds if `s` is not an accept state, and there is no epsilon transition to an accept state. + */ + predicate isRejectState(StateInPumpableRegexp s) { not epsilonSucc*(s) = Accept(_) } + + /** + * Holds if there is likely a non-empty suffix leading to rejection starting in `s`. + */ + pragma[noopt] + predicate hasEdgeToLikelyRejectable(StateInPumpableRegexp s) { + // all edges (at least one) with some char leads to another state that is rejectable. + // the `next` states might not share a common suffix, which can cause FPs. + exists(string char | char = hasEdgeToLikelyRejectableHelper(s) | + // noopt to force `hasEdgeToLikelyRejectableHelper` to be first in the join-order. + exists(State next | deltaClosedChar(s, char, next) | isLikelyRejectable(next)) and + forall(State next | deltaClosedChar(s, char, next) | isLikelyRejectable(next)) + ) + } + + /** + * Gets a char for there exists a transition away from `s`, + * and `s` has not been found to be rejectable by `hasRejectEdge` or `isRejectState`. + */ + pragma[noinline] + private string hasEdgeToLikelyRejectableHelper(StateInPumpableRegexp s) { + not hasRejectEdge(s) and + not isRejectState(s) and + deltaClosedChar(s, result, _) + } + + /** + * Holds if there is a state `next` that can be reached from `prev` + * along epsilon edges, such that there is a transition from + * `prev` to `next` that the character symbol `char`. + */ + predicate deltaClosedChar(StateInPumpableRegexp prev, string char, StateInPumpableRegexp next) { + deltaClosed(prev, getAnInputSymbolMatchingRelevant(char), next) + } + + pragma[noinline] + InputSymbol getAnInputSymbolMatchingRelevant(string char) { + char = relevant(_) and + result = getAnInputSymbolMatching(char) + } + + /** + * Gets a char used for finding possible suffixes inside `root`. + */ + pragma[noinline] + private string relevant(RegExpRoot root) { + exists(ascii(result)) + or + exists(InputSymbol s | belongsTo(s, root) | result = intersect(s, _)) + or + // The characters from `hasSimpleRejectEdge`. Only `\n` is really needed (as `\n` is not in the `ascii` relation). + // The three chars must be kept in sync with `hasSimpleRejectEdge`. + result = ["|", "\n", "Z"] + } + + /** + * Holds if there exists a `char` such that there is no edge from `s` labeled `char` in our NFA. + * The NFA does not model reject states, so the above is the same as saying there is a reject edge. + */ + private predicate hasRejectEdge(State s) { + hasSimpleRejectEdge(s) + or + not hasSimpleRejectEdge(s) and + exists(string char | char = relevant(getRoot(s.getRepr())) | not deltaClosedChar(s, char, _)) + } + + /** + * Holds if there is no edge from `s` labeled with "|", "\n", or "Z" in our NFA. + * This predicate is used as a cheap pre-processing to speed up `hasRejectEdge`. + */ + private predicate hasSimpleRejectEdge(State s) { + // The three chars were chosen arbitrarily. The three chars must be kept in sync with `relevant`. + exists(string char | char = ["|", "\n", "Z"] | not deltaClosedChar(s, char, _)) + } + + /** + * Gets a state that can be reached from pumpable `fork` consuming all + * chars in `w` any number of times followed by the first `i+1` characters of `w`. + */ + pragma[noopt] + private State process(State fork, string w, int i) { + exists(State prev | prev = getProcessPrevious(fork, i, w) | + exists(string char, InputSymbol sym | + char = w.charAt(i) and + deltaClosed(prev, sym, result) and + // noopt to prevent joining `prev` with all possible `chars` that could transition away from `prev`. + // Instead only join with the set of `chars` where a relevant `InputSymbol` has already been found. + sym = getAProcessInputSymbol(char) + ) + ) + } + + /** + * Gets a state that can be reached from pumpable `fork` consuming all + * chars in `w` any number of times followed by the first `i` characters of `w`. + */ + private State getProcessPrevious(State fork, int i, string w) { + isReDoSCandidate(fork, w) and + ( + i = 0 and result = fork + or + result = process(fork, w, i - 1) + or + // repeat until fixpoint + i = 0 and + result = process(fork, w, w.length() - 1) + ) + } + + /** + * Gets an InputSymbol that matches `char`. + * The predicate is specialized to only have a result for the `char`s that are relevant for the `process` predicate. + */ + private InputSymbol getAProcessInputSymbol(string char) { + char = getAProcessChar() and + result = getAnInputSymbolMatching(char) + } + + /** + * Gets a `char` that occurs in a `pump` string. + */ + private string getAProcessChar() { result = any(string s | isReDoSCandidate(_, s)).charAt(_) } +} + +/** + * Gets the result of backslash-escaping newlines, carriage-returns and + * backslashes in `s`. + */ +bindingset[s] +private string escape(string s) { + result = + s.replaceAll("\\", "\\\\") + .replaceAll("\n", "\\n") + .replaceAll("\r", "\\r") + .replaceAll("\t", "\\t") +} + +/** + * Gets `str` with the last `i` characters moved to the front. + * + * We use this to adjust the pump string to match with the beginning of + * a RegExpTerm, so it doesn't start in the middle of a constant. + */ +bindingset[str, i] +private string rotate(string str, int i) { + result = str.suffix(str.length() - i) + str.prefix(str.length() - i) +} + +/** + * Holds if `term` may cause superlinear backtracking on strings containing many repetitions of `pump`. + * Gets the shortest string that causes superlinear backtracking. + */ +private predicate isReDoSAttackable(RegExpTerm term, string pump, State s) { + exists(int i, string c | s = Match(term, i) | + c = + min(string w | + any(ReDoSConfiguration conf).isReDoSCandidate(s, w) and + SuffixConstruction::reachesOnlyRejectableSuffixes(s, w) + | + w order by w.length(), w + ) and + pump = escape(rotate(c, i)) + ) +} + +/** + * Holds if the state `s` (represented by the term `t`) can have backtracking with repetitions of `pump`. + * + * `prefixMsg` contains a friendly message for a prefix that reaches `s` (or `prefixMsg` is the empty string if the prefix is empty or if no prefix could be found). + */ +predicate hasReDoSResult(RegExpTerm t, string pump, State s, string prefixMsg) { + isReDoSAttackable(t, pump, s) and + ( + prefixMsg = "starting with '" + escape(PrefixConstruction::prefix(s)) + "' and " and + not PrefixConstruction::prefix(s) = "" + or + PrefixConstruction::prefix(s) = "" and prefixMsg = "" + or + not exists(PrefixConstruction::prefix(s)) and prefixMsg = "" + ) +} diff --git a/java/ql/lib/semmle/code/java/security/performance/RegExpTreeView.qll b/java/ql/lib/semmle/code/java/security/performance/RegExpTreeView.qll new file mode 100644 index 00000000000..ac220ec8a50 --- /dev/null +++ b/java/ql/lib/semmle/code/java/security/performance/RegExpTreeView.qll @@ -0,0 +1,49 @@ +/** + * This module should provide a class hierarchy corresponding to a parse tree of regular expressions. + */ + +import java +import semmle.code.java.regex.RegexTreeView + +/** + * Holds if the regular expression should not be considered. + * + * We make the pragmatic performance optimization to ignore regular expressions in files + * that does not belong to the project code (such as installed dependencies). + */ +predicate isExcluded(RegExpParent parent) { + not exists(parent.getRegex().getLocation().getFile().getRelativePath()) + or + // Regexes with many occurrences of ".*" may cause the polynomial ReDoS computation to explode, so + // we explicitly exclude these. + count(int i | exists(parent.getRegex().getText().regexpFind("\\.\\*", i, _)) | i) > 10 +} + +/** + * A module containing predicates for determining which flags a regular expression have. + */ +module RegExpFlags { + /** + * Holds if `root` has the `i` flag for case-insensitive matching. + */ + predicate isIgnoreCase(RegExpTerm root) { + root.isRootTerm() and + root.getLiteral().isIgnoreCase() + } + + /** + * Gets the flags for `root`, or the empty string if `root` has no flags. + */ + string getFlags(RegExpTerm root) { + root.isRootTerm() and + result = root.getLiteral().getFlags() + } + + /** + * Holds if `root` has the `s` flag for multi-line matching. + */ + predicate isDotAll(RegExpTerm root) { + root.isRootTerm() and + root.getLiteral().isDotAll() + } +} diff --git a/java/ql/lib/semmle/code/java/security/performance/SuperlinearBackTracking.qll b/java/ql/lib/semmle/code/java/security/performance/SuperlinearBackTracking.qll new file mode 100644 index 00000000000..2b42165ff7e --- /dev/null +++ b/java/ql/lib/semmle/code/java/security/performance/SuperlinearBackTracking.qll @@ -0,0 +1,420 @@ +/** + * Provides classes for working with regular expressions that can + * perform backtracking in superlinear time. + */ + +import ReDoSUtil + +/* + * This module implements the analysis described in the paper: + * Valentin Wustholz, Oswaldo Olivo, Marijn J. H. Heule, and Isil Dillig: + * Static Detection of DoS Vulnerabilities in + * Programs that use Regular Expressions + * (Extended Version). + * (https://arxiv.org/pdf/1701.04045.pdf) + * + * Theorem 3 from the paper describes the basic idea. + * + * The following explains the idea using variables and predicate names that are used in the implementation: + * We consider a pair of repetitions, which we will call `pivot` and `succ`. + * + * We create a product automaton of 3-tuples of states (see `StateTuple`). + * There exists a transition `(a,b,c) -> (d,e,f)` in the product automaton + * iff there exists three transitions in the NFA `a->d, b->e, c->f` where those three + * transitions all match a shared character `char`. (see `getAThreewayIntersect`) + * + * We start a search in the product automaton at `(pivot, pivot, succ)`, + * and search for a series of transitions (a `Trace`), such that we end + * at `(pivot, succ, succ)` (see `isReachableFromStartTuple`). + * + * For example, consider the regular expression `/^\d*5\w*$/`. + * The search will start at the tuple `(\d*, \d*, \w*)` and search + * for a path to `(\d*, \w*, \w*)`. + * This path exists, and consists of a single transition in the product automaton, + * where the three corresponding NFA edges all match the character `"5"`. + * + * The start-state in the NFA has an any-transition to itself, this allows us to + * flag regular expressions such as `/a*$/` - which does not have a start anchor - + * and can thus start matching anywhere. + * + * The implementation is not perfect. + * It has the same suffix detection issue as the `js/redos` query, which can cause false positives. + * It also doesn't find all transitions in the product automaton, which can cause false negatives. + */ + +/** + * An instantiaion of `ReDoSConfiguration` for superlinear ReDoS. + */ +class SuperLinearReDoSConfiguration extends ReDoSConfiguration { + SuperLinearReDoSConfiguration() { this = "SuperLinearReDoSConfiguration" } + + override predicate isReDoSCandidate(State state, string pump) { isPumpable(_, state, pump) } +} + +/** + * Gets any root (start) state of a regular expression. + */ +private State getRootState() { result = mkMatch(any(RegExpRoot r)) } + +private newtype TStateTuple = + MkStateTuple(State q1, State q2, State q3) { + // starts at (pivot, pivot, succ) + isStartLoops(q1, q3) and q1 = q2 + or + step(_, _, _, _, q1, q2, q3) and FeasibleTuple::isFeasibleTuple(q1, q2, q3) + } + +/** + * A state in the product automaton. + * The product automaton contains 3-tuples of states. + * + * We lazily only construct those states that we are actually + * going to need. + * Either a start state `(pivot, pivot, succ)`, or a state + * where there exists a transition from an already existing state. + * + * The exponential variant of this query (`js/redos`) uses an optimization + * trick where `q1 <= q2`. This trick cannot be used here as the order + * of the elements matter. + */ +class StateTuple extends TStateTuple { + State q1; + State q2; + State q3; + + StateTuple() { this = MkStateTuple(q1, q2, q3) } + + /** + * Gest a string repesentation of this tuple. + */ + string toString() { result = "(" + q1 + ", " + q2 + ", " + q3 + ")" } + + /** + * Holds if this tuple is `(r1, r2, r3)`. + */ + pragma[noinline] + predicate isTuple(State r1, State r2, State r3) { r1 = q1 and r2 = q2 and r3 = q3 } +} + +/** + * A module for determining feasible tuples for the product automaton. + * + * The implementation is split into many predicates for performance reasons. + */ +private module FeasibleTuple { + /** + * Holds if the tuple `(r1, r2, r3)` might be on path from a start-state to an end-state in the product automaton. + */ + pragma[inline] + predicate isFeasibleTuple(State r1, State r2, State r3) { + // The first element is either inside a repetition (or the start state itself) + isRepetitionOrStart(r1) and + // The last element is inside a repetition + stateInsideRepetition(r3) and + // The states are reachable in the NFA in the order r1 -> r2 -> r3 + delta+(r1) = r2 and + delta+(r2) = r3 and + // The first element can reach a beginning (the "pivot" state in a `(pivot, succ)` pair). + canReachABeginning(r1) and + // The last element can reach a target (the "succ" state in a `(pivot, succ)` pair). + canReachATarget(r3) + } + + /** + * Holds if `s` is either inside a repetition, or is the start state (which is a repetition). + */ + pragma[noinline] + private predicate isRepetitionOrStart(State s) { stateInsideRepetition(s) or s = getRootState() } + + /** + * Holds if state `s` might be inside a backtracking repetition. + */ + pragma[noinline] + private predicate stateInsideRepetition(State s) { + s.getRepr().getParent*() instanceof InfiniteRepetitionQuantifier + } + + /** + * Holds if there exists a path in the NFA from `s` to a "pivot" state + * (from a `(pivot, succ)` pair that starts the search). + */ + pragma[noinline] + private predicate canReachABeginning(State s) { + delta+(s) = any(State pivot | isStartLoops(pivot, _)) + } + + /** + * Holds if there exists a path in the NFA from `s` to a "succ" state + * (from a `(pivot, succ)` pair that starts the search). + */ + pragma[noinline] + private predicate canReachATarget(State s) { delta+(s) = any(State succ | isStartLoops(_, succ)) } +} + +/** + * Holds if `pivot` and `succ` are a pair of loops that could be the beginning of a quadratic blowup. + * + * There is a slight implementation difference compared to the paper: this predicate requires that `pivot != succ`. + * The case where `pivot = succ` causes exponential backtracking and is handled by the `js/redos` query. + */ +predicate isStartLoops(State pivot, State succ) { + pivot != succ and + succ.getRepr() instanceof InfiniteRepetitionQuantifier and + delta+(pivot) = succ and + ( + pivot.getRepr() instanceof InfiniteRepetitionQuantifier + or + pivot = mkMatch(any(RegExpRoot root)) + ) +} + +/** + * Gets a state for which there exists a transition in the NFA from `s'. + */ +State delta(State s) { delta(s, _, result) } + +/** + * Holds if there are transitions from the components of `q` to the corresponding + * components of `r` labelled with `s1`, `s2`, and `s3`, respectively. + */ +pragma[noinline] +predicate step(StateTuple q, InputSymbol s1, InputSymbol s2, InputSymbol s3, StateTuple r) { + exists(State r1, State r2, State r3 | + step(q, s1, s2, s3, r1, r2, r3) and r = MkStateTuple(r1, r2, r3) + ) +} + +/** + * Holds if there are transitions from the components of `q` to `r1`, `r2`, and `r3 + * labelled with `s1`, `s2`, and `s3`, respectively. + */ +pragma[noopt] +predicate step( + StateTuple q, InputSymbol s1, InputSymbol s2, InputSymbol s3, State r1, State r2, State r3 +) { + exists(State q1, State q2, State q3 | q.isTuple(q1, q2, q3) | + deltaClosed(q1, s1, r1) and + deltaClosed(q2, s2, r2) and + deltaClosed(q3, s3, r3) and + // use noopt to force the join on `getAThreewayIntersect` to happen last. + exists(getAThreewayIntersect(s1, s2, s3)) + ) +} + +/** + * Gets a char that is matched by all the edges `s1`, `s2`, and `s3`. + * + * The result is not complete, and might miss some combination of edges that share some character. + */ +pragma[noinline] +string getAThreewayIntersect(InputSymbol s1, InputSymbol s2, InputSymbol s3) { + result = minAndMaxIntersect(s1, s2) and result = [intersect(s2, s3), intersect(s1, s3)] + or + result = minAndMaxIntersect(s1, s3) and result = [intersect(s2, s3), intersect(s1, s2)] + or + result = minAndMaxIntersect(s2, s3) and result = [intersect(s1, s2), intersect(s1, s3)] +} + +/** + * Gets the minimum and maximum characters that intersect between `a` and `b`. + * This predicate is used to limit the size of `getAThreewayIntersect`. + */ +pragma[noinline] +string minAndMaxIntersect(InputSymbol a, InputSymbol b) { + result = [min(intersect(a, b)), max(intersect(a, b))] +} + +private newtype TTrace = + Nil() or + Step(InputSymbol s1, InputSymbol s2, InputSymbol s3, TTrace t) { + exists(StateTuple p | + isReachableFromStartTuple(_, _, p, t, _) and + step(p, s1, s2, s3, _) + ) + or + exists(State pivot, State succ | isStartLoops(pivot, succ) | + t = Nil() and step(MkStateTuple(pivot, pivot, succ), s1, s2, s3, _) + ) + } + +/** + * A list of tuples of input symbols that describe a path in the product automaton + * starting from some start state. + */ +class Trace extends TTrace { + /** + * Gets a string representation of this Trace that can be used for debug purposes. + */ + string toString() { + this = Nil() and result = "Nil()" + or + exists(InputSymbol s1, InputSymbol s2, InputSymbol s3, Trace t | this = Step(s1, s2, s3, t) | + result = "Step(" + s1 + ", " + s2 + ", " + s3 + ", " + t + ")" + ) + } +} + +/** + * Gets a string corresponding to the trace `t`. + */ +string concretise(Trace t) { + t = Nil() and result = "" + or + exists(InputSymbol s1, InputSymbol s2, InputSymbol s3, Trace rest | t = Step(s1, s2, s3, rest) | + result = concretise(rest) + getAThreewayIntersect(s1, s2, s3) + ) +} + +/** + * Holds if there exists a transition from `r` to `q` in the product automaton. + * Notice that the arguments are flipped, and thus the direction is backwards. + */ +pragma[noinline] +predicate tupleDeltaBackwards(StateTuple q, StateTuple r) { step(r, _, _, _, q) } + +/** + * Holds if `tuple` is an end state in our search. + * That means there exists a pair of loops `(pivot, succ)` such that `tuple = (pivot, succ, succ)`. + */ +predicate isEndTuple(StateTuple tuple) { tuple = getAnEndTuple(_, _) } + +/** + * Gets the minimum length of a path from `r` to some an end state `end`. + * + * The implementation searches backwards from the end-tuple. + * This approach was chosen because it is way more efficient if the first predicate given to `shortestDistances` is small. + * The `end` argument must always be an end state. + */ +int distBackFromEnd(StateTuple r, StateTuple end) = + shortestDistances(isEndTuple/1, tupleDeltaBackwards/2)(end, r, result) + +/** + * Holds if there exists a pair of repetitions `(pivot, succ)` in the regular expression such that: + * `tuple` is reachable from `(pivot, pivot, succ)` in the product automaton, + * and there is a distance of `dist` from `tuple` to the nearest end-tuple `(pivot, succ, succ)`, + * and a path from a start-state to `tuple` follows the transitions in `trace`. + */ +predicate isReachableFromStartTuple(State pivot, State succ, StateTuple tuple, Trace trace, int dist) { + // base case. The first step is inlined to start the search after all possible 1-steps, and not just the ones with the shortest path. + exists(InputSymbol s1, InputSymbol s2, InputSymbol s3, State q1, State q2, State q3 | + isStartLoops(pivot, succ) and + step(MkStateTuple(pivot, pivot, succ), s1, s2, s3, tuple) and + tuple = MkStateTuple(q1, q2, q3) and + trace = Step(s1, s2, s3, Nil()) and + dist = distBackFromEnd(tuple, MkStateTuple(pivot, succ, succ)) + ) + or + // recursive case + exists(StateTuple p, Trace v, InputSymbol s1, InputSymbol s2, InputSymbol s3 | + isReachableFromStartTuple(pivot, succ, p, v, dist + 1) and + dist = isReachableFromStartTupleHelper(pivot, succ, tuple, p, s1, s2, s3) and + trace = Step(s1, s2, s3, v) + ) +} + +/** + * Helper predicate for the recursive case in `isReachableFromStartTuple`. + */ +pragma[noinline] +private int isReachableFromStartTupleHelper( + State pivot, State succ, StateTuple r, StateTuple p, InputSymbol s1, InputSymbol s2, + InputSymbol s3 +) { + result = distBackFromEnd(r, MkStateTuple(pivot, succ, succ)) and + step(p, s1, s2, s3, r) +} + +/** + * Gets the tuple `(pivot, succ, succ)` from the product automaton. + */ +StateTuple getAnEndTuple(State pivot, State succ) { + isStartLoops(pivot, succ) and + result = MkStateTuple(pivot, succ, succ) +} + +/** + * Holds if matching repetitions of `pump` can: + * 1) Transition from `pivot` back to `pivot`. + * 2) Transition from `pivot` to `succ`. + * 3) Transition from `succ` to `succ`. + * + * From theorem 3 in the paper linked in the top of this file we can therefore conclude that + * the regular expression has polynomial backtracking - if a rejecting suffix exists. + * + * This predicate is used by `SuperLinearReDoSConfiguration`, and the final results are + * available in the `hasReDoSResult` predicate. + */ +predicate isPumpable(State pivot, State succ, string pump) { + exists(StateTuple q, Trace t | + isReachableFromStartTuple(pivot, succ, q, t, _) and + q = getAnEndTuple(pivot, succ) and + pump = concretise(t) + ) +} + +/** + * Holds if repetitions of `pump` at `t` will cause polynomial backtracking. + */ +predicate polynimalReDoS(RegExpTerm t, string pump, string prefixMsg, RegExpTerm prev) { + exists(State s, State pivot | + hasReDoSResult(t, pump, s, prefixMsg) and + isPumpable(pivot, s, _) and + prev = pivot.getRepr() + ) +} + +/** + * Gets a message for why `term` can cause polynomial backtracking. + */ +string getReasonString(RegExpTerm term, string pump, string prefixMsg, RegExpTerm prev) { + polynimalReDoS(term, pump, prefixMsg, prev) and + result = + "Strings " + prefixMsg + "with many repetitions of '" + pump + + "' can start matching anywhere after the start of the preceeding " + prev +} + +/** + * A term that may cause a regular expression engine to perform a + * polynomial number of match attempts, relative to the input length. + */ +class PolynomialBackTrackingTerm extends InfiniteRepetitionQuantifier { + string reason; + string pump; + string prefixMsg; + RegExpTerm prev; + + PolynomialBackTrackingTerm() { + reason = getReasonString(this, pump, prefixMsg, prev) and + // there might be many reasons for this term to have polynomial backtracking - we pick the shortest one. + reason = min(string msg | msg = getReasonString(this, _, _, _) | msg order by msg.length(), msg) + } + + /** + * Holds if all non-empty successors to the polynomial backtracking term matches the end of the line. + */ + predicate isAtEndLine() { + forall(RegExpTerm succ | this.getSuccessor+() = succ and not matchesEpsilon(succ) | + succ instanceof RegExpDollar + ) + } + + /** + * Gets the string that should be repeated to cause this regular expression to perform polynomially. + */ + string getPumpString() { result = pump } + + /** + * Gets a message for which prefix a matching string must start with for this term to cause polynomial backtracking. + */ + string getPrefixMessage() { result = prefixMsg } + + /** + * Gets a predecessor to `this`, which also loops on the pump string, and thereby causes polynomial backtracking. + */ + RegExpTerm getPreviousLoop() { result = prev } + + /** + * Gets the reason for the number of match attempts. + */ + string getReason() { result = reason } +} From 37240f01d267f659e6047affe87e52831aa34819 Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Mon, 6 Dec 2021 11:59:38 +0000 Subject: [PATCH 12/91] Copy Redos queries from python Todo: Implement dataflow for polynomialredos; update docs to reference java rather than python --- .../CWE/CWE-730/PolynomialReDoS.qhelp | 108 ++++++++++++++++++ .../Security/CWE/CWE-730/PolynomialReDoS.ql | 34 ++++++ java/ql/src/Security/CWE/CWE-730/ReDoS.qhelp | 34 ++++++ java/ql/src/Security/CWE/CWE-730/ReDoS.ql | 25 ++++ .../CWE/CWE-730/ReDoSIntroduction.inc.qhelp | 54 +++++++++ .../CWE/CWE-730/ReDoSReferences.inc.qhelp | 16 +++ 6 files changed, 271 insertions(+) create mode 100644 java/ql/src/Security/CWE/CWE-730/PolynomialReDoS.qhelp create mode 100644 java/ql/src/Security/CWE/CWE-730/PolynomialReDoS.ql create mode 100644 java/ql/src/Security/CWE/CWE-730/ReDoS.qhelp create mode 100644 java/ql/src/Security/CWE/CWE-730/ReDoS.ql create mode 100644 java/ql/src/Security/CWE/CWE-730/ReDoSIntroduction.inc.qhelp create mode 100644 java/ql/src/Security/CWE/CWE-730/ReDoSReferences.inc.qhelp diff --git a/java/ql/src/Security/CWE/CWE-730/PolynomialReDoS.qhelp b/java/ql/src/Security/CWE/CWE-730/PolynomialReDoS.qhelp new file mode 100644 index 00000000000..fa8a3563d23 --- /dev/null +++ b/java/ql/src/Security/CWE/CWE-730/PolynomialReDoS.qhelp @@ -0,0 +1,108 @@ + + + + + + + +

    + + Consider this use of a regular expression, which removes + all leading and trailing whitespace in a string: + +

    + + + re.sub(r"^\s+|\s+$", "", text) # BAD + + +

    + + The sub-expression "\s+$" will match the + whitespace characters in text from left to right, but it + can start matching anywhere within a whitespace sequence. This is + problematic for strings that do not end with a whitespace + character. Such a string will force the regular expression engine to + process each whitespace sequence once per whitespace character in the + sequence. + +

    + +

    + + This ultimately means that the time cost of trimming a + string is quadratic in the length of the string. So a string like + "a b" will take milliseconds to process, but a similar + string with a million spaces instead of just one will take several + minutes. + +

    + +

    + + Avoid this problem by rewriting the regular expression to + not contain the ambiguity about when to start matching whitespace + sequences. For instance, by using a negative look-behind + (^\s+|(?<!\s)\s+$), or just by using the built-in strip + method (text.strip()). + +

    + +

    + + Note that the sub-expression "^\s+" is + not problematic as the ^ anchor restricts + when that sub-expression can start matching, and as the regular + expression engine matches from left to right. + +

    + +     + + + +

    + + As a similar, but slightly subtler problem, consider the + regular expression that matches lines with numbers, possibly written + using scientific notation: +

    + + + ^0\.\d+E?\d+$ # BAD + + +

    + + The problem with this regular expression is in the + sub-expression \d+E?\d+ because the second + \d+ can start matching digits anywhere after the first + match of the first \d+ if there is no E in + the input string. + +

    + +

    + + This is problematic for strings that do not + end with a digit. Such a string will force the regular expression + engine to process each digit sequence once per digit in the sequence, + again leading to a quadratic time complexity. + +

    + +

    + + To make the processing faster, the regular expression + should be rewritten such that the two \d+ sub-expressions + do not have overlapping matches: ^0\.\d+(E\d+)?$. + +

    + +     + + + +     diff --git a/java/ql/src/Security/CWE/CWE-730/PolynomialReDoS.ql b/java/ql/src/Security/CWE/CWE-730/PolynomialReDoS.ql new file mode 100644 index 00000000000..13d5bb8e8a6 --- /dev/null +++ b/java/ql/src/Security/CWE/CWE-730/PolynomialReDoS.ql @@ -0,0 +1,34 @@ +/** + * @name Polynomial regular expression used on uncontrolled data + * @description A regular expression that can require polynomial time + * to match may be vulnerable to denial-of-service attacks. + * @kind path-problem + * @problem.severity warning + * @precision high + * @id java/polynomial-redos + * @tags security + * external/cwe/cwe-730 + * external/cwe/cwe-400 + */ + +import java +import semmle.code.java.security.performance.SuperlinearBackTracking +import semmle.code.java.dataflow.DataFlow +// import semmle.python.security.dataflow.PolynomialReDoS +import DataFlow::PathGraph + +from + PolynomialReDoS::Configuration config, DataFlow::PathNode source, DataFlow::PathNode sink, + PolynomialReDoS::Sink sinkNode, PolynomialBackTrackingTerm regexp +where + config.hasFlowPath(source, sink) and + sinkNode = sink.getNode() and + regexp.getRootTerm() = sinkNode.getRegExp() +// not ( +// source.getNode().(Source).getKind() = "url" and +// regexp.isAtEndLine() +// ) +select sinkNode.getHighlight(), source, sink, + "This $@ that depends on $@ may run slow on strings " + regexp.getPrefixMessage() + + "with many repetitions of '" + regexp.getPumpString() + "'.", regexp, "regular expression", + source.getNode(), "a user-provided value" diff --git a/java/ql/src/Security/CWE/CWE-730/ReDoS.qhelp b/java/ql/src/Security/CWE/CWE-730/ReDoS.qhelp new file mode 100644 index 00000000000..9cfbcc32354 --- /dev/null +++ b/java/ql/src/Security/CWE/CWE-730/ReDoS.qhelp @@ -0,0 +1,34 @@ + + + + + + + +

    + Consider this regular expression: +

    + + ^_(__|.)+_$ + +

    + Its sub-expression "(__|.)+?" can match the string "__" either by the + first alternative "__" to the left of the "|" operator, or by two + repetitions of the second alternative "." to the right. Thus, a string consisting + of an odd number of underscores followed by some other character will cause the regular + expression engine to run for an exponential amount of time before rejecting the input. +

    +

    + This problem can be avoided by rewriting the regular expression to remove the ambiguity between + the two branches of the alternative inside the repetition: +

    + + ^_(__|[^_])+_$ + +     + + + +     diff --git a/java/ql/src/Security/CWE/CWE-730/ReDoS.ql b/java/ql/src/Security/CWE/CWE-730/ReDoS.ql new file mode 100644 index 00000000000..f72bfc3fc13 --- /dev/null +++ b/java/ql/src/Security/CWE/CWE-730/ReDoS.ql @@ -0,0 +1,25 @@ +/** + * @name Inefficient regular expression + * @description A regular expression that requires exponential time to match certain inputs + * can be a performance bottleneck, and may be vulnerable to denial-of-service + * attacks. + * @kind problem + * @problem.severity error + * @precision high + * @id java/redos + * @tags security + * external/cwe/cwe-730 + * external/cwe/cwe-400 + */ + +import java +import semmle.code.java.security.performance.ExponentialBackTracking + +from RegExpTerm t, string pump, State s, string prefixMsg +where + hasReDoSResult(t, pump, s, prefixMsg) and + // exclude verbose mode regexes for now + not t.getRegex().getAMode() = "VERBOSE" +select t, + "This part of the regular expression may cause exponential backtracking on strings " + prefixMsg + + "containing many repetitions of '" + pump + "'." diff --git a/java/ql/src/Security/CWE/CWE-730/ReDoSIntroduction.inc.qhelp b/java/ql/src/Security/CWE/CWE-730/ReDoSIntroduction.inc.qhelp new file mode 100644 index 00000000000..f533097c222 --- /dev/null +++ b/java/ql/src/Security/CWE/CWE-730/ReDoSIntroduction.inc.qhelp @@ -0,0 +1,54 @@ + + + +

    + + Some regular expressions take a long time to match certain + input strings to the point where the time it takes to match a string + of length n is proportional to nk or even + 2n. Such regular expressions can negatively affect + performance, or even allow a malicious user to perform a Denial of + Service ("DoS") attack by crafting an expensive input string for the + regular expression to match. + +

    + +

    + + The regular expression engine provided by Python uses a backtracking non-deterministic finite + automata to implement regular expression matching. While this approach + is space-efficient and allows supporting advanced features like + capture groups, it is not time-efficient in general. The worst-case + time complexity of such an automaton can be polynomial or even + exponential, meaning that for strings of a certain shape, increasing + the input length by ten characters may make the automaton about 1000 + times slower. + +

    + +

    + + Typically, a regular expression is affected by this + problem if it contains a repetition of the form r* or + r+ where the sub-expression r is ambiguous + in the sense that it can match some string in multiple ways. More + information about the precise circumstances can be found in the + references. + +

    +     + + + +

    + + Modify the regular expression to remove the ambiguity, or + ensure that the strings matched with the regular expression are short + enough that the time-complexity does not matter. + +

    + +     +     diff --git a/java/ql/src/Security/CWE/CWE-730/ReDoSReferences.inc.qhelp b/java/ql/src/Security/CWE/CWE-730/ReDoSReferences.inc.qhelp new file mode 100644 index 00000000000..2b3e5f17c62 --- /dev/null +++ b/java/ql/src/Security/CWE/CWE-730/ReDoSReferences.inc.qhelp @@ -0,0 +1,16 @@ + + + +
  • + OWASP: + Regular expression Denial of Service - ReDoS. +
    • +
  • Wikipedia: ReDoS.
    • +
  • Wikipedia: Time complexity.
    • +
  • James Kirrage, Asiri Rathnayake, Hayo Thielecke: + Static Analysis for Regular Expression Denial-of-Service Attack. +
    • +     +     From 59945cd8b3e6d29d1500e8c729c770be57d0f7f1 Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Tue, 7 Dec 2021 12:33:52 +0000 Subject: [PATCH 13/91] Add dataflow logic to PolynomialRedDoS --- .../code/java/dataflow/ExternalFlow.qll | 2 +- .../code/java/regex/RegexFlowConfigs.qll | 192 ++++++++++++++++++ .../{RegexFlow.qll => RegexFlowModels.qll} | 14 +- java/ql/lib/semmle/code/java/regex/regex.qll | 20 +- .../Security/CWE/CWE-730/PolynomialReDoS.ql | 30 ++- 5 files changed, 223 insertions(+), 35 deletions(-) create mode 100644 java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll rename java/ql/lib/semmle/code/java/regex/{RegexFlow.qll => RegexFlowModels.qll} (82%) diff --git a/java/ql/lib/semmle/code/java/dataflow/ExternalFlow.qll b/java/ql/lib/semmle/code/java/dataflow/ExternalFlow.qll index 6d14dc5f95c..77e2d98c654 100644 --- a/java/ql/lib/semmle/code/java/dataflow/ExternalFlow.qll +++ b/java/ql/lib/semmle/code/java/dataflow/ExternalFlow.qll @@ -140,7 +140,7 @@ private module Frameworks { private import semmle.code.java.frameworks.jOOQ private import semmle.code.java.frameworks.JMS private import semmle.code.java.frameworks.RabbitMQ - private import semmle.code.java.regex.RegexFlow + private import semmle.code.java.regex.RegexFlowModels } private predicate sourceModelCsv(string row) { diff --git a/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll b/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll new file mode 100644 index 00000000000..9769a7ce8f7 --- /dev/null +++ b/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll @@ -0,0 +1,192 @@ +/** + * Defines configurations and steps for handling regexes + */ + +import java +private import semmle.code.java.dataflow.DataFlow +private import semmle.code.java.dataflow.DataFlow2 +private import semmle.code.java.dataflow.DataFlow3 +private import RegexFlowModels + +private class RegexCompileFlowConf extends DataFlow2::Configuration { + RegexCompileFlowConf() { this = "RegexCompileFlowConfig" } + + override predicate isSource(DataFlow::Node node) { node.asExpr() instanceof StringLiteral } + + override predicate isSink(DataFlow::Node node) { sinkNode(node, "regex-compile") } +} + +/** + * Holds if `s` is used as a regex, with the mode `mode` (if known). + * If regex mode is not known, `mode` will be `"None"`. + */ +predicate used_as_regex(Expr s, string mode) { + any(RegexCompileFlowConf c).hasFlow(DataFlow2::exprNode(s), _) and + mode = "None" // TODO: proper mode detection +} + +/** + * A method access that can match a regex against a string + */ +abstract class RegexMatchMethodAccess extends MethodAccess { + string package; + string type; + string name; + int regexArg; + int stringArg; + Method m; + + RegexMatchMethodAccess() { + this.getMethod().overrides*(m) and + m.hasQualifiedName(package, type, name) and + regexArg in [-1 .. m.getNumberOfParameters() - 1] and + stringArg in [-1 .. m.getNumberOfParameters() - 1] + } + + /** Gets the argument of this call that the regex to be matched against flows into */ + Expr getRegexArg() { result = argOf(this, regexArg) } + + /** Gets the argument of this call that the */ + Expr getStringArg() { result = argOf(this, stringArg) } +} + +private Expr argOf(MethodAccess ma, int arg) { + arg = -1 and result = ma.getQualifier() + or + result = ma.getArgument(arg) +} + +/** + * A unit class for adding additional regex flow steps. + * + * Extend this class to add additional flow steps that should apply to regex flow configurations. + */ +class RegexAdditionalFlowStep extends Unit { + /** + * Holds if the step from `node1` to `node2` should be considered a flow + * step for regex flow configurations. + */ + abstract predicate step(DataFlow::Node node1, DataFlow::Node node2); +} + +// TODO: can this be done with the models-as-data framework? +private class JdkRegexMatchMethodAccess extends RegexMatchMethodAccess { + JdkRegexMatchMethodAccess() { + package = "java.util.regex" and + type = "Pattern" and + ( + name = "matcher" and regexArg = -1 and stringArg = 0 + or + name = "matches" and regexArg = 0 and stringArg = 1 + or + name = "split" and regexArg = 0 and stringArg = 1 + or + name = "splitAsStream" and regexArg = 0 and stringArg = 1 + ) + or + package = "java.lang" and + type = "String" and + name = ["matches", "split"] and + regexArg = 0 and + stringArg = -1 + or + package = "java.util" and + type = "Predicate" and + name = "test" and + regexArg = -1 and + stringArg = 0 + } +} + +private class JdkRegexFlowStep extends RegexAdditionalFlowStep { + override predicate step(DataFlow::Node node1, DataFlow::Node node2) { + exists(MethodAccess ma, Method m, string package, string type, string name, int arg | + ma.getMethod().overrides*(m) and + m.hasQualifiedName(package, type, name) and + node1.asExpr() = argOf(ma, arg) and + node2.asExpr() = ma + | + package = "java.util.regex" and + type = "Pattern" and + ( + name = ["asMatchPredicate", "asPredicate"] and + arg = -1 + or + name = "compile" and + arg = 0 + ) + or + package = "java.util" and + type = "Predicate" and + name = ["and", "or", "not", "negate"] and + arg = [-1, 0] + ) + } +} + +private class GuavaRegexMatchMethodAccess extends RegexMatchMethodAccess { + GuavaRegexMatchMethodAccess() { + package = "com.google.common.collect" and + regexArg = -1 and + stringArg = 0 and + type = ["Splitter", "Splitter$MapSplitter"] and + name = ["split", "splitToList"] + } +} + +private class GuavaRegexFlowStep extends RegexAdditionalFlowStep { + override predicate step(DataFlow::Node node1, DataFlow::Node node2) { + exists(MethodAccess ma, Method m, string package, string type, string name, int arg | + ma.getMethod().overrides*(m) and + m.hasQualifiedName(package, type, name) and + node1.asExpr() = argOf(ma, arg) and + node2.asExpr() = ma + | + package = "com.google.common.base" and + type = "Splitter" and + ( + name = "on" and + m.getParameterType(0).(RefType).hasQualifiedName("java.util.regex", "Pattern") and + arg = 0 + or + name = "withKeyValueSeparator" and + m.getParameterType(0).(RefType).hasQualifiedName("com.google.common.base", "Splitter") and + arg = 0 + or + name = "onPattern" and + arg = 0 + or + name = ["limit", "omitEmptyStrings", "trimResults", "withKeyValueSeparator"] and + arg = -1 + ) + ) + } +} + +private class RegexMatchFlowConf extends DataFlow2::Configuration { + RegexMatchFlowConf() { this = "RegexMatchFlowConf" } + + override predicate isSource(DataFlow::Node src) { src.asExpr() instanceof StringLiteral } + + override predicate isSink(DataFlow::Node sink) { + exists(RegexMatchMethodAccess ma | sink.asExpr() = ma.getRegexArg()) + } + + override predicate isAdditionalFlowStep(DataFlow::Node node1, DataFlow::Node node2) { + any(RegexAdditionalFlowStep s).step(node1, node2) + } +} + +/** + * Holds if the string literal `regex` is matched against the expression `str`. + */ +predicate regex_match(StringLiteral regex, Expr str) { + exists( + DataFlow::Node src, DataFlow::Node sink, RegexMatchMethodAccess ma, RegexMatchFlowConf conf + | + src.asExpr() = regex and + sink.asExpr() = ma.getRegexArg() and + conf.hasFlow(src, sink) and + str = ma.getStringArg() + ) +} diff --git a/java/ql/lib/semmle/code/java/regex/RegexFlow.qll b/java/ql/lib/semmle/code/java/regex/RegexFlowModels.qll similarity index 82% rename from java/ql/lib/semmle/code/java/regex/RegexFlow.qll rename to java/ql/lib/semmle/code/java/regex/RegexFlowModels.qll index 54b16ae8a4b..65ff6199088 100644 --- a/java/ql/lib/semmle/code/java/regex/RegexFlow.qll +++ b/java/ql/lib/semmle/code/java/regex/RegexFlowModels.qll @@ -6,13 +6,13 @@ private class RegexSinkCsv extends SinkModelCsv { row = [ //"namespace;type;subtypes;name;signature;ext;input;kind" - "java.util.regex;Pattern;false;compile;(String);;Argument[0];regex-use", - "java.util.regex;Pattern;false;compile;(String,int);;Argument[0];regex-use", - "java.util.regex;Pattern;false;matches;(String,CharSequence);;Argument[0];regex-use", - "java.util;String;false;matches;(String);;Argument[0];regex-use", - "java.util;String;false;split;(String);;Argument[0];regex-use", - "java.util;String;false;split;(String,int);;Argument[0];regex-use", - "com.google.common.base;Splitter;false;onPattern;(String);;Argument[0];regex-use" + "java.util.regex;Pattern;false;compile;(String);;Argument[0];regex-compile", + "java.util.regex;Pattern;false;compile;(String,int);;Argument[0];regex-compile", + "java.util.regex;Pattern;false;matches;(String,CharSequence);;Argument[0];regex-compile", + "java.util;String;false;matches;(String);;Argument[0];regex-compile", + "java.util;String;false;split;(String);;Argument[0];regex-compile", + "java.util;String;false;split;(String,int);;Argument[0];regex-compile", + "com.google.common.base;Splitter;false;onPattern;(String);;Argument[0];regex-compile" ] } } diff --git a/java/ql/lib/semmle/code/java/regex/regex.qll b/java/ql/lib/semmle/code/java/regex/regex.qll index 5dae7020fd9..4ad795cdd39 100644 --- a/java/ql/lib/semmle/code/java/regex/regex.qll +++ b/java/ql/lib/semmle/code/java/regex/regex.qll @@ -1,23 +1,5 @@ import java -import semmle.code.java.dataflow.DataFlow2 -import semmle.code.java.dataflow.ExternalFlow - -class RegexFlowConf extends DataFlow2::Configuration { - RegexFlowConf() { this = "RegexFlowConf" } - - override predicate isSource(DataFlow2::Node node) { node.asExpr() instanceof StringLiteral } - - override predicate isSink(DataFlow2::Node node) { sinkNode(node, "regex-use") } -} - -/** - * Holds if `s` is used as a regex, with the mode `mode` (if known). - * If regex mode is not known, `mode` will be `"None"`. - */ -predicate used_as_regex(Expr s, string mode) { - any(RegexFlowConf c).hasFlow(DataFlow2::exprNode(s), _) and - mode = "None" // TODO: proper mode detection -} +private import RegexFlowConfigs /** * A string literal that is used as a regular exprssion. diff --git a/java/ql/src/Security/CWE/CWE-730/PolynomialReDoS.ql b/java/ql/src/Security/CWE/CWE-730/PolynomialReDoS.ql index 13d5bb8e8a6..40bc4845a7c 100644 --- a/java/ql/src/Security/CWE/CWE-730/PolynomialReDoS.ql +++ b/java/ql/src/Security/CWE/CWE-730/PolynomialReDoS.ql @@ -14,21 +14,35 @@ import java import semmle.code.java.security.performance.SuperlinearBackTracking import semmle.code.java.dataflow.DataFlow -// import semmle.python.security.dataflow.PolynomialReDoS +import semmle.code.java.regex.RegexTreeView +import semmle.code.java.regex.RegexFlowConfigs +import semmle.code.java.dataflow.FlowSources import DataFlow::PathGraph +class PolynomialRedosSink extends DataFlow::Node { + RegExpLiteral reg; + + PolynomialRedosSink() { regex_match(reg.getRegex(), this.asExpr()) } + + RegExpTerm getRegExp() { result = reg } +} + +class PolynomialRedosConfig extends DataFlow::Configuration { + PolynomialRedosConfig() { this = "PolynomialRodisConfig" } + + override predicate isSource(DataFlow::Node src) { src instanceof RemoteFlowSource } + + override predicate isSink(DataFlow::Node sink) { sink instanceof PolynomialRedosSink } +} + from - PolynomialReDoS::Configuration config, DataFlow::PathNode source, DataFlow::PathNode sink, - PolynomialReDoS::Sink sinkNode, PolynomialBackTrackingTerm regexp + PolynomialRedosConfig config, DataFlow::PathNode source, DataFlow::PathNode sink, + PolynomialRedosSink sinkNode, PolynomialBackTrackingTerm regexp where config.hasFlowPath(source, sink) and sinkNode = sink.getNode() and regexp.getRootTerm() = sinkNode.getRegExp() -// not ( -// source.getNode().(Source).getKind() = "url" and -// regexp.isAtEndLine() -// ) -select sinkNode.getHighlight(), source, sink, +select sinkNode, source, sink, "This $@ that depends on $@ may run slow on strings " + regexp.getPrefixMessage() + "with many repetitions of '" + regexp.getPumpString() + "'.", regexp, "regular expression", source.getNode(), "a user-provided value" From d04c99b0beba80bbfb70a24adbb5550eb7fc3ca7 Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Thu, 13 Jan 2022 14:15:06 +0000 Subject: [PATCH 14/91] Support quote sequences --- .../semmle/code/java/regex/RegexTreeView.qll | 35 +++++++++- java/ql/lib/semmle/code/java/regex/regex.qll | 65 ++++++++++++++++++- 2 files changed, 94 insertions(+), 6 deletions(-) diff --git a/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll b/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll index 1b6013b26a0..89242eea4d6 100644 --- a/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll +++ b/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll @@ -40,6 +40,8 @@ newtype TRegExpParent = TRegExpSpecialChar(Regex re, int start, int end) { re.specialCharacter(start, end, _) } or /** A normal character */ TRegExpNormalChar(Regex re, int start, int end) { re.normalCharacter(start, end) } or + /** A quoted sequence */ + TRegExpQuote(Regex re, int start, int end) { re.quote(start, end) } or /** A back reference */ TRegExpBackRef(Regex re, int start, int end) { re.backreference(start, end) } @@ -107,6 +109,8 @@ class RegExpTerm extends RegExpParent { or this = TRegExpNormalChar(re, start, end) or + this = TRegExpQuote(re, start, end) + or this = TRegExpGroup(re, start, end) or this = TRegExpQuantifier(re, start, end) @@ -675,9 +679,34 @@ class RegExpNormalChar extends RegExpTerm, TRegExpNormalChar { override string getPrimaryQLClass() { result = "RegExpNormalChar" } } +/** + * A quoted sequence. + * + * Example: + * ``` + * \Qabc\E + * ``` + */ +class RegExpQuote extends RegExpTerm, TRegExpQuote { + string value; + + RegExpQuote() { + exists(int inner_start, int inner_end | + this = TRegExpQuote(re, start, end) and + re.quote(start, end, inner_start, inner_end) and + value = re.getText().substring(inner_start, inner_end) + ) + } + + /** Gets the string matched by this quote term. */ + string getValue() { result = value } + + override string getPrimaryQLClass() { result = "RegExpQuote" } +} + /** * A constant regular expression term, that is, a regular expression - * term matching a single string. Currently, this will always be a single character. + * term matching a single string. This can be a single character or a quoted sequence. * * Example: * @@ -689,14 +718,14 @@ class RegExpConstant extends RegExpTerm { string value; RegExpConstant() { - this = TRegExpNormalChar(re, start, end) and + (this = TRegExpNormalChar(re, start, end) or this = TRegExpQuote(re, start, end)) and not this instanceof RegExpCharacterClassEscape and // exclude chars in qualifiers // TODO: push this into regex library not exists(int qstart, int qend | re.qualifiedPart(_, qstart, qend, _, _) | qstart <= start and end <= qend ) and - value = this.(RegExpNormalChar).getValue() + (value = this.(RegExpNormalChar).getValue() or value = this.(RegExpQuote).getValue()) } /** diff --git a/java/ql/lib/semmle/code/java/regex/regex.qll b/java/ql/lib/semmle/code/java/regex/regex.qll index 4ad795cdd39..185a4981b00 100644 --- a/java/ql/lib/semmle/code/java/regex/regex.qll +++ b/java/ql/lib/semmle/code/java/regex/regex.qll @@ -189,13 +189,17 @@ abstract class RegexString extends Expr { } /** Holds if the character at `pos` is a "\" that is actually escaping what comes after. */ - predicate escapingChar(int pos) { this.escaping(pos) = true } + predicate escapingChar(int pos) { + this.escaping(pos) = true and + not exists(int x, int y | this.quote(x, y) and pos in [x .. y - 1]) + } /** * Helper predicate for `escapingChar`. * In order to avoid negative recusrion, we return a boolean. * This way, we can refer to `escaping(pos - 1).booleanNot()` * rather than to a negated version of `escaping(pos)`. + * Does not take into account escape characters inside quote sequences. */ private boolean escaping(int pos) { pos = -1 and result = false @@ -205,6 +209,53 @@ abstract class RegexString extends Expr { this.getChar(pos) != "\\" and result = false } + /** + * Helper predicate for `quoteSequence`. + * Holds if the char at `pos` could be the beginning of a quote delimiter, i.e. `\Q` (non-escaped) or `\E` (escaping not checked, as quote sequences turn off escapes). + * Result is `true` for `\Q` and `false` for `\E`. + */ + private boolean quote_delimiter(int pos) { + result = true and + this.escaping(pos) = true and + this.getChar(pos + 1) = "Q" + or + result = false and + this.getChar(pos) = "\\" and + this.getChar(pos + 1) = "E" + } + + /** + * Helper predicate for `quoteSequence`. + * Holds if the char at `pos` is the one-based `index`th occourence of a quote delimiter (`\Q` or `\E`) + * Result is `true` for `\Q` and `false` for `\E`. + */ + private boolean quote_delimiter(int index, int pos) { + result = this.quote_delimiter(pos) and + pos = rank[index](int p | this.quote_delimiter(p) = [true, false]) + } + + /** Holds if a quoted sequence is found between `start` and `end` */ + predicate quote(int start, int end) { this.quote(start, end, _, _) } + + /** Holds if a quoted sequence is fund between `start` and `end`, with ontent found between `inner_start` and `inner_end`. */ + predicate quote(int start, int end, int inner_start, int inner_end) { + exists(int index | + this.quote_delimiter(index, start) = true and + ( + index = 1 + or + this.quote_delimiter(index - 1, _) = false + ) and + inner_start = start + 2 and + inner_end = end - 2 and + inner_end > inner_start and + this.quote_delimiter(inner_end) = false and + not exists(int mid | + this.quote_delimiter(mid) = false and mid in [inner_start .. inner_end - 1] + ) + ) + } + /** Gets the text of this regex */ string getText() { result = this.(StringLiteral).getValue() } @@ -212,7 +263,8 @@ abstract class RegexString extends Expr { string nonEscapedCharAt(int i) { result = this.getText().charAt(i) and - not exists(int x, int y | this.escapedCharacter(x, y) and i in [x .. y - 1]) + not exists(int x, int y | this.escapedCharacter(x, y) and i in [x .. y - 1]) and + not exists(int x, int y | this.quote(x, y) and i in [x .. y - 1]) } private predicate isOptionDivider(int i) { this.nonEscapedCharAt(i) = "|" } @@ -728,7 +780,8 @@ abstract class RegexString extends Expr { this.character(start, _) or this.isGroupStart(start) or this.charSet(start, _) or - this.backreference(start, _) + this.backreference(start, _) or + this.quote(start, _) } private predicate item_end(int end) { @@ -739,6 +792,8 @@ abstract class RegexString extends Expr { this.charSet(_, end) or this.qualifier(_, end, _, _) + or + this.quote(_, end) } private predicate top_level(int start, int end) { @@ -846,6 +901,8 @@ abstract class RegexString extends Expr { this.qualifiedItem(start, end, _, _) or this.charSet(start, end) + or + this.quote(start, end) ) and this.firstPart(start, end) } @@ -861,6 +918,8 @@ abstract class RegexString extends Expr { this.qualifiedItem(start, end, _, _) or this.charSet(start, end) + or + this.quote(start, end) ) and this.lastPart(start, end) } From 7530902ad70bf76e78f335d3858a01c1dea6fa5d Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Wed, 19 Jan 2022 11:43:46 +0000 Subject: [PATCH 15/91] Add approximate support for nested character classes. This shouldn't fail to parse on any correctly formed character class; but may give incorrect contents when nested classes are involved. --- java/ql/lib/semmle/code/java/regex/regex.qll | 126 +++++++------------ 1 file changed, 45 insertions(+), 81 deletions(-) diff --git a/java/ql/lib/semmle/code/java/regex/regex.qll b/java/ql/lib/semmle/code/java/regex/regex.qll index 185a4981b00..71b2cc7369e 100644 --- a/java/ql/lib/semmle/code/java/regex/regex.qll +++ b/java/ql/lib/semmle/code/java/regex/regex.qll @@ -3,87 +3,13 @@ private import RegexFlowConfigs /** * A string literal that is used as a regular exprssion. - * TODO: adjust parser for java regex syntax */ abstract class RegexString extends Expr { RegexString() { this instanceof StringLiteral } - /** - * Helper predicate for `char_set_start(int start, int end)`. - * - * In order to identify left brackets ('[') which actually start a character class, - * we perform a left to right scan of the string. - * - * To avoid negative recursion we return a boolean. See `escaping`, - * the helper for `escapingChar`, for a clean use of this pattern. - * - * result is true for those start chars that actually mark a start of a char set. - */ - boolean char_set_start(int pos) { - exists(int index | - // is opening bracket - this.char_set_delimiter(index, pos) = true and - ( - // if this is the first bracket, `pos` starts a char set - index = 1 and result = true - or - // if the previous char set delimiter was not a closing bracket, `pos` does - // not start a char set. This is needed to handle cases such as `[[]` (a - // char set that matches the `[` char) - index > 1 and - not this.char_set_delimiter(index - 1, _) = false and - result = false - or - // special handling of cases such as `[][]` (the character-set of the characters `]` and `[`). - exists(int prev_closing_bracket_pos | - // previous bracket is a closing bracket - this.char_set_delimiter(index - 1, prev_closing_bracket_pos) = false and - if - // check if the character that comes before the previous closing bracket - // is an opening bracket (taking `^` into account) - exists(int pos_before_prev_closing_bracket | - if this.getChar(prev_closing_bracket_pos - 1) = "^" - then pos_before_prev_closing_bracket = prev_closing_bracket_pos - 2 - else pos_before_prev_closing_bracket = prev_closing_bracket_pos - 1 - | - this.char_set_delimiter(index - 2, pos_before_prev_closing_bracket) = true - ) - then - // brackets without anything in between is not valid character ranges, so - // the first closing bracket in `[]]` and `[^]]` does not count, - // - // and we should _not_ mark the second opening bracket in `[][]` and `[^][]` - // as starting a new char set. ^ ^ - exists(int pos_before_prev_closing_bracket | - this.char_set_delimiter(index - 2, pos_before_prev_closing_bracket) = true - | - result = this.char_set_start(pos_before_prev_closing_bracket).booleanNot() - ) - else - // if not, `pos` does in fact mark a real start of a character range - result = true - ) - ) - ) - } - - /** - * Helper predicate for chars that could be character-set delimiters. - * Holds if the (non-escaped) char at `pos` in the string, is the (one-based) `index` occurrence of a bracket (`[` or `]`) in the string. - * Result if `true` is the char is `[`, and `false` if the char is `]`. - */ - boolean char_set_delimiter(int index, int pos) { - pos = rank[index](int p | this.nonEscapedCharAt(p) = "[" or this.nonEscapedCharAt(p) = "]") and - ( - this.nonEscapedCharAt(pos) = "[" and result = true - or - this.nonEscapedCharAt(pos) = "]" and result = false - ) - } - - /** Hold is a character set starts between `start` and `end`. */ - predicate char_set_start(int start, int end) { - this.char_set_start(start) = true and + /** Holds if a character set starts between `start` and `end`. */ + private predicate char_set_start0(int start, int end) { + this.nonEscapedCharAt(start) = "[" and ( this.getChar(start + 1) = "^" and end = start + 2 or @@ -91,7 +17,41 @@ abstract class RegexString extends Expr { ) } - /** Whether there is a character class, between start (inclusive) and end (exclusive) */ + /** Holds if the character at `pos` marks the end of a character class. */ + private predicate char_set_end0(int pos) { + this.nonEscapedCharAt(pos) = "]" and + /* special case: `[]]` and `[^]]` are valid char classes. */ + not char_set_start0(_, pos - 1) + } + + /** + * Gets the nesting depth of charcter classes at position `pos` + */ + private int char_set_depth(int pos) { + exists(this.getChar(pos)) and + result = + count(int i | i < pos and this.char_set_start0(i, _)) - + count(int i | i < pos and this.char_set_end0(i)) + } + + /** Hold if a top-level character set starts between `start` and `end`. */ + predicate char_set_start(int start, int end) { + this.char_set_start0(start, end) and + this.char_set_depth(start) = 0 + } + + /** Holds if a top-level character set ends at `pos`. */ + predicate char_set_end(int pos) { + this.char_set_end0(pos) and + this.char_set_depth(pos) = 1 + } + + /** + * Whether there is a top-level character class, between start (inclusive) and end (exclusive) + * + * For now, nested character classes are approximated by only considering the top-level class for parsing. + * This leads to very similar results for ReDoS queries. + */ predicate charSet(int start, int end) { exists(int inner_start, int inner_end | this.char_set_start(start, inner_start) and @@ -99,8 +59,8 @@ abstract class RegexString extends Expr { | end = inner_end + 1 and inner_end > inner_start and - this.nonEscapedCharAt(inner_end) = "]" and - not exists(int mid | this.nonEscapedCharAt(mid) = "]" | mid > inner_start and mid < inner_end) + this.char_set_end(inner_end) and + not exists(int mid | char_set_end(mid) | mid > inner_start and mid < inner_end) ) } @@ -118,6 +78,8 @@ abstract class RegexString extends Expr { this.escapedCharacter(start, end) or exists(this.nonEscapedCharAt(start)) and end = start + 1 + or + this.quote(start, end) ) or this.char_set_token(charset_start, _, start) and @@ -126,7 +88,9 @@ abstract class RegexString extends Expr { or exists(this.nonEscapedCharAt(start)) and end = start + 1 and - not this.getChar(start) = "]" + not this.char_set_end(start) + or + this.quote(start, end) ) } From 11e465f2acb63c823039f877d7df78597f3e6c8f Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Wed, 19 Jan 2022 16:57:23 +0000 Subject: [PATCH 16/91] Implement remaining syntax differences --- .../semmle/code/java/regex/RegexTreeView.qll | 10 +- java/ql/lib/semmle/code/java/regex/regex.qll | 95 ++++++++++++------- 2 files changed, 70 insertions(+), 35 deletions(-) diff --git a/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll b/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll index 89242eea4d6..5603ad82108 100644 --- a/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll +++ b/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll @@ -472,6 +472,7 @@ class RegExpEscape extends RegExpNormalChar { this.getUnescaped() = "t" and result = "\t" or // TODO: Find a way to include a formfeed character + // also the alert/bell character for \a and escape character for \e. // this.getUnescaped() = "f" and result = " " // or this.isUnicode() and @@ -479,7 +480,7 @@ class RegExpEscape extends RegExpNormalChar { } /** Holds if this terms name is given by the part following the escape character. */ - predicate isIdentityEscape() { not this.getUnescaped() in ["n", "r", "t", "f"] } + predicate isIdentityEscape() { not this.getUnescaped() in ["n", "r", "t", "f", "a", "e"] } override string getPrimaryQLClass() { result = "RegExpEscape" } @@ -494,7 +495,7 @@ class RegExpEscape extends RegExpNormalChar { /** * Holds if this is a unicode escape. */ - private predicate isUnicode() { this.getText().prefix(2) = ["\\u", "\\U"] } + private predicate isUnicode() { this.getText().prefix(2) = "\\u" } /** * Gets the unicode char for this escape. @@ -551,7 +552,10 @@ private int toHex(string hex) { * ``` */ class RegExpCharacterClassEscape extends RegExpEscape { - RegExpCharacterClassEscape() { this.getValue() in ["d", "D", "s", "S", "w", "W"] } + RegExpCharacterClassEscape() { + this.getValue() in ["d", "D", "s", "S", "w", "W", "h", "H", "v", "V"] or + this.getValue().charAt(0) in ["p", "P"] + } override RegExpTerm getChild(int i) { none() } diff --git a/java/ql/lib/semmle/code/java/regex/regex.qll b/java/ql/lib/semmle/code/java/regex/regex.qll index 71b2cc7369e..f3d96d828b9 100644 --- a/java/ql/lib/semmle/code/java/regex/regex.qll +++ b/java/ql/lib/semmle/code/java/regex/regex.qll @@ -30,8 +30,12 @@ abstract class RegexString extends Expr { private int char_set_depth(int pos) { exists(this.getChar(pos)) and result = - count(int i | i < pos and this.char_set_start0(i, _)) - - count(int i | i < pos and this.char_set_end0(i)) + max(int j | + j = 0 or + j = + count(int i | i < pos and this.char_set_start0(i, _)) - + count(int i | i < pos and this.char_set_end0(i)) + ) } /** Hold if a top-level character set starts between `start` and `end`. */ @@ -168,7 +172,12 @@ abstract class RegexString extends Expr { private boolean escaping(int pos) { pos = -1 and result = false or - this.getChar(pos) = "\\" and result = this.escaping(pos - 1).booleanNot() + this.getChar(pos) = "\\" and + ( + if this.getChar(pos - 1) = "c" // in `\c\`, the latter `\` isn't escaping + then result = this.escaping(pos - 2).booleanNot() + else result = this.escaping(pos - 1).booleanNot() + ) or this.getChar(pos) != "\\" and result = false } @@ -220,6 +229,16 @@ abstract class RegexString extends Expr { ) } + /** + * A control sequence, `\cx` + * `x` may be any ascii character including special characters. + */ + predicate controlEscape(int start, int end) { + this.escapingChar(start) and + this.getChar(start + 1) = "c" and + end = start + 3 + } + /** Gets the text of this regex */ string getText() { result = this.(StringLiteral).getValue() } @@ -228,7 +247,8 @@ abstract class RegexString extends Expr { string nonEscapedCharAt(int i) { result = this.getText().charAt(i) and not exists(int x, int y | this.escapedCharacter(x, y) and i in [x .. y - 1]) and - not exists(int x, int y | this.quote(x, y) and i in [x .. y - 1]) + not exists(int x, int y | this.quote(x, y) and i in [x .. y - 1]) and + not exists(int x, int y | this.controlEscape(x, y) and i in [x .. y - 1]) } private predicate isOptionDivider(int i) { this.nonEscapedCharAt(i) = "|" } @@ -246,10 +266,10 @@ abstract class RegexString extends Expr { ) } - /** Named unicode characters, eg \N{degree sign} */ - private predicate escapedName(int start, int end) { + /** An escape sequence that includes braces, such as named characters (\N{degree sign}), named classes (\p{Lower}), or hex values (\x{h..h}) */ + private predicate escapedBraces(int start, int end) { this.escapingChar(start) and - this.getChar(start + 1) = "N" and + this.getChar(start + 1) = ["N", "p", "P", "x"] and this.getChar(start + 2) = "{" and this.getChar(end - 1) = "}" and end > start and @@ -266,26 +286,38 @@ abstract class RegexString extends Expr { not this.numbered_backreference(start, _, _) and ( // hex value \xhh - this.getChar(start + 1) = "x" and end = start + 4 + this.getChar(start + 1) = "x" and + this.getChar(start + 2) != "{" and + end = start + 4 or - // octal value \o, \oo, or \ooo - end in [start + 2 .. start + 4] and + // octal value \0o, \0oo, or \0ooo. Max of 0377. + this.getChar(start + 1) = "0" and + end in [start + 3 .. start + 5] and forall(int i | i in [start + 1 .. end - 1] | this.isOctal(i)) and + (end = start + 5 implies this.getChar(start + 2) <= "3") and not ( - end < start + 4 and - this.isOctal(end) + end < start + 5 and + this.isOctal(end) and + (end = start + 4 implies this.getChar(start + 2) <= "3") ) or // 16-bit hex value \uhhhh this.getChar(start + 1) = "u" and end = start + 6 or - // 32-bit hex value \Uhhhhhhhh - this.getChar(start + 1) = "U" and end = start + 10 + escapedBraces(start, end) or - escapedName(start, end) + // Boundry matchers \b, \b{g} + this.getChar(start + 1) = "b" and + ( + if this.getText().substring(start + 2, start + 5) = "{g}" + then end = start + 5 + else end = start + 2 + ) + or + this.controlEscape(start, end) or // escape not handled above, update when adding a new case - not this.getChar(start + 1) in ["x", "u", "U", "N"] and + not this.getChar(start + 1) in ["x", "0", "u", "p", "P", "N", "b", "c"] and not exists(this.getChar(start + 1).toInt()) and end = start + 2 ) @@ -370,7 +402,7 @@ abstract class RegexString extends Expr { this.group(start, end) and exists(int name_end | this.named_group_start(start, name_end) and - result = this.getText().substring(start + 4, name_end - 1) + result = this.getText().substring(start + 3, name_end - 1) ) } @@ -464,7 +496,7 @@ abstract class RegexString extends Expr { or this.negative_lookbehind_assertion_start(start, end) or - this.comment_group_start(start, end) + this.atomic_group_start(start, end) or this.simple_group_start(start, end) } @@ -485,12 +517,11 @@ abstract class RegexString extends Expr { private predicate named_group_start(int start, int end) { this.isGroupStart(start) and this.getChar(start + 1) = "?" and - this.getChar(start + 2) = "P" and - this.getChar(start + 3) = "<" and - not this.getChar(start + 4) = "=" and - not this.getChar(start + 4) = "!" and + this.getChar(start + 2) = "<" and + not this.getChar(start + 3) = "=" and + not this.getChar(start + 3) = "!" and exists(int name_end | - name_end = min(int i | i > start + 4 and this.getChar(i) = ">") and + name_end = min(int i | i > start + 3 and this.getChar(i) = ">") and end = name_end + 1 ) } @@ -498,7 +529,7 @@ abstract class RegexString extends Expr { private predicate named_backreference_start(int start, int end) { this.isGroupStart(start) and this.getChar(start + 1) = "?" and - this.getChar(start + 2) = "P" and + this.getChar(start + 2) = "k" and this.getChar(start + 3) = "=" and // Should this be looking for unescaped ")"? // TODO: test this @@ -510,7 +541,7 @@ abstract class RegexString extends Expr { this.getChar(start + 1) = "?" and end = start + 3 and c = this.getChar(start + 2) and - c in ["i", "L", "m", "s", "u", "x"] + c in ["i", "m", "s", "u", "x", "U"] } /** @@ -521,8 +552,6 @@ abstract class RegexString extends Expr { exists(string c | this.flag_group_start(_, _, c) | c = "i" and result = "IGNORECASE" or - c = "L" and result = "LOCALE" - or c = "m" and result = "MULTILINE" or c = "s" and result = "DOTALL" @@ -530,6 +559,8 @@ abstract class RegexString extends Expr { c = "u" and result = "UNICODE" or c = "x" and result = "VERBOSE" + or + c = "U" and result = "UNICODECLASS" ) } @@ -563,10 +594,10 @@ abstract class RegexString extends Expr { end = start + 4 } - private predicate comment_group_start(int start, int end) { + private predicate atomic_group_start(int start, int end) { this.isGroupStart(start) and this.getChar(start + 1) = "?" and - this.getChar(start + 2) = "#" and + this.getChar(start + 2) = ">" and end = start + 3 } @@ -633,10 +664,10 @@ abstract class RegexString extends Expr { private predicate qualifier(int start, int end, boolean maybe_empty, boolean may_repeat_forever) { this.short_qualifier(start, end, maybe_empty, may_repeat_forever) and - not this.getChar(end) = "?" + not this.getChar(end) = ["?", "+"] or exists(int short_end | this.short_qualifier(start, short_end, maybe_empty, may_repeat_forever) | - if this.getChar(short_end) = "?" then end = short_end + 1 else end = short_end + if this.getChar(short_end) = ["?", "+"] then end = short_end + 1 else end = short_end ) } @@ -897,11 +928,11 @@ class Regex extends RegexString { * Gets a mode (if any) of this regular expression. Can be any of: * DEBUG * IGNORECASE - * LOCALE * MULTILINE * DOTALL * UNICODE * VERBOSE + * UNICODECLASS */ string getAMode() { result != "None" and From f9f7a01f578b3a1d4c6e5e586f906c5d3674240e Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Wed, 2 Feb 2022 13:53:41 +0000 Subject: [PATCH 17/91] Add Java ReDoS libraries to identical-files.json --- config/identical-files.json | 15 +- .../java/security/performance/ReDoSUtil.qll | 152 ++++++++++++------ .../security/performance/RegExpTreeView.qll | 8 + 3 files changed, 120 insertions(+), 55 deletions(-) diff --git a/config/identical-files.json b/config/identical-files.json index 2ff65c453f0..841274f3a3a 100644 --- a/config/identical-files.json +++ b/config/identical-files.json @@ -475,20 +475,23 @@ "python/ql/lib/semmle/python/security/internal/SensitiveDataHeuristics.qll", "ruby/ql/lib/codeql/ruby/security/internal/SensitiveDataHeuristics.qll" ], - "ReDoS Util Python/JS/Ruby": [ + "ReDoS Util Python/JS/Ruby/Java": [ "javascript/ql/lib/semmle/javascript/security/performance/ReDoSUtil.qll", "python/ql/lib/semmle/python/security/performance/ReDoSUtil.qll", - "ruby/ql/lib/codeql/ruby/security/performance/ReDoSUtil.qll" + "ruby/ql/lib/codeql/ruby/security/performance/ReDoSUtil.qll", + "java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll" ], - "ReDoS Exponential Python/JS/Ruby": [ + "ReDoS Exponential Python/JS/Ruby/Java": [ "javascript/ql/lib/semmle/javascript/security/performance/ExponentialBackTracking.qll", "python/ql/lib/semmle/python/security/performance/ExponentialBackTracking.qll", - "ruby/ql/lib/codeql/ruby/security/performance/ExponentialBackTracking.qll" + "ruby/ql/lib/codeql/ruby/security/performance/ExponentialBackTracking.qll", + "java/ql/lib/semmle/code/java/security/performance/ExponentialBackTracking.qll" ], - "ReDoS Polynomial Python/JS/Ruby": [ + "ReDoS Polynomial Python/JS/Ruby/Java": [ "javascript/ql/lib/semmle/javascript/security/performance/SuperlinearBackTracking.qll", "python/ql/lib/semmle/python/security/performance/SuperlinearBackTracking.qll", - "ruby/ql/lib/codeql/ruby/security/performance/SuperlinearBackTracking.qll" + "ruby/ql/lib/codeql/ruby/security/performance/SuperlinearBackTracking.qll", + "java/ql/lib/semmle/code/java/security/performance/SuperlinearBackTracking.qll" ], "BadTagFilterQuery Python/JS/Ruby": [ "javascript/ql/lib/semmle/javascript/security/BadTagFilterQuery.qll", diff --git a/java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll b/java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll index 2cd324ed8f7..54e69cc3178 100644 --- a/java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll +++ b/java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll @@ -140,9 +140,9 @@ class RegExpRoot extends RegExpTerm { // there is at least one repetition getRoot(any(InfiniteRepetitionQuantifier q)) = this and // is actually used as a RegExp - isUsedAsRegExp() and + this.isUsedAsRegExp() and // not excluded for library specific reasons - not isExcluded(getRootTerm().getParent()) + not isExcluded(this.getRootTerm().getParent()) } } @@ -218,7 +218,7 @@ private newtype TInputSymbol = recc instanceof RegExpCharacterClass and not recc.(RegExpCharacterClass).isUniversalClass() or - recc instanceof RegExpCharacterClassEscape + isEscapeClass(recc, _) ) } or /** An input symbol representing all characters matched by `.`. */ @@ -302,7 +302,7 @@ abstract class CharacterClass extends InputSymbol { /** * Gets a character matched by this character class. */ - string choose() { result = getARelevantChar() and matches(result) } + string choose() { result = this.getARelevantChar() and this.matches(result) } } /** @@ -340,13 +340,13 @@ private module CharacterClasses { char <= hi ) or - exists(RegExpCharacterClassEscape escape | escape = child | - escape.getValue() = escape.getValue().toLowerCase() and - classEscapeMatches(escape.getValue(), char) + exists(string charClass | isEscapeClass(child, charClass) | + charClass.toLowerCase() = charClass and + classEscapeMatches(charClass, char) or char = getARelevantChar() and - escape.getValue() = escape.getValue().toUpperCase() and - not classEscapeMatches(escape.getValue().toLowerCase(), char) + charClass.toUpperCase() = charClass and + not classEscapeMatches(charClass, char) ) ) } @@ -409,10 +409,10 @@ private module CharacterClasses { or child.(RegExpCharacterRange).isRange(_, result) or - exists(RegExpCharacterClassEscape escape | child = escape | - result = min(string s | classEscapeMatches(escape.getValue().toLowerCase(), s)) + exists(string charClass | isEscapeClass(child, charClass) | + result = min(string s | classEscapeMatches(charClass.toLowerCase(), s)) or - result = max(string s | classEscapeMatches(escape.getValue().toLowerCase(), s)) + result = max(string s | classEscapeMatches(charClass.toLowerCase(), s)) ) ) } @@ -466,33 +466,36 @@ private module CharacterClasses { * An implementation of `CharacterClass` for \d, \s, and \w. */ private class PositiveCharacterClassEscape extends CharacterClass { - RegExpCharacterClassEscape cc; + RegExpTerm cc; + string charClass; PositiveCharacterClassEscape() { - this = getCanonicalCharClass(cc) and cc.getValue() = ["d", "s", "w"] + isEscapeClass(cc, charClass) and + this = getCanonicalCharClass(cc) and + charClass = ["d", "s", "w"] } override string getARelevantChar() { - cc.getValue() = "d" and + charClass = "d" and result = ["0", "9"] or - cc.getValue() = "s" and + charClass = "s" and result = " " or - cc.getValue() = "w" and + charClass = "w" and result = ["a", "Z", "_", "0", "9"] } - override predicate matches(string char) { classEscapeMatches(cc.getValue(), char) } + override predicate matches(string char) { classEscapeMatches(charClass, char) } override string choose() { - cc.getValue() = "d" and + charClass = "d" and result = "9" or - cc.getValue() = "s" and + charClass = "s" and result = " " or - cc.getValue() = "w" and + charClass = "w" and result = "a" } } @@ -501,26 +504,29 @@ private module CharacterClasses { * An implementation of `CharacterClass` for \D, \S, and \W. */ private class NegativeCharacterClassEscape extends CharacterClass { - RegExpCharacterClassEscape cc; + RegExpTerm cc; + string charClass; NegativeCharacterClassEscape() { - this = getCanonicalCharClass(cc) and cc.getValue() = ["D", "S", "W"] + isEscapeClass(cc, charClass) and + this = getCanonicalCharClass(cc) and + charClass = ["D", "S", "W"] } override string getARelevantChar() { - cc.getValue() = "D" and + charClass = "D" and result = ["a", "Z", "!"] or - cc.getValue() = "S" and + charClass = "S" and result = ["a", "9", "!"] or - cc.getValue() = "W" and + charClass = "W" and result = [" ", "!"] } bindingset[char] override predicate matches(string char) { - not classEscapeMatches(cc.getValue().toLowerCase(), char) + not classEscapeMatches(charClass.toLowerCase(), char) } } } @@ -533,6 +539,55 @@ private class EdgeLabel extends TInputSymbol { } } +/** + * A RegExp term that acts like a plus. + * Either it's a RegExpPlus, or it is a range {1,X} where X is >= 30. + * 30 has been chosen as a threshold because for exponential blowup 2^30 is enough to get a decent DOS attack. + */ +private class EffectivelyPlus extends RegExpTerm { + EffectivelyPlus() { + this instanceof RegExpPlus + or + exists(RegExpRange range | + range.getLowerBound() = 1 and + (range.getUpperBound() >= 30 or not exists(range.getUpperBound())) + | + this = range + ) + } +} + +/** + * A RegExp term that acts like a star. + * Either it's a RegExpStar, or it is a range {0,X} where X is >= 30. + */ +private class EffectivelyStar extends RegExpTerm { + EffectivelyStar() { + this instanceof RegExpStar + or + exists(RegExpRange range | + range.getLowerBound() = 0 and + (range.getUpperBound() >= 30 or not exists(range.getUpperBound())) + | + this = range + ) + } +} + +/** + * A RegExp term that acts like a question mark. + * Either it's a RegExpQuestion, or it is a range {0,1}. + */ +private class EffectivelyQuestion extends RegExpTerm { + EffectivelyQuestion() { + this instanceof RegExpOpt + or + exists(RegExpRange range | range.getLowerBound() = 0 and range.getUpperBound() = 1 | + this = range + ) + } +} + /** * Gets the state before matching `t`. */ @@ -542,7 +597,7 @@ private State before(RegExpTerm t) { result = Match(t, 0) } /** * Gets a state the NFA may be in after matching `t`. */ -private State after(RegExpTerm t) { +State after(RegExpTerm t) { exists(RegExpAlt alt | t = alt.getAChild() | result = after(alt)) or exists(RegExpSequence seq, int i | t = seq.getChild(i) | @@ -553,14 +608,14 @@ private State after(RegExpTerm t) { or exists(RegExpGroup grp | t = grp.getAChild() | result = after(grp)) or - exists(RegExpStar star | t = star.getAChild() | result = before(star)) + exists(EffectivelyStar star | t = star.getAChild() | result = before(star)) or - exists(RegExpPlus plus | t = plus.getAChild() | + exists(EffectivelyPlus plus | t = plus.getAChild() | result = before(plus) or result = after(plus) ) or - exists(RegExpOpt opt | t = opt.getAChild() | result = after(opt)) + exists(EffectivelyQuestion opt | t = opt.getAChild() | result = after(opt)) or exists(RegExpRoot root | t = root | result = AcceptAnySuffix(root)) } @@ -599,7 +654,7 @@ predicate delta(State q1, EdgeLabel lbl, State q2) { q2 = after(cc) ) or - exists(RegExpCharacterClassEscape cc | + exists(RegExpTerm cc | isEscapeClass(cc, _) | q1 = before(cc) and lbl = CharClass(cc.getRawValue() + "|" + getCanonicalizationFlags(cc.getRootTerm())) and q2 = after(cc) @@ -611,15 +666,17 @@ predicate delta(State q1, EdgeLabel lbl, State q2) { or exists(RegExpGroup grp | lbl = Epsilon() | q1 = before(grp) and q2 = before(grp.getChild(0))) or - exists(RegExpStar star | lbl = Epsilon() | + exists(EffectivelyStar star | lbl = Epsilon() | q1 = before(star) and q2 = before(star.getChild(0)) or q1 = before(star) and q2 = after(star) ) or - exists(RegExpPlus plus | lbl = Epsilon() | q1 = before(plus) and q2 = before(plus.getChild(0))) + exists(EffectivelyPlus plus | lbl = Epsilon() | + q1 = before(plus) and q2 = before(plus.getChild(0)) + ) or - exists(RegExpOpt opt | lbl = Epsilon() | + exists(EffectivelyQuestion opt | lbl = Epsilon() | q1 = before(opt) and q2 = before(opt.getChild(0)) or q1 = before(opt) and q2 = after(opt) @@ -671,7 +728,7 @@ RegExpRoot getRoot(RegExpTerm term) { /** * A state in the NFA. */ -private newtype TState = +newtype TState = /** * A state representing that the NFA is about to match a term. * `i` is used to index into multi-char literals. @@ -801,29 +858,26 @@ InputSymbol getAnInputSymbolMatching(string char) { result = Any() } +/** + * Holds if `state` is a start state. + */ +predicate isStartState(State state) { + state = mkMatch(any(RegExpRoot r)) + or + exists(RegExpCaret car | state = after(car)) +} + /** * Predicates for constructing a prefix string that leads to a given state. */ private module PrefixConstruction { - /** - * Holds if `state` starts the string matched by the regular expression. - */ - private predicate isStartState(State state) { - state instanceof StateInPumpableRegexp and - ( - state = Match(any(RegExpRoot r), _) - or - exists(RegExpCaret car | state = after(car)) - ) - } - /** * Holds if `state` is the textually last start state for the regular expression. */ private predicate lastStartState(State state) { exists(RegExpRoot root | state = - max(State s, Location l | + max(StateInPumpableRegexp s, Location l | isStartState(s) and getRoot(s.getRepr()) = root and l = s.getRepr().getLocation() | s diff --git a/java/ql/lib/semmle/code/java/security/performance/RegExpTreeView.qll b/java/ql/lib/semmle/code/java/security/performance/RegExpTreeView.qll index ac220ec8a50..97c7fd5951e 100644 --- a/java/ql/lib/semmle/code/java/security/performance/RegExpTreeView.qll +++ b/java/ql/lib/semmle/code/java/security/performance/RegExpTreeView.qll @@ -5,6 +5,14 @@ import java import semmle.code.java.regex.RegexTreeView +/** + * Holds if `term` is an ecape class representing e.g. `\d`. + * `clazz` is which character class it represents, e.g. "d" for `\d`. + */ +predicate isEscapeClass(RegExpTerm term, string clazz) { + exists(RegExpCharacterClassEscape escape | term = escape | escape.getValue() = clazz) +} + /** * Holds if the regular expression should not be considered. * From ca422a2186e440c15189a0a23a66e868a60186eb Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Wed, 2 Feb 2022 13:59:31 +0000 Subject: [PATCH 18/91] Use explicit `this` --- java/ql/lib/semmle/code/java/regex/RegexTreeView.qll | 2 +- java/ql/lib/semmle/code/java/regex/regex.qll | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll b/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll index 5603ad82108..6e5c1785f8c 100644 --- a/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll +++ b/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll @@ -495,7 +495,7 @@ class RegExpEscape extends RegExpNormalChar { /** * Holds if this is a unicode escape. */ - private predicate isUnicode() { this.getText().prefix(2) = "\\u" } + private predicate isUnicode() { this.getText().matches("\\u%") } /** * Gets the unicode char for this escape. diff --git a/java/ql/lib/semmle/code/java/regex/regex.qll b/java/ql/lib/semmle/code/java/regex/regex.qll index f3d96d828b9..557c0cb495b 100644 --- a/java/ql/lib/semmle/code/java/regex/regex.qll +++ b/java/ql/lib/semmle/code/java/regex/regex.qll @@ -21,7 +21,7 @@ abstract class RegexString extends Expr { private predicate char_set_end0(int pos) { this.nonEscapedCharAt(pos) = "]" and /* special case: `[]]` and `[^]]` are valid char classes. */ - not char_set_start0(_, pos - 1) + not this.char_set_start0(_, pos - 1) } /** @@ -64,7 +64,7 @@ abstract class RegexString extends Expr { end = inner_end + 1 and inner_end > inner_start and this.char_set_end(inner_end) and - not exists(int mid | char_set_end(mid) | mid > inner_start and mid < inner_end) + not exists(int mid | this.char_set_end(mid) | mid > inner_start and mid < inner_end) ) } @@ -304,7 +304,7 @@ abstract class RegexString extends Expr { // 16-bit hex value \uhhhh this.getChar(start + 1) = "u" and end = start + 6 or - escapedBraces(start, end) + this.escapedBraces(start, end) or // Boundry matchers \b, \b{g} this.getChar(start + 1) = "b" and From 8e1918216e9683b5c58ed8e52e77023134474b1a Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Wed, 2 Feb 2022 17:18:14 +0000 Subject: [PATCH 19/91] Add PrintAst support for regex terms --- java/ql/lib/semmle/code/java/PrintAst.qll | 51 +++++++++++++++++++ .../semmle/code/java/regex/RegexTreeView.qll | 4 +- 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/java/ql/lib/semmle/code/java/PrintAst.qll b/java/ql/lib/semmle/code/java/PrintAst.qll index 9527787e3a4..4279c0e8e58 100644 --- a/java/ql/lib/semmle/code/java/PrintAst.qll +++ b/java/ql/lib/semmle/code/java/PrintAst.qll @@ -7,6 +7,7 @@ */ import java +import semmle.code.java.regex.RegexTreeView private newtype TPrintAstConfiguration = MkPrintAstConfiguration() @@ -131,6 +132,9 @@ private newtype TPrintAstNode = } or TImportsNode(CompilationUnit cu) { shouldPrint(cu, _) and exists(Import i | i.getCompilationUnit() = cu) + } or + TRegExpTermNode(RegExpTerm term) { + exists(StringLiteral str | term.getRootTerm() = getParsedRegExp(str) and shouldPrint(str, _)) } /** @@ -163,6 +167,12 @@ class PrintAstNode extends TPrintAstNode { */ Location getLocation() { none() } + predicate hasLocationInfo( + string filepath, int startline, int startcolumn, int endline, int endcolumn + ) { + this.getLocation().hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn) + } + /** * Gets the value of the property of this node, where the name of the property * is `key`. @@ -274,6 +284,47 @@ final class AnnotationPartNode extends ExprStmtNode { } } +/** + * A node representing a `StringLiteral`. + * It has a child if it is used as a regular expression, which is the root of the regular expression. + */ +final class StringLiteralNode extends ExprStmtNode { + StringLiteralNode() { element instanceof StringLiteral } + + override PrintAstNode getChild(int childIndex) { + childIndex = 0 and + result.(RegExpTermNode).getTerm() = getParsedRegExp(element) + } +} + +/** + * A node representing a regular expression term. + */ +class RegExpTermNode extends TRegExpTermNode, PrintAstNode { + RegExpTerm term; + + RegExpTermNode() { this = TRegExpTermNode(term) } + + /** Gets the `RegExpTerm` for this node. */ + RegExpTerm getTerm() { result = term } + + override PrintAstNode getChild(int childIndex) { + result.(RegExpTermNode).getTerm() = term.getChild(childIndex) + } + + override string toString() { + result = "[" + strictconcat(term.getPrimaryQLClass(), " | ") + "] " + term.toString() + } + + override Location getLocation() { result = term.getLocation() } + + override predicate hasLocationInfo( + string filepath, int startline, int startcolumn, int endline, int endcolumn + ) { + term.hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn) + } +} + /** * A node representing a `LocalVariableDeclExpr`. */ diff --git a/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll b/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll index 6e5c1785f8c..0bf2437c634 100644 --- a/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll +++ b/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll @@ -189,8 +189,8 @@ class RegExpTerm extends RegExpParent { ) { exists(int re_start, int re_end | re.getLocation().hasLocationInfo(filepath, startline, re_start, endline, re_end) and - startcolumn = re_start + start + 4 and - endcolumn = re_start + end + 3 + startcolumn = re_start + start + 1 and + endcolumn = re_start + end ) } From 28649da187e3e0f62ea39d1dc140e48a478b3b29 Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Wed, 9 Feb 2022 14:06:26 +0000 Subject: [PATCH 20/91] Add parser tests; fix some parser issues. [temporarily renamed existing regex/Test.java during rebasing to avoid conflict] --- java/ql/lib/semmle/code/java/regex/regex.qll | 29 ++--- .../regex/RegexParseTests.expected | 68 ++++++++++ .../library-tests/regex/RegexParseTests.ql | 10 ++ java/ql/test/library-tests/regex/Test.java | 117 +++--------------- java/ql/test/library-tests/regex/Test2.java | 104 ++++++++++++++++ 5 files changed, 211 insertions(+), 117 deletions(-) create mode 100644 java/ql/test/library-tests/regex/RegexParseTests.expected create mode 100644 java/ql/test/library-tests/regex/RegexParseTests.ql create mode 100644 java/ql/test/library-tests/regex/Test2.java diff --git a/java/ql/lib/semmle/code/java/regex/regex.qll b/java/ql/lib/semmle/code/java/regex/regex.qll index 557c0cb495b..35ca6fa9998 100644 --- a/java/ql/lib/semmle/code/java/regex/regex.qll +++ b/java/ql/lib/semmle/code/java/regex/regex.qll @@ -21,7 +21,7 @@ abstract class RegexString extends Expr { private predicate char_set_end0(int pos) { this.nonEscapedCharAt(pos) = "]" and /* special case: `[]]` and `[^]]` are valid char classes. */ - not this.char_set_start0(_, pos - 1) + not this.char_set_start0(_, pos) } /** @@ -283,7 +283,7 @@ abstract class RegexString extends Expr { */ predicate escapedCharacter(int start, int end) { this.escapingChar(start) and - not this.numbered_backreference(start, _, _) and + not this.backreference(start, _) and ( // hex value \xhh this.getChar(start + 1) = "x" and @@ -362,7 +362,8 @@ abstract class RegexString extends Expr { predicate character(int start, int end) { ( this.simpleCharacter(start, end) and - not exists(int x, int y | this.escapedCharacter(x, y) and x <= start and y >= end) + not exists(int x, int y | this.escapedCharacter(x, y) and x <= start and y >= end) and + not exists(int x, int y | this.quote(x, y) and x <= start and y >= end) or this.escapedCharacter(start, end) ) and @@ -486,8 +487,6 @@ abstract class RegexString extends Expr { or this.named_group_start(start, end) or - this.named_backreference_start(start, end) - or this.lookahead_assertion_start(start, end) or this.negative_lookahead_assertion_start(start, end) @@ -526,16 +525,6 @@ abstract class RegexString extends Expr { ) } - private predicate named_backreference_start(int start, int end) { - this.isGroupStart(start) and - this.getChar(start + 1) = "?" and - this.getChar(start + 2) = "k" and - this.getChar(start + 3) = "=" and - // Should this be looking for unescaped ")"? - // TODO: test this - end = min(int i | i > start + 4 and this.getChar(i) = "?") - } - private predicate flag_group_start(int start, int end, string c) { this.isGroupStart(start) and this.getChar(start + 1) = "?" and @@ -609,9 +598,11 @@ abstract class RegexString extends Expr { } private predicate named_backreference(int start, int end, string name) { - this.named_backreference_start(start, start + 4) and - end = min(int i | i > start + 4 and this.getChar(i) = ")") + 1 and - name = this.getText().substring(start + 4, end - 2) + this.escapingChar(start) and + this.getChar(start + 1) = "k" and + this.getChar(start + 2) = "<" and + end = min(int i | i > start + 2 and this.getChar(i) = ">") + 1 and + name = this.getText().substring(start + 3, end - 2) } private predicate numbered_backreference(int start, int end, int value) { @@ -660,6 +651,8 @@ abstract class RegexString extends Expr { this.charSet(start, end) or this.backreference(start, end) + or + this.quote(start, end) } private predicate qualifier(int start, int end, boolean maybe_empty, boolean may_repeat_forever) { diff --git a/java/ql/test/library-tests/regex/RegexParseTests.expected b/java/ql/test/library-tests/regex/RegexParseTests.expected new file mode 100644 index 00000000000..5a9b632d7c9 --- /dev/null +++ b/java/ql/test/library-tests/regex/RegexParseTests.expected @@ -0,0 +1,68 @@ +parseFailures +#select +| Test.java:5:10:5:16 | [A-Z\\d] | [RegExpCharacterClass] | +| Test.java:5:10:5:18 | [A-Z\\d]++ | [RegExpPlus] | +| Test.java:5:11:5:11 | A | [RegExpConstant,RegExpNormalChar] | +| Test.java:5:11:5:13 | A-Z | [RegExpCharacterRange] | +| Test.java:5:13:5:13 | Z | [RegExpConstant,RegExpNormalChar] | +| Test.java:5:14:5:15 | \\d | [RegExpCharacterClassEscape] | +| Test.java:6:10:6:39 | \\Q hello world [ *** \\Q ) ( \\E | [RegExpConstant,RegExpQuote] | +| Test.java:7:10:7:21 | [\\Q hi ] \\E] | [RegExpCharacterClass] | +| Test.java:7:11:7:20 | \\Q hi ] \\E | [RegExpConstant,RegExpQuote] | +| Test.java:8:10:8:12 | []] | [RegExpCharacterClass] | +| Test.java:8:11:8:11 | ] | [RegExpConstant,RegExpNormalChar] | +| Test.java:9:10:9:13 | [^]] | [RegExpCharacterClass] | +| Test.java:9:12:9:12 | ] | [RegExpConstant,RegExpNormalChar] | +| Test.java:10:10:10:20 | [abc[defg]] | [RegExpCharacterClass] | +| Test.java:10:11:10:11 | a | [RegExpConstant,RegExpNormalChar] | +| Test.java:10:12:10:12 | b | [RegExpConstant,RegExpNormalChar] | +| Test.java:10:13:10:13 | c | [RegExpConstant,RegExpNormalChar] | +| Test.java:10:14:10:14 | [ | [RegExpConstant,RegExpNormalChar] | +| Test.java:10:15:10:15 | d | [RegExpConstant,RegExpNormalChar] | +| Test.java:10:16:10:16 | e | [RegExpConstant,RegExpNormalChar] | +| Test.java:10:17:10:17 | f | [RegExpConstant,RegExpNormalChar] | +| Test.java:10:18:10:18 | g | [RegExpConstant,RegExpNormalChar] | +| Test.java:10:19:10:19 | ] | [RegExpConstant,RegExpNormalChar] | +| Test.java:11:10:11:53 | [abc&&[\\W\\p{Lower}\\P{Space}\\N{degree sign}]] | [RegExpCharacterClass] | +| Test.java:11:10:11:62 | [abc&&[\\W\\p{Lower}\\P{Space}\\N{degree sign}]]\\b7\\b{g}8 | [RegExpSequence] | +| Test.java:11:11:11:11 | a | [RegExpConstant,RegExpNormalChar] | +| Test.java:11:12:11:12 | b | [RegExpConstant,RegExpNormalChar] | +| Test.java:11:13:11:13 | c | [RegExpConstant,RegExpNormalChar] | +| Test.java:11:14:11:14 | & | [RegExpConstant,RegExpNormalChar] | +| Test.java:11:15:11:15 | & | [RegExpConstant,RegExpNormalChar] | +| Test.java:11:16:11:16 | [ | [RegExpConstant,RegExpNormalChar] | +| Test.java:11:17:11:18 | \\W | [RegExpCharacterClassEscape] | +| Test.java:11:19:11:27 | \\p{Lower} | [RegExpCharacterClassEscape] | +| Test.java:11:28:11:36 | \\P{Space} | [RegExpCharacterClassEscape] | +| Test.java:11:37:11:51 | \\N{degree sign} | [RegExpConstant,RegExpEscape] | +| Test.java:11:52:11:52 | ] | [RegExpConstant,RegExpNormalChar] | +| Test.java:11:54:11:55 | \\b | [RegExpConstant,RegExpEscape] | +| Test.java:11:56:11:56 | 7 | [RegExpConstant,RegExpNormalChar] | +| Test.java:11:57:11:61 | \\b{g} | [RegExpConstant,RegExpEscape] | +| Test.java:11:62:11:62 | 8 | [RegExpConstant,RegExpNormalChar] | +| Test.java:12:10:12:12 | \\cA | [RegExpConstant,RegExpEscape] | +| Test.java:13:10:13:12 | \\c( | [RegExpConstant,RegExpEscape] | +| Test.java:14:10:14:12 | \\c\\ | [RegExpConstant,RegExpEscape] | +| Test.java:14:10:14:16 | \\c\\(ab) | [RegExpSequence] | +| Test.java:14:13:14:16 | (ab) | [RegExpGroup] | +| Test.java:14:14:14:14 | a | [RegExpConstant,RegExpNormalChar] | +| Test.java:14:14:14:15 | ab | [RegExpSequence] | +| Test.java:14:15:14:15 | b | [RegExpConstant,RegExpNormalChar] | +| Test.java:15:10:15:15 | (?>hi) | [RegExpGroup] | +| Test.java:15:10:15:44 | (?>hi)(?hell*?o*+)123\\k | [RegExpSequence] | +| Test.java:15:13:15:13 | h | [RegExpConstant,RegExpNormalChar] | +| Test.java:15:13:15:14 | hi | [RegExpSequence] | +| Test.java:15:14:15:14 | i | [RegExpConstant,RegExpNormalChar] | +| Test.java:15:16:15:33 | (?hell*?o*+) | [RegExpGroup] | +| Test.java:15:24:15:24 | h | [RegExpConstant,RegExpNormalChar] | +| Test.java:15:24:15:32 | hell*?o*+ | [RegExpSequence] | +| Test.java:15:25:15:25 | e | [RegExpConstant,RegExpNormalChar] | +| Test.java:15:26:15:26 | l | [RegExpConstant,RegExpNormalChar] | +| Test.java:15:27:15:27 | l | [RegExpConstant,RegExpNormalChar] | +| Test.java:15:27:15:29 | l*? | [RegExpStar] | +| Test.java:15:30:15:30 | o | [RegExpConstant,RegExpNormalChar] | +| Test.java:15:30:15:32 | o*+ | [RegExpStar] | +| Test.java:15:34:15:34 | 1 | [RegExpConstant,RegExpNormalChar] | +| Test.java:15:35:15:35 | 2 | [RegExpConstant,RegExpNormalChar] | +| Test.java:15:36:15:36 | 3 | [RegExpConstant,RegExpNormalChar] | +| Test.java:15:37:15:44 | \\k | [RegExpBackRef] | diff --git a/java/ql/test/library-tests/regex/RegexParseTests.ql b/java/ql/test/library-tests/regex/RegexParseTests.ql new file mode 100644 index 00000000000..345031a3b2d --- /dev/null +++ b/java/ql/test/library-tests/regex/RegexParseTests.ql @@ -0,0 +1,10 @@ +import java +import semmle.code.java.regex.RegexTreeView +import semmle.code.java.regex.regex + +string getQLClases(RegExpTerm t) { result = "[" + strictconcat(t.getPrimaryQLClass(), ",") + "]" } + +query predicate parseFailures(Regex r, int i) { r.failedToParse(i) } + +from RegExpTerm t +select t, getQLClases(t) diff --git a/java/ql/test/library-tests/regex/Test.java b/java/ql/test/library-tests/regex/Test.java index b351c31812a..e061e48f9f9 100644 --- a/java/ql/test/library-tests/regex/Test.java +++ b/java/ql/test/library-tests/regex/Test.java @@ -1,104 +1,23 @@ -package generatedtest; - -import java.util.regex.Matcher; import java.util.regex.Pattern; -// Test case generated by GenerateFlowTestCase.ql -public class Test { +class Test { + static String[] regs = { + "[A-Z\\d]++", + "\\Q hello world [ *** \\Q ) ( \\E", + "[\\Q hi ] \\E]", + "[]]", + "[^]]", + "[abc[defg]]", + "[abc&&[\\W\\p{Lower}\\P{Space}\\N{degree sign}]]\\b7\\b{g}8", + "\\cA", + "\\c(", + "\\c\\(ab)", + "(?>hi)(?hell*?o*+)123\\k" + }; - private final String str_pattern = "\\$\\{(.*)\\}"; - private final Pattern pattern = Pattern.compile(str_pattern); - - Object source() { return null; } - void sink(Object o) { } - - public void test() throws Exception { - - { - // "java.util.regex;Matcher;false;group;;;Argument[-1];ReturnValue;taint" - String out = null; - String in = (String) source(); - Matcher m = pattern.matcher(in); - out = m.group("foo"); - sink(out); // $ hasTaintFlow + void test() { + for (int i = 0; i < regs.length; i++) { + Pattern.compile(regs[i]); + } } - { - // "java.util.regex;Matcher;false;group;;;Argument[-1];ReturnValue;taint" - String out = null; - String in = (String) source(); - Matcher m = pattern.matcher(in); - out = m.group(); - sink(out); // $ hasTaintFlow - } - { - // "java.util.regex;Matcher;false;group;;;Argument[-1];ReturnValue;taint" - String out = null; - String in = (String) source(); - Matcher m = pattern.matcher(in); - out = m.group(0); - sink(out); // $ hasTaintFlow - } - { - // "java.util.regex;Matcher;false;replaceAll;;;Argument[-1];ReturnValue;taint" - String out = null; - String in = (String) source(); - Matcher m = pattern.matcher(in); - out = m.replaceAll("foo"); - sink(out); // $ hasTaintFlow - } - { - // "java.util.regex;Matcher;false;replaceAll;;;Argument[0];ReturnValue;taint" - String out = null; - String in = (String) source(); - Matcher m = pattern.matcher("foo"); - out = m.replaceAll(in); - sink(out); // $ hasTaintFlow - } - { - // "java.util.regex;Matcher;false;replaceFirst;;;Argument[-1];ReturnValue;taint" - String out = null; - String in = (String) source(); - Matcher m = pattern.matcher(in); - out = m.replaceFirst("foo"); - sink(out); // $ hasTaintFlow - } - { - // "java.util.regex;Matcher;false;replaceFirst;;;Argument[0];ReturnValue;taint" - String out = null; - String in = (String) source(); - Matcher m = pattern.matcher("foo"); - out = m.replaceFirst(in); - sink(out); // $ hasTaintFlow - } - { - // "java.util.regex;Pattern;false;matcher;;;Argument[0];ReturnValue;taint" - Matcher out = null; - CharSequence in = (CharSequence)source(); - out = pattern.matcher(in); - sink(out); // $ hasTaintFlow - } - { - // "java.util.regex;Pattern;false;quote;;;Argument[0];ReturnValue;taint" - String out = null; - String in = (String)source(); - out = Pattern.quote(in); - sink(out); // $ hasTaintFlow - } - { - // "java.util.regex;Pattern;false;split;;;Argument[0];ReturnValue;taint" - String[] out = null; - CharSequence in = (CharSequence)source(); - out = pattern.split(in); - sink(out); // $ hasTaintFlow - } - { - // "java.util.regex;Pattern;false;split;;;Argument[0];ReturnValue;taint" - String[] out = null; - CharSequence in = (CharSequence)source(); - out = pattern.split(in, 0); - sink(out); // $ hasTaintFlow - } - - } - } diff --git a/java/ql/test/library-tests/regex/Test2.java b/java/ql/test/library-tests/regex/Test2.java new file mode 100644 index 00000000000..fd9be63b68b --- /dev/null +++ b/java/ql/test/library-tests/regex/Test2.java @@ -0,0 +1,104 @@ +package generatedtest; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +// Test case generated by GenerateFlowTestCase.ql +public class Test { + + private final String str_pattern = "\\$\\{(.*)\\}"; + private final Pattern pattern = Pattern.compile(str_pattern); + + Object source() { return null; } + void sink(Object o) { } + + public void test() throws Exception { + + { + // "java.util.regex;Matcher;false;group;;;Argument[-1];ReturnValue;taint" + String out = null; + String in = (String) source(); + Matcher m = pattern.matcher(in); + out = m.group("foo"); + sink(out); // $ hasTaintFlow + } + { + // "java.util.regex;Matcher;false;group;;;Argument[-1];ReturnValue;taint" + String out = null; + String in = (String) source(); + Matcher m = pattern.matcher(in); + out = m.group(); + sink(out); // $ hasTaintFlow + } + { + // "java.util.regex;Matcher;false;group;;;Argument[-1];ReturnValue;taint" + String out = null; + String in = (String) source(); + Matcher m = pattern.matcher(in); + out = m.group(0); + sink(out); // $ hasTaintFlow + } + { + // "java.util.regex;Matcher;false;replaceAll;;;Argument[-1];ReturnValue;taint" + String out = null; + String in = (String) source(); + Matcher m = pattern.matcher(in); + out = m.replaceAll("foo"); + sink(out); // $ hasTaintFlow + } + { + // "java.util.regex;Matcher;false;replaceAll;;;Argument[0];ReturnValue;taint" + String out = null; + String in = (String) source(); + Matcher m = pattern.matcher("foo"); + out = m.replaceAll(in); + sink(out); // $ hasTaintFlow + } + { + // "java.util.regex;Matcher;false;replaceFirst;;;Argument[-1];ReturnValue;taint" + String out = null; + String in = (String) source(); + Matcher m = pattern.matcher(in); + out = m.replaceFirst("foo"); + sink(out); // $ hasTaintFlow + } + { + // "java.util.regex;Matcher;false;replaceFirst;;;Argument[0];ReturnValue;taint" + String out = null; + String in = (String) source(); + Matcher m = pattern.matcher("foo"); + out = m.replaceFirst(in); + sink(out); // $ hasTaintFlow + } + { + // "java.util.regex;Pattern;false;matcher;;;Argument[0];ReturnValue;taint" + Matcher out = null; + CharSequence in = (CharSequence)source(); + out = pattern.matcher(in); + sink(out); // $ hasTaintFlow + } + { + // "java.util.regex;Pattern;false;quote;;;Argument[0];ReturnValue;taint" + String out = null; + String in = (String)source(); + out = Pattern.quote(in); + sink(out); // $ hasTaintFlow + } + { + // "java.util.regex;Pattern;false;split;;;Argument[0];ReturnValue;taint" + String[] out = null; + CharSequence in = (CharSequence)source(); + out = pattern.split(in); + sink(out); // $ hasTaintFlow + } + { + // "java.util.regex;Pattern;false;split;;;Argument[0];ReturnValue;taint" + String[] out = null; + CharSequence in = (CharSequence)source(); + out = pattern.split(in, 0); + sink(out); // $ hasTaintFlow + } + + } + +} \ No newline at end of file From 5b61de67de7fa12350cebf53896e0dd74eadbf8f Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Wed, 9 Feb 2022 14:54:13 +0000 Subject: [PATCH 21/91] Implement style/doc suggestions from code review --- java/ql/lib/semmle/code/java/PrintAst.qll | 2 +- .../code/java/regex/RegexFlowConfigs.qll | 10 +- .../semmle/code/java/regex/RegexTreeView.qll | 42 ++++----- java/ql/lib/semmle/code/java/regex/regex.qll | 91 +++++++++---------- .../Security/CWE/CWE-730/PolynomialReDoS.ql | 4 +- 5 files changed, 73 insertions(+), 76 deletions(-) diff --git a/java/ql/lib/semmle/code/java/PrintAst.qll b/java/ql/lib/semmle/code/java/PrintAst.qll index 4279c0e8e58..ee3a2800584 100644 --- a/java/ql/lib/semmle/code/java/PrintAst.qll +++ b/java/ql/lib/semmle/code/java/PrintAst.qll @@ -286,7 +286,7 @@ final class AnnotationPartNode extends ExprStmtNode { /** * A node representing a `StringLiteral`. - * It has a child if it is used as a regular expression, which is the root of the regular expression. + * If it is used as a regular expression, then it has a single child, the root of the parsed regular expression. */ final class StringLiteralNode extends ExprStmtNode { StringLiteralNode() { element instanceof StringLiteral } diff --git a/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll b/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll index 9769a7ce8f7..f86d787a96b 100644 --- a/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll +++ b/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll @@ -20,7 +20,7 @@ private class RegexCompileFlowConf extends DataFlow2::Configuration { * Holds if `s` is used as a regex, with the mode `mode` (if known). * If regex mode is not known, `mode` will be `"None"`. */ -predicate used_as_regex(Expr s, string mode) { +predicate usedAsRegex(StringLiteral s, string mode) { any(RegexCompileFlowConf c).hasFlow(DataFlow2::exprNode(s), _) and mode = "None" // TODO: proper mode detection } @@ -43,10 +43,10 @@ abstract class RegexMatchMethodAccess extends MethodAccess { stringArg in [-1 .. m.getNumberOfParameters() - 1] } - /** Gets the argument of this call that the regex to be matched against flows into */ + /** Gets the argument of this call that the regex to be matched against flows into. */ Expr getRegexArg() { result = argOf(this, regexArg) } - /** Gets the argument of this call that the */ + /** Gets the argument of this call that the string being matched flows into. */ Expr getStringArg() { result = argOf(this, stringArg) } } @@ -178,9 +178,9 @@ private class RegexMatchFlowConf extends DataFlow2::Configuration { } /** - * Holds if the string literal `regex` is matched against the expression `str`. + * Holds if the string literal `regex` is a regular expression that is matched against the expression `str`. */ -predicate regex_match(StringLiteral regex, Expr str) { +predicate regexMatchedAgainst(StringLiteral regex, Expr str) { exists( DataFlow::Node src, DataFlow::Node sink, RegexMatchMethodAccess ma, RegexMatchFlowConf conf | diff --git a/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll b/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll index 0bf2437c634..9c4ee401135 100644 --- a/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll +++ b/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll @@ -5,18 +5,18 @@ private import semmle.code.java.regex.regex /** * An element containing a regular expression term, that is, either - * a string literal (parsed as a regular expression) - * or another regular expression term. + * a string literal (parsed as a regular expression; the root of the parse tree) + * or another regular expression term (a decendent of the root). * - * For sequences and alternations, we require at least one child. + * For sequences and alternations, we require at least two children. * Otherwise, we wish to represent the term differently. * This avoids multiple representations of the same term. */ -newtype TRegExpParent = +private newtype TRegExpParent = /** A string literal used as a regular expression */ TRegExpLiteral(Regex re) or /** A quantified term */ - TRegExpQuantifier(Regex re, int start, int end) { re.qualifiedItem(start, end, _, _) } or + TRegExpQuantifier(Regex re, int start, int end) { re.quantifiedItem(start, end, _, _) } or /** A sequence term */ TRegExpSequence(Regex re, int start, int end) { re.sequence(start, end) and @@ -47,8 +47,8 @@ newtype TRegExpParent = /** * An element containing a regular expression term, that is, either - * a string literal (parsed as a regular expression) - * or another regular expression term. + * a string literal (parsed as a regular expression; the root of the parse tree) + * or another regular expression term (a decendent of the root). */ class RegExpParent extends TRegExpParent { /** Gets a textual representation of this element. */ @@ -92,6 +92,7 @@ class RegExpLiteral extends TRegExpLiteral, RegExpParent { /** * A regular expression term, that is, a syntactic part of a regular expression. + * These are the tree nodes that form the parse tree of a regular expression literal. */ class RegExpTerm extends RegExpParent { Regex re; @@ -187,6 +188,8 @@ class RegExpTerm extends RegExpParent { predicate hasLocationInfo( string filepath, int startline, int startcolumn, int endline, int endcolumn ) { + // This currently gives incorrect results for string literals including backslashes. TODO: fix that. + // There are also more complex cases where it fails. Handling all of them would be difficult for not much gain. exists(int re_start, int re_end | re.getLocation().hasLocationInfo(filepath, startline, re_start, endline, re_end) and startcolumn = re_start + start + 1 and @@ -245,7 +248,7 @@ class RegExpQuantifier extends RegExpTerm, TRegExpQuantifier { RegExpQuantifier() { this = TRegExpQuantifier(re, start, end) and - re.qualifiedPart(start, part_end, end, maybe_empty, may_repeat_forever) + re.quantifiedPart(start, part_end, end, maybe_empty, may_repeat_forever) } override RegExpTerm getChild(int i) { @@ -255,11 +258,11 @@ class RegExpQuantifier extends RegExpTerm, TRegExpQuantifier { result.getEnd() = part_end } - /** Hols if this term may match an unlimited number of times. */ + /** Holds if this term may match an unlimited number of times. */ predicate mayRepeatForever() { may_repeat_forever = true } - /** Gets the qualifier for this term. That is e.g "?" for "a?". */ - string getQualifier() { result = re.getText().substring(part_end, end) } + /** Gets the quantifier for this term. That is e.g "?" for "a?". */ + string getquantifier() { result = re.getText().substring(part_end, end) } override string getPrimaryQLClass() { result = "RegExpQuantifier" } } @@ -281,7 +284,7 @@ class InfiniteRepetitionQuantifier extends RegExpQuantifier { * ``` */ class RegExpStar extends InfiniteRepetitionQuantifier { - RegExpStar() { this.getQualifier().charAt(0) = "*" } + RegExpStar() { this.getquantifier().charAt(0) = "*" } override string getPrimaryQLClass() { result = "RegExpStar" } } @@ -296,7 +299,7 @@ class RegExpStar extends InfiniteRepetitionQuantifier { * ``` */ class RegExpPlus extends InfiniteRepetitionQuantifier { - RegExpPlus() { this.getQualifier().charAt(0) = "+" } + RegExpPlus() { this.getquantifier().charAt(0) = "+" } override string getPrimaryQLClass() { result = "RegExpPlus" } } @@ -311,7 +314,7 @@ class RegExpPlus extends InfiniteRepetitionQuantifier { * ``` */ class RegExpOpt extends RegExpQuantifier { - RegExpOpt() { this.getQualifier().charAt(0) = "?" } + RegExpOpt() { this.getquantifier().charAt(0) = "?" } override string getPrimaryQLClass() { result = "RegExpOpt" } } @@ -333,10 +336,10 @@ class RegExpRange extends RegExpQuantifier { RegExpRange() { re.multiples(part_end, end, lower, upper) } - /** Gets the string defining the upper bound of this range, if any. */ + /** Gets the string defining the upper bound of this range, which is empty when no such bound exists. */ string getUpper() { result = upper } - /** Gets the string defining the lower bound of this range, if any. */ + /** Gets the string defining the lower bound of this range, which is empty when no such bound exists. */ string getLower() { result = lower } /** @@ -578,9 +581,6 @@ class RegExpCharacterClass extends RegExpTerm, TRegExpCharacterClass { /** Holds if this character class is inverted, matching the opposite of its content. */ predicate isInverted() { re.getChar(start + 1) = "^" } - /** Gets the `i`th char inside this charater class. */ - string getCharThing(int i) { result = re.getChar(i + start) } - /** Holds if this character class can match anything. */ predicate isUniversalClass() { // [^] @@ -724,9 +724,9 @@ class RegExpConstant extends RegExpTerm { RegExpConstant() { (this = TRegExpNormalChar(re, start, end) or this = TRegExpQuote(re, start, end)) and not this instanceof RegExpCharacterClassEscape and - // exclude chars in qualifiers + // exclude chars in quantifiers // TODO: push this into regex library - not exists(int qstart, int qend | re.qualifiedPart(_, qstart, qend, _, _) | + not exists(int qstart, int qend | re.quantifiedPart(_, qstart, qend, _, _) | qstart <= start and end <= qend ) and (value = this.(RegExpNormalChar).getValue() or value = this.(RegExpQuote).getValue()) diff --git a/java/ql/lib/semmle/code/java/regex/regex.qll b/java/ql/lib/semmle/code/java/regex/regex.qll index 35ca6fa9998..a2a7f22c07e 100644 --- a/java/ql/lib/semmle/code/java/regex/regex.qll +++ b/java/ql/lib/semmle/code/java/regex/regex.qll @@ -1,20 +1,15 @@ import java private import RegexFlowConfigs +// In all ranges handled by this library, `start` is inclusive and `end` is exclusive. /** - * A string literal that is used as a regular exprssion. + * A string literal that is used as a regular expression. */ -abstract class RegexString extends Expr { - RegexString() { this instanceof StringLiteral } - - /** Holds if a character set starts between `start` and `end`. */ +abstract class RegexString extends StringLiteral { + /** Holds if a character set starts between `start` and `end`, including any negation character (`^`). */ private predicate char_set_start0(int start, int end) { this.nonEscapedCharAt(start) = "[" and - ( - this.getChar(start + 1) = "^" and end = start + 2 - or - not this.getChar(start + 1) = "^" and end = start + 1 - ) + (if this.getChar(start + 1) = "^" then end = start + 2 else end = start + 1) } /** Holds if the character at `pos` marks the end of a character class. */ @@ -25,7 +20,7 @@ abstract class RegexString extends Expr { } /** - * Gets the nesting depth of charcter classes at position `pos` + * Gets the nesting depth of character classes at position `pos` */ private int char_set_depth(int pos) { exists(this.getChar(pos)) and @@ -51,7 +46,7 @@ abstract class RegexString extends Expr { } /** - * Whether there is a top-level character class, between start (inclusive) and end (exclusive) + * Holds if there is a top-level character class beginning at `start` (inclusive) and ending at `end` (exclusive) * * For now, nested character classes are approximated by only considering the top-level class for parsing. * This leads to very similar results for ReDoS queries. @@ -355,7 +350,7 @@ abstract class RegexString extends Expr { not c = "[" and not c = ")" and not c = "|" and - not this.qualifier(start, _, _, _) + not this.quantifier(start, _, _, _) ) } @@ -384,7 +379,7 @@ abstract class RegexString extends Expr { not this.inCharSet(start) } - /** Whether the text in the range start,end is a group */ + /** Holds if the text in the range start,end is a group */ predicate group(int start, int end) { this.groupContents(start, end, _, _) or @@ -407,7 +402,7 @@ abstract class RegexString extends Expr { ) } - /** Whether the text in the range start, end is a group and can match the empty string. */ + /** Holds if the text in the range start, end is a group and can match the empty string. */ predicate zeroWidthMatch(int start, int end) { this.emptyGroup(start, end) or @@ -629,7 +624,7 @@ abstract class RegexString extends Expr { ) } - /** Whether the text in the range start,end is a back reference */ + /** Holds if the text in the range start,end is a back reference */ predicate backreference(int start, int end) { this.numbered_backreference(start, end, _) or @@ -655,16 +650,18 @@ abstract class RegexString extends Expr { this.quote(start, end) } - private predicate qualifier(int start, int end, boolean maybe_empty, boolean may_repeat_forever) { - this.short_qualifier(start, end, maybe_empty, may_repeat_forever) and + private predicate quantifier(int start, int end, boolean maybe_empty, boolean may_repeat_forever) { + this.short_quantifier(start, end, maybe_empty, may_repeat_forever) and not this.getChar(end) = ["?", "+"] or - exists(int short_end | this.short_qualifier(start, short_end, maybe_empty, may_repeat_forever) | + exists(int short_end | + this.short_quantifier(start, short_end, maybe_empty, may_repeat_forever) + | if this.getChar(short_end) = ["?", "+"] then end = short_end + 1 else end = short_end ) } - private predicate short_qualifier( + private predicate short_quantifier( int start, int end, boolean maybe_empty, boolean may_repeat_forever ) { ( @@ -708,32 +705,32 @@ abstract class RegexString extends Expr { } /** - * Whether the text in the range start,end is a qualified item, where item is a character, + * Holds if the text in the range start,end is a quantified item, where item is a character, * a character set or a group. */ - predicate qualifiedItem(int start, int end, boolean maybe_empty, boolean may_repeat_forever) { - this.qualifiedPart(start, _, end, maybe_empty, may_repeat_forever) + predicate quantifiedItem(int start, int end, boolean maybe_empty, boolean may_repeat_forever) { + this.quantifiedPart(start, _, end, maybe_empty, may_repeat_forever) } /** - * Holds if a qualified part is found between `start` and `part_end` and the qualifier is + * Holds if a quantified part is found between `start` and `part_end` and the quantifier is * found between `part_end` and `end`. * * `maybe_empty` is true if the part is optional. * `may_repeat_forever` is true if the part may be repeated unboundedly. */ - predicate qualifiedPart( + predicate quantifiedPart( int start, int part_end, int end, boolean maybe_empty, boolean may_repeat_forever ) { this.baseItem(start, part_end) and - this.qualifier(part_end, end, maybe_empty, may_repeat_forever) + this.quantifier(part_end, end, maybe_empty, may_repeat_forever) } /** Holds if the range `start`, `end` contains a character, a quantifier, a character set or a group. */ predicate item(int start, int end) { - this.qualifiedItem(start, end, _, _) + this.quantifiedItem(start, end, _, _) or - this.baseItem(start, end) and not this.qualifier(end, _, _, _) + this.baseItem(start, end) and not this.quantifier(end, _, _, _) } private predicate subsequence(int start, int end) { @@ -751,15 +748,15 @@ abstract class RegexString extends Expr { } /** - * Whether the text in the range start,end is a sequence of 1 or more items, where an item is a character, + * Holds if the text in the range start,end is a sequence of 1 or more items, where an item is a character, * a character set or a group. */ predicate sequence(int start, int end) { - this.sequenceOrQualified(start, end) and - not this.qualifiedItem(start, end, _, _) + this.sequenceOrquantified(start, end) and + not this.quantifiedItem(start, end, _, _) } - private predicate sequenceOrQualified(int start, int end) { + private predicate sequenceOrquantified(int start, int end) { this.subsequence(start, end) and not this.item_start(end) } @@ -779,7 +776,7 @@ abstract class RegexString extends Expr { or this.charSet(_, end) or - this.qualifier(_, end, _, _) + this.quantifier(_, end, _, _) or this.quote(_, end) } @@ -790,7 +787,7 @@ abstract class RegexString extends Expr { } private predicate subalternation(int start, int end, int item_start) { - this.sequenceOrQualified(start, end) and + this.sequenceOrquantified(start, end) and not this.isOptionDivider(start - 1) and item_start = start or @@ -804,14 +801,14 @@ abstract class RegexString extends Expr { this.isOptionDivider(mid) and item_start = mid + 1 | - this.sequenceOrQualified(item_start, end) + this.sequenceOrquantified(item_start, end) or not this.item_start(end) and end = item_start ) } /** - * Whether the text in the range start,end is an alternation + * Holds if the text in the range start,end is an alternation */ predicate alternation(int start, int end) { this.top_level(start, end) and @@ -819,7 +816,7 @@ abstract class RegexString extends Expr { } /** - * Whether the text in the range start,end is an alternation and the text in part_start, part_end is one of the + * Holds if the text in the range start,end is an alternation and the text in part_start, part_end is one of the * options in that alternation. */ predicate alternationOption(int start, int end, int part_start, int part_end) { @@ -833,14 +830,14 @@ abstract class RegexString extends Expr { or exists(int x | this.firstPart(x, end) | this.emptyMatchAtStartGroup(x, start) or - this.qualifiedItem(x, start, true, _) or + this.quantifiedItem(x, start, true, _) or this.specialCharacter(x, start, "^") ) or exists(int y | this.firstPart(start, y) | this.item(start, end) or - this.qualifiedPart(start, end, y, _, _) + this.quantifiedPart(start, end, y, _, _) ) or exists(int x, int y | this.firstPart(x, y) | @@ -857,7 +854,7 @@ abstract class RegexString extends Expr { exists(int y | this.lastPart(start, y) | this.emptyMatchAtEndGroup(end, y) or - this.qualifiedItem(end, y, true, _) + this.quantifiedItem(end, y, true, _) or this.specialCharacter(end, y, "$") or @@ -869,7 +866,7 @@ abstract class RegexString extends Expr { this.item(start, end) ) or - exists(int y | this.lastPart(start, y) | this.qualifiedPart(start, end, y, _, _)) + exists(int y | this.lastPart(start, y) | this.quantifiedPart(start, end, y, _, _)) or exists(int x, int y | this.lastPart(x, y) | this.groupContents(x, y, start, end) @@ -879,14 +876,14 @@ abstract class RegexString extends Expr { } /** - * Whether the item at [start, end) is one of the first items + * Holds if the item at [start, end) is one of the first items * to be matched. */ predicate firstItem(int start, int end) { ( this.character(start, end) or - this.qualifiedItem(start, end, _, _) + this.quantifiedItem(start, end, _, _) or this.charSet(start, end) or @@ -896,14 +893,14 @@ abstract class RegexString extends Expr { } /** - * Whether the item at [start, end) is one of the last items + * Holds if the item at [start, end) is one of the last items * to be matched. */ predicate lastItem(int start, int end) { ( this.character(start, end) or - this.qualifiedItem(start, end, _, _) + this.quantifiedItem(start, end, _, _) or this.charSet(start, end) or @@ -915,7 +912,7 @@ abstract class RegexString extends Expr { /** A string literal used as a regular expression */ class Regex extends RegexString { - Regex() { used_as_regex(this, _) } + Regex() { usedAsRegex(this, _) } /** * Gets a mode (if any) of this regular expression. Can be any of: @@ -929,7 +926,7 @@ class Regex extends RegexString { */ string getAMode() { result != "None" and - used_as_regex(this, result) + usedAsRegex(this, result) or result = this.getModeFromPrefix() } diff --git a/java/ql/src/Security/CWE/CWE-730/PolynomialReDoS.ql b/java/ql/src/Security/CWE/CWE-730/PolynomialReDoS.ql index 40bc4845a7c..1c8f3299f7f 100644 --- a/java/ql/src/Security/CWE/CWE-730/PolynomialReDoS.ql +++ b/java/ql/src/Security/CWE/CWE-730/PolynomialReDoS.ql @@ -22,12 +22,12 @@ import DataFlow::PathGraph class PolynomialRedosSink extends DataFlow::Node { RegExpLiteral reg; - PolynomialRedosSink() { regex_match(reg.getRegex(), this.asExpr()) } + PolynomialRedosSink() { regexMatchedAgainst(reg.getRegex(), this.asExpr()) } RegExpTerm getRegExp() { result = reg } } -class PolynomialRedosConfig extends DataFlow::Configuration { +class PolynomialRedosConfig extends TaintTracking::Configuration { PolynomialRedosConfig() { this = "PolynomialRodisConfig" } override predicate isSource(DataFlow::Node src) { src instanceof RemoteFlowSource } From e954db293a6791a4e0b081ab11c1e471cd569c0c Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Wed, 9 Feb 2022 15:11:01 +0000 Subject: [PATCH 22/91] Convert snake case predicates to camel case --- .../semmle/code/java/regex/RegexTreeView.qll | 6 +- java/ql/lib/semmle/code/java/regex/regex.qll | 190 +++++++++--------- 2 files changed, 98 insertions(+), 98 deletions(-) diff --git a/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll b/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll index 9c4ee401135..114e24f9d49 100644 --- a/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll +++ b/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll @@ -601,8 +601,8 @@ class RegExpCharacterClass extends RegExpTerm, TRegExpCharacterClass { result.getRegex() = re and exists(int itemStart, int itemEnd | result.getStart() = itemStart and - re.char_set_start(start, itemStart) and - re.char_set_child(start, itemStart, itemEnd) and + re.charSetStart(start, itemStart) and + re.charSetChild(start, itemStart, itemEnd) and result.getEnd() = itemEnd ) or @@ -610,7 +610,7 @@ class RegExpCharacterClass extends RegExpTerm, TRegExpCharacterClass { result.getRegex() = re and exists(int itemStart | itemStart = this.getChild(i - 1).getEnd() | result.getStart() = itemStart and - re.char_set_child(start, itemStart, result.getEnd()) + re.charSetChild(start, itemStart, result.getEnd()) ) } diff --git a/java/ql/lib/semmle/code/java/regex/regex.qll b/java/ql/lib/semmle/code/java/regex/regex.qll index a2a7f22c07e..3aed361ec9e 100644 --- a/java/ql/lib/semmle/code/java/regex/regex.qll +++ b/java/ql/lib/semmle/code/java/regex/regex.qll @@ -7,42 +7,42 @@ private import RegexFlowConfigs */ abstract class RegexString extends StringLiteral { /** Holds if a character set starts between `start` and `end`, including any negation character (`^`). */ - private predicate char_set_start0(int start, int end) { + private predicate charSetStart0(int start, int end) { this.nonEscapedCharAt(start) = "[" and (if this.getChar(start + 1) = "^" then end = start + 2 else end = start + 1) } /** Holds if the character at `pos` marks the end of a character class. */ - private predicate char_set_end0(int pos) { + private predicate charSetEnd0(int pos) { this.nonEscapedCharAt(pos) = "]" and /* special case: `[]]` and `[^]]` are valid char classes. */ - not this.char_set_start0(_, pos) + not this.charSetStart0(_, pos) } /** * Gets the nesting depth of character classes at position `pos` */ - private int char_set_depth(int pos) { + private int charSetDepth(int pos) { exists(this.getChar(pos)) and result = max(int j | j = 0 or j = - count(int i | i < pos and this.char_set_start0(i, _)) - - count(int i | i < pos and this.char_set_end0(i)) + count(int i | i < pos and this.charSetStart0(i, _)) - + count(int i | i < pos and this.charSetEnd0(i)) ) } /** Hold if a top-level character set starts between `start` and `end`. */ - predicate char_set_start(int start, int end) { - this.char_set_start0(start, end) and - this.char_set_depth(start) = 0 + predicate charSetStart(int start, int end) { + this.charSetStart0(start, end) and + this.charSetDepth(start) = 0 } /** Holds if a top-level character set ends at `pos`. */ - predicate char_set_end(int pos) { - this.char_set_end0(pos) and - this.char_set_depth(pos) = 1 + predicate charSetEnd(int pos) { + this.charSetEnd0(pos) and + this.charSetDepth(pos) = 1 } /** @@ -53,26 +53,26 @@ abstract class RegexString extends StringLiteral { */ predicate charSet(int start, int end) { exists(int inner_start, int inner_end | - this.char_set_start(start, inner_start) and - not this.char_set_start(_, start) + this.charSetStart(start, inner_start) and + not this.charSetStart(_, start) | end = inner_end + 1 and inner_end > inner_start and - this.char_set_end(inner_end) and - not exists(int mid | this.char_set_end(mid) | mid > inner_start and mid < inner_end) + this.charSetEnd(inner_end) and + not exists(int mid | this.charSetEnd(mid) | mid > inner_start and mid < inner_end) ) } - /** An indexed version of `char_set_token/3` */ - private predicate char_set_token(int charset_start, int index, int token_start, int token_end) { + /** An indexed version of `charSetToken/3` */ + private predicate charSetToken(int charset_start, int index, int token_start, int token_end) { token_start = - rank[index](int start, int end | this.char_set_token(charset_start, start, end) | start) and - this.char_set_token(charset_start, token_start, token_end) + rank[index](int start, int end | this.charSetToken(charset_start, start, end) | start) and + this.charSetToken(charset_start, token_start, token_end) } /** Either a char or a - */ - private predicate char_set_token(int charset_start, int start, int end) { - this.char_set_start(charset_start, start) and + private predicate charSetToken(int charset_start, int start, int end) { + this.charSetStart(charset_start, start) and ( this.escapedCharacter(start, end) or @@ -81,13 +81,13 @@ abstract class RegexString extends StringLiteral { this.quote(start, end) ) or - this.char_set_token(charset_start, _, start) and + this.charSetToken(charset_start, _, start) and ( this.escapedCharacter(start, end) or exists(this.nonEscapedCharAt(start)) and end = start + 1 and - not this.char_set_end(start) + not this.charSetEnd(start) or this.quote(start, end) ) @@ -97,8 +97,8 @@ abstract class RegexString extends StringLiteral { * Holds if the character set starting at `charset_start` contains either * a character or a range found between `start` and `end`. */ - predicate char_set_child(int charset_start, int start, int end) { - this.char_set_token(charset_start, start, end) and + predicate charSetChild(int charset_start, int start, int end) { + this.charSetToken(charset_start, start, end) and not exists(int range_start, int range_end | this.charRange(charset_start, range_start, _, _, range_end) and range_start <= start and @@ -116,8 +116,8 @@ abstract class RegexString extends StringLiteral { predicate charRange(int charset_start, int start, int lower_end, int upper_start, int end) { exists(int index | this.charRangeEnd(charset_start, index) = true and - this.char_set_token(charset_start, index - 2, start, lower_end) and - this.char_set_token(charset_start, index, upper_start, end) + this.charSetToken(charset_start, index - 2, start, lower_end) and + this.charSetToken(charset_start, index, upper_start, end) ) } @@ -129,13 +129,13 @@ abstract class RegexString extends StringLiteral { * the helper for `escapingChar`, for a clean use of this pattern. */ private boolean charRangeEnd(int charset_start, int index) { - this.char_set_token(charset_start, index, _, _) and + this.charSetToken(charset_start, index, _, _) and ( index in [1, 2] and result = false or index > 2 and exists(int connector_start | - this.char_set_token(charset_start, index - 1, connector_start, _) and + this.charSetToken(charset_start, index - 1, connector_start, _) and this.nonEscapedCharAt(connector_start) = "-" and result = this.charRangeEnd(charset_start, index - 2) @@ -144,7 +144,7 @@ abstract class RegexString extends StringLiteral { ) or not exists(int connector_start | - this.char_set_token(charset_start, index - 1, connector_start, _) and + this.charSetToken(charset_start, index - 1, connector_start, _) and this.nonEscapedCharAt(connector_start) = "-" ) and result = false @@ -182,7 +182,7 @@ abstract class RegexString extends StringLiteral { * Holds if the char at `pos` could be the beginning of a quote delimiter, i.e. `\Q` (non-escaped) or `\E` (escaping not checked, as quote sequences turn off escapes). * Result is `true` for `\Q` and `false` for `\E`. */ - private boolean quote_delimiter(int pos) { + private boolean quoteDelimiter(int pos) { result = true and this.escaping(pos) = true and this.getChar(pos + 1) = "Q" @@ -197,9 +197,9 @@ abstract class RegexString extends StringLiteral { * Holds if the char at `pos` is the one-based `index`th occourence of a quote delimiter (`\Q` or `\E`) * Result is `true` for `\Q` and `false` for `\E`. */ - private boolean quote_delimiter(int index, int pos) { - result = this.quote_delimiter(pos) and - pos = rank[index](int p | this.quote_delimiter(p) = [true, false]) + private boolean quoteDelimiter(int index, int pos) { + result = this.quoteDelimiter(pos) and + pos = rank[index](int p | this.quoteDelimiter(p) = [true, false]) } /** Holds if a quoted sequence is found between `start` and `end` */ @@ -208,18 +208,18 @@ abstract class RegexString extends StringLiteral { /** Holds if a quoted sequence is fund between `start` and `end`, with ontent found between `inner_start` and `inner_end`. */ predicate quote(int start, int end, int inner_start, int inner_end) { exists(int index | - this.quote_delimiter(index, start) = true and + this.quoteDelimiter(index, start) = true and ( index = 1 or - this.quote_delimiter(index - 1, _) = false + this.quoteDelimiter(index - 1, _) = false ) and inner_start = start + 2 and inner_end = end - 2 and inner_end > inner_start and - this.quote_delimiter(inner_end) = false and + this.quoteDelimiter(inner_end) = false and not exists(int mid | - this.quote_delimiter(mid) = false and mid in [inner_start .. inner_end - 1] + this.quoteDelimiter(mid) = false and mid in [inner_start .. inner_end - 1] ) ) } @@ -255,7 +255,7 @@ abstract class RegexString extends StringLiteral { predicate failedToParse(int i) { exists(this.getChar(i)) and not exists(int start, int end | - this.top_level(start, end) and + this.topLevel(start, end) and start <= i and end > i ) @@ -336,7 +336,7 @@ abstract class RegexString extends StringLiteral { exists(string c | c = this.getChar(start) | exists(int x, int y, int z | this.charSet(x, z) and - this.char_set_start(x, y) + this.charSetStart(x, y) | start = y or @@ -362,7 +362,7 @@ abstract class RegexString extends StringLiteral { or this.escapedCharacter(start, end) ) and - not exists(int x, int y | this.group_start(x, y) and x <= start and y >= end) and + not exists(int x, int y | this.groupStart(x, y) and x <= start and y >= end) and not exists(int x, int y | this.backreference(x, y) and x <= start and y >= end) } @@ -390,14 +390,14 @@ abstract class RegexString extends StringLiteral { int getGroupNumber(int start, int end) { this.group(start, end) and result = - count(int i | this.group(i, _) and i < start and not this.non_capturing_group_start(i, _)) + 1 + count(int i | this.group(i, _) and i < start and not this.nonCapturingGroupStart(i, _)) + 1 } /** Gets the name, if it has one, of the group in start,end */ string getGroupName(int start, int end) { this.group(start, end) and exists(int name_end | - this.named_group_start(start, name_end) and + this.namedGroupStart(start, name_end) and result = this.getText().substring(start + 3, name_end - 1) ) } @@ -416,7 +416,7 @@ abstract class RegexString extends StringLiteral { /** Holds if an empty group is found between `start` and `end`. */ predicate emptyGroup(int start, int end) { exists(int endm1 | end = endm1 + 1 | - this.group_start(start, endm1) and + this.groupStart(start, endm1) and this.isGroupEnd(endm1) ) } @@ -439,9 +439,9 @@ abstract class RegexString extends StringLiteral { private predicate negativeAssertionGroup(int start, int end) { exists(int in_start | - this.negative_lookahead_assertion_start(start, in_start) + this.negativeLookaheadAssertionStart(start, in_start) or - this.negative_lookbehind_assertion_start(start, in_start) + this.negativeLookbehindAssertionStart(start, in_start) | this.groupContents(start, end, in_start, _) ) @@ -449,66 +449,66 @@ abstract class RegexString extends StringLiteral { /** Holds if a negative lookahead is found between `start` and `end` */ predicate negativeLookaheadAssertionGroup(int start, int end) { - exists(int in_start | this.negative_lookahead_assertion_start(start, in_start) | + exists(int in_start | this.negativeLookaheadAssertionStart(start, in_start) | this.groupContents(start, end, in_start, _) ) } /** Holds if a negative lookbehind is found between `start` and `end` */ predicate negativeLookbehindAssertionGroup(int start, int end) { - exists(int in_start | this.negative_lookbehind_assertion_start(start, in_start) | + exists(int in_start | this.negativeLookbehindAssertionStart(start, in_start) | this.groupContents(start, end, in_start, _) ) } /** Holds if a positive lookahead is found between `start` and `end` */ predicate positiveLookaheadAssertionGroup(int start, int end) { - exists(int in_start | this.lookahead_assertion_start(start, in_start) | + exists(int in_start | this.lookaheadAssertionStart(start, in_start) | this.groupContents(start, end, in_start, _) ) } /** Holds if a positive lookbehind is found between `start` and `end` */ predicate positiveLookbehindAssertionGroup(int start, int end) { - exists(int in_start | this.lookbehind_assertion_start(start, in_start) | + exists(int in_start | this.lookbehindAssertionStart(start, in_start) | this.groupContents(start, end, in_start, _) ) } - private predicate group_start(int start, int end) { - this.non_capturing_group_start(start, end) + private predicate groupStart(int start, int end) { + this.nonCapturingGroupStart(start, end) or - this.flag_group_start(start, end, _) + this.flagGroupStart(start, end, _) or - this.named_group_start(start, end) + this.namedGroupStart(start, end) or - this.lookahead_assertion_start(start, end) + this.lookaheadAssertionStart(start, end) or - this.negative_lookahead_assertion_start(start, end) + this.negativeLookaheadAssertionStart(start, end) or - this.lookbehind_assertion_start(start, end) + this.lookbehindAssertionStart(start, end) or - this.negative_lookbehind_assertion_start(start, end) + this.negativeLookbehindAssertionStart(start, end) or - this.atomic_group_start(start, end) + this.atomicGroupStart(start, end) or - this.simple_group_start(start, end) + this.simpleGroupStart(start, end) } - private predicate non_capturing_group_start(int start, int end) { + private predicate nonCapturingGroupStart(int start, int end) { this.isGroupStart(start) and this.getChar(start + 1) = "?" and this.getChar(start + 2) = ":" and end = start + 3 } - private predicate simple_group_start(int start, int end) { + private predicate simpleGroupStart(int start, int end) { this.isGroupStart(start) and this.getChar(start + 1) != "?" and end = start + 1 } - private predicate named_group_start(int start, int end) { + private predicate namedGroupStart(int start, int end) { this.isGroupStart(start) and this.getChar(start + 1) = "?" and this.getChar(start + 2) = "<" and @@ -520,7 +520,7 @@ abstract class RegexString extends StringLiteral { ) } - private predicate flag_group_start(int start, int end, string c) { + private predicate flagGroupStart(int start, int end, string c) { this.isGroupStart(start) and this.getChar(start + 1) = "?" and end = start + 3 and @@ -533,7 +533,7 @@ abstract class RegexString extends StringLiteral { * it is defined by a prefix. */ string getModeFromPrefix() { - exists(string c | this.flag_group_start(_, _, c) | + exists(string c | this.flagGroupStart(_, _, c) | c = "i" and result = "IGNORECASE" or c = "m" and result = "MULTILINE" @@ -548,21 +548,21 @@ abstract class RegexString extends StringLiteral { ) } - private predicate lookahead_assertion_start(int start, int end) { + private predicate lookaheadAssertionStart(int start, int end) { this.isGroupStart(start) and this.getChar(start + 1) = "?" and this.getChar(start + 2) = "=" and end = start + 3 } - private predicate negative_lookahead_assertion_start(int start, int end) { + private predicate negativeLookaheadAssertionStart(int start, int end) { this.isGroupStart(start) and this.getChar(start + 1) = "?" and this.getChar(start + 2) = "!" and end = start + 3 } - private predicate lookbehind_assertion_start(int start, int end) { + private predicate lookbehindAssertionStart(int start, int end) { this.isGroupStart(start) and this.getChar(start + 1) = "?" and this.getChar(start + 2) = "<" and @@ -570,7 +570,7 @@ abstract class RegexString extends StringLiteral { end = start + 4 } - private predicate negative_lookbehind_assertion_start(int start, int end) { + private predicate negativeLookbehindAssertionStart(int start, int end) { this.isGroupStart(start) and this.getChar(start + 1) = "?" and this.getChar(start + 2) = "<" and @@ -578,7 +578,7 @@ abstract class RegexString extends StringLiteral { end = start + 4 } - private predicate atomic_group_start(int start, int end) { + private predicate atomicGroupStart(int start, int end) { this.isGroupStart(start) and this.getChar(start + 1) = "?" and this.getChar(start + 2) = ">" and @@ -586,13 +586,13 @@ abstract class RegexString extends StringLiteral { } predicate groupContents(int start, int end, int in_start, int in_end) { - this.group_start(start, in_start) and + this.groupStart(start, in_start) and end = in_end + 1 and - this.top_level(in_start, in_end) and + this.topLevel(in_start, in_end) and this.isGroupEnd(in_end) } - private predicate named_backreference(int start, int end, string name) { + private predicate namedBackreference(int start, int end, string name) { this.escapingChar(start) and this.getChar(start + 1) = "k" and this.getChar(start + 2) = "<" and @@ -600,7 +600,7 @@ abstract class RegexString extends StringLiteral { name = this.getText().substring(start + 3, end - 2) } - private predicate numbered_backreference(int start, int end, int value) { + private predicate numberedBackreference(int start, int end, int value) { this.escapingChar(start) and // starting with 0 makes it an octal escape not this.getChar(start + 1) = "0" and @@ -626,16 +626,16 @@ abstract class RegexString extends StringLiteral { /** Holds if the text in the range start,end is a back reference */ predicate backreference(int start, int end) { - this.numbered_backreference(start, end, _) + this.numberedBackreference(start, end, _) or - this.named_backreference(start, end, _) + this.namedBackreference(start, end, _) } /** Gets the number of the back reference in start,end */ - int getBackrefNumber(int start, int end) { this.numbered_backreference(start, end, result) } + int getBackrefNumber(int start, int end) { this.numberedBackreference(start, end, result) } /** Gets the name, if it has one, of the back reference in start,end */ - string getBackrefName(int start, int end) { this.named_backreference(start, end, result) } + string getBackrefName(int start, int end) { this.namedBackreference(start, end, result) } private predicate baseItem(int start, int end) { this.character(start, end) and @@ -651,17 +651,17 @@ abstract class RegexString extends StringLiteral { } private predicate quantifier(int start, int end, boolean maybe_empty, boolean may_repeat_forever) { - this.short_quantifier(start, end, maybe_empty, may_repeat_forever) and + this.shortQuantifier(start, end, maybe_empty, may_repeat_forever) and not this.getChar(end) = ["?", "+"] or exists(int short_end | - this.short_quantifier(start, short_end, maybe_empty, may_repeat_forever) + this.shortQuantifier(start, short_end, maybe_empty, may_repeat_forever) | if this.getChar(short_end) = ["?", "+"] then end = short_end + 1 else end = short_end ) } - private predicate short_quantifier( + private predicate shortQuantifier( int start, int end, boolean maybe_empty, boolean may_repeat_forever ) { ( @@ -736,7 +736,7 @@ abstract class RegexString extends StringLiteral { private predicate subsequence(int start, int end) { ( start = 0 or - this.group_start(_, start) or + this.groupStart(_, start) or this.isOptionDivider(start - 1) ) and this.item(start, end) @@ -758,10 +758,10 @@ abstract class RegexString extends StringLiteral { private predicate sequenceOrquantified(int start, int end) { this.subsequence(start, end) and - not this.item_start(end) + not this.itemStart(end) } - private predicate item_start(int start) { + private predicate itemStart(int start) { this.character(start, _) or this.isGroupStart(start) or this.charSet(start, _) or @@ -769,7 +769,7 @@ abstract class RegexString extends StringLiteral { this.quote(start, _) } - private predicate item_end(int end) { + private predicate itemEnd(int end) { this.character(_, end) or exists(int endm1 | this.isGroupEnd(endm1) and end = endm1 + 1) @@ -781,29 +781,29 @@ abstract class RegexString extends StringLiteral { this.quote(_, end) } - private predicate top_level(int start, int end) { + private predicate topLevel(int start, int end) { this.subalternation(start, end, _) and not this.isOptionDivider(end) } - private predicate subalternation(int start, int end, int item_start) { + private predicate subalternation(int start, int end, int itemStart) { this.sequenceOrquantified(start, end) and not this.isOptionDivider(start - 1) and - item_start = start + itemStart = start or start = end and - not this.item_end(start) and + not this.itemEnd(start) and this.isOptionDivider(end) and - item_start = start + itemStart = start or exists(int mid | this.subalternation(start, mid, _) and this.isOptionDivider(mid) and - item_start = mid + 1 + itemStart = mid + 1 | - this.sequenceOrquantified(item_start, end) + this.sequenceOrquantified(itemStart, end) or - not this.item_start(end) and end = item_start + not this.itemStart(end) and end = itemStart ) } @@ -811,7 +811,7 @@ abstract class RegexString extends StringLiteral { * Holds if the text in the range start,end is an alternation */ predicate alternation(int start, int end) { - this.top_level(start, end) and + this.topLevel(start, end) and exists(int less | this.subalternation(start, less, _) and less < end) } From aa1337db86c87571e4b1428aee5e54defe0f53cd Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Thu, 10 Feb 2022 11:16:37 +0000 Subject: [PATCH 23/91] Apply style suggestions from code review --- .../semmle/code/java/regex/RegexTreeView.qll | 5 +-- java/ql/lib/semmle/code/java/regex/regex.qll | 43 ++++++++----------- .../security/performance/RegExpTreeView.qll | 9 ++-- 3 files changed, 24 insertions(+), 33 deletions(-) diff --git a/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll b/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll index 114e24f9d49..b55548d2095 100644 --- a/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll +++ b/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll @@ -527,8 +527,7 @@ class RegExpEscape extends RegExpNormalChar { * Gets the hex number for the `hex` char. */ private int toHex(string hex) { - hex = [0 .. 9].toString() and - result = hex.toInt() + result = [0 .. 9] and hex = result.toString() or result = 10 and hex = ["a", "A"] or @@ -545,7 +544,7 @@ private int toHex(string hex) { /** * A character class escape in a regular expression. - * That is, an escaped charachter that denotes multiple characters. + * That is, an escaped character that denotes multiple characters. * * Examples: * diff --git a/java/ql/lib/semmle/code/java/regex/regex.qll b/java/ql/lib/semmle/code/java/regex/regex.qll index 3aed361ec9e..9116a883269 100644 --- a/java/ql/lib/semmle/code/java/regex/regex.qll +++ b/java/ql/lib/semmle/code/java/regex/regex.qll @@ -52,14 +52,10 @@ abstract class RegexString extends StringLiteral { * This leads to very similar results for ReDoS queries. */ predicate charSet(int start, int end) { - exists(int inner_start, int inner_end | - this.charSetStart(start, inner_start) and - not this.charSetStart(_, start) - | + exists(int inner_start, int inner_end | this.charSetStart(start, inner_start) | end = inner_end + 1 and - inner_end > inner_start and - this.charSetEnd(inner_end) and - not exists(int mid | this.charSetEnd(mid) | mid > inner_start and mid < inner_end) + inner_end = + min(int end_delimiter | this.charSetEnd(end_delimiter) and end_delimiter > inner_start) ) } @@ -159,7 +155,7 @@ abstract class RegexString extends StringLiteral { /** * Helper predicate for `escapingChar`. - * In order to avoid negative recusrion, we return a boolean. + * In order to avoid negative recursion, we return a boolean. * This way, we can refer to `escaping(pos - 1).booleanNot()` * rather than to a negated version of `escaping(pos)`. * Does not take into account escape characters inside quote sequences. @@ -199,13 +195,13 @@ abstract class RegexString extends StringLiteral { */ private boolean quoteDelimiter(int index, int pos) { result = this.quoteDelimiter(pos) and - pos = rank[index](int p | this.quoteDelimiter(p) = [true, false]) + pos = rank[index](int p | exists(this.quoteDelimiter(p))) } /** Holds if a quoted sequence is found between `start` and `end` */ predicate quote(int start, int end) { this.quote(start, end, _, _) } - /** Holds if a quoted sequence is fund between `start` and `end`, with ontent found between `inner_start` and `inner_end`. */ + /** Holds if a quoted sequence is found between `start` and `end`, with ontent found between `inner_start` and `inner_end`. */ predicate quote(int start, int end, int inner_start, int inner_end) { exists(int index | this.quoteDelimiter(index, start) = true and @@ -216,11 +212,10 @@ abstract class RegexString extends StringLiteral { ) and inner_start = start + 2 and inner_end = end - 2 and - inner_end > inner_start and - this.quoteDelimiter(inner_end) = false and - not exists(int mid | - this.quoteDelimiter(mid) = false and mid in [inner_start .. inner_end - 1] - ) + inner_end = + min(int end_delimiter | + this.quoteDelimiter(end_delimiter) = false and end_delimiter > inner_start + ) ) } @@ -266,9 +261,7 @@ abstract class RegexString extends StringLiteral { this.escapingChar(start) and this.getChar(start + 1) = ["N", "p", "P", "x"] and this.getChar(start + 2) = "{" and - this.getChar(end - 1) = "}" and - end > start and - not exists(int i | start + 2 < i and i < end - 1 | this.getChar(i) = "}") + end = min(int i | start + 2 < i and this.getChar(i - 1) = "}") } /** @@ -301,7 +294,7 @@ abstract class RegexString extends StringLiteral { or this.escapedBraces(start, end) or - // Boundry matchers \b, \b{g} + // Boundary matchers \b, \b{g} this.getChar(start + 1) = "b" and ( if this.getText().substring(start + 2, start + 5) = "{g}" @@ -654,9 +647,7 @@ abstract class RegexString extends StringLiteral { this.shortQuantifier(start, end, maybe_empty, may_repeat_forever) and not this.getChar(end) = ["?", "+"] or - exists(int short_end | - this.shortQuantifier(start, short_end, maybe_empty, may_repeat_forever) - | + exists(int short_end | this.shortQuantifier(start, short_end, maybe_empty, may_repeat_forever) | if this.getChar(short_end) = ["?", "+"] then end = short_end + 1 else end = short_end ) } @@ -752,11 +743,11 @@ abstract class RegexString extends StringLiteral { * a character set or a group. */ predicate sequence(int start, int end) { - this.sequenceOrquantified(start, end) and + this.sequenceOrQuantified(start, end) and not this.quantifiedItem(start, end, _, _) } - private predicate sequenceOrquantified(int start, int end) { + private predicate sequenceOrQuantified(int start, int end) { this.subsequence(start, end) and not this.itemStart(end) } @@ -787,7 +778,7 @@ abstract class RegexString extends StringLiteral { } private predicate subalternation(int start, int end, int itemStart) { - this.sequenceOrquantified(start, end) and + this.sequenceOrQuantified(start, end) and not this.isOptionDivider(start - 1) and itemStart = start or @@ -801,7 +792,7 @@ abstract class RegexString extends StringLiteral { this.isOptionDivider(mid) and itemStart = mid + 1 | - this.sequenceOrquantified(itemStart, end) + this.sequenceOrQuantified(itemStart, end) or not this.itemStart(end) and end = itemStart ) diff --git a/java/ql/lib/semmle/code/java/security/performance/RegExpTreeView.qll b/java/ql/lib/semmle/code/java/security/performance/RegExpTreeView.qll index 97c7fd5951e..ff3443acbca 100644 --- a/java/ql/lib/semmle/code/java/security/performance/RegExpTreeView.qll +++ b/java/ql/lib/semmle/code/java/security/performance/RegExpTreeView.qll @@ -1,30 +1,31 @@ /** * This module should provide a class hierarchy corresponding to a parse tree of regular expressions. + * This is the interface to the shared ReDoS library. */ import java import semmle.code.java.regex.RegexTreeView /** - * Holds if `term` is an ecape class representing e.g. `\d`. + * Holds if `term` is an escape class representing e.g. `\d`. * `clazz` is which character class it represents, e.g. "d" for `\d`. */ predicate isEscapeClass(RegExpTerm term, string clazz) { - exists(RegExpCharacterClassEscape escape | term = escape | escape.getValue() = clazz) + term.(RegExpCharacterClassEscape).getValue() = clazz } /** * Holds if the regular expression should not be considered. * * We make the pragmatic performance optimization to ignore regular expressions in files - * that does not belong to the project code (such as installed dependencies). + * that do not belong to the project code (such as installed dependencies). */ predicate isExcluded(RegExpParent parent) { not exists(parent.getRegex().getLocation().getFile().getRelativePath()) or // Regexes with many occurrences of ".*" may cause the polynomial ReDoS computation to explode, so // we explicitly exclude these. - count(int i | exists(parent.getRegex().getText().regexpFind("\\.\\*", i, _)) | i) > 10 + strictcount(int i | exists(parent.getRegex().getText().regexpFind("\\.\\*", i, _)) | i) > 10 } /** From 9e88c67c19f61214276908e847a9fae0c4e2a3e8 Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Thu, 10 Feb 2022 12:03:03 +0000 Subject: [PATCH 24/91] Add more test cases; make some fixes --- .../semmle/code/java/regex/RegexTreeView.qll | 3 - java/ql/lib/semmle/code/java/regex/regex.qll | 9 ++- .../regex/RegexParseTests.expected | 64 +++++++++++++++++++ java/ql/test/library-tests/regex/Test.java | 5 +- 4 files changed, 74 insertions(+), 7 deletions(-) diff --git a/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll b/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll index b55548d2095..fd9b93a142f 100644 --- a/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll +++ b/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll @@ -725,9 +725,6 @@ class RegExpConstant extends RegExpTerm { not this instanceof RegExpCharacterClassEscape and // exclude chars in quantifiers // TODO: push this into regex library - not exists(int qstart, int qend | re.quantifiedPart(_, qstart, qend, _, _) | - qstart <= start and end <= qend - ) and (value = this.(RegExpNormalChar).getValue() or value = this.(RegExpQuote).getValue()) } diff --git a/java/ql/lib/semmle/code/java/regex/regex.qll b/java/ql/lib/semmle/code/java/regex/regex.qll index 9116a883269..6646aa6f152 100644 --- a/java/ql/lib/semmle/code/java/regex/regex.qll +++ b/java/ql/lib/semmle/code/java/regex/regex.qll @@ -174,7 +174,7 @@ abstract class RegexString extends StringLiteral { } /** - * Helper predicate for `quoteSequence`. + * Helper predicate for `quote`. * Holds if the char at `pos` could be the beginning of a quote delimiter, i.e. `\Q` (non-escaped) or `\E` (escaping not checked, as quote sequences turn off escapes). * Result is `true` for `\Q` and `false` for `\E`. */ @@ -189,7 +189,7 @@ abstract class RegexString extends StringLiteral { } /** - * Helper predicate for `quoteSequence`. + * Helper predicate for `quote`. * Holds if the char at `pos` is the one-based `index`th occourence of a quote delimiter (`\Q` or `\E`) * Result is `true` for `\Q` and `false` for `\E`. */ @@ -343,7 +343,10 @@ abstract class RegexString extends StringLiteral { not c = "[" and not c = ")" and not c = "|" and - not this.quantifier(start, _, _, _) + not c = "{" and + not exists(int qstart, int qend | this.quantifier(qstart, qend, _, _) | + qstart <= start and start < qend + ) ) } diff --git a/java/ql/test/library-tests/regex/RegexParseTests.expected b/java/ql/test/library-tests/regex/RegexParseTests.expected index 5a9b632d7c9..e997975be95 100644 --- a/java/ql/test/library-tests/regex/RegexParseTests.expected +++ b/java/ql/test/library-tests/regex/RegexParseTests.expected @@ -66,3 +66,67 @@ parseFailures | Test.java:15:35:15:35 | 2 | [RegExpConstant,RegExpNormalChar] | | Test.java:15:36:15:36 | 3 | [RegExpConstant,RegExpNormalChar] | | Test.java:15:37:15:44 | \\k | [RegExpBackRef] | +| Test.java:16:10:16:10 | a | [RegExpConstant,RegExpNormalChar] | +| Test.java:16:10:16:11 | a+ | [RegExpPlus] | +| Test.java:16:10:16:108 | a+b*c?d{2}e{3,4}f{,5}g{6,}h+?i*?j??k{7}?l{8,9}?m{,10}?n{11,}?o++p*+q?+r{12}+s{13,14}+t{,15}+u{16,}+ | [RegExpSequence] | +| Test.java:16:12:16:12 | b | [RegExpConstant,RegExpNormalChar] | +| Test.java:16:12:16:13 | b* | [RegExpStar] | +| Test.java:16:14:16:14 | c | [RegExpConstant,RegExpNormalChar] | +| Test.java:16:14:16:15 | c? | [RegExpOpt] | +| Test.java:16:16:16:16 | d | [RegExpConstant,RegExpNormalChar] | +| Test.java:16:16:16:19 | d{2} | [RegExpRange] | +| Test.java:16:20:16:20 | e | [RegExpConstant,RegExpNormalChar] | +| Test.java:16:20:16:25 | e{3,4} | [RegExpRange] | +| Test.java:16:26:16:26 | f | [RegExpConstant,RegExpNormalChar] | +| Test.java:16:26:16:30 | f{,5} | [RegExpRange] | +| Test.java:16:31:16:31 | g | [RegExpConstant,RegExpNormalChar] | +| Test.java:16:31:16:35 | g{6,} | [RegExpRange] | +| Test.java:16:36:16:36 | h | [RegExpConstant,RegExpNormalChar] | +| Test.java:16:36:16:38 | h+? | [RegExpPlus] | +| Test.java:16:39:16:39 | i | [RegExpConstant,RegExpNormalChar] | +| Test.java:16:39:16:41 | i*? | [RegExpStar] | +| Test.java:16:42:16:42 | j | [RegExpConstant,RegExpNormalChar] | +| Test.java:16:42:16:44 | j?? | [RegExpOpt] | +| Test.java:16:45:16:45 | k | [RegExpConstant,RegExpNormalChar] | +| Test.java:16:45:16:49 | k{7}? | [RegExpQuantifier] | +| Test.java:16:50:16:50 | l | [RegExpConstant,RegExpNormalChar] | +| Test.java:16:50:16:56 | l{8,9}? | [RegExpQuantifier] | +| Test.java:16:57:16:57 | m | [RegExpConstant,RegExpNormalChar] | +| Test.java:16:57:16:63 | m{,10}? | [RegExpQuantifier] | +| Test.java:16:64:16:64 | n | [RegExpConstant,RegExpNormalChar] | +| Test.java:16:64:16:70 | n{11,}? | [RegExpQuantifier] | +| Test.java:16:71:16:71 | o | [RegExpConstant,RegExpNormalChar] | +| Test.java:16:71:16:73 | o++ | [RegExpPlus] | +| Test.java:16:74:16:74 | p | [RegExpConstant,RegExpNormalChar] | +| Test.java:16:74:16:76 | p*+ | [RegExpStar] | +| Test.java:16:77:16:77 | q | [RegExpConstant,RegExpNormalChar] | +| Test.java:16:77:16:79 | q?+ | [RegExpOpt] | +| Test.java:16:80:16:80 | r | [RegExpConstant,RegExpNormalChar] | +| Test.java:16:80:16:85 | r{12}+ | [RegExpQuantifier] | +| Test.java:16:86:16:86 | s | [RegExpConstant,RegExpNormalChar] | +| Test.java:16:86:16:94 | s{13,14}+ | [RegExpQuantifier] | +| Test.java:16:95:16:95 | t | [RegExpConstant,RegExpNormalChar] | +| Test.java:16:95:16:101 | t{,15}+ | [RegExpQuantifier] | +| Test.java:16:102:16:102 | u | [RegExpConstant,RegExpNormalChar] | +| Test.java:16:102:16:108 | u{16,}+ | [RegExpQuantifier] | +| Test.java:17:10:17:13 | (?i) | [RegExpZeroWidthMatch] | +| Test.java:17:10:17:35 | (?i)(?=a)(?!b)(?<=c)(?hi)(?hell*?o*+)123\\k" + "(?>hi)(?hell*?o*+)123\\k", + "a+b*c?d{2}e{3,4}f{,5}g{6,}h+?i*?j??k{7}?l{8,9}?m{,10}?n{11,}?o++p*+q?+r{12}+s{13,14}+t{,15}+u{16,}+", + "(?i)(?=a)(?!b)(?<=c)(? Date: Thu, 10 Feb 2022 12:12:38 +0000 Subject: [PATCH 25/91] Simplify octal handling --- java/ql/lib/semmle/code/java/regex/regex.qll | 15 ++++++++------- .../library-tests/regex/RegexParseTests.expected | 7 +++++++ java/ql/test/library-tests/regex/Test.java | 3 ++- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/java/ql/lib/semmle/code/java/regex/regex.qll b/java/ql/lib/semmle/code/java/regex/regex.qll index 6646aa6f152..975d8e28309 100644 --- a/java/ql/lib/semmle/code/java/regex/regex.qll +++ b/java/ql/lib/semmle/code/java/regex/regex.qll @@ -280,13 +280,14 @@ abstract class RegexString extends StringLiteral { or // octal value \0o, \0oo, or \0ooo. Max of 0377. this.getChar(start + 1) = "0" and - end in [start + 3 .. start + 5] and - forall(int i | i in [start + 1 .. end - 1] | this.isOctal(i)) and - (end = start + 5 implies this.getChar(start + 2) <= "3") and - not ( - end < start + 5 and - this.isOctal(end) and - (end = start + 4 implies this.getChar(start + 2) <= "3") + this.isOctal(start + 2) and + ( + if this.isOctal(start + 3) + then + if this.isOctal(start + 4) and this.getChar(start + 2) in ["0", "1", "2", "3"] + then end = start + 5 + else end = start + 4 + else end = start + 3 ) or // 16-bit hex value \uhhhh diff --git a/java/ql/test/library-tests/regex/RegexParseTests.expected b/java/ql/test/library-tests/regex/RegexParseTests.expected index e997975be95..ebd0317bcc2 100644 --- a/java/ql/test/library-tests/regex/RegexParseTests.expected +++ b/java/ql/test/library-tests/regex/RegexParseTests.expected @@ -130,3 +130,10 @@ parseFailures | Test.java:18:18:18:18 | e | [RegExpConstant,RegExpNormalChar] | | Test.java:18:20:18:20 | f | [RegExpConstant,RegExpNormalChar] | | Test.java:18:22:18:22 | g | [RegExpConstant,RegExpNormalChar] | +| Test.java:19:10:19:12 | \\01 | [RegExpConstant,RegExpEscape] | +| Test.java:19:10:19:27 | \\018\\033\\0377\\0777 | [RegExpSequence] | +| Test.java:19:13:19:13 | 8 | [RegExpConstant,RegExpNormalChar] | +| Test.java:19:14:19:17 | \\033 | [RegExpConstant,RegExpEscape] | +| Test.java:19:18:19:22 | \\0377 | [RegExpConstant,RegExpEscape] | +| Test.java:19:23:19:26 | \\077 | [RegExpConstant,RegExpEscape] | +| Test.java:19:27:19:27 | 7 | [RegExpConstant,RegExpNormalChar] | diff --git a/java/ql/test/library-tests/regex/Test.java b/java/ql/test/library-tests/regex/Test.java index a60a323996f..c7ec1ecf69a 100644 --- a/java/ql/test/library-tests/regex/Test.java +++ b/java/ql/test/library-tests/regex/Test.java @@ -15,7 +15,8 @@ class Test { "(?>hi)(?hell*?o*+)123\\k", "a+b*c?d{2}e{3,4}f{,5}g{6,}h+?i*?j??k{7}?l{8,9}?m{,10}?n{11,}?o++p*+q?+r{12}+s{13,14}+t{,15}+u{16,}+", "(?i)(?=a)(?!b)(?<=c)(? Date: Thu, 10 Feb 2022 13:44:24 +0000 Subject: [PATCH 26/91] Topologically sort RegexString --- java/ql/lib/semmle/code/java/regex/regex.qll | 697 ++++++++----------- 1 file changed, 300 insertions(+), 397 deletions(-) diff --git a/java/ql/lib/semmle/code/java/regex/regex.qll b/java/ql/lib/semmle/code/java/regex/regex.qll index 975d8e28309..ce441e0e477 100644 --- a/java/ql/lib/semmle/code/java/regex/regex.qll +++ b/java/ql/lib/semmle/code/java/regex/regex.qll @@ -6,153 +6,22 @@ private import RegexFlowConfigs * A string literal that is used as a regular expression. */ abstract class RegexString extends StringLiteral { - /** Holds if a character set starts between `start` and `end`, including any negation character (`^`). */ - private predicate charSetStart0(int start, int end) { - this.nonEscapedCharAt(start) = "[" and - (if this.getChar(start + 1) = "^" then end = start + 2 else end = start + 1) - } + /** Gets the text of this regex */ + string getText() { result = this.(StringLiteral).getValue() } - /** Holds if the character at `pos` marks the end of a character class. */ - private predicate charSetEnd0(int pos) { - this.nonEscapedCharAt(pos) = "]" and - /* special case: `[]]` and `[^]]` are valid char classes. */ - not this.charSetStart0(_, pos) - } + /** Gets the `i`th character of this regex. */ + string getChar(int i) { result = this.getText().charAt(i) } - /** - * Gets the nesting depth of character classes at position `pos` - */ - private int charSetDepth(int pos) { - exists(this.getChar(pos)) and - result = - max(int j | - j = 0 or - j = - count(int i | i < pos and this.charSetStart0(i, _)) - - count(int i | i < pos and this.charSetEnd0(i)) - ) - } - - /** Hold if a top-level character set starts between `start` and `end`. */ - predicate charSetStart(int start, int end) { - this.charSetStart0(start, end) and - this.charSetDepth(start) = 0 - } - - /** Holds if a top-level character set ends at `pos`. */ - predicate charSetEnd(int pos) { - this.charSetEnd0(pos) and - this.charSetDepth(pos) = 1 - } - - /** - * Holds if there is a top-level character class beginning at `start` (inclusive) and ending at `end` (exclusive) - * - * For now, nested character classes are approximated by only considering the top-level class for parsing. - * This leads to very similar results for ReDoS queries. - */ - predicate charSet(int start, int end) { - exists(int inner_start, int inner_end | this.charSetStart(start, inner_start) | - end = inner_end + 1 and - inner_end = - min(int end_delimiter | this.charSetEnd(end_delimiter) and end_delimiter > inner_start) + /** Holds if the regex failed to parse. */ + predicate failedToParse(int i) { + exists(this.getChar(i)) and + not exists(int start, int end | + this.topLevel(start, end) and + start <= i and + end > i ) } - /** An indexed version of `charSetToken/3` */ - private predicate charSetToken(int charset_start, int index, int token_start, int token_end) { - token_start = - rank[index](int start, int end | this.charSetToken(charset_start, start, end) | start) and - this.charSetToken(charset_start, token_start, token_end) - } - - /** Either a char or a - */ - private predicate charSetToken(int charset_start, int start, int end) { - this.charSetStart(charset_start, start) and - ( - this.escapedCharacter(start, end) - or - exists(this.nonEscapedCharAt(start)) and end = start + 1 - or - this.quote(start, end) - ) - or - this.charSetToken(charset_start, _, start) and - ( - this.escapedCharacter(start, end) - or - exists(this.nonEscapedCharAt(start)) and - end = start + 1 and - not this.charSetEnd(start) - or - this.quote(start, end) - ) - } - - /** - * Holds if the character set starting at `charset_start` contains either - * a character or a range found between `start` and `end`. - */ - predicate charSetChild(int charset_start, int start, int end) { - this.charSetToken(charset_start, start, end) and - not exists(int range_start, int range_end | - this.charRange(charset_start, range_start, _, _, range_end) and - range_start <= start and - range_end >= end - ) - or - this.charRange(charset_start, start, _, _, end) - } - - /** - * Holds if the character set starting at `charset_start` contains a character range - * with lower bound found between `start` and `lower_end` - * and upper bound found between `upper_start` and `end`. - */ - predicate charRange(int charset_start, int start, int lower_end, int upper_start, int end) { - exists(int index | - this.charRangeEnd(charset_start, index) = true and - this.charSetToken(charset_start, index - 2, start, lower_end) and - this.charSetToken(charset_start, index, upper_start, end) - ) - } - - /** - * Helper predicate for `charRange`. - * We can determine where character ranges end by a left to right sweep. - * - * To avoid negative recursion we return a boolean. See `escaping`, - * the helper for `escapingChar`, for a clean use of this pattern. - */ - private boolean charRangeEnd(int charset_start, int index) { - this.charSetToken(charset_start, index, _, _) and - ( - index in [1, 2] and result = false - or - index > 2 and - exists(int connector_start | - this.charSetToken(charset_start, index - 1, connector_start, _) and - this.nonEscapedCharAt(connector_start) = "-" and - result = - this.charRangeEnd(charset_start, index - 2) - .booleanNot() - .booleanAnd(this.charRangeEnd(charset_start, index - 1).booleanNot()) - ) - or - not exists(int connector_start | - this.charSetToken(charset_start, index - 1, connector_start, _) and - this.nonEscapedCharAt(connector_start) = "-" - ) and - result = false - ) - } - - /** Holds if the character at `pos` is a "\" that is actually escaping what comes after. */ - predicate escapingChar(int pos) { - this.escaping(pos) = true and - not exists(int x, int y | this.quote(x, y) and pos in [x .. y - 1]) - } - /** * Helper predicate for `escapingChar`. * In order to avoid negative recursion, we return a boolean. @@ -219,6 +88,12 @@ abstract class RegexString extends StringLiteral { ) } + /** Holds if the character at `pos` is a "\" that is actually escaping what comes after. */ + predicate escapingChar(int pos) { + this.escaping(pos) = true and + not exists(int x, int y | this.quote(x, y) and pos in [x .. y - 1]) + } + /** * A control sequence, `\cx` * `x` may be any ascii character including special characters. @@ -229,11 +104,6 @@ abstract class RegexString extends StringLiteral { end = start + 3 } - /** Gets the text of this regex */ - string getText() { result = this.(StringLiteral).getValue() } - - string getChar(int i) { result = this.getText().charAt(i) } - string nonEscapedCharAt(int i) { result = this.getText().charAt(i) and not exists(int x, int y | this.escapedCharacter(x, y) and i in [x .. y - 1]) and @@ -241,21 +111,152 @@ abstract class RegexString extends StringLiteral { not exists(int x, int y | this.controlEscape(x, y) and i in [x .. y - 1]) } - private predicate isOptionDivider(int i) { this.nonEscapedCharAt(i) = "|" } + /** Holds if a character set starts between `start` and `end`, including any negation character (`^`). */ + private predicate charSetStart0(int start, int end) { + this.nonEscapedCharAt(start) = "[" and + (if this.getChar(start + 1) = "^" then end = start + 2 else end = start + 1) + } - private predicate isGroupEnd(int i) { this.nonEscapedCharAt(i) = ")" and not this.inCharSet(i) } + /** Holds if the character at `pos` marks the end of a character class. */ + private predicate charSetEnd0(int pos) { + this.nonEscapedCharAt(pos) = "]" and + /* special case: `[]]` and `[^]]` are valid char classes. */ + not this.charSetStart0(_, pos) + } - private predicate isGroupStart(int i) { this.nonEscapedCharAt(i) = "(" and not this.inCharSet(i) } + /** + * Gets the nesting depth of character classes at position `pos` + */ + private int charSetDepth(int pos) { + pos = -1 and result = 0 + or + exists(this.getChar(pos)) and + result = + max(int j | + j = 0 or + j = + count(int i | i < pos and this.charSetStart0(i, _)) - + count(int i | i < pos and this.charSetEnd0(i)) + ) + } - predicate failedToParse(int i) { - exists(this.getChar(i)) and - not exists(int start, int end | - this.topLevel(start, end) and - start <= i and - end > i + /** Hold if a top-level character set starts between `start` and `end`. */ + predicate charSetStart(int start, int end) { + this.charSetStart0(start, end) and + this.charSetDepth(start) = 0 + } + + /** Holds if a top-level character set ends at `pos`. */ + predicate charSetEnd(int pos) { + this.charSetEnd0(pos) and + this.charSetDepth(pos) = 1 + } + + /** + * Holds if there is a top-level character class beginning at `start` (inclusive) and ending at `end` (exclusive) + * + * For now, nested character classes are approximated by only considering the top-level class for parsing. + * This leads to very similar results for ReDoS queries. + */ + predicate charSet(int start, int end) { + exists(int inner_start, int inner_end | this.charSetStart(start, inner_start) | + end = inner_end + 1 and + inner_end = + min(int end_delimiter | this.charSetEnd(end_delimiter) and end_delimiter > inner_start) ) } + /** Either a char or a - */ + private predicate charSetToken(int charset_start, int start, int end) { + this.charSetStart(charset_start, start) and + ( + this.escapedCharacter(start, end) + or + exists(this.nonEscapedCharAt(start)) and end = start + 1 + or + this.quote(start, end) + ) + or + this.charSetToken(charset_start, _, start) and + ( + this.escapedCharacter(start, end) + or + exists(this.nonEscapedCharAt(start)) and + end = start + 1 and + not this.charSetEnd(start) + or + this.quote(start, end) + ) + } + + /** An indexed version of `charSetToken/3` */ + private predicate charSetToken(int charset_start, int index, int token_start, int token_end) { + token_start = + rank[index](int start, int end | this.charSetToken(charset_start, start, end) | start) and + this.charSetToken(charset_start, token_start, token_end) + } + + /** + * Holds if the character set starting at `charset_start` contains either + * a character or a range found between `start` and `end`. + */ + predicate charSetChild(int charset_start, int start, int end) { + this.charSetToken(charset_start, start, end) and + not exists(int range_start, int range_end | + this.charRange(charset_start, range_start, _, _, range_end) and + range_start <= start and + range_end >= end + ) + or + this.charRange(charset_start, start, _, _, end) + } + + /** + * Helper predicate for `charRange`. + * We can determine where character ranges end by a left to right sweep. + * + * To avoid negative recursion we return a boolean. See `escaping`, + * the helper for `escapingChar`, for a clean use of this pattern. + */ + private boolean charRangeEnd(int charset_start, int index) { + this.charSetToken(charset_start, index, _, _) and + ( + index in [1, 2] and result = false + or + index > 2 and + exists(int connector_start | + this.charSetToken(charset_start, index - 1, connector_start, _) and + this.nonEscapedCharAt(connector_start) = "-" and + result = + this.charRangeEnd(charset_start, index - 2) + .booleanNot() + .booleanAnd(this.charRangeEnd(charset_start, index - 1).booleanNot()) + ) + or + not exists(int connector_start | + this.charSetToken(charset_start, index - 1, connector_start, _) and + this.nonEscapedCharAt(connector_start) = "-" + ) and + result = false + ) + } + + /** + * Holds if the character set starting at `charset_start` contains a character range + * with lower bound found between `start` and `lower_end` + * and upper bound found between `upper_start` and `end`. + */ + predicate charRange(int charset_start, int start, int lower_end, int upper_start, int end) { + exists(int index | + this.charRangeEnd(charset_start, index) = true and + this.charSetToken(charset_start, index - 2, start, lower_end) and + this.charSetToken(charset_start, index, upper_start, end) + ) + } + + pragma[inline] + private predicate isOctal(int index) { this.getChar(index) = [0 .. 7].toString() } + /** An escape sequence that includes braces, such as named characters (\N{degree sign}), named classes (\p{Lower}), or hex values (\x{h..h}) */ private predicate escapedBraces(int start, int end) { this.escapingChar(start) and @@ -312,9 +313,6 @@ abstract class RegexString extends StringLiteral { ) } - pragma[inline] - private predicate isOctal(int index) { this.getChar(index) = [0 .. 7].toString() } - /** Holds if `index` is inside a character set. */ predicate inCharSet(int index) { exists(int x, int y | this.charSet(x, y) and index in [x + 1 .. y - 2]) @@ -351,7 +349,7 @@ abstract class RegexString extends StringLiteral { ) } - predicate character(int start, int end) { + private predicate character(int start, int end) { ( this.simpleCharacter(start, end) and not exists(int x, int y | this.escapedCharacter(x, y) and x <= start and y >= end) and @@ -376,39 +374,9 @@ abstract class RegexString extends StringLiteral { not this.inCharSet(start) } - /** Holds if the text in the range start,end is a group */ - predicate group(int start, int end) { - this.groupContents(start, end, _, _) - or - this.emptyGroup(start, end) - } + private predicate isGroupEnd(int i) { this.nonEscapedCharAt(i) = ")" and not this.inCharSet(i) } - /** Gets the number of the group in start,end */ - int getGroupNumber(int start, int end) { - this.group(start, end) and - result = - count(int i | this.group(i, _) and i < start and not this.nonCapturingGroupStart(i, _)) + 1 - } - - /** Gets the name, if it has one, of the group in start,end */ - string getGroupName(int start, int end) { - this.group(start, end) and - exists(int name_end | - this.namedGroupStart(start, name_end) and - result = this.getText().substring(start + 3, name_end - 1) - ) - } - - /** Holds if the text in the range start, end is a group and can match the empty string. */ - predicate zeroWidthMatch(int start, int end) { - this.emptyGroup(start, end) - or - this.negativeAssertionGroup(start, end) - or - this.positiveLookaheadAssertionGroup(start, end) - or - this.positiveLookbehindAssertionGroup(start, end) - } + private predicate isGroupStart(int i) { this.nonEscapedCharAt(i) = "(" and not this.inCharSet(i) } /** Holds if an empty group is found between `start` and `end`. */ predicate emptyGroup(int start, int end) { @@ -418,80 +386,6 @@ abstract class RegexString extends StringLiteral { ) } - private predicate emptyMatchAtStartGroup(int start, int end) { - this.emptyGroup(start, end) - or - this.negativeAssertionGroup(start, end) - or - this.positiveLookaheadAssertionGroup(start, end) - } - - private predicate emptyMatchAtEndGroup(int start, int end) { - this.emptyGroup(start, end) - or - this.negativeAssertionGroup(start, end) - or - this.positiveLookbehindAssertionGroup(start, end) - } - - private predicate negativeAssertionGroup(int start, int end) { - exists(int in_start | - this.negativeLookaheadAssertionStart(start, in_start) - or - this.negativeLookbehindAssertionStart(start, in_start) - | - this.groupContents(start, end, in_start, _) - ) - } - - /** Holds if a negative lookahead is found between `start` and `end` */ - predicate negativeLookaheadAssertionGroup(int start, int end) { - exists(int in_start | this.negativeLookaheadAssertionStart(start, in_start) | - this.groupContents(start, end, in_start, _) - ) - } - - /** Holds if a negative lookbehind is found between `start` and `end` */ - predicate negativeLookbehindAssertionGroup(int start, int end) { - exists(int in_start | this.negativeLookbehindAssertionStart(start, in_start) | - this.groupContents(start, end, in_start, _) - ) - } - - /** Holds if a positive lookahead is found between `start` and `end` */ - predicate positiveLookaheadAssertionGroup(int start, int end) { - exists(int in_start | this.lookaheadAssertionStart(start, in_start) | - this.groupContents(start, end, in_start, _) - ) - } - - /** Holds if a positive lookbehind is found between `start` and `end` */ - predicate positiveLookbehindAssertionGroup(int start, int end) { - exists(int in_start | this.lookbehindAssertionStart(start, in_start) | - this.groupContents(start, end, in_start, _) - ) - } - - private predicate groupStart(int start, int end) { - this.nonCapturingGroupStart(start, end) - or - this.flagGroupStart(start, end, _) - or - this.namedGroupStart(start, end) - or - this.lookaheadAssertionStart(start, end) - or - this.negativeLookaheadAssertionStart(start, end) - or - this.lookbehindAssertionStart(start, end) - or - this.negativeLookbehindAssertionStart(start, end) - or - this.atomicGroupStart(start, end) - or - this.simpleGroupStart(start, end) - } - private predicate nonCapturingGroupStart(int start, int end) { this.isGroupStart(start) and this.getChar(start + 1) = "?" and @@ -582,6 +476,26 @@ abstract class RegexString extends StringLiteral { end = start + 3 } + private predicate groupStart(int start, int end) { + this.nonCapturingGroupStart(start, end) + or + this.flagGroupStart(start, end, _) + or + this.namedGroupStart(start, end) + or + this.lookaheadAssertionStart(start, end) + or + this.negativeLookaheadAssertionStart(start, end) + or + this.lookbehindAssertionStart(start, end) + or + this.negativeLookbehindAssertionStart(start, end) + or + this.atomicGroupStart(start, end) + or + this.simpleGroupStart(start, end) + } + predicate groupContents(int start, int end, int in_start, int in_end) { this.groupStart(start, in_start) and end = in_end + 1 and @@ -589,6 +503,78 @@ abstract class RegexString extends StringLiteral { this.isGroupEnd(in_end) } + /** Holds if the text in the range start,end is a group */ + predicate group(int start, int end) { + this.groupContents(start, end, _, _) + or + this.emptyGroup(start, end) + } + + /** Gets the number of the group in start,end */ + int getGroupNumber(int start, int end) { + this.group(start, end) and + result = + count(int i | this.group(i, _) and i < start and not this.nonCapturingGroupStart(i, _)) + 1 + } + + /** Gets the name, if it has one, of the group in start,end */ + string getGroupName(int start, int end) { + this.group(start, end) and + exists(int name_end | + this.namedGroupStart(start, name_end) and + result = this.getText().substring(start + 3, name_end - 1) + ) + } + + /** Holds if a negative lookahead is found between `start` and `end` */ + predicate negativeLookaheadAssertionGroup(int start, int end) { + exists(int in_start | this.negativeLookaheadAssertionStart(start, in_start) | + this.groupContents(start, end, in_start, _) + ) + } + + /** Holds if a negative lookbehind is found between `start` and `end` */ + predicate negativeLookbehindAssertionGroup(int start, int end) { + exists(int in_start | this.negativeLookbehindAssertionStart(start, in_start) | + this.groupContents(start, end, in_start, _) + ) + } + + private predicate negativeAssertionGroup(int start, int end) { + exists(int in_start | + this.negativeLookaheadAssertionStart(start, in_start) + or + this.negativeLookbehindAssertionStart(start, in_start) + | + this.groupContents(start, end, in_start, _) + ) + } + + /** Holds if a positive lookahead is found between `start` and `end` */ + predicate positiveLookaheadAssertionGroup(int start, int end) { + exists(int in_start | this.lookaheadAssertionStart(start, in_start) | + this.groupContents(start, end, in_start, _) + ) + } + + /** Holds if a positive lookbehind is found between `start` and `end` */ + predicate positiveLookbehindAssertionGroup(int start, int end) { + exists(int in_start | this.lookbehindAssertionStart(start, in_start) | + this.groupContents(start, end, in_start, _) + ) + } + + /** Holds if the text in the range start, end is a group and can match the empty string. */ + predicate zeroWidthMatch(int start, int end) { + this.emptyGroup(start, end) + or + this.negativeAssertionGroup(start, end) + or + this.positiveLookaheadAssertionGroup(start, end) + or + this.positiveLookbehindAssertionGroup(start, end) + } + private predicate namedBackreference(int start, int end, string name) { this.escapingChar(start) and this.getChar(start + 1) = "k" and @@ -647,15 +633,6 @@ abstract class RegexString extends StringLiteral { this.quote(start, end) } - private predicate quantifier(int start, int end, boolean maybe_empty, boolean may_repeat_forever) { - this.shortQuantifier(start, end, maybe_empty, may_repeat_forever) and - not this.getChar(end) = ["?", "+"] - or - exists(int short_end | this.shortQuantifier(start, short_end, maybe_empty, may_repeat_forever) | - if this.getChar(short_end) = ["?", "+"] then end = short_end + 1 else end = short_end - ) - } - private predicate shortQuantifier( int start, int end, boolean maybe_empty, boolean may_repeat_forever ) { @@ -699,12 +676,13 @@ abstract class RegexString extends StringLiteral { ) } - /** - * Holds if the text in the range start,end is a quantified item, where item is a character, - * a character set or a group. - */ - predicate quantifiedItem(int start, int end, boolean maybe_empty, boolean may_repeat_forever) { - this.quantifiedPart(start, _, end, maybe_empty, may_repeat_forever) + private predicate quantifier(int start, int end, boolean maybe_empty, boolean may_repeat_forever) { + this.shortQuantifier(start, end, maybe_empty, may_repeat_forever) and + not this.getChar(end) = ["?", "+"] + or + exists(int short_end | this.shortQuantifier(start, short_end, maybe_empty, may_repeat_forever) | + if this.getChar(short_end) = ["?", "+"] then end = short_end + 1 else end = short_end + ) } /** @@ -721,6 +699,14 @@ abstract class RegexString extends StringLiteral { this.quantifier(part_end, end, maybe_empty, may_repeat_forever) } + /** + * Holds if the text in the range start,end is a quantified item, where item is a character, + * a character set or a group. + */ + predicate quantifiedItem(int start, int end, boolean maybe_empty, boolean may_repeat_forever) { + this.quantifiedPart(start, _, end, maybe_empty, may_repeat_forever) + } + /** Holds if the range `start`, `end` contains a character, a quantifier, a character set or a group. */ predicate item(int start, int end) { this.quantifiedItem(start, end, _, _) @@ -728,34 +714,6 @@ abstract class RegexString extends StringLiteral { this.baseItem(start, end) and not this.quantifier(end, _, _, _) } - private predicate subsequence(int start, int end) { - ( - start = 0 or - this.groupStart(_, start) or - this.isOptionDivider(start - 1) - ) and - this.item(start, end) - or - exists(int mid | - this.subsequence(start, mid) and - this.item(mid, end) - ) - } - - /** - * Holds if the text in the range start,end is a sequence of 1 or more items, where an item is a character, - * a character set or a group. - */ - predicate sequence(int start, int end) { - this.sequenceOrQuantified(start, end) and - not this.quantifiedItem(start, end, _, _) - } - - private predicate sequenceOrQuantified(int start, int end) { - this.subsequence(start, end) and - not this.itemStart(end) - } - private predicate itemStart(int start) { this.character(start, _) or this.isGroupStart(start) or @@ -776,9 +734,34 @@ abstract class RegexString extends StringLiteral { this.quote(_, end) } - private predicate topLevel(int start, int end) { - this.subalternation(start, end, _) and - not this.isOptionDivider(end) + private predicate isOptionDivider(int i) { this.nonEscapedCharAt(i) = "|" } + + private predicate subsequence(int start, int end) { + ( + start = 0 or + this.groupStart(_, start) or + this.isOptionDivider(start - 1) + ) and + this.item(start, end) + or + exists(int mid | + this.subsequence(start, mid) and + this.item(mid, end) + ) + } + + private predicate sequenceOrQuantified(int start, int end) { + this.subsequence(start, end) and + not this.itemStart(end) + } + + /** + * Holds if the text in the range start,end is a sequence of 1 or more items, where an item is a character, + * a character set or a group. + */ + predicate sequence(int start, int end) { + this.sequenceOrQuantified(start, end) and + not this.quantifiedItem(start, end, _, _) } private predicate subalternation(int start, int end, int itemStart) { @@ -802,6 +785,11 @@ abstract class RegexString extends StringLiteral { ) } + private predicate topLevel(int start, int end) { + this.subalternation(start, end, _) and + not this.isOptionDivider(end) + } + /** * Holds if the text in the range start,end is an alternation */ @@ -818,91 +806,6 @@ abstract class RegexString extends StringLiteral { this.alternation(start, end) and this.subalternation(start, part_end, part_start) } - - /** A part of the regex that may match the start of the string. */ - private predicate firstPart(int start, int end) { - start = 0 and end = this.getText().length() - or - exists(int x | this.firstPart(x, end) | - this.emptyMatchAtStartGroup(x, start) or - this.quantifiedItem(x, start, true, _) or - this.specialCharacter(x, start, "^") - ) - or - exists(int y | this.firstPart(start, y) | - this.item(start, end) - or - this.quantifiedPart(start, end, y, _, _) - ) - or - exists(int x, int y | this.firstPart(x, y) | - this.groupContents(x, y, start, end) - or - this.alternationOption(x, y, start, end) - ) - } - - /** A part of the regex that may match the end of the string. */ - private predicate lastPart(int start, int end) { - start = 0 and end = this.getText().length() - or - exists(int y | this.lastPart(start, y) | - this.emptyMatchAtEndGroup(end, y) - or - this.quantifiedItem(end, y, true, _) - or - this.specialCharacter(end, y, "$") - or - y = end + 2 and this.escapingChar(end) and this.getChar(end + 1) = "Z" - ) - or - exists(int x | - this.lastPart(x, end) and - this.item(start, end) - ) - or - exists(int y | this.lastPart(start, y) | this.quantifiedPart(start, end, y, _, _)) - or - exists(int x, int y | this.lastPart(x, y) | - this.groupContents(x, y, start, end) - or - this.alternationOption(x, y, start, end) - ) - } - - /** - * Holds if the item at [start, end) is one of the first items - * to be matched. - */ - predicate firstItem(int start, int end) { - ( - this.character(start, end) - or - this.quantifiedItem(start, end, _, _) - or - this.charSet(start, end) - or - this.quote(start, end) - ) and - this.firstPart(start, end) - } - - /** - * Holds if the item at [start, end) is one of the last items - * to be matched. - */ - predicate lastItem(int start, int end) { - ( - this.character(start, end) - or - this.quantifiedItem(start, end, _, _) - or - this.charSet(start, end) - or - this.quote(start, end) - ) and - this.lastPart(start, end) - } } /** A string literal used as a regular expression */ From dd200e29d40126d043db442f6b475a75da4e895e Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Fri, 11 Feb 2022 16:32:00 +0000 Subject: [PATCH 27/91] Improve char set depth calculation --- java/ql/lib/semmle/code/java/regex/regex.qll | 42 ++++++++++++++------ 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/java/ql/lib/semmle/code/java/regex/regex.qll b/java/ql/lib/semmle/code/java/regex/regex.qll index ce441e0e477..71ec4d1f7d2 100644 --- a/java/ql/lib/semmle/code/java/regex/regex.qll +++ b/java/ql/lib/semmle/code/java/regex/regex.qll @@ -125,31 +125,47 @@ abstract class RegexString extends StringLiteral { } /** - * Gets the nesting depth of character classes at position `pos` + * Holds if the character at `pos` starts a character set delimiter. + * Result is 1 for `[` and 0 for `]`. */ - private int charSetDepth(int pos) { - pos = -1 and result = 0 + private int charSetDelimiter(int pos) { + result = 1 and this.charSetStart0(pos, _) or - exists(this.getChar(pos)) and - result = - max(int j | - j = 0 or - j = - count(int i | i < pos and this.charSetStart0(i, _)) - - count(int i | i < pos and this.charSetEnd0(i)) - ) + result = -1 and this.charSetEnd0(pos) + } + + /** + * Holds if the char at `pos` is the one-based `index`th occourence of a character set delimiter (`[` or `]`). + * Result is 1 for `[` and -1 for `]`. + */ + private int charSetDelimiter(int index, int pos) { + result = this.charSetDelimiter(pos) and + pos = rank[index](int p | exists(this.charSetDelimiter(p))) + } + + bindingset[x] + int max_zero(int x) { result = max([x, 0]) } + + /** + * Gets the nesting depth of character classes after position `pos`, + * where `pos` is the position of a character set delimiter. + */ + private int charSetDepth(int index, int pos) { + index = 1 and result = max_zero(charSetDelimiter(index, pos)) + or + result = max_zero(charSetDelimiter(index, pos) + charSetDepth(index - 1, _)) } /** Hold if a top-level character set starts between `start` and `end`. */ predicate charSetStart(int start, int end) { this.charSetStart0(start, end) and - this.charSetDepth(start) = 0 + this.charSetDepth(_, start) = 1 } /** Holds if a top-level character set ends at `pos`. */ predicate charSetEnd(int pos) { this.charSetEnd0(pos) and - this.charSetDepth(pos) = 1 + this.charSetDepth(_, pos) = 0 } /** From 9f4da6503042291b5828a0dfb4fe8be6609605cb Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Fri, 11 Feb 2022 17:35:38 +0000 Subject: [PATCH 28/91] Improve calculation of locations of regex terms --- .../semmle/code/java/regex/RegexTreeView.qll | 16 ++-- java/ql/lib/semmle/code/java/regex/regex.qll | 68 ++++++++++++-- .../regex/RegexParseTests.expected | 90 ++++++++++--------- java/ql/test/library-tests/regex/Test.java | 5 +- 4 files changed, 123 insertions(+), 56 deletions(-) diff --git a/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll b/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll index fd9b93a142f..c022ba6b2ac 100644 --- a/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll +++ b/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll @@ -188,12 +188,18 @@ class RegExpTerm extends RegExpParent { predicate hasLocationInfo( string filepath, int startline, int startcolumn, int endline, int endcolumn ) { - // This currently gives incorrect results for string literals including backslashes. TODO: fix that. - // There are also more complex cases where it fails. Handling all of them would be difficult for not much gain. - exists(int re_start, int re_end | + /* + * This is an approximation that handles the simple and common case of single, + * normal string literal written in the source, but does not give correct results in more complex cases + * such as compile-time concatenation, or multi-line string literals. + */ + + exists(int re_start, int re_end, int src_start, int src_end | re.getLocation().hasLocationInfo(filepath, startline, re_start, endline, re_end) and - startcolumn = re_start + start + 1 and - endcolumn = re_start + end + re.sourceCharacter(start, src_start, _) and + re.sourceCharacter(end - 1, _, src_end) and + startcolumn = re_start + src_start and + endcolumn = re_start + src_end - 1 ) } diff --git a/java/ql/lib/semmle/code/java/regex/regex.qll b/java/ql/lib/semmle/code/java/regex/regex.qll index 71ec4d1f7d2..9bd49037cdd 100644 --- a/java/ql/lib/semmle/code/java/regex/regex.qll +++ b/java/ql/lib/semmle/code/java/regex/regex.qll @@ -27,7 +27,6 @@ abstract class RegexString extends StringLiteral { * In order to avoid negative recursion, we return a boolean. * This way, we can refer to `escaping(pos - 1).booleanNot()` * rather than to a negated version of `escaping(pos)`. - * Does not take into account escape characters inside quote sequences. */ private boolean escaping(int pos) { pos = -1 and result = false @@ -104,11 +103,10 @@ abstract class RegexString extends StringLiteral { end = start + 3 } - string nonEscapedCharAt(int i) { - result = this.getText().charAt(i) and + private string nonEscapedCharAt(int i) { + result = this.getChar(i) and not exists(int x, int y | this.escapedCharacter(x, y) and i in [x .. y - 1]) and - not exists(int x, int y | this.quote(x, y) and i in [x .. y - 1]) and - not exists(int x, int y | this.controlEscape(x, y) and i in [x .. y - 1]) + not exists(int x, int y | this.quote(x, y) and i in [x .. y - 1]) } /** Holds if a character set starts between `start` and `end`, including any negation character (`^`). */ @@ -822,6 +820,66 @@ abstract class RegexString extends StringLiteral { this.alternation(start, end) and this.subalternation(start, part_end, part_start) } + + /** + * Gets the `i`th character of this literal as it was written in the source code. + */ + string getSourceChar(int i) { result = this.(StringLiteral).getLiteral().charAt(i) } + + /** + * Helper predicate for `sourceEscapingChar` that + * results in a boolean in order to avoid negative recursion. + */ + private boolean sourceEscaping(int pos) { + pos = -1 and result = false + or + this.getSourceChar(pos) = "\\" and + result = this.sourceEscaping(pos - 1).booleanNot() + or + this.getSourceChar(pos) != "\\" and result = false + } + + /** + * Equivalent of `escapingChar` for the literal source rather than the string value. + * Holds if the character at position `pos` in the source literal is a '\' that is + * actually escaping what comes after it. + */ + private predicate sourceEcapingChar(int pos) { this.sourceEscaping(pos) = true } + + /** + * Holds if an escaped character exists between `start` and `end` in the source iteral. + */ + private predicate sourceEscapedCharacter(int start, int end) { + this.sourceEcapingChar(start) and + (if this.getSourceChar(start + 1) = "u" then end = start + 6 else end = start + 2) + } + + private predicate sourceNonEscapedCharacter(int i) { + exists(this.getSourceChar(i)) and + not exists(int x, int y | this.sourceEscapedCharacter(x, y) and i in [x .. y - 1]) + } + + /** + * Holds if a character is represented between `start` and `end` in the source literal. + */ + private predicate sourceCharacter(int start, int end) { + sourceEscapedCharacter(start, end) + or + sourceNonEscapedCharacter(start) and + end = start + 1 + } + + /** + * Holds if the `i`th character of the string is represented between offsets + * `start` (inclusive) and `end` (exclusive) in the source code of this literal. + * This only gives correct results if the literal is written as a normal single-line string literal; + * without compile-time concatenation involved. + */ + predicate sourceCharacter(int pos, int start, int end) { + exists(this.getChar(pos)) and + sourceCharacter(start, end) and + start = rank[pos + 2](int s | sourceCharacter(s, _)) + } } /** A string literal used as a regular expression */ diff --git a/java/ql/test/library-tests/regex/RegexParseTests.expected b/java/ql/test/library-tests/regex/RegexParseTests.expected index ebd0317bcc2..c6d8322e5c1 100644 --- a/java/ql/test/library-tests/regex/RegexParseTests.expected +++ b/java/ql/test/library-tests/regex/RegexParseTests.expected @@ -1,14 +1,14 @@ parseFailures #select -| Test.java:5:10:5:16 | [A-Z\\d] | [RegExpCharacterClass] | -| Test.java:5:10:5:18 | [A-Z\\d]++ | [RegExpPlus] | +| Test.java:5:10:5:17 | [A-Z\\d] | [RegExpCharacterClass] | +| Test.java:5:10:5:19 | [A-Z\\d]++ | [RegExpPlus] | | Test.java:5:11:5:11 | A | [RegExpConstant,RegExpNormalChar] | | Test.java:5:11:5:13 | A-Z | [RegExpCharacterRange] | | Test.java:5:13:5:13 | Z | [RegExpConstant,RegExpNormalChar] | -| Test.java:5:14:5:15 | \\d | [RegExpCharacterClassEscape] | -| Test.java:6:10:6:39 | \\Q hello world [ *** \\Q ) ( \\E | [RegExpConstant,RegExpQuote] | -| Test.java:7:10:7:21 | [\\Q hi ] \\E] | [RegExpCharacterClass] | -| Test.java:7:11:7:20 | \\Q hi ] \\E | [RegExpConstant,RegExpQuote] | +| Test.java:5:14:5:16 | \\d | [RegExpCharacterClassEscape] | +| Test.java:6:10:6:42 | \\Q hello world [ *** \\Q ) ( \\E | [RegExpConstant,RegExpQuote] | +| Test.java:7:10:7:23 | [\\Q hi ] \\E] | [RegExpCharacterClass] | +| Test.java:7:11:7:22 | \\Q hi ] \\E | [RegExpConstant,RegExpQuote] | | Test.java:8:10:8:12 | []] | [RegExpCharacterClass] | | Test.java:8:11:8:11 | ] | [RegExpConstant,RegExpNormalChar] | | Test.java:9:10:9:13 | [^]] | [RegExpCharacterClass] | @@ -23,33 +23,33 @@ parseFailures | Test.java:10:17:10:17 | f | [RegExpConstant,RegExpNormalChar] | | Test.java:10:18:10:18 | g | [RegExpConstant,RegExpNormalChar] | | Test.java:10:19:10:19 | ] | [RegExpConstant,RegExpNormalChar] | -| Test.java:11:10:11:53 | [abc&&[\\W\\p{Lower}\\P{Space}\\N{degree sign}]] | [RegExpCharacterClass] | -| Test.java:11:10:11:62 | [abc&&[\\W\\p{Lower}\\P{Space}\\N{degree sign}]]\\b7\\b{g}8 | [RegExpSequence] | +| Test.java:11:10:11:57 | [abc&&[\\W\\p{Lower}\\P{Space}\\N{degree sign}]] | [RegExpCharacterClass] | +| Test.java:11:10:11:68 | [abc&&[\\W\\p{Lower}\\P{Space}\\N{degree sign}]]\\b7\\b{g}8 | [RegExpSequence] | | Test.java:11:11:11:11 | a | [RegExpConstant,RegExpNormalChar] | | Test.java:11:12:11:12 | b | [RegExpConstant,RegExpNormalChar] | | Test.java:11:13:11:13 | c | [RegExpConstant,RegExpNormalChar] | | Test.java:11:14:11:14 | & | [RegExpConstant,RegExpNormalChar] | | Test.java:11:15:11:15 | & | [RegExpConstant,RegExpNormalChar] | | Test.java:11:16:11:16 | [ | [RegExpConstant,RegExpNormalChar] | -| Test.java:11:17:11:18 | \\W | [RegExpCharacterClassEscape] | -| Test.java:11:19:11:27 | \\p{Lower} | [RegExpCharacterClassEscape] | -| Test.java:11:28:11:36 | \\P{Space} | [RegExpCharacterClassEscape] | -| Test.java:11:37:11:51 | \\N{degree sign} | [RegExpConstant,RegExpEscape] | -| Test.java:11:52:11:52 | ] | [RegExpConstant,RegExpNormalChar] | -| Test.java:11:54:11:55 | \\b | [RegExpConstant,RegExpEscape] | -| Test.java:11:56:11:56 | 7 | [RegExpConstant,RegExpNormalChar] | -| Test.java:11:57:11:61 | \\b{g} | [RegExpConstant,RegExpEscape] | -| Test.java:11:62:11:62 | 8 | [RegExpConstant,RegExpNormalChar] | -| Test.java:12:10:12:12 | \\cA | [RegExpConstant,RegExpEscape] | -| Test.java:13:10:13:12 | \\c( | [RegExpConstant,RegExpEscape] | -| Test.java:14:10:14:12 | \\c\\ | [RegExpConstant,RegExpEscape] | -| Test.java:14:10:14:16 | \\c\\(ab) | [RegExpSequence] | -| Test.java:14:13:14:16 | (ab) | [RegExpGroup] | -| Test.java:14:14:14:14 | a | [RegExpConstant,RegExpNormalChar] | -| Test.java:14:14:14:15 | ab | [RegExpSequence] | -| Test.java:14:15:14:15 | b | [RegExpConstant,RegExpNormalChar] | +| Test.java:11:17:11:19 | \\W | [RegExpCharacterClassEscape] | +| Test.java:11:20:11:29 | \\p{Lower} | [RegExpCharacterClassEscape] | +| Test.java:11:30:11:39 | \\P{Space} | [RegExpCharacterClassEscape] | +| Test.java:11:40:11:55 | \\N{degree sign} | [RegExpConstant,RegExpEscape] | +| Test.java:11:56:11:56 | ] | [RegExpConstant,RegExpNormalChar] | +| Test.java:11:58:11:60 | \\b | [RegExpConstant,RegExpEscape] | +| Test.java:11:61:11:61 | 7 | [RegExpConstant,RegExpNormalChar] | +| Test.java:11:62:11:67 | \\b{g} | [RegExpConstant,RegExpEscape] | +| Test.java:11:68:11:68 | 8 | [RegExpConstant,RegExpNormalChar] | +| Test.java:12:10:12:13 | \\cA | [RegExpConstant,RegExpEscape] | +| Test.java:13:10:13:13 | \\c( | [RegExpConstant,RegExpEscape] | +| Test.java:14:10:14:14 | \\c\\ | [RegExpConstant,RegExpEscape] | +| Test.java:14:10:14:18 | \\c\\(ab) | [RegExpSequence] | +| Test.java:14:15:14:18 | (ab) | [RegExpGroup] | +| Test.java:14:16:14:16 | a | [RegExpConstant,RegExpNormalChar] | +| Test.java:14:16:14:17 | ab | [RegExpSequence] | +| Test.java:14:17:14:17 | b | [RegExpConstant,RegExpNormalChar] | | Test.java:15:10:15:15 | (?>hi) | [RegExpGroup] | -| Test.java:15:10:15:44 | (?>hi)(?hell*?o*+)123\\k | [RegExpSequence] | +| Test.java:15:10:15:45 | (?>hi)(?hell*?o*+)123\\k | [RegExpSequence] | | Test.java:15:13:15:13 | h | [RegExpConstant,RegExpNormalChar] | | Test.java:15:13:15:14 | hi | [RegExpSequence] | | Test.java:15:14:15:14 | i | [RegExpConstant,RegExpNormalChar] | @@ -65,7 +65,7 @@ parseFailures | Test.java:15:34:15:34 | 1 | [RegExpConstant,RegExpNormalChar] | | Test.java:15:35:15:35 | 2 | [RegExpConstant,RegExpNormalChar] | | Test.java:15:36:15:36 | 3 | [RegExpConstant,RegExpNormalChar] | -| Test.java:15:37:15:44 | \\k | [RegExpBackRef] | +| Test.java:15:37:15:45 | \\k | [RegExpBackRef] | | Test.java:16:10:16:10 | a | [RegExpConstant,RegExpNormalChar] | | Test.java:16:10:16:11 | a+ | [RegExpPlus] | | Test.java:16:10:16:108 | a+b*c?d{2}e{3,4}f{,5}g{6,}h+?i*?j??k{7}?l{8,9}?m{,10}?n{11,}?o++p*+q?+r{12}+s{13,14}+t{,15}+u{16,}+ | [RegExpSequence] | @@ -120,20 +120,22 @@ parseFailures | Test.java:17:30:17:35 | (?hi)(?hell*?o*+)123\\k", "a+b*c?d{2}e{3,4}f{,5}g{6,}h+?i*?j??k{7}?l{8,9}?m{,10}?n{11,}?o++p*+q?+r{12}+s{13,14}+t{,15}+u{16,}+", "(?i)(?=a)(?!b)(?<=c)(? Date: Fri, 11 Feb 2022 17:36:44 +0000 Subject: [PATCH 29/91] Move test cases to their own directory to avoid conflict --- java/ql/test/library-tests/regex/Test.java | 122 ++++++++++++++---- java/ql/test/library-tests/regex/Test2.java | 104 --------------- .../{ => parser}/RegexParseTests.expected | 0 .../regex/{ => parser}/RegexParseTests.ql | 0 .../test/library-tests/regex/parser/Test.java | 28 ++++ 5 files changed, 127 insertions(+), 127 deletions(-) delete mode 100644 java/ql/test/library-tests/regex/Test2.java rename java/ql/test/library-tests/regex/{ => parser}/RegexParseTests.expected (100%) rename java/ql/test/library-tests/regex/{ => parser}/RegexParseTests.ql (100%) create mode 100644 java/ql/test/library-tests/regex/parser/Test.java diff --git a/java/ql/test/library-tests/regex/Test.java b/java/ql/test/library-tests/regex/Test.java index 4609856f682..fd9be63b68b 100644 --- a/java/ql/test/library-tests/regex/Test.java +++ b/java/ql/test/library-tests/regex/Test.java @@ -1,28 +1,104 @@ +package generatedtest; + +import java.util.regex.Matcher; import java.util.regex.Pattern; -class Test { - static String[] regs = { - "[A-Z\\d]++", - "\\Q hello world [ *** \\Q ) ( \\E", - "[\\Q hi ] \\E]", - "[]]", - "[^]]", - "[abc[defg]]", - "[abc&&[\\W\\p{Lower}\\P{Space}\\N{degree sign}]]\\b7\\b{g}8", - "\\cA", - "\\c(", - "\\c\\(ab)", - "(?>hi)(?hell*?o*+)123\\k", - "a+b*c?d{2}e{3,4}f{,5}g{6,}h+?i*?j??k{7}?l{8,9}?m{,10}?n{11,}?o++p*+q?+r{12}+s{13,14}+t{,15}+u{16,}+", - "(?i)(?=a)(?!b)(?<=c)(?hi)(?hell*?o*+)123\\k", + "a+b*c?d{2}e{3,4}f{,5}g{6,}h+?i*?j??k{7}?l{8,9}?m{,10}?n{11,}?o++p*+q?+r{12}+s{13,14}+t{,15}+u{16,}+", + "(?i)(?=a)(?!b)(?<=c)(? Date: Mon, 14 Feb 2022 15:15:48 +0000 Subject: [PATCH 30/91] Support more escaped characters --- .../semmle/code/java/regex/RegexTreeView.qll | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll b/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll index c022ba6b2ac..a6d459cfeb8 100644 --- a/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll +++ b/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll @@ -480,10 +480,12 @@ class RegExpEscape extends RegExpNormalChar { or this.getUnescaped() = "t" and result = "\t" or - // TODO: Find a way to include a formfeed character - // also the alert/bell character for \a and escape character for \e. - // this.getUnescaped() = "f" and result = " " - // or + this.getUnescaped() = "f" and result = 12.toUnicode() // form feed + or + this.getUnescaped() = "a" and result = 7.toUnicode() // alert/bell + or + this.getUnescaped() = "e" and result = 27.toUnicode() // escape (0x1B) + or this.isUnicode() and result = this.getUnicode() } @@ -664,6 +666,7 @@ class RegExpCharacterRange extends RegExpTerm, TRegExpCharacterRange { /** * A normal character in a regular expression, that is, a character * without special meaning. This includes escaped characters. + * It also includes escape sequences that represent character classes. * * Examples: * ``` @@ -727,11 +730,8 @@ class RegExpConstant extends RegExpTerm { string value; RegExpConstant() { - (this = TRegExpNormalChar(re, start, end) or this = TRegExpQuote(re, start, end)) and - not this instanceof RegExpCharacterClassEscape and - // exclude chars in quantifiers - // TODO: push this into regex library - (value = this.(RegExpNormalChar).getValue() or value = this.(RegExpQuote).getValue()) + (value = this.(RegExpNormalChar).getValue() or value = this.(RegExpQuote).getValue()) and + not this instanceof RegExpCharacterClassEscape } /** From 5a4316d94508b4d6467ee3858e6753c02af2063f Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Tue, 15 Feb 2022 13:06:28 +0000 Subject: [PATCH 31/91] Add test cases for exponential redos query --- .../security/CWE-730/PolynomialReDoS.expected | 0 .../security/CWE-730/PolynomialReDoS.ql | 29 ++ .../security/CWE-730/ReDoS.expected | 0 .../query-tests/security/CWE-730/ReDoS.ql | 29 ++ .../query-tests/security/CWE-730/Test.java | 393 ++++++++++++++++++ 5 files changed, 451 insertions(+) create mode 100644 java/ql/test/query-tests/security/CWE-730/PolynomialReDoS.expected create mode 100644 java/ql/test/query-tests/security/CWE-730/PolynomialReDoS.ql create mode 100644 java/ql/test/query-tests/security/CWE-730/ReDoS.expected create mode 100644 java/ql/test/query-tests/security/CWE-730/ReDoS.ql create mode 100644 java/ql/test/query-tests/security/CWE-730/Test.java diff --git a/java/ql/test/query-tests/security/CWE-730/PolynomialReDoS.expected b/java/ql/test/query-tests/security/CWE-730/PolynomialReDoS.expected new file mode 100644 index 00000000000..e69de29bb2d diff --git a/java/ql/test/query-tests/security/CWE-730/PolynomialReDoS.ql b/java/ql/test/query-tests/security/CWE-730/PolynomialReDoS.ql new file mode 100644 index 00000000000..98865781dbe --- /dev/null +++ b/java/ql/test/query-tests/security/CWE-730/PolynomialReDoS.ql @@ -0,0 +1,29 @@ +import java +import TestUtilities.InlineExpectationsTest +import TestUtilities.InlineFlowTest +import semmle.code.java.security.performance.SuperlinearBackTracking +import semmle.code.java.dataflow.DataFlow +import semmle.code.java.regex.RegexTreeView +import semmle.code.java.regex.RegexFlowConfigs +import semmle.code.java.dataflow.FlowSources + +class PolynomialRedosSink extends DataFlow::Node { + RegExpLiteral reg; + + PolynomialRedosSink() { regexMatchedAgainst(reg.getRegex(), this.asExpr()) } + // RegExpTerm getRegExp() { result = reg } +} + +class PolynomialRedosConfig extends TaintTracking::Configuration { + PolynomialRedosConfig() { this = "PolynomialRodisConfig" } + + override predicate isSource(DataFlow::Node src) { src instanceof RemoteFlowSource } + + override predicate isSink(DataFlow::Node sink) { sink instanceof PolynomialRedosSink } +} + +class HasFlowTest extends InlineFlowTest { + override DataFlow::Configuration getTaintFlowConfig() { result = any(PolynomialRedosConfig c) } + + override DataFlow::Configuration getValueFlowConfig() { none() } +} diff --git a/java/ql/test/query-tests/security/CWE-730/ReDoS.expected b/java/ql/test/query-tests/security/CWE-730/ReDoS.expected new file mode 100644 index 00000000000..e69de29bb2d diff --git a/java/ql/test/query-tests/security/CWE-730/ReDoS.ql b/java/ql/test/query-tests/security/CWE-730/ReDoS.ql new file mode 100644 index 00000000000..79cb8243cd7 --- /dev/null +++ b/java/ql/test/query-tests/security/CWE-730/ReDoS.ql @@ -0,0 +1,29 @@ +import java +import TestUtilities.InlineExpectationsTest +import semmle.code.java.security.performance.ExponentialBackTracking +import semmle.code.java.regex.regex + +class HasExpRedos extends InlineExpectationsTest { + HasExpRedos() { this = "HasExpRedos" } + + override string getARelevantTag() { result = ["hasExpRedos", "hasParseFailure"] } + + override predicate hasActualResult(Location location, string element, string tag, string value) { + tag = "hasExpRedos" and + exists(RegExpTerm t, string pump, State s, string prefixMsg | + hasReDoSResult(t, pump, s, prefixMsg) and + not t.getRegex().getAMode() = "VERBOSE" and + value = "" and + location = t.getLocation() and + element = t.toString() + ) + or + tag = "hasParseFailure" and + exists(Regex r | + r.failedToParse(_) and + value = "" and + location = r.getLocation() and + element = r.toString() + ) + } +} diff --git a/java/ql/test/query-tests/security/CWE-730/Test.java b/java/ql/test/query-tests/security/CWE-730/Test.java new file mode 100644 index 00000000000..e8e4a7bcf1d --- /dev/null +++ b/java/ql/test/query-tests/security/CWE-730/Test.java @@ -0,0 +1,393 @@ + +import java.util.regex.Pattern; + +class Test { + static String[] regs = { + + // NOT GOOD; attack: "_" + "__".repeat(100) + // Adapted from marked (https://github.com/markedjs/marked), which is licensed + // under the MIT license; see file marked-LICENSE. + "^\\b_((?:__|[\\s\\S])+?)_\\b|^\\*((?:\\*\\*|[\\s\\S])+?)\\*(?!\\*)", // $ hasExpRedos + + // GOOD + // Adapted from marked (https://github.com/markedjs/marked), which is licensed + // under the MIT license; see file marked-LICENSE. + "^\\b_((?:__|[^_])+?)_\\b|^\\*((?:\\*\\*|[^*])+?)\\*(?!\\*)", + + // GOOD - there is no witness in the end that could cause the regexp to not match + // Adapted from brace-expansion (https://github.com/juliangruber/brace-expansion), + // which is licensed under the MIT license; see file brace-expansion-LICENSE. + "(.*,)+.+", + + // NOT GOOD; attack: " '" + "\\\\".repeat(100) + // Adapted from CodeMirror (https://github.com/codemirror/codemirror), + // which is licensed under the MIT license; see file CodeMirror-LICENSE. + "^(?:\\s+(?:\"(?:[^\"\\\\]|\\\\\\\\|\\\\.)+\"|'(?:[^'\\\\]|\\\\\\\\|\\\\.)+'|\\((?:[^)\\\\]|\\\\\\\\|\\\\.)+\\)))?", // $ hasExpRedos + + // GOOD + // Adapted from lulucms2 (https://github.com/yiifans/lulucms2). + "\\(\\*(?:[\\s\\S]*?\\(\\*[\\s\\S]*?\\*\\))*[\\s\\S]*?\\*\\)", + + // GOOD + // Adapted from jest (https://github.com/facebook/jest), which is licensed + // under the MIT license; see file jest-LICENSE. + "^ *(\\S.*\\|.*)\\n *([-:]+ *\\|[-| :]*)\\n((?:.*\\|.*(?:\\n|$))*)\\n*", + + // NOT GOOD, variant of good3; attack: "a|\n:|\n" + "||\n".repeat(100) + "^ *(\\S.*\\|.*)\\n *([-:]+ *\\|[-| :]*)\\n((?:.*\\|.*(?:\\n|$))*)a", // $ hasExpRedos + + // NOT GOOD; attack: "/" + "\\/a".repeat(100) + // Adapted from ANodeBlog (https://github.com/gefangshuai/ANodeBlog), + // which is licensed under the Apache License 2.0; see file ANodeBlog-LICENSE. + "\\/(?![ *])(\\\\\\/|.)*?\\/[gim]*(?=\\W|$)", // $ hasExpRedos + + // NOT GOOD; attack: "##".repeat(100) + "\na" + // Adapted from CodeMirror (https://github.com/codemirror/codemirror), + // which is licensed under the MIT license; see file CodeMirror-LICENSE. + "^([\\s\\[\\{\\(]|#.*)*$", // $ hasExpRedos + + // GOOD + "(\\r\\n|\\r|\\n)+", + + // BAD - PoC: `node -e "/((?:[^\"\']|\".*?\"|\'.*?\')*?)([(,)]|$)/.test(\"'''''''''''''''''''''''''''''''''''''''''''''\\\"\");"`. It's complicated though, because the regexp still matches something, it just matches the empty-string after the attack string. + + // NOT GOOD; attack: "a" + "[]".repeat(100) + ".b\n" + // Adapted from Knockout (https://github.com/knockout/knockout), which is + // licensed under the MIT license; see file knockout-LICENSE + "^[\\_$a-z][\\_$a-z0-9]*(\\[.*?\\])*(\\.[\\_$a-z][\\_$a-z0-9]*(\\[.*?\\])*)*$", // $ hasExpRedos + + // GOOD + "(a|.)*", + + // Testing the NFA - only some of the below are detected. + "^([a-z]+)+$", // $ hasExpRedos + "^([a-z]*)*$", // $ hasExpRedos + "^([a-zA-Z0-9])(([\\\\-.]|[_]+)?([a-zA-Z0-9]+))*(@){1}[a-z0-9]+[.]{1}(([a-z]{2,3})|([a-z]{2,3}[.]{1}[a-z]{2,3}))$", // $ hasExpRedos + "^(([a-z])+.)+[A-Z]([a-z])+$", // $ hasExpRedos + + // NOT GOOD; attack: "[" + "][".repeat(100) + "]!" + // Adapted from Prototype.js (https://github.com/prototypejs/prototype), which + // is licensed under the MIT license; see file Prototype.js-LICENSE. + "(([\\w#:.~>+()\\s-]+|\\*|\\[.*?\\])+)\\s*(,|$)", // $ hasExpRedos + + // NOT GOOD; attack: "'" + "\\a".repeat(100) + '"' + // Adapted from Prism (https://github.com/PrismJS/prism), which is licensed + // under the MIT license; see file Prism-LICENSE. + "(\"|')(\\\\?.)*?\\1", // $ hasExpRedos + + // NOT GOOD + "(b|a?b)*c", // $ hasExpRedos + + // NOT GOOD + "(a|aa?)*b", // $ hasExpRedos + + // GOOD + "(.|\\n)*!", + + // NOT GOOD; attack: "\n".repeat(100) + "." + "(?s)(.|\\n)*!", // $ hasExpRedos + + // GOOD + "([\\w.]+)*", + + // NOT GOOD + "(a|aa?)*b", // $ hasExpRedos + + // NOT GOOD + "(([\\s\\S]|[^a])*)\"", // $ hasExpRedos + + // GOOD - there is no witness in the end that could cause the regexp to not match + "([^\"']+)*", + + // NOT GOOD + "((.|[^a])*)\"", // $ hasExpRedos + + // GOOD + "((a|[^a])*)\"", + + // NOT GOOD + "((b|[^a])*)\"", // $ hasExpRedos + + // NOT GOOD + "((G|[^a])*)\"", // $ hasExpRedos + + // NOT GOOD + "(([0-9]|[^a])*)\"", // $ hasExpRedos + + // NOT GOOD + "(?:=(?:([!#\\$%&'\\*\\+\\-\\.\\^_`\\|~0-9A-Za-z]+)|\"((?:\\\\[\\x00-\\x7f]|[^\\x00-\\x08\\x0a-\\x1f\\x7f\"])*)\"))?", // $ MISSING: hasExpRedos + + // NOT GOOD + "\"((?:\\\\[\\x00-\\x7f]|[^\\x00-\\x08\\x0a-\\x1f\\x7f\"])*)\"", // $ MISSING: hasExpRedos + + // GOOD + "\"((?:\\\\[\\x00-\\x7f]|[^\\x00-\\x08\\x0a-\\x1f\\x7f\"\\\\])*)\"", + + // NOT GOOD + "(([a-z]|[d-h])*)\"", // $ hasExpRedos + + // NOT GOOD + "(([^a-z]|[^0-9])*)\"", // $ hasExpRedos + + // NOT GOOD + "((\\d|[0-9])*)\"", // $ hasExpRedos + + // NOT GOOD + "((\\s|\\s)*)\"", // $ hasExpRedos + + // NOT GOOD + "((\\w|G)*)\"", // $ hasExpRedos + + // GOOD + "((\\s|\\d)*)\"", + + // NOT GOOD + "((\\d|\\w)*)\"", // $ hasExpRedos + + // NOT GOOD + "((\\d|5)*)\"", // $ hasExpRedos + + // NOT GOOD + "((\\s|[\\f])*)\"", // $ hasExpRedos + + // NOT GOOD - but not detected (likely because \v is a character class in Java rather than a specific character in other langs) + "((\\s|[\\v]|\\\\v)*)\"", // $ MISSING: hasExpRedos + + // NOT GOOD + "((\\f|[\\f])*)\"", // $ hasExpRedos + + // NOT GOOD + "((\\W|\\D)*)\"", // $ hasExpRedos + + // NOT GOOD + "((\\S|\\w)*)\"", // $ hasExpRedos + + // NOT GOOD + "((\\S|[\\w])*)\"", // $ hasExpRedos + + // NOT GOOD + "((1s|[\\da-z])*)\"", // $ hasExpRedos + + // NOT GOOD + "((0|[\\d])*)\"", // $ hasExpRedos + + // NOT GOOD + "(([\\d]+)*)\"", // $ hasExpRedos + + // GOOD - there is no witness in the end that could cause the regexp to not match + "(\\d+(X\\d+)?)+", + + // GOOD - there is no witness in the end that could cause the regexp to not match + "([0-9]+(X[0-9]*)?)*", + + // GOOD + "^([^>]+)*(>|$)", + + // NOT GOOD + "^([^>a]+)*(>|$)", // $ hasExpRedos + + // NOT GOOD + "(\\n\\s*)+$", // $ hasExpRedos + + // NOT GOOD + "^(?:\\s+|#.*|\\(\\?#[^)]*\\))*(?:[?*+]|\\{\\d+(?:,\\d*)?})", // $ hasExpRedos + + // NOT GOOD + "\\{\\[\\s*([a-zA-Z]+)\\(([a-zA-Z]+)\\)((\\s*([a-zA-Z]+)\\: ?([ a-zA-Z{}]+),?)+)*\\s*\\]\\}", // $ hasExpRedos + + // NOT GOOD + "(a+|b+|c+)*c", // $ hasExpRedos + + // NOT GOOD + "(((a+a?)*)+b+)", // $ hasExpRedos + + // NOT GOOD + "(a+)+bbbb", // $ hasExpRedos + + // GOOD + "(a+)+aaaaa*a+", + + // NOT GOOD + "(a+)+aaaaa$", // $ hasExpRedos + + // GOOD + "(\\n+)+\\n\\n", + + // NOT GOOD + "(\\n+)+\\n\\n$", // $ hasExpRedos + + // NOT GOOD + "([^X]+)*$", // $ hasExpRedos + + // NOT GOOD + "(([^X]b)+)*$", // $ hasExpRedos + + // GOOD + "(([^X]b)+)*($|[^X]b)", + + // NOT GOOD + "(([^X]b)+)*($|[^X]c)", // $ hasExpRedos + + // GOOD + "((ab)+)*ababab", + + // GOOD + "((ab)+)*abab(ab)*(ab)+", + + // GOOD + "((ab)+)*", + + // NOT GOOD + "((ab)+)*$", // $ hasExpRedos + + // GOOD + "((ab)+)*[a1][b1][a2][b2][a3][b3]", + + // NOT GOOD + "([\\n\\s]+)*(.)", // $ hasExpRedos + + // GOOD - any witness passes through the accept state. + "(A*A*X)*", + + // GOOD + "([^\\\\\\]]+)*", + + // NOT GOOD + "(\\w*foobarbaz\\w*foobarbaz\\w*foobarbaz\\w*foobarbaz\\s*foobarbaz\\d*foobarbaz\\w*)+-", // $ hasExpRedos + + // NOT GOOD + "(.thisisagoddamnlongstringforstresstestingthequery|\\sthisisagoddamnlongstringforstresstestingthequery)*-", // $ hasExpRedos + + // NOT GOOD + "(thisisagoddamnlongstringforstresstestingthequery|this\\w+query)*-", // $ hasExpRedos + + // GOOD + "(thisisagoddamnlongstringforstresstestingthequery|imanotherbutunrelatedstringcomparedtotheotherstring)*-", + + // GOOD (but false positive caused by the extractor converting all four unpaired surrogates to \uFFFD) + "foo([\uDC66\uDC67]|[\uDC68\uDC69])*foo", // $ SPURIOUS: hasExpRedos + + // GOOD (but false positive caused by the extractor converting all four unpaired surrogates to \uFFFD) + "foo((\uDC66|\uDC67)|(\uDC68|\uDC69))*foo", // $ SPURIOUS: hasExpRedos + + // NOT GOOD (but cannot currently construct a prefix) + "a{2,3}(b+)+X", // $ hasExpRedos + + // NOT GOOD (and a good prefix test) + "^<(\\w+)((?:\\s+\\w+(?:\\s*=\\s*(?:(?:\"[^\"]*\")|(?:'[^']*')|[^>\\s]+))?)*)\\s*(\\/?)>", // $ hasExpRedos + + // GOOD + "(a+)*[\\s\\S][\\s\\S][\\s\\S]?", + + // GOOD - but we fail to see that repeating the attack string ends in the "accept any" state (due to not parsing the range `[\s\S]{2,3}`). + "(a+)*[\\s\\S]{2,3}", // $ SPURIOUS: hasExpRedos + + // GOOD - but we spuriously conclude that a rejecting suffix exists (due to not parsing the range `[\s\S]{2,}` when constructing the NFA). + "(a+)*([\\s\\S]{2,}|X)$", // $ SPURIOUS: hasExpRedos + + // GOOD + "(a+)*([\\s\\S]*|X)$", + + // NOT GOOD + "((a+)*$|[\\s\\S]+)", // $ hasExpRedos + + // GOOD - but still flagged. The only change compared to the above is the order of alternatives, which we don't model. + "([\\s\\S]+|(a+)*$)", // $ SPURIOUS: hasExpRedos + + // GOOD + "((;|^)a+)+$", + + // NOT GOOD (a good prefix test) + "(^|;)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(e+)+f", // $ hasExpRedos + + // NOT GOOD + "^ab(c+)+$", // $ hasExpRedos + + // NOT GOOD + "(\\d(\\s+)*){20}", // $ hasExpRedos + + // GOOD - but we spuriously conclude that a rejecting suffix exists. + "(([^/]|X)+)(\\/[\\s\\S]*)*$", // $ SPURIOUS: hasExpRedos + + // GOOD - but we spuriously conclude that a rejecting suffix exists. + "^((x([^Y]+)?)*(Y|$))", // $ SPURIOUS: hasExpRedos + + // NOT GOOD + "(a*)+b", // $ hasExpRedos + + // NOT GOOD + "foo([\\w-]*)+bar", // $ hasExpRedos + + // NOT GOOD + "((ab)*)+c", // $ hasExpRedos + + // NOT GOOD + "(a?a?)*b", // $ hasExpRedos + + // GOOD + "(a?)*b", + + // NOT GOOD - but not detected + "(c?a?)*b", // $ MISSING: hasExpRedos + + // NOT GOOD + "(?:a|a?)+b", // $ hasExpRedos + + // NOT GOOD - but not detected. + "(a?b?)*$", // $ MISSING: hasExpRedos + + // NOT GOOD + "PRE(([a-c]|[c-d])T(e?e?e?e?|X))+(cTcT|cTXcTX$)", // $ hasExpRedos + + // NOT GOOD + "^((a)+\\w)+$", // $ hasExpRedos + + // NOT GOOD + "^(b+.)+$", // $ hasExpRedos + + // GOOD + "a*b", + + // All 4 bad combinations of nested * and + + "(a*)*b", // $ hasExpRedos + "(a+)*b", // $ hasExpRedos + "(a*)+b", // $ hasExpRedos + "(a+)+b", // $ hasExpRedos + + // GOOD + "(a|b)+", + "(?:[\\s;,\"'<>(){}|\\[\\]@=+*]|:(?![/\\\\]))+", + + "^((?:a{|-)|\\w\\{)+X$", // $ hasParseFailure + "^((?:a{0|-)|\\w\\{\\d)+X$", // $ hasParseFailure + "^((?:a{0,|-)|\\w\\{\\d,)+X$", // $ hasParseFailure + "^((?:a{0,2|-)|\\w\\{\\d,\\d)+X$", // $ hasParseFailure + + // GOOD + "^((?:a{0,2}|-)|\\w\\{\\d,\\d\\})+X$", + + // NOT GOOD + "X(\\u0061|a)*Y", // $ hasExpRedos + + // GOOD + "X(\\u0061|b)+Y", + + // GOOD + "(\"[^\"]*?\"|[^\"\\s]+)+(?=\\s*|\\s*$)", + + // BAD + "/(\"[^\"]*?\"|[^\"\\s]+)+(?=\\s*|\\s*$)X", // $ hasExpRedos + "/(\"[^\"]*?\"|[^\"\\s]+)+(?=X)", // $ hasExpRedos + + // BAD + "\\A(\\d|0)*x", // $ hasExpRedos + "(\\d|0)*\\Z", // $ hasExpRedos + "\\b(\\d|0)*x", // $ hasExpRedos + }; + + void test() { + for (int i = 0; i < regs.length; i++) { + Pattern.compile(regs[i]); + } + } +} From e23162d91bc450b05bb18b7aabc5d3b4039fbc17 Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Tue, 15 Feb 2022 17:50:50 +0000 Subject: [PATCH 32/91] Add test cases for PolynomialRedos dataflow logic; make fixes --- .../code/java/regex/RegexFlowConfigs.qll | 16 ++--- .../CWE-730/{Test.java => ExpRedosTest.java} | 3 +- .../security/CWE-730/PolyRedosTest.java | 35 ++++++++++ .../test/query-tests/security/CWE-730/options | 1 + .../com/google/common/base/CharMatcher.java | 53 +++++++++++++++ .../com/google/common/base/Splitter.java | 67 ++++++++----------- 6 files changed, 125 insertions(+), 50 deletions(-) rename java/ql/test/query-tests/security/CWE-730/{Test.java => ExpRedosTest.java} (99%) create mode 100644 java/ql/test/query-tests/security/CWE-730/PolyRedosTest.java create mode 100644 java/ql/test/query-tests/security/CWE-730/options create mode 100644 java/ql/test/stubs/guava-30.0/com/google/common/base/CharMatcher.java diff --git a/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll b/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll index f86d787a96b..bce55552acd 100644 --- a/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll +++ b/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll @@ -37,7 +37,7 @@ abstract class RegexMatchMethodAccess extends MethodAccess { Method m; RegexMatchMethodAccess() { - this.getMethod().overrides*(m) and + this.getMethod().getSourceDeclaration().overrides*(m) and m.hasQualifiedName(package, type, name) and regexArg in [-1 .. m.getNumberOfParameters() - 1] and stringArg in [-1 .. m.getNumberOfParameters() - 1] @@ -79,9 +79,9 @@ private class JdkRegexMatchMethodAccess extends RegexMatchMethodAccess { or name = "matches" and regexArg = 0 and stringArg = 1 or - name = "split" and regexArg = 0 and stringArg = 1 + name = "split" and regexArg = -1 and stringArg = 0 or - name = "splitAsStream" and regexArg = 0 and stringArg = 1 + name = "splitAsStream" and regexArg = -1 and stringArg = 0 ) or package = "java.lang" and @@ -90,7 +90,7 @@ private class JdkRegexMatchMethodAccess extends RegexMatchMethodAccess { regexArg = 0 and stringArg = -1 or - package = "java.util" and + package = "java.util.function" and type = "Predicate" and name = "test" and regexArg = -1 and @@ -101,7 +101,7 @@ private class JdkRegexMatchMethodAccess extends RegexMatchMethodAccess { private class JdkRegexFlowStep extends RegexAdditionalFlowStep { override predicate step(DataFlow::Node node1, DataFlow::Node node2) { exists(MethodAccess ma, Method m, string package, string type, string name, int arg | - ma.getMethod().overrides*(m) and + ma.getMethod().getSourceDeclaration().overrides*(m) and m.hasQualifiedName(package, type, name) and node1.asExpr() = argOf(ma, arg) and node2.asExpr() = ma @@ -116,7 +116,7 @@ private class JdkRegexFlowStep extends RegexAdditionalFlowStep { arg = 0 ) or - package = "java.util" and + package = "java.util.function" and type = "Predicate" and name = ["and", "or", "not", "negate"] and arg = [-1, 0] @@ -126,7 +126,7 @@ private class JdkRegexFlowStep extends RegexAdditionalFlowStep { private class GuavaRegexMatchMethodAccess extends RegexMatchMethodAccess { GuavaRegexMatchMethodAccess() { - package = "com.google.common.collect" and + package = "com.google.common.base" and regexArg = -1 and stringArg = 0 and type = ["Splitter", "Splitter$MapSplitter"] and @@ -137,7 +137,7 @@ private class GuavaRegexMatchMethodAccess extends RegexMatchMethodAccess { private class GuavaRegexFlowStep extends RegexAdditionalFlowStep { override predicate step(DataFlow::Node node1, DataFlow::Node node2) { exists(MethodAccess ma, Method m, string package, string type, string name, int arg | - ma.getMethod().overrides*(m) and + ma.getMethod().getSourceDeclaration().overrides*(m) and m.hasQualifiedName(package, type, name) and node1.asExpr() = argOf(ma, arg) and node2.asExpr() = ma diff --git a/java/ql/test/query-tests/security/CWE-730/Test.java b/java/ql/test/query-tests/security/CWE-730/ExpRedosTest.java similarity index 99% rename from java/ql/test/query-tests/security/CWE-730/Test.java rename to java/ql/test/query-tests/security/CWE-730/ExpRedosTest.java index e8e4a7bcf1d..846f6139d08 100644 --- a/java/ql/test/query-tests/security/CWE-730/Test.java +++ b/java/ql/test/query-tests/security/CWE-730/ExpRedosTest.java @@ -1,7 +1,6 @@ - import java.util.regex.Pattern; -class Test { +class ExpRedosTest { static String[] regs = { // NOT GOOD; attack: "_" + "__".repeat(100) diff --git a/java/ql/test/query-tests/security/CWE-730/PolyRedosTest.java b/java/ql/test/query-tests/security/CWE-730/PolyRedosTest.java new file mode 100644 index 00000000000..a3d9872c8e9 --- /dev/null +++ b/java/ql/test/query-tests/security/CWE-730/PolyRedosTest.java @@ -0,0 +1,35 @@ +import java.util.regex.Pattern; +import java.util.function.Predicate; +import javax.servlet.http.HttpServletRequest; +import com.google.common.base.Splitter; + +class PolyRedosTest { + void test(HttpServletRequest request) { + String tainted = request.getParameter("inp"); + String reg = "a\\.\\d+E?\\d+b"; + Predicate dummyPred = (s -> s.length() % 7 == 0); + + tainted.matches(reg); // $ hasTaintFlow + tainted.split(reg); // $ hasTaintFlow + tainted.split(reg, 7); // $ hasTaintFlow + Pattern.matches(reg, tainted); // $ hasTaintFlow + Pattern.compile(reg).matcher(tainted).matches(); // $ hasTaintFlow + Pattern.compile(reg).split(tainted); // $ hasTaintFlow + Pattern.compile(reg, Pattern.DOTALL).split(tainted); // $ hasTaintFlow + Pattern.compile(reg).split(tainted, 7); // $ hasTaintFlow + Pattern.compile(reg).splitAsStream(tainted); // $ hasTaintFlow + Pattern.compile(reg).asPredicate().test(tainted); // $ hasTaintFlow + Pattern.compile(reg).asMatchPredicate().negate().and(dummyPred).or(dummyPred).test(tainted); // $ hasTaintFlow + Predicate.not(dummyPred.and(dummyPred.or(Pattern.compile(reg).asPredicate()))).test(tainted); // $ hasTaintFlow + + Splitter.on(Pattern.compile(reg)).split(tainted); // $ hasTaintFlow + Splitter.on(reg).split(tainted); + Splitter.onPattern(reg).split(tainted); // $ hasTaintFlow + Splitter.onPattern(reg).splitToList(tainted); // $ hasTaintFlow + Splitter.onPattern(reg).limit(7).omitEmptyStrings().trimResults().split(tainted); // $ hasTaintFlow + Splitter.onPattern(reg).withKeyValueSeparator(" => ").split(tainted); // $ hasTaintFlow + Splitter.on(";").withKeyValueSeparator(reg).split(tainted); + Splitter.on(";").withKeyValueSeparator(Splitter.onPattern(reg)).split(tainted); // $ hasTaintFlow + + } +} \ No newline at end of file diff --git a/java/ql/test/query-tests/security/CWE-730/options b/java/ql/test/query-tests/security/CWE-730/options new file mode 100644 index 00000000000..2f7d22dc61c --- /dev/null +++ b/java/ql/test/query-tests/security/CWE-730/options @@ -0,0 +1 @@ +// semmle-extractor-options: --javac-args -cp ${testdir}/../../../stubs/servlet-api-2.4:${testdir}/../../../stubs/guava-30.0 \ No newline at end of file diff --git a/java/ql/test/stubs/guava-30.0/com/google/common/base/CharMatcher.java b/java/ql/test/stubs/guava-30.0/com/google/common/base/CharMatcher.java new file mode 100644 index 00000000000..bcc9a0f30b4 --- /dev/null +++ b/java/ql/test/stubs/guava-30.0/com/google/common/base/CharMatcher.java @@ -0,0 +1,53 @@ +// Generated automatically from com.google.common.base.CharMatcher for testing purposes + +package com.google.common.base; + +import com.google.common.base.Predicate; + +abstract public class CharMatcher implements Predicate +{ + protected CharMatcher(){} + public CharMatcher and(CharMatcher p0){ return null; } + public CharMatcher negate(){ return null; } + public CharMatcher or(CharMatcher p0){ return null; } + public CharMatcher precomputed(){ return null; } + public String collapseFrom(CharSequence p0, char p1){ return null; } + public String removeFrom(CharSequence p0){ return null; } + public String replaceFrom(CharSequence p0, CharSequence p1){ return null; } + public String replaceFrom(CharSequence p0, char p1){ return null; } + public String retainFrom(CharSequence p0){ return null; } + public String toString(){ return null; } + public String trimAndCollapseFrom(CharSequence p0, char p1){ return null; } + public String trimFrom(CharSequence p0){ return null; } + public String trimLeadingFrom(CharSequence p0){ return null; } + public String trimTrailingFrom(CharSequence p0){ return null; } + public abstract boolean matches(char p0); + public boolean apply(Character p0){ return false; } + public boolean matchesAllOf(CharSequence p0){ return false; } + public boolean matchesAnyOf(CharSequence p0){ return false; } + public boolean matchesNoneOf(CharSequence p0){ return false; } + public int countIn(CharSequence p0){ return 0; } + public int indexIn(CharSequence p0){ return 0; } + public int indexIn(CharSequence p0, int p1){ return 0; } + public int lastIndexIn(CharSequence p0){ return 0; } + public static CharMatcher any(){ return null; } + public static CharMatcher anyOf(CharSequence p0){ return null; } + public static CharMatcher ascii(){ return null; } + public static CharMatcher breakingWhitespace(){ return null; } + public static CharMatcher digit(){ return null; } + public static CharMatcher forPredicate(Predicate p0){ return null; } + public static CharMatcher inRange(char p0, char p1){ return null; } + public static CharMatcher invisible(){ return null; } + public static CharMatcher is(char p0){ return null; } + public static CharMatcher isNot(char p0){ return null; } + public static CharMatcher javaDigit(){ return null; } + public static CharMatcher javaIsoControl(){ return null; } + public static CharMatcher javaLetter(){ return null; } + public static CharMatcher javaLetterOrDigit(){ return null; } + public static CharMatcher javaLowerCase(){ return null; } + public static CharMatcher javaUpperCase(){ return null; } + public static CharMatcher none(){ return null; } + public static CharMatcher noneOf(CharSequence p0){ return null; } + public static CharMatcher singleWidth(){ return null; } + public static CharMatcher whitespace(){ return null; } +} diff --git a/java/ql/test/stubs/guava-30.0/com/google/common/base/Splitter.java b/java/ql/test/stubs/guava-30.0/com/google/common/base/Splitter.java index 521b6a605a5..0575f99cffd 100644 --- a/java/ql/test/stubs/guava-30.0/com/google/common/base/Splitter.java +++ b/java/ql/test/stubs/guava-30.0/com/google/common/base/Splitter.java @@ -1,48 +1,35 @@ -/* - * Copyright (C) 2009 The Guava Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ +// Generated automatically from com.google.common.base.Splitter for testing purposes package com.google.common.base; -import java.util.Iterator; +import com.google.common.base.CharMatcher; import java.util.List; import java.util.Map; +import java.util.regex.Pattern; +import java.util.stream.Stream; -public final class Splitter { - - public static Splitter on(final String separator) { - return null; - } - - public Splitter omitEmptyStrings() { - return null; - } - - public Iterable split(final CharSequence sequence) { - return null; - } - - public List splitToList(CharSequence sequence) { - return null; - } - - public MapSplitter withKeyValueSeparator(String separator) { - return null; - } - - public static final class MapSplitter { - public Map split(CharSequence sequence) { - return null; +public class Splitter +{ + protected Splitter() {} + public Iterable split(CharSequence p0){ return null; } + public List splitToList(CharSequence p0){ return null; } + public Splitter limit(int p0){ return null; } + public Splitter omitEmptyStrings(){ return null; } + public Splitter trimResults(){ return null; } + public Splitter trimResults(CharMatcher p0){ return null; } + public Splitter.MapSplitter withKeyValueSeparator(Splitter p0){ return null; } + public Splitter.MapSplitter withKeyValueSeparator(String p0){ return null; } + public Splitter.MapSplitter withKeyValueSeparator(char p0){ return null; } + public Stream splitToStream(CharSequence p0){ return null; } + public static Splitter fixedLength(int p0){ return null; } + public static Splitter on(CharMatcher p0){ return null; } + public static Splitter on(Pattern p0){ return null; } + public static Splitter on(String p0){ return null; } + public static Splitter on(char p0){ return null; } + public static Splitter onPattern(String p0){ return null; } + static public class MapSplitter + { + protected MapSplitter() {} + public Map split(CharSequence p0){ return null; } } - } } From 91887ab2299cd8105d68687b9fcfa0acef5c3d92 Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Wed, 16 Feb 2022 13:14:22 +0000 Subject: [PATCH 33/91] Sync shared files --- .../lib/semmle/code/java/security/performance/ReDoSUtil.qll | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll b/java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll index 54e69cc3178..824eb9de7ae 100644 --- a/java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll +++ b/java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll @@ -1052,13 +1052,13 @@ private module SuffixConstruction { */ pragma[noinline] private string relevant(RegExpRoot root) { - exists(ascii(result)) + exists(ascii(result)) and exists(root) or exists(InputSymbol s | belongsTo(s, root) | result = intersect(s, _)) or // The characters from `hasSimpleRejectEdge`. Only `\n` is really needed (as `\n` is not in the `ascii` relation). // The three chars must be kept in sync with `hasSimpleRejectEdge`. - result = ["|", "\n", "Z"] + result = ["|", "\n", "Z"] and exists(root) } /** From 51435850803f6c2ad30eb80ae0ea596b47ca12d1 Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Wed, 16 Feb 2022 16:31:50 +0000 Subject: [PATCH 34/91] Fix to PolynomialRedos not finding results and to test cases not finding that --- .../semmle/code/java/regex/RegexTreeView.qll | 2 + .../Security/CWE/CWE-730/PolynomialReDoS.ql | 4 +- .../security/CWE-730/PolyRedosTest.java | 38 +++++++++---------- .../security/CWE-730/PolynomialReDoS.ql | 27 ++++++++++--- 4 files changed, 44 insertions(+), 27 deletions(-) diff --git a/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll b/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll index a6d459cfeb8..f7e85fe3edf 100644 --- a/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll +++ b/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll @@ -73,6 +73,8 @@ class RegExpLiteral extends TRegExpLiteral, RegExpParent { RegExpLiteral() { this = TRegExpLiteral(re) } + override string toString() { result = re.toString() } + override RegExpTerm getChild(int i) { i = 0 and result.getRegex() = re and result.isRootTerm() } /** Holds if dot, `.`, matches all characters, including newlines. */ diff --git a/java/ql/src/Security/CWE/CWE-730/PolynomialReDoS.ql b/java/ql/src/Security/CWE/CWE-730/PolynomialReDoS.ql index 1c8f3299f7f..563be6febb0 100644 --- a/java/ql/src/Security/CWE/CWE-730/PolynomialReDoS.ql +++ b/java/ql/src/Security/CWE/CWE-730/PolynomialReDoS.ql @@ -24,11 +24,11 @@ class PolynomialRedosSink extends DataFlow::Node { PolynomialRedosSink() { regexMatchedAgainst(reg.getRegex(), this.asExpr()) } - RegExpTerm getRegExp() { result = reg } + RegExpTerm getRegExp() { result.getParent() = reg } } class PolynomialRedosConfig extends TaintTracking::Configuration { - PolynomialRedosConfig() { this = "PolynomialRodisConfig" } + PolynomialRedosConfig() { this = "PolynomialRedosConfig" } override predicate isSource(DataFlow::Node src) { src instanceof RemoteFlowSource } diff --git a/java/ql/test/query-tests/security/CWE-730/PolyRedosTest.java b/java/ql/test/query-tests/security/CWE-730/PolyRedosTest.java index a3d9872c8e9..ee6ede347da 100644 --- a/java/ql/test/query-tests/security/CWE-730/PolyRedosTest.java +++ b/java/ql/test/query-tests/security/CWE-730/PolyRedosTest.java @@ -6,30 +6,30 @@ import com.google.common.base.Splitter; class PolyRedosTest { void test(HttpServletRequest request) { String tainted = request.getParameter("inp"); - String reg = "a\\.\\d+E?\\d+b"; + String reg = "0\\.\\d+E?\\d+!"; Predicate dummyPred = (s -> s.length() % 7 == 0); - tainted.matches(reg); // $ hasTaintFlow - tainted.split(reg); // $ hasTaintFlow - tainted.split(reg, 7); // $ hasTaintFlow - Pattern.matches(reg, tainted); // $ hasTaintFlow - Pattern.compile(reg).matcher(tainted).matches(); // $ hasTaintFlow - Pattern.compile(reg).split(tainted); // $ hasTaintFlow - Pattern.compile(reg, Pattern.DOTALL).split(tainted); // $ hasTaintFlow - Pattern.compile(reg).split(tainted, 7); // $ hasTaintFlow - Pattern.compile(reg).splitAsStream(tainted); // $ hasTaintFlow - Pattern.compile(reg).asPredicate().test(tainted); // $ hasTaintFlow - Pattern.compile(reg).asMatchPredicate().negate().and(dummyPred).or(dummyPred).test(tainted); // $ hasTaintFlow - Predicate.not(dummyPred.and(dummyPred.or(Pattern.compile(reg).asPredicate()))).test(tainted); // $ hasTaintFlow + tainted.matches(reg); // $ hasPolyRedos + tainted.split(reg); // $ hasPolyRedos + tainted.split(reg, 7); // $ hasPolyRedos + Pattern.matches(reg, tainted); // $ hasPolyRedos + Pattern.compile(reg).matcher(tainted).matches(); // $ hasPolyRedos + Pattern.compile(reg).split(tainted); // $ hasPolyRedos + Pattern.compile(reg, Pattern.DOTALL).split(tainted); // $ hasPolyRedos + Pattern.compile(reg).split(tainted, 7); // $ hasPolyRedos + Pattern.compile(reg).splitAsStream(tainted); // $ hasPolyRedos + Pattern.compile(reg).asPredicate().test(tainted); // $ hasPolyRedos + Pattern.compile(reg).asMatchPredicate().negate().and(dummyPred).or(dummyPred).test(tainted); // $ hasPolyRedos + Predicate.not(dummyPred.and(dummyPred.or(Pattern.compile(reg).asPredicate()))).test(tainted); // $ hasPolyRedos - Splitter.on(Pattern.compile(reg)).split(tainted); // $ hasTaintFlow + Splitter.on(Pattern.compile(reg)).split(tainted); // $ hasPolyRedos Splitter.on(reg).split(tainted); - Splitter.onPattern(reg).split(tainted); // $ hasTaintFlow - Splitter.onPattern(reg).splitToList(tainted); // $ hasTaintFlow - Splitter.onPattern(reg).limit(7).omitEmptyStrings().trimResults().split(tainted); // $ hasTaintFlow - Splitter.onPattern(reg).withKeyValueSeparator(" => ").split(tainted); // $ hasTaintFlow + Splitter.onPattern(reg).split(tainted); // $ hasPolyRedos + Splitter.onPattern(reg).splitToList(tainted); // $ hasPolyRedos + Splitter.onPattern(reg).limit(7).omitEmptyStrings().trimResults().split(tainted); // $ hasPolyRedos + Splitter.onPattern(reg).withKeyValueSeparator(" => ").split(tainted); // $ hasPolyRedos Splitter.on(";").withKeyValueSeparator(reg).split(tainted); - Splitter.on(";").withKeyValueSeparator(Splitter.onPattern(reg)).split(tainted); // $ hasTaintFlow + Splitter.on(";").withKeyValueSeparator(Splitter.onPattern(reg)).split(tainted); // $ hasPolyRedos } } \ No newline at end of file diff --git a/java/ql/test/query-tests/security/CWE-730/PolynomialReDoS.ql b/java/ql/test/query-tests/security/CWE-730/PolynomialReDoS.ql index 98865781dbe..372f1792083 100644 --- a/java/ql/test/query-tests/security/CWE-730/PolynomialReDoS.ql +++ b/java/ql/test/query-tests/security/CWE-730/PolynomialReDoS.ql @@ -1,6 +1,5 @@ import java import TestUtilities.InlineExpectationsTest -import TestUtilities.InlineFlowTest import semmle.code.java.security.performance.SuperlinearBackTracking import semmle.code.java.dataflow.DataFlow import semmle.code.java.regex.RegexTreeView @@ -11,19 +10,35 @@ class PolynomialRedosSink extends DataFlow::Node { RegExpLiteral reg; PolynomialRedosSink() { regexMatchedAgainst(reg.getRegex(), this.asExpr()) } - // RegExpTerm getRegExp() { result = reg } + + RegExpTerm getRegExp() { result.getParent() = reg } } class PolynomialRedosConfig extends TaintTracking::Configuration { - PolynomialRedosConfig() { this = "PolynomialRodisConfig" } + PolynomialRedosConfig() { this = "PolynomialRedosConfig" } override predicate isSource(DataFlow::Node src) { src instanceof RemoteFlowSource } override predicate isSink(DataFlow::Node sink) { sink instanceof PolynomialRedosSink } } -class HasFlowTest extends InlineFlowTest { - override DataFlow::Configuration getTaintFlowConfig() { result = any(PolynomialRedosConfig c) } +class HasPolyRedos extends InlineExpectationsTest { + HasPolyRedos() { this = "HasPolyRedos" } - override DataFlow::Configuration getValueFlowConfig() { none() } + override string getARelevantTag() { result = ["hasPolyRedos"] } + + override predicate hasActualResult(Location location, string element, string tag, string value) { + tag = "hasPolyRedos" and + exists( + PolynomialRedosConfig config, DataFlow::PathNode source, DataFlow::PathNode sink, + PolynomialRedosSink sinkNode, PolynomialBackTrackingTerm regexp + | + config.hasFlowPath(source, sink) and + sinkNode = sink.getNode() and + regexp.getRootTerm() = sinkNode.getRegExp() and + location = sinkNode.getLocation() and + element = sinkNode.toString() and + value = "" + ) + } } From 57ba8a4d1b9d4f8a1c112834191fba5dc593ea5e Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Mon, 21 Feb 2022 16:39:59 +0000 Subject: [PATCH 35/91] Improve handling of hex escapes; and support some named character classes --- .../semmle/code/java/regex/RegexTreeView.qll | 81 +++++++++++++------ .../security/performance/RegExpTreeView.qll | 2 + .../security/CWE-730/ExpRedosTest.java | 36 +++++++++ 3 files changed, 96 insertions(+), 23 deletions(-) diff --git a/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll b/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll index f7e85fe3edf..04bda79e9ad 100644 --- a/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll +++ b/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll @@ -55,7 +55,7 @@ class RegExpParent extends TRegExpParent { string toString() { result = "RegExpParent" } /** Gets the `i`th child term. */ - abstract RegExpTerm getChild(int i); + RegExpTerm getChild(int i) { none() } /** Gets a child term . */ RegExpTerm getAChild() { result = this.getChild(_) } @@ -143,26 +143,6 @@ class RegExpTerm extends RegExpParent { */ predicate isRootTerm() { start = 0 and end = re.getText().length() } - override RegExpTerm getChild(int i) { - result = this.(RegExpAlt).getChild(i) - or - result = this.(RegExpBackRef).getChild(i) - or - result = this.(RegExpCharacterClass).getChild(i) - or - result = this.(RegExpCharacterRange).getChild(i) - or - result = this.(RegExpNormalChar).getChild(i) - or - result = this.(RegExpGroup).getChild(i) - or - result = this.(RegExpQuantifier).getChild(i) - or - result = this.(RegExpSequence).getChild(i) - or - result = this.(RegExpSpecialChar).getChild(i) - } - /** * Gets the parent term of this regular expression term, or the * regular expression literal if this is the root term. @@ -508,7 +488,7 @@ class RegExpEscape extends RegExpNormalChar { /** * Holds if this is a unicode escape. */ - private predicate isUnicode() { this.getText().matches("\\u%") } + private predicate isUnicode() { this.getText().matches(["\\u%", "\\x%"]) } /** * Gets the unicode char for this escape. @@ -520,13 +500,24 @@ class RegExpEscape extends RegExpNormalChar { ) } + /** Gets the part of this escape that is a hexidecimal string */ + private string getHexString() { + this.isUnicode() and + if this.getText().matches("\\u%") // \uhhhh + then result = this.getText().suffix(2) + else + if this.getText().matches("\\x{%") // \x{h..h} + then result = this.getText().substring(3, this.getText().length() - 1) + else result = this.getText().suffix(2) // \xhh + } + /** * Gets int value for the `index`th char in the hex number of the unicode escape. * E.g. for `\u0061` and `index = 2` this returns 96 (the number `6` interpreted as hex). */ private int getHexValueFromUnicode(int index) { this.isUnicode() and - exists(string hex, string char | hex = this.getText().suffix(2) | + exists(string hex, string char | hex = this.getHexString() | char = hex.charAt(index) and result = 16.pow(hex.length() - index - 1) * toHex(char) ) @@ -574,6 +565,50 @@ class RegExpCharacterClassEscape extends RegExpEscape { override string getPrimaryQLClass() { result = "RegExpCharacterClassEscape" } } +/** + * A named character class in a regular expression. + * + * Examples: + * + * ``` + * \p{Digit} + * \p{IsLowerCase} + */ +class RegExpNamedProperty extends RegExpCharacterClassEscape { + boolean inverted; + string name; + + RegExpNamedProperty() { + name = this.getValue().substring(2, this.getValue().length() - 1) and + ( + inverted = false and + this.getValue().charAt(0) = "p" + or + inverted = true and + this.getValue().charAt(0) = "P" + ) + } + + /** Holds if this class is inverted. */ + predicate isInverted() { inverted = true } + + /** Gets the name of this class. */ + string getClassName() { result = name } + + /** + * Gets an equivalent single-chcracter escape sequence for this class (e.g. \d) if possible, excluding the escape character. + */ + string getBackslashEquivalent() { + exists(string eq | if inverted = true then result = eq.toUpperCase() else result = eq | + name = ["Digit", "IsDigit"] and + eq = "d" + or + name = ["Space", "IsWhite_Space"] and + eq = "s" + ) + } +} + /** * A character class in a regular expression. * diff --git a/java/ql/lib/semmle/code/java/security/performance/RegExpTreeView.qll b/java/ql/lib/semmle/code/java/security/performance/RegExpTreeView.qll index ff3443acbca..608c03d006a 100644 --- a/java/ql/lib/semmle/code/java/security/performance/RegExpTreeView.qll +++ b/java/ql/lib/semmle/code/java/security/performance/RegExpTreeView.qll @@ -12,6 +12,8 @@ import semmle.code.java.regex.RegexTreeView */ predicate isEscapeClass(RegExpTerm term, string clazz) { term.(RegExpCharacterClassEscape).getValue() = clazz + or + term.(RegExpNamedProperty).getBackslashEquivalent() = clazz } /** diff --git a/java/ql/test/query-tests/security/CWE-730/ExpRedosTest.java b/java/ql/test/query-tests/security/CWE-730/ExpRedosTest.java index 846f6139d08..a9fc19e5d50 100644 --- a/java/ql/test/query-tests/security/CWE-730/ExpRedosTest.java +++ b/java/ql/test/query-tests/security/CWE-730/ExpRedosTest.java @@ -371,6 +371,42 @@ class ExpRedosTest { // GOOD "X(\\u0061|b)+Y", + // NOT GOOD + "X(\\x61|a)*Y", // $ hasExpRedos + + // GOOD + "X(\\x61|b)+Y", + + // NOT GOOD + "X(\\x{061}|a)*Y", // $ hasExpRedos + + // GOOD + "X(\\x{061}|b)+Y", + + // NOT GOOD + "X(\\p{Digit}|7)*Y", // $ hasExpRedos + + // GOOD + "X(\\p{Digit}|b)+Y", + + // NOT GOOD + "X(\\P{Digit}|b)*Y", // $ hasExpRedos + + // GOOD + "X(\\P{Digit}|7)+Y", + + // NOT GOOD + "X(\\p{IsDigit}|7)*Y", // $ hasExpRedos + + // GOOD + "X(\\p{IsDigit}|b)+Y", + + // NOT GOOD - but not detected + "X(\\p{Alpha}|a)*Y", // $ MISSING: hasExpRedos + + // GOOD + "X(\\p{Alpha}|7)+Y", + // GOOD "(\"[^\"]*?\"|[^\"\\s]+)+(?=\\s*|\\s*$)", From c312b4b6b0e131def14fbc5bac9587db1d9da520 Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Tue, 22 Feb 2022 14:51:38 +0000 Subject: [PATCH 36/91] Add missing qldoc --- java/ql/lib/semmle/code/java/PrintAst.qll | 7 +++++++ java/ql/lib/semmle/code/java/regex/RegexFlowModels.qll | 2 ++ java/ql/lib/semmle/code/java/regex/regex.qll | 9 ++++++++- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/java/ql/lib/semmle/code/java/PrintAst.qll b/java/ql/lib/semmle/code/java/PrintAst.qll index ee3a2800584..fa36821f6c7 100644 --- a/java/ql/lib/semmle/code/java/PrintAst.qll +++ b/java/ql/lib/semmle/code/java/PrintAst.qll @@ -167,6 +167,13 @@ class PrintAstNode extends TPrintAstNode { */ Location getLocation() { none() } + /** + * Holds if this node is at the specified location. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `filepath`. + * For more information, see + * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/). + */ predicate hasLocationInfo( string filepath, int startline, int startcolumn, int endline, int endcolumn ) { diff --git a/java/ql/lib/semmle/code/java/regex/RegexFlowModels.qll b/java/ql/lib/semmle/code/java/regex/RegexFlowModels.qll index 65ff6199088..c8ba5e882b6 100644 --- a/java/ql/lib/semmle/code/java/regex/RegexFlowModels.qll +++ b/java/ql/lib/semmle/code/java/regex/RegexFlowModels.qll @@ -1,3 +1,5 @@ +/** Definitions of data flow steps for determining flow of regular expressions. */ + import java import semmle.code.java.dataflow.ExternalFlow diff --git a/java/ql/lib/semmle/code/java/regex/regex.qll b/java/ql/lib/semmle/code/java/regex/regex.qll index 9bd49037cdd..8f168ea2255 100644 --- a/java/ql/lib/semmle/code/java/regex/regex.qll +++ b/java/ql/lib/semmle/code/java/regex/regex.qll @@ -1,3 +1,7 @@ +/** + * Definitions for parsing regular expressions. + */ + import java private import RegexFlowConfigs @@ -142,7 +146,7 @@ abstract class RegexString extends StringLiteral { } bindingset[x] - int max_zero(int x) { result = max([x, 0]) } + private int max_zero(int x) { result = max([x, 0]) } /** * Gets the nesting depth of character classes after position `pos`, @@ -375,11 +379,13 @@ abstract class RegexString extends StringLiteral { not exists(int x, int y | this.backreference(x, y) and x <= start and y >= end) } + /** Holds if a normal character or escape sequence is between `start` and `end`. */ predicate normalCharacter(int start, int end) { this.character(start, end) and not this.specialCharacter(start, end, _) } + /** Holds if a special character `char` is between `start` and `end`. */ predicate specialCharacter(int start, int end, string char) { this.character(start, end) and end = start + 1 and @@ -510,6 +516,7 @@ abstract class RegexString extends StringLiteral { this.simpleGroupStart(start, end) } + /** Holds if the text in the range start,end is a group with contents in the range in_start,in_end */ predicate groupContents(int start, int end, int in_start, int in_end) { this.groupStart(start, in_start) and end = in_end + 1 and From 5364001aa2e64846982fb80c29a0fa8bb6c31dce Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Tue, 22 Feb 2022 17:10:20 +0000 Subject: [PATCH 37/91] Update docs to be about Java --- .../Security/CWE/CWE-730/PolynomialReDoS.qhelp | 18 +++++++++--------- java/ql/src/Security/CWE/CWE-730/ReDoS.qhelp | 4 ++-- .../CWE/CWE-730/ReDoSIntroduction.inc.qhelp | 9 ++++++++- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/java/ql/src/Security/CWE/CWE-730/PolynomialReDoS.qhelp b/java/ql/src/Security/CWE/CWE-730/PolynomialReDoS.qhelp index fa8a3563d23..dbb1f4c37f5 100644 --- a/java/ql/src/Security/CWE/CWE-730/PolynomialReDoS.qhelp +++ b/java/ql/src/Security/CWE/CWE-730/PolynomialReDoS.qhelp @@ -14,13 +14,13 @@

    - - re.sub(r"^\s+|\s+$", "", text) # BAD + + Pattern.compile("^\\s+|\\s+$").matcher(text).replaceAll("") // BAD

    - The sub-expression "\s+$" will match the + The sub-expression "\\s+$" will match the whitespace characters in text from left to right, but it can start matching anywhere within a whitespace sequence. This is problematic for strings that do not end with a whitespace @@ -45,14 +45,14 @@ Avoid this problem by rewriting the regular expression to not contain the ambiguity about when to start matching whitespace sequences. For instance, by using a negative look-behind - (^\s+|(?<!\s)\s+$), or just by using the built-in strip - method (text.strip()). + ("^\\s+|(?<!\\s)\\s+$"), or just by using the built-in trim + method (text.trim()).

    - Note that the sub-expression "^\s+" is + Note that the sub-expression "^\\s+" is not problematic as the ^ anchor restricts when that sub-expression can start matching, and as the regular expression engine matches from left to right. @@ -70,8 +70,8 @@ using scientific notation:

    - - ^0\.\d+E?\d+$ # BAD + + "^0\\.\\d+E?\\d+$""

    @@ -97,7 +97,7 @@ To make the processing faster, the regular expression should be rewritten such that the two \d+ sub-expressions - do not have overlapping matches: ^0\.\d+(E\d+)?$. + do not have overlapping matches: "^0\\.\\d+(E\\d+)?$".

    diff --git a/java/ql/src/Security/CWE/CWE-730/ReDoS.qhelp b/java/ql/src/Security/CWE/CWE-730/ReDoS.qhelp index 9cfbcc32354..08b67acb638 100644 --- a/java/ql/src/Security/CWE/CWE-730/ReDoS.qhelp +++ b/java/ql/src/Security/CWE/CWE-730/ReDoS.qhelp @@ -10,7 +10,7 @@

    Consider this regular expression:

    - + ^_(__|.)+_$

    @@ -24,7 +24,7 @@ This problem can be avoided by rewriting the regular expression to remove the ambiguity between the two branches of the alternative inside the repetition:

    - + ^_(__|[^_])+_$ diff --git a/java/ql/src/Security/CWE/CWE-730/ReDoSIntroduction.inc.qhelp b/java/ql/src/Security/CWE/CWE-730/ReDoSIntroduction.inc.qhelp index f533097c222..f6e4dbd0a5f 100644 --- a/java/ql/src/Security/CWE/CWE-730/ReDoSIntroduction.inc.qhelp +++ b/java/ql/src/Security/CWE/CWE-730/ReDoSIntroduction.inc.qhelp @@ -17,7 +17,7 @@

    - The regular expression engine provided by Python uses a backtracking non-deterministic finite + The regular expression engine provided by Java uses a backtracking non-deterministic finite automata to implement regular expression matching. While this approach is space-efficient and allows supporting advanced features like capture groups, it is not time-efficient in general. The worst-case @@ -38,6 +38,11 @@ references.

    + +

    + Note that Java versions 9 and above have some mitigations against ReDoS; however they aren't perfect + and more complex regular expressions can still be affected by this problem. +

    @@ -48,6 +53,8 @@ ensure that the strings matched with the regular expression are short enough that the time-complexity does not matter. + Alternatively, an alternate regex library that guarantees linear time execution, such as Google's RE2J, may be used. +

    From 3ce0c2c23b66247f714a29417bcc701fcf7f5e2c Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Thu, 3 Mar 2022 11:51:14 +0000 Subject: [PATCH 38/91] Add more regex use functions in String --- java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll | 2 +- java/ql/lib/semmle/code/java/regex/RegexFlowModels.qll | 2 ++ java/ql/test/query-tests/security/CWE-730/PolyRedosTest.java | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll b/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll index bce55552acd..948cc557169 100644 --- a/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll +++ b/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll @@ -86,7 +86,7 @@ private class JdkRegexMatchMethodAccess extends RegexMatchMethodAccess { or package = "java.lang" and type = "String" and - name = ["matches", "split"] and + name = ["matches", "split", "replaceAll", "replaceFirst"] and regexArg = 0 and stringArg = -1 or diff --git a/java/ql/lib/semmle/code/java/regex/RegexFlowModels.qll b/java/ql/lib/semmle/code/java/regex/RegexFlowModels.qll index c8ba5e882b6..c6ee2ace865 100644 --- a/java/ql/lib/semmle/code/java/regex/RegexFlowModels.qll +++ b/java/ql/lib/semmle/code/java/regex/RegexFlowModels.qll @@ -14,6 +14,8 @@ private class RegexSinkCsv extends SinkModelCsv { "java.util;String;false;matches;(String);;Argument[0];regex-compile", "java.util;String;false;split;(String);;Argument[0];regex-compile", "java.util;String;false;split;(String,int);;Argument[0];regex-compile", + "java.util;String;false;replaceAll;(String,String);;Argument[0];regex-compile", + "java.util;String;false;replaceFirst;(String,String);;Argument[0];regex-compile", "com.google.common.base;Splitter;false;onPattern;(String);;Argument[0];regex-compile" ] } diff --git a/java/ql/test/query-tests/security/CWE-730/PolyRedosTest.java b/java/ql/test/query-tests/security/CWE-730/PolyRedosTest.java index ee6ede347da..e825e1ad2db 100644 --- a/java/ql/test/query-tests/security/CWE-730/PolyRedosTest.java +++ b/java/ql/test/query-tests/security/CWE-730/PolyRedosTest.java @@ -12,6 +12,8 @@ class PolyRedosTest { tainted.matches(reg); // $ hasPolyRedos tainted.split(reg); // $ hasPolyRedos tainted.split(reg, 7); // $ hasPolyRedos + tainted.replaceAll(reg, "a"); // $ hasPolyRedos + tainted.replaceFirst(reg, "a"); // $ hasPolyRedos Pattern.matches(reg, tainted); // $ hasPolyRedos Pattern.compile(reg).matcher(tainted).matches(); // $ hasPolyRedos Pattern.compile(reg).split(tainted); // $ hasPolyRedos From 9bd39168003b66df4e4dd35dbd430e0adffd4e65 Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Thu, 3 Mar 2022 12:06:29 +0000 Subject: [PATCH 39/91] Add change note --- java/ql/lib/change-notes/2022-03-03-redos.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 java/ql/lib/change-notes/2022-03-03-redos.md diff --git a/java/ql/lib/change-notes/2022-03-03-redos.md b/java/ql/lib/change-notes/2022-03-03-redos.md new file mode 100644 index 00000000000..daf1dd51be1 --- /dev/null +++ b/java/ql/lib/change-notes/2022-03-03-redos.md @@ -0,0 +1,6 @@ +--- +category: newQuery +--- + +* Two new queries "Inefficient regular expression" (`java/redos`) and "Polynomial regular expression used on uncontrolled data" (`java/polynomial-redos`) have been added. +These queries help find instances of Regular Expression Denial of Service vulnerabilities. \ No newline at end of file From 2d963176bf02d5c858e4719bef5977fd3a0bfd6a Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Thu, 3 Mar 2022 12:41:28 +0000 Subject: [PATCH 40/91] Fix change note --- java/ql/{lib => src}/change-notes/2022-03-03-redos.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename java/ql/{lib => src}/change-notes/2022-03-03-redos.md (100%) diff --git a/java/ql/lib/change-notes/2022-03-03-redos.md b/java/ql/src/change-notes/2022-03-03-redos.md similarity index 100% rename from java/ql/lib/change-notes/2022-03-03-redos.md rename to java/ql/src/change-notes/2022-03-03-redos.md From f5809a7440a7cbb26bef8a47c323672b9e8103a5 Mon Sep 17 00:00:00 2001 From: Chris Smowton Date: Thu, 3 Mar 2022 20:31:23 +0000 Subject: [PATCH 41/91] ReDoS performance fixes --- .../semmle/code/java/regex/RegexTreeView.qll | 39 +++++++++++-------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll b/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll index 04bda79e9ad..fbe1d10ea72 100644 --- a/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll +++ b/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll @@ -157,6 +157,11 @@ class RegExpTerm extends RegExpParent { /** Gets the offset at which this term ends. */ int getEnd() { result = end } + /** Holds if this term occurs in regex `inRe` offsets `startOffset` to `endOffset`. */ + predicate occursInRegex(Regex inRe, int startOffset, int endOffset) { + inRe = re and startOffset = start and endOffset = end + } + override string toString() { result = re.getText().substring(start, end) } /** @@ -385,18 +390,15 @@ private RegExpTerm seqChild(Regex re, int start, int end, int i) { re.sequence(start, end) and ( i = 0 and - result.getRegex() = re and - result.getStart() = start and exists(int itemEnd | re.item(start, itemEnd) and - result.getEnd() = itemEnd + result.occursInRegex(re, start, itemEnd) ) or i > 0 and - result.getRegex() = re and exists(int itemStart | itemStart = seqChildEnd(re, start, end, i - 1) | - result.getStart() = itemStart and - re.item(itemStart, result.getEnd()) + re.item(itemStart, result.getEnd()) and + result.occursInRegex(re, itemStart, _) ) ) } @@ -642,18 +644,15 @@ class RegExpCharacterClass extends RegExpTerm, TRegExpCharacterClass { override RegExpTerm getChild(int i) { i = 0 and - result.getRegex() = re and exists(int itemStart, int itemEnd | - result.getStart() = itemStart and re.charSetStart(start, itemStart) and re.charSetChild(start, itemStart, itemEnd) and - result.getEnd() = itemEnd + result.occursInRegex(re, itemStart, itemEnd) ) or i > 0 and - result.getRegex() = re and exists(int itemStart | itemStart = this.getChild(i - 1).getEnd() | - result.getStart() = itemStart and + result.occursInRegex(re, itemStart, _) and re.charSetChild(start, itemStart, result.getEnd()) ) } @@ -823,6 +822,16 @@ class RegExpGroup extends RegExpTerm, TRegExpGroup { } override string getPrimaryQLClass() { result = "RegExpGroup" } + + /** Holds if this is the `n`th numbered group of literal `lit`. */ + predicate isNumberedGroupOfLiteral(RegExpLiteral lit, int n) { + lit = this.getLiteral() and n = this.getNumber() + } + + /** Holds if this is a group with name `name` of literal `lit`. */ + predicate isNamedGroupOfLiteral(RegExpLiteral lit, string name) { + lit = this.getLiteral() and name = this.getName() + } } /** @@ -1054,11 +1063,9 @@ class RegExpBackRef extends RegExpTerm, TRegExpBackRef { /** Gets the capture group this back reference refers to. */ RegExpGroup getGroup() { - result.getLiteral() = this.getLiteral() and - ( - result.getNumber() = this.getNumber() or - result.getName() = this.getName() - ) + result.isNumberedGroupOfLiteral(this.getLiteral(), this.getNumber()) + or + result.isNamedGroupOfLiteral(this.getLiteral(), this.getName()) } override RegExpTerm getChild(int i) { none() } From 5ba6bafbef9873af76f2c678bda2f8a8bf5d6103 Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Fri, 4 Mar 2022 10:56:18 +0000 Subject: [PATCH 42/91] Use occursInRegex more ccnsistently throughout --- .../semmle/code/java/regex/RegexTreeView.qll | 44 +++++++------------ 1 file changed, 17 insertions(+), 27 deletions(-) diff --git a/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll b/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll index fbe1d10ea72..a7e3928f085 100644 --- a/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll +++ b/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll @@ -246,9 +246,7 @@ class RegExpQuantifier extends RegExpTerm, TRegExpQuantifier { override RegExpTerm getChild(int i) { i = 0 and - result.getRegex() = re and - result.getStart() = start and - result.getEnd() = part_end + result.occursInRegex(re, start, part_end) } /** Holds if this term may match an unlimited number of times. */ @@ -396,9 +394,9 @@ private RegExpTerm seqChild(Regex re, int start, int end, int i) { ) or i > 0 and - exists(int itemStart | itemStart = seqChildEnd(re, start, end, i - 1) | - re.item(itemStart, result.getEnd()) and - result.occursInRegex(re, itemStart, _) + exists(int itemStart, int itemEnd | itemStart = seqChildEnd(re, start, end, i - 1) | + re.item(itemStart, itemEnd) and + result.occursInRegex(re, itemStart, itemEnd) ) ) } @@ -417,20 +415,17 @@ class RegExpAlt extends RegExpTerm, TRegExpAlt { override RegExpTerm getChild(int i) { i = 0 and - result.getRegex() = re and - result.getStart() = start and exists(int part_end | re.alternationOption(start, end, start, part_end) and - result.getEnd() = part_end + result.occursInRegex(re, start, part_end) ) or i > 0 and - result.getRegex() = re and - exists(int part_start | + exists(int part_start, int part_end | part_start = this.getChild(i - 1).getEnd() + 1 // allow for the | | - result.getStart() = part_start and - re.alternationOption(start, end, part_start, result.getEnd()) + re.alternationOption(start, end, part_start, part_end) and + result.occursInRegex(re, part_start, part_end) ) } @@ -651,9 +646,9 @@ class RegExpCharacterClass extends RegExpTerm, TRegExpCharacterClass { ) or i > 0 and - exists(int itemStart | itemStart = this.getChild(i - 1).getEnd() | - result.occursInRegex(re, itemStart, _) and - re.charSetChild(start, itemStart, result.getEnd()) + exists(int itemStart, int itemEnd | itemStart = this.getChild(i - 1).getEnd() | + result.occursInRegex(re, itemStart, itemEnd) and + re.charSetChild(start, itemStart, itemEnd) ) } @@ -686,14 +681,10 @@ class RegExpCharacterRange extends RegExpTerm, TRegExpCharacterRange { override RegExpTerm getChild(int i) { i = 0 and - result.getRegex() = re and - result.getStart() = start and - result.getEnd() = lower_end + result.occursInRegex(re, start, lower_end) or i = 1 and - result.getRegex() = re and - result.getStart() = upper_start and - result.getEnd() = end + result.occursInRegex(re, upper_start, end) } override string getPrimaryQLClass() { result = "RegExpCharacterRange" } @@ -816,9 +807,10 @@ class RegExpGroup extends RegExpTerm, TRegExpGroup { string getName() { result = re.getGroupName(start, end) } override RegExpTerm getChild(int i) { - result.getRegex() = re and i = 0 and - re.groupContents(start, end, result.getStart(), result.getEnd()) + exists(int in_start, int in_end | re.groupContents(start, end, in_start, in_end) | + result.occursInRegex(re, in_start, in_end) + ) } override string getPrimaryQLClass() { result = "RegExpGroup" } @@ -946,9 +938,7 @@ class RegExpSubPattern extends RegExpZeroWidthMatch { /** Gets the lookahead term. */ RegExpTerm getOperand() { exists(int in_start, int in_end | re.groupContents(start, end, in_start, in_end) | - result.getRegex() = re and - result.getStart() = in_start and - result.getEnd() = in_end + result.occursInRegex(re, in_start, in_end) ) } } From 49374b877af3e0bf1e5dfc09a9cff9b5bb85e0a5 Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Tue, 8 Mar 2022 16:23:24 +0000 Subject: [PATCH 43/91] Fix parsing of alternations in character classes --- java/ql/lib/semmle/code/java/regex/regex.qll | 1 + .../ql/test/library-tests/regex/parser/RegexParseTests.expected | 2 ++ java/ql/test/library-tests/regex/parser/Test.java | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/java/ql/lib/semmle/code/java/regex/regex.qll b/java/ql/lib/semmle/code/java/regex/regex.qll index 8f168ea2255..2cd3b9a4035 100644 --- a/java/ql/lib/semmle/code/java/regex/regex.qll +++ b/java/ql/lib/semmle/code/java/regex/regex.qll @@ -807,6 +807,7 @@ abstract class RegexString extends StringLiteral { } private predicate topLevel(int start, int end) { + not this.inCharSet(start) and this.subalternation(start, end, _) and not this.isOptionDivider(end) } diff --git a/java/ql/test/library-tests/regex/parser/RegexParseTests.expected b/java/ql/test/library-tests/regex/parser/RegexParseTests.expected index c6d8322e5c1..66d63f70808 100644 --- a/java/ql/test/library-tests/regex/parser/RegexParseTests.expected +++ b/java/ql/test/library-tests/regex/parser/RegexParseTests.expected @@ -139,3 +139,5 @@ parseFailures | Test.java:19:26:19:30 | \\077 | [RegExpConstant,RegExpEscape] | | Test.java:19:31:19:31 | 7 | [RegExpConstant,RegExpNormalChar] | | Test.java:19:32:19:37 | \u1337 | [RegExpConstant,RegExpNormalChar] | +| Test.java:20:10:20:12 | [\|] | [RegExpCharacterClass] | +| Test.java:20:11:20:11 | \| | [RegExpConstant,RegExpNormalChar] | diff --git a/java/ql/test/library-tests/regex/parser/Test.java b/java/ql/test/library-tests/regex/parser/Test.java index 4609856f682..61822294397 100644 --- a/java/ql/test/library-tests/regex/parser/Test.java +++ b/java/ql/test/library-tests/regex/parser/Test.java @@ -17,7 +17,7 @@ class Test { "(?i)(?=a)(?!b)(?<=c)(? Date: Tue, 8 Mar 2022 16:47:39 +0000 Subject: [PATCH 44/91] Support possessive quantifiers, which cannot backtrack. They are approximated by limiting them to up to one repetition (effectively making *+ like ? and ++ like a no-op). --- .../ql/lib/semmle/code/java/regex/RegexTreeView.qll | 13 +++++++++---- .../code/java/security/performance/ReDoSUtil.qll | 9 +++++++-- .../java/security/performance/RegExpTreeView.qll | 5 +++++ .../query-tests/security/CWE-730/ExpRedosTest.java | 11 +++++++++++ 4 files changed, 32 insertions(+), 6 deletions(-) diff --git a/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll b/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll index a7e3928f085..27074913a7e 100644 --- a/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll +++ b/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll @@ -253,7 +253,12 @@ class RegExpQuantifier extends RegExpTerm, TRegExpQuantifier { predicate mayRepeatForever() { may_repeat_forever = true } /** Gets the quantifier for this term. That is e.g "?" for "a?". */ - string getquantifier() { result = re.getText().substring(part_end, end) } + string getQuantifier() { result = re.getText().substring(part_end, end) } + + /** Holds if this is a possessive quantifier, e.g. a*+. */ + predicate isPossessive() { + exists(string q | q = this.getQuantifier() | q.length() > 1 and q.charAt(q.length() - 1) = "+") + } override string getPrimaryQLClass() { result = "RegExpQuantifier" } } @@ -275,7 +280,7 @@ class InfiniteRepetitionQuantifier extends RegExpQuantifier { * ``` */ class RegExpStar extends InfiniteRepetitionQuantifier { - RegExpStar() { this.getquantifier().charAt(0) = "*" } + RegExpStar() { this.getQuantifier().charAt(0) = "*" } override string getPrimaryQLClass() { result = "RegExpStar" } } @@ -290,7 +295,7 @@ class RegExpStar extends InfiniteRepetitionQuantifier { * ``` */ class RegExpPlus extends InfiniteRepetitionQuantifier { - RegExpPlus() { this.getquantifier().charAt(0) = "+" } + RegExpPlus() { this.getQuantifier().charAt(0) = "+" } override string getPrimaryQLClass() { result = "RegExpPlus" } } @@ -305,7 +310,7 @@ class RegExpPlus extends InfiniteRepetitionQuantifier { * ``` */ class RegExpOpt extends RegExpQuantifier { - RegExpOpt() { this.getquantifier().charAt(0) = "?" } + RegExpOpt() { this.getQuantifier().charAt(0) = "?" } override string getPrimaryQLClass() { result = "RegExpOpt" } } diff --git a/java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll b/java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll index 824eb9de7ae..f0e26580158 100644 --- a/java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll +++ b/java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll @@ -608,10 +608,15 @@ State after(RegExpTerm t) { or exists(RegExpGroup grp | t = grp.getAChild() | result = after(grp)) or - exists(EffectivelyStar star | t = star.getAChild() | result = before(star)) + exists(EffectivelyStar star | t = star.getAChild() | + not isPossessive(star) and + result = before(star) + ) or exists(EffectivelyPlus plus | t = plus.getAChild() | - result = before(plus) or + not isPossessive(plus) and + result = before(plus) + or result = after(plus) ) or diff --git a/java/ql/lib/semmle/code/java/security/performance/RegExpTreeView.qll b/java/ql/lib/semmle/code/java/security/performance/RegExpTreeView.qll index 608c03d006a..f59b1f43ca9 100644 --- a/java/ql/lib/semmle/code/java/security/performance/RegExpTreeView.qll +++ b/java/ql/lib/semmle/code/java/security/performance/RegExpTreeView.qll @@ -16,6 +16,11 @@ predicate isEscapeClass(RegExpTerm term, string clazz) { term.(RegExpNamedProperty).getBackslashEquivalent() = clazz } +/** + * Holds if `term` is a possessive quantifier, e.g. `a*+`. + */ +predicate isPossessive(RegExpQuantifier term) { term.isPossessive() } + /** * Holds if the regular expression should not be considered. * diff --git a/java/ql/test/query-tests/security/CWE-730/ExpRedosTest.java b/java/ql/test/query-tests/security/CWE-730/ExpRedosTest.java index a9fc19e5d50..e7e876cb696 100644 --- a/java/ql/test/query-tests/security/CWE-730/ExpRedosTest.java +++ b/java/ql/test/query-tests/security/CWE-730/ExpRedosTest.java @@ -418,6 +418,17 @@ class ExpRedosTest { "\\A(\\d|0)*x", // $ hasExpRedos "(\\d|0)*\\Z", // $ hasExpRedos "\\b(\\d|0)*x", // $ hasExpRedos + + // GOOD - possessive quantifiers don't backtrack + "(a*+)*+b", + "(a*)*+b", + "(a*+)*b", + + // BAD + "(a*)*b", // $ hasExpRedos + + // BAD - but not detected due to the way possessive quantifiers are approximated + "((aa|a*+)b)*c" // $ MISSING: hasExpRedos }; void test() { From 0a5268aeb4e12ebcb2d75d80a9b51ecb6c18009d Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Tue, 8 Mar 2022 16:57:36 +0000 Subject: [PATCH 45/91] Sync shared library changes across languages. --- .../semmle/javascript/security/performance/ReDoSUtil.qll | 9 +++++++-- .../security/performance/ReDoSUtilSpecific.qll | 6 ++++++ .../lib/semmle/python/security/performance/ReDoSUtil.qll | 9 +++++++-- .../python/security/performance/ReDoSUtilSpecific.qll | 6 ++++++ .../lib/codeql/ruby/security/performance/ReDoSUtil.qll | 9 +++++++-- .../ruby/security/performance/ReDoSUtilSpecific.qll | 6 ++++++ 6 files changed, 39 insertions(+), 6 deletions(-) diff --git a/javascript/ql/lib/semmle/javascript/security/performance/ReDoSUtil.qll b/javascript/ql/lib/semmle/javascript/security/performance/ReDoSUtil.qll index 6f695b5035b..aea089f0715 100644 --- a/javascript/ql/lib/semmle/javascript/security/performance/ReDoSUtil.qll +++ b/javascript/ql/lib/semmle/javascript/security/performance/ReDoSUtil.qll @@ -610,10 +610,15 @@ State after(RegExpTerm t) { or exists(RegExpGroup grp | t = grp.getAChild() | result = after(grp)) or - exists(EffectivelyStar star | t = star.getAChild() | result = before(star)) + exists(EffectivelyStar star | t = star.getAChild() | + not isPossessive(star) and + result = before(star) + ) or exists(EffectivelyPlus plus | t = plus.getAChild() | - result = before(plus) or + not isPossessive(plus) and + result = before(plus) + or result = after(plus) ) or diff --git a/javascript/ql/lib/semmle/javascript/security/performance/ReDoSUtilSpecific.qll b/javascript/ql/lib/semmle/javascript/security/performance/ReDoSUtilSpecific.qll index 4f247b0ce50..bc5ef32536c 100644 --- a/javascript/ql/lib/semmle/javascript/security/performance/ReDoSUtilSpecific.qll +++ b/javascript/ql/lib/semmle/javascript/security/performance/ReDoSUtilSpecific.qll @@ -12,6 +12,12 @@ predicate isEscapeClass(RegExpTerm term, string clazz) { exists(RegExpCharacterClassEscape escape | term = escape | escape.getValue() = clazz) } +/** + * Holds if `term` is a possessive quantifier. + * As javascript's regexes do not support possessive quantifiers, this never holds, but is used by the shared library. + */ +predicate isPossessive(RegExpQuantifier term) { none() } + /** * Holds if the regular expression should not be considered. * diff --git a/python/ql/lib/semmle/python/security/performance/ReDoSUtil.qll b/python/ql/lib/semmle/python/security/performance/ReDoSUtil.qll index 6f695b5035b..aea089f0715 100644 --- a/python/ql/lib/semmle/python/security/performance/ReDoSUtil.qll +++ b/python/ql/lib/semmle/python/security/performance/ReDoSUtil.qll @@ -610,10 +610,15 @@ State after(RegExpTerm t) { or exists(RegExpGroup grp | t = grp.getAChild() | result = after(grp)) or - exists(EffectivelyStar star | t = star.getAChild() | result = before(star)) + exists(EffectivelyStar star | t = star.getAChild() | + not isPossessive(star) and + result = before(star) + ) or exists(EffectivelyPlus plus | t = plus.getAChild() | - result = before(plus) or + not isPossessive(plus) and + result = before(plus) + or result = after(plus) ) or diff --git a/python/ql/lib/semmle/python/security/performance/ReDoSUtilSpecific.qll b/python/ql/lib/semmle/python/security/performance/ReDoSUtilSpecific.qll index 4193fd5a1e5..2db1579126d 100644 --- a/python/ql/lib/semmle/python/security/performance/ReDoSUtilSpecific.qll +++ b/python/ql/lib/semmle/python/security/performance/ReDoSUtilSpecific.qll @@ -13,6 +13,12 @@ predicate isEscapeClass(RegExpTerm term, string clazz) { exists(RegExpCharacterClassEscape escape | term = escape | escape.getValue() = clazz) } +/** + * Holds if `term` is a possessive quantifier. + * As python's regexes do not support possessive quantifiers, this never holds, but is used by the shared library. + */ +predicate isPossessive(RegExpQuantifier term) { none() } + /** * Holds if the regular expression should not be considered. * diff --git a/ruby/ql/lib/codeql/ruby/security/performance/ReDoSUtil.qll b/ruby/ql/lib/codeql/ruby/security/performance/ReDoSUtil.qll index 6f695b5035b..aea089f0715 100644 --- a/ruby/ql/lib/codeql/ruby/security/performance/ReDoSUtil.qll +++ b/ruby/ql/lib/codeql/ruby/security/performance/ReDoSUtil.qll @@ -610,10 +610,15 @@ State after(RegExpTerm t) { or exists(RegExpGroup grp | t = grp.getAChild() | result = after(grp)) or - exists(EffectivelyStar star | t = star.getAChild() | result = before(star)) + exists(EffectivelyStar star | t = star.getAChild() | + not isPossessive(star) and + result = before(star) + ) or exists(EffectivelyPlus plus | t = plus.getAChild() | - result = before(plus) or + not isPossessive(plus) and + result = before(plus) + or result = after(plus) ) or diff --git a/ruby/ql/lib/codeql/ruby/security/performance/ReDoSUtilSpecific.qll b/ruby/ql/lib/codeql/ruby/security/performance/ReDoSUtilSpecific.qll index de125f4a9db..2df67573a9c 100644 --- a/ruby/ql/lib/codeql/ruby/security/performance/ReDoSUtilSpecific.qll +++ b/ruby/ql/lib/codeql/ruby/security/performance/ReDoSUtilSpecific.qll @@ -33,6 +33,12 @@ predicate isExcluded(RegExpParent parent) { parent.(RegExpTerm).getRegExp().(AST::RegExpLiteral).hasFreeSpacingFlag() // exclude free-spacing mode regexes } +/** + * Holds if `term` is a possessive quantifier. + * Not currently implemented, but is used by the shared library. + */ +predicate isPossessive(RegExpQuantifier term) { none() } + /** * A module containing predicates for determining which flags a regular expression have. */ From 5555985ad6fb9e29d526956c99de2b07f3204a47 Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Wed, 9 Mar 2022 14:29:49 +0000 Subject: [PATCH 46/91] Distingush between whether or not a regex is matched against a full string Also some fixes and additional tests --- .../code/java/regex/RegexFlowConfigs.qll | 40 +++++++++++++++++-- .../code/java/regex/RegexFlowModels.qll | 27 +++++++++---- java/ql/lib/semmle/code/java/regex/regex.qll | 12 +++++- .../java/security/performance/ReDoSUtil.qll | 6 ++- .../security/performance/RegExpTreeView.qll | 10 +++++ .../security/CWE-730/PolyRedosTest.java | 38 ++++++++++++++++++ 6 files changed, 118 insertions(+), 15 deletions(-) diff --git a/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll b/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll index 948cc557169..86f1172ff0a 100644 --- a/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll +++ b/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll @@ -13,16 +13,48 @@ private class RegexCompileFlowConf extends DataFlow2::Configuration { override predicate isSource(DataFlow::Node node) { node.asExpr() instanceof StringLiteral } - override predicate isSink(DataFlow::Node node) { sinkNode(node, "regex-compile") } + override predicate isSink(DataFlow::Node node) { + sinkNode(node, ["regex-compile", "regex-compile-match", "regex-compile-find"]) + } } /** * Holds if `s` is used as a regex, with the mode `mode` (if known). * If regex mode is not known, `mode` will be `"None"`. */ -predicate usedAsRegex(StringLiteral s, string mode) { - any(RegexCompileFlowConf c).hasFlow(DataFlow2::exprNode(s), _) and - mode = "None" // TODO: proper mode detection +predicate usedAsRegex(StringLiteral s, string mode, boolean match_full_string) { + exists(DataFlow::Node sink | + any(RegexCompileFlowConf c).hasFlow(DataFlow2::exprNode(s), sink) and + mode = "None" and // TODO: proper mode detection + (if matchesFullString(sink) then match_full_string = true else match_full_string = false) + ) +} + +/** + * Holds if the regex that flows to `sink` is used to match against a full string, + * as though it was implicitly surrounded by ^ and $. + */ +private predicate matchesFullString(DataFlow::Node sink) { + sinkNode(sink, "regex-compile-match") + or + exists(DataFlow::Node matchSource, RegexCompileToMatchConf conf | + matchSource.asExpr().(MethodAccess).getAnArgument() = sink.asExpr() and + conf.hasFlow(matchSource, _) + ) +} + +private class RegexCompileToMatchConf extends DataFlow2::Configuration { + RegexCompileToMatchConf() { this = "RegexCompileToMatchConfig" } + + override predicate isSource(DataFlow::Node node) { sourceNode(node, "regex-compile") } + + override predicate isSink(DataFlow::Node node) { sinkNode(node, "regex-match") } + + override predicate isAdditionalFlowStep(DataFlow::Node node1, DataFlow::Node node2) { + exists(MethodAccess ma | node2.asExpr() = ma and node1.asExpr() = ma.getQualifier() | + ma.getMethod().hasQualifiedName("java.util.regex", "Pattern", "matcher") + ) + } } /** diff --git a/java/ql/lib/semmle/code/java/regex/RegexFlowModels.qll b/java/ql/lib/semmle/code/java/regex/RegexFlowModels.qll index c6ee2ace865..fd0858639c4 100644 --- a/java/ql/lib/semmle/code/java/regex/RegexFlowModels.qll +++ b/java/ql/lib/semmle/code/java/regex/RegexFlowModels.qll @@ -3,6 +3,16 @@ import java import semmle.code.java.dataflow.ExternalFlow +private class RegexSourceCsv extends SourceModelCsv { + override predicate row(string row) { + row = + [ + //"namespace;type;subtypes;name;signature;ext;output;kind" + "java.util.regex;Pattern;false;compile;(String);;ReturnValue;regex-compile", + ] + } +} + private class RegexSinkCsv extends SinkModelCsv { override predicate row(string row) { row = @@ -10,13 +20,16 @@ private class RegexSinkCsv extends SinkModelCsv { //"namespace;type;subtypes;name;signature;ext;input;kind" "java.util.regex;Pattern;false;compile;(String);;Argument[0];regex-compile", "java.util.regex;Pattern;false;compile;(String,int);;Argument[0];regex-compile", - "java.util.regex;Pattern;false;matches;(String,CharSequence);;Argument[0];regex-compile", - "java.util;String;false;matches;(String);;Argument[0];regex-compile", - "java.util;String;false;split;(String);;Argument[0];regex-compile", - "java.util;String;false;split;(String,int);;Argument[0];regex-compile", - "java.util;String;false;replaceAll;(String,String);;Argument[0];regex-compile", - "java.util;String;false;replaceFirst;(String,String);;Argument[0];regex-compile", - "com.google.common.base;Splitter;false;onPattern;(String);;Argument[0];regex-compile" + "java.util.regex;Pattern;false;matches;(String,CharSequence);;Argument[0];regex-compile-match", + "java.lang;String;false;matches;(String);;Argument[0];regex-compile-match", + "java.lang;String;false;split;(String);;Argument[0];regex-compile-find", + "java.lang;String;false;split;(String,int);;Argument[0];regex-compile-find", + "java.lang;String;false;replaceAll;(String,String);;Argument[0];regex-compile-find", + "java.lang;String;false;replaceFirst;(String,String);;Argument[0];regex-compile-find", + "com.google.common.base;Splitter;false;onPattern;(String);;Argument[0];regex-compile-find", + // regex-match sinks + "java.util.regex;Pattern;false;asMatchPredicate;();;Argument[-1];regex-match", + "java.util.regex;Matcher;false;matches;();;Argument[-1];regex-match", ] } } diff --git a/java/ql/lib/semmle/code/java/regex/regex.qll b/java/ql/lib/semmle/code/java/regex/regex.qll index 2cd3b9a4035..9a7da15999e 100644 --- a/java/ql/lib/semmle/code/java/regex/regex.qll +++ b/java/ql/lib/semmle/code/java/regex/regex.qll @@ -892,7 +892,9 @@ abstract class RegexString extends StringLiteral { /** A string literal used as a regular expression */ class Regex extends RegexString { - Regex() { usedAsRegex(this, _) } + boolean matches_full_string; + + Regex() { usedAsRegex(this, _, matches_full_string) } /** * Gets a mode (if any) of this regular expression. Can be any of: @@ -906,8 +908,14 @@ class Regex extends RegexString { */ string getAMode() { result != "None" and - usedAsRegex(this, result) + usedAsRegex(this, result, _) or result = this.getModeFromPrefix() } + + /** + * Holds if this regex is used to match against a full string, + * as though it was implicitly surrounded by ^ and $. + */ + predicate matchesFullString() { matches_full_string = true } } diff --git a/java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll b/java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll index f0e26580158..a05261611c7 100644 --- a/java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll +++ b/java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll @@ -622,7 +622,9 @@ State after(RegExpTerm t) { or exists(EffectivelyQuestion opt | t = opt.getAChild() | result = after(opt)) or - exists(RegExpRoot root | t = root | result = AcceptAnySuffix(root)) + exists(RegExpRoot root | t = root | + if matchesAnySuffix(root) then result = AcceptAnySuffix(root) else result = Accept(root) + ) } /** @@ -693,7 +695,7 @@ predicate delta(State q1, EdgeLabel lbl, State q2) { lbl = Epsilon() and q2 = Accept(root) ) or - exists(RegExpRoot root | q1 = Match(root, 0) | lbl = Any() and q2 = q1) + exists(RegExpRoot root | q1 = Match(root, 0) | matchesAnyPrefix(root) and lbl = Any() and q2 = q1) or exists(RegExpDollar dollar | q1 = before(dollar) | lbl = Epsilon() and q2 = Accept(getRoot(dollar)) diff --git a/java/ql/lib/semmle/code/java/security/performance/RegExpTreeView.qll b/java/ql/lib/semmle/code/java/security/performance/RegExpTreeView.qll index f59b1f43ca9..daef79ceb1e 100644 --- a/java/ql/lib/semmle/code/java/security/performance/RegExpTreeView.qll +++ b/java/ql/lib/semmle/code/java/security/performance/RegExpTreeView.qll @@ -21,6 +21,16 @@ predicate isEscapeClass(RegExpTerm term, string clazz) { */ predicate isPossessive(RegExpQuantifier term) { term.isPossessive() } +/** + * Holds if the regex that `term` is part of is used in a way that ignores any leading prefix of the input it's matched against. + */ +predicate matchesAnyPrefix(RegExpTerm term) { not term.getRegex().matchesFullString() } + +/** + * Holds if the regex that `term` is part of is used in a way that ignores any trailing suffix of the input it's matched against. + */ +predicate matchesAnySuffix(RegExpTerm term) { not term.getRegex().matchesFullString() } + /** * Holds if the regular expression should not be considered. * diff --git a/java/ql/test/query-tests/security/CWE-730/PolyRedosTest.java b/java/ql/test/query-tests/security/CWE-730/PolyRedosTest.java index e825e1ad2db..dd6a77b5be0 100644 --- a/java/ql/test/query-tests/security/CWE-730/PolyRedosTest.java +++ b/java/ql/test/query-tests/security/CWE-730/PolyRedosTest.java @@ -34,4 +34,42 @@ class PolyRedosTest { Splitter.on(";").withKeyValueSeparator(Splitter.onPattern(reg)).split(tainted); // $ hasPolyRedos } + + void test2(HttpServletRequest request) { + String tainted = request.getParameter("inp"); + + Pattern p1 = Pattern.compile(".*a"); + Pattern p2 = Pattern.compile(".*b"); + + p1.matcher(tainted).matches(); + p2.matcher(tainted).find(); // $ hasPolyRedos + } + + void test3(HttpServletRequest request) { + String tainted = request.getParameter("inp"); + + Pattern p1 = Pattern.compile("ab*b*"); + Pattern p2 = Pattern.compile("cd*d*"); + + p1.matcher(tainted).matches(); // $ hasPolyRedos + p2.matcher(tainted).find(); + } + + void test4(HttpServletRequest request) { + String tainted = request.getParameter("inp"); + + tainted.matches(".*a"); + tainted.replaceAll(".*b", "c"); // $ hasPolyRedos + } + + static Pattern p3 = Pattern.compile(".*a"); + static Pattern p4 = Pattern.compile(".*b"); + + + void test5(HttpServletRequest request) { + String tainted = request.getParameter("inp"); + + p3.asMatchPredicate().test(tainted); + p4.asPredicate().test(tainted); // $ hasPolyRedos + } } \ No newline at end of file From c1290d9e2bae7a4f7b082c7249854ca5ddba66e9 Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Wed, 9 Mar 2022 14:34:50 +0000 Subject: [PATCH 47/91] Sync shared redos library files. --- .../javascript/security/performance/ReDoSUtil.qll | 6 ++++-- .../security/performance/ReDoSUtilSpecific.qll | 12 ++++++++++++ .../semmle/python/security/performance/ReDoSUtil.qll | 6 ++++-- .../security/performance/ReDoSUtilSpecific.qll | 12 ++++++++++++ .../codeql/ruby/security/performance/ReDoSUtil.qll | 6 ++++-- .../ruby/security/performance/ReDoSUtilSpecific.qll | 12 ++++++++++++ 6 files changed, 48 insertions(+), 6 deletions(-) diff --git a/javascript/ql/lib/semmle/javascript/security/performance/ReDoSUtil.qll b/javascript/ql/lib/semmle/javascript/security/performance/ReDoSUtil.qll index aea089f0715..8aa348bf62f 100644 --- a/javascript/ql/lib/semmle/javascript/security/performance/ReDoSUtil.qll +++ b/javascript/ql/lib/semmle/javascript/security/performance/ReDoSUtil.qll @@ -624,7 +624,9 @@ State after(RegExpTerm t) { or exists(EffectivelyQuestion opt | t = opt.getAChild() | result = after(opt)) or - exists(RegExpRoot root | t = root | result = AcceptAnySuffix(root)) + exists(RegExpRoot root | t = root | + if matchesAnySuffix(root) then result = AcceptAnySuffix(root) else result = Accept(root) + ) } /** @@ -695,7 +697,7 @@ predicate delta(State q1, EdgeLabel lbl, State q2) { lbl = Epsilon() and q2 = Accept(root) ) or - exists(RegExpRoot root | q1 = Match(root, 0) | lbl = Any() and q2 = q1) + exists(RegExpRoot root | q1 = Match(root, 0) | matchesAnyPrefix(root) and lbl = Any() and q2 = q1) or exists(RegExpDollar dollar | q1 = before(dollar) | lbl = Epsilon() and q2 = Accept(getRoot(dollar)) diff --git a/javascript/ql/lib/semmle/javascript/security/performance/ReDoSUtilSpecific.qll b/javascript/ql/lib/semmle/javascript/security/performance/ReDoSUtilSpecific.qll index bc5ef32536c..d363e25d83d 100644 --- a/javascript/ql/lib/semmle/javascript/security/performance/ReDoSUtilSpecific.qll +++ b/javascript/ql/lib/semmle/javascript/security/performance/ReDoSUtilSpecific.qll @@ -18,6 +18,18 @@ predicate isEscapeClass(RegExpTerm term, string clazz) { */ predicate isPossessive(RegExpQuantifier term) { none() } +/** + * Holds if the regex that `term` is part of is used in a way that ignores any leading prefix of the input it's matched against. + * Not yet implemented for Javascript. + */ +predicate matchesAnyPrefix(RegExpTerm term) { any() } + +/** + * Holds if the regex that `term` is part of is used in a way that ignores any trailing suffix of the input it's matched against. + * Not yet implemented for Javascript. + */ +predicate matchesAnySuffix(RegExpTerm term) { any() } + /** * Holds if the regular expression should not be considered. * diff --git a/python/ql/lib/semmle/python/security/performance/ReDoSUtil.qll b/python/ql/lib/semmle/python/security/performance/ReDoSUtil.qll index aea089f0715..8aa348bf62f 100644 --- a/python/ql/lib/semmle/python/security/performance/ReDoSUtil.qll +++ b/python/ql/lib/semmle/python/security/performance/ReDoSUtil.qll @@ -624,7 +624,9 @@ State after(RegExpTerm t) { or exists(EffectivelyQuestion opt | t = opt.getAChild() | result = after(opt)) or - exists(RegExpRoot root | t = root | result = AcceptAnySuffix(root)) + exists(RegExpRoot root | t = root | + if matchesAnySuffix(root) then result = AcceptAnySuffix(root) else result = Accept(root) + ) } /** @@ -695,7 +697,7 @@ predicate delta(State q1, EdgeLabel lbl, State q2) { lbl = Epsilon() and q2 = Accept(root) ) or - exists(RegExpRoot root | q1 = Match(root, 0) | lbl = Any() and q2 = q1) + exists(RegExpRoot root | q1 = Match(root, 0) | matchesAnyPrefix(root) and lbl = Any() and q2 = q1) or exists(RegExpDollar dollar | q1 = before(dollar) | lbl = Epsilon() and q2 = Accept(getRoot(dollar)) diff --git a/python/ql/lib/semmle/python/security/performance/ReDoSUtilSpecific.qll b/python/ql/lib/semmle/python/security/performance/ReDoSUtilSpecific.qll index 2db1579126d..bc495f88c3c 100644 --- a/python/ql/lib/semmle/python/security/performance/ReDoSUtilSpecific.qll +++ b/python/ql/lib/semmle/python/security/performance/ReDoSUtilSpecific.qll @@ -19,6 +19,18 @@ predicate isEscapeClass(RegExpTerm term, string clazz) { */ predicate isPossessive(RegExpQuantifier term) { none() } +/** + * Holds if the regex that `term` is part of is used in a way that ignores any leading prefix of the input it's matched against. + * Not yet implemented for Python. + */ +predicate matchesAnyPrefix(RegExpTerm term) { any() } + +/** + * Holds if the regex that `term` is part of is used in a way that ignores any trailing suffix of the input it's matched against. + * Not yet implemented for Python. + */ +predicate matchesAnySuffix(RegExpTerm term) { any() } + /** * Holds if the regular expression should not be considered. * diff --git a/ruby/ql/lib/codeql/ruby/security/performance/ReDoSUtil.qll b/ruby/ql/lib/codeql/ruby/security/performance/ReDoSUtil.qll index aea089f0715..8aa348bf62f 100644 --- a/ruby/ql/lib/codeql/ruby/security/performance/ReDoSUtil.qll +++ b/ruby/ql/lib/codeql/ruby/security/performance/ReDoSUtil.qll @@ -624,7 +624,9 @@ State after(RegExpTerm t) { or exists(EffectivelyQuestion opt | t = opt.getAChild() | result = after(opt)) or - exists(RegExpRoot root | t = root | result = AcceptAnySuffix(root)) + exists(RegExpRoot root | t = root | + if matchesAnySuffix(root) then result = AcceptAnySuffix(root) else result = Accept(root) + ) } /** @@ -695,7 +697,7 @@ predicate delta(State q1, EdgeLabel lbl, State q2) { lbl = Epsilon() and q2 = Accept(root) ) or - exists(RegExpRoot root | q1 = Match(root, 0) | lbl = Any() and q2 = q1) + exists(RegExpRoot root | q1 = Match(root, 0) | matchesAnyPrefix(root) and lbl = Any() and q2 = q1) or exists(RegExpDollar dollar | q1 = before(dollar) | lbl = Epsilon() and q2 = Accept(getRoot(dollar)) diff --git a/ruby/ql/lib/codeql/ruby/security/performance/ReDoSUtilSpecific.qll b/ruby/ql/lib/codeql/ruby/security/performance/ReDoSUtilSpecific.qll index 2df67573a9c..8d6b14607e0 100644 --- a/ruby/ql/lib/codeql/ruby/security/performance/ReDoSUtilSpecific.qll +++ b/ruby/ql/lib/codeql/ruby/security/performance/ReDoSUtilSpecific.qll @@ -39,6 +39,18 @@ predicate isExcluded(RegExpParent parent) { */ predicate isPossessive(RegExpQuantifier term) { none() } +/** + * Holds if the regex that `term` is part of is used in a way that ignores any leading prefix of the input it's matched against. + * Not yet implemented for Ruby. + */ +predicate matchesAnyPrefix(RegExpTerm term) { any() } + +/** + * Holds if the regex that `term` is part of is used in a way that ignores any trailing suffix of the input it's matched against. + * Not yet implemented for Ruby. + */ +predicate matchesAnySuffix(RegExpTerm term) { any() } + /** * A module containing predicates for determining which flags a regular expression have. */ From 6794268a3c3deb6b350315ec665ea1196f78c4ca Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Wed, 16 Mar 2022 13:30:06 +0000 Subject: [PATCH 48/91] Split PolynomialRedos definition into a library to avoid duplication in the tests --- .../performance/PolynomialReDosQuery.qll | 34 ++++++++++++++++++ .../Security/CWE/CWE-730/PolynomialReDoS.ql | 33 +++-------------- .../security/CWE-730/PolynomialReDoS.ql | 35 +++---------------- 3 files changed, 43 insertions(+), 59 deletions(-) create mode 100644 java/ql/lib/semmle/code/java/security/performance/PolynomialReDosQuery.qll diff --git a/java/ql/lib/semmle/code/java/security/performance/PolynomialReDosQuery.qll b/java/ql/lib/semmle/code/java/security/performance/PolynomialReDosQuery.qll new file mode 100644 index 00000000000..f13e8ffc8e7 --- /dev/null +++ b/java/ql/lib/semmle/code/java/security/performance/PolynomialReDosQuery.qll @@ -0,0 +1,34 @@ +/** Definitions and configurations for the Polynomial ReDos query */ + +import semmle.code.java.security.performance.SuperlinearBackTracking +import semmle.code.java.dataflow.DataFlow +import semmle.code.java.regex.RegexTreeView +import semmle.code.java.regex.RegexFlowConfigs +import semmle.code.java.dataflow.FlowSources + +/** A sink for polynomial redos queries, where a regex is matched. */ +class PolynomialRedosSink extends DataFlow::Node { + RegExpLiteral reg; + + PolynomialRedosSink() { regexMatchedAgainst(reg.getRegex(), this.asExpr()) } + + /** Gets the regex that is matched against this node. */ + RegExpTerm getRegExp() { result.getParent() = reg } +} + +/** A configuration for Polynomial ReDoS queries. */ +class PolynomialRedosConfig extends TaintTracking::Configuration { + PolynomialRedosConfig() { this = "PolynomialRedosConfig" } + + override predicate isSource(DataFlow::Node src) { src instanceof RemoteFlowSource } + + override predicate isSink(DataFlow::Node sink) { sink instanceof PolynomialRedosSink } +} + +/** Holds if there is flow from `source` to `sink` that is matched against the regexp term `regexp` that is vulnerable to Polynomial ReDoS. */ +predicate hasPolynomialReDosResult( + DataFlow::PathNode source, DataFlow::PathNode sink, PolynomialBackTrackingTerm regexp +) { + any(PolynomialRedosConfig config).hasFlowPath(source, sink) and + regexp.getRootTerm() = sink.getNode().(PolynomialRedosSink).getRegExp() +} diff --git a/java/ql/src/Security/CWE/CWE-730/PolynomialReDoS.ql b/java/ql/src/Security/CWE/CWE-730/PolynomialReDoS.ql index 563be6febb0..b37f51aec9a 100644 --- a/java/ql/src/Security/CWE/CWE-730/PolynomialReDoS.ql +++ b/java/ql/src/Security/CWE/CWE-730/PolynomialReDoS.ql @@ -12,37 +12,12 @@ */ import java -import semmle.code.java.security.performance.SuperlinearBackTracking -import semmle.code.java.dataflow.DataFlow -import semmle.code.java.regex.RegexTreeView -import semmle.code.java.regex.RegexFlowConfigs -import semmle.code.java.dataflow.FlowSources +import semmle.code.java.security.performance.PolynomialReDosQuery import DataFlow::PathGraph -class PolynomialRedosSink extends DataFlow::Node { - RegExpLiteral reg; - - PolynomialRedosSink() { regexMatchedAgainst(reg.getRegex(), this.asExpr()) } - - RegExpTerm getRegExp() { result.getParent() = reg } -} - -class PolynomialRedosConfig extends TaintTracking::Configuration { - PolynomialRedosConfig() { this = "PolynomialRedosConfig" } - - override predicate isSource(DataFlow::Node src) { src instanceof RemoteFlowSource } - - override predicate isSink(DataFlow::Node sink) { sink instanceof PolynomialRedosSink } -} - -from - PolynomialRedosConfig config, DataFlow::PathNode source, DataFlow::PathNode sink, - PolynomialRedosSink sinkNode, PolynomialBackTrackingTerm regexp -where - config.hasFlowPath(source, sink) and - sinkNode = sink.getNode() and - regexp.getRootTerm() = sinkNode.getRegExp() -select sinkNode, source, sink, +from DataFlow::PathNode source, DataFlow::PathNode sink, PolynomialBackTrackingTerm regexp +where hasPolynomialReDosResult(source, sink, regexp) +select sink, source, sink, "This $@ that depends on $@ may run slow on strings " + regexp.getPrefixMessage() + "with many repetitions of '" + regexp.getPumpString() + "'.", regexp, "regular expression", source.getNode(), "a user-provided value" diff --git a/java/ql/test/query-tests/security/CWE-730/PolynomialReDoS.ql b/java/ql/test/query-tests/security/CWE-730/PolynomialReDoS.ql index 372f1792083..698f658d508 100644 --- a/java/ql/test/query-tests/security/CWE-730/PolynomialReDoS.ql +++ b/java/ql/test/query-tests/security/CWE-730/PolynomialReDoS.ql @@ -1,26 +1,6 @@ import java import TestUtilities.InlineExpectationsTest -import semmle.code.java.security.performance.SuperlinearBackTracking -import semmle.code.java.dataflow.DataFlow -import semmle.code.java.regex.RegexTreeView -import semmle.code.java.regex.RegexFlowConfigs -import semmle.code.java.dataflow.FlowSources - -class PolynomialRedosSink extends DataFlow::Node { - RegExpLiteral reg; - - PolynomialRedosSink() { regexMatchedAgainst(reg.getRegex(), this.asExpr()) } - - RegExpTerm getRegExp() { result.getParent() = reg } -} - -class PolynomialRedosConfig extends TaintTracking::Configuration { - PolynomialRedosConfig() { this = "PolynomialRedosConfig" } - - override predicate isSource(DataFlow::Node src) { src instanceof RemoteFlowSource } - - override predicate isSink(DataFlow::Node sink) { sink instanceof PolynomialRedosSink } -} +import semmle.code.java.security.performance.PolynomialReDosQuery class HasPolyRedos extends InlineExpectationsTest { HasPolyRedos() { this = "HasPolyRedos" } @@ -29,15 +9,10 @@ class HasPolyRedos extends InlineExpectationsTest { override predicate hasActualResult(Location location, string element, string tag, string value) { tag = "hasPolyRedos" and - exists( - PolynomialRedosConfig config, DataFlow::PathNode source, DataFlow::PathNode sink, - PolynomialRedosSink sinkNode, PolynomialBackTrackingTerm regexp - | - config.hasFlowPath(source, sink) and - sinkNode = sink.getNode() and - regexp.getRootTerm() = sinkNode.getRegExp() and - location = sinkNode.getLocation() and - element = sinkNode.toString() and + exists(DataFlow::PathNode source, DataFlow::PathNode sink, PolynomialBackTrackingTerm regexp | + hasPolynomialReDosResult(source, sink, regexp) and + location = sink.getNode().getLocation() and + element = sink.getNode().toString() and value = "" ) } From 04edc10f1ee102a6aa5f6af0ffea6dc19e1cdb9c Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Wed, 16 Mar 2022 14:22:50 +0000 Subject: [PATCH 49/91] Exclude regexes from test code --- java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll b/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll index 86f1172ff0a..25d984c39ab 100644 --- a/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll +++ b/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll @@ -7,6 +7,7 @@ private import semmle.code.java.dataflow.DataFlow private import semmle.code.java.dataflow.DataFlow2 private import semmle.code.java.dataflow.DataFlow3 private import RegexFlowModels +private import semmle.code.java.security.SecurityTests private class RegexCompileFlowConf extends DataFlow2::Configuration { RegexCompileFlowConf() { this = "RegexCompileFlowConfig" } @@ -207,6 +208,10 @@ private class RegexMatchFlowConf extends DataFlow2::Configuration { override predicate isAdditionalFlowStep(DataFlow::Node node1, DataFlow::Node node2) { any(RegexAdditionalFlowStep s).step(node1, node2) } + + override predicate isBarrier(DataFlow::Node node) { + node.getEnclosingCallable().getDeclaringType() instanceof NonSecurityTestClass + } } /** From 1605d36ddff5d5723012fa99d2e8c9af6d02540b Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Wed, 16 Mar 2022 15:59:41 +0000 Subject: [PATCH 50/91] Refine polynomial redos sources to exclude length limited methods --- .../performance/PolynomialReDosQuery.qll | 27 +++++++++++++++++-- .../Security/CWE/CWE-730/PolynomialReDoS.ql | 4 +-- .../security/CWE-730/PolyRedosTest.java | 9 +++++++ .../security/CWE-730/PolynomialReDoS.ql | 4 +-- 4 files changed, 38 insertions(+), 6 deletions(-) diff --git a/java/ql/lib/semmle/code/java/security/performance/PolynomialReDosQuery.qll b/java/ql/lib/semmle/code/java/security/performance/PolynomialReDosQuery.qll index f13e8ffc8e7..1cd628e420a 100644 --- a/java/ql/lib/semmle/code/java/security/performance/PolynomialReDosQuery.qll +++ b/java/ql/lib/semmle/code/java/security/performance/PolynomialReDosQuery.qll @@ -1,4 +1,4 @@ -/** Definitions and configurations for the Polynomial ReDos query */ +/** Definitions and configurations for the Polynomial ReDoS query */ import semmle.code.java.security.performance.SuperlinearBackTracking import semmle.code.java.dataflow.DataFlow @@ -16,6 +16,22 @@ class PolynomialRedosSink extends DataFlow::Node { RegExpTerm getRegExp() { result.getParent() = reg } } +/** + * A method whose result typically has a limited length, + * such as HTTP headers, and values derrived from them. + */ +private class LengthRestrictedMethod extends Method { + LengthRestrictedMethod() { + this.getName().toLowerCase().matches(["%header%", "%requesturi%", "%requesturl%", "%cookie%"]) + or + this.getDeclaringType().getName().toLowerCase().matches("%cookie%") and + this.getName().matches("get%") + or + this.getDeclaringType().getName().toLowerCase().matches("%request%") and + this.getName().toLowerCase().matches(["%get%path%", "get%user%", "%querystring%"]) + } +} + /** A configuration for Polynomial ReDoS queries. */ class PolynomialRedosConfig extends TaintTracking::Configuration { PolynomialRedosConfig() { this = "PolynomialRedosConfig" } @@ -23,10 +39,17 @@ class PolynomialRedosConfig extends TaintTracking::Configuration { override predicate isSource(DataFlow::Node src) { src instanceof RemoteFlowSource } override predicate isSink(DataFlow::Node sink) { sink instanceof PolynomialRedosSink } + + override predicate isSanitizer(DataFlow::Node node) { + super.isSanitizer(node) or + node.getType() instanceof PrimitiveType or + node.getType() instanceof BoxedType or + node.asExpr().(MethodAccess).getMethod() instanceof LengthRestrictedMethod + } } /** Holds if there is flow from `source` to `sink` that is matched against the regexp term `regexp` that is vulnerable to Polynomial ReDoS. */ -predicate hasPolynomialReDosResult( +predicate hasPolynomialReDoSResult( DataFlow::PathNode source, DataFlow::PathNode sink, PolynomialBackTrackingTerm regexp ) { any(PolynomialRedosConfig config).hasFlowPath(source, sink) and diff --git a/java/ql/src/Security/CWE/CWE-730/PolynomialReDoS.ql b/java/ql/src/Security/CWE/CWE-730/PolynomialReDoS.ql index b37f51aec9a..e1907b39414 100644 --- a/java/ql/src/Security/CWE/CWE-730/PolynomialReDoS.ql +++ b/java/ql/src/Security/CWE/CWE-730/PolynomialReDoS.ql @@ -12,11 +12,11 @@ */ import java -import semmle.code.java.security.performance.PolynomialReDosQuery +import semmle.code.java.security.performance.PolynomialReDoSQuery import DataFlow::PathGraph from DataFlow::PathNode source, DataFlow::PathNode sink, PolynomialBackTrackingTerm regexp -where hasPolynomialReDosResult(source, sink, regexp) +where hasPolynomialReDoSResult(source, sink, regexp) select sink, source, sink, "This $@ that depends on $@ may run slow on strings " + regexp.getPrefixMessage() + "with many repetitions of '" + regexp.getPumpString() + "'.", regexp, "regular expression", diff --git a/java/ql/test/query-tests/security/CWE-730/PolyRedosTest.java b/java/ql/test/query-tests/security/CWE-730/PolyRedosTest.java index dd6a77b5be0..44931190460 100644 --- a/java/ql/test/query-tests/security/CWE-730/PolyRedosTest.java +++ b/java/ql/test/query-tests/security/CWE-730/PolyRedosTest.java @@ -72,4 +72,13 @@ class PolyRedosTest { p3.asMatchPredicate().test(tainted); p4.asPredicate().test(tainted); // $ hasPolyRedos } + + void test6(HttpServletRequest request) { + Pattern p = Pattern.compile("^a*a*$"); + + p.matcher(request.getParameter("inp")).matches(); // $ hasPolyRedos + p.matcher(request.getHeader("If-None-Match")).matches(); + p.matcher(request.getRequestURI()).matches(); + p.matcher(request.getCookies()[0].getName()).matches(); + } } \ No newline at end of file diff --git a/java/ql/test/query-tests/security/CWE-730/PolynomialReDoS.ql b/java/ql/test/query-tests/security/CWE-730/PolynomialReDoS.ql index 698f658d508..e5fb58d4794 100644 --- a/java/ql/test/query-tests/security/CWE-730/PolynomialReDoS.ql +++ b/java/ql/test/query-tests/security/CWE-730/PolynomialReDoS.ql @@ -1,6 +1,6 @@ import java import TestUtilities.InlineExpectationsTest -import semmle.code.java.security.performance.PolynomialReDosQuery +import semmle.code.java.security.performance.PolynomialReDoSQuery class HasPolyRedos extends InlineExpectationsTest { HasPolyRedos() { this = "HasPolyRedos" } @@ -10,7 +10,7 @@ class HasPolyRedos extends InlineExpectationsTest { override predicate hasActualResult(Location location, string element, string tag, string value) { tag = "hasPolyRedos" and exists(DataFlow::PathNode source, DataFlow::PathNode sink, PolynomialBackTrackingTerm regexp | - hasPolynomialReDosResult(source, sink, regexp) and + hasPolynomialReDoSResult(source, sink, regexp) and location = sink.getNode().getLocation() and element = sink.getNode().toString() and value = "" From 375ded4edeb30fd018da1b4d9e3e740a5b24475b Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Wed, 16 Mar 2022 16:04:48 +0000 Subject: [PATCH 51/91] Move check to exlude test cases so that it also covers exponential redos --- java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll b/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll index 25d984c39ab..adcda1e2516 100644 --- a/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll +++ b/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll @@ -17,6 +17,10 @@ private class RegexCompileFlowConf extends DataFlow2::Configuration { override predicate isSink(DataFlow::Node node) { sinkNode(node, ["regex-compile", "regex-compile-match", "regex-compile-find"]) } + + override predicate isBarrier(DataFlow::Node node) { + node.getEnclosingCallable().getDeclaringType() instanceof NonSecurityTestClass + } } /** @@ -208,10 +212,6 @@ private class RegexMatchFlowConf extends DataFlow2::Configuration { override predicate isAdditionalFlowStep(DataFlow::Node node1, DataFlow::Node node2) { any(RegexAdditionalFlowStep s).step(node1, node2) } - - override predicate isBarrier(DataFlow::Node node) { - node.getEnclosingCallable().getDeclaringType() instanceof NonSecurityTestClass - } } /** From 3d65a9cafc407a0e26c2a04aa794322ba5110dfb Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Wed, 16 Mar 2022 17:03:27 +0000 Subject: [PATCH 52/91] Update shared files --- .../java/security/performance/ReDoSUtil.qll | 38 ++++++++++--------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll b/java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll index a05261611c7..a7d843ac7f8 100644 --- a/java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll +++ b/java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll @@ -119,18 +119,18 @@ class EmptyPositiveSubPatttern extends RegExpSubPattern { * whose root node is not a disjunction. */ class RegExpRoot extends RegExpTerm { - RegExpParent parent; - RegExpRoot() { - exists(RegExpAlt alt | - alt.isRootTerm() and - this = alt.getAChild() and - parent = alt.getParent() + exists(RegExpParent parent | + exists(RegExpAlt alt | + alt.isRootTerm() and + this = alt.getAChild() and + parent = alt.getParent() + ) + or + this.isRootTerm() and + not this instanceof RegExpAlt and + parent = this.getParent() ) - or - this.isRootTerm() and - not this instanceof RegExpAlt and - parent = this.getParent() } /** @@ -466,13 +466,14 @@ private module CharacterClasses { * An implementation of `CharacterClass` for \d, \s, and \w. */ private class PositiveCharacterClassEscape extends CharacterClass { - RegExpTerm cc; string charClass; PositiveCharacterClassEscape() { - isEscapeClass(cc, charClass) and - this = getCanonicalCharClass(cc) and - charClass = ["d", "s", "w"] + exists(RegExpTerm cc | + isEscapeClass(cc, charClass) and + this = getCanonicalCharClass(cc) and + charClass = ["d", "s", "w"] + ) } override string getARelevantChar() { @@ -504,13 +505,14 @@ private module CharacterClasses { * An implementation of `CharacterClass` for \D, \S, and \W. */ private class NegativeCharacterClassEscape extends CharacterClass { - RegExpTerm cc; string charClass; NegativeCharacterClassEscape() { - isEscapeClass(cc, charClass) and - this = getCanonicalCharClass(cc) and - charClass = ["D", "S", "W"] + exists(RegExpTerm cc | + isEscapeClass(cc, charClass) and + this = getCanonicalCharClass(cc) and + charClass = ["D", "S", "W"] + ) } override string getARelevantChar() { From 522a8aff6fea78329b376a2a394f5c162f456c6f Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Thu, 17 Mar 2022 13:08:00 +0000 Subject: [PATCH 53/91] Fix filename case --- .../{PolynomialReDosQuery.qll => PolynomialReDoSQuery.qll} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename java/ql/lib/semmle/code/java/security/performance/{PolynomialReDosQuery.qll => PolynomialReDoSQuery.qll} (100%) diff --git a/java/ql/lib/semmle/code/java/security/performance/PolynomialReDosQuery.qll b/java/ql/lib/semmle/code/java/security/performance/PolynomialReDoSQuery.qll similarity index 100% rename from java/ql/lib/semmle/code/java/security/performance/PolynomialReDosQuery.qll rename to java/ql/lib/semmle/code/java/security/performance/PolynomialReDoSQuery.qll From 0f606d987dd4437dd8ab707a6628cb290f303049 Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Wed, 23 Mar 2022 12:28:08 +0000 Subject: [PATCH 54/91] Remove redundant `super` call. Co-authored-by: Tony Torralba --- .../code/java/security/performance/PolynomialReDoSQuery.qll | 1 - 1 file changed, 1 deletion(-) diff --git a/java/ql/lib/semmle/code/java/security/performance/PolynomialReDoSQuery.qll b/java/ql/lib/semmle/code/java/security/performance/PolynomialReDoSQuery.qll index 1cd628e420a..2a33e15c74a 100644 --- a/java/ql/lib/semmle/code/java/security/performance/PolynomialReDoSQuery.qll +++ b/java/ql/lib/semmle/code/java/security/performance/PolynomialReDoSQuery.qll @@ -41,7 +41,6 @@ class PolynomialRedosConfig extends TaintTracking::Configuration { override predicate isSink(DataFlow::Node sink) { sink instanceof PolynomialRedosSink } override predicate isSanitizer(DataFlow::Node node) { - super.isSanitizer(node) or node.getType() instanceof PrimitiveType or node.getType() instanceof BoxedType or node.asExpr().(MethodAccess).getMethod() instanceof LengthRestrictedMethod From 0d13864bc8f0b8343b12f45ec79ff909de17dff6 Mon Sep 17 00:00:00 2001 From: Chris Smowton Date: Mon, 28 Mar 2022 17:03:24 +0100 Subject: [PATCH 55/91] Restrict polynomial ReDoS' strings-parsed-as-regexes search to those that could possibly be interesting In practice for polynomial ReDoS this means those regexes containing at least one potentially-infinite quantifier (* or +). --- .../lib/semmle/code/java/regex/RegexFlowConfigs.qll | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll b/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll index adcda1e2516..77b6d40b791 100644 --- a/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll +++ b/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll @@ -9,10 +9,16 @@ private import semmle.code.java.dataflow.DataFlow3 private import RegexFlowModels private import semmle.code.java.security.SecurityTests +private class ExploitableStringLiteral extends StringLiteral { + ExploitableStringLiteral() { this.getValue().matches(["%+%", "%*%"]) } +} + private class RegexCompileFlowConf extends DataFlow2::Configuration { RegexCompileFlowConf() { this = "RegexCompileFlowConfig" } - override predicate isSource(DataFlow::Node node) { node.asExpr() instanceof StringLiteral } + override predicate isSource(DataFlow::Node node) { + node.asExpr() instanceof ExploitableStringLiteral + } override predicate isSink(DataFlow::Node node) { sinkNode(node, ["regex-compile", "regex-compile-match", "regex-compile-find"]) @@ -203,7 +209,9 @@ private class GuavaRegexFlowStep extends RegexAdditionalFlowStep { private class RegexMatchFlowConf extends DataFlow2::Configuration { RegexMatchFlowConf() { this = "RegexMatchFlowConf" } - override predicate isSource(DataFlow::Node src) { src.asExpr() instanceof StringLiteral } + override predicate isSource(DataFlow::Node src) { + src.asExpr() instanceof ExploitableStringLiteral + } override predicate isSink(DataFlow::Node sink) { exists(RegexMatchMethodAccess ma | sink.asExpr() = ma.getRegexArg()) From bc17d4b91f28e435dec320663a62f3d5dbd98dc2 Mon Sep 17 00:00:00 2001 From: Chris Smowton Date: Mon, 28 Mar 2022 17:05:09 +0100 Subject: [PATCH 56/91] Break the recursion between seqChild, RegExpTerm and TRegExpSequence --- java/ql/lib/semmle/code/java/regex/RegexTreeView.qll | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll b/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll index 27074913a7e..f4bd6682583 100644 --- a/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll +++ b/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll @@ -20,7 +20,11 @@ private newtype TRegExpParent = /** A sequence term */ TRegExpSequence(Regex re, int start, int end) { re.sequence(start, end) and - exists(seqChild(re, start, end, 1)) // if a sequence does not have more than one element, it should be treated as that element instead. + // Only create sequence nodes for sequences with two or more children. + exists(int mid | + re.item(start, mid) and + re.item(mid, _) + ) } or /** An alternation term */ TRegExpAlt(Regex re, int start, int end) { From e5ca92424029315be395ab860ccdc9190138b70f Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Tue, 29 Mar 2022 11:07:24 +0100 Subject: [PATCH 57/91] Allow quantifiers invoving {}; add comments --- java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll | 8 +++++++- java/ql/lib/semmle/code/java/regex/RegexTreeView.qll | 7 ++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll b/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll index 77b6d40b791..6339ff238f3 100644 --- a/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll +++ b/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll @@ -10,7 +10,7 @@ private import RegexFlowModels private import semmle.code.java.security.SecurityTests private class ExploitableStringLiteral extends StringLiteral { - ExploitableStringLiteral() { this.getValue().matches(["%+%", "%*%"]) } + ExploitableStringLiteral() { this.getValue().matches(["%+%", "%*%", "%{%}%"]) } } private class RegexCompileFlowConf extends DataFlow2::Configuration { @@ -32,6 +32,9 @@ private class RegexCompileFlowConf extends DataFlow2::Configuration { /** * Holds if `s` is used as a regex, with the mode `mode` (if known). * If regex mode is not known, `mode` will be `"None"`. + * + * As an optimisation, only regexes containing an infinite repitition quatifier (`+`, `*`, or `{x,}`) + * and therefore may be relevant for ReDoS queries are considered. */ predicate usedAsRegex(StringLiteral s, string mode, boolean match_full_string) { exists(DataFlow::Node sink | @@ -224,6 +227,9 @@ private class RegexMatchFlowConf extends DataFlow2::Configuration { /** * Holds if the string literal `regex` is a regular expression that is matched against the expression `str`. + * + * As an optimisation, only regexes containing an infinite repitition quatifier (`+`, `*`, or `{x,}`) + * and therefore may be relevant for ReDoS queries are considered. */ predicate regexMatchedAgainst(StringLiteral regex, Expr str) { exists( diff --git a/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll b/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll index f4bd6682583..c447774906e 100644 --- a/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll +++ b/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll @@ -71,7 +71,12 @@ class RegExpParent extends TRegExpParent { abstract Regex getRegex(); } -/** A string literal used as a regular expression */ +/** + * A string literal used as a regular expression. + * + * As an optimisation, only regexes containing an infinite repitition quatifier (`+`, `*`, or `{x,}`) + * and therefore may be relevant for ReDoS queries are considered. + */ class RegExpLiteral extends TRegExpLiteral, RegExpParent { Regex re; From 4ed2e8d1fdd870bcff8e658f497bd2444931b39f Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Tue, 29 Mar 2022 11:12:37 +0100 Subject: [PATCH 58/91] Update tests to account for only regexes with quantifiers being considered --- .../regex/parser/RegexParseTests.expected | 25 +++++++++++++----- .../test/library-tests/regex/parser/Test.java | 26 +++++++++---------- 2 files changed, 32 insertions(+), 19 deletions(-) diff --git a/java/ql/test/library-tests/regex/parser/RegexParseTests.expected b/java/ql/test/library-tests/regex/parser/RegexParseTests.expected index 66d63f70808..97e8c397682 100644 --- a/java/ql/test/library-tests/regex/parser/RegexParseTests.expected +++ b/java/ql/test/library-tests/regex/parser/RegexParseTests.expected @@ -7,13 +7,18 @@ parseFailures | Test.java:5:13:5:13 | Z | [RegExpConstant,RegExpNormalChar] | | Test.java:5:14:5:16 | \\d | [RegExpCharacterClassEscape] | | Test.java:6:10:6:42 | \\Q hello world [ *** \\Q ) ( \\E | [RegExpConstant,RegExpQuote] | +| Test.java:6:10:6:43 | \\Q hello world [ *** \\Q ) ( \\E+ | [RegExpPlus] | | Test.java:7:10:7:23 | [\\Q hi ] \\E] | [RegExpCharacterClass] | +| Test.java:7:10:7:24 | [\\Q hi ] \\E]+ | [RegExpPlus] | | Test.java:7:11:7:22 | \\Q hi ] \\E | [RegExpConstant,RegExpQuote] | | Test.java:8:10:8:12 | []] | [RegExpCharacterClass] | +| Test.java:8:10:8:13 | []]+ | [RegExpPlus] | | Test.java:8:11:8:11 | ] | [RegExpConstant,RegExpNormalChar] | | Test.java:9:10:9:13 | [^]] | [RegExpCharacterClass] | +| Test.java:9:10:9:14 | [^]]+ | [RegExpPlus] | | Test.java:9:12:9:12 | ] | [RegExpConstant,RegExpNormalChar] | | Test.java:10:10:10:20 | [abc[defg]] | [RegExpCharacterClass] | +| Test.java:10:10:10:21 | [abc[defg]]+ | [RegExpPlus] | | Test.java:10:11:10:11 | a | [RegExpConstant,RegExpNormalChar] | | Test.java:10:12:10:12 | b | [RegExpConstant,RegExpNormalChar] | | Test.java:10:13:10:13 | c | [RegExpConstant,RegExpNormalChar] | @@ -24,7 +29,7 @@ parseFailures | Test.java:10:18:10:18 | g | [RegExpConstant,RegExpNormalChar] | | Test.java:10:19:10:19 | ] | [RegExpConstant,RegExpNormalChar] | | Test.java:11:10:11:57 | [abc&&[\\W\\p{Lower}\\P{Space}\\N{degree sign}]] | [RegExpCharacterClass] | -| Test.java:11:10:11:68 | [abc&&[\\W\\p{Lower}\\P{Space}\\N{degree sign}]]\\b7\\b{g}8 | [RegExpSequence] | +| Test.java:11:10:11:69 | [abc&&[\\W\\p{Lower}\\P{Space}\\N{degree sign}]]\\b7\\b{g}8+ | [RegExpSequence] | | Test.java:11:11:11:11 | a | [RegExpConstant,RegExpNormalChar] | | Test.java:11:12:11:12 | b | [RegExpConstant,RegExpNormalChar] | | Test.java:11:13:11:13 | c | [RegExpConstant,RegExpNormalChar] | @@ -40,11 +45,15 @@ parseFailures | Test.java:11:61:11:61 | 7 | [RegExpConstant,RegExpNormalChar] | | Test.java:11:62:11:67 | \\b{g} | [RegExpConstant,RegExpEscape] | | Test.java:11:68:11:68 | 8 | [RegExpConstant,RegExpNormalChar] | +| Test.java:11:68:11:69 | 8+ | [RegExpPlus] | | Test.java:12:10:12:13 | \\cA | [RegExpConstant,RegExpEscape] | +| Test.java:12:10:12:14 | \\cA+ | [RegExpPlus] | | Test.java:13:10:13:13 | \\c( | [RegExpConstant,RegExpEscape] | +| Test.java:13:10:13:14 | \\c(+ | [RegExpPlus] | | Test.java:14:10:14:14 | \\c\\ | [RegExpConstant,RegExpEscape] | -| Test.java:14:10:14:18 | \\c\\(ab) | [RegExpSequence] | +| Test.java:14:10:14:19 | \\c\\(ab)+ | [RegExpSequence] | | Test.java:14:15:14:18 | (ab) | [RegExpGroup] | +| Test.java:14:15:14:19 | (ab)+ | [RegExpPlus] | | Test.java:14:16:14:16 | a | [RegExpConstant,RegExpNormalChar] | | Test.java:14:16:14:17 | ab | [RegExpSequence] | | Test.java:14:17:14:17 | b | [RegExpConstant,RegExpNormalChar] | @@ -110,7 +119,7 @@ parseFailures | Test.java:16:102:16:102 | u | [RegExpConstant,RegExpNormalChar] | | Test.java:16:102:16:108 | u{16,}+ | [RegExpQuantifier] | | Test.java:17:10:17:13 | (?i) | [RegExpZeroWidthMatch] | -| Test.java:17:10:17:35 | (?i)(?=a)(?!b)(?<=c)(?hi)(?hell*?o*+)123\\k", "a+b*c?d{2}e{3,4}f{,5}g{6,}h+?i*?j??k{7}?l{8,9}?m{,10}?n{11,}?o++p*+q?+r{12}+s{13,14}+t{,15}+u{16,}+", - "(?i)(?=a)(?!b)(?<=c)(? Date: Tue, 29 Mar 2022 11:20:30 +0100 Subject: [PATCH 59/91] Add a test for deeply nested sequences --- .../regex/parser/RegexParseTests.expected | 51 +++++++++++++++++++ .../test/library-tests/regex/parser/Test.java | 3 +- 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/java/ql/test/library-tests/regex/parser/RegexParseTests.expected b/java/ql/test/library-tests/regex/parser/RegexParseTests.expected index 97e8c397682..ad94d005289 100644 --- a/java/ql/test/library-tests/regex/parser/RegexParseTests.expected +++ b/java/ql/test/library-tests/regex/parser/RegexParseTests.expected @@ -154,3 +154,54 @@ parseFailures | Test.java:20:10:20:12 | [\|] | [RegExpCharacterClass] | | Test.java:20:10:20:13 | [\|]+ | [RegExpPlus] | | Test.java:20:11:20:11 | \| | [RegExpConstant,RegExpNormalChar] | +| Test.java:21:10:21:37 | (a(a(a(a(a(a((((c))))a)))))) | [RegExpGroup] | +| Test.java:21:10:21:68 | (a(a(a(a(a(a((((c))))a))))))((((((b(((((d)))))b)b)b)b)b)b)+ | [RegExpSequence] | +| Test.java:21:11:21:11 | a | [RegExpConstant,RegExpNormalChar] | +| Test.java:21:11:21:36 | a(a(a(a(a(a((((c))))a))))) | [RegExpSequence] | +| Test.java:21:12:21:36 | (a(a(a(a(a((((c))))a))))) | [RegExpGroup] | +| Test.java:21:13:21:13 | a | [RegExpConstant,RegExpNormalChar] | +| Test.java:21:13:21:35 | a(a(a(a(a((((c))))a)))) | [RegExpSequence] | +| Test.java:21:14:21:35 | (a(a(a(a((((c))))a)))) | [RegExpGroup] | +| Test.java:21:15:21:15 | a | [RegExpConstant,RegExpNormalChar] | +| Test.java:21:15:21:34 | a(a(a(a((((c))))a))) | [RegExpSequence] | +| Test.java:21:16:21:34 | (a(a(a((((c))))a))) | [RegExpGroup] | +| Test.java:21:17:21:17 | a | [RegExpConstant,RegExpNormalChar] | +| Test.java:21:17:21:33 | a(a(a((((c))))a)) | [RegExpSequence] | +| Test.java:21:18:21:33 | (a(a((((c))))a)) | [RegExpGroup] | +| Test.java:21:19:21:19 | a | [RegExpConstant,RegExpNormalChar] | +| Test.java:21:19:21:32 | a(a((((c))))a) | [RegExpSequence] | +| Test.java:21:20:21:32 | (a((((c))))a) | [RegExpGroup] | +| Test.java:21:21:21:21 | a | [RegExpConstant,RegExpNormalChar] | +| Test.java:21:21:21:31 | a((((c))))a | [RegExpSequence] | +| Test.java:21:22:21:30 | ((((c)))) | [RegExpGroup] | +| Test.java:21:23:21:29 | (((c))) | [RegExpGroup] | +| Test.java:21:24:21:28 | ((c)) | [RegExpGroup] | +| Test.java:21:25:21:27 | (c) | [RegExpGroup] | +| Test.java:21:26:21:26 | c | [RegExpConstant,RegExpNormalChar] | +| Test.java:21:31:21:31 | a | [RegExpConstant,RegExpNormalChar] | +| Test.java:21:38:21:67 | ((((((b(((((d)))))b)b)b)b)b)b) | [RegExpGroup] | +| Test.java:21:38:21:68 | ((((((b(((((d)))))b)b)b)b)b)b)+ | [RegExpPlus] | +| Test.java:21:39:21:65 | (((((b(((((d)))))b)b)b)b)b) | [RegExpGroup] | +| Test.java:21:39:21:66 | (((((b(((((d)))))b)b)b)b)b)b | [RegExpSequence] | +| Test.java:21:40:21:63 | ((((b(((((d)))))b)b)b)b) | [RegExpGroup] | +| Test.java:21:40:21:64 | ((((b(((((d)))))b)b)b)b)b | [RegExpSequence] | +| Test.java:21:41:21:61 | (((b(((((d)))))b)b)b) | [RegExpGroup] | +| Test.java:21:41:21:62 | (((b(((((d)))))b)b)b)b | [RegExpSequence] | +| Test.java:21:42:21:59 | ((b(((((d)))))b)b) | [RegExpGroup] | +| Test.java:21:42:21:60 | ((b(((((d)))))b)b)b | [RegExpSequence] | +| Test.java:21:43:21:57 | (b(((((d)))))b) | [RegExpGroup] | +| Test.java:21:43:21:58 | (b(((((d)))))b)b | [RegExpSequence] | +| Test.java:21:44:21:44 | b | [RegExpConstant,RegExpNormalChar] | +| Test.java:21:44:21:56 | b(((((d)))))b | [RegExpSequence] | +| Test.java:21:45:21:55 | (((((d))))) | [RegExpGroup] | +| Test.java:21:46:21:54 | ((((d)))) | [RegExpGroup] | +| Test.java:21:47:21:53 | (((d))) | [RegExpGroup] | +| Test.java:21:48:21:52 | ((d)) | [RegExpGroup] | +| Test.java:21:49:21:51 | (d) | [RegExpGroup] | +| Test.java:21:50:21:50 | d | [RegExpConstant,RegExpNormalChar] | +| Test.java:21:56:21:56 | b | [RegExpConstant,RegExpNormalChar] | +| Test.java:21:58:21:58 | b | [RegExpConstant,RegExpNormalChar] | +| Test.java:21:60:21:60 | b | [RegExpConstant,RegExpNormalChar] | +| Test.java:21:62:21:62 | b | [RegExpConstant,RegExpNormalChar] | +| Test.java:21:64:21:64 | b | [RegExpConstant,RegExpNormalChar] | +| Test.java:21:66:21:66 | b | [RegExpConstant,RegExpNormalChar] | diff --git a/java/ql/test/library-tests/regex/parser/Test.java b/java/ql/test/library-tests/regex/parser/Test.java index 52b354e1c87..713f150243f 100644 --- a/java/ql/test/library-tests/regex/parser/Test.java +++ b/java/ql/test/library-tests/regex/parser/Test.java @@ -17,7 +17,8 @@ class Test { "(?i)(?=a)(?!b)(?<=c)(? Date: Mon, 4 Apr 2022 14:56:03 +0100 Subject: [PATCH 60/91] Sync shared files --- .../performance/ExponentialBackTracking.qll | 55 +++++++++++++---- .../java/security/performance/ReDoSUtil.qll | 4 +- ...gExpTreeView.qll => ReDoSUtilSpecific.qll} | 0 .../performance/SuperlinearBackTracking.qll | 60 +++++++++++++++---- 4 files changed, 91 insertions(+), 28 deletions(-) rename java/ql/lib/semmle/code/java/security/performance/{RegExpTreeView.qll => ReDoSUtilSpecific.qll} (100%) diff --git a/java/ql/lib/semmle/code/java/security/performance/ExponentialBackTracking.qll b/java/ql/lib/semmle/code/java/security/performance/ExponentialBackTracking.qll index 8d308a93104..5e0fe18ea00 100644 --- a/java/ql/lib/semmle/code/java/security/performance/ExponentialBackTracking.qll +++ b/java/ql/lib/semmle/code/java/security/performance/ExponentialBackTracking.qll @@ -279,17 +279,6 @@ private class Trace extends TTrace { } } -/** - * Gets a string corresponding to the trace `t`. - */ -private string concretise(Trace t) { - t = Nil() and result = "" - or - exists(InputSymbol s1, InputSymbol s2, Trace rest | t = Step(s1, s2, rest) | - result = concretise(rest) + intersect(s1, s2) - ) -} - /** * Holds if `r` is reachable from `(fork, fork)` under input `w`, and there is * a path from `r` back to `(fork, fork)` with `rem` steps. @@ -321,14 +310,54 @@ private StatePair getAForkPair(State fork) { result = MkStatePair(epsilonPred*(fork), epsilonPred*(fork)) } +private predicate hasSuffix(Trace suffix, Trace t, int i) { + // Declaring `t` to be a `RelevantTrace` currently causes a redundant check in the + // recursive case, so instead we check it explicitly here. + t instanceof RelevantTrace and + i = 0 and + suffix = t + or + hasSuffix(Step(_, _, suffix), t, i - 1) +} + +pragma[noinline] +private predicate hasTuple(InputSymbol s1, InputSymbol s2, Trace t, int i) { + hasSuffix(Step(s1, s2, _), t, i) +} + +private class RelevantTrace extends Trace, Step { + RelevantTrace() { + exists(State fork, StatePair q | + isReachableFromFork(fork, q, this, _) and + q = getAForkPair(fork) + ) + } + + pragma[noinline] + private string intersect(int i) { + exists(InputSymbol s1, InputSymbol s2 | + hasTuple(s1, s2, this, i) and + result = intersect(s1, s2) + ) + } + + /** Gets a string corresponding to this trace. */ + // the pragma is needed for the case where `intersect(s1, s2)` has multiple values, + // not for recursion + language[monotonicAggregates] + string concretise() { + result = strictconcat(int i | hasTuple(_, _, this, i) | this.intersect(i) order by i desc) + } +} + /** * Holds if `fork` is a pumpable fork with word `w`. */ private predicate isPumpable(State fork, string w) { - exists(StatePair q, Trace t | + exists(StatePair q, RelevantTrace t | isReachableFromFork(fork, q, t, _) and q = getAForkPair(fork) and - w = concretise(t) + w = t.concretise() ) } diff --git a/java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll b/java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll index a7d843ac7f8..8aa348bf62f 100644 --- a/java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll +++ b/java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll @@ -12,7 +12,7 @@ * states that will cause backtracking (a rejecting suffix exists). */ -import RegExpTreeView +import ReDoSUtilSpecific /** * A configuration for which parts of a regular expression should be considered relevant for @@ -32,7 +32,7 @@ abstract class ReDoSConfiguration extends string { } /** - * Holds if repeating `pump' starting at `state` is a candidate for causing backtracking. + * Holds if repeating `pump` starting at `state` is a candidate for causing backtracking. * No check whether a rejected suffix exists has been made. */ private predicate isReDoSCandidate(State state, string pump) { diff --git a/java/ql/lib/semmle/code/java/security/performance/RegExpTreeView.qll b/java/ql/lib/semmle/code/java/security/performance/ReDoSUtilSpecific.qll similarity index 100% rename from java/ql/lib/semmle/code/java/security/performance/RegExpTreeView.qll rename to java/ql/lib/semmle/code/java/security/performance/ReDoSUtilSpecific.qll diff --git a/java/ql/lib/semmle/code/java/security/performance/SuperlinearBackTracking.qll b/java/ql/lib/semmle/code/java/security/performance/SuperlinearBackTracking.qll index 2b42165ff7e..4ba9520cdcc 100644 --- a/java/ql/lib/semmle/code/java/security/performance/SuperlinearBackTracking.qll +++ b/java/ql/lib/semmle/code/java/security/performance/SuperlinearBackTracking.qll @@ -254,17 +254,6 @@ class Trace extends TTrace { } } -/** - * Gets a string corresponding to the trace `t`. - */ -string concretise(Trace t) { - t = Nil() and result = "" - or - exists(InputSymbol s1, InputSymbol s2, InputSymbol s3, Trace rest | t = Step(s1, s2, s3, rest) | - result = concretise(rest) + getAThreewayIntersect(s1, s2, s3) - ) -} - /** * Holds if there exists a transition from `r` to `q` in the product automaton. * Notice that the arguments are flipped, and thus the direction is backwards. @@ -332,6 +321,51 @@ StateTuple getAnEndTuple(State pivot, State succ) { result = MkStateTuple(pivot, succ, succ) } +private predicate hasSuffix(Trace suffix, Trace t, int i) { + // Declaring `t` to be a `RelevantTrace` currently causes a redundant check in the + // recursive case, so instead we check it explicitly here. + t instanceof RelevantTrace and + i = 0 and + suffix = t + or + hasSuffix(Step(_, _, _, suffix), t, i - 1) +} + +pragma[noinline] +private predicate hasTuple(InputSymbol s1, InputSymbol s2, InputSymbol s3, Trace t, int i) { + hasSuffix(Step(s1, s2, s3, _), t, i) +} + +private class RelevantTrace extends Trace, Step { + RelevantTrace() { + exists(State pivot, State succ, StateTuple q | + isReachableFromStartTuple(pivot, succ, q, this, _) and + q = getAnEndTuple(pivot, succ) + ) + } + + pragma[noinline] + private string getAThreewayIntersect(int i) { + exists(InputSymbol s1, InputSymbol s2, InputSymbol s3 | + hasTuple(s1, s2, s3, this, i) and + result = getAThreewayIntersect(s1, s2, s3) + ) + } + + /** Gets a string corresponding to this trace. */ + // the pragma is needed for the case where `getAThreewayIntersect(s1, s2, s3)` has multiple values, + // not for recursion + language[monotonicAggregates] + string concretise() { + result = + strictconcat(int i | + hasTuple(_, _, _, this, i) + | + this.getAThreewayIntersect(i) order by i desc + ) + } +} + /** * Holds if matching repetitions of `pump` can: * 1) Transition from `pivot` back to `pivot`. @@ -345,10 +379,10 @@ StateTuple getAnEndTuple(State pivot, State succ) { * available in the `hasReDoSResult` predicate. */ predicate isPumpable(State pivot, State succ, string pump) { - exists(StateTuple q, Trace t | + exists(StateTuple q, RelevantTrace t | isReachableFromStartTuple(pivot, succ, q, t, _) and q = getAnEndTuple(pivot, succ) and - pump = concretise(t) + pump = t.concretise() ) } From eec57d4f25bfca3eb6e5b4339169df532df2ff44 Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Tue, 5 Apr 2022 15:39:46 +0100 Subject: [PATCH 61/91] Simplify dataflow logic by using only one configuration, and expessing more sinks with models-as-data --- .../code/java/regex/RegexFlowConfigs.qll | 198 +++++++----------- .../code/java/regex/RegexFlowModels.qll | 41 ++-- 2 files changed, 93 insertions(+), 146 deletions(-) diff --git a/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll b/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll index 6339ff238f3..c400b521f80 100644 --- a/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll +++ b/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll @@ -13,87 +13,53 @@ private class ExploitableStringLiteral extends StringLiteral { ExploitableStringLiteral() { this.getValue().matches(["%+%", "%*%", "%{%}%"]) } } -private class RegexCompileFlowConf extends DataFlow2::Configuration { - RegexCompileFlowConf() { this = "RegexCompileFlowConfig" } - - override predicate isSource(DataFlow::Node node) { - node.asExpr() instanceof ExploitableStringLiteral - } - - override predicate isSink(DataFlow::Node node) { - sinkNode(node, ["regex-compile", "regex-compile-match", "regex-compile-find"]) - } - - override predicate isBarrier(DataFlow::Node node) { - node.getEnclosingCallable().getDeclaringType() instanceof NonSecurityTestClass - } -} - /** - * Holds if `s` is used as a regex, with the mode `mode` (if known). - * If regex mode is not known, `mode` will be `"None"`. - * - * As an optimisation, only regexes containing an infinite repitition quatifier (`+`, `*`, or `{x,}`) - * and therefore may be relevant for ReDoS queries are considered. + * Holds if `kind` is an external sink kind that is relevant for regex flow. + * `full` is true if sinks with this kind match against the full string of its input. + * `strArg` is the index of the argument to methods with this sink kind that contan the string to be matched against, + * where -1 is the qualifier; or -2 if no such argument exists. */ -predicate usedAsRegex(StringLiteral s, string mode, boolean match_full_string) { - exists(DataFlow::Node sink | - any(RegexCompileFlowConf c).hasFlow(DataFlow2::exprNode(s), sink) and - mode = "None" and // TODO: proper mode detection - (if matchesFullString(sink) then match_full_string = true else match_full_string = false) +private predicate regexSinkKindInfo(string kind, boolean full, int strArg) { + sinkModel(_, _, _, _, _, _, _, kind) and + exists(string fullStr, string strArgStr | + ( + full = true and fullStr = "f" + or + full = false and fullStr = "" + ) and + ( + strArgStr.toInt() = strArg + or + strArg = -2 and + strArgStr = "" + ) + | + kind = "regex-use[" + fullStr + strArgStr + "]" ) } -/** - * Holds if the regex that flows to `sink` is used to match against a full string, - * as though it was implicitly surrounded by ^ and $. - */ -private predicate matchesFullString(DataFlow::Node sink) { - sinkNode(sink, "regex-compile-match") - or - exists(DataFlow::Node matchSource, RegexCompileToMatchConf conf | - matchSource.asExpr().(MethodAccess).getAnArgument() = sink.asExpr() and - conf.hasFlow(matchSource, _) - ) -} +/** A sink that is relevant for regex flow. */ +private class RegexFlowSink extends DataFlow::Node { + boolean full; + int strArg; -private class RegexCompileToMatchConf extends DataFlow2::Configuration { - RegexCompileToMatchConf() { this = "RegexCompileToMatchConfig" } - - override predicate isSource(DataFlow::Node node) { sourceNode(node, "regex-compile") } - - override predicate isSink(DataFlow::Node node) { sinkNode(node, "regex-match") } - - override predicate isAdditionalFlowStep(DataFlow::Node node1, DataFlow::Node node2) { - exists(MethodAccess ma | node2.asExpr() = ma and node1.asExpr() = ma.getQualifier() | - ma.getMethod().hasQualifiedName("java.util.regex", "Pattern", "matcher") + RegexFlowSink() { + exists(string kind | + regexSinkKindInfo(kind, full, strArg) and + sinkNode(this, kind) ) } -} -/** - * A method access that can match a regex against a string - */ -abstract class RegexMatchMethodAccess extends MethodAccess { - string package; - string type; - string name; - int regexArg; - int stringArg; - Method m; + /** Holds if a regex that flows here is matched against a full string (rather than a substring). */ + predicate matchesFullString() { full = true } - RegexMatchMethodAccess() { - this.getMethod().getSourceDeclaration().overrides*(m) and - m.hasQualifiedName(package, type, name) and - regexArg in [-1 .. m.getNumberOfParameters() - 1] and - stringArg in [-1 .. m.getNumberOfParameters() - 1] + /** Gets the string expression that a regex that flows here is matched against, if any. */ + Expr getStringArgument() { + exists(MethodAccess ma | + this.asExpr() = argOf(ma, _) and + result = argOf(ma, strArg) + ) } - - /** Gets the argument of this call that the regex to be matched against flows into. */ - Expr getRegexArg() { result = argOf(this, regexArg) } - - /** Gets the argument of this call that the string being matched flows into. */ - Expr getStringArg() { result = argOf(this, stringArg) } } private Expr argOf(MethodAccess ma, int arg) { @@ -115,35 +81,7 @@ class RegexAdditionalFlowStep extends Unit { abstract predicate step(DataFlow::Node node1, DataFlow::Node node2); } -// TODO: can this be done with the models-as-data framework? -private class JdkRegexMatchMethodAccess extends RegexMatchMethodAccess { - JdkRegexMatchMethodAccess() { - package = "java.util.regex" and - type = "Pattern" and - ( - name = "matcher" and regexArg = -1 and stringArg = 0 - or - name = "matches" and regexArg = 0 and stringArg = 1 - or - name = "split" and regexArg = -1 and stringArg = 0 - or - name = "splitAsStream" and regexArg = -1 and stringArg = 0 - ) - or - package = "java.lang" and - type = "String" and - name = ["matches", "split", "replaceAll", "replaceFirst"] and - regexArg = 0 and - stringArg = -1 - or - package = "java.util.function" and - type = "Predicate" and - name = "test" and - regexArg = -1 and - stringArg = 0 - } -} - +// TODO: This may be able to be done with models-as-data if query-specific flow steps beome supported. private class JdkRegexFlowStep extends RegexAdditionalFlowStep { override predicate step(DataFlow::Node node1, DataFlow::Node node2) { exists(MethodAccess ma, Method m, string package, string type, string name, int arg | @@ -155,7 +93,7 @@ private class JdkRegexFlowStep extends RegexAdditionalFlowStep { package = "java.util.regex" and type = "Pattern" and ( - name = ["asMatchPredicate", "asPredicate"] and + name = ["asMatchPredicate", "asPredicate", "matcher"] and arg = -1 or name = "compile" and @@ -170,16 +108,6 @@ private class JdkRegexFlowStep extends RegexAdditionalFlowStep { } } -private class GuavaRegexMatchMethodAccess extends RegexMatchMethodAccess { - GuavaRegexMatchMethodAccess() { - package = "com.google.common.base" and - regexArg = -1 and - stringArg = 0 and - type = ["Splitter", "Splitter$MapSplitter"] and - name = ["split", "splitToList"] - } -} - private class GuavaRegexFlowStep extends RegexAdditionalFlowStep { override predicate step(DataFlow::Node node1, DataFlow::Node node2) { exists(MethodAccess ma, Method m, string package, string type, string name, int arg | @@ -209,20 +137,46 @@ private class GuavaRegexFlowStep extends RegexAdditionalFlowStep { } } -private class RegexMatchFlowConf extends DataFlow2::Configuration { - RegexMatchFlowConf() { this = "RegexMatchFlowConf" } +private class RegexFlowConf extends DataFlow2::Configuration { + RegexFlowConf() { this = "RegexFlowConfig" } - override predicate isSource(DataFlow::Node src) { - src.asExpr() instanceof ExploitableStringLiteral + override predicate isSource(DataFlow::Node node) { + node.asExpr() instanceof ExploitableStringLiteral } - override predicate isSink(DataFlow::Node sink) { - exists(RegexMatchMethodAccess ma | sink.asExpr() = ma.getRegexArg()) - } + override predicate isSink(DataFlow::Node node) { node instanceof RegexFlowSink } override predicate isAdditionalFlowStep(DataFlow::Node node1, DataFlow::Node node2) { any(RegexAdditionalFlowStep s).step(node1, node2) } + + override predicate isBarrier(DataFlow::Node node) { + node.getEnclosingCallable().getDeclaringType() instanceof NonSecurityTestClass + } +} + +/** + * Holds if `regex` is used as a regex, with the mode `mode` (if known). + * If regex mode is not known, `mode` will be `"None"`. + * + * As an optimisation, only regexes containing an infinite repitition quatifier (`+`, `*`, or `{x,}`) + * and therefore may be relevant for ReDoS queries are considered. + */ +predicate usedAsRegex(StringLiteral regex, string mode, boolean match_full_string) { + any(RegexFlowConf c).hasFlow(DataFlow2::exprNode(regex), _) and + mode = "None" and // TODO: proper mode detection + (if matchesFullString(regex) then match_full_string = true else match_full_string = false) +} + +/** + * Holds if `regex` is used as a regular expression that is matched against a full string, + * as though it was implicitly surrounded by ^ and $. + */ +private predicate matchesFullString(StringLiteral regex) { + exists(RegexFlowConf c, RegexFlowSink sink | + sink.matchesFullString() and + c.hasFlow(DataFlow2::exprNode(regex), sink) + ) } /** @@ -232,12 +186,8 @@ private class RegexMatchFlowConf extends DataFlow2::Configuration { * and therefore may be relevant for ReDoS queries are considered. */ predicate regexMatchedAgainst(StringLiteral regex, Expr str) { - exists( - DataFlow::Node src, DataFlow::Node sink, RegexMatchMethodAccess ma, RegexMatchFlowConf conf - | - src.asExpr() = regex and - sink.asExpr() = ma.getRegexArg() and - conf.hasFlow(src, sink) and - str = ma.getStringArg() + exists(RegexFlowConf c, RegexFlowSink sink | + str = sink.getStringArgument() and + c.hasFlow(DataFlow2::exprNode(regex), sink) ) } diff --git a/java/ql/lib/semmle/code/java/regex/RegexFlowModels.qll b/java/ql/lib/semmle/code/java/regex/RegexFlowModels.qll index fd0858639c4..6934540116f 100644 --- a/java/ql/lib/semmle/code/java/regex/RegexFlowModels.qll +++ b/java/ql/lib/semmle/code/java/regex/RegexFlowModels.qll @@ -3,33 +3,30 @@ import java import semmle.code.java.dataflow.ExternalFlow -private class RegexSourceCsv extends SourceModelCsv { - override predicate row(string row) { - row = - [ - //"namespace;type;subtypes;name;signature;ext;output;kind" - "java.util.regex;Pattern;false;compile;(String);;ReturnValue;regex-compile", - ] - } -} - private class RegexSinkCsv extends SinkModelCsv { override predicate row(string row) { row = [ //"namespace;type;subtypes;name;signature;ext;input;kind" - "java.util.regex;Pattern;false;compile;(String);;Argument[0];regex-compile", - "java.util.regex;Pattern;false;compile;(String,int);;Argument[0];regex-compile", - "java.util.regex;Pattern;false;matches;(String,CharSequence);;Argument[0];regex-compile-match", - "java.lang;String;false;matches;(String);;Argument[0];regex-compile-match", - "java.lang;String;false;split;(String);;Argument[0];regex-compile-find", - "java.lang;String;false;split;(String,int);;Argument[0];regex-compile-find", - "java.lang;String;false;replaceAll;(String,String);;Argument[0];regex-compile-find", - "java.lang;String;false;replaceFirst;(String,String);;Argument[0];regex-compile-find", - "com.google.common.base;Splitter;false;onPattern;(String);;Argument[0];regex-compile-find", - // regex-match sinks - "java.util.regex;Pattern;false;asMatchPredicate;();;Argument[-1];regex-match", - "java.util.regex;Matcher;false;matches;();;Argument[-1];regex-match", + "java.util.regex;Matcher;false;matches;();;Argument[-1];regex-use[f]", + "java.util.regex;Pattern;false;asMatchPredicate;();;Argument[-1];regex-use[f]", + "java.util.regex;Pattern;false;compile;(String);;Argument[0];regex-use[]", + "java.util.regex;Pattern;false;compile;(String,int);;Argument[0];regex-use[]", + "java.util.regex;Pattern;false;matcher;(CharSequence);;Argument[-1];regex-use[0]", + "java.util.regex;Pattern;false;matches;(String,CharSequence);;Argument[0];regex-use[f1]", + "java.util.regex;Pattern;false;split;(CharSequence);;Argument[-1];regex-use[0]", + "java.util.regex;Pattern;false;split;(CharSequence,int);;Argument[-1];regex-use[0]", + "java.util.regex;Pattern;false;splitAsStream;(CharSequence);;Argument[-1];regex-use[0]", + "java.util.function;Predicate;false;test;(Object);;Argument[-1];regex-use[0]", + "java.lang;String;false;matches;(String);;Argument[0];regex-use[f-1]", + "java.lang;String;false;split;(String);;Argument[0];regex-use[-1]", + "java.lang;String;false;split;(String,int);;Argument[0];regex-use[-1]", + "java.lang;String;false;replaceAll;(String,String);;Argument[0];regex-use[-1]", + "java.lang;String;false;replaceFirst;(String,String);;Argument[0];regex-use[-1]", + "com.google.common.base;Splitter;false;onPattern;(String);;Argument[0];regex-use[]", + "com.google.common.base;Splitter;false;split;(CharSequence);;Argument[-1];regex-use[0]", + "com.google.common.base;Splitter;false;splitToList;(CharSequence);;Argument[-1];regex-use[0]", + "com.google.common.base;Splitter$MapSplitter;false;split;(CharSequence);;Argument[-1];regex-use[0]", ] } } From 66ab2bca75bd72427e42253b02c9528bbff4cabc Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Tue, 5 Apr 2022 16:29:01 +0100 Subject: [PATCH 62/91] Update PrintAst test output --- java/ql/test/library-tests/JDK/PrintAst.expected | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/java/ql/test/library-tests/JDK/PrintAst.expected b/java/ql/test/library-tests/JDK/PrintAst.expected index e6f240b325e..8074ef1b965 100644 --- a/java/ql/test/library-tests/JDK/PrintAst.expected +++ b/java/ql/test/library-tests/JDK/PrintAst.expected @@ -73,6 +73,11 @@ jdk/StringMatch.java: # 5| 0: [MethodAccess] matches(...) # 5| -1: [VarAccess] STR # 5| 0: [StringLiteral] "[a-z]+" +# 5| 0: [RegExpPlus] [a-z]+ +# 5| 0: [RegExpCharacterClass] [a-z] +# 5| 0: [RegExpCharacterRange] a-z +# 5| 0: [RegExpConstant | RegExpNormalChar] a +# 5| 1: [RegExpConstant | RegExpNormalChar] z # 8| 5: [Method] b # 8| 3: [TypeAccess] void # 8| 5: [BlockStmt] { ... } From b08f22c24dab0fe82630183d474de2023b40e876 Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Wed, 6 Apr 2022 12:35:24 +0100 Subject: [PATCH 63/91] Remove unnecassary import --- java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll | 1 - 1 file changed, 1 deletion(-) diff --git a/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll b/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll index c400b521f80..84187f1ba48 100644 --- a/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll +++ b/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll @@ -5,7 +5,6 @@ import java private import semmle.code.java.dataflow.DataFlow private import semmle.code.java.dataflow.DataFlow2 -private import semmle.code.java.dataflow.DataFlow3 private import RegexFlowModels private import semmle.code.java.security.SecurityTests From b854a2185e00c0e150db9e5dc066b8c4c52a71b8 Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Thu, 7 Apr 2022 12:20:17 +0100 Subject: [PATCH 64/91] Fix use of `sinkModel` --- java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll b/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll index 84187f1ba48..8936de5a923 100644 --- a/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll +++ b/java/ql/lib/semmle/code/java/regex/RegexFlowConfigs.qll @@ -3,6 +3,7 @@ */ import java +import semmle.code.java.dataflow.ExternalFlow private import semmle.code.java.dataflow.DataFlow private import semmle.code.java.dataflow.DataFlow2 private import RegexFlowModels @@ -19,7 +20,7 @@ private class ExploitableStringLiteral extends StringLiteral { * where -1 is the qualifier; or -2 if no such argument exists. */ private predicate regexSinkKindInfo(string kind, boolean full, int strArg) { - sinkModel(_, _, _, _, _, _, _, kind) and + sinkModel(_, _, _, _, _, _, _, kind, _) and exists(string fullStr, string strArgStr | ( full = true and fullStr = "f" From 9078e13f1c27740d02c4b7b11a1dc1bacb22c38d Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Tue, 3 May 2022 14:52:34 +0100 Subject: [PATCH 65/91] Apply reveiw suggestions - make java imports private - qdoc fixes - reorder predicates - simplifications --- .../semmle/code/java/regex/RegexTreeView.qll | 2 +- java/ql/lib/semmle/code/java/regex/regex.qll | 340 +++++++++--------- .../performance/ReDoSUtilSpecific.qll | 2 +- 3 files changed, 170 insertions(+), 174 deletions(-) diff --git a/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll b/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll index c447774906e..2db3da550e2 100644 --- a/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll +++ b/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll @@ -1,6 +1,6 @@ /** Provides a class hierarchy corresponding to a parse tree of regular expressions. */ -import java +private import java private import semmle.code.java.regex.regex /** diff --git a/java/ql/lib/semmle/code/java/regex/regex.qll b/java/ql/lib/semmle/code/java/regex/regex.qll index 9a7da15999e..a2cb535bda1 100644 --- a/java/ql/lib/semmle/code/java/regex/regex.qll +++ b/java/ql/lib/semmle/code/java/regex/regex.qll @@ -62,7 +62,7 @@ abstract class RegexString extends StringLiteral { /** * Helper predicate for `quote`. - * Holds if the char at `pos` is the one-based `index`th occourence of a quote delimiter (`\Q` or `\E`) + * Holds if the char at `pos` is the one-based `index`th occurence of a quote delimiter (`\Q` or `\E`) * Result is `true` for `\Q` and `false` for `\E`. */ private boolean quoteDelimiter(int index, int pos) { @@ -73,7 +73,7 @@ abstract class RegexString extends StringLiteral { /** Holds if a quoted sequence is found between `start` and `end` */ predicate quote(int start, int end) { this.quote(start, end, _, _) } - /** Holds if a quoted sequence is found between `start` and `end`, with ontent found between `inner_start` and `inner_end`. */ + /** Holds if a quoted sequence is found between `start` and `end`, with content found between `inner_start` and `inner_end`. */ predicate quote(int start, int end, int inner_start, int inner_end) { exists(int index | this.quoteDelimiter(index, start) = true and @@ -98,7 +98,7 @@ abstract class RegexString extends StringLiteral { } /** - * A control sequence, `\cx` + * Holds if there is a control sequence, `\cx`, between `start` and `end`. * `x` may be any ascii character including special characters. */ predicate controlEscape(int start, int end) { @@ -107,171 +107,6 @@ abstract class RegexString extends StringLiteral { end = start + 3 } - private string nonEscapedCharAt(int i) { - result = this.getChar(i) and - not exists(int x, int y | this.escapedCharacter(x, y) and i in [x .. y - 1]) and - not exists(int x, int y | this.quote(x, y) and i in [x .. y - 1]) - } - - /** Holds if a character set starts between `start` and `end`, including any negation character (`^`). */ - private predicate charSetStart0(int start, int end) { - this.nonEscapedCharAt(start) = "[" and - (if this.getChar(start + 1) = "^" then end = start + 2 else end = start + 1) - } - - /** Holds if the character at `pos` marks the end of a character class. */ - private predicate charSetEnd0(int pos) { - this.nonEscapedCharAt(pos) = "]" and - /* special case: `[]]` and `[^]]` are valid char classes. */ - not this.charSetStart0(_, pos) - } - - /** - * Holds if the character at `pos` starts a character set delimiter. - * Result is 1 for `[` and 0 for `]`. - */ - private int charSetDelimiter(int pos) { - result = 1 and this.charSetStart0(pos, _) - or - result = -1 and this.charSetEnd0(pos) - } - - /** - * Holds if the char at `pos` is the one-based `index`th occourence of a character set delimiter (`[` or `]`). - * Result is 1 for `[` and -1 for `]`. - */ - private int charSetDelimiter(int index, int pos) { - result = this.charSetDelimiter(pos) and - pos = rank[index](int p | exists(this.charSetDelimiter(p))) - } - - bindingset[x] - private int max_zero(int x) { result = max([x, 0]) } - - /** - * Gets the nesting depth of character classes after position `pos`, - * where `pos` is the position of a character set delimiter. - */ - private int charSetDepth(int index, int pos) { - index = 1 and result = max_zero(charSetDelimiter(index, pos)) - or - result = max_zero(charSetDelimiter(index, pos) + charSetDepth(index - 1, _)) - } - - /** Hold if a top-level character set starts between `start` and `end`. */ - predicate charSetStart(int start, int end) { - this.charSetStart0(start, end) and - this.charSetDepth(_, start) = 1 - } - - /** Holds if a top-level character set ends at `pos`. */ - predicate charSetEnd(int pos) { - this.charSetEnd0(pos) and - this.charSetDepth(_, pos) = 0 - } - - /** - * Holds if there is a top-level character class beginning at `start` (inclusive) and ending at `end` (exclusive) - * - * For now, nested character classes are approximated by only considering the top-level class for parsing. - * This leads to very similar results for ReDoS queries. - */ - predicate charSet(int start, int end) { - exists(int inner_start, int inner_end | this.charSetStart(start, inner_start) | - end = inner_end + 1 and - inner_end = - min(int end_delimiter | this.charSetEnd(end_delimiter) and end_delimiter > inner_start) - ) - } - - /** Either a char or a - */ - private predicate charSetToken(int charset_start, int start, int end) { - this.charSetStart(charset_start, start) and - ( - this.escapedCharacter(start, end) - or - exists(this.nonEscapedCharAt(start)) and end = start + 1 - or - this.quote(start, end) - ) - or - this.charSetToken(charset_start, _, start) and - ( - this.escapedCharacter(start, end) - or - exists(this.nonEscapedCharAt(start)) and - end = start + 1 and - not this.charSetEnd(start) - or - this.quote(start, end) - ) - } - - /** An indexed version of `charSetToken/3` */ - private predicate charSetToken(int charset_start, int index, int token_start, int token_end) { - token_start = - rank[index](int start, int end | this.charSetToken(charset_start, start, end) | start) and - this.charSetToken(charset_start, token_start, token_end) - } - - /** - * Holds if the character set starting at `charset_start` contains either - * a character or a range found between `start` and `end`. - */ - predicate charSetChild(int charset_start, int start, int end) { - this.charSetToken(charset_start, start, end) and - not exists(int range_start, int range_end | - this.charRange(charset_start, range_start, _, _, range_end) and - range_start <= start and - range_end >= end - ) - or - this.charRange(charset_start, start, _, _, end) - } - - /** - * Helper predicate for `charRange`. - * We can determine where character ranges end by a left to right sweep. - * - * To avoid negative recursion we return a boolean. See `escaping`, - * the helper for `escapingChar`, for a clean use of this pattern. - */ - private boolean charRangeEnd(int charset_start, int index) { - this.charSetToken(charset_start, index, _, _) and - ( - index in [1, 2] and result = false - or - index > 2 and - exists(int connector_start | - this.charSetToken(charset_start, index - 1, connector_start, _) and - this.nonEscapedCharAt(connector_start) = "-" and - result = - this.charRangeEnd(charset_start, index - 2) - .booleanNot() - .booleanAnd(this.charRangeEnd(charset_start, index - 1).booleanNot()) - ) - or - not exists(int connector_start | - this.charSetToken(charset_start, index - 1, connector_start, _) and - this.nonEscapedCharAt(connector_start) = "-" - ) and - result = false - ) - } - - /** - * Holds if the character set starting at `charset_start` contains a character range - * with lower bound found between `start` and `lower_end` - * and upper bound found between `upper_start` and `end`. - */ - predicate charRange(int charset_start, int start, int lower_end, int upper_start, int end) { - exists(int index | - this.charRangeEnd(charset_start, index) = true and - this.charSetToken(charset_start, index - 2, start, lower_end) and - this.charSetToken(charset_start, index, upper_start, end) - ) - } - pragma[inline] private predicate isOctal(int index) { this.getChar(index) = [0 .. 7].toString() } @@ -331,6 +166,167 @@ abstract class RegexString extends StringLiteral { ) } + private string nonEscapedCharAt(int i) { + result = this.getChar(i) and + not exists(int x, int y | this.escapedCharacter(x, y) and i in [x .. y - 1]) and + not exists(int x, int y | this.quote(x, y) and i in [x .. y - 1]) + } + + /** Holds if a character set starts between `start` and `end`, including any negation character (`^`). */ + private predicate charSetStart0(int start, int end) { + this.nonEscapedCharAt(start) = "[" and + (if this.getChar(start + 1) = "^" then end = start + 2 else end = start + 1) + } + + /** Holds if the character at `pos` marks the end of a character class. */ + private predicate charSetEnd0(int pos) { + this.nonEscapedCharAt(pos) = "]" and + /* special case: `[]]` and `[^]]` are valid char classes. */ + not this.charSetStart0(_, pos) + } + + /** + * Holds if the character at `pos` starts a character set delimiter. + * Result is 1 for `[` and -1 for `]`. + */ + private int charSetDelimiter(int pos) { + result = 1 and this.charSetStart0(pos, _) + or + result = -1 and this.charSetEnd0(pos) + } + + /** + * Holds if the char at `pos` is the one-based `index`th occourence of a character set delimiter (`[` or `]`). + * Result is 1 for `[` and -1 for `]`. + */ + private int charSetDelimiter(int index, int pos) { + result = this.charSetDelimiter(pos) and + pos = rank[index](int p | exists(this.charSetDelimiter(p))) + } + + /** + * Gets the nesting depth of character classes after position `pos`, + * where `pos` is the position of a character set delimiter. + */ + private int charSetDepth(int index, int pos) { + index = 1 and result = 0.maximum(this.charSetDelimiter(index, pos)) + or + result = 0.maximum(this.charSetDelimiter(index, pos) + this.charSetDepth(index - 1, _)) + } + + /** Hold if a top-level character set starts between `start` and `end`. */ + predicate charSetStart(int start, int end) { + this.charSetStart0(start, end) and + this.charSetDepth(_, start) = 1 + } + + /** Holds if a top-level character set ends at `pos`. */ + predicate charSetEnd(int pos) { + this.charSetEnd0(pos) and + this.charSetDepth(_, pos) = 0 + } + + /** + * Holds if there is a top-level character class beginning at `start` (inclusive) and ending at `end` (exclusive) + * + * For now, nested character classes are approximated by only considering the top-level class for parsing. + * This leads to very similar results for ReDoS queries. + */ + predicate charSet(int start, int end) { + exists(int inner_start, int inner_end | this.charSetStart(start, inner_start) | + end = inner_end + 1 and + inner_end = + min(int end_delimiter | this.charSetEnd(end_delimiter) and end_delimiter > inner_start) + ) + } + + /** Either a char or a - */ + private predicate charSetToken(int charset_start, int start, int end) { + this.charSetStart(charset_start, start) and + ( + this.escapedCharacter(start, end) + or + exists(this.nonEscapedCharAt(start)) and end = start + 1 + or + this.quote(start, end) + ) + or + this.charSetToken(charset_start, _, start) and + ( + this.escapedCharacter(start, end) + or + exists(this.nonEscapedCharAt(start)) and + end = start + 1 and + not this.charSetEnd(start) + or + this.quote(start, end) + ) + } + + /** An indexed version of `charSetToken/3` */ + private predicate charSetToken(int charset_start, int index, int token_start, int token_end) { + token_start = rank[index](int start | this.charSetToken(charset_start, start, _) | start) and + this.charSetToken(charset_start, token_start, token_end) + } + + /** + * Helper predicate for `charRange`. + * We can determine where character ranges end by a left to right sweep. + * + * To avoid negative recursion we return a boolean. See `escaping`, + * the helper for `escapingChar`, for a clean use of this pattern. + */ + private boolean charRangeEnd(int charset_start, int index) { + this.charSetToken(charset_start, index, _, _) and + ( + index in [1, 2] and result = false + or + index > 2 and + exists(int connector_start | + this.charSetToken(charset_start, index - 1, connector_start, _) and + this.nonEscapedCharAt(connector_start) = "-" and + result = + this.charRangeEnd(charset_start, index - 2) + .booleanNot() + .booleanAnd(this.charRangeEnd(charset_start, index - 1).booleanNot()) + ) + or + not exists(int connector_start | + this.charSetToken(charset_start, index - 1, connector_start, _) and + this.nonEscapedCharAt(connector_start) = "-" + ) and + result = false + ) + } + + /** + * Holds if the character set starting at `charset_start` contains a character range + * with lower bound found between `start` and `lower_end` + * and upper bound found between `upper_start` and `end`. + */ + predicate charRange(int charset_start, int start, int lower_end, int upper_start, int end) { + exists(int index | + this.charRangeEnd(charset_start, index) = true and + this.charSetToken(charset_start, index - 2, start, lower_end) and + this.charSetToken(charset_start, index, upper_start, end) + ) + } + + /** + * Holds if the character set starting at `charset_start` contains either + * a character or a range found between `start` and `end`. + */ + predicate charSetChild(int charset_start, int start, int end) { + this.charSetToken(charset_start, start, end) and + not exists(int range_start, int range_end | + this.charRange(charset_start, range_start, _, _, range_end) and + range_start <= start and + range_end >= end + ) + or + this.charRange(charset_start, start, _, _, end) + } + /** Holds if `index` is inside a character set. */ predicate inCharSet(int index) { exists(int x, int y | this.charSet(x, y) and index in [x + 1 .. y - 2]) @@ -871,9 +867,9 @@ abstract class RegexString extends StringLiteral { * Holds if a character is represented between `start` and `end` in the source literal. */ private predicate sourceCharacter(int start, int end) { - sourceEscapedCharacter(start, end) + this.sourceEscapedCharacter(start, end) or - sourceNonEscapedCharacter(start) and + this.sourceNonEscapedCharacter(start) and end = start + 1 } @@ -885,8 +881,8 @@ abstract class RegexString extends StringLiteral { */ predicate sourceCharacter(int pos, int start, int end) { exists(this.getChar(pos)) and - sourceCharacter(start, end) and - start = rank[pos + 2](int s | sourceCharacter(s, _)) + this.sourceCharacter(start, end) and + start = rank[pos + 2](int s | this.sourceCharacter(s, _)) } } diff --git a/java/ql/lib/semmle/code/java/security/performance/ReDoSUtilSpecific.qll b/java/ql/lib/semmle/code/java/security/performance/ReDoSUtilSpecific.qll index daef79ceb1e..541c3ca8f36 100644 --- a/java/ql/lib/semmle/code/java/security/performance/ReDoSUtilSpecific.qll +++ b/java/ql/lib/semmle/code/java/security/performance/ReDoSUtilSpecific.qll @@ -3,7 +3,7 @@ * This is the interface to the shared ReDoS library. */ -import java +private import java import semmle.code.java.regex.RegexTreeView /** From 2d82dfba38465c2c0abeefb39bdca65f8990a454 Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Tue, 3 May 2022 16:30:27 +0100 Subject: [PATCH 66/91] Reorder backreference predicates --- java/ql/lib/semmle/code/java/regex/regex.qll | 90 ++++++++++---------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/java/ql/lib/semmle/code/java/regex/regex.qll b/java/ql/lib/semmle/code/java/regex/regex.qll index a2cb535bda1..48f0f74580c 100644 --- a/java/ql/lib/semmle/code/java/regex/regex.qll +++ b/java/ql/lib/semmle/code/java/regex/regex.qll @@ -107,6 +107,51 @@ abstract class RegexString extends StringLiteral { end = start + 3 } + private predicate namedBackreference(int start, int end, string name) { + this.escapingChar(start) and + this.getChar(start + 1) = "k" and + this.getChar(start + 2) = "<" and + end = min(int i | i > start + 2 and this.getChar(i) = ">") + 1 and + name = this.getText().substring(start + 3, end - 2) + } + + private predicate numberedBackreference(int start, int end, int value) { + this.escapingChar(start) and + // starting with 0 makes it an octal escape + not this.getChar(start + 1) = "0" and + exists(string text, string svalue, int len | + end = start + len and + text = this.getText() and + len in [2 .. 3] + | + svalue = text.substring(start + 1, start + len) and + value = svalue.toInt() and + // value is composed of digits + forall(int i | i in [start + 1 .. start + len - 1] | this.getChar(i) = [0 .. 9].toString()) and + // a longer reference is not possible + not ( + len = 2 and + exists(text.substring(start + 1, start + len + 1).toInt()) + ) and + // 3 octal digits makes it an octal escape + not forall(int i | i in [start + 1 .. start + 4] | this.isOctal(i)) + // TODO: Inside a character set, all numeric escapes are treated as characters. + ) + } + + /** Holds if the text in the range start,end is a back reference */ + predicate backreference(int start, int end) { + this.numberedBackreference(start, end, _) + or + this.namedBackreference(start, end, _) + } + + /** Gets the number of the back reference in start,end */ + int getBackrefNumber(int start, int end) { this.numberedBackreference(start, end, result) } + + /** Gets the name, if it has one, of the back reference in start,end */ + string getBackrefName(int start, int end) { this.namedBackreference(start, end, result) } + pragma[inline] private predicate isOctal(int index) { this.getChar(index) = [0 .. 7].toString() } @@ -592,51 +637,6 @@ abstract class RegexString extends StringLiteral { this.positiveLookbehindAssertionGroup(start, end) } - private predicate namedBackreference(int start, int end, string name) { - this.escapingChar(start) and - this.getChar(start + 1) = "k" and - this.getChar(start + 2) = "<" and - end = min(int i | i > start + 2 and this.getChar(i) = ">") + 1 and - name = this.getText().substring(start + 3, end - 2) - } - - private predicate numberedBackreference(int start, int end, int value) { - this.escapingChar(start) and - // starting with 0 makes it an octal escape - not this.getChar(start + 1) = "0" and - exists(string text, string svalue, int len | - end = start + len and - text = this.getText() and - len in [2 .. 3] - | - svalue = text.substring(start + 1, start + len) and - value = svalue.toInt() and - // value is composed of digits - forall(int i | i in [start + 1 .. start + len - 1] | this.getChar(i) = [0 .. 9].toString()) and - // a longer reference is not possible - not ( - len = 2 and - exists(text.substring(start + 1, start + len + 1).toInt()) - ) and - // 3 octal digits makes it an octal escape - not forall(int i | i in [start + 1 .. start + 4] | this.isOctal(i)) - // TODO: Inside a character set, all numeric escapes are treated as characters. - ) - } - - /** Holds if the text in the range start,end is a back reference */ - predicate backreference(int start, int end) { - this.numberedBackreference(start, end, _) - or - this.namedBackreference(start, end, _) - } - - /** Gets the number of the back reference in start,end */ - int getBackrefNumber(int start, int end) { this.numberedBackreference(start, end, result) } - - /** Gets the name, if it has one, of the back reference in start,end */ - string getBackrefName(int start, int end) { this.namedBackreference(start, end, result) } - private predicate baseItem(int start, int end) { this.character(start, end) and not exists(int x, int y | this.charSet(x, y) and x <= start and y >= end) From c7d30087d11ff203b8af00a9cacb8331f1d34fe0 Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Wed, 4 May 2022 15:39:51 +0100 Subject: [PATCH 67/91] Fix issue with named backrefs; add needed import --- java/ql/lib/semmle/code/java/regex/RegexTreeView.qll | 3 +++ java/ql/lib/semmle/code/java/regex/regex.qll | 2 +- .../code/java/security/performance/ReDoSUtilSpecific.qll | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll b/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll index 2db3da550e2..c3592634fa0 100644 --- a/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll +++ b/java/ql/lib/semmle/code/java/regex/RegexTreeView.qll @@ -258,6 +258,9 @@ class RegExpQuantifier extends RegExpTerm, TRegExpQuantifier { result.occursInRegex(re, start, part_end) } + /** Holds if this term may match zero times. */ + predicate mayBeEmpty() { maybe_empty = true } + /** Holds if this term may match an unlimited number of times. */ predicate mayRepeatForever() { may_repeat_forever = true } diff --git a/java/ql/lib/semmle/code/java/regex/regex.qll b/java/ql/lib/semmle/code/java/regex/regex.qll index 48f0f74580c..ff20b17b6fa 100644 --- a/java/ql/lib/semmle/code/java/regex/regex.qll +++ b/java/ql/lib/semmle/code/java/regex/regex.qll @@ -112,7 +112,7 @@ abstract class RegexString extends StringLiteral { this.getChar(start + 1) = "k" and this.getChar(start + 2) = "<" and end = min(int i | i > start + 2 and this.getChar(i) = ">") + 1 and - name = this.getText().substring(start + 3, end - 2) + name = this.getText().substring(start + 3, end - 1) } private predicate numberedBackreference(int start, int end, int value) { diff --git a/java/ql/lib/semmle/code/java/security/performance/ReDoSUtilSpecific.qll b/java/ql/lib/semmle/code/java/security/performance/ReDoSUtilSpecific.qll index 541c3ca8f36..d72d6770848 100644 --- a/java/ql/lib/semmle/code/java/security/performance/ReDoSUtilSpecific.qll +++ b/java/ql/lib/semmle/code/java/security/performance/ReDoSUtilSpecific.qll @@ -4,6 +4,7 @@ */ private import java +import semmle.code.FileSystem import semmle.code.java.regex.RegexTreeView /** From 64227c91090bbd58315bfdfc20bb1bbc64eefe05 Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Wed, 4 May 2022 15:58:30 +0100 Subject: [PATCH 68/91] Fix codescanning alerts --- java/ql/src/Security/CWE/CWE-730/PolynomialReDoS.ql | 1 + java/ql/src/Security/CWE/CWE-730/ReDoS.ql | 1 + java/ql/test/query-tests/security/CWE-730/PolynomialReDoS.ql | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/java/ql/src/Security/CWE/CWE-730/PolynomialReDoS.ql b/java/ql/src/Security/CWE/CWE-730/PolynomialReDoS.ql index e1907b39414..1a52173183f 100644 --- a/java/ql/src/Security/CWE/CWE-730/PolynomialReDoS.ql +++ b/java/ql/src/Security/CWE/CWE-730/PolynomialReDoS.ql @@ -4,6 +4,7 @@ * to match may be vulnerable to denial-of-service attacks. * @kind path-problem * @problem.severity warning + * @security-severity 7.5 * @precision high * @id java/polynomial-redos * @tags security diff --git a/java/ql/src/Security/CWE/CWE-730/ReDoS.ql b/java/ql/src/Security/CWE/CWE-730/ReDoS.ql index f72bfc3fc13..c5d9661a63b 100644 --- a/java/ql/src/Security/CWE/CWE-730/ReDoS.ql +++ b/java/ql/src/Security/CWE/CWE-730/ReDoS.ql @@ -5,6 +5,7 @@ * attacks. * @kind problem * @problem.severity error + * @security-severity 7.5 * @precision high * @id java/redos * @tags security diff --git a/java/ql/test/query-tests/security/CWE-730/PolynomialReDoS.ql b/java/ql/test/query-tests/security/CWE-730/PolynomialReDoS.ql index e5fb58d4794..19096cf6f95 100644 --- a/java/ql/test/query-tests/security/CWE-730/PolynomialReDoS.ql +++ b/java/ql/test/query-tests/security/CWE-730/PolynomialReDoS.ql @@ -5,7 +5,7 @@ import semmle.code.java.security.performance.PolynomialReDoSQuery class HasPolyRedos extends InlineExpectationsTest { HasPolyRedos() { this = "HasPolyRedos" } - override string getARelevantTag() { result = ["hasPolyRedos"] } + override string getARelevantTag() { result = "hasPolyRedos" } override predicate hasActualResult(Location location, string element, string tag, string value) { tag = "hasPolyRedos" and From 1f00ba812afdf756f26516e8f3cc5a650e6fa598 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Thu, 5 May 2022 10:22:52 +0200 Subject: [PATCH 69/91] move YAMLMappingLikeNode to the standard library --- .../ql/lib/semmle/javascript/Actions.qll | 72 +------------------ javascript/ql/lib/semmle/javascript/YAML.qll | 71 ++++++++++++++++++ .../Security/CWE-094/UntrustedCheckout.ql | 4 +- 3 files changed, 74 insertions(+), 73 deletions(-) diff --git a/javascript/ql/lib/semmle/javascript/Actions.qll b/javascript/ql/lib/semmle/javascript/Actions.qll index c74c2576483..297f70ad860 100644 --- a/javascript/ql/lib/semmle/javascript/Actions.qll +++ b/javascript/ql/lib/semmle/javascript/Actions.qll @@ -20,76 +20,6 @@ module Actions { } } - /** - * A YAML node that may contain sub-nodes. - * - * Actions are quite flexible in parsing YAML. - * - * For example: - * ``` - * on: pull_request - * ``` - * and - * ``` - * on: [pull_request] - * ``` - * and - * ``` - * on: - * pull_request: - * ``` - * - * are equivalent. - */ - class MappingOrSequenceOrScalar extends YAMLNode { - MappingOrSequenceOrScalar() { - this instanceof YAMLMapping - or - this instanceof YAMLSequence - or - this instanceof YAMLScalar - } - - /** Gets sub-name identified by `name`. */ - YAMLNode getNode(string name) { - exists(YAMLMapping mapping | - mapping = this and - result = mapping.lookup(name) - ) - or - exists(YAMLSequence sequence, YAMLNode node | - sequence = this and - sequence.getAChildNode() = node and - node.eval().toString() = name and - result = node - ) - or - exists(YAMLScalar scalar | - scalar = this and - scalar.getValue() = name and - result = scalar - ) - } - - /** Gets the number of elements in this mapping or sequence. */ - int getElementCount() { - exists(YAMLMapping mapping | - mapping = this and - result = mapping.getNumChild() / 2 - ) - or - exists(YAMLSequence sequence | - sequence = this and - result = sequence.getNumChild() - ) - or - exists(YAMLScalar scalar | - scalar = this and - result = 1 - ) - } - } - /** * An Actions workflow. This is a mapping at the top level of an Actions YAML workflow file. * See https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions. @@ -112,7 +42,7 @@ module Actions { * An Actions On trigger within a workflow. * See https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#on. */ - class On extends YAMLNode, MappingOrSequenceOrScalar { + class On extends YAMLNode, YAMLMappingLikeNode { Workflow workflow; On() { workflow.lookup("on") = this } diff --git a/javascript/ql/lib/semmle/javascript/YAML.qll b/javascript/ql/lib/semmle/javascript/YAML.qll index ac1ac8380e4..49e0d28ae28 100644 --- a/javascript/ql/lib/semmle/javascript/YAML.qll +++ b/javascript/ql/lib/semmle/javascript/YAML.qll @@ -441,3 +441,74 @@ class YAMLParseError extends @yaml_error, Error { override string toString() { result = this.getMessage() } } + +/** + * A YAML node that may contain sub-nodes that can be identified by a name. + * I.e. a mapping, sequence, or scalar. + * + * Is used in e.g. GithHub Actions, which is quite flexible in parsing YAML. + * + * For example: + * ``` + * on: pull_request + * ``` + * and + * ``` + * on: [pull_request] + * ``` + * and + * ``` + * on: + * pull_request: + * ``` + * + * are equivalent. + */ +class YAMLMappingLikeNode extends YAMLNode { + YAMLMappingLikeNode() { + this instanceof YAMLMapping + or + this instanceof YAMLSequence + or + this instanceof YAMLScalar + } + + /** Gets sub-name identified by `name`. */ + YAMLNode getNode(string name) { + exists(YAMLMapping mapping | + mapping = this and + result = mapping.lookup(name) + ) + or + exists(YAMLSequence sequence, YAMLNode node | + sequence = this and + sequence.getAChildNode() = node and + node.eval().toString() = name and + result = node + ) + or + exists(YAMLScalar scalar | + scalar = this and + scalar.getValue() = name and + result = scalar + ) + } + + /** Gets the number of elements in this mapping or sequence. */ + int getElementCount() { + exists(YAMLMapping mapping | + mapping = this and + result = mapping.getNumChild() / 2 + ) + or + exists(YAMLSequence sequence | + sequence = this and + result = sequence.getNumChild() + ) + or + exists(YAMLScalar scalar | + scalar = this and + result = 1 + ) + } +} diff --git a/javascript/ql/src/experimental/Security/CWE-094/UntrustedCheckout.ql b/javascript/ql/src/experimental/Security/CWE-094/UntrustedCheckout.ql index cf19d07015d..0b91c83b502 100644 --- a/javascript/ql/src/experimental/Security/CWE-094/UntrustedCheckout.ql +++ b/javascript/ql/src/experimental/Security/CWE-094/UntrustedCheckout.ql @@ -78,7 +78,7 @@ class ProbableJob extends Actions::Job { /** * An action step that doesn't contain `actor` or `label` check in `if:` or */ -class ProbablePullRequestTarget extends Actions::On, Actions::MappingOrSequenceOrScalar { +class ProbablePullRequestTarget extends Actions::On, YAMLMappingLikeNode { ProbablePullRequestTarget() { exists(YAMLNode prtNode | // The `on:` is triggered on `pull_request_target` @@ -88,7 +88,7 @@ class ProbablePullRequestTarget extends Actions::On, Actions::MappingOrSequenceO not exists(prtNode.getAChild()) or // or has the filter, that is something else than just [labeled] - exists(Actions::MappingOrSequenceOrScalar prt, Actions::MappingOrSequenceOrScalar types | + exists(YAMLMappingLikeNode prt, YAMLMappingLikeNode types | types = prt.getNode("types") and prtNode = prt and ( From 9ea0f71581a3ebcc610018bc85fb8547d4bfb4c1 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Thu, 5 May 2022 10:28:00 +0200 Subject: [PATCH 70/91] convert TODO to a note in Actions::Uses --- javascript/ql/lib/semmle/javascript/Actions.qll | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/javascript/ql/lib/semmle/javascript/Actions.qll b/javascript/ql/lib/semmle/javascript/Actions.qll index 297f70ad860..cb50d410ac3 100644 --- a/javascript/ql/lib/semmle/javascript/Actions.qll +++ b/javascript/ql/lib/semmle/javascript/Actions.qll @@ -152,7 +152,8 @@ module Actions { * ``` * uses: actions/checkout@v2 * ``` - * TODO: Does not currently handle local repository references, e.g. `.github/actions/action-name`. + * + * Does not handle local repository references, e.g. `.github/actions/action-name`. */ class Uses extends YAMLNode, YAMLScalar { Step step; From dc1dc2a33ad209aeb359eecb4d930db2ac83ce8e Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Thu, 5 May 2022 10:40:08 +0200 Subject: [PATCH 71/91] parse the uses field in the getters instead of the charpred --- .../ql/lib/semmle/javascript/Actions.qll | 33 +++++++++---------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/javascript/ql/lib/semmle/javascript/Actions.qll b/javascript/ql/lib/semmle/javascript/Actions.qll index cb50d410ac3..9d5ac0c7e58 100644 --- a/javascript/ql/lib/semmle/javascript/Actions.qll +++ b/javascript/ql/lib/semmle/javascript/Actions.qll @@ -144,6 +144,15 @@ module Actions { Step getStep() { result = step } } + /** + * Gets a regular expression that parses an `owner/repo@version` reference within a `uses` field in an Actions job step. + * The capture groups are: + * 1: The owner of the repository where the Action comes from, e.g. `actions` in `actions/checkout@v2` + * 2: The name of the repository where the Action comes from, e.g. `checkout` in `actions/checkout@v2`. + * 3: The version reference used when checking out the Action, e.g. `v2` in `actions/checkout@v2`. + */ + private string usesParser() { result = "([^/]+)/([^/@]+)@(.+)" } + /** * A `uses` field within an Actions job step, which references an action as a reusable unit of code. * See https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idstepsuses. @@ -157,31 +166,21 @@ module Actions { */ class Uses extends YAMLNode, YAMLScalar { Step step; - /** The owner of the repository where the Action comes from, e.g. `actions` in `actions/checkout@v2`. */ - string repositoryOwner; - /** The name of the repository where the Action comes from, e.g. `checkout` in `actions/checkout@v2`. */ - string repositoryName; - /** The version reference used when checking out the Action, e.g. `v2` in `actions/checkout@v2`. */ - string version; - Uses() { - step.lookup("uses") = this and - // Simple regular expression to split up an Action reference `owner/repo@version` into its components. - exists(string regexp | regexp = "([^/]+)/([^/@]+)@(.+)" | - repositoryOwner = this.getValue().regexpCapture(regexp, 1) and - repositoryName = this.getValue().regexpCapture(regexp, 2) and - version = this.getValue().regexpCapture(regexp, 3) - ) - } + Uses() { step.lookup("uses") = this } /** Gets the step this field belongs to. */ Step getStep() { result = step } /** Gets the owner and name of the repository where the Action comes from, e.g. `actions/checkout` in `actions/checkout@v2`. */ - string getGitHubRepository() { result = repositoryOwner + "/" + repositoryName } + string getGitHubRepository() { + result = + this.getValue().regexpCapture(usesParser(), 1) + "/" + + this.getValue().regexpCapture(usesParser(), 2) + } /** Gets the version reference used when checking out the Action, e.g. `v2` in `actions/checkout@v2`. */ - string getVersion() { result = version } + string getVersion() { result = this.getValue().regexpCapture(usesParser(), 3) } } /** From c0152a46bc91e43e2cb20c64cf7527678f39a075 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Thu, 5 May 2022 11:02:47 +0200 Subject: [PATCH 72/91] rename getAReferencedExpression to getASimpleReferenceExpression and add examples of what it can parse --- javascript/ql/lib/semmle/javascript/Actions.qll | 6 +++--- javascript/ql/src/Security/CWE-094/ExpressionInjection.ql | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/javascript/ql/lib/semmle/javascript/Actions.qll b/javascript/ql/lib/semmle/javascript/Actions.qll index 9d5ac0c7e58..f5b2c39b064 100644 --- a/javascript/ql/lib/semmle/javascript/Actions.qll +++ b/javascript/ql/lib/semmle/javascript/Actions.qll @@ -237,12 +237,12 @@ module Actions { /** * Holds if `${{ e }}` is a GitHub Actions expression evaluated within this `run` command. * See https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions. + * Only finds simple expressions like `${{ github.event.comment.body }}`, where the expression contains only alphanumeric characters, underscores, dots, or dashes. + * Does not identify more complicated expressions like `${{ fromJSON(env.time) }}`, or ${{ format('{{Hello {0}!}}', github.event.head_commit.author.name) }} */ - string getAReferencedExpression() { + string getASimpleReferenceExpression() { // We use `regexpFind` to obtain *all* matches of `${{...}}`, // not just the last (greedy match) or first (reluctant match). - // TODO: This only handles expression strings that refer to contexts. - // It does not handle operators within the expression. result = this.getValue() .regexpFind("\\$\\{\\{\\s*[A-Za-z0-9_\\.\\-]+\\s*\\}\\}", _, _) diff --git a/javascript/ql/src/Security/CWE-094/ExpressionInjection.ql b/javascript/ql/src/Security/CWE-094/ExpressionInjection.ql index 80af8dae82a..399c9e4c9bd 100644 --- a/javascript/ql/src/Security/CWE-094/ExpressionInjection.ql +++ b/javascript/ql/src/Security/CWE-094/ExpressionInjection.ql @@ -80,7 +80,7 @@ private predicate isExternalUserControlledDiscussion(string context) { from Actions::Run run, string context, Actions::On on where - run.getAReferencedExpression() = context and + run.getASimpleReferenceExpression() = context and run.getStep().getJob().getWorkflow().getOn() = on and ( exists(on.getNode("issues")) and From 0c0e280637d2affc7f65fada41ec99e3585b64a0 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Thu, 5 May 2022 12:12:29 +0200 Subject: [PATCH 73/91] update the qhelp to mention that the GITHUB_TOKEN only sometimes has write-access --- .../CWE-094/ExpressionInjection.qhelp | 26 +++++-------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/javascript/ql/src/Security/CWE-094/ExpressionInjection.qhelp b/javascript/ql/src/Security/CWE-094/ExpressionInjection.qhelp index b9a3248408f..8068697d878 100644 --- a/javascript/ql/src/Security/CWE-094/ExpressionInjection.qhelp +++ b/javascript/ql/src/Security/CWE-094/ExpressionInjection.qhelp @@ -2,59 +2,47 @@ "-//Semmle//qhelp//EN" "qhelp.dtd"> - -

    - Using user-controlled input in GitHub Actions may lead to code injection in contexts like run: or script:. -

    -

    Code injection in GitHub actions may allow an attacker to exfiltrate the temporary GitHub repository authorization token. - The token has write access to the repository, and thus an attacker - can use it to modify the repository. + The token might have write access to the repository, and thus an attacker + might be able to use it to modify the repository.

    -     -

    The best practice to avoid code injection vulnerabilities in GitHub workflows is to set the untrusted input value of the expression to an intermediate environment variable.

    - +

    + It is also recommended to limit the permissions of any tokens used + by a workflow such as the the GITHUB_TOKEN. +

    -

    - The following example lets a user inject an arbitrary shell command: -

    -

    - The following example uses shell syntax to read the environment variable and will prevent the attack: -

    - -
  • GitHub Security Lab Research: Keeping your GitHub Actions and workflows secure: Untrusted input.
  • GitHub Docs: Security hardening for GitHub Actions.
    • +
  • GitHub Docs: Permissions for the GITHUB_TOKEN.
    •  -     From bfabfc3601e90caf1a079cd4c14a94b860766883 Mon Sep 17 00:00:00 2001 From: Tom Hvitved Date: Tue, 10 May 2022 12:40:46 +0200 Subject: [PATCH 74/91] Data flow: Add `Configuration::includeHiddenNodes()` --- .../ruby/dataflow/internal/DataFlowImpl.qll | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl.qll b/ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl.qll index 89a35b00fa6..c0fdf294dd4 100644 --- a/ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl.qll +++ b/ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl.qll @@ -170,6 +170,14 @@ abstract class Configuration extends string { */ int explorationLimit() { none() } + /** + * Holds if hidden nodes should be included in the data flow graph. + * + * This feature should only be used for debugging or when the data flow graph + * is not visualized (for example in a `path-problem` query). + */ + predicate includeHiddenNodes() { none() } + /** * Holds if there is a partial data flow path from `source` to `node`. The * approximate distance between `node` and the closest source is `dist` and @@ -3815,11 +3823,14 @@ abstract private class PathNodeImpl extends PathNode { abstract NodeEx getNodeEx(); predicate isHidden() { - hiddenNode(this.getNodeEx().asNode()) and - not this.isSource() and - not this instanceof PathNodeSink - or - this.getNodeEx() instanceof TNodeImplicitRead + not this.getConfiguration().includeHiddenNodes() and + ( + hiddenNode(this.getNodeEx().asNode()) and + not this.isSource() and + not this instanceof PathNodeSink + or + this.getNodeEx() instanceof TNodeImplicitRead + ) } private string ppAp() { From 712fe002b9ac17e334f431d2fe8aaa4e798663c9 Mon Sep 17 00:00:00 2001 From: Tom Hvitved Date: Tue, 10 May 2022 12:41:10 +0200 Subject: [PATCH 75/91] Data flow: Sync files --- .../cpp/dataflow/internal/DataFlowImpl.qll | 21 ++++++++++++++----- .../cpp/dataflow/internal/DataFlowImpl2.qll | 21 ++++++++++++++----- .../cpp/dataflow/internal/DataFlowImpl3.qll | 21 ++++++++++++++----- .../cpp/dataflow/internal/DataFlowImpl4.qll | 21 ++++++++++++++----- .../dataflow/internal/DataFlowImplLocal.qll | 21 ++++++++++++++----- .../cpp/ir/dataflow/internal/DataFlowImpl.qll | 21 ++++++++++++++----- .../ir/dataflow/internal/DataFlowImpl2.qll | 21 ++++++++++++++----- .../ir/dataflow/internal/DataFlowImpl3.qll | 21 ++++++++++++++----- .../ir/dataflow/internal/DataFlowImpl4.qll | 21 ++++++++++++++----- .../csharp/dataflow/internal/DataFlowImpl.qll | 21 ++++++++++++++----- .../dataflow/internal/DataFlowImpl2.qll | 21 ++++++++++++++----- .../dataflow/internal/DataFlowImpl3.qll | 21 ++++++++++++++----- .../dataflow/internal/DataFlowImpl4.qll | 21 ++++++++++++++----- .../dataflow/internal/DataFlowImpl5.qll | 21 ++++++++++++++----- .../java/dataflow/internal/DataFlowImpl.qll | 21 ++++++++++++++----- .../java/dataflow/internal/DataFlowImpl2.qll | 21 ++++++++++++++----- .../java/dataflow/internal/DataFlowImpl3.qll | 21 ++++++++++++++----- .../java/dataflow/internal/DataFlowImpl4.qll | 21 ++++++++++++++----- .../java/dataflow/internal/DataFlowImpl5.qll | 21 ++++++++++++++----- .../java/dataflow/internal/DataFlowImpl6.qll | 21 ++++++++++++++----- .../DataFlowImplForOnActivityResult.qll | 21 ++++++++++++++----- .../DataFlowImplForSerializability.qll | 21 ++++++++++++++----- .../dataflow/new/internal/DataFlowImpl.qll | 21 ++++++++++++++----- .../dataflow/new/internal/DataFlowImpl2.qll | 21 ++++++++++++++----- .../dataflow/new/internal/DataFlowImpl3.qll | 21 ++++++++++++++----- .../dataflow/new/internal/DataFlowImpl4.qll | 21 ++++++++++++++----- .../ruby/dataflow/internal/DataFlowImpl2.qll | 21 ++++++++++++++----- .../internal/DataFlowImplForLibraries.qll | 21 ++++++++++++++----- 28 files changed, 448 insertions(+), 140 deletions(-) diff --git a/cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl.qll b/cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl.qll index 89a35b00fa6..c0fdf294dd4 100644 --- a/cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl.qll +++ b/cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl.qll @@ -170,6 +170,14 @@ abstract class Configuration extends string { */ int explorationLimit() { none() } + /** + * Holds if hidden nodes should be included in the data flow graph. + * + * This feature should only be used for debugging or when the data flow graph + * is not visualized (for example in a `path-problem` query). + */ + predicate includeHiddenNodes() { none() } + /** * Holds if there is a partial data flow path from `source` to `node`. The * approximate distance between `node` and the closest source is `dist` and @@ -3815,11 +3823,14 @@ abstract private class PathNodeImpl extends PathNode { abstract NodeEx getNodeEx(); predicate isHidden() { - hiddenNode(this.getNodeEx().asNode()) and - not this.isSource() and - not this instanceof PathNodeSink - or - this.getNodeEx() instanceof TNodeImplicitRead + not this.getConfiguration().includeHiddenNodes() and + ( + hiddenNode(this.getNodeEx().asNode()) and + not this.isSource() and + not this instanceof PathNodeSink + or + this.getNodeEx() instanceof TNodeImplicitRead + ) } private string ppAp() { diff --git a/cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl2.qll b/cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl2.qll index 89a35b00fa6..c0fdf294dd4 100644 --- a/cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl2.qll +++ b/cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl2.qll @@ -170,6 +170,14 @@ abstract class Configuration extends string { */ int explorationLimit() { none() } + /** + * Holds if hidden nodes should be included in the data flow graph. + * + * This feature should only be used for debugging or when the data flow graph + * is not visualized (for example in a `path-problem` query). + */ + predicate includeHiddenNodes() { none() } + /** * Holds if there is a partial data flow path from `source` to `node`. The * approximate distance between `node` and the closest source is `dist` and @@ -3815,11 +3823,14 @@ abstract private class PathNodeImpl extends PathNode { abstract NodeEx getNodeEx(); predicate isHidden() { - hiddenNode(this.getNodeEx().asNode()) and - not this.isSource() and - not this instanceof PathNodeSink - or - this.getNodeEx() instanceof TNodeImplicitRead + not this.getConfiguration().includeHiddenNodes() and + ( + hiddenNode(this.getNodeEx().asNode()) and + not this.isSource() and + not this instanceof PathNodeSink + or + this.getNodeEx() instanceof TNodeImplicitRead + ) } private string ppAp() { diff --git a/cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl3.qll b/cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl3.qll index 89a35b00fa6..c0fdf294dd4 100644 --- a/cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl3.qll +++ b/cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl3.qll @@ -170,6 +170,14 @@ abstract class Configuration extends string { */ int explorationLimit() { none() } + /** + * Holds if hidden nodes should be included in the data flow graph. + * + * This feature should only be used for debugging or when the data flow graph + * is not visualized (for example in a `path-problem` query). + */ + predicate includeHiddenNodes() { none() } + /** * Holds if there is a partial data flow path from `source` to `node`. The * approximate distance between `node` and the closest source is `dist` and @@ -3815,11 +3823,14 @@ abstract private class PathNodeImpl extends PathNode { abstract NodeEx getNodeEx(); predicate isHidden() { - hiddenNode(this.getNodeEx().asNode()) and - not this.isSource() and - not this instanceof PathNodeSink - or - this.getNodeEx() instanceof TNodeImplicitRead + not this.getConfiguration().includeHiddenNodes() and + ( + hiddenNode(this.getNodeEx().asNode()) and + not this.isSource() and + not this instanceof PathNodeSink + or + this.getNodeEx() instanceof TNodeImplicitRead + ) } private string ppAp() { diff --git a/cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl4.qll b/cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl4.qll index 89a35b00fa6..c0fdf294dd4 100644 --- a/cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl4.qll +++ b/cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl4.qll @@ -170,6 +170,14 @@ abstract class Configuration extends string { */ int explorationLimit() { none() } + /** + * Holds if hidden nodes should be included in the data flow graph. + * + * This feature should only be used for debugging or when the data flow graph + * is not visualized (for example in a `path-problem` query). + */ + predicate includeHiddenNodes() { none() } + /** * Holds if there is a partial data flow path from `source` to `node`. The * approximate distance between `node` and the closest source is `dist` and @@ -3815,11 +3823,14 @@ abstract private class PathNodeImpl extends PathNode { abstract NodeEx getNodeEx(); predicate isHidden() { - hiddenNode(this.getNodeEx().asNode()) and - not this.isSource() and - not this instanceof PathNodeSink - or - this.getNodeEx() instanceof TNodeImplicitRead + not this.getConfiguration().includeHiddenNodes() and + ( + hiddenNode(this.getNodeEx().asNode()) and + not this.isSource() and + not this instanceof PathNodeSink + or + this.getNodeEx() instanceof TNodeImplicitRead + ) } private string ppAp() { diff --git a/cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplLocal.qll b/cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplLocal.qll index 89a35b00fa6..c0fdf294dd4 100644 --- a/cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplLocal.qll +++ b/cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplLocal.qll @@ -170,6 +170,14 @@ abstract class Configuration extends string { */ int explorationLimit() { none() } + /** + * Holds if hidden nodes should be included in the data flow graph. + * + * This feature should only be used for debugging or when the data flow graph + * is not visualized (for example in a `path-problem` query). + */ + predicate includeHiddenNodes() { none() } + /** * Holds if there is a partial data flow path from `source` to `node`. The * approximate distance between `node` and the closest source is `dist` and @@ -3815,11 +3823,14 @@ abstract private class PathNodeImpl extends PathNode { abstract NodeEx getNodeEx(); predicate isHidden() { - hiddenNode(this.getNodeEx().asNode()) and - not this.isSource() and - not this instanceof PathNodeSink - or - this.getNodeEx() instanceof TNodeImplicitRead + not this.getConfiguration().includeHiddenNodes() and + ( + hiddenNode(this.getNodeEx().asNode()) and + not this.isSource() and + not this instanceof PathNodeSink + or + this.getNodeEx() instanceof TNodeImplicitRead + ) } private string ppAp() { diff --git a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl.qll b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl.qll index 89a35b00fa6..c0fdf294dd4 100644 --- a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl.qll +++ b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl.qll @@ -170,6 +170,14 @@ abstract class Configuration extends string { */ int explorationLimit() { none() } + /** + * Holds if hidden nodes should be included in the data flow graph. + * + * This feature should only be used for debugging or when the data flow graph + * is not visualized (for example in a `path-problem` query). + */ + predicate includeHiddenNodes() { none() } + /** * Holds if there is a partial data flow path from `source` to `node`. The * approximate distance between `node` and the closest source is `dist` and @@ -3815,11 +3823,14 @@ abstract private class PathNodeImpl extends PathNode { abstract NodeEx getNodeEx(); predicate isHidden() { - hiddenNode(this.getNodeEx().asNode()) and - not this.isSource() and - not this instanceof PathNodeSink - or - this.getNodeEx() instanceof TNodeImplicitRead + not this.getConfiguration().includeHiddenNodes() and + ( + hiddenNode(this.getNodeEx().asNode()) and + not this.isSource() and + not this instanceof PathNodeSink + or + this.getNodeEx() instanceof TNodeImplicitRead + ) } private string ppAp() { diff --git a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl2.qll b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl2.qll index 89a35b00fa6..c0fdf294dd4 100644 --- a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl2.qll +++ b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl2.qll @@ -170,6 +170,14 @@ abstract class Configuration extends string { */ int explorationLimit() { none() } + /** + * Holds if hidden nodes should be included in the data flow graph. + * + * This feature should only be used for debugging or when the data flow graph + * is not visualized (for example in a `path-problem` query). + */ + predicate includeHiddenNodes() { none() } + /** * Holds if there is a partial data flow path from `source` to `node`. The * approximate distance between `node` and the closest source is `dist` and @@ -3815,11 +3823,14 @@ abstract private class PathNodeImpl extends PathNode { abstract NodeEx getNodeEx(); predicate isHidden() { - hiddenNode(this.getNodeEx().asNode()) and - not this.isSource() and - not this instanceof PathNodeSink - or - this.getNodeEx() instanceof TNodeImplicitRead + not this.getConfiguration().includeHiddenNodes() and + ( + hiddenNode(this.getNodeEx().asNode()) and + not this.isSource() and + not this instanceof PathNodeSink + or + this.getNodeEx() instanceof TNodeImplicitRead + ) } private string ppAp() { diff --git a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl3.qll b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl3.qll index 89a35b00fa6..c0fdf294dd4 100644 --- a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl3.qll +++ b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl3.qll @@ -170,6 +170,14 @@ abstract class Configuration extends string { */ int explorationLimit() { none() } + /** + * Holds if hidden nodes should be included in the data flow graph. + * + * This feature should only be used for debugging or when the data flow graph + * is not visualized (for example in a `path-problem` query). + */ + predicate includeHiddenNodes() { none() } + /** * Holds if there is a partial data flow path from `source` to `node`. The * approximate distance between `node` and the closest source is `dist` and @@ -3815,11 +3823,14 @@ abstract private class PathNodeImpl extends PathNode { abstract NodeEx getNodeEx(); predicate isHidden() { - hiddenNode(this.getNodeEx().asNode()) and - not this.isSource() and - not this instanceof PathNodeSink - or - this.getNodeEx() instanceof TNodeImplicitRead + not this.getConfiguration().includeHiddenNodes() and + ( + hiddenNode(this.getNodeEx().asNode()) and + not this.isSource() and + not this instanceof PathNodeSink + or + this.getNodeEx() instanceof TNodeImplicitRead + ) } private string ppAp() { diff --git a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl4.qll b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl4.qll index 89a35b00fa6..c0fdf294dd4 100644 --- a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl4.qll +++ b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl4.qll @@ -170,6 +170,14 @@ abstract class Configuration extends string { */ int explorationLimit() { none() } + /** + * Holds if hidden nodes should be included in the data flow graph. + * + * This feature should only be used for debugging or when the data flow graph + * is not visualized (for example in a `path-problem` query). + */ + predicate includeHiddenNodes() { none() } + /** * Holds if there is a partial data flow path from `source` to `node`. The * approximate distance between `node` and the closest source is `dist` and @@ -3815,11 +3823,14 @@ abstract private class PathNodeImpl extends PathNode { abstract NodeEx getNodeEx(); predicate isHidden() { - hiddenNode(this.getNodeEx().asNode()) and - not this.isSource() and - not this instanceof PathNodeSink - or - this.getNodeEx() instanceof TNodeImplicitRead + not this.getConfiguration().includeHiddenNodes() and + ( + hiddenNode(this.getNodeEx().asNode()) and + not this.isSource() and + not this instanceof PathNodeSink + or + this.getNodeEx() instanceof TNodeImplicitRead + ) } private string ppAp() { diff --git a/csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl.qll b/csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl.qll index 89a35b00fa6..c0fdf294dd4 100644 --- a/csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl.qll +++ b/csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl.qll @@ -170,6 +170,14 @@ abstract class Configuration extends string { */ int explorationLimit() { none() } + /** + * Holds if hidden nodes should be included in the data flow graph. + * + * This feature should only be used for debugging or when the data flow graph + * is not visualized (for example in a `path-problem` query). + */ + predicate includeHiddenNodes() { none() } + /** * Holds if there is a partial data flow path from `source` to `node`. The * approximate distance between `node` and the closest source is `dist` and @@ -3815,11 +3823,14 @@ abstract private class PathNodeImpl extends PathNode { abstract NodeEx getNodeEx(); predicate isHidden() { - hiddenNode(this.getNodeEx().asNode()) and - not this.isSource() and - not this instanceof PathNodeSink - or - this.getNodeEx() instanceof TNodeImplicitRead + not this.getConfiguration().includeHiddenNodes() and + ( + hiddenNode(this.getNodeEx().asNode()) and + not this.isSource() and + not this instanceof PathNodeSink + or + this.getNodeEx() instanceof TNodeImplicitRead + ) } private string ppAp() { diff --git a/csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl2.qll b/csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl2.qll index 89a35b00fa6..c0fdf294dd4 100644 --- a/csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl2.qll +++ b/csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl2.qll @@ -170,6 +170,14 @@ abstract class Configuration extends string { */ int explorationLimit() { none() } + /** + * Holds if hidden nodes should be included in the data flow graph. + * + * This feature should only be used for debugging or when the data flow graph + * is not visualized (for example in a `path-problem` query). + */ + predicate includeHiddenNodes() { none() } + /** * Holds if there is a partial data flow path from `source` to `node`. The * approximate distance between `node` and the closest source is `dist` and @@ -3815,11 +3823,14 @@ abstract private class PathNodeImpl extends PathNode { abstract NodeEx getNodeEx(); predicate isHidden() { - hiddenNode(this.getNodeEx().asNode()) and - not this.isSource() and - not this instanceof PathNodeSink - or - this.getNodeEx() instanceof TNodeImplicitRead + not this.getConfiguration().includeHiddenNodes() and + ( + hiddenNode(this.getNodeEx().asNode()) and + not this.isSource() and + not this instanceof PathNodeSink + or + this.getNodeEx() instanceof TNodeImplicitRead + ) } private string ppAp() { diff --git a/csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl3.qll b/csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl3.qll index 89a35b00fa6..c0fdf294dd4 100644 --- a/csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl3.qll +++ b/csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl3.qll @@ -170,6 +170,14 @@ abstract class Configuration extends string { */ int explorationLimit() { none() } + /** + * Holds if hidden nodes should be included in the data flow graph. + * + * This feature should only be used for debugging or when the data flow graph + * is not visualized (for example in a `path-problem` query). + */ + predicate includeHiddenNodes() { none() } + /** * Holds if there is a partial data flow path from `source` to `node`. The * approximate distance between `node` and the closest source is `dist` and @@ -3815,11 +3823,14 @@ abstract private class PathNodeImpl extends PathNode { abstract NodeEx getNodeEx(); predicate isHidden() { - hiddenNode(this.getNodeEx().asNode()) and - not this.isSource() and - not this instanceof PathNodeSink - or - this.getNodeEx() instanceof TNodeImplicitRead + not this.getConfiguration().includeHiddenNodes() and + ( + hiddenNode(this.getNodeEx().asNode()) and + not this.isSource() and + not this instanceof PathNodeSink + or + this.getNodeEx() instanceof TNodeImplicitRead + ) } private string ppAp() { diff --git a/csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl4.qll b/csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl4.qll index 89a35b00fa6..c0fdf294dd4 100644 --- a/csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl4.qll +++ b/csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl4.qll @@ -170,6 +170,14 @@ abstract class Configuration extends string { */ int explorationLimit() { none() } + /** + * Holds if hidden nodes should be included in the data flow graph. + * + * This feature should only be used for debugging or when the data flow graph + * is not visualized (for example in a `path-problem` query). + */ + predicate includeHiddenNodes() { none() } + /** * Holds if there is a partial data flow path from `source` to `node`. The * approximate distance between `node` and the closest source is `dist` and @@ -3815,11 +3823,14 @@ abstract private class PathNodeImpl extends PathNode { abstract NodeEx getNodeEx(); predicate isHidden() { - hiddenNode(this.getNodeEx().asNode()) and - not this.isSource() and - not this instanceof PathNodeSink - or - this.getNodeEx() instanceof TNodeImplicitRead + not this.getConfiguration().includeHiddenNodes() and + ( + hiddenNode(this.getNodeEx().asNode()) and + not this.isSource() and + not this instanceof PathNodeSink + or + this.getNodeEx() instanceof TNodeImplicitRead + ) } private string ppAp() { diff --git a/csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl5.qll b/csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl5.qll index 89a35b00fa6..c0fdf294dd4 100644 --- a/csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl5.qll +++ b/csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl5.qll @@ -170,6 +170,14 @@ abstract class Configuration extends string { */ int explorationLimit() { none() } + /** + * Holds if hidden nodes should be included in the data flow graph. + * + * This feature should only be used for debugging or when the data flow graph + * is not visualized (for example in a `path-problem` query). + */ + predicate includeHiddenNodes() { none() } + /** * Holds if there is a partial data flow path from `source` to `node`. The * approximate distance between `node` and the closest source is `dist` and @@ -3815,11 +3823,14 @@ abstract private class PathNodeImpl extends PathNode { abstract NodeEx getNodeEx(); predicate isHidden() { - hiddenNode(this.getNodeEx().asNode()) and - not this.isSource() and - not this instanceof PathNodeSink - or - this.getNodeEx() instanceof TNodeImplicitRead + not this.getConfiguration().includeHiddenNodes() and + ( + hiddenNode(this.getNodeEx().asNode()) and + not this.isSource() and + not this instanceof PathNodeSink + or + this.getNodeEx() instanceof TNodeImplicitRead + ) } private string ppAp() { diff --git a/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl.qll b/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl.qll index 89a35b00fa6..c0fdf294dd4 100644 --- a/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl.qll +++ b/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl.qll @@ -170,6 +170,14 @@ abstract class Configuration extends string { */ int explorationLimit() { none() } + /** + * Holds if hidden nodes should be included in the data flow graph. + * + * This feature should only be used for debugging or when the data flow graph + * is not visualized (for example in a `path-problem` query). + */ + predicate includeHiddenNodes() { none() } + /** * Holds if there is a partial data flow path from `source` to `node`. The * approximate distance between `node` and the closest source is `dist` and @@ -3815,11 +3823,14 @@ abstract private class PathNodeImpl extends PathNode { abstract NodeEx getNodeEx(); predicate isHidden() { - hiddenNode(this.getNodeEx().asNode()) and - not this.isSource() and - not this instanceof PathNodeSink - or - this.getNodeEx() instanceof TNodeImplicitRead + not this.getConfiguration().includeHiddenNodes() and + ( + hiddenNode(this.getNodeEx().asNode()) and + not this.isSource() and + not this instanceof PathNodeSink + or + this.getNodeEx() instanceof TNodeImplicitRead + ) } private string ppAp() { diff --git a/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl2.qll b/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl2.qll index 89a35b00fa6..c0fdf294dd4 100644 --- a/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl2.qll +++ b/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl2.qll @@ -170,6 +170,14 @@ abstract class Configuration extends string { */ int explorationLimit() { none() } + /** + * Holds if hidden nodes should be included in the data flow graph. + * + * This feature should only be used for debugging or when the data flow graph + * is not visualized (for example in a `path-problem` query). + */ + predicate includeHiddenNodes() { none() } + /** * Holds if there is a partial data flow path from `source` to `node`. The * approximate distance between `node` and the closest source is `dist` and @@ -3815,11 +3823,14 @@ abstract private class PathNodeImpl extends PathNode { abstract NodeEx getNodeEx(); predicate isHidden() { - hiddenNode(this.getNodeEx().asNode()) and - not this.isSource() and - not this instanceof PathNodeSink - or - this.getNodeEx() instanceof TNodeImplicitRead + not this.getConfiguration().includeHiddenNodes() and + ( + hiddenNode(this.getNodeEx().asNode()) and + not this.isSource() and + not this instanceof PathNodeSink + or + this.getNodeEx() instanceof TNodeImplicitRead + ) } private string ppAp() { diff --git a/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl3.qll b/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl3.qll index 89a35b00fa6..c0fdf294dd4 100644 --- a/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl3.qll +++ b/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl3.qll @@ -170,6 +170,14 @@ abstract class Configuration extends string { */ int explorationLimit() { none() } + /** + * Holds if hidden nodes should be included in the data flow graph. + * + * This feature should only be used for debugging or when the data flow graph + * is not visualized (for example in a `path-problem` query). + */ + predicate includeHiddenNodes() { none() } + /** * Holds if there is a partial data flow path from `source` to `node`. The * approximate distance between `node` and the closest source is `dist` and @@ -3815,11 +3823,14 @@ abstract private class PathNodeImpl extends PathNode { abstract NodeEx getNodeEx(); predicate isHidden() { - hiddenNode(this.getNodeEx().asNode()) and - not this.isSource() and - not this instanceof PathNodeSink - or - this.getNodeEx() instanceof TNodeImplicitRead + not this.getConfiguration().includeHiddenNodes() and + ( + hiddenNode(this.getNodeEx().asNode()) and + not this.isSource() and + not this instanceof PathNodeSink + or + this.getNodeEx() instanceof TNodeImplicitRead + ) } private string ppAp() { diff --git a/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl4.qll b/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl4.qll index 89a35b00fa6..c0fdf294dd4 100644 --- a/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl4.qll +++ b/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl4.qll @@ -170,6 +170,14 @@ abstract class Configuration extends string { */ int explorationLimit() { none() } + /** + * Holds if hidden nodes should be included in the data flow graph. + * + * This feature should only be used for debugging or when the data flow graph + * is not visualized (for example in a `path-problem` query). + */ + predicate includeHiddenNodes() { none() } + /** * Holds if there is a partial data flow path from `source` to `node`. The * approximate distance between `node` and the closest source is `dist` and @@ -3815,11 +3823,14 @@ abstract private class PathNodeImpl extends PathNode { abstract NodeEx getNodeEx(); predicate isHidden() { - hiddenNode(this.getNodeEx().asNode()) and - not this.isSource() and - not this instanceof PathNodeSink - or - this.getNodeEx() instanceof TNodeImplicitRead + not this.getConfiguration().includeHiddenNodes() and + ( + hiddenNode(this.getNodeEx().asNode()) and + not this.isSource() and + not this instanceof PathNodeSink + or + this.getNodeEx() instanceof TNodeImplicitRead + ) } private string ppAp() { diff --git a/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl5.qll b/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl5.qll index 89a35b00fa6..c0fdf294dd4 100644 --- a/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl5.qll +++ b/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl5.qll @@ -170,6 +170,14 @@ abstract class Configuration extends string { */ int explorationLimit() { none() } + /** + * Holds if hidden nodes should be included in the data flow graph. + * + * This feature should only be used for debugging or when the data flow graph + * is not visualized (for example in a `path-problem` query). + */ + predicate includeHiddenNodes() { none() } + /** * Holds if there is a partial data flow path from `source` to `node`. The * approximate distance between `node` and the closest source is `dist` and @@ -3815,11 +3823,14 @@ abstract private class PathNodeImpl extends PathNode { abstract NodeEx getNodeEx(); predicate isHidden() { - hiddenNode(this.getNodeEx().asNode()) and - not this.isSource() and - not this instanceof PathNodeSink - or - this.getNodeEx() instanceof TNodeImplicitRead + not this.getConfiguration().includeHiddenNodes() and + ( + hiddenNode(this.getNodeEx().asNode()) and + not this.isSource() and + not this instanceof PathNodeSink + or + this.getNodeEx() instanceof TNodeImplicitRead + ) } private string ppAp() { diff --git a/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl6.qll b/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl6.qll index 89a35b00fa6..c0fdf294dd4 100644 --- a/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl6.qll +++ b/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl6.qll @@ -170,6 +170,14 @@ abstract class Configuration extends string { */ int explorationLimit() { none() } + /** + * Holds if hidden nodes should be included in the data flow graph. + * + * This feature should only be used for debugging or when the data flow graph + * is not visualized (for example in a `path-problem` query). + */ + predicate includeHiddenNodes() { none() } + /** * Holds if there is a partial data flow path from `source` to `node`. The * approximate distance between `node` and the closest source is `dist` and @@ -3815,11 +3823,14 @@ abstract private class PathNodeImpl extends PathNode { abstract NodeEx getNodeEx(); predicate isHidden() { - hiddenNode(this.getNodeEx().asNode()) and - not this.isSource() and - not this instanceof PathNodeSink - or - this.getNodeEx() instanceof TNodeImplicitRead + not this.getConfiguration().includeHiddenNodes() and + ( + hiddenNode(this.getNodeEx().asNode()) and + not this.isSource() and + not this instanceof PathNodeSink + or + this.getNodeEx() instanceof TNodeImplicitRead + ) } private string ppAp() { diff --git a/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplForOnActivityResult.qll b/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplForOnActivityResult.qll index 89a35b00fa6..c0fdf294dd4 100644 --- a/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplForOnActivityResult.qll +++ b/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplForOnActivityResult.qll @@ -170,6 +170,14 @@ abstract class Configuration extends string { */ int explorationLimit() { none() } + /** + * Holds if hidden nodes should be included in the data flow graph. + * + * This feature should only be used for debugging or when the data flow graph + * is not visualized (for example in a `path-problem` query). + */ + predicate includeHiddenNodes() { none() } + /** * Holds if there is a partial data flow path from `source` to `node`. The * approximate distance between `node` and the closest source is `dist` and @@ -3815,11 +3823,14 @@ abstract private class PathNodeImpl extends PathNode { abstract NodeEx getNodeEx(); predicate isHidden() { - hiddenNode(this.getNodeEx().asNode()) and - not this.isSource() and - not this instanceof PathNodeSink - or - this.getNodeEx() instanceof TNodeImplicitRead + not this.getConfiguration().includeHiddenNodes() and + ( + hiddenNode(this.getNodeEx().asNode()) and + not this.isSource() and + not this instanceof PathNodeSink + or + this.getNodeEx() instanceof TNodeImplicitRead + ) } private string ppAp() { diff --git a/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplForSerializability.qll b/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplForSerializability.qll index 89a35b00fa6..c0fdf294dd4 100644 --- a/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplForSerializability.qll +++ b/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplForSerializability.qll @@ -170,6 +170,14 @@ abstract class Configuration extends string { */ int explorationLimit() { none() } + /** + * Holds if hidden nodes should be included in the data flow graph. + * + * This feature should only be used for debugging or when the data flow graph + * is not visualized (for example in a `path-problem` query). + */ + predicate includeHiddenNodes() { none() } + /** * Holds if there is a partial data flow path from `source` to `node`. The * approximate distance between `node` and the closest source is `dist` and @@ -3815,11 +3823,14 @@ abstract private class PathNodeImpl extends PathNode { abstract NodeEx getNodeEx(); predicate isHidden() { - hiddenNode(this.getNodeEx().asNode()) and - not this.isSource() and - not this instanceof PathNodeSink - or - this.getNodeEx() instanceof TNodeImplicitRead + not this.getConfiguration().includeHiddenNodes() and + ( + hiddenNode(this.getNodeEx().asNode()) and + not this.isSource() and + not this instanceof PathNodeSink + or + this.getNodeEx() instanceof TNodeImplicitRead + ) } private string ppAp() { diff --git a/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl.qll b/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl.qll index 89a35b00fa6..c0fdf294dd4 100644 --- a/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl.qll +++ b/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl.qll @@ -170,6 +170,14 @@ abstract class Configuration extends string { */ int explorationLimit() { none() } + /** + * Holds if hidden nodes should be included in the data flow graph. + * + * This feature should only be used for debugging or when the data flow graph + * is not visualized (for example in a `path-problem` query). + */ + predicate includeHiddenNodes() { none() } + /** * Holds if there is a partial data flow path from `source` to `node`. The * approximate distance between `node` and the closest source is `dist` and @@ -3815,11 +3823,14 @@ abstract private class PathNodeImpl extends PathNode { abstract NodeEx getNodeEx(); predicate isHidden() { - hiddenNode(this.getNodeEx().asNode()) and - not this.isSource() and - not this instanceof PathNodeSink - or - this.getNodeEx() instanceof TNodeImplicitRead + not this.getConfiguration().includeHiddenNodes() and + ( + hiddenNode(this.getNodeEx().asNode()) and + not this.isSource() and + not this instanceof PathNodeSink + or + this.getNodeEx() instanceof TNodeImplicitRead + ) } private string ppAp() { diff --git a/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl2.qll b/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl2.qll index 89a35b00fa6..c0fdf294dd4 100644 --- a/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl2.qll +++ b/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl2.qll @@ -170,6 +170,14 @@ abstract class Configuration extends string { */ int explorationLimit() { none() } + /** + * Holds if hidden nodes should be included in the data flow graph. + * + * This feature should only be used for debugging or when the data flow graph + * is not visualized (for example in a `path-problem` query). + */ + predicate includeHiddenNodes() { none() } + /** * Holds if there is a partial data flow path from `source` to `node`. The * approximate distance between `node` and the closest source is `dist` and @@ -3815,11 +3823,14 @@ abstract private class PathNodeImpl extends PathNode { abstract NodeEx getNodeEx(); predicate isHidden() { - hiddenNode(this.getNodeEx().asNode()) and - not this.isSource() and - not this instanceof PathNodeSink - or - this.getNodeEx() instanceof TNodeImplicitRead + not this.getConfiguration().includeHiddenNodes() and + ( + hiddenNode(this.getNodeEx().asNode()) and + not this.isSource() and + not this instanceof PathNodeSink + or + this.getNodeEx() instanceof TNodeImplicitRead + ) } private string ppAp() { diff --git a/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl3.qll b/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl3.qll index 89a35b00fa6..c0fdf294dd4 100644 --- a/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl3.qll +++ b/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl3.qll @@ -170,6 +170,14 @@ abstract class Configuration extends string { */ int explorationLimit() { none() } + /** + * Holds if hidden nodes should be included in the data flow graph. + * + * This feature should only be used for debugging or when the data flow graph + * is not visualized (for example in a `path-problem` query). + */ + predicate includeHiddenNodes() { none() } + /** * Holds if there is a partial data flow path from `source` to `node`. The * approximate distance between `node` and the closest source is `dist` and @@ -3815,11 +3823,14 @@ abstract private class PathNodeImpl extends PathNode { abstract NodeEx getNodeEx(); predicate isHidden() { - hiddenNode(this.getNodeEx().asNode()) and - not this.isSource() and - not this instanceof PathNodeSink - or - this.getNodeEx() instanceof TNodeImplicitRead + not this.getConfiguration().includeHiddenNodes() and + ( + hiddenNode(this.getNodeEx().asNode()) and + not this.isSource() and + not this instanceof PathNodeSink + or + this.getNodeEx() instanceof TNodeImplicitRead + ) } private string ppAp() { diff --git a/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl4.qll b/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl4.qll index 89a35b00fa6..c0fdf294dd4 100644 --- a/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl4.qll +++ b/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl4.qll @@ -170,6 +170,14 @@ abstract class Configuration extends string { */ int explorationLimit() { none() } + /** + * Holds if hidden nodes should be included in the data flow graph. + * + * This feature should only be used for debugging or when the data flow graph + * is not visualized (for example in a `path-problem` query). + */ + predicate includeHiddenNodes() { none() } + /** * Holds if there is a partial data flow path from `source` to `node`. The * approximate distance between `node` and the closest source is `dist` and @@ -3815,11 +3823,14 @@ abstract private class PathNodeImpl extends PathNode { abstract NodeEx getNodeEx(); predicate isHidden() { - hiddenNode(this.getNodeEx().asNode()) and - not this.isSource() and - not this instanceof PathNodeSink - or - this.getNodeEx() instanceof TNodeImplicitRead + not this.getConfiguration().includeHiddenNodes() and + ( + hiddenNode(this.getNodeEx().asNode()) and + not this.isSource() and + not this instanceof PathNodeSink + or + this.getNodeEx() instanceof TNodeImplicitRead + ) } private string ppAp() { diff --git a/ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl2.qll b/ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl2.qll index 89a35b00fa6..c0fdf294dd4 100644 --- a/ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl2.qll +++ b/ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl2.qll @@ -170,6 +170,14 @@ abstract class Configuration extends string { */ int explorationLimit() { none() } + /** + * Holds if hidden nodes should be included in the data flow graph. + * + * This feature should only be used for debugging or when the data flow graph + * is not visualized (for example in a `path-problem` query). + */ + predicate includeHiddenNodes() { none() } + /** * Holds if there is a partial data flow path from `source` to `node`. The * approximate distance between `node` and the closest source is `dist` and @@ -3815,11 +3823,14 @@ abstract private class PathNodeImpl extends PathNode { abstract NodeEx getNodeEx(); predicate isHidden() { - hiddenNode(this.getNodeEx().asNode()) and - not this.isSource() and - not this instanceof PathNodeSink - or - this.getNodeEx() instanceof TNodeImplicitRead + not this.getConfiguration().includeHiddenNodes() and + ( + hiddenNode(this.getNodeEx().asNode()) and + not this.isSource() and + not this instanceof PathNodeSink + or + this.getNodeEx() instanceof TNodeImplicitRead + ) } private string ppAp() { diff --git a/ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForLibraries.qll b/ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForLibraries.qll index 89a35b00fa6..c0fdf294dd4 100644 --- a/ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForLibraries.qll +++ b/ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForLibraries.qll @@ -170,6 +170,14 @@ abstract class Configuration extends string { */ int explorationLimit() { none() } + /** + * Holds if hidden nodes should be included in the data flow graph. + * + * This feature should only be used for debugging or when the data flow graph + * is not visualized (for example in a `path-problem` query). + */ + predicate includeHiddenNodes() { none() } + /** * Holds if there is a partial data flow path from `source` to `node`. The * approximate distance between `node` and the closest source is `dist` and @@ -3815,11 +3823,14 @@ abstract private class PathNodeImpl extends PathNode { abstract NodeEx getNodeEx(); predicate isHidden() { - hiddenNode(this.getNodeEx().asNode()) and - not this.isSource() and - not this instanceof PathNodeSink - or - this.getNodeEx() instanceof TNodeImplicitRead + not this.getConfiguration().includeHiddenNodes() and + ( + hiddenNode(this.getNodeEx().asNode()) and + not this.isSource() and + not this instanceof PathNodeSink + or + this.getNodeEx() instanceof TNodeImplicitRead + ) } private string ppAp() { From 88520435584f3ed3300e69f3d388a5dc0d07d11d Mon Sep 17 00:00:00 2001 From: Geoffrey White <40627776+geoffw0@users.noreply.github.com> Date: Thu, 5 May 2022 17:02:06 +0100 Subject: [PATCH 76/91] C++: Additional test cases. --- .../Security/CWE/CWE-611/tests5.cpp | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/cpp/ql/test/query-tests/Security/CWE/CWE-611/tests5.cpp b/cpp/ql/test/query-tests/Security/CWE/CWE-611/tests5.cpp index e98d5a99e60..b55b7bab1e9 100644 --- a/cpp/ql/test/query-tests/Security/CWE/CWE-611/tests5.cpp +++ b/cpp/ql/test/query-tests/Security/CWE/CWE-611/tests5.cpp @@ -76,3 +76,28 @@ void test5_6() { g_p1->parse(*g_data); // GOOD g_p2->parse(*g_data); // BAD (parser not correctly configured) [NOT DETECTED] } + +void test5_7(DOMImplementationLS *impl, InputSource &data) { + DOMLSParser *p = impl->createLSParser(); + + p->parse(data); // BAD (parser not correctly configured) [NOT DETECTED] + + p->getDomConfig()->setParameter(XMLUni::fgXercesDisableDefaultEntityResolution, true); + p->parse(data); // GOOD + + p->getDomConfig()->setParameter(XMLUni::fgXercesDisableDefaultEntityResolution, false); + p->parse(data); // BAD (parser not correctly configured) [NOT DETECTED] +} + +void test5_8(DOMImplementationLS *impl, InputSource &data) { + DOMLSParser *p = impl->createLSParser(); + DOMConfiguration *cfg = p->getDomConfig(); + + p->parse(data); // BAD (parser not correctly configured) [NOT DETECTED] + + cfg->setParameter(XMLUni::fgXercesDisableDefaultEntityResolution, true); + p->parse(data); // GOOD + + cfg->setParameter(XMLUni::fgXercesDisableDefaultEntityResolution, false); + p->parse(data); // BAD (parser not correctly configured) [NOT DETECTED] +} From e3be7749ea1baa80732e6e0ea94c52acdab377c0 Mon Sep 17 00:00:00 2001 From: Geoffrey White <40627776+geoffw0@users.noreply.github.com> Date: Thu, 5 May 2022 16:54:27 +0100 Subject: [PATCH 77/91] C++: Repair the LSParser sinks. --- cpp/ql/src/Security/CWE/CWE-611/XXE.ql | 13 ++++-- .../Security/CWE/CWE-611/XXE.expected | 40 +++++++++++++++++++ .../Security/CWE/CWE-611/tests5.cpp | 22 +++++----- 3 files changed, 61 insertions(+), 14 deletions(-) diff --git a/cpp/ql/src/Security/CWE/CWE-611/XXE.ql b/cpp/ql/src/Security/CWE/CWE-611/XXE.ql index c8b638ddecd..d56e6c56bc6 100644 --- a/cpp/ql/src/Security/CWE/CWE-611/XXE.ql +++ b/cpp/ql/src/Security/CWE/CWE-611/XXE.ql @@ -57,6 +57,13 @@ class XercesDOMParserClass extends Class { XercesDOMParserClass() { this.hasName("XercesDOMParser") } } +/** + * The `DOMLSParser` class. + */ +class DOMLSParserClass extends Class { + DOMLSParserClass() { this.hasName("DOMLSParser") } +} + /** * The `SAXParser` class. */ @@ -217,12 +224,12 @@ class SetFeatureTranformer extends XXEFlowStateTranformer { } /** - * The `AbstractDOMParser.parse`, `SAXParser.parse` or `SAX2XMLReader.parse` - * method. + * The `AbstractDOMParser.parse`, `DOMLSParserClass.parse`, `SAXParser.parse` or `SAX2XMLReader.parse` method. */ class ParseFunction extends Function { ParseFunction() { this.getClassAndName("parse") instanceof AbstractDOMParserClass or + this.getClassAndName("parse") instanceof DOMLSParserClass or this.getClassAndName("parse") instanceof SaxParserClass or this.getClassAndName("parse") instanceof Sax2XmlReader } @@ -235,7 +242,7 @@ class ParseFunction extends Function { class CreateLSParser extends Function { CreateLSParser() { this.hasName("createLSParser") and - this.getUnspecifiedType().(PointerType).getBaseType().getName() = "DOMLSParser" // returns a `DOMLSParser *`. + this.getUnspecifiedType().(PointerType).getBaseType() instanceof DOMLSParserClass // returns a `DOMLSParser *`. } } diff --git a/cpp/ql/test/query-tests/Security/CWE/CWE-611/XXE.expected b/cpp/ql/test/query-tests/Security/CWE/CWE-611/XXE.expected index 24371939674..3ffb719a96b 100644 --- a/cpp/ql/test/query-tests/Security/CWE/CWE-611/XXE.expected +++ b/cpp/ql/test/query-tests/Security/CWE/CWE-611/XXE.expected @@ -4,6 +4,17 @@ edges | tests3.cpp:23:21:23:53 | call to createXMLReader | tests3.cpp:25:2:25:2 | p | | tests3.cpp:60:21:60:53 | call to createXMLReader | tests3.cpp:63:2:63:2 | p | | tests3.cpp:67:21:67:53 | call to createXMLReader | tests3.cpp:70:2:70:2 | p | +| tests5.cpp:27:25:27:38 | call to createLSParser | tests5.cpp:29:2:29:2 | p | +| tests5.cpp:33:25:33:38 | call to createLSParser | tests5.cpp:36:2:36:2 | p | +| tests5.cpp:40:25:40:38 | call to createLSParser | tests5.cpp:43:2:43:2 | p | +| tests5.cpp:47:25:47:38 | call to createLSParser | tests5.cpp:51:2:51:2 | p | +| tests5.cpp:55:25:55:38 | call to createLSParser | tests5.cpp:59:2:59:2 | p | +| tests5.cpp:81:25:81:38 | call to createLSParser | tests5.cpp:83:2:83:2 | p | +| tests5.cpp:81:25:81:38 | call to createLSParser | tests5.cpp:86:2:86:2 | p | +| tests5.cpp:81:25:81:38 | call to createLSParser | tests5.cpp:89:2:89:2 | p | +| tests5.cpp:93:25:93:38 | call to createLSParser | tests5.cpp:96:2:96:2 | p | +| tests5.cpp:93:25:93:38 | call to createLSParser | tests5.cpp:99:2:99:2 | p | +| tests5.cpp:93:25:93:38 | call to createLSParser | tests5.cpp:102:2:102:2 | p | | tests.cpp:15:23:15:43 | XercesDOMParser output argument | tests.cpp:17:2:17:2 | p | | tests.cpp:28:23:28:43 | XercesDOMParser output argument | tests.cpp:31:2:31:2 | p | | tests.cpp:35:19:35:19 | VariableAddress [post update] | tests.cpp:37:2:37:2 | p | @@ -46,6 +57,24 @@ nodes | tests4.cpp:46:34:46:68 | ... \| ... | semmle.label | ... \| ... | | tests4.cpp:77:34:77:38 | flags | semmle.label | flags | | tests4.cpp:130:39:130:55 | (int)... | semmle.label | (int)... | +| tests5.cpp:27:25:27:38 | call to createLSParser | semmle.label | call to createLSParser | +| tests5.cpp:29:2:29:2 | p | semmle.label | p | +| tests5.cpp:33:25:33:38 | call to createLSParser | semmle.label | call to createLSParser | +| tests5.cpp:36:2:36:2 | p | semmle.label | p | +| tests5.cpp:40:25:40:38 | call to createLSParser | semmle.label | call to createLSParser | +| tests5.cpp:43:2:43:2 | p | semmle.label | p | +| tests5.cpp:47:25:47:38 | call to createLSParser | semmle.label | call to createLSParser | +| tests5.cpp:51:2:51:2 | p | semmle.label | p | +| tests5.cpp:55:25:55:38 | call to createLSParser | semmle.label | call to createLSParser | +| tests5.cpp:59:2:59:2 | p | semmle.label | p | +| tests5.cpp:81:25:81:38 | call to createLSParser | semmle.label | call to createLSParser | +| tests5.cpp:83:2:83:2 | p | semmle.label | p | +| tests5.cpp:86:2:86:2 | p | semmle.label | p | +| tests5.cpp:89:2:89:2 | p | semmle.label | p | +| tests5.cpp:93:25:93:38 | call to createLSParser | semmle.label | call to createLSParser | +| tests5.cpp:96:2:96:2 | p | semmle.label | p | +| tests5.cpp:99:2:99:2 | p | semmle.label | p | +| tests5.cpp:102:2:102:2 | p | semmle.label | p | | tests.cpp:15:23:15:43 | XercesDOMParser output argument | semmle.label | XercesDOMParser output argument | | tests.cpp:17:2:17:2 | p | semmle.label | p | | tests.cpp:28:23:28:43 | XercesDOMParser output argument | semmle.label | XercesDOMParser output argument | @@ -93,6 +122,17 @@ subpaths | tests4.cpp:46:34:46:68 | ... \| ... | tests4.cpp:46:34:46:68 | ... \| ... | tests4.cpp:46:34:46:68 | ... \| ... | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests4.cpp:46:34:46:68 | ... \| ... | XML parser | | tests4.cpp:77:34:77:38 | flags | tests4.cpp:77:34:77:38 | flags | tests4.cpp:77:34:77:38 | flags | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests4.cpp:77:34:77:38 | flags | XML parser | | tests4.cpp:130:39:130:55 | (int)... | tests4.cpp:130:39:130:55 | (int)... | tests4.cpp:130:39:130:55 | (int)... | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests4.cpp:130:39:130:55 | (int)... | XML parser | +| tests5.cpp:29:2:29:2 | p | tests5.cpp:27:25:27:38 | call to createLSParser | tests5.cpp:29:2:29:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests5.cpp:27:25:27:38 | call to createLSParser | XML parser | +| tests5.cpp:36:2:36:2 | p | tests5.cpp:33:25:33:38 | call to createLSParser | tests5.cpp:36:2:36:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests5.cpp:33:25:33:38 | call to createLSParser | XML parser | +| tests5.cpp:43:2:43:2 | p | tests5.cpp:40:25:40:38 | call to createLSParser | tests5.cpp:43:2:43:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests5.cpp:40:25:40:38 | call to createLSParser | XML parser | +| tests5.cpp:51:2:51:2 | p | tests5.cpp:47:25:47:38 | call to createLSParser | tests5.cpp:51:2:51:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests5.cpp:47:25:47:38 | call to createLSParser | XML parser | +| tests5.cpp:59:2:59:2 | p | tests5.cpp:55:25:55:38 | call to createLSParser | tests5.cpp:59:2:59:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests5.cpp:55:25:55:38 | call to createLSParser | XML parser | +| tests5.cpp:83:2:83:2 | p | tests5.cpp:81:25:81:38 | call to createLSParser | tests5.cpp:83:2:83:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests5.cpp:81:25:81:38 | call to createLSParser | XML parser | +| tests5.cpp:86:2:86:2 | p | tests5.cpp:81:25:81:38 | call to createLSParser | tests5.cpp:86:2:86:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests5.cpp:81:25:81:38 | call to createLSParser | XML parser | +| tests5.cpp:89:2:89:2 | p | tests5.cpp:81:25:81:38 | call to createLSParser | tests5.cpp:89:2:89:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests5.cpp:81:25:81:38 | call to createLSParser | XML parser | +| tests5.cpp:96:2:96:2 | p | tests5.cpp:93:25:93:38 | call to createLSParser | tests5.cpp:96:2:96:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests5.cpp:93:25:93:38 | call to createLSParser | XML parser | +| tests5.cpp:99:2:99:2 | p | tests5.cpp:93:25:93:38 | call to createLSParser | tests5.cpp:99:2:99:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests5.cpp:93:25:93:38 | call to createLSParser | XML parser | +| tests5.cpp:102:2:102:2 | p | tests5.cpp:93:25:93:38 | call to createLSParser | tests5.cpp:102:2:102:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests5.cpp:93:25:93:38 | call to createLSParser | XML parser | | tests.cpp:17:2:17:2 | p | tests.cpp:15:23:15:43 | XercesDOMParser output argument | tests.cpp:17:2:17:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests.cpp:15:23:15:43 | XercesDOMParser output argument | XML parser | | tests.cpp:31:2:31:2 | p | tests.cpp:28:23:28:43 | XercesDOMParser output argument | tests.cpp:31:2:31:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests.cpp:28:23:28:43 | XercesDOMParser output argument | XML parser | | tests.cpp:39:2:39:2 | p | tests.cpp:35:23:35:43 | XercesDOMParser output argument | tests.cpp:39:2:39:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests.cpp:35:23:35:43 | XercesDOMParser output argument | XML parser | diff --git a/cpp/ql/test/query-tests/Security/CWE/CWE-611/tests5.cpp b/cpp/ql/test/query-tests/Security/CWE/CWE-611/tests5.cpp index b55b7bab1e9..2716a708c32 100644 --- a/cpp/ql/test/query-tests/Security/CWE/CWE-611/tests5.cpp +++ b/cpp/ql/test/query-tests/Security/CWE/CWE-611/tests5.cpp @@ -26,21 +26,21 @@ public: void test5_1(DOMImplementationLS *impl, InputSource &data) { DOMLSParser *p = impl->createLSParser(); - p->parse(data); // BAD (parser not correctly configured) [NOT DETECTED] + p->parse(data); // BAD (parser not correctly configured) } void test5_2(DOMImplementationLS *impl, InputSource &data) { DOMLSParser *p = impl->createLSParser(); p->getDomConfig()->setParameter(XMLUni::fgXercesDisableDefaultEntityResolution, true); - p->parse(data); // GOOD + p->parse(data); // GOOD [FALSE POSITIVE] } void test5_3(DOMImplementationLS *impl, InputSource &data) { DOMLSParser *p = impl->createLSParser(); p->getDomConfig()->setParameter(XMLUni::fgXercesDisableDefaultEntityResolution, false); - p->parse(data); // BAD (parser not correctly configured) [NOT DETECTED] + p->parse(data); // BAD (parser not correctly configured) } void test5_4(DOMImplementationLS *impl, InputSource &data) { @@ -48,7 +48,7 @@ void test5_4(DOMImplementationLS *impl, InputSource &data) { DOMConfiguration *cfg = p->getDomConfig(); cfg->setParameter(XMLUni::fgXercesDisableDefaultEntityResolution, true); - p->parse(data); // GOOD + p->parse(data); // GOOD [FALSE POSITIVE] } void test5_5(DOMImplementationLS *impl, InputSource &data) { @@ -56,7 +56,7 @@ void test5_5(DOMImplementationLS *impl, InputSource &data) { DOMConfiguration *cfg = p->getDomConfig(); cfg->setParameter(XMLUni::fgXercesDisableDefaultEntityResolution, false); - p->parse(data); // BAD (parser not correctly configured) [NOT DETECTED] + p->parse(data); // BAD (parser not correctly configured) } DOMImplementationLS *g_impl; @@ -80,24 +80,24 @@ void test5_6() { void test5_7(DOMImplementationLS *impl, InputSource &data) { DOMLSParser *p = impl->createLSParser(); - p->parse(data); // BAD (parser not correctly configured) [NOT DETECTED] + p->parse(data); // BAD (parser not correctly configured) p->getDomConfig()->setParameter(XMLUni::fgXercesDisableDefaultEntityResolution, true); - p->parse(data); // GOOD + p->parse(data); // GOOD [FALSE POSITIVE] p->getDomConfig()->setParameter(XMLUni::fgXercesDisableDefaultEntityResolution, false); - p->parse(data); // BAD (parser not correctly configured) [NOT DETECTED] + p->parse(data); // BAD (parser not correctly configured) } void test5_8(DOMImplementationLS *impl, InputSource &data) { DOMLSParser *p = impl->createLSParser(); DOMConfiguration *cfg = p->getDomConfig(); - p->parse(data); // BAD (parser not correctly configured) [NOT DETECTED] + p->parse(data); // BAD (parser not correctly configured) cfg->setParameter(XMLUni::fgXercesDisableDefaultEntityResolution, true); - p->parse(data); // GOOD + p->parse(data); // GOOD [FALSE POSITIVE] cfg->setParameter(XMLUni::fgXercesDisableDefaultEntityResolution, false); - p->parse(data); // BAD (parser not correctly configured) [NOT DETECTED] + p->parse(data); // BAD (parser not correctly configured) } From 3dddc560a1a1470f9a7579a433a00eb988fe6f19 Mon Sep 17 00:00:00 2001 From: Geoffrey White <40627776+geoffw0@users.noreply.github.com> Date: Thu, 5 May 2022 18:16:50 +0100 Subject: [PATCH 78/91] C++: Add LSParser specific transformer. --- cpp/ql/src/Security/CWE/CWE-611/XXE.ql | 62 +++++++++++++++++++ .../Security/CWE/CWE-611/XXE.expected | 29 +++------ .../Security/CWE/CWE-611/tests5.cpp | 12 ++-- 3 files changed, 76 insertions(+), 27 deletions(-) diff --git a/cpp/ql/src/Security/CWE/CWE-611/XXE.ql b/cpp/ql/src/Security/CWE/CWE-611/XXE.ql index d56e6c56bc6..ab6ef6229de 100644 --- a/cpp/ql/src/Security/CWE/CWE-611/XXE.ql +++ b/cpp/ql/src/Security/CWE/CWE-611/XXE.ql @@ -223,6 +223,68 @@ class SetFeatureTranformer extends XXEFlowStateTranformer { } } +/** + * The `DOMLSParser.getDomConfig` function. + */ +class GetDomConfig extends Function { + GetDomConfig() { + this.hasName("getDomConfig") and + this.getDeclaringType() instanceof DOMLSParserClass + } +} + +/** + * The `DOMConfiguration.setParameter` function. + */ +class DomConfigurationSetParameter extends Function { + DomConfigurationSetParameter() { + this.hasName("setParameter") and + this.getDeclaringType().getName() = "DOMConfiguration" + } +} + +/** + * A flow state transformer for a call to `DOMConfiguration.setParameter` + * specifying the feature `XMLUni::fgXercesDisableDefaultEntityResolution`. + * This is a slightly more complex transformer because the qualifier is a + * `DOMConfiguration` pointer returned by `DOMLSParser.getDomConfig` - and it + * is *that* qualifier we want to transform the flow state of. + */ +class DOMConfigurationSetParameterTranformer extends XXEFlowStateTranformer { + Expr newValue; + + DOMConfigurationSetParameterTranformer() { + exists(FunctionCall getDomConfigCall, FunctionCall setParameterCall | + // this is the qualifier of a call to `DOMLSParser.getDomConfig`. + getDomConfigCall.getTarget() instanceof GetDomConfig and + this = getDomConfigCall.getQualifier() and + // `setParameterCall` is a call to `setParameter` on the return value of + // the same call to `DOMLSParser.getDomConfig`. + setParameterCall.getTarget() instanceof DomConfigurationSetParameter and + globalValueNumber(setParameterCall.getQualifier()).getAnExpr() = getDomConfigCall and + // the parameter being set is + // `XMLUni::fgXercesDisableDefaultEntityResolution`. + globalValueNumber(setParameterCall.getArgument(0)).getAnExpr().(VariableAccess).getTarget() + instanceof FeatureDisableDefaultEntityResolution and + // the value being set is `newValue`. + newValue = setParameterCall.getArgument(1) + ) + } + + final override XXEFlowState transform(XXEFlowState flowstate) { + exists(int createEntityReferenceNodes | + encodeXercesFlowState(flowstate, _, createEntityReferenceNodes) and + ( + globalValueNumber(newValue).getAnExpr().getValue().toInt() = 1 and // true + encodeXercesFlowState(result, 1, createEntityReferenceNodes) + or + not globalValueNumber(newValue).getAnExpr().getValue().toInt() = 1 and // false or unknown + encodeXercesFlowState(result, 0, createEntityReferenceNodes) + ) + ) + } +} + /** * The `AbstractDOMParser.parse`, `DOMLSParserClass.parse`, `SAXParser.parse` or `SAX2XMLReader.parse` method. */ diff --git a/cpp/ql/test/query-tests/Security/CWE/CWE-611/XXE.expected b/cpp/ql/test/query-tests/Security/CWE/CWE-611/XXE.expected index 3ffb719a96b..a4db6155e31 100644 --- a/cpp/ql/test/query-tests/Security/CWE/CWE-611/XXE.expected +++ b/cpp/ql/test/query-tests/Security/CWE/CWE-611/XXE.expected @@ -5,16 +5,14 @@ edges | tests3.cpp:60:21:60:53 | call to createXMLReader | tests3.cpp:63:2:63:2 | p | | tests3.cpp:67:21:67:53 | call to createXMLReader | tests3.cpp:70:2:70:2 | p | | tests5.cpp:27:25:27:38 | call to createLSParser | tests5.cpp:29:2:29:2 | p | -| tests5.cpp:33:25:33:38 | call to createLSParser | tests5.cpp:36:2:36:2 | p | | tests5.cpp:40:25:40:38 | call to createLSParser | tests5.cpp:43:2:43:2 | p | -| tests5.cpp:47:25:47:38 | call to createLSParser | tests5.cpp:51:2:51:2 | p | | tests5.cpp:55:25:55:38 | call to createLSParser | tests5.cpp:59:2:59:2 | p | | tests5.cpp:81:25:81:38 | call to createLSParser | tests5.cpp:83:2:83:2 | p | -| tests5.cpp:81:25:81:38 | call to createLSParser | tests5.cpp:86:2:86:2 | p | -| tests5.cpp:81:25:81:38 | call to createLSParser | tests5.cpp:89:2:89:2 | p | -| tests5.cpp:93:25:93:38 | call to createLSParser | tests5.cpp:96:2:96:2 | p | -| tests5.cpp:93:25:93:38 | call to createLSParser | tests5.cpp:99:2:99:2 | p | -| tests5.cpp:93:25:93:38 | call to createLSParser | tests5.cpp:102:2:102:2 | p | +| tests5.cpp:81:25:81:38 | call to createLSParser | tests5.cpp:83:2:83:2 | p | +| tests5.cpp:83:2:83:2 | p | tests5.cpp:85:2:85:2 | p | +| tests5.cpp:85:2:85:2 | p | tests5.cpp:86:2:86:2 | p | +| tests5.cpp:86:2:86:2 | p | tests5.cpp:88:2:88:2 | p | +| tests5.cpp:88:2:88:2 | p | tests5.cpp:89:2:89:2 | p | | tests.cpp:15:23:15:43 | XercesDOMParser output argument | tests.cpp:17:2:17:2 | p | | tests.cpp:28:23:28:43 | XercesDOMParser output argument | tests.cpp:31:2:31:2 | p | | tests.cpp:35:19:35:19 | VariableAddress [post update] | tests.cpp:37:2:37:2 | p | @@ -59,22 +57,17 @@ nodes | tests4.cpp:130:39:130:55 | (int)... | semmle.label | (int)... | | tests5.cpp:27:25:27:38 | call to createLSParser | semmle.label | call to createLSParser | | tests5.cpp:29:2:29:2 | p | semmle.label | p | -| tests5.cpp:33:25:33:38 | call to createLSParser | semmle.label | call to createLSParser | -| tests5.cpp:36:2:36:2 | p | semmle.label | p | | tests5.cpp:40:25:40:38 | call to createLSParser | semmle.label | call to createLSParser | | tests5.cpp:43:2:43:2 | p | semmle.label | p | -| tests5.cpp:47:25:47:38 | call to createLSParser | semmle.label | call to createLSParser | -| tests5.cpp:51:2:51:2 | p | semmle.label | p | | tests5.cpp:55:25:55:38 | call to createLSParser | semmle.label | call to createLSParser | | tests5.cpp:59:2:59:2 | p | semmle.label | p | | tests5.cpp:81:25:81:38 | call to createLSParser | semmle.label | call to createLSParser | | tests5.cpp:83:2:83:2 | p | semmle.label | p | +| tests5.cpp:83:2:83:2 | p | semmle.label | p | +| tests5.cpp:85:2:85:2 | p | semmle.label | p | | tests5.cpp:86:2:86:2 | p | semmle.label | p | +| tests5.cpp:88:2:88:2 | p | semmle.label | p | | tests5.cpp:89:2:89:2 | p | semmle.label | p | -| tests5.cpp:93:25:93:38 | call to createLSParser | semmle.label | call to createLSParser | -| tests5.cpp:96:2:96:2 | p | semmle.label | p | -| tests5.cpp:99:2:99:2 | p | semmle.label | p | -| tests5.cpp:102:2:102:2 | p | semmle.label | p | | tests.cpp:15:23:15:43 | XercesDOMParser output argument | semmle.label | XercesDOMParser output argument | | tests.cpp:17:2:17:2 | p | semmle.label | p | | tests.cpp:28:23:28:43 | XercesDOMParser output argument | semmle.label | XercesDOMParser output argument | @@ -123,16 +116,10 @@ subpaths | tests4.cpp:77:34:77:38 | flags | tests4.cpp:77:34:77:38 | flags | tests4.cpp:77:34:77:38 | flags | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests4.cpp:77:34:77:38 | flags | XML parser | | tests4.cpp:130:39:130:55 | (int)... | tests4.cpp:130:39:130:55 | (int)... | tests4.cpp:130:39:130:55 | (int)... | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests4.cpp:130:39:130:55 | (int)... | XML parser | | tests5.cpp:29:2:29:2 | p | tests5.cpp:27:25:27:38 | call to createLSParser | tests5.cpp:29:2:29:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests5.cpp:27:25:27:38 | call to createLSParser | XML parser | -| tests5.cpp:36:2:36:2 | p | tests5.cpp:33:25:33:38 | call to createLSParser | tests5.cpp:36:2:36:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests5.cpp:33:25:33:38 | call to createLSParser | XML parser | | tests5.cpp:43:2:43:2 | p | tests5.cpp:40:25:40:38 | call to createLSParser | tests5.cpp:43:2:43:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests5.cpp:40:25:40:38 | call to createLSParser | XML parser | -| tests5.cpp:51:2:51:2 | p | tests5.cpp:47:25:47:38 | call to createLSParser | tests5.cpp:51:2:51:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests5.cpp:47:25:47:38 | call to createLSParser | XML parser | | tests5.cpp:59:2:59:2 | p | tests5.cpp:55:25:55:38 | call to createLSParser | tests5.cpp:59:2:59:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests5.cpp:55:25:55:38 | call to createLSParser | XML parser | | tests5.cpp:83:2:83:2 | p | tests5.cpp:81:25:81:38 | call to createLSParser | tests5.cpp:83:2:83:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests5.cpp:81:25:81:38 | call to createLSParser | XML parser | -| tests5.cpp:86:2:86:2 | p | tests5.cpp:81:25:81:38 | call to createLSParser | tests5.cpp:86:2:86:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests5.cpp:81:25:81:38 | call to createLSParser | XML parser | | tests5.cpp:89:2:89:2 | p | tests5.cpp:81:25:81:38 | call to createLSParser | tests5.cpp:89:2:89:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests5.cpp:81:25:81:38 | call to createLSParser | XML parser | -| tests5.cpp:96:2:96:2 | p | tests5.cpp:93:25:93:38 | call to createLSParser | tests5.cpp:96:2:96:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests5.cpp:93:25:93:38 | call to createLSParser | XML parser | -| tests5.cpp:99:2:99:2 | p | tests5.cpp:93:25:93:38 | call to createLSParser | tests5.cpp:99:2:99:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests5.cpp:93:25:93:38 | call to createLSParser | XML parser | -| tests5.cpp:102:2:102:2 | p | tests5.cpp:93:25:93:38 | call to createLSParser | tests5.cpp:102:2:102:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests5.cpp:93:25:93:38 | call to createLSParser | XML parser | | tests.cpp:17:2:17:2 | p | tests.cpp:15:23:15:43 | XercesDOMParser output argument | tests.cpp:17:2:17:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests.cpp:15:23:15:43 | XercesDOMParser output argument | XML parser | | tests.cpp:31:2:31:2 | p | tests.cpp:28:23:28:43 | XercesDOMParser output argument | tests.cpp:31:2:31:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests.cpp:28:23:28:43 | XercesDOMParser output argument | XML parser | | tests.cpp:39:2:39:2 | p | tests.cpp:35:23:35:43 | XercesDOMParser output argument | tests.cpp:39:2:39:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests.cpp:35:23:35:43 | XercesDOMParser output argument | XML parser | diff --git a/cpp/ql/test/query-tests/Security/CWE/CWE-611/tests5.cpp b/cpp/ql/test/query-tests/Security/CWE/CWE-611/tests5.cpp index 2716a708c32..99027c9bd93 100644 --- a/cpp/ql/test/query-tests/Security/CWE/CWE-611/tests5.cpp +++ b/cpp/ql/test/query-tests/Security/CWE/CWE-611/tests5.cpp @@ -33,7 +33,7 @@ void test5_2(DOMImplementationLS *impl, InputSource &data) { DOMLSParser *p = impl->createLSParser(); p->getDomConfig()->setParameter(XMLUni::fgXercesDisableDefaultEntityResolution, true); - p->parse(data); // GOOD [FALSE POSITIVE] + p->parse(data); // GOOD } void test5_3(DOMImplementationLS *impl, InputSource &data) { @@ -48,7 +48,7 @@ void test5_4(DOMImplementationLS *impl, InputSource &data) { DOMConfiguration *cfg = p->getDomConfig(); cfg->setParameter(XMLUni::fgXercesDisableDefaultEntityResolution, true); - p->parse(data); // GOOD [FALSE POSITIVE] + p->parse(data); // GOOD } void test5_5(DOMImplementationLS *impl, InputSource &data) { @@ -83,7 +83,7 @@ void test5_7(DOMImplementationLS *impl, InputSource &data) { p->parse(data); // BAD (parser not correctly configured) p->getDomConfig()->setParameter(XMLUni::fgXercesDisableDefaultEntityResolution, true); - p->parse(data); // GOOD [FALSE POSITIVE] + p->parse(data); // GOOD p->getDomConfig()->setParameter(XMLUni::fgXercesDisableDefaultEntityResolution, false); p->parse(data); // BAD (parser not correctly configured) @@ -93,11 +93,11 @@ void test5_8(DOMImplementationLS *impl, InputSource &data) { DOMLSParser *p = impl->createLSParser(); DOMConfiguration *cfg = p->getDomConfig(); - p->parse(data); // BAD (parser not correctly configured) + p->parse(data); // BAD (parser not correctly configured) [NOT DETECTED] cfg->setParameter(XMLUni::fgXercesDisableDefaultEntityResolution, true); - p->parse(data); // GOOD [FALSE POSITIVE] + p->parse(data); // GOOD cfg->setParameter(XMLUni::fgXercesDisableDefaultEntityResolution, false); - p->parse(data); // BAD (parser not correctly configured) + p->parse(data); // BAD (parser not correctly configured) [NOT DETECTED] } From 00f7453fcb86d8d0b9fd3e8ea7e2e70165d336a4 Mon Sep 17 00:00:00 2001 From: Geoffrey White <40627776+geoffw0@users.noreply.github.com> Date: Wed, 11 May 2022 11:08:03 +0100 Subject: [PATCH 79/91] C++: Fix capitalization. --- cpp/ql/src/Security/CWE/CWE-611/XXE.ql | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/cpp/ql/src/Security/CWE/CWE-611/XXE.ql b/cpp/ql/src/Security/CWE/CWE-611/XXE.ql index ab6ef6229de..5f3ae65d78c 100644 --- a/cpp/ql/src/Security/CWE/CWE-611/XXE.ql +++ b/cpp/ql/src/Security/CWE/CWE-611/XXE.ql @@ -60,8 +60,8 @@ class XercesDOMParserClass extends Class { /** * The `DOMLSParser` class. */ -class DOMLSParserClass extends Class { - DOMLSParserClass() { this.hasName("DOMLSParser") } +class DomLSParserClass extends Class { + DomLSParserClass() { this.hasName("DOMLSParser") } } /** @@ -229,7 +229,7 @@ class SetFeatureTranformer extends XXEFlowStateTranformer { class GetDomConfig extends Function { GetDomConfig() { this.hasName("getDomConfig") and - this.getDeclaringType() instanceof DOMLSParserClass + this.getDeclaringType() instanceof DomLSParserClass } } @@ -286,12 +286,13 @@ class DOMConfigurationSetParameterTranformer extends XXEFlowStateTranformer { } /** - * The `AbstractDOMParser.parse`, `DOMLSParserClass.parse`, `SAXParser.parse` or `SAX2XMLReader.parse` method. + * The `AbstractDOMParser.parse`, `DOMLSParserClass.parse`, `SAXParser.parse` + * or `SAX2XMLReader.parse` method. */ class ParseFunction extends Function { ParseFunction() { this.getClassAndName("parse") instanceof AbstractDOMParserClass or - this.getClassAndName("parse") instanceof DOMLSParserClass or + this.getClassAndName("parse") instanceof DomLSParserClass or this.getClassAndName("parse") instanceof SaxParserClass or this.getClassAndName("parse") instanceof Sax2XmlReader } @@ -304,7 +305,7 @@ class ParseFunction extends Function { class CreateLSParser extends Function { CreateLSParser() { this.hasName("createLSParser") and - this.getUnspecifiedType().(PointerType).getBaseType() instanceof DOMLSParserClass // returns a `DOMLSParser *`. + this.getUnspecifiedType().(PointerType).getBaseType() instanceof DomLSParserClass // returns a `DOMLSParser *`. } } From f27c2f3031837416159d1752f3bdf6f29e360831 Mon Sep 17 00:00:00 2001 From: Geoffrey White <40627776+geoffw0@users.noreply.github.com> Date: Wed, 11 May 2022 11:27:57 +0100 Subject: [PATCH 80/91] C++: Fix more capitalization. --- cpp/ql/src/Security/CWE/CWE-611/XXE.ql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/ql/src/Security/CWE/CWE-611/XXE.ql b/cpp/ql/src/Security/CWE/CWE-611/XXE.ql index 5f3ae65d78c..2bb8839932e 100644 --- a/cpp/ql/src/Security/CWE/CWE-611/XXE.ql +++ b/cpp/ql/src/Security/CWE/CWE-611/XXE.ql @@ -250,10 +250,10 @@ class DomConfigurationSetParameter extends Function { * `DOMConfiguration` pointer returned by `DOMLSParser.getDomConfig` - and it * is *that* qualifier we want to transform the flow state of. */ -class DOMConfigurationSetParameterTranformer extends XXEFlowStateTranformer { +class DomConfigurationSetParameterTranformer extends XXEFlowStateTranformer { Expr newValue; - DOMConfigurationSetParameterTranformer() { + DomConfigurationSetParameterTranformer() { exists(FunctionCall getDomConfigCall, FunctionCall setParameterCall | // this is the qualifier of a call to `DOMLSParser.getDomConfig`. getDomConfigCall.getTarget() instanceof GetDomConfig and From 94e190c63af9977dfed1a33a8ed924e6de17388c Mon Sep 17 00:00:00 2001 From: Geoffrey White <40627776+geoffw0@users.noreply.github.com> Date: Wed, 11 May 2022 13:47:51 +0100 Subject: [PATCH 81/91] C++: getClassAndName. --- cpp/ql/src/Security/CWE/CWE-611/XXE.ql | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/cpp/ql/src/Security/CWE/CWE-611/XXE.ql b/cpp/ql/src/Security/CWE/CWE-611/XXE.ql index 2bb8839932e..3ce422c89b1 100644 --- a/cpp/ql/src/Security/CWE/CWE-611/XXE.ql +++ b/cpp/ql/src/Security/CWE/CWE-611/XXE.ql @@ -227,10 +227,7 @@ class SetFeatureTranformer extends XXEFlowStateTranformer { * The `DOMLSParser.getDomConfig` function. */ class GetDomConfig extends Function { - GetDomConfig() { - this.hasName("getDomConfig") and - this.getDeclaringType() instanceof DomLSParserClass - } + GetDomConfig() { this.getClassAndName("getDomConfig") instanceof DomLSParserClass } } /** @@ -238,8 +235,7 @@ class GetDomConfig extends Function { */ class DomConfigurationSetParameter extends Function { DomConfigurationSetParameter() { - this.hasName("setParameter") and - this.getDeclaringType().getName() = "DOMConfiguration" + this.getClassAndName("setParameter").getName() = "DOMConfiguration" } } From b1e8b3332c83d76819d6f3ce639bbb7778fda9d0 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Wed, 11 May 2022 12:45:03 +0200 Subject: [PATCH 82/91] resolve main module when there is a folder with the same name as the main file --- .../semmle/javascript/NodeModuleResolutionImpl.qll | 2 +- .../PrototypePollutingAssignment.expected | 12 ++++++++++++ .../PrototypePollutingAssignment/sublib/package.json | 4 ++++ .../PrototypePollutingAssignment/sublib/sub.js | 3 +++ .../PrototypePollutingAssignment/sublib/sub/empty.js | 1 + 5 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 javascript/ql/test/query-tests/Security/CWE-915/PrototypePollutingAssignment/sublib/package.json create mode 100644 javascript/ql/test/query-tests/Security/CWE-915/PrototypePollutingAssignment/sublib/sub.js create mode 100644 javascript/ql/test/query-tests/Security/CWE-915/PrototypePollutingAssignment/sublib/sub/empty.js diff --git a/javascript/ql/lib/semmle/javascript/NodeModuleResolutionImpl.qll b/javascript/ql/lib/semmle/javascript/NodeModuleResolutionImpl.qll index 6df85792a56..26aa1163e7e 100644 --- a/javascript/ql/lib/semmle/javascript/NodeModuleResolutionImpl.qll +++ b/javascript/ql/lib/semmle/javascript/NodeModuleResolutionImpl.qll @@ -96,7 +96,7 @@ File resolveMainModule(PackageJson pkg, int priority) { or result = tryExtensions(main.resolve(), "index", priority) or - not exists(main.resolve()) and + not main.resolve() instanceof File and exists(int n | n = main.getNumComponent() | result = tryExtensions(main.resolveUpTo(n - 1), getStem(main.getComponent(n - 1)), priority) ) diff --git a/javascript/ql/test/query-tests/Security/CWE-915/PrototypePollutingAssignment/PrototypePollutingAssignment.expected b/javascript/ql/test/query-tests/Security/CWE-915/PrototypePollutingAssignment/PrototypePollutingAssignment.expected index ec2bfb99862..7b4b6e81751 100644 --- a/javascript/ql/test/query-tests/Security/CWE-915/PrototypePollutingAssignment/PrototypePollutingAssignment.expected +++ b/javascript/ql/test/query-tests/Security/CWE-915/PrototypePollutingAssignment/PrototypePollutingAssignment.expected @@ -94,6 +94,12 @@ nodes | lib.js:108:3:108:10 | obj[one] | | lib.js:108:3:108:10 | obj[one] | | lib.js:108:7:108:9 | one | +| sublib/sub.js:1:37:1:40 | path | +| sublib/sub.js:1:37:1:40 | path | +| sublib/sub.js:2:3:2:14 | obj[path[0]] | +| sublib/sub.js:2:3:2:14 | obj[path[0]] | +| sublib/sub.js:2:7:2:10 | path | +| sublib/sub.js:2:7:2:13 | path[0] | | tst.js:5:9:5:38 | taint | | tst.js:5:17:5:38 | String( ... y.data) | | tst.js:5:24:5:37 | req.query.data | @@ -230,6 +236,11 @@ edges | lib.js:104:13:104:24 | arguments[1] | lib.js:104:7:104:24 | one | | lib.js:108:7:108:9 | one | lib.js:108:3:108:10 | obj[one] | | lib.js:108:7:108:9 | one | lib.js:108:3:108:10 | obj[one] | +| sublib/sub.js:1:37:1:40 | path | sublib/sub.js:2:7:2:10 | path | +| sublib/sub.js:1:37:1:40 | path | sublib/sub.js:2:7:2:10 | path | +| sublib/sub.js:2:7:2:10 | path | sublib/sub.js:2:7:2:13 | path[0] | +| sublib/sub.js:2:7:2:13 | path[0] | sublib/sub.js:2:3:2:14 | obj[path[0]] | +| sublib/sub.js:2:7:2:13 | path[0] | sublib/sub.js:2:3:2:14 | obj[path[0]] | | tst.js:5:9:5:38 | taint | tst.js:8:12:8:16 | taint | | tst.js:5:9:5:38 | taint | tst.js:9:12:9:16 | taint | | tst.js:5:9:5:38 | taint | tst.js:12:25:12:29 | taint | @@ -284,6 +295,7 @@ edges | lib.js:70:13:70:24 | obj[path[0]] | lib.js:59:18:59:18 | s | lib.js:70:13:70:24 | obj[path[0]] | This assignment may alter Object.prototype if a malicious '__proto__' string is injected from $@. | lib.js:59:18:59:18 | s | library input | | lib.js:87:10:87:14 | proto | lib.js:83:14:83:25 | arguments[1] | lib.js:87:10:87:14 | proto | This assignment may alter Object.prototype if a malicious '__proto__' string is injected from $@. | lib.js:83:14:83:25 | arguments[1] | library input | | lib.js:108:3:108:10 | obj[one] | lib.js:104:13:104:24 | arguments[1] | lib.js:108:3:108:10 | obj[one] | This assignment may alter Object.prototype if a malicious '__proto__' string is injected from $@. | lib.js:104:13:104:24 | arguments[1] | library input | +| sublib/sub.js:2:3:2:14 | obj[path[0]] | sublib/sub.js:1:37:1:40 | path | sublib/sub.js:2:3:2:14 | obj[path[0]] | This assignment may alter Object.prototype if a malicious '__proto__' string is injected from $@. | sublib/sub.js:1:37:1:40 | path | library input | | tst.js:8:5:8:17 | object[taint] | tst.js:5:24:5:37 | req.query.data | tst.js:8:5:8:17 | object[taint] | This assignment may alter Object.prototype if a malicious '__proto__' string is injected from $@. | tst.js:5:24:5:37 | req.query.data | user controlled input | | tst.js:9:5:9:17 | object[taint] | tst.js:5:24:5:37 | req.query.data | tst.js:9:5:9:17 | object[taint] | This assignment may alter Object.prototype if a malicious '__proto__' string is injected from $@. | tst.js:5:24:5:37 | req.query.data | user controlled input | | tst.js:14:5:14:32 | unsafeG ... taint) | tst.js:5:24:5:37 | req.query.data | tst.js:14:5:14:32 | unsafeG ... taint) | This assignment may alter Object.prototype if a malicious '__proto__' string is injected from $@. | tst.js:5:24:5:37 | req.query.data | user controlled input | diff --git a/javascript/ql/test/query-tests/Security/CWE-915/PrototypePollutingAssignment/sublib/package.json b/javascript/ql/test/query-tests/Security/CWE-915/PrototypePollutingAssignment/sublib/package.json new file mode 100644 index 00000000000..3f633a4ff71 --- /dev/null +++ b/javascript/ql/test/query-tests/Security/CWE-915/PrototypePollutingAssignment/sublib/package.json @@ -0,0 +1,4 @@ +{ + "name": "sublib", + "main": "./sub" +} \ No newline at end of file diff --git a/javascript/ql/test/query-tests/Security/CWE-915/PrototypePollutingAssignment/sublib/sub.js b/javascript/ql/test/query-tests/Security/CWE-915/PrototypePollutingAssignment/sublib/sub.js new file mode 100644 index 00000000000..d74de12493a --- /dev/null +++ b/javascript/ql/test/query-tests/Security/CWE-915/PrototypePollutingAssignment/sublib/sub.js @@ -0,0 +1,3 @@ +module.exports.set = function (obj, path, value) { + obj[path[0]][path[1]] = value; // NOT OK +} \ No newline at end of file diff --git a/javascript/ql/test/query-tests/Security/CWE-915/PrototypePollutingAssignment/sublib/sub/empty.js b/javascript/ql/test/query-tests/Security/CWE-915/PrototypePollutingAssignment/sublib/sub/empty.js new file mode 100644 index 00000000000..cc4c41e100b --- /dev/null +++ b/javascript/ql/test/query-tests/Security/CWE-915/PrototypePollutingAssignment/sublib/sub/empty.js @@ -0,0 +1 @@ +console.log("foo"); \ No newline at end of file From ebbd9c5b902d106276e2b7bf262a00850665b5c0 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Mon, 28 Mar 2022 10:02:42 +0200 Subject: [PATCH 83/91] C++: Handle C++17 if initializers --- cpp/ql/lib/semmle/code/cpp/PrintAST.qll | 4 + .../code/cpp/controlflow/internal/CFG.qll | 18 +- .../raw/internal/TranslatedStmt.qll | 27 ++- cpp/ql/lib/semmle/code/cpp/stmts/Stmt.qll | 48 +++- cpp/ql/lib/semmlecode.cpp.dbscheme | 10 + .../Unused Entities/UnusedLocals.ql | 3 +- .../library-tests/ir/ir/PrintAST.expected | 217 ++++++++++++++++++ cpp/ql/test/library-tests/ir/ir/ir.cpp | 28 +++ .../ir/ir/operand_locations.expected | 132 +++++++++++ .../test/library-tests/ir/ir/raw_ir.expected | 166 ++++++++++++++ .../successor-tests/ifstmt/ifstmt/ifstmt.cpp | 8 + .../ifstmt/ifstmt/ifstmt01.expected | 1 + .../ifstmt/ifstmt/ifstmt02.expected | 1 + .../ifstmt/ifstmt/ifstmt04.expected | 1 + .../successor-tests/ifstmt/ifstmt/ifstmt05.ql | 11 +- .../ifstmt/ifstmt/ifstmt10.expected | 15 ++ .../ifstmt/ifstmt/ifstmt11.expected | 0 .../successor-tests/ifstmt/ifstmt/ifstmt11.ql | 15 ++ 18 files changed, 692 insertions(+), 13 deletions(-) create mode 100644 cpp/ql/test/successor-tests/ifstmt/ifstmt/ifstmt.cpp create mode 100644 cpp/ql/test/successor-tests/ifstmt/ifstmt/ifstmt11.expected create mode 100644 cpp/ql/test/successor-tests/ifstmt/ifstmt/ifstmt11.ql diff --git a/cpp/ql/lib/semmle/code/cpp/PrintAST.qll b/cpp/ql/lib/semmle/code/cpp/PrintAST.qll index 7658d5cf17a..c7366128125 100644 --- a/cpp/ql/lib/semmle/code/cpp/PrintAST.qll +++ b/cpp/ql/lib/semmle/code/cpp/PrintAST.qll @@ -663,12 +663,16 @@ private predicate namedStmtChildPredicates(Locatable s, Element e, string pred) or s.(ComputedGotoStmt).getExpr() = e and pred = "getExpr()" or + s.(ConstexprIfStmt).getInitialization() = e and pred = "getInitialization()" + or s.(ConstexprIfStmt).getCondition() = e and pred = "getCondition()" or s.(ConstexprIfStmt).getThen() = e and pred = "getThen()" or s.(ConstexprIfStmt).getElse() = e and pred = "getElse()" or + s.(IfStmt).getInitialization() = e and pred = "getInitialization()" + or s.(IfStmt).getCondition() = e and pred = "getCondition()" or s.(IfStmt).getThen() = e and pred = "getThen()" diff --git a/cpp/ql/lib/semmle/code/cpp/controlflow/internal/CFG.qll b/cpp/ql/lib/semmle/code/cpp/controlflow/internal/CFG.qll index 96f03c931bf..4210db40a7b 100644 --- a/cpp/ql/lib/semmle/code/cpp/controlflow/internal/CFG.qll +++ b/cpp/ql/lib/semmle/code/cpp/controlflow/internal/CFG.qll @@ -836,8 +836,15 @@ private predicate subEdge(Pos p1, Node n1, Node n2, Pos p2) { p2.nodeAt(n2, f) ) or - // IfStmt -> condition ; { then, else } -> + // IfStmt -> [ init -> ] condition ; { then, else } -> exists(IfStmt s | + p1.nodeAt(n1, s) and + p2.nodeBefore(n2, s.getInitialization()) + or + p1.nodeAfter(n1, s.getInitialization()) and + p2.nodeBefore(n2, s.getCondition()) + or + not exists(s.getInitialization()) and p1.nodeAt(n1, s) and p2.nodeBefore(n2, s.getCondition()) or @@ -851,8 +858,15 @@ private predicate subEdge(Pos p1, Node n1, Node n2, Pos p2) { p2.nodeAfter(n2, s) ) or - // ConstexprIfStmt -> condition ; { then, else } -> // same as IfStmt + // ConstexprIfStmt -> [ init -> ] condition ; { then, else } -> // same as IfStmt exists(ConstexprIfStmt s | + p1.nodeAt(n1, s) and + p2.nodeBefore(n2, s.getInitialization()) + or + p1.nodeAfter(n1, s.getInitialization()) and + p2.nodeBefore(n2, s.getCondition()) + or + not exists(s.getInitialization()) and p1.nodeAt(n1, s) and p2.nodeBefore(n2, s.getCondition()) or diff --git a/cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/TranslatedStmt.qll b/cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/TranslatedStmt.qll index eb195286339..45d9f924b10 100644 --- a/cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/TranslatedStmt.qll +++ b/cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/TranslatedStmt.qll @@ -421,20 +421,36 @@ class TranslatedCatchAnyHandler extends TranslatedHandler { class TranslatedIfStmt extends TranslatedStmt, ConditionContext { override IfStmt stmt; - override Instruction getFirstInstruction() { result = getCondition().getFirstInstruction() } + override Instruction getFirstInstruction() { + if hasInitialization() + then result = getInitialization().getFirstInstruction() + else result = getFirstConditionInstruction() + } override TranslatedElement getChild(int id) { - id = 0 and result = getCondition() + id = 0 and result = getInitialization() or - id = 1 and result = getThen() + id = 1 and result = getCondition() or - id = 2 and result = getElse() + id = 2 and result = getThen() + or + id = 3 and result = getElse() + } + + private predicate hasInitialization() { exists(stmt.getInitialization()) } + + private TranslatedStmt getInitialization() { + result = getTranslatedStmt(stmt.getInitialization()) } private TranslatedCondition getCondition() { result = getTranslatedCondition(stmt.getCondition().getFullyConverted()) } + private Instruction getFirstConditionInstruction() { + result = getCondition().getFirstInstruction() + } + private TranslatedStmt getThen() { result = getTranslatedStmt(stmt.getThen()) } private TranslatedStmt getElse() { result = getTranslatedStmt(stmt.getElse()) } @@ -456,6 +472,9 @@ class TranslatedIfStmt extends TranslatedStmt, ConditionContext { } override Instruction getChildSuccessor(TranslatedElement child) { + child = getInitialization() and + result = getFirstConditionInstruction() + or (child = getThen() or child = getElse()) and result = getParent().getChildSuccessor(this) } diff --git a/cpp/ql/lib/semmle/code/cpp/stmts/Stmt.qll b/cpp/ql/lib/semmle/code/cpp/stmts/Stmt.qll index ef311e6fc45..084a4fb7bc7 100644 --- a/cpp/ql/lib/semmle/code/cpp/stmts/Stmt.qll +++ b/cpp/ql/lib/semmle/code/cpp/stmts/Stmt.qll @@ -213,6 +213,26 @@ class ConditionalStmt extends ControlStructure, TConditionalStmt { } class IfStmt extends ConditionalStmt, @stmt_if { override string getAPrimaryQlClass() { result = "IfStmt" } + /** + * Gets the initialization statement of this 'if' statement. + * + * For example, for + * ``` + * if (x = y; b) { f(); } + * ``` + * the result is `x = y;`. + * + * Does not hold if the initialization statement is missing or an empty statement, as in + * ``` + * if (b) { f(); } + * ``` + * or + * ``` + * if (; b) { f(); } + * ``` + */ + Stmt getInitialization() { if_initialization(underlyingElement(this), unresolveElement(result)) } + /** * Gets the condition expression of this 'if' statement. * @@ -222,7 +242,7 @@ class IfStmt extends ConditionalStmt, @stmt_if { * ``` * the result is `b`. */ - Expr getCondition() { result = this.getChild(0) } + Expr getCondition() { result = this.getChild(1) } override Expr getControllingExpr() { result = this.getCondition() } @@ -299,6 +319,28 @@ class IfStmt extends ConditionalStmt, @stmt_if { class ConstexprIfStmt extends ConditionalStmt, @stmt_constexpr_if { override string getAPrimaryQlClass() { result = "ConstexprIfStmt" } + /** + * Gets the initialization statement of this 'constexpr if' statement. + * + * For example, for + * ``` + * if constexpr (x = y; b) { f(); } + * ``` + * the result is `x = y;`. + * + * Does not hold if the initialization statement is missing or an empty statement, as in + * ``` + * if constexpr (b) { f(); } + * ``` + * or + * ``` + * if constexpr (; b) { f(); } + * ``` + */ + Stmt getInitialization() { + constexpr_if_initialization(underlyingElement(this), unresolveElement(result)) + } + /** * Gets the condition expression of this 'constexpr if' statement. * @@ -308,7 +350,7 @@ class ConstexprIfStmt extends ConditionalStmt, @stmt_constexpr_if { * ``` * the result is `b`. */ - Expr getCondition() { result = this.getChild(0) } + Expr getCondition() { result = this.getChild(1) } override Expr getControllingExpr() { result = this.getCondition() } @@ -926,7 +968,7 @@ class ForStmt extends Loop, @stmt_for { * * Does not hold if the initialization statement is an empty statement, as in * ``` - * for (; i < 10; i++) { j++ } + * for (; i < 10; i++) { j++; } * ``` */ Stmt getInitialization() { for_initialization(underlyingElement(this), unresolveElement(result)) } diff --git a/cpp/ql/lib/semmlecode.cpp.dbscheme b/cpp/ql/lib/semmlecode.cpp.dbscheme index e9a518baf14..233020bf9ad 100644 --- a/cpp/ql/lib/semmlecode.cpp.dbscheme +++ b/cpp/ql/lib/semmlecode.cpp.dbscheme @@ -1863,6 +1863,11 @@ variable_vla( int decl: @stmt_vla_decl ref ); +if_initialization( + unique int if_stmt: @stmt_if ref, + int init_id: @stmt ref +); + if_then( unique int if_stmt: @stmt_if ref, int then_id: @stmt ref @@ -1873,6 +1878,11 @@ if_else( int else_id: @stmt ref ); +constexpr_if_initialization( + unique int constexpr_if_stmt: @stmt_constexpr_if ref, + int init_id: @stmt ref +); + constexpr_if_then( unique int constexpr_if_stmt: @stmt_constexpr_if ref, int then_id: @stmt ref diff --git a/cpp/ql/src/Best Practices/Unused Entities/UnusedLocals.ql b/cpp/ql/src/Best Practices/Unused Entities/UnusedLocals.ql index 851d59f680f..cbb143dcebc 100644 --- a/cpp/ql/src/Best Practices/Unused Entities/UnusedLocals.ql +++ b/cpp/ql/src/Best Practices/Unused Entities/UnusedLocals.ql @@ -57,6 +57,5 @@ where not declarationHasSideEffects(v) and not exists(AsmStmt s | f = s.getEnclosingFunction()) and not v.getAnAttribute().getName() = "unused" and - not any(ErrorExpr e).getEnclosingFunction() = f and // unextracted expr may use `v` - not any(ConditionDeclExpr cde).getEnclosingFunction() = f // this case can be removed when the `if (a = b; a)` and `switch (a = b; a)` test cases don't depend on this exclusion + not any(ErrorExpr e).getEnclosingFunction() = f // unextracted expr may use `v` select v, "Variable " + v.getName() + " is not used" diff --git a/cpp/ql/test/library-tests/ir/ir/PrintAST.expected b/cpp/ql/test/library-tests/ir/ir/PrintAST.expected index af029bf3748..71fa09447b1 100644 --- a/cpp/ql/test/library-tests/ir/ir/PrintAST.expected +++ b/cpp/ql/test/library-tests/ir/ir/PrintAST.expected @@ -13559,6 +13559,223 @@ ir.cpp: # 1754| Type = [SpecifiedType] const CopyConstructorTestVirtualClass # 1754| ValueCategory = lvalue # 1755| getStmt(2): [ReturnStmt] return ... +# 1757| [TopLevelFunction] void if_initialization(int) +# 1757| : +# 1757| getParameter(0): [Parameter] x +# 1757| Type = [IntType] int +# 1757| getEntryPoint(): [BlockStmt] { ... } +# 1758| getStmt(0): [IfStmt] if (...) ... +# 1758| getInitialization(): [DeclStmt] declaration +# 1758| getDeclarationEntry(0): [VariableDeclarationEntry] definition of y +# 1758| Type = [IntType] int +# 1758| getVariable().getInitializer(): [Initializer] initializer for y +# 1758| getExpr(): [VariableAccess] x +# 1758| Type = [IntType] int +# 1758| ValueCategory = prvalue(load) +# 1758| getCondition(): [AddExpr] ... + ... +# 1758| Type = [IntType] int +# 1758| ValueCategory = prvalue +# 1758| getLeftOperand(): [VariableAccess] x +# 1758| Type = [IntType] int +# 1758| ValueCategory = prvalue(load) +# 1758| getRightOperand(): [Literal] 1 +# 1758| Type = [IntType] int +# 1758| Value = [Literal] 1 +# 1758| ValueCategory = prvalue +# 1758| getThen(): [BlockStmt] { ... } +# 1759| getStmt(0): [ExprStmt] ExprStmt +# 1759| getExpr(): [AssignExpr] ... = ... +# 1759| Type = [IntType] int +# 1759| ValueCategory = lvalue +# 1759| getLValue(): [VariableAccess] x +# 1759| Type = [IntType] int +# 1759| ValueCategory = lvalue +# 1759| getRValue(): [AddExpr] ... + ... +# 1759| Type = [IntType] int +# 1759| ValueCategory = prvalue +# 1759| getLeftOperand(): [VariableAccess] x +# 1759| Type = [IntType] int +# 1759| ValueCategory = prvalue(load) +# 1759| getRightOperand(): [VariableAccess] y +# 1759| Type = [IntType] int +# 1759| ValueCategory = prvalue(load) +# 1758| getCondition().getFullyConverted(): [CStyleCast] (bool)... +# 1758| Conversion = [BoolConversion] conversion to bool +# 1758| Type = [BoolType] bool +# 1758| ValueCategory = prvalue +# 1762| getStmt(1): [DeclStmt] declaration +# 1762| getDeclarationEntry(0): [VariableDeclarationEntry] definition of w +# 1762| Type = [IntType] int +# 1763| getStmt(2): [IfStmt] if (...) ... +# 1763| getInitialization(): [ExprStmt] ExprStmt +# 1763| getExpr(): [AssignExpr] ... = ... +# 1763| Type = [IntType] int +# 1763| ValueCategory = lvalue +# 1763| getLValue(): [VariableAccess] w +# 1763| Type = [IntType] int +# 1763| ValueCategory = lvalue +# 1763| getRValue(): [VariableAccess] x +# 1763| Type = [IntType] int +# 1763| ValueCategory = prvalue(load) +# 1763| getCondition(): [AddExpr] ... + ... +# 1763| Type = [IntType] int +# 1763| ValueCategory = prvalue +# 1763| getLeftOperand(): [VariableAccess] x +# 1763| Type = [IntType] int +# 1763| ValueCategory = prvalue(load) +# 1763| getRightOperand(): [Literal] 1 +# 1763| Type = [IntType] int +# 1763| Value = [Literal] 1 +# 1763| ValueCategory = prvalue +# 1763| getThen(): [BlockStmt] { ... } +# 1764| getStmt(0): [ExprStmt] ExprStmt +# 1764| getExpr(): [AssignExpr] ... = ... +# 1764| Type = [IntType] int +# 1764| ValueCategory = lvalue +# 1764| getLValue(): [VariableAccess] x +# 1764| Type = [IntType] int +# 1764| ValueCategory = lvalue +# 1764| getRValue(): [AddExpr] ... + ... +# 1764| Type = [IntType] int +# 1764| ValueCategory = prvalue +# 1764| getLeftOperand(): [VariableAccess] x +# 1764| Type = [IntType] int +# 1764| ValueCategory = prvalue(load) +# 1764| getRightOperand(): [VariableAccess] w +# 1764| Type = [IntType] int +# 1764| ValueCategory = prvalue(load) +# 1763| getCondition().getFullyConverted(): [CStyleCast] (bool)... +# 1763| Conversion = [BoolConversion] conversion to bool +# 1763| Type = [BoolType] bool +# 1763| ValueCategory = prvalue +# 1767| getStmt(3): [IfStmt] if (...) ... +# 1767| getInitialization(): [ExprStmt] ExprStmt +# 1767| getExpr(): [AssignExpr] ... = ... +# 1767| Type = [IntType] int +# 1767| ValueCategory = lvalue +# 1767| getLValue(): [VariableAccess] w +# 1767| Type = [IntType] int +# 1767| ValueCategory = lvalue +# 1767| getRValue(): [VariableAccess] x +# 1767| Type = [IntType] int +# 1767| ValueCategory = prvalue(load) +# 1767| getCondition(): [ConditionDeclExpr] (condition decl) +# 1767| Type = [BoolType] bool +# 1767| ValueCategory = prvalue +# 1767| getVariableAccess(): [VariableAccess] w2 +# 1767| Type = [IntType] int +# 1767| ValueCategory = prvalue(load) +# 1767| getVariableAccess().getFullyConverted(): [CStyleCast] (bool)... +# 1767| Conversion = [BoolConversion] conversion to bool +# 1767| Type = [BoolType] bool +# 1767| ValueCategory = prvalue +# 1767| getThen(): [BlockStmt] { ... } +# 1768| getStmt(0): [ExprStmt] ExprStmt +# 1768| getExpr(): [AssignExpr] ... = ... +# 1768| Type = [IntType] int +# 1768| ValueCategory = lvalue +# 1768| getLValue(): [VariableAccess] x +# 1768| Type = [IntType] int +# 1768| ValueCategory = lvalue +# 1768| getRValue(): [AddExpr] ... + ... +# 1768| Type = [IntType] int +# 1768| ValueCategory = prvalue +# 1768| getLeftOperand(): [VariableAccess] x +# 1768| Type = [IntType] int +# 1768| ValueCategory = prvalue(load) +# 1768| getRightOperand(): [VariableAccess] w +# 1768| Type = [IntType] int +# 1768| ValueCategory = prvalue(load) +# 1771| getStmt(4): [IfStmt] if (...) ... +# 1771| getInitialization(): [DeclStmt] declaration +# 1771| getDeclarationEntry(0): [VariableDeclarationEntry] definition of v +# 1771| Type = [IntType] int +# 1771| getVariable().getInitializer(): [Initializer] initializer for v +# 1771| getExpr(): [VariableAccess] x +# 1771| Type = [IntType] int +# 1771| ValueCategory = prvalue(load) +# 1771| getCondition(): [ConditionDeclExpr] (condition decl) +# 1771| Type = [BoolType] bool +# 1771| ValueCategory = prvalue +# 1771| getVariableAccess(): [VariableAccess] v2 +# 1771| Type = [IntType] int +# 1771| ValueCategory = prvalue(load) +# 1771| getVariableAccess().getFullyConverted(): [CStyleCast] (bool)... +# 1771| Conversion = [BoolConversion] conversion to bool +# 1771| Type = [BoolType] bool +# 1771| ValueCategory = prvalue +# 1771| getThen(): [BlockStmt] { ... } +# 1772| getStmt(0): [ExprStmt] ExprStmt +# 1772| getExpr(): [AssignExpr] ... = ... +# 1772| Type = [IntType] int +# 1772| ValueCategory = lvalue +# 1772| getLValue(): [VariableAccess] x +# 1772| Type = [IntType] int +# 1772| ValueCategory = lvalue +# 1772| getRValue(): [AddExpr] ... + ... +# 1772| Type = [IntType] int +# 1772| ValueCategory = prvalue +# 1772| getLeftOperand(): [VariableAccess] x +# 1772| Type = [IntType] int +# 1772| ValueCategory = prvalue(load) +# 1772| getRightOperand(): [VariableAccess] v +# 1772| Type = [IntType] int +# 1772| ValueCategory = prvalue(load) +# 1775| getStmt(5): [DeclStmt] declaration +# 1775| getDeclarationEntry(0): [VariableDeclarationEntry] definition of z +# 1775| Type = [IntType] int +# 1775| getVariable().getInitializer(): [Initializer] initializer for z +# 1775| getExpr(): [VariableAccess] x +# 1775| Type = [IntType] int +# 1775| ValueCategory = prvalue(load) +# 1776| getStmt(6): [IfStmt] if (...) ... +# 1776| getCondition(): [VariableAccess] z +# 1776| Type = [IntType] int +# 1776| ValueCategory = prvalue(load) +# 1776| getThen(): [BlockStmt] { ... } +# 1777| getStmt(0): [ExprStmt] ExprStmt +# 1777| getExpr(): [AssignExpr] ... = ... +# 1777| Type = [IntType] int +# 1777| ValueCategory = lvalue +# 1777| getLValue(): [VariableAccess] x +# 1777| Type = [IntType] int +# 1777| ValueCategory = lvalue +# 1777| getRValue(): [AddExpr] ... + ... +# 1777| Type = [IntType] int +# 1777| ValueCategory = prvalue +# 1777| getLeftOperand(): [VariableAccess] x +# 1777| Type = [IntType] int +# 1777| ValueCategory = prvalue(load) +# 1777| getRightOperand(): [VariableAccess] z +# 1777| Type = [IntType] int +# 1777| ValueCategory = prvalue(load) +# 1776| getCondition().getFullyConverted(): [CStyleCast] (bool)... +# 1776| Conversion = [BoolConversion] conversion to bool +# 1776| Type = [BoolType] bool +# 1776| ValueCategory = prvalue +# 1780| getStmt(7): [IfStmt] if (...) ... +# 1780| getCondition(): [ConditionDeclExpr] (condition decl) +# 1780| Type = [BoolType] bool +# 1780| ValueCategory = prvalue +# 1780| getVariableAccess(): [VariableAccess] z2 +# 1780| Type = [IntType] int +# 1780| ValueCategory = prvalue(load) +# 1780| getVariableAccess().getFullyConverted(): [CStyleCast] (bool)... +# 1780| Conversion = [BoolConversion] conversion to bool +# 1780| Type = [BoolType] bool +# 1780| ValueCategory = prvalue +# 1780| getThen(): [BlockStmt] { ... } +# 1781| getStmt(0): [ExprStmt] ExprStmt +# 1781| getExpr(): [AssignAddExpr] ... += ... +# 1781| Type = [IntType] int +# 1781| ValueCategory = lvalue +# 1781| getLValue(): [VariableAccess] x +# 1781| Type = [IntType] int +# 1781| ValueCategory = lvalue +# 1781| getRValue(): [VariableAccess] z2 +# 1781| Type = [IntType] int +# 1781| ValueCategory = prvalue(load) +# 1783| getStmt(8): [ReturnStmt] return ... perf-regression.cpp: # 4| [CopyAssignmentOperator] Big& Big::operator=(Big const&) # 4| : diff --git a/cpp/ql/test/library-tests/ir/ir/ir.cpp b/cpp/ql/test/library-tests/ir/ir/ir.cpp index 4e3006c8901..cf7cb477585 100644 --- a/cpp/ql/test/library-tests/ir/ir/ir.cpp +++ b/cpp/ql/test/library-tests/ir/ir/ir.cpp @@ -1754,4 +1754,32 @@ int implicit_copy_constructor_test( CopyConstructorTestVirtualClass cy = y; } +void if_initialization(int x) { + if (int y = x; x + 1) { + x = x + y; + } + + int w; + if (w = x; x + 1) { + x = x + w; + } + + if (w = x; int w2 = w) { + x = x + w; + } + + if (int v = x; int v2 = v) { + x = x + v; + } + + int z = x; + if (z) { + x = x + z; + } + + if (int z2 = z) { + x += z2; + } +} + // semmle-extractor-options: -std=c++17 --clang diff --git a/cpp/ql/test/library-tests/ir/ir/operand_locations.expected b/cpp/ql/test/library-tests/ir/ir/operand_locations.expected index 259cc553dbb..378a2c77603 100644 --- a/cpp/ql/test/library-tests/ir/ir/operand_locations.expected +++ b/cpp/ql/test/library-tests/ir/ir/operand_locations.expected @@ -8215,6 +8215,138 @@ | ir.cpp:1754:42:1754:42 | SideEffect | ~m1752_4 | | ir.cpp:1754:42:1754:42 | Unary | r1754_5 | | ir.cpp:1754:42:1754:42 | Unary | r1754_6 | +| ir.cpp:1757:6:1757:22 | ChiPartial | partial:m1757_3 | +| ir.cpp:1757:6:1757:22 | ChiTotal | total:m1757_2 | +| ir.cpp:1757:6:1757:22 | SideEffect | m1757_3 | +| ir.cpp:1757:28:1757:28 | Address | &:r1757_5 | +| ir.cpp:1758:13:1758:13 | Address | &:r1758_1 | +| ir.cpp:1758:17:1758:17 | Address | &:r1758_2 | +| ir.cpp:1758:17:1758:17 | Load | m1757_6 | +| ir.cpp:1758:17:1758:17 | StoreValue | r1758_3 | +| ir.cpp:1758:20:1758:20 | Address | &:r1758_5 | +| ir.cpp:1758:20:1758:20 | Left | r1758_6 | +| ir.cpp:1758:20:1758:20 | Load | m1757_6 | +| ir.cpp:1758:20:1758:24 | Condition | r1758_10 | +| ir.cpp:1758:20:1758:24 | Left | r1758_8 | +| ir.cpp:1758:20:1758:24 | Right | r1758_9 | +| ir.cpp:1758:24:1758:24 | Right | r1758_7 | +| ir.cpp:1759:9:1759:9 | Address | &:r1759_6 | +| ir.cpp:1759:13:1759:13 | Address | &:r1759_1 | +| ir.cpp:1759:13:1759:13 | Left | r1759_2 | +| ir.cpp:1759:13:1759:13 | Load | m1757_6 | +| ir.cpp:1759:13:1759:17 | StoreValue | r1759_5 | +| ir.cpp:1759:17:1759:17 | Address | &:r1759_3 | +| ir.cpp:1759:17:1759:17 | Load | m1758_4 | +| ir.cpp:1759:17:1759:17 | Right | r1759_4 | +| ir.cpp:1762:9:1762:9 | Address | &:r1762_2 | +| ir.cpp:1762:9:1762:9 | Phi | from 0:m1757_6 | +| ir.cpp:1762:9:1762:9 | Phi | from 1:m1759_7 | +| ir.cpp:1763:9:1763:9 | Address | &:r1763_3 | +| ir.cpp:1763:13:1763:13 | Address | &:r1763_1 | +| ir.cpp:1763:13:1763:13 | Load | m1762_1 | +| ir.cpp:1763:13:1763:13 | StoreValue | r1763_2 | +| ir.cpp:1763:16:1763:16 | Address | &:r1763_5 | +| ir.cpp:1763:16:1763:16 | Left | r1763_6 | +| ir.cpp:1763:16:1763:16 | Load | m1762_1 | +| ir.cpp:1763:16:1763:20 | Condition | r1763_10 | +| ir.cpp:1763:16:1763:20 | Left | r1763_8 | +| ir.cpp:1763:16:1763:20 | Right | r1763_9 | +| ir.cpp:1763:20:1763:20 | Right | r1763_7 | +| ir.cpp:1764:9:1764:9 | Address | &:r1764_6 | +| ir.cpp:1764:13:1764:13 | Address | &:r1764_1 | +| ir.cpp:1764:13:1764:13 | Left | r1764_2 | +| ir.cpp:1764:13:1764:13 | Load | m1762_1 | +| ir.cpp:1764:13:1764:17 | StoreValue | r1764_5 | +| ir.cpp:1764:17:1764:17 | Address | &:r1764_3 | +| ir.cpp:1764:17:1764:17 | Load | m1763_4 | +| ir.cpp:1764:17:1764:17 | Right | r1764_4 | +| ir.cpp:1767:9:1767:9 | Address | &:r1767_4 | +| ir.cpp:1767:13:1767:13 | Address | &:r1767_2 | +| ir.cpp:1767:13:1767:13 | Load | m1767_1 | +| ir.cpp:1767:13:1767:13 | Phi | from 2:m1762_1 | +| ir.cpp:1767:13:1767:13 | Phi | from 3:m1764_7 | +| ir.cpp:1767:13:1767:13 | StoreValue | r1767_3 | +| ir.cpp:1767:14:1767:25 | Address | &:r1767_6 | +| ir.cpp:1767:14:1767:25 | Condition | r1767_14 | +| ir.cpp:1767:20:1767:21 | Address | &:r1767_10 | +| ir.cpp:1767:20:1767:21 | Left | r1767_11 | +| ir.cpp:1767:20:1767:21 | Load | m1767_9 | +| ir.cpp:1767:20:1767:21 | Right | r1767_12 | +| ir.cpp:1767:20:1767:21 | Unary | r1767_13 | +| ir.cpp:1767:25:1767:25 | Address | &:r1767_7 | +| ir.cpp:1767:25:1767:25 | Load | m1767_5 | +| ir.cpp:1767:25:1767:25 | StoreValue | r1767_8 | +| ir.cpp:1768:9:1768:9 | Address | &:r1768_6 | +| ir.cpp:1768:13:1768:13 | Address | &:r1768_1 | +| ir.cpp:1768:13:1768:13 | Left | r1768_2 | +| ir.cpp:1768:13:1768:13 | Load | m1767_1 | +| ir.cpp:1768:13:1768:17 | StoreValue | r1768_5 | +| ir.cpp:1768:17:1768:17 | Address | &:r1768_3 | +| ir.cpp:1768:17:1768:17 | Load | m1767_5 | +| ir.cpp:1768:17:1768:17 | Right | r1768_4 | +| ir.cpp:1771:9:1771:29 | Address | &:r1771_6 | +| ir.cpp:1771:9:1771:29 | Condition | r1771_14 | +| ir.cpp:1771:13:1771:13 | Address | &:r1771_2 | +| ir.cpp:1771:13:1771:13 | Phi | from 4:m1767_1 | +| ir.cpp:1771:13:1771:13 | Phi | from 5:m1768_7 | +| ir.cpp:1771:17:1771:17 | Address | &:r1771_3 | +| ir.cpp:1771:17:1771:17 | Load | m1771_1 | +| ir.cpp:1771:17:1771:17 | StoreValue | r1771_4 | +| ir.cpp:1771:24:1771:25 | Address | &:r1771_10 | +| ir.cpp:1771:24:1771:25 | Left | r1771_11 | +| ir.cpp:1771:24:1771:25 | Load | m1771_9 | +| ir.cpp:1771:24:1771:25 | Right | r1771_12 | +| ir.cpp:1771:24:1771:25 | Unary | r1771_13 | +| ir.cpp:1771:29:1771:29 | Address | &:r1771_7 | +| ir.cpp:1771:29:1771:29 | Load | m1771_5 | +| ir.cpp:1771:29:1771:29 | StoreValue | r1771_8 | +| ir.cpp:1772:9:1772:9 | Address | &:r1772_6 | +| ir.cpp:1772:13:1772:13 | Address | &:r1772_1 | +| ir.cpp:1772:13:1772:13 | Left | r1772_2 | +| ir.cpp:1772:13:1772:13 | Load | m1771_1 | +| ir.cpp:1772:13:1772:17 | StoreValue | r1772_5 | +| ir.cpp:1772:17:1772:17 | Address | &:r1772_3 | +| ir.cpp:1772:17:1772:17 | Load | m1771_5 | +| ir.cpp:1772:17:1772:17 | Right | r1772_4 | +| ir.cpp:1775:9:1775:9 | Address | &:r1775_2 | +| ir.cpp:1775:9:1775:9 | Phi | from 6:m1771_1 | +| ir.cpp:1775:9:1775:9 | Phi | from 7:m1772_7 | +| ir.cpp:1775:13:1775:13 | Address | &:r1775_3 | +| ir.cpp:1775:13:1775:13 | Load | m1775_1 | +| ir.cpp:1775:13:1775:13 | StoreValue | r1775_4 | +| ir.cpp:1776:9:1776:9 | Address | &:r1776_1 | +| ir.cpp:1776:9:1776:9 | Condition | r1776_4 | +| ir.cpp:1776:9:1776:9 | Left | r1776_2 | +| ir.cpp:1776:9:1776:9 | Load | m1775_5 | +| ir.cpp:1776:9:1776:9 | Right | r1776_3 | +| ir.cpp:1777:9:1777:9 | Address | &:r1777_6 | +| ir.cpp:1777:13:1777:13 | Address | &:r1777_1 | +| ir.cpp:1777:13:1777:13 | Left | r1777_2 | +| ir.cpp:1777:13:1777:13 | Load | m1775_1 | +| ir.cpp:1777:13:1777:17 | StoreValue | r1777_5 | +| ir.cpp:1777:17:1777:17 | Address | &:r1777_3 | +| ir.cpp:1777:17:1777:17 | Load | m1775_5 | +| ir.cpp:1777:17:1777:17 | Right | r1777_4 | +| ir.cpp:1780:9:1780:18 | Address | &:r1780_2 | +| ir.cpp:1780:9:1780:18 | Condition | r1780_10 | +| ir.cpp:1780:9:1780:18 | Phi | from 8:m1775_1 | +| ir.cpp:1780:9:1780:18 | Phi | from 9:m1777_7 | +| ir.cpp:1780:13:1780:14 | Address | &:r1780_6 | +| ir.cpp:1780:13:1780:14 | Left | r1780_7 | +| ir.cpp:1780:13:1780:14 | Load | m1780_5 | +| ir.cpp:1780:13:1780:14 | Right | r1780_8 | +| ir.cpp:1780:13:1780:14 | Unary | r1780_9 | +| ir.cpp:1780:18:1780:18 | Address | &:r1780_3 | +| ir.cpp:1780:18:1780:18 | Load | m1775_5 | +| ir.cpp:1780:18:1780:18 | StoreValue | r1780_4 | +| ir.cpp:1781:9:1781:9 | Address | &:r1781_3 | +| ir.cpp:1781:9:1781:9 | Address | &:r1781_3 | +| ir.cpp:1781:9:1781:9 | Left | r1781_4 | +| ir.cpp:1781:9:1781:9 | Load | m1780_1 | +| ir.cpp:1781:9:1781:15 | StoreValue | r1781_5 | +| ir.cpp:1781:14:1781:15 | Address | &:r1781_1 | +| ir.cpp:1781:14:1781:15 | Load | m1780_5 | +| ir.cpp:1781:14:1781:15 | Right | r1781_2 | | perf-regression.cpp:6:3:6:5 | Address | &:r6_5 | | perf-regression.cpp:6:3:6:5 | Address | &:r6_5 | | perf-regression.cpp:6:3:6:5 | Address | &:r6_7 | diff --git a/cpp/ql/test/library-tests/ir/ir/raw_ir.expected b/cpp/ql/test/library-tests/ir/ir/raw_ir.expected index 3f49642c4fb..3c74321c413 100644 --- a/cpp/ql/test/library-tests/ir/ir/raw_ir.expected +++ b/cpp/ql/test/library-tests/ir/ir/raw_ir.expected @@ -9418,6 +9418,172 @@ ir.cpp: # 1750| v1750_6(void) = AliasedUse : ~m? # 1750| v1750_7(void) = ExitFunction : +# 1757| void if_initialization(int) +# 1757| Block 0 +# 1757| v1757_1(void) = EnterFunction : +# 1757| mu1757_2(unknown) = AliasedDefinition : +# 1757| mu1757_3(unknown) = InitializeNonLocal : +# 1757| r1757_4(glval) = VariableAddress[x] : +# 1757| mu1757_5(int) = InitializeParameter[x] : &:r1757_4 +# 1758| r1758_1(glval) = VariableAddress[y] : +# 1758| r1758_2(glval) = VariableAddress[x] : +# 1758| r1758_3(int) = Load[x] : &:r1758_2, ~m? +# 1758| mu1758_4(int) = Store[y] : &:r1758_1, r1758_3 +# 1758| r1758_5(glval) = VariableAddress[x] : +# 1758| r1758_6(int) = Load[x] : &:r1758_5, ~m? +# 1758| r1758_7(int) = Constant[1] : +# 1758| r1758_8(int) = Add : r1758_6, r1758_7 +# 1758| r1758_9(int) = Constant[0] : +# 1758| r1758_10(bool) = CompareNE : r1758_8, r1758_9 +# 1758| v1758_11(void) = ConditionalBranch : r1758_10 +#-----| False -> Block 2 +#-----| True -> Block 1 + +# 1759| Block 1 +# 1759| r1759_1(glval) = VariableAddress[x] : +# 1759| r1759_2(int) = Load[x] : &:r1759_1, ~m? +# 1759| r1759_3(glval) = VariableAddress[y] : +# 1759| r1759_4(int) = Load[y] : &:r1759_3, ~m? +# 1759| r1759_5(int) = Add : r1759_2, r1759_4 +# 1759| r1759_6(glval) = VariableAddress[x] : +# 1759| mu1759_7(int) = Store[x] : &:r1759_6, r1759_5 +#-----| Goto -> Block 2 + +# 1762| Block 2 +# 1762| r1762_1(glval) = VariableAddress[w] : +# 1762| mu1762_2(int) = Uninitialized[w] : &:r1762_1 +# 1763| r1763_1(glval) = VariableAddress[x] : +# 1763| r1763_2(int) = Load[x] : &:r1763_1, ~m? +# 1763| r1763_3(glval) = VariableAddress[w] : +# 1763| mu1763_4(int) = Store[w] : &:r1763_3, r1763_2 +# 1763| r1763_5(glval) = VariableAddress[x] : +# 1763| r1763_6(int) = Load[x] : &:r1763_5, ~m? +# 1763| r1763_7(int) = Constant[1] : +# 1763| r1763_8(int) = Add : r1763_6, r1763_7 +# 1763| r1763_9(int) = Constant[0] : +# 1763| r1763_10(bool) = CompareNE : r1763_8, r1763_9 +# 1763| v1763_11(void) = ConditionalBranch : r1763_10 +#-----| False -> Block 4 +#-----| True -> Block 3 + +# 1764| Block 3 +# 1764| r1764_1(glval) = VariableAddress[x] : +# 1764| r1764_2(int) = Load[x] : &:r1764_1, ~m? +# 1764| r1764_3(glval) = VariableAddress[w] : +# 1764| r1764_4(int) = Load[w] : &:r1764_3, ~m? +# 1764| r1764_5(int) = Add : r1764_2, r1764_4 +# 1764| r1764_6(glval) = VariableAddress[x] : +# 1764| mu1764_7(int) = Store[x] : &:r1764_6, r1764_5 +#-----| Goto -> Block 4 + +# 1767| Block 4 +# 1767| r1767_1(glval) = VariableAddress[x] : +# 1767| r1767_2(int) = Load[x] : &:r1767_1, ~m? +# 1767| r1767_3(glval) = VariableAddress[w] : +# 1767| mu1767_4(int) = Store[w] : &:r1767_3, r1767_2 +# 1767| r1767_5(glval) = VariableAddress[w2] : +# 1767| r1767_6(glval) = VariableAddress[w] : +# 1767| r1767_7(int) = Load[w] : &:r1767_6, ~m? +# 1767| mu1767_8(int) = Store[w2] : &:r1767_5, r1767_7 +# 1767| r1767_9(glval) = VariableAddress[w2] : +# 1767| r1767_10(int) = Load[w2] : &:r1767_9, ~m? +# 1767| r1767_11(int) = Constant[0] : +# 1767| r1767_12(bool) = CompareNE : r1767_10, r1767_11 +# 1767| r1767_13(bool) = CopyValue : r1767_12 +# 1767| v1767_14(void) = ConditionalBranch : r1767_13 +#-----| False -> Block 6 +#-----| True -> Block 5 + +# 1768| Block 5 +# 1768| r1768_1(glval) = VariableAddress[x] : +# 1768| r1768_2(int) = Load[x] : &:r1768_1, ~m? +# 1768| r1768_3(glval) = VariableAddress[w] : +# 1768| r1768_4(int) = Load[w] : &:r1768_3, ~m? +# 1768| r1768_5(int) = Add : r1768_2, r1768_4 +# 1768| r1768_6(glval) = VariableAddress[x] : +# 1768| mu1768_7(int) = Store[x] : &:r1768_6, r1768_5 +#-----| Goto -> Block 6 + +# 1771| Block 6 +# 1771| r1771_1(glval) = VariableAddress[v] : +# 1771| r1771_2(glval) = VariableAddress[x] : +# 1771| r1771_3(int) = Load[x] : &:r1771_2, ~m? +# 1771| mu1771_4(int) = Store[v] : &:r1771_1, r1771_3 +# 1771| r1771_5(glval) = VariableAddress[v2] : +# 1771| r1771_6(glval) = VariableAddress[v] : +# 1771| r1771_7(int) = Load[v] : &:r1771_6, ~m? +# 1771| mu1771_8(int) = Store[v2] : &:r1771_5, r1771_7 +# 1771| r1771_9(glval) = VariableAddress[v2] : +# 1771| r1771_10(int) = Load[v2] : &:r1771_9, ~m? +# 1771| r1771_11(int) = Constant[0] : +# 1771| r1771_12(bool) = CompareNE : r1771_10, r1771_11 +# 1771| r1771_13(bool) = CopyValue : r1771_12 +# 1771| v1771_14(void) = ConditionalBranch : r1771_13 +#-----| False -> Block 8 +#-----| True -> Block 7 + +# 1772| Block 7 +# 1772| r1772_1(glval) = VariableAddress[x] : +# 1772| r1772_2(int) = Load[x] : &:r1772_1, ~m? +# 1772| r1772_3(glval) = VariableAddress[v] : +# 1772| r1772_4(int) = Load[v] : &:r1772_3, ~m? +# 1772| r1772_5(int) = Add : r1772_2, r1772_4 +# 1772| r1772_6(glval) = VariableAddress[x] : +# 1772| mu1772_7(int) = Store[x] : &:r1772_6, r1772_5 +#-----| Goto -> Block 8 + +# 1775| Block 8 +# 1775| r1775_1(glval) = VariableAddress[z] : +# 1775| r1775_2(glval) = VariableAddress[x] : +# 1775| r1775_3(int) = Load[x] : &:r1775_2, ~m? +# 1775| mu1775_4(int) = Store[z] : &:r1775_1, r1775_3 +# 1776| r1776_1(glval) = VariableAddress[z] : +# 1776| r1776_2(int) = Load[z] : &:r1776_1, ~m? +# 1776| r1776_3(int) = Constant[0] : +# 1776| r1776_4(bool) = CompareNE : r1776_2, r1776_3 +# 1776| v1776_5(void) = ConditionalBranch : r1776_4 +#-----| False -> Block 10 +#-----| True -> Block 9 + +# 1777| Block 9 +# 1777| r1777_1(glval) = VariableAddress[x] : +# 1777| r1777_2(int) = Load[x] : &:r1777_1, ~m? +# 1777| r1777_3(glval) = VariableAddress[z] : +# 1777| r1777_4(int) = Load[z] : &:r1777_3, ~m? +# 1777| r1777_5(int) = Add : r1777_2, r1777_4 +# 1777| r1777_6(glval) = VariableAddress[x] : +# 1777| mu1777_7(int) = Store[x] : &:r1777_6, r1777_5 +#-----| Goto -> Block 10 + +# 1780| Block 10 +# 1780| r1780_1(glval) = VariableAddress[z2] : +# 1780| r1780_2(glval) = VariableAddress[z] : +# 1780| r1780_3(int) = Load[z] : &:r1780_2, ~m? +# 1780| mu1780_4(int) = Store[z2] : &:r1780_1, r1780_3 +# 1780| r1780_5(glval) = VariableAddress[z2] : +# 1780| r1780_6(int) = Load[z2] : &:r1780_5, ~m? +# 1780| r1780_7(int) = Constant[0] : +# 1780| r1780_8(bool) = CompareNE : r1780_6, r1780_7 +# 1780| r1780_9(bool) = CopyValue : r1780_8 +# 1780| v1780_10(void) = ConditionalBranch : r1780_9 +#-----| False -> Block 12 +#-----| True -> Block 11 + +# 1781| Block 11 +# 1781| r1781_1(glval) = VariableAddress[z2] : +# 1781| r1781_2(int) = Load[z2] : &:r1781_1, ~m? +# 1781| r1781_3(glval) = VariableAddress[x] : +# 1781| r1781_4(int) = Load[x] : &:r1781_3, ~m? +# 1781| r1781_5(int) = Add : r1781_4, r1781_2 +# 1781| mu1781_6(int) = Store[x] : &:r1781_3, r1781_5 +#-----| Goto -> Block 12 + +# 1783| Block 12 +# 1783| v1783_1(void) = NoOp : +# 1757| v1757_6(void) = ReturnVoid : +# 1757| v1757_7(void) = AliasedUse : ~m? +# 1757| v1757_8(void) = ExitFunction : + perf-regression.cpp: # 6| void Big::Big() # 6| Block 0 diff --git a/cpp/ql/test/successor-tests/ifstmt/ifstmt/ifstmt.cpp b/cpp/ql/test/successor-tests/ifstmt/ifstmt/ifstmt.cpp new file mode 100644 index 00000000000..b8d09edffca --- /dev/null +++ b/cpp/ql/test/successor-tests/ifstmt/ifstmt/ifstmt.cpp @@ -0,0 +1,8 @@ +void normal(int x, int y) { + if(int z = y; x == z) { + l1:; + } + l2:; +} + +// semmle-extractor-options: -std=c++17 diff --git a/cpp/ql/test/successor-tests/ifstmt/ifstmt/ifstmt01.expected b/cpp/ql/test/successor-tests/ifstmt/ifstmt/ifstmt01.expected index 9ed533ab840..73752ac3834 100644 --- a/cpp/ql/test/successor-tests/ifstmt/ifstmt/ifstmt01.expected +++ b/cpp/ql/test/successor-tests/ifstmt/ifstmt/ifstmt01.expected @@ -1 +1,2 @@ | ifstmt.c:28:6:28:11 | ... == ... | l2 | +| ifstmt.cpp:2:17:2:22 | ... == ... | l2 | diff --git a/cpp/ql/test/successor-tests/ifstmt/ifstmt/ifstmt02.expected b/cpp/ql/test/successor-tests/ifstmt/ifstmt/ifstmt02.expected index 1dfd5b6042e..fcc99833adc 100644 --- a/cpp/ql/test/successor-tests/ifstmt/ifstmt/ifstmt02.expected +++ b/cpp/ql/test/successor-tests/ifstmt/ifstmt/ifstmt02.expected @@ -1 +1,2 @@ | ifstmt.c:28:6:28:11 | ... == ... | l1 | +| ifstmt.cpp:2:17:2:22 | ... == ... | l1 | diff --git a/cpp/ql/test/successor-tests/ifstmt/ifstmt/ifstmt04.expected b/cpp/ql/test/successor-tests/ifstmt/ifstmt/ifstmt04.expected index a3eb12cc2c2..6de3a7b13f0 100644 --- a/cpp/ql/test/successor-tests/ifstmt/ifstmt/ifstmt04.expected +++ b/cpp/ql/test/successor-tests/ifstmt/ifstmt/ifstmt04.expected @@ -1 +1,2 @@ | ifstmt.c:29:8:29:8 | ; | l2 | +| ifstmt.cpp:3:8:3:8 | ; | l2 | diff --git a/cpp/ql/test/successor-tests/ifstmt/ifstmt/ifstmt05.ql b/cpp/ql/test/successor-tests/ifstmt/ifstmt/ifstmt05.ql index 46d6d103b8a..e59a59a01f4 100644 --- a/cpp/ql/test/successor-tests/ifstmt/ifstmt/ifstmt05.ql +++ b/cpp/ql/test/successor-tests/ifstmt/ifstmt/ifstmt05.ql @@ -1,6 +1,9 @@ /** * @name ifstmt05 - * @description Every if statement has its condition or one of the condition's descendants as its unique successor. + * @description Every if statement with an initialization has the initialization or one of the + * initialization's descendants as its unique successor. Every if statement without + * and initialization has its condition or one of the condition's descendants as + * its unique successor. */ import cpp @@ -8,7 +11,11 @@ import cpp from IfStmt is where not ( - is.getASuccessor() = is.getCondition().getAChild*() and + ( + if exists(is.getInitialization()) + then is.getASuccessor() = is.getInitialization().getAChild*() + else is.getASuccessor() = is.getCondition().getAChild*() + ) and count(is.getASuccessor()) = 1 ) select is diff --git a/cpp/ql/test/successor-tests/ifstmt/ifstmt/ifstmt10.expected b/cpp/ql/test/successor-tests/ifstmt/ifstmt/ifstmt10.expected index 8ffff11ac6e..a031d5f7d66 100644 --- a/cpp/ql/test/successor-tests/ifstmt/ifstmt/ifstmt10.expected +++ b/cpp/ql/test/successor-tests/ifstmt/ifstmt/ifstmt10.expected @@ -1,12 +1,27 @@ | 0 | ifstmt.c:27:27:32:1 | { ... } | 1 | ifstmt.c:28:3:30:3 | if (...) ... | +| 0 | ifstmt.cpp:1:27:6:1 | { ... } | 1 | ifstmt.cpp:2:3:4:3 | if (...) ... | | 1 | ifstmt.c:28:3:30:3 | if (...) ... | 1 | ifstmt.c:28:6:28:6 | x | | 1 | ifstmt.c:28:6:28:6 | x | 1 | ifstmt.c:28:11:28:11 | y | | 1 | ifstmt.c:28:6:28:11 | ... == ... | 1 | ifstmt.c:28:14:30:3 | { ... } | | 1 | ifstmt.c:28:6:28:11 | ... == ... | 4 | ifstmt.c:31:3:31:5 | label ...: | | 1 | ifstmt.c:28:11:28:11 | y | 1 | ifstmt.c:28:6:28:11 | ... == ... | | 1 | ifstmt.c:28:14:30:3 | { ... } | 2 | ifstmt.c:29:5:29:7 | label ...: | +| 1 | ifstmt.cpp:2:3:4:3 | if (...) ... | 1 | ifstmt.cpp:2:6:2:6 | declaration | +| 1 | ifstmt.cpp:2:6:2:6 | declaration | 1 | ifstmt.cpp:2:13:2:14 | initializer for z | +| 1 | ifstmt.cpp:2:13:2:14 | initializer for z | 1 | ifstmt.cpp:2:14:2:14 | y | +| 1 | ifstmt.cpp:2:14:2:14 | y | 1 | ifstmt.cpp:2:17:2:17 | x | +| 1 | ifstmt.cpp:2:17:2:17 | x | 1 | ifstmt.cpp:2:22:2:22 | z | +| 1 | ifstmt.cpp:2:17:2:22 | ... == ... | 1 | ifstmt.cpp:2:25:4:3 | { ... } | +| 1 | ifstmt.cpp:2:17:2:22 | ... == ... | 4 | ifstmt.cpp:5:3:5:5 | label ...: | +| 1 | ifstmt.cpp:2:22:2:22 | z | 1 | ifstmt.cpp:2:17:2:22 | ... == ... | +| 1 | ifstmt.cpp:2:25:4:3 | { ... } | 2 | ifstmt.cpp:3:5:3:7 | label ...: | | 2 | ifstmt.c:29:5:29:7 | label ...: | 2 | ifstmt.c:29:8:29:8 | ; | | 2 | ifstmt.c:29:8:29:8 | ; | 4 | ifstmt.c:31:3:31:5 | label ...: | +| 2 | ifstmt.cpp:3:5:3:7 | label ...: | 2 | ifstmt.cpp:3:8:3:8 | ; | +| 2 | ifstmt.cpp:3:8:3:8 | ; | 4 | ifstmt.cpp:5:3:5:5 | label ...: | | 4 | ifstmt.c:31:3:31:5 | label ...: | 4 | ifstmt.c:31:6:31:6 | ; | | 4 | ifstmt.c:31:6:31:6 | ; | 5 | ifstmt.c:32:1:32:1 | return ... | +| 4 | ifstmt.cpp:5:3:5:5 | label ...: | 4 | ifstmt.cpp:5:6:5:6 | ; | +| 4 | ifstmt.cpp:5:6:5:6 | ; | 5 | ifstmt.cpp:6:1:6:1 | return ... | | 5 | ifstmt.c:32:1:32:1 | return ... | 0 | ifstmt.c:27:6:27:11 | normal | +| 5 | ifstmt.cpp:6:1:6:1 | return ... | 0 | ifstmt.cpp:1:6:1:11 | normal | diff --git a/cpp/ql/test/successor-tests/ifstmt/ifstmt/ifstmt11.expected b/cpp/ql/test/successor-tests/ifstmt/ifstmt/ifstmt11.expected new file mode 100644 index 00000000000..e69de29bb2d diff --git a/cpp/ql/test/successor-tests/ifstmt/ifstmt/ifstmt11.ql b/cpp/ql/test/successor-tests/ifstmt/ifstmt/ifstmt11.ql new file mode 100644 index 00000000000..ad729b076e6 --- /dev/null +++ b/cpp/ql/test/successor-tests/ifstmt/ifstmt/ifstmt11.ql @@ -0,0 +1,15 @@ +/** + * @name ifstmt11 + * @description If an initialization exists, then the condition is a successor of the initialization. + */ + +import cpp + +from IfStmt is, Expr e, Stmt s, ControlFlowNode n +where + s = is.getInitialization() and + e = is.getCondition() and + n = s.getASuccessor*() and + not exists(ControlFlowNode m | m = e.getASuccessor*() | m = n) and + not exists(ControlFlowNode m | m = e.getAPredecessor*() | m = n) +select is From 71c019e126e8f61913759b5fb7cabda98eb63502 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Fri, 29 Apr 2022 16:18:25 +0200 Subject: [PATCH 84/91] C++: Handle C++17 switch initializers --- cpp/ql/lib/semmle/code/cpp/PrintAST.qll | 2 + .../code/cpp/controlflow/internal/CFG.qll | 19 +- .../raw/internal/TranslatedStmt.qll | 22 +- cpp/ql/lib/semmle/code/cpp/stmts/Stmt.qll | 24 ++- cpp/ql/lib/semmlecode.cpp.dbscheme | 5 + .../library-tests/ir/ir/PrintAST.expected | 199 ++++++++++++++++++ cpp/ql/test/library-tests/ir/ir/ir.cpp | 34 +++ .../ir/ir/operand_locations.expected | 110 ++++++++++ .../test/library-tests/ir/ir/raw_ir.expected | 136 ++++++++++++ .../switchstmt/switchstmt/cfg.expected | 17 ++ .../switchstmt/switchstmt/switchstmt.cpp | 12 ++ 11 files changed, 568 insertions(+), 12 deletions(-) create mode 100644 cpp/ql/test/successor-tests/switchstmt/switchstmt/switchstmt.cpp diff --git a/cpp/ql/lib/semmle/code/cpp/PrintAST.qll b/cpp/ql/lib/semmle/code/cpp/PrintAST.qll index c7366128125..106e49aff74 100644 --- a/cpp/ql/lib/semmle/code/cpp/PrintAST.qll +++ b/cpp/ql/lib/semmle/code/cpp/PrintAST.qll @@ -679,6 +679,8 @@ private predicate namedStmtChildPredicates(Locatable s, Element e, string pred) or s.(IfStmt).getElse() = e and pred = "getElse()" or + s.(SwitchStmt).getInitialization() = e and pred = "getInitialization()" + or s.(SwitchStmt).getExpr() = e and pred = "getExpr()" or s.(SwitchStmt).getStmt() = e and pred = "getStmt()" diff --git a/cpp/ql/lib/semmle/code/cpp/controlflow/internal/CFG.qll b/cpp/ql/lib/semmle/code/cpp/controlflow/internal/CFG.qll index 4210db40a7b..aa7d8b09f90 100644 --- a/cpp/ql/lib/semmle/code/cpp/controlflow/internal/CFG.qll +++ b/cpp/ql/lib/semmle/code/cpp/controlflow/internal/CFG.qll @@ -708,30 +708,33 @@ private predicate straightLineSparse(Node scope, int i, Node ni, Spec spec) { or scope = any(SwitchStmt s | + // SwitchStmt [-> init] -> expr i = -1 and ni = s and spec.isAt() or - i = 0 and ni = s.getExpr() and spec.isAround() + i = 0 and ni = s.getInitialization() and spec.isAround() + or + i = 1 and ni = s.getExpr() and spec.isAround() or // If the switch body is not a block then this step is skipped, and the // expression jumps directly to the cases. - i = 1 and ni = s.getStmt().(BlockStmt) and spec.isAt() + i = 2 and ni = s.getStmt().(BlockStmt) and spec.isAt() or - i = 2 and ni = s.getASwitchCase() and spec.isBefore() + i = 3 and ni = s.getASwitchCase() and spec.isBefore() or // If there is no default case, we can jump to after the block. Note: `i` // is same value as above. not s.getASwitchCase() instanceof DefaultCase and - i = 2 and + i = 3 and ni = s.getStmt() and spec.isAfter() or - i = 3 and /* BARRIER */ ni = s and spec.isBarrier() + i = 4 and /* BARRIER */ ni = s and spec.isBarrier() or - i = 4 and ni = s.getStmt() and spec.isAfter() + i = 5 and ni = s.getStmt() and spec.isAfter() or - i = 5 and ni = s and spec.isAroundDestructors() + i = 6 and ni = s and spec.isAroundDestructors() or - i = 6 and ni = s and spec.isAfter() + i = 7 and ni = s and spec.isAfter() ) or scope = diff --git a/cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/TranslatedStmt.qll b/cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/TranslatedStmt.qll index 45d9f924b10..187dbc2f994 100644 --- a/cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/TranslatedStmt.qll +++ b/cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/TranslatedStmt.qll @@ -717,14 +717,28 @@ class TranslatedSwitchStmt extends TranslatedStmt { result = getTranslatedExpr(stmt.getExpr().getFullyConverted()) } + private Instruction getFirstExprInstruction() { result = getExpr().getFirstInstruction() } + private TranslatedStmt getBody() { result = getTranslatedStmt(stmt.getStmt()) } - override Instruction getFirstInstruction() { result = getExpr().getFirstInstruction() } + override Instruction getFirstInstruction() { + if hasInitialization() + then result = getInitialization().getFirstInstruction() + else result = getFirstExprInstruction() + } override TranslatedElement getChild(int id) { - id = 0 and result = getExpr() + id = 0 and result = getInitialization() or - id = 1 and result = getBody() + id = 1 and result = getExpr() + or + id = 2 and result = getBody() + } + + private predicate hasInitialization() { exists(stmt.getInitialization()) } + + private TranslatedStmt getInitialization() { + result = getTranslatedStmt(stmt.getInitialization()) } override predicate hasInstruction(Opcode opcode, InstructionTag tag, CppType resultType) { @@ -754,6 +768,8 @@ class TranslatedSwitchStmt extends TranslatedStmt { } override Instruction getChildSuccessor(TranslatedElement child) { + child = getInitialization() and result = getFirstExprInstruction() + or child = getExpr() and result = getInstruction(SwitchBranchTag()) or child = getBody() and result = getParent().getChildSuccessor(this) diff --git a/cpp/ql/lib/semmle/code/cpp/stmts/Stmt.qll b/cpp/ql/lib/semmle/code/cpp/stmts/Stmt.qll index 084a4fb7bc7..9dda08ba206 100644 --- a/cpp/ql/lib/semmle/code/cpp/stmts/Stmt.qll +++ b/cpp/ql/lib/semmle/code/cpp/stmts/Stmt.qll @@ -1512,6 +1512,28 @@ class DefaultCase extends SwitchCase { class SwitchStmt extends ConditionalStmt, @stmt_switch { override string getAPrimaryQlClass() { result = "SwitchStmt" } + /** + * Gets the initialization statement of this 'switch' statement. + * + * For example, for + * ``` + * switch (x = y; b) { } + * ``` + * the result is `x = y;`. + * + * Does not hold if the initialization statement is missing or an empty statement, as in + * ``` + * switch (b) { } + * ``` + * or + * ``` + * switch (; b) { } + * ``` + */ + Stmt getInitialization() { + switch_initialization(underlyingElement(this), unresolveElement(result)) + } + /** * Gets the expression that this 'switch' statement switches on. * @@ -1527,7 +1549,7 @@ class SwitchStmt extends ConditionalStmt, @stmt_switch { * ``` * the result is `i`. */ - Expr getExpr() { result = this.getChild(0) } + Expr getExpr() { result = this.getChild(1) } override Expr getControllingExpr() { result = this.getExpr() } diff --git a/cpp/ql/lib/semmlecode.cpp.dbscheme b/cpp/ql/lib/semmlecode.cpp.dbscheme index 233020bf9ad..cf72c8898d1 100644 --- a/cpp/ql/lib/semmlecode.cpp.dbscheme +++ b/cpp/ql/lib/semmlecode.cpp.dbscheme @@ -1903,6 +1903,11 @@ do_body( int body_id: @stmt ref ); +switch_initialization( + unique int switch_stmt: @stmt_switch ref, + int init_id: @stmt ref +); + #keyset[switch_stmt, index] switch_case( int switch_stmt: @stmt_switch ref, diff --git a/cpp/ql/test/library-tests/ir/ir/PrintAST.expected b/cpp/ql/test/library-tests/ir/ir/PrintAST.expected index 71fa09447b1..9aedef96249 100644 --- a/cpp/ql/test/library-tests/ir/ir/PrintAST.expected +++ b/cpp/ql/test/library-tests/ir/ir/PrintAST.expected @@ -13776,6 +13776,205 @@ ir.cpp: # 1781| Type = [IntType] int # 1781| ValueCategory = prvalue(load) # 1783| getStmt(8): [ReturnStmt] return ... +# 1785| [TopLevelFunction] void switch_initialization(int) +# 1785| : +# 1785| getParameter(0): [Parameter] x +# 1785| Type = [IntType] int +# 1785| getEntryPoint(): [BlockStmt] { ... } +# 1786| getStmt(0): [SwitchStmt] switch (...) ... +# 1786| getInitialization(): [DeclStmt] declaration +# 1786| getDeclarationEntry(0): [VariableDeclarationEntry] definition of y +# 1786| Type = [IntType] int +# 1786| getVariable().getInitializer(): [Initializer] initializer for y +# 1786| getExpr(): [VariableAccess] x +# 1786| Type = [IntType] int +# 1786| ValueCategory = prvalue(load) +# 1786| getExpr(): [AddExpr] ... + ... +# 1786| Type = [IntType] int +# 1786| ValueCategory = prvalue +# 1786| getLeftOperand(): [VariableAccess] x +# 1786| Type = [IntType] int +# 1786| ValueCategory = prvalue(load) +# 1786| getRightOperand(): [Literal] 1 +# 1786| Type = [IntType] int +# 1786| Value = [Literal] 1 +# 1786| ValueCategory = prvalue +# 1786| getStmt(): [BlockStmt] { ... } +# 1787| getStmt(0): [SwitchCase] default: +# 1788| getStmt(1): [ExprStmt] ExprStmt +# 1788| getExpr(): [AssignExpr] ... = ... +# 1788| Type = [IntType] int +# 1788| ValueCategory = lvalue +# 1788| getLValue(): [VariableAccess] x +# 1788| Type = [IntType] int +# 1788| ValueCategory = lvalue +# 1788| getRValue(): [AddExpr] ... + ... +# 1788| Type = [IntType] int +# 1788| ValueCategory = prvalue +# 1788| getLeftOperand(): [VariableAccess] x +# 1788| Type = [IntType] int +# 1788| ValueCategory = prvalue(load) +# 1788| getRightOperand(): [VariableAccess] y +# 1788| Type = [IntType] int +# 1788| ValueCategory = prvalue(load) +# 1791| getStmt(1): [DeclStmt] declaration +# 1791| getDeclarationEntry(0): [VariableDeclarationEntry] definition of w +# 1791| Type = [IntType] int +# 1792| getStmt(2): [SwitchStmt] switch (...) ... +# 1792| getInitialization(): [ExprStmt] ExprStmt +# 1792| getExpr(): [AssignExpr] ... = ... +# 1792| Type = [IntType] int +# 1792| ValueCategory = lvalue +# 1792| getLValue(): [VariableAccess] w +# 1792| Type = [IntType] int +# 1792| ValueCategory = lvalue +# 1792| getRValue(): [VariableAccess] x +# 1792| Type = [IntType] int +# 1792| ValueCategory = prvalue(load) +# 1792| getExpr(): [AddExpr] ... + ... +# 1792| Type = [IntType] int +# 1792| ValueCategory = prvalue +# 1792| getLeftOperand(): [VariableAccess] x +# 1792| Type = [IntType] int +# 1792| ValueCategory = prvalue(load) +# 1792| getRightOperand(): [Literal] 1 +# 1792| Type = [IntType] int +# 1792| Value = [Literal] 1 +# 1792| ValueCategory = prvalue +# 1792| getStmt(): [BlockStmt] { ... } +# 1793| getStmt(0): [SwitchCase] default: +# 1794| getStmt(1): [ExprStmt] ExprStmt +# 1794| getExpr(): [AssignExpr] ... = ... +# 1794| Type = [IntType] int +# 1794| ValueCategory = lvalue +# 1794| getLValue(): [VariableAccess] x +# 1794| Type = [IntType] int +# 1794| ValueCategory = lvalue +# 1794| getRValue(): [AddExpr] ... + ... +# 1794| Type = [IntType] int +# 1794| ValueCategory = prvalue +# 1794| getLeftOperand(): [VariableAccess] x +# 1794| Type = [IntType] int +# 1794| ValueCategory = prvalue(load) +# 1794| getRightOperand(): [VariableAccess] w +# 1794| Type = [IntType] int +# 1794| ValueCategory = prvalue(load) +# 1797| getStmt(3): [SwitchStmt] switch (...) ... +# 1797| getInitialization(): [ExprStmt] ExprStmt +# 1797| getExpr(): [AssignExpr] ... = ... +# 1797| Type = [IntType] int +# 1797| ValueCategory = lvalue +# 1797| getLValue(): [VariableAccess] w +# 1797| Type = [IntType] int +# 1797| ValueCategory = lvalue +# 1797| getRValue(): [VariableAccess] x +# 1797| Type = [IntType] int +# 1797| ValueCategory = prvalue(load) +# 1797| getExpr(): [ConditionDeclExpr] (condition decl) +# 1797| Type = [IntType] int +# 1797| ValueCategory = prvalue +# 1797| getVariableAccess(): [VariableAccess] w2 +# 1797| Type = [IntType] int +# 1797| ValueCategory = prvalue(load) +# 1797| getStmt(): [BlockStmt] { ... } +# 1798| getStmt(0): [SwitchCase] default: +# 1799| getStmt(1): [ExprStmt] ExprStmt +# 1799| getExpr(): [AssignExpr] ... = ... +# 1799| Type = [IntType] int +# 1799| ValueCategory = lvalue +# 1799| getLValue(): [VariableAccess] x +# 1799| Type = [IntType] int +# 1799| ValueCategory = lvalue +# 1799| getRValue(): [AddExpr] ... + ... +# 1799| Type = [IntType] int +# 1799| ValueCategory = prvalue +# 1799| getLeftOperand(): [VariableAccess] x +# 1799| Type = [IntType] int +# 1799| ValueCategory = prvalue(load) +# 1799| getRightOperand(): [VariableAccess] w +# 1799| Type = [IntType] int +# 1799| ValueCategory = prvalue(load) +# 1802| getStmt(4): [SwitchStmt] switch (...) ... +# 1802| getInitialization(): [DeclStmt] declaration +# 1802| getDeclarationEntry(0): [VariableDeclarationEntry] definition of v +# 1802| Type = [IntType] int +# 1802| getVariable().getInitializer(): [Initializer] initializer for v +# 1802| getExpr(): [VariableAccess] x +# 1802| Type = [IntType] int +# 1802| ValueCategory = prvalue(load) +# 1802| getExpr(): [ConditionDeclExpr] (condition decl) +# 1802| Type = [IntType] int +# 1802| ValueCategory = prvalue +# 1802| getVariableAccess(): [VariableAccess] v2 +# 1802| Type = [IntType] int +# 1802| ValueCategory = prvalue(load) +# 1802| getStmt(): [BlockStmt] { ... } +# 1803| getStmt(0): [SwitchCase] default: +# 1804| getStmt(1): [ExprStmt] ExprStmt +# 1804| getExpr(): [AssignExpr] ... = ... +# 1804| Type = [IntType] int +# 1804| ValueCategory = lvalue +# 1804| getLValue(): [VariableAccess] x +# 1804| Type = [IntType] int +# 1804| ValueCategory = lvalue +# 1804| getRValue(): [AddExpr] ... + ... +# 1804| Type = [IntType] int +# 1804| ValueCategory = prvalue +# 1804| getLeftOperand(): [VariableAccess] x +# 1804| Type = [IntType] int +# 1804| ValueCategory = prvalue(load) +# 1804| getRightOperand(): [VariableAccess] v +# 1804| Type = [IntType] int +# 1804| ValueCategory = prvalue(load) +# 1807| getStmt(5): [DeclStmt] declaration +# 1807| getDeclarationEntry(0): [VariableDeclarationEntry] definition of z +# 1807| Type = [IntType] int +# 1807| getVariable().getInitializer(): [Initializer] initializer for z +# 1807| getExpr(): [VariableAccess] x +# 1807| Type = [IntType] int +# 1807| ValueCategory = prvalue(load) +# 1808| getStmt(6): [SwitchStmt] switch (...) ... +# 1808| getExpr(): [VariableAccess] z +# 1808| Type = [IntType] int +# 1808| ValueCategory = prvalue(load) +# 1808| getStmt(): [BlockStmt] { ... } +# 1809| getStmt(0): [SwitchCase] default: +# 1810| getStmt(1): [ExprStmt] ExprStmt +# 1810| getExpr(): [AssignExpr] ... = ... +# 1810| Type = [IntType] int +# 1810| ValueCategory = lvalue +# 1810| getLValue(): [VariableAccess] x +# 1810| Type = [IntType] int +# 1810| ValueCategory = lvalue +# 1810| getRValue(): [AddExpr] ... + ... +# 1810| Type = [IntType] int +# 1810| ValueCategory = prvalue +# 1810| getLeftOperand(): [VariableAccess] x +# 1810| Type = [IntType] int +# 1810| ValueCategory = prvalue(load) +# 1810| getRightOperand(): [VariableAccess] z +# 1810| Type = [IntType] int +# 1810| ValueCategory = prvalue(load) +# 1813| getStmt(7): [SwitchStmt] switch (...) ... +# 1813| getExpr(): [ConditionDeclExpr] (condition decl) +# 1813| Type = [IntType] int +# 1813| ValueCategory = prvalue +# 1813| getVariableAccess(): [VariableAccess] z2 +# 1813| Type = [IntType] int +# 1813| ValueCategory = prvalue(load) +# 1813| getStmt(): [BlockStmt] { ... } +# 1814| getStmt(0): [SwitchCase] default: +# 1815| getStmt(1): [ExprStmt] ExprStmt +# 1815| getExpr(): [AssignAddExpr] ... += ... +# 1815| Type = [IntType] int +# 1815| ValueCategory = lvalue +# 1815| getLValue(): [VariableAccess] x +# 1815| Type = [IntType] int +# 1815| ValueCategory = lvalue +# 1815| getRValue(): [VariableAccess] z2 +# 1815| Type = [IntType] int +# 1815| ValueCategory = prvalue(load) +# 1817| getStmt(8): [ReturnStmt] return ... perf-regression.cpp: # 4| [CopyAssignmentOperator] Big& Big::operator=(Big const&) # 4| : diff --git a/cpp/ql/test/library-tests/ir/ir/ir.cpp b/cpp/ql/test/library-tests/ir/ir/ir.cpp index cf7cb477585..e85c5f1b505 100644 --- a/cpp/ql/test/library-tests/ir/ir/ir.cpp +++ b/cpp/ql/test/library-tests/ir/ir/ir.cpp @@ -1782,4 +1782,38 @@ void if_initialization(int x) { } } +void switch_initialization(int x) { + switch (int y = x; x + 1) { + default: + x = x + y; + } + + int w; + switch (w = x; x + 1) { + default: + x = x + w; + } + + switch (w = x; int w2 = w) { + default: + x = x + w; + } + + switch (int v = x; int v2 = v) { + default: + x = x + v; + } + + int z = x; + switch (z) { + default: + x = x + z; + } + + switch (int z2 = z) { + default: + x += z2; + } +} + // semmle-extractor-options: -std=c++17 --clang diff --git a/cpp/ql/test/library-tests/ir/ir/operand_locations.expected b/cpp/ql/test/library-tests/ir/ir/operand_locations.expected index 378a2c77603..1581085efc6 100644 --- a/cpp/ql/test/library-tests/ir/ir/operand_locations.expected +++ b/cpp/ql/test/library-tests/ir/ir/operand_locations.expected @@ -8347,6 +8347,116 @@ | ir.cpp:1781:14:1781:15 | Address | &:r1781_1 | | ir.cpp:1781:14:1781:15 | Load | m1780_5 | | ir.cpp:1781:14:1781:15 | Right | r1781_2 | +| ir.cpp:1785:6:1785:26 | ChiPartial | partial:m1785_3 | +| ir.cpp:1785:6:1785:26 | ChiTotal | total:m1785_2 | +| ir.cpp:1785:6:1785:26 | SideEffect | m1785_3 | +| ir.cpp:1785:32:1785:32 | Address | &:r1785_5 | +| ir.cpp:1786:17:1786:17 | Address | &:r1786_1 | +| ir.cpp:1786:21:1786:21 | Address | &:r1786_2 | +| ir.cpp:1786:21:1786:21 | Load | m1785_6 | +| ir.cpp:1786:21:1786:21 | StoreValue | r1786_3 | +| ir.cpp:1786:24:1786:24 | Address | &:r1786_5 | +| ir.cpp:1786:24:1786:24 | Left | r1786_6 | +| ir.cpp:1786:24:1786:24 | Load | m1785_6 | +| ir.cpp:1786:24:1786:28 | Condition | r1786_8 | +| ir.cpp:1786:28:1786:28 | Right | r1786_7 | +| ir.cpp:1788:9:1788:9 | Address | &:r1788_6 | +| ir.cpp:1788:13:1788:13 | Address | &:r1788_1 | +| ir.cpp:1788:13:1788:13 | Left | r1788_2 | +| ir.cpp:1788:13:1788:13 | Load | m1785_6 | +| ir.cpp:1788:13:1788:17 | StoreValue | r1788_5 | +| ir.cpp:1788:17:1788:17 | Address | &:r1788_3 | +| ir.cpp:1788:17:1788:17 | Load | m1786_4 | +| ir.cpp:1788:17:1788:17 | Right | r1788_4 | +| ir.cpp:1791:9:1791:9 | Address | &:r1791_1 | +| ir.cpp:1792:13:1792:13 | Address | &:r1792_3 | +| ir.cpp:1792:17:1792:17 | Address | &:r1792_1 | +| ir.cpp:1792:17:1792:17 | Load | m1788_7 | +| ir.cpp:1792:17:1792:17 | StoreValue | r1792_2 | +| ir.cpp:1792:20:1792:20 | Address | &:r1792_5 | +| ir.cpp:1792:20:1792:20 | Left | r1792_6 | +| ir.cpp:1792:20:1792:20 | Load | m1788_7 | +| ir.cpp:1792:20:1792:24 | Condition | r1792_8 | +| ir.cpp:1792:24:1792:24 | Right | r1792_7 | +| ir.cpp:1794:9:1794:9 | Address | &:r1794_6 | +| ir.cpp:1794:13:1794:13 | Address | &:r1794_1 | +| ir.cpp:1794:13:1794:13 | Left | r1794_2 | +| ir.cpp:1794:13:1794:13 | Load | m1788_7 | +| ir.cpp:1794:13:1794:17 | StoreValue | r1794_5 | +| ir.cpp:1794:17:1794:17 | Address | &:r1794_3 | +| ir.cpp:1794:17:1794:17 | Load | m1792_4 | +| ir.cpp:1794:17:1794:17 | Right | r1794_4 | +| ir.cpp:1797:13:1797:13 | Address | &:r1797_3 | +| ir.cpp:1797:17:1797:17 | Address | &:r1797_1 | +| ir.cpp:1797:17:1797:17 | Load | m1794_7 | +| ir.cpp:1797:17:1797:17 | StoreValue | r1797_2 | +| ir.cpp:1797:18:1797:29 | Address | &:r1797_5 | +| ir.cpp:1797:18:1797:29 | Condition | r1797_11 | +| ir.cpp:1797:24:1797:25 | Address | &:r1797_9 | +| ir.cpp:1797:24:1797:25 | Load | m1797_8 | +| ir.cpp:1797:24:1797:25 | Unary | r1797_10 | +| ir.cpp:1797:29:1797:29 | Address | &:r1797_6 | +| ir.cpp:1797:29:1797:29 | Load | m1797_4 | +| ir.cpp:1797:29:1797:29 | StoreValue | r1797_7 | +| ir.cpp:1799:9:1799:9 | Address | &:r1799_6 | +| ir.cpp:1799:13:1799:13 | Address | &:r1799_1 | +| ir.cpp:1799:13:1799:13 | Left | r1799_2 | +| ir.cpp:1799:13:1799:13 | Load | m1794_7 | +| ir.cpp:1799:13:1799:17 | StoreValue | r1799_5 | +| ir.cpp:1799:17:1799:17 | Address | &:r1799_3 | +| ir.cpp:1799:17:1799:17 | Load | m1797_4 | +| ir.cpp:1799:17:1799:17 | Right | r1799_4 | +| ir.cpp:1802:13:1802:33 | Address | &:r1802_5 | +| ir.cpp:1802:13:1802:33 | Condition | r1802_11 | +| ir.cpp:1802:17:1802:17 | Address | &:r1802_1 | +| ir.cpp:1802:21:1802:21 | Address | &:r1802_2 | +| ir.cpp:1802:21:1802:21 | Load | m1799_7 | +| ir.cpp:1802:21:1802:21 | StoreValue | r1802_3 | +| ir.cpp:1802:28:1802:29 | Address | &:r1802_9 | +| ir.cpp:1802:28:1802:29 | Load | m1802_8 | +| ir.cpp:1802:28:1802:29 | Unary | r1802_10 | +| ir.cpp:1802:33:1802:33 | Address | &:r1802_6 | +| ir.cpp:1802:33:1802:33 | Load | m1802_4 | +| ir.cpp:1802:33:1802:33 | StoreValue | r1802_7 | +| ir.cpp:1804:9:1804:9 | Address | &:r1804_6 | +| ir.cpp:1804:13:1804:13 | Address | &:r1804_1 | +| ir.cpp:1804:13:1804:13 | Left | r1804_2 | +| ir.cpp:1804:13:1804:13 | Load | m1799_7 | +| ir.cpp:1804:13:1804:17 | StoreValue | r1804_5 | +| ir.cpp:1804:17:1804:17 | Address | &:r1804_3 | +| ir.cpp:1804:17:1804:17 | Load | m1802_4 | +| ir.cpp:1804:17:1804:17 | Right | r1804_4 | +| ir.cpp:1807:9:1807:9 | Address | &:r1807_1 | +| ir.cpp:1807:13:1807:13 | Address | &:r1807_2 | +| ir.cpp:1807:13:1807:13 | Load | m1804_7 | +| ir.cpp:1807:13:1807:13 | StoreValue | r1807_3 | +| ir.cpp:1808:13:1808:13 | Address | &:r1808_1 | +| ir.cpp:1808:13:1808:13 | Condition | r1808_2 | +| ir.cpp:1808:13:1808:13 | Load | m1807_4 | +| ir.cpp:1810:9:1810:9 | Address | &:r1810_6 | +| ir.cpp:1810:13:1810:13 | Address | &:r1810_1 | +| ir.cpp:1810:13:1810:13 | Left | r1810_2 | +| ir.cpp:1810:13:1810:13 | Load | m1804_7 | +| ir.cpp:1810:13:1810:17 | StoreValue | r1810_5 | +| ir.cpp:1810:17:1810:17 | Address | &:r1810_3 | +| ir.cpp:1810:17:1810:17 | Load | m1807_4 | +| ir.cpp:1810:17:1810:17 | Right | r1810_4 | +| ir.cpp:1813:13:1813:22 | Address | &:r1813_1 | +| ir.cpp:1813:13:1813:22 | Condition | r1813_7 | +| ir.cpp:1813:17:1813:18 | Address | &:r1813_5 | +| ir.cpp:1813:17:1813:18 | Load | m1813_4 | +| ir.cpp:1813:17:1813:18 | Unary | r1813_6 | +| ir.cpp:1813:22:1813:22 | Address | &:r1813_2 | +| ir.cpp:1813:22:1813:22 | Load | m1807_4 | +| ir.cpp:1813:22:1813:22 | StoreValue | r1813_3 | +| ir.cpp:1815:9:1815:9 | Address | &:r1815_3 | +| ir.cpp:1815:9:1815:9 | Address | &:r1815_3 | +| ir.cpp:1815:9:1815:9 | Left | r1815_4 | +| ir.cpp:1815:9:1815:9 | Load | m1810_7 | +| ir.cpp:1815:9:1815:15 | StoreValue | r1815_5 | +| ir.cpp:1815:14:1815:15 | Address | &:r1815_1 | +| ir.cpp:1815:14:1815:15 | Load | m1813_4 | +| ir.cpp:1815:14:1815:15 | Right | r1815_2 | | perf-regression.cpp:6:3:6:5 | Address | &:r6_5 | | perf-regression.cpp:6:3:6:5 | Address | &:r6_5 | | perf-regression.cpp:6:3:6:5 | Address | &:r6_7 | diff --git a/cpp/ql/test/library-tests/ir/ir/raw_ir.expected b/cpp/ql/test/library-tests/ir/ir/raw_ir.expected index 3c74321c413..17c59485eb9 100644 --- a/cpp/ql/test/library-tests/ir/ir/raw_ir.expected +++ b/cpp/ql/test/library-tests/ir/ir/raw_ir.expected @@ -9584,6 +9584,142 @@ ir.cpp: # 1757| v1757_7(void) = AliasedUse : ~m? # 1757| v1757_8(void) = ExitFunction : +# 1785| void switch_initialization(int) +# 1785| Block 0 +# 1785| v1785_1(void) = EnterFunction : +# 1785| mu1785_2(unknown) = AliasedDefinition : +# 1785| mu1785_3(unknown) = InitializeNonLocal : +# 1785| r1785_4(glval) = VariableAddress[x] : +# 1785| mu1785_5(int) = InitializeParameter[x] : &:r1785_4 +# 1786| r1786_1(glval) = VariableAddress[y] : +# 1786| r1786_2(glval) = VariableAddress[x] : +# 1786| r1786_3(int) = Load[x] : &:r1786_2, ~m? +# 1786| mu1786_4(int) = Store[y] : &:r1786_1, r1786_3 +# 1786| r1786_5(glval) = VariableAddress[x] : +# 1786| r1786_6(int) = Load[x] : &:r1786_5, ~m? +# 1786| r1786_7(int) = Constant[1] : +# 1786| r1786_8(int) = Add : r1786_6, r1786_7 +# 1786| v1786_9(void) = Switch : r1786_8 +#-----| Default -> Block 1 + +# 1787| Block 1 +# 1787| v1787_1(void) = NoOp : +# 1788| r1788_1(glval) = VariableAddress[x] : +# 1788| r1788_2(int) = Load[x] : &:r1788_1, ~m? +# 1788| r1788_3(glval) = VariableAddress[y] : +# 1788| r1788_4(int) = Load[y] : &:r1788_3, ~m? +# 1788| r1788_5(int) = Add : r1788_2, r1788_4 +# 1788| r1788_6(glval) = VariableAddress[x] : +# 1788| mu1788_7(int) = Store[x] : &:r1788_6, r1788_5 +# 1791| r1791_1(glval) = VariableAddress[w] : +# 1791| mu1791_2(int) = Uninitialized[w] : &:r1791_1 +# 1792| r1792_1(glval) = VariableAddress[x] : +# 1792| r1792_2(int) = Load[x] : &:r1792_1, ~m? +# 1792| r1792_3(glval) = VariableAddress[w] : +# 1792| mu1792_4(int) = Store[w] : &:r1792_3, r1792_2 +# 1792| r1792_5(glval) = VariableAddress[x] : +# 1792| r1792_6(int) = Load[x] : &:r1792_5, ~m? +# 1792| r1792_7(int) = Constant[1] : +# 1792| r1792_8(int) = Add : r1792_6, r1792_7 +# 1792| v1792_9(void) = Switch : r1792_8 +#-----| Default -> Block 2 + +# 1793| Block 2 +# 1793| v1793_1(void) = NoOp : +# 1794| r1794_1(glval) = VariableAddress[x] : +# 1794| r1794_2(int) = Load[x] : &:r1794_1, ~m? +# 1794| r1794_3(glval) = VariableAddress[w] : +# 1794| r1794_4(int) = Load[w] : &:r1794_3, ~m? +# 1794| r1794_5(int) = Add : r1794_2, r1794_4 +# 1794| r1794_6(glval) = VariableAddress[x] : +# 1794| mu1794_7(int) = Store[x] : &:r1794_6, r1794_5 +# 1797| r1797_1(glval) = VariableAddress[x] : +# 1797| r1797_2(int) = Load[x] : &:r1797_1, ~m? +# 1797| r1797_3(glval) = VariableAddress[w] : +# 1797| mu1797_4(int) = Store[w] : &:r1797_3, r1797_2 +# 1797| r1797_5(glval) = VariableAddress[w2] : +# 1797| r1797_6(glval) = VariableAddress[w] : +# 1797| r1797_7(int) = Load[w] : &:r1797_6, ~m? +# 1797| mu1797_8(int) = Store[w2] : &:r1797_5, r1797_7 +# 1797| r1797_9(glval) = VariableAddress[w2] : +# 1797| r1797_10(int) = Load[w2] : &:r1797_9, ~m? +# 1797| r1797_11(int) = CopyValue : r1797_10 +# 1797| v1797_12(void) = Switch : r1797_11 +#-----| Default -> Block 3 + +# 1798| Block 3 +# 1798| v1798_1(void) = NoOp : +# 1799| r1799_1(glval) = VariableAddress[x] : +# 1799| r1799_2(int) = Load[x] : &:r1799_1, ~m? +# 1799| r1799_3(glval) = VariableAddress[w] : +# 1799| r1799_4(int) = Load[w] : &:r1799_3, ~m? +# 1799| r1799_5(int) = Add : r1799_2, r1799_4 +# 1799| r1799_6(glval) = VariableAddress[x] : +# 1799| mu1799_7(int) = Store[x] : &:r1799_6, r1799_5 +# 1802| r1802_1(glval) = VariableAddress[v] : +# 1802| r1802_2(glval) = VariableAddress[x] : +# 1802| r1802_3(int) = Load[x] : &:r1802_2, ~m? +# 1802| mu1802_4(int) = Store[v] : &:r1802_1, r1802_3 +# 1802| r1802_5(glval) = VariableAddress[v2] : +# 1802| r1802_6(glval) = VariableAddress[v] : +# 1802| r1802_7(int) = Load[v] : &:r1802_6, ~m? +# 1802| mu1802_8(int) = Store[v2] : &:r1802_5, r1802_7 +# 1802| r1802_9(glval) = VariableAddress[v2] : +# 1802| r1802_10(int) = Load[v2] : &:r1802_9, ~m? +# 1802| r1802_11(int) = CopyValue : r1802_10 +# 1802| v1802_12(void) = Switch : r1802_11 +#-----| Default -> Block 4 + +# 1803| Block 4 +# 1803| v1803_1(void) = NoOp : +# 1804| r1804_1(glval) = VariableAddress[x] : +# 1804| r1804_2(int) = Load[x] : &:r1804_1, ~m? +# 1804| r1804_3(glval) = VariableAddress[v] : +# 1804| r1804_4(int) = Load[v] : &:r1804_3, ~m? +# 1804| r1804_5(int) = Add : r1804_2, r1804_4 +# 1804| r1804_6(glval) = VariableAddress[x] : +# 1804| mu1804_7(int) = Store[x] : &:r1804_6, r1804_5 +# 1807| r1807_1(glval) = VariableAddress[z] : +# 1807| r1807_2(glval) = VariableAddress[x] : +# 1807| r1807_3(int) = Load[x] : &:r1807_2, ~m? +# 1807| mu1807_4(int) = Store[z] : &:r1807_1, r1807_3 +# 1808| r1808_1(glval) = VariableAddress[z] : +# 1808| r1808_2(int) = Load[z] : &:r1808_1, ~m? +# 1808| v1808_3(void) = Switch : r1808_2 +#-----| Default -> Block 5 + +# 1809| Block 5 +# 1809| v1809_1(void) = NoOp : +# 1810| r1810_1(glval) = VariableAddress[x] : +# 1810| r1810_2(int) = Load[x] : &:r1810_1, ~m? +# 1810| r1810_3(glval) = VariableAddress[z] : +# 1810| r1810_4(int) = Load[z] : &:r1810_3, ~m? +# 1810| r1810_5(int) = Add : r1810_2, r1810_4 +# 1810| r1810_6(glval) = VariableAddress[x] : +# 1810| mu1810_7(int) = Store[x] : &:r1810_6, r1810_5 +# 1813| r1813_1(glval) = VariableAddress[z2] : +# 1813| r1813_2(glval) = VariableAddress[z] : +# 1813| r1813_3(int) = Load[z] : &:r1813_2, ~m? +# 1813| mu1813_4(int) = Store[z2] : &:r1813_1, r1813_3 +# 1813| r1813_5(glval) = VariableAddress[z2] : +# 1813| r1813_6(int) = Load[z2] : &:r1813_5, ~m? +# 1813| r1813_7(int) = CopyValue : r1813_6 +# 1813| v1813_8(void) = Switch : r1813_7 +#-----| Default -> Block 6 + +# 1814| Block 6 +# 1814| v1814_1(void) = NoOp : +# 1815| r1815_1(glval) = VariableAddress[z2] : +# 1815| r1815_2(int) = Load[z2] : &:r1815_1, ~m? +# 1815| r1815_3(glval) = VariableAddress[x] : +# 1815| r1815_4(int) = Load[x] : &:r1815_3, ~m? +# 1815| r1815_5(int) = Add : r1815_4, r1815_2 +# 1815| mu1815_6(int) = Store[x] : &:r1815_3, r1815_5 +# 1817| v1817_1(void) = NoOp : +# 1785| v1785_6(void) = ReturnVoid : +# 1785| v1785_7(void) = AliasedUse : ~m? +# 1785| v1785_8(void) = ExitFunction : + perf-regression.cpp: # 6| void Big::Big() # 6| Block 0 diff --git a/cpp/ql/test/successor-tests/switchstmt/switchstmt/cfg.expected b/cpp/ql/test/successor-tests/switchstmt/switchstmt/cfg.expected index ca3de0ffaff..607995cefd3 100644 --- a/cpp/ql/test/successor-tests/switchstmt/switchstmt/cfg.expected +++ b/cpp/ql/test/successor-tests/switchstmt/switchstmt/cfg.expected @@ -12,3 +12,20 @@ | switchstmt | switchstmt.c:1:6:1:6 | f | 7 | 8 | switchstmt.c:7:5:7:5 | switchstmt.c:7:5:7:5 | switchstmt.c:7:5:7:5 | ; | 8: return ... | | switchstmt | switchstmt.c:1:6:1:6 | f | 8 | 9 | switchstmt.c:8:1:8:1 | switchstmt.c:8:1:8:1 | switchstmt.c:8:1:8:1 | return ... | 8: f | | switchstmt | switchstmt.c:1:6:1:6 | f | 8 | 10 | switchstmt.c:1:6:1:6 | switchstmt.c:1:6:1:6 | switchstmt.c:1:6:1:6 | f | | +| switchstmt | switchstmt.cpp:3:6:3:6 | g | 3 | 1 | switchstmt.cpp:3:15:12:1 | switchstmt.cpp:3:15:12:1 | switchstmt.cpp:3:15:12:1 | { ... } | 4: switch (...) ... | +| switchstmt | switchstmt.cpp:3:6:3:6 | g | 4 | 2 | switchstmt.cpp:4:6:10:5 | switchstmt.cpp:4:6:10:5 | switchstmt.cpp:4:6:10:5 | switch (...) ... | 5: declaration | +| switchstmt | switchstmt.cpp:3:6:3:6 | g | 5 | 3 | switchstmt.cpp:5:10:5:10 | switchstmt.cpp:5:10:5:10 | switchstmt.cpp:5:10:5:10 | declaration | 5: initializer for y | +| switchstmt | switchstmt.cpp:3:6:3:6 | g | 5 | 4 | switchstmt.cpp:5:17:5:18 | switchstmt.cpp:5:17:5:18 | switchstmt.cpp:5:17:5:18 | initializer for y | 5: x | +| switchstmt | switchstmt.cpp:3:6:3:6 | g | 5 | 5 | switchstmt.cpp:5:18:5:18 | switchstmt.cpp:5:18:5:18 | switchstmt.cpp:5:18:5:18 | x | 6: y | +| switchstmt | switchstmt.cpp:3:6:3:6 | g | 6 | 6 | switchstmt.cpp:6:10:6:10 | switchstmt.cpp:6:10:6:10 | switchstmt.cpp:6:10:6:10 | y | 6: { ... } | +| switchstmt | switchstmt.cpp:3:6:3:6 | g | 6 | 7 | switchstmt.cpp:6:13:10:5 | switchstmt.cpp:6:13:10:5 | switchstmt.cpp:6:13:10:5 | { ... } | 7: case ...: | +| switchstmt | switchstmt.cpp:3:6:3:6 | g | 6 | 7 | switchstmt.cpp:6:13:10:5 | switchstmt.cpp:6:13:10:5 | switchstmt.cpp:6:13:10:5 | { ... } | 8: case ...: | +| switchstmt | switchstmt.cpp:3:6:3:6 | g | 6 | 7 | switchstmt.cpp:6:13:10:5 | switchstmt.cpp:6:13:10:5 | switchstmt.cpp:6:13:10:5 | { ... } | 9: default: | +| switchstmt | switchstmt.cpp:3:6:3:6 | g | 7 | 1 | switchstmt.cpp:7:14:7:14 | switchstmt.cpp:7:14:7:14 | switchstmt.cpp:7:14:7:14 | 1 | | +| switchstmt | switchstmt.cpp:3:6:3:6 | g | 7 | 8 | switchstmt.cpp:7:9:7:15 | switchstmt.cpp:7:9:7:15 | switchstmt.cpp:7:9:7:15 | case ...: | 8: case ...: | +| switchstmt | switchstmt.cpp:3:6:3:6 | g | 8 | 1 | switchstmt.cpp:8:14:8:14 | switchstmt.cpp:8:14:8:14 | switchstmt.cpp:8:14:8:14 | 2 | | +| switchstmt | switchstmt.cpp:3:6:3:6 | g | 8 | 9 | switchstmt.cpp:8:9:8:15 | switchstmt.cpp:8:9:8:15 | switchstmt.cpp:8:9:8:15 | case ...: | 9: default: | +| switchstmt | switchstmt.cpp:3:6:3:6 | g | 9 | 10 | switchstmt.cpp:9:9:9:16 | switchstmt.cpp:9:9:9:16 | switchstmt.cpp:9:9:9:16 | default: | 11: ; | +| switchstmt | switchstmt.cpp:3:6:3:6 | g | 11 | 11 | switchstmt.cpp:11:5:11:5 | switchstmt.cpp:11:5:11:5 | switchstmt.cpp:11:5:11:5 | ; | 12: return ... | +| switchstmt | switchstmt.cpp:3:6:3:6 | g | 12 | 12 | switchstmt.cpp:12:1:12:1 | switchstmt.cpp:12:1:12:1 | switchstmt.cpp:12:1:12:1 | return ... | 12: g | +| switchstmt | switchstmt.cpp:3:6:3:6 | g | 12 | 13 | switchstmt.cpp:3:6:3:6 | switchstmt.cpp:3:6:3:6 | switchstmt.cpp:3:6:3:6 | g | | diff --git a/cpp/ql/test/successor-tests/switchstmt/switchstmt/switchstmt.cpp b/cpp/ql/test/successor-tests/switchstmt/switchstmt/switchstmt.cpp new file mode 100644 index 00000000000..a88ad7e710c --- /dev/null +++ b/cpp/ql/test/successor-tests/switchstmt/switchstmt/switchstmt.cpp @@ -0,0 +1,12 @@ +// semmle-extractor-options: -std=c++17 + +void g(int x) { + switch ( + int y = x; + y) { + case 1: + case 2: + default: + } + ; +} From 97bba115da2152d3ddcc79233e0cc6d27693f7c7 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Fri, 29 Apr 2022 22:41:36 +0200 Subject: [PATCH 85/91] C++: Add upgrade and downgrade script --- .../exprparents.ql | 21 + .../old.dbscheme | 2111 +++++++++++++++++ .../semmlecode.cpp.dbscheme | 2096 ++++++++++++++++ .../stmtparents.ql | 22 + .../upgrade.properties | 6 + .../exprparents.ql | 21 + .../old.dbscheme | 2096 ++++++++++++++++ .../semmlecode.cpp.dbscheme | 2111 +++++++++++++++++ .../stmtparents.ql | 17 + .../upgrade.properties | 4 + 10 files changed, 8505 insertions(+) create mode 100644 cpp/downgrades/cf72c8898d19eb1b3374432cf79d8276cb07ad43/exprparents.ql create mode 100644 cpp/downgrades/cf72c8898d19eb1b3374432cf79d8276cb07ad43/old.dbscheme create mode 100644 cpp/downgrades/cf72c8898d19eb1b3374432cf79d8276cb07ad43/semmlecode.cpp.dbscheme create mode 100644 cpp/downgrades/cf72c8898d19eb1b3374432cf79d8276cb07ad43/stmtparents.ql create mode 100644 cpp/downgrades/cf72c8898d19eb1b3374432cf79d8276cb07ad43/upgrade.properties create mode 100644 cpp/ql/lib/upgrades/e9a518baf14f4322ac243578a8e1391386ff030f/exprparents.ql create mode 100644 cpp/ql/lib/upgrades/e9a518baf14f4322ac243578a8e1391386ff030f/old.dbscheme create mode 100644 cpp/ql/lib/upgrades/e9a518baf14f4322ac243578a8e1391386ff030f/semmlecode.cpp.dbscheme create mode 100644 cpp/ql/lib/upgrades/e9a518baf14f4322ac243578a8e1391386ff030f/stmtparents.ql create mode 100644 cpp/ql/lib/upgrades/e9a518baf14f4322ac243578a8e1391386ff030f/upgrade.properties diff --git a/cpp/downgrades/cf72c8898d19eb1b3374432cf79d8276cb07ad43/exprparents.ql b/cpp/downgrades/cf72c8898d19eb1b3374432cf79d8276cb07ad43/exprparents.ql new file mode 100644 index 00000000000..baddd72c144 --- /dev/null +++ b/cpp/downgrades/cf72c8898d19eb1b3374432cf79d8276cb07ad43/exprparents.ql @@ -0,0 +1,21 @@ +class Element extends @element { + string toString() { none() } +} + +class Expr extends @expr { + string toString() { none() } +} + +class Stmt extends @stmt { + string toString() { none() } +} + +predicate isStmtWithInitializer(Stmt stmt) { + exists(int kind | stmts(stmt, kind, _) | kind = 2 or kind = 11 or kind = 35) +} + +from Expr child, int index, int index_new, Element parent +where + exprparents(child, index, parent) and + if isStmtWithInitializer(parent) then index_new = index - 1 else index_new = index +select child, index_new, parent diff --git a/cpp/downgrades/cf72c8898d19eb1b3374432cf79d8276cb07ad43/old.dbscheme b/cpp/downgrades/cf72c8898d19eb1b3374432cf79d8276cb07ad43/old.dbscheme new file mode 100644 index 00000000000..cf72c8898d1 --- /dev/null +++ b/cpp/downgrades/cf72c8898d19eb1b3374432cf79d8276cb07ad43/old.dbscheme @@ -0,0 +1,2111 @@ + +/** + * An invocation of the compiler. Note that more than one file may be + * compiled per invocation. For example, this command compiles three + * source files: + * + * gcc -c f1.c f2.c f3.c + * + * The `id` simply identifies the invocation, while `cwd` is the working + * directory from which the compiler was invoked. + */ +compilations( + /** + * An invocation of the compiler. Note that more than one file may + * be compiled per invocation. For example, this command compiles + * three source files: + * + * gcc -c f1.c f2.c f3.c + */ + unique int id : @compilation, + string cwd : string ref +); + +/** + * The arguments that were passed to the extractor for a compiler + * invocation. If `id` is for the compiler invocation + * + * gcc -c f1.c f2.c f3.c + * + * then typically there will be rows for + * + * num | arg + * --- | --- + * 0 | *path to extractor* + * 1 | `--mimic` + * 2 | `/usr/bin/gcc` + * 3 | `-c` + * 4 | f1.c + * 5 | f2.c + * 6 | f3.c + */ +#keyset[id, num] +compilation_args( + int id : @compilation ref, + int num : int ref, + string arg : string ref +); + +/** + * The source files that are compiled by a compiler invocation. + * If `id` is for the compiler invocation + * + * gcc -c f1.c f2.c f3.c + * + * then there will be rows for + * + * num | arg + * --- | --- + * 0 | f1.c + * 1 | f2.c + * 2 | f3.c + * + * Note that even if those files `#include` headers, those headers + * do not appear as rows. + */ +#keyset[id, num] +compilation_compiling_files( + int id : @compilation ref, + int num : int ref, + int file : @file ref +); + +/** + * The time taken by the extractor for a compiler invocation. + * + * For each file `num`, there will be rows for + * + * kind | seconds + * ---- | --- + * 1 | CPU seconds used by the extractor frontend + * 2 | Elapsed seconds during the extractor frontend + * 3 | CPU seconds used by the extractor backend + * 4 | Elapsed seconds during the extractor backend + */ +#keyset[id, num, kind] +compilation_time( + int id : @compilation ref, + int num : int ref, + /* kind: + 1 = frontend_cpu_seconds + 2 = frontend_elapsed_seconds + 3 = extractor_cpu_seconds + 4 = extractor_elapsed_seconds + */ + int kind : int ref, + float seconds : float ref +); + +/** + * An error or warning generated by the extractor. + * The diagnostic message `diagnostic` was generated during compiler + * invocation `compilation`, and is the `file_number_diagnostic_number`th + * message generated while extracting the `file_number`th file of that + * invocation. + */ +#keyset[compilation, file_number, file_number_diagnostic_number] +diagnostic_for( + int diagnostic : @diagnostic ref, + int compilation : @compilation ref, + int file_number : int ref, + int file_number_diagnostic_number : int ref +); + +/** + * If extraction was successful, then `cpu_seconds` and + * `elapsed_seconds` are the CPU time and elapsed time (respectively) + * that extraction took for compiler invocation `id`. + */ +compilation_finished( + unique int id : @compilation ref, + float cpu_seconds : float ref, + float elapsed_seconds : float ref +); + + +/** + * External data, loaded from CSV files during snapshot creation. See + * [Tutorial: Incorporating external data](https://help.semmle.com/wiki/display/SD/Tutorial%3A+Incorporating+external+data) + * for more information. + */ +externalData( + int id : @externalDataElement, + string path : string ref, + int column: int ref, + string value : string ref +); + +/** + * The source location of the snapshot. + */ +sourceLocationPrefix(string prefix : string ref); + +/** + * Information about packages that provide code used during compilation. + * The `id` is just a unique identifier. + * The `namespace` is typically the name of the package manager that + * provided the package (e.g. "dpkg" or "yum"). + * The `package_name` is the name of the package, and `version` is its + * version (as a string). + */ +external_packages( + unique int id: @external_package, + string namespace : string ref, + string package_name : string ref, + string version : string ref +); + +/** + * Holds if File `fileid` was provided by package `package`. + */ +header_to_external_package( + int fileid : @file ref, + int package : @external_package ref +); + +/* + * Version history + */ + +svnentries( + unique int id : @svnentry, + string revision : string ref, + string author : string ref, + date revisionDate : date ref, + int changeSize : int ref +) + +svnaffectedfiles( + int id : @svnentry ref, + int file : @file ref, + string action : string ref +) + +svnentrymsg( + unique int id : @svnentry ref, + string message : string ref +) + +svnchurn( + int commit : @svnentry ref, + int file : @file ref, + int addedLines : int ref, + int deletedLines : int ref +) + +/* + * C++ dbscheme + */ + +@location = @location_stmt | @location_expr | @location_default ; + +/** + * The location of an element that is not an expression or a statement. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `file`. + * For more information, see + * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/). + */ +locations_default( + /** The location of an element that is not an expression or a statement. */ + unique int id: @location_default, + int container: @container ref, + int startLine: int ref, + int startColumn: int ref, + int endLine: int ref, + int endColumn: int ref +); + +/** + * The location of a statement. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `file`. + * For more information, see + * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/). + */ +locations_stmt( + /** The location of a statement. */ + unique int id: @location_stmt, + int container: @container ref, + int startLine: int ref, + int startColumn: int ref, + int endLine: int ref, + int endColumn: int ref +); + +/** + * The location of an expression. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `file`. + * For more information, see + * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/). + */ +locations_expr( + /** The location of an expression. */ + unique int id: @location_expr, + int container: @container ref, + int startLine: int ref, + int startColumn: int ref, + int endLine: int ref, + int endColumn: int ref +); + +/** An element for which line-count information is available. */ +@sourceline = @file | @function | @variable | @enumconstant | @xmllocatable; + +numlines( + int element_id: @sourceline ref, + int num_lines: int ref, + int num_code: int ref, + int num_comment: int ref +); + +diagnostics( + unique int id: @diagnostic, + int severity: int ref, + string error_tag: string ref, + string error_message: string ref, + string full_error_message: string ref, + int location: @location_default ref +); + +files( + unique int id: @file, + string name: string ref +); + +folders( + unique int id: @folder, + string name: string ref +); + +@container = @folder | @file + +containerparent( + int parent: @container ref, + unique int child: @container ref +); + +fileannotations( + int id: @file ref, + int kind: int ref, + string name: string ref, + string value: string ref +); + +inmacroexpansion( + int id: @element ref, + int inv: @macroinvocation ref +); + +affectedbymacroexpansion( + int id: @element ref, + int inv: @macroinvocation ref +); + +/* + case @macroinvocations.kind of + 1 = macro expansion + | 2 = other macro reference + ; +*/ +macroinvocations( + unique int id: @macroinvocation, + int macro_id: @ppd_define ref, + int location: @location_default ref, + int kind: int ref +); + +macroparent( + unique int id: @macroinvocation ref, + int parent_id: @macroinvocation ref +); + +// a macroinvocation may be part of another location +// the way to find a constant expression that uses a macro +// is thus to find a constant expression that has a location +// to which a macro invocation is bound +macrolocationbind( + int id: @macroinvocation ref, + int location: @location ref +); + +#keyset[invocation, argument_index] +macro_argument_unexpanded( + int invocation: @macroinvocation ref, + int argument_index: int ref, + string text: string ref +); + +#keyset[invocation, argument_index] +macro_argument_expanded( + int invocation: @macroinvocation ref, + int argument_index: int ref, + string text: string ref +); + +/* + case @function.kind of + 1 = normal + | 2 = constructor + | 3 = destructor + | 4 = conversion + | 5 = operator + | 6 = builtin // GCC built-in functions, e.g. __builtin___memcpy_chk + ; +*/ +functions( + unique int id: @function, + string name: string ref, + int kind: int ref +); + +function_entry_point(int id: @function ref, unique int entry_point: @stmt ref); + +function_return_type(int id: @function ref, int return_type: @type ref); + +/** If `function` is a coroutine, then this gives the + std::experimental::resumable_traits instance associated with it, + and the variables representing the `handle` and `promise` for it. */ +coroutine( + unique int function: @function ref, + int traits: @type ref, + int handle: @variable ref, + int promise: @variable ref +); + +/** The `new` function used for allocating the coroutine state, if any. */ +coroutine_new( + unique int function: @function ref, + int new: @function ref +); + +/** The `delete` function used for deallocating the coroutine state, if any. */ +coroutine_delete( + unique int function: @function ref, + int delete: @function ref +); + +purefunctions(unique int id: @function ref); + +function_deleted(unique int id: @function ref); + +function_defaulted(unique int id: @function ref); + +member_function_this_type(unique int id: @function ref, int this_type: @type ref); + +#keyset[id, type_id] +fun_decls( + int id: @fun_decl, + int function: @function ref, + int type_id: @type ref, + string name: string ref, + int location: @location_default ref +); +fun_def(unique int id: @fun_decl ref); +fun_specialized(unique int id: @fun_decl ref); +fun_implicit(unique int id: @fun_decl ref); +fun_decl_specifiers( + int id: @fun_decl ref, + string name: string ref +) +#keyset[fun_decl, index] +fun_decl_throws( + int fun_decl: @fun_decl ref, + int index: int ref, + int type_id: @type ref +); +/* an empty throw specification is different from none */ +fun_decl_empty_throws(unique int fun_decl: @fun_decl ref); +fun_decl_noexcept( + int fun_decl: @fun_decl ref, + int constant: @expr ref +); +fun_decl_empty_noexcept(int fun_decl: @fun_decl ref); +fun_decl_typedef_type( + unique int fun_decl: @fun_decl ref, + int typedeftype_id: @usertype ref +); + +param_decl_bind( + unique int id: @var_decl ref, + int index: int ref, + int fun_decl: @fun_decl ref +); + +#keyset[id, type_id] +var_decls( + int id: @var_decl, + int variable: @variable ref, + int type_id: @type ref, + string name: string ref, + int location: @location_default ref +); +var_def(unique int id: @var_decl ref); +var_decl_specifiers( + int id: @var_decl ref, + string name: string ref +) +is_structured_binding(unique int id: @variable ref); + +type_decls( + unique int id: @type_decl, + int type_id: @type ref, + int location: @location_default ref +); +type_def(unique int id: @type_decl ref); +type_decl_top( + unique int type_decl: @type_decl ref +); + +namespace_decls( + unique int id: @namespace_decl, + int namespace_id: @namespace ref, + int location: @location_default ref, + int bodylocation: @location_default ref +); + +usings( + unique int id: @using, + int element_id: @element ref, + int location: @location_default ref +); + +/** The element which contains the `using` declaration. */ +using_container( + int parent: @element ref, + int child: @using ref +); + +static_asserts( + unique int id: @static_assert, + int condition : @expr ref, + string message : string ref, + int location: @location_default ref, + int enclosing : @element ref +); + +// each function has an ordered list of parameters +#keyset[id, type_id] +#keyset[function, index, type_id] +params( + int id: @parameter, + int function: @functionorblock ref, + int index: int ref, + int type_id: @type ref +); + +overrides(int new: @function ref, int old: @function ref); + +#keyset[id, type_id] +membervariables( + int id: @membervariable, + int type_id: @type ref, + string name: string ref +); + +#keyset[id, type_id] +globalvariables( + int id: @globalvariable, + int type_id: @type ref, + string name: string ref +); + +#keyset[id, type_id] +localvariables( + int id: @localvariable, + int type_id: @type ref, + string name: string ref +); + +autoderivation( + unique int var: @variable ref, + int derivation_type: @type ref +); + +enumconstants( + unique int id: @enumconstant, + int parent: @usertype ref, + int index: int ref, + int type_id: @type ref, + string name: string ref, + int location: @location_default ref +); + +@variable = @localscopevariable | @globalvariable | @membervariable; + +@localscopevariable = @localvariable | @parameter; + +/* + Built-in types are the fundamental types, e.g., integral, floating, and void. + + case @builtintype.kind of + 1 = error + | 2 = unknown + | 3 = void + | 4 = boolean + | 5 = char + | 6 = unsigned_char + | 7 = signed_char + | 8 = short + | 9 = unsigned_short + | 10 = signed_short + | 11 = int + | 12 = unsigned_int + | 13 = signed_int + | 14 = long + | 15 = unsigned_long + | 16 = signed_long + | 17 = long_long + | 18 = unsigned_long_long + | 19 = signed_long_long + | 20 = __int8 // Microsoft-specific + | 21 = __int16 // Microsoft-specific + | 22 = __int32 // Microsoft-specific + | 23 = __int64 // Microsoft-specific + | 24 = float + | 25 = double + | 26 = long_double + | 27 = _Complex_float // C99-specific + | 28 = _Complex_double // C99-specific + | 29 = _Complex_long double // C99-specific + | 30 = _Imaginary_float // C99-specific + | 31 = _Imaginary_double // C99-specific + | 32 = _Imaginary_long_double // C99-specific + | 33 = wchar_t // Microsoft-specific + | 34 = decltype_nullptr // C++11 + | 35 = __int128 + | 36 = unsigned___int128 + | 37 = signed___int128 + | 38 = __float128 + | 39 = _Complex___float128 + | 40 = _Decimal32 + | 41 = _Decimal64 + | 42 = _Decimal128 + | 43 = char16_t + | 44 = char32_t + | 45 = _Float32 + | 46 = _Float32x + | 47 = _Float64 + | 48 = _Float64x + | 49 = _Float128 + | 50 = _Float128x + | 51 = char8_t + ; +*/ +builtintypes( + unique int id: @builtintype, + string name: string ref, + int kind: int ref, + int size: int ref, + int sign: int ref, + int alignment: int ref +); + +/* + Derived types are types that are directly derived from existing types and + point to, refer to, transform type data to return a new type. + + case @derivedtype.kind of + 1 = pointer + | 2 = reference + | 3 = type_with_specifiers + | 4 = array + | 5 = gnu_vector + | 6 = routineptr + | 7 = routinereference + | 8 = rvalue_reference // C++11 +// ... 9 type_conforming_to_protocols deprecated + | 10 = block + ; +*/ +derivedtypes( + unique int id: @derivedtype, + string name: string ref, + int kind: int ref, + int type_id: @type ref +); + +pointerishsize(unique int id: @derivedtype ref, + int size: int ref, + int alignment: int ref); + +arraysizes( + unique int id: @derivedtype ref, + int num_elements: int ref, + int bytesize: int ref, + int alignment: int ref +); + +typedefbase( + unique int id: @usertype ref, + int type_id: @type ref +); + +/** + * An instance of the C++11 `decltype` operator. For example: + * ``` + * int a; + * decltype(1+a) b; + * ``` + * Here `expr` is `1+a`. + * + * Sometimes an additional pair of parentheses around the expression + * would change the semantics of this decltype, e.g. + * ``` + * struct A { double x; }; + * const A* a = new A(); + * decltype( a->x ); // type is double + * decltype((a->x)); // type is const double& + * ``` + * (Please consult the C++11 standard for more details). + * `parentheses_would_change_meaning` is `true` iff that is the case. + */ +#keyset[id, expr] +decltypes( + int id: @decltype, + int expr: @expr ref, + int base_type: @type ref, + boolean parentheses_would_change_meaning: boolean ref +); + +/* + case @usertype.kind of + 1 = struct + | 2 = class + | 3 = union + | 4 = enum + | 5 = typedef // classic C: typedef typedef type name + | 6 = template + | 7 = template_parameter + | 8 = template_template_parameter + | 9 = proxy_class // a proxy class associated with a template parameter +// ... 10 objc_class deprecated +// ... 11 objc_protocol deprecated +// ... 12 objc_category deprecated + | 13 = scoped_enum + | 14 = using_alias // a using name = type style typedef + ; +*/ +usertypes( + unique int id: @usertype, + string name: string ref, + int kind: int ref +); + +usertypesize( + unique int id: @usertype ref, + int size: int ref, + int alignment: int ref +); + +usertype_final(unique int id: @usertype ref); + +usertype_uuid( + unique int id: @usertype ref, + string uuid: string ref +); + +mangled_name( + unique int id: @declaration ref, + int mangled_name : @mangledname +); + +is_pod_class(unique int id: @usertype ref); +is_standard_layout_class(unique int id: @usertype ref); + +is_complete(unique int id: @usertype ref); + +is_class_template(unique int id: @usertype ref); +class_instantiation( + int to: @usertype ref, + int from: @usertype ref +); +class_template_argument( + int type_id: @usertype ref, + int index: int ref, + int arg_type: @type ref +); +class_template_argument_value( + int type_id: @usertype ref, + int index: int ref, + int arg_value: @expr ref +); + +is_proxy_class_for( + unique int id: @usertype ref, + unique int templ_param_id: @usertype ref +); + +type_mentions( + unique int id: @type_mention, + int type_id: @type ref, + int location: @location ref, + // a_symbol_reference_kind from the EDG frontend. See symbol_ref.h there. + int kind: int ref +); + +is_function_template(unique int id: @function ref); +function_instantiation( + unique int to: @function ref, + int from: @function ref +); +function_template_argument( + int function_id: @function ref, + int index: int ref, + int arg_type: @type ref +); +function_template_argument_value( + int function_id: @function ref, + int index: int ref, + int arg_value: @expr ref +); + +is_variable_template(unique int id: @variable ref); +variable_instantiation( + unique int to: @variable ref, + int from: @variable ref +); +variable_template_argument( + int variable_id: @variable ref, + int index: int ref, + int arg_type: @type ref +); +variable_template_argument_value( + int variable_id: @variable ref, + int index: int ref, + int arg_value: @expr ref +); + +/* + Fixed point types + precision(1) = short, precision(2) = default, precision(3) = long + is_unsigned(1) = unsigned is_unsigned(2) = signed + is_fract_type(1) = declared with _Fract + saturating(1) = declared with _Sat +*/ +/* TODO +fixedpointtypes( + unique int id: @fixedpointtype, + int precision: int ref, + int is_unsigned: int ref, + int is_fract_type: int ref, + int saturating: int ref); +*/ + +routinetypes( + unique int id: @routinetype, + int return_type: @type ref +); + +routinetypeargs( + int routine: @routinetype ref, + int index: int ref, + int type_id: @type ref +); + +ptrtomembers( + unique int id: @ptrtomember, + int type_id: @type ref, + int class_id: @type ref +); + +/* + specifiers for types, functions, and variables + + "public", + "protected", + "private", + + "const", + "volatile", + "static", + + "pure", + "virtual", + "sealed", // Microsoft + "__interface", // Microsoft + "inline", + "explicit", + + "near", // near far extension + "far", // near far extension + "__ptr32", // Microsoft + "__ptr64", // Microsoft + "__sptr", // Microsoft + "__uptr", // Microsoft + "dllimport", // Microsoft + "dllexport", // Microsoft + "thread", // Microsoft + "naked", // Microsoft + "microsoft_inline", // Microsoft + "forceinline", // Microsoft + "selectany", // Microsoft + "nothrow", // Microsoft + "novtable", // Microsoft + "noreturn", // Microsoft + "noinline", // Microsoft + "noalias", // Microsoft + "restrict", // Microsoft +*/ + +specifiers( + unique int id: @specifier, + unique string str: string ref +); + +typespecifiers( + int type_id: @type ref, + int spec_id: @specifier ref +); + +funspecifiers( + int func_id: @function ref, + int spec_id: @specifier ref +); + +varspecifiers( + int var_id: @accessible ref, + int spec_id: @specifier ref +); + +attributes( + unique int id: @attribute, + int kind: int ref, + string name: string ref, + string name_space: string ref, + int location: @location_default ref +); + +case @attribute.kind of + 0 = @gnuattribute +| 1 = @stdattribute +| 2 = @declspec +| 3 = @msattribute +| 4 = @alignas +// ... 5 @objc_propertyattribute deprecated +; + +attribute_args( + unique int id: @attribute_arg, + int kind: int ref, + int attribute: @attribute ref, + int index: int ref, + int location: @location_default ref +); + +case @attribute_arg.kind of + 0 = @attribute_arg_empty +| 1 = @attribute_arg_token +| 2 = @attribute_arg_constant +| 3 = @attribute_arg_type +; + +attribute_arg_value( + unique int arg: @attribute_arg ref, + string value: string ref +); +attribute_arg_type( + unique int arg: @attribute_arg ref, + int type_id: @type ref +); +attribute_arg_name( + unique int arg: @attribute_arg ref, + string name: string ref +); + +typeattributes( + int type_id: @type ref, + int spec_id: @attribute ref +); + +funcattributes( + int func_id: @function ref, + int spec_id: @attribute ref +); + +varattributes( + int var_id: @accessible ref, + int spec_id: @attribute ref +); + +stmtattributes( + int stmt_id: @stmt ref, + int spec_id: @attribute ref +); + +@type = @builtintype + | @derivedtype + | @usertype + /* TODO | @fixedpointtype */ + | @routinetype + | @ptrtomember + | @decltype; + +unspecifiedtype( + unique int type_id: @type ref, + int unspecified_type_id: @type ref +); + +member( + int parent: @type ref, + int index: int ref, + int child: @member ref +); + +@enclosingfunction_child = @usertype | @variable | @namespace + +enclosingfunction( + unique int child: @enclosingfunction_child ref, + int parent: @function ref +); + +derivations( + unique int derivation: @derivation, + int sub: @type ref, + int index: int ref, + int super: @type ref, + int location: @location_default ref +); + +derspecifiers( + int der_id: @derivation ref, + int spec_id: @specifier ref +); + +/** + * Contains the byte offset of the base class subobject within the derived + * class. Only holds for non-virtual base classes, but see table + * `virtual_base_offsets` for offsets of virtual base class subobjects. + */ +direct_base_offsets( + unique int der_id: @derivation ref, + int offset: int ref +); + +/** + * Contains the byte offset of the virtual base class subobject for class + * `super` within a most-derived object of class `sub`. `super` can be either a + * direct or indirect base class. + */ +#keyset[sub, super] +virtual_base_offsets( + int sub: @usertype ref, + int super: @usertype ref, + int offset: int ref +); + +frienddecls( + unique int id: @frienddecl, + int type_id: @type ref, + int decl_id: @declaration ref, + int location: @location_default ref +); + +@declaredtype = @usertype ; + +@declaration = @function + | @declaredtype + | @variable + | @enumconstant + | @frienddecl; + +@member = @membervariable + | @function + | @declaredtype + | @enumconstant; + +@locatable = @diagnostic + | @declaration + | @ppd_include + | @ppd_define + | @macroinvocation + /*| @funcall*/ + | @xmllocatable + | @attribute + | @attribute_arg; + +@namedscope = @namespace | @usertype; + +@element = @locatable + | @file + | @folder + | @specifier + | @type + | @expr + | @namespace + | @initialiser + | @stmt + | @derivation + | @comment + | @preprocdirect + | @fun_decl + | @var_decl + | @type_decl + | @namespace_decl + | @using + | @namequalifier + | @specialnamequalifyingelement + | @static_assert + | @type_mention + | @lambdacapture; + +@exprparent = @element; + +comments( + unique int id: @comment, + string contents: string ref, + int location: @location_default ref +); + +commentbinding( + int id: @comment ref, + int element: @element ref +); + +exprconv( + int converted: @expr ref, + unique int conversion: @expr ref +); + +compgenerated(unique int id: @element ref); + +/** + * `destructor_call` destructs the `i`'th entity that should be + * destructed following `element`. Note that entities should be + * destructed in reverse construction order, so for a given `element` + * these should be called from highest to lowest `i`. + */ +#keyset[element, destructor_call] +#keyset[element, i] +synthetic_destructor_call( + int element: @element ref, + int i: int ref, + int destructor_call: @routineexpr ref +); + +namespaces( + unique int id: @namespace, + string name: string ref +); + +namespace_inline( + unique int id: @namespace ref +); + +namespacembrs( + int parentid: @namespace ref, + unique int memberid: @namespacembr ref +); + +@namespacembr = @declaration | @namespace; + +exprparents( + int expr_id: @expr ref, + int child_index: int ref, + int parent_id: @exprparent ref +); + +expr_isload(unique int expr_id: @expr ref); + +@cast = @c_style_cast + | @const_cast + | @dynamic_cast + | @reinterpret_cast + | @static_cast + ; + +/* +case @conversion.kind of + 0 = @simple_conversion // a numeric conversion, qualification conversion, or a reinterpret_cast +| 1 = @bool_conversion // conversion to 'bool' +| 2 = @base_class_conversion // a derived-to-base conversion +| 3 = @derived_class_conversion // a base-to-derived conversion +| 4 = @pm_base_class_conversion // a derived-to-base conversion of a pointer to member +| 5 = @pm_derived_class_conversion // a base-to-derived conversion of a pointer to member +| 6 = @glvalue_adjust // an adjustment of the type of a glvalue +| 7 = @prvalue_adjust // an adjustment of the type of a prvalue +; +*/ +/** + * Describes the semantics represented by a cast expression. This is largely + * independent of the source syntax of the cast, so it is separate from the + * regular expression kind. + */ +conversionkinds( + unique int expr_id: @cast ref, + int kind: int ref +); + +@conversion = @cast + | @array_to_pointer + | @parexpr + | @reference_to + | @ref_indirect + | @temp_init + ; + +/* +case @funbindexpr.kind of + 0 = @normal_call // a normal call +| 1 = @virtual_call // a virtual call +| 2 = @adl_call // a call whose target is only found by ADL +; +*/ +iscall(unique int caller: @funbindexpr ref, int kind: int ref); + +numtemplatearguments( + unique int expr_id: @expr ref, + int num: int ref +); + +specialnamequalifyingelements( + unique int id: @specialnamequalifyingelement, + unique string name: string ref +); + +@namequalifiableelement = @expr | @namequalifier; +@namequalifyingelement = @namespace + | @specialnamequalifyingelement + | @usertype; + +namequalifiers( + unique int id: @namequalifier, + unique int qualifiableelement: @namequalifiableelement ref, + int qualifyingelement: @namequalifyingelement ref, + int location: @location_default ref +); + +varbind( + int expr: @varbindexpr ref, + int var: @accessible ref +); + +funbind( + int expr: @funbindexpr ref, + int fun: @function ref +); + +@any_new_expr = @new_expr + | @new_array_expr; + +@new_or_delete_expr = @any_new_expr + | @delete_expr + | @delete_array_expr; + +@prefix_crement_expr = @preincrexpr | @predecrexpr; + +@postfix_crement_expr = @postincrexpr | @postdecrexpr; + +@increment_expr = @preincrexpr | @postincrexpr; + +@decrement_expr = @predecrexpr | @postdecrexpr; + +@crement_expr = @increment_expr | @decrement_expr; + +@un_arith_op_expr = @arithnegexpr + | @unaryplusexpr + | @conjugation + | @realpartexpr + | @imagpartexpr + | @crement_expr + ; + +@un_bitwise_op_expr = @complementexpr; + +@un_log_op_expr = @notexpr; + +@un_op_expr = @address_of + | @indirect + | @un_arith_op_expr + | @un_bitwise_op_expr + | @builtinaddressof + | @vec_fill + | @un_log_op_expr + | @co_await + | @co_yield + ; + +@bin_log_op_expr = @andlogicalexpr | @orlogicalexpr; + +@cmp_op_expr = @eq_op_expr | @rel_op_expr; + +@eq_op_expr = @eqexpr | @neexpr; + +@rel_op_expr = @gtexpr + | @ltexpr + | @geexpr + | @leexpr + | @spaceshipexpr + ; + +@bin_bitwise_op_expr = @lshiftexpr + | @rshiftexpr + | @andexpr + | @orexpr + | @xorexpr + ; + +@p_arith_op_expr = @paddexpr + | @psubexpr + | @pdiffexpr + ; + +@bin_arith_op_expr = @addexpr + | @subexpr + | @mulexpr + | @divexpr + | @remexpr + | @jmulexpr + | @jdivexpr + | @fjaddexpr + | @jfaddexpr + | @fjsubexpr + | @jfsubexpr + | @minexpr + | @maxexpr + | @p_arith_op_expr + ; + +@bin_op_expr = @bin_arith_op_expr + | @bin_bitwise_op_expr + | @cmp_op_expr + | @bin_log_op_expr + ; + +@op_expr = @un_op_expr + | @bin_op_expr + | @assign_expr + | @conditionalexpr + ; + +@assign_arith_expr = @assignaddexpr + | @assignsubexpr + | @assignmulexpr + | @assigndivexpr + | @assignremexpr + ; + +@assign_bitwise_expr = @assignandexpr + | @assignorexpr + | @assignxorexpr + | @assignlshiftexpr + | @assignrshiftexpr + | @assignpaddexpr + | @assignpsubexpr + ; + +@assign_op_expr = @assign_arith_expr | @assign_bitwise_expr + +@assign_expr = @assignexpr | @assign_op_expr + +/* + case @allocator.form of + 0 = plain + | 1 = alignment + ; +*/ + +/** + * The allocator function associated with a `new` or `new[]` expression. + * The `form` column specified whether the allocation call contains an alignment + * argument. + */ +expr_allocator( + unique int expr: @any_new_expr ref, + int func: @function ref, + int form: int ref +); + +/* + case @deallocator.form of + 0 = plain + | 1 = size + | 2 = alignment + | 3 = size_and_alignment + ; +*/ + +/** + * The deallocator function associated with a `delete`, `delete[]`, `new`, or + * `new[]` expression. For a `new` or `new[]` expression, the deallocator is the + * one used to free memory if the initialization throws an exception. + * The `form` column specifies whether the deallocation call contains a size + * argument, and alignment argument, or both. + */ +expr_deallocator( + unique int expr: @new_or_delete_expr ref, + int func: @function ref, + int form: int ref +); + +/** + * Holds if the `@conditionalexpr` is of the two operand form + * `guard ? : false`. + */ +expr_cond_two_operand( + unique int cond: @conditionalexpr ref +); + +/** + * The guard of `@conditionalexpr` `guard ? true : false` + */ +expr_cond_guard( + unique int cond: @conditionalexpr ref, + int guard: @expr ref +); + +/** + * The expression used when the guard of `@conditionalexpr` + * `guard ? true : false` holds. For the two operand form + * `guard ?: false` consider using `expr_cond_guard` instead. + */ +expr_cond_true( + unique int cond: @conditionalexpr ref, + int true: @expr ref +); + +/** + * The expression used when the guard of `@conditionalexpr` + * `guard ? true : false` does not hold. + */ +expr_cond_false( + unique int cond: @conditionalexpr ref, + int false: @expr ref +); + +/** A string representation of the value. */ +values( + unique int id: @value, + string str: string ref +); + +/** The actual text in the source code for the value, if any. */ +valuetext( + unique int id: @value ref, + string text: string ref +); + +valuebind( + int val: @value ref, + unique int expr: @expr ref +); + +fieldoffsets( + unique int id: @variable ref, + int byteoffset: int ref, + int bitoffset: int ref +); + +bitfield( + unique int id: @variable ref, + int bits: int ref, + int declared_bits: int ref +); + +/* TODO +memberprefix( + int member: @expr ref, + int prefix: @expr ref +); +*/ + +/* + kind(1) = mbrcallexpr + kind(2) = mbrptrcallexpr + kind(3) = mbrptrmbrcallexpr + kind(4) = ptrmbrptrmbrcallexpr + kind(5) = mbrreadexpr // x.y + kind(6) = mbrptrreadexpr // p->y + kind(7) = mbrptrmbrreadexpr // x.*pm + kind(8) = mbrptrmbrptrreadexpr // x->*pm + kind(9) = staticmbrreadexpr // static x.y + kind(10) = staticmbrptrreadexpr // static p->y +*/ +/* TODO +memberaccess( + int member: @expr ref, + int kind: int ref +); +*/ + +initialisers( + unique int init: @initialiser, + int var: @accessible ref, + unique int expr: @expr ref, + int location: @location_expr ref +); + +/** + * An ancestor for the expression, for cases in which we cannot + * otherwise find the expression's parent. + */ +expr_ancestor( + int exp: @expr ref, + int ancestor: @element ref +); + +exprs( + unique int id: @expr, + int kind: int ref, + int location: @location_expr ref +); + +/* + case @value.category of + 1 = prval + | 2 = xval + | 3 = lval + ; +*/ +expr_types( + int id: @expr ref, + int typeid: @type ref, + int value_category: int ref +); + +case @expr.kind of + 1 = @errorexpr +| 2 = @address_of // & AddressOfExpr +| 3 = @reference_to // ReferenceToExpr (implicit?) +| 4 = @indirect // * PointerDereferenceExpr +| 5 = @ref_indirect // ReferenceDereferenceExpr (implicit?) +// ... +| 8 = @array_to_pointer // (???) +| 9 = @vacuous_destructor_call // VacuousDestructorCall +// ... +| 11 = @assume // Microsoft +| 12 = @parexpr +| 13 = @arithnegexpr +| 14 = @unaryplusexpr +| 15 = @complementexpr +| 16 = @notexpr +| 17 = @conjugation // GNU ~ operator +| 18 = @realpartexpr // GNU __real +| 19 = @imagpartexpr // GNU __imag +| 20 = @postincrexpr +| 21 = @postdecrexpr +| 22 = @preincrexpr +| 23 = @predecrexpr +| 24 = @conditionalexpr +| 25 = @addexpr +| 26 = @subexpr +| 27 = @mulexpr +| 28 = @divexpr +| 29 = @remexpr +| 30 = @jmulexpr // C99 mul imaginary +| 31 = @jdivexpr // C99 div imaginary +| 32 = @fjaddexpr // C99 add real + imaginary +| 33 = @jfaddexpr // C99 add imaginary + real +| 34 = @fjsubexpr // C99 sub real - imaginary +| 35 = @jfsubexpr // C99 sub imaginary - real +| 36 = @paddexpr // pointer add (pointer + int or int + pointer) +| 37 = @psubexpr // pointer sub (pointer - integer) +| 38 = @pdiffexpr // difference between two pointers +| 39 = @lshiftexpr +| 40 = @rshiftexpr +| 41 = @andexpr +| 42 = @orexpr +| 43 = @xorexpr +| 44 = @eqexpr +| 45 = @neexpr +| 46 = @gtexpr +| 47 = @ltexpr +| 48 = @geexpr +| 49 = @leexpr +| 50 = @minexpr // GNU minimum +| 51 = @maxexpr // GNU maximum +| 52 = @assignexpr +| 53 = @assignaddexpr +| 54 = @assignsubexpr +| 55 = @assignmulexpr +| 56 = @assigndivexpr +| 57 = @assignremexpr +| 58 = @assignlshiftexpr +| 59 = @assignrshiftexpr +| 60 = @assignandexpr +| 61 = @assignorexpr +| 62 = @assignxorexpr +| 63 = @assignpaddexpr // assign pointer add +| 64 = @assignpsubexpr // assign pointer sub +| 65 = @andlogicalexpr +| 66 = @orlogicalexpr +| 67 = @commaexpr +| 68 = @subscriptexpr // access to member of an array, e.g., a[5] +// ... 69 @objc_subscriptexpr deprecated +// ... 70 @cmdaccess deprecated +// ... +| 73 = @virtfunptrexpr +| 74 = @callexpr +// ... 75 @msgexpr_normal deprecated +// ... 76 @msgexpr_super deprecated +// ... 77 @atselectorexpr deprecated +// ... 78 @atprotocolexpr deprecated +| 79 = @vastartexpr +| 80 = @vaargexpr +| 81 = @vaendexpr +| 82 = @vacopyexpr +// ... 83 @atencodeexpr deprecated +| 84 = @varaccess +| 85 = @thisaccess +// ... 86 @objc_box_expr deprecated +| 87 = @new_expr +| 88 = @delete_expr +| 89 = @throw_expr +| 90 = @condition_decl // a variable declared in a condition, e.g., if(int x = y > 2) +| 91 = @braced_init_list +| 92 = @type_id +| 93 = @runtime_sizeof +| 94 = @runtime_alignof +| 95 = @sizeof_pack +| 96 = @expr_stmt // GNU extension +| 97 = @routineexpr +| 98 = @type_operand // used to access a type in certain contexts (haven't found any examples yet....) +| 99 = @offsetofexpr // offsetof ::= type and field +| 100 = @hasassignexpr // __has_assign ::= type +| 101 = @hascopyexpr // __has_copy ::= type +| 102 = @hasnothrowassign // __has_nothrow_assign ::= type +| 103 = @hasnothrowconstr // __has_nothrow_constructor ::= type +| 104 = @hasnothrowcopy // __has_nothrow_copy ::= type +| 105 = @hastrivialassign // __has_trivial_assign ::= type +| 106 = @hastrivialconstr // __has_trivial_constructor ::= type +| 107 = @hastrivialcopy // __has_trivial_copy ::= type +| 108 = @hasuserdestr // __has_user_destructor ::= type +| 109 = @hasvirtualdestr // __has_virtual_destructor ::= type +| 110 = @isabstractexpr // __is_abstract ::= type +| 111 = @isbaseofexpr // __is_base_of ::= type type +| 112 = @isclassexpr // __is_class ::= type +| 113 = @isconvtoexpr // __is_convertible_to ::= type type +| 114 = @isemptyexpr // __is_empty ::= type +| 115 = @isenumexpr // __is_enum ::= type +| 116 = @ispodexpr // __is_pod ::= type +| 117 = @ispolyexpr // __is_polymorphic ::= type +| 118 = @isunionexpr // __is_union ::= type +| 119 = @typescompexpr // GNU __builtin_types_compatible ::= type type +| 120 = @intaddrexpr // EDG internal builtin, used to implement offsetof +// ... +| 122 = @hastrivialdestructor // __has_trivial_destructor ::= type +| 123 = @literal +| 124 = @uuidof +| 127 = @aggregateliteral +| 128 = @delete_array_expr +| 129 = @new_array_expr +// ... 130 @objc_array_literal deprecated +// ... 131 @objc_dictionary_literal deprecated +| 132 = @foldexpr +// ... +| 200 = @ctordirectinit +| 201 = @ctorvirtualinit +| 202 = @ctorfieldinit +| 203 = @ctordelegatinginit +| 204 = @dtordirectdestruct +| 205 = @dtorvirtualdestruct +| 206 = @dtorfielddestruct +// ... +| 210 = @static_cast +| 211 = @reinterpret_cast +| 212 = @const_cast +| 213 = @dynamic_cast +| 214 = @c_style_cast +| 215 = @lambdaexpr +| 216 = @param_ref +| 217 = @noopexpr +// ... +| 294 = @istriviallyconstructibleexpr +| 295 = @isdestructibleexpr +| 296 = @isnothrowdestructibleexpr +| 297 = @istriviallydestructibleexpr +| 298 = @istriviallyassignableexpr +| 299 = @isnothrowassignableexpr +| 300 = @istrivialexpr +| 301 = @isstandardlayoutexpr +| 302 = @istriviallycopyableexpr +| 303 = @isliteraltypeexpr +| 304 = @hastrivialmoveconstructorexpr +| 305 = @hastrivialmoveassignexpr +| 306 = @hasnothrowmoveassignexpr +| 307 = @isconstructibleexpr +| 308 = @isnothrowconstructibleexpr +| 309 = @hasfinalizerexpr +| 310 = @isdelegateexpr +| 311 = @isinterfaceclassexpr +| 312 = @isrefarrayexpr +| 313 = @isrefclassexpr +| 314 = @issealedexpr +| 315 = @issimplevalueclassexpr +| 316 = @isvalueclassexpr +| 317 = @isfinalexpr +| 319 = @noexceptexpr +| 320 = @builtinshufflevector +| 321 = @builtinchooseexpr +| 322 = @builtinaddressof +| 323 = @vec_fill +| 324 = @builtinconvertvector +| 325 = @builtincomplex +| 326 = @spaceshipexpr +| 327 = @co_await +| 328 = @co_yield +| 329 = @temp_init +; + +@var_args_expr = @vastartexpr + | @vaendexpr + | @vaargexpr + | @vacopyexpr + ; + +@builtin_op = @var_args_expr + | @noopexpr + | @offsetofexpr + | @intaddrexpr + | @hasassignexpr + | @hascopyexpr + | @hasnothrowassign + | @hasnothrowconstr + | @hasnothrowcopy + | @hastrivialassign + | @hastrivialconstr + | @hastrivialcopy + | @hastrivialdestructor + | @hasuserdestr + | @hasvirtualdestr + | @isabstractexpr + | @isbaseofexpr + | @isclassexpr + | @isconvtoexpr + | @isemptyexpr + | @isenumexpr + | @ispodexpr + | @ispolyexpr + | @isunionexpr + | @typescompexpr + | @builtinshufflevector + | @builtinconvertvector + | @builtinaddressof + | @istriviallyconstructibleexpr + | @isdestructibleexpr + | @isnothrowdestructibleexpr + | @istriviallydestructibleexpr + | @istriviallyassignableexpr + | @isnothrowassignableexpr + | @isstandardlayoutexpr + | @istriviallycopyableexpr + | @isliteraltypeexpr + | @hastrivialmoveconstructorexpr + | @hastrivialmoveassignexpr + | @hasnothrowmoveassignexpr + | @isconstructibleexpr + | @isnothrowconstructibleexpr + | @hasfinalizerexpr + | @isdelegateexpr + | @isinterfaceclassexpr + | @isrefarrayexpr + | @isrefclassexpr + | @issealedexpr + | @issimplevalueclassexpr + | @isvalueclassexpr + | @isfinalexpr + | @builtinchooseexpr + | @builtincomplex + ; + +new_allocated_type( + unique int expr: @new_expr ref, + int type_id: @type ref +); + +new_array_allocated_type( + unique int expr: @new_array_expr ref, + int type_id: @type ref +); + +/** + * The field being initialized by an initializer expression within an aggregate + * initializer for a class/struct/union. + */ +#keyset[aggregate, field] +aggregate_field_init( + int aggregate: @aggregateliteral ref, + int initializer: @expr ref, + int field: @membervariable ref +); + +/** + * The index of the element being initialized by an initializer expression + * within an aggregate initializer for an array. + */ +#keyset[aggregate, element_index] +aggregate_array_init( + int aggregate: @aggregateliteral ref, + int initializer: @expr ref, + int element_index: int ref +); + +@ctorinit = @ctordirectinit + | @ctorvirtualinit + | @ctorfieldinit + | @ctordelegatinginit; +@dtordestruct = @dtordirectdestruct + | @dtorvirtualdestruct + | @dtorfielddestruct; + + +condition_decl_bind( + unique int expr: @condition_decl ref, + unique int decl: @declaration ref +); + +typeid_bind( + unique int expr: @type_id ref, + int type_id: @type ref +); + +uuidof_bind( + unique int expr: @uuidof ref, + int type_id: @type ref +); + +@runtime_sizeof_or_alignof = @runtime_sizeof | @runtime_alignof; + +sizeof_bind( + unique int expr: @runtime_sizeof_or_alignof ref, + int type_id: @type ref +); + +code_block( + unique int block: @literal ref, + unique int routine: @function ref +); + +lambdas( + unique int expr: @lambdaexpr ref, + string default_capture: string ref, + boolean has_explicit_return_type: boolean ref +); + +lambda_capture( + unique int id: @lambdacapture, + int lambda: @lambdaexpr ref, + int index: int ref, + int field: @membervariable ref, + boolean captured_by_reference: boolean ref, + boolean is_implicit: boolean ref, + int location: @location_default ref +); + +@funbindexpr = @routineexpr + | @new_expr + | @delete_expr + | @delete_array_expr + | @ctordirectinit + | @ctorvirtualinit + | @ctordelegatinginit + | @dtordirectdestruct + | @dtorvirtualdestruct; + +@varbindexpr = @varaccess | @ctorfieldinit | @dtorfielddestruct; +@addressable = @function | @variable ; +@accessible = @addressable | @enumconstant ; + +@access = @varaccess | @routineexpr ; + +fold( + int expr: @foldexpr ref, + string operator: string ref, + boolean is_left_fold: boolean ref +); + +stmts( + unique int id: @stmt, + int kind: int ref, + int location: @location_stmt ref +); + +case @stmt.kind of + 1 = @stmt_expr +| 2 = @stmt_if +| 3 = @stmt_while +| 4 = @stmt_goto +| 5 = @stmt_label +| 6 = @stmt_return +| 7 = @stmt_block +| 8 = @stmt_end_test_while // do { ... } while ( ... ) +| 9 = @stmt_for +| 10 = @stmt_switch_case +| 11 = @stmt_switch +| 13 = @stmt_asm // "asm" statement or the body of an asm function +| 15 = @stmt_try_block +| 16 = @stmt_microsoft_try // Microsoft +| 17 = @stmt_decl +| 18 = @stmt_set_vla_size // C99 +| 19 = @stmt_vla_decl // C99 +| 25 = @stmt_assigned_goto // GNU +| 26 = @stmt_empty +| 27 = @stmt_continue +| 28 = @stmt_break +| 29 = @stmt_range_based_for // C++11 +// ... 30 @stmt_at_autoreleasepool_block deprecated +// ... 31 @stmt_objc_for_in deprecated +// ... 32 @stmt_at_synchronized deprecated +| 33 = @stmt_handler +// ... 34 @stmt_finally_end deprecated +| 35 = @stmt_constexpr_if +| 37 = @stmt_co_return +; + +type_vla( + int type_id: @type ref, + int decl: @stmt_vla_decl ref +); + +variable_vla( + int var: @variable ref, + int decl: @stmt_vla_decl ref +); + +if_initialization( + unique int if_stmt: @stmt_if ref, + int init_id: @stmt ref +); + +if_then( + unique int if_stmt: @stmt_if ref, + int then_id: @stmt ref +); + +if_else( + unique int if_stmt: @stmt_if ref, + int else_id: @stmt ref +); + +constexpr_if_initialization( + unique int constexpr_if_stmt: @stmt_constexpr_if ref, + int init_id: @stmt ref +); + +constexpr_if_then( + unique int constexpr_if_stmt: @stmt_constexpr_if ref, + int then_id: @stmt ref +); + +constexpr_if_else( + unique int constexpr_if_stmt: @stmt_constexpr_if ref, + int else_id: @stmt ref +); + +while_body( + unique int while_stmt: @stmt_while ref, + int body_id: @stmt ref +); + +do_body( + unique int do_stmt: @stmt_end_test_while ref, + int body_id: @stmt ref +); + +switch_initialization( + unique int switch_stmt: @stmt_switch ref, + int init_id: @stmt ref +); + +#keyset[switch_stmt, index] +switch_case( + int switch_stmt: @stmt_switch ref, + int index: int ref, + int case_id: @stmt_switch_case ref +); + +switch_body( + unique int switch_stmt: @stmt_switch ref, + int body_id: @stmt ref +); + +for_initialization( + unique int for_stmt: @stmt_for ref, + int init_id: @stmt ref +); + +for_condition( + unique int for_stmt: @stmt_for ref, + int condition_id: @expr ref +); + +for_update( + unique int for_stmt: @stmt_for ref, + int update_id: @expr ref +); + +for_body( + unique int for_stmt: @stmt_for ref, + int body_id: @stmt ref +); + +@stmtparent = @stmt | @expr_stmt ; +stmtparents( + unique int id: @stmt ref, + int index: int ref, + int parent: @stmtparent ref +); + +ishandler(unique int block: @stmt_block ref); + +@cfgnode = @stmt | @expr | @function | @initialiser ; + +stmt_decl_bind( + int stmt: @stmt_decl ref, + int num: int ref, + int decl: @declaration ref +); + +stmt_decl_entry_bind( + int stmt: @stmt_decl ref, + int num: int ref, + int decl_entry: @element ref +); + +@functionorblock = @function | @stmt_block; + +blockscope( + unique int block: @stmt_block ref, + int enclosing: @functionorblock ref +); + +@jump = @stmt_goto | @stmt_break | @stmt_continue; + +@jumporlabel = @jump | @stmt_label | @literal; + +jumpinfo( + unique int id: @jumporlabel ref, + string str: string ref, + int target: @stmt ref +); + +preprocdirects( + unique int id: @preprocdirect, + int kind: int ref, + int location: @location_default ref +); +case @preprocdirect.kind of + 0 = @ppd_if +| 1 = @ppd_ifdef +| 2 = @ppd_ifndef +| 3 = @ppd_elif +| 4 = @ppd_else +| 5 = @ppd_endif +| 6 = @ppd_plain_include +| 7 = @ppd_define +| 8 = @ppd_undef +| 9 = @ppd_line +| 10 = @ppd_error +| 11 = @ppd_pragma +| 12 = @ppd_objc_import +| 13 = @ppd_include_next +| 18 = @ppd_warning +; + +@ppd_include = @ppd_plain_include | @ppd_objc_import | @ppd_include_next; + +@ppd_branch = @ppd_if | @ppd_ifdef | @ppd_ifndef | @ppd_elif; + +preprocpair( + int begin : @ppd_branch ref, + int elseelifend : @preprocdirect ref +); + +preproctrue(int branch : @ppd_branch ref); +preprocfalse(int branch : @ppd_branch ref); + +preproctext( + unique int id: @preprocdirect ref, + string head: string ref, + string body: string ref +); + +includes( + unique int id: @ppd_include ref, + int included: @file ref +); + +link_targets( + unique int id: @link_target, + int binary: @file ref +); + +link_parent( + int element : @element ref, + int link_target : @link_target ref +); + +/* XML Files */ + +xmlEncoding(unique int id: @file ref, string encoding: string ref); + +xmlDTDs( + unique int id: @xmldtd, + string root: string ref, + string publicId: string ref, + string systemId: string ref, + int fileid: @file ref +); + +xmlElements( + unique int id: @xmlelement, + string name: string ref, + int parentid: @xmlparent ref, + int idx: int ref, + int fileid: @file ref +); + +xmlAttrs( + unique int id: @xmlattribute, + int elementid: @xmlelement ref, + string name: string ref, + string value: string ref, + int idx: int ref, + int fileid: @file ref +); + +xmlNs( + int id: @xmlnamespace, + string prefixName: string ref, + string URI: string ref, + int fileid: @file ref +); + +xmlHasNs( + int elementId: @xmlnamespaceable ref, + int nsId: @xmlnamespace ref, + int fileid: @file ref +); + +xmlComments( + unique int id: @xmlcomment, + string text: string ref, + int parentid: @xmlparent ref, + int fileid: @file ref +); + +xmlChars( + unique int id: @xmlcharacters, + string text: string ref, + int parentid: @xmlparent ref, + int idx: int ref, + int isCDATA: int ref, + int fileid: @file ref +); + +@xmlparent = @file | @xmlelement; +@xmlnamespaceable = @xmlelement | @xmlattribute; + +xmllocations( + int xmlElement: @xmllocatable ref, + int location: @location_default ref +); + +@xmllocatable = @xmlcharacters + | @xmlelement + | @xmlcomment + | @xmlattribute + | @xmldtd + | @file + | @xmlnamespace; diff --git a/cpp/downgrades/cf72c8898d19eb1b3374432cf79d8276cb07ad43/semmlecode.cpp.dbscheme b/cpp/downgrades/cf72c8898d19eb1b3374432cf79d8276cb07ad43/semmlecode.cpp.dbscheme new file mode 100644 index 00000000000..e9a518baf14 --- /dev/null +++ b/cpp/downgrades/cf72c8898d19eb1b3374432cf79d8276cb07ad43/semmlecode.cpp.dbscheme @@ -0,0 +1,2096 @@ + +/** + * An invocation of the compiler. Note that more than one file may be + * compiled per invocation. For example, this command compiles three + * source files: + * + * gcc -c f1.c f2.c f3.c + * + * The `id` simply identifies the invocation, while `cwd` is the working + * directory from which the compiler was invoked. + */ +compilations( + /** + * An invocation of the compiler. Note that more than one file may + * be compiled per invocation. For example, this command compiles + * three source files: + * + * gcc -c f1.c f2.c f3.c + */ + unique int id : @compilation, + string cwd : string ref +); + +/** + * The arguments that were passed to the extractor for a compiler + * invocation. If `id` is for the compiler invocation + * + * gcc -c f1.c f2.c f3.c + * + * then typically there will be rows for + * + * num | arg + * --- | --- + * 0 | *path to extractor* + * 1 | `--mimic` + * 2 | `/usr/bin/gcc` + * 3 | `-c` + * 4 | f1.c + * 5 | f2.c + * 6 | f3.c + */ +#keyset[id, num] +compilation_args( + int id : @compilation ref, + int num : int ref, + string arg : string ref +); + +/** + * The source files that are compiled by a compiler invocation. + * If `id` is for the compiler invocation + * + * gcc -c f1.c f2.c f3.c + * + * then there will be rows for + * + * num | arg + * --- | --- + * 0 | f1.c + * 1 | f2.c + * 2 | f3.c + * + * Note that even if those files `#include` headers, those headers + * do not appear as rows. + */ +#keyset[id, num] +compilation_compiling_files( + int id : @compilation ref, + int num : int ref, + int file : @file ref +); + +/** + * The time taken by the extractor for a compiler invocation. + * + * For each file `num`, there will be rows for + * + * kind | seconds + * ---- | --- + * 1 | CPU seconds used by the extractor frontend + * 2 | Elapsed seconds during the extractor frontend + * 3 | CPU seconds used by the extractor backend + * 4 | Elapsed seconds during the extractor backend + */ +#keyset[id, num, kind] +compilation_time( + int id : @compilation ref, + int num : int ref, + /* kind: + 1 = frontend_cpu_seconds + 2 = frontend_elapsed_seconds + 3 = extractor_cpu_seconds + 4 = extractor_elapsed_seconds + */ + int kind : int ref, + float seconds : float ref +); + +/** + * An error or warning generated by the extractor. + * The diagnostic message `diagnostic` was generated during compiler + * invocation `compilation`, and is the `file_number_diagnostic_number`th + * message generated while extracting the `file_number`th file of that + * invocation. + */ +#keyset[compilation, file_number, file_number_diagnostic_number] +diagnostic_for( + int diagnostic : @diagnostic ref, + int compilation : @compilation ref, + int file_number : int ref, + int file_number_diagnostic_number : int ref +); + +/** + * If extraction was successful, then `cpu_seconds` and + * `elapsed_seconds` are the CPU time and elapsed time (respectively) + * that extraction took for compiler invocation `id`. + */ +compilation_finished( + unique int id : @compilation ref, + float cpu_seconds : float ref, + float elapsed_seconds : float ref +); + + +/** + * External data, loaded from CSV files during snapshot creation. See + * [Tutorial: Incorporating external data](https://help.semmle.com/wiki/display/SD/Tutorial%3A+Incorporating+external+data) + * for more information. + */ +externalData( + int id : @externalDataElement, + string path : string ref, + int column: int ref, + string value : string ref +); + +/** + * The source location of the snapshot. + */ +sourceLocationPrefix(string prefix : string ref); + +/** + * Information about packages that provide code used during compilation. + * The `id` is just a unique identifier. + * The `namespace` is typically the name of the package manager that + * provided the package (e.g. "dpkg" or "yum"). + * The `package_name` is the name of the package, and `version` is its + * version (as a string). + */ +external_packages( + unique int id: @external_package, + string namespace : string ref, + string package_name : string ref, + string version : string ref +); + +/** + * Holds if File `fileid` was provided by package `package`. + */ +header_to_external_package( + int fileid : @file ref, + int package : @external_package ref +); + +/* + * Version history + */ + +svnentries( + unique int id : @svnentry, + string revision : string ref, + string author : string ref, + date revisionDate : date ref, + int changeSize : int ref +) + +svnaffectedfiles( + int id : @svnentry ref, + int file : @file ref, + string action : string ref +) + +svnentrymsg( + unique int id : @svnentry ref, + string message : string ref +) + +svnchurn( + int commit : @svnentry ref, + int file : @file ref, + int addedLines : int ref, + int deletedLines : int ref +) + +/* + * C++ dbscheme + */ + +@location = @location_stmt | @location_expr | @location_default ; + +/** + * The location of an element that is not an expression or a statement. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `file`. + * For more information, see + * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/). + */ +locations_default( + /** The location of an element that is not an expression or a statement. */ + unique int id: @location_default, + int container: @container ref, + int startLine: int ref, + int startColumn: int ref, + int endLine: int ref, + int endColumn: int ref +); + +/** + * The location of a statement. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `file`. + * For more information, see + * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/). + */ +locations_stmt( + /** The location of a statement. */ + unique int id: @location_stmt, + int container: @container ref, + int startLine: int ref, + int startColumn: int ref, + int endLine: int ref, + int endColumn: int ref +); + +/** + * The location of an expression. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `file`. + * For more information, see + * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/). + */ +locations_expr( + /** The location of an expression. */ + unique int id: @location_expr, + int container: @container ref, + int startLine: int ref, + int startColumn: int ref, + int endLine: int ref, + int endColumn: int ref +); + +/** An element for which line-count information is available. */ +@sourceline = @file | @function | @variable | @enumconstant | @xmllocatable; + +numlines( + int element_id: @sourceline ref, + int num_lines: int ref, + int num_code: int ref, + int num_comment: int ref +); + +diagnostics( + unique int id: @diagnostic, + int severity: int ref, + string error_tag: string ref, + string error_message: string ref, + string full_error_message: string ref, + int location: @location_default ref +); + +files( + unique int id: @file, + string name: string ref +); + +folders( + unique int id: @folder, + string name: string ref +); + +@container = @folder | @file + +containerparent( + int parent: @container ref, + unique int child: @container ref +); + +fileannotations( + int id: @file ref, + int kind: int ref, + string name: string ref, + string value: string ref +); + +inmacroexpansion( + int id: @element ref, + int inv: @macroinvocation ref +); + +affectedbymacroexpansion( + int id: @element ref, + int inv: @macroinvocation ref +); + +/* + case @macroinvocations.kind of + 1 = macro expansion + | 2 = other macro reference + ; +*/ +macroinvocations( + unique int id: @macroinvocation, + int macro_id: @ppd_define ref, + int location: @location_default ref, + int kind: int ref +); + +macroparent( + unique int id: @macroinvocation ref, + int parent_id: @macroinvocation ref +); + +// a macroinvocation may be part of another location +// the way to find a constant expression that uses a macro +// is thus to find a constant expression that has a location +// to which a macro invocation is bound +macrolocationbind( + int id: @macroinvocation ref, + int location: @location ref +); + +#keyset[invocation, argument_index] +macro_argument_unexpanded( + int invocation: @macroinvocation ref, + int argument_index: int ref, + string text: string ref +); + +#keyset[invocation, argument_index] +macro_argument_expanded( + int invocation: @macroinvocation ref, + int argument_index: int ref, + string text: string ref +); + +/* + case @function.kind of + 1 = normal + | 2 = constructor + | 3 = destructor + | 4 = conversion + | 5 = operator + | 6 = builtin // GCC built-in functions, e.g. __builtin___memcpy_chk + ; +*/ +functions( + unique int id: @function, + string name: string ref, + int kind: int ref +); + +function_entry_point(int id: @function ref, unique int entry_point: @stmt ref); + +function_return_type(int id: @function ref, int return_type: @type ref); + +/** If `function` is a coroutine, then this gives the + std::experimental::resumable_traits instance associated with it, + and the variables representing the `handle` and `promise` for it. */ +coroutine( + unique int function: @function ref, + int traits: @type ref, + int handle: @variable ref, + int promise: @variable ref +); + +/** The `new` function used for allocating the coroutine state, if any. */ +coroutine_new( + unique int function: @function ref, + int new: @function ref +); + +/** The `delete` function used for deallocating the coroutine state, if any. */ +coroutine_delete( + unique int function: @function ref, + int delete: @function ref +); + +purefunctions(unique int id: @function ref); + +function_deleted(unique int id: @function ref); + +function_defaulted(unique int id: @function ref); + +member_function_this_type(unique int id: @function ref, int this_type: @type ref); + +#keyset[id, type_id] +fun_decls( + int id: @fun_decl, + int function: @function ref, + int type_id: @type ref, + string name: string ref, + int location: @location_default ref +); +fun_def(unique int id: @fun_decl ref); +fun_specialized(unique int id: @fun_decl ref); +fun_implicit(unique int id: @fun_decl ref); +fun_decl_specifiers( + int id: @fun_decl ref, + string name: string ref +) +#keyset[fun_decl, index] +fun_decl_throws( + int fun_decl: @fun_decl ref, + int index: int ref, + int type_id: @type ref +); +/* an empty throw specification is different from none */ +fun_decl_empty_throws(unique int fun_decl: @fun_decl ref); +fun_decl_noexcept( + int fun_decl: @fun_decl ref, + int constant: @expr ref +); +fun_decl_empty_noexcept(int fun_decl: @fun_decl ref); +fun_decl_typedef_type( + unique int fun_decl: @fun_decl ref, + int typedeftype_id: @usertype ref +); + +param_decl_bind( + unique int id: @var_decl ref, + int index: int ref, + int fun_decl: @fun_decl ref +); + +#keyset[id, type_id] +var_decls( + int id: @var_decl, + int variable: @variable ref, + int type_id: @type ref, + string name: string ref, + int location: @location_default ref +); +var_def(unique int id: @var_decl ref); +var_decl_specifiers( + int id: @var_decl ref, + string name: string ref +) +is_structured_binding(unique int id: @variable ref); + +type_decls( + unique int id: @type_decl, + int type_id: @type ref, + int location: @location_default ref +); +type_def(unique int id: @type_decl ref); +type_decl_top( + unique int type_decl: @type_decl ref +); + +namespace_decls( + unique int id: @namespace_decl, + int namespace_id: @namespace ref, + int location: @location_default ref, + int bodylocation: @location_default ref +); + +usings( + unique int id: @using, + int element_id: @element ref, + int location: @location_default ref +); + +/** The element which contains the `using` declaration. */ +using_container( + int parent: @element ref, + int child: @using ref +); + +static_asserts( + unique int id: @static_assert, + int condition : @expr ref, + string message : string ref, + int location: @location_default ref, + int enclosing : @element ref +); + +// each function has an ordered list of parameters +#keyset[id, type_id] +#keyset[function, index, type_id] +params( + int id: @parameter, + int function: @functionorblock ref, + int index: int ref, + int type_id: @type ref +); + +overrides(int new: @function ref, int old: @function ref); + +#keyset[id, type_id] +membervariables( + int id: @membervariable, + int type_id: @type ref, + string name: string ref +); + +#keyset[id, type_id] +globalvariables( + int id: @globalvariable, + int type_id: @type ref, + string name: string ref +); + +#keyset[id, type_id] +localvariables( + int id: @localvariable, + int type_id: @type ref, + string name: string ref +); + +autoderivation( + unique int var: @variable ref, + int derivation_type: @type ref +); + +enumconstants( + unique int id: @enumconstant, + int parent: @usertype ref, + int index: int ref, + int type_id: @type ref, + string name: string ref, + int location: @location_default ref +); + +@variable = @localscopevariable | @globalvariable | @membervariable; + +@localscopevariable = @localvariable | @parameter; + +/* + Built-in types are the fundamental types, e.g., integral, floating, and void. + + case @builtintype.kind of + 1 = error + | 2 = unknown + | 3 = void + | 4 = boolean + | 5 = char + | 6 = unsigned_char + | 7 = signed_char + | 8 = short + | 9 = unsigned_short + | 10 = signed_short + | 11 = int + | 12 = unsigned_int + | 13 = signed_int + | 14 = long + | 15 = unsigned_long + | 16 = signed_long + | 17 = long_long + | 18 = unsigned_long_long + | 19 = signed_long_long + | 20 = __int8 // Microsoft-specific + | 21 = __int16 // Microsoft-specific + | 22 = __int32 // Microsoft-specific + | 23 = __int64 // Microsoft-specific + | 24 = float + | 25 = double + | 26 = long_double + | 27 = _Complex_float // C99-specific + | 28 = _Complex_double // C99-specific + | 29 = _Complex_long double // C99-specific + | 30 = _Imaginary_float // C99-specific + | 31 = _Imaginary_double // C99-specific + | 32 = _Imaginary_long_double // C99-specific + | 33 = wchar_t // Microsoft-specific + | 34 = decltype_nullptr // C++11 + | 35 = __int128 + | 36 = unsigned___int128 + | 37 = signed___int128 + | 38 = __float128 + | 39 = _Complex___float128 + | 40 = _Decimal32 + | 41 = _Decimal64 + | 42 = _Decimal128 + | 43 = char16_t + | 44 = char32_t + | 45 = _Float32 + | 46 = _Float32x + | 47 = _Float64 + | 48 = _Float64x + | 49 = _Float128 + | 50 = _Float128x + | 51 = char8_t + ; +*/ +builtintypes( + unique int id: @builtintype, + string name: string ref, + int kind: int ref, + int size: int ref, + int sign: int ref, + int alignment: int ref +); + +/* + Derived types are types that are directly derived from existing types and + point to, refer to, transform type data to return a new type. + + case @derivedtype.kind of + 1 = pointer + | 2 = reference + | 3 = type_with_specifiers + | 4 = array + | 5 = gnu_vector + | 6 = routineptr + | 7 = routinereference + | 8 = rvalue_reference // C++11 +// ... 9 type_conforming_to_protocols deprecated + | 10 = block + ; +*/ +derivedtypes( + unique int id: @derivedtype, + string name: string ref, + int kind: int ref, + int type_id: @type ref +); + +pointerishsize(unique int id: @derivedtype ref, + int size: int ref, + int alignment: int ref); + +arraysizes( + unique int id: @derivedtype ref, + int num_elements: int ref, + int bytesize: int ref, + int alignment: int ref +); + +typedefbase( + unique int id: @usertype ref, + int type_id: @type ref +); + +/** + * An instance of the C++11 `decltype` operator. For example: + * ``` + * int a; + * decltype(1+a) b; + * ``` + * Here `expr` is `1+a`. + * + * Sometimes an additional pair of parentheses around the expression + * would change the semantics of this decltype, e.g. + * ``` + * struct A { double x; }; + * const A* a = new A(); + * decltype( a->x ); // type is double + * decltype((a->x)); // type is const double& + * ``` + * (Please consult the C++11 standard for more details). + * `parentheses_would_change_meaning` is `true` iff that is the case. + */ +#keyset[id, expr] +decltypes( + int id: @decltype, + int expr: @expr ref, + int base_type: @type ref, + boolean parentheses_would_change_meaning: boolean ref +); + +/* + case @usertype.kind of + 1 = struct + | 2 = class + | 3 = union + | 4 = enum + | 5 = typedef // classic C: typedef typedef type name + | 6 = template + | 7 = template_parameter + | 8 = template_template_parameter + | 9 = proxy_class // a proxy class associated with a template parameter +// ... 10 objc_class deprecated +// ... 11 objc_protocol deprecated +// ... 12 objc_category deprecated + | 13 = scoped_enum + | 14 = using_alias // a using name = type style typedef + ; +*/ +usertypes( + unique int id: @usertype, + string name: string ref, + int kind: int ref +); + +usertypesize( + unique int id: @usertype ref, + int size: int ref, + int alignment: int ref +); + +usertype_final(unique int id: @usertype ref); + +usertype_uuid( + unique int id: @usertype ref, + string uuid: string ref +); + +mangled_name( + unique int id: @declaration ref, + int mangled_name : @mangledname +); + +is_pod_class(unique int id: @usertype ref); +is_standard_layout_class(unique int id: @usertype ref); + +is_complete(unique int id: @usertype ref); + +is_class_template(unique int id: @usertype ref); +class_instantiation( + int to: @usertype ref, + int from: @usertype ref +); +class_template_argument( + int type_id: @usertype ref, + int index: int ref, + int arg_type: @type ref +); +class_template_argument_value( + int type_id: @usertype ref, + int index: int ref, + int arg_value: @expr ref +); + +is_proxy_class_for( + unique int id: @usertype ref, + unique int templ_param_id: @usertype ref +); + +type_mentions( + unique int id: @type_mention, + int type_id: @type ref, + int location: @location ref, + // a_symbol_reference_kind from the EDG frontend. See symbol_ref.h there. + int kind: int ref +); + +is_function_template(unique int id: @function ref); +function_instantiation( + unique int to: @function ref, + int from: @function ref +); +function_template_argument( + int function_id: @function ref, + int index: int ref, + int arg_type: @type ref +); +function_template_argument_value( + int function_id: @function ref, + int index: int ref, + int arg_value: @expr ref +); + +is_variable_template(unique int id: @variable ref); +variable_instantiation( + unique int to: @variable ref, + int from: @variable ref +); +variable_template_argument( + int variable_id: @variable ref, + int index: int ref, + int arg_type: @type ref +); +variable_template_argument_value( + int variable_id: @variable ref, + int index: int ref, + int arg_value: @expr ref +); + +/* + Fixed point types + precision(1) = short, precision(2) = default, precision(3) = long + is_unsigned(1) = unsigned is_unsigned(2) = signed + is_fract_type(1) = declared with _Fract + saturating(1) = declared with _Sat +*/ +/* TODO +fixedpointtypes( + unique int id: @fixedpointtype, + int precision: int ref, + int is_unsigned: int ref, + int is_fract_type: int ref, + int saturating: int ref); +*/ + +routinetypes( + unique int id: @routinetype, + int return_type: @type ref +); + +routinetypeargs( + int routine: @routinetype ref, + int index: int ref, + int type_id: @type ref +); + +ptrtomembers( + unique int id: @ptrtomember, + int type_id: @type ref, + int class_id: @type ref +); + +/* + specifiers for types, functions, and variables + + "public", + "protected", + "private", + + "const", + "volatile", + "static", + + "pure", + "virtual", + "sealed", // Microsoft + "__interface", // Microsoft + "inline", + "explicit", + + "near", // near far extension + "far", // near far extension + "__ptr32", // Microsoft + "__ptr64", // Microsoft + "__sptr", // Microsoft + "__uptr", // Microsoft + "dllimport", // Microsoft + "dllexport", // Microsoft + "thread", // Microsoft + "naked", // Microsoft + "microsoft_inline", // Microsoft + "forceinline", // Microsoft + "selectany", // Microsoft + "nothrow", // Microsoft + "novtable", // Microsoft + "noreturn", // Microsoft + "noinline", // Microsoft + "noalias", // Microsoft + "restrict", // Microsoft +*/ + +specifiers( + unique int id: @specifier, + unique string str: string ref +); + +typespecifiers( + int type_id: @type ref, + int spec_id: @specifier ref +); + +funspecifiers( + int func_id: @function ref, + int spec_id: @specifier ref +); + +varspecifiers( + int var_id: @accessible ref, + int spec_id: @specifier ref +); + +attributes( + unique int id: @attribute, + int kind: int ref, + string name: string ref, + string name_space: string ref, + int location: @location_default ref +); + +case @attribute.kind of + 0 = @gnuattribute +| 1 = @stdattribute +| 2 = @declspec +| 3 = @msattribute +| 4 = @alignas +// ... 5 @objc_propertyattribute deprecated +; + +attribute_args( + unique int id: @attribute_arg, + int kind: int ref, + int attribute: @attribute ref, + int index: int ref, + int location: @location_default ref +); + +case @attribute_arg.kind of + 0 = @attribute_arg_empty +| 1 = @attribute_arg_token +| 2 = @attribute_arg_constant +| 3 = @attribute_arg_type +; + +attribute_arg_value( + unique int arg: @attribute_arg ref, + string value: string ref +); +attribute_arg_type( + unique int arg: @attribute_arg ref, + int type_id: @type ref +); +attribute_arg_name( + unique int arg: @attribute_arg ref, + string name: string ref +); + +typeattributes( + int type_id: @type ref, + int spec_id: @attribute ref +); + +funcattributes( + int func_id: @function ref, + int spec_id: @attribute ref +); + +varattributes( + int var_id: @accessible ref, + int spec_id: @attribute ref +); + +stmtattributes( + int stmt_id: @stmt ref, + int spec_id: @attribute ref +); + +@type = @builtintype + | @derivedtype + | @usertype + /* TODO | @fixedpointtype */ + | @routinetype + | @ptrtomember + | @decltype; + +unspecifiedtype( + unique int type_id: @type ref, + int unspecified_type_id: @type ref +); + +member( + int parent: @type ref, + int index: int ref, + int child: @member ref +); + +@enclosingfunction_child = @usertype | @variable | @namespace + +enclosingfunction( + unique int child: @enclosingfunction_child ref, + int parent: @function ref +); + +derivations( + unique int derivation: @derivation, + int sub: @type ref, + int index: int ref, + int super: @type ref, + int location: @location_default ref +); + +derspecifiers( + int der_id: @derivation ref, + int spec_id: @specifier ref +); + +/** + * Contains the byte offset of the base class subobject within the derived + * class. Only holds for non-virtual base classes, but see table + * `virtual_base_offsets` for offsets of virtual base class subobjects. + */ +direct_base_offsets( + unique int der_id: @derivation ref, + int offset: int ref +); + +/** + * Contains the byte offset of the virtual base class subobject for class + * `super` within a most-derived object of class `sub`. `super` can be either a + * direct or indirect base class. + */ +#keyset[sub, super] +virtual_base_offsets( + int sub: @usertype ref, + int super: @usertype ref, + int offset: int ref +); + +frienddecls( + unique int id: @frienddecl, + int type_id: @type ref, + int decl_id: @declaration ref, + int location: @location_default ref +); + +@declaredtype = @usertype ; + +@declaration = @function + | @declaredtype + | @variable + | @enumconstant + | @frienddecl; + +@member = @membervariable + | @function + | @declaredtype + | @enumconstant; + +@locatable = @diagnostic + | @declaration + | @ppd_include + | @ppd_define + | @macroinvocation + /*| @funcall*/ + | @xmllocatable + | @attribute + | @attribute_arg; + +@namedscope = @namespace | @usertype; + +@element = @locatable + | @file + | @folder + | @specifier + | @type + | @expr + | @namespace + | @initialiser + | @stmt + | @derivation + | @comment + | @preprocdirect + | @fun_decl + | @var_decl + | @type_decl + | @namespace_decl + | @using + | @namequalifier + | @specialnamequalifyingelement + | @static_assert + | @type_mention + | @lambdacapture; + +@exprparent = @element; + +comments( + unique int id: @comment, + string contents: string ref, + int location: @location_default ref +); + +commentbinding( + int id: @comment ref, + int element: @element ref +); + +exprconv( + int converted: @expr ref, + unique int conversion: @expr ref +); + +compgenerated(unique int id: @element ref); + +/** + * `destructor_call` destructs the `i`'th entity that should be + * destructed following `element`. Note that entities should be + * destructed in reverse construction order, so for a given `element` + * these should be called from highest to lowest `i`. + */ +#keyset[element, destructor_call] +#keyset[element, i] +synthetic_destructor_call( + int element: @element ref, + int i: int ref, + int destructor_call: @routineexpr ref +); + +namespaces( + unique int id: @namespace, + string name: string ref +); + +namespace_inline( + unique int id: @namespace ref +); + +namespacembrs( + int parentid: @namespace ref, + unique int memberid: @namespacembr ref +); + +@namespacembr = @declaration | @namespace; + +exprparents( + int expr_id: @expr ref, + int child_index: int ref, + int parent_id: @exprparent ref +); + +expr_isload(unique int expr_id: @expr ref); + +@cast = @c_style_cast + | @const_cast + | @dynamic_cast + | @reinterpret_cast + | @static_cast + ; + +/* +case @conversion.kind of + 0 = @simple_conversion // a numeric conversion, qualification conversion, or a reinterpret_cast +| 1 = @bool_conversion // conversion to 'bool' +| 2 = @base_class_conversion // a derived-to-base conversion +| 3 = @derived_class_conversion // a base-to-derived conversion +| 4 = @pm_base_class_conversion // a derived-to-base conversion of a pointer to member +| 5 = @pm_derived_class_conversion // a base-to-derived conversion of a pointer to member +| 6 = @glvalue_adjust // an adjustment of the type of a glvalue +| 7 = @prvalue_adjust // an adjustment of the type of a prvalue +; +*/ +/** + * Describes the semantics represented by a cast expression. This is largely + * independent of the source syntax of the cast, so it is separate from the + * regular expression kind. + */ +conversionkinds( + unique int expr_id: @cast ref, + int kind: int ref +); + +@conversion = @cast + | @array_to_pointer + | @parexpr + | @reference_to + | @ref_indirect + | @temp_init + ; + +/* +case @funbindexpr.kind of + 0 = @normal_call // a normal call +| 1 = @virtual_call // a virtual call +| 2 = @adl_call // a call whose target is only found by ADL +; +*/ +iscall(unique int caller: @funbindexpr ref, int kind: int ref); + +numtemplatearguments( + unique int expr_id: @expr ref, + int num: int ref +); + +specialnamequalifyingelements( + unique int id: @specialnamequalifyingelement, + unique string name: string ref +); + +@namequalifiableelement = @expr | @namequalifier; +@namequalifyingelement = @namespace + | @specialnamequalifyingelement + | @usertype; + +namequalifiers( + unique int id: @namequalifier, + unique int qualifiableelement: @namequalifiableelement ref, + int qualifyingelement: @namequalifyingelement ref, + int location: @location_default ref +); + +varbind( + int expr: @varbindexpr ref, + int var: @accessible ref +); + +funbind( + int expr: @funbindexpr ref, + int fun: @function ref +); + +@any_new_expr = @new_expr + | @new_array_expr; + +@new_or_delete_expr = @any_new_expr + | @delete_expr + | @delete_array_expr; + +@prefix_crement_expr = @preincrexpr | @predecrexpr; + +@postfix_crement_expr = @postincrexpr | @postdecrexpr; + +@increment_expr = @preincrexpr | @postincrexpr; + +@decrement_expr = @predecrexpr | @postdecrexpr; + +@crement_expr = @increment_expr | @decrement_expr; + +@un_arith_op_expr = @arithnegexpr + | @unaryplusexpr + | @conjugation + | @realpartexpr + | @imagpartexpr + | @crement_expr + ; + +@un_bitwise_op_expr = @complementexpr; + +@un_log_op_expr = @notexpr; + +@un_op_expr = @address_of + | @indirect + | @un_arith_op_expr + | @un_bitwise_op_expr + | @builtinaddressof + | @vec_fill + | @un_log_op_expr + | @co_await + | @co_yield + ; + +@bin_log_op_expr = @andlogicalexpr | @orlogicalexpr; + +@cmp_op_expr = @eq_op_expr | @rel_op_expr; + +@eq_op_expr = @eqexpr | @neexpr; + +@rel_op_expr = @gtexpr + | @ltexpr + | @geexpr + | @leexpr + | @spaceshipexpr + ; + +@bin_bitwise_op_expr = @lshiftexpr + | @rshiftexpr + | @andexpr + | @orexpr + | @xorexpr + ; + +@p_arith_op_expr = @paddexpr + | @psubexpr + | @pdiffexpr + ; + +@bin_arith_op_expr = @addexpr + | @subexpr + | @mulexpr + | @divexpr + | @remexpr + | @jmulexpr + | @jdivexpr + | @fjaddexpr + | @jfaddexpr + | @fjsubexpr + | @jfsubexpr + | @minexpr + | @maxexpr + | @p_arith_op_expr + ; + +@bin_op_expr = @bin_arith_op_expr + | @bin_bitwise_op_expr + | @cmp_op_expr + | @bin_log_op_expr + ; + +@op_expr = @un_op_expr + | @bin_op_expr + | @assign_expr + | @conditionalexpr + ; + +@assign_arith_expr = @assignaddexpr + | @assignsubexpr + | @assignmulexpr + | @assigndivexpr + | @assignremexpr + ; + +@assign_bitwise_expr = @assignandexpr + | @assignorexpr + | @assignxorexpr + | @assignlshiftexpr + | @assignrshiftexpr + | @assignpaddexpr + | @assignpsubexpr + ; + +@assign_op_expr = @assign_arith_expr | @assign_bitwise_expr + +@assign_expr = @assignexpr | @assign_op_expr + +/* + case @allocator.form of + 0 = plain + | 1 = alignment + ; +*/ + +/** + * The allocator function associated with a `new` or `new[]` expression. + * The `form` column specified whether the allocation call contains an alignment + * argument. + */ +expr_allocator( + unique int expr: @any_new_expr ref, + int func: @function ref, + int form: int ref +); + +/* + case @deallocator.form of + 0 = plain + | 1 = size + | 2 = alignment + | 3 = size_and_alignment + ; +*/ + +/** + * The deallocator function associated with a `delete`, `delete[]`, `new`, or + * `new[]` expression. For a `new` or `new[]` expression, the deallocator is the + * one used to free memory if the initialization throws an exception. + * The `form` column specifies whether the deallocation call contains a size + * argument, and alignment argument, or both. + */ +expr_deallocator( + unique int expr: @new_or_delete_expr ref, + int func: @function ref, + int form: int ref +); + +/** + * Holds if the `@conditionalexpr` is of the two operand form + * `guard ? : false`. + */ +expr_cond_two_operand( + unique int cond: @conditionalexpr ref +); + +/** + * The guard of `@conditionalexpr` `guard ? true : false` + */ +expr_cond_guard( + unique int cond: @conditionalexpr ref, + int guard: @expr ref +); + +/** + * The expression used when the guard of `@conditionalexpr` + * `guard ? true : false` holds. For the two operand form + * `guard ?: false` consider using `expr_cond_guard` instead. + */ +expr_cond_true( + unique int cond: @conditionalexpr ref, + int true: @expr ref +); + +/** + * The expression used when the guard of `@conditionalexpr` + * `guard ? true : false` does not hold. + */ +expr_cond_false( + unique int cond: @conditionalexpr ref, + int false: @expr ref +); + +/** A string representation of the value. */ +values( + unique int id: @value, + string str: string ref +); + +/** The actual text in the source code for the value, if any. */ +valuetext( + unique int id: @value ref, + string text: string ref +); + +valuebind( + int val: @value ref, + unique int expr: @expr ref +); + +fieldoffsets( + unique int id: @variable ref, + int byteoffset: int ref, + int bitoffset: int ref +); + +bitfield( + unique int id: @variable ref, + int bits: int ref, + int declared_bits: int ref +); + +/* TODO +memberprefix( + int member: @expr ref, + int prefix: @expr ref +); +*/ + +/* + kind(1) = mbrcallexpr + kind(2) = mbrptrcallexpr + kind(3) = mbrptrmbrcallexpr + kind(4) = ptrmbrptrmbrcallexpr + kind(5) = mbrreadexpr // x.y + kind(6) = mbrptrreadexpr // p->y + kind(7) = mbrptrmbrreadexpr // x.*pm + kind(8) = mbrptrmbrptrreadexpr // x->*pm + kind(9) = staticmbrreadexpr // static x.y + kind(10) = staticmbrptrreadexpr // static p->y +*/ +/* TODO +memberaccess( + int member: @expr ref, + int kind: int ref +); +*/ + +initialisers( + unique int init: @initialiser, + int var: @accessible ref, + unique int expr: @expr ref, + int location: @location_expr ref +); + +/** + * An ancestor for the expression, for cases in which we cannot + * otherwise find the expression's parent. + */ +expr_ancestor( + int exp: @expr ref, + int ancestor: @element ref +); + +exprs( + unique int id: @expr, + int kind: int ref, + int location: @location_expr ref +); + +/* + case @value.category of + 1 = prval + | 2 = xval + | 3 = lval + ; +*/ +expr_types( + int id: @expr ref, + int typeid: @type ref, + int value_category: int ref +); + +case @expr.kind of + 1 = @errorexpr +| 2 = @address_of // & AddressOfExpr +| 3 = @reference_to // ReferenceToExpr (implicit?) +| 4 = @indirect // * PointerDereferenceExpr +| 5 = @ref_indirect // ReferenceDereferenceExpr (implicit?) +// ... +| 8 = @array_to_pointer // (???) +| 9 = @vacuous_destructor_call // VacuousDestructorCall +// ... +| 11 = @assume // Microsoft +| 12 = @parexpr +| 13 = @arithnegexpr +| 14 = @unaryplusexpr +| 15 = @complementexpr +| 16 = @notexpr +| 17 = @conjugation // GNU ~ operator +| 18 = @realpartexpr // GNU __real +| 19 = @imagpartexpr // GNU __imag +| 20 = @postincrexpr +| 21 = @postdecrexpr +| 22 = @preincrexpr +| 23 = @predecrexpr +| 24 = @conditionalexpr +| 25 = @addexpr +| 26 = @subexpr +| 27 = @mulexpr +| 28 = @divexpr +| 29 = @remexpr +| 30 = @jmulexpr // C99 mul imaginary +| 31 = @jdivexpr // C99 div imaginary +| 32 = @fjaddexpr // C99 add real + imaginary +| 33 = @jfaddexpr // C99 add imaginary + real +| 34 = @fjsubexpr // C99 sub real - imaginary +| 35 = @jfsubexpr // C99 sub imaginary - real +| 36 = @paddexpr // pointer add (pointer + int or int + pointer) +| 37 = @psubexpr // pointer sub (pointer - integer) +| 38 = @pdiffexpr // difference between two pointers +| 39 = @lshiftexpr +| 40 = @rshiftexpr +| 41 = @andexpr +| 42 = @orexpr +| 43 = @xorexpr +| 44 = @eqexpr +| 45 = @neexpr +| 46 = @gtexpr +| 47 = @ltexpr +| 48 = @geexpr +| 49 = @leexpr +| 50 = @minexpr // GNU minimum +| 51 = @maxexpr // GNU maximum +| 52 = @assignexpr +| 53 = @assignaddexpr +| 54 = @assignsubexpr +| 55 = @assignmulexpr +| 56 = @assigndivexpr +| 57 = @assignremexpr +| 58 = @assignlshiftexpr +| 59 = @assignrshiftexpr +| 60 = @assignandexpr +| 61 = @assignorexpr +| 62 = @assignxorexpr +| 63 = @assignpaddexpr // assign pointer add +| 64 = @assignpsubexpr // assign pointer sub +| 65 = @andlogicalexpr +| 66 = @orlogicalexpr +| 67 = @commaexpr +| 68 = @subscriptexpr // access to member of an array, e.g., a[5] +// ... 69 @objc_subscriptexpr deprecated +// ... 70 @cmdaccess deprecated +// ... +| 73 = @virtfunptrexpr +| 74 = @callexpr +// ... 75 @msgexpr_normal deprecated +// ... 76 @msgexpr_super deprecated +// ... 77 @atselectorexpr deprecated +// ... 78 @atprotocolexpr deprecated +| 79 = @vastartexpr +| 80 = @vaargexpr +| 81 = @vaendexpr +| 82 = @vacopyexpr +// ... 83 @atencodeexpr deprecated +| 84 = @varaccess +| 85 = @thisaccess +// ... 86 @objc_box_expr deprecated +| 87 = @new_expr +| 88 = @delete_expr +| 89 = @throw_expr +| 90 = @condition_decl // a variable declared in a condition, e.g., if(int x = y > 2) +| 91 = @braced_init_list +| 92 = @type_id +| 93 = @runtime_sizeof +| 94 = @runtime_alignof +| 95 = @sizeof_pack +| 96 = @expr_stmt // GNU extension +| 97 = @routineexpr +| 98 = @type_operand // used to access a type in certain contexts (haven't found any examples yet....) +| 99 = @offsetofexpr // offsetof ::= type and field +| 100 = @hasassignexpr // __has_assign ::= type +| 101 = @hascopyexpr // __has_copy ::= type +| 102 = @hasnothrowassign // __has_nothrow_assign ::= type +| 103 = @hasnothrowconstr // __has_nothrow_constructor ::= type +| 104 = @hasnothrowcopy // __has_nothrow_copy ::= type +| 105 = @hastrivialassign // __has_trivial_assign ::= type +| 106 = @hastrivialconstr // __has_trivial_constructor ::= type +| 107 = @hastrivialcopy // __has_trivial_copy ::= type +| 108 = @hasuserdestr // __has_user_destructor ::= type +| 109 = @hasvirtualdestr // __has_virtual_destructor ::= type +| 110 = @isabstractexpr // __is_abstract ::= type +| 111 = @isbaseofexpr // __is_base_of ::= type type +| 112 = @isclassexpr // __is_class ::= type +| 113 = @isconvtoexpr // __is_convertible_to ::= type type +| 114 = @isemptyexpr // __is_empty ::= type +| 115 = @isenumexpr // __is_enum ::= type +| 116 = @ispodexpr // __is_pod ::= type +| 117 = @ispolyexpr // __is_polymorphic ::= type +| 118 = @isunionexpr // __is_union ::= type +| 119 = @typescompexpr // GNU __builtin_types_compatible ::= type type +| 120 = @intaddrexpr // EDG internal builtin, used to implement offsetof +// ... +| 122 = @hastrivialdestructor // __has_trivial_destructor ::= type +| 123 = @literal +| 124 = @uuidof +| 127 = @aggregateliteral +| 128 = @delete_array_expr +| 129 = @new_array_expr +// ... 130 @objc_array_literal deprecated +// ... 131 @objc_dictionary_literal deprecated +| 132 = @foldexpr +// ... +| 200 = @ctordirectinit +| 201 = @ctorvirtualinit +| 202 = @ctorfieldinit +| 203 = @ctordelegatinginit +| 204 = @dtordirectdestruct +| 205 = @dtorvirtualdestruct +| 206 = @dtorfielddestruct +// ... +| 210 = @static_cast +| 211 = @reinterpret_cast +| 212 = @const_cast +| 213 = @dynamic_cast +| 214 = @c_style_cast +| 215 = @lambdaexpr +| 216 = @param_ref +| 217 = @noopexpr +// ... +| 294 = @istriviallyconstructibleexpr +| 295 = @isdestructibleexpr +| 296 = @isnothrowdestructibleexpr +| 297 = @istriviallydestructibleexpr +| 298 = @istriviallyassignableexpr +| 299 = @isnothrowassignableexpr +| 300 = @istrivialexpr +| 301 = @isstandardlayoutexpr +| 302 = @istriviallycopyableexpr +| 303 = @isliteraltypeexpr +| 304 = @hastrivialmoveconstructorexpr +| 305 = @hastrivialmoveassignexpr +| 306 = @hasnothrowmoveassignexpr +| 307 = @isconstructibleexpr +| 308 = @isnothrowconstructibleexpr +| 309 = @hasfinalizerexpr +| 310 = @isdelegateexpr +| 311 = @isinterfaceclassexpr +| 312 = @isrefarrayexpr +| 313 = @isrefclassexpr +| 314 = @issealedexpr +| 315 = @issimplevalueclassexpr +| 316 = @isvalueclassexpr +| 317 = @isfinalexpr +| 319 = @noexceptexpr +| 320 = @builtinshufflevector +| 321 = @builtinchooseexpr +| 322 = @builtinaddressof +| 323 = @vec_fill +| 324 = @builtinconvertvector +| 325 = @builtincomplex +| 326 = @spaceshipexpr +| 327 = @co_await +| 328 = @co_yield +| 329 = @temp_init +; + +@var_args_expr = @vastartexpr + | @vaendexpr + | @vaargexpr + | @vacopyexpr + ; + +@builtin_op = @var_args_expr + | @noopexpr + | @offsetofexpr + | @intaddrexpr + | @hasassignexpr + | @hascopyexpr + | @hasnothrowassign + | @hasnothrowconstr + | @hasnothrowcopy + | @hastrivialassign + | @hastrivialconstr + | @hastrivialcopy + | @hastrivialdestructor + | @hasuserdestr + | @hasvirtualdestr + | @isabstractexpr + | @isbaseofexpr + | @isclassexpr + | @isconvtoexpr + | @isemptyexpr + | @isenumexpr + | @ispodexpr + | @ispolyexpr + | @isunionexpr + | @typescompexpr + | @builtinshufflevector + | @builtinconvertvector + | @builtinaddressof + | @istriviallyconstructibleexpr + | @isdestructibleexpr + | @isnothrowdestructibleexpr + | @istriviallydestructibleexpr + | @istriviallyassignableexpr + | @isnothrowassignableexpr + | @isstandardlayoutexpr + | @istriviallycopyableexpr + | @isliteraltypeexpr + | @hastrivialmoveconstructorexpr + | @hastrivialmoveassignexpr + | @hasnothrowmoveassignexpr + | @isconstructibleexpr + | @isnothrowconstructibleexpr + | @hasfinalizerexpr + | @isdelegateexpr + | @isinterfaceclassexpr + | @isrefarrayexpr + | @isrefclassexpr + | @issealedexpr + | @issimplevalueclassexpr + | @isvalueclassexpr + | @isfinalexpr + | @builtinchooseexpr + | @builtincomplex + ; + +new_allocated_type( + unique int expr: @new_expr ref, + int type_id: @type ref +); + +new_array_allocated_type( + unique int expr: @new_array_expr ref, + int type_id: @type ref +); + +/** + * The field being initialized by an initializer expression within an aggregate + * initializer for a class/struct/union. + */ +#keyset[aggregate, field] +aggregate_field_init( + int aggregate: @aggregateliteral ref, + int initializer: @expr ref, + int field: @membervariable ref +); + +/** + * The index of the element being initialized by an initializer expression + * within an aggregate initializer for an array. + */ +#keyset[aggregate, element_index] +aggregate_array_init( + int aggregate: @aggregateliteral ref, + int initializer: @expr ref, + int element_index: int ref +); + +@ctorinit = @ctordirectinit + | @ctorvirtualinit + | @ctorfieldinit + | @ctordelegatinginit; +@dtordestruct = @dtordirectdestruct + | @dtorvirtualdestruct + | @dtorfielddestruct; + + +condition_decl_bind( + unique int expr: @condition_decl ref, + unique int decl: @declaration ref +); + +typeid_bind( + unique int expr: @type_id ref, + int type_id: @type ref +); + +uuidof_bind( + unique int expr: @uuidof ref, + int type_id: @type ref +); + +@runtime_sizeof_or_alignof = @runtime_sizeof | @runtime_alignof; + +sizeof_bind( + unique int expr: @runtime_sizeof_or_alignof ref, + int type_id: @type ref +); + +code_block( + unique int block: @literal ref, + unique int routine: @function ref +); + +lambdas( + unique int expr: @lambdaexpr ref, + string default_capture: string ref, + boolean has_explicit_return_type: boolean ref +); + +lambda_capture( + unique int id: @lambdacapture, + int lambda: @lambdaexpr ref, + int index: int ref, + int field: @membervariable ref, + boolean captured_by_reference: boolean ref, + boolean is_implicit: boolean ref, + int location: @location_default ref +); + +@funbindexpr = @routineexpr + | @new_expr + | @delete_expr + | @delete_array_expr + | @ctordirectinit + | @ctorvirtualinit + | @ctordelegatinginit + | @dtordirectdestruct + | @dtorvirtualdestruct; + +@varbindexpr = @varaccess | @ctorfieldinit | @dtorfielddestruct; +@addressable = @function | @variable ; +@accessible = @addressable | @enumconstant ; + +@access = @varaccess | @routineexpr ; + +fold( + int expr: @foldexpr ref, + string operator: string ref, + boolean is_left_fold: boolean ref +); + +stmts( + unique int id: @stmt, + int kind: int ref, + int location: @location_stmt ref +); + +case @stmt.kind of + 1 = @stmt_expr +| 2 = @stmt_if +| 3 = @stmt_while +| 4 = @stmt_goto +| 5 = @stmt_label +| 6 = @stmt_return +| 7 = @stmt_block +| 8 = @stmt_end_test_while // do { ... } while ( ... ) +| 9 = @stmt_for +| 10 = @stmt_switch_case +| 11 = @stmt_switch +| 13 = @stmt_asm // "asm" statement or the body of an asm function +| 15 = @stmt_try_block +| 16 = @stmt_microsoft_try // Microsoft +| 17 = @stmt_decl +| 18 = @stmt_set_vla_size // C99 +| 19 = @stmt_vla_decl // C99 +| 25 = @stmt_assigned_goto // GNU +| 26 = @stmt_empty +| 27 = @stmt_continue +| 28 = @stmt_break +| 29 = @stmt_range_based_for // C++11 +// ... 30 @stmt_at_autoreleasepool_block deprecated +// ... 31 @stmt_objc_for_in deprecated +// ... 32 @stmt_at_synchronized deprecated +| 33 = @stmt_handler +// ... 34 @stmt_finally_end deprecated +| 35 = @stmt_constexpr_if +| 37 = @stmt_co_return +; + +type_vla( + int type_id: @type ref, + int decl: @stmt_vla_decl ref +); + +variable_vla( + int var: @variable ref, + int decl: @stmt_vla_decl ref +); + +if_then( + unique int if_stmt: @stmt_if ref, + int then_id: @stmt ref +); + +if_else( + unique int if_stmt: @stmt_if ref, + int else_id: @stmt ref +); + +constexpr_if_then( + unique int constexpr_if_stmt: @stmt_constexpr_if ref, + int then_id: @stmt ref +); + +constexpr_if_else( + unique int constexpr_if_stmt: @stmt_constexpr_if ref, + int else_id: @stmt ref +); + +while_body( + unique int while_stmt: @stmt_while ref, + int body_id: @stmt ref +); + +do_body( + unique int do_stmt: @stmt_end_test_while ref, + int body_id: @stmt ref +); + +#keyset[switch_stmt, index] +switch_case( + int switch_stmt: @stmt_switch ref, + int index: int ref, + int case_id: @stmt_switch_case ref +); + +switch_body( + unique int switch_stmt: @stmt_switch ref, + int body_id: @stmt ref +); + +for_initialization( + unique int for_stmt: @stmt_for ref, + int init_id: @stmt ref +); + +for_condition( + unique int for_stmt: @stmt_for ref, + int condition_id: @expr ref +); + +for_update( + unique int for_stmt: @stmt_for ref, + int update_id: @expr ref +); + +for_body( + unique int for_stmt: @stmt_for ref, + int body_id: @stmt ref +); + +@stmtparent = @stmt | @expr_stmt ; +stmtparents( + unique int id: @stmt ref, + int index: int ref, + int parent: @stmtparent ref +); + +ishandler(unique int block: @stmt_block ref); + +@cfgnode = @stmt | @expr | @function | @initialiser ; + +stmt_decl_bind( + int stmt: @stmt_decl ref, + int num: int ref, + int decl: @declaration ref +); + +stmt_decl_entry_bind( + int stmt: @stmt_decl ref, + int num: int ref, + int decl_entry: @element ref +); + +@functionorblock = @function | @stmt_block; + +blockscope( + unique int block: @stmt_block ref, + int enclosing: @functionorblock ref +); + +@jump = @stmt_goto | @stmt_break | @stmt_continue; + +@jumporlabel = @jump | @stmt_label | @literal; + +jumpinfo( + unique int id: @jumporlabel ref, + string str: string ref, + int target: @stmt ref +); + +preprocdirects( + unique int id: @preprocdirect, + int kind: int ref, + int location: @location_default ref +); +case @preprocdirect.kind of + 0 = @ppd_if +| 1 = @ppd_ifdef +| 2 = @ppd_ifndef +| 3 = @ppd_elif +| 4 = @ppd_else +| 5 = @ppd_endif +| 6 = @ppd_plain_include +| 7 = @ppd_define +| 8 = @ppd_undef +| 9 = @ppd_line +| 10 = @ppd_error +| 11 = @ppd_pragma +| 12 = @ppd_objc_import +| 13 = @ppd_include_next +| 18 = @ppd_warning +; + +@ppd_include = @ppd_plain_include | @ppd_objc_import | @ppd_include_next; + +@ppd_branch = @ppd_if | @ppd_ifdef | @ppd_ifndef | @ppd_elif; + +preprocpair( + int begin : @ppd_branch ref, + int elseelifend : @preprocdirect ref +); + +preproctrue(int branch : @ppd_branch ref); +preprocfalse(int branch : @ppd_branch ref); + +preproctext( + unique int id: @preprocdirect ref, + string head: string ref, + string body: string ref +); + +includes( + unique int id: @ppd_include ref, + int included: @file ref +); + +link_targets( + unique int id: @link_target, + int binary: @file ref +); + +link_parent( + int element : @element ref, + int link_target : @link_target ref +); + +/* XML Files */ + +xmlEncoding(unique int id: @file ref, string encoding: string ref); + +xmlDTDs( + unique int id: @xmldtd, + string root: string ref, + string publicId: string ref, + string systemId: string ref, + int fileid: @file ref +); + +xmlElements( + unique int id: @xmlelement, + string name: string ref, + int parentid: @xmlparent ref, + int idx: int ref, + int fileid: @file ref +); + +xmlAttrs( + unique int id: @xmlattribute, + int elementid: @xmlelement ref, + string name: string ref, + string value: string ref, + int idx: int ref, + int fileid: @file ref +); + +xmlNs( + int id: @xmlnamespace, + string prefixName: string ref, + string URI: string ref, + int fileid: @file ref +); + +xmlHasNs( + int elementId: @xmlnamespaceable ref, + int nsId: @xmlnamespace ref, + int fileid: @file ref +); + +xmlComments( + unique int id: @xmlcomment, + string text: string ref, + int parentid: @xmlparent ref, + int fileid: @file ref +); + +xmlChars( + unique int id: @xmlcharacters, + string text: string ref, + int parentid: @xmlparent ref, + int idx: int ref, + int isCDATA: int ref, + int fileid: @file ref +); + +@xmlparent = @file | @xmlelement; +@xmlnamespaceable = @xmlelement | @xmlattribute; + +xmllocations( + int xmlElement: @xmllocatable ref, + int location: @location_default ref +); + +@xmllocatable = @xmlcharacters + | @xmlelement + | @xmlcomment + | @xmlattribute + | @xmldtd + | @file + | @xmlnamespace; diff --git a/cpp/downgrades/cf72c8898d19eb1b3374432cf79d8276cb07ad43/stmtparents.ql b/cpp/downgrades/cf72c8898d19eb1b3374432cf79d8276cb07ad43/stmtparents.ql new file mode 100644 index 00000000000..85465fdf8a3 --- /dev/null +++ b/cpp/downgrades/cf72c8898d19eb1b3374432cf79d8276cb07ad43/stmtparents.ql @@ -0,0 +1,22 @@ +class Element extends @element { + string toString() { none() } +} + +class Stmt extends @stmt { + string toString() { none() } +} + +predicate isStmtWithInitializer(Stmt stmt) { + exists(int kind | stmts(stmt, kind, _) | kind = 2 or kind = 11 or kind = 35) +} + +from Stmt child, int index, int index_new, Element parent +where + stmtparents(child, index, parent) and + ( + not isStmtWithInitializer(parent) + or + index > 0 + ) and + if isStmtWithInitializer(parent) then index_new = index - 1 else index_new = index +select child, index_new, parent diff --git a/cpp/downgrades/cf72c8898d19eb1b3374432cf79d8276cb07ad43/upgrade.properties b/cpp/downgrades/cf72c8898d19eb1b3374432cf79d8276cb07ad43/upgrade.properties new file mode 100644 index 00000000000..784726b09de --- /dev/null +++ b/cpp/downgrades/cf72c8898d19eb1b3374432cf79d8276cb07ad43/upgrade.properties @@ -0,0 +1,6 @@ +description: Support C++17 if and switch initializers +compatibility: partial +if_initialization.rel: delete +switch_initialization.rel: delete +exprparents.rel: run exprparents.qlo +stmtparents.rel: run stmtparents.qlo diff --git a/cpp/ql/lib/upgrades/e9a518baf14f4322ac243578a8e1391386ff030f/exprparents.ql b/cpp/ql/lib/upgrades/e9a518baf14f4322ac243578a8e1391386ff030f/exprparents.ql new file mode 100644 index 00000000000..ca800c8e311 --- /dev/null +++ b/cpp/ql/lib/upgrades/e9a518baf14f4322ac243578a8e1391386ff030f/exprparents.ql @@ -0,0 +1,21 @@ +class Element extends @element { + string toString() { none() } +} + +class Expr extends @expr { + string toString() { none() } +} + +class Stmt extends @stmt { + string toString() { none() } +} + +predicate isStmtWithInitializer(Stmt stmt) { + exists(int kind | stmts(stmt, kind, _) | kind = 2 or kind = 11 or kind = 35) +} + +from Expr child, int index, int index_new, Element parent +where + exprparents(child, index, parent) and + if isStmtWithInitializer(parent) then index_new = index + 1 else index_new = index +select child, index_new, parent diff --git a/cpp/ql/lib/upgrades/e9a518baf14f4322ac243578a8e1391386ff030f/old.dbscheme b/cpp/ql/lib/upgrades/e9a518baf14f4322ac243578a8e1391386ff030f/old.dbscheme new file mode 100644 index 00000000000..e9a518baf14 --- /dev/null +++ b/cpp/ql/lib/upgrades/e9a518baf14f4322ac243578a8e1391386ff030f/old.dbscheme @@ -0,0 +1,2096 @@ + +/** + * An invocation of the compiler. Note that more than one file may be + * compiled per invocation. For example, this command compiles three + * source files: + * + * gcc -c f1.c f2.c f3.c + * + * The `id` simply identifies the invocation, while `cwd` is the working + * directory from which the compiler was invoked. + */ +compilations( + /** + * An invocation of the compiler. Note that more than one file may + * be compiled per invocation. For example, this command compiles + * three source files: + * + * gcc -c f1.c f2.c f3.c + */ + unique int id : @compilation, + string cwd : string ref +); + +/** + * The arguments that were passed to the extractor for a compiler + * invocation. If `id` is for the compiler invocation + * + * gcc -c f1.c f2.c f3.c + * + * then typically there will be rows for + * + * num | arg + * --- | --- + * 0 | *path to extractor* + * 1 | `--mimic` + * 2 | `/usr/bin/gcc` + * 3 | `-c` + * 4 | f1.c + * 5 | f2.c + * 6 | f3.c + */ +#keyset[id, num] +compilation_args( + int id : @compilation ref, + int num : int ref, + string arg : string ref +); + +/** + * The source files that are compiled by a compiler invocation. + * If `id` is for the compiler invocation + * + * gcc -c f1.c f2.c f3.c + * + * then there will be rows for + * + * num | arg + * --- | --- + * 0 | f1.c + * 1 | f2.c + * 2 | f3.c + * + * Note that even if those files `#include` headers, those headers + * do not appear as rows. + */ +#keyset[id, num] +compilation_compiling_files( + int id : @compilation ref, + int num : int ref, + int file : @file ref +); + +/** + * The time taken by the extractor for a compiler invocation. + * + * For each file `num`, there will be rows for + * + * kind | seconds + * ---- | --- + * 1 | CPU seconds used by the extractor frontend + * 2 | Elapsed seconds during the extractor frontend + * 3 | CPU seconds used by the extractor backend + * 4 | Elapsed seconds during the extractor backend + */ +#keyset[id, num, kind] +compilation_time( + int id : @compilation ref, + int num : int ref, + /* kind: + 1 = frontend_cpu_seconds + 2 = frontend_elapsed_seconds + 3 = extractor_cpu_seconds + 4 = extractor_elapsed_seconds + */ + int kind : int ref, + float seconds : float ref +); + +/** + * An error or warning generated by the extractor. + * The diagnostic message `diagnostic` was generated during compiler + * invocation `compilation`, and is the `file_number_diagnostic_number`th + * message generated while extracting the `file_number`th file of that + * invocation. + */ +#keyset[compilation, file_number, file_number_diagnostic_number] +diagnostic_for( + int diagnostic : @diagnostic ref, + int compilation : @compilation ref, + int file_number : int ref, + int file_number_diagnostic_number : int ref +); + +/** + * If extraction was successful, then `cpu_seconds` and + * `elapsed_seconds` are the CPU time and elapsed time (respectively) + * that extraction took for compiler invocation `id`. + */ +compilation_finished( + unique int id : @compilation ref, + float cpu_seconds : float ref, + float elapsed_seconds : float ref +); + + +/** + * External data, loaded from CSV files during snapshot creation. See + * [Tutorial: Incorporating external data](https://help.semmle.com/wiki/display/SD/Tutorial%3A+Incorporating+external+data) + * for more information. + */ +externalData( + int id : @externalDataElement, + string path : string ref, + int column: int ref, + string value : string ref +); + +/** + * The source location of the snapshot. + */ +sourceLocationPrefix(string prefix : string ref); + +/** + * Information about packages that provide code used during compilation. + * The `id` is just a unique identifier. + * The `namespace` is typically the name of the package manager that + * provided the package (e.g. "dpkg" or "yum"). + * The `package_name` is the name of the package, and `version` is its + * version (as a string). + */ +external_packages( + unique int id: @external_package, + string namespace : string ref, + string package_name : string ref, + string version : string ref +); + +/** + * Holds if File `fileid` was provided by package `package`. + */ +header_to_external_package( + int fileid : @file ref, + int package : @external_package ref +); + +/* + * Version history + */ + +svnentries( + unique int id : @svnentry, + string revision : string ref, + string author : string ref, + date revisionDate : date ref, + int changeSize : int ref +) + +svnaffectedfiles( + int id : @svnentry ref, + int file : @file ref, + string action : string ref +) + +svnentrymsg( + unique int id : @svnentry ref, + string message : string ref +) + +svnchurn( + int commit : @svnentry ref, + int file : @file ref, + int addedLines : int ref, + int deletedLines : int ref +) + +/* + * C++ dbscheme + */ + +@location = @location_stmt | @location_expr | @location_default ; + +/** + * The location of an element that is not an expression or a statement. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `file`. + * For more information, see + * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/). + */ +locations_default( + /** The location of an element that is not an expression or a statement. */ + unique int id: @location_default, + int container: @container ref, + int startLine: int ref, + int startColumn: int ref, + int endLine: int ref, + int endColumn: int ref +); + +/** + * The location of a statement. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `file`. + * For more information, see + * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/). + */ +locations_stmt( + /** The location of a statement. */ + unique int id: @location_stmt, + int container: @container ref, + int startLine: int ref, + int startColumn: int ref, + int endLine: int ref, + int endColumn: int ref +); + +/** + * The location of an expression. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `file`. + * For more information, see + * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/). + */ +locations_expr( + /** The location of an expression. */ + unique int id: @location_expr, + int container: @container ref, + int startLine: int ref, + int startColumn: int ref, + int endLine: int ref, + int endColumn: int ref +); + +/** An element for which line-count information is available. */ +@sourceline = @file | @function | @variable | @enumconstant | @xmllocatable; + +numlines( + int element_id: @sourceline ref, + int num_lines: int ref, + int num_code: int ref, + int num_comment: int ref +); + +diagnostics( + unique int id: @diagnostic, + int severity: int ref, + string error_tag: string ref, + string error_message: string ref, + string full_error_message: string ref, + int location: @location_default ref +); + +files( + unique int id: @file, + string name: string ref +); + +folders( + unique int id: @folder, + string name: string ref +); + +@container = @folder | @file + +containerparent( + int parent: @container ref, + unique int child: @container ref +); + +fileannotations( + int id: @file ref, + int kind: int ref, + string name: string ref, + string value: string ref +); + +inmacroexpansion( + int id: @element ref, + int inv: @macroinvocation ref +); + +affectedbymacroexpansion( + int id: @element ref, + int inv: @macroinvocation ref +); + +/* + case @macroinvocations.kind of + 1 = macro expansion + | 2 = other macro reference + ; +*/ +macroinvocations( + unique int id: @macroinvocation, + int macro_id: @ppd_define ref, + int location: @location_default ref, + int kind: int ref +); + +macroparent( + unique int id: @macroinvocation ref, + int parent_id: @macroinvocation ref +); + +// a macroinvocation may be part of another location +// the way to find a constant expression that uses a macro +// is thus to find a constant expression that has a location +// to which a macro invocation is bound +macrolocationbind( + int id: @macroinvocation ref, + int location: @location ref +); + +#keyset[invocation, argument_index] +macro_argument_unexpanded( + int invocation: @macroinvocation ref, + int argument_index: int ref, + string text: string ref +); + +#keyset[invocation, argument_index] +macro_argument_expanded( + int invocation: @macroinvocation ref, + int argument_index: int ref, + string text: string ref +); + +/* + case @function.kind of + 1 = normal + | 2 = constructor + | 3 = destructor + | 4 = conversion + | 5 = operator + | 6 = builtin // GCC built-in functions, e.g. __builtin___memcpy_chk + ; +*/ +functions( + unique int id: @function, + string name: string ref, + int kind: int ref +); + +function_entry_point(int id: @function ref, unique int entry_point: @stmt ref); + +function_return_type(int id: @function ref, int return_type: @type ref); + +/** If `function` is a coroutine, then this gives the + std::experimental::resumable_traits instance associated with it, + and the variables representing the `handle` and `promise` for it. */ +coroutine( + unique int function: @function ref, + int traits: @type ref, + int handle: @variable ref, + int promise: @variable ref +); + +/** The `new` function used for allocating the coroutine state, if any. */ +coroutine_new( + unique int function: @function ref, + int new: @function ref +); + +/** The `delete` function used for deallocating the coroutine state, if any. */ +coroutine_delete( + unique int function: @function ref, + int delete: @function ref +); + +purefunctions(unique int id: @function ref); + +function_deleted(unique int id: @function ref); + +function_defaulted(unique int id: @function ref); + +member_function_this_type(unique int id: @function ref, int this_type: @type ref); + +#keyset[id, type_id] +fun_decls( + int id: @fun_decl, + int function: @function ref, + int type_id: @type ref, + string name: string ref, + int location: @location_default ref +); +fun_def(unique int id: @fun_decl ref); +fun_specialized(unique int id: @fun_decl ref); +fun_implicit(unique int id: @fun_decl ref); +fun_decl_specifiers( + int id: @fun_decl ref, + string name: string ref +) +#keyset[fun_decl, index] +fun_decl_throws( + int fun_decl: @fun_decl ref, + int index: int ref, + int type_id: @type ref +); +/* an empty throw specification is different from none */ +fun_decl_empty_throws(unique int fun_decl: @fun_decl ref); +fun_decl_noexcept( + int fun_decl: @fun_decl ref, + int constant: @expr ref +); +fun_decl_empty_noexcept(int fun_decl: @fun_decl ref); +fun_decl_typedef_type( + unique int fun_decl: @fun_decl ref, + int typedeftype_id: @usertype ref +); + +param_decl_bind( + unique int id: @var_decl ref, + int index: int ref, + int fun_decl: @fun_decl ref +); + +#keyset[id, type_id] +var_decls( + int id: @var_decl, + int variable: @variable ref, + int type_id: @type ref, + string name: string ref, + int location: @location_default ref +); +var_def(unique int id: @var_decl ref); +var_decl_specifiers( + int id: @var_decl ref, + string name: string ref +) +is_structured_binding(unique int id: @variable ref); + +type_decls( + unique int id: @type_decl, + int type_id: @type ref, + int location: @location_default ref +); +type_def(unique int id: @type_decl ref); +type_decl_top( + unique int type_decl: @type_decl ref +); + +namespace_decls( + unique int id: @namespace_decl, + int namespace_id: @namespace ref, + int location: @location_default ref, + int bodylocation: @location_default ref +); + +usings( + unique int id: @using, + int element_id: @element ref, + int location: @location_default ref +); + +/** The element which contains the `using` declaration. */ +using_container( + int parent: @element ref, + int child: @using ref +); + +static_asserts( + unique int id: @static_assert, + int condition : @expr ref, + string message : string ref, + int location: @location_default ref, + int enclosing : @element ref +); + +// each function has an ordered list of parameters +#keyset[id, type_id] +#keyset[function, index, type_id] +params( + int id: @parameter, + int function: @functionorblock ref, + int index: int ref, + int type_id: @type ref +); + +overrides(int new: @function ref, int old: @function ref); + +#keyset[id, type_id] +membervariables( + int id: @membervariable, + int type_id: @type ref, + string name: string ref +); + +#keyset[id, type_id] +globalvariables( + int id: @globalvariable, + int type_id: @type ref, + string name: string ref +); + +#keyset[id, type_id] +localvariables( + int id: @localvariable, + int type_id: @type ref, + string name: string ref +); + +autoderivation( + unique int var: @variable ref, + int derivation_type: @type ref +); + +enumconstants( + unique int id: @enumconstant, + int parent: @usertype ref, + int index: int ref, + int type_id: @type ref, + string name: string ref, + int location: @location_default ref +); + +@variable = @localscopevariable | @globalvariable | @membervariable; + +@localscopevariable = @localvariable | @parameter; + +/* + Built-in types are the fundamental types, e.g., integral, floating, and void. + + case @builtintype.kind of + 1 = error + | 2 = unknown + | 3 = void + | 4 = boolean + | 5 = char + | 6 = unsigned_char + | 7 = signed_char + | 8 = short + | 9 = unsigned_short + | 10 = signed_short + | 11 = int + | 12 = unsigned_int + | 13 = signed_int + | 14 = long + | 15 = unsigned_long + | 16 = signed_long + | 17 = long_long + | 18 = unsigned_long_long + | 19 = signed_long_long + | 20 = __int8 // Microsoft-specific + | 21 = __int16 // Microsoft-specific + | 22 = __int32 // Microsoft-specific + | 23 = __int64 // Microsoft-specific + | 24 = float + | 25 = double + | 26 = long_double + | 27 = _Complex_float // C99-specific + | 28 = _Complex_double // C99-specific + | 29 = _Complex_long double // C99-specific + | 30 = _Imaginary_float // C99-specific + | 31 = _Imaginary_double // C99-specific + | 32 = _Imaginary_long_double // C99-specific + | 33 = wchar_t // Microsoft-specific + | 34 = decltype_nullptr // C++11 + | 35 = __int128 + | 36 = unsigned___int128 + | 37 = signed___int128 + | 38 = __float128 + | 39 = _Complex___float128 + | 40 = _Decimal32 + | 41 = _Decimal64 + | 42 = _Decimal128 + | 43 = char16_t + | 44 = char32_t + | 45 = _Float32 + | 46 = _Float32x + | 47 = _Float64 + | 48 = _Float64x + | 49 = _Float128 + | 50 = _Float128x + | 51 = char8_t + ; +*/ +builtintypes( + unique int id: @builtintype, + string name: string ref, + int kind: int ref, + int size: int ref, + int sign: int ref, + int alignment: int ref +); + +/* + Derived types are types that are directly derived from existing types and + point to, refer to, transform type data to return a new type. + + case @derivedtype.kind of + 1 = pointer + | 2 = reference + | 3 = type_with_specifiers + | 4 = array + | 5 = gnu_vector + | 6 = routineptr + | 7 = routinereference + | 8 = rvalue_reference // C++11 +// ... 9 type_conforming_to_protocols deprecated + | 10 = block + ; +*/ +derivedtypes( + unique int id: @derivedtype, + string name: string ref, + int kind: int ref, + int type_id: @type ref +); + +pointerishsize(unique int id: @derivedtype ref, + int size: int ref, + int alignment: int ref); + +arraysizes( + unique int id: @derivedtype ref, + int num_elements: int ref, + int bytesize: int ref, + int alignment: int ref +); + +typedefbase( + unique int id: @usertype ref, + int type_id: @type ref +); + +/** + * An instance of the C++11 `decltype` operator. For example: + * ``` + * int a; + * decltype(1+a) b; + * ``` + * Here `expr` is `1+a`. + * + * Sometimes an additional pair of parentheses around the expression + * would change the semantics of this decltype, e.g. + * ``` + * struct A { double x; }; + * const A* a = new A(); + * decltype( a->x ); // type is double + * decltype((a->x)); // type is const double& + * ``` + * (Please consult the C++11 standard for more details). + * `parentheses_would_change_meaning` is `true` iff that is the case. + */ +#keyset[id, expr] +decltypes( + int id: @decltype, + int expr: @expr ref, + int base_type: @type ref, + boolean parentheses_would_change_meaning: boolean ref +); + +/* + case @usertype.kind of + 1 = struct + | 2 = class + | 3 = union + | 4 = enum + | 5 = typedef // classic C: typedef typedef type name + | 6 = template + | 7 = template_parameter + | 8 = template_template_parameter + | 9 = proxy_class // a proxy class associated with a template parameter +// ... 10 objc_class deprecated +// ... 11 objc_protocol deprecated +// ... 12 objc_category deprecated + | 13 = scoped_enum + | 14 = using_alias // a using name = type style typedef + ; +*/ +usertypes( + unique int id: @usertype, + string name: string ref, + int kind: int ref +); + +usertypesize( + unique int id: @usertype ref, + int size: int ref, + int alignment: int ref +); + +usertype_final(unique int id: @usertype ref); + +usertype_uuid( + unique int id: @usertype ref, + string uuid: string ref +); + +mangled_name( + unique int id: @declaration ref, + int mangled_name : @mangledname +); + +is_pod_class(unique int id: @usertype ref); +is_standard_layout_class(unique int id: @usertype ref); + +is_complete(unique int id: @usertype ref); + +is_class_template(unique int id: @usertype ref); +class_instantiation( + int to: @usertype ref, + int from: @usertype ref +); +class_template_argument( + int type_id: @usertype ref, + int index: int ref, + int arg_type: @type ref +); +class_template_argument_value( + int type_id: @usertype ref, + int index: int ref, + int arg_value: @expr ref +); + +is_proxy_class_for( + unique int id: @usertype ref, + unique int templ_param_id: @usertype ref +); + +type_mentions( + unique int id: @type_mention, + int type_id: @type ref, + int location: @location ref, + // a_symbol_reference_kind from the EDG frontend. See symbol_ref.h there. + int kind: int ref +); + +is_function_template(unique int id: @function ref); +function_instantiation( + unique int to: @function ref, + int from: @function ref +); +function_template_argument( + int function_id: @function ref, + int index: int ref, + int arg_type: @type ref +); +function_template_argument_value( + int function_id: @function ref, + int index: int ref, + int arg_value: @expr ref +); + +is_variable_template(unique int id: @variable ref); +variable_instantiation( + unique int to: @variable ref, + int from: @variable ref +); +variable_template_argument( + int variable_id: @variable ref, + int index: int ref, + int arg_type: @type ref +); +variable_template_argument_value( + int variable_id: @variable ref, + int index: int ref, + int arg_value: @expr ref +); + +/* + Fixed point types + precision(1) = short, precision(2) = default, precision(3) = long + is_unsigned(1) = unsigned is_unsigned(2) = signed + is_fract_type(1) = declared with _Fract + saturating(1) = declared with _Sat +*/ +/* TODO +fixedpointtypes( + unique int id: @fixedpointtype, + int precision: int ref, + int is_unsigned: int ref, + int is_fract_type: int ref, + int saturating: int ref); +*/ + +routinetypes( + unique int id: @routinetype, + int return_type: @type ref +); + +routinetypeargs( + int routine: @routinetype ref, + int index: int ref, + int type_id: @type ref +); + +ptrtomembers( + unique int id: @ptrtomember, + int type_id: @type ref, + int class_id: @type ref +); + +/* + specifiers for types, functions, and variables + + "public", + "protected", + "private", + + "const", + "volatile", + "static", + + "pure", + "virtual", + "sealed", // Microsoft + "__interface", // Microsoft + "inline", + "explicit", + + "near", // near far extension + "far", // near far extension + "__ptr32", // Microsoft + "__ptr64", // Microsoft + "__sptr", // Microsoft + "__uptr", // Microsoft + "dllimport", // Microsoft + "dllexport", // Microsoft + "thread", // Microsoft + "naked", // Microsoft + "microsoft_inline", // Microsoft + "forceinline", // Microsoft + "selectany", // Microsoft + "nothrow", // Microsoft + "novtable", // Microsoft + "noreturn", // Microsoft + "noinline", // Microsoft + "noalias", // Microsoft + "restrict", // Microsoft +*/ + +specifiers( + unique int id: @specifier, + unique string str: string ref +); + +typespecifiers( + int type_id: @type ref, + int spec_id: @specifier ref +); + +funspecifiers( + int func_id: @function ref, + int spec_id: @specifier ref +); + +varspecifiers( + int var_id: @accessible ref, + int spec_id: @specifier ref +); + +attributes( + unique int id: @attribute, + int kind: int ref, + string name: string ref, + string name_space: string ref, + int location: @location_default ref +); + +case @attribute.kind of + 0 = @gnuattribute +| 1 = @stdattribute +| 2 = @declspec +| 3 = @msattribute +| 4 = @alignas +// ... 5 @objc_propertyattribute deprecated +; + +attribute_args( + unique int id: @attribute_arg, + int kind: int ref, + int attribute: @attribute ref, + int index: int ref, + int location: @location_default ref +); + +case @attribute_arg.kind of + 0 = @attribute_arg_empty +| 1 = @attribute_arg_token +| 2 = @attribute_arg_constant +| 3 = @attribute_arg_type +; + +attribute_arg_value( + unique int arg: @attribute_arg ref, + string value: string ref +); +attribute_arg_type( + unique int arg: @attribute_arg ref, + int type_id: @type ref +); +attribute_arg_name( + unique int arg: @attribute_arg ref, + string name: string ref +); + +typeattributes( + int type_id: @type ref, + int spec_id: @attribute ref +); + +funcattributes( + int func_id: @function ref, + int spec_id: @attribute ref +); + +varattributes( + int var_id: @accessible ref, + int spec_id: @attribute ref +); + +stmtattributes( + int stmt_id: @stmt ref, + int spec_id: @attribute ref +); + +@type = @builtintype + | @derivedtype + | @usertype + /* TODO | @fixedpointtype */ + | @routinetype + | @ptrtomember + | @decltype; + +unspecifiedtype( + unique int type_id: @type ref, + int unspecified_type_id: @type ref +); + +member( + int parent: @type ref, + int index: int ref, + int child: @member ref +); + +@enclosingfunction_child = @usertype | @variable | @namespace + +enclosingfunction( + unique int child: @enclosingfunction_child ref, + int parent: @function ref +); + +derivations( + unique int derivation: @derivation, + int sub: @type ref, + int index: int ref, + int super: @type ref, + int location: @location_default ref +); + +derspecifiers( + int der_id: @derivation ref, + int spec_id: @specifier ref +); + +/** + * Contains the byte offset of the base class subobject within the derived + * class. Only holds for non-virtual base classes, but see table + * `virtual_base_offsets` for offsets of virtual base class subobjects. + */ +direct_base_offsets( + unique int der_id: @derivation ref, + int offset: int ref +); + +/** + * Contains the byte offset of the virtual base class subobject for class + * `super` within a most-derived object of class `sub`. `super` can be either a + * direct or indirect base class. + */ +#keyset[sub, super] +virtual_base_offsets( + int sub: @usertype ref, + int super: @usertype ref, + int offset: int ref +); + +frienddecls( + unique int id: @frienddecl, + int type_id: @type ref, + int decl_id: @declaration ref, + int location: @location_default ref +); + +@declaredtype = @usertype ; + +@declaration = @function + | @declaredtype + | @variable + | @enumconstant + | @frienddecl; + +@member = @membervariable + | @function + | @declaredtype + | @enumconstant; + +@locatable = @diagnostic + | @declaration + | @ppd_include + | @ppd_define + | @macroinvocation + /*| @funcall*/ + | @xmllocatable + | @attribute + | @attribute_arg; + +@namedscope = @namespace | @usertype; + +@element = @locatable + | @file + | @folder + | @specifier + | @type + | @expr + | @namespace + | @initialiser + | @stmt + | @derivation + | @comment + | @preprocdirect + | @fun_decl + | @var_decl + | @type_decl + | @namespace_decl + | @using + | @namequalifier + | @specialnamequalifyingelement + | @static_assert + | @type_mention + | @lambdacapture; + +@exprparent = @element; + +comments( + unique int id: @comment, + string contents: string ref, + int location: @location_default ref +); + +commentbinding( + int id: @comment ref, + int element: @element ref +); + +exprconv( + int converted: @expr ref, + unique int conversion: @expr ref +); + +compgenerated(unique int id: @element ref); + +/** + * `destructor_call` destructs the `i`'th entity that should be + * destructed following `element`. Note that entities should be + * destructed in reverse construction order, so for a given `element` + * these should be called from highest to lowest `i`. + */ +#keyset[element, destructor_call] +#keyset[element, i] +synthetic_destructor_call( + int element: @element ref, + int i: int ref, + int destructor_call: @routineexpr ref +); + +namespaces( + unique int id: @namespace, + string name: string ref +); + +namespace_inline( + unique int id: @namespace ref +); + +namespacembrs( + int parentid: @namespace ref, + unique int memberid: @namespacembr ref +); + +@namespacembr = @declaration | @namespace; + +exprparents( + int expr_id: @expr ref, + int child_index: int ref, + int parent_id: @exprparent ref +); + +expr_isload(unique int expr_id: @expr ref); + +@cast = @c_style_cast + | @const_cast + | @dynamic_cast + | @reinterpret_cast + | @static_cast + ; + +/* +case @conversion.kind of + 0 = @simple_conversion // a numeric conversion, qualification conversion, or a reinterpret_cast +| 1 = @bool_conversion // conversion to 'bool' +| 2 = @base_class_conversion // a derived-to-base conversion +| 3 = @derived_class_conversion // a base-to-derived conversion +| 4 = @pm_base_class_conversion // a derived-to-base conversion of a pointer to member +| 5 = @pm_derived_class_conversion // a base-to-derived conversion of a pointer to member +| 6 = @glvalue_adjust // an adjustment of the type of a glvalue +| 7 = @prvalue_adjust // an adjustment of the type of a prvalue +; +*/ +/** + * Describes the semantics represented by a cast expression. This is largely + * independent of the source syntax of the cast, so it is separate from the + * regular expression kind. + */ +conversionkinds( + unique int expr_id: @cast ref, + int kind: int ref +); + +@conversion = @cast + | @array_to_pointer + | @parexpr + | @reference_to + | @ref_indirect + | @temp_init + ; + +/* +case @funbindexpr.kind of + 0 = @normal_call // a normal call +| 1 = @virtual_call // a virtual call +| 2 = @adl_call // a call whose target is only found by ADL +; +*/ +iscall(unique int caller: @funbindexpr ref, int kind: int ref); + +numtemplatearguments( + unique int expr_id: @expr ref, + int num: int ref +); + +specialnamequalifyingelements( + unique int id: @specialnamequalifyingelement, + unique string name: string ref +); + +@namequalifiableelement = @expr | @namequalifier; +@namequalifyingelement = @namespace + | @specialnamequalifyingelement + | @usertype; + +namequalifiers( + unique int id: @namequalifier, + unique int qualifiableelement: @namequalifiableelement ref, + int qualifyingelement: @namequalifyingelement ref, + int location: @location_default ref +); + +varbind( + int expr: @varbindexpr ref, + int var: @accessible ref +); + +funbind( + int expr: @funbindexpr ref, + int fun: @function ref +); + +@any_new_expr = @new_expr + | @new_array_expr; + +@new_or_delete_expr = @any_new_expr + | @delete_expr + | @delete_array_expr; + +@prefix_crement_expr = @preincrexpr | @predecrexpr; + +@postfix_crement_expr = @postincrexpr | @postdecrexpr; + +@increment_expr = @preincrexpr | @postincrexpr; + +@decrement_expr = @predecrexpr | @postdecrexpr; + +@crement_expr = @increment_expr | @decrement_expr; + +@un_arith_op_expr = @arithnegexpr + | @unaryplusexpr + | @conjugation + | @realpartexpr + | @imagpartexpr + | @crement_expr + ; + +@un_bitwise_op_expr = @complementexpr; + +@un_log_op_expr = @notexpr; + +@un_op_expr = @address_of + | @indirect + | @un_arith_op_expr + | @un_bitwise_op_expr + | @builtinaddressof + | @vec_fill + | @un_log_op_expr + | @co_await + | @co_yield + ; + +@bin_log_op_expr = @andlogicalexpr | @orlogicalexpr; + +@cmp_op_expr = @eq_op_expr | @rel_op_expr; + +@eq_op_expr = @eqexpr | @neexpr; + +@rel_op_expr = @gtexpr + | @ltexpr + | @geexpr + | @leexpr + | @spaceshipexpr + ; + +@bin_bitwise_op_expr = @lshiftexpr + | @rshiftexpr + | @andexpr + | @orexpr + | @xorexpr + ; + +@p_arith_op_expr = @paddexpr + | @psubexpr + | @pdiffexpr + ; + +@bin_arith_op_expr = @addexpr + | @subexpr + | @mulexpr + | @divexpr + | @remexpr + | @jmulexpr + | @jdivexpr + | @fjaddexpr + | @jfaddexpr + | @fjsubexpr + | @jfsubexpr + | @minexpr + | @maxexpr + | @p_arith_op_expr + ; + +@bin_op_expr = @bin_arith_op_expr + | @bin_bitwise_op_expr + | @cmp_op_expr + | @bin_log_op_expr + ; + +@op_expr = @un_op_expr + | @bin_op_expr + | @assign_expr + | @conditionalexpr + ; + +@assign_arith_expr = @assignaddexpr + | @assignsubexpr + | @assignmulexpr + | @assigndivexpr + | @assignremexpr + ; + +@assign_bitwise_expr = @assignandexpr + | @assignorexpr + | @assignxorexpr + | @assignlshiftexpr + | @assignrshiftexpr + | @assignpaddexpr + | @assignpsubexpr + ; + +@assign_op_expr = @assign_arith_expr | @assign_bitwise_expr + +@assign_expr = @assignexpr | @assign_op_expr + +/* + case @allocator.form of + 0 = plain + | 1 = alignment + ; +*/ + +/** + * The allocator function associated with a `new` or `new[]` expression. + * The `form` column specified whether the allocation call contains an alignment + * argument. + */ +expr_allocator( + unique int expr: @any_new_expr ref, + int func: @function ref, + int form: int ref +); + +/* + case @deallocator.form of + 0 = plain + | 1 = size + | 2 = alignment + | 3 = size_and_alignment + ; +*/ + +/** + * The deallocator function associated with a `delete`, `delete[]`, `new`, or + * `new[]` expression. For a `new` or `new[]` expression, the deallocator is the + * one used to free memory if the initialization throws an exception. + * The `form` column specifies whether the deallocation call contains a size + * argument, and alignment argument, or both. + */ +expr_deallocator( + unique int expr: @new_or_delete_expr ref, + int func: @function ref, + int form: int ref +); + +/** + * Holds if the `@conditionalexpr` is of the two operand form + * `guard ? : false`. + */ +expr_cond_two_operand( + unique int cond: @conditionalexpr ref +); + +/** + * The guard of `@conditionalexpr` `guard ? true : false` + */ +expr_cond_guard( + unique int cond: @conditionalexpr ref, + int guard: @expr ref +); + +/** + * The expression used when the guard of `@conditionalexpr` + * `guard ? true : false` holds. For the two operand form + * `guard ?: false` consider using `expr_cond_guard` instead. + */ +expr_cond_true( + unique int cond: @conditionalexpr ref, + int true: @expr ref +); + +/** + * The expression used when the guard of `@conditionalexpr` + * `guard ? true : false` does not hold. + */ +expr_cond_false( + unique int cond: @conditionalexpr ref, + int false: @expr ref +); + +/** A string representation of the value. */ +values( + unique int id: @value, + string str: string ref +); + +/** The actual text in the source code for the value, if any. */ +valuetext( + unique int id: @value ref, + string text: string ref +); + +valuebind( + int val: @value ref, + unique int expr: @expr ref +); + +fieldoffsets( + unique int id: @variable ref, + int byteoffset: int ref, + int bitoffset: int ref +); + +bitfield( + unique int id: @variable ref, + int bits: int ref, + int declared_bits: int ref +); + +/* TODO +memberprefix( + int member: @expr ref, + int prefix: @expr ref +); +*/ + +/* + kind(1) = mbrcallexpr + kind(2) = mbrptrcallexpr + kind(3) = mbrptrmbrcallexpr + kind(4) = ptrmbrptrmbrcallexpr + kind(5) = mbrreadexpr // x.y + kind(6) = mbrptrreadexpr // p->y + kind(7) = mbrptrmbrreadexpr // x.*pm + kind(8) = mbrptrmbrptrreadexpr // x->*pm + kind(9) = staticmbrreadexpr // static x.y + kind(10) = staticmbrptrreadexpr // static p->y +*/ +/* TODO +memberaccess( + int member: @expr ref, + int kind: int ref +); +*/ + +initialisers( + unique int init: @initialiser, + int var: @accessible ref, + unique int expr: @expr ref, + int location: @location_expr ref +); + +/** + * An ancestor for the expression, for cases in which we cannot + * otherwise find the expression's parent. + */ +expr_ancestor( + int exp: @expr ref, + int ancestor: @element ref +); + +exprs( + unique int id: @expr, + int kind: int ref, + int location: @location_expr ref +); + +/* + case @value.category of + 1 = prval + | 2 = xval + | 3 = lval + ; +*/ +expr_types( + int id: @expr ref, + int typeid: @type ref, + int value_category: int ref +); + +case @expr.kind of + 1 = @errorexpr +| 2 = @address_of // & AddressOfExpr +| 3 = @reference_to // ReferenceToExpr (implicit?) +| 4 = @indirect // * PointerDereferenceExpr +| 5 = @ref_indirect // ReferenceDereferenceExpr (implicit?) +// ... +| 8 = @array_to_pointer // (???) +| 9 = @vacuous_destructor_call // VacuousDestructorCall +// ... +| 11 = @assume // Microsoft +| 12 = @parexpr +| 13 = @arithnegexpr +| 14 = @unaryplusexpr +| 15 = @complementexpr +| 16 = @notexpr +| 17 = @conjugation // GNU ~ operator +| 18 = @realpartexpr // GNU __real +| 19 = @imagpartexpr // GNU __imag +| 20 = @postincrexpr +| 21 = @postdecrexpr +| 22 = @preincrexpr +| 23 = @predecrexpr +| 24 = @conditionalexpr +| 25 = @addexpr +| 26 = @subexpr +| 27 = @mulexpr +| 28 = @divexpr +| 29 = @remexpr +| 30 = @jmulexpr // C99 mul imaginary +| 31 = @jdivexpr // C99 div imaginary +| 32 = @fjaddexpr // C99 add real + imaginary +| 33 = @jfaddexpr // C99 add imaginary + real +| 34 = @fjsubexpr // C99 sub real - imaginary +| 35 = @jfsubexpr // C99 sub imaginary - real +| 36 = @paddexpr // pointer add (pointer + int or int + pointer) +| 37 = @psubexpr // pointer sub (pointer - integer) +| 38 = @pdiffexpr // difference between two pointers +| 39 = @lshiftexpr +| 40 = @rshiftexpr +| 41 = @andexpr +| 42 = @orexpr +| 43 = @xorexpr +| 44 = @eqexpr +| 45 = @neexpr +| 46 = @gtexpr +| 47 = @ltexpr +| 48 = @geexpr +| 49 = @leexpr +| 50 = @minexpr // GNU minimum +| 51 = @maxexpr // GNU maximum +| 52 = @assignexpr +| 53 = @assignaddexpr +| 54 = @assignsubexpr +| 55 = @assignmulexpr +| 56 = @assigndivexpr +| 57 = @assignremexpr +| 58 = @assignlshiftexpr +| 59 = @assignrshiftexpr +| 60 = @assignandexpr +| 61 = @assignorexpr +| 62 = @assignxorexpr +| 63 = @assignpaddexpr // assign pointer add +| 64 = @assignpsubexpr // assign pointer sub +| 65 = @andlogicalexpr +| 66 = @orlogicalexpr +| 67 = @commaexpr +| 68 = @subscriptexpr // access to member of an array, e.g., a[5] +// ... 69 @objc_subscriptexpr deprecated +// ... 70 @cmdaccess deprecated +// ... +| 73 = @virtfunptrexpr +| 74 = @callexpr +// ... 75 @msgexpr_normal deprecated +// ... 76 @msgexpr_super deprecated +// ... 77 @atselectorexpr deprecated +// ... 78 @atprotocolexpr deprecated +| 79 = @vastartexpr +| 80 = @vaargexpr +| 81 = @vaendexpr +| 82 = @vacopyexpr +// ... 83 @atencodeexpr deprecated +| 84 = @varaccess +| 85 = @thisaccess +// ... 86 @objc_box_expr deprecated +| 87 = @new_expr +| 88 = @delete_expr +| 89 = @throw_expr +| 90 = @condition_decl // a variable declared in a condition, e.g., if(int x = y > 2) +| 91 = @braced_init_list +| 92 = @type_id +| 93 = @runtime_sizeof +| 94 = @runtime_alignof +| 95 = @sizeof_pack +| 96 = @expr_stmt // GNU extension +| 97 = @routineexpr +| 98 = @type_operand // used to access a type in certain contexts (haven't found any examples yet....) +| 99 = @offsetofexpr // offsetof ::= type and field +| 100 = @hasassignexpr // __has_assign ::= type +| 101 = @hascopyexpr // __has_copy ::= type +| 102 = @hasnothrowassign // __has_nothrow_assign ::= type +| 103 = @hasnothrowconstr // __has_nothrow_constructor ::= type +| 104 = @hasnothrowcopy // __has_nothrow_copy ::= type +| 105 = @hastrivialassign // __has_trivial_assign ::= type +| 106 = @hastrivialconstr // __has_trivial_constructor ::= type +| 107 = @hastrivialcopy // __has_trivial_copy ::= type +| 108 = @hasuserdestr // __has_user_destructor ::= type +| 109 = @hasvirtualdestr // __has_virtual_destructor ::= type +| 110 = @isabstractexpr // __is_abstract ::= type +| 111 = @isbaseofexpr // __is_base_of ::= type type +| 112 = @isclassexpr // __is_class ::= type +| 113 = @isconvtoexpr // __is_convertible_to ::= type type +| 114 = @isemptyexpr // __is_empty ::= type +| 115 = @isenumexpr // __is_enum ::= type +| 116 = @ispodexpr // __is_pod ::= type +| 117 = @ispolyexpr // __is_polymorphic ::= type +| 118 = @isunionexpr // __is_union ::= type +| 119 = @typescompexpr // GNU __builtin_types_compatible ::= type type +| 120 = @intaddrexpr // EDG internal builtin, used to implement offsetof +// ... +| 122 = @hastrivialdestructor // __has_trivial_destructor ::= type +| 123 = @literal +| 124 = @uuidof +| 127 = @aggregateliteral +| 128 = @delete_array_expr +| 129 = @new_array_expr +// ... 130 @objc_array_literal deprecated +// ... 131 @objc_dictionary_literal deprecated +| 132 = @foldexpr +// ... +| 200 = @ctordirectinit +| 201 = @ctorvirtualinit +| 202 = @ctorfieldinit +| 203 = @ctordelegatinginit +| 204 = @dtordirectdestruct +| 205 = @dtorvirtualdestruct +| 206 = @dtorfielddestruct +// ... +| 210 = @static_cast +| 211 = @reinterpret_cast +| 212 = @const_cast +| 213 = @dynamic_cast +| 214 = @c_style_cast +| 215 = @lambdaexpr +| 216 = @param_ref +| 217 = @noopexpr +// ... +| 294 = @istriviallyconstructibleexpr +| 295 = @isdestructibleexpr +| 296 = @isnothrowdestructibleexpr +| 297 = @istriviallydestructibleexpr +| 298 = @istriviallyassignableexpr +| 299 = @isnothrowassignableexpr +| 300 = @istrivialexpr +| 301 = @isstandardlayoutexpr +| 302 = @istriviallycopyableexpr +| 303 = @isliteraltypeexpr +| 304 = @hastrivialmoveconstructorexpr +| 305 = @hastrivialmoveassignexpr +| 306 = @hasnothrowmoveassignexpr +| 307 = @isconstructibleexpr +| 308 = @isnothrowconstructibleexpr +| 309 = @hasfinalizerexpr +| 310 = @isdelegateexpr +| 311 = @isinterfaceclassexpr +| 312 = @isrefarrayexpr +| 313 = @isrefclassexpr +| 314 = @issealedexpr +| 315 = @issimplevalueclassexpr +| 316 = @isvalueclassexpr +| 317 = @isfinalexpr +| 319 = @noexceptexpr +| 320 = @builtinshufflevector +| 321 = @builtinchooseexpr +| 322 = @builtinaddressof +| 323 = @vec_fill +| 324 = @builtinconvertvector +| 325 = @builtincomplex +| 326 = @spaceshipexpr +| 327 = @co_await +| 328 = @co_yield +| 329 = @temp_init +; + +@var_args_expr = @vastartexpr + | @vaendexpr + | @vaargexpr + | @vacopyexpr + ; + +@builtin_op = @var_args_expr + | @noopexpr + | @offsetofexpr + | @intaddrexpr + | @hasassignexpr + | @hascopyexpr + | @hasnothrowassign + | @hasnothrowconstr + | @hasnothrowcopy + | @hastrivialassign + | @hastrivialconstr + | @hastrivialcopy + | @hastrivialdestructor + | @hasuserdestr + | @hasvirtualdestr + | @isabstractexpr + | @isbaseofexpr + | @isclassexpr + | @isconvtoexpr + | @isemptyexpr + | @isenumexpr + | @ispodexpr + | @ispolyexpr + | @isunionexpr + | @typescompexpr + | @builtinshufflevector + | @builtinconvertvector + | @builtinaddressof + | @istriviallyconstructibleexpr + | @isdestructibleexpr + | @isnothrowdestructibleexpr + | @istriviallydestructibleexpr + | @istriviallyassignableexpr + | @isnothrowassignableexpr + | @isstandardlayoutexpr + | @istriviallycopyableexpr + | @isliteraltypeexpr + | @hastrivialmoveconstructorexpr + | @hastrivialmoveassignexpr + | @hasnothrowmoveassignexpr + | @isconstructibleexpr + | @isnothrowconstructibleexpr + | @hasfinalizerexpr + | @isdelegateexpr + | @isinterfaceclassexpr + | @isrefarrayexpr + | @isrefclassexpr + | @issealedexpr + | @issimplevalueclassexpr + | @isvalueclassexpr + | @isfinalexpr + | @builtinchooseexpr + | @builtincomplex + ; + +new_allocated_type( + unique int expr: @new_expr ref, + int type_id: @type ref +); + +new_array_allocated_type( + unique int expr: @new_array_expr ref, + int type_id: @type ref +); + +/** + * The field being initialized by an initializer expression within an aggregate + * initializer for a class/struct/union. + */ +#keyset[aggregate, field] +aggregate_field_init( + int aggregate: @aggregateliteral ref, + int initializer: @expr ref, + int field: @membervariable ref +); + +/** + * The index of the element being initialized by an initializer expression + * within an aggregate initializer for an array. + */ +#keyset[aggregate, element_index] +aggregate_array_init( + int aggregate: @aggregateliteral ref, + int initializer: @expr ref, + int element_index: int ref +); + +@ctorinit = @ctordirectinit + | @ctorvirtualinit + | @ctorfieldinit + | @ctordelegatinginit; +@dtordestruct = @dtordirectdestruct + | @dtorvirtualdestruct + | @dtorfielddestruct; + + +condition_decl_bind( + unique int expr: @condition_decl ref, + unique int decl: @declaration ref +); + +typeid_bind( + unique int expr: @type_id ref, + int type_id: @type ref +); + +uuidof_bind( + unique int expr: @uuidof ref, + int type_id: @type ref +); + +@runtime_sizeof_or_alignof = @runtime_sizeof | @runtime_alignof; + +sizeof_bind( + unique int expr: @runtime_sizeof_or_alignof ref, + int type_id: @type ref +); + +code_block( + unique int block: @literal ref, + unique int routine: @function ref +); + +lambdas( + unique int expr: @lambdaexpr ref, + string default_capture: string ref, + boolean has_explicit_return_type: boolean ref +); + +lambda_capture( + unique int id: @lambdacapture, + int lambda: @lambdaexpr ref, + int index: int ref, + int field: @membervariable ref, + boolean captured_by_reference: boolean ref, + boolean is_implicit: boolean ref, + int location: @location_default ref +); + +@funbindexpr = @routineexpr + | @new_expr + | @delete_expr + | @delete_array_expr + | @ctordirectinit + | @ctorvirtualinit + | @ctordelegatinginit + | @dtordirectdestruct + | @dtorvirtualdestruct; + +@varbindexpr = @varaccess | @ctorfieldinit | @dtorfielddestruct; +@addressable = @function | @variable ; +@accessible = @addressable | @enumconstant ; + +@access = @varaccess | @routineexpr ; + +fold( + int expr: @foldexpr ref, + string operator: string ref, + boolean is_left_fold: boolean ref +); + +stmts( + unique int id: @stmt, + int kind: int ref, + int location: @location_stmt ref +); + +case @stmt.kind of + 1 = @stmt_expr +| 2 = @stmt_if +| 3 = @stmt_while +| 4 = @stmt_goto +| 5 = @stmt_label +| 6 = @stmt_return +| 7 = @stmt_block +| 8 = @stmt_end_test_while // do { ... } while ( ... ) +| 9 = @stmt_for +| 10 = @stmt_switch_case +| 11 = @stmt_switch +| 13 = @stmt_asm // "asm" statement or the body of an asm function +| 15 = @stmt_try_block +| 16 = @stmt_microsoft_try // Microsoft +| 17 = @stmt_decl +| 18 = @stmt_set_vla_size // C99 +| 19 = @stmt_vla_decl // C99 +| 25 = @stmt_assigned_goto // GNU +| 26 = @stmt_empty +| 27 = @stmt_continue +| 28 = @stmt_break +| 29 = @stmt_range_based_for // C++11 +// ... 30 @stmt_at_autoreleasepool_block deprecated +// ... 31 @stmt_objc_for_in deprecated +// ... 32 @stmt_at_synchronized deprecated +| 33 = @stmt_handler +// ... 34 @stmt_finally_end deprecated +| 35 = @stmt_constexpr_if +| 37 = @stmt_co_return +; + +type_vla( + int type_id: @type ref, + int decl: @stmt_vla_decl ref +); + +variable_vla( + int var: @variable ref, + int decl: @stmt_vla_decl ref +); + +if_then( + unique int if_stmt: @stmt_if ref, + int then_id: @stmt ref +); + +if_else( + unique int if_stmt: @stmt_if ref, + int else_id: @stmt ref +); + +constexpr_if_then( + unique int constexpr_if_stmt: @stmt_constexpr_if ref, + int then_id: @stmt ref +); + +constexpr_if_else( + unique int constexpr_if_stmt: @stmt_constexpr_if ref, + int else_id: @stmt ref +); + +while_body( + unique int while_stmt: @stmt_while ref, + int body_id: @stmt ref +); + +do_body( + unique int do_stmt: @stmt_end_test_while ref, + int body_id: @stmt ref +); + +#keyset[switch_stmt, index] +switch_case( + int switch_stmt: @stmt_switch ref, + int index: int ref, + int case_id: @stmt_switch_case ref +); + +switch_body( + unique int switch_stmt: @stmt_switch ref, + int body_id: @stmt ref +); + +for_initialization( + unique int for_stmt: @stmt_for ref, + int init_id: @stmt ref +); + +for_condition( + unique int for_stmt: @stmt_for ref, + int condition_id: @expr ref +); + +for_update( + unique int for_stmt: @stmt_for ref, + int update_id: @expr ref +); + +for_body( + unique int for_stmt: @stmt_for ref, + int body_id: @stmt ref +); + +@stmtparent = @stmt | @expr_stmt ; +stmtparents( + unique int id: @stmt ref, + int index: int ref, + int parent: @stmtparent ref +); + +ishandler(unique int block: @stmt_block ref); + +@cfgnode = @stmt | @expr | @function | @initialiser ; + +stmt_decl_bind( + int stmt: @stmt_decl ref, + int num: int ref, + int decl: @declaration ref +); + +stmt_decl_entry_bind( + int stmt: @stmt_decl ref, + int num: int ref, + int decl_entry: @element ref +); + +@functionorblock = @function | @stmt_block; + +blockscope( + unique int block: @stmt_block ref, + int enclosing: @functionorblock ref +); + +@jump = @stmt_goto | @stmt_break | @stmt_continue; + +@jumporlabel = @jump | @stmt_label | @literal; + +jumpinfo( + unique int id: @jumporlabel ref, + string str: string ref, + int target: @stmt ref +); + +preprocdirects( + unique int id: @preprocdirect, + int kind: int ref, + int location: @location_default ref +); +case @preprocdirect.kind of + 0 = @ppd_if +| 1 = @ppd_ifdef +| 2 = @ppd_ifndef +| 3 = @ppd_elif +| 4 = @ppd_else +| 5 = @ppd_endif +| 6 = @ppd_plain_include +| 7 = @ppd_define +| 8 = @ppd_undef +| 9 = @ppd_line +| 10 = @ppd_error +| 11 = @ppd_pragma +| 12 = @ppd_objc_import +| 13 = @ppd_include_next +| 18 = @ppd_warning +; + +@ppd_include = @ppd_plain_include | @ppd_objc_import | @ppd_include_next; + +@ppd_branch = @ppd_if | @ppd_ifdef | @ppd_ifndef | @ppd_elif; + +preprocpair( + int begin : @ppd_branch ref, + int elseelifend : @preprocdirect ref +); + +preproctrue(int branch : @ppd_branch ref); +preprocfalse(int branch : @ppd_branch ref); + +preproctext( + unique int id: @preprocdirect ref, + string head: string ref, + string body: string ref +); + +includes( + unique int id: @ppd_include ref, + int included: @file ref +); + +link_targets( + unique int id: @link_target, + int binary: @file ref +); + +link_parent( + int element : @element ref, + int link_target : @link_target ref +); + +/* XML Files */ + +xmlEncoding(unique int id: @file ref, string encoding: string ref); + +xmlDTDs( + unique int id: @xmldtd, + string root: string ref, + string publicId: string ref, + string systemId: string ref, + int fileid: @file ref +); + +xmlElements( + unique int id: @xmlelement, + string name: string ref, + int parentid: @xmlparent ref, + int idx: int ref, + int fileid: @file ref +); + +xmlAttrs( + unique int id: @xmlattribute, + int elementid: @xmlelement ref, + string name: string ref, + string value: string ref, + int idx: int ref, + int fileid: @file ref +); + +xmlNs( + int id: @xmlnamespace, + string prefixName: string ref, + string URI: string ref, + int fileid: @file ref +); + +xmlHasNs( + int elementId: @xmlnamespaceable ref, + int nsId: @xmlnamespace ref, + int fileid: @file ref +); + +xmlComments( + unique int id: @xmlcomment, + string text: string ref, + int parentid: @xmlparent ref, + int fileid: @file ref +); + +xmlChars( + unique int id: @xmlcharacters, + string text: string ref, + int parentid: @xmlparent ref, + int idx: int ref, + int isCDATA: int ref, + int fileid: @file ref +); + +@xmlparent = @file | @xmlelement; +@xmlnamespaceable = @xmlelement | @xmlattribute; + +xmllocations( + int xmlElement: @xmllocatable ref, + int location: @location_default ref +); + +@xmllocatable = @xmlcharacters + | @xmlelement + | @xmlcomment + | @xmlattribute + | @xmldtd + | @file + | @xmlnamespace; diff --git a/cpp/ql/lib/upgrades/e9a518baf14f4322ac243578a8e1391386ff030f/semmlecode.cpp.dbscheme b/cpp/ql/lib/upgrades/e9a518baf14f4322ac243578a8e1391386ff030f/semmlecode.cpp.dbscheme new file mode 100644 index 00000000000..cf72c8898d1 --- /dev/null +++ b/cpp/ql/lib/upgrades/e9a518baf14f4322ac243578a8e1391386ff030f/semmlecode.cpp.dbscheme @@ -0,0 +1,2111 @@ + +/** + * An invocation of the compiler. Note that more than one file may be + * compiled per invocation. For example, this command compiles three + * source files: + * + * gcc -c f1.c f2.c f3.c + * + * The `id` simply identifies the invocation, while `cwd` is the working + * directory from which the compiler was invoked. + */ +compilations( + /** + * An invocation of the compiler. Note that more than one file may + * be compiled per invocation. For example, this command compiles + * three source files: + * + * gcc -c f1.c f2.c f3.c + */ + unique int id : @compilation, + string cwd : string ref +); + +/** + * The arguments that were passed to the extractor for a compiler + * invocation. If `id` is for the compiler invocation + * + * gcc -c f1.c f2.c f3.c + * + * then typically there will be rows for + * + * num | arg + * --- | --- + * 0 | *path to extractor* + * 1 | `--mimic` + * 2 | `/usr/bin/gcc` + * 3 | `-c` + * 4 | f1.c + * 5 | f2.c + * 6 | f3.c + */ +#keyset[id, num] +compilation_args( + int id : @compilation ref, + int num : int ref, + string arg : string ref +); + +/** + * The source files that are compiled by a compiler invocation. + * If `id` is for the compiler invocation + * + * gcc -c f1.c f2.c f3.c + * + * then there will be rows for + * + * num | arg + * --- | --- + * 0 | f1.c + * 1 | f2.c + * 2 | f3.c + * + * Note that even if those files `#include` headers, those headers + * do not appear as rows. + */ +#keyset[id, num] +compilation_compiling_files( + int id : @compilation ref, + int num : int ref, + int file : @file ref +); + +/** + * The time taken by the extractor for a compiler invocation. + * + * For each file `num`, there will be rows for + * + * kind | seconds + * ---- | --- + * 1 | CPU seconds used by the extractor frontend + * 2 | Elapsed seconds during the extractor frontend + * 3 | CPU seconds used by the extractor backend + * 4 | Elapsed seconds during the extractor backend + */ +#keyset[id, num, kind] +compilation_time( + int id : @compilation ref, + int num : int ref, + /* kind: + 1 = frontend_cpu_seconds + 2 = frontend_elapsed_seconds + 3 = extractor_cpu_seconds + 4 = extractor_elapsed_seconds + */ + int kind : int ref, + float seconds : float ref +); + +/** + * An error or warning generated by the extractor. + * The diagnostic message `diagnostic` was generated during compiler + * invocation `compilation`, and is the `file_number_diagnostic_number`th + * message generated while extracting the `file_number`th file of that + * invocation. + */ +#keyset[compilation, file_number, file_number_diagnostic_number] +diagnostic_for( + int diagnostic : @diagnostic ref, + int compilation : @compilation ref, + int file_number : int ref, + int file_number_diagnostic_number : int ref +); + +/** + * If extraction was successful, then `cpu_seconds` and + * `elapsed_seconds` are the CPU time and elapsed time (respectively) + * that extraction took for compiler invocation `id`. + */ +compilation_finished( + unique int id : @compilation ref, + float cpu_seconds : float ref, + float elapsed_seconds : float ref +); + + +/** + * External data, loaded from CSV files during snapshot creation. See + * [Tutorial: Incorporating external data](https://help.semmle.com/wiki/display/SD/Tutorial%3A+Incorporating+external+data) + * for more information. + */ +externalData( + int id : @externalDataElement, + string path : string ref, + int column: int ref, + string value : string ref +); + +/** + * The source location of the snapshot. + */ +sourceLocationPrefix(string prefix : string ref); + +/** + * Information about packages that provide code used during compilation. + * The `id` is just a unique identifier. + * The `namespace` is typically the name of the package manager that + * provided the package (e.g. "dpkg" or "yum"). + * The `package_name` is the name of the package, and `version` is its + * version (as a string). + */ +external_packages( + unique int id: @external_package, + string namespace : string ref, + string package_name : string ref, + string version : string ref +); + +/** + * Holds if File `fileid` was provided by package `package`. + */ +header_to_external_package( + int fileid : @file ref, + int package : @external_package ref +); + +/* + * Version history + */ + +svnentries( + unique int id : @svnentry, + string revision : string ref, + string author : string ref, + date revisionDate : date ref, + int changeSize : int ref +) + +svnaffectedfiles( + int id : @svnentry ref, + int file : @file ref, + string action : string ref +) + +svnentrymsg( + unique int id : @svnentry ref, + string message : string ref +) + +svnchurn( + int commit : @svnentry ref, + int file : @file ref, + int addedLines : int ref, + int deletedLines : int ref +) + +/* + * C++ dbscheme + */ + +@location = @location_stmt | @location_expr | @location_default ; + +/** + * The location of an element that is not an expression or a statement. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `file`. + * For more information, see + * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/). + */ +locations_default( + /** The location of an element that is not an expression or a statement. */ + unique int id: @location_default, + int container: @container ref, + int startLine: int ref, + int startColumn: int ref, + int endLine: int ref, + int endColumn: int ref +); + +/** + * The location of a statement. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `file`. + * For more information, see + * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/). + */ +locations_stmt( + /** The location of a statement. */ + unique int id: @location_stmt, + int container: @container ref, + int startLine: int ref, + int startColumn: int ref, + int endLine: int ref, + int endColumn: int ref +); + +/** + * The location of an expression. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `file`. + * For more information, see + * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/). + */ +locations_expr( + /** The location of an expression. */ + unique int id: @location_expr, + int container: @container ref, + int startLine: int ref, + int startColumn: int ref, + int endLine: int ref, + int endColumn: int ref +); + +/** An element for which line-count information is available. */ +@sourceline = @file | @function | @variable | @enumconstant | @xmllocatable; + +numlines( + int element_id: @sourceline ref, + int num_lines: int ref, + int num_code: int ref, + int num_comment: int ref +); + +diagnostics( + unique int id: @diagnostic, + int severity: int ref, + string error_tag: string ref, + string error_message: string ref, + string full_error_message: string ref, + int location: @location_default ref +); + +files( + unique int id: @file, + string name: string ref +); + +folders( + unique int id: @folder, + string name: string ref +); + +@container = @folder | @file + +containerparent( + int parent: @container ref, + unique int child: @container ref +); + +fileannotations( + int id: @file ref, + int kind: int ref, + string name: string ref, + string value: string ref +); + +inmacroexpansion( + int id: @element ref, + int inv: @macroinvocation ref +); + +affectedbymacroexpansion( + int id: @element ref, + int inv: @macroinvocation ref +); + +/* + case @macroinvocations.kind of + 1 = macro expansion + | 2 = other macro reference + ; +*/ +macroinvocations( + unique int id: @macroinvocation, + int macro_id: @ppd_define ref, + int location: @location_default ref, + int kind: int ref +); + +macroparent( + unique int id: @macroinvocation ref, + int parent_id: @macroinvocation ref +); + +// a macroinvocation may be part of another location +// the way to find a constant expression that uses a macro +// is thus to find a constant expression that has a location +// to which a macro invocation is bound +macrolocationbind( + int id: @macroinvocation ref, + int location: @location ref +); + +#keyset[invocation, argument_index] +macro_argument_unexpanded( + int invocation: @macroinvocation ref, + int argument_index: int ref, + string text: string ref +); + +#keyset[invocation, argument_index] +macro_argument_expanded( + int invocation: @macroinvocation ref, + int argument_index: int ref, + string text: string ref +); + +/* + case @function.kind of + 1 = normal + | 2 = constructor + | 3 = destructor + | 4 = conversion + | 5 = operator + | 6 = builtin // GCC built-in functions, e.g. __builtin___memcpy_chk + ; +*/ +functions( + unique int id: @function, + string name: string ref, + int kind: int ref +); + +function_entry_point(int id: @function ref, unique int entry_point: @stmt ref); + +function_return_type(int id: @function ref, int return_type: @type ref); + +/** If `function` is a coroutine, then this gives the + std::experimental::resumable_traits instance associated with it, + and the variables representing the `handle` and `promise` for it. */ +coroutine( + unique int function: @function ref, + int traits: @type ref, + int handle: @variable ref, + int promise: @variable ref +); + +/** The `new` function used for allocating the coroutine state, if any. */ +coroutine_new( + unique int function: @function ref, + int new: @function ref +); + +/** The `delete` function used for deallocating the coroutine state, if any. */ +coroutine_delete( + unique int function: @function ref, + int delete: @function ref +); + +purefunctions(unique int id: @function ref); + +function_deleted(unique int id: @function ref); + +function_defaulted(unique int id: @function ref); + +member_function_this_type(unique int id: @function ref, int this_type: @type ref); + +#keyset[id, type_id] +fun_decls( + int id: @fun_decl, + int function: @function ref, + int type_id: @type ref, + string name: string ref, + int location: @location_default ref +); +fun_def(unique int id: @fun_decl ref); +fun_specialized(unique int id: @fun_decl ref); +fun_implicit(unique int id: @fun_decl ref); +fun_decl_specifiers( + int id: @fun_decl ref, + string name: string ref +) +#keyset[fun_decl, index] +fun_decl_throws( + int fun_decl: @fun_decl ref, + int index: int ref, + int type_id: @type ref +); +/* an empty throw specification is different from none */ +fun_decl_empty_throws(unique int fun_decl: @fun_decl ref); +fun_decl_noexcept( + int fun_decl: @fun_decl ref, + int constant: @expr ref +); +fun_decl_empty_noexcept(int fun_decl: @fun_decl ref); +fun_decl_typedef_type( + unique int fun_decl: @fun_decl ref, + int typedeftype_id: @usertype ref +); + +param_decl_bind( + unique int id: @var_decl ref, + int index: int ref, + int fun_decl: @fun_decl ref +); + +#keyset[id, type_id] +var_decls( + int id: @var_decl, + int variable: @variable ref, + int type_id: @type ref, + string name: string ref, + int location: @location_default ref +); +var_def(unique int id: @var_decl ref); +var_decl_specifiers( + int id: @var_decl ref, + string name: string ref +) +is_structured_binding(unique int id: @variable ref); + +type_decls( + unique int id: @type_decl, + int type_id: @type ref, + int location: @location_default ref +); +type_def(unique int id: @type_decl ref); +type_decl_top( + unique int type_decl: @type_decl ref +); + +namespace_decls( + unique int id: @namespace_decl, + int namespace_id: @namespace ref, + int location: @location_default ref, + int bodylocation: @location_default ref +); + +usings( + unique int id: @using, + int element_id: @element ref, + int location: @location_default ref +); + +/** The element which contains the `using` declaration. */ +using_container( + int parent: @element ref, + int child: @using ref +); + +static_asserts( + unique int id: @static_assert, + int condition : @expr ref, + string message : string ref, + int location: @location_default ref, + int enclosing : @element ref +); + +// each function has an ordered list of parameters +#keyset[id, type_id] +#keyset[function, index, type_id] +params( + int id: @parameter, + int function: @functionorblock ref, + int index: int ref, + int type_id: @type ref +); + +overrides(int new: @function ref, int old: @function ref); + +#keyset[id, type_id] +membervariables( + int id: @membervariable, + int type_id: @type ref, + string name: string ref +); + +#keyset[id, type_id] +globalvariables( + int id: @globalvariable, + int type_id: @type ref, + string name: string ref +); + +#keyset[id, type_id] +localvariables( + int id: @localvariable, + int type_id: @type ref, + string name: string ref +); + +autoderivation( + unique int var: @variable ref, + int derivation_type: @type ref +); + +enumconstants( + unique int id: @enumconstant, + int parent: @usertype ref, + int index: int ref, + int type_id: @type ref, + string name: string ref, + int location: @location_default ref +); + +@variable = @localscopevariable | @globalvariable | @membervariable; + +@localscopevariable = @localvariable | @parameter; + +/* + Built-in types are the fundamental types, e.g., integral, floating, and void. + + case @builtintype.kind of + 1 = error + | 2 = unknown + | 3 = void + | 4 = boolean + | 5 = char + | 6 = unsigned_char + | 7 = signed_char + | 8 = short + | 9 = unsigned_short + | 10 = signed_short + | 11 = int + | 12 = unsigned_int + | 13 = signed_int + | 14 = long + | 15 = unsigned_long + | 16 = signed_long + | 17 = long_long + | 18 = unsigned_long_long + | 19 = signed_long_long + | 20 = __int8 // Microsoft-specific + | 21 = __int16 // Microsoft-specific + | 22 = __int32 // Microsoft-specific + | 23 = __int64 // Microsoft-specific + | 24 = float + | 25 = double + | 26 = long_double + | 27 = _Complex_float // C99-specific + | 28 = _Complex_double // C99-specific + | 29 = _Complex_long double // C99-specific + | 30 = _Imaginary_float // C99-specific + | 31 = _Imaginary_double // C99-specific + | 32 = _Imaginary_long_double // C99-specific + | 33 = wchar_t // Microsoft-specific + | 34 = decltype_nullptr // C++11 + | 35 = __int128 + | 36 = unsigned___int128 + | 37 = signed___int128 + | 38 = __float128 + | 39 = _Complex___float128 + | 40 = _Decimal32 + | 41 = _Decimal64 + | 42 = _Decimal128 + | 43 = char16_t + | 44 = char32_t + | 45 = _Float32 + | 46 = _Float32x + | 47 = _Float64 + | 48 = _Float64x + | 49 = _Float128 + | 50 = _Float128x + | 51 = char8_t + ; +*/ +builtintypes( + unique int id: @builtintype, + string name: string ref, + int kind: int ref, + int size: int ref, + int sign: int ref, + int alignment: int ref +); + +/* + Derived types are types that are directly derived from existing types and + point to, refer to, transform type data to return a new type. + + case @derivedtype.kind of + 1 = pointer + | 2 = reference + | 3 = type_with_specifiers + | 4 = array + | 5 = gnu_vector + | 6 = routineptr + | 7 = routinereference + | 8 = rvalue_reference // C++11 +// ... 9 type_conforming_to_protocols deprecated + | 10 = block + ; +*/ +derivedtypes( + unique int id: @derivedtype, + string name: string ref, + int kind: int ref, + int type_id: @type ref +); + +pointerishsize(unique int id: @derivedtype ref, + int size: int ref, + int alignment: int ref); + +arraysizes( + unique int id: @derivedtype ref, + int num_elements: int ref, + int bytesize: int ref, + int alignment: int ref +); + +typedefbase( + unique int id: @usertype ref, + int type_id: @type ref +); + +/** + * An instance of the C++11 `decltype` operator. For example: + * ``` + * int a; + * decltype(1+a) b; + * ``` + * Here `expr` is `1+a`. + * + * Sometimes an additional pair of parentheses around the expression + * would change the semantics of this decltype, e.g. + * ``` + * struct A { double x; }; + * const A* a = new A(); + * decltype( a->x ); // type is double + * decltype((a->x)); // type is const double& + * ``` + * (Please consult the C++11 standard for more details). + * `parentheses_would_change_meaning` is `true` iff that is the case. + */ +#keyset[id, expr] +decltypes( + int id: @decltype, + int expr: @expr ref, + int base_type: @type ref, + boolean parentheses_would_change_meaning: boolean ref +); + +/* + case @usertype.kind of + 1 = struct + | 2 = class + | 3 = union + | 4 = enum + | 5 = typedef // classic C: typedef typedef type name + | 6 = template + | 7 = template_parameter + | 8 = template_template_parameter + | 9 = proxy_class // a proxy class associated with a template parameter +// ... 10 objc_class deprecated +// ... 11 objc_protocol deprecated +// ... 12 objc_category deprecated + | 13 = scoped_enum + | 14 = using_alias // a using name = type style typedef + ; +*/ +usertypes( + unique int id: @usertype, + string name: string ref, + int kind: int ref +); + +usertypesize( + unique int id: @usertype ref, + int size: int ref, + int alignment: int ref +); + +usertype_final(unique int id: @usertype ref); + +usertype_uuid( + unique int id: @usertype ref, + string uuid: string ref +); + +mangled_name( + unique int id: @declaration ref, + int mangled_name : @mangledname +); + +is_pod_class(unique int id: @usertype ref); +is_standard_layout_class(unique int id: @usertype ref); + +is_complete(unique int id: @usertype ref); + +is_class_template(unique int id: @usertype ref); +class_instantiation( + int to: @usertype ref, + int from: @usertype ref +); +class_template_argument( + int type_id: @usertype ref, + int index: int ref, + int arg_type: @type ref +); +class_template_argument_value( + int type_id: @usertype ref, + int index: int ref, + int arg_value: @expr ref +); + +is_proxy_class_for( + unique int id: @usertype ref, + unique int templ_param_id: @usertype ref +); + +type_mentions( + unique int id: @type_mention, + int type_id: @type ref, + int location: @location ref, + // a_symbol_reference_kind from the EDG frontend. See symbol_ref.h there. + int kind: int ref +); + +is_function_template(unique int id: @function ref); +function_instantiation( + unique int to: @function ref, + int from: @function ref +); +function_template_argument( + int function_id: @function ref, + int index: int ref, + int arg_type: @type ref +); +function_template_argument_value( + int function_id: @function ref, + int index: int ref, + int arg_value: @expr ref +); + +is_variable_template(unique int id: @variable ref); +variable_instantiation( + unique int to: @variable ref, + int from: @variable ref +); +variable_template_argument( + int variable_id: @variable ref, + int index: int ref, + int arg_type: @type ref +); +variable_template_argument_value( + int variable_id: @variable ref, + int index: int ref, + int arg_value: @expr ref +); + +/* + Fixed point types + precision(1) = short, precision(2) = default, precision(3) = long + is_unsigned(1) = unsigned is_unsigned(2) = signed + is_fract_type(1) = declared with _Fract + saturating(1) = declared with _Sat +*/ +/* TODO +fixedpointtypes( + unique int id: @fixedpointtype, + int precision: int ref, + int is_unsigned: int ref, + int is_fract_type: int ref, + int saturating: int ref); +*/ + +routinetypes( + unique int id: @routinetype, + int return_type: @type ref +); + +routinetypeargs( + int routine: @routinetype ref, + int index: int ref, + int type_id: @type ref +); + +ptrtomembers( + unique int id: @ptrtomember, + int type_id: @type ref, + int class_id: @type ref +); + +/* + specifiers for types, functions, and variables + + "public", + "protected", + "private", + + "const", + "volatile", + "static", + + "pure", + "virtual", + "sealed", // Microsoft + "__interface", // Microsoft + "inline", + "explicit", + + "near", // near far extension + "far", // near far extension + "__ptr32", // Microsoft + "__ptr64", // Microsoft + "__sptr", // Microsoft + "__uptr", // Microsoft + "dllimport", // Microsoft + "dllexport", // Microsoft + "thread", // Microsoft + "naked", // Microsoft + "microsoft_inline", // Microsoft + "forceinline", // Microsoft + "selectany", // Microsoft + "nothrow", // Microsoft + "novtable", // Microsoft + "noreturn", // Microsoft + "noinline", // Microsoft + "noalias", // Microsoft + "restrict", // Microsoft +*/ + +specifiers( + unique int id: @specifier, + unique string str: string ref +); + +typespecifiers( + int type_id: @type ref, + int spec_id: @specifier ref +); + +funspecifiers( + int func_id: @function ref, + int spec_id: @specifier ref +); + +varspecifiers( + int var_id: @accessible ref, + int spec_id: @specifier ref +); + +attributes( + unique int id: @attribute, + int kind: int ref, + string name: string ref, + string name_space: string ref, + int location: @location_default ref +); + +case @attribute.kind of + 0 = @gnuattribute +| 1 = @stdattribute +| 2 = @declspec +| 3 = @msattribute +| 4 = @alignas +// ... 5 @objc_propertyattribute deprecated +; + +attribute_args( + unique int id: @attribute_arg, + int kind: int ref, + int attribute: @attribute ref, + int index: int ref, + int location: @location_default ref +); + +case @attribute_arg.kind of + 0 = @attribute_arg_empty +| 1 = @attribute_arg_token +| 2 = @attribute_arg_constant +| 3 = @attribute_arg_type +; + +attribute_arg_value( + unique int arg: @attribute_arg ref, + string value: string ref +); +attribute_arg_type( + unique int arg: @attribute_arg ref, + int type_id: @type ref +); +attribute_arg_name( + unique int arg: @attribute_arg ref, + string name: string ref +); + +typeattributes( + int type_id: @type ref, + int spec_id: @attribute ref +); + +funcattributes( + int func_id: @function ref, + int spec_id: @attribute ref +); + +varattributes( + int var_id: @accessible ref, + int spec_id: @attribute ref +); + +stmtattributes( + int stmt_id: @stmt ref, + int spec_id: @attribute ref +); + +@type = @builtintype + | @derivedtype + | @usertype + /* TODO | @fixedpointtype */ + | @routinetype + | @ptrtomember + | @decltype; + +unspecifiedtype( + unique int type_id: @type ref, + int unspecified_type_id: @type ref +); + +member( + int parent: @type ref, + int index: int ref, + int child: @member ref +); + +@enclosingfunction_child = @usertype | @variable | @namespace + +enclosingfunction( + unique int child: @enclosingfunction_child ref, + int parent: @function ref +); + +derivations( + unique int derivation: @derivation, + int sub: @type ref, + int index: int ref, + int super: @type ref, + int location: @location_default ref +); + +derspecifiers( + int der_id: @derivation ref, + int spec_id: @specifier ref +); + +/** + * Contains the byte offset of the base class subobject within the derived + * class. Only holds for non-virtual base classes, but see table + * `virtual_base_offsets` for offsets of virtual base class subobjects. + */ +direct_base_offsets( + unique int der_id: @derivation ref, + int offset: int ref +); + +/** + * Contains the byte offset of the virtual base class subobject for class + * `super` within a most-derived object of class `sub`. `super` can be either a + * direct or indirect base class. + */ +#keyset[sub, super] +virtual_base_offsets( + int sub: @usertype ref, + int super: @usertype ref, + int offset: int ref +); + +frienddecls( + unique int id: @frienddecl, + int type_id: @type ref, + int decl_id: @declaration ref, + int location: @location_default ref +); + +@declaredtype = @usertype ; + +@declaration = @function + | @declaredtype + | @variable + | @enumconstant + | @frienddecl; + +@member = @membervariable + | @function + | @declaredtype + | @enumconstant; + +@locatable = @diagnostic + | @declaration + | @ppd_include + | @ppd_define + | @macroinvocation + /*| @funcall*/ + | @xmllocatable + | @attribute + | @attribute_arg; + +@namedscope = @namespace | @usertype; + +@element = @locatable + | @file + | @folder + | @specifier + | @type + | @expr + | @namespace + | @initialiser + | @stmt + | @derivation + | @comment + | @preprocdirect + | @fun_decl + | @var_decl + | @type_decl + | @namespace_decl + | @using + | @namequalifier + | @specialnamequalifyingelement + | @static_assert + | @type_mention + | @lambdacapture; + +@exprparent = @element; + +comments( + unique int id: @comment, + string contents: string ref, + int location: @location_default ref +); + +commentbinding( + int id: @comment ref, + int element: @element ref +); + +exprconv( + int converted: @expr ref, + unique int conversion: @expr ref +); + +compgenerated(unique int id: @element ref); + +/** + * `destructor_call` destructs the `i`'th entity that should be + * destructed following `element`. Note that entities should be + * destructed in reverse construction order, so for a given `element` + * these should be called from highest to lowest `i`. + */ +#keyset[element, destructor_call] +#keyset[element, i] +synthetic_destructor_call( + int element: @element ref, + int i: int ref, + int destructor_call: @routineexpr ref +); + +namespaces( + unique int id: @namespace, + string name: string ref +); + +namespace_inline( + unique int id: @namespace ref +); + +namespacembrs( + int parentid: @namespace ref, + unique int memberid: @namespacembr ref +); + +@namespacembr = @declaration | @namespace; + +exprparents( + int expr_id: @expr ref, + int child_index: int ref, + int parent_id: @exprparent ref +); + +expr_isload(unique int expr_id: @expr ref); + +@cast = @c_style_cast + | @const_cast + | @dynamic_cast + | @reinterpret_cast + | @static_cast + ; + +/* +case @conversion.kind of + 0 = @simple_conversion // a numeric conversion, qualification conversion, or a reinterpret_cast +| 1 = @bool_conversion // conversion to 'bool' +| 2 = @base_class_conversion // a derived-to-base conversion +| 3 = @derived_class_conversion // a base-to-derived conversion +| 4 = @pm_base_class_conversion // a derived-to-base conversion of a pointer to member +| 5 = @pm_derived_class_conversion // a base-to-derived conversion of a pointer to member +| 6 = @glvalue_adjust // an adjustment of the type of a glvalue +| 7 = @prvalue_adjust // an adjustment of the type of a prvalue +; +*/ +/** + * Describes the semantics represented by a cast expression. This is largely + * independent of the source syntax of the cast, so it is separate from the + * regular expression kind. + */ +conversionkinds( + unique int expr_id: @cast ref, + int kind: int ref +); + +@conversion = @cast + | @array_to_pointer + | @parexpr + | @reference_to + | @ref_indirect + | @temp_init + ; + +/* +case @funbindexpr.kind of + 0 = @normal_call // a normal call +| 1 = @virtual_call // a virtual call +| 2 = @adl_call // a call whose target is only found by ADL +; +*/ +iscall(unique int caller: @funbindexpr ref, int kind: int ref); + +numtemplatearguments( + unique int expr_id: @expr ref, + int num: int ref +); + +specialnamequalifyingelements( + unique int id: @specialnamequalifyingelement, + unique string name: string ref +); + +@namequalifiableelement = @expr | @namequalifier; +@namequalifyingelement = @namespace + | @specialnamequalifyingelement + | @usertype; + +namequalifiers( + unique int id: @namequalifier, + unique int qualifiableelement: @namequalifiableelement ref, + int qualifyingelement: @namequalifyingelement ref, + int location: @location_default ref +); + +varbind( + int expr: @varbindexpr ref, + int var: @accessible ref +); + +funbind( + int expr: @funbindexpr ref, + int fun: @function ref +); + +@any_new_expr = @new_expr + | @new_array_expr; + +@new_or_delete_expr = @any_new_expr + | @delete_expr + | @delete_array_expr; + +@prefix_crement_expr = @preincrexpr | @predecrexpr; + +@postfix_crement_expr = @postincrexpr | @postdecrexpr; + +@increment_expr = @preincrexpr | @postincrexpr; + +@decrement_expr = @predecrexpr | @postdecrexpr; + +@crement_expr = @increment_expr | @decrement_expr; + +@un_arith_op_expr = @arithnegexpr + | @unaryplusexpr + | @conjugation + | @realpartexpr + | @imagpartexpr + | @crement_expr + ; + +@un_bitwise_op_expr = @complementexpr; + +@un_log_op_expr = @notexpr; + +@un_op_expr = @address_of + | @indirect + | @un_arith_op_expr + | @un_bitwise_op_expr + | @builtinaddressof + | @vec_fill + | @un_log_op_expr + | @co_await + | @co_yield + ; + +@bin_log_op_expr = @andlogicalexpr | @orlogicalexpr; + +@cmp_op_expr = @eq_op_expr | @rel_op_expr; + +@eq_op_expr = @eqexpr | @neexpr; + +@rel_op_expr = @gtexpr + | @ltexpr + | @geexpr + | @leexpr + | @spaceshipexpr + ; + +@bin_bitwise_op_expr = @lshiftexpr + | @rshiftexpr + | @andexpr + | @orexpr + | @xorexpr + ; + +@p_arith_op_expr = @paddexpr + | @psubexpr + | @pdiffexpr + ; + +@bin_arith_op_expr = @addexpr + | @subexpr + | @mulexpr + | @divexpr + | @remexpr + | @jmulexpr + | @jdivexpr + | @fjaddexpr + | @jfaddexpr + | @fjsubexpr + | @jfsubexpr + | @minexpr + | @maxexpr + | @p_arith_op_expr + ; + +@bin_op_expr = @bin_arith_op_expr + | @bin_bitwise_op_expr + | @cmp_op_expr + | @bin_log_op_expr + ; + +@op_expr = @un_op_expr + | @bin_op_expr + | @assign_expr + | @conditionalexpr + ; + +@assign_arith_expr = @assignaddexpr + | @assignsubexpr + | @assignmulexpr + | @assigndivexpr + | @assignremexpr + ; + +@assign_bitwise_expr = @assignandexpr + | @assignorexpr + | @assignxorexpr + | @assignlshiftexpr + | @assignrshiftexpr + | @assignpaddexpr + | @assignpsubexpr + ; + +@assign_op_expr = @assign_arith_expr | @assign_bitwise_expr + +@assign_expr = @assignexpr | @assign_op_expr + +/* + case @allocator.form of + 0 = plain + | 1 = alignment + ; +*/ + +/** + * The allocator function associated with a `new` or `new[]` expression. + * The `form` column specified whether the allocation call contains an alignment + * argument. + */ +expr_allocator( + unique int expr: @any_new_expr ref, + int func: @function ref, + int form: int ref +); + +/* + case @deallocator.form of + 0 = plain + | 1 = size + | 2 = alignment + | 3 = size_and_alignment + ; +*/ + +/** + * The deallocator function associated with a `delete`, `delete[]`, `new`, or + * `new[]` expression. For a `new` or `new[]` expression, the deallocator is the + * one used to free memory if the initialization throws an exception. + * The `form` column specifies whether the deallocation call contains a size + * argument, and alignment argument, or both. + */ +expr_deallocator( + unique int expr: @new_or_delete_expr ref, + int func: @function ref, + int form: int ref +); + +/** + * Holds if the `@conditionalexpr` is of the two operand form + * `guard ? : false`. + */ +expr_cond_two_operand( + unique int cond: @conditionalexpr ref +); + +/** + * The guard of `@conditionalexpr` `guard ? true : false` + */ +expr_cond_guard( + unique int cond: @conditionalexpr ref, + int guard: @expr ref +); + +/** + * The expression used when the guard of `@conditionalexpr` + * `guard ? true : false` holds. For the two operand form + * `guard ?: false` consider using `expr_cond_guard` instead. + */ +expr_cond_true( + unique int cond: @conditionalexpr ref, + int true: @expr ref +); + +/** + * The expression used when the guard of `@conditionalexpr` + * `guard ? true : false` does not hold. + */ +expr_cond_false( + unique int cond: @conditionalexpr ref, + int false: @expr ref +); + +/** A string representation of the value. */ +values( + unique int id: @value, + string str: string ref +); + +/** The actual text in the source code for the value, if any. */ +valuetext( + unique int id: @value ref, + string text: string ref +); + +valuebind( + int val: @value ref, + unique int expr: @expr ref +); + +fieldoffsets( + unique int id: @variable ref, + int byteoffset: int ref, + int bitoffset: int ref +); + +bitfield( + unique int id: @variable ref, + int bits: int ref, + int declared_bits: int ref +); + +/* TODO +memberprefix( + int member: @expr ref, + int prefix: @expr ref +); +*/ + +/* + kind(1) = mbrcallexpr + kind(2) = mbrptrcallexpr + kind(3) = mbrptrmbrcallexpr + kind(4) = ptrmbrptrmbrcallexpr + kind(5) = mbrreadexpr // x.y + kind(6) = mbrptrreadexpr // p->y + kind(7) = mbrptrmbrreadexpr // x.*pm + kind(8) = mbrptrmbrptrreadexpr // x->*pm + kind(9) = staticmbrreadexpr // static x.y + kind(10) = staticmbrptrreadexpr // static p->y +*/ +/* TODO +memberaccess( + int member: @expr ref, + int kind: int ref +); +*/ + +initialisers( + unique int init: @initialiser, + int var: @accessible ref, + unique int expr: @expr ref, + int location: @location_expr ref +); + +/** + * An ancestor for the expression, for cases in which we cannot + * otherwise find the expression's parent. + */ +expr_ancestor( + int exp: @expr ref, + int ancestor: @element ref +); + +exprs( + unique int id: @expr, + int kind: int ref, + int location: @location_expr ref +); + +/* + case @value.category of + 1 = prval + | 2 = xval + | 3 = lval + ; +*/ +expr_types( + int id: @expr ref, + int typeid: @type ref, + int value_category: int ref +); + +case @expr.kind of + 1 = @errorexpr +| 2 = @address_of // & AddressOfExpr +| 3 = @reference_to // ReferenceToExpr (implicit?) +| 4 = @indirect // * PointerDereferenceExpr +| 5 = @ref_indirect // ReferenceDereferenceExpr (implicit?) +// ... +| 8 = @array_to_pointer // (???) +| 9 = @vacuous_destructor_call // VacuousDestructorCall +// ... +| 11 = @assume // Microsoft +| 12 = @parexpr +| 13 = @arithnegexpr +| 14 = @unaryplusexpr +| 15 = @complementexpr +| 16 = @notexpr +| 17 = @conjugation // GNU ~ operator +| 18 = @realpartexpr // GNU __real +| 19 = @imagpartexpr // GNU __imag +| 20 = @postincrexpr +| 21 = @postdecrexpr +| 22 = @preincrexpr +| 23 = @predecrexpr +| 24 = @conditionalexpr +| 25 = @addexpr +| 26 = @subexpr +| 27 = @mulexpr +| 28 = @divexpr +| 29 = @remexpr +| 30 = @jmulexpr // C99 mul imaginary +| 31 = @jdivexpr // C99 div imaginary +| 32 = @fjaddexpr // C99 add real + imaginary +| 33 = @jfaddexpr // C99 add imaginary + real +| 34 = @fjsubexpr // C99 sub real - imaginary +| 35 = @jfsubexpr // C99 sub imaginary - real +| 36 = @paddexpr // pointer add (pointer + int or int + pointer) +| 37 = @psubexpr // pointer sub (pointer - integer) +| 38 = @pdiffexpr // difference between two pointers +| 39 = @lshiftexpr +| 40 = @rshiftexpr +| 41 = @andexpr +| 42 = @orexpr +| 43 = @xorexpr +| 44 = @eqexpr +| 45 = @neexpr +| 46 = @gtexpr +| 47 = @ltexpr +| 48 = @geexpr +| 49 = @leexpr +| 50 = @minexpr // GNU minimum +| 51 = @maxexpr // GNU maximum +| 52 = @assignexpr +| 53 = @assignaddexpr +| 54 = @assignsubexpr +| 55 = @assignmulexpr +| 56 = @assigndivexpr +| 57 = @assignremexpr +| 58 = @assignlshiftexpr +| 59 = @assignrshiftexpr +| 60 = @assignandexpr +| 61 = @assignorexpr +| 62 = @assignxorexpr +| 63 = @assignpaddexpr // assign pointer add +| 64 = @assignpsubexpr // assign pointer sub +| 65 = @andlogicalexpr +| 66 = @orlogicalexpr +| 67 = @commaexpr +| 68 = @subscriptexpr // access to member of an array, e.g., a[5] +// ... 69 @objc_subscriptexpr deprecated +// ... 70 @cmdaccess deprecated +// ... +| 73 = @virtfunptrexpr +| 74 = @callexpr +// ... 75 @msgexpr_normal deprecated +// ... 76 @msgexpr_super deprecated +// ... 77 @atselectorexpr deprecated +// ... 78 @atprotocolexpr deprecated +| 79 = @vastartexpr +| 80 = @vaargexpr +| 81 = @vaendexpr +| 82 = @vacopyexpr +// ... 83 @atencodeexpr deprecated +| 84 = @varaccess +| 85 = @thisaccess +// ... 86 @objc_box_expr deprecated +| 87 = @new_expr +| 88 = @delete_expr +| 89 = @throw_expr +| 90 = @condition_decl // a variable declared in a condition, e.g., if(int x = y > 2) +| 91 = @braced_init_list +| 92 = @type_id +| 93 = @runtime_sizeof +| 94 = @runtime_alignof +| 95 = @sizeof_pack +| 96 = @expr_stmt // GNU extension +| 97 = @routineexpr +| 98 = @type_operand // used to access a type in certain contexts (haven't found any examples yet....) +| 99 = @offsetofexpr // offsetof ::= type and field +| 100 = @hasassignexpr // __has_assign ::= type +| 101 = @hascopyexpr // __has_copy ::= type +| 102 = @hasnothrowassign // __has_nothrow_assign ::= type +| 103 = @hasnothrowconstr // __has_nothrow_constructor ::= type +| 104 = @hasnothrowcopy // __has_nothrow_copy ::= type +| 105 = @hastrivialassign // __has_trivial_assign ::= type +| 106 = @hastrivialconstr // __has_trivial_constructor ::= type +| 107 = @hastrivialcopy // __has_trivial_copy ::= type +| 108 = @hasuserdestr // __has_user_destructor ::= type +| 109 = @hasvirtualdestr // __has_virtual_destructor ::= type +| 110 = @isabstractexpr // __is_abstract ::= type +| 111 = @isbaseofexpr // __is_base_of ::= type type +| 112 = @isclassexpr // __is_class ::= type +| 113 = @isconvtoexpr // __is_convertible_to ::= type type +| 114 = @isemptyexpr // __is_empty ::= type +| 115 = @isenumexpr // __is_enum ::= type +| 116 = @ispodexpr // __is_pod ::= type +| 117 = @ispolyexpr // __is_polymorphic ::= type +| 118 = @isunionexpr // __is_union ::= type +| 119 = @typescompexpr // GNU __builtin_types_compatible ::= type type +| 120 = @intaddrexpr // EDG internal builtin, used to implement offsetof +// ... +| 122 = @hastrivialdestructor // __has_trivial_destructor ::= type +| 123 = @literal +| 124 = @uuidof +| 127 = @aggregateliteral +| 128 = @delete_array_expr +| 129 = @new_array_expr +// ... 130 @objc_array_literal deprecated +// ... 131 @objc_dictionary_literal deprecated +| 132 = @foldexpr +// ... +| 200 = @ctordirectinit +| 201 = @ctorvirtualinit +| 202 = @ctorfieldinit +| 203 = @ctordelegatinginit +| 204 = @dtordirectdestruct +| 205 = @dtorvirtualdestruct +| 206 = @dtorfielddestruct +// ... +| 210 = @static_cast +| 211 = @reinterpret_cast +| 212 = @const_cast +| 213 = @dynamic_cast +| 214 = @c_style_cast +| 215 = @lambdaexpr +| 216 = @param_ref +| 217 = @noopexpr +// ... +| 294 = @istriviallyconstructibleexpr +| 295 = @isdestructibleexpr +| 296 = @isnothrowdestructibleexpr +| 297 = @istriviallydestructibleexpr +| 298 = @istriviallyassignableexpr +| 299 = @isnothrowassignableexpr +| 300 = @istrivialexpr +| 301 = @isstandardlayoutexpr +| 302 = @istriviallycopyableexpr +| 303 = @isliteraltypeexpr +| 304 = @hastrivialmoveconstructorexpr +| 305 = @hastrivialmoveassignexpr +| 306 = @hasnothrowmoveassignexpr +| 307 = @isconstructibleexpr +| 308 = @isnothrowconstructibleexpr +| 309 = @hasfinalizerexpr +| 310 = @isdelegateexpr +| 311 = @isinterfaceclassexpr +| 312 = @isrefarrayexpr +| 313 = @isrefclassexpr +| 314 = @issealedexpr +| 315 = @issimplevalueclassexpr +| 316 = @isvalueclassexpr +| 317 = @isfinalexpr +| 319 = @noexceptexpr +| 320 = @builtinshufflevector +| 321 = @builtinchooseexpr +| 322 = @builtinaddressof +| 323 = @vec_fill +| 324 = @builtinconvertvector +| 325 = @builtincomplex +| 326 = @spaceshipexpr +| 327 = @co_await +| 328 = @co_yield +| 329 = @temp_init +; + +@var_args_expr = @vastartexpr + | @vaendexpr + | @vaargexpr + | @vacopyexpr + ; + +@builtin_op = @var_args_expr + | @noopexpr + | @offsetofexpr + | @intaddrexpr + | @hasassignexpr + | @hascopyexpr + | @hasnothrowassign + | @hasnothrowconstr + | @hasnothrowcopy + | @hastrivialassign + | @hastrivialconstr + | @hastrivialcopy + | @hastrivialdestructor + | @hasuserdestr + | @hasvirtualdestr + | @isabstractexpr + | @isbaseofexpr + | @isclassexpr + | @isconvtoexpr + | @isemptyexpr + | @isenumexpr + | @ispodexpr + | @ispolyexpr + | @isunionexpr + | @typescompexpr + | @builtinshufflevector + | @builtinconvertvector + | @builtinaddressof + | @istriviallyconstructibleexpr + | @isdestructibleexpr + | @isnothrowdestructibleexpr + | @istriviallydestructibleexpr + | @istriviallyassignableexpr + | @isnothrowassignableexpr + | @isstandardlayoutexpr + | @istriviallycopyableexpr + | @isliteraltypeexpr + | @hastrivialmoveconstructorexpr + | @hastrivialmoveassignexpr + | @hasnothrowmoveassignexpr + | @isconstructibleexpr + | @isnothrowconstructibleexpr + | @hasfinalizerexpr + | @isdelegateexpr + | @isinterfaceclassexpr + | @isrefarrayexpr + | @isrefclassexpr + | @issealedexpr + | @issimplevalueclassexpr + | @isvalueclassexpr + | @isfinalexpr + | @builtinchooseexpr + | @builtincomplex + ; + +new_allocated_type( + unique int expr: @new_expr ref, + int type_id: @type ref +); + +new_array_allocated_type( + unique int expr: @new_array_expr ref, + int type_id: @type ref +); + +/** + * The field being initialized by an initializer expression within an aggregate + * initializer for a class/struct/union. + */ +#keyset[aggregate, field] +aggregate_field_init( + int aggregate: @aggregateliteral ref, + int initializer: @expr ref, + int field: @membervariable ref +); + +/** + * The index of the element being initialized by an initializer expression + * within an aggregate initializer for an array. + */ +#keyset[aggregate, element_index] +aggregate_array_init( + int aggregate: @aggregateliteral ref, + int initializer: @expr ref, + int element_index: int ref +); + +@ctorinit = @ctordirectinit + | @ctorvirtualinit + | @ctorfieldinit + | @ctordelegatinginit; +@dtordestruct = @dtordirectdestruct + | @dtorvirtualdestruct + | @dtorfielddestruct; + + +condition_decl_bind( + unique int expr: @condition_decl ref, + unique int decl: @declaration ref +); + +typeid_bind( + unique int expr: @type_id ref, + int type_id: @type ref +); + +uuidof_bind( + unique int expr: @uuidof ref, + int type_id: @type ref +); + +@runtime_sizeof_or_alignof = @runtime_sizeof | @runtime_alignof; + +sizeof_bind( + unique int expr: @runtime_sizeof_or_alignof ref, + int type_id: @type ref +); + +code_block( + unique int block: @literal ref, + unique int routine: @function ref +); + +lambdas( + unique int expr: @lambdaexpr ref, + string default_capture: string ref, + boolean has_explicit_return_type: boolean ref +); + +lambda_capture( + unique int id: @lambdacapture, + int lambda: @lambdaexpr ref, + int index: int ref, + int field: @membervariable ref, + boolean captured_by_reference: boolean ref, + boolean is_implicit: boolean ref, + int location: @location_default ref +); + +@funbindexpr = @routineexpr + | @new_expr + | @delete_expr + | @delete_array_expr + | @ctordirectinit + | @ctorvirtualinit + | @ctordelegatinginit + | @dtordirectdestruct + | @dtorvirtualdestruct; + +@varbindexpr = @varaccess | @ctorfieldinit | @dtorfielddestruct; +@addressable = @function | @variable ; +@accessible = @addressable | @enumconstant ; + +@access = @varaccess | @routineexpr ; + +fold( + int expr: @foldexpr ref, + string operator: string ref, + boolean is_left_fold: boolean ref +); + +stmts( + unique int id: @stmt, + int kind: int ref, + int location: @location_stmt ref +); + +case @stmt.kind of + 1 = @stmt_expr +| 2 = @stmt_if +| 3 = @stmt_while +| 4 = @stmt_goto +| 5 = @stmt_label +| 6 = @stmt_return +| 7 = @stmt_block +| 8 = @stmt_end_test_while // do { ... } while ( ... ) +| 9 = @stmt_for +| 10 = @stmt_switch_case +| 11 = @stmt_switch +| 13 = @stmt_asm // "asm" statement or the body of an asm function +| 15 = @stmt_try_block +| 16 = @stmt_microsoft_try // Microsoft +| 17 = @stmt_decl +| 18 = @stmt_set_vla_size // C99 +| 19 = @stmt_vla_decl // C99 +| 25 = @stmt_assigned_goto // GNU +| 26 = @stmt_empty +| 27 = @stmt_continue +| 28 = @stmt_break +| 29 = @stmt_range_based_for // C++11 +// ... 30 @stmt_at_autoreleasepool_block deprecated +// ... 31 @stmt_objc_for_in deprecated +// ... 32 @stmt_at_synchronized deprecated +| 33 = @stmt_handler +// ... 34 @stmt_finally_end deprecated +| 35 = @stmt_constexpr_if +| 37 = @stmt_co_return +; + +type_vla( + int type_id: @type ref, + int decl: @stmt_vla_decl ref +); + +variable_vla( + int var: @variable ref, + int decl: @stmt_vla_decl ref +); + +if_initialization( + unique int if_stmt: @stmt_if ref, + int init_id: @stmt ref +); + +if_then( + unique int if_stmt: @stmt_if ref, + int then_id: @stmt ref +); + +if_else( + unique int if_stmt: @stmt_if ref, + int else_id: @stmt ref +); + +constexpr_if_initialization( + unique int constexpr_if_stmt: @stmt_constexpr_if ref, + int init_id: @stmt ref +); + +constexpr_if_then( + unique int constexpr_if_stmt: @stmt_constexpr_if ref, + int then_id: @stmt ref +); + +constexpr_if_else( + unique int constexpr_if_stmt: @stmt_constexpr_if ref, + int else_id: @stmt ref +); + +while_body( + unique int while_stmt: @stmt_while ref, + int body_id: @stmt ref +); + +do_body( + unique int do_stmt: @stmt_end_test_while ref, + int body_id: @stmt ref +); + +switch_initialization( + unique int switch_stmt: @stmt_switch ref, + int init_id: @stmt ref +); + +#keyset[switch_stmt, index] +switch_case( + int switch_stmt: @stmt_switch ref, + int index: int ref, + int case_id: @stmt_switch_case ref +); + +switch_body( + unique int switch_stmt: @stmt_switch ref, + int body_id: @stmt ref +); + +for_initialization( + unique int for_stmt: @stmt_for ref, + int init_id: @stmt ref +); + +for_condition( + unique int for_stmt: @stmt_for ref, + int condition_id: @expr ref +); + +for_update( + unique int for_stmt: @stmt_for ref, + int update_id: @expr ref +); + +for_body( + unique int for_stmt: @stmt_for ref, + int body_id: @stmt ref +); + +@stmtparent = @stmt | @expr_stmt ; +stmtparents( + unique int id: @stmt ref, + int index: int ref, + int parent: @stmtparent ref +); + +ishandler(unique int block: @stmt_block ref); + +@cfgnode = @stmt | @expr | @function | @initialiser ; + +stmt_decl_bind( + int stmt: @stmt_decl ref, + int num: int ref, + int decl: @declaration ref +); + +stmt_decl_entry_bind( + int stmt: @stmt_decl ref, + int num: int ref, + int decl_entry: @element ref +); + +@functionorblock = @function | @stmt_block; + +blockscope( + unique int block: @stmt_block ref, + int enclosing: @functionorblock ref +); + +@jump = @stmt_goto | @stmt_break | @stmt_continue; + +@jumporlabel = @jump | @stmt_label | @literal; + +jumpinfo( + unique int id: @jumporlabel ref, + string str: string ref, + int target: @stmt ref +); + +preprocdirects( + unique int id: @preprocdirect, + int kind: int ref, + int location: @location_default ref +); +case @preprocdirect.kind of + 0 = @ppd_if +| 1 = @ppd_ifdef +| 2 = @ppd_ifndef +| 3 = @ppd_elif +| 4 = @ppd_else +| 5 = @ppd_endif +| 6 = @ppd_plain_include +| 7 = @ppd_define +| 8 = @ppd_undef +| 9 = @ppd_line +| 10 = @ppd_error +| 11 = @ppd_pragma +| 12 = @ppd_objc_import +| 13 = @ppd_include_next +| 18 = @ppd_warning +; + +@ppd_include = @ppd_plain_include | @ppd_objc_import | @ppd_include_next; + +@ppd_branch = @ppd_if | @ppd_ifdef | @ppd_ifndef | @ppd_elif; + +preprocpair( + int begin : @ppd_branch ref, + int elseelifend : @preprocdirect ref +); + +preproctrue(int branch : @ppd_branch ref); +preprocfalse(int branch : @ppd_branch ref); + +preproctext( + unique int id: @preprocdirect ref, + string head: string ref, + string body: string ref +); + +includes( + unique int id: @ppd_include ref, + int included: @file ref +); + +link_targets( + unique int id: @link_target, + int binary: @file ref +); + +link_parent( + int element : @element ref, + int link_target : @link_target ref +); + +/* XML Files */ + +xmlEncoding(unique int id: @file ref, string encoding: string ref); + +xmlDTDs( + unique int id: @xmldtd, + string root: string ref, + string publicId: string ref, + string systemId: string ref, + int fileid: @file ref +); + +xmlElements( + unique int id: @xmlelement, + string name: string ref, + int parentid: @xmlparent ref, + int idx: int ref, + int fileid: @file ref +); + +xmlAttrs( + unique int id: @xmlattribute, + int elementid: @xmlelement ref, + string name: string ref, + string value: string ref, + int idx: int ref, + int fileid: @file ref +); + +xmlNs( + int id: @xmlnamespace, + string prefixName: string ref, + string URI: string ref, + int fileid: @file ref +); + +xmlHasNs( + int elementId: @xmlnamespaceable ref, + int nsId: @xmlnamespace ref, + int fileid: @file ref +); + +xmlComments( + unique int id: @xmlcomment, + string text: string ref, + int parentid: @xmlparent ref, + int fileid: @file ref +); + +xmlChars( + unique int id: @xmlcharacters, + string text: string ref, + int parentid: @xmlparent ref, + int idx: int ref, + int isCDATA: int ref, + int fileid: @file ref +); + +@xmlparent = @file | @xmlelement; +@xmlnamespaceable = @xmlelement | @xmlattribute; + +xmllocations( + int xmlElement: @xmllocatable ref, + int location: @location_default ref +); + +@xmllocatable = @xmlcharacters + | @xmlelement + | @xmlcomment + | @xmlattribute + | @xmldtd + | @file + | @xmlnamespace; diff --git a/cpp/ql/lib/upgrades/e9a518baf14f4322ac243578a8e1391386ff030f/stmtparents.ql b/cpp/ql/lib/upgrades/e9a518baf14f4322ac243578a8e1391386ff030f/stmtparents.ql new file mode 100644 index 00000000000..e4865284f9c --- /dev/null +++ b/cpp/ql/lib/upgrades/e9a518baf14f4322ac243578a8e1391386ff030f/stmtparents.ql @@ -0,0 +1,17 @@ +class Element extends @element { + string toString() { none() } +} + +class Stmt extends @stmt { + string toString() { none() } +} + +predicate isStmtWithInitializer(Stmt stmt) { + exists(int kind | stmts(stmt, kind, _) | kind = 2 or kind = 11 or kind = 35) +} + +from Stmt child, int index, int index_new, Element parent +where + stmtparents(child, index, parent) and + if isStmtWithInitializer(parent) then index_new = index + 1 else index_new = index +select child, index_new, parent diff --git a/cpp/ql/lib/upgrades/e9a518baf14f4322ac243578a8e1391386ff030f/upgrade.properties b/cpp/ql/lib/upgrades/e9a518baf14f4322ac243578a8e1391386ff030f/upgrade.properties new file mode 100644 index 00000000000..46b0525f473 --- /dev/null +++ b/cpp/ql/lib/upgrades/e9a518baf14f4322ac243578a8e1391386ff030f/upgrade.properties @@ -0,0 +1,4 @@ +description: Support C++17 if and switch initializers +compatibility: partial +exprparents.rel: run exprparents.qlo +stmtparents.rel: run stmtparents.qlo From 894380d7014da9b9698f7427de3586e8aa8b1527 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Sat, 30 Apr 2022 09:29:26 +0200 Subject: [PATCH 86/91] C++: Update stats file --- cpp/ql/lib/semmlecode.cpp.dbscheme.stats | 10931 +++++++++++---------- 1 file changed, 5560 insertions(+), 5371 deletions(-) diff --git a/cpp/ql/lib/semmlecode.cpp.dbscheme.stats b/cpp/ql/lib/semmlecode.cpp.dbscheme.stats index 472bfd55aaf..52676255dbd 100644 --- a/cpp/ql/lib/semmlecode.cpp.dbscheme.stats +++ b/cpp/ql/lib/semmlecode.cpp.dbscheme.stats @@ -2,7 +2,7 @@ @compilation - 9999 + 9980 @externalDataElement @@ -18,67 +18,67 @@ @location_stmt - 3805195 - - - @location_default - 30199622 - - - @location_expr - 13127189 + 3805462 @diagnostic - 667106 + 632933 @file - 124610 + 124189 @folder - 15576 + 15994 + + + @location_expr + 13128112 + + + @location_default + 30128761 @macroinvocation - 34062185 + 33895024 @function - 4759741 + 4726273 @fun_decl - 5131684 - - - @type_decl - 3292736 - - - @namespace_decl - 307863 - - - @using - 375719 - - - @static_assert - 130790 + 5096962 @var_decl - 8586319 + 8543232 + + + @type_decl + 3283977 + + + @namespace_decl + 306979 + + + @using + 374921 + + + @static_assert + 130414 @parameter - 6696867 + 6660155 @membervariable - 1050009 + 1050083 @globalvariable @@ -86,63 +86,63 @@ @localvariable - 581746 + 581698 @enumconstant - 240596 + 240613 @builtintype - 22184 + 22109 @derivedtype - 4456239 + 4413446 @decltype - 31152 + 31047 @usertype - 5362968 + 5342989 @mangledname - 1733994 + 1730792 @type_mention - 4011226 + 4011508 @routinetype - 543059 + 546982 @ptrtomember - 38232 + 38103 @specifier - 25016 + 24932 @gnuattribute - 663644 + 664228 @stdattribute - 468476 + 469328 @alignas - 8968 + 8937 @declspec - 238528 + 238544 @msattribute @@ -150,15 +150,15 @@ @attribute_arg_token - 25960 + 25402 @attribute_arg_constant - 324742 + 326469 @attribute_arg_type - 1535 + 470 @attribute_arg_empty @@ -166,99 +166,207 @@ @derivation - 397280 + 402388 @frienddecl - 710038 + 715075 @comment - 9024167 + 9004230 @namespace - 12744 + 12701 @specialnamequalifyingelement - 472 + 470 @namequalifier - 1593457 + 1618961 @value - 10645398 + 10646146 @initialiser - 1736804 + 1731850 + + + @lambdacapture + 28224 + + + @stmt_expr + 1480414 + + + @stmt_if + 723174 + + + @stmt_while + 30207 + + + @stmt_label + 53046 + + + @stmt_return + 1306346 + + + @stmt_block + 1446530 + + + @stmt_end_test_while + 148604 + + + @stmt_for + 61324 + + + @stmt_switch_case + 191408 + + + @stmt_switch + 20901 + + + @stmt_try_block + 42701 + + + @stmt_decl + 608508 + + + @stmt_empty + 193487 + + + @stmt_continue + 22507 + + + @stmt_break + 102234 + + + @stmt_range_based_for + 8467 + + + @stmt_handler + 59432 + + + @stmt_constexpr_if + 52508 + + + @stmt_goto + 110490 + + + @stmt_asm + 110589 + + + @stmt_microsoft_try + 163 + + + @stmt_set_vla_size + 26 + + + @stmt_vla_decl + 21 + + + @stmt_assigned_goto + 9059 + + + @stmt_co_return + 2 @delete_array_expr - 1413 + 1410 @new_array_expr - 5113 + 5099 @ctordirectinit - 111984 + 112813 @ctorvirtualinit - 6546 + 6534 @ctorfieldinit - 199374 + 200789 @ctordelegatinginit - 7857 + 3347 @dtordirectdestruct - 41421 + 41715 @dtorvirtualdestruct - 4093 + 4122 @dtorfielddestruct - 41351 + 41644 @static_cast - 212170 + 211822 @reinterpret_cast - 30881 + 30813 @const_cast - 35398 + 35320 + + + @dynamic_cast + 1040 @c_style_cast - 4209375 + 4209393 @lambdaexpr - 21712 + 21639 @param_ref - 403726 + 375289 @errorexpr - 54470 + 46823 @address_of @@ -266,143 +374,139 @@ @reference_to - 1615323 + 1596143 @indirect - 292094 + 292115 @ref_indirect - 1941936 + 1934537 @array_to_pointer - 1424786 + 1424883 @vacuous_destructor_call - 8711 + 8138 @parexpr - 3572304 + 3572547 @arithnegexpr - 650877 - - - @unaryplusexpr - 2903 + 650874 @complementexpr - 27785 + 27787 @notexpr - 277973 + 277992 @postincrexpr - 61769 + 61774 @postdecrexpr - 41857 + 41860 @preincrexpr - 70302 + 70307 @predecrexpr - 26107 + 26108 @conditionalexpr - 654457 + 654502 @addexpr - 400330 + 400358 @subexpr - 339317 + 339340 @mulexpr - 305870 + 305891 @divexpr - 133721 + 133731 @remexpr - 15707 + 15819 @paddexpr - 86499 + 86505 @psubexpr - 49678 + 49681 @pdiffexpr - 35608 + 35529 @lshiftexpr - 562948 + 562988 @rshiftexpr - 141607 + 141617 @andexpr - 491783 + 491818 @orexpr - 146257 + 146267 @xorexpr - 53934 + 53938 @eqexpr - 468840 + 468873 @neexpr - 300391 + 300411 @gtexpr - 101009 + 100669 @ltexpr - 108562 + 106314 @geexpr - 58985 + 58989 @leexpr - 212126 + 212141 @assignexpr - 933354 + 933419 @assignaddexpr @@ -410,19 +514,19 @@ @assignsubexpr - 15624 + 11157 @assignmulexpr - 7586 + 7170 @assigndivexpr - 4971 + 4972 @assignremexpr - 1625 + 419 @assignlshiftexpr @@ -434,123 +538,119 @@ @assignandexpr - 5780 + 4852 @assignorexpr - 23853 + 23851 @assignxorexpr - 21802 + 21804 @assignpaddexpr - 13576 - - - @assignpsubexpr - 1148 + 13577 @andlogicalexpr - 248991 + 249008 @orlogicalexpr - 863957 + 864018 @commaexpr - 181070 + 181539 @subscriptexpr - 367945 + 367915 @callexpr - 310110 + 309063 @vastartexpr - 3713 + 3703 @vaendexpr - 2832 + 2822 + + + @vacopyexpr + 140 @varaccess - 6005942 + 6006364 @thisaccess - 1129204 + 1125933 @new_expr - 55240 + 47598 @delete_expr - 11649 + 11732 @throw_expr - 21806 + 21765 @condition_decl - 38495 + 38595 @braced_init_list - 1005 + 1008 @type_id - 36173 + 36430 @runtime_sizeof - 289248 + 289268 @runtime_alignof - 48208 + 48550 @sizeof_pack - 5664 + 5644 @routineexpr - 3065607 + 3047613 @type_operand - 1125950 - - - @offsetofexpr - 20101 + 1126029 @isemptyexpr - 2310 + 2305 @ispodexpr - 637 + 636 @hastrivialdestructor - 472 + 470 @literal - 4351842 + 4350894 @aggregateliteral @@ -558,31 +658,39 @@ @istrivialexpr - 944 + 940 @istriviallycopyableexpr - 3776 + 3763 + + + @isconstructibleexpr + 2724 @isfinalexpr - 2100 + 2096 @noexceptexpr - 35718 + 23574 @builtinaddressof - 13189 + 13282 @temp_init - 759580 + 760683 @assume - 3209 + 3200 + + + @unaryplusexpr + 2903 @conjugation @@ -628,6 +736,10 @@ @maxexpr 1 + + @assignpsubexpr + 1148 + @virtfunptrexpr 1 @@ -637,12 +749,12 @@ 950 - @vacopyexpr - 139 + @expr_stmt + 94929 - @expr_stmt - 94922 + @offsetofexpr + 20103 @hasassignexpr @@ -686,7 +798,7 @@ @isabstractexpr - 17 + 18 @isbaseofexpr @@ -694,15 +806,15 @@ @isclassexpr - 1841 + 1835 @isconvtoexpr - 1155 + 1152 @isenumexpr - 1260 + 1257 @ispolyexpr @@ -714,7 +826,7 @@ @typescompexpr - 562376 + 562415 @intaddrexpr @@ -722,23 +834,19 @@ @uuidof - 20051 + 19994 @foldexpr 4 - - @dynamic_cast - 1042 - @noopexpr 37 @istriviallyconstructibleexpr - 2836 + 2829 @isdestructibleexpr @@ -754,15 +862,15 @@ @istriviallyassignableexpr - 525 + 524 @isnothrowassignableexpr - 2100 + 2096 @isstandardlayoutexpr - 840 + 838 @isliteraltypeexpr @@ -780,13 +888,9 @@ @hasnothrowmoveassignexpr 4 - - @isconstructibleexpr - 2731 - @isnothrowconstructibleexpr - 4831 + 4821 @hasfinalizerexpr @@ -826,7 +930,7 @@ @builtinchooseexpr - 9135 + 9136 @vec_fill @@ -852,157 +956,53 @@ @co_yield 1 - - @lambdacapture - 28320 - - - @stmt_expr - 1480310 - - - @stmt_if - 723123 - - - @stmt_while - 30265 - - - @stmt_goto - 110482 - - - @stmt_label - 53042 - - - @stmt_return - 1310769 - - - @stmt_block - 1451428 - - - @stmt_end_test_while - 148593 - - - @stmt_for - 61319 - - - @stmt_switch_case - 190914 - - - @stmt_switch - 20900 - - - @stmt_asm - 110581 - - - @stmt_try_block - 42591 - - - @stmt_decl - 609675 - - - @stmt_empty - 193503 - - - @stmt_continue - 22506 - - - @stmt_break - 102528 - - - @stmt_range_based_for - 8496 - - - @stmt_handler - 59279 - - - @stmt_microsoft_try - 163 - - - @stmt_set_vla_size - 26 - - - @stmt_vla_decl - 21 - - - @stmt_assigned_goto - 9058 - - - @stmt_constexpr_if - 52624 - - - @stmt_co_return - 2 - @ppd_if - 671197 + 672225 @ppd_ifdef - 266213 + 265314 @ppd_ifndef - 269517 + 268607 @ppd_elif - 25488 + 25402 @ppd_else - 211460 + 210746 @ppd_endif - 1206927 + 1206147 @ppd_plain_include - 314830 + 313767 @ppd_define - 2443222 + 2437824 @ppd_undef - 264325 + 262021 @ppd_pragma - 313333 + 312641 @ppd_include_next - 1888 + 1881 @ppd_line - 27783 + 27780 @ppd_error @@ -1048,11 +1048,11 @@ compilations - 9999 + 9980 id - 9999 + 9980 cwd @@ -1070,7 +1070,7 @@ 1 2 - 9999 + 9980 @@ -1096,7 +1096,7 @@ compilation_args - 656105 + 656151 id @@ -1104,11 +1104,11 @@ num - 712 + 713 arg - 34648 + 34651 @@ -1346,7 +1346,7 @@ 1 2 - 32573 + 32576 2 @@ -1367,7 +1367,7 @@ 1 2 - 33435 + 33438 2 @@ -1382,7 +1382,7 @@ compilation_compiling_files - 11494 + 11495 id @@ -1394,7 +1394,7 @@ file - 9983 + 9984 @@ -1628,7 +1628,7 @@ compilation_time - 45979 + 45982 id @@ -1644,7 +1644,7 @@ seconds - 13762 + 10779 @@ -1725,48 +1725,48 @@ 3 4 - 278 + 596 4 5 - 715 + 397 6 - 7 - 119 - - - 9 - 11 - 119 - - - 11 - 12 - 238 - - - 13 - 17 - 119 - - - 18 - 20 - 119 - - - 21 - 26 + 9 159 - 44 - 132 + 9 + 10 + 79 + + + 10 + 11 + 159 + + + 11 + 16 + 159 + + + 17 + 19 119 + + 19 + 22 + 159 + + + 26 + 97 + 159 + @@ -1833,32 +1833,32 @@ 3 4 - 1073 + 1153 4 5 - 676 + 596 5 6 - 238 + 119 6 7 - 437 + 556 7 8 - 159 + 79 8 - 10 - 238 + 9 + 318 11 @@ -1866,8 +1866,8 @@ 278 - 29 - 93 + 28 + 98 198 @@ -1916,21 +1916,16 @@ 3 4 + 79 + + + 132 + 133 39 - 5 - 6 - 39 - - - 167 - 168 - 39 - - - 178 - 179 + 141 + 142 39 @@ -1947,22 +1942,27 @@ 1 2 - 8591 + 5887 2 3 - 3420 + 2505 3 4 - 1312 + 1431 4 - 40 - 437 + 7 + 835 + + + 29 + 45 + 119 @@ -1978,22 +1978,32 @@ 1 2 - 8591 + 5369 2 3 - 2744 + 2625 3 4 - 1551 + 875 4 + 5 + 715 + + + 5 + 7 + 914 + + + 7 64 - 875 + 278 @@ -2009,12 +2019,12 @@ 1 2 - 13483 + 10461 2 3 - 278 + 318 @@ -2024,23 +2034,23 @@ diagnostic_for - 917206 + 968526 diagnostic - 667106 + 632828 compilation - 1890 + 1991 file_number - 105 + 104 file_number_diagnostic_number - 105144 + 104912 @@ -2054,17 +2064,22 @@ 1 2 - 465220 + 405919 2 3 - 177937 + 151656 3 - 9 - 23949 + 4 + 42237 + + + 4 + 8 + 33014 @@ -2080,7 +2095,7 @@ 1 2 - 667106 + 632828 @@ -2096,12 +2111,17 @@ 1 2 - 633284 + 561454 2 - 5 - 33822 + 3 + 71164 + + + 3 + 4 + 209 @@ -2117,52 +2137,67 @@ 37 38 - 105 + 104 - 78 - 79 - 210 + 38 + 39 + 104 - 119 - 120 - 105 + 77 + 78 + 104 - 155 - 156 - 105 + 79 + 80 + 104 - 156 - 157 - 210 + 198 + 199 + 104 + + + 222 + 223 + 104 352 353 - 105 + 104 353 354 - 105 + 104 - 710 - 711 - 105 + 359 + 360 + 104 + + + 418 + 419 + 104 + + + 570 + 571 + 104 756 757 - 630 + 628 1001 1002 - 210 + 209 @@ -2178,7 +2213,7 @@ 1 2 - 1890 + 1991 @@ -2194,52 +2229,67 @@ 37 38 - 105 + 104 - 78 - 79 - 210 + 38 + 39 + 104 - 119 - 120 - 105 + 77 + 78 + 104 - 155 - 156 - 105 + 79 + 80 + 104 - 156 - 157 - 210 + 198 + 199 + 104 + + + 222 + 223 + 104 352 353 - 105 + 104 353 354 - 105 + 104 - 710 - 711 - 105 + 359 + 360 + 104 + + + 418 + 419 + 104 + + + 570 + 571 + 104 756 757 - 630 + 628 1001 1002 - 210 + 209 @@ -2253,9 +2303,9 @@ 12 - 6351 - 6352 - 105 + 6038 + 6039 + 104 @@ -2269,9 +2319,9 @@ 12 - 18 - 19 - 105 + 19 + 20 + 104 @@ -2287,7 +2337,7 @@ 1001 1002 - 105 + 104 @@ -2303,42 +2353,42 @@ 2 3 - 25734 + 25677 + + + 5 + 6 + 5974 6 7 - 12814 + 6183 7 8 - 105 + 17083 8 9 - 42015 + 9327 9 10 - 8193 + 22428 10 11 - 525 + 14358 11 - 12 - 9978 - - - 12 - 14 - 5777 + 13 + 3877 @@ -2354,37 +2404,42 @@ 2 3 - 25734 + 25677 8 9 - 4831 + 19494 9 10 - 37499 + 15930 10 - 11 - 105 - - - 11 - 12 - 20587 + 13 + 6917 13 - 16 - 8193 + 14 + 13624 - 17 - 19 - 8193 + 14 + 15 + 2515 + + + 15 + 16 + 12472 + + + 16 + 20 + 8279 @@ -2400,7 +2455,7 @@ 1 2 - 105144 + 104912 @@ -2410,19 +2465,19 @@ compilation_finished - 9999 + 9980 id - 9999 + 9980 cpu_seconds - 8319 + 7181 elapsed_seconds - 139 + 115 @@ -2436,7 +2491,7 @@ 1 2 - 9999 + 9980 @@ -2452,7 +2507,7 @@ 1 2 - 9999 + 9980 @@ -2468,17 +2523,17 @@ 1 2 - 7160 + 5666 2 3 - 915 + 1063 3 - 11 - 243 + 14 + 451 @@ -2494,12 +2549,12 @@ 1 2 - 8064 + 6661 2 3 - 254 + 520 @@ -2527,49 +2582,39 @@ 5 11 - - 6 - 7 - 11 - 7 8 11 - 13 - 14 + 18 + 19 11 - 23 - 24 + 27 + 28 11 - 64 - 65 + 83 + 84 11 - 95 - 96 + 190 + 191 11 - 136 - 137 + 258 + 259 11 - 254 - 255 - 11 - - - 257 - 258 + 272 + 273 11 @@ -2598,49 +2643,39 @@ 5 11 - - 6 - 7 - 11 - 7 8 11 - 13 - 14 + 18 + 19 11 - 22 - 23 + 25 + 26 11 - 63 - 64 + 79 + 80 11 - 91 - 92 + 135 + 136 11 - 94 - 95 + 169 + 170 11 - 203 - 204 - 11 - - - 233 - 234 + 225 + 226 11 @@ -2867,11 +2902,11 @@ sourceLocationPrefix - 472 + 470 prefix - 472 + 470 @@ -4365,31 +4400,31 @@ locations_default - 30199622 + 30128761 id - 30199622 + 30128761 container - 140186 + 140184 startLine - 2121681 + 2114992 startColumn - 37288 + 37162 endLine - 2124985 + 2117814 endColumn - 48616 + 48452 @@ -4403,7 +4438,7 @@ 1 2 - 30199622 + 30128761 @@ -4419,7 +4454,7 @@ 1 2 - 30199622 + 30128761 @@ -4435,7 +4470,7 @@ 1 2 - 30199622 + 30128761 @@ -4451,7 +4486,7 @@ 1 2 - 30199622 + 30128761 @@ -4467,7 +4502,7 @@ 1 2 - 30199622 + 30128761 @@ -4483,67 +4518,67 @@ 1 2 - 16048 + 16464 2 12 - 10856 + 10819 13 20 - 11800 + 11760 21 36 - 11328 + 11289 36 55 - 11328 + 11289 55 77 - 10856 + 10819 77 102 - 10856 + 10819 102 - 146 - 10856 + 149 + 10819 - 148 - 223 - 10856 + 149 + 227 + 11289 - 226 + 228 350 - 11800 + 11289 358 628 - 10856 + 10819 671 1926 - 10856 + 10819 - 2171 + 2168 2380 - 1888 + 1881 @@ -4559,67 +4594,67 @@ 1 2 - 16048 + 16464 2 9 - 10856 + 10819 9 16 - 11800 + 11760 16 25 - 11328 + 11289 25 40 - 10856 + 10819 40 57 - 10856 + 10819 58 72 - 10856 + 10819 73 - 99 - 10856 + 103 + 11289 - 101 + 106 141 - 12272 + 11760 148 226 - 10856 + 11289 226 - 365 - 10856 + 373 + 10819 - 372 - 1253 - 10856 + 381 + 1456 + 10819 - 1452 - 1616 - 1888 + 1464 + 1613 + 1411 @@ -4635,67 +4670,67 @@ 1 2 - 16048 + 16464 2 4 - 8968 + 8937 4 5 - 7552 + 7526 5 6 - 7552 + 7526 6 8 - 12272 + 11760 8 13 - 11800 + 12230 13 17 - 10856 + 10819 17 25 - 11328 + 11289 25 31 - 11800 + 11760 31 38 - 10856 + 10819 38 52 - 10856 + 10819 52 64 - 10856 + 10819 65 77 - 9440 + 9408 @@ -4711,67 +4746,67 @@ 1 2 - 16048 + 16464 2 9 - 10856 + 10819 9 16 - 11800 + 11760 16 25 - 11328 + 11289 25 40 - 10856 + 10819 40 57 - 10856 + 10819 58 71 - 10856 + 10819 72 - 97 - 10856 + 98 + 10819 - 97 + 101 140 - 11800 + 11760 140 - 225 - 11328 + 224 + 10819 - 225 - 365 - 10856 + 224 + 360 + 10819 - 372 - 1253 - 10856 + 364 + 1185 + 10819 - 1449 - 1613 - 1888 + 1254 + 1610 + 2352 @@ -4787,62 +4822,62 @@ 1 2 - 16048 + 16464 2 10 - 11328 + 11289 10 14 - 10856 + 10819 14 21 - 11328 + 11289 22 31 - 11328 + 11289 31 39 - 12744 + 12701 39 48 - 12272 + 12230 48 56 - 11800 + 11760 56 64 - 11800 + 11760 64 72 - 10856 + 10819 72 77 - 11328 + 11289 77 90 - 8496 + 8467 @@ -4858,719 +4893,59 @@ 1 2 - 598507 + 581905 2 3 - 302085 + 318001 3 4 - 190691 + 199456 4 6 - 171339 + 160882 6 10 - 192579 + 190048 10 16 - 161427 + 166057 16 - 24 - 159539 - - - 24 - 42 - 159539 - - - 42 - 142 - 159539 - - - 142 - 298 - 26432 - - - - - - - startLine - container - - - 12 - - - 1 - 2 - 881241 - - - 2 - 3 - 262437 - - - 3 - 4 - 135466 - - - 4 - 7 - 195883 - - - 7 - 11 - 177475 - - - 11 - 16 - 161899 - - - 16 - 29 - 159539 - - - 29 - 298 - 147738 - - - - - - - startLine - startColumn - - - 12 - - - 1 - 2 - 625884 - - - 2 - 3 - 296893 - - - 3 - 4 - 203435 - - - 4 - 6 - 195411 - - - 6 - 9 - 178891 - - - 9 - 13 - 167091 - - - 13 - 19 - 178419 - - - 19 - 29 - 162843 - - - 29 - 52 - 112810 - - - - - - - startLine - endLine - - - 12 - - - 1 - 2 - 1552910 - - - 2 - 3 - 345510 - - - 3 - 5 - 171811 - - - 5 - 16 - 51449 - - - - - - - startLine - endColumn - - - 12 - - - 1 - 2 - 601811 - - - 2 - 3 - 305389 - - - 3 - 4 - 194939 - - - 4 - 6 - 174171 - - - 6 - 9 - 165675 - - - 9 - 14 - 172755 - - - 14 - 20 - 164259 - - - 20 - 31 - 171339 - - - 31 - 58 - 160955 - - - 58 - 67 - 10384 - - - - - - - startColumn - id - - - 12 - - - 1 - 31 - 2832 - - - 42 - 85 - 2832 - - - 86 - 128 - 2832 - - - 129 - 230 - 2832 - - - 247 - 292 - 2832 - - - 292 - 360 - 2832 - - - 377 - 459 - 2832 - - - 476 - 559 - 2832 - - - 565 - 620 - 2832 - - - 625 - 689 - 2832 - - - 700 - 795 - 2832 - - - 820 - 1546 - 2832 - - - 1698 - 5668 - 2832 - - - 15394 - 15395 - 472 - - - - - - - startColumn - container - - - 12 - - - 1 - 18 - 2832 - - - 23 - 35 - 3304 - - - 37 - 43 - 2832 - - - 44 - 61 - 2832 - - - 65 - 73 - 2832 - - - 73 - 83 - 2832 - - - 83 - 96 - 2832 - - - 96 - 100 - 2832 - - - 101 - 103 - 2832 - - - 104 - 108 - 2832 - - - 108 - 115 - 2832 - - - 116 - 150 - 3304 - - - 152 - 298 - 2360 - - - - - - - startColumn - startLine - - - 12 - - - 1 - 19 - 2832 - - - 30 - 72 - 2832 - - - 83 - 122 - 2832 - - - 123 - 204 - 2832 - - - 213 - 262 - 2832 - - - 264 - 324 - 2832 - - - 324 - 380 - 2832 - - - 404 - 439 - 2832 - - - 455 - 475 - 2832 - - - 477 - 510 - 2832 - - - 517 - 579 - 2832 - - - 588 - 836 - 2832 - - - 1112 - 2199 - 2832 - - - 2401 - 2402 - 472 - - - - - - - startColumn - endLine - - - 12 - - - 1 - 19 - 2832 - - - 30 - 72 - 2832 - - - 83 - 122 - 2832 - - - 123 - 204 - 2832 - - - 213 - 262 - 2832 - - - 264 - 324 - 2832 - - - 324 - 380 - 2832 - - - 405 - 439 - 2832 - - - 455 - 475 - 2832 - - - 476 - 509 - 2832 - - - 520 - 578 - 2832 - - - 588 - 835 - 2832 - - - 1115 - 2200 - 2832 - - - 2409 - 2410 - 472 - - - - - - - startColumn - endColumn - - - 12 - - - 1 - 7 - 2832 - - - 7 - 10 - 1888 - - - 10 - 13 - 2832 - - - 13 - 19 - 2832 - - - 19 - 23 - 2832 - - - 23 25 - 3304 + 168879 25 - 31 - 2832 - - - 31 - 36 - 2832 - - - 38 - 41 - 2360 - - - 41 46 - 3304 + 163704 46 - 50 - 3304 + 169 + 159000 - 51 - 62 - 2832 - - - 63 - 81 - 2832 - - - 85 - 86 - 472 + 170 + 299 + 7056 - endLine - id - - - 12 - - - 1 - 2 - 600867 - - - 2 - 3 - 301141 - - - 3 - 4 - 198243 - - - 4 - 6 - 165203 - - - 6 - 10 - 189275 - - - 10 - 16 - 163315 - - - 16 - 24 - 161899 - - - 24 - 43 - 162371 - - - 43 - 145 - 160011 - - - 146 - 298 - 22656 - - - - - - - endLine + startLine container @@ -5579,42 +4954,702 @@ 1 2 - 883129 + 869799 2 3 - 275181 + 280838 3 5 - 194467 + 191459 5 8 - 177947 + 181110 8 12 - 159539 + 162293 12 18 - 171811 + 166527 18 39 - 160011 + 159941 39 - 298 - 102898 + 299 + 103021 + + + + + + + startLine + startColumn + + + 12 + + + 1 + 2 + 612482 + + + 2 + 3 + 313297 + + + 3 + 4 + 202749 + + + 4 + 6 + 184403 + + + 6 + 9 + 180639 + + + 9 + 13 + 166997 + + + 13 + 19 + 173583 + + + 19 + 29 + 167468 + + + 29 + 52 + 113370 + + + + + + + startLine + endLine + + + 12 + + + 1 + 2 + 1545788 + + + 2 + 3 + 351401 + + + 3 + 5 + 164645 + + + 5 + 16 + 53157 + + + + + + + startLine + endColumn + + + 12 + + + 1 + 2 + 586609 + + + 2 + 3 + 318942 + + + 3 + 4 + 201808 + + + 4 + 6 + 167468 + + + 6 + 9 + 160412 + + + 9 + 14 + 178287 + + + 14 + 21 + 176406 + + + 21 + 32 + 163234 + + + 32 + 61 + 159000 + + + 61 + 66 + 2822 + + + + + + + startColumn + id + + + 12 + + + 1 + 31 + 2822 + + + 42 + 85 + 2822 + + + 86 + 128 + 2822 + + + 129 + 229 + 2822 + + + 248 + 292 + 2822 + + + 292 + 360 + 2822 + + + 376 + 459 + 2822 + + + 476 + 558 + 2822 + + + 565 + 620 + 2822 + + + 626 + 699 + 2822 + + + 699 + 796 + 2822 + + + 819 + 1546 + 2822 + + + 1705 + 5645 + 2822 + + + 15306 + 15307 + 470 + + + + + + + startColumn + container + + + 12 + + + 1 + 18 + 2822 + + + 23 + 35 + 3292 + + + 38 + 43 + 2822 + + + 44 + 61 + 2822 + + + 65 + 73 + 2822 + + + 73 + 83 + 2822 + + + 83 + 95 + 2822 + + + 96 + 101 + 3292 + + + 101 + 105 + 3292 + + + 106 + 111 + 2822 + + + 111 + 123 + 2822 + + + 127 + 153 + 2822 + + + 169 + 299 + 1881 + + + + + + + startColumn + startLine + + + 12 + + + 1 + 19 + 2822 + + + 30 + 72 + 2822 + + + 83 + 122 + 2822 + + + 122 + 205 + 2822 + + + 214 + 261 + 2822 + + + 264 + 322 + 2822 + + + 325 + 380 + 2822 + + + 403 + 436 + 2822 + + + 454 + 474 + 2822 + + + 477 + 514 + 2822 + + + 517 + 586 + 2822 + + + 587 + 831 + 2822 + + + 1116 + 2197 + 2822 + + + 2387 + 2388 + 470 + + + + + + + startColumn + endLine + + + 12 + + + 1 + 19 + 2822 + + + 30 + 72 + 2822 + + + 83 + 122 + 2822 + + + 122 + 205 + 2822 + + + 214 + 261 + 2822 + + + 264 + 322 + 2822 + + + 325 + 380 + 2822 + + + 403 + 435 + 2822 + + + 454 + 474 + 2822 + + + 476 + 513 + 2822 + + + 520 + 585 + 2822 + + + 587 + 831 + 2822 + + + 1121 + 2205 + 2822 + + + 2383 + 2384 + 470 + + + + + + + startColumn + endColumn + + + 12 + + + 1 + 7 + 2822 + + + 7 + 11 + 3292 + + + 11 + 16 + 3292 + + + 16 + 22 + 2822 + + + 22 + 24 + 2352 + + + 24 + 27 + 3292 + + + 28 + 33 + 2822 + + + 33 + 40 + 3292 + + + 40 + 43 + 2822 + + + 43 + 49 + 2822 + + + 49 + 54 + 2822 + + + 54 + 74 + 2822 + + + 75 + 86 + 1881 + + + + + + + endLine + id + + + 12 + + + 1 + 2 + 589902 + + + 2 + 3 + 312356 + + + 3 + 4 + 198986 + + + 4 + 6 + 161352 + + + 6 + 10 + 189577 + + + 10 + 16 + 163704 + + + 16 + 25 + 171231 + + + 25 + 45 + 159471 + + + 45 + 160 + 159941 + + + 160 + 299 + 11289 + + + + + + + endLine + container + + + 12 + + + 1 + 2 + 881560 + + + 2 + 3 + 270019 + + + 3 + 4 + 123249 + + + 4 + 6 + 142065 + + + 6 + 10 + 195693 + + + 10 + 15 + 168409 + + + 15 + 26 + 165586 + + + 26 + 120 + 159471 + + + 121 + 299 + 11760 @@ -5630,22 +5665,22 @@ 1 2 - 1542998 + 1538732 2 3 - 354006 + 349048 3 5 - 170395 + 172172 5 10 - 57585 + 57861 @@ -5661,47 +5696,47 @@ 1 2 - 625412 + 621890 2 3 - 301613 + 304359 3 4 - 204852 + 204631 4 6 - 194467 + 186755 6 9 - 175115 + 177817 9 13 - 172755 + 169349 13 19 - 179363 + 174524 19 - 30 - 165203 + 29 + 162764 - 30 + 29 52 - 106202 + 115722 @@ -5717,52 +5752,52 @@ 1 2 - 605115 + 596488 2 3 - 305389 + 311415 3 4 - 200603 + 198986 4 6 - 168507 + 171701 6 9 - 164259 + 157589 9 14 - 174171 + 173583 14 - 20 - 161899 + 21 + 180169 - 20 - 30 - 159539 + 21 + 32 + 165116 - 30 - 54 - 159539 + 32 + 60 + 159000 - 54 - 66 - 25960 + 60 + 65 + 3763 @@ -5778,67 +5813,67 @@ 1 2 - 5192 + 5174 2 8 - 3776 + 3763 9 - 184 - 3776 + 186 + 3763 196 295 - 3776 + 3763 - 297 - 492 - 3776 + 298 + 498 + 3763 - 506 + 503 553 - 3776 + 3763 - 560 - 632 - 3776 + 563 + 634 + 3763 - 638 - 760 - 3776 + 639 + 762 + 3763 763 871 - 3776 + 3763 - 878 - 1082 - 3776 + 879 + 1081 + 3763 - 1085 + 1083 1286 - 3776 + 3763 - 1307 + 1309 1590 - 3776 + 3763 - 1674 - 2430 - 1888 + 1682 + 2419 + 1881 @@ -5854,67 +5889,67 @@ 1 2 - 5664 + 5644 2 6 - 3776 + 3763 6 65 - 3776 + 3763 - 69 - 99 - 3776 + 70 + 100 + 3763 - 99 - 112 - 4248 + 100 + 111 + 3763 - 113 + 112 122 - 3776 + 3763 - 124 - 140 - 3776 + 122 + 134 + 3763 - 143 - 153 - 3776 + 139 + 152 + 3763 - 153 + 152 160 - 3776 + 4233 160 172 - 3776 + 3763 172 176 - 3776 + 3763 176 208 - 3776 + 3763 240 - 298 - 944 + 299 + 940 @@ -5930,67 +5965,67 @@ 1 2 - 5664 + 5644 2 8 - 3776 + 3763 9 106 - 3776 + 3763 - 152 - 242 - 3776 + 155 + 241 + 3763 253 - 333 - 3776 + 335 + 3763 - 339 - 430 - 3776 + 340 + 426 + 3763 - 434 - 490 - 3776 + 437 + 488 + 3763 - 493 - 572 - 3776 + 488 + 574 + 3763 - 572 - 623 - 3776 + 575 + 628 + 3763 - 629 - 696 - 3776 + 630 + 698 + 4233 - 703 - 814 - 3776 + 701 + 817 + 3763 - 822 - 994 - 3776 + 843 + 1107 + 3763 - 1107 - 1175 - 1416 + 1160 + 1174 + 940 @@ -6006,67 +6041,67 @@ 1 2 - 6136 + 6115 2 4 - 3776 + 3763 4 8 - 3776 + 4233 8 - 14 - 3776 + 15 + 3763 - 14 - 22 - 3776 + 15 + 23 + 3763 - 22 + 23 29 - 4248 + 3763 29 35 - 4248 + 4233 35 39 - 3304 + 3292 39 42 - 3304 + 3763 42 44 - 3304 + 2822 44 46 - 3776 + 3763 46 - 50 - 4248 + 49 + 3763 - 51 + 49 53 - 944 + 1411 @@ -6082,67 +6117,67 @@ 1 2 - 5664 + 5644 2 8 - 3776 + 3763 9 - 153 - 3776 + 156 + 3763 - 157 - 243 - 3776 + 159 + 240 + 3763 - 250 - 333 - 3776 + 251 + 334 + 3763 - 341 - 432 - 3776 + 342 + 430 + 3763 - 432 - 488 - 3776 + 435 + 490 + 4233 - 495 - 571 - 3776 + 504 + 576 + 3763 - 573 - 621 - 3776 + 576 + 631 + 3763 - 628 - 700 - 3776 + 635 + 702 + 3763 - 703 - 815 - 3776 + 702 + 813 + 3763 - 819 - 994 - 3776 + 838 + 1109 + 3763 - 1108 - 1177 - 1416 + 1161 + 1181 + 940 @@ -6152,11 +6187,11 @@ locations_stmt - 3805195 + 3805462 id - 3805195 + 3805462 container @@ -6164,7 +6199,7 @@ startLine - 199402 + 199416 startColumn @@ -6172,7 +6207,7 @@ endLine - 193680 + 193694 endColumn @@ -6190,7 +6225,7 @@ 1 2 - 3805195 + 3805462 @@ -6206,7 +6241,7 @@ 1 2 - 3805195 + 3805462 @@ -6222,7 +6257,7 @@ 1 2 - 3805195 + 3805462 @@ -6238,7 +6273,7 @@ 1 2 - 3805195 + 3805462 @@ -6254,7 +6289,7 @@ 1 2 - 3805195 + 3805462 @@ -6655,67 +6690,67 @@ 1 2 - 21492 + 21494 2 3 - 15258 + 15259 3 4 - 12448 + 12449 4 6 - 14417 + 14418 6 8 - 12489 + 12490 8 11 - 16673 + 16674 11 16 - 17226 + 17228 16 22 - 15319 + 15320 22 29 - 16939 + 16941 29 37 - 17329 + 17330 37 45 - 15053 + 15054 45 56 - 16140 + 16141 56 73 - 8613 + 8614 @@ -6731,62 +6766,62 @@ 1 2 - 22251 + 22253 2 3 - 15688 + 15689 3 4 - 12653 + 12654 4 6 - 14355 + 14356 6 8 - 12694 + 12695 8 11 - 17534 + 17535 11 16 - 16324 + 16325 16 22 - 16181 + 16182 22 29 - 16919 + 16920 29 36 - 15955 + 15956 36 44 - 16283 + 16284 44 54 - 15606 + 15607 54 @@ -6807,52 +6842,52 @@ 1 2 - 26763 + 26765 2 3 - 20795 + 20796 3 4 - 16775 + 16776 4 5 - 16037 + 16038 5 6 - 17391 + 17392 6 7 - 19811 + 19812 7 8 - 22702 + 22704 8 9 - 20344 + 20345 9 10 - 14971 + 14972 10 12 - 16611 + 16612 12 @@ -6873,47 +6908,47 @@ 1 2 - 34515 + 34517 2 3 - 25737 + 25739 3 4 - 18395 + 18397 4 5 - 16181 + 16182 5 6 - 12756 + 12757 6 7 - 11997 + 11998 7 8 - 10151 + 10152 8 9 - 10951 + 10952 9 10 - 10705 + 10706 10 @@ -6923,17 +6958,17 @@ 11 12 - 10151 + 10152 12 14 - 15750 + 15751 14 24 - 11607 + 11608 @@ -6949,62 +6984,62 @@ 1 2 - 22087 + 22089 2 3 - 16160 + 16161 3 4 - 12920 + 12921 4 6 - 16037 + 16038 6 8 - 14663 + 14664 8 10 - 13166 + 13167 10 14 - 18252 + 18253 14 18 - 16980 + 16982 18 22 - 17534 + 17535 22 26 - 18457 + 18458 26 30 - 16345 + 16346 30 36 - 15196 + 15197 36 @@ -7395,12 +7430,12 @@ 1 2 - 17370 + 17371 2 3 - 14376 + 14377 3 @@ -7410,7 +7445,7 @@ 4 6 - 15565 + 15566 6 @@ -7420,42 +7455,42 @@ 8 11 - 15422 + 15423 11 15 - 14601 + 14602 15 21 - 16058 + 16059 21 27 - 15381 + 15382 27 34 - 14909 + 14910 34 42 - 15709 + 15710 42 52 - 15975 + 15977 52 130 - 14376 + 14377 @@ -7471,62 +7506,62 @@ 1 2 - 24897 + 24898 2 3 - 16099 + 16100 3 4 - 12735 + 12736 4 6 - 15627 + 15628 6 8 - 14971 + 14972 8 11 - 15852 + 15854 11 16 - 17411 + 17412 16 20 - 14560 + 14561 20 26 - 17124 + 17125 26 32 - 16222 + 16223 32 39 - 14827 + 14828 39 59 - 13350 + 13351 @@ -7542,62 +7577,62 @@ 1 2 - 32403 + 32405 2 3 - 23707 + 23709 3 4 - 18416 + 18417 4 5 - 15114 + 15115 5 6 - 13843 + 13844 6 7 - 11648 + 11649 7 8 - 11710 + 11711 8 9 - 10889 + 10890 9 10 - 10151 + 10152 10 12 - 17924 + 17925 12 15 - 17678 + 17679 15 100 - 10192 + 10193 @@ -7613,52 +7648,52 @@ 1 2 - 24897 + 24898 2 3 - 20344 + 20345 3 4 - 16796 + 16797 4 5 - 17760 + 17761 5 6 - 18539 + 18540 6 7 - 20385 + 20386 7 8 - 22374 + 22376 8 9 - 18703 + 18704 9 10 - 12899 + 12900 10 12 - 14991 + 14992 12 @@ -7679,12 +7714,12 @@ 1 2 - 24650 + 24652 2 3 - 16591 + 16592 3 @@ -7694,52 +7729,52 @@ 4 6 - 17780 + 17781 6 8 - 15299 + 15300 8 10 - 12797 + 12798 10 13 - 14376 + 14377 13 16 - 14991 + 14992 16 19 - 14622 + 14623 19 22 - 14007 + 14008 22 26 - 17083 + 17084 26 31 - 15299 + 15300 31 39 - 3670 + 3671 @@ -8134,27 +8169,27 @@ locations_expr - 13127189 + 13128112 id - 13127189 + 13128112 container - 4347 + 4348 startLine - 191486 + 191499 startColumn - 2460 + 2461 endLine - 191465 + 191479 endColumn @@ -8172,7 +8207,7 @@ 1 2 - 13127189 + 13128112 @@ -8188,7 +8223,7 @@ 1 2 - 13127189 + 13128112 @@ -8204,7 +8239,7 @@ 1 2 - 13127189 + 13128112 @@ -8220,7 +8255,7 @@ 1 2 - 13127189 + 13128112 @@ -8236,7 +8271,7 @@ 1 2 - 13127189 + 13128112 @@ -8642,67 +8677,67 @@ 1 5 - 16078 + 16079 5 9 - 16447 + 16448 9 15 - 15996 + 15997 15 23 - 15073 + 15074 23 32 - 15114 + 15115 32 44 - 14971 + 14972 44 60 - 14724 + 14726 60 80 - 14807 + 14808 80 103 - 14581 + 14582 103 130 - 14786 + 14787 130 159 - 14560 + 14561 159 194 - 14560 + 14561 194 302 - 9782 + 9783 @@ -8718,22 +8753,22 @@ 1 2 - 23461 + 23463 2 3 - 15586 + 15587 3 4 - 11320 + 11321 4 6 - 16324 + 16325 6 @@ -8743,32 +8778,32 @@ 8 11 - 16406 + 16407 11 16 - 17329 + 17330 16 21 - 16406 + 16407 21 28 - 16611 + 16612 28 35 - 15955 + 15956 35 43 - 15934 + 15936 43 @@ -8789,62 +8824,62 @@ 1 4 - 15934 + 15936 4 7 - 17493 + 17494 7 11 - 16652 + 16653 11 16 - 17370 + 17371 16 21 - 17473 + 17474 21 26 - 15032 + 15033 26 31 - 16140 + 16141 31 36 - 17698 + 17699 36 40 - 15791 + 15792 40 44 - 16529 + 16530 44 49 - 16591 + 16592 49 63 - 8777 + 8778 @@ -8860,22 +8895,22 @@ 1 2 - 101762 + 101769 2 3 - 44564 + 44567 3 4 - 27501 + 27503 4 6 - 14540 + 14541 6 @@ -8896,62 +8931,62 @@ 1 4 - 16919 + 16920 4 7 - 16611 + 16612 7 11 - 16386 + 16387 11 16 - 16181 + 16182 16 21 - 16406 + 16407 21 27 - 16734 + 16735 27 33 - 16406 + 16407 33 38 - 14458 + 14459 38 43 - 15504 + 15505 43 47 - 14745 + 14746 47 52 - 16714 + 16715 52 65 - 14376 + 14377 66 @@ -9352,67 +9387,67 @@ 1 5 - 16099 + 16100 5 9 - 16447 + 16448 9 15 - 15770 + 15772 15 23 - 15053 + 15054 23 32 - 15606 + 15607 32 44 - 14704 + 14705 44 60 - 14458 + 14459 60 80 - 15237 + 15238 80 103 - 14478 + 14479 103 130 - 14786 + 14787 130 159 - 14458 + 14459 159 193 - 14376 + 14377 193 299 - 9987 + 9988 @@ -9428,67 +9463,67 @@ 1 2 - 23461 + 23463 2 3 - 15524 + 15525 3 4 - 11320 + 11321 4 6 - 16016 + 16018 6 8 - 13453 + 13454 8 11 - 16468 + 16469 11 15 - 14458 + 14459 15 20 - 16673 + 16674 20 26 - 14971 + 14972 26 33 - 16119 + 16120 33 40 - 14724 + 14726 40 49 - 14724 + 14726 49 60 - 3547 + 3548 @@ -9504,22 +9539,22 @@ 1 2 - 95343 + 95349 2 3 - 49835 + 49838 3 4 - 29306 + 29308 4 6 - 15565 + 15566 6 @@ -9540,62 +9575,62 @@ 1 4 - 15791 + 15792 4 7 - 17411 + 17412 7 11 - 16447 + 16448 11 16 - 17309 + 17310 16 21 - 17268 + 17269 21 26 - 15114 + 15115 26 31 - 16263 + 16264 31 36 - 17678 + 17679 36 40 - 15340 + 15341 40 44 - 16591 + 16592 44 49 - 16734 + 16735 49 63 - 9515 + 9516 @@ -9611,62 +9646,62 @@ 1 4 - 17144 + 17146 4 7 - 16755 + 16756 7 11 - 16386 + 16387 11 16 - 16837 + 16838 16 21 - 15975 + 15977 21 26 - 14478 + 14479 26 32 - 16119 + 16120 32 37 - 14376 + 14377 37 42 - 15832 + 15833 42 46 - 14232 + 14233 46 51 - 17247 + 17248 51 59 - 14458 + 14459 59 @@ -9726,7 +9761,7 @@ 4177 - 7133 + 7135 225 @@ -9803,22 +9838,22 @@ 135 139 - 225 + 246 139 146 - 225 + 205 146 148 - 225 + 205 148 152 - 225 + 246 152 @@ -10056,23 +10091,23 @@ numlines - 1411307 + 1406545 element_id - 1404227 + 1399488 num_lines - 103842 + 102550 num_code - 85433 + 85615 num_comment - 59945 + 60213 @@ -10086,12 +10121,12 @@ 1 2 - 1397147 + 1392432 2 3 - 7080 + 7056 @@ -10107,12 +10142,12 @@ 1 2 - 1398091 + 1393373 2 3 - 6136 + 6115 @@ -10128,7 +10163,7 @@ 1 2 - 1404227 + 1399488 @@ -10144,27 +10179,27 @@ 1 2 - 69385 + 68680 2 3 - 14160 + 12230 3 - 5 - 8496 + 4 + 7526 - 5 - 63 - 8024 + 4 + 21 + 7997 - 79 + 29 926 - 3776 + 6115 @@ -10180,27 +10215,27 @@ 1 2 - 71745 + 71032 2 3 - 14160 + 12230 3 4 - 7552 + 8467 4 6 - 8968 + 9408 6 7 - 1416 + 1411 @@ -10216,22 +10251,22 @@ 1 2 - 70801 + 70092 2 3 - 16992 + 15053 3 4 - 9912 + 10819 4 7 - 6136 + 6585 @@ -10247,27 +10282,27 @@ 1 2 - 52393 + 53157 2 3 - 15104 + 14582 3 5 - 6608 + 6585 5 42 - 6608 + 6585 - 45 + 44 927 - 4720 + 4704 @@ -10283,27 +10318,27 @@ 1 2 - 52393 + 53157 2 3 - 17464 + 16934 3 5 - 6136 + 6115 5 8 - 7080 + 6585 8 12 - 2360 + 2822 @@ -10319,27 +10354,27 @@ 1 2 - 52865 + 53627 2 3 - 16520 + 15994 3 5 - 7552 + 7526 5 7 - 5192 + 5174 7 10 - 3304 + 3292 @@ -10355,32 +10390,32 @@ 1 2 - 34928 + 34810 2 3 - 8496 + 9408 3 4 - 4720 + 4233 4 6 - 4720 + 4704 6 11 - 5192 + 5174 17 2622 - 1888 + 1881 @@ -10396,32 +10431,32 @@ 1 2 - 34928 + 34810 2 3 - 8496 + 9408 3 4 - 4720 + 4233 4 6 - 4720 + 4704 6 8 - 4720 + 4704 10 - 37 - 2360 + 38 + 2352 @@ -10437,32 +10472,32 @@ 1 2 - 34928 + 34810 2 3 - 8496 + 9408 3 4 - 4720 + 4233 4 6 - 4720 + 4704 6 10 - 4720 + 4704 10 37 - 2360 + 2352 @@ -10472,31 +10507,31 @@ diagnostics - 667106 + 632933 id - 667106 + 632933 severity - 210 + 209 error_tag - 7142 + 7441 error_message - 50629 + 50202 full_error_message - 666371 + 632199 location - 366903 + 354878 @@ -10510,7 +10545,7 @@ 1 2 - 667106 + 632933 @@ -10526,7 +10561,7 @@ 1 2 - 667106 + 632933 @@ -10542,7 +10577,7 @@ 1 2 - 667106 + 632933 @@ -10558,7 +10593,7 @@ 1 2 - 667106 + 632933 @@ -10574,7 +10609,7 @@ 1 2 - 667106 + 632933 @@ -10588,14 +10623,14 @@ 12 - 375 - 376 - 105 + 354 + 355 + 104 - 5976 - 5977 - 105 + 5685 + 5686 + 104 @@ -10611,12 +10646,12 @@ 4 5 - 105 + 104 - 64 - 65 - 105 + 67 + 68 + 104 @@ -10632,12 +10667,12 @@ 6 7 - 105 + 104 - 476 - 477 - 105 + 473 + 474 + 104 @@ -10651,14 +10686,14 @@ 12 - 375 - 376 - 105 + 354 + 355 + 104 - 5969 - 5970 - 105 + 5678 + 5679 + 104 @@ -10672,14 +10707,14 @@ 12 - 211 - 212 - 105 + 198 + 199 + 104 - 3378 - 3379 - 105 + 3278 + 3279 + 104 @@ -10695,256 +10730,246 @@ 1 2 - 735 + 1048 2 3 - 735 + 1048 3 - 4 - 525 - - - 4 - 6 - 630 - - - 6 - 8 - 630 - - - 8 - 14 - 630 - - - 14 - 17 - 525 - - - 18 - 30 - 630 - - - 33 - 51 - 630 - - - 52 - 149 - 630 - - - 200 - 808 - 630 - - - 845 - 1335 - 210 - - - - - - - error_tag - severity - - - 12 - - - 1 - 2 - 7142 - - - - - - - error_tag - error_message - - - 12 - - - 1 - 2 - 4726 - - - 2 - 3 - 630 - - - 3 - 4 - 420 - - - 4 - 6 - 630 - - - 6 - 53 - 630 - - - 249 - 250 - 105 - - - - - - - error_tag - full_error_message - - - 12 - - - 1 - 2 - 840 - - - 2 - 3 - 735 - - - 3 - 4 - 525 - - - 4 - 6 - 630 - - - 6 - 8 - 630 - - - 8 - 14 - 525 - - - 14 - 17 - 525 - - - 18 - 30 - 630 - - - 33 - 51 - 630 - - - 52 - 149 - 630 - - - 200 - 808 - 630 - - - 845 - 1335 - 210 - - - - - - - error_tag - location - - - 12 - - - 1 - 2 - 1155 - - - 2 - 3 - 840 - - - 3 - 4 - 735 - - - 4 5 - 420 + 628 5 - 6 - 420 + 7 + 628 - 6 - 8 - 525 + 7 + 11 + 628 - 8 - 13 - 630 + 12 + 16 + 524 - 14 + 16 25 - 630 + 628 25 + 38 + 628 + + + 41 + 72 + 628 + + + 95 + 610 + 628 + + + 624 + 1277 + 419 + + + + + + + error_tag + severity + + + 12 + + + 1 + 2 + 7441 + + + + + + + error_tag + error_message + + + 12 + + + 1 + 2 + 5240 + + + 2 + 3 + 419 + + + 3 + 4 + 419 + + + 4 + 6 + 628 + + + 6 51 - 630 + 628 - 57 - 219 - 630 + 251 + 252 + 104 + + + + + + + error_tag + full_error_message + + + 12 + + + 1 + 2 + 1152 - 382 - 733 - 525 + 2 + 3 + 1048 + + + 3 + 5 + 628 + + + 5 + 7 + 628 + + + 7 + 13 + 628 + + + 13 + 16 + 419 + + + 16 + 25 + 628 + + + 25 + 38 + 628 + + + 41 + 72 + 628 + + + 95 + 610 + 628 + + + 624 + 1277 + 419 + + + + + + + error_tag + location + + + 12 + + + 1 + 2 + 1781 + + + 2 + 3 + 733 + + + 3 + 4 + 524 + + + 4 + 6 + 628 + + + 6 + 7 + 524 + + + 7 + 12 + 628 + + + 12 + 17 + 628 + + + 24 + 44 + 628 + + + 44 + 110 + 628 + + + 167 + 542 + 628 + + + 704 + 705 + 104 @@ -10960,42 +10985,42 @@ 1 2 - 20377 + 20437 2 3 - 10503 + 10899 3 4 - 2415 + 2829 4 5 - 4411 + 2515 5 - 7 - 3991 + 6 + 2829 - 7 - 14 - 4096 + 6 + 9 + 3773 - 14 - 67 - 3886 + 9 + 21 + 3773 - 75 - 1335 - 945 + 21 + 1277 + 3144 @@ -11011,7 +11036,7 @@ 1 2 - 50629 + 50202 @@ -11027,7 +11052,7 @@ 1 2 - 50629 + 50202 @@ -11043,42 +11068,42 @@ 1 2 - 20482 + 20542 2 3 - 10503 + 10899 3 4 - 2415 + 2829 4 5 - 4411 + 2515 5 - 7 - 3991 + 6 + 2829 - 7 - 14 - 3991 + 6 + 10 + 4087 - 14 - 67 - 3886 + 10 + 24 + 3877 - 75 - 1335 - 945 + 25 + 1277 + 2620 @@ -11094,32 +11119,32 @@ 1 2 - 32352 + 32280 2 3 - 5987 + 5345 3 5 - 3886 + 3563 5 - 9 - 3886 + 8 + 3877 - 9 - 86 - 3886 + 8 + 34 + 3773 - 130 - 733 - 630 + 34 + 705 + 1362 @@ -11135,12 +11160,12 @@ 1 2 - 666266 + 632094 8 9 - 105 + 104 @@ -11156,7 +11181,7 @@ 1 2 - 666371 + 632199 @@ -11172,7 +11197,7 @@ 1 2 - 666371 + 632199 @@ -11188,7 +11213,7 @@ 1 2 - 666371 + 632199 @@ -11204,7 +11229,7 @@ 1 2 - 666371 + 632199 @@ -11220,22 +11245,22 @@ 1 2 - 184554 + 180059 2 3 - 134345 + 135621 3 - 5 - 31721 + 6 + 31023 - 5 + 6 17 - 16281 + 8174 @@ -11251,12 +11276,12 @@ 1 2 - 356819 + 345445 2 3 - 10083 + 9432 @@ -11272,12 +11297,12 @@ 1 2 - 349361 + 338947 2 - 5 - 17541 + 6 + 15930 @@ -11293,12 +11318,12 @@ 1 2 - 348836 + 338423 2 - 5 - 18066 + 6 + 16454 @@ -11314,22 +11339,22 @@ 1 2 - 184659 + 180164 2 3 - 134345 + 135621 3 - 5 - 31721 + 6 + 31023 - 5 + 6 17 - 16176 + 8070 @@ -11339,15 +11364,15 @@ files - 124610 + 124189 id - 124610 + 124189 name - 124610 + 124189 @@ -11361,7 +11386,7 @@ 1 2 - 124610 + 124189 @@ -11377,7 +11402,7 @@ 1 2 - 124610 + 124189 @@ -11387,15 +11412,15 @@ folders - 15576 + 15994 id - 15576 + 15994 name - 15576 + 15994 @@ -11409,7 +11434,7 @@ 1 2 - 15576 + 15994 @@ -11425,7 +11450,7 @@ 1 2 - 15576 + 15994 @@ -11435,15 +11460,15 @@ containerparent - 139242 + 139243 parent - 15576 + 15994 child - 139242 + 139243 @@ -11457,32 +11482,32 @@ 1 2 - 6608 + 7056 2 3 - 3304 + 3292 3 5 - 1416 + 1411 5 12 - 1416 + 1411 23 28 - 1416 + 1411 40 67 - 1416 + 1411 @@ -11498,7 +11523,7 @@ 1 2 - 139242 + 139243 @@ -11508,11 +11533,11 @@ fileannotations - 5264964 + 5254893 id - 5028 + 5019 kind @@ -11520,11 +11545,11 @@ name - 56220 + 56112 value - 47263 + 47173 @@ -11543,7 +11568,7 @@ 2 3 - 4854 + 4845 @@ -11564,52 +11589,52 @@ 102 225 - 382 + 381 227 299 - 382 + 381 301 452 - 405 + 404 452 555 - 382 + 381 559 626 - 382 + 381 626 716 - 382 + 381 729 904 - 382 + 381 904 934 - 81 + 80 936 937 - 1459 + 1457 1083 2036 - 382 + 381 2293 @@ -11635,32 +11660,32 @@ 114 275 - 382 + 381 275 363 - 382 + 381 393 638 - 382 + 381 643 744 - 382 + 381 751 955 - 382 + 381 955 1087 - 382 + 381 1088 @@ -11670,17 +11695,17 @@ 1501 1502 - 1459 + 1457 1504 1874 - 382 + 381 1972 4080 - 243 + 242 @@ -11759,62 +11784,62 @@ 1 2 - 9095 + 9078 2 3 - 6384 + 6372 3 5 - 4287 + 4279 5 9 - 4379 + 4371 9 14 - 4090 + 4082 14 18 - 4287 + 4279 18 20 - 4843 + 4834 20 34 - 4333 + 4325 34 128 - 4623 + 4614 128 229 - 4229 + 4221 229 387 - 4356 + 4348 387 434 - 1309 + 1306 @@ -11830,7 +11855,7 @@ 1 2 - 56220 + 56112 @@ -11846,62 +11871,62 @@ 1 2 - 9107 + 9089 2 3 - 8273 + 8257 3 4 - 2630 + 2625 4 6 - 4634 + 4625 6 9 - 4240 + 4232 9 14 - 4321 + 4313 14 17 - 4240 + 4232 17 22 - 4715 + 4706 22 41 - 4321 + 4313 41 82 - 4275 + 4267 82 157 - 4217 + 4209 158 1895 - 1239 + 1237 @@ -11917,67 +11942,67 @@ 1 2 - 7346 + 7332 2 5 - 2294 + 2289 5 8 - 3418 + 3411 8 15 - 3626 + 3619 15 17 - 2607 + 2602 17 19 - 4252 + 4244 19 34 - 3418 + 3411 34 189 - 3719 + 3712 189 201 - 3707 + 3700 201 266 - 3649 + 3642 266 321 - 3777 + 3770 322 399 - 4055 + 4047 399 435 - 1390 + 1387 @@ -11993,7 +12018,7 @@ 1 2 - 47251 + 47161 2 @@ -12014,67 +12039,67 @@ 1 2 - 7369 + 7355 2 5 - 2653 + 2648 5 8 - 3603 + 3596 8 15 - 3649 + 3642 15 17 - 2908 + 2902 17 19 - 3684 + 3677 19 29 - 3603 + 3596 29 39 - 3765 + 3758 39 48 - 3707 + 3700 48 74 - 3661 + 3654 74 102 - 3545 + 3538 102 119 - 3696 + 3689 119 146 - 1413 + 1410 @@ -12084,15 +12109,15 @@ inmacroexpansion - 109305879 + 109313317 id - 17940692 + 17941916 inv - 2681885 + 2682068 @@ -12106,37 +12131,37 @@ 1 3 - 1565908 + 1566018 3 5 - 1071272 + 1071344 5 6 - 1179734 + 1179814 6 7 - 4799673 + 4800000 7 8 - 6358947 + 6359380 8 9 - 2595071 + 2595248 9 150 - 370084 + 370109 @@ -12152,32 +12177,32 @@ 1 2 - 371830 + 371855 2 3 - 540079 + 540113 3 4 - 349893 + 349917 4 7 - 199801 + 199815 7 8 - 206276 + 206290 8 9 - 240865 + 240882 9 @@ -12187,22 +12212,22 @@ 10 11 - 324110 + 324132 11 337 - 223895 + 223913 339 422 - 201369 + 201383 422 7616 - 21561 + 21563 @@ -12212,15 +12237,15 @@ affectedbymacroexpansion - 35538114 + 35540532 id - 5134928 + 5135277 inv - 2772993 + 2773181 @@ -12234,37 +12259,37 @@ 1 2 - 2804021 + 2804212 2 3 - 557759 + 557797 3 4 - 263786 + 263804 4 5 - 563401 + 563439 5 12 - 390245 + 390272 12 50 - 405678 + 405705 50 9900 - 150035 + 150045 @@ -12280,67 +12305,67 @@ 1 4 - 228146 + 228162 4 7 - 230808 + 230823 7 9 - 219545 + 219560 9 12 - 250025 + 250042 12 13 - 332565 + 332588 13 14 - 164888 + 164899 14 15 - 297580 + 297600 15 16 - 121327 + 121336 16 17 - 275438 + 275457 17 18 - 146319 + 146328 18 20 - 251068 + 251085 20 25 - 208095 + 208109 25 109 - 47183 + 47186 @@ -12350,19 +12375,19 @@ macroinvocations - 34062185 + 33895024 id - 34062185 + 33895024 macro_id - 81572 + 81382 location - 779877 + 778558 kind @@ -12380,7 +12405,7 @@ 1 2 - 34062185 + 33895024 @@ -12396,7 +12421,7 @@ 1 2 - 34062185 + 33895024 @@ -12412,7 +12437,7 @@ 1 2 - 34062185 + 33895024 @@ -12428,52 +12453,57 @@ 1 2 - 16835 + 17578 2 3 - 17322 + 16977 3 4 - 3846 + 3700 4 - 6 - 7323 + 5 + 4880 - 6 - 11 - 7265 + 5 + 8 + 6048 - 11 - 21 - 6546 + 8 + 14 + 6406 - 21 - 47 - 6257 + 14 + 29 + 6291 - 47 - 143 - 6117 + 29 + 72 + 6106 - 143 - 934 - 6152 + 72 + 247 + 6129 - 934 - 168391 - 3904 + 248 + 4166 + 6106 + + + 4220 + 168296 + 1156 @@ -12489,37 +12519,37 @@ 1 2 - 43648 + 43507 2 3 - 10648 + 10651 3 4 - 5295 + 5285 4 6 - 7010 + 6985 6 13 - 6627 + 6626 13 66 - 6152 + 6140 66 3614 - 2189 + 2185 @@ -12535,12 +12565,12 @@ 1 2 - 75732 + 75553 2 3 - 5839 + 5828 @@ -12556,42 +12586,37 @@ 1 2 - 297219 + 320983 2 3 - 195287 + 177752 3 4 - 48978 + 47300 4 5 - 57610 + 59605 5 - 8 - 59059 + 9 + 68533 - 8 - 17 - 59267 + 9 + 23 + 58425 - 17 - 934 - 58572 - - - 942 - 244474 - 3881 + 23 + 244365 + 45958 @@ -12607,12 +12632,12 @@ 1 2 - 732509 + 731281 2 350 - 47367 + 47277 @@ -12628,7 +12653,7 @@ 1 2 - 779877 + 778558 @@ -12642,13 +12667,13 @@ 12 - 21085 - 21086 + 20414 + 20415 11 - 2918595 - 2918596 + 2910446 + 2910447 11 @@ -12668,8 +12693,8 @@ 11 - 5421 - 5422 + 5418 + 5419 11 @@ -12684,13 +12709,13 @@ 12 - 6289 - 6290 + 6291 + 6292 11 - 61017 - 61018 + 61030 + 61031 11 @@ -12701,15 +12726,15 @@ macroparent - 30544569 + 30455631 id - 30544569 + 30455631 parent_id - 23768631 + 23698229 @@ -12723,7 +12748,7 @@ 1 2 - 30544569 + 30455631 @@ -12739,17 +12764,17 @@ 1 2 - 18363231 + 18307194 2 3 - 4552338 + 4540068 3 88 - 853060 + 850966 @@ -12759,15 +12784,15 @@ macrolocationbind - 3984360 + 3984640 id - 2778691 + 2778886 location - 1988314 + 1988454 @@ -12781,22 +12806,22 @@ 1 2 - 2182951 + 2183104 2 3 - 336419 + 336443 3 7 - 229799 + 229815 7 57 - 29521 + 29523 @@ -12812,22 +12837,22 @@ 1 2 - 1589477 + 1589588 2 3 - 169635 + 169647 3 8 - 154222 + 154233 8 723 - 74979 + 74984 @@ -12837,19 +12862,19 @@ macro_argument_unexpanded - 86422356 + 86176891 invocation - 26658798 + 26568376 argument_index - 764 + 763 text - 326615 + 326094 @@ -12863,22 +12888,22 @@ 1 2 - 7473686 + 7436803 2 3 - 10892801 + 10861705 3 4 - 6274590 + 6256816 4 67 - 2017720 + 2013051 @@ -12894,22 +12919,22 @@ 1 2 - 7545213 + 7508019 2 3 - 11043502 + 11011575 3 4 - 6104017 + 6087240 4 67 - 1966065 + 1961541 @@ -12923,18 +12948,18 @@ 12 - 41253 - 41254 - 672 + 41230 + 41231 + 670 - 41455 - 174137 + 41432 + 174067 57 - 715654 - 2300744 + 715085 + 2297335 34 @@ -12951,7 +12976,7 @@ 2 3 - 672 + 670 13 @@ -12960,7 +12985,7 @@ 6559 - 19569 + 19579 34 @@ -12977,57 +13002,57 @@ 1 2 - 39998 + 40858 2 3 - 66161 + 65607 3 4 - 14947 + 15184 4 5 - 44065 + 45103 5 - 7 - 24761 + 8 + 25569 - 7 + 8 12 - 18643 + 16075 12 16 - 22281 + 22297 16 23 - 26673 + 26518 23 43 - 24784 + 24748 43 164 - 24506 + 24459 164 - 521728 - 19790 + 521384 + 19671 @@ -13043,17 +13068,17 @@ 1 2 - 236190 + 235830 2 3 - 79869 + 79728 3 9 - 10555 + 10535 @@ -13063,19 +13088,19 @@ macro_argument_expanded - 86422356 + 86176891 invocation - 26658798 + 26568376 argument_index - 764 + 763 text - 197906 + 197597 @@ -13089,22 +13114,22 @@ 1 2 - 7473686 + 7436803 2 3 - 10892801 + 10861705 3 4 - 6274590 + 6256816 4 67 - 2017720 + 2013051 @@ -13120,22 +13145,22 @@ 1 2 - 10794971 + 10747757 2 3 - 9400379 + 9374150 3 4 - 5321625 + 5307004 4 9 - 1141821 + 1139463 @@ -13149,18 +13174,18 @@ 12 - 41253 - 41254 - 672 + 41230 + 41231 + 670 - 41455 - 174137 + 41432 + 174067 57 - 715654 - 2300744 + 715085 + 2297335 34 @@ -13177,7 +13202,7 @@ 1 2 - 660 + 659 2 @@ -13186,7 +13211,7 @@ 870 - 13871 + 13877 46 @@ -13203,62 +13228,62 @@ 1 2 - 24008 + 24552 2 3 - 41238 + 41147 3 4 - 6940 + 6927 4 5 - 15874 + 16364 5 6 - 3835 + 2995 6 7 - 23324 + 23291 7 9 - 15457 + 15982 9 15 - 17345 + 16699 15 - 30 - 14842 + 31 + 15589 - 30 - 89 - 14947 + 31 + 97 + 15080 - 90 - 505 - 14854 + 97 + 775 + 15485 - 510 - 1054483 - 5237 + 775 + 1052916 + 3481 @@ -13274,17 +13299,17 @@ 1 2 - 100112 + 99989 2 3 - 83009 + 82850 3 66 - 14785 + 14756 @@ -13294,19 +13319,19 @@ functions - 4759741 + 4726273 id - 4759741 + 4726273 name - 1941845 + 1934352 kind - 3304 + 3292 @@ -13320,7 +13345,7 @@ 1 2 - 4759741 + 4726273 @@ -13336,7 +13361,7 @@ 1 2 - 4759741 + 4726273 @@ -13352,22 +13377,22 @@ 1 2 - 1523173 + 1516622 2 3 - 153875 + 154296 3 5 - 151986 + 151003 5 - 1754 - 112810 + 1724 + 112429 @@ -13383,12 +13408,12 @@ 1 2 - 1941373 + 1933881 2 3 - 472 + 470 @@ -13404,37 +13429,37 @@ 6 7 - 472 + 470 64 65 - 472 + 470 173 174 - 472 + 470 195 196 - 472 + 470 - 1365 - 1366 - 472 + 1358 + 1359 + 470 - 2462 - 2463 - 472 + 2432 + 2433 + 470 5819 5820 - 472 + 470 @@ -13450,37 +13475,37 @@ 3 4 - 472 + 470 33 34 - 472 + 470 39 40 - 472 + 470 94 95 - 472 + 470 195 196 - 472 + 470 - 246 - 247 - 472 + 244 + 245 + 470 3505 3506 - 472 + 470 @@ -13490,15 +13515,15 @@ function_entry_point - 1180967 + 1176981 id - 1171054 + 1167103 entry_point - 1180967 + 1176981 @@ -13512,12 +13537,12 @@ 1 2 - 1161142 + 1157224 2 3 - 9912 + 9878 @@ -13533,7 +13558,7 @@ 1 2 - 1180967 + 1176981 @@ -13543,15 +13568,15 @@ function_return_type - 4768237 + 4734741 id - 4759741 + 4726273 return_type - 1027092 + 1016569 @@ -13565,12 +13590,12 @@ 1 2 - 4752660 + 4719217 2 5 - 7080 + 7056 @@ -13586,22 +13611,22 @@ 1 2 - 524874 + 523103 2 3 - 398847 + 390445 3 11 - 78825 + 78559 11 - 2523 - 24544 + 2516 + 24461 @@ -13923,48 +13948,48 @@ purefunctions - 99735 + 99448 id - 99735 + 99448 function_deleted - 141130 + 140654 id - 141130 + 140654 function_defaulted - 74577 + 74325 id - 74577 + 74325 member_function_this_type - 549916 + 553641 id - 549916 + 553641 this_type - 188354 + 189690 @@ -13978,7 +14003,7 @@ 1 2 - 549916 + 553641 @@ -13994,32 +14019,32 @@ 1 2 - 68044 + 68526 2 3 - 45024 + 45414 3 4 - 30331 + 30475 4 5 - 15428 + 15537 5 7 - 15497 + 15607 7 66 - 14028 + 14128 @@ -14029,27 +14054,27 @@ fun_decls - 5136876 + 5102136 id - 5131684 + 5096962 function - 4611530 + 4578563 type_id - 1023788 + 1013276 name - 1843196 + 1836035 location - 3473987 + 3461324 @@ -14063,7 +14088,7 @@ 1 2 - 5131684 + 5096962 @@ -14079,12 +14104,12 @@ 1 2 - 5126492 + 5091787 2 3 - 5192 + 5174 @@ -14100,7 +14125,7 @@ 1 2 - 5131684 + 5096962 @@ -14116,7 +14141,7 @@ 1 2 - 5131684 + 5096962 @@ -14132,17 +14157,17 @@ 1 2 - 4172561 + 4141075 2 3 - 364391 + 363161 3 7 - 74577 + 74325 @@ -14158,12 +14183,12 @@ 1 2 - 4569521 + 4536696 2 5 - 42008 + 41867 @@ -14179,7 +14204,7 @@ 1 2 - 4611530 + 4578563 @@ -14195,17 +14220,17 @@ 1 2 - 4229674 + 4197996 2 4 - 380439 + 379155 4 6 - 1416 + 1411 @@ -14221,22 +14246,22 @@ 1 2 - 447464 + 445954 2 3 - 462097 + 453481 3 9 - 79769 + 79500 9 - 2775 - 34456 + 2768 + 34340 @@ -14252,22 +14277,22 @@ 1 2 - 532426 + 530629 2 3 - 390351 + 381978 3 11 - 77409 + 77148 11 - 2484 - 23600 + 2477 + 23520 @@ -14283,17 +14308,17 @@ 1 2 - 893985 + 883912 2 5 - 90625 + 90319 5 - 824 - 39176 + 822 + 39044 @@ -14309,22 +14334,22 @@ 1 2 - 789199 + 779480 2 3 - 133578 + 133127 3 11 - 78353 + 78089 11 - 2032 - 22656 + 2030 + 22579 @@ -14340,27 +14365,27 @@ 1 2 - 1250824 + 1245192 2 3 - 269517 + 269548 3 4 - 80713 + 80441 4 6 - 139714 + 138772 6 - 1788 - 102426 + 1758 + 102080 @@ -14376,22 +14401,22 @@ 1 2 - 1432076 + 1425832 2 3 - 152930 + 153355 3 5 - 146322 + 145358 5 - 1738 - 111866 + 1708 + 111488 @@ -14407,17 +14432,17 @@ 1 2 - 1622295 + 1615880 2 4 - 135466 + 135009 4 - 969 - 85433 + 954 + 85145 @@ -14433,27 +14458,27 @@ 1 2 - 1272064 + 1266831 2 3 - 297365 + 296362 3 4 - 79769 + 79500 4 8 - 139714 + 139243 8 - 666 - 54281 + 664 + 54097 @@ -14469,17 +14494,17 @@ 1 2 - 3005754 + 2995611 2 4 - 304445 + 302007 4 55 - 163787 + 163704 @@ -14495,17 +14520,17 @@ 1 2 - 3073724 + 3063351 2 6 - 270933 + 268607 6 55 - 129330 + 129364 @@ -14521,12 +14546,12 @@ 1 2 - 3256863 + 3245873 2 27 - 217124 + 215450 @@ -14542,12 +14567,12 @@ 1 2 - 3296512 + 3285388 2 13 - 177475 + 175935 @@ -14557,22 +14582,22 @@ fun_def - 1970638 + 1963988 id - 1970638 + 1963988 fun_specialized - 26432 + 26343 id - 26432 + 26343 @@ -14590,15 +14615,15 @@ fun_decl_specifiers - 2947225 + 2937280 id - 1716697 + 1710904 name - 2832 + 2822 @@ -14612,17 +14637,17 @@ 1 2 - 505049 + 503345 2 3 - 1192767 + 1188742 3 4 - 18880 + 18816 @@ -14638,32 +14663,32 @@ 50 51 - 472 + 470 203 204 - 472 + 470 209 210 - 472 + 470 657 658 - 472 + 470 2561 2562 - 472 + 470 2564 2565 - 472 + 470 @@ -14794,26 +14819,26 @@ fun_decl_empty_throws - 2002263 + 1978101 fun_decl - 2002263 + 1978101 fun_decl_noexcept - 61343 + 61207 fun_decl - 61343 + 61207 constant - 61238 + 61102 @@ -14827,7 +14852,7 @@ 1 2 - 61343 + 61207 @@ -14843,12 +14868,12 @@ 1 2 - 61133 + 60998 2 3 - 105 + 104 @@ -14858,11 +14883,11 @@ fun_decl_empty_noexcept - 890209 + 888146 fun_decl - 890209 + 888146 @@ -14967,19 +14992,19 @@ param_decl_bind - 7511554 + 7472094 id - 7511554 + 7472094 index - 8024 + 7997 fun_decl - 4315108 + 4286434 @@ -14993,7 +15018,7 @@ 1 2 - 7511554 + 7472094 @@ -15009,7 +15034,7 @@ 1 2 - 7511554 + 7472094 @@ -15025,72 +15050,72 @@ 2 3 - 944 + 940 5 6 - 472 + 470 7 8 - 472 + 470 10 11 - 944 + 940 11 12 - 472 + 470 12 13 - 944 + 940 13 14 - 472 + 470 25 26 - 472 + 470 78 79 - 472 + 470 245 246 - 472 + 470 636 637 - 472 + 470 1713 1714 - 472 + 470 3991 3992 - 472 + 470 - 9142 - 9143 - 472 + 9112 + 9113 + 470 @@ -15106,72 +15131,72 @@ 2 3 - 944 + 940 5 6 - 472 + 470 7 8 - 472 + 470 10 11 - 944 + 940 11 12 - 472 + 470 12 13 - 944 + 940 13 14 - 472 + 470 25 26 - 472 + 470 78 79 - 472 + 470 245 246 - 472 + 470 636 637 - 472 + 470 1713 1714 - 472 + 470 3991 3992 - 472 + 470 - 9142 - 9143 - 472 + 9112 + 9113 + 470 @@ -15187,22 +15212,22 @@ 1 2 - 2431319 + 2409002 2 3 - 1075237 + 1071608 3 4 - 508353 + 506638 4 18 - 300197 + 299184 @@ -15218,22 +15243,22 @@ 1 2 - 2431319 + 2409002 2 3 - 1075237 + 1071608 3 4 - 508353 + 506638 4 18 - 300197 + 299184 @@ -15243,27 +15268,27 @@ var_decls - 8655233 + 8611913 id - 8586319 + 8543232 variable - 7559699 + 7520077 type_id - 2453031 + 2430641 name - 674973 + 672695 location - 5383265 + 5365099 @@ -15277,7 +15302,7 @@ 1 2 - 8586319 + 8543232 @@ -15293,12 +15318,12 @@ 1 2 - 8517406 + 8474552 2 3 - 68913 + 68680 @@ -15314,7 +15339,7 @@ 1 2 - 8586319 + 8543232 @@ -15330,7 +15355,7 @@ 1 2 - 8586319 + 8543232 @@ -15346,17 +15371,17 @@ 1 2 - 6694978 + 6658274 2 3 - 709429 + 707035 3 7 - 155291 + 154767 @@ -15372,12 +15397,12 @@ 1 2 - 7386000 + 7346963 2 4 - 173699 + 173113 @@ -15393,12 +15418,12 @@ 1 2 - 7442169 + 7402943 2 3 - 117530 + 117133 @@ -15414,12 +15439,12 @@ 1 2 - 7005089 + 6967337 2 4 - 554610 + 552739 @@ -15435,27 +15460,27 @@ 1 2 - 1525061 + 1505802 2 3 - 517794 + 516046 3 4 - 99121 + 98787 4 7 - 189275 + 188636 7 780 - 121778 + 121367 @@ -15471,22 +15496,22 @@ 1 2 - 1660056 + 1640342 2 3 - 492777 + 491114 3 7 - 188803 + 188166 7 742 - 111394 + 111018 @@ -15502,17 +15527,17 @@ 1 2 - 1939485 + 1918828 2 3 - 389879 + 388563 3 128 - 123666 + 123249 @@ -15528,22 +15553,22 @@ 1 2 - 1763898 + 1743833 2 3 - 408287 + 406910 3 8 - 190691 + 190048 8 595 - 90153 + 89849 @@ -15559,37 +15584,37 @@ 1 2 - 345038 + 343874 2 3 - 87793 + 87497 3 4 - 49088 + 48923 4 6 - 52393 + 52216 6 12 - 52865 + 52686 12 33 - 50976 + 50804 34 - 3311 - 36816 + 3281 + 36692 @@ -15605,37 +15630,37 @@ 1 2 - 372887 + 371628 2 3 - 78825 + 78559 3 4 - 45784 + 45630 4 6 - 50032 + 49864 6 14 - 53809 + 53627 14 56 - 51449 + 51275 56 - 3228 - 22184 + 3198 + 22109 @@ -15651,27 +15676,27 @@ 1 2 - 462097 + 460537 2 3 - 94873 + 94553 3 5 - 47200 + 47041 5 19 - 51449 + 51275 19 - 2009 - 19352 + 1979 + 19287 @@ -15687,32 +15712,32 @@ 1 2 - 383271 + 381978 2 3 - 91569 + 91260 3 5 - 60417 + 60213 5 9 - 51921 + 51745 9 21 - 50976 + 50804 21 1020 - 36816 + 36692 @@ -15728,17 +15753,17 @@ 1 2 - 4551113 + 4535755 2 3 - 552250 + 550387 3 - 1813 - 279901 + 1783 + 278956 @@ -15754,17 +15779,17 @@ 1 2 - 4956568 + 4939842 2 17 - 415840 + 414436 17 - 1809 - 10856 + 1779 + 10819 @@ -15780,12 +15805,12 @@ 1 2 - 5033506 + 5016520 2 - 1591 - 349758 + 1561 + 348578 @@ -15801,12 +15826,12 @@ 1 2 - 5379017 + 5360865 2 24 - 4248 + 4233 @@ -15816,26 +15841,26 @@ var_def - 4097512 + 4083685 id - 4097512 + 4083685 var_decl_specifiers - 336070 + 334936 id - 336070 + 334936 name - 1416 + 1411 @@ -15849,7 +15874,7 @@ 1 2 - 336070 + 334936 @@ -15865,17 +15890,17 @@ 15 16 - 472 + 470 66 67 - 472 + 470 631 632 - 472 + 470 @@ -15885,30 +15910,30 @@ is_structured_binding - 10 + 18 id - 10 + 18 type_decls - 3292736 + 3283977 id - 3292736 + 3283977 type_id - 3241759 + 3233172 location - 3212494 + 3204006 @@ -15922,7 +15947,7 @@ 1 2 - 3292736 + 3283977 @@ -15938,7 +15963,7 @@ 1 2 - 3292736 + 3283977 @@ -15954,12 +15979,12 @@ 1 2 - 3199750 + 3191305 2 5 - 42008 + 41867 @@ -15975,12 +16000,12 @@ 1 2 - 3199750 + 3191305 2 5 - 42008 + 41867 @@ -15996,12 +16021,12 @@ 1 2 - 3171430 + 3163080 2 20 - 41064 + 40926 @@ -16017,12 +16042,12 @@ 1 2 - 3171430 + 3163080 2 20 - 41064 + 40926 @@ -16032,45 +16057,45 @@ type_def - 2666852 + 2660675 id - 2666852 + 2660675 type_decl_top - 758990 + 755959 type_decl - 758990 + 755959 namespace_decls - 307863 + 306979 id - 307863 + 306979 namespace_id - 1418 + 1414 location - 307863 + 306979 bodylocation - 307863 + 306979 @@ -16084,7 +16109,7 @@ 1 2 - 307863 + 306979 @@ -16100,7 +16125,7 @@ 1 2 - 307863 + 306979 @@ -16116,7 +16141,7 @@ 1 2 - 307863 + 306979 @@ -16132,7 +16157,7 @@ 1 2 - 296 + 295 2 @@ -16142,17 +16167,17 @@ 3 6 - 126 + 125 6 14 - 107 + 106 14 30 - 107 + 106 30 @@ -16167,17 +16192,17 @@ 80 127 - 107 + 106 129 199 - 107 + 106 201 504 - 107 + 106 512 @@ -16198,7 +16223,7 @@ 1 2 - 296 + 295 2 @@ -16208,17 +16233,17 @@ 3 6 - 126 + 125 6 14 - 107 + 106 14 30 - 107 + 106 30 @@ -16233,17 +16258,17 @@ 80 127 - 107 + 106 129 199 - 107 + 106 201 504 - 107 + 106 512 @@ -16264,7 +16289,7 @@ 1 2 - 296 + 295 2 @@ -16274,17 +16299,17 @@ 3 6 - 126 + 125 6 14 - 107 + 106 14 30 - 107 + 106 30 @@ -16299,17 +16324,17 @@ 80 127 - 107 + 106 129 199 - 107 + 106 201 504 - 107 + 106 512 @@ -16330,7 +16355,7 @@ 1 2 - 307863 + 306979 @@ -16346,7 +16371,7 @@ 1 2 - 307863 + 306979 @@ -16362,7 +16387,7 @@ 1 2 - 307863 + 306979 @@ -16378,7 +16403,7 @@ 1 2 - 307863 + 306979 @@ -16394,7 +16419,7 @@ 1 2 - 307863 + 306979 @@ -16410,7 +16435,7 @@ 1 2 - 307863 + 306979 @@ -16420,19 +16445,19 @@ usings - 375719 + 374921 id - 375719 + 374921 element_id - 319078 + 318471 location - 250164 + 249791 @@ -16446,7 +16471,7 @@ 1 2 - 375719 + 374921 @@ -16462,7 +16487,7 @@ 1 2 - 375719 + 374921 @@ -16478,17 +16503,17 @@ 1 2 - 264325 + 263903 2 3 - 53337 + 53157 3 5 - 1416 + 1411 @@ -16504,17 +16529,17 @@ 1 2 - 264325 + 263903 2 3 - 53337 + 53157 3 5 - 1416 + 1411 @@ -16530,22 +16555,22 @@ 1 2 - 203907 + 203690 2 4 - 11328 + 11289 4 5 - 31624 + 31517 5 11 - 3304 + 3292 @@ -16561,22 +16586,22 @@ 1 2 - 203907 + 203690 2 4 - 11328 + 11289 4 5 - 31624 + 31517 5 11 - 3304 + 3292 @@ -16586,15 +16611,15 @@ using_container - 479100 + 478195 parent - 11308 + 11298 child - 303777 + 303207 @@ -16608,47 +16633,47 @@ 1 2 - 3348 + 3353 2 4 - 961 + 959 4 6 - 428 + 427 6 7 - 2560 + 2555 7 17 - 926 + 925 19 143 - 787 + 786 178 179 - 1332 + 1329 179 183 - 880 + 878 201 488 - 81 + 80 @@ -16664,22 +16689,22 @@ 1 2 - 224046 + 223629 2 3 - 53091 + 52990 3 11 - 24448 + 24401 13 41 - 2189 + 2185 @@ -16689,27 +16714,27 @@ static_asserts - 130790 + 130414 id - 130790 + 130414 condition - 130790 + 130414 message - 29535 + 29450 location - 16816 + 16768 enclosing - 1948 + 1942 @@ -16723,7 +16748,7 @@ 1 2 - 130790 + 130414 @@ -16739,7 +16764,7 @@ 1 2 - 130790 + 130414 @@ -16755,7 +16780,7 @@ 1 2 - 130790 + 130414 @@ -16771,7 +16796,7 @@ 1 2 - 130790 + 130414 @@ -16787,7 +16812,7 @@ 1 2 - 130790 + 130414 @@ -16803,7 +16828,7 @@ 1 2 - 130790 + 130414 @@ -16819,7 +16844,7 @@ 1 2 - 130790 + 130414 @@ -16835,7 +16860,7 @@ 1 2 - 130790 + 130414 @@ -16851,32 +16876,32 @@ 1 2 - 22006 + 21943 2 3 - 403 + 402 3 4 - 2774 + 2766 4 11 - 1425 + 1420 12 17 - 2383 + 2376 17 513 - 542 + 540 @@ -16892,32 +16917,32 @@ 1 2 - 22006 + 21943 2 3 - 403 + 402 3 4 - 2774 + 2766 4 11 - 1425 + 1420 12 17 - 2383 + 2376 17 513 - 542 + 540 @@ -16933,12 +16958,12 @@ 1 2 - 27416 + 27337 2 33 - 2118 + 2112 @@ -16954,27 +16979,27 @@ 1 2 - 23425 + 23357 2 3 - 189 + 188 3 4 - 2572 + 2565 4 11 - 1267 + 1263 12 21 - 2080 + 2074 @@ -16990,22 +17015,22 @@ 1 2 - 3133 + 3124 2 3 - 2705 + 2697 3 4 - 1311 + 1307 5 6 - 3632 + 3621 6 @@ -17015,7 +17040,7 @@ 14 15 - 2055 + 2049 16 @@ -17025,12 +17050,12 @@ 17 18 - 3411 + 3401 19 52 - 346 + 345 @@ -17046,22 +17071,22 @@ 1 2 - 3133 + 3124 2 3 - 2705 + 2697 3 4 - 1311 + 1307 5 6 - 3632 + 3621 6 @@ -17071,7 +17096,7 @@ 14 15 - 2055 + 2049 16 @@ -17081,12 +17106,12 @@ 17 18 - 3411 + 3401 19 52 - 346 + 345 @@ -17102,17 +17127,17 @@ 1 2 - 4634 + 4621 2 3 - 5958 + 5941 3 4 - 6040 + 6023 4 @@ -17133,32 +17158,32 @@ 1 2 - 3739 + 3728 2 3 - 6129 + 6111 3 4 - 1084 + 1081 4 5 - 3600 + 3590 5 6 - 189 + 188 13 14 - 2055 + 2049 16 @@ -17179,7 +17204,7 @@ 1 2 - 1380 + 1376 2 @@ -17199,7 +17224,7 @@ 209 11052 - 126 + 125 @@ -17215,7 +17240,7 @@ 1 2 - 1380 + 1376 2 @@ -17235,7 +17260,7 @@ 209 11052 - 126 + 125 @@ -17251,17 +17276,17 @@ 1 2 - 1551 + 1546 2 6 - 151 + 150 9 210 - 170 + 169 223 @@ -17282,7 +17307,7 @@ 1 2 - 1538 + 1534 2 @@ -17307,23 +17332,23 @@ params - 6863014 + 6825742 id - 6696867 + 6660155 function - 3967709 + 3940208 index - 8024 + 7997 type_id - 2256204 + 2234478 @@ -17337,7 +17362,7 @@ 1 2 - 6696867 + 6660155 @@ -17353,7 +17378,7 @@ 1 2 - 6696867 + 6660155 @@ -17369,12 +17394,12 @@ 1 2 - 6571312 + 6535025 2 4 - 125554 + 125130 @@ -17390,22 +17415,22 @@ 1 2 - 2325117 + 2303158 2 3 - 963842 + 960590 3 4 - 434720 + 433253 4 18 - 244028 + 243205 @@ -17421,22 +17446,22 @@ 1 2 - 2325117 + 2303158 2 3 - 963842 + 960590 3 4 - 434720 + 433253 4 18 - 244028 + 243205 @@ -17452,22 +17477,22 @@ 1 2 - 2628619 + 2605636 2 3 - 834040 + 831225 3 4 - 350702 + 349519 4 12 - 154347 + 153826 @@ -17483,72 +17508,72 @@ 2 3 - 944 + 940 4 5 - 472 + 470 6 7 - 472 + 470 8 9 - 944 + 940 9 10 - 472 + 470 10 11 - 944 + 940 11 12 - 472 + 470 19 20 - 472 + 470 64 65 - 472 + 470 194 195 - 472 + 470 517 518 - 472 + 470 1438 1439 - 472 + 470 3480 3481 - 472 + 470 - 8406 - 8407 - 472 + 8376 + 8377 + 470 @@ -17564,72 +17589,72 @@ 2 3 - 944 + 940 4 5 - 472 + 470 6 7 - 472 + 470 8 9 - 944 + 940 9 10 - 472 + 470 10 11 - 944 + 940 11 12 - 472 + 470 19 20 - 472 + 470 64 65 - 472 + 470 194 195 - 472 + 470 517 518 - 472 + 470 1438 1439 - 472 + 470 3480 3481 - 472 + 470 - 8406 - 8407 - 472 + 8376 + 8377 + 470 @@ -17645,67 +17670,67 @@ 1 2 - 944 + 940 3 4 - 472 + 470 4 5 - 472 + 470 5 6 - 472 + 470 6 7 - 1416 + 1411 7 8 - 944 + 940 11 12 - 472 + 470 42 43 - 472 + 470 106 107 - 472 + 470 228 229 - 472 + 470 582 583 - 472 + 470 1275 1276 - 472 + 470 - 3696 - 3697 - 472 + 3666 + 3667 + 470 @@ -17721,22 +17746,22 @@ 1 2 - 1544886 + 1525560 2 3 - 447936 + 446425 3 8 - 172283 + 171701 8 522 - 91097 + 90790 @@ -17752,22 +17777,22 @@ 1 2 - 1769090 + 1749008 2 3 - 251580 + 250731 3 9 - 170395 + 169820 9 506 - 65137 + 64917 @@ -17783,17 +17808,17 @@ 1 2 - 1821955 + 1801694 2 3 - 354478 + 353282 3 13 - 79769 + 79500 @@ -17803,15 +17828,15 @@ overrides - 160287 + 159827 new - 125386 + 125026 old - 15139 + 15096 @@ -17825,12 +17850,12 @@ 1 2 - 90491 + 90231 2 3 - 34888 + 34788 3 @@ -17851,37 +17876,37 @@ 1 2 - 7945 + 7922 2 3 - 1910 + 1905 3 4 - 989 + 987 4 5 - 1324 + 1320 5 11 - 1216 + 1213 11 60 - 1166 + 1163 61 231 - 586 + 584 @@ -17891,19 +17916,19 @@ membervariables - 1051799 + 1051873 id - 1050009 + 1050083 type_id - 326270 + 326293 name - 449611 + 449643 @@ -17917,7 +17942,7 @@ 1 2 - 1048298 + 1048372 2 @@ -17938,7 +17963,7 @@ 1 2 - 1050009 + 1050083 @@ -17954,22 +17979,22 @@ 1 2 - 241948 + 241965 2 3 - 51667 + 51670 3 10 - 25415 + 25417 10 4152 - 7238 + 7239 @@ -17985,17 +18010,17 @@ 1 2 - 254119 + 254137 2 3 - 46257 + 46261 3 40 - 24501 + 24502 41 @@ -18016,22 +18041,22 @@ 1 2 - 294013 + 294034 2 3 - 86151 + 86157 3 5 - 41007 + 41010 5 646 - 28438 + 28440 @@ -18047,17 +18072,17 @@ 1 2 - 366204 + 366230 2 3 - 51508 + 51511 3 650 - 31899 + 31901 @@ -18238,19 +18263,19 @@ localvariables - 581746 + 581698 id - 581746 + 581698 type_id - 37913 + 37905 name - 91411 + 91404 @@ -18264,7 +18289,7 @@ 1 2 - 581746 + 581698 @@ -18280,7 +18305,7 @@ 1 2 - 581746 + 581698 @@ -18296,12 +18321,12 @@ 1 2 - 21213 + 21207 2 3 - 5413 + 5417 3 @@ -18311,12 +18336,12 @@ 4 7 - 3413 + 3408 7 18 - 2879 + 2878 18 @@ -18337,7 +18362,7 @@ 1 2 - 27001 + 26999 2 @@ -18347,7 +18372,7 @@ 3 5 - 2946 + 2942 5 @@ -18373,17 +18398,17 @@ 1 2 - 57575 + 57570 2 3 - 14421 + 14420 3 5 - 8389 + 8388 5 @@ -18393,7 +18418,7 @@ 15 5176 - 3976 + 3975 @@ -18409,7 +18434,7 @@ 1 2 - 77221 + 77215 2 @@ -18419,7 +18444,7 @@ 3 1486 - 6708 + 6707 @@ -18429,15 +18454,15 @@ autoderivation - 149996 + 149665 var - 149996 + 149665 derivation_type - 525 + 524 @@ -18451,7 +18476,7 @@ 1 2 - 149996 + 149665 @@ -18467,27 +18492,27 @@ 33 34 - 105 + 104 90 91 - 105 + 104 353 354 - 105 + 104 392 393 - 105 + 104 560 561 - 105 + 104 @@ -18497,19 +18522,19 @@ enumconstants - 240596 + 240613 id - 240596 + 240613 parent - 28399 + 28401 index - 10182 + 10183 type_id @@ -18517,11 +18542,11 @@ name - 240318 + 240334 location - 220589 + 220605 @@ -18535,7 +18560,7 @@ 1 2 - 240596 + 240613 @@ -18551,7 +18576,7 @@ 1 2 - 240596 + 240613 @@ -18567,7 +18592,7 @@ 1 2 - 240596 + 240613 @@ -18583,7 +18608,7 @@ 1 2 - 240596 + 240613 @@ -18599,7 +18624,7 @@ 1 2 - 240596 + 240613 @@ -18630,7 +18655,7 @@ 4 5 - 3897 + 3898 5 @@ -18696,7 +18721,7 @@ 4 5 - 3897 + 3898 5 @@ -18747,7 +18772,7 @@ 1 2 - 28399 + 28401 @@ -18778,7 +18803,7 @@ 4 5 - 3897 + 3898 5 @@ -19002,7 +19027,7 @@ 1 2 - 10182 + 10183 @@ -19210,7 +19235,7 @@ 1 2 - 240039 + 240056 2 @@ -19231,7 +19256,7 @@ 1 2 - 240039 + 240056 2 @@ -19252,7 +19277,7 @@ 1 2 - 240318 + 240334 @@ -19268,7 +19293,7 @@ 1 2 - 240318 + 240334 @@ -19284,7 +19309,7 @@ 1 2 - 240039 + 240056 2 @@ -19305,7 +19330,7 @@ 1 2 - 219834 + 219849 2 @@ -19326,7 +19351,7 @@ 1 2 - 220589 + 220605 @@ -19342,7 +19367,7 @@ 1 2 - 219834 + 219849 2 @@ -19363,7 +19388,7 @@ 1 2 - 220589 + 220605 @@ -19379,7 +19404,7 @@ 1 2 - 219834 + 219849 2 @@ -19394,31 +19419,31 @@ builtintypes - 22184 + 22109 id - 22184 + 22109 name - 22184 + 22109 kind - 22184 + 22109 size - 3304 + 3292 sign - 1416 + 1411 alignment - 2360 + 2352 @@ -19432,7 +19457,7 @@ 1 2 - 22184 + 22109 @@ -19448,7 +19473,7 @@ 1 2 - 22184 + 22109 @@ -19464,7 +19489,7 @@ 1 2 - 22184 + 22109 @@ -19480,7 +19505,7 @@ 1 2 - 22184 + 22109 @@ -19496,7 +19521,7 @@ 1 2 - 22184 + 22109 @@ -19512,7 +19537,7 @@ 1 2 - 22184 + 22109 @@ -19528,7 +19553,7 @@ 1 2 - 22184 + 22109 @@ -19544,7 +19569,7 @@ 1 2 - 22184 + 22109 @@ -19560,7 +19585,7 @@ 1 2 - 22184 + 22109 @@ -19576,7 +19601,7 @@ 1 2 - 22184 + 22109 @@ -19592,7 +19617,7 @@ 1 2 - 22184 + 22109 @@ -19608,7 +19633,7 @@ 1 2 - 22184 + 22109 @@ -19624,7 +19649,7 @@ 1 2 - 22184 + 22109 @@ -19640,7 +19665,7 @@ 1 2 - 22184 + 22109 @@ -19656,7 +19681,7 @@ 1 2 - 22184 + 22109 @@ -19672,37 +19697,37 @@ 1 2 - 472 + 470 2 3 - 472 + 470 4 5 - 472 + 470 7 8 - 472 + 470 9 10 - 472 + 470 11 12 - 472 + 470 13 14 - 472 + 470 @@ -19718,37 +19743,37 @@ 1 2 - 472 + 470 2 3 - 472 + 470 4 5 - 472 + 470 7 8 - 472 + 470 9 10 - 472 + 470 11 12 - 472 + 470 13 14 - 472 + 470 @@ -19764,37 +19789,37 @@ 1 2 - 472 + 470 2 3 - 472 + 470 4 5 - 472 + 470 7 8 - 472 + 470 9 10 - 472 + 470 11 12 - 472 + 470 13 14 - 472 + 470 @@ -19810,12 +19835,12 @@ 1 2 - 944 + 940 3 4 - 2360 + 2352 @@ -19831,12 +19856,12 @@ 1 2 - 2360 + 2352 2 3 - 944 + 940 @@ -19852,17 +19877,17 @@ 6 7 - 472 + 470 12 13 - 472 + 470 29 30 - 472 + 470 @@ -19878,17 +19903,17 @@ 6 7 - 472 + 470 12 13 - 472 + 470 29 30 - 472 + 470 @@ -19904,17 +19929,17 @@ 6 7 - 472 + 470 12 13 - 472 + 470 29 30 - 472 + 470 @@ -19930,12 +19955,12 @@ 5 6 - 944 + 940 7 8 - 472 + 470 @@ -19951,7 +19976,7 @@ 5 6 - 1416 + 1411 @@ -19967,27 +19992,27 @@ 4 5 - 472 + 470 8 9 - 472 + 470 10 11 - 472 + 470 12 13 - 472 + 470 13 14 - 472 + 470 @@ -20003,27 +20028,27 @@ 4 5 - 472 + 470 8 9 - 472 + 470 10 11 - 472 + 470 12 13 - 472 + 470 13 14 - 472 + 470 @@ -20039,27 +20064,27 @@ 4 5 - 472 + 470 8 9 - 472 + 470 10 11 - 472 + 470 12 13 - 472 + 470 13 14 - 472 + 470 @@ -20075,12 +20100,12 @@ 1 2 - 472 + 470 2 3 - 1888 + 1881 @@ -20096,7 +20121,7 @@ 3 4 - 2360 + 2352 @@ -20106,23 +20131,23 @@ derivedtypes - 4456239 + 4413446 id - 4456239 + 4413446 name - 2235907 + 2205312 kind - 2832 + 2822 type_id - 2752285 + 2729356 @@ -20136,7 +20161,7 @@ 1 2 - 4456239 + 4413446 @@ -20152,7 +20177,7 @@ 1 2 - 4456239 + 4413446 @@ -20168,7 +20193,7 @@ 1 2 - 4456239 + 4413446 @@ -20184,17 +20209,17 @@ 1 2 - 1961670 + 1935763 2 5 - 175587 + 171231 5 1173 - 98649 + 98317 @@ -20210,12 +20235,12 @@ 1 2 - 2234963 + 2204371 2 3 - 944 + 940 @@ -20231,17 +20256,17 @@ 1 2 - 1961670 + 1935763 2 5 - 175587 + 171231 5 1155 - 98649 + 98317 @@ -20255,34 +20280,34 @@ 12 - 198 - 199 - 472 + 199 + 200 + 470 - 1118 - 1119 - 472 + 1103 + 1104 + 470 1154 1155 - 472 + 470 1223 1224 - 472 + 470 - 2208 - 2209 - 472 + 2193 + 2194 + 470 - 3540 - 3541 - 472 + 3510 + 3511 + 470 @@ -20298,32 +20323,32 @@ 1 2 - 472 + 470 164 165 - 472 + 470 611 612 - 472 + 470 - 796 - 797 - 472 + 783 + 784 + 470 - 1161 - 1162 - 472 + 1149 + 1150 + 470 - 2006 - 2007 - 472 + 1982 + 1983 + 470 @@ -20337,34 +20362,34 @@ 12 - 83 - 84 - 472 + 84 + 85 + 470 - 1118 - 1119 - 472 + 1103 + 1104 + 470 1154 1155 - 472 + 470 1223 1224 - 472 + 470 - 2163 - 2164 - 472 + 2148 + 2149 + 470 - 3540 - 3541 - 472 + 3510 + 3511 + 470 @@ -20380,22 +20405,22 @@ 1 2 - 1698289 + 1685972 2 3 - 570659 + 568733 3 4 - 375719 + 367395 4 54 - 107618 + 107254 @@ -20411,22 +20436,22 @@ 1 2 - 1709617 + 1697262 2 3 - 563107 + 561206 3 4 - 372887 + 364572 4 54 - 106674 + 106314 @@ -20442,22 +20467,22 @@ 1 2 - 1702537 + 1690206 2 3 - 574435 + 572496 3 4 - 374775 + 366454 4 6 - 100537 + 100198 @@ -20467,19 +20492,19 @@ pointerishsize - 3320584 + 3314342 id - 3320584 + 3314342 size - 472 + 35 alignment - 472 + 35 @@ -20493,7 +20518,7 @@ 1 2 - 3320584 + 3314342 @@ -20509,7 +20534,7 @@ 1 2 - 3320584 + 3314342 @@ -20523,9 +20548,9 @@ 12 - 7035 - 7036 - 472 + 94071 + 94072 + 35 @@ -20541,7 +20566,7 @@ 1 2 - 472 + 35 @@ -20555,9 +20580,9 @@ 12 - 7035 - 7036 - 472 + 94071 + 94072 + 35 @@ -20573,7 +20598,7 @@ 1 2 - 472 + 35 @@ -20583,23 +20608,23 @@ arraysizes - 71745 + 71503 id - 71745 + 71503 num_elements - 23600 + 23520 bytesize - 26432 + 26343 alignment - 1888 + 1881 @@ -20613,7 +20638,7 @@ 1 2 - 71745 + 71503 @@ -20629,7 +20654,7 @@ 1 2 - 71745 + 71503 @@ -20645,7 +20670,7 @@ 1 2 - 71745 + 71503 @@ -20661,32 +20686,32 @@ 1 2 - 2360 + 2352 2 3 - 15104 + 15053 3 4 - 1416 + 1411 4 6 - 1888 + 1881 6 11 - 1888 + 1881 12 14 - 944 + 940 @@ -20702,22 +20727,22 @@ 1 2 - 18408 + 18346 2 3 - 2360 + 2352 3 4 - 1888 + 1881 4 7 - 944 + 940 @@ -20733,22 +20758,22 @@ 1 2 - 18408 + 18346 2 3 - 2832 + 2822 3 4 - 1416 + 1411 4 5 - 944 + 940 @@ -20764,27 +20789,27 @@ 1 2 - 2832 + 2822 2 3 - 16992 + 16934 3 4 - 3304 + 3292 4 8 - 2360 + 2352 11 16 - 944 + 940 @@ -20800,17 +20825,17 @@ 1 2 - 21712 + 21639 2 3 - 3304 + 3292 3 5 - 1416 + 1411 @@ -20826,17 +20851,17 @@ 1 2 - 22184 + 22109 2 3 - 3304 + 3292 4 5 - 944 + 940 @@ -20852,22 +20877,22 @@ 5 6 - 472 + 470 16 17 - 472 + 470 31 32 - 472 + 470 100 101 - 472 + 470 @@ -20883,17 +20908,17 @@ 4 5 - 472 + 470 7 8 - 944 + 940 50 51 - 472 + 470 @@ -20909,22 +20934,22 @@ 4 5 - 472 + 470 7 8 - 472 + 470 8 9 - 472 + 470 50 51 - 472 + 470 @@ -20934,15 +20959,15 @@ typedefbase - 1731312 + 1724183 id - 1731312 + 1724183 type_id - 809076 + 803747 @@ -20956,7 +20981,7 @@ 1 2 - 1731312 + 1724183 @@ -20972,22 +20997,22 @@ 1 2 - 628341 + 623381 2 3 - 84469 + 84307 3 6 - 64632 + 64497 6 5443 - 31632 + 31560 @@ -20997,23 +21022,23 @@ decltypes - 354722 + 355640 id - 23891 + 23953 expr - 354722 + 355640 base_type - 17137 + 17181 parentheses_would_change_meaning - 17 + 18 @@ -21027,37 +21052,37 @@ 1 2 - 5945 + 5961 2 3 - 7472 + 7492 3 4 - 3251 + 3259 4 7 - 1993 + 1999 7 18 - 1993 + 1999 18 42 - 2011 + 2017 42 1767 - 1221 + 1224 @@ -21073,7 +21098,7 @@ 1 2 - 23891 + 23953 @@ -21089,7 +21114,7 @@ 1 2 - 23891 + 23953 @@ -21105,7 +21130,7 @@ 1 2 - 354722 + 355640 @@ -21121,7 +21146,7 @@ 1 2 - 354722 + 355640 @@ -21137,7 +21162,7 @@ 1 2 - 354722 + 355640 @@ -21153,17 +21178,17 @@ 1 2 - 14442 + 14479 2 3 - 2209 + 2215 3 149 - 485 + 486 @@ -21179,37 +21204,37 @@ 1 2 - 1796 + 1800 2 3 - 7329 + 7348 3 4 - 3071 + 3079 4 5 - 1419 + 1422 5 11 - 1365 + 1368 11 43 - 1562 + 1566 43 6569 - 592 + 594 @@ -21225,7 +21250,7 @@ 1 2 - 17137 + 17181 @@ -21241,7 +21266,7 @@ 1330 1331 - 17 + 18 @@ -21257,7 +21282,7 @@ 19747 19748 - 17 + 18 @@ -21273,7 +21298,7 @@ 954 955 - 17 + 18 @@ -21283,19 +21308,19 @@ usertypes - 5362968 + 5342989 id - 5362968 + 5342989 name - 1392427 + 1383024 kind - 5192 + 5174 @@ -21309,7 +21334,7 @@ 1 2 - 5362968 + 5342989 @@ -21325,7 +21350,7 @@ 1 2 - 5362968 + 5342989 @@ -21341,27 +21366,27 @@ 1 2 - 1009627 + 1001986 2 3 - 162371 + 161352 3 7 - 108562 + 107725 7 - 81 - 104786 + 80 + 104432 - 92 + 80 885 - 7080 + 7526 @@ -21377,17 +21402,17 @@ 1 2 - 1247992 + 1240488 2 3 - 128858 + 127012 3 7 - 15576 + 15523 @@ -21403,57 +21428,57 @@ 6 7 - 472 + 470 10 11 - 472 + 470 26 27 - 472 + 470 - 121 - 122 - 472 + 124 + 125 + 470 - 138 - 139 - 472 + 139 + 140 + 470 - 690 - 691 - 472 + 700 + 701 + 470 - 862 - 863 - 472 + 861 + 862 + 470 963 964 - 472 + 470 - 1770 - 1771 - 472 + 1762 + 1763 + 470 - 1913 - 1914 - 472 + 1899 + 1900 + 470 - 4863 - 4864 - 472 + 4868 + 4869 + 470 @@ -21469,57 +21494,57 @@ 5 6 - 472 + 470 6 7 - 472 + 470 14 15 - 472 + 470 30 31 - 472 + 470 44 45 - 472 + 470 - 124 - 125 - 472 + 126 + 127 + 470 269 270 - 472 + 470 373 374 - 472 + 470 433 434 - 472 + 470 - 749 - 750 - 472 + 748 + 749 + 470 - 1250 - 1251 - 472 + 1236 + 1237 + 470 @@ -21529,19 +21554,19 @@ usertypesize - 1766258 + 1755594 id - 1766258 + 1755594 size - 13688 + 13642 alignment - 2360 + 2352 @@ -21555,7 +21580,7 @@ 1 2 - 1766258 + 1755594 @@ -21571,7 +21596,7 @@ 1 2 - 1766258 + 1755594 @@ -21587,47 +21612,47 @@ 1 2 - 3304 + 3292 2 3 - 4248 + 4233 3 4 - 472 + 470 4 5 - 944 + 940 6 8 - 944 + 940 9 15 - 944 + 940 37 84 - 944 + 940 92 - 160 - 944 + 163 + 940 748 - 2552 - 944 + 2539 + 940 @@ -21643,17 +21668,17 @@ 1 2 - 10384 + 10349 2 3 - 2832 + 2822 3 4 - 472 + 470 @@ -21669,27 +21694,27 @@ 2 3 - 472 + 470 6 7 - 472 + 470 - 181 - 182 - 472 + 184 + 185 + 470 254 255 - 472 + 470 - 3299 - 3300 - 472 + 3286 + 3287 + 470 @@ -21705,27 +21730,27 @@ 1 2 - 472 + 470 2 3 - 472 + 470 3 4 - 472 + 470 9 10 - 472 + 470 22 23 - 472 + 470 @@ -21735,26 +21760,26 @@ usertype_final - 9558 + 9537 id - 9558 + 9537 usertype_uuid - 36206 + 36102 id - 36206 + 36102 uuid - 36206 + 35737 @@ -21768,7 +21793,7 @@ 1 2 - 36206 + 36102 @@ -21784,7 +21809,12 @@ 1 2 - 36206 + 35373 + + + 2 + 3 + 364 @@ -21794,15 +21824,15 @@ mangled_name - 5283199 + 5264430 id - 5283199 + 5264430 mangled_name - 1236664 + 1235313 @@ -21816,7 +21846,7 @@ 1 2 - 5283199 + 5264430 @@ -21832,32 +21862,32 @@ 1 2 - 733502 + 731027 2 3 - 178419 + 178287 3 4 - 82601 + 84674 4 6 - 85905 + 86086 6 - 14 - 99121 + 13 + 93142 - 14 + 13 885 - 57113 + 62094 @@ -21867,59 +21897,59 @@ is_pod_class - 559630 + 554327 id - 559630 + 554327 is_standard_layout_class - 1306521 + 1295997 id - 1306521 + 1295997 is_complete - 1706313 + 1694439 id - 1706313 + 1694439 is_class_template - 406871 + 405028 id - 406871 + 405028 class_instantiation - 1132350 + 1121943 to - 1132350 + 1121943 from - 171811 + 170761 @@ -21933,7 +21963,7 @@ 1 2 - 1132350 + 1121943 @@ -21949,42 +21979,42 @@ 1 2 - 60417 + 58802 2 3 - 29736 + 30106 3 4 - 16048 + 16464 4 5 - 14160 + 14582 5 7 - 15576 + 15523 7 - 14 - 15576 + 13 + 13171 - 14 - 34 - 13216 + 13 + 29 + 13171 - 34 + 30 84 - 7080 + 8937 @@ -21994,19 +22024,19 @@ class_template_argument - 2984763 + 2978116 type_id - 1362392 + 1355715 index - 1297 + 1295 arg_type - 873917 + 863387 @@ -22020,27 +22050,27 @@ 1 2 - 553906 + 551563 2 3 - 419705 + 411593 3 4 - 242829 + 246019 4 7 - 121721 + 122356 7 113 - 24228 + 24182 @@ -22056,22 +22086,22 @@ 1 2 - 578853 + 577422 2 3 - 432393 + 424939 3 4 - 256363 + 257885 4 113 - 94781 + 95468 @@ -22092,7 +22122,7 @@ 2 3 - 822 + 821 3 @@ -22111,12 +22141,12 @@ 592 - 8760 + 8835 104 - 12939 - 114739 + 13776 + 114840 46 @@ -22138,7 +22168,7 @@ 2 3 - 822 + 821 3 @@ -22157,12 +22187,12 @@ 195 - 3458 + 4197 104 - 11165 - 39733 + 10467 + 39739 34 @@ -22179,27 +22209,27 @@ 1 2 - 536062 + 535650 2 3 - 191313 + 181071 3 4 - 53972 + 52573 4 - 11 - 67088 + 10 + 65688 - 11 - 10817 - 25479 + 10 + 11334 + 28403 @@ -22215,17 +22245,17 @@ 1 2 - 762763 + 755683 2 3 - 94156 + 85741 3 22 - 16998 + 21961 @@ -22235,19 +22265,19 @@ class_template_argument_value - 506465 + 508520 type_id - 314830 + 316590 index - 1888 + 1881 arg_value - 506465 + 508520 @@ -22261,17 +22291,17 @@ 1 2 - 259133 + 261081 2 3 - 53809 + 53627 3 4 - 1888 + 1881 @@ -22287,22 +22317,22 @@ 1 2 - 199187 + 200397 2 3 - 81185 + 81852 3 - 4 - 12272 + 5 + 29165 - 4 + 5 9 - 22184 + 5174 @@ -22318,22 +22348,22 @@ 18 19 - 472 + 470 92 93 - 472 + 470 - 299 - 300 - 472 + 309 + 310 + 470 - 380 - 381 - 472 + 376 + 377 + 470 @@ -22349,22 +22379,22 @@ 19 20 - 472 + 470 124 125 - 472 + 470 - 410 - 411 - 472 + 425 + 426 + 470 - 520 - 521 - 472 + 513 + 514 + 470 @@ -22380,7 +22410,7 @@ 1 2 - 506465 + 508520 @@ -22396,7 +22426,7 @@ 1 2 - 506465 + 508520 @@ -22406,15 +22436,15 @@ is_proxy_class_for - 65137 + 65387 id - 65137 + 65387 templ_param_id - 65137 + 65387 @@ -22428,7 +22458,7 @@ 1 2 - 65137 + 65387 @@ -22444,7 +22474,7 @@ 1 2 - 65137 + 65387 @@ -22454,19 +22484,19 @@ type_mentions - 4011226 + 4011508 id - 4011226 + 4011508 type_id - 197321 + 197335 location - 3977856 + 3978135 kind @@ -22484,7 +22514,7 @@ 1 2 - 4011226 + 4011508 @@ -22500,7 +22530,7 @@ 1 2 - 4011226 + 4011508 @@ -22516,7 +22546,7 @@ 1 2 - 4011226 + 4011508 @@ -22532,17 +22562,17 @@ 1 2 - 97169 + 97176 2 3 - 21637 + 21638 3 4 - 8193 + 8194 4 @@ -22552,22 +22582,22 @@ 5 7 - 14318 + 14319 7 12 - 15790 + 15791 12 27 - 15114 + 15115 27 8555 - 14358 + 14359 @@ -22583,17 +22613,17 @@ 1 2 - 97169 + 97176 2 3 - 21637 + 21638 3 4 - 8193 + 8194 4 @@ -22603,22 +22633,22 @@ 5 7 - 14318 + 14319 7 12 - 15790 + 15791 12 27 - 15114 + 15115 27 8555 - 14358 + 14359 @@ -22634,7 +22664,7 @@ 1 2 - 197321 + 197335 @@ -22650,12 +22680,12 @@ 1 2 - 3944485 + 3944762 2 3 - 33370 + 33373 @@ -22671,12 +22701,12 @@ 1 2 - 3944485 + 3944762 2 3 - 33370 + 33373 @@ -22692,7 +22722,7 @@ 1 2 - 3977856 + 3978135 @@ -22750,26 +22780,26 @@ is_function_template - 1418387 + 1413601 id - 1418387 + 1413601 function_instantiation - 895034 + 906422 to - 895034 + 906422 from - 144729 + 146002 @@ -22783,7 +22813,7 @@ 1 2 - 895034 + 906422 @@ -22799,27 +22829,27 @@ 1 2 - 100124 + 101152 2 3 - 14448 + 14480 3 6 - 11964 + 12014 6 21 - 11964 + 12049 22 - 864 - 6227 + 869 + 6306 @@ -22829,19 +22859,19 @@ function_template_argument - 2313992 + 2339815 function_id - 1303055 + 1318394 index - 559 + 563 arg_type - 302683 + 305041 @@ -22855,22 +22885,22 @@ 1 2 - 668792 + 679667 2 3 - 388709 + 388084 3 4 - 175375 + 179966 4 15 - 70178 + 70676 @@ -22886,22 +22916,22 @@ 1 2 - 683870 + 694852 2 3 - 393991 + 393404 3 4 - 146583 + 150970 4 9 - 78609 + 79167 @@ -22917,57 +22947,57 @@ 1 2 - 209 + 211 7 8 - 34 + 35 35 36 - 34 + 35 108 109 - 34 + 35 164 165 - 34 + 35 294 295 - 34 + 35 849 850 - 34 + 35 - 3198 - 3199 - 34 + 3293 + 3294 + 35 - 8582 - 8583 - 34 + 8487 + 8488 + 35 - 17395 - 17396 - 34 + 17489 + 17490 + 35 - 34286 - 34287 - 34 + 34459 + 34460 + 35 @@ -22983,57 +23013,57 @@ 1 2 - 209 + 211 3 4 - 34 + 35 11 12 - 34 + 35 22 23 - 34 + 35 30 31 - 34 + 35 61 62 - 34 + 35 134 135 - 34 + 35 - 452 - 453 - 34 + 453 + 454 + 35 - 1127 - 1128 - 34 + 1126 + 1127 + 35 2404 2405 - 34 + 35 - 5836 - 5837 - 34 + 5842 + 5843 + 35 @@ -23049,32 +23079,32 @@ 1 2 - 185521 + 186978 2 3 - 44534 + 44850 3 5 - 23334 + 23218 5 16 - 23369 + 23535 16 - 113 - 25118 + 107 + 23006 - 126 + 108 955 - 804 + 3452 @@ -23090,17 +23120,17 @@ 1 2 - 272771 + 274918 2 4 - 25818 + 26001 4 17 - 4093 + 4122 @@ -23110,19 +23140,19 @@ function_template_argument_value - 358342 + 362998 function_id - 179083 + 181340 index - 559 + 563 arg_value - 355719 + 360356 @@ -23136,12 +23166,12 @@ 1 2 - 169778 + 171969 2 8 - 9305 + 9371 @@ -23157,17 +23187,17 @@ 1 2 - 149417 + 151428 2 3 - 20535 + 20681 3 97 - 9130 + 9230 @@ -23183,52 +23213,52 @@ 1 2 - 209 + 211 2 3 - 69 + 70 11 12 - 34 + 35 26 27 - 34 + 35 94 95 - 34 + 35 314 315 - 34 + 35 709 710 - 34 + 35 - 964 - 965 - 34 + 992 + 993 + 35 1187 1188 - 34 + 35 2148 2149 - 34 + 35 @@ -23244,52 +23274,52 @@ 1 2 - 209 + 211 2 3 - 69 + 70 60 61 - 34 + 35 80 81 - 34 + 35 141 142 - 34 + 35 533 534 - 34 + 35 - 1550 - 1551 - 34 + 1610 + 1611 + 35 1821 1822 - 34 + 35 2202 2203 - 34 + 35 3771 3772 - 34 + 35 @@ -23305,12 +23335,12 @@ 1 2 - 353095 + 357714 2 3 - 2623 + 2642 @@ -23326,7 +23356,7 @@ 1 2 - 355719 + 360356 @@ -23336,26 +23366,26 @@ is_variable_template - 42225 + 42132 id - 42225 + 42132 variable_instantiation - 48633 + 49154 to - 48633 + 49154 from - 24894 + 25049 @@ -23369,7 +23399,7 @@ 1 2 - 48633 + 49154 @@ -23385,22 +23415,22 @@ 1 2 - 14390 + 14358 2 3 - 7667 + 7650 3 8 - 1785 + 1991 8 14 - 1050 + 1048 @@ -23410,19 +23440,19 @@ variable_template_argument - 328248 + 327628 variable_id - 26469 + 26411 index - 1785 + 1781 arg_type - 217116 + 216742 @@ -23436,22 +23466,22 @@ 1 2 - 16176 + 16140 2 3 - 5672 + 5659 3 4 - 3886 + 3877 4 17 - 735 + 733 @@ -23467,52 +23497,52 @@ 1 2 - 5987 + 5974 2 3 - 5251 + 5240 3 4 - 1995 + 1991 4 5 - 1155 + 1152 5 6 - 2415 + 2410 6 8 - 2310 + 2305 8 11 - 2205 + 2200 11 18 - 2415 + 2410 18 67 - 1995 + 1991 80 - 515 - 735 + 516 + 733 @@ -23528,42 +23558,42 @@ 1 2 - 105 + 104 2 3 - 630 + 628 3 4 - 420 + 419 5 6 - 210 + 209 27 28 - 105 + 104 42 43 - 105 + 104 79 80 - 105 + 104 248 249 - 105 + 104 @@ -23579,52 +23609,52 @@ 1 2 - 105 + 104 12 13 - 630 + 628 13 14 - 105 + 104 14 15 - 315 + 314 24 25 - 105 + 104 32 33 - 105 + 104 128 129 - 105 + 104 - 436 - 437 - 105 + 437 + 438 + 104 636 637 - 105 + 104 890 891 - 105 + 104 @@ -23640,17 +23670,17 @@ 1 2 - 181088 + 180793 2 3 - 20902 + 20856 3 37 - 15125 + 15092 @@ -23666,17 +23696,17 @@ 1 2 - 199890 + 199553 2 5 - 16806 + 16769 5 6 - 420 + 419 @@ -23686,19 +23716,19 @@ variable_template_argument_value - 15650 + 15616 variable_id - 2836 + 2829 index - 420 + 419 arg_value - 12079 + 12052 @@ -23712,12 +23742,12 @@ 1 2 - 2625 + 2620 2 3 - 210 + 209 @@ -23733,42 +23763,42 @@ 2 3 - 420 + 419 3 4 - 105 + 104 4 5 - 1365 + 1362 5 6 - 210 + 209 6 7 - 210 + 209 8 9 - 210 + 209 12 17 - 210 + 209 20 21 - 105 + 104 @@ -23784,22 +23814,22 @@ 2 3 - 105 + 104 6 7 - 105 + 104 8 9 - 105 + 104 13 14 - 105 + 104 @@ -23815,22 +23845,22 @@ 12 13 - 105 + 104 30 31 - 105 + 104 33 34 - 105 + 104 40 41 - 105 + 104 @@ -23846,12 +23876,12 @@ 1 2 - 8508 + 8489 2 3 - 3571 + 3563 @@ -23867,7 +23897,7 @@ 1 2 - 12079 + 12052 @@ -23877,15 +23907,15 @@ routinetypes - 543059 + 546982 id - 543059 + 546982 return_type - 283826 + 285945 @@ -23899,7 +23929,7 @@ 1 2 - 543059 + 546982 @@ -23915,17 +23945,17 @@ 1 2 - 247373 + 249233 2 3 - 21165 + 21315 3 3594 - 15288 + 15396 @@ -23935,19 +23965,19 @@ routinetypeargs - 996883 + 993519 routine - 430472 + 429019 index - 8024 + 7997 type_id - 230340 + 229563 @@ -23961,27 +23991,27 @@ 1 2 - 156235 + 155707 2 3 - 135938 + 135479 3 4 - 64193 + 63976 4 5 - 46256 + 46100 5 18 - 27848 + 27754 @@ -23997,27 +24027,27 @@ 1 2 - 186443 + 185814 2 3 - 135466 + 135009 3 4 - 59473 + 59272 4 5 - 33984 + 33869 5 11 - 15104 + 15053 @@ -24033,67 +24063,67 @@ 2 3 - 944 + 940 4 5 - 472 + 470 6 7 - 472 + 470 8 9 - 944 + 940 9 10 - 472 + 470 10 11 - 1416 + 1411 13 14 - 472 + 470 28 29 - 472 + 470 59 60 - 472 + 470 157 158 - 472 + 470 293 294 - 472 + 470 581 582 - 472 + 470 912 913 - 472 + 470 @@ -24109,57 +24139,57 @@ 1 2 - 944 + 940 3 4 - 944 + 940 4 5 - 1416 + 1411 5 6 - 944 + 940 6 7 - 944 + 940 10 11 - 472 + 470 14 15 - 472 + 470 47 48 - 472 + 470 90 91 - 472 + 470 176 177 - 472 + 470 349 350 - 472 + 470 @@ -24175,27 +24205,27 @@ 1 2 - 149154 + 148651 2 3 - 31152 + 31047 3 5 - 16992 + 16934 5 12 - 18408 + 18346 12 113 - 14632 + 14582 @@ -24211,22 +24241,22 @@ 1 2 - 175587 + 174994 2 3 - 31152 + 31047 3 6 - 18880 + 18816 6 14 - 4720 + 4704 @@ -24236,19 +24266,19 @@ ptrtomembers - 38232 + 38103 id - 38232 + 38103 type_id - 38232 + 38103 class_id - 15576 + 15523 @@ -24262,7 +24292,7 @@ 1 2 - 38232 + 38103 @@ -24278,7 +24308,7 @@ 1 2 - 38232 + 38103 @@ -24294,7 +24324,7 @@ 1 2 - 38232 + 38103 @@ -24310,7 +24340,7 @@ 1 2 - 38232 + 38103 @@ -24326,17 +24356,17 @@ 1 2 - 13688 + 13642 8 9 - 1416 + 1411 28 29 - 472 + 470 @@ -24352,17 +24382,17 @@ 1 2 - 13688 + 13642 8 9 - 1416 + 1411 28 29 - 472 + 470 @@ -24372,15 +24402,15 @@ specifiers - 25016 + 24932 id - 25016 + 24932 str - 25016 + 24932 @@ -24394,7 +24424,7 @@ 1 2 - 25016 + 24932 @@ -24410,7 +24440,7 @@ 1 2 - 25016 + 24932 @@ -24420,15 +24450,15 @@ typespecifiers - 1326345 + 1317166 type_id - 1307937 + 1298819 spec_id - 3776 + 3763 @@ -24442,12 +24472,12 @@ 1 2 - 1289529 + 1280473 2 3 - 18408 + 18346 @@ -24463,42 +24493,42 @@ 8 9 - 472 + 470 36 37 - 472 + 470 51 52 - 472 + 470 86 87 - 472 + 470 105 106 - 472 + 470 219 220 - 472 + 470 - 221 - 222 - 472 + 226 + 227 + 470 - 2084 - 2085 - 472 + 2069 + 2070 + 470 @@ -24508,15 +24538,15 @@ funspecifiers - 12993924 + 13049498 func_id - 3955310 + 3975160 spec_id - 699 + 704 @@ -24530,27 +24560,27 @@ 1 2 - 312758 + 314977 2 3 - 540295 + 544551 3 4 - 1132682 + 1145438 4 5 - 1734095 + 1732127 5 8 - 235478 + 238064 @@ -24566,97 +24596,97 @@ 13 14 - 69 + 70 98 99 - 34 + 35 200 201 - 34 + 35 296 297 - 34 + 35 304 305 - 34 + 35 572 573 - 34 + 35 716 717 - 34 + 35 1599 1600 - 34 + 35 1646 1647 - 34 + 35 3782 3783 - 34 + 35 - 3872 - 3873 - 34 + 3902 + 3903 + 35 5095 5096 - 34 + 35 6814 6815 - 34 + 35 - 9270 - 9271 - 34 + 9303 + 9304 + 35 - 12058 - 12059 - 34 + 12218 + 12219 + 35 - 53280 - 53281 - 34 + 52896 + 52897 + 35 - 80172 - 80173 - 34 + 79931 + 79932 + 35 - 91598 - 91599 - 34 + 91328 + 91329 + 35 - 100025 - 100026 - 34 + 99658 + 99659 + 35 @@ -24666,15 +24696,15 @@ varspecifiers - 2352966 + 2347848 var_id - 1257904 + 1255071 spec_id - 3776 + 3763 @@ -24688,22 +24718,22 @@ 1 2 - 738222 + 735731 2 3 - 202491 + 203219 3 4 - 59001 + 58802 4 5 - 258189 + 257317 @@ -24719,42 +24749,42 @@ 112 113 - 472 + 470 315 316 - 472 + 470 - 411 - 412 - 472 + 414 + 415 + 470 560 561 - 472 + 470 692 693 - 472 + 470 700 701 - 472 + 470 732 733 - 472 + 470 - 1463 - 1464 - 472 + 1466 + 1467 + 470 @@ -24764,27 +24794,27 @@ attributes - 696623 + 696970 id - 696623 + 696970 kind - 315 + 314 name - 1575 + 1676 name_space - 210 + 209 location - 484652 + 484420 @@ -24798,7 +24828,7 @@ 1 2 - 696623 + 696970 @@ -24814,7 +24844,7 @@ 1 2 - 696623 + 696970 @@ -24830,7 +24860,7 @@ 1 2 - 696623 + 696970 @@ -24846,7 +24876,7 @@ 1 2 - 696623 + 696970 @@ -24862,17 +24892,17 @@ 4 5 - 105 + 104 2168 2169 - 105 + 104 - 4460 - 4461 - 105 + 4478 + 4479 + 104 @@ -24888,248 +24918,258 @@ 1 2 - 105 - - - 5 - 6 - 105 - - - 11 - 12 - 105 - - - - - - - kind - name_space - - - 12 - - - 1 - 2 - 210 - - - 2 - 3 - 105 - - - - - - - kind - location - - - 12 - - - 2 - 3 - 105 - - - 2055 - 2056 - 105 - - - 2557 - 2558 - 105 - - - - - - - name - id - - - 12 - - - 1 - 2 - 210 - - - 2 - 3 - 105 - - - 4 - 5 - 210 - - - 5 - 6 - 105 - - - 9 - 10 - 105 - - - 14 - 15 - 105 - - - 16 - 17 - 105 - - - 24 - 25 - 105 - - - 86 - 87 - 105 - - - 115 - 116 - 105 - - - 659 - 660 - 105 - - - 1760 - 1761 - 105 - - - 3932 - 3933 - 105 - - - - - - - name - kind - - - 12 - - - 1 - 2 - 1365 - - - 2 - 3 - 210 - - - - - - - name - name_space - - - 12 - - - 1 - 2 - 1575 - - - - - - - name - location - - - 12 - - - 1 - 2 - 315 - - - 2 - 3 - 210 - - - 4 - 5 - 105 + 104 6 7 - 105 + 104 + + + 11 + 12 + 104 + + + + + + + kind + name_space + + + 12 + + + 1 + 2 + 209 + + + 2 + 3 + 104 + + + + + + + kind + location + + + 12 + + + 2 + 3 + 104 + + + 2055 + 2056 + 104 + + + 2565 + 2566 + 104 + + + + + + + name + id + + + 12 + + + 1 + 2 + 209 + + + 2 + 3 + 104 + + + 4 + 5 + 209 + + + 5 + 6 + 104 9 10 - 105 + 104 14 15 - 105 + 104 + + + 16 + 17 + 104 18 19 - 105 + 104 + + + 24 + 25 + 104 + + + 86 + 87 + 104 + + + 115 + 116 + 104 + + + 659 + 660 + 104 + + + 1760 + 1761 + 104 + + + 3932 + 3933 + 104 + + + + + + + name + kind + + + 12 + + + 1 + 2 + 1467 + + + 2 + 3 + 209 + + + + + + + name + name_space + + + 12 + + + 1 + 2 + 1676 + + + + + + + name + location + + + 12 + + + 1 + 2 + 314 + + + 2 + 3 + 209 + + + 4 + 5 + 104 + + + 6 + 7 + 104 + + + 8 + 9 + 104 + + + 9 + 10 + 104 + + + 14 + 15 + 104 + + + 18 + 19 + 104 59 60 - 105 + 104 72 73 - 105 + 104 331 332 - 105 + 104 1756 1757 - 105 + 104 2379 2380 - 105 + 104 @@ -25143,14 +25183,14 @@ 12 - 5 - 6 - 105 + 23 + 24 + 104 6627 6628 - 105 + 104 @@ -25166,12 +25206,12 @@ 1 2 - 105 + 104 3 4 - 105 + 104 @@ -25185,14 +25225,14 @@ 12 - 1 - 2 - 105 + 2 + 3 + 104 14 15 - 105 + 104 @@ -25206,14 +25246,14 @@ 12 - 1 - 2 - 105 + 9 + 10 + 104 4613 4614 - 105 + 104 @@ -25229,17 +25269,17 @@ 1 2 - 443372 + 443126 2 9 - 36868 + 36787 9 201 - 4411 + 4506 @@ -25255,7 +25295,7 @@ 1 2 - 484652 + 484420 @@ -25271,12 +25311,12 @@ 1 2 - 480346 + 480123 2 3 - 4306 + 4297 @@ -25292,7 +25332,7 @@ 1 2 - 484652 + 484420 @@ -25302,27 +25342,27 @@ attribute_args - 351174 + 352341 id - 351174 + 352341 kind - 1416 + 1411 attribute - 269045 + 270489 index - 1416 + 1411 location - 328046 + 329291 @@ -25336,7 +25376,7 @@ 1 2 - 351174 + 352341 @@ -25352,7 +25392,7 @@ 1 2 - 351174 + 352341 @@ -25368,7 +25408,7 @@ 1 2 - 351174 + 352341 @@ -25384,7 +25424,7 @@ 1 2 - 351174 + 352341 @@ -25400,17 +25440,17 @@ 1 2 - 472 + 470 - 55 - 56 - 472 + 54 + 55 + 470 - 688 - 689 - 472 + 694 + 695 + 470 @@ -25426,17 +25466,17 @@ 1 2 - 472 + 470 - 55 - 56 - 472 + 54 + 55 + 470 - 536 - 537 - 472 + 542 + 543 + 470 @@ -25452,12 +25492,12 @@ 1 2 - 944 + 940 3 4 - 472 + 470 @@ -25473,17 +25513,17 @@ 1 2 - 472 + 470 - 55 - 56 - 472 + 54 + 55 + 470 - 666 - 667 - 472 + 672 + 673 + 470 @@ -25499,17 +25539,17 @@ 1 2 - 202963 + 204631 2 3 - 50032 + 49864 3 4 - 16048 + 15994 @@ -25525,12 +25565,12 @@ 1 2 - 258661 + 260140 2 3 - 10384 + 10349 @@ -25546,17 +25586,17 @@ 1 2 - 202963 + 204631 2 3 - 50032 + 49864 3 4 - 16048 + 15994 @@ -25572,17 +25612,17 @@ 1 2 - 202963 + 204631 2 3 - 50032 + 49864 3 4 - 16048 + 15994 @@ -25598,17 +25638,17 @@ 34 35 - 472 + 470 140 141 - 472 + 470 - 570 - 571 - 472 + 575 + 576 + 470 @@ -25624,12 +25664,12 @@ 1 2 - 944 + 940 3 4 - 472 + 470 @@ -25645,17 +25685,17 @@ 34 35 - 472 + 470 140 141 - 472 + 470 - 570 - 571 - 472 + 575 + 576 + 470 @@ -25671,17 +25711,17 @@ 34 35 - 472 + 470 140 141 - 472 + 470 - 521 - 522 - 472 + 526 + 527 + 470 @@ -25697,12 +25737,12 @@ 1 2 - 313886 + 315179 2 16 - 14160 + 14112 @@ -25718,12 +25758,12 @@ 1 2 - 315302 + 316590 2 3 - 12744 + 12701 @@ -25739,12 +25779,12 @@ 1 2 - 313886 + 315179 2 16 - 14160 + 14112 @@ -25760,7 +25800,7 @@ 1 2 - 328046 + 329291 @@ -25770,15 +25810,15 @@ attribute_arg_value - 350702 + 351871 arg - 350702 + 351871 value - 32568 + 34810 @@ -25792,7 +25832,7 @@ 1 2 - 350702 + 351871 @@ -25808,22 +25848,22 @@ 1 2 - 14632 + 16934 2 3 - 12272 + 12230 3 14 - 2832 + 2822 15 247 - 2832 + 2822 @@ -25833,15 +25873,15 @@ attribute_arg_type - 1535 + 470 arg - 1535 + 470 type_id - 1354 + 470 @@ -25855,7 +25895,7 @@ 1 2 - 1535 + 470 @@ -25871,12 +25911,7 @@ 1 2 - 1264 - - - 3 - 4 - 90 + 470 @@ -25939,15 +25974,15 @@ typeattributes - 60817 + 62570 type_id - 60397 + 62150 spec_id - 60817 + 62570 @@ -25961,12 +25996,12 @@ 1 2 - 59977 + 61731 2 3 - 420 + 419 @@ -25982,7 +26017,7 @@ 1 2 - 60817 + 62570 @@ -25992,15 +26027,15 @@ funcattributes - 637684 + 635532 func_id - 448880 + 447366 spec_id - 637684 + 635532 @@ -26014,22 +26049,22 @@ 1 2 - 342678 + 341522 2 3 - 65137 + 64917 3 6 - 40120 + 39985 6 7 - 944 + 940 @@ -26045,7 +26080,7 @@ 1 2 - 637684 + 635532 @@ -26113,15 +26148,15 @@ stmtattributes - 1008 + 1006 stmt_id - 1008 + 1006 spec_id - 1008 + 1006 @@ -26135,7 +26170,7 @@ 1 2 - 1008 + 1006 @@ -26151,7 +26186,7 @@ 1 2 - 1008 + 1006 @@ -26161,15 +26196,15 @@ unspecifiedtype - 10417715 + 10352924 type_id - 10417715 + 10352924 unspecified_type_id - 6996120 + 6956047 @@ -26183,7 +26218,7 @@ 1 2 - 10417715 + 10352924 @@ -26199,17 +26234,17 @@ 1 2 - 4696963 + 4675939 2 3 - 2054184 + 2037843 3 147 - 244972 + 242264 @@ -26219,19 +26254,19 @@ member - 5120282 + 5134480 parent - 692861 + 689567 index - 8746 + 8808 child - 5057871 + 5070710 @@ -26245,42 +26280,42 @@ 1 3 - 18716 + 18884 3 4 - 396265 + 390691 4 5 - 38727 + 39072 5 7 - 52616 + 53059 7 10 - 52476 + 52848 10 16 - 57164 + 57569 16 30 - 52581 + 52954 30 251 - 24313 + 24486 @@ -26296,42 +26331,42 @@ 1 3 - 18716 + 18884 3 4 - 396230 + 390656 4 5 - 38692 + 39107 5 7 - 52791 + 53165 7 10 - 52791 + 53165 10 16 - 56919 + 57323 16 29 - 52371 + 52742 29 253 - 24348 + 24521 @@ -26347,62 +26382,62 @@ 1 2 - 1399 + 1409 2 3 - 804 + 810 3 4 - 944 + 951 5 22 - 769 + 669 22 - 43 - 664 + 42 + 669 - 43 - 61 - 699 + 42 + 56 + 669 - 62 - 113 - 664 + 56 + 100 + 669 - 114 - 185 - 664 + 104 + 164 + 669 - 186 - 331 - 664 + 181 + 299 + 669 - 367 - 1282 - 664 + 300 + 727 + 669 - 1418 - 5994 - 664 + 845 + 4002 + 669 - 7270 - 19474 - 139 + 4606 + 19241 + 281 @@ -26418,62 +26453,62 @@ 1 2 - 804 + 810 2 3 - 874 + 880 3 4 - 1154 + 1162 4 - 16 - 734 + 15 + 669 16 - 33 - 664 + 35 + 739 - 34 - 58 - 664 + 36 + 55 + 669 - 58 + 57 93 - 699 + 739 97 135 - 664 + 669 140 - 253 - 664 + 256 + 669 - 253 - 601 - 664 + 268 + 612 + 669 - 611 - 2463 - 664 + 619 + 2611 + 669 - 2610 - 19486 - 489 + 2770 + 19253 + 458 @@ -26489,7 +26524,7 @@ 1 2 - 5057871 + 5070710 @@ -26505,12 +26540,12 @@ 1 2 - 4996578 + 5008419 2 - 5 - 61292 + 8 + 62290 @@ -26520,15 +26555,15 @@ enclosingfunction - 121976 + 121743 child - 121976 + 121743 parent - 69638 + 69504 @@ -26542,7 +26577,7 @@ 1 2 - 121976 + 121743 @@ -26558,22 +26593,22 @@ 1 2 - 36765 + 36695 2 3 - 21633 + 21591 3 4 - 6117 + 6106 4 45 - 5121 + 5111 @@ -26583,27 +26618,27 @@ derivations - 397280 + 402388 derivation - 397280 + 402388 sub - 376919 + 381883 index - 209 + 211 super - 228236 + 206461 location - 37887 + 38156 @@ -26617,7 +26652,7 @@ 1 2 - 397280 + 402388 @@ -26633,7 +26668,7 @@ 1 2 - 397280 + 402388 @@ -26649,7 +26684,7 @@ 1 2 - 397280 + 402388 @@ -26665,7 +26700,7 @@ 1 2 - 397280 + 402388 @@ -26681,12 +26716,12 @@ 1 2 - 361876 + 366733 2 7 - 15043 + 15149 @@ -26702,12 +26737,12 @@ 1 2 - 361876 + 366733 2 7 - 15043 + 15149 @@ -26723,12 +26758,12 @@ 1 2 - 361876 + 366733 2 7 - 15043 + 15149 @@ -26744,12 +26779,12 @@ 1 2 - 361876 + 366733 2 7 - 15043 + 15149 @@ -26765,22 +26800,22 @@ 25 26 - 104 + 105 77 78 - 34 + 35 430 431 - 34 + 35 - 10774 - 10775 - 34 + 10839 + 10840 + 35 @@ -26796,22 +26831,22 @@ 25 26 - 104 + 105 77 78 - 34 + 35 430 431 - 34 + 35 - 10774 - 10775 - 34 + 10839 + 10840 + 35 @@ -26827,27 +26862,27 @@ 23 24 - 34 + 35 25 26 - 69 + 70 35 36 - 34 + 35 261 262 - 34 + 35 - 6169 - 6170 - 34 + 5505 + 5506 + 35 @@ -26863,22 +26898,22 @@ 1 2 - 104 + 105 9 10 - 34 + 35 66 67 - 34 + 35 1005 1006 - 34 + 35 @@ -26894,12 +26929,12 @@ 1 2 - 220960 + 199133 2 - 1076 - 7276 + 1225 + 7328 @@ -26915,12 +26950,12 @@ 1 2 - 220960 + 199133 2 - 1076 - 7276 + 1225 + 7328 @@ -26936,12 +26971,12 @@ 1 2 - 227781 + 206003 2 4 - 454 + 458 @@ -26957,12 +26992,12 @@ 1 2 - 224668 + 202867 2 108 - 3568 + 3593 @@ -26978,27 +27013,27 @@ 1 2 - 28197 + 28326 2 5 - 3043 + 3135 5 16 - 2973 + 2994 17 133 - 2973 + 2994 - 154 - 656 - 699 + 142 + 474 + 704 @@ -27014,27 +27049,27 @@ 1 2 - 28197 + 28326 2 5 - 3043 + 3135 5 16 - 2973 + 2994 17 133 - 2973 + 2994 - 154 - 656 - 699 + 142 + 474 + 704 @@ -27050,7 +27085,7 @@ 1 2 - 37887 + 38156 @@ -27066,22 +27101,22 @@ 1 2 - 30611 + 30757 2 5 - 3218 + 3417 5 55 - 2868 + 2889 60 - 656 - 1189 + 420 + 1092 @@ -27091,15 +27126,15 @@ derspecifiers - 399169 + 404291 der_id - 396895 + 402001 spec_id - 139 + 140 @@ -27113,12 +27148,12 @@ 1 2 - 394621 + 399710 2 3 - 2273 + 2290 @@ -27134,22 +27169,22 @@ 65 66 - 34 + 35 93 94 - 34 + 35 1132 1133 - 34 + 35 - 10120 - 10121 - 34 + 10185 + 10186 + 35 @@ -27159,15 +27194,15 @@ direct_base_offsets - 368208 + 373110 der_id - 368208 + 373110 offset - 349 + 352 @@ -27181,7 +27216,7 @@ 1 2 - 368208 + 373110 @@ -27197,32 +27232,32 @@ 1 2 - 34 + 35 2 3 - 104 + 105 3 4 - 69 + 70 4 5 - 69 + 70 85 86 - 34 + 35 - 10419 - 10420 - 34 + 10484 + 10485 + 35 @@ -27232,15 +27267,15 @@ virtual_base_offsets - 6674 + 6661 sub - 3684 + 3677 super - 509 + 508 offset @@ -27258,17 +27293,17 @@ 1 2 - 2896 + 2891 2 4 - 324 + 323 4 7 - 266 + 265 7 @@ -27289,7 +27324,7 @@ 1 2 - 3105 + 3099 2 @@ -27299,7 +27334,7 @@ 4 8 - 266 + 265 @@ -27315,7 +27350,7 @@ 1 2 - 81 + 80 2 @@ -27381,7 +27416,7 @@ 2 3 - 81 + 80 4 @@ -27483,7 +27518,7 @@ 1 2 - 81 + 80 2 @@ -27523,23 +27558,23 @@ frienddecls - 710038 + 715075 id - 710038 + 715075 type_id - 42085 + 42384 decl_id - 69688 + 70182 location - 6297 + 6341 @@ -27553,7 +27588,7 @@ 1 2 - 710038 + 715075 @@ -27569,7 +27604,7 @@ 1 2 - 710038 + 715075 @@ -27585,7 +27620,7 @@ 1 2 - 710038 + 715075 @@ -27601,47 +27636,47 @@ 1 2 - 6157 + 6200 2 3 - 13119 + 13212 3 6 - 2938 + 2959 6 10 - 3183 + 3206 10 17 - 3253 + 3276 17 24 - 3323 + 3347 25 36 - 3288 + 3311 37 55 - 3218 + 3241 55 103 - 3603 + 3628 @@ -27657,47 +27692,47 @@ 1 2 - 6157 + 6200 2 3 - 13119 + 13212 3 6 - 2938 + 2959 6 10 - 3183 + 3206 10 17 - 3253 + 3276 17 24 - 3323 + 3347 25 36 - 3288 + 3311 37 55 - 3218 + 3241 55 103 - 3603 + 3628 @@ -27713,12 +27748,12 @@ 1 2 - 40651 + 40939 2 13 - 1434 + 1444 @@ -27734,37 +27769,37 @@ 1 2 - 40196 + 40481 2 3 - 5842 + 5883 3 8 - 5982 + 6024 8 15 - 5387 + 5425 15 32 - 5247 + 5284 32 71 - 5247 + 5284 72 160 - 1784 + 1796 @@ -27780,37 +27815,37 @@ 1 2 - 40196 + 40481 2 3 - 5842 + 5883 3 8 - 5982 + 6024 8 15 - 5387 + 5425 15 32 - 5247 + 5284 32 71 - 5247 + 5284 72 160 - 1784 + 1796 @@ -27826,12 +27861,12 @@ 1 2 - 69023 + 69513 2 5 - 664 + 669 @@ -27847,12 +27882,12 @@ 1 2 - 5912 + 5954 2 20106 - 384 + 387 @@ -27868,12 +27903,12 @@ 1 2 - 6157 + 6200 2 1105 - 139 + 140 @@ -27889,12 +27924,12 @@ 1 2 - 5947 + 5989 2 1837 - 349 + 352 @@ -27904,19 +27939,19 @@ comments - 9024167 + 9004230 id - 9024167 + 9004230 contents - 3546138 + 3538304 location - 9024167 + 9004230 @@ -27930,7 +27965,7 @@ 1 2 - 9024167 + 9004230 @@ -27946,7 +27981,7 @@ 1 2 - 9024167 + 9004230 @@ -27962,17 +27997,17 @@ 1 2 - 3255074 + 3247882 2 10 - 268901 + 268307 10 32841 - 22163 + 22114 @@ -27988,17 +28023,17 @@ 1 2 - 3255074 + 3247882 2 10 - 268901 + 268307 10 32841 - 22163 + 22114 @@ -28014,7 +28049,7 @@ 1 2 - 9024167 + 9004230 @@ -28030,7 +28065,7 @@ 1 2 - 9024167 + 9004230 @@ -28040,15 +28075,15 @@ commentbinding - 3153021 + 3145674 id - 2495512 + 2490384 element - 3075612 + 3068526 @@ -28062,12 +28097,12 @@ 1 2 - 2412911 + 2408061 2 97 - 82601 + 82322 @@ -28083,12 +28118,12 @@ 1 2 - 2998202 + 2991378 2 3 - 77409 + 77148 @@ -28098,15 +28133,15 @@ exprconv - 7003263 + 7003750 converted - 7003263 + 7003750 conversion - 7003263 + 7003750 @@ -28120,7 +28155,7 @@ 1 2 - 7003263 + 7003750 @@ -28136,7 +28171,7 @@ 1 2 - 7003263 + 7003750 @@ -28146,30 +28181,30 @@ compgenerated - 8432513 + 8494343 id - 8432513 + 8494343 synthetic_destructor_call - 132767 + 133110 element - 103217 + 103484 i - 305 + 306 destructor_call - 117462 + 117766 @@ -28183,17 +28218,17 @@ 1 2 - 85325 + 85546 2 3 - 11801 + 11832 3 18 - 6089 + 6105 @@ -28209,17 +28244,17 @@ 1 2 - 85325 + 85546 2 3 - 11801 + 11832 3 18 - 6089 + 6105 @@ -28235,67 +28270,67 @@ 1 2 - 17 + 18 2 3 - 53 + 54 3 4 - 17 + 18 4 5 - 53 + 54 6 7 - 17 + 18 11 12 - 17 + 18 20 21 - 17 + 18 34 35 - 17 + 18 65 66 - 17 + 18 152 153 - 17 + 18 339 340 - 17 + 18 996 997 - 17 + 18 5746 5747 - 17 + 18 @@ -28311,67 +28346,67 @@ 1 2 - 17 + 18 2 3 - 53 + 54 3 4 - 17 + 18 4 5 - 53 + 54 6 7 - 17 + 18 11 12 - 17 + 18 20 21 - 17 + 18 34 35 - 17 + 18 65 66 - 17 + 18 151 152 - 17 + 18 338 339 - 17 + 18 995 996 - 17 + 18 4897 4898 - 17 + 18 @@ -28387,12 +28422,12 @@ 1 2 - 115450 + 115749 2 26 - 2011 + 2017 @@ -28408,7 +28443,7 @@ 1 2 - 117462 + 117766 @@ -28418,15 +28453,15 @@ namespaces - 12744 + 12701 id - 12744 + 12701 name - 10384 + 10349 @@ -28440,7 +28475,7 @@ 1 2 - 12744 + 12701 @@ -28456,17 +28491,17 @@ 1 2 - 8968 + 8937 2 3 - 472 + 470 3 4 - 944 + 940 @@ -28476,26 +28511,26 @@ namespace_inline - 1416 + 1411 id - 1416 + 1411 namespacembrs - 2473800 + 2463100 parentid - 10856 + 10819 memberid - 2473800 + 2463100 @@ -28509,57 +28544,57 @@ 1 2 - 1888 + 1881 2 3 - 944 + 940 3 4 - 472 + 470 4 5 - 944 + 940 5 7 - 944 + 940 7 8 - 944 + 940 8 12 - 944 + 940 17 30 - 944 + 940 43 47 - 944 + 940 52 143 - 944 + 940 - 247 - 4603 - 944 + 253 + 4592 + 940 @@ -28575,7 +28610,7 @@ 1 2 - 2473800 + 2463100 @@ -28585,19 +28620,19 @@ exprparents - 14151887 + 14152882 expr_id - 14151887 + 14152882 child_index - 14601 + 14602 parent_id - 9417337 + 9417999 @@ -28611,7 +28646,7 @@ 1 2 - 14151887 + 14152882 @@ -28627,7 +28662,7 @@ 1 2 - 14151887 + 14152882 @@ -28663,7 +28698,7 @@ 5 8 - 1209 + 1210 8 @@ -28677,7 +28712,7 @@ 56 - 389706 + 354077 369 @@ -28714,7 +28749,7 @@ 5 8 - 1209 + 1210 8 @@ -28728,7 +28763,7 @@ 56 - 389706 + 354077 369 @@ -28745,17 +28780,17 @@ 1 2 - 5388560 + 5388939 2 3 - 3692338 + 3692597 3 712 - 336438 + 336462 @@ -28771,17 +28806,17 @@ 1 2 - 5388560 + 5388939 2 3 - 3692338 + 3692597 3 712 - 336438 + 336462 @@ -28791,22 +28826,22 @@ expr_isload - 4944556 + 4981789 expr_id - 4944556 + 4981789 conversionkinds - 4220603 + 4220621 expr_id - 4220603 + 4220621 kind @@ -28824,7 +28859,7 @@ 1 2 - 4220603 + 4220621 @@ -28848,23 +28883,23 @@ 1 - 13430 - 13431 + 13442 + 13443 1 - 26288 - 26289 + 26289 + 26290 1 - 44458 - 44459 + 44470 + 44471 1 - 4131036 - 4131037 + 4131029 + 4131030 1 @@ -28875,15 +28910,15 @@ iscall - 3095893 + 3078157 caller - 3095893 + 3078157 kind - 53 + 54 @@ -28897,7 +28932,7 @@ 1 2 - 3095893 + 3078157 @@ -28911,19 +28946,19 @@ 12 - 1389 - 1390 - 17 + 1378 + 1379 + 18 2512 2513 - 17 + 18 - 168444 - 168445 - 17 + 167025 + 167026 + 18 @@ -28933,15 +28968,15 @@ numtemplatearguments - 553810 + 543303 expr_id - 553810 + 543303 num - 71 + 72 @@ -28955,7 +28990,7 @@ 1 2 - 553810 + 543303 @@ -28971,22 +29006,22 @@ 26 27 - 17 + 18 28 29 - 17 + 18 220 221 - 17 + 18 - 30556 - 30557 - 17 + 29893 + 29894 + 18 @@ -28996,15 +29031,15 @@ specialnamequalifyingelements - 472 + 470 id - 472 + 470 name - 472 + 470 @@ -29018,7 +29053,7 @@ 1 2 - 472 + 470 @@ -29034,7 +29069,7 @@ 1 2 - 472 + 470 @@ -29044,23 +29079,23 @@ namequalifiers - 1593457 + 1618961 id - 1593457 + 1618961 qualifiableelement - 1593457 + 1618961 qualifyingelement - 74044 + 79675 location - 273366 + 282719 @@ -29074,7 +29109,7 @@ 1 2 - 1593457 + 1618961 @@ -29090,7 +29125,7 @@ 1 2 - 1593457 + 1618961 @@ -29106,7 +29141,7 @@ 1 2 - 1593457 + 1618961 @@ -29122,7 +29157,7 @@ 1 2 - 1593457 + 1618961 @@ -29138,7 +29173,7 @@ 1 2 - 1593457 + 1618961 @@ -29154,7 +29189,7 @@ 1 2 - 1593457 + 1618961 @@ -29170,27 +29205,27 @@ 1 2 - 46848 + 45546 2 3 - 12322 + 17163 3 4 - 5802 + 6195 4 - 11 - 5622 + 8 + 6087 - 11 - 31104 - 3448 + 8 + 31227 + 4682 @@ -29206,27 +29241,27 @@ 1 2 - 46848 + 45546 2 3 - 12322 + 17163 3 4 - 5802 + 6195 4 - 11 - 5622 + 8 + 6087 - 11 - 31104 - 3448 + 8 + 31227 + 4682 @@ -29242,27 +29277,27 @@ 1 2 - 50477 + 49725 2 3 - 10885 + 15992 3 4 - 6089 + 6411 4 - 15 - 5604 + 9 + 6123 - 15 + 9 7095 - 987 + 1422 @@ -29278,32 +29313,32 @@ 1 2 - 85792 + 91742 2 3 - 24160 + 25646 3 4 - 41621 + 42179 4 6 - 12736 + 12895 6 7 - 89547 + 89869 7 2135 - 19508 + 20387 @@ -29319,32 +29354,32 @@ 1 2 - 85792 + 91742 2 3 - 24160 + 25646 3 4 - 41621 + 42179 4 6 - 12736 + 12895 6 7 - 89547 + 89869 7 2135 - 19508 + 20387 @@ -29360,22 +29395,22 @@ 1 2 - 118270 + 125168 2 3 - 50782 + 52354 3 4 - 95942 + 96622 4 152 - 8370 + 8572 @@ -29385,15 +29420,15 @@ varbind - 6005942 + 6006364 expr - 6005942 + 6006364 var - 765575 + 765629 @@ -29407,7 +29442,7 @@ 1 2 - 6005942 + 6006364 @@ -29423,52 +29458,52 @@ 1 2 - 125736 + 125745 2 3 - 137344 + 137353 3 4 - 105884 + 105891 4 5 - 84883 + 84889 5 6 - 61053 + 61057 6 7 - 47927 + 47931 7 9 - 59392 + 59396 9 13 - 59043 + 59047 13 28 - 58653 + 58657 28 5137 - 25655 + 25657 @@ -29478,15 +29513,15 @@ funbind - 3098283 + 3080553 expr - 3094672 + 3076933 fun - 512404 + 514037 @@ -29500,12 +29535,12 @@ 1 2 - 3091061 + 3073313 2 3 - 3610 + 3619 @@ -29521,32 +29556,32 @@ 1 2 - 298514 + 306546 2 3 - 85810 + 78793 3 4 - 36160 + 37226 4 7 - 42968 + 43475 7 - 35 - 38603 + 38 + 38703 - 35 + 38 4943 - 10346 + 9293 @@ -29556,19 +29591,19 @@ expr_allocator - 54190 + 46541 expr - 54190 + 46541 func - 104 + 105 form - 34 + 35 @@ -29582,7 +29617,7 @@ 1 2 - 54190 + 46541 @@ -29598,7 +29633,7 @@ 1 2 - 54190 + 46541 @@ -29614,17 +29649,17 @@ 1 2 - 34 + 35 585 586 - 34 + 35 - 963 - 964 - 34 + 735 + 736 + 35 @@ -29640,7 +29675,7 @@ 1 2 - 104 + 105 @@ -29654,9 +29689,9 @@ 12 - 1549 - 1550 - 34 + 1321 + 1322 + 35 @@ -29672,7 +29707,7 @@ 3 4 - 34 + 35 @@ -29682,19 +29717,19 @@ expr_deallocator - 62901 + 55314 expr - 62901 + 55314 func - 104 + 105 form - 69 + 70 @@ -29708,7 +29743,7 @@ 1 2 - 62901 + 55314 @@ -29724,7 +29759,7 @@ 1 2 - 62901 + 55314 @@ -29740,17 +29775,17 @@ 1 2 - 34 + 35 + + + 722 + 723 + 35 847 848 - 34 - - - 950 - 951 - 34 + 35 @@ -29766,7 +29801,7 @@ 1 2 - 104 + 105 @@ -29780,14 +29815,14 @@ 12 - 848 - 849 - 34 + 722 + 723 + 35 - 950 - 951 - 34 + 848 + 849 + 35 @@ -29803,12 +29838,12 @@ 1 2 - 34 + 35 2 3 - 34 + 35 @@ -29829,15 +29864,15 @@ expr_cond_guard - 654457 + 654502 cond - 654457 + 654502 guard - 654457 + 654502 @@ -29851,7 +29886,7 @@ 1 2 - 654457 + 654502 @@ -29867,7 +29902,7 @@ 1 2 - 654457 + 654502 @@ -29877,15 +29912,15 @@ expr_cond_true - 654455 + 654499 cond - 654455 + 654499 true - 654455 + 654499 @@ -29899,7 +29934,7 @@ 1 2 - 654455 + 654499 @@ -29915,7 +29950,7 @@ 1 2 - 654455 + 654499 @@ -29925,15 +29960,15 @@ expr_cond_false - 654457 + 654502 cond - 654457 + 654502 false - 654457 + 654502 @@ -29947,7 +29982,7 @@ 1 2 - 654457 + 654502 @@ -29963,7 +29998,7 @@ 1 2 - 654457 + 654502 @@ -29973,15 +30008,15 @@ values - 10645398 + 10646146 id - 10645398 + 10646146 str - 86632 + 86639 @@ -29995,7 +30030,7 @@ 1 2 - 10645398 + 10646146 @@ -30011,7 +30046,7 @@ 1 2 - 58704 + 58708 2 @@ -30021,12 +30056,12 @@ 3 6 - 6746 + 6747 6 62 - 6512 + 6513 62 @@ -30041,15 +30076,15 @@ valuetext - 4756700 + 4756729 id - 4756700 + 4756729 text - 703927 + 703924 @@ -30063,7 +30098,7 @@ 1 2 - 4756700 + 4756729 @@ -30079,7 +30114,7 @@ 1 2 - 527530 + 527529 2 @@ -30089,7 +30124,7 @@ 3 7 - 56761 + 56759 7 @@ -30104,15 +30139,15 @@ valuebind - 11082271 + 11083050 val - 10645398 + 10646146 expr - 11082271 + 11083050 @@ -30126,12 +30161,12 @@ 1 2 - 10231303 + 10232022 2 7 - 414094 + 414124 @@ -30147,7 +30182,7 @@ 1 2 - 11082271 + 11083050 @@ -30157,15 +30192,15 @@ fieldoffsets - 1050009 + 1050083 id - 1050009 + 1050083 byteoffset - 22591 + 22593 bitoffset @@ -30183,7 +30218,7 @@ 1 2 - 1050009 + 1050083 @@ -30199,7 +30234,7 @@ 1 2 - 1050009 + 1050083 @@ -30215,7 +30250,7 @@ 1 2 - 12966 + 12967 2 @@ -30261,7 +30296,7 @@ 1 2 - 21915 + 21917 2 @@ -30358,19 +30393,19 @@ bitfield - 21007 + 20961 id - 21007 + 20961 bits - 2625 + 2620 declared_bits - 2625 + 2620 @@ -30384,7 +30419,7 @@ 1 2 - 21007 + 20961 @@ -30400,7 +30435,7 @@ 1 2 - 21007 + 20961 @@ -30416,42 +30451,42 @@ 1 2 - 735 + 733 2 3 - 630 + 628 3 4 - 210 + 209 4 5 - 210 + 209 5 6 - 210 + 209 6 8 - 210 + 209 8 11 - 210 + 209 12 115 - 210 + 209 @@ -30467,7 +30502,7 @@ 1 2 - 2625 + 2620 @@ -30483,42 +30518,42 @@ 1 2 - 735 + 733 2 3 - 630 + 628 3 4 - 210 + 209 4 5 - 210 + 209 5 6 - 210 + 209 6 8 - 210 + 209 8 11 - 210 + 209 12 115 - 210 + 209 @@ -30534,7 +30569,7 @@ 1 2 - 2625 + 2620 @@ -30544,23 +30579,23 @@ initialisers - 1736804 + 1731850 init - 1736804 + 1731850 var - 719782 + 717751 expr - 1736804 + 1731850 location - 391526 + 390432 @@ -30574,7 +30609,7 @@ 1 2 - 1736804 + 1731850 @@ -30590,7 +30625,7 @@ 1 2 - 1736804 + 1731850 @@ -30606,7 +30641,7 @@ 1 2 - 1736804 + 1731850 @@ -30622,17 +30657,17 @@ 1 2 - 631170 + 629393 2 16 - 31717 + 31625 16 25 - 56851 + 56687 25 @@ -30653,17 +30688,17 @@ 1 2 - 631170 + 629393 2 16 - 31717 + 31625 16 25 - 56851 + 56687 25 @@ -30684,7 +30719,7 @@ 1 2 - 719700 + 717669 2 @@ -30705,7 +30740,7 @@ 1 2 - 1736804 + 1731850 @@ -30721,7 +30756,7 @@ 1 2 - 1736804 + 1731850 @@ -30737,7 +30772,7 @@ 1 2 - 1736804 + 1731850 @@ -30753,22 +30788,22 @@ 1 2 - 318848 + 317950 2 3 - 23904 + 23835 3 15 - 30865 + 30789 15 - 111462 - 17907 + 111459 + 17856 @@ -30784,17 +30819,17 @@ 1 2 - 341914 + 340950 2 4 - 35670 + 35580 4 - 12741 - 13941 + 12738 + 13901 @@ -30810,22 +30845,22 @@ 1 2 - 318848 + 317950 2 3 - 23904 + 23835 3 15 - 30865 + 30789 15 - 111462 - 17907 + 111459 + 17856 @@ -30835,15 +30870,15 @@ expr_ancestor - 121360 + 121674 exp - 121360 + 121674 ancestor - 84661 + 84880 @@ -30857,7 +30892,7 @@ 1 2 - 121360 + 121674 @@ -30873,22 +30908,22 @@ 1 2 - 61219 + 61377 2 3 - 16741 + 16785 3 8 - 6466 + 6483 8 18 - 233 + 234 @@ -30898,19 +30933,19 @@ exprs - 18298854 + 18300140 id - 18298854 + 18300140 kind - 8670 + 3382 location - 7747846 + 3561673 @@ -30924,7 +30959,7 @@ 1 2 - 18298854 + 18300140 @@ -30940,7 +30975,7 @@ 1 2 - 18298854 + 18300140 @@ -30955,68 +30990,68 @@ 1 - 6 - 632 + 5 + 281 - 6 - 12 - 632 + 5 + 14 + 211 - 18 - 28 - 541 + 14 + 38 + 281 - 29 - 45 - 722 + 42 + 66 + 281 - 45 - 88 - 722 + 82 + 135 + 281 - 89 - 122 - 722 + 141 + 334 + 281 - 152 - 234 - 722 + 338 + 509 + 281 - 241 - 441 - 722 + 563 + 830 + 281 - 504 - 778 - 722 + 831 + 1183 + 281 - 802 - 1049 - 722 + 1184 + 2071 + 281 - 1117 - 2286 - 722 + 2627 + 5700 + 281 - 2424 - 9743 - 722 + 6591 + 63491 + 281 - 22722 - 53144 - 361 + 78915 + 109590 + 70 @@ -31031,68 +31066,68 @@ 1 + 2 + 281 + + + 2 3 - 722 + 176 3 - 4 - 541 + 6 + 281 - 4 + 6 13 - 722 + 281 - 13 - 24 - 722 + 14 + 26 + 281 - 24 - 37 - 722 + 28 + 62 + 246 - 41 - 101 - 722 + 63 + 83 + 281 - 102 - 175 - 722 + 91 + 183 + 281 - 180 - 279 - 722 + 206 + 342 + 281 - 281 - 499 - 722 + 353 + 448 + 281 - 565 - 774 - 722 + 468 + 1018 + 281 - 813 - 1643 - 722 + 1051 + 14609 + 281 - 1728 - 13946 - 722 - - - 15037 - 33170 - 180 + 16974 + 32757 + 140 @@ -31108,22 +31143,32 @@ 1 2 - 5161708 + 1936933 2 3 - 1378731 + 816932 3 - 5 - 662723 + 4 + 247260 - 5 - 14777 - 544682 + 4 + 8 + 280872 + + + 8 + 136 + 267131 + + + 136 + 54140 + 12542 @@ -31139,17 +31184,22 @@ 1 2 - 5999730 + 2363808 2 3 - 1427591 + 873691 3 - 26 - 320523 + 6 + 307261 + + + 6 + 25 + 16911 @@ -31159,19 +31209,19 @@ expr_types - 18356354 + 18357789 id - 18298854 + 18300140 typeid - 806338 + 829282 value_category - 53 + 54 @@ -31185,12 +31235,12 @@ 1 2 - 18241425 + 18242562 2 5 - 57428 + 57577 @@ -31206,7 +31256,7 @@ 1 2 - 18298854 + 18300140 @@ -31222,42 +31272,42 @@ 1 2 - 278288 + 293470 2 3 - 160556 + 160720 3 4 - 69787 + 69824 4 5 - 57051 + 60801 5 7 - 62027 + 66996 7 12 - 67236 + 65321 12 35 - 61758 + 62638 35 - 78725 - 49632 + 78674 + 49509 @@ -31273,17 +31323,17 @@ 1 2 - 695684 + 716180 2 3 - 100271 + 102314 3 4 - 10382 + 10787 @@ -31297,19 +31347,19 @@ 12 - 11948 - 11949 - 17 + 11828 + 11829 + 18 - 253514 - 253515 - 17 + 253738 + 253739 + 18 - 753215 - 753216 - 17 + 750551 + 750552 + 18 @@ -31323,19 +31373,19 @@ 12 - 1415 - 1416 - 17 + 1446 + 1447 + 18 - 11760 - 11761 - 17 + 11978 + 11979 + 18 - 38451 - 38452 - 17 + 39501 + 39502 + 18 @@ -31345,15 +31395,15 @@ new_allocated_type - 55240 + 47598 expr - 55240 + 47598 type_id - 27952 + 28150 @@ -31367,7 +31417,7 @@ 1 2 - 55240 + 47598 @@ -31383,22 +31433,17 @@ 1 2 - 9095 + 11767 2 3 - 15987 + 14903 3 - 6 - 2134 - - - 6 - 28 - 734 + 19 + 1479 @@ -31408,15 +31453,15 @@ new_array_allocated_type - 5113 + 5099 expr - 5113 + 5099 type_id - 2200 + 2194 @@ -31430,7 +31475,7 @@ 1 2 - 5113 + 5099 @@ -31451,12 +31496,12 @@ 2 3 - 1948 + 1942 3 7 - 170 + 169 8 @@ -32018,15 +32063,15 @@ condition_decl_bind - 38495 + 38595 expr - 38495 + 38595 decl - 38495 + 38595 @@ -32040,7 +32085,7 @@ 1 2 - 38495 + 38595 @@ -32056,7 +32101,7 @@ 1 2 - 38495 + 38595 @@ -32066,15 +32111,15 @@ typeid_bind - 36173 + 36430 expr - 36173 + 36430 type_id - 16267 + 16383 @@ -32088,7 +32133,7 @@ 1 2 - 36173 + 36430 @@ -32104,12 +32149,12 @@ 1 2 - 15847 + 15960 3 328 - 419 + 422 @@ -32119,15 +32164,15 @@ uuidof_bind - 20051 + 19994 expr - 20051 + 19994 type_id - 19856 + 19799 @@ -32141,7 +32186,7 @@ 1 2 - 20051 + 19994 @@ -32157,7 +32202,7 @@ 1 2 - 19692 + 19635 2 @@ -32172,15 +32217,15 @@ sizeof_bind - 191840 + 191854 expr - 191840 + 191854 type_id - 8193 + 8194 @@ -32194,7 +32239,7 @@ 1 2 - 191840 + 191854 @@ -32303,19 +32348,19 @@ lambdas - 21712 + 21639 expr - 21712 + 21639 default_capture - 472 + 470 has_explicit_return_type - 472 + 470 @@ -32329,7 +32374,7 @@ 1 2 - 21712 + 21639 @@ -32345,7 +32390,7 @@ 1 2 - 21712 + 21639 @@ -32361,7 +32406,7 @@ 46 47 - 472 + 470 @@ -32377,7 +32422,7 @@ 1 2 - 472 + 470 @@ -32393,7 +32438,7 @@ 46 47 - 472 + 470 @@ -32409,7 +32454,7 @@ 1 2 - 472 + 470 @@ -32419,35 +32464,35 @@ lambda_capture - 28320 + 28224 id - 28320 + 28224 lambda - 20768 + 20698 index - 944 + 940 field - 28320 + 28224 captured_by_reference - 472 + 470 is_implicit - 472 + 470 location - 2832 + 2822 @@ -32461,7 +32506,7 @@ 1 2 - 28320 + 28224 @@ -32477,7 +32522,7 @@ 1 2 - 28320 + 28224 @@ -32493,7 +32538,7 @@ 1 2 - 28320 + 28224 @@ -32509,7 +32554,7 @@ 1 2 - 28320 + 28224 @@ -32525,7 +32570,7 @@ 1 2 - 28320 + 28224 @@ -32541,7 +32586,7 @@ 1 2 - 28320 + 28224 @@ -32557,12 +32602,12 @@ 1 2 - 13216 + 13171 2 3 - 7552 + 7526 @@ -32578,12 +32623,12 @@ 1 2 - 13216 + 13171 2 3 - 7552 + 7526 @@ -32599,12 +32644,12 @@ 1 2 - 13216 + 13171 2 3 - 7552 + 7526 @@ -32620,7 +32665,7 @@ 1 2 - 20768 + 20698 @@ -32636,7 +32681,7 @@ 1 2 - 20768 + 20698 @@ -32652,12 +32697,12 @@ 1 2 - 13216 + 13171 2 3 - 7552 + 7526 @@ -32673,12 +32718,12 @@ 16 17 - 472 + 470 44 45 - 472 + 470 @@ -32694,12 +32739,12 @@ 16 17 - 472 + 470 44 45 - 472 + 470 @@ -32715,12 +32760,12 @@ 16 17 - 472 + 470 44 45 - 472 + 470 @@ -32736,7 +32781,7 @@ 1 2 - 944 + 940 @@ -32752,7 +32797,7 @@ 1 2 - 944 + 940 @@ -32768,12 +32813,12 @@ 2 3 - 472 + 470 4 5 - 472 + 470 @@ -32789,7 +32834,7 @@ 1 2 - 28320 + 28224 @@ -32805,7 +32850,7 @@ 1 2 - 28320 + 28224 @@ -32821,7 +32866,7 @@ 1 2 - 28320 + 28224 @@ -32837,7 +32882,7 @@ 1 2 - 28320 + 28224 @@ -32853,7 +32898,7 @@ 1 2 - 28320 + 28224 @@ -32869,7 +32914,7 @@ 1 2 - 28320 + 28224 @@ -32885,7 +32930,7 @@ 60 61 - 472 + 470 @@ -32901,7 +32946,7 @@ 44 45 - 472 + 470 @@ -32917,7 +32962,7 @@ 2 3 - 472 + 470 @@ -32933,7 +32978,7 @@ 60 61 - 472 + 470 @@ -32949,7 +32994,7 @@ 1 2 - 472 + 470 @@ -32965,7 +33010,7 @@ 6 7 - 472 + 470 @@ -32981,7 +33026,7 @@ 60 61 - 472 + 470 @@ -32997,7 +33042,7 @@ 44 45 - 472 + 470 @@ -33013,7 +33058,7 @@ 2 3 - 472 + 470 @@ -33029,7 +33074,7 @@ 60 61 - 472 + 470 @@ -33045,7 +33090,7 @@ 1 2 - 472 + 470 @@ -33061,7 +33106,7 @@ 6 7 - 472 + 470 @@ -33077,12 +33122,12 @@ 8 9 - 1888 + 1881 14 15 - 944 + 940 @@ -33098,12 +33143,12 @@ 8 9 - 1888 + 1881 14 15 - 944 + 940 @@ -33119,7 +33164,7 @@ 1 2 - 2832 + 2822 @@ -33135,12 +33180,12 @@ 8 9 - 1888 + 1881 14 15 - 944 + 940 @@ -33156,7 +33201,7 @@ 1 2 - 2832 + 2822 @@ -33172,7 +33217,7 @@ 1 2 - 2832 + 2822 @@ -33298,19 +33343,19 @@ stmts - 4689495 + 4659955 id - 4689495 + 4659955 kind - 1995 + 1991 location - 2293751 + 2288683 @@ -33324,7 +33369,7 @@ 1 2 - 4689495 + 4659955 @@ -33340,7 +33385,7 @@ 1 2 - 4689495 + 4659955 @@ -33356,97 +33401,97 @@ 1 2 - 105 + 104 18 19 - 105 + 104 22 23 - 105 + 104 46 47 - 105 + 104 75 76 - 105 + 104 83 84 - 105 + 104 102 103 - 105 + 104 154 155 - 105 + 104 242 243 - 105 + 104 284 285 - 105 + 104 383 384 - 105 + 104 418 419 - 105 + 104 501 502 - 105 + 104 1324 1325 - 105 + 104 2629 2630 - 105 + 104 - 4606 - 4607 - 105 + 4609 + 4610 + 104 - 8935 - 8936 - 105 + 8749 + 8750 + 104 11547 11548 - 105 + 104 13275 13276 - 105 + 104 @@ -33462,97 +33507,97 @@ 1 2 - 105 + 104 8 9 - 105 + 104 18 19 - 105 + 104 45 46 - 105 + 104 50 51 - 105 + 104 56 57 - 105 + 104 74 75 - 105 + 104 88 89 - 105 + 104 101 102 - 105 + 104 128 129 - 105 + 104 209 210 - 105 + 104 252 253 - 105 + 104 368 369 - 105 + 104 641 642 - 105 + 104 1742 1743 - 105 + 104 - 2183 - 2184 - 105 + 2186 + 2187 + 104 - 4210 - 4211 - 105 + 4191 + 4192 + 104 6065 6066 - 105 + 104 6529 6530 - 105 + 104 @@ -33568,22 +33613,22 @@ 1 2 - 1897016 + 1893663 2 4 - 177201 + 176600 4 12 - 176466 + 175762 12 684 - 43066 + 42656 @@ -33599,12 +33644,12 @@ 1 2 - 2234613 + 2231353 2 8 - 59137 + 57329 @@ -33709,16 +33754,64 @@ - if_then - 723123 + if_initialization + 314 if_stmt - 723123 + 314 + + + init_id + 314 + + + + + if_stmt + init_id + + + 12 + + + 1 + 2 + 314 + + + + + + + init_id + if_stmt + + + 12 + + + 1 + 2 + 314 + + + + + + + + + if_then + 723174 + + + if_stmt + 723174 then_id - 723123 + 723174 @@ -33732,7 +33825,7 @@ 1 2 - 723123 + 723174 @@ -33748,7 +33841,7 @@ 1 2 - 723123 + 723174 @@ -33758,15 +33851,15 @@ if_else - 183959 + 183972 if_stmt - 183959 + 183972 else_id - 183959 + 183972 @@ -33780,7 +33873,7 @@ 1 2 - 183959 + 183972 @@ -33796,7 +33889,55 @@ 1 2 - 183959 + 183972 + + + + + + + + + constexpr_if_initialization + 1 + + + constexpr_if_stmt + 1 + + + init_id + 1 + + + + + constexpr_if_stmt + init_id + + + 12 + + + 1 + 2 + 1 + + + + + + + init_id + constexpr_if_stmt + + + 12 + + + 1 + 2 + 1 @@ -33806,15 +33947,15 @@ constexpr_if_then - 52624 + 52508 constexpr_if_stmt - 52624 + 52508 then_id - 52624 + 52508 @@ -33828,7 +33969,7 @@ 1 2 - 52624 + 52508 @@ -33844,7 +33985,7 @@ 1 2 - 52624 + 52508 @@ -33854,15 +33995,15 @@ constexpr_if_else - 30986 + 30918 constexpr_if_stmt - 30986 + 30918 else_id - 30986 + 30918 @@ -33876,7 +34017,7 @@ 1 2 - 30986 + 30918 @@ -33892,7 +34033,7 @@ 1 2 - 30986 + 30918 @@ -33902,15 +34043,15 @@ while_body - 30265 + 30207 while_stmt - 30265 + 30207 body_id - 30265 + 30207 @@ -33924,7 +34065,7 @@ 1 2 - 30265 + 30207 @@ -33940,7 +34081,7 @@ 1 2 - 30265 + 30207 @@ -33950,15 +34091,15 @@ do_body - 148593 + 148604 do_stmt - 148593 + 148604 body_id - 148593 + 148604 @@ -33972,7 +34113,7 @@ 1 2 - 148593 + 148604 @@ -33988,7 +34129,55 @@ 1 2 - 148593 + 148604 + + + + + + + + + switch_initialization + 4 + + + switch_stmt + 4 + + + init_id + 4 + + + + + switch_stmt + init_id + + + 12 + + + 1 + 2 + 4 + + + + + + + init_id + switch_stmt + + + 12 + + + 1 + 2 + 4 @@ -33998,19 +34187,19 @@ switch_case - 190914 + 191408 switch_stmt - 10275 + 10301 index - 4418 + 4430 case_id - 190914 + 191408 @@ -34024,57 +34213,57 @@ 2 3 - 53 + 54 3 4 - 2263 + 2269 4 5 - 1652 + 1656 5 6 - 1005 + 1008 6 7 - 754 + 756 7 9 - 718 + 720 9 10 - 970 + 972 10 11 - 323 + 324 11 14 - 862 + 864 14 31 - 826 + 828 36 247 - 844 + 846 @@ -34090,57 +34279,57 @@ 2 3 - 53 + 54 3 4 - 2263 + 2269 4 5 - 1652 + 1656 5 6 - 1005 + 1008 6 7 - 754 + 756 7 9 - 718 + 720 9 10 - 970 + 972 10 11 - 323 + 324 11 14 - 862 + 864 14 31 - 826 + 828 36 247 - 844 + 846 @@ -34156,32 +34345,32 @@ 14 15 - 1167 + 1170 18 19 - 538 + 540 32 33 - 1904 + 1909 33 62 - 377 + 378 66 296 - 341 + 342 351 573 - 89 + 90 @@ -34197,32 +34386,32 @@ 14 15 - 1167 + 1170 18 19 - 538 + 540 32 33 - 1904 + 1909 33 62 - 377 + 378 66 296 - 341 + 342 351 573 - 89 + 90 @@ -34238,7 +34427,7 @@ 1 2 - 190914 + 191408 @@ -34254,7 +34443,7 @@ 1 2 - 190914 + 191408 @@ -34264,15 +34453,15 @@ switch_body - 20900 + 20901 switch_stmt - 20900 + 20901 body_id - 20900 + 20901 @@ -34286,7 +34475,7 @@ 1 2 - 20900 + 20901 @@ -34302,7 +34491,7 @@ 1 2 - 20900 + 20901 @@ -34312,15 +34501,15 @@ for_initialization - 53198 + 53202 for_stmt - 53198 + 53202 init_id - 53198 + 53202 @@ -34334,7 +34523,7 @@ 1 2 - 53198 + 53202 @@ -34350,7 +34539,7 @@ 1 2 - 53198 + 53202 @@ -34360,15 +34549,15 @@ for_condition - 55454 + 55458 for_stmt - 55454 + 55458 condition_id - 55454 + 55458 @@ -34382,7 +34571,7 @@ 1 2 - 55454 + 55458 @@ -34398,7 +34587,7 @@ 1 2 - 55454 + 55458 @@ -34408,15 +34597,15 @@ for_update - 53301 + 53304 for_stmt - 53301 + 53304 update_id - 53301 + 53304 @@ -34430,7 +34619,7 @@ 1 2 - 53301 + 53304 @@ -34446,7 +34635,7 @@ 1 2 - 53301 + 53304 @@ -34456,15 +34645,15 @@ for_body - 61319 + 61324 for_stmt - 61319 + 61324 body_id - 61319 + 61324 @@ -34478,7 +34667,7 @@ 1 2 - 61319 + 61324 @@ -34494,7 +34683,7 @@ 1 2 - 61319 + 61324 @@ -34504,19 +34693,19 @@ stmtparents - 4064694 + 4052307 id - 4064694 + 4052307 index - 12245 + 12210 parent - 1724432 + 1719451 @@ -34530,7 +34719,7 @@ 1 2 - 4064694 + 4052307 @@ -34546,7 +34735,7 @@ 1 2 - 4064694 + 4052307 @@ -34562,12 +34751,12 @@ 1 2 - 4022 + 4011 2 3 - 1002 + 999 3 @@ -34577,37 +34766,37 @@ 4 5 - 1557 + 1553 7 8 - 1021 + 1018 8 12 - 794 + 792 12 29 - 1078 + 1075 29 38 - 920 + 917 41 77 - 926 + 924 77 - 196938 - 699 + 196934 + 697 @@ -34623,12 +34812,12 @@ 1 2 - 4022 + 4011 2 3 - 1002 + 999 3 @@ -34638,37 +34827,37 @@ 4 5 - 1557 + 1553 7 8 - 1021 + 1018 8 12 - 794 + 792 12 29 - 1078 + 1075 29 38 - 920 + 917 41 77 - 926 + 924 77 - 196938 - 699 + 196934 + 697 @@ -34684,32 +34873,32 @@ 1 2 - 989567 + 987314 2 3 - 374640 + 372959 3 4 - 106047 + 105742 4 6 - 111564 + 111231 6 17 - 130216 + 129842 17 1943 - 12396 + 12361 @@ -34725,32 +34914,32 @@ 1 2 - 989567 + 987314 2 3 - 374640 + 372959 3 4 - 106047 + 105742 4 6 - 111564 + 111231 6 17 - 130216 + 129842 17 1943 - 12396 + 12361 @@ -34760,22 +34949,22 @@ ishandler - 59279 + 59432 block - 59279 + 59432 stmt_decl_bind - 585681 + 585632 stmt - 545527 + 545482 num @@ -34783,7 +34972,7 @@ decl - 585576 + 585527 @@ -34797,12 +34986,12 @@ 1 2 - 524646 + 524602 2 19 - 20881 + 20879 @@ -34818,12 +35007,12 @@ 1 2 - 524646 + 524602 2 19 - 20881 + 20879 @@ -35021,7 +35210,7 @@ 1 2 - 585538 + 585489 2 @@ -35042,7 +35231,7 @@ 1 2 - 585576 + 585527 @@ -35052,11 +35241,11 @@ stmt_decl_entry_bind - 528084 + 528040 stmt - 488233 + 488193 num @@ -35064,7 +35253,7 @@ decl_entry - 528025 + 527981 @@ -35078,12 +35267,12 @@ 1 2 - 467616 + 467578 2 19 - 20616 + 20615 @@ -35099,12 +35288,12 @@ 1 2 - 467616 + 467578 2 19 - 20616 + 20615 @@ -35302,7 +35491,7 @@ 1 2 - 528004 + 527960 3 @@ -35323,7 +35512,7 @@ 1 2 - 528025 + 527981 @@ -35333,15 +35522,15 @@ blockscope - 1442932 + 1438063 block - 1442932 + 1438063 enclosing - 1326345 + 1321870 @@ -35355,7 +35544,7 @@ 1 2 - 1442932 + 1438063 @@ -35371,12 +35560,12 @@ 1 2 - 1260264 + 1256011 2 13 - 66081 + 65858 @@ -35386,19 +35575,19 @@ jumpinfo - 253977 + 253995 id - 253977 + 253995 str - 21151 + 21152 target - 53042 + 53046 @@ -35412,7 +35601,7 @@ 1 2 - 253977 + 253995 @@ -35428,7 +35617,7 @@ 1 2 - 253977 + 253995 @@ -35444,7 +35633,7 @@ 2 3 - 9874 + 9875 3 @@ -35490,7 +35679,7 @@ 1 2 - 16716 + 16717 2 @@ -35526,12 +35715,12 @@ 2 3 - 26426 + 26428 3 4 - 12896 + 12897 4 @@ -35541,7 +35730,7 @@ 5 8 - 4690 + 4691 8 @@ -35562,7 +35751,7 @@ 1 2 - 53042 + 53046 @@ -35572,19 +35761,19 @@ preprocdirects - 4447274 + 4437448 id - 4447274 + 4437448 kind - 1050 + 1048 location - 4444753 + 4434933 @@ -35598,7 +35787,7 @@ 1 2 - 4447274 + 4437448 @@ -35614,7 +35803,7 @@ 1 2 - 4447274 + 4437448 @@ -35630,52 +35819,52 @@ 121 122 - 105 + 104 693 694 - 105 + 104 794 795 - 105 + 104 917 918 - 105 + 104 1697 1698 - 105 + 104 1785 1786 - 105 + 104 2983 2984 - 105 + 104 3799 3800 - 105 + 104 6290 6291 - 105 + 104 23260 23261 - 105 + 104 @@ -35691,52 +35880,52 @@ 121 122 - 105 + 104 693 694 - 105 + 104 794 795 - 105 + 104 917 918 - 105 + 104 1697 1698 - 105 + 104 1785 1786 - 105 + 104 2983 2984 - 105 + 104 3799 3800 - 105 + 104 6290 6291 - 105 + 104 23236 23237 - 105 + 104 @@ -35752,12 +35941,12 @@ 1 2 - 4444648 + 4434828 25 26 - 105 + 104 @@ -35773,7 +35962,7 @@ 1 2 - 4444753 + 4434933 @@ -35783,15 +35972,15 @@ preprocpair - 1443876 + 1442296 begin - 1206927 + 1206147 elseelifend - 1443876 + 1442296 @@ -35805,17 +35994,17 @@ 1 2 - 986027 + 985992 2 3 - 210516 + 209805 3 11 - 10384 + 10349 @@ -35831,7 +36020,7 @@ 1 2 - 1443876 + 1442296 @@ -35841,41 +36030,41 @@ preproctrue - 784479 + 782302 branch - 784479 + 782302 preprocfalse - 325686 + 327409 branch - 325686 + 327409 preproctext - 3585423 + 3577502 id - 3585423 + 3577502 head - 2599941 + 2594197 body - 1520449 + 1517194 @@ -35889,7 +36078,7 @@ 1 2 - 3585423 + 3577502 @@ -35905,7 +36094,7 @@ 1 2 - 3585423 + 3577502 @@ -35921,12 +36110,12 @@ 1 2 - 2452360 + 2446942 2 740 - 147580 + 147254 @@ -35942,12 +36131,12 @@ 1 2 - 2537548 + 2531941 2 5 - 62393 + 62255 @@ -35963,17 +36152,17 @@ 1 2 - 1376334 + 1373398 2 6 - 114073 + 113821 6 - 11582 - 30041 + 11581 + 29974 @@ -35989,17 +36178,17 @@ 1 2 - 1379380 + 1376438 2 7 - 114388 + 114135 7 - 2959 - 26680 + 2958 + 26621 @@ -36009,15 +36198,15 @@ includes - 316718 + 315649 id - 316718 + 315649 included - 118474 + 118074 @@ -36031,7 +36220,7 @@ 1 2 - 316718 + 315649 @@ -36047,32 +36236,32 @@ 1 2 - 61833 + 61624 2 3 - 22184 + 22109 3 4 - 12744 + 12701 4 6 - 10384 + 10349 6 14 - 8968 + 8937 14 47 - 2360 + 2352 @@ -36130,15 +36319,15 @@ link_parent - 40018704 + 40143068 element - 5094359 + 5115983 link_target - 349 + 352 @@ -36152,17 +36341,17 @@ 1 2 - 694015 + 701934 2 9 - 43135 + 44146 9 10 - 4357208 + 4369903 @@ -36178,52 +36367,52 @@ 3 4 - 34 + 35 - 124724 - 124725 - 34 + 124207 + 124208 + 35 - 124828 - 124829 - 34 + 124311 + 124312 + 35 - 124926 - 124927 - 34 + 124409 + 124410 + 35 - 124964 - 124965 - 34 + 124447 + 124448 + 35 - 124978 - 124979 - 34 + 124454 + 124455 + 35 - 124980 - 124981 - 34 + 124463 + 124464 + 35 - 126863 - 126864 - 34 + 126339 + 126340 + 35 - 132946 - 132947 - 34 + 132460 + 132461 + 35 - 134697 - 134698 - 34 + 134288 + 134289 + 35 From e23e5e5b12b0698d2500e004bb36c1ccfb790302 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Thu, 12 May 2022 08:28:34 +0200 Subject: [PATCH 87/91] C++: Add change notes for C++17 `if` and `switch` initializers --- .../lib/change-notes/2022-04-12-if-and-switch-initializers.md | 4 ++++ cpp/ql/src/change-notes/2022-04-12-unused-local-variable.md | 4 ++++ 2 files changed, 8 insertions(+) create mode 100644 cpp/ql/lib/change-notes/2022-04-12-if-and-switch-initializers.md create mode 100644 cpp/ql/src/change-notes/2022-04-12-unused-local-variable.md diff --git a/cpp/ql/lib/change-notes/2022-04-12-if-and-switch-initializers.md b/cpp/ql/lib/change-notes/2022-04-12-if-and-switch-initializers.md new file mode 100644 index 00000000000..dcfa69120fa --- /dev/null +++ b/cpp/ql/lib/change-notes/2022-04-12-if-and-switch-initializers.md @@ -0,0 +1,4 @@ +--- +category: feature +--- +* A `getInitialization` predicate was added to the `ConstexprIfStmt`, `IfStmt`, and `SwitchStmt` classes that yields the C++17-style initializer of the `if` or `switch` statement when it exists. diff --git a/cpp/ql/src/change-notes/2022-04-12-unused-local-variable.md b/cpp/ql/src/change-notes/2022-04-12-unused-local-variable.md new file mode 100644 index 00000000000..d4120401e1a --- /dev/null +++ b/cpp/ql/src/change-notes/2022-04-12-unused-local-variable.md @@ -0,0 +1,4 @@ +--- +category: minorAnalysis +--- +* The `cpp/unused-local-variable` no longer ignores functions that include `if` and `switch` statements with C++17-style initializers. From fef4455ccce370b8c16a990daffa41cf253b29d3 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Thu, 12 May 2022 13:28:45 +0200 Subject: [PATCH 88/91] apply suggestion from doc review Co-authored-by: Steve Guntrip <12534592+stevecat@users.noreply.github.com> --- .../ql/src/Security/CWE-094/ExpressionInjection.qhelp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/javascript/ql/src/Security/CWE-094/ExpressionInjection.qhelp b/javascript/ql/src/Security/CWE-094/ExpressionInjection.qhelp index 8068697d878..f355ef0aa6e 100644 --- a/javascript/ql/src/Security/CWE-094/ExpressionInjection.qhelp +++ b/javascript/ql/src/Security/CWE-094/ExpressionInjection.qhelp @@ -8,10 +8,10 @@ code injection in contexts like run: or script:.

    - Code injection in GitHub actions may allow an attacker to + Code injection in GitHub Actions may allow an attacker to exfiltrate the temporary GitHub repository authorization token. - The token might have write access to the repository, and thus an attacker - might be able to use it to modify the repository. + The token might have write access to the repository, allowing an attacker + to use the token to make changes to the repository.

    From 12b34bcf04318698b5c2574431ed2f85a6110abe Mon Sep 17 00:00:00 2001 From: Taus Date: Thu, 12 May 2022 12:17:04 +0000 Subject: [PATCH 89/91] Devcontainer: Install test dependencies These _should_ get installed automatically if missing, by in my experience this can be a bit flaky. Installing theme here should make this a bit more robust. --- .devcontainer/devcontainer.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index ff73bcb4e7b..7fd96b8d941 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -3,6 +3,8 @@ "rust-lang.rust", "bungcip.better-toml", "github.vscode-codeql", + "hbenl.vscode-test-explorer", + "ms-vscode.test-adapter-converter", "slevesque.vscode-zipexplorer" ], "settings": { From 72823e9576c8991be5adf18bc46b9c58473c6e52 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema <93738568+jketema@users.noreply.github.com> Date: Thu, 12 May 2022 14:54:43 +0200 Subject: [PATCH 90/91] Apply suggestions from code review Co-authored-by: Mathias Vorreiter Pedersen --- cpp/ql/lib/semmle/code/cpp/stmts/Stmt.qll | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp/ql/lib/semmle/code/cpp/stmts/Stmt.qll b/cpp/ql/lib/semmle/code/cpp/stmts/Stmt.qll index 9dda08ba206..7a79d27c1ba 100644 --- a/cpp/ql/lib/semmle/code/cpp/stmts/Stmt.qll +++ b/cpp/ql/lib/semmle/code/cpp/stmts/Stmt.qll @@ -214,7 +214,7 @@ class IfStmt extends ConditionalStmt, @stmt_if { override string getAPrimaryQlClass() { result = "IfStmt" } /** - * Gets the initialization statement of this 'if' statement. + * Gets the initialization statement of this 'if' statement, if any. * * For example, for * ``` @@ -320,7 +320,7 @@ class ConstexprIfStmt extends ConditionalStmt, @stmt_constexpr_if { override string getAPrimaryQlClass() { result = "ConstexprIfStmt" } /** - * Gets the initialization statement of this 'constexpr if' statement. + * Gets the initialization statement of this 'constexpr if' statement, if any. * * For example, for * ``` @@ -1513,7 +1513,7 @@ class SwitchStmt extends ConditionalStmt, @stmt_switch { override string getAPrimaryQlClass() { result = "SwitchStmt" } /** - * Gets the initialization statement of this 'switch' statement. + * Gets the initialization statement of this 'switch' statement, if any. * * For example, for * ``` From 723f3b09fe005e67f3aea61ee79d102aa8c75023 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Thu, 12 May 2022 15:09:06 +0200 Subject: [PATCH 91/91] C++: Address review comments --- cpp/ql/lib/semmle/code/cpp/stmts/Stmt.qll | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cpp/ql/lib/semmle/code/cpp/stmts/Stmt.qll b/cpp/ql/lib/semmle/code/cpp/stmts/Stmt.qll index 7a79d27c1ba..7ccd9f797dd 100644 --- a/cpp/ql/lib/semmle/code/cpp/stmts/Stmt.qll +++ b/cpp/ql/lib/semmle/code/cpp/stmts/Stmt.qll @@ -218,9 +218,9 @@ class IfStmt extends ConditionalStmt, @stmt_if { * * For example, for * ``` - * if (x = y; b) { f(); } + * if (int x = y; b) { f(); } * ``` - * the result is `x = y;`. + * the result is `int x = y;`. * * Does not hold if the initialization statement is missing or an empty statement, as in * ``` @@ -324,9 +324,9 @@ class ConstexprIfStmt extends ConditionalStmt, @stmt_constexpr_if { * * For example, for * ``` - * if constexpr (x = y; b) { f(); } + * if constexpr (int x = y; b) { f(); } * ``` - * the result is `x = y;`. + * the result is `int x = y;`. * * Does not hold if the initialization statement is missing or an empty statement, as in * ``` @@ -1517,9 +1517,9 @@ class SwitchStmt extends ConditionalStmt, @stmt_switch { * * For example, for * ``` - * switch (x = y; b) { } + * switch (int x = y; b) { } * ``` - * the result is `x = y;`. + * the result is `int x = y;`. * * Does not hold if the initialization statement is missing or an empty statement, as in * ```