From 0051ba1596e6b745be0214896faf062eb38f9403 Mon Sep 17 00:00:00 2001 From: Taus Date: Mon, 17 Oct 2022 13:10:56 +0000 Subject: [PATCH 001/106] Python: Add new module resolution implementation A fairly complicated bit of modelling, mostly due to the quirks of how imports are handled in Python. A few notes: - The handling of `__all__` is not actually needed (and perhaps not desirable, as it only pertains to `import *`, though it does match the current behaviour), but it might become useful at a later date, so I left it in. - Ideally, we would represent `foo as bar` in an `import` as a `DefinitionNode` in the CFG. I opted _not_ to do this, as it would also affect points-to, and I did not want to deal with any fallout arising from that. --- .../new/internal/ImportResolution.qll | 262 ++++++++++++++++++ .../dataflow/new/internal/ImportStar.qll | 2 +- 2 files changed, 263 insertions(+), 1 deletion(-) diff --git a/python/ql/lib/semmle/python/dataflow/new/internal/ImportResolution.qll b/python/ql/lib/semmle/python/dataflow/new/internal/ImportResolution.qll index 0c346fa2dd4..906460d76c1 100644 --- a/python/ql/lib/semmle/python/dataflow/new/internal/ImportResolution.qll +++ b/python/ql/lib/semmle/python/dataflow/new/internal/ImportResolution.qll @@ -1,14 +1,72 @@ +/** + * INTERNAL. DO NOT USE. + * + * Provides predicates for resolving imports. + */ + private import python private import semmle.python.dataflow.new.DataFlow private import semmle.python.dataflow.new.internal.ImportStar private import semmle.python.dataflow.new.TypeTracker +/** + * Python modules and the way imports are resolved are... complicated. Here's a crash course in how + * it works, as well as some caveats to bear in mind when looking at the implementation in this + * module. + * + * First, let's consider the humble `import` statement: + * ```python + * import foo + * import bar.baz + * import ham.eggs as spam + * ``` + * + * In the AST, all imports are aliased, as in the last import above. That is, `import foo` becomes + * `import foo as foo`, and `import bar.baz` becomes `import bar as bar`. Note that `import` is + * exclusively used to import modules -- if `eggs` is an attribute of the `ham` module (and not a + * submodule of the `ham` package), then the third line above is an error. + * + * Next, we have the `from` statement. This one is a bit more complicated, but still has the same + * aliasing desugaring as above applied to it. Thus, `from foo import bar` becomes + * `from foo import bar as bar`. + * + * In general, `from foo import bar` can mean two different things: + * + * 1. If `foo` is a module, and `bar` is an attribute of `foo`, then `from foo import bar` imports + * the attribute `bar` into the current module (binding it to the name `bar`). + * 2. If `foo` is a package, and `bar` is a submodule of `foo`, then `from foo import bar` first imports + * `foo.bar`, and then attempts to locate the `bar` attribute again. In most cases, that attribute + * will then point to the `bar` submodule. + * + * Now, when in comes to how these imports are represented in the AST, things get a bit complicated. + * First of all, both of the above forms of imports get mapped to the same kind of AST node: + * `Import`. An `Import` node has a sequence of names, each of which is an `Alias` node. This `Alias` + * node represents the `x as y` bit of each imported module. + * + * The same is true for `from` imports. So, how then do we distinguish between the two forms of + * imports? The distinguishing feature is the left hand side of the `as` node. If the left hand side + * is an `ImportExpr`, then it is a plain import. If it is an `ImportMember`, then it is a `from` + * import. (And to confuse matters even more, this `ImportMember` contains another `ImportExpr` for + * the bit between the `from` and `import` keywords.) + * + * Caveats: + * + * - A relative import of the form `from .foo import bar as baz` not only imports `bar` and binds it + * to the name `baz`, but also imports `foo` and binds it to the name `foo`. This only happens with + * relative imports. `from foo import bar as baz` only binds `bar` to `baz`. + * - Modules may also be packages, so e.g. `import foo.bar` may import the `bar` submodule in the `foo` + * package, or the `bar` subpackage of the `foo` package. The practical difference here is the name of + * the module that is imported, as the package `foo.bar` will have the "name" `foo.bar.__init__`, + * corresponding to the fact that the code that is executed is in the `__init__.py` file of the + * `bar` package. + */ module ImportResolution { /** * Holds if the module `m` defines a name `name` by assigning `defn` to it. This is an * overapproximation, as `name` may not in fact be exported (e.g. by defining an `__all__` that does * not include `name`). */ + pragma[nomagic] predicate module_export(Module m, string name, DataFlow::CfgNode defn) { exists(EssaVariable v | v.getName() = name and @@ -18,12 +76,216 @@ module ImportResolution { or defn.getNode() = v.getDefinition().(ArgumentRefinement).getArgument() ) + or + exists(Alias a | + defn.asExpr() = [a.getValue(), a.getValue().(ImportMember).getModule()] and + a.getAsname().(Name).getId() = name and + defn.getScope() = m + ) + } + + /** + * Holds if the module `m` explicitly exports the name `name` by listing it in `__all__`. Only + * handles simple cases where we can statically tell that this is the case. + */ + private predicate all_mentions_name(Module m, string name) { + exists(DefinitionNode def, SequenceNode n | + def.getValue() = n and + def.(NameNode).getId() = "__all__" and + def.getScope() = m and + any(StrConst s | s.getText() = name) = n.getAnElement().getNode() + ) + } + + /** + * Holds if the module `m` either does not set `__all__` (and so implicitly exports anything that + * doesn't start with an underscore), or sets `__all__` in a way that's too complicated for us to + * handle (in which case we _also_ pretend that it just exports all such names). + */ + private predicate no_or_complicated_all(Module m) { + // No mention of `__all__` in the module + not exists(DefinitionNode def | def.getScope() = m and def.(NameNode).getId() = "__all__") + or + // `__all__` is set to a non-sequence value + exists(DefinitionNode def | + def.(NameNode).getId() = "__all__" and + def.getScope() = m and + not def.getValue() instanceof SequenceNode + ) + or + // `__all__` is used in some way that doesn't involve storing a value in it. This usually means + // it is being mutated through `append` or `extend`, which we don't handle. + exists(NameNode n | n.getId() = "__all__" and n.getScope() = m and n.isLoad()) + } + + private predicate potential_module_export(Module m, string name) { + all_mentions_name(m, name) + or + no_or_complicated_all(m) and + ( + exists(NameNode n | n.getId() = name and n.getScope() = m and name.charAt(0) != "_") + or + exists(Alias a | a.getAsname().(Name).getId() = name and a.getValue().getScope() = m) + ) + } + + /** + * Holds if the module `reexporter` exports the module `reexported` under the name + * `reexported_name`. + */ + private predicate module_reexport(Module reexporter, string reexported_name, Module reexported) { + exists(DataFlow::Node ref | + ref = getImmediateModuleReference(reexported) and + module_export(reexporter, reexported_name, ref) and + potential_module_export(reexporter, reexported_name) + ) + } + + /** + * Gets a reference to `sys.modules`. + */ + private DataFlow::Node sys_modules_reference() { + result = + any(DataFlow::AttrRef a | + a.getAttributeName() = "modules" and a.getObject().asExpr().(Name).getId() = "sys" + ) + } + + /** Gets a module that may have been added to `sys.modules`. */ + private Module sys_modules_module_with_name(string name) { + exists(ControlFlowNode n, DataFlow::Node mod | + exists(SubscriptNode sub | + sub.getObject() = sys_modules_reference().asCfgNode() and + sub.getIndex() = n and + n.getNode().(StrConst).getText() = name and + sub.(DefinitionNode).getValue() = mod.asCfgNode() and + mod = getModuleReference(result) + ) + ) } Module getModule(DataFlow::CfgNode node) { exists(ModuleValue mv | node.getNode().pointsTo(mv) and result = mv.getScope() + Module getModuleImportedByImportStar(ImportStar i) { + isPreferredModuleForName(result.getFile(), i.getImportedModuleName()) + } + + /** Gets a data-flow node that may be a reference to a module with the name `module_name`. */ + DataFlow::Node getReferenceToModuleName(string module_name) { + // Regular import statements, e.g. + // import foo # implicitly `import foo as foo` + // import foo as foo_alias + exists(Import i, Alias a | a = i.getAName() | + result.asExpr() = a.getAsname() and + module_name = a.getValue().(ImportExpr).getImportedModuleName() + ) + or + // The module part of a `from ... import ...` statement, e.g. the `..foo.bar` in + // from ..foo.bar import baz # ..foo.bar might point to, say, package.subpackage.foo.bar + exists(ImportMember i | result.asExpr() = i.getModule() | + module_name = i.getModule().(ImportExpr).getImportedModuleName() + ) + or + // Modules (not attributes) imported via `from ... import ... statements`, e.g. + // from foo.bar import baz # imports foo.bar.baz as baz + // from foo.bar import baz as baz_alias # imports foo.bar.baz as baz_alias + exists(Import i, Alias a, ImportMember im | a = i.getAName() and im = a.getValue() | + i.isFromImport() and + result.asExpr() = a.getAsname() and + module_name = im.getModule().(ImportExpr).getImportedModuleName() + "." + im.getName() + ) + or + // For parity with the points-to based solution, the `ImportExpr` and `ImportMember` bits of the + // above cases should _also_ point to the right modules. + result.asExpr() = any(ImportExpr i | i.getImportedModuleName() = module_name) + or + result.asExpr() = + any(ImportMember i | + i.getModule().(ImportExpr).getImportedModuleName() = module_name + or + i.getModule().(ImportExpr).getImportedModuleName() + "." + i.getName() = module_name and + none() + ) + } + + /** Gets a dataflow node that is an immediate reference to the module `m`. */ + DataFlow::Node getImmediateModuleReference(Module m) { + exists(string module_name | result = getReferenceToModuleName(module_name) | + // Depending on whether the referenced module is a package or not, we may need to add a + // trailing `.__init__` to the module name. + isPreferredModuleForName(m.getFile(), module_name + ["", ".__init__"]) + or + // Module defined via `sys.modules` + m = sys_modules_module_with_name(module_name) + ) + or + // Reading an attribute on a module may return a submodule (or subpackage). + exists(DataFlow::AttrRead ar, Module p, string attr_name | + ar.getObject() = getModuleReference(p) and + attr_name = any(Module m0).getFile().getStem() and + ar.getAttributeName() = attr_name and + result = ar + | + isPreferredModuleForName(m.getFile(), p.getPackageName() + "." + attr_name + ["", ".__init__"]) + or + // This is also true for attributes that come from reexports. + module_reexport(p, attr_name, m) + ) + or + // Submodules that are implicitly defined when importing via `from ... import ...` statements. + // In practice, we create a definition for each module in a package, even if it is not imported. + exists(string submodule, Module package | + SsaSource::init_module_submodule_defn(result.asVar().getSourceVariable(), + package.getEntryNode()) and + isPreferredModuleForName(m.getFile(), + package.getPackageName() + "." + submodule + ["", ".__init__"]) ) } + + /** Join-order helper for `getModuleReference`. */ + pragma[nomagic] + private predicate module_name_in_scope(DataFlow::Node node, Scope s, string name, Module m) { + node.getScope() = s and + node.asExpr().(Name).getId() = name and + pragma[only_bind_into](node) = getImmediateModuleReference(pragma[only_bind_into](m)) + } + + /** Join-order helper for `getModuleReference`. */ + pragma[nomagic] + private predicate module_reference_in_scope(DataFlow::Node node, Scope s, string name) { + node.getScope() = s and + exists(Name n | n = node.asExpr() | + n.getId() = name and + pragma[only_bind_into](n).isUse() + ) + } + + /** + * Gets a reference to the module `m` (including through certain kinds of local and global flow). + */ + DataFlow::Node getModuleReference(Module m) { + // Immedate references to the module + result = getImmediateModuleReference(m) + or + // Flow (local or global) forward to a later reference to the module. + exists(DataFlow::Node ref | ref = getModuleReference(m) | + DataFlow::localFlow(ref, result) + or + exists(DataFlow::ModuleVariableNode mv | + mv.getAWrite() = ref and + result = mv.getARead() + ) + ) + or + // A reference to a name that is bound to a module in an enclosing scope. + exists(DataFlow::Node def, Scope def_scope, Scope use_scope, string name | + module_name_in_scope(pragma[only_bind_into](def), pragma[only_bind_into](def_scope), + pragma[only_bind_into](name), pragma[only_bind_into](m)) and + module_reference_in_scope(result, use_scope, name) and + use_scope.getEnclosingScope*() = def_scope + ) + } + } diff --git a/python/ql/lib/semmle/python/dataflow/new/internal/ImportStar.qll b/python/ql/lib/semmle/python/dataflow/new/internal/ImportStar.qll index ae115342dba..564630c47db 100644 --- a/python/ql/lib/semmle/python/dataflow/new/internal/ImportStar.qll +++ b/python/ql/lib/semmle/python/dataflow/new/internal/ImportStar.qll @@ -76,7 +76,7 @@ module ImportStar { exists(ImportStar i, DataFlow::CfgNode imported_module | imported_module.getNode().getNode() = i.getModule() and i.getScope() = m and - result = ImportResolution::getModule(imported_module) + result = ImportResolution::getModuleImportedByImportStar(i) ) } From 651afaf11b9ee9798a09f670dba1f1b97008d124 Mon Sep 17 00:00:00 2001 From: Taus Date: Mon, 17 Oct 2022 13:15:07 +0000 Subject: [PATCH 002/106] Python: Hook up new implementation Left as its own commit, as otherwise the diff would have been very confusing. --- .../semmle/python/dataflow/new/internal/ImportResolution.qll | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/python/ql/lib/semmle/python/dataflow/new/internal/ImportResolution.qll b/python/ql/lib/semmle/python/dataflow/new/internal/ImportResolution.qll index 906460d76c1..e1e4381519c 100644 --- a/python/ql/lib/semmle/python/dataflow/new/internal/ImportResolution.qll +++ b/python/ql/lib/semmle/python/dataflow/new/internal/ImportResolution.qll @@ -164,10 +164,6 @@ module ImportResolution { ) } - Module getModule(DataFlow::CfgNode node) { - exists(ModuleValue mv | - node.getNode().pointsTo(mv) and - result = mv.getScope() Module getModuleImportedByImportStar(ImportStar i) { isPreferredModuleForName(result.getFile(), i.getImportedModuleName()) } @@ -288,4 +284,5 @@ module ImportResolution { ) } + Module getModule(DataFlow::CfgNode node) { node = getModuleReference(result) } } From ad13fbaeb6b98d169dcfeac6ff5d95230a5b541d Mon Sep 17 00:00:00 2001 From: Taus Date: Mon, 17 Oct 2022 13:42:24 +0000 Subject: [PATCH 003/106] Python: Add tests A slightly complicated test setup. I wanted to both make sure I captured the semantics of Python and also the fact that the kinds of global flow we expect to see are indeed present. The code is executable, and prints out both when the execution reaches certain files, and also what values are assigned to the various attributes that are referenced throughout the program. These values are validated in the test as well. My original version used introspection to avoid referencing attributes directly (thus enabling better error diagnostics), but unfortunately that made it so that the model couldn't follow what was going on. The current setup is a bit clunky (and Python's scoping rules makes it especially so -- cf. the explicit calls to `globals` and `locals`), but I think it does the job okay. --- .../experimental/import-resolution/bar.py | 6 ++ .../experimental/import-resolution/foo.py | 14 ++++ .../import-resolution/importflow.expected | 0 .../import-resolution/importflow.ql | 41 ++++++++++++ .../import-resolution/imports.expected | 0 .../experimental/import-resolution/imports.ql | 49 ++++++++++++++ .../experimental/import-resolution/main.py | 65 +++++++++++++++++++ .../namespace_package/namespace_module.py | 6 ++ .../import-resolution/package/__init__.py | 7 ++ .../package/subpackage/__init__.py | 14 ++++ .../package/subpackage/submodule.py | 7 ++ .../experimental/import-resolution/trace.py | 49 ++++++++++++++ 12 files changed, 258 insertions(+) create mode 100644 python/ql/test/experimental/import-resolution/bar.py create mode 100644 python/ql/test/experimental/import-resolution/foo.py create mode 100644 python/ql/test/experimental/import-resolution/importflow.expected create mode 100644 python/ql/test/experimental/import-resolution/importflow.ql create mode 100644 python/ql/test/experimental/import-resolution/imports.expected create mode 100644 python/ql/test/experimental/import-resolution/imports.ql create mode 100644 python/ql/test/experimental/import-resolution/main.py create mode 100644 python/ql/test/experimental/import-resolution/namespace_package/namespace_module.py create mode 100644 python/ql/test/experimental/import-resolution/package/__init__.py create mode 100644 python/ql/test/experimental/import-resolution/package/subpackage/__init__.py create mode 100644 python/ql/test/experimental/import-resolution/package/subpackage/submodule.py create mode 100644 python/ql/test/experimental/import-resolution/trace.py diff --git a/python/ql/test/experimental/import-resolution/bar.py b/python/ql/test/experimental/import-resolution/bar.py new file mode 100644 index 00000000000..5cb6339be17 --- /dev/null +++ b/python/ql/test/experimental/import-resolution/bar.py @@ -0,0 +1,6 @@ +from trace import * +enter(__file__) + +bar_attr = "bar_attr" + +exit(__file__) diff --git a/python/ql/test/experimental/import-resolution/foo.py b/python/ql/test/experimental/import-resolution/foo.py new file mode 100644 index 00000000000..d112007ad03 --- /dev/null +++ b/python/ql/test/experimental/import-resolution/foo.py @@ -0,0 +1,14 @@ +from trace import * +enter(__file__) + +# A simple attribute. Used in main.py +foo_attr = "foo_attr" + +# A private attribute. Accessible from main.py despite this. +__private_foo_attr = "__private_foo_attr" + +# A reexport of bar under a new name. Used in main.py +import bar as bar_reexported #$ imports=bar as=bar_reexported +check("bar_reexported.bar_attr", bar_reexported.bar_attr, "bar_attr", globals()) #$ prints=bar_attr + +exit(__file__) diff --git a/python/ql/test/experimental/import-resolution/importflow.expected b/python/ql/test/experimental/import-resolution/importflow.expected new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/ql/test/experimental/import-resolution/importflow.ql b/python/ql/test/experimental/import-resolution/importflow.ql new file mode 100644 index 00000000000..0e7d300d328 --- /dev/null +++ b/python/ql/test/experimental/import-resolution/importflow.ql @@ -0,0 +1,41 @@ +import python +import semmle.python.dataflow.new.DataFlow +import semmle.python.ApiGraphs +import TestUtilities.InlineExpectationsTest + +private class SourceString extends DataFlow::Node { + string contents; + + SourceString() { + this.asExpr().(StrConst).getText() = contents and + this.asExpr().getParent() instanceof Assign + } + + string getContents() { result = contents } +} + +private class ImportConfiguration extends DataFlow::Configuration { + ImportConfiguration() { this = "ImportConfiguration" } + + override predicate isSource(DataFlow::Node source) { source instanceof SourceString } + + override predicate isSink(DataFlow::Node sink) { + sink = API::moduleImport("trace").getMember("check").getACall().getArg(1) + } +} + +class ResolutionTest extends InlineExpectationsTest { + ResolutionTest() { this = "ResolutionTest" } + + override string getARelevantTag() { result = "import" } + + override predicate hasActualResult(Location location, string element, string tag, string value) { + exists(DataFlow::PathNode source, DataFlow::PathNode sink, ImportConfiguration config | + config.hasFlowPath(source, sink) and + tag = "prints" and + location = sink.getNode().getLocation() and + value = source.getNode().(SourceString).getContents() and + element = sink.getNode().toString() + ) + } +} diff --git a/python/ql/test/experimental/import-resolution/imports.expected b/python/ql/test/experimental/import-resolution/imports.expected new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/ql/test/experimental/import-resolution/imports.ql b/python/ql/test/experimental/import-resolution/imports.ql new file mode 100644 index 00000000000..e5d357c5ef4 --- /dev/null +++ b/python/ql/test/experimental/import-resolution/imports.ql @@ -0,0 +1,49 @@ +import python +import TestUtilities.InlineExpectationsTest +import semmle.python.dataflow.new.DataFlow +import semmle.python.dataflow.new.internal.ImportResolution + +private class ImmediateModuleRef extends DataFlow::Node { + Module mod; + string alias; + + ImmediateModuleRef() { + this = ImportResolution::getImmediateModuleReference(mod) and + not mod.getName() in ["__future__", "trace"] and + this.asExpr() = any(Alias a | alias = a.getAsname().(Name).getId()).getAsname() + } + + Module getModule() { result = mod } + + string getAsname() { result = alias } +} + +class ImportTest extends InlineExpectationsTest { + ImportTest() { this = "ImportTest" } + + override string getARelevantTag() { result = "imports" } + + override predicate hasActualResult(Location location, string element, string tag, string value) { + exists(ImmediateModuleRef ref | + tag = "imports" and + location = ref.getLocation() and + value = ref.getModule().getName() and + element = ref.toString() + ) + } +} + +class AliasTest extends InlineExpectationsTest { + AliasTest() { this = "AliasTest" } + + override string getARelevantTag() { result = "as" } + + override predicate hasActualResult(Location location, string element, string tag, string value) { + exists(ImmediateModuleRef ref | + tag = "as" and + location = ref.getLocation() and + value = ref.getAsname() and + element = ref.toString() + ) + } +} diff --git a/python/ql/test/experimental/import-resolution/main.py b/python/ql/test/experimental/import-resolution/main.py new file mode 100644 index 00000000000..827bb78896b --- /dev/null +++ b/python/ql/test/experimental/import-resolution/main.py @@ -0,0 +1,65 @@ +#! /usr/bin/env python3 + +from __future__ import print_function +import sys +from trace import * +enter(__file__) + +# A simple import. Binds foo to the foo module +import foo #$ imports=foo as=foo +check("foo.foo_attr", foo.foo_attr, "foo_attr", globals()) #$ prints=foo_attr + +# Private attributes are still accessible. +check("foo.__private_foo_attr", foo.__private_foo_attr, "__private_foo_attr", globals()) #$ prints=__private_foo_attr + +# An aliased import, binding foo to foo_alias +import foo as foo_alias #$ imports=foo as=foo_alias +check("foo_alias.foo_attr", foo_alias.foo_attr, "foo_attr", globals()) #$ prints=foo_attr + +# A reference to a reexported module +check("foo.bar_reexported.bar_attr", foo.bar_reexported.bar_attr, "bar_attr", globals()) #$ prints=bar_attr + +# A simple "import from" statement. +from bar import bar_attr +check("bar_attr", bar_attr, "bar_attr", globals()) #$ prints=bar_attr + +# Importing an attribute from a subpackage of a package. +from package.subpackage import subpackage_attr +check("subpackage_attr", subpackage_attr, "subpackage_attr", globals()) #$ prints=subpackage_attr + +# Importing a package attribute under an alias. +from package import package_attr as package_attr_alias +check("package_attr_alias", package_attr_alias, "package_attr", globals()) #$ prints=package_attr + +# Importing a subpackage under an alias. +from package import subpackage as aliased_subpackage #$ imports=package.subpackage.__init__ as=aliased_subpackage +check("aliased_subpackage.subpackage_attr", aliased_subpackage.subpackage_attr, "subpackage_attr", globals()) #$ prints=subpackage_attr + +def local_import(): + # Same as above, but in a local scope. + import package.subpackage as local_subpackage #$ imports=package.subpackage.__init__ as=local_subpackage + check("local_subpackage.subpackage_attr", local_subpackage.subpackage_attr, "subpackage_attr", locals()) #$ prints=subpackage_attr + +local_import() + +# Importing a subpacking using `import` and binding it to a name. +import package.subpackage as aliased_subpackage #$ imports=package.subpackage.__init__ as=aliased_subpackage +check("aliased_subpackage.subpackage_attr", aliased_subpackage.subpackage_attr, "subpackage_attr", globals()) #$ prints=subpackage_attr + +# Importing without binding instead binds the top level name. +import package.subpackage #$ imports=package.__init__ as=package +check("package.package_attr", package.package_attr, "package_attr", globals()) #$ prints=package_attr + +if sys.version_info[0] >= 3: + # Importing from a namespace module. + from namespace_package.namespace_module import namespace_module_attr + check("namespace_module_attr", namespace_module_attr, "namespace_module_attr", globals()) #$ prints=namespace_module_attr + +exit(__file__) + +print() + +if status() == 0: + print("PASS") +else: + print("FAIL") diff --git a/python/ql/test/experimental/import-resolution/namespace_package/namespace_module.py b/python/ql/test/experimental/import-resolution/namespace_package/namespace_module.py new file mode 100644 index 00000000000..2c831662f47 --- /dev/null +++ b/python/ql/test/experimental/import-resolution/namespace_package/namespace_module.py @@ -0,0 +1,6 @@ +from trace import * +enter(__file__) + +namespace_module_attr = "namespace_module_attr" + +exit(__file__) diff --git a/python/ql/test/experimental/import-resolution/package/__init__.py b/python/ql/test/experimental/import-resolution/package/__init__.py new file mode 100644 index 00000000000..ee2e3211a2c --- /dev/null +++ b/python/ql/test/experimental/import-resolution/package/__init__.py @@ -0,0 +1,7 @@ +from trace import * +enter(__file__) + +attr_used_in_subpackage = "attr_used_in_subpackage" +package_attr = "package_attr" + +exit(__file__) diff --git a/python/ql/test/experimental/import-resolution/package/subpackage/__init__.py b/python/ql/test/experimental/import-resolution/package/subpackage/__init__.py new file mode 100644 index 00000000000..86cc92dd37b --- /dev/null +++ b/python/ql/test/experimental/import-resolution/package/subpackage/__init__.py @@ -0,0 +1,14 @@ +from trace import * +enter(__file__) + +subpackage_attr = "subpackage_attr" + +# Importing an attribute from the parent package. +from .. import attr_used_in_subpackage as imported_attr +check("imported_attr", imported_attr, "attr_used_in_subpackage", globals()) #$ prints=attr_used_in_subpackage + +# Importing an irrelevant attribute from a sibling module binds the name to the module. +from .submodule import irrelevant_attr +check("submodule.submodule_attr", submodule.submodule_attr, "submodule_attr", globals()) #$ prints=submodule_attr + +exit(__file__) diff --git a/python/ql/test/experimental/import-resolution/package/subpackage/submodule.py b/python/ql/test/experimental/import-resolution/package/subpackage/submodule.py new file mode 100644 index 00000000000..89f9dd497a4 --- /dev/null +++ b/python/ql/test/experimental/import-resolution/package/subpackage/submodule.py @@ -0,0 +1,7 @@ +from trace import * +enter(__file__) + +submodule_attr = "submodule_attr" +irrelevant_attr = "irrelevant_attr" + +exit(__file__) diff --git a/python/ql/test/experimental/import-resolution/trace.py b/python/ql/test/experimental/import-resolution/trace.py new file mode 100644 index 00000000000..04e847304e9 --- /dev/null +++ b/python/ql/test/experimental/import-resolution/trace.py @@ -0,0 +1,49 @@ +from __future__ import print_function + +_indent_level = 0 + +_print = print + +def print(*args, **kwargs): + _print(" " * _indent_level, end="") + _print(*args, **kwargs) + +def enter(file_name): + global _indent_level + print("Entering {}".format(file_name)) + _indent_level += 1 + +def exit(file_name): + global _indent_level + _indent_level -= 1 + print("Leaving {}".format(file_name)) + +_status = 0 + +def status(): + return _status + +def check(attr_path, actual_value, expected_value, bindings): + parts = attr_path.split(".") + base, parts = parts[0], parts[1:] + if base not in bindings: + print("Error: {} not in bindings".format(base)) + _status = 1 + return + val = bindings[base] + for part in parts: + if not hasattr(val, part): + print("Error: Unknown attribute {}".format(part)) + _status = 1 + return + val = getattr(val, part) + if val != actual_value: + print("Error: Value at path {} and actual value are out of sync! {} != {}".format(attr_path, val, actual_value)) + _status = 1 + if val != expected_value: + print("Error: Expected {} to be {}, got {}".format(attr_path, expected_value, val)) + _status = 1 + return + print("OK: {} = {}".format(attr_path, val)) + +__all__ = ["enter", "exit", "check", "status"] From 58754982cefc5d4b7c2b4082d233a81d06f35066 Mon Sep 17 00:00:00 2001 From: Taus Date: Mon, 17 Oct 2022 14:34:10 +0000 Subject: [PATCH 004/106] Python: Update type tracking tests No longer missing! :tada: --- .../experimental/dataflow/typetracking_imports/pkg/use.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/ql/test/experimental/dataflow/typetracking_imports/pkg/use.py b/python/ql/test/experimental/dataflow/typetracking_imports/pkg/use.py index 023af8684f2..a23ddc54b2d 100644 --- a/python/ql/test/experimental/dataflow/typetracking_imports/pkg/use.py +++ b/python/ql/test/experimental/dataflow/typetracking_imports/pkg/use.py @@ -6,8 +6,8 @@ test_direct_import() def test_alias_problem(): - from .alias_problem import foo # $ MISSING: tracked - print(foo) # $ MISSING: tracked + from .alias_problem import foo # $ tracked + print(foo) # $ tracked test_alias_problem() @@ -34,8 +34,8 @@ test_alias_only_direct() def test_problem_absolute_import(): - from pkg.problem_absolute_import import foo # $ MISSING: tracked - print(foo) # $ MISSING: tracked + from pkg.problem_absolute_import import foo # $ tracked + print(foo) # $ tracked test_problem_absolute_import() From d7f1491f419ca32b0197125755e77bf502bae58b Mon Sep 17 00:00:00 2001 From: erik-krogh Date: Fri, 4 Nov 2022 17:19:42 +0100 Subject: [PATCH 005/106] fix non-attached annotations for newtype branches --- ql/ql/src/codeql_ql/ast/Ast.qll | 4 +++ ql/ql/test/printAst/Foo.qll | 4 +++ ql/ql/test/printAst/printAst.expected | 46 ++++++++++++++++++--------- 3 files changed, 39 insertions(+), 15 deletions(-) diff --git a/ql/ql/src/codeql_ql/ast/Ast.qll b/ql/ql/src/codeql_ql/ast/Ast.qll index d6a5039a3c7..577fae69947 100644 --- a/ql/ql/src/codeql_ql/ast/Ast.qll +++ b/ql/ql/src/codeql_ql/ast/Ast.qll @@ -983,6 +983,8 @@ class NewTypeBranch extends TNewTypeBranch, Predicate, TypeDeclaration { override NewTypeBranchType getReturnType() { result.getDeclaration() = this } + override Annotation getAnAnnotation() { toQL(this).getAFieldOrChild() = toQL(result) } + override Type getParameterType(int i) { result = this.getField(i).getType() } override int getArity() { result = count(this.getField(_)) } @@ -2397,6 +2399,8 @@ private class AnnotationArg extends TAnnotationArg, AstNode { } override string toString() { result = this.getValue() } + + override string getAPrimaryQlClass() { result = "AnnotationArg" } } private class NoInlineArg extends AnnotationArg { diff --git a/ql/ql/test/printAst/Foo.qll b/ql/ql/test/printAst/Foo.qll index 17e4a4d636d..461402cd2eb 100644 --- a/ql/ql/test/printAst/Foo.qll +++ b/ql/ql/test/printAst/Foo.qll @@ -25,3 +25,7 @@ predicate calls(Foo f) { or true = false } + +newtype TPathNode = + pragma[assume_small_delta] + TPathNodeMid() diff --git a/ql/ql/test/printAst/printAst.expected b/ql/ql/test/printAst/printAst.expected index 333b42332e7..e53f9112569 100644 --- a/ql/ql/test/printAst/printAst.expected +++ b/ql/ql/test/printAst/printAst.expected @@ -1,8 +1,8 @@ nodes | Foo.qll:1:1:1:17 | Import | semmle.label | [Import] Import | | Foo.qll:1:1:1:17 | Import | semmle.order | 1 | -| Foo.qll:1:1:27:2 | TopLevel | semmle.label | [TopLevel] TopLevel | -| Foo.qll:1:1:27:2 | TopLevel | semmle.order | 1 | +| Foo.qll:1:1:31:17 | TopLevel | semmle.label | [TopLevel] TopLevel | +| Foo.qll:1:1:31:17 | TopLevel | semmle.order | 1 | | Foo.qll:1:8:1:17 | javascript | semmle.label | [ModuleExpr] javascript | | Foo.qll:1:8:1:17 | javascript | semmle.order | 3 | | Foo.qll:3:7:3:9 | Class Foo | semmle.label | [Class] Class Foo | @@ -153,6 +153,14 @@ nodes | Foo.qll:26:3:26:14 | ComparisonFormula | semmle.order | 75 | | Foo.qll:26:10:26:14 | Boolean | semmle.label | [Boolean] Boolean | | Foo.qll:26:10:26:14 | Boolean | semmle.order | 77 | +| Foo.qll:29:9:29:17 | NewType TPathNode | semmle.label | [NewType] NewType TPathNode | +| Foo.qll:29:9:29:17 | NewType TPathNode | semmle.order | 78 | +| Foo.qll:30:3:30:28 | annotation | semmle.label | [Annotation] annotation | +| Foo.qll:30:3:30:28 | annotation | semmle.order | 79 | +| Foo.qll:30:10:30:27 | assume_small_delta | semmle.label | [AnnotationArg] assume_small_delta | +| Foo.qll:30:10:30:27 | assume_small_delta | semmle.order | 80 | +| Foo.qll:31:3:31:14 | NewTypeBranch TPathNodeMid | semmle.label | [NewTypeBranch] NewTypeBranch TPathNodeMid | +| Foo.qll:31:3:31:14 | NewTypeBranch TPathNodeMid | semmle.order | 81 | | file://:0:0:0:0 | abs | semmle.label | [BuiltinPredicate] abs | | file://:0:0:0:0 | abs | semmle.label | [BuiltinPredicate] abs | | file://:0:0:0:0 | acos | semmle.label | [BuiltinPredicate] acos | @@ -235,22 +243,24 @@ nodes | file://:0:0:0:0 | trim | semmle.label | [BuiltinPredicate] trim | | file://:0:0:0:0 | ulp | semmle.label | [BuiltinPredicate] ulp | | printAst.ql:1:1:1:28 | Import | semmle.label | [Import] Import | -| printAst.ql:1:1:1:28 | Import | semmle.order | 78 | +| printAst.ql:1:1:1:28 | Import | semmle.order | 82 | | printAst.ql:1:1:1:29 | TopLevel | semmle.label | [TopLevel] TopLevel | -| printAst.ql:1:1:1:29 | TopLevel | semmle.order | 78 | +| printAst.ql:1:1:1:29 | TopLevel | semmle.order | 82 | | printAst.ql:1:18:1:28 | printAstAst | semmle.label | [ModuleExpr] printAstAst | -| printAst.ql:1:18:1:28 | printAstAst | semmle.order | 80 | +| printAst.ql:1:18:1:28 | printAstAst | semmle.order | 84 | edges | Foo.qll:1:1:1:17 | Import | Foo.qll:1:8:1:17 | javascript | semmle.label | getModuleExpr() | | Foo.qll:1:1:1:17 | Import | Foo.qll:1:8:1:17 | javascript | semmle.order | 3 | -| Foo.qll:1:1:27:2 | TopLevel | Foo.qll:1:1:1:17 | Import | semmle.label | getAnImport() | -| Foo.qll:1:1:27:2 | TopLevel | Foo.qll:1:1:1:17 | Import | semmle.order | 1 | -| Foo.qll:1:1:27:2 | TopLevel | Foo.qll:3:7:3:9 | Class Foo | semmle.label | getAClass() | -| Foo.qll:1:1:27:2 | TopLevel | Foo.qll:3:7:3:9 | Class Foo | semmle.order | 4 | -| Foo.qll:1:1:27:2 | TopLevel | Foo.qll:9:17:9:19 | ClasslessPredicate foo | semmle.label | getAPredicate() | -| Foo.qll:1:1:27:2 | TopLevel | Foo.qll:9:17:9:19 | ClasslessPredicate foo | semmle.order | 16 | -| Foo.qll:1:1:27:2 | TopLevel | Foo.qll:13:11:13:15 | ClasslessPredicate calls | semmle.label | getAPredicate() | -| Foo.qll:1:1:27:2 | TopLevel | Foo.qll:13:11:13:15 | ClasslessPredicate calls | semmle.order | 32 | +| Foo.qll:1:1:31:17 | TopLevel | Foo.qll:1:1:1:17 | Import | semmle.label | getAnImport() | +| Foo.qll:1:1:31:17 | TopLevel | Foo.qll:1:1:1:17 | Import | semmle.order | 1 | +| Foo.qll:1:1:31:17 | TopLevel | Foo.qll:3:7:3:9 | Class Foo | semmle.label | getAClass() | +| Foo.qll:1:1:31:17 | TopLevel | Foo.qll:3:7:3:9 | Class Foo | semmle.order | 4 | +| Foo.qll:1:1:31:17 | TopLevel | Foo.qll:9:17:9:19 | ClasslessPredicate foo | semmle.label | getAPredicate() | +| Foo.qll:1:1:31:17 | TopLevel | Foo.qll:9:17:9:19 | ClasslessPredicate foo | semmle.order | 16 | +| Foo.qll:1:1:31:17 | TopLevel | Foo.qll:13:11:13:15 | ClasslessPredicate calls | semmle.label | getAPredicate() | +| Foo.qll:1:1:31:17 | TopLevel | Foo.qll:13:11:13:15 | ClasslessPredicate calls | semmle.order | 32 | +| Foo.qll:1:1:31:17 | TopLevel | Foo.qll:29:9:29:17 | NewType TPathNode | semmle.label | getANewType() | +| Foo.qll:1:1:31:17 | TopLevel | Foo.qll:29:9:29:17 | NewType TPathNode | semmle.order | 78 | | Foo.qll:3:7:3:9 | Class Foo | Foo.qll:3:19:3:22 | TypeExpr | semmle.label | getASuperType() | | Foo.qll:3:7:3:9 | Class Foo | Foo.qll:3:19:3:22 | TypeExpr | semmle.order | 5 | | Foo.qll:3:7:3:9 | Class Foo | Foo.qll:4:3:4:17 | CharPred Foo | semmle.label | getCharPred() | @@ -393,9 +403,15 @@ edges | Foo.qll:26:3:26:14 | ComparisonFormula | Foo.qll:26:3:26:6 | Boolean | semmle.order | 75 | | Foo.qll:26:3:26:14 | ComparisonFormula | Foo.qll:26:10:26:14 | Boolean | semmle.label | getRightOperand() | | Foo.qll:26:3:26:14 | ComparisonFormula | Foo.qll:26:10:26:14 | Boolean | semmle.order | 77 | +| Foo.qll:29:9:29:17 | NewType TPathNode | Foo.qll:31:3:31:14 | NewTypeBranch TPathNodeMid | semmle.label | getABranch() | +| Foo.qll:29:9:29:17 | NewType TPathNode | Foo.qll:31:3:31:14 | NewTypeBranch TPathNodeMid | semmle.order | 81 | +| Foo.qll:30:3:30:28 | annotation | Foo.qll:30:10:30:27 | assume_small_delta | semmle.label | getArgs(_) | +| Foo.qll:30:3:30:28 | annotation | Foo.qll:30:10:30:27 | assume_small_delta | semmle.order | 80 | +| Foo.qll:31:3:31:14 | NewTypeBranch TPathNodeMid | Foo.qll:30:3:30:28 | annotation | semmle.label | getAnAnnotation() | +| Foo.qll:31:3:31:14 | NewTypeBranch TPathNodeMid | Foo.qll:30:3:30:28 | annotation | semmle.order | 79 | | printAst.ql:1:1:1:28 | Import | printAst.ql:1:18:1:28 | printAstAst | semmle.label | getModuleExpr() | -| printAst.ql:1:1:1:28 | Import | printAst.ql:1:18:1:28 | printAstAst | semmle.order | 80 | +| printAst.ql:1:1:1:28 | Import | printAst.ql:1:18:1:28 | printAstAst | semmle.order | 84 | | printAst.ql:1:1:1:29 | TopLevel | printAst.ql:1:1:1:28 | Import | semmle.label | getAnImport() | -| printAst.ql:1:1:1:29 | TopLevel | printAst.ql:1:1:1:28 | Import | semmle.order | 78 | +| printAst.ql:1:1:1:29 | TopLevel | printAst.ql:1:1:1:28 | Import | semmle.order | 82 | graphProperties | semmle.graphKind | tree | From dddf550593ee21e15c45c92cab409dec85375da6 Mon Sep 17 00:00:00 2001 From: erik-krogh Date: Tue, 1 Nov 2022 11:39:17 +0100 Subject: [PATCH 006/106] add codeql/regex as a dependency --- ruby/ql/lib/qlpack.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ruby/ql/lib/qlpack.yml b/ruby/ql/lib/qlpack.yml index 016f75260eb..bc6f3e513ee 100644 --- a/ruby/ql/lib/qlpack.yml +++ b/ruby/ql/lib/qlpack.yml @@ -7,4 +7,4 @@ upgrades: upgrades library: true dependencies: codeql/ssa: ${workspace} - + codeql/regex: ${workspace} From af922702c79458864a0ea6329de37dd7701acb50 Mon Sep 17 00:00:00 2001 From: erik-krogh Date: Tue, 1 Nov 2022 11:41:55 +0100 Subject: [PATCH 007/106] move existing regex-tree into a module --- .../lib/codeql/ruby/regexp/RegExpTreeView.qll | 2109 +++++++++-------- 1 file changed, 1058 insertions(+), 1051 deletions(-) diff --git a/ruby/ql/lib/codeql/ruby/regexp/RegExpTreeView.qll b/ruby/ql/lib/codeql/ruby/regexp/RegExpTreeView.qll index 1ff182e3dcc..fffa0044771 100644 --- a/ruby/ql/lib/codeql/ruby/regexp/RegExpTreeView.qll +++ b/ruby/ql/lib/codeql/ruby/regexp/RegExpTreeView.qll @@ -4,6 +4,12 @@ private import internal.ParseRegExp private import codeql.NumberUtils private import codeql.ruby.ast.Literal as Ast private import codeql.Locations +import Impl + +/** Gets the parse tree resulting from parsing `re`, if such has been constructed. */ +RegExpTerm getParsedRegExp(Ast::RegExpLiteral re) { + result.getRegExp() = re and result.isRootTerm() +} /** * An element containing a regular expression term, that is, either @@ -45,1109 +51,1110 @@ private newtype TRegExpParent = re.namedCharacterProperty(start, end, _) } -/** - * An element containing a regular expression term, that is, either - * a string literal (parsed as a regular expression) - * or another regular expression term. - */ -class RegExpParent extends TRegExpParent { - /** Gets a textual representation of this element. */ - string toString() { result = "RegExpParent" } - - /** Gets the `i`th child term. */ - RegExpTerm getChild(int i) { none() } - - /** Gets a child term . */ - final RegExpTerm getAChild() { result = this.getChild(_) } - - /** Gets the number of child terms. */ - int getNumChild() { result = count(this.getAChild()) } - - /** Gets the last child term of this element. */ - RegExpTerm getLastChild() { result = this.getChild(this.getNumChild() - 1) } - +/** An implementation that statisfies the RegexTreeView signature. */ +private module Impl { /** - * Gets the name of a primary CodeQL class to which this regular - * expression term belongs. + * An element containing a regular expression term, that is, either + * a string literal (parsed as a regular expression) + * or another regular expression term. */ - string getAPrimaryQlClass() { result = "RegExpParent" } + class RegExpParent extends TRegExpParent { + /** Gets a textual representation of this element. */ + string toString() { result = "RegExpParent" } - /** - * Gets a comma-separated list of the names of the primary CodeQL classes to - * which this regular expression term belongs. - */ - final string getPrimaryQlClasses() { result = concat(this.getAPrimaryQlClass(), ",") } -} + /** Gets the `i`th child term. */ + RegExpTerm getChild(int i) { none() } -/** A string literal used as a regular expression */ -class RegExpLiteral extends TRegExpLiteral, RegExpParent { - RegExp re; + /** Gets a child term . */ + final RegExpTerm getAChild() { result = this.getChild(_) } - RegExpLiteral() { this = TRegExpLiteral(re) } + /** Gets the number of child terms. */ + int getNumChild() { result = count(this.getAChild()) } - override RegExpTerm getChild(int i) { i = 0 and result.getRegExp() = re and result.isRootTerm() } + /** Gets the last child term of this element. */ + RegExpTerm getLastChild() { result = this.getChild(this.getNumChild() - 1) } - /** Holds if dot, `.`, matches all characters, including newlines. */ - predicate isDotAll() { re.isDotAll() } + /** + * Gets the name of a primary CodeQL class to which this regular + * expression term belongs. + */ + string getAPrimaryQlClass() { result = "RegExpParent" } - /** Holds if this regex matching is case-insensitive for this regex. */ - predicate isIgnoreCase() { re.isIgnoreCase() } + /** + * Gets a comma-separated list of the names of the primary CodeQL classes to + * which this regular expression term belongs. + */ + final string getPrimaryQlClasses() { result = concat(this.getAPrimaryQlClass(), ",") } + } - /** Get a string representing all modes for this regex. */ - string getFlags() { result = re.getFlags() } + /** A string literal used as a regular expression */ + class RegExpLiteral extends TRegExpLiteral, RegExpParent { + RegExp re; - /** Gets the primary QL class for this regex. */ - override string getAPrimaryQlClass() { result = "RegExpLiteral" } -} + RegExpLiteral() { this = TRegExpLiteral(re) } -/** - * A regular expression term, that is, a syntactic part of a regular expression. - */ -class RegExpTerm extends RegExpParent { - RegExp re; - int start; - int end; + override RegExpTerm getChild(int i) { + i = 0 and result.getRegExp() = re and result.isRootTerm() + } - RegExpTerm() { - this = TRegExpAlt(re, start, end) - or - this = TRegExpBackRef(re, start, end) - or - this = TRegExpCharacterClass(re, start, end) - or - this = TRegExpCharacterRange(re, start, end) - or - this = TRegExpNormalChar(re, start, end) - or - this = TRegExpGroup(re, start, end) - or - this = TRegExpQuantifier(re, start, end) - or - this = TRegExpSequence(re, start, end) and - exists(seqChild(re, start, end, 1)) // if a sequence does not have more than one element, it should be treated as that element instead. - or - this = TRegExpSpecialChar(re, start, end) - or - this = TRegExpNamedCharacterProperty(re, start, end) + /** Holds if dot, `.`, matches all characters, including newlines. */ + predicate isDotAll() { re.isDotAll() } + + /** Holds if this regex matching is case-insensitive for this regex. */ + predicate isIgnoreCase() { re.isIgnoreCase() } + + /** Get a string representing all modes for this regex. */ + string getFlags() { result = re.getFlags() } + + /** Gets the primary QL class for this regex. */ + override string getAPrimaryQlClass() { result = "RegExpLiteral" } } /** - * Gets the outermost term of this regular expression. + * A regular expression term, that is, a syntactic part of a regular expression. */ - RegExpTerm getRootTerm() { - this.isRootTerm() and result = this - or - result = this.getParent().(RegExpTerm).getRootTerm() - } + class RegExpTerm extends RegExpParent { + RegExp re; + int start; + int end; - /** - * Holds if this term is part of a string literal - * that is interpreted as a regular expression. - */ - predicate isUsedAsRegExp() { any() } - - /** - * Holds if this is the root term of a regular expression. - */ - predicate isRootTerm() { start = 0 and end = re.getText().length() } - - override RegExpTerm getChild(int i) { - result = this.(RegExpAlt).getChild(i) - or - result = this.(RegExpBackRef).getChild(i) - or - result = this.(RegExpCharacterClass).getChild(i) - or - result = this.(RegExpCharacterRange).getChild(i) - or - result = this.(RegExpNormalChar).getChild(i) - or - result = this.(RegExpGroup).getChild(i) - or - result = this.(RegExpQuantifier).getChild(i) - or - result = this.(RegExpSequence).getChild(i) - or - result = this.(RegExpSpecialChar).getChild(i) - or - result = this.(RegExpNamedCharacterProperty).getChild(i) - } - - /** - * Gets the parent term of this regular expression term, or the - * regular expression literal if this is the root term. - */ - RegExpParent getParent() { result.getAChild() = this } - - /** Gets the associated `RegExp`. */ - RegExp getRegExp() { result = re } - - /** Gets the offset at which this term starts. */ - int getStart() { result = start } - - /** Gets the offset at which this term ends. */ - int getEnd() { result = end } - - override string toString() { result = re.getText().substring(start, end) } - - /** - * Gets the location of the surrounding regex, as locations inside the regex do not exist. - * To get location information corresponding to the term inside the regex, - * use `hasLocationInfo`. - */ - Location getLocation() { result = re.getLocation() } - - pragma[noinline] - private predicate componentHasLocationInfo( - int i, string filepath, int startline, int startcolumn, int endline, int endcolumn - ) { - re.getComponent(i) - .getLocation() - .hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn) - } - - /** Holds if this term is found at the specified location offsets. */ - predicate hasLocationInfo( - string filepath, int startline, int startcolumn, int endline, int endcolumn - ) { - exists(int re_start, int re_end | - this.componentHasLocationInfo(0, filepath, startline, re_start, _, _) and - this.componentHasLocationInfo(re.getNumberOfComponents() - 1, filepath, _, _, endline, re_end) and - startcolumn = re_start + start and - endcolumn = re_start + end - 1 - ) - } - - /** Gets the file in which this term is found. */ - File getFile() { result = this.getLocation().getFile() } - - /** Gets the raw source text of this term. */ - string getRawValue() { result = this.toString() } - - /** Gets the string literal in which this term is found. */ - RegExpLiteral getLiteral() { result = TRegExpLiteral(re) } - - /** Gets the regular expression term that is matched (textually) before this one, if any. */ - RegExpTerm getPredecessor() { - exists(RegExpTerm parent | parent = this.getParent() | - result = parent.(RegExpSequence).previousElement(this) + RegExpTerm() { + this = TRegExpAlt(re, start, end) or - not exists(parent.(RegExpSequence).previousElement(this)) and - not parent instanceof RegExpSubPattern and - result = parent.getPredecessor() - ) - } - - /** Gets the regular expression term that is matched (textually) after this one, if any. */ - RegExpTerm getSuccessor() { - exists(RegExpTerm parent | parent = this.getParent() | - result = parent.(RegExpSequence).nextElement(this) + this = TRegExpBackRef(re, start, end) or - not exists(parent.(RegExpSequence).nextElement(this)) and - not parent instanceof RegExpSubPattern and - result = parent.getSuccessor() - ) + this = TRegExpCharacterClass(re, start, end) + or + this = TRegExpCharacterRange(re, start, end) + or + this = TRegExpNormalChar(re, start, end) + or + this = TRegExpGroup(re, start, end) + or + this = TRegExpQuantifier(re, start, end) + or + this = TRegExpSequence(re, start, end) and + exists(seqChild(re, start, end, 1)) // if a sequence does not have more than one element, it should be treated as that element instead. + or + this = TRegExpSpecialChar(re, start, end) + or + this = TRegExpNamedCharacterProperty(re, start, end) + } + + /** + * Gets the outermost term of this regular expression. + */ + RegExpTerm getRootTerm() { + this.isRootTerm() and result = this + or + result = this.getParent().(RegExpTerm).getRootTerm() + } + + /** + * Holds if this term is part of a string literal + * that is interpreted as a regular expression. + */ + predicate isUsedAsRegExp() { any() } + + /** + * Holds if this is the root term of a regular expression. + */ + predicate isRootTerm() { start = 0 and end = re.getText().length() } + + override RegExpTerm getChild(int i) { + result = this.(RegExpAlt).getChild(i) + or + result = this.(RegExpBackRef).getChild(i) + or + result = this.(RegExpCharacterClass).getChild(i) + or + result = this.(RegExpCharacterRange).getChild(i) + or + result = this.(RegExpNormalChar).getChild(i) + or + result = this.(RegExpGroup).getChild(i) + or + result = this.(RegExpQuantifier).getChild(i) + or + result = this.(RegExpSequence).getChild(i) + or + result = this.(RegExpSpecialChar).getChild(i) + or + result = this.(RegExpNamedCharacterProperty).getChild(i) + } + + /** + * Gets the parent term of this regular expression term, or the + * regular expression literal if this is the root term. + */ + RegExpParent getParent() { result.getAChild() = this } + + /** Gets the associated `RegExp`. */ + RegExp getRegExp() { result = re } + + /** Gets the offset at which this term starts. */ + int getStart() { result = start } + + /** Gets the offset at which this term ends. */ + int getEnd() { result = end } + + override string toString() { result = re.getText().substring(start, end) } + + /** + * Gets the location of the surrounding regex, as locations inside the regex do not exist. + * To get location information corresponding to the term inside the regex, + * use `hasLocationInfo`. + */ + Location getLocation() { result = re.getLocation() } + + pragma[noinline] + private predicate componentHasLocationInfo( + int i, string filepath, int startline, int startcolumn, int endline, int endcolumn + ) { + re.getComponent(i) + .getLocation() + .hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn) + } + + /** Holds if this term is found at the specified location offsets. */ + predicate hasLocationInfo( + string filepath, int startline, int startcolumn, int endline, int endcolumn + ) { + exists(int re_start, int re_end | + this.componentHasLocationInfo(0, filepath, startline, re_start, _, _) and + this.componentHasLocationInfo(re.getNumberOfComponents() - 1, filepath, _, _, endline, + re_end) and + startcolumn = re_start + start and + endcolumn = re_start + end - 1 + ) + } + + /** Gets the file in which this term is found. */ + File getFile() { result = this.getLocation().getFile() } + + /** Gets the raw source text of this term. */ + string getRawValue() { result = this.toString() } + + /** Gets the string literal in which this term is found. */ + RegExpLiteral getLiteral() { result = TRegExpLiteral(re) } + + /** Gets the regular expression term that is matched (textually) before this one, if any. */ + RegExpTerm getPredecessor() { + exists(RegExpTerm parent | parent = this.getParent() | + result = parent.(RegExpSequence).previousElement(this) + or + not exists(parent.(RegExpSequence).previousElement(this)) and + not parent instanceof RegExpSubPattern and + result = parent.getPredecessor() + ) + } + + /** Gets the regular expression term that is matched (textually) after this one, if any. */ + RegExpTerm getSuccessor() { + exists(RegExpTerm parent | parent = this.getParent() | + result = parent.(RegExpSequence).nextElement(this) + or + not exists(parent.(RegExpSequence).nextElement(this)) and + not parent instanceof RegExpSubPattern and + result = parent.getSuccessor() + ) + } + + /** + * Gets the single string this regular-expression term matches. + * + * This predicate is only defined for (sequences/groups of) constant regular + * expressions. In particular, terms involving zero-width assertions like `^` + * or `\b` are not considered to have a constant value. + * + * Note that this predicate does not take flags of the enclosing + * regular-expression literal into account. + */ + string getConstantValue() { none() } + + /** + * Gets a string that is matched by this regular-expression term. + */ + string getAMatchedString() { result = this.getConstantValue() } + + /** Gets the primary QL class for this term. */ + override string getAPrimaryQlClass() { result = "RegExpTerm" } + + /** Holds if this regular expression term can match the empty string. */ + predicate isNullable() { none() } } /** - * Gets the single string this regular-expression term matches. + * A quantified regular expression term. * - * This predicate is only defined for (sequences/groups of) constant regular - * expressions. In particular, terms involving zero-width assertions like `^` - * or `\b` are not considered to have a constant value. + * Example: * - * Note that this predicate does not take flags of the enclosing - * regular-expression literal into account. + * ``` + * ((ECMA|Java)[sS]cript)* + * ``` */ - string getConstantValue() { none() } + class RegExpQuantifier extends RegExpTerm, TRegExpQuantifier { + int part_end; + boolean may_repeat_forever; - /** - * Gets a string that is matched by this regular-expression term. - */ - string getAMatchedString() { result = this.getConstantValue() } + RegExpQuantifier() { + this = TRegExpQuantifier(re, start, end) and + re.qualifiedPart(start, part_end, end, _, may_repeat_forever) + } - /** Gets the primary QL class for this term. */ - override string getAPrimaryQlClass() { result = "RegExpTerm" } - - /** Holds if this regular expression term can match the empty string. */ - predicate isNullable() { none() } -} - -/** - * A quantified regular expression term. - * - * Example: - * - * ``` - * ((ECMA|Java)[sS]cript)* - * ``` - */ -class RegExpQuantifier extends RegExpTerm, TRegExpQuantifier { - int part_end; - boolean may_repeat_forever; - - RegExpQuantifier() { - this = TRegExpQuantifier(re, start, end) and - re.qualifiedPart(start, part_end, end, _, may_repeat_forever) - } - - override RegExpTerm getChild(int i) { - i = 0 and - result.getRegExp() = re and - result.getStart() = start and - result.getEnd() = part_end - } - - /** Hols if this term may match an unlimited number of times. */ - predicate mayRepeatForever() { may_repeat_forever = true } - - /** Gets the qualifier for this term. That is e.g "?" for "a?". */ - string getQualifier() { result = re.getText().substring(part_end, end) } - - override string getAPrimaryQlClass() { result = "RegExpQuantifier" } -} - -/** - * A regular expression term that permits unlimited repetitions. - */ -class InfiniteRepetitionQuantifier extends RegExpQuantifier { - InfiniteRepetitionQuantifier() { this.mayRepeatForever() } - - override string getAPrimaryQlClass() { result = "InfiniteRepetitionQuantifier" } -} - -/** - * A star-quantified term. - * - * Example: - * - * ``` - * \w* - * ``` - */ -class RegExpStar extends InfiniteRepetitionQuantifier { - RegExpStar() { this.getQualifier().charAt(0) = "*" } - - override string getAPrimaryQlClass() { result = "RegExpStar" } - - override predicate isNullable() { any() } -} - -/** - * A plus-quantified term. - * - * Example: - * - * ``` - * \w+ - * ``` - */ -class RegExpPlus extends InfiniteRepetitionQuantifier { - RegExpPlus() { this.getQualifier().charAt(0) = "+" } - - override string getAPrimaryQlClass() { result = "RegExpPlus" } - - override predicate isNullable() { this.getAChild().isNullable() } -} - -/** - * An optional term. - * - * Example: - * - * ``` - * ;? - * ``` - */ -class RegExpOpt extends RegExpQuantifier { - RegExpOpt() { this.getQualifier().charAt(0) = "?" } - - override string getAPrimaryQlClass() { result = "RegExpOpt" } - - override predicate isNullable() { any() } -} - -/** - * A range-quantified term - * - * Examples: - * - * ``` - * \w{2,4} - * \w{2,} - * \w{2} - * ``` - */ -class RegExpRange extends RegExpQuantifier { - string upper; - string lower; - - RegExpRange() { re.multiples(part_end, end, lower, upper) } - - override string getAPrimaryQlClass() { result = "RegExpRange" } - - /** Gets the string defining the upper bound of this range, if any. */ - string getUpper() { result = upper } - - /** Gets the string defining the lower bound of this range, if any. */ - string getLower() { result = lower } - - /** - * Gets the upper bound of the range, if any. - * - * If there is no upper bound, any number of repetitions is allowed. - * For a term of the form `r{lo}`, both the lower and the upper bound - * are `lo`. - */ - int getUpperBound() { result = this.getUpper().toInt() } - - /** Gets the lower bound of the range. */ - int getLowerBound() { result = this.getLower().toInt() } - - override predicate isNullable() { this.getAChild().isNullable() or this.getLowerBound() = 0 } -} - -/** - * A sequence term. - * - * Example: - * - * ``` - * (ECMA|Java)Script - * ``` - * - * This is a sequence with the elements `(ECMA|Java)` and `Script`. - */ -class RegExpSequence extends RegExpTerm, TRegExpSequence { - RegExpSequence() { - this = TRegExpSequence(re, start, end) and - exists(seqChild(re, start, end, 1)) // if a sequence does not have more than one element, it should be treated as that element instead. - } - - override RegExpTerm getChild(int i) { result = seqChild(re, start, end, i) } - - /** Gets the element preceding `element` in this sequence. */ - RegExpTerm previousElement(RegExpTerm element) { element = this.nextElement(result) } - - /** Gets the element following `element` in this sequence. */ - RegExpTerm nextElement(RegExpTerm element) { - exists(int i | - element = this.getChild(i) and - result = this.getChild(i + 1) - ) - } - - override string getConstantValue() { result = this.getConstantValue(0) } - - /** - * Gets the single string matched by the `i`th child and all following - * children of this sequence, if any. - */ - private string getConstantValue(int i) { - i = this.getNumChild() and - result = "" - or - result = this.getChild(i).getConstantValue() + this.getConstantValue(i + 1) - } - - override string getAPrimaryQlClass() { result = "RegExpSequence" } - - override predicate isNullable() { - forall(RegExpTerm child | child = this.getAChild() | child.isNullable()) - } -} - -pragma[nomagic] -private int seqChildEnd(RegExp re, int start, int end, int i) { - result = seqChild(re, start, end, i).getEnd() -} - -// moved out so we can use it in the charpred -private RegExpTerm seqChild(RegExp re, int start, int end, int i) { - re.sequence(start, end) and - ( - i = 0 and - result.getRegExp() = re and - result.getStart() = start and - exists(int itemEnd | - re.item(start, itemEnd) and - result.getEnd() = itemEnd - ) - or - i > 0 and - result.getRegExp() = re and - exists(int itemStart | itemStart = seqChildEnd(re, start, end, i - 1) | - result.getStart() = itemStart and - re.item(itemStart, result.getEnd()) - ) - ) -} - -/** - * An alternative term, that is, a term of the form `a|b`. - * - * Example: - * - * ``` - * ECMA|Java - * ``` - */ -class RegExpAlt extends RegExpTerm, TRegExpAlt { - RegExpAlt() { this = TRegExpAlt(re, start, end) } - - override RegExpTerm getChild(int i) { - i = 0 and - result.getRegExp() = re and - result.getStart() = start and - exists(int part_end | - re.alternationOption(start, end, start, part_end) and - result.getEnd() = part_end - ) - or - i > 0 and - result.getRegExp() = re and - exists(int part_start | - part_start = this.getChild(i - 1).getEnd() + 1 // allow for the | - | - result.getStart() = part_start and - re.alternationOption(start, end, part_start, result.getEnd()) - ) - } - - /** Gets an alternative of this term. */ - RegExpTerm getAlternative() { result = this.getAChild() } - - override string getAMatchedString() { result = this.getAlternative().getAMatchedString() } - - override string getAPrimaryQlClass() { result = "RegExpAlt" } - - override predicate isNullable() { this.getAChild().isNullable() } -} - -class RegExpCharEscape = RegExpEscape; - -/** - * An escaped regular expression term, that is, a regular expression - * term starting with a backslash, which is not a backreference. - * - * Example: - * - * ``` - * \. - * \w - * ``` - */ -class RegExpEscape extends RegExpNormalChar { - RegExpEscape() { re.escapedCharacter(start, end) } - - /** - * Gets the name of the escaped; for example, `w` for `\w`. - * TODO: Handle named escapes. - */ - override string getValue() { - not this.isUnicode() and - this.isIdentityEscape() and - result = this.getUnescaped() - or - this.getUnescaped() = "n" and result = "\n" - or - this.getUnescaped() = "r" and result = "\r" - or - this.getUnescaped() = "t" and result = "\t" - or - this.getUnescaped() = "f" and result = 12.toUnicode() - or - this.getUnescaped() = "v" and result = 11.toUnicode() - or - this.isUnicode() and - result = this.getUnicode() - } - - /** Holds if this terms name is given by the part following the escape character. */ - predicate isIdentityEscape() { - not this.getUnescaped() in ["n", "r", "t", "f", "v"] and not this.isUnicode() - } - - override string getAPrimaryQlClass() { result = "RegExpEscape" } - - /** Gets the part of the term following the escape character. That is e.g. "w" if the term is "\w". */ - string getUnescaped() { result = this.getText().suffix(1) } - - /** - * Gets the text for this escape. That is e.g. "\w". - */ - private string getText() { result = re.getText().substring(start, end) } - - /** - * Holds if this is a unicode escape. - */ - private predicate isUnicode() { this.getText().prefix(2) = ["\\u", "\\U"] } - - /** - * Gets the unicode char for this escape. - * E.g. for `\u0061` this returns "a". - */ - private string getUnicode() { - this.isUnicode() and - result = parseHexInt(this.getText().suffix(2)).toUnicode() - } -} - -/** - * A word boundary, that is, a regular expression term of the form `\b`. - */ -class RegExpWordBoundary extends RegExpSpecialChar { - RegExpWordBoundary() { this.getChar() = "\\b" } - - override predicate isNullable() { none() } -} - -/** - * A non-word boundary, that is, a regular expression term of the form `\B`. - */ -class RegExpNonWordBoundary extends RegExpSpecialChar { - RegExpNonWordBoundary() { this.getChar() = "\\B" } - - override string getAPrimaryQlClass() { result = "RegExpNonWordBoundary" } -} - -/** - * A character class escape in a regular expression. - * That is, an escaped character that denotes multiple characters. - * - * Examples: - * - * ``` - * \w - * \S - * ``` - */ -class RegExpCharacterClassEscape extends RegExpEscape { - RegExpCharacterClassEscape() { this.getValue() in ["d", "D", "s", "S", "w", "W", "h", "H"] } - - override RegExpTerm getChild(int i) { none() } - - override string getAPrimaryQlClass() { result = "RegExpCharacterClassEscape" } - - override predicate isNullable() { none() } -} - -/** - * A character class in a regular expression. - * - * Examples: - * - * ```rb - * /[a-fA-F0-9]/ - * /[^abc]/ - * ``` - */ -class RegExpCharacterClass extends RegExpTerm, TRegExpCharacterClass { - RegExpCharacterClass() { this = TRegExpCharacterClass(re, start, end) } - - /** Holds if this character class is inverted, matching the opposite of its content. */ - predicate isInverted() { re.getChar(start + 1) = "^" } - - /** Holds if this character class can match anything. */ - predicate isUniversalClass() { - // [^] - this.isInverted() and not exists(this.getAChild()) - or - // [\w\W] and similar - not this.isInverted() and - exists(string cce1, string cce2 | - cce1 = this.getAChild().(RegExpCharacterClassEscape).getValue() and - cce2 = this.getAChild().(RegExpCharacterClassEscape).getValue() - | - cce1 != cce2 and cce1.toLowerCase() = cce2.toLowerCase() - ) - } - - override RegExpTerm getChild(int i) { - i = 0 and - result.getRegExp() = re and - exists(int itemStart, int itemEnd | - result.getStart() = itemStart and - re.charSetStart(start, itemStart) and - re.charSetChild(start, itemStart, itemEnd) and - result.getEnd() = itemEnd - ) - or - i > 0 and - result.getRegExp() = re and - exists(int itemStart | itemStart = this.getChild(i - 1).getEnd() | - result.getStart() = itemStart and - re.charSetChild(start, itemStart, result.getEnd()) - ) - } - - override string getAMatchedString() { - not this.isInverted() and result = this.getAChild().getAMatchedString() - } - - override string getAPrimaryQlClass() { result = "RegExpCharacterClass" } - - override predicate isNullable() { none() } -} - -/** - * A character range in a character class in a regular expression. - * - * Example: - * - * ``` - * a-z - * ``` - */ -class RegExpCharacterRange extends RegExpTerm, TRegExpCharacterRange { - int lower_end; - int upper_start; - - RegExpCharacterRange() { - this = TRegExpCharacterRange(re, start, end) and - re.charRange(_, start, lower_end, upper_start, end) - } - - /** Holds if this range goes from `lo` to `hi`, in effect is `lo-hi`. */ - predicate isRange(string lo, string hi) { - lo = re.getText().substring(start, lower_end) and - hi = re.getText().substring(upper_start, end) - } - - override RegExpTerm getChild(int i) { - i = 0 and - result.getRegExp() = re and - result.getStart() = start and - result.getEnd() = lower_end - or - i = 1 and - result.getRegExp() = re and - result.getStart() = upper_start and - result.getEnd() = end - } - - override string getAPrimaryQlClass() { result = "RegExpCharacterRange" } - - override predicate isNullable() { none() } -} - -/** - * A normal character in a regular expression, that is, a character - * without special meaning. This includes escaped characters. - * - * Examples: - * ``` - * t - * \t - * ``` - */ -class RegExpNormalChar extends RegExpTerm, TRegExpNormalChar { - RegExpNormalChar() { this = TRegExpNormalChar(re, start, end) } - - /** - * Holds if this constant represents a valid Unicode character (as opposed - * to a surrogate code point that does not correspond to a character by itself.) - */ - predicate isCharacter() { any() } - - /** Gets the string representation of the char matched by this term. */ - string getValue() { result = re.getText().substring(start, end) } - - override RegExpTerm getChild(int i) { none() } - - override string getAPrimaryQlClass() { result = "RegExpNormalChar" } -} - -/** - * A constant regular expression term, that is, a regular expression - * term matching a single string. Currently, this will always be a single character. - * - * Example: - * - * ``` - * a - * ``` - */ -class RegExpConstant extends RegExpTerm { - string value; - - RegExpConstant() { - this = TRegExpNormalChar(re, start, end) and - not this instanceof RegExpCharacterClassEscape and - // exclude chars in qualifiers - // TODO: push this into regex library - not exists(int qstart, int qend | re.qualifiedPart(_, qstart, qend, _, _) | - qstart <= start and end <= qend - ) and - value = this.(RegExpNormalChar).getValue() - or - this = TRegExpSpecialChar(re, start, end) and - re.inCharSet(start) and - value = this.(RegExpSpecialChar).getChar() - } - - /** - * Holds if this constant represents a valid Unicode character (as opposed - * to a surrogate code point that does not correspond to a character by itself.) - */ - predicate isCharacter() { any() } - - /** Gets the string matched by this constant term. */ - string getValue() { result = value } - - override RegExpTerm getChild(int i) { none() } - - override string getConstantValue() { result = this.getValue() } - - override string getAPrimaryQlClass() { result = "RegExpConstant" } - - override predicate isNullable() { none() } -} - -/** - * A grouped regular expression. - * - * Examples: - * - * ``` - * (ECMA|Java) - * (?:ECMA|Java) - * (?['"]) - * ``` - */ -class RegExpGroup extends RegExpTerm, TRegExpGroup { - RegExpGroup() { this = TRegExpGroup(re, start, end) } - - /** - * Gets the index of this capture group within the enclosing regular - * expression literal. - * - * For example, in the regular expression `/((a?).)(?:b)/`, the - * group `((a?).)` has index 1, the group `(a?)` nested inside it - * has index 2, and the group `(?:b)` has no index, since it is - * not a capture group. - */ - int getNumber() { result = re.getGroupNumber(start, end) } - - /** Holds if this is a capture group. */ - predicate isCapture() { exists(this.getNumber()) } - - /** Holds if this is a named capture group. */ - predicate isNamed() { exists(this.getName()) } - - /** Gets the name of this capture group, if any. */ - string getName() { result = re.getGroupName(start, end) } - - override RegExpTerm getChild(int i) { - result.getRegExp() = re and - i = 0 and - re.groupContents(start, end, result.getStart(), result.getEnd()) - } - - override string getConstantValue() { result = this.getAChild().getConstantValue() } - - override string getAMatchedString() { result = this.getAChild().getAMatchedString() } - - override string getAPrimaryQlClass() { result = "RegExpGroup" } - - override predicate isNullable() { this.getAChild().isNullable() } -} - -/** - * A special character in a regular expression. - * - * Examples: - * ``` - * ^ - * $ - * . - * ``` - */ -class RegExpSpecialChar extends RegExpTerm, TRegExpSpecialChar { - string char; - - RegExpSpecialChar() { - this = TRegExpSpecialChar(re, start, end) and - re.specialCharacter(start, end, char) - } - - /** - * Holds if this constant represents a valid Unicode character (as opposed - * to a surrogate code point that does not correspond to a character by itself.) - */ - predicate isCharacter() { any() } - - /** Gets the char for this term. */ - string getChar() { result = char } - - override RegExpTerm getChild(int i) { none() } - - override string getAPrimaryQlClass() { result = "RegExpSpecialChar" } -} - -/** - * A dot regular expression. - * - * Example: - * - * ``` - * . - * ``` - */ -class RegExpDot extends RegExpSpecialChar { - RegExpDot() { this.getChar() = "." } - - override string getAPrimaryQlClass() { result = "RegExpDot" } - - override predicate isNullable() { none() } -} - -/** - * A term that matches a specific position between characters in the string. - * - * Example: - * - * ``` - * \A - * ``` - */ -class RegExpAnchor extends RegExpSpecialChar { - RegExpAnchor() { this.getChar() = ["^", "$", "\\A", "\\Z", "\\z"] } - - override string getAPrimaryQlClass() { result = "RegExpAnchor" } -} - -/** - * A dollar assertion `$` or `\Z` matching the end of a line. - * - * Example: - * - * ``` - * $ - * ``` - */ -class RegExpDollar extends RegExpAnchor { - RegExpDollar() { this.getChar() = ["$", "\\Z", "\\z"] } - - override string getAPrimaryQlClass() { result = "RegExpDollar" } - - override predicate isNullable() { any() } -} - -/** - * A caret assertion `^` or `\A` matching the beginning of a line. - * - * Example: - * - * ``` - * ^ - * ``` - */ -class RegExpCaret extends RegExpAnchor { - RegExpCaret() { this.getChar() = ["^", "\\A"] } - - override string getAPrimaryQlClass() { result = "RegExpCaret" } - - override predicate isNullable() { any() } -} - -/** - * A zero-width match, that is, either an empty group or an assertion. - * - * Examples: - * ``` - * () - * (?=\w) - * ``` - */ -class RegExpZeroWidthMatch extends RegExpGroup { - RegExpZeroWidthMatch() { re.zeroWidthMatch(start, end) } - - override RegExpTerm getChild(int i) { none() } - - override string getAPrimaryQlClass() { result = "RegExpZeroWidthMatch" } - - override predicate isNullable() { any() } -} - -/** - * A zero-width lookahead or lookbehind assertion. - * - * Examples: - * - * ``` - * (?=\w) - * (?!\n) - * (?<=\.) - * (?` - * in a regular expression. - * - * Examples: - * - * ``` - * \1 - * (?P=quote) - * ``` - */ -class RegExpBackRef extends RegExpTerm, TRegExpBackRef { - RegExpBackRef() { this = TRegExpBackRef(re, start, end) } + override string getAPrimaryQlClass() { result = "InfiniteRepetitionQuantifier" } + } /** - * Gets the number of the capture group this back reference refers to, if any. + * A star-quantified term. + * + * Example: + * + * ``` + * \w* + * ``` */ - int getNumber() { result = re.getBackRefNumber(start, end) } + class RegExpStar extends InfiniteRepetitionQuantifier { + RegExpStar() { this.getQualifier().charAt(0) = "*" } + + override string getAPrimaryQlClass() { result = "RegExpStar" } + + override predicate isNullable() { any() } + } /** - * Gets the name of the capture group this back reference refers to, if any. + * A plus-quantified term. + * + * Example: + * + * ``` + * \w+ + * ``` */ - string getName() { result = re.getBackRefName(start, end) } + class RegExpPlus extends InfiniteRepetitionQuantifier { + RegExpPlus() { this.getQualifier().charAt(0) = "+" } - /** Gets the capture group this back reference refers to. */ - RegExpGroup getGroup() { - result.getLiteral() = this.getLiteral() and + override string getAPrimaryQlClass() { result = "RegExpPlus" } + + override predicate isNullable() { this.getAChild().isNullable() } + } + + /** + * An optional term. + * + * Example: + * + * ``` + * ;? + * ``` + */ + class RegExpOpt extends RegExpQuantifier { + RegExpOpt() { this.getQualifier().charAt(0) = "?" } + + override string getAPrimaryQlClass() { result = "RegExpOpt" } + + override predicate isNullable() { any() } + } + + /** + * A range-quantified term + * + * Examples: + * + * ``` + * \w{2,4} + * \w{2,} + * \w{2} + * ``` + */ + class RegExpRange extends RegExpQuantifier { + string upper; + string lower; + + RegExpRange() { re.multiples(part_end, end, lower, upper) } + + override string getAPrimaryQlClass() { result = "RegExpRange" } + + /** Gets the string defining the upper bound of this range, if any. */ + string getUpper() { result = upper } + + /** Gets the string defining the lower bound of this range, if any. */ + string getLower() { result = lower } + + /** + * Gets the upper bound of the range, if any. + * + * If there is no upper bound, any number of repetitions is allowed. + * For a term of the form `r{lo}`, both the lower and the upper bound + * are `lo`. + */ + int getUpperBound() { result = this.getUpper().toInt() } + + /** Gets the lower bound of the range. */ + int getLowerBound() { result = this.getLower().toInt() } + + override predicate isNullable() { this.getAChild().isNullable() or this.getLowerBound() = 0 } + } + + /** + * A sequence term. + * + * Example: + * + * ``` + * (ECMA|Java)Script + * ``` + * + * This is a sequence with the elements `(ECMA|Java)` and `Script`. + */ + class RegExpSequence extends RegExpTerm, TRegExpSequence { + RegExpSequence() { + this = TRegExpSequence(re, start, end) and + exists(seqChild(re, start, end, 1)) // if a sequence does not have more than one element, it should be treated as that element instead. + } + + override RegExpTerm getChild(int i) { result = seqChild(re, start, end, i) } + + /** Gets the element preceding `element` in this sequence. */ + RegExpTerm previousElement(RegExpTerm element) { element = this.nextElement(result) } + + /** Gets the element following `element` in this sequence. */ + RegExpTerm nextElement(RegExpTerm element) { + exists(int i | + element = this.getChild(i) and + result = this.getChild(i + 1) + ) + } + + override string getConstantValue() { result = this.getConstantValue(0) } + + /** + * Gets the single string matched by the `i`th child and all following + * children of this sequence, if any. + */ + private string getConstantValue(int i) { + i = this.getNumChild() and + result = "" + or + result = this.getChild(i).getConstantValue() + this.getConstantValue(i + 1) + } + + override string getAPrimaryQlClass() { result = "RegExpSequence" } + + override predicate isNullable() { + forall(RegExpTerm child | child = this.getAChild() | child.isNullable()) + } + } + + pragma[nomagic] + private int seqChildEnd(RegExp re, int start, int end, int i) { + result = seqChild(re, start, end, i).getEnd() + } + + // moved out so we can use it in the charpred + private RegExpTerm seqChild(RegExp re, int start, int end, int i) { + re.sequence(start, end) and ( - result.getNumber() = this.getNumber() or - result.getName() = this.getName() + i = 0 and + result.getRegExp() = re and + result.getStart() = start and + exists(int itemEnd | + re.item(start, itemEnd) and + result.getEnd() = itemEnd + ) + or + i > 0 and + result.getRegExp() = re and + exists(int itemStart | itemStart = seqChildEnd(re, start, end, i - 1) | + result.getStart() = itemStart and + re.item(itemStart, result.getEnd()) + ) ) } - override RegExpTerm getChild(int i) { none() } + /** + * An alternative term, that is, a term of the form `a|b`. + * + * Example: + * + * ``` + * ECMA|Java + * ``` + */ + class RegExpAlt extends RegExpTerm, TRegExpAlt { + RegExpAlt() { this = TRegExpAlt(re, start, end) } - override string getAPrimaryQlClass() { result = "RegExpBackRef" } + override RegExpTerm getChild(int i) { + i = 0 and + result.getRegExp() = re and + result.getStart() = start and + exists(int part_end | + re.alternationOption(start, end, start, part_end) and + result.getEnd() = part_end + ) + or + i > 0 and + result.getRegExp() = re and + exists(int part_start | + part_start = this.getChild(i - 1).getEnd() + 1 // allow for the | + | + result.getStart() = part_start and + re.alternationOption(start, end, part_start, result.getEnd()) + ) + } - override predicate isNullable() { this.getGroup().isNullable() } -} + /** Gets an alternative of this term. */ + RegExpTerm getAlternative() { result = this.getAChild() } -/** - * A named character property. For example, the POSIX bracket expression - * `[[:digit:]]`. - */ -class RegExpNamedCharacterProperty extends RegExpTerm, TRegExpNamedCharacterProperty { - RegExpNamedCharacterProperty() { this = TRegExpNamedCharacterProperty(re, start, end) } + override string getAMatchedString() { result = this.getAlternative().getAMatchedString() } - override RegExpTerm getChild(int i) { none() } + override string getAPrimaryQlClass() { result = "RegExpAlt" } - override string getAPrimaryQlClass() { result = "RegExpNamedCharacterProperty" } + override predicate isNullable() { this.getAChild().isNullable() } + } + + class RegExpCharEscape = RegExpEscape; /** - * Gets the property name. For example, in `\p{Space}`, the result is - * `"Space"`. + * An escaped regular expression term, that is, a regular expression + * term starting with a backslash, which is not a backreference. + * + * Example: + * + * ``` + * \. + * \w + * ``` */ - string getName() { result = re.getCharacterPropertyName(start, end) } + class RegExpEscape extends RegExpNormalChar { + RegExpEscape() { re.escapedCharacter(start, end) } + + /** + * Gets the name of the escaped; for example, `w` for `\w`. + * TODO: Handle named escapes. + */ + override string getValue() { + not this.isUnicode() and + this.isIdentityEscape() and + result = this.getUnescaped() + or + this.getUnescaped() = "n" and result = "\n" + or + this.getUnescaped() = "r" and result = "\r" + or + this.getUnescaped() = "t" and result = "\t" + or + this.getUnescaped() = "f" and result = 12.toUnicode() + or + this.getUnescaped() = "v" and result = 11.toUnicode() + or + this.isUnicode() and + result = this.getUnicode() + } + + /** Holds if this terms name is given by the part following the escape character. */ + predicate isIdentityEscape() { + not this.getUnescaped() in ["n", "r", "t", "f", "v"] and not this.isUnicode() + } + + override string getAPrimaryQlClass() { result = "RegExpEscape" } + + /** Gets the part of the term following the escape character. That is e.g. "w" if the term is "\w". */ + string getUnescaped() { result = this.getText().suffix(1) } + + /** + * Gets the text for this escape. That is e.g. "\w". + */ + private string getText() { result = re.getText().substring(start, end) } + + /** + * Holds if this is a unicode escape. + */ + private predicate isUnicode() { this.getText().prefix(2) = ["\\u", "\\U"] } + + /** + * Gets the unicode char for this escape. + * E.g. for `\u0061` this returns "a". + */ + private string getUnicode() { + this.isUnicode() and + result = parseHexInt(this.getText().suffix(2)).toUnicode() + } + } /** - * Holds if the property is inverted. For example, it holds for `\p{^Digit}`, - * which matches non-digits. + * A word boundary, that is, a regular expression term of the form `\b`. */ - predicate isInverted() { re.namedCharacterPropertyIsInverted(start, end) } -} + class RegExpWordBoundary extends RegExpSpecialChar { + RegExpWordBoundary() { this.getChar() = "\\b" } -/** Gets the parse tree resulting from parsing `re`, if such has been constructed. */ -RegExpTerm getParsedRegExp(Ast::RegExpLiteral re) { - result.getRegExp() = re and result.isRootTerm() + override predicate isNullable() { none() } + } + + /** + * A non-word boundary, that is, a regular expression term of the form `\B`. + */ + class RegExpNonWordBoundary extends RegExpSpecialChar { + RegExpNonWordBoundary() { this.getChar() = "\\B" } + + override string getAPrimaryQlClass() { result = "RegExpNonWordBoundary" } + } + + /** + * A character class escape in a regular expression. + * That is, an escaped character that denotes multiple characters. + * + * Examples: + * + * ``` + * \w + * \S + * ``` + */ + class RegExpCharacterClassEscape extends RegExpEscape { + RegExpCharacterClassEscape() { this.getValue() in ["d", "D", "s", "S", "w", "W", "h", "H"] } + + override RegExpTerm getChild(int i) { none() } + + override string getAPrimaryQlClass() { result = "RegExpCharacterClassEscape" } + + override predicate isNullable() { none() } + } + + /** + * A character class in a regular expression. + * + * Examples: + * + * ```rb + * /[a-fA-F0-9]/ + * /[^abc]/ + * ``` + */ + class RegExpCharacterClass extends RegExpTerm, TRegExpCharacterClass { + RegExpCharacterClass() { this = TRegExpCharacterClass(re, start, end) } + + /** Holds if this character class is inverted, matching the opposite of its content. */ + predicate isInverted() { re.getChar(start + 1) = "^" } + + /** Holds if this character class can match anything. */ + predicate isUniversalClass() { + // [^] + this.isInverted() and not exists(this.getAChild()) + or + // [\w\W] and similar + not this.isInverted() and + exists(string cce1, string cce2 | + cce1 = this.getAChild().(RegExpCharacterClassEscape).getValue() and + cce2 = this.getAChild().(RegExpCharacterClassEscape).getValue() + | + cce1 != cce2 and cce1.toLowerCase() = cce2.toLowerCase() + ) + } + + override RegExpTerm getChild(int i) { + i = 0 and + result.getRegExp() = re and + exists(int itemStart, int itemEnd | + result.getStart() = itemStart and + re.charSetStart(start, itemStart) and + re.charSetChild(start, itemStart, itemEnd) and + result.getEnd() = itemEnd + ) + or + i > 0 and + result.getRegExp() = re and + exists(int itemStart | itemStart = this.getChild(i - 1).getEnd() | + result.getStart() = itemStart and + re.charSetChild(start, itemStart, result.getEnd()) + ) + } + + override string getAMatchedString() { + not this.isInverted() and result = this.getAChild().getAMatchedString() + } + + override string getAPrimaryQlClass() { result = "RegExpCharacterClass" } + + override predicate isNullable() { none() } + } + + /** + * A character range in a character class in a regular expression. + * + * Example: + * + * ``` + * a-z + * ``` + */ + class RegExpCharacterRange extends RegExpTerm, TRegExpCharacterRange { + int lower_end; + int upper_start; + + RegExpCharacterRange() { + this = TRegExpCharacterRange(re, start, end) and + re.charRange(_, start, lower_end, upper_start, end) + } + + /** Holds if this range goes from `lo` to `hi`, in effect is `lo-hi`. */ + predicate isRange(string lo, string hi) { + lo = re.getText().substring(start, lower_end) and + hi = re.getText().substring(upper_start, end) + } + + override RegExpTerm getChild(int i) { + i = 0 and + result.getRegExp() = re and + result.getStart() = start and + result.getEnd() = lower_end + or + i = 1 and + result.getRegExp() = re and + result.getStart() = upper_start and + result.getEnd() = end + } + + override string getAPrimaryQlClass() { result = "RegExpCharacterRange" } + + override predicate isNullable() { none() } + } + + /** + * A normal character in a regular expression, that is, a character + * without special meaning. This includes escaped characters. + * + * Examples: + * ``` + * t + * \t + * ``` + */ + class RegExpNormalChar extends RegExpTerm, TRegExpNormalChar { + RegExpNormalChar() { this = TRegExpNormalChar(re, start, end) } + + /** + * Holds if this constant represents a valid Unicode character (as opposed + * to a surrogate code point that does not correspond to a character by itself.) + */ + predicate isCharacter() { any() } + + /** Gets the string representation of the char matched by this term. */ + string getValue() { result = re.getText().substring(start, end) } + + override RegExpTerm getChild(int i) { none() } + + override string getAPrimaryQlClass() { result = "RegExpNormalChar" } + } + + /** + * A constant regular expression term, that is, a regular expression + * term matching a single string. Currently, this will always be a single character. + * + * Example: + * + * ``` + * a + * ``` + */ + class RegExpConstant extends RegExpTerm { + string value; + + RegExpConstant() { + this = TRegExpNormalChar(re, start, end) and + not this instanceof RegExpCharacterClassEscape and + // exclude chars in qualifiers + // TODO: push this into regex library + not exists(int qstart, int qend | re.qualifiedPart(_, qstart, qend, _, _) | + qstart <= start and end <= qend + ) and + value = this.(RegExpNormalChar).getValue() + or + this = TRegExpSpecialChar(re, start, end) and + re.inCharSet(start) and + value = this.(RegExpSpecialChar).getChar() + } + + /** + * Holds if this constant represents a valid Unicode character (as opposed + * to a surrogate code point that does not correspond to a character by itself.) + */ + predicate isCharacter() { any() } + + /** Gets the string matched by this constant term. */ + string getValue() { result = value } + + override RegExpTerm getChild(int i) { none() } + + override string getConstantValue() { result = this.getValue() } + + override string getAPrimaryQlClass() { result = "RegExpConstant" } + + override predicate isNullable() { none() } + } + + /** + * A grouped regular expression. + * + * Examples: + * + * ``` + * (ECMA|Java) + * (?:ECMA|Java) + * (?['"]) + * ``` + */ + class RegExpGroup extends RegExpTerm, TRegExpGroup { + RegExpGroup() { this = TRegExpGroup(re, start, end) } + + /** + * Gets the index of this capture group within the enclosing regular + * expression literal. + * + * For example, in the regular expression `/((a?).)(?:b)/`, the + * group `((a?).)` has index 1, the group `(a?)` nested inside it + * has index 2, and the group `(?:b)` has no index, since it is + * not a capture group. + */ + int getNumber() { result = re.getGroupNumber(start, end) } + + /** Holds if this is a capture group. */ + predicate isCapture() { exists(this.getNumber()) } + + /** Holds if this is a named capture group. */ + predicate isNamed() { exists(this.getName()) } + + /** Gets the name of this capture group, if any. */ + string getName() { result = re.getGroupName(start, end) } + + override RegExpTerm getChild(int i) { + result.getRegExp() = re and + i = 0 and + re.groupContents(start, end, result.getStart(), result.getEnd()) + } + + override string getConstantValue() { result = this.getAChild().getConstantValue() } + + override string getAMatchedString() { result = this.getAChild().getAMatchedString() } + + override string getAPrimaryQlClass() { result = "RegExpGroup" } + + override predicate isNullable() { this.getAChild().isNullable() } + } + + /** + * A special character in a regular expression. + * + * Examples: + * ``` + * ^ + * $ + * . + * ``` + */ + class RegExpSpecialChar extends RegExpTerm, TRegExpSpecialChar { + string char; + + RegExpSpecialChar() { + this = TRegExpSpecialChar(re, start, end) and + re.specialCharacter(start, end, char) + } + + /** + * Holds if this constant represents a valid Unicode character (as opposed + * to a surrogate code point that does not correspond to a character by itself.) + */ + predicate isCharacter() { any() } + + /** Gets the char for this term. */ + string getChar() { result = char } + + override RegExpTerm getChild(int i) { none() } + + override string getAPrimaryQlClass() { result = "RegExpSpecialChar" } + } + + /** + * A dot regular expression. + * + * Example: + * + * ``` + * . + * ``` + */ + class RegExpDot extends RegExpSpecialChar { + RegExpDot() { this.getChar() = "." } + + override string getAPrimaryQlClass() { result = "RegExpDot" } + + override predicate isNullable() { none() } + } + + /** + * A term that matches a specific position between characters in the string. + * + * Example: + * + * ``` + * \A + * ``` + */ + class RegExpAnchor extends RegExpSpecialChar { + RegExpAnchor() { this.getChar() = ["^", "$", "\\A", "\\Z", "\\z"] } + + override string getAPrimaryQlClass() { result = "RegExpAnchor" } + } + + /** + * A dollar assertion `$` or `\Z` matching the end of a line. + * + * Example: + * + * ``` + * $ + * ``` + */ + class RegExpDollar extends RegExpAnchor { + RegExpDollar() { this.getChar() = ["$", "\\Z", "\\z"] } + + override string getAPrimaryQlClass() { result = "RegExpDollar" } + + override predicate isNullable() { any() } + } + + /** + * A caret assertion `^` or `\A` matching the beginning of a line. + * + * Example: + * + * ``` + * ^ + * ``` + */ + class RegExpCaret extends RegExpAnchor { + RegExpCaret() { this.getChar() = ["^", "\\A"] } + + override string getAPrimaryQlClass() { result = "RegExpCaret" } + + override predicate isNullable() { any() } + } + + /** + * A zero-width match, that is, either an empty group or an assertion. + * + * Examples: + * ``` + * () + * (?=\w) + * ``` + */ + class RegExpZeroWidthMatch extends RegExpGroup { + RegExpZeroWidthMatch() { re.zeroWidthMatch(start, end) } + + override RegExpTerm getChild(int i) { none() } + + override string getAPrimaryQlClass() { result = "RegExpZeroWidthMatch" } + + override predicate isNullable() { any() } + } + + /** + * A zero-width lookahead or lookbehind assertion. + * + * Examples: + * + * ``` + * (?=\w) + * (?!\n) + * (?<=\.) + * (?` + * in a regular expression. + * + * Examples: + * + * ``` + * \1 + * (?P=quote) + * ``` + */ + class RegExpBackRef extends RegExpTerm, TRegExpBackRef { + RegExpBackRef() { this = TRegExpBackRef(re, start, end) } + + /** + * Gets the number of the capture group this back reference refers to, if any. + */ + int getNumber() { result = re.getBackRefNumber(start, end) } + + /** + * Gets the name of the capture group this back reference refers to, if any. + */ + string getName() { result = re.getBackRefName(start, end) } + + /** Gets the capture group this back reference refers to. */ + RegExpGroup getGroup() { + result.getLiteral() = this.getLiteral() and + ( + result.getNumber() = this.getNumber() or + result.getName() = this.getName() + ) + } + + override RegExpTerm getChild(int i) { none() } + + override string getAPrimaryQlClass() { result = "RegExpBackRef" } + + override predicate isNullable() { this.getGroup().isNullable() } + } + + /** + * A named character property. For example, the POSIX bracket expression + * `[[:digit:]]`. + */ + class RegExpNamedCharacterProperty extends RegExpTerm, TRegExpNamedCharacterProperty { + RegExpNamedCharacterProperty() { this = TRegExpNamedCharacterProperty(re, start, end) } + + override RegExpTerm getChild(int i) { none() } + + override string getAPrimaryQlClass() { result = "RegExpNamedCharacterProperty" } + + /** + * Gets the property name. For example, in `\p{Space}`, the result is + * `"Space"`. + */ + string getName() { result = re.getCharacterPropertyName(start, end) } + + /** + * Holds if the property is inverted. For example, it holds for `\p{^Digit}`, + * which matches non-digits. + */ + predicate isInverted() { re.namedCharacterPropertyIsInverted(start, end) } + } } From 3432e814c5b9f738be8ced3f2703e860699cdeb2 Mon Sep 17 00:00:00 2001 From: erik-krogh Date: Tue, 1 Nov 2022 11:43:15 +0100 Subject: [PATCH 008/106] add a Ruby implementation of `RegexTreeViewSig` --- .../lib/codeql/ruby/regexp/RegExpTreeView.qll | 69 ++++++++++++++++++- 1 file changed, 68 insertions(+), 1 deletion(-) diff --git a/ruby/ql/lib/codeql/ruby/regexp/RegExpTreeView.qll b/ruby/ql/lib/codeql/ruby/regexp/RegExpTreeView.qll index fffa0044771..ab21512c694 100644 --- a/ruby/ql/lib/codeql/ruby/regexp/RegExpTreeView.qll +++ b/ruby/ql/lib/codeql/ruby/regexp/RegExpTreeView.qll @@ -4,6 +4,10 @@ private import internal.ParseRegExp private import codeql.NumberUtils private import codeql.ruby.ast.Literal as Ast private import codeql.Locations +private import codeql.regex.nfa.NfaUtils as NfaUtils +private import codeql.regex.RegexTreeView +// exporting as RegexTreeView, and in the top-level scope. +import Impl as RegexTreeView import Impl /** Gets the parse tree resulting from parsing `re`, if such has been constructed. */ @@ -52,7 +56,7 @@ private newtype TRegExpParent = } /** An implementation that statisfies the RegexTreeView signature. */ -private module Impl { +private module Impl implements RegexTreeViewSig { /** * An element containing a regular expression term, that is, either * a string literal (parsed as a regular expression) @@ -1157,4 +1161,67 @@ private module Impl { */ predicate isInverted() { re.namedCharacterPropertyIsInverted(start, end) } } + + class Top = RegExpParent; + + /** + * Holds if `term` is an escape class representing e.g. `\d`. + * `clazz` is which character class it represents, e.g. "d" for `\d`. + */ + predicate isEscapeClass(RegExpTerm term, string clazz) { + exists(RegExpCharacterClassEscape escape | term = escape | escape.getValue() = clazz) + or + // TODO: expand to cover more properties + exists(RegExpNamedCharacterProperty escape | term = escape | + escape.getName().toLowerCase() = "digit" and + if escape.isInverted() then clazz = "D" else clazz = "d" + or + escape.getName().toLowerCase() = "space" and + if escape.isInverted() then clazz = "S" else clazz = "s" + or + escape.getName().toLowerCase() = "word" and + if escape.isInverted() then clazz = "W" else clazz = "w" + ) + } + + /** + * Holds if the regular expression should not be considered. + */ + predicate isExcluded(RegExpParent parent) { + parent.(RegExpTerm).getRegExp().(Ast::RegExpLiteral).hasFreeSpacingFlag() // exclude free-spacing mode regexes + } + + /** + * Holds if `term` is a possessive quantifier. + * Not currently implemented, but is used by the shared library. + */ + predicate isPossessive(RegExpQuantifier term) { none() } + + /** + * Holds if the regex that `term` is part of is used in a way that ignores any leading prefix of the input it's matched against. + * Not yet implemented for Ruby. + */ + predicate matchesAnyPrefix(RegExpTerm term) { any() } + + /** + * Holds if the regex that `term` is part of is used in a way that ignores any trailing suffix of the input it's matched against. + * Not yet implemented for Ruby. + */ + predicate matchesAnySuffix(RegExpTerm term) { any() } + + /** + * Holds if `root` has the `i` flag for case-insensitive matching. + */ + predicate isIgnoreCase(RegExpTerm root) { + root.isRootTerm() and + root.getLiteral().isIgnoreCase() + } + + /** + * Holds if `root` has the `s` flag for multi-line matching. + */ + predicate isDotAll(RegExpTerm root) { + root.isRootTerm() and + root.getLiteral().isDotAll() + } } From 40e43591731d6edf0a826b6631ddab923f8581a2 Mon Sep 17 00:00:00 2001 From: erik-krogh Date: Tue, 1 Nov 2022 11:47:32 +0100 Subject: [PATCH 009/106] port the Ruby regex/redos queries to use the shared pack --- .../ruby/security/BadTagFilterQuery.qll | 156 +- ...leteMultiCharacterSanitizationSpecific.qll | 5 +- .../ruby/security/OverlyLargeRangeQuery.qll | 289 +--- .../regexp/ExponentialBackTracking.qll | 285 +--- .../codeql/ruby/security/regexp/NfaUtils.qll | 1333 +---------------- .../ruby/security/regexp/NfaUtilsSpecific.qll | 73 - .../regexp/PolynomialReDoSCustomizations.qll | 6 +- .../ruby/security/regexp/RegexpMatching.qll | 156 +- .../regexp/SuperlinearBackTracking.qll | 385 +---- .../security/cwe-020/OverlyLargeRange.ql | 11 +- .../queries/security/cwe-116/BadTagFilter.ql | 3 +- .../security/cwe-1333/PolynomialReDoS.ql | 3 +- .../ql/src/queries/security/cwe-1333/ReDoS.ql | 7 +- 13 files changed, 43 insertions(+), 2669 deletions(-) delete mode 100644 ruby/ql/lib/codeql/ruby/security/regexp/NfaUtilsSpecific.qll diff --git a/ruby/ql/lib/codeql/ruby/security/BadTagFilterQuery.qll b/ruby/ql/lib/codeql/ruby/security/BadTagFilterQuery.qll index 95bfbeeeb5d..370baaf3a5d 100644 --- a/ruby/ql/lib/codeql/ruby/security/BadTagFilterQuery.qll +++ b/ruby/ql/lib/codeql/ruby/security/BadTagFilterQuery.qll @@ -2,155 +2,7 @@ * Provides predicates for reasoning about bad tag filter vulnerabilities. */ -import regexp.RegexpMatching - -/** - * Holds if the regexp `root` should be tested against `str`. - * Implements the `isRegexpMatchingCandidateSig` signature from `RegexpMatching`. - * `ignorePrefix` toggles whether the regular expression should be treated as accepting any prefix if it's unanchored. - * `testWithGroups` toggles whether it's tested which groups are filled by a given input string. - */ -private predicate isBadTagFilterCandidate( - RootTerm root, string str, boolean ignorePrefix, boolean testWithGroups -) { - // the regexp must mention "<" and ">" explicitly. - forall(string angleBracket | angleBracket = ["<", ">"] | - any(RegExpConstant term | term.getValue().matches("%" + angleBracket + "%")).getRootTerm() = - root - ) and - ignorePrefix = true and - ( - str = ["", "", "", "", "", - "", "", "", "", - "", "", - "", "", "", - "", "", "", - "", "") and - regexp.matches("") and - not regexp.matches("") and - ( - not regexp.matches("") and - msg = "This regular expression matches , but not " - or - not regexp.matches("") and - msg = "This regular expression matches , but not " - ) - or - regexp.matches("") and - regexp.matches("") and - not regexp.matches("") and - not regexp.matches("") and - msg = "This regular expression does not match script tags where the attribute uses single-quotes." - or - regexp.matches("") and - regexp.matches("") and - not regexp.matches("") and - not regexp.matches("") and - msg = "This regular expression does not match script tags where the attribute uses double-quotes." - or - regexp.matches("") and - regexp.matches("") and - not regexp.matches("") and - not regexp.matches("") and - not regexp.matches("") and - msg = "This regular expression does not match script tags where tabs are used between attributes." - or - regexp.matches("") and - not RegExpFlags::isIgnoreCase(regexp) and - not regexp.matches("") and - not regexp.matches("") and - ( - not regexp.matches("") and - msg = "This regular expression does not match upper case ") and - regexp.matches("") and - msg = "This regular expression does not match mixed case ") and - not regexp.matches("") and - not regexp.matches("") and - ( - not regexp.matches("") and - msg = "This regular expression does not match script end tags like ." - or - not regexp.matches("") and - msg = "This regular expression does not match script end tags like ." - or - not regexp.matches("