Merge branch 'main' into revert-import-change

cpp/ql/test/library-tests/dataflow/smart-pointers-taint/memory.h

@@ -0,0 +1,127 @@
+
+namespace std {
+  namespace detail {
+    template<typename T>
+    class compressed_pair_element {
+      T element;
+
+    public:
+      compressed_pair_element() = default;
+      compressed_pair_element(const T& t) : element(t) {}
+      
+      T& get() { return element; }
+
+      const T& get() const { return element; }
+    };
+
+    template<typename T, typename U>
+    struct compressed_pair : private compressed_pair_element<T>, private compressed_pair_element<U> {
+      compressed_pair() = default;
+      compressed_pair(T& t) : compressed_pair_element<T>(t), compressed_pair_element<U>() {}
+      compressed_pair(const compressed_pair&) = delete;
+      compressed_pair(compressed_pair<T, U>&&) noexcept = default;
+
+      T& first() { return static_cast<compressed_pair_element<T>&>(*this).get(); }
+      U& second() { return static_cast<compressed_pair_element<U>&>(*this).get(); }
+
+      const T& first() const { return static_cast<const compressed_pair_element<T>&>(*this).get(); }
+      const U& second() const { return static_cast<const compressed_pair_element<U>&>(*this).get(); }
+    };
+  }
+
+  template<class T>
+  struct default_delete {
+    void operator()(T* ptr) const { delete ptr; }
+  };
+
+  template<class T>
+  struct default_delete<T[]> {
+    template<class U>
+    void operator()(U* ptr) const { delete[] ptr; }
+  };
+
+  template<class T, class Deleter = default_delete<T> >
+  class unique_ptr {
+  private:
+    detail::compressed_pair<T*, Deleter> data;
+  public:
+    constexpr unique_ptr() noexcept {}
+    explicit unique_ptr(T* ptr) noexcept : data(ptr) {}
+    unique_ptr(const unique_ptr& ptr) = delete;
+    unique_ptr(unique_ptr&& ptr) noexcept = default;
+
+    unique_ptr& operator=(unique_ptr&& ptr) noexcept = default;
+
+    T& operator*() const { return *get(); }
+    T* operator->() const noexcept { return get(); }
+
+    T* get() const noexcept { return data.first(); }
+
+    ~unique_ptr() {
+      Deleter& d = data.second();
+      d(data.first());
+    }
+  };
+  
+  template<typename T, class... Args> unique_ptr<T> make_unique(Args&&... args) {
+    return unique_ptr<T>(new T(args...)); // std::forward calls elided for simplicity.
+  }
+
+  class ctrl_block {
+    unsigned uses;
+
+  public:
+    ctrl_block() : uses(1) {}
+
+    void inc() { ++uses; }
+    bool dec() { return --uses == 0; }
+
+    virtual void destroy() = 0;
+    virtual ~ctrl_block() {}
+  };
+
+  template<typename T, class Deleter = default_delete<T> >
+  struct ctrl_block_impl: public ctrl_block {
+    T* ptr;
+    Deleter d;
+
+    ctrl_block_impl(T* ptr, Deleter d) : ptr(ptr), d(d) {}
+    virtual void destroy() override { d(ptr); }
+  };
+
+  template<class T>
+  class shared_ptr {
+  private:
+    ctrl_block* ctrl;
+    T* ptr;
+
+    void dec() {
+      if(ctrl->dec()) {
+        ctrl->destroy();
+        delete ctrl;
+      }
+    }
+
+    void inc() {
+      ctrl->inc();
+    }
+
+  public:
+    constexpr shared_ptr() noexcept = default;
+    shared_ptr(T* ptr) : ctrl(new ctrl_block_impl<T>(ptr, default_delete<T>())) {}
+    shared_ptr(const shared_ptr& s) noexcept : ptr(s.ptr), ctrl(s.ctrl) {
+      inc();
+    }
+    shared_ptr(shared_ptr&& s) noexcept = default;
+
+    T* operator->() const { return ptr; }
+
+    T& operator*() const { return *ptr; }
+
+    ~shared_ptr() { dec(); }
+  };
+
+  template<typename T, class... Args> shared_ptr<T> make_shared(Args&&... args) {
+    return shared_ptr<T>(new T(args...)); // std::forward calls elided for simplicity.
+  }
+}

cpp/ql/test/library-tests/dataflow/smart-pointers-taint/taint.ql

@@ -0,0 +1,39 @@
+import TestUtilities.dataflow.FlowTestCommon
+
+module ASTTest {
+  private import semmle.code.cpp.dataflow.TaintTracking
+
+  class ASTSmartPointerTaintConfig extends TaintTracking::Configuration {
+    ASTSmartPointerTaintConfig() { this = "ASTSmartPointerTaintConfig" }
+
+    override predicate isSource(DataFlow::Node source) {
+      source.asExpr().(FunctionCall).getTarget().getName() = "source"
+    }
+
+    override predicate isSink(DataFlow::Node sink) {
+      exists(FunctionCall call |
+        call.getTarget().getName() = "sink" and
+        sink.asExpr() = call.getAnArgument()
+      )
+    }
+  }
+}
+
+module IRTest {
+  private import semmle.code.cpp.ir.dataflow.TaintTracking
+
+  class IRSmartPointerTaintConfig extends TaintTracking::Configuration {
+    IRSmartPointerTaintConfig() { this = "IRSmartPointerTaintConfig" }
+
+    override predicate isSource(DataFlow::Node source) {
+      source.asExpr().(FunctionCall).getTarget().getName() = "source"
+    }
+
+    override predicate isSink(DataFlow::Node sink) {
+      exists(FunctionCall call |
+        call.getTarget().getName() = "sink" and
+        sink.asExpr() = call.getAnArgument()
+      )
+    }
+  }
+}

cpp/ql/test/library-tests/dataflow/smart-pointers-taint/test.cpp

@@ -0,0 +1,46 @@
+#include "memory.h"
+
+int source();
+void sink(int);
+
+void test_unique_ptr_int() {
+	std::unique_ptr<int> p1(new int(source()));
+	std::unique_ptr<int> p2 = std::make_unique<int>(source());
+	
+	sink(*p1); // $ MISSING: ast,ir
+	sink(*p2); // $ ast ir=8:50
+}
+
+struct A {
+	int x, y;
+
+	A(int x, int y) : x(x), y(y) {}
+};
+
+void test_unique_ptr_struct() {
+	std::unique_ptr<A> p1(new A{source(), 0});
+	std::unique_ptr<A> p2 = std::make_unique<A>(source(), 0);
+	
+	sink(p1->x); // $ MISSING: ast,ir
+	sink(p1->y);
+	sink(p2->x); // $ MISSING: ast,ir
+	sink(p2->y);
+}
+
+void test_shared_ptr_int() {
+	std::shared_ptr<int> p1(new int(source()));
+	std::shared_ptr<int> p2 = std::make_shared<int>(source());
+	
+	sink(*p1); // $ ast
+	sink(*p2); // $ ast ir=32:50 
+}
+
+void test_shared_ptr_struct() {
+	std::shared_ptr<A> p1(new A{source(), 0});
+	std::shared_ptr<A> p2 = std::make_shared<A>(source(), 0);
+	
+	sink(p1->x); // $ MISSING: ast,ir
+	sink(p1->y);
+	sink(p2->x); // $ MISSING: ast,ir
+	sink(p2->y);
+}

cpp/ql/test/query-tests/Critical/MemoryFreed/MemoryNeverFreed.expected

@@ -10,3 +10,4 @@
 | test.cpp:89:18:89:23 | call to malloc | This memory is never freed |
 | test.cpp:156:3:156:26 | new | This memory is never freed |
 | test.cpp:157:3:157:26 | new[] | This memory is never freed |
+| test.cpp:167:14:167:19 | call to strdup | This memory is never freed |

cpp/ql/test/query-tests/Critical/MemoryFreed/test.cpp

@@ -156,3 +156,15 @@ int overloadedNew() {
   new(std::nothrow) int(3); // BAD
   new(std::nothrow) int[2]; // BAD
 }
+
+// --- strdup ---
+
+char *strdup(const char *s1);
+void output_msg(const char *msg);
+
+void test_strdup() {
+	char msg[] = "OctoCat";
+	char *cpy = strdup(msg); // BAD
+
+	output_msg(cpy);
+}

cpp/ql/test/query-tests/Security/CWE/CWE-764/semmle/tests/UnreleasedLock.expected

@@ -7,3 +7,4 @@
 | test.cpp:303:11:303:18 | call to try_lock | This lock might not be unlocked or might be locked more times than it is unlocked. |
 | test.cpp:313:11:313:18 | call to try_lock | This lock might not be unlocked or might be locked more times than it is unlocked. |
 | test.cpp:442:8:442:17 | call to mutex_lock | This lock might not be unlocked or might be locked more times than it is unlocked. |
+| test.cpp:482:2:482:19 | call to pthread_mutex_lock | This lock might not be unlocked or might be locked more times than it is unlocked. |

cpp/ql/test/query-tests/Security/CWE/CWE-764/semmle/tests/test.cpp

@@ -445,3 +445,46 @@ bool test_mutex(data_t *data)
 
 	return true;
 }
+
+// ---
+
+struct pthread_mutex
+{
+	// ...
+};
+
+void pthread_mutex_lock(pthread_mutex *m);
+void pthread_mutex_unlock(pthread_mutex *m);
+
+class MyClass
+{
+public:
+	pthread_mutex lock;
+};
+
+bool maybe();
+
+int test_MyClass_good(MyClass *obj)
+{
+	pthread_mutex_lock(&obj->lock);
+	
+	if (maybe()) {
+		pthread_mutex_unlock(&obj->lock);
+		return -1; // GOOD
+	}
+
+	pthread_mutex_unlock(&obj->lock); // GOOD
+	return 0;
+}
+
+int test_MyClass_bad(MyClass *obj)
+{
+	pthread_mutex_lock(&obj->lock);
+
+	if (maybe()) {
+		return -1; // BAD
+	}
+
+	pthread_mutex_unlock(&obj->lock); // GOOD
+	return 0;
+}

cpp/ql/test/query-tests/jsf/4.10 Classes/AV Rule 79/AV Rule 79.expected

@@ -18,9 +18,10 @@
 | NoDestructor.cpp:23:3:23:20 | ... = ... | Resource n is acquired by class MyClass5 but not released anywhere in this class. |
 | PlacementNew.cpp:36:3:36:36 | ... = ... | Resource p1 is acquired by class MyTestForPlacementNew but not released anywhere in this class. |
 | SelfRegistering.cpp:25:3:25:24 | ... = ... | Resource side is acquired by class MyOwner but not released anywhere in this class. |
-| Variants.cpp:25:3:25:13 | ... = ... | Resource f is acquired by class MyClass4 but not released anywhere in this class. |
-| Variants.cpp:65:3:65:17 | ... = ... | Resource a is acquired by class MyClass6 but not released anywhere in this class. |
-| Variants.cpp:66:3:66:36 | ... = ... | Resource b is acquired by class MyClass6 but not released anywhere in this class. |
-| Variants.cpp:67:3:67:41 | ... = ... | Resource c is acquired by class MyClass6 but not released anywhere in this class. |
+| Variants.cpp:26:3:26:13 | ... = ... | Resource f is acquired by class MyClass4 but not released anywhere in this class. |
+| Variants.cpp:69:3:69:17 | ... = ... | Resource a is acquired by class MyClass6 but not released anywhere in this class. |
+| Variants.cpp:70:3:70:36 | ... = ... | Resource b is acquired by class MyClass6 but not released anywhere in this class. |
+| Variants.cpp:71:3:71:41 | ... = ... | Resource c is acquired by class MyClass6 but not released anywhere in this class. |
+| Variants.cpp:72:3:72:22 | ... = ... | Resource d is acquired by class MyClass6 but not released anywhere in this class. |
 | Wrapped.cpp:46:3:46:22 | ... = ... | Resource ptr2 is acquired by class Wrapped2 but not released anywhere in this class. |
 | Wrapped.cpp:59:3:59:22 | ... = ... | Resource ptr4 is acquired by class Wrapped2 but not released anywhere in this class. |

cpp/ql/test/query-tests/jsf/4.10 Classes/AV Rule 79/Variants.cpp

@@ -5,6 +5,7 @@ void *malloc(size_t size);
 void *calloc(size_t nmemb, size_t size);
 void *realloc(void *ptr, size_t size);
 void free(void* ptr);
+char *strdup(const char *s1);
 
 int *ID(int *x) 
 {
@@ -45,6 +46,7 @@ public:
 		a = new int[10]; // GOOD
 		b = (int *)calloc(10, sizeof(int)); // GOOD
 		c = (int *)realloc(0, 10 * sizeof(int)); // GOOD
+		d = strdup("string");
 	}
 	
 	~MyClass5()
@@ -52,9 +54,11 @@ public:
 		delete [] a;
 		free(b);
 		free(c);
+		free(d);
 	}
 
 	int *a, *b, *c;
+	char *d;
 };
 
 class MyClass6
@@ -65,6 +69,7 @@ public:
 		a = new int[10]; // BAD
 		b = (int *)calloc(10, sizeof(int)); // BAD
 		c = (int *)realloc(0, 10 * sizeof(int)); // BAD
+		d = strdup("string"); // BAD
 	}
 	
 	~MyClass6()
@@ -72,6 +77,7 @@ public:
 	}
 
 	int *a, *b, *c;
+	char *d;
 };
 
 class MyClass7

docs/codeql/codeql-cli/about-ql-packs.rst

@@ -79,7 +79,7 @@ The following properties are supported in ``qlpack.yml`` files.
    * - ``version``
      - ``0.0.0``
      - All packs
-     - A version number for this QL pack. This field is not currently used by any commands, but may be required by future releases of CodeQL.
+     - A version number for this QL pack. This must be a valid semantic version that meets the `SemVer v2.0.0 specification <https://semver.org/spec/v2.0.0.html>`__.
    * - ``libraryPathDependencies``
      - ``codeql-javascript``
      - Optional

docs/codeql/codeql-cli/creating-codeql-databases.rst

@@ -168,7 +168,7 @@ build steps, you may need to explicitly define each step in the command line.
    For Go, you should always use the CodeQL autobuilder. Install the Go
    toolchain (version 1.11 or later) and, if there are dependencies, the
    appropriate dependency manager (such as `dep
-   <https://golang.github.io/dep/>`__ or `Glide <http://glide.sh/>`__).
+   <https://golang.github.io/dep/>`__).
    
    Do not specify any build commands, as you will override the autobuilder
    invocation, which will create an empty database.  

docs/codeql/codeql-cli/getting-started-with-the-codeql-cli.rst

@@ -121,24 +121,24 @@ see ":doc:`About QL packs <about-ql-packs>`."
 
    There are different versions of the CodeQL queries available for different
    users. Check out the correct version for your use case:
+
+   - For the queries used on `LGTM.com <https://lgtm.com>`__, check out the
+     ``lgtm.com`` branch. You should use this branch for databases you've built
+     using the CodeQL CLI, fetched from code scanning on GitHub, or recently downloaded from LGTM.com.
+     The queries on the ``lgtm.com`` branch are more likely to be compatible 
+     with the ``latest`` CLI, so you'll be less likely to have to upgrade 
+     newly-created databases than if you use the ``main`` branch. Older databases 
+     may need to be upgraded before you can analyze them.
    
    - For the most up to date CodeQL queries, check out the ``main`` branch. 
      This branch represents the very latest version of CodeQL's analysis. Even
      databases created using the most recent version of the CLI may have to be
      upgraded before you can analyze them. For more information, see
      ":doc:`Upgrading CodeQL databases <upgrading-codeql-databases>`."
-    
-   - For the queries used on `LGTM.com <https://lgtm.com>`__, check out the
-     ``lgtm.com`` branch. You can run these queries on databases you've recently
-     downloaded from LGTM.com. Older databases may need to be upgraded before
-     you can analyze them. The queries on the ``lgtm.com`` branch are also more
-     likely to be compatible with the ``latest`` CLI, so you'll be less likely
-     to have to upgrade newly-created databases than if you use the ``main``
-     branch.
-           
+        
    - For the queries used in a particular LGTM Enterprise release, check out the
      branch tagged with the relevant release number. For example, the branch
-     tagged ``v1.23.0`` corresponds to LGTM Enterprise 1.23. You must use this
+     tagged ``v1.27.0`` corresponds to LGTM Enterprise 1.27. You must use this
      version if you want to upload data to LGTM Enterprise. For further
      information, see `Preparing CodeQL databases to upload to LGTM 
      <https://help.semmle.com/lgtm-enterprise/admin/help/prepare-database-upload.html>`__

docs/codeql/codeql-cli/testing-custom-queries.rst

@@ -13,20 +13,6 @@ on the query and the expected results until the actual results and the expected
 results exactly match. This topic shows you how to create test files and execute
 tests on them using the ``test run`` subcommand.
 
-.. container:: toggle
-
-   .. container:: name
-
-      **Information for LGTM Enterprise 1.23 users**
-
-   CodeQL tests are a new feature in CodeQL CLI version 2.0.2.
-   
-   If you are an LGTM Enterprise 1.23 user, you should still use CodeQL CLI
-   version 2.0.1 to prepare databases to upload to your instance of LGTM. You
-   can use version 2.0.2 (or newer) to test your custom queries, but databases
-   created with versions newer than 2.0.1 are not compatible with LGTM
-   Enterprise 1.23. For more information, see ":ref:`Getting started with the CodeQL CLI <using-two-versions-of-the-codeql-cli>`."
-
 Setting up a test QL pack for custom queries
 --------------------------------------------
 

docs/codeql/codeql-cli/testing-query-help-files.rst

@@ -38,7 +38,7 @@ where ``<qhelp|query|dir|suite>`` is one of:
 - the path to a ``.ql`` file.
 - the path to a directory containing queries and query help files.
 - the path to a query suite, or the name of a well-known query suite for a QL pack. 
-  For more information, see "`Creating CodeQL query suites <creating-codeql-query-suites.html#specifying-well-known-query-suites>`__."
+  For more information, see "`Creating CodeQL query suites <https://codeql.github.com/docs/codeql-cli/creating-codeql-query-suites#specifying-well-known-query-suites>`__."
 
 You must specify a ``--format`` option, which defines how the query help is rendered. 
 Currently, you must specify ``markdown`` to render the query help as markdown. 

docs/codeql/codeql-for-visual-studio-code/analyzing-your-projects.rst

@@ -14,7 +14,7 @@ To analyze a project, you need to add a :ref:`CodeQL database <codeql-database>`
 
 #. Open the CodeQL Databases view in the sidebar.
 
-#. Hover over the **Databases** title bar and click the appropriate icon to add your database. You can add a database from a local ZIP archive or folder, from a public URL, or from a project URL on LGTM.com.
+#. Hover over the **Databases** title bar and click the appropriate icon to add your database. You can add a database from a local ZIP archive or folder, from a public URL, or from a project slug or URL on LGTM.com.
 
    .. image:: ../images/codeql-for-visual-studio-code/choose-database.png
       :width: 350

docs/codeql/codeql-for-visual-studio-code/setting-up-codeql-in-visual-studio-code.rst

@@ -64,7 +64,7 @@ There are two ways to do this:
       **Click to show information for LGTM Enterprise users**
 
    Your local version of the CodeQL queries and libraries should match your version of LGTM Enterprise. For example, if you
-   use LGTM Enterprise 1.23, then you should clone the ``1.23.0`` branch of the `starter workspace <https://github.com/github/vscode-codeql-starter/>`__ (or the appropriate ``1.23.x`` branch, corresponding to each maintenance release).
+   use LGTM Enterprise 1.27, then you should clone the ``1.27.0`` branch of the `starter workspace <https://github.com/github/vscode-codeql-starter/>`__ (or the appropriate ``1.27.x`` branch, corresponding to each maintenance release).
    
    This ensures that the queries and libraries you write in VS Code also work in the query console on LGTM Enterprise.
 

docs/codeql/ql-language-reference/annotations.rst

@@ -385,21 +385,23 @@ For more information, see ":ref:`monotonic-aggregates`."
 Binding sets
 ============
 
-**Available for**: |characteristic predicates|, |member predicates|, |non-member predicates|
+**Available for**: |classes|, |characteristic predicates|, |member predicates|, |non-member predicates|
 
 ``bindingset[...]``
 -------------------
 
-You can use this annotation to explicitly state the binding sets for a predicate. A binding set
-is a subset of the predicate's arguments such that, if those arguments are constrained to a
-finite set of values, then the predicate itself is finite (that is, it evaluates to a finite 
+You can use this annotation to explicitly state the binding sets for a predicate or class. A binding set
+is a subset of a predicate's or class body's arguments such that, if those arguments are constrained to a
+finite set of values, then the predicate or class itself is finite (that is, it evaluates to a finite 
 set of tuples).
 
-The ``bindingset`` annotation takes a comma-separated list of variables. Each variable must be
-an argument of the predicate, possibly including ``this`` (for characteristic predicates and 
-member predicates) and ``result`` (for predicates that return a result).
+The ``bindingset`` annotation takes a comma-separated list of variables.
 
-For more information, see ":ref:`predicate-binding`."
+- When you annotate a predicate, each variable must be an argument of the predicate, possibly including ``this``
+  (for characteristic predicates and member predicates) and ``result`` (for predicates that return a result). 
+  For more information, see ":ref:`predicate-binding`."
+- When you annotate a class, each variable must be ``this`` or a field in the class. 
+  Binding sets for classes are supported from release 2.3.0 of the CodeQL CLI, and release 1.26 of LGTM Enterprise.
 
 .. Links to use in substitutions
 

javascript/change-notes/2021-03-29-misc-steps.md

@@ -0,0 +1,6 @@
+lgtm,codescanning
+* The `lodash-es` package is now recognized as a variant of `lodash`.
+* Taint is now propagated through the `babel.transform` function.
+* Improved data flow through React applications using `redux-form` or `react-router`.
+* Base64 decoding using the `react-native-base64` package is now recognized.
+* An expression of form `o[o.length] = y` is now recognized as appending to an array.

javascript/change-notes/2021-03-29-pg-promise.md

@@ -0,0 +1,2 @@
+lgtm,codescanning
+* SQL injection sinks from the `pg-promise` library are now recognized.

javascript/ql/src/semmle/javascript/Base64.qll

@@ -150,7 +150,8 @@ private class NpmBase64Encode extends Base64::Encode::Range, DataFlow::CallNode
       enc = DataFlow::moduleMember("base64url", "toBase64") or
       enc = DataFlow::moduleMember("js-base64", "Base64").getAPropertyRead("encode") or
       enc = DataFlow::moduleMember("js-base64", "Base64").getAPropertyRead("encodeURI") or
-      enc = DataFlow::moduleMember("urlsafe-base64", "encode")
+      enc = DataFlow::moduleMember("urlsafe-base64", "encode") or
+      enc = DataFlow::moduleMember("react-native-base64", ["encode", "encodeFromByteArray"])
     |
       this = enc.getACall()
     )
@@ -186,7 +187,8 @@ private class NpmBase64Decode extends Base64::Decode::Range, DataFlow::CallNode
       dec = DataFlow::moduleMember("base64url", "decode") or
       dec = DataFlow::moduleMember("base64url", "fromBase64") or
       dec = DataFlow::moduleMember("js-base64", "Base64").getAPropertyRead("decode") or
-      dec = DataFlow::moduleMember("urlsafe-base64", "decode")
+      dec = DataFlow::moduleMember("urlsafe-base64", "decode") or
+      dec = DataFlow::moduleMember("react-native-base64", "decode")
     |
       this = dec.getACall()
     )

javascript/ql/src/semmle/javascript/NodeJS.qll

@@ -202,6 +202,42 @@ private class RequireVariable extends Variable {
  */
 private predicate moduleInFile(Module m, File f) { m.getFile() = f }
 
+private predicate isModuleModule(DataFlow::Node nd) {
+  exists(ImportDeclaration imp |
+    imp.getImportedPath().getValue() = "module" and
+    nd =
+      [
+        DataFlow::destructuredModuleImportNode(imp),
+        DataFlow::valueNode(imp.getASpecifier().(ImportNamespaceSpecifier))
+      ]
+  )
+  or
+  isModuleModule(nd.getAPredecessor())
+}
+
+private predicate isCreateRequire(DataFlow::Node nd) {
+  exists(PropAccess prop |
+    isModuleModule(prop.getBase().flow()) and
+    prop.getPropertyName() = "createRequire" and
+    nd = prop.flow()
+  )
+  or
+  exists(PropertyPattern prop |
+    isModuleModule(prop.getObjectPattern().flow()) and
+    prop.getName() = "createRequire" and
+    nd = prop.getValuePattern().flow()
+  )
+  or
+  exists(ImportDeclaration decl, NamedImportSpecifier spec |
+    decl.getImportedPath().getValue() = "module" and
+    spec = decl.getASpecifier() and
+    spec.getImportedName() = "createRequire" and
+    nd = spec.flow()
+  )
+  or
+  isCreateRequire(nd.getAPredecessor())
+}
+
 /**
  * Holds if `nd` may refer to `require`, either directly or modulo local data flow.
  */
@@ -215,16 +251,11 @@ private predicate isRequire(DataFlow::Node nd) {
   or
   // `import { createRequire } from 'module';`.
   // specialized to ES2015 modules to avoid recursion in the `DataFlow::moduleImport()` predicate and to avoid
-  // negative recursion between `Import.getImportedModuleNode()` and `Import.getImportedModule()`.
-  exists(ImportDeclaration imp, DataFlow::SourceNode baseObj |
-    imp.getImportedPath().getValue() = "module"
-  |
-    baseObj =
-      [
-        DataFlow::destructuredModuleImportNode(imp),
-        DataFlow::valueNode(imp.getASpecifier().(ImportNamespaceSpecifier))
-      ] and
-    nd = baseObj.getAPropertyRead("createRequire").getACall()
+  // negative recursion between `Import.getImportedModuleNode()` and `Import.getImportedModule()`, and
+  // to avoid depending on `SourceNode` as this would make `SourceNode::Range` recursive.
+  exists(CallExpr call |
+    isCreateRequire(call.getCallee().flow()) and
+    nd = call.flow()
   )
 }
 

javascript/ql/src/semmle/javascript/dataflow/DataFlow.qll

@@ -492,6 +492,7 @@ module DataFlow {
      * Gets the data flow node corresponding to the base object
      * whose property is read from or written to.
      */
+    cached
     abstract Node getBase();
 
     /**
@@ -595,7 +596,10 @@ module DataFlow {
 
     PropLValueAsPropWrite() { astNode instanceof LValue }
 
-    override Node getBase() { result = valueNode(astNode.getBase()) }
+    override Node getBase() {
+      result = valueNode(astNode.getBase()) and
+      Stages::DataFlowStage::ref()
+    }
 
     override Expr getPropertyNameExpr() { result = astNode.getPropertyNameExpr() }
 
@@ -724,7 +728,7 @@ module DataFlow {
     override ParameterField prop;
 
     override Node getBase() {
-      result = thisNode(prop.getDeclaringClass().getConstructor().getBody())
+      thisNode(result, prop.getDeclaringClass().getConstructor().getBody())
     }
 
     override Expr getPropertyNameExpr() {
@@ -758,7 +762,7 @@ module DataFlow {
     }
 
     override Node getBase() {
-      result = thisNode(prop.getDeclaringClass().getConstructor().getBody())
+      thisNode(result, prop.getDeclaringClass().getConstructor().getBody())
     }
 
     override Expr getPropertyNameExpr() { result = prop.getNameExpr() }

javascript/ql/src/semmle/javascript/dataflow/Sources.qll

@@ -34,7 +34,11 @@ private import semmle.javascript.internal.CachedStages
  * ```
  */
 class SourceNode extends DataFlow::Node {
-  SourceNode() { this instanceof SourceNode::Range }
+  SourceNode() {
+    this instanceof SourceNode::Range
+    or
+    none() and this instanceof SourceNode::Internal::RecursionGuard
+  }
 
   /**
    * Holds if this node flows into `sink` in zero or more local (that is,
@@ -329,6 +333,12 @@ module SourceNode {
       DataFlow::functionReturnNode(this, _)
     }
   }
+
+  /** INTERNAL. DO NOT USE. */
+  module Internal {
+    /** An empty class that some tests are using to enforce that SourceNode is non-recursive. */
+    abstract class RecursionGuard extends DataFlow::Node { }
+  }
 }
 
 deprecated class DefaultSourceNode extends SourceNode {

javascript/ql/src/semmle/javascript/dataflow/TaintTracking.qll

@@ -584,22 +584,26 @@ module TaintTracking {
 
   /**
    * A taint propagating data flow edge for assignments of the form `o[k] = v`, where
-   * `k` is not a constant and `o` refers to some object literal; in this case, we consider
-   * taint to flow from `v` to that object literal.
+   * one of the following holds:
    *
-   * The rationale for this heuristic is that if properties of `o` are accessed by
-   * computed (that is, non-constant) names, then `o` is most likely being treated as
-   * a map, not as a real object. In this case, it makes sense to consider the entire
-   * map to be tainted as soon as one of its entries is.
+   * - `k` is not a constant and `o` refers to some object literal. The rationale
+   *   here is that `o` is most likely being used like a dictionary object.
+   *
+   * - `k` refers to `o.length`, that is, the assignment is of form `o[o.length] = v`.
+   *   In this case, the assignment behaves like `o.push(v)`.
    */
-  private class DictionaryTaintStep extends SharedTaintStep {
+  private class ComputedPropWriteTaintStep extends SharedTaintStep {
     override predicate heapStep(DataFlow::Node pred, DataFlow::Node succ) {
-      exists(AssignExpr assgn, IndexExpr idx, DataFlow::ObjectLiteralNode obj |
+      exists(AssignExpr assgn, IndexExpr idx, DataFlow::SourceNode obj |
         assgn.getTarget() = idx and
         obj.flowsToExpr(idx.getBase()) and
         not exists(idx.getPropertyName()) and
         pred = DataFlow::valueNode(assgn.getRhs()) and
         succ = obj
+      |
+        obj instanceof DataFlow::ObjectLiteralNode
+        or
+        obj.getAPropertyRead("length").flowsToExpr(idx.getPropertyNameExpr())
       )
     }
   }

javascript/ql/src/semmle/javascript/frameworks/Babel.qll

@@ -188,4 +188,20 @@ module Babel {
     /** Gets the name of the variable used to create JSX elements. */
     string getJsxFactoryVariableName() { result = getOption("pragma").(JSONString).getValue() }
   }
+
+  /**
+   * A taint step through a call to the Babel `transform` function.
+   */
+  private class TransformTaintStep extends TaintTracking::SharedTaintStep {
+    override predicate step(DataFlow::Node pred, DataFlow::Node succ) {
+      exists(API::CallNode call |
+        call =
+          API::moduleImport(["@babel/standalone", "@babel/core"])
+              .getMember(["transform", "transformSync", "transformAsync"])
+              .getACall() and
+        pred = call.getArgument(0) and
+        succ = [call, call.getParameter(2).getParameter(0).getAnImmediateUse()]
+      )
+    }
+  }
 }

javascript/ql/src/semmle/javascript/frameworks/LodashUnderscore.qll

@@ -21,11 +21,9 @@ module LodashUnderscore {
     string name;
 
     DefaultMember() {
-      this = DataFlow::moduleMember("underscore", name)
+      this = DataFlow::moduleMember(["underscore", "lodash", "lodash-es"], name)
       or
-      this = DataFlow::moduleMember("lodash", name)
-      or
-      this = DataFlow::moduleImport("lodash/" + name)
+      this = DataFlow::moduleImport(["lodash/", "lodash-es/"] + name)
       or
       this = DataFlow::moduleImport("lodash." + name.toLowerCase()) and isLodashMember(name)
       or

javascript/ql/src/semmle/javascript/frameworks/PropertyProjection.qll

@@ -75,10 +75,6 @@ private DataFlow::SourceNode getASimplePropertyProjectionCallee(
 ) {
   singleton = false and
   (
-    result = LodashUnderscore::member("pick") and
-    objectIndex = 0 and
-    selectorIndex = [1 .. max(result.getACall().getNumArgument())]
-    or
     result = LodashUnderscore::member("pickBy") and
     objectIndex = 0 and
     selectorIndex = 1
@@ -131,6 +127,19 @@ private class SimplePropertyProjection extends PropertyProjection::Range {
   override predicate isSingletonProjection() { singleton = true }
 }
 
+/**
+ * A property projection with a variable number of selector indices.
+ */
+private class VarArgsPropertyProjection extends PropertyProjection::Range {
+  VarArgsPropertyProjection() { this = LodashUnderscore::member("pick").getACall() }
+
+  override DataFlow::Node getObject() { result = getArgument(0) }
+
+  override DataFlow::Node getASelector() { result = getArgument(any(int i | i > 0)) }
+
+  override predicate isSingletonProjection() { none() }
+}
+
 /**
  * A taint step for a property projection.
  */

javascript/ql/src/semmle/javascript/frameworks/React.qll

@@ -691,15 +691,22 @@ private DataFlow::SourceNode reactRouterDom() {
   result = DataFlow::moduleImport("react-router-dom")
 }
 
+private DataFlow::SourceNode reactRouterMatchObject() {
+  result = reactRouterDom().getAMemberCall(["useRouteMatch", "matchPath"])
+  or
+  exists(ReactComponent c |
+    dependedOnByReactRouterClient(c.getTopLevel()) and
+    result = c.getAPropRead("match")
+  )
+}
+
 private class ReactRouterSource extends ClientSideRemoteFlowSource {
   ClientSideRemoteFlowKind kind;
 
   ReactRouterSource() {
     this = reactRouterDom().getAMemberCall("useParams") and kind.isPath()
     or
-    exists(string prop |
-      this = reactRouterDom().getAMemberCall("useRouteMatch").getAPropertyRead(prop)
-    |
+    exists(string prop | this = reactRouterMatchObject().getAPropertyRead(prop) |
       prop = "params" and kind.isPath()
       or
       prop = "url" and kind.isUrl()
@@ -713,9 +720,6 @@ private class ReactRouterSource extends ClientSideRemoteFlowSource {
 
 /**
  * Holds if `mod` transitively depends on `react-router-dom`.
- *
- * We assume any React component in such a file may be used in a context where react-router
- * injects the `location` property in its `props` object.
  */
 private predicate dependsOnReactRouter(Module mod) {
   mod.getAnImport().getImportedPath().getValue() = "react-router-dom"
@@ -723,6 +727,18 @@ private predicate dependsOnReactRouter(Module mod) {
   dependsOnReactRouter(mod.getAnImportedModule())
 }
 
+/**
+ * Holds if `mod` is imported from a module that transitively depends on `react-router-dom`.
+ *
+ * We assume any React component in such a file may be used in a context where react-router
+ * injects the `location` and `match` properties in its `props` object.
+ */
+private predicate dependedOnByReactRouterClient(Module mod) {
+  dependsOnReactRouter(mod)
+  or
+  dependedOnByReactRouterClient(any(Module m | m.getAnImportedModule() = mod))
+}
+
 /**
  * A reference to the DOM location obtained through `react-router-dom`
  *
@@ -740,7 +756,7 @@ private class ReactRouterLocationSource extends DOM::LocationSource::Range {
     this = reactRouterDom().getAMemberCall("useLocation")
     or
     exists(ReactComponent component |
-      dependsOnReactRouter(component.getTopLevel()) and
+      dependedOnByReactRouterClient(component.getTopLevel()) and
       this = component.getAPropRead("location")
     )
   }
@@ -759,6 +775,8 @@ private DataFlow::SourceNode higherOrderComponentBuilder() {
   or
   result = DataFlow::moduleMember(["react-hot-loader", "react-hot-loader/root"], "hot").getACall()
   or
+  result = DataFlow::moduleMember("redux-form", "reduxForm").getACall()
+  or
   result = reactRouterDom().getAPropertyRead("withRouter")
   or
   exists(FunctionCompositionCall compose |

javascript/ql/src/semmle/javascript/frameworks/SQL.qll

@@ -93,11 +93,17 @@ private module MySql {
 }
 
 /**
- * Provides classes modelling the `pg` package.
+ * Provides classes modelling the PostgreSQL packages, such as `pg` and `pg-promise`.
  */
 private module Postgres {
+  API::Node pg() {
+    result = API::moduleImport("pg")
+    or
+    result = pgpMain().getMember("pg")
+  }
+
   /** Gets a reference to the `Client` constructor in the `pg` package, for example `require('pg').Client`. */
-  API::Node newClient() { result = API::moduleImport("pg").getMember("Client") }
+  API::Node newClient() { result = pg().getMember("Client") }
 
   /** Gets a freshly created Postgres client instance. */
   API::Node client() {
@@ -105,19 +111,25 @@ private module Postgres {
     or
     // pool.connect(function(err, client) { ... })
     result = pool().getMember("connect").getParameter(0).getParameter(1)
+    or
+    result = pgpConnection().getMember("client")
   }
 
   /** Gets a constructor that when invoked constructs a new connection pool. */
   API::Node newPool() {
     // new require('pg').Pool()
-    result = API::moduleImport("pg").getMember("Pool")
+    result = pg().getMember("Pool")
     or
     // new require('pg-pool')
     result = API::moduleImport("pg-pool")
   }
 
-  /** Gets an expression that constructs a new connection pool. */
-  API::Node pool() { result = newPool().getInstance() }
+  /** Gets an API node that refers to a connection pool. */
+  API::Node pool() {
+    result = newPool().getInstance()
+    or
+    result = pgpDatabase().getMember("$pool")
+  }
 
   /** A call to the Postgres `query` method. */
   private class QueryCall extends DatabaseAccess, DataFlow::MethodCallNode {
@@ -137,17 +149,142 @@ private module Postgres {
 
     Credentials() {
       exists(string prop |
-        this = [newClient(), newPool()].getParameter(0).getMember(prop).getARhs().asExpr() and
-        (
-          prop = "user" and kind = "user name"
-          or
-          prop = "password" and kind = prop
-        )
+        this = [newClient(), newPool()].getParameter(0).getMember(prop).getARhs().asExpr()
+        or
+        this = pgPromise().getParameter(0).getMember(prop).getARhs().asExpr()
+      |
+        prop = "user" and kind = "user name"
+        or
+        prop = "password" and kind = prop
       )
     }
 
     override string getCredentialsKind() { result = kind }
   }
+
+  /** Gets a node referring to the `pg-promise` library (which is not itself a Promise). */
+  API::Node pgPromise() { result = API::moduleImport("pg-promise") }
+
+  /** Gets an initialized `pg-promise` library. */
+  API::Node pgpMain() {
+    result = pgPromise().getReturn()
+    or
+    result = API::Node::ofType("pg-promise", "IMain")
+  }
+
+  /** Gets a database from `pg-promise`. */
+  API::Node pgpDatabase() {
+    result = pgpMain().getReturn()
+    or
+    result = API::Node::ofType("pg-promise", "IDatabase")
+  }
+
+  /** Gets a connection created from a `pg-promise` database. */
+  API::Node pgpConnection() {
+    result = pgpDatabase().getMember("connect").getReturn().getPromised()
+    or
+    result = API::Node::ofType("pg-promise", "IConnected")
+  }
+
+  /** Gets a `pg-promise` task object. */
+  API::Node pgpTask() {
+    exists(API::Node taskMethod |
+      taskMethod = pgpObject().getMember(["task", "taskIf", "tx", "txIf"])
+    |
+      result = taskMethod.getParameter([0, 1]).getParameter(0)
+      or
+      result = taskMethod.getParameter(0).getMember("cnd").getParameter(0)
+    )
+    or
+    result = API::Node::ofType("pg-promise", "ITask")
+  }
+
+  /** Gets a `pg-promise` object which supports querying (database, connection, or task). */
+  API::Node pgpObject() {
+    result = [pgpDatabase(), pgpConnection(), pgpTask()]
+    or
+    result = API::Node::ofType("pg-promise", "IBaseProtocol")
+  }
+
+  private string pgpQueryMethodName() {
+    result =
+      [
+        "any", "each", "many", "manyOrNone", "map", "multi", "multiResult", "none", "one",
+        "oneOrNone", "query", "result"
+      ]
+  }
+
+  /** A call that executes a SQL query via `pg-promise`. */
+  private class PgPromiseQueryCall extends DatabaseAccess, DataFlow::MethodCallNode {
+    PgPromiseQueryCall() { this = pgpObject().getMember(pgpQueryMethodName()).getACall() }
+
+    /** Gets an argument interpreted as a SQL string, not including raw interpolation variables. */
+    private DataFlow::Node getADirectQueryArgument() {
+      result = getArgument(0)
+      or
+      result = getOptionArgument(0, "text")
+    }
+
+    /**
+     * Gets an interpolation parameter whose value is interpreted literally, or is not escaped appropriately for its context.
+     *
+     * For example, the following are raw placeholders: $1:raw, $1^, ${prop}:raw, $(prop)^
+     */
+    private string getARawParameterName() {
+      exists(string sqlString, string placeholderRegexp, string regexp |
+        placeholderRegexp = "\\$(\\d+|[{(\\[/]\\w+[})\\]/])" and // For example: $1 or ${prop}
+        sqlString = getADirectQueryArgument().getStringValue()
+      |
+        // Match $1:raw or ${prop}:raw
+        regexp = placeholderRegexp + "(:raw|\\^)" and
+        result =
+          sqlString
+              .regexpFind(regexp, _, _)
+              .regexpCapture(regexp, 1)
+              .regexpReplaceAll("[^\\w\\d]", "")
+        or
+        // Match $1:value or ${prop}:value unless enclosed by single quotes (:value prevents breaking out of single quotes)
+        regexp = placeholderRegexp + "(:value|\\#)" and
+        result =
+          sqlString
+              .regexpReplaceAll("'[^']*'", "''")
+              .regexpFind(regexp, _, _)
+              .regexpCapture(regexp, 1)
+              .regexpReplaceAll("[^\\w\\d]", "")
+      )
+    }
+
+    /** Gets the argument holding the values to plug into placeholders. */
+    private DataFlow::Node getValues() {
+      result = getArgument(1)
+      or
+      result = getOptionArgument(0, "values")
+    }
+
+    /** Gets a value that is plugged into a raw placeholder variable, making it a sink for SQL injection. */
+    private DataFlow::Node getARawValue() {
+      result = getValues() and getARawParameterName() = "1" // Special case: if the argument is not an array or object, it's just plugged into $1
+      or
+      exists(DataFlow::SourceNode values | values = getValues().getALocalSource() |
+        result = values.getAPropertyWrite(getARawParameterName()).getRhs()
+        or
+        // Array literals do not have PropWrites with property names so handle them separately,
+        // and also translate to 0-based indexing.
+        result = values.(DataFlow::ArrayCreationNode).getElement(getARawParameterName().toInt() - 1)
+      )
+    }
+
+    override DataFlow::Node getAQueryArgument() {
+      result = getADirectQueryArgument()
+      or
+      result = getARawValue()
+    }
+  }
+
+  /** An expression that is interpreted as SQL by `pg-promise`. */
+  class PgPromiseQueryString extends SQL::SqlString {
+    PgPromiseQueryString() { this = any(PgPromiseQueryCall qc).getAQueryArgument().asExpr() }
+  }
 }
 
 /**

javascript/ql/src/semmle/javascript/internal/CachedStages.qll

@@ -133,6 +133,8 @@ module Stages {
       exists(any(DataFlow::Node node).toString())
       or
       exists(any(AccessPath a).getAnInstanceIn(_))
+      or
+      exists(any(DataFlow::PropRef ref).getBase())
     }
   }
 

javascript/ql/test/library-tests/RecursionPrevention/SourceNodeRange.expected

@@ -0,0 +1 @@
+| Success |

javascript/ql/test/library-tests/RecursionPrevention/SourceNodeRange.ql

@@ -0,0 +1,14 @@
+/**
+ * Test that fails to compile if the domain of `SourceNode::Range` depends on `SourceNode` (recursively).
+ *
+ * This tests adds a negative dependency `SourceNode --!--> SourceNode::Range`
+ * so that the undesired edge `SourceNode::Range --> SourceNode` completes a negative cycle.
+ */
+
+import javascript
+
+class BadSourceNodeRange extends DataFlow::SourceNode::Internal::RecursionGuard {
+  BadSourceNodeRange() { not this instanceof DataFlow::SourceNode::Range }
+}
+
+select "Success"

javascript/ql/test/library-tests/TaintTracking/BasicTaintTracking.expected

@@ -11,6 +11,7 @@ typeInferenceMismatch
 | array-mutation.js:27:16:27:23 | source() | array-mutation.js:28:8:28:8 | g |
 | array-mutation.js:31:33:31:40 | source() | array-mutation.js:32:8:32:8 | h |
 | array-mutation.js:35:36:35:43 | source() | array-mutation.js:36:8:36:8 | i |
+| array-mutation.js:39:17:39:24 | source() | array-mutation.js:40:8:40:8 | j |
 | booleanOps.js:2:11:2:18 | source() | booleanOps.js:4:8:4:8 | x |
 | booleanOps.js:2:11:2:18 | source() | booleanOps.js:13:10:13:10 | x |
 | booleanOps.js:2:11:2:18 | source() | booleanOps.js:19:10:19:10 | x |

javascript/ql/test/library-tests/TaintTracking/array-mutation.js

@@ -34,4 +34,8 @@ function test(x, y) {
   let i = [];
   Array.prototype.unshift.apply(i, source());
   sink(i); // NOT OK
+
+  let j = [];
+  j[j.length] = source();
+  sink(j); // NOT OK
 }

javascript/ql/test/library-tests/frameworks/ReactJS/importedComponent.jsx

@@ -1,5 +1,5 @@
 import { MyComponent } from "./exportedComponent";
 
-export function render(color) {
+export function render({color, location}) {
     return <MyComponent color={color}/>
 }

javascript/ql/test/library-tests/frameworks/ReactJS/tests.expected

@@ -16,6 +16,7 @@ test_ReactComponent_getInstanceMethod
 | es5.js:18:33:22:1 | {\\n  ren ... ;\\n  }\\n} | render | es5.js:19:11:21:3 | functio ... 1>;\\n  } |
 | es6.js:1:1:8:1 | class H ... ;\\n  }\\n} | render | es6.js:2:9:4:3 | () {\\n   ... v>;\\n  } |
 | exportedComponent.jsx:1:8:3:1 | functio ... r}}/>\\n} | render | exportedComponent.jsx:1:8:3:1 | functio ... r}}/>\\n} |
+| importedComponent.jsx:3:8:5:1 | functio ... or}/>\\n} | render | importedComponent.jsx:3:8:5:1 | functio ... or}/>\\n} |
 | plainfn.js:1:1:3:1 | functio ... div>;\\n} | render | plainfn.js:1:1:3:1 | functio ... div>;\\n} |
 | plainfn.js:5:1:7:1 | functio ... iv");\\n} | render | plainfn.js:5:1:7:1 | functio ... iv");\\n} |
 | plainfn.js:9:1:12:1 | functio ... rn x;\\n} | render | plainfn.js:9:1:12:1 | functio ... rn x;\\n} |
@@ -97,6 +98,7 @@ test_ReactComponent_ref
 | es6.js:14:1:20:1 | class H ...     }\\n} | es6.js:17:9:17:12 | this |
 | es6.js:14:1:20:1 | class H ...     }\\n} | es6.js:18:9:18:12 | this |
 | exportedComponent.jsx:1:8:3:1 | functio ... r}}/>\\n} | exportedComponent.jsx:1:8:1:7 | this |
+| importedComponent.jsx:3:8:5:1 | functio ... or}/>\\n} | importedComponent.jsx:3:8:3:7 | this |
 | namedImport.js:3:1:3:28 | class C ... nent {} | namedImport.js:3:27:3:26 | this |
 | namedImport.js:5:1:5:20 | class D extends C {} | namedImport.js:5:19:5:18 | this |
 | plainfn.js:1:1:3:1 | functio ... div>;\\n} | plainfn.js:1:1:1:0 | this |
@@ -198,6 +200,7 @@ test_ReactComponent_getADirectPropsSource
 | es6.js:1:1:8:1 | class H ... ;\\n  }\\n} | es6.js:1:37:1:36 | args |
 | es6.js:1:1:8:1 | class H ... ;\\n  }\\n} | es6.js:3:24:3:33 | this.props |
 | exportedComponent.jsx:1:8:3:1 | functio ... r}}/>\\n} | exportedComponent.jsx:1:29:1:33 | props |
+| importedComponent.jsx:3:8:5:1 | functio ... or}/>\\n} | importedComponent.jsx:3:24:3:40 | {color, location} |
 | namedImport.js:3:1:3:28 | class C ... nent {} | namedImport.js:3:27:3:26 | args |
 | namedImport.js:5:1:5:20 | class D extends C {} | namedImport.js:5:19:5:18 | args |
 | plainfn.js:1:1:3:1 | functio ... div>;\\n} | plainfn.js:1:16:1:20 | props |
@@ -236,6 +239,7 @@ test_ReactComponent
 | es6.js:1:1:8:1 | class H ... ;\\n  }\\n} |
 | es6.js:14:1:20:1 | class H ...     }\\n} |
 | exportedComponent.jsx:1:8:3:1 | functio ... r}}/>\\n} |
+| importedComponent.jsx:3:8:5:1 | functio ... or}/>\\n} |
 | namedImport.js:3:1:3:28 | class C ... nent {} |
 | namedImport.js:5:1:5:20 | class D extends C {} |
 | plainfn.js:1:1:3:1 | functio ... div>;\\n} |
@@ -264,6 +268,8 @@ test_ReactComponent_getAPropRead
 | es5.js:18:33:22:1 | {\\n  ren ... ;\\n  }\\n} | name | es5.js:20:24:20:38 | this.props.name |
 | es6.js:1:1:8:1 | class H ... ;\\n  }\\n} | name | es6.js:3:24:3:38 | this.props.name |
 | exportedComponent.jsx:1:8:3:1 | functio ... r}}/>\\n} | color | exportedComponent.jsx:2:32:2:42 | props.color |
+| importedComponent.jsx:3:8:5:1 | functio ... or}/>\\n} | color | importedComponent.jsx:3:25:3:29 | color |
+| importedComponent.jsx:3:8:5:1 | functio ... or}/>\\n} | location | importedComponent.jsx:3:32:3:39 | location |
 | plainfn.js:1:1:3:1 | functio ... div>;\\n} | name | plainfn.js:2:22:2:31 | props.name |
 | preact.js:1:1:7:1 | class H ...     }\\n} | name | preact.js:3:9:3:18 | props.name |
 | probably-a-component.js:1:1:6:1 | class H ...     }\\n} | name | probably-a-component.js:3:9:3:23 | this.props.name |
@@ -288,6 +294,9 @@ test_JSXname
 | thisAccesses.js:60:19:60:41 | <this.n ... s.name> | thisAccesses.js:60:20:60:28 | this.name | this.name | dot |
 | thisAccesses.js:61:19:61:41 | <this.t ... s.this> | thisAccesses.js:61:20:61:28 | this.this | this.this | dot |
 | thisAccesses_importedMappers.js:13:16:13:21 | <div/> | thisAccesses_importedMappers.js:13:17:13:19 | div | div | Identifier |
+| use-react-router.jsx:5:17:5:87 | <Router ... Router> | use-react-router.jsx:5:18:5:23 | Router | Router | Identifier |
+| use-react-router.jsx:5:25:5:78 | <Route> ... /Route> | use-react-router.jsx:5:26:5:30 | Route | Route | Identifier |
+| use-react-router.jsx:5:32:5:70 | <Import ... ponent> | use-react-router.jsx:5:33:5:49 | ImportedComponent | ImportedComponent | Identifier |
 | useHigherOrderComponent.jsx:5:12:5:39 | <SomeCo ... "red"/> | useHigherOrderComponent.jsx:5:13:5:25 | SomeComponent | SomeComponent | Identifier |
 | useHigherOrderComponent.jsx:11:12:11:46 | <LazyLo ... lazy"/> | useHigherOrderComponent.jsx:11:13:11:31 | LazyLoadedComponent | LazyLoadedComponent | Identifier |
 | useHigherOrderComponent.jsx:17:12:17:48 | <LazyLo ... azy2"/> | useHigherOrderComponent.jsx:17:13:17:32 | LazyLoadedComponent2 | LazyLoadedComponent2 | Identifier |
@@ -298,3 +307,5 @@ test_JSXName_this
 | statePropertyWrites.js:38:12:38:45 | <div>He ... }</div> | statePropertyWrites.js:38:24:38:27 | this |
 | thisAccesses.js:60:19:60:41 | <this.n ... s.name> | thisAccesses.js:60:20:60:23 | this |
 | thisAccesses.js:61:19:61:41 | <this.t ... s.this> | thisAccesses.js:61:20:61:23 | this |
+locationSource
+| importedComponent.jsx:3:32:3:39 | location |

javascript/ql/test/library-tests/frameworks/ReactJS/tests.ql

@@ -9,3 +9,5 @@ import ReactComponent_getACandidatePropsValue
 import ReactComponent
 import ReactComponent_getAPropRead
 import ReactName
+
+query DataFlow::SourceNode locationSource() { result = DOM::locationSource() }

javascript/ql/test/library-tests/frameworks/ReactJS/use-react-router.jsx

@@ -0,0 +1,5 @@
+import * as ReactDOM from "react-dom"
+import { Route, Router } from "react-router-dom";
+import ImportedComponent from "./importedComponent";
+
+ReactDOM.render(<Router><Route><ImportedComponent></ImportedComponent></Route></Router>);

javascript/ql/test/query-tests/Security/CWE-089/untyped/DatabaseAccesses.expected

@@ -37,6 +37,28 @@
 | mongoose.js:97:2:97:52 | Documen ... query)) |
 | mongoose.js:99:2:99:50 | Documen ... query)) |
 | mongoose.js:113:2:113:53 | Documen ... () { }) |
+| pg-promise-types.ts:8:5:8:22 | this.db.one(taint) |
+| pg-promise.js:9:3:9:15 | db.any(query) |
+| pg-promise.js:10:3:10:16 | db.many(query) |
+| pg-promise.js:11:3:11:22 | db.manyOrNone(query) |
+| pg-promise.js:12:3:12:15 | db.map(query) |
+| pg-promise.js:13:3:13:17 | db.multi(query) |
+| pg-promise.js:14:3:14:23 | db.mult ... (query) |
+| pg-promise.js:15:3:15:16 | db.none(query) |
+| pg-promise.js:16:3:16:15 | db.one(query) |
+| pg-promise.js:17:3:17:21 | db.oneOrNone(query) |
+| pg-promise.js:18:3:18:17 | db.query(query) |
+| pg-promise.js:19:3:19:18 | db.result(query) |
+| pg-promise.js:21:3:23:4 | db.one( ... OK\\n  }) |
+| pg-promise.js:24:3:27:4 | db.one( ... OK\\n  }) |
+| pg-promise.js:28:3:31:4 | db.one( ... er\\n  }) |
+| pg-promise.js:32:3:35:4 | db.one( ... OK\\n  }) |
+| pg-promise.js:36:3:43:4 | db.one( ...  ]\\n  }) |
+| pg-promise.js:44:3:50:4 | db.one( ...  }\\n  }) |
+| pg-promise.js:51:3:58:4 | db.one( ...  }\\n  }) |
+| pg-promise.js:60:14:60:25 | t.one(query) |
+| pg-promise.js:63:17:63:28 | t.one(query) |
+| pg-promise.js:64:10:64:21 | t.one(query) |
 | socketio.js:11:5:11:54 | db.run( ... ndle}`) |
 | tst2.js:7:3:7:62 | sql.que ... ms.id}` |
 | tst2.js:9:3:9:85 | new sql ...  + "'") |

javascript/ql/test/query-tests/Security/CWE-089/untyped/SqlInjection.expected

@@ -206,6 +206,70 @@ nodes
 | mongooseModelClient.js:12:22:12:29 | req.body |
 | mongooseModelClient.js:12:22:12:29 | req.body |
 | mongooseModelClient.js:12:22:12:32 | req.body.id |
+| pg-promise-types.ts:7:9:7:28 | taint |
+| pg-promise-types.ts:7:17:7:28 | req.params.x |
+| pg-promise-types.ts:7:17:7:28 | req.params.x |
+| pg-promise-types.ts:8:17:8:21 | taint |
+| pg-promise-types.ts:8:17:8:21 | taint |
+| pg-promise.js:6:7:7:55 | query |
+| pg-promise.js:6:15:7:55 | "SELECT ...  PRICE" |
+| pg-promise.js:7:16:7:34 | req.params.category |
+| pg-promise.js:7:16:7:34 | req.params.category |
+| pg-promise.js:9:10:9:14 | query |
+| pg-promise.js:9:10:9:14 | query |
+| pg-promise.js:10:11:10:15 | query |
+| pg-promise.js:10:11:10:15 | query |
+| pg-promise.js:11:17:11:21 | query |
+| pg-promise.js:11:17:11:21 | query |
+| pg-promise.js:12:10:12:14 | query |
+| pg-promise.js:12:10:12:14 | query |
+| pg-promise.js:13:12:13:16 | query |
+| pg-promise.js:13:12:13:16 | query |
+| pg-promise.js:14:18:14:22 | query |
+| pg-promise.js:14:18:14:22 | query |
+| pg-promise.js:15:11:15:15 | query |
+| pg-promise.js:15:11:15:15 | query |
+| pg-promise.js:16:10:16:14 | query |
+| pg-promise.js:16:10:16:14 | query |
+| pg-promise.js:17:16:17:20 | query |
+| pg-promise.js:17:16:17:20 | query |
+| pg-promise.js:18:12:18:16 | query |
+| pg-promise.js:18:12:18:16 | query |
+| pg-promise.js:19:13:19:17 | query |
+| pg-promise.js:19:13:19:17 | query |
+| pg-promise.js:22:11:22:15 | query |
+| pg-promise.js:22:11:22:15 | query |
+| pg-promise.js:30:13:30:25 | req.params.id |
+| pg-promise.js:30:13:30:25 | req.params.id |
+| pg-promise.js:30:13:30:25 | req.params.id |
+| pg-promise.js:34:13:34:25 | req.params.id |
+| pg-promise.js:34:13:34:25 | req.params.id |
+| pg-promise.js:34:13:34:25 | req.params.id |
+| pg-promise.js:38:13:42:5 | [\\n      ... n\\n    ] |
+| pg-promise.js:38:13:42:5 | [\\n      ... n\\n    ] |
+| pg-promise.js:39:7:39:19 | req.params.id |
+| pg-promise.js:39:7:39:19 | req.params.id |
+| pg-promise.js:39:7:39:19 | req.params.id |
+| pg-promise.js:40:7:40:21 | req.params.name |
+| pg-promise.js:40:7:40:21 | req.params.name |
+| pg-promise.js:40:7:40:21 | req.params.name |
+| pg-promise.js:41:7:41:20 | req.params.foo |
+| pg-promise.js:41:7:41:20 | req.params.foo |
+| pg-promise.js:47:11:47:23 | req.params.id |
+| pg-promise.js:47:11:47:23 | req.params.id |
+| pg-promise.js:47:11:47:23 | req.params.id |
+| pg-promise.js:54:11:54:23 | req.params.id |
+| pg-promise.js:54:11:54:23 | req.params.id |
+| pg-promise.js:54:11:54:23 | req.params.id |
+| pg-promise.js:56:14:56:29 | req.params.title |
+| pg-promise.js:56:14:56:29 | req.params.title |
+| pg-promise.js:56:14:56:29 | req.params.title |
+| pg-promise.js:60:20:60:24 | query |
+| pg-promise.js:60:20:60:24 | query |
+| pg-promise.js:63:23:63:27 | query |
+| pg-promise.js:63:23:63:27 | query |
+| pg-promise.js:64:16:64:20 | query |
+| pg-promise.js:64:16:64:20 | query |
 | redis.js:10:16:10:23 | req.body |
 | redis.js:10:16:10:23 | req.body |
 | redis.js:10:16:10:27 | req.body.key |
@@ -553,6 +617,62 @@ edges
 | mongooseModelClient.js:12:22:12:29 | req.body | mongooseModelClient.js:12:22:12:32 | req.body.id |
 | mongooseModelClient.js:12:22:12:32 | req.body.id | mongooseModelClient.js:12:16:12:34 | { id: req.body.id } |
 | mongooseModelClient.js:12:22:12:32 | req.body.id | mongooseModelClient.js:12:16:12:34 | { id: req.body.id } |
+| pg-promise-types.ts:7:9:7:28 | taint | pg-promise-types.ts:8:17:8:21 | taint |
+| pg-promise-types.ts:7:9:7:28 | taint | pg-promise-types.ts:8:17:8:21 | taint |
+| pg-promise-types.ts:7:17:7:28 | req.params.x | pg-promise-types.ts:7:9:7:28 | taint |
+| pg-promise-types.ts:7:17:7:28 | req.params.x | pg-promise-types.ts:7:9:7:28 | taint |
+| pg-promise.js:6:7:7:55 | query | pg-promise.js:9:10:9:14 | query |
+| pg-promise.js:6:7:7:55 | query | pg-promise.js:9:10:9:14 | query |
+| pg-promise.js:6:7:7:55 | query | pg-promise.js:10:11:10:15 | query |
+| pg-promise.js:6:7:7:55 | query | pg-promise.js:10:11:10:15 | query |
+| pg-promise.js:6:7:7:55 | query | pg-promise.js:11:17:11:21 | query |
+| pg-promise.js:6:7:7:55 | query | pg-promise.js:11:17:11:21 | query |
+| pg-promise.js:6:7:7:55 | query | pg-promise.js:12:10:12:14 | query |
+| pg-promise.js:6:7:7:55 | query | pg-promise.js:12:10:12:14 | query |
+| pg-promise.js:6:7:7:55 | query | pg-promise.js:13:12:13:16 | query |
+| pg-promise.js:6:7:7:55 | query | pg-promise.js:13:12:13:16 | query |
+| pg-promise.js:6:7:7:55 | query | pg-promise.js:14:18:14:22 | query |
+| pg-promise.js:6:7:7:55 | query | pg-promise.js:14:18:14:22 | query |
+| pg-promise.js:6:7:7:55 | query | pg-promise.js:15:11:15:15 | query |
+| pg-promise.js:6:7:7:55 | query | pg-promise.js:15:11:15:15 | query |
+| pg-promise.js:6:7:7:55 | query | pg-promise.js:16:10:16:14 | query |
+| pg-promise.js:6:7:7:55 | query | pg-promise.js:16:10:16:14 | query |
+| pg-promise.js:6:7:7:55 | query | pg-promise.js:17:16:17:20 | query |
+| pg-promise.js:6:7:7:55 | query | pg-promise.js:17:16:17:20 | query |
+| pg-promise.js:6:7:7:55 | query | pg-promise.js:18:12:18:16 | query |
+| pg-promise.js:6:7:7:55 | query | pg-promise.js:18:12:18:16 | query |
+| pg-promise.js:6:7:7:55 | query | pg-promise.js:19:13:19:17 | query |
+| pg-promise.js:6:7:7:55 | query | pg-promise.js:19:13:19:17 | query |
+| pg-promise.js:6:7:7:55 | query | pg-promise.js:22:11:22:15 | query |
+| pg-promise.js:6:7:7:55 | query | pg-promise.js:22:11:22:15 | query |
+| pg-promise.js:6:7:7:55 | query | pg-promise.js:60:20:60:24 | query |
+| pg-promise.js:6:7:7:55 | query | pg-promise.js:60:20:60:24 | query |
+| pg-promise.js:6:7:7:55 | query | pg-promise.js:63:23:63:27 | query |
+| pg-promise.js:6:7:7:55 | query | pg-promise.js:63:23:63:27 | query |
+| pg-promise.js:6:7:7:55 | query | pg-promise.js:64:16:64:20 | query |
+| pg-promise.js:6:7:7:55 | query | pg-promise.js:64:16:64:20 | query |
+| pg-promise.js:6:15:7:55 | "SELECT ...  PRICE" | pg-promise.js:6:7:7:55 | query |
+| pg-promise.js:7:16:7:34 | req.params.category | pg-promise.js:6:15:7:55 | "SELECT ...  PRICE" |
+| pg-promise.js:7:16:7:34 | req.params.category | pg-promise.js:6:15:7:55 | "SELECT ...  PRICE" |
+| pg-promise.js:30:13:30:25 | req.params.id | pg-promise.js:30:13:30:25 | req.params.id |
+| pg-promise.js:34:13:34:25 | req.params.id | pg-promise.js:34:13:34:25 | req.params.id |
+| pg-promise.js:39:7:39:19 | req.params.id | pg-promise.js:38:13:42:5 | [\\n      ... n\\n    ] |
+| pg-promise.js:39:7:39:19 | req.params.id | pg-promise.js:38:13:42:5 | [\\n      ... n\\n    ] |
+| pg-promise.js:39:7:39:19 | req.params.id | pg-promise.js:38:13:42:5 | [\\n      ... n\\n    ] |
+| pg-promise.js:39:7:39:19 | req.params.id | pg-promise.js:38:13:42:5 | [\\n      ... n\\n    ] |
+| pg-promise.js:39:7:39:19 | req.params.id | pg-promise.js:39:7:39:19 | req.params.id |
+| pg-promise.js:40:7:40:21 | req.params.name | pg-promise.js:38:13:42:5 | [\\n      ... n\\n    ] |
+| pg-promise.js:40:7:40:21 | req.params.name | pg-promise.js:38:13:42:5 | [\\n      ... n\\n    ] |
+| pg-promise.js:40:7:40:21 | req.params.name | pg-promise.js:38:13:42:5 | [\\n      ... n\\n    ] |
+| pg-promise.js:40:7:40:21 | req.params.name | pg-promise.js:38:13:42:5 | [\\n      ... n\\n    ] |
+| pg-promise.js:40:7:40:21 | req.params.name | pg-promise.js:40:7:40:21 | req.params.name |
+| pg-promise.js:41:7:41:20 | req.params.foo | pg-promise.js:38:13:42:5 | [\\n      ... n\\n    ] |
+| pg-promise.js:41:7:41:20 | req.params.foo | pg-promise.js:38:13:42:5 | [\\n      ... n\\n    ] |
+| pg-promise.js:41:7:41:20 | req.params.foo | pg-promise.js:38:13:42:5 | [\\n      ... n\\n    ] |
+| pg-promise.js:41:7:41:20 | req.params.foo | pg-promise.js:38:13:42:5 | [\\n      ... n\\n    ] |
+| pg-promise.js:47:11:47:23 | req.params.id | pg-promise.js:47:11:47:23 | req.params.id |
+| pg-promise.js:54:11:54:23 | req.params.id | pg-promise.js:54:11:54:23 | req.params.id |
+| pg-promise.js:56:14:56:29 | req.params.title | pg-promise.js:56:14:56:29 | req.params.title |
 | redis.js:10:16:10:23 | req.body | redis.js:10:16:10:27 | req.body.key |
 | redis.js:10:16:10:23 | req.body | redis.js:10:16:10:27 | req.body.key |
 | redis.js:10:16:10:23 | req.body | redis.js:10:16:10:27 | req.body.key |
@@ -665,6 +785,32 @@ edges
 | mongooseJsonParse.js:23:19:23:23 | query | mongooseJsonParse.js:20:30:20:43 | req.query.data | mongooseJsonParse.js:23:19:23:23 | query | This query depends on $@. | mongooseJsonParse.js:20:30:20:43 | req.query.data | a user-provided value |
 | mongooseModelClient.js:11:16:11:24 | { id: v } | mongooseModelClient.js:10:22:10:29 | req.body | mongooseModelClient.js:11:16:11:24 | { id: v } | This query depends on $@. | mongooseModelClient.js:10:22:10:29 | req.body | a user-provided value |
 | mongooseModelClient.js:12:16:12:34 | { id: req.body.id } | mongooseModelClient.js:12:22:12:29 | req.body | mongooseModelClient.js:12:16:12:34 | { id: req.body.id } | This query depends on $@. | mongooseModelClient.js:12:22:12:29 | req.body | a user-provided value |
+| pg-promise-types.ts:8:17:8:21 | taint | pg-promise-types.ts:7:17:7:28 | req.params.x | pg-promise-types.ts:8:17:8:21 | taint | This query depends on $@. | pg-promise-types.ts:7:17:7:28 | req.params.x | a user-provided value |
+| pg-promise.js:9:10:9:14 | query | pg-promise.js:7:16:7:34 | req.params.category | pg-promise.js:9:10:9:14 | query | This query depends on $@. | pg-promise.js:7:16:7:34 | req.params.category | a user-provided value |
+| pg-promise.js:10:11:10:15 | query | pg-promise.js:7:16:7:34 | req.params.category | pg-promise.js:10:11:10:15 | query | This query depends on $@. | pg-promise.js:7:16:7:34 | req.params.category | a user-provided value |
+| pg-promise.js:11:17:11:21 | query | pg-promise.js:7:16:7:34 | req.params.category | pg-promise.js:11:17:11:21 | query | This query depends on $@. | pg-promise.js:7:16:7:34 | req.params.category | a user-provided value |
+| pg-promise.js:12:10:12:14 | query | pg-promise.js:7:16:7:34 | req.params.category | pg-promise.js:12:10:12:14 | query | This query depends on $@. | pg-promise.js:7:16:7:34 | req.params.category | a user-provided value |
+| pg-promise.js:13:12:13:16 | query | pg-promise.js:7:16:7:34 | req.params.category | pg-promise.js:13:12:13:16 | query | This query depends on $@. | pg-promise.js:7:16:7:34 | req.params.category | a user-provided value |
+| pg-promise.js:14:18:14:22 | query | pg-promise.js:7:16:7:34 | req.params.category | pg-promise.js:14:18:14:22 | query | This query depends on $@. | pg-promise.js:7:16:7:34 | req.params.category | a user-provided value |
+| pg-promise.js:15:11:15:15 | query | pg-promise.js:7:16:7:34 | req.params.category | pg-promise.js:15:11:15:15 | query | This query depends on $@. | pg-promise.js:7:16:7:34 | req.params.category | a user-provided value |
+| pg-promise.js:16:10:16:14 | query | pg-promise.js:7:16:7:34 | req.params.category | pg-promise.js:16:10:16:14 | query | This query depends on $@. | pg-promise.js:7:16:7:34 | req.params.category | a user-provided value |
+| pg-promise.js:17:16:17:20 | query | pg-promise.js:7:16:7:34 | req.params.category | pg-promise.js:17:16:17:20 | query | This query depends on $@. | pg-promise.js:7:16:7:34 | req.params.category | a user-provided value |
+| pg-promise.js:18:12:18:16 | query | pg-promise.js:7:16:7:34 | req.params.category | pg-promise.js:18:12:18:16 | query | This query depends on $@. | pg-promise.js:7:16:7:34 | req.params.category | a user-provided value |
+| pg-promise.js:19:13:19:17 | query | pg-promise.js:7:16:7:34 | req.params.category | pg-promise.js:19:13:19:17 | query | This query depends on $@. | pg-promise.js:7:16:7:34 | req.params.category | a user-provided value |
+| pg-promise.js:22:11:22:15 | query | pg-promise.js:7:16:7:34 | req.params.category | pg-promise.js:22:11:22:15 | query | This query depends on $@. | pg-promise.js:7:16:7:34 | req.params.category | a user-provided value |
+| pg-promise.js:30:13:30:25 | req.params.id | pg-promise.js:30:13:30:25 | req.params.id | pg-promise.js:30:13:30:25 | req.params.id | This query depends on $@. | pg-promise.js:30:13:30:25 | req.params.id | a user-provided value |
+| pg-promise.js:34:13:34:25 | req.params.id | pg-promise.js:34:13:34:25 | req.params.id | pg-promise.js:34:13:34:25 | req.params.id | This query depends on $@. | pg-promise.js:34:13:34:25 | req.params.id | a user-provided value |
+| pg-promise.js:38:13:42:5 | [\\n      ... n\\n    ] | pg-promise.js:39:7:39:19 | req.params.id | pg-promise.js:38:13:42:5 | [\\n      ... n\\n    ] | This query depends on $@. | pg-promise.js:39:7:39:19 | req.params.id | a user-provided value |
+| pg-promise.js:38:13:42:5 | [\\n      ... n\\n    ] | pg-promise.js:40:7:40:21 | req.params.name | pg-promise.js:38:13:42:5 | [\\n      ... n\\n    ] | This query depends on $@. | pg-promise.js:40:7:40:21 | req.params.name | a user-provided value |
+| pg-promise.js:38:13:42:5 | [\\n      ... n\\n    ] | pg-promise.js:41:7:41:20 | req.params.foo | pg-promise.js:38:13:42:5 | [\\n      ... n\\n    ] | This query depends on $@. | pg-promise.js:41:7:41:20 | req.params.foo | a user-provided value |
+| pg-promise.js:39:7:39:19 | req.params.id | pg-promise.js:39:7:39:19 | req.params.id | pg-promise.js:39:7:39:19 | req.params.id | This query depends on $@. | pg-promise.js:39:7:39:19 | req.params.id | a user-provided value |
+| pg-promise.js:40:7:40:21 | req.params.name | pg-promise.js:40:7:40:21 | req.params.name | pg-promise.js:40:7:40:21 | req.params.name | This query depends on $@. | pg-promise.js:40:7:40:21 | req.params.name | a user-provided value |
+| pg-promise.js:47:11:47:23 | req.params.id | pg-promise.js:47:11:47:23 | req.params.id | pg-promise.js:47:11:47:23 | req.params.id | This query depends on $@. | pg-promise.js:47:11:47:23 | req.params.id | a user-provided value |
+| pg-promise.js:54:11:54:23 | req.params.id | pg-promise.js:54:11:54:23 | req.params.id | pg-promise.js:54:11:54:23 | req.params.id | This query depends on $@. | pg-promise.js:54:11:54:23 | req.params.id | a user-provided value |
+| pg-promise.js:56:14:56:29 | req.params.title | pg-promise.js:56:14:56:29 | req.params.title | pg-promise.js:56:14:56:29 | req.params.title | This query depends on $@. | pg-promise.js:56:14:56:29 | req.params.title | a user-provided value |
+| pg-promise.js:60:20:60:24 | query | pg-promise.js:7:16:7:34 | req.params.category | pg-promise.js:60:20:60:24 | query | This query depends on $@. | pg-promise.js:7:16:7:34 | req.params.category | a user-provided value |
+| pg-promise.js:63:23:63:27 | query | pg-promise.js:7:16:7:34 | req.params.category | pg-promise.js:63:23:63:27 | query | This query depends on $@. | pg-promise.js:7:16:7:34 | req.params.category | a user-provided value |
+| pg-promise.js:64:16:64:20 | query | pg-promise.js:7:16:7:34 | req.params.category | pg-promise.js:64:16:64:20 | query | This query depends on $@. | pg-promise.js:7:16:7:34 | req.params.category | a user-provided value |
 | redis.js:10:16:10:27 | req.body.key | redis.js:10:16:10:23 | req.body | redis.js:10:16:10:27 | req.body.key | This query depends on $@. | redis.js:10:16:10:23 | req.body | a user-provided value |
 | redis.js:18:16:18:18 | key | redis.js:12:15:12:22 | req.body | redis.js:18:16:18:18 | key | This query depends on $@. | redis.js:12:15:12:22 | req.body | a user-provided value |
 | redis.js:19:43:19:45 | key | redis.js:12:15:12:22 | req.body | redis.js:19:43:19:45 | key | This query depends on $@. | redis.js:12:15:12:22 | req.body | a user-provided value |

javascript/ql/test/query-tests/Security/CWE-089/untyped/pg-promise-types.ts

@@ -0,0 +1,13 @@
+import { IDatabase } from "pg-promise";
+
+export class Foo {
+  db: IDatabase;
+
+  onRequest(req, res) {
+    let taint = req.params.x;
+    this.db.one(taint); // NOT OK
+    res.end();
+  }
+}
+
+require('express')().get('/foo', (req, res) => new Foo().onRequest(req, res));

javascript/ql/test/query-tests/Security/CWE-089/untyped/pg-promise.js

@@ -0,0 +1,66 @@
+const pgp = require('pg-promise')();
+
+require('express')().get('/foo', (req, res) => {
+  const db = pgp(process.env['DB_CONNECTION_STRING']);
+
+  var query = "SELECT ITEM,PRICE FROM PRODUCT WHERE ITEM_CATEGORY='"
+             + req.params.category + "' ORDER BY PRICE";
+  
+  db.any(query); // NOT OK
+  db.many(query); // NOT OK
+  db.manyOrNone(query); // NOT OK
+  db.map(query); // NOT OK
+  db.multi(query); // NOT OK
+  db.multiResult(query); // NOT OK
+  db.none(query); // NOT OK
+  db.one(query); // NOT OK
+  db.oneOrNone(query); // NOT OK
+  db.query(query); // NOT OK
+  db.result(query); // NOT OK
+  
+  db.one({
+    text: query // NOT OK
+  });
+  db.one({
+    text: 'SELECT * FROM news where id = $1', // OK
+    values: req.params.id, // OK
+  });
+  db.one({
+    text: 'SELECT * FROM news where id = $1:raw',
+    values: req.params.id, // NOT OK - interpreted as raw parameter
+  });
+  db.one({
+    text: 'SELECT * FROM news where id = $1^',
+    values: req.params.id, // NOT OK
+  });
+  db.one({
+    text: 'SELECT * FROM news where id = $1:raw AND name = $2:raw AND foo = $3',
+    values: [
+      req.params.id, // NOT OK
+      req.params.name, // NOT OK
+      req.params.foo, // OK - not using raw interpolation
+    ]
+  });
+  db.one({
+    text: 'SELECT * FROM news where id = ${id}:raw AND name = ${name}',
+    values: {
+      id: req.params.id, // NOT OK
+      name: req.params.name, // OK - not using raw interpolation
+    }
+  });
+  db.one({
+    text: "SELECT * FROM news where id = ${id}:value AND name LIKE '%${name}:value%' AND title LIKE \"%${title}:value%\"",
+    values: {
+      id: req.params.id, // NOT OK
+      name: req.params.name, // OK - :value cannot break out of single quotes
+      title: req.params.title, // NOT OK - enclosed by wrong type of quote
+    }
+  });
+  db.task(t => {
+      return t.one(query); // NOT OK
+  });
+  db.task(
+    { cnd: t => t.one(query) }, // NOT OK
+    t => t.one(query) // NOT OK
+  );
+});

python/change-notes/2021-03-25-remove-legacy.md

@@ -0,0 +1,3 @@
+lgtm,codescanning
+* The legacy code duplication library has been removed.
+* Legacy filter queries have been removed.

python/ql/src/Filters/NotGenerated.ql

@@ -1,14 +0,0 @@
-/**
- * @name Filter: non-generated files
- * @description Only keep results that aren't (or don't appear to be) generated.
- * @kind problem
- * @id py/not-generated-file-filter
- */
-
-import python
-import external.DefectFilter
-import semmle.python.filters.GeneratedCode
-
-from DefectResult res
-where not exists(GeneratedFile f | res.getFile() = f)
-select res, res.getMessage()

python/ql/src/Filters/NotTest.ql

@@ -1,14 +0,0 @@
-/**
- * @name Filter: non-test files
- * @description Only keep results that aren't in tests
- * @kind problem
- * @id py/not-test-file-filter
- */
-
-import python
-import external.DefectFilter
-import semmle.python.filters.Tests
-
-from DefectResult res
-where not exists(TestScope s | contains(s.getLocation(), res))
-select res, res.getMessage()

python/ql/src/Lexical/FCommentedOutCode.ql

@@ -4,7 +4,6 @@
  * @kind treemap
  * @treemap.warnOn highValues
  * @metricType file
- * @precision high
  * @tags maintainability
  * @id py/lines-of-commented-out-code-in-files
  */

python/ql/src/Metrics/Dependencies/ExternalDependencies.ql

@@ -5,7 +5,6 @@
  * @kind treemap
  * @treemap.warnOn highValues
  * @metricType externalDependency
- * @precision medium
  * @id py/external-dependencies
  */
 

python/ql/src/Metrics/FLinesOfCode.ql

@@ -6,7 +6,6 @@
  * @treemap.warnOn highValues
  * @metricType file
  * @metricAggregate avg sum max
- * @precision very-high
  * @tags maintainability
  * @id py/lines-of-code-in-files
  */

python/ql/src/Metrics/FLinesOfComments.ql

@@ -6,7 +6,6 @@
  * @treemap.warnOn lowValues
  * @metricType file
  * @metricAggregate avg sum max
- * @precision very-high
  * @id py/lines-of-comments-in-files
  */
 

python/ql/src/Metrics/FLinesOfDuplicatedCode.ql

@@ -7,21 +7,12 @@
  * @treemap.warnOn highValues
  * @metricType file
  * @metricAggregate avg sum max
- * @precision high
  * @tags testability
  * @id py/duplicated-lines-in-files
  */
 
 import python
-import external.CodeDuplication
 
 from File f, int n
-where
-  n =
-    count(int line |
-      exists(DuplicateBlock d | d.sourceFile() = f |
-        line in [d.sourceStartLine() .. d.sourceEndLine()] and
-        not allowlistedLineForDuplication(f, line)
-      )
-    )
+where none()
 select f, n order by n desc

python/ql/src/Metrics/FLinesOfSimilarCode.ql

@@ -7,21 +7,12 @@
  * @treemap.warnOn highValues
  * @metricType file
  * @metricAggregate avg sum max
- * @precision high
  * @tags testability
  * @id py/similar-lines-in-files
  */
 
 import python
-import external.CodeDuplication
 
 from File f, int n
-where
-  n =
-    count(int line |
-      exists(SimilarBlock d | d.sourceFile() = f |
-        line in [d.sourceStartLine() .. d.sourceEndLine()] and
-        not allowlistedLineForDuplication(f, line)
-      )
-    )
+where none()
 select f, n order by n desc

python/ql/src/Metrics/FNumberOfTests.ql

@@ -5,7 +5,6 @@
  * @treemap.warnOn lowValues
  * @metricType file
  * @metricAggregate avg sum max
- * @precision medium
  * @id py/tests-in-files
  */
 

python/ql/src/experimental/Security-old-dataflow/CVE-2018-1281/BindToAllInterfaces.ql

@@ -3,6 +3,8 @@
  * @description Binding a socket to all interfaces opens it up to traffic from any IPv4 address
  * and is therefore associated with security risks.
  * @kind problem
+ * @id py/old/bind-socket-all-network-interfaces
+ * @problem.severity error
  */
 
 import python

python/ql/src/experimental/Security-old-dataflow/CWE-022/PathInjection.ql

@@ -2,6 +2,8 @@
  * @name OLD QUERY: Uncontrolled data used in path expression
  * @description Accessing paths influenced by users can allow an attacker to access unexpected resources.
  * @kind path-problem
+ * @problem.severity error
+ * @id py/old/path-injection
  */
 
 import python

python/ql/src/experimental/Security-old-dataflow/CWE-078/CommandInjection.ql

@@ -3,6 +3,8 @@
  * @description Using externally controlled strings in a command line may allow a malicious
  *              user to change the meaning of the command.
  * @kind path-problem
+ * @problem.severity error
+ * @id py/old/command-line-injection
  */
 
 import python

python/ql/src/experimental/Security-old-dataflow/CWE-079/ReflectedXss.ql

@@ -3,6 +3,8 @@
  * @description Writing user input directly to a web page
  *              allows for a cross-site scripting vulnerability.
  * @kind path-problem
+ * @problem.severity error
+ * @id py/old/reflective-xss
  */
 
 import python

python/ql/src/experimental/Security-old-dataflow/CWE-089/SqlInjection.ql

@@ -3,6 +3,8 @@
  * @description Building a SQL query from user-controlled sources is vulnerable to insertion of
  *              malicious SQL code by the user.
  * @kind path-problem
+ * @problem.severity error
+ * @id py/old/sql-injection
  */
 
 import python

python/ql/src/experimental/Security-old-dataflow/CWE-094/CodeInjection.ql

@@ -1,8 +1,10 @@
 /**
  * @name Code injection
- * @description Interpreting unsanitized user input as code allows a malicious user arbitrary
+ * @description OLD QUERY: Interpreting unsanitized user input as code allows a malicious user arbitrary
  *              code execution.
  * @kind path-problem
+ * @problem.severity error
+ * @id py/old/code-injection
  */
 
 import python

python/ql/src/experimental/Security-old-dataflow/CWE-326/WeakCrypto.ql

@@ -1,12 +1,9 @@
 /**
- * @name Use of weak cryptographic key
+ * @name OLD QUERY: Use of weak cryptographic key
  * @description Use of a cryptographic key that is too small may allow the encryption to be broken.
  * @kind problem
  * @problem.severity error
- * @precision high
- * @id py/weak-crypto-key
- * @tags security
- *       external/cwe/cwe-326
+ * @id py/old/weak-crypto-key
  */
 
 import python

python/ql/src/experimental/Security-old-dataflow/CWE-502/UnsafeDeserialization.ql

@@ -2,6 +2,8 @@
  * @name OLD QUERY: Deserializing untrusted input
  * @description Deserializing user-controlled data may allow attackers to execute arbitrary code.
  * @kind path-problem
+ * @id py/old/unsafe-deserialization
+ * @problem.severity error
  */
 
 import python

python/ql/src/experimental/Security-old-dataflow/CWE-601/UrlRedirect.ql

@@ -3,6 +3,8 @@
  * @description URL redirection based on unvalidated user input
  *              may cause redirection to malicious web sites.
  * @kind path-problem
+ * @problem.severity error
+ * @id py/old/url-redirection
  */
 
 import python

python/ql/src/external/CodeDuplication.qll

@@ -1,273 +0,0 @@
-/** Provides classes for detecting duplicate or similar code. */
-
-import python
-
-/** Gets the relative path of `file`, with backslashes replaced by forward slashes. */
-private string relativePath(File file) { result = file.getRelativePath().replaceAll("\\", "/") }
-
-/**
- * Holds if the `index`-th token of block `copy` is in file `file`, spanning
- * column `sc` of line `sl` to column `ec` of line `el`.
- *
- * For more information, see [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
- */
-pragma[noinline, nomagic]
-private predicate tokenLocation(File file, int sl, int sc, int ec, int el, Copy copy, int index) {
-  file = copy.sourceFile() and
-  tokens(copy, index, sl, sc, ec, el)
-}
-
-/** A token block used for detection of duplicate and similar code. */
-class Copy extends @duplication_or_similarity {
-  private int lastToken() { result = max(int i | tokens(this, i, _, _, _, _) | i) }
-
-  /** Gets the index of the token in this block starting at the location `loc`, if any. */
-  int tokenStartingAt(Location loc) {
-    tokenLocation(loc.getFile(), loc.getStartLine(), loc.getStartColumn(), _, _, this, result)
-  }
-
-  /** Gets the index of the token in this block ending at the location `loc`, if any. */
-  int tokenEndingAt(Location loc) {
-    tokenLocation(loc.getFile(), _, _, loc.getEndLine(), loc.getEndColumn(), this, result)
-  }
-
-  /** Gets the line on which the first token in this block starts. */
-  int sourceStartLine() { tokens(this, 0, result, _, _, _) }
-
-  /** Gets the column on which the first token in this block starts. */
-  int sourceStartColumn() { tokens(this, 0, _, result, _, _) }
-
-  /** Gets the line on which the last token in this block ends. */
-  int sourceEndLine() { tokens(this, this.lastToken(), _, _, result, _) }
-
-  /** Gets the column on which the last token in this block ends. */
-  int sourceEndColumn() { tokens(this, this.lastToken(), _, _, _, result) }
-
-  /** Gets the number of lines containing at least (part of) one token in this block. */
-  int sourceLines() { result = this.sourceEndLine() + 1 - this.sourceStartLine() }
-
-  /** Gets an opaque identifier for the equivalence class of this block. */
-  int getEquivalenceClass() { duplicateCode(this, _, result) or similarCode(this, _, result) }
-
-  /** Gets the source file in which this block appears. */
-  File sourceFile() {
-    exists(string name | duplicateCode(this, name, _) or similarCode(this, name, _) |
-      name.replaceAll("\\", "/") = relativePath(result)
-    )
-  }
-
-  /**
-   * Holds if this element is at the specified location.
-   * The location spans column `startcolumn` of line `startline` to
-   * column `endcolumn` of line `endline` in file `filepath`.
-   * For more information, see
-   * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
-   */
-  predicate hasLocationInfo(
-    string filepath, int startline, int startcolumn, int endline, int endcolumn
-  ) {
-    sourceFile().getAbsolutePath() = filepath and
-    startline = sourceStartLine() and
-    startcolumn = sourceStartColumn() and
-    endline = sourceEndLine() and
-    endcolumn = sourceEndColumn()
-  }
-
-  /** Gets a textual representation of this element. */
-  string toString() { result = "Copy" }
-
-  /**
-   * Gets a block that extends this one, that is, its first token is also
-   * covered by this block, but they are not the same block.
-   */
-  Copy extendingBlock() {
-    exists(File file, int sl, int sc, int ec, int el |
-      tokenLocation(file, sl, sc, ec, el, this, _) and
-      tokenLocation(file, sl, sc, ec, el, result, 0)
-    ) and
-    this != result
-  }
-}
-
-/**
- * Holds if there is a sequence of `SimilarBlock`s `start1, ..., end1` and another sequence
- * `start2, ..., end2` such that each block extends the previous one and corresponding blocks
- * have the same equivalence class, with `start` being the equivalence class of `start1` and
- * `start2`, and `end` the equivalence class of `end1` and `end2`.
- */
-predicate similar_extension(
-  SimilarBlock start1, SimilarBlock start2, SimilarBlock ext1, SimilarBlock ext2, int start, int ext
-) {
-  start1.getEquivalenceClass() = start and
-  start2.getEquivalenceClass() = start and
-  ext1.getEquivalenceClass() = ext and
-  ext2.getEquivalenceClass() = ext and
-  start1 != start2 and
-  (
-    ext1 = start1 and ext2 = start2
-    or
-    similar_extension(start1.extendingBlock(), start2.extendingBlock(), ext1, ext2, _, ext)
-  )
-}
-
-/**
- * Holds if there is a sequence of `DuplicateBlock`s `start1, ..., end1` and another sequence
- * `start2, ..., end2` such that each block extends the previous one and corresponding blocks
- * have the same equivalence class, with `start` being the equivalence class of `start1` and
- * `start2`, and `end` the equivalence class of `end1` and `end2`.
- */
-predicate duplicate_extension(
-  DuplicateBlock start1, DuplicateBlock start2, DuplicateBlock ext1, DuplicateBlock ext2, int start,
-  int ext
-) {
-  start1.getEquivalenceClass() = start and
-  start2.getEquivalenceClass() = start and
-  ext1.getEquivalenceClass() = ext and
-  ext2.getEquivalenceClass() = ext and
-  start1 != start2 and
-  (
-    ext1 = start1 and ext2 = start2
-    or
-    duplicate_extension(start1.extendingBlock(), start2.extendingBlock(), ext1, ext2, _, ext)
-  )
-}
-
-/** A block of duplicated code. */
-class DuplicateBlock extends Copy, @duplication {
-  override string toString() { result = "Duplicate code: " + sourceLines() + " duplicated lines." }
-}
-
-/** A block of similar code. */
-class SimilarBlock extends Copy, @similarity {
-  override string toString() {
-    result = "Similar code: " + sourceLines() + " almost duplicated lines."
-  }
-}
-
-/**
- * Holds if `stmt1` and `stmt2` are duplicate statements in function or toplevel `sc1` and `sc2`,
- * respectively, where `scope1` and `scope2` are not the same.
- */
-predicate duplicateStatement(Scope scope1, Scope scope2, Stmt stmt1, Stmt stmt2) {
-  exists(int equivstart, int equivend, int first, int last |
-    scope1.contains(stmt1) and
-    scope2.contains(stmt2) and
-    duplicateCoversStatement(equivstart, equivend, first, last, stmt1) and
-    duplicateCoversStatement(equivstart, equivend, first, last, stmt2) and
-    stmt1 != stmt2 and
-    scope1 != scope2
-  )
-}
-
-/**
- * Holds if statement `stmt` is covered by a sequence of `DuplicateBlock`s, where `first`
- * is the index of the token in the first block that starts at the beginning of `stmt`,
- * while `last` is the index of the token in the last block that ends at the end of `stmt`,
- * and `equivstart` and `equivend` are the equivalence classes of the first and the last
- * block, respectively.
- */
-private predicate duplicateCoversStatement(
-  int equivstart, int equivend, int first, int last, Stmt stmt
-) {
-  exists(DuplicateBlock b1, DuplicateBlock b2, Location startloc, Location endloc |
-    stmt.getLocation() = startloc and
-    stmt.getLastStatement().getLocation() = endloc and
-    first = b1.tokenStartingAt(startloc) and
-    last = b2.tokenEndingAt(endloc) and
-    b1.getEquivalenceClass() = equivstart and
-    b2.getEquivalenceClass() = equivend and
-    duplicate_extension(b1, _, b2, _, equivstart, equivend)
-  )
-}
-
-/**
- * Holds if `sc1` is a function or toplevel with `total` lines, and `scope2` is a function or
- * toplevel that has `duplicate` lines in common with `scope1`.
- */
-predicate duplicateStatements(Scope scope1, Scope scope2, int duplicate, int total) {
-  duplicate = strictcount(Stmt stmt | duplicateStatement(scope1, scope2, stmt, _)) and
-  total = strictcount(Stmt stmt | scope1.contains(stmt))
-}
-
-/**
- * Find pairs of scopes that are identical or almost identical
- */
-predicate duplicateScopes(Scope s, Scope other, float percent, string message) {
-  exists(int total, int duplicate | duplicateStatements(s, other, duplicate, total) |
-    percent = 100.0 * duplicate / total and
-    percent >= 80.0 and
-    if duplicate = total
-    then message = "All " + total + " statements in " + s.getName() + " are identical in $@."
-    else
-      message =
-        duplicate + " out of " + total + " statements in " + s.getName() + " are duplicated in $@."
-  )
-}
-
-/**
- * Holds if `stmt1` and `stmt2` are similar statements in function or toplevel `scope1` and `scope2`,
- * respectively, where `scope1` and `scope2` are not the same.
- */
-private predicate similarStatement(Scope scope1, Scope scope2, Stmt stmt1, Stmt stmt2) {
-  exists(int start, int end, int first, int last |
-    scope1.contains(stmt1) and
-    scope2.contains(stmt2) and
-    similarCoversStatement(start, end, first, last, stmt1) and
-    similarCoversStatement(start, end, first, last, stmt2) and
-    stmt1 != stmt2 and
-    scope1 != scope2
-  )
-}
-
-/**
- * Holds if statement `stmt` is covered by a sequence of `SimilarBlock`s, where `first`
- * is the index of the token in the first block that starts at the beginning of `stmt`,
- * while `last` is the index of the token in the last block that ends at the end of `stmt`,
- * and `equivstart` and `equivend` are the equivalence classes of the first and the last
- * block, respectively.
- */
-private predicate similarCoversStatement(
-  int equivstart, int equivend, int first, int last, Stmt stmt
-) {
-  exists(SimilarBlock b1, SimilarBlock b2, Location startloc, Location endloc |
-    stmt.getLocation() = startloc and
-    stmt.getLastStatement().getLocation() = endloc and
-    first = b1.tokenStartingAt(startloc) and
-    last = b2.tokenEndingAt(endloc) and
-    b1.getEquivalenceClass() = equivstart and
-    b2.getEquivalenceClass() = equivend and
-    similar_extension(b1, _, b2, _, equivstart, equivend)
-  )
-}
-
-/**
- * Holds if `sc1` is a function or toplevel with `total` lines, and `scope2` is a function or
- * toplevel that has `similar` similar lines to `scope1`.
- */
-private predicate similarStatements(Scope scope1, Scope scope2, int similar, int total) {
-  similar = strictcount(Stmt stmt | similarStatement(scope1, scope2, stmt, _)) and
-  total = strictcount(Stmt stmt | scope1.contains(stmt))
-}
-
-/**
- * Find pairs of scopes that are similar
- */
-predicate similarScopes(Scope s, Scope other, float percent, string message) {
-  exists(int total, int similar | similarStatements(s, other, similar, total) |
-    percent = 100.0 * similar / total and
-    percent >= 80.0 and
-    if similar = total
-    then message = "All statements in " + s.getName() + " are similar in $@."
-    else
-      message =
-        similar + " out of " + total + " statements in " + s.getName() + " are similar in $@."
-  )
-}
-
-/**
- * Holds if the line is acceptable as a duplicate.
- * This is true for blocks of import statements.
- */
-predicate allowlistedLineForDuplication(File f, int line) {
-  exists(ImportingStmt i | i.getLocation().getFile() = f and i.getLocation().getStartLine() = line)
-}

python/ql/src/external/DuplicateBlock.ql

@@ -16,19 +16,7 @@
  */
 
 import python
-import CodeDuplication
 
-predicate sorted_by_location(DuplicateBlock x, DuplicateBlock y) {
-  if x.sourceFile() = y.sourceFile()
-  then x.sourceStartLine() < y.sourceStartLine()
-  else x.sourceFile().getAbsolutePath() < y.sourceFile().getAbsolutePath()
-}
-
-from DuplicateBlock d, DuplicateBlock other
-where
-  d.sourceLines() > 10 and
-  other.getEquivalenceClass() = d.getEquivalenceClass() and
-  sorted_by_location(other, d)
-select d,
-  "Duplicate code: " + d.sourceLines() + " lines are duplicated at " +
-    other.sourceFile().getShortName() + ":" + other.sourceStartLine().toString()
+from BasicBlock d
+where none()
+select d, "Duplicate code: " + "-1" + " lines are duplicated at " + "<file>" + ":" + "-1"

python/ql/src/external/DuplicateFunction.ql

@@ -16,15 +16,7 @@
  */
 
 import python
-import CodeDuplication
 
-predicate relevant(Function m) { m.getMetrics().getNumberOfLinesOfCode() > 5 }
-
-from Function m, Function other, string message, int percent
-where
-  duplicateScopes(m, other, percent, message) and
-  relevant(m) and
-  percent > 95.0 and
-  not duplicateScopes(m.getEnclosingModule(), other.getEnclosingModule(), _, _) and
-  not duplicateScopes(m.getScope(), other.getScope(), _, _)
+from Function m, Function other, string message
+where none()
 select m, message, other, other.getName()

python/ql/src/external/MostlyDuplicateClass.ql

@@ -16,11 +16,7 @@
  */
 
 import python
-import CodeDuplication
 
 from Class c, Class other, string message
-where
-  duplicateScopes(c, other, _, message) and
-  count(c.getAStmt()) > 3 and
-  not duplicateScopes(c.getEnclosingModule(), _, _, _)
+where none()
 select c, message, other, other.getName()

python/ql/src/external/MostlyDuplicateFile.ql

@@ -16,8 +16,7 @@
  */
 
 import python
-import CodeDuplication
 
-from Module m, Module other, int percent, string message
-where duplicateScopes(m, other, percent, message)
+from Module m, Module other, string message
+where none()
 select m, message, other, other.getName()

python/ql/src/external/MostlySimilarFile.ql

@@ -16,8 +16,7 @@
  */
 
 import python
-import CodeDuplication
 
 from Module m, Module other, string message
-where similarScopes(m, other, _, message)
+where none()
 select m, message, other, other.getName()

python/ql/src/external/SimilarFunction.ql

@@ -16,16 +16,7 @@
  */
 
 import python
-import CodeDuplication
 
-predicate relevant(Function m) { m.getMetrics().getNumberOfLinesOfCode() > 10 }
-
-from Function m, Function other, string message, int percent
-where
-  similarScopes(m, other, percent, message) and
-  relevant(m) and
-  percent > 95.0 and
-  not duplicateScopes(m, other, _, _) and
-  not duplicateScopes(m.getEnclosingModule(), other.getEnclosingModule(), _, _) and
-  not duplicateScopes(m.getScope(), other.getScope(), _, _)
+from Function m, Function other, string message
+where none()
 select m, message, other, other.getName()

python/ql/src/semmle/python/dataflow/new/internal/DataFlowPrivate.qll

@@ -1517,10 +1517,13 @@ predicate forReadStep(CfgNode nodeFrom, Content c, Node nodeTo) {
     or
     c instanceof SetElementContent
     or
-    c instanceof TupleElementContent
+    c = small_tuple()
   )
 }
 
+pragma[noinline]
+TupleElementContent small_tuple() { result.getIndex() <= 7 }
+
 /**
  * Holds if `nodeTo` is a read of an attribute (corresponding to `c`) of the object in `nodeFrom`.
  *

python/ql/src/semmle/python/dataflow/new/internal/DataFlowPublic.qll

@@ -467,14 +467,22 @@ class BarrierGuard extends GuardNode {
   }
 }
 
+private predicate comes_from_cfgnode(Node node) {
+  exists(CfgNode first, Node second |
+    simpleLocalFlowStep(first, second) and
+    simpleLocalFlowStep*(second, node)
+  )
+}
+
 /**
  * A data flow node that is a source of local flow. This includes things like
  * - Expressions
  * - Function parameters
  */
 class LocalSourceNode extends Node {
+  cached
   LocalSourceNode() {
-    not simpleLocalFlowStep+(any(CfgNode n), this) and
+    not comes_from_cfgnode(this) and
     not this instanceof ModuleVariableNode
     or
     this = any(ModuleVariableNode mvn).getARead()
@@ -522,15 +530,12 @@ private module Cached {
    * The slightly backwards parametering ordering is to force correct indexing.
    */
   cached
-  predicate hasLocalSource(Node sink, Node source) {
-    // Declaring `source` to be a `SourceNode` currently causes a redundant check in the
-    // recursive case, so instead we check it explicitly here.
-    source = sink and
-    source instanceof LocalSourceNode
+  predicate hasLocalSource(Node sink, LocalSourceNode source) {
+    source = sink
     or
-    exists(Node mid |
-      hasLocalSource(mid, source) and
-      simpleLocalFlowStep(mid, sink)
+    exists(Node second |
+      simpleLocalFlowStep(source, second) and
+      simpleLocalFlowStep*(second, sink)
     )
   }
 

python/ql/test/library-tests/DuplicateCode/Duplicate.expected

@@ -1,2 +0,0 @@
-| Duplicate code: 34 duplicated lines. | Duplicate code: 34 duplicated lines. | duplicate_test.py | 9 | 42 |
-| Duplicate code: 80 duplicated lines. | Duplicate code: 80 duplicated lines. | duplicate_test.py | 84 | 163 |

python/ql/test/library-tests/DuplicateCode/Duplicate.ql

@@ -1,23 +0,0 @@
-/**
- * @name Duplicate
- * @description Insert description here...
- * @kind table
- * @problem.severity warning
- */
-
-import python
-import external.CodeDuplication
-
-predicate lexically_sorted(DuplicateBlock dup1, DuplicateBlock dup2) {
-  dup1.sourceFile().getAbsolutePath() < dup2.sourceFile().getAbsolutePath()
-  or
-  dup1.sourceFile().getAbsolutePath() = dup2.sourceFile().getAbsolutePath() and
-  dup1.sourceStartLine() < dup2.sourceStartLine()
-}
-
-from DuplicateBlock dup1, DuplicateBlock dup2
-where
-  dup1.getEquivalenceClass() = dup2.getEquivalenceClass() and
-  lexically_sorted(dup1, dup2)
-select dup1.toString(), dup2.toString(), dup1.sourceFile().getShortName(), dup1.sourceStartLine(),
-  dup1.sourceEndLine()

python/ql/test/library-tests/DuplicateCode/DuplicateStatements.ql

@@ -1,26 +0,0 @@
-/**
- * @name DuplicateStatements
- * @description Insert description here...
- * @kind problem
- * @problem.severity warning
- */
-
-import python
-import external.CodeDuplication
-
-predicate mostlyDuplicateFunction(Function f) {
-  exists(int covered, int total, Function other, int percent |
-    duplicateStatements(f, other, covered, total) and
-    covered != total and
-    total > 5 and
-    covered * 100 / total = percent and
-    percent > 80 and
-    not exists(Scope s | s = f.getScope*() | duplicateScopes(s, _, _, _))
-  )
-}
-
-from Stmt s
-where
-  mostlyDuplicateFunction(s.getScope()) and
-  not duplicateStatement(s.getScope(), _, s, _)
-select s.toString(), s.getLocation().toString()

python/ql/test/library-tests/DuplicateCode/Similar.expected

@@ -1,23 +0,0 @@
-| duplicate_test.py:9:1:20:17 | Similar code: 12 almost duplicated lines. | duplicate_test.py:47:1:58:17 | Similar code: 12 almost duplicated lines. | duplicate_test.py | 9 | 20 |
-| duplicate_test.py:9:1:20:17 | Similar code: 12 almost duplicated lines. | duplicate_test.py:249:1:260:17 | Similar code: 12 almost duplicated lines. | duplicate_test.py | 9 | 20 |
-| duplicate_test.py:9:1:20:17 | Similar code: 12 almost duplicated lines. | duplicate_test.py:287:1:298:17 | Similar code: 12 almost duplicated lines. | duplicate_test.py | 9 | 20 |
-| duplicate_test.py:14:8:25:13 | Similar code: 12 almost duplicated lines. | duplicate_test.py:52:8:63:13 | Similar code: 12 almost duplicated lines. | duplicate_test.py | 14 | 25 |
-| duplicate_test.py:14:8:25:13 | Similar code: 12 almost duplicated lines. | duplicate_test.py:254:8:265:13 | Similar code: 12 almost duplicated lines. | duplicate_test.py | 14 | 25 |
-| duplicate_test.py:20:28:42:31 | Similar code: 23 almost duplicated lines. | duplicate_test.py:58:28:80:31 | Similar code: 23 almost duplicated lines. | duplicate_test.py | 20 | 42 |
-| duplicate_test.py:20:28:42:31 | Similar code: 23 almost duplicated lines. | duplicate_test.py:260:28:282:31 | Similar code: 23 almost duplicated lines. | duplicate_test.py | 20 | 42 |
-| duplicate_test.py:20:28:42:31 | Similar code: 23 almost duplicated lines. | duplicate_test.py:296:40:318:31 | Similar code: 23 almost duplicated lines. | duplicate_test.py | 20 | 42 |
-| duplicate_test.py:36:1:47:0 | Similar code: 12 almost duplicated lines. | duplicate_test.py:74:1:84:0 | Similar code: 11 almost duplicated lines. | duplicate_test.py | 36 | 47 |
-| duplicate_test.py:36:1:47:0 | Similar code: 12 almost duplicated lines. | duplicate_test.py:276:1:287:0 | Similar code: 12 almost duplicated lines. | duplicate_test.py | 36 | 47 |
-| duplicate_test.py:36:22:56:26 | Similar code: 21 almost duplicated lines. | duplicate_test.py:276:21:296:26 | Similar code: 21 almost duplicated lines. | duplicate_test.py | 36 | 56 |
-| duplicate_test.py:42:22:57:9 | Similar code: 16 almost duplicated lines. | duplicate_test.py:245:20:259:9 | Similar code: 15 almost duplicated lines. | duplicate_test.py | 42 | 57 |
-| duplicate_test.py:42:22:57:9 | Similar code: 16 almost duplicated lines. | duplicate_test.py:282:22:297:9 | Similar code: 16 almost duplicated lines. | duplicate_test.py | 42 | 57 |
-| duplicate_test.py:47:1:58:17 | Similar code: 12 almost duplicated lines. | duplicate_test.py:249:1:260:17 | Similar code: 12 almost duplicated lines. | duplicate_test.py | 47 | 58 |
-| duplicate_test.py:47:1:58:17 | Similar code: 12 almost duplicated lines. | duplicate_test.py:287:1:298:17 | Similar code: 12 almost duplicated lines. | duplicate_test.py | 47 | 58 |
-| duplicate_test.py:52:8:63:13 | Similar code: 12 almost duplicated lines. | duplicate_test.py:254:8:265:13 | Similar code: 12 almost duplicated lines. | duplicate_test.py | 52 | 63 |
-| duplicate_test.py:58:28:80:31 | Similar code: 23 almost duplicated lines. | duplicate_test.py:260:28:282:31 | Similar code: 23 almost duplicated lines. | duplicate_test.py | 58 | 80 |
-| duplicate_test.py:58:28:80:31 | Similar code: 23 almost duplicated lines. | duplicate_test.py:296:40:318:31 | Similar code: 23 almost duplicated lines. | duplicate_test.py | 58 | 80 |
-| duplicate_test.py:74:1:84:0 | Similar code: 11 almost duplicated lines. | duplicate_test.py:276:1:287:0 | Similar code: 12 almost duplicated lines. | duplicate_test.py | 74 | 84 |
-| duplicate_test.py:82:25:163:24 | Similar code: 82 almost duplicated lines. | duplicate_test.py:163:24:245:24 | Similar code: 83 almost duplicated lines. | duplicate_test.py | 82 | 163 |
-| duplicate_test.py:245:20:259:9 | Similar code: 15 almost duplicated lines. | duplicate_test.py:282:22:297:9 | Similar code: 16 almost duplicated lines. | duplicate_test.py | 245 | 259 |
-| duplicate_test.py:249:1:260:17 | Similar code: 12 almost duplicated lines. | duplicate_test.py:287:1:298:17 | Similar code: 12 almost duplicated lines. | duplicate_test.py | 249 | 260 |
-| duplicate_test.py:260:28:282:31 | Similar code: 23 almost duplicated lines. | duplicate_test.py:296:40:318:31 | Similar code: 23 almost duplicated lines. | duplicate_test.py | 260 | 282 |

python/ql/test/library-tests/DuplicateCode/Similar.ql

@@ -1,22 +0,0 @@
-/**
- * @name Similar
- * @description Insert description here...
- * @kind table
- * @problem.severity warning
- */
-
-import python
-import external.CodeDuplication
-
-predicate lexically_sorted(SimilarBlock dup1, SimilarBlock dup2) {
-  dup1.sourceFile().getAbsolutePath() < dup2.sourceFile().getAbsolutePath()
-  or
-  dup1.sourceFile().getAbsolutePath() = dup2.sourceFile().getAbsolutePath() and
-  dup1.sourceStartLine() < dup2.sourceStartLine()
-}
-
-from SimilarBlock dup1, SimilarBlock dup2
-where
-  dup1.getEquivalenceClass() = dup2.getEquivalenceClass() and
-  lexically_sorted(dup1, dup2)
-select dup1, dup2, dup1.sourceFile().getShortName(), dup1.sourceStartLine(), dup1.sourceEndLine()

python/ql/test/library-tests/DuplicateCode/duplicate_test.py

@@ -1,321 +0,0 @@
-#Code Duplication
-
-
-#Exact duplication of function
-
-#Code copied from stdlib, copyright PSF.
-#See http://www.python.org/download/releases/2.7/license/
-
-def dis(x=None):
-    """Disassemble classes, methods, functions, or code.
-
-    With no argument, disassemble the last traceback.
-
-    """
-    if x is None:
-        distb()
-        return
-    if isinstance(x, types.InstanceType):
-        x = x.__class__
-    if hasattr(x, 'im_func'):
-        x = x.im_func
-    if hasattr(x, 'func_code'):
-        x = x.func_code
-    if hasattr(x, '__dict__'):
-        items = x.__dict__.items()
-        items.sort()
-        for name, x1 in items:
-            if isinstance(x1, _have_code):
-                print "Disassembly of %s:" % name
-                try:
-                    dis(x1)
-                except TypeError, msg:
-                    print "Sorry:", msg
-                print
-    elif hasattr(x, 'co_code'):
-        disassemble(x)
-    elif isinstance(x, str):
-        disassemble_string(x)
-    else:
-        raise TypeError, \
-              "don't know how to disassemble %s objects" % \
-              type(x).__name__
-
-
-#And duplicate version
-
-def dis2(x=None):
-    """Disassemble classes, methods, functions, or code.
-
-    With no argument, disassemble the last traceback.
-
-    """
-    if x is None:
-        distb()
-        return
-    if isinstance(x, types.InstanceType):
-        x = x.__class__
-    if hasattr(x, 'im_func'):
-        x = x.im_func
-    if hasattr(x, 'func_code'):
-        x = x.func_code
-    if hasattr(x, '__dict__'):
-        items = x.__dict__.items()
-        items.sort()
-        for name, x1 in items:
-            if isinstance(x1, _have_code):
-                print "Disassembly of %s:" % name
-                try:
-                    dis(x1)
-                except TypeError, msg:
-                    print "Sorry:", msg
-                print
-    elif hasattr(x, 'co_code'):
-        disassemble(x)
-    elif isinstance(x, str):
-        disassemble_string(x)
-    else:
-        raise TypeError, \
-              "don't know how to disassemble %s objects" % \
-              type(x).__name__
-
-#Exactly duplicate class
-
-class Popen3:
-    """Class representing a child process.  Normally, instances are created
-    internally by the functions popen2() and popen3()."""
-
-    sts = -1                    # Child not completed yet
-
-    def __init__(self, cmd, capturestderr=False, bufsize=-1):
-        """The parameter 'cmd' is the shell command to execute in a
-        sub-process.  On UNIX, 'cmd' may be a sequence, in which case arguments
-        will be passed directly to the program without shell intervention (as
-        with os.spawnv()).  If 'cmd' is a string it will be passed to the shell
-        (as with os.system()).   The 'capturestderr' flag, if true, specifies
-        that the object should capture standard error output of the child
-        process.  The default is false.  If the 'bufsize' parameter is
-        specified, it specifies the size of the I/O buffers to/from the child
-        process."""
-        _cleanup()
-        self.cmd = cmd
-        p2cread, p2cwrite = os.pipe()
-        c2pread, c2pwrite = os.pipe()
-        if capturestderr:
-            errout, errin = os.pipe()
-        self.pid = os.fork()
-        if self.pid == 0:
-            # Child
-            os.dup2(p2cread, 0)
-            os.dup2(c2pwrite, 1)
-            if capturestderr:
-                os.dup2(errin, 2)
-            self._run_child(cmd)
-        os.close(p2cread)
-        self.tochild = os.fdopen(p2cwrite, 'w', bufsize)
-        os.close(c2pwrite)
-        self.fromchild = os.fdopen(c2pread, 'r', bufsize)
-        if capturestderr:
-            os.close(errin)
-            self.childerr = os.fdopen(errout, 'r', bufsize)
-        else:
-            self.childerr = None
-
-    def __del__(self):
-        # In case the child hasn't been waited on, check if it's done.
-        self.poll(_deadstate=sys.maxint)
-        if self.sts < 0:
-            if _active is not None:
-                # Child is still running, keep us alive until we can wait on it.
-                _active.append(self)
-
-    def _run_child(self, cmd):
-        if isinstance(cmd, basestring):
-            cmd = ['/bin/sh', '-c', cmd]
-        os.closerange(3, MAXFD)
-        try:
-            os.execvp(cmd[0], cmd)
-        finally:
-            os._exit(1)
-
-    def poll(self, _deadstate=None):
-        """Return the exit status of the child process if it has finished,
-        or -1 if it hasn't finished yet."""
-        if self.sts < 0:
-            try:
-                pid, sts = os.waitpid(self.pid, os.WNOHANG)
-                # pid will be 0 if self.pid hasn't terminated
-                if pid == self.pid:
-                    self.sts = sts
-            except os.error:
-                if _deadstate is not None:
-                    self.sts = _deadstate
-        return self.sts
-
-    def wait(self):
-        """Wait for and return the exit status of the child process."""
-        if self.sts < 0:
-            pid, sts = os.waitpid(self.pid, 0)
-            # This used to be a test, but it is believed to be
-            # always true, so I changed it to an assertion - mvl
-            assert pid == self.pid
-            self.sts = sts
-        return self.sts
-
-
-class Popen3Again:
-    """Class representing a child process.  Normally, instances are created
-    internally by the functions popen2() and popen3()."""
-
-    sts = -1                    # Child not completed yet
-
-    def __init__(self, cmd, capturestderr=False, bufsize=-1):
-        """The parameter 'cmd' is the shell command to execute in a
-        sub-process.  On UNIX, 'cmd' may be a sequence, in which case arguments
-        will be passed directly to the program without shell intervention (as
-        with os.spawnv()).  If 'cmd' is a string it will be passed to the shell
-        (as with os.system()).   The 'capturestderr' flag, if true, specifies
-        that the object should capture standard error output of the child
-        process.  The default is false.  If the 'bufsize' parameter is
-        specified, it specifies the size of the I/O buffers to/from the child
-        process."""
-        _cleanup()
-        self.cmd = cmd
-        p2cread, p2cwrite = os.pipe()
-        c2pread, c2pwrite = os.pipe()
-        if capturestderr:
-            errout, errin = os.pipe()
-        self.pid = os.fork()
-        if self.pid == 0:
-            # Child
-            os.dup2(p2cread, 0)
-            os.dup2(c2pwrite, 1)
-            if capturestderr:
-                os.dup2(errin, 2)
-            self._run_child(cmd)
-        os.close(p2cread)
-        self.tochild = os.fdopen(p2cwrite, 'w', bufsize)
-        os.close(c2pwrite)
-        self.fromchild = os.fdopen(c2pread, 'r', bufsize)
-        if capturestderr:
-            os.close(errin)
-            self.childerr = os.fdopen(errout, 'r', bufsize)
-        else:
-            self.childerr = None
-
-    def __del__(self):
-        # In case the child hasn't been waited on, check if it's done.
-        self.poll(_deadstate=sys.maxint)
-        if self.sts < 0:
-            if _active is not None:
-                # Child is still running, keep us alive until we can wait on it.
-                _active.append(self)
-
-    def _run_child(self, cmd):
-        if isinstance(cmd, basestring):
-            cmd = ['/bin/sh', '-c', cmd]
-        os.closerange(3, MAXFD)
-        try:
-            os.execvp(cmd[0], cmd)
-        finally:
-            os._exit(1)
-
-    def poll(self, _deadstate=None):
-        """Return the exit status of the child process if it has finished,
-        or -1 if it hasn't finished yet."""
-        if self.sts < 0:
-            try:
-                pid, sts = os.waitpid(self.pid, os.WNOHANG)
-                # pid will be 0 if self.pid hasn't terminated
-                if pid == self.pid:
-                    self.sts = sts
-            except os.error:
-                if _deadstate is not None:
-                    self.sts = _deadstate
-        return self.sts
-
-    def wait(self):
-        """Wait for and return the exit status of the child process."""
-        if self.sts < 0:
-            pid, sts = os.waitpid(self.pid, 0)
-            # This used to be a test, but it is believed to be
-            # always true, so I changed it to an assertion - mvl
-            assert pid == self.pid
-            self.sts = sts
-        return self.sts
-
-#Duplicate function with identifiers changed
-
-def dis3(y=None):
-    """frobnicate classes, methods, functions, or code.
-
-    With no argument, frobnicate the last traceback.
-
-    """
-    if y is None:
-        distb()
-        return
-    if isinstance(y, types.InstanceType):
-        y = y.__class__
-    if hasattr(y, 'im_func'):
-        y = y.im_func
-    if hasattr(y, 'func_code'):
-        y = y.func_code
-    if hasattr(y, '__dict__'):
-        items = y.__dict__.items()
-        items.sort()
-        for name, y1 in items:
-            if isinstance(y1, _have_code):
-                print "Disassembly of %s:" % name
-                try:
-                    dis(y1)
-                except TypeError, msg:
-                    print "Sorry:", msg
-                print
-    elif hasattr(y, 'co_code'):
-        frobnicate(y)
-    elif isinstance(y, str):
-        frobnicate_string(y)
-    else:
-        raise TypeError, \
-              "don't know how to frobnicate %s objects" % \
-              type(y).__name__
-
-
-#Mostly similar function with changed identifiers
-
-def dis5(z=None):
-    """splat classes, methods, functions, or code.
-
-    With no argument, splat the last traceback.
-
-    """
-    if z is None:
-        distb()
-        return
-    if isinstance(z, types.InstanceType):
-        z = z.__class__
-    if hasattr(y, 'func_code'):
-        y = y.func_code
-    if hasattr(z, '__dict__'):
-        items = z.__dict__.items()
-        items.sort()
-        for name, z1 in items:
-            if isinstance(z1, _have_code):
-                print "Disassembly of %s:" % name
-                try:
-                    dis(z1)
-                except TypeError, msg:
-                    print "Sorry:", msg
-                print
-    elif hasattr(z, 'co_code'):
-        splat(z)
-    elif isinstance(z, str):
-        splat_string(z)
-    else:
-        raise TypeError, \
-              "don't know how to splat %s objects" % \
-              type(z).__name__
-
-
-

python/ql/test/query-tests/UselessCode/DuplicateCode/DuplicateBlock.expected

@@ -1,8 +0,0 @@
-| duplicate_test.py:47:9:60:17 | Duplicate code: 14 duplicated lines. | Duplicate code: 14 lines are duplicated at duplicate_test.py:9 |
-| duplicate_test.py:56:18:66:25 | Duplicate code: 11 duplicated lines. | Duplicate code: 11 lines are duplicated at duplicate_test.py:18 |
-| duplicate_test.py:61:24:80:32 | Duplicate code: 20 duplicated lines. | Duplicate code: 20 lines are duplicated at duplicate_test.py:23 |
-| duplicate_test.py:166:18:245:24 | Duplicate code: 80 duplicated lines. | Duplicate code: 80 lines are duplicated at duplicate_test.py:84 |
-| duplicate_test.py:287:9:300:17 | Duplicate code: 14 duplicated lines. | Duplicate code: 14 lines are duplicated at duplicate_test.py:9 |
-| duplicate_test.py:287:9:300:17 | Duplicate code: 14 duplicated lines. | Duplicate code: 14 lines are duplicated at duplicate_test.py:47 |
-| duplicate_test.py:299:22:318:32 | Duplicate code: 20 duplicated lines. | Duplicate code: 20 lines are duplicated at duplicate_test.py:23 |
-| duplicate_test.py:299:22:318:32 | Duplicate code: 20 duplicated lines. | Duplicate code: 20 lines are duplicated at duplicate_test.py:61 |

python/ql/test/query-tests/UselessCode/DuplicateCode/DuplicateBlock.qlref

@@ -1 +0,0 @@
-external/DuplicateBlock.ql

python/ql/test/query-tests/UselessCode/DuplicateCode/DuplicateFunction.expected

@@ -1,4 +0,0 @@
-| duplicate_test.py:9:1:9:16 | Function dis | All 26 statements in dis are identical in $@. | duplicate_test.py:47:1:47:17 | Function dis2 | dis2 |
-| duplicate_test.py:47:1:47:17 | Function dis2 | All 26 statements in dis2 are identical in $@. | duplicate_test.py:9:1:9:16 | Function dis | dis |
-| duplicate_test.py:287:1:287:17 | Function dis4 | All 24 statements in dis4 are identical in $@. | duplicate_test.py:9:1:9:16 | Function dis | dis |
-| duplicate_test.py:287:1:287:17 | Function dis4 | All 24 statements in dis4 are identical in $@. | duplicate_test.py:47:1:47:17 | Function dis2 | dis2 |

python/ql/test/query-tests/UselessCode/DuplicateCode/DuplicateFunction.qlref

@@ -1 +0,0 @@
-external/DuplicateFunction.ql

python/ql/test/query-tests/UselessCode/DuplicateCode/MostlyDuplicateClass.expected

@@ -1,2 +0,0 @@
-| duplicate_test.py:84:1:84:13 | Class Popen3 | All 55 statements in Popen3 are identical in $@. | duplicate_test.py:166:1:166:18 | Class Popen3Again | Popen3Again |
-| duplicate_test.py:166:1:166:18 | Class Popen3Again | All 55 statements in Popen3Again are identical in $@. | duplicate_test.py:84:1:84:13 | Class Popen3 | Popen3 |

python/ql/test/query-tests/UselessCode/DuplicateCode/MostlyDuplicateClass.qlref

@@ -1 +0,0 @@
-external/MostlyDuplicateClass.ql

python/ql/test/query-tests/UselessCode/DuplicateCode/MostlyDuplicateFile.qlref

@@ -1 +0,0 @@
-external/MostlyDuplicateFile.ql

python/ql/test/query-tests/UselessCode/DuplicateCode/MostlySimilarFile.qlref

@@ -1 +0,0 @@
-external/MostlySimilarFile.ql

python/ql/test/query-tests/UselessCode/DuplicateCode/SimilarFunction.expected

@@ -1,12 +0,0 @@
-| duplicate_test.py:9:1:9:16 | Function dis | All statements in dis are similar in $@. | duplicate_test.py:249:1:249:17 | Function dis3 | dis3 |
-| duplicate_test.py:9:1:9:16 | Function dis | All statements in dis are similar in $@. | duplicate_test.py:323:1:323:17 | Function dis5 | dis5 |
-| duplicate_test.py:47:1:47:17 | Function dis2 | All statements in dis2 are similar in $@. | duplicate_test.py:249:1:249:17 | Function dis3 | dis3 |
-| duplicate_test.py:47:1:47:17 | Function dis2 | All statements in dis2 are similar in $@. | duplicate_test.py:323:1:323:17 | Function dis5 | dis5 |
-| duplicate_test.py:249:1:249:17 | Function dis3 | All statements in dis3 are similar in $@. | duplicate_test.py:9:1:9:16 | Function dis | dis |
-| duplicate_test.py:249:1:249:17 | Function dis3 | All statements in dis3 are similar in $@. | duplicate_test.py:47:1:47:17 | Function dis2 | dis2 |
-| duplicate_test.py:249:1:249:17 | Function dis3 | All statements in dis3 are similar in $@. | duplicate_test.py:323:1:323:17 | Function dis5 | dis5 |
-| duplicate_test.py:287:1:287:17 | Function dis4 | All statements in dis4 are similar in $@. | duplicate_test.py:249:1:249:17 | Function dis3 | dis3 |
-| duplicate_test.py:287:1:287:17 | Function dis4 | All statements in dis4 are similar in $@. | duplicate_test.py:323:1:323:17 | Function dis5 | dis5 |
-| duplicate_test.py:323:1:323:17 | Function dis5 | All statements in dis5 are similar in $@. | duplicate_test.py:9:1:9:16 | Function dis | dis |
-| duplicate_test.py:323:1:323:17 | Function dis5 | All statements in dis5 are similar in $@. | duplicate_test.py:47:1:47:17 | Function dis2 | dis2 |
-| duplicate_test.py:323:1:323:17 | Function dis5 | All statements in dis5 are similar in $@. | duplicate_test.py:249:1:249:17 | Function dis3 | dis3 |

python/ql/test/query-tests/UselessCode/DuplicateCode/SimilarFunction.qlref

@@ -1 +0,0 @@
-external/SimilarFunction.ql

python/ql/test/query-tests/UselessCode/DuplicateCode/duplicate_test.py

@@ -1,358 +0,0 @@
-#Code Duplication
-
-
-#Exact duplication of function
-
-#Code copied from stdlib, copyright PSF.
-#See http://www.python.org/download/releases/2.7/license/
-
-def dis(x=None):
-    """Disassemble classes, methods, functions, or code.
-
-    With no argument, disassemble the last traceback.
-
-    """
-    if x is None:
-        distb()
-        return
-    if isinstance(x, types.InstanceType):
-        x = x.__class__
-    if hasattr(x, 'im_func'):
-        x = x.im_func
-    if hasattr(x, 'func_code'):
-        x = x.func_code
-    if hasattr(x, '__dict__'):
-        items = x.__dict__.items()
-        items.sort()
-        for name, x1 in items:
-            if isinstance(x1, _have_code):
-                print("Disassembly of %s:" % name)
-                try:
-                    dis(x1)
-                except TypeError(msg):
-                    print("Sorry:", msg)
-                print()
-    elif hasattr(x, 'co_code'):
-        disassemble(x)
-    elif isinstance(x, str):
-        disassemble_string(x)
-    else:
-        raise TypeError(
-              "don't know how to disassemble %s objects" %
-              type(x).__name__)
-
-
-#And duplicate version
-
-def dis2(x=None):
-    """Disassemble classes, methods, functions, or code.
-
-    With no argument, disassemble the last traceback.
-
-    """
-    if x is None:
-        distb()
-        return
-    if isinstance(x, types.InstanceType):
-        x = x.__class__
-    if hasattr(x, 'im_func'):
-        x = x.im_func
-    if hasattr(x, 'func_code'):
-        x = x.func_code
-    if hasattr(x, '__dict__'):
-        items = x.__dict__.items()
-        items.sort()
-        for name, x1 in items:
-            if isinstance(x1, _have_code):
-                print("Disassembly of %s:" % name)
-                try:
-                    dis(x1)
-                except TypeError(msg):
-                    print("Sorry:", msg)
-                print()
-    elif hasattr(x, 'co_code'):
-        disassemble(x)
-    elif isinstance(x, str):
-        disassemble_string(x)
-    else:
-        raise TypeError(
-              "don't know how to disassemble %s objects" %
-              type(x).__name__)
-
-#Exactly duplicate class
-
-class Popen3:
-    """Class representing a child process.  Normally, instances are created
-    internally by the functions popen2() and popen3()."""
-
-    sts = -1                    # Child not completed yet
-
-    def __init__(self, cmd, capturestderr=False, bufsize=-1):
-        """The parameter 'cmd' is the shell command to execute in a
-        sub-process.  On UNIX, 'cmd' may be a sequence, in which case arguments
-        will be passed directly to the program without shell intervention (as
-        with os.spawnv()).  If 'cmd' is a string it will be passed to the shell
-        (as with os.system()).   The 'capturestderr' flag, if true, specifies
-        that the object should capture standard error output of the child
-        process.  The default is false.  If the 'bufsize' parameter is
-        specified, it specifies the size of the I/O buffers to/from the child
-        process."""
-        _cleanup()
-        self.cmd = cmd
-        p2cread, p2cwrite = os.pipe()
-        c2pread, c2pwrite = os.pipe()
-        if capturestderr:
-            errout, errin = os.pipe()
-        self.pid = os.fork()
-        if self.pid == 0:
-            # Child
-            os.dup2(p2cread, 0)
-            os.dup2(c2pwrite, 1)
-            if capturestderr:
-                os.dup2(errin, 2)
-            self._run_child(cmd)
-        os.close(p2cread)
-        self.tochild = os.fdopen(p2cwrite, 'w', bufsize)
-        os.close(c2pwrite)
-        self.fromchild = os.fdopen(c2pread, 'r', bufsize)
-        if capturestderr:
-            os.close(errin)
-            self.childerr = os.fdopen(errout, 'r', bufsize)
-        else:
-            self.childerr = None
-
-    def __del__(self):
-        # In case the child hasn't been waited on, check if it's done.
-        self.poll(_deadstate=sys.maxint)
-        if self.sts < 0:
-            if _active is not None:
-                # Child is still running, keep us alive until we can wait on it.
-                _active.append(self)
-
-    def _run_child(self, cmd):
-        if isinstance(cmd, basestring):
-            cmd = ['/bin/sh', '-c', cmd]
-        os.closerange(3, MAXFD)
-        try:
-            os.execvp(cmd[0], cmd)
-        finally:
-            os._exit(1)
-
-    def poll(self, _deadstate=None):
-        """Return the exit status of the child process if it has finished,
-        or -1 if it hasn't finished yet."""
-        if self.sts < 0:
-            try:
-                pid, sts = os.waitpid(self.pid, os.WNOHANG)
-                # pid will be 0 if self.pid hasn't terminated
-                if pid == self.pid:
-                    self.sts = sts
-            except os.error:
-                if _deadstate is not None:
-                    self.sts = _deadstate
-        return self.sts
-
-    def wait(self):
-        """Wait for and return the exit status of the child process."""
-        if self.sts < 0:
-            pid, sts = os.waitpid(self.pid, 0)
-            # This used to be a test, but it is believed to be
-            # always true, so I changed it to an assertion - mvl
-            assert pid == self.pid
-            self.sts = sts
-        return self.sts
-
-
-class Popen3Again:
-    """Class representing a child process.  Normally, instances are created
-    internally by the functions popen2() and popen3()."""
-
-    sts = -1                    # Child not completed yet
-
-    def __init__(self, cmd, capturestderr=False, bufsize=-1):
-        """The parameter 'cmd' is the shell command to execute in a
-        sub-process.  On UNIX, 'cmd' may be a sequence, in which case arguments
-        will be passed directly to the program without shell intervention (as
-        with os.spawnv()).  If 'cmd' is a string it will be passed to the shell
-        (as with os.system()).   The 'capturestderr' flag, if true, specifies
-        that the object should capture standard error output of the child
-        process.  The default is false.  If the 'bufsize' parameter is
-        specified, it specifies the size of the I/O buffers to/from the child
-        process."""
-        _cleanup()
-        self.cmd = cmd
-        p2cread, p2cwrite = os.pipe()
-        c2pread, c2pwrite = os.pipe()
-        if capturestderr:
-            errout, errin = os.pipe()
-        self.pid = os.fork()
-        if self.pid == 0:
-            # Child
-            os.dup2(p2cread, 0)
-            os.dup2(c2pwrite, 1)
-            if capturestderr:
-                os.dup2(errin, 2)
-            self._run_child(cmd)
-        os.close(p2cread)
-        self.tochild = os.fdopen(p2cwrite, 'w', bufsize)
-        os.close(c2pwrite)
-        self.fromchild = os.fdopen(c2pread, 'r', bufsize)
-        if capturestderr:
-            os.close(errin)
-            self.childerr = os.fdopen(errout, 'r', bufsize)
-        else:
-            self.childerr = None
-
-    def __del__(self):
-        # In case the child hasn't been waited on, check if it's done.
-        self.poll(_deadstate=sys.maxint)
-        if self.sts < 0:
-            if _active is not None:
-                # Child is still running, keep us alive until we can wait on it.
-                _active.append(self)
-
-    def _run_child(self, cmd):
-        if isinstance(cmd, basestring):
-            cmd = ['/bin/sh', '-c', cmd]
-        os.closerange(3, MAXFD)
-        try:
-            os.execvp(cmd[0], cmd)
-        finally:
-            os._exit(1)
-
-    def poll(self, _deadstate=None):
-        """Return the exit status of the child process if it has finished,
-        or -1 if it hasn't finished yet."""
-        if self.sts < 0:
-            try:
-                pid, sts = os.waitpid(self.pid, os.WNOHANG)
-                # pid will be 0 if self.pid hasn't terminated
-                if pid == self.pid:
-                    self.sts = sts
-            except os.error:
-                if _deadstate is not None:
-                    self.sts = _deadstate
-        return self.sts
-
-    def wait(self):
-        """Wait for and return the exit status of the child process."""
-        if self.sts < 0:
-            pid, sts = os.waitpid(self.pid, 0)
-            # This used to be a test, but it is believed to be
-            # always true, so I changed it to an assertion - mvl
-            assert pid == self.pid
-            self.sts = sts
-        return self.sts
-
-#Duplicate function with identifiers changed
-
-def dis3(y=None):
-    """frobnicate classes, methods, functions, or code.
-
-    With no argument, frobnicate the last traceback.
-
-    """
-    if y is None:
-        distb()
-        return
-    if isinstance(y, types.InstanceType):
-        y = y.__class__
-    if hasattr(y, 'im_func'):
-        y = y.im_func
-    if hasattr(y, 'func_code'):
-        y = y.func_code
-    if hasattr(y, '__dict__'):
-        items = y.__dict__.items()
-        items.sort()
-        for name, y1 in items:
-            if isinstance(y1, _have_code):
-                print("Disassembly of %s:" % name)
-                try:
-                    dis(y1)
-                except TypeError(msg):
-                    print("Sorry:", msg)
-                print()
-    elif hasattr(y, 'co_code'):
-        frobnicate(y)
-    elif isinstance(y, str):
-        frobnicate_string(y)
-    else:
-        raise TypeError(
-              "don't know how to frobnicate %s objects" %
-              type(y).__name__)
-
-
-#Mostly similar function
-
-def dis4(x=None):
-    """Disassemble classes, methods, functions, or code.
-
-    With no argument, disassemble the last traceback.
-
-    """
-    if x is None:
-        distb()
-        return
-    if isinstance(x, types.InstanceType):
-        x = x.__class__
-    if hasattr(x, 'im_func'):
-        x = x.im_func
-    if hasattr(x, '__dict__'):
-        items = x.__dict__.items()
-        items.sort()
-        for name, x1 in items:
-            if isinstance(x1, _have_code):
-                print("Disassembly of %s:" % name)
-                try:
-                    dis(x1)
-                except TypeError(msg):
-                    print("Sorry:", msg)
-                print()
-    elif hasattr(x, 'co_code'):
-        disassemble(x)
-    elif isinstance(x, str):
-        disassemble_string(x)
-    else:
-        raise TypeError(
-              "don't know how to disassemble %s objects" %
-              type(x).__name__)
-
-
-#Similar function with changed identifiers
-
-def dis5(z=None):
-    """splat classes, methods, functions, or code.
-
-    With no argument, splat the last traceback.
-
-    """
-    if z is None:
-        distb()
-        return
-    if isinstance(z, types.InstanceType):
-        z = z.__class__
-    if hasattr(z, 'im_func'):
-        z = z.im_func
-    if hasattr(y, 'func_code'):
-        y = y.func_code
-    if hasattr(z, '__dict__'):
-        items = z.__dict__.items()
-        items.sort()
-        for name, z1 in items:
-            if isinstance(z1, _have_code):
-                print("Disassembly of %s:" % name)
-                try:
-                    dis(z1)
-                except TypeError(msg):
-                    print("Sorry:", msg)
-                print()
-    elif hasattr(z, 'co_code'):
-        splat(z)
-    elif isinstance(z, str):
-        splat_string(z)
-    else:
-        raise TypeError(
-              "don't know how to splat %s objects" %
-              type(z).__name__)
-
-

python/ql/test/query-tests/UselessCode/DuplicateCode/similar.py

@@ -1,63 +0,0 @@
-
-
-def original(the_ast):
-    def walk(node, in_function, in_name_main):
-        def flags():
-            return in_function * 2 + in_name_main
-        if isinstance(node, ast.Module):
-            for import_node in walk(node.body, in_function, in_name_main):
-                yield import_node
-        elif isinstance(node, ast.ImportFrom):
-            aliases = [ Alias(a.name, a.asname) for a in node.names]
-            yield FromImport(node.level, node.module, aliases, flags())
-        elif isinstance(node, ast.Import):
-            aliases = [ Alias(a.name, a.asname) for a in node.names]
-            yield Import(aliases, flags())
-        elif isinstance(node, ast.FunctionDef):
-            for _, child in ast.iter_fields(node):
-                for import_node in walk(child, True, in_name_main):
-                    yield import_node
-        elif isinstance(node, list):
-            for n in node:
-                for import_node in walk(n, in_function, in_name_main):
-                    yield import_node
-    return list(walk(the_ast, False, False))
-
-def similar_1(the_ast):
-    def walk(node, in_function, in_name_main):
-        def flags():
-            return in_function * 2 + in_name_main
-        if isinstance(node, ast.Module):
-            for import_node in walk(node.body, in_function, in_name_main):
-                yield import_node
-        elif isinstance(node, ast.ImportFrom):
-            aliases = [ Alias(a.name, a.asname) for a in node.names]
-            yield FromImport(node.level, node.module, aliases, flags())
-        elif isinstance(node, ast.Import):
-            aliases = [ Alias(a.name, a.asname) for a in node.names]
-            yield Import(aliases, flags())
-        elif isinstance(node, ast.FunctionDef):
-            for _, child in ast.iter_fields(node):
-                for import_node in walk(child, True, in_name_main):
-                    yield import_node
-    return list(walk(the_ast, False, False))
-
-def similar_2(the_ast):
-    def walk(node, in_function, in_name_main):
-        def flags():
-            return in_function * 2 + in_name_main
-        if isinstance(node, ast.Module):
-            for import_node in walk(node.body, in_function, in_name_main):
-                yield import_node
-        elif isinstance(node, ast.Import):
-            aliases = [ Alias(a.name, a.asname) for a in node.names]
-            yield Import(aliases, flags())
-        elif isinstance(node, ast.FunctionDef):
-            for _, child in ast.iter_fields(node):
-                for import_node in walk(child, True, in_name_main):
-                    yield import_node
-        elif isinstance(node, list):
-            for n in node:
-                for import_node in walk(n, in_function, in_name_main):
-                    yield import_node
-    return list(walk(the_ast, False, False))