Merge branch 'rc/3.2' into mergeback-rc/3.2-to-main

python/change-notes/2021-07-12-add-typetrackingnode.md

@@ -1,3 +1,2 @@
 lgtm,codescanning
-* The `track` and `backtrack` methods on `LocalSourceNode` have been deprecated. When writing
-  type trackers, the corresponding methods on `TypeTrackingNode` should be used instead.
+* The `track` and `backtrack` methods on `LocalSourceNode` are in the process of being deprecated. When using type trackers, the corresponding methods on `TypeTrackingNode` should be used instead.

python/change-notes/2021-07-28-port-RoDoS-queries.md

@@ -1,3 +1,3 @@
 lgtm,codescanning
-* Added _Inefficient regular expression_ (`py/redos`) query, which is already available in JavaScript.
-* Added _Polynomial regular expression used on uncontrolled data_ (`py/polynomial-redos`), which is already available in JavaScript.
+* Added an experimental _Inefficient regular expression_ (`py/redos`) query, which is already available in JavaScript.
+* Added an experimental _Polynomial regular expression used on uncontrolled data_ (`py/polynomial-redos`), which is already available in JavaScript.

python/ql/src/semmle/python/dataflow/new/internal/LocalSources.qll

@@ -104,26 +104,20 @@ class LocalSourceNode extends Node {
   }
 
   /**
-   * DEPRECATED. Use `TypeTrackingNode::track` instead.
-   *
    * Gets a node that this node may flow to using one heap and/or interprocedural step.
    *
    * See `TypeTracker` for more details about how to use this.
    */
   pragma[inline]
-  deprecated LocalSourceNode track(TypeTracker t2, TypeTracker t) { t = t2.step(this, result) }
+  LocalSourceNode track(TypeTracker t2, TypeTracker t) { t = t2.step(this, result) }
 
   /**
-   * DEPRECATED. Use `TypeTrackingNode::backtrack` instead.
-   *
    * Gets a node that may flow into this one using one heap and/or interprocedural step.
    *
    * See `TypeBackTracker` for more details about how to use this.
    */
   pragma[inline]
-  deprecated LocalSourceNode backtrack(TypeBackTracker t2, TypeBackTracker t) {
-    t2 = t.step(result, this)
-  }
+  LocalSourceNode backtrack(TypeBackTracker t2, TypeBackTracker t) { t2 = t.step(result, this) }
 }
 
 /**
@@ -131,40 +125,46 @@ class LocalSourceNode extends Node {
  *
  * All steps made during type tracking should be between instances of this class.
  */
-class TypeTrackingNode extends Node {
-  TypeTrackingNode() {
-    this instanceof LocalSourceNode
-    or
-    this instanceof ModuleVariableNode
+class TypeTrackingNode = LocalSourceNode;
+
+/** Temporary holding ground for the `TypeTrackingNode` class. */
+private module FutureWork {
+  class FutureTypeTrackingNode extends Node {
+    FutureTypeTrackingNode() {
+      this instanceof LocalSourceNode
+      or
+      this instanceof ModuleVariableNode
+    }
+
+    /**
+     * Holds if this node can flow to `nodeTo` in one or more local flow steps.
+     *
+     * For `ModuleVariableNode`s, the only "local" step is to the node itself.
+     * For `LocalSourceNode`s, this is the usual notion of local flow.
+     */
+    pragma[inline]
+    predicate flowsTo(Node node) {
+      this instanceof ModuleVariableNode and this = node
+      or
+      this.(LocalSourceNode).flowsTo(node)
+    }
+
+    /**
+     * Gets a node that this node may flow to using one heap and/or interprocedural step.
+     *
+     * See `TypeTracker` for more details about how to use this.
+     */
+    pragma[inline]
+    TypeTrackingNode track(TypeTracker t2, TypeTracker t) { t = t2.step(this, result) }
+
+    /**
+     * Gets a node that may flow into this one using one heap and/or interprocedural step.
+     *
+     * See `TypeBackTracker` for more details about how to use this.
+     */
+    pragma[inline]
+    TypeTrackingNode backtrack(TypeBackTracker t2, TypeBackTracker t) { t2 = t.step(result, this) }
   }
-
-  /**
-   * Holds if this node can flow to `nodeTo` in one or more local flow steps.
-   *
-   * For `ModuleVariableNode`s, the only "local" step is to the node itself.
-   * For `LocalSourceNode`s, this is the usual notion of local flow.
-   */
-  predicate flowsTo(Node node) {
-    this instanceof ModuleVariableNode and this = node
-    or
-    this.(LocalSourceNode).flowsTo(node)
-  }
-
-  /**
-   * Gets a node that this node may flow to using one heap and/or interprocedural step.
-   *
-   * See `TypeTracker` for more details about how to use this.
-   */
-  pragma[inline]
-  TypeTrackingNode track(TypeTracker t2, TypeTracker t) { t = t2.step(this, result) }
-
-  /**
-   * Gets a node that may flow into this one using one heap and/or interprocedural step.
-   *
-   * See `TypeBackTracker` for more details about how to use this.
-   */
-  pragma[inline]
-  TypeTrackingNode backtrack(TypeBackTracker t2, TypeBackTracker t) { t2 = t.step(result, this) }
 }
 
 cached
@@ -179,11 +179,21 @@ private module Cached {
     source = sink
     or
     exists(Node second |
-      simpleLocalFlowStep(source, second) and
-      simpleLocalFlowStep*(second, sink)
+      localSourceFlowStep(source, second) and
+      localSourceFlowStep*(second, sink)
     )
   }
 
+  /**
+   * Helper predicate for `hasLocalSource`. Removes any steps go to module variable reads, as these
+   * are already local source nodes in their own right.
+   */
+  cached
+  private predicate localSourceFlowStep(Node nodeFrom, Node nodeTo) {
+    simpleLocalFlowStep(nodeFrom, nodeTo) and
+    not nodeTo = any(ModuleVariableNode v).getARead()
+  }
+
   /**
    * Holds if `base` flows to the base of `ref` and `ref` has attribute name `attr`.
    */

python/ql/test/query-tests/Security/CWE-730-PolynomialReDoS/PolynomialReDoS.qlref

@@ -1 +1 @@
-Security/CWE-730/PolynomialReDoS.ql
+experimental/Security/CWE-730/PolynomialReDoS.ql

python/ql/test/query-tests/Security/CWE-730-ReDoS/ReDoS.qlref

@@ -1 +1 @@
-Security/CWE-730/ReDoS.ql
+experimental/Security/CWE-730/ReDoS.ql