Merge branch 'main' into labelNaming

python/ql/lib/CHANGELOG.md

@@ -1,3 +1,26 @@
+## 0.0.13
+
+## 0.0.12
+
+### Breaking Changes
+
+* The flow state variants of `isBarrier` and `isAdditionalFlowStep` are no longer exposed in the taint tracking library. The `isSanitizer` and `isAdditionalTaintStep` predicates should be used instead.
+
+### Deprecated APIs
+
+* Many classes/predicates/modules that had upper-case acronyms have been renamed to follow our style-guide. 
+  The old name still exists as a deprecated alias.
+* Some modules that started with a lowercase letter have been renamed to follow our style-guide. 
+  The old name still exists as a deprecated alias.
+
+### New Features
+
+* The data flow and taint tracking libraries have been extended with versions of `isBarrierIn`, `isBarrierOut`, and `isBarrierGuard`, respectively `isSanitizerIn`, `isSanitizerOut`, and `isSanitizerGuard`, that support flow states.
+
+### Minor Analysis Improvements
+
+* All deprecated predicates/classes/modules that have been deprecated for over a year have been deleted.
+
 ## 0.0.11
 
 ### Minor Analysis Improvements

python/ql/lib/change-notes/2022-02-07-deleted-deprecations.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* All deprecated predicates/classes/modules that have been deprecated for over a year have been deleted.

python/ql/lib/change-notes/2022-02-07-deprecated-acronyms.md

@@ -1,5 +0,0 @@
----
-category: deprecated
----
-* Many classes/predicates/modules that had upper-case acronyms have been renamed to follow our style-guide. 
-  The old name still exists as a deprecated alias.

python/ql/lib/change-notes/2022-02-07-deprecated-modules.md

@@ -1,5 +0,0 @@
----
-category: deprecated
----
-* Some modules that started with a lowercase letter have been renamed to follow our style-guide. 
-  The old name still exists as a deprecated alias.

python/ql/lib/change-notes/2022-03-14-flow-state-barriers.md

@@ -1,4 +0,0 @@
----
-category: feature
----
-* The data flow and taint tracking libraries have been extended with versions of `isBarrierIn`, `isBarrierOut`, and `isBarrierGuard`, respectively `isSanitizerIn`, `isSanitizerOut`, and `isSanitizerGuard`, that support flow states.

python/ql/lib/change-notes/2022-03-14-taint-interface-cleanup.md

@@ -1,4 +0,0 @@
----
-category: breaking
----
-* The flow state variants of `isBarrier` and `isAdditionalFlowStep` are no longer exposed in the taint tracking library. The `isSanitizer` and `isAdditionalTaintStep` predicates should be used instead.

python/ql/lib/change-notes/2022-03-21-query-suffix-convention.md

@@ -0,0 +1,8 @@
+---
+category: deprecated
+---
+* Queries importing a data-flow configuration from `semmle.python.security.dataflow`
+  should ensure that the imported file ends with `Query`, and only import its top-level
+  module. For example, a query that used `CommandInjection::Configuration` from
+  `semmle.python.security.dataflow.CommandInjection` should from now use `Configuration`
+  from `semmle.python.security.dataflow.CommandInjectionQuery` instead.

python/ql/lib/change-notes/2022-03-30-flask-recognize-body-param.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Improved modeling of Flask `Response` objects, so passing a response body with the keyword argument `response` is now recognized.

python/ql/lib/change-notes/2022-04-08-flow-state-barriers.md

@@ -0,0 +1,4 @@
+---
+category: breaking
+---
+The recently added flow-state versions of `isBarrierIn`, `isBarrierOut`, `isSanitizerIn`, and `isSanitizerOut` in the data flow and taint tracking libraries have been removed.

python/ql/lib/change-notes/released/0.0.12.md

@@ -0,0 +1,20 @@
+## 0.0.12
+
+### Breaking Changes
+
+* The flow state variants of `isBarrier` and `isAdditionalFlowStep` are no longer exposed in the taint tracking library. The `isSanitizer` and `isAdditionalTaintStep` predicates should be used instead.
+
+### Deprecated APIs
+
+* Many classes/predicates/modules that had upper-case acronyms have been renamed to follow our style-guide. 
+  The old name still exists as a deprecated alias.
+* Some modules that started with a lowercase letter have been renamed to follow our style-guide. 
+  The old name still exists as a deprecated alias.
+
+### New Features
+
+* The data flow and taint tracking libraries have been extended with versions of `isBarrierIn`, `isBarrierOut`, and `isBarrierGuard`, respectively `isSanitizerIn`, `isSanitizerOut`, and `isSanitizerGuard`, that support flow states.
+
+### Minor Analysis Improvements
+
+* All deprecated predicates/classes/modules that have been deprecated for over a year have been deleted.

python/ql/lib/change-notes/released/0.0.13.md

@@ -0,0 +1 @@
+## 0.0.13

python/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.0.11
+lastReleaseVersion: 0.0.13

python/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/python-all
-version: 0.0.12-dev
+version: 0.1.0-dev
 groups: python
 dbscheme: semmlecode.python.dbscheme
 extractor: python

python/ql/lib/semmle/python/ApiGraphs.qll

@@ -7,8 +7,9 @@
  */
 
 // Importing python under the `py` namespace to avoid importing `CallNode` from `Flow.qll` and thereby having a naming conflict with `API::CallNode`.
-private import python as py
+private import python as PY
 import semmle.python.dataflow.new.DataFlow
+private import semmle.python.internal.CachedStages
 
 /**
  * Provides classes and predicates for working with APIs used in a database.
@@ -427,13 +428,13 @@ module API {
       /** An abstract representative for imports of the module called `name`. */
       MkModuleImport(string name) {
         // Ignore the following module name for Python 2, as we alias `__builtin__` to `builtins` elsewhere
-        (name != "__builtin__" or py::major_version() = 3) and
+        (name != "__builtin__" or PY::major_version() = 3) and
         (
           imports(_, name)
           or
           // When we `import foo.bar.baz` we want to create API graph nodes also for the prefixes
           // `foo` and `foo.bar`:
-          name = any(py::ImportExpr e | not e.isRelative()).getAnImportedModuleName()
+          name = any(PY::ImportExpr e | not e.isRelative()).getAnImportedModuleName()
         )
         or
         // The `builtins` module should always be implicitly available
@@ -469,7 +470,7 @@ module API {
      * Ignores relative imports, such as `from ..foo.bar import baz`.
      */
     private predicate imports(DataFlow::Node imp, string name) {
-      exists(py::ImportExprNode iexpr |
+      exists(PY::ImportExprNode iexpr |
         imp.asCfgNode() = iexpr and
         not iexpr.getNode().isRelative() and
         name = iexpr.getNode().getImportedModuleName()
@@ -492,7 +493,7 @@ module API {
      *
      * `moduleImport("foo").getMember("bar")`
      */
-    private TApiNode potential_import_star_base(py::Scope s) {
+    private TApiNode potential_import_star_base(PY::Scope s) {
       exists(DataFlow::Node n |
         n.asCfgNode() = ImportStar::potentialImportStarBase(s) and
         use(result, n)
@@ -515,17 +516,17 @@ module API {
         )
         or
         // TODO: I had expected `DataFlow::AttrWrite` to contain the attribute writes from a dict, that's how JS works.
-        exists(py::Dict dict, py::KeyValuePair item |
+        exists(PY::Dict dict, PY::KeyValuePair item |
           dict = pred.asExpr() and
           dict.getItem(_) = item and
-          lbl = Label::member(item.getKey().(py::StrConst).getS()) and
+          lbl = Label::member(item.getKey().(PY::StrConst).getS()) and
           rhs.asExpr() = item.getValue()
         )
         or
-        exists(py::CallableExpr fn | fn = pred.asExpr() |
+        exists(PY::CallableExpr fn | fn = pred.asExpr() |
           not fn.getInnerScope().isAsync() and
           lbl = Label::return() and
-          exists(py::Return ret |
+          exists(PY::Return ret |
             rhs.asExpr() = ret.getValue() and
             ret.getScope() = fn.getInnerScope()
           )
@@ -568,7 +569,7 @@ module API {
         // Subclassing a node
         lbl = Label::subclass() and
         exists(DataFlow::Node superclass | pred.flowsTo(superclass) |
-          ref.asExpr().(py::ClassExpr).getABase() = superclass.asExpr()
+          ref.asExpr().(PY::ClassExpr).getABase() = superclass.asExpr()
         )
         or
         // awaiting
@@ -579,7 +580,7 @@ module API {
         )
       )
       or
-      exists(DataFlow::Node def, py::CallableExpr fn |
+      exists(DataFlow::Node def, PY::CallableExpr fn |
         rhs(base, def) and fn = trackDefNode(def).asExpr()
       |
         exists(int i |
@@ -598,7 +599,7 @@ module API {
       lbl = Label::member(any(string name | ref = Builtins::likelyBuiltin(name)))
       or
       // Unknown variables that may belong to a module imported with `import *`
-      exists(py::Scope s |
+      exists(PY::Scope s |
         base = potential_import_star_base(s) and
         lbl =
           Label::member(any(string name |
@@ -618,7 +619,7 @@ module API {
       )
       or
       // Ensure the Python 2 `__builtin__` module gets the name of the Python 3 `builtins` module.
-      py::major_version() = 2 and
+      PY::major_version() = 2 and
       nd = MkModuleImport("builtins") and
       imports(ref, "__builtin__")
       or
@@ -683,6 +684,7 @@ module API {
      */
     cached
     DataFlow::LocalSourceNode trackUseNode(DataFlow::LocalSourceNode src) {
+      Stages::TypeTracking::ref() and
       result = trackUseNode(src, DataFlow::TypeTracker::end()) and
       not result instanceof DataFlow::ModuleVariableNode
     }
@@ -759,25 +761,25 @@ module API {
           exists(Builtins::likelyBuiltin(member)) or
           ImportStar::namePossiblyDefinedInImportStar(_, member, _) or
           Impl::prefix_member(_, member, _) or
-          member = any(py::Dict d).getAnItem().(py::KeyValuePair).getKey().(py::StrConst).getS()
+          member = any(PY::Dict d).getAnItem().(PY::KeyValuePair).getKey().(PY::StrConst).getS()
         } or
         MkLabelUnknownMember() or
         MkLabelParameter(int i) {
           exists(any(DataFlow::CallCfgNode c).getArg(i))
           or
-          exists(any(py::Function f).getArg(i))
+          exists(any(PY::Function f).getArg(i))
         } or
         MkLabelKeywordParameter(string name) {
           exists(any(DataFlow::CallCfgNode c).getArgByName(name))
           or
-          exists(any(py::Function f).getArgByName(name))
+          exists(any(PY::Function f).getArgByName(name))
         } or
         MkLabelReturn() or
         MkLabelSubclass() or
         MkLabelAwait()
 
       /** A label for a module. */
-      class LabelModule extends ApiLabel {
+      class LabelModule extends ApiLabel, MkLabelModule {
         string mod;
 
         LabelModule() { this = MkLabelModule(mod) }
@@ -789,7 +791,7 @@ module API {
       }
 
       /** A label for the member named `prop`. */
-      class LabelMember extends ApiLabel {
+      class LabelMember extends ApiLabel, MkLabelMember {
         string member;
 
         LabelMember() { this = MkLabelMember(member) }
@@ -801,14 +803,12 @@ module API {
       }
 
       /** A label for a member with an unknown name. */
-      class LabelUnknownMember extends ApiLabel {
-        LabelUnknownMember() { this = MkLabelUnknownMember() }
-
+      class LabelUnknownMember extends ApiLabel, MkLabelUnknownMember {
         override string toString() { result = "getUnknownMember()" }
       }
 
       /** A label for parameter `i`. */
-      class LabelParameter extends ApiLabel {
+      class LabelParameter extends ApiLabel, MkLabelParameter {
         int i;
 
         LabelParameter() { this = MkLabelParameter(i) }
@@ -820,7 +820,7 @@ module API {
       }
 
       /** A label for a keyword parameter `name`. */
-      class LabelKeywordParameter extends ApiLabel {
+      class LabelKeywordParameter extends ApiLabel, MkLabelKeywordParameter {
         string name;
 
         LabelKeywordParameter() { this = MkLabelKeywordParameter(name) }
@@ -832,23 +832,17 @@ module API {
       }
 
       /** A label that gets the return value of a function. */
-      class LabelReturn extends ApiLabel {
-        LabelReturn() { this = MkLabelReturn() }
-
+      class LabelReturn extends ApiLabel, MkLabelReturn {
         override string toString() { result = "getReturn()" }
       }
 
       /** A label that gets the subclass of a class. */
-      class LabelSubclass extends ApiLabel {
-        LabelSubclass() { this = MkLabelSubclass() }
-
+      class LabelSubclass extends ApiLabel, MkLabelSubclass {
         override string toString() { result = "getASubclass()" }
       }
 
       /** A label for awaited values. */
-      class LabelAwait extends ApiLabel {
-        LabelAwait() { this = MkLabelAwait() }
-
+      class LabelAwait extends ApiLabel, MkLabelAwait {
         override string toString() { result = "getAwaited()" }
       }
     }

python/ql/lib/semmle/python/AstExtended.qll

@@ -1,4 +1,5 @@
 import python
+private import semmle.python.internal.CachedStages
 
 /** A syntactic node (Class, Function, Module, Expr, Stmt or Comprehension) corresponding to a flow node */
 abstract class AstNode extends AstNode_ {
@@ -17,9 +18,14 @@ abstract class AstNode extends AstNode_ {
    * NOTE: For some statements and other purely syntactic elements,
    * there may not be a `ControlFlowNode`
    */
-  ControlFlowNode getAFlowNode() { py_flow_bb_node(result, this, _, _) }
+  cached
+  ControlFlowNode getAFlowNode() {
+    Stages::AST::ref() and
+    py_flow_bb_node(result, this, _, _)
+  }
 
   /** Gets the location for this AST node */
+  cached
   Location getLocation() { none() }
 
   /**
@@ -35,6 +41,7 @@ abstract class AstNode extends AstNode_ {
    * Expr.getASubExpression(), Stmt.getASubStatement(), Stmt.getASubExpression() or
    * Scope.getAStmt().
    */
+  cached
   abstract AstNode getAChildNode();
 
   /**
@@ -44,12 +51,16 @@ abstract class AstNode extends AstNode_ {
    * Expr.getASubExpression(), Stmt.getASubStatement(), Stmt.getASubExpression() or
    * Scope.getAStmt() applied to the parent.
    */
-  AstNode getParentNode() { result.getAChildNode() = this }
+  cached
+  AstNode getParentNode() {
+    Stages::AST::ref() and
+    result.getAChildNode() = this
+  }
 
   /** Whether this contains `inner` syntactically */
   predicate contains(AstNode inner) { this.getAChildNode+() = inner }
 
-  pragma[noinline]
+  pragma[nomagic]
   private predicate containsInScope(AstNode inner, Scope scope) {
     this.contains(inner) and
     not inner instanceof Scope and
@@ -106,9 +117,16 @@ class Comprehension extends Comprehension_, AstNode {
 
   override string toString() { result = "Comprehension" }
 
-  override Location getLocation() { result = Comprehension_.super.getLocation() }
+  override Location getLocation() {
+    Stages::AST::ref() and
+    result = Comprehension_.super.getLocation()
+  }
 
-  override AstNode getAChildNode() { result = this.getASubExpression() }
+  pragma[nomagic]
+  override AstNode getAChildNode() {
+    Stages::AST::ref() and
+    result = this.getASubExpression()
+  }
 
   Expr getASubExpression() {
     result = this.getIter() or

python/ql/lib/semmle/python/Concepts.qll

@@ -17,13 +17,9 @@ private import semmle.python.Frameworks
  * Extend this class to refine existing API models. If you want to model new APIs,
  * extend `SystemCommandExecution::Range` instead.
  */
-class SystemCommandExecution extends DataFlow::Node {
-  SystemCommandExecution::Range range;
-
-  SystemCommandExecution() { this = range }
-
+class SystemCommandExecution extends DataFlow::Node instanceof SystemCommandExecution::Range {
   /** Gets the argument that specifies the command to be executed. */
-  DataFlow::Node getCommand() { result = range.getCommand() }
+  DataFlow::Node getCommand() { result = super.getCommand() }
 }
 
 /** Provides a class for modeling new system-command execution APIs. */
@@ -48,13 +44,9 @@ module SystemCommandExecution {
  * Extend this class to refine existing API models. If you want to model new APIs,
  * extend `FileSystemAccess::Range` instead.
  */
-class FileSystemAccess extends DataFlow::Node {
-  FileSystemAccess::Range range;
-
-  FileSystemAccess() { this = range }
-
+class FileSystemAccess extends DataFlow::Node instanceof FileSystemAccess::Range {
   /** Gets an argument to this file system access that is interpreted as a path. */
-  DataFlow::Node getAPathArgument() { result = range.getAPathArgument() }
+  DataFlow::Node getAPathArgument() { result = super.getAPathArgument() }
 }
 
 /** Provides a class for modeling new file system access APIs. */
@@ -78,14 +70,12 @@ module FileSystemAccess {
  * Extend this class to refine existing API models. If you want to model new APIs,
  * extend `FileSystemWriteAccess::Range` instead.
  */
-class FileSystemWriteAccess extends FileSystemAccess {
-  override FileSystemWriteAccess::Range range;
-
+class FileSystemWriteAccess extends FileSystemAccess instanceof FileSystemWriteAccess::Range {
   /**
    * Gets a node that represents data to be written to the file system (possibly with
    * some transformation happening before it is written, like JSON encoding).
    */
-  DataFlow::Node getADataNode() { result = range.getADataNode() }
+  DataFlow::Node getADataNode() { result = super.getADataNode() }
 }
 
 /** Provides a class for modeling new file system writes. */
@@ -111,13 +101,9 @@ module Path {
    * A data-flow node that performs path normalization. This is often needed in order
    * to safely access paths.
    */
-  class PathNormalization extends DataFlow::Node {
-    PathNormalization::Range range;
-
-    PathNormalization() { this = range }
-
+  class PathNormalization extends DataFlow::Node instanceof PathNormalization::Range {
     /** Gets an argument to this path normalization that is interpreted as a path. */
-    DataFlow::Node getPathArg() { result = range.getPathArg() }
+    DataFlow::Node getPathArg() { result = super.getPathArg() }
   }
 
   /** Provides a class for modeling new path normalization APIs. */
@@ -133,12 +119,10 @@ module Path {
   }
 
   /** A data-flow node that checks that a path is safe to access. */
-  class SafeAccessCheck extends DataFlow::BarrierGuard {
-    SafeAccessCheck::Range range;
-
-    SafeAccessCheck() { this = range }
-
-    override predicate checks(ControlFlowNode node, boolean branch) { range.checks(node, branch) }
+  class SafeAccessCheck extends DataFlow::BarrierGuard instanceof SafeAccessCheck::Range {
+    override predicate checks(ControlFlowNode node, boolean branch) {
+      SafeAccessCheck::Range.super.checks(node, branch)
+    }
   }
 
   /** Provides a class for modeling new path safety checks. */
@@ -160,22 +144,18 @@ module Path {
  * Extend this class to refine existing API models. If you want to model new APIs,
  * extend `Decoding::Range` instead.
  */
-class Decoding extends DataFlow::Node {
-  Decoding::Range range;
-
-  Decoding() { this = range }
-
+class Decoding extends DataFlow::Node instanceof Decoding::Range {
   /** Holds if this call may execute code embedded in its input. */
-  predicate mayExecuteInput() { range.mayExecuteInput() }
+  predicate mayExecuteInput() { super.mayExecuteInput() }
 
   /** Gets an input that is decoded by this function. */
-  DataFlow::Node getAnInput() { result = range.getAnInput() }
+  DataFlow::Node getAnInput() { result = super.getAnInput() }
 
   /** Gets the output that contains the decoded data produced by this function. */
-  DataFlow::Node getOutput() { result = range.getOutput() }
+  DataFlow::Node getOutput() { result = super.getOutput() }
 
   /** Gets an identifier for the format this function decodes from, such as "JSON". */
-  string getFormat() { result = range.getFormat() }
+  string getFormat() { result = super.getFormat() }
 }
 
 /** Provides a class for modeling new decoding mechanisms. */
@@ -226,19 +206,15 @@ private class DecodingAdditionalTaintStep extends TaintTracking::AdditionalTaint
  * Extend this class to refine existing API models. If you want to model new APIs,
  * extend `Encoding::Range` instead.
  */
-class Encoding extends DataFlow::Node {
-  Encoding::Range range;
-
-  Encoding() { this = range }
-
+class Encoding extends DataFlow::Node instanceof Encoding::Range {
   /** Gets an input that is encoded by this function. */
-  DataFlow::Node getAnInput() { result = range.getAnInput() }
+  DataFlow::Node getAnInput() { result = super.getAnInput() }
 
   /** Gets the output that contains the encoded data produced by this function. */
-  DataFlow::Node getOutput() { result = range.getOutput() }
+  DataFlow::Node getOutput() { result = super.getOutput() }
 
   /** Gets an identifier for the format this function decodes from, such as "JSON". */
-  string getFormat() { result = range.getFormat() }
+  string getFormat() { result = super.getFormat() }
 }
 
 /** Provides a class for modeling new encoding mechanisms. */
@@ -280,13 +256,9 @@ private class EncodingAdditionalTaintStep extends TaintTracking::AdditionalTaint
  * Extend this class to refine existing API models. If you want to model new APIs,
  * extend `Logging::Range` instead.
  */
-class Logging extends DataFlow::Node {
-  Logging::Range range;
-
-  Logging() { this = range }
-
+class Logging extends DataFlow::Node instanceof Logging::Range {
   /** Gets an input that is logged. */
-  DataFlow::Node getAnInput() { result = range.getAnInput() }
+  DataFlow::Node getAnInput() { result = super.getAnInput() }
 }
 
 /** Provides a class for modeling new logging mechanisms. */
@@ -309,13 +281,9 @@ module Logging {
  * Extend this class to refine existing API models. If you want to model new APIs,
  * extend `CodeExecution::Range` instead.
  */
-class CodeExecution extends DataFlow::Node {
-  CodeExecution::Range range;
-
-  CodeExecution() { this = range }
-
+class CodeExecution extends DataFlow::Node instanceof CodeExecution::Range {
   /** Gets the argument that specifies the code to be executed. */
-  DataFlow::Node getCode() { result = range.getCode() }
+  DataFlow::Node getCode() { result = super.getCode() }
 }
 
 /** Provides a class for modeling new dynamic code execution APIs. */
@@ -343,13 +311,9 @@ module CodeExecution {
  * Extend this class to refine existing API models. If you want to model new APIs,
  * extend `SqlConstruction::Range` instead.
  */
-class SqlConstruction extends DataFlow::Node {
-  SqlConstruction::Range range;
-
-  SqlConstruction() { this = range }
-
+class SqlConstruction extends DataFlow::Node instanceof SqlConstruction::Range {
   /** Gets the argument that specifies the SQL statements to be constructed. */
-  DataFlow::Node getSql() { result = range.getSql() }
+  DataFlow::Node getSql() { result = super.getSql() }
 }
 
 /** Provides a class for modeling new SQL execution APIs. */
@@ -380,13 +344,9 @@ module SqlConstruction {
  * Extend this class to refine existing API models. If you want to model new APIs,
  * extend `SqlExecution::Range` instead.
  */
-class SqlExecution extends DataFlow::Node {
-  SqlExecution::Range range;
-
-  SqlExecution() { this = range }
-
+class SqlExecution extends DataFlow::Node instanceof SqlExecution::Range {
   /** Gets the argument that specifies the SQL statements to be executed. */
-  DataFlow::Node getSql() { result = range.getSql() }
+  DataFlow::Node getSql() { result = super.getSql() }
 }
 
 /** Provides a class for modeling new SQL execution APIs. */
@@ -412,22 +372,18 @@ module SqlExecution {
  * Extend this class to refine existing API models. If you want to model new APIs,
  * extend `RegexExecution::Range` instead.
  */
-class RegexExecution extends DataFlow::Node {
-  RegexExecution::Range range;
-
-  RegexExecution() { this = range }
-
+class RegexExecution extends DataFlow::Node instanceof RegexExecution::Range {
   /** Gets the data flow node for the regex being executed by this node. */
-  DataFlow::Node getRegex() { result = range.getRegex() }
+  DataFlow::Node getRegex() { result = super.getRegex() }
 
   /** Gets a dataflow node for the string to be searched or matched against. */
-  DataFlow::Node getString() { result = range.getString() }
+  DataFlow::Node getString() { result = super.getString() }
 
   /**
    * Gets the name of this regex execution, typically the name of an executing method.
    * This is used for nice alert messages and should include the module if possible.
    */
-  string getName() { result = range.getName() }
+  string getName() { result = super.getName() }
 }
 
 /** Provides classes for modeling new regular-expression execution APIs. */
@@ -466,19 +422,15 @@ module XML {
    * Extend this class to refine existing API models. If you want to model new APIs,
    * extend `XPathConstruction::Range` instead.
    */
-  class XPathConstruction extends DataFlow::Node {
-    XPathConstruction::Range range;
-
-    XPathConstruction() { this = range }
-
+  class XPathConstruction extends DataFlow::Node instanceof XPathConstruction::Range {
     /** Gets the argument that specifies the XPath expressions to be constructed. */
-    DataFlow::Node getXPath() { result = range.getXPath() }
+    DataFlow::Node getXPath() { result = super.getXPath() }
 
     /**
      * Gets the name of this XPath expression construction, typically the name of an executing method.
      * This is used for nice alert messages and should include the module if possible.
      */
-    string getName() { result = range.getName() }
+    string getName() { result = super.getName() }
   }
 
   /** Provides a class for modeling new XPath construction APIs. */
@@ -513,19 +465,15 @@ module XML {
    * Extend this class to refine existing API models. If you want to model new APIs,
    * extend `XPathExecution::Range` instead.
    */
-  class XPathExecution extends DataFlow::Node {
-    XPathExecution::Range range;
-
-    XPathExecution() { this = range }
-
+  class XPathExecution extends DataFlow::Node instanceof XPathExecution::Range {
     /** Gets the data flow node for the XPath expression being executed by this node. */
-    DataFlow::Node getXPath() { result = range.getXPath() }
+    DataFlow::Node getXPath() { result = super.getXPath() }
 
     /**
      * Gets the name of this XPath expression execution, typically the name of an executing method.
      * This is used for nice alert messages and should include the module if possible.
      */
-    string getName() { result = range.getName() }
+    string getName() { result = super.getName() }
   }
 
   /** Provides classes for modeling new regular-expression execution APIs. */
@@ -560,16 +508,12 @@ module LDAP {
    * Extend this class to refine existing API models. If you want to model new APIs,
    * extend `LDAPQuery::Range` instead.
    */
-  class LdapExecution extends DataFlow::Node {
-    LdapExecution::Range range;
-
-    LdapExecution() { this = range }
-
+  class LdapExecution extends DataFlow::Node instanceof LdapExecution::Range {
     /** Gets the argument containing the filter string. */
-    DataFlow::Node getFilter() { result = range.getFilter() }
+    DataFlow::Node getFilter() { result = super.getFilter() }
 
     /** Gets the argument containing the base DN. */
-    DataFlow::Node getBaseDn() { result = range.getBaseDn() }
+    DataFlow::Node getBaseDn() { result = super.getBaseDn() }
   }
 
   /** Provides classes for modeling new LDAP query execution-related APIs. */
@@ -597,26 +541,23 @@ module LDAP {
  * Extend this class to refine existing API models. If you want to model new APIs,
  * extend `Escaping::Range` instead.
  */
-class Escaping extends DataFlow::Node {
-  Escaping::Range range;
-
+class Escaping extends DataFlow::Node instanceof Escaping::Range {
   Escaping() {
-    this = range and
     // escapes that don't have _both_ input/output defined are not valid
-    exists(range.getAnInput()) and
-    exists(range.getOutput())
+    exists(super.getAnInput()) and
+    exists(super.getOutput())
   }
 
   /** Gets an input that will be escaped. */
-  DataFlow::Node getAnInput() { result = range.getAnInput() }
+  DataFlow::Node getAnInput() { result = super.getAnInput() }
 
   /** Gets the output that contains the escaped data. */
-  DataFlow::Node getOutput() { result = range.getOutput() }
+  DataFlow::Node getOutput() { result = super.getOutput() }
 
   /**
    * Gets the context that this function escapes for, such as `html`, or `url`.
    */
-  string getKind() { result = range.getKind() }
+  string getKind() { result = super.getKind() }
 }
 
 /** Provides a class for modeling new escaping APIs. */
@@ -674,7 +615,7 @@ module Escaping {
  * `<p>{}</p>`.
  */
 class HtmlEscaping extends Escaping {
-  HtmlEscaping() { range.getKind() = Escaping::getHtmlKind() }
+  HtmlEscaping() { super.getKind() = Escaping::getHtmlKind() }
 }
 
 /**
@@ -682,7 +623,7 @@ class HtmlEscaping extends Escaping {
  * the body of a regex.
  */
 class RegexEscaping extends Escaping {
-  RegexEscaping() { range.getKind() = Escaping::getRegexKind() }
+  RegexEscaping() { super.getKind() = Escaping::getRegexKind() }
 }
 
 /**
@@ -690,14 +631,14 @@ class RegexEscaping extends Escaping {
  * in an LDAP search.
  */
 class LdapDnEscaping extends Escaping {
-  LdapDnEscaping() { range.getKind() = Escaping::getLdapDnKind() }
+  LdapDnEscaping() { super.getKind() = Escaping::getLdapDnKind() }
 }
 
 /**
  * An escape of a string so it can be safely used as a filter in an LDAP search.
  */
 class LdapFilterEscaping extends Escaping {
-  LdapFilterEscaping() { range.getKind() = Escaping::getLdapFilterKind() }
+  LdapFilterEscaping() { super.getKind() = Escaping::getLdapFilterKind() }
 }
 
 /** Provides classes for modeling HTTP-related APIs. */
@@ -716,29 +657,25 @@ module HTTP {
      * Extend this class to refine existing API models. If you want to model new APIs,
      * extend `RouteSetup::Range` instead.
      */
-    class RouteSetup extends DataFlow::Node {
-      RouteSetup::Range range;
-
-      RouteSetup() { this = range }
-
+    class RouteSetup extends DataFlow::Node instanceof RouteSetup::Range {
       /** Gets the URL pattern for this route, if it can be statically determined. */
-      string getUrlPattern() { result = range.getUrlPattern() }
+      string getUrlPattern() { result = super.getUrlPattern() }
 
       /**
        * Gets a function that will handle incoming requests for this route, if any.
        *
        * NOTE: This will be modified in the near future to have a `RequestHandler` result, instead of a `Function`.
        */
-      Function getARequestHandler() { result = range.getARequestHandler() }
+      Function getARequestHandler() { result = super.getARequestHandler() }
 
       /**
        * Gets a parameter that will receive parts of the url when handling incoming
        * requests for this route, if any. These automatically become a `RemoteFlowSource`.
        */
-      Parameter getARoutedParameter() { result = range.getARoutedParameter() }
+      Parameter getARoutedParameter() { result = super.getARoutedParameter() }
 
       /** Gets a string that identifies the framework used for this route setup. */
-      string getFramework() { result = range.getFramework() }
+      string getFramework() { result = super.getFramework() }
     }
 
     /** Provides a class for modeling new HTTP routing APIs. */
@@ -785,19 +722,15 @@ module HTTP {
      * Extend this class to refine existing API models. If you want to model new APIs,
      * extend `RequestHandler::Range` instead.
      */
-    class RequestHandler extends Function {
-      RequestHandler::Range range;
-
-      RequestHandler() { this = range }
-
+    class RequestHandler extends Function instanceof RequestHandler::Range {
       /**
        * Gets a parameter that could receive parts of the url when handling incoming
        * requests, if any. These automatically become a `RemoteFlowSource`.
        */
-      Parameter getARoutedParameter() { result = range.getARoutedParameter() }
+      Parameter getARoutedParameter() { result = super.getARoutedParameter() }
 
       /** Gets a string that identifies the framework used for this route setup. */
-      string getFramework() { result = range.getFramework() }
+      string getFramework() { result = super.getFramework() }
     }
 
     /** Provides a class for modeling new HTTP request handlers. */
@@ -853,16 +786,12 @@ module HTTP {
      * Extend this class to refine existing API models. If you want to model new APIs,
      * extend `HttpResponse::Range` instead.
      */
-    class HttpResponse extends DataFlow::Node {
-      HttpResponse::Range range;
-
-      HttpResponse() { this = range }
-
+    class HttpResponse extends DataFlow::Node instanceof HttpResponse::Range {
       /** Gets the data-flow node that specifies the body of this HTTP response. */
-      DataFlow::Node getBody() { result = range.getBody() }
+      DataFlow::Node getBody() { result = super.getBody() }
 
       /** Gets the mimetype of this HTTP response, if it can be statically determined. */
-      string getMimetype() { result = range.getMimetype() }
+      string getMimetype() { result = super.getMimetype() }
     }
 
     /** Provides a class for modeling new HTTP response APIs. */
@@ -908,13 +837,9 @@ module HTTP {
      * Extend this class to refine existing API models. If you want to model new APIs,
      * extend `HttpRedirectResponse::Range` instead.
      */
-    class HttpRedirectResponse extends HttpResponse {
-      override HttpRedirectResponse::Range range;
-
-      HttpRedirectResponse() { this = range }
-
+    class HttpRedirectResponse extends HttpResponse instanceof HttpRedirectResponse::Range {
       /** Gets the data-flow node that specifies the location of this HTTP redirect response. */
-      DataFlow::Node getRedirectLocation() { result = range.getRedirectLocation() }
+      DataFlow::Node getRedirectLocation() { result = super.getRedirectLocation() }
     }
 
     /** Provides a class for modeling new HTTP redirect response APIs. */
@@ -940,25 +865,21 @@ module HTTP {
      * Extend this class to refine existing API models. If you want to model new APIs,
      * extend `HTTP::CookieWrite::Range` instead.
      */
-    class CookieWrite extends DataFlow::Node {
-      CookieWrite::Range range;
-
-      CookieWrite() { this = range }
-
+    class CookieWrite extends DataFlow::Node instanceof CookieWrite::Range {
       /**
        * Gets the argument, if any, specifying the raw cookie header.
        */
-      DataFlow::Node getHeaderArg() { result = range.getHeaderArg() }
+      DataFlow::Node getHeaderArg() { result = super.getHeaderArg() }
 
       /**
        * Gets the argument, if any, specifying the cookie name.
        */
-      DataFlow::Node getNameArg() { result = range.getNameArg() }
+      DataFlow::Node getNameArg() { result = super.getNameArg() }
 
       /**
        * Gets the argument, if any, specifying the cookie value.
        */
-      DataFlow::Node getValueArg() { result = range.getValueArg() }
+      DataFlow::Node getValueArg() { result = super.getValueArg() }
     }
 
     /** Provides a class for modeling new cookie writes on HTTP responses. */
@@ -1075,27 +996,23 @@ module Cryptography {
      * Extend this class to refine existing API models. If you want to model new APIs,
      * extend `KeyGeneration::Range` instead.
      */
-    class KeyGeneration extends DataFlow::Node {
-      KeyGeneration::Range range;
-
-      KeyGeneration() { this = range }
-
+    class KeyGeneration extends DataFlow::Node instanceof KeyGeneration::Range {
       /** Gets the name of the cryptographic algorithm (for example `"RSA"` or `"AES"`). */
-      string getName() { result = range.getName() }
+      string getName() { result = super.getName() }
 
       /** Gets the argument that specifies the size of the key in bits, if available. */
-      DataFlow::Node getKeySizeArg() { result = range.getKeySizeArg() }
+      DataFlow::Node getKeySizeArg() { result = super.getKeySizeArg() }
 
       /**
        * Gets the size of the key generated (in bits), as well as the `origin` that
        * explains how we obtained this specific key size.
        */
       int getKeySizeWithOrigin(DataFlow::Node origin) {
-        result = range.getKeySizeWithOrigin(origin)
+        result = super.getKeySizeWithOrigin(origin)
       }
 
       /** Gets the minimum key size (in bits) for this algorithm to be considered secure. */
-      int minimumSecureKeySize() { result = range.minimumSecureKeySize() }
+      int minimumSecureKeySize() { result = super.minimumSecureKeySize() }
     }
 
     /** Provides classes for modeling new key-pair generation APIs. */
@@ -1174,16 +1091,12 @@ module Cryptography {
    * Extend this class to refine existing API models. If you want to model new APIs,
    * extend `CryptographicOperation::Range` instead.
    */
-  class CryptographicOperation extends DataFlow::Node {
-    CryptographicOperation::Range range;
-
-    CryptographicOperation() { this = range }
-
+  class CryptographicOperation extends DataFlow::Node instanceof CryptographicOperation::Range {
     /** Gets the algorithm used, if it matches a known `CryptographicAlgorithm`. */
-    CryptographicAlgorithm getAlgorithm() { result = range.getAlgorithm() }
+    CryptographicAlgorithm getAlgorithm() { result = super.getAlgorithm() }
 
     /** Gets an input the algorithm is used on, for example the plain text input to be encrypted. */
-    DataFlow::Node getAnInput() { result = range.getAnInput() }
+    DataFlow::Node getAnInput() { result = super.getAnInput() }
   }
 
   /** Provides classes for modeling new applications of a cryptographic algorithms. */

python/ql/lib/semmle/python/Exprs.qll

@@ -1,6 +1,7 @@
 import python
 private import semmle.python.pointsto.PointsTo
 private import semmle.python.objects.ObjectInternal
+private import semmle.python.internal.CachedStages
 
 /** An expression */
 class Expr extends Expr_, AstNode {
@@ -8,7 +9,11 @@ class Expr extends Expr_, AstNode {
   override Scope getScope() { py_scopes(this, result) }
 
   /** Gets a textual representation of this element. */
-  override string toString() { result = "Expression" }
+  cached
+  override string toString() {
+    Stages::AST::ref() and
+    result = "Expression"
+  }
 
   /** Gets the module in which this expression occurs */
   Module getEnclosingModule() { result = this.getScope().getEnclosingModule() }

python/ql/lib/semmle/python/Flow.qll

@@ -1,5 +1,6 @@
 import python
 private import semmle.python.pointsto.PointsTo
+private import semmle.python.internal.CachedStages
 
 /*
  * Note about matching parent and child nodes and CFG splitting:
@@ -122,7 +123,9 @@ class ControlFlowNode extends @py_flow_node {
   AstNode getNode() { py_flow_bb_node(this, result, _, _) }
 
   /** Gets a textual representation of this element. */
+  cached
   string toString() {
+    Stages::DataFlow::ref() and
     exists(Scope s | s.getEntryNode() = this | result = "Entry node for " + s.toString())
     or
     exists(Scope s | s.getANormalExit() = this | result = "Exit node for " + s.toString())
@@ -191,7 +194,9 @@ class ControlFlowNode extends @py_flow_node {
   BasicBlock getBasicBlock() { result.contains(this) }
 
   /** Gets the scope containing this flow node */
+  cached
   Scope getScope() {
+    Stages::AST::ref() and
     if this.getNode() instanceof Scope
     then
       /* Entry or exit node */
@@ -614,7 +619,9 @@ class UnaryExprNode extends ControlFlowNode {
  * and nodes implicitly assigned in class and function definitions and imports.
  */
 class DefinitionNode extends ControlFlowNode {
+  cached
   DefinitionNode() {
+    Stages::AST::ref() and
     exists(Assign a | a.getATarget().getAFlowNode() = this)
     or
     exists(AnnAssign a | a.getTarget().getAFlowNode() = this and exists(a.getValue()))
@@ -673,6 +680,7 @@ abstract class SequenceNode extends ControlFlowNode {
   ControlFlowNode getAnElement() { result = this.getElement(_) }
 
   /** Gets the control flow node for the nth element of this sequence */
+  cached
   abstract ControlFlowNode getElement(int n);
 }
 
@@ -681,6 +689,7 @@ class TupleNode extends SequenceNode {
   TupleNode() { toAst(this) instanceof Tuple }
 
   override ControlFlowNode getElement(int n) {
+    Stages::AST::ref() and
     exists(Tuple t | this.getNode() = t and result.getNode() = t.getElt(n)) and
     (
       result.getBasicBlock().dominates(this.getBasicBlock())
@@ -998,11 +1007,13 @@ class BasicBlock extends @py_flow_node {
   string toString() { result = "BasicBlock" }
 
   /** Whether this basic block strictly dominates the other */
-  pragma[nomagic]
-  predicate strictlyDominates(BasicBlock other) { other.getImmediateDominator+() = this }
+  cached
+  predicate strictlyDominates(BasicBlock other) {
+    Stages::AST::ref() and
+    other.getImmediateDominator+() = this
+  }
 
   /** Whether this basic block dominates the other */
-  pragma[nomagic]
   predicate dominates(BasicBlock other) {
     this = other
     or
@@ -1011,6 +1022,7 @@ class BasicBlock extends @py_flow_node {
 
   cached
   BasicBlock getImmediateDominator() {
+    Stages::AST::ref() and
     this.firstNode().getImmediateDominator().getBasicBlock() = result
   }
 
@@ -1048,7 +1060,11 @@ class BasicBlock extends @py_flow_node {
   }
 
   /** Gets a successor to this basic block */
-  BasicBlock getASuccessor() { result = this.getLastNode().getASuccessor().getBasicBlock() }
+  cached
+  BasicBlock getASuccessor() {
+    Stages::AST::ref() and
+    result = this.getLastNode().getASuccessor().getBasicBlock()
+  }
 
   /** Gets a predecessor to this basic block */
   BasicBlock getAPredecessor() { result.getASuccessor() = this }
@@ -1118,7 +1134,11 @@ class BasicBlock extends @py_flow_node {
   }
 
   /** Holds if this basic block strictly reaches the other. Is the start of other reachable from the end of this. */
-  predicate strictlyReaches(BasicBlock other) { this.getASuccessor+() = other }
+  cached
+  predicate strictlyReaches(BasicBlock other) {
+    Stages::AST::ref() and
+    this.getASuccessor+() = other
+  }
 
   /** Holds if this basic block reaches the other. Is the start of other reachable from the end of this. */
   predicate reaches(BasicBlock other) { this = other or this.strictlyReaches(other) }

python/ql/lib/semmle/python/Function.qll

@@ -167,18 +167,18 @@ class Function extends Function_, Scope, AstNode {
 
 /** A def statement. Note that FunctionDef extends Assign as a function definition binds the newly created function */
 class FunctionDef extends Assign {
+  FunctionExpr f;
+
   /* syntax: def name(...): ... */
   FunctionDef() {
     /* This is an artificial assignment the rhs of which is a (possibly decorated) FunctionExpr */
-    exists(FunctionExpr f | this.getValue() = f or this.getValue() = f.getADecoratorCall())
+    this.getValue() = f or this.getValue() = f.getADecoratorCall()
   }
 
   override string toString() { result = "FunctionDef" }
 
   /** Gets the function for this statement */
-  Function getDefinedFunction() {
-    exists(FunctionExpr func | this.containsInScope(func) and result = func.getInnerScope())
-  }
+  Function getDefinedFunction() { result = f.getInnerScope() }
 
   override Stmt getLastStatement() { result = this.getDefinedFunction().getLastStatement() }
 }

python/ql/lib/semmle/python/Import.qll

@@ -1,5 +1,6 @@
 import python
 private import semmle.python.types.Builtins
+private import semmle.python.internal.CachedStages
 
 /**
  * An alias in an import statement, the `mod as name` part of `import mod as name`. May be artificial;
@@ -203,7 +204,9 @@ class Import extends Import_ {
 /** An import * statement */
 class ImportStar extends ImportStar_ {
   /* syntax: from modname import * */
+  cached
   ImportExpr getModuleExpr() {
+    Stages::AST::ref() and
     result = this.getModule()
     or
     result = this.getModule().(ImportMember).getModule()

python/ql/lib/semmle/python/Module.qll

@@ -1,6 +1,7 @@
 import python
 private import semmle.python.objects.ObjectAPI
 private import semmle.python.objects.Modules
+private import semmle.python.internal.CachedStages
 
 /**
  * A module. This is the top level element in an AST, corresponding to a source file.
@@ -221,7 +222,9 @@ private predicate transitively_imported_from_entry_point(File file) {
   )
 }
 
+cached
 string moduleNameFromFile(Container file) {
+  Stages::AST::ref() and
   exists(string basename |
     basename = moduleNameFromBase(file) and
     legalShortName(basename)

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowDispatchPointsTo.qll

@@ -0,0 +1,558 @@
+/**
+ * INTERNAL: Do not use.
+ *
+ * Points-to based call-graph.
+ */
+
+private import python
+private import DataFlowPublic
+private import semmle.python.SpecialMethods
+
+/** A parameter position represented by an integer. */
+class ParameterPosition extends int {
+  ParameterPosition() { exists(any(DataFlowCallable c).getParameter(this)) }
+}
+
+/** An argument position represented by an integer. */
+class ArgumentPosition extends int {
+  ArgumentPosition() { exists(any(DataFlowCall c).getArg(this)) }
+}
+
+/** Holds if arguments at position `apos` match parameters at position `ppos`. */
+pragma[inline]
+predicate parameterMatch(ParameterPosition ppos, ArgumentPosition apos) { ppos = apos }
+
+/**
+ * Computes routing of arguments to parameters
+ *
+ * When a call contains more positional arguments than there are positional parameters,
+ * the extra positional arguments are passed as a tuple to a starred parameter. This is
+ * achieved by synthesizing a node `TPosOverflowNode(call, callable)`
+ * that represents the tuple of extra positional arguments. There is a store step from each
+ * extra positional argument to this node.
+ *
+ * CURRENTLY NOT SUPPORTED:
+ * When a call contains an iterable unpacking argument, such as `func(*args)`, it is expanded into positional arguments.
+ *
+ * CURRENTLY NOT SUPPORTED:
+ * If a call contains an iterable unpacking argument, such as `func(*args)`, and the callee contains a starred argument, any extra
+ * positional arguments are passed to the starred argument.
+ *
+ * When a call contains keyword arguments that do not correspond to keyword parameters, these
+ * extra keyword arguments are passed as a dictionary to a doubly starred parameter. This is
+ * achieved by synthesizing a node `TKwOverflowNode(call, callable)`
+ * that represents the dictionary of extra keyword arguments. There is a store step from each
+ * extra keyword argument to this node.
+ *
+ * When a call contains a dictionary unpacking argument, such as `func(**kwargs)`, with entries corresponding to a keyword parameter,
+ * the value at such a key is unpacked and passed to the parameter. This is achieved
+ * by synthesizing an argument node `TKwUnpacked(call, callable, name)` representing the unpacked
+ * value. This node is used as the argument passed to the matching keyword parameter. There is a read
+ * step from the dictionary argument to the synthesized argument node.
+ *
+ * When a call contains a dictionary unpacking argument, such as `func(**kwargs)`, and the callee contains a doubly starred parameter,
+ * entries which are not unpacked are passed to the doubly starred parameter. This is achieved by
+ * adding a dataflow step from the dictionary argument to `TKwOverflowNode(call, callable)` and a
+ * step to clear content of that node at any unpacked keys.
+ *
+ * ## Examples:
+ * Assume that we have the callable
+ * ```python
+ * def f(x, y, *t, **d):
+ *   pass
+ * ```
+ * Then the call
+ * ```python
+ * f(0, 1, 2, a=3)
+ * ```
+ * will be modeled as
+ * ```python
+ * f(0, 1, [*t], [**d])
+ * ```
+ * where `[` and `]` denotes synthesized nodes, so `[*t]` is the synthesized tuple argument
+ * `TPosOverflowNode` and `[**d]` is the synthesized dictionary argument `TKwOverflowNode`.
+ * There will be a store step from `2` to `[*t]` at pos `0` and one from `3` to `[**d]` at key
+ * `a`.
+ *
+ * For the call
+ * ```python
+ * f(0, **{"y": 1, "a": 3})
+ * ```
+ * no tuple argument is synthesized. It is modeled as
+ * ```python
+ * f(0, [y=1], [**d])
+ * ```
+ * where `[y=1]` is the synthesized unpacked argument `TKwUnpacked` (with `name` = `y`). There is
+ * a read step from `**{"y": 1, "a": 3}` to `[y=1]` at key `y` to get the value passed to the parameter
+ * `y`. There is a dataflow step from `**{"y": 1, "a": 3}` to `[**d]` to transfer the content and
+ * a clearing of content at key `y` for node `[**d]`, since that value has been unpacked.
+ */
+module ArgumentPassing {
+  /**
+   * Holds if `call` represents a `DataFlowCall` to a `DataFlowCallable` represented by `callable`.
+   *
+   * It _may not_ be the case that `call = callable.getACall()`, i.e. if `call` represents a `ClassCall`.
+   *
+   * Used to limit the size of predicates.
+   */
+  predicate connects(CallNode call, CallableValue callable) {
+    exists(DataFlowCall c |
+      call = c.getNode() and
+      callable = c.getCallable().getCallableValue()
+    )
+  }
+
+  /**
+   * Gets the `n`th parameter of `callable`.
+   * If the callable has a starred parameter, say `*tuple`, that is matched with `n=-1`.
+   * If the callable has a doubly starred parameter, say `**dict`, that is matched with `n=-2`.
+   * Note that, unlike other languages, we do _not_ use -1 for the position of `self` in Python,
+   * as it is an explicit parameter at position 0.
+   */
+  NameNode getParameter(CallableValue callable, int n) {
+    // positional parameter
+    result = callable.getParameter(n)
+    or
+    // starred parameter, `*tuple`
+    exists(Function f |
+      f = callable.getScope() and
+      n = -1 and
+      result = f.getVararg().getAFlowNode()
+    )
+    or
+    // doubly starred parameter, `**dict`
+    exists(Function f |
+      f = callable.getScope() and
+      n = -2 and
+      result = f.getKwarg().getAFlowNode()
+    )
+  }
+
+  /**
+   * A type representing a mapping from argument indices to parameter indices.
+   * We currently use two mappings: NoShift, the identity, used for ordinary
+   * function calls, and ShiftOneUp which is used for calls where an extra argument
+   * is inserted. These include method calls, constructor calls and class calls.
+   * In these calls, the argument at index `n` is mapped to the parameter at position `n+1`.
+   */
+  newtype TArgParamMapping =
+    TNoShift() or
+    TShiftOneUp()
+
+  /** A mapping used for parameter passing. */
+  abstract class ArgParamMapping extends TArgParamMapping {
+    /** Gets the index of the parameter that corresponds to the argument at index `argN`. */
+    bindingset[argN]
+    abstract int getParamN(int argN);
+
+    /** Gets a textual representation of this element. */
+    abstract string toString();
+  }
+
+  /** A mapping that passes argument `n` to parameter `n`. */
+  class NoShift extends ArgParamMapping, TNoShift {
+    NoShift() { this = TNoShift() }
+
+    override string toString() { result = "NoShift [n -> n]" }
+
+    bindingset[argN]
+    override int getParamN(int argN) { result = argN }
+  }
+
+  /** A mapping that passes argument `n` to parameter `n+1`. */
+  class ShiftOneUp extends ArgParamMapping, TShiftOneUp {
+    ShiftOneUp() { this = TShiftOneUp() }
+
+    override string toString() { result = "ShiftOneUp [n -> n+1]" }
+
+    bindingset[argN]
+    override int getParamN(int argN) { result = argN + 1 }
+  }
+
+  /**
+   * Gets the node representing the argument to `call` that is passed to the parameter at
+   * (zero-based) index `paramN` in `callable`. If this is a positional argument, it must appear
+   * at an index, `argN`, in `call` wich satisfies `paramN = mapping.getParamN(argN)`.
+   *
+   * `mapping` will be the identity for function calls, but not for method- or constructor calls,
+   * where the first parameter is `self` and the first positional argument is passed to the second positional parameter.
+   * Similarly for classmethod calls, where the first parameter is `cls`.
+   *
+   * NOT SUPPORTED: Keyword-only parameters.
+   */
+  Node getArg(CallNode call, ArgParamMapping mapping, CallableValue callable, int paramN) {
+    connects(call, callable) and
+    (
+      // positional argument
+      exists(int argN |
+        paramN = mapping.getParamN(argN) and
+        result = TCfgNode(call.getArg(argN))
+      )
+      or
+      // keyword argument
+      // TODO: Since `getArgName` have no results for keyword-only parameters,
+      // these are currently not supported.
+      exists(Function f, string argName |
+        f = callable.getScope() and
+        f.getArgName(paramN) = argName and
+        result = TCfgNode(call.getArgByName(unbind_string(argName)))
+      )
+      or
+      // a synthezised argument passed to the starred parameter (at position -1)
+      callable.getScope().hasVarArg() and
+      paramN = -1 and
+      result = TPosOverflowNode(call, callable)
+      or
+      // a synthezised argument passed to the doubly starred parameter (at position -2)
+      callable.getScope().hasKwArg() and
+      paramN = -2 and
+      result = TKwOverflowNode(call, callable)
+      or
+      // argument unpacked from dict
+      exists(string name |
+        call_unpacks(call, mapping, callable, name, paramN) and
+        result = TKwUnpackedNode(call, callable, name)
+      )
+    )
+  }
+
+  /** Currently required in `getArg` in order to prevent a bad join. */
+  bindingset[result, s]
+  private string unbind_string(string s) { result <= s and s <= result }
+
+  /** Gets the control flow node that is passed as the `n`th overflow positional argument. */
+  ControlFlowNode getPositionalOverflowArg(CallNode call, CallableValue callable, int n) {
+    connects(call, callable) and
+    exists(Function f, int posCount, int argNr |
+      f = callable.getScope() and
+      f.hasVarArg() and
+      posCount = f.getPositionalParameterCount() and
+      result = call.getArg(argNr) and
+      argNr >= posCount and
+      argNr = posCount + n
+    )
+  }
+
+  /** Gets the control flow node that is passed as the overflow keyword argument with key `key`. */
+  ControlFlowNode getKeywordOverflowArg(CallNode call, CallableValue callable, string key) {
+    connects(call, callable) and
+    exists(Function f |
+      f = callable.getScope() and
+      f.hasKwArg() and
+      not exists(f.getArgByName(key)) and
+      result = call.getArgByName(key)
+    )
+  }
+
+  /**
+   * Holds if `call` unpacks a dictionary argument in order to pass it via `name`.
+   * It will then be passed to the parameter of `callable` at index `paramN`.
+   */
+  predicate call_unpacks(
+    CallNode call, ArgParamMapping mapping, CallableValue callable, string name, int paramN
+  ) {
+    connects(call, callable) and
+    exists(Function f |
+      f = callable.getScope() and
+      not exists(int argN | paramN = mapping.getParamN(argN) | exists(call.getArg(argN))) and // no positional argument available
+      name = f.getArgName(paramN) and
+      // not exists(call.getArgByName(name)) and // only matches keyword arguments not preceded by **
+      // TODO: make the below logic respect control flow splitting (by not going to the AST).
+      not call.getNode().getANamedArg().(Keyword).getArg() = name and // no keyword argument available
+      paramN >= 0 and
+      paramN < f.getPositionalParameterCount() + f.getKeywordOnlyParameterCount() and
+      exists(call.getNode().getKwargs()) // dict argument available
+    )
+  }
+}
+
+import ArgumentPassing
+
+/**
+ * IPA type for DataFlowCallable.
+ *
+ * A callable is either a function value, a class value, or a module (for enclosing `ModuleVariableNode`s).
+ * A module has no calls.
+ */
+newtype TDataFlowCallable =
+  TCallableValue(CallableValue callable) {
+    callable instanceof FunctionValue and
+    not callable.(FunctionValue).isLambda()
+    or
+    callable instanceof ClassValue
+  } or
+  TLambda(Function lambda) { lambda.isLambda() } or
+  TModule(Module m)
+
+/** A callable. */
+abstract class DataFlowCallable extends TDataFlowCallable {
+  /** Gets a textual representation of this element. */
+  abstract string toString();
+
+  /** Gets a call to this callable. */
+  abstract CallNode getACall();
+
+  /** Gets the scope of this callable */
+  abstract Scope getScope();
+
+  /** Gets the specified parameter of this callable */
+  abstract NameNode getParameter(int n);
+
+  /** Gets the name of this callable. */
+  abstract string getName();
+
+  /** Gets a callable value for this callable, if one exists. */
+  abstract CallableValue getCallableValue();
+}
+
+/** A class representing a callable value. */
+class DataFlowCallableValue extends DataFlowCallable, TCallableValue {
+  CallableValue callable;
+
+  DataFlowCallableValue() { this = TCallableValue(callable) }
+
+  override string toString() { result = callable.toString() }
+
+  override CallNode getACall() { result = callable.getACall() }
+
+  override Scope getScope() { result = callable.getScope() }
+
+  override NameNode getParameter(int n) { result = getParameter(callable, n) }
+
+  override string getName() { result = callable.getName() }
+
+  override CallableValue getCallableValue() { result = callable }
+}
+
+/** A class representing a callable lambda. */
+class DataFlowLambda extends DataFlowCallable, TLambda {
+  Function lambda;
+
+  DataFlowLambda() { this = TLambda(lambda) }
+
+  override string toString() { result = lambda.toString() }
+
+  override CallNode getACall() { result = this.getCallableValue().getACall() }
+
+  override Scope getScope() { result = lambda.getEvaluatingScope() }
+
+  override NameNode getParameter(int n) { result = getParameter(this.getCallableValue(), n) }
+
+  override string getName() { result = "Lambda callable" }
+
+  override FunctionValue getCallableValue() {
+    result.getOrigin().getNode() = lambda.getDefinition()
+  }
+}
+
+/** A class representing the scope in which a `ModuleVariableNode` appears. */
+class DataFlowModuleScope extends DataFlowCallable, TModule {
+  Module mod;
+
+  DataFlowModuleScope() { this = TModule(mod) }
+
+  override string toString() { result = mod.toString() }
+
+  override CallNode getACall() { none() }
+
+  override Scope getScope() { result = mod }
+
+  override NameNode getParameter(int n) { none() }
+
+  override string getName() { result = mod.getName() }
+
+  override CallableValue getCallableValue() { none() }
+}
+
+/**
+ * IPA type for DataFlowCall.
+ *
+ * Calls corresponding to `CallNode`s are either to callable values or to classes.
+ * The latter is directed to the callable corresponding to the `__init__` method of the class.
+ *
+ * An `__init__` method can also be called directly, so that the callable can be targeted by
+ * different types of calls. In that case, the parameter mappings will be different,
+ * as the class call will synthesize an argument node to be mapped to the `self` parameter.
+ *
+ * A call corresponding to a special method call is handled by the corresponding `SpecialMethodCallNode`.
+ *
+ * TODO: Add `TClassMethodCall` mapping `cls` appropriately.
+ */
+newtype TDataFlowCall =
+  TFunctionCall(CallNode call) { call = any(FunctionValue f).getAFunctionCall() } or
+  /** Bound methods need to make room for the explicit self parameter */
+  TMethodCall(CallNode call) { call = any(FunctionValue f).getAMethodCall() } or
+  TClassCall(CallNode call) { call = any(ClassValue c | not c.isAbsent()).getACall() } or
+  TSpecialCall(SpecialMethodCallNode special)
+
+/** A call. */
+abstract class DataFlowCall extends TDataFlowCall {
+  /** Gets a textual representation of this element. */
+  abstract string toString();
+
+  /** Get the callable to which this call goes. */
+  abstract DataFlowCallable getCallable();
+
+  /**
+   * Gets the argument to this call that will be sent
+   * to the `n`th parameter of the callable.
+   */
+  abstract Node getArg(int n);
+
+  /** Get the control flow node representing this call. */
+  abstract ControlFlowNode getNode();
+
+  /** Gets the enclosing callable of this call. */
+  abstract DataFlowCallable getEnclosingCallable();
+
+  /** Gets the location of this dataflow call. */
+  Location getLocation() { result = this.getNode().getLocation() }
+}
+
+/**
+ * A call to a function/lambda.
+ * This excludes calls to bound methods, classes, and special methods.
+ * Bound method calls and class calls insert an argument for the explicit
+ * `self` parameter, and special method calls have special argument passing.
+ */
+class FunctionCall extends DataFlowCall, TFunctionCall {
+  CallNode call;
+  DataFlowCallable callable;
+
+  FunctionCall() {
+    this = TFunctionCall(call) and
+    call = callable.getACall()
+  }
+
+  override string toString() { result = call.toString() }
+
+  override Node getArg(int n) { result = getArg(call, TNoShift(), callable.getCallableValue(), n) }
+
+  override ControlFlowNode getNode() { result = call }
+
+  override DataFlowCallable getCallable() { result = callable }
+
+  override DataFlowCallable getEnclosingCallable() { result.getScope() = call.getNode().getScope() }
+}
+
+/**
+ * Represents a call to a bound method call.
+ * The node representing the instance is inserted as argument to the `self` parameter.
+ */
+class MethodCall extends DataFlowCall, TMethodCall {
+  CallNode call;
+  FunctionValue bm;
+
+  MethodCall() {
+    this = TMethodCall(call) and
+    call = bm.getACall()
+  }
+
+  private CallableValue getCallableValue() { result = bm }
+
+  override string toString() { result = call.toString() }
+
+  override Node getArg(int n) {
+    n > 0 and result = getArg(call, TShiftOneUp(), this.getCallableValue(), n)
+    or
+    n = 0 and result = TCfgNode(call.getFunction().(AttrNode).getObject())
+  }
+
+  override ControlFlowNode getNode() { result = call }
+
+  override DataFlowCallable getCallable() { result = TCallableValue(this.getCallableValue()) }
+
+  override DataFlowCallable getEnclosingCallable() { result.getScope() = call.getScope() }
+}
+
+/**
+ * Represents a call to a class.
+ * The pre-update node for the call is inserted as argument to the `self` parameter.
+ * That makes the call node be the post-update node holding the value of the object
+ * after the constructor has run.
+ */
+class ClassCall extends DataFlowCall, TClassCall {
+  CallNode call;
+  ClassValue c;
+
+  ClassCall() {
+    this = TClassCall(call) and
+    call = c.getACall()
+  }
+
+  private CallableValue getCallableValue() { c.getScope().getInitMethod() = result.getScope() }
+
+  override string toString() { result = call.toString() }
+
+  override Node getArg(int n) {
+    n > 0 and result = getArg(call, TShiftOneUp(), this.getCallableValue(), n)
+    or
+    n = 0 and result = TSyntheticPreUpdateNode(TCfgNode(call))
+  }
+
+  override ControlFlowNode getNode() { result = call }
+
+  override DataFlowCallable getCallable() { result = TCallableValue(this.getCallableValue()) }
+
+  override DataFlowCallable getEnclosingCallable() { result.getScope() = call.getScope() }
+}
+
+/** A call to a special method. */
+class SpecialCall extends DataFlowCall, TSpecialCall {
+  SpecialMethodCallNode special;
+
+  SpecialCall() { this = TSpecialCall(special) }
+
+  override string toString() { result = special.toString() }
+
+  override Node getArg(int n) { result = TCfgNode(special.(SpecialMethod::Potential).getArg(n)) }
+
+  override ControlFlowNode getNode() { result = special }
+
+  override DataFlowCallable getCallable() {
+    result = TCallableValue(special.getResolvedSpecialMethod())
+  }
+
+  override DataFlowCallable getEnclosingCallable() {
+    result.getScope() = special.getNode().getScope()
+  }
+}
+
+/** Gets a viable run-time target for the call `call`. */
+DataFlowCallable viableCallable(DataFlowCall call) { result = call.getCallable() }
+
+private newtype TReturnKind = TNormalReturnKind()
+
+/**
+ * A return kind. A return kind describes how a value can be returned
+ * from a callable. For Python, this is simply a method return.
+ */
+class ReturnKind extends TReturnKind {
+  /** Gets a textual representation of this element. */
+  string toString() { result = "return" }
+}
+
+/** A data flow node that represents a value returned by a callable. */
+class ReturnNode extends CfgNode {
+  Return ret;
+
+  // See `TaintTrackingImplementation::returnFlowStep`
+  ReturnNode() { node = ret.getValue().getAFlowNode() }
+
+  /** Gets the kind of this return node. */
+  ReturnKind getKind() { any() }
+}
+
+/** A data flow node that represents the output of a call. */
+class OutNode extends CfgNode {
+  OutNode() { node instanceof CallNode }
+}
+
+/**
+ * Gets a node that can read the value returned from `call` with return kind
+ * `kind`.
+ */
+OutNode getAnOutNode(DataFlowCall call, ReturnKind kind) {
+  call.getNode() = result.getNode() and
+  kind = TNormalReturnKind()
+}

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl.qll

@@ -87,21 +87,9 @@ abstract class Configuration extends string {
   /** Holds if data flow into `node` is prohibited. */
   predicate isBarrierIn(Node node) { none() }
 
-  /**
-   * Holds if data flow into `node` is prohibited when the flow state is
-   * `state`
-   */
-  predicate isBarrierIn(Node node, FlowState state) { none() }
-
   /** Holds if data flow out of `node` is prohibited. */
   predicate isBarrierOut(Node node) { none() }
 
-  /**
-   * Holds if data flow out of `node` is prohibited when the flow state is
-   * `state`
-   */
-  predicate isBarrierOut(Node node, FlowState state) { none() }
-
   /** Holds if data flow through nodes guarded by `guard` is prohibited. */
   predicate isBarrierGuard(BarrierGuard guard) { none() }
 
@@ -112,15 +100,13 @@ abstract class Configuration extends string {
   predicate isBarrierGuard(BarrierGuard guard, FlowState state) { none() }
 
   /**
-   * Holds if the additional flow step from `node1` to `node2` must be taken
-   * into account in the analysis.
+   * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps.
    */
   predicate isAdditionalFlowStep(Node node1, Node node2) { none() }
 
   /**
-   * Holds if the additional flow step from `node1` to `node2` must be taken
-   * into account in the analysis. This step is only applicable in `state1` and
-   * updates the flow state to `state2`.
+   * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps.
+   * This step is only applicable in `state1` and updates the flow state to `state2`.
    */
   predicate isAdditionalFlowStep(Node node1, FlowState state1, Node node2, FlowState state2) {
     none()
@@ -323,7 +309,7 @@ private class RetNodeEx extends NodeEx {
   ReturnKindExt getKind() { result = this.asNode().(ReturnNodeExt).getKind() }
 }
 
-private predicate fullInBarrier(NodeEx node, Configuration config) {
+private predicate inBarrier(NodeEx node, Configuration config) {
   exists(Node n |
     node.asNode() = n and
     config.isBarrierIn(n)
@@ -332,16 +318,7 @@ private predicate fullInBarrier(NodeEx node, Configuration config) {
   )
 }
 
-private predicate stateInBarrier(NodeEx node, FlowState state, Configuration config) {
-  exists(Node n |
-    node.asNode() = n and
-    config.isBarrierIn(n, state)
-  |
-    config.isSource(n, state)
-  )
-}
-
-private predicate fullOutBarrier(NodeEx node, Configuration config) {
+private predicate outBarrier(NodeEx node, Configuration config) {
   exists(Node n |
     node.asNode() = n and
     config.isBarrierOut(n)
@@ -350,15 +327,6 @@ private predicate fullOutBarrier(NodeEx node, Configuration config) {
   )
 }
 
-private predicate stateOutBarrier(NodeEx node, FlowState state, Configuration config) {
-  exists(Node n |
-    node.asNode() = n and
-    config.isBarrierOut(n, state)
-  |
-    config.isSink(n, state)
-  )
-}
-
 pragma[nomagic]
 private predicate fullBarrier(NodeEx node, Configuration config) {
   exists(Node n | node.asNode() = n |
@@ -384,12 +352,6 @@ private predicate stateBarrier(NodeEx node, FlowState state, Configuration confi
   exists(Node n | node.asNode() = n |
     config.isBarrier(n, state)
     or
-    config.isBarrierIn(n, state) and
-    not config.isSource(n, state)
-    or
-    config.isBarrierOut(n, state) and
-    not config.isSink(n, state)
-    or
     exists(BarrierGuard g |
       config.isBarrierGuard(g, state) and
       n = g.getAGuardedNode()
@@ -422,8 +384,8 @@ private predicate sinkNode(NodeEx node, FlowState state, Configuration config) {
 /** Provides the relevant barriers for a step from `node1` to `node2`. */
 pragma[inline]
 private predicate stepFilter(NodeEx node1, NodeEx node2, Configuration config) {
-  not fullOutBarrier(node1, config) and
-  not fullInBarrier(node2, config) and
+  not outBarrier(node1, config) and
+  not inBarrier(node2, config) and
   not fullBarrier(node1, config) and
   not fullBarrier(node2, config)
 }
@@ -476,8 +438,6 @@ private predicate additionalLocalStateStep(
     config.isAdditionalFlowStep(n1, s1, n2, s2) and
     getNodeEnclosingCallable(n1) = getNodeEnclosingCallable(n2) and
     stepFilter(node1, node2, config) and
-    not stateOutBarrier(node1, s1, config) and
-    not stateInBarrier(node2, s2, config) and
     not stateBarrier(node1, s1, config) and
     not stateBarrier(node2, s2, config)
   )
@@ -519,8 +479,6 @@ private predicate additionalJumpStateStep(
     config.isAdditionalFlowStep(n1, s1, n2, s2) and
     getNodeEnclosingCallable(n1) != getNodeEnclosingCallable(n2) and
     stepFilter(node1, node2, config) and
-    not stateOutBarrier(node1, s1, config) and
-    not stateInBarrier(node2, s2, config) and
     not stateBarrier(node1, s1, config) and
     not stateBarrier(node2, s2, config) and
     not config.getAFeature() instanceof FeatureEqualSourceSinkCallContext
@@ -920,8 +878,8 @@ private module Stage1 {
   private predicate throughFlowNodeCand(NodeEx node, Configuration config) {
     revFlow(node, true, config) and
     fwdFlow(node, true, config) and
-    not fullInBarrier(node, config) and
-    not fullOutBarrier(node, config)
+    not inBarrier(node, config) and
+    not outBarrier(node, config)
   }
 
   /** Holds if flow may return from `callable`. */
@@ -1016,8 +974,8 @@ private predicate flowOutOfCallNodeCand1(
 ) {
   viableReturnPosOutNodeCand1(call, ret.getReturnPosition(), out, config) and
   Stage1::revFlow(ret, config) and
-  not fullOutBarrier(ret, config) and
-  not fullInBarrier(out, config)
+  not outBarrier(ret, config) and
+  not inBarrier(out, config)
 }
 
 pragma[nomagic]
@@ -1038,8 +996,8 @@ private predicate flowIntoCallNodeCand1(
 ) {
   viableParamArgNodeCand1(call, p, arg, config) and
   Stage1::revFlow(p, config) and
-  not fullOutBarrier(arg, config) and
-  not fullInBarrier(p, config)
+  not outBarrier(arg, config) and
+  not inBarrier(p, config)
 }
 
 /**
@@ -1160,8 +1118,8 @@ private module Stage2 {
     if reducedViableImplInReturn(c, call) then result = TReturn(c, call) else result = ccNone()
   }
 
-  bindingset[node, cc, config]
-  private LocalCc getLocalCc(NodeEx node, Cc cc, Configuration config) { any() }
+  bindingset[node, cc]
+  private LocalCc getLocalCc(NodeEx node, Cc cc) { any() }
 
   bindingset[node1, state1, config]
   bindingset[node2, state2, config]
@@ -1248,7 +1206,7 @@ private module Stage2 {
     or
     exists(NodeEx mid, FlowState state0, Ap ap0, LocalCc localCc |
       fwdFlow(mid, state0, cc, argAp, ap0, config) and
-      localCc = getLocalCc(mid, cc, config)
+      localCc = getLocalCc(mid, cc)
     |
       localStep(mid, state0, node, state, true, _, config, localCc) and
       ap = ap0
@@ -1953,8 +1911,8 @@ private module Stage3 {
   bindingset[call, c, innercc]
   private CcNoCall getCallContextReturn(DataFlowCallable c, DataFlowCall call, Cc innercc) { any() }
 
-  bindingset[node, cc, config]
-  private LocalCc getLocalCc(NodeEx node, Cc cc, Configuration config) { any() }
+  bindingset[node, cc]
+  private LocalCc getLocalCc(NodeEx node, Cc cc) { any() }
 
   private predicate localStep(
     NodeEx node1, FlowState state1, NodeEx node2, FlowState state2, boolean preservesValue,
@@ -2037,7 +1995,7 @@ private module Stage3 {
     or
     exists(NodeEx mid, FlowState state0, Ap ap0, LocalCc localCc |
       fwdFlow(mid, state0, cc, argAp, ap0, config) and
-      localCc = getLocalCc(mid, cc, config)
+      localCc = getLocalCc(mid, cc)
     |
       localStep(mid, state0, node, state, true, _, config, localCc) and
       ap = ap0
@@ -2767,12 +2725,11 @@ private module Stage4 {
     if reducedViableImplInReturn(c, call) then result = TReturn(c, call) else result = ccNone()
   }
 
-  bindingset[node, cc, config]
-  private LocalCc getLocalCc(NodeEx node, Cc cc, Configuration config) {
+  bindingset[node, cc]
+  private LocalCc getLocalCc(NodeEx node, Cc cc) {
     result =
       getLocalCallContext(pragma[only_bind_into](pragma[only_bind_out](cc)),
-        node.getEnclosingCallable()) and
-    exists(config)
+        node.getEnclosingCallable())
   }
 
   private predicate localStep(
@@ -2865,7 +2822,7 @@ private module Stage4 {
     or
     exists(NodeEx mid, FlowState state0, Ap ap0, LocalCc localCc |
       fwdFlow(mid, state0, cc, argAp, ap0, config) and
-      localCc = getLocalCc(mid, cc, config)
+      localCc = getLocalCc(mid, cc)
     |
       localStep(mid, state0, node, state, true, _, config, localCc) and
       ap = ap0
@@ -5050,6 +5007,7 @@ private module FlowExploration {
     )
   }
 
+  pragma[nomagic]
   private predicate revPartialPathStep(
     PartialPathNodeRev mid, NodeEx node, FlowState state, TRevSummaryCtx1 sc1, TRevSummaryCtx2 sc2,
     TRevSummaryCtx3 sc3, RevPartialAccessPath ap, Configuration config

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl2.qll

@@ -87,21 +87,9 @@ abstract class Configuration extends string {
   /** Holds if data flow into `node` is prohibited. */
   predicate isBarrierIn(Node node) { none() }
 
-  /**
-   * Holds if data flow into `node` is prohibited when the flow state is
-   * `state`
-   */
-  predicate isBarrierIn(Node node, FlowState state) { none() }
-
   /** Holds if data flow out of `node` is prohibited. */
   predicate isBarrierOut(Node node) { none() }
 
-  /**
-   * Holds if data flow out of `node` is prohibited when the flow state is
-   * `state`
-   */
-  predicate isBarrierOut(Node node, FlowState state) { none() }
-
   /** Holds if data flow through nodes guarded by `guard` is prohibited. */
   predicate isBarrierGuard(BarrierGuard guard) { none() }
 
@@ -112,15 +100,13 @@ abstract class Configuration extends string {
   predicate isBarrierGuard(BarrierGuard guard, FlowState state) { none() }
 
   /**
-   * Holds if the additional flow step from `node1` to `node2` must be taken
-   * into account in the analysis.
+   * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps.
    */
   predicate isAdditionalFlowStep(Node node1, Node node2) { none() }
 
   /**
-   * Holds if the additional flow step from `node1` to `node2` must be taken
-   * into account in the analysis. This step is only applicable in `state1` and
-   * updates the flow state to `state2`.
+   * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps.
+   * This step is only applicable in `state1` and updates the flow state to `state2`.
    */
   predicate isAdditionalFlowStep(Node node1, FlowState state1, Node node2, FlowState state2) {
     none()
@@ -323,7 +309,7 @@ private class RetNodeEx extends NodeEx {
   ReturnKindExt getKind() { result = this.asNode().(ReturnNodeExt).getKind() }
 }
 
-private predicate fullInBarrier(NodeEx node, Configuration config) {
+private predicate inBarrier(NodeEx node, Configuration config) {
   exists(Node n |
     node.asNode() = n and
     config.isBarrierIn(n)
@@ -332,16 +318,7 @@ private predicate fullInBarrier(NodeEx node, Configuration config) {
   )
 }
 
-private predicate stateInBarrier(NodeEx node, FlowState state, Configuration config) {
-  exists(Node n |
-    node.asNode() = n and
-    config.isBarrierIn(n, state)
-  |
-    config.isSource(n, state)
-  )
-}
-
-private predicate fullOutBarrier(NodeEx node, Configuration config) {
+private predicate outBarrier(NodeEx node, Configuration config) {
   exists(Node n |
     node.asNode() = n and
     config.isBarrierOut(n)
@@ -350,15 +327,6 @@ private predicate fullOutBarrier(NodeEx node, Configuration config) {
   )
 }
 
-private predicate stateOutBarrier(NodeEx node, FlowState state, Configuration config) {
-  exists(Node n |
-    node.asNode() = n and
-    config.isBarrierOut(n, state)
-  |
-    config.isSink(n, state)
-  )
-}
-
 pragma[nomagic]
 private predicate fullBarrier(NodeEx node, Configuration config) {
   exists(Node n | node.asNode() = n |
@@ -384,12 +352,6 @@ private predicate stateBarrier(NodeEx node, FlowState state, Configuration confi
   exists(Node n | node.asNode() = n |
     config.isBarrier(n, state)
     or
-    config.isBarrierIn(n, state) and
-    not config.isSource(n, state)
-    or
-    config.isBarrierOut(n, state) and
-    not config.isSink(n, state)
-    or
     exists(BarrierGuard g |
       config.isBarrierGuard(g, state) and
       n = g.getAGuardedNode()
@@ -422,8 +384,8 @@ private predicate sinkNode(NodeEx node, FlowState state, Configuration config) {
 /** Provides the relevant barriers for a step from `node1` to `node2`. */
 pragma[inline]
 private predicate stepFilter(NodeEx node1, NodeEx node2, Configuration config) {
-  not fullOutBarrier(node1, config) and
-  not fullInBarrier(node2, config) and
+  not outBarrier(node1, config) and
+  not inBarrier(node2, config) and
   not fullBarrier(node1, config) and
   not fullBarrier(node2, config)
 }
@@ -476,8 +438,6 @@ private predicate additionalLocalStateStep(
     config.isAdditionalFlowStep(n1, s1, n2, s2) and
     getNodeEnclosingCallable(n1) = getNodeEnclosingCallable(n2) and
     stepFilter(node1, node2, config) and
-    not stateOutBarrier(node1, s1, config) and
-    not stateInBarrier(node2, s2, config) and
     not stateBarrier(node1, s1, config) and
     not stateBarrier(node2, s2, config)
   )
@@ -519,8 +479,6 @@ private predicate additionalJumpStateStep(
     config.isAdditionalFlowStep(n1, s1, n2, s2) and
     getNodeEnclosingCallable(n1) != getNodeEnclosingCallable(n2) and
     stepFilter(node1, node2, config) and
-    not stateOutBarrier(node1, s1, config) and
-    not stateInBarrier(node2, s2, config) and
     not stateBarrier(node1, s1, config) and
     not stateBarrier(node2, s2, config) and
     not config.getAFeature() instanceof FeatureEqualSourceSinkCallContext
@@ -920,8 +878,8 @@ private module Stage1 {
   private predicate throughFlowNodeCand(NodeEx node, Configuration config) {
     revFlow(node, true, config) and
     fwdFlow(node, true, config) and
-    not fullInBarrier(node, config) and
-    not fullOutBarrier(node, config)
+    not inBarrier(node, config) and
+    not outBarrier(node, config)
   }
 
   /** Holds if flow may return from `callable`. */
@@ -1016,8 +974,8 @@ private predicate flowOutOfCallNodeCand1(
 ) {
   viableReturnPosOutNodeCand1(call, ret.getReturnPosition(), out, config) and
   Stage1::revFlow(ret, config) and
-  not fullOutBarrier(ret, config) and
-  not fullInBarrier(out, config)
+  not outBarrier(ret, config) and
+  not inBarrier(out, config)
 }
 
 pragma[nomagic]
@@ -1038,8 +996,8 @@ private predicate flowIntoCallNodeCand1(
 ) {
   viableParamArgNodeCand1(call, p, arg, config) and
   Stage1::revFlow(p, config) and
-  not fullOutBarrier(arg, config) and
-  not fullInBarrier(p, config)
+  not outBarrier(arg, config) and
+  not inBarrier(p, config)
 }
 
 /**
@@ -1160,8 +1118,8 @@ private module Stage2 {
     if reducedViableImplInReturn(c, call) then result = TReturn(c, call) else result = ccNone()
   }
 
-  bindingset[node, cc, config]
-  private LocalCc getLocalCc(NodeEx node, Cc cc, Configuration config) { any() }
+  bindingset[node, cc]
+  private LocalCc getLocalCc(NodeEx node, Cc cc) { any() }
 
   bindingset[node1, state1, config]
   bindingset[node2, state2, config]
@@ -1248,7 +1206,7 @@ private module Stage2 {
     or
     exists(NodeEx mid, FlowState state0, Ap ap0, LocalCc localCc |
       fwdFlow(mid, state0, cc, argAp, ap0, config) and
-      localCc = getLocalCc(mid, cc, config)
+      localCc = getLocalCc(mid, cc)
     |
       localStep(mid, state0, node, state, true, _, config, localCc) and
       ap = ap0
@@ -1953,8 +1911,8 @@ private module Stage3 {
   bindingset[call, c, innercc]
   private CcNoCall getCallContextReturn(DataFlowCallable c, DataFlowCall call, Cc innercc) { any() }
 
-  bindingset[node, cc, config]
-  private LocalCc getLocalCc(NodeEx node, Cc cc, Configuration config) { any() }
+  bindingset[node, cc]
+  private LocalCc getLocalCc(NodeEx node, Cc cc) { any() }
 
   private predicate localStep(
     NodeEx node1, FlowState state1, NodeEx node2, FlowState state2, boolean preservesValue,
@@ -2037,7 +1995,7 @@ private module Stage3 {
     or
     exists(NodeEx mid, FlowState state0, Ap ap0, LocalCc localCc |
       fwdFlow(mid, state0, cc, argAp, ap0, config) and
-      localCc = getLocalCc(mid, cc, config)
+      localCc = getLocalCc(mid, cc)
     |
       localStep(mid, state0, node, state, true, _, config, localCc) and
       ap = ap0
@@ -2767,12 +2725,11 @@ private module Stage4 {
     if reducedViableImplInReturn(c, call) then result = TReturn(c, call) else result = ccNone()
   }
 
-  bindingset[node, cc, config]
-  private LocalCc getLocalCc(NodeEx node, Cc cc, Configuration config) {
+  bindingset[node, cc]
+  private LocalCc getLocalCc(NodeEx node, Cc cc) {
     result =
       getLocalCallContext(pragma[only_bind_into](pragma[only_bind_out](cc)),
-        node.getEnclosingCallable()) and
-    exists(config)
+        node.getEnclosingCallable())
   }
 
   private predicate localStep(
@@ -2865,7 +2822,7 @@ private module Stage4 {
     or
     exists(NodeEx mid, FlowState state0, Ap ap0, LocalCc localCc |
       fwdFlow(mid, state0, cc, argAp, ap0, config) and
-      localCc = getLocalCc(mid, cc, config)
+      localCc = getLocalCc(mid, cc)
     |
       localStep(mid, state0, node, state, true, _, config, localCc) and
       ap = ap0
@@ -5050,6 +5007,7 @@ private module FlowExploration {
     )
   }
 
+  pragma[nomagic]
   private predicate revPartialPathStep(
     PartialPathNodeRev mid, NodeEx node, FlowState state, TRevSummaryCtx1 sc1, TRevSummaryCtx2 sc2,
     TRevSummaryCtx3 sc3, RevPartialAccessPath ap, Configuration config

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl3.qll

@@ -87,21 +87,9 @@ abstract class Configuration extends string {
   /** Holds if data flow into `node` is prohibited. */
   predicate isBarrierIn(Node node) { none() }
 
-  /**
-   * Holds if data flow into `node` is prohibited when the flow state is
-   * `state`
-   */
-  predicate isBarrierIn(Node node, FlowState state) { none() }
-
   /** Holds if data flow out of `node` is prohibited. */
   predicate isBarrierOut(Node node) { none() }
 
-  /**
-   * Holds if data flow out of `node` is prohibited when the flow state is
-   * `state`
-   */
-  predicate isBarrierOut(Node node, FlowState state) { none() }
-
   /** Holds if data flow through nodes guarded by `guard` is prohibited. */
   predicate isBarrierGuard(BarrierGuard guard) { none() }
 
@@ -112,15 +100,13 @@ abstract class Configuration extends string {
   predicate isBarrierGuard(BarrierGuard guard, FlowState state) { none() }
 
   /**
-   * Holds if the additional flow step from `node1` to `node2` must be taken
-   * into account in the analysis.
+   * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps.
    */
   predicate isAdditionalFlowStep(Node node1, Node node2) { none() }
 
   /**
-   * Holds if the additional flow step from `node1` to `node2` must be taken
-   * into account in the analysis. This step is only applicable in `state1` and
-   * updates the flow state to `state2`.
+   * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps.
+   * This step is only applicable in `state1` and updates the flow state to `state2`.
    */
   predicate isAdditionalFlowStep(Node node1, FlowState state1, Node node2, FlowState state2) {
     none()
@@ -323,7 +309,7 @@ private class RetNodeEx extends NodeEx {
   ReturnKindExt getKind() { result = this.asNode().(ReturnNodeExt).getKind() }
 }
 
-private predicate fullInBarrier(NodeEx node, Configuration config) {
+private predicate inBarrier(NodeEx node, Configuration config) {
   exists(Node n |
     node.asNode() = n and
     config.isBarrierIn(n)
@@ -332,16 +318,7 @@ private predicate fullInBarrier(NodeEx node, Configuration config) {
   )
 }
 
-private predicate stateInBarrier(NodeEx node, FlowState state, Configuration config) {
-  exists(Node n |
-    node.asNode() = n and
-    config.isBarrierIn(n, state)
-  |
-    config.isSource(n, state)
-  )
-}
-
-private predicate fullOutBarrier(NodeEx node, Configuration config) {
+private predicate outBarrier(NodeEx node, Configuration config) {
   exists(Node n |
     node.asNode() = n and
     config.isBarrierOut(n)
@@ -350,15 +327,6 @@ private predicate fullOutBarrier(NodeEx node, Configuration config) {
   )
 }
 
-private predicate stateOutBarrier(NodeEx node, FlowState state, Configuration config) {
-  exists(Node n |
-    node.asNode() = n and
-    config.isBarrierOut(n, state)
-  |
-    config.isSink(n, state)
-  )
-}
-
 pragma[nomagic]
 private predicate fullBarrier(NodeEx node, Configuration config) {
   exists(Node n | node.asNode() = n |
@@ -384,12 +352,6 @@ private predicate stateBarrier(NodeEx node, FlowState state, Configuration confi
   exists(Node n | node.asNode() = n |
     config.isBarrier(n, state)
     or
-    config.isBarrierIn(n, state) and
-    not config.isSource(n, state)
-    or
-    config.isBarrierOut(n, state) and
-    not config.isSink(n, state)
-    or
     exists(BarrierGuard g |
       config.isBarrierGuard(g, state) and
       n = g.getAGuardedNode()
@@ -422,8 +384,8 @@ private predicate sinkNode(NodeEx node, FlowState state, Configuration config) {
 /** Provides the relevant barriers for a step from `node1` to `node2`. */
 pragma[inline]
 private predicate stepFilter(NodeEx node1, NodeEx node2, Configuration config) {
-  not fullOutBarrier(node1, config) and
-  not fullInBarrier(node2, config) and
+  not outBarrier(node1, config) and
+  not inBarrier(node2, config) and
   not fullBarrier(node1, config) and
   not fullBarrier(node2, config)
 }
@@ -476,8 +438,6 @@ private predicate additionalLocalStateStep(
     config.isAdditionalFlowStep(n1, s1, n2, s2) and
     getNodeEnclosingCallable(n1) = getNodeEnclosingCallable(n2) and
     stepFilter(node1, node2, config) and
-    not stateOutBarrier(node1, s1, config) and
-    not stateInBarrier(node2, s2, config) and
     not stateBarrier(node1, s1, config) and
     not stateBarrier(node2, s2, config)
   )
@@ -519,8 +479,6 @@ private predicate additionalJumpStateStep(
     config.isAdditionalFlowStep(n1, s1, n2, s2) and
     getNodeEnclosingCallable(n1) != getNodeEnclosingCallable(n2) and
     stepFilter(node1, node2, config) and
-    not stateOutBarrier(node1, s1, config) and
-    not stateInBarrier(node2, s2, config) and
     not stateBarrier(node1, s1, config) and
     not stateBarrier(node2, s2, config) and
     not config.getAFeature() instanceof FeatureEqualSourceSinkCallContext
@@ -920,8 +878,8 @@ private module Stage1 {
   private predicate throughFlowNodeCand(NodeEx node, Configuration config) {
     revFlow(node, true, config) and
     fwdFlow(node, true, config) and
-    not fullInBarrier(node, config) and
-    not fullOutBarrier(node, config)
+    not inBarrier(node, config) and
+    not outBarrier(node, config)
   }
 
   /** Holds if flow may return from `callable`. */
@@ -1016,8 +974,8 @@ private predicate flowOutOfCallNodeCand1(
 ) {
   viableReturnPosOutNodeCand1(call, ret.getReturnPosition(), out, config) and
   Stage1::revFlow(ret, config) and
-  not fullOutBarrier(ret, config) and
-  not fullInBarrier(out, config)
+  not outBarrier(ret, config) and
+  not inBarrier(out, config)
 }
 
 pragma[nomagic]
@@ -1038,8 +996,8 @@ private predicate flowIntoCallNodeCand1(
 ) {
   viableParamArgNodeCand1(call, p, arg, config) and
   Stage1::revFlow(p, config) and
-  not fullOutBarrier(arg, config) and
-  not fullInBarrier(p, config)
+  not outBarrier(arg, config) and
+  not inBarrier(p, config)
 }
 
 /**
@@ -1160,8 +1118,8 @@ private module Stage2 {
     if reducedViableImplInReturn(c, call) then result = TReturn(c, call) else result = ccNone()
   }
 
-  bindingset[node, cc, config]
-  private LocalCc getLocalCc(NodeEx node, Cc cc, Configuration config) { any() }
+  bindingset[node, cc]
+  private LocalCc getLocalCc(NodeEx node, Cc cc) { any() }
 
   bindingset[node1, state1, config]
   bindingset[node2, state2, config]
@@ -1248,7 +1206,7 @@ private module Stage2 {
     or
     exists(NodeEx mid, FlowState state0, Ap ap0, LocalCc localCc |
       fwdFlow(mid, state0, cc, argAp, ap0, config) and
-      localCc = getLocalCc(mid, cc, config)
+      localCc = getLocalCc(mid, cc)
     |
       localStep(mid, state0, node, state, true, _, config, localCc) and
       ap = ap0
@@ -1953,8 +1911,8 @@ private module Stage3 {
   bindingset[call, c, innercc]
   private CcNoCall getCallContextReturn(DataFlowCallable c, DataFlowCall call, Cc innercc) { any() }
 
-  bindingset[node, cc, config]
-  private LocalCc getLocalCc(NodeEx node, Cc cc, Configuration config) { any() }
+  bindingset[node, cc]
+  private LocalCc getLocalCc(NodeEx node, Cc cc) { any() }
 
   private predicate localStep(
     NodeEx node1, FlowState state1, NodeEx node2, FlowState state2, boolean preservesValue,
@@ -2037,7 +1995,7 @@ private module Stage3 {
     or
     exists(NodeEx mid, FlowState state0, Ap ap0, LocalCc localCc |
       fwdFlow(mid, state0, cc, argAp, ap0, config) and
-      localCc = getLocalCc(mid, cc, config)
+      localCc = getLocalCc(mid, cc)
     |
       localStep(mid, state0, node, state, true, _, config, localCc) and
       ap = ap0
@@ -2767,12 +2725,11 @@ private module Stage4 {
     if reducedViableImplInReturn(c, call) then result = TReturn(c, call) else result = ccNone()
   }
 
-  bindingset[node, cc, config]
-  private LocalCc getLocalCc(NodeEx node, Cc cc, Configuration config) {
+  bindingset[node, cc]
+  private LocalCc getLocalCc(NodeEx node, Cc cc) {
     result =
       getLocalCallContext(pragma[only_bind_into](pragma[only_bind_out](cc)),
-        node.getEnclosingCallable()) and
-    exists(config)
+        node.getEnclosingCallable())
   }
 
   private predicate localStep(
@@ -2865,7 +2822,7 @@ private module Stage4 {
     or
     exists(NodeEx mid, FlowState state0, Ap ap0, LocalCc localCc |
       fwdFlow(mid, state0, cc, argAp, ap0, config) and
-      localCc = getLocalCc(mid, cc, config)
+      localCc = getLocalCc(mid, cc)
     |
       localStep(mid, state0, node, state, true, _, config, localCc) and
       ap = ap0
@@ -5050,6 +5007,7 @@ private module FlowExploration {
     )
   }
 
+  pragma[nomagic]
   private predicate revPartialPathStep(
     PartialPathNodeRev mid, NodeEx node, FlowState state, TRevSummaryCtx1 sc1, TRevSummaryCtx2 sc2,
     TRevSummaryCtx3 sc3, RevPartialAccessPath ap, Configuration config

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl4.qll

@@ -87,21 +87,9 @@ abstract class Configuration extends string {
   /** Holds if data flow into `node` is prohibited. */
   predicate isBarrierIn(Node node) { none() }
 
-  /**
-   * Holds if data flow into `node` is prohibited when the flow state is
-   * `state`
-   */
-  predicate isBarrierIn(Node node, FlowState state) { none() }
-
   /** Holds if data flow out of `node` is prohibited. */
   predicate isBarrierOut(Node node) { none() }
 
-  /**
-   * Holds if data flow out of `node` is prohibited when the flow state is
-   * `state`
-   */
-  predicate isBarrierOut(Node node, FlowState state) { none() }
-
   /** Holds if data flow through nodes guarded by `guard` is prohibited. */
   predicate isBarrierGuard(BarrierGuard guard) { none() }
 
@@ -112,15 +100,13 @@ abstract class Configuration extends string {
   predicate isBarrierGuard(BarrierGuard guard, FlowState state) { none() }
 
   /**
-   * Holds if the additional flow step from `node1` to `node2` must be taken
-   * into account in the analysis.
+   * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps.
    */
   predicate isAdditionalFlowStep(Node node1, Node node2) { none() }
 
   /**
-   * Holds if the additional flow step from `node1` to `node2` must be taken
-   * into account in the analysis. This step is only applicable in `state1` and
-   * updates the flow state to `state2`.
+   * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps.
+   * This step is only applicable in `state1` and updates the flow state to `state2`.
    */
   predicate isAdditionalFlowStep(Node node1, FlowState state1, Node node2, FlowState state2) {
     none()
@@ -323,7 +309,7 @@ private class RetNodeEx extends NodeEx {
   ReturnKindExt getKind() { result = this.asNode().(ReturnNodeExt).getKind() }
 }
 
-private predicate fullInBarrier(NodeEx node, Configuration config) {
+private predicate inBarrier(NodeEx node, Configuration config) {
   exists(Node n |
     node.asNode() = n and
     config.isBarrierIn(n)
@@ -332,16 +318,7 @@ private predicate fullInBarrier(NodeEx node, Configuration config) {
   )
 }
 
-private predicate stateInBarrier(NodeEx node, FlowState state, Configuration config) {
-  exists(Node n |
-    node.asNode() = n and
-    config.isBarrierIn(n, state)
-  |
-    config.isSource(n, state)
-  )
-}
-
-private predicate fullOutBarrier(NodeEx node, Configuration config) {
+private predicate outBarrier(NodeEx node, Configuration config) {
   exists(Node n |
     node.asNode() = n and
     config.isBarrierOut(n)
@@ -350,15 +327,6 @@ private predicate fullOutBarrier(NodeEx node, Configuration config) {
   )
 }
 
-private predicate stateOutBarrier(NodeEx node, FlowState state, Configuration config) {
-  exists(Node n |
-    node.asNode() = n and
-    config.isBarrierOut(n, state)
-  |
-    config.isSink(n, state)
-  )
-}
-
 pragma[nomagic]
 private predicate fullBarrier(NodeEx node, Configuration config) {
   exists(Node n | node.asNode() = n |
@@ -384,12 +352,6 @@ private predicate stateBarrier(NodeEx node, FlowState state, Configuration confi
   exists(Node n | node.asNode() = n |
     config.isBarrier(n, state)
     or
-    config.isBarrierIn(n, state) and
-    not config.isSource(n, state)
-    or
-    config.isBarrierOut(n, state) and
-    not config.isSink(n, state)
-    or
     exists(BarrierGuard g |
       config.isBarrierGuard(g, state) and
       n = g.getAGuardedNode()
@@ -422,8 +384,8 @@ private predicate sinkNode(NodeEx node, FlowState state, Configuration config) {
 /** Provides the relevant barriers for a step from `node1` to `node2`. */
 pragma[inline]
 private predicate stepFilter(NodeEx node1, NodeEx node2, Configuration config) {
-  not fullOutBarrier(node1, config) and
-  not fullInBarrier(node2, config) and
+  not outBarrier(node1, config) and
+  not inBarrier(node2, config) and
   not fullBarrier(node1, config) and
   not fullBarrier(node2, config)
 }
@@ -476,8 +438,6 @@ private predicate additionalLocalStateStep(
     config.isAdditionalFlowStep(n1, s1, n2, s2) and
     getNodeEnclosingCallable(n1) = getNodeEnclosingCallable(n2) and
     stepFilter(node1, node2, config) and
-    not stateOutBarrier(node1, s1, config) and
-    not stateInBarrier(node2, s2, config) and
     not stateBarrier(node1, s1, config) and
     not stateBarrier(node2, s2, config)
   )
@@ -519,8 +479,6 @@ private predicate additionalJumpStateStep(
     config.isAdditionalFlowStep(n1, s1, n2, s2) and
     getNodeEnclosingCallable(n1) != getNodeEnclosingCallable(n2) and
     stepFilter(node1, node2, config) and
-    not stateOutBarrier(node1, s1, config) and
-    not stateInBarrier(node2, s2, config) and
     not stateBarrier(node1, s1, config) and
     not stateBarrier(node2, s2, config) and
     not config.getAFeature() instanceof FeatureEqualSourceSinkCallContext
@@ -920,8 +878,8 @@ private module Stage1 {
   private predicate throughFlowNodeCand(NodeEx node, Configuration config) {
     revFlow(node, true, config) and
     fwdFlow(node, true, config) and
-    not fullInBarrier(node, config) and
-    not fullOutBarrier(node, config)
+    not inBarrier(node, config) and
+    not outBarrier(node, config)
   }
 
   /** Holds if flow may return from `callable`. */
@@ -1016,8 +974,8 @@ private predicate flowOutOfCallNodeCand1(
 ) {
   viableReturnPosOutNodeCand1(call, ret.getReturnPosition(), out, config) and
   Stage1::revFlow(ret, config) and
-  not fullOutBarrier(ret, config) and
-  not fullInBarrier(out, config)
+  not outBarrier(ret, config) and
+  not inBarrier(out, config)
 }
 
 pragma[nomagic]
@@ -1038,8 +996,8 @@ private predicate flowIntoCallNodeCand1(
 ) {
   viableParamArgNodeCand1(call, p, arg, config) and
   Stage1::revFlow(p, config) and
-  not fullOutBarrier(arg, config) and
-  not fullInBarrier(p, config)
+  not outBarrier(arg, config) and
+  not inBarrier(p, config)
 }
 
 /**
@@ -1160,8 +1118,8 @@ private module Stage2 {
     if reducedViableImplInReturn(c, call) then result = TReturn(c, call) else result = ccNone()
   }
 
-  bindingset[node, cc, config]
-  private LocalCc getLocalCc(NodeEx node, Cc cc, Configuration config) { any() }
+  bindingset[node, cc]
+  private LocalCc getLocalCc(NodeEx node, Cc cc) { any() }
 
   bindingset[node1, state1, config]
   bindingset[node2, state2, config]
@@ -1248,7 +1206,7 @@ private module Stage2 {
     or
     exists(NodeEx mid, FlowState state0, Ap ap0, LocalCc localCc |
       fwdFlow(mid, state0, cc, argAp, ap0, config) and
-      localCc = getLocalCc(mid, cc, config)
+      localCc = getLocalCc(mid, cc)
     |
       localStep(mid, state0, node, state, true, _, config, localCc) and
       ap = ap0
@@ -1953,8 +1911,8 @@ private module Stage3 {
   bindingset[call, c, innercc]
   private CcNoCall getCallContextReturn(DataFlowCallable c, DataFlowCall call, Cc innercc) { any() }
 
-  bindingset[node, cc, config]
-  private LocalCc getLocalCc(NodeEx node, Cc cc, Configuration config) { any() }
+  bindingset[node, cc]
+  private LocalCc getLocalCc(NodeEx node, Cc cc) { any() }
 
   private predicate localStep(
     NodeEx node1, FlowState state1, NodeEx node2, FlowState state2, boolean preservesValue,
@@ -2037,7 +1995,7 @@ private module Stage3 {
     or
     exists(NodeEx mid, FlowState state0, Ap ap0, LocalCc localCc |
       fwdFlow(mid, state0, cc, argAp, ap0, config) and
-      localCc = getLocalCc(mid, cc, config)
+      localCc = getLocalCc(mid, cc)
     |
       localStep(mid, state0, node, state, true, _, config, localCc) and
       ap = ap0
@@ -2767,12 +2725,11 @@ private module Stage4 {
     if reducedViableImplInReturn(c, call) then result = TReturn(c, call) else result = ccNone()
   }
 
-  bindingset[node, cc, config]
-  private LocalCc getLocalCc(NodeEx node, Cc cc, Configuration config) {
+  bindingset[node, cc]
+  private LocalCc getLocalCc(NodeEx node, Cc cc) {
     result =
       getLocalCallContext(pragma[only_bind_into](pragma[only_bind_out](cc)),
-        node.getEnclosingCallable()) and
-    exists(config)
+        node.getEnclosingCallable())
   }
 
   private predicate localStep(
@@ -2865,7 +2822,7 @@ private module Stage4 {
     or
     exists(NodeEx mid, FlowState state0, Ap ap0, LocalCc localCc |
       fwdFlow(mid, state0, cc, argAp, ap0, config) and
-      localCc = getLocalCc(mid, cc, config)
+      localCc = getLocalCc(mid, cc)
     |
       localStep(mid, state0, node, state, true, _, config, localCc) and
       ap = ap0
@@ -5050,6 +5007,7 @@ private module FlowExploration {
     )
   }
 
+  pragma[nomagic]
   private predicate revPartialPathStep(
     PartialPathNodeRev mid, NodeEx node, FlowState state, TRevSummaryCtx1 sc1, TRevSummaryCtx2 sc2,
     TRevSummaryCtx3 sc3, RevPartialAccessPath ap, Configuration config

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPublic.qll

@@ -110,13 +110,19 @@ private DataFlowCallable getCallableScope(Scope s) {
   result = getCallableScope(s.getEnclosingScope())
 }
 
+private import semmle.python.internal.CachedStages
+
 /**
  * An element, viewed as a node in a data flow graph. Either an SSA variable
  * (`EssaNode`) or a control flow node (`CfgNode`).
  */
 class Node extends TNode {
   /** Gets a textual representation of this element. */
-  string toString() { result = "Data flow node" }
+  cached
+  string toString() {
+    Stages::DataFlow::ref() and
+    result = "Data flow node"
+  }
 
   /** Gets the scope of this node. */
   Scope getScope() { none() }
@@ -134,9 +140,11 @@ class Node extends TNode {
    * For more information, see
    * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
    */
+  cached
   predicate hasLocationInfo(
     string filepath, int startline, int startcolumn, int endline, int endcolumn
   ) {
+    Stages::DataFlow::ref() and
     this.getLocation().hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn)
   }
 

python/ql/lib/semmle/python/dataflow/new/internal/IterableUnpacking.qll

@@ -0,0 +1,396 @@
+/**
+ * The unpacking assignment takes the general form
+ * ```python
+ *   sequence = iterable
+ * ```
+ * where `sequence` is either a tuple or a list and it can contain wildcards.
+ * The iterable can be any iterable, which means that (CodeQL modeling of) content
+ * will need to change type if it should be transferred from the LHS to the RHS.
+ *
+ * Note that (CodeQL modeling of) content does not have to change type on data-flow
+ * paths _inside_ the LHS, as the different allowed syntaxes here are merely a convenience.
+ * Consequently, we model all LHS sequences as tuples, which have the more precise content
+ * model, making flow to the elements more precise. If an element is a starred variable,
+ * we will have to mutate the content type to be list content.
+ *
+ * We may for instance have
+ * ```python
+ *    (a, b) = ["a", SOURCE]  # RHS has content `ListElementContent`
+ * ```
+ * Due to the abstraction for list content, we do not know whether `SOURCE`
+ * ends up in `a` or in `b`, so we want to overapproximate and see it in both.
+ *
+ * Using wildcards we may have
+ * ```python
+ *   (a, *b) = ("a", "b", SOURCE)  # RHS has content `TupleElementContent(2)`
+ * ```
+ * Since the starred variables are always assigned (Python-)type list, `*b` will be
+ * `["b", SOURCE]`, and we will again overapproximate and assign it
+ * content corresponding to anything found in the RHS.
+ *
+ * For a precise transfer
+ * ```python
+ *    (a, b) = ("a", SOURCE)  # RHS has content `TupleElementContent(1)`
+ * ```
+ * we wish to keep the precision, so only `b` receives the tuple content at index 1.
+ *
+ * Finally, `sequence` is actually a pattern and can have a more complicated structure,
+ * such as
+ * ```python
+ *   (a, [b, *c]) = ("a", ["b", SOURCE])  # RHS has content `TupleElementContent(1); ListElementContent`
+ * ```
+ * where `a` should not receive content, but `b` and `c` should. `c` will be `[SOURCE]` so
+ * should have the content transferred, while `b` should read it.
+ *
+ * To transfer content from RHS to the elements of the LHS in the expression `sequence = iterable`,
+ * we use two synthetic nodes:
+ *
+ * - `TIterableSequence(sequence)` which captures the content-modeling the entire `sequence` will have
+ * (essentially just a copy of the content-modeling the RHS has)
+ *
+ * - `TIterableElement(sequence)` which captures the content-modeling that will be assigned to an element.
+ * Note that an empty access path means that the value we are tracking flows directly to the element.
+ *
+ *
+ * The `TIterableSequence(sequence)` is at this point superflous but becomes useful when handling recursive
+ * structures in the LHS, where `sequence` is some internal sequence node. We can have a uniform treatment
+ * by always having these two synthetic nodes. So we transfer to (or, in the recursive case, read into)
+ * `TIterableSequence(sequence)`, from which we take a read step to `TIterableElement(sequence)` and then a
+ * store step to `sequence`.
+ *
+ * This allows the unknown content from the RHS to be read into `TIterableElement(sequence)` and tuple content
+ * to then be stored into `sequence`. If the content is already tuple content, this inderection creates crosstalk
+ * between indices. Therefore, tuple content is never read into `TIterableElement(sequence)`; it is instead
+ * transferred directly from `TIterableSequence(sequence)` to `sequence` via a flow step. Such a flow step will
+ * also transfer other content, but only tuple content is further read from `sequence` into its elements.
+ *
+ * The strategy is then via several read-, store-, and flow steps:
+ * 1. a) [Flow] Content is transferred from `iterable` to `TIterableSequence(sequence)` via a
+ *    flow step. From here, everything happens on the LHS.
+ *
+ *    b) [Read] If the unpacking happens inside a for as in
+ *    ```python
+ *       for sequence in iterable
+ *    ```
+ *    then content is read from `iterable` to `TIterableSequence(sequence)`.
+ *
+ * 2. [Flow] Content is transferred from `TIterableSequence(sequence)` to `sequence` via a
+ *    flow step. (Here only tuple content is relevant.)
+ *
+ * 3. [Read] Content is read from `TIterableSequence(sequence)` into  `TIterableElement(sequence)`.
+ *    As `sequence` is modeled as a tuple, we will not read tuple content as that would allow
+ *    crosstalk.
+ *
+ * 4. [Store] Content is stored from `TIterableElement(sequence)` to `sequence`.
+ *    Content type is `TupleElementContent` with indices taken from the syntax.
+ *    For instance, if `sequence` is `(a, *b, c)`, content is written to index 0, 1, and 2.
+ *    This is adequate as the route through `TIterableElement(sequence)` does not transfer precise content.
+ *
+ * 5. [Read] Content is read from `sequence` to its elements.
+ *    a) If the element is a plain variable, the target is the corresponding essa node.
+ *
+ *    b) If the element is itself a sequence, with control-flow node `seq`, the target is `TIterableSequence(seq)`.
+ *
+ *    c) If the element is a starred variable, with control-flow node `v`, the target is `TIterableElement(v)`.
+ *
+ * 6. [Store] Content is stored from `TIterableElement(v)` to the essa variable for `v`, with
+ *    content type `ListElementContent`.
+ *
+ * 7. [Flow, Read, Store] Steps 2 through 7 are repeated for all recursive elements which are sequences.
+ *
+ *
+ * We illustrate the above steps on the assignment
+ *
+ * ```python
+ * (a, b) = ["a", SOURCE]
+ * ```
+ *
+ * Looking at the content propagation to `a`:
+ *   `["a", SOURCE]`: [ListElementContent]
+ *
+ * --Step 1a-->
+ *
+ *   `TIterableSequence((a, b))`: [ListElementContent]
+ *
+ * --Step 3-->
+ *
+ *   `TIterableElement((a, b))`: []
+ *
+ * --Step 4-->
+ *
+ *   `(a, b)`: [TupleElementContent(0)]
+ *
+ * --Step 5a-->
+ *
+ *   `a`: []
+ *
+ * Meaning there is data-flow from the RHS to `a` (an over approximation). The same logic would be applied to show there is data-flow to `b`. Note that _Step 3_ and _Step 4_ would not have been needed if the RHS had been a tuple (since that would have been able to use _Step 2_ instead).
+ *
+ * Another, more complicated example:
+ * ```python
+ *   (a, [b, *c]) = ["a", [SOURCE]]
+ * ```
+ * where the path to `c` is
+ *
+ *   `["a", [SOURCE]]`: [ListElementContent; ListElementContent]
+ *
+ * --Step 1a-->
+ *
+ *   `TIterableSequence((a, [b, *c]))`: [ListElementContent; ListElementContent]
+ *
+ * --Step 3-->
+ *
+ *   `TIterableElement((a, [b, *c]))`: [ListElementContent]
+ *
+ * --Step 4-->
+ *
+ *   `(a, [b, *c])`: [TupleElementContent(1); ListElementContent]
+ *
+ * --Step 5b-->
+ *
+ *   `TIterableSequence([b, *c])`: [ListElementContent]
+ *
+ * --Step 3-->
+ *
+ *   `TIterableElement([b, *c])`: []
+ *
+ * --Step 4-->
+ *
+ *   `[b, *c]`: [TupleElementContent(1)]
+ *
+ * --Step 5c-->
+ *
+ *   `TIterableElement(c)`: []
+ *
+ * --Step 6-->
+ *
+ *  `c`: [ListElementContent]
+ */
+
+private import python
+private import DataFlowPublic
+
+/**
+ * The target of a `for`, e.g. `x` in `for x in list` or in `[42 for x in list]`.
+ * This class also records the source, which in both above cases is `list`.
+ * This class abstracts away the differing representations of comprehensions and
+ * for statements.
+ */
+class ForTarget extends ControlFlowNode {
+  Expr source;
+
+  ForTarget() {
+    exists(For for |
+      source = for.getIter() and
+      this.getNode() = for.getTarget() and
+      not for = any(Comp comp).getNthInnerLoop(0)
+    )
+    or
+    exists(Comp comp |
+      source = comp.getIterable() and
+      this.getNode() = comp.getNthInnerLoop(0).getTarget()
+    )
+  }
+
+  Expr getSource() { result = source }
+}
+
+/** The LHS of an assignment, it also records the assigned value. */
+class AssignmentTarget extends ControlFlowNode {
+  Expr value;
+
+  AssignmentTarget() {
+    exists(Assign assign | this.getNode() = assign.getATarget() | value = assign.getValue())
+  }
+
+  Expr getValue() { result = value }
+}
+
+/** A direct (or top-level) target of an unpacking assignment. */
+class UnpackingAssignmentDirectTarget extends ControlFlowNode {
+  Expr value;
+
+  UnpackingAssignmentDirectTarget() {
+    this instanceof SequenceNode and
+    (
+      value = this.(AssignmentTarget).getValue()
+      or
+      value = this.(ForTarget).getSource()
+    )
+  }
+
+  Expr getValue() { result = value }
+}
+
+/** A (possibly recursive) target of an unpacking assignment. */
+class UnpackingAssignmentTarget extends ControlFlowNode {
+  UnpackingAssignmentTarget() {
+    this instanceof UnpackingAssignmentDirectTarget
+    or
+    this = any(UnpackingAssignmentSequenceTarget parent).getAnElement()
+  }
+}
+
+/** A (possibly recursive) target of an unpacking assignment which is also a sequence. */
+class UnpackingAssignmentSequenceTarget extends UnpackingAssignmentTarget instanceof SequenceNode {
+  ControlFlowNode getElement(int i) { result = super.getElement(i) }
+
+  ControlFlowNode getAnElement() { result = this.getElement(_) }
+}
+
+/**
+ * Step 1a
+ * Data flows from `iterable` to `TIterableSequence(sequence)`
+ */
+predicate iterableUnpackingAssignmentFlowStep(Node nodeFrom, Node nodeTo) {
+  exists(AssignmentTarget target |
+    nodeFrom.asExpr() = target.getValue() and
+    nodeTo = TIterableSequenceNode(target)
+  )
+}
+
+/**
+ * Step 1b
+ * Data is read from `iterable` to `TIterableSequence(sequence)`
+ */
+predicate iterableUnpackingForReadStep(CfgNode nodeFrom, Content c, Node nodeTo) {
+  exists(ForTarget target |
+    nodeFrom.asExpr() = target.getSource() and
+    target instanceof SequenceNode and
+    nodeTo = TIterableSequenceNode(target)
+  ) and
+  (
+    c instanceof ListElementContent
+    or
+    c instanceof SetElementContent
+  )
+}
+
+/**
+ * Step 2
+ * Data flows from `TIterableSequence(sequence)` to `sequence`
+ */
+predicate iterableUnpackingTupleFlowStep(Node nodeFrom, Node nodeTo) {
+  exists(UnpackingAssignmentSequenceTarget target |
+    nodeFrom = TIterableSequenceNode(target) and
+    nodeTo.asCfgNode() = target
+  )
+}
+
+/**
+ * Step 3
+ * Data flows from `TIterableSequence(sequence)` into  `TIterableElement(sequence)`.
+ * As `sequence` is modeled as a tuple, we will not read tuple content as that would allow
+ * crosstalk.
+ */
+predicate iterableUnpackingConvertingReadStep(Node nodeFrom, Content c, Node nodeTo) {
+  exists(UnpackingAssignmentSequenceTarget target |
+    nodeFrom = TIterableSequenceNode(target) and
+    nodeTo = TIterableElementNode(target) and
+    (
+      c instanceof ListElementContent
+      or
+      c instanceof SetElementContent
+      // TODO: dict content in iterable unpacking not handled
+    )
+  )
+}
+
+/**
+ * Step 4
+ * Data flows from `TIterableElement(sequence)` to `sequence`.
+ * Content type is `TupleElementContent` with indices taken from the syntax.
+ * For instance, if `sequence` is `(a, *b, c)`, content is written to index 0, 1, and 2.
+ */
+predicate iterableUnpackingConvertingStoreStep(Node nodeFrom, Content c, Node nodeTo) {
+  exists(UnpackingAssignmentSequenceTarget target |
+    nodeFrom = TIterableElementNode(target) and
+    nodeTo.asCfgNode() = target and
+    exists(int index | exists(target.getElement(index)) |
+      c.(TupleElementContent).getIndex() = index
+    )
+  )
+}
+
+/**
+ * Step 5
+ * For a sequence node inside an iterable unpacking, data flows from the sequence to its elements. There are
+ * three cases for what `toNode` should be:
+ *    a) If the element is a plain variable, `toNode` is the corresponding essa node.
+ *
+ *    b) If the element is itself a sequence, with control-flow node `seq`, `toNode` is `TIterableSequence(seq)`.
+ *
+ *    c) If the element is a starred variable, with control-flow node `v`, `toNode` is `TIterableElement(v)`.
+ */
+predicate iterableUnpackingElementReadStep(Node nodeFrom, Content c, Node nodeTo) {
+  exists(
+    UnpackingAssignmentSequenceTarget target, int index, ControlFlowNode element, int starIndex
+  |
+    target.getElement(starIndex) instanceof StarredNode
+    or
+    not exists(target.getAnElement().(StarredNode)) and
+    starIndex = -1
+  |
+    nodeFrom.asCfgNode() = target and
+    element = target.getElement(index) and
+    (
+      if starIndex = -1 or index < starIndex
+      then c.(TupleElementContent).getIndex() = index
+      else
+        // This could get big if big tuples exist
+        if index = starIndex
+        then c.(TupleElementContent).getIndex() >= index
+        else c.(TupleElementContent).getIndex() >= index - 1
+    ) and
+    (
+      if element instanceof SequenceNode
+      then
+        // Step 5b
+        nodeTo = TIterableSequenceNode(element)
+      else
+        if element instanceof StarredNode
+        then
+          // Step 5c
+          nodeTo = TIterableElementNode(element)
+        else
+          // Step 5a
+          nodeTo.asVar().getDefinition().(MultiAssignmentDefinition).getDefiningNode() = element
+    )
+  )
+}
+
+/**
+ * Step 6
+ * Data flows from `TIterableElement(v)` to the essa variable for `v`, with
+ * content type `ListElementContent`.
+ */
+predicate iterableUnpackingStarredElementStoreStep(Node nodeFrom, Content c, Node nodeTo) {
+  exists(ControlFlowNode starred | starred.getNode() instanceof Starred |
+    nodeFrom = TIterableElementNode(starred) and
+    nodeTo.asVar().getDefinition().(MultiAssignmentDefinition).getDefiningNode() = starred and
+    c instanceof ListElementContent
+  )
+}
+
+/** All read steps associated with unpacking assignment. */
+predicate iterableUnpackingReadStep(Node nodeFrom, Content c, Node nodeTo) {
+  iterableUnpackingForReadStep(nodeFrom, c, nodeTo)
+  or
+  iterableUnpackingElementReadStep(nodeFrom, c, nodeTo)
+  or
+  iterableUnpackingConvertingReadStep(nodeFrom, c, nodeTo)
+}
+
+/** All store steps associated with unpacking assignment. */
+predicate iterableUnpackingStoreStep(Node nodeFrom, Content c, Node nodeTo) {
+  iterableUnpackingStarredElementStoreStep(nodeFrom, c, nodeTo)
+  or
+  iterableUnpackingConvertingStoreStep(nodeFrom, c, nodeTo)
+}
+
+/** All flow steps associated with unpacking assignment. */
+predicate iterableUnpackingFlowStep(Node nodeFrom, Node nodeTo) {
+  iterableUnpackingAssignmentFlowStep(nodeFrom, nodeTo)
+  or
+  iterableUnpackingTupleFlowStep(nodeFrom, nodeTo)
+}

python/ql/lib/semmle/python/dataflow/new/internal/LocalSources.qll

@@ -9,6 +9,7 @@
 import python
 import DataFlowPublic
 private import DataFlowPrivate
+private import semmle.python.internal.CachedStages
 
 /**
  * A data flow node that is a source of local flow. This includes things like
@@ -33,6 +34,7 @@ private import DataFlowPrivate
 class LocalSourceNode extends Node {
   cached
   LocalSourceNode() {
+    Stages::DataFlow::ref() and
     this instanceof ExprNode and
     not simpleLocalFlowStep(_, this)
     or
@@ -176,6 +178,7 @@ private module Cached {
    */
   cached
   predicate hasLocalSource(Node sink, LocalSourceNode source) {
+    Stages::DataFlow::ref() and
     source = sink
     or
     exists(Node second |

python/ql/lib/semmle/python/dataflow/new/internal/MatchUnpacking.qll

@@ -0,0 +1,311 @@
+/**
+ * There are a number of patterns available for the match statement.
+ * Each one transfers data and content differently to its parts.
+ *
+ * Furthermore, given a successful match, we can infer some data about
+ * the subject. Consider the example:
+ * ```python
+ * match choice:
+ *   case 'Y':
+ *     ...body
+ * ```
+ * Inside `body`, we know that `choice` has the value `'Y'`.
+ *
+ * A similar thing happens with the "as pattern". Consider the example:
+ * ```python
+ * match choice:
+ *  case ('y'|'Y') as c:
+ *   ...body
+ * ```
+ * By the binding rules, there is data flow from `choice` to `c`. But we
+ * can infer the value of `c` to be either `'y'` or `'Y'` if the match succeeds.
+ *
+ * We will treat such inferences separately as guards. First we will model the data flow
+ * stemming from the bindings and the matching of shape. Below, 'subject' is not necessarily the
+ * top-level subject of the match, but rather the part recursively matched by the current pattern.
+ * For instance, in the example:
+ * ```python
+ * match command:
+ *  case ('quit' as c) | ('go', ('up'|'down') as c):
+ *   ...body
+ * ```
+ * `command` is the subject of first the as-pattern, while the second component of `command`
+ * is the subject of the second as-pattern. As such, 'subject' refers to the pattern under evaluation.
+ *
+ * - as pattern: subject flows to alias as well as to the interior pattern
+ * - or pattern: subject flows to each alternative
+ * - literal pattern: flow from the literal to the pattern, to add information
+ * - capture pattern: subject flows to the variable
+ * - wildcard pattern: no flow
+ * - value pattern: flow from the value to the pattern, to add information
+ * - sequence pattern: each element reads from subject at the associated index
+ * - star pattern: subject flows to the variable, possibly via a conversion
+ * - mapping pattern: each value reads from subject at the associated key
+ * - double star pattern: subject flows to the variable, possibly via a conversion
+ * - key-value pattern: the value reads from the subject at the key (see mapping pattern)
+ * - class pattern: all keywords read the appropriate attribute from the subject
+ * - keyword pattern: the appropriate attribute is read from the subject (see class pattern)
+ *
+ * Inside the class pattern, we also find positional arguments. They are converted to
+ * keyword arguments using the `__match_args__` attribute on the class. We do not
+ * currently model this.
+ */
+
+private import python
+private import DataFlowPublic
+
+/**
+ * Holds when there is flow from the subject `nodeFrom` to the (top-level) pattern `nodeTo` of a `match` statement.
+ *
+ * The subject of a match flows to each top-level pattern
+ * (a pattern directly under a `case` statement).
+ *
+ * We could consider a model closer to use-use-flow, where the subject
+ * only flows to the first top-level pattern and from there to the
+ * following ones.
+ */
+predicate matchSubjectFlowStep(Node nodeFrom, Node nodeTo) {
+  exists(MatchStmt match, Expr subject, Pattern target |
+    subject = match.getSubject() and
+    target = match.getCase(_).(Case).getPattern()
+  |
+    nodeFrom.asExpr() = subject and
+    nodeTo.asCfgNode().getNode() = target
+  )
+}
+
+/**
+ * as pattern: subject flows to alias as well as to the interior pattern
+ * syntax (toplevel): `case pattern as alias:`
+ */
+predicate matchAsFlowStep(Node nodeFrom, Node nodeTo) {
+  exists(MatchAsPattern subject, Name alias | alias = subject.getAlias() |
+    // We make the subject flow to the interior pattern via the alias.
+    // That way, information can propagate from the interior pattern to the alias.
+    //
+    // the subject flows to the interior pattern
+    nodeFrom.asCfgNode().getNode() = subject and
+    nodeTo.asCfgNode().getNode() = subject.getPattern()
+    or
+    // the interior pattern flows to the alias
+    nodeFrom.asCfgNode().getNode() = subject.getPattern() and
+    nodeTo.asVar().getDefinition().(PatternAliasDefinition).getDefiningNode().getNode() = alias
+  )
+}
+
+/**
+ * or pattern: subject flows to each alternative
+ * syntax (toplevel): `case alt1 | alt2:`
+ */
+predicate matchOrFlowStep(Node nodeFrom, Node nodeTo) {
+  exists(MatchOrPattern subject, Pattern pattern | pattern = subject.getAPattern() |
+    nodeFrom.asCfgNode().getNode() = subject and
+    nodeTo.asCfgNode().getNode() = pattern
+  )
+}
+
+/**
+ * literal pattern: flow from the literal to the pattern, to add information
+ * syntax (toplevel): `case literal:`
+ */
+predicate matchLiteralFlowStep(Node nodeFrom, Node nodeTo) {
+  exists(MatchLiteralPattern pattern, Expr literal | literal = pattern.getLiteral() |
+    nodeFrom.asExpr() = literal and
+    nodeTo.asCfgNode().getNode() = pattern
+  )
+}
+
+/**
+ * capture pattern: subject flows to the variable
+ * syntax (toplevel): `case var:`
+ */
+predicate matchCaptureFlowStep(Node nodeFrom, Node nodeTo) {
+  exists(MatchCapturePattern capture, Name var | capture.getVariable() = var |
+    nodeFrom.asCfgNode().getNode() = capture and
+    nodeTo.asVar().getDefinition().(PatternCaptureDefinition).getDefiningNode().getNode() = var
+  )
+}
+
+/**
+ * value pattern: flow from the value to the pattern, to add information
+ * syntax (toplevel): `case Dotted.value:`
+ */
+predicate matchValueFlowStep(Node nodeFrom, Node nodeTo) {
+  exists(MatchValuePattern pattern, Expr value | value = pattern.getValue() |
+    nodeFrom.asExpr() = value and
+    nodeTo.asCfgNode().getNode() = pattern
+  )
+}
+
+/**
+ * sequence pattern: each element reads from subject at the associated index
+ * syntax (toplevel): `case [a, b]:`
+ */
+predicate matchSequenceReadStep(Node nodeFrom, Content c, Node nodeTo) {
+  exists(MatchSequencePattern subject, int index, Pattern element |
+    element = subject.getPattern(index)
+  |
+    nodeFrom.asCfgNode().getNode() = subject and
+    nodeTo.asCfgNode().getNode() = element and
+    (
+      // tuple content
+      c.(TupleElementContent).getIndex() = index
+      or
+      // list content
+      c instanceof ListElementContent
+      // set content is excluded from sequence patterns,
+      // see https://www.python.org/dev/peps/pep-0635/#sequence-patterns
+    )
+  )
+}
+
+/**
+ * star pattern: subject flows to the variable, possibly via a conversion
+ * syntax (toplevel): `case *var:`
+ *
+ * We decompose this flow into a read step and a store step. The read step
+ * reads both tuple and list content, the store step only stores list content.
+ * This way, we convert all content to list content.
+ *
+ * This is the read step.
+ */
+predicate matchStarReadStep(Node nodeFrom, Content c, Node nodeTo) {
+  exists(MatchSequencePattern subject, int index, MatchStarPattern star |
+    star = subject.getPattern(index)
+  |
+    nodeFrom.asCfgNode().getNode() = subject and
+    nodeTo = TStarPatternElementNode(star) and
+    (
+      // tuple content
+      c.(TupleElementContent).getIndex() >= index
+      or
+      // list content
+      c instanceof ListElementContent
+      // set content is excluded from sequence patterns,
+      // see https://www.python.org/dev/peps/pep-0635/#sequence-patterns
+    )
+  )
+}
+
+/**
+ * star pattern: subject flows to the variable, possibly via a conversion
+ * syntax (toplevel): `case *var:`
+ *
+ * We decompose this flow into a read step and a store step. The read step
+ * reads both tuple and list content, the store step only stores list content.
+ * This way, we convert all content to list content.
+ *
+ * This is the store step.
+ */
+predicate matchStarStoreStep(Node nodeFrom, Content c, Node nodeTo) {
+  exists(MatchStarPattern star |
+    nodeFrom = TStarPatternElementNode(star) and
+    nodeTo.asCfgNode().getNode() = star.getTarget() and
+    c instanceof ListElementContent
+  )
+}
+
+/**
+ * mapping pattern: each value reads from subject at the associated key
+ * syntax (toplevel): `case {"color": c, "height": x}:`
+ */
+predicate matchMappingReadStep(Node nodeFrom, Content c, Node nodeTo) {
+  exists(
+    MatchMappingPattern subject, MatchKeyValuePattern keyValue, MatchLiteralPattern key,
+    Pattern value
+  |
+    keyValue = subject.getAMapping() and
+    key = keyValue.getKey() and
+    value = keyValue.getValue()
+  |
+    nodeFrom.asCfgNode().getNode() = subject and
+    nodeTo.asCfgNode().getNode() = value and
+    c.(DictionaryElementContent).getKey() = key.getLiteral().(StrConst).getText()
+  )
+}
+
+/**
+ * double star pattern: subject flows to the variable, possibly via a conversion
+ * syntax (toplevel): `case {**var}:`
+ *
+ * Dictionary content flows to the double star, but all mentioned keys in the
+ * mapping pattern should be cleared.
+ */
+predicate matchMappingFlowStep(Node nodeFrom, Node nodeTo) {
+  exists(MatchMappingPattern subject, MatchDoubleStarPattern dstar | dstar = subject.getAMapping() |
+    nodeFrom.asCfgNode().getNode() = subject and
+    nodeTo.asCfgNode().getNode() = dstar.getTarget()
+  )
+}
+
+/**
+ * Bindings that are mentioned in a mapping pattern will not be available
+ * to a double star pattern in the same mapping pattern.
+ */
+predicate matchMappingClearStep(Node n, Content c) {
+  exists(
+    MatchMappingPattern subject, MatchKeyValuePattern keyValue, MatchLiteralPattern key,
+    MatchDoubleStarPattern dstar
+  |
+    keyValue = subject.getAMapping() and
+    key = keyValue.getKey() and
+    dstar = subject.getAMapping()
+  |
+    n.asCfgNode().getNode() = dstar.getTarget() and
+    c.(DictionaryElementContent).getKey() = key.getLiteral().(StrConst).getText()
+  )
+}
+
+/**
+ * class pattern: all keywords read the appropriate attribute from the subject
+ * syntax (toplevel): `case ClassName(attr = val):`
+ */
+predicate matchClassReadStep(Node nodeFrom, Content c, Node nodeTo) {
+  exists(MatchClassPattern subject, MatchKeywordPattern keyword, Name attr, Pattern value |
+    keyword = subject.getKeyword(_) and
+    attr = keyword.getAttribute() and
+    value = keyword.getValue()
+  |
+    nodeFrom.asCfgNode().getNode() = subject and
+    nodeTo.asCfgNode().getNode() = value and
+    c.(AttributeContent).getAttribute() = attr.getId()
+  )
+}
+
+/** All flow steps associated with match. */
+predicate matchFlowStep(Node nodeFrom, Node nodeTo) {
+  matchSubjectFlowStep(nodeFrom, nodeTo)
+  or
+  matchAsFlowStep(nodeFrom, nodeTo)
+  or
+  matchOrFlowStep(nodeFrom, nodeTo)
+  or
+  matchLiteralFlowStep(nodeFrom, nodeTo)
+  or
+  matchCaptureFlowStep(nodeFrom, nodeTo)
+  or
+  matchValueFlowStep(nodeFrom, nodeTo)
+  or
+  matchMappingFlowStep(nodeFrom, nodeTo)
+}
+
+/** All read steps associated with match. */
+predicate matchReadStep(Node nodeFrom, Content c, Node nodeTo) {
+  matchClassReadStep(nodeFrom, c, nodeTo)
+  or
+  matchSequenceReadStep(nodeFrom, c, nodeTo)
+  or
+  matchMappingReadStep(nodeFrom, c, nodeTo)
+  or
+  matchStarReadStep(nodeFrom, c, nodeTo)
+}
+
+/** All store steps associated with match. */
+predicate matchStoreStep(Node nodeFrom, Content c, Node nodeTo) {
+  matchStarStoreStep(nodeFrom, c, nodeTo)
+}
+
+/**
+ * All clear steps associated with match
+ */
+predicate matchClearStep(Node n, Content c) { matchMappingClearStep(n, c) }

python/ql/lib/semmle/python/dataflow/new/internal/TypeTrackerSpecific.qll

@@ -5,6 +5,7 @@
 private import python
 private import semmle.python.dataflow.new.internal.DataFlowPublic as DataFlowPublic
 private import semmle.python.dataflow.new.internal.DataFlowPrivate as DataFlowPrivate
+import semmle.python.internal.CachedStages
 
 class Node = DataFlowPublic::Node;
 
@@ -21,7 +22,10 @@ predicate levelStep(Node pred, Node succ) { none() }
  * Gets the name of a possible piece of content. For Python, this is currently only attribute names,
  * using the name of the attribute for the corresponding content.
  */
-string getPossibleContentName() { result = any(DataFlowPublic::AttrRef a).getAttributeName() }
+string getPossibleContentName() {
+  Stages::TypeTracking::ref() and // the TypeTracking::append() etc. predicates that we want to cache depend on this predicate, so we can place the `ref()` call here to get around identical files.
+  result = any(DataFlowPublic::AttrRef a).getAttributeName()
+}
 
 /**
  * Gets a callable for the call where `nodeFrom` is used as the `i`'th argument.

python/ql/lib/semmle/python/dataflow/new/internal/readme.md

@@ -1,138 +0,0 @@
-# Using the shared dataflow library
-
-## File organisation
-
-The files currently live in `experimental` (whereas the existing implementation lives in `semmle\python\dataflow`).
-
-In there is found `DataFlow.qll`, `DataFlow2.qll` etc. which refer to `internal\DataFlowImpl`, `internal\DataFlowImpl2` etc. respectively. The `DataFlowImplN`-files are all identical copies to avoid mutual recursion. They start off by including two files `internal\DataFlowImplCommon` and `internal\DataFlowImplSpecific`. The former contains all the language-agnostic definitions, while the latter is where we describe our favorite language. `Sepcific` simply forwards to two other files `internal\DataFlowPrivate.qll` and `internal\DataFlowPublic.qll`. Definitions in the former will be hidden behind a `private` modifier, while those in the latter can be referred to in data flow queries. For instance, the definition of `DataFlow::Node` should likely be in `DataFlowPublic.qll`.
-
-## Define the dataflow graph
-
-In order to use the dataflow library, we need to define the dataflow graph,
-that is define the nodes and the edges.
-
-### Define the nodes
-
-The nodes are defined in the type `DataFlow::Node` (found in `DataFlowPublic.qll`).
-This should likely be an IPA type, so we can extend it as needed.
-
-Typical cases needed to construct the call graph include
- - argument node
- - parameter node
- - return node
-
-Typical extensions include
- - postupdate nodes
- - implicit `this`-nodes
-
-### Define the edges
-
-The edges split into local flow (within a function) and global flow (the call graph, between functions/procedures).
-
-Extra flow, such as reading from and writing to global variables, can be captured in `jumpStep`.
-The local flow should be obtainalble from an SSA computation.
-Local flow nodes are generally either control flow nodes or SSA variables.
-Flow from control flow nodes to SSA variables comes from SSA variable definitions, while flow from SSA variables to control flow nodes comes from def-use pairs.
-
-The global flow should be obtainable from a `PointsTo` analysis. It is specified via `viableCallable` and
-`getAnOutNode`. Consider making `ReturnKind` a singleton IPA type as in java.
-
-Global flow includes local flow within a consistent call context. Thus, for local flow to count as global flow, all relevant nodes should implement `getEnclosingCallable`.
-
-If complicated dispatch needs to be modelled, try using the `[reduced|pruned]viable*` predicates.
-
-## Field flow
-
-To track flow through fields we need to provide a model of fields, that is the `Content` class.
-
-Field access is specified via `read_step` and `store_step`.
-
-Work is being done to make field flow handle lists and dictionaries and the like.
-
-`PostUpdateNode`s become important when field flow is used, as they track modifications to fields resulting from function calls.
-
-## Type pruning
-
-If type information is available, flows can be discarded on the grounds of type mismatch.
-
-Tracked types are given by the class `DataFlowType` and the predicate `getTypeBound`, and compatibility is recorded in the predicate `compatibleTypes`.
-If type pruning is not used, `compatibleTypes` should be implemented as `any`; if it is implemented, say, as `none`, all flows will be pruned.
-
-Further, possible casts are given by the class `CastNode`.
-
----
-
-# Plan
-
-## Stage I, data flow
-
-### Phase 0, setup
-Define minimal IPA type for `DataFlow::Node`
-Define all required predicates empty (via `none()`),
-except `compatibleTypes` which should be `any()`.
-Define `ReturnKind`, `DataFlowType`, and `Content` as singleton IPA types.
-
-
-### Phase 1, local flow
-Implement `simpleLocalFlowStep` based on the existing SSA computation
-
-### Phase 2, local flow
-Implement `viableCallable` and `getAnOutNode` based on the existing predicate `PointsTo`.
-
-### Phase 3, field flow
-Redefine `Content` and implement `read_step` and `store_step`.
-
-Review use of post-update nodes.
-
-### Phase 4, type pruning
-Use type trackers to obtain relevant type information and redefine `DataFlowType` to contain appropriate cases. Record the type information in `getTypeBound`.
-
-Implement `compatibleTypes` (perhaps simply as the identity).
-
-If necessary, re-implement `getErasedRepr` and `ppReprType`.
-
-If necessary, redefine `CastNode`.
-
-### Phase 5, bonus
-Review possible use of `[reduced|pruned]viable*` predicates.
-
-Review need for more elaborate `ReturnKind`.
-
-Review need for non-empty `jumpStep`.
-
-Review need for non-empty `isUnreachableInCall`.
-
-## Stage II, taint tracking
-
-# Phase 0, setup
-Implement all predicates empty.
-
-# Phase 1, experiments
-Try recovering an existing taint tracking query by implementing sources, sinks, sanitizers, and barriers.
-
----
-
-# Status
-
-## Achieved
-
-- Copy of shared library; implemented enough predicates to make it compile.
-- Simple flow into, out of, and through functions.
-- Some tests, in particular a sceleton for something comprehensive.
-
-## TODO
-
-- Implementation has largely been done by finding a plausibly-sounding predicate in the python library to refer to. We should review that we actually have the intended semantics in all places.
-- Comprehensive testing.
-- The regression tests track the value of guards in order to eliminate impossible data flow. We currently have regressions because of this. We cannot readily replicate the existing method, as it uses the interdefinedness of data flow and taint tracking (there is a boolean taint kind). C++ [does something similar](https://github.com/github/codeql/blob/master/cpp/ql/src/semmle/code/cpp/controlflow/internal/ConstantExprs.qll#L27-L36) for eliminating impossible control flow, which we might be able to replicate (they infer values of "interesting" control flow nodes, which are those needed to determine values of guards).
-- Flow for some syntactic constructs are done via extra taint steps in the existing implementation, we should find a way to get data flow for it. Some of this should be covered by field flow.
-- A document is being written about proper use of the shared data flow library, this should be adhered to. In particular, we should consider replacing def-use with def-to-first-use and use-to-next-use in local flow.
-- We seem to get duplicated results for global flow, as well as flow with and without type (so four times the "unique" results).
-- We currently consider control flow nodes like exit nodes for functions, we should probably filter down which ones are of interest.
-- We should probably override ToString for a number of data flow nodes.
-- Test flow through classes, constructors and methods.
-- What happens with named arguments? What does C# do?
-- What should the enclosable callable for global variables be? C++ [makes it the variable itself](https://github.com/github/codeql/blob/master/cpp/ql/src/semmle/code/cpp/ir/dataflow/internal/DataFlowUtil.qll#L417), C# seems to not have nodes for these but only for their reads and writes.
-- Is `yield` another return type? If not, how is it handled?
-- Should `OutNode` include magic function calls?
-- Consider creating an internal abstract class for nodes as C# does. Among other things, this can help the optimizer by stating that `getEnclosingCallable` [is functional](https://github.com/github/codeql/blob/master/csharp/ql/src/semmle/code/csharp/dataflow/internal/DataFlowPublic.qll#L62).

python/ql/lib/semmle/python/dataflow/new/internal/tainttracking1/TaintTrackingImpl.qll

@@ -109,16 +109,6 @@ abstract class Configuration extends DataFlow::Configuration {
   /** Holds if taint propagation into `node` is prohibited. */
   predicate isSanitizerIn(DataFlow::Node node) { none() }
 
-  /**
-   * Holds if taint propagation into `node` is prohibited when the flow state is
-   * `state`.
-   */
-  predicate isSanitizerIn(DataFlow::Node node, DataFlow::FlowState state) { none() }
-
-  final override predicate isBarrierIn(DataFlow::Node node, DataFlow::FlowState state) {
-    this.isSanitizerIn(node, state)
-  }
-
   final override predicate isBarrierIn(DataFlow::Node node) { this.isSanitizerIn(node) }
 
   /** Holds if taint propagation out of `node` is prohibited. */
@@ -126,16 +116,6 @@ abstract class Configuration extends DataFlow::Configuration {
 
   final override predicate isBarrierOut(DataFlow::Node node) { this.isSanitizerOut(node) }
 
-  /**
-   * Holds if taint propagation out of `node` is prohibited when the flow state is
-   * `state`.
-   */
-  predicate isSanitizerOut(DataFlow::Node node, DataFlow::FlowState state) { none() }
-
-  final override predicate isBarrierOut(DataFlow::Node node, DataFlow::FlowState state) {
-    this.isSanitizerOut(node, state)
-  }
-
   /** Holds if taint propagation through nodes guarded by `guard` is prohibited. */
   predicate isSanitizerGuard(DataFlow::BarrierGuard guard) { none() }
 
@@ -154,8 +134,7 @@ abstract class Configuration extends DataFlow::Configuration {
   }
 
   /**
-   * Holds if the additional taint propagation step from `node1` to `node2`
-   * must be taken into account in the analysis.
+   * Holds if taint may propagate from `node1` to `node2` in addition to the normal data-flow and taint steps.
    */
   predicate isAdditionalTaintStep(DataFlow::Node node1, DataFlow::Node node2) { none() }
 
@@ -165,9 +144,8 @@ abstract class Configuration extends DataFlow::Configuration {
   }
 
   /**
-   * Holds if the additional taint propagation step from `node1` to `node2`
-   * must be taken into account in the analysis. This step is only applicable
-   * in `state1` and updates the flow state to `state2`.
+   * Holds if taint may propagate from `node1` to `node2` in addition to the normal data-flow and taint steps.
+   * This step is only applicable in `state1` and updates the flow state to `state2`.
    */
   predicate isAdditionalTaintStep(
     DataFlow::Node node1, DataFlow::FlowState state1, DataFlow::Node node2,

python/ql/lib/semmle/python/dataflow/new/internal/tainttracking2/TaintTrackingImpl.qll

@@ -109,16 +109,6 @@ abstract class Configuration extends DataFlow::Configuration {
   /** Holds if taint propagation into `node` is prohibited. */
   predicate isSanitizerIn(DataFlow::Node node) { none() }
 
-  /**
-   * Holds if taint propagation into `node` is prohibited when the flow state is
-   * `state`.
-   */
-  predicate isSanitizerIn(DataFlow::Node node, DataFlow::FlowState state) { none() }
-
-  final override predicate isBarrierIn(DataFlow::Node node, DataFlow::FlowState state) {
-    this.isSanitizerIn(node, state)
-  }
-
   final override predicate isBarrierIn(DataFlow::Node node) { this.isSanitizerIn(node) }
 
   /** Holds if taint propagation out of `node` is prohibited. */
@@ -126,16 +116,6 @@ abstract class Configuration extends DataFlow::Configuration {
 
   final override predicate isBarrierOut(DataFlow::Node node) { this.isSanitizerOut(node) }
 
-  /**
-   * Holds if taint propagation out of `node` is prohibited when the flow state is
-   * `state`.
-   */
-  predicate isSanitizerOut(DataFlow::Node node, DataFlow::FlowState state) { none() }
-
-  final override predicate isBarrierOut(DataFlow::Node node, DataFlow::FlowState state) {
-    this.isSanitizerOut(node, state)
-  }
-
   /** Holds if taint propagation through nodes guarded by `guard` is prohibited. */
   predicate isSanitizerGuard(DataFlow::BarrierGuard guard) { none() }
 
@@ -154,8 +134,7 @@ abstract class Configuration extends DataFlow::Configuration {
   }
 
   /**
-   * Holds if the additional taint propagation step from `node1` to `node2`
-   * must be taken into account in the analysis.
+   * Holds if taint may propagate from `node1` to `node2` in addition to the normal data-flow and taint steps.
    */
   predicate isAdditionalTaintStep(DataFlow::Node node1, DataFlow::Node node2) { none() }
 
@@ -165,9 +144,8 @@ abstract class Configuration extends DataFlow::Configuration {
   }
 
   /**
-   * Holds if the additional taint propagation step from `node1` to `node2`
-   * must be taken into account in the analysis. This step is only applicable
-   * in `state1` and updates the flow state to `state2`.
+   * Holds if taint may propagate from `node1` to `node2` in addition to the normal data-flow and taint steps.
+   * This step is only applicable in `state1` and updates the flow state to `state2`.
    */
   predicate isAdditionalTaintStep(
     DataFlow::Node node1, DataFlow::FlowState state1, DataFlow::Node node2,

python/ql/lib/semmle/python/dataflow/new/internal/tainttracking3/TaintTrackingImpl.qll

@@ -109,16 +109,6 @@ abstract class Configuration extends DataFlow::Configuration {
   /** Holds if taint propagation into `node` is prohibited. */
   predicate isSanitizerIn(DataFlow::Node node) { none() }
 
-  /**
-   * Holds if taint propagation into `node` is prohibited when the flow state is
-   * `state`.
-   */
-  predicate isSanitizerIn(DataFlow::Node node, DataFlow::FlowState state) { none() }
-
-  final override predicate isBarrierIn(DataFlow::Node node, DataFlow::FlowState state) {
-    this.isSanitizerIn(node, state)
-  }
-
   final override predicate isBarrierIn(DataFlow::Node node) { this.isSanitizerIn(node) }
 
   /** Holds if taint propagation out of `node` is prohibited. */
@@ -126,16 +116,6 @@ abstract class Configuration extends DataFlow::Configuration {
 
   final override predicate isBarrierOut(DataFlow::Node node) { this.isSanitizerOut(node) }
 
-  /**
-   * Holds if taint propagation out of `node` is prohibited when the flow state is
-   * `state`.
-   */
-  predicate isSanitizerOut(DataFlow::Node node, DataFlow::FlowState state) { none() }
-
-  final override predicate isBarrierOut(DataFlow::Node node, DataFlow::FlowState state) {
-    this.isSanitizerOut(node, state)
-  }
-
   /** Holds if taint propagation through nodes guarded by `guard` is prohibited. */
   predicate isSanitizerGuard(DataFlow::BarrierGuard guard) { none() }
 
@@ -154,8 +134,7 @@ abstract class Configuration extends DataFlow::Configuration {
   }
 
   /**
-   * Holds if the additional taint propagation step from `node1` to `node2`
-   * must be taken into account in the analysis.
+   * Holds if taint may propagate from `node1` to `node2` in addition to the normal data-flow and taint steps.
    */
   predicate isAdditionalTaintStep(DataFlow::Node node1, DataFlow::Node node2) { none() }
 
@@ -165,9 +144,8 @@ abstract class Configuration extends DataFlow::Configuration {
   }
 
   /**
-   * Holds if the additional taint propagation step from `node1` to `node2`
-   * must be taken into account in the analysis. This step is only applicable
-   * in `state1` and updates the flow state to `state2`.
+   * Holds if taint may propagate from `node1` to `node2` in addition to the normal data-flow and taint steps.
+   * This step is only applicable in `state1` and updates the flow state to `state2`.
    */
   predicate isAdditionalTaintStep(
     DataFlow::Node node1, DataFlow::FlowState state1, DataFlow::Node node2,

python/ql/lib/semmle/python/dataflow/new/internal/tainttracking4/TaintTrackingImpl.qll

@@ -109,16 +109,6 @@ abstract class Configuration extends DataFlow::Configuration {
   /** Holds if taint propagation into `node` is prohibited. */
   predicate isSanitizerIn(DataFlow::Node node) { none() }
 
-  /**
-   * Holds if taint propagation into `node` is prohibited when the flow state is
-   * `state`.
-   */
-  predicate isSanitizerIn(DataFlow::Node node, DataFlow::FlowState state) { none() }
-
-  final override predicate isBarrierIn(DataFlow::Node node, DataFlow::FlowState state) {
-    this.isSanitizerIn(node, state)
-  }
-
   final override predicate isBarrierIn(DataFlow::Node node) { this.isSanitizerIn(node) }
 
   /** Holds if taint propagation out of `node` is prohibited. */
@@ -126,16 +116,6 @@ abstract class Configuration extends DataFlow::Configuration {
 
   final override predicate isBarrierOut(DataFlow::Node node) { this.isSanitizerOut(node) }
 
-  /**
-   * Holds if taint propagation out of `node` is prohibited when the flow state is
-   * `state`.
-   */
-  predicate isSanitizerOut(DataFlow::Node node, DataFlow::FlowState state) { none() }
-
-  final override predicate isBarrierOut(DataFlow::Node node, DataFlow::FlowState state) {
-    this.isSanitizerOut(node, state)
-  }
-
   /** Holds if taint propagation through nodes guarded by `guard` is prohibited. */
   predicate isSanitizerGuard(DataFlow::BarrierGuard guard) { none() }
 
@@ -154,8 +134,7 @@ abstract class Configuration extends DataFlow::Configuration {
   }
 
   /**
-   * Holds if the additional taint propagation step from `node1` to `node2`
-   * must be taken into account in the analysis.
+   * Holds if taint may propagate from `node1` to `node2` in addition to the normal data-flow and taint steps.
    */
   predicate isAdditionalTaintStep(DataFlow::Node node1, DataFlow::Node node2) { none() }
 
@@ -165,9 +144,8 @@ abstract class Configuration extends DataFlow::Configuration {
   }
 
   /**
-   * Holds if the additional taint propagation step from `node1` to `node2`
-   * must be taken into account in the analysis. This step is only applicable
-   * in `state1` and updates the flow state to `state2`.
+   * Holds if taint may propagate from `node1` to `node2` in addition to the normal data-flow and taint steps.
+   * This step is only applicable in `state1` and updates the flow state to `state2`.
    */
   predicate isAdditionalTaintStep(
     DataFlow::Node node1, DataFlow::FlowState state1, DataFlow::Node node2,

python/ql/lib/semmle/python/essa/Essa.qll

@@ -5,6 +5,7 @@
 import python
 private import SsaCompute
 import semmle.python.essa.Definitions
+private import semmle.python.internal.CachedStages
 
 /** An (enhanced) SSA variable derived from `SsaSourceVariable`. */
 class EssaVariable extends TEssaDefinition {
@@ -270,6 +271,7 @@ class PhiFunction extends EssaDefinition, TPhiFunction {
   /** Gets the input variable for this phi node on the edge `pred` -> `this.getBasicBlock()`, if any. */
   cached
   EssaVariable getInput(BasicBlock pred) {
+    Stages::AST::ref() and
     result.getDefinition() = this.reachingDefinition(pred)
     or
     result.getDefinition() = this.inputEdgeRefinement(pred)

python/ql/lib/semmle/python/essa/SsaCompute.qll

@@ -90,6 +90,7 @@
  */
 
 import python
+private import semmle.python.internal.CachedStages
 
 cached
 private module SsaComputeImpl {
@@ -308,6 +309,7 @@ private module SsaComputeImpl {
      */
     cached
     predicate reachesEndOfBlock(SsaSourceVariable v, BasicBlock defbb, int defindex, BasicBlock b) {
+      Stages::AST::ref() and
       Liveness::liveAtExit(v, b) and
       (
         defbb = b and

python/ql/lib/semmle/python/essa/SsaDefinitions.qll

@@ -5,12 +5,14 @@
 
 import python
 private import semmle.python.pointsto.Base
+private import semmle.python.internal.CachedStages
 
 cached
 module SsaSource {
   /** Holds if `v` is used as the receiver in a method call. */
   cached
   predicate method_call_refinement(Variable v, ControlFlowNode use, CallNode call) {
+    Stages::AST::ref() and
     use = v.getAUse() and
     call.getFunction().(AttrNode).getObject() = use and
     not test_contains(_, call)

python/ql/lib/semmle/python/filters/GeneratedCode.qll

@@ -32,18 +32,26 @@ class GenericGeneratedFile extends GeneratedFile {
   override string getTool() { lax_generated_by(this, result) or strict_generated_by(this, result) }
 }
 
+pragma[nomagic]
+private int minStmtLine(File file) {
+  result =
+    min(int line |
+      line = any(Stmt s | s.getLocation().getFile() = file).getLocation().getStartLine()
+    )
+}
+
+pragma[nomagic]
+private predicate isCommentAfterCode(Comment c, File f) {
+  f = c.getLocation().getFile() and
+  minStmtLine(f) < c.getLocation().getStartLine()
+}
+
 private string comment_or_docstring(File f, boolean before_code) {
   exists(Comment c |
     c.getLocation().getFile() = f and
     result = c.getText()
   |
-    if
-      exists(Stmt s |
-        s.getEnclosingModule().getFile() = f and
-        s.getLocation().getStartLine() < c.getLocation().getStartLine()
-      )
-    then before_code = false
-    else before_code = true
+    if isCommentAfterCode(c, f) then before_code = false else before_code = true
   )
   or
   exists(Module m | m.getFile() = f |

python/ql/lib/semmle/python/frameworks/Aiohttp.qll

@@ -243,7 +243,9 @@ module AiohttpWebModel {
 
   /** A class that has a super-type which is an aiohttp.web View class. */
   class AiohttpViewClassFromSuperClass extends AiohttpViewClass {
-    AiohttpViewClassFromSuperClass() { this.getABase() = View::subclassRef().getAUse().asExpr() }
+    AiohttpViewClassFromSuperClass() {
+      this.getParent() = View::subclassRef().getAnImmediateUse().asExpr()
+    }
   }
 
   /** A class that is used in a route-setup, therefore being considered an aiohttp.web View class. */

python/ql/lib/semmle/python/frameworks/Django.qll

@@ -829,7 +829,7 @@ module PrivateDjango {
 
           /** Gets the (AST) class of the Django model class `modelClass`. */
           Class getModelClassClass(API::Node modelClass) {
-            result.getParent() = modelClass.getAUse().asExpr().(ClassExpr) and
+            result.getParent() = modelClass.getAnImmediateUse().asExpr() and
             modelClass = Model::subclassRef()
           }
 
@@ -2162,7 +2162,9 @@ module PrivateDjango {
    * thereby handling user input.
    */
   class DjangoFormClass extends Class, SelfRefMixin {
-    DjangoFormClass() { this.getABase() = Django::Forms::Form::subclassRef().getAUse().asExpr() }
+    DjangoFormClass() {
+      this.getParent() = Django::Forms::Form::subclassRef().getAnImmediateUse().asExpr()
+    }
   }
 
   /**
@@ -2195,7 +2197,7 @@ module PrivateDjango {
    */
   class DjangoFormFieldClass extends Class {
     DjangoFormFieldClass() {
-      this.getABase() = Django::Forms::Field::subclassRef().getAUse().asExpr()
+      this.getParent() = Django::Forms::Field::subclassRef().getAnImmediateUse().asExpr()
     }
   }
 
@@ -2298,7 +2300,7 @@ module PrivateDjango {
    */
   class DjangoViewClassFromSuperClass extends DjangoViewClass {
     DjangoViewClassFromSuperClass() {
-      this.getABase() = Django::Views::View::subclassRef().getAUse().asExpr()
+      this.getParent() = Django::Views::View::subclassRef().getAnImmediateUse().asExpr()
     }
   }
 

python/ql/lib/semmle/python/frameworks/Flask.qll

@@ -122,7 +122,9 @@ module Flask {
     private class ClassInstantiation extends InstanceSource, DataFlow::CallCfgNode {
       ClassInstantiation() { this = classRef().getACall() }
 
-      override DataFlow::Node getBody() { result = this.getArg(0) }
+      override DataFlow::Node getBody() {
+        result in [this.getArg(0), this.getArgByName("response")]
+      }
 
       override string getMimetypeDefault() { result = "text/html" }
 
@@ -192,8 +194,8 @@ module Flask {
     API::Node api_node;
 
     FlaskViewClass() {
-      this.getABase() = Views::View::subclassRef().getAUse().asExpr() and
-      api_node.getAnImmediateUse().asExpr() = this.getParent()
+      api_node = Views::View::subclassRef() and
+      this.getParent() = api_node.getAnImmediateUse().asExpr()
     }
 
     /** Gets a function that could handle incoming requests, if any. */
@@ -217,8 +219,8 @@ module Flask {
    */
   class FlaskMethodViewClass extends FlaskViewClass {
     FlaskMethodViewClass() {
-      this.getABase() = Views::MethodView::subclassRef().getAUse().asExpr() and
-      api_node.getAnImmediateUse().asExpr() = this.getParent()
+      api_node = Views::MethodView::subclassRef() and
+      this.getParent() = api_node.getAnImmediateUse().asExpr()
     }
 
     override Function getARequestHandler() {

python/ql/lib/semmle/python/frameworks/RestFramework.qll

@@ -115,7 +115,7 @@ private module RestFramework {
    */
   class RestFrameworkApiViewClass extends PrivateDjango::DjangoViewClassFromSuperClass {
     RestFrameworkApiViewClass() {
-      this.getABase() = any(ModeledApiViewClasses c).getASubclass*().getAUse().asExpr()
+      this.getParent() = any(ModeledApiViewClasses c).getASubclass*().getAnImmediateUse().asExpr()
     }
 
     override Function getARequestHandler() {

python/ql/lib/semmle/python/frameworks/Stdlib.qll

@@ -1934,7 +1934,7 @@ private module StdlibPrivate {
 
     /** A HttpRequestHandler class definition (most likely in project code). */
     class HttpRequestHandlerClassDef extends Class {
-      HttpRequestHandlerClassDef() { this.getParent() = subclassRef().getAUse().asExpr() }
+      HttpRequestHandlerClassDef() { this.getParent() = subclassRef().getAnImmediateUse().asExpr() }
     }
 
     /** DEPRECATED: Alias for HttpRequestHandlerClassDef */
@@ -2027,12 +2027,12 @@ private module StdlibPrivate {
   private module WsgirefSimpleServer {
     class WsgiServerSubclass extends Class, SelfRefMixin {
       WsgiServerSubclass() {
-        this.getABase() =
+        this.getParent() =
           API::moduleImport("wsgiref")
               .getMember("simple_server")
               .getMember("WSGIServer")
               .getASubclass*()
-              .getAUse()
+              .getAnImmediateUse()
               .asExpr()
       }
     }

python/ql/lib/semmle/python/frameworks/Tornado.qll

@@ -92,7 +92,7 @@ private module Tornado {
 
         /** A RequestHandler class (most likely in project code). */
         class RequestHandlerClass extends Class {
-          RequestHandlerClass() { this.getParent() = subclassRef().getAUse().asExpr() }
+          RequestHandlerClass() { this.getParent() = subclassRef().getAnImmediateUse().asExpr() }
 
           /** Gets a function that could handle incoming requests, if any. */
           Function getARequestHandler() {

python/ql/lib/semmle/python/frameworks/Twisted.qll

@@ -27,13 +27,13 @@ private module Twisted {
    */
   class TwistedResourceSubclass extends Class {
     TwistedResourceSubclass() {
-      this.getABase() =
+      this.getParent() =
         API::moduleImport("twisted")
             .getMember("web")
             .getMember("resource")
             .getMember("Resource")
             .getASubclass*()
-            .getAUse()
+            .getAnImmediateUse()
             .asExpr()
     }
 

python/ql/lib/semmle/python/internal/Awaited.qll

@@ -6,6 +6,7 @@
 
 private import python
 private import semmle.python.dataflow.new.DataFlow
+private import semmle.python.internal.CachedStages
 
 /**
  * INTERNAL: Do not use.
@@ -14,6 +15,7 @@ private import semmle.python.dataflow.new.DataFlow
  */
 cached
 DataFlow::Node awaited(DataFlow::Node awaitedValue) {
+  Stages::DataFlow::ref() and
   // `await` x
   // - `awaitedValue` is `x`
   // - `result` is `await x`

python/ql/lib/semmle/python/internal/CachedStages.qll

@@ -0,0 +1,179 @@
+/**
+ * INTERNAL: Do not use.
+ *
+ * The purpose of this file is to control which cached predicates belong to the same stage.
+ *
+ * Combining stages can improve performance as we are more likely to reuse shared, non-cached predicates.
+ *
+ * To make a predicate `p` belong to a stage `A`:
+ * - make `p` depend on `A::ref()`, and
+ * - make `A::backref()` depend on `p`.
+ *
+ * Since `A` is a cached module, `ref` and `backref` must be in the same stage, and the dependency
+ * chain above thus forces `p` to be in that stage as well.
+ *
+ * With these two predicates in a `cached module` we ensure that all the cached predicates will be in a single stage at runtime.
+ *
+ * Grouping stages can cause unnecessary computation, as a concrete query might not depend on
+ * all the cached predicates in a stage.
+ * Care should therefore be taken not to combine two stages, if it is likely that a query only depend
+ * on some but not all the cached predicates in the combined stage.
+ */
+
+/**
+ * Contains a `cached module` for each stage.
+ * Each `cached module` ensures that predicates that are supposed to be in the same stage, are in the same stage.
+ *
+ * Each `cached module` contain two predicates:
+ * The first, `ref`, always holds, and is referenced from `cached` predicates.
+ * The second, `backref`, contains references to the same `cached` predicates.
+ * The `backref` predicate starts with `1 = 1 or` to ensure that the predicate will be optimized down to a constant by the optimizer.
+ */
+module Stages {
+  /**
+   * The `AST` stage.
+   * Computes predicates based on the AST.
+   * These include SSA and basic-blocks.
+   */
+  cached
+  module AST {
+    /**
+     * Always holds.
+     * Ensures that a predicate is evaluated as part of the Ast stage.
+     */
+    cached
+    predicate ref() { 1 = 1 }
+
+    private import semmle.python.essa.SsaDefinitions as SsaDefinitions
+    private import semmle.python.essa.SsaCompute as SsaCompute
+    private import semmle.python.essa.Essa as Essa
+    private import semmle.python.Module as PyModule
+    private import semmle.python.Exprs as Exprs
+    private import semmle.python.AstExtended as AstExtended
+    private import semmle.python.Flow as PyFlow
+
+    /**
+     * DONT USE!
+     * Contains references to each predicate that use the above `ref` predicate.
+     */
+    cached
+    predicate backref() {
+      1 = 1
+      or
+      SsaDefinitions::SsaSource::method_call_refinement(_, _, _)
+      or
+      SsaCompute::SsaDefinitions::reachesEndOfBlock(_, _, _, _)
+      or
+      exists(any(Essa::PhiFunction p).getInput(_))
+      or
+      exists(PyModule::moduleNameFromFile(_))
+      or
+      exists(any(Exprs::Expr e).toString())
+      or
+      exists(any(AstExtended::AstNode n).getLocation())
+      or
+      exists(any(AstExtended::AstNode n).getAChildNode())
+      or
+      exists(any(AstExtended::AstNode n).getParentNode())
+      or
+      exists(any(AstExtended::AstNode n).getAFlowNode())
+      or
+      exists(any(PyFlow::BasicBlock b).getImmediateDominator())
+      or
+      exists(any(PyFlow::BasicBlock b).getScope())
+      or
+      any(PyFlow::BasicBlock b).strictlyDominates(_)
+      or
+      any(PyFlow::BasicBlock b).strictlyReaches(_)
+      or
+      exists(any(PyFlow::BasicBlock b).getASuccessor())
+      or
+      exists(any(PyFlow::ControlFlowNode b).getScope())
+      or
+      exists(PyFlow::DefinitionNode b)
+      or
+      exists(any(PyFlow::SequenceNode n).getElement(_))
+    }
+  }
+
+  /**
+   * The `TypeTracking` stage.
+   */
+  cached
+  module TypeTracking {
+    /**
+     * Always holds.
+     * Ensures that a predicate is evaluated as part of the Ast stage.
+     */
+    cached
+    predicate ref() { 1 = 1 }
+
+    private import semmle.python.dataflow.new.DataFlow::DataFlow as NewDataFlow
+    private import semmle.python.ApiGraphs::API as API
+
+    /**
+     * DONT USE!
+     * Contains references to each predicate that use the above `ref` predicate.
+     */
+    cached
+    predicate backref() {
+      1 = 1
+      or
+      exists(any(NewDataFlow::TypeTracker t).append(_))
+      or
+      exists(any(API::Node n).getAMember().getAUse())
+    }
+  }
+
+  /**
+   * The `dataflow` stage.
+   */
+  cached
+  module DataFlow {
+    /**
+     * Always holds.
+     * Ensures that a predicate is evaluated as part of the DataFlow stage.
+     */
+    cached
+    predicate ref() { 1 = 1 }
+
+    private import semmle.python.dataflow.new.internal.DataFlowPublic as DataFlowPublic
+    private import semmle.python.dataflow.new.internal.LocalSources as LocalSources
+    private import semmle.python.internal.Awaited as Awaited
+    private import semmle.python.pointsto.Base as PointsToBase
+    private import semmle.python.types.Object as TypeObject
+    private import semmle.python.objects.TObject as TObject
+    private import semmle.python.Flow as Flow
+    private import semmle.python.objects.ObjectInternal as ObjectInternal
+    private import semmle.python.pointsto.PointsTo as PointsTo
+
+    /**
+     * DONT USE!
+     * Contains references to each predicate that use the above `ref` predicate.
+     */
+    cached
+    predicate backref() {
+      1 = 1
+      or
+      exists(any(DataFlowPublic::Node node).toString())
+      or
+      any(DataFlowPublic::Node node).hasLocationInfo(_, _, _, _, _)
+      or
+      any(LocalSources::LocalSourceNode n).flowsTo(_)
+      or
+      exists(Awaited::awaited(_))
+      or
+      PointsToBase::BaseFlow::scope_entry_value_transfer_from_earlier(_, _, _, _)
+      or
+      exists(TypeObject::Object a)
+      or
+      exists(TObject::TObject f)
+      or
+      exists(any(Flow::ControlFlowNode c).toString())
+      or
+      exists(any(ObjectInternal::ObjectInternal o).toString())
+      or
+      PointsTo::AttributePointsTo::variableAttributePointsTo(_, _, _, _, _)
+    }
+  }
+}

python/ql/lib/semmle/python/objects/ObjectInternal.qll

@@ -15,9 +15,11 @@ import semmle.python.objects.Callables
 import semmle.python.objects.Constants
 import semmle.python.objects.Sequences
 import semmle.python.objects.Descriptors
+private import semmle.python.internal.CachedStages
 
 class ObjectInternal extends TObject {
   /** Gets a textual representation of this element. */
+  cached
   abstract string toString();
 
   /**
@@ -213,7 +215,10 @@ class ObjectInternal extends TObject {
 class BuiltinOpaqueObjectInternal extends ObjectInternal, TBuiltinOpaqueObject {
   override Builtin getBuiltin() { this = TBuiltinOpaqueObject(result) }
 
-  override string toString() { result = this.getBuiltin().getClass().getName() + " object" }
+  override string toString() {
+    Stages::DataFlow::ref() and
+    result = this.getBuiltin().getClass().getName() + " object"
+  }
 
   override boolean booleanValue() {
     // TO DO ... Depends on class. `result = this.getClass().instancesBooleanValue()`

python/ql/lib/semmle/python/objects/TObject.qll

@@ -5,6 +5,7 @@ private import semmle.python.types.Builtins
 private import semmle.python.objects.ObjectInternal
 private import semmle.python.pointsto.PointsTo
 private import semmle.python.pointsto.PointsToContext
+private import semmle.python.internal.CachedStages
 
 /**
  * Internal type backing `ObjectInternal` and `Value`

python/ql/lib/semmle/python/pointsto/Base.qll

@@ -11,6 +11,7 @@
 import python
 import semmle.python.essa.SsaDefinitions
 private import semmle.python.types.Builtins
+private import semmle.python.internal.CachedStages
 
 module BasePointsTo {
   /** INTERNAL -- Use n.refersTo(value, _, origin) instead */
@@ -280,6 +281,7 @@ module BaseFlow {
   predicate scope_entry_value_transfer_from_earlier(
     EssaVariable pred_var, Scope pred_scope, ScopeEntryDefinition succ_def, Scope succ_scope
   ) {
+    Stages::DataFlow::ref() and
     exists(SsaSourceVariable var |
       reaches_exit(pred_var) and
       pred_var.getScope() = pred_scope and

python/ql/lib/semmle/python/pointsto/PointsTo.qll

@@ -6,6 +6,7 @@ private import semmle.python.pointsto.PointsToContext
 private import semmle.python.pointsto.MRO
 private import semmle.python.types.Builtins
 private import semmle.python.types.Extensions
+private import semmle.python.internal.CachedStages
 
 /* Use this version for speed */
 class CfgOrigin extends @py_object {
@@ -274,7 +275,6 @@ module PointsToInternal {
   }
 
   /** Holds if `var` refers to `(value, origin)` given the context `context`. */
-  pragma[noinline]
   cached
   predicate variablePointsTo(
     EssaVariable var, PointsToContext context, ObjectInternal value, CfgOrigin origin
@@ -924,6 +924,7 @@ private module InterModulePointsTo {
   }
 }
 
+cached
 module InterProceduralPointsTo {
   cached
   predicate call(CallNode call, PointsToContext caller, ObjectInternal value) {
@@ -938,7 +939,7 @@ module InterProceduralPointsTo {
     PointsToInternal::pointsTo(call.getFunction(), caller, value, _)
   }
 
-  pragma[noinline]
+  cached
   predicate call_points_to(
     CallNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin
   ) {
@@ -987,7 +988,7 @@ module InterProceduralPointsTo {
   }
 
   /** Points-to for parameter. `def foo(param): ...`. */
-  pragma[noinline]
+  cached
   predicate parameter_points_to(
     ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin
   ) {
@@ -1034,6 +1035,7 @@ module InterProceduralPointsTo {
     )
   }
 
+  cached
   predicate selfMethodCall(
     SelfCallsiteRefinement def, PointsToContext caller, Function func, PointsToContext callee
   ) {
@@ -1104,6 +1106,7 @@ module InterProceduralPointsTo {
    * that the number of position arguments (including expansion of `*` argument) exceeds the number of positional arguments by
    * `length` and that the excess arguments start at `start`.
    */
+  cached
   predicate varargs_tuple(
     CallNode call, PointsToContext caller, Function scope, PointsToContext callee, int start,
     int length
@@ -1117,7 +1120,7 @@ module InterProceduralPointsTo {
   }
 
   /** Holds if for function scope `func` in context `callee` the `*` parameter will hold the empty tuple. */
-  predicate varargs_empty_tuple(Function func, PointsToContext callee) {
+  private predicate varargs_empty_tuple(Function func, PointsToContext callee) {
     exists(CallNode call, PointsToContext caller, int parameter_offset |
       callsite_calls_function(call, caller, func, callee, parameter_offset) and
       func.getPositionalParameterCount() - parameter_offset >=
@@ -1136,6 +1139,7 @@ module InterProceduralPointsTo {
    * Holds if the `n`th argument in call `call` with context `caller` points-to `value` from `origin`, including values in tuples
    * expanded by a `*` argument. For example, for the call `f('a', *(`x`,`y`))` the arguments are `('a', 'x', y')`
    */
+  cached
   predicate positional_argument_points_to(
     CallNode call, int n, PointsToContext caller, ObjectInternal value, ControlFlowNode origin
   ) {
@@ -1163,7 +1167,7 @@ module InterProceduralPointsTo {
   }
 
   /** Holds if the parameter definition `def` points-to `value` from `origin` given the context `context` */
-  predicate positional_parameter_points_to(
+  private predicate positional_parameter_points_to(
     ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin
   ) {
     exists(CallNode call, int argument, PointsToContext caller, Function func, int offset |
@@ -1312,7 +1316,7 @@ module InterProceduralPointsTo {
    * `var = ...; foo(); use(var)`
    * Where var may be redefined in call to `foo` if `var` escapes (is global or non-local).
    */
-  pragma[noinline]
+  cached
   predicate callsite_points_to(
     CallsiteRefinement def, PointsToContext context, ObjectInternal value, CfgOrigin origin
   ) {
@@ -2552,10 +2556,11 @@ module AttributePointsTo {
     f.isLoad() and var.getASourceUse() = f.(AttrNode).getObject(name)
   }
 
-  pragma[nomagic]
+  cached
   predicate variableAttributePointsTo(
     EssaVariable var, Context context, string name, ObjectInternal value, CfgOrigin origin
   ) {
+    Stages::DataFlow::ref() and
     definitionAttributePointsTo(var.getDefinition(), context, name, value, origin)
     or
     exists(EssaVariable prev |

python/ql/lib/semmle/python/security/dataflow/CleartextLogging.qll

@@ -1,10 +1,4 @@
-/**
- * Provides a taint-tracking configuration for "Clear-text logging of sensitive information".
- *
- * Note, for performance reasons: only import this file if
- * `CleartextLogging::Configuration` is needed, otherwise
- * `CleartextLoggingCustomizations` should be imported instead.
- */
+/** DEPRECATED. Import `CleartextLoggingQuery` instead. */
 
 private import python
 private import semmle.python.dataflow.new.DataFlow
@@ -14,26 +8,7 @@ private import semmle.python.dataflow.new.RemoteFlowSources
 private import semmle.python.dataflow.new.BarrierGuards
 private import semmle.python.dataflow.new.SensitiveDataSources
 
-/**
- * Provides a taint-tracking configuration for detecting "Clear-text logging of sensitive information".
- */
-module CleartextLogging {
-  import CleartextLoggingCustomizations::CleartextLogging
-
-  /**
-   * A taint-tracking configuration for detecting "Clear-text logging of sensitive information".
-   */
-  class Configuration extends TaintTracking::Configuration {
-    Configuration() { this = "CleartextLogging" }
-
-    override predicate isSource(DataFlow::Node source) { source instanceof Source }
-
-    override predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
-
-    override predicate isSanitizer(DataFlow::Node node) {
-      super.isSanitizer(node)
-      or
-      node instanceof Sanitizer
-    }
-  }
+/** DEPRECATED. Import `CleartextLoggingQuery` instead. */
+deprecated module CleartextLogging {
+  import CleartextLoggingQuery // ignore-query-import
 }

python/ql/lib/semmle/python/security/dataflow/CleartextLoggingQuery.qll

@@ -0,0 +1,33 @@
+/**
+ * Provides a taint-tracking configuration for "Clear-text logging of sensitive information".
+ *
+ * Note, for performance reasons: only import this file if
+ * `CleartextLogging::Configuration` is needed, otherwise
+ * `CleartextLoggingCustomizations` should be imported instead.
+ */
+
+private import python
+private import semmle.python.dataflow.new.DataFlow
+private import semmle.python.dataflow.new.TaintTracking
+private import semmle.python.Concepts
+private import semmle.python.dataflow.new.RemoteFlowSources
+private import semmle.python.dataflow.new.BarrierGuards
+private import semmle.python.dataflow.new.SensitiveDataSources
+import CleartextLoggingCustomizations::CleartextLogging
+
+/**
+ * A taint-tracking configuration for detecting "Clear-text logging of sensitive information".
+ */
+class Configuration extends TaintTracking::Configuration {
+  Configuration() { this = "CleartextLogging" }
+
+  override predicate isSource(DataFlow::Node source) { source instanceof Source }
+
+  override predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
+
+  override predicate isSanitizer(DataFlow::Node node) {
+    super.isSanitizer(node)
+    or
+    node instanceof Sanitizer
+  }
+}

python/ql/lib/semmle/python/security/dataflow/CleartextStorage.qll

@@ -1,10 +1,4 @@
-/**
- * Provides a taint-tracking configuration for "Clear-text storage of sensitive information".
- *
- * Note, for performance reasons: only import this file if
- * `CleartextStorage::Configuration` is needed, otherwise
- * `CleartextStorageCustomizations` should be imported instead.
- */
+/** DEPRECATED. Import `CleartextStorageQuery` instead. */
 
 private import python
 private import semmle.python.dataflow.new.DataFlow
@@ -14,26 +8,7 @@ private import semmle.python.dataflow.new.RemoteFlowSources
 private import semmle.python.dataflow.new.BarrierGuards
 private import semmle.python.dataflow.new.SensitiveDataSources
 
-/**
- * Provides a taint-tracking configuration for detecting "Clear-text storage of sensitive information".
- */
-module CleartextStorage {
-  import CleartextStorageCustomizations::CleartextStorage
-
-  /**
-   * A taint-tracking configuration for detecting "Clear-text storage of sensitive information".
-   */
-  class Configuration extends TaintTracking::Configuration {
-    Configuration() { this = "CleartextStorage" }
-
-    override predicate isSource(DataFlow::Node source) { source instanceof Source }
-
-    override predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
-
-    override predicate isSanitizer(DataFlow::Node node) {
-      super.isSanitizer(node)
-      or
-      node instanceof Sanitizer
-    }
-  }
+/** DEPRECATED. Import `CleartextStorageQuery` instead. */
+deprecated module CleartextStorage {
+  import CleartextStorageQuery // ignore-query-import
 }

python/ql/lib/semmle/python/security/dataflow/CleartextStorageQuery.qll

@@ -0,0 +1,33 @@
+/**
+ * Provides a taint-tracking configuration for "Clear-text storage of sensitive information".
+ *
+ * Note, for performance reasons: only import this file if
+ * `CleartextStorage::Configuration` is needed, otherwise
+ * `CleartextStorageCustomizations` should be imported instead.
+ */
+
+private import python
+private import semmle.python.dataflow.new.DataFlow
+private import semmle.python.dataflow.new.TaintTracking
+private import semmle.python.Concepts
+private import semmle.python.dataflow.new.RemoteFlowSources
+private import semmle.python.dataflow.new.BarrierGuards
+private import semmle.python.dataflow.new.SensitiveDataSources
+import CleartextStorageCustomizations::CleartextStorage
+
+/**
+ * A taint-tracking configuration for detecting "Clear-text storage of sensitive information".
+ */
+class Configuration extends TaintTracking::Configuration {
+  Configuration() { this = "CleartextStorage" }
+
+  override predicate isSource(DataFlow::Node source) { source instanceof Source }
+
+  override predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
+
+  override predicate isSanitizer(DataFlow::Node node) {
+    super.isSanitizer(node)
+    or
+    node instanceof Sanitizer
+  }
+}

python/ql/lib/semmle/python/security/dataflow/CodeInjection.qll

@@ -1,42 +1,13 @@
-/**
- * Provides a taint-tracking configuration for detecting "code injection" vulnerabilities.
- *
- * Note, for performance reasons: only import this file if
- * `CodeInjection::Configuration` is needed, otherwise
- * `CodeInjectionCustomizations` should be imported instead.
- */
+/** DEPRECATED. Import `CodeInjectionQuery` instead. */
 
 private import python
 import semmle.python.dataflow.new.DataFlow
 import semmle.python.dataflow.new.TaintTracking
 
-/**
- * Provides a taint-tracking configuration for detecting "code injection" vulnerabilities.
- */
-module CodeInjection {
-  import CodeInjectionCustomizations::CodeInjection
-
-  /**
-   * A taint-tracking configuration for detecting "code injection" vulnerabilities.
-   */
-  class Configuration extends TaintTracking::Configuration {
-    Configuration() { this = "CodeInjection" }
-
-    override predicate isSource(DataFlow::Node source) { source instanceof Source }
-
-    override predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
-
-    override predicate isSanitizer(DataFlow::Node node) { node instanceof Sanitizer }
-
-    override predicate isSanitizerGuard(DataFlow::BarrierGuard guard) {
-      guard instanceof SanitizerGuard
-    }
-  }
+/** DEPRECATED. Import `CodeInjectionQuery` instead. */
+deprecated module CodeInjection {
+  import CodeInjectionQuery // ignore-query-import
 }
 
-/**
- * DEPRECATED: Don't extend this class for customization, since this will lead to bad
- * performance, instead use the new `CodeInjectionCustomizations.qll` file, and extend
- * its' classes.
- */
+/** DEPRECATED. Import `CodeInjectionQuery` instead. */
 deprecated class CodeInjectionConfiguration = CodeInjection::Configuration;

python/ql/lib/semmle/python/security/dataflow/CodeInjectionQuery.qll

@@ -0,0 +1,29 @@
+/**
+ * Provides a taint-tracking configuration for detecting "code injection" vulnerabilities.
+ *
+ * Note, for performance reasons: only import this file if
+ * `CodeInjection::Configuration` is needed, otherwise
+ * `CodeInjectionCustomizations` should be imported instead.
+ */
+
+private import python
+import semmle.python.dataflow.new.DataFlow
+import semmle.python.dataflow.new.TaintTracking
+import CodeInjectionCustomizations::CodeInjection
+
+/**
+ * A taint-tracking configuration for detecting "code injection" vulnerabilities.
+ */
+class Configuration extends TaintTracking::Configuration {
+  Configuration() { this = "CodeInjection" }
+
+  override predicate isSource(DataFlow::Node source) { source instanceof Source }
+
+  override predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
+
+  override predicate isSanitizer(DataFlow::Node node) { node instanceof Sanitizer }
+
+  override predicate isSanitizerGuard(DataFlow::BarrierGuard guard) {
+    guard instanceof SanitizerGuard
+  }
+}

python/ql/lib/semmle/python/security/dataflow/CommandInjection.qll

@@ -1,42 +1,13 @@
-/**
- * Provides a taint-tracking configuration for detecting "command injection" vulnerabilities.
- *
- * Note, for performance reasons: only import this file if
- * `CommandInjection::Configuration` is needed, otherwise
- * `CommandInjectionCustomizations` should be imported instead.
- */
+/** DEPRECATED. Import `CommandInjectionQuery` instead. */
 
 private import python
 import semmle.python.dataflow.new.DataFlow
 import semmle.python.dataflow.new.TaintTracking
 
-/**
- * Provides a taint-tracking configuration for detecting "command injection" vulnerabilities.
- */
-module CommandInjection {
-  import CommandInjectionCustomizations::CommandInjection
-
-  /**
-   * A taint-tracking configuration for detecting "command injection" vulnerabilities.
-   */
-  class Configuration extends TaintTracking::Configuration {
-    Configuration() { this = "CommandInjection" }
-
-    override predicate isSource(DataFlow::Node source) { source instanceof Source }
-
-    override predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
-
-    override predicate isSanitizer(DataFlow::Node node) { node instanceof Sanitizer }
-
-    override predicate isSanitizerGuard(DataFlow::BarrierGuard guard) {
-      guard instanceof SanitizerGuard
-    }
-  }
+/** DEPRECATED. Import `CommandInjectionQuery` instead. */
+deprecated module CommandInjection {
+  import CommandInjectionQuery // ignore-query-import
 }
 
-/**
- * DEPRECATED: Don't extend this class for customization, since this will lead to bad
- * performance, instead use the new `CommandInjectionCustomizations.qll` file, and extend
- * its' classes.
- */
+/** DEPRECATED. Import `CommandInjectionQuery` instead. */
 deprecated class CommandInjectionConfiguration = CommandInjection::Configuration;

python/ql/lib/semmle/python/security/dataflow/CommandInjectionQuery.qll

@@ -0,0 +1,29 @@
+/**
+ * Provides a taint-tracking configuration for detecting "command injection" vulnerabilities.
+ *
+ * Note, for performance reasons: only import this file if
+ * `CommandInjection::Configuration` is needed, otherwise
+ * `CommandInjectionCustomizations` should be imported instead.
+ */
+
+private import python
+import semmle.python.dataflow.new.DataFlow
+import semmle.python.dataflow.new.TaintTracking
+import CommandInjectionCustomizations::CommandInjection
+
+/**
+ * A taint-tracking configuration for detecting "command injection" vulnerabilities.
+ */
+class Configuration extends TaintTracking::Configuration {
+  Configuration() { this = "CommandInjection" }
+
+  override predicate isSource(DataFlow::Node source) { source instanceof Source }
+
+  override predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
+
+  override predicate isSanitizer(DataFlow::Node node) { node instanceof Sanitizer }
+
+  override predicate isSanitizerGuard(DataFlow::BarrierGuard guard) {
+    guard instanceof SanitizerGuard
+  }
+}

python/ql/lib/semmle/python/security/dataflow/LdapInjection.qll

@@ -1,10 +1,4 @@
-/**
- * Provides taint-tracking configurations for detecting LDAP injection vulnerabilities
- *
- * Note, for performance reasons: only import this file if
- * `LdapInjection::Configuration` is needed, otherwise
- * `LdapInjectionCustomizations` should be imported instead.
- */
+/** DEPRECATED. Import `LdapInjectionQuery` instead. */
 
 import python
 import semmle.python.Concepts
@@ -12,49 +6,7 @@ import semmle.python.dataflow.new.DataFlow
 import semmle.python.dataflow.new.TaintTracking
 import semmle.python.dataflow.new.RemoteFlowSources
 
-/**
- * Provides aint-tracking configurations for detecting LDAP injection vulnerabilities.class
- *
- * Two configurations are provided. One is for detecting LDAP injection
- * via the distinguished name (DN). The other is for detecting LDAP injection
- * via the filter. These require different escapings.
- */
-module LdapInjection {
-  import LdapInjectionCustomizations::LdapInjection
-
-  /**
-   * A taint-tracking configuration for detecting LDAP injection vulnerabilities
-   * via the distinguished name (DN) parameter of an LDAP search.
-   */
-  class DnConfiguration extends TaintTracking::Configuration {
-    DnConfiguration() { this = "LdapDnInjection" }
-
-    override predicate isSource(DataFlow::Node source) { source instanceof Source }
-
-    override predicate isSink(DataFlow::Node sink) { sink instanceof DnSink }
-
-    override predicate isSanitizer(DataFlow::Node node) { node instanceof DnSanitizer }
-
-    override predicate isSanitizerGuard(DataFlow::BarrierGuard guard) {
-      guard instanceof DnSanitizerGuard
-    }
-  }
-
-  /**
-   * A taint-tracking configuration for detecting LDAP injection vulnerabilities
-   * via the filter parameter of an LDAP search.
-   */
-  class FilterConfiguration extends TaintTracking::Configuration {
-    FilterConfiguration() { this = "LdapFilterInjection" }
-
-    override predicate isSource(DataFlow::Node source) { source instanceof Source }
-
-    override predicate isSink(DataFlow::Node sink) { sink instanceof FilterSink }
-
-    override predicate isSanitizer(DataFlow::Node node) { node instanceof FilterSanitizer }
-
-    override predicate isSanitizerGuard(DataFlow::BarrierGuard guard) {
-      guard instanceof FilterSanitizerGuard
-    }
-  }
+/** DEPRECATED. Import `LdapInjectionQuery` instead. */
+deprecated module LdapInjection {
+  import LdapInjectionQuery // ignore-query-import
 }

python/ql/lib/semmle/python/security/dataflow/LdapInjectionQuery.qll

@@ -0,0 +1,50 @@
+/**
+ * Provides taint-tracking configurations for detecting LDAP injection vulnerabilities
+ *
+ * Note, for performance reasons: only import this file if
+ * `LdapInjection::Configuration` is needed, otherwise
+ * `LdapInjectionCustomizations` should be imported instead.
+ */
+
+import python
+import semmle.python.Concepts
+import semmle.python.dataflow.new.DataFlow
+import semmle.python.dataflow.new.TaintTracking
+import semmle.python.dataflow.new.RemoteFlowSources
+import LdapInjectionCustomizations::LdapInjection
+
+/**
+ * A taint-tracking configuration for detecting LDAP injection vulnerabilities
+ * via the distinguished name (DN) parameter of an LDAP search.
+ */
+class DnConfiguration extends TaintTracking::Configuration {
+  DnConfiguration() { this = "LdapDnInjection" }
+
+  override predicate isSource(DataFlow::Node source) { source instanceof Source }
+
+  override predicate isSink(DataFlow::Node sink) { sink instanceof DnSink }
+
+  override predicate isSanitizer(DataFlow::Node node) { node instanceof DnSanitizer }
+
+  override predicate isSanitizerGuard(DataFlow::BarrierGuard guard) {
+    guard instanceof DnSanitizerGuard
+  }
+}
+
+/**
+ * A taint-tracking configuration for detecting LDAP injection vulnerabilities
+ * via the filter parameter of an LDAP search.
+ */
+class FilterConfiguration extends TaintTracking::Configuration {
+  FilterConfiguration() { this = "LdapFilterInjection" }
+
+  override predicate isSource(DataFlow::Node source) { source instanceof Source }
+
+  override predicate isSink(DataFlow::Node sink) { sink instanceof FilterSink }
+
+  override predicate isSanitizer(DataFlow::Node node) { node instanceof FilterSanitizer }
+
+  override predicate isSanitizerGuard(DataFlow::BarrierGuard guard) {
+    guard instanceof FilterSanitizerGuard
+  }
+}

python/ql/lib/semmle/python/security/dataflow/LogInjection.qll

@@ -1,35 +1,10 @@
-/**
- * Provides a taint-tracking configuration for tracking untrusted user input used in log entries.
- *
- * Note, for performance reasons: only import this file if
- * `LogInjection::Configuration` is needed, otherwise
- * `LogInjectionCustomizations` should be imported instead.
- */
+/** DEPRECATED. Import `LogInjectionQuery` instead. */
 
 import python
 import semmle.python.dataflow.new.DataFlow
 import semmle.python.dataflow.new.TaintTracking
 
-/**
- * Provides a taint-tracking configuration for tracking untrusted user input used in log entries.
- */
-module LogInjection {
-  import LogInjectionCustomizations::LogInjection
-
-  /**
-   * A taint-tracking configuration for tracking untrusted user input used in log entries.
-   */
-  class Configuration extends TaintTracking::Configuration {
-    Configuration() { this = "LogInjection" }
-
-    override predicate isSource(DataFlow::Node source) { source instanceof Source }
-
-    override predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
-
-    override predicate isSanitizer(DataFlow::Node node) { node instanceof Sanitizer }
-
-    override predicate isSanitizerGuard(DataFlow::BarrierGuard guard) {
-      guard instanceof SanitizerGuard
-    }
-  }
+/** DEPRECATED. Import `LogInjectionQuery` instead. */
+deprecated module LogInjection {
+  import LogInjectionQuery // ignore-query-import
 }

python/ql/lib/semmle/python/security/dataflow/LogInjectionQuery.qll

@@ -0,0 +1,29 @@
+/**
+ * Provides a taint-tracking configuration for tracking untrusted user input used in log entries.
+ *
+ * Note, for performance reasons: only import this file if
+ * `LogInjection::Configuration` is needed, otherwise
+ * `LogInjectionCustomizations` should be imported instead.
+ */
+
+import python
+import semmle.python.dataflow.new.DataFlow
+import semmle.python.dataflow.new.TaintTracking
+import LogInjectionCustomizations::LogInjection
+
+/**
+ * A taint-tracking configuration for tracking untrusted user input used in log entries.
+ */
+class Configuration extends TaintTracking::Configuration {
+  Configuration() { this = "LogInjection" }
+
+  override predicate isSource(DataFlow::Node source) { source instanceof Source }
+
+  override predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
+
+  override predicate isSanitizer(DataFlow::Node node) { node instanceof Sanitizer }
+
+  override predicate isSanitizerGuard(DataFlow::BarrierGuard guard) {
+    guard instanceof SanitizerGuard
+  }
+}

python/ql/lib/semmle/python/security/dataflow/PathInjection.qll

@@ -1,84 +1,13 @@
-/**
- * Provides taint-tracking configurations for detecting "path injection" vulnerabilities.
- *
- * Note, for performance reasons: only import this file if
- * `PathInjection::Configuration` is needed, otherwise
- * `PathInjectionCustomizations` should be imported instead.
- */
+/** DEPRECATED. Import `PathInjectionQuery` instead. */
 
 private import python
 private import semmle.python.Concepts
 import semmle.python.dataflow.new.DataFlow
 import semmle.python.dataflow.new.TaintTracking
 
-/**
- * Provides a taint-tracking configuration for detecting "path injection" vulnerabilities.
- */
-module PathInjection {
-  import PathInjectionCustomizations::PathInjection
-
-  /**
-   * A taint-tracking configuration for detecting "path injection" vulnerabilities.
-   *
-   * This configuration uses two flow states, `NotNormalized` and `NormalizedUnchecked`,
-   * to track the requirement that a file path must be first normalized and then checked
-   * before it is safe to use.
-   *
-   * At sources, paths are assumed not normalized. At normalization points, they change
-   * state to `NormalizedUnchecked` after which they can be made safe by an appropriate
-   * check of the prefix.
-   *
-   * Such checks are ineffective in the `NotNormalized` state.
-   */
-  class Configuration extends TaintTracking::Configuration {
-    Configuration() { this = "PathInjection" }
-
-    override predicate isSource(DataFlow::Node source, DataFlow::FlowState state) {
-      source instanceof Source and state instanceof NotNormalized
-    }
-
-    override predicate isSink(DataFlow::Node sink, DataFlow::FlowState state) {
-      sink instanceof Sink and
-      (
-        state instanceof NotNormalized or
-        state instanceof NormalizedUnchecked
-      )
-    }
-
-    override predicate isSanitizer(DataFlow::Node node) { node instanceof Sanitizer }
-
-    override predicate isSanitizer(DataFlow::Node node, DataFlow::FlowState state) {
-      // Block `NotNormalized` paths here, since they change state to `NormalizedUnchecked`
-      node instanceof Path::PathNormalization and
-      state instanceof NotNormalized
-      or
-      node = any(Path::SafeAccessCheck c).getAGuardedNode() and
-      state instanceof NormalizedUnchecked
-    }
-
-    override predicate isSanitizerGuard(DataFlow::BarrierGuard guard) {
-      guard instanceof SanitizerGuard
-    }
-
-    override predicate isAdditionalTaintStep(
-      DataFlow::Node nodeFrom, DataFlow::FlowState stateFrom, DataFlow::Node nodeTo,
-      DataFlow::FlowState stateTo
-    ) {
-      nodeFrom = nodeTo.(Path::PathNormalization).getPathArg() and
-      stateFrom instanceof NotNormalized and
-      stateTo instanceof NormalizedUnchecked
-    }
-  }
-
-  /** A state signifying that the file path has not been normalized. */
-  class NotNormalized extends DataFlow::FlowState {
-    NotNormalized() { this = "NotNormalized" }
-  }
-
-  /** A state signifying that the file path has been normalized, but not checked. */
-  class NormalizedUnchecked extends DataFlow::FlowState {
-    NormalizedUnchecked() { this = "NormalizedUnchecked" }
-  }
+/** DEPRECATED. Import `PathInjectionQuery` instead. */
+deprecated module PathInjection {
+  import PathInjectionQuery // ignore-query-import
 }
 
 // ---------------------------------------------------------------------------
@@ -93,7 +22,7 @@ import PathInjectionCustomizations::PathInjection
 // Case 1. The path is never normalized.
 // ---------------------------------------------------------------------------
 /**
- * DEPRECATED: Use `PathInjection::Configuration` instead
+ * DEPRECATED: Import `PathInjectionQuery` instead.
  *
  * Configuration to find paths from sources to sinks that contain no normalization.
  */
@@ -116,7 +45,7 @@ deprecated class PathNotNormalizedConfiguration extends TaintTracking::Configura
 }
 
 /**
- * DEPRECATED: Use `PathInjection::Configuration` instead
+ * DEPRECATED: Import `PathInjectionQuery` instead.
  *
  * Holds if there is a path injection from source to sink, where the (python) path is
  * not normalized.
@@ -129,7 +58,7 @@ deprecated predicate pathNotNormalized(CustomPathNode source, CustomPathNode sin
 // Case 2. The path is normalized at least once, but never checked afterwards.
 // ---------------------------------------------------------------------------
 /**
- * DEPRECATED: Use `PathInjection::Configuration` instead
+ * DEPRECATED: Import `PathInjectionQuery` instead.
  *
  * Configuration to find paths from sources to normalizations that contain no prior normalizations.
  */
@@ -150,7 +79,7 @@ deprecated class FirstNormalizationConfiguration extends TaintTracking::Configur
 }
 
 /**
- * DEPRECATED: Use `PathInjection::Configuration` instead
+ * DEPRECATED: Import `PathInjectionQuery` instead.
  *
  * Configuration to find paths from normalizations to sinks that do not go through a check.
  */
@@ -171,7 +100,7 @@ deprecated class NormalizedPathNotCheckedConfiguration extends TaintTracking2::C
 }
 
 /**
- * DEPRECATED: Use `PathInjection::Configuration` instead
+ * DEPRECATED: Import `PathInjectionQuery` instead.
  *
  * Holds if there is a path injection from source to sink, where the (python) path is
  * normalized at least once, but never checked afterwards.
@@ -191,7 +120,7 @@ deprecated predicate pathNotCheckedAfterNormalization(CustomPathNode source, Cus
 // Query: Either case 1 or case 2.
 // ---------------------------------------------------------------------------
 /**
- * DEPRECATED: Use `PathInjection::Configuration` instead
+ * DEPRECATED: Import `PathInjectionQuery` instead.
  *
  * Holds if there is a path injection from source to sink
  */

python/ql/lib/semmle/python/security/dataflow/PathInjectionQuery.qll

@@ -0,0 +1,76 @@
+/**
+ * Provides taint-tracking configurations for detecting "path injection" vulnerabilities.
+ *
+ * Note, for performance reasons: only import this file if
+ * `PathInjection::Configuration` is needed, otherwise
+ * `PathInjectionCustomizations` should be imported instead.
+ */
+
+private import python
+private import semmle.python.Concepts
+import semmle.python.dataflow.new.DataFlow
+import semmle.python.dataflow.new.TaintTracking
+import PathInjectionCustomizations::PathInjection
+
+/**
+ * A taint-tracking configuration for detecting "path injection" vulnerabilities.
+ *
+ * This configuration uses two flow states, `NotNormalized` and `NormalizedUnchecked`,
+ * to track the requirement that a file path must be first normalized and then checked
+ * before it is safe to use.
+ *
+ * At sources, paths are assumed not normalized. At normalization points, they change
+ * state to `NormalizedUnchecked` after which they can be made safe by an appropriate
+ * check of the prefix.
+ *
+ * Such checks are ineffective in the `NotNormalized` state.
+ */
+class Configuration extends TaintTracking::Configuration {
+  Configuration() { this = "PathInjection" }
+
+  override predicate isSource(DataFlow::Node source, DataFlow::FlowState state) {
+    source instanceof Source and state instanceof NotNormalized
+  }
+
+  override predicate isSink(DataFlow::Node sink, DataFlow::FlowState state) {
+    sink instanceof Sink and
+    (
+      state instanceof NotNormalized or
+      state instanceof NormalizedUnchecked
+    )
+  }
+
+  override predicate isSanitizer(DataFlow::Node node) { node instanceof Sanitizer }
+
+  override predicate isSanitizer(DataFlow::Node node, DataFlow::FlowState state) {
+    // Block `NotNormalized` paths here, since they change state to `NormalizedUnchecked`
+    node instanceof Path::PathNormalization and
+    state instanceof NotNormalized
+    or
+    node = any(Path::SafeAccessCheck c).getAGuardedNode() and
+    state instanceof NormalizedUnchecked
+  }
+
+  override predicate isSanitizerGuard(DataFlow::BarrierGuard guard) {
+    guard instanceof SanitizerGuard
+  }
+
+  override predicate isAdditionalTaintStep(
+    DataFlow::Node nodeFrom, DataFlow::FlowState stateFrom, DataFlow::Node nodeTo,
+    DataFlow::FlowState stateTo
+  ) {
+    nodeFrom = nodeTo.(Path::PathNormalization).getPathArg() and
+    stateFrom instanceof NotNormalized and
+    stateTo instanceof NormalizedUnchecked
+  }
+}
+
+/** A state signifying that the file path has not been normalized. */
+class NotNormalized extends DataFlow::FlowState {
+  NotNormalized() { this = "NotNormalized" }
+}
+
+/** A state signifying that the file path has been normalized, but not checked. */
+class NormalizedUnchecked extends DataFlow::FlowState {
+  NormalizedUnchecked() { this = "NormalizedUnchecked" }
+}

python/ql/lib/semmle/python/security/dataflow/PolynomialReDoS.qll

@@ -1,35 +1,10 @@
-/**
- * Provides a taint-tracking configuration for detecting "polynomial regular expression denial of service (ReDoS)" vulnerabilities.
- *
- * Note, for performance reasons: only import this file if
- * `PolynomialReDoS::Configuration` is needed, otherwise
- * `PolynomialReDoSCustomizations` should be imported instead.
- */
+/** DEPRECATED. Import `PolynomialReDoSQuery` instead. */
 
 private import python
 import semmle.python.dataflow.new.DataFlow
 import semmle.python.dataflow.new.TaintTracking
 
-/**
- * Provides a taint-tracking configuration for detecting "polynomial regular expression denial of service (ReDoS)" vulnerabilities.
- */
-module PolynomialReDoS {
-  import PolynomialReDoSCustomizations::PolynomialReDoS
-
-  /**
-   * A taint-tracking configuration for detecting "polynomial regular expression denial of service (ReDoS)" vulnerabilities.
-   */
-  class Configuration extends TaintTracking::Configuration {
-    Configuration() { this = "PolynomialReDoS" }
-
-    override predicate isSource(DataFlow::Node source) { source instanceof Source }
-
-    override predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
-
-    override predicate isSanitizer(DataFlow::Node node) { node instanceof Sanitizer }
-
-    override predicate isSanitizerGuard(DataFlow::BarrierGuard guard) {
-      guard instanceof SanitizerGuard
-    }
-  }
+/** DEPRECATED. Import `PolynomialReDoSQuery` instead. */
+deprecated module PolynomialReDoS {
+  import PolynomialReDoSQuery // ignore-query-import
 }

python/ql/lib/semmle/python/security/dataflow/PolynomialReDoSQuery.qll

@@ -0,0 +1,29 @@
+/**
+ * Provides a taint-tracking configuration for detecting "polynomial regular expression denial of service (ReDoS)" vulnerabilities.
+ *
+ * Note, for performance reasons: only import this file if
+ * `PolynomialReDoS::Configuration` is needed, otherwise
+ * `PolynomialReDoSCustomizations` should be imported instead.
+ */
+
+private import python
+import semmle.python.dataflow.new.DataFlow
+import semmle.python.dataflow.new.TaintTracking
+import PolynomialReDoSCustomizations::PolynomialReDoS
+
+/**
+ * A taint-tracking configuration for detecting "polynomial regular expression denial of service (ReDoS)" vulnerabilities.
+ */
+class Configuration extends TaintTracking::Configuration {
+  Configuration() { this = "PolynomialReDoS" }
+
+  override predicate isSource(DataFlow::Node source) { source instanceof Source }
+
+  override predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
+
+  override predicate isSanitizer(DataFlow::Node node) { node instanceof Sanitizer }
+
+  override predicate isSanitizerGuard(DataFlow::BarrierGuard guard) {
+    guard instanceof SanitizerGuard
+  }
+}

python/ql/lib/semmle/python/security/dataflow/ReflectedXSS.qll

@@ -1,45 +1,16 @@
-/**
- * Provides a taint-tracking configuration for detecting "reflected server-side cross-site scripting" vulnerabilities.
- *
- * Note, for performance reasons: only import this file if
- * `ReflectedXSS::Configuration` is needed, otherwise
- * `ReflectedXSSCustomizations` should be imported instead.
- */
+/** DEPRECATED. Import `ReflectedXSSQuery` instead. */
 
 private import python
 import semmle.python.dataflow.new.DataFlow
 import semmle.python.dataflow.new.TaintTracking
 
-/**
- * Provides a taint-tracking configuration for detecting "reflected server-side cross-site scripting" vulnerabilities.
- */
-module ReflectedXss {
-  import ReflectedXSSCustomizations::ReflectedXss
-
-  /**
-   * A taint-tracking configuration for detecting "reflected server-side cross-site scripting" vulnerabilities.
-   */
-  class Configuration extends TaintTracking::Configuration {
-    Configuration() { this = "ReflectedXSS" }
-
-    override predicate isSource(DataFlow::Node source) { source instanceof Source }
-
-    override predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
-
-    override predicate isSanitizer(DataFlow::Node node) { node instanceof Sanitizer }
-
-    override predicate isSanitizerGuard(DataFlow::BarrierGuard guard) {
-      guard instanceof SanitizerGuard
-    }
-  }
+/** DEPRECATED. Import `ReflectedXSSQuery` instead. */
+deprecated module ReflectedXss {
+  import ReflectedXssQuery // ignore-query-import
 }
 
-/** DEPRECATED: Alias for ReflectedXss */
+/** DEPRECATED. Import `ReflectedXSSQuery` instead. */
 deprecated module ReflectedXSS = ReflectedXss;
 
-/**
- * DEPRECATED: Don't extend this class for customization, since this will lead to bad
- * performance, instead use the new `ReflectedXSSCustomizations.qll` file, and extend
- * its' classes.
- */
+/** DEPRECATED. Import `ReflectedXSSQuery` instead. */
 deprecated class ReflectedXssConfiguration = ReflectedXss::Configuration;

python/ql/lib/semmle/python/security/dataflow/ReflectedXssQuery.qll

@@ -0,0 +1,29 @@
+/**
+ * Provides a taint-tracking configuration for detecting "reflected server-side cross-site scripting" vulnerabilities.
+ *
+ * Note, for performance reasons: only import this file if
+ * `ReflectedXSS::Configuration` is needed, otherwise
+ * `ReflectedXSSCustomizations` should be imported instead.
+ */
+
+private import python
+import semmle.python.dataflow.new.DataFlow
+import semmle.python.dataflow.new.TaintTracking
+import ReflectedXSSCustomizations::ReflectedXss
+
+/**
+ * A taint-tracking configuration for detecting "reflected server-side cross-site scripting" vulnerabilities.
+ */
+class Configuration extends TaintTracking::Configuration {
+  Configuration() { this = "ReflectedXSS" }
+
+  override predicate isSource(DataFlow::Node source) { source instanceof Source }
+
+  override predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
+
+  override predicate isSanitizer(DataFlow::Node node) { node instanceof Sanitizer }
+
+  override predicate isSanitizerGuard(DataFlow::BarrierGuard guard) {
+    guard instanceof SanitizerGuard
+  }
+}

python/ql/lib/semmle/python/security/dataflow/RegexInjection.qll

@@ -1,37 +1,10 @@
-/**
- * Provides a taint-tracking configuration for detecting regular expression injection
- * vulnerabilities.
- *
- * Note, for performance reasons: only import this file if
- * `RegexInjection::Configuration` is needed, otherwise
- * `RegexInjectionCustomizations` should be imported instead.
- */
+/** DEPRECATED. Import `RegexInjectionQuery` instead. */
 
 private import python
 import semmle.python.dataflow.new.DataFlow
 import semmle.python.dataflow.new.TaintTracking
 
-/**
- * Provides a taint-tracking configuration for detecting regular expression injection
- * vulnerabilities.
- */
-module RegexInjection {
-  import RegexInjectionCustomizations::RegexInjection
-
-  /**
-   * A taint-tracking configuration for detecting "reflected server-side cross-site scripting" vulnerabilities.
-   */
-  class Configuration extends TaintTracking::Configuration {
-    Configuration() { this = "RegexInjection" }
-
-    override predicate isSource(DataFlow::Node source) { source instanceof Source }
-
-    override predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
-
-    override predicate isSanitizer(DataFlow::Node node) { node instanceof Sanitizer }
-
-    override predicate isSanitizerGuard(DataFlow::BarrierGuard guard) {
-      guard instanceof SanitizerGuard
-    }
-  }
+/** DEPRECATED. Import `RegexInjectionQuery` instead. */
+deprecated module RegexInjection {
+  import RegexInjectionQuery // ignore-query-import
 }

python/ql/lib/semmle/python/security/dataflow/RegexInjectionQuery.qll

@@ -0,0 +1,30 @@
+/**
+ * Provides a taint-tracking configuration for detecting regular expression injection
+ * vulnerabilities.
+ *
+ * Note, for performance reasons: only import this file if
+ * `RegexInjection::Configuration` is needed, otherwise
+ * `RegexInjectionCustomizations` should be imported instead.
+ */
+
+private import python
+import semmle.python.dataflow.new.DataFlow
+import semmle.python.dataflow.new.TaintTracking
+import RegexInjectionCustomizations::RegexInjection
+
+/**
+ * A taint-tracking configuration for detecting "reflected server-side cross-site scripting" vulnerabilities.
+ */
+class Configuration extends TaintTracking::Configuration {
+  Configuration() { this = "RegexInjection" }
+
+  override predicate isSource(DataFlow::Node source) { source instanceof Source }
+
+  override predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
+
+  override predicate isSanitizer(DataFlow::Node node) { node instanceof Sanitizer }
+
+  override predicate isSanitizerGuard(DataFlow::BarrierGuard guard) {
+    guard instanceof SanitizerGuard
+  }
+}

python/ql/lib/semmle/python/security/dataflow/ServerSideRequestForgery.qll

@@ -1,84 +1,25 @@
-/**
- * Provides a taint-tracking configuration for detecting "Server-side request forgery" vulnerabilities.
- *
- * Note, for performance reasons: only import this file if
- * `ServerSideRequestForgery::Configuration` is needed, otherwise
- * `ServerSideRequestForgeryCustomizations` should be imported instead.
- */
+/** DEPRECATED. Import `ServerSideRequestForgeryQuery` instead. */
 
 private import python
 import semmle.python.dataflow.new.DataFlow
 import semmle.python.dataflow.new.TaintTracking
 import semmle.python.Concepts
+import ServerSideRequestForgeryQuery as ServerSideRequestForgeryQuery // ignore-query-import
 
-/**
- * Provides a taint-tracking configuration for detecting "Server-side request forgery" vulnerabilities.
- *
- * This configuration has a sanitizer to limit results to cases where attacker has full control of URL.
- * See `PartialServerSideRequestForgery` for a variant without this requirement.
- *
- * You should use the `partOfFullyControlledRequest` to only select results where all
- * URL parts are fully controlled.
- */
-module FullServerSideRequestForgery {
+/** DEPRECATED. Import `ServerSideRequestForgeryQuery` instead. */
+deprecated module FullServerSideRequestForgery {
   import ServerSideRequestForgeryCustomizations::ServerSideRequestForgery
 
-  /**
-   * A taint-tracking configuration for detecting "Server-side request forgery" vulnerabilities.
-   */
-  class Configuration extends TaintTracking::Configuration {
-    Configuration() { this = "FullServerSideRequestForgery" }
-
-    override predicate isSource(DataFlow::Node source) { source instanceof Source }
-
-    override predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
-
-    override predicate isSanitizer(DataFlow::Node node) {
-      node instanceof Sanitizer
-      or
-      node instanceof FullUrlControlSanitizer
-    }
-
-    override predicate isSanitizerGuard(DataFlow::BarrierGuard guard) {
-      guard instanceof SanitizerGuard
-    }
-  }
+  class Configuration = ServerSideRequestForgeryQuery::FullServerSideRequestForgeryConfiguration;
 }
 
-/**
- * Holds if all URL parts of `request` is fully user controlled.
- */
-predicate fullyControlledRequest(HTTP::Client::Request request) {
-  exists(FullServerSideRequestForgery::Configuration fullConfig |
-    forall(DataFlow::Node urlPart | urlPart = request.getAUrlPart() |
-      fullConfig.hasFlow(_, urlPart)
-    )
-  )
-}
+/** DEPRECATED. Import `ServerSideRequestForgeryQuery` instead. */
+deprecated predicate fullyControlledRequest =
+  ServerSideRequestForgeryQuery::fullyControlledRequest/1;
 
-/**
- * Provides a taint-tracking configuration for detecting "Server-side request forgery" vulnerabilities.
- *
- * This configuration has results, even when the attacker does not have full control over the URL.
- * See `FullServerSideRequestForgery` for variant that has this requirement.
- */
-module PartialServerSideRequestForgery {
+/** DEPRECATED. Import `ServerSideRequestForgeryQuery` instead. */
+deprecated module PartialServerSideRequestForgery {
   import ServerSideRequestForgeryCustomizations::ServerSideRequestForgery
 
-  /**
-   * A taint-tracking configuration for detecting "Server-side request forgery" vulnerabilities.
-   */
-  class Configuration extends TaintTracking::Configuration {
-    Configuration() { this = "PartialServerSideRequestForgery" }
-
-    override predicate isSource(DataFlow::Node source) { source instanceof Source }
-
-    override predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
-
-    override predicate isSanitizer(DataFlow::Node node) { node instanceof Sanitizer }
-
-    override predicate isSanitizerGuard(DataFlow::BarrierGuard guard) {
-      guard instanceof SanitizerGuard
-    }
-  }
+  class Configuration = ServerSideRequestForgeryQuery::PartialServerSideRequestForgeryConfiguration;
 }

python/ql/lib/semmle/python/security/dataflow/ServerSideRequestForgeryQuery.qll

@@ -0,0 +1,71 @@
+/**
+ * Provides a taint-tracking configuration for detecting "Server-side request forgery" vulnerabilities.
+ *
+ * Note, for performance reasons: only import this file if
+ * `ServerSideRequestForgery::Configuration` is needed, otherwise
+ * `ServerSideRequestForgeryCustomizations` should be imported instead.
+ */
+
+private import python
+import semmle.python.dataflow.new.DataFlow
+import semmle.python.dataflow.new.TaintTracking
+import semmle.python.Concepts
+import ServerSideRequestForgeryCustomizations::ServerSideRequestForgery
+
+/**
+ * A taint-tracking configuration for detecting "Server-side request forgery" vulnerabilities.
+ *
+ * This configuration has a sanitizer to limit results to cases where attacker has full control of URL.
+ * See `PartialServerSideRequestForgery` for a variant without this requirement.
+ *
+ * You should use the `fullyControlledRequest` to only select results where all
+ * URL parts are fully controlled.
+ */
+class FullServerSideRequestForgeryConfiguration extends TaintTracking::Configuration {
+  FullServerSideRequestForgeryConfiguration() { this = "FullServerSideRequestForgery" }
+
+  override predicate isSource(DataFlow::Node source) { source instanceof Source }
+
+  override predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
+
+  override predicate isSanitizer(DataFlow::Node node) {
+    node instanceof Sanitizer
+    or
+    node instanceof FullUrlControlSanitizer
+  }
+
+  override predicate isSanitizerGuard(DataFlow::BarrierGuard guard) {
+    guard instanceof SanitizerGuard
+  }
+}
+
+/**
+ * Holds if all URL parts of `request` is fully user controlled.
+ */
+predicate fullyControlledRequest(HTTP::Client::Request request) {
+  exists(FullServerSideRequestForgeryConfiguration fullConfig |
+    forall(DataFlow::Node urlPart | urlPart = request.getAUrlPart() |
+      fullConfig.hasFlow(_, urlPart)
+    )
+  )
+}
+
+/**
+ * A taint-tracking configuration for detecting "Server-side request forgery" vulnerabilities.
+ *
+ * This configuration has results, even when the attacker does not have full control over the URL.
+ * See `FullServerSideRequestForgeryConfiguration`, and the `fullyControlledRequest` predicate.
+ */
+class PartialServerSideRequestForgeryConfiguration extends TaintTracking::Configuration {
+  PartialServerSideRequestForgeryConfiguration() { this = "PartialServerSideRequestForgery" }
+
+  override predicate isSource(DataFlow::Node source) { source instanceof Source }
+
+  override predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
+
+  override predicate isSanitizer(DataFlow::Node node) { node instanceof Sanitizer }
+
+  override predicate isSanitizerGuard(DataFlow::BarrierGuard guard) {
+    guard instanceof SanitizerGuard
+  }
+}

python/ql/lib/semmle/python/security/dataflow/SqlInjection.qll

@@ -1,45 +1,16 @@
-/**
- * Provides a taint-tracking configuration for detecting "SQL injection" vulnerabilities.
- *
- * Note, for performance reasons: only import this file if
- * `SqlInjection::Configuration` is needed, otherwise
- * `SqlInjectionCustomizations` should be imported instead.
- */
+/** DEPRECATED. Import `SqlInjectionQuery` instead. */
 
 private import python
 import semmle.python.dataflow.new.DataFlow
 import semmle.python.dataflow.new.TaintTracking
 
-/**
- * Provides a taint-tracking configuration for detecting "SQL injection" vulnerabilities.
- */
-module SqlInjection {
-  import SqlInjectionCustomizations::SqlInjection
-
-  /**
-   * A taint-tracking configuration for detecting "SQL injection" vulnerabilities.
-   */
-  class Configuration extends TaintTracking::Configuration {
-    Configuration() { this = "SqlInjection" }
-
-    override predicate isSource(DataFlow::Node source) { source instanceof Source }
-
-    override predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
-
-    override predicate isSanitizer(DataFlow::Node node) { node instanceof Sanitizer }
-
-    override predicate isSanitizerGuard(DataFlow::BarrierGuard guard) {
-      guard instanceof SanitizerGuard
-    }
-  }
+/** DEPRECATED. Import `SqlInjectionQuery` instead. */
+deprecated module SqlInjection {
+  import SqlInjectionQuery // ignore-query-import
 }
 
-/**
- * DEPRECATED: Don't extend this class for customization, since this will lead to bad
- * performance, instead use the new `SqlInjectionCustomizations.qll` file, and extend
- * its' classes.
- */
+/** DEPRECATED. Import `SqlInjectionQuery` instead. */
 deprecated class SqlInjectionConfiguration = SqlInjection::Configuration;
 
-/** DEPRECATED: Alias for SqlInjectionConfiguration */
+/** DEPRECATED. Import `SqlInjectionQuery` instead. */
 deprecated class SQLInjectionConfiguration = SqlInjectionConfiguration;

python/ql/lib/semmle/python/security/dataflow/SqlInjectionQuery.qll

@@ -0,0 +1,29 @@
+/**
+ * Provides a taint-tracking configuration for detecting "SQL injection" vulnerabilities.
+ *
+ * Note, for performance reasons: only import this file if
+ * `SqlInjection::Configuration` is needed, otherwise
+ * `SqlInjectionCustomizations` should be imported instead.
+ */
+
+private import python
+import semmle.python.dataflow.new.DataFlow
+import semmle.python.dataflow.new.TaintTracking
+import SqlInjectionCustomizations::SqlInjection
+
+/**
+ * A taint-tracking configuration for detecting "SQL injection" vulnerabilities.
+ */
+class Configuration extends TaintTracking::Configuration {
+  Configuration() { this = "SqlInjection" }
+
+  override predicate isSource(DataFlow::Node source) { source instanceof Source }
+
+  override predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
+
+  override predicate isSanitizer(DataFlow::Node node) { node instanceof Sanitizer }
+
+  override predicate isSanitizerGuard(DataFlow::BarrierGuard guard) {
+    guard instanceof SanitizerGuard
+  }
+}

python/ql/lib/semmle/python/security/dataflow/StackTraceExposure.qll

@@ -1,51 +1,13 @@
-/**
- * Provides a taint-tracking configuration for detecting "stack trace exposure" vulnerabilities.
- *
- * Note, for performance reasons: only import this file if
- * `StackTraceExposure::Configuration` is needed, otherwise
- * `StackTraceExposureCustomizations` should be imported instead.
- */
+/** DEPRECATED. Import `StackTraceExposureQuery` instead. */
 
 private import python
 import semmle.python.dataflow.new.DataFlow
 import semmle.python.dataflow.new.TaintTracking
 
-/**
- * Provides a taint-tracking configuration for detecting "stack trace exposure" vulnerabilities.
- */
-module StackTraceExposure {
-  import StackTraceExposureCustomizations::StackTraceExposure
-
-  /**
-   * A taint-tracking configuration for detecting "stack trace exposure" vulnerabilities.
-   */
-  class Configuration extends TaintTracking::Configuration {
-    Configuration() { this = "StackTraceExposure" }
-
-    override predicate isSource(DataFlow::Node source) { source instanceof Source }
-
-    override predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
-
-    override predicate isSanitizer(DataFlow::Node node) { node instanceof Sanitizer }
-
-    override predicate isSanitizerGuard(DataFlow::BarrierGuard guard) {
-      guard instanceof SanitizerGuard
-    }
-
-    // A stack trace is accessible as the `__traceback__` attribute of a caught exception.
-    //  seehttps://docs.python.org/3/reference/datamodel.html#traceback-objects
-    override predicate isAdditionalTaintStep(DataFlow::Node nodeFrom, DataFlow::Node nodeTo) {
-      exists(DataFlow::AttrRead attr | attr.getAttributeName() = "__traceback__" |
-        nodeFrom = attr.getObject() and
-        nodeTo = attr
-      )
-    }
-  }
+/** DEPRECATED. Import `StackTraceExposureQuery` instead. */
+deprecated module StackTraceExposure {
+  import StackTraceExposureQuery // ignore-query-import
 }
 
-/**
- * DEPRECATED: Don't extend this class for customization, since this will lead to bad
- * performance, instead use the new `StackTraceExposureCustomizations.qll` file, and extend
- * its' classes.
- */
+/** DEPRECATED. Import `StackTraceExposureQuery` instead. */
 deprecated class StackTraceExposureConfiguration = StackTraceExposure::Configuration;

python/ql/lib/semmle/python/security/dataflow/StackTraceExposureQuery.qll

@@ -0,0 +1,38 @@
+/**
+ * Provides a taint-tracking configuration for detecting "stack trace exposure" vulnerabilities.
+ *
+ * Note, for performance reasons: only import this file if
+ * `StackTraceExposure::Configuration` is needed, otherwise
+ * `StackTraceExposureCustomizations` should be imported instead.
+ */
+
+private import python
+import semmle.python.dataflow.new.DataFlow
+import semmle.python.dataflow.new.TaintTracking
+import StackTraceExposureCustomizations::StackTraceExposure
+
+/**
+ * A taint-tracking configuration for detecting "stack trace exposure" vulnerabilities.
+ */
+class Configuration extends TaintTracking::Configuration {
+  Configuration() { this = "StackTraceExposure" }
+
+  override predicate isSource(DataFlow::Node source) { source instanceof Source }
+
+  override predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
+
+  override predicate isSanitizer(DataFlow::Node node) { node instanceof Sanitizer }
+
+  override predicate isSanitizerGuard(DataFlow::BarrierGuard guard) {
+    guard instanceof SanitizerGuard
+  }
+
+  // A stack trace is accessible as the `__traceback__` attribute of a caught exception.
+  //  seehttps://docs.python.org/3/reference/datamodel.html#traceback-objects
+  override predicate isAdditionalTaintStep(DataFlow::Node nodeFrom, DataFlow::Node nodeTo) {
+    exists(DataFlow::AttrRead attr | attr.getAttributeName() = "__traceback__" |
+      nodeFrom = attr.getObject() and
+      nodeTo = attr
+    )
+  }
+}

python/ql/lib/semmle/python/security/dataflow/UnsafeDeserialization.qll

@@ -1,42 +1,13 @@
-/**
- * Provides a taint-tracking configuration for detecting "code execution from deserialization" vulnerabilities.
- *
- * Note, for performance reasons: only import this file if
- * `UnsafeDeserialization::Configuration` is needed, otherwise
- * `UnsafeDeserializationCustomizations` should be imported instead.
- */
+/** DEPRECATED. Import `UnsafeDeserializationQuery` instead. */
 
 private import python
 import semmle.python.dataflow.new.DataFlow
 import semmle.python.dataflow.new.TaintTracking
 
-/**
- * Provides a taint-tracking configuration for detecting "code execution from deserialization" vulnerabilities.
- */
-module UnsafeDeserialization {
-  import UnsafeDeserializationCustomizations::UnsafeDeserialization
-
-  /**
-   * A taint-tracking configuration for detecting "code execution from deserialization" vulnerabilities.
-   */
-  class Configuration extends TaintTracking::Configuration {
-    Configuration() { this = "UnsafeDeserialization" }
-
-    override predicate isSource(DataFlow::Node source) { source instanceof Source }
-
-    override predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
-
-    override predicate isSanitizer(DataFlow::Node node) { node instanceof Sanitizer }
-
-    override predicate isSanitizerGuard(DataFlow::BarrierGuard guard) {
-      guard instanceof SanitizerGuard
-    }
-  }
+/** DEPRECATED. Import `UnsafeDeserializationQuery` instead. */
+deprecated module UnsafeDeserialization {
+  import UnsafeDeserializationQuery // ignore-query-import
 }
 
-/**
- * DEPRECATED: Don't extend this class for customization, since this will lead to bad
- * performance, instead use the new `UnsafeDeserializationCustomizations.qll` file, and extend
- * its' classes.
- */
+/** DEPRECATED. Import `UnsafeDeserializationQuery` instead. */
 deprecated class UnsafeDeserializationConfiguration = UnsafeDeserialization::Configuration;

python/ql/lib/semmle/python/security/dataflow/UnsafeDeserializationQuery.qll

@@ -0,0 +1,29 @@
+/**
+ * Provides a taint-tracking configuration for detecting "code execution from deserialization" vulnerabilities.
+ *
+ * Note, for performance reasons: only import this file if
+ * `UnsafeDeserialization::Configuration` is needed, otherwise
+ * `UnsafeDeserializationCustomizations` should be imported instead.
+ */
+
+private import python
+import semmle.python.dataflow.new.DataFlow
+import semmle.python.dataflow.new.TaintTracking
+import UnsafeDeserializationCustomizations::UnsafeDeserialization
+
+/**
+ * A taint-tracking configuration for detecting "code execution from deserialization" vulnerabilities.
+ */
+class Configuration extends TaintTracking::Configuration {
+  Configuration() { this = "UnsafeDeserialization" }
+
+  override predicate isSource(DataFlow::Node source) { source instanceof Source }
+
+  override predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
+
+  override predicate isSanitizer(DataFlow::Node node) { node instanceof Sanitizer }
+
+  override predicate isSanitizerGuard(DataFlow::BarrierGuard guard) {
+    guard instanceof SanitizerGuard
+  }
+}

python/ql/lib/semmle/python/security/dataflow/UrlRedirect.qll

@@ -1,42 +1,13 @@
-/**
- * Provides a taint-tracking configuration for detecting "URL redirection" vulnerabilities.
- *
- * Note, for performance reasons: only import this file if
- * `UrlRedirect::Configuration` is needed, otherwise
- * `UrlRedirectCustomizations` should be imported instead.
- */
+/** DEPRECATED. Import `UrlRedirectQuery` instead. */
 
 private import python
 import semmle.python.dataflow.new.DataFlow
 import semmle.python.dataflow.new.TaintTracking
 
-/**
- * Provides a taint-tracking configuration for detecting "URL redirection" vulnerabilities.
- */
-module UrlRedirect {
-  import UrlRedirectCustomizations::UrlRedirect
-
-  /**
-   * A taint-tracking configuration for detecting "URL redirection" vulnerabilities.
-   */
-  class Configuration extends TaintTracking::Configuration {
-    Configuration() { this = "UrlRedirect" }
-
-    override predicate isSource(DataFlow::Node source) { source instanceof Source }
-
-    override predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
-
-    override predicate isSanitizer(DataFlow::Node node) { node instanceof Sanitizer }
-
-    override predicate isSanitizerGuard(DataFlow::BarrierGuard guard) {
-      guard instanceof SanitizerGuard
-    }
-  }
+/** DEPRECATED. Import `UrlRedirectQuery` instead. */
+deprecated module UrlRedirect {
+  import UrlRedirectQuery // ignore-query-import
 }
 
-/**
- * DEPRECATED: Don't extend this class for customization, since this will lead to bad
- * performance, instead use the new `UrlRedirectCustomizations.qll` file, and extend
- * its' classes.
- */
+/** DEPRECATED. Import `UrlRedirectQuery` instead. */
 deprecated class UrlRedirectConfiguration = UrlRedirect::Configuration;

python/ql/lib/semmle/python/security/dataflow/UrlRedirectQuery.qll

@@ -0,0 +1,29 @@
+/**
+ * Provides a taint-tracking configuration for detecting "URL redirection" vulnerabilities.
+ *
+ * Note, for performance reasons: only import this file if
+ * `UrlRedirect::Configuration` is needed, otherwise
+ * `UrlRedirectCustomizations` should be imported instead.
+ */
+
+private import python
+import semmle.python.dataflow.new.DataFlow
+import semmle.python.dataflow.new.TaintTracking
+import UrlRedirectCustomizations::UrlRedirect
+
+/**
+ * A taint-tracking configuration for detecting "URL redirection" vulnerabilities.
+ */
+class Configuration extends TaintTracking::Configuration {
+  Configuration() { this = "UrlRedirect" }
+
+  override predicate isSource(DataFlow::Node source) { source instanceof Source }
+
+  override predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
+
+  override predicate isSanitizer(DataFlow::Node node) { node instanceof Sanitizer }
+
+  override predicate isSanitizerGuard(DataFlow::BarrierGuard guard) {
+    guard instanceof SanitizerGuard
+  }
+}

python/ql/lib/semmle/python/security/dataflow/WeakSensitiveDataHashing.qll

@@ -1,11 +1,4 @@
-/**
- * Provides a taint-tracking configuration for detecting use of a broken or weak
- * cryptographic hashing algorithm on sensitive data.
- *
- * Note, for performance reasons: only import this file if
- * `WeakSensitiveDataHashing::Configuration` is needed, otherwise
- * `WeakSensitiveDataHashingCustomizations` should be imported instead.
- */
+/** DEPRECATED. Import `WeakSensitiveDataHashingQuery` instead. */
 
 private import python
 private import semmle.python.dataflow.new.DataFlow
@@ -15,69 +8,12 @@ private import semmle.python.dataflow.new.RemoteFlowSources
 private import semmle.python.dataflow.new.BarrierGuards
 private import semmle.python.dataflow.new.SensitiveDataSources
 
-/**
- * Provides a taint-tracking configuration for detecting use of a broken or weak
- * cryptographic hash function on sensitive data, that does NOT require a
- * computationally expensive hash function.
- */
-module NormalHashFunction {
-  import WeakSensitiveDataHashingCustomizations::NormalHashFunction
-
-  /**
-   * A taint-tracking configuration for detecting use of a broken or weak
-   * cryptographic hashing algorithm on sensitive data.
-   */
-  class Configuration extends TaintTracking::Configuration {
-    Configuration() { this = "NormalHashFunction" }
-
-    override predicate isSource(DataFlow::Node source) { source instanceof Source }
-
-    override predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
-
-    override predicate isSanitizer(DataFlow::Node node) {
-      super.isSanitizer(node)
-      or
-      node instanceof Sanitizer
-    }
-
-    override predicate isAdditionalTaintStep(DataFlow::Node node1, DataFlow::Node node2) {
-      sensitiveDataExtraStepForCalls(node1, node2)
-    }
-  }
+/** DEPRECATED. Import `WeakSensitiveDataHashingQuery` instead. */
+deprecated module NormalHashFunction {
+  import WeakSensitiveDataHashingQuery::NormalHashFunction // ignore-query-import
 }
 
-/**
- * Provides a taint-tracking configuration for detecting use of a broken or weak
- * cryptographic hashing algorithm on passwords.
- *
- * Passwords has stricter requirements on the hashing algorithm used (must be
- * computationally expensive to prevent brute-force attacks).
- */
-module ComputationallyExpensiveHashFunction {
-  import WeakSensitiveDataHashingCustomizations::ComputationallyExpensiveHashFunction
-
-  /**
-   * A taint-tracking configuration for detecting use of a broken or weak
-   * cryptographic hashing algorithm on passwords.
-   *
-   * Passwords has stricter requirements on the hashing algorithm used (must be
-   * computationally expensive to prevent brute-force attacks).
-   */
-  class Configuration extends TaintTracking::Configuration {
-    Configuration() { this = "ComputationallyExpensiveHashFunction" }
-
-    override predicate isSource(DataFlow::Node source) { source instanceof Source }
-
-    override predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
-
-    override predicate isSanitizer(DataFlow::Node node) {
-      super.isSanitizer(node)
-      or
-      node instanceof Sanitizer
-    }
-
-    override predicate isAdditionalTaintStep(DataFlow::Node node1, DataFlow::Node node2) {
-      sensitiveDataExtraStepForCalls(node1, node2)
-    }
-  }
+/** DEPRECATED. Import `WeakSensitiveDataHashingQuery` instead. */
+deprecated module ComputationallyExpensiveHashFunction {
+  import WeakSensitiveDataHashingQuery::ComputationallyExpensiveHashFunction // ignore-query-import
 }

python/ql/lib/semmle/python/security/dataflow/WeakSensitiveDataHashingQuery.qll

@@ -0,0 +1,83 @@
+/**
+ * Provides a taint-tracking configuration for detecting use of a broken or weak
+ * cryptographic hashing algorithm on sensitive data.
+ *
+ * Note, for performance reasons: only import this file if
+ * `WeakSensitiveDataHashing::Configuration` is needed, otherwise
+ * `WeakSensitiveDataHashingCustomizations` should be imported instead.
+ */
+
+private import python
+private import semmle.python.dataflow.new.DataFlow
+private import semmle.python.dataflow.new.TaintTracking
+private import semmle.python.Concepts
+private import semmle.python.dataflow.new.RemoteFlowSources
+private import semmle.python.dataflow.new.BarrierGuards
+private import semmle.python.dataflow.new.SensitiveDataSources
+
+/**
+ * Provides a taint-tracking configuration for detecting use of a broken or weak
+ * cryptographic hash function on sensitive data, that does NOT require a
+ * computationally expensive hash function.
+ */
+module NormalHashFunction {
+  import WeakSensitiveDataHashingCustomizations::NormalHashFunction
+
+  /**
+   * A taint-tracking configuration for detecting use of a broken or weak
+   * cryptographic hashing algorithm on sensitive data.
+   */
+  class Configuration extends TaintTracking::Configuration {
+    Configuration() { this = "NormalHashFunction" }
+
+    override predicate isSource(DataFlow::Node source) { source instanceof Source }
+
+    override predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
+
+    override predicate isSanitizer(DataFlow::Node node) {
+      super.isSanitizer(node)
+      or
+      node instanceof Sanitizer
+    }
+
+    override predicate isAdditionalTaintStep(DataFlow::Node node1, DataFlow::Node node2) {
+      sensitiveDataExtraStepForCalls(node1, node2)
+    }
+  }
+}
+
+/**
+ * Provides a taint-tracking configuration for detecting use of a broken or weak
+ * cryptographic hashing algorithm on passwords.
+ *
+ * Passwords has stricter requirements on the hashing algorithm used (must be
+ * computationally expensive to prevent brute-force attacks).
+ */
+module ComputationallyExpensiveHashFunction {
+  import WeakSensitiveDataHashingCustomizations::ComputationallyExpensiveHashFunction
+
+  /**
+   * A taint-tracking configuration for detecting use of a broken or weak
+   * cryptographic hashing algorithm on passwords.
+   *
+   * Passwords has stricter requirements on the hashing algorithm used (must be
+   * computationally expensive to prevent brute-force attacks).
+   */
+  class Configuration extends TaintTracking::Configuration {
+    Configuration() { this = "ComputationallyExpensiveHashFunction" }
+
+    override predicate isSource(DataFlow::Node source) { source instanceof Source }
+
+    override predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
+
+    override predicate isSanitizer(DataFlow::Node node) {
+      super.isSanitizer(node)
+      or
+      node instanceof Sanitizer
+    }
+
+    override predicate isAdditionalTaintStep(DataFlow::Node node1, DataFlow::Node node2) {
+      sensitiveDataExtraStepForCalls(node1, node2)
+    }
+  }
+}

python/ql/lib/semmle/python/security/dataflow/XpathInjection.qll

@@ -1,35 +1,10 @@
-/**
- * Provides a taint-tracking configuration for detecting "Xpath Injection" vulnerabilities.
- *
- * Note, for performance reasons: only import this file if
- * `XpathInjection::Configuration` is needed, otherwise
- * `XpathInjectionCustomizations` should be imported instead.
- */
+/** DEPRECATED. Import `XpathInjectionQuery` instead. */
 
 private import python
 import semmle.python.dataflow.new.DataFlow
 import semmle.python.dataflow.new.TaintTracking
 
-/**
- * Provides a taint-tracking configuration for detecting "Xpath Injection" vulnerabilities.
- */
-module XpathInjection {
-  import XpathInjectionCustomizations::XpathInjection
-
-  /**
-   * A taint-tracking configuration for detecting "Xpath Injection" vulnerabilities.
-   */
-  class Configuration extends TaintTracking::Configuration {
-    Configuration() { this = "Xpath Injection" }
-
-    override predicate isSource(DataFlow::Node source) { source instanceof Source }
-
-    override predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
-
-    override predicate isSanitizer(DataFlow::Node node) { node instanceof Sanitizer }
-
-    override predicate isSanitizerGuard(DataFlow::BarrierGuard guard) {
-      guard instanceof SanitizerGuard
-    }
-  }
+/** DEPRECATED. Import `XpathInjectionQuery` instead. */
+deprecated module XpathInjection {
+  import XpathInjectionQuery // ignore-query-import
 }

python/ql/lib/semmle/python/security/dataflow/XpathInjectionQuery.qll

@@ -0,0 +1,29 @@
+/**
+ * Provides a taint-tracking configuration for detecting "Xpath Injection" vulnerabilities.
+ *
+ * Note, for performance reasons: only import this file if
+ * `XpathInjection::Configuration` is needed, otherwise
+ * `XpathInjectionCustomizations` should be imported instead.
+ */
+
+private import python
+import semmle.python.dataflow.new.DataFlow
+import semmle.python.dataflow.new.TaintTracking
+import XpathInjectionCustomizations::XpathInjection
+
+/**
+ * A taint-tracking configuration for detecting "Xpath Injection" vulnerabilities.
+ */
+class Configuration extends TaintTracking::Configuration {
+  Configuration() { this = "Xpath Injection" }
+
+  override predicate isSource(DataFlow::Node source) { source instanceof Source }
+
+  override predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
+
+  override predicate isSanitizer(DataFlow::Node node) { node instanceof Sanitizer }
+
+  override predicate isSanitizerGuard(DataFlow::BarrierGuard guard) {
+    guard instanceof SanitizerGuard
+  }
+}

python/ql/lib/semmle/python/security/performance/ExponentialBackTracking.qll

@@ -279,17 +279,6 @@ private class Trace extends TTrace {
   }
 }
 
-/**
- * Gets a string corresponding to the trace `t`.
- */
-private string concretise(Trace t) {
-  t = Nil() and result = ""
-  or
-  exists(InputSymbol s1, InputSymbol s2, Trace rest | t = Step(s1, s2, rest) |
-    result = concretise(rest) + intersect(s1, s2)
-  )
-}
-
 /**
  * Holds if `r` is reachable from `(fork, fork)` under input `w`, and there is
  * a path from `r` back to `(fork, fork)` with `rem` steps.
@@ -321,14 +310,54 @@ private StatePair getAForkPair(State fork) {
   result = MkStatePair(epsilonPred*(fork), epsilonPred*(fork))
 }
 
+private predicate hasSuffix(Trace suffix, Trace t, int i) {
+  // Declaring `t` to be a `RelevantTrace` currently causes a redundant check in the
+  // recursive case, so instead we check it explicitly here.
+  t instanceof RelevantTrace and
+  i = 0 and
+  suffix = t
+  or
+  hasSuffix(Step(_, _, suffix), t, i - 1)
+}
+
+pragma[noinline]
+private predicate hasTuple(InputSymbol s1, InputSymbol s2, Trace t, int i) {
+  hasSuffix(Step(s1, s2, _), t, i)
+}
+
+private class RelevantTrace extends Trace, Step {
+  RelevantTrace() {
+    exists(State fork, StatePair q |
+      isReachableFromFork(fork, q, this, _) and
+      q = getAForkPair(fork)
+    )
+  }
+
+  pragma[noinline]
+  private string intersect(int i) {
+    exists(InputSymbol s1, InputSymbol s2 |
+      hasTuple(s1, s2, this, i) and
+      result = intersect(s1, s2)
+    )
+  }
+
+  /** Gets a string corresponding to this trace. */
+  // the pragma is needed for the case where `intersect(s1, s2)` has multiple values,
+  // not for recursion
+  language[monotonicAggregates]
+  string concretise() {
+    result = strictconcat(int i | hasTuple(_, _, this, i) | this.intersect(i) order by i desc)
+  }
+}
+
 /**
  * Holds if `fork` is a pumpable fork with word `w`.
  */
 private predicate isPumpable(State fork, string w) {
-  exists(StatePair q, Trace t |
+  exists(StatePair q, RelevantTrace t |
     isReachableFromFork(fork, q, t, _) and
     q = getAForkPair(fork) and
-    w = concretise(t)
+    w = t.concretise()
   )
 }
 

python/ql/lib/semmle/python/security/performance/ReDoSUtil.qll

@@ -12,7 +12,7 @@
  * states that will cause backtracking (a rejecting suffix exists).
  */
 
-import RegExpTreeView
+import ReDoSUtilSpecific
 
 /**
  * A configuration for which parts of a regular expression should be considered relevant for
@@ -32,7 +32,7 @@ abstract class ReDoSConfiguration extends string {
 }
 
 /**
- * Holds if repeating `pump' starting at `state` is a candidate for causing backtracking.
+ * Holds if repeating `pump` starting at `state` is a candidate for causing backtracking.
  * No check whether a rejected suffix exists has been made.
  */
 private predicate isReDoSCandidate(State state, string pump) {

python/ql/lib/semmle/python/security/performance/ReDoSUtilSpecific.qll

@@ -1,5 +1,5 @@
 /**
- * This module should provide a class hierarchy corresponding to a parse tree of regular expressions.
+ * Provides Python-specific definitions for use in the ReDoSUtil module.
  */
 
 import python

python/ql/lib/semmle/python/security/performance/SuperlinearBackTracking.qll

@@ -254,17 +254,6 @@ class Trace extends TTrace {
   }
 }
 
-/**
- * Gets a string corresponding to the trace `t`.
- */
-string concretise(Trace t) {
-  t = Nil() and result = ""
-  or
-  exists(InputSymbol s1, InputSymbol s2, InputSymbol s3, Trace rest | t = Step(s1, s2, s3, rest) |
-    result = concretise(rest) + getAThreewayIntersect(s1, s2, s3)
-  )
-}
-
 /**
  * Holds if there exists a transition from `r` to `q` in the product automaton.
  * Notice that the arguments are flipped, and thus the direction is backwards.
@@ -332,6 +321,51 @@ StateTuple getAnEndTuple(State pivot, State succ) {
   result = MkStateTuple(pivot, succ, succ)
 }
 
+private predicate hasSuffix(Trace suffix, Trace t, int i) {
+  // Declaring `t` to be a `RelevantTrace` currently causes a redundant check in the
+  // recursive case, so instead we check it explicitly here.
+  t instanceof RelevantTrace and
+  i = 0 and
+  suffix = t
+  or
+  hasSuffix(Step(_, _, _, suffix), t, i - 1)
+}
+
+pragma[noinline]
+private predicate hasTuple(InputSymbol s1, InputSymbol s2, InputSymbol s3, Trace t, int i) {
+  hasSuffix(Step(s1, s2, s3, _), t, i)
+}
+
+private class RelevantTrace extends Trace, Step {
+  RelevantTrace() {
+    exists(State pivot, State succ, StateTuple q |
+      isReachableFromStartTuple(pivot, succ, q, this, _) and
+      q = getAnEndTuple(pivot, succ)
+    )
+  }
+
+  pragma[noinline]
+  private string getAThreewayIntersect(int i) {
+    exists(InputSymbol s1, InputSymbol s2, InputSymbol s3 |
+      hasTuple(s1, s2, s3, this, i) and
+      result = getAThreewayIntersect(s1, s2, s3)
+    )
+  }
+
+  /** Gets a string corresponding to this trace. */
+  // the pragma is needed for the case where `getAThreewayIntersect(s1, s2, s3)` has multiple values,
+  // not for recursion
+  language[monotonicAggregates]
+  string concretise() {
+    result =
+      strictconcat(int i |
+        hasTuple(_, _, _, this, i)
+      |
+        this.getAThreewayIntersect(i) order by i desc
+      )
+  }
+}
+
 /**
  * Holds if matching repetitions of `pump` can:
  * 1) Transition from `pivot` back to `pivot`.
@@ -345,10 +379,10 @@ StateTuple getAnEndTuple(State pivot, State succ) {
  * available in the `hasReDoSResult` predicate.
  */
 predicate isPumpable(State pivot, State succ, string pump) {
-  exists(StateTuple q, Trace t |
+  exists(StateTuple q, RelevantTrace t |
     isReachableFromStartTuple(pivot, succ, q, t, _) and
     q = getAnEndTuple(pivot, succ) and
-    pump = concretise(t)
+    pump = t.concretise()
   )
 }
 

python/ql/lib/semmle/python/types/Object.qll

@@ -2,9 +2,11 @@ import python
 private import semmle.python.objects.ObjectAPI
 private import semmle.python.objects.ObjectInternal
 private import semmle.python.types.Builtins
+private import semmle.python.internal.CachedStages
 
 cached
 private predicate is_an_object(@py_object obj) {
+  Stages::DataFlow::ref() and
   /* CFG nodes for numeric literals, all of which have a @py_cobject for the value of that literal */
   obj instanceof ControlFlowNode and
   not obj.(ControlFlowNode).getNode() instanceof IntegerLiteral and
@@ -73,9 +75,11 @@ class Object extends @py_object {
    * For more information, see
    * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
    */
+  cached
   predicate hasLocationInfo(
     string filepath, int startline, int startcolumn, int endline, int endcolumn
   ) {
+    Stages::DataFlow::ref() and
     this.hasOrigin() and
     this.getOrigin()
         .getLocation()
@@ -93,7 +97,9 @@ class Object extends @py_object {
   Builtin asBuiltin() { result = this }
 
   /** Gets a textual representation of this element. */
+  cached
   string toString() {
+    Stages::DataFlow::ref() and
     not this = undefinedVariable() and
     not this = unknownValue() and
     exists(ClassObject type | type.asBuiltin() = this.asBuiltin().getClass() |

python/ql/src/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.0.13
+
+## 0.0.12
+
 ## 0.0.11
 
 ### New Queries

python/ql/src/Security/CWE-020/HostnameRegexpSpecific.qll

@@ -1,3 +1,3 @@
-import semmle.python.security.performance.RegExpTreeView
+import semmle.python.RegexTreeView
 import semmle.python.dataflow.new.DataFlow
 import semmle.python.dataflow.new.Regexp

python/ql/src/Security/CWE-022/PathInjection.ql

@@ -17,10 +17,10 @@
  */
 
 import python
-import semmle.python.security.dataflow.PathInjection
+import semmle.python.security.dataflow.PathInjectionQuery
 import DataFlow::PathGraph
 
-from PathInjection::Configuration config, DataFlow::PathNode source, DataFlow::PathNode sink
+from Configuration config, DataFlow::PathNode source, DataFlow::PathNode sink
 where config.hasFlowPath(source, sink)
 select sink.getNode(), source, sink, "This path depends on $@.", source.getNode(),
   "a user-provided value"

python/ql/src/Security/CWE-078/CommandInjection.ql

@@ -15,10 +15,10 @@
  */
 
 import python
-import semmle.python.security.dataflow.CommandInjection
+import semmle.python.security.dataflow.CommandInjectionQuery
 import DataFlow::PathGraph
 
-from CommandInjection::Configuration config, DataFlow::PathNode source, DataFlow::PathNode sink
+from Configuration config, DataFlow::PathNode source, DataFlow::PathNode sink
 where config.hasFlowPath(source, sink)
 select sink.getNode(), source, sink, "This command depends on $@.", source.getNode(),
   "a user-provided value"

python/ql/src/Security/CWE-079/ReflectedXss.ql

@@ -14,10 +14,10 @@
  */
 
 import python
-import semmle.python.security.dataflow.ReflectedXSS
+import semmle.python.security.dataflow.ReflectedXssQuery
 import DataFlow::PathGraph
 
-from ReflectedXss::Configuration config, DataFlow::PathNode source, DataFlow::PathNode sink
+from Configuration config, DataFlow::PathNode source, DataFlow::PathNode sink
 where config.hasFlowPath(source, sink)
 select sink.getNode(), source, sink, "Cross-site scripting vulnerability due to $@.",
   source.getNode(), "a user-provided value"

python/ql/src/Security/CWE-089/SqlInjection.ql

@@ -12,10 +12,10 @@
  */
 
 import python
-import semmle.python.security.dataflow.SqlInjection
+import semmle.python.security.dataflow.SqlInjectionQuery
 import DataFlow::PathGraph
 
-from SqlInjection::Configuration config, DataFlow::PathNode source, DataFlow::PathNode sink
+from Configuration config, DataFlow::PathNode source, DataFlow::PathNode sink
 where config.hasFlowPath(source, sink)
 select sink.getNode(), source, sink, "This SQL query depends on $@.", source.getNode(),
   "a user-provided value"

python/ql/src/Security/CWE-090/LdapInjection.ql

@@ -13,15 +13,15 @@
 
 // Determine precision above
 import python
-import semmle.python.security.dataflow.LdapInjection
+import semmle.python.security.dataflow.LdapInjectionQuery
 import DataFlow::PathGraph
 
 from DataFlow::PathNode source, DataFlow::PathNode sink, string parameterName
 where
-  any(LdapInjection::DnConfiguration dnConfig).hasFlowPath(source, sink) and
+  any(DnConfiguration dnConfig).hasFlowPath(source, sink) and
   parameterName = "DN"
   or
-  any(LdapInjection::FilterConfiguration filterConfig).hasFlowPath(source, sink) and
+  any(FilterConfiguration filterConfig).hasFlowPath(source, sink) and
   parameterName = "filter"
 select sink.getNode(), source, sink,
   "$@ LDAP query parameter (" + parameterName + ") comes from $@.", sink.getNode(), "This",