Merge branch 'main' into turbo/experimental/combined

python/config/suites/lgtm/python-queries-lgtm

@@ -1,6 +1,6 @@
 + semmlecode-python-queries/analysis/Definitions.ql
     @_namespace com.lgtm/python-queries
-+ semmlecode-python-queries/analysis/AlertSuppression.ql
++ semmlecode-python-queries/AlertSuppression.ql
     @_namespace com.lgtm/python-queries
 + semmlecode-python-queries/Filters/ClassifyFiles.ql
     @_namespace com.lgtm/python-queries

python/downgrades/47e552c4357a04c5735355fad818630daee4a5ac/py_stmts.ql

@@ -0,0 +1,31 @@
+// First we need to wrap some database types
+class Stmt_ extends @py_stmt {
+  string toString() { result = "Stmt" }
+}
+
+class StmtList_ extends @py_stmt_list {
+  string toString() { result = "StmtList" }
+}
+
+/**
+ * New kinds have been inserted such that
+ * `@py_Exec` which used to have index 7 now has index 8.
+ * Entries with lower indices are unchanged.
+ */
+bindingset[new_index]
+int old_index(int new_index) {
+  not new_index = 7 and
+  if new_index < 7 then result = new_index else result + (8 - 7) = new_index
+}
+
+// The schema for py_stmts is:
+//
+// py_stmts(unique int id : @py_stmt,
+//   int kind: int ref,
+//   int parent : @py_stmt_list ref,
+//   int idx : int ref);
+from Stmt_ expr, int new_kind, StmtList_ parent, int idx, int old_kind
+where
+  py_stmts(expr, new_kind, parent, idx) and
+  old_kind = old_index(new_kind)
+select expr, old_kind, parent, idx

python/downgrades/47e552c4357a04c5735355fad818630daee4a5ac/upgrade.properties

@@ -0,0 +1,3 @@
+description: Add support for `except*`
+compatibility: backwards
+py_stmts.rel: run py_stmts.qlo

python/downgrades/503c0516fba2e5da9570f00eb34ef43025ecb8fb/py_exprs.ql

@@ -23,10 +23,17 @@ class ExprParent_ extends @py_expr_parent {
  * New kinds have been inserted such that
  * `@py_Name` which used to have index 18 now has index 19.
  * Entries with lower indices are unchanged.
+ *
+ * Note that if `18 <= new_index < 19`, it does not correspond
+ * to an old index.
  */
 bindingset[new_index]
 int old_index(int new_index) {
-  if new_index < 18 then result = new_index else result + (19 - 18) = new_index
+  // before inserted range
+  new_index < 18 and result = new_index
+  or
+  // after inserted range
+  new_index >= 19 and result + (19 - 18) = new_index
 }
 
 // The schema for py_exprs is:

python/downgrades/503c0516fba2e5da9570f00eb34ef43025ecb8fb/py_stmts.ql

@@ -23,10 +23,17 @@ class StmtList_ extends @py_stmt_list {
  * New kinds have been inserted such that
  * `@py_Nonlocal` which used to have index 14 now has index 16.
  * Entries with lower indices are unchanged.
+ *
+ * Note that if `14 <= new_index < 16`, it does not correspond
+ * to an old index.
  */
 bindingset[new_index]
 int old_index(int new_index) {
-  if new_index < 14 then result = new_index else result + (16 - 14) = new_index
+  // before inserted range
+  new_index < 14 and result = new_index
+  or
+  // after inserted range
+  new_index >= 16 and result + (16 - 14) = new_index
 }
 
 // The schema for py_stmts is:

python/ql/lib/CHANGELOG.md

@@ -1,3 +1,27 @@
+## 0.7.0
+
+### Major Analysis Improvements
+
+* The _PAM authorization bypass due to incorrect usage_ (`py/pam-auth-bypass`) query has been converted to a taint-tracking query, resulting in significantly fewer false positives.
+
+### Minor Analysis Improvements
+
+* Added `subprocess.getoutput` and `subprocess.getoutputstatus` as new command injection sinks for the StdLib.
+* The data-flow library has been rewritten to no longer rely on the points-to analysis in order to resolve references to modules. Improvements in the module resolution can lead to more results.
+* Deleted the deprecated `importNode` predicate from the `DataFlowUtil.qll` file.
+* Deleted the deprecated features from `PEP249.qll` that were not inside the `PEP249` module.
+* Deleted the deprecated `werkzeug` from the `Werkzeug` module in `Werkzeug.qll`.
+* Deleted the deprecated `methodResult` predicate from `PEP249::Cursor`.
+
+### Bug Fixes
+
+* `except*` is now supported.
+* The result of `Try.getAHandler` and `Try.getHandler(<index>)` is no longer of type `ExceptStmt`, as handlers may also be `ExceptGroupStmt`s (After Python 3.11 introduced PEP 654). Instead, it is of the new type `ExceptionHandler` of which `ExceptStmt` and `ExceptGroupStmt` are subtypes. To support selecting only one type of handler, `Try.getANormalHandler` and `Try.getAGroupHandler` have been added. Existing uses of `Try.getAHandler` for which it is important to select only normal handlers, will need to be updated to `Try.getANormalHandler`.
+
+## 0.6.6
+
+No user-facing changes.
+
 ## 0.6.5
 
 No user-facing changes.

python/ql/lib/analysis/DefinitionTracking.qll

@@ -221,10 +221,8 @@ pragma[noinline]
 private predicate module_and_name_for_import_star(
   ModuleObject mod, string name, ImportStarRefinement def
 ) {
-  exists(ImportStarNode im_star |
-    module_and_name_for_import_star_helper(mod, name, im_star, def) and
-    mod.exports(name)
-  )
+  module_and_name_for_import_star_helper(mod, name, _, def) and
+  mod.exports(name)
 }
 
 pragma[noinline]
@@ -484,9 +482,9 @@ class NiceLocationExpr extends Expr {
    */
   predicate hasLocationInfo(string f, int bl, int bc, int el, int ec) {
     /* Attribute location for x.y is that of 'y' so that url does not overlap with that of 'x' */
-    exists(int abl, int abc | this.(Attribute).getLocation().hasLocationInfo(f, abl, abc, el, ec) |
-      bl = el and bc = ec - this.(Attribute).getName().length() + 1
-    )
+    this.(Attribute).getLocation().hasLocationInfo(f, _, _, el, ec) and
+    bl = el and
+    bc = ec - this.(Attribute).getName().length() + 1
     or
     this.(Name).getLocation().hasLocationInfo(f, bl, bc, el, ec)
     or

python/ql/lib/change-notes/2022-11-17-deleted-deps.md

@@ -1,7 +0,0 @@
----
-category: minorAnalysis
----
-* Deleted the deprecated `importNode` predicate from the `DataFlowUtil.qll` file.
-* Deleted the deprecated features from `PEP249.qll` that were not inside the `PEP249` module.
-* Deleted the deprecated `werkzeug` from the `Werkzeug` module in `Werkzeug.qll`.
-* Deleted the deprecated `methodResult` predicate from `PEP249::Cursor`.

python/ql/lib/change-notes/2022-11-17-py-pam-improve.md

@@ -1,4 +0,0 @@
----
- category: majorAnalysis
----
-* The _PAM authorization bypass due to incorrect usage_ (`py/pam-auth-bypass`) query has been converted to a taint-tracking query, resulting in significantly fewer false positives.

python/ql/lib/change-notes/2022-11-21-module-resolution-rewrite.md

@@ -1,5 +0,0 @@
----
- category: minorAnalysis
----
- * The data-flow library has been rewritten to no longer rely on the points-to analysis in order to
-   resolve references to modules. Improvements in the module resolution can lead to more results.

python/ql/lib/change-notes/2022-12-03-new-stdlib-cmdi-sinks.md

@@ -1,5 +0,0 @@
----
-category: minorAnalysis
----
-- Added `subprocess.getoutput` and `subprocess.getoutputstatus` as new command injection sinks for the StdLib.
-

python/ql/lib/change-notes/released/0.6.6.md

@@ -0,0 +1,3 @@
+## 0.6.6
+
+No user-facing changes.

python/ql/lib/change-notes/released/0.7.0.md

@@ -0,0 +1,19 @@
+## 0.7.0
+
+### Major Analysis Improvements
+
+* The _PAM authorization bypass due to incorrect usage_ (`py/pam-auth-bypass`) query has been converted to a taint-tracking query, resulting in significantly fewer false positives.
+
+### Minor Analysis Improvements
+
+* Added `subprocess.getoutput` and `subprocess.getoutputstatus` as new command injection sinks for the StdLib.
+* The data-flow library has been rewritten to no longer rely on the points-to analysis in order to resolve references to modules. Improvements in the module resolution can lead to more results.
+* Deleted the deprecated `importNode` predicate from the `DataFlowUtil.qll` file.
+* Deleted the deprecated features from `PEP249.qll` that were not inside the `PEP249` module.
+* Deleted the deprecated `werkzeug` from the `Werkzeug` module in `Werkzeug.qll`.
+* Deleted the deprecated `methodResult` predicate from `PEP249::Cursor`.
+
+### Bug Fixes
+
+* `except*` is now supported.
+* The result of `Try.getAHandler` and `Try.getHandler(<index>)` is no longer of type `ExceptStmt`, as handlers may also be `ExceptGroupStmt`s (After Python 3.11 introduced PEP 654). Instead, it is of the new type `ExceptionHandler` of which `ExceptStmt` and `ExceptGroupStmt` are subtypes. To support selecting only one type of handler, `Try.getANormalHandler` and `Try.getAGroupHandler` have been added. Existing uses of `Try.getAHandler` for which it is important to select only normal handlers, will need to be updated to `Try.getANormalHandler`.

python/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.6.5
+lastReleaseVersion: 0.7.0

python/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/python-all
-version: 0.6.6-dev
+version: 0.7.1-dev
 groups: python
 dbscheme: semmlecode.python.dbscheme
 extractor: python
@@ -7,5 +7,6 @@ library: true
 upgrades: upgrades
 dependencies:
   codeql/regex: ${workspace}
+  codeql/tutorial: ${workspace}
 dataExtensions:
   - semmle/python/frameworks/**/model.yml

python/ql/lib/semmle/python/AstGenerated.qll

@@ -389,6 +389,26 @@ class Eq_ extends @py_Eq, Cmpop {
   override string toString() { result = "Eq" }
 }
 
+/** INTERNAL: See the class `ExceptGroupStmt` for further information. */
+class ExceptGroupStmt_ extends @py_ExceptGroupStmt, Stmt {
+  /** Gets the type of this except group block. */
+  Expr getType() { py_exprs(result, _, this, 1) }
+
+  /** Gets the name of this except group block. */
+  Expr getName() { py_exprs(result, _, this, 2) }
+
+  /** Gets the body of this except group block. */
+  StmtList getBody() { py_stmt_lists(result, this, 3) }
+
+  /** Gets the nth statement of this except group block. */
+  Stmt getStmt(int index) { result = this.getBody().getItem(index) }
+
+  /** Gets a statement of this except group block. */
+  Stmt getAStmt() { result = this.getBody().getAnItem() }
+
+  override string toString() { result = "ExceptGroupStmt" }
+}
+
 /** INTERNAL: See the class `ExceptStmt` for further information. */
 class ExceptStmt_ extends @py_ExceptStmt, Stmt {
   /** Gets the type of this except block. */

python/ql/lib/semmle/python/Flow.qll

@@ -385,9 +385,9 @@ class CallNode extends ControlFlowNode {
 
   /** Gets the flow node corresponding to an argument of the call corresponding to this flow node */
   ControlFlowNode getAnArg() {
-    exists(int n | result = this.getArg(n))
+    result = this.getArg(_)
     or
-    exists(string name | result = this.getArgByName(name))
+    result = this.getArgByName(_)
   }
 
   override Call getNode() { result = super.getNode() }

python/ql/lib/semmle/python/RegexTreeView.qll

@@ -102,6 +102,9 @@ module Impl implements RegexTreeViewSig {
     /** Gets the number of child terms. */
     int getNumChild() { result = count(this.getAChild()) }
 
+    /** Gets the last child term of this element. */
+    RegExpTerm getLastChild() { result = this.getChild(this.getNumChild() - 1) }
+
     /** Gets the associated regex. */
     abstract Regex getRegex();
   }
@@ -224,8 +227,8 @@ module Impl implements RegexTreeViewSig {
     predicate hasLocationInfo(
       string filepath, int startline, int startcolumn, int endline, int endcolumn
     ) {
-      exists(int re_start, int re_end |
-        re.getLocation().hasLocationInfo(filepath, startline, re_start, endline, re_end) and
+      exists(int re_start |
+        re.getLocation().hasLocationInfo(filepath, startline, re_start, endline, _) and
         startcolumn = re_start + start + 4 and
         endcolumn = re_start + end + 3
       )
@@ -454,7 +457,16 @@ module Impl implements RegexTreeViewSig {
     override string getPrimaryQLClass() { result = "RegExpAlt" }
   }
 
-  additional class RegExpCharEscape = RegExpEscape;
+  /**
+   * A character escape in a regular expression.
+   *
+   * Example:
+   *
+   * ```
+   * \.
+   * ```
+   */
+  class RegExpCharEscape = RegExpEscape;
 
   /**
    * An escaped regular expression term, that is, a regular expression
@@ -561,6 +573,13 @@ module Impl implements RegexTreeViewSig {
     RegExpWordBoundary() { this.getChar() = "\\b" }
   }
 
+  /**
+   * A non-word boundary, that is, a regular expression term of the form `\B`.
+   */
+  class RegExpNonWordBoundary extends RegExpSpecialChar {
+    RegExpNonWordBoundary() { this.getChar() = "\\B" }
+  }
+
   /**
    * A character class escape in a regular expression.
    * That is, an escaped character that denotes multiple characters.
@@ -829,6 +848,19 @@ module Impl implements RegexTreeViewSig {
     override string getPrimaryQLClass() { result = "RegExpDot" }
   }
 
+  /**
+   * A term that matches a specific position between characters in the string.
+   *
+   * Example:
+   *
+   * ```
+   * \A
+   * ```
+   */
+  class RegExpAnchor extends RegExpSpecialChar {
+    RegExpAnchor() { this.getChar() = ["\\A", "^", "$", "\\Z"] }
+  }
+
   /**
    * A dollar assertion `$` or `\Z` matching the end of a line.
    *
@@ -838,7 +870,7 @@ module Impl implements RegexTreeViewSig {
    * $
    * ```
    */
-  class RegExpDollar extends RegExpSpecialChar {
+  class RegExpDollar extends RegExpAnchor {
     RegExpDollar() { this.getChar() = ["$", "\\Z"] }
 
     override string getPrimaryQLClass() { result = "RegExpDollar" }
@@ -853,7 +885,7 @@ module Impl implements RegexTreeViewSig {
    * ^
    * ```
    */
-  class RegExpCaret extends RegExpSpecialChar {
+  class RegExpCaret extends RegExpAnchor {
     RegExpCaret() { this.getChar() = ["^", "\\A"] }
 
     override string getPrimaryQLClass() { result = "RegExpCaret" }

python/ql/lib/semmle/python/SelfAttribute.qll

@@ -78,8 +78,8 @@ private predicate attr_assigned_in_method_arg_n(FunctionObject method, string na
       attr.isStore()
     )
     or
-    exists(CallNode call, FunctionObject callee, int m |
-      callee.getArgumentForCall(call, m) = param.getAUse() and
+    exists(FunctionObject callee, int m |
+      callee.getArgumentForCall(_, m) = param.getAUse() and
       attr_assigned_in_method_arg_n(callee, name, m)
     )
   )

python/ql/lib/semmle/python/Stmts.qll

@@ -143,12 +143,30 @@ class Exec extends Exec_ {
   override Stmt getASubStatement() { none() }
 }
 
-/** An except statement (part of a `try` statement), such as `except IOError as err:` */
-class ExceptStmt extends ExceptStmt_ {
-  /* syntax: except Expr [ as Expr ]: */
+/**
+ * An exception handler such as an `except` or an `except*` statement
+ * in a `try` statement.
+ */
+class ExceptionHandler extends Stmt {
+  ExceptionHandler() {
+    this instanceof ExceptStmt_
+    or
+    this instanceof ExceptGroupStmt_
+  }
+
   /** Gets the immediately enclosing try statement */
   Try getTry() { result.getAHandler() = this }
 
+  /** Gets the name of this except group block. */
+  abstract Expr getName();
+
+  /** Gets the type of this except group block. */
+  abstract Expr getType();
+}
+
+/** An except group statement (part of a `try` statement), such as `except* IOError as err:` */
+class ExceptGroupStmt extends ExceptGroupStmt_, ExceptionHandler {
+  /* syntax: except Expr [ as Expr ]: */
   override Expr getASubExpression() {
     result = this.getName()
     or
@@ -159,10 +177,34 @@ class ExceptStmt extends ExceptStmt_ {
 
   override Stmt getLastStatement() { result = this.getBody().getLastItem().getLastStatement() }
 
+  override Expr getName() { result = ExceptGroupStmt_.super.getName() }
+
   override Expr getType() {
-    result = super.getType() and not result instanceof Tuple
+    result = ExceptGroupStmt_.super.getType() and not result instanceof Tuple
     or
-    result = super.getType().(Tuple).getAnElt()
+    result = ExceptGroupStmt_.super.getType().(Tuple).getAnElt()
+  }
+}
+
+/** An except statement (part of a `try` statement), such as `except IOError as err:` */
+class ExceptStmt extends ExceptStmt_, ExceptionHandler {
+  /* syntax: except Expr [ as Expr ]: */
+  override Expr getASubExpression() {
+    result = this.getName()
+    or
+    result = this.getType()
+  }
+
+  override Stmt getASubStatement() { result = this.getAStmt() }
+
+  override Stmt getLastStatement() { result = this.getBody().getLastItem().getLastStatement() }
+
+  override Expr getName() { result = ExceptStmt_.super.getName() }
+
+  override Expr getType() {
+    result = ExceptStmt_.super.getType() and not result instanceof Tuple
+    or
+    result = ExceptStmt_.super.getType().(Tuple).getAnElt()
   }
 }
 
@@ -364,10 +406,15 @@ class Try extends Try_ {
     result = this.getAnOrelse()
   }
 
-  override ExceptStmt getHandler(int i) { result = Try_.super.getHandler(i) }
+  override ExceptionHandler getHandler(int i) { result = Try_.super.getHandler(i) }
 
-  /** Gets an exception handler of this try statement. */
-  override ExceptStmt getAHandler() { result = Try_.super.getAHandler() }
+  override ExceptionHandler getAHandler() { result = Try_.super.getAHandler() }
+
+  /** Gets a normal exception handler, `except`, of this try statement. */
+  ExceptStmt getANormalHandler() { result = this.getAHandler() }
+
+  /** Gets a group exception handler, `except*`, of this try statement. */
+  ExceptGroupStmt getAGroupHandler() { result = this.getAHandler() }
 
   override Stmt getLastStatement() {
     result = this.getFinalbody().getLastItem().getLastStatement()

python/ql/lib/semmle/python/dataflow/new/Regexp.qll

@@ -9,11 +9,12 @@ private import semmle.python.dataflow.new.DataFlow
 /**
  * Provides utility predicates related to regular expressions.
  */
-module RegExpPatterns {
+deprecated module RegExpPatterns {
   /**
    * Gets a pattern that matches common top-level domain names in lower case.
+   * DEPRECATED: use the similarly named predicate from `HostnameRegex` from the `regex` pack instead.
    */
-  string getACommonTld() {
+  deprecated string getACommonTld() {
     // according to ranking by http://google.com/search?q=site:.<<TLD>>
     result = "(?:com|org|edu|gov|uk|net|io)(?![a-z0-9])"
   }

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl.qll

@@ -622,7 +622,11 @@ private predicate parameterFlowThroughAllowed(ParamNodeEx p, ReturnKindExt kind)
 }
 
 private module Stage1 implements StageSig {
-  class Ap = Unit;
+  class Ap extends int {
+    // workaround for bad functionality-induced joins (happens when using `Unit`)
+    pragma[nomagic]
+    Ap() { this in [0 .. 1] and this < 1 }
+  }
 
   private class Cc = boolean;
 
@@ -872,9 +876,9 @@ private module Stage1 implements StageSig {
 
   pragma[nomagic]
   private predicate revFlowOut(ReturnPosition pos, Configuration config) {
-    exists(DataFlowCall call, NodeEx out |
+    exists(NodeEx out |
       revFlow(out, _, config) and
-      viableReturnPosOutNodeCandFwd1(call, pos, out, config)
+      viableReturnPosOutNodeCandFwd1(_, pos, out, config)
     )
   }
 
@@ -1327,8 +1331,8 @@ private module MkStage<StageSig PrevStage> {
      */
     pragma[nomagic]
     additional predicate fwdFlow(
-      NodeEx node, FlowState state, Cc cc, ParameterPositionOption summaryCtx, ApOption argAp,
-      Ap ap, ApApprox apa, Configuration config
+      NodeEx node, FlowState state, Cc cc, ParamNodeOption summaryCtx, ApOption argAp, Ap ap,
+      ApApprox apa, Configuration config
     ) {
       fwdFlow0(node, state, cc, summaryCtx, argAp, ap, apa, config) and
       PrevStage::revFlow(node, state, apa, config) and
@@ -1337,21 +1341,21 @@ private module MkStage<StageSig PrevStage> {
 
     pragma[inline]
     additional predicate fwdFlow(
-      NodeEx node, FlowState state, Cc cc, ParameterPositionOption summaryCtx, ApOption argAp,
-      Ap ap, Configuration config
+      NodeEx node, FlowState state, Cc cc, ParamNodeOption summaryCtx, ApOption argAp, Ap ap,
+      Configuration config
     ) {
       fwdFlow(node, state, cc, summaryCtx, argAp, ap, _, config)
     }
 
     pragma[nomagic]
     private predicate fwdFlow0(
-      NodeEx node, FlowState state, Cc cc, ParameterPositionOption summaryCtx, ApOption argAp,
-      Ap ap, ApApprox apa, Configuration config
+      NodeEx node, FlowState state, Cc cc, ParamNodeOption summaryCtx, ApOption argAp, Ap ap,
+      ApApprox apa, Configuration config
     ) {
       sourceNode(node, state, config) and
       (if hasSourceCallCtx(config) then cc = ccSomeCall() else cc = ccNone()) and
       argAp = apNone() and
-      summaryCtx = TParameterPositionNone() and
+      summaryCtx = TParamNodeNone() and
       ap = getApNil(node) and
       apa = getApprox(ap)
       or
@@ -1372,7 +1376,7 @@ private module MkStage<StageSig PrevStage> {
         fwdFlow(mid, pragma[only_bind_into](state), _, _, _, ap, apa, pragma[only_bind_into](config)) and
         jumpStep(mid, node, config) and
         cc = ccNone() and
-        summaryCtx = TParameterPositionNone() and
+        summaryCtx = TParamNodeNone() and
         argAp = apNone()
       )
       or
@@ -1380,7 +1384,7 @@ private module MkStage<StageSig PrevStage> {
         fwdFlow(mid, state, _, _, _, nil, pragma[only_bind_into](config)) and
         additionalJumpStep(mid, node, config) and
         cc = ccNone() and
-        summaryCtx = TParameterPositionNone() and
+        summaryCtx = TParamNodeNone() and
         argAp = apNone() and
         ap = getApNil(node) and
         apa = getApprox(ap)
@@ -1390,7 +1394,7 @@ private module MkStage<StageSig PrevStage> {
         fwdFlow(mid, state0, _, _, _, nil, pragma[only_bind_into](config)) and
         additionalJumpStateStep(mid, state0, node, state, config) and
         cc = ccNone() and
-        summaryCtx = TParameterPositionNone() and
+        summaryCtx = TParamNodeNone() and
         argAp = apNone() and
         ap = getApNil(node) and
         apa = getApprox(ap)
@@ -1414,10 +1418,10 @@ private module MkStage<StageSig PrevStage> {
       fwdFlowIn(_, node, state, _, cc, _, _, ap, apa, config) and
       if PrevStage::parameterMayFlowThrough(node, apa, config)
       then (
-        summaryCtx = TParameterPositionSome(node.(ParamNodeEx).getPosition()) and
+        summaryCtx = TParamNodeSome(node.asNode()) and
         argAp = apSome(ap)
       ) else (
-        summaryCtx = TParameterPositionNone() and argAp = apNone()
+        summaryCtx = TParamNodeNone() and argAp = apNone()
       )
       or
       // flow out of a callable
@@ -1433,16 +1437,19 @@ private module MkStage<StageSig PrevStage> {
       )
       or
       // flow through a callable
-      exists(DataFlowCall call, ParameterPosition summaryCtx0, Ap argAp0 |
-        fwdFlowOutFromArg(call, node, state, summaryCtx0, argAp0, ap, apa, config) and
-        fwdFlowIsEntered(call, cc, summaryCtx, argAp, summaryCtx0, argAp0, config)
+      exists(
+        DataFlowCall call, CcCall ccc, RetNodeEx ret, boolean allowsFieldFlow, ApApprox innerArgApa
+      |
+        fwdFlowThrough(call, cc, state, ccc, summaryCtx, argAp, ap, apa, ret, innerArgApa, config) and
+        flowThroughOutOfCall(call, ccc, ret, node, allowsFieldFlow, innerArgApa, apa, config) and
+        if allowsFieldFlow = false then ap instanceof ApNil else any()
       )
     }
 
     pragma[nomagic]
     private predicate fwdFlowStore(
       NodeEx node1, Ap ap1, TypedContent tc, NodeEx node2, FlowState state, Cc cc,
-      ParameterPositionOption summaryCtx, ApOption argAp, Configuration config
+      ParamNodeOption summaryCtx, ApOption argAp, Configuration config
     ) {
       exists(DataFlowType contentType, ApApprox apa1 |
         fwdFlow(node1, state, cc, summaryCtx, argAp, ap1, apa1, config) and
@@ -1473,8 +1480,8 @@ private module MkStage<StageSig PrevStage> {
 
     pragma[nomagic]
     private predicate fwdFlowRead0(
-      NodeEx node1, FlowState state, Cc cc, ParameterPositionOption summaryCtx, ApOption argAp,
-      ApNonNil ap, Configuration config
+      NodeEx node1, FlowState state, Cc cc, ParamNodeOption summaryCtx, ApOption argAp, ApNonNil ap,
+      Configuration config
     ) {
       fwdFlow(node1, state, cc, summaryCtx, argAp, ap, config) and
       PrevStage::readStepCand(node1, _, _, config)
@@ -1483,7 +1490,7 @@ private module MkStage<StageSig PrevStage> {
     pragma[nomagic]
     private predicate fwdFlowRead(
       Ap ap, Content c, NodeEx node1, NodeEx node2, FlowState state, Cc cc,
-      ParameterPositionOption summaryCtx, ApOption argAp, Configuration config
+      ParamNodeOption summaryCtx, ApOption argAp, Configuration config
     ) {
       fwdFlowRead0(node1, state, cc, summaryCtx, argAp, ap, config) and
       PrevStage::readStepCand(node1, c, node2, config) and
@@ -1493,7 +1500,7 @@ private module MkStage<StageSig PrevStage> {
     pragma[nomagic]
     private predicate fwdFlowIn(
       DataFlowCall call, ParamNodeEx p, FlowState state, Cc outercc, CcCall innercc,
-      ParameterPositionOption summaryCtx, ApOption argAp, Ap ap, ApApprox apa, Configuration config
+      ParamNodeOption summaryCtx, ApOption argAp, Ap ap, ApApprox apa, Configuration config
     ) {
       exists(ArgNodeEx arg, boolean allowsFieldFlow |
         fwdFlow(arg, state, outercc, summaryCtx, argAp, ap, apa, config) and
@@ -1505,64 +1512,38 @@ private module MkStage<StageSig PrevStage> {
 
     pragma[nomagic]
     private predicate fwdFlowRetFromArg(
-      RetNodeEx ret, FlowState state, CcCall ccc, ParameterPosition summaryCtx, ParamNodeEx p,
-      Ap argAp, ApApprox argApa, Ap ap, ApApprox apa, Configuration config
+      RetNodeEx ret, FlowState state, CcCall ccc, ParamNodeEx summaryCtx, Ap argAp, ApApprox argApa,
+      Ap ap, ApApprox apa, Configuration config
     ) {
-      exists(DataFlowCallable c, ReturnKindExt kind |
+      exists(ReturnKindExt kind |
         fwdFlow(pragma[only_bind_into](ret), state, ccc,
-          TParameterPositionSome(pragma[only_bind_into](summaryCtx)), apSome(argAp), ap, apa, config) and
-        getApprox(argAp) = argApa and
-        c = ret.getEnclosingCallable() and
+          TParamNodeSome(pragma[only_bind_into](summaryCtx.asNode())),
+          pragma[only_bind_into](apSome(argAp)), ap, pragma[only_bind_into](apa),
+          pragma[only_bind_into](config)) and
         kind = ret.getKind() and
-        p.isParameterOf(c, pragma[only_bind_into](summaryCtx)) and
-        parameterFlowThroughAllowed(p, kind)
+        parameterFlowThroughAllowed(summaryCtx, kind) and
+        argApa = getApprox(argAp) and
+        PrevStage::returnMayFlowThrough(ret, argApa, apa, kind, pragma[only_bind_into](config))
       )
     }
 
     pragma[inline]
-    private predicate fwdFlowInMayFlowThrough(
-      DataFlowCall call, Cc cc, CcCall innerCc, ParameterPositionOption summaryCtx, ApOption argAp,
-      ParamNodeEx param, Ap ap, ApApprox apa, Configuration config
+    private predicate fwdFlowThrough0(
+      DataFlowCall call, Cc cc, FlowState state, CcCall ccc, ParamNodeOption summaryCtx,
+      ApOption argAp, Ap ap, ApApprox apa, RetNodeEx ret, ParamNodeEx innerSummaryCtx,
+      Ap innerArgAp, ApApprox innerArgApa, Configuration config
     ) {
-      fwdFlowIn(call, pragma[only_bind_into](param), _, cc, innerCc, summaryCtx, argAp, ap,
-        pragma[only_bind_into](apa), pragma[only_bind_into](config)) and
-      PrevStage::parameterMayFlowThrough(param, apa, config)
-    }
-
-    // dedup before joining with `flowThroughOutOfCall`
-    pragma[nomagic]
-    private predicate fwdFlowInMayFlowThroughProj(
-      DataFlowCall call, CcCall innerCc, ApApprox apa, Configuration config
-    ) {
-      fwdFlowInMayFlowThrough(call, _, innerCc, _, _, _, _, apa, config)
-    }
-
-    /**
-     * Same as `flowThroughOutOfCall`, but restricted to calls that are reached
-     * in the flow covered by `fwdFlow`, where data might flow through the target
-     * callable and back out at `call`.
-     */
-    pragma[nomagic]
-    private predicate fwdFlowThroughOutOfCall(
-      DataFlowCall call, CcCall ccc, RetNodeEx ret, NodeEx out, boolean allowsFieldFlow,
-      ApApprox argApa, ApApprox apa, Configuration config
-    ) {
-      fwdFlowInMayFlowThroughProj(call, ccc, argApa, config) and
-      flowThroughOutOfCall(call, ccc, ret, out, allowsFieldFlow, argApa, apa, config)
+      fwdFlowRetFromArg(ret, state, ccc, innerSummaryCtx, innerArgAp, innerArgApa, ap, apa, config) and
+      fwdFlowIsEntered(call, cc, ccc, summaryCtx, argAp, innerSummaryCtx, innerArgAp, config)
     }
 
     pragma[nomagic]
-    private predicate fwdFlowOutFromArg(
-      DataFlowCall call, NodeEx out, FlowState state, ParameterPosition summaryCtx, Ap argAp, Ap ap,
-      ApApprox apa, Configuration config
+    private predicate fwdFlowThrough(
+      DataFlowCall call, Cc cc, FlowState state, CcCall ccc, ParamNodeOption summaryCtx,
+      ApOption argAp, Ap ap, ApApprox apa, RetNodeEx ret, ApApprox innerArgApa, Configuration config
     ) {
-      exists(RetNodeEx ret, boolean allowsFieldFlow, CcCall ccc, ApApprox argApa |
-        fwdFlowRetFromArg(pragma[only_bind_into](ret), state, pragma[only_bind_into](ccc),
-          summaryCtx, _, argAp, pragma[only_bind_into](argApa), ap, pragma[only_bind_into](apa),
-          config) and
-        fwdFlowThroughOutOfCall(call, ccc, ret, out, allowsFieldFlow, argApa, apa, config) and
-        (if allowsFieldFlow = false then ap instanceof ApNil else any())
-      )
+      fwdFlowThrough0(call, cc, state, ccc, summaryCtx, argAp, ap, apa, ret, _, _, innerArgApa,
+        config)
     }
 
     /**
@@ -1571,12 +1552,14 @@ private module MkStage<StageSig PrevStage> {
      */
     pragma[nomagic]
     private predicate fwdFlowIsEntered(
-      DataFlowCall call, Cc cc, ParameterPositionOption summaryCtx, ApOption argAp,
-      ParameterPosition pos, Ap ap, Configuration config
+      DataFlowCall call, Cc cc, CcCall innerCc, ParamNodeOption summaryCtx, ApOption argAp,
+      ParamNodeEx p, Ap ap, Configuration config
     ) {
-      exists(ParamNodeEx param |
-        fwdFlowInMayFlowThrough(call, cc, _, summaryCtx, argAp, param, ap, _, config) and
-        pos = param.getPosition()
+      exists(ApApprox apa |
+        fwdFlowIn(call, pragma[only_bind_into](p), _, cc, innerCc, summaryCtx, argAp, ap,
+          pragma[only_bind_into](apa), pragma[only_bind_into](config)) and
+        PrevStage::parameterMayFlowThrough(p, apa, config) and
+        PrevStage::callMayFlowThroughRev(call, pragma[only_bind_into](config))
       )
     }
 
@@ -1596,23 +1579,31 @@ private module MkStage<StageSig PrevStage> {
       fwdFlowConsCand(ap1, c, ap2, config)
     }
 
+    pragma[nomagic]
+    private predicate returnFlowsThrough0(
+      DataFlowCall call, FlowState state, CcCall ccc, Ap ap, ApApprox apa, RetNodeEx ret,
+      ParamNodeEx innerSummaryCtx, Ap innerArgAp, ApApprox innerArgApa, Configuration config
+    ) {
+      fwdFlowThrough0(call, _, state, ccc, _, _, ap, apa, ret, innerSummaryCtx, innerArgAp,
+        innerArgApa, config)
+    }
+
     pragma[nomagic]
     private predicate returnFlowsThrough(
-      RetNodeEx ret, ReturnKindExt kind, FlowState state, CcCall ccc, ParamNodeEx p, Ap argAp,
+      RetNodeEx ret, ReturnPosition pos, FlowState state, CcCall ccc, ParamNodeEx p, Ap argAp,
       Ap ap, Configuration config
     ) {
-      exists(boolean allowsFieldFlow, ApApprox argApa, ApApprox apa |
-        fwdFlowRetFromArg(pragma[only_bind_into](ret), state, pragma[only_bind_into](ccc), _, p,
-          argAp, pragma[only_bind_into](argApa), ap, pragma[only_bind_into](apa), config) and
-        kind = ret.getKind() and
-        fwdFlowThroughOutOfCall(_, ccc, ret, _, allowsFieldFlow, argApa, apa, config) and
-        (if allowsFieldFlow = false then ap instanceof ApNil else any())
+      exists(DataFlowCall call, ApApprox apa, boolean allowsFieldFlow, ApApprox innerArgApa |
+        returnFlowsThrough0(call, state, ccc, ap, apa, ret, p, argAp, innerArgApa, config) and
+        flowThroughOutOfCall(call, ccc, ret, _, allowsFieldFlow, innerArgApa, apa, config) and
+        pos = ret.getReturnPosition() and
+        if allowsFieldFlow = false then ap instanceof ApNil else any()
       )
     }
 
     pragma[nomagic]
     private predicate flowThroughIntoCall(
-      DataFlowCall call, ArgNodeEx arg, ParamNodeEx p, boolean allowsFieldFlow, Ap argAp,
+      DataFlowCall call, ArgNodeEx arg, ParamNodeEx p, boolean allowsFieldFlow, Ap argAp, Ap ap,
       Configuration config
     ) {
       exists(ApApprox argApa |
@@ -1620,7 +1611,7 @@ private module MkStage<StageSig PrevStage> {
           allowsFieldFlow, argApa, pragma[only_bind_into](config)) and
         fwdFlow(arg, _, _, _, _, pragma[only_bind_into](argAp), argApa,
           pragma[only_bind_into](config)) and
-        returnFlowsThrough(_, _, _, _, p, pragma[only_bind_into](argAp), _,
+        returnFlowsThrough(_, _, _, _, p, pragma[only_bind_into](argAp), ap,
           pragma[only_bind_into](config)) and
         if allowsFieldFlow = false then argAp instanceof ApNil else any()
       )
@@ -1639,12 +1630,13 @@ private module MkStage<StageSig PrevStage> {
 
     pragma[nomagic]
     private predicate flowOutOfCallAp(
-      DataFlowCall call, RetNodeEx ret, ReturnKindExt kind, NodeEx out, boolean allowsFieldFlow,
+      DataFlowCall call, RetNodeEx ret, ReturnPosition pos, NodeEx out, boolean allowsFieldFlow,
       Ap ap, Configuration config
     ) {
       exists(ApApprox apa |
-        flowOutOfCallApa(call, ret, kind, out, allowsFieldFlow, apa, config) and
-        fwdFlow(ret, _, _, _, _, ap, apa, config)
+        flowOutOfCallApa(call, ret, _, out, allowsFieldFlow, apa, config) and
+        fwdFlow(ret, _, _, _, _, ap, apa, config) and
+        pos = ret.getReturnPosition()
       )
     }
 
@@ -1739,17 +1731,17 @@ private module MkStage<StageSig PrevStage> {
       )
       or
       // flow through a callable
-      exists(DataFlowCall call, ReturnKindExt returnKind0, Ap returnAp0 |
-        revFlowInToReturn(call, node, state, returnKind0, returnAp0, ap, config) and
-        revFlowIsReturned(call, returnCtx, returnAp, returnKind0, returnAp0, config)
+      exists(DataFlowCall call, ParamNodeEx p, Ap innerReturnAp |
+        revFlowThrough(call, returnCtx, p, state, _, returnAp, ap, innerReturnAp, config) and
+        flowThroughIntoCall(call, node, p, _, ap, innerReturnAp, config)
       )
       or
       // flow out of a callable
-      exists(ReturnKindExt kind |
-        revFlowOut(_, node, kind, state, _, _, ap, config) and
-        if returnFlowsThrough(node, kind, state, _, _, _, ap, config)
+      exists(ReturnPosition pos |
+        revFlowOut(_, node, pos, state, _, _, ap, config) and
+        if returnFlowsThrough(node, pos, state, _, _, _, ap, config)
         then (
-          returnCtx = TReturnCtxMaybeFlowThrough(kind) and
+          returnCtx = TReturnCtxMaybeFlowThrough(pos) and
           returnAp = apSome(ap)
         ) else (
           returnCtx = TReturnCtxNoFlowThrough() and returnAp = apNone()
@@ -1782,47 +1774,33 @@ private module MkStage<StageSig PrevStage> {
 
     pragma[nomagic]
     private predicate revFlowOut(
-      DataFlowCall call, RetNodeEx ret, ReturnKindExt kind, FlowState state, ReturnCtx returnCtx,
+      DataFlowCall call, RetNodeEx ret, ReturnPosition pos, FlowState state, ReturnCtx returnCtx,
       ApOption returnAp, Ap ap, Configuration config
     ) {
       exists(NodeEx out, boolean allowsFieldFlow |
         revFlow(out, state, returnCtx, returnAp, ap, config) and
-        flowOutOfCallAp(call, ret, kind, out, allowsFieldFlow, ap, config) and
+        flowOutOfCallAp(call, ret, pos, out, allowsFieldFlow, ap, config) and
         if allowsFieldFlow = false then ap instanceof ApNil else any()
       )
     }
 
-    /**
-     * Same as `flowThroughIntoCall`, but restricted to calls that are reached
-     * in the flow covered by `revFlow`, where data might flow through the target
-     * callable and back out at `call`.
-     */
-    pragma[nomagic]
-    private predicate revFlowThroughIntoCall(
-      DataFlowCall call, ArgNodeEx arg, ParamNodeEx p, boolean allowsFieldFlow, Ap argAp,
-      Configuration config
-    ) {
-      flowThroughIntoCall(call, arg, p, allowsFieldFlow, argAp, config) and
-      revFlowIsReturned(call, _, _, _, _, config)
-    }
-
     pragma[nomagic]
     private predicate revFlowParamToReturn(
-      ParamNodeEx p, FlowState state, ReturnKindExt kind, Ap returnAp, Ap ap, Configuration config
+      ParamNodeEx p, FlowState state, ReturnPosition pos, Ap returnAp, Ap ap, Configuration config
     ) {
-      revFlow(p, state, TReturnCtxMaybeFlowThrough(kind), apSome(returnAp), ap, config) and
-      parameterFlowThroughAllowed(p, kind)
+      revFlow(pragma[only_bind_into](p), state, TReturnCtxMaybeFlowThrough(pos), apSome(returnAp),
+        pragma[only_bind_into](ap), pragma[only_bind_into](config)) and
+      parameterFlowThroughAllowed(p, pos.getKind()) and
+      PrevStage::parameterMayFlowThrough(p, getApprox(ap), config)
     }
 
     pragma[nomagic]
-    private predicate revFlowInToReturn(
-      DataFlowCall call, ArgNodeEx arg, FlowState state, ReturnKindExt kind, Ap returnAp, Ap ap,
-      Configuration config
+    private predicate revFlowThrough(
+      DataFlowCall call, ReturnCtx returnCtx, ParamNodeEx p, FlowState state, ReturnPosition pos,
+      ApOption returnAp, Ap ap, Ap innerReturnAp, Configuration config
     ) {
-      exists(ParamNodeEx p, boolean allowsFieldFlow |
-        revFlowParamToReturn(p, state, kind, returnAp, ap, config) and
-        revFlowThroughIntoCall(call, arg, p, allowsFieldFlow, ap, config)
-      )
+      revFlowParamToReturn(p, state, pos, innerReturnAp, ap, config) and
+      revFlowIsReturned(call, returnCtx, returnAp, pos, innerReturnAp, config)
     }
 
     /**
@@ -1832,12 +1810,12 @@ private module MkStage<StageSig PrevStage> {
      */
     pragma[nomagic]
     private predicate revFlowIsReturned(
-      DataFlowCall call, ReturnCtx returnCtx, ApOption returnAp, ReturnKindExt kind, Ap ap,
+      DataFlowCall call, ReturnCtx returnCtx, ApOption returnAp, ReturnPosition pos, Ap ap,
       Configuration config
     ) {
       exists(RetNodeEx ret, FlowState state, CcCall ccc |
-        revFlowOut(call, ret, kind, state, returnCtx, returnAp, ap, config) and
-        returnFlowsThrough(ret, kind, state, ccc, _, _, ap, config) and
+        revFlowOut(call, ret, pos, state, returnCtx, returnAp, ap, config) and
+        returnFlowsThrough(ret, pos, state, ccc, _, _, ap, config) and
         matchesCall(ccc, call)
       )
     }
@@ -1915,17 +1893,17 @@ private module MkStage<StageSig PrevStage> {
 
     pragma[nomagic]
     private predicate parameterFlowsThroughRev(
-      ParamNodeEx p, Ap ap, ReturnKindExt kind, Ap returnAp, Configuration config
+      ParamNodeEx p, Ap ap, ReturnPosition pos, Ap returnAp, Configuration config
     ) {
-      revFlow(p, _, TReturnCtxMaybeFlowThrough(kind), apSome(returnAp), ap, config) and
-      parameterFlowThroughAllowed(p, kind)
+      revFlow(p, _, TReturnCtxMaybeFlowThrough(pos), apSome(returnAp), ap, config) and
+      parameterFlowThroughAllowed(p, pos.getKind())
     }
 
     pragma[nomagic]
     predicate parameterMayFlowThrough(ParamNodeEx p, Ap ap, Configuration config) {
-      exists(RetNodeEx ret, ReturnKindExt kind |
-        returnFlowsThrough(ret, kind, _, _, p, ap, _, config) and
-        parameterFlowsThroughRev(p, ap, kind, _, config)
+      exists(ReturnPosition pos |
+        returnFlowsThrough(_, pos, _, _, p, ap, _, config) and
+        parameterFlowsThroughRev(p, ap, pos, _, config)
       )
     }
 
@@ -1933,20 +1911,21 @@ private module MkStage<StageSig PrevStage> {
     predicate returnMayFlowThrough(
       RetNodeEx ret, Ap argAp, Ap ap, ReturnKindExt kind, Configuration config
     ) {
-      exists(ParamNodeEx p |
-        returnFlowsThrough(ret, kind, _, _, p, argAp, ap, config) and
-        parameterFlowsThroughRev(p, argAp, kind, ap, config)
+      exists(ParamNodeEx p, ReturnPosition pos |
+        returnFlowsThrough(ret, pos, _, _, p, argAp, ap, config) and
+        parameterFlowsThroughRev(p, argAp, pos, ap, config) and
+        kind = pos.getKind()
       )
     }
 
     pragma[nomagic]
-    predicate revFlowInToReturnIsReturned(
+    private predicate revFlowThroughArg(
       DataFlowCall call, ArgNodeEx arg, FlowState state, ReturnCtx returnCtx, ApOption returnAp,
       Ap ap, Configuration config
     ) {
-      exists(ReturnKindExt returnKind0, Ap returnAp0 |
-        revFlowInToReturn(call, arg, state, returnKind0, returnAp0, ap, config) and
-        revFlowIsReturned(call, returnCtx, returnAp, returnKind0, returnAp0, config)
+      exists(ParamNodeEx p, Ap innerReturnAp |
+        revFlowThrough(call, returnCtx, p, state, _, returnAp, ap, innerReturnAp, config) and
+        flowThroughIntoCall(call, arg, p, _, ap, innerReturnAp, config)
       )
     }
 
@@ -1954,7 +1933,7 @@ private module MkStage<StageSig PrevStage> {
     predicate callMayFlowThroughRev(DataFlowCall call, Configuration config) {
       exists(ArgNodeEx arg, FlowState state, ReturnCtx returnCtx, ApOption returnAp, Ap ap |
         revFlow(arg, state, returnCtx, returnAp, ap, config) and
-        revFlowInToReturnIsReturned(call, arg, state, returnCtx, returnAp, ap, config)
+        revFlowThroughArg(call, arg, state, returnCtx, returnAp, ap, config)
       )
     }
 
@@ -1967,8 +1946,9 @@ private module MkStage<StageSig PrevStage> {
       conscand = count(TypedContent f0, Ap ap | fwdConsCand(f0, ap, config)) and
       states = count(FlowState state | fwdFlow(_, state, _, _, _, _, config)) and
       tuples =
-        count(NodeEx n, FlowState state, Cc cc, ParameterPositionOption summaryCtx, ApOption argAp,
-          Ap ap | fwdFlow(n, state, cc, summaryCtx, argAp, ap, config))
+        count(NodeEx n, FlowState state, Cc cc, ParamNodeOption summaryCtx, ApOption argAp, Ap ap |
+          fwdFlow(n, state, cc, summaryCtx, argAp, ap, config)
+        )
       or
       fwd = false and
       nodes = count(NodeEx node | revFlow(node, _, _, _, _, config)) and
@@ -2823,13 +2803,12 @@ private Configuration unbindConf(Configuration conf) {
 
 pragma[nomagic]
 private predicate nodeMayUseSummary0(
-  NodeEx n, DataFlowCallable c, ParameterPosition pos, FlowState state, AccessPathApprox apa,
-  Configuration config
+  NodeEx n, ParamNodeEx p, FlowState state, AccessPathApprox apa, Configuration config
 ) {
   exists(AccessPathApprox apa0 |
-    c = n.getEnclosingCallable() and
+    Stage5::parameterMayFlowThrough(p, _, _) and
     Stage5::revFlow(n, state, TReturnCtxMaybeFlowThrough(_), _, apa0, config) and
-    Stage5::fwdFlow(n, state, any(CallContextCall ccc), TParameterPositionSome(pos),
+    Stage5::fwdFlow(n, state, any(CallContextCall ccc), TParamNodeSome(p.asNode()),
       TAccessPathApproxSome(apa), apa0, config)
   )
 }
@@ -2838,10 +2817,9 @@ pragma[nomagic]
 private predicate nodeMayUseSummary(
   NodeEx n, FlowState state, AccessPathApprox apa, Configuration config
 ) {
-  exists(DataFlowCallable c, ParameterPosition pos, ParamNodeEx p |
+  exists(ParamNodeEx p |
     Stage5::parameterMayFlowThrough(p, apa, config) and
-    nodeMayUseSummary0(n, c, pos, state, apa, config) and
-    p.isParameterOf(c, pos)
+    nodeMayUseSummary0(n, p, state, apa, config)
   )
 }
 
@@ -3771,8 +3749,8 @@ private predicate paramFlowsThrough(
   ReturnKindExt kind, FlowState state, CallContextCall cc, SummaryCtxSome sc, AccessPath ap,
   AccessPathApprox apa, Configuration config
 ) {
-  exists(PathNodeMid mid, RetNodeEx ret |
-    pathNode(mid, ret, state, cc, sc, ap, config, _) and
+  exists(RetNodeEx ret |
+    pathNode(_, ret, state, cc, sc, ap, config, _) and
     kind = ret.getKind() and
     apa = ap.getApprox() and
     parameterFlowThroughAllowed(sc.getParamNode(), kind)
@@ -4234,17 +4212,15 @@ private module FlowExploration {
       ap = TRevPartialNil() and
       exists(config.explorationLimit())
       or
-      exists(PartialPathNodeRev mid |
-        revPartialPathStep(mid, node, state, sc1, sc2, sc3, ap, config) and
-        not clearsContentEx(node, ap.getHead()) and
-        (
-          notExpectsContent(node) or
-          expectsContentEx(node, ap.getHead())
-        ) and
-        not fullBarrier(node, config) and
-        not stateBarrier(node, state, config) and
-        distSink(node.getEnclosingCallable(), config) <= config.explorationLimit()
-      )
+      revPartialPathStep(_, node, state, sc1, sc2, sc3, ap, config) and
+      not clearsContentEx(node, ap.getHead()) and
+      (
+        notExpectsContent(node) or
+        expectsContentEx(node, ap.getHead())
+      ) and
+      not fullBarrier(node, config) and
+      not stateBarrier(node, state, config) and
+      distSink(node.getEnclosingCallable(), config) <= config.explorationLimit()
     }
 
   pragma[nomagic]
@@ -4252,19 +4228,17 @@ private module FlowExploration {
     NodeEx node, FlowState state, CallContext cc, TSummaryCtx1 sc1, TSummaryCtx2 sc2,
     TSummaryCtx3 sc3, PartialAccessPath ap, Configuration config
   ) {
-    exists(PartialPathNodeFwd mid |
-      partialPathStep(mid, node, state, cc, sc1, sc2, sc3, ap, config) and
-      not fullBarrier(node, config) and
-      not stateBarrier(node, state, config) and
-      not clearsContentEx(node, ap.getHead().getContent()) and
-      (
-        notExpectsContent(node) or
-        expectsContentEx(node, ap.getHead().getContent())
-      ) and
-      if node.asNode() instanceof CastingNode
-      then compatibleTypes(node.getDataFlowType(), ap.getType())
-      else any()
-    )
+    partialPathStep(_, node, state, cc, sc1, sc2, sc3, ap, config) and
+    not fullBarrier(node, config) and
+    not stateBarrier(node, state, config) and
+    not clearsContentEx(node, ap.getHead().getContent()) and
+    (
+      notExpectsContent(node) or
+      expectsContentEx(node, ap.getHead().getContent())
+    ) and
+    if node.asNode() instanceof CastingNode
+    then compatibleTypes(node.getDataFlowType(), ap.getType())
+    else any()
   }
 
   /**

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl2.qll

@@ -622,7 +622,11 @@ private predicate parameterFlowThroughAllowed(ParamNodeEx p, ReturnKindExt kind)
 }
 
 private module Stage1 implements StageSig {
-  class Ap = Unit;
+  class Ap extends int {
+    // workaround for bad functionality-induced joins (happens when using `Unit`)
+    pragma[nomagic]
+    Ap() { this in [0 .. 1] and this < 1 }
+  }
 
   private class Cc = boolean;
 
@@ -872,9 +876,9 @@ private module Stage1 implements StageSig {
 
   pragma[nomagic]
   private predicate revFlowOut(ReturnPosition pos, Configuration config) {
-    exists(DataFlowCall call, NodeEx out |
+    exists(NodeEx out |
       revFlow(out, _, config) and
-      viableReturnPosOutNodeCandFwd1(call, pos, out, config)
+      viableReturnPosOutNodeCandFwd1(_, pos, out, config)
     )
   }
 
@@ -1327,8 +1331,8 @@ private module MkStage<StageSig PrevStage> {
      */
     pragma[nomagic]
     additional predicate fwdFlow(
-      NodeEx node, FlowState state, Cc cc, ParameterPositionOption summaryCtx, ApOption argAp,
-      Ap ap, ApApprox apa, Configuration config
+      NodeEx node, FlowState state, Cc cc, ParamNodeOption summaryCtx, ApOption argAp, Ap ap,
+      ApApprox apa, Configuration config
     ) {
       fwdFlow0(node, state, cc, summaryCtx, argAp, ap, apa, config) and
       PrevStage::revFlow(node, state, apa, config) and
@@ -1337,21 +1341,21 @@ private module MkStage<StageSig PrevStage> {
 
     pragma[inline]
     additional predicate fwdFlow(
-      NodeEx node, FlowState state, Cc cc, ParameterPositionOption summaryCtx, ApOption argAp,
-      Ap ap, Configuration config
+      NodeEx node, FlowState state, Cc cc, ParamNodeOption summaryCtx, ApOption argAp, Ap ap,
+      Configuration config
     ) {
       fwdFlow(node, state, cc, summaryCtx, argAp, ap, _, config)
     }
 
     pragma[nomagic]
     private predicate fwdFlow0(
-      NodeEx node, FlowState state, Cc cc, ParameterPositionOption summaryCtx, ApOption argAp,
-      Ap ap, ApApprox apa, Configuration config
+      NodeEx node, FlowState state, Cc cc, ParamNodeOption summaryCtx, ApOption argAp, Ap ap,
+      ApApprox apa, Configuration config
     ) {
       sourceNode(node, state, config) and
       (if hasSourceCallCtx(config) then cc = ccSomeCall() else cc = ccNone()) and
       argAp = apNone() and
-      summaryCtx = TParameterPositionNone() and
+      summaryCtx = TParamNodeNone() and
       ap = getApNil(node) and
       apa = getApprox(ap)
       or
@@ -1372,7 +1376,7 @@ private module MkStage<StageSig PrevStage> {
         fwdFlow(mid, pragma[only_bind_into](state), _, _, _, ap, apa, pragma[only_bind_into](config)) and
         jumpStep(mid, node, config) and
         cc = ccNone() and
-        summaryCtx = TParameterPositionNone() and
+        summaryCtx = TParamNodeNone() and
         argAp = apNone()
       )
       or
@@ -1380,7 +1384,7 @@ private module MkStage<StageSig PrevStage> {
         fwdFlow(mid, state, _, _, _, nil, pragma[only_bind_into](config)) and
         additionalJumpStep(mid, node, config) and
         cc = ccNone() and
-        summaryCtx = TParameterPositionNone() and
+        summaryCtx = TParamNodeNone() and
         argAp = apNone() and
         ap = getApNil(node) and
         apa = getApprox(ap)
@@ -1390,7 +1394,7 @@ private module MkStage<StageSig PrevStage> {
         fwdFlow(mid, state0, _, _, _, nil, pragma[only_bind_into](config)) and
         additionalJumpStateStep(mid, state0, node, state, config) and
         cc = ccNone() and
-        summaryCtx = TParameterPositionNone() and
+        summaryCtx = TParamNodeNone() and
         argAp = apNone() and
         ap = getApNil(node) and
         apa = getApprox(ap)
@@ -1414,10 +1418,10 @@ private module MkStage<StageSig PrevStage> {
       fwdFlowIn(_, node, state, _, cc, _, _, ap, apa, config) and
       if PrevStage::parameterMayFlowThrough(node, apa, config)
       then (
-        summaryCtx = TParameterPositionSome(node.(ParamNodeEx).getPosition()) and
+        summaryCtx = TParamNodeSome(node.asNode()) and
         argAp = apSome(ap)
       ) else (
-        summaryCtx = TParameterPositionNone() and argAp = apNone()
+        summaryCtx = TParamNodeNone() and argAp = apNone()
       )
       or
       // flow out of a callable
@@ -1433,16 +1437,19 @@ private module MkStage<StageSig PrevStage> {
       )
       or
       // flow through a callable
-      exists(DataFlowCall call, ParameterPosition summaryCtx0, Ap argAp0 |
-        fwdFlowOutFromArg(call, node, state, summaryCtx0, argAp0, ap, apa, config) and
-        fwdFlowIsEntered(call, cc, summaryCtx, argAp, summaryCtx0, argAp0, config)
+      exists(
+        DataFlowCall call, CcCall ccc, RetNodeEx ret, boolean allowsFieldFlow, ApApprox innerArgApa
+      |
+        fwdFlowThrough(call, cc, state, ccc, summaryCtx, argAp, ap, apa, ret, innerArgApa, config) and
+        flowThroughOutOfCall(call, ccc, ret, node, allowsFieldFlow, innerArgApa, apa, config) and
+        if allowsFieldFlow = false then ap instanceof ApNil else any()
       )
     }
 
     pragma[nomagic]
     private predicate fwdFlowStore(
       NodeEx node1, Ap ap1, TypedContent tc, NodeEx node2, FlowState state, Cc cc,
-      ParameterPositionOption summaryCtx, ApOption argAp, Configuration config
+      ParamNodeOption summaryCtx, ApOption argAp, Configuration config
     ) {
       exists(DataFlowType contentType, ApApprox apa1 |
         fwdFlow(node1, state, cc, summaryCtx, argAp, ap1, apa1, config) and
@@ -1473,8 +1480,8 @@ private module MkStage<StageSig PrevStage> {
 
     pragma[nomagic]
     private predicate fwdFlowRead0(
-      NodeEx node1, FlowState state, Cc cc, ParameterPositionOption summaryCtx, ApOption argAp,
-      ApNonNil ap, Configuration config
+      NodeEx node1, FlowState state, Cc cc, ParamNodeOption summaryCtx, ApOption argAp, ApNonNil ap,
+      Configuration config
     ) {
       fwdFlow(node1, state, cc, summaryCtx, argAp, ap, config) and
       PrevStage::readStepCand(node1, _, _, config)
@@ -1483,7 +1490,7 @@ private module MkStage<StageSig PrevStage> {
     pragma[nomagic]
     private predicate fwdFlowRead(
       Ap ap, Content c, NodeEx node1, NodeEx node2, FlowState state, Cc cc,
-      ParameterPositionOption summaryCtx, ApOption argAp, Configuration config
+      ParamNodeOption summaryCtx, ApOption argAp, Configuration config
     ) {
       fwdFlowRead0(node1, state, cc, summaryCtx, argAp, ap, config) and
       PrevStage::readStepCand(node1, c, node2, config) and
@@ -1493,7 +1500,7 @@ private module MkStage<StageSig PrevStage> {
     pragma[nomagic]
     private predicate fwdFlowIn(
       DataFlowCall call, ParamNodeEx p, FlowState state, Cc outercc, CcCall innercc,
-      ParameterPositionOption summaryCtx, ApOption argAp, Ap ap, ApApprox apa, Configuration config
+      ParamNodeOption summaryCtx, ApOption argAp, Ap ap, ApApprox apa, Configuration config
     ) {
       exists(ArgNodeEx arg, boolean allowsFieldFlow |
         fwdFlow(arg, state, outercc, summaryCtx, argAp, ap, apa, config) and
@@ -1505,64 +1512,38 @@ private module MkStage<StageSig PrevStage> {
 
     pragma[nomagic]
     private predicate fwdFlowRetFromArg(
-      RetNodeEx ret, FlowState state, CcCall ccc, ParameterPosition summaryCtx, ParamNodeEx p,
-      Ap argAp, ApApprox argApa, Ap ap, ApApprox apa, Configuration config
+      RetNodeEx ret, FlowState state, CcCall ccc, ParamNodeEx summaryCtx, Ap argAp, ApApprox argApa,
+      Ap ap, ApApprox apa, Configuration config
     ) {
-      exists(DataFlowCallable c, ReturnKindExt kind |
+      exists(ReturnKindExt kind |
         fwdFlow(pragma[only_bind_into](ret), state, ccc,
-          TParameterPositionSome(pragma[only_bind_into](summaryCtx)), apSome(argAp), ap, apa, config) and
-        getApprox(argAp) = argApa and
-        c = ret.getEnclosingCallable() and
+          TParamNodeSome(pragma[only_bind_into](summaryCtx.asNode())),
+          pragma[only_bind_into](apSome(argAp)), ap, pragma[only_bind_into](apa),
+          pragma[only_bind_into](config)) and
         kind = ret.getKind() and
-        p.isParameterOf(c, pragma[only_bind_into](summaryCtx)) and
-        parameterFlowThroughAllowed(p, kind)
+        parameterFlowThroughAllowed(summaryCtx, kind) and
+        argApa = getApprox(argAp) and
+        PrevStage::returnMayFlowThrough(ret, argApa, apa, kind, pragma[only_bind_into](config))
       )
     }
 
     pragma[inline]
-    private predicate fwdFlowInMayFlowThrough(
-      DataFlowCall call, Cc cc, CcCall innerCc, ParameterPositionOption summaryCtx, ApOption argAp,
-      ParamNodeEx param, Ap ap, ApApprox apa, Configuration config
+    private predicate fwdFlowThrough0(
+      DataFlowCall call, Cc cc, FlowState state, CcCall ccc, ParamNodeOption summaryCtx,
+      ApOption argAp, Ap ap, ApApprox apa, RetNodeEx ret, ParamNodeEx innerSummaryCtx,
+      Ap innerArgAp, ApApprox innerArgApa, Configuration config
     ) {
-      fwdFlowIn(call, pragma[only_bind_into](param), _, cc, innerCc, summaryCtx, argAp, ap,
-        pragma[only_bind_into](apa), pragma[only_bind_into](config)) and
-      PrevStage::parameterMayFlowThrough(param, apa, config)
-    }
-
-    // dedup before joining with `flowThroughOutOfCall`
-    pragma[nomagic]
-    private predicate fwdFlowInMayFlowThroughProj(
-      DataFlowCall call, CcCall innerCc, ApApprox apa, Configuration config
-    ) {
-      fwdFlowInMayFlowThrough(call, _, innerCc, _, _, _, _, apa, config)
-    }
-
-    /**
-     * Same as `flowThroughOutOfCall`, but restricted to calls that are reached
-     * in the flow covered by `fwdFlow`, where data might flow through the target
-     * callable and back out at `call`.
-     */
-    pragma[nomagic]
-    private predicate fwdFlowThroughOutOfCall(
-      DataFlowCall call, CcCall ccc, RetNodeEx ret, NodeEx out, boolean allowsFieldFlow,
-      ApApprox argApa, ApApprox apa, Configuration config
-    ) {
-      fwdFlowInMayFlowThroughProj(call, ccc, argApa, config) and
-      flowThroughOutOfCall(call, ccc, ret, out, allowsFieldFlow, argApa, apa, config)
+      fwdFlowRetFromArg(ret, state, ccc, innerSummaryCtx, innerArgAp, innerArgApa, ap, apa, config) and
+      fwdFlowIsEntered(call, cc, ccc, summaryCtx, argAp, innerSummaryCtx, innerArgAp, config)
     }
 
     pragma[nomagic]
-    private predicate fwdFlowOutFromArg(
-      DataFlowCall call, NodeEx out, FlowState state, ParameterPosition summaryCtx, Ap argAp, Ap ap,
-      ApApprox apa, Configuration config
+    private predicate fwdFlowThrough(
+      DataFlowCall call, Cc cc, FlowState state, CcCall ccc, ParamNodeOption summaryCtx,
+      ApOption argAp, Ap ap, ApApprox apa, RetNodeEx ret, ApApprox innerArgApa, Configuration config
     ) {
-      exists(RetNodeEx ret, boolean allowsFieldFlow, CcCall ccc, ApApprox argApa |
-        fwdFlowRetFromArg(pragma[only_bind_into](ret), state, pragma[only_bind_into](ccc),
-          summaryCtx, _, argAp, pragma[only_bind_into](argApa), ap, pragma[only_bind_into](apa),
-          config) and
-        fwdFlowThroughOutOfCall(call, ccc, ret, out, allowsFieldFlow, argApa, apa, config) and
-        (if allowsFieldFlow = false then ap instanceof ApNil else any())
-      )
+      fwdFlowThrough0(call, cc, state, ccc, summaryCtx, argAp, ap, apa, ret, _, _, innerArgApa,
+        config)
     }
 
     /**
@@ -1571,12 +1552,14 @@ private module MkStage<StageSig PrevStage> {
      */
     pragma[nomagic]
     private predicate fwdFlowIsEntered(
-      DataFlowCall call, Cc cc, ParameterPositionOption summaryCtx, ApOption argAp,
-      ParameterPosition pos, Ap ap, Configuration config
+      DataFlowCall call, Cc cc, CcCall innerCc, ParamNodeOption summaryCtx, ApOption argAp,
+      ParamNodeEx p, Ap ap, Configuration config
     ) {
-      exists(ParamNodeEx param |
-        fwdFlowInMayFlowThrough(call, cc, _, summaryCtx, argAp, param, ap, _, config) and
-        pos = param.getPosition()
+      exists(ApApprox apa |
+        fwdFlowIn(call, pragma[only_bind_into](p), _, cc, innerCc, summaryCtx, argAp, ap,
+          pragma[only_bind_into](apa), pragma[only_bind_into](config)) and
+        PrevStage::parameterMayFlowThrough(p, apa, config) and
+        PrevStage::callMayFlowThroughRev(call, pragma[only_bind_into](config))
       )
     }
 
@@ -1596,23 +1579,31 @@ private module MkStage<StageSig PrevStage> {
       fwdFlowConsCand(ap1, c, ap2, config)
     }
 
+    pragma[nomagic]
+    private predicate returnFlowsThrough0(
+      DataFlowCall call, FlowState state, CcCall ccc, Ap ap, ApApprox apa, RetNodeEx ret,
+      ParamNodeEx innerSummaryCtx, Ap innerArgAp, ApApprox innerArgApa, Configuration config
+    ) {
+      fwdFlowThrough0(call, _, state, ccc, _, _, ap, apa, ret, innerSummaryCtx, innerArgAp,
+        innerArgApa, config)
+    }
+
     pragma[nomagic]
     private predicate returnFlowsThrough(
-      RetNodeEx ret, ReturnKindExt kind, FlowState state, CcCall ccc, ParamNodeEx p, Ap argAp,
+      RetNodeEx ret, ReturnPosition pos, FlowState state, CcCall ccc, ParamNodeEx p, Ap argAp,
       Ap ap, Configuration config
     ) {
-      exists(boolean allowsFieldFlow, ApApprox argApa, ApApprox apa |
-        fwdFlowRetFromArg(pragma[only_bind_into](ret), state, pragma[only_bind_into](ccc), _, p,
-          argAp, pragma[only_bind_into](argApa), ap, pragma[only_bind_into](apa), config) and
-        kind = ret.getKind() and
-        fwdFlowThroughOutOfCall(_, ccc, ret, _, allowsFieldFlow, argApa, apa, config) and
-        (if allowsFieldFlow = false then ap instanceof ApNil else any())
+      exists(DataFlowCall call, ApApprox apa, boolean allowsFieldFlow, ApApprox innerArgApa |
+        returnFlowsThrough0(call, state, ccc, ap, apa, ret, p, argAp, innerArgApa, config) and
+        flowThroughOutOfCall(call, ccc, ret, _, allowsFieldFlow, innerArgApa, apa, config) and
+        pos = ret.getReturnPosition() and
+        if allowsFieldFlow = false then ap instanceof ApNil else any()
       )
     }
 
     pragma[nomagic]
     private predicate flowThroughIntoCall(
-      DataFlowCall call, ArgNodeEx arg, ParamNodeEx p, boolean allowsFieldFlow, Ap argAp,
+      DataFlowCall call, ArgNodeEx arg, ParamNodeEx p, boolean allowsFieldFlow, Ap argAp, Ap ap,
       Configuration config
     ) {
       exists(ApApprox argApa |
@@ -1620,7 +1611,7 @@ private module MkStage<StageSig PrevStage> {
           allowsFieldFlow, argApa, pragma[only_bind_into](config)) and
         fwdFlow(arg, _, _, _, _, pragma[only_bind_into](argAp), argApa,
           pragma[only_bind_into](config)) and
-        returnFlowsThrough(_, _, _, _, p, pragma[only_bind_into](argAp), _,
+        returnFlowsThrough(_, _, _, _, p, pragma[only_bind_into](argAp), ap,
           pragma[only_bind_into](config)) and
         if allowsFieldFlow = false then argAp instanceof ApNil else any()
       )
@@ -1639,12 +1630,13 @@ private module MkStage<StageSig PrevStage> {
 
     pragma[nomagic]
     private predicate flowOutOfCallAp(
-      DataFlowCall call, RetNodeEx ret, ReturnKindExt kind, NodeEx out, boolean allowsFieldFlow,
+      DataFlowCall call, RetNodeEx ret, ReturnPosition pos, NodeEx out, boolean allowsFieldFlow,
       Ap ap, Configuration config
     ) {
       exists(ApApprox apa |
-        flowOutOfCallApa(call, ret, kind, out, allowsFieldFlow, apa, config) and
-        fwdFlow(ret, _, _, _, _, ap, apa, config)
+        flowOutOfCallApa(call, ret, _, out, allowsFieldFlow, apa, config) and
+        fwdFlow(ret, _, _, _, _, ap, apa, config) and
+        pos = ret.getReturnPosition()
       )
     }
 
@@ -1739,17 +1731,17 @@ private module MkStage<StageSig PrevStage> {
       )
       or
       // flow through a callable
-      exists(DataFlowCall call, ReturnKindExt returnKind0, Ap returnAp0 |
-        revFlowInToReturn(call, node, state, returnKind0, returnAp0, ap, config) and
-        revFlowIsReturned(call, returnCtx, returnAp, returnKind0, returnAp0, config)
+      exists(DataFlowCall call, ParamNodeEx p, Ap innerReturnAp |
+        revFlowThrough(call, returnCtx, p, state, _, returnAp, ap, innerReturnAp, config) and
+        flowThroughIntoCall(call, node, p, _, ap, innerReturnAp, config)
       )
       or
       // flow out of a callable
-      exists(ReturnKindExt kind |
-        revFlowOut(_, node, kind, state, _, _, ap, config) and
-        if returnFlowsThrough(node, kind, state, _, _, _, ap, config)
+      exists(ReturnPosition pos |
+        revFlowOut(_, node, pos, state, _, _, ap, config) and
+        if returnFlowsThrough(node, pos, state, _, _, _, ap, config)
         then (
-          returnCtx = TReturnCtxMaybeFlowThrough(kind) and
+          returnCtx = TReturnCtxMaybeFlowThrough(pos) and
           returnAp = apSome(ap)
         ) else (
           returnCtx = TReturnCtxNoFlowThrough() and returnAp = apNone()
@@ -1782,47 +1774,33 @@ private module MkStage<StageSig PrevStage> {
 
     pragma[nomagic]
     private predicate revFlowOut(
-      DataFlowCall call, RetNodeEx ret, ReturnKindExt kind, FlowState state, ReturnCtx returnCtx,
+      DataFlowCall call, RetNodeEx ret, ReturnPosition pos, FlowState state, ReturnCtx returnCtx,
       ApOption returnAp, Ap ap, Configuration config
     ) {
       exists(NodeEx out, boolean allowsFieldFlow |
         revFlow(out, state, returnCtx, returnAp, ap, config) and
-        flowOutOfCallAp(call, ret, kind, out, allowsFieldFlow, ap, config) and
+        flowOutOfCallAp(call, ret, pos, out, allowsFieldFlow, ap, config) and
         if allowsFieldFlow = false then ap instanceof ApNil else any()
       )
     }
 
-    /**
-     * Same as `flowThroughIntoCall`, but restricted to calls that are reached
-     * in the flow covered by `revFlow`, where data might flow through the target
-     * callable and back out at `call`.
-     */
-    pragma[nomagic]
-    private predicate revFlowThroughIntoCall(
-      DataFlowCall call, ArgNodeEx arg, ParamNodeEx p, boolean allowsFieldFlow, Ap argAp,
-      Configuration config
-    ) {
-      flowThroughIntoCall(call, arg, p, allowsFieldFlow, argAp, config) and
-      revFlowIsReturned(call, _, _, _, _, config)
-    }
-
     pragma[nomagic]
     private predicate revFlowParamToReturn(
-      ParamNodeEx p, FlowState state, ReturnKindExt kind, Ap returnAp, Ap ap, Configuration config
+      ParamNodeEx p, FlowState state, ReturnPosition pos, Ap returnAp, Ap ap, Configuration config
     ) {
-      revFlow(p, state, TReturnCtxMaybeFlowThrough(kind), apSome(returnAp), ap, config) and
-      parameterFlowThroughAllowed(p, kind)
+      revFlow(pragma[only_bind_into](p), state, TReturnCtxMaybeFlowThrough(pos), apSome(returnAp),
+        pragma[only_bind_into](ap), pragma[only_bind_into](config)) and
+      parameterFlowThroughAllowed(p, pos.getKind()) and
+      PrevStage::parameterMayFlowThrough(p, getApprox(ap), config)
     }
 
     pragma[nomagic]
-    private predicate revFlowInToReturn(
-      DataFlowCall call, ArgNodeEx arg, FlowState state, ReturnKindExt kind, Ap returnAp, Ap ap,
-      Configuration config
+    private predicate revFlowThrough(
+      DataFlowCall call, ReturnCtx returnCtx, ParamNodeEx p, FlowState state, ReturnPosition pos,
+      ApOption returnAp, Ap ap, Ap innerReturnAp, Configuration config
     ) {
-      exists(ParamNodeEx p, boolean allowsFieldFlow |
-        revFlowParamToReturn(p, state, kind, returnAp, ap, config) and
-        revFlowThroughIntoCall(call, arg, p, allowsFieldFlow, ap, config)
-      )
+      revFlowParamToReturn(p, state, pos, innerReturnAp, ap, config) and
+      revFlowIsReturned(call, returnCtx, returnAp, pos, innerReturnAp, config)
     }
 
     /**
@@ -1832,12 +1810,12 @@ private module MkStage<StageSig PrevStage> {
      */
     pragma[nomagic]
     private predicate revFlowIsReturned(
-      DataFlowCall call, ReturnCtx returnCtx, ApOption returnAp, ReturnKindExt kind, Ap ap,
+      DataFlowCall call, ReturnCtx returnCtx, ApOption returnAp, ReturnPosition pos, Ap ap,
       Configuration config
     ) {
       exists(RetNodeEx ret, FlowState state, CcCall ccc |
-        revFlowOut(call, ret, kind, state, returnCtx, returnAp, ap, config) and
-        returnFlowsThrough(ret, kind, state, ccc, _, _, ap, config) and
+        revFlowOut(call, ret, pos, state, returnCtx, returnAp, ap, config) and
+        returnFlowsThrough(ret, pos, state, ccc, _, _, ap, config) and
         matchesCall(ccc, call)
       )
     }
@@ -1915,17 +1893,17 @@ private module MkStage<StageSig PrevStage> {
 
     pragma[nomagic]
     private predicate parameterFlowsThroughRev(
-      ParamNodeEx p, Ap ap, ReturnKindExt kind, Ap returnAp, Configuration config
+      ParamNodeEx p, Ap ap, ReturnPosition pos, Ap returnAp, Configuration config
     ) {
-      revFlow(p, _, TReturnCtxMaybeFlowThrough(kind), apSome(returnAp), ap, config) and
-      parameterFlowThroughAllowed(p, kind)
+      revFlow(p, _, TReturnCtxMaybeFlowThrough(pos), apSome(returnAp), ap, config) and
+      parameterFlowThroughAllowed(p, pos.getKind())
     }
 
     pragma[nomagic]
     predicate parameterMayFlowThrough(ParamNodeEx p, Ap ap, Configuration config) {
-      exists(RetNodeEx ret, ReturnKindExt kind |
-        returnFlowsThrough(ret, kind, _, _, p, ap, _, config) and
-        parameterFlowsThroughRev(p, ap, kind, _, config)
+      exists(ReturnPosition pos |
+        returnFlowsThrough(_, pos, _, _, p, ap, _, config) and
+        parameterFlowsThroughRev(p, ap, pos, _, config)
       )
     }
 
@@ -1933,20 +1911,21 @@ private module MkStage<StageSig PrevStage> {
     predicate returnMayFlowThrough(
       RetNodeEx ret, Ap argAp, Ap ap, ReturnKindExt kind, Configuration config
     ) {
-      exists(ParamNodeEx p |
-        returnFlowsThrough(ret, kind, _, _, p, argAp, ap, config) and
-        parameterFlowsThroughRev(p, argAp, kind, ap, config)
+      exists(ParamNodeEx p, ReturnPosition pos |
+        returnFlowsThrough(ret, pos, _, _, p, argAp, ap, config) and
+        parameterFlowsThroughRev(p, argAp, pos, ap, config) and
+        kind = pos.getKind()
       )
     }
 
     pragma[nomagic]
-    predicate revFlowInToReturnIsReturned(
+    private predicate revFlowThroughArg(
       DataFlowCall call, ArgNodeEx arg, FlowState state, ReturnCtx returnCtx, ApOption returnAp,
       Ap ap, Configuration config
     ) {
-      exists(ReturnKindExt returnKind0, Ap returnAp0 |
-        revFlowInToReturn(call, arg, state, returnKind0, returnAp0, ap, config) and
-        revFlowIsReturned(call, returnCtx, returnAp, returnKind0, returnAp0, config)
+      exists(ParamNodeEx p, Ap innerReturnAp |
+        revFlowThrough(call, returnCtx, p, state, _, returnAp, ap, innerReturnAp, config) and
+        flowThroughIntoCall(call, arg, p, _, ap, innerReturnAp, config)
       )
     }
 
@@ -1954,7 +1933,7 @@ private module MkStage<StageSig PrevStage> {
     predicate callMayFlowThroughRev(DataFlowCall call, Configuration config) {
       exists(ArgNodeEx arg, FlowState state, ReturnCtx returnCtx, ApOption returnAp, Ap ap |
         revFlow(arg, state, returnCtx, returnAp, ap, config) and
-        revFlowInToReturnIsReturned(call, arg, state, returnCtx, returnAp, ap, config)
+        revFlowThroughArg(call, arg, state, returnCtx, returnAp, ap, config)
       )
     }
 
@@ -1967,8 +1946,9 @@ private module MkStage<StageSig PrevStage> {
       conscand = count(TypedContent f0, Ap ap | fwdConsCand(f0, ap, config)) and
       states = count(FlowState state | fwdFlow(_, state, _, _, _, _, config)) and
       tuples =
-        count(NodeEx n, FlowState state, Cc cc, ParameterPositionOption summaryCtx, ApOption argAp,
-          Ap ap | fwdFlow(n, state, cc, summaryCtx, argAp, ap, config))
+        count(NodeEx n, FlowState state, Cc cc, ParamNodeOption summaryCtx, ApOption argAp, Ap ap |
+          fwdFlow(n, state, cc, summaryCtx, argAp, ap, config)
+        )
       or
       fwd = false and
       nodes = count(NodeEx node | revFlow(node, _, _, _, _, config)) and
@@ -2823,13 +2803,12 @@ private Configuration unbindConf(Configuration conf) {
 
 pragma[nomagic]
 private predicate nodeMayUseSummary0(
-  NodeEx n, DataFlowCallable c, ParameterPosition pos, FlowState state, AccessPathApprox apa,
-  Configuration config
+  NodeEx n, ParamNodeEx p, FlowState state, AccessPathApprox apa, Configuration config
 ) {
   exists(AccessPathApprox apa0 |
-    c = n.getEnclosingCallable() and
+    Stage5::parameterMayFlowThrough(p, _, _) and
     Stage5::revFlow(n, state, TReturnCtxMaybeFlowThrough(_), _, apa0, config) and
-    Stage5::fwdFlow(n, state, any(CallContextCall ccc), TParameterPositionSome(pos),
+    Stage5::fwdFlow(n, state, any(CallContextCall ccc), TParamNodeSome(p.asNode()),
       TAccessPathApproxSome(apa), apa0, config)
   )
 }
@@ -2838,10 +2817,9 @@ pragma[nomagic]
 private predicate nodeMayUseSummary(
   NodeEx n, FlowState state, AccessPathApprox apa, Configuration config
 ) {
-  exists(DataFlowCallable c, ParameterPosition pos, ParamNodeEx p |
+  exists(ParamNodeEx p |
     Stage5::parameterMayFlowThrough(p, apa, config) and
-    nodeMayUseSummary0(n, c, pos, state, apa, config) and
-    p.isParameterOf(c, pos)
+    nodeMayUseSummary0(n, p, state, apa, config)
   )
 }
 
@@ -3771,8 +3749,8 @@ private predicate paramFlowsThrough(
   ReturnKindExt kind, FlowState state, CallContextCall cc, SummaryCtxSome sc, AccessPath ap,
   AccessPathApprox apa, Configuration config
 ) {
-  exists(PathNodeMid mid, RetNodeEx ret |
-    pathNode(mid, ret, state, cc, sc, ap, config, _) and
+  exists(RetNodeEx ret |
+    pathNode(_, ret, state, cc, sc, ap, config, _) and
     kind = ret.getKind() and
     apa = ap.getApprox() and
     parameterFlowThroughAllowed(sc.getParamNode(), kind)
@@ -4234,17 +4212,15 @@ private module FlowExploration {
       ap = TRevPartialNil() and
       exists(config.explorationLimit())
       or
-      exists(PartialPathNodeRev mid |
-        revPartialPathStep(mid, node, state, sc1, sc2, sc3, ap, config) and
-        not clearsContentEx(node, ap.getHead()) and
-        (
-          notExpectsContent(node) or
-          expectsContentEx(node, ap.getHead())
-        ) and
-        not fullBarrier(node, config) and
-        not stateBarrier(node, state, config) and
-        distSink(node.getEnclosingCallable(), config) <= config.explorationLimit()
-      )
+      revPartialPathStep(_, node, state, sc1, sc2, sc3, ap, config) and
+      not clearsContentEx(node, ap.getHead()) and
+      (
+        notExpectsContent(node) or
+        expectsContentEx(node, ap.getHead())
+      ) and
+      not fullBarrier(node, config) and
+      not stateBarrier(node, state, config) and
+      distSink(node.getEnclosingCallable(), config) <= config.explorationLimit()
     }
 
   pragma[nomagic]
@@ -4252,19 +4228,17 @@ private module FlowExploration {
     NodeEx node, FlowState state, CallContext cc, TSummaryCtx1 sc1, TSummaryCtx2 sc2,
     TSummaryCtx3 sc3, PartialAccessPath ap, Configuration config
   ) {
-    exists(PartialPathNodeFwd mid |
-      partialPathStep(mid, node, state, cc, sc1, sc2, sc3, ap, config) and
-      not fullBarrier(node, config) and
-      not stateBarrier(node, state, config) and
-      not clearsContentEx(node, ap.getHead().getContent()) and
-      (
-        notExpectsContent(node) or
-        expectsContentEx(node, ap.getHead().getContent())
-      ) and
-      if node.asNode() instanceof CastingNode
-      then compatibleTypes(node.getDataFlowType(), ap.getType())
-      else any()
-    )
+    partialPathStep(_, node, state, cc, sc1, sc2, sc3, ap, config) and
+    not fullBarrier(node, config) and
+    not stateBarrier(node, state, config) and
+    not clearsContentEx(node, ap.getHead().getContent()) and
+    (
+      notExpectsContent(node) or
+      expectsContentEx(node, ap.getHead().getContent())
+    ) and
+    if node.asNode() instanceof CastingNode
+    then compatibleTypes(node.getDataFlowType(), ap.getType())
+    else any()
   }
 
   /**

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl3.qll

@@ -622,7 +622,11 @@ private predicate parameterFlowThroughAllowed(ParamNodeEx p, ReturnKindExt kind)
 }
 
 private module Stage1 implements StageSig {
-  class Ap = Unit;
+  class Ap extends int {
+    // workaround for bad functionality-induced joins (happens when using `Unit`)
+    pragma[nomagic]
+    Ap() { this in [0 .. 1] and this < 1 }
+  }
 
   private class Cc = boolean;
 
@@ -872,9 +876,9 @@ private module Stage1 implements StageSig {
 
   pragma[nomagic]
   private predicate revFlowOut(ReturnPosition pos, Configuration config) {
-    exists(DataFlowCall call, NodeEx out |
+    exists(NodeEx out |
       revFlow(out, _, config) and
-      viableReturnPosOutNodeCandFwd1(call, pos, out, config)
+      viableReturnPosOutNodeCandFwd1(_, pos, out, config)
     )
   }
 
@@ -1327,8 +1331,8 @@ private module MkStage<StageSig PrevStage> {
      */
     pragma[nomagic]
     additional predicate fwdFlow(
-      NodeEx node, FlowState state, Cc cc, ParameterPositionOption summaryCtx, ApOption argAp,
-      Ap ap, ApApprox apa, Configuration config
+      NodeEx node, FlowState state, Cc cc, ParamNodeOption summaryCtx, ApOption argAp, Ap ap,
+      ApApprox apa, Configuration config
     ) {
       fwdFlow0(node, state, cc, summaryCtx, argAp, ap, apa, config) and
       PrevStage::revFlow(node, state, apa, config) and
@@ -1337,21 +1341,21 @@ private module MkStage<StageSig PrevStage> {
 
     pragma[inline]
     additional predicate fwdFlow(
-      NodeEx node, FlowState state, Cc cc, ParameterPositionOption summaryCtx, ApOption argAp,
-      Ap ap, Configuration config
+      NodeEx node, FlowState state, Cc cc, ParamNodeOption summaryCtx, ApOption argAp, Ap ap,
+      Configuration config
     ) {
       fwdFlow(node, state, cc, summaryCtx, argAp, ap, _, config)
     }
 
     pragma[nomagic]
     private predicate fwdFlow0(
-      NodeEx node, FlowState state, Cc cc, ParameterPositionOption summaryCtx, ApOption argAp,
-      Ap ap, ApApprox apa, Configuration config
+      NodeEx node, FlowState state, Cc cc, ParamNodeOption summaryCtx, ApOption argAp, Ap ap,
+      ApApprox apa, Configuration config
     ) {
       sourceNode(node, state, config) and
       (if hasSourceCallCtx(config) then cc = ccSomeCall() else cc = ccNone()) and
       argAp = apNone() and
-      summaryCtx = TParameterPositionNone() and
+      summaryCtx = TParamNodeNone() and
       ap = getApNil(node) and
       apa = getApprox(ap)
       or
@@ -1372,7 +1376,7 @@ private module MkStage<StageSig PrevStage> {
         fwdFlow(mid, pragma[only_bind_into](state), _, _, _, ap, apa, pragma[only_bind_into](config)) and
         jumpStep(mid, node, config) and
         cc = ccNone() and
-        summaryCtx = TParameterPositionNone() and
+        summaryCtx = TParamNodeNone() and
         argAp = apNone()
       )
       or
@@ -1380,7 +1384,7 @@ private module MkStage<StageSig PrevStage> {
         fwdFlow(mid, state, _, _, _, nil, pragma[only_bind_into](config)) and
         additionalJumpStep(mid, node, config) and
         cc = ccNone() and
-        summaryCtx = TParameterPositionNone() and
+        summaryCtx = TParamNodeNone() and
         argAp = apNone() and
         ap = getApNil(node) and
         apa = getApprox(ap)
@@ -1390,7 +1394,7 @@ private module MkStage<StageSig PrevStage> {
         fwdFlow(mid, state0, _, _, _, nil, pragma[only_bind_into](config)) and
         additionalJumpStateStep(mid, state0, node, state, config) and
         cc = ccNone() and
-        summaryCtx = TParameterPositionNone() and
+        summaryCtx = TParamNodeNone() and
         argAp = apNone() and
         ap = getApNil(node) and
         apa = getApprox(ap)
@@ -1414,10 +1418,10 @@ private module MkStage<StageSig PrevStage> {
       fwdFlowIn(_, node, state, _, cc, _, _, ap, apa, config) and
       if PrevStage::parameterMayFlowThrough(node, apa, config)
       then (
-        summaryCtx = TParameterPositionSome(node.(ParamNodeEx).getPosition()) and
+        summaryCtx = TParamNodeSome(node.asNode()) and
         argAp = apSome(ap)
       ) else (
-        summaryCtx = TParameterPositionNone() and argAp = apNone()
+        summaryCtx = TParamNodeNone() and argAp = apNone()
       )
       or
       // flow out of a callable
@@ -1433,16 +1437,19 @@ private module MkStage<StageSig PrevStage> {
       )
       or
       // flow through a callable
-      exists(DataFlowCall call, ParameterPosition summaryCtx0, Ap argAp0 |
-        fwdFlowOutFromArg(call, node, state, summaryCtx0, argAp0, ap, apa, config) and
-        fwdFlowIsEntered(call, cc, summaryCtx, argAp, summaryCtx0, argAp0, config)
+      exists(
+        DataFlowCall call, CcCall ccc, RetNodeEx ret, boolean allowsFieldFlow, ApApprox innerArgApa
+      |
+        fwdFlowThrough(call, cc, state, ccc, summaryCtx, argAp, ap, apa, ret, innerArgApa, config) and
+        flowThroughOutOfCall(call, ccc, ret, node, allowsFieldFlow, innerArgApa, apa, config) and
+        if allowsFieldFlow = false then ap instanceof ApNil else any()
       )
     }
 
     pragma[nomagic]
     private predicate fwdFlowStore(
       NodeEx node1, Ap ap1, TypedContent tc, NodeEx node2, FlowState state, Cc cc,
-      ParameterPositionOption summaryCtx, ApOption argAp, Configuration config
+      ParamNodeOption summaryCtx, ApOption argAp, Configuration config
     ) {
       exists(DataFlowType contentType, ApApprox apa1 |
         fwdFlow(node1, state, cc, summaryCtx, argAp, ap1, apa1, config) and
@@ -1473,8 +1480,8 @@ private module MkStage<StageSig PrevStage> {
 
     pragma[nomagic]
     private predicate fwdFlowRead0(
-      NodeEx node1, FlowState state, Cc cc, ParameterPositionOption summaryCtx, ApOption argAp,
-      ApNonNil ap, Configuration config
+      NodeEx node1, FlowState state, Cc cc, ParamNodeOption summaryCtx, ApOption argAp, ApNonNil ap,
+      Configuration config
     ) {
       fwdFlow(node1, state, cc, summaryCtx, argAp, ap, config) and
       PrevStage::readStepCand(node1, _, _, config)
@@ -1483,7 +1490,7 @@ private module MkStage<StageSig PrevStage> {
     pragma[nomagic]
     private predicate fwdFlowRead(
       Ap ap, Content c, NodeEx node1, NodeEx node2, FlowState state, Cc cc,
-      ParameterPositionOption summaryCtx, ApOption argAp, Configuration config
+      ParamNodeOption summaryCtx, ApOption argAp, Configuration config
     ) {
       fwdFlowRead0(node1, state, cc, summaryCtx, argAp, ap, config) and
       PrevStage::readStepCand(node1, c, node2, config) and
@@ -1493,7 +1500,7 @@ private module MkStage<StageSig PrevStage> {
     pragma[nomagic]
     private predicate fwdFlowIn(
       DataFlowCall call, ParamNodeEx p, FlowState state, Cc outercc, CcCall innercc,
-      ParameterPositionOption summaryCtx, ApOption argAp, Ap ap, ApApprox apa, Configuration config
+      ParamNodeOption summaryCtx, ApOption argAp, Ap ap, ApApprox apa, Configuration config
     ) {
       exists(ArgNodeEx arg, boolean allowsFieldFlow |
         fwdFlow(arg, state, outercc, summaryCtx, argAp, ap, apa, config) and
@@ -1505,64 +1512,38 @@ private module MkStage<StageSig PrevStage> {
 
     pragma[nomagic]
     private predicate fwdFlowRetFromArg(
-      RetNodeEx ret, FlowState state, CcCall ccc, ParameterPosition summaryCtx, ParamNodeEx p,
-      Ap argAp, ApApprox argApa, Ap ap, ApApprox apa, Configuration config
+      RetNodeEx ret, FlowState state, CcCall ccc, ParamNodeEx summaryCtx, Ap argAp, ApApprox argApa,
+      Ap ap, ApApprox apa, Configuration config
     ) {
-      exists(DataFlowCallable c, ReturnKindExt kind |
+      exists(ReturnKindExt kind |
         fwdFlow(pragma[only_bind_into](ret), state, ccc,
-          TParameterPositionSome(pragma[only_bind_into](summaryCtx)), apSome(argAp), ap, apa, config) and
-        getApprox(argAp) = argApa and
-        c = ret.getEnclosingCallable() and
+          TParamNodeSome(pragma[only_bind_into](summaryCtx.asNode())),
+          pragma[only_bind_into](apSome(argAp)), ap, pragma[only_bind_into](apa),
+          pragma[only_bind_into](config)) and
         kind = ret.getKind() and
-        p.isParameterOf(c, pragma[only_bind_into](summaryCtx)) and
-        parameterFlowThroughAllowed(p, kind)
+        parameterFlowThroughAllowed(summaryCtx, kind) and
+        argApa = getApprox(argAp) and
+        PrevStage::returnMayFlowThrough(ret, argApa, apa, kind, pragma[only_bind_into](config))
       )
     }
 
     pragma[inline]
-    private predicate fwdFlowInMayFlowThrough(
-      DataFlowCall call, Cc cc, CcCall innerCc, ParameterPositionOption summaryCtx, ApOption argAp,
-      ParamNodeEx param, Ap ap, ApApprox apa, Configuration config
+    private predicate fwdFlowThrough0(
+      DataFlowCall call, Cc cc, FlowState state, CcCall ccc, ParamNodeOption summaryCtx,
+      ApOption argAp, Ap ap, ApApprox apa, RetNodeEx ret, ParamNodeEx innerSummaryCtx,
+      Ap innerArgAp, ApApprox innerArgApa, Configuration config
     ) {
-      fwdFlowIn(call, pragma[only_bind_into](param), _, cc, innerCc, summaryCtx, argAp, ap,
-        pragma[only_bind_into](apa), pragma[only_bind_into](config)) and
-      PrevStage::parameterMayFlowThrough(param, apa, config)
-    }
-
-    // dedup before joining with `flowThroughOutOfCall`
-    pragma[nomagic]
-    private predicate fwdFlowInMayFlowThroughProj(
-      DataFlowCall call, CcCall innerCc, ApApprox apa, Configuration config
-    ) {
-      fwdFlowInMayFlowThrough(call, _, innerCc, _, _, _, _, apa, config)
-    }
-
-    /**
-     * Same as `flowThroughOutOfCall`, but restricted to calls that are reached
-     * in the flow covered by `fwdFlow`, where data might flow through the target
-     * callable and back out at `call`.
-     */
-    pragma[nomagic]
-    private predicate fwdFlowThroughOutOfCall(
-      DataFlowCall call, CcCall ccc, RetNodeEx ret, NodeEx out, boolean allowsFieldFlow,
-      ApApprox argApa, ApApprox apa, Configuration config
-    ) {
-      fwdFlowInMayFlowThroughProj(call, ccc, argApa, config) and
-      flowThroughOutOfCall(call, ccc, ret, out, allowsFieldFlow, argApa, apa, config)
+      fwdFlowRetFromArg(ret, state, ccc, innerSummaryCtx, innerArgAp, innerArgApa, ap, apa, config) and
+      fwdFlowIsEntered(call, cc, ccc, summaryCtx, argAp, innerSummaryCtx, innerArgAp, config)
     }
 
     pragma[nomagic]
-    private predicate fwdFlowOutFromArg(
-      DataFlowCall call, NodeEx out, FlowState state, ParameterPosition summaryCtx, Ap argAp, Ap ap,
-      ApApprox apa, Configuration config
+    private predicate fwdFlowThrough(
+      DataFlowCall call, Cc cc, FlowState state, CcCall ccc, ParamNodeOption summaryCtx,
+      ApOption argAp, Ap ap, ApApprox apa, RetNodeEx ret, ApApprox innerArgApa, Configuration config
     ) {
-      exists(RetNodeEx ret, boolean allowsFieldFlow, CcCall ccc, ApApprox argApa |
-        fwdFlowRetFromArg(pragma[only_bind_into](ret), state, pragma[only_bind_into](ccc),
-          summaryCtx, _, argAp, pragma[only_bind_into](argApa), ap, pragma[only_bind_into](apa),
-          config) and
-        fwdFlowThroughOutOfCall(call, ccc, ret, out, allowsFieldFlow, argApa, apa, config) and
-        (if allowsFieldFlow = false then ap instanceof ApNil else any())
-      )
+      fwdFlowThrough0(call, cc, state, ccc, summaryCtx, argAp, ap, apa, ret, _, _, innerArgApa,
+        config)
     }
 
     /**
@@ -1571,12 +1552,14 @@ private module MkStage<StageSig PrevStage> {
      */
     pragma[nomagic]
     private predicate fwdFlowIsEntered(
-      DataFlowCall call, Cc cc, ParameterPositionOption summaryCtx, ApOption argAp,
-      ParameterPosition pos, Ap ap, Configuration config
+      DataFlowCall call, Cc cc, CcCall innerCc, ParamNodeOption summaryCtx, ApOption argAp,
+      ParamNodeEx p, Ap ap, Configuration config
     ) {
-      exists(ParamNodeEx param |
-        fwdFlowInMayFlowThrough(call, cc, _, summaryCtx, argAp, param, ap, _, config) and
-        pos = param.getPosition()
+      exists(ApApprox apa |
+        fwdFlowIn(call, pragma[only_bind_into](p), _, cc, innerCc, summaryCtx, argAp, ap,
+          pragma[only_bind_into](apa), pragma[only_bind_into](config)) and
+        PrevStage::parameterMayFlowThrough(p, apa, config) and
+        PrevStage::callMayFlowThroughRev(call, pragma[only_bind_into](config))
       )
     }
 
@@ -1596,23 +1579,31 @@ private module MkStage<StageSig PrevStage> {
       fwdFlowConsCand(ap1, c, ap2, config)
     }
 
+    pragma[nomagic]
+    private predicate returnFlowsThrough0(
+      DataFlowCall call, FlowState state, CcCall ccc, Ap ap, ApApprox apa, RetNodeEx ret,
+      ParamNodeEx innerSummaryCtx, Ap innerArgAp, ApApprox innerArgApa, Configuration config
+    ) {
+      fwdFlowThrough0(call, _, state, ccc, _, _, ap, apa, ret, innerSummaryCtx, innerArgAp,
+        innerArgApa, config)
+    }
+
     pragma[nomagic]
     private predicate returnFlowsThrough(
-      RetNodeEx ret, ReturnKindExt kind, FlowState state, CcCall ccc, ParamNodeEx p, Ap argAp,
+      RetNodeEx ret, ReturnPosition pos, FlowState state, CcCall ccc, ParamNodeEx p, Ap argAp,
       Ap ap, Configuration config
     ) {
-      exists(boolean allowsFieldFlow, ApApprox argApa, ApApprox apa |
-        fwdFlowRetFromArg(pragma[only_bind_into](ret), state, pragma[only_bind_into](ccc), _, p,
-          argAp, pragma[only_bind_into](argApa), ap, pragma[only_bind_into](apa), config) and
-        kind = ret.getKind() and
-        fwdFlowThroughOutOfCall(_, ccc, ret, _, allowsFieldFlow, argApa, apa, config) and
-        (if allowsFieldFlow = false then ap instanceof ApNil else any())
+      exists(DataFlowCall call, ApApprox apa, boolean allowsFieldFlow, ApApprox innerArgApa |
+        returnFlowsThrough0(call, state, ccc, ap, apa, ret, p, argAp, innerArgApa, config) and
+        flowThroughOutOfCall(call, ccc, ret, _, allowsFieldFlow, innerArgApa, apa, config) and
+        pos = ret.getReturnPosition() and
+        if allowsFieldFlow = false then ap instanceof ApNil else any()
       )
     }
 
     pragma[nomagic]
     private predicate flowThroughIntoCall(
-      DataFlowCall call, ArgNodeEx arg, ParamNodeEx p, boolean allowsFieldFlow, Ap argAp,
+      DataFlowCall call, ArgNodeEx arg, ParamNodeEx p, boolean allowsFieldFlow, Ap argAp, Ap ap,
       Configuration config
     ) {
       exists(ApApprox argApa |
@@ -1620,7 +1611,7 @@ private module MkStage<StageSig PrevStage> {
           allowsFieldFlow, argApa, pragma[only_bind_into](config)) and
         fwdFlow(arg, _, _, _, _, pragma[only_bind_into](argAp), argApa,
           pragma[only_bind_into](config)) and
-        returnFlowsThrough(_, _, _, _, p, pragma[only_bind_into](argAp), _,
+        returnFlowsThrough(_, _, _, _, p, pragma[only_bind_into](argAp), ap,
           pragma[only_bind_into](config)) and
         if allowsFieldFlow = false then argAp instanceof ApNil else any()
       )
@@ -1639,12 +1630,13 @@ private module MkStage<StageSig PrevStage> {
 
     pragma[nomagic]
     private predicate flowOutOfCallAp(
-      DataFlowCall call, RetNodeEx ret, ReturnKindExt kind, NodeEx out, boolean allowsFieldFlow,
+      DataFlowCall call, RetNodeEx ret, ReturnPosition pos, NodeEx out, boolean allowsFieldFlow,
       Ap ap, Configuration config
     ) {
       exists(ApApprox apa |
-        flowOutOfCallApa(call, ret, kind, out, allowsFieldFlow, apa, config) and
-        fwdFlow(ret, _, _, _, _, ap, apa, config)
+        flowOutOfCallApa(call, ret, _, out, allowsFieldFlow, apa, config) and
+        fwdFlow(ret, _, _, _, _, ap, apa, config) and
+        pos = ret.getReturnPosition()
       )
     }
 
@@ -1739,17 +1731,17 @@ private module MkStage<StageSig PrevStage> {
       )
       or
       // flow through a callable
-      exists(DataFlowCall call, ReturnKindExt returnKind0, Ap returnAp0 |
-        revFlowInToReturn(call, node, state, returnKind0, returnAp0, ap, config) and
-        revFlowIsReturned(call, returnCtx, returnAp, returnKind0, returnAp0, config)
+      exists(DataFlowCall call, ParamNodeEx p, Ap innerReturnAp |
+        revFlowThrough(call, returnCtx, p, state, _, returnAp, ap, innerReturnAp, config) and
+        flowThroughIntoCall(call, node, p, _, ap, innerReturnAp, config)
       )
       or
       // flow out of a callable
-      exists(ReturnKindExt kind |
-        revFlowOut(_, node, kind, state, _, _, ap, config) and
-        if returnFlowsThrough(node, kind, state, _, _, _, ap, config)
+      exists(ReturnPosition pos |
+        revFlowOut(_, node, pos, state, _, _, ap, config) and
+        if returnFlowsThrough(node, pos, state, _, _, _, ap, config)
         then (
-          returnCtx = TReturnCtxMaybeFlowThrough(kind) and
+          returnCtx = TReturnCtxMaybeFlowThrough(pos) and
           returnAp = apSome(ap)
         ) else (
           returnCtx = TReturnCtxNoFlowThrough() and returnAp = apNone()
@@ -1782,47 +1774,33 @@ private module MkStage<StageSig PrevStage> {
 
     pragma[nomagic]
     private predicate revFlowOut(
-      DataFlowCall call, RetNodeEx ret, ReturnKindExt kind, FlowState state, ReturnCtx returnCtx,
+      DataFlowCall call, RetNodeEx ret, ReturnPosition pos, FlowState state, ReturnCtx returnCtx,
       ApOption returnAp, Ap ap, Configuration config
     ) {
       exists(NodeEx out, boolean allowsFieldFlow |
         revFlow(out, state, returnCtx, returnAp, ap, config) and
-        flowOutOfCallAp(call, ret, kind, out, allowsFieldFlow, ap, config) and
+        flowOutOfCallAp(call, ret, pos, out, allowsFieldFlow, ap, config) and
         if allowsFieldFlow = false then ap instanceof ApNil else any()
       )
     }
 
-    /**
-     * Same as `flowThroughIntoCall`, but restricted to calls that are reached
-     * in the flow covered by `revFlow`, where data might flow through the target
-     * callable and back out at `call`.
-     */
-    pragma[nomagic]
-    private predicate revFlowThroughIntoCall(
-      DataFlowCall call, ArgNodeEx arg, ParamNodeEx p, boolean allowsFieldFlow, Ap argAp,
-      Configuration config
-    ) {
-      flowThroughIntoCall(call, arg, p, allowsFieldFlow, argAp, config) and
-      revFlowIsReturned(call, _, _, _, _, config)
-    }
-
     pragma[nomagic]
     private predicate revFlowParamToReturn(
-      ParamNodeEx p, FlowState state, ReturnKindExt kind, Ap returnAp, Ap ap, Configuration config
+      ParamNodeEx p, FlowState state, ReturnPosition pos, Ap returnAp, Ap ap, Configuration config
     ) {
-      revFlow(p, state, TReturnCtxMaybeFlowThrough(kind), apSome(returnAp), ap, config) and
-      parameterFlowThroughAllowed(p, kind)
+      revFlow(pragma[only_bind_into](p), state, TReturnCtxMaybeFlowThrough(pos), apSome(returnAp),
+        pragma[only_bind_into](ap), pragma[only_bind_into](config)) and
+      parameterFlowThroughAllowed(p, pos.getKind()) and
+      PrevStage::parameterMayFlowThrough(p, getApprox(ap), config)
     }
 
     pragma[nomagic]
-    private predicate revFlowInToReturn(
-      DataFlowCall call, ArgNodeEx arg, FlowState state, ReturnKindExt kind, Ap returnAp, Ap ap,
-      Configuration config
+    private predicate revFlowThrough(
+      DataFlowCall call, ReturnCtx returnCtx, ParamNodeEx p, FlowState state, ReturnPosition pos,
+      ApOption returnAp, Ap ap, Ap innerReturnAp, Configuration config
     ) {
-      exists(ParamNodeEx p, boolean allowsFieldFlow |
-        revFlowParamToReturn(p, state, kind, returnAp, ap, config) and
-        revFlowThroughIntoCall(call, arg, p, allowsFieldFlow, ap, config)
-      )
+      revFlowParamToReturn(p, state, pos, innerReturnAp, ap, config) and
+      revFlowIsReturned(call, returnCtx, returnAp, pos, innerReturnAp, config)
     }
 
     /**
@@ -1832,12 +1810,12 @@ private module MkStage<StageSig PrevStage> {
      */
     pragma[nomagic]
     private predicate revFlowIsReturned(
-      DataFlowCall call, ReturnCtx returnCtx, ApOption returnAp, ReturnKindExt kind, Ap ap,
+      DataFlowCall call, ReturnCtx returnCtx, ApOption returnAp, ReturnPosition pos, Ap ap,
       Configuration config
     ) {
       exists(RetNodeEx ret, FlowState state, CcCall ccc |
-        revFlowOut(call, ret, kind, state, returnCtx, returnAp, ap, config) and
-        returnFlowsThrough(ret, kind, state, ccc, _, _, ap, config) and
+        revFlowOut(call, ret, pos, state, returnCtx, returnAp, ap, config) and
+        returnFlowsThrough(ret, pos, state, ccc, _, _, ap, config) and
         matchesCall(ccc, call)
       )
     }
@@ -1915,17 +1893,17 @@ private module MkStage<StageSig PrevStage> {
 
     pragma[nomagic]
     private predicate parameterFlowsThroughRev(
-      ParamNodeEx p, Ap ap, ReturnKindExt kind, Ap returnAp, Configuration config
+      ParamNodeEx p, Ap ap, ReturnPosition pos, Ap returnAp, Configuration config
     ) {
-      revFlow(p, _, TReturnCtxMaybeFlowThrough(kind), apSome(returnAp), ap, config) and
-      parameterFlowThroughAllowed(p, kind)
+      revFlow(p, _, TReturnCtxMaybeFlowThrough(pos), apSome(returnAp), ap, config) and
+      parameterFlowThroughAllowed(p, pos.getKind())
     }
 
     pragma[nomagic]
     predicate parameterMayFlowThrough(ParamNodeEx p, Ap ap, Configuration config) {
-      exists(RetNodeEx ret, ReturnKindExt kind |
-        returnFlowsThrough(ret, kind, _, _, p, ap, _, config) and
-        parameterFlowsThroughRev(p, ap, kind, _, config)
+      exists(ReturnPosition pos |
+        returnFlowsThrough(_, pos, _, _, p, ap, _, config) and
+        parameterFlowsThroughRev(p, ap, pos, _, config)
       )
     }
 
@@ -1933,20 +1911,21 @@ private module MkStage<StageSig PrevStage> {
     predicate returnMayFlowThrough(
       RetNodeEx ret, Ap argAp, Ap ap, ReturnKindExt kind, Configuration config
     ) {
-      exists(ParamNodeEx p |
-        returnFlowsThrough(ret, kind, _, _, p, argAp, ap, config) and
-        parameterFlowsThroughRev(p, argAp, kind, ap, config)
+      exists(ParamNodeEx p, ReturnPosition pos |
+        returnFlowsThrough(ret, pos, _, _, p, argAp, ap, config) and
+        parameterFlowsThroughRev(p, argAp, pos, ap, config) and
+        kind = pos.getKind()
       )
     }
 
     pragma[nomagic]
-    predicate revFlowInToReturnIsReturned(
+    private predicate revFlowThroughArg(
       DataFlowCall call, ArgNodeEx arg, FlowState state, ReturnCtx returnCtx, ApOption returnAp,
       Ap ap, Configuration config
     ) {
-      exists(ReturnKindExt returnKind0, Ap returnAp0 |
-        revFlowInToReturn(call, arg, state, returnKind0, returnAp0, ap, config) and
-        revFlowIsReturned(call, returnCtx, returnAp, returnKind0, returnAp0, config)
+      exists(ParamNodeEx p, Ap innerReturnAp |
+        revFlowThrough(call, returnCtx, p, state, _, returnAp, ap, innerReturnAp, config) and
+        flowThroughIntoCall(call, arg, p, _, ap, innerReturnAp, config)
       )
     }
 
@@ -1954,7 +1933,7 @@ private module MkStage<StageSig PrevStage> {
     predicate callMayFlowThroughRev(DataFlowCall call, Configuration config) {
       exists(ArgNodeEx arg, FlowState state, ReturnCtx returnCtx, ApOption returnAp, Ap ap |
         revFlow(arg, state, returnCtx, returnAp, ap, config) and
-        revFlowInToReturnIsReturned(call, arg, state, returnCtx, returnAp, ap, config)
+        revFlowThroughArg(call, arg, state, returnCtx, returnAp, ap, config)
       )
     }
 
@@ -1967,8 +1946,9 @@ private module MkStage<StageSig PrevStage> {
       conscand = count(TypedContent f0, Ap ap | fwdConsCand(f0, ap, config)) and
       states = count(FlowState state | fwdFlow(_, state, _, _, _, _, config)) and
       tuples =
-        count(NodeEx n, FlowState state, Cc cc, ParameterPositionOption summaryCtx, ApOption argAp,
-          Ap ap | fwdFlow(n, state, cc, summaryCtx, argAp, ap, config))
+        count(NodeEx n, FlowState state, Cc cc, ParamNodeOption summaryCtx, ApOption argAp, Ap ap |
+          fwdFlow(n, state, cc, summaryCtx, argAp, ap, config)
+        )
       or
       fwd = false and
       nodes = count(NodeEx node | revFlow(node, _, _, _, _, config)) and
@@ -2823,13 +2803,12 @@ private Configuration unbindConf(Configuration conf) {
 
 pragma[nomagic]
 private predicate nodeMayUseSummary0(
-  NodeEx n, DataFlowCallable c, ParameterPosition pos, FlowState state, AccessPathApprox apa,
-  Configuration config
+  NodeEx n, ParamNodeEx p, FlowState state, AccessPathApprox apa, Configuration config
 ) {
   exists(AccessPathApprox apa0 |
-    c = n.getEnclosingCallable() and
+    Stage5::parameterMayFlowThrough(p, _, _) and
     Stage5::revFlow(n, state, TReturnCtxMaybeFlowThrough(_), _, apa0, config) and
-    Stage5::fwdFlow(n, state, any(CallContextCall ccc), TParameterPositionSome(pos),
+    Stage5::fwdFlow(n, state, any(CallContextCall ccc), TParamNodeSome(p.asNode()),
       TAccessPathApproxSome(apa), apa0, config)
   )
 }
@@ -2838,10 +2817,9 @@ pragma[nomagic]
 private predicate nodeMayUseSummary(
   NodeEx n, FlowState state, AccessPathApprox apa, Configuration config
 ) {
-  exists(DataFlowCallable c, ParameterPosition pos, ParamNodeEx p |
+  exists(ParamNodeEx p |
     Stage5::parameterMayFlowThrough(p, apa, config) and
-    nodeMayUseSummary0(n, c, pos, state, apa, config) and
-    p.isParameterOf(c, pos)
+    nodeMayUseSummary0(n, p, state, apa, config)
   )
 }
 
@@ -3771,8 +3749,8 @@ private predicate paramFlowsThrough(
   ReturnKindExt kind, FlowState state, CallContextCall cc, SummaryCtxSome sc, AccessPath ap,
   AccessPathApprox apa, Configuration config
 ) {
-  exists(PathNodeMid mid, RetNodeEx ret |
-    pathNode(mid, ret, state, cc, sc, ap, config, _) and
+  exists(RetNodeEx ret |
+    pathNode(_, ret, state, cc, sc, ap, config, _) and
     kind = ret.getKind() and
     apa = ap.getApprox() and
     parameterFlowThroughAllowed(sc.getParamNode(), kind)
@@ -4234,17 +4212,15 @@ private module FlowExploration {
       ap = TRevPartialNil() and
       exists(config.explorationLimit())
       or
-      exists(PartialPathNodeRev mid |
-        revPartialPathStep(mid, node, state, sc1, sc2, sc3, ap, config) and
-        not clearsContentEx(node, ap.getHead()) and
-        (
-          notExpectsContent(node) or
-          expectsContentEx(node, ap.getHead())
-        ) and
-        not fullBarrier(node, config) and
-        not stateBarrier(node, state, config) and
-        distSink(node.getEnclosingCallable(), config) <= config.explorationLimit()
-      )
+      revPartialPathStep(_, node, state, sc1, sc2, sc3, ap, config) and
+      not clearsContentEx(node, ap.getHead()) and
+      (
+        notExpectsContent(node) or
+        expectsContentEx(node, ap.getHead())
+      ) and
+      not fullBarrier(node, config) and
+      not stateBarrier(node, state, config) and
+      distSink(node.getEnclosingCallable(), config) <= config.explorationLimit()
     }
 
   pragma[nomagic]
@@ -4252,19 +4228,17 @@ private module FlowExploration {
     NodeEx node, FlowState state, CallContext cc, TSummaryCtx1 sc1, TSummaryCtx2 sc2,
     TSummaryCtx3 sc3, PartialAccessPath ap, Configuration config
   ) {
-    exists(PartialPathNodeFwd mid |
-      partialPathStep(mid, node, state, cc, sc1, sc2, sc3, ap, config) and
-      not fullBarrier(node, config) and
-      not stateBarrier(node, state, config) and
-      not clearsContentEx(node, ap.getHead().getContent()) and
-      (
-        notExpectsContent(node) or
-        expectsContentEx(node, ap.getHead().getContent())
-      ) and
-      if node.asNode() instanceof CastingNode
-      then compatibleTypes(node.getDataFlowType(), ap.getType())
-      else any()
-    )
+    partialPathStep(_, node, state, cc, sc1, sc2, sc3, ap, config) and
+    not fullBarrier(node, config) and
+    not stateBarrier(node, state, config) and
+    not clearsContentEx(node, ap.getHead().getContent()) and
+    (
+      notExpectsContent(node) or
+      expectsContentEx(node, ap.getHead().getContent())
+    ) and
+    if node.asNode() instanceof CastingNode
+    then compatibleTypes(node.getDataFlowType(), ap.getType())
+    else any()
   }
 
   /**

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl4.qll

@@ -622,7 +622,11 @@ private predicate parameterFlowThroughAllowed(ParamNodeEx p, ReturnKindExt kind)
 }
 
 private module Stage1 implements StageSig {
-  class Ap = Unit;
+  class Ap extends int {
+    // workaround for bad functionality-induced joins (happens when using `Unit`)
+    pragma[nomagic]
+    Ap() { this in [0 .. 1] and this < 1 }
+  }
 
   private class Cc = boolean;
 
@@ -872,9 +876,9 @@ private module Stage1 implements StageSig {
 
   pragma[nomagic]
   private predicate revFlowOut(ReturnPosition pos, Configuration config) {
-    exists(DataFlowCall call, NodeEx out |
+    exists(NodeEx out |
       revFlow(out, _, config) and
-      viableReturnPosOutNodeCandFwd1(call, pos, out, config)
+      viableReturnPosOutNodeCandFwd1(_, pos, out, config)
     )
   }
 
@@ -1327,8 +1331,8 @@ private module MkStage<StageSig PrevStage> {
      */
     pragma[nomagic]
     additional predicate fwdFlow(
-      NodeEx node, FlowState state, Cc cc, ParameterPositionOption summaryCtx, ApOption argAp,
-      Ap ap, ApApprox apa, Configuration config
+      NodeEx node, FlowState state, Cc cc, ParamNodeOption summaryCtx, ApOption argAp, Ap ap,
+      ApApprox apa, Configuration config
     ) {
       fwdFlow0(node, state, cc, summaryCtx, argAp, ap, apa, config) and
       PrevStage::revFlow(node, state, apa, config) and
@@ -1337,21 +1341,21 @@ private module MkStage<StageSig PrevStage> {
 
     pragma[inline]
     additional predicate fwdFlow(
-      NodeEx node, FlowState state, Cc cc, ParameterPositionOption summaryCtx, ApOption argAp,
-      Ap ap, Configuration config
+      NodeEx node, FlowState state, Cc cc, ParamNodeOption summaryCtx, ApOption argAp, Ap ap,
+      Configuration config
     ) {
       fwdFlow(node, state, cc, summaryCtx, argAp, ap, _, config)
     }
 
     pragma[nomagic]
     private predicate fwdFlow0(
-      NodeEx node, FlowState state, Cc cc, ParameterPositionOption summaryCtx, ApOption argAp,
-      Ap ap, ApApprox apa, Configuration config
+      NodeEx node, FlowState state, Cc cc, ParamNodeOption summaryCtx, ApOption argAp, Ap ap,
+      ApApprox apa, Configuration config
     ) {
       sourceNode(node, state, config) and
       (if hasSourceCallCtx(config) then cc = ccSomeCall() else cc = ccNone()) and
       argAp = apNone() and
-      summaryCtx = TParameterPositionNone() and
+      summaryCtx = TParamNodeNone() and
       ap = getApNil(node) and
       apa = getApprox(ap)
       or
@@ -1372,7 +1376,7 @@ private module MkStage<StageSig PrevStage> {
         fwdFlow(mid, pragma[only_bind_into](state), _, _, _, ap, apa, pragma[only_bind_into](config)) and
         jumpStep(mid, node, config) and
         cc = ccNone() and
-        summaryCtx = TParameterPositionNone() and
+        summaryCtx = TParamNodeNone() and
         argAp = apNone()
       )
       or
@@ -1380,7 +1384,7 @@ private module MkStage<StageSig PrevStage> {
         fwdFlow(mid, state, _, _, _, nil, pragma[only_bind_into](config)) and
         additionalJumpStep(mid, node, config) and
         cc = ccNone() and
-        summaryCtx = TParameterPositionNone() and
+        summaryCtx = TParamNodeNone() and
         argAp = apNone() and
         ap = getApNil(node) and
         apa = getApprox(ap)
@@ -1390,7 +1394,7 @@ private module MkStage<StageSig PrevStage> {
         fwdFlow(mid, state0, _, _, _, nil, pragma[only_bind_into](config)) and
         additionalJumpStateStep(mid, state0, node, state, config) and
         cc = ccNone() and
-        summaryCtx = TParameterPositionNone() and
+        summaryCtx = TParamNodeNone() and
         argAp = apNone() and
         ap = getApNil(node) and
         apa = getApprox(ap)
@@ -1414,10 +1418,10 @@ private module MkStage<StageSig PrevStage> {
       fwdFlowIn(_, node, state, _, cc, _, _, ap, apa, config) and
       if PrevStage::parameterMayFlowThrough(node, apa, config)
       then (
-        summaryCtx = TParameterPositionSome(node.(ParamNodeEx).getPosition()) and
+        summaryCtx = TParamNodeSome(node.asNode()) and
         argAp = apSome(ap)
       ) else (
-        summaryCtx = TParameterPositionNone() and argAp = apNone()
+        summaryCtx = TParamNodeNone() and argAp = apNone()
       )
       or
       // flow out of a callable
@@ -1433,16 +1437,19 @@ private module MkStage<StageSig PrevStage> {
       )
       or
       // flow through a callable
-      exists(DataFlowCall call, ParameterPosition summaryCtx0, Ap argAp0 |
-        fwdFlowOutFromArg(call, node, state, summaryCtx0, argAp0, ap, apa, config) and
-        fwdFlowIsEntered(call, cc, summaryCtx, argAp, summaryCtx0, argAp0, config)
+      exists(
+        DataFlowCall call, CcCall ccc, RetNodeEx ret, boolean allowsFieldFlow, ApApprox innerArgApa
+      |
+        fwdFlowThrough(call, cc, state, ccc, summaryCtx, argAp, ap, apa, ret, innerArgApa, config) and
+        flowThroughOutOfCall(call, ccc, ret, node, allowsFieldFlow, innerArgApa, apa, config) and
+        if allowsFieldFlow = false then ap instanceof ApNil else any()
       )
     }
 
     pragma[nomagic]
     private predicate fwdFlowStore(
       NodeEx node1, Ap ap1, TypedContent tc, NodeEx node2, FlowState state, Cc cc,
-      ParameterPositionOption summaryCtx, ApOption argAp, Configuration config
+      ParamNodeOption summaryCtx, ApOption argAp, Configuration config
     ) {
       exists(DataFlowType contentType, ApApprox apa1 |
         fwdFlow(node1, state, cc, summaryCtx, argAp, ap1, apa1, config) and
@@ -1473,8 +1480,8 @@ private module MkStage<StageSig PrevStage> {
 
     pragma[nomagic]
     private predicate fwdFlowRead0(
-      NodeEx node1, FlowState state, Cc cc, ParameterPositionOption summaryCtx, ApOption argAp,
-      ApNonNil ap, Configuration config
+      NodeEx node1, FlowState state, Cc cc, ParamNodeOption summaryCtx, ApOption argAp, ApNonNil ap,
+      Configuration config
     ) {
       fwdFlow(node1, state, cc, summaryCtx, argAp, ap, config) and
       PrevStage::readStepCand(node1, _, _, config)
@@ -1483,7 +1490,7 @@ private module MkStage<StageSig PrevStage> {
     pragma[nomagic]
     private predicate fwdFlowRead(
       Ap ap, Content c, NodeEx node1, NodeEx node2, FlowState state, Cc cc,
-      ParameterPositionOption summaryCtx, ApOption argAp, Configuration config
+      ParamNodeOption summaryCtx, ApOption argAp, Configuration config
     ) {
       fwdFlowRead0(node1, state, cc, summaryCtx, argAp, ap, config) and
       PrevStage::readStepCand(node1, c, node2, config) and
@@ -1493,7 +1500,7 @@ private module MkStage<StageSig PrevStage> {
     pragma[nomagic]
     private predicate fwdFlowIn(
       DataFlowCall call, ParamNodeEx p, FlowState state, Cc outercc, CcCall innercc,
-      ParameterPositionOption summaryCtx, ApOption argAp, Ap ap, ApApprox apa, Configuration config
+      ParamNodeOption summaryCtx, ApOption argAp, Ap ap, ApApprox apa, Configuration config
     ) {
       exists(ArgNodeEx arg, boolean allowsFieldFlow |
         fwdFlow(arg, state, outercc, summaryCtx, argAp, ap, apa, config) and
@@ -1505,64 +1512,38 @@ private module MkStage<StageSig PrevStage> {
 
     pragma[nomagic]
     private predicate fwdFlowRetFromArg(
-      RetNodeEx ret, FlowState state, CcCall ccc, ParameterPosition summaryCtx, ParamNodeEx p,
-      Ap argAp, ApApprox argApa, Ap ap, ApApprox apa, Configuration config
+      RetNodeEx ret, FlowState state, CcCall ccc, ParamNodeEx summaryCtx, Ap argAp, ApApprox argApa,
+      Ap ap, ApApprox apa, Configuration config
     ) {
-      exists(DataFlowCallable c, ReturnKindExt kind |
+      exists(ReturnKindExt kind |
         fwdFlow(pragma[only_bind_into](ret), state, ccc,
-          TParameterPositionSome(pragma[only_bind_into](summaryCtx)), apSome(argAp), ap, apa, config) and
-        getApprox(argAp) = argApa and
-        c = ret.getEnclosingCallable() and
+          TParamNodeSome(pragma[only_bind_into](summaryCtx.asNode())),
+          pragma[only_bind_into](apSome(argAp)), ap, pragma[only_bind_into](apa),
+          pragma[only_bind_into](config)) and
         kind = ret.getKind() and
-        p.isParameterOf(c, pragma[only_bind_into](summaryCtx)) and
-        parameterFlowThroughAllowed(p, kind)
+        parameterFlowThroughAllowed(summaryCtx, kind) and
+        argApa = getApprox(argAp) and
+        PrevStage::returnMayFlowThrough(ret, argApa, apa, kind, pragma[only_bind_into](config))
       )
     }
 
     pragma[inline]
-    private predicate fwdFlowInMayFlowThrough(
-      DataFlowCall call, Cc cc, CcCall innerCc, ParameterPositionOption summaryCtx, ApOption argAp,
-      ParamNodeEx param, Ap ap, ApApprox apa, Configuration config
+    private predicate fwdFlowThrough0(
+      DataFlowCall call, Cc cc, FlowState state, CcCall ccc, ParamNodeOption summaryCtx,
+      ApOption argAp, Ap ap, ApApprox apa, RetNodeEx ret, ParamNodeEx innerSummaryCtx,
+      Ap innerArgAp, ApApprox innerArgApa, Configuration config
     ) {
-      fwdFlowIn(call, pragma[only_bind_into](param), _, cc, innerCc, summaryCtx, argAp, ap,
-        pragma[only_bind_into](apa), pragma[only_bind_into](config)) and
-      PrevStage::parameterMayFlowThrough(param, apa, config)
-    }
-
-    // dedup before joining with `flowThroughOutOfCall`
-    pragma[nomagic]
-    private predicate fwdFlowInMayFlowThroughProj(
-      DataFlowCall call, CcCall innerCc, ApApprox apa, Configuration config
-    ) {
-      fwdFlowInMayFlowThrough(call, _, innerCc, _, _, _, _, apa, config)
-    }
-
-    /**
-     * Same as `flowThroughOutOfCall`, but restricted to calls that are reached
-     * in the flow covered by `fwdFlow`, where data might flow through the target
-     * callable and back out at `call`.
-     */
-    pragma[nomagic]
-    private predicate fwdFlowThroughOutOfCall(
-      DataFlowCall call, CcCall ccc, RetNodeEx ret, NodeEx out, boolean allowsFieldFlow,
-      ApApprox argApa, ApApprox apa, Configuration config
-    ) {
-      fwdFlowInMayFlowThroughProj(call, ccc, argApa, config) and
-      flowThroughOutOfCall(call, ccc, ret, out, allowsFieldFlow, argApa, apa, config)
+      fwdFlowRetFromArg(ret, state, ccc, innerSummaryCtx, innerArgAp, innerArgApa, ap, apa, config) and
+      fwdFlowIsEntered(call, cc, ccc, summaryCtx, argAp, innerSummaryCtx, innerArgAp, config)
     }
 
     pragma[nomagic]
-    private predicate fwdFlowOutFromArg(
-      DataFlowCall call, NodeEx out, FlowState state, ParameterPosition summaryCtx, Ap argAp, Ap ap,
-      ApApprox apa, Configuration config
+    private predicate fwdFlowThrough(
+      DataFlowCall call, Cc cc, FlowState state, CcCall ccc, ParamNodeOption summaryCtx,
+      ApOption argAp, Ap ap, ApApprox apa, RetNodeEx ret, ApApprox innerArgApa, Configuration config
     ) {
-      exists(RetNodeEx ret, boolean allowsFieldFlow, CcCall ccc, ApApprox argApa |
-        fwdFlowRetFromArg(pragma[only_bind_into](ret), state, pragma[only_bind_into](ccc),
-          summaryCtx, _, argAp, pragma[only_bind_into](argApa), ap, pragma[only_bind_into](apa),
-          config) and
-        fwdFlowThroughOutOfCall(call, ccc, ret, out, allowsFieldFlow, argApa, apa, config) and
-        (if allowsFieldFlow = false then ap instanceof ApNil else any())
-      )
+      fwdFlowThrough0(call, cc, state, ccc, summaryCtx, argAp, ap, apa, ret, _, _, innerArgApa,
+        config)
     }
 
     /**
@@ -1571,12 +1552,14 @@ private module MkStage<StageSig PrevStage> {
      */
     pragma[nomagic]
     private predicate fwdFlowIsEntered(
-      DataFlowCall call, Cc cc, ParameterPositionOption summaryCtx, ApOption argAp,
-      ParameterPosition pos, Ap ap, Configuration config
+      DataFlowCall call, Cc cc, CcCall innerCc, ParamNodeOption summaryCtx, ApOption argAp,
+      ParamNodeEx p, Ap ap, Configuration config
     ) {
-      exists(ParamNodeEx param |
-        fwdFlowInMayFlowThrough(call, cc, _, summaryCtx, argAp, param, ap, _, config) and
-        pos = param.getPosition()
+      exists(ApApprox apa |
+        fwdFlowIn(call, pragma[only_bind_into](p), _, cc, innerCc, summaryCtx, argAp, ap,
+          pragma[only_bind_into](apa), pragma[only_bind_into](config)) and
+        PrevStage::parameterMayFlowThrough(p, apa, config) and
+        PrevStage::callMayFlowThroughRev(call, pragma[only_bind_into](config))
       )
     }
 
@@ -1596,23 +1579,31 @@ private module MkStage<StageSig PrevStage> {
       fwdFlowConsCand(ap1, c, ap2, config)
     }
 
+    pragma[nomagic]
+    private predicate returnFlowsThrough0(
+      DataFlowCall call, FlowState state, CcCall ccc, Ap ap, ApApprox apa, RetNodeEx ret,
+      ParamNodeEx innerSummaryCtx, Ap innerArgAp, ApApprox innerArgApa, Configuration config
+    ) {
+      fwdFlowThrough0(call, _, state, ccc, _, _, ap, apa, ret, innerSummaryCtx, innerArgAp,
+        innerArgApa, config)
+    }
+
     pragma[nomagic]
     private predicate returnFlowsThrough(
-      RetNodeEx ret, ReturnKindExt kind, FlowState state, CcCall ccc, ParamNodeEx p, Ap argAp,
+      RetNodeEx ret, ReturnPosition pos, FlowState state, CcCall ccc, ParamNodeEx p, Ap argAp,
       Ap ap, Configuration config
     ) {
-      exists(boolean allowsFieldFlow, ApApprox argApa, ApApprox apa |
-        fwdFlowRetFromArg(pragma[only_bind_into](ret), state, pragma[only_bind_into](ccc), _, p,
-          argAp, pragma[only_bind_into](argApa), ap, pragma[only_bind_into](apa), config) and
-        kind = ret.getKind() and
-        fwdFlowThroughOutOfCall(_, ccc, ret, _, allowsFieldFlow, argApa, apa, config) and
-        (if allowsFieldFlow = false then ap instanceof ApNil else any())
+      exists(DataFlowCall call, ApApprox apa, boolean allowsFieldFlow, ApApprox innerArgApa |
+        returnFlowsThrough0(call, state, ccc, ap, apa, ret, p, argAp, innerArgApa, config) and
+        flowThroughOutOfCall(call, ccc, ret, _, allowsFieldFlow, innerArgApa, apa, config) and
+        pos = ret.getReturnPosition() and
+        if allowsFieldFlow = false then ap instanceof ApNil else any()
       )
     }
 
     pragma[nomagic]
     private predicate flowThroughIntoCall(
-      DataFlowCall call, ArgNodeEx arg, ParamNodeEx p, boolean allowsFieldFlow, Ap argAp,
+      DataFlowCall call, ArgNodeEx arg, ParamNodeEx p, boolean allowsFieldFlow, Ap argAp, Ap ap,
       Configuration config
     ) {
       exists(ApApprox argApa |
@@ -1620,7 +1611,7 @@ private module MkStage<StageSig PrevStage> {
           allowsFieldFlow, argApa, pragma[only_bind_into](config)) and
         fwdFlow(arg, _, _, _, _, pragma[only_bind_into](argAp), argApa,
           pragma[only_bind_into](config)) and
-        returnFlowsThrough(_, _, _, _, p, pragma[only_bind_into](argAp), _,
+        returnFlowsThrough(_, _, _, _, p, pragma[only_bind_into](argAp), ap,
           pragma[only_bind_into](config)) and
         if allowsFieldFlow = false then argAp instanceof ApNil else any()
       )
@@ -1639,12 +1630,13 @@ private module MkStage<StageSig PrevStage> {
 
     pragma[nomagic]
     private predicate flowOutOfCallAp(
-      DataFlowCall call, RetNodeEx ret, ReturnKindExt kind, NodeEx out, boolean allowsFieldFlow,
+      DataFlowCall call, RetNodeEx ret, ReturnPosition pos, NodeEx out, boolean allowsFieldFlow,
       Ap ap, Configuration config
     ) {
       exists(ApApprox apa |
-        flowOutOfCallApa(call, ret, kind, out, allowsFieldFlow, apa, config) and
-        fwdFlow(ret, _, _, _, _, ap, apa, config)
+        flowOutOfCallApa(call, ret, _, out, allowsFieldFlow, apa, config) and
+        fwdFlow(ret, _, _, _, _, ap, apa, config) and
+        pos = ret.getReturnPosition()
       )
     }
 
@@ -1739,17 +1731,17 @@ private module MkStage<StageSig PrevStage> {
       )
       or
       // flow through a callable
-      exists(DataFlowCall call, ReturnKindExt returnKind0, Ap returnAp0 |
-        revFlowInToReturn(call, node, state, returnKind0, returnAp0, ap, config) and
-        revFlowIsReturned(call, returnCtx, returnAp, returnKind0, returnAp0, config)
+      exists(DataFlowCall call, ParamNodeEx p, Ap innerReturnAp |
+        revFlowThrough(call, returnCtx, p, state, _, returnAp, ap, innerReturnAp, config) and
+        flowThroughIntoCall(call, node, p, _, ap, innerReturnAp, config)
       )
       or
       // flow out of a callable
-      exists(ReturnKindExt kind |
-        revFlowOut(_, node, kind, state, _, _, ap, config) and
-        if returnFlowsThrough(node, kind, state, _, _, _, ap, config)
+      exists(ReturnPosition pos |
+        revFlowOut(_, node, pos, state, _, _, ap, config) and
+        if returnFlowsThrough(node, pos, state, _, _, _, ap, config)
         then (
-          returnCtx = TReturnCtxMaybeFlowThrough(kind) and
+          returnCtx = TReturnCtxMaybeFlowThrough(pos) and
           returnAp = apSome(ap)
         ) else (
           returnCtx = TReturnCtxNoFlowThrough() and returnAp = apNone()
@@ -1782,47 +1774,33 @@ private module MkStage<StageSig PrevStage> {
 
     pragma[nomagic]
     private predicate revFlowOut(
-      DataFlowCall call, RetNodeEx ret, ReturnKindExt kind, FlowState state, ReturnCtx returnCtx,
+      DataFlowCall call, RetNodeEx ret, ReturnPosition pos, FlowState state, ReturnCtx returnCtx,
       ApOption returnAp, Ap ap, Configuration config
     ) {
       exists(NodeEx out, boolean allowsFieldFlow |
         revFlow(out, state, returnCtx, returnAp, ap, config) and
-        flowOutOfCallAp(call, ret, kind, out, allowsFieldFlow, ap, config) and
+        flowOutOfCallAp(call, ret, pos, out, allowsFieldFlow, ap, config) and
         if allowsFieldFlow = false then ap instanceof ApNil else any()
       )
     }
 
-    /**
-     * Same as `flowThroughIntoCall`, but restricted to calls that are reached
-     * in the flow covered by `revFlow`, where data might flow through the target
-     * callable and back out at `call`.
-     */
-    pragma[nomagic]
-    private predicate revFlowThroughIntoCall(
-      DataFlowCall call, ArgNodeEx arg, ParamNodeEx p, boolean allowsFieldFlow, Ap argAp,
-      Configuration config
-    ) {
-      flowThroughIntoCall(call, arg, p, allowsFieldFlow, argAp, config) and
-      revFlowIsReturned(call, _, _, _, _, config)
-    }
-
     pragma[nomagic]
     private predicate revFlowParamToReturn(
-      ParamNodeEx p, FlowState state, ReturnKindExt kind, Ap returnAp, Ap ap, Configuration config
+      ParamNodeEx p, FlowState state, ReturnPosition pos, Ap returnAp, Ap ap, Configuration config
     ) {
-      revFlow(p, state, TReturnCtxMaybeFlowThrough(kind), apSome(returnAp), ap, config) and
-      parameterFlowThroughAllowed(p, kind)
+      revFlow(pragma[only_bind_into](p), state, TReturnCtxMaybeFlowThrough(pos), apSome(returnAp),
+        pragma[only_bind_into](ap), pragma[only_bind_into](config)) and
+      parameterFlowThroughAllowed(p, pos.getKind()) and
+      PrevStage::parameterMayFlowThrough(p, getApprox(ap), config)
     }
 
     pragma[nomagic]
-    private predicate revFlowInToReturn(
-      DataFlowCall call, ArgNodeEx arg, FlowState state, ReturnKindExt kind, Ap returnAp, Ap ap,
-      Configuration config
+    private predicate revFlowThrough(
+      DataFlowCall call, ReturnCtx returnCtx, ParamNodeEx p, FlowState state, ReturnPosition pos,
+      ApOption returnAp, Ap ap, Ap innerReturnAp, Configuration config
     ) {
-      exists(ParamNodeEx p, boolean allowsFieldFlow |
-        revFlowParamToReturn(p, state, kind, returnAp, ap, config) and
-        revFlowThroughIntoCall(call, arg, p, allowsFieldFlow, ap, config)
-      )
+      revFlowParamToReturn(p, state, pos, innerReturnAp, ap, config) and
+      revFlowIsReturned(call, returnCtx, returnAp, pos, innerReturnAp, config)
     }
 
     /**
@@ -1832,12 +1810,12 @@ private module MkStage<StageSig PrevStage> {
      */
     pragma[nomagic]
     private predicate revFlowIsReturned(
-      DataFlowCall call, ReturnCtx returnCtx, ApOption returnAp, ReturnKindExt kind, Ap ap,
+      DataFlowCall call, ReturnCtx returnCtx, ApOption returnAp, ReturnPosition pos, Ap ap,
       Configuration config
     ) {
       exists(RetNodeEx ret, FlowState state, CcCall ccc |
-        revFlowOut(call, ret, kind, state, returnCtx, returnAp, ap, config) and
-        returnFlowsThrough(ret, kind, state, ccc, _, _, ap, config) and
+        revFlowOut(call, ret, pos, state, returnCtx, returnAp, ap, config) and
+        returnFlowsThrough(ret, pos, state, ccc, _, _, ap, config) and
         matchesCall(ccc, call)
       )
     }
@@ -1915,17 +1893,17 @@ private module MkStage<StageSig PrevStage> {
 
     pragma[nomagic]
     private predicate parameterFlowsThroughRev(
-      ParamNodeEx p, Ap ap, ReturnKindExt kind, Ap returnAp, Configuration config
+      ParamNodeEx p, Ap ap, ReturnPosition pos, Ap returnAp, Configuration config
     ) {
-      revFlow(p, _, TReturnCtxMaybeFlowThrough(kind), apSome(returnAp), ap, config) and
-      parameterFlowThroughAllowed(p, kind)
+      revFlow(p, _, TReturnCtxMaybeFlowThrough(pos), apSome(returnAp), ap, config) and
+      parameterFlowThroughAllowed(p, pos.getKind())
     }
 
     pragma[nomagic]
     predicate parameterMayFlowThrough(ParamNodeEx p, Ap ap, Configuration config) {
-      exists(RetNodeEx ret, ReturnKindExt kind |
-        returnFlowsThrough(ret, kind, _, _, p, ap, _, config) and
-        parameterFlowsThroughRev(p, ap, kind, _, config)
+      exists(ReturnPosition pos |
+        returnFlowsThrough(_, pos, _, _, p, ap, _, config) and
+        parameterFlowsThroughRev(p, ap, pos, _, config)
       )
     }
 
@@ -1933,20 +1911,21 @@ private module MkStage<StageSig PrevStage> {
     predicate returnMayFlowThrough(
       RetNodeEx ret, Ap argAp, Ap ap, ReturnKindExt kind, Configuration config
     ) {
-      exists(ParamNodeEx p |
-        returnFlowsThrough(ret, kind, _, _, p, argAp, ap, config) and
-        parameterFlowsThroughRev(p, argAp, kind, ap, config)
+      exists(ParamNodeEx p, ReturnPosition pos |
+        returnFlowsThrough(ret, pos, _, _, p, argAp, ap, config) and
+        parameterFlowsThroughRev(p, argAp, pos, ap, config) and
+        kind = pos.getKind()
       )
     }
 
     pragma[nomagic]
-    predicate revFlowInToReturnIsReturned(
+    private predicate revFlowThroughArg(
       DataFlowCall call, ArgNodeEx arg, FlowState state, ReturnCtx returnCtx, ApOption returnAp,
       Ap ap, Configuration config
     ) {
-      exists(ReturnKindExt returnKind0, Ap returnAp0 |
-        revFlowInToReturn(call, arg, state, returnKind0, returnAp0, ap, config) and
-        revFlowIsReturned(call, returnCtx, returnAp, returnKind0, returnAp0, config)
+      exists(ParamNodeEx p, Ap innerReturnAp |
+        revFlowThrough(call, returnCtx, p, state, _, returnAp, ap, innerReturnAp, config) and
+        flowThroughIntoCall(call, arg, p, _, ap, innerReturnAp, config)
       )
     }
 
@@ -1954,7 +1933,7 @@ private module MkStage<StageSig PrevStage> {
     predicate callMayFlowThroughRev(DataFlowCall call, Configuration config) {
       exists(ArgNodeEx arg, FlowState state, ReturnCtx returnCtx, ApOption returnAp, Ap ap |
         revFlow(arg, state, returnCtx, returnAp, ap, config) and
-        revFlowInToReturnIsReturned(call, arg, state, returnCtx, returnAp, ap, config)
+        revFlowThroughArg(call, arg, state, returnCtx, returnAp, ap, config)
       )
     }
 
@@ -1967,8 +1946,9 @@ private module MkStage<StageSig PrevStage> {
       conscand = count(TypedContent f0, Ap ap | fwdConsCand(f0, ap, config)) and
       states = count(FlowState state | fwdFlow(_, state, _, _, _, _, config)) and
       tuples =
-        count(NodeEx n, FlowState state, Cc cc, ParameterPositionOption summaryCtx, ApOption argAp,
-          Ap ap | fwdFlow(n, state, cc, summaryCtx, argAp, ap, config))
+        count(NodeEx n, FlowState state, Cc cc, ParamNodeOption summaryCtx, ApOption argAp, Ap ap |
+          fwdFlow(n, state, cc, summaryCtx, argAp, ap, config)
+        )
       or
       fwd = false and
       nodes = count(NodeEx node | revFlow(node, _, _, _, _, config)) and
@@ -2823,13 +2803,12 @@ private Configuration unbindConf(Configuration conf) {
 
 pragma[nomagic]
 private predicate nodeMayUseSummary0(
-  NodeEx n, DataFlowCallable c, ParameterPosition pos, FlowState state, AccessPathApprox apa,
-  Configuration config
+  NodeEx n, ParamNodeEx p, FlowState state, AccessPathApprox apa, Configuration config
 ) {
   exists(AccessPathApprox apa0 |
-    c = n.getEnclosingCallable() and
+    Stage5::parameterMayFlowThrough(p, _, _) and
     Stage5::revFlow(n, state, TReturnCtxMaybeFlowThrough(_), _, apa0, config) and
-    Stage5::fwdFlow(n, state, any(CallContextCall ccc), TParameterPositionSome(pos),
+    Stage5::fwdFlow(n, state, any(CallContextCall ccc), TParamNodeSome(p.asNode()),
       TAccessPathApproxSome(apa), apa0, config)
   )
 }
@@ -2838,10 +2817,9 @@ pragma[nomagic]
 private predicate nodeMayUseSummary(
   NodeEx n, FlowState state, AccessPathApprox apa, Configuration config
 ) {
-  exists(DataFlowCallable c, ParameterPosition pos, ParamNodeEx p |
+  exists(ParamNodeEx p |
     Stage5::parameterMayFlowThrough(p, apa, config) and
-    nodeMayUseSummary0(n, c, pos, state, apa, config) and
-    p.isParameterOf(c, pos)
+    nodeMayUseSummary0(n, p, state, apa, config)
   )
 }
 
@@ -3771,8 +3749,8 @@ private predicate paramFlowsThrough(
   ReturnKindExt kind, FlowState state, CallContextCall cc, SummaryCtxSome sc, AccessPath ap,
   AccessPathApprox apa, Configuration config
 ) {
-  exists(PathNodeMid mid, RetNodeEx ret |
-    pathNode(mid, ret, state, cc, sc, ap, config, _) and
+  exists(RetNodeEx ret |
+    pathNode(_, ret, state, cc, sc, ap, config, _) and
     kind = ret.getKind() and
     apa = ap.getApprox() and
     parameterFlowThroughAllowed(sc.getParamNode(), kind)
@@ -4234,17 +4212,15 @@ private module FlowExploration {
       ap = TRevPartialNil() and
       exists(config.explorationLimit())
       or
-      exists(PartialPathNodeRev mid |
-        revPartialPathStep(mid, node, state, sc1, sc2, sc3, ap, config) and
-        not clearsContentEx(node, ap.getHead()) and
-        (
-          notExpectsContent(node) or
-          expectsContentEx(node, ap.getHead())
-        ) and
-        not fullBarrier(node, config) and
-        not stateBarrier(node, state, config) and
-        distSink(node.getEnclosingCallable(), config) <= config.explorationLimit()
-      )
+      revPartialPathStep(_, node, state, sc1, sc2, sc3, ap, config) and
+      not clearsContentEx(node, ap.getHead()) and
+      (
+        notExpectsContent(node) or
+        expectsContentEx(node, ap.getHead())
+      ) and
+      not fullBarrier(node, config) and
+      not stateBarrier(node, state, config) and
+      distSink(node.getEnclosingCallable(), config) <= config.explorationLimit()
     }
 
   pragma[nomagic]
@@ -4252,19 +4228,17 @@ private module FlowExploration {
     NodeEx node, FlowState state, CallContext cc, TSummaryCtx1 sc1, TSummaryCtx2 sc2,
     TSummaryCtx3 sc3, PartialAccessPath ap, Configuration config
   ) {
-    exists(PartialPathNodeFwd mid |
-      partialPathStep(mid, node, state, cc, sc1, sc2, sc3, ap, config) and
-      not fullBarrier(node, config) and
-      not stateBarrier(node, state, config) and
-      not clearsContentEx(node, ap.getHead().getContent()) and
-      (
-        notExpectsContent(node) or
-        expectsContentEx(node, ap.getHead().getContent())
-      ) and
-      if node.asNode() instanceof CastingNode
-      then compatibleTypes(node.getDataFlowType(), ap.getType())
-      else any()
-    )
+    partialPathStep(_, node, state, cc, sc1, sc2, sc3, ap, config) and
+    not fullBarrier(node, config) and
+    not stateBarrier(node, state, config) and
+    not clearsContentEx(node, ap.getHead().getContent()) and
+    (
+      notExpectsContent(node) or
+      expectsContentEx(node, ap.getHead().getContent())
+    ) and
+    if node.asNode() instanceof CastingNode
+    then compatibleTypes(node.getDataFlowType(), ap.getType())
+    else any()
   }
 
   /**

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplCommon.qll

@@ -916,15 +916,15 @@ private module Cached {
     TDataFlowCallSome(DataFlowCall call)
 
   cached
-  newtype TParameterPositionOption =
-    TParameterPositionNone() or
-    TParameterPositionSome(ParameterPosition pos)
+  newtype TParamNodeOption =
+    TParamNodeNone() or
+    TParamNodeSome(ParamNode p)
 
   cached
   newtype TReturnCtx =
     TReturnCtxNone() or
     TReturnCtxNoFlowThrough() or
-    TReturnCtxMaybeFlowThrough(ReturnKindExt kind)
+    TReturnCtxMaybeFlowThrough(ReturnPosition pos)
 
   cached
   newtype TTypedContentApprox =
@@ -1343,15 +1343,15 @@ class DataFlowCallOption extends TDataFlowCallOption {
   }
 }
 
-/** An optional `ParameterPosition`. */
-class ParameterPositionOption extends TParameterPositionOption {
+/** An optional `ParamNode`. */
+class ParamNodeOption extends TParamNodeOption {
   string toString() {
-    this = TParameterPositionNone() and
+    this = TParamNodeNone() and
     result = "(none)"
     or
-    exists(ParameterPosition pos |
-      this = TParameterPositionSome(pos) and
-      result = pos.toString()
+    exists(ParamNode p |
+      this = TParamNodeSome(p) and
+      result = p.toString()
     )
   }
 }
@@ -1363,7 +1363,7 @@ class ParameterPositionOption extends TParameterPositionOption {
  *
  * - `TReturnCtxNone()`: no return flow.
  * - `TReturnCtxNoFlowThrough()`: return flow, but flow through is not possible.
- * - `TReturnCtxMaybeFlowThrough(ReturnKindExt kind)`: return flow, of kind `kind`, and
+ * - `TReturnCtxMaybeFlowThrough(ReturnPosition pos)`: return flow, of kind `pos`, and
  *    flow through may be possible.
  */
 class ReturnCtx extends TReturnCtx {
@@ -1374,9 +1374,9 @@ class ReturnCtx extends TReturnCtx {
     this = TReturnCtxNoFlowThrough() and
     result = "(no flow through)"
     or
-    exists(ReturnKindExt kind |
-      this = TReturnCtxMaybeFlowThrough(kind) and
-      result = kind.toString()
+    exists(ReturnPosition pos |
+      this = TReturnCtxMaybeFlowThrough(pos) and
+      result = pos.toString()
     )
   }
 }

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplConsistency.qll

@@ -101,9 +101,7 @@ module Consistency {
     exists(int c |
       c =
         strictcount(Node n |
-          not exists(string filepath, int startline, int startcolumn, int endline, int endcolumn |
-            n.hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn)
-          ) and
+          not n.hasLocationInfo(_, _, _, _, _) and
           not any(ConsistencyConfiguration conf).missingLocationExclude(n)
         ) and
       msg = "Nodes without location: " + c

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPrivate.qll

@@ -819,9 +819,9 @@ predicate attributeReadStep(Node nodeFrom, AttributeContent c, AttrRead nodeTo)
  * synthesized unpacked argument with the name indicated by `c`.
  */
 predicate kwUnpackReadStep(CfgNode nodeFrom, DictionaryElementContent c, Node nodeTo) {
-  exists(CallNode call, CallableValue callable, string name |
+  exists(CallNode call, string name |
     nodeFrom.asCfgNode() = call.getNode().getKwargs().getAFlowNode() and
-    nodeTo = TKwUnpackedNode(call, callable, name) and
+    nodeTo = TKwUnpackedNode(call, _, name) and
     name = c.getKey()
   )
 }

python/ql/lib/semmle/python/dataflow/new/internal/FlowSummaryImpl.qll

@@ -241,9 +241,15 @@ module Public {
     }
 
     /**
-     * Holds if the summary is auto generated.
+     * Holds if the summary is auto generated and not manually generated.
      */
     predicate isAutoGenerated() { none() }
+
+    /**
+     * Holds if the summary has the given provenance where `true` is
+     * `generated` and `false` is `manual`.
+     */
+    predicate hasProvenance(boolean generated) { none() }
   }
 
   /** A callable where there is no flow via the callable. */
@@ -254,6 +260,12 @@ module Public {
      *  Holds if the neutral is auto generated.
      */
     predicate isAutoGenerated() { neutralElement(this, true) }
+
+    /**
+     * Holds if the neutral has the given provenance where `true` is
+     * `generated` and `false` is `manual`.
+     */
+    predicate hasProvenance(boolean generated) { neutralElement(this, generated) }
   }
 }
 
@@ -568,10 +580,8 @@ module Private {
           head = TWithContentSummaryComponent(cont)
         )
         or
-        exists(ContentSet cont |
-          head = TWithoutContentSummaryComponent(cont) and
-          result = getNodeType(summaryNodeInputState(c, s.tail()))
-        )
+        head = TWithoutContentSummaryComponent(_) and
+        result = getNodeType(summaryNodeInputState(c, s.tail()))
         or
         exists(ReturnKind rk |
           head = TReturnSummaryComponent(rk) and
@@ -646,8 +656,8 @@ module Private {
 
   /** Holds if summary node `ret` is a return node of kind `rk`. */
   predicate summaryReturnNode(Node ret, ReturnKind rk) {
-    exists(SummarizedCallable callable, SummaryComponentStack s |
-      ret = summaryNodeOutputState(callable, s) and
+    exists(SummaryComponentStack s |
+      ret = summaryNodeOutputState(_, s) and
       s = TSingletonSummaryComponentStack(TReturnSummaryComponent(rk))
     )
   }
@@ -1012,6 +1022,10 @@ module Private {
       }
 
       override predicate isAutoGenerated() { this.relevantSummaryElementGenerated(_, _, _) }
+
+      override predicate hasProvenance(boolean generated) {
+        summaryElement(this, _, _, _, generated)
+      }
     }
 
     /** Holds if component `c` of specification `spec` cannot be parsed. */

python/ql/lib/semmle/python/dataflow/old/Implementation.qll

@@ -531,8 +531,8 @@ class TaintTrackingImplementation extends string instanceof TaintTracking::Confi
     TaintTrackingNode src, DataFlow::Node node, TaintTrackingContext context, AttributePath path,
     TaintKind kind, string edgeLabel
   ) {
-    exists(DataFlow::Node srcnode, CallNode call, TaintKind srckind, string name |
-      src = TTaintTrackingNode_(srcnode, context, path, srckind, this) and
+    exists(CallNode call, TaintKind srckind, string name |
+      src = TTaintTrackingNode_(_, context, path, srckind, this) and
       call.getFunction().(AttrNode).getObject(name) = src.getNode().asCfgNode() and
       kind = srckind.getTaintOfMethodResult(name) and
       node.asCfgNode() = call
@@ -560,8 +560,8 @@ class TaintTrackingImplementation extends string instanceof TaintTracking::Confi
     TaintTrackingNode src, DataFlow::Node node, TaintTrackingContext context, AttributePath path,
     TaintKind kind, string edgeLabel
   ) {
-    exists(CallNode call, PythonFunctionObjectInternal pyfunc, int arg |
-      this.callWithTaintedArgument(src, call, _, pyfunc, arg, path, kind) and
+    exists(PythonFunctionObjectInternal pyfunc, int arg |
+      this.callWithTaintedArgument(src, _, _, pyfunc, arg, path, kind) and
       node.asCfgNode() = pyfunc.getParameter(arg) and
       context = TParamContext(kind, path, arg)
     ) and
@@ -795,11 +795,9 @@ private class EssaTaintTracking extends string instanceof TaintTracking::Configu
     TaintTrackingNode src, PyEdgeRefinement defn, TaintTrackingContext context, AttributePath path,
     TaintKind kind
   ) {
-    exists(DataFlow::Node srcnode, ControlFlowNode use |
-      src = TTaintTrackingNode_(srcnode, context, path, kind, this) and
-      not super.isBarrierTest(defn.getTest(), defn.getSense()) and
-      defn.getSense() = this.testEvaluates(defn, defn.getTest(), use, src)
-    )
+    src = TTaintTrackingNode_(_, context, path, kind, this) and
+    not super.isBarrierTest(defn.getTest(), defn.getSense()) and
+    defn.getSense() = this.testEvaluates(defn, defn.getTest(), _, src)
   }
 
   pragma[noinline]

python/ql/lib/semmle/python/dataflow/old/StateTracking.qll

@@ -99,9 +99,7 @@ module StateTracking {
         exists(int n |
           f = b.getNode(n) and
           appliesToNode(state, b.getNode(n - 1), ctx, sense) and
-          not exists(PythonFunctionObjectInternal func, Context callee |
-            callee.fromCall(f, func, ctx)
-          )
+          not exists(Context callee | callee.fromCall(f, _, ctx))
         )
       )
       or

python/ql/lib/semmle/python/essa/Definitions.qll

@@ -69,6 +69,8 @@ abstract class SsaSourceVariable extends @py_variable {
     or
     SsaSource::exception_capture(this, def)
     or
+    SsaSource::exception_group_capture(this, def)
+    or
     SsaSource::with_definition(this, def)
     or
     SsaSource::pattern_capture_definition(this, def)

python/ql/lib/semmle/python/essa/Essa.qll

@@ -108,7 +108,7 @@ private string location_string(EssaVariable v) {
 
 /* Helper to compute an index for this SSA variable. */
 private int var_index(EssaVariable v) {
-  location_string(v) = rank[result](string s | exists(EssaVariable x | location_string(x) = s) | s)
+  location_string(v) = rank[result](string s | location_string(_) = s | s)
 }
 
 /* Helper for `v.getRepresentation()` */
@@ -511,12 +511,16 @@ class AssignmentDefinition extends EssaNodeDefinition {
   override string getAPrimaryQlClass() { result = "AssignmentDefinition" }
 }
 
-/** A capture of a raised exception `except ExceptionType ex:` */
+/** A capture of a raised exception `except ExceptionType as ex:` */
 class ExceptionCapture extends EssaNodeDefinition {
   ExceptionCapture() {
     SsaSource::exception_capture(this.getSourceVariable(), this.getDefiningNode())
   }
 
+  /**
+   * Gets the type handled by this exception handler
+   * `ExceptionType` in `except ExceptionType as ex:`.
+   */
   ControlFlowNode getType() {
     exists(ExceptFlowNode ex |
       ex.getName() = this.getDefiningNode() and
@@ -529,6 +533,28 @@ class ExceptionCapture extends EssaNodeDefinition {
   override string getAPrimaryQlClass() { result = "ExceptionCapture" }
 }
 
+/** A capture of a raised exception group `except* ExceptionType as ex:` */
+class ExceptionGroupCapture extends EssaNodeDefinition {
+  ExceptionGroupCapture() {
+    SsaSource::exception_group_capture(this.getSourceVariable(), this.getDefiningNode())
+  }
+
+  /**
+   * Gets the type handled by this exception handler
+   * `ExceptionType` in `except* ExceptionType as ex:`.
+   */
+  ControlFlowNode getType() {
+    exists(ExceptGroupFlowNode ex |
+      ex.getName() = this.getDefiningNode() and
+      result = ex.getType()
+    )
+  }
+
+  override string getRepresentation() { result = "except* " + this.getSourceVariable().getName() }
+
+  override string getAPrimaryQlClass() { result = "ExceptionGroupCapture" }
+}
+
 /** An assignment to a variable as part of a multiple assignment `..., v, ... = val` */
 class MultiAssignmentDefinition extends EssaNodeDefinition {
   MultiAssignmentDefinition() {

python/ql/lib/semmle/python/essa/SsaDefinitions.qll

@@ -30,6 +30,13 @@ module SsaSource {
     exists(ExceptFlowNode ex | ex.getName() = defn)
   }
 
+  /** Holds if `v` is defined by assignment of the captured exception group. */
+  cached
+  predicate exception_group_capture(Variable v, NameNode defn) {
+    defn.defines(v) and
+    exists(ExceptGroupFlowNode ex | ex.getName() = defn)
+  }
+
   /** Holds if `v` is defined by a with statement. */
   cached
   predicate with_definition(Variable v, ControlFlowNode defn) {

python/ql/lib/semmle/python/objects/Constants.qll

@@ -35,10 +35,8 @@ abstract class ConstantObjectInternal extends ObjectInternal {
   pragma[noinline]
   override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) {
     PointsToInternal::attributeRequired(pragma[only_bind_into](this), pragma[only_bind_into](name)) and
-    exists(ObjectInternal cls_attr, CfgOrigin attr_orig |
-      this.getClass()
-          .(ClassObjectInternal)
-          .lookup(pragma[only_bind_into](name), cls_attr, attr_orig) and
+    exists(ObjectInternal cls_attr |
+      this.getClass().(ClassObjectInternal).lookup(pragma[only_bind_into](name), cls_attr, _) and
       cls_attr.isDescriptor() = true and
       cls_attr.descriptorGetInstance(this, value, origin)
     )

python/ql/lib/semmle/python/objects/Instances.qll

@@ -246,8 +246,8 @@ class SelfInstanceInternal extends TSelfInstance, InstanceObject {
 
   pragma[noinline]
   override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) {
-    exists(AttrNode attr, ClassObjectInternal cls |
-      receiver_type(attr, name, this, cls) and
+    exists(ClassObjectInternal cls |
+      receiver_type(_, name, this, cls) and
       cls_descriptor(cls, name, descriptor)
     ) and
     instance = this
@@ -349,8 +349,8 @@ class UnknownInstanceInternal extends TUnknownInstance, ObjectInternal {
 
   pragma[noinline]
   override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) {
-    exists(AttrNode attr, ClassObjectInternal cls |
-      receiver_type(attr, name, this, cls) and
+    exists(ClassObjectInternal cls |
+      receiver_type(_, name, this, cls) and
       cls_descriptor(cls, name, descriptor)
     ) and
     instance = this

python/ql/lib/semmle/python/objects/ObjectAPI.qll

@@ -674,7 +674,7 @@ abstract class FunctionValue extends CallableValue {
   /** Gets the maximum number of parameters that can be correctly passed to this function */
   abstract int maxParameters();
 
-  predicate isOverridingMethod() { exists(Value f | this.overrides(f)) }
+  predicate isOverridingMethod() { this.overrides(_) }
 
   predicate isOverriddenMethod() { exists(Value f | f.overrides(this)) }
 

python/ql/lib/semmle/python/pointsto/PointsTo.qll

@@ -1026,8 +1026,8 @@ module InterProceduralPointsTo {
     ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin
   ) {
     def.isSelf() and
-    exists(CallNode call, BoundMethodObjectInternal method, Function func, PointsToContext caller |
-      callWithContext(call, caller, method, context) and
+    exists(BoundMethodObjectInternal method, Function func |
+      callWithContext(_, _, method, context) and
       func = method.getScope() and
       def.getScope() = func and
       value = method.getSelf() and
@@ -1062,8 +1062,8 @@ module InterProceduralPointsTo {
   private predicate context_for_default_value(ParameterDefinition def, PointsToContext context) {
     context.isRuntime() and exists(def)
     or
-    exists(PointsToContext caller, CallNode call, PythonFunctionObjectInternal func, int n |
-      context.fromCall(call, func, caller) and
+    exists(CallNode call, PythonFunctionObjectInternal func, int n |
+      context.fromCall(call, func, _) and
       func.getScope().getArg(n) = def.getParameter() and
       not exists(call.getArg(n)) and
       not exists(call.getArgByName(def.getVariable().getName())) and
@@ -1184,9 +1184,7 @@ module InterProceduralPointsTo {
     PointsToContext callee
   ) {
     PointsToInternal::pointsTo(argument, caller, _, _) and
-    exists(CallNode call, Function func, int offset |
-      callsite_calls_function(call, caller, func, callee, offset)
-    |
+    exists(CallNode call, Function func | callsite_calls_function(call, caller, func, callee, _) |
       exists(string name |
         argument = call.getArgByName(name) and
         function_parameter_name(func, param, name)
@@ -1460,10 +1458,9 @@ module Expressions {
     SubscriptNode subscr, PointsToContext context, ObjectInternal value, ControlFlowNode obj,
     ObjectInternal objvalue
   ) {
-    exists(ControlFlowNode index | subscriptObjectAndIndex(subscr, context, obj, objvalue, index) |
-      objvalue.subscriptUnknown() and
-      value = ObjectInternal::unknown()
-    )
+    subscriptObjectAndIndex(subscr, context, obj, objvalue, _) and
+    objvalue.subscriptUnknown() and
+    value = ObjectInternal::unknown()
     or
     exists(int n |
       subscriptObjectAndIndexPointsToInt(subscr, context, obj, objvalue, n) and
@@ -1567,9 +1564,9 @@ module Expressions {
       b.operands(other, op, operand)
     |
       op instanceof BitOr and
-      exists(ObjectInternal obj, int i1, int i2 |
+      exists(int i1, int i2 |
         pointsToInt(operand, context, opvalue, i1) and
-        pointsToInt(other, context, obj, i2) and
+        pointsToInt(other, context, _, i2) and
         value = TInt(i1.bitOr(i2))
       )
     )
@@ -2071,7 +2068,7 @@ module Expressions {
     exists(ObjectInternal sup_or_tuple |
       issubclass_call(_, _, _, sub, sup_or_tuple) and sub.isClass() = true
       or
-      exists(ObjectInternal val | isinstance_call(_, _, _, val, sub, sup_or_tuple))
+      isinstance_call(_, _, _, _, sub, sup_or_tuple)
     |
       sup = sup_or_tuple
       or
@@ -2759,8 +2756,8 @@ module ModuleAttributes {
     )
     or
     /* Retain value held before import */
-    exists(ModuleObjectInternal mod, EssaVariable input |
-      importStarDef(def, input, mod) and
+    exists(ModuleObjectInternal mod |
+      importStarDef(def, _, mod) and
       (InterModulePointsTo::moduleExportsBoolean(mod, name) = false or name.charAt(0) = "_") and
       attributePointsTo(def.getInput(), name, value, origin)
     )
@@ -2787,8 +2784,8 @@ module ModuleAttributes {
     CallsiteRefinement def, string name, ObjectInternal value, CfgOrigin origin
   ) {
     def.getVariable().isMetaVariable() and
-    exists(EssaVariable var, Function func, PointsToContext callee |
-      InterProceduralPointsTo::callsite_calls_function(def.getCall(), _, func, callee, _) and
+    exists(EssaVariable var, Function func |
+      InterProceduralPointsTo::callsite_calls_function(def.getCall(), _, func, _, _) and
       var = moduleStateVariable(func.getANormalExit()) and
       attributePointsTo(var, name, value, origin)
     )

python/ql/lib/semmle/python/regex.qll

@@ -994,10 +994,8 @@ abstract class RegexString extends Expr {
       this.specialCharacter(end, y, ["$", "\\Z"])
     )
     or
-    exists(int x |
-      this.lastPart(x, end) and
-      this.item(start, end)
-    )
+    this.lastPart(_, end) and
+    this.item(start, end)
     or
     exists(int y | this.lastPart(start, y) | this.qualifiedPart(start, end, y, _, _))
     or

python/ql/lib/semmle/python/security/regexp/HostnameRegex.qll

@@ -0,0 +1,18 @@
+/**
+ * Provides predicates for reasoning about regular expressions
+ * that match URLs and hostname patterns.
+ */
+
+private import python
+private import semmle.python.dataflow.new.DataFlow
+private import semmle.python.RegexTreeView::RegexTreeView as TreeImpl
+private import semmle.python.dataflow.new.Regexp as Regexp
+private import codeql.regex.HostnameRegexp as Shared
+
+private module Impl implements Shared::HostnameRegexpSig<TreeImpl> {
+  class DataFlowNode = DataFlow::Node;
+
+  class RegExpPatternSource = Regexp::RegExpPatternSource;
+}
+
+import Shared::Make<TreeImpl, Impl>

python/ql/lib/semmle/python/types/Exceptions.qll

@@ -367,6 +367,10 @@ predicate scope_raises_unknown(Scope s) {
 class ExceptFlowNode extends ControlFlowNode {
   ExceptFlowNode() { this.getNode() instanceof ExceptStmt }
 
+  /**
+   * Gets the type handled by this exception handler.
+   * `ExceptionType` in `except ExceptionType as e:`
+   */
   ControlFlowNode getType() {
     exists(ExceptStmt ex |
       this.getBasicBlock().dominates(result.getBasicBlock()) and
@@ -375,6 +379,10 @@ class ExceptFlowNode extends ControlFlowNode {
     )
   }
 
+  /**
+   * Gets the name assigned to the handled exception, if any.
+   * `e` in `except ExceptionType as e:`
+   */
   ControlFlowNode getName() {
     exists(ExceptStmt ex |
       this.getBasicBlock().dominates(result.getBasicBlock()) and
@@ -439,6 +447,29 @@ class ExceptFlowNode extends ControlFlowNode {
   }
 }
 
+/** The ControlFlowNode for an 'except*' statement. */
+class ExceptGroupFlowNode extends ControlFlowNode {
+  ExceptGroupFlowNode() { this.getNode() instanceof ExceptGroupStmt }
+
+  /**
+   * Gets the type handled by this exception handler.
+   * `ExceptionType` in `except* ExceptionType as e:`
+   */
+  ControlFlowNode getType() {
+    this.getBasicBlock().dominates(result.getBasicBlock()) and
+    result = this.getNode().(ExceptGroupStmt).getType().getAFlowNode()
+  }
+
+  /**
+   * Gets the name assigned to the handled exception, if any.
+   * `e` in `except* ExceptionType as e:`
+   */
+  ControlFlowNode getName() {
+    this.getBasicBlock().dominates(result.getBasicBlock()) and
+    result = this.getNode().(ExceptGroupStmt).getName().getAFlowNode()
+  }
+}
+
 private ControlFlowNode element_from_tuple_objectapi(Object tuple) {
   exists(Tuple t | t = tuple.getOrigin() and result = t.getAnElt().getAFlowNode())
 }

python/ql/lib/semmle/python/types/FunctionObject.qll

@@ -10,7 +10,7 @@ private import semmle.python.types.Builtins
 abstract class FunctionObject extends Object {
   CallableValue theCallable() { result.(ObjectInternal).getSource() = this }
 
-  predicate isOverridingMethod() { exists(Object f | this.overrides(f)) }
+  predicate isOverridingMethod() { this.overrides(_) }
 
   predicate isOverriddenMethod() { exists(Object f | f.overrides(this)) }
 

python/ql/lib/semmle/python/web/turbogears/TurboGears.qll

@@ -28,10 +28,10 @@ deprecated class TurboGearsControllerMethod extends Function {
   predicate isTemplated() { exists(this.templateName()) }
 
   Dict getValidationDict() {
-    exists(Call call, Value dict |
+    exists(Call call |
       call = this.getADecorator() and
       call.getFunc().(Name).getId() = "validate" and
-      call.getArg(0).pointsTo(dict, result)
+      call.getArg(0).pointsTo(_, result)
     )
   }
 }

python/ql/lib/semmlecode.python.dbscheme

@@ -273,6 +273,11 @@ py_extracted_version(int module : @py_Module ref,
 /* <Field> Ellipsis.location = 0, location */
 /* <Field> Ellipsis.parenthesised = 1, bool */
 
+/* <Field> ExceptGroupStmt.location = 0, location */
+/* <Field> ExceptGroupStmt.type = 1, expr */
+/* <Field> ExceptGroupStmt.name = 2, expr */
+/* <Field> ExceptGroupStmt.body = 3, stmt_list */
+
 /* <Field> ExceptStmt.location = 0, location */
 /* <Field> ExceptStmt.type = 1, expr */
 /* <Field> ExceptStmt.name = 2, expr */
@@ -863,25 +868,26 @@ case @py_stmt.kind of
 |   4 = @py_Continue
 |   5 = @py_Delete
 |   6 = @py_ExceptStmt
-|   7 = @py_Exec
-|   8 = @py_Expr_stmt
-|   9 = @py_For
-|   10 = @py_Global
-|   11 = @py_If
-|   12 = @py_Import
-|   13 = @py_ImportStar
-|   14 = @py_MatchStmt
-|   15 = @py_Case
-|   16 = @py_Nonlocal
-|   17 = @py_Pass
-|   18 = @py_Print
-|   19 = @py_Raise
-|   20 = @py_Return
-|   21 = @py_Try
-|   22 = @py_While
-|   23 = @py_With
-|   24 = @py_TemplateWrite
-|   25 = @py_AnnAssign;
+|   7 = @py_ExceptGroupStmt
+|   8 = @py_Exec
+|   9 = @py_Expr_stmt
+|   10 = @py_For
+|   11 = @py_Global
+|   12 = @py_If
+|   13 = @py_Import
+|   14 = @py_ImportStar
+|   15 = @py_MatchStmt
+|   16 = @py_Case
+|   17 = @py_Nonlocal
+|   18 = @py_Pass
+|   19 = @py_Print
+|   20 = @py_Raise
+|   21 = @py_Return
+|   22 = @py_Try
+|   23 = @py_While
+|   24 = @py_With
+|   25 = @py_TemplateWrite
+|   26 = @py_AnnAssign;
 
 case @py_unaryop.kind of
     0 = @py_Invert
@@ -907,7 +913,7 @@ case @py_unaryop.kind of
 
 @py_expr_or_stmt = @py_expr | @py_stmt;
 
-@py_expr_parent = @py_AnnAssign | @py_Assert | @py_Assign | @py_AssignExpr | @py_Attribute | @py_AugAssign | @py_Await | @py_BinaryExpr | @py_Call | @py_Case | @py_Compare | @py_DictComp | @py_DictUnpacking | @py_ExceptStmt | @py_Exec | @py_Expr_stmt | @py_Filter | @py_For | @py_FormattedValue | @py_Function | @py_FunctionExpr | @py_GeneratorExp | @py_Guard | @py_If | @py_IfExp | @py_ImportMember | @py_ImportStar | @py_KeyValuePair | @py_ListComp | @py_MatchAsPattern | @py_MatchCapturePattern | @py_MatchClassPattern | @py_MatchKeywordPattern | @py_MatchLiteralPattern | @py_MatchStmt | @py_MatchValuePattern | @py_Print | @py_Raise | @py_Repr | @py_Return | @py_SetComp | @py_Slice | @py_Starred | @py_Subscript | @py_TemplateDottedNotation | @py_TemplateWrite | @py_UnaryExpr | @py_While | @py_With | @py_Yield | @py_YieldFrom | @py_alias | @py_arguments | @py_comprehension | @py_expr_list | @py_keyword | @py_parameter_list;
+@py_expr_parent = @py_AnnAssign | @py_Assert | @py_Assign | @py_AssignExpr | @py_Attribute | @py_AugAssign | @py_Await | @py_BinaryExpr | @py_Call | @py_Case | @py_Compare | @py_DictComp | @py_DictUnpacking | @py_ExceptGroupStmt | @py_ExceptStmt | @py_Exec | @py_Expr_stmt | @py_Filter | @py_For | @py_FormattedValue | @py_Function | @py_FunctionExpr | @py_GeneratorExp | @py_Guard | @py_If | @py_IfExp | @py_ImportMember | @py_ImportStar | @py_KeyValuePair | @py_ListComp | @py_MatchAsPattern | @py_MatchCapturePattern | @py_MatchClassPattern | @py_MatchKeywordPattern | @py_MatchLiteralPattern | @py_MatchStmt | @py_MatchValuePattern | @py_Print | @py_Raise | @py_Repr | @py_Return | @py_SetComp | @py_Slice | @py_Starred | @py_Subscript | @py_TemplateDottedNotation | @py_TemplateWrite | @py_UnaryExpr | @py_While | @py_With | @py_Yield | @py_YieldFrom | @py_alias | @py_arguments | @py_comprehension | @py_expr_list | @py_keyword | @py_parameter_list;
 
 @py_location_parent = @py_DictUnpacking | @py_KeyValuePair | @py_StringPart | @py_comprehension | @py_expr | @py_keyword | @py_pattern | @py_stmt;
 
@@ -919,7 +925,7 @@ case @py_unaryop.kind of
 
 @py_scope = @py_Class | @py_Function | @py_Module;
 
-@py_stmt_list_parent = @py_Case | @py_Class | @py_ExceptStmt | @py_For | @py_Function | @py_If | @py_MatchStmt | @py_Module | @py_Try | @py_While | @py_With;
+@py_stmt_list_parent = @py_Case | @py_Class | @py_ExceptGroupStmt | @py_ExceptStmt | @py_For | @py_Function | @py_If | @py_MatchStmt | @py_Module | @py_Try | @py_While | @py_With;
 
 @py_str_list_parent = @py_Global | @py_Nonlocal;
 

python/ql/lib/semmlecode.python.dbscheme.stats

@@ -469,6 +469,10 @@
 <v>5610</v>
 </e>
 <e>
+<k>@py_ExceptGroupStmt</k>
+<v>1000</v>
+</e>
+<e>
 <k>@py_Expr_stmt</k>
 <v>76750</v>
 </e>

python/ql/lib/upgrades/503c0516fba2e5da9570f00eb34ef43025ecb8fb/py_stmts.ql

@@ -0,0 +1,30 @@
+// First we need to wrap some database types
+class Stmt_ extends @py_stmt {
+  string toString() { result = "Stmt" }
+}
+
+class StmtList_ extends @py_stmt_list {
+  string toString() { result = "StmtList" }
+}
+
+/**
+ * New kinds have been inserted such that
+ * `@py_Exec` which used to have index 7 now has index 8.
+ * Entries with lower indices are unchanged.
+ */
+bindingset[old_index]
+int new_index(int old_index) {
+  if old_index < 7 then result = old_index else result = (8 - 7) + old_index
+}
+
+// The schema for py_stmts is:
+//
+// py_stmts(unique int id : @py_stmt,
+//   int kind: int ref,
+//   int parent : @py_stmt_list ref,
+//   int idx : int ref);
+from Stmt_ expr, int old_kind, StmtList_ parent, int idx, int new_kind
+where
+  py_stmts(expr, old_kind, parent, idx) and
+  new_kind = new_index(old_kind)
+select expr, new_kind, parent, idx

python/ql/lib/upgrades/503c0516fba2e5da9570f00eb34ef43025ecb8fb/upgrade.properties

@@ -0,0 +1,3 @@
+description: Add support for `except*`
+compatibility: backwards
+py_stmts.rel: run py_stmts.qlo

python/ql/src/AlertSuppression.ql

@@ -0,0 +1,51 @@
+/**
+ * @name Alert suppression
+ * @description Generates information about alert suppressions.
+ * @kind alert-suppression
+ * @id py/alert-suppression
+ */
+
+private import codeql.util.suppression.AlertSuppression as AS
+private import semmle.python.Comment as P
+
+class AstNode instanceof P::AstNode {
+  predicate hasLocationInfo(
+    string filepath, int startline, int startcolumn, int endline, int endcolumn
+  ) {
+    super.getLocation().hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn)
+  }
+
+  string toString() { result = super.toString() }
+}
+
+class SingleLineComment instanceof P::Comment {
+  predicate hasLocationInfo(
+    string filepath, int startline, int startcolumn, int endline, int endcolumn
+  ) {
+    super.getLocation().hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn)
+  }
+
+  string getText() { result = super.getContents() }
+
+  string toString() { result = super.toString() }
+}
+
+import AS::Make<AstNode, SingleLineComment>
+
+/**
+ * A noqa suppression comment. Both pylint and pyflakes respect this, so lgtm ought to too.
+ */
+class NoqaSuppressionComment extends SuppressionComment instanceof SingleLineComment {
+  NoqaSuppressionComment() {
+    SingleLineComment.super.getText().regexpMatch("(?i)\\s*noqa\\s*([^:].*)?")
+  }
+
+  override string getAnnotation() { result = "lgtm" }
+
+  override predicate covers(
+    string filepath, int startline, int startcolumn, int endline, int endcolumn
+  ) {
+    this.hasLocationInfo(filepath, startline, _, endline, endcolumn) and
+    startcolumn = 1
+  }
+}

python/ql/src/CHANGELOG.md

@@ -1,3 +1,15 @@
+## 0.6.0
+
+### Minor Analysis Improvements
+
+* The `analysis/AlertSuppression.ql` query has moved to the root folder. Users that refer to this query by path should update their configurations. The query has been updated to support the new `# codeql[query-id]` supression comments. These comments can be used to suppress an alert and must be placed on a blank line before the alert. In addition the legacy `# lgtm` and `# lgtm[query-id]` comments can now also be placed on the line before an alert.
+* Bumped the minimum keysize we consider secure for elliptic curve cryptography from 224 to 256 bits, following current best practices. This might effect results from the _Use of weak cryptographic key_ (`py/weak-crypto-key`) query.
+* Added modeling of `getpass.getpass` as a source of passwords, which will be an additional source for `py/clear-text-logging-sensitive-data`, `py/clear-text-storage-sensitive-data`, and `py/weak-sensitive-data-hashing`.
+
+## 0.5.6
+
+No user-facing changes.
+
 ## 0.5.5
 
 No user-facing changes.

python/ql/src/Expressions/WrongNumberArgumentsForFormat.ql

@@ -17,15 +17,15 @@ import semmle.python.strings
 
 predicate string_format(BinaryExpr operation, StrConst str, Value args, AstNode origin) {
   operation.getOp() instanceof Mod and
-  exists(Value fmt, Context ctx |
-    operation.getLeft().pointsTo(ctx, fmt, str) and
+  exists(Context ctx |
+    operation.getLeft().pointsTo(ctx, _, str) and
     operation.getRight().pointsTo(ctx, args, origin)
   )
 }
 
 int sequence_length(Value args) {
   /* Guess length of sequence */
-  exists(Tuple seq, AstNode origin | seq.pointsTo(args, origin) |
+  exists(Tuple seq | seq.pointsTo(args, _) |
     result = strictcount(seq.getAnElt()) and
     not seq.getAnElt() instanceof Starred
   )

python/ql/src/Security/CWE-020-ExternalAPIs/ExternalAPIs.qll

@@ -87,12 +87,12 @@ class ExternalApiDataNode extends DataFlow::Node {
       not exists(cv.(CallableValue).getScope().getLocation().getFile().getRelativePath())
     ) and
     // Not already modeled as a taint step
-    not exists(DataFlow::Node next | TaintTrackingPrivate::defaultAdditionalTaintStep(this, next)) and
+    not TaintTrackingPrivate::defaultAdditionalTaintStep(this, _) and
     // for `list.append(x)`, we have a additional taint step from x -> [post] list.
     // Since we have modeled this explicitly, I don't see any cases where we would want to report this.
-    not exists(DataFlow::Node prev, DataFlow::PostUpdateNode post |
+    not exists(DataFlow::PostUpdateNode post |
       post.getPreUpdateNode() = this and
-      TaintTrackingPrivate::defaultAdditionalTaintStep(prev, post)
+      TaintTrackingPrivate::defaultAdditionalTaintStep(_, post)
     )
   }
 

python/ql/src/Security/CWE-020/HostnameRegexpShared.qll

@@ -3,200 +3,6 @@
  * that match URLs and hostname patterns.
  */
 
-private import HostnameRegexpSpecific
-
-/**
- * Holds if the given constant is unlikely to occur in the origin part of a URL.
- */
-predicate isConstantInvalidInsideOrigin(RegExpConstant term) {
-  // Look for any of these cases:
-  // - A character that can't occur in the origin
-  // - Two dashes in a row
-  // - A colon that is not part of port or scheme separator
-  // - A slash that is not part of scheme separator
-  term.getValue().regexpMatch(".*(?:[^a-zA-Z0-9.:/-]|--|:[^0-9/]|(?<![/:]|^)/).*")
-}
-
-/** Holds if `term` is a dot constant of form `\.` or `[.]`. */
-predicate isDotConstant(RegExpTerm term) {
-  term.(RegExpCharEscape).getValue() = "."
-  or
-  exists(RegExpCharacterClass cls |
-    term = cls and
-    not cls.isInverted() and
-    cls.getNumChild() = 1 and
-    cls.getAChild().(RegExpConstant).getValue() = "."
-  )
-}
-
-/** Holds if `term` is a wildcard `.` or an actual `.` character. */
-predicate isDotLike(RegExpTerm term) {
-  term instanceof RegExpDot
-  or
-  isDotConstant(term)
-}
-
-/** Holds if `term` will only ever be matched against the beginning of the input. */
-predicate matchesBeginningOfString(RegExpTerm term) {
-  term.isRootTerm()
-  or
-  exists(RegExpTerm parent | matchesBeginningOfString(parent) |
-    term = parent.(RegExpSequence).getChild(0)
-    or
-    parent.(RegExpSequence).getChild(0) instanceof RegExpCaret and
-    term = parent.(RegExpSequence).getChild(1)
-    or
-    term = parent.(RegExpAlt).getAChild()
-    or
-    term = parent.(RegExpGroup).getAChild()
-  )
-}
-
-/**
- * Holds if the given sequence `seq` contains top-level domain preceded by a dot, such as `.com`,
- * excluding cases where this is at the very beginning of the regexp.
- *
- * `i` is bound to the index of the last child in the top-level domain part.
- */
-predicate hasTopLevelDomainEnding(RegExpSequence seq, int i) {
-  seq.getChild(i)
-      .(RegExpConstant)
-      .getValue()
-      .regexpMatch("(?i)" + RegExpPatterns::getACommonTld() + "(:\\d+)?([/?#].*)?") and
-  isDotLike(seq.getChild(i - 1)) and
-  not (i = 1 and matchesBeginningOfString(seq))
-}
-
-/**
- * Holds if the given regular expression term contains top-level domain preceded by a dot,
- * such as `.com`.
- */
-predicate hasTopLevelDomainEnding(RegExpSequence seq) { hasTopLevelDomainEnding(seq, _) }
-
-/**
- * Holds if `term` will always match a hostname, that is, all disjunctions contain
- * a hostname pattern that isn't inside a quantifier.
- */
-predicate alwaysMatchesHostname(RegExpTerm term) {
-  hasTopLevelDomainEnding(term, _)
-  or
-  // `localhost` is considered a hostname pattern, but has no TLD
-  term.(RegExpConstant).getValue().regexpMatch("\\blocalhost\\b")
-  or
-  not term instanceof RegExpAlt and
-  not term instanceof RegExpQuantifier and
-  alwaysMatchesHostname(term.getAChild())
-  or
-  alwaysMatchesHostnameAlt(term)
-}
-
-/** Holds if every child of `alt` contains a hostname pattern. */
-predicate alwaysMatchesHostnameAlt(RegExpAlt alt) {
-  alwaysMatchesHostnameAlt(alt, alt.getNumChild() - 1)
-}
-
-/**
- * Holds if the first `i` children of `alt` contains a hostname pattern.
- *
- * This is used instead of `forall` to avoid materializing the set of alternatives
- * that don't contains hostnames, which is much larger.
- */
-predicate alwaysMatchesHostnameAlt(RegExpAlt alt, int i) {
-  alwaysMatchesHostname(alt.getChild(0)) and i = 0
-  or
-  alwaysMatchesHostnameAlt(alt, i - 1) and
-  alwaysMatchesHostname(alt.getChild(i))
-}
-
-/**
- * Holds if `term` occurs inside a quantifier or alternative (and thus
- * can not be expected to correspond to a unique match), or as part of
- * a lookaround assertion (which are rarely used for capture groups).
- */
-predicate isInsideChoiceOrSubPattern(RegExpTerm term) {
-  exists(RegExpParent parent | parent = term.getParent() |
-    parent instanceof RegExpAlt
-    or
-    parent instanceof RegExpQuantifier
-    or
-    parent instanceof RegExpSubPattern
-    or
-    isInsideChoiceOrSubPattern(parent)
-  )
-}
-
-/**
- * Holds if `group` is likely to be used as a capture group.
- */
-predicate isLikelyCaptureGroup(RegExpGroup group) {
-  group.isCapture() and
-  not isInsideChoiceOrSubPattern(group)
-}
-
-/**
- * Holds if `seq` contains two consecutive dots `..` or escaped dots.
- *
- * At least one of these dots is not intended to be a subdomain separator,
- * so we avoid flagging the pattern in this case.
- */
-predicate hasConsecutiveDots(RegExpSequence seq) {
-  exists(int i |
-    isDotLike(seq.getChild(i)) and
-    isDotLike(seq.getChild(i + 1))
-  )
-}
-
-predicate isIncompleteHostNameRegExpPattern(RegExpTerm regexp, RegExpSequence seq, string msg) {
-  seq = regexp.getAChild*() and
-  exists(RegExpDot unescapedDot, int i, string hostname |
-    hasTopLevelDomainEnding(seq, i) and
-    not isConstantInvalidInsideOrigin(seq.getChild([0 .. i - 1]).getAChild*()) and
-    not isLikelyCaptureGroup(seq.getChild([i .. seq.getNumChild() - 1]).getAChild*()) and
-    unescapedDot = seq.getChild([0 .. i - 1]).getAChild*() and
-    unescapedDot != seq.getChild(i - 1) and // Should not be the '.' immediately before the TLD
-    not hasConsecutiveDots(unescapedDot.getParent()) and
-    hostname =
-      seq.getChild(i - 2).getRawValue() + seq.getChild(i - 1).getRawValue() +
-        seq.getChild(i).getRawValue()
-  |
-    if unescapedDot.getParent() instanceof RegExpQuantifier
-    then
-      // `.*\.example.com` can match `evil.com/?x=.example.com`
-      //
-      // This problem only occurs when the pattern is applied against a full URL, not just a hostname/origin.
-      // We therefore check if the pattern includes a suffix after the TLD, such as `.*\.example.com/`.
-      // Note that a post-anchored pattern (`.*\.example.com$`) will usually fail to match a full URL,
-      // and patterns with neither a suffix nor an anchor fall under the purview of MissingRegExpAnchor.
-      seq.getChild(0) instanceof RegExpCaret and
-      not seq.getAChild() instanceof RegExpDollar and
-      seq.getChild([i .. i + 1]).(RegExpConstant).getValue().regexpMatch(".*[/?#].*") and
-      msg =
-        "has an unrestricted wildcard '" + unescapedDot.getParent().(RegExpQuantifier).getRawValue()
-          + "' which may cause '" + hostname +
-          "' to be matched anywhere in the URL, outside the hostname."
-    else
-      msg =
-        "has an unescaped '.' before '" + hostname +
-          "', so it might match more hosts than expected."
-  )
-}
-
-predicate incompleteHostnameRegExp(
-  RegExpSequence hostSequence, string message, DataFlow::Node aux, string label
-) {
-  exists(RegExpPatternSource re, RegExpTerm regexp, string msg, string kind |
-    regexp = re.getRegExpTerm() and
-    isIncompleteHostNameRegExpPattern(regexp, hostSequence, msg) and
-    (
-      if re.getAParse() != re
-      then (
-        kind = "string, which is used as a regular expression $@," and
-        aux = re.getAParse()
-      ) else (
-        kind = "regular expression" and aux = re
-      )
-    )
-  |
-    message = "This " + kind + " " + msg and label = "here"
-  )
-}
+// HostnameRegexp should be used directly from the shared regex pack, and not from this file.
+deprecated private import semmle.python.security.regexp.HostnameRegex as Dep
+import Dep

python/ql/src/Security/CWE-020/HostnameRegexpSpecific.qll

@@ -1,3 +0,0 @@
-import semmle.python.RegexTreeView
-import semmle.python.dataflow.new.DataFlow
-import semmle.python.dataflow.new.Regexp

python/ql/src/Security/CWE-020/IncompleteHostnameRegExp.ql

@@ -11,6 +11,6 @@
  *       external/cwe/cwe-020
  */
 
-import HostnameRegexpShared
+private import semmle.python.security.regexp.HostnameRegex as HostnameRegex
 
-query predicate problems = incompleteHostnameRegExp/4;
+query predicate problems = HostnameRegex::incompleteHostnameRegExp/4;

python/ql/src/Statements/StatementNoEffect.ql

@@ -22,8 +22,8 @@ predicate understood_attribute(Attribute attr, ClassValue cls, ClassValue attr_c
 
 /* Conservative estimate of whether attribute lookup has a side effect */
 predicate side_effecting_attribute(Attribute attr) {
-  exists(ClassValue cls, ClassValue attr_cls |
-    understood_attribute(attr, cls, attr_cls) and
+  exists(ClassValue attr_cls |
+    understood_attribute(attr, _, attr_cls) and
     side_effecting_descriptor_type(attr_cls)
   )
 }

python/ql/src/analysis/AlertSuppression.ql

@@ -1,105 +0,0 @@
-/**
- * @name Alert suppression
- * @description Generates information about alert suppressions.
- * @kind alert-suppression
- * @id py/alert-suppression
- */
-
-import python
-
-/**
- * An alert suppression comment.
- */
-abstract class SuppressionComment extends Comment {
-  /** Gets the scope of this suppression. */
-  abstract SuppressionScope getScope();
-
-  /** Gets the suppression annotation in this comment. */
-  abstract string getAnnotation();
-
-  /**
-   * Holds if this comment applies to the range from column `startcolumn` of line `startline`
-   * to column `endcolumn` of line `endline` in file `filepath`.
-   */
-  abstract predicate covers(
-    string filepath, int startline, int startcolumn, int endline, int endcolumn
-  );
-}
-
-/**
- * An alert comment that applies to a single line
- */
-abstract class LineSuppressionComment extends SuppressionComment {
-  LineSuppressionComment() {
-    exists(string filepath, int l |
-      this.getLocation().hasLocationInfo(filepath, l, _, _, _) and
-      any(AstNode a).getLocation().hasLocationInfo(filepath, l, _, _, _)
-    )
-  }
-
-  /** Gets the scope of this suppression. */
-  override SuppressionScope getScope() { result = this }
-
-  override predicate covers(
-    string filepath, int startline, int startcolumn, int endline, int endcolumn
-  ) {
-    this.getLocation().hasLocationInfo(filepath, startline, _, endline, endcolumn) and
-    startcolumn = 1
-  }
-}
-
-/**
- * An lgtm suppression comment.
- */
-class LgtmSuppressionComment extends LineSuppressionComment {
-  string annotation;
-
-  LgtmSuppressionComment() {
-    exists(string all | all = this.getContents() |
-      // match `lgtm[...]` anywhere in the comment
-      annotation = all.regexpFind("(?i)\\blgtm\\s*\\[[^\\]]*\\]", _, _)
-      or
-      // match `lgtm` at the start of the comment and after semicolon
-      annotation = all.regexpFind("(?i)(?<=^|;)\\s*lgtm(?!\\B|\\s*\\[)", _, _).trim()
-    )
-  }
-
-  /** Gets the suppression annotation in this comment. */
-  override string getAnnotation() { result = annotation }
-}
-
-/**
- * A noqa suppression comment. Both pylint and pyflakes respect this, so lgtm ought to too.
- */
-class NoqaSuppressionComment extends LineSuppressionComment {
-  NoqaSuppressionComment() { this.getContents().toLowerCase().regexpMatch("\\s*noqa\\s*([^:].*)?") }
-
-  override string getAnnotation() { result = "lgtm" }
-}
-
-/**
- * The scope of an alert suppression comment.
- */
-class SuppressionScope extends @py_comment instanceof SuppressionComment {
-  /**
-   * Holds if this element is at the specified location.
-   * The location spans column `startcolumn` of line `startline` to
-   * column `endcolumn` of line `endline` in file `filepath`.
-   * For more information, see
-   * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
-   */
-  predicate hasLocationInfo(
-    string filepath, int startline, int startcolumn, int endline, int endcolumn
-  ) {
-    super.covers(filepath, startline, startcolumn, endline, endcolumn)
-  }
-
-  /** Gets a textual representation of this element. */
-  string toString() { result = "suppression range" }
-}
-
-from SuppressionComment c
-select c, // suppression comment
-  c.getContents(), // text of suppression comment (excluding delimiters)
-  c.getAnnotation(), // text of suppression annotation
-  c.getScope() // scope of suppression

python/ql/src/analysis/Consistency.ql

@@ -231,8 +231,8 @@ predicate points_to_consistency(string clsname, string problem, string what) {
     what = obj.toString()
   )
   or
-  exists(ControlFlowNode use, ControlFlowNode inter, Object obj |
-    intermediate_origins(use, inter, obj) and
+  exists(ControlFlowNode use, ControlFlowNode inter |
+    intermediate_origins(use, inter, _) and
     clsname = use.getAQlClass() and
     problem = "has intermediate origin " + inter and
     what = use.toString()

python/ql/src/analysis/TypeInferenceFailure.ql

@@ -12,5 +12,5 @@ import python
 from ControlFlowNode f, Object o
 where
   f.refersTo(o) and
-  not exists(ClassObject c | f.refersTo(o, c, _))
+  not f.refersTo(o, _, _)
 select o, "Type inference fails for 'object'."

python/ql/src/change-notes/2022-11-22-getpass.md

@@ -1,4 +0,0 @@
----
- category: minorAnalysis
----
- * Added modeling of `getpass.getpass` as a source of passwords, which will be an additional source for `py/clear-text-logging-sensitive-data`, `py/clear-text-storage-sensitive-data`, and `py/weak-sensitive-data-hashing`.

python/ql/src/change-notes/2022-11-29-ecc-min-secure-keysize.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* Bumped the minimum keysize we consider secure for elliptic curve cryptography from 224 to 256 bits, following current best practices. This might effect results from the _Use of weak cryptographic key_ (`py/weak-crypto-key`) query.

python/ql/src/change-notes/released/0.5.6.md

@@ -0,0 +1,3 @@
+## 0.5.6
+
+No user-facing changes.

python/ql/src/change-notes/released/0.6.0.md

@@ -0,0 +1,7 @@
+## 0.6.0
+
+### Minor Analysis Improvements
+
+* The `analysis/AlertSuppression.ql` query has moved to the root folder. Users that refer to this query by path should update their configurations. The query has been updated to support the new `# codeql[query-id]` supression comments. These comments can be used to suppress an alert and must be placed on a blank line before the alert. In addition the legacy `# lgtm` and `# lgtm[query-id]` comments can now also be placed on the line before an alert.
+* Bumped the minimum keysize we consider secure for elliptic curve cryptography from 224 to 256 bits, following current best practices. This might effect results from the _Use of weak cryptographic key_ (`py/weak-crypto-key`) query.
+* Added modeling of `getpass.getpass` as a source of passwords, which will be an additional source for `py/clear-text-logging-sensitive-data`, `py/clear-text-storage-sensitive-data`, and `py/weak-sensitive-data-hashing`.

python/ql/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.5.5
+lastReleaseVersion: 0.6.0

python/ql/src/external/VCS.qll

@@ -33,7 +33,7 @@ class Commit extends @svnentry {
 
   File getAnAffectedFile(string action) { svnaffectedfiles(this, result, action) }
 
-  File getAnAffectedFile() { exists(string action | result = this.getAnAffectedFile(action)) }
+  File getAnAffectedFile() { result = this.getAnAffectedFile(_) }
 
   predicate isRecent() { recentCommit(this) }
 

python/ql/src/qlpack.yml

@@ -1,11 +1,12 @@
 name: codeql/python-queries
-version: 0.5.6-dev
+version: 0.6.1-dev
 groups: 
   - python
   - queries
 dependencies:
   codeql/python-all: ${workspace}
   codeql/suite-helpers: ${workspace}
+  codeql/util: ${workspace}
 suites: codeql-suites
 extractor: python
 defaultSuiteFile: codeql-suites/python-code-scanning.qls

python/ql/test/3/library-tests/PointsTo/typehints/Values.expected

@@ -19,3 +19,10 @@
 | test.py:10:6:10:18 | test.py:10 | ControlFlowNode for Subscript | import | file://:0:0:0:0 | _Optional()[builtin-class int] |
 | test.py:10:15:10:17 | test.py:10 | ControlFlowNode for int | import | file://:0:0:0:0 | builtin-class int |
 | test.py:10:20:10:22 | test.py:10 | ControlFlowNode for int | import | file://:0:0:0:0 | builtin-class int |
+| test.py:12:1:12:12 | test.py:12 | ControlFlowNode for ClassExpr | import | test.py:12:1:12:12 | class baz |
+| test.py:15:7:15:10 | test.py:15 | ControlFlowNode for True | import | file://:0:0:0:0 | bool True |
+| test.py:16:11:16:13 | test.py:16 | ControlFlowNode for baz | import | file://:0:0:0:0 | class baz[class baz] |
+| test.py:16:11:16:13 | test.py:16 | ControlFlowNode for baz | import | test.py:12:1:12:12 | class baz |
+| test.py:16:11:16:18 | test.py:16 | ControlFlowNode for Subscript | import | file://:0:0:0:0 | class baz[class baz] |
+| test.py:16:15:16:17 | test.py:16 | ControlFlowNode for baz | import | file://:0:0:0:0 | class baz[class baz] |
+| test.py:16:15:16:17 | test.py:16 | ControlFlowNode for baz | import | test.py:12:1:12:12 | class baz |

python/ql/test/3/library-tests/PointsTo/typehints/test.py

@@ -9,10 +9,8 @@ def bar(s:set)->Set:
 t1 = Optional[Optional[int]]
 t2 = Optional[int][int]
 
-# ODASA-8075
-# Commented out until the fix has been pushed to LGTM.com
-#class baz():
-#    pass
-#
-#while True:
-#    baz = baz[baz]
+class baz():
+    pass
+
+while True:
+    baz = baz[baz]

python/ql/test/TestUtilities/InlineExpectationsTest.qll

@@ -1,399 +1,19 @@
 /**
- * Provides a library for writing QL tests whose success or failure is based on expected results
- * embedded in the test source code as comments, rather than the contents of an `.expected` file
- * (in that the `.expected` file should always be empty).
- *
- * To add this framework to a new language:
- * - Add a file `InlineExpectationsTestPrivate.qll` that defines a `ExpectationComment` class. This class
- *   must support a `getContents` method that returns the contents of the given comment, _excluding_
- *   the comment indicator itself. It should also define `toString` and `getLocation` as usual.
- *
- * To create a new inline expectations test:
- * - Declare a class that extends `InlineExpectationsTest`. In the characteristic predicate of the
- * new class, bind `this` to a unique string (usually the name of the test).
- * - Override the `hasActualResult()` predicate to produce the actual results of the query. For each
- * result, specify a `Location`, a text description of the element for which the result was
- * reported, a short string to serve as the tag to identify expected results for this test, and the
- * expected value of the result.
- * - Override `getARelevantTag()` to return the set of tags that can be produced by
- * `hasActualResult()`. Often this is just a single tag.
- *
- * Example:
- * ```ql
- * class ConstantValueTest extends InlineExpectationsTest {
- *   ConstantValueTest() { this = "ConstantValueTest" }
- *
- *   override string getARelevantTag() {
- *     // We only use one tag for this test.
- *     result = "const"
- *   }
- *
- *   override predicate hasActualResult(
- *     Location location, string element, string tag, string value
- *   ) {
- *     exists(Expr e |
- *       tag = "const" and // The tag for this test.
- *       value = e.getValue() and // The expected value. Will only hold for constant expressions.
- *       location = e.getLocation() and // The location of the result to be reported.
- *       element = e.toString() // The display text for the result.
- *     )
- *   }
- * }
- * ```
- *
- * There is no need to write a `select` clause or query predicate. All of the differences between
- * expected results and actual results will be reported in the `failures()` query predicate.
- *
- * To annotate the test source code with an expected result, place a comment starting with a `$` on the
- * same line as the expected result, with text of the following format as the body of the comment:
- *
- * `tag=expected-value`
- *
- * Where `tag` is the value of the `tag` parameter from `hasActualResult()`, and `expected-value` is
- * the value of the `value` parameter from `hasActualResult()`. The `=expected-value` portion may be
- * omitted, in which case `expected-value` is treated as the empty string. Multiple expectations may
- * be placed in the same comment. Any actual result that
- * appears on a line that does not contain a matching expected result comment will be reported with
- * a message of the form "Unexpected result: tag=value". Any expected result comment for which there
- * is no matching actual result will be reported with a message of the form
- * "Missing result: tag=expected-value".
- *
- * Example:
- * ```cpp
- * int i = x + 5;  // $ const=5
- * int j = y + (7 - 3)  // $ const=7 const=3 const=4  // The result of the subtraction is a constant.
- * ```
- *
- * For tests that contain known missing and spurious results, it is possible to further
- * annotate that a particular expected result is known to be spurious, or that a particular
- * missing result is known to be missing:
- *
- * `$ SPURIOUS: tag=expected-value`  // Spurious result
- * `$ MISSING: tag=expected-value`  // Missing result
- *
- * A spurious expectation is treated as any other expected result, except that if there is no
- * matching actual result, the message will be of the form "Fixed spurious result: tag=value". A
- * missing expectation is treated as if there were no expected result, except that if a
- * matching expected result is found, the message will be of the form
- * "Fixed missing result: tag=value".
- *
- * A single line can contain all the expected, spurious and missing results of that line. For instance:
- * `$ tag1=value1 SPURIOUS: tag2=value2 MISSING: tag3=value3`.
- *
- * If the same result value is expected for two or more tags on the same line, there is a shorthand
- * notation available:
- *
- * `tag1,tag2=expected-value`
- *
- * is equivalent to:
- *
- * `tag1=expected-value tag2=expected-value`
+ * Inline expectation tests for Python.
+ * See `shared/util/codeql/util/test/InlineExpectationsTest.qll`
  */
 
-private import InlineExpectationsTestPrivate
-
-/**
- * The base class for tests with inline expectations. The test extends this class to provide the actual
- * results of the query, which are then compared with the expected results in comments to produce a
- * list of failure messages that point out where the actual results differ from the expected
- * results.
- */
-abstract class InlineExpectationsTest extends string {
-  bindingset[this]
-  InlineExpectationsTest() { any() }
+private import python as PY
+private import codeql.util.test.InlineExpectationsTest
 
+private module Impl implements InlineExpectationsTestSig {
   /**
-   * Returns all tags that can be generated by this test. Most tests will only ever produce a single
-   * tag. Any expected result comments for a tag that is not returned by the `getARelevantTag()`
-   * predicate for an active test will be ignored. This makes it possible to write multiple tests in
-   * different `.ql` files that all query the same source code.
+   * A class representing line comments in Python. As this is the only form of comment Python
+   * permits, we simply reuse the `Comment` class.
    */
-  abstract string getARelevantTag();
+  class ExpectationComment = PY::Comment;
 
-  /**
-   * Returns the actual results of the query that is being tested. Each result consist of the
-   * following values:
-   * - `location` - The source code location of the result. Any expected result comment must appear
-   *   on the start line of this location.
-   * - `element` - Display text for the element on which the result is reported.
-   * - `tag` - The tag that marks this result as coming from this test. This must be one of the tags
-   *   returned by `getARelevantTag()`.
-   * - `value` - The value of the result, which will be matched against the value associated with
-   *   `tag` in any expected result comment on that line.
-   */
-  abstract predicate hasActualResult(Location location, string element, string tag, string value);
-
-  /**
-   * Holds if there is an optional result on the specified location.
-   *
-   * This is similar to `hasActualResult`, but returns results that do not require a matching annotation.
-   * A failure will still arise if there is an annotation that does not match any results, but not vice versa.
-   * Override this predicate to specify optional results.
-   */
-  predicate hasOptionalResult(Location location, string element, string tag, string value) {
-    none()
-  }
-
-  final predicate hasFailureMessage(FailureLocatable element, string message) {
-    exists(ActualResult actualResult |
-      actualResult.getTest() = this and
-      actualResult.getTag() = this.getARelevantTag() and
-      element = actualResult and
-      (
-        exists(FalseNegativeExpectation falseNegative |
-          falseNegative.matchesActualResult(actualResult) and
-          message = "Fixed missing result:" + falseNegative.getExpectationText()
-        )
-        or
-        not exists(ValidExpectation expectation | expectation.matchesActualResult(actualResult)) and
-        message = "Unexpected result: " + actualResult.getExpectationText() and
-        not actualResult.isOptional()
-      )
-    )
-    or
-    exists(ActualResult actualResult |
-      actualResult.getTest() = this and
-      not actualResult.getTag() = this.getARelevantTag() and
-      element = actualResult and
-      message =
-        "Tag mismatch: Actual result with tag '" + actualResult.getTag() +
-          "' that is not part of getARelevantTag()"
-    )
-    or
-    exists(ValidExpectation expectation |
-      not exists(ActualResult actualResult | expectation.matchesActualResult(actualResult)) and
-      expectation.getTag() = this.getARelevantTag() and
-      element = expectation and
-      (
-        expectation instanceof GoodExpectation and
-        message = "Missing result:" + expectation.getExpectationText()
-        or
-        expectation instanceof FalsePositiveExpectation and
-        message = "Fixed spurious result:" + expectation.getExpectationText()
-      )
-    )
-    or
-    exists(InvalidExpectation expectation |
-      element = expectation and
-      message = "Invalid expectation syntax: " + expectation.getExpectation()
-    )
-  }
+  class Location = PY::Location;
 }
 
-/**
- * RegEx pattern to match a comment containing one or more expected results. The comment must have
- * `$` as its first non-whitespace character. Any subsequent character
- * is treated as part of the expected results, except that the comment may contain a `//` sequence
- * to treat the remainder of the line as a regular (non-interpreted) comment.
- */
-private string expectationCommentPattern() { result = "\\s*\\$((?:[^/]|/[^/])*)(?://.*)?" }
-
-/**
- * The possible columns in an expectation comment. The `TDefaultColumn` branch represents the first
- * column in a comment. This column is not precedeeded by a name. `TNamedColumn(name)` represents a
- * column containing expected results preceded by the string `name:`.
- */
-private newtype TColumn =
-  TDefaultColumn() or
-  TNamedColumn(string name) { name = ["MISSING", "SPURIOUS"] }
-
-bindingset[start, content]
-private int getEndOfColumnPosition(int start, string content) {
-  result =
-    min(string name, int cand |
-      exists(TNamedColumn(name)) and
-      cand = content.indexOf(name + ":") and
-      cand >= start
-    |
-      cand
-    )
-  or
-  not exists(string name |
-    exists(TNamedColumn(name)) and
-    content.indexOf(name + ":") >= start
-  ) and
-  result = content.length()
-}
-
-private predicate getAnExpectation(
-  ExpectationComment comment, TColumn column, string expectation, string tags, string value
-) {
-  exists(string content |
-    content = comment.getContents().regexpCapture(expectationCommentPattern(), 1) and
-    (
-      column = TDefaultColumn() and
-      exists(int end |
-        end = getEndOfColumnPosition(0, content) and
-        expectation = content.prefix(end).regexpFind(expectationPattern(), _, _).trim()
-      )
-      or
-      exists(string name, int start, int end |
-        column = TNamedColumn(name) and
-        start = content.indexOf(name + ":") + name.length() + 1 and
-        end = getEndOfColumnPosition(start, content) and
-        expectation = content.substring(start, end).regexpFind(expectationPattern(), _, _).trim()
-      )
-    )
-  ) and
-  tags = expectation.regexpCapture(expectationPattern(), 1) and
-  if exists(expectation.regexpCapture(expectationPattern(), 2))
-  then value = expectation.regexpCapture(expectationPattern(), 2)
-  else value = ""
-}
-
-private string getColumnString(TColumn column) {
-  column = TDefaultColumn() and result = ""
-  or
-  column = TNamedColumn(result)
-}
-
-/**
- * RegEx pattern to match a single expected result, not including the leading `$`. It consists of one or
- * more comma-separated tags optionally followed by `=` and the expected value.
- *
- * Tags must be only letters, digits, `-` and `_` (note that the first character
- * must not be a digit), but can contain anything enclosed in a single set of
- * square brackets.
- *
- * Examples:
- * - `tag`
- * - `tag=value`
- * - `tag,tag2=value`
- * - `tag[foo bar]=value`
- *
- * Not allowed:
- * - `tag[[[foo bar]`
- */
-private string expectationPattern() {
-  exists(string tag, string tags, string value |
-    tag = "[A-Za-z-_](?:[A-Za-z-_0-9]|\\[[^\\]\\]]*\\])*" and
-    tags = "((?:" + tag + ")(?:\\s*,\\s*" + tag + ")*)" and
-    // In Python, we allow both `"` and `'` for strings, as well as the prefixes `bru`.
-    // For example, `b"foo"`.
-    value = "((?:[bru]*\"[^\"]*\"|[bru]*'[^']*'|\\S+)*)" and
-    result = tags + "(?:=" + value + ")?"
-  )
-}
-
-private newtype TFailureLocatable =
-  TActualResult(
-    InlineExpectationsTest test, Location location, string element, string tag, string value,
-    boolean optional
-  ) {
-    test.hasActualResult(location, element, tag, value) and
-    optional = false
-    or
-    test.hasOptionalResult(location, element, tag, value) and optional = true
-  } or
-  TValidExpectation(ExpectationComment comment, string tag, string value, string knownFailure) {
-    exists(TColumn column, string tags |
-      getAnExpectation(comment, column, _, tags, value) and
-      tag = tags.splitAt(",") and
-      knownFailure = getColumnString(column)
-    )
-  } or
-  TInvalidExpectation(ExpectationComment comment, string expectation) {
-    getAnExpectation(comment, _, expectation, _, _) and
-    not expectation.regexpMatch(expectationPattern())
-  }
-
-class FailureLocatable extends TFailureLocatable {
-  string toString() { none() }
-
-  Location getLocation() { none() }
-
-  final string getExpectationText() { result = getTag() + "=" + getValue() }
-
-  string getTag() { none() }
-
-  string getValue() { none() }
-}
-
-class ActualResult extends FailureLocatable, TActualResult {
-  InlineExpectationsTest test;
-  Location location;
-  string element;
-  string tag;
-  string value;
-  boolean optional;
-
-  ActualResult() { this = TActualResult(test, location, element, tag, value, optional) }
-
-  override string toString() { result = element }
-
-  override Location getLocation() { result = location }
-
-  InlineExpectationsTest getTest() { result = test }
-
-  override string getTag() { result = tag }
-
-  override string getValue() { result = value }
-
-  predicate isOptional() { optional = true }
-}
-
-abstract private class Expectation extends FailureLocatable {
-  ExpectationComment comment;
-
-  override string toString() { result = comment.toString() }
-
-  override Location getLocation() { result = comment.getLocation() }
-}
-
-private predicate onSameLine(ValidExpectation a, ActualResult b) {
-  exists(string fname, int line, Location la, Location lb |
-    // Join order intent:
-    // Take the locations of ActualResults,
-    // join with locations in the same file / on the same line,
-    // then match those against ValidExpectations.
-    la = a.getLocation() and
-    pragma[only_bind_into](lb) = b.getLocation() and
-    pragma[only_bind_into](la).hasLocationInfo(fname, line, _, _, _) and
-    lb.hasLocationInfo(fname, line, _, _, _)
-  )
-}
-
-private class ValidExpectation extends Expectation, TValidExpectation {
-  string tag;
-  string value;
-  string knownFailure;
-
-  ValidExpectation() { this = TValidExpectation(comment, tag, value, knownFailure) }
-
-  override string getTag() { result = tag }
-
-  override string getValue() { result = value }
-
-  string getKnownFailure() { result = knownFailure }
-
-  predicate matchesActualResult(ActualResult actualResult) {
-    onSameLine(pragma[only_bind_into](this), actualResult) and
-    getTag() = actualResult.getTag() and
-    getValue() = actualResult.getValue()
-  }
-}
-
-/* Note: These next three classes correspond to all the possible values of type `TColumn`. */
-class GoodExpectation extends ValidExpectation {
-  GoodExpectation() { getKnownFailure() = "" }
-}
-
-class FalsePositiveExpectation extends ValidExpectation {
-  FalsePositiveExpectation() { getKnownFailure() = "SPURIOUS" }
-}
-
-class FalseNegativeExpectation extends ValidExpectation {
-  FalseNegativeExpectation() { getKnownFailure() = "MISSING" }
-}
-
-class InvalidExpectation extends Expectation, TInvalidExpectation {
-  string expectation;
-
-  InvalidExpectation() { this = TInvalidExpectation(comment, expectation) }
-
-  string getExpectation() { result = expectation }
-}
-
-query predicate failures(FailureLocatable element, string message) {
-  exists(InlineExpectationsTest test | test.hasFailureMessage(element, message))
-}
+import Make<Impl>

python/ql/test/TestUtilities/InlineExpectationsTestPrivate.qll

@@ -1,7 +0,0 @@
-import python
-
-/**
- * A class representing line comments in Python. As this is the only form of comment Python
- * permits, we simply reuse the `Comment` class.
- */
-class ExpectationComment = Comment;

python/ql/test/experimental/dataflow/TestUtil/MaximalFlowTest.qll

@@ -29,7 +29,7 @@ class MaximalFlowsConfig extends DataFlow::Configuration {
     not node instanceof DataFlow::PostUpdateNode and
     // not node.asExpr() instanceof FunctionExpr and
     // not node.asExpr() instanceof ClassExpr and
-    not exists(DataFlow::Node pred | DataFlow::localFlowStep(pred, node))
+    not DataFlow::localFlowStep(_, node)
   }
 
   override predicate isSink(DataFlow::Node node) {
@@ -37,6 +37,6 @@ class MaximalFlowsConfig extends DataFlow::Configuration {
     not any(CallNode c).getArg(_) = node.asCfgNode() and
     not node instanceof DataFlow::ArgumentNode and
     not node.asCfgNode().(NameNode).getId().matches("SINK%") and
-    not exists(DataFlow::Node succ | DataFlow::localFlowStep(node, succ))
+    not DataFlow::localFlowStep(node, _)
   }
 }

python/ql/test/experimental/dataflow/coverage/classes.py

@@ -1521,7 +1521,7 @@ class With_await:
     def __await__(self):
         SINK1(self)
         OK()  # Call not found
-        return (yield from asyncio.coroutine(lambda: "")())
+        return (yield from [])
 
 
 async def atest_await():

python/ql/test/experimental/dataflow/exceptions/NormalDataflowTest.expected

@@ -0,0 +1,2 @@
+missingAnnotationOnSink
+failures

python/ql/test/experimental/dataflow/exceptions/NormalDataflowTest.ql

@@ -0,0 +1,2 @@
+import python
+import experimental.dataflow.TestUtil.NormalDataflowTest

python/ql/test/experimental/dataflow/exceptions/dataflow-consistency.expected

@@ -0,0 +1,24 @@
+uniqueEnclosingCallable
+uniqueType
+uniqueNodeLocation
+missingLocation
+uniqueNodeToString
+missingToString
+parameterCallable
+localFlowIsLocal
+readStepIsLocal
+storeStepIsLocal
+compatibleTypesReflexive
+unreachableNodeCCtx
+localCallNodes
+postIsNotPre
+postHasUniquePre
+uniquePostUpdate
+postIsInSameCallable
+reverseRead
+argHasPostUpdate
+postWithInFlow
+viableImplInCallContextTooLarge
+uniqueParameterNodeAtPosition
+uniqueParameterNodePosition
+uniqueContentApprox

python/ql/test/experimental/dataflow/exceptions/dataflow-consistency.ql

@@ -0,0 +1 @@
+import semmle.python.dataflow.new.internal.DataFlowImplConsistency::Consistency

python/ql/test/experimental/dataflow/exceptions/test.py

@@ -0,0 +1,35 @@
+import sys
+import os
+
+sys.path.append(os.path.dirname(os.path.dirname((__file__))))
+from testlib import expects
+
+# These are defined so that we can evaluate the test code.
+NONSOURCE = "not a source"
+SOURCE = "source"
+
+
+def is_source(x):
+    return x == "source" or x == b"source" or x == 42 or x == 42.0 or x == 42j
+
+
+def SINK(x):
+    if is_source(x):
+        print("OK")
+    else:
+        print("Unexpected flow", x)
+
+
+def SINK_F(x):
+    if is_source(x):
+        print("Unexpected flow", x)
+    else:
+        print("OK")
+
+def test_as_binding():
+    try:
+        e_with_source = Exception()
+        e_with_source.a = SOURCE
+        raise e_with_source
+    except Exception as e:
+        SINK(e.a) # $ MISSING: flow

python/ql/test/experimental/dataflow/exceptions/test_group.py

@@ -0,0 +1,77 @@
+import sys
+import os
+
+sys.path.append(os.path.dirname(os.path.dirname((__file__))))
+from testlib import expects
+
+# These are defined so that we can evaluate the test code.
+NONSOURCE = "not a source"
+SOURCE = "source"
+
+
+def is_source(x):
+    return x == "source" or x == b"source" or x == 42 or x == 42.0 or x == 42j
+
+
+def SINK(x):
+    if is_source(x):
+        print("OK")
+    else:
+        print("Unexpected flow", x)
+
+
+def SINK_F(x):
+    if is_source(x):
+        print("Unexpected flow", x)
+    else:
+        print("OK")
+
+def test_as_binding():
+    try:
+        e_with_source = Exception()
+        e_with_source.a = SOURCE
+        raise e_with_source
+    except* Exception as eg:
+        SINK(eg.exceptions[0].a) # $ MISSING: flow
+
+@expects(4)
+def test_group():
+    value_error_with_source = ValueError()
+    value_error_with_source.a = SOURCE
+
+    type_error_without_source = TypeError()
+    type_error_without_source.a = NONSOURCE
+
+    os_error_without_source = OSError()
+    os_error_without_source.a = NONSOURCE
+
+    eg = ExceptionGroup(
+        "one",
+        [
+            type_error_without_source,
+            ExceptionGroup(
+                "two",
+                    [type_error_without_source, value_error_with_source]
+            ),
+            ExceptionGroup(
+                    "three",
+                    [os_error_without_source]
+            )
+        ]
+    )
+    try:
+        raise eg
+    except* (TypeError, OSError) as es:
+        # The matched sub-group, represented by `es` is filtered,
+        # but the nesting is in place.
+        SINK_F(es.exceptions[0].a)
+        SINK_F(es.exceptions[1].exceptions[0].a)
+        SINK_F(es.exceptions[2].exceptions[0].a)
+    except* ValueError as es:
+        # The matched sub-group, represented by `es` is filtered,
+        # but the nesting is in place.
+        # So the ValueError was originally found in
+        # `eg.exceptions[1].exceptions[1].a`
+        # but now it is found in
+        # `es.exceptions[0].exceptions[0].a`
+        SINK(es.exceptions[0].exceptions[0].a) # $ MISSING: flow

python/ql/test/experimental/dataflow/validTest.py

@@ -51,6 +51,14 @@ def check_tests_valid(testFile):
                 check_async_test_function(item)
 
 
+def check_tests_valid_after_version(testFile, version):
+
+    if sys.version_info[:2] >= version:
+        print("INFO: Will run tests in", testFile, "since we're running Python", version, "or newer")
+        check_tests_valid(testFile)
+    else:
+        print("WARN: Will not run tests in", testFile, "since we're running Python", sys.version_info[:2], "and need", version, "or newer")
+
 if __name__ == "__main__":
     check_tests_valid("coverage.classes")
     check_tests_valid("coverage.test")
@@ -61,12 +69,9 @@ if __name__ == "__main__":
     check_tests_valid("variable-capture.dict")
     check_tests_valid("module-initialization.multiphase")
     check_tests_valid("fieldflow.test")
-
-    if sys.version_info[:2] >= (3, 10):
-        print("INFO: Will run `match` tests since we're running Python 3.10 or newer")
-        check_tests_valid("match.test")
-    else:
-        print("WARN: Skipping `match` tests since we're not running 3.10 or newer")
+    check_tests_valid_after_version("match.test", (3, 10))
+    check_tests_valid("exceptions.test")
+    check_tests_valid_after_version("exceptions.test_group", (3, 11))
 
     # The below fails when trying to import modules
     # check_tests_valid("module-initialization.test")

python/ql/test/experimental/dataflow/variable-capture/CaptureTest.ql

@@ -9,9 +9,7 @@ class CaptureTest extends InlineExpectationsTest {
   override string getARelevantTag() { result = "captured" }
 
   override predicate hasActualResult(Location location, string element, string tag, string value) {
-    exists(DataFlow::Node source, DataFlow::Node sink |
-      exists(TestConfiguration cfg | cfg.hasFlow(source, sink))
-    |
+    exists(DataFlow::Node sink | exists(TestConfiguration cfg | cfg.hasFlowTo(sink)) |
       location = sink.getLocation() and
       tag = "captured" and
       value = "" and

python/ql/test/library-tests/ApiGraphs/py2/use.ql

@@ -13,7 +13,7 @@ class ApiUseTest extends InlineExpectationsTest {
   }
 
   override predicate hasActualResult(Location location, string element, string tag, string value) {
-    exists(API::Node a, DataFlow::Node n | relevant_node(a, n, location) |
+    exists(DataFlow::Node n | relevant_node(_, n, location) |
       tag = "use" and
       // Only report the longest path on this line:
       value =

python/ql/test/library-tests/regex/charSetTest.py

@@ -25,14 +25,12 @@ except re.error:
 
 re.compile("]]][[[[]") #$ charSet=3:8
 
-
-#ODASA-3985
 #Half Surrogate pairs
 re.compile(u'[\uD800-\uDBFF][\uDC00-\uDFFF]') #$ charSet=0:5 charSet=5:10
 #Outside BMP
 re.compile(u'[\U00010000-\U0010ffff]') #$ charSet=0:5
 
-#Misparsed on LGTM
+#Misparsed
 re.compile(r"\[(?P<txt>[^[]*)\]\((?P<uri>[^)]*)") #$ charSet=10:14 charSet=28:32
 
  # parses wrongly, sees this   \|/ as a char set start

python/ql/test/library-tests/regex/escapedCharacterTest.py

@@ -21,7 +21,7 @@ re.compile(u'[\uD800-\uDBFF][\uDC00-\uDFFF]') # not escapes
 #Outside BMP
 re.compile(u'[\U00010000-\U0010ffff]') # not escapes
 
-#Misparsed on LGTM
+#Misparsed
 re.compile(r"\[(?P<txt>[^[]*)\]\((?P<uri>[^)]*)") #$ escapedCharacter=0:2 escapedCharacter=16:18 escapedCharacter=18:20
 
 #Non-raw string

python/ql/test/library-tests/regex/test.py

@@ -62,7 +62,7 @@ re.compile(r'x|')
 #Named group with caret and empty choice.
 re.compile(r'(?:(?P<n1>^(?:|x)))')
 
-#Misparsed on LGTM
+#Misparsed
 re.compile(r"\[(?P<txt>[^[]*)\]\((?P<uri>[^)]*)")
 
 re.compile("", re.M) # ODASA-8056

python/ql/test/library-tests/regexparser/unittests.py

@@ -11,7 +11,7 @@ re.compile(r'(?:.|\n)*b', re.DOTALL) # Has ReDoS.
 # minimal example constructed by @erik-krogh
 baz = re.compile(r'\+0')
 
-# exerpts from LGTM.com
+# exerpts from real-world results
 re.compile(r'\+0x')
 re.compile(r'\+0x.*')
 re.compile(r'+\-0+\.')

python/ql/test/query-tests/Classes/conflicting/odasa6643.py

@@ -1,8 +1,6 @@
 #This code has conflicting attributes,
 #but the documentation in the standard library tells you do it this way :(
 
-#See https://discuss.lgtm.com/t/warning-on-normal-use-of-python-socketserver-mixins/677
-
 class ThreadingMixIn(object):
 
     def process_request(selfself, req):

python/ql/test/query-tests/Classes/missing-init/missing_init.py

@@ -39,7 +39,7 @@ class UVT(UT, PU):
 class IUVT(IUT, UVT):
     pass
 
-#False positive observed on LGTM
+#False positive
 class M1(object):
     def __init__(self):
         print("A")

python/ql/test/query-tests/Expressions/eq/expressions_test.py

@@ -78,7 +78,7 @@ class C(object):
         '''
         return self is other
 
-#Was FP -- https://github.com/lgtmhq/lgtm-queries/issues/13
+#Was FP
 def both_sides_known(zero_based="auto", query_id=False):
     if query_id and zero_based == "auto":
         zero_based = True

python/ql/test/query-tests/Functions/general/explicit_return_in_init.py

@@ -46,7 +46,7 @@ class InitCallsBadInit(ExplicitReturnInInit):
 
 # OK as procedure implicitly returns None
 #
-# this was seen in the wild: https://lgtm.com/projects/b/jjburton/cgmtools/snapshot/0d8a429b7ea17854a5e5341df98b1cbd54d7fe6c/files/mayaTools/cgm/lib/classes/AttrFactory.py?sort=name&dir=ASC&mode=heatmap#L90
+# this was seen in the wild:
 # using a pattern of `return procedure_that_logs_error()`
 
 def procedure():

python/ql/test/query-tests/Functions/general/parameter_names.py

@@ -127,7 +127,7 @@ Z().meth(0)
 # makes it such. As a consequence, the query `py/not-named-self` and other
 # relevant queries need to account for this.
 #
-# This has come up in the wild via LGTM as a false positive. For example,
+# This has come up in the wild as a false positive. For example,
 # `__init_subclass__`:
 #   https://docs.python.org/3/reference/datamodel.html#customizing-class-creation
 # `__class_getitem__`:

python/ql/test/query-tests/Variables/unused/variables_test.py

@@ -127,7 +127,7 @@ def decorated_inner_function():
 
 
 
-#FP observed https://lgtm.com/projects/g/torchbox/wagtail/alerts/ 
+#FP observed
 def test_dict_unpacking(queryset, field_name, value):
     #True positive
     for tag in value.split(','):

python/ql/test/query-tests/analysis/suppression/AlertSuppression.expected

@@ -14,7 +14,11 @@
 | test.py:18:4:18:12 | Comment #    lgtm |     lgtm | lgtm | test.py:18:1:18:12 | suppression range |
 | test.py:19:4:19:31 | Comment # lgtm    [py/line-too-long] |  lgtm    [py/line-too-long] | lgtm    [py/line-too-long] | test.py:19:1:19:31 | suppression range |
 | test.py:20:4:20:14 | Comment # lgtm lgtm |  lgtm lgtm | lgtm | test.py:20:1:20:14 | suppression range |
+| test.py:23:1:23:41 | Comment #lgtm -- Ignore this -- No line or scope. | lgtm -- Ignore this -- No line or scope. | lgtm | test.py:23:1:23:41 | suppression range |
+| test.py:23:1:23:41 | Comment #lgtm -- Ignore this -- No line or scope. | lgtm -- Ignore this -- No line or scope. | lgtm | test.py:24:0:24:0 | suppression range |
 | test.py:27:12:27:23 | Comment #lgtm [func] | lgtm [func] | lgtm [func] | test.py:27:1:27:23 | suppression range |
+| test.py:28:5:28:70 | Comment # lgtm -- Blank line (ignore for now, maybe scope wide in future). |  lgtm -- Blank line (ignore for now, maybe scope wide in future). | lgtm | test.py:28:1:28:70 | suppression range |
+| test.py:28:5:28:70 | Comment # lgtm -- Blank line (ignore for now, maybe scope wide in future). |  lgtm -- Blank line (ignore for now, maybe scope wide in future). | lgtm | test.py:29:0:29:0 | suppression range |
 | test.py:29:17:29:35 | Comment # lgtm on docstring |  lgtm on docstring | lgtm | test.py:29:1:29:35 | suppression range |
 | test.py:30:16:30:47 | Comment #lgtm [py/duplicate-key-in-dict] | lgtm [py/duplicate-key-in-dict] | lgtm [py/duplicate-key-in-dict] | test.py:30:1:30:47 | suppression range |
 | test.py:35:10:35:21 | Comment # lgtm class |  lgtm class | lgtm | test.py:35:1:35:21 | suppression range |
@@ -22,6 +26,8 @@
 | test.py:39:4:39:8 | Comment #noqa | noqa | lgtm | test.py:39:1:39:8 | suppression range |
 | test.py:40:4:40:9 | Comment # noqa |  noqa | lgtm | test.py:40:1:40:9 | suppression range |
 | test.py:45:4:45:31 | Comment # noqa -- Some extra detail. |  noqa -- Some extra detail. | lgtm | test.py:45:1:45:31 | suppression range |
+| test.py:49:1:49:10 | Comment #LGTM-1929 | LGTM-1929 | LGTM | test.py:49:1:49:10 | suppression range |
+| test.py:49:1:49:10 | Comment #LGTM-1929 | LGTM-1929 | LGTM | test.py:50:0:50:0 | suppression range |
 | test.py:50:34:50:117 | Comment # noqa: E501; (line too long) pylint: disable=invalid-name; lgtm [py/missing-equals] |  noqa: E501; (line too long) pylint: disable=invalid-name; lgtm [py/missing-equals] | lgtm [py/missing-equals] | test.py:50:1:50:117 | suppression range |
 | test.py:52:4:52:67 | Comment # noqa: E501; (line too long) pylint: disable=invalid-name; lgtm |  noqa: E501; (line too long) pylint: disable=invalid-name; lgtm | lgtm | test.py:52:1:52:67 | suppression range |
 | test.py:53:4:53:78 | Comment # random nonsense lgtm [py/missing-equals] and then some more commentary... |  random nonsense lgtm [py/missing-equals] and then some more commentary... | lgtm [py/missing-equals] | test.py:53:1:53:78 | suppression range |
@@ -31,6 +37,9 @@
 | test.py:65:4:65:60 | Comment # lgtm[py/line-too-long] and lgtm[py/non-callable-called] |  lgtm[py/line-too-long] and lgtm[py/non-callable-called] | lgtm[py/non-callable-called] | test.py:65:1:65:60 | suppression range |
 | test.py:66:4:66:33 | Comment # lgtm[py/line-too-long]; lgtm |  lgtm[py/line-too-long]; lgtm | lgtm | test.py:66:1:66:33 | suppression range |
 | test.py:66:4:66:33 | Comment # lgtm[py/line-too-long]; lgtm |  lgtm[py/line-too-long]; lgtm | lgtm[py/line-too-long] | test.py:66:1:66:33 | suppression range |
+| test.py:69:1:69:26 | Comment # codeql[py/line-too-long] |  codeql[py/line-too-long] | lgtm[py/line-too-long] | test.py:70:0:70:0 | suppression range |
+| test.py:71:1:71:25 | Comment #CODEQL[py/line-too-long] | CODEQL[py/line-too-long] | lgtm[py/line-too-long] | test.py:72:0:72:0 | suppression range |
+| test.py:73:1:73:63 | Comment # codeql[py/line-too-long] -- because I know better than codeql |  codeql[py/line-too-long] -- because I know better than codeql | lgtm[py/line-too-long] | test.py:74:0:74:0 | suppression range |
 | testWindows.py:4:4:4:9 | Comment # lgtm |  lgtm | lgtm | testWindows.py:4:1:4:9 | suppression range |
 | testWindows.py:5:4:5:27 | Comment # lgtm[py/line-too-long] |  lgtm[py/line-too-long] | lgtm[py/line-too-long] | testWindows.py:5:1:5:27 | suppression range |
 | testWindows.py:6:4:6:51 | Comment # lgtm[py/line-too-long, py/non-callable-called] |  lgtm[py/line-too-long, py/non-callable-called] | lgtm[py/line-too-long, py/non-callable-called] | testWindows.py:6:1:6:51 | suppression range |
@@ -47,14 +56,22 @@
 | testWindows.py:18:4:18:12 | Comment #    lgtm |     lgtm | lgtm | testWindows.py:18:1:18:12 | suppression range |
 | testWindows.py:19:4:19:31 | Comment # lgtm    [py/line-too-long] |  lgtm    [py/line-too-long] | lgtm    [py/line-too-long] | testWindows.py:19:1:19:31 | suppression range |
 | testWindows.py:20:4:20:14 | Comment # lgtm lgtm |  lgtm lgtm | lgtm | testWindows.py:20:1:20:14 | suppression range |
+| testWindows.py:23:1:23:41 | Comment #lgtm -- Ignore this -- No line or scope. | lgtm -- Ignore this -- No line or scope. | lgtm | testWindows.py:23:1:23:41 | suppression range |
+| testWindows.py:23:1:23:41 | Comment #lgtm -- Ignore this -- No line or scope. | lgtm -- Ignore this -- No line or scope. | lgtm | testWindows.py:24:0:24:0 | suppression range |
 | testWindows.py:27:12:27:23 | Comment #lgtm [func] | lgtm [func] | lgtm [func] | testWindows.py:27:1:27:23 | suppression range |
+| testWindows.py:28:5:28:70 | Comment # lgtm -- Blank line (ignore for now, maybe scope wide in future). |  lgtm -- Blank line (ignore for now, maybe scope wide in future). | lgtm | testWindows.py:28:1:28:70 | suppression range |
+| testWindows.py:28:5:28:70 | Comment # lgtm -- Blank line (ignore for now, maybe scope wide in future). |  lgtm -- Blank line (ignore for now, maybe scope wide in future). | lgtm | testWindows.py:29:0:29:0 | suppression range |
 | testWindows.py:29:17:29:35 | Comment # lgtm on docstring |  lgtm on docstring | lgtm | testWindows.py:29:1:29:35 | suppression range |
 | testWindows.py:30:16:30:47 | Comment #lgtm [py/duplicate-key-in-dict] | lgtm [py/duplicate-key-in-dict] | lgtm [py/duplicate-key-in-dict] | testWindows.py:30:1:30:47 | suppression range |
 | testWindows.py:35:10:35:21 | Comment # lgtm class |  lgtm class | lgtm | testWindows.py:35:1:35:21 | suppression range |
 | testWindows.py:36:21:36:33 | Comment # lgtm method |  lgtm method | lgtm | testWindows.py:36:1:36:33 | suppression range |
 | testWindows.py:39:3:39:7 | Comment #noqa | noqa | lgtm | testWindows.py:39:1:39:7 | suppression range |
 | testWindows.py:40:4:40:9 | Comment # noqa |  noqa | lgtm | testWindows.py:40:1:40:9 | suppression range |
+| testWindows.py:45:1:45:28 | Comment # noqa -- Some extra detail. |  noqa -- Some extra detail. | lgtm | testWindows.py:45:1:45:28 | suppression range |
 | testWindows.py:48:4:48:60 | Comment # lgtm[py/line-too-long] and lgtm[py/non-callable-called] |  lgtm[py/line-too-long] and lgtm[py/non-callable-called] | lgtm[py/line-too-long] | testWindows.py:48:1:48:60 | suppression range |
 | testWindows.py:48:4:48:60 | Comment # lgtm[py/line-too-long] and lgtm[py/non-callable-called] |  lgtm[py/line-too-long] and lgtm[py/non-callable-called] | lgtm[py/non-callable-called] | testWindows.py:48:1:48:60 | suppression range |
 | testWindows.py:49:4:49:33 | Comment # lgtm[py/line-too-long]; lgtm |  lgtm[py/line-too-long]; lgtm | lgtm | testWindows.py:49:1:49:33 | suppression range |
 | testWindows.py:49:4:49:33 | Comment # lgtm[py/line-too-long]; lgtm |  lgtm[py/line-too-long]; lgtm | lgtm[py/line-too-long] | testWindows.py:49:1:49:33 | suppression range |
+| testWindows.py:52:1:52:26 | Comment # codeql[py/line-too-long] |  codeql[py/line-too-long] | lgtm[py/line-too-long] | testWindows.py:53:0:53:0 | suppression range |
+| testWindows.py:54:1:54:25 | Comment #CODEQL[py/line-too-long] | CODEQL[py/line-too-long] | lgtm[py/line-too-long] | testWindows.py:55:0:55:0 | suppression range |
+| testWindows.py:56:1:56:63 | Comment # codeql[py/line-too-long] -- because I know better than codeql |  codeql[py/line-too-long] -- because I know better than codeql | lgtm[py/line-too-long] | testWindows.py:57:0:57:0 | suppression range |

python/ql/test/query-tests/analysis/suppression/AlertSuppression.qlref

@@ -1 +1 @@
-analysis/AlertSuppression.ql
+AlertSuppression.ql

python/ql/test/query-tests/analysis/suppression/test.py

@@ -64,3 +64,12 @@ class frozenbidict(BidictBase):  # noqa: E501; (line too long) pylint: disable=i
 
 "" # lgtm[py/line-too-long] and lgtm[py/non-callable-called]
 "" # lgtm[py/line-too-long]; lgtm
+
+#CodeQL comments
+# codeql[py/line-too-long]
+""
+#CODEQL[py/line-too-long]
+""
+# codeql[py/line-too-long] -- because I know better than codeql
+""
+"" # codeql[py/line-too-long]

python/ql/test/query-tests/analysis/suppression/testWindows.py

@@ -47,3 +47,12 @@ class C: # lgtm class
 
 "" # lgtm[py/line-too-long] and lgtm[py/non-callable-called]
 "" # lgtm[py/line-too-long]; lgtm
+
+#CodeQL comments
+# codeql[py/line-too-long]
+""
+#CODEQL[py/line-too-long]
+""
+# codeql[py/line-too-long] -- because I know better than codeql
+""
+"" # codeql[py/line-too-long]