Merge branch 'main' of https://github.com/github/codeql into python/captured-variables-basic

python/ql/lib/change-notes/2023-12-20-add-scope-entry-definition-nodes.md

@@ -0,0 +1,5 @@
+---
+category: fix
+---
+
+- We would previously confuse all captured variables into a single scope entry node. Now they each get their own node so they can be tracked properly.

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPrivate.qll

@@ -361,7 +361,10 @@ module LocalFlow {
     //   nodeFrom is `y` on first line
     //   nodeTo is `y` on second line
     exists(EssaDefinition def |
-      nodeFrom.(CfgNode).getNode() = def.(EssaNodeDefinition).getDefiningNode() and
+      nodeFrom.(CfgNode).getNode() = def.(EssaNodeDefinition).getDefiningNode()
+      or
+      nodeFrom.(ScopeEntryDefinitionNode).getDefinition() = def
+    |
       AdjacentUses::firstUse(def, nodeTo.(CfgNode).getNode())
     )
     or
@@ -492,8 +495,7 @@ predicate simpleLocalFlowStep(Node nodeFrom, Node nodeTo) {
  * or at runtime when callables in the module are called.
  */
 predicate simpleLocalFlowStepForTypetracking(Node nodeFrom, Node nodeTo) {
-  IncludePostUpdateFlow<PhaseDependentFlow<LocalFlow::localFlowStep/2>::step/2>::step(nodeFrom,
-    nodeTo)
+  LocalFlow::localFlowStep(nodeFrom, nodeTo)
 }
 
 private predicate summaryLocalStep(Node nodeFrom, Node nodeTo) {

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPublic.qll

@@ -27,9 +27,12 @@ newtype TNode =
     isExpressionNode(node)
     or
     node.getNode() instanceof Pattern
-    or
-    node = any(ScopeEntryDefinition def | not def.getScope() instanceof Module).getDefiningNode()
   } or
+  /**
+   * A node corresponding to a scope entry definition. That is, the value of a variable
+   * as it enters a scope.
+   */
+  TScopeEntryDefinitionNode(ScopeEntryDefinition def) { not def.getScope() instanceof Module } or
   /**
    * A synthetic node representing the value of an object before a state change.
    *
@@ -265,6 +268,28 @@ class ExprNode extends CfgNode {
 /** Gets a node corresponding to expression `e`. */
 ExprNode exprNode(DataFlowExpr e) { result.getNode().getNode() = e }
 
+/**
+ * A node corresponding to a scope entry definition. That is, the value of a variable
+ * as it enters a scope.
+ */
+class ScopeEntryDefinitionNode extends Node, TScopeEntryDefinitionNode {
+  ScopeEntryDefinition def;
+
+  ScopeEntryDefinitionNode() { this = TScopeEntryDefinitionNode(def) }
+
+  /** Gets the `ScopeEntryDefinition` associated with this node. */
+  ScopeEntryDefinition getDefinition() { result = def }
+
+  /** Gets the source variable represented by this node. */
+  SsaSourceVariable getVariable() { result = def.getSourceVariable() }
+
+  override Location getLocation() { result = def.getLocation() }
+
+  override Scope getScope() { result = def.getScope() }
+
+  override string toString() { result = "Entry definition for " + this.getVariable().toString() }
+}
+
 /**
  * The value of a parameter at function entry, viewed as a node in a data
  * flow graph.

python/ql/lib/semmle/python/dataflow/new/internal/LocalSources.qll

@@ -71,7 +71,7 @@ class LocalSourceNode extends Node {
     or
     // We include all scope entry definitions, as these act as the local source within the scope they
     // enter.
-    this.asCfgNode() = any(ScopeEntryDefinition def).getDefiningNode()
+    this instanceof ScopeEntryDefinitionNode
     or
     this instanceof ParameterNode
   }
@@ -167,7 +167,7 @@ class LocalSourceNodeNotModuleVariableNode extends LocalSourceNode {
   LocalSourceNodeNotModuleVariableNode() {
     this instanceof ExprNode
     or
-    this.asCfgNode() = any(ScopeEntryDefinition def).getDefiningNode()
+    this instanceof ScopeEntryDefinitionNode
   }
 }
 

python/ql/lib/semmle/python/dataflow/new/internal/TypeTrackingImpl.qll

@@ -251,7 +251,7 @@ module TypeTrackingInput implements Shared::TypeTrackingInput {
       e.getSourceVariable() = var and
       var.hasDefiningNode(def)
     |
-      nodeTo.asCfgNode() = e.getDefiningNode() and
+      nodeTo.(DataFlowPublic::ScopeEntryDefinitionNode).getDefinition() = e and
       nodeFrom.asCfgNode() = def.getValue() and
       var.getScope().getScope*() = nodeFrom.getScope()
     )