Merge pull request #15711 from RasmusWL/tt-content

Python: Add type tracking for content

python/ql/consistency-queries/TypeTrackingConsistency.ql

@@ -27,15 +27,18 @@ private module ConsistencyChecksInput implements ConsistencyChecksInputSig {
       TypeTrackingInput::simpleLocalSmallStep*(m, n)
     )
     or
-    // TODO: when adding support for proper content, handle iterable unpacking better
-    // such as `for k,v in items:`, or `a, (b,c) = ...`
-    n instanceof DataFlow::IterableSequenceNode
-    or
     // We have missing use-use flow in
     // https://github.com/python/cpython/blob/0fb18b02c8ad56299d6a2910be0bab8ad601ef24/Lib/socketserver.py#L276-L303
     // which I couldn't just fix. We ignore the problems here, and instead rely on the
     // test-case added in https://github.com/github/codeql/pull/15841
     n.getLocation().getFile().getAbsolutePath().matches("%/socketserver.py")
+    or
+    // for iterable unpacking like `a,b = some_list`, we currently don't want to allow
+    // type-tracking... however, in the future when we allow tracking list indexes
+    // precisely (that is, move away from ListElementContent), we should ensure we have
+    // proper flow to the synthetic `IterableElementNode`.
+    exists(DataFlow::ListElementContent c) and
+    n instanceof DataFlow::IterableElementNode
   }
 }
 

python/ql/lib/change-notes/2024-03-12-typetracking-content.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Improved the type-tracking capabilities (and therefore also API graphs) to allow tracking items in tuples and dictionaries.

python/ql/lib/semmle/python/dataflow/new/TypeTracking.qll

@@ -5,9 +5,14 @@
 
 private import internal.TypeTrackingImpl as Impl
 import Impl::Shared::TypeTracking<Impl::TypeTrackingInput>
+private import semmle.python.dataflow.new.internal.DataFlowPublic as DataFlowPublic
 
-/** A string that may appear as the name of an attribute or access path. */
+/**
-class AttributeName = Impl::TypeTrackingInput::Content;
+ * DEPRECATED.
+ *
+ * A string that may appear as the name of an attribute or access path.
+ */
+deprecated class AttributeName = Impl::TypeTrackingInput::Content;
 
 /**
  * A summary of the steps needed to track a value to a given dataflow node.
@@ -40,7 +45,11 @@ class TypeTracker extends Impl::TypeTracker {
    * Holds if this is the starting point of type tracking, and the value starts in the attribute named `attrName`.
    * The type tracking only ends after the attribute has been loaded.
    */
-  predicate startInAttr(string attrName) { this.startInContent(attrName) }
+  predicate startInAttr(string attrName) {
+    exists(DataFlowPublic::AttributeContent content | content.getAttribute() = attrName |
+      this.startInContent(content)
+    )
+  }
 
   /**
    * INTERNAL. DO NOT USE.
@@ -48,9 +57,8 @@ class TypeTracker extends Impl::TypeTracker {
    * Gets the attribute associated with this type tracker.
    */
   string getAttr() {
-    result = this.getContent().asSome()
+    if this.getContent().asSome() instanceof DataFlowPublic::AttributeContent
-    or
+    then result = this.getContent().asSome().(DataFlowPublic::AttributeContent).getAttribute()
-    this.getContent().isNone() and
+    else result = ""
-    result = ""
   }
 }

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPrivate.qll

@@ -642,23 +642,37 @@ predicate jumpStepNotSharedWithTypeTracker(Node nodeFrom, Node nodeTo) {
 // Field flow
 //--------
 /**
- * Holds if data can flow from `nodeFrom` to `nodeTo` via an assignment to
+ * Subset of `storeStep` that should be shared with type-tracking.
- * content `c`.
+ *
+ * NOTE: This does not include attributeStoreStep right now, since it has its' own
+ * modeling in the type-tracking library (which is slightly different due to
+ * PostUpdateNodes).
+ *
+ * As of 2024-04-02 the type-tracking library only supports precise content, so there is
+ * no reason to include steps for list content right now.
  */
-predicate storeStep(Node nodeFrom, ContentSet c, Node nodeTo) {
+predicate storeStepCommon(Node nodeFrom, ContentSet c, Node nodeTo) {
-  listStoreStep(nodeFrom, c, nodeTo)
-  or
-  setStoreStep(nodeFrom, c, nodeTo)
-  or
   tupleStoreStep(nodeFrom, c, nodeTo)
   or
   dictStoreStep(nodeFrom, c, nodeTo)
   or
   moreDictStoreSteps(nodeFrom, c, nodeTo)
   or
-  comprehensionStoreStep(nodeFrom, c, nodeTo)
-  or
   iterableUnpackingStoreStep(nodeFrom, c, nodeTo)
+}
+
+/**
+ * Holds if data can flow from `nodeFrom` to `nodeTo` via an assignment to
+ * content `c`.
+ */
+predicate storeStep(Node nodeFrom, ContentSet c, Node nodeTo) {
+  storeStepCommon(nodeFrom, c, nodeTo)
+  or
+  listStoreStep(nodeFrom, c, nodeTo)
+  or
+  setStoreStep(nodeFrom, c, nodeTo)
+  or
+  comprehensionStoreStep(nodeFrom, c, nodeTo)
   or
   attributeStoreStep(nodeFrom, c, nodeTo)
   or
@@ -892,12 +906,19 @@ predicate attributeStoreStep(Node nodeFrom, AttributeContent c, Node nodeTo) {
 }
 
 /**
- * Holds if data can flow from `nodeFrom` to `nodeTo` via a read of content `c`.
+ * Subset of `readStep` that should be shared with type-tracking.
  */
-predicate readStep(Node nodeFrom, ContentSet c, Node nodeTo) {
+predicate readStepCommon(Node nodeFrom, ContentSet c, Node nodeTo) {
   subscriptReadStep(nodeFrom, c, nodeTo)
   or
   iterableUnpackingReadStep(nodeFrom, c, nodeTo)
+}
+
+/**
+ * Holds if data can flow from `nodeFrom` to `nodeTo` via a read of content `c`.
+ */
+predicate readStep(Node nodeFrom, ContentSet c, Node nodeTo) {
+  readStepCommon(nodeFrom, c, nodeTo)
   or
   matchReadStep(nodeFrom, c, nodeTo)
   or

python/ql/lib/semmle/python/dataflow/new/internal/TypeTracker.qll

@@ -1,6 +1,7 @@
 /** Step Summaries and Type Tracking */
 
 private import TypeTrackerSpecific
+private import semmle.python.dataflow.new.internal.DataFlowPublic as DataFlowPublic
 
 cached
 private module Cached {
@@ -12,10 +13,22 @@ private module Cached {
     LevelStep() or
     CallStep() or
     ReturnStep() or
-    deprecated StoreStep(TypeTrackerContent content) { basicStoreStep(_, _, content) } or
+    deprecated StoreStep(TypeTrackerContent content) {
-    deprecated LoadStep(TypeTrackerContent content) { basicLoadStep(_, _, content) } or
+      exists(DataFlowPublic::AttributeContent dfc | dfc.getAttribute() = content |
+        basicStoreStep(_, _, dfc)
+      )
+    } or
+    deprecated LoadStep(TypeTrackerContent content) {
+      exists(DataFlowPublic::AttributeContent dfc | dfc.getAttribute() = content |
+        basicLoadStep(_, _, dfc)
+      )
+    } or
     deprecated LoadStoreStep(TypeTrackerContent load, TypeTrackerContent store) {
-      basicLoadStoreStep(_, _, load, store)
+      exists(DataFlowPublic::AttributeContent dfcLoad, DataFlowPublic::AttributeContent dfcStore |
+        dfcLoad.getAttribute() = load and dfcStore.getAttribute() = store
+      |
+        basicLoadStoreStep(_, _, dfcLoad, dfcStore)
+      )
     } or
     deprecated WithContent(ContentFilter filter) { basicWithContentStep(_, _, filter) } or
     deprecated WithoutContent(ContentFilter filter) { basicWithoutContentStep(_, _, filter) } or
@@ -29,13 +42,13 @@ private module Cached {
       // Restrict `content` to those that might eventually match a load.
       // We can't rely on `basicStoreStep` since `startInContent` might be used with
       // a content that has no corresponding store.
-      exists(TypeTrackerContent loadContents |
+      exists(DataFlowPublic::AttributeContent loadContents |
         (
           basicLoadStep(_, _, loadContents)
           or
           basicLoadStoreStep(_, _, loadContents, _)
         ) and
-        compatibleContents(content, loadContents)
+        compatibleContents(content, loadContents.getAttribute())
       )
     }
 
@@ -45,13 +58,13 @@ private module Cached {
       content = noContent()
       or
       // As in MkTypeTracker, restrict `content` to those that might eventually match a store.
-      exists(TypeTrackerContent storeContent |
+      exists(DataFlowPublic::AttributeContent storeContent |
         (
           basicStoreStep(_, _, storeContent)
           or
           basicLoadStoreStep(_, _, _, storeContent)
         ) and
-        compatibleContents(storeContent, content)
+        compatibleContents(storeContent.getAttribute(), content)
       )
     }
 
@@ -198,7 +211,10 @@ private module Cached {
       flowsToStoreStep(nodeFrom, nodeTo, content) and
       summary = StoreStep(content)
       or
-      basicLoadStep(nodeFrom, nodeTo, content) and summary = LoadStep(content)
+      exists(DataFlowPublic::AttributeContent dfc | dfc.getAttribute() = content |
+        basicLoadStep(nodeFrom, nodeTo, dfc)
+      ) and
+      summary = LoadStep(content)
     )
     or
     exists(TypeTrackerContent loadContent, TypeTrackerContent storeContent |
@@ -281,7 +297,12 @@ deprecated private predicate smallstepProj(Node nodeFrom, StepSummary summary) {
 deprecated private predicate flowsToStoreStep(
   Node nodeFrom, TypeTrackingNode nodeTo, TypeTrackerContent content
 ) {
-  exists(Node obj | nodeTo.flowsTo(obj) and basicStoreStep(nodeFrom, obj, content))
+  exists(Node obj |
+    nodeTo.flowsTo(obj) and
+    exists(DataFlowPublic::AttributeContent dfc | dfc.getAttribute() = content |
+      basicStoreStep(nodeFrom, obj, dfc)
+    )
+  )
 }
 
 /**
@@ -292,7 +313,12 @@ deprecated private predicate flowsToLoadStoreStep(
   TypeTrackerContent storeContent
 ) {
   exists(Node obj |
-    nodeTo.flowsTo(obj) and basicLoadStoreStep(nodeFrom, obj, loadContent, storeContent)
+    nodeTo.flowsTo(obj) and
+    exists(DataFlowPublic::AttributeContent loadDfc, DataFlowPublic::AttributeContent storeDfc |
+      loadDfc.getAttribute() = loadContent and storeDfc.getAttribute() = storeContent
+    |
+      basicLoadStoreStep(nodeFrom, obj, loadDfc, storeDfc)
+    )
   )
 }
 

python/ql/lib/semmle/python/dataflow/new/internal/TypeTrackerSpecific.qll

@@ -15,7 +15,7 @@ deprecated class OptionalTypeTrackerContent extends string {
   OptionalTypeTrackerContent() {
     this = ""
     or
-    this instanceof TypeTrackingImpl::TypeTrackingInput::Content
+    this = any(DataFlowPublic::AttributeContent dfc).getAttribute()
   }
 }
 

python/ql/lib/semmle/python/dataflow/new/internal/TypeTrackingImpl.qll

@@ -8,6 +8,7 @@ private import semmle.python.dataflow.new.internal.DataFlowPrivate as DataFlowPr
 private import codeql.typetracking.internal.SummaryTypeTracker as SummaryTypeTracker
 private import semmle.python.dataflow.new.internal.FlowSummaryImpl as FlowSummaryImpl
 private import semmle.python.dataflow.new.internal.DataFlowDispatch as DataFlowDispatch
+private import semmle.python.dataflow.new.internal.IterableUnpacking as IterableUnpacking
 
 private module SummaryTypeTrackerInput implements SummaryTypeTracker::Input {
   // Dataflow nodes
@@ -97,24 +98,25 @@ private module SummaryTypeTrackerInput implements SummaryTypeTracker::Input {
 
 private module TypeTrackerSummaryFlow = SummaryTypeTracker::SummaryFlow<SummaryTypeTrackerInput>;
 
-/**
- * Gets the name of a possible piece of content. For Python, this is currently only attribute names,
- * using the name of the attribute for the corresponding content.
- */
-private string getPossibleContentName() {
-  Stages::TypeTracking::ref() and // the TypeTracking::append() etc. predicates that we want to cache depend on this predicate, so we can place the `ref()` call here to get around identical files.
-  result = any(DataFlowPublic::AttrRef a).getAttributeName()
-}
-
 module TypeTrackingInput implements Shared::TypeTrackingInput {
   class Node = DataFlowPublic::Node;
 
   class LocalSourceNode = DataFlowPublic::LocalSourceNode;
 
-  class Content instanceof string {
+  class Content extends DataFlowPublic::Content {
-    Content() { this = getPossibleContentName() }
+    Content() {
-
+      // TODO: for now, it's not 100% clear if should support non-precise content in
-    string toString() { result = this }
+      // type-tracking, or if it will lead to bad results. We start with only allowing
+      // precise content, which should always be a good improvement! It also simplifies
+      // the process of examining new results from non-precise content steps in the
+      // future, since you will _only_ have to look over the results from the new
+      // non-precise steps.
+      this instanceof DataFlowPublic::AttributeContent
+      or
+      this instanceof DataFlowPublic::DictionaryElementContent
+      or
+      this instanceof DataFlowPublic::TupleElementContent
+    }
   }
 
   /**
@@ -134,7 +136,27 @@ module TypeTrackingInput implements Shared::TypeTrackingInput {
   }
 
   /** Holds if there is a simple local flow step from `nodeFrom` to `nodeTo` */
-  predicate simpleLocalSmallStep = DataFlowPrivate::simpleLocalFlowStepForTypetracking/2;
+  predicate simpleLocalSmallStep(Node nodeFrom, Node nodeTo) {
+    DataFlowPrivate::simpleLocalFlowStepForTypetracking(nodeFrom, nodeTo) and
+    // for `for k,v in foo` no need to do local flow step from the synthetic sequence
+    // node for `k,v` to the tuple `k,v` -- since type-tracking only supports one level
+    // of content tracking, and there is one read-step from `foo` the synthetic sequence
+    // node required, we can skip the flow step from the synthetic sequence node to the
+    // tuple itself, since the read-step from the tuple to the tuple elements will not
+    // matter.
+    not (
+      IterableUnpacking::iterableUnpackingForReadStep(_, _, nodeFrom) and
+      IterableUnpacking::iterableUnpackingTupleFlowStep(nodeFrom, nodeTo)
+    ) and
+    // for nested iterable unpacking, such as `[[a]] = foo` or `((a,b),) = bar`, we can
+    // ignore the flow steps from the synthetic sequence node to the real sequence node,
+    // since we only support one level of content in type-trackers, and the nested
+    // structure requires two levels at least to be useful.
+    not exists(SequenceNode outer |
+      outer.getAnElement() = nodeTo.asCfgNode() and
+      IterableUnpacking::iterableUnpackingTupleFlowStep(nodeFrom, nodeTo)
+    )
+  }
 
   /** Holds if there is a level step from `nodeFrom` to `nodeTo`, which may depend on the call graph. */
   predicate levelStepCall(Node nodeFrom, LocalSourceNode nodeTo) { none() }
@@ -181,46 +203,68 @@ module TypeTrackingInput implements Shared::TypeTrackingInput {
    * Holds if `nodeFrom` is being written to the `content` content of the object in `nodeTo`.
    */
   predicate storeStep(Node nodeFrom, Node nodeTo, Content content) {
-    exists(DataFlowPublic::AttrWrite a |
+    exists(DataFlowPublic::AttrWrite a, string attrName |
-      a.mayHaveAttributeName(content) and
+      content.(DataFlowPublic::AttributeContent).getAttribute() = attrName and
+      a.mayHaveAttributeName(attrName) and
       nodeFrom = a.getValue() and
       nodeTo = a.getObject()
     )
     or
-    exists(DataFlowPublic::ContentSet contents |
+    // type-tracking doesn't really handle PostUpdateNodes, so for some assignment steps
-      contents.(DataFlowPublic::AttributeContent).getAttribute() = content
+    // like `my_dict["foo"] = foo` the data-flow step targets the PostUpdateNode for
+    // `my_dict`, where we want to translate that into a type-tracking step that targets
+    // the normal/non-PostUpdateNode for `my_dict`.
+    exists(DataFlowPublic::Node storeTarget |
+      DataFlowPrivate::storeStepCommon(nodeFrom, content, storeTarget)
     |
-      TypeTrackerSummaryFlow::basicStoreStep(nodeFrom, nodeTo, contents)
+      not storeTarget instanceof DataFlowPrivate::SyntheticPostUpdateNode and
-    )
+      nodeTo = storeTarget
+      or
+      nodeTo = storeTarget.(DataFlowPrivate::SyntheticPostUpdateNode).getPreUpdateNode()
+    ) and
+    // when only supporting precise content, no need for IterableElementNode (since it
+    // is only fed set/list content)
+    not nodeFrom instanceof DataFlowPublic::IterableElementNode
+    or
+    TypeTrackerSummaryFlow::basicStoreStep(nodeFrom, nodeTo, content)
   }
 
   /**
    * Holds if `nodeTo` is the result of accessing the `content` content of `nodeFrom`.
    */
   predicate loadStep(Node nodeFrom, LocalSourceNode nodeTo, Content content) {
-    exists(DataFlowPublic::AttrRead a |
+    exists(DataFlowPublic::AttrRead a, string attrName |
-      a.mayHaveAttributeName(content) and
+      content.(DataFlowPublic::AttributeContent).getAttribute() = attrName and
+      a.mayHaveAttributeName(attrName) and
       nodeFrom = a.getObject() and
       nodeTo = a
     )
     or
-    exists(DataFlowPublic::ContentSet contents |
+    DataFlowPrivate::readStepCommon(nodeFrom, content, nodeTo) and
-      contents.(DataFlowPublic::AttributeContent).getAttribute() = content
+    // Since we only support one level of content in type-trackers we don't actually
-    |
+    // support `(aa, ab), (ba, bb) = ...`. Therefore we exclude the read-step from `(aa,
-      TypeTrackerSummaryFlow::basicLoadStep(nodeFrom, nodeTo, contents)
+    // ab)` to `aa` (since it is not needed).
+    not exists(SequenceNode outer |
+      outer.getAnElement() = nodeFrom.asCfgNode() and
+      IterableUnpacking::iterableUnpackingTupleFlowStep(_, nodeFrom)
+    ) and
+    // Again, due to only supporting one level deep, for `for (k,v) in ...` we exclude read-step from
+    // the tuple to `k` and `v`.
+    not exists(DataFlowPublic::IterableSequenceNode seq, DataFlowPublic::IterableElementNode elem |
+      IterableUnpacking::iterableUnpackingForReadStep(_, _, seq) and
+      IterableUnpacking::iterableUnpackingConvertingReadStep(seq, _, elem) and
+      IterableUnpacking::iterableUnpackingConvertingStoreStep(elem, _, nodeFrom) and
+      nodeFrom.asCfgNode() instanceof SequenceNode
     )
+    or
+    TypeTrackerSummaryFlow::basicLoadStep(nodeFrom, nodeTo, content)
   }
 
   /**
    * Holds if the `loadContent` of `nodeFrom` is stored in the `storeContent` of `nodeTo`.
    */
   predicate loadStoreStep(Node nodeFrom, Node nodeTo, Content loadContent, Content storeContent) {
-    exists(DataFlowPublic::ContentSet loadContents, DataFlowPublic::ContentSet storeContents |
+    TypeTrackerSummaryFlow::basicLoadStoreStep(nodeFrom, nodeTo, loadContent, storeContent)
-      loadContents.(DataFlowPublic::AttributeContent).getAttribute() = loadContent and
-      storeContents.(DataFlowPublic::AttributeContent).getAttribute() = storeContent
-    |
-      TypeTrackerSummaryFlow::basicLoadStoreStep(nodeFrom, nodeTo, loadContents, storeContents)
-    )
   }
 
   /**

python/ql/test/experimental/dataflow/typetracking/content_test.py

@@ -0,0 +1,100 @@
+# test of other content types than attributes
+
+def test_tuple(index_arg):
+    tup = (tracked, other) # $tracked
+
+    tup[0] # $ tracked
+    tup[1]
+
+    a,b = tup # $tracked
+    a # $ tracked
+    b
+
+    # non-precise access is not supported right now (and it's not 100% clear if we want
+    # to support it, or if it will lead to bad results)
+    tup[index_arg]
+
+    for x in tup:
+        print(x)
+
+    for i in range(len(tup)):
+        print(tup[i])
+
+
+    # nested tuples
+    nested_tuples = ((tracked, other), (other, tracked)) # $tracked
+
+    nested_tuples[0][0] # $ MISSING: tracked
+    nested_tuples[0][1]
+    nested_tuples[1][0]
+    nested_tuples[1][1] # $ MISSING: tracked
+
+    (aa, ab), (ba, bb) = nested_tuples
+    aa # $ MISSING: tracked
+    ab
+    ba
+    bb # $ MISSING: tracked
+
+
+    # non-precise access is not supported right now (and it's not 100% clear if we want
+    # to support it, or if it will lead to bad results)
+    for (x, y) in nested_tuples:
+        x
+        y
+
+
+def test_dict(key_arg):
+    d1 = {"t": tracked, "o": other} # $tracked
+    d1["t"] # $ tracked
+    d1.get("t") # $ MISSING: tracked
+    d1.setdefault("t") # $ MISSING: tracked
+
+    d1["o"]
+    d1.get("o")
+    d1.setdefault("o")
+
+
+    # non-precise access is not supported right now (and it's not 100% clear if we want
+    # to support it, or if it will lead to bad results)
+    d1[key_arg]
+
+    for k in d1:
+        d1[k]
+
+    for v in d1.values():
+        v
+
+    for k, v in d1.items():
+        v
+
+
+    # construction with inline updates
+    d2 = dict()
+    d2["t"] = tracked # $ tracked
+    d2["o"] = other
+
+    d2["t"] # $ tracked
+    d2["o"]
+
+    # notice that time-travel is also possible (just as with attributes)
+    d3 = dict()
+    d3["t"] # $ SPURIOUS: tracked
+    d3["t"] = tracked # $ tracked
+    d3["t"] # $ tracked
+
+
+def test_list(index_arg):
+    l = [tracked, other] # $tracked
+
+    l[0] # $ MISSING: tracked
+    l[1]
+
+    # non-precise access is not supported right now (and it's not 100% clear if we want
+    # to support it, or if it will lead to bad results)
+    l[index_arg]
+
+    for x in l:
+        print(x)
+
+    for i in range(len(l)):
+        print(l[i])

python/ql/test/experimental/dataflow/typetracking/tracked.ql

@@ -30,6 +30,14 @@ module TrackedTest implements TestSig {
       not e instanceof DataFlow::ScopeEntryDefinitionNode and
       // ...same for `SynthCaptureNode`s
       not e instanceof DP::SynthCaptureNode and
+      // after starting to track all kinds of content, we generally just want to show
+      // annotations after reading the tracked data out again. (we keep the old
+      // attribute logic to not rewrite all our tests)
+      (
+        t.getContent().isNone()
+        or
+        t.getContent().asSome() instanceof DataFlow::AttributeContent
+      ) and
       tag = "tracked" and
       location = e.getLocation() and
       value = t.getAttr() and

python/ql/test/experimental/library-tests/CallGraph/InlineCallGraphTest.expected

@@ -16,7 +16,6 @@ pointsTo_found_typeTracker_notFound
 | code/func_defined_outside_class.py:42:1:42:7 | ControlFlowNode for Attribute() | B._gen.func |
 | code/func_defined_outside_class.py:43:1:43:7 | ControlFlowNode for Attribute() | B._gen.func |
 | code/funky_regression.py:15:9:15:17 | ControlFlowNode for Attribute() | Wat.f2 |
-| code/tuple_function_return.py:15:1:15:4 | ControlFlowNode for f2() | func |
 | code/type_tracking_limitation.py:8:1:8:3 | ControlFlowNode for x() | my_func |
 typeTracker_found_pointsTo_notFound
 | code/callable_as_argument.py:29:5:29:12 | ControlFlowNode for Attribute() | test_class.InsideTestFunc.sm |
@@ -38,6 +37,10 @@ typeTracker_found_pointsTo_notFound
 | code/class_super.py:101:1:101:7 | ControlFlowNode for Attribute() | Z.foo |
 | code/class_super.py:108:1:108:8 | ControlFlowNode for Attribute() | Z.foo |
 | code/def_in_function.py:22:5:22:11 | ControlFlowNode for Attribute() | test.A.foo |
+| code/func_ref_in_content.py:32:1:32:4 | ControlFlowNode for f4() | func |
+| code/func_ref_in_content.py:46:1:46:4 | ControlFlowNode for f5() | func |
+| code/func_ref_in_content.py:48:1:48:15 | ControlFlowNode for Subscript() | func2 |
+| code/func_ref_in_content.py:50:1:50:19 | ControlFlowNode for Subscript() | func2 |
 | code/isinstance.py:9:13:9:22 | ControlFlowNode for Attribute() | A.foo |
 | code/isinstance.py:9:13:9:22 | ControlFlowNode for Attribute() | ASub.foo |
 | code/isinstance.py:14:13:14:22 | ControlFlowNode for Attribute() | A.foo |

python/ql/test/experimental/library-tests/CallGraph/code/func_ref_in_content.py

@@ -0,0 +1,74 @@
+def func():
+    print("func()")
+
+def func2():
+    print("func2()")
+
+def return_func():
+    return func
+
+f1 = return_func() # $ pt,tt=return_func
+f1() # $ pt,tt=func
+
+
+def return_func_in_tuple():
+    return (func, 42)
+
+tup = return_func_in_tuple() # $ pt,tt=return_func_in_tuple
+
+f2, _ = tup
+f2() # $ pt,tt=func
+
+f3 = tup[0]
+f3() # $ tt,pt=func
+
+
+def return_func_in_dict():
+    return {'func': func, 'val': 42}
+
+dct = return_func_in_dict() # $ pt,tt=return_func_in_dict
+
+f4 = dct['func']
+f4() # $ tt=func
+
+
+def return_func_in_dict_update():
+    d = {}
+    d["func"] = func
+    d["func2"] = func2
+    d["contested"] = func
+    d["contested"] = func2
+    return d
+
+dct2 = return_func_in_dict_update() # $ pt,tt=return_func_in_dict_update
+
+f5 = dct2['func']
+f5() # $ tt=func
+
+dct2['func2']() # $ tt=func2
+
+dct2['contested']() # $ tt=func2 SPURIOUS: tt=func
+
+
+## non-precise access is not supported right now
+for k in dct2:
+    dct2[k]() # $ MISSING: tt=func tt=func2
+
+for v in dct2.values():
+    v() # $ MISSING: tt=func tt=func2
+
+for k, v in dct2.items():
+    v() # $ MISSING: tt=func tt=func2
+
+
+def return_func_in_list():
+    return [func, 42]
+
+lst = return_func_in_list() # $ pt,tt=return_func_in_list
+
+f6 = lst[0]
+f6() # $ MISSING: pt,tt=func
+
+if eval("False"): # don't run this, but fool analysis to still consider it (doesn't wok if you just to `if False:`)
+    f7 = lst[1]
+    f7()

python/ql/test/experimental/library-tests/CallGraph/code/tuple_function_return.py

@@ -1,15 +0,0 @@
-def func():
-    print("func()")
-
-def return_func():
-    return func
-
-def return_func_in_tuple():
-    return (func, 42)
-
-f1 = return_func() # $ pt,tt=return_func
-f1() # $ pt,tt=func
-
-
-f2, _ = return_func_in_tuple() # $ pt,tt=return_func_in_tuple
-f2() # $ pt=func MISSING: tt

python/ql/test/library-tests/essa/ssa-compute/CONSISTENCY/TypeTrackingConsistency.expected

@@ -1,6 +1,6 @@
 unreachableNode
-| test2.py:16:17:16:17 | ControlFlowNode for y | Unreachable node in step of kind load bar. |
+| test2.py:16:17:16:17 | ControlFlowNode for y | Unreachable node in step of kind load Attribute bar. |
-| test2.py:25:23:25:23 | ControlFlowNode for x | Unreachable node in step of kind load attribute. |
+| test2.py:25:23:25:23 | ControlFlowNode for x | Unreachable node in step of kind load Attribute attribute. |
 | test2.py:25:23:25:23 | ControlFlowNode for x | Unreachable node in step of kind simpleLocalSmallStep. |
-| test2.py:26:17:26:17 | ControlFlowNode for y | Unreachable node in step of kind load bar. |
+| test2.py:26:17:26:17 | ControlFlowNode for y | Unreachable node in step of kind load Attribute bar. |
 | test2.py:27:23:27:23 | ControlFlowNode for x | Unreachable node in step of kind simpleLocalSmallStep. |