Merge pull request #15711 from RasmusWL/tt-content

Python: Add type tracking for content

python/ql/test/experimental/dataflow/typetracking/content_test.py

@@ -0,0 +1,100 @@
+# test of other content types than attributes
+
+def test_tuple(index_arg):
+    tup = (tracked, other) # $tracked
+
+    tup[0] # $ tracked
+    tup[1]
+
+    a,b = tup # $tracked
+    a # $ tracked
+    b
+
+    # non-precise access is not supported right now (and it's not 100% clear if we want
+    # to support it, or if it will lead to bad results)
+    tup[index_arg]
+
+    for x in tup:
+        print(x)
+
+    for i in range(len(tup)):
+        print(tup[i])
+
+
+    # nested tuples
+    nested_tuples = ((tracked, other), (other, tracked)) # $tracked
+
+    nested_tuples[0][0] # $ MISSING: tracked
+    nested_tuples[0][1]
+    nested_tuples[1][0]
+    nested_tuples[1][1] # $ MISSING: tracked
+
+    (aa, ab), (ba, bb) = nested_tuples
+    aa # $ MISSING: tracked
+    ab
+    ba
+    bb # $ MISSING: tracked
+
+
+    # non-precise access is not supported right now (and it's not 100% clear if we want
+    # to support it, or if it will lead to bad results)
+    for (x, y) in nested_tuples:
+        x
+        y
+
+
+def test_dict(key_arg):
+    d1 = {"t": tracked, "o": other} # $tracked
+    d1["t"] # $ tracked
+    d1.get("t") # $ MISSING: tracked
+    d1.setdefault("t") # $ MISSING: tracked
+
+    d1["o"]
+    d1.get("o")
+    d1.setdefault("o")
+
+
+    # non-precise access is not supported right now (and it's not 100% clear if we want
+    # to support it, or if it will lead to bad results)
+    d1[key_arg]
+
+    for k in d1:
+        d1[k]
+
+    for v in d1.values():
+        v
+
+    for k, v in d1.items():
+        v
+
+
+    # construction with inline updates
+    d2 = dict()
+    d2["t"] = tracked # $ tracked
+    d2["o"] = other
+
+    d2["t"] # $ tracked
+    d2["o"]
+
+    # notice that time-travel is also possible (just as with attributes)
+    d3 = dict()
+    d3["t"] # $ SPURIOUS: tracked
+    d3["t"] = tracked # $ tracked
+    d3["t"] # $ tracked
+
+
+def test_list(index_arg):
+    l = [tracked, other] # $tracked
+
+    l[0] # $ MISSING: tracked
+    l[1]
+
+    # non-precise access is not supported right now (and it's not 100% clear if we want
+    # to support it, or if it will lead to bad results)
+    l[index_arg]
+
+    for x in l:
+        print(x)
+
+    for i in range(len(l)):
+        print(l[i])

python/ql/test/experimental/dataflow/typetracking/tracked.ql

@@ -30,6 +30,14 @@ module TrackedTest implements TestSig {
       not e instanceof DataFlow::ScopeEntryDefinitionNode and
       // ...same for `SynthCaptureNode`s
       not e instanceof DP::SynthCaptureNode and
+      // after starting to track all kinds of content, we generally just want to show
+      // annotations after reading the tracked data out again. (we keep the old
+      // attribute logic to not rewrite all our tests)
+      (
+        t.getContent().isNone()
+        or
+        t.getContent().asSome() instanceof DataFlow::AttributeContent
+      ) and
       tag = "tracked" and
       location = e.getLocation() and
       value = t.getAttr() and

python/ql/test/experimental/library-tests/CallGraph/InlineCallGraphTest.expected

@@ -16,7 +16,6 @@ pointsTo_found_typeTracker_notFound
 | code/func_defined_outside_class.py:42:1:42:7 | ControlFlowNode for Attribute() | B._gen.func |
 | code/func_defined_outside_class.py:43:1:43:7 | ControlFlowNode for Attribute() | B._gen.func |
 | code/funky_regression.py:15:9:15:17 | ControlFlowNode for Attribute() | Wat.f2 |
-| code/tuple_function_return.py:15:1:15:4 | ControlFlowNode for f2() | func |
 | code/type_tracking_limitation.py:8:1:8:3 | ControlFlowNode for x() | my_func |
 typeTracker_found_pointsTo_notFound
 | code/callable_as_argument.py:29:5:29:12 | ControlFlowNode for Attribute() | test_class.InsideTestFunc.sm |
@@ -38,6 +37,10 @@ typeTracker_found_pointsTo_notFound
 | code/class_super.py:101:1:101:7 | ControlFlowNode for Attribute() | Z.foo |
 | code/class_super.py:108:1:108:8 | ControlFlowNode for Attribute() | Z.foo |
 | code/def_in_function.py:22:5:22:11 | ControlFlowNode for Attribute() | test.A.foo |
+| code/func_ref_in_content.py:32:1:32:4 | ControlFlowNode for f4() | func |
+| code/func_ref_in_content.py:46:1:46:4 | ControlFlowNode for f5() | func |
+| code/func_ref_in_content.py:48:1:48:15 | ControlFlowNode for Subscript() | func2 |
+| code/func_ref_in_content.py:50:1:50:19 | ControlFlowNode for Subscript() | func2 |
 | code/isinstance.py:9:13:9:22 | ControlFlowNode for Attribute() | A.foo |
 | code/isinstance.py:9:13:9:22 | ControlFlowNode for Attribute() | ASub.foo |
 | code/isinstance.py:14:13:14:22 | ControlFlowNode for Attribute() | A.foo |

python/ql/test/experimental/library-tests/CallGraph/code/func_ref_in_content.py

@@ -0,0 +1,74 @@
+def func():
+    print("func()")
+
+def func2():
+    print("func2()")
+
+def return_func():
+    return func
+
+f1 = return_func() # $ pt,tt=return_func
+f1() # $ pt,tt=func
+
+
+def return_func_in_tuple():
+    return (func, 42)
+
+tup = return_func_in_tuple() # $ pt,tt=return_func_in_tuple
+
+f2, _ = tup
+f2() # $ pt,tt=func
+
+f3 = tup[0]
+f3() # $ tt,pt=func
+
+
+def return_func_in_dict():
+    return {'func': func, 'val': 42}
+
+dct = return_func_in_dict() # $ pt,tt=return_func_in_dict
+
+f4 = dct['func']
+f4() # $ tt=func
+
+
+def return_func_in_dict_update():
+    d = {}
+    d["func"] = func
+    d["func2"] = func2
+    d["contested"] = func
+    d["contested"] = func2
+    return d
+
+dct2 = return_func_in_dict_update() # $ pt,tt=return_func_in_dict_update
+
+f5 = dct2['func']
+f5() # $ tt=func
+
+dct2['func2']() # $ tt=func2
+
+dct2['contested']() # $ tt=func2 SPURIOUS: tt=func
+
+
+## non-precise access is not supported right now
+for k in dct2:
+    dct2[k]() # $ MISSING: tt=func tt=func2
+
+for v in dct2.values():
+    v() # $ MISSING: tt=func tt=func2
+
+for k, v in dct2.items():
+    v() # $ MISSING: tt=func tt=func2
+
+
+def return_func_in_list():
+    return [func, 42]
+
+lst = return_func_in_list() # $ pt,tt=return_func_in_list
+
+f6 = lst[0]
+f6() # $ MISSING: pt,tt=func
+
+if eval("False"): # don't run this, but fool analysis to still consider it (doesn't wok if you just to `if False:`)
+    f7 = lst[1]
+    f7()

python/ql/test/experimental/library-tests/CallGraph/code/tuple_function_return.py

@@ -1,15 +0,0 @@
-def func():
-    print("func()")
-
-def return_func():
-    return func
-
-def return_func_in_tuple():
-    return (func, 42)
-
-f1 = return_func() # $ pt,tt=return_func
-f1() # $ pt,tt=func
-
-
-f2, _ = return_func_in_tuple() # $ pt,tt=return_func_in_tuple
-f2() # $ pt=func MISSING: tt