Merge branch 'master' of github.com:github/codeql into SharedDataflow_SequenceFlow

2025-12-24 04:36:35 +01:00 · 2020-08-03 07:05:34 +02:00
parent b21da86ac1 595ab442e6
commit 6debc48e79
50 changed files with 1271 additions and 165 deletions
--- a/python/ql/examples/qlpack.yml
+++ b/python/ql/examples/qlpack.yml
@@ -0,0 +1,3 @@
+name: codeql-python-examples
+version: 0.0.0
+libraryPathDependencies: codeql-python
--- a/python/ql/src/codeql-suites/python-lgtm-full.qls
+++ b/python/ql/src/codeql-suites/python-lgtm-full.qls
@@ -7,3 +7,11 @@
    tags contain:
      - ide-contextual-queries/local-definitions
      - ide-contextual-queries/local-references
+- query: Lexical/FCommentedOutCode.ql
+- query: Metrics/Dependencies/ExternalDependencies.ql
+- query: Metrics/Dependencies/ExternalDependenciesSourceLinks.ql
+- query: Metrics/FLinesOfCode.ql
+- query: Metrics/FLinesOfComments.ql
+- query: Metrics/FLinesOfDuplicatedCode.ql
+- query: Metrics/FLinesOfSimilarCode.ql
+- query: Metrics/FNumberOfTests.ql
--- a/python/ql/test/experimental/dataflow/coverage/dataflow.expected
+++ b/python/ql/test/experimental/dataflow/coverage/dataflow.expected
@@ -1,53 +1,39 @@
 edges
-| test.py:17:5:17:5 | SSA variable x [Content] | test.py:18:9:18:9 | ControlFlowNode for x [Content] |
-| test.py:17:13:17:18 | ControlFlowNode for SOURCE | test.py:17:5:17:5 | SSA variable x [Content] |
-| test.py:18:9:18:9 | ControlFlowNode for x [Content] | test.py:18:9:18:12 | ControlFlowNode for Subscript |
-| test.py:18:9:18:12 | ControlFlowNode for Subscript | test.py:19:10:19:10 | ControlFlowNode for y |
-| test.py:23:9:23:14 | ControlFlowNode for SOURCE | test.py:24:10:24:10 | ControlFlowNode for x |
-| test.py:28:9:28:16 | ControlFlowNode for Str | test.py:29:10:29:10 | ControlFlowNode for x |
-| test.py:32:9:32:17 | ControlFlowNode for Str | test.py:33:10:33:10 | ControlFlowNode for x |
-| test.py:36:9:36:10 | ControlFlowNode for IntegerLiteral | test.py:37:10:37:10 | ControlFlowNode for x |
-| test.py:40:9:40:12 | ControlFlowNode for FloatLiteral | test.py:41:10:41:10 | ControlFlowNode for x |
-| test.py:49:10:49:15 | ControlFlowNode for SOURCE | test.py:50:10:50:10 | ControlFlowNode for x |
-| test.py:54:5:54:5 | SSA variable x [Content] | test.py:55:10:55:10 | ControlFlowNode for x [Content] |
-| test.py:54:10:54:15 | ControlFlowNode for SOURCE | test.py:54:5:54:5 | SSA variable x [Content] |
-| test.py:55:10:55:10 | ControlFlowNode for x [Content] | test.py:55:10:55:13 | ControlFlowNode for Subscript |
-| test.py:226:15:226:20 | ControlFlowNode for SOURCE | test.py:226:10:226:21 | ControlFlowNode for f() |
-| test.py:282:12:282:17 | ControlFlowNode for SOURCE | test.py:282:10:282:18 | ControlFlowNode for f() |
+| test.py:35:9:35:14 | ControlFlowNode for SOURCE | test.py:36:10:36:10 | ControlFlowNode for x |
+| test.py:40:9:40:16 | ControlFlowNode for Str | test.py:41:10:41:10 | ControlFlowNode for x |
+| test.py:44:9:44:17 | ControlFlowNode for Str | test.py:45:10:45:10 | ControlFlowNode for x |
+| test.py:48:9:48:10 | ControlFlowNode for IntegerLiteral | test.py:49:10:49:10 | ControlFlowNode for x |
+| test.py:52:9:52:12 | ControlFlowNode for FloatLiteral | test.py:53:10:53:10 | ControlFlowNode for x |
+| test.py:61:10:61:15 | ControlFlowNode for SOURCE | test.py:62:10:62:10 | ControlFlowNode for x |
+| test.py:238:28:238:33 | ControlFlowNode for SOURCE | test.py:238:10:238:34 | ControlFlowNode for second() |
+| test.py:297:12:297:17 | ControlFlowNode for SOURCE | test.py:297:10:297:18 | ControlFlowNode for f() |
+| test.py:301:28:301:33 | ControlFlowNode for SOURCE | test.py:301:10:301:34 | ControlFlowNode for second() |
 nodes
-| test.py:17:5:17:5 | SSA variable x [Content] | semmle.label | SSA variable x [Content] |
-| test.py:17:13:17:18 | ControlFlowNode for SOURCE | semmle.label | ControlFlowNode for SOURCE |
-| test.py:18:9:18:9 | ControlFlowNode for x [Content] | semmle.label | ControlFlowNode for x [Content] |
-| test.py:18:9:18:12 | ControlFlowNode for Subscript | semmle.label | ControlFlowNode for Subscript |
-| test.py:19:10:19:10 | ControlFlowNode for y | semmle.label | ControlFlowNode for y |
-| test.py:23:9:23:14 | ControlFlowNode for SOURCE | semmle.label | ControlFlowNode for SOURCE |
-| test.py:24:10:24:10 | ControlFlowNode for x | semmle.label | ControlFlowNode for x |
-| test.py:28:9:28:16 | ControlFlowNode for Str | semmle.label | ControlFlowNode for Str |
-| test.py:29:10:29:10 | ControlFlowNode for x | semmle.label | ControlFlowNode for x |
-| test.py:32:9:32:17 | ControlFlowNode for Str | semmle.label | ControlFlowNode for Str |
-| test.py:33:10:33:10 | ControlFlowNode for x | semmle.label | ControlFlowNode for x |
-| test.py:36:9:36:10 | ControlFlowNode for IntegerLiteral | semmle.label | ControlFlowNode for IntegerLiteral |
-| test.py:37:10:37:10 | ControlFlowNode for x | semmle.label | ControlFlowNode for x |
-| test.py:40:9:40:12 | ControlFlowNode for FloatLiteral | semmle.label | ControlFlowNode for FloatLiteral |
+| test.py:35:9:35:14 | ControlFlowNode for SOURCE | semmle.label | ControlFlowNode for SOURCE |
+| test.py:36:10:36:10 | ControlFlowNode for x | semmle.label | ControlFlowNode for x |
+| test.py:40:9:40:16 | ControlFlowNode for Str | semmle.label | ControlFlowNode for Str |
 | test.py:41:10:41:10 | ControlFlowNode for x | semmle.label | ControlFlowNode for x |
-| test.py:49:10:49:15 | ControlFlowNode for SOURCE | semmle.label | ControlFlowNode for SOURCE |
-| test.py:50:10:50:10 | ControlFlowNode for x | semmle.label | ControlFlowNode for x |
-| test.py:54:5:54:5 | SSA variable x [Content] | semmle.label | SSA variable x [Content] |
-| test.py:54:10:54:15 | ControlFlowNode for SOURCE | semmle.label | ControlFlowNode for SOURCE |
-| test.py:55:10:55:10 | ControlFlowNode for x [Content] | semmle.label | ControlFlowNode for x [Content] |
-| test.py:55:10:55:13 | ControlFlowNode for Subscript | semmle.label | ControlFlowNode for Subscript |
-| test.py:226:10:226:21 | ControlFlowNode for f() | semmle.label | ControlFlowNode for f() |
-| test.py:226:15:226:20 | ControlFlowNode for SOURCE | semmle.label | ControlFlowNode for SOURCE |
-| test.py:282:10:282:18 | ControlFlowNode for f() | semmle.label | ControlFlowNode for f() |
-| test.py:282:12:282:17 | ControlFlowNode for SOURCE | semmle.label | ControlFlowNode for SOURCE |
+| test.py:44:9:44:17 | ControlFlowNode for Str | semmle.label | ControlFlowNode for Str |
+| test.py:45:10:45:10 | ControlFlowNode for x | semmle.label | ControlFlowNode for x |
+| test.py:48:9:48:10 | ControlFlowNode for IntegerLiteral | semmle.label | ControlFlowNode for IntegerLiteral |
+| test.py:49:10:49:10 | ControlFlowNode for x | semmle.label | ControlFlowNode for x |
+| test.py:52:9:52:12 | ControlFlowNode for FloatLiteral | semmle.label | ControlFlowNode for FloatLiteral |
+| test.py:53:10:53:10 | ControlFlowNode for x | semmle.label | ControlFlowNode for x |
+| test.py:61:10:61:15 | ControlFlowNode for SOURCE | semmle.label | ControlFlowNode for SOURCE |
+| test.py:62:10:62:10 | ControlFlowNode for x | semmle.label | ControlFlowNode for x |
+| test.py:238:10:238:34 | ControlFlowNode for second() | semmle.label | ControlFlowNode for second() |
+| test.py:238:28:238:33 | ControlFlowNode for SOURCE | semmle.label | ControlFlowNode for SOURCE |
+| test.py:297:10:297:18 | ControlFlowNode for f() | semmle.label | ControlFlowNode for f() |
+| test.py:297:12:297:17 | ControlFlowNode for SOURCE | semmle.label | ControlFlowNode for SOURCE |
+| test.py:301:10:301:34 | ControlFlowNode for second() | semmle.label | ControlFlowNode for second() |
+| test.py:301:28:301:33 | ControlFlowNode for SOURCE | semmle.label | ControlFlowNode for SOURCE |
 #select
-| test.py:19:10:19:10 | ControlFlowNode for y | test.py:17:13:17:18 | ControlFlowNode for SOURCE | test.py:19:10:19:10 | ControlFlowNode for y | <message> |
-| test.py:24:10:24:10 | ControlFlowNode for x | test.py:23:9:23:14 | ControlFlowNode for SOURCE | test.py:24:10:24:10 | ControlFlowNode for x | <message> |
-| test.py:29:10:29:10 | ControlFlowNode for x | test.py:28:9:28:16 | ControlFlowNode for Str | test.py:29:10:29:10 | ControlFlowNode for x | <message> |
-| test.py:33:10:33:10 | ControlFlowNode for x | test.py:32:9:32:17 | ControlFlowNode for Str | test.py:33:10:33:10 | ControlFlowNode for x | <message> |
-| test.py:37:10:37:10 | ControlFlowNode for x | test.py:36:9:36:10 | ControlFlowNode for IntegerLiteral | test.py:37:10:37:10 | ControlFlowNode for x | <message> |
-| test.py:41:10:41:10 | ControlFlowNode for x | test.py:40:9:40:12 | ControlFlowNode for FloatLiteral | test.py:41:10:41:10 | ControlFlowNode for x | <message> |
-| test.py:50:10:50:10 | ControlFlowNode for x | test.py:49:10:49:15 | ControlFlowNode for SOURCE | test.py:50:10:50:10 | ControlFlowNode for x | <message> |
-| test.py:55:10:55:13 | ControlFlowNode for Subscript | test.py:54:10:54:15 | ControlFlowNode for SOURCE | test.py:55:10:55:13 | ControlFlowNode for Subscript | <message> |
-| test.py:226:10:226:21 | ControlFlowNode for f() | test.py:226:15:226:20 | ControlFlowNode for SOURCE | test.py:226:10:226:21 | ControlFlowNode for f() | <message> |
-| test.py:282:10:282:18 | ControlFlowNode for f() | test.py:282:12:282:17 | ControlFlowNode for SOURCE | test.py:282:10:282:18 | ControlFlowNode for f() | <message> |
+| test.py:36:10:36:10 | ControlFlowNode for x | test.py:35:9:35:14 | ControlFlowNode for SOURCE | test.py:36:10:36:10 | ControlFlowNode for x | <message> |
+| test.py:41:10:41:10 | ControlFlowNode for x | test.py:40:9:40:16 | ControlFlowNode for Str | test.py:41:10:41:10 | ControlFlowNode for x | <message> |
+| test.py:45:10:45:10 | ControlFlowNode for x | test.py:44:9:44:17 | ControlFlowNode for Str | test.py:45:10:45:10 | ControlFlowNode for x | <message> |
+| test.py:49:10:49:10 | ControlFlowNode for x | test.py:48:9:48:10 | ControlFlowNode for IntegerLiteral | test.py:49:10:49:10 | ControlFlowNode for x | <message> |
+| test.py:53:10:53:10 | ControlFlowNode for x | test.py:52:9:52:12 | ControlFlowNode for FloatLiteral | test.py:53:10:53:10 | ControlFlowNode for x | <message> |
+| test.py:62:10:62:10 | ControlFlowNode for x | test.py:61:10:61:15 | ControlFlowNode for SOURCE | test.py:62:10:62:10 | ControlFlowNode for x | <message> |
+| test.py:238:10:238:34 | ControlFlowNode for second() | test.py:238:28:238:33 | ControlFlowNode for SOURCE | test.py:238:10:238:34 | ControlFlowNode for second() | <message> |
+| test.py:297:10:297:18 | ControlFlowNode for f() | test.py:297:12:297:17 | ControlFlowNode for SOURCE | test.py:297:10:297:18 | ControlFlowNode for f() | <message> |
+| test.py:301:10:301:34 | ControlFlowNode for second() | test.py:301:28:301:33 | ControlFlowNode for SOURCE | test.py:301:10:301:34 | ControlFlowNode for second() | <message> |
--- a/python/ql/test/experimental/dataflow/coverage/localFlow.expected
+++ b/python/ql/test/experimental/dataflow/coverage/localFlow.expected
@@ -1,7 +1,7 @@
-| test.py:17:5:17:5 | SSA variable x | test.py:16:1:16:33 | Exit node for Function test_tuple_with_local_flow |
-| test.py:17:5:17:5 | SSA variable x | test.py:18:9:18:9 | ControlFlowNode for x |
-| test.py:17:10:17:18 | ControlFlowNode for Tuple | test.py:17:5:17:5 | SSA variable x |
-| test.py:18:5:18:5 | SSA variable y | test.py:19:5:19:11 | SSA variable y |
-| test.py:18:5:18:5 | SSA variable y | test.py:19:10:19:10 | ControlFlowNode for y |
-| test.py:18:9:18:12 | ControlFlowNode for Subscript | test.py:18:5:18:5 | SSA variable y |
-| test.py:19:5:19:11 | SSA variable y | test.py:16:1:16:33 | Exit node for Function test_tuple_with_local_flow |
+| test.py:24:5:24:5 | SSA variable x | test.py:23:1:23:33 | Exit node for Function test_tuple_with_local_flow |
+| test.py:24:5:24:5 | SSA variable x | test.py:25:9:25:9 | ControlFlowNode for x |
+| test.py:24:10:24:26 | ControlFlowNode for Tuple | test.py:24:5:24:5 | SSA variable x |
+| test.py:25:5:25:5 | SSA variable y | test.py:26:5:26:11 | SSA variable y |
+| test.py:25:5:25:5 | SSA variable y | test.py:26:10:26:10 | ControlFlowNode for y |
+| test.py:25:9:25:12 | ControlFlowNode for Subscript | test.py:25:5:25:5 | SSA variable y |
+| test.py:26:5:26:11 | SSA variable y | test.py:23:1:23:33 | Exit node for Function test_tuple_with_local_flow |
--- a/python/ql/test/experimental/dataflow/coverage/test.py
+++ b/python/ql/test/experimental/dataflow/coverage/test.py
@@ -6,18 +6,30 @@
 #
 # Functions whose name ends with "_with_local_flow" will also be tested for local flow.
 #
-# All functions starting with "test_" should run and print a source (sources are defined in testConfig.qll).
+# All functions starting with "test_" should run and either
+#  - print a source (sources are defined in testConfig.qll).
+#  - print "Unexpected flow: " and a non-source
+# (The idea is to later write a script to autimatically confirm this.)

 # These are defined so that we can evaluate the test code.
+NONSOURCE = "not a source"
 SOURCE = "source"
 def SINK(x):
    print(x)

+def SINK_F(x):
+    print("Unexpected flow: ", x)
+
 def test_tuple_with_local_flow():
-    x = (3, SOURCE)
+    x = (NONSOURCE, SOURCE)
    y = x[1]
    SINK(y) # Flow missing

+def test_tuple_negative():
+    x = (NONSOURCE, SOURCE)
+    y = x[0]
+    SINK_F(y)
+
 # 6.2.1. Identifiers (Names)
 def test_names():
    x = SOURCE
@@ -54,8 +66,12 @@ def test_list_display():
    x = [SOURCE]
    SINK(x[0]) # Flow missing

+def test_list_display_negative():
+    x = [SOURCE]
+    SINK_F(x)
+
 def test_list_comprehension():
-    x = [SOURCE for y in [3]]
+    x = [SOURCE for y in [NONSOURCE]]
    SINK(x[0]) # Flow missing

 def test_nested_list_display():
@@ -68,7 +84,7 @@ def test_set_display():
    SINK(x.pop()) # Flow missing

 def test_set_comprehension():
-    x = {SOURCE for y in [3]}
+    x = {SOURCE for y in [NONSOURCE]}
    SINK(x.pop()) # Flow missing

 def test_nested_set_display():
@@ -90,7 +106,7 @@ def test_nested_dict_display():

 # 6.2.8. Generator expressions
 def test_generator():
-    x = (SOURCE for y in [3])
+    x = (SOURCE for y in [NONSOURCE])
    SINK([*x][0]) # Flow missing

 # 6.2.9. Yield expressions
@@ -123,13 +139,13 @@ def gen2(x):
    yield m

 def test_send():
-    g = gen2(3)
+    g = gen2(NONSOURCE)
    n = next(g)
    SINK(g.send(SOURCE)) # Flow missing

 def gen_ex(x):
    try:
-        yield 3
+        yield NONSOURCE
    except:
        yield x # `x` has to flow to call to `throw`

@@ -163,7 +179,7 @@ async def agen2(x):
    yield m

 async def atest_asend():
-    g = agen2(3)
+    g = agen2(NONSOURCE)
    n = await g.__anext__()
    SINK(await g.asend(SOURCE)) # Flow missing

@@ -172,7 +188,7 @@ def test_asend():

 async def agen_ex(x):
    try:
-        yield 3
+        yield NONSOURCE
    except:
        yield x # `x` has to flow to call to `athrow`

@@ -194,10 +210,6 @@ def test_attribute_reference():
 # overriding __getattr__ should be tested by the class coverage tests

 # 6.3.2. Subscriptions
-# This does not constitute dataflow (but could be taint flow)
-def example_subscription_string():
-    SINK("source"[0]) # Flow not expected
-
 def test_subscription_tuple():
    SINK((SOURCE,)[0]) # Flow missing

@@ -219,32 +231,35 @@ def test_slicing():
 # The grammar seems to allow `l[0:1:1, 0:1]`, but the interpreter does not like it

 # 6.3.4. Calls
-def f(a, b):
+def second(a, b):
    return b

 def test_call_positional():
-    SINK(f(3, SOURCE))
+    SINK(second(NONSOURCE, SOURCE))
+
+def test_call_positional_negative():
+    SINK_F(second(SOURCE, NONSOURCE))

 def test_call_keyword():
-    SINK(f(3, b=SOURCE)) # Flow missing
+    SINK(second(NONSOURCE, b=SOURCE)) # Flow missing

 def test_call_unpack_iterable():
-    SINK(f(3, *[SOURCE])) # Flow missing
+    SINK(second(NONSOURCE, *[SOURCE])) # Flow missing

 def test_call_unpack_mapping():
-    SINK(f(3, **{"b": SOURCE})) # Flow missing
+    SINK(second(NONSOURCE, **{"b": SOURCE})) # Flow missing

 def f_extra_pos(a, *b):
    return b[0]

 def test_call_extra_pos():
-    SINK(f_extra_pos(3, SOURCE)) # Flow missing
+    SINK(f_extra_pos(NONSOURCE, SOURCE)) # Flow missing

 def f_extra_keyword(a, **b):
    return b["b"]

 def test_call_extra_keyword():
-    SINK(f_extra_keyword(3, b=SOURCE)) # Flow missing
+    SINK(f_extra_keyword(NONSOURCE, b=SOURCE)) # Flow missing

 # return the name of the first extra keyword argument
 def f_extra_keyword_flow(**a):
@@ -256,27 +271,60 @@ def test_call_extra_keyword_flow():

 # 6.12. Assignment expressions
 def test_assignment_expression():
-    x = 3
+    x = NONSOURCE
    SINK(x := SOURCE) # Flow missing

 # 6.13. Conditional expressions
 def test_conditional_true():
-    SINK(SOURCE if True else 3) # Flow missing
+    SINK(SOURCE if True else NONSOURCE) # Flow missing

 def test_conditional_false():
-    SINK(3 if False else SOURCE) # Flow missing
+    SINK(NONSOURCE if False else SOURCE) # Flow missing

 # Condition is evaluated first, so x is SOURCE once chosen
 def test_conditional_evaluation_true():
-    x = 3
-    SINK(x if (SOURCE == (x := SOURCE)) else 3) # Flow missing
+    x = NONSOURCE
+    SINK(x if (SOURCE == (x := SOURCE)) else NONSOURCE) # Flow missing

 # Condition is evaluated first, so x is SOURCE once chosen
 def test_conditional_evaluation_false():
-    x = 3
-    SINK(3 if (3 == (x := SOURCE)) else x) # Flow missing
+    x = NONSOURCE
+    SINK(NONSOURCE if (NONSOURCE == (x := SOURCE)) else x) # Flow missing

 # 6.14. Lambdas
 def test_lambda():
    f = lambda x : x
    SINK(f(SOURCE))
+
+def test_lambda_positional():
+    second = lambda a, b : b
+    SINK(second(NONSOURCE, SOURCE))
+
+def test_lambda_positional_negative():
+    second = lambda a, b : b
+    SINK_F(second(SOURCE, NONSOURCE))
+
+def test_lambda_keyword():
+    second = lambda a, b : b
+    SINK(second(NONSOURCE, b=SOURCE)) # Flow missing
+
+def test_lambda_unpack_iterable():
+    second = lambda a, b : b
+    SINK(second(NONSOURCE, *[SOURCE])) # Flow missing
+
+def test_lambda_unpack_mapping():
+    second = lambda a, b : b
+    SINK(second(NONSOURCE, **{"b": SOURCE})) # Flow missing
+
+def test_lambda_extra_pos():
+    f_extra_pos = lambda a, *b : b[0]
+    SINK(f_extra_pos(NONSOURCE, SOURCE)) # Flow missing
+
+def test_lambda_extra_keyword():
+    f_extra_keyword = lambda a, **b : b["b"]
+    SINK(f_extra_keyword(NONSOURCE, b=SOURCE)) # Flow missing
+
+# call the function with our source as the name of the keyword arguemnt
+def test_lambda_extra_keyword_flow():
+    f_extra_keyword_flow = lambda **a : [*a][0] # return the name of the first extra keyword argument
+    SINK(f_extra_keyword_flow(**{SOURCE: None})) # Flow missing