resolve merge conflict

python/ql/test/2/query-tests/Expressions/UseofApply.expected

@@ -1,2 +1,2 @@
 | UseofApply.py:19:3:19:17 | ControlFlowNode for apply() | Call to the obsolete builtin function 'apply'. |
-| expressions_test.py:3:5:3:21 | ControlFlowNode for apply() | Call to the obsolete builtin function 'apply'. |
+| expressions_test.py:2:5:2:21 | ControlFlowNode for apply() | Call to the obsolete builtin function 'apply'. |

python/ql/test/2/query-tests/Expressions/expressions_test.py

@@ -1,7 +1,18 @@
-
 def use_of_apply(func, args):
     apply(func, args)
 
-def use_of_input():
-    return input()
 
+def use_of_input():
+    return input() # NOT OK
+
+
+def not_use_of_input():
+    input = raw_input
+    return input() # OK
+
+
+if __name__ == "__main__":
+    # if you enter 4+4 each time, you'll see that results are: 8, '4+4', 8
+    print("result:", use_of_input())
+    print("result:", not_use_of_input())
+    print("result:", use_of_input())

python/ql/test/experimental/dataflow/ApiGraphs/test.py

@@ -122,14 +122,18 @@ def redefine_print():
     print = my_print
     print("these words")
 
-def local_redefine_range():
-    range = 5
-    return range
+def local_redefine_chr():
+    chr = 5
+    return chr
 
-def global_redefine_range():
-    global range
-    range = 6
-    return range #$ SPURIOUS: use=moduleImport("builtins").getMember("range")
+def global_redefine_chr():
+    global chr
+    chr = 6
+    return chr
+
+def what_is_chr_now():
+    # If global_redefine_chr has been run, then the following is _not_ a reference to the built-in chr
+    return chr(123) #$ MISSING: use=moduleImport("builtins").getMember("chr").getReturn()
 
 def obscured_print():
     p = print #$ use=moduleImport("builtins").getMember("print")

python/ql/test/experimental/dataflow/TestUtil/PrintNode.qll

@@ -1,7 +1,7 @@
 import python
 import semmle.python.dataflow.new.DataFlow
 
-string prettyExp(Expr e) {
+string prettyExpr(Expr e) {
   not e instanceof Num and
   not e instanceof StrConst and
   not e instanceof Subscript and
@@ -15,17 +15,41 @@ string prettyExp(Expr e) {
     e.(StrConst).getPrefix() + e.(StrConst).getText() +
       e.(StrConst).getPrefix().regexpReplaceAll("[a-zA-Z]+", "")
   or
-  result = prettyExp(e.(Subscript).getObject()) + "[" + prettyExp(e.(Subscript).getIndex()) + "]"
+  result = prettyExpr(e.(Subscript).getObject()) + "[" + prettyExpr(e.(Subscript).getIndex()) + "]"
   or
   (
     if exists(e.(Call).getAnArg()) or exists(e.(Call).getANamedArg())
-    then result = prettyExp(e.(Call).getFunc()) + "(..)"
-    else result = prettyExp(e.(Call).getFunc()) + "()"
+    then result = prettyExpr(e.(Call).getFunc()) + "(..)"
+    else result = prettyExpr(e.(Call).getFunc()) + "()"
   )
   or
-  result = prettyExp(e.(Attribute).getObject()) + "." + e.(Attribute).getName()
+  result = prettyExpr(e.(Attribute).getObject()) + "." + e.(Attribute).getName()
 }
 
+/**
+ * Gets pretty-printed version of the DataFlow::Node `node`
+ */
+bindingset[node]
 string prettyNode(DataFlow::Node node) {
-  if exists(node.asExpr()) then result = prettyExp(node.asExpr()) else result = node.toString()
+  if exists(node.asExpr()) then result = prettyExpr(node.asExpr()) else result = node.toString()
+}
+
+/**
+ * Gets pretty-printed version of the DataFlow::Node `node`, that is suitable for use
+ * with `TestUtilities.InlineExpectationsTest` (that is, no spaces unless required).
+ */
+bindingset[node]
+string prettyNodeForInlineTest(DataFlow::Node node) {
+  exists(node.asExpr()) and
+  result = prettyExpr(node.asExpr())
+  or
+  exists(Expr e | e = node.(DataFlow::PostUpdateNode).getPreUpdateNode().asExpr() |
+    // since PostUpdateNode both has space in the `[post <thing>]` annotation, and does
+    // not pretty print the pre-update node, we do custom handling of this.
+    result = "[post]" + prettyExpr(e)
+  )
+  or
+  not exists(node.asExpr()) and
+  not exists(Expr e | e = node.(DataFlow::PostUpdateNode).getPreUpdateNode().asExpr()) and
+  result = node.toString()
 }

python/ql/test/experimental/dataflow/sensitive-data/TestSensitiveDataSources.ql

@@ -0,0 +1,20 @@
+import python
+import semmle.python.dataflow.new.DataFlow
+import TestUtilities.InlineExpectationsTest
+import semmle.python.dataflow.new.SensitiveDataSources
+
+class SensitiveDataSourcesTest extends InlineExpectationsTest {
+  SensitiveDataSourcesTest() { this = "SensitiveDataSourcesTest" }
+
+  override string getARelevantTag() { result = "SensitiveDataSource" }
+
+  override predicate hasActualResult(Location location, string element, string tag, string value) {
+    exists(location.getFile().getRelativePath()) and
+    exists(SensitiveDataSource source |
+      location = source.getLocation() and
+      element = source.toString() and
+      value = source.getClassification() and
+      tag = "SensitiveDataSource"
+    )
+  }
+}

python/ql/test/experimental/dataflow/sensitive-data/test.py

@@ -0,0 +1,33 @@
+
+from not_found import get_passwd, account_id
+
+def get_password():
+    pass
+
+def get_secret():
+    pass
+
+def fetch_certificate():
+    pass
+
+def encrypt_password(pwd):
+    pass
+
+get_password() # $ SensitiveDataSource=password
+get_passwd() # $ SensitiveDataSource=password
+get_secret() # $ SensitiveDataSource=secret
+fetch_certificate() # $ SensitiveDataSource=certificate
+account_id() # $ SensitiveDataSource=id
+safe_to_store = encrypt_password(pwd)
+
+# attributes
+foo = ObjectFromDatabase()
+foo.secret # $ SensitiveDataSource=secret
+foo.username # $ SensitiveDataSource=id
+
+# Special handling of lookups of sensitive properties
+request.args["password"], # $ MISSING: SensitiveDataSource=password
+request.args.get("password") # $ SensitiveDataSource=password
+
+# I don't think handling `getlist` is super important, just included it to show what we don't handle
+request.args.getlist("password")[0] # $ MISSING: SensitiveDataSource=password

python/ql/test/experimental/dataflow/tainttracking/TestTaintLib.qll

@@ -46,6 +46,6 @@ query predicate test_taint(string arg_location, string test_res, string scope_na
     arg_location = arg.getLocation().toString() and
     test_res = test_res and
     scope_name = call.getScope().getName() and
-    repr = prettyExp(arg)
+    repr = prettyExpr(arg)
   )
 }

python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep/test_json.py

@@ -1,64 +0,0 @@
-# Add taintlib to PATH so it can be imported during runtime without any hassle
-import sys; import os; sys.path.append(os.path.dirname(os.path.dirname((__file__))))
-from taintlib import *
-
-# This has no runtime impact, but allows autocomplete to work
-from typing import TYPE_CHECKING
-if TYPE_CHECKING:
-    from ..taintlib import *
-
-
-# Actual tests
-
-from io import StringIO
-
-# Workaround for Python3 not having unicode
-import sys
-if sys.version_info[0] == 3:
-    unicode = str
-
-def test():
-    print("\n# test")
-    ts = TAINTED_STRING
-    import json
-
-    ensure_tainted(
-        json.dumps(ts), # $ tainted
-        json.loads(json.dumps(ts)), # $ tainted
-    )
-
-    # For Python2, need to convert to unicode for StringIO to work
-    tainted_filelike = StringIO(unicode(json.dumps(ts)))
-
-    ensure_tainted(
-        tainted_filelike, # $ MISSING: tainted
-        json.load(tainted_filelike), # $ MISSING: tainted
-    )
-
-def non_syntacical():
-    print("\n# non_syntacical")
-    ts = TAINTED_STRING
-
-    # a less syntactical approach
-    from json import load, loads, dumps
-
-    dumps_alias = dumps
-
-    ensure_tainted(
-        dumps(ts), # $ tainted
-        dumps_alias(ts), # $ tainted
-        loads(dumps(ts)), # $ tainted
-    )
-
-    # For Python2, need to convert to unicode for StringIO to work
-    tainted_filelike = StringIO(unicode(dumps(ts)))
-
-    ensure_tainted(
-        tainted_filelike, # $ MISSING: tainted
-        load(tainted_filelike), # $ MISSING: tainted
-    )
-
-# Make tests runable
-
-test()
-non_syntacical()

python/ql/test/experimental/dataflow/typetracking/attribute_tests.py

@@ -30,8 +30,28 @@ def test_incompatible_types():
     x.field = str("Hello") # $str=field str SPURIOUS: int=field int
     expects_string(x) # $ str=field SPURIOUS: int=field
 
+# set in different function
+def set_foo(some_class_instance): # $ tracked=foo
+    some_class_instance.foo = tracked # $ tracked=foo tracked
 
+def test_set_x():
+    x = SomeClass() # $ MISSING: tracked=foo
+    set_foo(x) # $ MISSING: tracked=foo
+    print(x.foo) # $ MISSING: tracked=foo tracked
+
+# return from a different function
+def create_with_foo():
+    x = SomeClass() # $ tracked=foo
+    x.foo = tracked # $ tracked=foo tracked
+    return x # $ tracked=foo
+
+def test_create_with_foo():
+    x = create_with_foo() # $ tracked=foo
+    print(x.foo) # $ tracked=foo tracked
+
+# ------------------------------------------------------------------------------
 # Attributes assigned statically to a class
+# ------------------------------------------------------------------------------
 
 class MyClass: # $tracked=field
     field = tracked # $tracked
@@ -40,7 +60,9 @@ lookup = MyClass.field # $tracked tracked=field
 instance = MyClass() # $tracked=field
 lookup2 = instance.field # MISSING: tracked
 
-## Dynamic attribute access
+# ------------------------------------------------------------------------------
+# Dynamic attribute access
+# ------------------------------------------------------------------------------
 
 # Via `getattr`/`setattr`
 
@@ -99,3 +121,41 @@ def dunder_dict_indirect_read():
     do_stuff(y) # $ MISSING: tracked
 
 
+# ------------------------------------------------------------------------------
+# Tracking of attribute on class instance
+# ------------------------------------------------------------------------------
+
+# attribute set in method
+# inspired by https://github.com/github/codeql/pull/6023
+
+class MyClass2(object):
+    def __init__(self): # $ tracked=foo
+        self.foo = tracked # $ tracked=foo tracked
+
+    def print_foo(self): # $ MISSING: tracked=foo
+        print(self.foo) # $ MISSING: tracked=foo tracked
+
+    def possibly_uncalled_method(self): # $ MISSING: tracked=foo
+        print(self.foo) # $ MISSING: tracked=foo tracked
+
+instance = MyClass2()
+print(instance.foo) # $ MISSING: tracked=foo tracked
+instance.print_foo() # $ MISSING: tracked=foo
+
+
+# attribute set from outside of class
+
+class MyClass3(object):
+    def print_self(self): # $ tracked=foo
+        print(self) # $ tracked=foo
+
+    def print_foo(self): # $ tracked=foo
+        print(self.foo) # $ tracked=foo tracked
+
+    def possibly_uncalled_method(self): # $ MISSING: tracked=foo
+        print(self.foo) # $ MISSING: tracked=foo tracked
+
+instance = MyClass3() # $ tracked=foo
+instance.print_self() # $ tracked=foo
+instance.foo = tracked # $ tracked=foo tracked
+instance.print_foo() # $ tracked=foo

python/ql/test/experimental/meta/ConceptsTest.qll

@@ -2,19 +2,7 @@ import python
 import semmle.python.dataflow.new.DataFlow
 import semmle.python.Concepts
 import TestUtilities.InlineExpectationsTest
-
-string value_from_expr(Expr e) {
-  // TODO: This one is starting to look like `repr` predicate from TestTaintLib
-  result =
-    e.(StrConst).getPrefix() + e.(StrConst).getText() +
-      e.(StrConst).getPrefix().regexpReplaceAll("[a-zA-Z]+", "")
-  or
-  result = e.(Name).getId()
-  or
-  not e instanceof StrConst and
-  not e instanceof Name and
-  result = e.toString()
-}
+import experimental.dataflow.TestUtil.PrintNode
 
 class SystemCommandExecutionTest extends InlineExpectationsTest {
   SystemCommandExecutionTest() { this = "SystemCommandExecutionTest" }
@@ -27,7 +15,7 @@ class SystemCommandExecutionTest extends InlineExpectationsTest {
       command = sce.getCommand() and
       location = command.getLocation() and
       element = command.toString() and
-      value = value_from_expr(command.asExpr()) and
+      value = prettyNodeForInlineTest(command) and
       tag = "getCommand"
     )
   }
@@ -46,7 +34,7 @@ class DecodingTest extends InlineExpectationsTest {
       exists(DataFlow::Node data |
         location = data.getLocation() and
         element = data.toString() and
-        value = value_from_expr(data.asExpr()) and
+        value = prettyNodeForInlineTest(data) and
         (
           data = d.getAnInput() and
           tag = "decodeInput"
@@ -84,7 +72,7 @@ class EncodingTest extends InlineExpectationsTest {
       exists(DataFlow::Node data |
         location = data.getLocation() and
         element = data.toString() and
-        value = value_from_expr(data.asExpr()) and
+        value = prettyNodeForInlineTest(data) and
         (
           data = e.getAnInput() and
           tag = "encodeInput"
@@ -117,7 +105,7 @@ class CodeExecutionTest extends InlineExpectationsTest {
       code = ce.getCode() and
       location = code.getLocation() and
       element = code.toString() and
-      value = value_from_expr(code.asExpr()) and
+      value = prettyNodeForInlineTest(code) and
       tag = "getCode"
     )
   }
@@ -135,7 +123,7 @@ class SqlExecutionTest extends InlineExpectationsTest {
       sql = e.getSql() and
       location = e.getLocation() and
       element = sql.toString() and
-      value = value_from_expr(sql.asExpr()) and
+      value = prettyNodeForInlineTest(sql) and
       tag = "getSql"
     )
   }
@@ -218,7 +206,7 @@ class HttpServerHttpResponseTest extends InlineExpectationsTest {
       exists(HTTP::Server::HttpResponse response |
         location = response.getLocation() and
         element = response.toString() and
-        value = value_from_expr(response.getBody().asExpr()) and
+        value = prettyNodeForInlineTest(response.getBody()) and
         tag = "responseBody"
       )
       or
@@ -257,7 +245,7 @@ class HttpServerHttpRedirectResponseTest extends InlineExpectationsTest {
       exists(HTTP::Server::HttpRedirectResponse redirect |
         location = redirect.getLocation() and
         element = redirect.toString() and
-        value = value_from_expr(redirect.getRedirectLocation().asExpr()) and
+        value = prettyNodeForInlineTest(redirect.getRedirectLocation()) and
         tag = "redirectLocation"
       )
     )
@@ -275,7 +263,7 @@ class FileSystemAccessTest extends InlineExpectationsTest {
       path = a.getAPathArgument() and
       location = a.getLocation() and
       element = path.toString() and
-      value = value_from_expr(path.asExpr()) and
+      value = prettyNodeForInlineTest(path) and
       tag = "getAPathArgument"
     )
   }
@@ -309,7 +297,7 @@ class SafeAccessCheckTest extends InlineExpectationsTest {
       location = c.getLocation() and
       (
         element = checks.toString() and
-        value = value_from_expr(checks.asExpr()) and
+        value = prettyNodeForInlineTest(checks) and
         tag = "checks"
         or
         element = branch.toString() and
@@ -341,3 +329,33 @@ class PublicKeyGenerationTest extends InlineExpectationsTest {
     )
   }
 }
+
+class CryptographicOperationTest extends InlineExpectationsTest {
+  CryptographicOperationTest() { this = "CryptographicOperationTest" }
+
+  override string getARelevantTag() {
+    result in [
+        "CryptographicOperation", "CryptographicOperationInput", "CryptographicOperationAlgorithm"
+      ]
+  }
+
+  override predicate hasActualResult(Location location, string element, string tag, string value) {
+    exists(location.getFile().getRelativePath()) and
+    exists(Cryptography::CryptographicOperation cryptoOperation |
+      location = cryptoOperation.getLocation() and
+      (
+        element = cryptoOperation.toString() and
+        value = "" and
+        tag = "CryptographicOperation"
+        or
+        element = cryptoOperation.toString() and
+        value = prettyNodeForInlineTest(cryptoOperation.getAnInput()) and
+        tag = "CryptographicOperationInput"
+        or
+        element = cryptoOperation.toString() and
+        value = cryptoOperation.getAlgorithm().getName() and
+        tag = "CryptographicOperationAlgorithm"
+      )
+    )
+  }
+}

python/ql/test/experimental/meta/InlineTaintTest.qll

@@ -58,7 +58,7 @@ class InlineTaintTest extends InlineExpectationsTest {
     exists(DataFlow::Node sink |
       any(TestTaintTrackingConfiguration config).hasFlow(_, sink) and
       location = sink.getLocation() and
-      element = prettyExp(sink.asExpr()) and
+      element = prettyExpr(sink.asExpr()) and
       value = "" and
       tag = "tainted"
     )
@@ -84,7 +84,7 @@ query predicate untaintedArgumentToEnsureTaintedNotMarkedAsMissing(
   error = "ERROR, you should add `# $ MISSING: tainted` annotation" and
   exists(DataFlow::Node sink |
     sink = shouldBeTainted() and
-    element = prettyExp(sink.asExpr()) and
+    element = prettyExpr(sink.asExpr()) and
     not any(TestTaintTrackingConfiguration config).hasFlow(_, sink) and
     location = sink.getLocation() and
     not exists(FalseNegativeExpectation missingResult |

python/ql/test/experimental/query-tests/Security/CWE-090/LDAPInjection.expected

@@ -0,0 +1,98 @@
+edges
+| ldap3_bad.py:13:17:13:23 | ControlFlowNode for request | ldap3_bad.py:13:17:13:28 | ControlFlowNode for Attribute |
+| ldap3_bad.py:13:17:13:23 | ControlFlowNode for request | ldap3_bad.py:14:21:14:27 | ControlFlowNode for request |
+| ldap3_bad.py:13:17:13:23 | ControlFlowNode for request | ldap3_bad.py:14:21:14:32 | ControlFlowNode for Attribute |
+| ldap3_bad.py:13:17:13:28 | ControlFlowNode for Attribute | ldap3_bad.py:13:17:13:34 | ControlFlowNode for Subscript |
+| ldap3_bad.py:13:17:13:34 | ControlFlowNode for Subscript | ldap3_bad.py:21:17:21:18 | ControlFlowNode for dn |
+| ldap3_bad.py:14:21:14:27 | ControlFlowNode for request | ldap3_bad.py:14:21:14:32 | ControlFlowNode for Attribute |
+| ldap3_bad.py:14:21:14:32 | ControlFlowNode for Attribute | ldap3_bad.py:14:21:14:44 | ControlFlowNode for Subscript |
+| ldap3_bad.py:14:21:14:44 | ControlFlowNode for Subscript | ldap3_bad.py:21:21:21:33 | ControlFlowNode for search_filter |
+| ldap3_bad.py:30:17:30:23 | ControlFlowNode for request | ldap3_bad.py:30:17:30:28 | ControlFlowNode for Attribute |
+| ldap3_bad.py:30:17:30:23 | ControlFlowNode for request | ldap3_bad.py:31:21:31:27 | ControlFlowNode for request |
+| ldap3_bad.py:30:17:30:23 | ControlFlowNode for request | ldap3_bad.py:31:21:31:32 | ControlFlowNode for Attribute |
+| ldap3_bad.py:30:17:30:28 | ControlFlowNode for Attribute | ldap3_bad.py:30:17:30:34 | ControlFlowNode for Subscript |
+| ldap3_bad.py:30:17:30:34 | ControlFlowNode for Subscript | ldap3_bad.py:38:9:38:10 | ControlFlowNode for dn |
+| ldap3_bad.py:31:21:31:27 | ControlFlowNode for request | ldap3_bad.py:31:21:31:32 | ControlFlowNode for Attribute |
+| ldap3_bad.py:31:21:31:32 | ControlFlowNode for Attribute | ldap3_bad.py:31:21:31:44 | ControlFlowNode for Subscript |
+| ldap3_bad.py:31:21:31:44 | ControlFlowNode for Subscript | ldap3_bad.py:38:13:38:25 | ControlFlowNode for search_filter |
+| ldap_bad.py:13:17:13:23 | ControlFlowNode for request | ldap_bad.py:13:17:13:28 | ControlFlowNode for Attribute |
+| ldap_bad.py:13:17:13:23 | ControlFlowNode for request | ldap_bad.py:14:21:14:27 | ControlFlowNode for request |
+| ldap_bad.py:13:17:13:23 | ControlFlowNode for request | ldap_bad.py:14:21:14:32 | ControlFlowNode for Attribute |
+| ldap_bad.py:13:17:13:28 | ControlFlowNode for Attribute | ldap_bad.py:13:17:13:34 | ControlFlowNode for Subscript |
+| ldap_bad.py:13:17:13:34 | ControlFlowNode for Subscript | ldap_bad.py:21:9:21:10 | ControlFlowNode for dn |
+| ldap_bad.py:14:21:14:27 | ControlFlowNode for request | ldap_bad.py:14:21:14:32 | ControlFlowNode for Attribute |
+| ldap_bad.py:14:21:14:32 | ControlFlowNode for Attribute | ldap_bad.py:14:21:14:44 | ControlFlowNode for Subscript |
+| ldap_bad.py:14:21:14:44 | ControlFlowNode for Subscript | ldap_bad.py:21:33:21:45 | ControlFlowNode for search_filter |
+| ldap_bad.py:30:17:30:23 | ControlFlowNode for request | ldap_bad.py:30:17:30:28 | ControlFlowNode for Attribute |
+| ldap_bad.py:30:17:30:23 | ControlFlowNode for request | ldap_bad.py:31:21:31:27 | ControlFlowNode for request |
+| ldap_bad.py:30:17:30:23 | ControlFlowNode for request | ldap_bad.py:31:21:31:32 | ControlFlowNode for Attribute |
+| ldap_bad.py:30:17:30:28 | ControlFlowNode for Attribute | ldap_bad.py:30:17:30:34 | ControlFlowNode for Subscript |
+| ldap_bad.py:30:17:30:34 | ControlFlowNode for Subscript | ldap_bad.py:37:9:37:10 | ControlFlowNode for dn |
+| ldap_bad.py:31:21:31:27 | ControlFlowNode for request | ldap_bad.py:31:21:31:32 | ControlFlowNode for Attribute |
+| ldap_bad.py:31:21:31:32 | ControlFlowNode for Attribute | ldap_bad.py:31:21:31:44 | ControlFlowNode for Subscript |
+| ldap_bad.py:31:21:31:44 | ControlFlowNode for Subscript | ldap_bad.py:37:33:37:45 | ControlFlowNode for search_filter |
+| ldap_bad.py:47:17:47:23 | ControlFlowNode for request | ldap_bad.py:47:17:47:28 | ControlFlowNode for Attribute |
+| ldap_bad.py:47:17:47:23 | ControlFlowNode for request | ldap_bad.py:48:21:48:27 | ControlFlowNode for request |
+| ldap_bad.py:47:17:47:23 | ControlFlowNode for request | ldap_bad.py:48:21:48:32 | ControlFlowNode for Attribute |
+| ldap_bad.py:47:17:47:28 | ControlFlowNode for Attribute | ldap_bad.py:47:17:47:34 | ControlFlowNode for Subscript |
+| ldap_bad.py:47:17:47:34 | ControlFlowNode for Subscript | ldap_bad.py:55:9:55:10 | ControlFlowNode for dn |
+| ldap_bad.py:48:21:48:27 | ControlFlowNode for request | ldap_bad.py:48:21:48:32 | ControlFlowNode for Attribute |
+| ldap_bad.py:48:21:48:32 | ControlFlowNode for Attribute | ldap_bad.py:48:21:48:44 | ControlFlowNode for Subscript |
+| ldap_bad.py:48:21:48:44 | ControlFlowNode for Subscript | ldap_bad.py:55:43:55:55 | ControlFlowNode for search_filter |
+nodes
+| ldap3_bad.py:13:17:13:23 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
+| ldap3_bad.py:13:17:13:28 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
+| ldap3_bad.py:13:17:13:34 | ControlFlowNode for Subscript | semmle.label | ControlFlowNode for Subscript |
+| ldap3_bad.py:14:21:14:27 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
+| ldap3_bad.py:14:21:14:32 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
+| ldap3_bad.py:14:21:14:44 | ControlFlowNode for Subscript | semmle.label | ControlFlowNode for Subscript |
+| ldap3_bad.py:21:17:21:18 | ControlFlowNode for dn | semmle.label | ControlFlowNode for dn |
+| ldap3_bad.py:21:21:21:33 | ControlFlowNode for search_filter | semmle.label | ControlFlowNode for search_filter |
+| ldap3_bad.py:30:17:30:23 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
+| ldap3_bad.py:30:17:30:28 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
+| ldap3_bad.py:30:17:30:34 | ControlFlowNode for Subscript | semmle.label | ControlFlowNode for Subscript |
+| ldap3_bad.py:31:21:31:27 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
+| ldap3_bad.py:31:21:31:32 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
+| ldap3_bad.py:31:21:31:44 | ControlFlowNode for Subscript | semmle.label | ControlFlowNode for Subscript |
+| ldap3_bad.py:38:9:38:10 | ControlFlowNode for dn | semmle.label | ControlFlowNode for dn |
+| ldap3_bad.py:38:13:38:25 | ControlFlowNode for search_filter | semmle.label | ControlFlowNode for search_filter |
+| ldap_bad.py:13:17:13:23 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
+| ldap_bad.py:13:17:13:28 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
+| ldap_bad.py:13:17:13:34 | ControlFlowNode for Subscript | semmle.label | ControlFlowNode for Subscript |
+| ldap_bad.py:14:21:14:27 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
+| ldap_bad.py:14:21:14:32 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
+| ldap_bad.py:14:21:14:44 | ControlFlowNode for Subscript | semmle.label | ControlFlowNode for Subscript |
+| ldap_bad.py:21:9:21:10 | ControlFlowNode for dn | semmle.label | ControlFlowNode for dn |
+| ldap_bad.py:21:33:21:45 | ControlFlowNode for search_filter | semmle.label | ControlFlowNode for search_filter |
+| ldap_bad.py:30:17:30:23 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
+| ldap_bad.py:30:17:30:28 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
+| ldap_bad.py:30:17:30:34 | ControlFlowNode for Subscript | semmle.label | ControlFlowNode for Subscript |
+| ldap_bad.py:31:21:31:27 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
+| ldap_bad.py:31:21:31:32 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
+| ldap_bad.py:31:21:31:44 | ControlFlowNode for Subscript | semmle.label | ControlFlowNode for Subscript |
+| ldap_bad.py:37:9:37:10 | ControlFlowNode for dn | semmle.label | ControlFlowNode for dn |
+| ldap_bad.py:37:33:37:45 | ControlFlowNode for search_filter | semmle.label | ControlFlowNode for search_filter |
+| ldap_bad.py:47:17:47:23 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
+| ldap_bad.py:47:17:47:28 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
+| ldap_bad.py:47:17:47:34 | ControlFlowNode for Subscript | semmle.label | ControlFlowNode for Subscript |
+| ldap_bad.py:48:21:48:27 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
+| ldap_bad.py:48:21:48:32 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
+| ldap_bad.py:48:21:48:44 | ControlFlowNode for Subscript | semmle.label | ControlFlowNode for Subscript |
+| ldap_bad.py:55:9:55:10 | ControlFlowNode for dn | semmle.label | ControlFlowNode for dn |
+| ldap_bad.py:55:43:55:55 | ControlFlowNode for search_filter | semmle.label | ControlFlowNode for search_filter |
+#select
+| ldap3_bad.py:21:17:21:18 | ControlFlowNode for dn | ldap3_bad.py:13:17:13:23 | ControlFlowNode for request | ldap3_bad.py:21:17:21:18 | ControlFlowNode for dn | $@ LDAP query parameter comes from $@. | ldap3_bad.py:21:17:21:18 | ControlFlowNode for dn | This | ldap3_bad.py:13:17:13:23 | ControlFlowNode for request | a user-provided value |
+| ldap3_bad.py:21:21:21:33 | ControlFlowNode for search_filter | ldap3_bad.py:13:17:13:23 | ControlFlowNode for request | ldap3_bad.py:21:21:21:33 | ControlFlowNode for search_filter | $@ LDAP query parameter comes from $@. | ldap3_bad.py:21:21:21:33 | ControlFlowNode for search_filter | This | ldap3_bad.py:13:17:13:23 | ControlFlowNode for request | a user-provided value |
+| ldap3_bad.py:21:21:21:33 | ControlFlowNode for search_filter | ldap3_bad.py:14:21:14:27 | ControlFlowNode for request | ldap3_bad.py:21:21:21:33 | ControlFlowNode for search_filter | $@ LDAP query parameter comes from $@. | ldap3_bad.py:21:21:21:33 | ControlFlowNode for search_filter | This | ldap3_bad.py:14:21:14:27 | ControlFlowNode for request | a user-provided value |
+| ldap3_bad.py:38:9:38:10 | ControlFlowNode for dn | ldap3_bad.py:30:17:30:23 | ControlFlowNode for request | ldap3_bad.py:38:9:38:10 | ControlFlowNode for dn | $@ LDAP query parameter comes from $@. | ldap3_bad.py:38:9:38:10 | ControlFlowNode for dn | This | ldap3_bad.py:30:17:30:23 | ControlFlowNode for request | a user-provided value |
+| ldap3_bad.py:38:13:38:25 | ControlFlowNode for search_filter | ldap3_bad.py:30:17:30:23 | ControlFlowNode for request | ldap3_bad.py:38:13:38:25 | ControlFlowNode for search_filter | $@ LDAP query parameter comes from $@. | ldap3_bad.py:38:13:38:25 | ControlFlowNode for search_filter | This | ldap3_bad.py:30:17:30:23 | ControlFlowNode for request | a user-provided value |
+| ldap3_bad.py:38:13:38:25 | ControlFlowNode for search_filter | ldap3_bad.py:31:21:31:27 | ControlFlowNode for request | ldap3_bad.py:38:13:38:25 | ControlFlowNode for search_filter | $@ LDAP query parameter comes from $@. | ldap3_bad.py:38:13:38:25 | ControlFlowNode for search_filter | This | ldap3_bad.py:31:21:31:27 | ControlFlowNode for request | a user-provided value |
+| ldap_bad.py:21:9:21:10 | ControlFlowNode for dn | ldap_bad.py:13:17:13:23 | ControlFlowNode for request | ldap_bad.py:21:9:21:10 | ControlFlowNode for dn | $@ LDAP query parameter comes from $@. | ldap_bad.py:21:9:21:10 | ControlFlowNode for dn | This | ldap_bad.py:13:17:13:23 | ControlFlowNode for request | a user-provided value |
+| ldap_bad.py:21:33:21:45 | ControlFlowNode for search_filter | ldap_bad.py:13:17:13:23 | ControlFlowNode for request | ldap_bad.py:21:33:21:45 | ControlFlowNode for search_filter | $@ LDAP query parameter comes from $@. | ldap_bad.py:21:33:21:45 | ControlFlowNode for search_filter | This | ldap_bad.py:13:17:13:23 | ControlFlowNode for request | a user-provided value |
+| ldap_bad.py:21:33:21:45 | ControlFlowNode for search_filter | ldap_bad.py:14:21:14:27 | ControlFlowNode for request | ldap_bad.py:21:33:21:45 | ControlFlowNode for search_filter | $@ LDAP query parameter comes from $@. | ldap_bad.py:21:33:21:45 | ControlFlowNode for search_filter | This | ldap_bad.py:14:21:14:27 | ControlFlowNode for request | a user-provided value |
+| ldap_bad.py:37:9:37:10 | ControlFlowNode for dn | ldap_bad.py:30:17:30:23 | ControlFlowNode for request | ldap_bad.py:37:9:37:10 | ControlFlowNode for dn | $@ LDAP query parameter comes from $@. | ldap_bad.py:37:9:37:10 | ControlFlowNode for dn | This | ldap_bad.py:30:17:30:23 | ControlFlowNode for request | a user-provided value |
+| ldap_bad.py:37:33:37:45 | ControlFlowNode for search_filter | ldap_bad.py:30:17:30:23 | ControlFlowNode for request | ldap_bad.py:37:33:37:45 | ControlFlowNode for search_filter | $@ LDAP query parameter comes from $@. | ldap_bad.py:37:33:37:45 | ControlFlowNode for search_filter | This | ldap_bad.py:30:17:30:23 | ControlFlowNode for request | a user-provided value |
+| ldap_bad.py:37:33:37:45 | ControlFlowNode for search_filter | ldap_bad.py:31:21:31:27 | ControlFlowNode for request | ldap_bad.py:37:33:37:45 | ControlFlowNode for search_filter | $@ LDAP query parameter comes from $@. | ldap_bad.py:37:33:37:45 | ControlFlowNode for search_filter | This | ldap_bad.py:31:21:31:27 | ControlFlowNode for request | a user-provided value |
+| ldap_bad.py:55:9:55:10 | ControlFlowNode for dn | ldap_bad.py:47:17:47:23 | ControlFlowNode for request | ldap_bad.py:55:9:55:10 | ControlFlowNode for dn | $@ LDAP query parameter comes from $@. | ldap_bad.py:55:9:55:10 | ControlFlowNode for dn | This | ldap_bad.py:47:17:47:23 | ControlFlowNode for request | a user-provided value |
+| ldap_bad.py:55:43:55:55 | ControlFlowNode for search_filter | ldap_bad.py:47:17:47:23 | ControlFlowNode for request | ldap_bad.py:55:43:55:55 | ControlFlowNode for search_filter | $@ LDAP query parameter comes from $@. | ldap_bad.py:55:43:55:55 | ControlFlowNode for search_filter | This | ldap_bad.py:47:17:47:23 | ControlFlowNode for request | a user-provided value |
+| ldap_bad.py:55:43:55:55 | ControlFlowNode for search_filter | ldap_bad.py:48:21:48:27 | ControlFlowNode for request | ldap_bad.py:55:43:55:55 | ControlFlowNode for search_filter | $@ LDAP query parameter comes from $@. | ldap_bad.py:55:43:55:55 | ControlFlowNode for search_filter | This | ldap_bad.py:48:21:48:27 | ControlFlowNode for request | a user-provided value |

python/ql/test/experimental/query-tests/Security/CWE-090/LDAPInjection.qlref

@@ -0,0 +1 @@
+experimental/Security/CWE-090/LDAPInjection.ql

python/ql/test/experimental/query-tests/Security/CWE-090/ldap3_bad.py

@@ -0,0 +1,41 @@
+from flask import request, Flask
+import ldap3
+
+app = Flask(__name__)
+
+
+@app.route("/normal")
+def normal():
+    """
+    A RemoteFlowSource is used directly as DN and search filter
+    """
+
+    unsafe_dc = request.args['dc']
+    unsafe_filter = request.args['username']
+
+    dn = "dc={}".format(unsafe_dc)
+    search_filter = "(user={})".format(unsafe_filter)
+
+    srv = ldap3.Server('ldap://127.0.0.1')
+    conn = ldap3.Connection(srv, user=dn, auto_bind=True)
+    conn.search(dn, search_filter)
+
+
+@app.route("/direct")
+def direct():
+    """
+    A RemoteFlowSource is used directly as DN and search filter using a oneline call to .search
+    """
+
+    unsafe_dc = request.args['dc']
+    unsafe_filter = request.args['username']
+
+    dn = "dc={}".format(unsafe_dc)
+    search_filter = "(user={})".format(unsafe_filter)
+
+    srv = ldap3.Server('ldap://127.0.0.1')
+    conn = ldap3.Connection(srv, user=dn, auto_bind=True).search(
+        dn, search_filter)
+
+# if __name__ == "__main__":
+#     app.run(debug=True)

python/ql/test/experimental/query-tests/Security/CWE-090/ldap3_good.py

@@ -0,0 +1,49 @@
+from flask import request, Flask
+import ldap3
+from ldap3.utils.dn import escape_rdn
+from ldap3.utils.conv import escape_filter_chars
+
+app = Flask(__name__)
+
+
+@app.route("/normal")
+def normal():
+    """
+    A RemoteFlowSource is sanitized and used as DN and search filter
+    """
+
+    unsafe_dc = request.args['dc']
+    unsafe_filter = request.args['username']
+
+    safe_dc = escape_rdn(unsafe_dc)
+    safe_filter = escape_filter_chars(unsafe_filter)
+
+    dn = "dc={}".format(safe_dc)
+    search_filter = "(user={})".format(safe_filter)
+
+    srv = ldap3.Server('ldap://127.0.0.1')
+    conn = ldap3.Connection(srv, user=dn, auto_bind=True)
+    conn.search(dn, search_filter)
+
+
+@app.route("/direct")
+def direct():
+    """
+    A RemoteFlowSource is sanitized and used as DN and search filter using a oneline call to .search
+    """
+
+    unsafe_dc = request.args['dc']
+    unsafe_filter = request.args['username']
+
+    safe_dc = escape_rdn(unsafe_dc)
+    safe_filter = escape_filter_chars(unsafe_filter)
+
+    dn = "dc={}".format(safe_dc)
+    search_filter = "(user={})".format(safe_filter)
+
+    srv = ldap3.Server('ldap://127.0.0.1')
+    conn = ldap3.Connection(srv, user=dn, auto_bind=True).search(
+        dn, search_filter)
+
+# if __name__ == "__main__":
+#     app.run(debug=True)

python/ql/test/experimental/query-tests/Security/CWE-090/ldap_bad.py

@@ -0,0 +1,59 @@
+from flask import request, Flask
+import ldap
+
+app = Flask(__name__)
+
+
+@app.route("/normal")
+def normal():
+    """
+    A RemoteFlowSource is used directly as DN and search filter
+    """
+
+    unsafe_dc = request.args['dc']
+    unsafe_filter = request.args['username']
+
+    dn = "dc={}".format(unsafe_dc)
+    search_filter = "(user={})".format(unsafe_filter)
+
+    ldap_connection = ldap.initialize("ldap://127.0.0.1")
+    user = ldap_connection.search_s(
+        dn, ldap.SCOPE_SUBTREE, search_filter)
+
+
+@app.route("/direct")
+def direct():
+    """
+    A RemoteFlowSource is used directly as DN and search filter using a oneline call to .search_s
+    """
+
+    unsafe_dc = request.args['dc']
+    unsafe_filter = request.args['username']
+
+    dn = "dc={}".format(unsafe_dc)
+    search_filter = "(user={})".format(unsafe_filter)
+
+    user = ldap.initialize("ldap://127.0.0.1").search_s(
+        dn, ldap.SCOPE_SUBTREE, search_filter)
+
+
+@app.route("/normal_argbyname")
+def normal_argbyname():
+    """
+    A RemoteFlowSource is used directly as DN and search filter, while the search filter is specified as
+    an argument by name
+    """
+
+    unsafe_dc = request.args['dc']
+    unsafe_filter = request.args['username']
+
+    dn = "dc={}".format(unsafe_dc)
+    search_filter = "(user={})".format(unsafe_filter)
+
+    ldap_connection = ldap.initialize("ldap://127.0.0.1")
+    user = ldap_connection.search_s(
+        dn, ldap.SCOPE_SUBTREE, filterstr=search_filter)
+
+
+# if __name__ == "__main__":
+#     app.run(debug=True)

python/ql/test/experimental/query-tests/Security/CWE-090/ldap_good.py

@@ -0,0 +1,70 @@
+from flask import request, Flask
+import ldap
+import ldap.filter
+import ldap.dn
+
+app = Flask(__name__)
+
+
+@app.route("/normal")
+def normal():
+    """
+    A RemoteFlowSource is sanitized and used as DN and search filter
+    """
+
+    unsafe_dc = request.args['dc']
+    unsafe_filter = request.args['username']
+
+    safe_dc = ldap.dn.escape_dn_chars(unsafe_dc)
+    safe_filter = ldap.filter.escape_filter_chars(unsafe_filter)
+
+    dn = "dc={}".format(safe_dc)
+    search_filter = "(user={})".format(safe_filter)
+
+    ldap_connection = ldap.initialize("ldap://127.0.0.1")
+    user = ldap_connection.search_s(
+        dn, ldap.SCOPE_SUBTREE, search_filter)
+
+
+@app.route("/direct")
+def direct():
+    """
+    A RemoteFlowSource is sanitized and used as DN and search filter using a oneline call to .search_s
+    """
+
+    unsafe_dc = request.args['dc']
+    unsafe_filter = request.args['username']
+
+    safe_dc = ldap.dn.escape_dn_chars(unsafe_dc)
+    safe_filter = ldap.filter.escape_filter_chars(unsafe_filter)
+
+    dn = "dc={}".format(safe_dc)
+    search_filter = "(user={})".format(safe_filter)
+
+    user = ldap.initialize("ldap://127.0.0.1").search_s(
+        dn, ldap.SCOPE_SUBTREE, search_filter, ["testAttr1", "testAttr2"])
+
+
+@app.route("/normal_argbyname")
+def normal_argbyname():
+    """
+    A RemoteFlowSource is sanitized and used as DN and search filter, while the search filter is specified as
+    an argument by name
+    """
+
+    unsafe_dc = request.args['dc']
+    unsafe_filter = request.args['username']
+
+    safe_dc = ldap.dn.escape_dn_chars(unsafe_dc)
+    safe_filter = ldap.filter.escape_filter_chars(unsafe_filter)
+
+    dn = "dc={}".format(safe_dc)
+    search_filter = "(user={})".format(safe_filter)
+
+    ldap_connection = ldap.initialize("ldap://127.0.0.1")
+    user = ldap_connection.search_s(
+        dn, ldap.SCOPE_SUBTREE, filterstr=search_filter)
+
+
+# if __name__ == "__main__":
+#     app.run(debug=True)

python/ql/test/experimental/query-tests/Security/CWE-730/RegexInjection.expected

@@ -0,0 +1,27 @@
+edges
+| re_bad.py:13:22:13:28 | ControlFlowNode for request | re_bad.py:13:22:13:33 | ControlFlowNode for Attribute |
+| re_bad.py:13:22:13:33 | ControlFlowNode for Attribute | re_bad.py:13:22:13:44 | ControlFlowNode for Subscript |
+| re_bad.py:13:22:13:44 | ControlFlowNode for Subscript | re_bad.py:14:15:14:28 | ControlFlowNode for unsafe_pattern |
+| re_bad.py:24:22:24:28 | ControlFlowNode for request | re_bad.py:24:22:24:33 | ControlFlowNode for Attribute |
+| re_bad.py:24:22:24:33 | ControlFlowNode for Attribute | re_bad.py:24:22:24:44 | ControlFlowNode for Subscript |
+| re_bad.py:24:22:24:44 | ControlFlowNode for Subscript | re_bad.py:25:35:25:48 | ControlFlowNode for unsafe_pattern |
+| re_bad.py:36:22:36:28 | ControlFlowNode for request | re_bad.py:36:22:36:33 | ControlFlowNode for Attribute |
+| re_bad.py:36:22:36:33 | ControlFlowNode for Attribute | re_bad.py:36:22:36:44 | ControlFlowNode for Subscript |
+| re_bad.py:36:22:36:44 | ControlFlowNode for Subscript | re_bad.py:37:16:37:29 | ControlFlowNode for unsafe_pattern |
+nodes
+| re_bad.py:13:22:13:28 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
+| re_bad.py:13:22:13:33 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
+| re_bad.py:13:22:13:44 | ControlFlowNode for Subscript | semmle.label | ControlFlowNode for Subscript |
+| re_bad.py:14:15:14:28 | ControlFlowNode for unsafe_pattern | semmle.label | ControlFlowNode for unsafe_pattern |
+| re_bad.py:24:22:24:28 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
+| re_bad.py:24:22:24:33 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
+| re_bad.py:24:22:24:44 | ControlFlowNode for Subscript | semmle.label | ControlFlowNode for Subscript |
+| re_bad.py:25:35:25:48 | ControlFlowNode for unsafe_pattern | semmle.label | ControlFlowNode for unsafe_pattern |
+| re_bad.py:36:22:36:28 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
+| re_bad.py:36:22:36:33 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
+| re_bad.py:36:22:36:44 | ControlFlowNode for Subscript | semmle.label | ControlFlowNode for Subscript |
+| re_bad.py:37:16:37:29 | ControlFlowNode for unsafe_pattern | semmle.label | ControlFlowNode for unsafe_pattern |
+#select
+| re_bad.py:14:15:14:28 | ControlFlowNode for unsafe_pattern | re_bad.py:13:22:13:28 | ControlFlowNode for request | re_bad.py:14:15:14:28 | ControlFlowNode for unsafe_pattern | $@ regular expression is constructed from a $@ and executed by $@. | re_bad.py:14:15:14:28 | ControlFlowNode for unsafe_pattern | This | re_bad.py:13:22:13:28 | ControlFlowNode for request | user-provided value | re_bad.py:14:5:14:13 | Attribute | re.search |
+| re_bad.py:25:35:25:48 | ControlFlowNode for unsafe_pattern | re_bad.py:24:22:24:28 | ControlFlowNode for request | re_bad.py:25:35:25:48 | ControlFlowNode for unsafe_pattern | $@ regular expression is constructed from a $@ and executed by $@. | re_bad.py:25:35:25:48 | ControlFlowNode for unsafe_pattern | This | re_bad.py:24:22:24:28 | ControlFlowNode for request | user-provided value | re_bad.py:26:5:26:27 | Attribute | re.search |
+| re_bad.py:37:16:37:29 | ControlFlowNode for unsafe_pattern | re_bad.py:36:22:36:28 | ControlFlowNode for request | re_bad.py:37:16:37:29 | ControlFlowNode for unsafe_pattern | $@ regular expression is constructed from a $@ and executed by $@. | re_bad.py:37:16:37:29 | ControlFlowNode for unsafe_pattern | This | re_bad.py:36:22:36:28 | ControlFlowNode for request | user-provided value | re_bad.py:37:5:37:37 | Attribute | re.search |

python/ql/test/experimental/query-tests/Security/CWE-730/RegexInjection.qlref

@@ -0,0 +1 @@
+experimental/Security/CWE-730/RegexInjection.ql

python/ql/test/experimental/query-tests/Security/CWE-730/re_bad.py

@@ -0,0 +1,40 @@
+from flask import request, Flask
+import re
+
+app = Flask(__name__)
+
+
+@app.route("/direct")
+def direct():
+    """
+    A RemoteFlowSource is used directly as re.search's pattern
+    """
+
+    unsafe_pattern = request.args["pattern"]
+    re.search(unsafe_pattern, "")
+
+
+@app.route("/compile")
+def compile():
+    """
+    A RemoteFlowSource is used directly as re.compile's pattern
+    which also executes .search() 
+    """
+
+    unsafe_pattern = request.args["pattern"]
+    compiled_pattern = re.compile(unsafe_pattern)
+    compiled_pattern.search("")
+
+
+@app.route("/compile_direct")
+def compile_direct():
+    """
+    A RemoteFlowSource is used directly as re.compile's pattern
+    which also executes .search() in the same line
+    """
+
+    unsafe_pattern = request.args["pattern"]
+    re.compile(unsafe_pattern).search("")
+
+# if __name__ == "__main__":
+#     app.run(debug=True)

python/ql/test/experimental/query-tests/Security/CWE-730/re_good.py

@@ -0,0 +1,45 @@
+from flask import request, Flask
+import re
+
+app = Flask(__name__)
+
+
+@app.route("/direct")
+def direct():
+    """
+    A RemoteFlowSource is escaped by re.escape and then used as
+    re'search pattern
+    """
+
+    unsafe_pattern = request.args['pattern']
+    safe_pattern = re.escape(unsafe_pattern)
+    re.search(safe_pattern, "")
+
+
+@app.route("/compile")
+def compile():
+    """
+    A RemoteFlowSource is escaped by re.escape and used as re.compile's
+    pattern which also executes .search()
+    """
+
+    unsafe_pattern = request.args['pattern']
+    safe_pattern = re.escape(unsafe_pattern)
+    compiled_pattern = re.compile(safe_pattern)
+    compiled_pattern.search("")
+
+
+@app.route("/compile_direct")
+def compile_direct():
+    """
+    A RemoteFlowSource is escaped by re.escape and then used as re.compile's
+    pattern which also executes .search() in the same line
+    """
+
+    unsafe_pattern = request.args['pattern']
+    safe_pattern = re.escape(unsafe_pattern)
+    re.compile(safe_pattern).search("")
+
+
+# if __name__ == "__main__":
+#     app.run(debug=True)

python/ql/test/experimental/semmle/python/frameworks/clickhouse-driver/ClickHouseDriver.expected

@@ -0,0 +1,5 @@
+| ClickHouseDriver.py:15:22:15:106 | ControlFlowNode for Attribute() | ClickHouseDriver.py:15:52:15:105 | ControlFlowNode for BinaryExpr |
+| ClickHouseDriver.py:18:5:18:87 | ControlFlowNode for Attribute() | ClickHouseDriver.py:18:33:18:86 | ControlFlowNode for BinaryExpr |
+| ClickHouseDriver.py:22:5:22:62 | ControlFlowNode for Attribute() | ClickHouseDriver.py:22:33:22:37 | ControlFlowNode for query |
+| ClickHouseDriver.py:27:5:27:74 | ControlFlowNode for Attribute() | ClickHouseDriver.py:27:20:27:73 | ControlFlowNode for BinaryExpr |
+| ClickHouseDriver.py:30:5:30:89 | ControlFlowNode for Attribute() | ClickHouseDriver.py:30:35:30:88 | ControlFlowNode for BinaryExpr |

python/ql/test/experimental/semmle/python/frameworks/clickhouse-driver/ClickHouseDriver.py

@@ -0,0 +1,32 @@
+from django.conf.urls import url
+from clickhouse_driver import Client
+from clickhouse_driver import connect
+from aioch import Client as aiochClient
+
+# Dummy Client subclass
+class MyClient(Client):
+    def dummy(self):
+        return None
+
+def show_user(request, username):
+
+    # BAD -- Untrusted user input is directly injected into the sql query using async library 'aioch'
+    aclient = aiochClient("localhost")
+    progress = await aclient.execute_with_progress("SELECT * FROM users WHERE username = '%s'" % username)
+
+    # BAD -- Untrusted user input is directly injected into the sql query using native client of library 'clickhouse_driver'
+    Client('localhost').execute("SELECT * FROM users WHERE username = '%s'" % username)
+
+    # GOOD -- query uses prepared statements
+    query = "SELECT * FROM users WHERE username = %(username)s"
+    Client('localhost').execute(query, {"username": username})
+
+    # BAD -- Untrusted user input is directly injected into the sql query using PEP249 interface
+    conn = connect('clickhouse://localhost')
+    cursor = conn.cursor()
+    cursor.execute("SELECT * FROM users WHERE username = '%s'" % username)
+
+    # BAD -- Untrusted user input is directly injected into the sql query using MyClient, which is a subclass of Client
+    MyClient('localhost').execute("SELECT * FROM users WHERE username = '%s'" % username)
+
+urlpatterns = [url(r'^users/(?P<username>[^/]+)$', show_user)]

python/ql/test/experimental/semmle/python/frameworks/clickhouse-driver/ClickHouseDriver.ql

@@ -0,0 +1,6 @@
+import python
+import experimental.semmle.python.frameworks.ClickHouseDriver
+import semmle.python.Concepts
+
+from SqlExecution s
+select s, s.getSql()

python/ql/test/library-tests/frameworks/aiohttp/ConceptsTest.ql

@@ -0,0 +1,12 @@
+import python
+import experimental.meta.ConceptsTest
+
+class DedicatedResponseTest extends HttpServerHttpResponseTest {
+  DedicatedResponseTest() { file.getShortName() = "response_test.py" }
+}
+
+class OtherResponseTest extends HttpServerHttpResponseTest {
+  OtherResponseTest() { not this instanceof DedicatedResponseTest }
+
+  override string getARelevantTag() { result = "HttpResponse" }
+}

python/ql/test/library-tests/frameworks/aiohttp/InlineTaintTest.expected

@@ -0,0 +1,3 @@
+argumentToEnsureNotTaintedNotMarkedAsSpurious
+untaintedArgumentToEnsureTaintedNotMarkedAsMissing
+failures

python/ql/test/library-tests/frameworks/aiohttp/InlineTaintTest.ql

@@ -0,0 +1 @@
+import experimental.meta.InlineTaintTest

python/ql/test/library-tests/frameworks/aiohttp/app_conf_test.py

@@ -0,0 +1,48 @@
+"""
+This file is a test of an extra data-flow step that we want to have for
+aiohttp.web.Application
+
+We don't really have an established way to test extra data-flow steps in external
+libraries right now, so for now I've just used our normal taint-flow testing ¯\_(ツ)_/¯
+
+see https://docs.aiohttp.org/en/stable/web_advanced.html#application-s-config
+"""
+
+from aiohttp import web
+
+# to make code runable
+TAINTED_STRING = "TAINTED_STRING"
+def ensure_tainted(*args, **kwargs):
+    pass
+
+ensure_tainted(
+    TAINTED_STRING # $ tainted
+)
+
+
+async def example(request: web.Request): # $ requestHandler
+    return web.Response(text=f'example {request.app["foo"]=}') # $ HttpResponse
+
+
+async def also_works(request: web.Request): # $ requestHandler
+    return web.Response(text=f'also_works {request.config_dict["foo"]=}') # $ HttpResponse
+
+
+async def taint_test(request: web.Request): # $ requestHandler
+    ensure_tainted(
+        request.app["ts"], # $ MISSING: tainted
+        request.config_dict["ts"], # $ MISSING: tainted
+    )
+    return web.Response(text="ok") # $ HttpResponse
+
+
+app = web.Application()
+app.router.add_get("", example) # $ routeSetup=""
+app.router.add_get("/also-works", also_works) # $ routeSetup="/also-works"
+app.router.add_get("/taint-test", taint_test) # $ routeSetup="/taint-test"
+app["foo"] = 42
+app["ts"] = TAINTED_STRING
+
+
+if __name__ == "__main__":
+    web.run_app(app)

python/ql/test/library-tests/frameworks/aiohttp/options

@@ -0,0 +1 @@
+semmle-extractor-options: --max-import-depth=1 --lang=3

python/ql/test/library-tests/frameworks/aiohttp/response_test.py

@@ -0,0 +1,72 @@
+from aiohttp import web
+
+
+routes = web.RouteTableDef()
+
+
+@routes.get("/raw_text") # $ routeSetup="/raw_text"
+async def raw_text(request): # $ requestHandler
+    return web.Response(text="foo") # $ HttpResponse mimetype=text/plain responseBody="foo"
+
+
+@routes.get("/raw_body") # $ routeSetup="/raw_body"
+async def raw_body(request): # $ requestHandler
+    return web.Response(body=b"foo") # $ HttpResponse mimetype=application/octet-stream responseBody=b"foo"
+
+
+@routes.get("/html_text") # $ routeSetup="/html_text"
+async def html_text(request): # $ requestHandler
+    return web.Response(text="foo", content_type="text/html") # $ HttpResponse mimetype=text/html responseBody="foo"
+
+
+@routes.get("/html_body") # $ routeSetup="/html_body"
+async def html_body(request): # $ requestHandler
+    return web.Response(body=b"foo", content_type="text/html") # $ HttpResponse mimetype=text/html responseBody=b"foo"
+
+
+@routes.get("/html_body_set_later") # $ routeSetup="/html_body_set_later"
+async def html_body_set_later(request): # $ requestHandler
+    resp = web.Response(body=b"foo") # $ HttpResponse mimetype=application/octet-stream responseBody=b"foo"
+    resp.content_type = "text/html" # $ MISSING: mimetype=text/html
+    return resp
+
+# Each HTTP status code has an exception
+# see https://docs.aiohttp.org/en/stable/web_quickstart.html#exceptions
+
+@routes.get("/through_200_exception") # $ routeSetup="/through_200_exception"
+async def through_200_exception(request): # $ requestHandler
+    raise web.HTTPOk(text="foo") # $ HttpResponse mimetype=text/plain responseBody="foo"
+
+
+@routes.get("/through_200_exception_html") # $ routeSetup="/through_200_exception_html"
+async def through_200_exception(request): # $ requestHandler
+    exception = web.HTTPOk(text="foo") # $ HttpResponse mimetype=text/plain responseBody="foo"
+    exception.content_type = "text/html" # $ MISSING: mimetype=text/html
+    raise exception
+
+
+@routes.get("/through_404_exception") # $ routeSetup="/through_404_exception"
+async def through_404_exception(request): # $ requestHandler
+    raise web.HTTPNotFound(text="foo") # $ HttpResponse mimetype=text/plain responseBody="foo"
+
+
+@routes.get("/redirect_301") # $ routeSetup="/redirect_301"
+async def redirect_301(request): # $ requestHandler
+    if not "kwarg" in request.url.query:
+        raise web.HTTPMovedPermanently("/login") # $ HttpResponse HttpRedirectResponse mimetype=application/octet-stream redirectLocation="/login"
+    else:
+        raise web.HTTPMovedPermanently(location="/logout") # $ HttpResponse HttpRedirectResponse mimetype=application/octet-stream redirectLocation="/logout"
+
+
+@routes.get("/redirect_302") # $ routeSetup="/redirect_302"
+async def redirect_302(request): # $ requestHandler
+    if not "kwarg" in request.url.query:
+        raise web.HTTPFound("/login") # $ HttpResponse HttpRedirectResponse mimetype=application/octet-stream redirectLocation="/login"
+    else:
+        raise web.HTTPFound(location="/logout") # $ HttpResponse HttpRedirectResponse mimetype=application/octet-stream redirectLocation="/logout"
+
+
+if __name__ == "__main__":
+    app = web.Application()
+    app.add_routes(routes)
+    web.run_app(app)

python/ql/test/library-tests/frameworks/aiohttp/routing_test.py

@@ -0,0 +1,199 @@
+# Inspired by https://docs.aiohttp.org/en/stable/web_quickstart.html
+# and https://docs.aiohttp.org/en/stable/web_quickstart.html#resources-and-routes
+
+from aiohttp import web
+
+
+app = web.Application()
+
+
+## ================================= ##
+## Ways to specify routes / handlers ##
+## ================================= ##
+
+## Using coroutines
+if True:
+
+    # `app.add_routes` with list
+    async def foo(request):  # $ requestHandler
+        return web.Response(text="foo") # $ HttpResponse
+
+    async def foo2(request):  # $ requestHandler
+        return web.Response(text="foo2") # $ HttpResponse
+
+    async def foo3(request):  # $ requestHandler
+        return web.Response(text="foo3") # $ HttpResponse
+
+    app.add_routes([
+        web.get("/foo", foo),  # $ routeSetup="/foo"
+        web.route("*", "/foo2", foo2), # $ routeSetup="/foo2"
+        web.get(path="/foo3", handler=foo3), # $ routeSetup="/foo3"
+    ])
+
+
+    # using decorator
+    routes = web.RouteTableDef()
+
+    @routes.get("/bar")  # $ routeSetup="/bar"
+    async def bar(request):  # $ requestHandler
+        return web.Response(text="bar") # $ HttpResponse
+
+    @routes.route("*", "/bar2")  # $ routeSetup="/bar2"
+    async def bar2(request):  # $ requestHandler
+        return web.Response(text="bar2") # $ HttpResponse
+
+    @routes.get(path="/bar3")  # $ routeSetup="/bar3"
+    async def bar3(request):  # $ requestHandler
+        return web.Response(text="bar3") # $ HttpResponse
+
+    app.add_routes(routes)
+
+
+    # `app.router.add_get` / `app.router.add_route`
+    async def baz(request):  # $ requestHandler
+        return web.Response(text="baz") # $ HttpResponse
+
+    app.router.add_get("/baz", baz)  # $ routeSetup="/baz"
+
+    async def baz2(request):  # $ requestHandler
+        return web.Response(text="baz2") # $ HttpResponse
+
+    app.router.add_route("*", "/baz2", baz2)  # $ routeSetup="/baz2"
+
+    async def baz3(request):  # $ requestHandler
+        return web.Response(text="baz3") # $ HttpResponse
+
+    app.router.add_get(path="/baz3", handler=baz3)  # $ routeSetup="/baz3"
+
+
+## Using classes / views
+if True:
+    # see https://docs.aiohttp.org/en/stable/web_quickstart.html#organizing-handlers-in-classes
+
+    class MyCustomHandlerClass:
+
+        async def foo_handler(self, request):  # $ MISSING: requestHandler
+            return web.Response(text="MyCustomHandlerClass.foo") # $ HttpResponse
+
+    my_custom_handler = MyCustomHandlerClass()
+    app.router.add_get("/MyCustomHandlerClass/foo", my_custom_handler.foo_handler)   # $ routeSetup="/MyCustomHandlerClass/foo"
+
+    # Using `web.View`
+    # ---------------
+
+    # `app.add_routes` with list
+    class MyWebView1(web.View):
+        async def get(self):  # $ requestHandler
+            return web.Response(text="MyWebView1.get") # $ HttpResponse
+
+    app.add_routes([
+        web.view("/MyWebView1", MyWebView1)   # $ routeSetup="/MyWebView1"
+    ])
+
+
+    # using decorator
+    routes = web.RouteTableDef()
+
+    @routes.view("/MyWebView2")  # $ routeSetup="/MyWebView2"
+    class MyWebView2(web.View):
+        async def get(self):  # $ requestHandler
+            return web.Response(text="MyWebView2.get") # $ HttpResponse
+
+    app.add_routes(routes)
+
+
+    # `app.router.add_view`
+    class MyWebView3(web.View):
+        async def get(self):  # $ requestHandler
+            return web.Response(text="MyWebView3.get") # $ HttpResponse
+
+    app.router.add_view("/MyWebView3", MyWebView3)  # $ routeSetup="/MyWebView3"
+
+    # no route-setup
+    class MyWebViewNoRoute(web.View):
+        async def get(self):  # $ requestHandler
+            return web.Response(text="MyWebViewNoRoute.get") # $ HttpResponse
+
+    if len(__name__) < 0: # avoid running, but fool analysis to not consider dead code
+        # no explicit-view subclass (but route-setup)
+        class MyWebViewNoSubclassButRoute(somelib.someclass):
+            async def get(self):  # $ requestHandler
+                return web.Response(text="MyWebViewNoSubclassButRoute.get") # $ HttpResponse
+
+        app.router.add_view("/MyWebViewNoSubclassButRoute", MyWebViewNoSubclassButRoute)  # $ routeSetup="/MyWebViewNoSubclassButRoute"
+
+
+# Apparently there is no enforcement that `add_view` is only for views, and vice-versa
+# for `add_get` only being for async functions.
+if True:
+    async def no_rules(request): # $ requestHandler
+        return web.Response(text="no_rules") # $ HttpResponse
+
+    app.router.add_view("/no_rules", no_rules) # $ routeSetup="/no_rules"
+
+
+    class NoRulesView(web.View):
+        async def get(self):  # $ requestHandler
+            return web.Response(text="NoRulesView.get") # $ HttpResponse
+
+    app.router.add_get("/NoRulesView", NoRulesView) # $ routeSetup="/NoRulesView"
+
+
+## =================== ##
+## "Routed parameters" ##
+## =================== ##
+
+if True:
+    # see https://docs.aiohttp.org/en/stable/web_quickstart.html#variable-resources
+
+    async def matching(request: web.Request):  # $ requestHandler
+        name = request.match_info['name']
+        number = request.match_info['number']
+        return web.Response(text="matching name={} number={}".format(name, number)) # $ HttpResponse
+
+    app.router.add_get(r"/matching/{name}/{number:\d+}", matching)  # $ routeSetup="/matching/{name}/{number:\d+}"
+
+## ======= ##
+## subapps ##
+## ======= ##
+
+if True:
+    subapp = web.Application()
+
+    async def subapp_handler(request):  # $ requestHandler
+        return web.Response(text="subapp_handler") # $ HttpResponse
+
+    subapp.router.add_get("/subapp_handler", subapp_handler)  # $ routeSetup="/subapp_handler"
+
+    app.add_subapp("/my_subapp", subapp)
+
+    # similar behavior is possible with `app.add_domain`, but since I don't think we'll have special handling
+    # for any kind of subapps, I have not created a test for this.
+
+
+## ================================ ##
+## Constructing UrlDispatcher first ##
+## ================================ ##
+
+if True:
+    async def manual_dispatcher_instance(request):  # $ requestHandler
+        return web.Response(text="manual_dispatcher_instance") # $ HttpResponse
+
+    url_dispatcher = web.UrlDispatcher()
+    url_dispatcher.add_get("/manual_dispatcher_instance", manual_dispatcher_instance)  # $ routeSetup="/manual_dispatcher_instance"
+
+    subapp2 = web.Application(router=url_dispatcher)
+    app.add_subapp("/manual_dispatcher_instance_app", subapp2)
+
+
+## =========== ##
+## Run the app ##
+## =========== ##
+
+if __name__ == "__main__":
+    print("For auto-reloading server you can use:")
+    print(f"aiohttp-devtools runserver {__file__}")
+    print("after doing `pip install aiohttp-devtools`")
+    print()
+
+    web.run_app(app)

python/ql/test/library-tests/frameworks/aiohttp/taint_test.py

@@ -0,0 +1,152 @@
+from aiohttp import web
+
+async def test_taint(request: web.Request): # $ requestHandler
+
+    ensure_tainted(
+        # see https://docs.aiohttp.org/en/stable/web_reference.html#request-and-base-request
+        request, # $ tainted
+
+        # yarl.URL (see `yarl` framework tests)
+        request.url, # $ tainted
+        request.url.human_repr(), # $ tainted
+        request.rel_url, # $ tainted
+        request.rel_url.human_repr(), # $ tainted
+
+        request.forwarded, # $ tainted
+
+        request.host, # $ tainted
+        request.remote, # $ tainted
+        request.path, # $ tainted
+        request.path_qs, # $ tainted
+        request.raw_path, # $ tainted
+
+        # dict-like for captured parts of the URL
+        request.match_info, # $ tainted
+        request.match_info["key"], # $ tainted
+        request.match_info.get("key"), # $ tainted
+
+        # multidict.MultiDictProxy[str] (see `multidict` framework tests)
+        request.query, # $ tainted
+        request.query.getone("key"), # $ tainted
+
+        # multidict.CIMultiDictProxy[str] (see `multidict` framework tests)
+        request.headers, # $ tainted
+        request.headers.getone("key"), # $ tainted
+
+
+
+        # dict-like (readonly)
+        request.cookies, # $ tainted
+        request.cookies["key"], # $ tainted
+        request.cookies.get("key"), # $ tainted
+        request.cookies.keys(), # $ MISSING: tainted
+        request.cookies.values(), # $ tainted
+        request.cookies.items(), # $ tainted
+        list(request.cookies), # $ tainted
+        iter(request.cookies), # $ tainted
+
+
+        # aiohttp.StreamReader
+        # see https://docs.aiohttp.org/en/stable/streams.html#aiohttp.StreamReader
+        request.content, # $ tainted
+        await request.content.read(), # $ tainted
+        await request.content.readany(), # $ tainted
+        await request.content.readexactly(42), # $ tainted
+        await request.content.readline(), # $ tainted
+        await request.content.readchunk(), # $ tainted
+        (await request.content.readchunk())[0], # $ tainted
+        [line async for line in request.content], # $ MISSING: tainted
+        [data async for data in request.content.iter_chunked(1024)], # $ MISSING: tainted
+        [data async for data in request.content.iter_any()], # $ MISSING: tainted
+        [data async for data, _ in request.content.iter_chunks()], # $ MISSING: tainted
+        request.content.read_nowait(), # $ tainted
+
+        # aiohttp.StreamReader
+        request._payload, # $ tainted
+        await request._payload.readany(), # $ tainted
+
+        request.content_type, # $ tainted
+        request.charset, # $ tainted
+
+        request.http_range, # $ tainted
+
+        # Optional[datetime]
+        request.if_modified_since, # $ tainted
+        request.if_unmodified_since, # $ tainted
+        request.if_range, # $ tainted
+
+        request.clone(scheme="https"), # $ tainted
+
+        # asyncio.Transport
+        # https://docs.python.org/3/library/asyncio-protocol.html#asyncio-transport
+        # example given in https://docs.aiohttp.org/en/stable/web_reference.html#aiohttp.web.BaseRequest.transport
+        # uses `peername` to get IP address of client
+        request.transport, # $ tainted
+        request.transport.get_extra_info("key"), # $ MISSING: tainted
+
+        # Like request.transport.get_extra_info
+        request.get_extra_info("key"), # $ tainted
+
+        # Like request.transport.get_extra_info
+        request.protocol.transport.get_extra_info("key"), # $ MISSING: tainted
+
+        # bytes
+        await request.read(), # $ tainted
+
+        # str
+        await request.text(), # $ tainted
+
+        # obj
+        await request.json(), # $ tainted
+
+        # aiohttp.multipart.MultipartReader
+        await request.multipart(), # $ tainted
+
+        # multidict.MultiDictProxy[str] (see `multidict` framework tests)
+        await request.post(), # $ tainted
+        (await request.post()).getone("key"), # $ tainted
+    )
+
+    import yarl
+    assert isinstance(request.url, yarl.URL)
+    assert isinstance(request.rel_url, yarl.URL)
+
+
+    # things that are technically controlled by sender of request,
+    # but doesn't seem that likely for exploitation.
+    ensure_not_tainted(
+        request.method,
+        request.version,
+        request.scheme,
+        request.secure,
+        request.keep_alive,
+
+        request.content_length,
+        request.body_exists,
+        request.has_body,
+        request.can_read_body,
+    )
+
+    ensure_not_tainted(
+        request.loop,
+
+        request.app,
+        request.config_dict,
+    )
+
+
+class TaintTestClass(web.View):
+    def get(self): # $ requestHandler
+        ensure_tainted(
+            self.request, # $ tainted
+            self.request.url # $ tainted
+        )
+
+
+app = web.Application()
+app.router.add_get(r"/test_taint/{name}/{number:\d+}", test_taint)  # $ routeSetup="/test_taint/{name}/{number:\d+}"
+app.router.add_view(r"/test_taint_class", TaintTestClass)  # $ routeSetup="/test_taint_class"
+
+
+if __name__ == "__main__":
+    web.run_app(app)

python/ql/test/library-tests/frameworks/crypto/README.md

@@ -0,0 +1,13 @@
+These tests are a copy of the tests in [../cryptodome](../cryptodome) with `Cryptodome` replaced by `Crypto`.
+
+You can run the following command to update the tests:
+
+```sh
+rm *.py && cp ../cryptodome/*.py . && sed -i -e 's/Cryptodome/Crypto/' *.py
+```
+
+The original [`pycrypto` PyPI package](https://pypi.org/project/pycrypto/) that provided the `Crypto` Python package has not been updated since 2013, so it is reasonable to assume that people will use the replacement [`pycryptodome` PyPI package](https://pypi.org/project/pycryptodome/) that also provides a `Crypto` Python package and has a (mostly) compatible API.
+
+The pycryptodome functionality is also available in the [`pycryptodomex` PyPI package](https://pypi.org/project/pycryptodomex/) which provides the `Cryptodome` Python package.
+
+To ensure our modeling actually covers _both_ ways of importing the same functionality, we have this convoluted test setup.

python/ql/test/library-tests/frameworks/crypto/test_aes.py

@@ -0,0 +1,36 @@
+# https://pycryptodome.readthedocs.io/en/latest/src/cipher/aes.html
+from Crypto.Cipher import AES
+
+import os
+
+key = os.urandom(256//8)
+iv = os.urandom(16)
+
+# ------------------------------------------------------------------------------
+# encrypt/decrypt
+# ------------------------------------------------------------------------------
+
+
+print("encrypt/decrypt")
+
+secret_message = b"secret message"
+
+padding_len = 16 - (len(secret_message) % 16)
+padding = b"\0"*padding_len
+
+cipher = AES.new(key, AES.MODE_CBC, iv=iv)
+# using separate .encrypt calls on individual lines does not work
+whole_plantext = secret_message + padding
+encrypted = cipher.encrypt(whole_plantext) # $ CryptographicOperation CryptographicOperationAlgorithm=AES CryptographicOperationInput=whole_plantext
+
+print("encrypted={}".format(encrypted))
+
+print()
+
+cipher = AES.new(key, AES.MODE_CBC, iv=iv)
+decrypted = cipher.decrypt(encrypted) # $ CryptographicOperation CryptographicOperationAlgorithm=AES CryptographicOperationInput=encrypted
+
+decrypted = decrypted[:-padding_len]
+
+print("decrypted={}".format(decrypted))
+assert decrypted == secret_message

python/ql/test/library-tests/frameworks/crypto/test_dsa.py

@@ -20,8 +20,8 @@ message = b"message"
 
 signer = DSS.new(private_key, mode='fips-186-3')
 
-hasher = SHA256.new(message)
-signature = signer.sign(hasher)
+hasher = SHA256.new(message) # $ CryptographicOperation CryptographicOperationAlgorithm=SHA256 CryptographicOperationInput=message
+signature = signer.sign(hasher) # $ CryptographicOperation CryptographicOperationInput=hasher # MISSING: CryptographicOperationAlgorithm=DSA
 
 print("signature={}".format(signature))
 
@@ -29,13 +29,13 @@ print()
 
 verifier = DSS.new(public_key, mode='fips-186-3')
 
-hasher = SHA256.new(message)
-verifier.verify(hasher, signature)
+hasher = SHA256.new(message) # $ CryptographicOperation CryptographicOperationAlgorithm=SHA256 CryptographicOperationInput=message
+verifier.verify(hasher, signature) # $ CryptographicOperation CryptographicOperationInput=hasher CryptographicOperationInput=signature # MISSING: CryptographicOperationAlgorithm=DSA
 print("Signature verified (as expected)")
 
 try:
-    hasher = SHA256.new(b"other message")
-    verifier.verify(hasher, signature)
+    hasher = SHA256.new(b"other message") # $ CryptographicOperation CryptographicOperationAlgorithm=SHA256 CryptographicOperationInput=b"other message"
+    verifier.verify(hasher, signature) # $ CryptographicOperation CryptographicOperationInput=hasher CryptographicOperationInput=signature # MISSING: CryptographicOperationAlgorithm=DSA
     raise Exception("Signature verified (unexpected)")
 except ValueError:
     print("Signature mismatch (as expected)")

python/ql/test/library-tests/frameworks/crypto/test_ec.py

@@ -17,8 +17,8 @@ message = b"message"
 
 signer = DSS.new(private_key, mode='fips-186-3')
 
-hasher = SHA256.new(message)
-signature = signer.sign(hasher)
+hasher = SHA256.new(message) # $ CryptographicOperation CryptographicOperationAlgorithm=SHA256 CryptographicOperationInput=message
+signature = signer.sign(hasher) # $ CryptographicOperation CryptographicOperationInput=hasher # MISSING: CryptographicOperationAlgorithm=ECDSA
 
 print("signature={}".format(signature))
 
@@ -26,13 +26,13 @@ print()
 
 verifier = DSS.new(public_key, mode='fips-186-3')
 
-hasher = SHA256.new(message)
-verifier.verify(hasher, signature)
+hasher = SHA256.new(message) # $ CryptographicOperation CryptographicOperationAlgorithm=SHA256 CryptographicOperationInput=message
+verifier.verify(hasher, signature) # $ CryptographicOperation CryptographicOperationInput=hasher CryptographicOperationInput=signature
 print("Signature verified (as expected)")
 
 try:
-    hasher = SHA256.new(b"other message")
-    verifier.verify(hasher, signature)
+    hasher = SHA256.new(b"other message") # $ CryptographicOperation CryptographicOperationAlgorithm=SHA256 CryptographicOperationInput=b"other message"
+    verifier.verify(hasher, signature) # $ CryptographicOperation CryptographicOperationInput=hasher CryptographicOperationInput=signature # MISSING: CryptographicOperationAlgorithm=ECDSA
     raise Exception("Signature verified (unexpected)")
 except ValueError:
     print("Signature mismatch (as expected)")

python/ql/test/library-tests/frameworks/crypto/test_md5.py

@@ -0,0 +1,10 @@
+from Crypto.Hash import MD5
+
+hasher = MD5.new(b"secret message") # $ CryptographicOperation CryptographicOperationInput=b"secret message" CryptographicOperationAlgorithm=MD5
+print(hasher.hexdigest())
+
+
+hasher = MD5.new() # $ CryptographicOperation CryptographicOperationAlgorithm=MD5
+hasher.update(b"secret") # $ CryptographicOperation CryptographicOperationInput=b"secret" CryptographicOperationAlgorithm=MD5
+hasher.update(b" message") # $ CryptographicOperation CryptographicOperationInput=b" message" CryptographicOperationAlgorithm=MD5
+print(hasher.hexdigest())

python/ql/test/library-tests/frameworks/crypto/test_rc4.py

@@ -0,0 +1,30 @@
+# https://pycryptodome.readthedocs.io/en/latest/src/cipher/arc4.html
+from Crypto.Cipher import ARC4
+
+import os
+
+key = os.urandom(256//8)
+
+
+# ------------------------------------------------------------------------------
+# encrypt/decrypt
+# ------------------------------------------------------------------------------
+
+
+
+print("encrypt/decrypt")
+
+secret_message = b"secret message"
+
+cipher = ARC4.new(key)
+encrypted = cipher.encrypt(secret_message) # $ CryptographicOperation CryptographicOperationAlgorithm=ARC4 CryptographicOperationInput=secret_message
+
+print("encrypted={}".format(encrypted))
+
+print()
+
+cipher = ARC4.new(key)
+decrypted = cipher.decrypt(encrypted) # $ CryptographicOperation CryptographicOperationAlgorithm=ARC4 CryptographicOperationInput=encrypted
+
+print("decrypted={}".format(decrypted))
+assert decrypted == secret_message

python/ql/test/library-tests/frameworks/crypto/test_rsa.py

@@ -23,7 +23,7 @@ secret_message = b"secret message"
 
 encrypt_cipher = PKCS1_OAEP.new(public_key)
 
-encrypted = encrypt_cipher.encrypt(secret_message)
+encrypted = encrypt_cipher.encrypt(secret_message) # $ CryptographicOperation CryptographicOperationInput=secret_message # MISSING: CryptographicOperationAlgorithm=RSA-OAEP?
 
 print("encrypted={}".format(encrypted))
 
@@ -31,9 +31,7 @@ print()
 
 decrypt_cipher = PKCS1_OAEP.new(private_key)
 
-decrypted = decrypt_cipher.decrypt(
-    encrypted,
-)
+decrypted = decrypt_cipher.decrypt(encrypted) # $ CryptographicOperation CryptographicOperationInput=encrypted # MISSING: CryptographicOperationAlgorithm=RSA-OAEP?
 
 print("decrypted={}".format(decrypted))
 assert decrypted == secret_message
@@ -51,23 +49,23 @@ message = b"message"
 
 signer = pss.new(private_key)
 
-hasher = SHA256.new(message)
-signature = signer.sign(hasher)
+hasher = SHA256.new(message) # $ CryptographicOperation CryptographicOperationAlgorithm=SHA256 CryptographicOperationInput=message
+signature = signer.sign(hasher) # $ CryptographicOperation CryptographicOperationInput=hasher # MISSING: CryptographicOperationAlgorithm=RSA-PSS?
 
 print("signature={}".format(signature))
 
 print()
 
-
 verifier = pss.new(public_key)
-hasher = SHA256.new(message)
-verifier.verify(hasher, signature)
+
+hasher = SHA256.new(message) # $ CryptographicOperation CryptographicOperationAlgorithm=SHA256 CryptographicOperationInput=message
+verifier.verify(hasher, signature) # $ CryptographicOperation CryptographicOperationInput=hasher CryptographicOperationInput=signature # MISSING: CryptographicOperationAlgorithm=RSA-PSS?
 print("Signature verified (as expected)")
 
 try:
     verifier = pss.new(public_key)
-    hasher = SHA256.new(b"other message")
-    verifier.verify(hasher, signature)
+    hasher = SHA256.new(b"other message") # $ CryptographicOperation CryptographicOperationAlgorithm=SHA256 CryptographicOperationInput=b"other message"
+    verifier.verify(hasher, signature) # $ CryptographicOperation CryptographicOperationInput=hasher CryptographicOperationInput=signature # MISSING: CryptographicOperationAlgorithm=RSA-PSS?
     raise Exception("Signature verified (unexpected)")
 except ValueError:
     print("Signature mismatch (as expected)")

python/ql/test/library-tests/frameworks/cryptodome/test_aes.py

@@ -0,0 +1,36 @@
+# https://pycryptodome.readthedocs.io/en/latest/src/cipher/aes.html
+from Cryptodome.Cipher import AES
+
+import os
+
+key = os.urandom(256//8)
+iv = os.urandom(16)
+
+# ------------------------------------------------------------------------------
+# encrypt/decrypt
+# ------------------------------------------------------------------------------
+
+
+print("encrypt/decrypt")
+
+secret_message = b"secret message"
+
+padding_len = 16 - (len(secret_message) % 16)
+padding = b"\0"*padding_len
+
+cipher = AES.new(key, AES.MODE_CBC, iv=iv)
+# using separate .encrypt calls on individual lines does not work
+whole_plantext = secret_message + padding
+encrypted = cipher.encrypt(whole_plantext) # $ CryptographicOperation CryptographicOperationAlgorithm=AES CryptographicOperationInput=whole_plantext
+
+print("encrypted={}".format(encrypted))
+
+print()
+
+cipher = AES.new(key, AES.MODE_CBC, iv=iv)
+decrypted = cipher.decrypt(encrypted) # $ CryptographicOperation CryptographicOperationAlgorithm=AES CryptographicOperationInput=encrypted
+
+decrypted = decrypted[:-padding_len]
+
+print("decrypted={}".format(decrypted))
+assert decrypted == secret_message

python/ql/test/library-tests/frameworks/cryptodome/test_dsa.py

@@ -20,8 +20,8 @@ message = b"message"
 
 signer = DSS.new(private_key, mode='fips-186-3')
 
-hasher = SHA256.new(message)
-signature = signer.sign(hasher)
+hasher = SHA256.new(message) # $ CryptographicOperation CryptographicOperationAlgorithm=SHA256 CryptographicOperationInput=message
+signature = signer.sign(hasher) # $ CryptographicOperation CryptographicOperationInput=hasher # MISSING: CryptographicOperationAlgorithm=DSA
 
 print("signature={}".format(signature))
 
@@ -29,13 +29,13 @@ print()
 
 verifier = DSS.new(public_key, mode='fips-186-3')
 
-hasher = SHA256.new(message)
-verifier.verify(hasher, signature)
+hasher = SHA256.new(message) # $ CryptographicOperation CryptographicOperationAlgorithm=SHA256 CryptographicOperationInput=message
+verifier.verify(hasher, signature) # $ CryptographicOperation CryptographicOperationInput=hasher CryptographicOperationInput=signature # MISSING: CryptographicOperationAlgorithm=DSA
 print("Signature verified (as expected)")
 
 try:
-    hasher = SHA256.new(b"other message")
-    verifier.verify(hasher, signature)
+    hasher = SHA256.new(b"other message") # $ CryptographicOperation CryptographicOperationAlgorithm=SHA256 CryptographicOperationInput=b"other message"
+    verifier.verify(hasher, signature) # $ CryptographicOperation CryptographicOperationInput=hasher CryptographicOperationInput=signature # MISSING: CryptographicOperationAlgorithm=DSA
     raise Exception("Signature verified (unexpected)")
 except ValueError:
     print("Signature mismatch (as expected)")

python/ql/test/library-tests/frameworks/cryptodome/test_ec.py

@@ -17,8 +17,8 @@ message = b"message"
 
 signer = DSS.new(private_key, mode='fips-186-3')
 
-hasher = SHA256.new(message)
-signature = signer.sign(hasher)
+hasher = SHA256.new(message) # $ CryptographicOperation CryptographicOperationAlgorithm=SHA256 CryptographicOperationInput=message
+signature = signer.sign(hasher) # $ CryptographicOperation CryptographicOperationInput=hasher # MISSING: CryptographicOperationAlgorithm=ECDSA
 
 print("signature={}".format(signature))
 
@@ -26,13 +26,13 @@ print()
 
 verifier = DSS.new(public_key, mode='fips-186-3')
 
-hasher = SHA256.new(message)
-verifier.verify(hasher, signature)
+hasher = SHA256.new(message) # $ CryptographicOperation CryptographicOperationAlgorithm=SHA256 CryptographicOperationInput=message
+verifier.verify(hasher, signature) # $ CryptographicOperation CryptographicOperationInput=hasher CryptographicOperationInput=signature
 print("Signature verified (as expected)")
 
 try:
-    hasher = SHA256.new(b"other message")
-    verifier.verify(hasher, signature)
+    hasher = SHA256.new(b"other message") # $ CryptographicOperation CryptographicOperationAlgorithm=SHA256 CryptographicOperationInput=b"other message"
+    verifier.verify(hasher, signature) # $ CryptographicOperation CryptographicOperationInput=hasher CryptographicOperationInput=signature # MISSING: CryptographicOperationAlgorithm=ECDSA
     raise Exception("Signature verified (unexpected)")
 except ValueError:
     print("Signature mismatch (as expected)")

python/ql/test/library-tests/frameworks/cryptodome/test_md5.py

@@ -0,0 +1,10 @@
+from Cryptodome.Hash import MD5
+
+hasher = MD5.new(b"secret message") # $ CryptographicOperation CryptographicOperationInput=b"secret message" CryptographicOperationAlgorithm=MD5
+print(hasher.hexdigest())
+
+
+hasher = MD5.new() # $ CryptographicOperation CryptographicOperationAlgorithm=MD5
+hasher.update(b"secret") # $ CryptographicOperation CryptographicOperationInput=b"secret" CryptographicOperationAlgorithm=MD5
+hasher.update(b" message") # $ CryptographicOperation CryptographicOperationInput=b" message" CryptographicOperationAlgorithm=MD5
+print(hasher.hexdigest())

python/ql/test/library-tests/frameworks/cryptodome/test_rc4.py

@@ -0,0 +1,30 @@
+# https://pycryptodome.readthedocs.io/en/latest/src/cipher/arc4.html
+from Cryptodome.Cipher import ARC4
+
+import os
+
+key = os.urandom(256//8)
+
+
+# ------------------------------------------------------------------------------
+# encrypt/decrypt
+# ------------------------------------------------------------------------------
+
+
+
+print("encrypt/decrypt")
+
+secret_message = b"secret message"
+
+cipher = ARC4.new(key)
+encrypted = cipher.encrypt(secret_message) # $ CryptographicOperation CryptographicOperationAlgorithm=ARC4 CryptographicOperationInput=secret_message
+
+print("encrypted={}".format(encrypted))
+
+print()
+
+cipher = ARC4.new(key)
+decrypted = cipher.decrypt(encrypted) # $ CryptographicOperation CryptographicOperationAlgorithm=ARC4 CryptographicOperationInput=encrypted
+
+print("decrypted={}".format(decrypted))
+assert decrypted == secret_message

python/ql/test/library-tests/frameworks/cryptodome/test_rsa.py

@@ -23,7 +23,7 @@ secret_message = b"secret message"
 
 encrypt_cipher = PKCS1_OAEP.new(public_key)
 
-encrypted = encrypt_cipher.encrypt(secret_message)
+encrypted = encrypt_cipher.encrypt(secret_message) # $ CryptographicOperation CryptographicOperationInput=secret_message # MISSING: CryptographicOperationAlgorithm=RSA-OAEP?
 
 print("encrypted={}".format(encrypted))
 
@@ -31,9 +31,7 @@ print()
 
 decrypt_cipher = PKCS1_OAEP.new(private_key)
 
-decrypted = decrypt_cipher.decrypt(
-    encrypted,
-)
+decrypted = decrypt_cipher.decrypt(encrypted) # $ CryptographicOperation CryptographicOperationInput=encrypted # MISSING: CryptographicOperationAlgorithm=RSA-OAEP?
 
 print("decrypted={}".format(decrypted))
 assert decrypted == secret_message
@@ -51,8 +49,8 @@ message = b"message"
 
 signer = pss.new(private_key)
 
-hasher = SHA256.new(message)
-signature = signer.sign(hasher)
+hasher = SHA256.new(message) # $ CryptographicOperation CryptographicOperationAlgorithm=SHA256 CryptographicOperationInput=message
+signature = signer.sign(hasher) # $ CryptographicOperation CryptographicOperationInput=hasher # MISSING: CryptographicOperationAlgorithm=RSA-PSS?
 
 print("signature={}".format(signature))
 
@@ -60,14 +58,14 @@ print()
 
 verifier = pss.new(public_key)
 
-hasher = SHA256.new(message)
-verifier.verify(hasher, signature)
+hasher = SHA256.new(message) # $ CryptographicOperation CryptographicOperationAlgorithm=SHA256 CryptographicOperationInput=message
+verifier.verify(hasher, signature) # $ CryptographicOperation CryptographicOperationInput=hasher CryptographicOperationInput=signature # MISSING: CryptographicOperationAlgorithm=RSA-PSS?
 print("Signature verified (as expected)")
 
 try:
     verifier = pss.new(public_key)
-    hasher = SHA256.new(b"other message")
-    verifier.verify(hasher, signature)
+    hasher = SHA256.new(b"other message") # $ CryptographicOperation CryptographicOperationAlgorithm=SHA256 CryptographicOperationInput=b"other message"
+    verifier.verify(hasher, signature) # $ CryptographicOperation CryptographicOperationInput=hasher CryptographicOperationInput=signature # MISSING: CryptographicOperationAlgorithm=RSA-PSS?
     raise Exception("Signature verified (unexpected)")
 except ValueError:
     print("Signature mismatch (as expected)")

python/ql/test/library-tests/frameworks/cryptography/EcKeygenOrigin.expected

@@ -0,0 +1,5 @@
+| ec_keygen_origin.py:8:1:8:45 | ControlFlowNode for Attribute() | 384 | ec_keygen_origin.py:8:31:8:42 | ControlFlowNode for Attribute |
+| ec_keygen_origin.py:9:1:9:43 | ControlFlowNode for Attribute() | 384 | ec_keygen_origin.py:9:31:9:42 | ControlFlowNode for Attribute |
+| ec_keygen_origin.py:12:1:12:36 | ControlFlowNode for Attribute() | 384 | ec_keygen_origin.py:11:9:11:20 | ControlFlowNode for Attribute |
+| ec_keygen_origin.py:15:1:15:39 | ControlFlowNode for Attribute() | 384 | ec_keygen_origin.py:11:9:11:20 | ControlFlowNode for Attribute |
+| ec_keygen_origin.py:20:1:20:32 | ControlFlowNode for Attribute() | 384 | ec_keygen_origin.py:6:58:6:66 | ControlFlowNode for ImportMember |

python/ql/test/library-tests/frameworks/cryptography/EcKeygenOrigin.ql

@@ -0,0 +1,8 @@
+import semmle.python.dataflow.new.DataFlow
+import semmle.python.Concepts
+
+from Cryptography::PublicKey::KeyGeneration keyGen, int keySize, DataFlow::Node origin
+where
+  keyGen.getLocation().getFile().getShortName() = "ec_keygen_origin.py" and
+  keySize = keyGen.getKeySizeWithOrigin(origin)
+select keyGen, keySize, origin

python/ql/test/library-tests/frameworks/cryptography/ec_keygen_origin.py

@@ -0,0 +1,20 @@
+# Since key-size is not specified explicitly as an integer for the predefined
+# classes in the `cryptography.hazmat.primitives.asymmetric.ec` module, we need
+# special handling of the origin... this test is simply to show off how we handle this.
+
+from cryptography.hazmat.primitives.asymmetric import ec
+from cryptography.hazmat.primitives.asymmetric.ec import SECP384R1
+
+ec.generate_private_key(curve=ec.SECP384R1()) # $ PublicKeyGeneration keySize=384
+ec.generate_private_key(curve=ec.SECP384R1) # $ PublicKeyGeneration keySize=384
+
+alias = ec.SECP384R1
+ec.generate_private_key(curve=alias) # $ PublicKeyGeneration keySize=384
+
+instance = alias()
+ec.generate_private_key(curve=instance) # $ PublicKeyGeneration keySize=384
+
+
+x = SECP384R1
+y = x
+ec.generate_private_key(curve=y) # $ PublicKeyGeneration keySize=384

python/ql/test/library-tests/frameworks/cryptography/test_aes.py

@@ -0,0 +1,40 @@
+from cryptography.hazmat.primitives.ciphers import algorithms, Cipher, modes
+import os
+
+key = os.urandom(256//8)
+iv = os.urandom(16)
+
+algorithm = algorithms.AES(key)
+cipher = Cipher(algorithm, mode=modes.CBC(iv))
+
+# ------------------------------------------------------------------------------
+# encrypt/decrypt
+# ------------------------------------------------------------------------------
+
+# following https://cryptography.io/en/latest/hazmat/primitives/symmetric-encryption/#cryptography.hazmat.primitives.ciphers.Cipher
+
+print("encrypt/decrypt")
+
+secret_message = b"secret message"
+
+padding_len = 16 - (len(secret_message) % 16)
+padding = b"\0"*padding_len
+
+encryptor = cipher.encryptor()
+print(padding_len)
+encrypted = encryptor.update(secret_message) # $ CryptographicOperation CryptographicOperationAlgorithm=AES CryptographicOperationInput=secret_message
+encrypted += encryptor.update(padding) # $ CryptographicOperation CryptographicOperationAlgorithm=AES CryptographicOperationInput=padding
+encrypted += encryptor.finalize()
+
+print("encrypted={}".format(encrypted))
+
+print()
+
+decryptor = cipher.decryptor()
+decrypted = decryptor.update(encrypted) # $ CryptographicOperation CryptographicOperationAlgorithm=AES CryptographicOperationInput=encrypted
+decrypted += decryptor.finalize()
+
+decrypted = decrypted[:-padding_len]
+
+print("decrypted={}".format(decrypted))
+assert decrypted == secret_message

python/ql/test/library-tests/frameworks/cryptography/test_ec.py

@@ -6,6 +6,7 @@ from cryptography.exceptions import InvalidSignature
 
 
 private_key = ec.generate_private_key(curve=ec.SECP384R1()) # $ PublicKeyGeneration keySize=384
+private_key = ec.generate_private_key(curve=ec.SECP384R1) # $ PublicKeyGeneration keySize=384
 public_key = private_key.public_key()
 
 HASH_ALGORITHM = hashes.SHA256()

python/ql/test/library-tests/frameworks/cryptography/test_md5.py

@@ -0,0 +1,10 @@
+from cryptography.hazmat.primitives import hashes
+
+from binascii import hexlify
+
+
+hasher = hashes.Hash(hashes.MD5())
+hasher.update(b"secret message") # $ CryptographicOperation CryptographicOperationInput=b"secret message" CryptographicOperationAlgorithm=MD5
+
+digest = hasher.finalize()
+print(hexlify(digest).decode('utf-8'))

python/ql/test/library-tests/frameworks/cryptography/test_rc4.py

@@ -0,0 +1,32 @@
+from cryptography.hazmat.primitives.ciphers import algorithms, Cipher
+import os
+
+key = os.urandom(256//8)
+
+algorithm = algorithms.ARC4(key)
+cipher = Cipher(algorithm, mode=None)
+
+# ------------------------------------------------------------------------------
+# encrypt/decrypt
+# ------------------------------------------------------------------------------
+
+# following https://cryptography.io/en/latest/hazmat/primitives/symmetric-encryption.html#cryptography.hazmat.primitives.ciphers.algorithms.ARC4
+
+print("encrypt/decrypt")
+
+secret_message = b"secret message"
+
+encryptor = cipher.encryptor()
+encrypted = encryptor.update(secret_message) # $ CryptographicOperation CryptographicOperationAlgorithm=ARC4 CryptographicOperationInput=secret_message
+encrypted += encryptor.finalize()
+
+print("encrypted={}".format(encrypted))
+
+print()
+
+decryptor = cipher.decryptor()
+decrypted = decryptor.update(encrypted) # $ CryptographicOperation CryptographicOperationAlgorithm=ARC4 CryptographicOperationInput=encrypted
+decrypted += decryptor.finalize()
+
+print("decrypted={}".format(decrypted))
+assert decrypted == secret_message

python/ql/test/library-tests/frameworks/dill/Decoding.py

@@ -1,3 +1,3 @@
 import dill
 
-dill.loads(payload)  # $decodeInput=payload decodeOutput=Attribute() decodeFormat=dill decodeMayExecuteInput
+dill.loads(payload)  # $decodeInput=payload decodeOutput=dill.loads(..) decodeFormat=dill decodeMayExecuteInput

python/ql/test/library-tests/frameworks/django-v2-v3/response_test.py

@@ -74,20 +74,20 @@ class CustomRedirectView(RedirectView):
 
 # Ensure that simple subclasses are still vuln to XSS
 def xss__not_found(request):
-    return HttpResponseNotFound(request.GET.get("name"))  # $HttpResponse mimetype=text/html responseBody=Attribute()
+    return HttpResponseNotFound(request.GET.get("name"))  # $HttpResponse mimetype=text/html responseBody=request.GET.get(..)
 
 # Ensure we still have an XSS sink when manually setting the content_type to HTML
 def xss__manual_response_type(request):
-    return HttpResponse(request.GET.get("name"), content_type="text/html; charset=utf-8")  # $HttpResponse mimetype=text/html responseBody=Attribute()
+    return HttpResponse(request.GET.get("name"), content_type="text/html; charset=utf-8")  # $HttpResponse mimetype=text/html responseBody=request.GET.get(..)
 
 def xss__write(request):
     response = HttpResponse()  # $HttpResponse mimetype=text/html
-    response.write(request.GET.get("name"))  # $HttpResponse mimetype=text/html responseBody=Attribute()
+    response.write(request.GET.get("name"))  # $HttpResponse mimetype=text/html responseBody=request.GET.get(..)
 
 # This is safe but probably a bug if the argument to `write` is not a result of `json.dumps` or similar.
 def safe__write_json(request):
     response = JsonResponse()  # $HttpResponse mimetype=application/json
-    response.write(request.GET.get("name"))  # $HttpResponse mimetype=application/json responseBody=Attribute()
+    response.write(request.GET.get("name"))  # $HttpResponse mimetype=application/json responseBody=request.GET.get(..)
 
 # Ensure manual subclasses are vulnerable
 class CustomResponse(HttpResponse):
@@ -95,7 +95,7 @@ class CustomResponse(HttpResponse):
         super().__init__(content, *args, content_type="text/html", **kwargs)
 
 def xss__custom_response(request):
-    return CustomResponse("ACME Responses", request.GET("name"))  # $HttpResponse MISSING: mimetype=text/html responseBody=Attribute() SPURIOUS: responseBody="ACME Responses"
+    return CustomResponse("ACME Responses", request.GET("name"))  # $HttpResponse MISSING: mimetype=text/html responseBody=request.GET.get(..) SPURIOUS: responseBody="ACME Responses"
 
 class CustomJsonResponse(JsonResponse):
     def __init__(self, banner, content, *args, **kwargs):

python/ql/test/library-tests/frameworks/django-v2-v3/testproj/settings.py

@@ -13,7 +13,7 @@ https://docs.djangoproject.com/en/3.1/ref/settings/
 from pathlib import Path
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
-BASE_DIR = Path(__file__).resolve().parent.parent  #$ getAPathArgument=Path()
+BASE_DIR = Path(__file__).resolve().parent.parent  #$ getAPathArgument=Path(..)
 
 
 # Quick-start development settings - unsuitable for production

python/ql/test/library-tests/frameworks/idna/ConceptsTest.ql

@@ -0,0 +1,2 @@
+import python
+import experimental.meta.ConceptsTest

python/ql/test/library-tests/frameworks/idna/InlineTaintTest.expected

@@ -0,0 +1,3 @@
+argumentToEnsureNotTaintedNotMarkedAsSpurious
+untaintedArgumentToEnsureTaintedNotMarkedAsMissing
+failures

python/ql/test/library-tests/frameworks/idna/InlineTaintTest.ql

@@ -0,0 +1 @@
+import experimental.meta.InlineTaintTest

python/ql/test/library-tests/frameworks/idna/taint_test.py

@@ -0,0 +1,13 @@
+import idna
+
+def test_idna():
+    ts = TAINTED_STRING
+    tb = TAINTED_BYTES
+
+    ensure_tainted(
+        idna.encode(ts),  # $ tainted encodeInput=ts encodeOutput=idna.encode(..) encodeFormat=IDNA
+        idna.encode(s=ts),  # $ tainted encodeInput=ts encodeOutput=idna.encode(..) encodeFormat=IDNA
+
+        idna.decode(tb),  # $ tainted decodeInput=tb decodeOutput=idna.decode(..) decodeFormat=IDNA
+        idna.decode(s=tb),  # $ tainted decodeInput=tb decodeOutput=idna.decode(..) decodeFormat=IDNA
+    )

python/ql/test/library-tests/frameworks/multidict/ConceptsTest.ql

@@ -0,0 +1,2 @@
+import python
+import experimental.meta.ConceptsTest

python/ql/test/library-tests/frameworks/multidict/InlineTaintTest.expected

@@ -0,0 +1,3 @@
+argumentToEnsureNotTaintedNotMarkedAsSpurious
+untaintedArgumentToEnsureTaintedNotMarkedAsMissing
+failures

python/ql/test/library-tests/frameworks/multidict/InlineTaintTest.ql

@@ -0,0 +1 @@
+import experimental.meta.InlineTaintTest

python/ql/test/library-tests/frameworks/multidict/options

@@ -0,0 +1 @@
+semmle-extractor-options: --max-import-depth=1 --lang=3

python/ql/test/library-tests/frameworks/multidict/taint_test.py

@@ -0,0 +1,41 @@
+import multidict
+
+# TODO: This is an invalid MultiDictProxy construction... but for the purpose of
+# taint-test, this should be good enough.
+mdp = multidict.MultiDictProxy(TAINTED_STRING)
+
+ensure_tainted(
+    # see https://multidict.readthedocs.io/en/stable/multidict.html#multidict.MultiDictProxy
+
+    mdp, # $ tainted
+    mdp["key"], # $ tainted
+    mdp.get("key"), # $ tainted
+    mdp.getone("key"), # $ tainted
+    mdp.getall("key"), # $ tainted
+    mdp.keys(), # $ MISSING: tainted
+    mdp.values(), # $ tainted
+    mdp.items(), # $ tainted
+    mdp.copy(), # $ tainted
+    list(mdp), # $ tainted
+    iter(mdp), # $ tainted
+)
+
+# TODO: This is an invalid CIMultiDictProxy construction... but for the purpose of
+# taint-test, this should be good enough.
+ci_mdp = multidict.CIMultiDictProxy(TAINTED_STRING)
+
+ensure_tainted(
+    # see https://multidict.readthedocs.io/en/stable/multidict.html#multidict.CIMultiDictProxy
+
+    ci_mdp, # $ tainted
+    ci_mdp["key"], # $ tainted
+    ci_mdp.get("key"), # $ tainted
+    ci_mdp.getone("key"), # $ tainted
+    ci_mdp.getall("key"), # $ tainted
+    ci_mdp.keys(), # $ MISSING: tainted
+    ci_mdp.values(), # $ tainted
+    ci_mdp.items(), # $ tainted
+    ci_mdp.copy(), # $ tainted
+    list(ci_mdp), # $ tainted
+    iter(ci_mdp), # $ tainted
+)

python/ql/test/library-tests/frameworks/simplejson/ConceptsTest.ql

@@ -0,0 +1,2 @@
+import python
+import experimental.meta.ConceptsTest

python/ql/test/library-tests/frameworks/simplejson/InlineTaintTest.expected

@@ -0,0 +1,3 @@
+argumentToEnsureNotTaintedNotMarkedAsSpurious
+untaintedArgumentToEnsureTaintedNotMarkedAsMissing
+failures

python/ql/test/library-tests/frameworks/simplejson/InlineTaintTest.ql

@@ -0,0 +1 @@
+import experimental.meta.InlineTaintTest

python/ql/test/library-tests/frameworks/simplejson/taint_test.py

@@ -0,0 +1,46 @@
+import simplejson
+from io import StringIO
+
+def test():
+    ts = TAINTED_STRING
+    tainted_obj = {"foo": ts}
+
+    encoded = simplejson.dumps(tainted_obj) # $ encodeOutput=simplejson.dumps(..) encodeFormat=JSON encodeInput=tainted_obj
+
+    ensure_tainted(
+        encoded, # $ tainted
+        simplejson.dumps(tainted_obj), # $ tainted encodeOutput=simplejson.dumps(..) encodeFormat=JSON encodeInput=tainted_obj
+        simplejson.dumps(obj=tainted_obj), # $ tainted encodeOutput=simplejson.dumps(..) encodeFormat=JSON encodeInput=tainted_obj
+        simplejson.loads(encoded), # $ tainted decodeOutput=simplejson.loads(..) decodeFormat=JSON decodeInput=encoded
+        simplejson.loads(s=encoded), # $ tainted decodeOutput=simplejson.loads(..) decodeFormat=JSON decodeInput=encoded
+    )
+
+    # load/dump with file-like
+    tainted_filelike = StringIO()
+    simplejson.dump(tainted_obj, tainted_filelike) # $ encodeFormat=JSON encodeInput=tainted_obj
+
+    tainted_filelike.seek(0)
+    ensure_tainted(
+        tainted_filelike, # $ MISSING: tainted
+        simplejson.load(tainted_filelike), # $ decodeOutput=simplejson.load(..) decodeFormat=JSON decodeInput=tainted_filelike MISSING: tainted
+    )
+
+    # load/dump with file-like using keyword-args
+    tainted_filelike = StringIO()
+    simplejson.dump(obj=tainted_obj, fp=tainted_filelike) # $ encodeFormat=JSON encodeInput=tainted_obj
+
+    tainted_filelike.seek(0)
+    ensure_tainted(
+        tainted_filelike, # $ MISSING: tainted
+        simplejson.load(fp=tainted_filelike), # $ decodeOutput=simplejson.load(..) decodeFormat=JSON decodeInput=tainted_filelike MISSING: tainted
+    )
+
+# To make things runable
+
+TAINTED_STRING = "TAINTED_STRING"
+def ensure_tainted(*args):
+    print("- ensure_tainted")
+    for i, arg in enumerate(args):
+        print("arg {}: {!r}".format(i, arg))
+
+test()

python/ql/test/library-tests/frameworks/stdlib-py3/Decoding.py

@@ -1,6 +1,6 @@
 import base64
 
 # TODO: These tests should be merged with python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep-py3/test_string.py
-base64.a85decode(payload)  # $ decodeInput=payload decodeOutput=Attribute() decodeFormat=Ascii85
-base64.b85decode(payload)  # $ decodeInput=payload decodeOutput=Attribute() decodeFormat=Base85
-base64.decodebytes(payload)  # $ decodeInput=payload decodeOutput=Attribute() decodeFormat=Base64
+base64.a85decode(payload)  # $ decodeInput=payload decodeOutput=base64.a85decode(..) decodeFormat=Ascii85
+base64.b85decode(payload)  # $ decodeInput=payload decodeOutput=base64.b85decode(..) decodeFormat=Base85
+base64.decodebytes(payload)  # $ decodeInput=payload decodeOutput=base64.decodebytes(..) decodeFormat=Base64

python/ql/test/library-tests/frameworks/stdlib-py3/Encoding.py

@@ -1,6 +1,6 @@
 import base64
 
 # TODO: These tests should be merged with python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep-py3/test_string.py
-base64.a85encode(bs) # $ encodeInput=bs encodeOutput=Attribute() encodeFormat=Ascii85
-base64.b85encode(bs)# $ encodeInput=bs encodeOutput=Attribute() encodeFormat=Base85
-base64.encodebytes(bs)# $ encodeInput=bs encodeOutput=Attribute() encodeFormat=Base64
+base64.a85encode(bs) # $ encodeInput=bs encodeOutput=base64.a85encode(..) encodeFormat=Ascii85
+base64.b85encode(bs)# $ encodeInput=bs encodeOutput=base64.b85encode(..) encodeFormat=Base85
+base64.encodebytes(bs)# $ encodeInput=bs encodeOutput=base64.encodebytes(..) encodeFormat=Base64

python/ql/test/library-tests/frameworks/stdlib/Decoding.py

@@ -2,14 +2,14 @@ import pickle
 import marshal
 import base64
 
-pickle.loads(payload)  # $ decodeInput=payload decodeOutput=Attribute() decodeFormat=pickle decodeMayExecuteInput
-marshal.loads(payload)  # $ decodeInput=payload decodeOutput=Attribute() decodeFormat=marshal decodeMayExecuteInput
+pickle.loads(payload)  # $ decodeInput=payload decodeOutput=pickle.loads(..) decodeFormat=pickle decodeMayExecuteInput
+marshal.loads(payload)  # $ decodeInput=payload decodeOutput=marshal.loads(..) decodeFormat=marshal decodeMayExecuteInput
 
 # TODO: These tests should be merged with python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep/test_string.py
-base64.b64decode(payload)  # $ decodeInput=payload decodeOutput=Attribute() decodeFormat=Base64
-base64.standard_b64decode(payload)  # $ decodeInput=payload decodeOutput=Attribute() decodeFormat=Base64
-base64.urlsafe_b64decode(payload)  # $ decodeInput=payload decodeOutput=Attribute() decodeFormat=Base64
-base64.b32decode(payload)  # $ decodeInput=payload decodeOutput=Attribute() decodeFormat=Base32
-base64.b16decode(payload)  # $ decodeInput=payload decodeOutput=Attribute() decodeFormat=Base16
+base64.b64decode(payload)  # $ decodeInput=payload decodeOutput=base64.b64decode(..) decodeFormat=Base64
+base64.standard_b64decode(payload)  # $ decodeInput=payload decodeOutput=base64.standard_b64decode(..) decodeFormat=Base64
+base64.urlsafe_b64decode(payload)  # $ decodeInput=payload decodeOutput=base64.urlsafe_b64decode(..) decodeFormat=Base64
+base64.b32decode(payload)  # $ decodeInput=payload decodeOutput=base64.b32decode(..) decodeFormat=Base32
+base64.b16decode(payload)  # $ decodeInput=payload decodeOutput=base64.b16decode(..) decodeFormat=Base16
 # deprecated since Python 3.1, but still works
-base64.decodestring(payload)  # $ decodeInput=payload decodeOutput=Attribute() decodeFormat=Base64
+base64.decodestring(payload)  # $ decodeInput=payload decodeOutput=base64.decodestring(..) decodeFormat=Base64

python/ql/test/library-tests/frameworks/stdlib/Encoding.py

@@ -2,14 +2,14 @@ import pickle
 import marshal
 import base64
 
-pickle.dumps(obj)  # $ MISSING: f-:encodeInput=obj f-:encodeOutput=Attribute() f-:encodeFormat=pickle f-:encodeMayExecuteInput
-marshal.dumps(obj)  # $ MISSING: f-:encodeInput=obj f-:encodeOutput=Attribute() f-:encodeFormat=marshal f-:encodeMayExecuteInput
+pickle.dumps(obj)  # $ MISSING: encodeInput=obj encodeOutput=pickle.dumps(..) encodeFormat=pickle encodeMayExecuteInput
+marshal.dumps(obj)  # $ MISSING: encodeInput=obj encodeOutput=marshal.dumps(..) encodeFormat=marshal encodeMayExecuteInput
 
 # TODO: These tests should be merged with python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep/test_string.py
-base64.b64encode(bs)  # $ encodeInput=bs encodeOutput=Attribute() encodeFormat=Base64
-base64.standard_b64encode(bs)  # $ encodeInput=bs encodeOutput=Attribute() encodeFormat=Base64
-base64.urlsafe_b64encode(bs)  # $ encodeInput=bs encodeOutput=Attribute() encodeFormat=Base64
-base64.b32encode(bs)  # $ encodeInput=bs encodeOutput=Attribute() encodeFormat=Base32
-base64.b16encode(bs)  # $ encodeInput=bs encodeOutput=Attribute() encodeFormat=Base16
+base64.b64encode(bs)  # $ encodeInput=bs encodeOutput=base64.b64encode(..) encodeFormat=Base64
+base64.standard_b64encode(bs)  # $ encodeInput=bs encodeOutput=base64.standard_b64encode(..) encodeFormat=Base64
+base64.urlsafe_b64encode(bs)  # $ encodeInput=bs encodeOutput=base64.urlsafe_b64encode(..) encodeFormat=Base64
+base64.b32encode(bs)  # $ encodeInput=bs encodeOutput=base64.b32encode(..) encodeFormat=Base32
+base64.b16encode(bs)  # $ encodeInput=bs encodeOutput=base64.b16encode(..) encodeFormat=Base16
 # deprecated since Python 3.1, but still works
-base64.encodestring(bs)  # $ encodeInput=bs encodeOutput=Attribute() encodeFormat=Base64
+base64.encodestring(bs)  # $ encodeInput=bs encodeOutput=base64.encodestring(..) encodeFormat=Base64

python/ql/test/library-tests/frameworks/stdlib/test_json.py

@@ -0,0 +1,40 @@
+from io import StringIO
+import json
+
+def test():
+    print("\n# test")
+    ts = TAINTED_STRING
+
+    encoded = json.dumps(ts) # $ encodeOutput=json.dumps(..) encodeFormat=JSON encodeInput=ts
+
+    ensure_tainted(
+        encoded, # $ tainted
+        json.dumps(ts), # $ tainted encodeOutput=json.dumps(..) encodeFormat=JSON encodeInput=ts
+        json.dumps(obj=ts), # $ tainted encodeOutput=json.dumps(..) encodeFormat=JSON encodeInput=ts
+        json.loads(encoded), # $ tainted decodeOutput=json.loads(..) decodeFormat=JSON decodeInput=encoded
+        json.loads(s=encoded), # $ tainted decodeOutput=json.loads(..) decodeFormat=JSON decodeInput=encoded
+    )
+
+    # load/dump with file-like
+    tainted_filelike = StringIO()
+    json.dump(ts, tainted_filelike) # $ encodeOutput=[post]tainted_filelike encodeFormat=JSON encodeInput=ts
+
+    tainted_filelike.seek(0)
+    ensure_tainted(
+        tainted_filelike, # $ tainted
+        json.load(tainted_filelike), # $ tainted decodeOutput=json.load(..) decodeFormat=JSON decodeInput=tainted_filelike
+    )
+
+    # load/dump with file-like using keyword-args
+    tainted_filelike = StringIO()
+    json.dump(obj=ts, fp=tainted_filelike) # $ encodeOutput=[post]tainted_filelike encodeFormat=JSON encodeInput=ts
+
+    tainted_filelike.seek(0)
+    ensure_tainted(
+        tainted_filelike, # $ tainted
+        json.load(fp=tainted_filelike), # $ tainted decodeOutput=json.load(..) decodeFormat=JSON decodeInput=tainted_filelike
+    )
+
+
+# Make tests runable
+test()

python/ql/test/library-tests/frameworks/stdlib/test_md5.py

@@ -0,0 +1,29 @@
+import hashlib
+
+
+hasher = hashlib.md5(b"secret message") # $ CryptographicOperation CryptographicOperationInput=b"secret message" CryptographicOperationAlgorithm=MD5
+print(hasher.hexdigest())
+
+
+hasher = hashlib.md5(string=b"secret message") # $ CryptographicOperation CryptographicOperationInput=b"secret message" CryptographicOperationAlgorithm=MD5
+print(hasher.hexdigest())
+
+
+hasher = hashlib.md5()
+hasher.update(b"secret") # $ CryptographicOperation CryptographicOperationInput=b"secret" CryptographicOperationAlgorithm=MD5
+hasher.update(b" message") # $ CryptographicOperation CryptographicOperationInput=b" message" CryptographicOperationAlgorithm=MD5
+print(hasher.hexdigest())
+
+
+hasher = hashlib.new('md5', b"secret message") # $ CryptographicOperation CryptographicOperationInput=b"secret message" CryptographicOperationAlgorithm=MD5
+print(hasher.hexdigest())
+
+
+hasher = hashlib.new('md5', data=b"secret message") # $ CryptographicOperation CryptographicOperationInput=b"secret message" CryptographicOperationAlgorithm=MD5
+print(hasher.hexdigest())
+
+
+hasher = hashlib.new('md5')
+hasher.update(b"secret") # $ CryptographicOperation CryptographicOperationInput=b"secret" CryptographicOperationAlgorithm=MD5
+hasher.update(b" message") # $ CryptographicOperation CryptographicOperationInput=b" message" CryptographicOperationAlgorithm=MD5
+print(hasher.hexdigest())

python/ql/test/library-tests/frameworks/ujson/ConceptsTest.ql

@@ -0,0 +1,2 @@
+import python
+import experimental.meta.ConceptsTest

python/ql/test/library-tests/frameworks/ujson/InlineTaintTest.expected

@@ -0,0 +1,3 @@
+argumentToEnsureNotTaintedNotMarkedAsSpurious
+untaintedArgumentToEnsureTaintedNotMarkedAsMissing
+failures

python/ql/test/library-tests/frameworks/ujson/InlineTaintTest.ql

@@ -0,0 +1 @@
+import experimental.meta.InlineTaintTest

python/ql/test/library-tests/frameworks/ujson/taint_test.py

@@ -0,0 +1,44 @@
+import ujson
+from io import StringIO
+
+def test():
+    ts = TAINTED_STRING
+    tainted_obj = {"foo": ts}
+
+    encoded = ujson.dumps(tainted_obj) # $ encodeOutput=ujson.dumps(..) encodeFormat=JSON encodeInput=tainted_obj
+
+    ensure_tainted(
+        encoded, # $ tainted
+        ujson.dumps(tainted_obj), # $ tainted encodeOutput=ujson.dumps(..) encodeFormat=JSON encodeInput=tainted_obj
+        ujson.dumps(obj=tainted_obj), # $ tainted encodeOutput=ujson.dumps(..) encodeFormat=JSON encodeInput=tainted_obj
+        ujson.loads(encoded), # $ tainted decodeOutput=ujson.loads(..) decodeFormat=JSON decodeInput=encoded
+        ujson.loads(obj=encoded), # $ tainted decodeOutput=ujson.loads(..) decodeFormat=JSON decodeInput=encoded
+
+        ujson.encode(tainted_obj), # $ tainted encodeOutput=ujson.encode(..) encodeFormat=JSON encodeInput=tainted_obj
+        ujson.encode(obj=tainted_obj), # $ tainted encodeOutput=ujson.encode(..) encodeFormat=JSON encodeInput=tainted_obj
+        ujson.decode(encoded), # $ tainted decodeOutput=ujson.decode(..) decodeFormat=JSON decodeInput=encoded
+        ujson.decode(obj=encoded), # $ tainted decodeOutput=ujson.decode(..) decodeFormat=JSON decodeInput=encoded
+    )
+
+    # load/dump with file-like
+    tainted_filelike = StringIO()
+    ujson.dump(tainted_obj, tainted_filelike) # $ encodeFormat=JSON encodeInput=tainted_obj
+
+    tainted_filelike.seek(0)
+    ensure_tainted(
+        tainted_filelike, # $ MISSING: tainted
+        ujson.load(tainted_filelike), # $ decodeOutput=ujson.load(..) decodeFormat=JSON decodeInput=tainted_filelike MISSING: tainted
+    )
+
+    # load/dump with file-like using keyword-args does not work in `ujson`
+
+
+# To make things runable
+
+TAINTED_STRING = "TAINTED_STRING"
+def ensure_tainted(*args):
+    print("- ensure_tainted")
+    for i, arg in enumerate(args):
+        print("arg {}: {!r}".format(i, arg))
+
+test()

python/ql/test/library-tests/frameworks/yaml/Decoding.py

@@ -1,37 +1,37 @@
 import yaml
 
 # Unsafe:
-yaml.load(payload)  # $decodeInput=payload decodeOutput=Attribute() decodeFormat=YAML decodeMayExecuteInput
-yaml.load(payload, yaml.Loader)  # $decodeInput=payload decodeOutput=Attribute() decodeFormat=YAML decodeMayExecuteInput
-yaml.unsafe_load(payload) # $ decodeInput=payload decodeOutput=Attribute() decodeFormat=YAML decodeMayExecuteInput
-yaml.full_load(payload) # $ decodeInput=payload decodeOutput=Attribute() decodeFormat=YAML decodeMayExecuteInput
+yaml.load(payload)  # $decodeInput=payload decodeOutput=yaml.load(..) decodeFormat=YAML decodeMayExecuteInput
+yaml.load(payload, yaml.Loader)  # $decodeInput=payload decodeOutput=yaml.load(..) decodeFormat=YAML decodeMayExecuteInput
+yaml.unsafe_load(payload) # $ decodeInput=payload decodeOutput=yaml.unsafe_load(..) decodeFormat=YAML decodeMayExecuteInput
+yaml.full_load(payload) # $ decodeInput=payload decodeOutput=yaml.full_load(..) decodeFormat=YAML decodeMayExecuteInput
 
 # Safe:
-yaml.load(payload, yaml.SafeLoader)  # $ decodeInput=payload decodeOutput=Attribute() decodeFormat=YAML
-yaml.load(payload, Loader=yaml.SafeLoader)  # $decodeInput=payload decodeOutput=Attribute() decodeFormat=YAML
-yaml.load(payload, yaml.BaseLoader)  # $decodeInput=payload decodeOutput=Attribute() decodeFormat=YAML
-yaml.safe_load(payload) # $ decodeInput=payload decodeOutput=Attribute() decodeFormat=YAML
+yaml.load(payload, yaml.SafeLoader)  # $ decodeInput=payload decodeOutput=yaml.load(..) decodeFormat=YAML
+yaml.load(payload, Loader=yaml.SafeLoader)  # $decodeInput=payload decodeOutput=yaml.load(..) decodeFormat=YAML
+yaml.load(payload, yaml.BaseLoader)  # $decodeInput=payload decodeOutput=yaml.load(..) decodeFormat=YAML
+yaml.safe_load(payload) # $ decodeInput=payload decodeOutput=yaml.safe_load(..) decodeFormat=YAML
 
 ################################################################################
 # load_all variants
 ################################################################################
 
 # Unsafe:
-yaml.load_all(payload) # $ decodeInput=payload decodeOutput=Attribute() decodeFormat=YAML decodeMayExecuteInput
-yaml.unsafe_load_all(payload) # $ decodeInput=payload decodeOutput=Attribute() decodeFormat=YAML decodeMayExecuteInput
-yaml.full_load_all(payload) # $ decodeInput=payload decodeOutput=Attribute() decodeFormat=YAML decodeMayExecuteInput
+yaml.load_all(payload) # $ decodeInput=payload decodeOutput=yaml.load_all(..) decodeFormat=YAML decodeMayExecuteInput
+yaml.unsafe_load_all(payload) # $ decodeInput=payload decodeOutput=yaml.unsafe_load_all(..) decodeFormat=YAML decodeMayExecuteInput
+yaml.full_load_all(payload) # $ decodeInput=payload decodeOutput=yaml.full_load_all(..) decodeFormat=YAML decodeMayExecuteInput
 
 # Safe:
-yaml.safe_load_all(payload) # $ decodeInput=payload decodeOutput=Attribute() decodeFormat=YAML
+yaml.safe_load_all(payload) # $ decodeInput=payload decodeOutput=yaml.safe_load_all(..) decodeFormat=YAML
 
 ################################################################################
 # C-based loaders with `libyaml`
 ################################################################################
 
 # Unsafe:
-yaml.load(payload, yaml.CLoader)  # $ decodeInput=payload decodeOutput=Attribute() decodeFormat=YAML decodeMayExecuteInput
-yaml.load(payload, yaml.CFullLoader)  # $ decodeInput=payload decodeOutput=Attribute() decodeFormat=YAML decodeMayExecuteInput
+yaml.load(payload, yaml.CLoader)  # $ decodeInput=payload decodeOutput=yaml.load(..) decodeFormat=YAML decodeMayExecuteInput
+yaml.load(payload, yaml.CFullLoader)  # $ decodeInput=payload decodeOutput=yaml.load(..) decodeFormat=YAML decodeMayExecuteInput
 
 # Safe:
-yaml.load(payload, yaml.CSafeLoader)  # $decodeInput=payload decodeOutput=Attribute() decodeFormat=YAML
-yaml.load(payload, yaml.CBaseLoader)  # $decodeInput=payload decodeOutput=Attribute() decodeFormat=YAML
+yaml.load(payload, yaml.CSafeLoader)  # $decodeInput=payload decodeOutput=yaml.load(..) decodeFormat=YAML
+yaml.load(payload, yaml.CBaseLoader)  # $decodeInput=payload decodeOutput=yaml.load(..) decodeFormat=YAML

python/ql/test/library-tests/frameworks/yarl/ConceptsTest.ql

@@ -0,0 +1,2 @@
+import python
+import experimental.meta.ConceptsTest

python/ql/test/library-tests/frameworks/yarl/InlineTaintTest.expected

@@ -0,0 +1,3 @@
+argumentToEnsureNotTaintedNotMarkedAsSpurious
+untaintedArgumentToEnsureTaintedNotMarkedAsMissing
+failures

python/ql/test/library-tests/frameworks/yarl/InlineTaintTest.ql

@@ -0,0 +1 @@
+import experimental.meta.InlineTaintTest

python/ql/test/library-tests/frameworks/yarl/options

@@ -0,0 +1 @@
+semmle-extractor-options: --max-import-depth=1 --lang=3

python/ql/test/library-tests/frameworks/yarl/taint_test.py

@@ -0,0 +1,64 @@
+import yarl
+
+
+url = yarl.URL(TAINTED_STRING)
+
+
+ensure_tainted(
+    url, # $ tainted
+
+    # see https://yarl.readthedocs.io/en/stable/api.html#yarl.URL
+    url.user, # $ tainted
+    url.raw_user, # $ tainted
+
+    url.password, # $ tainted
+    url.raw_password, # $ tainted
+
+    url.host, # $ tainted
+    url.raw_host, # $ tainted
+
+    url.port, # $ tainted
+    url.explicit_port, # $ tainted
+
+    url.authority, # $ tainted
+    url.raw_authority, # $ tainted
+
+    url.path, # $ tainted
+    url.raw_path, # $ tainted
+
+    url.path_qs, # $ tainted
+    url.raw_path_qs, # $ tainted
+
+    url.query_string, # $ tainted
+    url.raw_query_string, # $ tainted
+
+    url.fragment, # $ tainted
+    url.raw_fragment, # $ tainted
+
+    url.parts, # $ tainted
+    url.raw_parts, # $ tainted
+
+    url.name, # $ tainted
+    url.raw_name, # $ tainted
+
+    # multidict.MultiDictProxy[str]
+    url.query, # $ tainted
+    url.query.getone("key"), # $ tainted
+
+    url.with_scheme("foo"), # $ tainted
+    url.with_user("foo"), # $ tainted
+    url.with_password("foo"), # $ tainted
+    url.with_host("foo"), # $ tainted
+    url.with_port("foo"), # $ tainted
+    url.with_path("foo"), # $ tainted
+    url.with_query({"foo": 42}), # $ tainted
+    url.with_query(foo=42), # $ tainted
+    url.update_query({"foo": 42}), # $ tainted
+    url.update_query(foo=42), # $ tainted
+    url.with_fragment("foo"), # $ tainted
+    url.with_name("foo"), # $ tainted
+
+    url.join(yarl.URL("wat.html")), # $ tainted
+
+    url.human_repr(), # $ tainted
+)

python/ql/test/query-tests/Security/CWE-326/WeakCryptoKey.expected

@@ -1,8 +1,8 @@
 | weak_crypto.py:68:1:68:21 | ControlFlowNode for dsa_gen_key() | Creation of an DSA key uses $@ bits, which is below 2048 and considered breakable. | weak_crypto.py:16:12:16:15 | ControlFlowNode for IntegerLiteral | 1024 |
-| weak_crypto.py:69:1:69:19 | ControlFlowNode for ec_gen_key() | Creation of an ECC key uses $@ bits, which is below 224 and considered breakable. | weak_crypto.py:22:11:22:24 | ControlFlowNode for Attribute() | 163 |
+| weak_crypto.py:69:1:69:19 | ControlFlowNode for ec_gen_key() | Creation of an ECC key uses $@ bits, which is below 224 and considered breakable. | weak_crypto.py:22:11:22:22 | ControlFlowNode for Attribute | 163 |
 | weak_crypto.py:70:1:70:28 | ControlFlowNode for rsa_gen_key() | Creation of an RSA key uses $@ bits, which is below 2048 and considered breakable. | weak_crypto.py:12:12:12:15 | ControlFlowNode for IntegerLiteral | 1024 |
 | weak_crypto.py:72:1:72:30 | ControlFlowNode for dsa_gen_key() | Creation of an DSA key uses $@ bits, which is below 2048 and considered breakable. | weak_crypto.py:16:12:16:15 | ControlFlowNode for IntegerLiteral | 1024 |
-| weak_crypto.py:73:1:73:25 | ControlFlowNode for ec_gen_key() | Creation of an ECC key uses $@ bits, which is below 224 and considered breakable. | weak_crypto.py:22:11:22:24 | ControlFlowNode for Attribute() | 163 |
+| weak_crypto.py:73:1:73:25 | ControlFlowNode for ec_gen_key() | Creation of an ECC key uses $@ bits, which is below 224 and considered breakable. | weak_crypto.py:22:11:22:22 | ControlFlowNode for Attribute | 163 |
 | weak_crypto.py:74:1:74:37 | ControlFlowNode for rsa_gen_key() | Creation of an RSA key uses $@ bits, which is below 2048 and considered breakable. | weak_crypto.py:12:12:12:15 | ControlFlowNode for IntegerLiteral | 1024 |
 | weak_crypto.py:76:1:76:22 | ControlFlowNode for Attribute() | Creation of an DSA key uses $@ bits, which is below 2048 and considered breakable. | weak_crypto.py:16:12:16:15 | ControlFlowNode for IntegerLiteral | 1024 |
 | weak_crypto.py:77:1:77:22 | ControlFlowNode for Attribute() | Creation of an RSA key uses $@ bits, which is below 2048 and considered breakable. | weak_crypto.py:12:12:12:15 | ControlFlowNode for IntegerLiteral | 1024 |

python/ql/test/query-tests/Security/CWE-327-BrokenCryptoAlgorithm/BrokenCryptoAlgorithm.expected

@@ -0,0 +1,2 @@
+| test_cryptodome.py:11:13:11:42 | ControlFlowNode for Attribute() | The cryptographic algorithm ARC4 is broken or weak, and should not be used. |
+| test_cryptography.py:13:13:13:44 | ControlFlowNode for Attribute() | The cryptographic algorithm ARC4 is broken or weak, and should not be used. |

python/ql/test/query-tests/Security/CWE-327-BrokenCryptoAlgorithm/README.md

@@ -0,0 +1,3 @@
+Note that the tests in this directory are very shallow, and simply show that the query is able to produce alerts.
+
+More in-depth tests can be found for the individual frameworks that we have modeled `Cryptography::CryptographicOperation` for.

python/ql/test/query-tests/Security/CWE-327-BrokenCryptoAlgorithm/test_cryptodome.py

@@ -0,0 +1,13 @@
+# snippet from python/ql/test/experimental/library-tests/frameworks/cryptodome/test_rc4.py
+from Cryptodome.Cipher import ARC4
+
+import os
+
+key = os.urandom(256//8)
+
+secret_message = b"secret message"
+
+cipher = ARC4.new(key)
+encrypted = cipher.encrypt(secret_message) # NOT OK
+
+print(secret_message, encrypted)

python/ql/test/query-tests/Security/CWE-327-BrokenCryptoAlgorithm/test_cryptography.py

@@ -0,0 +1,16 @@
+# snippet from python/ql/test/experimental/library-tests/frameworks/cryptography/test_rc4.py
+from cryptography.hazmat.primitives.ciphers import algorithms, Cipher
+import os
+
+key = os.urandom(256//8)
+
+algorithm = algorithms.ARC4(key)
+cipher = Cipher(algorithm, mode=None)
+
+secret_message = b"secret message"
+
+encryptor = cipher.encryptor()
+encrypted = encryptor.update(secret_message) # NOT OK
+encrypted += encryptor.finalize()
+
+print(secret_message, encrypted)

python/ql/test/query-tests/Security/CWE-327-WeakSensitiveDataHashing/README.md

@@ -0,0 +1,3 @@
+Note that the tests in this directory are very shallow, and simply show that the query is able to produce alerts.
+
+More in-depth tests can be found for the individual frameworks that we have modeled `Cryptography::CryptographicOperation` for.

python/ql/test/query-tests/Security/CWE-327-WeakSensitiveDataHashing/WeakSensitiveDataHashing.expected

@@ -0,0 +1,27 @@
+edges
+| test_cryptodome.py:6:17:6:33 | ControlFlowNode for get_certificate() | test_cryptodome.py:8:19:8:27 | ControlFlowNode for dangerous |
+| test_cryptodome.py:13:17:13:30 | ControlFlowNode for get_password() | test_cryptodome.py:15:19:15:27 | ControlFlowNode for dangerous |
+| test_cryptodome.py:20:17:20:30 | ControlFlowNode for get_password() | test_cryptodome.py:24:19:24:27 | ControlFlowNode for dangerous |
+| test_cryptography.py:7:17:7:33 | ControlFlowNode for get_certificate() | test_cryptography.py:9:19:9:27 | ControlFlowNode for dangerous |
+| test_cryptography.py:15:17:15:30 | ControlFlowNode for get_password() | test_cryptography.py:17:19:17:27 | ControlFlowNode for dangerous |
+| test_cryptography.py:23:17:23:30 | ControlFlowNode for get_password() | test_cryptography.py:27:19:27:27 | ControlFlowNode for dangerous |
+nodes
+| test_cryptodome.py:6:17:6:33 | ControlFlowNode for get_certificate() | semmle.label | ControlFlowNode for get_certificate() |
+| test_cryptodome.py:8:19:8:27 | ControlFlowNode for dangerous | semmle.label | ControlFlowNode for dangerous |
+| test_cryptodome.py:13:17:13:30 | ControlFlowNode for get_password() | semmle.label | ControlFlowNode for get_password() |
+| test_cryptodome.py:15:19:15:27 | ControlFlowNode for dangerous | semmle.label | ControlFlowNode for dangerous |
+| test_cryptodome.py:20:17:20:30 | ControlFlowNode for get_password() | semmle.label | ControlFlowNode for get_password() |
+| test_cryptodome.py:24:19:24:27 | ControlFlowNode for dangerous | semmle.label | ControlFlowNode for dangerous |
+| test_cryptography.py:7:17:7:33 | ControlFlowNode for get_certificate() | semmle.label | ControlFlowNode for get_certificate() |
+| test_cryptography.py:9:19:9:27 | ControlFlowNode for dangerous | semmle.label | ControlFlowNode for dangerous |
+| test_cryptography.py:15:17:15:30 | ControlFlowNode for get_password() | semmle.label | ControlFlowNode for get_password() |
+| test_cryptography.py:17:19:17:27 | ControlFlowNode for dangerous | semmle.label | ControlFlowNode for dangerous |
+| test_cryptography.py:23:17:23:30 | ControlFlowNode for get_password() | semmle.label | ControlFlowNode for get_password() |
+| test_cryptography.py:27:19:27:27 | ControlFlowNode for dangerous | semmle.label | ControlFlowNode for dangerous |
+#select
+| test_cryptodome.py:8:19:8:27 | ControlFlowNode for dangerous | test_cryptodome.py:6:17:6:33 | ControlFlowNode for get_certificate() | test_cryptodome.py:8:19:8:27 | ControlFlowNode for dangerous | $@ is used in a hashing algorithm (MD5) that is insecure. | test_cryptodome.py:6:17:6:33 | ControlFlowNode for get_certificate() | Sensitive data (certificate) |
+| test_cryptodome.py:15:19:15:27 | ControlFlowNode for dangerous | test_cryptodome.py:13:17:13:30 | ControlFlowNode for get_password() | test_cryptodome.py:15:19:15:27 | ControlFlowNode for dangerous | $@ is used in a hashing algorithm (MD5) that is insecure for password hashing, since it is not a computationally expensive hash function. | test_cryptodome.py:13:17:13:30 | ControlFlowNode for get_password() | Sensitive data (password) |
+| test_cryptodome.py:24:19:24:27 | ControlFlowNode for dangerous | test_cryptodome.py:20:17:20:30 | ControlFlowNode for get_password() | test_cryptodome.py:24:19:24:27 | ControlFlowNode for dangerous | $@ is used in a hashing algorithm (SHA256) that is insecure for password hashing, since it is not a computationally expensive hash function. | test_cryptodome.py:20:17:20:30 | ControlFlowNode for get_password() | Sensitive data (password) |
+| test_cryptography.py:9:19:9:27 | ControlFlowNode for dangerous | test_cryptography.py:7:17:7:33 | ControlFlowNode for get_certificate() | test_cryptography.py:9:19:9:27 | ControlFlowNode for dangerous | $@ is used in a hashing algorithm (MD5) that is insecure. | test_cryptography.py:7:17:7:33 | ControlFlowNode for get_certificate() | Sensitive data (certificate) |
+| test_cryptography.py:17:19:17:27 | ControlFlowNode for dangerous | test_cryptography.py:15:17:15:30 | ControlFlowNode for get_password() | test_cryptography.py:17:19:17:27 | ControlFlowNode for dangerous | $@ is used in a hashing algorithm (MD5) that is insecure for password hashing, since it is not a computationally expensive hash function. | test_cryptography.py:15:17:15:30 | ControlFlowNode for get_password() | Sensitive data (password) |
+| test_cryptography.py:27:19:27:27 | ControlFlowNode for dangerous | test_cryptography.py:23:17:23:30 | ControlFlowNode for get_password() | test_cryptography.py:27:19:27:27 | ControlFlowNode for dangerous | $@ is used in a hashing algorithm (SHA256) that is insecure for password hashing, since it is not a computationally expensive hash function. | test_cryptography.py:23:17:23:30 | ControlFlowNode for get_password() | Sensitive data (password) |

python/ql/test/query-tests/Security/CWE-327-WeakSensitiveDataHashing/WeakSensitiveDataHashing.qlref

@@ -0,0 +1 @@
+Security/CWE-327/WeakSensitiveDataHashing.ql