Merge branch 'main' of https://github.com/github/codeql into python/remove-ssa-nodes-from-dataflow-graph

python/ql/test/library-tests/ApiGraphs/py3/dataflow-consistency.expected

@@ -1,27 +0,0 @@
-uniqueEnclosingCallable
-uniqueCallEnclosingCallable
-uniqueType
-uniqueNodeLocation
-missingLocation
-uniqueNodeToString
-parameterCallable
-localFlowIsLocal
-readStepIsLocal
-storeStepIsLocal
-compatibleTypesReflexive
-unreachableNodeCCtx
-localCallNodes
-postIsNotPre
-postHasUniquePre
-uniquePostUpdate
-postIsInSameCallable
-reverseRead
-argHasPostUpdate
-postWithInFlow
-viableImplInCallContextTooLarge
-uniqueParameterNodeAtPosition
-uniqueParameterNodePosition
-uniqueContentApprox
-identityLocalStep
-missingArgumentCall
-multipleArgumentCall

python/ql/test/library-tests/ApiGraphs/py3/dataflow-consistency.ql

@@ -1,2 +0,0 @@
-import python
-import experimental.dataflow.TestUtil.DataFlowConsistency

python/ql/test/library-tests/frameworks/django-orm/CONSISTENCY/DataFlowConsistency.expected

@@ -1,12 +1,3 @@
-uniqueEnclosingCallable
-uniqueCallEnclosingCallable
-uniqueType
-uniqueNodeLocation
-missingLocation
-uniqueNodeToString
-parameterCallable
-localFlowIsLocal
-readStepIsLocal
 storeStepIsLocal
 | testapp/orm_form_test.py:6:1:6:28 | [orm-model] Class MyModel | testapp/tests.py:83:16:83:36 | ControlFlowNode for Attribute() | Store step does not preserve enclosing callable. |
 | testapp/orm_form_test.py:6:1:6:28 | [orm-model] Class MyModel | testapp/tests.py:84:16:84:43 | ControlFlowNode for Attribute() | Store step does not preserve enclosing callable. |
@@ -56,20 +47,3 @@ storeStepIsLocal
 | testapp/orm_tests.py:294:1:294:29 | [orm-model] Class TestLoad | testapp/orm_tests.py:356:12:356:33 | ControlFlowNode for Attribute() | Store step does not preserve enclosing callable. |
 | testapp/orm_tests.py:294:1:294:29 | [orm-model] Class TestLoad | testapp/orm_tests.py:363:9:363:37 | ControlFlowNode for Attribute() | Store step does not preserve enclosing callable. |
 | testapp/tests.py:81:33:81:37 | ControlFlowNode for Str | testapp/orm_form_test.py:6:1:6:28 | [orm-model] Class MyModel | Store step does not preserve enclosing callable. |
-compatibleTypesReflexive
-unreachableNodeCCtx
-localCallNodes
-postIsNotPre
-postHasUniquePre
-uniquePostUpdate
-postIsInSameCallable
-reverseRead
-argHasPostUpdate
-postWithInFlow
-viableImplInCallContextTooLarge
-uniqueParameterNodeAtPosition
-uniqueParameterNodePosition
-uniqueContentApprox
-identityLocalStep
-missingArgumentCall
-multipleArgumentCall

python/ql/test/library-tests/frameworks/django-orm/dataflow-consistency.ql

@@ -1,2 +0,0 @@
-import python
-import experimental.dataflow.TestUtil.DataFlowConsistency

python/ql/test/library-tests/frameworks/django-v2-v3/routing_test.py

@@ -150,3 +150,40 @@ class UnknownViewSubclass(UnknownViewSuperclass):
 urlpatterns = [
     path("UnknownViewSubclass/", UnknownViewSubclass.as_view()),  # $ routeSetup="UnknownViewSubclass/"
 ]
+
+################################################################################
+# Routing to *args and **kwargs
+################################################################################
+
+def kwargs_param(request, **kwargs): # $ requestHandler routedParameter=kwargs
+    ensure_tainted(
+        kwargs, # $ tainted
+        kwargs["foo"], # $ tainted
+        kwargs["bar"]  # $ tainted
+    )
+
+    ensure_tainted(request) # $ tainted
+
+
+def star_args_param(request, *args): # $ requestHandler routedParameter=args
+    ensure_tainted(
+        args, # $ tainted
+        args[0], # $ tainted
+        args[1], # $ tainted
+    )
+    ensure_tainted(request) # $ tainted
+
+
+def star_args_param_check(request, foo, bar): # $ requestHandler routedParameter=foo routedParameter=bar
+    ensure_tainted(
+        foo, # $ tainted
+        bar, # $ tainted
+    )
+    ensure_tainted(request) # $ tainted
+
+
+urlpatterns = [
+    path("test-kwargs_param/<foo>/<bar>", kwargs_param),  # $ routeSetup="test-kwargs_param/<foo>/<bar>"
+    re_path("test-star_args_param/([^/]+)/(.+)", star_args_param),  # $ routeSetup="test-star_args_param/([^/]+)/(.+)"
+    re_path("test-star_args_param_check/([^/]+)/(.+)", star_args_param_check),  # $ routeSetup="test-star_args_param_check/([^/]+)/(.+)"
+]

python/ql/test/library-tests/frameworks/django/CONSISTENCY/DataFlowConsistency.expected

@@ -0,0 +1,13 @@
+storeStepIsLocal
+| SqlExecution.py:16:1:16:25 | [orm-model] Class User | SqlExecution.py:21:5:21:32 | ControlFlowNode for Attribute() | Store step does not preserve enclosing callable. |
+| SqlExecution.py:16:1:16:25 | [orm-model] Class User | SqlExecution.py:23:5:23:45 | ControlFlowNode for Attribute() | Store step does not preserve enclosing callable. |
+| SqlExecution.py:16:1:16:25 | [orm-model] Class User | SqlExecution.py:24:5:24:55 | ControlFlowNode for Attribute() | Store step does not preserve enclosing callable. |
+| SqlExecution.py:16:1:16:25 | [orm-model] Class User | SqlExecution.py:25:5:25:49 | ControlFlowNode for Attribute() | Store step does not preserve enclosing callable. |
+| SqlExecution.py:16:1:16:25 | [orm-model] Class User | SqlExecution.py:27:5:27:52 | ControlFlowNode for Attribute() | Store step does not preserve enclosing callable. |
+| SqlExecution.py:16:1:16:25 | [orm-model] Class User | SqlExecution.py:28:5:28:46 | ControlFlowNode for Attribute() | Store step does not preserve enclosing callable. |
+| SqlExecution.py:16:1:16:25 | [orm-model] Class User | SqlExecution.py:30:5:30:34 | ControlFlowNode for Attribute() | Store step does not preserve enclosing callable. |
+| SqlExecution.py:16:1:16:25 | [orm-model] Class User | SqlExecution.py:31:5:31:92 | ControlFlowNode for Attribute() | Store step does not preserve enclosing callable. |
+| SqlExecution.py:16:1:16:25 | [orm-model] Class User | SqlExecution.py:34:5:34:34 | ControlFlowNode for Attribute() | Store step does not preserve enclosing callable. |
+| SqlExecution.py:16:1:16:25 | [orm-model] Class User | SqlExecution.py:37:5:37:33 | ControlFlowNode for Attribute() | Store step does not preserve enclosing callable. |
+| SqlExecution.py:16:1:16:25 | [orm-model] Class User | SqlExecution.py:37:5:37:59 | ControlFlowNode for Attribute() | Store step does not preserve enclosing callable. |
+| SqlExecution.py:16:1:16:25 | [orm-model] Class User | SqlExecution.py:37:5:37:77 | ControlFlowNode for Attribute() | Store step does not preserve enclosing callable. |

python/ql/test/library-tests/frameworks/rest_framework/CONSISTENCY/DataFlowConsistency.expected

@@ -0,0 +1,4 @@
+storeStepIsLocal
+| testapp/models.py:6:1:6:24 | [orm-model] Class Foo | testapp/views.py:14:16:14:32 | ControlFlowNode for Attribute() | Store step does not preserve enclosing callable. |
+| testapp/models.py:11:1:11:24 | [orm-model] Class Bar | testapp/views.py:19:16:19:32 | ControlFlowNode for Attribute() | Store step does not preserve enclosing callable. |
+| testapp/models.py:11:1:11:24 | [orm-model] Class Bar | testapp/views.py:23:16:23:32 | ControlFlowNode for Attribute() | Store step does not preserve enclosing callable. |

python/ql/test/library-tests/frameworks/rest_framework/taint_test.py

@@ -1,5 +1,6 @@
 from rest_framework.decorators import api_view, parser_classes
 from rest_framework.views import APIView
+from rest_framework.viewsets import ModelViewSet
 from rest_framework.request import Request
 from rest_framework.response import Response
 from rest_framework.parsers import JSONParser
@@ -89,7 +90,7 @@ def test_taint(request: Request, routed_param): # $ requestHandler routedParamet
 
 
 class MyClass(APIView):
-    def initial(self, request, *args, **kwargs): # $ requestHandler
+    def initial(self, request, *args, **kwargs): # $ requestHandler routedParameter=kwargs
         # this method will be called before processing any request
         ensure_tainted(request) # $ tainted
 
@@ -107,12 +108,31 @@ class MyClass(APIView):
 
         return Response("ok") # $ HttpResponse
 
+# Viewsets
+# see https://www.django-rest-framework.org/api-guide/viewsets/
+
+class MyModelViewSet(ModelViewSet):
+    def retrieve(self, request, routed_param): # $ requestHandler routedParameter=routed_param
+        ensure_tainted(
+            request, # $ tainted
+            request.GET, # $ tainted
+            request.GET.get("pk"), # $ tainted
+            request.data # $ tainted
+        )
+
+        ensure_tainted(routed_param) # $ tainted
+
+        # same as for standard Django view
+        ensure_tainted(self.args, self.kwargs) # $ tainted
+
+        return Response("retrieve") # $ HttpResponse
 
 
 # fake setup, you can't actually run this
 urlpatterns = [
     path("test-taint/<routed_param>", test_taint),  # $ routeSetup="test-taint/<routed_param>"
     path("ClassView/<routed_param>", MyClass.as_view()), # $ routeSetup="ClassView/<routed_param>"
+    path("MyModelViewSet/<routed_param>", MyModelViewSet.as_view()) # $ routeSetup="MyModelViewSet/<routed_param>"
 ]
 
 # tests with no route-setup, but we can still tell that these are using Django REST

python/ql/test/library-tests/frameworks/stdlib/test_re.py

@@ -0,0 +1,82 @@
+import re
+
+ts = TAINTED_STRING
+
+pat = ... # some pattern
+compiled_pat = re.compile(pat)
+
+# see https://docs.python.org/3/library/re.html#functions
+ensure_not_tainted(
+    # returns Match object, which is tested properly below. (note: with the flow summary
+    # modeling, objects containing tainted values are not themselves tainted).
+    re.search(pat, ts),
+    re.match(pat, ts),
+    re.fullmatch(pat, ts),
+
+    compiled_pat.search(ts),
+    compiled_pat.match(ts),
+    compiled_pat.fullmatch(ts),
+)
+
+# Match object
+tainted_match = re.match(pat, ts)
+safe_match = re.match(pat, "safe")
+ensure_tainted(
+    tainted_match.expand("Hello \1"), # $ tainted
+    safe_match.expand(ts), # $ tainted
+    tainted_match.group(), # $ tainted
+    tainted_match.group(1, 2), # $ tainted
+    tainted_match.group(1, 2)[0], # $ tainted
+    tainted_match[0], # $ tainted
+    tainted_match["key"], # $ tainted
+
+    tainted_match.groups()[0], # $ tainted
+    tainted_match.groupdict()["key"], # $ tainted
+
+    re.match(pat, ts).string, # $ tainted
+    re.match(ts, "safe").re.pattern, # $ tainted
+
+    compiled_pat.match(ts).string, # $ tainted
+    re.compile(ts).match("safe").re.pattern, # $ tainted
+)
+ensure_not_tainted(
+    safe_match.expand("Hello \1"),
+    safe_match.group(),
+
+    re.match(pat, "safe").re,
+    re.match(pat, "safe").string,
+)
+
+ensure_tainted(
+    # other functions not returning Match objects
+    re.split(pat, ts), # $ tainted
+    re.split(pat, ts)[0], # $ tainted
+
+    re.findall(pat, ts), # $ tainted
+    re.findall(pat, ts)[0], # $ tainted
+
+    re.finditer(pat, ts), # $ tainted
+    [x for x in re.finditer(pat, ts)], # $ tainted
+
+    re.sub(pat, repl="safe", string=ts), # $ tainted
+    re.sub(pat, repl=lambda m: ..., string=ts), # $ tainted
+    re.sub(pat, repl=ts, string="safe"), # $ tainted
+    re.sub(pat, repl=lambda m: ts, string="safe"), # $ tainted
+
+    # same for compiled patterns
+    compiled_pat.split(ts), # $ tainted
+    compiled_pat.split(ts)[0], # $ tainted
+    # ...
+
+    # user-controlled compiled pattern
+    re.compile(ts), # $ tainted
+    re.compile(ts).pattern, # $ tainted
+)
+
+ensure_not_tainted(
+    re.subn(pat, repl="safe", string=ts),
+    re.subn(pat, repl="safe", string=ts)[1], # // the number of substitutions made
+)
+ensure_tainted(
+    re.subn(pat, repl="safe", string=ts)[0], # $ tainted // the string
+)