Merge branch 'main' into java/merge-5226

python/change-notes/2021-02-12-django-get_redirect_url.md

@@ -0,0 +1,2 @@
+lgtm,codescanning
+* Improved modeling of `django` to recognize request redirects from `get_redirect_url` on a `RedirectView` subclass.

python/change-notes/2021-03-01-fluent-interface-data-flow.md

@@ -0,0 +1,2 @@
+lgtm,codescanning
+* The data-flow library now recognises more side-effects of method chaining (e.g. `someObject.setX(clean).setY(tainted).setZ...` having a side-effect on `someObject`), as well as other related circumstances where a function input is directly passed to its output. All queries that use data-flow analysis, including most security queries, may return more results accordingly.

python/ql/src/semmle/crypto/Crypto.qll

@@ -1,10 +1,7 @@
 /**
- * Provides classes for modeling cryptographic libraries.
- */
-
-/*
- * The following information is copied from `/semmlecode-javascript-queries/semmle/javascript/frameworks/CryptoLibraries.qll`
- * which should be considered the definitive version (as of Feb 2018)
+ * Provides classes modeling cryptographic algorithms, separated into strong and weak variants.
+ *
+ * The classification into strong and weak are based on Wikipedia, OWASP and google (2017).
  */
 
 /**
@@ -13,6 +10,8 @@
  * The names are normalized: upper-case, no spaces, dashes or underscores.
  *
  * The names are inspired by the names used in real world crypto libraries.
+ *
+ * The classification into strong and weak are based on Wikipedia, OWASP and google (2017).
  */
 private module AlgorithmNames {
   predicate isStrongHashingAlgorithm(string name) {
@@ -81,14 +80,6 @@ private module AlgorithmNames {
   }
 
   predicate isWeakPasswordHashingAlgorithm(string name) { none() }
-
-  /**
-   * Normalizes `name`: upper-case, no spaces, dashes or underscores.
-   *
-   * All names of this module are in this normalized form.
-   */
-  bindingset[name]
-  string normalizeName(string name) { result = name.toUpperCase().regexpReplaceAll("[-_ ]", "") }
 }
 
 private import AlgorithmNames
@@ -121,10 +112,19 @@ abstract class CryptographicAlgorithm extends TCryptographicAlgorithm {
   string toString() { result = getName() }
 
   /**
-   * Gets the name of the algorithm.
+   * Gets the normalized name of this algorithm (upper-case, no spaces, dashes or underscores).
    */
   abstract string getName();
 
+  /**
+   * Holds if the name of this algorithm matches `name` modulo case,
+   * white space, dashes, and underscores.
+   */
+  bindingset[name]
+  predicate matchesName(string name) {
+    name.toUpperCase().regexpReplaceAll("[-_ ]", "") = getName()
+  }
+
   /**
    * Holds if this algorithm is weak.
    */

python/ql/src/semmle/python/dataflow/new/internal/DataFlowImpl.qll

@@ -223,7 +223,10 @@ private predicate isAdditionalFlowStep(
  * Holds if data can flow in one local step from `node1` to `node2`.
  */
 private predicate localFlowStep(Node node1, Node node2, Configuration config) {
-  simpleLocalFlowStep(node1, node2) and
+  (
+    simpleLocalFlowStep(node1, node2) or
+    reverseStepThroughInputOutputAlias(node1, node2)
+  ) and
   not outBarrier(node1, config) and
   not inBarrier(node2, config) and
   not fullBarrier(node1, config) and

python/ql/src/semmle/python/dataflow/new/internal/DataFlowImpl2.qll

@@ -223,7 +223,10 @@ private predicate isAdditionalFlowStep(
  * Holds if data can flow in one local step from `node1` to `node2`.
  */
 private predicate localFlowStep(Node node1, Node node2, Configuration config) {
-  simpleLocalFlowStep(node1, node2) and
+  (
+    simpleLocalFlowStep(node1, node2) or
+    reverseStepThroughInputOutputAlias(node1, node2)
+  ) and
   not outBarrier(node1, config) and
   not inBarrier(node2, config) and
   not fullBarrier(node1, config) and

python/ql/src/semmle/python/dataflow/new/internal/DataFlowImpl3.qll

@@ -223,7 +223,10 @@ private predicate isAdditionalFlowStep(
  * Holds if data can flow in one local step from `node1` to `node2`.
  */
 private predicate localFlowStep(Node node1, Node node2, Configuration config) {
-  simpleLocalFlowStep(node1, node2) and
+  (
+    simpleLocalFlowStep(node1, node2) or
+    reverseStepThroughInputOutputAlias(node1, node2)
+  ) and
   not outBarrier(node1, config) and
   not inBarrier(node2, config) and
   not fullBarrier(node1, config) and

python/ql/src/semmle/python/dataflow/new/internal/DataFlowImpl4.qll

@@ -223,7 +223,10 @@ private predicate isAdditionalFlowStep(
  * Holds if data can flow in one local step from `node1` to `node2`.
  */
 private predicate localFlowStep(Node node1, Node node2, Configuration config) {
-  simpleLocalFlowStep(node1, node2) and
+  (
+    simpleLocalFlowStep(node1, node2) or
+    reverseStepThroughInputOutputAlias(node1, node2)
+  ) and
   not outBarrier(node1, config) and
   not inBarrier(node2, config) and
   not fullBarrier(node1, config) and

python/ql/src/semmle/python/dataflow/new/internal/DataFlowImplCommon.qll

@@ -415,6 +415,30 @@ private module Cached {
     store(node1, tc.getContent(), node2, contentType, tc.getContainerType())
   }
 
+  /**
+   * Holds if data can flow from `fromNode` to `toNode` because they are the post-update
+   * nodes of some function output and input respectively, where the output and input
+   * are aliases. A typical example is a function returning `this`, implementing a fluent
+   * interface.
+   */
+  cached
+  predicate reverseStepThroughInputOutputAlias(PostUpdateNode fromNode, PostUpdateNode toNode) {
+    exists(Node fromPre, Node toPre |
+      fromPre = fromNode.getPreUpdateNode() and
+      toPre = toNode.getPreUpdateNode()
+    |
+      exists(DataFlowCall c |
+        // Does the language-specific simpleLocalFlowStep already model flow
+        // from function input to output?
+        fromPre = getAnOutNode(c, _) and
+        toPre.(ArgumentNode).argumentOf(c, _) and
+        simpleLocalFlowStep(toPre.(ArgumentNode), fromPre)
+      )
+      or
+      argumentValueFlowsThrough(toPre, TReadStepTypesNone(), fromPre)
+    )
+  }
+
   /**
    * Holds if the call context `call` either improves virtual dispatch in
    * `callable` or if it allows us to prune unreachable nodes in `callable`.

python/ql/src/semmle/python/frameworks/Django.qll

@@ -2074,7 +2074,11 @@ private module Django {
       // TODO: This doesn't handle attribute assignment. Should be OK, but analysis is not as complete as with
       // points-to and `.lookup`, which would handle `post = my_post_handler` inside class def
       result = this.getAMethod() and
-      result.getName() = HTTP::httpVerbLower()
+      (
+        result.getName() = HTTP::httpVerbLower()
+        or
+        result.getName() = "get_redirect_url"
+      )
     }
 
     /**
@@ -2124,6 +2128,8 @@ private module Django {
   /**
    * A function that is a django route handler, meaning it handles incoming requests
    * with the django framework.
+   *
+   * Most functions take a django HttpRequest as a parameter (but not all).
    */
   private class DjangoRouteHandler extends Function {
     DjangoRouteHandler() {
@@ -2132,6 +2138,12 @@ private module Django {
       any(DjangoViewClass vc).getARequestHandler() = this
     }
 
+    /**
+     * Gets the index of the parameter where the first routed parameter can be passed --
+     * that is, the one just after any possible `self` or HttpRequest parameters.
+     */
+    int getFirstPossibleRoutedParamIndex() { result = 1 + this.getRequestParamIndex() }
+
     /** Gets the index of the request parameter. */
     int getRequestParamIndex() {
       not this.isMethod() and
@@ -2145,6 +2157,26 @@ private module Django {
     Parameter getRequestParam() { result = this.getArg(this.getRequestParamIndex()) }
   }
 
+  /**
+   * A method named `get_redirect_url` on a django view class.
+   *
+   * See https://docs.djangoproject.com/en/3.1/ref/class-based-views/base/#django.views.generic.base.RedirectView.get_redirect_url
+   *
+   * Note: this function only does something on a subclass of `RedirectView`, but since
+   * classes can be considered django view classes without us knowing their super-classes,
+   * we need to consider _any_ django view class. I don't expect any problems to come from this.
+   */
+  private class GetRedirectUrlFunction extends DjangoRouteHandler {
+    GetRedirectUrlFunction() {
+      this.getName() = "get_redirect_url" and
+      any(DjangoViewClass vc).getARequestHandler() = this
+    }
+
+    override int getFirstPossibleRoutedParamIndex() { result = 1 }
+
+    override int getRequestParamIndex() { none() }
+  }
+
   /** A data-flow node that sets up a route on a server, using the django framework. */
   abstract private class DjangoRouteSetup extends HTTP::Server::RouteSetup::Range, DataFlow::CfgNode {
     /** Gets the data-flow node that is used as the argument for the view handler. */
@@ -2175,7 +2207,7 @@ private module Django {
       // parameter. This should give us more RemoteFlowSources but could also lead to
       // more FPs. If this turns out to be the wrong tradeoff, we can always change our mind.
       result in [this.getArg(_), this.getArgByName(_)] and
-      not result = any(int i | i <= this.getRequestParamIndex() | this.getArg(i))
+      not result = any(int i | i < this.getFirstPossibleRoutedParamIndex() | this.getArg(i))
     }
 
     override string getFramework() { result = "Django" }
@@ -2215,7 +2247,8 @@ private module Django {
       exists(DjangoRouteHandler routeHandler | routeHandler = this.getARequestHandler() |
         not exists(this.getUrlPattern()) and
         result in [routeHandler.getArg(_), routeHandler.getArgByName(_)] and
-        not result = any(int i | i <= routeHandler.getRequestParamIndex() | routeHandler.getArg(i))
+        not result =
+          any(int i | i < routeHandler.getFirstPossibleRoutedParamIndex() | routeHandler.getArg(i))
       )
       or
       exists(string name |
@@ -2237,7 +2270,8 @@ private module Django {
       exists(DjangoRouteHandler routeHandler | routeHandler = this.getARequestHandler() |
         not exists(this.getUrlPattern()) and
         result in [routeHandler.getArg(_), routeHandler.getArgByName(_)] and
-        not result = any(int i | i <= routeHandler.getRequestParamIndex() | routeHandler.getArg(i))
+        not result =
+          any(int i | i < routeHandler.getFirstPossibleRoutedParamIndex() | routeHandler.getArg(i))
       )
       or
       exists(DjangoRouteHandler routeHandler, DjangoRouteRegex regex |
@@ -2249,7 +2283,9 @@ private module Django {
         not exists(regex.getGroupName(_, _)) and
         // first group will have group number 1
         result =
-          routeHandler.getArg(routeHandler.getRequestParamIndex() + regex.getGroupNumber(_, _))
+          routeHandler
+              .getArg(routeHandler.getFirstPossibleRoutedParamIndex() - 1 +
+                  regex.getGroupNumber(_, _))
         or
         result = routeHandler.getArgByName(regex.getGroupName(_, _))
       )
@@ -2445,4 +2481,31 @@ private module Django {
 
     override string getMimetypeDefault() { none() }
   }
+
+  // ---------------------------------------------------------------------------
+  // RedirectView handling
+  // ---------------------------------------------------------------------------
+  /**
+   * A return from a method named `get_redirect_url` on a django view class.
+   *
+   * Note that in reality, this only does something on a subclass of `RedirectView` --
+   * but until API graphs makes this easy to model, I took a shortcut in modeling
+   * preciseness.
+   *
+   * See https://docs.djangoproject.com/en/3.1/ref/class-based-views/base/#redirectview
+   */
+  private class DjangoRedirectViewGetRedirectUrlReturn extends HTTP::Server::HttpRedirectResponse::Range,
+    DataFlow::CfgNode {
+    DjangoRedirectViewGetRedirectUrlReturn() {
+      node = any(GetRedirectUrlFunction f).getAReturnValueFlowNode()
+    }
+
+    override DataFlow::Node getRedirectLocation() { result = this }
+
+    override DataFlow::Node getBody() { none() }
+
+    override DataFlow::Node getMimetypeOrContentTypeArg() { none() }
+
+    override string getMimetypeDefault() { none() }
+  }
 }

python/ql/test/experimental/library-tests/frameworks/django-v2-v3/TestTaint.expected

@@ -1,3 +1,4 @@
+| response_test.py:61 | ok   | get_redirect_url | foo |
 | taint_test.py:8 | ok   | test_taint | bar |
 | taint_test.py:8 | ok   | test_taint | foo |
 | taint_test.py:9 | ok   | test_taint | baz |

python/ql/test/experimental/library-tests/frameworks/django-v2-v3/response_test.py

@@ -1,4 +1,5 @@
 from django.http.response import HttpResponse, HttpResponseRedirect, HttpResponsePermanentRedirect, JsonResponse, HttpResponseNotFound
+from django.views.generic import RedirectView
 import django.shortcuts
 
 # Not an XSS sink, since the Content-Type is not "text/html"
@@ -54,6 +55,14 @@ def redirect_shortcut(request):
     return django.shortcuts.redirect(next) # $ HttpResponse HttpRedirectResponse redirectLocation=next
 
 
+class CustomRedirectView(RedirectView):
+
+    def get_redirect_url(self, foo): # $ requestHandler routedParameter=foo
+        ensure_tainted(foo)
+        next = "https://example.com/{}".format(foo)
+        return next # $ HttpResponse HttpRedirectResponse redirectLocation=next
+
+
 # Ensure that simple subclasses are still vuln to XSS
 def xss__not_found(request):
     return HttpResponseNotFound(request.GET.get("name"))  # $HttpResponse mimetype=text/html responseBody=Attribute()

python/ql/test/experimental/library-tests/frameworks/django-v2-v3/testapp/urls.py

@@ -15,4 +15,7 @@ urlpatterns = [
 
     path("basic-view-handler/", views.MyBasicViewHandler.as_view()),  # $routeSetup="basic-view-handler/"
     path("custom-inheritance-view-handler/", views.MyViewHandlerWithCustomInheritance.as_view()),  # $routeSetup="custom-inheritance-view-handler/"
+
+    path("CustomRedirectView/<foo>", views.CustomRedirectView.as_view()),  # $routeSetup="CustomRedirectView/<foo>"
+    path("CustomRedirectView2/<foo>", views.CustomRedirectView2.as_view()),  # $routeSetup="CustomRedirectView2/<foo>"
 ]

python/ql/test/experimental/library-tests/frameworks/django-v2-v3/testapp/views.py

@@ -1,5 +1,5 @@
 from django.http import HttpRequest, HttpResponse
-from django.views import View
+from django.views.generic import View, RedirectView
 from django.views.decorators.csrf import csrf_exempt
 
 
@@ -32,3 +32,16 @@ class MyViewHandlerWithCustomInheritance(MyCustomViewBaseClass):
     def get(self, request: HttpRequest): # $ requestHandler
         print(self.request.GET)
         return HttpResponse("MyViewHandlerWithCustomInheritance: GET") # $ HttpResponse
+
+# RedirectView
+# See docs at https://docs.djangoproject.com/en/3.1/ref/class-based-views/base/#redirectview
+class CustomRedirectView(RedirectView):
+
+    def get_redirect_url(self, foo): # $ requestHandler routedParameter=foo
+        next = "https://example.com/{}".format(foo)
+        return next # $ HttpResponse HttpRedirectResponse redirectLocation=next
+
+
+class CustomRedirectView2(RedirectView):
+
+    url = "https://example.com/%(foo)s"