Merge pull request #2129 from RasmusWL/python-update-django

Python: update django support

python/ql/test/3/library-tests/web/django/Sinks.expected

@@ -1,6 +0,0 @@
-| models.py:9 | key | externally controlled string |
-| rawsql.py:4 | BinaryExpr | externally controlled string |
-| rawsql.py:13 | BinaryExpr | externally controlled string |
-| rawsql.py:18 | BinaryExpr | externally controlled string |
-| rawsql.py:22 | BinaryExpr | externally controlled string |
-| views.py:8 | Attribute() | externally controlled string |

python/ql/test/3/library-tests/web/django/Sinks.ql

@@ -1,13 +0,0 @@
-
-import python
-
-
-import semmle.python.web.django.Request
-import semmle.python.web.django.Model
-import semmle.python.web.django.Db
-import semmle.python.web.django.Response
-import semmle.python.security.strings.Untrusted
-
-from TaintSink sink, TaintKind kind
-where sink.sinks(kind)
-select sink.getLocation().toString(), sink.(ControlFlowNode).getNode().toString(), kind.toString()

python/ql/test/3/library-tests/web/django/Sources.expected

@@ -1,8 +0,0 @@
-| models.py:9 | Attribute | django.db.models.Model.objects |
-| rawsql.py:13 | Attribute | django.db.models.Model.objects |
-| rawsql.py:16 | Attribute | django.db.models.Model.objects |
-| rawsql.py:21 | Attribute | django.db.models.Model.objects |
-| views.py:6 | request | django.request.HttpRequest |
-| views.py:8 | HttpResponse() | django.response.HttpResponse |
-| views.py:11 | path | externally controlled string |
-| views.py:11 | request | django.request.HttpRequest |

python/ql/test/3/library-tests/web/django/Sources.ql

@@ -1,12 +0,0 @@
-
-import python
-
-
-import semmle.python.web.django.Request
-import semmle.python.web.django.Model
-import semmle.python.web.django.Response
-import semmle.python.security.strings.Untrusted
-
-from TaintSource src, TaintKind kind
-where src.isSourceOf(kind)
-select src.getLocation().toString(), src.(ControlFlowNode).getNode().toString(), kind.toString()

python/ql/test/3/library-tests/web/django/Taint.expected

@@ -1,24 +0,0 @@
-| models.py:9 | Attribute | django.db.models.Model.objects |
-| rawsql.py:13 | Attribute | django.db.models.Model.objects |
-| rawsql.py:13 | Attribute() | django.db.models.Model.objects |
-| rawsql.py:16 | Attribute | django.db.models.Model.objects |
-| rawsql.py:16 | Attribute() | django.db.models.Model.objects |
-| rawsql.py:17 | Attribute() | django.db.models.Model.objects |
-| rawsql.py:17 | m | django.db.models.Model.objects |
-| rawsql.py:18 | Attribute() | django.db.models.Model.objects |
-| rawsql.py:18 | m | django.db.models.Model.objects |
-| rawsql.py:21 | Attribute | django.db.models.Model.objects |
-| rawsql.py:21 | Attribute() | django.db.models.Model.objects |
-| rawsql.py:22 | Attribute() | django.db.models.Model.objects |
-| rawsql.py:22 | m | django.db.models.Model.objects |
-| views.py:6 | request | django.request.HttpRequest |
-| views.py:8 | Attribute | django.http.request.QueryDict |
-| views.py:8 | Attribute() | externally controlled string |
-| views.py:8 | HttpResponse() | django.response.HttpResponse |
-| views.py:8 | request | django.request.HttpRequest |
-| views.py:11 | path | externally controlled string |
-| views.py:11 | request | django.request.HttpRequest |
-| views.py:12 | Dict | {externally controlled string} |
-| views.py:12 | path | externally controlled string |
-| views.py:13 | env | {externally controlled string} |
-| views.py:13 | request | django.request.HttpRequest |

python/ql/test/3/library-tests/web/django/Taint.ql

@@ -1,14 +0,0 @@
-
-import python
-
-
-import semmle.python.web.django.Request
-import semmle.python.web.django.Model
-import semmle.python.web.django.Response
-import semmle.python.security.strings.Untrusted
-
-
-from TaintedNode node
-
-select node.getLocation().toString(), node.getAstNode().toString(), node.getTaintKind().toString()
-

python/ql/test/3/library-tests/web/django/django/__init__.py

@@ -1 +0,0 @@
-#Fake django package

python/ql/test/3/library-tests/web/django/django/conf/__init__.py

@@ -1 +0,0 @@
-#Fake django package

python/ql/test/3/library-tests/web/django/django/conf/urls.py

@@ -1,3 +0,0 @@
-
-def url(regex, view):
-    pass

python/ql/test/3/library-tests/web/django/django/db/__init__.py

@@ -1 +0,0 @@
-#Fake django package

python/ql/test/3/library-tests/web/django/django/db/models/__init__.py

@@ -1,2 +0,0 @@
-class Model:
-    pass

python/ql/test/3/library-tests/web/django/django/db/models/expressions.py

@@ -1,2 +0,0 @@
-class RawSQL:
-    pass

python/ql/test/3/library-tests/web/django/django/http/__init__.py

@@ -1,2 +0,0 @@
-
-from .response import HttpResponse

python/ql/test/3/library-tests/web/django/django/http/response.py

@@ -1,5 +0,0 @@
-
-class HttpResponse:
-
-    def __init__(self, *args):
-        pass

python/ql/test/3/library-tests/web/django/models.py

@@ -1,10 +0,0 @@
-
-from django.db import models
-
-class MyModel(models.Model):
-    title = models.CharField(max_length=500)
-    summary = models.TextField(blank=True)
-
-def update_my_model(key, title):
-    item = MyModel.objects.get(pk=key)
-    item.title = title

python/ql/test/3/library-tests/web/django/rawsql.py

@@ -1,23 +0,0 @@
-from django.db.models.expressions import RawSQL
-
-def raw1(arg):
-    return RawSQL("select foo from bar where baz = %s" % arg, "")
-
-
-from django.db import models
-
-class MyModel(models.Model):
-    pass
-
-def raw2(arg):
-    MyModel.objects.raw("select foo from bar where baz = %s" % arg)
-
-def raw3(arg):
-    m = MyModel.objects.filter('foo')
-    m = m.filter('bar')
-    m.raw("select foo from bar where baz = %s" % arg)
-
-def raw4(arg):
-    m = MyModel.objects.filter('foo')
-    m.extra("select foo from bar where baz = %s" % arg)
-

python/ql/test/3/library-tests/web/django/urls.py

@@ -1,9 +0,0 @@
-from django.conf.urls import url
-import views
-
-urlpatterns = [
-
-    url(r'^route1$', views.view_func1),
-    url(r'^(?P<path>.*)$', views.view_func2),
-    url(r'^route2$', views.ClassView.as_view())
-]

python/ql/test/3/library-tests/web/django/views.py

@@ -1,19 +0,0 @@
-
-from django.http import HttpResponse
-from django.shortcuts import redirect, render
-from django.views.generic import View
-
-def view_func1(request):
-    # Whether this is safe depends on template.html -- annoyingly
-    return HttpResponse(request.GET.get("untrusted"))
-
-
-def view_func2(request, path='default'):
-    env = {'path': path}
-    return render(request, 'vulnerable-path.html', env)
-
-
-class ClassView(View):
-
-    def get(self, request):
-        pass

python/ql/test/3/library-tests/web/options

@@ -1 +0,0 @@
-semmle-extractor-options: --max-import-depth=3 --lang=3

python/ql/test/library-tests/web/django/Sinks.expected

@@ -1,6 +1,16 @@
-| test.py:18 | Str | externally controlled string |
-| test.py:21 | BinaryExpr | externally controlled string |
-| test.py:24 | BinaryExpr | externally controlled string |
-| test.py:25 | BinaryExpr | externally controlled string |
-| test.py:26 | BinaryExpr | externally controlled string |
-| test.py:34 | BinaryExpr | externally controlled string |
+| sql.py:13 | Str | externally controlled string |
+| sql.py:14 | Str | externally controlled string |
+| sql.py:17 | BinaryExpr | externally controlled string |
+| sql.py:20 | BinaryExpr | externally controlled string |
+| sql.py:21 | BinaryExpr | externally controlled string |
+| sql.py:22 | BinaryExpr | externally controlled string |
+| sql.py:36 | Str | externally controlled string |
+| sql.py:42 | BinaryExpr | externally controlled string |
+| sql.py:47 | BinaryExpr | externally controlled string |
+| views.py:7 | Attribute() | externally controlled string |
+| views.py:11 | Attribute() | externally controlled string |
+| views.py:15 | Attribute() | externally controlled string |
+| views.py:23 | Attribute() | externally controlled string |
+| views.py:29 | Attribute() | externally controlled string |
+| views.py:34 | Attribute() | externally controlled string |
+| views.py:38 | Attribute() | externally controlled string |

python/ql/test/library-tests/web/django/Sources.expected

@@ -1,2 +1,19 @@
+| test.py:5 | path | externally controlled string |
+| test.py:5 | request | django.request.HttpRequest |
+| test.py:11 | path | externally controlled string |
 | test.py:11 | request | django.request.HttpRequest |
-| test.py:31 | request | django.request.HttpRequest |
+| views.py:6 | bar | externally controlled string |
+| views.py:6 | foo | externally controlled string |
+| views.py:6 | request | django.request.HttpRequest |
+| views.py:10 | request | django.request.HttpRequest |
+| views.py:14 | request | django.request.HttpRequest |
+| views.py:22 | request | django.request.HttpRequest |
+| views.py:28 | request | django.request.HttpRequest |
+| views.py:32 | page_number | externally controlled string |
+| views.py:32 | request | django.request.HttpRequest |
+| views.py:37 | arg0 | externally controlled string |
+| views.py:37 | arg1 | externally controlled string |
+| views.py:37 | request | django.request.HttpRequest |
+| views.py:57 | request | django.request.HttpRequest |
+| views.py:57 | username | externally controlled string |
+| views.py:66 | request | django.request.HttpRequest |

python/ql/test/library-tests/web/django/sql.py

@@ -0,0 +1,53 @@
+from django.db import connection, models
+from django.db.models.expressions import RawSQL
+
+
+class User(models.Model):
+    username = models.CharField(max_length=100)
+    description = models.TextField(blank=True)
+
+
+def show_user(username):
+    with connection.cursor() as cursor:
+        # GOOD -- Using parameters
+        cursor.execute("SELECT * FROM users WHERE username = %s", username)
+        User.objects.raw("SELECT * FROM users WHERE username = %s", (username,))
+
+        # BAD -- Using string formatting
+        cursor.execute("SELECT * FROM users WHERE username = '%s'" % username)
+
+        # BAD -- other ways of executing raw SQL code with string interpolation
+        User.objects.annotate(RawSQL("insert into names_file ('name') values ('%s')" % username))
+        User.objects.raw("insert into names_file ('name') values ('%s')" % username)
+        User.objects.extra("insert into names_file ('name') values ('%s')" % username)
+
+        # BAD (but currently no custom query to find this)
+        #
+        # It is exposed to SQL injection (https://docs.djangoproject.com/en/2.2/ref/models/querysets/#extra)
+        # For example, using name = "; DROP ALL TABLES -- "
+        # will result in SQL: SELECT * FROM name WHERE name = ''; DROP ALL TABLES -- ''
+        #
+        # This shouldn't be very widespread, since using a normal string will result in invalid SQL
+        # Using name = "example", will result in SQL: SELECT * FROM name WHERE name = ''example''
+        # which in MySQL will give a syntax error
+        #
+        # When testing this out locally, none of the queries worked against SQLite3, but I could use
+        # the SQL injection against MySQL.
+        User.objects.raw("SELECT * FROM users WHERE username = '%s'", (username,))
+
+
+def raw3(arg):
+    m = User.objects.filter('foo')
+    m = m.filter('bar')
+    m.raw("select foo from bar where baz = %s" % arg)
+
+
+def raw4(arg):
+    m = User.objects.filter('foo')
+    m.extra("select foo from bar where baz = %s" % arg)
+
+
+def update_user(key, description1):
+    # Neither of these are exposed to sql-injections
+    user = User.objects.get(pk=key)
+    item.description = description

python/ql/test/library-tests/web/django/test.py

@@ -1,40 +1,18 @@
+from django.conf.urls import url
+from django.shortcuts import redirect, render
 
-from django.conf.urls import patterns, url
-from django.db import connection, models
-from django.db.models.expressions import RawSQL
-from django.http.response import HttpResponse
-import base64
 
-class Name(models.Model):
-    pass
+def with_template(request, path='default'):
+    env = {'path': path}
+    # We would need to understand django templates to know if this is safe or not
+    return render(request, 'possibly-vulnerable-template.html', env)
 
-def save_name(request):
 
-    if request.method == 'POST':
-        name = request.POST.get('name')
-        curs = connection.cursor()
-        #GOOD -- Using parameters
-        curs.execute(
-            "insert into names_file ('name') values ('%s')", name)
-        #BAD -- Using string formatting
-        curs.execute(
-            "insert into names_file ('name') values ('%s')" % name)
+def vuln_redirect(request, path):
+    return redirect(path)
 
-        #BAD -- other ways of executing raw SQL code with string interpolation
-        Name.objects.annotate(RawSQL("insert into names_file ('name') values ('%s')" % name))
-        Name.objects.raw("insert into names_file ('name') values ('%s')" % name)
-        Name.objects.extra("insert into names_file ('name') values ('%s')" % name)
 
-urlpatterns1 = patterns(url(r'^save_name/$',
-                           save_name, name='save_name'))
-
-def maybe_xss(request):
-    first_name = request.POST.get('first_name', '')
-    resp = HttpResponse()
-    resp.write("first name is " + first_name)
-    return resp
-
-urlpatterns2 = [
-    # Route to code_execution
-    url(r'^maybe_xss$', maybe_xss, name='maybe_xss')
+urlpatterns = [
+    url(r'^(?P<path>.*)$', with_template),
+    url(r'^redirect/(?P<path>.*)$', vuln_redirect),
 ]

python/ql/test/library-tests/web/django/views.py

@@ -0,0 +1,72 @@
+from django.conf.urls import patterns, url
+from django.http.response import HttpResponse
+from django.views.generic import View
+
+
+def url_match_xss(request, foo, bar, no_taint=None):
+    return HttpResponse('url_match_xss: {} {}'.format(foo, bar))
+
+
+def get_params_xss(request):
+    return HttpResponse(request.GET.get("untrusted"))
+
+
+def post_params_xss(request):
+    return HttpResponse(request.POST.get("untrusted"))
+
+
+class Foo(object):
+    # Note: since Foo is used as the super type in a class view, it will be able to handle requests.
+
+    # TODO: Currently we don't flag `untrusted` as a DjangoRequestParameter
+    def post(self, request, untrusted):
+        return HttpResponse('Foo post: {}'.format(untrusted))
+
+
+class ClassView(View, Foo):
+    # TODO: Currently we don't flag `untrusted` as a DjangoRequestParameter
+    def get(self, request, untrusted):
+        return HttpResponse('ClassView get: {}'.format(untrusted))
+
+
+def show_articles(request, page_number=1):
+    page_number = int(page_number)
+    return HttpResponse('articles page: {}'.format(page_number))
+
+
+def xxs_positional_arg(request, arg0, arg1, no_taint=None):
+    return HttpResponse('xxs_positional_arg: {} {}'.format(arg0, arg1))
+
+
+urlpatterns = [
+    url(r'^url_match/(?P<foo>[^/]+)/(?P<bar>[^/]+)$', url_match_xss),
+    url(r'^get_params$', get_params_xss),
+    url(r'^post_params$', post_params_xss),
+    url(r'^class_view/(?P<untrusted>.+)$', ClassView.as_view()),
+
+    # one pattern to support `articles/page-<n>` and ensuring that articles/ goes to page-1
+    url(r'articles/^(?:page-(?P<page_number>\d+)/)?$', show_articles),
+    # passing as positional argument is not the recommended way of doing things, but it is certainly
+    # possible
+    url(r'^([^/]+)/(?:foo|bar)/([^/]+)$', xxs_positional_arg, name='xxs_positional_arg'),
+]
+
+
+# Using patterns() for routing
+
+def show_user(request, username):
+    pass
+
+
+urlpatterns = patterns(url(r'^users/(?P<username>[^/]+)$', show_user))
+
+
+# Show we understand the keyword arguments to django.conf.urls.url
+
+def we_understand_url_kwargs(request):
+    pass
+
+
+urlpatterns = [
+    url(view=we_understand_url_kwargs, regex=r'^specifying-as-kwargs-is-not-a-problem$')
+]

python/ql/test/query-tests/Security/CWE-089/SqlInjection.expected

@@ -1,17 +1,14 @@
 edges
-| sql_injection.py:9:15:9:21 | django.request.HttpRequest | sql_injection.py:12:16:12:22 | django.request.HttpRequest |
-| sql_injection.py:12:16:12:22 | django.request.HttpRequest | sql_injection.py:12:16:12:27 | django.http.request.QueryDict |
-| sql_injection.py:12:16:12:27 | django.http.request.QueryDict | sql_injection.py:12:16:12:39 | externally controlled string |
-| sql_injection.py:12:16:12:39 | externally controlled string | sql_injection.py:19:63:19:66 | externally controlled string |
-| sql_injection.py:12:16:12:39 | externally controlled string | sql_injection.py:22:88:22:91 | externally controlled string |
-| sql_injection.py:12:16:12:39 | externally controlled string | sql_injection.py:23:76:23:79 | externally controlled string |
-| sql_injection.py:12:16:12:39 | externally controlled string | sql_injection.py:24:78:24:81 | externally controlled string |
-| sql_injection.py:19:63:19:66 | externally controlled string | sql_injection.py:19:13:19:66 | externally controlled string |
-| sql_injection.py:22:88:22:91 | externally controlled string | sql_injection.py:22:38:22:91 | externally controlled string |
-| sql_injection.py:23:76:23:79 | externally controlled string | sql_injection.py:23:26:23:79 | externally controlled string |
-| sql_injection.py:24:78:24:81 | externally controlled string | sql_injection.py:24:28:24:81 | externally controlled string |
+| sql_injection.py:12:24:12:31 | externally controlled string | sql_injection.py:19:70:19:77 | externally controlled string |
+| sql_injection.py:12:24:12:31 | externally controlled string | sql_injection.py:22:88:22:95 | externally controlled string |
+| sql_injection.py:12:24:12:31 | externally controlled string | sql_injection.py:23:76:23:83 | externally controlled string |
+| sql_injection.py:12:24:12:31 | externally controlled string | sql_injection.py:24:78:24:85 | externally controlled string |
+| sql_injection.py:19:70:19:77 | externally controlled string | sql_injection.py:19:24:19:77 | externally controlled string |
+| sql_injection.py:22:88:22:95 | externally controlled string | sql_injection.py:22:38:22:95 | externally controlled string |
+| sql_injection.py:23:76:23:83 | externally controlled string | sql_injection.py:23:26:23:83 | externally controlled string |
+| sql_injection.py:24:78:24:85 | externally controlled string | sql_injection.py:24:28:24:85 | externally controlled string |
 #select
-| sql_injection.py:19:13:19:66 | BinaryExpr | sql_injection.py:9:15:9:21 | django.request.HttpRequest | sql_injection.py:19:13:19:66 | externally controlled string | This SQL query depends on $@. | sql_injection.py:9:15:9:21 | request | a user-provided value |
-| sql_injection.py:22:38:22:91 | BinaryExpr | sql_injection.py:9:15:9:21 | django.request.HttpRequest | sql_injection.py:22:38:22:91 | externally controlled string | This SQL query depends on $@. | sql_injection.py:9:15:9:21 | request | a user-provided value |
-| sql_injection.py:23:26:23:79 | BinaryExpr | sql_injection.py:9:15:9:21 | django.request.HttpRequest | sql_injection.py:23:26:23:79 | externally controlled string | This SQL query depends on $@. | sql_injection.py:9:15:9:21 | request | a user-provided value |
-| sql_injection.py:24:28:24:81 | BinaryExpr | sql_injection.py:9:15:9:21 | django.request.HttpRequest | sql_injection.py:24:28:24:81 | externally controlled string | This SQL query depends on $@. | sql_injection.py:9:15:9:21 | request | a user-provided value |
+| sql_injection.py:19:24:19:77 | BinaryExpr | sql_injection.py:12:24:12:31 | externally controlled string | sql_injection.py:19:24:19:77 | externally controlled string | This SQL query depends on $@. | sql_injection.py:12:24:12:31 | username | a user-provided value |
+| sql_injection.py:22:38:22:95 | BinaryExpr | sql_injection.py:12:24:12:31 | externally controlled string | sql_injection.py:22:38:22:95 | externally controlled string | This SQL query depends on $@. | sql_injection.py:12:24:12:31 | username | a user-provided value |
+| sql_injection.py:23:26:23:83 | BinaryExpr | sql_injection.py:12:24:12:31 | externally controlled string | sql_injection.py:23:26:23:83 | externally controlled string | This SQL query depends on $@. | sql_injection.py:12:24:12:31 | username | a user-provided value |
+| sql_injection.py:24:28:24:85 | BinaryExpr | sql_injection.py:12:24:12:31 | externally controlled string | sql_injection.py:24:28:24:85 | externally controlled string | This SQL query depends on $@. | sql_injection.py:12:24:12:31 | username | a user-provided value |

python/ql/test/query-tests/Security/CWE-089/sql_injection.py

@@ -1,28 +1,40 @@
+"""This is copied from ql/python/ql/test/library-tests/web/django/test.py
+and a only a slight extension of ql/python/ql/src/Security/CWE-089/examples/sql_injection.py
+"""
 
-from django.conf.urls import patterns, url
+from django.conf.urls import url
 from django.db import connection, models
 from django.db.models.expressions import RawSQL
 
-class Name(models.Model):
+class User(models.Model):
     pass
 
-def save_name(request):
+def show_user(request, username):
+    with connection.cursor() as cursor:
+        # GOOD -- Using parameters
+        cursor.execute("SELECT * FROM users WHERE username = %s", username)
+        User.objects.raw("SELECT * FROM users WHERE username = %s", (username,))
 
-    if request.method == 'POST':
-        name = request.POST.get('name')
-        curs = connection.cursor()
-        #GOOD -- Using parameters
-        curs.execute(
-            "insert into names_file ('name') values ('%s')", name)
-        #BAD -- Using string formatting
-        curs.execute(
-            "insert into names_file ('name') values ('%s')" % name)
+        # BAD -- Using string formatting
+        cursor.execute("SELECT * FROM users WHERE username = '%s'" % username)
 
-        #BAD -- other ways of executing raw SQL code with string interpolation
-        Name.objects.annotate(RawSQL("insert into names_file ('name') values ('%s')" % name))
-        Name.objects.raw("insert into names_file ('name') values ('%s')" % name)
-        Name.objects.extra("insert into names_file ('name') values ('%s')" % name)
+        # BAD -- other ways of executing raw SQL code with string interpolation
+        User.objects.annotate(RawSQL("insert into names_file ('name') values ('%s')" % username))
+        User.objects.raw("insert into names_file ('name') values ('%s')" % username)
+        User.objects.extra("insert into names_file ('name') values ('%s')" % username)
 
-urlpatterns = patterns(url(r'^save_name/$',
-                           save_name, name='save_name'))
+        # BAD (but currently no custom query to find this)
+        #
+        # It is exposed to SQL injection (https://docs.djangoproject.com/en/2.2/ref/models/querysets/#extra)
+        # For example, using name = "; DROP ALL TABLES -- "
+        # will result in SQL: SELECT * FROM name WHERE name = ''; DROP ALL TABLES -- ''
+        #
+        # This shouldn't be very widespread, since using a normal string will result in invalid SQL
+        # Using name = "example", will result in SQL: SELECT * FROM name WHERE name = ''example''
+        # which in MySQL will give a syntax error
+        #
+        # When testing this out locally, none of the queries worked against SQLite3, but I could use
+        # the SQL injection against MySQL.
+        User.objects.raw("SELECT * FROM users WHERE username = '%s'", (username,))
 
+urlpatterns = [url(r'^users/(?P<username>[^/]+)$', show_user)]

python/ql/test/query-tests/Security/lib/django/conf/urls.py

@@ -1,7 +1,6 @@
-
-def url(pattern, *args):
+# https://docs.djangoproject.com/en/1.11/_modules/django/conf/urls/#url
+def url(regex, view, kwargs=None, name=None):
     pass
 
 def patterns(*urls):
     pass
-

python/ql/test/query-tests/Security/lib/django/views/generic.py

@@ -0,0 +1,2 @@
+class View:
+    pass