Merge branch 'master' into python-autoformat-almost-everything

python/ql/src/Security/CWE-022/TarSlip.ql

@@ -121,9 +121,19 @@ class ExtractMembersSink extends TaintSink {
 class TarFileInfoSanitizer extends Sanitizer {
     TarFileInfoSanitizer() { this = "TarInfo sanitizer" }
 
+    /** The test `if <path_sanitizing_test>:` clears taint on its `false` edge. */
     override predicate sanitizingEdge(TaintKind taint, PyEdgeRefinement test) {
-        path_sanitizing_test(test.getTest()) and
-        taint instanceof TarFileInfo
+        taint instanceof TarFileInfo and
+        clears_taint_on_false_edge(test.getTest(), test.getSense())
+    }
+
+    private predicate clears_taint_on_false_edge(ControlFlowNode test, boolean sense) {
+        path_sanitizing_test(test) and
+        sense = false
+        or
+        // handle `not` (also nested)
+        test.(UnaryExprNode).getNode().getOp() instanceof Not and
+        clears_taint_on_false_edge(test.(UnaryExprNode).getOperand(), sense.booleanNot())
     }
 }
 

python/ql/src/Security/CWE-078/CommandInjection.ql

@@ -29,8 +29,7 @@ class CommandInjectionConfiguration extends TaintTracking::Configuration {
     }
 
     override predicate isSink(TaintTracking::Sink sink) {
-        sink instanceof OsCommandFirstArgument or
-        sink instanceof ShellCommand
+        sink instanceof CommandSink
     }
 
     override predicate isExtension(TaintTracking::Extension extension) {

python/ql/src/semmle/python/dataflow/Implementation.qll

@@ -710,33 +710,16 @@ private class EssaTaintTracking extends string {
         TaintTrackingNode src, MultiAssignmentDefinition defn, TaintTrackingContext context,
         AttributePath path, TaintKind kind
     ) {
-        exists(DataFlow::Node srcnode, TaintKind srckind, Assign assign |
+        exists(DataFlow::Node srcnode, TaintKind srckind, Assign assign, int depth |
             src = TTaintTrackingNode_(srcnode, context, path, srckind, this) and
             path.noAttribute()
         |
             assign.getValue().getAFlowNode() = srcnode.asCfgNode() and
-            kind =
-                iterable_unpacking_descent(assign.getATarget().getAFlowNode(), defn.getDefiningNode(),
-                    srckind)
+            depth = iterable_unpacking_descent(assign.getATarget().getAFlowNode(), defn.getDefiningNode()) and
+            kind = taint_at_depth(srckind, depth)
         )
     }
 
-    /** `((x,y), ...) = value` with any nesting on LHS */
-    private TaintKind iterable_unpacking_descent(
-        SequenceNode left_parent, ControlFlowNode left_defn, CollectionKind parent_kind
-    ) {
-        //TODO: Fix the cartesian product in this predicate
-        none() and
-        left_parent.getAnElement() = left_defn and
-        // Handle `a, *b = some_iterable`
-        if left_defn instanceof StarredNode
-        then result = parent_kind
-        else result = parent_kind.getMember()
-        or
-        result =
-            iterable_unpacking_descent(left_parent.getAnElement(), left_defn, parent_kind.getMember())
-    }
-
     pragma[noinline]
     private predicate taintedAttributeAssignment(
         TaintTrackingNode src, AttributeAssignment defn, TaintTrackingContext context,
@@ -966,6 +949,46 @@ private predicate piNodeTestAndUse(PyEdgeRefinement defn, ControlFlowNode test,
     test = defn.getTest() and use = defn.getInput().getASourceUse() and test.getAChild*() = use
 }
 
+/** Helper predicate for taintedMultiAssignment */
+private TaintKind taint_at_depth(SequenceKind parent_kind, int depth) {
+    depth >= 0 and
+    (
+        // base-case #0
+        depth = 0 and
+        result = parent_kind
+        or
+        // base-case #1
+        depth = 1 and
+        result = parent_kind.getMember()
+        or
+        // recursive case
+        depth > 1 and
+        result = taint_at_depth(parent_kind.getMember(), depth-1)
+    )
+}
+
+/** Helper predicate for taintedMultiAssignment
+ *
+ * Returns the `depth` the elements that are assigned to `left_defn` with iterable unpacking has,
+ * compared to `left_parent`. Special care is taken for `StarredNode` that is assigned a sequence of items.
+ *
+ * For example, `((x, *y), ...) = value` with any nesting on LHS
+ * - with `left_defn` = `x`, `left_parent` = `(x, *y)`, result = 1
+ * - with `left_defn` = `x`, `left_parent` = `((x, *y), ...)`, result = 2
+ * - with `left_defn` = `*y`, `left_parent` = `(x, *y)`, result = 0
+ * - with `left_defn` = `*y`, `left_parent` = `((x, *y), ...)`, result = 1
+ */
+int iterable_unpacking_descent(SequenceNode left_parent, ControlFlowNode left_defn) {
+    exists(Assign a | a.getATarget().getASubExpression*().getAFlowNode() = left_parent) and
+    left_parent.getAnElement() = left_defn and
+    // Handle `a, *b = some_iterable`
+    if left_defn instanceof StarredNode
+    then result = 0
+    else result = 1
+    or
+    result = 1 + iterable_unpacking_descent(left_parent.getAnElement(), left_defn)
+}
+
 module Implementation {
     /* A call that returns a copy (or similar) of the argument */
     predicate copyCall(ControlFlowNode fromnode, CallNode tonode) {

python/ql/src/semmle/python/security/injection/Command.qll

@@ -10,6 +10,9 @@ import python
 import semmle.python.security.TaintTracking
 import semmle.python.security.strings.Untrusted
 
+/** Abstract taint sink that is potentially vulnerable to malicious shell commands. */
+abstract class CommandSink extends TaintSink { }
+
 private ModuleObject osOrPopenModule() {
     result.getName() = "os" or
     result.getName() = "popen2"
@@ -47,7 +50,7 @@ class FirstElementFlow extends DataFlowExtension::DataFlowNode {
  * A taint sink that is potentially vulnerable to malicious shell commands.
  * The `vuln` in `subprocess.call(shell=vuln)` and similar calls.
  */
-class ShellCommand extends TaintSink {
+class ShellCommand extends CommandSink {
     override string toString() { result = "shell command" }
 
     ShellCommand() {
@@ -86,7 +89,7 @@ class ShellCommand extends TaintSink {
  * A taint sink that is potentially vulnerable to malicious shell commands.
  * The `vuln` in `subprocess.call(vuln, ...)` and similar calls.
  */
-class OsCommandFirstArgument extends TaintSink {
+class OsCommandFirstArgument extends CommandSink {
     override string toString() { result = "OS command first argument" }
 
     OsCommandFirstArgument() {
@@ -105,3 +108,126 @@ class OsCommandFirstArgument extends TaintSink {
         kind instanceof FirstElementKind
     }
 }
+
+// -------------------------------------------------------------------------- //
+// Modeling of the 'invoke' package and 'fabric' package (v 2.x)
+//
+// Since fabric build so closely upon invoke, we model them together to avoid
+// duplication
+// -------------------------------------------------------------------------- //
+/**
+ * A taint sink that is potentially vulnerable to malicious shell commands.
+ * The `vuln` in `invoke.run(vuln, ...)` and similar calls.
+ */
+class InvokeRun extends CommandSink {
+    InvokeRun() {
+        this = Value::named("invoke.run").(FunctionValue).getArgumentForCall(_, 0)
+        or
+        this = Value::named("invoke.sudo").(FunctionValue).getArgumentForCall(_, 0)
+    }
+
+    override string toString() { result = "InvokeRun" }
+
+    override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind }
+}
+
+/**
+ * Internal TaintKind to track the invoke.Context instance passed to functions
+ * marked with @invoke.task
+ */
+private class InvokeContextArg extends TaintKind {
+    InvokeContextArg() { this = "InvokeContextArg" }
+}
+
+/** Internal TaintSource to track the context passed to functions marked with @invoke.task */
+private class InvokeContextArgSource extends TaintSource {
+    InvokeContextArgSource() {
+        exists(Function f, Expr decorator |
+            count(f.getADecorator()) = 1 and
+            (
+                decorator = f.getADecorator() and not decorator instanceof Call
+                or
+                decorator = f.getADecorator().(Call).getFunc()
+            ) and
+            (
+                decorator.pointsTo(Value::named("invoke.task"))
+                or
+                decorator.pointsTo(Value::named("fabric.task"))
+            )
+        |
+            this.(ControlFlowNode).getNode() = f.getArg(0)
+        )
+    }
+
+    override predicate isSourceOf(TaintKind kind) { kind instanceof InvokeContextArg }
+}
+
+/**
+ * A taint sink that is potentially vulnerable to malicious shell commands.
+ * The `vuln` in `invoke.Context().run(vuln, ...)` and similar calls.
+ */
+class InvokeContextRun extends CommandSink {
+    InvokeContextRun() {
+        exists(CallNode call |
+            any(InvokeContextArg k).taints(call.getFunction().(AttrNode).getObject("run"))
+            or
+            call = Value::named("invoke.Context").(ClassValue).lookup("run").getACall()
+            or
+            // fabric.connection.Connection is a subtype of invoke.context.Context
+            // since fabric.Connection.run has a decorator, it doesn't work with FunctionValue :|
+            // and `Value::named("fabric.Connection").(ClassValue).lookup("run").getACall()` returned no results,
+            // so here is the hacky solution that works :\
+            call.getFunction().(AttrNode).getObject("run").pointsTo().getClass() =
+                Value::named("fabric.Connection")
+        |
+            this = call.getArg(0)
+            or
+            this = call.getArgByName("command")
+        )
+    }
+
+    override string toString() { result = "InvokeContextRun" }
+
+    override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind }
+}
+
+/**
+ * A taint sink that is potentially vulnerable to malicious shell commands.
+ * The `vuln` in `fabric.Group().run(vuln, ...)` and similar calls.
+ */
+class FabricGroupRun extends CommandSink {
+    FabricGroupRun() {
+        exists(ClassValue cls |
+            cls.getASuperType() = Value::named("fabric.Group") and
+            this = cls.lookup("run").(FunctionValue).getArgumentForCall(_, 1)
+        )
+    }
+
+    override string toString() { result = "FabricGroupRun" }
+
+    override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind }
+}
+
+// -------------------------------------------------------------------------- //
+// Modeling of the 'invoke' package and 'fabric' package (v 1.x)
+// -------------------------------------------------------------------------- //
+class FabricV1Commands extends CommandSink {
+    FabricV1Commands() {
+        // since `run` and `sudo` are decorated, we can't use FunctionValue's :(
+        exists(CallNode call |
+            call = Value::named("fabric.api.local").getACall()
+            or
+            call = Value::named("fabric.api.run").getACall()
+            or
+            call = Value::named("fabric.api.sudo").getACall()
+        |
+            this = call.getArg(0)
+            or
+            this = call.getArgByName("command")
+        )
+    }
+
+    override string toString() { result = "FabricV1Commands" }
+
+    override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind }
+}

python/ql/src/semmle/python/web/bottle/General.qll

@@ -36,7 +36,7 @@ class BottleRoute extends ControlFlowNode {
 
     Function getFunction() { bottle_route(this, _, result) }
 
-    Parameter getNamedArgument() {
+    Parameter getANamedArgument() {
         exists(string name, Function func |
             func = this.getFunction() and
             func.getArgByName(name) = result and

python/ql/src/semmle/python/web/bottle/Request.qll

@@ -71,7 +71,7 @@ class UntrustedFile extends TaintKind {
 /** Parameter to a bottle request handler function */
 class BottleRequestParameter extends HttpRequestTaintSource {
     BottleRequestParameter() {
-        exists(BottleRoute route | route.getNamedArgument() = this.(ControlFlowNode).getNode())
+        exists(BottleRoute route | route.getANamedArgument() = this.(ControlFlowNode).getNode())
     }
 
     override predicate isSourceOf(TaintKind kind) { kind instanceof UntrustedStringKind }

python/ql/src/semmle/python/web/django/General.qll

@@ -2,39 +2,142 @@ import python
 import semmle.python.regex
 import semmle.python.web.Http
 
-predicate django_route(CallNode call, ControlFlowNode regex, FunctionValue view) {
-    exists(FunctionValue url |
-        Value::named("django.conf.urls.url") = url and
-        url.getArgumentForCall(call, 0) = regex and
-        url.getArgumentForCall(call, 1).pointsTo(view)
-    )
-}
-
-class DjangoRouteRegex extends RegexString {
-    DjangoRouteRegex() { django_route(_, this.getAFlowNode(), _) }
-}
-
-class DjangoRoute extends CallNode {
-    DjangoRoute() { django_route(this, _, _) }
-
-    FunctionValue getViewFunction() { django_route(this, _, result) }
-
-    string getNamedArgument() {
-        exists(DjangoRouteRegex regex |
-            django_route(this, regex.getAFlowNode(), _) and
-            regex.getGroupName(_, _) = result
-        )
+// TODO: Since django uses `path = partial(...)`, our analysis doesn't understand this is
+// a FunctionValue, so we can't use `FunctionValue.getArgumentForCall`
+// https://github.com/django/django/blob/master/django/urls/conf.py#L76
+abstract class DjangoRoute extends CallNode {
+    DjangoViewHandler getViewHandler() {
+        result = view_handler_from_view_arg(this.getArg(1))
+        or
+        result = view_handler_from_view_arg(this.getArgByName("view"))
     }
 
+    abstract string getANamedArgument();
+
     /**
      * Get the number of positional arguments that will be passed to the view.
      * Will only return a result if there are no named arguments.
      */
-    int getNumPositionalArguments() {
-        exists(DjangoRouteRegex regex |
-            django_route(this, regex.getAFlowNode(), _) and
-            not exists(string s | s = regex.getGroupName(_, _)) and
+    abstract int getNumPositionalArguments();
+}
+
+/**
+ * For function based views -- also see `DjangoClassBasedViewHandler`
+ * https://docs.djangoproject.com/en/1.11/topics/http/views/
+ * https://docs.djangoproject.com/en/3.0/topics/http/views/
+ */
+class DjangoViewHandler extends PythonFunctionValue {
+
+    /** Gets the index of the 'request' argument */
+    int getRequestArgIndex() {
+        result = 0
+    }
+}
+
+/**
+ * Class based views
+ * https://docs.djangoproject.com/en/1.11/topics/class-based-views/
+ * https://docs.djangoproject.com/en/3.0/topics/class-based-views/
+ */
+private class DjangoViewClass extends ClassValue {
+    DjangoViewClass() {
+        Value::named("django.views.generic.View") = this.getASuperType()
+        or
+        Value::named("django.views.View") = this.getASuperType()
+    }
+}
+
+class DjangoClassBasedViewHandler extends DjangoViewHandler {
+    DjangoClassBasedViewHandler() {
+        exists(DjangoViewClass cls |
+            cls.lookup(httpVerbLower()) = this
+        )
+    }
+
+    override int getRequestArgIndex() {
+        // due to `self` being the first parameter
+        result = 1
+    }
+}
+
+/**
+ * Gets the function that will handle requests when `view_arg` is used as the view argument to a
+ * django route. That is, this methods handles Class-based Views and its `as_view()` function.
+ */
+private DjangoViewHandler view_handler_from_view_arg(ControlFlowNode view_arg) {
+    // Function-based view
+    result = view_arg.pointsTo()
+    or
+    // Class-based view
+    exists(ClassValue cls |
+        cls = view_arg.(CallNode).getFunction().(AttrNode).getObject("as_view").pointsTo() and
+        result = cls.lookup(httpVerbLower())
+    )
+}
+
+// We need this "dummy" class, since otherwise the regex argument would not be considered
+// a regex (RegexString is abstract)
+class DjangoRouteRegex extends RegexString {
+    DjangoRouteRegex() { exists(DjangoRegexRoute route | route.getRouteArg() = this.getAFlowNode()) }
+}
+
+class DjangoRegexRoute extends DjangoRoute {
+    ControlFlowNode route;
+
+    DjangoRegexRoute() {
+        exists(FunctionValue route_maker |
+            // Django 1.x: https://docs.djangoproject.com/en/1.11/ref/urls/#django.conf.urls.url
+            Value::named("django.conf.urls.url") = route_maker and
+            route_maker.getArgumentForCall(this, 0) = route
+        )
+        or
+        // Django 2.x and 3.x: https://docs.djangoproject.com/en/3.0/ref/urls/#re-path
+        this = Value::named("django.urls.re_path").getACall() and
+        (
+            route = this.getArg(0)
+            or
+            route = this.getArgByName("route")
+        )
+    }
+
+    ControlFlowNode getRouteArg() { result = route }
+
+    override string getANamedArgument() {
+        exists(DjangoRouteRegex regex | regex.getAFlowNode() = route |
+            result = regex.getGroupName(_, _)
+        )
+    }
+
+    override int getNumPositionalArguments() {
+        not exists(this.getANamedArgument()) and
+        exists(DjangoRouteRegex regex | regex.getAFlowNode() = route |
             result = count(regex.getGroupNumber(_, _))
         )
     }
 }
+
+class DjangoPathRoute extends DjangoRoute {
+    ControlFlowNode route;
+
+    DjangoPathRoute() {
+        // Django 2.x and 3.x: https://docs.djangoproject.com/en/3.0/ref/urls/#path
+        this = Value::named("django.urls.path").getACall() and
+        (
+            route = this.getArg(0)
+            or
+            route = this.getArgByName("route")
+        )
+    }
+
+    override string getANamedArgument() {
+        // regexp taken from django:
+        // https://github.com/django/django/blob/7d1bf29977bb368d7c28e7c6eb146db3b3009ae7/django/urls/resolvers.py#L199
+        exists(StrConst route_str, string match |
+            route_str = route.getNode() and
+            match = route_str.getText().regexpFind("<(?:(?<converter>[^>:]+):)?(?<parameter>\\w+)>", _, _) and
+            result = match.regexpCapture("<(?:(?<converter>[^>:]+):)?(?<parameter>\\w+)>", 2)
+        )
+    }
+
+    override int getNumPositionalArguments() { none() }
+}

python/ql/src/semmle/python/web/django/Redirect.qll

@@ -17,9 +17,6 @@ class DjangoRedirect extends HttpRedirectTaintSink {
     override string toString() { result = "django.redirect" }
 
     DjangoRedirect() {
-        exists(CallNode call |
-            redirect().getACall() = call and
-            this = call.getAnArg()
-        )
+        this = redirect().getACall().getAnArg()
     }
 }

python/ql/src/semmle/python/web/django/Request.qll

@@ -39,53 +39,35 @@ class DjangoQueryDict extends TaintKind {
     }
 }
 
-abstract class DjangoRequestSource extends HttpRequestTaintSource {
+/** A Django request parameter */
+class DjangoRequestSource extends HttpRequestTaintSource {
+    DjangoRequestSource() {
+        exists(DjangoRoute route, DjangoViewHandler view, int request_arg_index |
+            route.getViewHandler() = view and
+            request_arg_index = view.getRequestArgIndex() and
+            this = view.getScope().getArg(request_arg_index).asName().getAFlowNode()
+        )
+    }
+
     override string toString() { result = "Django request source" }
 
     override predicate isSourceOf(TaintKind kind) { kind instanceof DjangoRequest }
 }
 
-/**
- * Function based views
- * https://docs.djangoproject.com/en/1.11/topics/http/views/
- */
-private class DjangoFunctionBasedViewRequestArgument extends DjangoRequestSource {
-    DjangoFunctionBasedViewRequestArgument() {
-        exists(FunctionValue view |
-            django_route(_, _, view) and
-            this = view.getScope().getArg(0).asName().getAFlowNode()
-        )
-    }
-}
-
-/**
- * Class based views
- * https://docs.djangoproject.com/en/1.11/topics/class-based-views/
- */
-private class DjangoView extends ClassValue {
-    DjangoView() { Value::named("django.views.generic.View") = this.getASuperType() }
-}
-
-private FunctionValue djangoViewHttpMethod() {
-    exists(DjangoView view | view.lookup(httpVerbLower()) = result)
-}
-
-class DjangoClassBasedViewRequestArgument extends DjangoRequestSource {
-    DjangoClassBasedViewRequestArgument() {
-        this = djangoViewHttpMethod().getScope().getArg(1).asName().getAFlowNode()
-    }
-}
-
 /** An argument specified in a url routing table */
 class DjangoRequestParameter extends HttpRequestTaintSource {
     DjangoRequestParameter() {
-        exists(DjangoRoute route, Function f | f = route.getViewFunction().getScope() |
-            this.(ControlFlowNode).getNode() = f.getArgByName(route.getNamedArgument())
+        exists(DjangoRoute route, Function f, DjangoViewHandler view, int request_arg_index |
+            route.getViewHandler() = view and
+            request_arg_index = view.getRequestArgIndex() and
+            f = view.getScope()
+        |
+            this.(ControlFlowNode).getNode() = f.getArgByName(route.getANamedArgument())
             or
             exists(int i | i >= 0 |
                 i < route.getNumPositionalArguments() and
                 // +1 because first argument is always the request
-                this.(ControlFlowNode).getNode() = f.getArg(i + 1)
+                this.(ControlFlowNode).getNode() = f.getArg(request_arg_index + 1 + i)
             )
         )
     }

python/ql/src/semmle/python/web/django/Response.qll

@@ -14,7 +14,15 @@ class DjangoResponse extends TaintKind {
 }
 
 private ClassValue theDjangoHttpResponseClass() {
-    result = Value::named("django.http.response.HttpResponse") and
+    (
+        // version 1.x
+        result = Value::named("django.http.response.HttpResponse")
+        or
+        // version 2.x
+        // https://docs.djangoproject.com/en/2.2/ref/request-response/#httpresponse-objects
+        result = Value::named("django.http.HttpResponse")
+    ) and
+    // TODO: does this do anything? when could they be the same???
     not result = theDjangoHttpRedirectClass()
 }
 

python/ql/src/semmle/python/web/django/Shared.qll

@@ -4,5 +4,9 @@ import python
 FunctionValue redirect() { result = Value::named("django.shortcuts.redirect") }
 
 ClassValue theDjangoHttpRedirectClass() {
+    // version 1.x
     result = Value::named("django.http.response.HttpResponseRedirectBase")
+    or
+    // version 2.x
+    result = Value::named("django.http.HttpResponseRedirectBase")
 }

python/ql/test/3/library-tests/taint/unpacking/TestTaint.expected

@@ -1,9 +1,9 @@
-| test.py:11 | extended_unpacking | first | NO TAINT |
-| test.py:11 | extended_unpacking | last | NO TAINT |
-| test.py:11 | extended_unpacking | rest | NO TAINT |
-| test.py:16 | also_allowed | a | NO TAINT |
+| test.py:11 | extended_unpacking | first | externally controlled string |
+| test.py:11 | extended_unpacking | last | externally controlled string |
+| test.py:11 | extended_unpacking | rest | [externally controlled string] |
+| test.py:16 | also_allowed | a | [externally controlled string] |
 | test.py:24 | also_allowed | b | NO TAINT |
 | test.py:24 | also_allowed | c | NO TAINT |
-| test.py:31 | nested | x | NO TAINT |
-| test.py:31 | nested | xs | NO TAINT |
-| test.py:31 | nested | ys | NO TAINT |
+| test.py:31 | nested | x | externally controlled string |
+| test.py:31 | nested | xs | [externally controlled string] |
+| test.py:31 | nested | ys | [externally controlled string] |

python/ql/test/library-tests/PointsTo/regressions/wrong/classmethod/Test.expected

@@ -0,0 +1,7 @@
+| test.py:10:15:10:17 | ControlFlowNode for cls | class Foo |
+| test.py:17:15:17:17 | ControlFlowNode for cls | class Foo |
+| test.py:17:15:17:17 | ControlFlowNode for cls | self instance of Foo |
+| test.py:22:15:22:17 | ControlFlowNode for cls | class Foo |
+| test.py:22:15:22:17 | ControlFlowNode for cls | self instance of Foo |
+| test.py:27:15:27:17 | ControlFlowNode for cls | class Foo |
+| test.py:27:15:27:17 | ControlFlowNode for cls | self instance of Foo |

python/ql/test/library-tests/PointsTo/regressions/wrong/classmethod/Test.ql

@@ -0,0 +1,10 @@
+import python
+
+from NameNode name, CallNode call, string debug
+where
+    call.getAnArg() = name and
+    call.getFunction().(NameNode).getId() = "check" and
+    if exists(name.pointsTo())
+    then debug = name.pointsTo().toString()
+    else debug = "<MISSING pointsTo()>"
+select name, debug

python/ql/test/library-tests/PointsTo/regressions/wrong/classmethod/test.py

@@ -0,0 +1,35 @@
+# See https://github.com/Semmle/ql/issues/3113
+def some_decorator(func):
+    print("this could be tricky for our analysis")
+    return func
+
+class Foo(object):
+
+    @classmethod
+    def no_problem(cls):
+        check(cls) # analysis says 'cls' can only point-to Class Foo
+
+    @some_decorator
+    @classmethod
+    def problem_through_instance(cls):
+        # Problem is that our analysis says that 'cls' can point to EITHER the
+        # Class Foo (correct) or an instance of Foo (wrong)
+        check(cls)
+
+    @some_decorator
+    @classmethod
+    def problem_through_class(cls):
+        check(cls) # same as above
+
+    @classmethod
+    @some_decorator
+    def also_problem(cls):
+        check(cls) # same as above
+
+# We need to call the methods before our analysis works
+f1 = Foo()
+f1.no_problem()
+f1.problem_through_instance()
+f1.also_problem()
+
+Foo.problem_through_class()

python/ql/test/library-tests/security/command-execution/CommandSinks.expected

@@ -0,0 +1,17 @@
+| fabric_v1_test.py:8:7:8:28 | FabricV1Commands | externally controlled string |
+| fabric_v1_test.py:9:5:9:27 | FabricV1Commands | externally controlled string |
+| fabric_v1_test.py:10:6:10:38 | FabricV1Commands | externally controlled string |
+| fabric_v2_test.py:10:16:10:25 | InvokeContextRun | externally controlled string |
+| fabric_v2_test.py:12:15:12:36 | InvokeContextRun | externally controlled string |
+| fabric_v2_test.py:16:45:16:54 | FabricGroupRun | externally controlled string |
+| fabric_v2_test.py:21:10:21:13 | FabricGroupRun | externally controlled string |
+| fabric_v2_test.py:31:14:31:41 | InvokeContextRun | externally controlled string |
+| fabric_v2_test.py:33:15:33:64 | InvokeContextRun | externally controlled string |
+| invoke_test.py:8:12:8:21 | InvokeRun | externally controlled string |
+| invoke_test.py:9:20:9:40 | InvokeRun | externally controlled string |
+| invoke_test.py:12:17:12:24 | InvokeRun | externally controlled string |
+| invoke_test.py:13:25:13:32 | InvokeRun | externally controlled string |
+| invoke_test.py:17:11:17:40 | InvokeContextRun | externally controlled string |
+| invoke_test.py:21:11:21:32 | InvokeContextRun | externally controlled string |
+| invoke_test.py:27:11:27:25 | InvokeContextRun | externally controlled string |
+| invoke_test.py:32:11:32:25 | InvokeContextRun | externally controlled string |

python/ql/test/library-tests/security/command-execution/CommandSinks.ql

@@ -0,0 +1,7 @@
+import python
+import semmle.python.security.injection.Command
+import semmle.python.security.strings.Untrusted
+
+from CommandSink sink, TaintKind kind
+where sink.sinks(kind)
+select sink, kind

python/ql/test/library-tests/security/command-execution/fabric-LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2020 Jeff Forcier.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice,
+      this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright notice,
+      this list of conditions and the following disclaimer in the documentation
+      and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

python/ql/test/library-tests/security/command-execution/fabric_v1_test.py

@@ -0,0 +1,10 @@
+"""tests for the 'fabric' package (v1.x)
+
+See http://docs.fabfile.org/en/1.14/tutorial.html
+"""
+
+from fabric.api import run, local, sudo
+
+local('echo local execution')
+run('echo remote execution')
+sudo('echo remote execution with sudo')

python/ql/test/library-tests/security/command-execution/fabric_v2_test.py

@@ -0,0 +1,33 @@
+"""tests for the 'fabric' package (v2.x)
+
+Most of these examples are taken from the fabric documentation: http://docs.fabfile.org/en/2.5/getting-started.html
+See fabric-LICENSE for its' license.
+"""
+
+from fabric import Connection
+
+c = Connection('web1')
+result = c.run('uname -s')
+
+c.run(command='echo run with kwargs')
+
+
+from fabric import SerialGroup as Group
+results = Group('web1', 'web2', 'mac1').run('uname -s')
+
+
+from fabric import SerialGroup as Group
+pool = Group('web1', 'web2', 'web3')
+pool.run('ls')
+
+
+
+# using the 'fab' command-line tool
+
+from fabric import task
+
+@task
+def upload_and_unpack(c):
+    if c.run('test -f /opt/mydata/myfile', warn=True).failed:
+        c.put('myfiles.tgz', '/opt/mydata')
+        c.run('tar -C /opt/mydata -xzvf /opt/mydata/myfiles.tgz')

python/ql/test/library-tests/security/command-execution/invoke_test.py

@@ -0,0 +1,32 @@
+"""tests for the 'invoke' package
+
+see https://www.pyinvoke.org/
+"""
+
+import invoke
+
+invoke.run('echo run')
+invoke.run(command='echo run with kwarg')
+
+def with_sudo():
+    invoke.sudo('whoami')
+    invoke.sudo(command='whoami')
+
+def manual_context():
+    c = invoke.Context()
+    c.run('echo run from manual context')
+manual_context()
+
+def foo_helper(c):
+    c.run('echo from foo_helper')
+
+# for use with the 'invoke' command-line tool
+@invoke.task
+def foo(c):
+    # 'c' is a invoke.context.Context
+    c.run('echo task foo')
+    foo_helper(c)
+
+@invoke.task()
+def bar(c):
+    c.run('echo task bar')

python/ql/test/library-tests/security/command-execution/options

@@ -0,0 +1 @@
+semmle-extractor-options: --max-import-depth=2 -p ../../../query-tests/Security/lib/

python/ql/test/library-tests/taint/collections/TestStep.expected

@@ -26,6 +26,9 @@
 | Taint [externally controlled string] | test.py:29 | test.py:29:9:29:25 | Subscript |  |  -->  | Taint [externally controlled string] | test.py:32 | test.py:32:16:32:16 | c |  |
 | Taint [externally controlled string] | test.py:30 | test.py:30:9:30:20 | tainted_list |  |  -->  | Taint [externally controlled string] | test.py:30 | test.py:30:9:30:27 | Attribute() |  |
 | Taint [externally controlled string] | test.py:30 | test.py:30:9:30:27 | Attribute() |  |  -->  | Taint [externally controlled string] | test.py:32 | test.py:32:19:32:19 | d |  |
+| Taint [externally controlled string] | test.py:31 | test.py:31:15:31:26 | tainted_list |  |  -->  | Taint externally controlled string | test.py:32 | test.py:32:22:32:22 | e |  |
+| Taint [externally controlled string] | test.py:31 | test.py:31:15:31:26 | tainted_list |  |  -->  | Taint externally controlled string | test.py:32 | test.py:32:25:32:25 | f |  |
+| Taint [externally controlled string] | test.py:31 | test.py:31:15:31:26 | tainted_list |  |  -->  | Taint externally controlled string | test.py:32 | test.py:32:28:32:28 | g |  |
 | Taint [externally controlled string] | test.py:33 | test.py:33:14:33:25 | tainted_list |  |  -->  | Taint externally controlled string | test.py:33 | test.py:33:5:33:26 | For |  |
 | Taint [externally controlled string] | test.py:35 | test.py:35:14:35:35 | reversed() |  |  -->  | Taint externally controlled string | test.py:35 | test.py:35:5:35:36 | For |  |
 | Taint [externally controlled string] | test.py:35 | test.py:35:23:35:34 | tainted_list |  |  -->  | Taint [externally controlled string] | test.py:35 | test.py:35:14:35:35 | reversed() |  |

python/ql/test/library-tests/taint/collections/TestTaint.expected

@@ -10,9 +10,9 @@
 | test.py:32 | test_access | b | externally controlled string |
 | test.py:32 | test_access | c | [externally controlled string] |
 | test.py:32 | test_access | d | [externally controlled string] |
-| test.py:32 | test_access | e | NO TAINT |
-| test.py:32 | test_access | f | NO TAINT |
-| test.py:32 | test_access | g | NO TAINT |
+| test.py:32 | test_access | e | externally controlled string |
+| test.py:32 | test_access | f | externally controlled string |
+| test.py:32 | test_access | g | externally controlled string |
 | test.py:34 | test_access | h | externally controlled string |
 | test.py:36 | test_access | i | externally controlled string |
 | test.py:43 | test_dict_access | a | externally controlled string |

python/ql/test/library-tests/taint/namedtuple/TestTaint.expected

@@ -1,7 +1,7 @@
 | test.py:13 | test_basic | a | externally controlled string |
 | test.py:13 | test_basic | b | externally controlled string |
 | test.py:13 | test_basic | c | externally controlled string |
-| test.py:13 | test_basic | d | NO TAINT |
+| test.py:13 | test_basic | d | externally controlled string |
 | test.py:13 | test_basic | urlsplit_res | [externally controlled string] |
 | test.py:19 | test_sanitizer | Attribute | externally controlled string |
 | test.py:22 | test_sanitizer | Attribute | NO TAINT |

python/ql/test/library-tests/taint/unpacking/TestStep.expected

@@ -1,8 +1,35 @@
 | Taint [[externally controlled string]] | test.py:19 | test.py:19:10:19:18 | List |  |  -->  | Taint [[externally controlled string]] | test.py:22 | test.py:22:28:22:29 | ll |  |
 | Taint [[externally controlled string]] | test.py:19 | test.py:19:10:19:18 | List |  |  -->  | Taint [[externally controlled string]] | test.py:26 | test.py:26:28:26:29 | ll |  |
 | Taint [[externally controlled string]] | test.py:19 | test.py:19:10:19:18 | List |  |  -->  | Taint [[externally controlled string]] | test.py:30 | test.py:30:28:30:29 | ll |  |
+| Taint [[externally controlled string]] | test.py:22 | test.py:22:28:22:29 | ll |  |  -->  | Taint [externally controlled string] | test.py:23 | test.py:23:22:23:22 | b |  |
+| Taint [[externally controlled string]] | test.py:22 | test.py:22:28:22:29 | ll |  |  -->  | Taint [externally controlled string] | test.py:23 | test.py:23:25:23:25 | c |  |
+| Taint [[externally controlled string]] | test.py:22 | test.py:22:28:22:29 | ll |  |  -->  | Taint externally controlled string | test.py:23 | test.py:23:10:23:11 | a1 |  |
+| Taint [[externally controlled string]] | test.py:22 | test.py:22:28:22:29 | ll |  |  -->  | Taint externally controlled string | test.py:23 | test.py:23:14:23:15 | a2 |  |
+| Taint [[externally controlled string]] | test.py:22 | test.py:22:28:22:29 | ll |  |  -->  | Taint externally controlled string | test.py:23 | test.py:23:18:23:19 | a3 |  |
+| Taint [[externally controlled string]] | test.py:26 | test.py:26:28:26:29 | ll |  |  -->  | Taint [externally controlled string] | test.py:27 | test.py:27:22:27:22 | b |  |
+| Taint [[externally controlled string]] | test.py:26 | test.py:26:28:26:29 | ll |  |  -->  | Taint [externally controlled string] | test.py:27 | test.py:27:25:27:25 | c |  |
+| Taint [[externally controlled string]] | test.py:26 | test.py:26:28:26:29 | ll |  |  -->  | Taint externally controlled string | test.py:27 | test.py:27:10:27:11 | a1 |  |
+| Taint [[externally controlled string]] | test.py:26 | test.py:26:28:26:29 | ll |  |  -->  | Taint externally controlled string | test.py:27 | test.py:27:14:27:15 | a2 |  |
+| Taint [[externally controlled string]] | test.py:26 | test.py:26:28:26:29 | ll |  |  -->  | Taint externally controlled string | test.py:27 | test.py:27:18:27:19 | a3 |  |
+| Taint [[externally controlled string]] | test.py:30 | test.py:30:28:30:29 | ll |  |  -->  | Taint [externally controlled string] | test.py:31 | test.py:31:22:31:22 | b |  |
+| Taint [[externally controlled string]] | test.py:30 | test.py:30:28:30:29 | ll |  |  -->  | Taint [externally controlled string] | test.py:31 | test.py:31:25:31:25 | c |  |
+| Taint [[externally controlled string]] | test.py:30 | test.py:30:28:30:29 | ll |  |  -->  | Taint externally controlled string | test.py:31 | test.py:31:10:31:11 | a1 |  |
+| Taint [[externally controlled string]] | test.py:30 | test.py:30:28:30:29 | ll |  |  -->  | Taint externally controlled string | test.py:31 | test.py:31:14:31:15 | a2 |  |
+| Taint [[externally controlled string]] | test.py:30 | test.py:30:28:30:29 | ll |  |  -->  | Taint externally controlled string | test.py:31 | test.py:31:18:31:19 | a3 |  |
+| Taint [[externally controlled string]] | test.py:47 | test.py:47:28:47:54 | Tuple |  |  -->  | Taint externally controlled string | test.py:48 | test.py:48:10:48:10 | a |  |
+| Taint [[externally controlled string]] | test.py:47 | test.py:47:28:47:54 | Tuple |  |  -->  | Taint externally controlled string | test.py:48 | test.py:48:13:48:13 | b |  |
+| Taint [[externally controlled string]] | test.py:47 | test.py:47:28:47:54 | Tuple |  |  -->  | Taint externally controlled string | test.py:48 | test.py:48:16:48:16 | c |  |
+| Taint [[externally controlled string]] | test.py:47 | test.py:47:28:47:54 | Tuple |  |  -->  | Taint externally controlled string | test.py:48 | test.py:48:19:48:19 | d |  |
+| Taint [[externally controlled string]] | test.py:47 | test.py:47:28:47:54 | Tuple |  |  -->  | Taint externally controlled string | test.py:48 | test.py:48:22:48:22 | e |  |
+| Taint [[externally controlled string]] | test.py:47 | test.py:47:28:47:54 | Tuple |  |  -->  | Taint externally controlled string | test.py:48 | test.py:48:25:48:25 | f |  |
 | Taint [externally controlled string] | test.py:6 | test.py:6:9:6:20 | TAINTED_LIST |  |  -->  | Taint [externally controlled string] | test.py:7 | test.py:7:15:7:15 | l |  |
+| Taint [externally controlled string] | test.py:7 | test.py:7:15:7:15 | l |  |  -->  | Taint externally controlled string | test.py:8 | test.py:8:10:8:10 | a |  |
+| Taint [externally controlled string] | test.py:7 | test.py:7:15:7:15 | l |  |  -->  | Taint externally controlled string | test.py:8 | test.py:8:13:8:13 | b |  |
+| Taint [externally controlled string] | test.py:7 | test.py:7:15:7:15 | l |  |  -->  | Taint externally controlled string | test.py:8 | test.py:8:16:8:16 | c |  |
 | Taint [externally controlled string] | test.py:12 | test.py:12:9:12:20 | TAINTED_LIST |  |  -->  | Taint [externally controlled string] | test.py:13 | test.py:13:17:13:17 | l |  |
+| Taint [externally controlled string] | test.py:13 | test.py:13:17:13:17 | l |  |  -->  | Taint externally controlled string | test.py:14 | test.py:14:10:14:10 | a |  |
+| Taint [externally controlled string] | test.py:13 | test.py:13:17:13:17 | l |  |  -->  | Taint externally controlled string | test.py:14 | test.py:14:13:14:13 | b |  |
+| Taint [externally controlled string] | test.py:13 | test.py:13:17:13:17 | l |  |  -->  | Taint externally controlled string | test.py:14 | test.py:14:16:14:16 | c |  |
 | Taint [externally controlled string] | test.py:18 | test.py:18:9:18:20 | TAINTED_LIST |  |  -->  | Taint [externally controlled string] | test.py:19 | test.py:19:11:19:11 | l |  |
 | Taint [externally controlled string] | test.py:18 | test.py:18:9:18:20 | TAINTED_LIST |  |  -->  | Taint [externally controlled string] | test.py:19 | test.py:19:14:19:14 | l |  |
 | Taint [externally controlled string] | test.py:18 | test.py:18:9:18:20 | TAINTED_LIST |  |  -->  | Taint [externally controlled string] | test.py:19 | test.py:19:17:19:17 | l |  |

python/ql/test/library-tests/taint/unpacking/TestTaint.expected

@@ -1,33 +1,33 @@
-| test.py:8 | unpacking | a | NO TAINT |
-| test.py:8 | unpacking | b | NO TAINT |
-| test.py:8 | unpacking | c | NO TAINT |
-| test.py:14 | unpacking_to_list | a | NO TAINT |
-| test.py:14 | unpacking_to_list | b | NO TAINT |
-| test.py:14 | unpacking_to_list | c | NO TAINT |
-| test.py:23 | nested | a1 | NO TAINT |
-| test.py:23 | nested | a2 | NO TAINT |
-| test.py:23 | nested | a3 | NO TAINT |
-| test.py:23 | nested | b | NO TAINT |
-| test.py:23 | nested | c | NO TAINT |
-| test.py:27 | nested | a1 | NO TAINT |
-| test.py:27 | nested | a2 | NO TAINT |
-| test.py:27 | nested | a3 | NO TAINT |
-| test.py:27 | nested | b | NO TAINT |
-| test.py:27 | nested | c | NO TAINT |
-| test.py:31 | nested | a1 | NO TAINT |
-| test.py:31 | nested | a2 | NO TAINT |
-| test.py:31 | nested | a3 | NO TAINT |
-| test.py:31 | nested | b | NO TAINT |
-| test.py:31 | nested | c | NO TAINT |
+| test.py:8 | unpacking | a | externally controlled string |
+| test.py:8 | unpacking | b | externally controlled string |
+| test.py:8 | unpacking | c | externally controlled string |
+| test.py:14 | unpacking_to_list | a | externally controlled string |
+| test.py:14 | unpacking_to_list | b | externally controlled string |
+| test.py:14 | unpacking_to_list | c | externally controlled string |
+| test.py:23 | nested | a1 | externally controlled string |
+| test.py:23 | nested | a2 | externally controlled string |
+| test.py:23 | nested | a3 | externally controlled string |
+| test.py:23 | nested | b | [externally controlled string] |
+| test.py:23 | nested | c | [externally controlled string] |
+| test.py:27 | nested | a1 | externally controlled string |
+| test.py:27 | nested | a2 | externally controlled string |
+| test.py:27 | nested | a3 | externally controlled string |
+| test.py:27 | nested | b | [externally controlled string] |
+| test.py:27 | nested | c | [externally controlled string] |
+| test.py:31 | nested | a1 | externally controlled string |
+| test.py:31 | nested | a2 | externally controlled string |
+| test.py:31 | nested | a3 | externally controlled string |
+| test.py:31 | nested | b | [externally controlled string] |
+| test.py:31 | nested | c | [externally controlled string] |
 | test.py:38 | unpack_from_set | a | NO TAINT |
 | test.py:38 | unpack_from_set | b | NO TAINT |
 | test.py:38 | unpack_from_set | c | NO TAINT |
-| test.py:48 | contrived_1 | a | NO TAINT |
-| test.py:48 | contrived_1 | b | NO TAINT |
-| test.py:48 | contrived_1 | c | NO TAINT |
-| test.py:48 | contrived_1 | d | NO TAINT |
-| test.py:48 | contrived_1 | e | NO TAINT |
-| test.py:48 | contrived_1 | f | NO TAINT |
+| test.py:48 | contrived_1 | a | externally controlled string |
+| test.py:48 | contrived_1 | b | externally controlled string |
+| test.py:48 | contrived_1 | c | externally controlled string |
+| test.py:48 | contrived_1 | d | externally controlled string |
+| test.py:48 | contrived_1 | e | externally controlled string |
+| test.py:48 | contrived_1 | f | externally controlled string |
 | test.py:56 | contrived_2 | a | NO TAINT |
 | test.py:56 | contrived_2 | b | NO TAINT |
 | test.py:56 | contrived_2 | c | NO TAINT |

python/ql/test/library-tests/web/django/HttpRedirectSinks.expected

@@ -0,0 +1,2 @@
+| test_1x.py:13:21:13:24 | django.redirect | externally controlled string |
+| test_2x_3x.py:13:21:13:24 | django.redirect | externally controlled string |

python/ql/test/library-tests/web/django/HttpRedirectSinks.ql

@@ -0,0 +1,7 @@
+import python
+import semmle.python.web.HttpRedirect
+import semmle.python.security.strings.Untrusted
+
+from HttpRedirectTaintSink sink, TaintKind kind
+where sink.sinks(kind)
+select sink, kind

python/ql/test/library-tests/web/django/HttpResponseSinks.expected

@@ -1,7 +1,23 @@
-| views.py:7:25:7:63 | django.Response(...) | externally controlled string |
-| views.py:11:25:11:52 | django.Response(...) | externally controlled string |
-| views.py:15:25:15:53 | django.Response(...) | externally controlled string |
-| views.py:23:29:23:60 | django.Response(...) | externally controlled string |
-| views.py:29:29:29:65 | django.Response(...) | externally controlled string |
-| views.py:34:25:34:63 | django.Response(...) | externally controlled string |
-| views.py:38:25:38:70 | django.Response(...) | externally controlled string |
+| views_1x.py:8:25:8:63 | django.Response(...) | externally controlled string |
+| views_1x.py:12:25:12:52 | django.Response(...) | externally controlled string |
+| views_1x.py:16:25:16:53 | django.Response(...) | externally controlled string |
+| views_1x.py:21:15:21:42 | django.Response.write(...) | externally controlled string |
+| views_1x.py:30:29:30:60 | django.Response(...) | externally controlled string |
+| views_1x.py:36:29:36:65 | django.Response(...) | externally controlled string |
+| views_1x.py:41:25:41:63 | django.Response(...) | externally controlled string |
+| views_1x.py:45:25:45:70 | django.Response(...) | externally controlled string |
+| views_1x.py:66:25:66:55 | django.Response(...) | externally controlled string |
+| views_1x.py:75:25:75:33 | django.Response(...) | externally controlled string |
+| views_2x_3x.py:8:25:8:63 | django.Response(...) | externally controlled string |
+| views_2x_3x.py:12:25:12:52 | django.Response(...) | externally controlled string |
+| views_2x_3x.py:16:25:16:53 | django.Response(...) | externally controlled string |
+| views_2x_3x.py:21:15:21:42 | django.Response.write(...) | externally controlled string |
+| views_2x_3x.py:30:29:30:60 | django.Response(...) | externally controlled string |
+| views_2x_3x.py:36:29:36:65 | django.Response(...) | externally controlled string |
+| views_2x_3x.py:41:25:41:63 | django.Response(...) | externally controlled string |
+| views_2x_3x.py:45:25:45:70 | django.Response(...) | externally controlled string |
+| views_2x_3x.py:66:25:66:40 | django.Response(...) | externally controlled string |
+| views_2x_3x.py:79:25:79:61 | django.Response(...) | externally controlled string |
+| views_2x_3x.py:82:25:82:69 | django.Response(...) | externally controlled string |
+| views_2x_3x.py:85:25:85:64 | django.Response(...) | externally controlled string |
+| views_2x_3x.py:88:25:88:32 | django.Response(...) | externally controlled string |

python/ql/test/library-tests/web/django/HttpSources.expected

@@ -1,19 +1,51 @@
-| test.py:5:19:5:25 | request | django.request.HttpRequest |
-| test.py:5:28:5:31 | path | externally controlled string |
-| test.py:11:19:11:25 | request | django.request.HttpRequest |
-| test.py:11:28:11:31 | path | externally controlled string |
-| views.py:6:19:6:25 | request | django.request.HttpRequest |
-| views.py:6:28:6:30 | foo | externally controlled string |
-| views.py:6:33:6:35 | bar | externally controlled string |
-| views.py:10:20:10:26 | request | django.request.HttpRequest |
-| views.py:14:21:14:27 | request | django.request.HttpRequest |
-| views.py:22:20:22:26 | request | django.request.HttpRequest |
-| views.py:28:19:28:25 | request | django.request.HttpRequest |
-| views.py:32:19:32:25 | request | django.request.HttpRequest |
-| views.py:32:28:32:38 | page_number | externally controlled string |
-| views.py:37:24:37:30 | request | django.request.HttpRequest |
-| views.py:37:33:37:36 | arg0 | externally controlled string |
-| views.py:37:39:37:42 | arg1 | externally controlled string |
-| views.py:57:15:57:21 | request | django.request.HttpRequest |
-| views.py:57:24:57:31 | username | externally controlled string |
-| views.py:66:30:66:36 | request | django.request.HttpRequest |
+| test_1x.py:6:19:6:25 | request | django.request.HttpRequest |
+| test_1x.py:6:28:6:31 | path | externally controlled string |
+| test_1x.py:12:19:12:25 | request | django.request.HttpRequest |
+| test_1x.py:12:28:12:31 | path | externally controlled string |
+| test_2x_3x.py:6:19:6:25 | request | django.request.HttpRequest |
+| test_2x_3x.py:6:28:6:31 | path | externally controlled string |
+| test_2x_3x.py:12:19:12:25 | request | django.request.HttpRequest |
+| test_2x_3x.py:12:28:12:31 | path | externally controlled string |
+| views_1x.py:7:19:7:25 | request | django.request.HttpRequest |
+| views_1x.py:7:28:7:30 | foo | externally controlled string |
+| views_1x.py:7:33:7:35 | bar | externally controlled string |
+| views_1x.py:11:20:11:26 | request | django.request.HttpRequest |
+| views_1x.py:15:21:15:27 | request | django.request.HttpRequest |
+| views_1x.py:19:21:19:27 | request | django.request.HttpRequest |
+| views_1x.py:29:20:29:26 | request | django.request.HttpRequest |
+| views_1x.py:29:29:29:37 | untrusted | externally controlled string |
+| views_1x.py:35:19:35:25 | request | django.request.HttpRequest |
+| views_1x.py:35:28:35:36 | untrusted | externally controlled string |
+| views_1x.py:39:19:39:25 | request | django.request.HttpRequest |
+| views_1x.py:39:28:39:38 | page_number | externally controlled string |
+| views_1x.py:44:24:44:30 | request | django.request.HttpRequest |
+| views_1x.py:44:33:44:36 | arg0 | externally controlled string |
+| views_1x.py:44:39:44:42 | arg1 | externally controlled string |
+| views_1x.py:65:15:65:21 | request | django.request.HttpRequest |
+| views_1x.py:65:24:65:31 | username | externally controlled string |
+| views_1x.py:74:13:74:19 | request | django.request.HttpRequest |
+| views_2x_3x.py:7:19:7:25 | request | django.request.HttpRequest |
+| views_2x_3x.py:7:28:7:30 | foo | externally controlled string |
+| views_2x_3x.py:7:33:7:35 | bar | externally controlled string |
+| views_2x_3x.py:11:20:11:26 | request | django.request.HttpRequest |
+| views_2x_3x.py:15:21:15:27 | request | django.request.HttpRequest |
+| views_2x_3x.py:19:21:19:27 | request | django.request.HttpRequest |
+| views_2x_3x.py:29:20:29:26 | request | django.request.HttpRequest |
+| views_2x_3x.py:29:29:29:37 | untrusted | externally controlled string |
+| views_2x_3x.py:35:19:35:25 | request | django.request.HttpRequest |
+| views_2x_3x.py:35:28:35:36 | untrusted | externally controlled string |
+| views_2x_3x.py:39:19:39:25 | request | django.request.HttpRequest |
+| views_2x_3x.py:39:28:39:38 | page_number | externally controlled string |
+| views_2x_3x.py:44:24:44:30 | request | django.request.HttpRequest |
+| views_2x_3x.py:44:33:44:36 | arg0 | externally controlled string |
+| views_2x_3x.py:44:39:44:42 | arg1 | externally controlled string |
+| views_2x_3x.py:78:17:78:23 | request | django.request.HttpRequest |
+| views_2x_3x.py:78:26:78:36 | page_number | externally controlled string |
+| views_2x_3x.py:81:17:81:23 | request | django.request.HttpRequest |
+| views_2x_3x.py:81:26:81:28 | foo | externally controlled string |
+| views_2x_3x.py:81:31:81:33 | bar | externally controlled string |
+| views_2x_3x.py:81:36:81:38 | baz | externally controlled string |
+| views_2x_3x.py:84:17:84:23 | request | django.request.HttpRequest |
+| views_2x_3x.py:84:26:84:28 | foo | externally controlled string |
+| views_2x_3x.py:84:31:84:33 | bar | externally controlled string |
+| views_2x_3x.py:87:26:87:32 | request | django.request.HttpRequest |

python/ql/test/library-tests/web/django/test_1x.py

@@ -1,3 +1,4 @@
+"""tests for Django 1.x"""
 from django.conf.urls import url
 from django.shortcuts import redirect, render
 

python/ql/test/library-tests/web/django/test_2x_3x.py

@@ -0,0 +1,19 @@
+"""tests for Django 2.x and 3.x"""
+from django.urls import path
+from django.shortcuts import redirect, render
+
+
+def with_template(request, path='default'):
+    env = {'path': path}
+    # We would need to understand django templates to know if this is safe or not
+    return render(request, 'possibly-vulnerable-template.html', env)
+
+
+def vuln_redirect(request, path):
+    return redirect(path)
+
+
+urlpatterns = [
+    path('/<path>', with_template),
+    path('/redirect/<path>', vuln_redirect),
+]

python/ql/test/library-tests/web/django/views_1x.py

@@ -1,3 +1,4 @@
+"""test of views for Django 1.x"""
 from django.conf.urls import patterns, url
 from django.http.response import HttpResponse
 from django.views.generic import View
@@ -15,16 +16,22 @@ def post_params_xss(request):
     return HttpResponse(request.POST.get("untrusted"))
 
 
+def http_resp_write(request):
+    rsp = HttpResponse()
+    rsp.write(request.GET.get("untrusted"))
+    return rsp
+
+
 class Foo(object):
     # Note: since Foo is used as the super type in a class view, it will be able to handle requests.
 
-    # TODO: Currently we don't flag `untrusted` as a DjangoRequestParameter
+
     def post(self, request, untrusted):
         return HttpResponse('Foo post: {}'.format(untrusted))
 
 
 class ClassView(View, Foo):
-    # TODO: Currently we don't flag `untrusted` as a DjangoRequestParameter
+
     def get(self, request, untrusted):
         return HttpResponse('ClassView get: {}'.format(untrusted))
 
@@ -42,6 +49,7 @@ urlpatterns = [
     url(r'^url_match/(?P<foo>[^/]+)/(?P<bar>[^/]+)$', url_match_xss),
     url(r'^get_params$', get_params_xss),
     url(r'^post_params$', post_params_xss),
+    url(r'^http_resp_write$', http_resp_write),
     url(r'^class_view/(?P<untrusted>.+)$', ClassView.as_view()),
 
     # one pattern to support `articles/page-<n>` and ensuring that articles/ goes to page-1
@@ -51,22 +59,21 @@ urlpatterns = [
     url(r'^([^/]+)/(?:foo|bar)/([^/]+)$', xxs_positional_arg, name='xxs_positional_arg'),
 ]
 
-
+################################################################################
 # Using patterns() for routing
 
 def show_user(request, username):
-    pass
+    return HttpResponse('show_user {}'.format(username))
 
 
 urlpatterns = patterns(url(r'^users/(?P<username>[^/]+)$', show_user))
 
-
+################################################################################
 # Show we understand the keyword arguments to django.conf.urls.url
 
-def we_understand_url_kwargs(request):
-    pass
-
+def kw_args(request):
+    return HttpResponse('kw_args')
 
 urlpatterns = [
-    url(view=we_understand_url_kwargs, regex=r'^specifying-as-kwargs-is-not-a-problem$')
+    url(view=kw_args, regex=r'^kw_args$')
 ]

python/ql/test/library-tests/web/django/views_2x_3x.py

@@ -0,0 +1,122 @@
+"""testing views for Django 2.x and 3.x"""
+from django.urls import path, re_path
+from django.http import HttpResponse
+from django.views import View
+
+
+def url_match_xss(request, foo, bar, no_taint=None):
+    return HttpResponse('url_match_xss: {} {}'.format(foo, bar))
+
+
+def get_params_xss(request):
+    return HttpResponse(request.GET.get("untrusted"))
+
+
+def post_params_xss(request):
+    return HttpResponse(request.POST.get("untrusted"))
+
+
+def http_resp_write(request):
+    rsp = HttpResponse()
+    rsp.write(request.GET.get("untrusted"))
+    return rsp
+
+
+class Foo(object):
+    # Note: since Foo is used as the super type in a class view, it will be able to handle requests.
+
+
+    def post(self, request, untrusted):
+        return HttpResponse('Foo post: {}'.format(untrusted))
+
+
+class ClassView(View, Foo):
+
+    def get(self, request, untrusted):
+        return HttpResponse('ClassView get: {}'.format(untrusted))
+
+
+def show_articles(request, page_number=1):
+    page_number = int(page_number)
+    return HttpResponse('articles page: {}'.format(page_number))
+
+
+def xxs_positional_arg(request, arg0, arg1, no_taint=None):
+    return HttpResponse('xxs_positional_arg: {} {}'.format(arg0, arg1))
+
+
+urlpatterns = [
+    re_path(r'^url_match/(?P<foo>[^/]+)/(?P<bar>[^/]+)$', url_match_xss),
+    re_path(r'^get_params$', get_params_xss),
+    re_path(r'^post_params$', post_params_xss),
+    re_path(r'^http_resp_write$', http_resp_write),
+    re_path(r'^class_view/(?P<untrusted>.+)$', ClassView.as_view()),
+
+    # one pattern to support `articles/page-<n>` and ensuring that articles/ goes to page-1
+    re_path(r'articles/^(?:page-(?P<page_number>\d+)/)?$', show_articles),
+    # passing as positional argument is not the recommended way of doing things, but it is certainly
+    # possible
+    re_path(r'^([^/]+)/(?:foo|bar)/([^/]+)$', xxs_positional_arg, name='xxs_positional_arg'),
+]
+
+
+# Show we understand the keyword arguments to from django.urls.re_path
+
+def re_path_kwargs(request):
+    return HttpResponse('re_path_kwargs')
+
+
+urlpatterns = [
+    re_path(view=re_path_kwargs, regex=r'^specifying-as-kwargs-is-not-a-problem$')
+]
+
+################################################################################
+# Using path
+################################################################################
+
+# saying page_number is an externally controlled *string* is a bit strange, when we have an int converter :O
+def page_number(request, page_number=1):
+    return HttpResponse('page_number: {}'.format(page_number))
+
+def foo_bar_baz(request, foo, bar, baz):
+    return HttpResponse('foo_bar_baz: {} {} {}'.format(foo, bar, baz))
+
+def path_kwargs(request, foo, bar):
+    return HttpResponse('path_kwargs: {} {} {}'.format(foo, bar))
+
+def not_valid_identifier(request):
+    return HttpResponse('<foo!>')
+
+urlpatterns = [
+    path('articles/', page_number),
+    path('articles/page-<int:page_number>', page_number),
+    path('<int:foo>/<str:bar>/<baz>', foo_bar_baz, name='foo-bar-baz'),
+
+    path(view=path_kwargs, route='<foo>/<bar>'),
+
+    # We should not report there is a request parameter called `not_valid!`
+    path('not_valid/<not_valid!>', not_valid_identifier),
+]
+
+
+################################################################################
+
+
+# We should abort if a decorator is used. As demonstrated below, anything might happen
+
+# def reverse_kwargs(f):
+#     @wraps(f)
+#     def f_(*args, **kwargs):
+#         new_kwargs = dict()
+#         for key, value in kwargs.items():
+#             new_kwargs[key[::-1]] = value
+#         return f(*args, **new_kwargs)
+#     return f_
+
+# @reverse_kwargs
+# def decorators_can_do_anything(request, oof, foo=None):
+#     return HttpResponse('This is a mess'[::-1])
+
+# urlpatterns = [
+#     path('rev/<foo>', decorators_can_do_anything),
+# ]

python/ql/test/query-tests/Expressions/callable/NonCallableCalled.expected

@@ -3,3 +3,4 @@
 | test.py:18:5:18:8 | List() | Call to a $@ of $@. | test.py:18:5:18:6 | List | non-callable | file://:0:0:0:0 | builtin-class list | builtin-class list |
 | test.py:26:9:26:16 | non() | Call to a $@ of $@. | test.py:15:11:15:23 | NonCallable() | non-callable | test.py:3:1:3:26 | class NonCallable | class NonCallable |
 | test.py:47:12:47:27 | NotImplemented() | Call to a $@ of $@. | test.py:47:12:47:25 | NotImplemented | non-callable | file://:0:0:0:0 | builtin-class NotImplementedType | builtin-class NotImplementedType |
+| test.py:63:16:63:27 | cls() | Call to a $@ of $@. | test.py:62:22:62:24 | cls | non-callable | test.py:56:1:56:18 | class Foo | class Foo |

python/ql/test/query-tests/Expressions/callable/test.py

@@ -46,3 +46,21 @@ def foo():
 def bar():
     return NotImplemented()
 
+
+# FP due to decorator
+# https://github.com/Semmle/ql/issues/3113
+def some_decorator(func):
+    print("this could be tricky for our analysis")
+    return func
+
+class Foo(object):
+    def __init__(self, arg):
+        self.arg = arg
+
+    @some_decorator
+    @classmethod
+    def new_instance(cls, new_arg):
+        return cls(new_arg) # TODO: FP
+
+f1 = Foo(1)
+f2 = f1.new_instance(2)

python/ql/test/query-tests/Security/CWE-022/TarSlip.expected

@@ -15,8 +15,15 @@ edges
 | tarslip.py:34:14:34:16 | tarfile.open | tarslip.py:34:1:34:17 | tarfile.entry |
 | tarslip.py:40:7:40:39 | tarfile.open | tarslip.py:41:24:41:26 | tarfile.open |
 | tarslip.py:40:7:40:39 | tarfile.open | tarslip.py:41:24:41:26 | tarfile.open |
+| tarslip.py:56:7:56:39 | tarfile.open | tarslip.py:57:14:57:16 | tarfile.open |
+| tarslip.py:56:7:56:39 | tarfile.open | tarslip.py:57:14:57:16 | tarfile.open |
+| tarslip.py:57:1:57:17 | tarfile.entry | tarslip.py:59:21:59:25 | tarfile.entry |
+| tarslip.py:57:1:57:17 | tarfile.entry | tarslip.py:59:21:59:25 | tarfile.entry |
+| tarslip.py:57:14:57:16 | tarfile.open | tarslip.py:57:1:57:17 | tarfile.entry |
+| tarslip.py:57:14:57:16 | tarfile.open | tarslip.py:57:1:57:17 | tarfile.entry |
 #select
 | tarslip.py:13:1:13:3 | tar | tarslip.py:12:7:12:39 | tarfile.open | tarslip.py:13:1:13:3 | tarfile.open | Extraction of tarfile from $@ | tarslip.py:12:7:12:39 | Attribute() | a potentially untrusted source |
 | tarslip.py:18:17:18:21 | entry | tarslip.py:16:7:16:39 | tarfile.open | tarslip.py:18:17:18:21 | tarfile.entry | Extraction of tarfile from $@ | tarslip.py:16:7:16:39 | Attribute() | a potentially untrusted source |
 | tarslip.py:37:17:37:21 | entry | tarslip.py:33:7:33:39 | tarfile.open | tarslip.py:37:17:37:21 | tarfile.entry | Extraction of tarfile from $@ | tarslip.py:33:7:33:39 | Attribute() | a potentially untrusted source |
 | tarslip.py:41:24:41:26 | tar | tarslip.py:40:7:40:39 | tarfile.open | tarslip.py:41:24:41:26 | tarfile.open | Extraction of tarfile from $@ | tarslip.py:40:7:40:39 | Attribute() | a potentially untrusted source |
+| tarslip.py:59:21:59:25 | entry | tarslip.py:56:7:56:39 | tarfile.open | tarslip.py:59:21:59:25 | tarfile.entry | Extraction of tarfile from $@ | tarslip.py:56:7:56:39 | Attribute() | a potentially untrusted source |

python/ql/test/query-tests/Security/CWE-022/tarslip.py

@@ -50,3 +50,33 @@ def safemembers(members):
 
 tar = tarfile.open(unsafe_filename_tar)
 tar.extractall(members=safemembers(tar))
+
+
+# Wrong sanitizer (is missing not)
+tar = tarfile.open(unsafe_filename_tar)
+for entry in tar:
+    if os.path.isabs(entry.name) or ".." in entry.name:
+        tar.extract(entry, "/tmp/unpack/")
+
+
+# OK Sanitized using not
+tar = tarfile.open(unsafe_filename_tar)
+for entry in tar:
+    if not (os.path.isabs(entry.name) or ".." in entry.name):
+        tar.extract(entry, "/tmp/unpack/")
+
+# The following two variants are included by purpose, since by default there is a
+# difference in handling `not x` and `not (x or False)` when overriding
+# Sanitizer.sanitizingEdge. We want to ensure we handle both consistently.
+
+# Not reported, although vulnerable to '..'
+tar = tarfile.open(unsafe_filename_tar)
+for entry in tar:
+    if not (os.path.isabs(entry.name) or False):
+        tar.extract(entry, "/tmp/unpack/")
+
+# Not reported, although vulnerable to '..'
+tar = tarfile.open(unsafe_filename_tar)
+for entry in tar:
+    if not os.path.isabs(entry.name):
+        tar.extract(entry, "/tmp/unpack/")

python/ql/test/query-tests/Security/lib/django/conf/urls.py

@@ -2,5 +2,6 @@
 def url(regex, view, kwargs=None, name=None):
     pass
 
+
 def patterns(*urls):
     pass

python/ql/test/query-tests/Security/lib/django/shortcuts.py

@@ -0,0 +1,8 @@
+# see https://docs.djangoproject.com/en/1.11/_modules/django/shortcuts/#redirect
+# https://github.com/django/django/blob/86908785076b2bbc31b908781da6b6ad1779b18b/django/shortcuts.py
+
+def render(request, template_name, context=None, content_type=None, status=None, using=None):
+    pass
+
+def redirect(to, *args, **kwargs):
+    pass

python/ql/test/query-tests/Security/lib/django/urls.py

@@ -0,0 +1,7 @@
+from functools import partial
+
+def _path(route, view, kwargs=None, name=None, Pattern=None):
+   pass
+
+path = partial(_path, Pattern='RoutePattern (but this is a mock)')
+re_path = partial(_path, Pattern='RegexPattern (but this is a mock)')

python/ql/test/query-tests/Security/lib/django/views/__init__.py

@@ -0,0 +1,3 @@
+# For django 2.x and 3.x
+class View:
+    pass

python/ql/test/query-tests/Security/lib/django/views/generic.py

@@ -1,2 +1,3 @@
+# For django 1.x
 class View:
     pass

python/ql/test/query-tests/Security/lib/fabric/__init__.py

@@ -0,0 +1,3 @@
+from .connection import Connection
+from .group import Group, SerialGroup, ThreadingGroup
+from .tasks import task

python/ql/test/query-tests/Security/lib/fabric/api.py

@@ -0,0 +1,25 @@
+# For the 1.x version
+
+def needs_host(func):
+    @wraps(func)
+    def inner(*args, **kwargs):
+        return func(*args, **kwargs)
+    return inner
+
+
+def local(command, capture=False, shell=None):
+    pass
+
+
+@needs_host
+def run(command, shell=True, pty=True, combine_stderr=None, quiet=False,
+    warn_only=False, stdout=None, stderr=None, timeout=None, shell_escape=None,
+    capture_buffer_size=None):
+    pass
+
+
+@needs_host
+def sudo(command, shell=True, pty=True, combine_stderr=None, user=None,
+    quiet=False, warn_only=False, stdout=None, stderr=None, group=None,
+    timeout=None, shell_escape=None, capture_buffer_size=None):
+    pass

python/ql/test/query-tests/Security/lib/fabric/connection.py

@@ -0,0 +1,15 @@
+from invoke import Context
+
+@decorator
+def opens(method, self, *args, **kwargs):
+    self.open()
+    return method(self, *args, **kwargs)
+
+class Connection(Context):
+
+    def open(self):
+        pass
+
+    @opens
+    def run(self, command, **kwargs):
+        pass

python/ql/test/query-tests/Security/lib/fabric/group.py

@@ -0,0 +1,11 @@
+class Group(list):
+    def run(self, *args, **kwargs):
+        raise NotImplementedError
+
+class SerialGroup(Group):
+    def run(self, *args, **kwargs):
+        pass
+
+class ThreadingGroup(Group):
+    def run(self, *args, **kwargs):
+        pass

python/ql/test/query-tests/Security/lib/fabric/tasks.py

@@ -0,0 +1,2 @@
+def task(*args, **kwargs):
+    pass

python/ql/test/query-tests/Security/lib/invoke/__init__.py

@@ -0,0 +1,8 @@
+from .context import Context
+from .tasks import task
+
+def run(command, **kwargs):
+    pass
+
+def sudo(command, **kwargs):
+    pass

python/ql/test/query-tests/Security/lib/invoke/context.py

@@ -0,0 +1,3 @@
+class Context(object):
+    def run(self, command, **kwargs):
+        pass

python/ql/test/query-tests/Security/lib/invoke/tasks.py

@@ -0,0 +1,2 @@
+def task(*args, **kwargs):
+    pass