Merge branch 'main' into fastapi

python/change-notes/2021-10-26-ruamel.yaml-modeling.md

@@ -0,0 +1,2 @@
+lgtm,codescanning
+* Added modeling of the `ruamel.yaml` PyPI package, resulting in additional sinks for the _Deserializing untrusted input_ (`py/unsafe-deserialization`) query (since `ruamel.yaml.load` can lead to code execution).

python/ql/lib/semmle/python/Frameworks.qll

@@ -27,6 +27,7 @@ private import semmle.python.frameworks.Psycopg2
 private import semmle.python.frameworks.Pydantic
 private import semmle.python.frameworks.PyMySQL
 private import semmle.python.frameworks.Rsa
+private import semmle.python.frameworks.RuamelYaml
 private import semmle.python.frameworks.Simplejson
 private import semmle.python.frameworks.SqlAlchemy
 private import semmle.python.frameworks.Starlette

python/ql/lib/semmle/python/frameworks/RuamelYaml.qll

@@ -0,0 +1,57 @@
+/**
+ * Provides classes modeling security-relevant aspects of the `ruamel.yaml` PyPI package
+ *
+ * See
+ * - https://pypi.org/project/ruamel.yaml/
+ */
+
+private import python
+private import semmle.python.dataflow.new.DataFlow
+private import semmle.python.Concepts
+private import semmle.python.ApiGraphs
+
+/**
+ * Provides models for the `ruamel.yaml` PyPI package.
+ *
+ * See
+ * - https://pypi.org/project/ruamel.yaml/
+ */
+private module RuamelYaml {
+  // Note: `ruamel.yaml` is a fork of the `PyYAML` PyPI package, so that's why the
+  // interface is so similar.
+  /**
+   * A call to any of the loading functions in `yaml` (`load`, `load_all`, `safe_load`, `safe_load_all`)
+   *
+   * See https://pyyaml.org/wiki/PyYAMLDocumentation (you will have to scroll down).
+   */
+  private class RuamelYamlLoadCall extends Decoding::Range, DataFlow::CallCfgNode {
+    string func_name;
+
+    RuamelYamlLoadCall() {
+      func_name in ["load", "load_all", "safe_load", "safe_load_all"] and
+      this = API::moduleImport("ruamel").getMember("yaml").getMember(func_name).getACall()
+    }
+
+    override predicate mayExecuteInput() {
+      func_name in ["load", "load_all"] and
+      // If the `Loader` argument is not set, the default loader will be used, which is
+      // not safe. The only safe loaders are `SafeLoader` or `BaseLoader` (and their
+      // variants with C implementation).
+      not exists(DataFlow::Node loader_arg |
+        loader_arg in [this.getArg(1), this.getArgByName("Loader")]
+      |
+        loader_arg =
+          API::moduleImport("ruamel")
+              .getMember("yaml")
+              .getMember(["SafeLoader", "BaseLoader", "CSafeLoader", "CBaseLoader"])
+              .getAUse()
+      )
+    }
+
+    override DataFlow::Node getAnInput() { result in [this.getArg(0), this.getArgByName("stream")] }
+
+    override DataFlow::Node getOutput() { result = this }
+
+    override string getFormat() { result = "YAML" }
+  }
+}

python/ql/lib/semmle/python/frameworks/Yaml.qll

@@ -9,7 +9,6 @@
 
 private import python
 private import semmle.python.dataflow.new.DataFlow
-private import semmle.python.dataflow.new.RemoteFlowSources
 private import semmle.python.Concepts
 private import semmle.python.ApiGraphs
 
@@ -41,11 +40,17 @@ private module Yaml {
     }
 
     /**
-     * This function was thought safe from the 5.1 release in 2017, when the default loader was changed to `FullLoader`.
-     * In 2020 new exploits were found, meaning it's not safe. The Current plan is to change the default to `SafeLoader` in release 6.0
-     * (as explained in https://github.com/yaml/pyyaml/issues/420#issuecomment-696752389).
-     * Until 6.0 is released, we will mark `yaml.load` as possibly leading to arbitrary code execution.
-     * See https://github.com/yaml/pyyaml/wiki/PyYAML-yaml.load(input)-Deprecation for more details.
+     * This function was thought safe from the 5.1 release in 2017, when the default
+     * loader was changed to `FullLoader` (see
+     * https://github.com/yaml/pyyaml/wiki/PyYAML-yaml.load(input)-Deprecation).
+     *
+     * In 2020 new exploits were found, meaning it's not safe. With the 6.0 release (see
+     * https://github.com/yaml/pyyaml/commit/8cdff2c80573b8be8e8ad28929264a913a63aa33),
+     * when using `load` and `load_all` you are now required to specify a Loader. But
+     * from what I (@RasmusWL) can gather, `FullLoader` is not to be considered safe,
+     * although known exploits have been mitigated (is at least my impression). Also see
+     * https://github.com/yaml/pyyaml/issues/420#issuecomment-696752389 for more
+     * details.
      */
     override predicate mayExecuteInput() {
       func_name in ["full_load", "full_load_all", "unsafe_load", "unsafe_load_all"]
@@ -63,7 +68,7 @@ private module Yaml {
       )
     }
 
-    override DataFlow::Node getAnInput() { result = this.getArg(0) }
+    override DataFlow::Node getAnInput() { result in [this.getArg(0), this.getArgByName("stream")] }
 
     override DataFlow::Node getOutput() { result = this }
 

python/ql/lib/semmle/python/regex.qll

@@ -102,15 +102,7 @@ string mode_from_node(DataFlow::Node node) { node = re_flag_tracker(result) }
  * Gets a regular expression mode flag associated with the given value.
  */
 deprecated string mode_from_mode_object(Value obj) {
-  (
-    result = "DEBUG" or
-    result = "IGNORECASE" or
-    result = "LOCALE" or
-    result = "MULTILINE" or
-    result = "DOTALL" or
-    result = "UNICODE" or
-    result = "VERBOSE"
-  ) and
+  result in ["DEBUG", "IGNORECASE", "LOCALE", "MULTILINE", "DOTALL", "UNICODE", "VERBOSE"] and
   exists(int flag |
     flag = Value::named("sre_constants.SRE_FLAG_" + result).(OI::ObjectInternal).intValue() and
     obj.(OI::ObjectInternal).intValue().bitAnd(flag) = flag
@@ -611,14 +603,7 @@ abstract class RegexString extends Expr {
     this.getChar(start + 1) = "?" and
     end = start + 3 and
     c = this.getChar(start + 2) and
-    (
-      c = "i" or
-      c = "L" or
-      c = "m" or
-      c = "s" or
-      c = "u" or
-      c = "x"
-    )
+    c in ["i", "L", "m", "s", "u", "x"]
   }
 
   /**

python/ql/lib/semmle/python/security/Exceptions.qll

@@ -74,13 +74,10 @@ class ExceptionInfoSequence extends SequenceKind {
 class CallToTracebackFunction extends ErrorInfoSource {
   CallToTracebackFunction() {
     exists(string name |
-      name = "extract_tb" or
-      name = "extract_stack" or
-      name = "format_list" or
-      name = "format_exception_only" or
-      name = "format_exception" or
-      name = "format_tb" or
-      name = "format_stack"
+      name in [
+          "extract_tb", "extract_stack", "format_list", "format_exception_only", "format_exception",
+          "format_tb", "format_stack"
+        ]
     |
       this = traceback_function(name).getACall()
     )

python/ql/lib/semmle/python/types/Extensions.qll

@@ -112,14 +112,7 @@ class BottleRoutePointToExtension extends PointsToExtension {
 
 /* Python 3.6+ regex module constants */
 string short_flag(string flag) {
-  (
-    flag = "ASCII" or
-    flag = "IGNORECASE" or
-    flag = "LOCALE" or
-    flag = "UNICODE" or
-    flag = "MULTILINE" or
-    flag = "TEMPLATE"
-  ) and
+  flag in ["ASCII", "IGNORECASE", "LOCALE", "UNICODE", "MULTILINE", "TEMPLATE"] and
   result = flag.prefix(1)
   or
   flag = "DOTALL" and result = "S"

python/ql/lib/semmle/python/types/FunctionObject.qll

@@ -183,6 +183,7 @@ class PyFunctionObject extends FunctionObject {
   }
 
   /** Factored out to help join ordering */
+  pragma[noinline]
   private predicate implicitlyReturns(Object none_, ClassObject noneType) {
     noneType = theNoneType() and
     not this.getFunction().isGenerator() and

python/ql/lib/semmle/python/web/HttpConstants.qll

@@ -1,13 +1,5 @@
 /** Gets an HTTP verb, in upper case */
-string httpVerb() {
-  result = "GET" or
-  result = "POST" or
-  result = "PUT" or
-  result = "PATCH" or
-  result = "DELETE" or
-  result = "OPTIONS" or
-  result = "HEAD"
-}
+string httpVerb() { result in ["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD"] }
 
 /** Gets an HTTP verb, in lower case */
 string httpVerbLower() { result = httpVerb().toLowerCase() }

python/ql/lib/semmle/python/web/django/Model.qll

@@ -15,31 +15,11 @@ class DjangoDbTableObjects extends TaintKind {
 
   override TaintKind getTaintOfMethodResult(string name) {
     result = this and
-    (
-      name = "filter" or
-      name = "exclude" or
-      name = "annotate" or
-      name = "order_by" or
-      name = "reverse" or
-      name = "distinct" or
-      name = "values" or
-      name = "values_list" or
-      name = "dates" or
-      name = "datetimes" or
-      name = "none" or
-      name = "all" or
-      name = "union" or
-      name = "intersection" or
-      name = "difference" or
-      name = "select_related" or
-      name = "prefetch_related" or
-      name = "extra" or
-      name = "defer" or
-      name = "only" or
-      name = "using" or
-      name = "select_for_update" or
-      name = "raw"
-    )
+    name in [
+        "filter", "exclude", "none", "all", "union", "intersection", "difference", "select_related",
+        "prefetch_related", "extra", "defer", "only", "annotate", "using", "select_for_update",
+        "raw", "order_by", "reverse", "distinct", "values", "values_list", "dates", "datetimes"
+      ]
   }
 }
 

python/ql/lib/semmle/python/web/falcon/Request.qll

@@ -12,13 +12,7 @@ class FalconRequest extends TaintKind {
     name = "env" and result instanceof WsgiEnvironment
     or
     result instanceof ExternalStringKind and
-    (
-      name = "uri" or
-      name = "url" or
-      name = "forwarded_uri" or
-      name = "relative_uri" or
-      name = "query_string"
-    )
+    name in ["uri", "url", "forwarded_uri", "relative_uri", "query_string"]
     or
     result instanceof ExternalStringDictKind and
     (name = "cookies" or name = "params")

python/ql/lib/semmle/python/web/flask/Request.qll

@@ -32,12 +32,7 @@ class FlaskRequestData extends HttpRequestTaintSource {
 class FlaskRequestArgs extends HttpRequestTaintSource {
   FlaskRequestArgs() {
     exists(string attr | flask_request_attr(this, attr) |
-      attr = "args" or
-      attr = "form" or
-      attr = "values" or
-      attr = "files" or
-      attr = "headers" or
-      attr = "json"
+      attr in ["args", "form", "values", "files", "headers", "json"]
     )
   }
 

python/ql/src/Classes/UselessClass.ql

@@ -52,11 +52,7 @@ predicate is_stateful(Class c) {
     call.getFunc() = a and
     a.getName() = name
   |
-    name = "pop" or
-    name = "remove" or
-    name = "discard" or
-    name = "extend" or
-    name = "append"
+    name in ["pop", "remove", "discard", "extend", "append"]
   )
 }
 

python/ql/src/Functions/IncorrectRaiseInSpecialMethod.ql

@@ -22,49 +22,14 @@ private predicate indexing_method(string name) {
 }
 
 private predicate arithmetic_method(string name) {
-  name = "__add__" or
-  name = "__sub__" or
-  name = "__div__" or
-  name = "__pos__" or
-  name = "__abs__" or
-  name = "__floordiv__" or
-  name = "__div__" or
-  name = "__divmod__" or
-  name = "__lshift__" or
-  name = "__and__" or
-  name = "__or__" or
-  name = "__xor__" or
-  name = "__rshift__" or
-  name = "__pow__" or
-  name = "__mul__" or
-  name = "__neg__" or
-  name = "__radd__" or
-  name = "__rsub__" or
-  name = "__rdiv__" or
-  name = "__rfloordiv__" or
-  name = "__rdiv__" or
-  name = "__rlshift__" or
-  name = "__rand__" or
-  name = "__ror__" or
-  name = "__rxor__" or
-  name = "__rrshift__" or
-  name = "__rpow__" or
-  name = "__rmul__" or
-  name = "__truediv__" or
-  name = "__rtruediv__" or
-  name = "__iadd__" or
-  name = "__isub__" or
-  name = "__idiv__" or
-  name = "__ifloordiv__" or
-  name = "__idiv__" or
-  name = "__ilshift__" or
-  name = "__iand__" or
-  name = "__ior__" or
-  name = "__ixor__" or
-  name = "__irshift__" or
-  name = "__ipow__" or
-  name = "__imul__" or
-  name = "__itruediv__"
+  name in [
+      "__add__", "__sub__", "__or__", "__xor__", "__rshift__", "__pow__", "__mul__", "__neg__",
+      "__radd__", "__rsub__", "__rdiv__", "__rfloordiv__", "__div__", "__rdiv__", "__rlshift__",
+      "__rand__", "__ror__", "__rxor__", "__rrshift__", "__rpow__", "__rmul__", "__truediv__",
+      "__rtruediv__", "__pos__", "__iadd__", "__isub__", "__idiv__", "__ifloordiv__", "__idiv__",
+      "__ilshift__", "__iand__", "__ior__", "__ixor__", "__irshift__", "__abs__", "__ipow__",
+      "__imul__", "__itruediv__", "__floordiv__", "__div__", "__divmod__", "__lshift__", "__and__"
+    ]
 }
 
 private predicate ordering_method(string name) {

python/ql/src/Functions/SignatureSpecialMethods.ql

@@ -13,98 +13,29 @@
 import python
 
 predicate is_unary_op(string name) {
-  name = "__del__" or
-  name = "__repr__" or
-  name = "__str__" or
-  name = "__hash__" or
-  name = "__bool__" or
-  name = "__nonzero__" or
-  name = "__unicode__" or
-  name = "__len__" or
-  name = "__iter__" or
-  name = "__reversed__" or
-  name = "__neg__" or
-  name = "__pos__" or
-  name = "__abs__" or
-  name = "__invert__" or
-  name = "__complex__" or
-  name = "__int__" or
-  name = "__float__" or
-  name = "__long__" or
-  name = "__oct__" or
-  name = "__hex__" or
-  name = "__index__" or
-  name = "__enter__"
+  name in [
+      "__del__", "__repr__", "__neg__", "__pos__", "__abs__", "__invert__", "__complex__",
+      "__int__", "__float__", "__long__", "__oct__", "__hex__", "__str__", "__index__", "__enter__",
+      "__hash__", "__bool__", "__nonzero__", "__unicode__", "__len__", "__iter__", "__reversed__"
+    ]
 }
 
 predicate is_binary_op(string name) {
-  name = "__lt__" or
-  name = "__le__" or
-  name = "__eq__" or
-  name = "__ne__" or
-  name = "__gt__" or
-  name = "__ge__" or
-  name = "__cmp__" or
-  name = "__rcmp__" or
-  name = "__getattr___" or
-  name = "__getattribute___" or
-  name = "__delattr__" or
-  name = "__delete__" or
-  name = "__instancecheck__" or
-  name = "__subclasscheck__" or
-  name = "__getitem__" or
-  name = "__delitem__" or
-  name = "__contains__" or
-  name = "__add__" or
-  name = "__sub__" or
-  name = "__mul__" or
-  name = "__floordiv__" or
-  name = "__div__" or
-  name = "__truediv__" or
-  name = "__mod__" or
-  name = "__divmod__" or
-  name = "__lshift__" or
-  name = "__rshift__" or
-  name = "__and__" or
-  name = "__xor__" or
-  name = "__or__" or
-  name = "__radd__" or
-  name = "__rsub__" or
-  name = "__rmul__" or
-  name = "__rfloordiv__" or
-  name = "__rdiv__" or
-  name = "__rtruediv__" or
-  name = "__rmod__" or
-  name = "__rdivmod__" or
-  name = "__rpow__" or
-  name = "__rlshift__" or
-  name = "__rrshift__" or
-  name = "__rand__" or
-  name = "__rxor__" or
-  name = "__ror__" or
-  name = "__iadd__" or
-  name = "__isub__" or
-  name = "__imul__" or
-  name = "__ifloordiv__" or
-  name = "__idiv__" or
-  name = "__itruediv__" or
-  name = "__imod__" or
-  name = "__idivmod__" or
-  name = "__ipow__" or
-  name = "__ilshift__" or
-  name = "__irshift__" or
-  name = "__iand__" or
-  name = "__ixor__" or
-  name = "__ior__" or
-  name = "__coerce__"
+  name in [
+      "__lt__", "__le__", "__delattr__", "__delete__", "__instancecheck__", "__subclasscheck__",
+      "__getitem__", "__delitem__", "__contains__", "__add__", "__sub__", "__mul__", "__eq__",
+      "__floordiv__", "__div__", "__truediv__", "__mod__", "__divmod__", "__lshift__", "__rshift__",
+      "__and__", "__xor__", "__or__", "__ne__", "__radd__", "__rsub__", "__rmul__", "__rfloordiv__",
+      "__rdiv__", "__rtruediv__", "__rmod__", "__rdivmod__", "__rpow__", "__rlshift__", "__gt__",
+      "__rrshift__", "__rand__", "__rxor__", "__ror__", "__iadd__", "__isub__", "__imul__",
+      "__ifloordiv__", "__idiv__", "__itruediv__", "__ge__", "__imod__", "__idivmod__", "__ipow__",
+      "__ilshift__", "__irshift__", "__iand__", "__ixor__", "__ior__", "__coerce__", "__cmp__",
+      "__rcmp__", "__getattr___", "__getattribute___"
+    ]
 }
 
 predicate is_ternary_op(string name) {
-  name = "__setattr__" or
-  name = "__set__" or
-  name = "__setitem__" or
-  name = "__getslice__" or
-  name = "__delslice__"
+  name in ["__setattr__", "__set__", "__setitem__", "__getslice__", "__delslice__"]
 }
 
 predicate is_quad_op(string name) { name = "__setslice__" or name = "__exit__" }

python/ql/src/Lexical/CommentedOutCode.qll

@@ -297,41 +297,17 @@ private predicate file_or_url(Comment c) {
   c.getText().regexpMatch("#[^'\"]+(\\[a-zA-Z]\\w*)+\\.[a-zA-Z]+.*")
 }
 
-private string operator_keyword() {
-  result = "import" or
-  result = "and" or
-  result = "is" or
-  result = "or" or
-  result = "in" or
-  result = "not" or
-  result = "as"
-}
+private string operator_keyword() { result in ["import", "and", "is", "or", "in", "not", "as"] }
 
 private string keyword_requiring_colon() {
-  result = "try" or
-  result = "while" or
-  result = "elif" or
-  result = "else" or
-  result = "if" or
-  result = "except" or
-  result = "def" or
-  result = "class"
+  result in ["try", "while", "elif", "else", "if", "except", "def", "class"]
 }
 
 private string other_keyword() {
-  result = "del" or
-  result = "lambda" or
-  result = "from" or
-  result = "global" or
-  result = "with" or
-  result = "assert" or
-  result = "yield" or
-  result = "finally" or
-  result = "print" or
-  result = "exec" or
-  result = "raise" or
-  result = "return" or
-  result = "for"
+  result in [
+      "del", "lambda", "raise", "return", "for", "from", "global", "with", "assert", "yield",
+      "finally", "print", "exec"
+    ]
 }
 
 private string a_keyword() {

python/ql/src/Statements/SideEffectInAssert.ql

@@ -15,16 +15,9 @@ import python
 
 predicate func_with_side_effects(Expr e) {
   exists(string name | name = e.(Attribute).getName() or name = e.(Name).getId() |
-    name = "print" or
-    name = "write" or
-    name = "append" or
-    name = "pop" or
-    name = "remove" or
-    name = "discard" or
-    name = "delete" or
-    name = "close" or
-    name = "open" or
-    name = "exit"
+    name in [
+        "print", "write", "append", "pop", "remove", "discard", "delete", "close", "open", "exit"
+      ]
   )
 }
 

python/ql/src/analysis/Consistency.ql

@@ -8,15 +8,10 @@ import python
 import DefinitionTracking
 
 predicate uniqueness_error(int number, string what, string problem) {
-  (
-    what = "toString" or
-    what = "getLocation" or
-    what = "getNode" or
-    what = "getDefinition" or
-    what = "getEntryNode" or
-    what = "getOrigin" or
-    what = "getAnInferredType"
-  ) and
+  what in [
+      "toString", "getLocation", "getNode", "getDefinition", "getEntryNode", "getOrigin",
+      "getAnInferredType"
+    ] and
   (
     number = 0 and problem = "no results for " + what + "()"
     or

python/ql/test/library-tests/PointsTo/api/Value.ql

@@ -3,11 +3,5 @@ import python
 from Value val, string name
 where
   val = Value::named(name) and
-  (
-    name = "bool" or
-    name = "sys" or
-    name = "sys.argv" or
-    name = "ValueError" or
-    name = "slice"
-  )
+  name in ["bool", "sys", "sys.argv", "ValueError", "slice"]
 select val, name

python/ql/test/library-tests/frameworks/ruamel.yaml/ConceptsTest.ql

@@ -0,0 +1,2 @@
+import python
+import experimental.meta.ConceptsTest

python/ql/test/library-tests/frameworks/ruamel.yaml/Decoding.py

@@ -0,0 +1,33 @@
+import ruamel.yaml
+
+# Unsafe:
+ruamel.yaml.load(payload)  # $ decodeInput=payload decodeOutput=ruamel.yaml.load(..) decodeFormat=YAML decodeMayExecuteInput
+ruamel.yaml.load(stream=payload)  # $ decodeInput=payload decodeOutput=ruamel.yaml.load(..) decodeFormat=YAML decodeMayExecuteInput
+ruamel.yaml.load(payload, ruamel.yaml.Loader)  # $ decodeInput=payload decodeOutput=ruamel.yaml.load(..) decodeFormat=YAML decodeMayExecuteInput
+
+# Safe:
+ruamel.yaml.load(payload, ruamel.yaml.SafeLoader)  # $  decodeInput=payload decodeOutput=ruamel.yaml.load(..) decodeFormat=YAML
+ruamel.yaml.load(payload, Loader=ruamel.yaml.SafeLoader)  # $ decodeInput=payload decodeOutput=ruamel.yaml.load(..) decodeFormat=YAML
+ruamel.yaml.load(payload, ruamel.yaml.BaseLoader)  # $ decodeInput=payload decodeOutput=ruamel.yaml.load(..) decodeFormat=YAML
+ruamel.yaml.safe_load(payload) # $  decodeInput=payload decodeOutput=ruamel.yaml.safe_load(..) decodeFormat=YAML
+
+################################################################################
+# load_all variants
+################################################################################
+
+# Unsafe:
+ruamel.yaml.load_all(payload) # $  decodeInput=payload decodeOutput=ruamel.yaml.load_all(..) decodeFormat=YAML decodeMayExecuteInput
+
+# Safe:
+ruamel.yaml.safe_load_all(payload) # $  decodeInput=payload decodeOutput=ruamel.yaml.safe_load_all(..) decodeFormat=YAML
+
+################################################################################
+# C-based loaders with `libyaml`
+################################################################################
+
+# Unsafe:
+ruamel.yaml.load(payload, ruamel.yaml.CLoader)  # $  decodeInput=payload decodeOutput=ruamel.yaml.load(..) decodeFormat=YAML decodeMayExecuteInput
+
+# Safe:
+ruamel.yaml.load(payload, ruamel.yaml.CSafeLoader)  # $ decodeInput=payload decodeOutput=ruamel.yaml.load(..) decodeFormat=YAML
+ruamel.yaml.load(payload, ruamel.yaml.CBaseLoader)  # $ decodeInput=payload decodeOutput=ruamel.yaml.load(..) decodeFormat=YAML

python/ql/test/library-tests/frameworks/ruamel.yaml/PoC

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+
+# this file doesn't have a .py extension so the extractor doesn't pick it up, so it
+# doesn't have to be annotated
+
+# This file is just a Proof of Concept for how code execution can be triggered.
+
+import os
+import ruamel.yaml
+
+class Exploit(object):
+    def __reduce__(self):
+        return (os.system, ('ls',))
+
+data = Exploit()
+serialized_data = ruamel.yaml.dump(data)
+
+# All these will execute `ls`
+print("!!! ruamel.yaml.load")
+ruamel.yaml.load(serialized_data)
+
+print("!!! ruamel.yaml.load kwarg")
+ruamel.yaml.load(stream=serialized_data)
+
+print("!!! ruamel.yaml.load with Loader=ruamel.yaml.Loader")
+ruamel.yaml.load(serialized_data, ruamel.yaml.Loader)
+
+print("!!! ruamel.yaml.load with Loader=ruamel.yaml.UnsafeLoader")
+ruamel.yaml.load(serialized_data, ruamel.yaml.UnsafeLoader)
+
+print("!!! ruamel.yaml.load with Loader=ruamel.yaml.CLoader")
+ruamel.yaml.load(serialized_data, ruamel.yaml.CLoader)
+
+# you need to iterate through the result for it to execute... but it still works
+print("!!! ruamel.yaml.load_all")
+for _ in ruamel.yaml.load_all(serialized_data):
+    pass
+
+# check that the safe version is actually safe
+print("\n" + "-"*80)
+print("safe versions")
+print("-" * 80)
+
+print("!!! ruamel.yaml.safe_load")
+try:
+    ruamel.yaml.safe_load(serialized_data)
+    raise Exception("should not happen")
+except ruamel.yaml.constructor.ConstructorError:
+    pass
+
+print("!!! ruamel.yaml.load with Loader=ruamel.yaml.SafeLoader")
+try:
+    ruamel.yaml.load(serialized_data, ruamel.yaml.SafeLoader)
+    raise Exception("should not happen")
+except ruamel.yaml.constructor.ConstructorError:
+    pass
+
+print("!!! ruamel.yaml.load with Loader=ruamel.yaml.CSafeLoader")
+try:
+    ruamel.yaml.load(serialized_data, ruamel.yaml.CSafeLoader)
+    raise Exception("should not happen")
+except ruamel.yaml.constructor.ConstructorError:
+    pass

python/ql/test/library-tests/frameworks/yaml/Decoding.py

@@ -2,6 +2,7 @@ import yaml
 
 # Unsafe:
 yaml.load(payload)  # $decodeInput=payload decodeOutput=yaml.load(..) decodeFormat=YAML decodeMayExecuteInput
+yaml.load(stream=payload)  # $decodeInput=payload decodeOutput=yaml.load(..) decodeFormat=YAML decodeMayExecuteInput
 yaml.load(payload, yaml.Loader)  # $decodeInput=payload decodeOutput=yaml.load(..) decodeFormat=YAML decodeMayExecuteInput
 yaml.unsafe_load(payload) # $ decodeInput=payload decodeOutput=yaml.unsafe_load(..) decodeFormat=YAML decodeMayExecuteInput
 yaml.full_load(payload) # $ decodeInput=payload decodeOutput=yaml.full_load(..) decodeFormat=YAML decodeMayExecuteInput

python/ql/test/library-tests/frameworks/yaml/PoC

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+
+# this file doesn't have a .py extension so the extractor doesn't pick it up, so it
+# doesn't have to be annotated
+
+# This file is just a Proof of Concept for how code execution can be triggered.
+
+
+import os
+import yaml
+
+class Exploit(object):
+    def __reduce__(self):
+        return (os.system, ('ls',))
+
+data = Exploit()
+serialized_data = yaml.dump(data)
+
+# All these will execute `ls`
+print("!!! yaml.unsafe_load")
+yaml.unsafe_load(serialized_data)
+
+print("!!! yaml.unsafe_load kwarg")
+yaml.unsafe_load(stream=serialized_data)
+
+print("!!! yaml.load with Loader=yaml.UnsafeLoader")
+yaml.load(serialized_data, yaml.UnsafeLoader)
+
+# you need to iterate through the result for it to execute... but it still works
+print("!!! yaml.unsafe_load_all")
+for _ in yaml.unsafe_load_all(serialized_data):
+    pass
+
+# check that the safe version is actually safe
+print("\n" + "-"*80)
+print("safe versions")
+print("-" * 80)
+
+print("!!! yaml.load")
+try:
+    yaml.load(serialized_data)
+    raise Exception("should not happen")
+except yaml.constructor.ConstructorError:
+    pass
+
+print("!!! yaml.safe_load")
+try:
+    yaml.safe_load(serialized_data)
+    raise Exception("should not happen")
+except yaml.constructor.ConstructorError:
+    pass
+
+print("!!! yaml.load with Loader=yaml.SafeLoader")
+try:
+    yaml.load(serialized_data, yaml.SafeLoader)
+    raise Exception("should not happen")
+except yaml.constructor.ConstructorError:
+    pass

python/ql/test/library-tests/objects/Name.ql

@@ -3,19 +3,7 @@ import python
 from Object o, string name
 where
   o.hasLongName(name) and
-  (
-    name = "sys.modules"
-    or
-    name = "test.n"
-    or
-    name = "test.l"
-    or
-    name = "test.d"
-    or
-    name = "test.C.meth"
-    or
-    name = "test.C.cmeth"
-    or
-    name = "test.C.smeth"
-  )
+  name in [
+      "sys.modules", "test.n", "test.l", "test.d", "test.C.meth", "test.C.cmeth", "test.C.smeth"
+    ]
 select name, o.toString()