Merge remote-tracking branch 'origin/main' into jorgectf/python/insecure-cookie

python/change-notes/2021-10-26-ruamel.yaml-modeling.md

@@ -0,0 +1,2 @@
+lgtm,codescanning
+* Added modeling of the `ruamel.yaml` PyPI package, resulting in additional sinks for the _Deserializing untrusted input_ (`py/unsafe-deserialization`) query (since `ruamel.yaml.load` can lead to code execution).

python/ql/lib/semmle/python/Frameworks.qll

@@ -25,6 +25,7 @@ private import semmle.python.frameworks.Peewee
 private import semmle.python.frameworks.Psycopg2
 private import semmle.python.frameworks.PyMySQL
 private import semmle.python.frameworks.Rsa
+private import semmle.python.frameworks.RuamelYaml
 private import semmle.python.frameworks.Simplejson
 private import semmle.python.frameworks.SqlAlchemy
 private import semmle.python.frameworks.Stdlib

python/ql/lib/semmle/python/frameworks/Flask.qll

@@ -519,4 +519,32 @@ module Flask {
 
     override DataFlow::Node getValueArg() { none() }
   }
+
+  /**
+   * A call to `flask.send_from_directory`.
+   *
+   * See https://flask.palletsprojects.com/en/1.1.x/api/#flask.send_from_directory
+   */
+  class FlaskSendFromDirectory extends FileSystemAccess::Range, DataFlow::CallCfgNode {
+    FlaskSendFromDirectory() {
+      this = API::moduleImport("flask").getMember("send_from_directory").getACall()
+    }
+
+    override DataFlow::Node getAPathArgument() {
+      result in [this.getArg(_), this.getArgByName(["directory", "filename"])]
+    }
+  }
+
+  /**
+   * A call to `flask.send_file`.
+   *
+   * See https://flask.palletsprojects.com/en/1.1.x/api/#flask.send_file
+   */
+  class FlaskSendFile extends FileSystemAccess::Range, DataFlow::CallCfgNode {
+    FlaskSendFile() { this = API::moduleImport("flask").getMember("send_file").getACall() }
+
+    override DataFlow::Node getAPathArgument() {
+      result in [this.getArg(0), this.getArgByName("filename_or_fp")]
+    }
+  }
 }

python/ql/lib/semmle/python/frameworks/RuamelYaml.qll

@@ -0,0 +1,57 @@
+/**
+ * Provides classes modeling security-relevant aspects of the `ruamel.yaml` PyPI package
+ *
+ * See
+ * - https://pypi.org/project/ruamel.yaml/
+ */
+
+private import python
+private import semmle.python.dataflow.new.DataFlow
+private import semmle.python.Concepts
+private import semmle.python.ApiGraphs
+
+/**
+ * Provides models for the `ruamel.yaml` PyPI package.
+ *
+ * See
+ * - https://pypi.org/project/ruamel.yaml/
+ */
+private module RuamelYaml {
+  // Note: `ruamel.yaml` is a fork of the `PyYAML` PyPI package, so that's why the
+  // interface is so similar.
+  /**
+   * A call to any of the loading functions in `yaml` (`load`, `load_all`, `safe_load`, `safe_load_all`)
+   *
+   * See https://pyyaml.org/wiki/PyYAMLDocumentation (you will have to scroll down).
+   */
+  private class RuamelYamlLoadCall extends Decoding::Range, DataFlow::CallCfgNode {
+    string func_name;
+
+    RuamelYamlLoadCall() {
+      func_name in ["load", "load_all", "safe_load", "safe_load_all"] and
+      this = API::moduleImport("ruamel").getMember("yaml").getMember(func_name).getACall()
+    }
+
+    override predicate mayExecuteInput() {
+      func_name in ["load", "load_all"] and
+      // If the `Loader` argument is not set, the default loader will be used, which is
+      // not safe. The only safe loaders are `SafeLoader` or `BaseLoader` (and their
+      // variants with C implementation).
+      not exists(DataFlow::Node loader_arg |
+        loader_arg in [this.getArg(1), this.getArgByName("Loader")]
+      |
+        loader_arg =
+          API::moduleImport("ruamel")
+              .getMember("yaml")
+              .getMember(["SafeLoader", "BaseLoader", "CSafeLoader", "CBaseLoader"])
+              .getAUse()
+      )
+    }
+
+    override DataFlow::Node getAnInput() { result in [this.getArg(0), this.getArgByName("stream")] }
+
+    override DataFlow::Node getOutput() { result = this }
+
+    override string getFormat() { result = "YAML" }
+  }
+}

python/ql/lib/semmle/python/frameworks/Yaml.qll

@@ -9,7 +9,6 @@
 
 private import python
 private import semmle.python.dataflow.new.DataFlow
-private import semmle.python.dataflow.new.RemoteFlowSources
 private import semmle.python.Concepts
 private import semmle.python.ApiGraphs
 
@@ -41,11 +40,17 @@ private module Yaml {
     }
 
     /**
-     * This function was thought safe from the 5.1 release in 2017, when the default loader was changed to `FullLoader`.
-     * In 2020 new exploits were found, meaning it's not safe. The Current plan is to change the default to `SafeLoader` in release 6.0
-     * (as explained in https://github.com/yaml/pyyaml/issues/420#issuecomment-696752389).
-     * Until 6.0 is released, we will mark `yaml.load` as possibly leading to arbitrary code execution.
-     * See https://github.com/yaml/pyyaml/wiki/PyYAML-yaml.load(input)-Deprecation for more details.
+     * This function was thought safe from the 5.1 release in 2017, when the default
+     * loader was changed to `FullLoader` (see
+     * https://github.com/yaml/pyyaml/wiki/PyYAML-yaml.load(input)-Deprecation).
+     *
+     * In 2020 new exploits were found, meaning it's not safe. With the 6.0 release (see
+     * https://github.com/yaml/pyyaml/commit/8cdff2c80573b8be8e8ad28929264a913a63aa33),
+     * when using `load` and `load_all` you are now required to specify a Loader. But
+     * from what I (@RasmusWL) can gather, `FullLoader` is not to be considered safe,
+     * although known exploits have been mitigated (is at least my impression). Also see
+     * https://github.com/yaml/pyyaml/issues/420#issuecomment-696752389 for more
+     * details.
      */
     override predicate mayExecuteInput() {
       func_name in ["full_load", "full_load_all", "unsafe_load", "unsafe_load_all"]
@@ -63,7 +68,7 @@ private module Yaml {
       )
     }
 
-    override DataFlow::Node getAnInput() { result = this.getArg(0) }
+    override DataFlow::Node getAnInput() { result in [this.getArg(0), this.getArgByName("stream")] }
 
     override DataFlow::Node getOutput() { result = this }
 

python/ql/test/library-tests/frameworks/flask/save_uploaded_file.py

@@ -1,6 +1,17 @@
-from flask import Flask, request
+from flask import Flask, request, send_from_directory, send_file
 app = Flask(__name__)
 
 @app.route("/save-uploaded-file")  # $routeSetup="/save-uploaded-file"
 def test_taint():  # $requestHandler
     request.files['key'].save("path") # $ getAPathArgument="path"
+
+
+@app.route("/path-injection")  # $routeSetup="/path-injection"
+def test_path():  # $requestHandler
+
+    send_from_directory("filepath","file")  # $ getAPathArgument="filepath" getAPathArgument="file"
+    send_file("file")  # $ getAPathArgument="file"
+
+    send_from_directory(directory="filepath","file") # $ getAPathArgument="filepath" getAPathArgument="file"
+    send_from_directory(filename="filepath","file") # $ getAPathArgument="filepath" getAPathArgument="file"
+    send_file(filename_or_fp="file")  # $ getAPathArgument="file"

python/ql/test/library-tests/frameworks/ruamel.yaml/ConceptsTest.ql

@@ -0,0 +1,2 @@
+import python
+import experimental.meta.ConceptsTest

python/ql/test/library-tests/frameworks/ruamel.yaml/Decoding.py

@@ -0,0 +1,33 @@
+import ruamel.yaml
+
+# Unsafe:
+ruamel.yaml.load(payload)  # $ decodeInput=payload decodeOutput=ruamel.yaml.load(..) decodeFormat=YAML decodeMayExecuteInput
+ruamel.yaml.load(stream=payload)  # $ decodeInput=payload decodeOutput=ruamel.yaml.load(..) decodeFormat=YAML decodeMayExecuteInput
+ruamel.yaml.load(payload, ruamel.yaml.Loader)  # $ decodeInput=payload decodeOutput=ruamel.yaml.load(..) decodeFormat=YAML decodeMayExecuteInput
+
+# Safe:
+ruamel.yaml.load(payload, ruamel.yaml.SafeLoader)  # $  decodeInput=payload decodeOutput=ruamel.yaml.load(..) decodeFormat=YAML
+ruamel.yaml.load(payload, Loader=ruamel.yaml.SafeLoader)  # $ decodeInput=payload decodeOutput=ruamel.yaml.load(..) decodeFormat=YAML
+ruamel.yaml.load(payload, ruamel.yaml.BaseLoader)  # $ decodeInput=payload decodeOutput=ruamel.yaml.load(..) decodeFormat=YAML
+ruamel.yaml.safe_load(payload) # $  decodeInput=payload decodeOutput=ruamel.yaml.safe_load(..) decodeFormat=YAML
+
+################################################################################
+# load_all variants
+################################################################################
+
+# Unsafe:
+ruamel.yaml.load_all(payload) # $  decodeInput=payload decodeOutput=ruamel.yaml.load_all(..) decodeFormat=YAML decodeMayExecuteInput
+
+# Safe:
+ruamel.yaml.safe_load_all(payload) # $  decodeInput=payload decodeOutput=ruamel.yaml.safe_load_all(..) decodeFormat=YAML
+
+################################################################################
+# C-based loaders with `libyaml`
+################################################################################
+
+# Unsafe:
+ruamel.yaml.load(payload, ruamel.yaml.CLoader)  # $  decodeInput=payload decodeOutput=ruamel.yaml.load(..) decodeFormat=YAML decodeMayExecuteInput
+
+# Safe:
+ruamel.yaml.load(payload, ruamel.yaml.CSafeLoader)  # $ decodeInput=payload decodeOutput=ruamel.yaml.load(..) decodeFormat=YAML
+ruamel.yaml.load(payload, ruamel.yaml.CBaseLoader)  # $ decodeInput=payload decodeOutput=ruamel.yaml.load(..) decodeFormat=YAML

python/ql/test/library-tests/frameworks/ruamel.yaml/PoC

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+
+# this file doesn't have a .py extension so the extractor doesn't pick it up, so it
+# doesn't have to be annotated
+
+# This file is just a Proof of Concept for how code execution can be triggered.
+
+import os
+import ruamel.yaml
+
+class Exploit(object):
+    def __reduce__(self):
+        return (os.system, ('ls',))
+
+data = Exploit()
+serialized_data = ruamel.yaml.dump(data)
+
+# All these will execute `ls`
+print("!!! ruamel.yaml.load")
+ruamel.yaml.load(serialized_data)
+
+print("!!! ruamel.yaml.load kwarg")
+ruamel.yaml.load(stream=serialized_data)
+
+print("!!! ruamel.yaml.load with Loader=ruamel.yaml.Loader")
+ruamel.yaml.load(serialized_data, ruamel.yaml.Loader)
+
+print("!!! ruamel.yaml.load with Loader=ruamel.yaml.UnsafeLoader")
+ruamel.yaml.load(serialized_data, ruamel.yaml.UnsafeLoader)
+
+print("!!! ruamel.yaml.load with Loader=ruamel.yaml.CLoader")
+ruamel.yaml.load(serialized_data, ruamel.yaml.CLoader)
+
+# you need to iterate through the result for it to execute... but it still works
+print("!!! ruamel.yaml.load_all")
+for _ in ruamel.yaml.load_all(serialized_data):
+    pass
+
+# check that the safe version is actually safe
+print("\n" + "-"*80)
+print("safe versions")
+print("-" * 80)
+
+print("!!! ruamel.yaml.safe_load")
+try:
+    ruamel.yaml.safe_load(serialized_data)
+    raise Exception("should not happen")
+except ruamel.yaml.constructor.ConstructorError:
+    pass
+
+print("!!! ruamel.yaml.load with Loader=ruamel.yaml.SafeLoader")
+try:
+    ruamel.yaml.load(serialized_data, ruamel.yaml.SafeLoader)
+    raise Exception("should not happen")
+except ruamel.yaml.constructor.ConstructorError:
+    pass
+
+print("!!! ruamel.yaml.load with Loader=ruamel.yaml.CSafeLoader")
+try:
+    ruamel.yaml.load(serialized_data, ruamel.yaml.CSafeLoader)
+    raise Exception("should not happen")
+except ruamel.yaml.constructor.ConstructorError:
+    pass

python/ql/test/library-tests/frameworks/yaml/Decoding.py

@@ -2,6 +2,7 @@ import yaml
 
 # Unsafe:
 yaml.load(payload)  # $decodeInput=payload decodeOutput=yaml.load(..) decodeFormat=YAML decodeMayExecuteInput
+yaml.load(stream=payload)  # $decodeInput=payload decodeOutput=yaml.load(..) decodeFormat=YAML decodeMayExecuteInput
 yaml.load(payload, yaml.Loader)  # $decodeInput=payload decodeOutput=yaml.load(..) decodeFormat=YAML decodeMayExecuteInput
 yaml.unsafe_load(payload) # $ decodeInput=payload decodeOutput=yaml.unsafe_load(..) decodeFormat=YAML decodeMayExecuteInput
 yaml.full_load(payload) # $ decodeInput=payload decodeOutput=yaml.full_load(..) decodeFormat=YAML decodeMayExecuteInput

python/ql/test/library-tests/frameworks/yaml/PoC

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+
+# this file doesn't have a .py extension so the extractor doesn't pick it up, so it
+# doesn't have to be annotated
+
+# This file is just a Proof of Concept for how code execution can be triggered.
+
+
+import os
+import yaml
+
+class Exploit(object):
+    def __reduce__(self):
+        return (os.system, ('ls',))
+
+data = Exploit()
+serialized_data = yaml.dump(data)
+
+# All these will execute `ls`
+print("!!! yaml.unsafe_load")
+yaml.unsafe_load(serialized_data)
+
+print("!!! yaml.unsafe_load kwarg")
+yaml.unsafe_load(stream=serialized_data)
+
+print("!!! yaml.load with Loader=yaml.UnsafeLoader")
+yaml.load(serialized_data, yaml.UnsafeLoader)
+
+# you need to iterate through the result for it to execute... but it still works
+print("!!! yaml.unsafe_load_all")
+for _ in yaml.unsafe_load_all(serialized_data):
+    pass
+
+# check that the safe version is actually safe
+print("\n" + "-"*80)
+print("safe versions")
+print("-" * 80)
+
+print("!!! yaml.load")
+try:
+    yaml.load(serialized_data)
+    raise Exception("should not happen")
+except yaml.constructor.ConstructorError:
+    pass
+
+print("!!! yaml.safe_load")
+try:
+    yaml.safe_load(serialized_data)
+    raise Exception("should not happen")
+except yaml.constructor.ConstructorError:
+    pass
+
+print("!!! yaml.load with Loader=yaml.SafeLoader")
+try:
+    yaml.load(serialized_data, yaml.SafeLoader)
+    raise Exception("should not happen")
+except yaml.constructor.ConstructorError:
+    pass