Merge pull request #8275 from haby0/py/add-ssrf-sinks

Python: Add Server-side Request Forgery sinks

python/ql/lib/change-notes/2022-03-04-add-ssrf-sinks.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Added new SSRF sinks for `httpx`, `pycurl`, `urllib`, `urllib2`, `urllib3`, and `libtaxii`. This improvement was [submitted by @haby0](https://github.com/github/codeql/pull/8275).

python/ql/lib/semmle/python/Frameworks.qll

@@ -19,11 +19,13 @@ private import semmle.python.frameworks.FastApi
 private import semmle.python.frameworks.Flask
 private import semmle.python.frameworks.FlaskAdmin
 private import semmle.python.frameworks.FlaskSqlAlchemy
+private import semmle.python.frameworks.Httpx
 private import semmle.python.frameworks.Idna
 private import semmle.python.frameworks.Invoke
 private import semmle.python.frameworks.Jmespath
 private import semmle.python.frameworks.Ldap
 private import semmle.python.frameworks.Ldap3
+private import semmle.python.frameworks.Libtaxii
 private import semmle.python.frameworks.Libxml2
 private import semmle.python.frameworks.Lxml
 private import semmle.python.frameworks.MarkupSafe
@@ -32,6 +34,7 @@ private import semmle.python.frameworks.Mysql
 private import semmle.python.frameworks.MySQLdb
 private import semmle.python.frameworks.Peewee
 private import semmle.python.frameworks.Psycopg2
+private import semmle.python.frameworks.Pycurl
 private import semmle.python.frameworks.Pydantic
 private import semmle.python.frameworks.PyMySQL
 private import semmle.python.frameworks.Requests
@@ -46,5 +49,6 @@ private import semmle.python.frameworks.Toml
 private import semmle.python.frameworks.Tornado
 private import semmle.python.frameworks.Twisted
 private import semmle.python.frameworks.Ujson
+private import semmle.python.frameworks.Urllib3
 private import semmle.python.frameworks.Yaml
 private import semmle.python.frameworks.Yarl

python/ql/lib/semmle/python/frameworks/Aiohttp.qll

@@ -639,3 +639,53 @@ module AiohttpWebModel {
     override DataFlow::Node getValueArg() { result = value }
   }
 }
+
+/**
+ * Provides models for the web server part (`aiohttp.client`) of the `aiohttp` PyPI package.
+ * See https://docs.aiohttp.org/en/stable/client.html
+ */
+private module AiohttpClientModel {
+  /**
+   * Provides models for the `aiohttp.ClientSession` class
+   *
+   * See https://docs.aiohttp.org/en/stable/client_reference.html#aiohttp.ClientSession.
+   */
+  module ClientSession {
+    /** Gets a reference to the `aiohttp.ClientSession` class. */
+    private API::Node classRef() {
+      result = API::moduleImport("aiohttp").getMember("ClientSession")
+    }
+
+    /** Gets a reference to an instance of `aiohttp.ClientSession`. */
+    private API::Node instance() { result = classRef().getReturn() }
+
+    /** A method call on a ClientSession that sends off a request */
+    private class OutgoingRequestCall extends HTTP::Client::Request::Range, DataFlow::CallCfgNode {
+      string methodName;
+
+      OutgoingRequestCall() {
+        methodName in [HTTP::httpVerbLower(), "request"] and
+        this = instance().getMember(methodName).getACall()
+      }
+
+      override DataFlow::Node getAUrlPart() {
+        result = this.getArgByName("url")
+        or
+        not methodName = "request" and
+        result = this.getArg(0)
+        or
+        methodName = "request" and
+        result = this.getArg(1)
+      }
+
+      override string getFramework() { result = "aiohttp.ClientSession" }
+
+      override predicate disablesCertificateValidation(
+        DataFlow::Node disablingNode, DataFlow::Node argumentOrigin
+      ) {
+        // TODO: Look into disabling certificate validation
+        none()
+      }
+    }
+  }
+}

python/ql/lib/semmle/python/frameworks/Httpx.qll

@@ -0,0 +1,88 @@
+/**
+ * Provides classes modeling security-relevant aspects of the `httpx` PyPI package.
+ *
+ * See
+ * - https://pypi.org/project/httpx/
+ * - https://www.python-httpx.org/
+ */
+
+private import python
+private import semmle.python.Concepts
+private import semmle.python.ApiGraphs
+
+/**
+ * Provides models for the `httpx` PyPI package.
+ *
+ * See
+ * - https://pypi.org/project/httpx/
+ * - https://www.python-httpx.org/
+ */
+private module HttpxModel {
+  private class RequestCall extends HTTP::Client::Request::Range, DataFlow::CallCfgNode {
+    string methodName;
+
+    RequestCall() {
+      methodName in [HTTP::httpVerbLower(), "request", "stream"] and
+      this = API::moduleImport("httpx").getMember(methodName).getACall()
+    }
+
+    override DataFlow::Node getAUrlPart() {
+      result = this.getArgByName("url")
+      or
+      if methodName in ["request", "stream"]
+      then result = this.getArg(1)
+      else result = this.getArg(0)
+    }
+
+    override string getFramework() { result = "httpx" }
+
+    override predicate disablesCertificateValidation(
+      DataFlow::Node disablingNode, DataFlow::Node argumentOrigin
+    ) {
+      // TODO: Look into disabling certificate validation
+      none()
+    }
+  }
+
+  /**
+   * Provides models for the `httpx.[Async]Client` class
+   *
+   * See https://www.python-httpx.org/async/
+   */
+  module Client {
+    /** Get a reference to the `httpx.Client` or `httpx.AsyncClient` class. */
+    private API::Node classRef() {
+      result = API::moduleImport("httpx").getMember(["Client", "AsyncClient"])
+    }
+
+    /** Get a reference to an `httpx.Client` or `httpx.AsyncClient` instance. */
+    private API::Node instance() { result = classRef().getReturn() }
+
+    /** A method call on a Client that sends off a request */
+    private class OutgoingRequestCall extends HTTP::Client::Request::Range, DataFlow::CallCfgNode {
+      string methodName;
+
+      OutgoingRequestCall() {
+        methodName in [HTTP::httpVerbLower(), "request", "stream"] and
+        this = instance().getMember(methodName).getACall()
+      }
+
+      override DataFlow::Node getAUrlPart() {
+        result = this.getArgByName("url")
+        or
+        if methodName in ["request", "stream"]
+        then result = this.getArg(1)
+        else result = this.getArg(0)
+      }
+
+      override string getFramework() { result = "httpx.[Async]Client" }
+
+      override predicate disablesCertificateValidation(
+        DataFlow::Node disablingNode, DataFlow::Node argumentOrigin
+      ) {
+        // TODO: Look into disabling certificate validation
+        none()
+      }
+    }
+  }
+}

python/ql/lib/semmle/python/frameworks/Libtaxii.qll

@@ -0,0 +1,42 @@
+/**
+ * Provides classes modeling security-relevant aspects of the `libtaxii` PyPI package.
+ *
+ * See
+ * - https://pypi.org/project/libtaxii/
+ * - https://github.com/TAXIIProject/libtaxii
+ */
+
+private import python
+private import semmle.python.Concepts
+private import semmle.python.ApiGraphs
+
+/**
+ * Provides models for the `libtaxii` PyPI package.
+ *
+ * See
+ * - https://pypi.org/project/libtaxii/
+ * - https://github.com/TAXIIProject/libtaxii
+ */
+private module Libtaxii {
+  /**
+   * A call to `libtaxii.common.parse`.
+   * When the `allow_url` parameter value is set to `True`, there is an SSRF vulnerability..
+   */
+  private class ParseCall extends HTTP::Client::Request::Range, DataFlow::CallCfgNode {
+    ParseCall() {
+      this = API::moduleImport("libtaxii").getMember("common").getMember("parse").getACall() and
+      this.getArgByName("allow_url").getALocalSource().asExpr() = any(True t)
+    }
+
+    override DataFlow::Node getAUrlPart() { result in [this.getArg(0), this.getArgByName("s")] }
+
+    override string getFramework() { result = "libtaxii.common.parse" }
+
+    override predicate disablesCertificateValidation(
+      DataFlow::Node disablingNode, DataFlow::Node argumentOrigin
+    ) {
+      // TODO: Look into disabling certificate validation
+      none()
+    }
+  }
+}

python/ql/lib/semmle/python/frameworks/Pycurl.qll

@@ -0,0 +1,59 @@
+/**
+ * Provides classes modeling security-relevant aspects of the `pycurl` PyPI package.
+ *
+ * See
+ * - https://pypi.org/project/pycurl/
+ * - https://pycurl.io/docs/latest/
+ */
+
+private import python
+private import semmle.python.Concepts
+private import semmle.python.ApiGraphs
+
+/**
+ * Provides models for the `pycurl` PyPI package.
+ *
+ * See
+ * - https://pypi.org/project/pycurl/
+ * - https://pycurl.io/docs/latest/
+ */
+private module Pycurl {
+  /**
+   * Provides models for the `pycurl.Curl` class
+   *
+   * See https://pycurl.io/docs/latest/curl.html.
+   */
+  module Curl {
+    /** Gets a reference to the `pycurl.Curl` class. */
+    private API::Node classRef() { result = API::moduleImport("pycurl").getMember("Curl") }
+
+    /** Gets a reference to an instance of `pycurl.Curl`. */
+    private API::Node instance() { result = classRef().getReturn() }
+
+    /**
+     * When the first parameter value of the `setopt` function is set to `pycurl.URL`,
+     * the second parameter value is the request resource link.
+     *
+     * See http://pycurl.io/docs/latest/curlobject.html#pycurl.Curl.setopt.
+     */
+    private class OutgoingRequestCall extends HTTP::Client::Request::Range, DataFlow::CallCfgNode {
+      OutgoingRequestCall() {
+        this = instance().getMember("setopt").getACall() and
+        this.getArg(0).asCfgNode().(AttrNode).getName() = "URL"
+      }
+
+      override DataFlow::Node getAUrlPart() {
+        result in [this.getArg(1), this.getArgByName("value")]
+      }
+
+      override string getFramework() { result = "pycurl.Curl" }
+
+      override predicate disablesCertificateValidation(
+        DataFlow::Node disablingNode, DataFlow::Node argumentOrigin
+      ) {
+        // TODO: Look into disabling certificate validation
+        none()
+      }
+    }
+  }
+}

python/ql/lib/semmle/python/frameworks/Stdlib.qll

@@ -13,6 +13,9 @@ private import semmle.python.frameworks.PEP249
 private import semmle.python.frameworks.internal.PoorMansFunctionResolution
 private import semmle.python.frameworks.internal.SelfRefMixin
 private import semmle.python.frameworks.internal.InstanceTaintStepsHelper
+// modeling split over multiple files to keep this file from becoming too big
+private import semmle.python.frameworks.Stdlib.Urllib
+private import semmle.python.frameworks.Stdlib.Urllib2
 
 /** Provides models for the Python standard library. */
 module Stdlib {

python/ql/lib/semmle/python/frameworks/Stdlib/Urllib.qll

@@ -0,0 +1,71 @@
+/**
+ * Provides classes modeling security-relevant aspects of the `urllib` module, part of
+ * the Python standard library.
+ *
+ * See
+ * - https://docs.python.org/2/library/urllib.html
+ * - https://docs.python.org/3/library/urllib.html
+ */
+
+private import python
+private import semmle.python.Concepts
+private import semmle.python.ApiGraphs
+
+/**
+ * Provides models for the `urllib` module, part of
+ * the Python standard library.
+ *
+ * See
+ * - https://docs.python.org/2/library/urllib.html
+ * - https://docs.python.org/3/library/urllib.html
+ */
+private module Urllib {
+  /**
+   * Provides models for the `urllib.request` extension library
+   *
+   * See https://docs.python.org/3.9/library/urllib.request.html
+   */
+  module Request {
+    /**
+     * See
+     * - https://docs.python.org/3.9/library/urllib.request.html#urllib.request.Request
+     */
+    private class RequestCall extends HTTP::Client::Request::Range, DataFlow::CallCfgNode {
+      RequestCall() {
+        this = API::moduleImport("urllib").getMember("request").getMember("Request").getACall()
+      }
+
+      override DataFlow::Node getAUrlPart() { result in [this.getArg(0), this.getArgByName("url")] }
+
+      override string getFramework() { result = "urllib.request.Request" }
+
+      override predicate disablesCertificateValidation(
+        DataFlow::Node disablingNode, DataFlow::Node argumentOrigin
+      ) {
+        // TODO: Look into disabling certificate validation
+        none()
+      }
+    }
+
+    /**
+     * See
+     * - https://docs.python.org/3.9/library/urllib.request.html#urllib.request.urlopen
+     */
+    private class UrlOpenCall extends HTTP::Client::Request::Range, DataFlow::CallCfgNode {
+      UrlOpenCall() {
+        this = API::moduleImport("urllib").getMember("request").getMember("urlopen").getACall()
+      }
+
+      override DataFlow::Node getAUrlPart() { result in [this.getArg(0), this.getArgByName("url")] }
+
+      override string getFramework() { result = "urllib.request.urlopen" }
+
+      override predicate disablesCertificateValidation(
+        DataFlow::Node disablingNode, DataFlow::Node argumentOrigin
+      ) {
+        // TODO: Look into disabling certificate validation
+        none()
+      }
+    }
+  }
+}

python/ql/lib/semmle/python/frameworks/Stdlib/Urllib2.qll

@@ -0,0 +1,56 @@
+/**
+ * Provides classes modeling security-relevant aspects of the `urllib2` module, part of
+ * the Python 2 standard library.
+ *
+ * See https://docs.python.org/2/library/urllib2.html
+ */
+
+private import python
+private import semmle.python.Concepts
+private import semmle.python.ApiGraphs
+
+/**
+ * Provides models for the the `urllib2` module, part of
+ * the Python 2 standard library.
+ *
+ * See https://docs.python.org/2/library/urllib2.html
+ */
+private module Urllib2 {
+  /**
+   * See
+   * - https://docs.python.org/2/library/urllib2.html#urllib2.Request
+   */
+  private class RequestCall extends HTTP::Client::Request::Range, DataFlow::CallCfgNode {
+    RequestCall() { this = API::moduleImport("urllib2").getMember("Request").getACall() }
+
+    override DataFlow::Node getAUrlPart() { result in [this.getArg(0), this.getArgByName("url")] }
+
+    override string getFramework() { result = "urllib2.Request" }
+
+    override predicate disablesCertificateValidation(
+      DataFlow::Node disablingNode, DataFlow::Node argumentOrigin
+    ) {
+      // TODO: Look into disabling certificate validation
+      none()
+    }
+  }
+
+  /**
+   * See
+   * - https://docs.python.org/2/library/urllib2.html#urllib2.urlopen
+   */
+  private class UrlOpenCall extends HTTP::Client::Request::Range, DataFlow::CallCfgNode {
+    UrlOpenCall() { this = API::moduleImport("urllib2").getMember("urlopen").getACall() }
+
+    override DataFlow::Node getAUrlPart() { result in [this.getArg(0), this.getArgByName("url")] }
+
+    override string getFramework() { result = "urllib2.urlopen" }
+
+    override predicate disablesCertificateValidation(
+      DataFlow::Node disablingNode, DataFlow::Node argumentOrigin
+    ) {
+      // TODO: Look into disabling certificate validation
+      none()
+    }
+  }
+}

python/ql/lib/semmle/python/frameworks/Urllib3.qll

@@ -0,0 +1,75 @@
+/**
+ * Provides classes modeling security-relevant aspects of the `urllib3` PyPI package.
+ *
+ * See
+ * - https://pypi.org/project/urllib3/
+ * - https://urllib3.readthedocs.io/en/stable/reference/
+ */
+
+private import python
+private import semmle.python.Concepts
+private import semmle.python.ApiGraphs
+
+/**
+ * Provides models for the `urllib3` PyPI package.
+ *
+ * See
+ * - https://pypi.org/project/urllib3/
+ * - https://urllib3.readthedocs.io/en/stable/reference/
+ */
+private module Urllib3 {
+  /**
+   * Provides models for the `urllib3.request.RequestMethods` class and subclasses, such
+   * as the `urllib3.PoolManager` class
+   *
+   * See
+   * - https://urllib3.readthedocs.io/en/stable/reference/urllib3.request.html#urllib3.request.RequestMethods
+   *
+   *
+   * https://urllib3.readthedocs.io/en/stable/reference/urllib3.poolmanager.html.
+   */
+  module PoolManager {
+    /** Gets a reference to the `urllib3.PoolManager` class. */
+    private API::Node classRef() {
+      result =
+        API::moduleImport("urllib3")
+            .getMember(["PoolManager", "ProxyManager", "HTTPConnectionPool", "HTTPSConnectionPool"])
+      or
+      result =
+        API::moduleImport("urllib3")
+            .getMember("request")
+            .getMember("RequestMethods")
+            .getASubclass+()
+    }
+
+    /** Gets a reference to an instance of a `urllib3.request.RequestMethods` subclass. */
+    private API::Node instance() { result = classRef().getReturn() }
+
+    /**
+     * A call to a method making an outgoing request.
+     *
+     * See
+     * - https://urllib3.readthedocs.io/en/stable/reference/urllib3.request.html#urllib3.request.RequestMethods
+     * - https://urllib3.readthedocs.io/en/stable/reference/urllib3.connectionpool.html#urllib3.HTTPConnectionPool.urlopen
+     */
+    private class RequestCall extends HTTP::Client::Request::Range, DataFlow::CallCfgNode {
+      RequestCall() {
+        this =
+          instance()
+              .getMember(["request", "request_encode_url", "request_encode_body", "urlopen"])
+              .getACall()
+      }
+
+      override DataFlow::Node getAUrlPart() { result in [this.getArg(1), this.getArgByName("url")] }
+
+      override string getFramework() { result = "urllib3.PoolManager" }
+
+      override predicate disablesCertificateValidation(
+        DataFlow::Node disablingNode, DataFlow::Node argumentOrigin
+      ) {
+        // TODO: Look into disabling certificate validation
+        none()
+      }
+    }
+  }
+}

python/ql/test/library-tests/frameworks/aiohttp/client_request.py

@@ -0,0 +1,16 @@
+import aiohttp
+import asyncio
+
+s = aiohttp.ClientSession()
+resp = s.request("method", "url") # $ clientRequestUrlPart="url"
+resp = s.request("method", url="url") # $ clientRequestUrlPart="url"
+
+with aiohttp.ClientSession() as session:
+    resp = session.get("url") # $ clientRequestUrlPart="url"
+    resp = session.request(method="GET", url="url") # $ clientRequestUrlPart="url"
+
+# other methods than GET
+s = aiohttp.ClientSession()
+resp = s.post("url") # $ clientRequestUrlPart="url"
+resp = s.patch("url") # $ clientRequestUrlPart="url"
+resp = s.options("url") # $ clientRequestUrlPart="url"

python/ql/test/library-tests/frameworks/httpx/ConceptsTest.ql

@@ -0,0 +1,2 @@
+import python
+import experimental.meta.ConceptsTest

python/ql/test/library-tests/frameworks/httpx/test.py

@@ -0,0 +1,24 @@
+import httpx
+
+httpx.get("url") # $ clientRequestUrlPart="url"
+httpx.post("url") # $ clientRequestUrlPart="url"
+httpx.patch("url") # $ clientRequestUrlPart="url"
+httpx.options("url") # $ clientRequestUrlPart="url"
+httpx.request("method", url="url") # $ clientRequestUrlPart="url"
+httpx.stream("method", url="url") # $ clientRequestUrlPart="url"
+
+client = httpx.Client()
+response = client.get("url") # $ clientRequestUrlPart="url"
+response = client.post("url") # $ clientRequestUrlPart="url"
+response = client.patch("url") # $ clientRequestUrlPart="url"
+response = client.options("url") # $ clientRequestUrlPart="url"
+response = client.request("method", url="url") # $ clientRequestUrlPart="url"
+response = client.stream("method", url="url") # $ clientRequestUrlPart="url"
+
+client = httpx.AsyncClient()
+response = client.get("url") # $ clientRequestUrlPart="url"
+response = client.post("url") # $ clientRequestUrlPart="url"
+response = client.patch("url") # $ clientRequestUrlPart="url"
+response = client.options("url") # $ clientRequestUrlPart="url"
+response = client.request("method", url="url") # $ clientRequestUrlPart="url"
+response = client.stream("method", url="url") # $ clientRequestUrlPart="url"

python/ql/test/library-tests/frameworks/libtaxii/ConceptsTest.ql

@@ -0,0 +1,2 @@
+import python
+import experimental.meta.ConceptsTest

python/ql/test/library-tests/frameworks/libtaxii/test.py

@@ -0,0 +1,4 @@
+from libtaxii.common import parse
+
+result = parse("url", allow_url=True) # $ clientRequestUrlPart="url"
+result = parse(s="url", allow_url=True) # $ clientRequestUrlPart="url"

python/ql/test/library-tests/frameworks/pycurl/ConceptsTest.ql

@@ -0,0 +1,2 @@
+import python
+import experimental.meta.ConceptsTest

python/ql/test/library-tests/frameworks/pycurl/test.py

@@ -0,0 +1,4 @@
+import pycurl
+
+c = pycurl.Curl()
+c.setopt(pycurl.URL, "url") # $ clientRequestUrlPart="url"

python/ql/test/library-tests/frameworks/stdlib-py2/urllib2.py

@@ -0,0 +1,7 @@
+import urllib2
+
+resp = urllib2.Request("url") # $ clientRequestUrlPart="url"
+resp = urllib2.Request(url="url") # $ clientRequestUrlPart="url"
+
+resp = urllib2.urlopen("url") # $ clientRequestUrlPart="url"
+resp = urllib2.urlopen(url="url") # $ clientRequestUrlPart="url"

python/ql/test/library-tests/frameworks/stdlib/urllib.py

@@ -0,0 +1,7 @@
+from urllib.request import Request, urlopen
+
+Request("url") # $ clientRequestUrlPart="url"
+Request(url="url") # $ clientRequestUrlPart="url"
+
+urlopen("url") # $ clientRequestUrlPart="url"
+urlopen(url="url") # $ clientRequestUrlPart="url"

python/ql/test/library-tests/frameworks/urllib3/ConceptsTest.ql

@@ -0,0 +1,2 @@
+import python
+import experimental.meta.ConceptsTest

python/ql/test/library-tests/frameworks/urllib3/test.py

@@ -0,0 +1,29 @@
+import urllib3
+
+pool = urllib3.PoolManager()
+
+resp = pool.request("method", "url") # $ clientRequestUrlPart="url"
+resp = pool.request("method", url="url") # $ clientRequestUrlPart="url"
+resp = pool.urlopen("method", "url") # $ clientRequestUrlPart="url"
+resp = pool.urlopen("method", url="url") # $ clientRequestUrlPart="url"
+
+pool = urllib3.ProxyManager("http://proxy")
+
+resp = pool.request("method", "url") # $ clientRequestUrlPart="url"
+resp = pool.request("method", url="url") # $ clientRequestUrlPart="url"
+resp = pool.urlopen("method", "url") # $ clientRequestUrlPart="url"
+resp = pool.urlopen("method", url="url") # $ clientRequestUrlPart="url"
+
+pool = urllib3.HTTPConnectionPool("host")
+
+resp = pool.request("method", "url") # $ clientRequestUrlPart="url"
+resp = pool.request("method", url="url") # $ clientRequestUrlPart="url"
+resp = pool.urlopen("method", "url") # $ clientRequestUrlPart="url"
+resp = pool.urlopen("method", url="url") # $ clientRequestUrlPart="url"
+
+pool = urllib3.HTTPSConnectionPool("host")
+
+resp = pool.request("method", "url") # $ clientRequestUrlPart="url"
+resp = pool.request("method", url="url") # $ clientRequestUrlPart="url"
+resp = pool.urlopen("method", "url") # $ clientRequestUrlPart="url"
+resp = pool.urlopen("method", url="url") # $ clientRequestUrlPart="url"