From 8a3e4f14d1ab4149befdf84c8e4cf546d13bd3f5 Mon Sep 17 00:00:00 2001
From: jorgectf
Date: Sun, 25 Jul 2021 04:06:02 +0200
Subject: [PATCH 001/171] Add tests and `.qlref`
---
.../Security/CWE-614/InsecureCookie.qlref | 1 +
.../Security/CWE-614/django_bad.py | 13 +++++++
.../Security/CWE-614/django_good.py | 19 +++++++++++
.../query-tests/Security/CWE-614/flask_bad.py | 34 +++++++++++++++++++
.../Security/CWE-614/flask_good.py | 34 +++++++++++++++++++
5 files changed, 101 insertions(+)
create mode 100644 python/ql/test/experimental/query-tests/Security/CWE-614/InsecureCookie.qlref
create mode 100644 python/ql/test/experimental/query-tests/Security/CWE-614/django_bad.py
create mode 100644 python/ql/test/experimental/query-tests/Security/CWE-614/django_good.py
create mode 100644 python/ql/test/experimental/query-tests/Security/CWE-614/flask_bad.py
create mode 100644 python/ql/test/experimental/query-tests/Security/CWE-614/flask_good.py
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-614/InsecureCookie.qlref b/python/ql/test/experimental/query-tests/Security/CWE-614/InsecureCookie.qlref
new file mode 100644
index 00000000000..378d5dcae1a
--- /dev/null
+++ b/python/ql/test/experimental/query-tests/Security/CWE-614/InsecureCookie.qlref
@@ -0,0 +1 @@
+experimental/Security/CWE-614/InsecureCookie.ql
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-614/django_bad.py b/python/ql/test/experimental/query-tests/Security/CWE-614/django_bad.py
new file mode 100644
index 00000000000..877231f8f14
--- /dev/null
+++ b/python/ql/test/experimental/query-tests/Security/CWE-614/django_bad.py
@@ -0,0 +1,13 @@
+import django.http
+
+
+def django_response(request):
+ resp = django.http.HttpResponse()
+ resp.set_cookie("name", "value", secure=None)
+ return resp
+
+
+def django_response(request):
+ resp = django.http.HttpResponse()
+ resp.set_cookie("name", "value")
+ return resp
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-614/django_good.py b/python/ql/test/experimental/query-tests/Security/CWE-614/django_good.py
new file mode 100644
index 00000000000..ebf16236de2
--- /dev/null
+++ b/python/ql/test/experimental/query-tests/Security/CWE-614/django_good.py
@@ -0,0 +1,19 @@
+import django.http
+
+
+def django_response(request):
+ resp = django.http.HttpResponse()
+ resp['Set-Cookie'] = "name=value; Secure;"
+ return resp
+
+
+def django_response(request):
+ resp = django.http.HttpResponse()
+ resp.set_cookie("name", "value", secure=True)
+ return resp
+
+
+def indeterminate(secure):
+ resp = django.http.HttpResponse()
+ resp.set_cookie("name", "value", secure)
+ return resp
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-614/flask_bad.py b/python/ql/test/experimental/query-tests/Security/CWE-614/flask_bad.py
new file mode 100644
index 00000000000..7c7d6e8acd0
--- /dev/null
+++ b/python/ql/test/experimental/query-tests/Security/CWE-614/flask_bad.py
@@ -0,0 +1,34 @@
+from flask import Flask, request, make_response, Response
+
+app = Flask(__name__)
+
+
+@app.route("/false")
+def false():
+ resp = make_response()
+ resp.set_cookie("name", value="value", secure=False)
+ return resp
+
+
+@app.route("/none")
+def none():
+ resp = make_response()
+ resp.set_cookie("name", value="value", secure=None)
+ return resp
+
+
+@app.route("/flask_Response")
+def flask_Response():
+ resp = Response()
+ resp.headers['Set-Cookie'] = "name=value;"
+ return resp
+
+
+@app.route("/flask_make_response")
+def flask_make_response():
+ resp = make_response("hello")
+ resp.headers['Set-Cookie'] = "name=value;"
+ return resp
+
+# if __name__ == "__main__":
+# app.run(debug=True)
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-614/flask_good.py b/python/ql/test/experimental/query-tests/Security/CWE-614/flask_good.py
new file mode 100644
index 00000000000..05ee3f28657
--- /dev/null
+++ b/python/ql/test/experimental/query-tests/Security/CWE-614/flask_good.py
@@ -0,0 +1,34 @@
+from flask import Flask, request, make_response, Response
+
+app = Flask(__name__)
+
+
+@app.route("/true")
+def true():
+ resp = make_response()
+ resp.set_cookie("name", value="value", secure=True)
+ return resp
+
+
+@app.route("/flask_Response")
+def flask_Response():
+ resp = Response()
+ resp.headers['Set-Cookie'] = "name=value; Secure;"
+ return resp
+
+
+@app.route("/flask_make_response")
+def flask_make_response():
+ resp = make_response("hello")
+ resp.headers['Set-Cookie'] = "name=value; Secure;"
+ return resp
+
+
+def indeterminate(secure):
+ resp = make_response()
+ resp.set_cookie("name", value="value", secure=secure)
+ return resp
+
+
+# if __name__ == "__main__":
+# app.run(debug=True)
From c8983be947adac8c3faba902056c9c898cf848af Mon Sep 17 00:00:00 2001
From: jorgectf
Date: Sun, 25 Jul 2021 04:06:44 +0200
Subject: [PATCH 002/171] Add query
---
.../Security/CWE-614/InsecureCookie.ql | 30 +++++++++++++++++++
1 file changed, 30 insertions(+)
create mode 100644 python/ql/src/experimental/Security/CWE-614/InsecureCookie.ql
diff --git a/python/ql/src/experimental/Security/CWE-614/InsecureCookie.ql b/python/ql/src/experimental/Security/CWE-614/InsecureCookie.ql
new file mode 100644
index 00000000000..2405d6ddd4e
--- /dev/null
+++ b/python/ql/src/experimental/Security/CWE-614/InsecureCookie.ql
@@ -0,0 +1,30 @@
+/**
+ * @name Failure to use secure cookies
+ * @description Insecure cookies may be sent in cleartext, which makes them vulnerable to
+ * interception.
+ * @kind problem
+ * @problem.severity error
+ * @id py/insecure-cookie
+ * @tags security
+ * external/cwe/cwe-614
+ */
+
+// determine precision above
+import python
+import semmle.python.dataflow.new.DataFlow
+import semmle.python.Concepts
+import experimental.semmle.python.Concepts
+
+from HeaderDeclaration headerWrite, False f, None n
+where
+ exists(StrConst headerName, StrConst headerValue |
+ headerName.getText() = "Set-Cookie" and
+ DataFlow::exprNode(headerName).(DataFlow::LocalSourceNode).flowsTo(headerWrite.getNameArg()) and
+ not headerValue.getText().regexpMatch(".*; *Secure;.*") and
+ DataFlow::exprNode(headerValue).(DataFlow::LocalSourceNode).flowsTo(headerWrite.getValueArg())
+ )
+ or
+ [DataFlow::exprNode(f), DataFlow::exprNode(n)]
+ .(DataFlow::LocalSourceNode)
+ .flowsTo(headerWrite.(DataFlow::CallCfgNode).getArgByName("secure"))
+select headerWrite, "Cookie is added to response without the 'secure' flag being set."
From 4f68a1777ce5efe87507dce2847b3c4dcfb15066 Mon Sep 17 00:00:00 2001
From: jorgectf
Date: Sun, 25 Jul 2021 04:07:05 +0200
Subject: [PATCH 003/171] Write documentation and example
---
.../Security/CWE-614/InsecureCookie.py | 15 +++++++++++
.../Security/CWE-614/InsecureCookie.qhelp | 26 +++++++++++++++++++
2 files changed, 41 insertions(+)
create mode 100644 python/ql/src/experimental/Security/CWE-614/InsecureCookie.py
create mode 100644 python/ql/src/experimental/Security/CWE-614/InsecureCookie.qhelp
diff --git a/python/ql/src/experimental/Security/CWE-614/InsecureCookie.py b/python/ql/src/experimental/Security/CWE-614/InsecureCookie.py
new file mode 100644
index 00000000000..54bbeff7d12
--- /dev/null
+++ b/python/ql/src/experimental/Security/CWE-614/InsecureCookie.py
@@ -0,0 +1,15 @@
+from flask import Flask, request, make_response, Response
+
+
+@app.route("/true")
+def true():
+ resp = make_response()
+ resp.set_cookie("name", value="value", secure=True)
+ return resp
+
+
+@app.route("/flask_make_response")
+def flask_make_response():
+ resp = make_response("hello")
+ resp.headers['Set-Cookie'] = "name=value; Secure;"
+ return resp
\ No newline at end of file
diff --git a/python/ql/src/experimental/Security/CWE-614/InsecureCookie.qhelp b/python/ql/src/experimental/Security/CWE-614/InsecureCookie.qhelp
new file mode 100644
index 00000000000..ab5e3031629
--- /dev/null
+++ b/python/ql/src/experimental/Security/CWE-614/InsecureCookie.qhelp
@@ -0,0 +1,26 @@
+
+
+
+
+Failing to set the 'secure' flag on a cookie can cause it to be sent in cleartext.
+This makes it easier for an attacker to intercept.
+
+
+
+Always set secure to True or add "; Secure;" to the cookie's raw value.
+
+
+
+This example shows two ways of adding a cookie to a Flask response. The first way uses set_cookie's
+secure flag and the second adds the secure flag in the cookie's raw value.
+
+
+
+
+Detectify: Cookie lack Secure flag .
+PortSwigger: TLS cookie without secure flag set .
+
+
+
\ No newline at end of file
From 65044293dd680552ac9782656d48354e0fa06b42 Mon Sep 17 00:00:00 2001
From: jorgectf
Date: Sun, 25 Jul 2021 17:53:58 +0200
Subject: [PATCH 004/171] Add `CookieWrite` concept
---
.../experimental/semmle/python/Concepts.qll | 58 +++++++++++++++++++
.../semmle/python/frameworks/Django.qll | 11 ++++
2 files changed, 69 insertions(+)
diff --git a/python/ql/src/experimental/semmle/python/Concepts.qll b/python/ql/src/experimental/semmle/python/Concepts.qll
index 809176a9d52..56543246784 100644
--- a/python/ql/src/experimental/semmle/python/Concepts.qll
+++ b/python/ql/src/experimental/semmle/python/Concepts.qll
@@ -252,3 +252,61 @@ class HeaderDeclaration extends DataFlow::Node {
*/
DataFlow::Node getValueArg() { result = range.getValueArg() }
}
+
+module ExperimentalHTTP {
+ /**
+ * A data-flow node that sets a cookie in an HTTP response.
+ *
+ * Extend this class to refine existing API models. If you want to model new APIs,
+ * extend `HTTP::CookieWrite::Range` instead.
+ */
+ class CookieWrite extends DataFlow::Node {
+ CookieWrite::Range range;
+
+ CookieWrite() { this = range }
+
+ /**
+ * Gets the argument, if any, specifying the raw cookie header.
+ */
+ DataFlow::Node getHeaderArg() { result = range.getHeaderArg() }
+
+ /**
+ * Gets the argument, if any, specifying the cookie name.
+ */
+ DataFlow::Node getNameArg() { result = range.getNameArg() }
+
+ /**
+ * Gets the argument, if any, specifying the cookie value.
+ */
+ DataFlow::Node getValueArg() { result = range.getValueArg() }
+ }
+
+ /** Provides a class for modeling new cookie writes on HTTP responses. */
+ module CookieWrite {
+ /**
+ * A data-flow node that sets a cookie in an HTTP response.
+ *
+ * Note: we don't require that this redirect must be sent to a client (a kind of
+ * "if a tree falls in a forest and nobody hears it" situation).
+ *
+ * Extend this class to model new APIs. If you want to refine existing API models,
+ * extend `HttpResponse` instead.
+ */
+ abstract class Range extends DataFlow::Node {
+ /**
+ * Gets the argument, if any, specifying the raw cookie header.
+ */
+ abstract DataFlow::Node getHeaderArg();
+
+ /**
+ * Gets the argument, if any, specifying the cookie name.
+ */
+ abstract DataFlow::Node getNameArg();
+
+ /**
+ * Gets the argument, if any, specifying the cookie value.
+ */
+ abstract DataFlow::Node getValueArg();
+ }
+ }
+}
diff --git a/python/ql/src/experimental/semmle/python/frameworks/Django.qll b/python/ql/src/experimental/semmle/python/frameworks/Django.qll
index c525b73b40e..da7db6fd18b 100644
--- a/python/ql/src/experimental/semmle/python/frameworks/Django.qll
+++ b/python/ql/src/experimental/semmle/python/frameworks/Django.qll
@@ -75,6 +75,17 @@ private module PrivateDjango {
override DataFlow::Node getValueArg() { result = headerInput }
}
+
+ class DjangoSetCookieCall extends DataFlow::CallCfgNode,
+ ExperimentalHTTP::CookieWrite::Range {
+ DjangoSetCookieCall() { this = baseClassRef().getMember("set_cookie").getACall() }
+
+ override DataFlow::Node getHeaderArg() { none() }
+
+ override DataFlow::Node getNameArg() { result = this.getArg(0) }
+
+ override DataFlow::Node getValueArg() { result = this.getArg(1) }
+ }
}
}
}
From 983465963aeeb403673ffce9ab4b61c275bd1af3 Mon Sep 17 00:00:00 2001
From: jorgectf
Date: Sun, 25 Jul 2021 18:18:29 +0200
Subject: [PATCH 005/171] Polish `CookieWrite`
---
.../Security/CWE-614/InsecureCookie.ql | 18 +++++++++++-------
.../semmle/python/frameworks/Flask.qll | 16 ++++++++++++++++
.../query-tests/Security/CWE-614/django_bad.py | 2 +-
3 files changed, 28 insertions(+), 8 deletions(-)
diff --git a/python/ql/src/experimental/Security/CWE-614/InsecureCookie.ql b/python/ql/src/experimental/Security/CWE-614/InsecureCookie.ql
index 2405d6ddd4e..8a57aea8d69 100644
--- a/python/ql/src/experimental/Security/CWE-614/InsecureCookie.ql
+++ b/python/ql/src/experimental/Security/CWE-614/InsecureCookie.ql
@@ -15,16 +15,20 @@ import semmle.python.dataflow.new.DataFlow
import semmle.python.Concepts
import experimental.semmle.python.Concepts
-from HeaderDeclaration headerWrite, False f, None n
+from Expr cookieExpr, False f, None n
where
- exists(StrConst headerName, StrConst headerValue |
+ exists(HeaderDeclaration headerWrite, StrConst headerName, StrConst headerValue |
headerName.getText() = "Set-Cookie" and
DataFlow::exprNode(headerName).(DataFlow::LocalSourceNode).flowsTo(headerWrite.getNameArg()) and
not headerValue.getText().regexpMatch(".*; *Secure;.*") and
- DataFlow::exprNode(headerValue).(DataFlow::LocalSourceNode).flowsTo(headerWrite.getValueArg())
+ DataFlow::exprNode(headerValue).(DataFlow::LocalSourceNode).flowsTo(headerWrite.getValueArg()) and
+ cookieExpr = headerWrite.asExpr()
)
or
- [DataFlow::exprNode(f), DataFlow::exprNode(n)]
- .(DataFlow::LocalSourceNode)
- .flowsTo(headerWrite.(DataFlow::CallCfgNode).getArgByName("secure"))
-select headerWrite, "Cookie is added to response without the 'secure' flag being set."
+ exists(ExperimentalHTTP::CookieWrite cookieWrite |
+ [DataFlow::exprNode(f), DataFlow::exprNode(n)]
+ .(DataFlow::LocalSourceNode)
+ .flowsTo(cookieWrite.(DataFlow::CallCfgNode).getArgByName("secure")) and
+ cookieExpr = cookieWrite.asExpr()
+ )
+select cookieExpr, "Cookie is added to response without the 'secure' flag being set."
diff --git a/python/ql/src/experimental/semmle/python/frameworks/Flask.qll b/python/ql/src/experimental/semmle/python/frameworks/Flask.qll
index 9c66d9a4601..2a2cc68fe84 100644
--- a/python/ql/src/experimental/semmle/python/frameworks/Flask.qll
+++ b/python/ql/src/experimental/semmle/python/frameworks/Flask.qll
@@ -81,4 +81,20 @@ module ExperimentalFlask {
override DataFlow::Node getValueArg() { result.asExpr() = item.getValue() }
}
+
+ class DjangoSetCookieCall extends DataFlow::CallCfgNode, ExperimentalHTTP::CookieWrite::Range {
+ DjangoSetCookieCall() {
+ this =
+ [Flask::Response::classRef(), flaskMakeResponse()]
+ .getReturn()
+ .getMember("set_cookie")
+ .getACall()
+ }
+
+ override DataFlow::Node getHeaderArg() { none() }
+
+ override DataFlow::Node getNameArg() { result = this.getArg(0) }
+
+ override DataFlow::Node getValueArg() { result = this.getArg(1) }
+ }
}
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-614/django_bad.py b/python/ql/test/experimental/query-tests/Security/CWE-614/django_bad.py
index 877231f8f14..02752d32f99 100644
--- a/python/ql/test/experimental/query-tests/Security/CWE-614/django_bad.py
+++ b/python/ql/test/experimental/query-tests/Security/CWE-614/django_bad.py
@@ -9,5 +9,5 @@ def django_response(request):
def django_response(request):
resp = django.http.HttpResponse()
- resp.set_cookie("name", "value")
+ resp.set_cookie("name", "value", secure=False)
return resp
From c8a7f48d6efa57bd9907086742b0860a4d3bb4a0 Mon Sep 17 00:00:00 2001
From: jorgectf
Date: Sun, 25 Jul 2021 18:18:38 +0200
Subject: [PATCH 006/171] Add `.expected`
---
.../query-tests/Security/CWE-614/InsecureCookie.expected | 6 ++++++
1 file changed, 6 insertions(+)
create mode 100644 python/ql/test/experimental/query-tests/Security/CWE-614/InsecureCookie.expected
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-614/InsecureCookie.expected b/python/ql/test/experimental/query-tests/Security/CWE-614/InsecureCookie.expected
new file mode 100644
index 00000000000..5c157a11976
--- /dev/null
+++ b/python/ql/test/experimental/query-tests/Security/CWE-614/InsecureCookie.expected
@@ -0,0 +1,6 @@
+| django_bad.py:6:5:6:49 | Attribute() | Cookie is added to response without the 'secure' flag being set. |
+| django_bad.py:12:5:12:50 | Attribute() | Cookie is added to response without the 'secure' flag being set. |
+| flask_bad.py:9:5:9:56 | Attribute() | Cookie is added to response without the 'secure' flag being set. |
+| flask_bad.py:16:5:16:55 | Attribute() | Cookie is added to response without the 'secure' flag being set. |
+| flask_bad.py:23:5:23:30 | Subscript | Cookie is added to response without the 'secure' flag being set. |
+| flask_bad.py:30:5:30:30 | Subscript | Cookie is added to response without the 'secure' flag being set. |
From 54ed25a925caf2e05e2c5189dc3c0e4f99d463d9 Mon Sep 17 00:00:00 2001
From: jorgectf
Date: Sun, 25 Jul 2021 18:21:16 +0200
Subject: [PATCH 007/171] Change `False` and `None` scopes
---
python/ql/src/experimental/Security/CWE-614/InsecureCookie.ql | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/python/ql/src/experimental/Security/CWE-614/InsecureCookie.ql b/python/ql/src/experimental/Security/CWE-614/InsecureCookie.ql
index 8a57aea8d69..02b1280abab 100644
--- a/python/ql/src/experimental/Security/CWE-614/InsecureCookie.ql
+++ b/python/ql/src/experimental/Security/CWE-614/InsecureCookie.ql
@@ -15,7 +15,7 @@ import semmle.python.dataflow.new.DataFlow
import semmle.python.Concepts
import experimental.semmle.python.Concepts
-from Expr cookieExpr, False f, None n
+from Expr cookieExpr
where
exists(HeaderDeclaration headerWrite, StrConst headerName, StrConst headerValue |
headerName.getText() = "Set-Cookie" and
@@ -25,7 +25,7 @@ where
cookieExpr = headerWrite.asExpr()
)
or
- exists(ExperimentalHTTP::CookieWrite cookieWrite |
+ exists(ExperimentalHTTP::CookieWrite cookieWrite, False f, None n |
[DataFlow::exprNode(f), DataFlow::exprNode(n)]
.(DataFlow::LocalSourceNode)
.flowsTo(cookieWrite.(DataFlow::CallCfgNode).getArgByName("secure")) and
From 48c3c3d8a8df6cf3d0a65d00fe400aae1537314a Mon Sep 17 00:00:00 2001
From: jorgectf
Date: Wed, 27 Oct 2021 21:00:50 +0200
Subject: [PATCH 008/171] Broaden scope
---
.../Security/CWE-614/InsecureCookie.ql | 25 +++++-------
.../experimental/semmle/python/Concepts.qll | 39 +++++++++----------
.../semmle/python/CookieHeader.qll | 26 +++++++++++++
.../semmle/python/frameworks/Django.qll | 19 ++++++---
.../semmle/python/frameworks/Flask.qll | 20 +++++++---
5 files changed, 82 insertions(+), 47 deletions(-)
create mode 100644 python/ql/src/experimental/semmle/python/CookieHeader.qll
diff --git a/python/ql/src/experimental/Security/CWE-614/InsecureCookie.ql b/python/ql/src/experimental/Security/CWE-614/InsecureCookie.ql
index 02b1280abab..bf0ff22d45e 100644
--- a/python/ql/src/experimental/Security/CWE-614/InsecureCookie.ql
+++ b/python/ql/src/experimental/Security/CWE-614/InsecureCookie.ql
@@ -12,23 +12,16 @@
// determine precision above
import python
import semmle.python.dataflow.new.DataFlow
-import semmle.python.Concepts
import experimental.semmle.python.Concepts
-from Expr cookieExpr
+from Cookie cookie, string alert
where
- exists(HeaderDeclaration headerWrite, StrConst headerName, StrConst headerValue |
- headerName.getText() = "Set-Cookie" and
- DataFlow::exprNode(headerName).(DataFlow::LocalSourceNode).flowsTo(headerWrite.getNameArg()) and
- not headerValue.getText().regexpMatch(".*; *Secure;.*") and
- DataFlow::exprNode(headerValue).(DataFlow::LocalSourceNode).flowsTo(headerWrite.getValueArg()) and
- cookieExpr = headerWrite.asExpr()
- )
+ cookie.isSecure() and
+ alert = "secure"
or
- exists(ExperimentalHTTP::CookieWrite cookieWrite, False f, None n |
- [DataFlow::exprNode(f), DataFlow::exprNode(n)]
- .(DataFlow::LocalSourceNode)
- .flowsTo(cookieWrite.(DataFlow::CallCfgNode).getArgByName("secure")) and
- cookieExpr = cookieWrite.asExpr()
- )
-select cookieExpr, "Cookie is added to response without the 'secure' flag being set."
+ not cookie.isHttpOnly() and
+ alert = "httponly"
+ or
+ cookie.isSameSite() and
+ alert = "samesite"
+select cookie, "Cookie is added without the ", alert, " flag properly set."
diff --git a/python/ql/src/experimental/semmle/python/Concepts.qll b/python/ql/src/experimental/semmle/python/Concepts.qll
index 9da35b854a5..b6a15cb025b 100644
--- a/python/ql/src/experimental/semmle/python/Concepts.qll
+++ b/python/ql/src/experimental/semmle/python/Concepts.qll
@@ -301,54 +301,51 @@ class HeaderDeclaration extends DataFlow::Node {
* A data-flow node that sets a cookie in an HTTP response.
*
* Extend this class to refine existing API models. If you want to model new APIs,
- * extend `HTTP::CookieWrite::Range` instead.
+ * extend `Cookie::Range` instead.
*/
-class CookieWrite extends DataFlow::Node {
- CookieWrite::Range range;
+class Cookie extends DataFlow::Node {
+ Cookie::Range range;
- CookieWrite() { this = range }
+ Cookie() { this = range }
/**
- * Gets the argument, if any, specifying the raw cookie header.
+ * Holds if this cookie is secure.
*/
- DataFlow::Node getHeaderArg() { result = range.getHeaderArg() }
+ predicate isSecure() { range.isSecure() }
/**
- * Gets the argument, if any, specifying the cookie name.
+ * Holds if this cookie is HttpOnly.
*/
- DataFlow::Node getNameArg() { result = range.getNameArg() }
+ predicate isHttpOnly() { range.isHttpOnly() }
/**
- * Gets the argument, if any, specifying the cookie value.
+ * Holds if the cookie is SameSite
*/
- DataFlow::Node getValueArg() { result = range.getValueArg() }
+ predicate isSameSite() { range.isSameSite() }
}
/** Provides a class for modeling new cookie writes on HTTP responses. */
-module CookieWrite {
+module Cookie {
/**
* A data-flow node that sets a cookie in an HTTP response.
*
- * Note: we don't require that this redirect must be sent to a client (a kind of
- * "if a tree falls in a forest and nobody hears it" situation).
- *
* Extend this class to model new APIs. If you want to refine existing API models,
- * extend `HttpResponse` instead.
+ * extend `Cookie` instead.
*/
abstract class Range extends DataFlow::Node {
/**
- * Gets the argument, if any, specifying the raw cookie header.
+ * Holds if this cookie is secure.
*/
- abstract DataFlow::Node getHeaderArg();
+ abstract predicate isSecure();
/**
- * Gets the argument, if any, specifying the cookie name.
+ * Holds if this cookie is HttpOnly.
*/
- abstract DataFlow::Node getNameArg();
+ abstract predicate isHttpOnly();
/**
- * Gets the argument, if any, specifying the cookie value.
+ * Holds if the cookie is SameSite.
*/
- abstract DataFlow::Node getValueArg();
+ abstract predicate isSameSite();
}
}
diff --git a/python/ql/src/experimental/semmle/python/CookieHeader.qll b/python/ql/src/experimental/semmle/python/CookieHeader.qll
new file mode 100644
index 00000000000..610fa311310
--- /dev/null
+++ b/python/ql/src/experimental/semmle/python/CookieHeader.qll
@@ -0,0 +1,26 @@
+/**
+ * Temporary: provides a class to extend current cookies to header declarations
+ */
+
+import python
+import semmle.python.dataflow.new.DataFlow
+import semmle.python.dataflow.new.TaintTracking
+import experimental.semmle.python.Concepts
+
+class CookieHeader extends HeaderDeclaration, Cookie::Range {
+ CookieHeader() {
+ this instanceof HeaderDeclaration and this.getNameArg().asExpr().(Str_).getS() = "Set-Cookie"
+ }
+
+ override predicate isSecure() {
+ this.getValueArg().asExpr().(Str_).getS().regexpMatch(".*; *Secure;.*")
+ }
+
+ override predicate isHttpOnly() {
+ this.getValueArg().asExpr().(Str_).getS().regexpMatch(".*; *HttpOnly;.*")
+ }
+
+ override predicate isSameSite() {
+ this.getValueArg().asExpr().(Str_).getS().regexpMatch(".*; *SameSite=(Strict|Lax);.*")
+ }
+}
diff --git a/python/ql/src/experimental/semmle/python/frameworks/Django.qll b/python/ql/src/experimental/semmle/python/frameworks/Django.qll
index 1a3c5c1cf10..fb6762d3dc3 100644
--- a/python/ql/src/experimental/semmle/python/frameworks/Django.qll
+++ b/python/ql/src/experimental/semmle/python/frameworks/Django.qll
@@ -87,15 +87,24 @@ private module PrivateDjango {
override DataFlow::Node getValueArg() { result = headerInput }
}
- class DjangoSetCookieCall extends DataFlow::CallCfgNode,
- ExperimentalHTTP::CookieWrite::Range {
+ class DjangoSetCookieCall extends DataFlow::CallCfgNode, Cookie::Range {
DjangoSetCookieCall() { this = baseClassRef().getMember("set_cookie").getACall() }
- override DataFlow::Node getHeaderArg() { none() }
+ override predicate isSecure() {
+ DataFlow::exprNode(any(True t))
+ .(DataFlow::LocalSourceNode)
+ .flowsTo(this.getArgByName("secure"))
+ }
- override DataFlow::Node getNameArg() { result = this.getArg(0) }
+ override predicate isHttpOnly() {
+ DataFlow::exprNode(any(True t))
+ .(DataFlow::LocalSourceNode)
+ .flowsTo(this.getArgByName("httponly"))
+ }
- override DataFlow::Node getValueArg() { result = this.getArg(1) }
+ override predicate isSameSite() {
+ this.getArgByName("samesite").asExpr().(Str_).getS() in ["Strict", "Lax"]
+ }
}
}
}
diff --git a/python/ql/src/experimental/semmle/python/frameworks/Flask.qll b/python/ql/src/experimental/semmle/python/frameworks/Flask.qll
index 2a2cc68fe84..614e0738bcc 100644
--- a/python/ql/src/experimental/semmle/python/frameworks/Flask.qll
+++ b/python/ql/src/experimental/semmle/python/frameworks/Flask.qll
@@ -82,8 +82,8 @@ module ExperimentalFlask {
override DataFlow::Node getValueArg() { result.asExpr() = item.getValue() }
}
- class DjangoSetCookieCall extends DataFlow::CallCfgNode, ExperimentalHTTP::CookieWrite::Range {
- DjangoSetCookieCall() {
+ class FlaskSetCookieCall extends DataFlow::CallCfgNode, Cookie::Range {
+ FlaskSetCookieCall() {
this =
[Flask::Response::classRef(), flaskMakeResponse()]
.getReturn()
@@ -91,10 +91,20 @@ module ExperimentalFlask {
.getACall()
}
- override DataFlow::Node getHeaderArg() { none() }
+ override predicate isSecure() {
+ DataFlow::exprNode(any(True t))
+ .(DataFlow::LocalSourceNode)
+ .flowsTo(this.getArgByName("secure"))
+ }
- override DataFlow::Node getNameArg() { result = this.getArg(0) }
+ override predicate isHttpOnly() {
+ DataFlow::exprNode(any(True t))
+ .(DataFlow::LocalSourceNode)
+ .flowsTo(this.getArgByName("httponly"))
+ }
- override DataFlow::Node getValueArg() { result = this.getArg(1) }
+ override predicate isSameSite() {
+ this.getArgByName("samesite").asExpr().(Str_).getS() in ["Strict", "Lax"]
+ }
}
}
From 0f2b81e0d2ae3ed0b56763870d0f7787696ae54e Mon Sep 17 00:00:00 2001
From: jorgectf
Date: Thu, 28 Oct 2021 09:24:47 +0200
Subject: [PATCH 009/171] Polish tests
---
.../query-tests/Security/CWE-614/django_bad.py | 6 ++++--
.../query-tests/Security/CWE-614/django_good.py | 5 +++--
.../query-tests/Security/CWE-614/flask_bad.py | 12 +++---------
.../query-tests/Security/CWE-614/flask_good.py | 7 ++++---
4 files changed, 14 insertions(+), 16 deletions(-)
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-614/django_bad.py b/python/ql/test/experimental/query-tests/Security/CWE-614/django_bad.py
index 02752d32f99..340291a6b9c 100644
--- a/python/ql/test/experimental/query-tests/Security/CWE-614/django_bad.py
+++ b/python/ql/test/experimental/query-tests/Security/CWE-614/django_bad.py
@@ -3,11 +3,13 @@ import django.http
def django_response(request):
resp = django.http.HttpResponse()
- resp.set_cookie("name", "value", secure=None)
+ resp.set_cookie("name", "value", secure=False,
+ httponly=False, samesite='None')
return resp
def django_response(request):
resp = django.http.HttpResponse()
- resp.set_cookie("name", "value", secure=False)
+ resp.set_cookie("name", "value", secure=False,
+ httponly=False, samesite='None')
return resp
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-614/django_good.py b/python/ql/test/experimental/query-tests/Security/CWE-614/django_good.py
index ebf16236de2..7476971cbb5 100644
--- a/python/ql/test/experimental/query-tests/Security/CWE-614/django_good.py
+++ b/python/ql/test/experimental/query-tests/Security/CWE-614/django_good.py
@@ -3,13 +3,14 @@ import django.http
def django_response(request):
resp = django.http.HttpResponse()
- resp['Set-Cookie'] = "name=value; Secure;"
+ resp['Set-Cookie'] = "name=value; Secure; HttpOnly; SameSite=Lax;"
return resp
def django_response(request):
resp = django.http.HttpResponse()
- resp.set_cookie("name", "value", secure=True)
+ resp.set_cookie("name", "value", secure=True,
+ httponly=True, samesite='Lax')
return resp
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-614/flask_bad.py b/python/ql/test/experimental/query-tests/Security/CWE-614/flask_bad.py
index 7c7d6e8acd0..f32d28a6f65 100644
--- a/python/ql/test/experimental/query-tests/Security/CWE-614/flask_bad.py
+++ b/python/ql/test/experimental/query-tests/Security/CWE-614/flask_bad.py
@@ -6,14 +6,8 @@ app = Flask(__name__)
@app.route("/false")
def false():
resp = make_response()
- resp.set_cookie("name", value="value", secure=False)
- return resp
-
-
-@app.route("/none")
-def none():
- resp = make_response()
- resp.set_cookie("name", value="value", secure=None)
+ resp.set_cookie("name", value="value", secure=False,
+ httponly=False, samesite='None')
return resp
@@ -27,7 +21,7 @@ def flask_Response():
@app.route("/flask_make_response")
def flask_make_response():
resp = make_response("hello")
- resp.headers['Set-Cookie'] = "name=value;"
+ resp.headers['Set-Cookie'] = "name=value; SameSite=None;"
return resp
# if __name__ == "__main__":
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-614/flask_good.py b/python/ql/test/experimental/query-tests/Security/CWE-614/flask_good.py
index 05ee3f28657..5b9f83e1a63 100644
--- a/python/ql/test/experimental/query-tests/Security/CWE-614/flask_good.py
+++ b/python/ql/test/experimental/query-tests/Security/CWE-614/flask_good.py
@@ -6,21 +6,22 @@ app = Flask(__name__)
@app.route("/true")
def true():
resp = make_response()
- resp.set_cookie("name", value="value", secure=True)
+ resp.set_cookie("name", value="value", secure=True,
+ httponly=True, samesite='Lax')
return resp
@app.route("/flask_Response")
def flask_Response():
resp = Response()
- resp.headers['Set-Cookie'] = "name=value; Secure;"
+ resp.headers['Set-Cookie'] = "name=value; Secure; HttpOnly; SameSite=Lax;"
return resp
@app.route("/flask_make_response")
def flask_make_response():
resp = make_response("hello")
- resp.headers['Set-Cookie'] = "name=value; Secure;"
+ resp.headers['Set-Cookie'] = "name=value; Secure; HttpOnly; SameSite=Lax;"
return resp
From 5dc1ad6f8ab9e3c656e5faac12e5b918a54ac730 Mon Sep 17 00:00:00 2001
From: jorgectf
Date: Thu, 28 Oct 2021 09:25:47 +0200
Subject: [PATCH 010/171] Polish `.ql`
---
.../src/experimental/Security/CWE-614/InsecureCookie.qhelp | 2 +-
.../ql/src/experimental/Security/CWE-614/InsecureCookie.ql | 5 +++--
2 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/python/ql/src/experimental/Security/CWE-614/InsecureCookie.qhelp b/python/ql/src/experimental/Security/CWE-614/InsecureCookie.qhelp
index ab5e3031629..97df2e49e13 100644
--- a/python/ql/src/experimental/Security/CWE-614/InsecureCookie.qhelp
+++ b/python/ql/src/experimental/Security/CWE-614/InsecureCookie.qhelp
@@ -23,4 +23,4 @@ secure flag and the second adds the secure flag in the cookie's raw value.
PortSwigger: TLS cookie without secure flag set .
-
\ No newline at end of file
+
diff --git a/python/ql/src/experimental/Security/CWE-614/InsecureCookie.ql b/python/ql/src/experimental/Security/CWE-614/InsecureCookie.ql
index bf0ff22d45e..ee22243e5c3 100644
--- a/python/ql/src/experimental/Security/CWE-614/InsecureCookie.ql
+++ b/python/ql/src/experimental/Security/CWE-614/InsecureCookie.ql
@@ -13,15 +13,16 @@
import python
import semmle.python.dataflow.new.DataFlow
import experimental.semmle.python.Concepts
+import experimental.semmle.python.CookieHeader
from Cookie cookie, string alert
where
- cookie.isSecure() and
+ not cookie.isSecure() and
alert = "secure"
or
not cookie.isHttpOnly() and
alert = "httponly"
or
- cookie.isSameSite() and
+ not cookie.isSameSite() and
alert = "samesite"
select cookie, "Cookie is added without the ", alert, " flag properly set."
From 129edd605ea7092fa3d309b3f0e322f983d69b3e Mon Sep 17 00:00:00 2001
From: jorgectf
Date: Thu, 28 Oct 2021 09:25:56 +0200
Subject: [PATCH 011/171] Update `.expected`
---
.../Security/CWE-614/InsecureCookie.expected | 27 ++++++++++++++-----
1 file changed, 21 insertions(+), 6 deletions(-)
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-614/InsecureCookie.expected b/python/ql/test/experimental/query-tests/Security/CWE-614/InsecureCookie.expected
index 5c157a11976..a04ad9cdafe 100644
--- a/python/ql/test/experimental/query-tests/Security/CWE-614/InsecureCookie.expected
+++ b/python/ql/test/experimental/query-tests/Security/CWE-614/InsecureCookie.expected
@@ -1,6 +1,21 @@
-| django_bad.py:6:5:6:49 | Attribute() | Cookie is added to response without the 'secure' flag being set. |
-| django_bad.py:12:5:12:50 | Attribute() | Cookie is added to response without the 'secure' flag being set. |
-| flask_bad.py:9:5:9:56 | Attribute() | Cookie is added to response without the 'secure' flag being set. |
-| flask_bad.py:16:5:16:55 | Attribute() | Cookie is added to response without the 'secure' flag being set. |
-| flask_bad.py:23:5:23:30 | Subscript | Cookie is added to response without the 'secure' flag being set. |
-| flask_bad.py:30:5:30:30 | Subscript | Cookie is added to response without the 'secure' flag being set. |
+| django_bad.py:6:5:7:52 | ControlFlowNode for Attribute() | Cookie is added without the | httponly | flag properly set. |
+| django_bad.py:6:5:7:52 | ControlFlowNode for Attribute() | Cookie is added without the | samesite | flag properly set. |
+| django_bad.py:6:5:7:52 | ControlFlowNode for Attribute() | Cookie is added without the | secure | flag properly set. |
+| django_bad.py:13:5:14:52 | ControlFlowNode for Attribute() | Cookie is added without the | httponly | flag properly set. |
+| django_bad.py:13:5:14:52 | ControlFlowNode for Attribute() | Cookie is added without the | samesite | flag properly set. |
+| django_bad.py:13:5:14:52 | ControlFlowNode for Attribute() | Cookie is added without the | secure | flag properly set. |
+| django_good.py:19:5:19:44 | ControlFlowNode for Attribute() | Cookie is added without the | httponly | flag properly set. |
+| django_good.py:19:5:19:44 | ControlFlowNode for Attribute() | Cookie is added without the | samesite | flag properly set. |
+| django_good.py:19:5:19:44 | ControlFlowNode for Attribute() | Cookie is added without the | secure | flag properly set. |
+| flask_bad.py:9:5:10:52 | ControlFlowNode for Attribute() | Cookie is added without the | httponly | flag properly set. |
+| flask_bad.py:9:5:10:52 | ControlFlowNode for Attribute() | Cookie is added without the | samesite | flag properly set. |
+| flask_bad.py:9:5:10:52 | ControlFlowNode for Attribute() | Cookie is added without the | secure | flag properly set. |
+| flask_bad.py:17:5:17:30 | ControlFlowNode for Subscript | Cookie is added without the | httponly | flag properly set. |
+| flask_bad.py:17:5:17:30 | ControlFlowNode for Subscript | Cookie is added without the | samesite | flag properly set. |
+| flask_bad.py:17:5:17:30 | ControlFlowNode for Subscript | Cookie is added without the | secure | flag properly set. |
+| flask_bad.py:24:5:24:30 | ControlFlowNode for Subscript | Cookie is added without the | httponly | flag properly set. |
+| flask_bad.py:24:5:24:30 | ControlFlowNode for Subscript | Cookie is added without the | samesite | flag properly set. |
+| flask_bad.py:24:5:24:30 | ControlFlowNode for Subscript | Cookie is added without the | secure | flag properly set. |
+| flask_good.py:31:5:31:57 | ControlFlowNode for Attribute() | Cookie is added without the | httponly | flag properly set. |
+| flask_good.py:31:5:31:57 | ControlFlowNode for Attribute() | Cookie is added without the | samesite | flag properly set. |
+| flask_good.py:31:5:31:57 | ControlFlowNode for Attribute() | Cookie is added without the | secure | flag properly set. |
From cf9e9f9dd4b0ffc99d169d910e914cef78725bae Mon Sep 17 00:00:00 2001
From: jorgectf
Date: Thu, 28 Oct 2021 10:28:45 +0200
Subject: [PATCH 012/171] Add cookie injection query missing proper tests
---
.../Security/CWE-614/CookieInjection.ql | 28 +++++++++++++
.../experimental/semmle/python/Concepts.qll | 20 ++++++++++
.../semmle/python/CookieHeader.qll | 18 +++++++--
.../semmle/python/frameworks/Django.qll | 4 ++
.../semmle/python/frameworks/Flask.qll | 4 ++
.../security/injection/CookieInjection.qll | 40 +++++++++++++++++++
.../query-tests/Security/CWE-614/flask_bad.py | 2 +-
7 files changed, 111 insertions(+), 5 deletions(-)
create mode 100644 python/ql/src/experimental/Security/CWE-614/CookieInjection.ql
create mode 100644 python/ql/src/experimental/semmle/python/security/injection/CookieInjection.qll
diff --git a/python/ql/src/experimental/Security/CWE-614/CookieInjection.ql b/python/ql/src/experimental/Security/CWE-614/CookieInjection.ql
new file mode 100644
index 00000000000..e97ff962919
--- /dev/null
+++ b/python/ql/src/experimental/Security/CWE-614/CookieInjection.ql
@@ -0,0 +1,28 @@
+/**
+ * @name Failure to use secure cookies
+ * @description Insecure cookies may be sent in cleartext, which makes them vulnerable to
+ * interception.
+ * @kind problem
+ * @problem.severity error
+ * @id py/insecure-cookie
+ * @tags security
+ * external/cwe/cwe-614
+ */
+
+// determine precision above
+import python
+import semmle.python.dataflow.new.DataFlow
+import experimental.semmle.python.Concepts
+import experimental.semmle.python.CookieHeader
+import experimental.semmle.python.security.injection.CookieInjection
+
+from
+ CookieInjectionFlowConfig config, DataFlow::PathNode source, DataFlow::PathNode sink,
+ string insecure
+where
+ config.hasFlowPath(source, sink) and
+ if exists(sink.getNode().(CookieSink))
+ then insecure = "and it's " + sink.getNode().(CookieSink).getFlag() + " flag is not properly set"
+ else insecure = ""
+select sink.getNode(), "Cookie is constructed from a", source.getNode(), "user-supplied input",
+ insecure
diff --git a/python/ql/src/experimental/semmle/python/Concepts.qll b/python/ql/src/experimental/semmle/python/Concepts.qll
index b6a15cb025b..64aa755d9cf 100644
--- a/python/ql/src/experimental/semmle/python/Concepts.qll
+++ b/python/ql/src/experimental/semmle/python/Concepts.qll
@@ -322,6 +322,16 @@ class Cookie extends DataFlow::Node {
* Holds if the cookie is SameSite
*/
predicate isSameSite() { range.isSameSite() }
+
+ /**
+ * Gets the argument containing the header name.
+ */
+ DataFlow::Node getName() { result = range.getName() }
+
+ /**
+ * Gets the argument containing the header value.
+ */
+ DataFlow::Node getValue() { result = range.getValue() }
}
/** Provides a class for modeling new cookie writes on HTTP responses. */
@@ -347,5 +357,15 @@ module Cookie {
* Holds if the cookie is SameSite.
*/
abstract predicate isSameSite();
+
+ /**
+ * Gets the argument containing the header name.
+ */
+ abstract DataFlow::Node getName();
+
+ /**
+ * Gets the argument containing the header value.
+ */
+ abstract DataFlow::Node getValue();
}
}
diff --git a/python/ql/src/experimental/semmle/python/CookieHeader.qll b/python/ql/src/experimental/semmle/python/CookieHeader.qll
index 610fa311310..c7779aadd80 100644
--- a/python/ql/src/experimental/semmle/python/CookieHeader.qll
+++ b/python/ql/src/experimental/semmle/python/CookieHeader.qll
@@ -9,18 +9,28 @@ import experimental.semmle.python.Concepts
class CookieHeader extends HeaderDeclaration, Cookie::Range {
CookieHeader() {
- this instanceof HeaderDeclaration and this.getNameArg().asExpr().(Str_).getS() = "Set-Cookie"
+ this instanceof HeaderDeclaration and
+ this.(HeaderDeclaration).getNameArg().asExpr().(Str_).getS() = "Set-Cookie"
}
override predicate isSecure() {
- this.getValueArg().asExpr().(Str_).getS().regexpMatch(".*; *Secure;.*")
+ this.(HeaderDeclaration).getValueArg().asExpr().(Str_).getS().regexpMatch(".*; *Secure;.*")
}
override predicate isHttpOnly() {
- this.getValueArg().asExpr().(Str_).getS().regexpMatch(".*; *HttpOnly;.*")
+ this.(HeaderDeclaration).getValueArg().asExpr().(Str_).getS().regexpMatch(".*; *HttpOnly;.*")
}
override predicate isSameSite() {
- this.getValueArg().asExpr().(Str_).getS().regexpMatch(".*; *SameSite=(Strict|Lax);.*")
+ this.(HeaderDeclaration)
+ .getValueArg()
+ .asExpr()
+ .(Str_)
+ .getS()
+ .regexpMatch(".*; *SameSite=(Strict|Lax);.*")
}
+
+ override DataFlow::Node getName() { result = this.(HeaderDeclaration).getValueArg() }
+
+ override DataFlow::Node getValue() { result = this.(HeaderDeclaration).getValueArg() }
}
diff --git a/python/ql/src/experimental/semmle/python/frameworks/Django.qll b/python/ql/src/experimental/semmle/python/frameworks/Django.qll
index fb6762d3dc3..ba99ddaa800 100644
--- a/python/ql/src/experimental/semmle/python/frameworks/Django.qll
+++ b/python/ql/src/experimental/semmle/python/frameworks/Django.qll
@@ -90,6 +90,10 @@ private module PrivateDjango {
class DjangoSetCookieCall extends DataFlow::CallCfgNode, Cookie::Range {
DjangoSetCookieCall() { this = baseClassRef().getMember("set_cookie").getACall() }
+ override DataFlow::Node getName() { result = this.getArg(0) }
+
+ override DataFlow::Node getValue() { result = this.getArgByName("value") }
+
override predicate isSecure() {
DataFlow::exprNode(any(True t))
.(DataFlow::LocalSourceNode)
diff --git a/python/ql/src/experimental/semmle/python/frameworks/Flask.qll b/python/ql/src/experimental/semmle/python/frameworks/Flask.qll
index 614e0738bcc..b93e6713846 100644
--- a/python/ql/src/experimental/semmle/python/frameworks/Flask.qll
+++ b/python/ql/src/experimental/semmle/python/frameworks/Flask.qll
@@ -91,6 +91,10 @@ module ExperimentalFlask {
.getACall()
}
+ override DataFlow::Node getName() { result = this.getArg(0) }
+
+ override DataFlow::Node getValue() { result = this.getArgByName("value") }
+
override predicate isSecure() {
DataFlow::exprNode(any(True t))
.(DataFlow::LocalSourceNode)
diff --git a/python/ql/src/experimental/semmle/python/security/injection/CookieInjection.qll b/python/ql/src/experimental/semmle/python/security/injection/CookieInjection.qll
new file mode 100644
index 00000000000..41f7c2af7d3
--- /dev/null
+++ b/python/ql/src/experimental/semmle/python/security/injection/CookieInjection.qll
@@ -0,0 +1,40 @@
+import python
+import experimental.semmle.python.Concepts
+import semmle.python.dataflow.new.DataFlow
+import semmle.python.dataflow.new.TaintTracking
+import semmle.python.dataflow.new.RemoteFlowSources
+
+class CookieSink extends DataFlow::Node {
+ string flag;
+
+ CookieSink() {
+ exists(Cookie cookie |
+ this in [cookie.getName(), cookie.getValue()] and
+ (
+ not cookie.isSecure() and
+ flag = "secure"
+ or
+ not cookie.isHttpOnly() and
+ flag = "httponly"
+ or
+ not cookie.isSameSite() and
+ flag = "samesite"
+ )
+ )
+ }
+
+ string getFlag() { result = flag }
+}
+
+/**
+ * A taint-tracking configuration for detecting Cookie injections.
+ */
+class CookieInjectionFlowConfig extends TaintTracking::Configuration {
+ CookieInjectionFlowConfig() { this = "CookieInjectionFlowConfig" }
+
+ override predicate isSource(DataFlow::Node source) { source instanceof RemoteFlowSource }
+
+ override predicate isSink(DataFlow::Node sink) {
+ exists(Cookie c | sink in [c.getName(), c.getValue()])
+ }
+}
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-614/flask_bad.py b/python/ql/test/experimental/query-tests/Security/CWE-614/flask_bad.py
index f32d28a6f65..fc0177e3012 100644
--- a/python/ql/test/experimental/query-tests/Security/CWE-614/flask_bad.py
+++ b/python/ql/test/experimental/query-tests/Security/CWE-614/flask_bad.py
@@ -6,7 +6,7 @@ app = Flask(__name__)
@app.route("/false")
def false():
resp = make_response()
- resp.set_cookie("name", value="value", secure=False,
+ resp.set_cookie(request.args["name"], value=request.args["value"], secure=False,
httponly=False, samesite='None')
return resp
From 4cb78ac654981ba68960bb13352da99b9590bc1c Mon Sep 17 00:00:00 2001
From: jorgectf
Date: Fri, 5 Nov 2021 20:08:37 +0100
Subject: [PATCH 013/171] Fix typo
---
python/ql/src/experimental/Security/CWE-614/CookieInjection.ql | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/python/ql/src/experimental/Security/CWE-614/CookieInjection.ql b/python/ql/src/experimental/Security/CWE-614/CookieInjection.ql
index e97ff962919..b4a880e9a58 100644
--- a/python/ql/src/experimental/Security/CWE-614/CookieInjection.ql
+++ b/python/ql/src/experimental/Security/CWE-614/CookieInjection.ql
@@ -22,7 +22,7 @@ from
where
config.hasFlowPath(source, sink) and
if exists(sink.getNode().(CookieSink))
- then insecure = "and it's " + sink.getNode().(CookieSink).getFlag() + " flag is not properly set"
+ then insecure = "and its " + sink.getNode().(CookieSink).getFlag() + " flag is not properly set"
else insecure = ""
select sink.getNode(), "Cookie is constructed from a", source.getNode(), "user-supplied input",
insecure
From d7a79469e62e8cc98e2ccff0b6f53b2f43a0058e Mon Sep 17 00:00:00 2001
From: jorgectf
Date: Fri, 5 Nov 2021 20:08:52 +0100
Subject: [PATCH 014/171] Improve tests
---
.../Security/CWE-614/django_bad.py | 17 ++++++++++++--
.../query-tests/Security/CWE-614/flask_bad.py | 23 +++++++++++++------
.../Security/CWE-614/flask_good.py | 7 ------
3 files changed, 31 insertions(+), 16 deletions(-)
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-614/django_bad.py b/python/ql/test/experimental/query-tests/Security/CWE-614/django_bad.py
index 340291a6b9c..6f1916930e5 100644
--- a/python/ql/test/experimental/query-tests/Security/CWE-614/django_bad.py
+++ b/python/ql/test/experimental/query-tests/Security/CWE-614/django_bad.py
@@ -8,8 +8,21 @@ def django_response(request):
return resp
+def django_response():
+ response = django.http.HttpResponse()
+ response['Set-Cookie'] = "name=value; SameSite=None;"
+ return response
+
+
def django_response(request):
resp = django.http.HttpResponse()
- resp.set_cookie("name", "value", secure=False,
- httponly=False, samesite='None')
+ resp.set_cookie(django.http.request.GET.get("name"),
+ django.http.request.GET.get("value"),
+ secure=False, httponly=False, samesite='None')
return resp
+
+
+def django_response():
+ response = django.http.HttpResponse()
+ response['Set-Cookie'] = f"{django.http.request.GET.get('name')}={django.http.request.GET.get('value')}; SameSite=None;"
+ return response
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-614/flask_bad.py b/python/ql/test/experimental/query-tests/Security/CWE-614/flask_bad.py
index fc0177e3012..740070a7b53 100644
--- a/python/ql/test/experimental/query-tests/Security/CWE-614/flask_bad.py
+++ b/python/ql/test/experimental/query-tests/Security/CWE-614/flask_bad.py
@@ -3,6 +3,21 @@ from flask import Flask, request, make_response, Response
app = Flask(__name__)
+@app.route("/false")
+def false():
+ resp = make_response()
+ resp.set_cookie("name", value="value", secure=False,
+ httponly=False, samesite='None')
+ return resp
+
+
+@app.route("/flask_Response")
+def flask_Response():
+ resp = Response()
+ resp.headers['Set-Cookie'] = "name=value; SameSite=None;"
+ return resp
+
+
@app.route("/false")
def false():
resp = make_response()
@@ -14,15 +29,9 @@ def false():
@app.route("/flask_Response")
def flask_Response():
resp = Response()
- resp.headers['Set-Cookie'] = "name=value;"
+ resp.headers['Set-Cookie'] = f"{request.args['name']}={request.args['value']}; SameSite=None;"
return resp
-@app.route("/flask_make_response")
-def flask_make_response():
- resp = make_response("hello")
- resp.headers['Set-Cookie'] = "name=value; SameSite=None;"
- return resp
-
# if __name__ == "__main__":
# app.run(debug=True)
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-614/flask_good.py b/python/ql/test/experimental/query-tests/Security/CWE-614/flask_good.py
index 5b9f83e1a63..724f8de8289 100644
--- a/python/ql/test/experimental/query-tests/Security/CWE-614/flask_good.py
+++ b/python/ql/test/experimental/query-tests/Security/CWE-614/flask_good.py
@@ -18,13 +18,6 @@ def flask_Response():
return resp
-@app.route("/flask_make_response")
-def flask_make_response():
- resp = make_response("hello")
- resp.headers['Set-Cookie'] = "name=value; Secure; HttpOnly; SameSite=Lax;"
- return resp
-
-
def indeterminate(secure):
resp = make_response()
resp.set_cookie("name", value="value", secure=secure)
From b3258ce20f64abf22d72282ed28bcaffeca007db Mon Sep 17 00:00:00 2001
From: jorgectf
Date: Fri, 5 Nov 2021 20:12:05 +0100
Subject: [PATCH 015/171] Add `CookieInjection` sample and `.qhelp`
---
.../Security/CWE-614/CookieInjection.py | 16 +++++++++++
.../Security/CWE-614/CookieInjection.qhelp | 28 +++++++++++++++++++
2 files changed, 44 insertions(+)
create mode 100644 python/ql/src/experimental/Security/CWE-614/CookieInjection.py
create mode 100644 python/ql/src/experimental/Security/CWE-614/CookieInjection.qhelp
diff --git a/python/ql/src/experimental/Security/CWE-614/CookieInjection.py b/python/ql/src/experimental/Security/CWE-614/CookieInjection.py
new file mode 100644
index 00000000000..55d211bafc1
--- /dev/null
+++ b/python/ql/src/experimental/Security/CWE-614/CookieInjection.py
@@ -0,0 +1,16 @@
+from flask import request, make_response
+
+
+@app.route("/1")
+def true():
+ resp = make_response()
+ resp.set_cookie(request.args["name"],
+ value=request.args["name"])
+ return resp
+
+
+@app.route("/2")
+def flask_make_response():
+ resp = make_response("hello")
+ resp.headers['Set-Cookie'] = f"{request.args['name']}={request.args['name']};"
+ return resp
diff --git a/python/ql/src/experimental/Security/CWE-614/CookieInjection.qhelp b/python/ql/src/experimental/Security/CWE-614/CookieInjection.qhelp
new file mode 100644
index 00000000000..4b5ec11726c
--- /dev/null
+++ b/python/ql/src/experimental/Security/CWE-614/CookieInjection.qhelp
@@ -0,0 +1,28 @@
+
+
+
+
+Constructing cookies from user input may allow an attacker to perform a Cookie Poisoning attack.
+It is possible, however, to perform other parameter-like attacks through cookie poisoning techniques,
+such as SQL Injection, Directory Traversal, or Stealth Commanding, etc. Additionally,
+cookie injection may relate to attempts to perform Access of Administrative Interface.
+
+
+
+
+Do not use raw user input to construct cookies.
+
+
+
+This example shows two ways of adding a cookie to a Flask response. The first way uses set_cookie's
+and the second sets a cookie's raw value through a header, both using user-supplied input.
+
+
+
+
+Imperva: Cookie injection .
+
+
+
From cf47e8eb9ce3d6a33e38c905f70c06c6dbea754e Mon Sep 17 00:00:00 2001
From: jorgectf
Date: Fri, 5 Nov 2021 20:12:35 +0100
Subject: [PATCH 016/171] Fix endpoints' naming
---
.../src/experimental/Security/CWE-614/InsecureCookie.py | 6 +++---
.../query-tests/Security/CWE-614/flask_bad.py | 8 ++++----
.../query-tests/Security/CWE-614/flask_good.py | 4 ++--
3 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/python/ql/src/experimental/Security/CWE-614/InsecureCookie.py b/python/ql/src/experimental/Security/CWE-614/InsecureCookie.py
index 54bbeff7d12..4087830f7eb 100644
--- a/python/ql/src/experimental/Security/CWE-614/InsecureCookie.py
+++ b/python/ql/src/experimental/Security/CWE-614/InsecureCookie.py
@@ -1,15 +1,15 @@
from flask import Flask, request, make_response, Response
-@app.route("/true")
+@app.route("/1")
def true():
resp = make_response()
resp.set_cookie("name", value="value", secure=True)
return resp
-@app.route("/flask_make_response")
+@app.route("/2")
def flask_make_response():
resp = make_response("hello")
resp.headers['Set-Cookie'] = "name=value; Secure;"
- return resp
\ No newline at end of file
+ return resp
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-614/flask_bad.py b/python/ql/test/experimental/query-tests/Security/CWE-614/flask_bad.py
index 740070a7b53..431df5eb4d8 100644
--- a/python/ql/test/experimental/query-tests/Security/CWE-614/flask_bad.py
+++ b/python/ql/test/experimental/query-tests/Security/CWE-614/flask_bad.py
@@ -3,7 +3,7 @@ from flask import Flask, request, make_response, Response
app = Flask(__name__)
-@app.route("/false")
+@app.route("/1")
def false():
resp = make_response()
resp.set_cookie("name", value="value", secure=False,
@@ -11,14 +11,14 @@ def false():
return resp
-@app.route("/flask_Response")
+@app.route("/2")
def flask_Response():
resp = Response()
resp.headers['Set-Cookie'] = "name=value; SameSite=None;"
return resp
-@app.route("/false")
+@app.route("/3")
def false():
resp = make_response()
resp.set_cookie(request.args["name"], value=request.args["value"], secure=False,
@@ -26,7 +26,7 @@ def false():
return resp
-@app.route("/flask_Response")
+@app.route("/4")
def flask_Response():
resp = Response()
resp.headers['Set-Cookie'] = f"{request.args['name']}={request.args['value']}; SameSite=None;"
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-614/flask_good.py b/python/ql/test/experimental/query-tests/Security/CWE-614/flask_good.py
index 724f8de8289..4cb23bd84b3 100644
--- a/python/ql/test/experimental/query-tests/Security/CWE-614/flask_good.py
+++ b/python/ql/test/experimental/query-tests/Security/CWE-614/flask_good.py
@@ -3,7 +3,7 @@ from flask import Flask, request, make_response, Response
app = Flask(__name__)
-@app.route("/true")
+@app.route("/1")
def true():
resp = make_response()
resp.set_cookie("name", value="value", secure=True,
@@ -11,7 +11,7 @@ def true():
return resp
-@app.route("/flask_Response")
+@app.route("/2")
def flask_Response():
resp = Response()
resp.headers['Set-Cookie'] = "name=value; Secure; HttpOnly; SameSite=Lax;"
From a420e6e18dae60e7013ad314d0df8e78eb840c45 Mon Sep 17 00:00:00 2001
From: jorgectf
Date: Fri, 5 Nov 2021 20:12:56 +0100
Subject: [PATCH 017/171] Add `CookieInjection.qlref`
---
.../query-tests/Security/CWE-614/CookieInjection.qlref | 1 +
1 file changed, 1 insertion(+)
create mode 100644 python/ql/test/experimental/query-tests/Security/CWE-614/CookieInjection.qlref
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-614/CookieInjection.qlref b/python/ql/test/experimental/query-tests/Security/CWE-614/CookieInjection.qlref
new file mode 100644
index 00000000000..5710a3322de
--- /dev/null
+++ b/python/ql/test/experimental/query-tests/Security/CWE-614/CookieInjection.qlref
@@ -0,0 +1 @@
+experimental/Security/CWE-614/CookieInjection.ql
From 86aac7c215408e41389866da890842052a39fcfd Mon Sep 17 00:00:00 2001
From: jorgectf
Date: Fri, 5 Nov 2021 20:13:12 +0100
Subject: [PATCH 018/171] Add/Update `.expected` files.
---
.../Security/CWE-614/CookieInjection.expected | 24 +++++++++++++++++
.../Security/CWE-614/InsecureCookie.expected | 27 ++++++++++++-------
2 files changed, 42 insertions(+), 9 deletions(-)
create mode 100644 python/ql/test/experimental/query-tests/Security/CWE-614/CookieInjection.expected
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-614/CookieInjection.expected b/python/ql/test/experimental/query-tests/Security/CWE-614/CookieInjection.expected
new file mode 100644
index 00000000000..fc368c95323
--- /dev/null
+++ b/python/ql/test/experimental/query-tests/Security/CWE-614/CookieInjection.expected
@@ -0,0 +1,24 @@
+| django_bad.py:19:21:19:55 | ControlFlowNode for Attribute() | Cookie is constructed from a | django_bad.py:19:21:19:55 | ControlFlowNode for Attribute() | user-supplied input | and its httponly flag is not properly set |
+| django_bad.py:19:21:19:55 | ControlFlowNode for Attribute() | Cookie is constructed from a | django_bad.py:19:21:19:55 | ControlFlowNode for Attribute() | user-supplied input | and its samesite flag is not properly set |
+| django_bad.py:19:21:19:55 | ControlFlowNode for Attribute() | Cookie is constructed from a | django_bad.py:19:21:19:55 | ControlFlowNode for Attribute() | user-supplied input | and its secure flag is not properly set |
+| django_bad.py:27:30:27:124 | ControlFlowNode for Fstring | Cookie is constructed from a | django_bad.py:27:33:27:67 | ControlFlowNode for Attribute() | user-supplied input | and its httponly flag is not properly set |
+| django_bad.py:27:30:27:124 | ControlFlowNode for Fstring | Cookie is constructed from a | django_bad.py:27:33:27:67 | ControlFlowNode for Attribute() | user-supplied input | and its samesite flag is not properly set |
+| django_bad.py:27:30:27:124 | ControlFlowNode for Fstring | Cookie is constructed from a | django_bad.py:27:33:27:67 | ControlFlowNode for Attribute() | user-supplied input | and its secure flag is not properly set |
+| django_bad.py:27:30:27:124 | ControlFlowNode for Fstring | Cookie is constructed from a | django_bad.py:27:71:27:106 | ControlFlowNode for Attribute() | user-supplied input | and its httponly flag is not properly set |
+| django_bad.py:27:30:27:124 | ControlFlowNode for Fstring | Cookie is constructed from a | django_bad.py:27:71:27:106 | ControlFlowNode for Attribute() | user-supplied input | and its samesite flag is not properly set |
+| django_bad.py:27:30:27:124 | ControlFlowNode for Fstring | Cookie is constructed from a | django_bad.py:27:71:27:106 | ControlFlowNode for Attribute() | user-supplied input | and its secure flag is not properly set |
+| flask_bad.py:24:21:24:40 | ControlFlowNode for Subscript | Cookie is constructed from a | flask_bad.py:24:21:24:27 | ControlFlowNode for request | user-supplied input | and its httponly flag is not properly set |
+| flask_bad.py:24:21:24:40 | ControlFlowNode for Subscript | Cookie is constructed from a | flask_bad.py:24:21:24:27 | ControlFlowNode for request | user-supplied input | and its samesite flag is not properly set |
+| flask_bad.py:24:21:24:40 | ControlFlowNode for Subscript | Cookie is constructed from a | flask_bad.py:24:21:24:27 | ControlFlowNode for request | user-supplied input | and its secure flag is not properly set |
+| flask_bad.py:24:49:24:69 | ControlFlowNode for Subscript | Cookie is constructed from a | flask_bad.py:24:21:24:27 | ControlFlowNode for request | user-supplied input | and its httponly flag is not properly set |
+| flask_bad.py:24:49:24:69 | ControlFlowNode for Subscript | Cookie is constructed from a | flask_bad.py:24:21:24:27 | ControlFlowNode for request | user-supplied input | and its samesite flag is not properly set |
+| flask_bad.py:24:49:24:69 | ControlFlowNode for Subscript | Cookie is constructed from a | flask_bad.py:24:21:24:27 | ControlFlowNode for request | user-supplied input | and its secure flag is not properly set |
+| flask_bad.py:24:49:24:69 | ControlFlowNode for Subscript | Cookie is constructed from a | flask_bad.py:24:49:24:55 | ControlFlowNode for request | user-supplied input | and its httponly flag is not properly set |
+| flask_bad.py:24:49:24:69 | ControlFlowNode for Subscript | Cookie is constructed from a | flask_bad.py:24:49:24:55 | ControlFlowNode for request | user-supplied input | and its samesite flag is not properly set |
+| flask_bad.py:24:49:24:69 | ControlFlowNode for Subscript | Cookie is constructed from a | flask_bad.py:24:49:24:55 | ControlFlowNode for request | user-supplied input | and its secure flag is not properly set |
+| flask_bad.py:32:34:32:98 | ControlFlowNode for Fstring | Cookie is constructed from a | flask_bad.py:32:37:32:43 | ControlFlowNode for request | user-supplied input | and its httponly flag is not properly set |
+| flask_bad.py:32:34:32:98 | ControlFlowNode for Fstring | Cookie is constructed from a | flask_bad.py:32:37:32:43 | ControlFlowNode for request | user-supplied input | and its samesite flag is not properly set |
+| flask_bad.py:32:34:32:98 | ControlFlowNode for Fstring | Cookie is constructed from a | flask_bad.py:32:37:32:43 | ControlFlowNode for request | user-supplied input | and its secure flag is not properly set |
+| flask_bad.py:32:34:32:98 | ControlFlowNode for Fstring | Cookie is constructed from a | flask_bad.py:32:60:32:66 | ControlFlowNode for request | user-supplied input | and its httponly flag is not properly set |
+| flask_bad.py:32:34:32:98 | ControlFlowNode for Fstring | Cookie is constructed from a | flask_bad.py:32:60:32:66 | ControlFlowNode for request | user-supplied input | and its samesite flag is not properly set |
+| flask_bad.py:32:34:32:98 | ControlFlowNode for Fstring | Cookie is constructed from a | flask_bad.py:32:60:32:66 | ControlFlowNode for request | user-supplied input | and its secure flag is not properly set |
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-614/InsecureCookie.expected b/python/ql/test/experimental/query-tests/Security/CWE-614/InsecureCookie.expected
index a04ad9cdafe..1ece5048db8 100644
--- a/python/ql/test/experimental/query-tests/Security/CWE-614/InsecureCookie.expected
+++ b/python/ql/test/experimental/query-tests/Security/CWE-614/InsecureCookie.expected
@@ -1,9 +1,15 @@
| django_bad.py:6:5:7:52 | ControlFlowNode for Attribute() | Cookie is added without the | httponly | flag properly set. |
| django_bad.py:6:5:7:52 | ControlFlowNode for Attribute() | Cookie is added without the | samesite | flag properly set. |
| django_bad.py:6:5:7:52 | ControlFlowNode for Attribute() | Cookie is added without the | secure | flag properly set. |
-| django_bad.py:13:5:14:52 | ControlFlowNode for Attribute() | Cookie is added without the | httponly | flag properly set. |
-| django_bad.py:13:5:14:52 | ControlFlowNode for Attribute() | Cookie is added without the | samesite | flag properly set. |
-| django_bad.py:13:5:14:52 | ControlFlowNode for Attribute() | Cookie is added without the | secure | flag properly set. |
+| django_bad.py:13:5:13:26 | ControlFlowNode for Subscript | Cookie is added without the | httponly | flag properly set. |
+| django_bad.py:13:5:13:26 | ControlFlowNode for Subscript | Cookie is added without the | samesite | flag properly set. |
+| django_bad.py:13:5:13:26 | ControlFlowNode for Subscript | Cookie is added without the | secure | flag properly set. |
+| django_bad.py:19:5:21:66 | ControlFlowNode for Attribute() | Cookie is added without the | httponly | flag properly set. |
+| django_bad.py:19:5:21:66 | ControlFlowNode for Attribute() | Cookie is added without the | samesite | flag properly set. |
+| django_bad.py:19:5:21:66 | ControlFlowNode for Attribute() | Cookie is added without the | secure | flag properly set. |
+| django_bad.py:27:5:27:26 | ControlFlowNode for Subscript | Cookie is added without the | httponly | flag properly set. |
+| django_bad.py:27:5:27:26 | ControlFlowNode for Subscript | Cookie is added without the | samesite | flag properly set. |
+| django_bad.py:27:5:27:26 | ControlFlowNode for Subscript | Cookie is added without the | secure | flag properly set. |
| django_good.py:19:5:19:44 | ControlFlowNode for Attribute() | Cookie is added without the | httponly | flag properly set. |
| django_good.py:19:5:19:44 | ControlFlowNode for Attribute() | Cookie is added without the | samesite | flag properly set. |
| django_good.py:19:5:19:44 | ControlFlowNode for Attribute() | Cookie is added without the | secure | flag properly set. |
@@ -13,9 +19,12 @@
| flask_bad.py:17:5:17:30 | ControlFlowNode for Subscript | Cookie is added without the | httponly | flag properly set. |
| flask_bad.py:17:5:17:30 | ControlFlowNode for Subscript | Cookie is added without the | samesite | flag properly set. |
| flask_bad.py:17:5:17:30 | ControlFlowNode for Subscript | Cookie is added without the | secure | flag properly set. |
-| flask_bad.py:24:5:24:30 | ControlFlowNode for Subscript | Cookie is added without the | httponly | flag properly set. |
-| flask_bad.py:24:5:24:30 | ControlFlowNode for Subscript | Cookie is added without the | samesite | flag properly set. |
-| flask_bad.py:24:5:24:30 | ControlFlowNode for Subscript | Cookie is added without the | secure | flag properly set. |
-| flask_good.py:31:5:31:57 | ControlFlowNode for Attribute() | Cookie is added without the | httponly | flag properly set. |
-| flask_good.py:31:5:31:57 | ControlFlowNode for Attribute() | Cookie is added without the | samesite | flag properly set. |
-| flask_good.py:31:5:31:57 | ControlFlowNode for Attribute() | Cookie is added without the | secure | flag properly set. |
+| flask_bad.py:24:5:25:52 | ControlFlowNode for Attribute() | Cookie is added without the | httponly | flag properly set. |
+| flask_bad.py:24:5:25:52 | ControlFlowNode for Attribute() | Cookie is added without the | samesite | flag properly set. |
+| flask_bad.py:24:5:25:52 | ControlFlowNode for Attribute() | Cookie is added without the | secure | flag properly set. |
+| flask_bad.py:32:5:32:30 | ControlFlowNode for Subscript | Cookie is added without the | httponly | flag properly set. |
+| flask_bad.py:32:5:32:30 | ControlFlowNode for Subscript | Cookie is added without the | samesite | flag properly set. |
+| flask_bad.py:32:5:32:30 | ControlFlowNode for Subscript | Cookie is added without the | secure | flag properly set. |
+| flask_good.py:23:5:23:57 | ControlFlowNode for Attribute() | Cookie is added without the | httponly | flag properly set. |
+| flask_good.py:23:5:23:57 | ControlFlowNode for Attribute() | Cookie is added without the | samesite | flag properly set. |
+| flask_good.py:23:5:23:57 | ControlFlowNode for Attribute() | Cookie is added without the | secure | flag properly set. |
From 83e3de1fed2ccb90bc16325a4ba940a8feeb3253 Mon Sep 17 00:00:00 2001
From: jorgectf
Date: Fri, 5 Nov 2021 21:05:33 +0100
Subject: [PATCH 019/171] Polish documentation.
---
.../Security/CWE-614/CookieInjection.ql | 7 +++----
.../Security/CWE-614/InsecureCookie.qhelp | 9 +++++++--
.../semmle/python/CookieHeader.qll | 19 ++++++++++++++++++
.../semmle/python/frameworks/Django.qll | 19 ++++++++++++++++++
.../semmle/python/frameworks/Flask.qll | 20 +++++++++++++++++++
5 files changed, 68 insertions(+), 6 deletions(-)
diff --git a/python/ql/src/experimental/Security/CWE-614/CookieInjection.ql b/python/ql/src/experimental/Security/CWE-614/CookieInjection.ql
index b4a880e9a58..8f7ef99789b 100644
--- a/python/ql/src/experimental/Security/CWE-614/CookieInjection.ql
+++ b/python/ql/src/experimental/Security/CWE-614/CookieInjection.ql
@@ -1,10 +1,9 @@
/**
- * @name Failure to use secure cookies
- * @description Insecure cookies may be sent in cleartext, which makes them vulnerable to
- * interception.
+ * @name Construction of a cookie using user-supplied input.
+ * @description Constructing cookies from user input may allow an attacker to perform a Cookie Poisoning attack.
* @kind problem
* @problem.severity error
- * @id py/insecure-cookie
+ * @id py/cookie-injection
* @tags security
* external/cwe/cwe-614
*/
diff --git a/python/ql/src/experimental/Security/CWE-614/InsecureCookie.qhelp b/python/ql/src/experimental/Security/CWE-614/InsecureCookie.qhelp
index 97df2e49e13..c76ab17954a 100644
--- a/python/ql/src/experimental/Security/CWE-614/InsecureCookie.qhelp
+++ b/python/ql/src/experimental/Security/CWE-614/InsecureCookie.qhelp
@@ -4,12 +4,17 @@
-Failing to set the 'secure' flag on a cookie can cause it to be sent in cleartext.
-This makes it easier for an attacker to intercept.
+Setting the 'secure' flag on a cookie to False can cause it to be sent in cleartext.
+Setting the 'httponly' flag on a cookie to False may allow attackers access it via JavaScript.
+Setting the 'samesite' flag on a cookie to 'None' will make the cookie to be sent in third-party
+contexts which may be attacker-controlled.
Always set secure to True or add "; Secure;" to the cookie's raw value.
+Always set httponly to True or add "; HttpOnly;" to the cookie's raw value.
+Always set samesite to Lax or Strict, or add "; SameSite=Lax;", or
+"; Samesite=Strict;" to the cookie's raw header value.
diff --git a/python/ql/src/experimental/semmle/python/CookieHeader.qll b/python/ql/src/experimental/semmle/python/CookieHeader.qll
index c7779aadd80..ab6bde4bb82 100644
--- a/python/ql/src/experimental/semmle/python/CookieHeader.qll
+++ b/python/ql/src/experimental/semmle/python/CookieHeader.qll
@@ -7,6 +7,25 @@ import semmle.python.dataflow.new.DataFlow
import semmle.python.dataflow.new.TaintTracking
import experimental.semmle.python.Concepts
+/**
+ * Gets a header setting a cookie.
+ *
+ * Given the following example:
+ *
+ * ```py
+ * @app.route("/")
+ * def flask_make_response():
+ * resp = make_response("")
+ * resp.headers['Set-Cookie'] = "name=value; Secure;"
+ * return resp
+ * ```
+ *
+ * * `this` would be `resp.headers['Set-Cookie'] = "name=value; Secure;"`.
+ * * `isSecure()` predicate would succeed.
+ * * `isHttpOnly()` predicate would fail.
+ * * `isSameSite()` predicate would fail.
+ * * `getName()` and `getValue()` results would be `"name=value; Secure;"`.
+ */
class CookieHeader extends HeaderDeclaration, Cookie::Range {
CookieHeader() {
this instanceof HeaderDeclaration and
diff --git a/python/ql/src/experimental/semmle/python/frameworks/Django.qll b/python/ql/src/experimental/semmle/python/frameworks/Django.qll
index ba99ddaa800..7fb7b65989e 100644
--- a/python/ql/src/experimental/semmle/python/frameworks/Django.qll
+++ b/python/ql/src/experimental/semmle/python/frameworks/Django.qll
@@ -87,6 +87,25 @@ private module PrivateDjango {
override DataFlow::Node getValueArg() { result = headerInput }
}
+ /**
+ * Gets a call to `set_cookie()`.
+ *
+ * Given the following example:
+ *
+ * ```py
+ * def django_response(request):
+ * resp = django.http.HttpResponse()
+ * resp.set_cookie("name", "value", secure=True, httponly=True, samesite='Lax')
+ * return resp
+ * ```
+ *
+ * * `this` would be `resp.set_cookie("name", "value", secure=False, httponly=False, samesite='None')`.
+ * * `getName()`'s result would be `"name"`.
+ * * `getValue()`'s result would be `"value"`.
+ * * `isSecure()` predicate would succeed.
+ * * `isHttpOnly()` predicate would succeed.
+ * * `isSameSite()` predicate would succeed.
+ */
class DjangoSetCookieCall extends DataFlow::CallCfgNode, Cookie::Range {
DjangoSetCookieCall() { this = baseClassRef().getMember("set_cookie").getACall() }
diff --git a/python/ql/src/experimental/semmle/python/frameworks/Flask.qll b/python/ql/src/experimental/semmle/python/frameworks/Flask.qll
index b93e6713846..c07092ee761 100644
--- a/python/ql/src/experimental/semmle/python/frameworks/Flask.qll
+++ b/python/ql/src/experimental/semmle/python/frameworks/Flask.qll
@@ -82,6 +82,26 @@ module ExperimentalFlask {
override DataFlow::Node getValueArg() { result.asExpr() = item.getValue() }
}
+ /**
+ * Gets a call to `set_cookie()`.
+ *
+ * Given the following example:
+ *
+ * ```py
+ * @app.route("/")
+ * def false():
+ * resp = make_response()
+ * resp.set_cookie("name", value="value", secure=True, httponly=True, samesite='Lax')
+ * return resp
+ * ```
+ *
+ * * `this` would be `resp.set_cookie("name", value="value", secure=False, httponly=False, samesite='None')`.
+ * * `getName()`'s result would be `"name"`.
+ * * `getValue()`'s result would be `"value"`.
+ * * `isSecure()` predicate would succeed.
+ * * `isHttpOnly()` predicate would succeed.
+ * * `isSameSite()` predicate would succeed.
+ */
class FlaskSetCookieCall extends DataFlow::CallCfgNode, Cookie::Range {
FlaskSetCookieCall() {
this =
From e7d649f36dd74321f4588cfd9c3d0e69e5b00ce1 Mon Sep 17 00:00:00 2001
From: jorgectf
Date: Tue, 16 Nov 2021 13:54:25 +0100
Subject: [PATCH 020/171] Make `Cookie` concept extend
`HTTP::Server::CookieWrite`
---
.../experimental/semmle/python/Concepts.qll | 35 ++++---------------
.../semmle/python/CookieHeader.qll | 8 +++--
.../semmle/python/frameworks/Django.qll | 6 ++--
.../semmle/python/frameworks/Flask.qll | 6 ++--
.../security/injection/CookieInjection.qll | 4 +--
.../Security/CWE-614/CookieInjection.expected | 3 ++
6 files changed, 24 insertions(+), 38 deletions(-)
diff --git a/python/ql/src/experimental/semmle/python/Concepts.qll b/python/ql/src/experimental/semmle/python/Concepts.qll
index 64aa755d9cf..91b1a5f777d 100644
--- a/python/ql/src/experimental/semmle/python/Concepts.qll
+++ b/python/ql/src/experimental/semmle/python/Concepts.qll
@@ -13,6 +13,7 @@ private import semmle.python.dataflow.new.DataFlow
private import semmle.python.dataflow.new.RemoteFlowSources
private import semmle.python.dataflow.new.TaintTracking
private import experimental.semmle.python.Frameworks
+private import semmle.python.Concepts
/** Provides classes for modeling log related APIs. */
module LogOutput {
@@ -303,35 +304,21 @@ class HeaderDeclaration extends DataFlow::Node {
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `Cookie::Range` instead.
*/
-class Cookie extends DataFlow::Node {
- Cookie::Range range;
-
- Cookie() { this = range }
-
+class Cookie extends HTTP::Server::CookieWrite instanceof Cookie::Range {
/**
* Holds if this cookie is secure.
*/
- predicate isSecure() { range.isSecure() }
+ predicate isSecure() { super.isSecure() }
/**
* Holds if this cookie is HttpOnly.
*/
- predicate isHttpOnly() { range.isHttpOnly() }
+ predicate isHttpOnly() { super.isHttpOnly() }
/**
* Holds if the cookie is SameSite
*/
- predicate isSameSite() { range.isSameSite() }
-
- /**
- * Gets the argument containing the header name.
- */
- DataFlow::Node getName() { result = range.getName() }
-
- /**
- * Gets the argument containing the header value.
- */
- DataFlow::Node getValue() { result = range.getValue() }
+ predicate isSameSite() { super.isSameSite() }
}
/** Provides a class for modeling new cookie writes on HTTP responses. */
@@ -342,7 +329,7 @@ module Cookie {
* Extend this class to model new APIs. If you want to refine existing API models,
* extend `Cookie` instead.
*/
- abstract class Range extends DataFlow::Node {
+ abstract class Range extends HTTP::Server::CookieWrite::Range {
/**
* Holds if this cookie is secure.
*/
@@ -357,15 +344,5 @@ module Cookie {
* Holds if the cookie is SameSite.
*/
abstract predicate isSameSite();
-
- /**
- * Gets the argument containing the header name.
- */
- abstract DataFlow::Node getName();
-
- /**
- * Gets the argument containing the header value.
- */
- abstract DataFlow::Node getValue();
}
}
diff --git a/python/ql/src/experimental/semmle/python/CookieHeader.qll b/python/ql/src/experimental/semmle/python/CookieHeader.qll
index ab6bde4bb82..2fda527c69f 100644
--- a/python/ql/src/experimental/semmle/python/CookieHeader.qll
+++ b/python/ql/src/experimental/semmle/python/CookieHeader.qll
@@ -26,7 +26,7 @@ import experimental.semmle.python.Concepts
* * `isSameSite()` predicate would fail.
* * `getName()` and `getValue()` results would be `"name=value; Secure;"`.
*/
-class CookieHeader extends HeaderDeclaration, Cookie::Range {
+class CookieHeader extends Cookie::Range instanceof HeaderDeclaration {
CookieHeader() {
this instanceof HeaderDeclaration and
this.(HeaderDeclaration).getNameArg().asExpr().(Str_).getS() = "Set-Cookie"
@@ -49,7 +49,9 @@ class CookieHeader extends HeaderDeclaration, Cookie::Range {
.regexpMatch(".*; *SameSite=(Strict|Lax);.*")
}
- override DataFlow::Node getName() { result = this.(HeaderDeclaration).getValueArg() }
+ override DataFlow::Node getNameArg() { result = this.(HeaderDeclaration).getValueArg() }
- override DataFlow::Node getValue() { result = this.(HeaderDeclaration).getValueArg() }
+ override DataFlow::Node getValueArg() { result = this.(HeaderDeclaration).getValueArg() }
+
+ override DataFlow::Node getHeaderArg() { none() }
}
diff --git a/python/ql/src/experimental/semmle/python/frameworks/Django.qll b/python/ql/src/experimental/semmle/python/frameworks/Django.qll
index 7fb7b65989e..11b4665d6c8 100644
--- a/python/ql/src/experimental/semmle/python/frameworks/Django.qll
+++ b/python/ql/src/experimental/semmle/python/frameworks/Django.qll
@@ -109,9 +109,9 @@ private module PrivateDjango {
class DjangoSetCookieCall extends DataFlow::CallCfgNode, Cookie::Range {
DjangoSetCookieCall() { this = baseClassRef().getMember("set_cookie").getACall() }
- override DataFlow::Node getName() { result = this.getArg(0) }
+ override DataFlow::Node getNameArg() { result = this.getArg(0) }
- override DataFlow::Node getValue() { result = this.getArgByName("value") }
+ override DataFlow::Node getValueArg() { result = this.getArgByName("value") }
override predicate isSecure() {
DataFlow::exprNode(any(True t))
@@ -128,6 +128,8 @@ private module PrivateDjango {
override predicate isSameSite() {
this.getArgByName("samesite").asExpr().(Str_).getS() in ["Strict", "Lax"]
}
+
+ override DataFlow::Node getHeaderArg() { none() }
}
}
}
diff --git a/python/ql/src/experimental/semmle/python/frameworks/Flask.qll b/python/ql/src/experimental/semmle/python/frameworks/Flask.qll
index c07092ee761..92a019599c0 100644
--- a/python/ql/src/experimental/semmle/python/frameworks/Flask.qll
+++ b/python/ql/src/experimental/semmle/python/frameworks/Flask.qll
@@ -111,9 +111,9 @@ module ExperimentalFlask {
.getACall()
}
- override DataFlow::Node getName() { result = this.getArg(0) }
+ override DataFlow::Node getNameArg() { result = this.getArg(0) }
- override DataFlow::Node getValue() { result = this.getArgByName("value") }
+ override DataFlow::Node getValueArg() { result = this.getArgByName("value") }
override predicate isSecure() {
DataFlow::exprNode(any(True t))
@@ -130,5 +130,7 @@ module ExperimentalFlask {
override predicate isSameSite() {
this.getArgByName("samesite").asExpr().(Str_).getS() in ["Strict", "Lax"]
}
+
+ override DataFlow::Node getHeaderArg() { none() }
}
}
diff --git a/python/ql/src/experimental/semmle/python/security/injection/CookieInjection.qll b/python/ql/src/experimental/semmle/python/security/injection/CookieInjection.qll
index 41f7c2af7d3..87f3b1fd76b 100644
--- a/python/ql/src/experimental/semmle/python/security/injection/CookieInjection.qll
+++ b/python/ql/src/experimental/semmle/python/security/injection/CookieInjection.qll
@@ -9,7 +9,7 @@ class CookieSink extends DataFlow::Node {
CookieSink() {
exists(Cookie cookie |
- this in [cookie.getName(), cookie.getValue()] and
+ this in [cookie.getNameArg(), cookie.getValueArg()] and
(
not cookie.isSecure() and
flag = "secure"
@@ -35,6 +35,6 @@ class CookieInjectionFlowConfig extends TaintTracking::Configuration {
override predicate isSource(DataFlow::Node source) { source instanceof RemoteFlowSource }
override predicate isSink(DataFlow::Node sink) {
- exists(Cookie c | sink in [c.getName(), c.getValue()])
+ exists(Cookie c | sink in [c.getNameArg(), c.getValueArg()])
}
}
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-614/CookieInjection.expected b/python/ql/test/experimental/query-tests/Security/CWE-614/CookieInjection.expected
index fc368c95323..879b1088002 100644
--- a/python/ql/test/experimental/query-tests/Security/CWE-614/CookieInjection.expected
+++ b/python/ql/test/experimental/query-tests/Security/CWE-614/CookieInjection.expected
@@ -1,6 +1,9 @@
| django_bad.py:19:21:19:55 | ControlFlowNode for Attribute() | Cookie is constructed from a | django_bad.py:19:21:19:55 | ControlFlowNode for Attribute() | user-supplied input | and its httponly flag is not properly set |
| django_bad.py:19:21:19:55 | ControlFlowNode for Attribute() | Cookie is constructed from a | django_bad.py:19:21:19:55 | ControlFlowNode for Attribute() | user-supplied input | and its samesite flag is not properly set |
| django_bad.py:19:21:19:55 | ControlFlowNode for Attribute() | Cookie is constructed from a | django_bad.py:19:21:19:55 | ControlFlowNode for Attribute() | user-supplied input | and its secure flag is not properly set |
+| django_bad.py:20:21:20:56 | ControlFlowNode for Attribute() | Cookie is constructed from a | django_bad.py:20:21:20:56 | ControlFlowNode for Attribute() | user-supplied input | and its httponly flag is not properly set |
+| django_bad.py:20:21:20:56 | ControlFlowNode for Attribute() | Cookie is constructed from a | django_bad.py:20:21:20:56 | ControlFlowNode for Attribute() | user-supplied input | and its samesite flag is not properly set |
+| django_bad.py:20:21:20:56 | ControlFlowNode for Attribute() | Cookie is constructed from a | django_bad.py:20:21:20:56 | ControlFlowNode for Attribute() | user-supplied input | and its secure flag is not properly set |
| django_bad.py:27:30:27:124 | ControlFlowNode for Fstring | Cookie is constructed from a | django_bad.py:27:33:27:67 | ControlFlowNode for Attribute() | user-supplied input | and its httponly flag is not properly set |
| django_bad.py:27:30:27:124 | ControlFlowNode for Fstring | Cookie is constructed from a | django_bad.py:27:33:27:67 | ControlFlowNode for Attribute() | user-supplied input | and its samesite flag is not properly set |
| django_bad.py:27:30:27:124 | ControlFlowNode for Fstring | Cookie is constructed from a | django_bad.py:27:33:27:67 | ControlFlowNode for Attribute() | user-supplied input | and its secure flag is not properly set |
From 6ecb6d1a1b599104590c11d7e6febbbaf4e06e8b Mon Sep 17 00:00:00 2001
From: jorgectf
Date: Tue, 16 Nov 2021 14:59:41 +0100
Subject: [PATCH 021/171] Adapt Django and Flask to their main modelings
---
.../semmle/python/frameworks/Django.qll | 73 ++++++++++++++++---
.../semmle/python/frameworks/Flask.qll | 23 +++---
2 files changed, 71 insertions(+), 25 deletions(-)
diff --git a/python/ql/src/experimental/semmle/python/frameworks/Django.qll b/python/ql/src/experimental/semmle/python/frameworks/Django.qll
index 11b4665d6c8..1c2d13f76cf 100644
--- a/python/ql/src/experimental/semmle/python/frameworks/Django.qll
+++ b/python/ql/src/experimental/semmle/python/frameworks/Django.qll
@@ -9,6 +9,7 @@ private import semmle.python.dataflow.new.DataFlow
private import experimental.semmle.python.Concepts
private import semmle.python.ApiGraphs
import semmle.python.dataflow.new.RemoteFlowSources
+private import semmle.python.Concepts
private module PrivateDjango {
private module django {
@@ -32,22 +33,64 @@ private module PrivateDjango {
module response {
module HttpResponse {
API::Node baseClassRef() {
- result = response().getMember("HttpResponse").getReturn()
+ result = response().getMember("HttpResponse")
or
// Handle `django.http.HttpResponse` alias
- result = http().getMember("HttpResponse").getReturn()
+ result = http().getMember("HttpResponse")
}
+ /** Gets a reference to the `django.http.response.HttpResponse` class. */
+ API::Node classRef() { result = baseClassRef().getASubclass*() }
+
+ /**
+ * A source of instances of `django.http.response.HttpResponse`, extend this class to model new instances.
+ *
+ * This can include instantiations of the class, return values from function
+ * calls, or a special parameter that will be set when functions are called by an external
+ * library.
+ *
+ * Use the predicate `HttpResponse::instance()` to get references to instances of `django.http.response.HttpResponse`.
+ */
+ abstract class InstanceSource extends HTTP::Server::HttpResponse::Range, DataFlow::Node {
+ }
+
+ /** A direct instantiation of `django.http.response.HttpResponse`. */
+ private class ClassInstantiation extends InstanceSource, DataFlow::CallCfgNode {
+ ClassInstantiation() { this = classRef().getACall() }
+
+ override DataFlow::Node getBody() {
+ result in [this.getArg(0), this.getArgByName("content")]
+ }
+
+ // How to support the `headers` argument here?
+ override DataFlow::Node getMimetypeOrContentTypeArg() {
+ result in [this.getArg(1), this.getArgByName("content_type")]
+ }
+
+ override string getMimetypeDefault() { result = "text/html" }
+ }
+
+ /** Gets a reference to an instance of `django.http.response.HttpResponse`. */
+ private DataFlow::TypeTrackingNode instance(DataFlow::TypeTracker t) {
+ t.start() and
+ result instanceof InstanceSource
+ or
+ exists(DataFlow::TypeTracker t2 | result = instance(t2).track(t2, t))
+ }
+
+ /** Gets a reference to an instance of `django.http.response.HttpResponse`. */
+ DataFlow::Node instance() { instance(DataFlow::TypeTracker::end()).flowsTo(result) }
+
/** Gets a reference to a header instance. */
private DataFlow::LocalSourceNode headerInstance(DataFlow::TypeTracker t) {
t.start() and
(
exists(SubscriptNode subscript |
- subscript.getObject() = baseClassRef().getAUse().asCfgNode() and
+ subscript.getObject() = baseClassRef().getReturn().getAUse().asCfgNode() and
result.asCfgNode() = subscript
)
or
- result.(DataFlow::AttrRead).getObject() = baseClassRef().getAUse()
+ result.(DataFlow::AttrRead).getObject() = baseClassRef().getReturn().getAUse()
)
or
exists(DataFlow::TypeTracker t2 | result = headerInstance(t2).track(t2, t))
@@ -106,27 +149,35 @@ private module PrivateDjango {
* * `isHttpOnly()` predicate would succeed.
* * `isSameSite()` predicate would succeed.
*/
- class DjangoSetCookieCall extends DataFlow::CallCfgNode, Cookie::Range {
- DjangoSetCookieCall() { this = baseClassRef().getMember("set_cookie").getACall() }
+ class DjangoResponseSetCookieCall extends DataFlow::MethodCallNode, Cookie::Range {
+ DjangoResponseSetCookieCall() {
+ this.calls(django::http::response::HttpResponse::instance(), "set_cookie")
+ }
- override DataFlow::Node getNameArg() { result = this.getArg(0) }
+ override DataFlow::Node getNameArg() {
+ result in [this.getArg(0), this.getArgByName("key")]
+ }
- override DataFlow::Node getValueArg() { result = this.getArgByName("value") }
+ override DataFlow::Node getValueArg() {
+ result in [this.getArg(1), this.getArgByName("value")]
+ }
override predicate isSecure() {
DataFlow::exprNode(any(True t))
.(DataFlow::LocalSourceNode)
- .flowsTo(this.getArgByName("secure"))
+ .flowsTo(this.(DataFlow::CallCfgNode).getArgByName("secure"))
}
override predicate isHttpOnly() {
DataFlow::exprNode(any(True t))
.(DataFlow::LocalSourceNode)
- .flowsTo(this.getArgByName("httponly"))
+ .flowsTo(this.(DataFlow::CallCfgNode).getArgByName("httponly"))
}
override predicate isSameSite() {
- this.getArgByName("samesite").asExpr().(Str_).getS() in ["Strict", "Lax"]
+ this.(DataFlow::CallCfgNode).getArgByName("samesite").asExpr().(Str_).getS() in [
+ "Strict", "Lax"
+ ]
}
override DataFlow::Node getHeaderArg() { none() }
diff --git a/python/ql/src/experimental/semmle/python/frameworks/Flask.qll b/python/ql/src/experimental/semmle/python/frameworks/Flask.qll
index 92a019599c0..c07abc0e177 100644
--- a/python/ql/src/experimental/semmle/python/frameworks/Flask.qll
+++ b/python/ql/src/experimental/semmle/python/frameworks/Flask.qll
@@ -8,6 +8,7 @@ private import semmle.python.frameworks.Flask
private import semmle.python.dataflow.new.DataFlow
private import experimental.semmle.python.Concepts
private import semmle.python.ApiGraphs
+private import semmle.python.frameworks.Flask
module ExperimentalFlask {
/**
@@ -102,33 +103,27 @@ module ExperimentalFlask {
* * `isHttpOnly()` predicate would succeed.
* * `isSameSite()` predicate would succeed.
*/
- class FlaskSetCookieCall extends DataFlow::CallCfgNode, Cookie::Range {
- FlaskSetCookieCall() {
- this =
- [Flask::Response::classRef(), flaskMakeResponse()]
- .getReturn()
- .getMember("set_cookie")
- .getACall()
- }
+ class FlaskSetCookieCall extends Cookie::Range instanceof Flask::FlaskResponseSetCookieCall {
+ override DataFlow::Node getNameArg() { result = this.getNameArg() }
- override DataFlow::Node getNameArg() { result = this.getArg(0) }
-
- override DataFlow::Node getValueArg() { result = this.getArgByName("value") }
+ override DataFlow::Node getValueArg() { result = this.getValueArg() }
override predicate isSecure() {
DataFlow::exprNode(any(True t))
.(DataFlow::LocalSourceNode)
- .flowsTo(this.getArgByName("secure"))
+ .flowsTo(this.(DataFlow::CallCfgNode).getArgByName("secure"))
}
override predicate isHttpOnly() {
DataFlow::exprNode(any(True t))
.(DataFlow::LocalSourceNode)
- .flowsTo(this.getArgByName("httponly"))
+ .flowsTo(this.(DataFlow::CallCfgNode).getArgByName("httponly"))
}
override predicate isSameSite() {
- this.getArgByName("samesite").asExpr().(Str_).getS() in ["Strict", "Lax"]
+ this.(DataFlow::CallCfgNode).getArgByName("samesite").asExpr().(Str_).getS() in [
+ "Strict", "Lax"
+ ]
}
override DataFlow::Node getHeaderArg() { none() }
From a4204cc04f3dd7886017e1db6d7787354ca673d4 Mon Sep 17 00:00:00 2001
From: jorgectf
Date: Tue, 16 Nov 2021 19:00:04 +0100
Subject: [PATCH 022/171] Avoid using `Str_` internal class
---
.../src/experimental/semmle/python/frameworks/Django.qll | 9 ++++++---
.../src/experimental/semmle/python/frameworks/Flask.qll | 9 ++++++---
2 files changed, 12 insertions(+), 6 deletions(-)
diff --git a/python/ql/src/experimental/semmle/python/frameworks/Django.qll b/python/ql/src/experimental/semmle/python/frameworks/Django.qll
index 1c2d13f76cf..2fef35d276c 100644
--- a/python/ql/src/experimental/semmle/python/frameworks/Django.qll
+++ b/python/ql/src/experimental/semmle/python/frameworks/Django.qll
@@ -175,9 +175,12 @@ private module PrivateDjango {
}
override predicate isSameSite() {
- this.(DataFlow::CallCfgNode).getArgByName("samesite").asExpr().(Str_).getS() in [
- "Strict", "Lax"
- ]
+ exists(StrConst str |
+ str.getText() in ["Strict", "Lax"] and
+ DataFlow::exprNode(str)
+ .(DataFlow::LocalSourceNode)
+ .flowsTo(this.(DataFlow::CallCfgNode).getArgByName("samesite"))
+ )
}
override DataFlow::Node getHeaderArg() { none() }
diff --git a/python/ql/src/experimental/semmle/python/frameworks/Flask.qll b/python/ql/src/experimental/semmle/python/frameworks/Flask.qll
index c07abc0e177..b9283dafd92 100644
--- a/python/ql/src/experimental/semmle/python/frameworks/Flask.qll
+++ b/python/ql/src/experimental/semmle/python/frameworks/Flask.qll
@@ -121,9 +121,12 @@ module ExperimentalFlask {
}
override predicate isSameSite() {
- this.(DataFlow::CallCfgNode).getArgByName("samesite").asExpr().(Str_).getS() in [
- "Strict", "Lax"
- ]
+ exists(StrConst str |
+ str.getText() in ["Strict", "Lax"] and
+ DataFlow::exprNode(str)
+ .(DataFlow::LocalSourceNode)
+ .flowsTo(this.(DataFlow::CallCfgNode).getArgByName("samesite"))
+ )
}
override DataFlow::Node getHeaderArg() { none() }
From 840cded9b022c300db528f3e3b7bf73789fdafad Mon Sep 17 00:00:00 2001
From: jorgectf
Date: Tue, 16 Nov 2021 19:18:00 +0100
Subject: [PATCH 023/171] Avoid using `Str_` in `CookieHeader`
---
.../semmle/python/CookieHeader.qll | 33 ++++++++++++++-----
1 file changed, 24 insertions(+), 9 deletions(-)
diff --git a/python/ql/src/experimental/semmle/python/CookieHeader.qll b/python/ql/src/experimental/semmle/python/CookieHeader.qll
index 2fda527c69f..1d28548e5a4 100644
--- a/python/ql/src/experimental/semmle/python/CookieHeader.qll
+++ b/python/ql/src/experimental/semmle/python/CookieHeader.qll
@@ -29,24 +29,39 @@ import experimental.semmle.python.Concepts
class CookieHeader extends Cookie::Range instanceof HeaderDeclaration {
CookieHeader() {
this instanceof HeaderDeclaration and
- this.(HeaderDeclaration).getNameArg().asExpr().(Str_).getS() = "Set-Cookie"
+ exists(StrConst str |
+ str.getText() = "Set-Cookie" and
+ DataFlow::exprNode(str)
+ .(DataFlow::LocalSourceNode)
+ .flowsTo(this.(HeaderDeclaration).getNameArg())
+ )
}
override predicate isSecure() {
- this.(HeaderDeclaration).getValueArg().asExpr().(Str_).getS().regexpMatch(".*; *Secure;.*")
+ exists(StrConst str |
+ str.getText().regexpMatch(".*; *Secure;.*") and
+ DataFlow::exprNode(str)
+ .(DataFlow::LocalSourceNode)
+ .flowsTo(this.(HeaderDeclaration).getValueArg())
+ )
}
override predicate isHttpOnly() {
- this.(HeaderDeclaration).getValueArg().asExpr().(Str_).getS().regexpMatch(".*; *HttpOnly;.*")
+ exists(StrConst str |
+ str.getText().regexpMatch(".*; *HttpOnly;.*") and
+ DataFlow::exprNode(str)
+ .(DataFlow::LocalSourceNode)
+ .flowsTo(this.(HeaderDeclaration).getValueArg())
+ )
}
override predicate isSameSite() {
- this.(HeaderDeclaration)
- .getValueArg()
- .asExpr()
- .(Str_)
- .getS()
- .regexpMatch(".*; *SameSite=(Strict|Lax);.*")
+ exists(StrConst str |
+ str.getText().regexpMatch(".*; *SameSite=(Strict|Lax);.*") and
+ DataFlow::exprNode(str)
+ .(DataFlow::LocalSourceNode)
+ .flowsTo(this.(HeaderDeclaration).getValueArg())
+ )
}
override DataFlow::Node getNameArg() { result = this.(HeaderDeclaration).getValueArg() }
From 93750fe17fbc1133642d6a7af63826b314d19d15 Mon Sep 17 00:00:00 2001
From: Rasmus Lerchedahl Petersen
Date: Fri, 4 Mar 2022 12:47:23 +0100
Subject: [PATCH 024/171] python: minimal CSRF implementation - currectly only
looks for custom django middleware
---
python/ql/lib/semmle/python/Concepts.qll | 31 ++++++++++
.../lib/semmle/python/frameworks/Django.qll | 31 ++++++++++
.../CWE-352/CSRFProtectionDisabled.qhelp | 60 +++++++++++++++++++
.../CWE-352/CSRFProtectionDisabled.ql | 19 ++++++
.../src/Security/CWE-352/examples/setting.py | 9 +++
5 files changed, 150 insertions(+)
create mode 100644 python/ql/src/Security/CWE-352/CSRFProtectionDisabled.qhelp
create mode 100644 python/ql/src/Security/CWE-352/CSRFProtectionDisabled.ql
create mode 100644 python/ql/src/Security/CWE-352/examples/setting.py
diff --git a/python/ql/lib/semmle/python/Concepts.qll b/python/ql/lib/semmle/python/Concepts.qll
index 6c67b0e5d91..8e4f810d4a0 100644
--- a/python/ql/lib/semmle/python/Concepts.qll
+++ b/python/ql/lib/semmle/python/Concepts.qll
@@ -105,6 +105,37 @@ module FileSystemWriteAccess {
}
}
+/**
+ * A data-flow node that may set or unset Cross-site request forgery protection.
+ *
+ * Extend this class to refine existing API models. If you want to model new APIs,
+ * extend `CSRFProtectionSetting::Range` instead.
+ */
+class CSRFProtectionSetting extends DataFlow::Node instanceof CSRFProtectionSetting::Range {
+ /**
+ * Gets the boolean value corresponding to if CSRF protection is enabled
+ * (`true`) or disabled (`false`) by this node.
+ */
+ boolean getVerificationSetting() { result = super.getVerificationSetting() }
+}
+
+/** Provides a class for modeling new CSRF protection setting APIs. */
+module CSRFProtectionSetting {
+ /**
+ * A data-flow node that may set or unset Cross-site request forgery protection.
+ *
+ * Extend this class to model new APIs. If you want to refine existing API models,
+ * extend `CSRFProtectionSetting` instead.
+ */
+ abstract class Range extends DataFlow::Node {
+ /**
+ * Gets the boolean value corresponding to if CSRF protection is enabled
+ * (`true`) or disabled (`false`) by this node.
+ */
+ abstract boolean getVerificationSetting();
+ }
+}
+
/** Provides classes for modeling path-related APIs. */
module Path {
/**
diff --git a/python/ql/lib/semmle/python/frameworks/Django.qll b/python/ql/lib/semmle/python/frameworks/Django.qll
index 8f34043f093..f5989badfe4 100644
--- a/python/ql/lib/semmle/python/frameworks/Django.qll
+++ b/python/ql/lib/semmle/python/frameworks/Django.qll
@@ -2313,4 +2313,35 @@ module PrivateDjango {
.getAnImmediateUse()
}
}
+
+ // ---------------------------------------------------------------------------
+ // Settings
+ // ---------------------------------------------------------------------------
+ /**
+ * A custom middleware stack
+ */
+ private class DjangoSettingsMiddlewareStack extends CSRFProtectionSetting::Range {
+ List list;
+
+ DjangoSettingsMiddlewareStack() {
+ this.asExpr() = list and
+ // we look for an assignment to the `MIDDLEWARE` setting
+ exists(DataFlow::Node mw, string djangomw |
+ mw.asVar().getName() = "MIDDLEWARE" and
+ DataFlow::localFlow(this, mw)
+ |
+ // check that the list contains at least one reference to `django`
+ list.getAnElt().(StrConst).getText() = djangomw and
+ // TODO: Consider requiring `django.middleware.security.SecurityMiddleware`
+ // or something indicating that a security middleware is enabled.
+ djangomw.matches("django.%")
+ )
+ }
+
+ override boolean getVerificationSetting() {
+ if list.getAnElt().(StrConst).getText() = "django.middleware.csrf.CsrfViewMiddleware"
+ then result = true
+ else result = false
+ }
+ }
}
diff --git a/python/ql/src/Security/CWE-352/CSRFProtectionDisabled.qhelp b/python/ql/src/Security/CWE-352/CSRFProtectionDisabled.qhelp
new file mode 100644
index 00000000000..98a5dae20ba
--- /dev/null
+++ b/python/ql/src/Security/CWE-352/CSRFProtectionDisabled.qhelp
@@ -0,0 +1,60 @@
+
+
+
+
+
+ Cross-site request forgery (CSRF) is a type of vulnerability in which an
+ attacker is able to force a user carry out an action that the user did
+ not intend.
+
+
+
+ The attacker tricks an authenticated user into submitting a request to the
+ web application. Typically this request will result in a state change on
+ the server, such as changing the user's password. The request can be
+ initiated when the user visits a site controlled by the attacker. If the
+ web application relies only on cookies for authentication, or on other
+ credentials that are automatically included in the request, then this
+ request will appear as legitimate to the server.
+
+
+
+ A common countermeasure for CSRF is to generate a unique token to be
+ included in the HTML sent from the server to a user. This token can be
+ used as a hidden field to be sent back with requests to the server, where
+ the server can then check that the token is valid and associated with the
+ relevant user session.
+
+
+
+
+
+ In many web frameworks, CSRF protection is enabled by default. In these
+ cases, using the default configuration is sufficient to guard against most
+ CSRF attacks.
+
+
+
+
+
+ The following example shows a case where CSRF protection is disabled by
+ overriding the default middleware stack and not including the one protecting against CSRF.
+
+
+
+
+
+ The protecting middleware was probably commented out during a testing phase, when server-side token generation was not set up.
+ Simply commenting it back in (or remove the custom middleware stack) will enable CSRF protection.
+
+
+
+
+
+ Wikipedia: Cross-site request forgery
+ OWASP: Cross-site request forgery
+
+
+
diff --git a/python/ql/src/Security/CWE-352/CSRFProtectionDisabled.ql b/python/ql/src/Security/CWE-352/CSRFProtectionDisabled.ql
new file mode 100644
index 00000000000..00f2cad5050
--- /dev/null
+++ b/python/ql/src/Security/CWE-352/CSRFProtectionDisabled.ql
@@ -0,0 +1,19 @@
+/**
+ * @name CSRF protection weakened or disabled
+ * @description Disabling or weakening CSRF protection may make the application
+ * vulnerable to a Cross-Site Request Forgery (CSRF) attack.
+ * @kind problem
+ * @problem.severity warning
+ * @security-severity 8.8
+ * @precision high
+ * @id py/csrf-protection-disabled
+ * @tags security
+ * external/cwe/cwe-352
+ */
+
+import python
+import semmle.python.Concepts
+
+from CSRFProtectionSetting s
+where s.getVerificationSetting() = false
+select s, "Potential CSRF vulnerability due to forgery protection being disabled or weakened."
diff --git a/python/ql/src/Security/CWE-352/examples/setting.py b/python/ql/src/Security/CWE-352/examples/setting.py
new file mode 100644
index 00000000000..d1f1f983cef
--- /dev/null
+++ b/python/ql/src/Security/CWE-352/examples/setting.py
@@ -0,0 +1,9 @@
+MIDDLEWARE = [
+ 'django.middleware.security.SecurityMiddleware',
+ 'django.contrib.sessions.middleware.SessionMiddleware',
+ 'django.middleware.common.CommonMiddleware',
+ # 'django.middleware.csrf.CsrfViewMiddleware',
+ 'django.contrib.auth.middleware.AuthenticationMiddleware',
+ 'django.contrib.messages.middleware.MessageMiddleware',
+ 'django.middleware.clickjacking.XFrameOptionsMiddleware',
+]
From 895ce755c1891da7285adcec45e095eca1667975 Mon Sep 17 00:00:00 2001
From: Rasmus Lerchedahl Petersen
Date: Mon, 7 Mar 2022 13:03:04 +0100
Subject: [PATCH 025/171] python: correct file name
---
.../ql/src/Security/CWE-352/examples/{setting.py => settings.py} | 0
1 file changed, 0 insertions(+), 0 deletions(-)
rename python/ql/src/Security/CWE-352/examples/{setting.py => settings.py} (100%)
diff --git a/python/ql/src/Security/CWE-352/examples/setting.py b/python/ql/src/Security/CWE-352/examples/settings.py
similarity index 100%
rename from python/ql/src/Security/CWE-352/examples/setting.py
rename to python/ql/src/Security/CWE-352/examples/settings.py
From f5b53083ae77a6a9ed459e9189877cb7326f2ed1 Mon Sep 17 00:00:00 2001
From: Rasmus Lerchedahl Petersen
Date: Tue, 22 Mar 2022 08:44:19 +0100
Subject: [PATCH 026/171] python: require authentication middleware for CSRF to
be relevant
---
python/ql/lib/semmle/python/frameworks/Django.qll | 13 +++++++------
1 file changed, 7 insertions(+), 6 deletions(-)
diff --git a/python/ql/lib/semmle/python/frameworks/Django.qll b/python/ql/lib/semmle/python/frameworks/Django.qll
index f5989badfe4..ee273a9f2a6 100644
--- a/python/ql/lib/semmle/python/frameworks/Django.qll
+++ b/python/ql/lib/semmle/python/frameworks/Django.qll
@@ -2326,15 +2326,16 @@ module PrivateDjango {
DjangoSettingsMiddlewareStack() {
this.asExpr() = list and
// we look for an assignment to the `MIDDLEWARE` setting
- exists(DataFlow::Node mw, string djangomw |
+ exists(DataFlow::Node mw |
mw.asVar().getName() = "MIDDLEWARE" and
DataFlow::localFlow(this, mw)
|
- // check that the list contains at least one reference to `django`
- list.getAnElt().(StrConst).getText() = djangomw and
- // TODO: Consider requiring `django.middleware.security.SecurityMiddleware`
- // or something indicating that a security middleware is enabled.
- djangomw.matches("django.%")
+ // it only counts as setting the CSRF protection, if the app uses authentication,
+ // so check that the list contains the django authentication middleware.
+ //
+ // This also strongly implies that we are actually looking at the `MIDDLEWARE` setting.
+ list.getAnElt().(StrConst).getText() =
+ "django.contrib.auth.middleware.AuthenticationMiddleware"
)
}
From 0f2c21c8bd6f953a41ed13721e93de8245b2fd1a Mon Sep 17 00:00:00 2001
From: Rasmus Lerchedahl Petersen
Date: Tue, 22 Mar 2022 13:42:52 +0100
Subject: [PATCH 027/171] python: require local protection to be absent for
CSRF to be likely
---
python/ql/lib/semmle/python/Concepts.qll | 39 ++++++++++++++++++-
.../lib/semmle/python/frameworks/Django.qll | 17 ++++++++
.../CWE-352/CSRFProtectionDisabled.ql | 4 +-
.../frameworks/django-v2-v3/response_test.py | 2 +
4 files changed, 59 insertions(+), 3 deletions(-)
diff --git a/python/ql/lib/semmle/python/Concepts.qll b/python/ql/lib/semmle/python/Concepts.qll
index 8e4f810d4a0..04d4d63aca3 100644
--- a/python/ql/lib/semmle/python/Concepts.qll
+++ b/python/ql/lib/semmle/python/Concepts.qll
@@ -106,7 +106,8 @@ module FileSystemWriteAccess {
}
/**
- * A data-flow node that may set or unset Cross-site request forgery protection.
+ * A data-flow node that may set or unset Cross-site request forgery protection
+ * in a global manner.
*
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `CSRFProtectionSetting::Range` instead.
@@ -122,7 +123,8 @@ class CSRFProtectionSetting extends DataFlow::Node instanceof CSRFProtectionSett
/** Provides a class for modeling new CSRF protection setting APIs. */
module CSRFProtectionSetting {
/**
- * A data-flow node that may set or unset Cross-site request forgery protection.
+ * A data-flow node that may set or unset Cross-site request forgery protection
+ * in a global manner.
*
* Extend this class to model new APIs. If you want to refine existing API models,
* extend `CSRFProtectionSetting` instead.
@@ -136,6 +138,39 @@ module CSRFProtectionSetting {
}
}
+/**
+ * A data-flow node that provides Cross-site request forgery protection
+ * for a specific part of an application.
+ *
+ * Extend this class to refine existing API models. If you want to model new APIs,
+ * extend `CSRFProtection::Range` instead.
+ */
+class CSRFProtection extends DataFlow::Node instanceof CSRFProtection::Range {
+ /**
+ * Gets a `Function` representing the protected interaction
+ * (probably a request handler).
+ */
+ Function getProtected() { result = super.getProtected() }
+}
+
+/** Provides a class for modeling new CSRF protection setting APIs. */
+module CSRFProtection {
+ /**
+ * A data-flow node that provides Cross-site request forgery protection
+ * for a specific part of an application.
+ *
+ * Extend this class to model new APIs. If you want to refine existing API models,
+ * extend `CSRFProtection` instead.
+ */
+ abstract class Range extends DataFlow::Node {
+ /**
+ * Gets a `Function` representing the protected interaction
+ * (probably a request handler).
+ */
+ abstract Function getProtected();
+ }
+}
+
/** Provides classes for modeling path-related APIs. */
module Path {
/**
diff --git a/python/ql/lib/semmle/python/frameworks/Django.qll b/python/ql/lib/semmle/python/frameworks/Django.qll
index ee273a9f2a6..baa81c682ea 100644
--- a/python/ql/lib/semmle/python/frameworks/Django.qll
+++ b/python/ql/lib/semmle/python/frameworks/Django.qll
@@ -2346,3 +2346,20 @@ module PrivateDjango {
}
}
}
+
+private class DjangoCSRFDecorator extends CSRFProtection::Range {
+ Function function;
+
+ DjangoCSRFDecorator() {
+ this =
+ API::moduleImport("django")
+ .getMember("views")
+ .getMember("decorators")
+ .getMember("csrf")
+ .getMember("csrf_protect")
+ .getAUse() and
+ this.asExpr() = function.getADecorator()
+ }
+
+ override Function getProtected() { result = function }
+}
diff --git a/python/ql/src/Security/CWE-352/CSRFProtectionDisabled.ql b/python/ql/src/Security/CWE-352/CSRFProtectionDisabled.ql
index 00f2cad5050..489ed1ea53c 100644
--- a/python/ql/src/Security/CWE-352/CSRFProtectionDisabled.ql
+++ b/python/ql/src/Security/CWE-352/CSRFProtectionDisabled.ql
@@ -15,5 +15,7 @@ import python
import semmle.python.Concepts
from CSRFProtectionSetting s
-where s.getVerificationSetting() = false
+where
+ s.getVerificationSetting() = false and
+ not exists(CSRFProtection p)
select s, "Potential CSRF vulnerability due to forgery protection being disabled or weakened."
diff --git a/python/ql/test/library-tests/frameworks/django-v2-v3/response_test.py b/python/ql/test/library-tests/frameworks/django-v2-v3/response_test.py
index 74f306e8357..4007b2d8063 100644
--- a/python/ql/test/library-tests/frameworks/django-v2-v3/response_test.py
+++ b/python/ql/test/library-tests/frameworks/django-v2-v3/response_test.py
@@ -1,5 +1,6 @@
from django.http.response import HttpResponse, HttpResponseRedirect, HttpResponsePermanentRedirect, JsonResponse, HttpResponseNotFound
from django.views.generic import RedirectView
+from django.views.decorators.csrf import csrf_protect
import django.shortcuts
import json
@@ -117,6 +118,7 @@ class CustomJsonResponse(JsonResponse):
def __init__(self, banner, content, *args, **kwargs):
super().__init__(content, *args, content_type="text/html", **kwargs)
+@csrf_protect
def safe__custom_json_response(request):
return CustomJsonResponse("ACME Responses", {"foo": request.GET.get("foo")}) # $HttpResponse mimetype=application/json MISSING: responseBody=Dict SPURIOUS: responseBody="ACME Responses"
From 53de8287f532d14beae84b1398c18a4b033b03b6 Mon Sep 17 00:00:00 2001
From: Rasmus Lerchedahl Petersen
Date: Tue, 22 Mar 2022 14:57:05 +0100
Subject: [PATCH 028/171] python: rule out test code for CSRF
---
python/ql/src/Security/CWE-352/CSRFProtectionDisabled.ql | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/python/ql/src/Security/CWE-352/CSRFProtectionDisabled.ql b/python/ql/src/Security/CWE-352/CSRFProtectionDisabled.ql
index 489ed1ea53c..5caa19d3d88 100644
--- a/python/ql/src/Security/CWE-352/CSRFProtectionDisabled.ql
+++ b/python/ql/src/Security/CWE-352/CSRFProtectionDisabled.ql
@@ -17,5 +17,7 @@ import semmle.python.Concepts
from CSRFProtectionSetting s
where
s.getVerificationSetting() = false and
- not exists(CSRFProtection p)
+ not exists(CSRFProtection p) and
+ // rule out test code as this is a common place to turn off CSRF protection
+ not s.getLocation().getFile().getAbsolutePath().matches("%test%")
select s, "Potential CSRF vulnerability due to forgery protection being disabled or weakened."
From 441e206cfaa776cdafa775dfdee567052b5dea94 Mon Sep 17 00:00:00 2001
From: Rasmus Lerchedahl Petersen
Date: Wed, 23 Mar 2022 11:29:27 +0100
Subject: [PATCH 029/171] python: CSRF -> Csrf
---
python/ql/lib/semmle/python/Concepts.qll | 16 +++++-----
.../lib/semmle/python/frameworks/Django.qll | 30 +++++++++----------
.../CWE-352/CSRFProtectionDisabled.ql | 4 +--
3 files changed, 25 insertions(+), 25 deletions(-)
diff --git a/python/ql/lib/semmle/python/Concepts.qll b/python/ql/lib/semmle/python/Concepts.qll
index 04d4d63aca3..1f4aca0b21a 100644
--- a/python/ql/lib/semmle/python/Concepts.qll
+++ b/python/ql/lib/semmle/python/Concepts.qll
@@ -110,9 +110,9 @@ module FileSystemWriteAccess {
* in a global manner.
*
* Extend this class to refine existing API models. If you want to model new APIs,
- * extend `CSRFProtectionSetting::Range` instead.
+ * extend `CsrfProtectionSetting::Range` instead.
*/
-class CSRFProtectionSetting extends DataFlow::Node instanceof CSRFProtectionSetting::Range {
+class CsrfProtectionSetting extends DataFlow::Node instanceof CsrfProtectionSetting::Range {
/**
* Gets the boolean value corresponding to if CSRF protection is enabled
* (`true`) or disabled (`false`) by this node.
@@ -121,13 +121,13 @@ class CSRFProtectionSetting extends DataFlow::Node instanceof CSRFProtectionSett
}
/** Provides a class for modeling new CSRF protection setting APIs. */
-module CSRFProtectionSetting {
+module CsrfProtectionSetting {
/**
* A data-flow node that may set or unset Cross-site request forgery protection
* in a global manner.
*
* Extend this class to model new APIs. If you want to refine existing API models,
- * extend `CSRFProtectionSetting` instead.
+ * extend `CsrfProtectionSetting` instead.
*/
abstract class Range extends DataFlow::Node {
/**
@@ -143,9 +143,9 @@ module CSRFProtectionSetting {
* for a specific part of an application.
*
* Extend this class to refine existing API models. If you want to model new APIs,
- * extend `CSRFProtection::Range` instead.
+ * extend `CsrfLocalProtection::Range` instead.
*/
-class CSRFProtection extends DataFlow::Node instanceof CSRFProtection::Range {
+class CsrfLocalProtection extends DataFlow::Node instanceof CsrfLocalProtection::Range {
/**
* Gets a `Function` representing the protected interaction
* (probably a request handler).
@@ -154,13 +154,13 @@ class CSRFProtection extends DataFlow::Node instanceof CSRFProtection::Range {
}
/** Provides a class for modeling new CSRF protection setting APIs. */
-module CSRFProtection {
+module CsrfLocalProtection {
/**
* A data-flow node that provides Cross-site request forgery protection
* for a specific part of an application.
*
* Extend this class to model new APIs. If you want to refine existing API models,
- * extend `CSRFProtection` instead.
+ * extend `CsrfLocalProtection` instead.
*/
abstract class Range extends DataFlow::Node {
/**
diff --git a/python/ql/lib/semmle/python/frameworks/Django.qll b/python/ql/lib/semmle/python/frameworks/Django.qll
index baa81c682ea..efa1a0eaa48 100644
--- a/python/ql/lib/semmle/python/frameworks/Django.qll
+++ b/python/ql/lib/semmle/python/frameworks/Django.qll
@@ -2320,7 +2320,7 @@ module PrivateDjango {
/**
* A custom middleware stack
*/
- private class DjangoSettingsMiddlewareStack extends CSRFProtectionSetting::Range {
+ private class DjangoSettingsMiddlewareStack extends CsrfProtectionSetting::Range {
List list;
DjangoSettingsMiddlewareStack() {
@@ -2345,21 +2345,21 @@ module PrivateDjango {
else result = false
}
}
-}
-private class DjangoCSRFDecorator extends CSRFProtection::Range {
- Function function;
+ private class DjangoCsrfDecorator extends CsrfLocalProtection::Range {
+ Function function;
- DjangoCSRFDecorator() {
- this =
- API::moduleImport("django")
- .getMember("views")
- .getMember("decorators")
- .getMember("csrf")
- .getMember("csrf_protect")
- .getAUse() and
- this.asExpr() = function.getADecorator()
+ DjangoCsrfDecorator() {
+ this =
+ API::moduleImport("django")
+ .getMember("views")
+ .getMember("decorators")
+ .getMember("csrf")
+ .getMember("csrf_protect")
+ .getAUse() and
+ this.asExpr() = function.getADecorator()
+ }
+
+ override Function getProtected() { result = function }
}
-
- override Function getProtected() { result = function }
}
diff --git a/python/ql/src/Security/CWE-352/CSRFProtectionDisabled.ql b/python/ql/src/Security/CWE-352/CSRFProtectionDisabled.ql
index 5caa19d3d88..91609c25adb 100644
--- a/python/ql/src/Security/CWE-352/CSRFProtectionDisabled.ql
+++ b/python/ql/src/Security/CWE-352/CSRFProtectionDisabled.ql
@@ -14,10 +14,10 @@
import python
import semmle.python.Concepts
-from CSRFProtectionSetting s
+from CsrfProtectionSetting s
where
s.getVerificationSetting() = false and
- not exists(CSRFProtection p) and
+ not exists(CsrfLocalProtection p) and
// rule out test code as this is a common place to turn off CSRF protection
not s.getLocation().getFile().getAbsolutePath().matches("%test%")
select s, "Potential CSRF vulnerability due to forgery protection being disabled or weakened."
From 6c2449564a768a96989e8c60fae4d041a39bd80b Mon Sep 17 00:00:00 2001
From: Rasmus Lerchedahl Petersen
Date: Wed, 23 Mar 2022 12:05:09 +0100
Subject: [PATCH 030/171] python: add concept tests
---
.../test/experimental/meta/ConceptsTest.qll | 32 +++++++++++++++++++
.../frameworks/django-v2-v3/response_test.py | 2 +-
.../django-v2-v3/testproj/settings.py | 2 +-
3 files changed, 34 insertions(+), 2 deletions(-)
diff --git a/python/ql/test/experimental/meta/ConceptsTest.qll b/python/ql/test/experimental/meta/ConceptsTest.qll
index 517f3a50bf7..6fd15e586f5 100644
--- a/python/ql/test/experimental/meta/ConceptsTest.qll
+++ b/python/ql/test/experimental/meta/ConceptsTest.qll
@@ -503,3 +503,35 @@ class HttpClientRequestTest extends InlineExpectationsTest {
)
}
}
+
+class CsrfProtectionSettingTest extends InlineExpectationsTest {
+ CsrfProtectionSettingTest() { this = "CsrfProtectionSettingTest" }
+
+ override string getARelevantTag() { result = "CsrfProtectionSetting" }
+
+ override predicate hasActualResult(Location location, string element, string tag, string value) {
+ exists(location.getFile().getRelativePath()) and
+ exists(CsrfProtectionSetting setting |
+ location = setting.getLocation() and
+ element = setting.toString() and
+ value = setting.getVerificationSetting().toString() and
+ tag = "CsrfProtectionSetting"
+ )
+ }
+}
+
+class CsrfLocalProtectionTest extends InlineExpectationsTest {
+ CsrfLocalProtectionTest() { this = "CsrfLocalProtectionTest" }
+
+ override string getARelevantTag() { result = "CsrfLocalProtection" }
+
+ override predicate hasActualResult(Location location, string element, string tag, string value) {
+ exists(location.getFile().getRelativePath()) and
+ exists(CsrfLocalProtection p |
+ location = p.getLocation() and
+ element = p.toString() and
+ value = p.getProtected().getName().toString() and
+ tag = "CsrfLocalProtection"
+ )
+ }
+}
diff --git a/python/ql/test/library-tests/frameworks/django-v2-v3/response_test.py b/python/ql/test/library-tests/frameworks/django-v2-v3/response_test.py
index 4007b2d8063..73517f261fd 100644
--- a/python/ql/test/library-tests/frameworks/django-v2-v3/response_test.py
+++ b/python/ql/test/library-tests/frameworks/django-v2-v3/response_test.py
@@ -118,7 +118,7 @@ class CustomJsonResponse(JsonResponse):
def __init__(self, banner, content, *args, **kwargs):
super().__init__(content, *args, content_type="text/html", **kwargs)
-@csrf_protect
+@csrf_protect # $CsrfLocalProtection=safe__custom_json_response
def safe__custom_json_response(request):
return CustomJsonResponse("ACME Responses", {"foo": request.GET.get("foo")}) # $HttpResponse mimetype=application/json MISSING: responseBody=Dict SPURIOUS: responseBody="ACME Responses"
diff --git a/python/ql/test/library-tests/frameworks/django-v2-v3/testproj/settings.py b/python/ql/test/library-tests/frameworks/django-v2-v3/testproj/settings.py
index 5343182c1c9..7a0c10a21f6 100644
--- a/python/ql/test/library-tests/frameworks/django-v2-v3/testproj/settings.py
+++ b/python/ql/test/library-tests/frameworks/django-v2-v3/testproj/settings.py
@@ -40,7 +40,7 @@ INSTALLED_APPS = [
'django.contrib.staticfiles',
]
-MIDDLEWARE = [
+MIDDLEWARE = [ # $CsrfProtectionSetting=false
'django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
From 93336bcb16cedf671e8a1665c3b7ffeb45dfa0b5 Mon Sep 17 00:00:00 2001
From: Rasmus Lerchedahl Petersen
Date: Wed, 23 Mar 2022 12:27:51 +0100
Subject: [PATCH 031/171] python: allow alternative middleware (observed [on
LGTM](https://lgtm.com/projects/g/mozilla/mozillians/snapshot/9d6a7ee180addf7652fce96c21bafbad14d1dda7/files/mozillians/settings.py?sort=name&dir=ASC&mode=heatmap#L96))
---
python/ql/lib/semmle/python/frameworks/Django.qll | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/python/ql/lib/semmle/python/frameworks/Django.qll b/python/ql/lib/semmle/python/frameworks/Django.qll
index efa1a0eaa48..d623c663442 100644
--- a/python/ql/lib/semmle/python/frameworks/Django.qll
+++ b/python/ql/lib/semmle/python/frameworks/Django.qll
@@ -2340,7 +2340,12 @@ module PrivateDjango {
}
override boolean getVerificationSetting() {
- if list.getAnElt().(StrConst).getText() = "django.middleware.csrf.CsrfViewMiddleware"
+ if
+ list.getAnElt().(StrConst).getText() in [
+ "django.middleware.csrf.CsrfViewMiddleware",
+ // see https://github.com/mozilla/django-session-csrf
+ "session_csrf.CsrfMiddleware"
+ ]
then result = true
else result = false
}
From aecf4e48f8cd18526440fcdcfd400be2c733c013 Mon Sep 17 00:00:00 2001
From: Rasmus Lerchedahl Petersen
Date: Thu, 24 Mar 2022 11:43:07 +0100
Subject: [PATCH 032/171] python: add change note
---
python/ql/src/change-notes/2022-03-24-csrf-protection.md | 4 ++++
1 file changed, 4 insertions(+)
create mode 100644 python/ql/src/change-notes/2022-03-24-csrf-protection.md
diff --git a/python/ql/src/change-notes/2022-03-24-csrf-protection.md b/python/ql/src/change-notes/2022-03-24-csrf-protection.md
new file mode 100644
index 00000000000..fc733b7b030
--- /dev/null
+++ b/python/ql/src/change-notes/2022-03-24-csrf-protection.md
@@ -0,0 +1,4 @@
+---
+ category: newQuery
+ ---
+ * The query "CSRF protection weakened or disabled" (`py/csrf-protection-disabled`) has been implemented. Its results will now appear by default.
\ No newline at end of file
From ce017394e6e8fd4242639412f5db14126c6dcb71 Mon Sep 17 00:00:00 2001
From: Rasmus Lerchedahl Petersen
Date: Thu, 24 Mar 2022 12:01:46 +0100
Subject: [PATCH 033/171] python: fix change note (hepofully)
---
python/ql/src/change-notes/2022-03-24-csrf-protection.md | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/python/ql/src/change-notes/2022-03-24-csrf-protection.md b/python/ql/src/change-notes/2022-03-24-csrf-protection.md
index fc733b7b030..14a291d5f78 100644
--- a/python/ql/src/change-notes/2022-03-24-csrf-protection.md
+++ b/python/ql/src/change-notes/2022-03-24-csrf-protection.md
@@ -1,4 +1,4 @@
---
category: newQuery
- ---
- * The query "CSRF protection weakened or disabled" (`py/csrf-protection-disabled`) has been implemented. Its results will now appear by default.
\ No newline at end of file
+---
+* The query "CSRF protection weakened or disabled" (`py/csrf-protection-disabled`) has been implemented. Its results will now appear by default.
From 85f1d92a0dfe7bb2617f440eb03ae14dba1a4e7b Mon Sep 17 00:00:00 2001
From: yoff
Date: Fri, 25 Mar 2022 11:42:32 +0100
Subject: [PATCH 034/171] Apply suggestions from code review
Co-authored-by: Rasmus Wriedt Larsen
---
python/ql/lib/semmle/python/Concepts.qll | 2 +-
python/ql/lib/semmle/python/frameworks/Django.qll | 8 ++++++--
.../ql/src/Security/CWE-352/CSRFProtectionDisabled.qhelp | 2 +-
3 files changed, 8 insertions(+), 4 deletions(-)
diff --git a/python/ql/lib/semmle/python/Concepts.qll b/python/ql/lib/semmle/python/Concepts.qll
index 1f4aca0b21a..31fd2a5cf0e 100644
--- a/python/ql/lib/semmle/python/Concepts.qll
+++ b/python/ql/lib/semmle/python/Concepts.qll
@@ -106,7 +106,7 @@ module FileSystemWriteAccess {
}
/**
- * A data-flow node that may set or unset Cross-site request forgery protection
+ * A data-flow node that enables or disables Cross-site request forgery protection
* in a global manner.
*
* Extend this class to refine existing API models. If you want to model new APIs,
diff --git a/python/ql/lib/semmle/python/frameworks/Django.qll b/python/ql/lib/semmle/python/frameworks/Django.qll
index d623c663442..5e7226a2f3a 100644
--- a/python/ql/lib/semmle/python/frameworks/Django.qll
+++ b/python/ql/lib/semmle/python/frameworks/Django.qll
@@ -2330,8 +2330,12 @@ module PrivateDjango {
mw.asVar().getName() = "MIDDLEWARE" and
DataFlow::localFlow(this, mw)
|
- // it only counts as setting the CSRF protection, if the app uses authentication,
- // so check that the list contains the django authentication middleware.
+ // To only include results where CSRF protection matters, we only care about CSRF
+ // protection when the django authentication middleware is enabled.
+ // Since an active session cookie is exactly what would allow an attacker to perform
+ // a CSRF attack.
+ // Notice that this does not ensure that this is not a FP, since the authentication
+ // middleware might be unused.
//
// This also strongly implies that we are actually looking at the `MIDDLEWARE` setting.
list.getAnElt().(StrConst).getText() =
diff --git a/python/ql/src/Security/CWE-352/CSRFProtectionDisabled.qhelp b/python/ql/src/Security/CWE-352/CSRFProtectionDisabled.qhelp
index 98a5dae20ba..c9a6d4f0f16 100644
--- a/python/ql/src/Security/CWE-352/CSRFProtectionDisabled.qhelp
+++ b/python/ql/src/Security/CWE-352/CSRFProtectionDisabled.qhelp
@@ -6,7 +6,7 @@
Cross-site request forgery (CSRF) is a type of vulnerability in which an
- attacker is able to force a user carry out an action that the user did
+ attacker is able to force a user to carry out an action that the user did
not intend.
From 778a88f32c7575ab273d2472b6552940646b9899 Mon Sep 17 00:00:00 2001
From: Rasmus Lerchedahl Petersen
Date: Fri, 25 Mar 2022 11:49:06 +0100
Subject: [PATCH 035/171] python: update qhelp removing custom middleware stack
will _not_ enable CSRF protection
---
python/ql/src/Security/CWE-352/CSRFProtectionDisabled.qhelp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/python/ql/src/Security/CWE-352/CSRFProtectionDisabled.qhelp b/python/ql/src/Security/CWE-352/CSRFProtectionDisabled.qhelp
index c9a6d4f0f16..51745c11632 100644
--- a/python/ql/src/Security/CWE-352/CSRFProtectionDisabled.qhelp
+++ b/python/ql/src/Security/CWE-352/CSRFProtectionDisabled.qhelp
@@ -47,7 +47,7 @@
The protecting middleware was probably commented out during a testing phase, when server-side token generation was not set up.
- Simply commenting it back in (or remove the custom middleware stack) will enable CSRF protection.
+ Simply commenting it back in will enable CSRF protection.
From 179f77b123958ba4f07e381b7b0b2e51e3f7f29f Mon Sep 17 00:00:00 2001
From: Rasmus Lerchedahl Petersen
Date: Fri, 25 Mar 2022 11:51:24 +0100
Subject: [PATCH 036/171] python: clearer comment
---
python/ql/lib/semmle/python/frameworks/Django.qll | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/python/ql/lib/semmle/python/frameworks/Django.qll b/python/ql/lib/semmle/python/frameworks/Django.qll
index 5e7226a2f3a..f2d0019e3fc 100644
--- a/python/ql/lib/semmle/python/frameworks/Django.qll
+++ b/python/ql/lib/semmle/python/frameworks/Django.qll
@@ -2337,7 +2337,8 @@ module PrivateDjango {
// Notice that this does not ensure that this is not a FP, since the authentication
// middleware might be unused.
//
- // This also strongly implies that we are actually looking at the `MIDDLEWARE` setting.
+ // This also strongly implies that `mw` is in fact a Django middleware setting and
+ // not just a variable named `MIDDLEWARE`.
list.getAnElt().(StrConst).getText() =
"django.contrib.auth.middleware.AuthenticationMiddleware"
)
From 1e9840d7794f800037d169365ba7ec82d99e1a95 Mon Sep 17 00:00:00 2001
From: Rasmus Lerchedahl Petersen
Date: Fri, 25 Mar 2022 12:28:33 +0100
Subject: [PATCH 037/171] python: broaden local protection concept
---
python/ql/lib/semmle/python/Concepts.qll | 30 +++++++++++--------
.../lib/semmle/python/frameworks/Django.qll | 10 +++++--
.../CWE-352/CSRFProtectionDisabled.ql | 2 +-
.../test/experimental/meta/ConceptsTest.qll | 14 +++++----
.../frameworks/django-v2-v3/response_test.py | 2 +-
5 files changed, 34 insertions(+), 24 deletions(-)
diff --git a/python/ql/lib/semmle/python/Concepts.qll b/python/ql/lib/semmle/python/Concepts.qll
index 31fd2a5cf0e..01aa16d2094 100644
--- a/python/ql/lib/semmle/python/Concepts.qll
+++ b/python/ql/lib/semmle/python/Concepts.qll
@@ -123,7 +123,7 @@ class CsrfProtectionSetting extends DataFlow::Node instanceof CsrfProtectionSett
/** Provides a class for modeling new CSRF protection setting APIs. */
module CsrfProtectionSetting {
/**
- * A data-flow node that may set or unset Cross-site request forgery protection
+ * A data-flow node that enables or disables Cross-site request forgery protection
* in a global manner.
*
* Extend this class to model new APIs. If you want to refine existing API models,
@@ -139,35 +139,39 @@ module CsrfProtectionSetting {
}
/**
- * A data-flow node that provides Cross-site request forgery protection
+ * A data-flow node that enables or disables Cross-site request forgery protection
* for a specific part of an application.
*
* Extend this class to refine existing API models. If you want to model new APIs,
- * extend `CsrfLocalProtection::Range` instead.
+ * extend `CsrfLocalProtectionSetting::Range` instead.
*/
-class CsrfLocalProtection extends DataFlow::Node instanceof CsrfLocalProtection::Range {
+class CsrfLocalProtectionSetting extends DataFlow::Node instanceof CsrfLocalProtectionSetting::Range {
/**
- * Gets a `Function` representing the protected interaction
- * (probably a request handler).
+ * Gets a request handler whose CSRF protection is changed.
*/
- Function getProtected() { result = super.getProtected() }
+ Function getRequestHandler() { result = super.getRequestHandler() }
+
+ /** Holds if CSRF protection is enabled by this setting */
+ predicate csrfEnabled() { super.csrfEnabled() }
}
/** Provides a class for modeling new CSRF protection setting APIs. */
-module CsrfLocalProtection {
+module CsrfLocalProtectionSetting {
/**
- * A data-flow node that provides Cross-site request forgery protection
+ * A data-flow node that enables or disables Cross-site request forgery protection
* for a specific part of an application.
*
* Extend this class to model new APIs. If you want to refine existing API models,
- * extend `CsrfLocalProtection` instead.
+ * extend `CsrfLocalProtectionSetting` instead.
*/
abstract class Range extends DataFlow::Node {
/**
- * Gets a `Function` representing the protected interaction
- * (probably a request handler).
+ * Gets a request handler whose CSRF protection is changed.
*/
- abstract Function getProtected();
+ abstract Function getRequestHandler();
+
+ /** Holds if CSRF protection is enabled by this setting */
+ abstract predicate csrfEnabled();
}
}
diff --git a/python/ql/lib/semmle/python/frameworks/Django.qll b/python/ql/lib/semmle/python/frameworks/Django.qll
index f2d0019e3fc..c2852187b48 100644
--- a/python/ql/lib/semmle/python/frameworks/Django.qll
+++ b/python/ql/lib/semmle/python/frameworks/Django.qll
@@ -2356,20 +2356,24 @@ module PrivateDjango {
}
}
- private class DjangoCsrfDecorator extends CsrfLocalProtection::Range {
+ private class DjangoCsrfDecorator extends CsrfLocalProtectionSetting::Range {
+ string decoratorName;
Function function;
DjangoCsrfDecorator() {
+ decoratorName in ["csrf_protect", "csrf_exempt", "requires_csrf_token", "ensure_csrf_cookie"] and
this =
API::moduleImport("django")
.getMember("views")
.getMember("decorators")
.getMember("csrf")
- .getMember("csrf_protect")
+ .getMember(decoratorName)
.getAUse() and
this.asExpr() = function.getADecorator()
}
- override Function getProtected() { result = function }
+ override Function getRequestHandler() { result = function }
+
+ override predicate csrfEnabled() { decoratorName in ["csrf_protect", "requires_csrf_token"] }
}
}
diff --git a/python/ql/src/Security/CWE-352/CSRFProtectionDisabled.ql b/python/ql/src/Security/CWE-352/CSRFProtectionDisabled.ql
index 91609c25adb..e7202a361e5 100644
--- a/python/ql/src/Security/CWE-352/CSRFProtectionDisabled.ql
+++ b/python/ql/src/Security/CWE-352/CSRFProtectionDisabled.ql
@@ -17,7 +17,7 @@ import semmle.python.Concepts
from CsrfProtectionSetting s
where
s.getVerificationSetting() = false and
- not exists(CsrfLocalProtection p) and
+ not exists(CsrfLocalProtectionSetting p | p.csrfEnabled()) and
// rule out test code as this is a common place to turn off CSRF protection
not s.getLocation().getFile().getAbsolutePath().matches("%test%")
select s, "Potential CSRF vulnerability due to forgery protection being disabled or weakened."
diff --git a/python/ql/test/experimental/meta/ConceptsTest.qll b/python/ql/test/experimental/meta/ConceptsTest.qll
index 6fd15e586f5..339087d50d6 100644
--- a/python/ql/test/experimental/meta/ConceptsTest.qll
+++ b/python/ql/test/experimental/meta/ConceptsTest.qll
@@ -520,18 +520,20 @@ class CsrfProtectionSettingTest extends InlineExpectationsTest {
}
}
-class CsrfLocalProtectionTest extends InlineExpectationsTest {
- CsrfLocalProtectionTest() { this = "CsrfLocalProtectionTest" }
+class CsrfLocalProtectionSettingTest extends InlineExpectationsTest {
+ CsrfLocalProtectionSettingTest() { this = "CsrfLocalProtectionSettingTest" }
- override string getARelevantTag() { result = "CsrfLocalProtection" }
+ override string getARelevantTag() { result = "CsrfLocalProtection" + ["Enabled", "Disabled"] }
override predicate hasActualResult(Location location, string element, string tag, string value) {
exists(location.getFile().getRelativePath()) and
- exists(CsrfLocalProtection p |
+ exists(CsrfLocalProtectionSetting p |
location = p.getLocation() and
element = p.toString() and
- value = p.getProtected().getName().toString() and
- tag = "CsrfLocalProtection"
+ value = p.getRequestHandler().getName().toString() and
+ if p.csrfEnabled()
+ then tag = "CsrfLocalProtectionEnabled"
+ else tag = "CsrfLocalProtectionDisabled"
)
}
}
diff --git a/python/ql/test/library-tests/frameworks/django-v2-v3/response_test.py b/python/ql/test/library-tests/frameworks/django-v2-v3/response_test.py
index 73517f261fd..dd78cd51016 100644
--- a/python/ql/test/library-tests/frameworks/django-v2-v3/response_test.py
+++ b/python/ql/test/library-tests/frameworks/django-v2-v3/response_test.py
@@ -118,7 +118,7 @@ class CustomJsonResponse(JsonResponse):
def __init__(self, banner, content, *args, **kwargs):
super().__init__(content, *args, content_type="text/html", **kwargs)
-@csrf_protect # $CsrfLocalProtection=safe__custom_json_response
+@csrf_protect # $CsrfLocalProtectionEnabled=safe__custom_json_response
def safe__custom_json_response(request):
return CustomJsonResponse("ACME Responses", {"foo": request.GET.get("foo")}) # $HttpResponse mimetype=application/json MISSING: responseBody=Dict SPURIOUS: responseBody="ACME Responses"
From f19ade3446319c4d9ac9b72f068a4a8c4cccc9c4 Mon Sep 17 00:00:00 2001
From: Marcono1234
Date: Sun, 27 Mar 2022 01:28:00 +0100
Subject: [PATCH 038/171] Java: Add `StmtExpr`
---
.../2022-03-27-statement-expression.md | 4 ++
java/ql/lib/semmle/code/java/Collections.qll | 2 +-
java/ql/lib/semmle/code/java/Expr.qll | 38 +++++++++++
java/ql/lib/semmle/code/java/Maps.qll | 2 +-
.../Statements/ReturnValueIgnored.ql | 2 +-
.../IgnoreExceptionalReturn.ql | 2 +-
.../CWE-297/IgnoredHostnameVerification.ql | 2 +-
.../library-tests/StmtExpr/StmtExpr.expected | 14 ++++
.../test/library-tests/StmtExpr/StmtExpr.java | 68 +++++++++++++++++++
.../test/library-tests/StmtExpr/StmtExpr.ql | 4 ++
java/ql/test/library-tests/StmtExpr/options | 1 +
11 files changed, 134 insertions(+), 5 deletions(-)
create mode 100644 java/ql/lib/change-notes/2022-03-27-statement-expression.md
create mode 100644 java/ql/test/library-tests/StmtExpr/StmtExpr.expected
create mode 100644 java/ql/test/library-tests/StmtExpr/StmtExpr.java
create mode 100644 java/ql/test/library-tests/StmtExpr/StmtExpr.ql
create mode 100644 java/ql/test/library-tests/StmtExpr/options
diff --git a/java/ql/lib/change-notes/2022-03-27-statement-expression.md b/java/ql/lib/change-notes/2022-03-27-statement-expression.md
new file mode 100644
index 00000000000..bb261f66878
--- /dev/null
+++ b/java/ql/lib/change-notes/2022-03-27-statement-expression.md
@@ -0,0 +1,4 @@
+---
+category: feature
+---
+* The QL class `StmtExpr` has been added to model statement expressions, that is, expressions whose result is discarded.
diff --git a/java/ql/lib/semmle/code/java/Collections.qll b/java/ql/lib/semmle/code/java/Collections.qll
index d557d6281de..e6da65faa04 100644
--- a/java/ql/lib/semmle/code/java/Collections.qll
+++ b/java/ql/lib/semmle/code/java/Collections.qll
@@ -84,7 +84,7 @@ class CollectionMutation extends MethodAccess {
CollectionMutation() { this.getMethod() instanceof CollectionMutator }
/** Holds if the result of this call is not immediately discarded. */
- predicate resultIsChecked() { not this.getParent() instanceof ExprStmt }
+ predicate resultIsChecked() { not this instanceof StmtExpr }
}
/** A method that queries the contents of a collection without mutating it. */
diff --git a/java/ql/lib/semmle/code/java/Expr.qll b/java/ql/lib/semmle/code/java/Expr.qll
index 8f77b1800a2..4659d0e78fc 100755
--- a/java/ql/lib/semmle/code/java/Expr.qll
+++ b/java/ql/lib/semmle/code/java/Expr.qll
@@ -2133,3 +2133,41 @@ class Argument extends Expr {
)
}
}
+
+/**
+ * A statement expression, as specified by JLS 17 section 14.8.
+ * The result of a statement expression, if any, is discarded.
+ *
+ * Not to be confused with `ExprStmt`; while the child of an `ExprStmt` is always
+ * a `StmtExpr`, the opposite is not true. A `StmtExpr` occurs for example also
+ * as 'init' of a `for` statement.
+ */
+class StmtExpr extends Expr {
+ StmtExpr() {
+ this = any(ExprStmt s).getExpr()
+ or
+ this = any(ForStmt s).getAnInit() and not this instanceof LocalVariableDeclExpr
+ or
+ this = any(ForStmt s).getAnUpdate()
+ or
+ // Only applies to SwitchStmt, but not to SwitchExpr, see JLS 17 section 14.11.2
+ // TODO: Possibly redundant depending on how https://github.com/github/codeql/issues/8570 is resolved
+ this = any(SwitchStmt s).getACase().getRuleExpression()
+ or
+ // TODO: Workarounds for https://github.com/github/codeql/issues/3605
+ exists(LambdaExpr lambda |
+ this = lambda.getExprBody() and
+ lambda.asMethod().getReturnType() instanceof VoidType
+ )
+ or
+ exists(MemberRefExpr memberRef, Method implicitMethod, Method overridden |
+ implicitMethod = memberRef.asMethod()
+ |
+ this.getParent().(ReturnStmt).getEnclosingCallable() = implicitMethod and
+ // asMethod() has bogus method with wrong return type as result, e.g. `run(): String` (overriding `Runnable.run(): void`)
+ // Therefore need to check the overridden method
+ implicitMethod.getSourceDeclaration().overridesOrInstantiates*(overridden) and
+ overridden.getReturnType() instanceof VoidType
+ )
+ }
+}
diff --git a/java/ql/lib/semmle/code/java/Maps.qll b/java/ql/lib/semmle/code/java/Maps.qll
index 784db84fb98..f768ee3642b 100644
--- a/java/ql/lib/semmle/code/java/Maps.qll
+++ b/java/ql/lib/semmle/code/java/Maps.qll
@@ -53,7 +53,7 @@ class MapMutation extends MethodAccess {
MapMutation() { this.getMethod() instanceof MapMutator }
/** Holds if the result of this call is not immediately discarded. */
- predicate resultIsChecked() { not this.getParent() instanceof ExprStmt }
+ predicate resultIsChecked() { not this instanceof StmtExpr }
}
/** A method that queries the contents of the map it belongs to without mutating it. */
diff --git a/java/ql/src/Likely Bugs/Statements/ReturnValueIgnored.ql b/java/ql/src/Likely Bugs/Statements/ReturnValueIgnored.ql
index 39d6e7fe16b..1c2905c1d61 100644
--- a/java/ql/src/Likely Bugs/Statements/ReturnValueIgnored.ql
+++ b/java/ql/src/Likely Bugs/Statements/ReturnValueIgnored.ql
@@ -18,7 +18,7 @@ import Chaining
predicate checkedMethodCall(MethodAccess ma) {
relevantMethodCall(ma, _) and
- not ma.getParent() instanceof ExprStmt
+ not ma instanceof StmtExpr
}
/**
diff --git a/java/ql/src/Violations of Best Practice/Exception Handling/IgnoreExceptionalReturn.ql b/java/ql/src/Violations of Best Practice/Exception Handling/IgnoreExceptionalReturn.ql
index cbdaddf3b45..ed712eb2504 100644
--- a/java/ql/src/Violations of Best Practice/Exception Handling/IgnoreExceptionalReturn.ql
+++ b/java/ql/src/Violations of Best Practice/Exception Handling/IgnoreExceptionalReturn.ql
@@ -45,7 +45,7 @@ predicate unboundedQueue(RefType t) {
from MethodAccess ma, SpecialMethod m
where
- ma.getParent() instanceof ExprStmt and
+ ma instanceof StmtExpr and
m = ma.getMethod() and
(
m.isMethod("java.util", "Queue", "offer", 1) and not unboundedQueue(m.getDeclaringType())
diff --git a/java/ql/src/experimental/Security/CWE/CWE-297/IgnoredHostnameVerification.ql b/java/ql/src/experimental/Security/CWE/CWE-297/IgnoredHostnameVerification.ql
index 55d51a19a8c..38e2cb79998 100644
--- a/java/ql/src/experimental/Security/CWE/CWE-297/IgnoredHostnameVerification.ql
+++ b/java/ql/src/experimental/Security/CWE/CWE-297/IgnoredHostnameVerification.ql
@@ -21,7 +21,7 @@ private class HostnameVerificationCall extends MethodAccess {
}
/** Holds if the result of the call is not used. */
- predicate isIgnored() { this = any(ExprStmt es).getExpr() }
+ predicate isIgnored() { this instanceof StmtExpr }
}
from HostnameVerificationCall verification
diff --git a/java/ql/test/library-tests/StmtExpr/StmtExpr.expected b/java/ql/test/library-tests/StmtExpr/StmtExpr.expected
new file mode 100644
index 00000000000..ecb5e338238
--- /dev/null
+++ b/java/ql/test/library-tests/StmtExpr/StmtExpr.expected
@@ -0,0 +1,14 @@
+| StmtExpr.java:7:9:7:18 | toString(...) |
+| StmtExpr.java:13:9:13:13 | ...=... |
+| StmtExpr.java:14:9:14:11 | ...++ |
+| StmtExpr.java:15:9:15:11 | ++... |
+| StmtExpr.java:16:9:16:11 | ...-- |
+| StmtExpr.java:17:9:17:11 | --... |
+| StmtExpr.java:19:9:19:20 | new Object(...) |
+| StmtExpr.java:22:9:22:28 | clone(...) |
+| StmtExpr.java:25:14:25:39 | println(...) |
+| StmtExpr.java:30:17:30:44 | println(...) |
+| StmtExpr.java:45:24:45:33 | toString(...) |
+| StmtExpr.java:58:28:58:37 | toString(...) |
+| StmtExpr.java:60:13:60:22 | toString(...) |
+| StmtExpr.java:66:23:66:36 | toString(...) |
diff --git a/java/ql/test/library-tests/StmtExpr/StmtExpr.java b/java/ql/test/library-tests/StmtExpr/StmtExpr.java
new file mode 100644
index 00000000000..c35e24ea122
--- /dev/null
+++ b/java/ql/test/library-tests/StmtExpr/StmtExpr.java
@@ -0,0 +1,68 @@
+package StmtExpr;
+
+import java.util.function.Supplier;
+
+class StmtExpr {
+ void test() {
+ toString();
+
+ // LocalVariableDeclarationStatement with init is not a StatementExpression
+ String s = toString();
+
+ int i;
+ i = 0;
+ i++;
+ ++i;
+ i--;
+ --i;
+
+ new Object();
+ // ArrayCreationExpression cannot be a StatementExpression, but a method access
+ // on it can be
+ new int[] {}.clone();
+
+ // for statement init can be StatementExpression
+ for (System.out.println("init");;) {
+ break;
+ }
+
+ // for statement update is StatementExpression
+ for (;; System.out.println("update")) {
+ break;
+ }
+
+ // variable declaration and condition are not StatementExpressions
+ for (int i1 = 0; i1 < 10;) { }
+ for (int i1, i2 = 0; i2 < 10;) { }
+ for (;;) {
+ break;
+ }
+
+ // Not a StatementExpression
+ for (int i2 : new int[] {1}) { }
+
+ switch(1) {
+ default -> toString(); // StatementExpression
+ }
+ // SwitchExpression has no StatementExpression
+ String s2 = switch(1) {
+ default -> toString();
+ };
+
+ // Lambda with non-void return type has no StatementExpression
+ Supplier supplier1 = () -> toString();
+ Supplier supplier2 = () -> {
+ return toString();
+ };
+ // Lambda with void return type has StatementExpression
+ Runnable r = () -> toString();
+ Runnable r2 = () -> {
+ toString();
+ };
+
+ // Method reference with non-void return type has no StatementExpression
+ Supplier supplier3 = StmtExpr::new;
+ // Method reference with void return type has StatementExpression in implicit method body
+ Runnable r3 = this::toString;
+ }
+}
diff --git a/java/ql/test/library-tests/StmtExpr/StmtExpr.ql b/java/ql/test/library-tests/StmtExpr/StmtExpr.ql
new file mode 100644
index 00000000000..c624e738d71
--- /dev/null
+++ b/java/ql/test/library-tests/StmtExpr/StmtExpr.ql
@@ -0,0 +1,4 @@
+import java
+
+from StmtExpr e
+select e
diff --git a/java/ql/test/library-tests/StmtExpr/options b/java/ql/test/library-tests/StmtExpr/options
new file mode 100644
index 00000000000..03edcc8fcc0
--- /dev/null
+++ b/java/ql/test/library-tests/StmtExpr/options
@@ -0,0 +1 @@
+//semmle-extractor-options: --javac-args -source 14 -target 14
\ No newline at end of file
From 774c811e972f38a363948ed7f663abbb22804a91 Mon Sep 17 00:00:00 2001
From: Rasmus Lerchedahl Petersen
Date: Fri, 25 Mar 2022 12:58:19 +0100
Subject: [PATCH 039/171] python: move CSRF concepts inside `HTTP::Server`
---
python/ql/lib/semmle/python/Concepts.qll | 140 +++++++++---------
.../lib/semmle/python/frameworks/Django.qll | 4 +-
.../CWE-352/CSRFProtectionDisabled.ql | 4 +-
.../test/experimental/meta/ConceptsTest.qll | 4 +-
4 files changed, 76 insertions(+), 76 deletions(-)
diff --git a/python/ql/lib/semmle/python/Concepts.qll b/python/ql/lib/semmle/python/Concepts.qll
index 01aa16d2094..fe3cad338bb 100644
--- a/python/ql/lib/semmle/python/Concepts.qll
+++ b/python/ql/lib/semmle/python/Concepts.qll
@@ -105,76 +105,6 @@ module FileSystemWriteAccess {
}
}
-/**
- * A data-flow node that enables or disables Cross-site request forgery protection
- * in a global manner.
- *
- * Extend this class to refine existing API models. If you want to model new APIs,
- * extend `CsrfProtectionSetting::Range` instead.
- */
-class CsrfProtectionSetting extends DataFlow::Node instanceof CsrfProtectionSetting::Range {
- /**
- * Gets the boolean value corresponding to if CSRF protection is enabled
- * (`true`) or disabled (`false`) by this node.
- */
- boolean getVerificationSetting() { result = super.getVerificationSetting() }
-}
-
-/** Provides a class for modeling new CSRF protection setting APIs. */
-module CsrfProtectionSetting {
- /**
- * A data-flow node that enables or disables Cross-site request forgery protection
- * in a global manner.
- *
- * Extend this class to model new APIs. If you want to refine existing API models,
- * extend `CsrfProtectionSetting` instead.
- */
- abstract class Range extends DataFlow::Node {
- /**
- * Gets the boolean value corresponding to if CSRF protection is enabled
- * (`true`) or disabled (`false`) by this node.
- */
- abstract boolean getVerificationSetting();
- }
-}
-
-/**
- * A data-flow node that enables or disables Cross-site request forgery protection
- * for a specific part of an application.
- *
- * Extend this class to refine existing API models. If you want to model new APIs,
- * extend `CsrfLocalProtectionSetting::Range` instead.
- */
-class CsrfLocalProtectionSetting extends DataFlow::Node instanceof CsrfLocalProtectionSetting::Range {
- /**
- * Gets a request handler whose CSRF protection is changed.
- */
- Function getRequestHandler() { result = super.getRequestHandler() }
-
- /** Holds if CSRF protection is enabled by this setting */
- predicate csrfEnabled() { super.csrfEnabled() }
-}
-
-/** Provides a class for modeling new CSRF protection setting APIs. */
-module CsrfLocalProtectionSetting {
- /**
- * A data-flow node that enables or disables Cross-site request forgery protection
- * for a specific part of an application.
- *
- * Extend this class to model new APIs. If you want to refine existing API models,
- * extend `CsrfLocalProtectionSetting` instead.
- */
- abstract class Range extends DataFlow::Node {
- /**
- * Gets a request handler whose CSRF protection is changed.
- */
- abstract Function getRequestHandler();
-
- /** Holds if CSRF protection is enabled by this setting */
- abstract predicate csrfEnabled();
- }
-}
-
/** Provides classes for modeling path-related APIs. */
module Path {
/**
@@ -956,6 +886,76 @@ module HTTP {
abstract DataFlow::Node getValueArg();
}
}
+
+ /**
+ * A data-flow node that enables or disables Cross-site request forgery protection
+ * in a global manner.
+ *
+ * Extend this class to refine existing API models. If you want to model new APIs,
+ * extend `CsrfProtectionSetting::Range` instead.
+ */
+ class CsrfProtectionSetting extends DataFlow::Node instanceof CsrfProtectionSetting::Range {
+ /**
+ * Gets the boolean value corresponding to if CSRF protection is enabled
+ * (`true`) or disabled (`false`) by this node.
+ */
+ boolean getVerificationSetting() { result = super.getVerificationSetting() }
+ }
+
+ /** Provides a class for modeling new CSRF protection setting APIs. */
+ module CsrfProtectionSetting {
+ /**
+ * A data-flow node that enables or disables Cross-site request forgery protection
+ * in a global manner.
+ *
+ * Extend this class to model new APIs. If you want to refine existing API models,
+ * extend `CsrfProtectionSetting` instead.
+ */
+ abstract class Range extends DataFlow::Node {
+ /**
+ * Gets the boolean value corresponding to if CSRF protection is enabled
+ * (`true`) or disabled (`false`) by this node.
+ */
+ abstract boolean getVerificationSetting();
+ }
+ }
+
+ /**
+ * A data-flow node that enables or disables Cross-site request forgery protection
+ * for a specific part of an application.
+ *
+ * Extend this class to refine existing API models. If you want to model new APIs,
+ * extend `CsrfLocalProtectionSetting::Range` instead.
+ */
+ class CsrfLocalProtectionSetting extends DataFlow::Node instanceof CsrfLocalProtectionSetting::Range {
+ /**
+ * Gets a request handler whose CSRF protection is changed.
+ */
+ Function getRequestHandler() { result = super.getRequestHandler() }
+
+ /** Holds if CSRF protection is enabled by this setting */
+ predicate csrfEnabled() { super.csrfEnabled() }
+ }
+
+ /** Provides a class for modeling new CSRF protection setting APIs. */
+ module CsrfLocalProtectionSetting {
+ /**
+ * A data-flow node that enables or disables Cross-site request forgery protection
+ * for a specific part of an application.
+ *
+ * Extend this class to model new APIs. If you want to refine existing API models,
+ * extend `CsrfLocalProtectionSetting` instead.
+ */
+ abstract class Range extends DataFlow::Node {
+ /**
+ * Gets a request handler whose CSRF protection is changed.
+ */
+ abstract Function getRequestHandler();
+
+ /** Holds if CSRF protection is enabled by this setting */
+ abstract predicate csrfEnabled();
+ }
+ }
}
/** Provides classes for modeling HTTP clients. */
diff --git a/python/ql/lib/semmle/python/frameworks/Django.qll b/python/ql/lib/semmle/python/frameworks/Django.qll
index c2852187b48..f8b4272a741 100644
--- a/python/ql/lib/semmle/python/frameworks/Django.qll
+++ b/python/ql/lib/semmle/python/frameworks/Django.qll
@@ -2320,7 +2320,7 @@ module PrivateDjango {
/**
* A custom middleware stack
*/
- private class DjangoSettingsMiddlewareStack extends CsrfProtectionSetting::Range {
+ private class DjangoSettingsMiddlewareStack extends HTTP::Server::CsrfProtectionSetting::Range {
List list;
DjangoSettingsMiddlewareStack() {
@@ -2356,7 +2356,7 @@ module PrivateDjango {
}
}
- private class DjangoCsrfDecorator extends CsrfLocalProtectionSetting::Range {
+ private class DjangoCsrfDecorator extends HTTP::Server::CsrfLocalProtectionSetting::Range {
string decoratorName;
Function function;
diff --git a/python/ql/src/Security/CWE-352/CSRFProtectionDisabled.ql b/python/ql/src/Security/CWE-352/CSRFProtectionDisabled.ql
index e7202a361e5..44353c9b322 100644
--- a/python/ql/src/Security/CWE-352/CSRFProtectionDisabled.ql
+++ b/python/ql/src/Security/CWE-352/CSRFProtectionDisabled.ql
@@ -14,10 +14,10 @@
import python
import semmle.python.Concepts
-from CsrfProtectionSetting s
+from HTTP::Server::CsrfProtectionSetting s
where
s.getVerificationSetting() = false and
- not exists(CsrfLocalProtectionSetting p | p.csrfEnabled()) and
+ not exists(HTTP::Server::CsrfLocalProtectionSetting p | p.csrfEnabled()) and
// rule out test code as this is a common place to turn off CSRF protection
not s.getLocation().getFile().getAbsolutePath().matches("%test%")
select s, "Potential CSRF vulnerability due to forgery protection being disabled or weakened."
diff --git a/python/ql/test/experimental/meta/ConceptsTest.qll b/python/ql/test/experimental/meta/ConceptsTest.qll
index 339087d50d6..b7fbea26264 100644
--- a/python/ql/test/experimental/meta/ConceptsTest.qll
+++ b/python/ql/test/experimental/meta/ConceptsTest.qll
@@ -511,7 +511,7 @@ class CsrfProtectionSettingTest extends InlineExpectationsTest {
override predicate hasActualResult(Location location, string element, string tag, string value) {
exists(location.getFile().getRelativePath()) and
- exists(CsrfProtectionSetting setting |
+ exists(HTTP::Server::CsrfProtectionSetting setting |
location = setting.getLocation() and
element = setting.toString() and
value = setting.getVerificationSetting().toString() and
@@ -527,7 +527,7 @@ class CsrfLocalProtectionSettingTest extends InlineExpectationsTest {
override predicate hasActualResult(Location location, string element, string tag, string value) {
exists(location.getFile().getRelativePath()) and
- exists(CsrfLocalProtectionSetting p |
+ exists(HTTP::Server::CsrfLocalProtectionSetting p |
location = p.getLocation() and
element = p.toString() and
value = p.getRequestHandler().getName().toString() and
From d39410aa2d54edc7f5f680a8d1f4e0f766b5d323 Mon Sep 17 00:00:00 2001
From: Rasmus Lerchedahl Petersen
Date: Fri, 25 Mar 2022 13:03:00 +0100
Subject: [PATCH 040/171] python: backport review comment to Ruby
---
.../src/queries/security/cwe-352/CSRFProtectionDisabled.qhelp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ruby/ql/src/queries/security/cwe-352/CSRFProtectionDisabled.qhelp b/ruby/ql/src/queries/security/cwe-352/CSRFProtectionDisabled.qhelp
index 8b5ff38d5a3..7656a676d64 100644
--- a/ruby/ql/src/queries/security/cwe-352/CSRFProtectionDisabled.qhelp
+++ b/ruby/ql/src/queries/security/cwe-352/CSRFProtectionDisabled.qhelp
@@ -6,7 +6,7 @@
Cross-site request forgery (CSRF) is a type of vulnerability in which an
- attacker is able to force a user carry out an action that the user did
+ attacker is able to force a user to carry out an action that the user did
not intend.
From 3416f074e8365ea250e5b3abcb1dae4e8a10dc96 Mon Sep 17 00:00:00 2001
From: yoff
Date: Tue, 29 Mar 2022 13:59:04 +0200
Subject: [PATCH 041/171] Update
python/ql/src/Security/CWE-352/CSRFProtectionDisabled.ql
Explain why `TestScope` is not used.
Co-authored-by: Rasmus Wriedt Larsen
---
python/ql/src/Security/CWE-352/CSRFProtectionDisabled.ql | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/python/ql/src/Security/CWE-352/CSRFProtectionDisabled.ql b/python/ql/src/Security/CWE-352/CSRFProtectionDisabled.ql
index 44353c9b322..24917411fb4 100644
--- a/python/ql/src/Security/CWE-352/CSRFProtectionDisabled.ql
+++ b/python/ql/src/Security/CWE-352/CSRFProtectionDisabled.ql
@@ -18,6 +18,8 @@ from HTTP::Server::CsrfProtectionSetting s
where
s.getVerificationSetting() = false and
not exists(HTTP::Server::CsrfLocalProtectionSetting p | p.csrfEnabled()) and
- // rule out test code as this is a common place to turn off CSRF protection
+ // rule out test code as this is a common place to turn off CSRF protection.
+ // We don't use normal `TestScope` to find test files, since we also want to match
+ // a settings file such as `.../integration-tests/settings.py`
not s.getLocation().getFile().getAbsolutePath().matches("%test%")
select s, "Potential CSRF vulnerability due to forgery protection being disabled or weakened."
From 92033047a51018285b081cba1d234b704078d9e3 Mon Sep 17 00:00:00 2001
From: Porcupiney Hairs
Date: Tue, 29 Mar 2022 23:32:33 +0530
Subject: [PATCH 042/171] Python : Add query to detect PAM authorization bypass
Using only a call to `pam_authenticate` to check the validity of a login can
lead to authorization bypass vulnerabilities. A `pam_authenticate` only
verifies the credentials of a user. It does not check if a user has an
appropriate authorization to actually login. This means a user with a
expired login or a password can still access the system.
This PR includes a qhelp describing the issue, a query which detects instances where a call to
`pam_acc_mgmt` does not follow a call to `pam_authenticate` and it's
corresponding tests.
This PR has multiple detections. Some of the public one I can find are :
* [CVE-2022-0860](https://nvd.nist.gov/vuln/detail/CVE-2022-0860) found
in [cobbler/cobbler](https://www.github.com/cobbler/cobbler)
* [fredhutch/motuz](https://www.huntr.dev/bounties/d46f91ca-b8ef-4b67-a79a-2420c4c6d52b/)
---
.../Security/CWE-285/PamAuthorization.qhelp | 49 ++++++++++
.../Security/CWE-285/PamAuthorization.ql | 58 +++++++++++
.../Security/CWE-285/PamAuthorizationBad.py | 15 +++
.../Security/CWE-285/PamAuthorizationGood.py | 17 ++++
.../CWE-285/PamAuthorization.expected | 1 +
.../Security/CWE-285/PamAuthorization.qlref | 1 +
.../query-tests/Security/CWE-285/bad.py | 95 ++++++++++++++++++
.../query-tests/Security/CWE-285/good.py | 97 +++++++++++++++++++
8 files changed, 333 insertions(+)
create mode 100644 python/ql/src/experimental/Security/CWE-285/PamAuthorization.qhelp
create mode 100644 python/ql/src/experimental/Security/CWE-285/PamAuthorization.ql
create mode 100644 python/ql/src/experimental/Security/CWE-285/PamAuthorizationBad.py
create mode 100644 python/ql/src/experimental/Security/CWE-285/PamAuthorizationGood.py
create mode 100644 python/ql/test/experimental/query-tests/Security/CWE-285/PamAuthorization.expected
create mode 100644 python/ql/test/experimental/query-tests/Security/CWE-285/PamAuthorization.qlref
create mode 100644 python/ql/test/experimental/query-tests/Security/CWE-285/bad.py
create mode 100644 python/ql/test/experimental/query-tests/Security/CWE-285/good.py
diff --git a/python/ql/src/experimental/Security/CWE-285/PamAuthorization.qhelp b/python/ql/src/experimental/Security/CWE-285/PamAuthorization.qhelp
new file mode 100644
index 00000000000..8e0f829f33e
--- /dev/null
+++ b/python/ql/src/experimental/Security/CWE-285/PamAuthorization.qhelp
@@ -0,0 +1,49 @@
+
+
+
+
+ Using only a call to
+ pam_authenticate
+ to check the validity of a login can lead to authorization bypass vulnerabilities.
+
+
+ A
+ pam_authenticate
+ only verifies the credentials of a user. It does not check if a user has an appropriate authorization to actually login. This means a user with a expired login or a password can still access the system.
+
+
+
+
+
+
+ A call to
+ pam_authenticate
+ should be followed by a call to
+ pam_acct_mgmt
+ to check if a user is allowed to login.
+
+
+
+
+
+ In the following example, the code only checks the credentials of a user. Hence, in this case, a user expired with expired creds can still login. This can be verified by creating a new user account, expiring it with
+ chage -E0 `username`
+ and then trying to log in.
+
+
+
+
+ This can be avoided by calling
+ pam_acct_mgmt
+ call to verify access as has been done in the snippet shown below.
+
+
+
+
+
+
+ Man-Page:
+ pam_acct_mgmt
+
+
+
\ No newline at end of file
diff --git a/python/ql/src/experimental/Security/CWE-285/PamAuthorization.ql b/python/ql/src/experimental/Security/CWE-285/PamAuthorization.ql
new file mode 100644
index 00000000000..e67745cceac
--- /dev/null
+++ b/python/ql/src/experimental/Security/CWE-285/PamAuthorization.ql
@@ -0,0 +1,58 @@
+/**
+ * @name Authorization bypass due to incorrect usage of PAM
+ * @description Using only the `pam_authenticate` call to check the validity of a login can lead to a authorization bypass.
+ * @kind problem
+ * @problem.severity warning
+ * @id py/pam-auth-bypass
+ * @tags security
+ * external/cwe/cwe-285
+ */
+
+import python
+import semmle.python.ApiGraphs
+import experimental.semmle.python.Concepts
+import semmle.python.dataflow.new.TaintTracking
+
+private class LibPam extends API::Node {
+ LibPam() {
+ exists(
+ API::Node cdll, API::Node find_library, API::Node libpam, API::CallNode cdll_call,
+ API::CallNode find_lib_call, StrConst str
+ |
+ API::moduleImport("ctypes").getMember("CDLL") = cdll and
+ find_library = API::moduleImport("ctypes.util").getMember("find_library") and
+ cdll_call = cdll.getACall() and
+ find_lib_call = find_library.getACall() and
+ DataFlow::localFlow(DataFlow::exprNode(str), find_lib_call.getArg(0)) and
+ str.getText() = "pam" and
+ cdll_call.getArg(0) = find_lib_call and
+ libpam = cdll_call.getReturn()
+ |
+ libpam = this
+ )
+ }
+
+ override string toString() { result = "libpam" }
+}
+
+class PamAuthCall extends API::Node {
+ PamAuthCall() { exists(LibPam pam | pam.getMember("pam_authenticate") = this) }
+
+ override string toString() { result = "pam_authenticate" }
+}
+
+class PamActMgt extends API::Node {
+ PamActMgt() { exists(LibPam pam | pam.getMember("pam_acct_mgmt") = this) }
+
+ override string toString() { result = "pam_acct_mgmt" }
+}
+
+from PamAuthCall p, API::CallNode u, Expr handle
+where
+ u = p.getACall() and
+ handle = u.asExpr().(Call).getArg(0) and
+ not exists(PamActMgt pam |
+ DataFlow::localFlow(DataFlow::exprNode(handle),
+ DataFlow::exprNode(pam.getACall().asExpr().(Call).getArg(0)))
+ )
+select u, "This PAM authentication call may be lead to an authorization bypass."
diff --git a/python/ql/src/experimental/Security/CWE-285/PamAuthorizationBad.py b/python/ql/src/experimental/Security/CWE-285/PamAuthorizationBad.py
new file mode 100644
index 00000000000..3b06156f551
--- /dev/null
+++ b/python/ql/src/experimental/Security/CWE-285/PamAuthorizationBad.py
@@ -0,0 +1,15 @@
+def authenticate(self, username, password, service='login', encoding='utf-8', resetcreds=True):
+ libpam = CDLL(find_library("pam"))
+ pam_authenticate = libpam.pam_authenticate
+ pam_acct_mgmt = libpam.pam_acct_mgmt
+ pam_authenticate.restype = c_int
+ pam_authenticate.argtypes = [PamHandle, c_int]
+ pam_acct_mgmt.restype = c_int
+ pam_acct_mgmt.argtypes = [PamHandle, c_int]
+
+ handle = PamHandle()
+ conv = PamConv(my_conv, 0)
+ retval = pam_start(service, username, byref(conv), byref(handle))
+
+ retval = pam_authenticate(handle, 0)
+ return retval == 0
\ No newline at end of file
diff --git a/python/ql/src/experimental/Security/CWE-285/PamAuthorizationGood.py b/python/ql/src/experimental/Security/CWE-285/PamAuthorizationGood.py
new file mode 100644
index 00000000000..0f047c6ac65
--- /dev/null
+++ b/python/ql/src/experimental/Security/CWE-285/PamAuthorizationGood.py
@@ -0,0 +1,17 @@
+def authenticate(self, username, password, service='login', encoding='utf-8', resetcreds=True):
+ libpam = CDLL(find_library("pam"))
+ pam_authenticate = libpam.pam_authenticate
+ pam_acct_mgmt = libpam.pam_acct_mgmt
+ pam_authenticate.restype = c_int
+ pam_authenticate.argtypes = [PamHandle, c_int]
+ pam_acct_mgmt.restype = c_int
+ pam_acct_mgmt.argtypes = [PamHandle, c_int]
+
+ handle = PamHandle()
+ conv = PamConv(my_conv, 0)
+ retval = pam_start(service, username, byref(conv), byref(handle))
+
+ retval = pam_authenticate(handle, 0)
+ if retval == 0:
+ retval = pam_acct_mgmt(handle, 0)
+ return retval == 0
\ No newline at end of file
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-285/PamAuthorization.expected b/python/ql/test/experimental/query-tests/Security/CWE-285/PamAuthorization.expected
new file mode 100644
index 00000000000..52c4c8ac669
--- /dev/null
+++ b/python/ql/test/experimental/query-tests/Security/CWE-285/PamAuthorization.expected
@@ -0,0 +1 @@
+| bad.py:92:18:92:44 | ControlFlowNode for pam_authenticate() | This PAM authentication call may be lead to an authorization bypass. |
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-285/PamAuthorization.qlref b/python/ql/test/experimental/query-tests/Security/CWE-285/PamAuthorization.qlref
new file mode 100644
index 00000000000..38fac298b1e
--- /dev/null
+++ b/python/ql/test/experimental/query-tests/Security/CWE-285/PamAuthorization.qlref
@@ -0,0 +1 @@
+experimental/Security/CWE-285/PamAuthorization.ql
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-285/bad.py b/python/ql/test/experimental/query-tests/Security/CWE-285/bad.py
new file mode 100644
index 00000000000..84527d6f6fb
--- /dev/null
+++ b/python/ql/test/experimental/query-tests/Security/CWE-285/bad.py
@@ -0,0 +1,95 @@
+from ctypes import CDLL, POINTER, Structure, CFUNCTYPE, cast, byref, sizeof
+from ctypes import c_void_p, c_size_t, c_char_p, c_char, c_int
+from ctypes import memmove
+from ctypes.util import find_library
+
+class PamHandle(Structure):
+ _fields_ = [ ("handle", c_void_p) ]
+
+ def __init__(self):
+ Structure.__init__(self)
+ self.handle = 0
+
+class PamMessage(Structure):
+ """wrapper class for pam_message structure"""
+ _fields_ = [ ("msg_style", c_int), ("msg", c_char_p) ]
+
+ def __repr__(self):
+ return "" % (self.msg_style, self.msg)
+
+class PamResponse(Structure):
+ """wrapper class for pam_response structure"""
+ _fields_ = [ ("resp", c_char_p), ("resp_retcode", c_int) ]
+
+ def __repr__(self):
+ return "" % (self.resp_retcode, self.resp)
+
+conv_func = CFUNCTYPE(c_int, c_int, POINTER(POINTER(PamMessage)), POINTER(POINTER(PamResponse)), c_void_p)
+
+class PamConv(Structure):
+ """wrapper class for pam_conv structure"""
+ _fields_ = [ ("conv", conv_func), ("appdata_ptr", c_void_p) ]
+
+# Various constants
+PAM_PROMPT_ECHO_OFF = 1
+PAM_PROMPT_ECHO_ON = 2
+PAM_ERROR_MSG = 3
+PAM_TEXT_INFO = 4
+PAM_REINITIALIZE_CRED = 8
+
+libc = CDLL(find_library("c"))
+libpam = CDLL(find_library("pam"))
+
+calloc = libc.calloc
+calloc.restype = c_void_p
+calloc.argtypes = [c_size_t, c_size_t]
+
+# bug #6 (@NIPE-SYSTEMS), some libpam versions don't include this function
+if hasattr(libpam, 'pam_end'):
+ pam_end = libpam.pam_end
+ pam_end.restype = c_int
+ pam_end.argtypes = [PamHandle, c_int]
+
+pam_start = libpam.pam_start
+pam_start.restype = c_int
+pam_start.argtypes = [c_char_p, c_char_p, POINTER(PamConv), POINTER(PamHandle)]
+
+pam_setcred = libpam.pam_setcred
+pam_setcred.restype = c_int
+pam_setcred.argtypes = [PamHandle, c_int]
+
+pam_strerror = libpam.pam_strerror
+pam_strerror.restype = c_char_p
+pam_strerror.argtypes = [PamHandle, c_int]
+
+pam_authenticate = libpam.pam_authenticate
+pam_authenticate.restype = c_int
+pam_authenticate.argtypes = [PamHandle, c_int]
+
+pam_acct_mgmt = libpam.pam_acct_mgmt
+pam_acct_mgmt.restype = c_int
+pam_acct_mgmt.argtypes = [PamHandle, c_int]
+
+class pam():
+ code = 0
+ reason = None
+
+ def __init__(self):
+ pass
+
+ def authenticate(self, username, password, service='login', encoding='utf-8', resetcreds=True):
+ @conv_func
+ def my_conv(n_messages, messages, p_response, app_data):
+ return 0
+
+
+ cpassword = c_char_p(password)
+
+ handle = PamHandle()
+ conv = PamConv(my_conv, 0)
+ retval = pam_start(service, username, byref(conv), byref(handle))
+
+ retval = pam_authenticate(handle, 0)
+ auth_success = retval == 0
+
+ return auth_success
\ No newline at end of file
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-285/good.py b/python/ql/test/experimental/query-tests/Security/CWE-285/good.py
new file mode 100644
index 00000000000..e9996c770ed
--- /dev/null
+++ b/python/ql/test/experimental/query-tests/Security/CWE-285/good.py
@@ -0,0 +1,97 @@
+from ctypes import CDLL, POINTER, Structure, CFUNCTYPE, cast, byref, sizeof
+from ctypes import c_void_p, c_size_t, c_char_p, c_char, c_int
+from ctypes import memmove
+from ctypes.util import find_library
+
+class PamHandle(Structure):
+ _fields_ = [ ("handle", c_void_p) ]
+
+ def __init__(self):
+ Structure.__init__(self)
+ self.handle = 0
+
+class PamMessage(Structure):
+ """wrapper class for pam_message structure"""
+ _fields_ = [ ("msg_style", c_int), ("msg", c_char_p) ]
+
+ def __repr__(self):
+ return "" % (self.msg_style, self.msg)
+
+class PamResponse(Structure):
+ """wrapper class for pam_response structure"""
+ _fields_ = [ ("resp", c_char_p), ("resp_retcode", c_int) ]
+
+ def __repr__(self):
+ return "" % (self.resp_retcode, self.resp)
+
+conv_func = CFUNCTYPE(c_int, c_int, POINTER(POINTER(PamMessage)), POINTER(POINTER(PamResponse)), c_void_p)
+
+class PamConv(Structure):
+ """wrapper class for pam_conv structure"""
+ _fields_ = [ ("conv", conv_func), ("appdata_ptr", c_void_p) ]
+
+# Various constants
+PAM_PROMPT_ECHO_OFF = 1
+PAM_PROMPT_ECHO_ON = 2
+PAM_ERROR_MSG = 3
+PAM_TEXT_INFO = 4
+PAM_REINITIALIZE_CRED = 8
+
+libc = CDLL(find_library("c"))
+libpam = CDLL(find_library("pam"))
+
+calloc = libc.calloc
+calloc.restype = c_void_p
+calloc.argtypes = [c_size_t, c_size_t]
+
+# bug #6 (@NIPE-SYSTEMS), some libpam versions don't include this function
+if hasattr(libpam, 'pam_end'):
+ pam_end = libpam.pam_end
+ pam_end.restype = c_int
+ pam_end.argtypes = [PamHandle, c_int]
+
+pam_start = libpam.pam_start
+pam_start.restype = c_int
+pam_start.argtypes = [c_char_p, c_char_p, POINTER(PamConv), POINTER(PamHandle)]
+
+pam_setcred = libpam.pam_setcred
+pam_setcred.restype = c_int
+pam_setcred.argtypes = [PamHandle, c_int]
+
+pam_strerror = libpam.pam_strerror
+pam_strerror.restype = c_char_p
+pam_strerror.argtypes = [PamHandle, c_int]
+
+pam_authenticate = libpam.pam_authenticate
+pam_authenticate.restype = c_int
+pam_authenticate.argtypes = [PamHandle, c_int]
+
+pam_acct_mgmt = libpam.pam_acct_mgmt
+pam_acct_mgmt.restype = c_int
+pam_acct_mgmt.argtypes = [PamHandle, c_int]
+
+class pam():
+ code = 0
+ reason = None
+
+ def __init__(self):
+ pass
+
+ def authenticate(self, username, password, service='login', encoding='utf-8', resetcreds=True):
+ @conv_func
+ def my_conv(n_messages, messages, p_response, app_data):
+ return 0
+
+
+ cpassword = c_char_p(password)
+
+ handle = PamHandle()
+ conv = PamConv(my_conv, 0)
+ retval = pam_start(service, username, byref(conv), byref(handle))
+
+ retval = pam_authenticate(handle, 0)
+ if retval == 0:
+ retval = pam_acct_mgmt(handle, 0)
+ auth_success = retval == 0
+
+ return auth_success
\ No newline at end of file
From 65907c97620e2824ffbd008db54a98d9ee1a1060 Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Thu, 24 Mar 2022 14:13:12 +0100
Subject: [PATCH 043/171] Python: Copy Xxe/XmlBomb queries from JS
After internal discussion, these will replace the `XmlEntityInjection`
query, so we can have separate severities on DoS and the other (more
serious) attacks.
Note: These clearly don't work, since they are verbatim copies of the JS
code, but I split it into multiple commits to clearly highlight what
changes were made.
---
.../Security/NEW/CWE-611/Xxe.qhelp | 57 ++++++++++++++++++
.../experimental/Security/NEW/CWE-611/Xxe.ql | 23 +++++++
.../Security/NEW/CWE-611/examples/Xxe.js | 7 +++
.../Security/NEW/CWE-611/examples/XxeGood.js | 7 +++
.../Security/NEW/CWE-776/XmlBomb.qhelp | 60 +++++++++++++++++++
.../Security/NEW/CWE-776/XmlBomb.ql | 23 +++++++
.../Security/NEW/CWE-776/examples/XmlBomb.js | 10 ++++
.../NEW/CWE-776/examples/XmlBombGood.js | 10 ++++
.../dataflow/XmlBombCustomizations.qll | 49 +++++++++++++++
.../python/security/dataflow/XmlBombQuery.qll | 27 +++++++++
.../security/dataflow/XxeCustomizations.qll | 52 ++++++++++++++++
.../python/security/dataflow/XxeQuery.qll | 27 +++++++++
12 files changed, 352 insertions(+)
create mode 100644 python/ql/src/experimental/Security/NEW/CWE-611/Xxe.qhelp
create mode 100644 python/ql/src/experimental/Security/NEW/CWE-611/Xxe.ql
create mode 100644 python/ql/src/experimental/Security/NEW/CWE-611/examples/Xxe.js
create mode 100644 python/ql/src/experimental/Security/NEW/CWE-611/examples/XxeGood.js
create mode 100644 python/ql/src/experimental/Security/NEW/CWE-776/XmlBomb.qhelp
create mode 100644 python/ql/src/experimental/Security/NEW/CWE-776/XmlBomb.ql
create mode 100644 python/ql/src/experimental/Security/NEW/CWE-776/examples/XmlBomb.js
create mode 100644 python/ql/src/experimental/Security/NEW/CWE-776/examples/XmlBombGood.js
create mode 100644 python/ql/src/experimental/semmle/python/security/dataflow/XmlBombCustomizations.qll
create mode 100644 python/ql/src/experimental/semmle/python/security/dataflow/XmlBombQuery.qll
create mode 100644 python/ql/src/experimental/semmle/python/security/dataflow/XxeCustomizations.qll
create mode 100644 python/ql/src/experimental/semmle/python/security/dataflow/XxeQuery.qll
diff --git a/python/ql/src/experimental/Security/NEW/CWE-611/Xxe.qhelp b/python/ql/src/experimental/Security/NEW/CWE-611/Xxe.qhelp
new file mode 100644
index 00000000000..1e859eb121f
--- /dev/null
+++ b/python/ql/src/experimental/Security/NEW/CWE-611/Xxe.qhelp
@@ -0,0 +1,57 @@
+
+
+
+
+
+Parsing untrusted XML files with a weakly configured XML parser may lead to an
+XML External Entity (XXE) attack. This type of attack uses external entity references
+to access arbitrary files on a system, carry out denial-of-service (DoS) attacks, or server-side
+request forgery. Even when the result of parsing is not returned to the user, DoS attacks are still possible
+and out-of-band data retrieval techniques may allow attackers to steal sensitive data.
+
+
+
+
+
+The easiest way to prevent XXE attacks is to disable external entity handling when
+parsing untrusted data. How this is done depends on the library being used. Note that some
+libraries, such as recent versions of libxml, disable entity expansion by default,
+so unless you have explicitly enabled entity expansion, no further action needs to be taken.
+
+
+
+
+
+The following example uses the libxml XML parser to parse a string xmlSrc.
+If that string is from an untrusted source, this code may be vulnerable to an XXE attack, since
+the parser is invoked with the noent option set to true:
+
+
+
+
+To guard against XXE attacks, the noent option should be omitted or set to
+false. This means that no entity expansion is undertaken at all, not even for standard
+internal entities such as & or >. If desired, these
+entities can be expanded in a separate step using utility functions provided by libraries such
+as underscore ,
+lodash or
+he .
+
+
+
+
+
+
+OWASP:
+XML External Entity (XXE) Processing .
+
+
+Timothy Morgen:
+XML Schema, DTD, and Entity Attacks .
+
+
+Timur Yunusov, Alexey Osipov:
+XML Out-Of-Band Data Retrieval .
+
+
+
diff --git a/python/ql/src/experimental/Security/NEW/CWE-611/Xxe.ql b/python/ql/src/experimental/Security/NEW/CWE-611/Xxe.ql
new file mode 100644
index 00000000000..01e518b6df7
--- /dev/null
+++ b/python/ql/src/experimental/Security/NEW/CWE-611/Xxe.ql
@@ -0,0 +1,23 @@
+/**
+ * @name XML external entity expansion
+ * @description Parsing user input as an XML document with external
+ * entity expansion is vulnerable to XXE attacks.
+ * @kind path-problem
+ * @problem.severity error
+ * @security-severity 9.1
+ * @precision high
+ * @id js/xxe
+ * @tags security
+ * external/cwe/cwe-611
+ * external/cwe/cwe-827
+ */
+
+import javascript
+import semmle.javascript.security.dataflow.XxeQuery
+import DataFlow::PathGraph
+
+from Configuration cfg, DataFlow::PathNode source, DataFlow::PathNode sink
+where cfg.hasFlowPath(source, sink)
+select sink.getNode(), source, sink,
+ "A $@ is parsed as XML without guarding against external entity expansion.", source.getNode(),
+ "user-provided value"
diff --git a/python/ql/src/experimental/Security/NEW/CWE-611/examples/Xxe.js b/python/ql/src/experimental/Security/NEW/CWE-611/examples/Xxe.js
new file mode 100644
index 00000000000..99fa02cc42f
--- /dev/null
+++ b/python/ql/src/experimental/Security/NEW/CWE-611/examples/Xxe.js
@@ -0,0 +1,7 @@
+const app = require("express")(),
+ libxml = require("libxmljs");
+
+app.post("upload", (req, res) => {
+ let xmlSrc = req.body,
+ doc = libxml.parseXml(xmlSrc, { noent: true });
+});
diff --git a/python/ql/src/experimental/Security/NEW/CWE-611/examples/XxeGood.js b/python/ql/src/experimental/Security/NEW/CWE-611/examples/XxeGood.js
new file mode 100644
index 00000000000..8317dcac98f
--- /dev/null
+++ b/python/ql/src/experimental/Security/NEW/CWE-611/examples/XxeGood.js
@@ -0,0 +1,7 @@
+const app = require("express")(),
+ libxml = require("libxmljs");
+
+app.post("upload", (req, res) => {
+ let xmlSrc = req.body,
+ doc = libxml.parseXml(xmlSrc);
+});
diff --git a/python/ql/src/experimental/Security/NEW/CWE-776/XmlBomb.qhelp b/python/ql/src/experimental/Security/NEW/CWE-776/XmlBomb.qhelp
new file mode 100644
index 00000000000..c0714b3f96f
--- /dev/null
+++ b/python/ql/src/experimental/Security/NEW/CWE-776/XmlBomb.qhelp
@@ -0,0 +1,60 @@
+
+
+
+
+
+Parsing untrusted XML files with a weakly configured XML parser may be vulnerable to
+denial-of-service (DoS) attacks exploiting uncontrolled internal entity expansion.
+
+
+In XML, so-called internal entities are a mechanism for introducing an abbreviation
+for a piece of text or part of a document. When a parser that has been configured
+to expand entities encounters a reference to an internal entity, it replaces the entity
+by the data it represents. The replacement text may itself contain other entity references,
+which are expanded recursively. This means that entity expansion can increase document size
+dramatically.
+
+
+If untrusted XML is parsed with entity expansion enabled, a malicious attacker could
+submit a document that contains very deeply nested entity definitions, causing the parser
+to take a very long time or use large amounts of memory. This is sometimes called an
+XML bomb attack.
+
+
+
+
+
+The safest way to prevent XML bomb attacks is to disable entity expansion when parsing untrusted
+data. How this is done depends on the library being used. Note that some libraries, such as
+recent versions of libxmljs (though not its SAX parser API), disable entity expansion
+by default, so unless you have explicitly enabled entity expansion, no further action is needed.
+
+
+
+
+
+The following example uses the XML parser provided by the node-expat package to
+parse a string xmlSrc. If that string is from an untrusted source, this code may be
+vulnerable to a DoS attack, since node-expat expands internal entities by default:
+
+
+
+
+At the time of writing, node-expat does not provide a way of controlling entity
+expansion, but the example could be rewritten to use the sax package instead,
+which only expands standard entities such as &:
+
+
+
+
+
+
+Wikipedia:
+Billion Laughs .
+
+
+Bryan Sullivan:
+Security Briefs - XML Denial of Service Attacks and Defenses .
+
+
+
diff --git a/python/ql/src/experimental/Security/NEW/CWE-776/XmlBomb.ql b/python/ql/src/experimental/Security/NEW/CWE-776/XmlBomb.ql
new file mode 100644
index 00000000000..c340eee68cc
--- /dev/null
+++ b/python/ql/src/experimental/Security/NEW/CWE-776/XmlBomb.ql
@@ -0,0 +1,23 @@
+/**
+ * @name XML internal entity expansion
+ * @description Parsing user input as an XML document with arbitrary internal
+ * entity expansion is vulnerable to denial-of-service attacks.
+ * @kind path-problem
+ * @problem.severity warning
+ * @security-severity 7.5
+ * @precision high
+ * @id js/xml-bomb
+ * @tags security
+ * external/cwe/cwe-776
+ * external/cwe/cwe-400
+ */
+
+import javascript
+import semmle.javascript.security.dataflow.XmlBombQuery
+import DataFlow::PathGraph
+
+from Configuration cfg, DataFlow::PathNode source, DataFlow::PathNode sink
+where cfg.hasFlowPath(source, sink)
+select sink.getNode(), source, sink,
+ "A $@ is parsed as XML without guarding against uncontrolled entity expansion.", source.getNode(),
+ "user-provided value"
diff --git a/python/ql/src/experimental/Security/NEW/CWE-776/examples/XmlBomb.js b/python/ql/src/experimental/Security/NEW/CWE-776/examples/XmlBomb.js
new file mode 100644
index 00000000000..f72902a5304
--- /dev/null
+++ b/python/ql/src/experimental/Security/NEW/CWE-776/examples/XmlBomb.js
@@ -0,0 +1,10 @@
+const app = require("express")(),
+ expat = require("node-expat");
+
+app.post("upload", (req, res) => {
+ let xmlSrc = req.body,
+ parser = new expat.Parser();
+ parser.on("startElement", handleStart);
+ parser.on("text", handleText);
+ parser.write(xmlSrc);
+});
diff --git a/python/ql/src/experimental/Security/NEW/CWE-776/examples/XmlBombGood.js b/python/ql/src/experimental/Security/NEW/CWE-776/examples/XmlBombGood.js
new file mode 100644
index 00000000000..a8c5bc97e63
--- /dev/null
+++ b/python/ql/src/experimental/Security/NEW/CWE-776/examples/XmlBombGood.js
@@ -0,0 +1,10 @@
+const app = require("express")(),
+ sax = require("sax");
+
+app.post("upload", (req, res) => {
+ let xmlSrc = req.body,
+ parser = sax.parser(true);
+ parser.onopentag = handleStart;
+ parser.ontext = handleText;
+ parser.write(xmlSrc);
+});
diff --git a/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombCustomizations.qll b/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombCustomizations.qll
new file mode 100644
index 00000000000..1d159b057ad
--- /dev/null
+++ b/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombCustomizations.qll
@@ -0,0 +1,49 @@
+/**
+ * Provides default sources, sinks and sanitizers for reasoning about
+ * XML-bomb vulnerabilities, as well as extension points for adding
+ * your own.
+ */
+
+import javascript
+import semmle.javascript.security.dataflow.DOM
+
+module XmlBomb {
+ /**
+ * A data flow source for XML-bomb vulnerabilities.
+ */
+ abstract class Source extends DataFlow::Node { }
+
+ /**
+ * A data flow sink for XML-bomb vulnerabilities.
+ */
+ abstract class Sink extends DataFlow::Node { }
+
+ /**
+ * A sanitizer for XML-bomb vulnerabilities.
+ */
+ abstract class Sanitizer extends DataFlow::Node { }
+
+ /** A source of remote user input, considered as a flow source for XML bomb vulnerabilities. */
+ class RemoteFlowSourceAsSource extends Source {
+ RemoteFlowSourceAsSource() { this instanceof RemoteFlowSource }
+ }
+
+ /**
+ * An access to `document.location`, considered as a flow source for XML bomb vulnerabilities.
+ */
+ class LocationAsSource extends Source, DataFlow::ValueNode {
+ LocationAsSource() { isLocation(astNode) }
+ }
+
+ /**
+ * A call to an XML parser that performs internal entity expansion, viewed
+ * as a data flow sink for XML-bomb vulnerabilities.
+ */
+ class XmlParsingWithEntityResolution extends Sink, DataFlow::ValueNode {
+ XmlParsingWithEntityResolution() {
+ exists(XML::ParserInvocation parse | astNode = parse.getSourceArgument() |
+ parse.resolvesEntities(XML::InternalEntity())
+ )
+ }
+ }
+}
diff --git a/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombQuery.qll b/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombQuery.qll
new file mode 100644
index 00000000000..951b927f86e
--- /dev/null
+++ b/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombQuery.qll
@@ -0,0 +1,27 @@
+/**
+ * Provides a taint tracking configuration for reasoning about
+ * XML-bomb vulnerabilities.
+ *
+ * Note, for performance reasons: only import this file if
+ * `XmlBomb::Configuration` is needed, otherwise
+ * `XmlBombCustomizations` should be imported instead.
+ */
+
+import javascript
+import XmlBombCustomizations::XmlBomb
+
+/**
+ * A taint-tracking configuration for reasoning about XML-bomb vulnerabilities.
+ */
+class Configuration extends TaintTracking::Configuration {
+ Configuration() { this = "XmlBomb" }
+
+ override predicate isSource(DataFlow::Node source) { source instanceof Source }
+
+ override predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
+
+ override predicate isSanitizer(DataFlow::Node node) {
+ super.isSanitizer(node) or
+ node instanceof Sanitizer
+ }
+}
diff --git a/python/ql/src/experimental/semmle/python/security/dataflow/XxeCustomizations.qll b/python/ql/src/experimental/semmle/python/security/dataflow/XxeCustomizations.qll
new file mode 100644
index 00000000000..4e7bb5e730c
--- /dev/null
+++ b/python/ql/src/experimental/semmle/python/security/dataflow/XxeCustomizations.qll
@@ -0,0 +1,52 @@
+/**
+ * Provides default sources, sinks and sanitizers for reasoning about
+ * XML External Entity (XXE) vulnerabilities, as well as extension
+ * points for adding your own.
+ */
+
+import javascript
+import semmle.javascript.security.dataflow.DOM
+
+module Xxe {
+ /**
+ * A data flow source for XXE vulnerabilities.
+ */
+ abstract class Source extends DataFlow::Node { }
+
+ /**
+ * A data flow sink for XXE vulnerabilities.
+ */
+ abstract class Sink extends DataFlow::Node { }
+
+ /**
+ * A sanitizer for XXE vulnerabilities.
+ */
+ abstract class Sanitizer extends DataFlow::Node { }
+
+ /** A source of remote user input, considered as a flow source for XXE vulnerabilities. */
+ class RemoteFlowSourceAsSource extends Source {
+ RemoteFlowSourceAsSource() { this instanceof RemoteFlowSource }
+ }
+
+ /**
+ * An access to `document.location`, considered as a flow source for XXE vulnerabilities.
+ */
+ class LocationAsSource extends Source, DataFlow::ValueNode {
+ LocationAsSource() { isLocation(astNode) }
+ }
+
+ /**
+ * A call to an XML parser that performs external entity expansion, viewed
+ * as a data flow sink for XXE vulnerabilities.
+ */
+ class XmlParsingWithExternalEntityResolution extends Sink, DataFlow::ValueNode {
+ XmlParsingWithExternalEntityResolution() {
+ exists(XML::ParserInvocation parse | astNode = parse.getSourceArgument() |
+ parse.resolvesEntities(XML::ExternalEntity(_))
+ or
+ parse.resolvesEntities(XML::ParameterEntity(true)) and
+ parse.resolvesEntities(XML::InternalEntity())
+ )
+ }
+ }
+}
diff --git a/python/ql/src/experimental/semmle/python/security/dataflow/XxeQuery.qll b/python/ql/src/experimental/semmle/python/security/dataflow/XxeQuery.qll
new file mode 100644
index 00000000000..82d3fb4f6cc
--- /dev/null
+++ b/python/ql/src/experimental/semmle/python/security/dataflow/XxeQuery.qll
@@ -0,0 +1,27 @@
+/**
+ * Provides a taint tracking configuration for reasoning about XML
+ * External Entity (XXE) vulnerabilities.
+ *
+ * Note, for performance reasons: only import this file if
+ * `Xxe::Configuration` is needed, otherwise `XxeCustomizations`
+ * should be imported instead.
+ */
+
+import javascript
+import XxeCustomizations::Xxe
+
+/**
+ * A taint-tracking configuration for reasoning about XXE vulnerabilities.
+ */
+class Configuration extends TaintTracking::Configuration {
+ Configuration() { this = "Xxe" }
+
+ override predicate isSource(DataFlow::Node source) { source instanceof Source }
+
+ override predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
+
+ override predicate isSanitizer(DataFlow::Node node) {
+ super.isSanitizer(node) or
+ node instanceof Sanitizer
+ }
+}
From e45f9d69ccb44a2109518f3c8334e21f5c193a43 Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Thu, 24 Mar 2022 14:15:54 +0100
Subject: [PATCH 044/171] Python: Adjust Xxe/XmlBomb for Python
I changed a few QLdocs so they fit the style we have used in Python...
although I surely do regret having introduced a new style for how these
QLDocs look :D
---
.../experimental/Security/NEW/CWE-611/Xxe.ql | 6 ++--
.../Security/NEW/CWE-776/XmlBomb.ql | 6 ++--
.../dataflow/XmlBombCustomizations.qll | 31 +++++++++--------
.../python/security/dataflow/XmlBombQuery.qll | 11 +++---
.../security/dataflow/XxeCustomizations.qll | 34 +++++++++----------
.../python/security/dataflow/XxeQuery.qll | 13 +++----
6 files changed, 51 insertions(+), 50 deletions(-)
diff --git a/python/ql/src/experimental/Security/NEW/CWE-611/Xxe.ql b/python/ql/src/experimental/Security/NEW/CWE-611/Xxe.ql
index 01e518b6df7..f706ea6e909 100644
--- a/python/ql/src/experimental/Security/NEW/CWE-611/Xxe.ql
+++ b/python/ql/src/experimental/Security/NEW/CWE-611/Xxe.ql
@@ -6,14 +6,14 @@
* @problem.severity error
* @security-severity 9.1
* @precision high
- * @id js/xxe
+ * @id py/xxe
* @tags security
* external/cwe/cwe-611
* external/cwe/cwe-827
*/
-import javascript
-import semmle.javascript.security.dataflow.XxeQuery
+import python
+import experimental.semmle.python.security.dataflow.XxeQuery
import DataFlow::PathGraph
from Configuration cfg, DataFlow::PathNode source, DataFlow::PathNode sink
diff --git a/python/ql/src/experimental/Security/NEW/CWE-776/XmlBomb.ql b/python/ql/src/experimental/Security/NEW/CWE-776/XmlBomb.ql
index c340eee68cc..2a1ea5916c4 100644
--- a/python/ql/src/experimental/Security/NEW/CWE-776/XmlBomb.ql
+++ b/python/ql/src/experimental/Security/NEW/CWE-776/XmlBomb.ql
@@ -6,14 +6,14 @@
* @problem.severity warning
* @security-severity 7.5
* @precision high
- * @id js/xml-bomb
+ * @id py/xml-bomb
* @tags security
* external/cwe/cwe-776
* external/cwe/cwe-400
*/
-import javascript
-import semmle.javascript.security.dataflow.XmlBombQuery
+import python
+import experimental.semmle.python.security.dataflow.XmlBombQuery
import DataFlow::PathGraph
from Configuration cfg, DataFlow::PathNode source, DataFlow::PathNode sink
diff --git a/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombCustomizations.qll b/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombCustomizations.qll
index 1d159b057ad..66a16a4494a 100644
--- a/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombCustomizations.qll
+++ b/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombCustomizations.qll
@@ -1,12 +1,18 @@
/**
- * Provides default sources, sinks and sanitizers for reasoning about
- * XML-bomb vulnerabilities, as well as extension points for adding
- * your own.
+ * Provides default sources, sinks and sanitizers for detecting
+ * "XML bomb"
+ * vulnerabilities, as well as extension points for adding your own.
*/
-import javascript
-import semmle.javascript.security.dataflow.DOM
+private import python
+private import semmle.python.dataflow.new.DataFlow
+private import experimental.semmle.python.Concepts
+private import semmle.python.dataflow.new.RemoteFlowSources
+/**
+ * Provides default sources, sinks and sanitizers for detecting "XML bomb"
+ * vulnerabilities, as well as extension points for adding your own.
+ */
module XmlBomb {
/**
* A data flow source for XML-bomb vulnerabilities.
@@ -28,21 +34,16 @@ module XmlBomb {
RemoteFlowSourceAsSource() { this instanceof RemoteFlowSource }
}
- /**
- * An access to `document.location`, considered as a flow source for XML bomb vulnerabilities.
- */
- class LocationAsSource extends Source, DataFlow::ValueNode {
- LocationAsSource() { isLocation(astNode) }
- }
-
/**
* A call to an XML parser that performs internal entity expansion, viewed
* as a data flow sink for XML-bomb vulnerabilities.
*/
- class XmlParsingWithEntityResolution extends Sink, DataFlow::ValueNode {
+ class XmlParsingWithEntityResolution extends Sink {
XmlParsingWithEntityResolution() {
- exists(XML::ParserInvocation parse | astNode = parse.getSourceArgument() |
- parse.resolvesEntities(XML::InternalEntity())
+ exists(ExperimentalXML::XMLParsing parsing, ExperimentalXML::XMLVulnerabilityKind kind |
+ (kind.isBillionLaughs() or kind.isQuadraticBlowup()) and
+ parsing.vulnerableTo(kind) and
+ this = parsing.getAnInput()
)
}
}
diff --git a/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombQuery.qll b/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombQuery.qll
index 951b927f86e..d0c0b85d84f 100644
--- a/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombQuery.qll
+++ b/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombQuery.qll
@@ -1,17 +1,18 @@
/**
- * Provides a taint tracking configuration for reasoning about
- * XML-bomb vulnerabilities.
+ * Provides a taint-tracking configuration for detecting "XML bomb" vulnerabilities.
*
* Note, for performance reasons: only import this file if
- * `XmlBomb::Configuration` is needed, otherwise
+ * `Configuration` is needed, otherwise
* `XmlBombCustomizations` should be imported instead.
*/
-import javascript
+import python
+import semmle.python.dataflow.new.DataFlow
+import semmle.python.dataflow.new.TaintTracking
import XmlBombCustomizations::XmlBomb
/**
- * A taint-tracking configuration for reasoning about XML-bomb vulnerabilities.
+ * A taint-tracking configuration for detecting "XML bomb" vulnerabilities.
*/
class Configuration extends TaintTracking::Configuration {
Configuration() { this = "XmlBomb" }
diff --git a/python/ql/src/experimental/semmle/python/security/dataflow/XxeCustomizations.qll b/python/ql/src/experimental/semmle/python/security/dataflow/XxeCustomizations.qll
index 4e7bb5e730c..b2992dd335f 100644
--- a/python/ql/src/experimental/semmle/python/security/dataflow/XxeCustomizations.qll
+++ b/python/ql/src/experimental/semmle/python/security/dataflow/XxeCustomizations.qll
@@ -1,12 +1,18 @@
/**
- * Provides default sources, sinks and sanitizers for reasoning about
- * XML External Entity (XXE) vulnerabilities, as well as extension
- * points for adding your own.
+ * Provides default sources, sinks and sanitizers for detecting
+ * "XML External Entity (XXE)"
+ * vulnerabilities, as well as extension points for adding your own.
*/
-import javascript
-import semmle.javascript.security.dataflow.DOM
+private import python
+private import semmle.python.dataflow.new.DataFlow
+private import experimental.semmle.python.Concepts
+private import semmle.python.dataflow.new.RemoteFlowSources
+/**
+ * Provides default sources, sinks and sanitizers for detecting "XML External Entity (XXE)"
+ * vulnerabilities, as well as extension points for adding your own.
+ */
module Xxe {
/**
* A data flow source for XXE vulnerabilities.
@@ -28,24 +34,16 @@ module Xxe {
RemoteFlowSourceAsSource() { this instanceof RemoteFlowSource }
}
- /**
- * An access to `document.location`, considered as a flow source for XXE vulnerabilities.
- */
- class LocationAsSource extends Source, DataFlow::ValueNode {
- LocationAsSource() { isLocation(astNode) }
- }
-
/**
* A call to an XML parser that performs external entity expansion, viewed
* as a data flow sink for XXE vulnerabilities.
*/
- class XmlParsingWithExternalEntityResolution extends Sink, DataFlow::ValueNode {
+ class XmlParsingWithExternalEntityResolution extends Sink {
XmlParsingWithExternalEntityResolution() {
- exists(XML::ParserInvocation parse | astNode = parse.getSourceArgument() |
- parse.resolvesEntities(XML::ExternalEntity(_))
- or
- parse.resolvesEntities(XML::ParameterEntity(true)) and
- parse.resolvesEntities(XML::InternalEntity())
+ exists(ExperimentalXML::XMLParsing parsing, ExperimentalXML::XMLVulnerabilityKind kind |
+ kind.isXxe() and
+ parsing.vulnerableTo(kind) and
+ this = parsing.getAnInput()
)
}
}
diff --git a/python/ql/src/experimental/semmle/python/security/dataflow/XxeQuery.qll b/python/ql/src/experimental/semmle/python/security/dataflow/XxeQuery.qll
index 82d3fb4f6cc..dd2409f2a3c 100644
--- a/python/ql/src/experimental/semmle/python/security/dataflow/XxeQuery.qll
+++ b/python/ql/src/experimental/semmle/python/security/dataflow/XxeQuery.qll
@@ -1,17 +1,18 @@
/**
- * Provides a taint tracking configuration for reasoning about XML
- * External Entity (XXE) vulnerabilities.
+ * Provides a taint-tracking configuration for detecting "XML External Entity (XXE)" vulnerabilities.
*
* Note, for performance reasons: only import this file if
- * `Xxe::Configuration` is needed, otherwise `XxeCustomizations`
- * should be imported instead.
+ * `Configuration` is needed, otherwise
+ * `XxeCustomizations` should be imported instead.
*/
-import javascript
+import python
+import semmle.python.dataflow.new.DataFlow
+import semmle.python.dataflow.new.TaintTracking
import XxeCustomizations::Xxe
/**
- * A taint-tracking configuration for reasoning about XXE vulnerabilities.
+ * A taint-tracking configuration for detecting "XML External Entity (XXE)" vulnerabilities.
*/
class Configuration extends TaintTracking::Configuration {
Configuration() { this = "Xxe" }
From 91795b857756a4912e6a280e4e53f65f4fbaf76a Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Thu, 24 Mar 2022 14:16:38 +0100
Subject: [PATCH 045/171] Python: Add simple test of Xxe/XmlBomb
Note that most of the testing happens in the framework specific tests,
with an inline-expectation test
---
.../Security/CWE-611-Xxe/Xxe.expected | 20 +++++++++++++
.../Security/CWE-611-Xxe/Xxe.qlref | 1 +
.../query-tests/Security/CWE-611-Xxe/test.py | 30 +++++++++++++++++++
.../Security/CWE-776-XmlBomb/XmlBomb.expected | 12 ++++++++
.../Security/CWE-776-XmlBomb/XmlBomb.qlref | 1 +
.../Security/CWE-776-XmlBomb/test.py | 30 +++++++++++++++++++
6 files changed, 94 insertions(+)
create mode 100644 python/ql/test/experimental/query-tests/Security/CWE-611-Xxe/Xxe.expected
create mode 100644 python/ql/test/experimental/query-tests/Security/CWE-611-Xxe/Xxe.qlref
create mode 100644 python/ql/test/experimental/query-tests/Security/CWE-611-Xxe/test.py
create mode 100644 python/ql/test/experimental/query-tests/Security/CWE-776-XmlBomb/XmlBomb.expected
create mode 100644 python/ql/test/experimental/query-tests/Security/CWE-776-XmlBomb/XmlBomb.qlref
create mode 100644 python/ql/test/experimental/query-tests/Security/CWE-776-XmlBomb/test.py
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-611-Xxe/Xxe.expected b/python/ql/test/experimental/query-tests/Security/CWE-611-Xxe/Xxe.expected
new file mode 100644
index 00000000000..004369d79cf
--- /dev/null
+++ b/python/ql/test/experimental/query-tests/Security/CWE-611-Xxe/Xxe.expected
@@ -0,0 +1,20 @@
+edges
+| test.py:8:19:8:25 | ControlFlowNode for request | test.py:8:19:8:30 | ControlFlowNode for Attribute |
+| test.py:8:19:8:30 | ControlFlowNode for Attribute | test.py:8:19:8:45 | ControlFlowNode for Subscript |
+| test.py:8:19:8:45 | ControlFlowNode for Subscript | test.py:9:34:9:44 | ControlFlowNode for xml_content |
+| test.py:19:19:19:25 | ControlFlowNode for request | test.py:19:19:19:30 | ControlFlowNode for Attribute |
+| test.py:19:19:19:30 | ControlFlowNode for Attribute | test.py:19:19:19:45 | ControlFlowNode for Subscript |
+| test.py:19:19:19:45 | ControlFlowNode for Subscript | test.py:30:34:30:44 | ControlFlowNode for xml_content |
+nodes
+| test.py:8:19:8:25 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
+| test.py:8:19:8:30 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
+| test.py:8:19:8:45 | ControlFlowNode for Subscript | semmle.label | ControlFlowNode for Subscript |
+| test.py:9:34:9:44 | ControlFlowNode for xml_content | semmle.label | ControlFlowNode for xml_content |
+| test.py:19:19:19:25 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
+| test.py:19:19:19:30 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
+| test.py:19:19:19:45 | ControlFlowNode for Subscript | semmle.label | ControlFlowNode for Subscript |
+| test.py:30:34:30:44 | ControlFlowNode for xml_content | semmle.label | ControlFlowNode for xml_content |
+subpaths
+#select
+| test.py:9:34:9:44 | ControlFlowNode for xml_content | test.py:8:19:8:25 | ControlFlowNode for request | test.py:9:34:9:44 | ControlFlowNode for xml_content | A $@ is parsed as XML without guarding against external entity expansion. | test.py:8:19:8:25 | ControlFlowNode for request | user-provided value |
+| test.py:30:34:30:44 | ControlFlowNode for xml_content | test.py:19:19:19:25 | ControlFlowNode for request | test.py:30:34:30:44 | ControlFlowNode for xml_content | A $@ is parsed as XML without guarding against external entity expansion. | test.py:19:19:19:25 | ControlFlowNode for request | user-provided value |
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-611-Xxe/Xxe.qlref b/python/ql/test/experimental/query-tests/Security/CWE-611-Xxe/Xxe.qlref
new file mode 100644
index 00000000000..f8a07d7d2ee
--- /dev/null
+++ b/python/ql/test/experimental/query-tests/Security/CWE-611-Xxe/Xxe.qlref
@@ -0,0 +1 @@
+experimental/Security/NEW/CWE-611/Xxe.ql
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-611-Xxe/test.py b/python/ql/test/experimental/query-tests/Security/CWE-611-Xxe/test.py
new file mode 100644
index 00000000000..d9181c4cf34
--- /dev/null
+++ b/python/ql/test/experimental/query-tests/Security/CWE-611-Xxe/test.py
@@ -0,0 +1,30 @@
+from flask import Flask, request
+import lxml.etree
+
+app = Flask(__name__)
+
+@app.route("/vuln-handler")
+def vuln_handler():
+ xml_content = request.args['xml_content']
+ return lxml.etree.fromstring(xml_content).text
+
+@app.route("/safe-handler")
+def safe_handler():
+ xml_content = request.args['xml_content']
+ parser = lxml.etree.XMLParser(resolve_entities=False)
+ return lxml.etree.fromstring(xml_content, parser=parser).text
+
+@app.route("/super-vuln-handler")
+def super_vuln_handler():
+ xml_content = request.args['xml_content']
+ parser = lxml.etree.XMLParser(
+ # allows XXE
+ resolve_entities=True,
+ # allows remote XXE
+ no_network=False,
+ # together with `no_network=False`, allows DTD-retrival
+ load_dtd=True,
+ # allows DoS attacks
+ huge_tree=True,
+ )
+ return lxml.etree.fromstring(xml_content, parser=parser).text
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-776-XmlBomb/XmlBomb.expected b/python/ql/test/experimental/query-tests/Security/CWE-776-XmlBomb/XmlBomb.expected
new file mode 100644
index 00000000000..15c439d0761
--- /dev/null
+++ b/python/ql/test/experimental/query-tests/Security/CWE-776-XmlBomb/XmlBomb.expected
@@ -0,0 +1,12 @@
+edges
+| test.py:19:19:19:25 | ControlFlowNode for request | test.py:19:19:19:30 | ControlFlowNode for Attribute |
+| test.py:19:19:19:30 | ControlFlowNode for Attribute | test.py:19:19:19:45 | ControlFlowNode for Subscript |
+| test.py:19:19:19:45 | ControlFlowNode for Subscript | test.py:30:34:30:44 | ControlFlowNode for xml_content |
+nodes
+| test.py:19:19:19:25 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
+| test.py:19:19:19:30 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
+| test.py:19:19:19:45 | ControlFlowNode for Subscript | semmle.label | ControlFlowNode for Subscript |
+| test.py:30:34:30:44 | ControlFlowNode for xml_content | semmle.label | ControlFlowNode for xml_content |
+subpaths
+#select
+| test.py:30:34:30:44 | ControlFlowNode for xml_content | test.py:19:19:19:25 | ControlFlowNode for request | test.py:30:34:30:44 | ControlFlowNode for xml_content | A $@ is parsed as XML without guarding against uncontrolled entity expansion. | test.py:19:19:19:25 | ControlFlowNode for request | user-provided value |
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-776-XmlBomb/XmlBomb.qlref b/python/ql/test/experimental/query-tests/Security/CWE-776-XmlBomb/XmlBomb.qlref
new file mode 100644
index 00000000000..5eadbb1f26f
--- /dev/null
+++ b/python/ql/test/experimental/query-tests/Security/CWE-776-XmlBomb/XmlBomb.qlref
@@ -0,0 +1 @@
+experimental/Security/NEW/CWE-776/XmlBomb.ql
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-776-XmlBomb/test.py b/python/ql/test/experimental/query-tests/Security/CWE-776-XmlBomb/test.py
new file mode 100644
index 00000000000..d9181c4cf34
--- /dev/null
+++ b/python/ql/test/experimental/query-tests/Security/CWE-776-XmlBomb/test.py
@@ -0,0 +1,30 @@
+from flask import Flask, request
+import lxml.etree
+
+app = Flask(__name__)
+
+@app.route("/vuln-handler")
+def vuln_handler():
+ xml_content = request.args['xml_content']
+ return lxml.etree.fromstring(xml_content).text
+
+@app.route("/safe-handler")
+def safe_handler():
+ xml_content = request.args['xml_content']
+ parser = lxml.etree.XMLParser(resolve_entities=False)
+ return lxml.etree.fromstring(xml_content, parser=parser).text
+
+@app.route("/super-vuln-handler")
+def super_vuln_handler():
+ xml_content = request.args['xml_content']
+ parser = lxml.etree.XMLParser(
+ # allows XXE
+ resolve_entities=True,
+ # allows remote XXE
+ no_network=False,
+ # together with `no_network=False`, allows DTD-retrival
+ load_dtd=True,
+ # allows DoS attacks
+ huge_tree=True,
+ )
+ return lxml.etree.fromstring(xml_content, parser=parser).text
From a1d88e39a77f4c16ca0e292ca5e6311828745b2e Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Thu, 24 Mar 2022 15:36:20 +0100
Subject: [PATCH 046/171] Python: Adjust XXE PoC for newer lxml versions
Which doesn't raise that syntax error (at least not on my laptop)
---
.../experimental/library-tests/frameworks/XML/poc/PoC.py | 6 +-----
1 file changed, 1 insertion(+), 5 deletions(-)
diff --git a/python/ql/test/experimental/library-tests/frameworks/XML/poc/PoC.py b/python/ql/test/experimental/library-tests/frameworks/XML/poc/PoC.py
index adcace1aa0a..77d6c032683 100644
--- a/python/ql/test/experimental/library-tests/frameworks/XML/poc/PoC.py
+++ b/python/ql/test/experimental/library-tests/frameworks/XML/poc/PoC.py
@@ -361,11 +361,7 @@ class TestLxml:
hit_xxe = False
parser = lxml.etree.XMLParser()
- try:
- root = lxml.etree.fromstring(remote_xxe, parser=parser)
- assert False
- except lxml.etree.XMLSyntaxError as e:
- assert "Failure to process entity remote_xxe" in str(e)
+ root = lxml.etree.fromstring(remote_xxe, parser=parser)
assert hit_xxe == False
@staticmethod
From 57b97804283545dbe986c019660ae5171ba8e7ed Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Thu, 24 Mar 2022 15:37:14 +0100
Subject: [PATCH 047/171] Python: XXE: Add example of exfiltrating data through
dtd-retrival
---
.../library-tests/frameworks/XML/poc/PoC.py | 32 ++++++++++++++++++-
.../library-tests/frameworks/XML/poc/flag | 2 +-
2 files changed, 32 insertions(+), 2 deletions(-)
diff --git a/python/ql/test/experimental/library-tests/frameworks/XML/poc/PoC.py b/python/ql/test/experimental/library-tests/frameworks/XML/poc/PoC.py
index 77d6c032683..b4cb2faf304 100644
--- a/python/ql/test/experimental/library-tests/frameworks/XML/poc/PoC.py
+++ b/python/ql/test/experimental/library-tests/frameworks/XML/poc/PoC.py
@@ -70,6 +70,10 @@ dtd_retrieval = f"""
bar
"""
+exfiltrate_through_dtd_retrieval = f"""
+ %xxe; ]>
+"""
+
# ==============================================================================
# other setup
@@ -95,6 +99,22 @@ def test_xxe():
hit_xxe = True
return "ok"
+@app.route("/exfiltrate-through.dtd")
+def exfiltrate_through_dtd():
+ return f"""
+">
+%eval;
+%exfiltrate;
+ """
+
+exfiltrated_data = None
+@app.route("/exfiltrate-data")
+def exfiltrate_data():
+ from flask import request
+ global exfiltrated_data
+ exfiltrated_data = request.args["data"]
+ return "ok"
+
def run_app():
app.run(host=HOST, port=PORT)
@@ -346,7 +366,7 @@ class TestLxml:
parser = lxml.etree.XMLParser()
root = lxml.etree.fromstring(local_xxe, parser=parser)
assert root.tag == "test"
- assert root.text == "SECRET_FLAG\n", root.text
+ assert root.text == "SECRET_FLAG", root.text
@staticmethod
def test_local_xxe_disabled():
@@ -412,6 +432,16 @@ class TestLxml:
pass
assert hit_dtd == False
+ @staticmethod
+ def test_exfiltrate_through_dtd():
+ # note that this only works when the data to exfiltrate does not contain a newline :|
+ global exfiltrated_data
+ exfiltrated_data = None
+ parser = lxml.etree.XMLParser(load_dtd=True, no_network=False)
+ with pytest.raises(lxml.etree.XMLSyntaxError):
+ lxml.etree.fromstring(exfiltrate_through_dtd_retrieval, parser=parser)
+
+ assert exfiltrated_data == "SECRET_FLAG"
# ==============================================================================
diff --git a/python/ql/test/experimental/library-tests/frameworks/XML/poc/flag b/python/ql/test/experimental/library-tests/frameworks/XML/poc/flag
index 45c9436ee9f..b8bd6838774 100644
--- a/python/ql/test/experimental/library-tests/frameworks/XML/poc/flag
+++ b/python/ql/test/experimental/library-tests/frameworks/XML/poc/flag
@@ -1 +1 @@
-SECRET_FLAG
+SECRET_FLAG
\ No newline at end of file
From 769f5691d08dd8288e4eb6432e163b0a53c8ac21 Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Tue, 29 Mar 2022 17:18:06 +0200
Subject: [PATCH 048/171] Python: Add taint for `StringIO` and `BytesIO`
---
.../2022-03-29-add-taint-for-StringIO.md | 4 ++
.../lib/semmle/python/frameworks/Stdlib.qll | 58 +++++++++++++++++++
.../frameworks/stdlib/io_test.py | 47 +++++++++++++++
3 files changed, 109 insertions(+)
create mode 100644 python/ql/lib/change-notes/2022-03-29-add-taint-for-StringIO.md
create mode 100644 python/ql/test/library-tests/frameworks/stdlib/io_test.py
diff --git a/python/ql/lib/change-notes/2022-03-29-add-taint-for-StringIO.md b/python/ql/lib/change-notes/2022-03-29-add-taint-for-StringIO.md
new file mode 100644
index 00000000000..7857e6f9ca6
--- /dev/null
+++ b/python/ql/lib/change-notes/2022-03-29-add-taint-for-StringIO.md
@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Added taint propagation for `io.StringIO` and `io.BytesIO`. This addition was originally [submitted as part of an experimental query by @jorgectf](https://github.com/github/codeql/pull/6112).
diff --git a/python/ql/lib/semmle/python/frameworks/Stdlib.qll b/python/ql/lib/semmle/python/frameworks/Stdlib.qll
index 22dce5427ae..234a8802f0f 100644
--- a/python/ql/lib/semmle/python/frameworks/Stdlib.qll
+++ b/python/ql/lib/semmle/python/frameworks/Stdlib.qll
@@ -3116,6 +3116,64 @@ private module StdlibPrivate {
result in [this.getArg(0), this.getArgByName("path")]
}
}
+
+ // ---------------------------------------------------------------------------
+ // io
+ // ---------------------------------------------------------------------------
+ /**
+ * Provides models for the `io.StringIO`/`io.BytesIO` classes
+ *
+ * See https://docs.python.org/3.10/library/io.html#io.StringIO.
+ */
+ module StringIO {
+ /** Gets a reference to the `io.StringIO` class. */
+ private API::Node classRef() {
+ result = API::moduleImport("io").getMember(["StringIO", "BytesIO"])
+ }
+
+ /**
+ * A source of instances of `io.StringIO`/`io.BytesIO`, extend this class to model new instances.
+ *
+ * This can include instantiations of the class, return values from function
+ * calls, or a special parameter that will be set when functions are called by an external
+ * library.
+ *
+ * Use the predicate `StringIO::instance()` to get references to instances of `io.StringIO`.
+ */
+ abstract class InstanceSource extends Stdlib::FileLikeObject::InstanceSource { }
+
+ /** A direct instantiation of `io.StringIO`/`io.BytesIO`. */
+ private class ClassInstantiation extends InstanceSource, DataFlow::CallCfgNode {
+ ClassInstantiation() { this = classRef().getACall() }
+
+ DataFlow::Node getInitialValue() {
+ result = this.getArg(0)
+ or
+ // `initial_value` for StringIO, `initial_bytes` for BytesIO
+ result = this.getArgByName(["initial_value", "initial_bytes"])
+ }
+ }
+
+ /** Gets a reference to an instance of `io.StringIO`/`io.BytesIO`. */
+ private DataFlow::TypeTrackingNode instance(DataFlow::TypeTracker t) {
+ t.start() and
+ result instanceof InstanceSource
+ or
+ exists(DataFlow::TypeTracker t2 | result = instance(t2).track(t2, t))
+ }
+
+ /** Gets a reference to an instance of `io.StringIO`/`io.BytesIO`. */
+ DataFlow::Node instance() { instance(DataFlow::TypeTracker::end()).flowsTo(result) }
+
+ /**
+ * Extra taint propagation for `io.StringIO`/`io.BytesIO`.
+ */
+ private class AdditionalTaintStep extends TaintTracking::AdditionalTaintStep {
+ override predicate step(DataFlow::Node nodeFrom, DataFlow::Node nodeTo) {
+ nodeTo.(ClassInstantiation).getInitialValue() = nodeFrom
+ }
+ }
+ }
}
// ---------------------------------------------------------------------------
diff --git a/python/ql/test/library-tests/frameworks/stdlib/io_test.py b/python/ql/test/library-tests/frameworks/stdlib/io_test.py
new file mode 100644
index 00000000000..98d60445e1c
--- /dev/null
+++ b/python/ql/test/library-tests/frameworks/stdlib/io_test.py
@@ -0,0 +1,47 @@
+from io import StringIO, BytesIO
+
+TAINTED_STRING = "TS"
+TAINTED_BYTES = b"TB"
+
+def ensure_tainted(*args):
+ print("ensure_tainted")
+ for arg in args:
+ print("", repr(arg))
+
+
+def test_stringio():
+ ts = TAINTED_STRING
+
+ x = StringIO()
+ x.write(ts)
+ x.seek(0)
+
+ ensure_tainted(
+ StringIO(ts), # $ tainted
+ StringIO(initial_value=ts), # $ tainted
+ x, # $ tainted
+
+ x.read(), # $ tainted
+ StringIO(ts).read(), # $ tainted
+ )
+
+
+def test_bytesio():
+ tb = TAINTED_BYTES
+
+ x = BytesIO()
+ x.write(tb)
+ x.seek(0)
+
+ ensure_tainted(
+ BytesIO(tb), # $ tainted
+ BytesIO(initial_bytes=tb), # $ tainted
+ x, # $ tainted
+
+ x.read(), # $ tainted
+ BytesIO(tb).read(), # $ tainted
+ )
+
+
+test_stringio()
+test_bytesio()
From c3653378671f7e8c39c20993f16f64224945bf97 Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Tue, 29 Mar 2022 11:20:38 +0200
Subject: [PATCH 049/171] Python: Delete `XmlEntityInjection.ql`
Kept the test of SimpleXmlRpcServer, and kept the qhelp so it can be
used to write the new qhelp files
---
.../src/experimental/Security/CWE-611/XXE.xml | 4 -
.../Security/CWE-611/XmlEntityInjection.py | 25 ------
.../Security/CWE-611/XmlEntityInjection.ql | 31 -------
.../{CWE-611 => NEW}/XmlEntityInjection.qhelp | 0
.../security/dataflow/XmlEntityInjection.qll | 28 ------
.../XmlEntityInjectionCustomizations.qll | 86 -------------------
.../SimpleXmlRpcServer.expected | 0
.../SimpleXmlRpcServer.qlref | 0
.../xmlrpc_server.py | 0
.../CWE-611/XmlEntityInjection.expected | 27 ------
.../Security/CWE-611/XmlEntityInjection.qlref | 1 -
.../query-tests/Security/CWE-611/test.py | 30 -------
12 files changed, 232 deletions(-)
delete mode 100644 python/ql/src/experimental/Security/CWE-611/XXE.xml
delete mode 100644 python/ql/src/experimental/Security/CWE-611/XmlEntityInjection.py
delete mode 100644 python/ql/src/experimental/Security/CWE-611/XmlEntityInjection.ql
rename python/ql/src/experimental/Security/{CWE-611 => NEW}/XmlEntityInjection.qhelp (100%)
delete mode 100644 python/ql/src/experimental/semmle/python/security/dataflow/XmlEntityInjection.qll
delete mode 100644 python/ql/src/experimental/semmle/python/security/dataflow/XmlEntityInjectionCustomizations.qll
rename python/ql/test/experimental/query-tests/Security/{CWE-611 => CWE-611-SimpleXmlRpcServer}/SimpleXmlRpcServer.expected (100%)
rename python/ql/test/experimental/query-tests/Security/{CWE-611 => CWE-611-SimpleXmlRpcServer}/SimpleXmlRpcServer.qlref (100%)
rename python/ql/test/experimental/query-tests/Security/{CWE-611 => CWE-611-SimpleXmlRpcServer}/xmlrpc_server.py (100%)
delete mode 100644 python/ql/test/experimental/query-tests/Security/CWE-611/XmlEntityInjection.expected
delete mode 100644 python/ql/test/experimental/query-tests/Security/CWE-611/XmlEntityInjection.qlref
delete mode 100644 python/ql/test/experimental/query-tests/Security/CWE-611/test.py
diff --git a/python/ql/src/experimental/Security/CWE-611/XXE.xml b/python/ql/src/experimental/Security/CWE-611/XXE.xml
deleted file mode 100644
index ddd196f2f13..00000000000
--- a/python/ql/src/experimental/Security/CWE-611/XXE.xml
+++ /dev/null
@@ -1,4 +0,0 @@
-
-]>
-&xxe;
\ No newline at end of file
diff --git a/python/ql/src/experimental/Security/CWE-611/XmlEntityInjection.py b/python/ql/src/experimental/Security/CWE-611/XmlEntityInjection.py
deleted file mode 100644
index 0e9eec933d7..00000000000
--- a/python/ql/src/experimental/Security/CWE-611/XmlEntityInjection.py
+++ /dev/null
@@ -1,25 +0,0 @@
-from flask import request, Flask
-import lxml.etree
-import xml.etree.ElementTree
-
-app = Flask(__name__)
-
-# BAD
-@app.route("/bad")
-def bad():
- xml_content = request.args['xml_content']
-
- parser = lxml.etree.XMLParser()
- parsed_xml = xml.etree.ElementTree.fromstring(xml_content, parser=parser)
-
- return parsed_xml.text
-
-# GOOD
-@app.route("/good")
-def good():
- xml_content = request.args['xml_content']
-
- parser = lxml.etree.XMLParser(resolve_entities=False)
- parsed_xml = xml.etree.ElementTree.fromstring(xml_content, parser=parser)
-
- return parsed_xml.text
\ No newline at end of file
diff --git a/python/ql/src/experimental/Security/CWE-611/XmlEntityInjection.ql b/python/ql/src/experimental/Security/CWE-611/XmlEntityInjection.ql
deleted file mode 100644
index 922ca346b17..00000000000
--- a/python/ql/src/experimental/Security/CWE-611/XmlEntityInjection.ql
+++ /dev/null
@@ -1,31 +0,0 @@
-/**
- * @name XML Entity injection
- * @description User input should not be parsed allowing the injection of entities.
- * @kind path-problem
- * @problem.severity error
- * @id py/xml-entity-injection
- * @tags security
- * external/cwe/cwe-611
- * external/cwe/cwe-776
- * external/cwe/cwe-827
- */
-
-// determine precision above
-import python
-import experimental.semmle.python.security.dataflow.XmlEntityInjection
-import DataFlow::PathGraph
-
-from
- XmlEntityInjection::XmlEntityInjectionConfiguration config, DataFlow::PathNode source,
- DataFlow::PathNode sink, string kinds
-where
- config.hasFlowPath(source, sink) and
- kinds =
- strictconcat(string kind |
- kind = sink.getNode().(XmlEntityInjection::Sink).getVulnerableKind()
- |
- kind, ", "
- )
-select sink.getNode(), source, sink,
- "$@ XML input is constructed from a $@ and is vulnerable to: " + kinds + ".", sink.getNode(),
- "This", source.getNode(), "user-provided value"
diff --git a/python/ql/src/experimental/Security/CWE-611/XmlEntityInjection.qhelp b/python/ql/src/experimental/Security/NEW/XmlEntityInjection.qhelp
similarity index 100%
rename from python/ql/src/experimental/Security/CWE-611/XmlEntityInjection.qhelp
rename to python/ql/src/experimental/Security/NEW/XmlEntityInjection.qhelp
diff --git a/python/ql/src/experimental/semmle/python/security/dataflow/XmlEntityInjection.qll b/python/ql/src/experimental/semmle/python/security/dataflow/XmlEntityInjection.qll
deleted file mode 100644
index 35220e153d1..00000000000
--- a/python/ql/src/experimental/semmle/python/security/dataflow/XmlEntityInjection.qll
+++ /dev/null
@@ -1,28 +0,0 @@
-import python
-import experimental.semmle.python.Concepts
-import semmle.python.dataflow.new.DataFlow
-import semmle.python.dataflow.new.TaintTracking
-import semmle.python.dataflow.new.RemoteFlowSources
-import semmle.python.dataflow.new.BarrierGuards
-
-module XmlEntityInjection {
- import XmlEntityInjectionCustomizations::XmlEntityInjection
-
- class XmlEntityInjectionConfiguration extends TaintTracking::Configuration {
- XmlEntityInjectionConfiguration() { this = "XmlEntityInjectionConfiguration" }
-
- override predicate isSource(DataFlow::Node source) {
- source instanceof RemoteFlowSourceAsSource
- }
-
- override predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
-
- override predicate isSanitizerGuard(DataFlow::BarrierGuard guard) {
- guard instanceof SanitizerGuard
- }
-
- override predicate isAdditionalTaintStep(DataFlow::Node nodeFrom, DataFlow::Node nodeTo) {
- any(AdditionalTaintStep s).step(nodeFrom, nodeTo)
- }
- }
-}
diff --git a/python/ql/src/experimental/semmle/python/security/dataflow/XmlEntityInjectionCustomizations.qll b/python/ql/src/experimental/semmle/python/security/dataflow/XmlEntityInjectionCustomizations.qll
deleted file mode 100644
index e420c738a97..00000000000
--- a/python/ql/src/experimental/semmle/python/security/dataflow/XmlEntityInjectionCustomizations.qll
+++ /dev/null
@@ -1,86 +0,0 @@
-/**
- * Provides default sources, sinks and sanitizers for detecting
- * "ldap injection"
- * vulnerabilities, as well as extension points for adding your own.
- */
-
-private import python
-private import semmle.python.dataflow.new.DataFlow
-private import experimental.semmle.python.Concepts
-private import semmle.python.dataflow.new.RemoteFlowSources
-private import semmle.python.dataflow.new.BarrierGuards
-private import semmle.python.ApiGraphs
-
-/**
- * Provides default sources, sinks and sanitizers for detecting "xml injection"
- * vulnerabilities, as well as extension points for adding your own.
- */
-module XmlEntityInjection {
- /**
- * A data flow source for "xml injection" vulnerabilities.
- */
- abstract class Source extends DataFlow::Node { }
-
- /**
- * A data flow sink for "xml injection" vulnerabilities.
- */
- abstract class Sink extends DataFlow::Node {
- /** Gets the kind of XML injection that this sink is vulnerable to. */
- abstract string getVulnerableKind();
- }
-
- /**
- * A sanitizer guard for "xml injection" vulnerabilities.
- */
- abstract class SanitizerGuard extends DataFlow::BarrierGuard { }
-
- /**
- * A unit class for adding additional taint steps.
- *
- * Extend this class to add additional taint steps that should apply to `XmlEntityInjection`
- * taint configuration.
- */
- class AdditionalTaintStep extends Unit {
- /**
- * Holds if the step from `nodeFrom` to `nodeTo` should be considered a taint
- * step for `XmlEntityInjection` configuration.
- */
- abstract predicate step(DataFlow::Node nodeFrom, DataFlow::Node nodeTo);
- }
-
- /**
- * An input to a direct XML parsing function, considered as a flow sink.
- *
- * See `XML::XMLParsing`.
- */
- class XMLParsingInputAsSink extends Sink {
- ExperimentalXML::XMLParsing xmlParsing;
-
- XMLParsingInputAsSink() { this = xmlParsing.getAnInput() }
-
- override string getVulnerableKind() { xmlParsing.vulnerableTo(result) }
- }
-
- /**
- * A source of remote user input, considered as a flow source.
- */
- class RemoteFlowSourceAsSource extends Source, RemoteFlowSource { }
-
- /**
- * A comparison with a constant string, considered as a sanitizer-guard.
- */
- class StringConstCompareAsSanitizerGuard extends SanitizerGuard, StringConstCompare { }
-
- /**
- * A taint step for `io`'s `StringIO` and `BytesIO` methods.
- */
- class IoAdditionalTaintStep extends AdditionalTaintStep {
- override predicate step(DataFlow::Node nodeFrom, DataFlow::Node nodeTo) {
- exists(DataFlow::CallCfgNode ioCalls |
- ioCalls = API::moduleImport("io").getMember(["StringIO", "BytesIO"]).getACall() and
- nodeFrom = ioCalls.getArg(0) and
- nodeTo = ioCalls
- )
- }
- }
-}
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-611/SimpleXmlRpcServer.expected b/python/ql/test/experimental/query-tests/Security/CWE-611-SimpleXmlRpcServer/SimpleXmlRpcServer.expected
similarity index 100%
rename from python/ql/test/experimental/query-tests/Security/CWE-611/SimpleXmlRpcServer.expected
rename to python/ql/test/experimental/query-tests/Security/CWE-611-SimpleXmlRpcServer/SimpleXmlRpcServer.expected
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-611/SimpleXmlRpcServer.qlref b/python/ql/test/experimental/query-tests/Security/CWE-611-SimpleXmlRpcServer/SimpleXmlRpcServer.qlref
similarity index 100%
rename from python/ql/test/experimental/query-tests/Security/CWE-611/SimpleXmlRpcServer.qlref
rename to python/ql/test/experimental/query-tests/Security/CWE-611-SimpleXmlRpcServer/SimpleXmlRpcServer.qlref
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-611/xmlrpc_server.py b/python/ql/test/experimental/query-tests/Security/CWE-611-SimpleXmlRpcServer/xmlrpc_server.py
similarity index 100%
rename from python/ql/test/experimental/query-tests/Security/CWE-611/xmlrpc_server.py
rename to python/ql/test/experimental/query-tests/Security/CWE-611-SimpleXmlRpcServer/xmlrpc_server.py
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-611/XmlEntityInjection.expected b/python/ql/test/experimental/query-tests/Security/CWE-611/XmlEntityInjection.expected
deleted file mode 100644
index 25594b4ddaa..00000000000
--- a/python/ql/test/experimental/query-tests/Security/CWE-611/XmlEntityInjection.expected
+++ /dev/null
@@ -1,27 +0,0 @@
-edges
-| test.py:8:19:8:25 | ControlFlowNode for request | test.py:8:19:8:30 | ControlFlowNode for Attribute |
-| test.py:8:19:8:30 | ControlFlowNode for Attribute | test.py:8:19:8:45 | ControlFlowNode for Subscript |
-| test.py:8:19:8:45 | ControlFlowNode for Subscript | test.py:9:34:9:44 | ControlFlowNode for xml_content |
-| test.py:13:19:13:25 | ControlFlowNode for request | test.py:13:19:13:30 | ControlFlowNode for Attribute |
-| test.py:13:19:13:30 | ControlFlowNode for Attribute | test.py:13:19:13:45 | ControlFlowNode for Subscript |
-| test.py:13:19:13:45 | ControlFlowNode for Subscript | test.py:15:34:15:44 | ControlFlowNode for xml_content |
-| test.py:19:19:19:25 | ControlFlowNode for request | test.py:19:19:19:30 | ControlFlowNode for Attribute |
-| test.py:19:19:19:30 | ControlFlowNode for Attribute | test.py:19:19:19:45 | ControlFlowNode for Subscript |
-| test.py:19:19:19:45 | ControlFlowNode for Subscript | test.py:30:34:30:44 | ControlFlowNode for xml_content |
-nodes
-| test.py:8:19:8:25 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
-| test.py:8:19:8:30 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
-| test.py:8:19:8:45 | ControlFlowNode for Subscript | semmle.label | ControlFlowNode for Subscript |
-| test.py:9:34:9:44 | ControlFlowNode for xml_content | semmle.label | ControlFlowNode for xml_content |
-| test.py:13:19:13:25 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
-| test.py:13:19:13:30 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
-| test.py:13:19:13:45 | ControlFlowNode for Subscript | semmle.label | ControlFlowNode for Subscript |
-| test.py:15:34:15:44 | ControlFlowNode for xml_content | semmle.label | ControlFlowNode for xml_content |
-| test.py:19:19:19:25 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
-| test.py:19:19:19:30 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
-| test.py:19:19:19:45 | ControlFlowNode for Subscript | semmle.label | ControlFlowNode for Subscript |
-| test.py:30:34:30:44 | ControlFlowNode for xml_content | semmle.label | ControlFlowNode for xml_content |
-subpaths
-#select
-| test.py:9:34:9:44 | ControlFlowNode for xml_content | test.py:8:19:8:25 | ControlFlowNode for request | test.py:9:34:9:44 | ControlFlowNode for xml_content | $@ XML input is constructed from a $@ and is vulnerable to: XXE. | test.py:9:34:9:44 | ControlFlowNode for xml_content | This | test.py:8:19:8:25 | ControlFlowNode for request | user-provided value |
-| test.py:30:34:30:44 | ControlFlowNode for xml_content | test.py:19:19:19:25 | ControlFlowNode for request | test.py:30:34:30:44 | ControlFlowNode for xml_content | $@ XML input is constructed from a $@ and is vulnerable to: Billion Laughs, DTD retrieval, Quadratic Blowup, XXE. | test.py:30:34:30:44 | ControlFlowNode for xml_content | This | test.py:19:19:19:25 | ControlFlowNode for request | user-provided value |
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-611/XmlEntityInjection.qlref b/python/ql/test/experimental/query-tests/Security/CWE-611/XmlEntityInjection.qlref
deleted file mode 100644
index 36a7c8845fb..00000000000
--- a/python/ql/test/experimental/query-tests/Security/CWE-611/XmlEntityInjection.qlref
+++ /dev/null
@@ -1 +0,0 @@
-experimental/Security/CWE-611/XmlEntityInjection.ql
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-611/test.py b/python/ql/test/experimental/query-tests/Security/CWE-611/test.py
deleted file mode 100644
index d9181c4cf34..00000000000
--- a/python/ql/test/experimental/query-tests/Security/CWE-611/test.py
+++ /dev/null
@@ -1,30 +0,0 @@
-from flask import Flask, request
-import lxml.etree
-
-app = Flask(__name__)
-
-@app.route("/vuln-handler")
-def vuln_handler():
- xml_content = request.args['xml_content']
- return lxml.etree.fromstring(xml_content).text
-
-@app.route("/safe-handler")
-def safe_handler():
- xml_content = request.args['xml_content']
- parser = lxml.etree.XMLParser(resolve_entities=False)
- return lxml.etree.fromstring(xml_content, parser=parser).text
-
-@app.route("/super-vuln-handler")
-def super_vuln_handler():
- xml_content = request.args['xml_content']
- parser = lxml.etree.XMLParser(
- # allows XXE
- resolve_entities=True,
- # allows remote XXE
- no_network=False,
- # together with `no_network=False`, allows DTD-retrival
- load_dtd=True,
- # allows DoS attacks
- huge_tree=True,
- )
- return lxml.etree.fromstring(xml_content, parser=parser).text
From b00766b054d1b58a06dce48bd631a5b0eaacb7b7 Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Tue, 29 Mar 2022 13:51:00 +0200
Subject: [PATCH 050/171] Python: Adjust XXE qhelp
and remove the old copy, we don't need it anymore :)
---
.../Security/NEW/CWE-611/Xxe.qhelp | 39 ++++++++++-----
.../Security/NEW/CWE-611/examples/Xxe.js | 7 ---
.../Security/NEW/CWE-611/examples/XxeBad.py | 10 ++++
.../Security/NEW/CWE-611/examples/XxeGood.js | 7 ---
.../Security/NEW/CWE-611/examples/XxeGood.py | 11 +++++
.../Security/NEW/XmlEntityInjection.qhelp | 48 -------------------
.../library-tests/frameworks/XML/poc/PoC.py | 11 +++++
7 files changed, 58 insertions(+), 75 deletions(-)
delete mode 100644 python/ql/src/experimental/Security/NEW/CWE-611/examples/Xxe.js
create mode 100644 python/ql/src/experimental/Security/NEW/CWE-611/examples/XxeBad.py
delete mode 100644 python/ql/src/experimental/Security/NEW/CWE-611/examples/XxeGood.js
create mode 100644 python/ql/src/experimental/Security/NEW/CWE-611/examples/XxeGood.py
delete mode 100644 python/ql/src/experimental/Security/NEW/XmlEntityInjection.qhelp
diff --git a/python/ql/src/experimental/Security/NEW/CWE-611/Xxe.qhelp b/python/ql/src/experimental/Security/NEW/CWE-611/Xxe.qhelp
index 1e859eb121f..7254e292309 100644
--- a/python/ql/src/experimental/Security/NEW/CWE-611/Xxe.qhelp
+++ b/python/ql/src/experimental/Security/NEW/CWE-611/Xxe.qhelp
@@ -15,29 +15,34 @@ and out-of-band data retrieval techniques may allow attackers to steal sensitive
The easiest way to prevent XXE attacks is to disable external entity handling when
parsing untrusted data. How this is done depends on the library being used. Note that some
-libraries, such as recent versions of libxml, disable entity expansion by default,
+libraries, such as recent versions of the XML libraries in the standard library of Python 3,
+disable entity expansion by default,
so unless you have explicitly enabled entity expansion, no further action needs to be taken.
+
+
+We recommend using the defusedxml
+PyPI package, which has been created to prevent XML attacks (both XXE and XML bombs).
+
-The following example uses the libxml XML parser to parse a string xmlSrc.
-If that string is from an untrusted source, this code may be vulnerable to an XXE attack, since
-the parser is invoked with the noent option set to true:
+The following example uses the lxml XML parser to parse a string
+xml_src. That string is from an untrusted source, so this code is
+vulnerable to an XXE attack, since the
+default parser from lxml.etree allows local external entities to be resolved.
-
+
-To guard against XXE attacks, the noent option should be omitted or set to
-false. This means that no entity expansion is undertaken at all, not even for standard
-internal entities such as & or >. If desired, these
-entities can be expanded in a separate step using utility functions provided by libraries such
-as underscore ,
-lodash or
-he .
+To guard against XXE attacks with the lxml library, you should create a
+parser with resolve_entities set to false. This means that no
+entity expansion is undertaken, althuogh standard predefined entities such as
+>, for writing > inside the text of an XML element,
+are still allowed.
-
+
@@ -53,5 +58,13 @@ Timothy Morgen:
Timur Yunusov, Alexey Osipov:
XML Out-Of-Band Data Retrieval .
+
+Python 3 standard library:
+XML Vulnerabilities .
+
+
+Python 2 standard library:
+XML Vulnerabilities .
+
diff --git a/python/ql/src/experimental/Security/NEW/CWE-611/examples/Xxe.js b/python/ql/src/experimental/Security/NEW/CWE-611/examples/Xxe.js
deleted file mode 100644
index 99fa02cc42f..00000000000
--- a/python/ql/src/experimental/Security/NEW/CWE-611/examples/Xxe.js
+++ /dev/null
@@ -1,7 +0,0 @@
-const app = require("express")(),
- libxml = require("libxmljs");
-
-app.post("upload", (req, res) => {
- let xmlSrc = req.body,
- doc = libxml.parseXml(xmlSrc, { noent: true });
-});
diff --git a/python/ql/src/experimental/Security/NEW/CWE-611/examples/XxeBad.py b/python/ql/src/experimental/Security/NEW/CWE-611/examples/XxeBad.py
new file mode 100644
index 00000000000..4b2121ab4a6
--- /dev/null
+++ b/python/ql/src/experimental/Security/NEW/CWE-611/examples/XxeBad.py
@@ -0,0 +1,10 @@
+from flask import Flask, request
+import lxml.etree
+
+app = Flask(__name__)
+
+@app.post("/upload")
+def upload():
+ xml_src = request.get_data()
+ doc = lxml.etree.fromstring(xml_src)
+ return lxml.etree.tostring(doc)
diff --git a/python/ql/src/experimental/Security/NEW/CWE-611/examples/XxeGood.js b/python/ql/src/experimental/Security/NEW/CWE-611/examples/XxeGood.js
deleted file mode 100644
index 8317dcac98f..00000000000
--- a/python/ql/src/experimental/Security/NEW/CWE-611/examples/XxeGood.js
+++ /dev/null
@@ -1,7 +0,0 @@
-const app = require("express")(),
- libxml = require("libxmljs");
-
-app.post("upload", (req, res) => {
- let xmlSrc = req.body,
- doc = libxml.parseXml(xmlSrc);
-});
diff --git a/python/ql/src/experimental/Security/NEW/CWE-611/examples/XxeGood.py b/python/ql/src/experimental/Security/NEW/CWE-611/examples/XxeGood.py
new file mode 100644
index 00000000000..20844032fa3
--- /dev/null
+++ b/python/ql/src/experimental/Security/NEW/CWE-611/examples/XxeGood.py
@@ -0,0 +1,11 @@
+from flask import Flask, request
+import lxml.etree
+
+app = Flask(__name__)
+
+@app.post("/upload")
+def upload():
+ xml_src = request.get_data()
+ parser = lxml.etree.XMLParser(resolve_entities=False)
+ doc = lxml.etree.fromstring(xml_src, parser=parser)
+ return lxml.etree.tostring(doc)
diff --git a/python/ql/src/experimental/Security/NEW/XmlEntityInjection.qhelp b/python/ql/src/experimental/Security/NEW/XmlEntityInjection.qhelp
deleted file mode 100644
index 6da1bf1d306..00000000000
--- a/python/ql/src/experimental/Security/NEW/XmlEntityInjection.qhelp
+++ /dev/null
@@ -1,48 +0,0 @@
-
-
-
-
-
-Parsing untrusted XML files with a weakly configured XML parser may lead to attacks such as XML External Entity (XXE),
-Billion Laughs, Quadratic Blowup and DTD retrieval.
-This type of attack uses external entity references to access arbitrary files on a system, carry out denial of
-service, or server side request forgery. Even when the result of parsing is not returned to the user, out-of-band
-data retrieval techniques may allow attackers to steal sensitive data. Denial of services can also be carried out
-in this situation.
-
-
-
-
-
-Use defusedxml , a Python package aimed
-to prevent any potentially malicious operation.
-
-
-
-
-
-The following example calls xml.etree.ElementTree.fromstring using a parser (lxml.etree.XMLParser)
-that is not safely configured on untrusted data, and is therefore inherently unsafe.
-
-
-
-Providing an input (xml_content) like the following XML content against /bad, the request response would contain the contents of
-/etc/passwd.
-
-
-
-
-
-Python 3 XML Vulnerabilities .
-Python 2 XML Vulnerabilities .
-Python XML Parsing .
-OWASP vulnerability description: XML External Entity (XXE) Processing .
-OWASP guidance on parsing xml files: XXE Prevention Cheat Sheet .
-Paper by Timothy Morgen: XML Schema, DTD, and Entity Attacks
-Out-of-band data retrieval: Timur Yunusov & Alexey Osipov, Black hat EU 2013: XML Out-Of-Band Data Retrieval .
-Denial of service attack (Billion laughs): Billion Laughs.
-
-
-
diff --git a/python/ql/test/experimental/library-tests/frameworks/XML/poc/PoC.py b/python/ql/test/experimental/library-tests/frameworks/XML/poc/PoC.py
index b4cb2faf304..a4de65084ae 100644
--- a/python/ql/test/experimental/library-tests/frameworks/XML/poc/PoC.py
+++ b/python/ql/test/experimental/library-tests/frameworks/XML/poc/PoC.py
@@ -74,6 +74,10 @@ exfiltrate_through_dtd_retrieval = f"""
%xxe; ]>
"""
+predefined_entity_xml = """
+<
+"""
+
# ==============================================================================
# other setup
@@ -443,6 +447,13 @@ class TestLxml:
assert exfiltrated_data == "SECRET_FLAG"
+ @staticmethod
+ def test_predefined_entity():
+ parser = lxml.etree.XMLParser(resolve_entities=False)
+ root = lxml.etree.fromstring(predefined_entity_xml, parser=parser)
+ assert root.tag == "test"
+ assert root.text == "<"
+
# ==============================================================================
import xmltodict
From 56b9c891d85636c543bf9529dd7b191908248be7 Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Tue, 29 Mar 2022 15:30:04 +0200
Subject: [PATCH 051/171] Python: Adjust `XmlBomb.qhelp` from JS
---
.../Security/NEW/CWE-776/XmlBomb.qhelp | 36 +++++++++++++------
.../Security/NEW/CWE-776/examples/XmlBomb.js | 10 ------
.../NEW/CWE-776/examples/XmlBombBad.py | 10 ++++++
.../NEW/CWE-776/examples/XmlBombGood.js | 10 ------
.../NEW/CWE-776/examples/XmlBombGood.py | 10 ++++++
5 files changed, 45 insertions(+), 31 deletions(-)
delete mode 100644 python/ql/src/experimental/Security/NEW/CWE-776/examples/XmlBomb.js
create mode 100644 python/ql/src/experimental/Security/NEW/CWE-776/examples/XmlBombBad.py
delete mode 100644 python/ql/src/experimental/Security/NEW/CWE-776/examples/XmlBombGood.js
create mode 100644 python/ql/src/experimental/Security/NEW/CWE-776/examples/XmlBombGood.py
diff --git a/python/ql/src/experimental/Security/NEW/CWE-776/XmlBomb.qhelp b/python/ql/src/experimental/Security/NEW/CWE-776/XmlBomb.qhelp
index c0714b3f96f..f20dd526fdd 100644
--- a/python/ql/src/experimental/Security/NEW/CWE-776/XmlBomb.qhelp
+++ b/python/ql/src/experimental/Security/NEW/CWE-776/XmlBomb.qhelp
@@ -25,26 +25,32 @@ to take a very long time or use large amounts of memory. This is sometimes calle
The safest way to prevent XML bomb attacks is to disable entity expansion when parsing untrusted
-data. How this is done depends on the library being used. Note that some libraries, such as
-recent versions of libxmljs (though not its SAX parser API), disable entity expansion
-by default, so unless you have explicitly enabled entity expansion, no further action is needed.
+data. Whether this can be done depends on the library being used. Note that some libraries, such as
+lxml, have measures enabled by default to prevent such DoS XML attacks, so
+unless you have explicitly set huge_tree to True, no further action is needed.
+
+
+
+We recommend using the defusedxml
+PyPI package, which has been created to prevent XML attacks (both XXE and XML bombs).
-The following example uses the XML parser provided by the node-expat package to
-parse a string xmlSrc. If that string is from an untrusted source, this code may be
-vulnerable to a DoS attack, since node-expat expands internal entities by default:
+The following example uses the xml.etree XML parser provided by the Python standard library to
+parse a string xml_src. That string is from an untrusted source, so this code is be
+vulnerable to a DoS attack, since the xml.etree XML parser expands internal entities by default:
-
+
-At the time of writing, node-expat does not provide a way of controlling entity
-expansion, but the example could be rewritten to use the sax package instead,
-which only expands standard entities such as &:
+It is not possible to guard against internal entity expansion with
+xml.etree, so to guard against these attacks, the following example uses
+the defusedxml
+PyPI package instead, which is not exposed to such internal entity expansion attacks.
-
+
@@ -56,5 +62,13 @@ Wikipedia:
Bryan Sullivan:
Security Briefs - XML Denial of Service Attacks and Defenses .
+
+Python 3 standard library:
+XML Vulnerabilities .
+
+
+Python 2 standard library:
+XML Vulnerabilities .
+
diff --git a/python/ql/src/experimental/Security/NEW/CWE-776/examples/XmlBomb.js b/python/ql/src/experimental/Security/NEW/CWE-776/examples/XmlBomb.js
deleted file mode 100644
index f72902a5304..00000000000
--- a/python/ql/src/experimental/Security/NEW/CWE-776/examples/XmlBomb.js
+++ /dev/null
@@ -1,10 +0,0 @@
-const app = require("express")(),
- expat = require("node-expat");
-
-app.post("upload", (req, res) => {
- let xmlSrc = req.body,
- parser = new expat.Parser();
- parser.on("startElement", handleStart);
- parser.on("text", handleText);
- parser.write(xmlSrc);
-});
diff --git a/python/ql/src/experimental/Security/NEW/CWE-776/examples/XmlBombBad.py b/python/ql/src/experimental/Security/NEW/CWE-776/examples/XmlBombBad.py
new file mode 100644
index 00000000000..d52054d9492
--- /dev/null
+++ b/python/ql/src/experimental/Security/NEW/CWE-776/examples/XmlBombBad.py
@@ -0,0 +1,10 @@
+from flask import Flask, request
+import xml.etree.ElementTree as ET
+
+app = Flask(__name__)
+
+@app.post("/upload")
+def upload():
+ xml_src = request.get_data()
+ doc = ET.fromstring(xml_src)
+ return ET.tostring(doc)
diff --git a/python/ql/src/experimental/Security/NEW/CWE-776/examples/XmlBombGood.js b/python/ql/src/experimental/Security/NEW/CWE-776/examples/XmlBombGood.js
deleted file mode 100644
index a8c5bc97e63..00000000000
--- a/python/ql/src/experimental/Security/NEW/CWE-776/examples/XmlBombGood.js
+++ /dev/null
@@ -1,10 +0,0 @@
-const app = require("express")(),
- sax = require("sax");
-
-app.post("upload", (req, res) => {
- let xmlSrc = req.body,
- parser = sax.parser(true);
- parser.onopentag = handleStart;
- parser.ontext = handleText;
- parser.write(xmlSrc);
-});
diff --git a/python/ql/src/experimental/Security/NEW/CWE-776/examples/XmlBombGood.py b/python/ql/src/experimental/Security/NEW/CWE-776/examples/XmlBombGood.py
new file mode 100644
index 00000000000..5e4261e35da
--- /dev/null
+++ b/python/ql/src/experimental/Security/NEW/CWE-776/examples/XmlBombGood.py
@@ -0,0 +1,10 @@
+from flask import Flask, request
+import defusedxml.ElementTree as ET
+
+app = Flask(__name__)
+
+@app.post("/upload")
+def upload():
+ xml_src = request.get_data()
+ doc = ET.fromstring(xml_src)
+ return ET.tostring(doc)
From 9caf4be21be7370a0317ae44205dfb35a5169073 Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Tue, 29 Mar 2022 15:33:57 +0200
Subject: [PATCH 052/171] Python: Add PortSwigger link to `Xxe.qhelp`
I found this resource quite good myself at least :)
---
python/ql/src/experimental/Security/NEW/CWE-611/Xxe.qhelp | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/python/ql/src/experimental/Security/NEW/CWE-611/Xxe.qhelp b/python/ql/src/experimental/Security/NEW/CWE-611/Xxe.qhelp
index 7254e292309..19bbc955fd6 100644
--- a/python/ql/src/experimental/Security/NEW/CWE-611/Xxe.qhelp
+++ b/python/ql/src/experimental/Security/NEW/CWE-611/Xxe.qhelp
@@ -66,5 +66,9 @@ Python 3 standard library:
Python 2 standard library:
XML Vulnerabilities .
+
+PortSwigger:
+XML external entity (XXE) injection .
+
From e005a5c0ab7409ebb6fb0002cdc7ab11a1b54bd1 Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Tue, 29 Mar 2022 15:50:24 +0200
Subject: [PATCH 053/171] Python: Promote `XMLParsing` concept
---
python/ql/lib/semmle/python/Concepts.qll | 62 +++++++++++++++++
.../experimental/semmle/python/Concepts.qll | 68 -------------------
.../semmle/python/frameworks/Xml.qll | 4 +-
.../dataflow/XmlBombCustomizations.qll | 5 +-
.../security/dataflow/XxeCustomizations.qll | 5 +-
.../XML/ExperimentalXmlConceptsTests.ql | 2 +-
6 files changed, 70 insertions(+), 76 deletions(-)
diff --git a/python/ql/lib/semmle/python/Concepts.qll b/python/ql/lib/semmle/python/Concepts.qll
index a768f29795c..3d83ec100a5 100644
--- a/python/ql/lib/semmle/python/Concepts.qll
+++ b/python/ql/lib/semmle/python/Concepts.qll
@@ -550,6 +550,68 @@ module XML {
abstract string getName();
}
}
+
+ /**
+ * A kind of XML vulnerability.
+ *
+ * See overview of kinds at https://pypi.org/project/defusedxml/#python-xml-libraries
+ */
+ class XMLVulnerabilityKind extends string {
+ XMLVulnerabilityKind() {
+ this in ["Billion Laughs", "Quadratic Blowup", "XXE", "DTD retrieval"]
+ }
+
+ /** Holds for Billion Laughs vulnerability kind. */
+ predicate isBillionLaughs() { this = "Billion Laughs" }
+
+ /** Holds for Quadratic Blowup vulnerability kind. */
+ predicate isQuadraticBlowup() { this = "Quadratic Blowup" }
+
+ /** Holds for XXE vulnerability kind. */
+ predicate isXxe() { this = "XXE" }
+
+ /** Holds for DTD retrieval vulnerability kind. */
+ predicate isDtdRetrieval() { this = "DTD retrieval" }
+ }
+
+ /**
+ * A data-flow node that parses XML.
+ *
+ * Extend this class to model new APIs. If you want to refine existing API models,
+ * extend `XMLParsing` instead.
+ */
+ class XMLParsing extends DataFlow::Node instanceof XMLParsing::Range {
+ /**
+ * Gets the argument containing the content to parse.
+ */
+ DataFlow::Node getAnInput() { result = super.getAnInput() }
+
+ /**
+ * Holds if this XML parsing is vulnerable to `kind`.
+ */
+ predicate vulnerableTo(XMLVulnerabilityKind kind) { super.vulnerableTo(kind) }
+ }
+
+ /** Provides classes for modeling XML parsing APIs. */
+ module XMLParsing {
+ /**
+ * A data-flow node that parses XML.
+ *
+ * Extend this class to model new APIs. If you want to refine existing API models,
+ * extend `XMLParsing` instead.
+ */
+ abstract class Range extends DataFlow::Node {
+ /**
+ * Gets the argument containing the content to parse.
+ */
+ abstract DataFlow::Node getAnInput();
+
+ /**
+ * Holds if this XML parsing is vulnerable to `kind`.
+ */
+ abstract predicate vulnerableTo(XMLVulnerabilityKind kind);
+ }
+ }
}
/** Provides classes for modeling LDAP-related APIs. */
diff --git a/python/ql/src/experimental/semmle/python/Concepts.qll b/python/ql/src/experimental/semmle/python/Concepts.qll
index 6fdba4d3627..09b44d95e89 100644
--- a/python/ql/src/experimental/semmle/python/Concepts.qll
+++ b/python/ql/src/experimental/semmle/python/Concepts.qll
@@ -14,74 +14,6 @@ private import semmle.python.dataflow.new.RemoteFlowSources
private import semmle.python.dataflow.new.TaintTracking
private import experimental.semmle.python.Frameworks
-/**
- * Since there is both XML module in normal and experimental Concepts,
- * we have to rename the experimental module as this.
- */
-module ExperimentalXML {
- /**
- * A kind of XML vulnerability.
- *
- * See https://pypi.org/project/defusedxml/#python-xml-libraries
- */
- class XMLVulnerabilityKind extends string {
- XMLVulnerabilityKind() {
- this in ["Billion Laughs", "Quadratic Blowup", "XXE", "DTD retrieval"]
- }
-
- /** Holds for Billion Laughs vulnerability kind. */
- predicate isBillionLaughs() { this = "Billion Laughs" }
-
- /** Holds for Quadratic Blowup vulnerability kind. */
- predicate isQuadraticBlowup() { this = "Quadratic Blowup" }
-
- /** Holds for XXE vulnerability kind. */
- predicate isXxe() { this = "XXE" }
-
- /** Holds for DTD retrieval vulnerability kind. */
- predicate isDtdRetrieval() { this = "DTD retrieval" }
- }
-
- /**
- * A data-flow node that parses XML.
- *
- * Extend this class to model new APIs. If you want to refine existing API models,
- * extend `XMLParsing` instead.
- */
- class XMLParsing extends DataFlow::Node instanceof XMLParsing::Range {
- /**
- * Gets the argument containing the content to parse.
- */
- DataFlow::Node getAnInput() { result = super.getAnInput() }
-
- /**
- * Holds if this XML parsing is vulnerable to `kind`.
- */
- predicate vulnerableTo(XMLVulnerabilityKind kind) { super.vulnerableTo(kind) }
- }
-
- /** Provides classes for modeling XML parsing APIs. */
- module XMLParsing {
- /**
- * A data-flow node that parses XML.
- *
- * Extend this class to model new APIs. If you want to refine existing API models,
- * extend `XMLParsing` instead.
- */
- abstract class Range extends DataFlow::Node {
- /**
- * Gets the argument containing the content to parse.
- */
- abstract DataFlow::Node getAnInput();
-
- /**
- * Holds if this XML parsing is vulnerable to `kind`.
- */
- abstract predicate vulnerableTo(XMLVulnerabilityKind kind);
- }
- }
-}
-
/** Provides classes for modeling LDAP query execution-related APIs. */
module LdapQuery {
/**
diff --git a/python/ql/src/experimental/semmle/python/frameworks/Xml.qll b/python/ql/src/experimental/semmle/python/frameworks/Xml.qll
index a2f36f66f2e..87aa236804d 100644
--- a/python/ql/src/experimental/semmle/python/frameworks/Xml.qll
+++ b/python/ql/src/experimental/semmle/python/frameworks/Xml.qll
@@ -5,11 +5,9 @@
private import python
private import semmle.python.dataflow.new.DataFlow
-private import experimental.semmle.python.Concepts
+private import semmle.python.Concepts
private import semmle.python.ApiGraphs
-module XML = ExperimentalXML;
-
private module XmlEtree {
/**
* Provides models for `xml.etree` parsers
diff --git a/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombCustomizations.qll b/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombCustomizations.qll
index 66a16a4494a..a4cbfe61821 100644
--- a/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombCustomizations.qll
+++ b/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombCustomizations.qll
@@ -6,7 +6,8 @@
private import python
private import semmle.python.dataflow.new.DataFlow
-private import experimental.semmle.python.Concepts
+private import semmle.python.Concepts
+import experimental.semmle.python.frameworks.Xml // needed until modeling have been promoted
private import semmle.python.dataflow.new.RemoteFlowSources
/**
@@ -40,7 +41,7 @@ module XmlBomb {
*/
class XmlParsingWithEntityResolution extends Sink {
XmlParsingWithEntityResolution() {
- exists(ExperimentalXML::XMLParsing parsing, ExperimentalXML::XMLVulnerabilityKind kind |
+ exists(XML::XMLParsing parsing, XML::XMLVulnerabilityKind kind |
(kind.isBillionLaughs() or kind.isQuadraticBlowup()) and
parsing.vulnerableTo(kind) and
this = parsing.getAnInput()
diff --git a/python/ql/src/experimental/semmle/python/security/dataflow/XxeCustomizations.qll b/python/ql/src/experimental/semmle/python/security/dataflow/XxeCustomizations.qll
index b2992dd335f..c118e1b2ff9 100644
--- a/python/ql/src/experimental/semmle/python/security/dataflow/XxeCustomizations.qll
+++ b/python/ql/src/experimental/semmle/python/security/dataflow/XxeCustomizations.qll
@@ -6,7 +6,8 @@
private import python
private import semmle.python.dataflow.new.DataFlow
-private import experimental.semmle.python.Concepts
+private import semmle.python.Concepts
+import experimental.semmle.python.frameworks.Xml // needed until modeling have been promoted
private import semmle.python.dataflow.new.RemoteFlowSources
/**
@@ -40,7 +41,7 @@ module Xxe {
*/
class XmlParsingWithExternalEntityResolution extends Sink {
XmlParsingWithExternalEntityResolution() {
- exists(ExperimentalXML::XMLParsing parsing, ExperimentalXML::XMLVulnerabilityKind kind |
+ exists(XML::XMLParsing parsing, XML::XMLVulnerabilityKind kind |
kind.isXxe() and
parsing.vulnerableTo(kind) and
this = parsing.getAnInput()
diff --git a/python/ql/test/experimental/library-tests/frameworks/XML/ExperimentalXmlConceptsTests.ql b/python/ql/test/experimental/library-tests/frameworks/XML/ExperimentalXmlConceptsTests.ql
index 81bc391d0e5..679dbc3456c 100644
--- a/python/ql/test/experimental/library-tests/frameworks/XML/ExperimentalXmlConceptsTests.ql
+++ b/python/ql/test/experimental/library-tests/frameworks/XML/ExperimentalXmlConceptsTests.ql
@@ -1,5 +1,5 @@
import python
-import experimental.semmle.python.Concepts
+import semmle.python.Concepts
import experimental.semmle.python.frameworks.Xml
import semmle.python.dataflow.new.DataFlow
import TestUtilities.InlineExpectationsTest
From e45288e812a0cd0f87cb909768b60847ec5aa997 Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Tue, 29 Mar 2022 15:51:27 +0200
Subject: [PATCH 054/171] Python: => `XMLParsingVulnerabilityKind`
Since there are other XML vulnerabilities that are not about parsing,
this is more correct.
---
python/ql/lib/semmle/python/Concepts.qll | 8 +++----
.../Security/CWE-611/SimpleXmlRpcServer.ql | 2 +-
.../semmle/python/frameworks/Xml.qll | 24 +++++++++----------
.../dataflow/XmlBombCustomizations.qll | 2 +-
.../security/dataflow/XxeCustomizations.qll | 2 +-
.../XML/ExperimentalXmlConceptsTests.ql | 2 +-
6 files changed, 20 insertions(+), 20 deletions(-)
diff --git a/python/ql/lib/semmle/python/Concepts.qll b/python/ql/lib/semmle/python/Concepts.qll
index 3d83ec100a5..c430594d05b 100644
--- a/python/ql/lib/semmle/python/Concepts.qll
+++ b/python/ql/lib/semmle/python/Concepts.qll
@@ -556,8 +556,8 @@ module XML {
*
* See overview of kinds at https://pypi.org/project/defusedxml/#python-xml-libraries
*/
- class XMLVulnerabilityKind extends string {
- XMLVulnerabilityKind() {
+ class XMLParsingVulnerabilityKind extends string {
+ XMLParsingVulnerabilityKind() {
this in ["Billion Laughs", "Quadratic Blowup", "XXE", "DTD retrieval"]
}
@@ -589,7 +589,7 @@ module XML {
/**
* Holds if this XML parsing is vulnerable to `kind`.
*/
- predicate vulnerableTo(XMLVulnerabilityKind kind) { super.vulnerableTo(kind) }
+ predicate vulnerableTo(XMLParsingVulnerabilityKind kind) { super.vulnerableTo(kind) }
}
/** Provides classes for modeling XML parsing APIs. */
@@ -609,7 +609,7 @@ module XML {
/**
* Holds if this XML parsing is vulnerable to `kind`.
*/
- abstract predicate vulnerableTo(XMLVulnerabilityKind kind);
+ abstract predicate vulnerableTo(XMLParsingVulnerabilityKind kind);
}
}
}
diff --git a/python/ql/src/experimental/Security/CWE-611/SimpleXmlRpcServer.ql b/python/ql/src/experimental/Security/CWE-611/SimpleXmlRpcServer.ql
index cda0633690c..3d2a736ed49 100644
--- a/python/ql/src/experimental/Security/CWE-611/SimpleXmlRpcServer.ql
+++ b/python/ql/src/experimental/Security/CWE-611/SimpleXmlRpcServer.ql
@@ -17,7 +17,7 @@ from DataFlow::CallCfgNode call, string kinds
where
call = API::moduleImport("xmlrpc").getMember("server").getMember("SimpleXMLRPCServer").getACall() and
kinds =
- strictconcat(ExperimentalXML::XMLVulnerabilityKind kind |
+ strictconcat(ExperimentalXML::XMLParsingVulnerabilityKind kind |
kind.isBillionLaughs() or kind.isQuadraticBlowup()
|
kind, ", "
diff --git a/python/ql/src/experimental/semmle/python/frameworks/Xml.qll b/python/ql/src/experimental/semmle/python/frameworks/Xml.qll
index 87aa236804d..4987e24bce4 100644
--- a/python/ql/src/experimental/semmle/python/frameworks/Xml.qll
+++ b/python/ql/src/experimental/semmle/python/frameworks/Xml.qll
@@ -66,7 +66,7 @@ private module XmlEtree {
override DataFlow::Node getAnInput() { result in [this.getArg(0), this.getArgByName("data")] }
- override predicate vulnerableTo(XML::XMLVulnerabilityKind kind) {
+ override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
kind.isBillionLaughs() or kind.isQuadraticBlowup()
}
}
@@ -103,7 +103,7 @@ private module XmlEtree {
]
}
- override predicate vulnerableTo(XML::XMLVulnerabilityKind kind) {
+ override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
// note: it does not matter what `xml.etree` parser you are using, you cannot
// change the security features anyway :|
kind.isBillionLaughs() or kind.isQuadraticBlowup()
@@ -218,7 +218,7 @@ private module SaxBasedParsing {
override DataFlow::Node getAnInput() { result in [this.getArg(0), this.getArgByName("source")] }
- override predicate vulnerableTo(XML::XMLVulnerabilityKind kind) {
+ override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
// always vuln to these
(kind.isBillionLaughs() or kind.isQuadraticBlowup())
or
@@ -251,7 +251,7 @@ private module SaxBasedParsing {
]
}
- override predicate vulnerableTo(XML::XMLVulnerabilityKind kind) {
+ override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
// always vuln to these
(kind.isBillionLaughs() or kind.isQuadraticBlowup())
or
@@ -290,7 +290,7 @@ private module SaxBasedParsing {
DataFlow::Node getParserArg() { result in [this.getArg(1), this.getArgByName("parser")] }
- override predicate vulnerableTo(XML::XMLVulnerabilityKind kind) {
+ override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
this.getParserArg() = saxParserWithFeatureExternalGesTurnedOn() and
(kind.isXxe() or kind.isDtdRetrieval())
or
@@ -317,7 +317,7 @@ private module Lxml {
*/
abstract class InstanceSource extends DataFlow::LocalSourceNode {
/** Holds if this instance is vulnerable to `kind`. */
- abstract predicate vulnerableTo(XML::XMLVulnerabilityKind kind);
+ abstract predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind);
}
/**
@@ -331,7 +331,7 @@ private module Lxml {
}
// NOTE: it's not possible to change settings of a parser after constructing it
- override predicate vulnerableTo(XML::XMLVulnerabilityKind kind) {
+ override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
kind.isXxe() and
(
// resolve_entities has default True
@@ -361,7 +361,7 @@ private module Lxml {
API::moduleImport("lxml").getMember("etree").getMember("get_default_parser").getACall()
}
- override predicate vulnerableTo(XML::XMLVulnerabilityKind kind) {
+ override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
// as highlighted by
// https://lxml.de/apidoc/lxml.etree.html?highlight=xmlparser#lxml.etree.XMLParser
// by default XXE is allow. so as long as the default parser has not been
@@ -385,7 +385,7 @@ private module Lxml {
}
/** Gets a reference to an `lxml.etree` parser instance, that is vulnerable to `kind`. */
- DataFlow::Node instanceVulnerableTo(XML::XMLVulnerabilityKind kind) {
+ DataFlow::Node instanceVulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
exists(InstanceSource origin | result = instance(origin) and origin.vulnerableTo(kind))
}
@@ -397,7 +397,7 @@ private module Lxml {
override DataFlow::Node getAnInput() { result in [this.getArg(0), this.getArgByName("data")] }
- override predicate vulnerableTo(XML::XMLVulnerabilityKind kind) {
+ override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
this.calls(instanceVulnerableTo(kind), "feed")
}
}
@@ -436,7 +436,7 @@ private module Lxml {
DataFlow::Node getParserArg() { result in [this.getArg(1), this.getArgByName("parser")] }
- override predicate vulnerableTo(XML::XMLVulnerabilityKind kind) {
+ override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
this.getParserArg() = XMLParser::instanceVulnerableTo(kind)
or
kind.isXxe() and
@@ -456,7 +456,7 @@ private module Xmltodict {
result in [this.getArg(0), this.getArgByName("xml_input")]
}
- override predicate vulnerableTo(XML::XMLVulnerabilityKind kind) {
+ override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
(kind.isBillionLaughs() or kind.isQuadraticBlowup()) and
this.getArgByName("disable_entities").getALocalSource().asExpr() = any(False f)
}
diff --git a/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombCustomizations.qll b/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombCustomizations.qll
index a4cbfe61821..c5e69c1e0e3 100644
--- a/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombCustomizations.qll
+++ b/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombCustomizations.qll
@@ -41,7 +41,7 @@ module XmlBomb {
*/
class XmlParsingWithEntityResolution extends Sink {
XmlParsingWithEntityResolution() {
- exists(XML::XMLParsing parsing, XML::XMLVulnerabilityKind kind |
+ exists(XML::XMLParsing parsing, XML::XMLParsingVulnerabilityKind kind |
(kind.isBillionLaughs() or kind.isQuadraticBlowup()) and
parsing.vulnerableTo(kind) and
this = parsing.getAnInput()
diff --git a/python/ql/src/experimental/semmle/python/security/dataflow/XxeCustomizations.qll b/python/ql/src/experimental/semmle/python/security/dataflow/XxeCustomizations.qll
index c118e1b2ff9..27d011625a6 100644
--- a/python/ql/src/experimental/semmle/python/security/dataflow/XxeCustomizations.qll
+++ b/python/ql/src/experimental/semmle/python/security/dataflow/XxeCustomizations.qll
@@ -41,7 +41,7 @@ module Xxe {
*/
class XmlParsingWithExternalEntityResolution extends Sink {
XmlParsingWithExternalEntityResolution() {
- exists(XML::XMLParsing parsing, XML::XMLVulnerabilityKind kind |
+ exists(XML::XMLParsing parsing, XML::XMLParsingVulnerabilityKind kind |
kind.isXxe() and
parsing.vulnerableTo(kind) and
this = parsing.getAnInput()
diff --git a/python/ql/test/experimental/library-tests/frameworks/XML/ExperimentalXmlConceptsTests.ql b/python/ql/test/experimental/library-tests/frameworks/XML/ExperimentalXmlConceptsTests.ql
index 679dbc3456c..98237b447ea 100644
--- a/python/ql/test/experimental/library-tests/frameworks/XML/ExperimentalXmlConceptsTests.ql
+++ b/python/ql/test/experimental/library-tests/frameworks/XML/ExperimentalXmlConceptsTests.ql
@@ -21,7 +21,7 @@ class XmlParsingTest extends InlineExpectationsTest {
tag = "input"
)
or
- exists(XML::XMLVulnerabilityKind kind |
+ exists(XML::XMLParsingVulnerabilityKind kind |
parsing.vulnerableTo(kind) and
location = parsing.getLocation() and
element = parsing.toString() and
From 35ccba2ec10b2610969ac790d8ca8fa76a282ad9 Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Tue, 29 Mar 2022 15:57:00 +0200
Subject: [PATCH 055/171] Python: Promote `XMLParsing` concept test
---
...tsTests.expected => ConceptsTest.expected} | 0
.../frameworks/XML/ConceptsTest.ql | 3 ++
.../XML/ExperimentalXmlConceptsTests.ql | 33 ---------------
.../frameworks/XML/lxml_etree.py | 40 +++++++++----------
.../library-tests/frameworks/XML/xml_dom.py | 24 +++++------
.../library-tests/frameworks/XML/xml_etree.py | 34 ++++++++--------
.../library-tests/frameworks/XML/xml_sax.py | 26 ++++++------
.../library-tests/frameworks/XML/xmltodict.py | 6 +--
.../test/experimental/meta/ConceptsTest.qll | 27 +++++++++++++
9 files changed, 95 insertions(+), 98 deletions(-)
rename python/ql/test/experimental/library-tests/frameworks/XML/{ExperimentalXmlConceptsTests.expected => ConceptsTest.expected} (100%)
create mode 100644 python/ql/test/experimental/library-tests/frameworks/XML/ConceptsTest.ql
delete mode 100644 python/ql/test/experimental/library-tests/frameworks/XML/ExperimentalXmlConceptsTests.ql
diff --git a/python/ql/test/experimental/library-tests/frameworks/XML/ExperimentalXmlConceptsTests.expected b/python/ql/test/experimental/library-tests/frameworks/XML/ConceptsTest.expected
similarity index 100%
rename from python/ql/test/experimental/library-tests/frameworks/XML/ExperimentalXmlConceptsTests.expected
rename to python/ql/test/experimental/library-tests/frameworks/XML/ConceptsTest.expected
diff --git a/python/ql/test/experimental/library-tests/frameworks/XML/ConceptsTest.ql b/python/ql/test/experimental/library-tests/frameworks/XML/ConceptsTest.ql
new file mode 100644
index 00000000000..95728bd6dc8
--- /dev/null
+++ b/python/ql/test/experimental/library-tests/frameworks/XML/ConceptsTest.ql
@@ -0,0 +1,3 @@
+import python
+import experimental.meta.ConceptsTest
+import experimental.semmle.python.frameworks.Xml // needed until modeling have been promoted
diff --git a/python/ql/test/experimental/library-tests/frameworks/XML/ExperimentalXmlConceptsTests.ql b/python/ql/test/experimental/library-tests/frameworks/XML/ExperimentalXmlConceptsTests.ql
deleted file mode 100644
index 98237b447ea..00000000000
--- a/python/ql/test/experimental/library-tests/frameworks/XML/ExperimentalXmlConceptsTests.ql
+++ /dev/null
@@ -1,33 +0,0 @@
-import python
-import semmle.python.Concepts
-import experimental.semmle.python.frameworks.Xml
-import semmle.python.dataflow.new.DataFlow
-import TestUtilities.InlineExpectationsTest
-private import semmle.python.dataflow.new.internal.PrintNode
-
-class XmlParsingTest extends InlineExpectationsTest {
- XmlParsingTest() { this = "XmlParsingTest" }
-
- override string getARelevantTag() { result in ["input", "vuln"] }
-
- override predicate hasActualResult(Location location, string element, string tag, string value) {
- exists(location.getFile().getRelativePath()) and
- exists(XML::XMLParsing parsing |
- exists(DataFlow::Node input |
- input = parsing.getAnInput() and
- location = input.getLocation() and
- element = input.toString() and
- value = prettyNodeForInlineTest(input) and
- tag = "input"
- )
- or
- exists(XML::XMLParsingVulnerabilityKind kind |
- parsing.vulnerableTo(kind) and
- location = parsing.getLocation() and
- element = parsing.toString() and
- value = "'" + kind + "'" and
- tag = "vuln"
- )
- )
- }
-}
diff --git a/python/ql/test/experimental/library-tests/frameworks/XML/lxml_etree.py b/python/ql/test/experimental/library-tests/frameworks/XML/lxml_etree.py
index 22930a58af3..ee8f3fc69c1 100644
--- a/python/ql/test/experimental/library-tests/frameworks/XML/lxml_etree.py
+++ b/python/ql/test/experimental/library-tests/frameworks/XML/lxml_etree.py
@@ -4,51 +4,51 @@ import lxml.etree
x = "some xml"
# different parsing methods
-lxml.etree.fromstring(x) # $ input=x vuln='XXE'
-lxml.etree.fromstring(text=x) # $ input=x vuln='XXE'
+lxml.etree.fromstring(x) # $ xmlInput=x xmlVuln='XXE'
+lxml.etree.fromstring(text=x) # $ xmlInput=x xmlVuln='XXE'
-lxml.etree.fromstringlist([x]) # $ input=List vuln='XXE'
-lxml.etree.fromstringlist(strings=[x]) # $ input=List vuln='XXE'
+lxml.etree.fromstringlist([x]) # $ xmlInput=List xmlVuln='XXE'
+lxml.etree.fromstringlist(strings=[x]) # $ xmlInput=List xmlVuln='XXE'
-lxml.etree.XML(x) # $ input=x vuln='XXE'
-lxml.etree.XML(text=x) # $ input=x vuln='XXE'
+lxml.etree.XML(x) # $ xmlInput=x xmlVuln='XXE'
+lxml.etree.XML(text=x) # $ xmlInput=x xmlVuln='XXE'
-lxml.etree.parse(StringIO(x)) # $ input=StringIO(..) vuln='XXE'
-lxml.etree.parse(source=StringIO(x)) # $ input=StringIO(..) vuln='XXE'
+lxml.etree.parse(StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='XXE'
+lxml.etree.parse(source=StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='XXE'
-lxml.etree.parseid(StringIO(x)) # $ input=StringIO(..) vuln='XXE'
-lxml.etree.parseid(source=StringIO(x)) # $ input=StringIO(..) vuln='XXE'
+lxml.etree.parseid(StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='XXE'
+lxml.etree.parseid(source=StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='XXE'
# With default parsers (nothing changed)
parser = lxml.etree.XMLParser()
-lxml.etree.fromstring(x, parser=parser) # $ input=x vuln='XXE'
+lxml.etree.fromstring(x, parser=parser) # $ xmlInput=x xmlVuln='XXE'
parser = lxml.etree.get_default_parser()
-lxml.etree.fromstring(x, parser=parser) # $ input=x vuln='XXE'
+lxml.etree.fromstring(x, parser=parser) # $ xmlInput=x xmlVuln='XXE'
# manual use of feed method
parser = lxml.etree.XMLParser()
-parser.feed(x) # $ input=x vuln='XXE'
-parser.feed(data=x) # $ input=x vuln='XXE'
+parser.feed(x) # $ xmlInput=x xmlVuln='XXE'
+parser.feed(data=x) # $ xmlInput=x xmlVuln='XXE'
parser.close()
# XXE-safe
parser = lxml.etree.XMLParser(resolve_entities=False)
-lxml.etree.fromstring(x, parser) # $ input=x
-lxml.etree.fromstring(x, parser=parser) # $ input=x
+lxml.etree.fromstring(x, parser) # $ xmlInput=x
+lxml.etree.fromstring(x, parser=parser) # $ xmlInput=x
# XXE-vuln
parser = lxml.etree.XMLParser(resolve_entities=True)
-lxml.etree.fromstring(x, parser=parser) # $ input=x vuln='XXE'
+lxml.etree.fromstring(x, parser=parser) # $ xmlInput=x xmlVuln='XXE'
# Billion laughs vuln (also XXE)
parser = lxml.etree.XMLParser(huge_tree=True)
-lxml.etree.fromstring(x, parser=parser) # $ input=x vuln='Billion Laughs' vuln='Quadratic Blowup' vuln='XXE'
+lxml.etree.fromstring(x, parser=parser) # $ xmlInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' xmlVuln='XXE'
# Safe for both Billion laughs and XXE
parser = lxml.etree.XMLParser(resolve_entities=False, huge_tree=True)
-lxml.etree.fromstring(x, parser=parser) # $ input=x
+lxml.etree.fromstring(x, parser=parser) # $ xmlInput=x
# DTD retrival vuln (also XXE)
parser = lxml.etree.XMLParser(load_dtd=True, no_network=False)
-lxml.etree.fromstring(x, parser=parser) # $ input=x vuln='DTD retrieval' vuln='XXE'
+lxml.etree.fromstring(x, parser=parser) # $ xmlInput=x xmlVuln='DTD retrieval' xmlVuln='XXE'
diff --git a/python/ql/test/experimental/library-tests/frameworks/XML/xml_dom.py b/python/ql/test/experimental/library-tests/frameworks/XML/xml_dom.py
index 7dce29fc7b9..b86770b8d6c 100644
--- a/python/ql/test/experimental/library-tests/frameworks/XML/xml_dom.py
+++ b/python/ql/test/experimental/library-tests/frameworks/XML/xml_dom.py
@@ -6,26 +6,26 @@ import xml.sax
x = "some xml"
# minidom
-xml.dom.minidom.parse(StringIO(x)) # $ input=StringIO(..) vuln='Billion Laughs' vuln='Quadratic Blowup'
-xml.dom.minidom.parse(file=StringIO(x)) # $ input=StringIO(..) vuln='Billion Laughs' vuln='Quadratic Blowup'
+xml.dom.minidom.parse(StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+xml.dom.minidom.parse(file=StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
-xml.dom.minidom.parseString(x) # $ input=x vuln='Billion Laughs' vuln='Quadratic Blowup'
-xml.dom.minidom.parseString(string=x) # $ input=x vuln='Billion Laughs' vuln='Quadratic Blowup'
+xml.dom.minidom.parseString(x) # $ xmlInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+xml.dom.minidom.parseString(string=x) # $ xmlInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
# pulldom
-xml.dom.pulldom.parse(StringIO(x))['START_DOCUMENT'][1] # $ input=StringIO(..) vuln='Billion Laughs' vuln='Quadratic Blowup'
-xml.dom.pulldom.parse(stream_or_string=StringIO(x))['START_DOCUMENT'][1] # $ input=StringIO(..) vuln='Billion Laughs' vuln='Quadratic Blowup'
+xml.dom.pulldom.parse(StringIO(x))['START_DOCUMENT'][1] # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+xml.dom.pulldom.parse(stream_or_string=StringIO(x))['START_DOCUMENT'][1] # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
-xml.dom.pulldom.parseString(x)['START_DOCUMENT'][1] # $ input=x vuln='Billion Laughs' vuln='Quadratic Blowup'
-xml.dom.pulldom.parseString(string=x)['START_DOCUMENT'][1] # $ input=x vuln='Billion Laughs' vuln='Quadratic Blowup'
+xml.dom.pulldom.parseString(x)['START_DOCUMENT'][1] # $ xmlInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+xml.dom.pulldom.parseString(string=x)['START_DOCUMENT'][1] # $ xmlInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
# These are based on SAX parses, and you can specify your own, so you can expose yourself to XXE (yay/)
parser = xml.sax.make_parser()
parser.setFeature(xml.sax.handler.feature_external_ges, True)
-xml.dom.minidom.parse(StringIO(x), parser) # $ input=StringIO(..) vuln='Billion Laughs' vuln='DTD retrieval' vuln='Quadratic Blowup' vuln='XXE'
-xml.dom.minidom.parse(StringIO(x), parser=parser) # $ input=StringIO(..) vuln='Billion Laughs' vuln='DTD retrieval' vuln='Quadratic Blowup' vuln='XXE'
+xml.dom.minidom.parse(StringIO(x), parser) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE'
+xml.dom.minidom.parse(StringIO(x), parser=parser) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE'
-xml.dom.pulldom.parse(StringIO(x), parser) # $ input=StringIO(..) vuln='Billion Laughs' vuln='DTD retrieval' vuln='Quadratic Blowup' vuln='XXE'
-xml.dom.pulldom.parse(StringIO(x), parser=parser) # $ input=StringIO(..) vuln='Billion Laughs' vuln='DTD retrieval' vuln='Quadratic Blowup' vuln='XXE'
+xml.dom.pulldom.parse(StringIO(x), parser) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE'
+xml.dom.pulldom.parse(StringIO(x), parser=parser) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE'
diff --git a/python/ql/test/experimental/library-tests/frameworks/XML/xml_etree.py b/python/ql/test/experimental/library-tests/frameworks/XML/xml_etree.py
index df126e458e2..c5d141a3715 100644
--- a/python/ql/test/experimental/library-tests/frameworks/XML/xml_etree.py
+++ b/python/ql/test/experimental/library-tests/frameworks/XML/xml_etree.py
@@ -4,39 +4,39 @@ import xml.etree.ElementTree
x = "some xml"
# Parsing in different ways
-xml.etree.ElementTree.fromstring(x) # $ input=x vuln='Billion Laughs' vuln='Quadratic Blowup'
-xml.etree.ElementTree.fromstring(text=x) # $ input=x vuln='Billion Laughs' vuln='Quadratic Blowup'
+xml.etree.ElementTree.fromstring(x) # $ xmlInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+xml.etree.ElementTree.fromstring(text=x) # $ xmlInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
-xml.etree.ElementTree.fromstringlist([x]) # $ input=List vuln='Billion Laughs' vuln='Quadratic Blowup'
-xml.etree.ElementTree.fromstringlist(sequence=[x]) # $ input=List vuln='Billion Laughs' vuln='Quadratic Blowup'
+xml.etree.ElementTree.fromstringlist([x]) # $ xmlInput=List xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+xml.etree.ElementTree.fromstringlist(sequence=[x]) # $ xmlInput=List xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
-xml.etree.ElementTree.XML(x) # $ input=x vuln='Billion Laughs' vuln='Quadratic Blowup'
-xml.etree.ElementTree.XML(text=x) # $ input=x vuln='Billion Laughs' vuln='Quadratic Blowup'
+xml.etree.ElementTree.XML(x) # $ xmlInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+xml.etree.ElementTree.XML(text=x) # $ xmlInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
-xml.etree.ElementTree.XMLID(x) # $ input=x vuln='Billion Laughs' vuln='Quadratic Blowup'
-xml.etree.ElementTree.XMLID(text=x) # $ input=x vuln='Billion Laughs' vuln='Quadratic Blowup'
+xml.etree.ElementTree.XMLID(x) # $ xmlInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+xml.etree.ElementTree.XMLID(text=x) # $ xmlInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
-xml.etree.ElementTree.parse(StringIO(x)) # $ input=StringIO(..) vuln='Billion Laughs' vuln='Quadratic Blowup'
-xml.etree.ElementTree.parse(source=StringIO(x)) # $ input=StringIO(..) vuln='Billion Laughs' vuln='Quadratic Blowup'
+xml.etree.ElementTree.parse(StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+xml.etree.ElementTree.parse(source=StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
-xml.etree.ElementTree.iterparse(StringIO(x)) # $ input=StringIO(..) vuln='Billion Laughs' vuln='Quadratic Blowup'
-xml.etree.ElementTree.iterparse(source=StringIO(x)) # $ input=StringIO(..) vuln='Billion Laughs' vuln='Quadratic Blowup'
+xml.etree.ElementTree.iterparse(StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+xml.etree.ElementTree.iterparse(source=StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
# With parsers (no options available to disable/enable security features)
parser = xml.etree.ElementTree.XMLParser()
-xml.etree.ElementTree.fromstring(x, parser=parser) # $ input=x vuln='Billion Laughs' vuln='Quadratic Blowup'
+xml.etree.ElementTree.fromstring(x, parser=parser) # $ xmlInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
# manual use of feed method
parser = xml.etree.ElementTree.XMLParser()
-parser.feed(x) # $ input=x vuln='Billion Laughs' vuln='Quadratic Blowup'
-parser.feed(data=x) # $ input=x vuln='Billion Laughs' vuln='Quadratic Blowup'
+parser.feed(x) # $ xmlInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+parser.feed(data=x) # $ xmlInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
parser.close()
# manual use of feed method on XMLPullParser
parser = xml.etree.ElementTree.XMLPullParser()
-parser.feed(x) # $ input=x vuln='Billion Laughs' vuln='Quadratic Blowup'
-parser.feed(data=x) # $ input=x vuln='Billion Laughs' vuln='Quadratic Blowup'
+parser.feed(x) # $ xmlInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+parser.feed(data=x) # $ xmlInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
parser.close()
# note: it's technically possible to use the thing wrapper func `fromstring` with an
diff --git a/python/ql/test/experimental/library-tests/frameworks/XML/xml_sax.py b/python/ql/test/experimental/library-tests/frameworks/XML/xml_sax.py
index 158e62ffae6..c0e5923c5c0 100644
--- a/python/ql/test/experimental/library-tests/frameworks/XML/xml_sax.py
+++ b/python/ql/test/experimental/library-tests/frameworks/XML/xml_sax.py
@@ -10,41 +10,41 @@ class MainHandler(xml.sax.ContentHandler):
def characters(self, data):
self._result.append(data)
-xml.sax.parse(StringIO(x)) # $ input=StringIO(..) vuln='Billion Laughs' vuln='Quadratic Blowup'
-xml.sax.parse(source=StringIO(x)) # $ input=StringIO(..) vuln='Billion Laughs' vuln='Quadratic Blowup'
+xml.sax.parse(StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+xml.sax.parse(source=StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
-xml.sax.parseString(x) # $ input=x vuln='Billion Laughs' vuln='Quadratic Blowup'
-xml.sax.parseString(string=x) # $ input=x vuln='Billion Laughs' vuln='Quadratic Blowup'
+xml.sax.parseString(x) # $ xmlInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+xml.sax.parseString(string=x) # $ xmlInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
parser = xml.sax.make_parser()
-parser.parse(StringIO(x)) # $ input=StringIO(..) vuln='Billion Laughs' vuln='Quadratic Blowup'
-parser.parse(source=StringIO(x)) # $ input=StringIO(..) vuln='Billion Laughs' vuln='Quadratic Blowup'
+parser.parse(StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+parser.parse(source=StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
# You can make it vuln to both XXE and DTD retrieval by setting this flag
# see https://docs.python.org/3/library/xml.sax.handler.html#xml.sax.handler.feature_external_ges
parser = xml.sax.make_parser()
parser.setFeature(xml.sax.handler.feature_external_ges, True)
-parser.parse(StringIO(x)) # $ input=StringIO(..) vuln='Billion Laughs' vuln='DTD retrieval' vuln='Quadratic Blowup' vuln='XXE'
+parser.parse(StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE'
parser = xml.sax.make_parser()
parser.setFeature(xml.sax.handler.feature_external_ges, False)
-parser.parse(StringIO(x)) # $ input=StringIO(..) vuln='Billion Laughs' vuln='Quadratic Blowup'
+parser.parse(StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
# Forward Type Tracking test
def func(cond):
parser = xml.sax.make_parser()
if cond:
parser.setFeature(xml.sax.handler.feature_external_ges, True)
- parser.parse(StringIO(x)) # $ input=StringIO(..) vuln='Billion Laughs' vuln='DTD retrieval' vuln='Quadratic Blowup' vuln='XXE'
+ parser.parse(StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE'
else:
- parser.parse(StringIO(x)) # $ input=StringIO(..) vuln='Billion Laughs' vuln='Quadratic Blowup'
+ parser.parse(StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
# make it vuln, then making it safe
# a bit of an edge-case, but is nice to be able to handle.
parser = xml.sax.make_parser()
parser.setFeature(xml.sax.handler.feature_external_ges, True)
parser.setFeature(xml.sax.handler.feature_external_ges, False)
-parser.parse(StringIO(x)) # $ input=StringIO(..) vuln='Billion Laughs' vuln='Quadratic Blowup'
+parser.parse(StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
def check_conditional_assignment(cond):
parser = xml.sax.make_parser()
@@ -52,7 +52,7 @@ def check_conditional_assignment(cond):
parser.setFeature(xml.sax.handler.feature_external_ges, True)
else:
parser.setFeature(xml.sax.handler.feature_external_ges, False)
- parser.parse(StringIO(x)) # $ input=StringIO(..) vuln='Billion Laughs' vuln='DTD retrieval' vuln='Quadratic Blowup' vuln='XXE'
+ parser.parse(StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE'
def check_conditional_assignment2(cond):
parser = xml.sax.make_parser()
@@ -61,4 +61,4 @@ def check_conditional_assignment2(cond):
else:
flag_value = False
parser.setFeature(xml.sax.handler.feature_external_ges, flag_value)
- parser.parse(StringIO(x)) # $ input=StringIO(..) vuln='Billion Laughs' vuln='DTD retrieval' vuln='Quadratic Blowup' vuln='XXE'
+ parser.parse(StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE'
diff --git a/python/ql/test/experimental/library-tests/frameworks/XML/xmltodict.py b/python/ql/test/experimental/library-tests/frameworks/XML/xmltodict.py
index 473e51c9fe6..27d04862f83 100644
--- a/python/ql/test/experimental/library-tests/frameworks/XML/xmltodict.py
+++ b/python/ql/test/experimental/library-tests/frameworks/XML/xmltodict.py
@@ -2,7 +2,7 @@ import xmltodict
x = "some xml"
-xmltodict.parse(x) # $ input=x
-xmltodict.parse(xml_input=x) # $ input=x
+xmltodict.parse(x) # $ xmlInput=x
+xmltodict.parse(xml_input=x) # $ xmlInput=x
-xmltodict.parse(x, disable_entities=False) # $ input=x vuln='Billion Laughs' vuln='Quadratic Blowup'
+xmltodict.parse(x, disable_entities=False) # $ xmlInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
diff --git a/python/ql/test/experimental/meta/ConceptsTest.qll b/python/ql/test/experimental/meta/ConceptsTest.qll
index 8f9435f633f..e9f71356963 100644
--- a/python/ql/test/experimental/meta/ConceptsTest.qll
+++ b/python/ql/test/experimental/meta/ConceptsTest.qll
@@ -539,3 +539,30 @@ class HttpClientRequestTest extends InlineExpectationsTest {
)
}
}
+
+class XmlParsingTest extends InlineExpectationsTest {
+ XmlParsingTest() { this = "XmlParsingTest" }
+
+ override string getARelevantTag() { result in ["xmlInput", "xmlVuln"] }
+
+ override predicate hasActualResult(Location location, string element, string tag, string value) {
+ exists(location.getFile().getRelativePath()) and
+ exists(XML::XMLParsing parsing |
+ exists(DataFlow::Node input |
+ input = parsing.getAnInput() and
+ location = input.getLocation() and
+ element = input.toString() and
+ value = prettyNodeForInlineTest(input) and
+ tag = "xmlInput"
+ )
+ or
+ exists(XML::XMLParsingVulnerabilityKind kind |
+ parsing.vulnerableTo(kind) and
+ location = parsing.getLocation() and
+ element = parsing.toString() and
+ value = "'" + kind + "'" and
+ tag = "xmlVuln"
+ )
+ )
+ }
+}
From 1ea4bcc59f4ccbfe02e454ae3223a8fa34ac33e3 Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Tue, 29 Mar 2022 16:48:30 +0200
Subject: [PATCH 056/171] Python: Make `XMLParsing` a `Decoding` subclass
---
python/ql/lib/semmle/python/Concepts.qll | 16 ++----
.../semmle/python/frameworks/Xml.qll | 52 +++++++++++++++++++
.../frameworks/XML/lxml_etree.py | 42 +++++++--------
.../library-tests/frameworks/XML/xml_dom.py | 24 ++++-----
.../library-tests/frameworks/XML/xml_etree.py | 38 +++++++-------
.../library-tests/frameworks/XML/xml_sax.py | 26 +++++-----
.../library-tests/frameworks/XML/xmltodict.py | 6 +--
.../test/experimental/meta/ConceptsTest.qll | 8 ---
8 files changed, 124 insertions(+), 88 deletions(-)
diff --git a/python/ql/lib/semmle/python/Concepts.qll b/python/ql/lib/semmle/python/Concepts.qll
index c430594d05b..b553c8d927d 100644
--- a/python/ql/lib/semmle/python/Concepts.qll
+++ b/python/ql/lib/semmle/python/Concepts.qll
@@ -580,12 +580,7 @@ module XML {
* Extend this class to model new APIs. If you want to refine existing API models,
* extend `XMLParsing` instead.
*/
- class XMLParsing extends DataFlow::Node instanceof XMLParsing::Range {
- /**
- * Gets the argument containing the content to parse.
- */
- DataFlow::Node getAnInput() { result = super.getAnInput() }
-
+ class XMLParsing extends Decoding instanceof XMLParsing::Range {
/**
* Holds if this XML parsing is vulnerable to `kind`.
*/
@@ -600,16 +595,13 @@ module XML {
* Extend this class to model new APIs. If you want to refine existing API models,
* extend `XMLParsing` instead.
*/
- abstract class Range extends DataFlow::Node {
- /**
- * Gets the argument containing the content to parse.
- */
- abstract DataFlow::Node getAnInput();
-
+ abstract class Range extends Decoding::Range {
/**
* Holds if this XML parsing is vulnerable to `kind`.
*/
abstract predicate vulnerableTo(XMLParsingVulnerabilityKind kind);
+
+ override string getFormat() { result = "XML" }
}
}
}
diff --git a/python/ql/src/experimental/semmle/python/frameworks/Xml.qll b/python/ql/src/experimental/semmle/python/frameworks/Xml.qll
index 4987e24bce4..c072295c461 100644
--- a/python/ql/src/experimental/semmle/python/frameworks/Xml.qll
+++ b/python/ql/src/experimental/semmle/python/frameworks/Xml.qll
@@ -69,6 +69,15 @@ private module XmlEtree {
override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
kind.isBillionLaughs() or kind.isQuadraticBlowup()
}
+
+ override predicate mayExecuteInput() { none() }
+
+ override DataFlow::Node getOutput() {
+ exists(DataFlow::Node objRef |
+ DataFlow::localFlow(this.getObject(), objRef) and
+ result.(DataFlow::MethodCallNode).calls(objRef, "close")
+ )
+ }
}
}
@@ -108,6 +117,10 @@ private module XmlEtree {
// change the security features anyway :|
kind.isBillionLaughs() or kind.isQuadraticBlowup()
}
+
+ override predicate mayExecuteInput() { none() }
+
+ override DataFlow::Node getOutput() { result = this }
}
}
@@ -226,6 +239,15 @@ private module SaxBasedParsing {
this.getObject() = saxParserWithFeatureExternalGesTurnedOn() and
(kind.isXxe() or kind.isDtdRetrieval())
}
+
+ override predicate mayExecuteInput() { none() }
+
+ override DataFlow::Node getOutput() {
+ // note: the output of parsing with SAX is that the content handler gets the
+ // data... but we don't currently model this (it's not trivial to do, and won't
+ // really give us any value, at least not as of right now).
+ none()
+ }
}
/**
@@ -259,6 +281,15 @@ private module SaxBasedParsing {
this.getObject() = saxParserWithFeatureExternalGesTurnedOn() and
(kind.isXxe() or kind.isDtdRetrieval())
}
+
+ override predicate mayExecuteInput() { none() }
+
+ override DataFlow::Node getOutput() {
+ // note: the output of parsing with SAX is that the content handler gets the
+ // data... but we don't currently model this (it's not trivial to do, and won't
+ // really give us any value, at least not as of right now).
+ none()
+ }
}
/**
@@ -296,6 +327,10 @@ private module SaxBasedParsing {
or
(kind.isBillionLaughs() or kind.isQuadraticBlowup())
}
+
+ override predicate mayExecuteInput() { none() }
+
+ override DataFlow::Node getOutput() { result = this }
}
}
@@ -400,6 +435,15 @@ private module Lxml {
override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
this.calls(instanceVulnerableTo(kind), "feed")
}
+
+ override predicate mayExecuteInput() { none() }
+
+ override DataFlow::Node getOutput() {
+ exists(DataFlow::Node objRef |
+ DataFlow::localFlow(this.getObject(), objRef) and
+ result.(DataFlow::MethodCallNode).calls(objRef, "close")
+ )
+ }
}
}
@@ -442,6 +486,10 @@ private module Lxml {
kind.isXxe() and
not exists(this.getParserArg())
}
+
+ override predicate mayExecuteInput() { none() }
+
+ override DataFlow::Node getOutput() { result = this }
}
}
@@ -460,5 +508,9 @@ private module Xmltodict {
(kind.isBillionLaughs() or kind.isQuadraticBlowup()) and
this.getArgByName("disable_entities").getALocalSource().asExpr() = any(False f)
}
+
+ override predicate mayExecuteInput() { none() }
+
+ override DataFlow::Node getOutput() { result = this }
}
}
diff --git a/python/ql/test/experimental/library-tests/frameworks/XML/lxml_etree.py b/python/ql/test/experimental/library-tests/frameworks/XML/lxml_etree.py
index ee8f3fc69c1..f1dbd5390ad 100644
--- a/python/ql/test/experimental/library-tests/frameworks/XML/lxml_etree.py
+++ b/python/ql/test/experimental/library-tests/frameworks/XML/lxml_etree.py
@@ -4,51 +4,51 @@ import lxml.etree
x = "some xml"
# different parsing methods
-lxml.etree.fromstring(x) # $ xmlInput=x xmlVuln='XXE'
-lxml.etree.fromstring(text=x) # $ xmlInput=x xmlVuln='XXE'
+lxml.etree.fromstring(x) # $ decodeFormat=XML decodeInput=x xmlVuln='XXE' decodeOutput=lxml.etree.fromstring(..)
+lxml.etree.fromstring(text=x) # $ decodeFormat=XML decodeInput=x xmlVuln='XXE' decodeOutput=lxml.etree.fromstring(..)
-lxml.etree.fromstringlist([x]) # $ xmlInput=List xmlVuln='XXE'
-lxml.etree.fromstringlist(strings=[x]) # $ xmlInput=List xmlVuln='XXE'
+lxml.etree.fromstringlist([x]) # $ decodeFormat=XML decodeInput=List xmlVuln='XXE' decodeOutput=lxml.etree.fromstringlist(..)
+lxml.etree.fromstringlist(strings=[x]) # $ decodeFormat=XML decodeInput=List xmlVuln='XXE' decodeOutput=lxml.etree.fromstringlist(..)
-lxml.etree.XML(x) # $ xmlInput=x xmlVuln='XXE'
-lxml.etree.XML(text=x) # $ xmlInput=x xmlVuln='XXE'
+lxml.etree.XML(x) # $ decodeFormat=XML decodeInput=x xmlVuln='XXE' decodeOutput=lxml.etree.XML(..)
+lxml.etree.XML(text=x) # $ decodeFormat=XML decodeInput=x xmlVuln='XXE' decodeOutput=lxml.etree.XML(..)
-lxml.etree.parse(StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='XXE'
-lxml.etree.parse(source=StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='XXE'
+lxml.etree.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='XXE' decodeOutput=lxml.etree.parse(..)
+lxml.etree.parse(source=StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='XXE' decodeOutput=lxml.etree.parse(..)
-lxml.etree.parseid(StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='XXE'
-lxml.etree.parseid(source=StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='XXE'
+lxml.etree.parseid(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='XXE' decodeOutput=lxml.etree.parseid(..)
+lxml.etree.parseid(source=StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='XXE' decodeOutput=lxml.etree.parseid(..)
# With default parsers (nothing changed)
parser = lxml.etree.XMLParser()
-lxml.etree.fromstring(x, parser=parser) # $ xmlInput=x xmlVuln='XXE'
+lxml.etree.fromstring(x, parser=parser) # $ decodeFormat=XML decodeInput=x xmlVuln='XXE' decodeOutput=lxml.etree.fromstring(..)
parser = lxml.etree.get_default_parser()
-lxml.etree.fromstring(x, parser=parser) # $ xmlInput=x xmlVuln='XXE'
+lxml.etree.fromstring(x, parser=parser) # $ decodeFormat=XML decodeInput=x xmlVuln='XXE' decodeOutput=lxml.etree.fromstring(..)
# manual use of feed method
parser = lxml.etree.XMLParser()
-parser.feed(x) # $ xmlInput=x xmlVuln='XXE'
-parser.feed(data=x) # $ xmlInput=x xmlVuln='XXE'
-parser.close()
+parser.feed(x) # $ decodeFormat=XML decodeInput=x xmlVuln='XXE'
+parser.feed(data=x) # $ decodeFormat=XML decodeInput=x xmlVuln='XXE'
+parser.close() # $ decodeOutput=parser.close()
# XXE-safe
parser = lxml.etree.XMLParser(resolve_entities=False)
-lxml.etree.fromstring(x, parser) # $ xmlInput=x
-lxml.etree.fromstring(x, parser=parser) # $ xmlInput=x
+lxml.etree.fromstring(x, parser) # $ decodeFormat=XML decodeInput=x decodeOutput=lxml.etree.fromstring(..)
+lxml.etree.fromstring(x, parser=parser) # $ decodeFormat=XML decodeInput=x decodeOutput=lxml.etree.fromstring(..)
# XXE-vuln
parser = lxml.etree.XMLParser(resolve_entities=True)
-lxml.etree.fromstring(x, parser=parser) # $ xmlInput=x xmlVuln='XXE'
+lxml.etree.fromstring(x, parser=parser) # $ decodeFormat=XML decodeInput=x xmlVuln='XXE' decodeOutput=lxml.etree.fromstring(..)
# Billion laughs vuln (also XXE)
parser = lxml.etree.XMLParser(huge_tree=True)
-lxml.etree.fromstring(x, parser=parser) # $ xmlInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' xmlVuln='XXE'
+lxml.etree.fromstring(x, parser=parser) # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' xmlVuln='XXE' decodeOutput=lxml.etree.fromstring(..)
# Safe for both Billion laughs and XXE
parser = lxml.etree.XMLParser(resolve_entities=False, huge_tree=True)
-lxml.etree.fromstring(x, parser=parser) # $ xmlInput=x
+lxml.etree.fromstring(x, parser=parser) # $ decodeFormat=XML decodeInput=x decodeOutput=lxml.etree.fromstring(..)
# DTD retrival vuln (also XXE)
parser = lxml.etree.XMLParser(load_dtd=True, no_network=False)
-lxml.etree.fromstring(x, parser=parser) # $ xmlInput=x xmlVuln='DTD retrieval' xmlVuln='XXE'
+lxml.etree.fromstring(x, parser=parser) # $ decodeFormat=XML decodeInput=x xmlVuln='DTD retrieval' xmlVuln='XXE' decodeOutput=lxml.etree.fromstring(..)
diff --git a/python/ql/test/experimental/library-tests/frameworks/XML/xml_dom.py b/python/ql/test/experimental/library-tests/frameworks/XML/xml_dom.py
index b86770b8d6c..c6152c75807 100644
--- a/python/ql/test/experimental/library-tests/frameworks/XML/xml_dom.py
+++ b/python/ql/test/experimental/library-tests/frameworks/XML/xml_dom.py
@@ -6,26 +6,26 @@ import xml.sax
x = "some xml"
# minidom
-xml.dom.minidom.parse(StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
-xml.dom.minidom.parse(file=StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+xml.dom.minidom.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.dom.minidom.parse(..)
+xml.dom.minidom.parse(file=StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.dom.minidom.parse(..)
-xml.dom.minidom.parseString(x) # $ xmlInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
-xml.dom.minidom.parseString(string=x) # $ xmlInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+xml.dom.minidom.parseString(x) # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.dom.minidom.parseString(..)
+xml.dom.minidom.parseString(string=x) # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.dom.minidom.parseString(..)
# pulldom
-xml.dom.pulldom.parse(StringIO(x))['START_DOCUMENT'][1] # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
-xml.dom.pulldom.parse(stream_or_string=StringIO(x))['START_DOCUMENT'][1] # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+xml.dom.pulldom.parse(StringIO(x))['START_DOCUMENT'][1] # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.dom.pulldom.parse(..)
+xml.dom.pulldom.parse(stream_or_string=StringIO(x))['START_DOCUMENT'][1] # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.dom.pulldom.parse(..)
-xml.dom.pulldom.parseString(x)['START_DOCUMENT'][1] # $ xmlInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
-xml.dom.pulldom.parseString(string=x)['START_DOCUMENT'][1] # $ xmlInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+xml.dom.pulldom.parseString(x)['START_DOCUMENT'][1] # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.dom.pulldom.parseString(..)
+xml.dom.pulldom.parseString(string=x)['START_DOCUMENT'][1] # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.dom.pulldom.parseString(..)
# These are based on SAX parses, and you can specify your own, so you can expose yourself to XXE (yay/)
parser = xml.sax.make_parser()
parser.setFeature(xml.sax.handler.feature_external_ges, True)
-xml.dom.minidom.parse(StringIO(x), parser) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE'
-xml.dom.minidom.parse(StringIO(x), parser=parser) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE'
+xml.dom.minidom.parse(StringIO(x), parser) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE' decodeOutput=xml.dom.minidom.parse(..)
+xml.dom.minidom.parse(StringIO(x), parser=parser) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE' decodeOutput=xml.dom.minidom.parse(..)
-xml.dom.pulldom.parse(StringIO(x), parser) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE'
-xml.dom.pulldom.parse(StringIO(x), parser=parser) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE'
+xml.dom.pulldom.parse(StringIO(x), parser) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE' decodeOutput=xml.dom.pulldom.parse(..)
+xml.dom.pulldom.parse(StringIO(x), parser=parser) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE' decodeOutput=xml.dom.pulldom.parse(..)
diff --git a/python/ql/test/experimental/library-tests/frameworks/XML/xml_etree.py b/python/ql/test/experimental/library-tests/frameworks/XML/xml_etree.py
index c5d141a3715..0ed750ba8c7 100644
--- a/python/ql/test/experimental/library-tests/frameworks/XML/xml_etree.py
+++ b/python/ql/test/experimental/library-tests/frameworks/XML/xml_etree.py
@@ -4,40 +4,40 @@ import xml.etree.ElementTree
x = "some xml"
# Parsing in different ways
-xml.etree.ElementTree.fromstring(x) # $ xmlInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
-xml.etree.ElementTree.fromstring(text=x) # $ xmlInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+xml.etree.ElementTree.fromstring(x) # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.etree.ElementTree.fromstring(..)
+xml.etree.ElementTree.fromstring(text=x) # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.etree.ElementTree.fromstring(..)
-xml.etree.ElementTree.fromstringlist([x]) # $ xmlInput=List xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
-xml.etree.ElementTree.fromstringlist(sequence=[x]) # $ xmlInput=List xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+xml.etree.ElementTree.fromstringlist([x]) # $ decodeFormat=XML decodeInput=List xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.etree.ElementTree.fromstringlist(..)
+xml.etree.ElementTree.fromstringlist(sequence=[x]) # $ decodeFormat=XML decodeInput=List xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.etree.ElementTree.fromstringlist(..)
-xml.etree.ElementTree.XML(x) # $ xmlInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
-xml.etree.ElementTree.XML(text=x) # $ xmlInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+xml.etree.ElementTree.XML(x) # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.etree.ElementTree.XML(..)
+xml.etree.ElementTree.XML(text=x) # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.etree.ElementTree.XML(..)
-xml.etree.ElementTree.XMLID(x) # $ xmlInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
-xml.etree.ElementTree.XMLID(text=x) # $ xmlInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+xml.etree.ElementTree.XMLID(x) # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.etree.ElementTree.XMLID(..)
+xml.etree.ElementTree.XMLID(text=x) # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.etree.ElementTree.XMLID(..)
-xml.etree.ElementTree.parse(StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
-xml.etree.ElementTree.parse(source=StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+xml.etree.ElementTree.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.etree.ElementTree.parse(..)
+xml.etree.ElementTree.parse(source=StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.etree.ElementTree.parse(..)
-xml.etree.ElementTree.iterparse(StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
-xml.etree.ElementTree.iterparse(source=StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+xml.etree.ElementTree.iterparse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.etree.ElementTree.iterparse(..)
+xml.etree.ElementTree.iterparse(source=StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.etree.ElementTree.iterparse(..)
# With parsers (no options available to disable/enable security features)
parser = xml.etree.ElementTree.XMLParser()
-xml.etree.ElementTree.fromstring(x, parser=parser) # $ xmlInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+xml.etree.ElementTree.fromstring(x, parser=parser) # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.etree.ElementTree.fromstring(..)
# manual use of feed method
parser = xml.etree.ElementTree.XMLParser()
-parser.feed(x) # $ xmlInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
-parser.feed(data=x) # $ xmlInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
-parser.close()
+parser.feed(x) # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+parser.feed(data=x) # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+parser.close() # $ decodeOutput=parser.close()
# manual use of feed method on XMLPullParser
parser = xml.etree.ElementTree.XMLPullParser()
-parser.feed(x) # $ xmlInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
-parser.feed(data=x) # $ xmlInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
-parser.close()
+parser.feed(x) # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+parser.feed(data=x) # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+parser.close() # $ decodeOutput=parser.close()
# note: it's technically possible to use the thing wrapper func `fromstring` with an
# `lxml` parser, and thereby change what vulnerabilities you are exposed to.. but it
diff --git a/python/ql/test/experimental/library-tests/frameworks/XML/xml_sax.py b/python/ql/test/experimental/library-tests/frameworks/XML/xml_sax.py
index c0e5923c5c0..8dbe9d4ae99 100644
--- a/python/ql/test/experimental/library-tests/frameworks/XML/xml_sax.py
+++ b/python/ql/test/experimental/library-tests/frameworks/XML/xml_sax.py
@@ -10,41 +10,41 @@ class MainHandler(xml.sax.ContentHandler):
def characters(self, data):
self._result.append(data)
-xml.sax.parse(StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
-xml.sax.parse(source=StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+xml.sax.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+xml.sax.parse(source=StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
-xml.sax.parseString(x) # $ xmlInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
-xml.sax.parseString(string=x) # $ xmlInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+xml.sax.parseString(x) # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+xml.sax.parseString(string=x) # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
parser = xml.sax.make_parser()
-parser.parse(StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
-parser.parse(source=StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+parser.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+parser.parse(source=StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
# You can make it vuln to both XXE and DTD retrieval by setting this flag
# see https://docs.python.org/3/library/xml.sax.handler.html#xml.sax.handler.feature_external_ges
parser = xml.sax.make_parser()
parser.setFeature(xml.sax.handler.feature_external_ges, True)
-parser.parse(StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE'
+parser.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE'
parser = xml.sax.make_parser()
parser.setFeature(xml.sax.handler.feature_external_ges, False)
-parser.parse(StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+parser.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
# Forward Type Tracking test
def func(cond):
parser = xml.sax.make_parser()
if cond:
parser.setFeature(xml.sax.handler.feature_external_ges, True)
- parser.parse(StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE'
+ parser.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE'
else:
- parser.parse(StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+ parser.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
# make it vuln, then making it safe
# a bit of an edge-case, but is nice to be able to handle.
parser = xml.sax.make_parser()
parser.setFeature(xml.sax.handler.feature_external_ges, True)
parser.setFeature(xml.sax.handler.feature_external_ges, False)
-parser.parse(StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+parser.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
def check_conditional_assignment(cond):
parser = xml.sax.make_parser()
@@ -52,7 +52,7 @@ def check_conditional_assignment(cond):
parser.setFeature(xml.sax.handler.feature_external_ges, True)
else:
parser.setFeature(xml.sax.handler.feature_external_ges, False)
- parser.parse(StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE'
+ parser.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE'
def check_conditional_assignment2(cond):
parser = xml.sax.make_parser()
@@ -61,4 +61,4 @@ def check_conditional_assignment2(cond):
else:
flag_value = False
parser.setFeature(xml.sax.handler.feature_external_ges, flag_value)
- parser.parse(StringIO(x)) # $ xmlInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE'
+ parser.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE'
diff --git a/python/ql/test/experimental/library-tests/frameworks/XML/xmltodict.py b/python/ql/test/experimental/library-tests/frameworks/XML/xmltodict.py
index 27d04862f83..01dc2f3c484 100644
--- a/python/ql/test/experimental/library-tests/frameworks/XML/xmltodict.py
+++ b/python/ql/test/experimental/library-tests/frameworks/XML/xmltodict.py
@@ -2,7 +2,7 @@ import xmltodict
x = "some xml"
-xmltodict.parse(x) # $ xmlInput=x
-xmltodict.parse(xml_input=x) # $ xmlInput=x
+xmltodict.parse(x) # $ decodeFormat=XML decodeInput=x decodeOutput=xmltodict.parse(..)
+xmltodict.parse(xml_input=x) # $ decodeFormat=XML decodeInput=x decodeOutput=xmltodict.parse(..)
-xmltodict.parse(x, disable_entities=False) # $ xmlInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+xmltodict.parse(x, disable_entities=False) # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xmltodict.parse(..)
diff --git a/python/ql/test/experimental/meta/ConceptsTest.qll b/python/ql/test/experimental/meta/ConceptsTest.qll
index e9f71356963..24cbbab2d44 100644
--- a/python/ql/test/experimental/meta/ConceptsTest.qll
+++ b/python/ql/test/experimental/meta/ConceptsTest.qll
@@ -548,14 +548,6 @@ class XmlParsingTest extends InlineExpectationsTest {
override predicate hasActualResult(Location location, string element, string tag, string value) {
exists(location.getFile().getRelativePath()) and
exists(XML::XMLParsing parsing |
- exists(DataFlow::Node input |
- input = parsing.getAnInput() and
- location = input.getLocation() and
- element = input.toString() and
- value = prettyNodeForInlineTest(input) and
- tag = "xmlInput"
- )
- or
exists(XML::XMLParsingVulnerabilityKind kind |
parsing.vulnerableTo(kind) and
location = parsing.getLocation() and
From c4473c5f6506e6dcb8e6736f7d3ddd0acea022d4 Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Thu, 31 Mar 2022 10:08:02 +0200
Subject: [PATCH 057/171] Python: Rename lxml XPath tests
---
.../ql/test/library-tests/frameworks/lxml/{test.py => xpath.py} | 0
1 file changed, 0 insertions(+), 0 deletions(-)
rename python/ql/test/library-tests/frameworks/lxml/{test.py => xpath.py} (100%)
diff --git a/python/ql/test/library-tests/frameworks/lxml/test.py b/python/ql/test/library-tests/frameworks/lxml/xpath.py
similarity index 100%
rename from python/ql/test/library-tests/frameworks/lxml/test.py
rename to python/ql/test/library-tests/frameworks/lxml/xpath.py
From 3040adfd9bdc26a0c54ef04453a1c8b2420bb4c5 Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Thu, 31 Mar 2022 10:08:26 +0200
Subject: [PATCH 058/171] Python: Handle `XMLParser().close()` for XPath
---
.../ql/lib/semmle/python/frameworks/Lxml.qll | 26 ++++++++++++++-----
1 file changed, 19 insertions(+), 7 deletions(-)
diff --git a/python/ql/lib/semmle/python/frameworks/Lxml.qll b/python/ql/lib/semmle/python/frameworks/Lxml.qll
index 9259668a5c8..ab29f33e7cf 100644
--- a/python/ql/lib/semmle/python/frameworks/Lxml.qll
+++ b/python/ql/lib/semmle/python/frameworks/Lxml.qll
@@ -57,13 +57,25 @@ private module Lxml {
*/
class XPathCall extends XML::XPathExecution::Range, DataFlow::CallCfgNode {
XPathCall() {
- this =
- API::moduleImport("lxml")
- .getMember("etree")
- .getMember(["parse", "fromstring", "fromstringlist", "HTML", "XML"])
- .getReturn()
- .getMember("xpath")
- .getACall()
+ exists(API::Node parseResult |
+ parseResult =
+ API::moduleImport("lxml")
+ .getMember("etree")
+ .getMember(["parse", "fromstring", "fromstringlist", "HTML", "XML"])
+ .getReturn()
+ or
+ // TODO: lxml.etree.parseid()[0] will contain the root element from parsing
+ // but we don't really have a way to model that nicely.
+ parseResult =
+ API::moduleImport("lxml")
+ .getMember("etree")
+ .getMember("XMLParser")
+ .getReturn()
+ .getMember("close")
+ .getReturn()
+ |
+ this = parseResult.getMember("xpath").getACall()
+ )
}
override DataFlow::Node getXPath() { result in [this.getArg(0), this.getArgByName("_path")] }
From 80b5cde3a2d3123029630450e41475f89253938c Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Thu, 31 Mar 2022 10:19:08 +0200
Subject: [PATCH 059/171] Python: Promote lxml parsing modeling
---
.../ql/lib/semmle/python/frameworks/Lxml.qll | 163 ++++++++++++++++++
.../semmle/python/frameworks/Xml.qll | 159 -----------------
.../frameworks/lxml/parsing.py} | 0
.../library-tests/frameworks/lxml/xpath.py | 8 +-
4 files changed, 167 insertions(+), 163 deletions(-)
rename python/ql/test/{experimental/library-tests/frameworks/XML/lxml_etree.py => library-tests/frameworks/lxml/parsing.py} (100%)
diff --git a/python/ql/lib/semmle/python/frameworks/Lxml.qll b/python/ql/lib/semmle/python/frameworks/Lxml.qll
index ab29f33e7cf..de89345a7d6 100644
--- a/python/ql/lib/semmle/python/frameworks/Lxml.qll
+++ b/python/ql/lib/semmle/python/frameworks/Lxml.qll
@@ -19,6 +19,9 @@ private import semmle.python.ApiGraphs
* - https://lxml.de/tutorial.html
*/
private module Lxml {
+ // ---------------------------------------------------------------------------
+ // XPath
+ // ---------------------------------------------------------------------------
/**
* A class constructor compiling an XPath expression.
*
@@ -97,4 +100,164 @@ private module Lxml {
override string getName() { result = "lxml.etree" }
}
+
+ // ---------------------------------------------------------------------------
+ // Parsing
+ // ---------------------------------------------------------------------------
+ /**
+ * Provides models for `lxml.etree` parsers.
+ *
+ * See https://lxml.de/apidoc/lxml.etree.html?highlight=xmlparser#lxml.etree.XMLParser
+ */
+ module XMLParser {
+ /**
+ * A source of instances of `lxml.etree` parsers, extend this class to model new instances.
+ *
+ * This can include instantiations of the class, return values from function
+ * calls, or a special parameter that will be set when functions are called by an external
+ * library.
+ *
+ * Use the predicate `XMLParser::instance()` to get references to instances of `lxml.etree` parsers.
+ */
+ abstract class InstanceSource extends DataFlow::LocalSourceNode {
+ /** Holds if this instance is vulnerable to `kind`. */
+ abstract predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind);
+ }
+
+ /**
+ * A call to `lxml.etree.XMLParser`.
+ *
+ * See https://lxml.de/apidoc/lxml.etree.html?highlight=xmlparser#lxml.etree.XMLParser
+ */
+ private class LXMLParser extends InstanceSource, DataFlow::CallCfgNode {
+ LXMLParser() {
+ this = API::moduleImport("lxml").getMember("etree").getMember("XMLParser").getACall()
+ }
+
+ // NOTE: it's not possible to change settings of a parser after constructing it
+ override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
+ kind.isXxe() and
+ (
+ // resolve_entities has default True
+ not exists(this.getArgByName("resolve_entities"))
+ or
+ this.getArgByName("resolve_entities").getALocalSource().asExpr() = any(True t)
+ )
+ or
+ (kind.isBillionLaughs() or kind.isQuadraticBlowup()) and
+ this.getArgByName("huge_tree").getALocalSource().asExpr() = any(True t) and
+ not this.getArgByName("resolve_entities").getALocalSource().asExpr() = any(False t)
+ or
+ kind.isDtdRetrieval() and
+ this.getArgByName("load_dtd").getALocalSource().asExpr() = any(True t) and
+ this.getArgByName("no_network").getALocalSource().asExpr() = any(False t)
+ }
+ }
+
+ /**
+ * A call to `lxml.etree.get_default_parser`.
+ *
+ * See https://lxml.de/apidoc/lxml.etree.html?highlight=xmlparser#lxml.etree.get_default_parser
+ */
+ private class LXMLDefaultParser extends InstanceSource, DataFlow::CallCfgNode {
+ LXMLDefaultParser() {
+ this =
+ API::moduleImport("lxml").getMember("etree").getMember("get_default_parser").getACall()
+ }
+
+ override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
+ // as highlighted by
+ // https://lxml.de/apidoc/lxml.etree.html?highlight=xmlparser#lxml.etree.XMLParser
+ // by default XXE is allow. so as long as the default parser has not been
+ // overridden, the result is also vuln to XXE.
+ kind.isXxe()
+ // TODO: take into account that you can override the default parser with `lxml.etree.set_default_parser`.
+ }
+ }
+
+ /** Gets a reference to an `lxml.etree` parsers instance, with origin in `origin` */
+ private DataFlow::TypeTrackingNode instance(DataFlow::TypeTracker t, InstanceSource origin) {
+ t.start() and
+ result = origin
+ or
+ exists(DataFlow::TypeTracker t2 | result = instance(t2, origin).track(t2, t))
+ }
+
+ /** Gets a reference to an `lxml.etree` parsers instance, with origin in `origin` */
+ DataFlow::Node instance(InstanceSource origin) {
+ instance(DataFlow::TypeTracker::end(), origin).flowsTo(result)
+ }
+
+ /** Gets a reference to an `lxml.etree` parser instance, that is vulnerable to `kind`. */
+ DataFlow::Node instanceVulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
+ exists(InstanceSource origin | result = instance(origin) and origin.vulnerableTo(kind))
+ }
+
+ /**
+ * A call to the `feed` method of an `lxml` parser.
+ */
+ private class LXMLParserFeedCall extends DataFlow::MethodCallNode, XML::XMLParsing::Range {
+ LXMLParserFeedCall() { this.calls(instance(_), "feed") }
+
+ override DataFlow::Node getAnInput() { result in [this.getArg(0), this.getArgByName("data")] }
+
+ override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
+ this.calls(instanceVulnerableTo(kind), "feed")
+ }
+
+ override predicate mayExecuteInput() { none() }
+
+ override DataFlow::Node getOutput() {
+ exists(DataFlow::Node objRef |
+ DataFlow::localFlow(this.getObject(), objRef) and
+ result.(DataFlow::MethodCallNode).calls(objRef, "close")
+ )
+ }
+ }
+ }
+
+ /**
+ * A call to either of:
+ * - `lxml.etree.fromstring`
+ * - `lxml.etree.fromstringlist`
+ * - `lxml.etree.XML`
+ * - `lxml.etree.parse`
+ * - `lxml.etree.parseid`
+ *
+ * See https://lxml.de/apidoc/lxml.etree.html?highlight=parseids#lxml.etree.fromstring
+ */
+ private class LXMLParsing extends DataFlow::CallCfgNode, XML::XMLParsing::Range {
+ LXMLParsing() {
+ this =
+ API::moduleImport("lxml")
+ .getMember("etree")
+ .getMember(["fromstring", "fromstringlist", "XML", "parse", "parseid"])
+ .getACall()
+ }
+
+ override DataFlow::Node getAnInput() {
+ result in [
+ this.getArg(0),
+ // fromstring / XML
+ this.getArgByName("text"),
+ // fromstringlist
+ this.getArgByName("strings"),
+ // parse / parseid
+ this.getArgByName("source"),
+ ]
+ }
+
+ DataFlow::Node getParserArg() { result in [this.getArg(1), this.getArgByName("parser")] }
+
+ override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
+ this.getParserArg() = XMLParser::instanceVulnerableTo(kind)
+ or
+ kind.isXxe() and
+ not exists(this.getParserArg())
+ }
+
+ override predicate mayExecuteInput() { none() }
+
+ override DataFlow::Node getOutput() { result = this }
+ }
}
diff --git a/python/ql/src/experimental/semmle/python/frameworks/Xml.qll b/python/ql/src/experimental/semmle/python/frameworks/Xml.qll
index c072295c461..b31151eed1a 100644
--- a/python/ql/src/experimental/semmle/python/frameworks/Xml.qll
+++ b/python/ql/src/experimental/semmle/python/frameworks/Xml.qll
@@ -334,165 +334,6 @@ private module SaxBasedParsing {
}
}
-private module Lxml {
- /**
- * Provides models for `lxml.etree` parsers.
- *
- * See https://lxml.de/apidoc/lxml.etree.html?highlight=xmlparser#lxml.etree.XMLParser
- */
- module XMLParser {
- /**
- * A source of instances of `lxml.etree` parsers, extend this class to model new instances.
- *
- * This can include instantiations of the class, return values from function
- * calls, or a special parameter that will be set when functions are called by an external
- * library.
- *
- * Use the predicate `XMLParser::instance()` to get references to instances of `lxml.etree` parsers.
- */
- abstract class InstanceSource extends DataFlow::LocalSourceNode {
- /** Holds if this instance is vulnerable to `kind`. */
- abstract predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind);
- }
-
- /**
- * A call to `lxml.etree.XMLParser`.
- *
- * See https://lxml.de/apidoc/lxml.etree.html?highlight=xmlparser#lxml.etree.XMLParser
- */
- private class LXMLParser extends InstanceSource, DataFlow::CallCfgNode {
- LXMLParser() {
- this = API::moduleImport("lxml").getMember("etree").getMember("XMLParser").getACall()
- }
-
- // NOTE: it's not possible to change settings of a parser after constructing it
- override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
- kind.isXxe() and
- (
- // resolve_entities has default True
- not exists(this.getArgByName("resolve_entities"))
- or
- this.getArgByName("resolve_entities").getALocalSource().asExpr() = any(True t)
- )
- or
- (kind.isBillionLaughs() or kind.isQuadraticBlowup()) and
- this.getArgByName("huge_tree").getALocalSource().asExpr() = any(True t) and
- not this.getArgByName("resolve_entities").getALocalSource().asExpr() = any(False t)
- or
- kind.isDtdRetrieval() and
- this.getArgByName("load_dtd").getALocalSource().asExpr() = any(True t) and
- this.getArgByName("no_network").getALocalSource().asExpr() = any(False t)
- }
- }
-
- /**
- * A call to `lxml.etree.get_default_parser`.
- *
- * See https://lxml.de/apidoc/lxml.etree.html?highlight=xmlparser#lxml.etree.get_default_parser
- */
- private class LXMLDefaultParser extends InstanceSource, DataFlow::CallCfgNode {
- LXMLDefaultParser() {
- this =
- API::moduleImport("lxml").getMember("etree").getMember("get_default_parser").getACall()
- }
-
- override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
- // as highlighted by
- // https://lxml.de/apidoc/lxml.etree.html?highlight=xmlparser#lxml.etree.XMLParser
- // by default XXE is allow. so as long as the default parser has not been
- // overridden, the result is also vuln to XXE.
- kind.isXxe()
- // TODO: take into account that you can override the default parser with `lxml.etree.set_default_parser`.
- }
- }
-
- /** Gets a reference to an `lxml.etree` parsers instance, with origin in `origin` */
- private DataFlow::TypeTrackingNode instance(DataFlow::TypeTracker t, InstanceSource origin) {
- t.start() and
- result = origin
- or
- exists(DataFlow::TypeTracker t2 | result = instance(t2, origin).track(t2, t))
- }
-
- /** Gets a reference to an `lxml.etree` parsers instance, with origin in `origin` */
- DataFlow::Node instance(InstanceSource origin) {
- instance(DataFlow::TypeTracker::end(), origin).flowsTo(result)
- }
-
- /** Gets a reference to an `lxml.etree` parser instance, that is vulnerable to `kind`. */
- DataFlow::Node instanceVulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
- exists(InstanceSource origin | result = instance(origin) and origin.vulnerableTo(kind))
- }
-
- /**
- * A call to the `feed` method of an `lxml` parser.
- */
- private class LXMLParserFeedCall extends DataFlow::MethodCallNode, XML::XMLParsing::Range {
- LXMLParserFeedCall() { this.calls(instance(_), "feed") }
-
- override DataFlow::Node getAnInput() { result in [this.getArg(0), this.getArgByName("data")] }
-
- override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
- this.calls(instanceVulnerableTo(kind), "feed")
- }
-
- override predicate mayExecuteInput() { none() }
-
- override DataFlow::Node getOutput() {
- exists(DataFlow::Node objRef |
- DataFlow::localFlow(this.getObject(), objRef) and
- result.(DataFlow::MethodCallNode).calls(objRef, "close")
- )
- }
- }
- }
-
- /**
- * A call to either of:
- * - `lxml.etree.fromstring`
- * - `lxml.etree.fromstringlist`
- * - `lxml.etree.XML`
- * - `lxml.etree.parse`
- * - `lxml.etree.parseid`
- *
- * See https://lxml.de/apidoc/lxml.etree.html?highlight=parseids#lxml.etree.fromstring
- */
- private class LXMLParsing extends DataFlow::CallCfgNode, XML::XMLParsing::Range {
- LXMLParsing() {
- this =
- API::moduleImport("lxml")
- .getMember("etree")
- .getMember(["fromstring", "fromstringlist", "XML", "parse", "parseid"])
- .getACall()
- }
-
- override DataFlow::Node getAnInput() {
- result in [
- this.getArg(0),
- // fromstring / XML
- this.getArgByName("text"),
- // fromstringlist
- this.getArgByName("strings"),
- // parse / parseid
- this.getArgByName("source"),
- ]
- }
-
- DataFlow::Node getParserArg() { result in [this.getArg(1), this.getArgByName("parser")] }
-
- override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
- this.getParserArg() = XMLParser::instanceVulnerableTo(kind)
- or
- kind.isXxe() and
- not exists(this.getParserArg())
- }
-
- override predicate mayExecuteInput() { none() }
-
- override DataFlow::Node getOutput() { result = this }
- }
-}
-
private module Xmltodict {
/**
* A call to `xmltodict.parse`.
diff --git a/python/ql/test/experimental/library-tests/frameworks/XML/lxml_etree.py b/python/ql/test/library-tests/frameworks/lxml/parsing.py
similarity index 100%
rename from python/ql/test/experimental/library-tests/frameworks/XML/lxml_etree.py
rename to python/ql/test/library-tests/frameworks/lxml/parsing.py
diff --git a/python/ql/test/library-tests/frameworks/lxml/xpath.py b/python/ql/test/library-tests/frameworks/lxml/xpath.py
index e8ce583503a..9cf3a0883bd 100644
--- a/python/ql/test/library-tests/frameworks/lxml/xpath.py
+++ b/python/ql/test/library-tests/frameworks/lxml/xpath.py
@@ -2,20 +2,20 @@ from lxml import etree
from io import StringIO
def test_parse():
- tree = etree.parse(StringIO(' '))
+ tree = etree.parse(StringIO(' ')) # $ decodeFormat=XML decodeInput=StringIO(..) decodeOutput=etree.parse(..) xmlVuln='XXE'
r = tree.xpath('/foo/bar') # $ getXPath='/foo/bar'
def test_XPath_class():
- root = etree.XML("TEXT ")
+ root = etree.XML("TEXT ") # $ decodeFormat=XML decodeInput="TEXT " decodeOutput=etree.XML(..) xmlVuln='XXE'
find_text = etree.XPath("path") # $ constructedXPath="path"
text = find_text(root)[0]
def test_ETXpath_class():
- root = etree.XML("TEXT ")
+ root = etree.XML("TEXT ") # $ decodeFormat=XML decodeInput="TEXT " decodeOutput=etree.XML(..) xmlVuln='XXE'
find_text = etree.ETXPath("path") # $ constructedXPath="path"
text = find_text(root)[0]
def test_XPathEvaluator_class():
- root = etree.XML("TEXT ")
+ root = etree.XML("TEXT ") # $ decodeFormat=XML decodeInput="TEXT " decodeOutput=etree.XML(..) xmlVuln='XXE'
search_root = etree.XPathEvaluator(root)
text = search_root("path")[0] # $ getXPath="path"
From 7f5f7679f8f9f14db7fac551bfb6071c08c41767 Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Thu, 31 Mar 2022 10:28:34 +0200
Subject: [PATCH 060/171] Python: Promote `xmltodict` modeling
---
docs/codeql/support/reusables/frameworks.rst | 1 +
python/ql/lib/semmle/python/Frameworks.qll | 1 +
.../semmle/python/frameworks/Xmltodict.qll | 39 +++++++++++++++++++
.../semmle/python/frameworks/Xml.qll | 22 -----------
.../xmltodict/ConceptsTest.expected | 0
.../frameworks/xmltodict/ConceptsTest.ql | 2 +
.../frameworks/xmltodict/test.py} | 0
7 files changed, 43 insertions(+), 22 deletions(-)
create mode 100644 python/ql/lib/semmle/python/frameworks/Xmltodict.qll
create mode 100644 python/ql/test/library-tests/frameworks/xmltodict/ConceptsTest.expected
create mode 100644 python/ql/test/library-tests/frameworks/xmltodict/ConceptsTest.ql
rename python/ql/test/{experimental/library-tests/frameworks/XML/xmltodict.py => library-tests/frameworks/xmltodict/test.py} (100%)
diff --git a/docs/codeql/support/reusables/frameworks.rst b/docs/codeql/support/reusables/frameworks.rst
index 93280c6732a..12bcd5af8e6 100644
--- a/docs/codeql/support/reusables/frameworks.rst
+++ b/docs/codeql/support/reusables/frameworks.rst
@@ -214,3 +214,4 @@ Python built-in support
libtaxii, TAXII utility library
libxml2, XML processing library
lxml, XML processing library
+ xmltodict, XML processing library
diff --git a/python/ql/lib/semmle/python/Frameworks.qll b/python/ql/lib/semmle/python/Frameworks.qll
index b94b8aee5d9..4812628d262 100644
--- a/python/ql/lib/semmle/python/Frameworks.qll
+++ b/python/ql/lib/semmle/python/Frameworks.qll
@@ -52,3 +52,4 @@ private import semmle.python.frameworks.Ujson
private import semmle.python.frameworks.Urllib3
private import semmle.python.frameworks.Yaml
private import semmle.python.frameworks.Yarl
+private import semmle.python.frameworks.Xmltodict
diff --git a/python/ql/lib/semmle/python/frameworks/Xmltodict.qll b/python/ql/lib/semmle/python/frameworks/Xmltodict.qll
new file mode 100644
index 00000000000..bb65607251f
--- /dev/null
+++ b/python/ql/lib/semmle/python/frameworks/Xmltodict.qll
@@ -0,0 +1,39 @@
+/**
+ * Provides classes modeling security-relevant aspects of the `xmltodict` PyPI package.
+ *
+ * See
+ * - https://pypi.org/project/xmltodict/
+ */
+
+private import python
+private import semmle.python.dataflow.new.DataFlow
+private import semmle.python.Concepts
+private import semmle.python.ApiGraphs
+
+/**
+ * Provides classes modeling security-relevant aspects of the `xmltodict` PyPI package
+ *
+ * See
+ * - https://pypi.org/project/xmltodict/
+ */
+private module Xmltodict {
+ /**
+ * A call to `xmltodict.parse`.
+ */
+ private class XMLtoDictParsing extends DataFlow::CallCfgNode, XML::XMLParsing::Range {
+ XMLtoDictParsing() { this = API::moduleImport("xmltodict").getMember("parse").getACall() }
+
+ override DataFlow::Node getAnInput() {
+ result in [this.getArg(0), this.getArgByName("xml_input")]
+ }
+
+ override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
+ (kind.isBillionLaughs() or kind.isQuadraticBlowup()) and
+ this.getArgByName("disable_entities").getALocalSource().asExpr() = any(False f)
+ }
+
+ override predicate mayExecuteInput() { none() }
+
+ override DataFlow::Node getOutput() { result = this }
+ }
+}
diff --git a/python/ql/src/experimental/semmle/python/frameworks/Xml.qll b/python/ql/src/experimental/semmle/python/frameworks/Xml.qll
index b31151eed1a..c98370ba85a 100644
--- a/python/ql/src/experimental/semmle/python/frameworks/Xml.qll
+++ b/python/ql/src/experimental/semmle/python/frameworks/Xml.qll
@@ -333,25 +333,3 @@ private module SaxBasedParsing {
override DataFlow::Node getOutput() { result = this }
}
}
-
-private module Xmltodict {
- /**
- * A call to `xmltodict.parse`.
- */
- private class XMLtoDictParsing extends DataFlow::CallCfgNode, XML::XMLParsing::Range {
- XMLtoDictParsing() { this = API::moduleImport("xmltodict").getMember("parse").getACall() }
-
- override DataFlow::Node getAnInput() {
- result in [this.getArg(0), this.getArgByName("xml_input")]
- }
-
- override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
- (kind.isBillionLaughs() or kind.isQuadraticBlowup()) and
- this.getArgByName("disable_entities").getALocalSource().asExpr() = any(False f)
- }
-
- override predicate mayExecuteInput() { none() }
-
- override DataFlow::Node getOutput() { result = this }
- }
-}
diff --git a/python/ql/test/library-tests/frameworks/xmltodict/ConceptsTest.expected b/python/ql/test/library-tests/frameworks/xmltodict/ConceptsTest.expected
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/python/ql/test/library-tests/frameworks/xmltodict/ConceptsTest.ql b/python/ql/test/library-tests/frameworks/xmltodict/ConceptsTest.ql
new file mode 100644
index 00000000000..b557a0bccb6
--- /dev/null
+++ b/python/ql/test/library-tests/frameworks/xmltodict/ConceptsTest.ql
@@ -0,0 +1,2 @@
+import python
+import experimental.meta.ConceptsTest
diff --git a/python/ql/test/experimental/library-tests/frameworks/XML/xmltodict.py b/python/ql/test/library-tests/frameworks/xmltodict/test.py
similarity index 100%
rename from python/ql/test/experimental/library-tests/frameworks/XML/xmltodict.py
rename to python/ql/test/library-tests/frameworks/xmltodict/test.py
From 64aa503cc3b6374744efbaa2d6f4c322d03a3faa Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Thu, 31 Mar 2022 10:42:05 +0200
Subject: [PATCH 061/171] Python: Promote `xml.etree` modeling
---
.../lib/semmle/python/frameworks/Stdlib.qll | 117 +++++++++++++++++
.../semmle/python/frameworks/Xml.qll | 118 +-----------------
.../frameworks/stdlib/XPathExecution.py | 2 +-
.../frameworks/stdlib}/xml_etree.py | 0
4 files changed, 119 insertions(+), 118 deletions(-)
rename python/ql/test/{experimental/library-tests/frameworks/XML => library-tests/frameworks/stdlib}/xml_etree.py (100%)
diff --git a/python/ql/lib/semmle/python/frameworks/Stdlib.qll b/python/ql/lib/semmle/python/frameworks/Stdlib.qll
index 234a8802f0f..263cdfcd0b3 100644
--- a/python/ql/lib/semmle/python/frameworks/Stdlib.qll
+++ b/python/ql/lib/semmle/python/frameworks/Stdlib.qll
@@ -3174,6 +3174,123 @@ private module StdlibPrivate {
}
}
}
+
+ // ---------------------------------------------------------------------------
+ // xml.etree
+ // ---------------------------------------------------------------------------
+ /**
+ * Provides models for `xml.etree` parsers
+ *
+ * See
+ * - https://docs.python.org/3.10/library/xml.etree.elementtree.html#xml.etree.ElementTree.XMLParser
+ * - https://docs.python.org/3.10/library/xml.etree.elementtree.html#xml.etree.ElementTree.XMLPullParser
+ */
+ module XMLParser {
+ /**
+ * A source of instances of `xml.etree` parsers, extend this class to model new instances.
+ *
+ * This can include instantiations of the class, return values from function
+ * calls, or a special parameter that will be set when functions are called by an external
+ * library.
+ *
+ * Use the predicate `XMLParser::instance()` to get references to instances of `xml.etree` parsers.
+ */
+ abstract class InstanceSource extends DataFlow::LocalSourceNode { }
+
+ /** A direct instantiation of `xml.etree` parsers. */
+ private class ClassInstantiation extends InstanceSource, DataFlow::CallCfgNode {
+ ClassInstantiation() {
+ this =
+ API::moduleImport("xml")
+ .getMember("etree")
+ .getMember("ElementTree")
+ .getMember("XMLParser")
+ .getACall()
+ or
+ this =
+ API::moduleImport("xml")
+ .getMember("etree")
+ .getMember("ElementTree")
+ .getMember("XMLPullParser")
+ .getACall()
+ }
+ }
+
+ /** Gets a reference to an `xml.etree` parser instance. */
+ private DataFlow::TypeTrackingNode instance(DataFlow::TypeTracker t) {
+ t.start() and
+ result instanceof InstanceSource
+ or
+ exists(DataFlow::TypeTracker t2 | result = instance(t2).track(t2, t))
+ }
+
+ /** Gets a reference to an `xml.etree` parser instance. */
+ DataFlow::Node instance() { instance(DataFlow::TypeTracker::end()).flowsTo(result) }
+
+ /**
+ * A call to the `feed` method of an `xml.etree` parser.
+ */
+ private class XMLEtreeParserFeedCall extends DataFlow::MethodCallNode, XML::XMLParsing::Range {
+ XMLEtreeParserFeedCall() { this.calls(instance(), "feed") }
+
+ override DataFlow::Node getAnInput() { result in [this.getArg(0), this.getArgByName("data")] }
+
+ override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
+ kind.isBillionLaughs() or kind.isQuadraticBlowup()
+ }
+
+ override predicate mayExecuteInput() { none() }
+
+ override DataFlow::Node getOutput() {
+ exists(DataFlow::Node objRef |
+ DataFlow::localFlow(this.getObject(), objRef) and
+ result.(DataFlow::MethodCallNode).calls(objRef, "close")
+ )
+ }
+ }
+ }
+
+ /**
+ * A call to either of:
+ * - `xml.etree.ElementTree.fromstring`
+ * - `xml.etree.ElementTree.fromstringlist`
+ * - `xml.etree.ElementTree.XML`
+ * - `xml.etree.ElementTree.XMLID`
+ * - `xml.etree.ElementTree.parse`
+ * - `xml.etree.ElementTree.iterparse`
+ */
+ private class XMLEtreeParsing extends DataFlow::CallCfgNode, XML::XMLParsing::Range {
+ XMLEtreeParsing() {
+ this =
+ API::moduleImport("xml")
+ .getMember("etree")
+ .getMember("ElementTree")
+ .getMember(["fromstring", "fromstringlist", "XML", "XMLID", "parse", "iterparse"])
+ .getACall()
+ }
+
+ override DataFlow::Node getAnInput() {
+ result in [
+ this.getArg(0),
+ // fromstring / XML / XMLID
+ this.getArgByName("text"),
+ // fromstringlist
+ this.getArgByName("sequence"),
+ // parse / iterparse
+ this.getArgByName("source"),
+ ]
+ }
+
+ override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
+ // note: it does not matter what `xml.etree` parser you are using, you cannot
+ // change the security features anyway :|
+ kind.isBillionLaughs() or kind.isQuadraticBlowup()
+ }
+
+ override predicate mayExecuteInput() { none() }
+
+ override DataFlow::Node getOutput() { result = this }
+ }
}
// ---------------------------------------------------------------------------
diff --git a/python/ql/src/experimental/semmle/python/frameworks/Xml.qll b/python/ql/src/experimental/semmle/python/frameworks/Xml.qll
index c98370ba85a..88def863824 100644
--- a/python/ql/src/experimental/semmle/python/frameworks/Xml.qll
+++ b/python/ql/src/experimental/semmle/python/frameworks/Xml.qll
@@ -8,129 +8,13 @@ private import semmle.python.dataflow.new.DataFlow
private import semmle.python.Concepts
private import semmle.python.ApiGraphs
-private module XmlEtree {
- /**
- * Provides models for `xml.etree` parsers
- *
- * See
- * - https://docs.python.org/3.10/library/xml.etree.elementtree.html#xml.etree.ElementTree.XMLParser
- * - https://docs.python.org/3.10/library/xml.etree.elementtree.html#xml.etree.ElementTree.XMLPullParser
- */
- module XMLParser {
- /**
- * A source of instances of `xml.etree` parsers, extend this class to model new instances.
- *
- * This can include instantiations of the class, return values from function
- * calls, or a special parameter that will be set when functions are called by an external
- * library.
- *
- * Use the predicate `XMLParser::instance()` to get references to instances of `xml.etree` parsers.
- */
- abstract class InstanceSource extends DataFlow::LocalSourceNode { }
-
- /** A direct instantiation of `xml.etree` parsers. */
- private class ClassInstantiation extends InstanceSource, DataFlow::CallCfgNode {
- ClassInstantiation() {
- this =
- API::moduleImport("xml")
- .getMember("etree")
- .getMember("ElementTree")
- .getMember("XMLParser")
- .getACall()
- or
- this =
- API::moduleImport("xml")
- .getMember("etree")
- .getMember("ElementTree")
- .getMember("XMLPullParser")
- .getACall()
- }
- }
-
- /** Gets a reference to an `xml.etree` parser instance. */
- private DataFlow::TypeTrackingNode instance(DataFlow::TypeTracker t) {
- t.start() and
- result instanceof InstanceSource
- or
- exists(DataFlow::TypeTracker t2 | result = instance(t2).track(t2, t))
- }
-
- /** Gets a reference to an `xml.etree` parser instance. */
- DataFlow::Node instance() { instance(DataFlow::TypeTracker::end()).flowsTo(result) }
-
- /**
- * A call to the `feed` method of an `xml.etree` parser.
- */
- private class XMLEtreeParserFeedCall extends DataFlow::MethodCallNode, XML::XMLParsing::Range {
- XMLEtreeParserFeedCall() { this.calls(instance(), "feed") }
-
- override DataFlow::Node getAnInput() { result in [this.getArg(0), this.getArgByName("data")] }
-
- override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
- kind.isBillionLaughs() or kind.isQuadraticBlowup()
- }
-
- override predicate mayExecuteInput() { none() }
-
- override DataFlow::Node getOutput() {
- exists(DataFlow::Node objRef |
- DataFlow::localFlow(this.getObject(), objRef) and
- result.(DataFlow::MethodCallNode).calls(objRef, "close")
- )
- }
- }
- }
-
- /**
- * A call to either of:
- * - `xml.etree.ElementTree.fromstring`
- * - `xml.etree.ElementTree.fromstringlist`
- * - `xml.etree.ElementTree.XML`
- * - `xml.etree.ElementTree.XMLID`
- * - `xml.etree.ElementTree.parse`
- * - `xml.etree.ElementTree.iterparse`
- */
- private class XMLEtreeParsing extends DataFlow::CallCfgNode, XML::XMLParsing::Range {
- XMLEtreeParsing() {
- this =
- API::moduleImport("xml")
- .getMember("etree")
- .getMember("ElementTree")
- .getMember(["fromstring", "fromstringlist", "XML", "XMLID", "parse", "iterparse"])
- .getACall()
- }
-
- override DataFlow::Node getAnInput() {
- result in [
- this.getArg(0),
- // fromstring / XML / XMLID
- this.getArgByName("text"),
- // fromstringlist
- this.getArgByName("sequence"),
- // parse / iterparse
- this.getArgByName("source"),
- ]
- }
-
- override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
- // note: it does not matter what `xml.etree` parser you are using, you cannot
- // change the security features anyway :|
- kind.isBillionLaughs() or kind.isQuadraticBlowup()
- }
-
- override predicate mayExecuteInput() { none() }
-
- override DataFlow::Node getOutput() { result = this }
- }
-}
-
private module SaxBasedParsing {
/**
* A call to the `setFeature` method on a XML sax parser.
*
* See https://docs.python.org/3.10/library/xml.sax.reader.html#xml.sax.xmlreader.XMLReader.setFeature
*/
- class SaxParserSetFeatureCall extends DataFlow::MethodCallNode {
+ private class SaxParserSetFeatureCall extends DataFlow::MethodCallNode {
SaxParserSetFeatureCall() {
this =
API::moduleImport("xml")
diff --git a/python/ql/test/library-tests/frameworks/stdlib/XPathExecution.py b/python/ql/test/library-tests/frameworks/stdlib/XPathExecution.py
index 98bdaefac27..d39b0e04888 100644
--- a/python/ql/test/library-tests/frameworks/stdlib/XPathExecution.py
+++ b/python/ql/test/library-tests/frameworks/stdlib/XPathExecution.py
@@ -2,7 +2,7 @@ match = "dc:title"
ns = {'dc': 'http://purl.org/dc/elements/1.1/'}
import xml.etree.ElementTree as ET
-tree = ET.parse('country_data.xml')
+tree = ET.parse('country_data.xml') # $ decodeFormat=XML decodeInput='country_data.xml' decodeOutput=ET.parse(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
root = tree.getroot()
root.find(match, namespaces=ns) # $ getXPath=match
diff --git a/python/ql/test/experimental/library-tests/frameworks/XML/xml_etree.py b/python/ql/test/library-tests/frameworks/stdlib/xml_etree.py
similarity index 100%
rename from python/ql/test/experimental/library-tests/frameworks/XML/xml_etree.py
rename to python/ql/test/library-tests/frameworks/stdlib/xml_etree.py
From a315aa84b2bdfad3cd3196336bbc1bc6fc658415 Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Thu, 31 Mar 2022 11:13:12 +0200
Subject: [PATCH 062/171] Python: Add some links in QLDocs
---
python/ql/lib/semmle/python/frameworks/Lxml.qll | 7 ++++++-
python/ql/lib/semmle/python/frameworks/Stdlib.qll | 8 ++++++++
2 files changed, 14 insertions(+), 1 deletion(-)
diff --git a/python/ql/lib/semmle/python/frameworks/Lxml.qll b/python/ql/lib/semmle/python/frameworks/Lxml.qll
index de89345a7d6..e1052efbf99 100644
--- a/python/ql/lib/semmle/python/frameworks/Lxml.qll
+++ b/python/ql/lib/semmle/python/frameworks/Lxml.qll
@@ -224,7 +224,12 @@ private module Lxml {
* - `lxml.etree.parse`
* - `lxml.etree.parseid`
*
- * See https://lxml.de/apidoc/lxml.etree.html?highlight=parseids#lxml.etree.fromstring
+ * See
+ * - https://lxml.de/apidoc/lxml.etree.html?highlight=parseids#lxml.etree.fromstring
+ * - https://lxml.de/apidoc/lxml.etree.html?highlight=parseids#lxml.etree.fromstringlist
+ * - https://lxml.de/apidoc/lxml.etree.html?highlight=parseids#lxml.etree.XML
+ * - https://lxml.de/apidoc/lxml.etree.html?highlight=parseids#lxml.etree.parse
+ * - https://lxml.de/apidoc/lxml.etree.html?highlight=parseids#lxml.etree.parseid
*/
private class LXMLParsing extends DataFlow::CallCfgNode, XML::XMLParsing::Range {
LXMLParsing() {
diff --git a/python/ql/lib/semmle/python/frameworks/Stdlib.qll b/python/ql/lib/semmle/python/frameworks/Stdlib.qll
index 263cdfcd0b3..6c8de064852 100644
--- a/python/ql/lib/semmle/python/frameworks/Stdlib.qll
+++ b/python/ql/lib/semmle/python/frameworks/Stdlib.qll
@@ -3258,6 +3258,14 @@ private module StdlibPrivate {
* - `xml.etree.ElementTree.XMLID`
* - `xml.etree.ElementTree.parse`
* - `xml.etree.ElementTree.iterparse`
+ *
+ * See
+ * - https://docs.python.org/3/library/xml.etree.elementtree.html#xml.etree.ElementTree.fromstring
+ * - https://docs.python.org/3/library/xml.etree.elementtree.html#xml.etree.ElementTree.fromstringlist
+ * - https://docs.python.org/3/library/xml.etree.elementtree.html#xml.etree.ElementTree.XML
+ * - https://docs.python.org/3/library/xml.etree.elementtree.html#xml.etree.ElementTree.XMLID
+ * - https://docs.python.org/3/library/xml.etree.elementtree.html#xml.etree.ElementTree.parse
+ * - https://docs.python.org/3/library/xml.etree.elementtree.html#xml.etree.ElementTree.iterparse
*/
private class XMLEtreeParsing extends DataFlow::CallCfgNode, XML::XMLParsing::Range {
XMLEtreeParsing() {
From 6774085e7af76b7faa952d2b23cbc9232a57273d Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Thu, 31 Mar 2022 11:19:25 +0200
Subject: [PATCH 063/171] Python: Add note about parseid/XMLID
---
python/ql/lib/semmle/python/frameworks/Lxml.qll | 9 ++++++++-
python/ql/lib/semmle/python/frameworks/Stdlib.qll | 9 ++++++++-
2 files changed, 16 insertions(+), 2 deletions(-)
diff --git a/python/ql/lib/semmle/python/frameworks/Lxml.qll b/python/ql/lib/semmle/python/frameworks/Lxml.qll
index e1052efbf99..e090b9dbf05 100644
--- a/python/ql/lib/semmle/python/frameworks/Lxml.qll
+++ b/python/ql/lib/semmle/python/frameworks/Lxml.qll
@@ -263,6 +263,13 @@ private module Lxml {
override predicate mayExecuteInput() { none() }
- override DataFlow::Node getOutput() { result = this }
+ override DataFlow::Node getOutput() {
+ // Note: for `parseid` the result of the call is a tuple with `(root, dict)`, so
+ // maybe we should not just say that the entire tuple is the decoding output... my
+ // gut feeling is that THIS instance doesn't matter too much, but that it would be
+ // nice to be able to do this in general. (this is a problem for both `lxml.etree`
+ // and `xml.etree`)
+ result = this
+ }
}
}
diff --git a/python/ql/lib/semmle/python/frameworks/Stdlib.qll b/python/ql/lib/semmle/python/frameworks/Stdlib.qll
index 6c8de064852..77ec1b5f9da 100644
--- a/python/ql/lib/semmle/python/frameworks/Stdlib.qll
+++ b/python/ql/lib/semmle/python/frameworks/Stdlib.qll
@@ -3297,7 +3297,14 @@ private module StdlibPrivate {
override predicate mayExecuteInput() { none() }
- override DataFlow::Node getOutput() { result = this }
+ override DataFlow::Node getOutput() {
+ // Note: for `XMLID` the result of the call is a tuple with `(root, dict)`, so
+ // maybe we should not just say that the entire tuple is the decoding output... my
+ // gut feeling is that THIS instance doesn't matter too much, but that it would be
+ // nice to be able to do this in general. (this is a problem for both `lxml.etree`
+ // and `xml.etree`)
+ result = this
+ }
}
}
From 12cbdcde284e4e8fbce7a02ae0f65cedeee7e4eb Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Thu, 31 Mar 2022 11:21:24 +0200
Subject: [PATCH 064/171] Python: Model `lxml.etree.XMLID`
---
python/ql/lib/semmle/python/frameworks/Lxml.qll | 8 +++++---
python/ql/test/library-tests/frameworks/lxml/parsing.py | 3 +++
2 files changed, 8 insertions(+), 3 deletions(-)
diff --git a/python/ql/lib/semmle/python/frameworks/Lxml.qll b/python/ql/lib/semmle/python/frameworks/Lxml.qll
index e090b9dbf05..60cc850fd34 100644
--- a/python/ql/lib/semmle/python/frameworks/Lxml.qll
+++ b/python/ql/lib/semmle/python/frameworks/Lxml.qll
@@ -221,6 +221,7 @@ private module Lxml {
* - `lxml.etree.fromstring`
* - `lxml.etree.fromstringlist`
* - `lxml.etree.XML`
+ * - `lxml.etree.XMLID`
* - `lxml.etree.parse`
* - `lxml.etree.parseid`
*
@@ -228,6 +229,7 @@ private module Lxml {
* - https://lxml.de/apidoc/lxml.etree.html?highlight=parseids#lxml.etree.fromstring
* - https://lxml.de/apidoc/lxml.etree.html?highlight=parseids#lxml.etree.fromstringlist
* - https://lxml.de/apidoc/lxml.etree.html?highlight=parseids#lxml.etree.XML
+ * - https://lxml.de/apidoc/lxml.etree.html?highlight=parseids#lxml.etree.XMLID
* - https://lxml.de/apidoc/lxml.etree.html?highlight=parseids#lxml.etree.parse
* - https://lxml.de/apidoc/lxml.etree.html?highlight=parseids#lxml.etree.parseid
*/
@@ -236,14 +238,14 @@ private module Lxml {
this =
API::moduleImport("lxml")
.getMember("etree")
- .getMember(["fromstring", "fromstringlist", "XML", "parse", "parseid"])
+ .getMember(["fromstring", "fromstringlist", "XML", "XMLID", "parse", "parseid"])
.getACall()
}
override DataFlow::Node getAnInput() {
result in [
this.getArg(0),
- // fromstring / XML
+ // fromstring / XML / XMLID
this.getArgByName("text"),
// fromstringlist
this.getArgByName("strings"),
@@ -264,7 +266,7 @@ private module Lxml {
override predicate mayExecuteInput() { none() }
override DataFlow::Node getOutput() {
- // Note: for `parseid` the result of the call is a tuple with `(root, dict)`, so
+ // Note: for `parseid`/XMLID the result of the call is a tuple with `(root, dict)`, so
// maybe we should not just say that the entire tuple is the decoding output... my
// gut feeling is that THIS instance doesn't matter too much, but that it would be
// nice to be able to do this in general. (this is a problem for both `lxml.etree`
diff --git a/python/ql/test/library-tests/frameworks/lxml/parsing.py b/python/ql/test/library-tests/frameworks/lxml/parsing.py
index f1dbd5390ad..e69a68a6ad2 100644
--- a/python/ql/test/library-tests/frameworks/lxml/parsing.py
+++ b/python/ql/test/library-tests/frameworks/lxml/parsing.py
@@ -13,6 +13,9 @@ lxml.etree.fromstringlist(strings=[x]) # $ decodeFormat=XML decodeInput=List xml
lxml.etree.XML(x) # $ decodeFormat=XML decodeInput=x xmlVuln='XXE' decodeOutput=lxml.etree.XML(..)
lxml.etree.XML(text=x) # $ decodeFormat=XML decodeInput=x xmlVuln='XXE' decodeOutput=lxml.etree.XML(..)
+lxml.etree.XMLID(x) # $ decodeFormat=XML decodeInput=x xmlVuln='XXE' decodeOutput=lxml.etree.XMLID(..)
+lxml.etree.XMLID(text=x) # $ decodeFormat=XML decodeInput=x xmlVuln='XXE' decodeOutput=lxml.etree.XMLID(..)
+
lxml.etree.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='XXE' decodeOutput=lxml.etree.parse(..)
lxml.etree.parse(source=StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='XXE' decodeOutput=lxml.etree.parse(..)
From 386ff5361415f17c248285300de71ca735e92f7a Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Thu, 31 Mar 2022 11:32:22 +0200
Subject: [PATCH 065/171] Python: Model `lxml.iterparse`
---
.../ql/lib/semmle/python/frameworks/Lxml.qll | 30 +++++++++++++++++++
.../library-tests/frameworks/lxml/parsing.py | 18 ++++++++---
2 files changed, 44 insertions(+), 4 deletions(-)
diff --git a/python/ql/lib/semmle/python/frameworks/Lxml.qll b/python/ql/lib/semmle/python/frameworks/Lxml.qll
index 60cc850fd34..821fc6bac80 100644
--- a/python/ql/lib/semmle/python/frameworks/Lxml.qll
+++ b/python/ql/lib/semmle/python/frameworks/Lxml.qll
@@ -274,4 +274,34 @@ private module Lxml {
result = this
}
}
+
+ /**
+ * A call to `lxml.etree.iterparse`
+ *
+ * See
+ * - https://lxml.de/apidoc/lxml.etree.html?highlight=parseids#lxml.etree.iterparse
+ */
+ private class LXMLIterparseCall extends DataFlow::CallCfgNode, XML::XMLParsing::Range {
+ LXMLIterparseCall() {
+ this = API::moduleImport("lxml").getMember("etree").getMember("iterparse").getACall()
+ }
+
+ override DataFlow::Node getAnInput() { result in [this.getArg(0), this.getArgByName("source")] }
+
+ override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
+ // note that there is no `resolve_entities` argument, so it's not possible to turn off XXE :O
+ kind.isXxe()
+ or
+ (kind.isBillionLaughs() or kind.isQuadraticBlowup()) and
+ this.getArgByName("huge_tree").getALocalSource().asExpr() = any(True t)
+ or
+ kind.isDtdRetrieval() and
+ this.getArgByName("load_dtd").getALocalSource().asExpr() = any(True t) and
+ this.getArgByName("no_network").getALocalSource().asExpr() = any(False t)
+ }
+
+ override predicate mayExecuteInput() { none() }
+
+ override DataFlow::Node getOutput() { result = this }
+ }
}
diff --git a/python/ql/test/library-tests/frameworks/lxml/parsing.py b/python/ql/test/library-tests/frameworks/lxml/parsing.py
index e69a68a6ad2..5abd626caf4 100644
--- a/python/ql/test/library-tests/frameworks/lxml/parsing.py
+++ b/python/ql/test/library-tests/frameworks/lxml/parsing.py
@@ -16,11 +16,15 @@ lxml.etree.XML(text=x) # $ decodeFormat=XML decodeInput=x xmlVuln='XXE' decodeOu
lxml.etree.XMLID(x) # $ decodeFormat=XML decodeInput=x xmlVuln='XXE' decodeOutput=lxml.etree.XMLID(..)
lxml.etree.XMLID(text=x) # $ decodeFormat=XML decodeInput=x xmlVuln='XXE' decodeOutput=lxml.etree.XMLID(..)
-lxml.etree.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='XXE' decodeOutput=lxml.etree.parse(..)
-lxml.etree.parse(source=StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='XXE' decodeOutput=lxml.etree.parse(..)
+xml_file = 'xml_file'
+lxml.etree.parse(xml_file) # $ decodeFormat=XML decodeInput=xml_file xmlVuln='XXE' decodeOutput=lxml.etree.parse(..)
+lxml.etree.parse(source=xml_file) # $ decodeFormat=XML decodeInput=xml_file xmlVuln='XXE' decodeOutput=lxml.etree.parse(..)
-lxml.etree.parseid(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='XXE' decodeOutput=lxml.etree.parseid(..)
-lxml.etree.parseid(source=StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='XXE' decodeOutput=lxml.etree.parseid(..)
+lxml.etree.parseid(xml_file) # $ decodeFormat=XML decodeInput=xml_file xmlVuln='XXE' decodeOutput=lxml.etree.parseid(..)
+lxml.etree.parseid(source=xml_file) # $ decodeFormat=XML decodeInput=xml_file xmlVuln='XXE' decodeOutput=lxml.etree.parseid(..)
+
+lxml.etree.iterparse(xml_file) # $ decodeFormat=XML decodeInput=xml_file xmlVuln='XXE' decodeOutput=lxml.etree.iterparse(..)
+lxml.etree.iterparse(source=xml_file) # $ decodeFormat=XML decodeInput=xml_file xmlVuln='XXE' decodeOutput=lxml.etree.iterparse(..)
# With default parsers (nothing changed)
parser = lxml.etree.XMLParser()
@@ -55,3 +59,9 @@ lxml.etree.fromstring(x, parser=parser) # $ decodeFormat=XML decodeInput=x decod
# DTD retrival vuln (also XXE)
parser = lxml.etree.XMLParser(load_dtd=True, no_network=False)
lxml.etree.fromstring(x, parser=parser) # $ decodeFormat=XML decodeInput=x xmlVuln='DTD retrieval' xmlVuln='XXE' decodeOutput=lxml.etree.fromstring(..)
+
+# iterparse configurations ... this doesn't use a parser argument but takes MOST (!) of
+# the normal XMLParser arguments. Specifically, it doesn't allow disabling XXE :O
+
+lxml.etree.iterparse(xml_file, huge_tree=True) # $ decodeFormat=XML decodeInput=xml_file xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' xmlVuln='XXE' decodeOutput=lxml.etree.iterparse(..)
+lxml.etree.iterparse(xml_file, load_dtd=True, no_network=False) # $ decodeFormat=XML decodeInput=xml_file xmlVuln='DTD retrieval' xmlVuln='XXE' decodeOutput=lxml.etree.iterparse(..)
From 543454eff234ac2d403b932cb82b38309dea8002 Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Thu, 31 Mar 2022 11:47:29 +0200
Subject: [PATCH 066/171] Python: Model file access from XML parsing
---
.../ql/lib/semmle/python/frameworks/Lxml.qll | 29 ++++++++++++++++++-
.../lib/semmle/python/frameworks/Stdlib.qll | 29 +++++++++++++++++++
.../library-tests/frameworks/lxml/parsing.py | 16 +++++-----
.../library-tests/frameworks/lxml/xpath.py | 2 +-
.../frameworks/stdlib/XPathExecution.py | 2 +-
.../frameworks/stdlib/xml_etree.py | 8 ++---
6 files changed, 71 insertions(+), 15 deletions(-)
diff --git a/python/ql/lib/semmle/python/frameworks/Lxml.qll b/python/ql/lib/semmle/python/frameworks/Lxml.qll
index 821fc6bac80..a3825a70db0 100644
--- a/python/ql/lib/semmle/python/frameworks/Lxml.qll
+++ b/python/ql/lib/semmle/python/frameworks/Lxml.qll
@@ -275,13 +275,38 @@ private module Lxml {
}
}
+ /**
+ * A call to `lxml.etree.ElementTree.parse` or `lxml.etree.ElementTree.parseid`, which
+ * takes either a filename or a file-like object as argument. To capture the filename
+ * for path-injection, we have this subclass.
+ *
+ * See
+ * - https://lxml.de/apidoc/lxml.etree.html?highlight=parseids#lxml.etree.parse
+ * - https://lxml.de/apidoc/lxml.etree.html?highlight=parseids#lxml.etree.parseid
+ */
+ private class FileAccessFromLXMLParsing extends LXMLParsing, FileSystemAccess::Range {
+ FileAccessFromLXMLParsing() {
+ this = API::moduleImport("lxml").getMember("etree").getMember(["parse", "parseid"]).getACall()
+ // I considered whether we should try to reduce FPs from people passing file-like
+ // objects, which will not be a file system access (and couldn't cause a
+ // path-injection).
+ //
+ // I suppose that once we have proper flow-summary support for file-like objects,
+ // we can make the XXE/XML-bomb sinks allow an access-path, while the
+ // path-injection sink wouldn't, and then we will not end up with such FPs.
+ }
+
+ override DataFlow::Node getAPathArgument() { result = this.getAnInput() }
+ }
+
/**
* A call to `lxml.etree.iterparse`
*
* See
* - https://lxml.de/apidoc/lxml.etree.html?highlight=parseids#lxml.etree.iterparse
*/
- private class LXMLIterparseCall extends DataFlow::CallCfgNode, XML::XMLParsing::Range {
+ private class LXMLIterparseCall extends DataFlow::CallCfgNode, XML::XMLParsing::Range,
+ FileSystemAccess::Range {
LXMLIterparseCall() {
this = API::moduleImport("lxml").getMember("etree").getMember("iterparse").getACall()
}
@@ -303,5 +328,7 @@ private module Lxml {
override predicate mayExecuteInput() { none() }
override DataFlow::Node getOutput() { result = this }
+
+ override DataFlow::Node getAPathArgument() { result = this.getAnInput() }
}
}
diff --git a/python/ql/lib/semmle/python/frameworks/Stdlib.qll b/python/ql/lib/semmle/python/frameworks/Stdlib.qll
index 77ec1b5f9da..3afbf71f495 100644
--- a/python/ql/lib/semmle/python/frameworks/Stdlib.qll
+++ b/python/ql/lib/semmle/python/frameworks/Stdlib.qll
@@ -3306,6 +3306,35 @@ private module StdlibPrivate {
result = this
}
}
+
+ /**
+ * A call to `xml.etree.ElementTree.parse` or `xml.etree.ElementTree.iterparse`, which
+ * takes either a filename or a file-like object as argument. To capture the filename
+ * for path-injection, we have this subclass.
+ *
+ * See
+ * - https://docs.python.org/3/library/xml.etree.elementtree.html#xml.etree.ElementTree.parse
+ * - https://docs.python.org/3/library/xml.etree.elementtree.html#xml.etree.ElementTree.iterparse
+ */
+ private class FileAccessFromXMLEtreeParsing extends XMLEtreeParsing, FileSystemAccess::Range {
+ FileAccessFromXMLEtreeParsing() {
+ this =
+ API::moduleImport("xml")
+ .getMember("etree")
+ .getMember("ElementTree")
+ .getMember(["parse", "iterparse"])
+ .getACall()
+ // I considered whether we should try to reduce FPs from people passing file-like
+ // objects, which will not be a file system access (and couldn't cause a
+ // path-injection).
+ //
+ // I suppose that once we have proper flow-summary support for file-like objects,
+ // we can make the XXE/XML-bomb sinks allow an access-path, while the
+ // path-injection sink wouldn't, and then we will not end up with such FPs.
+ }
+
+ override DataFlow::Node getAPathArgument() { result = this.getAnInput() }
+ }
}
// ---------------------------------------------------------------------------
diff --git a/python/ql/test/library-tests/frameworks/lxml/parsing.py b/python/ql/test/library-tests/frameworks/lxml/parsing.py
index 5abd626caf4..ca68c99a90e 100644
--- a/python/ql/test/library-tests/frameworks/lxml/parsing.py
+++ b/python/ql/test/library-tests/frameworks/lxml/parsing.py
@@ -17,14 +17,14 @@ lxml.etree.XMLID(x) # $ decodeFormat=XML decodeInput=x xmlVuln='XXE' decodeOutpu
lxml.etree.XMLID(text=x) # $ decodeFormat=XML decodeInput=x xmlVuln='XXE' decodeOutput=lxml.etree.XMLID(..)
xml_file = 'xml_file'
-lxml.etree.parse(xml_file) # $ decodeFormat=XML decodeInput=xml_file xmlVuln='XXE' decodeOutput=lxml.etree.parse(..)
-lxml.etree.parse(source=xml_file) # $ decodeFormat=XML decodeInput=xml_file xmlVuln='XXE' decodeOutput=lxml.etree.parse(..)
+lxml.etree.parse(xml_file) # $ decodeFormat=XML decodeInput=xml_file xmlVuln='XXE' decodeOutput=lxml.etree.parse(..) getAPathArgument=xml_file
+lxml.etree.parse(source=xml_file) # $ decodeFormat=XML decodeInput=xml_file xmlVuln='XXE' decodeOutput=lxml.etree.parse(..) getAPathArgument=xml_file
-lxml.etree.parseid(xml_file) # $ decodeFormat=XML decodeInput=xml_file xmlVuln='XXE' decodeOutput=lxml.etree.parseid(..)
-lxml.etree.parseid(source=xml_file) # $ decodeFormat=XML decodeInput=xml_file xmlVuln='XXE' decodeOutput=lxml.etree.parseid(..)
+lxml.etree.parseid(xml_file) # $ decodeFormat=XML decodeInput=xml_file xmlVuln='XXE' decodeOutput=lxml.etree.parseid(..) getAPathArgument=xml_file
+lxml.etree.parseid(source=xml_file) # $ decodeFormat=XML decodeInput=xml_file xmlVuln='XXE' decodeOutput=lxml.etree.parseid(..) getAPathArgument=xml_file
-lxml.etree.iterparse(xml_file) # $ decodeFormat=XML decodeInput=xml_file xmlVuln='XXE' decodeOutput=lxml.etree.iterparse(..)
-lxml.etree.iterparse(source=xml_file) # $ decodeFormat=XML decodeInput=xml_file xmlVuln='XXE' decodeOutput=lxml.etree.iterparse(..)
+lxml.etree.iterparse(xml_file) # $ decodeFormat=XML decodeInput=xml_file xmlVuln='XXE' decodeOutput=lxml.etree.iterparse(..) getAPathArgument=xml_file
+lxml.etree.iterparse(source=xml_file) # $ decodeFormat=XML decodeInput=xml_file xmlVuln='XXE' decodeOutput=lxml.etree.iterparse(..) getAPathArgument=xml_file
# With default parsers (nothing changed)
parser = lxml.etree.XMLParser()
@@ -63,5 +63,5 @@ lxml.etree.fromstring(x, parser=parser) # $ decodeFormat=XML decodeInput=x xmlVu
# iterparse configurations ... this doesn't use a parser argument but takes MOST (!) of
# the normal XMLParser arguments. Specifically, it doesn't allow disabling XXE :O
-lxml.etree.iterparse(xml_file, huge_tree=True) # $ decodeFormat=XML decodeInput=xml_file xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' xmlVuln='XXE' decodeOutput=lxml.etree.iterparse(..)
-lxml.etree.iterparse(xml_file, load_dtd=True, no_network=False) # $ decodeFormat=XML decodeInput=xml_file xmlVuln='DTD retrieval' xmlVuln='XXE' decodeOutput=lxml.etree.iterparse(..)
+lxml.etree.iterparse(xml_file, huge_tree=True) # $ decodeFormat=XML decodeInput=xml_file xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' xmlVuln='XXE' decodeOutput=lxml.etree.iterparse(..) getAPathArgument=xml_file
+lxml.etree.iterparse(xml_file, load_dtd=True, no_network=False) # $ decodeFormat=XML decodeInput=xml_file xmlVuln='DTD retrieval' xmlVuln='XXE' decodeOutput=lxml.etree.iterparse(..) getAPathArgument=xml_file
diff --git a/python/ql/test/library-tests/frameworks/lxml/xpath.py b/python/ql/test/library-tests/frameworks/lxml/xpath.py
index 9cf3a0883bd..f67c8dae17c 100644
--- a/python/ql/test/library-tests/frameworks/lxml/xpath.py
+++ b/python/ql/test/library-tests/frameworks/lxml/xpath.py
@@ -2,7 +2,7 @@ from lxml import etree
from io import StringIO
def test_parse():
- tree = etree.parse(StringIO(' ')) # $ decodeFormat=XML decodeInput=StringIO(..) decodeOutput=etree.parse(..) xmlVuln='XXE'
+ tree = etree.parse(StringIO(' ')) # $ decodeFormat=XML decodeInput=StringIO(..) decodeOutput=etree.parse(..) xmlVuln='XXE' getAPathArgument=StringIO(..)
r = tree.xpath('/foo/bar') # $ getXPath='/foo/bar'
def test_XPath_class():
diff --git a/python/ql/test/library-tests/frameworks/stdlib/XPathExecution.py b/python/ql/test/library-tests/frameworks/stdlib/XPathExecution.py
index d39b0e04888..b501e2d4ccb 100644
--- a/python/ql/test/library-tests/frameworks/stdlib/XPathExecution.py
+++ b/python/ql/test/library-tests/frameworks/stdlib/XPathExecution.py
@@ -2,7 +2,7 @@ match = "dc:title"
ns = {'dc': 'http://purl.org/dc/elements/1.1/'}
import xml.etree.ElementTree as ET
-tree = ET.parse('country_data.xml') # $ decodeFormat=XML decodeInput='country_data.xml' decodeOutput=ET.parse(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+tree = ET.parse('country_data.xml') # $ decodeFormat=XML decodeInput='country_data.xml' decodeOutput=ET.parse(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' getAPathArgument='country_data.xml'
root = tree.getroot()
root.find(match, namespaces=ns) # $ getXPath=match
diff --git a/python/ql/test/library-tests/frameworks/stdlib/xml_etree.py b/python/ql/test/library-tests/frameworks/stdlib/xml_etree.py
index 0ed750ba8c7..684aaaa4a9c 100644
--- a/python/ql/test/library-tests/frameworks/stdlib/xml_etree.py
+++ b/python/ql/test/library-tests/frameworks/stdlib/xml_etree.py
@@ -16,11 +16,11 @@ xml.etree.ElementTree.XML(text=x) # $ decodeFormat=XML decodeInput=x xmlVuln='Bi
xml.etree.ElementTree.XMLID(x) # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.etree.ElementTree.XMLID(..)
xml.etree.ElementTree.XMLID(text=x) # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.etree.ElementTree.XMLID(..)
-xml.etree.ElementTree.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.etree.ElementTree.parse(..)
-xml.etree.ElementTree.parse(source=StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.etree.ElementTree.parse(..)
+xml.etree.ElementTree.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.etree.ElementTree.parse(..) getAPathArgument=StringIO(..)
+xml.etree.ElementTree.parse(source=StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.etree.ElementTree.parse(..) getAPathArgument=StringIO(..)
-xml.etree.ElementTree.iterparse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.etree.ElementTree.iterparse(..)
-xml.etree.ElementTree.iterparse(source=StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.etree.ElementTree.iterparse(..)
+xml.etree.ElementTree.iterparse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.etree.ElementTree.iterparse(..) getAPathArgument=StringIO(..)
+xml.etree.ElementTree.iterparse(source=StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.etree.ElementTree.iterparse(..) getAPathArgument=StringIO(..)
# With parsers (no options available to disable/enable security features)
From db43d043c4cdd59c65424f20fdcdc0a7d79a632c Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Thu, 31 Mar 2022 11:54:08 +0200
Subject: [PATCH 067/171] Python: Add test showing misalignment of xml.etree
modeling
---
.../test/library-tests/frameworks/stdlib/XPathExecution.py | 5 +++++
python/ql/test/library-tests/frameworks/stdlib/xml_etree.py | 4 ++++
2 files changed, 9 insertions(+)
diff --git a/python/ql/test/library-tests/frameworks/stdlib/XPathExecution.py b/python/ql/test/library-tests/frameworks/stdlib/XPathExecution.py
index b501e2d4ccb..37043d7049c 100644
--- a/python/ql/test/library-tests/frameworks/stdlib/XPathExecution.py
+++ b/python/ql/test/library-tests/frameworks/stdlib/XPathExecution.py
@@ -15,3 +15,8 @@ tree.parse("index.xhtml")
tree.find(match, namespaces=ns) # $ getXPath=match
tree.findall(match, namespaces=ns) # $ getXPath=match
tree.findtext(match, default=None, namespaces=ns) # $ getXPath=match
+
+parser = ET.XMLParser()
+parser.feed("bar ") # $ decodeFormat=XML decodeInput="bar " xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+tree = parser.close() # $ decodeOutput=parser.close()
+tree.find(match, namespaces=ns) # $ MISSING: getXPath=match
diff --git a/python/ql/test/library-tests/frameworks/stdlib/xml_etree.py b/python/ql/test/library-tests/frameworks/stdlib/xml_etree.py
index 684aaaa4a9c..da04cedbdfc 100644
--- a/python/ql/test/library-tests/frameworks/stdlib/xml_etree.py
+++ b/python/ql/test/library-tests/frameworks/stdlib/xml_etree.py
@@ -22,6 +22,10 @@ xml.etree.ElementTree.parse(source=StringIO(x)) # $ decodeFormat=XML decodeInput
xml.etree.ElementTree.iterparse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.etree.ElementTree.iterparse(..) getAPathArgument=StringIO(..)
xml.etree.ElementTree.iterparse(source=StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.etree.ElementTree.iterparse(..) getAPathArgument=StringIO(..)
+tree = xml.etree.ElementTree.ElementTree()
+tree.parse("file.xml") # $ MISSING: decodeFormat=XML decodeInput="file.xml" xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=tree.parse(..) getAPathArgument="file.xml"
+tree.parse(source="file.xml") # $ MISSING: decodeFormat=XML decodeInput="file.xml" xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=tree.parse(..) getAPathArgument="file.xml"
+
# With parsers (no options available to disable/enable security features)
parser = xml.etree.ElementTree.XMLParser()
From 70b3eecdd506fcb2e17f3eb027e7b05073c257df Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Thu, 31 Mar 2022 17:13:11 +0200
Subject: [PATCH 068/171] Python: Merge `xml.etree.ElementTree` models
I forgot about the existing ones when I promoted it
---
.../lib/semmle/python/frameworks/Stdlib.qll | 127 +++++++++---------
1 file changed, 62 insertions(+), 65 deletions(-)
diff --git a/python/ql/lib/semmle/python/frameworks/Stdlib.qll b/python/ql/lib/semmle/python/frameworks/Stdlib.qll
index 3afbf71f495..85cf61cdbaf 100644
--- a/python/ql/lib/semmle/python/frameworks/Stdlib.qll
+++ b/python/ql/lib/semmle/python/frameworks/Stdlib.qll
@@ -2835,70 +2835,6 @@ private module StdlibPrivate {
override string getKind() { result = Escaping::getRegexKind() }
}
- // ---------------------------------------------------------------------------
- // xml.etree.ElementTree
- // ---------------------------------------------------------------------------
- /**
- * An instance of `xml.etree.ElementTree.ElementTree`.
- *
- * See https://docs.python.org/3.10/library/xml.etree.elementtree.html#xml.etree.ElementTree.ElementTree
- */
- private API::Node elementTreeInstance() {
- //parse to a tree
- result =
- API::moduleImport("xml")
- .getMember("etree")
- .getMember("ElementTree")
- .getMember("parse")
- .getReturn()
- or
- // construct a tree without parsing
- result =
- API::moduleImport("xml")
- .getMember("etree")
- .getMember("ElementTree")
- .getMember("ElementTree")
- .getReturn()
- }
-
- /**
- * An instance of `xml.etree.ElementTree.Element`.
- *
- * See https://docs.python.org/3.10/library/xml.etree.elementtree.html#xml.etree.ElementTree.Element
- */
- private API::Node elementInstance() {
- // parse or go to the root of a tree
- result = elementTreeInstance().getMember(["parse", "getroot"]).getReturn()
- or
- // parse directly to an element
- result =
- API::moduleImport("xml")
- .getMember("etree")
- .getMember("ElementTree")
- .getMember(["fromstring", "fromstringlist", "XML"])
- .getReturn()
- }
-
- /**
- * A call to a find method on a tree or an element will execute an XPath expression.
- */
- private class ElementTreeFindCall extends XML::XPathExecution::Range, DataFlow::CallCfgNode {
- string methodName;
-
- ElementTreeFindCall() {
- methodName in ["find", "findall", "findtext"] and
- (
- this = elementTreeInstance().getMember(methodName).getACall()
- or
- this = elementInstance().getMember(methodName).getACall()
- )
- }
-
- override DataFlow::Node getXPath() { result in [this.getArg(0), this.getArgByName("match")] }
-
- override string getName() { result = "xml.etree" }
- }
-
// ---------------------------------------------------------------------------
// urllib
// ---------------------------------------------------------------------------
@@ -3176,8 +3112,69 @@ private module StdlibPrivate {
}
// ---------------------------------------------------------------------------
- // xml.etree
+ // xml.etree.ElementTree
// ---------------------------------------------------------------------------
+ /**
+ * An instance of `xml.etree.ElementTree.ElementTree`.
+ *
+ * See https://docs.python.org/3.10/library/xml.etree.elementtree.html#xml.etree.ElementTree.ElementTree
+ */
+ private API::Node elementTreeInstance() {
+ //parse to a tree
+ result =
+ API::moduleImport("xml")
+ .getMember("etree")
+ .getMember("ElementTree")
+ .getMember("parse")
+ .getReturn()
+ or
+ // construct a tree without parsing
+ result =
+ API::moduleImport("xml")
+ .getMember("etree")
+ .getMember("ElementTree")
+ .getMember("ElementTree")
+ .getReturn()
+ }
+
+ /**
+ * An instance of `xml.etree.ElementTree.Element`.
+ *
+ * See https://docs.python.org/3.10/library/xml.etree.elementtree.html#xml.etree.ElementTree.Element
+ */
+ private API::Node elementInstance() {
+ // parse or go to the root of a tree
+ result = elementTreeInstance().getMember(["parse", "getroot"]).getReturn()
+ or
+ // parse directly to an element
+ result =
+ API::moduleImport("xml")
+ .getMember("etree")
+ .getMember("ElementTree")
+ .getMember(["fromstring", "fromstringlist", "XML"])
+ .getReturn()
+ }
+
+ /**
+ * A call to a find method on a tree or an element will execute an XPath expression.
+ */
+ private class ElementTreeFindCall extends XML::XPathExecution::Range, DataFlow::CallCfgNode {
+ string methodName;
+
+ ElementTreeFindCall() {
+ methodName in ["find", "findall", "findtext"] and
+ (
+ this = elementTreeInstance().getMember(methodName).getACall()
+ or
+ this = elementInstance().getMember(methodName).getACall()
+ )
+ }
+
+ override DataFlow::Node getXPath() { result in [this.getArg(0), this.getArgByName("match")] }
+
+ override string getName() { result = "xml.etree" }
+ }
+
/**
* Provides models for `xml.etree` parsers
*
From 05bb0ef97688627eacd4b6ed247b84a707385ed5 Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Thu, 31 Mar 2022 17:24:16 +0200
Subject: [PATCH 069/171] Python: Align `xml.etree.ElementTree` modeling
I didn't find a good way to actually share the stuff, so we kinda just
have 2 things that look very similar :|
---
python/ql/lib/semmle/python/frameworks/Stdlib.qll | 14 ++++++++++++++
.../frameworks/stdlib/XPathExecution.py | 4 ++--
.../library-tests/frameworks/stdlib/xml_etree.py | 4 ++--
3 files changed, 18 insertions(+), 4 deletions(-)
diff --git a/python/ql/lib/semmle/python/frameworks/Stdlib.qll b/python/ql/lib/semmle/python/frameworks/Stdlib.qll
index 85cf61cdbaf..1118133d215 100644
--- a/python/ql/lib/semmle/python/frameworks/Stdlib.qll
+++ b/python/ql/lib/semmle/python/frameworks/Stdlib.qll
@@ -3153,6 +3153,15 @@ private module StdlibPrivate {
.getMember("ElementTree")
.getMember(["fromstring", "fromstringlist", "XML"])
.getReturn()
+ or
+ result =
+ API::moduleImport("xml")
+ .getMember("etree")
+ .getMember("ElementTree")
+ .getMember("XMLParser")
+ .getReturn()
+ .getMember("close")
+ .getReturn()
}
/**
@@ -3255,6 +3264,7 @@ private module StdlibPrivate {
* - `xml.etree.ElementTree.XMLID`
* - `xml.etree.ElementTree.parse`
* - `xml.etree.ElementTree.iterparse`
+ * - `parse` method on an `xml.etree.ElementTree.ElementTree` instance
*
* See
* - https://docs.python.org/3/library/xml.etree.elementtree.html#xml.etree.ElementTree.fromstring
@@ -3272,6 +3282,8 @@ private module StdlibPrivate {
.getMember("ElementTree")
.getMember(["fromstring", "fromstringlist", "XML", "XMLID", "parse", "iterparse"])
.getACall()
+ or
+ this = elementTreeInstance().getMember("parse").getACall()
}
override DataFlow::Node getAnInput() {
@@ -3321,6 +3333,8 @@ private module StdlibPrivate {
.getMember("ElementTree")
.getMember(["parse", "iterparse"])
.getACall()
+ or
+ this = elementTreeInstance().getMember("parse").getACall()
// I considered whether we should try to reduce FPs from people passing file-like
// objects, which will not be a file system access (and couldn't cause a
// path-injection).
diff --git a/python/ql/test/library-tests/frameworks/stdlib/XPathExecution.py b/python/ql/test/library-tests/frameworks/stdlib/XPathExecution.py
index 37043d7049c..5faff5ed868 100644
--- a/python/ql/test/library-tests/frameworks/stdlib/XPathExecution.py
+++ b/python/ql/test/library-tests/frameworks/stdlib/XPathExecution.py
@@ -10,7 +10,7 @@ root.findall(match, namespaces=ns) # $ getXPath=match
root.findtext(match, default=None, namespaces=ns) # $ getXPath=match
tree = ET.ElementTree()
-tree.parse("index.xhtml")
+tree.parse("index.xhtml") # $ decodeFormat=XML decodeInput="index.xhtml" decodeOutput=tree.parse(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' getAPathArgument="index.xhtml"
tree.find(match, namespaces=ns) # $ getXPath=match
tree.findall(match, namespaces=ns) # $ getXPath=match
@@ -19,4 +19,4 @@ tree.findtext(match, default=None, namespaces=ns) # $ getXPath=match
parser = ET.XMLParser()
parser.feed("bar ") # $ decodeFormat=XML decodeInput="bar " xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
tree = parser.close() # $ decodeOutput=parser.close()
-tree.find(match, namespaces=ns) # $ MISSING: getXPath=match
+tree.find(match, namespaces=ns) # $ getXPath=match
diff --git a/python/ql/test/library-tests/frameworks/stdlib/xml_etree.py b/python/ql/test/library-tests/frameworks/stdlib/xml_etree.py
index da04cedbdfc..00f3b964b18 100644
--- a/python/ql/test/library-tests/frameworks/stdlib/xml_etree.py
+++ b/python/ql/test/library-tests/frameworks/stdlib/xml_etree.py
@@ -23,8 +23,8 @@ xml.etree.ElementTree.iterparse(StringIO(x)) # $ decodeFormat=XML decodeInput=St
xml.etree.ElementTree.iterparse(source=StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.etree.ElementTree.iterparse(..) getAPathArgument=StringIO(..)
tree = xml.etree.ElementTree.ElementTree()
-tree.parse("file.xml") # $ MISSING: decodeFormat=XML decodeInput="file.xml" xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=tree.parse(..) getAPathArgument="file.xml"
-tree.parse(source="file.xml") # $ MISSING: decodeFormat=XML decodeInput="file.xml" xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=tree.parse(..) getAPathArgument="file.xml"
+tree.parse("file.xml") # $ decodeFormat=XML decodeInput="file.xml" xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=tree.parse(..) getAPathArgument="file.xml"
+tree.parse(source="file.xml") # $ decodeFormat=XML decodeInput="file.xml" xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=tree.parse(..) getAPathArgument="file.xml"
# With parsers (no options available to disable/enable security features)
From e11269715dc55e3509625489267601b736c324f1 Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Thu, 31 Mar 2022 17:44:00 +0200
Subject: [PATCH 070/171] Python: Promote `xml.sax` and `xml.dom.*` modeling
---
.../lib/semmle/python/frameworks/Stdlib.qll | 214 ++++++++++++++++++
.../semmle/python/frameworks/Xml.qll | 210 -----------------
.../frameworks/stdlib}/xml_dom.py | 0
.../frameworks/stdlib}/xml_sax.py | 0
4 files changed, 214 insertions(+), 210 deletions(-)
rename python/ql/test/{experimental/library-tests/frameworks/XML => library-tests/frameworks/stdlib}/xml_dom.py (100%)
rename python/ql/test/{experimental/library-tests/frameworks/XML => library-tests/frameworks/stdlib}/xml_sax.py (100%)
diff --git a/python/ql/lib/semmle/python/frameworks/Stdlib.qll b/python/ql/lib/semmle/python/frameworks/Stdlib.qll
index 1118133d215..418f3475c1e 100644
--- a/python/ql/lib/semmle/python/frameworks/Stdlib.qll
+++ b/python/ql/lib/semmle/python/frameworks/Stdlib.qll
@@ -3346,6 +3346,220 @@ private module StdlibPrivate {
override DataFlow::Node getAPathArgument() { result = this.getAnInput() }
}
+
+ // ---------------------------------------------------------------------------
+ // xml.sax
+ // ---------------------------------------------------------------------------
+ /**
+ * A call to the `setFeature` method on a XML sax parser.
+ *
+ * See https://docs.python.org/3.10/library/xml.sax.reader.html#xml.sax.xmlreader.XMLReader.setFeature
+ */
+ private class SaxParserSetFeatureCall extends DataFlow::MethodCallNode {
+ SaxParserSetFeatureCall() {
+ this =
+ API::moduleImport("xml")
+ .getMember("sax")
+ .getMember("make_parser")
+ .getReturn()
+ .getMember("setFeature")
+ .getACall()
+ }
+
+ // The keyword argument names does not match documentation. I checked (with Python
+ // 3.9.5) that the names used here actually works.
+ DataFlow::Node getFeatureArg() { result in [this.getArg(0), this.getArgByName("name")] }
+
+ DataFlow::Node getStateArg() { result in [this.getArg(1), this.getArgByName("state")] }
+ }
+
+ /** Gets a back-reference to the `setFeature` state argument `arg`. */
+ private DataFlow::TypeTrackingNode saxParserSetFeatureStateArgBacktracker(
+ DataFlow::TypeBackTracker t, DataFlow::Node arg
+ ) {
+ t.start() and
+ arg = any(SaxParserSetFeatureCall c).getStateArg() and
+ result = arg.getALocalSource()
+ or
+ exists(DataFlow::TypeBackTracker t2 |
+ result = saxParserSetFeatureStateArgBacktracker(t2, arg).backtrack(t2, t)
+ )
+ }
+
+ /** Gets a back-reference to the `setFeature` state argument `arg`. */
+ DataFlow::LocalSourceNode saxParserSetFeatureStateArgBacktracker(DataFlow::Node arg) {
+ result = saxParserSetFeatureStateArgBacktracker(DataFlow::TypeBackTracker::end(), arg)
+ }
+
+ /**
+ * Gets a reference to a XML sax parser that has `feature_external_ges` turned on.
+ *
+ * See https://docs.python.org/3/library/xml.sax.handler.html#xml.sax.handler.feature_external_ges
+ */
+ private DataFlow::Node saxParserWithFeatureExternalGesTurnedOn(DataFlow::TypeTracker t) {
+ t.start() and
+ exists(SaxParserSetFeatureCall call |
+ call.getFeatureArg() =
+ API::moduleImport("xml")
+ .getMember("sax")
+ .getMember("handler")
+ .getMember("feature_external_ges")
+ .getAUse() and
+ saxParserSetFeatureStateArgBacktracker(call.getStateArg())
+ .asExpr()
+ .(BooleanLiteral)
+ .booleanValue() = true and
+ result = call.getObject()
+ )
+ or
+ exists(DataFlow::TypeTracker t2 |
+ t = t2.smallstep(saxParserWithFeatureExternalGesTurnedOn(t2), result)
+ ) and
+ // take account of that we can set the feature to False, which makes the parser safe again
+ not exists(SaxParserSetFeatureCall call |
+ call.getObject() = result and
+ call.getFeatureArg() =
+ API::moduleImport("xml")
+ .getMember("sax")
+ .getMember("handler")
+ .getMember("feature_external_ges")
+ .getAUse() and
+ saxParserSetFeatureStateArgBacktracker(call.getStateArg())
+ .asExpr()
+ .(BooleanLiteral)
+ .booleanValue() = false
+ )
+ }
+
+ /**
+ * Gets a reference to a XML sax parser that has `feature_external_ges` turned on.
+ *
+ * See https://docs.python.org/3/library/xml.sax.handler.html#xml.sax.handler.feature_external_ges
+ */
+ DataFlow::Node saxParserWithFeatureExternalGesTurnedOn() {
+ result = saxParserWithFeatureExternalGesTurnedOn(DataFlow::TypeTracker::end())
+ }
+
+ /**
+ * A call to the `parse` method on a SAX XML parser.
+ */
+ private class XMLSaxInstanceParsing extends DataFlow::MethodCallNode, XML::XMLParsing::Range {
+ XMLSaxInstanceParsing() {
+ this =
+ API::moduleImport("xml")
+ .getMember("sax")
+ .getMember("make_parser")
+ .getReturn()
+ .getMember("parse")
+ .getACall()
+ }
+
+ override DataFlow::Node getAnInput() { result in [this.getArg(0), this.getArgByName("source")] }
+
+ override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
+ // always vuln to these
+ (kind.isBillionLaughs() or kind.isQuadraticBlowup())
+ or
+ // can be vuln to other things if features has been turned on
+ this.getObject() = saxParserWithFeatureExternalGesTurnedOn() and
+ (kind.isXxe() or kind.isDtdRetrieval())
+ }
+
+ override predicate mayExecuteInput() { none() }
+
+ override DataFlow::Node getOutput() {
+ // note: the output of parsing with SAX is that the content handler gets the
+ // data... but we don't currently model this (it's not trivial to do, and won't
+ // really give us any value, at least not as of right now).
+ none()
+ }
+ }
+
+ /**
+ * A call to either `parse` or `parseString` from `xml.sax` module.
+ *
+ * See:
+ * - https://docs.python.org/3.10/library/xml.sax.html#xml.sax.parse
+ * - https://docs.python.org/3.10/library/xml.sax.html#xml.sax.parseString
+ */
+ private class XMLSaxParsing extends DataFlow::MethodCallNode, XML::XMLParsing::Range {
+ XMLSaxParsing() {
+ this =
+ API::moduleImport("xml").getMember("sax").getMember(["parse", "parseString"]).getACall()
+ }
+
+ override DataFlow::Node getAnInput() {
+ result in [
+ this.getArg(0),
+ // parseString
+ this.getArgByName("string"),
+ // parse
+ this.getArgByName("source"),
+ ]
+ }
+
+ override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
+ // always vuln to these
+ (kind.isBillionLaughs() or kind.isQuadraticBlowup())
+ or
+ // can be vuln to other things if features has been turned on
+ this.getObject() = saxParserWithFeatureExternalGesTurnedOn() and
+ (kind.isXxe() or kind.isDtdRetrieval())
+ }
+
+ override predicate mayExecuteInput() { none() }
+
+ override DataFlow::Node getOutput() {
+ // note: the output of parsing with SAX is that the content handler gets the
+ // data... but we don't currently model this (it's not trivial to do, and won't
+ // really give us any value, at least not as of right now).
+ none()
+ }
+ }
+
+ // ---------------------------------------------------------------------------
+ // xml.dom.*
+ // ---------------------------------------------------------------------------
+ /**
+ * A call to the `parse` or `parseString` methods from `xml.dom.minidom` or `xml.dom.pulldom`.
+ *
+ * Both of these modules are based on SAX parsers.
+ */
+ private class XMLDomParsing extends DataFlow::CallCfgNode, XML::XMLParsing::Range {
+ XMLDomParsing() {
+ this =
+ API::moduleImport("xml")
+ .getMember("dom")
+ .getMember(["minidom", "pulldom"])
+ .getMember(["parse", "parseString"])
+ .getACall()
+ }
+
+ override DataFlow::Node getAnInput() {
+ result in [
+ this.getArg(0),
+ // parseString
+ this.getArgByName("string"),
+ // minidom.parse
+ this.getArgByName("file"),
+ // pulldom.parse
+ this.getArgByName("stream_or_string"),
+ ]
+ }
+
+ DataFlow::Node getParserArg() { result in [this.getArg(1), this.getArgByName("parser")] }
+
+ override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
+ this.getParserArg() = saxParserWithFeatureExternalGesTurnedOn() and
+ (kind.isXxe() or kind.isDtdRetrieval())
+ or
+ (kind.isBillionLaughs() or kind.isQuadraticBlowup())
+ }
+
+ override predicate mayExecuteInput() { none() }
+
+ override DataFlow::Node getOutput() { result = this }
+ }
}
// ---------------------------------------------------------------------------
diff --git a/python/ql/src/experimental/semmle/python/frameworks/Xml.qll b/python/ql/src/experimental/semmle/python/frameworks/Xml.qll
index 88def863824..344a19a0109 100644
--- a/python/ql/src/experimental/semmle/python/frameworks/Xml.qll
+++ b/python/ql/src/experimental/semmle/python/frameworks/Xml.qll
@@ -7,213 +7,3 @@ private import python
private import semmle.python.dataflow.new.DataFlow
private import semmle.python.Concepts
private import semmle.python.ApiGraphs
-
-private module SaxBasedParsing {
- /**
- * A call to the `setFeature` method on a XML sax parser.
- *
- * See https://docs.python.org/3.10/library/xml.sax.reader.html#xml.sax.xmlreader.XMLReader.setFeature
- */
- private class SaxParserSetFeatureCall extends DataFlow::MethodCallNode {
- SaxParserSetFeatureCall() {
- this =
- API::moduleImport("xml")
- .getMember("sax")
- .getMember("make_parser")
- .getReturn()
- .getMember("setFeature")
- .getACall()
- }
-
- // The keyword argument names does not match documentation. I checked (with Python
- // 3.9.5) that the names used here actually works.
- DataFlow::Node getFeatureArg() { result in [this.getArg(0), this.getArgByName("name")] }
-
- DataFlow::Node getStateArg() { result in [this.getArg(1), this.getArgByName("state")] }
- }
-
- /** Gets a back-reference to the `setFeature` state argument `arg`. */
- private DataFlow::TypeTrackingNode saxParserSetFeatureStateArgBacktracker(
- DataFlow::TypeBackTracker t, DataFlow::Node arg
- ) {
- t.start() and
- arg = any(SaxParserSetFeatureCall c).getStateArg() and
- result = arg.getALocalSource()
- or
- exists(DataFlow::TypeBackTracker t2 |
- result = saxParserSetFeatureStateArgBacktracker(t2, arg).backtrack(t2, t)
- )
- }
-
- /** Gets a back-reference to the `setFeature` state argument `arg`. */
- DataFlow::LocalSourceNode saxParserSetFeatureStateArgBacktracker(DataFlow::Node arg) {
- result = saxParserSetFeatureStateArgBacktracker(DataFlow::TypeBackTracker::end(), arg)
- }
-
- /**
- * Gets a reference to a XML sax parser that has `feature_external_ges` turned on.
- *
- * See https://docs.python.org/3/library/xml.sax.handler.html#xml.sax.handler.feature_external_ges
- */
- private DataFlow::Node saxParserWithFeatureExternalGesTurnedOn(DataFlow::TypeTracker t) {
- t.start() and
- exists(SaxParserSetFeatureCall call |
- call.getFeatureArg() =
- API::moduleImport("xml")
- .getMember("sax")
- .getMember("handler")
- .getMember("feature_external_ges")
- .getAUse() and
- saxParserSetFeatureStateArgBacktracker(call.getStateArg())
- .asExpr()
- .(BooleanLiteral)
- .booleanValue() = true and
- result = call.getObject()
- )
- or
- exists(DataFlow::TypeTracker t2 |
- t = t2.smallstep(saxParserWithFeatureExternalGesTurnedOn(t2), result)
- ) and
- // take account of that we can set the feature to False, which makes the parser safe again
- not exists(SaxParserSetFeatureCall call |
- call.getObject() = result and
- call.getFeatureArg() =
- API::moduleImport("xml")
- .getMember("sax")
- .getMember("handler")
- .getMember("feature_external_ges")
- .getAUse() and
- saxParserSetFeatureStateArgBacktracker(call.getStateArg())
- .asExpr()
- .(BooleanLiteral)
- .booleanValue() = false
- )
- }
-
- /**
- * Gets a reference to a XML sax parser that has `feature_external_ges` turned on.
- *
- * See https://docs.python.org/3/library/xml.sax.handler.html#xml.sax.handler.feature_external_ges
- */
- DataFlow::Node saxParserWithFeatureExternalGesTurnedOn() {
- result = saxParserWithFeatureExternalGesTurnedOn(DataFlow::TypeTracker::end())
- }
-
- /**
- * A call to the `parse` method on a SAX XML parser.
- */
- private class XMLSaxInstanceParsing extends DataFlow::MethodCallNode, XML::XMLParsing::Range {
- XMLSaxInstanceParsing() {
- this =
- API::moduleImport("xml")
- .getMember("sax")
- .getMember("make_parser")
- .getReturn()
- .getMember("parse")
- .getACall()
- }
-
- override DataFlow::Node getAnInput() { result in [this.getArg(0), this.getArgByName("source")] }
-
- override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
- // always vuln to these
- (kind.isBillionLaughs() or kind.isQuadraticBlowup())
- or
- // can be vuln to other things if features has been turned on
- this.getObject() = saxParserWithFeatureExternalGesTurnedOn() and
- (kind.isXxe() or kind.isDtdRetrieval())
- }
-
- override predicate mayExecuteInput() { none() }
-
- override DataFlow::Node getOutput() {
- // note: the output of parsing with SAX is that the content handler gets the
- // data... but we don't currently model this (it's not trivial to do, and won't
- // really give us any value, at least not as of right now).
- none()
- }
- }
-
- /**
- * A call to either `parse` or `parseString` from `xml.sax` module.
- *
- * See:
- * - https://docs.python.org/3.10/library/xml.sax.html#xml.sax.parse
- * - https://docs.python.org/3.10/library/xml.sax.html#xml.sax.parseString
- */
- private class XMLSaxParsing extends DataFlow::MethodCallNode, XML::XMLParsing::Range {
- XMLSaxParsing() {
- this =
- API::moduleImport("xml").getMember("sax").getMember(["parse", "parseString"]).getACall()
- }
-
- override DataFlow::Node getAnInput() {
- result in [
- this.getArg(0),
- // parseString
- this.getArgByName("string"),
- // parse
- this.getArgByName("source"),
- ]
- }
-
- override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
- // always vuln to these
- (kind.isBillionLaughs() or kind.isQuadraticBlowup())
- or
- // can be vuln to other things if features has been turned on
- this.getObject() = saxParserWithFeatureExternalGesTurnedOn() and
- (kind.isXxe() or kind.isDtdRetrieval())
- }
-
- override predicate mayExecuteInput() { none() }
-
- override DataFlow::Node getOutput() {
- // note: the output of parsing with SAX is that the content handler gets the
- // data... but we don't currently model this (it's not trivial to do, and won't
- // really give us any value, at least not as of right now).
- none()
- }
- }
-
- /**
- * A call to the `parse` or `parseString` methods from `xml.dom.minidom` or `xml.dom.pulldom`.
- *
- * Both of these modules are based on SAX parsers.
- */
- private class XMLDomParsing extends DataFlow::CallCfgNode, XML::XMLParsing::Range {
- XMLDomParsing() {
- this =
- API::moduleImport("xml")
- .getMember("dom")
- .getMember(["minidom", "pulldom"])
- .getMember(["parse", "parseString"])
- .getACall()
- }
-
- override DataFlow::Node getAnInput() {
- result in [
- this.getArg(0),
- // parseString
- this.getArgByName("string"),
- // minidom.parse
- this.getArgByName("file"),
- // pulldom.parse
- this.getArgByName("stream_or_string"),
- ]
- }
-
- DataFlow::Node getParserArg() { result in [this.getArg(1), this.getArgByName("parser")] }
-
- override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
- this.getParserArg() = saxParserWithFeatureExternalGesTurnedOn() and
- (kind.isXxe() or kind.isDtdRetrieval())
- or
- (kind.isBillionLaughs() or kind.isQuadraticBlowup())
- }
-
- override predicate mayExecuteInput() { none() }
-
- override DataFlow::Node getOutput() { result = this }
- }
-}
diff --git a/python/ql/test/experimental/library-tests/frameworks/XML/xml_dom.py b/python/ql/test/library-tests/frameworks/stdlib/xml_dom.py
similarity index 100%
rename from python/ql/test/experimental/library-tests/frameworks/XML/xml_dom.py
rename to python/ql/test/library-tests/frameworks/stdlib/xml_dom.py
diff --git a/python/ql/test/experimental/library-tests/frameworks/XML/xml_sax.py b/python/ql/test/library-tests/frameworks/stdlib/xml_sax.py
similarity index 100%
rename from python/ql/test/experimental/library-tests/frameworks/XML/xml_sax.py
rename to python/ql/test/library-tests/frameworks/stdlib/xml_sax.py
From 1d7cec60ae09489618b7e561845b5a361c274583 Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Thu, 31 Mar 2022 17:50:23 +0200
Subject: [PATCH 071/171] Python: `xml.sax.parse` is not a method call
And it's not possible to provide a parser argument either
---
python/ql/lib/semmle/python/frameworks/Stdlib.qll | 6 +-----
1 file changed, 1 insertion(+), 5 deletions(-)
diff --git a/python/ql/lib/semmle/python/frameworks/Stdlib.qll b/python/ql/lib/semmle/python/frameworks/Stdlib.qll
index 418f3475c1e..5659c7c8e91 100644
--- a/python/ql/lib/semmle/python/frameworks/Stdlib.qll
+++ b/python/ql/lib/semmle/python/frameworks/Stdlib.qll
@@ -3482,7 +3482,7 @@ private module StdlibPrivate {
* - https://docs.python.org/3.10/library/xml.sax.html#xml.sax.parse
* - https://docs.python.org/3.10/library/xml.sax.html#xml.sax.parseString
*/
- private class XMLSaxParsing extends DataFlow::MethodCallNode, XML::XMLParsing::Range {
+ private class XMLSaxParsing extends DataFlow::CallCfgNode, XML::XMLParsing::Range {
XMLSaxParsing() {
this =
API::moduleImport("xml").getMember("sax").getMember(["parse", "parseString"]).getACall()
@@ -3501,10 +3501,6 @@ private module StdlibPrivate {
override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
// always vuln to these
(kind.isBillionLaughs() or kind.isQuadraticBlowup())
- or
- // can be vuln to other things if features has been turned on
- this.getObject() = saxParserWithFeatureExternalGesTurnedOn() and
- (kind.isXxe() or kind.isDtdRetrieval())
}
override predicate mayExecuteInput() { none() }
From b4c0065aeb160839129d25cc3ee1818564670d21 Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Thu, 31 Mar 2022 18:03:35 +0200
Subject: [PATCH 072/171] Python: Extend FileSystemAccess for `xml.sax` and
`xml.dom.*` parsing
---
.../lib/semmle/python/frameworks/Stdlib.qll | 72 ++++++++++++++++++-
.../frameworks/stdlib/xml_dom.py | 16 ++---
.../frameworks/stdlib/xml_sax.py | 22 +++---
3 files changed, 90 insertions(+), 20 deletions(-)
diff --git a/python/ql/lib/semmle/python/frameworks/Stdlib.qll b/python/ql/lib/semmle/python/frameworks/Stdlib.qll
index 5659c7c8e91..38fe32a3b3c 100644
--- a/python/ql/lib/semmle/python/frameworks/Stdlib.qll
+++ b/python/ql/lib/semmle/python/frameworks/Stdlib.qll
@@ -3442,8 +3442,11 @@ private module StdlibPrivate {
/**
* A call to the `parse` method on a SAX XML parser.
+ *
+ * See https://docs.python.org/3/library/xml.sax.reader.html#xml.sax.xmlreader.XMLReader.parse
*/
- private class XMLSaxInstanceParsing extends DataFlow::MethodCallNode, XML::XMLParsing::Range {
+ private class XMLSaxInstanceParsing extends DataFlow::MethodCallNode, XML::XMLParsing::Range,
+ FileSystemAccess::Range {
XMLSaxInstanceParsing() {
this =
API::moduleImport("xml")
@@ -3473,6 +3476,17 @@ private module StdlibPrivate {
// really give us any value, at least not as of right now).
none()
}
+
+ override DataFlow::Node getAPathArgument() {
+ // I considered whether we should try to reduce FPs from people passing file-like
+ // objects, which will not be a file system access (and couldn't cause a
+ // path-injection).
+ //
+ // I suppose that once we have proper flow-summary support for file-like objects,
+ // we can make the XXE/XML-bomb sinks allow an access-path, while the
+ // path-injection sink wouldn't, and then we will not end up with such FPs.
+ result = this.getAnInput()
+ }
}
/**
@@ -3513,6 +3527,29 @@ private module StdlibPrivate {
}
}
+ /**
+ * A call to `xml.sax.parse`, which takes either a filename or a file-like object as
+ * argument. To capture the filename for path-injection, we have this subclass.
+ *
+ * See
+ * - https://docs.python.org/3/library/xml.etree.elementtree.html#xml.etree.ElementTree.parse
+ * - https://docs.python.org/3/library/xml.etree.elementtree.html#xml.etree.ElementTree.iterparse
+ */
+ private class FileAccessFromXMLSaxParsing extends XMLSaxParsing, FileSystemAccess::Range {
+ FileAccessFromXMLSaxParsing() {
+ this = API::moduleImport("xml").getMember("sax").getMember("parse").getACall()
+ // I considered whether we should try to reduce FPs from people passing file-like
+ // objects, which will not be a file system access (and couldn't cause a
+ // path-injection).
+ //
+ // I suppose that once we have proper flow-summary support for file-like objects,
+ // we can make the XXE/XML-bomb sinks allow an access-path, while the
+ // path-injection sink wouldn't, and then we will not end up with such FPs.
+ }
+
+ override DataFlow::Node getAPathArgument() { result = this.getAnInput() }
+ }
+
// ---------------------------------------------------------------------------
// xml.dom.*
// ---------------------------------------------------------------------------
@@ -3520,6 +3557,10 @@ private module StdlibPrivate {
* A call to the `parse` or `parseString` methods from `xml.dom.minidom` or `xml.dom.pulldom`.
*
* Both of these modules are based on SAX parsers.
+ *
+ * See
+ * - https://docs.python.org/3/library/xml.dom.minidom.html#xml.dom.minidom.parse
+ * - https://docs.python.org/3/library/xml.dom.pulldom.html#xml.dom.pulldom.parse
*/
private class XMLDomParsing extends DataFlow::CallCfgNode, XML::XMLParsing::Range {
XMLDomParsing() {
@@ -3556,6 +3597,35 @@ private module StdlibPrivate {
override DataFlow::Node getOutput() { result = this }
}
+
+ /**
+ * A call to the `parse` or `parseString` methods from `xml.dom.minidom` or
+ * `xml.dom.pulldom`, which takes either a filename or a file-like object as argument.
+ * To capture the filename for path-injection, we have this subclass.
+ *
+ * See
+ * - https://docs.python.org/3/library/xml.dom.minidom.html#xml.dom.minidom.parse
+ * - https://docs.python.org/3/library/xml.dom.pulldom.html#xml.dom.pulldom.parse
+ */
+ private class FileAccessFromXMLDomParsing extends XMLDomParsing, FileSystemAccess::Range {
+ FileAccessFromXMLDomParsing() {
+ this =
+ API::moduleImport("xml")
+ .getMember("dom")
+ .getMember(["minidom", "pulldom"])
+ .getMember("parse")
+ .getACall()
+ // I considered whether we should try to reduce FPs from people passing file-like
+ // objects, which will not be a file system access (and couldn't cause a
+ // path-injection).
+ //
+ // I suppose that once we have proper flow-summary support for file-like objects,
+ // we can make the XXE/XML-bomb sinks allow an access-path, while the
+ // path-injection sink wouldn't, and then we will not end up with such FPs.
+ }
+
+ override DataFlow::Node getAPathArgument() { result = this.getAnInput() }
+ }
}
// ---------------------------------------------------------------------------
diff --git a/python/ql/test/library-tests/frameworks/stdlib/xml_dom.py b/python/ql/test/library-tests/frameworks/stdlib/xml_dom.py
index c6152c75807..b3a1ab7f930 100644
--- a/python/ql/test/library-tests/frameworks/stdlib/xml_dom.py
+++ b/python/ql/test/library-tests/frameworks/stdlib/xml_dom.py
@@ -6,16 +6,16 @@ import xml.sax
x = "some xml"
# minidom
-xml.dom.minidom.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.dom.minidom.parse(..)
-xml.dom.minidom.parse(file=StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.dom.minidom.parse(..)
+xml.dom.minidom.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.dom.minidom.parse(..) getAPathArgument=StringIO(..)
+xml.dom.minidom.parse(file=StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.dom.minidom.parse(..) getAPathArgument=StringIO(..)
xml.dom.minidom.parseString(x) # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.dom.minidom.parseString(..)
xml.dom.minidom.parseString(string=x) # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.dom.minidom.parseString(..)
# pulldom
-xml.dom.pulldom.parse(StringIO(x))['START_DOCUMENT'][1] # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.dom.pulldom.parse(..)
-xml.dom.pulldom.parse(stream_or_string=StringIO(x))['START_DOCUMENT'][1] # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.dom.pulldom.parse(..)
+xml.dom.pulldom.parse(StringIO(x))['START_DOCUMENT'][1] # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.dom.pulldom.parse(..) getAPathArgument=StringIO(..)
+xml.dom.pulldom.parse(stream_or_string=StringIO(x))['START_DOCUMENT'][1] # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.dom.pulldom.parse(..) getAPathArgument=StringIO(..)
xml.dom.pulldom.parseString(x)['START_DOCUMENT'][1] # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.dom.pulldom.parseString(..)
xml.dom.pulldom.parseString(string=x)['START_DOCUMENT'][1] # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.dom.pulldom.parseString(..)
@@ -24,8 +24,8 @@ xml.dom.pulldom.parseString(string=x)['START_DOCUMENT'][1] # $ decodeFormat=XML
# These are based on SAX parses, and you can specify your own, so you can expose yourself to XXE (yay/)
parser = xml.sax.make_parser()
parser.setFeature(xml.sax.handler.feature_external_ges, True)
-xml.dom.minidom.parse(StringIO(x), parser) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE' decodeOutput=xml.dom.minidom.parse(..)
-xml.dom.minidom.parse(StringIO(x), parser=parser) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE' decodeOutput=xml.dom.minidom.parse(..)
+xml.dom.minidom.parse(StringIO(x), parser) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE' decodeOutput=xml.dom.minidom.parse(..) getAPathArgument=StringIO(..)
+xml.dom.minidom.parse(StringIO(x), parser=parser) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE' decodeOutput=xml.dom.minidom.parse(..) getAPathArgument=StringIO(..)
-xml.dom.pulldom.parse(StringIO(x), parser) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE' decodeOutput=xml.dom.pulldom.parse(..)
-xml.dom.pulldom.parse(StringIO(x), parser=parser) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE' decodeOutput=xml.dom.pulldom.parse(..)
+xml.dom.pulldom.parse(StringIO(x), parser) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE' decodeOutput=xml.dom.pulldom.parse(..) getAPathArgument=StringIO(..)
+xml.dom.pulldom.parse(StringIO(x), parser=parser) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE' decodeOutput=xml.dom.pulldom.parse(..) getAPathArgument=StringIO(..)
diff --git a/python/ql/test/library-tests/frameworks/stdlib/xml_sax.py b/python/ql/test/library-tests/frameworks/stdlib/xml_sax.py
index 8dbe9d4ae99..c08034907a4 100644
--- a/python/ql/test/library-tests/frameworks/stdlib/xml_sax.py
+++ b/python/ql/test/library-tests/frameworks/stdlib/xml_sax.py
@@ -10,41 +10,41 @@ class MainHandler(xml.sax.ContentHandler):
def characters(self, data):
self._result.append(data)
-xml.sax.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
-xml.sax.parse(source=StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+xml.sax.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' getAPathArgument=StringIO(..)
+xml.sax.parse(source=StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' getAPathArgument=StringIO(..)
xml.sax.parseString(x) # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
xml.sax.parseString(string=x) # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
parser = xml.sax.make_parser()
-parser.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
-parser.parse(source=StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+parser.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' getAPathArgument=StringIO(..)
+parser.parse(source=StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' getAPathArgument=StringIO(..)
# You can make it vuln to both XXE and DTD retrieval by setting this flag
# see https://docs.python.org/3/library/xml.sax.handler.html#xml.sax.handler.feature_external_ges
parser = xml.sax.make_parser()
parser.setFeature(xml.sax.handler.feature_external_ges, True)
-parser.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE'
+parser.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE' getAPathArgument=StringIO(..)
parser = xml.sax.make_parser()
parser.setFeature(xml.sax.handler.feature_external_ges, False)
-parser.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+parser.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' getAPathArgument=StringIO(..)
# Forward Type Tracking test
def func(cond):
parser = xml.sax.make_parser()
if cond:
parser.setFeature(xml.sax.handler.feature_external_ges, True)
- parser.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE'
+ parser.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE' getAPathArgument=StringIO(..)
else:
- parser.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+ parser.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' getAPathArgument=StringIO(..)
# make it vuln, then making it safe
# a bit of an edge-case, but is nice to be able to handle.
parser = xml.sax.make_parser()
parser.setFeature(xml.sax.handler.feature_external_ges, True)
parser.setFeature(xml.sax.handler.feature_external_ges, False)
-parser.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+parser.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' getAPathArgument=StringIO(..)
def check_conditional_assignment(cond):
parser = xml.sax.make_parser()
@@ -52,7 +52,7 @@ def check_conditional_assignment(cond):
parser.setFeature(xml.sax.handler.feature_external_ges, True)
else:
parser.setFeature(xml.sax.handler.feature_external_ges, False)
- parser.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE'
+ parser.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE' getAPathArgument=StringIO(..)
def check_conditional_assignment2(cond):
parser = xml.sax.make_parser()
@@ -61,4 +61,4 @@ def check_conditional_assignment2(cond):
else:
flag_value = False
parser.setFeature(xml.sax.handler.feature_external_ges, flag_value)
- parser.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE'
+ parser.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE' getAPathArgument=StringIO(..)
From 673220b231fdfcd225f4d29cbc76be67f156b21c Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Thu, 31 Mar 2022 18:18:35 +0200
Subject: [PATCH 073/171] Python: Minor cleanup of `XmlParsingTest`
---
.../ql/test/experimental/meta/ConceptsTest.qll | 16 +++++++---------
1 file changed, 7 insertions(+), 9 deletions(-)
diff --git a/python/ql/test/experimental/meta/ConceptsTest.qll b/python/ql/test/experimental/meta/ConceptsTest.qll
index 24cbbab2d44..cd90d716dd4 100644
--- a/python/ql/test/experimental/meta/ConceptsTest.qll
+++ b/python/ql/test/experimental/meta/ConceptsTest.qll
@@ -543,18 +543,16 @@ class HttpClientRequestTest extends InlineExpectationsTest {
class XmlParsingTest extends InlineExpectationsTest {
XmlParsingTest() { this = "XmlParsingTest" }
- override string getARelevantTag() { result in ["xmlInput", "xmlVuln"] }
+ override string getARelevantTag() { result in ["xmlVuln"] }
override predicate hasActualResult(Location location, string element, string tag, string value) {
exists(location.getFile().getRelativePath()) and
- exists(XML::XMLParsing parsing |
- exists(XML::XMLParsingVulnerabilityKind kind |
- parsing.vulnerableTo(kind) and
- location = parsing.getLocation() and
- element = parsing.toString() and
- value = "'" + kind + "'" and
- tag = "xmlVuln"
- )
+ exists(XML::XMLParsing parsing, XML::XMLParsingVulnerabilityKind kind |
+ parsing.vulnerableTo(kind) and
+ location = parsing.getLocation() and
+ element = parsing.toString() and
+ value = "'" + kind + "'" and
+ tag = "xmlVuln"
)
}
}
From 5083023aa80238c58811aa7e56df6dddf4e6b33a Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Thu, 31 Mar 2022 18:37:47 +0200
Subject: [PATCH 074/171] Python: Move XML parsing PoC
Since the folder where it used to live is now empty otherwise :O
---
python/PoCs/README.md | 1 +
.../library-tests/frameworks/XML/poc => PoCs/XmlParsing}/PoC.py | 0
.../library-tests/frameworks/XML/poc => PoCs/XmlParsing}/flag | 0
python/ql/lib/semmle/python/Concepts.qll | 2 ++
.../library-tests/frameworks/XML/poc/this-dir-is-not-extracted | 1 -
5 files changed, 3 insertions(+), 1 deletion(-)
create mode 100644 python/PoCs/README.md
rename python/{ql/test/experimental/library-tests/frameworks/XML/poc => PoCs/XmlParsing}/PoC.py (100%)
rename python/{ql/test/experimental/library-tests/frameworks/XML/poc => PoCs/XmlParsing}/flag (100%)
delete mode 100644 python/ql/test/experimental/library-tests/frameworks/XML/poc/this-dir-is-not-extracted
diff --git a/python/PoCs/README.md b/python/PoCs/README.md
new file mode 100644
index 00000000000..20eeb5dbd78
--- /dev/null
+++ b/python/PoCs/README.md
@@ -0,0 +1 @@
+A place to collect proof of concept for how certain vulnerabilities work.
diff --git a/python/ql/test/experimental/library-tests/frameworks/XML/poc/PoC.py b/python/PoCs/XmlParsing/PoC.py
similarity index 100%
rename from python/ql/test/experimental/library-tests/frameworks/XML/poc/PoC.py
rename to python/PoCs/XmlParsing/PoC.py
diff --git a/python/ql/test/experimental/library-tests/frameworks/XML/poc/flag b/python/PoCs/XmlParsing/flag
similarity index 100%
rename from python/ql/test/experimental/library-tests/frameworks/XML/poc/flag
rename to python/PoCs/XmlParsing/flag
diff --git a/python/ql/lib/semmle/python/Concepts.qll b/python/ql/lib/semmle/python/Concepts.qll
index b553c8d927d..b1727e4829d 100644
--- a/python/ql/lib/semmle/python/Concepts.qll
+++ b/python/ql/lib/semmle/python/Concepts.qll
@@ -555,6 +555,8 @@ module XML {
* A kind of XML vulnerability.
*
* See overview of kinds at https://pypi.org/project/defusedxml/#python-xml-libraries
+ *
+ * See PoC at `python/PoCs/XmlParsing/PoC.py` for some tests of vulnerable XML parsing.
*/
class XMLParsingVulnerabilityKind extends string {
XMLParsingVulnerabilityKind() {
diff --git a/python/ql/test/experimental/library-tests/frameworks/XML/poc/this-dir-is-not-extracted b/python/ql/test/experimental/library-tests/frameworks/XML/poc/this-dir-is-not-extracted
deleted file mode 100644
index b1925ade1d3..00000000000
--- a/python/ql/test/experimental/library-tests/frameworks/XML/poc/this-dir-is-not-extracted
+++ /dev/null
@@ -1 +0,0 @@
-just FYI
From b8d3c5e96fbfc0b5770591d699b94695f3d15a26 Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Thu, 31 Mar 2022 18:40:26 +0200
Subject: [PATCH 075/171] Python: Remove last bits of experimental XML modeling
---
python/ql/src/experimental/semmle/python/Frameworks.qll | 1 -
.../ql/src/experimental/semmle/python/frameworks/Xml.qll | 9 ---------
.../python/security/dataflow/XmlBombCustomizations.qll | 1 -
.../python/security/dataflow/XxeCustomizations.qll | 1 -
.../library-tests/frameworks/XML/ConceptsTest.expected | 0
.../library-tests/frameworks/XML/ConceptsTest.ql | 3 ---
6 files changed, 15 deletions(-)
delete mode 100644 python/ql/src/experimental/semmle/python/frameworks/Xml.qll
delete mode 100644 python/ql/test/experimental/library-tests/frameworks/XML/ConceptsTest.expected
delete mode 100644 python/ql/test/experimental/library-tests/frameworks/XML/ConceptsTest.ql
diff --git a/python/ql/src/experimental/semmle/python/Frameworks.qll b/python/ql/src/experimental/semmle/python/Frameworks.qll
index edbed61c41c..81b2c1bee23 100644
--- a/python/ql/src/experimental/semmle/python/Frameworks.qll
+++ b/python/ql/src/experimental/semmle/python/Frameworks.qll
@@ -3,7 +3,6 @@
*/
private import experimental.semmle.python.frameworks.Stdlib
-private import experimental.semmle.python.frameworks.Xml
private import experimental.semmle.python.frameworks.Flask
private import experimental.semmle.python.frameworks.Django
private import experimental.semmle.python.frameworks.Werkzeug
diff --git a/python/ql/src/experimental/semmle/python/frameworks/Xml.qll b/python/ql/src/experimental/semmle/python/frameworks/Xml.qll
deleted file mode 100644
index 344a19a0109..00000000000
--- a/python/ql/src/experimental/semmle/python/frameworks/Xml.qll
+++ /dev/null
@@ -1,9 +0,0 @@
-/**
- * Provides class and predicates to track external data that
- * may represent malicious XML objects.
- */
-
-private import python
-private import semmle.python.dataflow.new.DataFlow
-private import semmle.python.Concepts
-private import semmle.python.ApiGraphs
diff --git a/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombCustomizations.qll b/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombCustomizations.qll
index c5e69c1e0e3..d6f2e0791f9 100644
--- a/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombCustomizations.qll
+++ b/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombCustomizations.qll
@@ -7,7 +7,6 @@
private import python
private import semmle.python.dataflow.new.DataFlow
private import semmle.python.Concepts
-import experimental.semmle.python.frameworks.Xml // needed until modeling have been promoted
private import semmle.python.dataflow.new.RemoteFlowSources
/**
diff --git a/python/ql/src/experimental/semmle/python/security/dataflow/XxeCustomizations.qll b/python/ql/src/experimental/semmle/python/security/dataflow/XxeCustomizations.qll
index 27d011625a6..a4473285b8d 100644
--- a/python/ql/src/experimental/semmle/python/security/dataflow/XxeCustomizations.qll
+++ b/python/ql/src/experimental/semmle/python/security/dataflow/XxeCustomizations.qll
@@ -7,7 +7,6 @@
private import python
private import semmle.python.dataflow.new.DataFlow
private import semmle.python.Concepts
-import experimental.semmle.python.frameworks.Xml // needed until modeling have been promoted
private import semmle.python.dataflow.new.RemoteFlowSources
/**
diff --git a/python/ql/test/experimental/library-tests/frameworks/XML/ConceptsTest.expected b/python/ql/test/experimental/library-tests/frameworks/XML/ConceptsTest.expected
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/python/ql/test/experimental/library-tests/frameworks/XML/ConceptsTest.ql b/python/ql/test/experimental/library-tests/frameworks/XML/ConceptsTest.ql
deleted file mode 100644
index 95728bd6dc8..00000000000
--- a/python/ql/test/experimental/library-tests/frameworks/XML/ConceptsTest.ql
+++ /dev/null
@@ -1,3 +0,0 @@
-import python
-import experimental.meta.ConceptsTest
-import experimental.semmle.python.frameworks.Xml // needed until modeling have been promoted
From 4abab2206618b950509b45ed516b8a9c11f7732d Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Thu, 31 Mar 2022 18:47:50 +0200
Subject: [PATCH 076/171] Python: Promote XXE and XML-bomb queries
Need to write a change-note as well, but will do that tomorrow
---
.../{experimental/Security/NEW => Security}/CWE-611/Xxe.qhelp | 0
.../src/{experimental/Security/NEW => Security}/CWE-611/Xxe.ql | 0
.../Security/NEW => Security}/CWE-611/examples/XxeBad.py | 0
.../Security/NEW => Security}/CWE-611/examples/XxeGood.py | 0
.../Security/NEW => Security}/CWE-776/XmlBomb.qhelp | 0
.../{experimental/Security/NEW => Security}/CWE-776/XmlBomb.ql | 0
.../Security/NEW => Security}/CWE-776/examples/XmlBombBad.py | 0
.../Security/NEW => Security}/CWE-776/examples/XmlBombGood.py | 0
.../test/experimental/query-tests/Security/CWE-611-Xxe/Xxe.qlref | 1 -
.../query-tests/Security/CWE-776-XmlBomb/XmlBomb.qlref | 1 -
.../query-tests/Security/CWE-611-Xxe/Xxe.expected | 0
python/ql/test/query-tests/Security/CWE-611-Xxe/Xxe.qlref | 1 +
.../{experimental => }/query-tests/Security/CWE-611-Xxe/test.py | 0
.../query-tests/Security/CWE-776-XmlBomb/XmlBomb.expected | 0
.../ql/test/query-tests/Security/CWE-776-XmlBomb/XmlBomb.qlref | 1 +
.../query-tests/Security/CWE-776-XmlBomb/test.py | 0
16 files changed, 2 insertions(+), 2 deletions(-)
rename python/ql/src/{experimental/Security/NEW => Security}/CWE-611/Xxe.qhelp (100%)
rename python/ql/src/{experimental/Security/NEW => Security}/CWE-611/Xxe.ql (100%)
rename python/ql/src/{experimental/Security/NEW => Security}/CWE-611/examples/XxeBad.py (100%)
rename python/ql/src/{experimental/Security/NEW => Security}/CWE-611/examples/XxeGood.py (100%)
rename python/ql/src/{experimental/Security/NEW => Security}/CWE-776/XmlBomb.qhelp (100%)
rename python/ql/src/{experimental/Security/NEW => Security}/CWE-776/XmlBomb.ql (100%)
rename python/ql/src/{experimental/Security/NEW => Security}/CWE-776/examples/XmlBombBad.py (100%)
rename python/ql/src/{experimental/Security/NEW => Security}/CWE-776/examples/XmlBombGood.py (100%)
delete mode 100644 python/ql/test/experimental/query-tests/Security/CWE-611-Xxe/Xxe.qlref
delete mode 100644 python/ql/test/experimental/query-tests/Security/CWE-776-XmlBomb/XmlBomb.qlref
rename python/ql/test/{experimental => }/query-tests/Security/CWE-611-Xxe/Xxe.expected (100%)
create mode 100644 python/ql/test/query-tests/Security/CWE-611-Xxe/Xxe.qlref
rename python/ql/test/{experimental => }/query-tests/Security/CWE-611-Xxe/test.py (100%)
rename python/ql/test/{experimental => }/query-tests/Security/CWE-776-XmlBomb/XmlBomb.expected (100%)
create mode 100644 python/ql/test/query-tests/Security/CWE-776-XmlBomb/XmlBomb.qlref
rename python/ql/test/{experimental => }/query-tests/Security/CWE-776-XmlBomb/test.py (100%)
diff --git a/python/ql/src/experimental/Security/NEW/CWE-611/Xxe.qhelp b/python/ql/src/Security/CWE-611/Xxe.qhelp
similarity index 100%
rename from python/ql/src/experimental/Security/NEW/CWE-611/Xxe.qhelp
rename to python/ql/src/Security/CWE-611/Xxe.qhelp
diff --git a/python/ql/src/experimental/Security/NEW/CWE-611/Xxe.ql b/python/ql/src/Security/CWE-611/Xxe.ql
similarity index 100%
rename from python/ql/src/experimental/Security/NEW/CWE-611/Xxe.ql
rename to python/ql/src/Security/CWE-611/Xxe.ql
diff --git a/python/ql/src/experimental/Security/NEW/CWE-611/examples/XxeBad.py b/python/ql/src/Security/CWE-611/examples/XxeBad.py
similarity index 100%
rename from python/ql/src/experimental/Security/NEW/CWE-611/examples/XxeBad.py
rename to python/ql/src/Security/CWE-611/examples/XxeBad.py
diff --git a/python/ql/src/experimental/Security/NEW/CWE-611/examples/XxeGood.py b/python/ql/src/Security/CWE-611/examples/XxeGood.py
similarity index 100%
rename from python/ql/src/experimental/Security/NEW/CWE-611/examples/XxeGood.py
rename to python/ql/src/Security/CWE-611/examples/XxeGood.py
diff --git a/python/ql/src/experimental/Security/NEW/CWE-776/XmlBomb.qhelp b/python/ql/src/Security/CWE-776/XmlBomb.qhelp
similarity index 100%
rename from python/ql/src/experimental/Security/NEW/CWE-776/XmlBomb.qhelp
rename to python/ql/src/Security/CWE-776/XmlBomb.qhelp
diff --git a/python/ql/src/experimental/Security/NEW/CWE-776/XmlBomb.ql b/python/ql/src/Security/CWE-776/XmlBomb.ql
similarity index 100%
rename from python/ql/src/experimental/Security/NEW/CWE-776/XmlBomb.ql
rename to python/ql/src/Security/CWE-776/XmlBomb.ql
diff --git a/python/ql/src/experimental/Security/NEW/CWE-776/examples/XmlBombBad.py b/python/ql/src/Security/CWE-776/examples/XmlBombBad.py
similarity index 100%
rename from python/ql/src/experimental/Security/NEW/CWE-776/examples/XmlBombBad.py
rename to python/ql/src/Security/CWE-776/examples/XmlBombBad.py
diff --git a/python/ql/src/experimental/Security/NEW/CWE-776/examples/XmlBombGood.py b/python/ql/src/Security/CWE-776/examples/XmlBombGood.py
similarity index 100%
rename from python/ql/src/experimental/Security/NEW/CWE-776/examples/XmlBombGood.py
rename to python/ql/src/Security/CWE-776/examples/XmlBombGood.py
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-611-Xxe/Xxe.qlref b/python/ql/test/experimental/query-tests/Security/CWE-611-Xxe/Xxe.qlref
deleted file mode 100644
index f8a07d7d2ee..00000000000
--- a/python/ql/test/experimental/query-tests/Security/CWE-611-Xxe/Xxe.qlref
+++ /dev/null
@@ -1 +0,0 @@
-experimental/Security/NEW/CWE-611/Xxe.ql
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-776-XmlBomb/XmlBomb.qlref b/python/ql/test/experimental/query-tests/Security/CWE-776-XmlBomb/XmlBomb.qlref
deleted file mode 100644
index 5eadbb1f26f..00000000000
--- a/python/ql/test/experimental/query-tests/Security/CWE-776-XmlBomb/XmlBomb.qlref
+++ /dev/null
@@ -1 +0,0 @@
-experimental/Security/NEW/CWE-776/XmlBomb.ql
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-611-Xxe/Xxe.expected b/python/ql/test/query-tests/Security/CWE-611-Xxe/Xxe.expected
similarity index 100%
rename from python/ql/test/experimental/query-tests/Security/CWE-611-Xxe/Xxe.expected
rename to python/ql/test/query-tests/Security/CWE-611-Xxe/Xxe.expected
diff --git a/python/ql/test/query-tests/Security/CWE-611-Xxe/Xxe.qlref b/python/ql/test/query-tests/Security/CWE-611-Xxe/Xxe.qlref
new file mode 100644
index 00000000000..62a3f7f22d9
--- /dev/null
+++ b/python/ql/test/query-tests/Security/CWE-611-Xxe/Xxe.qlref
@@ -0,0 +1 @@
+Security/CWE-611/Xxe.ql
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-611-Xxe/test.py b/python/ql/test/query-tests/Security/CWE-611-Xxe/test.py
similarity index 100%
rename from python/ql/test/experimental/query-tests/Security/CWE-611-Xxe/test.py
rename to python/ql/test/query-tests/Security/CWE-611-Xxe/test.py
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-776-XmlBomb/XmlBomb.expected b/python/ql/test/query-tests/Security/CWE-776-XmlBomb/XmlBomb.expected
similarity index 100%
rename from python/ql/test/experimental/query-tests/Security/CWE-776-XmlBomb/XmlBomb.expected
rename to python/ql/test/query-tests/Security/CWE-776-XmlBomb/XmlBomb.expected
diff --git a/python/ql/test/query-tests/Security/CWE-776-XmlBomb/XmlBomb.qlref b/python/ql/test/query-tests/Security/CWE-776-XmlBomb/XmlBomb.qlref
new file mode 100644
index 00000000000..c983b357446
--- /dev/null
+++ b/python/ql/test/query-tests/Security/CWE-776-XmlBomb/XmlBomb.qlref
@@ -0,0 +1 @@
+Security/CWE-776/XmlBomb.ql
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-776-XmlBomb/test.py b/python/ql/test/query-tests/Security/CWE-776-XmlBomb/test.py
similarity index 100%
rename from python/ql/test/experimental/query-tests/Security/CWE-776-XmlBomb/test.py
rename to python/ql/test/query-tests/Security/CWE-776-XmlBomb/test.py
From d2b03bb4809b1156d1d0799ca739da4265c68ba7 Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Thu, 31 Mar 2022 20:37:28 +0200
Subject: [PATCH 077/171] Python: Fix `SimpleXmlRpcServer.ql`
---
.../src/experimental/Security/CWE-611/SimpleXmlRpcServer.ql | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/python/ql/src/experimental/Security/CWE-611/SimpleXmlRpcServer.ql b/python/ql/src/experimental/Security/CWE-611/SimpleXmlRpcServer.ql
index 3d2a736ed49..53ff6eeedb8 100644
--- a/python/ql/src/experimental/Security/CWE-611/SimpleXmlRpcServer.ql
+++ b/python/ql/src/experimental/Security/CWE-611/SimpleXmlRpcServer.ql
@@ -10,14 +10,14 @@
*/
private import python
-private import experimental.semmle.python.Concepts
+private import semmle.python.Concepts
private import semmle.python.ApiGraphs
from DataFlow::CallCfgNode call, string kinds
where
call = API::moduleImport("xmlrpc").getMember("server").getMember("SimpleXMLRPCServer").getACall() and
kinds =
- strictconcat(ExperimentalXML::XMLParsingVulnerabilityKind kind |
+ strictconcat(XML::XMLParsingVulnerabilityKind kind |
kind.isBillionLaughs() or kind.isQuadraticBlowup()
|
kind, ", "
From ab59d5c786893d71dc044107af0517e56b460171 Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Tue, 5 Apr 2022 11:06:22 +0200
Subject: [PATCH 078/171] Python: Rename to `XmlParsing`
To follow our style guide
---
python/ql/lib/semmle/python/Concepts.qll | 8 ++++----
python/ql/lib/semmle/python/frameworks/Lxml.qll | 6 +++---
python/ql/lib/semmle/python/frameworks/Stdlib.qll | 10 +++++-----
python/ql/lib/semmle/python/frameworks/Xmltodict.qll | 2 +-
.../python/security/dataflow/XmlBombCustomizations.qll | 2 +-
.../python/security/dataflow/XxeCustomizations.qll | 2 +-
python/ql/test/experimental/meta/ConceptsTest.qll | 2 +-
7 files changed, 16 insertions(+), 16 deletions(-)
diff --git a/python/ql/lib/semmle/python/Concepts.qll b/python/ql/lib/semmle/python/Concepts.qll
index b1727e4829d..b0a5e1766a2 100644
--- a/python/ql/lib/semmle/python/Concepts.qll
+++ b/python/ql/lib/semmle/python/Concepts.qll
@@ -580,9 +580,9 @@ module XML {
* A data-flow node that parses XML.
*
* Extend this class to model new APIs. If you want to refine existing API models,
- * extend `XMLParsing` instead.
+ * extend `XmlParsing` instead.
*/
- class XMLParsing extends Decoding instanceof XMLParsing::Range {
+ class XmlParsing extends Decoding instanceof XmlParsing::Range {
/**
* Holds if this XML parsing is vulnerable to `kind`.
*/
@@ -590,12 +590,12 @@ module XML {
}
/** Provides classes for modeling XML parsing APIs. */
- module XMLParsing {
+ module XmlParsing {
/**
* A data-flow node that parses XML.
*
* Extend this class to model new APIs. If you want to refine existing API models,
- * extend `XMLParsing` instead.
+ * extend `XmlParsing` instead.
*/
abstract class Range extends Decoding::Range {
/**
diff --git a/python/ql/lib/semmle/python/frameworks/Lxml.qll b/python/ql/lib/semmle/python/frameworks/Lxml.qll
index a3825a70db0..05dfd388dac 100644
--- a/python/ql/lib/semmle/python/frameworks/Lxml.qll
+++ b/python/ql/lib/semmle/python/frameworks/Lxml.qll
@@ -196,7 +196,7 @@ private module Lxml {
/**
* A call to the `feed` method of an `lxml` parser.
*/
- private class LXMLParserFeedCall extends DataFlow::MethodCallNode, XML::XMLParsing::Range {
+ private class LXMLParserFeedCall extends DataFlow::MethodCallNode, XML::XmlParsing::Range {
LXMLParserFeedCall() { this.calls(instance(_), "feed") }
override DataFlow::Node getAnInput() { result in [this.getArg(0), this.getArgByName("data")] }
@@ -233,7 +233,7 @@ private module Lxml {
* - https://lxml.de/apidoc/lxml.etree.html?highlight=parseids#lxml.etree.parse
* - https://lxml.de/apidoc/lxml.etree.html?highlight=parseids#lxml.etree.parseid
*/
- private class LXMLParsing extends DataFlow::CallCfgNode, XML::XMLParsing::Range {
+ private class LXMLParsing extends DataFlow::CallCfgNode, XML::XmlParsing::Range {
LXMLParsing() {
this =
API::moduleImport("lxml")
@@ -305,7 +305,7 @@ private module Lxml {
* See
* - https://lxml.de/apidoc/lxml.etree.html?highlight=parseids#lxml.etree.iterparse
*/
- private class LXMLIterparseCall extends DataFlow::CallCfgNode, XML::XMLParsing::Range,
+ private class LXMLIterparseCall extends DataFlow::CallCfgNode, XML::XmlParsing::Range,
FileSystemAccess::Range {
LXMLIterparseCall() {
this = API::moduleImport("lxml").getMember("etree").getMember("iterparse").getACall()
diff --git a/python/ql/lib/semmle/python/frameworks/Stdlib.qll b/python/ql/lib/semmle/python/frameworks/Stdlib.qll
index 38fe32a3b3c..e45e8e3a879 100644
--- a/python/ql/lib/semmle/python/frameworks/Stdlib.qll
+++ b/python/ql/lib/semmle/python/frameworks/Stdlib.qll
@@ -3236,7 +3236,7 @@ private module StdlibPrivate {
/**
* A call to the `feed` method of an `xml.etree` parser.
*/
- private class XMLEtreeParserFeedCall extends DataFlow::MethodCallNode, XML::XMLParsing::Range {
+ private class XMLEtreeParserFeedCall extends DataFlow::MethodCallNode, XML::XmlParsing::Range {
XMLEtreeParserFeedCall() { this.calls(instance(), "feed") }
override DataFlow::Node getAnInput() { result in [this.getArg(0), this.getArgByName("data")] }
@@ -3274,7 +3274,7 @@ private module StdlibPrivate {
* - https://docs.python.org/3/library/xml.etree.elementtree.html#xml.etree.ElementTree.parse
* - https://docs.python.org/3/library/xml.etree.elementtree.html#xml.etree.ElementTree.iterparse
*/
- private class XMLEtreeParsing extends DataFlow::CallCfgNode, XML::XMLParsing::Range {
+ private class XMLEtreeParsing extends DataFlow::CallCfgNode, XML::XmlParsing::Range {
XMLEtreeParsing() {
this =
API::moduleImport("xml")
@@ -3445,7 +3445,7 @@ private module StdlibPrivate {
*
* See https://docs.python.org/3/library/xml.sax.reader.html#xml.sax.xmlreader.XMLReader.parse
*/
- private class XMLSaxInstanceParsing extends DataFlow::MethodCallNode, XML::XMLParsing::Range,
+ private class XMLSaxInstanceParsing extends DataFlow::MethodCallNode, XML::XmlParsing::Range,
FileSystemAccess::Range {
XMLSaxInstanceParsing() {
this =
@@ -3496,7 +3496,7 @@ private module StdlibPrivate {
* - https://docs.python.org/3.10/library/xml.sax.html#xml.sax.parse
* - https://docs.python.org/3.10/library/xml.sax.html#xml.sax.parseString
*/
- private class XMLSaxParsing extends DataFlow::CallCfgNode, XML::XMLParsing::Range {
+ private class XMLSaxParsing extends DataFlow::CallCfgNode, XML::XmlParsing::Range {
XMLSaxParsing() {
this =
API::moduleImport("xml").getMember("sax").getMember(["parse", "parseString"]).getACall()
@@ -3562,7 +3562,7 @@ private module StdlibPrivate {
* - https://docs.python.org/3/library/xml.dom.minidom.html#xml.dom.minidom.parse
* - https://docs.python.org/3/library/xml.dom.pulldom.html#xml.dom.pulldom.parse
*/
- private class XMLDomParsing extends DataFlow::CallCfgNode, XML::XMLParsing::Range {
+ private class XMLDomParsing extends DataFlow::CallCfgNode, XML::XmlParsing::Range {
XMLDomParsing() {
this =
API::moduleImport("xml")
diff --git a/python/ql/lib/semmle/python/frameworks/Xmltodict.qll b/python/ql/lib/semmle/python/frameworks/Xmltodict.qll
index bb65607251f..84b0b0fe03f 100644
--- a/python/ql/lib/semmle/python/frameworks/Xmltodict.qll
+++ b/python/ql/lib/semmle/python/frameworks/Xmltodict.qll
@@ -20,7 +20,7 @@ private module Xmltodict {
/**
* A call to `xmltodict.parse`.
*/
- private class XMLtoDictParsing extends DataFlow::CallCfgNode, XML::XMLParsing::Range {
+ private class XMLtoDictParsing extends DataFlow::CallCfgNode, XML::XmlParsing::Range {
XMLtoDictParsing() { this = API::moduleImport("xmltodict").getMember("parse").getACall() }
override DataFlow::Node getAnInput() {
diff --git a/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombCustomizations.qll b/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombCustomizations.qll
index d6f2e0791f9..5da602173a1 100644
--- a/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombCustomizations.qll
+++ b/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombCustomizations.qll
@@ -40,7 +40,7 @@ module XmlBomb {
*/
class XmlParsingWithEntityResolution extends Sink {
XmlParsingWithEntityResolution() {
- exists(XML::XMLParsing parsing, XML::XMLParsingVulnerabilityKind kind |
+ exists(XML::XmlParsing parsing, XML::XMLParsingVulnerabilityKind kind |
(kind.isBillionLaughs() or kind.isQuadraticBlowup()) and
parsing.vulnerableTo(kind) and
this = parsing.getAnInput()
diff --git a/python/ql/src/experimental/semmle/python/security/dataflow/XxeCustomizations.qll b/python/ql/src/experimental/semmle/python/security/dataflow/XxeCustomizations.qll
index a4473285b8d..355b3aeefc9 100644
--- a/python/ql/src/experimental/semmle/python/security/dataflow/XxeCustomizations.qll
+++ b/python/ql/src/experimental/semmle/python/security/dataflow/XxeCustomizations.qll
@@ -40,7 +40,7 @@ module Xxe {
*/
class XmlParsingWithExternalEntityResolution extends Sink {
XmlParsingWithExternalEntityResolution() {
- exists(XML::XMLParsing parsing, XML::XMLParsingVulnerabilityKind kind |
+ exists(XML::XmlParsing parsing, XML::XMLParsingVulnerabilityKind kind |
kind.isXxe() and
parsing.vulnerableTo(kind) and
this = parsing.getAnInput()
diff --git a/python/ql/test/experimental/meta/ConceptsTest.qll b/python/ql/test/experimental/meta/ConceptsTest.qll
index cd90d716dd4..24c3c270413 100644
--- a/python/ql/test/experimental/meta/ConceptsTest.qll
+++ b/python/ql/test/experimental/meta/ConceptsTest.qll
@@ -547,7 +547,7 @@ class XmlParsingTest extends InlineExpectationsTest {
override predicate hasActualResult(Location location, string element, string tag, string value) {
exists(location.getFile().getRelativePath()) and
- exists(XML::XMLParsing parsing, XML::XMLParsingVulnerabilityKind kind |
+ exists(XML::XmlParsing parsing, XML::XMLParsingVulnerabilityKind kind |
parsing.vulnerableTo(kind) and
location = parsing.getLocation() and
element = parsing.toString() and
From 1f285b8983c15e31b08886aa4080f4fad3c8b42b Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Tue, 5 Apr 2022 11:07:12 +0200
Subject: [PATCH 079/171] Python: Rename to `XmlParsingVulnerabilityKind`
To keep up with style guide
---
python/ql/lib/semmle/python/Concepts.qll | 8 ++++----
python/ql/lib/semmle/python/frameworks/Lxml.qll | 14 +++++++-------
python/ql/lib/semmle/python/frameworks/Stdlib.qll | 10 +++++-----
.../ql/lib/semmle/python/frameworks/Xmltodict.qll | 2 +-
.../Security/CWE-611/SimpleXmlRpcServer.ql | 2 +-
.../security/dataflow/XmlBombCustomizations.qll | 2 +-
.../python/security/dataflow/XxeCustomizations.qll | 2 +-
python/ql/test/experimental/meta/ConceptsTest.qll | 2 +-
8 files changed, 21 insertions(+), 21 deletions(-)
diff --git a/python/ql/lib/semmle/python/Concepts.qll b/python/ql/lib/semmle/python/Concepts.qll
index b0a5e1766a2..091ce31a157 100644
--- a/python/ql/lib/semmle/python/Concepts.qll
+++ b/python/ql/lib/semmle/python/Concepts.qll
@@ -558,8 +558,8 @@ module XML {
*
* See PoC at `python/PoCs/XmlParsing/PoC.py` for some tests of vulnerable XML parsing.
*/
- class XMLParsingVulnerabilityKind extends string {
- XMLParsingVulnerabilityKind() {
+ class XmlParsingVulnerabilityKind extends string {
+ XmlParsingVulnerabilityKind() {
this in ["Billion Laughs", "Quadratic Blowup", "XXE", "DTD retrieval"]
}
@@ -586,7 +586,7 @@ module XML {
/**
* Holds if this XML parsing is vulnerable to `kind`.
*/
- predicate vulnerableTo(XMLParsingVulnerabilityKind kind) { super.vulnerableTo(kind) }
+ predicate vulnerableTo(XmlParsingVulnerabilityKind kind) { super.vulnerableTo(kind) }
}
/** Provides classes for modeling XML parsing APIs. */
@@ -601,7 +601,7 @@ module XML {
/**
* Holds if this XML parsing is vulnerable to `kind`.
*/
- abstract predicate vulnerableTo(XMLParsingVulnerabilityKind kind);
+ abstract predicate vulnerableTo(XmlParsingVulnerabilityKind kind);
override string getFormat() { result = "XML" }
}
diff --git a/python/ql/lib/semmle/python/frameworks/Lxml.qll b/python/ql/lib/semmle/python/frameworks/Lxml.qll
index 05dfd388dac..6d310563ade 100644
--- a/python/ql/lib/semmle/python/frameworks/Lxml.qll
+++ b/python/ql/lib/semmle/python/frameworks/Lxml.qll
@@ -121,7 +121,7 @@ private module Lxml {
*/
abstract class InstanceSource extends DataFlow::LocalSourceNode {
/** Holds if this instance is vulnerable to `kind`. */
- abstract predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind);
+ abstract predicate vulnerableTo(XML::XmlParsingVulnerabilityKind kind);
}
/**
@@ -135,7 +135,7 @@ private module Lxml {
}
// NOTE: it's not possible to change settings of a parser after constructing it
- override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
+ override predicate vulnerableTo(XML::XmlParsingVulnerabilityKind kind) {
kind.isXxe() and
(
// resolve_entities has default True
@@ -165,7 +165,7 @@ private module Lxml {
API::moduleImport("lxml").getMember("etree").getMember("get_default_parser").getACall()
}
- override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
+ override predicate vulnerableTo(XML::XmlParsingVulnerabilityKind kind) {
// as highlighted by
// https://lxml.de/apidoc/lxml.etree.html?highlight=xmlparser#lxml.etree.XMLParser
// by default XXE is allow. so as long as the default parser has not been
@@ -189,7 +189,7 @@ private module Lxml {
}
/** Gets a reference to an `lxml.etree` parser instance, that is vulnerable to `kind`. */
- DataFlow::Node instanceVulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
+ DataFlow::Node instanceVulnerableTo(XML::XmlParsingVulnerabilityKind kind) {
exists(InstanceSource origin | result = instance(origin) and origin.vulnerableTo(kind))
}
@@ -201,7 +201,7 @@ private module Lxml {
override DataFlow::Node getAnInput() { result in [this.getArg(0), this.getArgByName("data")] }
- override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
+ override predicate vulnerableTo(XML::XmlParsingVulnerabilityKind kind) {
this.calls(instanceVulnerableTo(kind), "feed")
}
@@ -256,7 +256,7 @@ private module Lxml {
DataFlow::Node getParserArg() { result in [this.getArg(1), this.getArgByName("parser")] }
- override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
+ override predicate vulnerableTo(XML::XmlParsingVulnerabilityKind kind) {
this.getParserArg() = XMLParser::instanceVulnerableTo(kind)
or
kind.isXxe() and
@@ -313,7 +313,7 @@ private module Lxml {
override DataFlow::Node getAnInput() { result in [this.getArg(0), this.getArgByName("source")] }
- override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
+ override predicate vulnerableTo(XML::XmlParsingVulnerabilityKind kind) {
// note that there is no `resolve_entities` argument, so it's not possible to turn off XXE :O
kind.isXxe()
or
diff --git a/python/ql/lib/semmle/python/frameworks/Stdlib.qll b/python/ql/lib/semmle/python/frameworks/Stdlib.qll
index e45e8e3a879..91ba7bc75b5 100644
--- a/python/ql/lib/semmle/python/frameworks/Stdlib.qll
+++ b/python/ql/lib/semmle/python/frameworks/Stdlib.qll
@@ -3241,7 +3241,7 @@ private module StdlibPrivate {
override DataFlow::Node getAnInput() { result in [this.getArg(0), this.getArgByName("data")] }
- override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
+ override predicate vulnerableTo(XML::XmlParsingVulnerabilityKind kind) {
kind.isBillionLaughs() or kind.isQuadraticBlowup()
}
@@ -3298,7 +3298,7 @@ private module StdlibPrivate {
]
}
- override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
+ override predicate vulnerableTo(XML::XmlParsingVulnerabilityKind kind) {
// note: it does not matter what `xml.etree` parser you are using, you cannot
// change the security features anyway :|
kind.isBillionLaughs() or kind.isQuadraticBlowup()
@@ -3459,7 +3459,7 @@ private module StdlibPrivate {
override DataFlow::Node getAnInput() { result in [this.getArg(0), this.getArgByName("source")] }
- override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
+ override predicate vulnerableTo(XML::XmlParsingVulnerabilityKind kind) {
// always vuln to these
(kind.isBillionLaughs() or kind.isQuadraticBlowup())
or
@@ -3512,7 +3512,7 @@ private module StdlibPrivate {
]
}
- override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
+ override predicate vulnerableTo(XML::XmlParsingVulnerabilityKind kind) {
// always vuln to these
(kind.isBillionLaughs() or kind.isQuadraticBlowup())
}
@@ -3586,7 +3586,7 @@ private module StdlibPrivate {
DataFlow::Node getParserArg() { result in [this.getArg(1), this.getArgByName("parser")] }
- override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
+ override predicate vulnerableTo(XML::XmlParsingVulnerabilityKind kind) {
this.getParserArg() = saxParserWithFeatureExternalGesTurnedOn() and
(kind.isXxe() or kind.isDtdRetrieval())
or
diff --git a/python/ql/lib/semmle/python/frameworks/Xmltodict.qll b/python/ql/lib/semmle/python/frameworks/Xmltodict.qll
index 84b0b0fe03f..db2c443d8e9 100644
--- a/python/ql/lib/semmle/python/frameworks/Xmltodict.qll
+++ b/python/ql/lib/semmle/python/frameworks/Xmltodict.qll
@@ -27,7 +27,7 @@ private module Xmltodict {
result in [this.getArg(0), this.getArgByName("xml_input")]
}
- override predicate vulnerableTo(XML::XMLParsingVulnerabilityKind kind) {
+ override predicate vulnerableTo(XML::XmlParsingVulnerabilityKind kind) {
(kind.isBillionLaughs() or kind.isQuadraticBlowup()) and
this.getArgByName("disable_entities").getALocalSource().asExpr() = any(False f)
}
diff --git a/python/ql/src/experimental/Security/CWE-611/SimpleXmlRpcServer.ql b/python/ql/src/experimental/Security/CWE-611/SimpleXmlRpcServer.ql
index 53ff6eeedb8..e638c13853f 100644
--- a/python/ql/src/experimental/Security/CWE-611/SimpleXmlRpcServer.ql
+++ b/python/ql/src/experimental/Security/CWE-611/SimpleXmlRpcServer.ql
@@ -17,7 +17,7 @@ from DataFlow::CallCfgNode call, string kinds
where
call = API::moduleImport("xmlrpc").getMember("server").getMember("SimpleXMLRPCServer").getACall() and
kinds =
- strictconcat(XML::XMLParsingVulnerabilityKind kind |
+ strictconcat(XML::XmlParsingVulnerabilityKind kind |
kind.isBillionLaughs() or kind.isQuadraticBlowup()
|
kind, ", "
diff --git a/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombCustomizations.qll b/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombCustomizations.qll
index 5da602173a1..05f6fc57a34 100644
--- a/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombCustomizations.qll
+++ b/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombCustomizations.qll
@@ -40,7 +40,7 @@ module XmlBomb {
*/
class XmlParsingWithEntityResolution extends Sink {
XmlParsingWithEntityResolution() {
- exists(XML::XmlParsing parsing, XML::XMLParsingVulnerabilityKind kind |
+ exists(XML::XmlParsing parsing, XML::XmlParsingVulnerabilityKind kind |
(kind.isBillionLaughs() or kind.isQuadraticBlowup()) and
parsing.vulnerableTo(kind) and
this = parsing.getAnInput()
diff --git a/python/ql/src/experimental/semmle/python/security/dataflow/XxeCustomizations.qll b/python/ql/src/experimental/semmle/python/security/dataflow/XxeCustomizations.qll
index 355b3aeefc9..0fc139ec4f3 100644
--- a/python/ql/src/experimental/semmle/python/security/dataflow/XxeCustomizations.qll
+++ b/python/ql/src/experimental/semmle/python/security/dataflow/XxeCustomizations.qll
@@ -40,7 +40,7 @@ module Xxe {
*/
class XmlParsingWithExternalEntityResolution extends Sink {
XmlParsingWithExternalEntityResolution() {
- exists(XML::XmlParsing parsing, XML::XMLParsingVulnerabilityKind kind |
+ exists(XML::XmlParsing parsing, XML::XmlParsingVulnerabilityKind kind |
kind.isXxe() and
parsing.vulnerableTo(kind) and
this = parsing.getAnInput()
diff --git a/python/ql/test/experimental/meta/ConceptsTest.qll b/python/ql/test/experimental/meta/ConceptsTest.qll
index 24c3c270413..73bcf8b4aa9 100644
--- a/python/ql/test/experimental/meta/ConceptsTest.qll
+++ b/python/ql/test/experimental/meta/ConceptsTest.qll
@@ -547,7 +547,7 @@ class XmlParsingTest extends InlineExpectationsTest {
override predicate hasActualResult(Location location, string element, string tag, string value) {
exists(location.getFile().getRelativePath()) and
- exists(XML::XmlParsing parsing, XML::XMLParsingVulnerabilityKind kind |
+ exists(XML::XmlParsing parsing, XML::XmlParsingVulnerabilityKind kind |
parsing.vulnerableTo(kind) and
location = parsing.getLocation() and
element = parsing.toString() and
From a7dab53ed2df129e7bdab97cd04f73b9b133574b Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Tue, 5 Apr 2022 11:46:49 +0200
Subject: [PATCH 080/171] Python: Add change-note
---
python/ql/src/change-notes/2022-04-05-add-xxe-and-xmlbomb.md | 5 +++++
1 file changed, 5 insertions(+)
create mode 100644 python/ql/src/change-notes/2022-04-05-add-xxe-and-xmlbomb.md
diff --git a/python/ql/src/change-notes/2022-04-05-add-xxe-and-xmlbomb.md b/python/ql/src/change-notes/2022-04-05-add-xxe-and-xmlbomb.md
new file mode 100644
index 00000000000..bd867091aea
--- /dev/null
+++ b/python/ql/src/change-notes/2022-04-05-add-xxe-and-xmlbomb.md
@@ -0,0 +1,5 @@
+---
+category: newQuery
+---
+* "XML external entity expansion" (`py/xxe`). Results will appear by default. This query was based on [an experimental query by @jorgectf](https://github.com/github/codeql/pull/6112).
+* "XML internal entity expansion" (`py/xml-bomb`). Results will appear by default. This query was based on [an experimental query by @jorgectf](https://github.com/github/codeql/pull/6112).
From b7f56dd17e982ddace861a561ba851e8e7cf7e5c Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Tue, 5 Apr 2022 12:31:09 +0200
Subject: [PATCH 081/171] Python: Rewrite concepts to use `extends ...
instanceof ...`
This caused compilation time for `ConceptsTest.ql` to go from 1m24s to
7s
---
python/ql/lib/semmle/python/Concepts.qll | 241 ++++++++---------------
1 file changed, 77 insertions(+), 164 deletions(-)
diff --git a/python/ql/lib/semmle/python/Concepts.qll b/python/ql/lib/semmle/python/Concepts.qll
index 091ce31a157..eec0cd0d1a0 100644
--- a/python/ql/lib/semmle/python/Concepts.qll
+++ b/python/ql/lib/semmle/python/Concepts.qll
@@ -17,13 +17,9 @@ private import semmle.python.Frameworks
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `SystemCommandExecution::Range` instead.
*/
-class SystemCommandExecution extends DataFlow::Node {
- SystemCommandExecution::Range range;
-
- SystemCommandExecution() { this = range }
-
+class SystemCommandExecution extends DataFlow::Node instanceof SystemCommandExecution::Range {
/** Gets the argument that specifies the command to be executed. */
- DataFlow::Node getCommand() { result = range.getCommand() }
+ DataFlow::Node getCommand() { result = super.getCommand() }
}
/** Provides a class for modeling new system-command execution APIs. */
@@ -48,13 +44,9 @@ module SystemCommandExecution {
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `FileSystemAccess::Range` instead.
*/
-class FileSystemAccess extends DataFlow::Node {
- FileSystemAccess::Range range;
-
- FileSystemAccess() { this = range }
-
+class FileSystemAccess extends DataFlow::Node instanceof FileSystemAccess::Range {
/** Gets an argument to this file system access that is interpreted as a path. */
- DataFlow::Node getAPathArgument() { result = range.getAPathArgument() }
+ DataFlow::Node getAPathArgument() { result = super.getAPathArgument() }
}
/** Provides a class for modeling new file system access APIs. */
@@ -78,14 +70,12 @@ module FileSystemAccess {
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `FileSystemWriteAccess::Range` instead.
*/
-class FileSystemWriteAccess extends FileSystemAccess {
- override FileSystemWriteAccess::Range range;
-
+class FileSystemWriteAccess extends FileSystemAccess instanceof FileSystemWriteAccess::Range {
/**
* Gets a node that represents data to be written to the file system (possibly with
* some transformation happening before it is written, like JSON encoding).
*/
- DataFlow::Node getADataNode() { result = range.getADataNode() }
+ DataFlow::Node getADataNode() { result = super.getADataNode() }
}
/** Provides a class for modeling new file system writes. */
@@ -111,13 +101,9 @@ module Path {
* A data-flow node that performs path normalization. This is often needed in order
* to safely access paths.
*/
- class PathNormalization extends DataFlow::Node {
- PathNormalization::Range range;
-
- PathNormalization() { this = range }
-
+ class PathNormalization extends DataFlow::Node instanceof PathNormalization::Range {
/** Gets an argument to this path normalization that is interpreted as a path. */
- DataFlow::Node getPathArg() { result = range.getPathArg() }
+ DataFlow::Node getPathArg() { result = super.getPathArg() }
}
/** Provides a class for modeling new path normalization APIs. */
@@ -133,12 +119,10 @@ module Path {
}
/** A data-flow node that checks that a path is safe to access. */
- class SafeAccessCheck extends DataFlow::BarrierGuard {
- SafeAccessCheck::Range range;
-
- SafeAccessCheck() { this = range }
-
- override predicate checks(ControlFlowNode node, boolean branch) { range.checks(node, branch) }
+ class SafeAccessCheck extends DataFlow::BarrierGuard instanceof SafeAccessCheck::Range {
+ override predicate checks(ControlFlowNode node, boolean branch) {
+ SafeAccessCheck::Range.super.checks(node, branch)
+ }
}
/** Provides a class for modeling new path safety checks. */
@@ -160,22 +144,18 @@ module Path {
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `Decoding::Range` instead.
*/
-class Decoding extends DataFlow::Node {
- Decoding::Range range;
-
- Decoding() { this = range }
-
+class Decoding extends DataFlow::Node instanceof Decoding::Range {
/** Holds if this call may execute code embedded in its input. */
- predicate mayExecuteInput() { range.mayExecuteInput() }
+ predicate mayExecuteInput() { super.mayExecuteInput() }
/** Gets an input that is decoded by this function. */
- DataFlow::Node getAnInput() { result = range.getAnInput() }
+ DataFlow::Node getAnInput() { result = super.getAnInput() }
/** Gets the output that contains the decoded data produced by this function. */
- DataFlow::Node getOutput() { result = range.getOutput() }
+ DataFlow::Node getOutput() { result = super.getOutput() }
/** Gets an identifier for the format this function decodes from, such as "JSON". */
- string getFormat() { result = range.getFormat() }
+ string getFormat() { result = super.getFormat() }
}
/** Provides a class for modeling new decoding mechanisms. */
@@ -226,19 +206,15 @@ private class DecodingAdditionalTaintStep extends TaintTracking::AdditionalTaint
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `Encoding::Range` instead.
*/
-class Encoding extends DataFlow::Node {
- Encoding::Range range;
-
- Encoding() { this = range }
-
+class Encoding extends DataFlow::Node instanceof Encoding::Range {
/** Gets an input that is encoded by this function. */
- DataFlow::Node getAnInput() { result = range.getAnInput() }
+ DataFlow::Node getAnInput() { result = super.getAnInput() }
/** Gets the output that contains the encoded data produced by this function. */
- DataFlow::Node getOutput() { result = range.getOutput() }
+ DataFlow::Node getOutput() { result = super.getOutput() }
/** Gets an identifier for the format this function decodes from, such as "JSON". */
- string getFormat() { result = range.getFormat() }
+ string getFormat() { result = super.getFormat() }
}
/** Provides a class for modeling new encoding mechanisms. */
@@ -280,13 +256,9 @@ private class EncodingAdditionalTaintStep extends TaintTracking::AdditionalTaint
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `Logging::Range` instead.
*/
-class Logging extends DataFlow::Node {
- Logging::Range range;
-
- Logging() { this = range }
-
+class Logging extends DataFlow::Node instanceof Logging::Range {
/** Gets an input that is logged. */
- DataFlow::Node getAnInput() { result = range.getAnInput() }
+ DataFlow::Node getAnInput() { result = super.getAnInput() }
}
/** Provides a class for modeling new logging mechanisms. */
@@ -309,13 +281,9 @@ module Logging {
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `CodeExecution::Range` instead.
*/
-class CodeExecution extends DataFlow::Node {
- CodeExecution::Range range;
-
- CodeExecution() { this = range }
-
+class CodeExecution extends DataFlow::Node instanceof CodeExecution::Range {
/** Gets the argument that specifies the code to be executed. */
- DataFlow::Node getCode() { result = range.getCode() }
+ DataFlow::Node getCode() { result = super.getCode() }
}
/** Provides a class for modeling new dynamic code execution APIs. */
@@ -343,13 +311,9 @@ module CodeExecution {
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `SqlConstruction::Range` instead.
*/
-class SqlConstruction extends DataFlow::Node {
- SqlConstruction::Range range;
-
- SqlConstruction() { this = range }
-
+class SqlConstruction extends DataFlow::Node instanceof SqlConstruction::Range {
/** Gets the argument that specifies the SQL statements to be constructed. */
- DataFlow::Node getSql() { result = range.getSql() }
+ DataFlow::Node getSql() { result = super.getSql() }
}
/** Provides a class for modeling new SQL execution APIs. */
@@ -380,13 +344,9 @@ module SqlConstruction {
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `SqlExecution::Range` instead.
*/
-class SqlExecution extends DataFlow::Node {
- SqlExecution::Range range;
-
- SqlExecution() { this = range }
-
+class SqlExecution extends DataFlow::Node instanceof SqlExecution::Range {
/** Gets the argument that specifies the SQL statements to be executed. */
- DataFlow::Node getSql() { result = range.getSql() }
+ DataFlow::Node getSql() { result = super.getSql() }
}
/** Provides a class for modeling new SQL execution APIs. */
@@ -412,22 +372,18 @@ module SqlExecution {
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `RegexExecution::Range` instead.
*/
-class RegexExecution extends DataFlow::Node {
- RegexExecution::Range range;
-
- RegexExecution() { this = range }
-
+class RegexExecution extends DataFlow::Node instanceof RegexExecution::Range {
/** Gets the data flow node for the regex being executed by this node. */
- DataFlow::Node getRegex() { result = range.getRegex() }
+ DataFlow::Node getRegex() { result = super.getRegex() }
/** Gets a dataflow node for the string to be searched or matched against. */
- DataFlow::Node getString() { result = range.getString() }
+ DataFlow::Node getString() { result = super.getString() }
/**
* Gets the name of this regex execution, typically the name of an executing method.
* This is used for nice alert messages and should include the module if possible.
*/
- string getName() { result = range.getName() }
+ string getName() { result = super.getName() }
}
/** Provides classes for modeling new regular-expression execution APIs. */
@@ -466,19 +422,15 @@ module XML {
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `XPathConstruction::Range` instead.
*/
- class XPathConstruction extends DataFlow::Node {
- XPathConstruction::Range range;
-
- XPathConstruction() { this = range }
-
+ class XPathConstruction extends DataFlow::Node instanceof XPathConstruction::Range {
/** Gets the argument that specifies the XPath expressions to be constructed. */
- DataFlow::Node getXPath() { result = range.getXPath() }
+ DataFlow::Node getXPath() { result = super.getXPath() }
/**
* Gets the name of this XPath expression construction, typically the name of an executing method.
* This is used for nice alert messages and should include the module if possible.
*/
- string getName() { result = range.getName() }
+ string getName() { result = super.getName() }
}
/** Provides a class for modeling new XPath construction APIs. */
@@ -513,19 +465,15 @@ module XML {
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `XPathExecution::Range` instead.
*/
- class XPathExecution extends DataFlow::Node {
- XPathExecution::Range range;
-
- XPathExecution() { this = range }
-
+ class XPathExecution extends DataFlow::Node instanceof XPathExecution::Range {
/** Gets the data flow node for the XPath expression being executed by this node. */
- DataFlow::Node getXPath() { result = range.getXPath() }
+ DataFlow::Node getXPath() { result = super.getXPath() }
/**
* Gets the name of this XPath expression execution, typically the name of an executing method.
* This is used for nice alert messages and should include the module if possible.
*/
- string getName() { result = range.getName() }
+ string getName() { result = super.getName() }
}
/** Provides classes for modeling new regular-expression execution APIs. */
@@ -616,16 +564,12 @@ module LDAP {
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `LDAPQuery::Range` instead.
*/
- class LdapExecution extends DataFlow::Node {
- LdapExecution::Range range;
-
- LdapExecution() { this = range }
-
+ class LdapExecution extends DataFlow::Node instanceof LdapExecution::Range {
/** Gets the argument containing the filter string. */
- DataFlow::Node getFilter() { result = range.getFilter() }
+ DataFlow::Node getFilter() { result = super.getFilter() }
/** Gets the argument containing the base DN. */
- DataFlow::Node getBaseDn() { result = range.getBaseDn() }
+ DataFlow::Node getBaseDn() { result = super.getBaseDn() }
}
/** Provides classes for modeling new LDAP query execution-related APIs. */
@@ -653,26 +597,23 @@ module LDAP {
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `Escaping::Range` instead.
*/
-class Escaping extends DataFlow::Node {
- Escaping::Range range;
-
+class Escaping extends DataFlow::Node instanceof Escaping::Range {
Escaping() {
- this = range and
// escapes that don't have _both_ input/output defined are not valid
- exists(range.getAnInput()) and
- exists(range.getOutput())
+ exists(super.getAnInput()) and
+ exists(super.getOutput())
}
/** Gets an input that will be escaped. */
- DataFlow::Node getAnInput() { result = range.getAnInput() }
+ DataFlow::Node getAnInput() { result = super.getAnInput() }
/** Gets the output that contains the escaped data. */
- DataFlow::Node getOutput() { result = range.getOutput() }
+ DataFlow::Node getOutput() { result = super.getOutput() }
/**
* Gets the context that this function escapes for, such as `html`, or `url`.
*/
- string getKind() { result = range.getKind() }
+ string getKind() { result = super.getKind() }
}
/** Provides a class for modeling new escaping APIs. */
@@ -730,7 +671,7 @@ module Escaping {
* `{}
`.
*/
class HtmlEscaping extends Escaping {
- HtmlEscaping() { range.getKind() = Escaping::getHtmlKind() }
+ HtmlEscaping() { super.getKind() = Escaping::getHtmlKind() }
}
/**
@@ -738,7 +679,7 @@ class HtmlEscaping extends Escaping {
* the body of a regex.
*/
class RegexEscaping extends Escaping {
- RegexEscaping() { range.getKind() = Escaping::getRegexKind() }
+ RegexEscaping() { super.getKind() = Escaping::getRegexKind() }
}
/**
@@ -746,14 +687,14 @@ class RegexEscaping extends Escaping {
* in an LDAP search.
*/
class LdapDnEscaping extends Escaping {
- LdapDnEscaping() { range.getKind() = Escaping::getLdapDnKind() }
+ LdapDnEscaping() { super.getKind() = Escaping::getLdapDnKind() }
}
/**
* An escape of a string so it can be safely used as a filter in an LDAP search.
*/
class LdapFilterEscaping extends Escaping {
- LdapFilterEscaping() { range.getKind() = Escaping::getLdapFilterKind() }
+ LdapFilterEscaping() { super.getKind() = Escaping::getLdapFilterKind() }
}
/** Provides classes for modeling HTTP-related APIs. */
@@ -772,29 +713,25 @@ module HTTP {
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `RouteSetup::Range` instead.
*/
- class RouteSetup extends DataFlow::Node {
- RouteSetup::Range range;
-
- RouteSetup() { this = range }
-
+ class RouteSetup extends DataFlow::Node instanceof RouteSetup::Range {
/** Gets the URL pattern for this route, if it can be statically determined. */
- string getUrlPattern() { result = range.getUrlPattern() }
+ string getUrlPattern() { result = super.getUrlPattern() }
/**
* Gets a function that will handle incoming requests for this route, if any.
*
* NOTE: This will be modified in the near future to have a `RequestHandler` result, instead of a `Function`.
*/
- Function getARequestHandler() { result = range.getARequestHandler() }
+ Function getARequestHandler() { result = super.getARequestHandler() }
/**
* Gets a parameter that will receive parts of the url when handling incoming
* requests for this route, if any. These automatically become a `RemoteFlowSource`.
*/
- Parameter getARoutedParameter() { result = range.getARoutedParameter() }
+ Parameter getARoutedParameter() { result = super.getARoutedParameter() }
/** Gets a string that identifies the framework used for this route setup. */
- string getFramework() { result = range.getFramework() }
+ string getFramework() { result = super.getFramework() }
}
/** Provides a class for modeling new HTTP routing APIs. */
@@ -841,19 +778,15 @@ module HTTP {
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `RequestHandler::Range` instead.
*/
- class RequestHandler extends Function {
- RequestHandler::Range range;
-
- RequestHandler() { this = range }
-
+ class RequestHandler extends Function instanceof RequestHandler::Range {
/**
* Gets a parameter that could receive parts of the url when handling incoming
* requests, if any. These automatically become a `RemoteFlowSource`.
*/
- Parameter getARoutedParameter() { result = range.getARoutedParameter() }
+ Parameter getARoutedParameter() { result = super.getARoutedParameter() }
/** Gets a string that identifies the framework used for this route setup. */
- string getFramework() { result = range.getFramework() }
+ string getFramework() { result = super.getFramework() }
}
/** Provides a class for modeling new HTTP request handlers. */
@@ -909,16 +842,12 @@ module HTTP {
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `HttpResponse::Range` instead.
*/
- class HttpResponse extends DataFlow::Node {
- HttpResponse::Range range;
-
- HttpResponse() { this = range }
-
+ class HttpResponse extends DataFlow::Node instanceof HttpResponse::Range {
/** Gets the data-flow node that specifies the body of this HTTP response. */
- DataFlow::Node getBody() { result = range.getBody() }
+ DataFlow::Node getBody() { result = super.getBody() }
/** Gets the mimetype of this HTTP response, if it can be statically determined. */
- string getMimetype() { result = range.getMimetype() }
+ string getMimetype() { result = super.getMimetype() }
}
/** Provides a class for modeling new HTTP response APIs. */
@@ -964,13 +893,9 @@ module HTTP {
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `HttpRedirectResponse::Range` instead.
*/
- class HttpRedirectResponse extends HttpResponse {
- override HttpRedirectResponse::Range range;
-
- HttpRedirectResponse() { this = range }
-
+ class HttpRedirectResponse extends HttpResponse instanceof HttpRedirectResponse::Range {
/** Gets the data-flow node that specifies the location of this HTTP redirect response. */
- DataFlow::Node getRedirectLocation() { result = range.getRedirectLocation() }
+ DataFlow::Node getRedirectLocation() { result = super.getRedirectLocation() }
}
/** Provides a class for modeling new HTTP redirect response APIs. */
@@ -996,25 +921,21 @@ module HTTP {
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `HTTP::CookieWrite::Range` instead.
*/
- class CookieWrite extends DataFlow::Node {
- CookieWrite::Range range;
-
- CookieWrite() { this = range }
-
+ class CookieWrite extends DataFlow::Node instanceof CookieWrite::Range {
/**
* Gets the argument, if any, specifying the raw cookie header.
*/
- DataFlow::Node getHeaderArg() { result = range.getHeaderArg() }
+ DataFlow::Node getHeaderArg() { result = super.getHeaderArg() }
/**
* Gets the argument, if any, specifying the cookie name.
*/
- DataFlow::Node getNameArg() { result = range.getNameArg() }
+ DataFlow::Node getNameArg() { result = super.getNameArg() }
/**
* Gets the argument, if any, specifying the cookie value.
*/
- DataFlow::Node getValueArg() { result = range.getValueArg() }
+ DataFlow::Node getValueArg() { result = super.getValueArg() }
}
/** Provides a class for modeling new cookie writes on HTTP responses. */
@@ -1131,27 +1052,23 @@ module Cryptography {
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `KeyGeneration::Range` instead.
*/
- class KeyGeneration extends DataFlow::Node {
- KeyGeneration::Range range;
-
- KeyGeneration() { this = range }
-
+ class KeyGeneration extends DataFlow::Node instanceof KeyGeneration::Range {
/** Gets the name of the cryptographic algorithm (for example `"RSA"` or `"AES"`). */
- string getName() { result = range.getName() }
+ string getName() { result = super.getName() }
/** Gets the argument that specifies the size of the key in bits, if available. */
- DataFlow::Node getKeySizeArg() { result = range.getKeySizeArg() }
+ DataFlow::Node getKeySizeArg() { result = super.getKeySizeArg() }
/**
* Gets the size of the key generated (in bits), as well as the `origin` that
* explains how we obtained this specific key size.
*/
int getKeySizeWithOrigin(DataFlow::Node origin) {
- result = range.getKeySizeWithOrigin(origin)
+ result = super.getKeySizeWithOrigin(origin)
}
/** Gets the minimum key size (in bits) for this algorithm to be considered secure. */
- int minimumSecureKeySize() { result = range.minimumSecureKeySize() }
+ int minimumSecureKeySize() { result = super.minimumSecureKeySize() }
}
/** Provides classes for modeling new key-pair generation APIs. */
@@ -1230,16 +1147,12 @@ module Cryptography {
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `CryptographicOperation::Range` instead.
*/
- class CryptographicOperation extends DataFlow::Node {
- CryptographicOperation::Range range;
-
- CryptographicOperation() { this = range }
-
+ class CryptographicOperation extends DataFlow::Node instanceof CryptographicOperation::Range {
/** Gets the algorithm used, if it matches a known `CryptographicAlgorithm`. */
- CryptographicAlgorithm getAlgorithm() { result = range.getAlgorithm() }
+ CryptographicAlgorithm getAlgorithm() { result = super.getAlgorithm() }
/** Gets an input the algorithm is used on, for example the plain text input to be encrypted. */
- DataFlow::Node getAnInput() { result = range.getAnInput() }
+ DataFlow::Node getAnInput() { result = super.getAnInput() }
}
/** Provides classes for modeling new applications of a cryptographic algorithms. */
From c784f15762b8ea2f749e1f3d92fe29d498b63de3 Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Wed, 6 Apr 2022 15:40:04 +0200
Subject: [PATCH 082/171] Python: Rename more XML classes to follow convention
- `XMLEtree` to `XmlEtree`
- `XMLSax` to `XmlSax`
- `LXML` to `Lxml`
- `XMLParser` to `XmlParser`
---
.../ql/lib/semmle/python/frameworks/Lxml.qll | 30 +++++++++----------
.../lib/semmle/python/frameworks/Stdlib.qll | 28 ++++++++---------
2 files changed, 29 insertions(+), 29 deletions(-)
diff --git a/python/ql/lib/semmle/python/frameworks/Lxml.qll b/python/ql/lib/semmle/python/frameworks/Lxml.qll
index 6d310563ade..24afbd199df 100644
--- a/python/ql/lib/semmle/python/frameworks/Lxml.qll
+++ b/python/ql/lib/semmle/python/frameworks/Lxml.qll
@@ -109,7 +109,7 @@ private module Lxml {
*
* See https://lxml.de/apidoc/lxml.etree.html?highlight=xmlparser#lxml.etree.XMLParser
*/
- module XMLParser {
+ module XmlParser {
/**
* A source of instances of `lxml.etree` parsers, extend this class to model new instances.
*
@@ -117,7 +117,7 @@ private module Lxml {
* calls, or a special parameter that will be set when functions are called by an external
* library.
*
- * Use the predicate `XMLParser::instance()` to get references to instances of `lxml.etree` parsers.
+ * Use the predicate `XmlParser::instance()` to get references to instances of `lxml.etree` parsers.
*/
abstract class InstanceSource extends DataFlow::LocalSourceNode {
/** Holds if this instance is vulnerable to `kind`. */
@@ -129,8 +129,8 @@ private module Lxml {
*
* See https://lxml.de/apidoc/lxml.etree.html?highlight=xmlparser#lxml.etree.XMLParser
*/
- private class LXMLParser extends InstanceSource, DataFlow::CallCfgNode {
- LXMLParser() {
+ private class LxmlParser extends InstanceSource, DataFlow::CallCfgNode {
+ LxmlParser() {
this = API::moduleImport("lxml").getMember("etree").getMember("XMLParser").getACall()
}
@@ -159,8 +159,8 @@ private module Lxml {
*
* See https://lxml.de/apidoc/lxml.etree.html?highlight=xmlparser#lxml.etree.get_default_parser
*/
- private class LXMLDefaultParser extends InstanceSource, DataFlow::CallCfgNode {
- LXMLDefaultParser() {
+ private class LxmlDefaultParser extends InstanceSource, DataFlow::CallCfgNode {
+ LxmlDefaultParser() {
this =
API::moduleImport("lxml").getMember("etree").getMember("get_default_parser").getACall()
}
@@ -196,8 +196,8 @@ private module Lxml {
/**
* A call to the `feed` method of an `lxml` parser.
*/
- private class LXMLParserFeedCall extends DataFlow::MethodCallNode, XML::XmlParsing::Range {
- LXMLParserFeedCall() { this.calls(instance(_), "feed") }
+ private class LxmlParserFeedCall extends DataFlow::MethodCallNode, XML::XmlParsing::Range {
+ LxmlParserFeedCall() { this.calls(instance(_), "feed") }
override DataFlow::Node getAnInput() { result in [this.getArg(0), this.getArgByName("data")] }
@@ -233,8 +233,8 @@ private module Lxml {
* - https://lxml.de/apidoc/lxml.etree.html?highlight=parseids#lxml.etree.parse
* - https://lxml.de/apidoc/lxml.etree.html?highlight=parseids#lxml.etree.parseid
*/
- private class LXMLParsing extends DataFlow::CallCfgNode, XML::XmlParsing::Range {
- LXMLParsing() {
+ private class LxmlParsing extends DataFlow::CallCfgNode, XML::XmlParsing::Range {
+ LxmlParsing() {
this =
API::moduleImport("lxml")
.getMember("etree")
@@ -257,7 +257,7 @@ private module Lxml {
DataFlow::Node getParserArg() { result in [this.getArg(1), this.getArgByName("parser")] }
override predicate vulnerableTo(XML::XmlParsingVulnerabilityKind kind) {
- this.getParserArg() = XMLParser::instanceVulnerableTo(kind)
+ this.getParserArg() = XmlParser::instanceVulnerableTo(kind)
or
kind.isXxe() and
not exists(this.getParserArg())
@@ -284,8 +284,8 @@ private module Lxml {
* - https://lxml.de/apidoc/lxml.etree.html?highlight=parseids#lxml.etree.parse
* - https://lxml.de/apidoc/lxml.etree.html?highlight=parseids#lxml.etree.parseid
*/
- private class FileAccessFromLXMLParsing extends LXMLParsing, FileSystemAccess::Range {
- FileAccessFromLXMLParsing() {
+ private class FileAccessFromLxmlParsing extends LxmlParsing, FileSystemAccess::Range {
+ FileAccessFromLxmlParsing() {
this = API::moduleImport("lxml").getMember("etree").getMember(["parse", "parseid"]).getACall()
// I considered whether we should try to reduce FPs from people passing file-like
// objects, which will not be a file system access (and couldn't cause a
@@ -305,9 +305,9 @@ private module Lxml {
* See
* - https://lxml.de/apidoc/lxml.etree.html?highlight=parseids#lxml.etree.iterparse
*/
- private class LXMLIterparseCall extends DataFlow::CallCfgNode, XML::XmlParsing::Range,
+ private class LxmlIterparseCall extends DataFlow::CallCfgNode, XML::XmlParsing::Range,
FileSystemAccess::Range {
- LXMLIterparseCall() {
+ LxmlIterparseCall() {
this = API::moduleImport("lxml").getMember("etree").getMember("iterparse").getACall()
}
diff --git a/python/ql/lib/semmle/python/frameworks/Stdlib.qll b/python/ql/lib/semmle/python/frameworks/Stdlib.qll
index 91ba7bc75b5..8508aaef5f0 100644
--- a/python/ql/lib/semmle/python/frameworks/Stdlib.qll
+++ b/python/ql/lib/semmle/python/frameworks/Stdlib.qll
@@ -3191,7 +3191,7 @@ private module StdlibPrivate {
* - https://docs.python.org/3.10/library/xml.etree.elementtree.html#xml.etree.ElementTree.XMLParser
* - https://docs.python.org/3.10/library/xml.etree.elementtree.html#xml.etree.ElementTree.XMLPullParser
*/
- module XMLParser {
+ module XmlParser {
/**
* A source of instances of `xml.etree` parsers, extend this class to model new instances.
*
@@ -3199,7 +3199,7 @@ private module StdlibPrivate {
* calls, or a special parameter that will be set when functions are called by an external
* library.
*
- * Use the predicate `XMLParser::instance()` to get references to instances of `xml.etree` parsers.
+ * Use the predicate `XmlParser::instance()` to get references to instances of `xml.etree` parsers.
*/
abstract class InstanceSource extends DataFlow::LocalSourceNode { }
@@ -3236,8 +3236,8 @@ private module StdlibPrivate {
/**
* A call to the `feed` method of an `xml.etree` parser.
*/
- private class XMLEtreeParserFeedCall extends DataFlow::MethodCallNode, XML::XmlParsing::Range {
- XMLEtreeParserFeedCall() { this.calls(instance(), "feed") }
+ private class XmlEtreeParserFeedCall extends DataFlow::MethodCallNode, XML::XmlParsing::Range {
+ XmlEtreeParserFeedCall() { this.calls(instance(), "feed") }
override DataFlow::Node getAnInput() { result in [this.getArg(0), this.getArgByName("data")] }
@@ -3274,8 +3274,8 @@ private module StdlibPrivate {
* - https://docs.python.org/3/library/xml.etree.elementtree.html#xml.etree.ElementTree.parse
* - https://docs.python.org/3/library/xml.etree.elementtree.html#xml.etree.ElementTree.iterparse
*/
- private class XMLEtreeParsing extends DataFlow::CallCfgNode, XML::XmlParsing::Range {
- XMLEtreeParsing() {
+ private class XmlEtreeParsing extends DataFlow::CallCfgNode, XML::XmlParsing::Range {
+ XmlEtreeParsing() {
this =
API::moduleImport("xml")
.getMember("etree")
@@ -3325,8 +3325,8 @@ private module StdlibPrivate {
* - https://docs.python.org/3/library/xml.etree.elementtree.html#xml.etree.ElementTree.parse
* - https://docs.python.org/3/library/xml.etree.elementtree.html#xml.etree.ElementTree.iterparse
*/
- private class FileAccessFromXMLEtreeParsing extends XMLEtreeParsing, FileSystemAccess::Range {
- FileAccessFromXMLEtreeParsing() {
+ private class FileAccessFromXmlEtreeParsing extends XmlEtreeParsing, FileSystemAccess::Range {
+ FileAccessFromXmlEtreeParsing() {
this =
API::moduleImport("xml")
.getMember("etree")
@@ -3445,9 +3445,9 @@ private module StdlibPrivate {
*
* See https://docs.python.org/3/library/xml.sax.reader.html#xml.sax.xmlreader.XMLReader.parse
*/
- private class XMLSaxInstanceParsing extends DataFlow::MethodCallNode, XML::XmlParsing::Range,
+ private class XmlSaxInstanceParsing extends DataFlow::MethodCallNode, XML::XmlParsing::Range,
FileSystemAccess::Range {
- XMLSaxInstanceParsing() {
+ XmlSaxInstanceParsing() {
this =
API::moduleImport("xml")
.getMember("sax")
@@ -3496,8 +3496,8 @@ private module StdlibPrivate {
* - https://docs.python.org/3.10/library/xml.sax.html#xml.sax.parse
* - https://docs.python.org/3.10/library/xml.sax.html#xml.sax.parseString
*/
- private class XMLSaxParsing extends DataFlow::CallCfgNode, XML::XmlParsing::Range {
- XMLSaxParsing() {
+ private class XmlSaxParsing extends DataFlow::CallCfgNode, XML::XmlParsing::Range {
+ XmlSaxParsing() {
this =
API::moduleImport("xml").getMember("sax").getMember(["parse", "parseString"]).getACall()
}
@@ -3535,8 +3535,8 @@ private module StdlibPrivate {
* - https://docs.python.org/3/library/xml.etree.elementtree.html#xml.etree.ElementTree.parse
* - https://docs.python.org/3/library/xml.etree.elementtree.html#xml.etree.ElementTree.iterparse
*/
- private class FileAccessFromXMLSaxParsing extends XMLSaxParsing, FileSystemAccess::Range {
- FileAccessFromXMLSaxParsing() {
+ private class FileAccessFromXmlSaxParsing extends XmlSaxParsing, FileSystemAccess::Range {
+ FileAccessFromXmlSaxParsing() {
this = API::moduleImport("xml").getMember("sax").getMember("parse").getACall()
// I considered whether we should try to reduce FPs from people passing file-like
// objects, which will not be a file system access (and couldn't cause a
From f2f0873d911dc9bb685fa708707e3f4c1de6fc9d Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Wed, 6 Apr 2022 15:49:06 +0200
Subject: [PATCH 083/171] Python: Use new `API::CallNode` for XML constant
check
This also means that the detection of the values passed to these keyword
arguments will no longer just be from a local scope, but can also be
across function boundaries.
---
.../ql/lib/semmle/python/frameworks/Lxml.qll | 21 ++++++++++---------
.../semmle/python/frameworks/Xmltodict.qll | 4 ++--
2 files changed, 13 insertions(+), 12 deletions(-)
diff --git a/python/ql/lib/semmle/python/frameworks/Lxml.qll b/python/ql/lib/semmle/python/frameworks/Lxml.qll
index 24afbd199df..a77da9e7915 100644
--- a/python/ql/lib/semmle/python/frameworks/Lxml.qll
+++ b/python/ql/lib/semmle/python/frameworks/Lxml.qll
@@ -129,7 +129,7 @@ private module Lxml {
*
* See https://lxml.de/apidoc/lxml.etree.html?highlight=xmlparser#lxml.etree.XMLParser
*/
- private class LxmlParser extends InstanceSource, DataFlow::CallCfgNode {
+ private class LxmlParser extends InstanceSource, API::CallNode {
LxmlParser() {
this = API::moduleImport("lxml").getMember("etree").getMember("XMLParser").getACall()
}
@@ -141,16 +141,17 @@ private module Lxml {
// resolve_entities has default True
not exists(this.getArgByName("resolve_entities"))
or
- this.getArgByName("resolve_entities").getALocalSource().asExpr() = any(True t)
+ this.getKeywordParameter("resolve_entities").getAValueReachingRhs().asExpr() = any(True t)
)
or
(kind.isBillionLaughs() or kind.isQuadraticBlowup()) and
- this.getArgByName("huge_tree").getALocalSource().asExpr() = any(True t) and
- not this.getArgByName("resolve_entities").getALocalSource().asExpr() = any(False t)
+ this.getKeywordParameter("huge_tree").getAValueReachingRhs().asExpr() = any(True t) and
+ not this.getKeywordParameter("resolve_entities").getAValueReachingRhs().asExpr() =
+ any(False t)
or
kind.isDtdRetrieval() and
- this.getArgByName("load_dtd").getALocalSource().asExpr() = any(True t) and
- this.getArgByName("no_network").getALocalSource().asExpr() = any(False t)
+ this.getKeywordParameter("load_dtd").getAValueReachingRhs().asExpr() = any(True t) and
+ this.getKeywordParameter("no_network").getAValueReachingRhs().asExpr() = any(False t)
}
}
@@ -305,7 +306,7 @@ private module Lxml {
* See
* - https://lxml.de/apidoc/lxml.etree.html?highlight=parseids#lxml.etree.iterparse
*/
- private class LxmlIterparseCall extends DataFlow::CallCfgNode, XML::XmlParsing::Range,
+ private class LxmlIterparseCall extends API::CallNode, XML::XmlParsing::Range,
FileSystemAccess::Range {
LxmlIterparseCall() {
this = API::moduleImport("lxml").getMember("etree").getMember("iterparse").getACall()
@@ -318,11 +319,11 @@ private module Lxml {
kind.isXxe()
or
(kind.isBillionLaughs() or kind.isQuadraticBlowup()) and
- this.getArgByName("huge_tree").getALocalSource().asExpr() = any(True t)
+ this.getKeywordParameter("huge_tree").getAValueReachingRhs().asExpr() = any(True t)
or
kind.isDtdRetrieval() and
- this.getArgByName("load_dtd").getALocalSource().asExpr() = any(True t) and
- this.getArgByName("no_network").getALocalSource().asExpr() = any(False t)
+ this.getKeywordParameter("load_dtd").getAValueReachingRhs().asExpr() = any(True t) and
+ this.getKeywordParameter("no_network").getAValueReachingRhs().asExpr() = any(False t)
}
override predicate mayExecuteInput() { none() }
diff --git a/python/ql/lib/semmle/python/frameworks/Xmltodict.qll b/python/ql/lib/semmle/python/frameworks/Xmltodict.qll
index db2c443d8e9..95d44d6d1b0 100644
--- a/python/ql/lib/semmle/python/frameworks/Xmltodict.qll
+++ b/python/ql/lib/semmle/python/frameworks/Xmltodict.qll
@@ -20,7 +20,7 @@ private module Xmltodict {
/**
* A call to `xmltodict.parse`.
*/
- private class XMLtoDictParsing extends DataFlow::CallCfgNode, XML::XmlParsing::Range {
+ private class XMLtoDictParsing extends API::CallNode, XML::XmlParsing::Range {
XMLtoDictParsing() { this = API::moduleImport("xmltodict").getMember("parse").getACall() }
override DataFlow::Node getAnInput() {
@@ -29,7 +29,7 @@ private module Xmltodict {
override predicate vulnerableTo(XML::XmlParsingVulnerabilityKind kind) {
(kind.isBillionLaughs() or kind.isQuadraticBlowup()) and
- this.getArgByName("disable_entities").getALocalSource().asExpr() = any(False f)
+ this.getKeywordParameter("disable_entities").getAValueReachingRhs().asExpr() = any(False f)
}
override predicate mayExecuteInput() { none() }
From 7728b6cf1b750eadf462606dbc3ca0660e86417d Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Thu, 7 Apr 2022 10:45:43 +0200
Subject: [PATCH 084/171] Python: Change XmlBomb vulnerability kind
---
python/ql/lib/semmle/python/Concepts.qll | 19 ++++++----
.../ql/lib/semmle/python/frameworks/Lxml.qll | 4 +-
.../lib/semmle/python/frameworks/Stdlib.qll | 12 +++---
.../semmle/python/frameworks/Xmltodict.qll | 2 +-
.../Security/CWE-611/SimpleXmlRpcServer.ql | 12 ++----
.../dataflow/XmlBombCustomizations.qll | 2 +-
.../library-tests/frameworks/lxml/parsing.py | 4 +-
.../frameworks/stdlib/XPathExecution.py | 6 +--
.../frameworks/stdlib/xml_dom.py | 24 ++++++------
.../frameworks/stdlib/xml_etree.py | 38 +++++++++----------
.../frameworks/stdlib/xml_sax.py | 26 ++++++-------
.../frameworks/xmltodict/test.py | 2 +-
12 files changed, 73 insertions(+), 78 deletions(-)
diff --git a/python/ql/lib/semmle/python/Concepts.qll b/python/ql/lib/semmle/python/Concepts.qll
index eec0cd0d1a0..4fadc953c3b 100644
--- a/python/ql/lib/semmle/python/Concepts.qll
+++ b/python/ql/lib/semmle/python/Concepts.qll
@@ -507,15 +507,18 @@ module XML {
* See PoC at `python/PoCs/XmlParsing/PoC.py` for some tests of vulnerable XML parsing.
*/
class XmlParsingVulnerabilityKind extends string {
- XmlParsingVulnerabilityKind() {
- this in ["Billion Laughs", "Quadratic Blowup", "XXE", "DTD retrieval"]
- }
+ XmlParsingVulnerabilityKind() { this in ["XML bomb", "XXE", "DTD retrieval"] }
- /** Holds for Billion Laughs vulnerability kind. */
- predicate isBillionLaughs() { this = "Billion Laughs" }
-
- /** Holds for Quadratic Blowup vulnerability kind. */
- predicate isQuadraticBlowup() { this = "Quadratic Blowup" }
+ /**
+ * Holds for XML bomb vulnerability kind, such as 'Billion Laughs' and 'Quadratic
+ * Blowup'.
+ *
+ * While a parser could technically be vulnerable to one and not the other, from our
+ * point of view the interesting part is that it IS vulnerable to these types of
+ * attacks, and not so much which one specifically works. In practice I haven't seen
+ * a parser that is vulnerable to one and not the other.
+ */
+ predicate isXmlBomb() { this = "XML bomb" }
/** Holds for XXE vulnerability kind. */
predicate isXxe() { this = "XXE" }
diff --git a/python/ql/lib/semmle/python/frameworks/Lxml.qll b/python/ql/lib/semmle/python/frameworks/Lxml.qll
index a77da9e7915..cfb83fd5732 100644
--- a/python/ql/lib/semmle/python/frameworks/Lxml.qll
+++ b/python/ql/lib/semmle/python/frameworks/Lxml.qll
@@ -144,7 +144,7 @@ private module Lxml {
this.getKeywordParameter("resolve_entities").getAValueReachingRhs().asExpr() = any(True t)
)
or
- (kind.isBillionLaughs() or kind.isQuadraticBlowup()) and
+ kind.isXmlBomb() and
this.getKeywordParameter("huge_tree").getAValueReachingRhs().asExpr() = any(True t) and
not this.getKeywordParameter("resolve_entities").getAValueReachingRhs().asExpr() =
any(False t)
@@ -318,7 +318,7 @@ private module Lxml {
// note that there is no `resolve_entities` argument, so it's not possible to turn off XXE :O
kind.isXxe()
or
- (kind.isBillionLaughs() or kind.isQuadraticBlowup()) and
+ kind.isXmlBomb() and
this.getKeywordParameter("huge_tree").getAValueReachingRhs().asExpr() = any(True t)
or
kind.isDtdRetrieval() and
diff --git a/python/ql/lib/semmle/python/frameworks/Stdlib.qll b/python/ql/lib/semmle/python/frameworks/Stdlib.qll
index 8508aaef5f0..f4b6915d440 100644
--- a/python/ql/lib/semmle/python/frameworks/Stdlib.qll
+++ b/python/ql/lib/semmle/python/frameworks/Stdlib.qll
@@ -3241,9 +3241,7 @@ private module StdlibPrivate {
override DataFlow::Node getAnInput() { result in [this.getArg(0), this.getArgByName("data")] }
- override predicate vulnerableTo(XML::XmlParsingVulnerabilityKind kind) {
- kind.isBillionLaughs() or kind.isQuadraticBlowup()
- }
+ override predicate vulnerableTo(XML::XmlParsingVulnerabilityKind kind) { kind.isXmlBomb() }
override predicate mayExecuteInput() { none() }
@@ -3301,7 +3299,7 @@ private module StdlibPrivate {
override predicate vulnerableTo(XML::XmlParsingVulnerabilityKind kind) {
// note: it does not matter what `xml.etree` parser you are using, you cannot
// change the security features anyway :|
- kind.isBillionLaughs() or kind.isQuadraticBlowup()
+ kind.isXmlBomb()
}
override predicate mayExecuteInput() { none() }
@@ -3461,7 +3459,7 @@ private module StdlibPrivate {
override predicate vulnerableTo(XML::XmlParsingVulnerabilityKind kind) {
// always vuln to these
- (kind.isBillionLaughs() or kind.isQuadraticBlowup())
+ kind.isXmlBomb()
or
// can be vuln to other things if features has been turned on
this.getObject() = saxParserWithFeatureExternalGesTurnedOn() and
@@ -3514,7 +3512,7 @@ private module StdlibPrivate {
override predicate vulnerableTo(XML::XmlParsingVulnerabilityKind kind) {
// always vuln to these
- (kind.isBillionLaughs() or kind.isQuadraticBlowup())
+ kind.isXmlBomb()
}
override predicate mayExecuteInput() { none() }
@@ -3590,7 +3588,7 @@ private module StdlibPrivate {
this.getParserArg() = saxParserWithFeatureExternalGesTurnedOn() and
(kind.isXxe() or kind.isDtdRetrieval())
or
- (kind.isBillionLaughs() or kind.isQuadraticBlowup())
+ kind.isXmlBomb()
}
override predicate mayExecuteInput() { none() }
diff --git a/python/ql/lib/semmle/python/frameworks/Xmltodict.qll b/python/ql/lib/semmle/python/frameworks/Xmltodict.qll
index 95d44d6d1b0..f63fec7afe4 100644
--- a/python/ql/lib/semmle/python/frameworks/Xmltodict.qll
+++ b/python/ql/lib/semmle/python/frameworks/Xmltodict.qll
@@ -28,7 +28,7 @@ private module Xmltodict {
}
override predicate vulnerableTo(XML::XmlParsingVulnerabilityKind kind) {
- (kind.isBillionLaughs() or kind.isQuadraticBlowup()) and
+ kind.isXmlBomb() and
this.getKeywordParameter("disable_entities").getAValueReachingRhs().asExpr() = any(False f)
}
diff --git a/python/ql/src/experimental/Security/CWE-611/SimpleXmlRpcServer.ql b/python/ql/src/experimental/Security/CWE-611/SimpleXmlRpcServer.ql
index e638c13853f..e31fdc88629 100644
--- a/python/ql/src/experimental/Security/CWE-611/SimpleXmlRpcServer.ql
+++ b/python/ql/src/experimental/Security/CWE-611/SimpleXmlRpcServer.ql
@@ -13,13 +13,7 @@ private import python
private import semmle.python.Concepts
private import semmle.python.ApiGraphs
-from DataFlow::CallCfgNode call, string kinds
+from DataFlow::CallCfgNode call
where
- call = API::moduleImport("xmlrpc").getMember("server").getMember("SimpleXMLRPCServer").getACall() and
- kinds =
- strictconcat(XML::XmlParsingVulnerabilityKind kind |
- kind.isBillionLaughs() or kind.isQuadraticBlowup()
- |
- kind, ", "
- )
-select call, "SimpleXMLRPCServer is vulnerable to: " + kinds + "."
+ call = API::moduleImport("xmlrpc").getMember("server").getMember("SimpleXMLRPCServer").getACall()
+select call, "SimpleXMLRPCServer is vulnerable to XML bombs"
diff --git a/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombCustomizations.qll b/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombCustomizations.qll
index 05f6fc57a34..7cc4ec5bad5 100644
--- a/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombCustomizations.qll
+++ b/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombCustomizations.qll
@@ -41,7 +41,7 @@ module XmlBomb {
class XmlParsingWithEntityResolution extends Sink {
XmlParsingWithEntityResolution() {
exists(XML::XmlParsing parsing, XML::XmlParsingVulnerabilityKind kind |
- (kind.isBillionLaughs() or kind.isQuadraticBlowup()) and
+ kind.isXmlBomb() and
parsing.vulnerableTo(kind) and
this = parsing.getAnInput()
)
diff --git a/python/ql/test/library-tests/frameworks/lxml/parsing.py b/python/ql/test/library-tests/frameworks/lxml/parsing.py
index ca68c99a90e..63cdc79b4c1 100644
--- a/python/ql/test/library-tests/frameworks/lxml/parsing.py
+++ b/python/ql/test/library-tests/frameworks/lxml/parsing.py
@@ -50,7 +50,7 @@ lxml.etree.fromstring(x, parser=parser) # $ decodeFormat=XML decodeInput=x xmlVu
# Billion laughs vuln (also XXE)
parser = lxml.etree.XMLParser(huge_tree=True)
-lxml.etree.fromstring(x, parser=parser) # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' xmlVuln='XXE' decodeOutput=lxml.etree.fromstring(..)
+lxml.etree.fromstring(x, parser=parser) # $ decodeFormat=XML decodeInput=x xmlVuln='XML bomb' xmlVuln='XXE' decodeOutput=lxml.etree.fromstring(..)
# Safe for both Billion laughs and XXE
parser = lxml.etree.XMLParser(resolve_entities=False, huge_tree=True)
@@ -63,5 +63,5 @@ lxml.etree.fromstring(x, parser=parser) # $ decodeFormat=XML decodeInput=x xmlVu
# iterparse configurations ... this doesn't use a parser argument but takes MOST (!) of
# the normal XMLParser arguments. Specifically, it doesn't allow disabling XXE :O
-lxml.etree.iterparse(xml_file, huge_tree=True) # $ decodeFormat=XML decodeInput=xml_file xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' xmlVuln='XXE' decodeOutput=lxml.etree.iterparse(..) getAPathArgument=xml_file
+lxml.etree.iterparse(xml_file, huge_tree=True) # $ decodeFormat=XML decodeInput=xml_file xmlVuln='XML bomb' xmlVuln='XXE' decodeOutput=lxml.etree.iterparse(..) getAPathArgument=xml_file
lxml.etree.iterparse(xml_file, load_dtd=True, no_network=False) # $ decodeFormat=XML decodeInput=xml_file xmlVuln='DTD retrieval' xmlVuln='XXE' decodeOutput=lxml.etree.iterparse(..) getAPathArgument=xml_file
diff --git a/python/ql/test/library-tests/frameworks/stdlib/XPathExecution.py b/python/ql/test/library-tests/frameworks/stdlib/XPathExecution.py
index 5faff5ed868..bf7dd08185b 100644
--- a/python/ql/test/library-tests/frameworks/stdlib/XPathExecution.py
+++ b/python/ql/test/library-tests/frameworks/stdlib/XPathExecution.py
@@ -2,7 +2,7 @@ match = "dc:title"
ns = {'dc': 'http://purl.org/dc/elements/1.1/'}
import xml.etree.ElementTree as ET
-tree = ET.parse('country_data.xml') # $ decodeFormat=XML decodeInput='country_data.xml' decodeOutput=ET.parse(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' getAPathArgument='country_data.xml'
+tree = ET.parse('country_data.xml') # $ decodeFormat=XML decodeInput='country_data.xml' decodeOutput=ET.parse(..) xmlVuln='XML bomb' getAPathArgument='country_data.xml'
root = tree.getroot()
root.find(match, namespaces=ns) # $ getXPath=match
@@ -10,13 +10,13 @@ root.findall(match, namespaces=ns) # $ getXPath=match
root.findtext(match, default=None, namespaces=ns) # $ getXPath=match
tree = ET.ElementTree()
-tree.parse("index.xhtml") # $ decodeFormat=XML decodeInput="index.xhtml" decodeOutput=tree.parse(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' getAPathArgument="index.xhtml"
+tree.parse("index.xhtml") # $ decodeFormat=XML decodeInput="index.xhtml" decodeOutput=tree.parse(..) xmlVuln='XML bomb' getAPathArgument="index.xhtml"
tree.find(match, namespaces=ns) # $ getXPath=match
tree.findall(match, namespaces=ns) # $ getXPath=match
tree.findtext(match, default=None, namespaces=ns) # $ getXPath=match
parser = ET.XMLParser()
-parser.feed("bar ") # $ decodeFormat=XML decodeInput="bar " xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+parser.feed("bar ") # $ decodeFormat=XML decodeInput="bar " xmlVuln='XML bomb'
tree = parser.close() # $ decodeOutput=parser.close()
tree.find(match, namespaces=ns) # $ getXPath=match
diff --git a/python/ql/test/library-tests/frameworks/stdlib/xml_dom.py b/python/ql/test/library-tests/frameworks/stdlib/xml_dom.py
index b3a1ab7f930..8d511c51733 100644
--- a/python/ql/test/library-tests/frameworks/stdlib/xml_dom.py
+++ b/python/ql/test/library-tests/frameworks/stdlib/xml_dom.py
@@ -6,26 +6,26 @@ import xml.sax
x = "some xml"
# minidom
-xml.dom.minidom.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.dom.minidom.parse(..) getAPathArgument=StringIO(..)
-xml.dom.minidom.parse(file=StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.dom.minidom.parse(..) getAPathArgument=StringIO(..)
+xml.dom.minidom.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='XML bomb' decodeOutput=xml.dom.minidom.parse(..) getAPathArgument=StringIO(..)
+xml.dom.minidom.parse(file=StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='XML bomb' decodeOutput=xml.dom.minidom.parse(..) getAPathArgument=StringIO(..)
-xml.dom.minidom.parseString(x) # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.dom.minidom.parseString(..)
-xml.dom.minidom.parseString(string=x) # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.dom.minidom.parseString(..)
+xml.dom.minidom.parseString(x) # $ decodeFormat=XML decodeInput=x xmlVuln='XML bomb' decodeOutput=xml.dom.minidom.parseString(..)
+xml.dom.minidom.parseString(string=x) # $ decodeFormat=XML decodeInput=x xmlVuln='XML bomb' decodeOutput=xml.dom.minidom.parseString(..)
# pulldom
-xml.dom.pulldom.parse(StringIO(x))['START_DOCUMENT'][1] # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.dom.pulldom.parse(..) getAPathArgument=StringIO(..)
-xml.dom.pulldom.parse(stream_or_string=StringIO(x))['START_DOCUMENT'][1] # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.dom.pulldom.parse(..) getAPathArgument=StringIO(..)
+xml.dom.pulldom.parse(StringIO(x))['START_DOCUMENT'][1] # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='XML bomb' decodeOutput=xml.dom.pulldom.parse(..) getAPathArgument=StringIO(..)
+xml.dom.pulldom.parse(stream_or_string=StringIO(x))['START_DOCUMENT'][1] # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='XML bomb' decodeOutput=xml.dom.pulldom.parse(..) getAPathArgument=StringIO(..)
-xml.dom.pulldom.parseString(x)['START_DOCUMENT'][1] # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.dom.pulldom.parseString(..)
-xml.dom.pulldom.parseString(string=x)['START_DOCUMENT'][1] # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.dom.pulldom.parseString(..)
+xml.dom.pulldom.parseString(x)['START_DOCUMENT'][1] # $ decodeFormat=XML decodeInput=x xmlVuln='XML bomb' decodeOutput=xml.dom.pulldom.parseString(..)
+xml.dom.pulldom.parseString(string=x)['START_DOCUMENT'][1] # $ decodeFormat=XML decodeInput=x xmlVuln='XML bomb' decodeOutput=xml.dom.pulldom.parseString(..)
# These are based on SAX parses, and you can specify your own, so you can expose yourself to XXE (yay/)
parser = xml.sax.make_parser()
parser.setFeature(xml.sax.handler.feature_external_ges, True)
-xml.dom.minidom.parse(StringIO(x), parser) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE' decodeOutput=xml.dom.minidom.parse(..) getAPathArgument=StringIO(..)
-xml.dom.minidom.parse(StringIO(x), parser=parser) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE' decodeOutput=xml.dom.minidom.parse(..) getAPathArgument=StringIO(..)
+xml.dom.minidom.parse(StringIO(x), parser) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='XML bomb' xmlVuln='DTD retrieval' xmlVuln='XXE' decodeOutput=xml.dom.minidom.parse(..) getAPathArgument=StringIO(..)
+xml.dom.minidom.parse(StringIO(x), parser=parser) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='XML bomb' xmlVuln='DTD retrieval' xmlVuln='XXE' decodeOutput=xml.dom.minidom.parse(..) getAPathArgument=StringIO(..)
-xml.dom.pulldom.parse(StringIO(x), parser) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE' decodeOutput=xml.dom.pulldom.parse(..) getAPathArgument=StringIO(..)
-xml.dom.pulldom.parse(StringIO(x), parser=parser) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE' decodeOutput=xml.dom.pulldom.parse(..) getAPathArgument=StringIO(..)
+xml.dom.pulldom.parse(StringIO(x), parser) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='XML bomb' xmlVuln='DTD retrieval' xmlVuln='XXE' decodeOutput=xml.dom.pulldom.parse(..) getAPathArgument=StringIO(..)
+xml.dom.pulldom.parse(StringIO(x), parser=parser) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='XML bomb' xmlVuln='DTD retrieval' xmlVuln='XXE' decodeOutput=xml.dom.pulldom.parse(..) getAPathArgument=StringIO(..)
diff --git a/python/ql/test/library-tests/frameworks/stdlib/xml_etree.py b/python/ql/test/library-tests/frameworks/stdlib/xml_etree.py
index 00f3b964b18..441f9adc87a 100644
--- a/python/ql/test/library-tests/frameworks/stdlib/xml_etree.py
+++ b/python/ql/test/library-tests/frameworks/stdlib/xml_etree.py
@@ -4,43 +4,43 @@ import xml.etree.ElementTree
x = "some xml"
# Parsing in different ways
-xml.etree.ElementTree.fromstring(x) # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.etree.ElementTree.fromstring(..)
-xml.etree.ElementTree.fromstring(text=x) # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.etree.ElementTree.fromstring(..)
+xml.etree.ElementTree.fromstring(x) # $ decodeFormat=XML decodeInput=x xmlVuln='XML bomb' decodeOutput=xml.etree.ElementTree.fromstring(..)
+xml.etree.ElementTree.fromstring(text=x) # $ decodeFormat=XML decodeInput=x xmlVuln='XML bomb' decodeOutput=xml.etree.ElementTree.fromstring(..)
-xml.etree.ElementTree.fromstringlist([x]) # $ decodeFormat=XML decodeInput=List xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.etree.ElementTree.fromstringlist(..)
-xml.etree.ElementTree.fromstringlist(sequence=[x]) # $ decodeFormat=XML decodeInput=List xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.etree.ElementTree.fromstringlist(..)
+xml.etree.ElementTree.fromstringlist([x]) # $ decodeFormat=XML decodeInput=List xmlVuln='XML bomb' decodeOutput=xml.etree.ElementTree.fromstringlist(..)
+xml.etree.ElementTree.fromstringlist(sequence=[x]) # $ decodeFormat=XML decodeInput=List xmlVuln='XML bomb' decodeOutput=xml.etree.ElementTree.fromstringlist(..)
-xml.etree.ElementTree.XML(x) # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.etree.ElementTree.XML(..)
-xml.etree.ElementTree.XML(text=x) # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.etree.ElementTree.XML(..)
+xml.etree.ElementTree.XML(x) # $ decodeFormat=XML decodeInput=x xmlVuln='XML bomb' decodeOutput=xml.etree.ElementTree.XML(..)
+xml.etree.ElementTree.XML(text=x) # $ decodeFormat=XML decodeInput=x xmlVuln='XML bomb' decodeOutput=xml.etree.ElementTree.XML(..)
-xml.etree.ElementTree.XMLID(x) # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.etree.ElementTree.XMLID(..)
-xml.etree.ElementTree.XMLID(text=x) # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.etree.ElementTree.XMLID(..)
+xml.etree.ElementTree.XMLID(x) # $ decodeFormat=XML decodeInput=x xmlVuln='XML bomb' decodeOutput=xml.etree.ElementTree.XMLID(..)
+xml.etree.ElementTree.XMLID(text=x) # $ decodeFormat=XML decodeInput=x xmlVuln='XML bomb' decodeOutput=xml.etree.ElementTree.XMLID(..)
-xml.etree.ElementTree.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.etree.ElementTree.parse(..) getAPathArgument=StringIO(..)
-xml.etree.ElementTree.parse(source=StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.etree.ElementTree.parse(..) getAPathArgument=StringIO(..)
+xml.etree.ElementTree.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='XML bomb' decodeOutput=xml.etree.ElementTree.parse(..) getAPathArgument=StringIO(..)
+xml.etree.ElementTree.parse(source=StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='XML bomb' decodeOutput=xml.etree.ElementTree.parse(..) getAPathArgument=StringIO(..)
-xml.etree.ElementTree.iterparse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.etree.ElementTree.iterparse(..) getAPathArgument=StringIO(..)
-xml.etree.ElementTree.iterparse(source=StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.etree.ElementTree.iterparse(..) getAPathArgument=StringIO(..)
+xml.etree.ElementTree.iterparse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='XML bomb' decodeOutput=xml.etree.ElementTree.iterparse(..) getAPathArgument=StringIO(..)
+xml.etree.ElementTree.iterparse(source=StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='XML bomb' decodeOutput=xml.etree.ElementTree.iterparse(..) getAPathArgument=StringIO(..)
tree = xml.etree.ElementTree.ElementTree()
-tree.parse("file.xml") # $ decodeFormat=XML decodeInput="file.xml" xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=tree.parse(..) getAPathArgument="file.xml"
-tree.parse(source="file.xml") # $ decodeFormat=XML decodeInput="file.xml" xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=tree.parse(..) getAPathArgument="file.xml"
+tree.parse("file.xml") # $ decodeFormat=XML decodeInput="file.xml" xmlVuln='XML bomb' decodeOutput=tree.parse(..) getAPathArgument="file.xml"
+tree.parse(source="file.xml") # $ decodeFormat=XML decodeInput="file.xml" xmlVuln='XML bomb' decodeOutput=tree.parse(..) getAPathArgument="file.xml"
# With parsers (no options available to disable/enable security features)
parser = xml.etree.ElementTree.XMLParser()
-xml.etree.ElementTree.fromstring(x, parser=parser) # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xml.etree.ElementTree.fromstring(..)
+xml.etree.ElementTree.fromstring(x, parser=parser) # $ decodeFormat=XML decodeInput=x xmlVuln='XML bomb' decodeOutput=xml.etree.ElementTree.fromstring(..)
# manual use of feed method
parser = xml.etree.ElementTree.XMLParser()
-parser.feed(x) # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
-parser.feed(data=x) # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+parser.feed(x) # $ decodeFormat=XML decodeInput=x xmlVuln='XML bomb'
+parser.feed(data=x) # $ decodeFormat=XML decodeInput=x xmlVuln='XML bomb'
parser.close() # $ decodeOutput=parser.close()
# manual use of feed method on XMLPullParser
parser = xml.etree.ElementTree.XMLPullParser()
-parser.feed(x) # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
-parser.feed(data=x) # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+parser.feed(x) # $ decodeFormat=XML decodeInput=x xmlVuln='XML bomb'
+parser.feed(data=x) # $ decodeFormat=XML decodeInput=x xmlVuln='XML bomb'
parser.close() # $ decodeOutput=parser.close()
# note: it's technically possible to use the thing wrapper func `fromstring` with an
diff --git a/python/ql/test/library-tests/frameworks/stdlib/xml_sax.py b/python/ql/test/library-tests/frameworks/stdlib/xml_sax.py
index c08034907a4..6199fd76cc1 100644
--- a/python/ql/test/library-tests/frameworks/stdlib/xml_sax.py
+++ b/python/ql/test/library-tests/frameworks/stdlib/xml_sax.py
@@ -10,41 +10,41 @@ class MainHandler(xml.sax.ContentHandler):
def characters(self, data):
self._result.append(data)
-xml.sax.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' getAPathArgument=StringIO(..)
-xml.sax.parse(source=StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' getAPathArgument=StringIO(..)
+xml.sax.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='XML bomb' getAPathArgument=StringIO(..)
+xml.sax.parse(source=StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='XML bomb' getAPathArgument=StringIO(..)
-xml.sax.parseString(x) # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
-xml.sax.parseString(string=x) # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup'
+xml.sax.parseString(x) # $ decodeFormat=XML decodeInput=x xmlVuln='XML bomb'
+xml.sax.parseString(string=x) # $ decodeFormat=XML decodeInput=x xmlVuln='XML bomb'
parser = xml.sax.make_parser()
-parser.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' getAPathArgument=StringIO(..)
-parser.parse(source=StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' getAPathArgument=StringIO(..)
+parser.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='XML bomb' getAPathArgument=StringIO(..)
+parser.parse(source=StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='XML bomb' getAPathArgument=StringIO(..)
# You can make it vuln to both XXE and DTD retrieval by setting this flag
# see https://docs.python.org/3/library/xml.sax.handler.html#xml.sax.handler.feature_external_ges
parser = xml.sax.make_parser()
parser.setFeature(xml.sax.handler.feature_external_ges, True)
-parser.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE' getAPathArgument=StringIO(..)
+parser.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='XML bomb' xmlVuln='DTD retrieval' xmlVuln='XXE' getAPathArgument=StringIO(..)
parser = xml.sax.make_parser()
parser.setFeature(xml.sax.handler.feature_external_ges, False)
-parser.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' getAPathArgument=StringIO(..)
+parser.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='XML bomb' getAPathArgument=StringIO(..)
# Forward Type Tracking test
def func(cond):
parser = xml.sax.make_parser()
if cond:
parser.setFeature(xml.sax.handler.feature_external_ges, True)
- parser.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE' getAPathArgument=StringIO(..)
+ parser.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='XML bomb' xmlVuln='DTD retrieval' xmlVuln='XXE' getAPathArgument=StringIO(..)
else:
- parser.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' getAPathArgument=StringIO(..)
+ parser.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='XML bomb' getAPathArgument=StringIO(..)
# make it vuln, then making it safe
# a bit of an edge-case, but is nice to be able to handle.
parser = xml.sax.make_parser()
parser.setFeature(xml.sax.handler.feature_external_ges, True)
parser.setFeature(xml.sax.handler.feature_external_ges, False)
-parser.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' getAPathArgument=StringIO(..)
+parser.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='XML bomb' getAPathArgument=StringIO(..)
def check_conditional_assignment(cond):
parser = xml.sax.make_parser()
@@ -52,7 +52,7 @@ def check_conditional_assignment(cond):
parser.setFeature(xml.sax.handler.feature_external_ges, True)
else:
parser.setFeature(xml.sax.handler.feature_external_ges, False)
- parser.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE' getAPathArgument=StringIO(..)
+ parser.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='XML bomb' xmlVuln='DTD retrieval' xmlVuln='XXE' getAPathArgument=StringIO(..)
def check_conditional_assignment2(cond):
parser = xml.sax.make_parser()
@@ -61,4 +61,4 @@ def check_conditional_assignment2(cond):
else:
flag_value = False
parser.setFeature(xml.sax.handler.feature_external_ges, flag_value)
- parser.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='Billion Laughs' xmlVuln='DTD retrieval' xmlVuln='Quadratic Blowup' xmlVuln='XXE' getAPathArgument=StringIO(..)
+ parser.parse(StringIO(x)) # $ decodeFormat=XML decodeInput=StringIO(..) xmlVuln='XML bomb' xmlVuln='DTD retrieval' xmlVuln='XXE' getAPathArgument=StringIO(..)
diff --git a/python/ql/test/library-tests/frameworks/xmltodict/test.py b/python/ql/test/library-tests/frameworks/xmltodict/test.py
index 01dc2f3c484..ef236f7796c 100644
--- a/python/ql/test/library-tests/frameworks/xmltodict/test.py
+++ b/python/ql/test/library-tests/frameworks/xmltodict/test.py
@@ -5,4 +5,4 @@ x = "some xml"
xmltodict.parse(x) # $ decodeFormat=XML decodeInput=x decodeOutput=xmltodict.parse(..)
xmltodict.parse(xml_input=x) # $ decodeFormat=XML decodeInput=x decodeOutput=xmltodict.parse(..)
-xmltodict.parse(x, disable_entities=False) # $ decodeFormat=XML decodeInput=x xmlVuln='Billion Laughs' xmlVuln='Quadratic Blowup' decodeOutput=xmltodict.parse(..)
+xmltodict.parse(x, disable_entities=False) # $ decodeFormat=XML decodeInput=x xmlVuln='XML bomb' decodeOutput=xmltodict.parse(..)
From 405480c41045f943e025aa7d21a33b971b231cf2 Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Thu, 7 Apr 2022 15:34:56 +0200
Subject: [PATCH 085/171] Python: Rename sink definitions for XXE/XML bomb
---
.../python/security/dataflow/XmlBombCustomizations.qll | 7 +++----
.../semmle/python/security/dataflow/XxeCustomizations.qll | 7 +++----
2 files changed, 6 insertions(+), 8 deletions(-)
diff --git a/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombCustomizations.qll b/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombCustomizations.qll
index 7cc4ec5bad5..a2fe1b8ecb2 100644
--- a/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombCustomizations.qll
+++ b/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombCustomizations.qll
@@ -35,11 +35,10 @@ module XmlBomb {
}
/**
- * A call to an XML parser that performs internal entity expansion, viewed
- * as a data flow sink for XML-bomb vulnerabilities.
+ * A call to an XML parser that is vulnerable to XML bombs.
*/
- class XmlParsingWithEntityResolution extends Sink {
- XmlParsingWithEntityResolution() {
+ class XmlParsingVulnerableToXmlBomb extends Sink {
+ XmlParsingVulnerableToXmlBomb() {
exists(XML::XmlParsing parsing, XML::XmlParsingVulnerabilityKind kind |
kind.isXmlBomb() and
parsing.vulnerableTo(kind) and
diff --git a/python/ql/src/experimental/semmle/python/security/dataflow/XxeCustomizations.qll b/python/ql/src/experimental/semmle/python/security/dataflow/XxeCustomizations.qll
index 0fc139ec4f3..1d1ad087f84 100644
--- a/python/ql/src/experimental/semmle/python/security/dataflow/XxeCustomizations.qll
+++ b/python/ql/src/experimental/semmle/python/security/dataflow/XxeCustomizations.qll
@@ -35,11 +35,10 @@ module Xxe {
}
/**
- * A call to an XML parser that performs external entity expansion, viewed
- * as a data flow sink for XXE vulnerabilities.
+ * A call to an XML parser that is vulnerable to XXE.
*/
- class XmlParsingWithExternalEntityResolution extends Sink {
- XmlParsingWithExternalEntityResolution() {
+ class XmlParsingVulnerableToXxe extends Sink {
+ XmlParsingVulnerableToXxe() {
exists(XML::XmlParsing parsing, XML::XmlParsingVulnerabilityKind kind |
kind.isXxe() and
parsing.vulnerableTo(kind) and
From 8191be9d7506bec7909a19f001276d2716d4f600 Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Thu, 7 Apr 2022 15:36:04 +0200
Subject: [PATCH 086/171] Python: Move last XXE/XML bomb out of experimental
---
.../semmle/python/security/dataflow/XmlBombCustomizations.qll | 0
.../semmle/python/security/dataflow/XmlBombQuery.qll | 0
.../semmle/python/security/dataflow/XxeCustomizations.qll | 0
.../semmle/python/security/dataflow/XxeQuery.qll | 0
python/ql/src/Security/CWE-611/Xxe.ql | 2 +-
python/ql/src/Security/CWE-776/XmlBomb.ql | 2 +-
6 files changed, 2 insertions(+), 2 deletions(-)
rename python/ql/{src/experimental => lib}/semmle/python/security/dataflow/XmlBombCustomizations.qll (100%)
rename python/ql/{src/experimental => lib}/semmle/python/security/dataflow/XmlBombQuery.qll (100%)
rename python/ql/{src/experimental => lib}/semmle/python/security/dataflow/XxeCustomizations.qll (100%)
rename python/ql/{src/experimental => lib}/semmle/python/security/dataflow/XxeQuery.qll (100%)
diff --git a/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombCustomizations.qll b/python/ql/lib/semmle/python/security/dataflow/XmlBombCustomizations.qll
similarity index 100%
rename from python/ql/src/experimental/semmle/python/security/dataflow/XmlBombCustomizations.qll
rename to python/ql/lib/semmle/python/security/dataflow/XmlBombCustomizations.qll
diff --git a/python/ql/src/experimental/semmle/python/security/dataflow/XmlBombQuery.qll b/python/ql/lib/semmle/python/security/dataflow/XmlBombQuery.qll
similarity index 100%
rename from python/ql/src/experimental/semmle/python/security/dataflow/XmlBombQuery.qll
rename to python/ql/lib/semmle/python/security/dataflow/XmlBombQuery.qll
diff --git a/python/ql/src/experimental/semmle/python/security/dataflow/XxeCustomizations.qll b/python/ql/lib/semmle/python/security/dataflow/XxeCustomizations.qll
similarity index 100%
rename from python/ql/src/experimental/semmle/python/security/dataflow/XxeCustomizations.qll
rename to python/ql/lib/semmle/python/security/dataflow/XxeCustomizations.qll
diff --git a/python/ql/src/experimental/semmle/python/security/dataflow/XxeQuery.qll b/python/ql/lib/semmle/python/security/dataflow/XxeQuery.qll
similarity index 100%
rename from python/ql/src/experimental/semmle/python/security/dataflow/XxeQuery.qll
rename to python/ql/lib/semmle/python/security/dataflow/XxeQuery.qll
diff --git a/python/ql/src/Security/CWE-611/Xxe.ql b/python/ql/src/Security/CWE-611/Xxe.ql
index f706ea6e909..5cc6da25467 100644
--- a/python/ql/src/Security/CWE-611/Xxe.ql
+++ b/python/ql/src/Security/CWE-611/Xxe.ql
@@ -13,7 +13,7 @@
*/
import python
-import experimental.semmle.python.security.dataflow.XxeQuery
+import semmle.python.security.dataflow.XxeQuery
import DataFlow::PathGraph
from Configuration cfg, DataFlow::PathNode source, DataFlow::PathNode sink
diff --git a/python/ql/src/Security/CWE-776/XmlBomb.ql b/python/ql/src/Security/CWE-776/XmlBomb.ql
index 2a1ea5916c4..54d483db17e 100644
--- a/python/ql/src/Security/CWE-776/XmlBomb.ql
+++ b/python/ql/src/Security/CWE-776/XmlBomb.ql
@@ -13,7 +13,7 @@
*/
import python
-import experimental.semmle.python.security.dataflow.XmlBombQuery
+import semmle.python.security.dataflow.XmlBombQuery
import DataFlow::PathGraph
from Configuration cfg, DataFlow::PathNode source, DataFlow::PathNode sink
From 30fff1cf8b23e57d32417e4d89b516b0180d5810 Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Thu, 7 Apr 2022 16:02:41 +0200
Subject: [PATCH 087/171] Python: Merge pymongo NoSQL tests
---
.../Security/CWE-943/NoSQLInjection.expected | 24 +++++++++----------
.../Security/CWE-943/pymongo_bad.py | 17 -------------
.../{pymongo_good.py => pymongo_test.py} | 17 +++++++++----
3 files changed, 25 insertions(+), 33 deletions(-)
delete mode 100644 python/ql/test/experimental/query-tests/Security/CWE-943/pymongo_bad.py
rename python/ql/test/experimental/query-tests/Security/CWE-943/{pymongo_good.py => pymongo_test.py} (57%)
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-943/NoSQLInjection.expected b/python/ql/test/experimental/query-tests/Security/CWE-943/NoSQLInjection.expected
index 6fa158370a6..5213b12744d 100644
--- a/python/ql/test/experimental/query-tests/Security/CWE-943/NoSQLInjection.expected
+++ b/python/ql/test/experimental/query-tests/Security/CWE-943/NoSQLInjection.expected
@@ -44,11 +44,11 @@ edges
| mongoengine_bad.py:57:21:57:42 | ControlFlowNode for Subscript | mongoengine_bad.py:58:30:58:42 | ControlFlowNode for unsafe_search |
| mongoengine_bad.py:58:19:58:43 | ControlFlowNode for Attribute() | mongoengine_bad.py:61:29:61:49 | ControlFlowNode for Dict |
| mongoengine_bad.py:58:30:58:42 | ControlFlowNode for unsafe_search | mongoengine_bad.py:58:19:58:43 | ControlFlowNode for Attribute() |
-| pymongo_bad.py:11:21:11:27 | ControlFlowNode for request | pymongo_bad.py:11:21:11:32 | ControlFlowNode for Attribute |
-| pymongo_bad.py:11:21:11:32 | ControlFlowNode for Attribute | pymongo_bad.py:11:21:11:42 | ControlFlowNode for Subscript |
-| pymongo_bad.py:11:21:11:42 | ControlFlowNode for Subscript | pymongo_bad.py:12:30:12:42 | ControlFlowNode for unsafe_search |
-| pymongo_bad.py:12:19:12:43 | ControlFlowNode for Attribute() | pymongo_bad.py:14:42:14:62 | ControlFlowNode for Dict |
-| pymongo_bad.py:12:30:12:42 | ControlFlowNode for unsafe_search | pymongo_bad.py:12:19:12:43 | ControlFlowNode for Attribute() |
+| pymongo_test.py:12:21:12:27 | ControlFlowNode for request | pymongo_test.py:12:21:12:32 | ControlFlowNode for Attribute |
+| pymongo_test.py:12:21:12:32 | ControlFlowNode for Attribute | pymongo_test.py:12:21:12:42 | ControlFlowNode for Subscript |
+| pymongo_test.py:12:21:12:42 | ControlFlowNode for Subscript | pymongo_test.py:13:30:13:42 | ControlFlowNode for unsafe_search |
+| pymongo_test.py:13:19:13:43 | ControlFlowNode for Attribute() | pymongo_test.py:15:42:15:62 | ControlFlowNode for Dict |
+| pymongo_test.py:13:30:13:42 | ControlFlowNode for unsafe_search | pymongo_test.py:13:19:13:43 | ControlFlowNode for Attribute() |
nodes
| flask_mongoengine_bad.py:19:21:19:27 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
| flask_mongoengine_bad.py:19:21:19:32 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
@@ -104,12 +104,12 @@ nodes
| mongoengine_bad.py:58:19:58:43 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() |
| mongoengine_bad.py:58:30:58:42 | ControlFlowNode for unsafe_search | semmle.label | ControlFlowNode for unsafe_search |
| mongoengine_bad.py:61:29:61:49 | ControlFlowNode for Dict | semmle.label | ControlFlowNode for Dict |
-| pymongo_bad.py:11:21:11:27 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
-| pymongo_bad.py:11:21:11:32 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
-| pymongo_bad.py:11:21:11:42 | ControlFlowNode for Subscript | semmle.label | ControlFlowNode for Subscript |
-| pymongo_bad.py:12:19:12:43 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() |
-| pymongo_bad.py:12:30:12:42 | ControlFlowNode for unsafe_search | semmle.label | ControlFlowNode for unsafe_search |
-| pymongo_bad.py:14:42:14:62 | ControlFlowNode for Dict | semmle.label | ControlFlowNode for Dict |
+| pymongo_test.py:12:21:12:27 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
+| pymongo_test.py:12:21:12:32 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
+| pymongo_test.py:12:21:12:42 | ControlFlowNode for Subscript | semmle.label | ControlFlowNode for Subscript |
+| pymongo_test.py:13:19:13:43 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() |
+| pymongo_test.py:13:30:13:42 | ControlFlowNode for unsafe_search | semmle.label | ControlFlowNode for unsafe_search |
+| pymongo_test.py:15:42:15:62 | ControlFlowNode for Dict | semmle.label | ControlFlowNode for Dict |
subpaths
#select
| flask_mongoengine_bad.py:22:34:22:44 | ControlFlowNode for json_search | flask_mongoengine_bad.py:19:21:19:27 | ControlFlowNode for request | flask_mongoengine_bad.py:22:34:22:44 | ControlFlowNode for json_search | $@ NoSQL query contains an unsanitized $@ | flask_mongoengine_bad.py:22:34:22:44 | ControlFlowNode for json_search | This | flask_mongoengine_bad.py:19:21:19:27 | ControlFlowNode for request | user-provided value |
@@ -121,4 +121,4 @@ subpaths
| mongoengine_bad.py:46:26:46:46 | ControlFlowNode for Dict | mongoengine_bad.py:42:21:42:27 | ControlFlowNode for request | mongoengine_bad.py:46:26:46:46 | ControlFlowNode for Dict | $@ NoSQL query contains an unsanitized $@ | mongoengine_bad.py:46:26:46:46 | ControlFlowNode for Dict | This | mongoengine_bad.py:42:21:42:27 | ControlFlowNode for request | user-provided value |
| mongoengine_bad.py:53:34:53:44 | ControlFlowNode for json_search | mongoengine_bad.py:50:21:50:27 | ControlFlowNode for request | mongoengine_bad.py:53:34:53:44 | ControlFlowNode for json_search | $@ NoSQL query contains an unsanitized $@ | mongoengine_bad.py:53:34:53:44 | ControlFlowNode for json_search | This | mongoengine_bad.py:50:21:50:27 | ControlFlowNode for request | user-provided value |
| mongoengine_bad.py:61:29:61:49 | ControlFlowNode for Dict | mongoengine_bad.py:57:21:57:27 | ControlFlowNode for request | mongoengine_bad.py:61:29:61:49 | ControlFlowNode for Dict | $@ NoSQL query contains an unsanitized $@ | mongoengine_bad.py:61:29:61:49 | ControlFlowNode for Dict | This | mongoengine_bad.py:57:21:57:27 | ControlFlowNode for request | user-provided value |
-| pymongo_bad.py:14:42:14:62 | ControlFlowNode for Dict | pymongo_bad.py:11:21:11:27 | ControlFlowNode for request | pymongo_bad.py:14:42:14:62 | ControlFlowNode for Dict | $@ NoSQL query contains an unsanitized $@ | pymongo_bad.py:14:42:14:62 | ControlFlowNode for Dict | This | pymongo_bad.py:11:21:11:27 | ControlFlowNode for request | user-provided value |
+| pymongo_test.py:15:42:15:62 | ControlFlowNode for Dict | pymongo_test.py:12:21:12:27 | ControlFlowNode for request | pymongo_test.py:15:42:15:62 | ControlFlowNode for Dict | $@ NoSQL query contains an unsanitized $@ | pymongo_test.py:15:42:15:62 | ControlFlowNode for Dict | This | pymongo_test.py:12:21:12:27 | ControlFlowNode for request | user-provided value |
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-943/pymongo_bad.py b/python/ql/test/experimental/query-tests/Security/CWE-943/pymongo_bad.py
deleted file mode 100644
index da16df828b8..00000000000
--- a/python/ql/test/experimental/query-tests/Security/CWE-943/pymongo_bad.py
+++ /dev/null
@@ -1,17 +0,0 @@
-from flask import Flask, request
-from pymongo import MongoClient
-import json
-
-app = Flask(__name__)
-client = MongoClient()
-
-
-@app.route("/")
-def home_page():
- unsafe_search = request.args['search']
- json_search = json.loads(unsafe_search)
-
- return client.db.collection.find_one({'data': json_search})
-
-# if __name__ == "__main__":
-# app.run(debug=True)
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-943/pymongo_good.py b/python/ql/test/experimental/query-tests/Security/CWE-943/pymongo_test.py
similarity index 57%
rename from python/ql/test/experimental/query-tests/Security/CWE-943/pymongo_good.py
rename to python/ql/test/experimental/query-tests/Security/CWE-943/pymongo_test.py
index 5b112e12385..300567ac8c3 100644
--- a/python/ql/test/experimental/query-tests/Security/CWE-943/pymongo_good.py
+++ b/python/ql/test/experimental/query-tests/Security/CWE-943/pymongo_test.py
@@ -7,13 +7,22 @@ app = Flask(__name__)
client = MongoClient()
-@app.route("/")
-def home_page():
+@app.route("/bad")
+def bad():
+ unsafe_search = request.args['search']
+ json_search = json.loads(unsafe_search)
+
+ return client.db.collection.find_one({'data': json_search})
+
+
+@app.route("/good")
+def good():
unsafe_search = request.args['search']
json_search = json.loads(unsafe_search)
safe_search = sanitize(json_search)
return client.db.collection.find_one({'data': safe_search})
-# if __name__ == "__main__":
-# app.run(debug=True)
+
+if __name__ == "__main__":
+ app.run(debug=True)
From 81fdc1bd78c55bfb215b608b700a88dc21cdcdb0 Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Thu, 7 Apr 2022 16:07:02 +0200
Subject: [PATCH 088/171] Python: Add more `pymongo` NoSQL tests
---
.../Security/CWE-943/NoSQLInjection.expected | 10 ++++++++++
.../Security/CWE-943/pymongo_test.py | 18 ++++++++++++++++++
2 files changed, 28 insertions(+)
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-943/NoSQLInjection.expected b/python/ql/test/experimental/query-tests/Security/CWE-943/NoSQLInjection.expected
index 5213b12744d..c39aea2345d 100644
--- a/python/ql/test/experimental/query-tests/Security/CWE-943/NoSQLInjection.expected
+++ b/python/ql/test/experimental/query-tests/Security/CWE-943/NoSQLInjection.expected
@@ -49,6 +49,10 @@ edges
| pymongo_test.py:12:21:12:42 | ControlFlowNode for Subscript | pymongo_test.py:13:30:13:42 | ControlFlowNode for unsafe_search |
| pymongo_test.py:13:19:13:43 | ControlFlowNode for Attribute() | pymongo_test.py:15:42:15:62 | ControlFlowNode for Dict |
| pymongo_test.py:13:30:13:42 | ControlFlowNode for unsafe_search | pymongo_test.py:13:19:13:43 | ControlFlowNode for Attribute() |
+| pymongo_test.py:29:16:29:51 | ControlFlowNode for Attribute() | pymongo_test.py:33:34:33:73 | ControlFlowNode for Dict |
+| pymongo_test.py:29:27:29:33 | ControlFlowNode for request | pymongo_test.py:29:27:29:38 | ControlFlowNode for Attribute |
+| pymongo_test.py:29:27:29:38 | ControlFlowNode for Attribute | pymongo_test.py:29:27:29:50 | ControlFlowNode for Subscript |
+| pymongo_test.py:29:27:29:50 | ControlFlowNode for Subscript | pymongo_test.py:29:16:29:51 | ControlFlowNode for Attribute() |
nodes
| flask_mongoengine_bad.py:19:21:19:27 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
| flask_mongoengine_bad.py:19:21:19:32 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
@@ -110,6 +114,11 @@ nodes
| pymongo_test.py:13:19:13:43 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() |
| pymongo_test.py:13:30:13:42 | ControlFlowNode for unsafe_search | semmle.label | ControlFlowNode for unsafe_search |
| pymongo_test.py:15:42:15:62 | ControlFlowNode for Dict | semmle.label | ControlFlowNode for Dict |
+| pymongo_test.py:29:16:29:51 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() |
+| pymongo_test.py:29:27:29:33 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
+| pymongo_test.py:29:27:29:38 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
+| pymongo_test.py:29:27:29:50 | ControlFlowNode for Subscript | semmle.label | ControlFlowNode for Subscript |
+| pymongo_test.py:33:34:33:73 | ControlFlowNode for Dict | semmle.label | ControlFlowNode for Dict |
subpaths
#select
| flask_mongoengine_bad.py:22:34:22:44 | ControlFlowNode for json_search | flask_mongoengine_bad.py:19:21:19:27 | ControlFlowNode for request | flask_mongoengine_bad.py:22:34:22:44 | ControlFlowNode for json_search | $@ NoSQL query contains an unsanitized $@ | flask_mongoengine_bad.py:22:34:22:44 | ControlFlowNode for json_search | This | flask_mongoengine_bad.py:19:21:19:27 | ControlFlowNode for request | user-provided value |
@@ -122,3 +131,4 @@ subpaths
| mongoengine_bad.py:53:34:53:44 | ControlFlowNode for json_search | mongoengine_bad.py:50:21:50:27 | ControlFlowNode for request | mongoengine_bad.py:53:34:53:44 | ControlFlowNode for json_search | $@ NoSQL query contains an unsanitized $@ | mongoengine_bad.py:53:34:53:44 | ControlFlowNode for json_search | This | mongoengine_bad.py:50:21:50:27 | ControlFlowNode for request | user-provided value |
| mongoengine_bad.py:61:29:61:49 | ControlFlowNode for Dict | mongoengine_bad.py:57:21:57:27 | ControlFlowNode for request | mongoengine_bad.py:61:29:61:49 | ControlFlowNode for Dict | $@ NoSQL query contains an unsanitized $@ | mongoengine_bad.py:61:29:61:49 | ControlFlowNode for Dict | This | mongoengine_bad.py:57:21:57:27 | ControlFlowNode for request | user-provided value |
| pymongo_test.py:15:42:15:62 | ControlFlowNode for Dict | pymongo_test.py:12:21:12:27 | ControlFlowNode for request | pymongo_test.py:15:42:15:62 | ControlFlowNode for Dict | $@ NoSQL query contains an unsanitized $@ | pymongo_test.py:15:42:15:62 | ControlFlowNode for Dict | This | pymongo_test.py:12:21:12:27 | ControlFlowNode for request | user-provided value |
+| pymongo_test.py:33:34:33:73 | ControlFlowNode for Dict | pymongo_test.py:29:27:29:33 | ControlFlowNode for request | pymongo_test.py:33:34:33:73 | ControlFlowNode for Dict | $@ NoSQL query contains an unsanitized $@ | pymongo_test.py:33:34:33:73 | ControlFlowNode for Dict | This | pymongo_test.py:29:27:29:33 | ControlFlowNode for request | user-provided value |
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-943/pymongo_test.py b/python/ql/test/experimental/query-tests/Security/CWE-943/pymongo_test.py
index 300567ac8c3..052c1c65d4f 100644
--- a/python/ql/test/experimental/query-tests/Security/CWE-943/pymongo_test.py
+++ b/python/ql/test/experimental/query-tests/Security/CWE-943/pymongo_test.py
@@ -24,5 +24,23 @@ def good():
return client.db.collection.find_one({'data': safe_search})
+@app.route("/bad2")
+def bad2():
+ event_id = json.loads(request.args['event_id'])
+ client = MongoClient("localhost", 27017, maxPoolSize=50)
+ db = client.localhost
+ collection = db['collection']
+ cursor = collection.find_one({"$where": f"this._id == '${event_id}'"})
+
+
+@app.route("/bad3")
+def bad3():
+ event_id = json.loads(request.args['event_id'])
+ client = MongoClient("localhost", 27017, maxPoolSize=50)
+ db = client.get_database(name="localhost")
+ collection = db['collection']
+ cursor = collection.find_one({"$where": f"this._id == '${event_id}'"})
+
+
if __name__ == "__main__":
app.run(debug=True)
From 0ce2ced1aadf28e38c518a8786eeab42fa829197 Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Thu, 7 Apr 2022 16:16:10 +0200
Subject: [PATCH 089/171] Python: Model `pymongo.mongo_client.MongoClient`
---
python/ql/src/experimental/semmle/python/frameworks/NoSQL.qll | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/python/ql/src/experimental/semmle/python/frameworks/NoSQL.qll b/python/ql/src/experimental/semmle/python/frameworks/NoSQL.qll
index bdd067218b3..99681c8502d 100644
--- a/python/ql/src/experimental/semmle/python/frameworks/NoSQL.qll
+++ b/python/ql/src/experimental/semmle/python/frameworks/NoSQL.qll
@@ -15,6 +15,10 @@ private module NoSql {
/** Gets a reference to `pymongo.MongoClient` */
private API::Node pyMongo() {
result = API::moduleImport("pymongo").getMember("MongoClient").getReturn()
+ or
+ // see https://pymongo.readthedocs.io/en/stable/api/pymongo/mongo_client.html#pymongo.mongo_client.MongoClient
+ result =
+ API::moduleImport("pymongo").getMember("mongo_client").getMember("MongoClient").getReturn()
}
/** Gets a reference to `flask_pymongo.PyMongo` */
From e58e9a273bd5b4d9aa61f02345fe1c3cff234ffe Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Thu, 7 Apr 2022 16:17:12 +0200
Subject: [PATCH 090/171] Python: `mongoClientInstance` refactoring
---
.../src/experimental/semmle/python/frameworks/NoSQL.qll | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/python/ql/src/experimental/semmle/python/frameworks/NoSQL.qll b/python/ql/src/experimental/semmle/python/frameworks/NoSQL.qll
index 99681c8502d..bfb350915eb 100644
--- a/python/ql/src/experimental/semmle/python/frameworks/NoSQL.qll
+++ b/python/ql/src/experimental/semmle/python/frameworks/NoSQL.qll
@@ -38,7 +38,7 @@ private module NoSql {
* Gets a reference to an initialized `Mongo` instance.
* See `pyMongo()`, `flask_PyMongo()`
*/
- private API::Node mongoInstance() {
+ private API::Node mongoClientInstance() {
result = pyMongo() or
result = flask_PyMongo()
}
@@ -56,17 +56,17 @@ private module NoSql {
/**
* Gets a reference to a `Mongo` DB use.
*
- * See `mongoInstance()`, `mongoDBInstance()`.
+ * See `mongoClientInstance()`, `mongoDBInstance()`.
*/
private DataFlow::LocalSourceNode mongoDB(DataFlow::TypeTracker t) {
t.start() and
(
exists(SubscriptNode subscript |
- subscript.getObject() = mongoInstance().getAUse().asCfgNode() and
+ subscript.getObject() = mongoClientInstance().getAUse().asCfgNode() and
result.asCfgNode() = subscript
)
or
- result.(DataFlow::AttrRead).getObject() = mongoInstance().getAUse()
+ result.(DataFlow::AttrRead).getObject() = mongoClientInstance().getAUse()
or
result = mongoDBInstance().getAUse()
)
From 7ca19653dfd3242da06820e56392c8d5da2ee600 Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Thu, 7 Apr 2022 16:22:57 +0200
Subject: [PATCH 091/171] Python: `mongoDBInstance` refactor
---
.../semmle/python/frameworks/NoSQL.qll | 34 +++++++------------
1 file changed, 13 insertions(+), 21 deletions(-)
diff --git a/python/ql/src/experimental/semmle/python/frameworks/NoSQL.qll b/python/ql/src/experimental/semmle/python/frameworks/NoSQL.qll
index bfb350915eb..1fd1075b7d4 100644
--- a/python/ql/src/experimental/semmle/python/frameworks/NoSQL.qll
+++ b/python/ql/src/experimental/semmle/python/frameworks/NoSQL.qll
@@ -44,21 +44,9 @@ private module NoSql {
}
/**
- * Gets a reference to an initialized `Mongo` DB instance.
- * See `mongoEngine()`, `flask_MongoEngine()`
+ * Gets a reference to a `Mongo` DB instance.
*/
- private API::Node mongoDBInstance() {
- result = mongoEngine().getMember(["get_db", "connect"]).getReturn() or
- result = mongoEngine().getMember("connection").getMember(["get_db", "connect"]).getReturn() or
- result = flask_MongoEngine().getMember("get_db").getReturn()
- }
-
- /**
- * Gets a reference to a `Mongo` DB use.
- *
- * See `mongoClientInstance()`, `mongoDBInstance()`.
- */
- private DataFlow::LocalSourceNode mongoDB(DataFlow::TypeTracker t) {
+ private DataFlow::LocalSourceNode mongoDBInstance(DataFlow::TypeTracker t) {
t.start() and
(
exists(SubscriptNode subscript |
@@ -68,10 +56,14 @@ private module NoSql {
or
result.(DataFlow::AttrRead).getObject() = mongoClientInstance().getAUse()
or
- result = mongoDBInstance().getAUse()
+ result = mongoEngine().getMember(["get_db", "connect"]).getACall()
+ or
+ result = mongoEngine().getMember("connection").getMember(["get_db", "connect"]).getACall()
+ or
+ result = flask_MongoEngine().getMember("get_db").getACall()
)
or
- exists(DataFlow::TypeTracker t2 | result = mongoDB(t2).track(t2, t))
+ exists(DataFlow::TypeTracker t2 | result = mongoDBInstance(t2).track(t2, t))
}
/**
@@ -85,21 +77,21 @@ private module NoSql {
*
* `mongo.db` would be a use of a `Mongo` instance, and so the result.
*/
- private DataFlow::Node mongoDB() { mongoDB(DataFlow::TypeTracker::end()).flowsTo(result) }
+ private DataFlow::Node mongoDBInstance() {
+ mongoDBInstance(DataFlow::TypeTracker::end()).flowsTo(result)
+ }
/**
* Gets a reference to a `Mongo` collection use.
- *
- * See `mongoDB()`.
*/
private DataFlow::LocalSourceNode mongoCollection(DataFlow::TypeTracker t) {
t.start() and
(
exists(SubscriptNode subscript | result.asCfgNode() = subscript |
- subscript.getObject() = mongoDB().asCfgNode()
+ subscript.getObject() = mongoDBInstance().asCfgNode()
)
or
- result.(DataFlow::AttrRead).getObject() = mongoDB()
+ result.(DataFlow::AttrRead).getObject() = mongoDBInstance()
)
or
exists(DataFlow::TypeTracker t2 | result = mongoCollection(t2).track(t2, t))
From 89eeaf85d50d487a81c464b10a63372c21389e89 Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Thu, 7 Apr 2022 16:24:21 +0200
Subject: [PATCH 092/171] Python: Handle `get_database` on `MongoClient`
instance
---
.../experimental/semmle/python/frameworks/NoSQL.qll | 4 ++++
.../Security/CWE-943/NoSQLInjection.expected | 10 ++++++++++
2 files changed, 14 insertions(+)
diff --git a/python/ql/src/experimental/semmle/python/frameworks/NoSQL.qll b/python/ql/src/experimental/semmle/python/frameworks/NoSQL.qll
index 1fd1075b7d4..1be6cc6f74b 100644
--- a/python/ql/src/experimental/semmle/python/frameworks/NoSQL.qll
+++ b/python/ql/src/experimental/semmle/python/frameworks/NoSQL.qll
@@ -61,6 +61,10 @@ private module NoSql {
result = mongoEngine().getMember("connection").getMember(["get_db", "connect"]).getACall()
or
result = flask_MongoEngine().getMember("get_db").getACall()
+ or
+ // see https://pymongo.readthedocs.io/en/stable/api/pymongo/mongo_client.html#pymongo.mongo_client.MongoClient.get_default_database
+ // see https://pymongo.readthedocs.io/en/stable/api/pymongo/mongo_client.html#pymongo.mongo_client.MongoClient.get_database
+ result = mongoClientInstance().getMember(["get_default_database", "get_database"]).getACall()
)
or
exists(DataFlow::TypeTracker t2 | result = mongoDBInstance(t2).track(t2, t))
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-943/NoSQLInjection.expected b/python/ql/test/experimental/query-tests/Security/CWE-943/NoSQLInjection.expected
index c39aea2345d..677d21b69e7 100644
--- a/python/ql/test/experimental/query-tests/Security/CWE-943/NoSQLInjection.expected
+++ b/python/ql/test/experimental/query-tests/Security/CWE-943/NoSQLInjection.expected
@@ -53,6 +53,10 @@ edges
| pymongo_test.py:29:27:29:33 | ControlFlowNode for request | pymongo_test.py:29:27:29:38 | ControlFlowNode for Attribute |
| pymongo_test.py:29:27:29:38 | ControlFlowNode for Attribute | pymongo_test.py:29:27:29:50 | ControlFlowNode for Subscript |
| pymongo_test.py:29:27:29:50 | ControlFlowNode for Subscript | pymongo_test.py:29:16:29:51 | ControlFlowNode for Attribute() |
+| pymongo_test.py:38:16:38:51 | ControlFlowNode for Attribute() | pymongo_test.py:42:34:42:73 | ControlFlowNode for Dict |
+| pymongo_test.py:38:27:38:33 | ControlFlowNode for request | pymongo_test.py:38:27:38:38 | ControlFlowNode for Attribute |
+| pymongo_test.py:38:27:38:38 | ControlFlowNode for Attribute | pymongo_test.py:38:27:38:50 | ControlFlowNode for Subscript |
+| pymongo_test.py:38:27:38:50 | ControlFlowNode for Subscript | pymongo_test.py:38:16:38:51 | ControlFlowNode for Attribute() |
nodes
| flask_mongoengine_bad.py:19:21:19:27 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
| flask_mongoengine_bad.py:19:21:19:32 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
@@ -119,6 +123,11 @@ nodes
| pymongo_test.py:29:27:29:38 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
| pymongo_test.py:29:27:29:50 | ControlFlowNode for Subscript | semmle.label | ControlFlowNode for Subscript |
| pymongo_test.py:33:34:33:73 | ControlFlowNode for Dict | semmle.label | ControlFlowNode for Dict |
+| pymongo_test.py:38:16:38:51 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() |
+| pymongo_test.py:38:27:38:33 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
+| pymongo_test.py:38:27:38:38 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
+| pymongo_test.py:38:27:38:50 | ControlFlowNode for Subscript | semmle.label | ControlFlowNode for Subscript |
+| pymongo_test.py:42:34:42:73 | ControlFlowNode for Dict | semmle.label | ControlFlowNode for Dict |
subpaths
#select
| flask_mongoengine_bad.py:22:34:22:44 | ControlFlowNode for json_search | flask_mongoengine_bad.py:19:21:19:27 | ControlFlowNode for request | flask_mongoengine_bad.py:22:34:22:44 | ControlFlowNode for json_search | $@ NoSQL query contains an unsanitized $@ | flask_mongoengine_bad.py:22:34:22:44 | ControlFlowNode for json_search | This | flask_mongoengine_bad.py:19:21:19:27 | ControlFlowNode for request | user-provided value |
@@ -132,3 +141,4 @@ subpaths
| mongoengine_bad.py:61:29:61:49 | ControlFlowNode for Dict | mongoengine_bad.py:57:21:57:27 | ControlFlowNode for request | mongoengine_bad.py:61:29:61:49 | ControlFlowNode for Dict | $@ NoSQL query contains an unsanitized $@ | mongoengine_bad.py:61:29:61:49 | ControlFlowNode for Dict | This | mongoengine_bad.py:57:21:57:27 | ControlFlowNode for request | user-provided value |
| pymongo_test.py:15:42:15:62 | ControlFlowNode for Dict | pymongo_test.py:12:21:12:27 | ControlFlowNode for request | pymongo_test.py:15:42:15:62 | ControlFlowNode for Dict | $@ NoSQL query contains an unsanitized $@ | pymongo_test.py:15:42:15:62 | ControlFlowNode for Dict | This | pymongo_test.py:12:21:12:27 | ControlFlowNode for request | user-provided value |
| pymongo_test.py:33:34:33:73 | ControlFlowNode for Dict | pymongo_test.py:29:27:29:33 | ControlFlowNode for request | pymongo_test.py:33:34:33:73 | ControlFlowNode for Dict | $@ NoSQL query contains an unsanitized $@ | pymongo_test.py:33:34:33:73 | ControlFlowNode for Dict | This | pymongo_test.py:29:27:29:33 | ControlFlowNode for request | user-provided value |
+| pymongo_test.py:42:34:42:73 | ControlFlowNode for Dict | pymongo_test.py:38:27:38:33 | ControlFlowNode for request | pymongo_test.py:42:34:42:73 | ControlFlowNode for Dict | $@ NoSQL query contains an unsanitized $@ | pymongo_test.py:42:34:42:73 | ControlFlowNode for Dict | This | pymongo_test.py:38:27:38:33 | ControlFlowNode for request | user-provided value |
From ec66f26ade803c7d999dd852d93f282df1e11b4f Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Thu, 7 Apr 2022 16:32:20 +0200
Subject: [PATCH 093/171] Python: Handle `get_collection` on pymongo DB
---
.../semmle/python/frameworks/NoSQL.qll | 6 ++++++
.../Security/CWE-943/NoSQLInjection.expected | 20 +++++++++----------
.../Security/CWE-943/pymongo_test.py | 3 ++-
3 files changed, 18 insertions(+), 11 deletions(-)
diff --git a/python/ql/src/experimental/semmle/python/frameworks/NoSQL.qll b/python/ql/src/experimental/semmle/python/frameworks/NoSQL.qll
index 1be6cc6f74b..fa135009ed0 100644
--- a/python/ql/src/experimental/semmle/python/frameworks/NoSQL.qll
+++ b/python/ql/src/experimental/semmle/python/frameworks/NoSQL.qll
@@ -96,6 +96,12 @@ private module NoSql {
)
or
result.(DataFlow::AttrRead).getObject() = mongoDBInstance()
+ or
+ // see https://pymongo.readthedocs.io/en/stable/api/pymongo/database.html#pymongo.database.Database.get_collection
+ // see https://pymongo.readthedocs.io/en/stable/api/pymongo/database.html#pymongo.database.Database.create_collection
+ result
+ .(DataFlow::MethodCallNode)
+ .calls(mongoDBInstance(), ["get_collection", "create_collection"])
)
or
exists(DataFlow::TypeTracker t2 | result = mongoCollection(t2).track(t2, t))
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-943/NoSQLInjection.expected b/python/ql/test/experimental/query-tests/Security/CWE-943/NoSQLInjection.expected
index 677d21b69e7..2922cc9f97e 100644
--- a/python/ql/test/experimental/query-tests/Security/CWE-943/NoSQLInjection.expected
+++ b/python/ql/test/experimental/query-tests/Security/CWE-943/NoSQLInjection.expected
@@ -53,10 +53,10 @@ edges
| pymongo_test.py:29:27:29:33 | ControlFlowNode for request | pymongo_test.py:29:27:29:38 | ControlFlowNode for Attribute |
| pymongo_test.py:29:27:29:38 | ControlFlowNode for Attribute | pymongo_test.py:29:27:29:50 | ControlFlowNode for Subscript |
| pymongo_test.py:29:27:29:50 | ControlFlowNode for Subscript | pymongo_test.py:29:16:29:51 | ControlFlowNode for Attribute() |
-| pymongo_test.py:38:16:38:51 | ControlFlowNode for Attribute() | pymongo_test.py:42:34:42:73 | ControlFlowNode for Dict |
-| pymongo_test.py:38:27:38:33 | ControlFlowNode for request | pymongo_test.py:38:27:38:38 | ControlFlowNode for Attribute |
-| pymongo_test.py:38:27:38:38 | ControlFlowNode for Attribute | pymongo_test.py:38:27:38:50 | ControlFlowNode for Subscript |
-| pymongo_test.py:38:27:38:50 | ControlFlowNode for Subscript | pymongo_test.py:38:16:38:51 | ControlFlowNode for Attribute() |
+| pymongo_test.py:39:16:39:51 | ControlFlowNode for Attribute() | pymongo_test.py:43:34:43:73 | ControlFlowNode for Dict |
+| pymongo_test.py:39:27:39:33 | ControlFlowNode for request | pymongo_test.py:39:27:39:38 | ControlFlowNode for Attribute |
+| pymongo_test.py:39:27:39:38 | ControlFlowNode for Attribute | pymongo_test.py:39:27:39:50 | ControlFlowNode for Subscript |
+| pymongo_test.py:39:27:39:50 | ControlFlowNode for Subscript | pymongo_test.py:39:16:39:51 | ControlFlowNode for Attribute() |
nodes
| flask_mongoengine_bad.py:19:21:19:27 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
| flask_mongoengine_bad.py:19:21:19:32 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
@@ -123,11 +123,11 @@ nodes
| pymongo_test.py:29:27:29:38 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
| pymongo_test.py:29:27:29:50 | ControlFlowNode for Subscript | semmle.label | ControlFlowNode for Subscript |
| pymongo_test.py:33:34:33:73 | ControlFlowNode for Dict | semmle.label | ControlFlowNode for Dict |
-| pymongo_test.py:38:16:38:51 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() |
-| pymongo_test.py:38:27:38:33 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
-| pymongo_test.py:38:27:38:38 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
-| pymongo_test.py:38:27:38:50 | ControlFlowNode for Subscript | semmle.label | ControlFlowNode for Subscript |
-| pymongo_test.py:42:34:42:73 | ControlFlowNode for Dict | semmle.label | ControlFlowNode for Dict |
+| pymongo_test.py:39:16:39:51 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() |
+| pymongo_test.py:39:27:39:33 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
+| pymongo_test.py:39:27:39:38 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
+| pymongo_test.py:39:27:39:50 | ControlFlowNode for Subscript | semmle.label | ControlFlowNode for Subscript |
+| pymongo_test.py:43:34:43:73 | ControlFlowNode for Dict | semmle.label | ControlFlowNode for Dict |
subpaths
#select
| flask_mongoengine_bad.py:22:34:22:44 | ControlFlowNode for json_search | flask_mongoengine_bad.py:19:21:19:27 | ControlFlowNode for request | flask_mongoengine_bad.py:22:34:22:44 | ControlFlowNode for json_search | $@ NoSQL query contains an unsanitized $@ | flask_mongoengine_bad.py:22:34:22:44 | ControlFlowNode for json_search | This | flask_mongoengine_bad.py:19:21:19:27 | ControlFlowNode for request | user-provided value |
@@ -141,4 +141,4 @@ subpaths
| mongoengine_bad.py:61:29:61:49 | ControlFlowNode for Dict | mongoengine_bad.py:57:21:57:27 | ControlFlowNode for request | mongoengine_bad.py:61:29:61:49 | ControlFlowNode for Dict | $@ NoSQL query contains an unsanitized $@ | mongoengine_bad.py:61:29:61:49 | ControlFlowNode for Dict | This | mongoengine_bad.py:57:21:57:27 | ControlFlowNode for request | user-provided value |
| pymongo_test.py:15:42:15:62 | ControlFlowNode for Dict | pymongo_test.py:12:21:12:27 | ControlFlowNode for request | pymongo_test.py:15:42:15:62 | ControlFlowNode for Dict | $@ NoSQL query contains an unsanitized $@ | pymongo_test.py:15:42:15:62 | ControlFlowNode for Dict | This | pymongo_test.py:12:21:12:27 | ControlFlowNode for request | user-provided value |
| pymongo_test.py:33:34:33:73 | ControlFlowNode for Dict | pymongo_test.py:29:27:29:33 | ControlFlowNode for request | pymongo_test.py:33:34:33:73 | ControlFlowNode for Dict | $@ NoSQL query contains an unsanitized $@ | pymongo_test.py:33:34:33:73 | ControlFlowNode for Dict | This | pymongo_test.py:29:27:29:33 | ControlFlowNode for request | user-provided value |
-| pymongo_test.py:42:34:42:73 | ControlFlowNode for Dict | pymongo_test.py:38:27:38:33 | ControlFlowNode for request | pymongo_test.py:42:34:42:73 | ControlFlowNode for Dict | $@ NoSQL query contains an unsanitized $@ | pymongo_test.py:42:34:42:73 | ControlFlowNode for Dict | This | pymongo_test.py:38:27:38:33 | ControlFlowNode for request | user-provided value |
+| pymongo_test.py:43:34:43:73 | ControlFlowNode for Dict | pymongo_test.py:39:27:39:33 | ControlFlowNode for request | pymongo_test.py:43:34:43:73 | ControlFlowNode for Dict | $@ NoSQL query contains an unsanitized $@ | pymongo_test.py:43:34:43:73 | ControlFlowNode for Dict | This | pymongo_test.py:39:27:39:33 | ControlFlowNode for request | user-provided value |
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-943/pymongo_test.py b/python/ql/test/experimental/query-tests/Security/CWE-943/pymongo_test.py
index 052c1c65d4f..ecf53ec4f9a 100644
--- a/python/ql/test/experimental/query-tests/Security/CWE-943/pymongo_test.py
+++ b/python/ql/test/experimental/query-tests/Security/CWE-943/pymongo_test.py
@@ -35,10 +35,11 @@ def bad2():
@app.route("/bad3")
def bad3():
+ # using `get_` methods instead of subscript/attribute lookups
event_id = json.loads(request.args['event_id'])
client = MongoClient("localhost", 27017, maxPoolSize=50)
db = client.get_database(name="localhost")
- collection = db['collection']
+ collection = db.get_collection("collection")
cursor = collection.find_one({"$where": f"this._id == '${event_id}'"})
From 517444b5ff3067a178c57bdda5d523bd8c16316c Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Thu, 7 Apr 2022 16:42:40 +0200
Subject: [PATCH 094/171] Python: Fix `SimpleXmlRpcServer.expected`
---
.../CWE-611-SimpleXmlRpcServer/SimpleXmlRpcServer.expected | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-611-SimpleXmlRpcServer/SimpleXmlRpcServer.expected b/python/ql/test/experimental/query-tests/Security/CWE-611-SimpleXmlRpcServer/SimpleXmlRpcServer.expected
index 4a08d61c47a..5f848fb56bb 100644
--- a/python/ql/test/experimental/query-tests/Security/CWE-611-SimpleXmlRpcServer/SimpleXmlRpcServer.expected
+++ b/python/ql/test/experimental/query-tests/Security/CWE-611-SimpleXmlRpcServer/SimpleXmlRpcServer.expected
@@ -1 +1 @@
-| xmlrpc_server.py:7:10:7:48 | ControlFlowNode for SimpleXMLRPCServer() | SimpleXMLRPCServer is vulnerable to: Billion Laughs, Quadratic Blowup. |
+| xmlrpc_server.py:7:10:7:48 | ControlFlowNode for SimpleXMLRPCServer() | SimpleXMLRPCServer is vulnerable to XML bombs |
From bc5dc6ad50abf2985198118c1c86ebf79f1c844a Mon Sep 17 00:00:00 2001
From: Marcono1234
Date: Sun, 10 Apr 2022 18:24:26 +0200
Subject: [PATCH 095/171] Java: Remove TODO comment for `getRuleExpression()`
behavior
Predicate behavior has been fixed on `main`.
---
java/ql/lib/semmle/code/java/Expr.qll | 1 -
1 file changed, 1 deletion(-)
diff --git a/java/ql/lib/semmle/code/java/Expr.qll b/java/ql/lib/semmle/code/java/Expr.qll
index 4659d0e78fc..1e6b14fc9e7 100755
--- a/java/ql/lib/semmle/code/java/Expr.qll
+++ b/java/ql/lib/semmle/code/java/Expr.qll
@@ -2151,7 +2151,6 @@ class StmtExpr extends Expr {
this = any(ForStmt s).getAnUpdate()
or
// Only applies to SwitchStmt, but not to SwitchExpr, see JLS 17 section 14.11.2
- // TODO: Possibly redundant depending on how https://github.com/github/codeql/issues/8570 is resolved
this = any(SwitchStmt s).getACase().getRuleExpression()
or
// TODO: Workarounds for https://github.com/github/codeql/issues/3605
From 785dc1af3c49140a7b4e9ad936ecc80d037b0335 Mon Sep 17 00:00:00 2001
From: Porcupiney Hairs
Date: Tue, 12 Apr 2022 21:09:05 +0530
Subject: [PATCH 096/171] Include changes from review
---
.../Security/CWE-285/PamAuthorization.ql | 62 ++++--------
.../Security/CWE-285/PamAuthorizationBad.py | 4 +-
.../CWE-285/PamAuthorization.expected | 2 +-
.../query-tests/Security/CWE-285/bad.py | 95 ------------------
.../query-tests/Security/CWE-285/good.py | 97 -------------------
.../query-tests/Security/CWE-285/pam_test.py | 59 +++++++++++
6 files changed, 81 insertions(+), 238 deletions(-)
delete mode 100644 python/ql/test/experimental/query-tests/Security/CWE-285/bad.py
delete mode 100644 python/ql/test/experimental/query-tests/Security/CWE-285/good.py
create mode 100644 python/ql/test/experimental/query-tests/Security/CWE-285/pam_test.py
diff --git a/python/ql/src/experimental/Security/CWE-285/PamAuthorization.ql b/python/ql/src/experimental/Security/CWE-285/PamAuthorization.ql
index e67745cceac..3f11728de4e 100644
--- a/python/ql/src/experimental/Security/CWE-285/PamAuthorization.ql
+++ b/python/ql/src/experimental/Security/CWE-285/PamAuthorization.ql
@@ -13,46 +13,24 @@ import semmle.python.ApiGraphs
import experimental.semmle.python.Concepts
import semmle.python.dataflow.new.TaintTracking
-private class LibPam extends API::Node {
- LibPam() {
- exists(
- API::Node cdll, API::Node find_library, API::Node libpam, API::CallNode cdll_call,
- API::CallNode find_lib_call, StrConst str
- |
- API::moduleImport("ctypes").getMember("CDLL") = cdll and
- find_library = API::moduleImport("ctypes.util").getMember("find_library") and
- cdll_call = cdll.getACall() and
- find_lib_call = find_library.getACall() and
- DataFlow::localFlow(DataFlow::exprNode(str), find_lib_call.getArg(0)) and
- str.getText() = "pam" and
- cdll_call.getArg(0) = find_lib_call and
- libpam = cdll_call.getReturn()
- |
- libpam = this
- )
- }
-
- override string toString() { result = "libpam" }
-}
-
-class PamAuthCall extends API::Node {
- PamAuthCall() { exists(LibPam pam | pam.getMember("pam_authenticate") = this) }
-
- override string toString() { result = "pam_authenticate" }
-}
-
-class PamActMgt extends API::Node {
- PamActMgt() { exists(LibPam pam | pam.getMember("pam_acct_mgmt") = this) }
-
- override string toString() { result = "pam_acct_mgmt" }
-}
-
-from PamAuthCall p, API::CallNode u, Expr handle
-where
- u = p.getACall() and
- handle = u.asExpr().(Call).getArg(0) and
- not exists(PamActMgt pam |
- DataFlow::localFlow(DataFlow::exprNode(handle),
- DataFlow::exprNode(pam.getACall().asExpr().(Call).getArg(0)))
+API::Node libPam() {
+ exists(API::CallNode findLibCall, API::CallNode cdllCall, StrConst str |
+ findLibCall = API::moduleImport("ctypes.util").getMember("find_library").getACall() and
+ cdllCall = API::moduleImport("ctypes").getMember("CDLL").getACall() and
+ DataFlow::localFlow(DataFlow::exprNode(str), findLibCall.getArg(0)) and
+ str.getText() = "pam" and
+ cdllCall.getArg(0) = findLibCall
+ |
+ result = cdllCall.getReturn()
)
-select u, "This PAM authentication call may be lead to an authorization bypass."
+}
+
+from API::CallNode authenticateCall, DataFlow::Node handle
+where
+ authenticateCall = libPam().getMember("pam_authenticate").getACall() and
+ handle = authenticateCall.getArg(0) and
+ not exists(API::CallNode acctMgmtCall |
+ acctMgmtCall = libPam().getMember("pam_acct_mgmt").getACall() and
+ DataFlow::localFlow(handle, acctMgmtCall.getArg(0))
+ )
+select authenticateCall, "This PAM authentication call may be lead to an authorization bypass."
diff --git a/python/ql/src/experimental/Security/CWE-285/PamAuthorizationBad.py b/python/ql/src/experimental/Security/CWE-285/PamAuthorizationBad.py
index 3b06156f551..257f9b99729 100644
--- a/python/ql/src/experimental/Security/CWE-285/PamAuthorizationBad.py
+++ b/python/ql/src/experimental/Security/CWE-285/PamAuthorizationBad.py
@@ -1,11 +1,9 @@
def authenticate(self, username, password, service='login', encoding='utf-8', resetcreds=True):
libpam = CDLL(find_library("pam"))
pam_authenticate = libpam.pam_authenticate
- pam_acct_mgmt = libpam.pam_acct_mgmt
pam_authenticate.restype = c_int
pam_authenticate.argtypes = [PamHandle, c_int]
- pam_acct_mgmt.restype = c_int
- pam_acct_mgmt.argtypes = [PamHandle, c_int]
+
handle = PamHandle()
conv = PamConv(my_conv, 0)
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-285/PamAuthorization.expected b/python/ql/test/experimental/query-tests/Security/CWE-285/PamAuthorization.expected
index 52c4c8ac669..cde7271874a 100644
--- a/python/ql/test/experimental/query-tests/Security/CWE-285/PamAuthorization.expected
+++ b/python/ql/test/experimental/query-tests/Security/CWE-285/PamAuthorization.expected
@@ -1 +1 @@
-| bad.py:92:18:92:44 | ControlFlowNode for pam_authenticate() | This PAM authentication call may be lead to an authorization bypass. |
+| pam_test.py:44:18:44:44 | ControlFlowNode for pam_authenticate() | This PAM authentication call may be lead to an authorization bypass. |
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-285/bad.py b/python/ql/test/experimental/query-tests/Security/CWE-285/bad.py
deleted file mode 100644
index 84527d6f6fb..00000000000
--- a/python/ql/test/experimental/query-tests/Security/CWE-285/bad.py
+++ /dev/null
@@ -1,95 +0,0 @@
-from ctypes import CDLL, POINTER, Structure, CFUNCTYPE, cast, byref, sizeof
-from ctypes import c_void_p, c_size_t, c_char_p, c_char, c_int
-from ctypes import memmove
-from ctypes.util import find_library
-
-class PamHandle(Structure):
- _fields_ = [ ("handle", c_void_p) ]
-
- def __init__(self):
- Structure.__init__(self)
- self.handle = 0
-
-class PamMessage(Structure):
- """wrapper class for pam_message structure"""
- _fields_ = [ ("msg_style", c_int), ("msg", c_char_p) ]
-
- def __repr__(self):
- return "" % (self.msg_style, self.msg)
-
-class PamResponse(Structure):
- """wrapper class for pam_response structure"""
- _fields_ = [ ("resp", c_char_p), ("resp_retcode", c_int) ]
-
- def __repr__(self):
- return "" % (self.resp_retcode, self.resp)
-
-conv_func = CFUNCTYPE(c_int, c_int, POINTER(POINTER(PamMessage)), POINTER(POINTER(PamResponse)), c_void_p)
-
-class PamConv(Structure):
- """wrapper class for pam_conv structure"""
- _fields_ = [ ("conv", conv_func), ("appdata_ptr", c_void_p) ]
-
-# Various constants
-PAM_PROMPT_ECHO_OFF = 1
-PAM_PROMPT_ECHO_ON = 2
-PAM_ERROR_MSG = 3
-PAM_TEXT_INFO = 4
-PAM_REINITIALIZE_CRED = 8
-
-libc = CDLL(find_library("c"))
-libpam = CDLL(find_library("pam"))
-
-calloc = libc.calloc
-calloc.restype = c_void_p
-calloc.argtypes = [c_size_t, c_size_t]
-
-# bug #6 (@NIPE-SYSTEMS), some libpam versions don't include this function
-if hasattr(libpam, 'pam_end'):
- pam_end = libpam.pam_end
- pam_end.restype = c_int
- pam_end.argtypes = [PamHandle, c_int]
-
-pam_start = libpam.pam_start
-pam_start.restype = c_int
-pam_start.argtypes = [c_char_p, c_char_p, POINTER(PamConv), POINTER(PamHandle)]
-
-pam_setcred = libpam.pam_setcred
-pam_setcred.restype = c_int
-pam_setcred.argtypes = [PamHandle, c_int]
-
-pam_strerror = libpam.pam_strerror
-pam_strerror.restype = c_char_p
-pam_strerror.argtypes = [PamHandle, c_int]
-
-pam_authenticate = libpam.pam_authenticate
-pam_authenticate.restype = c_int
-pam_authenticate.argtypes = [PamHandle, c_int]
-
-pam_acct_mgmt = libpam.pam_acct_mgmt
-pam_acct_mgmt.restype = c_int
-pam_acct_mgmt.argtypes = [PamHandle, c_int]
-
-class pam():
- code = 0
- reason = None
-
- def __init__(self):
- pass
-
- def authenticate(self, username, password, service='login', encoding='utf-8', resetcreds=True):
- @conv_func
- def my_conv(n_messages, messages, p_response, app_data):
- return 0
-
-
- cpassword = c_char_p(password)
-
- handle = PamHandle()
- conv = PamConv(my_conv, 0)
- retval = pam_start(service, username, byref(conv), byref(handle))
-
- retval = pam_authenticate(handle, 0)
- auth_success = retval == 0
-
- return auth_success
\ No newline at end of file
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-285/good.py b/python/ql/test/experimental/query-tests/Security/CWE-285/good.py
deleted file mode 100644
index e9996c770ed..00000000000
--- a/python/ql/test/experimental/query-tests/Security/CWE-285/good.py
+++ /dev/null
@@ -1,97 +0,0 @@
-from ctypes import CDLL, POINTER, Structure, CFUNCTYPE, cast, byref, sizeof
-from ctypes import c_void_p, c_size_t, c_char_p, c_char, c_int
-from ctypes import memmove
-from ctypes.util import find_library
-
-class PamHandle(Structure):
- _fields_ = [ ("handle", c_void_p) ]
-
- def __init__(self):
- Structure.__init__(self)
- self.handle = 0
-
-class PamMessage(Structure):
- """wrapper class for pam_message structure"""
- _fields_ = [ ("msg_style", c_int), ("msg", c_char_p) ]
-
- def __repr__(self):
- return "" % (self.msg_style, self.msg)
-
-class PamResponse(Structure):
- """wrapper class for pam_response structure"""
- _fields_ = [ ("resp", c_char_p), ("resp_retcode", c_int) ]
-
- def __repr__(self):
- return "" % (self.resp_retcode, self.resp)
-
-conv_func = CFUNCTYPE(c_int, c_int, POINTER(POINTER(PamMessage)), POINTER(POINTER(PamResponse)), c_void_p)
-
-class PamConv(Structure):
- """wrapper class for pam_conv structure"""
- _fields_ = [ ("conv", conv_func), ("appdata_ptr", c_void_p) ]
-
-# Various constants
-PAM_PROMPT_ECHO_OFF = 1
-PAM_PROMPT_ECHO_ON = 2
-PAM_ERROR_MSG = 3
-PAM_TEXT_INFO = 4
-PAM_REINITIALIZE_CRED = 8
-
-libc = CDLL(find_library("c"))
-libpam = CDLL(find_library("pam"))
-
-calloc = libc.calloc
-calloc.restype = c_void_p
-calloc.argtypes = [c_size_t, c_size_t]
-
-# bug #6 (@NIPE-SYSTEMS), some libpam versions don't include this function
-if hasattr(libpam, 'pam_end'):
- pam_end = libpam.pam_end
- pam_end.restype = c_int
- pam_end.argtypes = [PamHandle, c_int]
-
-pam_start = libpam.pam_start
-pam_start.restype = c_int
-pam_start.argtypes = [c_char_p, c_char_p, POINTER(PamConv), POINTER(PamHandle)]
-
-pam_setcred = libpam.pam_setcred
-pam_setcred.restype = c_int
-pam_setcred.argtypes = [PamHandle, c_int]
-
-pam_strerror = libpam.pam_strerror
-pam_strerror.restype = c_char_p
-pam_strerror.argtypes = [PamHandle, c_int]
-
-pam_authenticate = libpam.pam_authenticate
-pam_authenticate.restype = c_int
-pam_authenticate.argtypes = [PamHandle, c_int]
-
-pam_acct_mgmt = libpam.pam_acct_mgmt
-pam_acct_mgmt.restype = c_int
-pam_acct_mgmt.argtypes = [PamHandle, c_int]
-
-class pam():
- code = 0
- reason = None
-
- def __init__(self):
- pass
-
- def authenticate(self, username, password, service='login', encoding='utf-8', resetcreds=True):
- @conv_func
- def my_conv(n_messages, messages, p_response, app_data):
- return 0
-
-
- cpassword = c_char_p(password)
-
- handle = PamHandle()
- conv = PamConv(my_conv, 0)
- retval = pam_start(service, username, byref(conv), byref(handle))
-
- retval = pam_authenticate(handle, 0)
- if retval == 0:
- retval = pam_acct_mgmt(handle, 0)
- auth_success = retval == 0
-
- return auth_success
\ No newline at end of file
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-285/pam_test.py b/python/ql/test/experimental/query-tests/Security/CWE-285/pam_test.py
new file mode 100644
index 00000000000..60408ade722
--- /dev/null
+++ b/python/ql/test/experimental/query-tests/Security/CWE-285/pam_test.py
@@ -0,0 +1,59 @@
+from ctypes import CDLL, POINTER, Structure, byref
+from ctypes import c_char_p, c_int
+from ctypes.util import find_library
+
+
+class PamHandle(Structure):
+ pass
+
+
+class PamMessage(Structure):
+ pass
+
+
+class PamResponse(Structure):
+ pass
+
+
+class PamConv(Structure):
+ pass
+
+
+libpam = CDLL(find_library("pam"))
+
+pam_start = libpam.pam_start
+pam_start.restype = c_int
+pam_start.argtypes = [c_char_p, c_char_p, POINTER(PamConv), POINTER(PamHandle)]
+
+pam_authenticate = libpam.pam_authenticate
+pam_authenticate.restype = c_int
+pam_authenticate.argtypes = [PamHandle, c_int]
+
+pam_acct_mgmt = libpam.pam_acct_mgmt
+pam_acct_mgmt.restype = c_int
+pam_acct_mgmt.argtypes = [PamHandle, c_int]
+
+
+class pam():
+
+ def authenticate_bad(self, username, service='login'):
+ handle = PamHandle()
+ conv = PamConv(None, 0)
+ retval = pam_start(service, username, byref(conv), byref(handle))
+
+ retval = pam_authenticate(handle, 0)
+ auth_success = retval == 0
+
+ return auth_success
+
+ def authenticate_good(self, username, service='login'):
+ handle = PamHandle()
+ conv = PamConv(None, 0)
+ retval = pam_start(service, username, byref(conv), byref(handle))
+
+ retval = pam_authenticate(handle, 0)
+ if retval == 0:
+ retval = pam_acct_mgmt(handle, 0)
+ auth_success = retval == 0
+
+ return auth_success
From 6235dc503965deafd3f30ff76e3c7fcd2d5fceb6 Mon Sep 17 00:00:00 2001
From: Rasmus Wriedt Larsen
Date: Wed, 13 Apr 2022 11:44:15 +0200
Subject: [PATCH 097/171] Python: Handle `find_library` assignment to temp
variable
---
.../src/experimental/Security/CWE-285/PamAuthorization.ql | 7 +++----
.../query-tests/Security/CWE-285/PamAuthorization.expected | 2 +-
.../experimental/query-tests/Security/CWE-285/pam_test.py | 6 +++++-
3 files changed, 9 insertions(+), 6 deletions(-)
diff --git a/python/ql/src/experimental/Security/CWE-285/PamAuthorization.ql b/python/ql/src/experimental/Security/CWE-285/PamAuthorization.ql
index 3f11728de4e..595d1af13a4 100644
--- a/python/ql/src/experimental/Security/CWE-285/PamAuthorization.ql
+++ b/python/ql/src/experimental/Security/CWE-285/PamAuthorization.ql
@@ -14,12 +14,11 @@ import experimental.semmle.python.Concepts
import semmle.python.dataflow.new.TaintTracking
API::Node libPam() {
- exists(API::CallNode findLibCall, API::CallNode cdllCall, StrConst str |
+ exists(API::CallNode findLibCall, API::CallNode cdllCall |
findLibCall = API::moduleImport("ctypes.util").getMember("find_library").getACall() and
+ findLibCall.getParameter(0).getAValueReachingRhs().asExpr().(StrConst).getText() = "pam" and
cdllCall = API::moduleImport("ctypes").getMember("CDLL").getACall() and
- DataFlow::localFlow(DataFlow::exprNode(str), findLibCall.getArg(0)) and
- str.getText() = "pam" and
- cdllCall.getArg(0) = findLibCall
+ cdllCall.getParameter(0).getAValueReachingRhs() = findLibCall
|
result = cdllCall.getReturn()
)
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-285/PamAuthorization.expected b/python/ql/test/experimental/query-tests/Security/CWE-285/PamAuthorization.expected
index cde7271874a..1b6c23291be 100644
--- a/python/ql/test/experimental/query-tests/Security/CWE-285/PamAuthorization.expected
+++ b/python/ql/test/experimental/query-tests/Security/CWE-285/PamAuthorization.expected
@@ -1 +1 @@
-| pam_test.py:44:18:44:44 | ControlFlowNode for pam_authenticate() | This PAM authentication call may be lead to an authorization bypass. |
+| pam_test.py:48:18:48:44 | ControlFlowNode for pam_authenticate() | This PAM authentication call may be lead to an authorization bypass. |
diff --git a/python/ql/test/experimental/query-tests/Security/CWE-285/pam_test.py b/python/ql/test/experimental/query-tests/Security/CWE-285/pam_test.py
index 60408ade722..966e13cb991 100644
--- a/python/ql/test/experimental/query-tests/Security/CWE-285/pam_test.py
+++ b/python/ql/test/experimental/query-tests/Security/CWE-285/pam_test.py
@@ -18,9 +18,13 @@ class PamResponse(Structure):
class PamConv(Structure):
pass
-
+# this is normal way to do things
libpam = CDLL(find_library("pam"))
+# but we also handle assignment to temp variable
+temp = find_library("pam")
+libpam = CDLL(temp)
+
pam_start = libpam.pam_start
pam_start.restype = c_int
pam_start.argtypes = [c_char_p, c_char_p, POINTER(PamConv), POINTER(PamHandle)]
From 40da7a10553c76ad98bb0f6c6a5af652fc3f4e5e Mon Sep 17 00:00:00 2001
From: Geoffrey White <40627776+geoffw0@users.noreply.github.com>
Date: Thu, 21 Apr 2022 16:55:50 +0100
Subject: [PATCH 098/171] C++: Add a test of NoCheckBeforeUnsafePutUser.ql.
---
.../NoCheckBeforeUnsafePutUser.expected | 1 +
.../NoCheckBeforeUnsafePutUser.qlref | 1 +
.../NoCheckBeforeUnsafePutUser/test.cpp | 82 +++++++++++++++++++
3 files changed, 84 insertions(+)
create mode 100644 cpp/ql/test/experimental/query-tests/Security/CWE/CWE-020/NoCheckBeforeUnsafePutUser/NoCheckBeforeUnsafePutUser.expected
create mode 100644 cpp/ql/test/experimental/query-tests/Security/CWE/CWE-020/NoCheckBeforeUnsafePutUser/NoCheckBeforeUnsafePutUser.qlref
create mode 100644 cpp/ql/test/experimental/query-tests/Security/CWE/CWE-020/NoCheckBeforeUnsafePutUser/test.cpp
diff --git a/cpp/ql/test/experimental/query-tests/Security/CWE/CWE-020/NoCheckBeforeUnsafePutUser/NoCheckBeforeUnsafePutUser.expected b/cpp/ql/test/experimental/query-tests/Security/CWE/CWE-020/NoCheckBeforeUnsafePutUser/NoCheckBeforeUnsafePutUser.expected
new file mode 100644
index 00000000000..ffb7941f1cd
--- /dev/null
+++ b/cpp/ql/test/experimental/query-tests/Security/CWE/CWE-020/NoCheckBeforeUnsafePutUser/NoCheckBeforeUnsafePutUser.expected
@@ -0,0 +1 @@
+| test.cpp:14:16:14:16 | p | unsafe_put_user write user-mode pointer $@ without check. | test.cpp:14:16:14:16 | p | p |
diff --git a/cpp/ql/test/experimental/query-tests/Security/CWE/CWE-020/NoCheckBeforeUnsafePutUser/NoCheckBeforeUnsafePutUser.qlref b/cpp/ql/test/experimental/query-tests/Security/CWE/CWE-020/NoCheckBeforeUnsafePutUser/NoCheckBeforeUnsafePutUser.qlref
new file mode 100644
index 00000000000..a4543b332dd
--- /dev/null
+++ b/cpp/ql/test/experimental/query-tests/Security/CWE/CWE-020/NoCheckBeforeUnsafePutUser/NoCheckBeforeUnsafePutUser.qlref
@@ -0,0 +1 @@
+experimental/Security/CWE/CWE-020/NoCheckBeforeUnsafePutUser.ql
diff --git a/cpp/ql/test/experimental/query-tests/Security/CWE/CWE-020/NoCheckBeforeUnsafePutUser/test.cpp b/cpp/ql/test/experimental/query-tests/Security/CWE/CWE-020/NoCheckBeforeUnsafePutUser/test.cpp
new file mode 100644
index 00000000000..755a73864c7
--- /dev/null
+++ b/cpp/ql/test/experimental/query-tests/Security/CWE/CWE-020/NoCheckBeforeUnsafePutUser/test.cpp
@@ -0,0 +1,82 @@
+
+typedef unsigned long size_t;
+
+void SYSC_SOMESYSTEMCALL(void *param);
+
+bool user_access_begin_impl(const void *where, size_t sz);
+void user_access_end_impl();
+#define user_access_begin(where, sz) user_access_begin_impl(where, sz)
+#define user_access_end() user_access_end_impl()
+
+void unsafe_put_user_impl(int what, const void *where, size_t sz);
+#define unsafe_put_user(what, where) unsafe_put_user_impl( (what), (where), sizeof(*(where)) )
+
+void test1(int p)
+{
+ SYSC_SOMESYSTEMCALL(&p);
+
+ unsafe_put_user(123, &p); // BAD
+}
+
+void test2(int p)
+{
+ SYSC_SOMESYSTEMCALL(&p);
+
+ if (user_access_begin(&p, sizeof(p)))
+ {
+ unsafe_put_user(123, &p); // GOOD
+
+ user_access_end();
+ }
+}
+
+void test3()
+{
+ int v;
+
+ SYSC_SOMESYSTEMCALL(&v);
+
+ unsafe_put_user(123, &v); // BAD [NOT DETECTED]
+}
+
+void test4()
+{
+ int v;
+
+ SYSC_SOMESYSTEMCALL(&v);
+
+ if (user_access_begin(&v, sizeof(v)))
+ {
+ unsafe_put_user(123, &v); // GOOD
+
+ user_access_end();
+ }
+}
+
+struct data
+{
+ int x;
+};
+
+void test5()
+{
+ data myData;
+
+ SYSC_SOMESYSTEMCALL(&myData);
+
+ unsafe_put_user(123, &(myData.x)); // BAD [NOT DETECTED]
+}
+
+void test6()
+{
+ data myData;
+
+ SYSC_SOMESYSTEMCALL(&myData);
+
+ if (user_access_begin(&myData, sizeof(myData)))
+ {
+ unsafe_put_user(123, &(myData.x)); // GOOD
+
+ user_access_end();
+ }
+}
From 7359ffaa2ea1be1aa45bb7957fdcacf9b8eb73c5 Mon Sep 17 00:00:00 2001
From: Arthur Baars
Date: Thu, 28 Apr 2022 11:36:01 +0200
Subject: [PATCH 099/171] Ruby: add tree-sitter test case
---
.../library-tests/ast/TreeSitter.expected | 5846 +++++++++++++++++
ruby/ql/test/library-tests/ast/TreeSitter.ql | 31 +
2 files changed, 5877 insertions(+)
create mode 100644 ruby/ql/test/library-tests/ast/TreeSitter.expected
create mode 100644 ruby/ql/test/library-tests/ast/TreeSitter.ql
diff --git a/ruby/ql/test/library-tests/ast/TreeSitter.expected b/ruby/ql/test/library-tests/ast/TreeSitter.expected
new file mode 100644
index 00000000000..ecf00e54895
--- /dev/null
+++ b/ruby/ql/test/library-tests/ast/TreeSitter.expected
@@ -0,0 +1,5846 @@
+calls/calls.rb:
+# 1| [Program] Program
+# 2| 0: [Call] Call
+# 2| 0: [Identifier] foo
+# 2| 1: [ArgumentList] ArgumentList
+# 2| 0: [ReservedWord] (
+# 2| 1: [ReservedWord] )
+# 5| 1: [Call] Call
+# 5| 0: [ScopeResolution] ScopeResolution
+# 5| 0: [Constant] Foo
+# 5| 1: [ReservedWord] ::
+# 5| 2: [Identifier] bar
+# 5| 1: [ArgumentList] ArgumentList
+# 5| 0: [ReservedWord] (
+# 5| 1: [ReservedWord] )
+# 8| 2: [Call] Call
+# 8| 0: [ScopeResolution] ScopeResolution
+# 8| 0: [ReservedWord] ::
+# 8| 1: [Identifier] bar
+# 8| 1: [ArgumentList] ArgumentList
+# 8| 0: [ReservedWord] (
+# 8| 1: [ReservedWord] )
+# 11| 3: [Call] Call
+# 11| 0: [Integer] 123
+# 11| 1: [ReservedWord] .
+# 11| 2: [Identifier] bar
+# 14| 4: [Call] Call
+# 14| 0: [Identifier] foo
+# 14| 1: [ArgumentList] ArgumentList
+# 14| 0: [Integer] 0
+# 14| 1: [ReservedWord] ,
+# 14| 2: [Integer] 1
+# 14| 3: [ReservedWord] ,
+# 14| 4: [Integer] 2
+# 17| 5: [Call] Call
+# 17| 0: [Identifier] foo
+# 17| 1: [Block] Block
+# 17| 0: [ReservedWord] {
+# 17| 1: [BlockParameters] BlockParameters
+# 17| 0: [ReservedWord] |
+# 17| 1: [Identifier] x
+# 17| 2: [ReservedWord] |
+# 17| 2: [Binary] Binary
+# 17| 0: [Identifier] x
+# 17| 1: [ReservedWord] +
+# 17| 2: [Integer] 1
+# 17| 3: [ReservedWord] }
+# 20| 6: [Call] Call
+# 20| 0: [Identifier] foo
+# 20| 1: [DoBlock] DoBlock
+# 20| 0: [ReservedWord] do
+# 20| 1: [BlockParameters] BlockParameters
+# 20| 0: [ReservedWord] |
+# 20| 1: [Identifier] x
+# 20| 2: [ReservedWord] |
+# 21| 2: [Binary] Binary
+# 21| 0: [Identifier] x
+# 21| 1: [ReservedWord] +
+# 21| 2: [Integer] 1
+# 22| 3: [ReservedWord] end
+# 25| 7: [Call] Call
+# 25| 0: [Integer] 123
+# 25| 1: [ReservedWord] .
+# 25| 2: [Identifier] bar
+# 25| 3: [ArgumentList] ArgumentList
+# 25| 0: [ReservedWord] (
+# 25| 1: [String] String
+# 25| 0: [ReservedWord] '
+# 25| 1: [StringContent] foo
+# 25| 2: [ReservedWord] '
+# 25| 2: [ReservedWord] )
+# 25| 4: [DoBlock] DoBlock
+# 25| 0: [ReservedWord] do
+# 25| 1: [BlockParameters] BlockParameters
+# 25| 0: [ReservedWord] |
+# 25| 1: [Identifier] x
+# 25| 2: [ReservedWord] |
+# 26| 2: [Binary] Binary
+# 26| 0: [Identifier] x
+# 26| 1: [ReservedWord] +
+# 26| 2: [Integer] 1
+# 27| 3: [ReservedWord] end
+# 30| 8: [Method] Method
+# 30| 0: [ReservedWord] def
+# 30| 1: [Identifier] method_that_yields
+# 31| 2: [Yield] Yield
+# 31| 0: [ReservedWord] yield
+# 32| 3: [ReservedWord] end
+# 35| 9: [Method] Method
+# 35| 0: [ReservedWord] def
+# 35| 1: [Identifier] another_method_that_yields
+# 36| 2: [Yield] Yield
+# 36| 0: [ReservedWord] yield
+# 36| 1: [ArgumentList] ArgumentList
+# 36| 0: [Integer] 100
+# 36| 1: [ReservedWord] ,
+# 36| 2: [Integer] 200
+# 37| 3: [ReservedWord] end
+# 46| 10: [Identifier] foo
+# 47| 11: [ScopeResolution] ScopeResolution
+# 47| 0: [Constant] X
+# 47| 1: [ReservedWord] ::
+# 47| 2: [Identifier] foo
+# 50| 12: [ParenthesizedStatements] ParenthesizedStatements
+# 50| 0: [ReservedWord] (
+# 50| 1: [Identifier] foo
+# 50| 2: [ReservedWord] )
+# 51| 13: [ParenthesizedStatements] ParenthesizedStatements
+# 51| 0: [ReservedWord] (
+# 51| 1: [ScopeResolution] ScopeResolution
+# 51| 0: [Constant] X
+# 51| 1: [ReservedWord] ::
+# 51| 2: [Identifier] foo
+# 51| 2: [ReservedWord] )
+# 54| 14: [Call] Call
+# 54| 0: [Identifier] some_func
+# 54| 1: [ArgumentList] ArgumentList
+# 54| 0: [ReservedWord] (
+# 54| 1: [Identifier] foo
+# 54| 2: [ReservedWord] )
+# 55| 15: [Call] Call
+# 55| 0: [Identifier] some_func
+# 55| 1: [ArgumentList] ArgumentList
+# 55| 0: [ReservedWord] (
+# 55| 1: [ScopeResolution] ScopeResolution
+# 55| 0: [Constant] X
+# 55| 1: [ReservedWord] ::
+# 55| 2: [Identifier] foo
+# 55| 2: [ReservedWord] )
+# 58| 16: [Array] Array
+# 58| 0: [ReservedWord] [
+# 58| 1: [Identifier] foo
+# 58| 2: [ReservedWord] ]
+# 59| 17: [Array] Array
+# 59| 0: [ReservedWord] [
+# 59| 1: [ScopeResolution] ScopeResolution
+# 59| 0: [Constant] X
+# 59| 1: [ReservedWord] ::
+# 59| 2: [Identifier] foo
+# 59| 2: [ReservedWord] ]
+# 62| 18: [Assignment] Assignment
+# 62| 0: [Identifier] var1
+# 62| 1: [ReservedWord] =
+# 62| 2: [Identifier] foo
+# 63| 19: [Assignment] Assignment
+# 63| 0: [Identifier] var1
+# 63| 1: [ReservedWord] =
+# 63| 2: [ScopeResolution] ScopeResolution
+# 63| 0: [Constant] X
+# 63| 1: [ReservedWord] ::
+# 63| 2: [Identifier] foo
+# 66| 20: [OperatorAssignment] OperatorAssignment
+# 66| 0: [Identifier] var1
+# 66| 1: [ReservedWord] +=
+# 66| 2: [Identifier] bar
+# 67| 21: [OperatorAssignment] OperatorAssignment
+# 67| 0: [Identifier] var1
+# 67| 1: [ReservedWord] +=
+# 67| 2: [ScopeResolution] ScopeResolution
+# 67| 0: [Constant] X
+# 67| 1: [ReservedWord] ::
+# 67| 2: [Identifier] bar
+# 70| 22: [Assignment] Assignment
+# 70| 0: [Identifier] var1
+# 70| 1: [ReservedWord] =
+# 70| 2: [RightAssignmentList] RightAssignmentList
+# 70| 0: [Identifier] foo
+# 70| 1: [ReservedWord] ,
+# 70| 2: [ScopeResolution] ScopeResolution
+# 70| 0: [Constant] X
+# 70| 1: [ReservedWord] ::
+# 70| 2: [Identifier] bar
+# 73| 23: [Begin] Begin
+# 73| 0: [ReservedWord] begin
+# 74| 1: [Identifier] foo
+# 75| 2: [ScopeResolution] ScopeResolution
+# 75| 0: [Constant] X
+# 75| 1: [ReservedWord] ::
+# 75| 2: [Identifier] foo
+# 76| 3: [ReservedWord] end
+# 79| 24: [BeginBlock] BeginBlock
+# 79| 0: [ReservedWord] BEGIN
+# 79| 1: [ReservedWord] {
+# 79| 2: [Identifier] foo
+# 79| 3: [ReservedWord] ;
+# 79| 4: [ScopeResolution] ScopeResolution
+# 79| 0: [Constant] X
+# 79| 1: [ReservedWord] ::
+# 79| 2: [Identifier] bar
+# 79| 5: [ReservedWord] }
+# 82| 25: [EndBlock] EndBlock
+# 82| 0: [ReservedWord] END
+# 82| 1: [ReservedWord] {
+# 82| 2: [Identifier] foo
+# 82| 3: [ReservedWord] ;
+# 82| 4: [ScopeResolution] ScopeResolution
+# 82| 0: [Constant] X
+# 82| 1: [ReservedWord] ::
+# 82| 2: [Identifier] bar
+# 82| 5: [ReservedWord] }
+# 85| 26: [Binary] Binary
+# 85| 0: [Identifier] foo
+# 85| 1: [ReservedWord] +
+# 85| 2: [ScopeResolution] ScopeResolution
+# 85| 0: [Constant] X
+# 85| 1: [ReservedWord] ::
+# 85| 2: [Identifier] bar
+# 88| 27: [Unary] Unary
+# 88| 0: [ReservedWord] !
+# 88| 1: [Identifier] foo
+# 89| 28: [Unary] Unary
+# 89| 0: [ReservedWord] ~
+# 89| 1: [ScopeResolution] ScopeResolution
+# 89| 0: [Constant] X
+# 89| 1: [ReservedWord] ::
+# 89| 2: [Identifier] bar
+# 92| 29: [Call] Call
+# 92| 0: [Identifier] foo
+# 92| 1: [ArgumentList] ArgumentList
+# 92| 0: [ReservedWord] (
+# 92| 1: [ReservedWord] )
+# 92| 2: [Block] Block
+# 92| 0: [ReservedWord] {
+# 92| 1: [Identifier] bar
+# 92| 2: [ReservedWord] ;
+# 92| 3: [ScopeResolution] ScopeResolution
+# 92| 0: [Constant] X
+# 92| 1: [ReservedWord] ::
+# 92| 2: [Identifier] baz
+# 92| 4: [ReservedWord] }
+# 95| 30: [Call] Call
+# 95| 0: [Identifier] foo
+# 95| 1: [ArgumentList] ArgumentList
+# 95| 0: [ReservedWord] (
+# 95| 1: [ReservedWord] )
+# 95| 2: [DoBlock] DoBlock
+# 95| 0: [ReservedWord] do
+# 96| 1: [Identifier] bar
+# 97| 2: [ScopeResolution] ScopeResolution
+# 97| 0: [Constant] X
+# 97| 1: [ReservedWord] ::
+# 97| 2: [Identifier] baz
+# 98| 3: [ReservedWord] end
+# 101| 31: [Call] Call
+# 101| 0: [Identifier] foo
+# 101| 1: [ReservedWord] .
+# 101| 2: [Identifier] bar
+# 101| 3: [ArgumentList] ArgumentList
+# 101| 0: [ReservedWord] (
+# 101| 1: [ReservedWord] )
+# 102| 32: [Call] Call
+# 102| 0: [Identifier] bar
+# 102| 1: [ReservedWord] .
+# 102| 2: [Identifier] baz
+# 102| 3: [ArgumentList] ArgumentList
+# 102| 0: [ReservedWord] (
+# 102| 1: [ReservedWord] )
+# 106| 33: [Case] Case
+# 106| 0: [ReservedWord] case
+# 106| 1: [Identifier] foo
+# 107| 2: [When] When
+# 107| 0: [ReservedWord] when
+# 107| 1: [Pattern] Pattern
+# 107| 0: [Identifier] bar
+# 107| 2: [Then] Then
+# 108| 0: [Identifier] baz
+# 109| 3: [ReservedWord] end
+# 110| 34: [Case] Case
+# 110| 0: [ReservedWord] case
+# 110| 1: [ScopeResolution] ScopeResolution
+# 110| 0: [Constant] X
+# 110| 1: [ReservedWord] ::
+# 110| 2: [Identifier] foo
+# 111| 2: [When] When
+# 111| 0: [ReservedWord] when
+# 111| 1: [Pattern] Pattern
+# 111| 0: [ScopeResolution] ScopeResolution
+# 111| 0: [Constant] X
+# 111| 1: [ReservedWord] ::
+# 111| 2: [Identifier] bar
+# 111| 2: [Then] Then
+# 112| 0: [ScopeResolution] ScopeResolution
+# 112| 0: [Constant] X
+# 112| 1: [ReservedWord] ::
+# 112| 2: [Identifier] baz
+# 113| 3: [ReservedWord] end
+# 116| 35: [Class] Class
+# 116| 0: [ReservedWord] class
+# 116| 1: [Constant] MyClass
+# 117| 2: [Identifier] foo
+# 118| 3: [ScopeResolution] ScopeResolution
+# 118| 0: [Constant] X
+# 118| 1: [ReservedWord] ::
+# 118| 2: [Identifier] bar
+# 119| 4: [ReservedWord] end
+# 122| 36: [Class] Class
+# 122| 0: [ReservedWord] class
+# 122| 1: [Constant] MyClass
+# 122| 2: [Superclass] Superclass
+# 122| 0: [ReservedWord] <
+# 122| 1: [Identifier] foo
+# 123| 3: [ReservedWord] end
+# 124| 37: [Class] Class
+# 124| 0: [ReservedWord] class
+# 124| 1: [Constant] MyClass2
+# 124| 2: [Superclass] Superclass
+# 124| 0: [ReservedWord] <
+# 124| 1: [ScopeResolution] ScopeResolution
+# 124| 0: [Constant] X
+# 124| 1: [ReservedWord] ::
+# 124| 2: [Identifier] foo
+# 125| 3: [ReservedWord] end
+# 128| 38: [SingletonClass] SingletonClass
+# 128| 0: [ReservedWord] class
+# 128| 1: [ReservedWord] <<
+# 128| 2: [Identifier] foo
+# 129| 3: [Identifier] bar
+# 130| 4: [ReservedWord] end
+# 131| 39: [SingletonClass] SingletonClass
+# 131| 0: [ReservedWord] class
+# 131| 1: [ReservedWord] <<
+# 131| 2: [ScopeResolution] ScopeResolution
+# 131| 0: [Constant] X
+# 131| 1: [ReservedWord] ::
+# 131| 2: [Identifier] foo
+# 132| 3: [ScopeResolution] ScopeResolution
+# 132| 0: [Constant] X
+# 132| 1: [ReservedWord] ::
+# 132| 2: [Identifier] bar
+# 133| 4: [ReservedWord] end
+# 136| 40: [Method] Method
+# 136| 0: [ReservedWord] def
+# 136| 1: [Identifier] some_method
+# 137| 2: [Identifier] foo
+# 138| 3: [ScopeResolution] ScopeResolution
+# 138| 0: [Constant] X
+# 138| 1: [ReservedWord] ::
+# 138| 2: [Identifier] bar
+# 139| 4: [ReservedWord] end
+# 142| 41: [SingletonMethod] SingletonMethod
+# 142| 0: [ReservedWord] def
+# 142| 1: [Identifier] foo
+# 142| 2: [ReservedWord] .
+# 142| 3: [Identifier] some_method
+# 143| 4: [Identifier] bar
+# 144| 5: [ScopeResolution] ScopeResolution
+# 144| 0: [Constant] X
+# 144| 1: [ReservedWord] ::
+# 144| 2: [Identifier] baz
+# 145| 6: [ReservedWord] end
+# 148| 42: [Method] Method
+# 148| 0: [ReservedWord] def
+# 148| 1: [Identifier] method_with_keyword_param
+# 148| 2: [MethodParameters] MethodParameters
+# 148| 0: [ReservedWord] (
+# 148| 1: [KeywordParameter] KeywordParameter
+# 148| 0: [Identifier] keyword
+# 148| 1: [ReservedWord] :
+# 148| 2: [Identifier] foo
+# 148| 2: [ReservedWord] )
+# 149| 3: [ReservedWord] end
+# 150| 43: [Method] Method
+# 150| 0: [ReservedWord] def
+# 150| 1: [Identifier] method_with_keyword_param2
+# 150| 2: [MethodParameters] MethodParameters
+# 150| 0: [ReservedWord] (
+# 150| 1: [KeywordParameter] KeywordParameter
+# 150| 0: [Identifier] keyword
+# 150| 1: [ReservedWord] :
+# 150| 2: [ScopeResolution] ScopeResolution
+# 150| 0: [Constant] X
+# 150| 1: [ReservedWord] ::
+# 150| 2: [Identifier] foo
+# 150| 2: [ReservedWord] )
+# 151| 3: [ReservedWord] end
+# 154| 44: [Method] Method
+# 154| 0: [ReservedWord] def
+# 154| 1: [Identifier] method_with_optional_param
+# 154| 2: [MethodParameters] MethodParameters
+# 154| 0: [ReservedWord] (
+# 154| 1: [OptionalParameter] OptionalParameter
+# 154| 0: [Identifier] param
+# 154| 1: [ReservedWord] =
+# 154| 2: [Identifier] foo
+# 154| 2: [ReservedWord] )
+# 155| 3: [ReservedWord] end
+# 156| 45: [Method] Method
+# 156| 0: [ReservedWord] def
+# 156| 1: [Identifier] method_with_optional_param2
+# 156| 2: [MethodParameters] MethodParameters
+# 156| 0: [ReservedWord] (
+# 156| 1: [OptionalParameter] OptionalParameter
+# 156| 0: [Identifier] param
+# 156| 1: [ReservedWord] =
+# 156| 2: [ScopeResolution] ScopeResolution
+# 156| 0: [Constant] X
+# 156| 1: [ReservedWord] ::
+# 156| 2: [Identifier] foo
+# 156| 2: [ReservedWord] )
+# 157| 3: [ReservedWord] end
+# 160| 46: [Module] Module
+# 160| 0: [ReservedWord] module
+# 160| 1: [Constant] SomeModule
+# 161| 2: [Identifier] foo
+# 162| 3: [ScopeResolution] ScopeResolution
+# 162| 0: [Constant] X
+# 162| 1: [ReservedWord] ::
+# 162| 2: [Identifier] bar
+# 163| 4: [ReservedWord] end
+# 166| 47: [Conditional] Conditional
+# 166| 0: [Identifier] foo
+# 166| 1: [ReservedWord] ?
+# 166| 2: [Identifier] bar
+# 166| 3: [ReservedWord] :
+# 166| 4: [Identifier] baz
+# 167| 48: [Conditional] Conditional
+# 167| 0: [ScopeResolution] ScopeResolution
+# 167| 0: [Constant] X
+# 167| 1: [ReservedWord] ::
+# 167| 2: [Identifier] foo
+# 167| 1: [ReservedWord] ?
+# 167| 2: [ScopeResolution] ScopeResolution
+# 167| 0: [Constant] X
+# 167| 1: [ReservedWord] ::
+# 167| 2: [Identifier] bar
+# 167| 3: [ReservedWord] :
+# 167| 4: [ScopeResolution] ScopeResolution
+# 167| 0: [Constant] X
+# 167| 1: [ReservedWord] ::
+# 167| 2: [Identifier] baz
+# 170| 49: [If] If
+# 170| 0: [ReservedWord] if
+# 170| 1: [Identifier] foo
+# 170| 2: [Then] Then
+# 171| 0: [Identifier] wibble
+# 172| 3: [Elsif] Elsif
+# 172| 0: [ReservedWord] elsif
+# 172| 1: [Identifier] bar
+# 172| 2: [Then] Then
+# 173| 0: [Identifier] wobble
+# 174| 3: [Else] Else
+# 174| 0: [ReservedWord] else
+# 175| 1: [Identifier] wabble
+# 176| 4: [ReservedWord] end
+# 177| 50: [If] If
+# 177| 0: [ReservedWord] if
+# 177| 1: [ScopeResolution] ScopeResolution
+# 177| 0: [Constant] X
+# 177| 1: [ReservedWord] ::
+# 177| 2: [Identifier] foo
+# 177| 2: [Then] Then
+# 178| 0: [ScopeResolution] ScopeResolution
+# 178| 0: [Constant] X
+# 178| 1: [ReservedWord] ::
+# 178| 2: [Identifier] wibble
+# 179| 3: [Elsif] Elsif
+# 179| 0: [ReservedWord] elsif
+# 179| 1: [ScopeResolution] ScopeResolution
+# 179| 0: [Constant] X
+# 179| 1: [ReservedWord] ::
+# 179| 2: [Identifier] bar
+# 179| 2: [Then] Then
+# 180| 0: [ScopeResolution] ScopeResolution
+# 180| 0: [Constant] X
+# 180| 1: [ReservedWord] ::
+# 180| 2: [Identifier] wobble
+# 181| 3: [Else] Else
+# 181| 0: [ReservedWord] else
+# 182| 1: [ScopeResolution] ScopeResolution
+# 182| 0: [Constant] X
+# 182| 1: [ReservedWord] ::
+# 182| 2: [Identifier] wabble
+# 183| 4: [ReservedWord] end
+# 186| 51: [IfModifier] IfModifier
+# 186| 0: [Identifier] bar
+# 186| 1: [ReservedWord] if
+# 186| 2: [Identifier] foo
+# 187| 52: [IfModifier] IfModifier
+# 187| 0: [ScopeResolution] ScopeResolution
+# 187| 0: [Constant] X
+# 187| 1: [ReservedWord] ::
+# 187| 2: [Identifier] bar
+# 187| 1: [ReservedWord] if
+# 187| 2: [ScopeResolution] ScopeResolution
+# 187| 0: [Constant] X
+# 187| 1: [ReservedWord] ::
+# 187| 2: [Identifier] foo
+# 190| 53: [Unless] Unless
+# 190| 0: [ReservedWord] unless
+# 190| 1: [Identifier] foo
+# 190| 2: [Then] Then
+# 191| 0: [Identifier] bar
+# 192| 3: [ReservedWord] end
+# 193| 54: [Unless] Unless
+# 193| 0: [ReservedWord] unless
+# 193| 1: [ScopeResolution] ScopeResolution
+# 193| 0: [Constant] X
+# 193| 1: [ReservedWord] ::
+# 193| 2: [Identifier] foo
+# 193| 2: [Then] Then
+# 194| 0: [ScopeResolution] ScopeResolution
+# 194| 0: [Constant] X
+# 194| 1: [ReservedWord] ::
+# 194| 2: [Identifier] bar
+# 195| 3: [ReservedWord] end
+# 198| 55: [UnlessModifier] UnlessModifier
+# 198| 0: [Identifier] bar
+# 198| 1: [ReservedWord] unless
+# 198| 2: [Identifier] foo
+# 199| 56: [UnlessModifier] UnlessModifier
+# 199| 0: [ScopeResolution] ScopeResolution
+# 199| 0: [Constant] X
+# 199| 1: [ReservedWord] ::
+# 199| 2: [Identifier] bar
+# 199| 1: [ReservedWord] unless
+# 199| 2: [ScopeResolution] ScopeResolution
+# 199| 0: [Constant] X
+# 199| 1: [ReservedWord] ::
+# 199| 2: [Identifier] foo
+# 202| 57: [While] While
+# 202| 0: [ReservedWord] while
+# 202| 1: [Identifier] foo
+# 202| 2: [Do] Do
+# 202| 0: [ReservedWord] do
+# 203| 1: [Identifier] bar
+# 204| 2: [ReservedWord] end
+# 205| 58: [While] While
+# 205| 0: [ReservedWord] while
+# 205| 1: [ScopeResolution] ScopeResolution
+# 205| 0: [Constant] X
+# 205| 1: [ReservedWord] ::
+# 205| 2: [Identifier] foo
+# 205| 2: [Do] Do
+# 205| 0: [ReservedWord] do
+# 206| 1: [ScopeResolution] ScopeResolution
+# 206| 0: [Constant] X
+# 206| 1: [ReservedWord] ::
+# 206| 2: [Identifier] bar
+# 207| 2: [ReservedWord] end
+# 210| 59: [WhileModifier] WhileModifier
+# 210| 0: [Identifier] bar
+# 210| 1: [ReservedWord] while
+# 210| 2: [Identifier] foo
+# 211| 60: [WhileModifier] WhileModifier
+# 211| 0: [ScopeResolution] ScopeResolution
+# 211| 0: [Constant] X
+# 211| 1: [ReservedWord] ::
+# 211| 2: [Identifier] bar
+# 211| 1: [ReservedWord] while
+# 211| 2: [ScopeResolution] ScopeResolution
+# 211| 0: [Constant] X
+# 211| 1: [ReservedWord] ::
+# 211| 2: [Identifier] foo
+# 214| 61: [Until] Until
+# 214| 0: [ReservedWord] until
+# 214| 1: [Identifier] foo
+# 214| 2: [Do] Do
+# 214| 0: [ReservedWord] do
+# 215| 1: [Identifier] bar
+# 216| 2: [ReservedWord] end
+# 217| 62: [Until] Until
+# 217| 0: [ReservedWord] until
+# 217| 1: [ScopeResolution] ScopeResolution
+# 217| 0: [Constant] X
+# 217| 1: [ReservedWord] ::
+# 217| 2: [Identifier] foo
+# 217| 2: [Do] Do
+# 217| 0: [ReservedWord] do
+# 218| 1: [ScopeResolution] ScopeResolution
+# 218| 0: [Constant] X
+# 218| 1: [ReservedWord] ::
+# 218| 2: [Identifier] bar
+# 219| 2: [ReservedWord] end
+# 222| 63: [UntilModifier] UntilModifier
+# 222| 0: [Identifier] bar
+# 222| 1: [ReservedWord] until
+# 222| 2: [Identifier] foo
+# 223| 64: [UntilModifier] UntilModifier
+# 223| 0: [ScopeResolution] ScopeResolution
+# 223| 0: [Constant] X
+# 223| 1: [ReservedWord] ::
+# 223| 2: [Identifier] bar
+# 223| 1: [ReservedWord] until
+# 223| 2: [ScopeResolution] ScopeResolution
+# 223| 0: [Constant] X
+# 223| 1: [ReservedWord] ::
+# 223| 2: [Identifier] foo
+# 226| 65: [For] For
+# 226| 0: [ReservedWord] for
+# 226| 1: [Identifier] x
+# 226| 2: [In] In
+# 226| 0: [ReservedWord] in
+# 226| 1: [Identifier] bar
+# 226| 3: [Do] Do
+# 227| 0: [Identifier] baz
+# 228| 1: [ReservedWord] end
+# 229| 66: [For] For
+# 229| 0: [ReservedWord] for
+# 229| 1: [Identifier] x
+# 229| 2: [In] In
+# 229| 0: [ReservedWord] in
+# 229| 1: [ScopeResolution] ScopeResolution
+# 229| 0: [Constant] X
+# 229| 1: [ReservedWord] ::
+# 229| 2: [Identifier] bar
+# 229| 3: [Do] Do
+# 230| 0: [ScopeResolution] ScopeResolution
+# 230| 0: [Constant] X
+# 230| 1: [ReservedWord] ::
+# 230| 2: [Identifier] baz
+# 231| 1: [ReservedWord] end
+# 234| 67: [ElementReference] ElementReference
+# 234| 0: [Identifier] foo
+# 234| 1: [ReservedWord] [
+# 234| 2: [Identifier] bar
+# 234| 3: [ReservedWord] ]
+# 235| 68: [ElementReference] ElementReference
+# 235| 0: [ScopeResolution] ScopeResolution
+# 235| 0: [Constant] X
+# 235| 1: [ReservedWord] ::
+# 235| 2: [Identifier] foo
+# 235| 1: [ReservedWord] [
+# 235| 2: [ScopeResolution] ScopeResolution
+# 235| 0: [Constant] X
+# 235| 1: [ReservedWord] ::
+# 235| 2: [Identifier] bar
+# 235| 3: [ReservedWord] ]
+# 238| 69: [String] String
+# 238| 0: [ReservedWord] "
+# 238| 1: [StringContent] foo-
+# 238| 2: [Interpolation] Interpolation
+# 238| 0: [ReservedWord] #{
+# 238| 1: [Identifier] bar
+# 238| 2: [ReservedWord] }
+# 238| 3: [StringContent] -
+# 238| 4: [Interpolation] Interpolation
+# 238| 0: [ReservedWord] #{
+# 238| 1: [ScopeResolution] ScopeResolution
+# 238| 0: [Constant] X
+# 238| 1: [ReservedWord] ::
+# 238| 2: [Identifier] baz
+# 238| 2: [ReservedWord] }
+# 238| 5: [ReservedWord] "
+# 241| 70: [ScopeResolution] ScopeResolution
+# 241| 0: [Identifier] foo
+# 241| 1: [ReservedWord] ::
+# 241| 2: [Constant] Bar
+# 242| 71: [ScopeResolution] ScopeResolution
+# 242| 0: [ScopeResolution] ScopeResolution
+# 242| 0: [Constant] X
+# 242| 1: [ReservedWord] ::
+# 242| 2: [Identifier] foo
+# 242| 1: [ReservedWord] ::
+# 242| 2: [Constant] Bar
+# 245| 72: [Range] Range
+# 245| 0: [Identifier] foo
+# 245| 1: [ReservedWord] ..
+# 245| 2: [Identifier] bar
+# 246| 73: [Range] Range
+# 246| 0: [ScopeResolution] ScopeResolution
+# 246| 0: [Constant] X
+# 246| 1: [ReservedWord] ::
+# 246| 2: [Identifier] foo
+# 246| 1: [ReservedWord] ..
+# 246| 2: [ScopeResolution] ScopeResolution
+# 246| 0: [Constant] X
+# 246| 1: [ReservedWord] ::
+# 246| 2: [Identifier] bar
+# 249| 74: [Hash] Hash
+# 249| 0: [ReservedWord] {
+# 249| 1: [Pair] Pair
+# 249| 0: [Identifier] foo
+# 249| 1: [ReservedWord] =>
+# 249| 2: [Identifier] bar
+# 249| 2: [ReservedWord] ,
+# 249| 3: [Pair] Pair
+# 249| 0: [ScopeResolution] ScopeResolution
+# 249| 0: [Constant] X
+# 249| 1: [ReservedWord] ::
+# 249| 2: [Identifier] foo
+# 249| 1: [ReservedWord] =>
+# 249| 2: [ScopeResolution] ScopeResolution
+# 249| 0: [Constant] X
+# 249| 1: [ReservedWord] ::
+# 249| 2: [Identifier] bar
+# 249| 4: [ReservedWord] }
+# 252| 75: [Begin] Begin
+# 252| 0: [ReservedWord] begin
+# 253| 1: [Rescue] Rescue
+# 253| 0: [ReservedWord] rescue
+# 253| 1: [Exceptions] Exceptions
+# 253| 0: [Identifier] foo
+# 254| 2: [Ensure] Ensure
+# 254| 0: [ReservedWord] ensure
+# 254| 1: [Identifier] bar
+# 255| 3: [ReservedWord] end
+# 256| 76: [Begin] Begin
+# 256| 0: [ReservedWord] begin
+# 257| 1: [Rescue] Rescue
+# 257| 0: [ReservedWord] rescue
+# 257| 1: [Exceptions] Exceptions
+# 257| 0: [ScopeResolution] ScopeResolution
+# 257| 0: [Constant] X
+# 257| 1: [ReservedWord] ::
+# 257| 2: [Identifier] foo
+# 258| 2: [Ensure] Ensure
+# 258| 0: [ReservedWord] ensure
+# 258| 1: [ScopeResolution] ScopeResolution
+# 258| 0: [Constant] X
+# 258| 1: [ReservedWord] ::
+# 258| 2: [Identifier] bar
+# 259| 3: [ReservedWord] end
+# 262| 77: [RescueModifier] RescueModifier
+# 262| 0: [Identifier] foo
+# 262| 1: [ReservedWord] rescue
+# 262| 2: [Identifier] bar
+# 263| 78: [RescueModifier] RescueModifier
+# 263| 0: [ScopeResolution] ScopeResolution
+# 263| 0: [Constant] X
+# 263| 1: [ReservedWord] ::
+# 263| 2: [Identifier] foo
+# 263| 1: [ReservedWord] rescue
+# 263| 2: [ScopeResolution] ScopeResolution
+# 263| 0: [Constant] X
+# 263| 1: [ReservedWord] ::
+# 263| 2: [Identifier] bar
+# 266| 79: [Call] Call
+# 266| 0: [Identifier] foo
+# 266| 1: [ArgumentList] ArgumentList
+# 266| 0: [ReservedWord] (
+# 266| 1: [BlockArgument] BlockArgument
+# 266| 0: [ReservedWord] &
+# 266| 1: [Identifier] bar
+# 266| 2: [ReservedWord] )
+# 267| 80: [Call] Call
+# 267| 0: [Identifier] foo
+# 267| 1: [ArgumentList] ArgumentList
+# 267| 0: [ReservedWord] (
+# 267| 1: [BlockArgument] BlockArgument
+# 267| 0: [ReservedWord] &
+# 267| 1: [ScopeResolution] ScopeResolution
+# 267| 0: [Constant] X
+# 267| 1: [ReservedWord] ::
+# 267| 2: [Identifier] bar
+# 267| 2: [ReservedWord] )
+# 268| 81: [Call] Call
+# 268| 0: [Identifier] foo
+# 268| 1: [ArgumentList] ArgumentList
+# 268| 0: [ReservedWord] (
+# 268| 1: [BlockArgument] BlockArgument
+# 268| 0: [ReservedWord] &
+# 268| 2: [ReservedWord] )
+# 270| 82: [Call] Call
+# 270| 0: [Identifier] foo
+# 270| 1: [ArgumentList] ArgumentList
+# 270| 0: [ReservedWord] (
+# 270| 1: [SplatArgument] SplatArgument
+# 270| 0: [ReservedWord] *
+# 270| 1: [Identifier] bar
+# 270| 2: [ReservedWord] )
+# 271| 83: [Call] Call
+# 271| 0: [Identifier] foo
+# 271| 1: [ArgumentList] ArgumentList
+# 271| 0: [ReservedWord] (
+# 271| 1: [SplatArgument] SplatArgument
+# 271| 0: [ReservedWord] *
+# 271| 1: [ScopeResolution] ScopeResolution
+# 271| 0: [Constant] X
+# 271| 1: [ReservedWord] ::
+# 271| 2: [Identifier] bar
+# 271| 2: [ReservedWord] )
+# 274| 84: [Call] Call
+# 274| 0: [Identifier] foo
+# 274| 1: [ArgumentList] ArgumentList
+# 274| 0: [ReservedWord] (
+# 274| 1: [HashSplatArgument] HashSplatArgument
+# 274| 0: [ReservedWord] **
+# 274| 1: [Identifier] bar
+# 274| 2: [ReservedWord] )
+# 275| 85: [Call] Call
+# 275| 0: [Identifier] foo
+# 275| 1: [ArgumentList] ArgumentList
+# 275| 0: [ReservedWord] (
+# 275| 1: [HashSplatArgument] HashSplatArgument
+# 275| 0: [ReservedWord] **
+# 275| 1: [ScopeResolution] ScopeResolution
+# 275| 0: [Constant] X
+# 275| 1: [ReservedWord] ::
+# 275| 2: [Identifier] bar
+# 275| 2: [ReservedWord] )
+# 278| 86: [Call] Call
+# 278| 0: [Identifier] foo
+# 278| 1: [ArgumentList] ArgumentList
+# 278| 0: [ReservedWord] (
+# 278| 1: [Pair] Pair
+# 278| 0: [HashKeySymbol] blah
+# 278| 1: [ReservedWord] :
+# 278| 2: [Identifier] bar
+# 278| 2: [ReservedWord] )
+# 279| 87: [Call] Call
+# 279| 0: [Identifier] foo
+# 279| 1: [ArgumentList] ArgumentList
+# 279| 0: [ReservedWord] (
+# 279| 1: [Pair] Pair
+# 279| 0: [HashKeySymbol] blah
+# 279| 1: [ReservedWord] :
+# 279| 2: [ScopeResolution] ScopeResolution
+# 279| 0: [Constant] X
+# 279| 1: [ReservedWord] ::
+# 279| 2: [Identifier] bar
+# 279| 2: [ReservedWord] )
+# 284| 88: [Class] Class
+# 284| 0: [ReservedWord] class
+# 284| 1: [Constant] MyClass
+# 285| 2: [Method] Method
+# 285| 0: [ReservedWord] def
+# 285| 1: [Identifier] my_method
+# 286| 2: [Super] super
+# 287| 3: [Call] Call
+# 287| 0: [Super] super
+# 287| 1: [ArgumentList] ArgumentList
+# 287| 0: [ReservedWord] (
+# 287| 1: [ReservedWord] )
+# 288| 4: [Call] Call
+# 288| 0: [Super] super
+# 288| 1: [ArgumentList] ArgumentList
+# 288| 0: [String] String
+# 288| 0: [ReservedWord] '
+# 288| 1: [StringContent] blah
+# 288| 2: [ReservedWord] '
+# 289| 5: [Call] Call
+# 289| 0: [Super] super
+# 289| 1: [ArgumentList] ArgumentList
+# 289| 0: [Integer] 1
+# 289| 1: [ReservedWord] ,
+# 289| 2: [Integer] 2
+# 289| 3: [ReservedWord] ,
+# 289| 4: [Integer] 3
+# 290| 6: [Call] Call
+# 290| 0: [Super] super
+# 290| 1: [Block] Block
+# 290| 0: [ReservedWord] {
+# 290| 1: [BlockParameters] BlockParameters
+# 290| 0: [ReservedWord] |
+# 290| 1: [Identifier] x
+# 290| 2: [ReservedWord] |
+# 290| 2: [Binary] Binary
+# 290| 0: [Identifier] x
+# 290| 1: [ReservedWord] +
+# 290| 2: [Integer] 1
+# 290| 3: [ReservedWord] }
+# 291| 7: [Call] Call
+# 291| 0: [Super] super
+# 291| 1: [DoBlock] DoBlock
+# 291| 0: [ReservedWord] do
+# 291| 1: [BlockParameters] BlockParameters
+# 291| 0: [ReservedWord] |
+# 291| 1: [Identifier] x
+# 291| 2: [ReservedWord] |
+# 291| 2: [Binary] Binary
+# 291| 0: [Identifier] x
+# 291| 1: [ReservedWord] *
+# 291| 2: [Integer] 2
+# 291| 3: [ReservedWord] end
+# 292| 8: [Call] Call
+# 292| 0: [Super] super
+# 292| 1: [ArgumentList] ArgumentList
+# 292| 0: [Integer] 4
+# 292| 1: [ReservedWord] ,
+# 292| 2: [Integer] 5
+# 292| 2: [Block] Block
+# 292| 0: [ReservedWord] {
+# 292| 1: [BlockParameters] BlockParameters
+# 292| 0: [ReservedWord] |
+# 292| 1: [Identifier] x
+# 292| 2: [ReservedWord] |
+# 292| 2: [Binary] Binary
+# 292| 0: [Identifier] x
+# 292| 1: [ReservedWord] +
+# 292| 2: [Integer] 100
+# 292| 3: [ReservedWord] }
+# 293| 9: [Call] Call
+# 293| 0: [Super] super
+# 293| 1: [ArgumentList] ArgumentList
+# 293| 0: [Integer] 6
+# 293| 1: [ReservedWord] ,
+# 293| 2: [Integer] 7
+# 293| 2: [DoBlock] DoBlock
+# 293| 0: [ReservedWord] do
+# 293| 1: [BlockParameters] BlockParameters
+# 293| 0: [ReservedWord] |
+# 293| 1: [Identifier] x
+# 293| 2: [ReservedWord] |
+# 293| 2: [Binary] Binary
+# 293| 0: [Identifier] x
+# 293| 1: [ReservedWord] +
+# 293| 2: [Integer] 200
+# 293| 3: [ReservedWord] end
+# 294| 10: [ReservedWord] end
+# 295| 3: [ReservedWord] end
+# 301| 89: [Class] Class
+# 301| 0: [ReservedWord] class
+# 301| 1: [Constant] AnotherClass
+# 302| 2: [Method] Method
+# 302| 0: [ReservedWord] def
+# 302| 1: [Identifier] another_method
+# 303| 2: [Call] Call
+# 303| 0: [Identifier] foo
+# 303| 1: [ReservedWord] .
+# 303| 2: [Identifier] super
+# 304| 3: [Call] Call
+# 304| 0: [Self] self
+# 304| 1: [ReservedWord] .
+# 304| 2: [Identifier] super
+# 305| 4: [Call] Call
+# 305| 0: [Super] super
+# 305| 1: [ReservedWord] .
+# 305| 2: [Identifier] super
+# 306| 5: [ReservedWord] end
+# 307| 3: [ReservedWord] end
+# 310| 90: [Call] Call
+# 310| 0: [Identifier] foo
+# 310| 1: [ReservedWord] .
+# 310| 2: [ArgumentList] ArgumentList
+# 310| 0: [ReservedWord] (
+# 310| 1: [ReservedWord] )
+# 311| 91: [Call] Call
+# 311| 0: [Identifier] foo
+# 311| 1: [ReservedWord] .
+# 311| 2: [ArgumentList] ArgumentList
+# 311| 0: [ReservedWord] (
+# 311| 1: [Integer] 1
+# 311| 2: [ReservedWord] )
+# 314| 92: [Assignment] Assignment
+# 314| 0: [Call] Call
+# 314| 0: [Self] self
+# 314| 1: [ReservedWord] .
+# 314| 2: [Identifier] foo
+# 314| 1: [ReservedWord] =
+# 314| 2: [Integer] 10
+# 315| 93: [Assignment] Assignment
+# 315| 0: [ElementReference] ElementReference
+# 315| 0: [Identifier] foo
+# 315| 1: [ReservedWord] [
+# 315| 2: [Integer] 0
+# 315| 3: [ReservedWord] ]
+# 315| 1: [ReservedWord] =
+# 315| 2: [Integer] 10
+# 316| 94: [Assignment] Assignment
+# 316| 0: [LeftAssignmentList] LeftAssignmentList
+# 316| 0: [Call] Call
+# 316| 0: [Self] self
+# 316| 1: [ReservedWord] .
+# 316| 2: [Identifier] foo
+# 316| 1: [ReservedWord] ,
+# 316| 2: [RestAssignment] RestAssignment
+# 316| 0: [ReservedWord] *
+# 316| 1: [Call] Call
+# 316| 0: [Self] self
+# 316| 1: [ReservedWord] .
+# 316| 2: [Identifier] bar
+# 316| 3: [ReservedWord] ,
+# 316| 4: [ElementReference] ElementReference
+# 316| 0: [Identifier] foo
+# 316| 1: [ReservedWord] [
+# 316| 2: [Integer] 4
+# 316| 3: [ReservedWord] ]
+# 316| 1: [ReservedWord] =
+# 316| 2: [Array] Array
+# 316| 0: [ReservedWord] [
+# 316| 1: [Integer] 1
+# 316| 2: [ReservedWord] ,
+# 316| 3: [Integer] 2
+# 316| 4: [ReservedWord] ,
+# 316| 5: [Integer] 3
+# 316| 6: [ReservedWord] ,
+# 316| 7: [Integer] 4
+# 316| 8: [ReservedWord] ]
+# 317| 95: [Assignment] Assignment
+# 317| 0: [LeftAssignmentList] LeftAssignmentList
+# 317| 0: [Identifier] a
+# 317| 1: [ReservedWord] ,
+# 317| 2: [RestAssignment] RestAssignment
+# 317| 0: [ReservedWord] *
+# 317| 1: [ElementReference] ElementReference
+# 317| 0: [Identifier] foo
+# 317| 1: [ReservedWord] [
+# 317| 2: [Integer] 5
+# 317| 3: [ReservedWord] ]
+# 317| 1: [ReservedWord] =
+# 317| 2: [Array] Array
+# 317| 0: [ReservedWord] [
+# 317| 1: [Integer] 1
+# 317| 2: [ReservedWord] ,
+# 317| 3: [Integer] 2
+# 317| 4: [ReservedWord] ,
+# 317| 5: [Integer] 3
+# 317| 6: [ReservedWord] ]
+# 318| 96: [OperatorAssignment] OperatorAssignment
+# 318| 0: [Call] Call
+# 318| 0: [Self] self
+# 318| 1: [ReservedWord] .
+# 318| 2: [Identifier] count
+# 318| 1: [ReservedWord] +=
+# 318| 2: [Integer] 1
+# 319| 97: [OperatorAssignment] OperatorAssignment
+# 319| 0: [ElementReference] ElementReference
+# 319| 0: [Identifier] foo
+# 319| 1: [ReservedWord] [
+# 319| 2: [Integer] 0
+# 319| 3: [ReservedWord] ]
+# 319| 1: [ReservedWord] +=
+# 319| 2: [Integer] 1
+# 320| 98: [OperatorAssignment] OperatorAssignment
+# 320| 0: [ElementReference] ElementReference
+# 320| 0: [Call] Call
+# 320| 0: [Identifier] foo
+# 320| 1: [ReservedWord] .
+# 320| 2: [Identifier] bar
+# 320| 1: [ReservedWord] [
+# 320| 2: [Integer] 0
+# 320| 3: [ReservedWord] ,
+# 320| 4: [Call] Call
+# 320| 0: [Identifier] foo
+# 320| 1: [ReservedWord] .
+# 320| 2: [Identifier] baz
+# 320| 5: [ReservedWord] ,
+# 320| 6: [Binary] Binary
+# 320| 0: [Call] Call
+# 320| 0: [Identifier] foo
+# 320| 1: [ReservedWord] .
+# 320| 2: [Identifier] boo
+# 320| 1: [ReservedWord] +
+# 320| 2: [Integer] 1
+# 320| 7: [ReservedWord] ]
+# 320| 1: [ReservedWord] *=
+# 320| 2: [Integer] 2
+# 323| 99: [Method] Method
+# 323| 0: [ReservedWord] def
+# 323| 1: [Identifier] foo
+# 323| 2: [ReservedWord] =
+# 323| 3: [Identifier] bar
+# 324| 100: [Method] Method
+# 324| 0: [ReservedWord] def
+# 324| 1: [Identifier] foo
+# 324| 2: [MethodParameters] MethodParameters
+# 324| 0: [ReservedWord] (
+# 324| 1: [ReservedWord] )
+# 324| 3: [ReservedWord] =
+# 324| 4: [Identifier] bar
+# 325| 101: [Method] Method
+# 325| 0: [ReservedWord] def
+# 325| 1: [Identifier] foo
+# 325| 2: [MethodParameters] MethodParameters
+# 325| 0: [ReservedWord] (
+# 325| 1: [Identifier] x
+# 325| 2: [ReservedWord] )
+# 325| 3: [ReservedWord] =
+# 325| 4: [Identifier] bar
+# 326| 102: [SingletonMethod] SingletonMethod
+# 326| 0: [ReservedWord] def
+# 326| 1: [Constant] Object
+# 326| 2: [ReservedWord] .
+# 326| 3: [Identifier] foo
+# 326| 4: [ReservedWord] =
+# 326| 5: [Identifier] bar
+# 327| 103: [SingletonMethod] SingletonMethod
+# 327| 0: [ReservedWord] def
+# 327| 1: [Constant] Object
+# 327| 2: [ReservedWord] .
+# 327| 3: [Identifier] foo
+# 327| 4: [MethodParameters] MethodParameters
+# 327| 0: [ReservedWord] (
+# 327| 1: [Identifier] x
+# 327| 2: [ReservedWord] )
+# 327| 5: [ReservedWord] =
+# 327| 6: [Identifier] bar
+# 328| 104: [Method] Method
+# 328| 0: [ReservedWord] def
+# 328| 1: [Identifier] foo
+# 328| 2: [MethodParameters] MethodParameters
+# 328| 0: [ReservedWord] (
+# 328| 1: [ReservedWord] )
+# 328| 3: [ReservedWord] =
+# 328| 4: [RescueModifier] RescueModifier
+# 328| 0: [Identifier] bar
+# 328| 1: [ReservedWord] rescue
+# 328| 2: [ParenthesizedStatements] ParenthesizedStatements
+# 328| 0: [ReservedWord] (
+# 328| 1: [Call] Call
+# 328| 0: [Identifier] print
+# 328| 1: [ArgumentList] ArgumentList
+# 328| 0: [String] String
+# 328| 0: [ReservedWord] "
+# 328| 1: [StringContent] error
+# 328| 2: [ReservedWord] "
+# 328| 2: [ReservedWord] )
+# 331| 105: [Method] Method
+# 331| 0: [ReservedWord] def
+# 331| 1: [Identifier] foo
+# 331| 2: [MethodParameters] MethodParameters
+# 331| 0: [ReservedWord] (
+# 331| 1: [ForwardParameter] ...
+# 331| 0: [ReservedWord] ...
+# 331| 2: [ReservedWord] )
+# 332| 3: [Call] Call
+# 332| 0: [Super] super
+# 332| 1: [ArgumentList] ArgumentList
+# 332| 0: [ReservedWord] (
+# 332| 1: [ForwardArgument] ...
+# 332| 0: [ReservedWord] ...
+# 332| 2: [ReservedWord] )
+# 333| 4: [ReservedWord] end
+# 335| 106: [Method] Method
+# 335| 0: [ReservedWord] def
+# 335| 1: [Identifier] foo
+# 335| 2: [MethodParameters] MethodParameters
+# 335| 0: [ReservedWord] (
+# 335| 1: [Identifier] a
+# 335| 2: [ReservedWord] ,
+# 335| 3: [Identifier] b
+# 335| 4: [ReservedWord] ,
+# 335| 5: [ForwardParameter] ...
+# 335| 0: [ReservedWord] ...
+# 335| 6: [ReservedWord] )
+# 336| 3: [Call] Call
+# 336| 0: [Identifier] bar
+# 336| 1: [ArgumentList] ArgumentList
+# 336| 0: [ReservedWord] (
+# 336| 1: [Identifier] b
+# 336| 2: [ReservedWord] ,
+# 336| 3: [ForwardArgument] ...
+# 336| 0: [ReservedWord] ...
+# 336| 4: [ReservedWord] )
+# 337| 4: [ReservedWord] end
+# 340| 107: [For] For
+# 340| 0: [ReservedWord] for
+# 340| 1: [LeftAssignmentList] LeftAssignmentList
+# 340| 0: [Identifier] x
+# 340| 1: [ReservedWord] ,
+# 340| 2: [Identifier] y
+# 340| 3: [ReservedWord] ,
+# 340| 4: [Identifier] z
+# 340| 2: [In] In
+# 340| 0: [ReservedWord] in
+# 340| 1: [Array] Array
+# 340| 0: [ReservedWord] [
+# 340| 1: [Array] Array
+# 340| 0: [ReservedWord] [
+# 340| 1: [Integer] 1
+# 340| 2: [ReservedWord] ,
+# 340| 3: [Integer] 2
+# 340| 4: [ReservedWord] ,
+# 340| 5: [Integer] 3
+# 340| 6: [ReservedWord] ]
+# 340| 2: [ReservedWord] ,
+# 340| 3: [Array] Array
+# 340| 0: [ReservedWord] [
+# 340| 1: [Integer] 4
+# 340| 2: [ReservedWord] ,
+# 340| 3: [Integer] 5
+# 340| 4: [ReservedWord] ,
+# 340| 5: [Integer] 6
+# 340| 6: [ReservedWord] ]
+# 340| 4: [ReservedWord] ]
+# 340| 3: [Do] Do
+# 341| 0: [Call] Call
+# 341| 0: [Identifier] foo
+# 341| 1: [ArgumentList] ArgumentList
+# 341| 0: [Identifier] x
+# 341| 1: [ReservedWord] ,
+# 341| 2: [Identifier] y
+# 341| 3: [ReservedWord] ,
+# 341| 4: [Identifier] z
+# 342| 1: [ReservedWord] end
+# 344| 108: [Call] Call
+# 344| 0: [Identifier] foo
+# 344| 1: [ArgumentList] ArgumentList
+# 344| 0: [ReservedWord] (
+# 344| 1: [Pair] Pair
+# 344| 0: [HashKeySymbol] x
+# 344| 1: [ReservedWord] :
+# 344| 2: [Integer] 42
+# 344| 2: [ReservedWord] )
+# 345| 109: [Call] Call
+# 345| 0: [Identifier] foo
+# 345| 1: [ArgumentList] ArgumentList
+# 345| 0: [ReservedWord] (
+# 345| 1: [Pair] Pair
+# 345| 0: [HashKeySymbol] x
+# 345| 1: [ReservedWord] :
+# 345| 2: [ReservedWord] ,
+# 345| 3: [Pair] Pair
+# 345| 0: [HashKeySymbol] novar
+# 345| 1: [ReservedWord] :
+# 345| 4: [ReservedWord] )
+# 346| 110: [Call] Call
+# 346| 0: [Identifier] foo
+# 346| 1: [ArgumentList] ArgumentList
+# 346| 0: [ReservedWord] (
+# 346| 1: [Pair] Pair
+# 346| 0: [HashKeySymbol] X
+# 346| 1: [ReservedWord] :
+# 346| 2: [Integer] 42
+# 346| 2: [ReservedWord] )
+# 347| 111: [Call] Call
+# 347| 0: [Identifier] foo
+# 347| 1: [ArgumentList] ArgumentList
+# 347| 0: [ReservedWord] (
+# 347| 1: [Pair] Pair
+# 347| 0: [HashKeySymbol] X
+# 347| 1: [ReservedWord] :
+# 347| 2: [ReservedWord] )
+# 350| 112: [Assignment] Assignment
+# 350| 0: [Identifier] y
+# 350| 1: [ReservedWord] =
+# 350| 2: [Integer] 1
+# 351| 113: [Assignment] Assignment
+# 351| 0: [Identifier] one
+# 351| 1: [ReservedWord] =
+# 351| 2: [Lambda] Lambda
+# 351| 0: [ReservedWord] ->
+# 351| 1: [LambdaParameters] LambdaParameters
+# 351| 0: [ReservedWord] (
+# 351| 1: [Identifier] x
+# 351| 2: [ReservedWord] )
+# 351| 2: [Block] Block
+# 351| 0: [ReservedWord] {
+# 351| 1: [Identifier] y
+# 351| 2: [ReservedWord] }
+# 352| 114: [Assignment] Assignment
+# 352| 0: [Identifier] f
+# 352| 1: [ReservedWord] =
+# 352| 2: [Lambda] Lambda
+# 352| 0: [ReservedWord] ->
+# 352| 1: [LambdaParameters] LambdaParameters
+# 352| 0: [ReservedWord] (
+# 352| 1: [Identifier] x
+# 352| 2: [ReservedWord] )
+# 352| 2: [Block] Block
+# 352| 0: [ReservedWord] {
+# 352| 1: [Call] Call
+# 352| 0: [Identifier] foo
+# 352| 1: [ArgumentList] ArgumentList
+# 352| 0: [Identifier] x
+# 352| 2: [ReservedWord] }
+# 353| 115: [Assignment] Assignment
+# 353| 0: [Identifier] g
+# 353| 1: [ReservedWord] =
+# 353| 2: [Lambda] Lambda
+# 353| 0: [ReservedWord] ->
+# 353| 1: [LambdaParameters] LambdaParameters
+# 353| 0: [ReservedWord] (
+# 353| 1: [Identifier] x
+# 353| 2: [ReservedWord] )
+# 353| 2: [Block] Block
+# 353| 0: [ReservedWord] {
+# 353| 1: [Identifier] unknown_call
+# 353| 2: [ReservedWord] }
+# 354| 116: [Assignment] Assignment
+# 354| 0: [Identifier] h
+# 354| 1: [ReservedWord] =
+# 354| 2: [Lambda] Lambda
+# 354| 0: [ReservedWord] ->
+# 354| 1: [LambdaParameters] LambdaParameters
+# 354| 0: [ReservedWord] (
+# 354| 1: [Identifier] x
+# 354| 2: [ReservedWord] )
+# 354| 2: [DoBlock] DoBlock
+# 354| 0: [ReservedWord] do
+# 355| 1: [Identifier] x
+# 356| 2: [Identifier] y
+# 357| 3: [Identifier] unknown_call
+# 358| 4: [ReservedWord] end
+# 1| [Comment] # call with no receiver, arguments, or block
+# 4| [Comment] # call whose name is a scope resolution
+# 7| [Comment] # call whose name is a global scope resolution
+# 10| [Comment] # call with a receiver, no arguments or block
+# 13| [Comment] # call with arguments
+# 16| [Comment] # call with curly brace block
+# 19| [Comment] # call with do block
+# 24| [Comment] # call with receiver, arguments, and a block
+# 29| [Comment] # a yield call
+# 34| [Comment] # a yield call with arguments
+# 39| [Comment] # ------------------------------------------------------------------------------
+# 40| [Comment] # Calls without parentheses or arguments are parsed by tree-sitter simply as
+# 41| [Comment] # `identifier` nodes (or `scope_resolution` nodes whose `name` field is an
+# 42| [Comment] # `identifier), so here we test that our AST library correctly represents them
+# 43| [Comment] # as calls in all the following contexts.
+# 45| [Comment] # root level (child of program)
+# 49| [Comment] # in a parenthesized statement
+# 53| [Comment] # in an argument list
+# 57| [Comment] # in an array
+# 61| [Comment] # RHS of an assignment
+# 65| [Comment] # RHS an operator assignment
+# 69| [Comment] # RHS assignment list
+# 72| [Comment] # in a begin-end block
+# 78| [Comment] # in a BEGIN block
+# 81| [Comment] # in an END block
+# 84| [Comment] # both operands of a binary operation
+# 87| [Comment] # unary operand
+# 91| [Comment] # in a curly brace block
+# 94| [Comment] # in a do-end block
+# 100| [Comment] # the receiver in a call can itself be a call
+# 104| [Comment] # the value for a case expr
+# 105| [Comment] # and the when pattern and body
+# 115| [Comment] # in a class definition
+# 121| [Comment] # in a superclass
+# 127| [Comment] # in a singleton class value or body
+# 135| [Comment] # in a method body
+# 141| [Comment] # in a singleton method object or body
+# 147| [Comment] # in the default value for a keyword parameter
+# 153| [Comment] # in the default value for an optional parameter
+# 159| [Comment] # in a module
+# 165| [Comment] # ternary if: condition, consequence, and alternative can all be calls
+# 169| [Comment] # if/elsif/else conditions and bodies
+# 185| [Comment] # if-modifier condition/body
+# 189| [Comment] # unless condition/body
+# 197| [Comment] # unless-modifier condition/body
+# 201| [Comment] # while loop condition/body
+# 209| [Comment] # while-modifier loop condition/body
+# 213| [Comment] # until loop condition/body
+# 221| [Comment] # until-modifier loop condition/body
+# 225| [Comment] # the collection being iterated over in a for loop, and the body
+# 233| [Comment] # in an array indexing operation, both the object and the index can be calls
+# 237| [Comment] # interpolation
+# 240| [Comment] # the scope in a scope resolution
+# 244| [Comment] # in a range
+# 248| [Comment] # the key/value in a hash pair
+# 251| [Comment] # rescue exceptions and ensure
+# 261| [Comment] # rescue-modifier body and handler
+# 265| [Comment] # block argument
+# 269| [Comment] # splat argument
+# 273| [Comment] # hash-splat argument
+# 277| [Comment] # the value in a keyword argument
+# 281| [Comment] # ------------------------------------------------------------------------------
+# 282| [Comment] # calls to `super`
+# 297| [Comment] # ------------------------------------------------------------------------------
+# 298| [Comment] # calls to methods simply named `super`, i.e. *not* calls to the same method in
+# 299| [Comment] # a parent classs, so these should be Call but not SuperCall
+# 305| [Comment] # we expect the receiver to be a SuperCall, while the outer call should not (it's just a regular Call)
+# 309| [Comment] # calls without method name
+# 313| [Comment] # setter calls
+# 322| [Comment] # endless method definitions
+# 330| [Comment] # forward parameter and forwarded arguments
+# 339| [Comment] # for loop over nested array
+# 349| [Comment] # calls inside lambdas
+constants/constants.rb:
+# 1| [Program] Program
+# 1| 0: [Module] Module
+# 1| 0: [ReservedWord] module
+# 1| 1: [Constant] ModuleA
+# 2| 2: [Class] Class
+# 2| 0: [ReservedWord] class
+# 2| 1: [Constant] ClassA
+# 3| 2: [Assignment] Assignment
+# 3| 0: [Constant] CONST_A
+# 3| 1: [ReservedWord] =
+# 3| 2: [String] String
+# 3| 0: [ReservedWord] "
+# 3| 1: [StringContent] const_a
+# 3| 2: [ReservedWord] "
+# 4| 3: [ReservedWord] end
+# 6| 3: [Assignment] Assignment
+# 6| 0: [Constant] CONST_B
+# 6| 1: [ReservedWord] =
+# 6| 2: [String] String
+# 6| 0: [ReservedWord] "
+# 6| 1: [StringContent] const_b
+# 6| 2: [ReservedWord] "
+# 8| 4: [Module] Module
+# 8| 0: [ReservedWord] module
+# 8| 1: [Constant] ModuleB
+# 9| 2: [Class] Class
+# 9| 0: [ReservedWord] class
+# 9| 1: [Constant] ClassB
+# 9| 2: [Superclass] Superclass
+# 9| 0: [ReservedWord] <
+# 9| 1: [Constant] Base
+# 10| 3: [ReservedWord] end
+# 12| 3: [Class] Class
+# 12| 0: [ReservedWord] class
+# 12| 1: [Constant] ClassC
+# 12| 2: [Superclass] Superclass
+# 12| 0: [ReservedWord] <
+# 12| 1: [ScopeResolution] ScopeResolution
+# 12| 0: [ScopeResolution] ScopeResolution
+# 12| 0: [Constant] X
+# 12| 1: [ReservedWord] ::
+# 12| 2: [Constant] Y
+# 12| 1: [ReservedWord] ::
+# 12| 2: [Constant] Z
+# 13| 3: [ReservedWord] end
+# 14| 4: [ReservedWord] end
+# 15| 5: [ReservedWord] end
+# 17| 1: [Assignment] Assignment
+# 17| 0: [Constant] GREETING
+# 17| 1: [ReservedWord] =
+# 17| 2: [Binary] Binary
+# 17| 0: [Binary] Binary
+# 17| 0: [String] String
+# 17| 0: [ReservedWord] '
+# 17| 1: [StringContent] Hello
+# 17| 2: [ReservedWord] '
+# 17| 1: [ReservedWord] +
+# 17| 2: [ScopeResolution] ScopeResolution
+# 17| 0: [ScopeResolution] ScopeResolution
+# 17| 0: [Constant] ModuleA
+# 17| 1: [ReservedWord] ::
+# 17| 2: [Constant] ClassA
+# 17| 1: [ReservedWord] ::
+# 17| 2: [Constant] CONST_A
+# 17| 1: [ReservedWord] +
+# 17| 2: [ScopeResolution] ScopeResolution
+# 17| 0: [Constant] ModuleA
+# 17| 1: [ReservedWord] ::
+# 17| 2: [Constant] CONST_B
+# 19| 2: [Method] Method
+# 19| 0: [ReservedWord] def
+# 19| 1: [Identifier] foo
+# 20| 2: [Assignment] Assignment
+# 20| 0: [Constant] Names
+# 20| 1: [ReservedWord] =
+# 20| 2: [Array] Array
+# 20| 0: [ReservedWord] [
+# 20| 1: [String] String
+# 20| 0: [ReservedWord] '
+# 20| 1: [StringContent] Vera
+# 20| 2: [ReservedWord] '
+# 20| 2: [ReservedWord] ,
+# 20| 3: [String] String
+# 20| 0: [ReservedWord] '
+# 20| 1: [StringContent] Chuck
+# 20| 2: [ReservedWord] '
+# 20| 4: [ReservedWord] ,
+# 20| 5: [String] String
+# 20| 0: [ReservedWord] '
+# 20| 1: [StringContent] Dave
+# 20| 2: [ReservedWord] '
+# 20| 6: [ReservedWord] ]
+# 22| 3: [Call] Call
+# 22| 0: [Constant] Names
+# 22| 1: [ReservedWord] .
+# 22| 2: [Identifier] each
+# 22| 3: [DoBlock] DoBlock
+# 22| 0: [ReservedWord] do
+# 22| 1: [BlockParameters] BlockParameters
+# 22| 0: [ReservedWord] |
+# 22| 1: [Identifier] name
+# 22| 2: [ReservedWord] |
+# 23| 2: [Call] Call
+# 23| 0: [Identifier] puts
+# 23| 1: [ArgumentList] ArgumentList
+# 23| 0: [String] String
+# 23| 0: [ReservedWord] "
+# 23| 1: [Interpolation] Interpolation
+# 23| 0: [ReservedWord] #{
+# 23| 1: [Constant] GREETING
+# 23| 2: [ReservedWord] }
+# 23| 2: [StringContent]
+# 23| 3: [Interpolation] Interpolation
+# 23| 0: [ReservedWord] #{
+# 23| 1: [Identifier] name
+# 23| 2: [ReservedWord] }
+# 23| 4: [ReservedWord] "
+# 24| 3: [ReservedWord] end
+# 28| 4: [Call] Call
+# 28| 0: [Constant] Array
+# 28| 1: [ArgumentList] ArgumentList
+# 28| 0: [ReservedWord] (
+# 28| 1: [String] String
+# 28| 0: [ReservedWord] '
+# 28| 1: [StringContent] foo
+# 28| 2: [ReservedWord] '
+# 28| 2: [ReservedWord] )
+# 29| 5: [ReservedWord] end
+# 31| 3: [Class] Class
+# 31| 0: [ReservedWord] class
+# 31| 1: [ScopeResolution] ScopeResolution
+# 31| 0: [Constant] ModuleA
+# 31| 1: [ReservedWord] ::
+# 31| 2: [Constant] ClassD
+# 31| 2: [Superclass] Superclass
+# 31| 0: [ReservedWord] <
+# 31| 1: [ScopeResolution] ScopeResolution
+# 31| 0: [Constant] ModuleA
+# 31| 1: [ReservedWord] ::
+# 31| 2: [Constant] ClassA
+# 32| 3: [Assignment] Assignment
+# 32| 0: [Constant] FOURTY_TWO
+# 32| 1: [ReservedWord] =
+# 32| 2: [Integer] 42
+# 33| 4: [ReservedWord] end
+# 35| 4: [Module] Module
+# 35| 0: [ReservedWord] module
+# 35| 1: [ScopeResolution] ScopeResolution
+# 35| 0: [Constant] ModuleA
+# 35| 1: [ReservedWord] ::
+# 35| 2: [Constant] ModuleC
+# 36| 2: [Assignment] Assignment
+# 36| 0: [Constant] FOURTY_THREE
+# 36| 1: [ReservedWord] =
+# 36| 2: [Integer] 43
+# 37| 3: [ReservedWord] end
+# 39| 5: [Assignment] Assignment
+# 39| 0: [ScopeResolution] ScopeResolution
+# 39| 0: [ScopeResolution] ScopeResolution
+# 39| 0: [Constant] ModuleA
+# 39| 1: [ReservedWord] ::
+# 39| 2: [Constant] ModuleB
+# 39| 1: [ReservedWord] ::
+# 39| 2: [Constant] MAX_SIZE
+# 39| 1: [ReservedWord] =
+# 39| 2: [Integer] 1024
+# 41| 6: [Call] Call
+# 41| 0: [Identifier] puts
+# 41| 1: [ArgumentList] ArgumentList
+# 41| 0: [ScopeResolution] ScopeResolution
+# 41| 0: [ScopeResolution] ScopeResolution
+# 41| 0: [Constant] ModuleA
+# 41| 1: [ReservedWord] ::
+# 41| 2: [Constant] ModuleB
+# 41| 1: [ReservedWord] ::
+# 41| 2: [Constant] MAX_SIZE
+# 43| 7: [Call] Call
+# 43| 0: [Identifier] puts
+# 43| 1: [ArgumentList] ArgumentList
+# 43| 0: [Constant] GREETING
+# 44| 8: [Call] Call
+# 44| 0: [Identifier] puts
+# 44| 1: [ArgumentList] ArgumentList
+# 44| 0: [ScopeResolution] ScopeResolution
+# 44| 0: [ReservedWord] ::
+# 44| 1: [Constant] GREETING
+# 46| 9: [Module] Module
+# 46| 0: [ReservedWord] module
+# 46| 1: [ScopeResolution] ScopeResolution
+# 46| 0: [Constant] ModuleA
+# 46| 1: [ReservedWord] ::
+# 46| 2: [Constant] ModuleB
+# 47| 2: [Class] Class
+# 47| 0: [ReservedWord] class
+# 47| 1: [Constant] ClassB
+# 47| 2: [Superclass] Superclass
+# 47| 0: [ReservedWord] <
+# 47| 1: [Constant] Base
+# 48| 3: [Assignment] Assignment
+# 48| 0: [Constant] FOURTY_ONE
+# 48| 1: [ReservedWord] =
+# 48| 2: [Integer] 41
+# 49| 4: [ReservedWord] end
+# 50| 3: [ReservedWord] end
+# 52| 10: [Module] Module
+# 52| 0: [ReservedWord] module
+# 52| 1: [Constant] ModuleA
+# 53| 2: [Assignment] Assignment
+# 53| 0: [Constant] FOURTY_FOUR
+# 53| 1: [ReservedWord] =
+# 53| 2: [String] String
+# 53| 0: [ReservedWord] "
+# 53| 1: [StringContent] fourty-four
+# 53| 2: [ReservedWord] "
+# 54| 3: [Class] Class
+# 54| 0: [ReservedWord] class
+# 54| 1: [ScopeResolution] ScopeResolution
+# 54| 0: [Constant] ModuleB
+# 54| 1: [ReservedWord] ::
+# 54| 2: [Constant] ClassB
+# 54| 2: [Superclass] Superclass
+# 54| 0: [ReservedWord] <
+# 54| 1: [Constant] Base
+# 55| 3: [Assignment] Assignment
+# 55| 0: [ClassVariable] @@fourty_four
+# 55| 1: [ReservedWord] =
+# 55| 2: [Constant] FOURTY_FOUR
+# 56| 4: [Assignment] Assignment
+# 56| 0: [Constant] FOURTY_FOUR
+# 56| 1: [ReservedWord] =
+# 56| 2: [Integer] 44
+# 57| 5: [Assignment] Assignment
+# 57| 0: [ClassVariable] @@fourty_four
+# 57| 1: [ReservedWord] =
+# 57| 2: [Constant] FOURTY_FOUR
+# 58| 6: [ReservedWord] end
+# 59| 4: [ReservedWord] end
+# 61| 11: [Module] Module
+# 61| 0: [ReservedWord] module
+# 61| 1: [Constant] Mod1
+# 62| 2: [Module] Module
+# 62| 0: [ReservedWord] module
+# 62| 1: [Constant] Mod3
+# 63| 2: [Assignment] Assignment
+# 63| 0: [Constant] FOURTY_FIVE
+# 63| 1: [ReservedWord] =
+# 63| 2: [Integer] 45
+# 64| 3: [ReservedWord] end
+# 65| 3: [Assignment] Assignment
+# 65| 0: [ClassVariable] @@fourty_five
+# 65| 1: [ReservedWord] =
+# 65| 2: [ScopeResolution] ScopeResolution
+# 65| 0: [Constant] Mod3
+# 65| 1: [ReservedWord] ::
+# 65| 2: [Constant] FOURTY_FIVE
+# 66| 4: [ReservedWord] end
+# 68| 12: [Module] Module
+# 68| 0: [ReservedWord] module
+# 68| 1: [Constant] Mod4
+# 69| 2: [Call] Call
+# 69| 0: [Identifier] include
+# 69| 1: [ArgumentList] ArgumentList
+# 69| 0: [Constant] Mod1
+# 70| 3: [Module] Module
+# 70| 0: [ReservedWord] module
+# 70| 1: [ScopeResolution] ScopeResolution
+# 70| 0: [Constant] Mod3
+# 70| 1: [ReservedWord] ::
+# 70| 2: [Constant] Mod5
+# 71| 2: [Assignment] Assignment
+# 71| 0: [Constant] FOURTY_SIX
+# 71| 1: [ReservedWord] =
+# 71| 2: [Integer] 46
+# 72| 3: [ReservedWord] end
+# 73| 4: [Assignment] Assignment
+# 73| 0: [ClassVariable] @@fourty_six
+# 73| 1: [ReservedWord] =
+# 73| 2: [ScopeResolution] ScopeResolution
+# 73| 0: [Constant] Mod3
+# 73| 1: [ReservedWord] ::
+# 73| 2: [Constant] FOURTY_SIX
+# 74| 5: [ReservedWord] end
+# 26| [Comment] # A call to Kernel::Array; despite beginning with an upper-case character,
+# 27| [Comment] # we don't consider this to be a constant access.
+# 55| [Comment] # refers to ::ModuleA::FOURTY_FOUR
+# 57| [Comment] # refers to ::ModuleA::ModuleB::ClassB::FOURTY_FOUR
+control/cases.rb:
+# 1| [Program] Program
+# 2| 0: [Assignment] Assignment
+# 2| 0: [Identifier] a
+# 2| 1: [ReservedWord] =
+# 2| 2: [Integer] 0
+# 3| 1: [Assignment] Assignment
+# 3| 0: [Identifier] b
+# 3| 1: [ReservedWord] =
+# 3| 2: [Integer] 0
+# 4| 2: [Assignment] Assignment
+# 4| 0: [Identifier] c
+# 4| 1: [ReservedWord] =
+# 4| 2: [Integer] 0
+# 5| 3: [Assignment] Assignment
+# 5| 0: [Identifier] d
+# 5| 1: [ReservedWord] =
+# 5| 2: [Integer] 0
+# 8| 4: [Case] Case
+# 8| 0: [ReservedWord] case
+# 8| 1: [Identifier] a
+# 9| 2: [When] When
+# 9| 0: [ReservedWord] when
+# 9| 1: [Pattern] Pattern
+# 9| 0: [Identifier] b
+# 9| 2: [Then] Then
+# 10| 0: [Integer] 100
+# 11| 3: [When] When
+# 11| 0: [ReservedWord] when
+# 11| 1: [Pattern] Pattern
+# 11| 0: [Identifier] c
+# 11| 2: [ReservedWord] ,
+# 11| 3: [Pattern] Pattern
+# 11| 0: [Identifier] d
+# 11| 4: [Then] Then
+# 12| 0: [Integer] 200
+# 13| 4: [Else] Else
+# 13| 0: [ReservedWord] else
+# 14| 1: [Integer] 300
+# 15| 5: [ReservedWord] end
+# 18| 5: [Case] Case
+# 18| 0: [ReservedWord] case
+# 19| 1: [When] When
+# 19| 0: [ReservedWord] when
+# 19| 1: [Pattern] Pattern
+# 19| 0: [Binary] Binary
+# 19| 0: [Identifier] a
+# 19| 1: [ReservedWord] >
+# 19| 2: [Identifier] b
+# 19| 2: [Then] Then
+# 19| 0: [ReservedWord] then
+# 19| 1: [Integer] 10
+# 20| 2: [When] When
+# 20| 0: [ReservedWord] when
+# 20| 1: [Pattern] Pattern
+# 20| 0: [Binary] Binary
+# 20| 0: [Identifier] a
+# 20| 1: [ReservedWord] ==
+# 20| 2: [Identifier] b
+# 20| 2: [Then] Then
+# 20| 0: [ReservedWord] then
+# 20| 1: [Integer] 20
+# 21| 3: [When] When
+# 21| 0: [ReservedWord] when
+# 21| 1: [Pattern] Pattern
+# 21| 0: [Binary] Binary
+# 21| 0: [Identifier] a
+# 21| 1: [ReservedWord] <
+# 21| 2: [Identifier] b
+# 21| 2: [Then] Then
+# 21| 0: [ReservedWord] then
+# 21| 1: [Integer] 30
+# 22| 4: [ReservedWord] end
+# 26| 6: [CaseMatch] CaseMatch
+# 26| 0: [ReservedWord] case
+# 26| 1: [Identifier] expr
+# 27| 2: [InClause] InClause
+# 27| 0: [ReservedWord] in
+# 27| 1: [Integer] 5
+# 27| 2: [Then] Then
+# 27| 0: [ReservedWord] then
+# 27| 1: [True] true
+# 28| 3: [Else] Else
+# 28| 0: [ReservedWord] else
+# 28| 1: [False] false
+# 29| 4: [ReservedWord] end
+# 31| 7: [CaseMatch] CaseMatch
+# 31| 0: [ReservedWord] case
+# 31| 1: [Identifier] expr
+# 32| 2: [InClause] InClause
+# 32| 0: [ReservedWord] in
+# 32| 1: [Identifier] x
+# 32| 2: [UnlessGuard] UnlessGuard
+# 32| 0: [ReservedWord] unless
+# 32| 1: [Binary] Binary
+# 32| 0: [Identifier] x
+# 32| 1: [ReservedWord] <
+# 32| 2: [Integer] 0
+# 32| 3: [Then] Then
+# 33| 0: [ReservedWord] then
+# 33| 1: [True] true
+# 34| 3: [InClause] InClause
+# 34| 0: [ReservedWord] in
+# 34| 1: [Identifier] x
+# 34| 2: [IfGuard] IfGuard
+# 34| 0: [ReservedWord] if
+# 34| 1: [Binary] Binary
+# 34| 0: [Identifier] x
+# 34| 1: [ReservedWord] <
+# 34| 2: [Integer] 0
+# 34| 3: [Then] Then
+# 35| 0: [ReservedWord] then
+# 35| 1: [True] true
+# 36| 4: [Else] Else
+# 36| 0: [ReservedWord] else
+# 36| 1: [False] false
+# 37| 5: [ReservedWord] end
+# 39| 8: [CaseMatch] CaseMatch
+# 39| 0: [ReservedWord] case
+# 39| 1: [Identifier] expr
+# 40| 2: [InClause] InClause
+# 40| 0: [ReservedWord] in
+# 40| 1: [Integer] 5
+# 41| 3: [InClause] InClause
+# 41| 0: [ReservedWord] in
+# 41| 1: [ArrayPattern] ArrayPattern
+# 41| 0: [Integer] 5
+# 41| 1: [SplatParameter] SplatParameter
+# 42| 4: [InClause] InClause
+# 42| 0: [ReservedWord] in
+# 42| 1: [ArrayPattern] ArrayPattern
+# 42| 0: [Integer] 1
+# 42| 1: [ReservedWord] ,
+# 42| 2: [Integer] 2
+# 43| 5: [InClause] InClause
+# 43| 0: [ReservedWord] in
+# 43| 1: [ArrayPattern] ArrayPattern
+# 43| 0: [Integer] 1
+# 43| 1: [ReservedWord] ,
+# 43| 2: [Integer] 2
+# 43| 3: [SplatParameter] SplatParameter
+# 44| 6: [InClause] InClause
+# 44| 0: [ReservedWord] in
+# 44| 1: [ArrayPattern] ArrayPattern
+# 44| 0: [Integer] 1
+# 44| 1: [ReservedWord] ,
+# 44| 2: [Integer] 2
+# 44| 3: [ReservedWord] ,
+# 44| 4: [Integer] 3
+# 45| 7: [InClause] InClause
+# 45| 0: [ReservedWord] in
+# 45| 1: [ArrayPattern] ArrayPattern
+# 45| 0: [Integer] 1
+# 45| 1: [ReservedWord] ,
+# 45| 2: [Integer] 2
+# 45| 3: [ReservedWord] ,
+# 45| 4: [Integer] 3
+# 45| 5: [SplatParameter] SplatParameter
+# 46| 8: [InClause] InClause
+# 46| 0: [ReservedWord] in
+# 46| 1: [ArrayPattern] ArrayPattern
+# 46| 0: [Integer] 1
+# 46| 1: [ReservedWord] ,
+# 46| 2: [Integer] 2
+# 46| 3: [ReservedWord] ,
+# 46| 4: [Integer] 3
+# 46| 5: [ReservedWord] ,
+# 46| 6: [SplatParameter] SplatParameter
+# 46| 0: [ReservedWord] *
+# 47| 9: [InClause] InClause
+# 47| 0: [ReservedWord] in
+# 47| 1: [ArrayPattern] ArrayPattern
+# 47| 0: [Integer] 1
+# 47| 1: [ReservedWord] ,
+# 47| 2: [SplatParameter] SplatParameter
+# 47| 0: [ReservedWord] *
+# 47| 1: [Identifier] x
+# 47| 3: [ReservedWord] ,
+# 47| 4: [Integer] 3
+# 48| 10: [InClause] InClause
+# 48| 0: [ReservedWord] in
+# 48| 1: [ArrayPattern] ArrayPattern
+# 48| 0: [SplatParameter] SplatParameter
+# 48| 0: [ReservedWord] *
+# 49| 11: [InClause] InClause
+# 49| 0: [ReservedWord] in
+# 49| 1: [ArrayPattern] ArrayPattern
+# 49| 0: [SplatParameter] SplatParameter
+# 49| 0: [ReservedWord] *
+# 49| 1: [ReservedWord] ,
+# 49| 2: [Integer] 3
+# 49| 3: [ReservedWord] ,
+# 49| 4: [Integer] 4
+# 50| 12: [InClause] InClause
+# 50| 0: [ReservedWord] in
+# 50| 1: [FindPattern] FindPattern
+# 50| 0: [SplatParameter] SplatParameter
+# 50| 0: [ReservedWord] *
+# 50| 1: [ReservedWord] ,
+# 50| 2: [Integer] 3
+# 50| 3: [ReservedWord] ,
+# 50| 4: [SplatParameter] SplatParameter
+# 50| 0: [ReservedWord] *
+# 51| 13: [InClause] InClause
+# 51| 0: [ReservedWord] in
+# 51| 1: [FindPattern] FindPattern
+# 51| 0: [SplatParameter] SplatParameter
+# 51| 0: [ReservedWord] *
+# 51| 1: [Identifier] a
+# 51| 1: [ReservedWord] ,
+# 51| 2: [Integer] 3
+# 51| 3: [ReservedWord] ,
+# 51| 4: [SplatParameter] SplatParameter
+# 51| 0: [ReservedWord] *
+# 51| 1: [Identifier] b
+# 52| 14: [InClause] InClause
+# 52| 0: [ReservedWord] in
+# 52| 1: [HashPattern] HashPattern
+# 52| 0: [KeywordPattern] KeywordPattern
+# 52| 0: [HashKeySymbol] a
+# 52| 1: [ReservedWord] :
+# 53| 15: [InClause] InClause
+# 53| 0: [ReservedWord] in
+# 53| 1: [HashPattern] HashPattern
+# 53| 0: [KeywordPattern] KeywordPattern
+# 53| 0: [HashKeySymbol] a
+# 53| 1: [ReservedWord] :
+# 53| 2: [Integer] 5
+# 54| 16: [InClause] InClause
+# 54| 0: [ReservedWord] in
+# 54| 1: [HashPattern] HashPattern
+# 54| 0: [KeywordPattern] KeywordPattern
+# 54| 0: [HashKeySymbol] a
+# 54| 1: [ReservedWord] :
+# 54| 2: [Integer] 5
+# 54| 1: [ReservedWord] ,
+# 55| 17: [InClause] InClause
+# 55| 0: [ReservedWord] in
+# 55| 1: [HashPattern] HashPattern
+# 55| 0: [KeywordPattern] KeywordPattern
+# 55| 0: [HashKeySymbol] a
+# 55| 1: [ReservedWord] :
+# 55| 2: [Integer] 5
+# 55| 1: [ReservedWord] ,
+# 55| 2: [KeywordPattern] KeywordPattern
+# 55| 0: [HashKeySymbol] b
+# 55| 1: [ReservedWord] :
+# 55| 3: [ReservedWord] ,
+# 55| 4: [HashSplatParameter] HashSplatParameter
+# 55| 0: [ReservedWord] **
+# 56| 18: [InClause] InClause
+# 56| 0: [ReservedWord] in
+# 56| 1: [HashPattern] HashPattern
+# 56| 0: [KeywordPattern] KeywordPattern
+# 56| 0: [HashKeySymbol] a
+# 56| 1: [ReservedWord] :
+# 56| 2: [Integer] 5
+# 56| 1: [ReservedWord] ,
+# 56| 2: [KeywordPattern] KeywordPattern
+# 56| 0: [HashKeySymbol] b
+# 56| 1: [ReservedWord] :
+# 56| 3: [ReservedWord] ,
+# 56| 4: [HashSplatParameter] HashSplatParameter
+# 56| 0: [ReservedWord] **
+# 56| 1: [Identifier] map
+# 57| 19: [InClause] InClause
+# 57| 0: [ReservedWord] in
+# 57| 1: [HashPattern] HashPattern
+# 57| 0: [KeywordPattern] KeywordPattern
+# 57| 0: [HashKeySymbol] a
+# 57| 1: [ReservedWord] :
+# 57| 2: [Integer] 5
+# 57| 1: [ReservedWord] ,
+# 57| 2: [KeywordPattern] KeywordPattern
+# 57| 0: [HashKeySymbol] b
+# 57| 1: [ReservedWord] :
+# 57| 3: [ReservedWord] ,
+# 57| 4: [HashSplatNil] **nil
+# 57| 0: [ReservedWord] **
+# 57| 1: [ReservedWord] nil
+# 58| 20: [InClause] InClause
+# 58| 0: [ReservedWord] in
+# 58| 1: [HashPattern] HashPattern
+# 58| 0: [HashSplatNil] **nil
+# 58| 0: [ReservedWord] **
+# 58| 1: [ReservedWord] nil
+# 59| 21: [InClause] InClause
+# 59| 0: [ReservedWord] in
+# 59| 1: [ArrayPattern] ArrayPattern
+# 59| 0: [ReservedWord] [
+# 59| 1: [Integer] 5
+# 59| 2: [ReservedWord] ]
+# 60| 22: [InClause] InClause
+# 60| 0: [ReservedWord] in
+# 60| 1: [ArrayPattern] ArrayPattern
+# 60| 0: [ReservedWord] [
+# 60| 1: [Integer] 5
+# 60| 2: [SplatParameter] SplatParameter
+# 60| 3: [ReservedWord] ]
+# 61| 23: [InClause] InClause
+# 61| 0: [ReservedWord] in
+# 61| 1: [ArrayPattern] ArrayPattern
+# 61| 0: [ReservedWord] [
+# 61| 1: [Integer] 1
+# 61| 2: [ReservedWord] ,
+# 61| 3: [Integer] 2
+# 61| 4: [ReservedWord] ]
+# 62| 24: [InClause] InClause
+# 62| 0: [ReservedWord] in
+# 62| 1: [ArrayPattern] ArrayPattern
+# 62| 0: [ReservedWord] [
+# 62| 1: [Integer] 1
+# 62| 2: [ReservedWord] ,
+# 62| 3: [Integer] 2
+# 62| 4: [SplatParameter] SplatParameter
+# 62| 5: [ReservedWord] ]
+# 63| 25: [InClause] InClause
+# 63| 0: [ReservedWord] in
+# 63| 1: [ArrayPattern] ArrayPattern
+# 63| 0: [ReservedWord] [
+# 63| 1: [Integer] 1
+# 63| 2: [ReservedWord] ,
+# 63| 3: [Integer] 2
+# 63| 4: [ReservedWord] ,
+# 63| 5: [Integer] 3
+# 63| 6: [ReservedWord] ]
+# 64| 26: [InClause] InClause
+# 64| 0: [ReservedWord] in
+# 64| 1: [ArrayPattern] ArrayPattern
+# 64| 0: [ReservedWord] [
+# 64| 1: [Integer] 1
+# 64| 2: [ReservedWord] ,
+# 64| 3: [Integer] 2
+# 64| 4: [ReservedWord] ,
+# 64| 5: [Integer] 3
+# 64| 6: [SplatParameter] SplatParameter
+# 64| 7: [ReservedWord] ]
+# 65| 27: [InClause] InClause
+# 65| 0: [ReservedWord] in
+# 65| 1: [ArrayPattern] ArrayPattern
+# 65| 0: [ReservedWord] [
+# 65| 1: [Integer] 1
+# 65| 2: [ReservedWord] ,
+# 65| 3: [Integer] 2
+# 65| 4: [ReservedWord] ,
+# 65| 5: [Integer] 3
+# 65| 6: [ReservedWord] ,
+# 65| 7: [SplatParameter] SplatParameter
+# 65| 0: [ReservedWord] *
+# 65| 8: [ReservedWord] ]
+# 66| 28: [InClause] InClause
+# 66| 0: [ReservedWord] in
+# 66| 1: [ArrayPattern] ArrayPattern
+# 66| 0: [ReservedWord] [
+# 66| 1: [Integer] 1
+# 66| 2: [ReservedWord] ,
+# 66| 3: [SplatParameter] SplatParameter
+# 66| 0: [ReservedWord] *
+# 66| 1: [Identifier] x
+# 66| 4: [ReservedWord] ,
+# 66| 5: [Integer] 3
+# 66| 6: [ReservedWord] ]
+# 67| 29: [InClause] InClause
+# 67| 0: [ReservedWord] in
+# 67| 1: [ArrayPattern] ArrayPattern
+# 67| 0: [ReservedWord] [
+# 67| 1: [SplatParameter] SplatParameter
+# 67| 0: [ReservedWord] *
+# 67| 2: [ReservedWord] ]
+# 68| 30: [InClause] InClause
+# 68| 0: [ReservedWord] in
+# 68| 1: [ArrayPattern] ArrayPattern
+# 68| 0: [ReservedWord] [
+# 68| 1: [SplatParameter] SplatParameter
+# 68| 0: [ReservedWord] *
+# 68| 1: [Identifier] x
+# 68| 2: [ReservedWord] ,
+# 68| 3: [Integer] 3
+# 68| 4: [ReservedWord] ,
+# 68| 5: [Integer] 4
+# 68| 6: [ReservedWord] ]
+# 69| 31: [InClause] InClause
+# 69| 0: [ReservedWord] in
+# 69| 1: [FindPattern] FindPattern
+# 69| 0: [ReservedWord] [
+# 69| 1: [SplatParameter] SplatParameter
+# 69| 0: [ReservedWord] *
+# 69| 2: [ReservedWord] ,
+# 69| 3: [Integer] 3
+# 69| 4: [ReservedWord] ,
+# 69| 5: [SplatParameter] SplatParameter
+# 69| 0: [ReservedWord] *
+# 69| 6: [ReservedWord] ]
+# 70| 32: [InClause] InClause
+# 70| 0: [ReservedWord] in
+# 70| 1: [FindPattern] FindPattern
+# 70| 0: [ReservedWord] [
+# 70| 1: [SplatParameter] SplatParameter
+# 70| 0: [ReservedWord] *
+# 70| 1: [Identifier] a
+# 70| 2: [ReservedWord] ,
+# 70| 3: [Integer] 3
+# 70| 4: [ReservedWord] ,
+# 70| 5: [SplatParameter] SplatParameter
+# 70| 0: [ReservedWord] *
+# 70| 1: [Identifier] b
+# 70| 6: [ReservedWord] ]
+# 71| 33: [InClause] InClause
+# 71| 0: [ReservedWord] in
+# 71| 1: [HashPattern] HashPattern
+# 71| 0: [ReservedWord] {
+# 71| 1: [KeywordPattern] KeywordPattern
+# 71| 0: [HashKeySymbol] a
+# 71| 1: [ReservedWord] :
+# 71| 2: [ReservedWord] }
+# 72| 34: [InClause] InClause
+# 72| 0: [ReservedWord] in
+# 72| 1: [HashPattern] HashPattern
+# 72| 0: [ReservedWord] {
+# 72| 1: [KeywordPattern] KeywordPattern
+# 72| 0: [HashKeySymbol] a
+# 72| 1: [ReservedWord] :
+# 72| 2: [Integer] 5
+# 72| 2: [ReservedWord] }
+# 73| 35: [InClause] InClause
+# 73| 0: [ReservedWord] in
+# 73| 1: [HashPattern] HashPattern
+# 73| 0: [ReservedWord] {
+# 73| 1: [KeywordPattern] KeywordPattern
+# 73| 0: [HashKeySymbol] a
+# 73| 1: [ReservedWord] :
+# 73| 2: [Integer] 5
+# 73| 2: [ReservedWord] ,
+# 73| 3: [ReservedWord] }
+# 74| 36: [InClause] InClause
+# 74| 0: [ReservedWord] in
+# 74| 1: [HashPattern] HashPattern
+# 74| 0: [ReservedWord] {
+# 74| 1: [KeywordPattern] KeywordPattern
+# 74| 0: [HashKeySymbol] a
+# 74| 1: [ReservedWord] :
+# 74| 2: [Integer] 5
+# 74| 2: [ReservedWord] ,
+# 74| 3: [KeywordPattern] KeywordPattern
+# 74| 0: [HashKeySymbol] b
+# 74| 1: [ReservedWord] :
+# 74| 4: [ReservedWord] ,
+# 74| 5: [HashSplatParameter] HashSplatParameter
+# 74| 0: [ReservedWord] **
+# 74| 6: [ReservedWord] }
+# 75| 37: [InClause] InClause
+# 75| 0: [ReservedWord] in
+# 75| 1: [HashPattern] HashPattern
+# 75| 0: [ReservedWord] {
+# 75| 1: [KeywordPattern] KeywordPattern
+# 75| 0: [HashKeySymbol] a
+# 75| 1: [ReservedWord] :
+# 75| 2: [Integer] 5
+# 75| 2: [ReservedWord] ,
+# 75| 3: [KeywordPattern] KeywordPattern
+# 75| 0: [HashKeySymbol] b
+# 75| 1: [ReservedWord] :
+# 75| 4: [ReservedWord] ,
+# 75| 5: [HashSplatParameter] HashSplatParameter
+# 75| 0: [ReservedWord] **
+# 75| 1: [Identifier] map
+# 75| 6: [ReservedWord] }
+# 76| 38: [InClause] InClause
+# 76| 0: [ReservedWord] in
+# 76| 1: [HashPattern] HashPattern
+# 76| 0: [ReservedWord] {
+# 76| 1: [KeywordPattern] KeywordPattern
+# 76| 0: [HashKeySymbol] a
+# 76| 1: [ReservedWord] :
+# 76| 2: [Integer] 5
+# 76| 2: [ReservedWord] ,
+# 76| 3: [KeywordPattern] KeywordPattern
+# 76| 0: [HashKeySymbol] b
+# 76| 1: [ReservedWord] :
+# 76| 4: [ReservedWord] ,
+# 76| 5: [HashSplatNil] **nil
+# 76| 0: [ReservedWord] **
+# 76| 1: [ReservedWord] nil
+# 76| 6: [ReservedWord] }
+# 77| 39: [InClause] InClause
+# 77| 0: [ReservedWord] in
+# 77| 1: [HashPattern] HashPattern
+# 77| 0: [ReservedWord] {
+# 77| 1: [HashSplatNil] **nil
+# 77| 0: [ReservedWord] **
+# 77| 1: [ReservedWord] nil
+# 77| 2: [ReservedWord] }
+# 78| 40: [InClause] InClause
+# 78| 0: [ReservedWord] in
+# 78| 1: [HashPattern] HashPattern
+# 78| 0: [ReservedWord] {
+# 78| 1: [ReservedWord] }
+# 79| 41: [InClause] InClause
+# 79| 0: [ReservedWord] in
+# 79| 1: [ArrayPattern] ArrayPattern
+# 79| 0: [ReservedWord] [
+# 79| 1: [ReservedWord] ]
+# 80| 42: [ReservedWord] end
+# 84| 9: [Assignment] Assignment
+# 84| 0: [Identifier] foo
+# 84| 1: [ReservedWord] =
+# 84| 2: [Integer] 42
+# 86| 10: [CaseMatch] CaseMatch
+# 86| 0: [ReservedWord] case
+# 86| 1: [Identifier] expr
+# 87| 2: [InClause] InClause
+# 87| 0: [ReservedWord] in
+# 87| 1: [Integer] 5
+# 88| 3: [InClause] InClause
+# 88| 0: [ReservedWord] in
+# 88| 1: [VariableReferencePattern] VariableReferencePattern
+# 88| 0: [ReservedWord] ^
+# 88| 1: [Identifier] foo
+# 89| 4: [InClause] InClause
+# 89| 0: [ReservedWord] in
+# 89| 1: [String] String
+# 89| 0: [ReservedWord] "
+# 89| 1: [StringContent] string
+# 89| 2: [ReservedWord] "
+# 90| 5: [InClause] InClause
+# 90| 0: [ReservedWord] in
+# 90| 1: [StringArray] StringArray
+# 90| 0: [ReservedWord] %w(
+# 90| 1: [BareString] BareString
+# 90| 0: [StringContent] foo
+# 90| 2: [BareString] BareString
+# 90| 0: [StringContent] bar
+# 90| 3: [ReservedWord] )
+# 91| 6: [InClause] InClause
+# 91| 0: [ReservedWord] in
+# 91| 1: [SymbolArray] SymbolArray
+# 91| 0: [ReservedWord] %i(
+# 91| 1: [BareSymbol] BareSymbol
+# 91| 0: [StringContent] foo
+# 91| 2: [BareSymbol] BareSymbol
+# 91| 0: [StringContent] bar
+# 91| 3: [ReservedWord] )
+# 92| 7: [InClause] InClause
+# 92| 0: [ReservedWord] in
+# 92| 1: [Regex] Regex
+# 92| 0: [ReservedWord] /
+# 92| 1: [StringContent] .*abc[0-9]
+# 92| 2: [ReservedWord] /
+# 93| 8: [InClause] InClause
+# 93| 0: [ReservedWord] in
+# 93| 1: [Range] Range
+# 93| 0: [Integer] 5
+# 93| 1: [ReservedWord] ..
+# 93| 2: [Integer] 10
+# 94| 9: [InClause] InClause
+# 94| 0: [ReservedWord] in
+# 94| 1: [Range] Range
+# 94| 0: [ReservedWord] ..
+# 94| 1: [Integer] 10
+# 95| 10: [InClause] InClause
+# 95| 0: [ReservedWord] in
+# 95| 1: [Range] Range
+# 95| 0: [Integer] 5
+# 95| 1: [ReservedWord] ..
+# 96| 11: [InClause] InClause
+# 96| 0: [ReservedWord] in
+# 96| 1: [AsPattern] AsPattern
+# 96| 0: [Integer] 5
+# 96| 1: [ReservedWord] =>
+# 96| 2: [Identifier] x
+# 97| 12: [InClause] InClause
+# 97| 0: [ReservedWord] in
+# 97| 1: [Constant] Foo
+# 98| 13: [InClause] InClause
+# 98| 0: [ReservedWord] in
+# 98| 1: [ScopeResolution] ScopeResolution
+# 98| 0: [Constant] Foo
+# 98| 1: [ReservedWord] ::
+# 98| 2: [Constant] Bar
+# 99| 14: [InClause] InClause
+# 99| 0: [ReservedWord] in
+# 99| 1: [ScopeResolution] ScopeResolution
+# 99| 0: [ScopeResolution] ScopeResolution
+# 99| 0: [ReservedWord] ::
+# 99| 1: [Constant] Foo
+# 99| 1: [ReservedWord] ::
+# 99| 2: [Constant] Bar
+# 100| 15: [InClause] InClause
+# 100| 0: [ReservedWord] in
+# 100| 1: [AlternativePattern] AlternativePattern
+# 100| 0: [Nil] nil
+# 100| 1: [ReservedWord] |
+# 100| 2: [Self] self
+# 100| 3: [ReservedWord] |
+# 100| 4: [True] true
+# 100| 5: [ReservedWord] |
+# 100| 6: [False] false
+# 100| 7: [ReservedWord] |
+# 100| 8: [Line] __LINE__
+# 100| 9: [ReservedWord] |
+# 100| 10: [File] __FILE__
+# 100| 11: [ReservedWord] |
+# 100| 12: [Encoding] __ENCODING__
+# 101| 16: [InClause] InClause
+# 101| 0: [ReservedWord] in
+# 101| 1: [Lambda] Lambda
+# 101| 0: [ReservedWord] ->
+# 101| 1: [LambdaParameters] LambdaParameters
+# 101| 0: [Identifier] x
+# 101| 2: [Block] Block
+# 101| 0: [ReservedWord] {
+# 101| 1: [Binary] Binary
+# 101| 0: [Identifier] x
+# 101| 1: [ReservedWord] ==
+# 101| 2: [Integer] 10
+# 101| 2: [ReservedWord] }
+# 102| 17: [InClause] InClause
+# 102| 0: [ReservedWord] in
+# 102| 1: [SimpleSymbol] :foo
+# 103| 18: [InClause] InClause
+# 103| 0: [ReservedWord] in
+# 103| 1: [DelimitedSymbol] DelimitedSymbol
+# 103| 0: [ReservedWord] :"
+# 103| 1: [StringContent] foo bar
+# 103| 2: [ReservedWord] "
+# 104| 19: [InClause] InClause
+# 104| 0: [ReservedWord] in
+# 104| 1: [AlternativePattern] AlternativePattern
+# 104| 0: [Unary] Unary
+# 104| 0: [ReservedWord] -
+# 104| 1: [Integer] 5
+# 104| 1: [ReservedWord] |
+# 104| 2: [Unary] Unary
+# 104| 0: [ReservedWord] +
+# 104| 1: [Integer] 10
+# 105| 20: [InClause] InClause
+# 105| 0: [ReservedWord] in
+# 105| 1: [ParenthesizedPattern] ParenthesizedPattern
+# 105| 0: [ReservedWord] (
+# 105| 1: [Range] Range
+# 105| 0: [Integer] 1
+# 105| 1: [ReservedWord] ..
+# 105| 2: [ReservedWord] )
+# 106| 21: [InClause] InClause
+# 106| 0: [ReservedWord] in
+# 106| 1: [ParenthesizedPattern] ParenthesizedPattern
+# 106| 0: [ReservedWord] (
+# 106| 1: [AlternativePattern] AlternativePattern
+# 106| 0: [Integer] 0
+# 106| 1: [ReservedWord] |
+# 106| 2: [String] String
+# 106| 0: [ReservedWord] "
+# 106| 1: [ReservedWord] "
+# 106| 3: [ReservedWord] |
+# 106| 4: [ArrayPattern] ArrayPattern
+# 106| 0: [ReservedWord] [
+# 106| 1: [ReservedWord] ]
+# 106| 5: [ReservedWord] |
+# 106| 6: [HashPattern] HashPattern
+# 106| 0: [ReservedWord] {
+# 106| 1: [ReservedWord] }
+# 106| 2: [ReservedWord] )
+# 107| 22: [InClause] InClause
+# 107| 0: [ReservedWord] in
+# 107| 1: [Identifier] var
+# 108| 23: [ReservedWord] end
+# 110| 11: [CaseMatch] CaseMatch
+# 110| 0: [ReservedWord] case
+# 110| 1: [Identifier] expr
+# 111| 2: [InClause] InClause
+# 111| 0: [ReservedWord] in
+# 111| 1: [AlternativePattern] AlternativePattern
+# 111| 0: [Integer] 5
+# 111| 1: [ReservedWord] |
+# 111| 2: [VariableReferencePattern] VariableReferencePattern
+# 111| 0: [ReservedWord] ^
+# 111| 1: [Identifier] foo
+# 111| 3: [ReservedWord] |
+# 111| 4: [String] String
+# 111| 0: [ReservedWord] "
+# 111| 1: [StringContent] string
+# 111| 2: [ReservedWord] "
+# 111| 5: [ReservedWord] |
+# 111| 6: [Identifier] var
+# 112| 3: [ReservedWord] end
+# 116| 12: [CaseMatch] CaseMatch
+# 116| 0: [ReservedWord] case
+# 116| 1: [Identifier] expr
+# 117| 2: [InClause] InClause
+# 117| 0: [ReservedWord] in
+# 117| 1: [ArrayPattern] ArrayPattern
+# 117| 0: [ReservedWord] [
+# 117| 1: [ReservedWord] ]
+# 117| 2: [ReservedWord] ;
+# 118| 3: [InClause] InClause
+# 118| 0: [ReservedWord] in
+# 118| 1: [ArrayPattern] ArrayPattern
+# 118| 0: [ReservedWord] [
+# 118| 1: [Identifier] x
+# 118| 2: [ReservedWord] ]
+# 118| 2: [ReservedWord] ;
+# 119| 4: [InClause] InClause
+# 119| 0: [ReservedWord] in
+# 119| 1: [ArrayPattern] ArrayPattern
+# 119| 0: [ReservedWord] [
+# 119| 1: [Identifier] x
+# 119| 2: [SplatParameter] SplatParameter
+# 119| 3: [ReservedWord] ]
+# 119| 2: [ReservedWord] ;
+# 120| 5: [InClause] InClause
+# 120| 0: [ReservedWord] in
+# 120| 1: [ArrayPattern] ArrayPattern
+# 120| 0: [ScopeResolution] ScopeResolution
+# 120| 0: [Constant] Foo
+# 120| 1: [ReservedWord] ::
+# 120| 2: [Constant] Bar
+# 120| 1: [ReservedWord] [
+# 120| 2: [ReservedWord] ]
+# 120| 2: [ReservedWord] ;
+# 121| 6: [InClause] InClause
+# 121| 0: [ReservedWord] in
+# 121| 1: [ArrayPattern] ArrayPattern
+# 121| 0: [Constant] Foo
+# 121| 1: [ReservedWord] (
+# 121| 2: [ReservedWord] )
+# 121| 2: [ReservedWord] ;
+# 122| 7: [InClause] InClause
+# 122| 0: [ReservedWord] in
+# 122| 1: [ArrayPattern] ArrayPattern
+# 122| 0: [Constant] Bar
+# 122| 1: [ReservedWord] (
+# 122| 2: [SplatParameter] SplatParameter
+# 122| 0: [ReservedWord] *
+# 122| 3: [ReservedWord] )
+# 122| 2: [ReservedWord] ;
+# 123| 8: [InClause] InClause
+# 123| 0: [ReservedWord] in
+# 123| 1: [ArrayPattern] ArrayPattern
+# 123| 0: [Constant] Bar
+# 123| 1: [ReservedWord] (
+# 123| 2: [Identifier] a
+# 123| 3: [ReservedWord] ,
+# 123| 4: [Identifier] b
+# 123| 5: [ReservedWord] ,
+# 123| 6: [SplatParameter] SplatParameter
+# 123| 0: [ReservedWord] *
+# 123| 1: [Identifier] c
+# 123| 7: [ReservedWord] ,
+# 123| 8: [Identifier] d
+# 123| 9: [ReservedWord] ,
+# 123| 10: [Identifier] e
+# 123| 11: [ReservedWord] )
+# 123| 2: [ReservedWord] ;
+# 124| 9: [ReservedWord] end
+# 128| 13: [CaseMatch] CaseMatch
+# 128| 0: [ReservedWord] case
+# 128| 1: [Identifier] expr
+# 129| 2: [InClause] InClause
+# 129| 0: [ReservedWord] in
+# 129| 1: [FindPattern] FindPattern
+# 129| 0: [ReservedWord] [
+# 129| 1: [SplatParameter] SplatParameter
+# 129| 0: [ReservedWord] *
+# 129| 2: [ReservedWord] ,
+# 129| 3: [Identifier] x
+# 129| 4: [ReservedWord] ,
+# 129| 5: [SplatParameter] SplatParameter
+# 129| 0: [ReservedWord] *
+# 129| 6: [ReservedWord] ]
+# 129| 2: [ReservedWord] ;
+# 130| 3: [InClause] InClause
+# 130| 0: [ReservedWord] in
+# 130| 1: [FindPattern] FindPattern
+# 130| 0: [ReservedWord] [
+# 130| 1: [SplatParameter] SplatParameter
+# 130| 0: [ReservedWord] *
+# 130| 1: [Identifier] x
+# 130| 2: [ReservedWord] ,
+# 130| 3: [Integer] 1
+# 130| 4: [ReservedWord] ,
+# 130| 5: [Integer] 2
+# 130| 6: [ReservedWord] ,
+# 130| 7: [SplatParameter] SplatParameter
+# 130| 0: [ReservedWord] *
+# 130| 1: [Identifier] y
+# 130| 8: [ReservedWord] ]
+# 130| 2: [ReservedWord] ;
+# 131| 4: [InClause] InClause
+# 131| 0: [ReservedWord] in
+# 131| 1: [FindPattern] FindPattern
+# 131| 0: [ScopeResolution] ScopeResolution
+# 131| 0: [Constant] Foo
+# 131| 1: [ReservedWord] ::
+# 131| 2: [Constant] Bar
+# 131| 1: [ReservedWord] [
+# 131| 2: [SplatParameter] SplatParameter
+# 131| 0: [ReservedWord] *
+# 131| 3: [ReservedWord] ,
+# 131| 4: [Integer] 1
+# 131| 5: [ReservedWord] ,
+# 131| 6: [SplatParameter] SplatParameter
+# 131| 0: [ReservedWord] *
+# 131| 7: [ReservedWord] ]
+# 131| 2: [ReservedWord] ;
+# 132| 5: [InClause] InClause
+# 132| 0: [ReservedWord] in
+# 132| 1: [FindPattern] FindPattern
+# 132| 0: [Constant] Foo
+# 132| 1: [ReservedWord] (
+# 132| 2: [SplatParameter] SplatParameter
+# 132| 0: [ReservedWord] *
+# 132| 3: [ReservedWord] ,
+# 132| 4: [Constant] Bar
+# 132| 5: [ReservedWord] ,
+# 132| 6: [SplatParameter] SplatParameter
+# 132| 0: [ReservedWord] *
+# 132| 7: [ReservedWord] )
+# 132| 2: [ReservedWord] ;
+# 133| 6: [ReservedWord] end
+# 137| 14: [CaseMatch] CaseMatch
+# 137| 0: [ReservedWord] case
+# 137| 1: [Identifier] expr
+# 138| 2: [InClause] InClause
+# 138| 0: [ReservedWord] in
+# 138| 1: [HashPattern] HashPattern
+# 138| 0: [ReservedWord] {
+# 138| 1: [ReservedWord] }
+# 138| 2: [ReservedWord] ;
+# 139| 3: [InClause] InClause
+# 139| 0: [ReservedWord] in
+# 139| 1: [HashPattern] HashPattern
+# 139| 0: [ReservedWord] {
+# 139| 1: [KeywordPattern] KeywordPattern
+# 139| 0: [HashKeySymbol] x
+# 139| 1: [ReservedWord] :
+# 139| 2: [ReservedWord] }
+# 139| 2: [ReservedWord] ;
+# 140| 4: [InClause] InClause
+# 140| 0: [ReservedWord] in
+# 140| 1: [HashPattern] HashPattern
+# 140| 0: [ScopeResolution] ScopeResolution
+# 140| 0: [Constant] Foo
+# 140| 1: [ReservedWord] ::
+# 140| 2: [Constant] Bar
+# 140| 1: [ReservedWord] [
+# 140| 2: [KeywordPattern] KeywordPattern
+# 140| 0: [HashKeySymbol] x
+# 140| 1: [ReservedWord] :
+# 140| 2: [Integer] 1
+# 140| 3: [ReservedWord] ]
+# 140| 2: [ReservedWord] ;
+# 141| 5: [InClause] InClause
+# 141| 0: [ReservedWord] in
+# 141| 1: [HashPattern] HashPattern
+# 141| 0: [ScopeResolution] ScopeResolution
+# 141| 0: [Constant] Foo
+# 141| 1: [ReservedWord] ::
+# 141| 2: [Constant] Bar
+# 141| 1: [ReservedWord] [
+# 141| 2: [KeywordPattern] KeywordPattern
+# 141| 0: [HashKeySymbol] x
+# 141| 1: [ReservedWord] :
+# 141| 2: [Integer] 1
+# 141| 3: [ReservedWord] ,
+# 141| 4: [KeywordPattern] KeywordPattern
+# 141| 0: [HashKeySymbol] a
+# 141| 1: [ReservedWord] :
+# 141| 5: [ReservedWord] ,
+# 141| 6: [HashSplatParameter] HashSplatParameter
+# 141| 0: [ReservedWord] **
+# 141| 1: [Identifier] rest
+# 141| 7: [ReservedWord] ]
+# 141| 2: [ReservedWord] ;
+# 142| 6: [InClause] InClause
+# 142| 0: [ReservedWord] in
+# 142| 1: [HashPattern] HashPattern
+# 142| 0: [Constant] Foo
+# 142| 1: [ReservedWord] (
+# 142| 2: [KeywordPattern] KeywordPattern
+# 142| 0: [HashKeySymbol] y
+# 142| 1: [ReservedWord] :
+# 142| 3: [ReservedWord] )
+# 142| 2: [ReservedWord] ;
+# 143| 7: [InClause] InClause
+# 143| 0: [ReservedWord] in
+# 143| 1: [HashPattern] HashPattern
+# 143| 0: [Constant] Bar
+# 143| 1: [ReservedWord] (
+# 143| 2: [HashSplatParameter] HashSplatParameter
+# 143| 0: [ReservedWord] **
+# 143| 3: [ReservedWord] )
+# 143| 2: [ReservedWord] ;
+# 144| 8: [InClause] InClause
+# 144| 0: [ReservedWord] in
+# 144| 1: [HashPattern] HashPattern
+# 144| 0: [Constant] Bar
+# 144| 1: [ReservedWord] (
+# 144| 2: [KeywordPattern] KeywordPattern
+# 144| 0: [HashKeySymbol] a
+# 144| 1: [ReservedWord] :
+# 144| 2: [Integer] 1
+# 144| 3: [ReservedWord] ,
+# 144| 4: [HashSplatNil] **nil
+# 144| 0: [ReservedWord] **
+# 144| 1: [ReservedWord] nil
+# 144| 5: [ReservedWord] )
+# 144| 2: [ReservedWord] ;
+# 145| 9: [ReservedWord] end
+# 147| 15: [CaseMatch] CaseMatch
+# 147| 0: [ReservedWord] case
+# 147| 1: [Identifier] expr
+# 148| 2: [InClause] InClause
+# 148| 0: [ReservedWord] in
+# 148| 1: [VariableReferencePattern] VariableReferencePattern
+# 148| 0: [ReservedWord] ^
+# 148| 1: [Identifier] foo
+# 148| 2: [ReservedWord] ;
+# 149| 3: [InClause] InClause
+# 149| 0: [ReservedWord] in
+# 149| 1: [VariableReferencePattern] VariableReferencePattern
+# 149| 0: [ReservedWord] ^
+# 149| 1: [GlobalVariable] $foo
+# 149| 2: [ReservedWord] ;
+# 150| 4: [InClause] InClause
+# 150| 0: [ReservedWord] in
+# 150| 1: [VariableReferencePattern] VariableReferencePattern
+# 150| 0: [ReservedWord] ^
+# 150| 1: [InstanceVariable] @foo
+# 150| 2: [ReservedWord] ;
+# 151| 5: [InClause] InClause
+# 151| 0: [ReservedWord] in
+# 151| 1: [VariableReferencePattern] VariableReferencePattern
+# 151| 0: [ReservedWord] ^
+# 151| 1: [ClassVariable] @@foo
+# 151| 2: [ReservedWord] ;
+# 152| 6: [ReservedWord] end
+# 154| 16: [CaseMatch] CaseMatch
+# 154| 0: [ReservedWord] case
+# 154| 1: [Identifier] expr
+# 155| 2: [InClause] InClause
+# 155| 0: [ReservedWord] in
+# 155| 1: [ExpressionReferencePattern] ExpressionReferencePattern
+# 155| 0: [ReservedWord] ^
+# 155| 1: [ReservedWord] (
+# 155| 2: [Identifier] foo
+# 155| 3: [ReservedWord] )
+# 155| 2: [ReservedWord] ;
+# 156| 3: [InClause] InClause
+# 156| 0: [ReservedWord] in
+# 156| 1: [ExpressionReferencePattern] ExpressionReferencePattern
+# 156| 0: [ReservedWord] ^
+# 156| 1: [ReservedWord] (
+# 156| 2: [InstanceVariable] @foo
+# 156| 3: [ReservedWord] )
+# 156| 2: [ReservedWord] ;
+# 157| 4: [InClause] InClause
+# 157| 0: [ReservedWord] in
+# 157| 1: [ExpressionReferencePattern] ExpressionReferencePattern
+# 157| 0: [ReservedWord] ^
+# 157| 1: [ReservedWord] (
+# 157| 2: [Binary] Binary
+# 157| 0: [Integer] 1
+# 157| 1: [ReservedWord] +
+# 157| 2: [Integer] 1
+# 157| 3: [ReservedWord] )
+# 157| 2: [ReservedWord] ;
+# 158| 5: [ReservedWord] end
+# 1| [Comment] # Define some variables used below
+# 7| [Comment] # A case expr with a value and an else branch
+# 17| [Comment] # A case expr without a value or else branch
+# 24| [Comment] # pattern matching
+# 82| [Comment] # more pattern matching
+# 114| [Comment] # array patterns
+# 126| [Comment] # find patterns
+# 135| [Comment] # hash patterns
+control/conditionals.rb:
+# 1| [Program] Program
+# 2| 0: [Assignment] Assignment
+# 2| 0: [Identifier] a
+# 2| 1: [ReservedWord] =
+# 2| 2: [Integer] 0
+# 3| 1: [Assignment] Assignment
+# 3| 0: [Identifier] b
+# 3| 1: [ReservedWord] =
+# 3| 2: [Integer] 0
+# 4| 2: [Assignment] Assignment
+# 4| 0: [Identifier] c
+# 4| 1: [ReservedWord] =
+# 4| 2: [Integer] 0
+# 5| 3: [Assignment] Assignment
+# 5| 0: [Identifier] d
+# 5| 1: [ReservedWord] =
+# 5| 2: [Integer] 0
+# 6| 4: [Assignment] Assignment
+# 6| 0: [Identifier] e
+# 6| 1: [ReservedWord] =
+# 6| 2: [Integer] 0
+# 7| 5: [Assignment] Assignment
+# 7| 0: [Identifier] f
+# 7| 1: [ReservedWord] =
+# 7| 2: [Integer] 0
+# 10| 6: [If] If
+# 10| 0: [ReservedWord] if
+# 10| 1: [Binary] Binary
+# 10| 0: [Identifier] a
+# 10| 1: [ReservedWord] >
+# 10| 2: [Identifier] b
+# 10| 2: [Then] Then
+# 10| 0: [ReservedWord] then
+# 11| 1: [Identifier] c
+# 12| 3: [ReservedWord] end
+# 15| 7: [If] If
+# 15| 0: [ReservedWord] if
+# 15| 1: [Binary] Binary
+# 15| 0: [Identifier] a
+# 15| 1: [ReservedWord] ==
+# 15| 2: [Identifier] b
+# 15| 2: [Then] Then
+# 16| 0: [Identifier] c
+# 17| 3: [Else] Else
+# 17| 0: [ReservedWord] else
+# 18| 1: [Identifier] d
+# 19| 4: [ReservedWord] end
+# 22| 8: [If] If
+# 22| 0: [ReservedWord] if
+# 22| 1: [Binary] Binary
+# 22| 0: [Identifier] a
+# 22| 1: [ReservedWord] ==
+# 22| 2: [Integer] 0
+# 22| 2: [Then] Then
+# 22| 0: [ReservedWord] then
+# 23| 1: [Identifier] c
+# 24| 3: [Elsif] Elsif
+# 24| 0: [ReservedWord] elsif
+# 24| 1: [Binary] Binary
+# 24| 0: [Identifier] a
+# 24| 1: [ReservedWord] ==
+# 24| 2: [Integer] 1
+# 24| 2: [Then] Then
+# 24| 0: [ReservedWord] then
+# 25| 1: [Identifier] d
+# 26| 3: [Elsif] Elsif
+# 26| 0: [ReservedWord] elsif
+# 26| 1: [Binary] Binary
+# 26| 0: [Identifier] a
+# 26| 1: [ReservedWord] ==
+# 26| 2: [Integer] 2
+# 26| 2: [Then] Then
+# 26| 0: [ReservedWord] then
+# 27| 1: [Identifier] e
+# 28| 3: [Else] Else
+# 28| 0: [ReservedWord] else
+# 29| 1: [Identifier] f
+# 30| 4: [ReservedWord] end
+# 33| 9: [If] If
+# 33| 0: [ReservedWord] if
+# 33| 1: [Binary] Binary
+# 33| 0: [Identifier] a
+# 33| 1: [ReservedWord] ==
+# 33| 2: [Integer] 0
+# 33| 2: [Then] Then
+# 34| 0: [Identifier] b
+# 35| 3: [Elsif] Elsif
+# 35| 0: [ReservedWord] elsif
+# 35| 1: [Binary] Binary
+# 35| 0: [Identifier] a
+# 35| 1: [ReservedWord] ==
+# 35| 2: [Integer] 1
+# 35| 2: [Then] Then
+# 36| 0: [Identifier] c
+# 37| 4: [ReservedWord] end
+# 40| 10: [Unless] Unless
+# 40| 0: [ReservedWord] unless
+# 40| 1: [Binary] Binary
+# 40| 0: [Identifier] a
+# 40| 1: [ReservedWord] >
+# 40| 2: [Identifier] b
+# 40| 2: [Then] Then
+# 40| 0: [ReservedWord] then
+# 41| 1: [Identifier] c
+# 42| 3: [ReservedWord] end
+# 45| 11: [Unless] Unless
+# 45| 0: [ReservedWord] unless
+# 45| 1: [Binary] Binary
+# 45| 0: [Identifier] a
+# 45| 1: [ReservedWord] ==
+# 45| 2: [Identifier] b
+# 45| 2: [Then] Then
+# 46| 0: [Identifier] c
+# 47| 3: [Else] Else
+# 47| 0: [ReservedWord] else
+# 48| 1: [Identifier] d
+# 49| 4: [ReservedWord] end
+# 52| 12: [IfModifier] IfModifier
+# 52| 0: [Assignment] Assignment
+# 52| 0: [Identifier] a
+# 52| 1: [ReservedWord] =
+# 52| 2: [Identifier] b
+# 52| 1: [ReservedWord] if
+# 52| 2: [Binary] Binary
+# 52| 0: [Identifier] c
+# 52| 1: [ReservedWord] >
+# 52| 2: [Identifier] d
+# 55| 13: [UnlessModifier] UnlessModifier
+# 55| 0: [Assignment] Assignment
+# 55| 0: [Identifier] a
+# 55| 1: [ReservedWord] =
+# 55| 2: [Identifier] b
+# 55| 1: [ReservedWord] unless
+# 55| 2: [Binary] Binary
+# 55| 0: [Identifier] c
+# 55| 1: [ReservedWord] <
+# 55| 2: [Identifier] d
+# 58| 14: [Assignment] Assignment
+# 58| 0: [Identifier] a
+# 58| 1: [ReservedWord] =
+# 58| 2: [Conditional] Conditional
+# 58| 0: [Binary] Binary
+# 58| 0: [Identifier] b
+# 58| 1: [ReservedWord] >
+# 58| 2: [Identifier] c
+# 58| 1: [ReservedWord] ?
+# 58| 2: [Binary] Binary
+# 58| 0: [Identifier] d
+# 58| 1: [ReservedWord] +
+# 58| 2: [Integer] 1
+# 58| 3: [ReservedWord] :
+# 58| 4: [Binary] Binary
+# 58| 0: [Identifier] e
+# 58| 1: [ReservedWord] -
+# 58| 2: [Integer] 2
+# 61| 15: [If] If
+# 61| 0: [ReservedWord] if
+# 61| 1: [Binary] Binary
+# 61| 0: [Identifier] a
+# 61| 1: [ReservedWord] >
+# 61| 2: [Identifier] b
+# 61| 2: [Then] Then
+# 61| 0: [ReservedWord] then
+# 62| 1: [Identifier] c
+# 63| 3: [Else] Else
+# 63| 0: [ReservedWord] else
+# 64| 4: [ReservedWord] end
+# 67| 16: [If] If
+# 67| 0: [ReservedWord] if
+# 67| 1: [Binary] Binary
+# 67| 0: [Identifier] a
+# 67| 1: [ReservedWord] >
+# 67| 2: [Identifier] b
+# 67| 2: [Then] Then
+# 67| 0: [ReservedWord] then
+# 68| 3: [Else] Else
+# 68| 0: [ReservedWord] else
+# 69| 1: [Identifier] c
+# 70| 4: [ReservedWord] end
+# 1| [Comment] # Define some variables used below
+# 9| [Comment] # If expr with no else
+# 14| [Comment] # If expr with single else
+# 21| [Comment] # If expr with multiple nested elsif branches
+# 32| [Comment] # If expr with elsif and then no else
+# 39| [Comment] # Unless expr with no else
+# 44| [Comment] # Unless expr with else
+# 51| [Comment] # If-modified expr
+# 54| [Comment] # Unless-modified expr
+# 57| [Comment] # Ternary if expr
+# 60| [Comment] # If expr with empty else (treated as no else)
+# 66| [Comment] # If expr with empty then (treated as no then)
+control/loops.rb:
+# 1| [Program] Program
+# 2| 0: [Assignment] Assignment
+# 2| 0: [Identifier] foo
+# 2| 1: [ReservedWord] =
+# 2| 2: [Integer] 0
+# 3| 1: [Assignment] Assignment
+# 3| 0: [Identifier] sum
+# 3| 1: [ReservedWord] =
+# 3| 2: [Integer] 0
+# 4| 2: [Assignment] Assignment
+# 4| 0: [Identifier] x
+# 4| 1: [ReservedWord] =
+# 4| 2: [Integer] 0
+# 5| 3: [Assignment] Assignment
+# 5| 0: [Identifier] y
+# 5| 1: [ReservedWord] =
+# 5| 2: [Integer] 0
+# 6| 4: [Assignment] Assignment
+# 6| 0: [Identifier] z
+# 6| 1: [ReservedWord] =
+# 6| 2: [Integer] 0
+# 9| 5: [For] For
+# 9| 0: [ReservedWord] for
+# 9| 1: [Identifier] n
+# 9| 2: [In] In
+# 9| 0: [ReservedWord] in
+# 9| 1: [Range] Range
+# 9| 0: [Integer] 1
+# 9| 1: [ReservedWord] ..
+# 9| 2: [Integer] 10
+# 9| 3: [Do] Do
+# 10| 0: [OperatorAssignment] OperatorAssignment
+# 10| 0: [Identifier] sum
+# 10| 1: [ReservedWord] +=
+# 10| 2: [Identifier] n
+# 11| 1: [Assignment] Assignment
+# 11| 0: [Identifier] foo
+# 11| 1: [ReservedWord] =
+# 11| 2: [Identifier] n
+# 12| 2: [ReservedWord] end
+# 16| 6: [For] For
+# 16| 0: [ReservedWord] for
+# 16| 1: [Identifier] n
+# 16| 2: [In] In
+# 16| 0: [ReservedWord] in
+# 16| 1: [Range] Range
+# 16| 0: [Integer] 1
+# 16| 1: [ReservedWord] ..
+# 16| 2: [Integer] 10
+# 16| 3: [Do] Do
+# 17| 0: [OperatorAssignment] OperatorAssignment
+# 17| 0: [Identifier] sum
+# 17| 1: [ReservedWord] +=
+# 17| 2: [Identifier] n
+# 18| 1: [OperatorAssignment] OperatorAssignment
+# 18| 0: [Identifier] foo
+# 18| 1: [ReservedWord] -=
+# 18| 2: [Identifier] n
+# 19| 2: [ReservedWord] end
+# 22| 7: [For] For
+# 22| 0: [ReservedWord] for
+# 22| 1: [LeftAssignmentList] LeftAssignmentList
+# 22| 0: [Identifier] key
+# 22| 1: [ReservedWord] ,
+# 22| 2: [Identifier] value
+# 22| 2: [In] In
+# 22| 0: [ReservedWord] in
+# 22| 1: [Hash] Hash
+# 22| 0: [ReservedWord] {
+# 22| 1: [Pair] Pair
+# 22| 0: [HashKeySymbol] foo
+# 22| 1: [ReservedWord] :
+# 22| 2: [Integer] 0
+# 22| 2: [ReservedWord] ,
+# 22| 3: [Pair] Pair
+# 22| 0: [HashKeySymbol] bar
+# 22| 1: [ReservedWord] :
+# 22| 2: [Integer] 1
+# 22| 4: [ReservedWord] }
+# 22| 3: [Do] Do
+# 23| 0: [OperatorAssignment] OperatorAssignment
+# 23| 0: [Identifier] sum
+# 23| 1: [ReservedWord] +=
+# 23| 2: [Identifier] value
+# 24| 1: [OperatorAssignment] OperatorAssignment
+# 24| 0: [Identifier] foo
+# 24| 1: [ReservedWord] *=
+# 24| 2: [Identifier] value
+# 25| 2: [ReservedWord] end
+# 28| 8: [For] For
+# 28| 0: [ReservedWord] for
+# 28| 1: [LeftAssignmentList] LeftAssignmentList
+# 28| 0: [DestructuredLeftAssignment] DestructuredLeftAssignment
+# 28| 0: [ReservedWord] (
+# 28| 1: [Identifier] key
+# 28| 2: [ReservedWord] ,
+# 28| 3: [Identifier] value
+# 28| 4: [ReservedWord] )
+# 28| 2: [In] In
+# 28| 0: [ReservedWord] in
+# 28| 1: [Hash] Hash
+# 28| 0: [ReservedWord] {
+# 28| 1: [Pair] Pair
+# 28| 0: [HashKeySymbol] foo
+# 28| 1: [ReservedWord] :
+# 28| 2: [Integer] 0
+# 28| 2: [ReservedWord] ,
+# 28| 3: [Pair] Pair
+# 28| 0: [HashKeySymbol] bar
+# 28| 1: [ReservedWord] :
+# 28| 2: [Integer] 1
+# 28| 4: [ReservedWord] }
+# 28| 3: [Do] Do
+# 29| 0: [OperatorAssignment] OperatorAssignment
+# 29| 0: [Identifier] sum
+# 29| 1: [ReservedWord] +=
+# 29| 2: [Identifier] value
+# 30| 1: [OperatorAssignment] OperatorAssignment
+# 30| 0: [Identifier] foo
+# 30| 1: [ReservedWord] /=
+# 30| 2: [Identifier] value
+# 31| 2: [Break] Break
+# 31| 0: [ReservedWord] break
+# 32| 3: [ReservedWord] end
+# 35| 9: [While] While
+# 35| 0: [ReservedWord] while
+# 35| 1: [Binary] Binary
+# 35| 0: [Identifier] x
+# 35| 1: [ReservedWord] <
+# 35| 2: [Identifier] y
+# 35| 2: [Do] Do
+# 36| 0: [OperatorAssignment] OperatorAssignment
+# 36| 0: [Identifier] x
+# 36| 1: [ReservedWord] +=
+# 36| 2: [Integer] 1
+# 37| 1: [OperatorAssignment] OperatorAssignment
+# 37| 0: [Identifier] z
+# 37| 1: [ReservedWord] +=
+# 37| 2: [Integer] 1
+# 38| 2: [Next] Next
+# 38| 0: [ReservedWord] next
+# 39| 3: [ReservedWord] end
+# 42| 10: [While] While
+# 42| 0: [ReservedWord] while
+# 42| 1: [Binary] Binary
+# 42| 0: [Identifier] x
+# 42| 1: [ReservedWord] <
+# 42| 2: [Identifier] y
+# 42| 2: [Do] Do
+# 42| 0: [ReservedWord] do
+# 43| 1: [OperatorAssignment] OperatorAssignment
+# 43| 0: [Identifier] x
+# 43| 1: [ReservedWord] +=
+# 43| 2: [Integer] 1
+# 44| 2: [OperatorAssignment] OperatorAssignment
+# 44| 0: [Identifier] z
+# 44| 1: [ReservedWord] +=
+# 44| 2: [Integer] 2
+# 45| 3: [ReservedWord] end
+# 48| 11: [WhileModifier] WhileModifier
+# 48| 0: [OperatorAssignment] OperatorAssignment
+# 48| 0: [Identifier] x
+# 48| 1: [ReservedWord] +=
+# 48| 2: [Integer] 1
+# 48| 1: [ReservedWord] while
+# 48| 2: [Binary] Binary
+# 48| 0: [Identifier] y
+# 48| 1: [ReservedWord] >=
+# 48| 2: [Identifier] x
+# 51| 12: [Until] Until
+# 51| 0: [ReservedWord] until
+# 51| 1: [Binary] Binary
+# 51| 0: [Identifier] x
+# 51| 1: [ReservedWord] ==
+# 51| 2: [Identifier] y
+# 51| 2: [Do] Do
+# 52| 0: [OperatorAssignment] OperatorAssignment
+# 52| 0: [Identifier] x
+# 52| 1: [ReservedWord] +=
+# 52| 2: [Integer] 1
+# 53| 1: [OperatorAssignment] OperatorAssignment
+# 53| 0: [Identifier] z
+# 53| 1: [ReservedWord] -=
+# 53| 2: [Integer] 1
+# 54| 2: [ReservedWord] end
+# 57| 13: [Until] Until
+# 57| 0: [ReservedWord] until
+# 57| 1: [Binary] Binary
+# 57| 0: [Identifier] x
+# 57| 1: [ReservedWord] >
+# 57| 2: [Identifier] y
+# 57| 2: [Do] Do
+# 57| 0: [ReservedWord] do
+# 58| 1: [OperatorAssignment] OperatorAssignment
+# 58| 0: [Identifier] x
+# 58| 1: [ReservedWord] +=
+# 58| 2: [Integer] 1
+# 59| 2: [OperatorAssignment] OperatorAssignment
+# 59| 0: [Identifier] z
+# 59| 1: [ReservedWord] -=
+# 59| 2: [Integer] 4
+# 60| 3: [ReservedWord] end
+# 63| 14: [UntilModifier] UntilModifier
+# 63| 0: [OperatorAssignment] OperatorAssignment
+# 63| 0: [Identifier] x
+# 63| 1: [ReservedWord] -=
+# 63| 2: [Integer] 1
+# 63| 1: [ReservedWord] until
+# 63| 2: [Binary] Binary
+# 63| 0: [Identifier] x
+# 63| 1: [ReservedWord] ==
+# 63| 2: [Integer] 0
+# 66| 15: [While] While
+# 66| 0: [ReservedWord] while
+# 66| 1: [Binary] Binary
+# 66| 0: [Identifier] x
+# 66| 1: [ReservedWord] <
+# 66| 2: [Identifier] y
+# 66| 2: [Do] Do
+# 66| 0: [ReservedWord] do
+# 67| 1: [ReservedWord] end
+# 1| [Comment] # Define some variables used below.
+# 8| [Comment] # For loop with a single variable as the iteration argument
+# 14| [Comment] # For loop with a single variable and a trailing comma as the iteration
+# 15| [Comment] # argument
+# 21| [Comment] # For loop with a tuple pattern as the iteration argument
+# 27| [Comment] # Same, but with parentheses around the pattern
+# 34| [Comment] # While loop
+# 41| [Comment] # While loop with `do` keyword
+# 47| [Comment] # While-modified expression
+# 50| [Comment] # Until loop
+# 56| [Comment] # Until loop with `do` keyword
+# 62| [Comment] # Until-modified expression
+# 65| [Comment] # While loop with empty `do` block
+erb/template.html.erb:
+# 19| [Program] Program
+# 19| 0: [String] String
+# 19| 0: [ReservedWord] "
+# 19| 1: [StringContent] hello world
+# 19| 2: [ReservedWord] "
+# 25| 1: [Assignment] Assignment
+# 25| 0: [Identifier] xs
+# 25| 1: [ReservedWord] =
+# 25| 2: [String] String
+# 25| 0: [ReservedWord] "
+# 25| 1: [ReservedWord] "
+# 27| 2: [For] For
+# 27| 0: [ReservedWord] for
+# 27| 1: [Identifier] x
+# 27| 2: [In] In
+# 27| 0: [ReservedWord] in
+# 27| 1: [Array] Array
+# 27| 0: [ReservedWord] [
+# 27| 1: [String] String
+# 27| 0: [ReservedWord] "
+# 27| 1: [StringContent] foo
+# 27| 2: [ReservedWord] "
+# 27| 2: [ReservedWord] ,
+# 27| 3: [String] String
+# 27| 0: [ReservedWord] "
+# 27| 1: [StringContent] bar
+# 27| 2: [ReservedWord] "
+# 27| 4: [ReservedWord] ,
+# 27| 5: [String] String
+# 27| 0: [ReservedWord] "
+# 27| 1: [StringContent] baz
+# 27| 2: [ReservedWord] "
+# 27| 6: [ReservedWord] ]
+# 27| 3: [Do] Do
+# 27| 0: [ReservedWord] do
+# 28| 1: [OperatorAssignment] OperatorAssignment
+# 28| 0: [Identifier] xs
+# 28| 1: [ReservedWord] +=
+# 28| 2: [Identifier] x
+# 29| 2: [Identifier] xs
+# 31| 3: [ReservedWord] end
+escape_sequences/escapes.rb:
+# 1| [Program] Program
+# 6| 0: [String] String
+# 6| 0: [ReservedWord] '
+# 6| 1: [StringContent] \'
+# 6| 2: [ReservedWord] '
+# 7| 1: [String] String
+# 7| 0: [ReservedWord] '
+# 7| 1: [StringContent] \"
+# 7| 2: [ReservedWord] '
+# 8| 2: [String] String
+# 8| 0: [ReservedWord] '
+# 8| 1: [StringContent] \\
+# 8| 2: [ReservedWord] '
+# 9| 3: [String] String
+# 9| 0: [ReservedWord] '
+# 9| 1: [StringContent] \1
+# 9| 2: [ReservedWord] '
+# 10| 4: [String] String
+# 10| 0: [ReservedWord] '
+# 10| 1: [StringContent] \\1
+# 10| 2: [ReservedWord] '
+# 11| 5: [String] String
+# 11| 0: [ReservedWord] '
+# 11| 1: [StringContent] \141
+# 11| 2: [ReservedWord] '
+# 12| 6: [String] String
+# 12| 0: [ReservedWord] '
+# 12| 1: [StringContent] \n
+# 12| 2: [ReservedWord] '
+# 15| 7: [String] String
+# 15| 0: [ReservedWord] "
+# 15| 1: [EscapeSequence] \'
+# 15| 2: [ReservedWord] "
+# 16| 8: [String] String
+# 16| 0: [ReservedWord] "
+# 16| 1: [EscapeSequence] \"
+# 16| 2: [ReservedWord] "
+# 17| 9: [String] String
+# 17| 0: [ReservedWord] "
+# 17| 1: [EscapeSequence] \\
+# 17| 2: [ReservedWord] "
+# 18| 10: [String] String
+# 18| 0: [ReservedWord] "
+# 18| 1: [EscapeSequence] \1
+# 18| 2: [ReservedWord] "
+# 19| 11: [String] String
+# 19| 0: [ReservedWord] "
+# 19| 1: [EscapeSequence] \\
+# 19| 2: [StringContent] 1
+# 19| 3: [ReservedWord] "
+# 20| 12: [String] String
+# 20| 0: [ReservedWord] "
+# 20| 1: [EscapeSequence] \141
+# 20| 2: [ReservedWord] "
+# 21| 13: [String] String
+# 21| 0: [ReservedWord] "
+# 21| 1: [EscapeSequence] \x6d
+# 21| 2: [ReservedWord] "
+# 22| 14: [String] String
+# 22| 0: [ReservedWord] "
+# 22| 1: [EscapeSequence] \x6E
+# 22| 2: [ReservedWord] "
+# 23| 15: [String] String
+# 23| 0: [ReservedWord] "
+# 23| 1: [EscapeSequence] \X
+# 23| 2: [StringContent] 6d
+# 23| 3: [ReservedWord] "
+# 24| 16: [String] String
+# 24| 0: [ReservedWord] "
+# 24| 1: [EscapeSequence] \X
+# 24| 2: [StringContent] 6E
+# 24| 3: [ReservedWord] "
+# 25| 17: [String] String
+# 25| 0: [ReservedWord] "
+# 25| 1: [EscapeSequence] \u203d
+# 25| 2: [ReservedWord] "
+# 26| 18: [String] String
+# 26| 0: [ReservedWord] "
+# 26| 1: [EscapeSequence] \u{62}
+# 26| 2: [ReservedWord] "
+# 27| 19: [String] String
+# 27| 0: [ReservedWord] "
+# 27| 1: [EscapeSequence] \u{1f60a}
+# 27| 2: [ReservedWord] "
+# 28| 20: [String] String
+# 28| 0: [ReservedWord] "
+# 28| 1: [EscapeSequence] \a
+# 28| 2: [ReservedWord] "
+# 29| 21: [String] String
+# 29| 0: [ReservedWord] "
+# 29| 1: [EscapeSequence] \b
+# 29| 2: [ReservedWord] "
+# 30| 22: [String] String
+# 30| 0: [ReservedWord] "
+# 30| 1: [EscapeSequence] \t
+# 30| 2: [ReservedWord] "
+# 31| 23: [String] String
+# 31| 0: [ReservedWord] "
+# 31| 1: [EscapeSequence] \n
+# 31| 2: [ReservedWord] "
+# 32| 24: [String] String
+# 32| 0: [ReservedWord] "
+# 32| 1: [EscapeSequence] \v
+# 32| 2: [ReservedWord] "
+# 33| 25: [String] String
+# 33| 0: [ReservedWord] "
+# 33| 1: [EscapeSequence] \f
+# 33| 2: [ReservedWord] "
+# 34| 26: [String] String
+# 34| 0: [ReservedWord] "
+# 34| 1: [EscapeSequence] \r
+# 34| 2: [ReservedWord] "
+# 35| 27: [String] String
+# 35| 0: [ReservedWord] "
+# 35| 1: [EscapeSequence] \e
+# 35| 2: [ReservedWord] "
+# 36| 28: [String] String
+# 36| 0: [ReservedWord] "
+# 36| 1: [EscapeSequence] \s
+# 36| 2: [ReservedWord] "
+# 37| 29: [String] String
+# 37| 0: [ReservedWord] "
+# 37| 1: [EscapeSequence] \c
+# 37| 2: [StringContent] ?
+# 37| 3: [ReservedWord] "
+# 38| 30: [String] String
+# 38| 0: [ReservedWord] "
+# 38| 1: [EscapeSequence] \C
+# 38| 2: [StringContent] -?
+# 38| 3: [ReservedWord] "
+# 43| 31: [Assignment] Assignment
+# 43| 0: [Identifier] a
+# 43| 1: [ReservedWord] =
+# 43| 2: [String] String
+# 43| 0: [ReservedWord] "
+# 43| 1: [EscapeSequence] \\
+# 43| 2: [StringContent] .
+# 43| 3: [ReservedWord] "
+# 44| 32: [String] String
+# 44| 0: [ReservedWord] "
+# 44| 1: [Interpolation] Interpolation
+# 44| 0: [ReservedWord] #{
+# 44| 1: [Identifier] a
+# 44| 2: [ReservedWord] }
+# 44| 2: [ReservedWord] "
+# 48| 33: [Regex] Regex
+# 48| 0: [ReservedWord] /
+# 48| 1: [EscapeSequence] \n
+# 48| 2: [ReservedWord] /
+# 49| 34: [Regex] Regex
+# 49| 0: [ReservedWord] /
+# 49| 1: [EscapeSequence] \p
+# 49| 2: [ReservedWord] /
+# 50| 35: [Regex] Regex
+# 50| 0: [ReservedWord] /
+# 50| 1: [EscapeSequence] \u0061
+# 50| 2: [ReservedWord] /
+# 53| 36: [Assignment] Assignment
+# 53| 0: [Identifier] a
+# 53| 1: [ReservedWord] =
+# 53| 2: [String] String
+# 53| 0: [ReservedWord] "
+# 53| 1: [EscapeSequence] \\
+# 53| 2: [StringContent] .
+# 53| 3: [ReservedWord] "
+# 54| 37: [Assignment] Assignment
+# 54| 0: [Identifier] b
+# 54| 1: [ReservedWord] =
+# 54| 2: [Regex] Regex
+# 54| 0: [ReservedWord] /
+# 54| 1: [EscapeSequence] \.
+# 54| 2: [ReservedWord] /
+# 55| 38: [Regex] Regex
+# 55| 0: [ReservedWord] /
+# 55| 1: [Interpolation] Interpolation
+# 55| 0: [ReservedWord] #{
+# 55| 1: [Identifier] a
+# 55| 2: [ReservedWord] }
+# 55| 2: [Interpolation] Interpolation
+# 55| 0: [ReservedWord] #{
+# 55| 1: [Identifier] b
+# 55| 2: [ReservedWord] }
+# 55| 3: [ReservedWord] /
+# 58| 39: [StringArray] StringArray
+# 58| 0: [ReservedWord] %w[
+# 58| 1: [BareString] BareString
+# 58| 0: [StringContent] foo
+# 58| 1: [EscapeSequence] \n
+# 58| 2: [BareString] BareString
+# 58| 0: [StringContent] bar
+# 58| 3: [ReservedWord] ]
+# 61| 40: [DelimitedSymbol] DelimitedSymbol
+# 61| 0: [ReservedWord] :'
+# 61| 1: [StringContent] \'
+# 61| 2: [ReservedWord] '
+# 62| 41: [DelimitedSymbol] DelimitedSymbol
+# 62| 0: [ReservedWord] :'
+# 62| 1: [StringContent] \"
+# 62| 2: [ReservedWord] '
+# 63| 42: [DelimitedSymbol] DelimitedSymbol
+# 63| 0: [ReservedWord] :'
+# 63| 1: [StringContent] \\
+# 63| 2: [ReservedWord] '
+# 64| 43: [DelimitedSymbol] DelimitedSymbol
+# 64| 0: [ReservedWord] :'
+# 64| 1: [StringContent] \1
+# 64| 2: [ReservedWord] '
+# 65| 44: [DelimitedSymbol] DelimitedSymbol
+# 65| 0: [ReservedWord] :'
+# 65| 1: [StringContent] \\1
+# 65| 2: [ReservedWord] '
+# 66| 45: [DelimitedSymbol] DelimitedSymbol
+# 66| 0: [ReservedWord] :'
+# 66| 1: [StringContent] \141
+# 66| 2: [ReservedWord] '
+# 67| 46: [DelimitedSymbol] DelimitedSymbol
+# 67| 0: [ReservedWord] :'
+# 67| 1: [StringContent] \n
+# 67| 2: [ReservedWord] '
+# 70| 47: [DelimitedSymbol] DelimitedSymbol
+# 70| 0: [ReservedWord] :"
+# 70| 1: [EscapeSequence] \'
+# 70| 2: [ReservedWord] "
+# 71| 48: [DelimitedSymbol] DelimitedSymbol
+# 71| 0: [ReservedWord] :"
+# 71| 1: [EscapeSequence] \"
+# 71| 2: [ReservedWord] "
+# 72| 49: [DelimitedSymbol] DelimitedSymbol
+# 72| 0: [ReservedWord] :"
+# 72| 1: [EscapeSequence] \\
+# 72| 2: [ReservedWord] "
+# 73| 50: [DelimitedSymbol] DelimitedSymbol
+# 73| 0: [ReservedWord] :"
+# 73| 1: [EscapeSequence] \1
+# 73| 2: [ReservedWord] "
+# 74| 51: [DelimitedSymbol] DelimitedSymbol
+# 74| 0: [ReservedWord] :"
+# 74| 1: [EscapeSequence] \\
+# 74| 2: [StringContent] 1
+# 74| 3: [ReservedWord] "
+# 75| 52: [DelimitedSymbol] DelimitedSymbol
+# 75| 0: [ReservedWord] :"
+# 75| 1: [EscapeSequence] \141
+# 75| 2: [ReservedWord] "
+# 76| 53: [DelimitedSymbol] DelimitedSymbol
+# 76| 0: [ReservedWord] :"
+# 76| 1: [EscapeSequence] \x6d
+# 76| 2: [ReservedWord] "
+# 77| 54: [DelimitedSymbol] DelimitedSymbol
+# 77| 0: [ReservedWord] :"
+# 77| 1: [EscapeSequence] \x6E
+# 77| 2: [ReservedWord] "
+# 78| 55: [DelimitedSymbol] DelimitedSymbol
+# 78| 0: [ReservedWord] :"
+# 78| 1: [EscapeSequence] \X
+# 78| 2: [StringContent] 6d
+# 78| 3: [ReservedWord] "
+# 79| 56: [DelimitedSymbol] DelimitedSymbol
+# 79| 0: [ReservedWord] :"
+# 79| 1: [EscapeSequence] \X
+# 79| 2: [StringContent] 6E
+# 79| 3: [ReservedWord] "
+# 80| 57: [DelimitedSymbol] DelimitedSymbol
+# 80| 0: [ReservedWord] :"
+# 80| 1: [EscapeSequence] \u203d
+# 80| 2: [ReservedWord] "
+# 81| 58: [DelimitedSymbol] DelimitedSymbol
+# 81| 0: [ReservedWord] :"
+# 81| 1: [EscapeSequence] \u{62}
+# 81| 2: [ReservedWord] "
+# 82| 59: [DelimitedSymbol] DelimitedSymbol
+# 82| 0: [ReservedWord] :"
+# 82| 1: [EscapeSequence] \u{1f60a}
+# 82| 2: [ReservedWord] "
+# 83| 60: [DelimitedSymbol] DelimitedSymbol
+# 83| 0: [ReservedWord] :"
+# 83| 1: [EscapeSequence] \a
+# 83| 2: [ReservedWord] "
+# 84| 61: [DelimitedSymbol] DelimitedSymbol
+# 84| 0: [ReservedWord] :"
+# 84| 1: [EscapeSequence] \b
+# 84| 2: [ReservedWord] "
+# 85| 62: [DelimitedSymbol] DelimitedSymbol
+# 85| 0: [ReservedWord] :"
+# 85| 1: [EscapeSequence] \t
+# 85| 2: [ReservedWord] "
+# 86| 63: [DelimitedSymbol] DelimitedSymbol
+# 86| 0: [ReservedWord] :"
+# 86| 1: [EscapeSequence] \n
+# 86| 2: [ReservedWord] "
+# 87| 64: [DelimitedSymbol] DelimitedSymbol
+# 87| 0: [ReservedWord] :"
+# 87| 1: [EscapeSequence] \v
+# 87| 2: [ReservedWord] "
+# 88| 65: [DelimitedSymbol] DelimitedSymbol
+# 88| 0: [ReservedWord] :"
+# 88| 1: [EscapeSequence] \f
+# 88| 2: [ReservedWord] "
+# 89| 66: [DelimitedSymbol] DelimitedSymbol
+# 89| 0: [ReservedWord] :"
+# 89| 1: [EscapeSequence] \r
+# 89| 2: [ReservedWord] "
+# 90| 67: [DelimitedSymbol] DelimitedSymbol
+# 90| 0: [ReservedWord] :"
+# 90| 1: [EscapeSequence] \e
+# 90| 2: [ReservedWord] "
+# 91| 68: [DelimitedSymbol] DelimitedSymbol
+# 91| 0: [ReservedWord] :"
+# 91| 1: [EscapeSequence] \s
+# 91| 2: [ReservedWord] "
+# 92| 69: [DelimitedSymbol] DelimitedSymbol
+# 92| 0: [ReservedWord] :"
+# 92| 1: [EscapeSequence] \c
+# 92| 2: [StringContent] ?
+# 92| 3: [ReservedWord] "
+# 93| 70: [DelimitedSymbol] DelimitedSymbol
+# 93| 0: [ReservedWord] :"
+# 93| 1: [EscapeSequence] \C
+# 93| 2: [StringContent] -?
+# 93| 3: [ReservedWord] "
+# 1| [Comment] # Most comments indicate the contents of the string after MRI has parsed the
+# 2| [Comment] # escape sequences (i.e. what gets printed by `puts`), and that's what we expect
+# 3| [Comment] # `getConstantValue().getString()` to return.
+# 5| [Comment] # The only escapes in single-quoted strings are backslash and single-quote.
+# 6| [Comment] # '
+# 7| [Comment] # \"
+# 8| [Comment] # \
+# 9| [Comment] # \1
+# 10| [Comment] # \1
+# 11| [Comment] # \141
+# 12| [Comment] # \n
+# 14| [Comment] # Double-quoted strings
+# 15| [Comment] # '
+# 16| [Comment] # "
+# 17| [Comment] # \
+# 18| [Comment] #
+# 19| [Comment] # \1
+# 20| [Comment] # a
+# 21| [Comment] # m
+# 22| [Comment] # n
+# 23| [Comment] # X6d
+# 24| [Comment] # X6E
+# 25| [Comment] # ‽
+# 26| [Comment] # b
+# 27| [Comment] # 😊
+# 28| [Comment] #
+# 29| [Comment] #
+# 30| [Comment] #
+# 31| [Comment] #
+# 32| [Comment] #
+# 33| [Comment] #