Merge branch 'main' into util/inline-expect-test-use-end-line

python/ql/lib/CHANGELOG.md

@@ -1,3 +1,16 @@
+## 0.8.0
+
+### Breaking Changes
+
+- Python 2 is no longer supported for extracting databases using the CodeQL CLI. As a consequence,
+  the previously deprecated support for `pyxl` and `spitfire` templates has also been removed. When
+  extracting Python 2 code, having Python 2 installed is still recommended, as this ensures the
+  correct version of the Python standard library is extracted.
+
+### Minor Analysis Improvements
+
+* Fixed module resolution so we properly recognize that in `from <pkg> import *`, where `<pkg>` is a package, the actual imports are made from the `<pkg>/__init__.py` file.
+
 ## 0.7.2
 
 No user-facing changes.

python/ql/lib/change-notes/2023-02-13-hmac-modeling.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Added modeling of cryptographic operations in the `hmac` library.

python/ql/lib/change-notes/2023-02-15-import-star-package.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* Fixed module resolution so we properly recognize that in `from <pkg> import *`, where `<pkg>` is a package, the actual imports are made from the `<pkg>/__init__.py` file.

python/ql/lib/change-notes/released/0.8.0.md

@@ -1,7 +1,12 @@
----
-category: breaking
----
+## 0.8.0
+
+### Breaking Changes
+
 - Python 2 is no longer supported for extracting databases using the CodeQL CLI. As a consequence,
   the previously deprecated support for `pyxl` and `spitfire` templates has also been removed. When
   extracting Python 2 code, having Python 2 installed is still recommended, as this ensures the
   correct version of the Python standard library is extracted.
+
+### Minor Analysis Improvements
+
+* Fixed module resolution so we properly recognize that in `from <pkg> import *`, where `<pkg>` is a package, the actual imports are made from the `<pkg>/__init__.py` file.

python/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.7.2
+lastReleaseVersion: 0.8.0

python/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/python-all
-version: 0.7.3-dev
+version: 0.8.1-dev
 groups: python
 dbscheme: semmlecode.python.dbscheme
 extractor: python

python/ql/lib/semmle/python/frameworks/Stdlib.qll

@@ -2669,6 +2669,7 @@ private module StdlibPrivate {
 
     HashlibNewCall() {
       this = hashlibNewCall(hashName) and
+      // we only want to consider it as an cryptographic operation if the input is available
       exists(this.getParameter(1, "data"))
     }
 
@@ -2751,6 +2752,78 @@ private module StdlibPrivate {
     }
   }
 
+  // ---------------------------------------------------------------------------
+  // hmac
+  // ---------------------------------------------------------------------------
+  abstract class HmacCryptographicOperation extends Cryptography::CryptographicOperation::Range,
+    API::CallNode {
+    abstract API::Node getDigestArg();
+
+    override Cryptography::CryptographicAlgorithm getAlgorithm() {
+      exists(string algorithmName | result.matchesName(algorithmName) |
+        this.getDigestArg().asSink() = hashlibMember(algorithmName).asSource()
+        or
+        this.getDigestArg().getAValueReachingSink().asExpr().(StrConst).getText() = algorithmName
+      )
+    }
+
+    override Cryptography::BlockMode getBlockMode() { none() }
+  }
+
+  API::CallNode getHmacConstructorCall(API::Node digestArg) {
+    result = API::moduleImport("hmac").getMember(["new", "HMAC"]).getACall() and
+    digestArg = result.getParameter(2, "digestmod")
+  }
+
+  /**
+   * A call to `hmac.new`/`hmac.HMAC`.
+   *
+   * See https://docs.python.org/3.11/library/hmac.html#hmac.new
+   */
+  class HmacNewCall extends HmacCryptographicOperation {
+    API::Node digestArg;
+
+    HmacNewCall() {
+      this = getHmacConstructorCall(digestArg) and
+      // we only want to consider it as an cryptographic operation if the input is available
+      exists(this.getParameter(1, "msg").asSink())
+    }
+
+    override API::Node getDigestArg() { result = digestArg }
+
+    override DataFlow::Node getAnInput() { result = this.getParameter(1, "msg").asSink() }
+  }
+
+  /**
+   * A call to `.update` on an HMAC object.
+   *
+   * See https://docs.python.org/3.11/library/hmac.html#hmac.HMAC.update
+   */
+  class HmacUpdateCall extends HmacCryptographicOperation {
+    API::Node digestArg;
+
+    HmacUpdateCall() {
+      this = getHmacConstructorCall(digestArg).getReturn().getMember("update").getACall()
+    }
+
+    override API::Node getDigestArg() { result = digestArg }
+
+    override DataFlow::Node getAnInput() { result = this.getParameter(0, "msg").asSink() }
+  }
+
+  /**
+   * A call to `hmac.digest`.
+   *
+   * See https://docs.python.org/3.11/library/hmac.html#hmac.digest
+   */
+  class HmacDigestCall extends HmacCryptographicOperation {
+    HmacDigestCall() { this = API::moduleImport("hmac").getMember("digest").getACall() }
+
+    override API::Node getDigestArg() { result = this.getParameter(2, "digest") }
+
+    override DataFlow::Node getAnInput() { result = this.getParameter(1, "msg").asSink() }
+  }
+
   // ---------------------------------------------------------------------------
   // logging
   // ---------------------------------------------------------------------------

python/ql/src/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.6.3
+
+No user-facing changes.
+
 ## 0.6.2
 
 No user-facing changes.

python/ql/src/Security/CWE-022/examples/tainted_path.py

@@ -1,36 +1,30 @@
 import os.path
+from flask import Flask, request, abort
 
+app = Flask(__name__)
 
-urlpatterns = [
-    # Route to user_picture
-    url(r'^user-pic1$', user_picture1, name='user-picture1'),
-    url(r'^user-pic2$', user_picture2, name='user-picture2'),
-    url(r'^user-pic3$', user_picture3, name='user-picture3')
-]
-
-
-def user_picture1(request):
-    """A view that is vulnerable to malicious file access."""
-    filename = request.GET.get('p')
+@app.route("/user_picture1")
+def user_picture1():
+    filename = request.args.get('p')
     # BAD: This could read any file on the file system
     data = open(filename, 'rb').read()
-    return HttpResponse(data)
+    return data
 
-def user_picture2(request):
-    """A view that is vulnerable to malicious file access."""
+@app.route("/user_picture2")
+def user_picture2():
     base_path = '/server/static/images'
-    filename = request.GET.get('p')
+    filename = request.args.get('p')
     # BAD: This could still read any file on the file system
     data = open(os.path.join(base_path, filename), 'rb').read()
-    return HttpResponse(data)
+    return data
 
-def user_picture3(request):
-    """A view that is not vulnerable to malicious file access."""
+@app.route("/user_picture3")
+def user_picture3():
     base_path = '/server/static/images'
-    filename = request.GET.get('p')
+    filename = request.args.get('p')
     #GOOD -- Verify with normalised version of path
     fullpath = os.path.normpath(os.path.join(base_path, filename))
     if not fullpath.startswith(base_path):
-        raise SecurityException()
+        raise Exception("not allowed")
     data = open(fullpath, 'rb').read()
-    return HttpResponse(data)
+    return data

python/ql/src/Security/CWE-022/examples/tarslip_bad.py

@@ -1,7 +1,7 @@
-
+import sys
 import tarfile
 
-with tarfile.open('archive.zip') as tar:
+with tarfile.open(sys.argv[1]) as tar:
     #BAD : This could write any file on the filesystem.
     for entry in tar:
         tar.extract(entry, "/tmp/unpack/")

python/ql/src/Security/CWE-022/examples/tarslip_good.py

@@ -1,8 +1,8 @@
-
+import sys
 import tarfile
 import os.path
 
-with tarfile.open('archive.zip') as tar:
+with tarfile.open(sys.argv[1]) as tar:
     for entry in tar:
         #GOOD: Check that entry is safe
         if os.path.isabs(entry.name) or ".." in entry.name:

python/ql/src/change-notes/released/0.6.3.md

@@ -0,0 +1,3 @@
+## 0.6.3
+
+No user-facing changes.

python/ql/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.6.2
+lastReleaseVersion: 0.6.3

python/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/python-queries
-version: 0.6.3-dev
+version: 0.6.4-dev
 groups: 
   - python
   - queries

python/ql/test/library-tests/frameworks/stdlib/test_hmac.py

@@ -0,0 +1,33 @@
+import hmac
+import hashlib
+
+key = b"<secret key>"
+
+hmac_obj = hmac.new(key, b"secret message", "sha256") # $ CryptographicOperation CryptographicOperationInput=b"secret message" CryptographicOperationAlgorithm=SHA256
+print(hmac_obj.digest())
+print(hmac_obj.hexdigest())
+
+hmac_obj = hmac.new(key, msg=b"secret message", digestmod="sha256") # $ CryptographicOperation CryptographicOperationInput=b"secret message" CryptographicOperationAlgorithm=SHA256
+print(hmac_obj.hexdigest())
+
+
+hmac_obj = hmac.new(key, digestmod="sha256")
+hmac_obj.update(b"secret") # $ CryptographicOperation CryptographicOperationInput=b"secret" CryptographicOperationAlgorithm=SHA256
+hmac_obj.update(msg=b" message") # $ CryptographicOperation CryptographicOperationInput=b" message" CryptographicOperationAlgorithm=SHA256
+print(hmac_obj.hexdigest())
+
+
+hmac_obj = hmac.new(key, b"secret message", hashlib.sha256) # $ CryptographicOperation CryptographicOperationInput=b"secret message" CryptographicOperationAlgorithm=SHA256
+print(hmac_obj.hexdigest())
+
+
+# like hmac.new
+hmac_obj = hmac.HMAC(key, digestmod="sha256")
+hmac_obj.update(b"secret message") # $ CryptographicOperation CryptographicOperationInput=b"secret message" CryptographicOperationAlgorithm=SHA256
+print(hmac_obj.hexdigest())
+
+
+dig = hmac.digest(key, b"secret message", "sha256") # $ CryptographicOperation CryptographicOperationInput=b"secret message" CryptographicOperationAlgorithm=SHA256
+print(dig)
+dig = hmac.digest(key, msg=b"secret message", digest="sha256") # $ CryptographicOperation CryptographicOperationInput=b"secret message" CryptographicOperationAlgorithm=SHA256
+print(dig)