Merge branch 'main' into clean-tests

python/ql/lib/semmle/python/Frameworks.qll

@@ -5,13 +5,18 @@
 // If you add modeling of a new framework/library, remember to add it to the docs in
 // `docs/codeql/reusables/supported-frameworks.rst`
 private import semmle.python.frameworks.Aioch
+private import semmle.python.frameworks.Aiofile
+private import semmle.python.frameworks.Aiofiles
 private import semmle.python.frameworks.Aiohttp
 private import semmle.python.frameworks.Aiomysql
 private import semmle.python.frameworks.Aiopg
 private import semmle.python.frameworks.Aiosqlite
+private import semmle.python.frameworks.Anyio
 private import semmle.python.frameworks.Asyncpg
+private import semmle.python.frameworks.Baize
 private import semmle.python.frameworks.BSon
 private import semmle.python.frameworks.CassandraDriver
+private import semmle.python.frameworks.Cherrypy
 private import semmle.python.frameworks.ClickhouseDriver
 private import semmle.python.frameworks.Cryptodome
 private import semmle.python.frameworks.Cryptography
@@ -54,6 +59,7 @@ private import semmle.python.frameworks.Requests
 private import semmle.python.frameworks.RestFramework
 private import semmle.python.frameworks.Rsa
 private import semmle.python.frameworks.RuamelYaml
+private import semmle.python.frameworks.Sanic
 private import semmle.python.frameworks.ServerLess
 private import semmle.python.frameworks.Setuptools
 private import semmle.python.frameworks.Simplejson

python/ql/lib/semmle/python/frameworks/Aiofile.qll

@@ -0,0 +1,42 @@
+/**
+ * Provides classes modeling security-relevant aspects of the `aiofile` PyPI package.
+ *
+ * See https://pypi.org/project/aiofile.
+ */
+
+private import python
+private import semmle.python.dataflow.new.DataFlow
+private import semmle.python.dataflow.new.RemoteFlowSources
+private import semmle.python.dataflow.new.TaintTracking
+private import semmle.python.Concepts
+private import semmle.python.ApiGraphs
+
+/**
+ * Provides models for the `aiofile` PyPI package.
+ *
+ * See https://pypi.org/project/aiofile.
+ */
+private module Aiofile {
+  /**
+   * A call to the `async_open` function or `AIOFile` constructor from `aiofile` as a sink for Filesystem access.
+   */
+  class FileResponseCall extends FileSystemAccess::Range, API::CallNode {
+    string methodName;
+
+    FileResponseCall() {
+      this = API::moduleImport("aiofile").getMember("async_open").getACall() and
+      methodName = "async_open"
+      or
+      this = API::moduleImport("aiofile").getMember("AIOFile").getACall() and
+      methodName = "AIOFile"
+    }
+
+    override DataFlow::Node getAPathArgument() {
+      result = this.getParameter(0, "file_specifier").asSink() and
+      methodName = "async_open"
+      or
+      result = this.getParameter(0, "filename").asSink() and
+      methodName = "AIOFile"
+    }
+  }
+}

python/ql/lib/semmle/python/frameworks/Aiofiles.qll

@@ -0,0 +1,28 @@
+/**
+ * Provides classes modeling security-relevant aspects of the `aiofiles` PyPI package.
+ *
+ * See https://pypi.org/project/aiofiles.
+ */
+
+private import python
+private import semmle.python.dataflow.new.DataFlow
+private import semmle.python.dataflow.new.RemoteFlowSources
+private import semmle.python.dataflow.new.TaintTracking
+private import semmle.python.Concepts
+private import semmle.python.ApiGraphs
+
+/**
+ * Provides models for the `aiofiles` PyPI package.
+ *
+ * See https://pypi.org/project/aiofiles.
+ */
+private module Aiofiles {
+  /**
+   * A call to the `open` function from `aiofiles` as a sink for Filesystem access.
+   */
+  class FileResponseCall extends FileSystemAccess::Range, API::CallNode {
+    FileResponseCall() { this = API::moduleImport("aiofiles").getMember("open").getACall() }
+
+    override DataFlow::Node getAPathArgument() { result = this.getParameter(0, "file").asSink() }
+  }
+}

python/ql/lib/semmle/python/frameworks/Anyio.qll

@@ -0,0 +1,54 @@
+/**
+ * Provides classes modeling security-relevant aspects of the `anyio` PyPI package.
+ *
+ * See https://pypi.org/project/anyio.
+ */
+
+private import python
+private import semmle.python.dataflow.new.DataFlow
+private import semmle.python.dataflow.new.RemoteFlowSources
+private import semmle.python.dataflow.new.TaintTracking
+private import semmle.python.Concepts
+private import semmle.python.ApiGraphs
+
+/**
+ * Provides models for the `anyio` PyPI package.
+ *
+ * See https://pypi.org/project/anyio.
+ */
+private module Anyio {
+  /**
+   * A call to the `from_path` function from `FileReadStream` or `FileWriteStream` constructors of `anyio.streams.file` as a sink for Filesystem access.
+   */
+  class FileStreamCall extends FileSystemAccess::Range, API::CallNode {
+    FileStreamCall() {
+      this =
+        API::moduleImport("anyio")
+            .getMember("streams")
+            .getMember("file")
+            .getMember(["FileReadStream", "FileWriteStream"])
+            .getMember("from_path")
+            .getACall()
+    }
+
+    override DataFlow::Node getAPathArgument() { result = this.getParameter(0, "path").asSink() }
+  }
+
+  /**
+   * A call to the `Path` constructor from `anyio` as a sink for Filesystem access.
+   */
+  class PathCall extends FileSystemAccess::Range, API::CallNode {
+    PathCall() { this = API::moduleImport("anyio").getMember("Path").getACall() }
+
+    override DataFlow::Node getAPathArgument() { result = this.getParameter(0).asSink() }
+  }
+
+  /**
+   * A call to the `open_file` function from `anyio` as a sink for Filesystem access.
+   */
+  class OpenFileCall extends FileSystemAccess::Range, API::CallNode {
+    OpenFileCall() { this = API::moduleImport("anyio").getMember("open_file").getACall() }
+
+    override DataFlow::Node getAPathArgument() { result = this.getParameter(0, "file").asSink() }
+  }
+}

python/ql/lib/semmle/python/frameworks/Baize.qll

@@ -0,0 +1,35 @@
+/**
+ * Provides classes modeling security-relevant aspects of the `baize` PyPI package.
+ *
+ * See https://pypi.org/project/baize.
+ */
+
+private import python
+private import semmle.python.dataflow.new.DataFlow
+private import semmle.python.dataflow.new.TaintTracking
+private import semmle.python.Concepts
+private import semmle.python.ApiGraphs
+private import semmle.python.frameworks.internal.InstanceTaintStepsHelper
+private import semmle.python.frameworks.Stdlib
+
+/**
+ * Provides models for `baize` PyPI package.
+ *
+ * See https://pypi.org/project/baize.
+ */
+module Baize {
+  /**
+   * A call to the `baize.asgi.FileResponse` constructor as a sink for Filesystem access.
+   *
+   * it is not contained to Starlette source code but it is mentioned in documents as an alternative to Starlette FileResponse
+   */
+  class BaizeFileResponseCall extends FileSystemAccess::Range, API::CallNode {
+    BaizeFileResponseCall() {
+      this = API::moduleImport("baize").getMember("asgi").getMember("FileResponse").getACall()
+    }
+
+    override DataFlow::Node getAPathArgument() {
+      result = this.getParameter(0, "filepath").asSink()
+    }
+  }
+}

python/ql/lib/semmle/python/frameworks/Cherrypy.qll

@@ -0,0 +1,48 @@
+/**
+ * Provides classes modeling security-relevant aspects of the `cherrypy` PyPI package.
+ */
+
+private import python
+private import semmle.python.dataflow.new.DataFlow
+private import semmle.python.dataflow.new.RemoteFlowSources
+private import semmle.python.dataflow.new.TaintTracking
+private import semmle.python.Concepts
+private import semmle.python.ApiGraphs
+
+/**
+ * Provides models for the `cherrypy` PyPI package.
+ * See https://cherrypy.dev/.
+ */
+private module Cherrypy {
+  /**
+   * Holds for an instance of `cherrypy.lib.static`
+   */
+  API::Node libStatic() {
+    result = API::moduleImport("cherrypy").getMember("lib").getMember("static")
+  }
+
+  /**
+   * A call to the `serve_file` or `serve_download`or `staticfile` functions of `cherrypy.lib.static` as a sink for Filesystem access.
+   */
+  class FileResponseCall extends FileSystemAccess::Range, API::CallNode {
+    string funcName;
+
+    FileResponseCall() {
+      this = libStatic().getMember("staticfile").getACall() and
+      funcName = "staticfile"
+      or
+      this = libStatic().getMember("serve_file").getACall() and
+      funcName = "serve_file"
+      or
+      this = libStatic().getMember("serve_download").getACall() and
+      funcName = "serve_download"
+    }
+
+    override DataFlow::Node getAPathArgument() {
+      result = this.getParameter(0, "path").asSink() and funcName = ["serve_download", "serve_file"]
+      or
+      result = this.getParameter(0, "filename").asSink() and
+      funcName = "staticfile"
+    }
+  }
+}

python/ql/lib/semmle/python/frameworks/Sanic.qll

@@ -0,0 +1,42 @@
+/**
+ * Provides classes modeling security-relevant aspects of the `sanic` PyPI package.
+ * See https://sanic.dev/.
+ */
+
+private import python
+private import semmle.python.dataflow.new.DataFlow
+private import semmle.python.dataflow.new.RemoteFlowSources
+private import semmle.python.dataflow.new.TaintTracking
+private import semmle.python.Concepts
+private import semmle.python.ApiGraphs
+
+/**
+ * Provides models for the `sanic` PyPI package.
+ * See https://sanic.dev/.
+ */
+private module Sanic {
+  /**
+   * Provides models for Sanic applications (an instance of `sanic.Sanic`).
+   */
+  module App {
+    /** Gets a reference to a Sanic application (an instance of `sanic.Sanic`). */
+    API::Node instance() { result = API::moduleImport("sanic").getMember("Sanic").getReturn() }
+  }
+
+  /**
+   * A call to the `file` or `file_stream` functions of `sanic.response` as a sink for Filesystem access.
+   */
+  class FileResponseCall extends FileSystemAccess::Range, API::CallNode {
+    FileResponseCall() {
+      this =
+        API::moduleImport("sanic")
+            .getMember("response")
+            .getMember(["file", "file_stream"])
+            .getACall()
+    }
+
+    override DataFlow::Node getAPathArgument() {
+      result = this.getParameter(0, "location").asSink()
+    }
+  }
+}

python/ql/lib/semmle/python/frameworks/Starlette.qll

@@ -163,4 +163,16 @@ module Starlette {
 
   /** DEPRECATED: Alias for Url */
   deprecated module URL = Url;
+
+  /**
+   * A call to the `starlette.responses.FileResponse` constructor as a sink for Filesystem access.
+   */
+  class FileResponseCall extends FileSystemAccess::Range, API::CallNode {
+    FileResponseCall() {
+      this =
+        API::moduleImport("starlette").getMember("responses").getMember("FileResponse").getACall()
+    }
+
+    override DataFlow::Node getAPathArgument() { result = this.getParameter(0, "path").asSink() }
+  }
 }

python/ql/lib/semmle/python/frameworks/Stdlib.qll

@@ -1479,6 +1479,26 @@ private module StdlibPrivate {
     }
   }
 
+  /**
+   * A call to the `io.FileIO` constructor.
+   * See https://docs.python.org/3/library/io.html#io.FileIO
+   */
+  private class FileIOCall extends FileSystemAccess::Range, API::CallNode {
+    FileIOCall() { this = API::moduleImport("io").getMember("FileIO").getACall() }
+
+    override DataFlow::Node getAPathArgument() { result = this.getParameter(0, "file").asSink() }
+  }
+
+  /**
+   * A call to the `io.open_code` function.
+   * See https://docs.python.org/3.11/library/io.html#io.open_code
+   */
+  private class OpenCodeCall extends FileSystemAccess::Range, API::CallNode {
+    OpenCodeCall() { this = API::moduleImport("io").getMember("open_code").getACall() }
+
+    override DataFlow::Node getAPathArgument() { result = this.getParameter(0, "path").asSink() }
+  }
+
   /** Gets a reference to an open file. */
   private DataFlow::TypeTrackingNode openFile(DataFlow::TypeTracker t, FileSystemAccess openCall) {
     t.start() and

python/ql/src/Summary/options

@@ -0,0 +1 @@
+semmle-extractor-options: --lang=3

python/ql/src/change-notes/2023-11-06-more-filesystem-modeling.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Added modeling of more `FileSystemAccess` in packages `cherrypy`, `aiofile`, `aiofiles`, `anyio`, `sanic`, `starlette`, `baize`, and `io`. This will mainly affect the _Uncontrolled data used in path expression_ (`py/path-injection`) query.

python/ql/test/2/library-tests/PointsTo/imports2/options

@@ -0,0 +1 @@
+semmle-extractor-options: --lang=2 --max-import-depth=2 -r package

python/ql/test/2/library-tests/modules/usage/options

@@ -0,0 +1 @@
+semmle-extractor-options: --lang=2 -F script

python/ql/test/3/library-tests/PointsTo/imports/Runtime.expected

@@ -0,0 +1,57 @@
+| __init__.py | 1 | ControlFlowNode for ImportExpr | Module package.module | ControlFlowNode for ImportExpr |
+| __init__.py | 2 | ControlFlowNode for ImportMember | Function module | ControlFlowNode for FunctionExpr |
+| __init__.py | 2 | ControlFlowNode for module | Function module | ControlFlowNode for FunctionExpr |
+| __init__.py | 4 | ControlFlowNode for ImportExpr | Module package | ControlFlowNode for ImportExpr |
+| __init__.py | 4 | ControlFlowNode for ImportMember | Module package.module2 | Entry node for Module package.module2 |
+| __init__.py | 4 | ControlFlowNode for module3 | Module package.module2 | Entry node for Module package.module2 |
+| __init__.py | 5 | ControlFlowNode for IntegerLiteral | int 7 | ControlFlowNode for IntegerLiteral |
+| __init__.py | 5 | ControlFlowNode for module2 | int 7 | ControlFlowNode for IntegerLiteral |
+| __init__.py | 6 | ControlFlowNode for ImportExpr | Module package | ControlFlowNode for ImportExpr |
+| __init__.py | 6 | ControlFlowNode for ImportMember | int 7 | ControlFlowNode for IntegerLiteral |
+| __init__.py | 6 | ControlFlowNode for module4 | int 7 | ControlFlowNode for IntegerLiteral |
+| __init__.py | 7 | ControlFlowNode for ImportExpr | Module package | ControlFlowNode for ImportExpr |
+| __init__.py | 7 | ControlFlowNode for ImportMember | Module package.module2 | Entry node for Module package.module2 |
+| __init__.py | 7 | ControlFlowNode for module5 | Module package.module2 | Entry node for Module package.module2 |
+| __init__.py | 8 | ControlFlowNode for ImportExpr | Module package | ControlFlowNode for ImportExpr |
+| __init__.py | 8 | ControlFlowNode for ImportMember | Module package.moduleX | Entry node for Module package.moduleX |
+| __init__.py | 8 | ControlFlowNode for moduleX | Module package.moduleX | Entry node for Module package.moduleX |
+| module2.py | 1 | ControlFlowNode for IntegerLiteral | int 0 | ControlFlowNode for IntegerLiteral |
+| module2.py | 1 | ControlFlowNode for x | int 0 | ControlFlowNode for IntegerLiteral |
+| module.py | 2 | ControlFlowNode for FunctionExpr | Function module | ControlFlowNode for FunctionExpr |
+| module.py | 2 | ControlFlowNode for module | Function module | ControlFlowNode for FunctionExpr |
+| moduleX.py | 1 | ControlFlowNode for ClassExpr | class Y | ControlFlowNode for ClassExpr |
+| moduleX.py | 1 | ControlFlowNode for Y | class Y | ControlFlowNode for ClassExpr |
+| moduleX.py | 1 | ControlFlowNode for object | builtin-class object | ControlFlowNode for object |
+| test.py | 1 | ControlFlowNode for ImportExpr | Module package | ControlFlowNode for ImportExpr |
+| test.py | 2 | ControlFlowNode for ImportMember | Function module | ControlFlowNode for FunctionExpr |
+| test.py | 2 | ControlFlowNode for module | Function module | ControlFlowNode for FunctionExpr |
+| test.py | 4 | ControlFlowNode for ImportExpr | Module package | ControlFlowNode for ImportExpr |
+| test.py | 5 | ControlFlowNode for ImportMember | Module package.x | Entry node for Module package.x |
+| test.py | 5 | ControlFlowNode for x | Module package.x | Entry node for Module package.x |
+| test.py | 8 | ControlFlowNode for C | class C | ControlFlowNode for ClassExpr |
+| test.py | 8 | ControlFlowNode for ClassExpr | class C | ControlFlowNode for ClassExpr |
+| test.py | 8 | ControlFlowNode for object | builtin-class object | ControlFlowNode for object |
+| test.py | 10 | ControlFlowNode for ImportExpr | Module package | ControlFlowNode for ImportExpr |
+| test.py | 10 | ControlFlowNode for ImportMember | int 7 | ControlFlowNode for IntegerLiteral |
+| test.py | 10 | ControlFlowNode for module2 | int 7 | ControlFlowNode for IntegerLiteral |
+| test.py | 12 | ControlFlowNode for FunctionExpr | Function f | ControlFlowNode for FunctionExpr |
+| test.py | 12 | ControlFlowNode for f | Function f | ControlFlowNode for FunctionExpr |
+| test.py | 13 | ControlFlowNode for ImportExpr | Module package | ControlFlowNode for ImportExpr |
+| test.py | 13 | ControlFlowNode for ImportMember | Module package.x | Entry node for Module package.x |
+| test.py | 13 | ControlFlowNode for x | Module package.x | Entry node for Module package.x |
+| test.py | 15 | ControlFlowNode for ImportExpr | Module package | ControlFlowNode for ImportExpr |
+| test.py | 15 | ControlFlowNode for ImportMember | Module package.moduleX | Entry node for Module package.moduleX |
+| test.py | 15 | ControlFlowNode for moduleX | Module package.moduleX | Entry node for Module package.moduleX |
+| test.py | 16 | ControlFlowNode for Attribute | class Y | ControlFlowNode for ClassExpr |
+| test.py | 16 | ControlFlowNode for moduleX | Module package.moduleX | Entry node for Module package.moduleX |
+| test.py | 19 | ControlFlowNode for ImportExpr | Module tty | ControlFlowNode for ImportExpr |
+| test.py | 19 | ControlFlowNode for tty | Module tty | ControlFlowNode for ImportExpr |
+| test.py | 22 | ControlFlowNode for Attribute | Builtin-function exc_info | ControlFlowNode for from sys import * |
+| test.py | 22 | ControlFlowNode for x | Module package.x | Entry node for Module package.x |
+| test.py | 24 | ControlFlowNode for IntegerLiteral | int 0 | ControlFlowNode for IntegerLiteral |
+| test.py | 24 | ControlFlowNode for argv | int 0 | ControlFlowNode for IntegerLiteral |
+| test.py | 27 | ControlFlowNode for ImportExpr | Module sys | ControlFlowNode for ImportExpr |
+| test.py | 31 | ControlFlowNode for argv | list object | ControlFlowNode for from sys import * |
+| test.py | 33 | ControlFlowNode for ImportExpr | Module socket | ControlFlowNode for ImportExpr |
+| test.py | 34 | ControlFlowNode for timeout | builtin-class TimeoutError | ControlFlowNode for from _socket import * |
+| x.py | 2 | ControlFlowNode for ImportExpr | Module sys | ControlFlowNode for ImportExpr |

python/ql/test/3/library-tests/PointsTo/imports/Runtime.ql

@@ -0,0 +1,9 @@
+import python
+
+from int line, ControlFlowNode f, Object o, ControlFlowNode orig
+where
+  not f.getLocation().getFile().inStdlib() and
+  f.refersTo(o, orig) and
+  line = f.getLocation().getStartLine() and
+  line != 0
+select f.getLocation().getFile().getShortName(), line, f.toString(), o.toString(), orig.toString()

python/ql/test/3/library-tests/PointsTo/imports/RuntimeWithType.expected

@@ -0,0 +1,57 @@
+| __init__.py | 1 | ControlFlowNode for ImportExpr | Module package.module | builtin-class module | ControlFlowNode for ImportExpr |
+| __init__.py | 2 | ControlFlowNode for ImportMember | Function module | builtin-class function | ControlFlowNode for FunctionExpr |
+| __init__.py | 2 | ControlFlowNode for module | Function module | builtin-class function | ControlFlowNode for FunctionExpr |
+| __init__.py | 4 | ControlFlowNode for ImportExpr | Module package | builtin-class module | ControlFlowNode for ImportExpr |
+| __init__.py | 4 | ControlFlowNode for ImportMember | Module package.module2 | builtin-class module | Entry node for Module package.module2 |
+| __init__.py | 4 | ControlFlowNode for module3 | Module package.module2 | builtin-class module | Entry node for Module package.module2 |
+| __init__.py | 5 | ControlFlowNode for IntegerLiteral | int 7 | builtin-class int | ControlFlowNode for IntegerLiteral |
+| __init__.py | 5 | ControlFlowNode for module2 | int 7 | builtin-class int | ControlFlowNode for IntegerLiteral |
+| __init__.py | 6 | ControlFlowNode for ImportExpr | Module package | builtin-class module | ControlFlowNode for ImportExpr |
+| __init__.py | 6 | ControlFlowNode for ImportMember | int 7 | builtin-class int | ControlFlowNode for IntegerLiteral |
+| __init__.py | 6 | ControlFlowNode for module4 | int 7 | builtin-class int | ControlFlowNode for IntegerLiteral |
+| __init__.py | 7 | ControlFlowNode for ImportExpr | Module package | builtin-class module | ControlFlowNode for ImportExpr |
+| __init__.py | 7 | ControlFlowNode for ImportMember | Module package.module2 | builtin-class module | Entry node for Module package.module2 |
+| __init__.py | 7 | ControlFlowNode for module5 | Module package.module2 | builtin-class module | Entry node for Module package.module2 |
+| __init__.py | 8 | ControlFlowNode for ImportExpr | Module package | builtin-class module | ControlFlowNode for ImportExpr |
+| __init__.py | 8 | ControlFlowNode for ImportMember | Module package.moduleX | builtin-class module | Entry node for Module package.moduleX |
+| __init__.py | 8 | ControlFlowNode for moduleX | Module package.moduleX | builtin-class module | Entry node for Module package.moduleX |
+| module2.py | 1 | ControlFlowNode for IntegerLiteral | int 0 | builtin-class int | ControlFlowNode for IntegerLiteral |
+| module2.py | 1 | ControlFlowNode for x | int 0 | builtin-class int | ControlFlowNode for IntegerLiteral |
+| module.py | 2 | ControlFlowNode for FunctionExpr | Function module | builtin-class function | ControlFlowNode for FunctionExpr |
+| module.py | 2 | ControlFlowNode for module | Function module | builtin-class function | ControlFlowNode for FunctionExpr |
+| moduleX.py | 1 | ControlFlowNode for ClassExpr | class Y | builtin-class type | ControlFlowNode for ClassExpr |
+| moduleX.py | 1 | ControlFlowNode for Y | class Y | builtin-class type | ControlFlowNode for ClassExpr |
+| moduleX.py | 1 | ControlFlowNode for object | builtin-class object | builtin-class type | ControlFlowNode for object |
+| test.py | 1 | ControlFlowNode for ImportExpr | Module package | builtin-class module | ControlFlowNode for ImportExpr |
+| test.py | 2 | ControlFlowNode for ImportMember | Function module | builtin-class function | ControlFlowNode for FunctionExpr |
+| test.py | 2 | ControlFlowNode for module | Function module | builtin-class function | ControlFlowNode for FunctionExpr |
+| test.py | 4 | ControlFlowNode for ImportExpr | Module package | builtin-class module | ControlFlowNode for ImportExpr |
+| test.py | 5 | ControlFlowNode for ImportMember | Module package.x | builtin-class module | Entry node for Module package.x |
+| test.py | 5 | ControlFlowNode for x | Module package.x | builtin-class module | Entry node for Module package.x |
+| test.py | 8 | ControlFlowNode for C | class C | builtin-class type | ControlFlowNode for ClassExpr |
+| test.py | 8 | ControlFlowNode for ClassExpr | class C | builtin-class type | ControlFlowNode for ClassExpr |
+| test.py | 8 | ControlFlowNode for object | builtin-class object | builtin-class type | ControlFlowNode for object |
+| test.py | 10 | ControlFlowNode for ImportExpr | Module package | builtin-class module | ControlFlowNode for ImportExpr |
+| test.py | 10 | ControlFlowNode for ImportMember | int 7 | builtin-class int | ControlFlowNode for IntegerLiteral |
+| test.py | 10 | ControlFlowNode for module2 | int 7 | builtin-class int | ControlFlowNode for IntegerLiteral |
+| test.py | 12 | ControlFlowNode for FunctionExpr | Function f | builtin-class function | ControlFlowNode for FunctionExpr |
+| test.py | 12 | ControlFlowNode for f | Function f | builtin-class function | ControlFlowNode for FunctionExpr |
+| test.py | 13 | ControlFlowNode for ImportExpr | Module package | builtin-class module | ControlFlowNode for ImportExpr |
+| test.py | 13 | ControlFlowNode for ImportMember | Module package.x | builtin-class module | Entry node for Module package.x |
+| test.py | 13 | ControlFlowNode for x | Module package.x | builtin-class module | Entry node for Module package.x |
+| test.py | 15 | ControlFlowNode for ImportExpr | Module package | builtin-class module | ControlFlowNode for ImportExpr |
+| test.py | 15 | ControlFlowNode for ImportMember | Module package.moduleX | builtin-class module | Entry node for Module package.moduleX |
+| test.py | 15 | ControlFlowNode for moduleX | Module package.moduleX | builtin-class module | Entry node for Module package.moduleX |
+| test.py | 16 | ControlFlowNode for Attribute | class Y | builtin-class type | ControlFlowNode for ClassExpr |
+| test.py | 16 | ControlFlowNode for moduleX | Module package.moduleX | builtin-class module | Entry node for Module package.moduleX |
+| test.py | 19 | ControlFlowNode for ImportExpr | Module tty | builtin-class module | ControlFlowNode for ImportExpr |
+| test.py | 19 | ControlFlowNode for tty | Module tty | builtin-class module | ControlFlowNode for ImportExpr |
+| test.py | 22 | ControlFlowNode for Attribute | Builtin-function exc_info | builtin-class builtin_function_or_method | ControlFlowNode for from sys import * |
+| test.py | 22 | ControlFlowNode for x | Module package.x | builtin-class module | Entry node for Module package.x |
+| test.py | 24 | ControlFlowNode for IntegerLiteral | int 0 | builtin-class int | ControlFlowNode for IntegerLiteral |
+| test.py | 24 | ControlFlowNode for argv | int 0 | builtin-class int | ControlFlowNode for IntegerLiteral |
+| test.py | 27 | ControlFlowNode for ImportExpr | Module sys | builtin-class module | ControlFlowNode for ImportExpr |
+| test.py | 31 | ControlFlowNode for argv | list object | builtin-class list | ControlFlowNode for from sys import * |
+| test.py | 33 | ControlFlowNode for ImportExpr | Module socket | builtin-class module | ControlFlowNode for ImportExpr |
+| test.py | 34 | ControlFlowNode for timeout | builtin-class TimeoutError | builtin-class type | ControlFlowNode for from _socket import * |
+| x.py | 2 | ControlFlowNode for ImportExpr | Module sys | builtin-class module | ControlFlowNode for ImportExpr |

python/ql/test/3/library-tests/PointsTo/imports/RuntimeWithType.ql

@@ -0,0 +1,10 @@
+import python
+
+from int line, ControlFlowNode f, Object o, ClassObject cls, ControlFlowNode orig
+where
+  not f.getLocation().getFile().inStdlib() and
+  f.refersTo(o, cls, orig) and
+  line = f.getLocation().getStartLine() and
+  line != 0
+select f.getLocation().getFile().getShortName(), line, f.toString(), o.toString(), cls.toString(),
+  orig.toString()

python/ql/test/3/library-tests/PointsTo/imports/options

@@ -0,0 +1 @@
+semmle-extractor-options: --lang=3 --max-import-depth=2 -r package

python/ql/test/3/library-tests/PointsTo/imports/package/__init__.py

@@ -0,0 +1,15 @@
+from .module \
+import module
+
+from . import module2 as module3
+module2 = 7
+from . import module2 as module4
+from . import module3 as module5
+from package import moduleX
+
+#We should now have:
+#module2 = 7
+#module3 = package.module2
+#module4 = 7
+#module5 = package.module2
+#moduleX = package.moduleX

python/ql/test/3/library-tests/PointsTo/imports/package/module.py

@@ -0,0 +1,3 @@
+
+def module(args):
+    pass

python/ql/test/3/library-tests/PointsTo/imports/package/module2.py

@@ -0,0 +1 @@
+x = 0

python/ql/test/3/library-tests/PointsTo/imports/package/moduleX.py

@@ -0,0 +1,2 @@
+class Y(object):
+    pass

python/ql/test/3/library-tests/PointsTo/imports/package/x.py

@@ -0,0 +1,2 @@
+
+from sys import *

python/ql/test/3/library-tests/PointsTo/imports/test.py

@@ -0,0 +1,34 @@
+from package \
+import module
+
+from package \
+import x
+#Should work correctly in nested scopes as well.
+
+class C(object):
+
+    from package import module2
+
+    def f(self):
+        from package import x
+
+from package import moduleX
+moduleX.Y
+
+#A small stdlib module to test version handling.
+import tty
+
+#Check imports of builtin-objects using import * with no corresponding variable.
+x.exc_info
+
+argv = 0
+
+try:
+    from sys import *
+except:
+    pass
+
+argv
+
+from socket import *
+timeout

python/ql/test/3/library-tests/modules/usage/ModuleUsage.expected

@@ -0,0 +1,6 @@
+| file://:0:0:0:0 | Module sys | isUsedAsModule |
+| file://:0:0:0:0 | Module sys.monitoring | isUsedAsModule |
+| imported.py:0:0:0:0 | Module imported | isUsedAsModule |
+| main.py:0:0:0:0 | Module main | isUsedAsScript |
+| myscript.py:0:0:0:0 | Script myscript | isUsedAsScript |
+| script:0:0:0:0 | Script script | isUsedAsScript |

python/ql/test/3/library-tests/modules/usage/ModuleUsage.ql

@@ -0,0 +1,16 @@
+import python
+
+from ModuleValue mv, string usage
+where
+  // builtin module has different name in Python 2 and 3
+  not mv = Module::builtinModule() and
+  (
+    mv.isUsedAsModule() and usage = "isUsedAsModule"
+    or
+    mv.isUsedAsScript() and usage = "isUsedAsScript"
+    or
+    not mv.isUsedAsModule() and
+    not mv.isUsedAsScript() and
+    usage = "<UNKNOWN>"
+  )
+select mv, usage

python/ql/test/3/library-tests/modules/usage/imported.py

@@ -0,0 +1,6 @@
+def func():
+    pass
+
+if __name__ == "__main__":
+    print("I could have done something interesting...")
+    print("but I didn't")

python/ql/test/3/library-tests/modules/usage/main.py

@@ -0,0 +1,5 @@
+import imported
+
+if __name__ == "__main__":
+    imported.func()
+    print('Done')

python/ql/test/3/library-tests/modules/usage/myscript.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+
+print("I'm actually a script you see ;)")

python/ql/test/3/library-tests/modules/usage/options

@@ -0,0 +1 @@
+semmle-extractor-options: --lang=3 -F script

python/ql/test/3/library-tests/modules/usage/script

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+
+print('Under construction :)')

python/ql/test/3/library-tests/types/properties/BuiltinProperties.expected

@@ -1,9 +1,12 @@
 | builtin-class object | __class__ | Property __class__ | method-wrapper __get__ | method-wrapper __set__ | method-wrapper __delete__ |
 | builtin-class type | __abstractmethods__ | Property __abstractmethods__ | method-wrapper __get__ | method-wrapper __set__ | method-wrapper __delete__ |
+| builtin-class type | __annotations__ | Property __annotations__ | method-wrapper __get__ | method-wrapper __set__ | method-wrapper __delete__ |
 | builtin-class type | __bases__ | Property __bases__ | method-wrapper __get__ | method-wrapper __set__ | method-wrapper __delete__ |
 | builtin-class type | __dict__ | Property __dict__ | method-wrapper __get__ | method-wrapper __set__ | method-wrapper __delete__ |
 | builtin-class type | __doc__ | Property __doc__ | method-wrapper __get__ | method-wrapper __set__ | method-wrapper __delete__ |
 | builtin-class type | __module__ | Property __module__ | method-wrapper __get__ | method-wrapper __set__ | method-wrapper __delete__ |
+| builtin-class type | __mro__ | Property __mro__ | method-wrapper __get__ | method-wrapper __set__ | method-wrapper __delete__ |
 | builtin-class type | __name__ | Property __name__ | method-wrapper __get__ | method-wrapper __set__ | method-wrapper __delete__ |
 | builtin-class type | __qualname__ | Property __qualname__ | method-wrapper __get__ | method-wrapper __set__ | method-wrapper __delete__ |
 | builtin-class type | __text_signature__ | Property __text_signature__ | method-wrapper __get__ | method-wrapper __set__ | method-wrapper __delete__ |
+| builtin-class type | __type_params__ | Property __type_params__ | method-wrapper __get__ | method-wrapper __set__ | method-wrapper __delete__ |

python/ql/test/3/query-tests/Summary/LinesOfCode.expected

@@ -0,0 +1 @@
+| 51 |

python/ql/test/3/query-tests/Summary/LinesOfCode.qlref

@@ -0,0 +1 @@
+Summary/LinesOfCode.ql

python/ql/test/3/query-tests/Summary/LinesOfUserCode.expected

@@ -0,0 +1 @@
+| 11 |

python/ql/test/3/query-tests/Summary/LinesOfUserCode.qlref

@@ -0,0 +1 @@
+Summary/LinesOfUserCode.ql

python/ql/test/3/query-tests/Summary/also_python_code

@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+
+# although this is actually Python code, it is not included by the extractor by default.
+
+print("this is also code")
+
+print("but just dummy code")

python/ql/test/3/query-tests/Summary/my_file.py

@@ -0,0 +1,26 @@
+"""
+module level docstring
+
+is not included
+"""
+# this line is not code
+
+# `tty` was chosen for stability over python versions (so we don't get diffrent results
+# on different computers, that has different versions of Python).
+#
+# According to https://github.com/python/cpython/tree/master/Lib (at 2021-04-23) `tty`
+# was last changed in 2001, so chances of this being changed in the future are slim.
+import tty
+
+s = """
+all these lines are code
+"""
+
+print(s)
+
+def func():
+    """
+    this string is a doc-string. Although the module-level docstring is not considered
+    code, this one apparently is ¯\_(ツ)_/¯
+    """
+    pass

python/ql/test/3/query-tests/Summary/not_python

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# Although this is valid python code, it should not be counted as such.
+
+print("foo")

python/ql/test/experimental/dataflow/coverage/datamodel.py

@@ -186,14 +186,20 @@ SINK(asyncio.run(c.coro(SOURCE))) # $ MISSING: flow
 class A:
 
   def __await__(self):
-    # yield SOURCE  -- see https://groups.google.com/g/dev-python/c/_lrrc-vp9TI?pli=1
-    return (yield from asyncio.coroutine(lambda: SOURCE)())
+    fut = asyncio.Future()
+    fut.set_result(SOURCE)
+    yield from fut
 
-async def agen(x):
+async def atest_custom_await_impl():
   a = A()
-  return await a
+  x = await a
+  # TODO: Figure out how to actually return something from our custom __await__
+  # implementation. The problem is we have to play nicely with the asyncio framework,
+  # which have their own expectations on what a return value from __await__ should look
+  # like.
+  assert x is None
+  SINK_F(x)
 
-SINK(asyncio.run(agen(SOURCE))) # $ MISSING: flow
 
 # Asynchronous generator functions
 # A function or method which is defined using async def and which uses the yield statement is called a asynchronous generator function. Such a function, when called, returns an asynchronous iterator object which can be used in an async for statement to execute the body of the function.

python/ql/test/library-tests/PointsTo/imports/options

@@ -1 +0,0 @@
-semmle-extractor-options: --max-import-depth=2 -r package

python/ql/test/library-tests/frameworks/aiofile/ConceptsTest.expected

@@ -0,0 +1,2 @@
+testFailures
+failures

python/ql/test/library-tests/frameworks/aiofile/ConceptsTest.ql

@@ -0,0 +1,2 @@
+import python
+import experimental.meta.ConceptsTest

python/ql/test/library-tests/frameworks/aiofile/FileSystemAccess.py

@@ -0,0 +1,4 @@
+from aiofile import async_open, AIOFile
+
+AIOFile("file", 'r') # $ getAPathArgument="file"
+async_open("file", "r") # $ getAPathArgument="file"

python/ql/test/library-tests/frameworks/aiofiles/ConceptsTest.expected

@@ -0,0 +1,2 @@
+testFailures
+failures

python/ql/test/library-tests/frameworks/aiofiles/ConceptsTest.ql

@@ -0,0 +1,2 @@
+import python
+import experimental.meta.ConceptsTest

python/ql/test/library-tests/frameworks/aiofiles/FileSystemAccess.py

@@ -0,0 +1,3 @@
+import aiofiles
+
+aiofiles.open("file", mode='r')  # $ getAPathArgument="file"

python/ql/test/library-tests/frameworks/anyio/ConceptsTest.expected

@@ -0,0 +1,2 @@
+testFailures
+failures

python/ql/test/library-tests/frameworks/anyio/ConceptsTest.ql

@@ -0,0 +1,2 @@
+import python
+import experimental.meta.ConceptsTest

python/ql/test/library-tests/frameworks/anyio/FileSystemAccess.py

@@ -0,0 +1,8 @@
+import anyio
+from anyio.streams.file import FileReadStream, FileWriteStream
+from anyio import Path
+
+anyio.open_file("file", 'r') # $ getAPathArgument="file"
+FileReadStream.from_path("file") # $ getAPathArgument="file"
+FileWriteStream.from_path("file") # $ getAPathArgument="file"
+Path("file") # $ getAPathArgument="file"

python/ql/test/library-tests/frameworks/baize/ConceptsTest.expected

@@ -0,0 +1,2 @@
+testFailures
+failures

python/ql/test/library-tests/frameworks/baize/ConceptsTest.ql

@@ -0,0 +1,2 @@
+import python
+import experimental.meta.ConceptsTest

python/ql/test/library-tests/frameworks/baize/FileSystemAccess.py

@@ -0,0 +1,3 @@
+from baize.asgi import FileResponse as baizeFileResponse
+
+baizeFileResponse("file")  # $ getAPathArgument="file"

python/ql/test/library-tests/frameworks/cherrypy/ConceptsTest.expected

@@ -0,0 +1,2 @@
+testFailures
+failures

python/ql/test/library-tests/frameworks/cherrypy/ConceptsTest.ql

@@ -0,0 +1,2 @@
+import python
+import experimental.meta.ConceptsTest

python/ql/test/library-tests/frameworks/cherrypy/FileSystemAccess.py

@@ -0,0 +1,8 @@
+import cherrypy
+from cherrypy.lib.static import serve_file, serve_download, staticfile
+
+serve_file("file")  # $ getAPathArgument="file"
+serve_download("file")  # $ getAPathArgument="file"
+staticfile("file")  # $ getAPathArgument="file"
+# root won't make this safe
+staticfile("file", root="/path/to/safe/dir") # $ getAPathArgument="file"

python/ql/test/library-tests/frameworks/sanic/ConceptsTest.expected

@@ -0,0 +1,2 @@
+testFailures
+failures

python/ql/test/library-tests/frameworks/sanic/ConceptsTest.ql

@@ -0,0 +1,2 @@
+import python
+import experimental.meta.ConceptsTest

python/ql/test/library-tests/frameworks/sanic/FileSystemAccess.py

@@ -0,0 +1,4 @@
+from sanic import response
+
+response.file("file")  # $ getAPathArgument="file"
+response.file_stream("file")  # $ getAPathArgument="file"

python/ql/test/library-tests/frameworks/starlette/ConceptsTest.expected

@@ -0,0 +1,2 @@
+testFailures
+failures

python/ql/test/library-tests/frameworks/starlette/ConceptsTest.ql

@@ -0,0 +1,2 @@
+import python
+import experimental.meta.ConceptsTest

python/ql/test/library-tests/frameworks/starlette/FileSystemAccess.py

@@ -0,0 +1,3 @@
+from starlette.responses import FileResponse
+
+FileResponse("file")  # $ getAPathArgument="file"

python/ql/test/library-tests/frameworks/stdlib/FileSystemAccess.py

@@ -20,6 +20,8 @@ builtins.open(file="file")  # $ getAPathArgument="file"
 
 io.open("file")  # $ getAPathArgument="file"
 io.open(file="file")  # $ getAPathArgument="file"
+io.open_code("file")  # $ getAPathArgument="file"
+io.FileIO("file")  # $ getAPathArgument="file"
 
 f = open("path") # $ getAPathArgument="path"
 f.write("foo") # $ getAPathArgument="path" fileWriteData="foo"

python/ql/test/library-tests/modules/usage/options

@@ -1 +0,0 @@
-semmle-extractor-options: -F script