Merge branch 'main' into jorgectf/python/ldapinsecureauth

python/change-notes/2021-08-30-port-modifying-default-query.md

@@ -0,0 +1,2 @@
+lgtm,codescanning
+* Updated _Modification of parameter with default_ (`py/modification-of-default-value`) query to use the new data flow library instead of the old taint tracking library and to remove the use of points-to analysis. You may see differences in the results found by the query, but overall this change should result in a more robust and accurate analysis.

python/change-notes/2021-09-02-add-Flask-SQLAlchemy-modeling.md

@@ -0,0 +1,2 @@
+lgtm,codescanning
+* Added modeling of SQL execution in the `Flask-SQLAlchemy` PyPI package, resulting in additional sinks for the SQL Injection query (`py/sql-injection`).

python/change-notes/2021-09-02-add-SQLAlchemy-modeling.md

@@ -0,0 +1,2 @@
+lgtm,codescanning
+* Added modeling of SQL execution in the `SQLAlchemy` PyPI package, resulting in additional sinks for the SQL Injection query (`py/sql-injection`). This modeling was originally [submitted as a contribution by @mrthankyou](https://github.com/github/codeql/pull/5680).

python/change-notes/2021-09-02-add-SQLAlchemyTextClauseInjection.md

@@ -0,0 +1,2 @@
+lgtm,codescanning
+* Expanded the query _SQL query built from user-controlled sources_ (`py/sql-injection`) to alert if user-input is added to a TextClause from SQLAlchemy, since that can lead to SQL injection.

python/ql/lib/semmle/python/Files.qll

@@ -1,9 +1,7 @@
 import python
 
 /** A file */
-class File extends Container {
-  File() { files(this, _, _, _, _) }
-
+class File extends Container, @file {
   /** DEPRECATED: Use `getAbsolutePath` instead. */
   deprecated override string getName() { result = this.getAbsolutePath() }
 
@@ -34,9 +32,7 @@ class File extends Container {
   }
 
   /** Gets a short name for this file (just the file name) */
-  string getShortName() {
-    exists(string simple, string ext | files(this, _, simple, ext, _) | result = simple + ext)
-  }
+  string getShortName() { result = this.getBaseName() }
 
   private int lastLine() {
     result = max(int i | exists(Location l | l.getFile() = this and l.getEndLine() = i))
@@ -55,7 +51,7 @@ class File extends Container {
     )
   }
 
-  override string getAbsolutePath() { files(this, result, _, _, _) }
+  override string getAbsolutePath() { files(this, result) }
 
   /** Gets the URL of this file. */
   override string getURL() { result = "file://" + this.getAbsolutePath() + ":0:0:0:0" }
@@ -118,15 +114,10 @@ private predicate occupied_line(File f, int n) {
 }
 
 /** A folder (directory) */
-class Folder extends Container {
-  Folder() { folders(this, _, _) }
-
+class Folder extends Container, @folder {
   /** DEPRECATED: Use `getAbsolutePath` instead. */
   deprecated override string getName() { result = this.getAbsolutePath() }
 
-  /** DEPRECATED: Use `getBaseName` instead. */
-  deprecated string getSimple() { folders(this, _, result) }
-
   /**
    * Holds if this element is at the specified location.
    * The location spans column `startcolumn` of line `startline` to
@@ -144,7 +135,7 @@ class Folder extends Container {
     endcolumn = 0
   }
 
-  override string getAbsolutePath() { folders(this, result, _) }
+  override string getAbsolutePath() { folders(this, result) }
 
   /** Gets the URL of this folder. */
   override string getURL() { result = "folder://" + this.getAbsolutePath() }

python/ql/lib/semmle/python/Flow.qll

@@ -653,6 +653,8 @@ class DefinitionNode extends ControlFlowNode {
   DefinitionNode() {
     exists(Assign a | a.getATarget().getAFlowNode() = this)
     or
+    exists(AnnAssign a | a.getTarget().getAFlowNode() = this and exists(a.getValue()))
+    or
     exists(Alias a | a.getAsname().getAFlowNode() = this)
     or
     augstore(_, this)
@@ -795,6 +797,9 @@ private AstNode assigned_value(Expr lhs) {
   /* lhs = result */
   exists(Assign a | a.getATarget() = lhs and result = a.getValue())
   or
+  /* lhs : annotation = result */
+  exists(AnnAssign a | a.getTarget() = lhs and result = a.getValue())
+  or
   /* import result as lhs */
   exists(Alias a | a.getAsname() = lhs and result = a.getValue())
   or

python/ql/lib/semmle/python/Frameworks.qll

@@ -13,6 +13,7 @@ private import semmle.python.frameworks.Dill
 private import semmle.python.frameworks.Django
 private import semmle.python.frameworks.Fabric
 private import semmle.python.frameworks.Flask
+private import semmle.python.frameworks.FlaskSqlAlchemy
 private import semmle.python.frameworks.Idna
 private import semmle.python.frameworks.Invoke
 private import semmle.python.frameworks.Jmespath
@@ -20,13 +21,14 @@ private import semmle.python.frameworks.MarkupSafe
 private import semmle.python.frameworks.Multidict
 private import semmle.python.frameworks.Mysql
 private import semmle.python.frameworks.MySQLdb
+private import semmle.python.frameworks.Peewee
 private import semmle.python.frameworks.Psycopg2
 private import semmle.python.frameworks.PyMySQL
 private import semmle.python.frameworks.Rsa
 private import semmle.python.frameworks.Simplejson
+private import semmle.python.frameworks.SqlAlchemy
 private import semmle.python.frameworks.Stdlib
 private import semmle.python.frameworks.Tornado
-private import semmle.python.frameworks.Peewee
 private import semmle.python.frameworks.Twisted
 private import semmle.python.frameworks.Ujson
 private import semmle.python.frameworks.Yaml

python/ql/lib/semmle/python/Stmts.qll

@@ -153,6 +153,12 @@ class ExceptStmt extends ExceptStmt_ {
   override Stmt getASubStatement() { result = this.getAStmt() }
 
   override Stmt getLastStatement() { result = this.getBody().getLastItem().getLastStatement() }
+
+  override Expr getType() {
+    result = super.getType() and not result instanceof Tuple
+    or
+    result = super.getType().(Tuple).getAnElt()
+  }
 }
 
 /** An assert statement, such as `assert a == b, "A is not equal to b"` */

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl.qll

@@ -3643,9 +3643,10 @@ private module Subpaths {
     PathNode arg, ParamNodeEx par, SummaryCtxSome sc, CallContext innercc, ReturnKindExt kind,
     NodeEx out, AccessPath apout
   ) {
-    pathThroughCallable(arg, out, _, apout) and
+    pathThroughCallable(arg, out, _, pragma[only_bind_into](apout)) and
     pathIntoCallable(arg, par, _, innercc, sc, _) and
-    paramFlowsThrough(kind, innercc, sc, apout, _, unbindConf(arg.getConfiguration()))
+    paramFlowsThrough(kind, innercc, sc, pragma[only_bind_into](apout), _,
+      unbindConf(arg.getConfiguration()))
   }
 
   /**
@@ -3690,8 +3691,8 @@ private module Subpaths {
    */
   predicate subpaths(PathNode arg, PathNodeImpl par, PathNodeMid ret, PathNodeMid out) {
     exists(ParamNodeEx p, NodeEx o, AccessPath apout |
-      arg.getASuccessor() = par and
-      arg.getASuccessor() = out and
+      pragma[only_bind_into](arg).getASuccessor() = par and
+      pragma[only_bind_into](arg).getASuccessor() = out and
       subpaths03(arg, p, ret, o, apout) and
       par.getNodeEx() = p and
       out.getNodeEx() = o and

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl2.qll

@@ -3643,9 +3643,10 @@ private module Subpaths {
     PathNode arg, ParamNodeEx par, SummaryCtxSome sc, CallContext innercc, ReturnKindExt kind,
     NodeEx out, AccessPath apout
   ) {
-    pathThroughCallable(arg, out, _, apout) and
+    pathThroughCallable(arg, out, _, pragma[only_bind_into](apout)) and
     pathIntoCallable(arg, par, _, innercc, sc, _) and
-    paramFlowsThrough(kind, innercc, sc, apout, _, unbindConf(arg.getConfiguration()))
+    paramFlowsThrough(kind, innercc, sc, pragma[only_bind_into](apout), _,
+      unbindConf(arg.getConfiguration()))
   }
 
   /**
@@ -3690,8 +3691,8 @@ private module Subpaths {
    */
   predicate subpaths(PathNode arg, PathNodeImpl par, PathNodeMid ret, PathNodeMid out) {
     exists(ParamNodeEx p, NodeEx o, AccessPath apout |
-      arg.getASuccessor() = par and
-      arg.getASuccessor() = out and
+      pragma[only_bind_into](arg).getASuccessor() = par and
+      pragma[only_bind_into](arg).getASuccessor() = out and
       subpaths03(arg, p, ret, o, apout) and
       par.getNodeEx() = p and
       out.getNodeEx() = o and

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl3.qll

@@ -3643,9 +3643,10 @@ private module Subpaths {
     PathNode arg, ParamNodeEx par, SummaryCtxSome sc, CallContext innercc, ReturnKindExt kind,
     NodeEx out, AccessPath apout
   ) {
-    pathThroughCallable(arg, out, _, apout) and
+    pathThroughCallable(arg, out, _, pragma[only_bind_into](apout)) and
     pathIntoCallable(arg, par, _, innercc, sc, _) and
-    paramFlowsThrough(kind, innercc, sc, apout, _, unbindConf(arg.getConfiguration()))
+    paramFlowsThrough(kind, innercc, sc, pragma[only_bind_into](apout), _,
+      unbindConf(arg.getConfiguration()))
   }
 
   /**
@@ -3690,8 +3691,8 @@ private module Subpaths {
    */
   predicate subpaths(PathNode arg, PathNodeImpl par, PathNodeMid ret, PathNodeMid out) {
     exists(ParamNodeEx p, NodeEx o, AccessPath apout |
-      arg.getASuccessor() = par and
-      arg.getASuccessor() = out and
+      pragma[only_bind_into](arg).getASuccessor() = par and
+      pragma[only_bind_into](arg).getASuccessor() = out and
       subpaths03(arg, p, ret, o, apout) and
       par.getNodeEx() = p and
       out.getNodeEx() = o and

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl4.qll

@@ -3643,9 +3643,10 @@ private module Subpaths {
     PathNode arg, ParamNodeEx par, SummaryCtxSome sc, CallContext innercc, ReturnKindExt kind,
     NodeEx out, AccessPath apout
   ) {
-    pathThroughCallable(arg, out, _, apout) and
+    pathThroughCallable(arg, out, _, pragma[only_bind_into](apout)) and
     pathIntoCallable(arg, par, _, innercc, sc, _) and
-    paramFlowsThrough(kind, innercc, sc, apout, _, unbindConf(arg.getConfiguration()))
+    paramFlowsThrough(kind, innercc, sc, pragma[only_bind_into](apout), _,
+      unbindConf(arg.getConfiguration()))
   }
 
   /**
@@ -3690,8 +3691,8 @@ private module Subpaths {
    */
   predicate subpaths(PathNode arg, PathNodeImpl par, PathNodeMid ret, PathNodeMid out) {
     exists(ParamNodeEx p, NodeEx o, AccessPath apout |
-      arg.getASuccessor() = par and
-      arg.getASuccessor() = out and
+      pragma[only_bind_into](arg).getASuccessor() = par and
+      pragma[only_bind_into](arg).getASuccessor() = out and
       subpaths03(arg, p, ret, o, apout) and
       par.getNodeEx() = p and
       out.getNodeEx() = o and

python/ql/lib/semmle/python/frameworks/FlaskSqlAlchemy.qll

@@ -0,0 +1,56 @@
+/**
+ * Provides classes modeling security-relevant aspects of the `Flask-SQLAlchemy` PyPI package
+ * (imported by `flask_sqlalchemy`).
+ * See
+ *  - https://pypi.org/project/Flask-SQLAlchemy/
+ *  - https://flask-sqlalchemy.palletsprojects.com/en/2.x/
+ */
+
+private import python
+private import semmle.python.dataflow.new.DataFlow
+private import semmle.python.dataflow.new.TaintTracking
+private import semmle.python.ApiGraphs
+private import semmle.python.Concepts
+private import semmle.python.frameworks.SqlAlchemy
+
+/**
+ * INTERNAL: Do not use.
+ *
+ * Provides models for the `Flask-SQLAlchemy` PyPI package (imported by `flask_sqlalchemy`).
+ * See
+ *  - https://pypi.org/project/Flask-SQLAlchemy/
+ *  - https://flask-sqlalchemy.palletsprojects.com/en/2.x/
+ */
+private module FlaskSqlAlchemy {
+  /** Gets an instance of `flask_sqlalchemy.SQLAlchemy` */
+  private API::Node dbInstance() {
+    result = API::moduleImport("flask_sqlalchemy").getMember("SQLAlchemy").getReturn()
+  }
+
+  /** A call to the `text` method on a DB. */
+  private class DbTextCall extends SqlAlchemy::TextClause::TextClauseConstruction {
+    DbTextCall() { this = dbInstance().getMember("text").getACall() }
+  }
+
+  /** Access on a DB resulting in an Engine */
+  private class DbEngine extends SqlAlchemy::Engine::InstanceSource {
+    DbEngine() {
+      this = dbInstance().getMember("engine").getAUse()
+      or
+      this = dbInstance().getMember("get_engine").getACall()
+    }
+  }
+
+  /** Access on a DB resulting in a Session */
+  private class DbSession extends SqlAlchemy::Session::InstanceSource {
+    DbSession() {
+      this = dbInstance().getMember("session").getAUse()
+      or
+      this = dbInstance().getMember("create_session").getReturn().getACall()
+      or
+      this = dbInstance().getMember("create_session").getReturn().getMember("begin").getACall()
+      or
+      this = dbInstance().getMember("create_scoped_session").getACall()
+    }
+  }
+}

python/ql/lib/semmle/python/frameworks/SqlAlchemy.qll

@@ -0,0 +1,345 @@
+/**
+ * Provides classes modeling security-relevant aspects of the `SQLAlchemy` PyPI package.
+ * See
+ *  - https://pypi.org/project/SQLAlchemy/
+ *  - https://docs.sqlalchemy.org/en/14/index.html
+ */
+
+private import python
+private import semmle.python.dataflow.new.DataFlow
+private import semmle.python.dataflow.new.TaintTracking
+private import semmle.python.ApiGraphs
+private import semmle.python.Concepts
+// This import is done like this to avoid importing the deprecated top-level things that
+// would pollute the namespace
+private import semmle.python.frameworks.PEP249::PEP249 as PEP249
+
+/**
+ * INTERNAL: Do not use.
+ *
+ * Provides models for the `SQLAlchemy` PyPI package.
+ * See
+ *  - https://pypi.org/project/SQLAlchemy/
+ *  - https://docs.sqlalchemy.org/en/14/index.html
+ */
+module SqlAlchemy {
+  /**
+   * Provides models for the `sqlalchemy.engine.Engine` and `sqlalchemy.future.Engine` classes.
+   *
+   * These are so similar that we model both in the same way.
+   *
+   * See
+   *  - https://docs.sqlalchemy.org/en/14/core/connections.html#sqlalchemy.engine.Engine
+   *  - https://docs.sqlalchemy.org/en/14/core/future.html#sqlalchemy.future.Engine
+   */
+  module Engine {
+    /** Gets a reference to a SQLAlchemy Engine class. */
+    private API::Node classRef() {
+      result = API::moduleImport("sqlalchemy").getMember("engine").getMember("Engine")
+      or
+      result = API::moduleImport("sqlalchemy").getMember("future").getMember("Engine")
+    }
+
+    /**
+     * A source of instances of a SQLAlchemy Engine, extend this class to model new instances.
+     *
+     * This can include instantiations of the class, return values from function
+     * calls, or a special parameter that will be set when functions are called by an external
+     * library.
+     *
+     * Use the predicate `Engine::instance()` to get references to instances of a SQLAlchemy Engine.
+     */
+    abstract class InstanceSource extends DataFlow::LocalSourceNode { }
+
+    private class EngineConstruction extends InstanceSource, DataFlow::CallCfgNode {
+      EngineConstruction() {
+        this = classRef().getACall()
+        or
+        this = API::moduleImport("sqlalchemy").getMember("create_engine").getACall()
+        or
+        this =
+          API::moduleImport("sqlalchemy").getMember("future").getMember("create_engine").getACall()
+        or
+        this.(DataFlow::MethodCallNode).calls(instance(), "execution_options")
+      }
+    }
+
+    /** Gets a reference to an instance of a SQLAlchemy Engine. */
+    private DataFlow::TypeTrackingNode instance(DataFlow::TypeTracker t) {
+      t.start() and
+      result instanceof InstanceSource
+      or
+      exists(DataFlow::TypeTracker t2 | result = instance(t2).track(t2, t))
+    }
+
+    /** Gets a reference to an instance of a SQLAlchemy Engine. */
+    DataFlow::Node instance() { instance(DataFlow::TypeTracker::end()).flowsTo(result) }
+  }
+
+  /**
+   * Provides models for the `sqlalchemy.engine.base.Connection` and `sqlalchemy.future.Connection` classes.
+   *
+   * These are so similar that we model both in the same way.
+   *
+   * See
+   * - https://docs.sqlalchemy.org/en/14/core/connections.html#sqlalchemy.engine.Connection
+   * - https://docs.sqlalchemy.org/en/14/core/future.html#sqlalchemy.future.Connection
+   */
+  module Connection {
+    /** Gets a reference to a SQLAlchemy Connection class. */
+    private API::Node classRef() {
+      result =
+        API::moduleImport("sqlalchemy")
+            .getMember("engine")
+            .getMember("base")
+            .getMember("Connection")
+      or
+      result = API::moduleImport("sqlalchemy").getMember("future").getMember("Connection")
+    }
+
+    /**
+     * A source of instances of a SQLAlchemy Connection, extend this class to model new instances.
+     *
+     * This can include instantiations of the class, return values from function
+     * calls, or a special parameter that will be set when functions are called by an external
+     * library.
+     *
+     * Use the predicate `Connection::instance()` to get references to instances of a SQLAlchemy Connection.
+     */
+    abstract class InstanceSource extends DataFlow::LocalSourceNode { }
+
+    private class ConnectionConstruction extends InstanceSource, DataFlow::CallCfgNode {
+      ConnectionConstruction() {
+        this = classRef().getACall()
+        or
+        this.(DataFlow::MethodCallNode).calls(Engine::instance(), ["begin", "connect"])
+        or
+        this.(DataFlow::MethodCallNode).calls(instance(), "connect")
+        or
+        this.(DataFlow::MethodCallNode).calls(instance(), "execution_options")
+      }
+    }
+
+    /** Gets a reference to an instance of a SQLAlchemy Connection. */
+    private DataFlow::TypeTrackingNode instance(DataFlow::TypeTracker t) {
+      t.start() and
+      result instanceof InstanceSource
+      or
+      exists(DataFlow::TypeTracker t2 | result = instance(t2).track(t2, t))
+    }
+
+    /** Gets a reference to an instance of a SQLAlchemy Connection. */
+    DataFlow::Node instance() { instance(DataFlow::TypeTracker::end()).flowsTo(result) }
+  }
+
+  /**
+   * Provides models for the underlying DB-API Connection of a SQLAlchemy Connection.
+   *
+   * See https://docs.sqlalchemy.org/en/14/core/connections.html#dbapi-connections.
+   */
+  module DBAPIConnection {
+    /**
+     * A source of instances of DB-API Connections, extend this class to model new instances.
+     *
+     * This can include instantiations of the class, return values from function
+     * calls, or a special parameter that will be set when functions are called by an external
+     * library.
+     *
+     * Use the predicate `DBAPIConnection::instance()` to get references to instances of DB-API Connections.
+     */
+    abstract class InstanceSource extends DataFlow::LocalSourceNode { }
+
+    private class DBAPIConnectionSources extends InstanceSource, PEP249::Connection::InstanceSource {
+      DBAPIConnectionSources() {
+        this.(DataFlow::MethodCallNode).calls(Engine::instance(), "raw_connection")
+        or
+        this.(DataFlow::AttrRead).accesses(Connection::instance(), "connection")
+      }
+    }
+
+    /** Gets a reference to an instance of DB-API Connections. */
+    private DataFlow::TypeTrackingNode instance(DataFlow::TypeTracker t) {
+      t.start() and
+      result instanceof InstanceSource
+      or
+      exists(DataFlow::TypeTracker t2 | result = instance(t2).track(t2, t))
+    }
+
+    /** Gets a reference to an instance of DB-API Connections. */
+    DataFlow::Node instance() { instance(DataFlow::TypeTracker::end()).flowsTo(result) }
+  }
+
+  /**
+   * Provides models for the `sqlalchemy.orm.Session` class
+   *
+   * See
+   * - https://docs.sqlalchemy.org/en/14/orm/session_api.html#sqlalchemy.orm.Session
+   * - https://docs.sqlalchemy.org/en/14/orm/session_basics.html
+   */
+  module Session {
+    /** Gets a reference to the `sqlalchemy.orm.Session` class. */
+    private API::Node classRef() {
+      result = API::moduleImport("sqlalchemy").getMember("orm").getMember("Session")
+    }
+
+    /**
+     * A source of instances of `sqlalchemy.orm.Session`, extend this class to model new instances.
+     *
+     * This can include instantiations of the class, return values from function
+     * calls, or a special parameter that will be set when functions are called by an external
+     * library.
+     *
+     * Use the predicate `Session::instance()` to get references to instances of `sqlalchemy.orm.Session`.
+     */
+    abstract class InstanceSource extends DataFlow::LocalSourceNode { }
+
+    private class SessionConstruction extends InstanceSource, DataFlow::CallCfgNode {
+      SessionConstruction() {
+        this = classRef().getACall()
+        or
+        this =
+          API::moduleImport("sqlalchemy")
+              .getMember("orm")
+              .getMember("sessionmaker")
+              .getReturn()
+              .getACall()
+        or
+        this =
+          API::moduleImport("sqlalchemy")
+              .getMember("orm")
+              .getMember("sessionmaker")
+              .getReturn()
+              .getMember("begin")
+              .getACall()
+      }
+    }
+
+    /** Gets a reference to an instance of `sqlalchemy.orm.Session`. */
+    private DataFlow::TypeTrackingNode instance(DataFlow::TypeTracker t) {
+      t.start() and
+      result instanceof InstanceSource
+      or
+      exists(DataFlow::TypeTracker t2 | result = instance(t2).track(t2, t))
+    }
+
+    /** Gets a reference to an instance of `sqlalchemy.orm.Session`. */
+    DataFlow::Node instance() { instance(DataFlow::TypeTracker::end()).flowsTo(result) }
+  }
+
+  /**
+   * A call to `execute` on a SQLAlchemy Engine, Connection, or Session.
+   * See
+   *  - https://docs.sqlalchemy.org/en/14/core/connections.html#sqlalchemy.engine.Engine.execute
+   *  - https://docs.sqlalchemy.org/en/14/core/connections.html#sqlalchemy.engine.Connection.execute
+   *  - https://docs.sqlalchemy.org/en/14/core/future.html#sqlalchemy.future.Connection.execute
+   *  - https://docs.sqlalchemy.org/en/14/orm/session_api.html#sqlalchemy.orm.Session.execute
+   */
+  private class SqlAlchemyExecuteCall extends DataFlow::MethodCallNode, SqlExecution::Range {
+    SqlAlchemyExecuteCall() {
+      this.calls(Engine::instance(), "execute")
+      or
+      this.calls(Connection::instance(), "execute")
+      or
+      this.calls(Session::instance(), "execute")
+    }
+
+    override DataFlow::Node getSql() { result in [this.getArg(0), this.getArgByName("statement")] }
+  }
+
+  /**
+   * A call to `exec_driver_sql` on a SQLAlchemy Connection.
+   * See
+   *  - https://docs.sqlalchemy.org/en/14/core/connections.html#sqlalchemy.engine.Connection.exec_driver_sql
+   *  - https://docs.sqlalchemy.org/en/14/core/future.html#sqlalchemy.future.Connection.exec_driver_sql
+   */
+  private class SqlAlchemyExecDriverSqlCall extends DataFlow::MethodCallNode, SqlExecution::Range {
+    SqlAlchemyExecDriverSqlCall() { this.calls(Connection::instance(), "exec_driver_sql") }
+
+    override DataFlow::Node getSql() { result in [this.getArg(0), this.getArgByName("statement")] }
+  }
+
+  /**
+   * A call to `scalar` on a SQLAlchemy Engine, Connection, or Session.
+   * See
+   *  - https://docs.sqlalchemy.org/en/14/core/connections.html#sqlalchemy.engine.Engine.scalar
+   *  - https://docs.sqlalchemy.org/en/14/core/connections.html#sqlalchemy.engine.Connection.scalar
+   *  - https://docs.sqlalchemy.org/en/14/core/future.html#sqlalchemy.future.Connection.scalar
+   *  - https://docs.sqlalchemy.org/en/14/orm/session_api.html#sqlalchemy.orm.Session.scalar
+   */
+  private class SqlAlchemyScalarCall extends DataFlow::MethodCallNode, SqlExecution::Range {
+    SqlAlchemyScalarCall() {
+      this.calls(Engine::instance(), "scalar")
+      or
+      this.calls(Connection::instance(), "scalar")
+      or
+      this.calls(Session::instance(), "scalar")
+    }
+
+    override DataFlow::Node getSql() {
+      result in [this.getArg(0), this.getArgByName("statement"), this.getArgByName("object_")]
+    }
+  }
+
+  /**
+   * Provides models for the `sqlalchemy.sql.expression.TextClause` class,
+   * which represents a textual SQL string directly.
+   *
+   * ```py
+   * session.query(For14).filter_by(description=sqlalchemy.text(f"'{user_input}'")).all()
+   * ```
+   *
+   * Initially I wanted to add lots of additional taint steps for such that the normal
+   * SQL injection query would be able to find cases as the one above where an ORM query
+   * includes a TextClause that includes user-input directly... But that presented 2
+   * problems:
+   *
+   * - which part of the query construction above should be marked as SQL to fit our
+   *   `SqlExecution` concept. Nothing really fits this well, since all the SQL
+   *   execution happens under the hood.
+   * - This would require a LOT of modeling for these additional taint steps, since
+   *   there are many many constructs we would need to have models for. (see the 2
+   *   examples below)
+   *
+   * So instead we flag user-input to a TextClause with its' own query
+   * (`py/sqlalchemy-textclause-injection`). And so we don't highlight any parts of an
+   * ORM constructed query such as these as containing SQL, and don't need the additional
+   * taint steps either.
+   *
+   * See
+   * - https://docs.sqlalchemy.org/en/14/core/sqlelement.html#sqlalchemy.sql.expression.TextClause.
+   * - https://docs.sqlalchemy.org/en/14/core/sqlelement.html#sqlalchemy.sql.expression.text
+   */
+  module TextClause {
+    /**
+     * A construction of a `sqlalchemy.sql.expression.TextClause`, which represents a
+     * textual SQL string directly.
+     */
+    abstract class TextClauseConstruction extends DataFlow::CallCfgNode {
+      /** Gets the argument that specifies the SQL text. */
+      DataFlow::Node getTextArg() { result in [this.getArg(0), this.getArgByName("text")] }
+    }
+
+    /** `TextClause` constructions from the `sqlalchemy` package. */
+    private class DefaultTextClauseConstruction extends TextClauseConstruction {
+      DefaultTextClauseConstruction() {
+        this = API::moduleImport("sqlalchemy").getMember("text").getACall()
+        or
+        this = API::moduleImport("sqlalchemy").getMember("sql").getMember("text").getACall()
+        or
+        this =
+          API::moduleImport("sqlalchemy")
+              .getMember("sql")
+              .getMember("expression")
+              .getMember("text")
+              .getACall()
+        or
+        this =
+          API::moduleImport("sqlalchemy")
+              .getMember("sql")
+              .getMember("expression")
+              .getMember("TextClause")
+              .getACall()
+      }
+    }
+  }
+}

python/ql/lib/semmle/python/regex.qll

@@ -773,15 +773,18 @@ abstract class RegexString extends Expr {
    * string is empty.
    */
   predicate multiples(int start, int end, string lower, string upper) {
-    this.getChar(start) = "{" and
-    this.getChar(end - 1) = "}" and
-    exists(string inner | inner = this.getText().substring(start + 1, end - 1) |
-      inner.regexpMatch("[0-9]+") and
+    exists(string text, string match, string inner |
+      text = this.getText() and
+      end = start + match.length() and
+      inner = match.substring(1, match.length() - 1)
+    |
+      match = text.regexpFind("\\{[0-9]+\\}", _, start) and
       lower = inner and
       upper = lower
       or
-      inner.regexpMatch("[0-9]*,[0-9]*") and
-      exists(int commaIndex | commaIndex = inner.indexOf(",") |
+      match = text.regexpFind("\\{[0-9]*,[0-9]*\\}", _, start) and
+      exists(int commaIndex |
+        commaIndex = inner.indexOf(",") and
         lower = inner.prefix(commaIndex) and
         upper = inner.suffix(commaIndex + 1)
       )

python/ql/lib/semmle/python/security/dataflow/SqlInjectionCustomizations.qll

@@ -9,6 +9,7 @@ private import semmle.python.dataflow.new.DataFlow
 private import semmle.python.Concepts
 private import semmle.python.dataflow.new.RemoteFlowSources
 private import semmle.python.dataflow.new.BarrierGuards
+private import semmle.python.frameworks.SqlAlchemy
 
 /**
  * Provides default sources, sinks and sanitizers for detecting
@@ -48,6 +49,13 @@ module SqlInjection {
     SqlExecutionAsSink() { this = any(SqlExecution e).getSql() }
   }
 
+  /**
+   * The text argument of a SQLAlchemy TextClause construction, considered as a flow sink.
+   */
+  class TextArgAsSink extends Sink {
+    TextArgAsSink() { this = any(SqlAlchemy::TextClause::TextClauseConstruction tcc).getTextArg() }
+  }
+
   /**
    * A comparison with a constant string, considered as a sanitizer-guard.
    */

python/ql/lib/semmlecode.python.dbscheme

@@ -120,16 +120,11 @@ svnchurn(
         Python dbscheme
 ****************************/
 
-/* fromSource is ignored */
 files(unique int id: @file,
-      varchar(900) name: string ref,
-      varchar(900) simple: string ref,
-      varchar(900) ext: string ref,
-      int fromSource: int ref);
+      varchar(900) name: string ref);
 
 folders(unique int id: @folder,
-        varchar(900) name: string ref,
-        varchar(900) simple: string ref);
+        varchar(900) name: string ref);
 
 @container = @folder | @file;
 

python/ql/lib/semmlecode.python.dbscheme.stats

@@ -4331,18 +4331,6 @@
 <k>name</k>
 <v>3066</v>
 </e>
-<e>
-<k>simple</k>
-<v>1294</v>
-</e>
-<e>
-<k>ext</k>
-<v>1</v>
-</e>
-<e>
-<k>fromSource</k>
-<v>1</v>
-</e>
 </columnsizes>
 <dependencies>
 <dep>
@@ -4362,54 +4350,6 @@
 </val>
 </dep>
 <dep>
-<src>id</src>
-<trg>simple</trg>
-<val>
-<hist>
-<budget>12</budget>
-<bs>
-<b>
-<a>1</a>
-<b>2</b>
-<v>3066</v>
-</b>
-</bs>
-</hist>
-</val>
-</dep>
-<dep>
-<src>id</src>
-<trg>ext</trg>
-<val>
-<hist>
-<budget>12</budget>
-<bs>
-<b>
-<a>1</a>
-<b>2</b>
-<v>3066</v>
-</b>
-</bs>
-</hist>
-</val>
-</dep>
-<dep>
-<src>id</src>
-<trg>fromSource</trg>
-<val>
-<hist>
-<budget>12</budget>
-<bs>
-<b>
-<a>1</a>
-<b>2</b>
-<v>3066</v>
-</b>
-</bs>
-</hist>
-</val>
-</dep>
-<dep>
 <src>name</src>
 <trg>id</trg>
 <val>
@@ -4425,276 +4365,6 @@
 </hist>
 </val>
 </dep>
-<dep>
-<src>name</src>
-<trg>simple</trg>
-<val>
-<hist>
-<budget>12</budget>
-<bs>
-<b>
-<a>1</a>
-<b>2</b>
-<v>3066</v>
-</b>
-</bs>
-</hist>
-</val>
-</dep>
-<dep>
-<src>name</src>
-<trg>ext</trg>
-<val>
-<hist>
-<budget>12</budget>
-<bs>
-<b>
-<a>1</a>
-<b>2</b>
-<v>3066</v>
-</b>
-</bs>
-</hist>
-</val>
-</dep>
-<dep>
-<src>name</src>
-<trg>fromSource</trg>
-<val>
-<hist>
-<budget>12</budget>
-<bs>
-<b>
-<a>1</a>
-<b>2</b>
-<v>3066</v>
-</b>
-</bs>
-</hist>
-</val>
-</dep>
-<dep>
-<src>simple</src>
-<trg>id</trg>
-<val>
-<hist>
-<budget>12</budget>
-<bs>
-<b>
-<a>1</a>
-<b>2</b>
-<v>1058</v>
-</b>
-<b>
-<a>2</a>
-<b>3</b>
-<v>132</v>
-</b>
-<b>
-<a>3</a>
-<b>38</b>
-<v>98</v>
-</b>
-<b>
-<a>47</a>
-<b>646</b>
-<v>6</v>
-</b>
-</bs>
-</hist>
-</val>
-</dep>
-<dep>
-<src>simple</src>
-<trg>name</trg>
-<val>
-<hist>
-<budget>12</budget>
-<bs>
-<b>
-<a>1</a>
-<b>2</b>
-<v>1058</v>
-</b>
-<b>
-<a>2</a>
-<b>3</b>
-<v>132</v>
-</b>
-<b>
-<a>3</a>
-<b>38</b>
-<v>98</v>
-</b>
-<b>
-<a>47</a>
-<b>646</b>
-<v>6</v>
-</b>
-</bs>
-</hist>
-</val>
-</dep>
-<dep>
-<src>simple</src>
-<trg>ext</trg>
-<val>
-<hist>
-<budget>12</budget>
-<bs>
-<b>
-<a>1</a>
-<b>2</b>
-<v>1294</v>
-</b>
-</bs>
-</hist>
-</val>
-</dep>
-<dep>
-<src>simple</src>
-<trg>fromSource</trg>
-<val>
-<hist>
-<budget>12</budget>
-<bs>
-<b>
-<a>1</a>
-<b>2</b>
-<v>1294</v>
-</b>
-</bs>
-</hist>
-</val>
-</dep>
-<dep>
-<src>ext</src>
-<trg>id</trg>
-<val>
-<hist>
-<budget>12</budget>
-<bs>
-<b>
-<a>3066</a>
-<b>3067</b>
-<v>1</v>
-</b>
-</bs>
-</hist>
-</val>
-</dep>
-<dep>
-<src>ext</src>
-<trg>name</trg>
-<val>
-<hist>
-<budget>12</budget>
-<bs>
-<b>
-<a>3066</a>
-<b>3067</b>
-<v>1</v>
-</b>
-</bs>
-</hist>
-</val>
-</dep>
-<dep>
-<src>ext</src>
-<trg>simple</trg>
-<val>
-<hist>
-<budget>12</budget>
-<bs>
-<b>
-<a>1294</a>
-<b>1295</b>
-<v>1</v>
-</b>
-</bs>
-</hist>
-</val>
-</dep>
-<dep>
-<src>ext</src>
-<trg>fromSource</trg>
-<val>
-<hist>
-<budget>12</budget>
-<bs>
-<b>
-<a>1</a>
-<b>2</b>
-<v>1</v>
-</b>
-</bs>
-</hist>
-</val>
-</dep>
-<dep>
-<src>fromSource</src>
-<trg>id</trg>
-<val>
-<hist>
-<budget>12</budget>
-<bs>
-<b>
-<a>3066</a>
-<b>3067</b>
-<v>1</v>
-</b>
-</bs>
-</hist>
-</val>
-</dep>
-<dep>
-<src>fromSource</src>
-<trg>name</trg>
-<val>
-<hist>
-<budget>12</budget>
-<bs>
-<b>
-<a>3066</a>
-<b>3067</b>
-<v>1</v>
-</b>
-</bs>
-</hist>
-</val>
-</dep>
-<dep>
-<src>fromSource</src>
-<trg>simple</trg>
-<val>
-<hist>
-<budget>12</budget>
-<bs>
-<b>
-<a>1294</a>
-<b>1295</b>
-<v>1</v>
-</b>
-</bs>
-</hist>
-</val>
-</dep>
-<dep>
-<src>fromSource</src>
-<trg>ext</trg>
-<val>
-<hist>
-<budget>12</budget>
-<bs>
-<b>
-<a>1</a>
-<b>2</b>
-<v>1</v>
-</b>
-</bs>
-</hist>
-</val>
-</dep>
 </dependencies>
 </relation>
 <relation>
@@ -4709,10 +4379,6 @@
 <k>name</k>
 <v>686</v>
 </e>
-<e>
-<k>simple</k>
-<v>538</v>
-</e>
 </columnsizes>
 <dependencies>
 <dep>
@@ -4732,22 +4398,6 @@
 </val>
 </dep>
 <dep>
-<src>id</src>
-<trg>simple</trg>
-<val>
-<hist>
-<budget>12</budget>
-<bs>
-<b>
-<a>1</a>
-<b>2</b>
-<v>686</v>
-</b>
-</bs>
-</hist>
-</val>
-</dep>
-<dep>
 <src>name</src>
 <trg>id</trg>
 <val>
@@ -4763,74 +4413,6 @@
 </hist>
 </val>
 </dep>
-<dep>
-<src>name</src>
-<trg>simple</trg>
-<val>
-<hist>
-<budget>12</budget>
-<bs>
-<b>
-<a>1</a>
-<b>2</b>
-<v>686</v>
-</b>
-</bs>
-</hist>
-</val>
-</dep>
-<dep>
-<src>simple</src>
-<trg>id</trg>
-<val>
-<hist>
-<budget>12</budget>
-<bs>
-<b>
-<a>1</a>
-<b>2</b>
-<v>481</v>
-</b>
-<b>
-<a>2</a>
-<b>4</b>
-<v>45</v>
-</b>
-<b>
-<a>4</a>
-<b>27</b>
-<v>12</v>
-</b>
-</bs>
-</hist>
-</val>
-</dep>
-<dep>
-<src>simple</src>
-<trg>name</trg>
-<val>
-<hist>
-<budget>12</budget>
-<bs>
-<b>
-<a>1</a>
-<b>2</b>
-<v>481</v>
-</b>
-<b>
-<a>2</a>
-<b>4</b>
-<v>45</v>
-</b>
-<b>
-<a>4</a>
-<b>27</b>
-<v>12</v>
-</b>
-</bs>
-</hist>
-</val>
-</dep>
 </dependencies>
 </relation>
 <relation>

python/ql/src/Functions/ModificationOfParameterWithDefault.ql

@@ -12,88 +12,12 @@
  */
 
 import python
-import semmle.python.security.Paths
+import semmle.python.functions.ModificationOfParameterWithDefault
+import DataFlow::PathGraph
 
-predicate safe_method(string name) {
-  name = "count" or
-  name = "index" or
-  name = "copy" or
-  name = "get" or
-  name = "has_key" or
-  name = "items" or
-  name = "keys" or
-  name = "values" or
-  name = "iteritems" or
-  name = "iterkeys" or
-  name = "itervalues" or
-  name = "__contains__" or
-  name = "__getitem__" or
-  name = "__getattribute__"
-}
-
-/** Gets the truthiness (non emptyness) of the default of `p` if that value is mutable */
-private boolean mutableDefaultValue(Parameter p) {
-  exists(Dict d | p.getDefault() = d |
-    exists(d.getAKey()) and result = true
-    or
-    not exists(d.getAKey()) and result = false
-  )
-  or
-  exists(List l | p.getDefault() = l |
-    exists(l.getAnElt()) and result = true
-    or
-    not exists(l.getAnElt()) and result = false
-  )
-}
-
-class NonEmptyMutableValue extends TaintKind {
-  NonEmptyMutableValue() { this = "non-empty mutable value" }
-}
-
-class EmptyMutableValue extends TaintKind {
-  EmptyMutableValue() { this = "empty mutable value" }
-
-  override boolean booleanValue() { result = false }
-}
-
-class MutableDefaultValue extends TaintSource {
-  boolean nonEmpty;
-
-  MutableDefaultValue() { nonEmpty = mutableDefaultValue(this.(NameNode).getNode()) }
-
-  override string toString() { result = "mutable default value" }
-
-  override predicate isSourceOf(TaintKind kind) {
-    nonEmpty = false and kind instanceof EmptyMutableValue
-    or
-    nonEmpty = true and kind instanceof NonEmptyMutableValue
-  }
-}
-
-private ClassValue mutable_class() {
-  result = Value::named("list") or
-  result = Value::named("dict")
-}
-
-class Mutation extends TaintSink {
-  Mutation() {
-    exists(AugAssign a | a.getTarget().getAFlowNode() = this)
-    or
-    exists(Call c, Attribute a | c.getFunc() = a |
-      a.getObject().getAFlowNode() = this and
-      not safe_method(a.getName()) and
-      this.(ControlFlowNode).pointsTo().getClass() = mutable_class()
-    )
-  }
-
-  override predicate sinks(TaintKind kind) {
-    kind instanceof EmptyMutableValue
-    or
-    kind instanceof NonEmptyMutableValue
-  }
-}
-
-from TaintedPathSource src, TaintedPathSink sink
-where src.flowsTo(sink)
-select sink.getSink(), src, sink, "$@ flows to here and is mutated.", src.getSource(),
+from
+  ModificationOfParameterWithDefault::Configuration config, DataFlow::PathNode source,
+  DataFlow::PathNode sink
+where config.hasFlowPath(source, sink)
+select sink.getNode(), source, sink, "$@ flows to here and is mutated.", source.getNode(),
   "Default value"

python/ql/src/Security/CWE-089/SqlInjection.qhelp

@@ -9,6 +9,13 @@ If a database query (such as a SQL or NoSQL query) is built from
 user-provided data without sufficient sanitization, a user
 may be able to run malicious database queries.
 </p>
+
+<p>
+This also includes using the <code>TextClause</code> class in the
+<code><a href="https://pypi.org/project/SQLAlchemy/">SQLAlchemy</a></code> PyPI package,
+which is used to represent a literal SQL fragment and is inserted directly into the
+final SQL when used in a query built using the ORM.
+</p>
 </overview>
 
 <recommendation>
@@ -52,5 +59,6 @@ vulnerable to SQL injection attacks. In this example, if <code>username</code> w
 <references>
 <li>Wikipedia: <a href="https://en.wikipedia.org/wiki/SQL_injection">SQL injection</a>.</li>
 <li>OWASP: <a href="https://cheatsheetseries.owasp.org/cheatsheets/SQL_Injection_Prevention_Cheat_Sheet.html">SQL Injection Prevention Cheat Sheet</a>.</li>
+<li><a href="https://docs.sqlalchemy.org/en/14/core/sqlelement.html#sqlalchemy.sql.expression.text.params.text">SQLAlchemy documentation for TextClause</a>.</li>
 </references>
 </qhelp>

python/ql/src/Variables/UnusedLocalVariable.ql

@@ -19,6 +19,7 @@ predicate unused_local(Name unused, LocalVariable v) {
     def.getVariable() = v and
     def.isUnused() and
     not exists(def.getARedef()) and
+    not exists(annotation_without_assignment(v)) and
     def.isRelevant() and
     not v = any(Nonlocal n).getAVariable() and
     not exists(def.getNode().getParentNode().(FunctionDef).getDefinedFunction().getADecorator()) and
@@ -26,6 +27,17 @@ predicate unused_local(Name unused, LocalVariable v) {
   )
 }
 
+/**
+ * Gets any annotation of the local variable `v` that does not also reassign its value.
+ *
+ * TODO: This predicate should not be needed. Rather, annotated "assignments" that do not actually
+ * assign a value should not result in the creation of an SSA variable (which then goes unused).
+ */
+private AnnAssign annotation_without_assignment(LocalVariable v) {
+  result.getTarget() = v.getAStore() and
+  not exists(result.getValue())
+}
+
 from Name unused, LocalVariable v
 where
   unused_local(unused, v) and

python/ql/src/experimental/semmle/python/frameworks/SqlAlchemy.qll

@@ -1,148 +0,0 @@
-/**
- * Provides classes modeling security-relevant aspects of the 'SqlAlchemy' package.
- * See https://pypi.org/project/SQLAlchemy/.
- */
-
-private import python
-private import semmle.python.dataflow.new.DataFlow
-private import semmle.python.dataflow.new.TaintTracking
-private import semmle.python.ApiGraphs
-private import semmle.python.Concepts
-private import experimental.semmle.python.Concepts
-
-private module SqlAlchemy {
-  /**
-   * Returns an instantization of a SqlAlchemy Session object.
-   * See https://docs.sqlalchemy.org/en/14/orm/session_api.html#sqlalchemy.orm.Session and
-   * https://docs.sqlalchemy.org/en/14/orm/session_api.html#sqlalchemy.orm.sessionmaker
-   */
-  private API::Node getSqlAlchemySessionInstance() {
-    result = API::moduleImport("sqlalchemy.orm").getMember("Session").getReturn() or
-    result = API::moduleImport("sqlalchemy.orm").getMember("sessionmaker").getReturn().getReturn()
-  }
-
-  /**
-   * Returns an instantization of a SqlAlchemy Engine object.
-   * See https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine
-   */
-  private API::Node getSqlAlchemyEngineInstance() {
-    result = API::moduleImport("sqlalchemy").getMember("create_engine").getReturn()
-  }
-
-  /**
-   * Returns an instantization of a SqlAlchemy Query object.
-   * See https://docs.sqlalchemy.org/en/14/orm/query.html?highlight=query#sqlalchemy.orm.Query
-   */
-  private API::Node getSqlAlchemyQueryInstance() {
-    result = getSqlAlchemySessionInstance().getMember("query").getReturn()
-  }
-
-  /**
-   * A call to `execute` meant to execute an SQL expression
-   * See the following links:
-   *   - https://docs.sqlalchemy.org/en/14/core/connections.html?highlight=execute#sqlalchemy.engine.Connection.execute
-   *   - https://docs.sqlalchemy.org/en/14/core/connections.html?highlight=execute#sqlalchemy.engine.Engine.execute
-   *   - https://docs.sqlalchemy.org/en/14/orm/session_api.html?highlight=execute#sqlalchemy.orm.Session.execute
-   */
-  private class SqlAlchemyExecuteCall extends DataFlow::CallCfgNode, SqlExecution::Range {
-    SqlAlchemyExecuteCall() {
-      // new way
-      this = getSqlAlchemySessionInstance().getMember("execute").getACall() or
-      this =
-        getSqlAlchemyEngineInstance()
-            .getMember(["connect", "begin"])
-            .getReturn()
-            .getMember("execute")
-            .getACall()
-    }
-
-    override DataFlow::Node getSql() { result = this.getArg(0) }
-  }
-
-  /**
-   * A call to `scalar` meant to execute an SQL expression
-   * See https://docs.sqlalchemy.org/en/14/orm/session_api.html#sqlalchemy.orm.Session.scalar and
-   * https://docs.sqlalchemy.org/en/14/core/connections.html?highlight=scalar#sqlalchemy.engine.Engine.scalar
-   */
-  private class SqlAlchemyScalarCall extends DataFlow::CallCfgNode, SqlExecution::Range {
-    SqlAlchemyScalarCall() {
-      this =
-        [getSqlAlchemySessionInstance(), getSqlAlchemyEngineInstance()]
-            .getMember("scalar")
-            .getACall()
-    }
-
-    override DataFlow::Node getSql() { result = this.getArg(0) }
-  }
-
-  /**
-   * A call on a Query object
-   * See https://docs.sqlalchemy.org/en/14/orm/query.html?highlight=query#sqlalchemy.orm.Query
-   */
-  private class SqlAlchemyQueryCall extends DataFlow::CallCfgNode, SqlExecution::Range {
-    SqlAlchemyQueryCall() {
-      this =
-        getSqlAlchemyQueryInstance()
-            .getMember(any(SqlAlchemyVulnerableMethodNames methodName))
-            .getACall()
-    }
-
-    override DataFlow::Node getSql() { result = this.getArg(0) }
-  }
-
-  /**
-   * This class represents a list of methods vulnerable to sql injection.
-   *
-   * See https://github.com/jty-team/codeql/pull/2#issue-611592361
-   */
-  private class SqlAlchemyVulnerableMethodNames extends string {
-    SqlAlchemyVulnerableMethodNames() { this in ["filter", "filter_by", "group_by", "order_by"] }
-  }
-
-  /**
-   * Additional taint-steps for `sqlalchemy.text()`
-   *
-   * See https://docs.sqlalchemy.org/en/14/core/sqlelement.html#sqlalchemy.sql.expression.text
-   * See https://docs.sqlalchemy.org/en/14/core/sqlelement.html#sqlalchemy.sql.expression.TextClause
-   */
-  class SqlAlchemyTextAdditionalTaintSteps extends TaintTracking::AdditionalTaintStep {
-    override predicate step(DataFlow::Node nodeFrom, DataFlow::Node nodeTo) {
-      exists(DataFlow::CallCfgNode call |
-        (
-          call = API::moduleImport("sqlalchemy").getMember("text").getACall()
-          or
-          call = API::moduleImport("sqlalchemy").getMember("sql").getMember("text").getACall()
-          or
-          call =
-            API::moduleImport("sqlalchemy")
-                .getMember("sql")
-                .getMember("expression")
-                .getMember("text")
-                .getACall()
-          or
-          call =
-            API::moduleImport("sqlalchemy")
-                .getMember("sql")
-                .getMember("expression")
-                .getMember("TextClause")
-                .getACall()
-        ) and
-        nodeFrom in [call.getArg(0), call.getArgByName("text")] and
-        nodeTo = call
-      )
-    }
-  }
-
-  /**
-   * Gets a reference to `sqlescapy.sqlescape`.
-   *
-   * See https://pypi.org/project/sqlescapy/
-   */
-  class SQLEscapySanitizerCall extends DataFlow::CallCfgNode, SQLEscape::Range {
-    SQLEscapySanitizerCall() {
-      this = API::moduleImport("sqlescapy").getMember("sqlescape").getACall()
-    }
-
-    override DataFlow::Node getAnInput() { result = this.getArg(0) }
-  }
-}

python/ql/src/semmle/python/functions/ModificationOfParameterWithDefault.qll

@@ -0,0 +1,46 @@
+/**
+ * Provides a data-flow configuration for detecting modifications of a parameters default value.
+ *
+ * Note, for performance reasons: only import this file if
+ * `ModificationOfParameterWithDefault::Configuration` is needed, otherwise
+ * `ModificationOfParameterWithDefaultCustomizations` should be imported instead.
+ */
+
+private import python
+import semmle.python.dataflow.new.DataFlow
+
+/**
+ * Provides a data-flow configuration for detecting modifications of a parameters default value.
+ */
+module ModificationOfParameterWithDefault {
+  import ModificationOfParameterWithDefaultCustomizations::ModificationOfParameterWithDefault
+
+  /**
+   * A data-flow configuration for detecting modifications of a parameters default value.
+   */
+  class Configuration extends DataFlow::Configuration {
+    /** Record whether the default value being tracked is non-empty. */
+    boolean nonEmptyDefault;
+
+    Configuration() {
+      nonEmptyDefault in [true, false] and
+      this = "ModificationOfParameterWithDefault:" + nonEmptyDefault.toString()
+    }
+
+    override predicate isSource(DataFlow::Node source) {
+      source.(Source).isNonEmpty() = nonEmptyDefault
+    }
+
+    override predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
+
+    override predicate isBarrier(DataFlow::Node node) {
+      // if we are tracking a non-empty default, then it is ok to modify empty values,
+      // so our tracking ends at those.
+      nonEmptyDefault = true and node instanceof MustBeEmpty
+      or
+      // if we are tracking a empty default, then it is ok to modify non-empty values,
+      // so our tracking ends at those.
+      nonEmptyDefault = false and node instanceof MustBeNonEmpty
+    }
+  }
+}

python/ql/src/semmle/python/functions/ModificationOfParameterWithDefaultCustomizations.qll

@@ -0,0 +1,190 @@
+/**
+ * Provides default sources, sinks and sanitizers for detecting
+ * modifications of a parameters default value, as well as extension points for adding your own.
+ */
+
+private import python
+private import semmle.python.dataflow.new.DataFlow
+private import semmle.python.dataflow.new.BarrierGuards
+
+/**
+ * Provides default sources, sinks and sanitizers for detecting
+ * "command injection"
+ * vulnerabilities, as well as extension points for adding your own.
+ */
+module ModificationOfParameterWithDefault {
+  /**
+   * A data flow source for detecting modifications of a parameters default value,
+   * that is a default value for some parameter.
+   */
+  abstract class Source extends DataFlow::Node {
+    /** Result is true if the default value is non-empty for this source and false if not. */
+    abstract boolean isNonEmpty();
+  }
+
+  /**
+   * A data flow sink for detecting modifications of a parameters default value,
+   * that is a node representing a modification.
+   */
+  abstract class Sink extends DataFlow::Node { }
+
+  /**
+   * A sanitizer for detecting modifications of a parameters default value
+   * should determine if the node (which is perhaps about to be modified)
+   * can be the default value or not.
+   *
+   * In this query we do not track the default value exactly, but rather wheter
+   * it is empty or not (see `Source`).
+   *
+   * This is the extension point for determining that a node must be empty and
+   * therefor is allowed to be modified if the tracked default value is non-empty.
+   */
+  abstract class MustBeEmpty extends DataFlow::Node { }
+
+  /**
+   * A sanitizer for detecting modifications of a parameters default value
+   * should determine if the node (which is perhaps about to be modified)
+   * can be the default value or not.
+   *
+   * In this query we do not track the default value exactly, but rather wheter
+   * it is empty or not (see `Source`).
+   *
+   * This is the extension point for determining that a node must be non-empty
+   * and therefor is allowed to be modified if the tracked default value is empty.
+   */
+  abstract class MustBeNonEmpty extends DataFlow::Node { }
+
+  /** Gets the truthiness (non emptyness) of the default of `p` if that value is mutable */
+  private boolean mutableDefaultValue(Parameter p) {
+    exists(Dict d | p.getDefault() = d |
+      exists(d.getAKey()) and result = true
+      or
+      not exists(d.getAKey()) and result = false
+    )
+    or
+    exists(List l | p.getDefault() = l |
+      exists(l.getAnElt()) and result = true
+      or
+      not exists(l.getAnElt()) and result = false
+    )
+  }
+
+  /**
+   * A mutable default value for a parameter, considered as a flow source.
+   */
+  class MutableDefaultValue extends Source {
+    boolean nonEmpty;
+
+    MutableDefaultValue() { nonEmpty = mutableDefaultValue(this.asCfgNode().(NameNode).getNode()) }
+
+    override boolean isNonEmpty() { result = nonEmpty }
+  }
+
+  /**
+   * A name of a list function that modifies the list.
+   * See https://docs.python.org/3/tutorial/datastructures.html#more-on-lists
+   */
+  string list_modifying_method() {
+    result in ["append", "extend", "insert", "remove", "pop", "clear", "sort", "reverse"]
+  }
+
+  /**
+   * A name of a dict function that modifies the dict.
+   * See https://docs.python.org/3/library/stdtypes.html#dict
+   */
+  string dict_modifying_method() { result in ["clear", "pop", "popitem", "setdefault", "update"] }
+
+  /**
+   * A mutation of the default value is a flow sink.
+   *
+   * Syntactic constructs that modify a list are:
+   * - s[i] = x
+   * - s[i:j] = t
+   * - del s[i:j]
+   * - s[i:j:k] = t
+   * - del s[i:j:k]
+   * - s += t
+   * - s *= n
+   * See https://docs.python.org/3/library/stdtypes.html#mutable-sequence-types
+   *
+   * Syntactic constructs that modify a dictionary are:
+   * - d[key] = value
+   * - del d[key]
+   * - d |= other
+   * See https://docs.python.org/3/library/stdtypes.html#dict
+   *
+   * These are all covered by:
+   * - assignment to a subscript (includes slices)
+   * - deletion of a subscript
+   * - augmented assignment to the value
+   */
+  class Mutation extends Sink {
+    Mutation() {
+      // assignment to a subscript (includes slices)
+      exists(DefinitionNode d | d.(SubscriptNode).getObject() = this.asCfgNode())
+      or
+      // deletion of a subscript
+      exists(DeletionNode d | d.getTarget().(SubscriptNode).getObject() = this.asCfgNode())
+      or
+      // augmented assignment to the value
+      exists(AugAssign a | a.getTarget().getAFlowNode() = this.asCfgNode())
+      or
+      // modifying function call
+      exists(DataFlow::CallCfgNode c, DataFlow::AttrRead a | c.getFunction() = a |
+        a.getObject() = this and
+        a.getAttributeName() in [list_modifying_method(), dict_modifying_method()]
+      )
+    }
+  }
+
+  // This to reimplement some of the functionality of the DataFlow::BarrierGuard
+  private import semmle.python.essa.SsaCompute
+
+  /**
+   * A data-flow node that is known to be either truthy or falsey.
+   *
+   * It handles the cases `if x` and `if not x`.
+   *
+   * For example, in the following code, `this` will be the `x` that is printed,
+   * which we will know is truthy:
+   *
+   * ```py
+   * if x:
+   *     print(x)
+   * ```
+   */
+  private class MustBe extends DataFlow::Node {
+    boolean truthy;
+
+    MustBe() {
+      exists(DataFlow::GuardNode guard, NameNode guarded, boolean branch |
+        // case: if x
+        guard = guarded and
+        branch = truthy
+        or
+        // case: if not x
+        guard.(UnaryExprNode).getNode().getOp() instanceof Not and
+        guarded = guard.(UnaryExprNode).getOperand() and
+        branch = truthy.booleanNot()
+      |
+        // guard controls this
+        guard.controlsBlock(this.asCfgNode().getBasicBlock(), branch) and
+        // there is a definition tying the guarded value to this
+        exists(EssaDefinition def |
+          AdjacentUses::useOfDef(def, this.asCfgNode()) and
+          AdjacentUses::useOfDef(def, guarded)
+        )
+      )
+    }
+  }
+
+  /** Simple guard detecting truthy values. */
+  private class MustBeTruthy extends MustBe, MustBeNonEmpty {
+    MustBeTruthy() { truthy = true }
+  }
+
+  /** Simple guard detecting falsey values. */
+  private class MustBeFalsey extends MustBe, MustBeEmpty {
+    MustBeFalsey() { truthy = false }
+  }
+}

python/ql/test/experimental/library-tests/frameworks/sqlalchemy/ConceptsTest.ql

@@ -1,3 +0,0 @@
-import python
-import experimental.meta.ConceptsTest
-import experimental.semmle.python.frameworks.SqlAlchemy

python/ql/test/experimental/library-tests/frameworks/sqlalchemy/InlineTaintTest.ql

@@ -1,2 +0,0 @@
-import experimental.meta.InlineTaintTest
-import experimental.semmle.python.frameworks.SqlAlchemy

python/ql/test/experimental/library-tests/frameworks/sqlalchemy/taint_test.py

@@ -1,12 +0,0 @@
-import sqlalchemy
-
-def test_taint():
-    ts = TAINTED_STRING
-
-    ensure_tainted(
-        ts, # $ tainted
-        sqlalchemy.text(ts), # $ tainted
-        sqlalchemy.sql.text(ts),# $ tainted
-        sqlalchemy.sql.expression.text(ts),# $ tainted
-        sqlalchemy.sql.expression.TextClause(ts),# $ tainted
-    )

python/ql/test/library-tests/frameworks/flask_sqlalchemy/ConceptsTest.ql

@@ -0,0 +1,2 @@
+import python
+import experimental.meta.ConceptsTest

python/ql/test/library-tests/frameworks/flask_sqlalchemy/InlineTaintTest.ql

@@ -0,0 +1 @@
+import experimental.meta.InlineTaintTest

python/ql/test/library-tests/frameworks/flask_sqlalchemy/SqlExecution.py

@@ -0,0 +1,51 @@
+# pip install Flask-SQLAlchemy
+from flask import Flask
+from flask_sqlalchemy import SQLAlchemy
+import sqlalchemy
+
+app = Flask(__name__)
+app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite+pysqlite:///:memory:"
+db = SQLAlchemy(app)
+
+# re-exports all things from `sqlalchemy` and `sqlalchemy.orm` under instances of `SQLAlchemy`
+# see
+# - https://github.com/pallets/flask-sqlalchemy/blob/931ec00d1e27f51508e05706eef41cc4419a0b32/src/flask_sqlalchemy/__init__.py#L765
+# - https://github.com/pallets/flask-sqlalchemy/blob/931ec00d1e27f51508e05706eef41cc4419a0b32/src/flask_sqlalchemy/__init__.py#L99-L109
+
+assert str(type(db.text("Foo"))) == "<class 'sqlalchemy.sql.elements.TextClause'>"
+
+# also has engine/session instantiated
+
+raw_sql = "SELECT 'Foo'"
+
+conn = db.engine.connect()
+result = conn.execute(raw_sql) # $ getSql=raw_sql
+assert result.fetchall() == [("Foo",)]
+
+conn = db.get_engine().connect()
+result = conn.execute(raw_sql) # $ getSql=raw_sql
+assert result.fetchall() == [("Foo",)]
+
+result = db.session.execute(raw_sql) # $ getSql=raw_sql
+assert result.fetchall() == [("Foo",)]
+
+Session = db.create_session(options={})
+session = Session()
+result = session.execute(raw_sql) # $ getSql=raw_sql
+assert result.fetchall() == [("Foo",)]
+
+Session = db.create_session(options={})
+with Session.begin() as session:
+    result = session.execute(raw_sql) # $ getSql=raw_sql
+    assert result.fetchall() == [("Foo",)]
+
+result = db.create_scoped_session().execute(raw_sql) # $ getSql=raw_sql
+assert result.fetchall() == [("Foo",)]
+
+
+# text
+t = db.text("foo")
+assert isinstance(t, sqlalchemy.sql.expression.TextClause)
+
+t = db.text(text="foo")
+assert isinstance(t, sqlalchemy.sql.expression.TextClause)

python/ql/test/library-tests/frameworks/sqlalchemy/ConceptsTest.ql

@@ -0,0 +1,2 @@
+import python
+import experimental.meta.ConceptsTest

python/ql/test/library-tests/frameworks/sqlalchemy/InlineTaintTest.expected

@@ -0,0 +1,3 @@
+argumentToEnsureNotTaintedNotMarkedAsSpurious
+untaintedArgumentToEnsureTaintedNotMarkedAsMissing
+failures

python/ql/test/library-tests/frameworks/sqlalchemy/InlineTaintTest.ql

@@ -0,0 +1 @@
+import experimental.meta.InlineTaintTest

python/ql/test/library-tests/frameworks/sqlalchemy/SqlExecution.py

@@ -47,10 +47,10 @@ with engine.begin() as connection:
 
 # Injection requiring the text() taint-step
 t = text("some sql")
-session.query(User).filter(t)  # $ getSql=t
-session.query(User).group_by(User.id).having(t)  # $ getSql=User.id MISSING: getSql=t
-session.query(User).group_by(t).first()  # $ getSql=t
-session.query(User).order_by(t).first()  # $ getSql=t
+session.query(User).filter(t)
+session.query(User).group_by(User.id).having(t)
+session.query(User).group_by(t).first()
+session.query(User).order_by(t).first()
 
 query = select(User).where(User.name == t)  # $ MISSING: getSql=t
 with engine.connect() as conn:

python/ql/test/library-tests/frameworks/sqlalchemy/new_tests.py

@@ -0,0 +1,388 @@
+import sqlalchemy
+import sqlalchemy.orm
+
+# SQLAlchemy is slowly migrating to a 2.0 version, and as part of 1.4 release have a 2.0
+# style (forwards compatible) API that _can_ be adopted. So these tests are marked with
+# either v1.4 or v2.0, such that we cover both.
+
+raw_sql = "select 'FOO'"
+text_sql = sqlalchemy.text(raw_sql)
+
+Base = sqlalchemy.orm.declarative_base()
+
+# ==============================================================================
+# v1.4
+# ==============================================================================
+
+print("v1.4")
+
+# Engine see https://docs.sqlalchemy.org/en/14/core/connections.html#sqlalchemy.engine.Engine
+
+engine = sqlalchemy.create_engine("sqlite+pysqlite:///:memory:", echo=True)
+
+result = engine.execute(raw_sql) # $ getSql=raw_sql
+assert result.fetchall() == [("FOO",)]
+result = engine.execute(statement=raw_sql) # $ getSql=raw_sql
+assert result.fetchall() == [("FOO",)]
+result = engine.execute(text_sql) # $ getSql=text_sql
+assert result.fetchall() == [("FOO",)]
+
+scalar_result = engine.scalar(raw_sql) # $ getSql=raw_sql
+assert scalar_result == "FOO"
+scalar_result = engine.scalar(statement=raw_sql) # $ getSql=raw_sql
+assert scalar_result == "FOO"
+
+# engine with custom execution options
+# see https://docs.sqlalchemy.org/en/14/core/connections.html#sqlalchemy.engine.Engine.execution_options
+engine_with_custom_exe_opts = engine.execution_options(foo=42)
+result = engine_with_custom_exe_opts.execute(raw_sql) # $ getSql=raw_sql
+assert result.fetchall() == [("FOO",)]
+
+even_more_opts = engine_with_custom_exe_opts.execution_options(bar=43)
+result = even_more_opts.execute(raw_sql) # $ getSql=raw_sql
+assert result.fetchall() == [("FOO",)]
+
+# Connection see https://docs.sqlalchemy.org/en/14/core/connections.html#sqlalchemy.engine.Connection
+conn = engine.connect()
+conn: sqlalchemy.engine.base.Connection
+
+result = conn.execute(raw_sql) # $ getSql=raw_sql
+assert result.fetchall() == [("FOO",)]
+result = conn.execute(statement=raw_sql) # $ getSql=raw_sql
+assert result.fetchall() == [("FOO",)]
+
+result = conn.execute(text_sql) # $ getSql=text_sql
+assert result.fetchall() == [("FOO",)]
+result = conn.execute(statement=text_sql) # $ getSql=text_sql
+assert result.fetchall() == [("FOO",)]
+
+# scalar
+scalar_result = conn.scalar(raw_sql) # $ getSql=raw_sql
+assert scalar_result == "FOO"
+scalar_result = conn.scalar(object_=raw_sql) # $ getSql=raw_sql
+assert scalar_result == "FOO"
+
+scalar_result = conn.scalar(text_sql) # $ getSql=text_sql
+assert scalar_result == "FOO"
+scalar_result = conn.scalar(object_=text_sql) # $ getSql=text_sql
+assert scalar_result == "FOO"
+
+
+# exec_driver_sql
+result = conn.exec_driver_sql(raw_sql) # $ getSql=raw_sql
+assert result.fetchall() == [("FOO",)]
+
+# construction by object
+conn = sqlalchemy.engine.base.Connection(engine)
+result = conn.execute(raw_sql) # $ getSql=raw_sql
+assert result.fetchall() == [("FOO",)]
+
+# branched connection
+branched_conn = conn.connect()
+result = branched_conn.execute(text_sql) # $ getSql=text_sql
+assert result.fetchall() == [("FOO",)]
+
+# raw connection
+raw_conn = conn.connection
+result = raw_conn.execute(raw_sql) # $ getSql=raw_sql
+assert result.fetchall() == [("FOO",)]
+
+cursor = raw_conn.cursor()
+cursor.execute(raw_sql) # $ getSql=raw_sql
+assert cursor.fetchall() == [("FOO",)]
+cursor.close()
+
+raw_conn = engine.raw_connection()
+result = raw_conn.execute(raw_sql) # $ getSql=raw_sql
+assert result.fetchall() == [("FOO",)]
+
+# connection with custom execution options
+conn_with_custom_exe_opts = conn.execution_options(bar=1337)
+result = conn_with_custom_exe_opts.execute(text_sql) # $ getSql=text_sql
+assert result.fetchall() == [("FOO",)]
+
+# Session -- is what you use to work with the ORM layer
+# see https://docs.sqlalchemy.org/en/14/orm/session_basics.html
+# and https://docs.sqlalchemy.org/en/14/orm/session_api.html#sqlalchemy.orm.Session
+
+session = sqlalchemy.orm.Session(engine)
+
+result = session.execute(raw_sql) # $ getSql=raw_sql
+assert result.fetchall() == [("FOO",)]
+result = session.execute(statement=raw_sql) # $ getSql=raw_sql
+assert result.fetchall() == [("FOO",)]
+
+result = session.execute(text_sql) # $ getSql=text_sql
+assert result.fetchall() == [("FOO",)]
+result = session.execute(statement=text_sql) # $ getSql=text_sql
+assert result.fetchall() == [("FOO",)]
+
+# scalar
+scalar_result = session.scalar(raw_sql) # $ getSql=raw_sql
+assert scalar_result == "FOO"
+scalar_result = session.scalar(statement=raw_sql) # $ getSql=raw_sql
+assert scalar_result == "FOO"
+
+scalar_result = session.scalar(text_sql) # $ getSql=text_sql
+assert scalar_result == "FOO"
+scalar_result = session.scalar(statement=text_sql) # $ getSql=text_sql
+assert scalar_result == "FOO"
+
+# other ways to construct a session
+with sqlalchemy.orm.Session(engine) as session:
+    result = session.execute(raw_sql) # $ getSql=raw_sql
+    assert result.fetchall() == [("FOO",)]
+
+Session = sqlalchemy.orm.sessionmaker(engine)
+session = Session()
+
+result = session.execute(raw_sql) # $ getSql=raw_sql
+assert result.fetchall() == [("FOO",)]
+
+with Session() as session:
+    result = session.execute(raw_sql) # $ getSql=raw_sql
+    assert result.fetchall() == [("FOO",)]
+
+with Session.begin() as session:
+    result = session.execute(raw_sql) # $ getSql=raw_sql
+    assert result.fetchall() == [("FOO",)]
+
+# Querying (1.4)
+# see https://docs.sqlalchemy.org/en/14/orm/session_basics.html#querying-1-x-style
+
+# to do so we first need a model
+
+class For14(Base):
+    __tablename__ = "for14"
+
+    id = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True)
+    description = sqlalchemy.Column(sqlalchemy.String)
+
+Base.metadata.create_all(engine)
+
+# add a test-entry
+test_entry = For14(id=14, description="test")
+session = sqlalchemy.orm.Session(engine)
+session.add(test_entry)
+session.commit()
+assert session.query(For14).all()[0].id == 14
+
+# and now we can do the actual querying
+
+text_foo = sqlalchemy.text("'FOO'")
+
+# filter_by is only vulnerable to injection if sqlalchemy.text is used, which is evident
+# from the logs produced if this file is run
+# that is, first filter_by results in the SQL
+#
+# SELECT for14.id AS for14_id, for14.description AS for14_description
+# FROM for14
+# WHERE for14.description = ?
+#
+# which is then called with the argument `'FOO'`
+#
+# and the second filter_by results in the SQL
+#
+# SELECT for14.id AS for14_id, for14.description AS for14_description
+# FROM for14
+# WHERE for14.description = 'FOO'
+#
+# which is then called without any arguments
+assert session.query(For14).filter_by(description="'FOO'").all() == []
+query = session.query(For14).filter_by(description=text_foo)
+assert query.all() == []
+
+# Initially I wanted to add lots of additional taint steps such that the normal SQL
+# injection query would find these cases where an ORM query includes a TextClause that
+# includes user-input directly... But that presented 2 problems:
+#
+# - which part of the query construction above should be marked as SQL to fit our
+#   `SqlExecution` concept. Nothing really fits this well, since all the SQL execution
+#   happens under the hood.
+# - This would require a LOT of modeling for these additional taint steps, since there
+#   are many many constructs we would need to have models for. (see the 2 examples below)
+#
+# So instead we flag user-input to a TextClause with its' own query. And so we don't
+# highlight any parts of an ORM constructed query such as these as containing SQL.
+
+# `filter` provides more general filtering
+# see https://docs.sqlalchemy.org/en/14/orm/tutorial.html#common-filter-operators
+# and https://docs.sqlalchemy.org/en/14/orm/query.html#sqlalchemy.orm.Query.filter
+assert session.query(For14).filter(For14.description == "'FOO'").all() == []
+query = session.query(For14).filter(For14.description == text_foo)
+assert query.all() == []
+
+assert session.query(For14).filter(For14.description.like("'FOO'")).all() == []
+query = session.query(For14).filter(For14.description.like(text_foo))
+assert query.all() == []
+
+# There are many other possibilities for ending up with SQL injection, including the
+# following (not an exhaustive list):
+# - `where` (alias for `filter`)
+# - `group_by`
+# - `having`
+# - `order_by`
+# - `join`
+# - `outerjoin`
+
+# ==============================================================================
+# v2.0
+# ==============================================================================
+import sqlalchemy.future
+
+print("-"*80)
+print("v2.0 style")
+
+# For Engine, see https://docs.sqlalchemy.org/en/14/core/future.html#sqlalchemy.future.Engine
+engine = sqlalchemy.create_engine("sqlite+pysqlite:///:memory:", echo=True, future=True)
+future_engine = sqlalchemy.future.create_engine("sqlite+pysqlite:///:memory:", echo=True)
+
+# in 2.0 you are not allowed to execute things directly on the engine
+try:
+    engine.execute(raw_sql) # $ SPURIOUS: getSql=raw_sql
+    raise Exception("above not allowed in 2.0")
+except NotImplementedError:
+    pass
+try:
+    engine.execute(text_sql) # $ SPURIOUS: getSql=text_sql
+    raise Exception("above not allowed in 2.0")
+except NotImplementedError:
+    pass
+
+
+# `connect` returns a new Connection object.
+# see https://docs.sqlalchemy.org/en/14/core/future.html#sqlalchemy.future.Connection
+print("v2.0 engine.connect")
+with engine.connect() as conn:
+    conn: sqlalchemy.future.Connection
+
+    # in 2.0 you are not allowed to use raw strings like this:
+    try:
+        conn.execute(raw_sql) # $ SPURIOUS: getSql=raw_sql
+        raise Exception("above not allowed in 2.0")
+    except sqlalchemy.exc.ObjectNotExecutableError:
+        pass
+
+    result = conn.execute(text_sql) # $ getSql=text_sql
+    assert result.fetchall() == [("FOO",)]
+    result = conn.execute(statement=text_sql) # $ getSql=text_sql
+    assert result.fetchall() == [("FOO",)]
+
+    result = conn.exec_driver_sql(raw_sql) # $ getSql=raw_sql
+    assert result.fetchall() == [("FOO",)]
+
+    raw_conn = conn.connection
+    result = raw_conn.execute(raw_sql) # $ getSql=raw_sql
+    assert result.fetchall() == [("FOO",)]
+
+    # branching not allowed in 2.0
+    try:
+        branched_conn = conn.connect()
+        raise Exception("above not allowed in 2.0")
+    except NotImplementedError:
+        pass
+
+    # connection with custom execution options
+    conn_with_custom_exe_opts = conn.execution_options(bar=1337)
+    result = conn_with_custom_exe_opts.execute(text_sql) # $ getSql=text_sql
+    assert result.fetchall() == [("FOO",)]
+
+    # `scalar` is shorthand helper
+    try:
+        conn.scalar(raw_sql) # $ SPURIOUS: getSql=raw_sql
+    except sqlalchemy.exc.ObjectNotExecutableError:
+        pass
+    scalar_result = conn.scalar(text_sql) # $ getSql=text_sql
+    assert scalar_result == "FOO"
+    scalar_result = conn.scalar(statement=text_sql) # $ getSql=text_sql
+    assert scalar_result == "FOO"
+
+    # This is a contrived example
+    select = sqlalchemy.select(sqlalchemy.text("'BAR'"))
+    result = conn.execute(select) # $ getSql=select
+    assert result.fetchall() == [("BAR",)]
+
+    # This is a contrived example
+    select = sqlalchemy.select(sqlalchemy.literal_column("'BAZ'"))
+    result = conn.execute(select) # $ getSql=select
+    assert result.fetchall() == [("BAZ",)]
+
+with future_engine.connect() as conn:
+    result = conn.execute(text_sql) # $ getSql=text_sql
+    assert result.fetchall() == [("FOO",)]
+
+# `begin` returns a new Connection object with a transaction begun.
+print("v2.0 engine.begin")
+with engine.begin() as conn:
+    result = conn.execute(text_sql) # $ getSql=text_sql
+    assert result.fetchall() == [("FOO",)]
+
+# construction by object
+conn = sqlalchemy.future.Connection(engine)
+result = conn.execute(text_sql) # $ getSql=text_sql
+assert result.fetchall() == [("FOO",)]
+
+# raw_connection
+
+raw_conn = engine.raw_connection()
+result = raw_conn.execute(raw_sql) # $ getSql=raw_sql
+assert result.fetchall() == [("FOO",)]
+
+cursor = raw_conn.cursor()
+cursor.execute(raw_sql) # $ getSql=raw_sql
+assert cursor.fetchall() == [("FOO",)]
+cursor.close()
+
+# Session (2.0)
+session = sqlalchemy.orm.Session(engine, future=True)
+
+result = session.execute(raw_sql) # $ getSql=raw_sql
+assert result.fetchall() == [("FOO",)]
+result = session.execute(statement=raw_sql) # $ getSql=raw_sql
+assert result.fetchall() == [("FOO",)]
+
+result = session.execute(text_sql) # $ getSql=text_sql
+assert result.fetchall() == [("FOO",)]
+result = session.execute(statement=text_sql) # $ getSql=text_sql
+assert result.fetchall() == [("FOO",)]
+
+# scalar
+scalar_result = session.scalar(raw_sql) # $ getSql=raw_sql
+assert scalar_result == "FOO"
+scalar_result = session.scalar(statement=raw_sql) # $ getSql=raw_sql
+assert scalar_result == "FOO"
+
+scalar_result = session.scalar(text_sql) # $ getSql=text_sql
+assert scalar_result == "FOO"
+scalar_result = session.scalar(statement=text_sql) # $ getSql=text_sql
+assert scalar_result == "FOO"
+
+# Querying (2.0)
+# uses a slightly different style than 1.4 -- see note about not modeling
+# ORM query construction as SQL execution at the 1.4 query tests.
+
+class For20(Base):
+    __tablename__ = "for20"
+
+    id = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True)
+    description = sqlalchemy.Column(sqlalchemy.String)
+
+For20.metadata.create_all(engine)
+
+# add a test-entry
+test_entry = For20(id=20, description="test")
+session = sqlalchemy.orm.Session(engine, future=True)
+session.add(test_entry)
+session.commit()
+assert session.query(For20).all()[0].id == 20
+
+# and now we can do the actual querying
+# see https://docs.sqlalchemy.org/en/14/orm/session_basics.html#querying-2-0-style
+
+statement = sqlalchemy.select(For20)
+result = session.execute(statement) # $ getSql=statement
+assert result.scalars().all()[0].id == 20
+
+statement = sqlalchemy.select(For20).where(For20.description == text_foo)
+result = session.execute(statement) # $ getSql=statement
+assert result.scalars().all() == []

python/ql/test/library-tests/frameworks/sqlalchemy/taint_test.py

@@ -0,0 +1,28 @@
+import sqlalchemy
+
+ensure_tainted = ensure_not_tainted = print
+TAINTED_STRING = "TAINTED_STRING"
+
+def test_taint():
+    ts = TAINTED_STRING
+
+    ensure_tainted(ts) # $ tainted
+
+    t1 = sqlalchemy.text(ts)
+    t2 = sqlalchemy.text(text=ts)
+    t3 = sqlalchemy.sql.text(ts)
+    t4 = sqlalchemy.sql.text(text=ts)
+    t5 = sqlalchemy.sql.expression.text(ts)
+    t6 = sqlalchemy.sql.expression.text(text=ts)
+    t7 = sqlalchemy.sql.expression.TextClause(ts)
+    t8 = sqlalchemy.sql.expression.TextClause(text=ts)
+
+    # Since we flag user-input to a TextClause with its' own query, we don't want to
+    # have a taint-step for it as that would lead to us also giving an alert for normal
+    # SQL-injection... and double alerting like this does not seem desireable.
+    ensure_not_tainted(t1, t2, t3, t4, t5, t6, t7, t8)
+
+    for text in [t1, t2, t3, t4, t5, t6, t7, t8]:
+        assert isinstance(text, sqlalchemy.sql.expression.TextClause)
+
+test_taint()

python/ql/test/query-tests/Exceptions/generators/test.py

@@ -53,3 +53,12 @@ def ok5(seq):
 
 def ok6(seq):
     yield next(iter([]), default='foo')    
+
+# Handling for multiple exception types, one of which is `StopIteration`
+# Reported as a false positive in github/codeql#6227
+def ok7(seq, ctx):
+    try:
+        with ctx:
+            yield next(iter)
+    except (StopIteration, MemoryError):
+        return

python/ql/test/query-tests/Functions/ModificationOfParameterWithDefault/ModificationOfParameterWithDefault.expected

@@ -0,0 +1,108 @@
+edges
+| test.py:2:12:2:12 | ControlFlowNode for l | test.py:3:5:3:5 | ControlFlowNode for l |
+| test.py:7:11:7:11 | ControlFlowNode for l | test.py:8:5:8:5 | ControlFlowNode for l |
+| test.py:12:14:12:14 | ControlFlowNode for l | test.py:13:9:13:9 | ControlFlowNode for l |
+| test.py:17:15:17:15 | ControlFlowNode for l | test.py:18:5:18:5 | ControlFlowNode for l |
+| test.py:22:15:22:15 | ControlFlowNode for l | test.py:23:5:23:5 | ControlFlowNode for l |
+| test.py:27:12:27:12 | ControlFlowNode for l | test.py:28:5:28:5 | ControlFlowNode for l |
+| test.py:38:13:38:13 | ControlFlowNode for l | test.py:39:5:39:5 | ControlFlowNode for l |
+| test.py:43:14:43:14 | ControlFlowNode for l | test.py:44:13:44:13 | ControlFlowNode for l |
+| test.py:44:13:44:13 | ControlFlowNode for l | test.py:38:13:38:13 | ControlFlowNode for l |
+| test.py:48:14:48:14 | ControlFlowNode for l | test.py:49:5:49:5 | ControlFlowNode for l |
+| test.py:53:10:53:10 | ControlFlowNode for d | test.py:54:5:54:5 | ControlFlowNode for d |
+| test.py:58:19:58:19 | ControlFlowNode for d | test.py:59:5:59:5 | ControlFlowNode for d |
+| test.py:63:28:63:28 | ControlFlowNode for d | test.py:64:5:64:5 | ControlFlowNode for d |
+| test.py:67:14:67:14 | ControlFlowNode for d | test.py:68:5:68:5 | ControlFlowNode for d |
+| test.py:72:19:72:19 | ControlFlowNode for d | test.py:73:14:73:14 | ControlFlowNode for d |
+| test.py:73:14:73:14 | ControlFlowNode for d | test.py:67:14:67:14 | ControlFlowNode for d |
+| test.py:77:17:77:17 | ControlFlowNode for d | test.py:78:5:78:5 | ControlFlowNode for d |
+| test.py:82:26:82:26 | ControlFlowNode for d | test.py:83:5:83:5 | ControlFlowNode for d |
+| test.py:87:35:87:35 | ControlFlowNode for d | test.py:88:5:88:5 | ControlFlowNode for d |
+| test.py:91:21:91:21 | ControlFlowNode for d | test.py:92:5:92:5 | ControlFlowNode for d |
+| test.py:96:26:96:26 | ControlFlowNode for d | test.py:97:21:97:21 | ControlFlowNode for d |
+| test.py:97:21:97:21 | ControlFlowNode for d | test.py:91:21:91:21 | ControlFlowNode for d |
+| test.py:108:14:108:14 | ControlFlowNode for d | test.py:109:9:109:9 | ControlFlowNode for d |
+| test.py:113:20:113:20 | ControlFlowNode for d | test.py:115:5:115:5 | ControlFlowNode for d |
+| test.py:119:29:119:29 | ControlFlowNode for d | test.py:121:5:121:5 | ControlFlowNode for d |
+| test.py:124:15:124:15 | ControlFlowNode for l | test.py:128:9:128:9 | ControlFlowNode for l |
+| test.py:131:23:131:23 | ControlFlowNode for l | test.py:135:9:135:9 | ControlFlowNode for l |
+| test.py:138:15:138:15 | ControlFlowNode for l | test.py:140:9:140:9 | ControlFlowNode for l |
+| test.py:145:23:145:23 | ControlFlowNode for l | test.py:147:9:147:9 | ControlFlowNode for l |
+nodes
+| test.py:2:12:2:12 | ControlFlowNode for l | semmle.label | ControlFlowNode for l |
+| test.py:3:5:3:5 | ControlFlowNode for l | semmle.label | ControlFlowNode for l |
+| test.py:7:11:7:11 | ControlFlowNode for l | semmle.label | ControlFlowNode for l |
+| test.py:8:5:8:5 | ControlFlowNode for l | semmle.label | ControlFlowNode for l |
+| test.py:12:14:12:14 | ControlFlowNode for l | semmle.label | ControlFlowNode for l |
+| test.py:13:9:13:9 | ControlFlowNode for l | semmle.label | ControlFlowNode for l |
+| test.py:17:15:17:15 | ControlFlowNode for l | semmle.label | ControlFlowNode for l |
+| test.py:18:5:18:5 | ControlFlowNode for l | semmle.label | ControlFlowNode for l |
+| test.py:22:15:22:15 | ControlFlowNode for l | semmle.label | ControlFlowNode for l |
+| test.py:23:5:23:5 | ControlFlowNode for l | semmle.label | ControlFlowNode for l |
+| test.py:27:12:27:12 | ControlFlowNode for l | semmle.label | ControlFlowNode for l |
+| test.py:28:5:28:5 | ControlFlowNode for l | semmle.label | ControlFlowNode for l |
+| test.py:38:13:38:13 | ControlFlowNode for l | semmle.label | ControlFlowNode for l |
+| test.py:39:5:39:5 | ControlFlowNode for l | semmle.label | ControlFlowNode for l |
+| test.py:43:14:43:14 | ControlFlowNode for l | semmle.label | ControlFlowNode for l |
+| test.py:44:13:44:13 | ControlFlowNode for l | semmle.label | ControlFlowNode for l |
+| test.py:48:14:48:14 | ControlFlowNode for l | semmle.label | ControlFlowNode for l |
+| test.py:49:5:49:5 | ControlFlowNode for l | semmle.label | ControlFlowNode for l |
+| test.py:53:10:53:10 | ControlFlowNode for d | semmle.label | ControlFlowNode for d |
+| test.py:54:5:54:5 | ControlFlowNode for d | semmle.label | ControlFlowNode for d |
+| test.py:58:19:58:19 | ControlFlowNode for d | semmle.label | ControlFlowNode for d |
+| test.py:59:5:59:5 | ControlFlowNode for d | semmle.label | ControlFlowNode for d |
+| test.py:63:28:63:28 | ControlFlowNode for d | semmle.label | ControlFlowNode for d |
+| test.py:64:5:64:5 | ControlFlowNode for d | semmle.label | ControlFlowNode for d |
+| test.py:67:14:67:14 | ControlFlowNode for d | semmle.label | ControlFlowNode for d |
+| test.py:68:5:68:5 | ControlFlowNode for d | semmle.label | ControlFlowNode for d |
+| test.py:72:19:72:19 | ControlFlowNode for d | semmle.label | ControlFlowNode for d |
+| test.py:73:14:73:14 | ControlFlowNode for d | semmle.label | ControlFlowNode for d |
+| test.py:77:17:77:17 | ControlFlowNode for d | semmle.label | ControlFlowNode for d |
+| test.py:78:5:78:5 | ControlFlowNode for d | semmle.label | ControlFlowNode for d |
+| test.py:82:26:82:26 | ControlFlowNode for d | semmle.label | ControlFlowNode for d |
+| test.py:83:5:83:5 | ControlFlowNode for d | semmle.label | ControlFlowNode for d |
+| test.py:87:35:87:35 | ControlFlowNode for d | semmle.label | ControlFlowNode for d |
+| test.py:88:5:88:5 | ControlFlowNode for d | semmle.label | ControlFlowNode for d |
+| test.py:91:21:91:21 | ControlFlowNode for d | semmle.label | ControlFlowNode for d |
+| test.py:92:5:92:5 | ControlFlowNode for d | semmle.label | ControlFlowNode for d |
+| test.py:96:26:96:26 | ControlFlowNode for d | semmle.label | ControlFlowNode for d |
+| test.py:97:21:97:21 | ControlFlowNode for d | semmle.label | ControlFlowNode for d |
+| test.py:108:14:108:14 | ControlFlowNode for d | semmle.label | ControlFlowNode for d |
+| test.py:109:9:109:9 | ControlFlowNode for d | semmle.label | ControlFlowNode for d |
+| test.py:113:20:113:20 | ControlFlowNode for d | semmle.label | ControlFlowNode for d |
+| test.py:115:5:115:5 | ControlFlowNode for d | semmle.label | ControlFlowNode for d |
+| test.py:119:29:119:29 | ControlFlowNode for d | semmle.label | ControlFlowNode for d |
+| test.py:121:5:121:5 | ControlFlowNode for d | semmle.label | ControlFlowNode for d |
+| test.py:124:15:124:15 | ControlFlowNode for l | semmle.label | ControlFlowNode for l |
+| test.py:128:9:128:9 | ControlFlowNode for l | semmle.label | ControlFlowNode for l |
+| test.py:131:23:131:23 | ControlFlowNode for l | semmle.label | ControlFlowNode for l |
+| test.py:135:9:135:9 | ControlFlowNode for l | semmle.label | ControlFlowNode for l |
+| test.py:138:15:138:15 | ControlFlowNode for l | semmle.label | ControlFlowNode for l |
+| test.py:140:9:140:9 | ControlFlowNode for l | semmle.label | ControlFlowNode for l |
+| test.py:145:23:145:23 | ControlFlowNode for l | semmle.label | ControlFlowNode for l |
+| test.py:147:9:147:9 | ControlFlowNode for l | semmle.label | ControlFlowNode for l |
+subpaths
+#select
+| test.py:3:5:3:5 | ControlFlowNode for l | test.py:2:12:2:12 | ControlFlowNode for l | test.py:3:5:3:5 | ControlFlowNode for l | $@ flows to here and is mutated. | test.py:2:12:2:12 | ControlFlowNode for l | Default value |
+| test.py:8:5:8:5 | ControlFlowNode for l | test.py:7:11:7:11 | ControlFlowNode for l | test.py:8:5:8:5 | ControlFlowNode for l | $@ flows to here and is mutated. | test.py:7:11:7:11 | ControlFlowNode for l | Default value |
+| test.py:13:9:13:9 | ControlFlowNode for l | test.py:12:14:12:14 | ControlFlowNode for l | test.py:13:9:13:9 | ControlFlowNode for l | $@ flows to here and is mutated. | test.py:12:14:12:14 | ControlFlowNode for l | Default value |
+| test.py:18:5:18:5 | ControlFlowNode for l | test.py:17:15:17:15 | ControlFlowNode for l | test.py:18:5:18:5 | ControlFlowNode for l | $@ flows to here and is mutated. | test.py:17:15:17:15 | ControlFlowNode for l | Default value |
+| test.py:23:5:23:5 | ControlFlowNode for l | test.py:22:15:22:15 | ControlFlowNode for l | test.py:23:5:23:5 | ControlFlowNode for l | $@ flows to here and is mutated. | test.py:22:15:22:15 | ControlFlowNode for l | Default value |
+| test.py:28:5:28:5 | ControlFlowNode for l | test.py:27:12:27:12 | ControlFlowNode for l | test.py:28:5:28:5 | ControlFlowNode for l | $@ flows to here and is mutated. | test.py:27:12:27:12 | ControlFlowNode for l | Default value |
+| test.py:39:5:39:5 | ControlFlowNode for l | test.py:43:14:43:14 | ControlFlowNode for l | test.py:39:5:39:5 | ControlFlowNode for l | $@ flows to here and is mutated. | test.py:43:14:43:14 | ControlFlowNode for l | Default value |
+| test.py:49:5:49:5 | ControlFlowNode for l | test.py:48:14:48:14 | ControlFlowNode for l | test.py:49:5:49:5 | ControlFlowNode for l | $@ flows to here and is mutated. | test.py:48:14:48:14 | ControlFlowNode for l | Default value |
+| test.py:54:5:54:5 | ControlFlowNode for d | test.py:53:10:53:10 | ControlFlowNode for d | test.py:54:5:54:5 | ControlFlowNode for d | $@ flows to here and is mutated. | test.py:53:10:53:10 | ControlFlowNode for d | Default value |
+| test.py:59:5:59:5 | ControlFlowNode for d | test.py:58:19:58:19 | ControlFlowNode for d | test.py:59:5:59:5 | ControlFlowNode for d | $@ flows to here and is mutated. | test.py:58:19:58:19 | ControlFlowNode for d | Default value |
+| test.py:64:5:64:5 | ControlFlowNode for d | test.py:63:28:63:28 | ControlFlowNode for d | test.py:64:5:64:5 | ControlFlowNode for d | $@ flows to here and is mutated. | test.py:63:28:63:28 | ControlFlowNode for d | Default value |
+| test.py:68:5:68:5 | ControlFlowNode for d | test.py:72:19:72:19 | ControlFlowNode for d | test.py:68:5:68:5 | ControlFlowNode for d | $@ flows to here and is mutated. | test.py:72:19:72:19 | ControlFlowNode for d | Default value |
+| test.py:78:5:78:5 | ControlFlowNode for d | test.py:77:17:77:17 | ControlFlowNode for d | test.py:78:5:78:5 | ControlFlowNode for d | $@ flows to here and is mutated. | test.py:77:17:77:17 | ControlFlowNode for d | Default value |
+| test.py:83:5:83:5 | ControlFlowNode for d | test.py:82:26:82:26 | ControlFlowNode for d | test.py:83:5:83:5 | ControlFlowNode for d | $@ flows to here and is mutated. | test.py:82:26:82:26 | ControlFlowNode for d | Default value |
+| test.py:88:5:88:5 | ControlFlowNode for d | test.py:87:35:87:35 | ControlFlowNode for d | test.py:88:5:88:5 | ControlFlowNode for d | $@ flows to here and is mutated. | test.py:87:35:87:35 | ControlFlowNode for d | Default value |
+| test.py:92:5:92:5 | ControlFlowNode for d | test.py:96:26:96:26 | ControlFlowNode for d | test.py:92:5:92:5 | ControlFlowNode for d | $@ flows to here and is mutated. | test.py:96:26:96:26 | ControlFlowNode for d | Default value |
+| test.py:109:9:109:9 | ControlFlowNode for d | test.py:108:14:108:14 | ControlFlowNode for d | test.py:109:9:109:9 | ControlFlowNode for d | $@ flows to here and is mutated. | test.py:108:14:108:14 | ControlFlowNode for d | Default value |
+| test.py:115:5:115:5 | ControlFlowNode for d | test.py:113:20:113:20 | ControlFlowNode for d | test.py:115:5:115:5 | ControlFlowNode for d | $@ flows to here and is mutated. | test.py:113:20:113:20 | ControlFlowNode for d | Default value |
+| test.py:121:5:121:5 | ControlFlowNode for d | test.py:119:29:119:29 | ControlFlowNode for d | test.py:121:5:121:5 | ControlFlowNode for d | $@ flows to here and is mutated. | test.py:119:29:119:29 | ControlFlowNode for d | Default value |
+| test.py:128:9:128:9 | ControlFlowNode for l | test.py:124:15:124:15 | ControlFlowNode for l | test.py:128:9:128:9 | ControlFlowNode for l | $@ flows to here and is mutated. | test.py:124:15:124:15 | ControlFlowNode for l | Default value |
+| test.py:135:9:135:9 | ControlFlowNode for l | test.py:131:23:131:23 | ControlFlowNode for l | test.py:135:9:135:9 | ControlFlowNode for l | $@ flows to here and is mutated. | test.py:131:23:131:23 | ControlFlowNode for l | Default value |
+| test.py:140:9:140:9 | ControlFlowNode for l | test.py:138:15:138:15 | ControlFlowNode for l | test.py:140:9:140:9 | ControlFlowNode for l | $@ flows to here and is mutated. | test.py:138:15:138:15 | ControlFlowNode for l | Default value |
+| test.py:147:9:147:9 | ControlFlowNode for l | test.py:145:23:145:23 | ControlFlowNode for l | test.py:147:9:147:9 | ControlFlowNode for l | $@ flows to here and is mutated. | test.py:145:23:145:23 | ControlFlowNode for l | Default value |

python/ql/test/query-tests/Functions/ModificationOfParameterWithDefault/ModificationOfParameterWithDefault.qlref

@@ -0,0 +1 @@
+Functions/ModificationOfParameterWithDefault.ql

python/ql/test/query-tests/Functions/ModificationOfParameterWithDefault/test.py

@@ -0,0 +1,150 @@
+# Not OK
+def simple(l = [0]):
+    l[0] = 1  #$ modification=l
+    return l
+
+# Not OK
+def slice(l = [0]):
+    l[0:1] = 1  #$ modification=l
+    return l
+
+# Not OK
+def list_del(l = [0]):
+    del l[0]  #$ modification=l
+    return l
+
+# Not OK
+def append_op(l = []):
+    l += [1, 2, 3]  #$ modification=l
+    return l
+
+# Not OK
+def repeat_op(l = [0]):
+    l *= 3  #$ modification=l
+    return l
+
+# Not OK
+def append(l = []):
+    l.append(1)  #$ modification=l
+    return l
+
+# OK
+def includes(l = []):
+    x = [0]
+    x.extend(l)
+    x.extend([1])
+    return x
+
+def extends(l):
+    l.extend([1])  #$ modification=l
+    return l
+
+# Not OK
+def deferred(l = []):
+    extends(l)
+    return l
+
+# Not OK
+def nonempty(l = [5]):
+    l.append(1)  #$ modification=l
+    return l
+
+# Not OK
+def dict(d = {}):
+    d['a'] = 1  #$ modification=d
+    return d
+
+# Not OK
+def dict_nonempty(d = {'a': 1}):
+    d['a'] = 2  #$ modification=d
+    return d
+
+# OK
+def dict_nonempty_nochange(d = {'a': 1}):
+    d['a'] = 1  #$ SPURIOUS: modification=d
+    return d
+
+def modifies(d):
+    d['a'] = 1  #$ modification=d
+    return d
+
+# Not OK
+def dict_deferred(d = {}):
+    modifies(d)
+    return d
+
+# Not OK
+def dict_method(d = {}):
+    d.update({'a': 1})  #$ modification=d
+    return d
+
+# Not OK
+def dict_method_nonempty(d = {'a': 1}):
+    d.update({'a': 2})  #$ modification=d
+    return d
+
+# OK
+def dict_method_nonempty_nochange(d = {'a': 1}):
+    d.update({'a': 1})  #$ SPURIOUS:modification=d
+    return d
+
+def modifies_method(d):
+    d.update({'a': 1})  #$ modification=d
+    return d
+
+# Not OK
+def dict_deferred_method(d = {}):
+    modifies_method(d)
+    return d
+
+# OK
+def dict_includes(d = {}):
+    x = {}
+    x.update(d)
+    x.update({'a': 1})
+    return x
+
+# Not OK
+def dict_del(d = {'a': 1}):
+    del d['a']  #$ modification=d
+    return d
+
+# Not OK
+def dict_update_op(d = {}):
+    x = {'a': 1}
+    d |= x  #$ modification=d
+    return d
+
+# OK
+def dict_update_op_nochange(d = {}):
+    x = {}
+    d |= x  #$ SPURIOUS: modification=d
+    return d
+
+def sanitizer(l = []):
+    if l:
+        l.append(1)
+    else:
+        l.append(1)  #$ modification=l
+    return l
+
+def sanitizer_negated(l = [1]):
+    if not l:
+        l.append(1)
+    else:
+        l.append(1)  #$ modification=l
+    return l
+
+def sanitizer(l = []):
+    if not l:
+        l.append(1)  #$ modification=l
+    else:
+        l.append(1)
+    return l
+
+def sanitizer_negated(l = [1]):
+    if l:
+        l.append(1)  #$ modification=l
+    else:
+        l.append(1)
+    return l

python/ql/test/query-tests/Functions/ModificationOfParameterWithDefault/test.ql

@@ -0,0 +1,24 @@
+import python
+import semmle.python.dataflow.new.DataFlow
+import TestUtilities.InlineExpectationsTest
+import semmle.python.functions.ModificationOfParameterWithDefault
+private import semmle.python.dataflow.new.internal.PrintNode
+
+class ModificationOfParameterWithDefaultTest extends InlineExpectationsTest {
+  ModificationOfParameterWithDefaultTest() { this = "ModificationOfParameterWithDefaultTest" }
+
+  override string getARelevantTag() { result = "modification" }
+
+  predicate relevant_node(DataFlow::Node sink) {
+    exists(ModificationOfParameterWithDefault::Configuration cfg | cfg.hasFlowTo(sink))
+  }
+
+  override predicate hasActualResult(Location location, string element, string tag, string value) {
+    exists(DataFlow::Node n | relevant_node(n) |
+      n.getLocation() = location and
+      tag = "modification" and
+      value = prettyNode(n) and
+      element = n.toString()
+    )
+  }
+}

python/ql/test/query-tests/Functions/general/ModificationOfParameterWithDefault.expected

@@ -1,24 +1,44 @@
 edges
-| functions_test.py:39:9:39:9 | empty mutable value | functions_test.py:40:5:40:5 | empty mutable value |
-| functions_test.py:133:15:133:15 | empty mutable value | functions_test.py:134:5:134:5 | empty mutable value |
-| functions_test.py:151:25:151:25 | empty mutable value | functions_test.py:152:5:152:5 | empty mutable value |
-| functions_test.py:154:21:154:21 | empty mutable value | functions_test.py:155:5:155:5 | empty mutable value |
-| functions_test.py:157:27:157:27 | empty mutable value | functions_test.py:158:25:158:25 | empty mutable value |
-| functions_test.py:157:27:157:27 | empty mutable value | functions_test.py:159:21:159:21 | empty mutable value |
-| functions_test.py:158:25:158:25 | empty mutable value | functions_test.py:151:25:151:25 | empty mutable value |
-| functions_test.py:159:21:159:21 | empty mutable value | functions_test.py:154:21:154:21 | empty mutable value |
-| functions_test.py:175:28:175:28 | non-empty mutable value | functions_test.py:179:9:179:9 | non-empty mutable value |
-| functions_test.py:175:28:175:28 | non-empty mutable value | functions_test.py:181:9:181:9 | non-empty mutable value |
-| functions_test.py:188:18:188:18 | non-empty mutable value | functions_test.py:189:28:189:28 | non-empty mutable value |
-| functions_test.py:189:28:189:28 | non-empty mutable value | functions_test.py:175:28:175:28 | non-empty mutable value |
-| functions_test.py:191:18:191:18 | non-empty mutable value | functions_test.py:192:28:192:28 | non-empty mutable value |
-| functions_test.py:192:28:192:28 | non-empty mutable value | functions_test.py:175:28:175:28 | non-empty mutable value |
+| functions_test.py:39:9:39:9 | ControlFlowNode for x | functions_test.py:40:5:40:5 | ControlFlowNode for x |
+| functions_test.py:133:15:133:15 | ControlFlowNode for x | functions_test.py:134:5:134:5 | ControlFlowNode for x |
+| functions_test.py:151:25:151:25 | ControlFlowNode for x | functions_test.py:152:5:152:5 | ControlFlowNode for x |
+| functions_test.py:154:21:154:21 | ControlFlowNode for x | functions_test.py:155:5:155:5 | ControlFlowNode for x |
+| functions_test.py:157:27:157:27 | ControlFlowNode for y | functions_test.py:158:25:158:25 | ControlFlowNode for y |
+| functions_test.py:157:27:157:27 | ControlFlowNode for y | functions_test.py:159:21:159:21 | ControlFlowNode for y |
+| functions_test.py:158:25:158:25 | ControlFlowNode for y | functions_test.py:151:25:151:25 | ControlFlowNode for x |
+| functions_test.py:159:21:159:21 | ControlFlowNode for y | functions_test.py:154:21:154:21 | ControlFlowNode for x |
+| functions_test.py:179:28:179:28 | ControlFlowNode for x | functions_test.py:183:9:183:9 | ControlFlowNode for x |
+| functions_test.py:179:28:179:28 | ControlFlowNode for x | functions_test.py:185:9:185:9 | ControlFlowNode for x |
+| functions_test.py:192:18:192:18 | ControlFlowNode for x | functions_test.py:193:28:193:28 | ControlFlowNode for x |
+| functions_test.py:193:28:193:28 | ControlFlowNode for x | functions_test.py:179:28:179:28 | ControlFlowNode for x |
+| functions_test.py:195:18:195:18 | ControlFlowNode for x | functions_test.py:196:28:196:28 | ControlFlowNode for x |
+| functions_test.py:196:28:196:28 | ControlFlowNode for x | functions_test.py:179:28:179:28 | ControlFlowNode for x |
+nodes
+| functions_test.py:39:9:39:9 | ControlFlowNode for x | semmle.label | ControlFlowNode for x |
+| functions_test.py:40:5:40:5 | ControlFlowNode for x | semmle.label | ControlFlowNode for x |
+| functions_test.py:133:15:133:15 | ControlFlowNode for x | semmle.label | ControlFlowNode for x |
+| functions_test.py:134:5:134:5 | ControlFlowNode for x | semmle.label | ControlFlowNode for x |
+| functions_test.py:151:25:151:25 | ControlFlowNode for x | semmle.label | ControlFlowNode for x |
+| functions_test.py:152:5:152:5 | ControlFlowNode for x | semmle.label | ControlFlowNode for x |
+| functions_test.py:154:21:154:21 | ControlFlowNode for x | semmle.label | ControlFlowNode for x |
+| functions_test.py:155:5:155:5 | ControlFlowNode for x | semmle.label | ControlFlowNode for x |
+| functions_test.py:157:27:157:27 | ControlFlowNode for y | semmle.label | ControlFlowNode for y |
+| functions_test.py:158:25:158:25 | ControlFlowNode for y | semmle.label | ControlFlowNode for y |
+| functions_test.py:159:21:159:21 | ControlFlowNode for y | semmle.label | ControlFlowNode for y |
+| functions_test.py:179:28:179:28 | ControlFlowNode for x | semmle.label | ControlFlowNode for x |
+| functions_test.py:183:9:183:9 | ControlFlowNode for x | semmle.label | ControlFlowNode for x |
+| functions_test.py:185:9:185:9 | ControlFlowNode for x | semmle.label | ControlFlowNode for x |
+| functions_test.py:192:18:192:18 | ControlFlowNode for x | semmle.label | ControlFlowNode for x |
+| functions_test.py:193:28:193:28 | ControlFlowNode for x | semmle.label | ControlFlowNode for x |
+| functions_test.py:195:18:195:18 | ControlFlowNode for x | semmle.label | ControlFlowNode for x |
+| functions_test.py:196:28:196:28 | ControlFlowNode for x | semmle.label | ControlFlowNode for x |
+subpaths
 #select
-| functions_test.py:40:5:40:5 | x | functions_test.py:39:9:39:9 | empty mutable value | functions_test.py:40:5:40:5 | empty mutable value | $@ flows to here and is mutated. | functions_test.py:39:9:39:9 | x | Default value |
-| functions_test.py:134:5:134:5 | x | functions_test.py:133:15:133:15 | empty mutable value | functions_test.py:134:5:134:5 | empty mutable value | $@ flows to here and is mutated. | functions_test.py:133:15:133:15 | x | Default value |
-| functions_test.py:152:5:152:5 | x | functions_test.py:157:27:157:27 | empty mutable value | functions_test.py:152:5:152:5 | empty mutable value | $@ flows to here and is mutated. | functions_test.py:157:27:157:27 | y | Default value |
-| functions_test.py:155:5:155:5 | x | functions_test.py:157:27:157:27 | empty mutable value | functions_test.py:155:5:155:5 | empty mutable value | $@ flows to here and is mutated. | functions_test.py:157:27:157:27 | y | Default value |
-| functions_test.py:179:9:179:9 | x | functions_test.py:188:18:188:18 | non-empty mutable value | functions_test.py:179:9:179:9 | non-empty mutable value | $@ flows to here and is mutated. | functions_test.py:188:18:188:18 | x | Default value |
-| functions_test.py:179:9:179:9 | x | functions_test.py:191:18:191:18 | non-empty mutable value | functions_test.py:179:9:179:9 | non-empty mutable value | $@ flows to here and is mutated. | functions_test.py:191:18:191:18 | x | Default value |
-| functions_test.py:181:9:181:9 | x | functions_test.py:188:18:188:18 | non-empty mutable value | functions_test.py:181:9:181:9 | non-empty mutable value | $@ flows to here and is mutated. | functions_test.py:188:18:188:18 | x | Default value |
-| functions_test.py:181:9:181:9 | x | functions_test.py:191:18:191:18 | non-empty mutable value | functions_test.py:181:9:181:9 | non-empty mutable value | $@ flows to here and is mutated. | functions_test.py:191:18:191:18 | x | Default value |
+| functions_test.py:40:5:40:5 | ControlFlowNode for x | functions_test.py:39:9:39:9 | ControlFlowNode for x | functions_test.py:40:5:40:5 | ControlFlowNode for x | $@ flows to here and is mutated. | functions_test.py:39:9:39:9 | ControlFlowNode for x | Default value |
+| functions_test.py:134:5:134:5 | ControlFlowNode for x | functions_test.py:133:15:133:15 | ControlFlowNode for x | functions_test.py:134:5:134:5 | ControlFlowNode for x | $@ flows to here and is mutated. | functions_test.py:133:15:133:15 | ControlFlowNode for x | Default value |
+| functions_test.py:152:5:152:5 | ControlFlowNode for x | functions_test.py:157:27:157:27 | ControlFlowNode for y | functions_test.py:152:5:152:5 | ControlFlowNode for x | $@ flows to here and is mutated. | functions_test.py:157:27:157:27 | ControlFlowNode for y | Default value |
+| functions_test.py:155:5:155:5 | ControlFlowNode for x | functions_test.py:157:27:157:27 | ControlFlowNode for y | functions_test.py:155:5:155:5 | ControlFlowNode for x | $@ flows to here and is mutated. | functions_test.py:157:27:157:27 | ControlFlowNode for y | Default value |
+| functions_test.py:183:9:183:9 | ControlFlowNode for x | functions_test.py:192:18:192:18 | ControlFlowNode for x | functions_test.py:183:9:183:9 | ControlFlowNode for x | $@ flows to here and is mutated. | functions_test.py:192:18:192:18 | ControlFlowNode for x | Default value |
+| functions_test.py:183:9:183:9 | ControlFlowNode for x | functions_test.py:195:18:195:18 | ControlFlowNode for x | functions_test.py:183:9:183:9 | ControlFlowNode for x | $@ flows to here and is mutated. | functions_test.py:195:18:195:18 | ControlFlowNode for x | Default value |
+| functions_test.py:185:9:185:9 | ControlFlowNode for x | functions_test.py:192:18:192:18 | ControlFlowNode for x | functions_test.py:185:9:185:9 | ControlFlowNode for x | $@ flows to here and is mutated. | functions_test.py:192:18:192:18 | ControlFlowNode for x | Default value |
+| functions_test.py:185:9:185:9 | ControlFlowNode for x | functions_test.py:195:18:195:18 | ControlFlowNode for x | functions_test.py:185:9:185:9 | ControlFlowNode for x | $@ flows to here and is mutated. | functions_test.py:195:18:195:18 | ControlFlowNode for x | Default value |

python/ql/test/query-tests/Functions/general/functions_test.py

@@ -163,11 +163,15 @@ def guarded_modification(z=[]):
         z.append(0)
     return z
 
-def issue1143(expr, param=[]):
-    if not param:
-        return result
-    for i in param:
-        param.remove(i) # Mutation here
+# This function causes a discrepancy between the
+# Python 2 and 3 versions of the analysis.
+# We comment it out until we have resoved the issue.
+#
+# def issue1143(expr, param=[]):
+#     if not param:
+#         return result
+#     for i in param:
+#         param.remove(i) # Mutation here
 
 
 # Type guarding of modification of parameter with default:

python/ql/test/query-tests/Functions/general/options

@@ -0,0 +1 @@
+semmle-extractor-options: --max-import-depth=1 --dont-split-graph

python/ql/test/query-tests/Security/CWE-089-SqlInjection/SqlInjection.expected

@@ -3,15 +3,52 @@ edges
 | sql_injection.py:14:15:14:22 | ControlFlowNode for username | sql_injection.py:24:38:24:95 | ControlFlowNode for BinaryExpr |
 | sql_injection.py:14:15:14:22 | ControlFlowNode for username | sql_injection.py:25:26:25:83 | ControlFlowNode for BinaryExpr |
 | sql_injection.py:14:15:14:22 | ControlFlowNode for username | sql_injection.py:26:28:26:85 | ControlFlowNode for BinaryExpr |
+| sqlalchemy_textclause.py:23:15:23:22 | ControlFlowNode for username | sqlalchemy_textclause.py:27:28:27:87 | ControlFlowNode for Attribute() |
+| sqlalchemy_textclause.py:23:15:23:22 | ControlFlowNode for username | sqlalchemy_textclause.py:31:50:31:72 | ControlFlowNode for Attribute() |
+| sqlalchemy_textclause.py:23:15:23:22 | ControlFlowNode for username | sqlalchemy_textclause.py:41:26:41:33 | ControlFlowNode for username |
+| sqlalchemy_textclause.py:23:15:23:22 | ControlFlowNode for username | sqlalchemy_textclause.py:42:31:42:38 | ControlFlowNode for username |
+| sqlalchemy_textclause.py:23:15:23:22 | ControlFlowNode for username | sqlalchemy_textclause.py:43:30:43:37 | ControlFlowNode for username |
+| sqlalchemy_textclause.py:23:15:23:22 | ControlFlowNode for username | sqlalchemy_textclause.py:44:35:44:42 | ControlFlowNode for username |
+| sqlalchemy_textclause.py:23:15:23:22 | ControlFlowNode for username | sqlalchemy_textclause.py:45:41:45:48 | ControlFlowNode for username |
+| sqlalchemy_textclause.py:23:15:23:22 | ControlFlowNode for username | sqlalchemy_textclause.py:46:46:46:53 | ControlFlowNode for username |
+| sqlalchemy_textclause.py:23:15:23:22 | ControlFlowNode for username | sqlalchemy_textclause.py:47:47:47:54 | ControlFlowNode for username |
+| sqlalchemy_textclause.py:23:15:23:22 | ControlFlowNode for username | sqlalchemy_textclause.py:48:52:48:59 | ControlFlowNode for username |
+| sqlalchemy_textclause.py:23:15:23:22 | ControlFlowNode for username | sqlalchemy_textclause.py:50:18:50:25 | ControlFlowNode for username |
+| sqlalchemy_textclause.py:23:15:23:22 | ControlFlowNode for username | sqlalchemy_textclause.py:51:24:51:31 | ControlFlowNode for username |
 nodes
 | sql_injection.py:14:15:14:22 | ControlFlowNode for username | semmle.label | ControlFlowNode for username |
 | sql_injection.py:21:24:21:77 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr |
 | sql_injection.py:24:38:24:95 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr |
 | sql_injection.py:25:26:25:83 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr |
 | sql_injection.py:26:28:26:85 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr |
+| sqlalchemy_textclause.py:23:15:23:22 | ControlFlowNode for username | semmle.label | ControlFlowNode for username |
+| sqlalchemy_textclause.py:27:28:27:87 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() |
+| sqlalchemy_textclause.py:31:50:31:72 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() |
+| sqlalchemy_textclause.py:41:26:41:33 | ControlFlowNode for username | semmle.label | ControlFlowNode for username |
+| sqlalchemy_textclause.py:42:31:42:38 | ControlFlowNode for username | semmle.label | ControlFlowNode for username |
+| sqlalchemy_textclause.py:43:30:43:37 | ControlFlowNode for username | semmle.label | ControlFlowNode for username |
+| sqlalchemy_textclause.py:44:35:44:42 | ControlFlowNode for username | semmle.label | ControlFlowNode for username |
+| sqlalchemy_textclause.py:45:41:45:48 | ControlFlowNode for username | semmle.label | ControlFlowNode for username |
+| sqlalchemy_textclause.py:46:46:46:53 | ControlFlowNode for username | semmle.label | ControlFlowNode for username |
+| sqlalchemy_textclause.py:47:47:47:54 | ControlFlowNode for username | semmle.label | ControlFlowNode for username |
+| sqlalchemy_textclause.py:48:52:48:59 | ControlFlowNode for username | semmle.label | ControlFlowNode for username |
+| sqlalchemy_textclause.py:50:18:50:25 | ControlFlowNode for username | semmle.label | ControlFlowNode for username |
+| sqlalchemy_textclause.py:51:24:51:31 | ControlFlowNode for username | semmle.label | ControlFlowNode for username |
 subpaths
 #select
 | sql_injection.py:21:24:21:77 | ControlFlowNode for BinaryExpr | sql_injection.py:14:15:14:22 | ControlFlowNode for username | sql_injection.py:21:24:21:77 | ControlFlowNode for BinaryExpr | This SQL query depends on $@. | sql_injection.py:14:15:14:22 | ControlFlowNode for username | a user-provided value |
 | sql_injection.py:24:38:24:95 | ControlFlowNode for BinaryExpr | sql_injection.py:14:15:14:22 | ControlFlowNode for username | sql_injection.py:24:38:24:95 | ControlFlowNode for BinaryExpr | This SQL query depends on $@. | sql_injection.py:14:15:14:22 | ControlFlowNode for username | a user-provided value |
 | sql_injection.py:25:26:25:83 | ControlFlowNode for BinaryExpr | sql_injection.py:14:15:14:22 | ControlFlowNode for username | sql_injection.py:25:26:25:83 | ControlFlowNode for BinaryExpr | This SQL query depends on $@. | sql_injection.py:14:15:14:22 | ControlFlowNode for username | a user-provided value |
 | sql_injection.py:26:28:26:85 | ControlFlowNode for BinaryExpr | sql_injection.py:14:15:14:22 | ControlFlowNode for username | sql_injection.py:26:28:26:85 | ControlFlowNode for BinaryExpr | This SQL query depends on $@. | sql_injection.py:14:15:14:22 | ControlFlowNode for username | a user-provided value |
+| sqlalchemy_textclause.py:27:28:27:87 | ControlFlowNode for Attribute() | sqlalchemy_textclause.py:23:15:23:22 | ControlFlowNode for username | sqlalchemy_textclause.py:27:28:27:87 | ControlFlowNode for Attribute() | This SQL query depends on $@. | sqlalchemy_textclause.py:23:15:23:22 | ControlFlowNode for username | a user-provided value |
+| sqlalchemy_textclause.py:31:50:31:72 | ControlFlowNode for Attribute() | sqlalchemy_textclause.py:23:15:23:22 | ControlFlowNode for username | sqlalchemy_textclause.py:31:50:31:72 | ControlFlowNode for Attribute() | This SQL query depends on $@. | sqlalchemy_textclause.py:23:15:23:22 | ControlFlowNode for username | a user-provided value |
+| sqlalchemy_textclause.py:41:26:41:33 | ControlFlowNode for username | sqlalchemy_textclause.py:23:15:23:22 | ControlFlowNode for username | sqlalchemy_textclause.py:41:26:41:33 | ControlFlowNode for username | This SQL query depends on $@. | sqlalchemy_textclause.py:23:15:23:22 | ControlFlowNode for username | a user-provided value |
+| sqlalchemy_textclause.py:42:31:42:38 | ControlFlowNode for username | sqlalchemy_textclause.py:23:15:23:22 | ControlFlowNode for username | sqlalchemy_textclause.py:42:31:42:38 | ControlFlowNode for username | This SQL query depends on $@. | sqlalchemy_textclause.py:23:15:23:22 | ControlFlowNode for username | a user-provided value |
+| sqlalchemy_textclause.py:43:30:43:37 | ControlFlowNode for username | sqlalchemy_textclause.py:23:15:23:22 | ControlFlowNode for username | sqlalchemy_textclause.py:43:30:43:37 | ControlFlowNode for username | This SQL query depends on $@. | sqlalchemy_textclause.py:23:15:23:22 | ControlFlowNode for username | a user-provided value |
+| sqlalchemy_textclause.py:44:35:44:42 | ControlFlowNode for username | sqlalchemy_textclause.py:23:15:23:22 | ControlFlowNode for username | sqlalchemy_textclause.py:44:35:44:42 | ControlFlowNode for username | This SQL query depends on $@. | sqlalchemy_textclause.py:23:15:23:22 | ControlFlowNode for username | a user-provided value |
+| sqlalchemy_textclause.py:45:41:45:48 | ControlFlowNode for username | sqlalchemy_textclause.py:23:15:23:22 | ControlFlowNode for username | sqlalchemy_textclause.py:45:41:45:48 | ControlFlowNode for username | This SQL query depends on $@. | sqlalchemy_textclause.py:23:15:23:22 | ControlFlowNode for username | a user-provided value |
+| sqlalchemy_textclause.py:46:46:46:53 | ControlFlowNode for username | sqlalchemy_textclause.py:23:15:23:22 | ControlFlowNode for username | sqlalchemy_textclause.py:46:46:46:53 | ControlFlowNode for username | This SQL query depends on $@. | sqlalchemy_textclause.py:23:15:23:22 | ControlFlowNode for username | a user-provided value |
+| sqlalchemy_textclause.py:47:47:47:54 | ControlFlowNode for username | sqlalchemy_textclause.py:23:15:23:22 | ControlFlowNode for username | sqlalchemy_textclause.py:47:47:47:54 | ControlFlowNode for username | This SQL query depends on $@. | sqlalchemy_textclause.py:23:15:23:22 | ControlFlowNode for username | a user-provided value |
+| sqlalchemy_textclause.py:48:52:48:59 | ControlFlowNode for username | sqlalchemy_textclause.py:23:15:23:22 | ControlFlowNode for username | sqlalchemy_textclause.py:48:52:48:59 | ControlFlowNode for username | This SQL query depends on $@. | sqlalchemy_textclause.py:23:15:23:22 | ControlFlowNode for username | a user-provided value |
+| sqlalchemy_textclause.py:50:18:50:25 | ControlFlowNode for username | sqlalchemy_textclause.py:23:15:23:22 | ControlFlowNode for username | sqlalchemy_textclause.py:50:18:50:25 | ControlFlowNode for username | This SQL query depends on $@. | sqlalchemy_textclause.py:23:15:23:22 | ControlFlowNode for username | a user-provided value |
+| sqlalchemy_textclause.py:51:24:51:31 | ControlFlowNode for username | sqlalchemy_textclause.py:23:15:23:22 | ControlFlowNode for username | sqlalchemy_textclause.py:51:24:51:31 | ControlFlowNode for username | This SQL query depends on $@. | sqlalchemy_textclause.py:23:15:23:22 | ControlFlowNode for username | a user-provided value |

python/ql/test/query-tests/Security/CWE-089-SqlInjection/sqlalchemy_textclause.py

@@ -0,0 +1,51 @@
+from flask import Flask, request
+import sqlalchemy
+import sqlalchemy.orm
+from flask_sqlalchemy import SQLAlchemy
+
+app = Flask(__name__)
+engine = sqlalchemy.create_engine(...)
+Base = sqlalchemy.orm.declarative_base()
+
+app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite+pysqlite:///:memory:"
+db = SQLAlchemy(app)
+
+
+
+class User(Base):
+    __tablename__ = "users"
+
+    id = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True)
+    username = sqlalchemy.Column(sqlalchemy.String)
+
+
+@app.route("/users/<username>")
+def show_user(username):
+    session = sqlalchemy.orm.Session(engine)
+
+    # BAD, normal SQL injection
+    stmt = sqlalchemy.text("SELECT * FROM users WHERE username = '{}'".format(username))
+    results = session.execute(stmt).fetchall()
+
+    # BAD, allows SQL injection
+    username_formatted_for_sql = sqlalchemy.text("'{}'".format(username))
+    stmt = sqlalchemy.select(User).where(User.username == username_formatted_for_sql)
+    results = session.execute(stmt).scalars().all()
+
+    # GOOD, does not allow for SQL injection
+    stmt = sqlalchemy.select(User).where(User.username == username)
+    results = session.execute(stmt).scalars().all()
+
+
+    # All of these should be flagged by query
+    t1 = sqlalchemy.text(username)
+    t2 = sqlalchemy.text(text=username)
+    t3 = sqlalchemy.sql.text(username)
+    t4 = sqlalchemy.sql.text(text=username)
+    t5 = sqlalchemy.sql.expression.text(username)
+    t6 = sqlalchemy.sql.expression.text(text=username)
+    t7 = sqlalchemy.sql.expression.TextClause(username)
+    t8 = sqlalchemy.sql.expression.TextClause(text=username)
+
+    t9 = db.text(username)
+    t10 = db.text(text=username)

python/ql/test/query-tests/Variables/undefined/UninitializedLocal.py

@@ -288,3 +288,8 @@ def avoid_redundant_split(a):
         var = False
     if var:
         foo.bar() #foo is defined here.
+
+def type_annotation_fp():
+    annotated : annotation = [1,2,3]
+    for x in annotated:
+        print(x)

python/ql/test/query-tests/Variables/unused/UnusedLocalVariable.expected

@@ -1,4 +1,3 @@
-| type_annotation_fp.py:5:5:5:7 | foo | The value assigned to local variable 'foo' is never used. |
 | variables_test.py:29:5:29:5 | x | The value assigned to local variable 'x' is never used. |
 | variables_test.py:89:5:89:5 | a | The value assigned to local variable 'a' is never used. |
 | variables_test.py:89:7:89:7 | b | The value assigned to local variable 'b' is never used. |

python/ql/test/query-tests/Variables/unused/type_annotation_fp.py

@@ -9,3 +9,8 @@ def type_annotation(x):
     else:
         foo : float
         do_other_stuff_with(foo)
+
+def type_annotation_fn():
+    # False negative: the value of `bar` is never used, but this is masked by the presence of the type annotation.
+    bar = 5
+    bar : int

python/upgrades/4f1806347d7fafe2f78508da01c01e5aff5f7cbb/old.dbscheme

@@ -0,0 +1,999 @@
+/*
+ *      This dbscheme is auto-generated by 'semmle/dbscheme_gen.py'.
+ *      WARNING: Any modifications to this file will be lost.
+ *      Relations can be changed by modifying master.py or
+ *      by adding rules to dbscheme.template
+ */
+
+/* This is a dummy line to alter the dbscheme, so we can make a database upgrade
+ * without actually changing any of the dbscheme predicates. It contains a date
+ * to allow for such updates in the future as well.
+ *
+ * 2020-07-02
+ *
+ * DO NOT remove this comment carelessly, since it can revert the dbscheme back to a
+ * previously seen state (matching a previously seen SHA), which would make the upgrade
+ * mechanism not work properly.
+ */
+
+ /*
+ * External artifacts
+ */
+
+externalDefects(
+  unique int id : @externalDefect,
+  varchar(900) queryPath : string ref,
+  int location : @location ref,
+  varchar(900) message : string ref,
+  float severity : float ref
+);
+
+externalMetrics(
+  unique int id : @externalMetric,
+  varchar(900) queryPath : string ref,
+  int location : @location ref,
+  float value : float ref
+);
+
+externalData(
+  int id : @externalDataElement,
+  varchar(900) queryPath : string ref,
+  int column: int ref,
+  varchar(900) data : string ref
+);
+
+snapshotDate(unique date snapshotDate : date ref);
+
+sourceLocationPrefix(varchar(900) prefix : string ref);
+
+
+/*
+ * Duplicate code
+ */
+
+duplicateCode(
+  unique int id : @duplication,
+  varchar(900) relativePath : string ref,
+  int equivClass : int ref);
+
+similarCode(
+  unique int id : @similarity,
+  varchar(900) relativePath : string ref,
+  int equivClass : int ref);
+
+@duplication_or_similarity = @duplication | @similarity
+
+tokens(
+  int id : @duplication_or_similarity ref,
+  int offset : int ref,
+  int beginLine : int ref,
+  int beginColumn : int ref,
+  int endLine : int ref,
+  int endColumn : int ref);
+
+/*
+ * Line metrics
+ */
+py_codelines(int id : @py_scope ref,
+             int count : int ref);
+
+py_commentlines(int id : @py_scope ref,
+                int count : int ref);
+
+py_docstringlines(int id : @py_scope ref,
+                int count : int ref);
+
+py_alllines(int id : @py_scope ref,
+                int count : int ref);
+
+/*
+ * Version history
+ */
+
+svnentries(
+  int id : @svnentry,
+  varchar(500) revision : string ref,
+  varchar(500) author : string ref,
+  date revisionDate : date ref,
+  int changeSize : int ref
+)
+
+svnaffectedfiles(
+  int id : @svnentry ref,
+  int file : @file ref,
+  varchar(500) action : string ref
+)
+
+svnentrymsg(
+  int id : @svnentry ref,
+  varchar(500) message : string ref
+)
+
+svnchurn(
+  int commit : @svnentry ref,
+  int file : @file ref,
+  int addedLines : int ref,
+  int deletedLines : int ref
+)
+
+/****************************
+        Python dbscheme
+****************************/
+
+/* fromSource is ignored */
+files(unique int id: @file,
+      varchar(900) name: string ref,
+      varchar(900) simple: string ref,
+      varchar(900) ext: string ref,
+      int fromSource: int ref);
+
+folders(unique int id: @folder,
+        varchar(900) name: string ref,
+        varchar(900) simple: string ref);
+
+@container = @folder | @file;
+
+containerparent(int parent: @container ref,
+                unique int child: @container ref);
+
+@sourceline = @file | @py_Module | @xmllocatable;
+
+numlines(int element_id: @sourceline ref,
+         int num_lines: int ref,
+         int num_code: int ref,
+         int num_comment: int ref
+        );
+
+@location = @location_ast | @location_default ;
+
+locations_default(unique int id: @location_default,
+          int file: @file ref,
+          int beginLine: int ref,
+          int beginColumn: int ref,
+          int endLine: int ref,
+          int endColumn: int ref);
+
+locations_ast(unique int id: @location_ast,
+          int module: @py_Module ref,
+          int beginLine: int ref,
+          int beginColumn: int ref,
+          int endLine: int ref,
+          int endColumn: int ref);
+
+file_contents(unique int file: @file ref, string contents: string ref);
+
+py_module_path(int module: @py_Module ref, int file: @container ref);
+
+variable(unique int id : @py_variable,
+         int scope : @py_scope ref,
+         varchar(1) name : string ref);
+
+py_line_lengths(unique int id : @py_line,
+                int file: @py_Module ref,
+                int line : int ref,
+                int length : int ref);
+
+py_extracted_version(int module : @py_Module ref,
+                     varchar(1) version : string ref);
+
+/* AUTO GENERATED PART STARTS HERE */
+
+
+/* <Field> AnnAssign.location = 0, location */
+/* <Field> AnnAssign.value = 1, expr */
+/* <Field> AnnAssign.annotation = 2, expr */
+/* <Field> AnnAssign.target = 3, expr */
+
+/* <Field> Assert.location = 0, location */
+/* <Field> Assert.test = 1, expr */
+/* <Field> Assert.msg = 2, expr */
+
+/* <Field> Assign.location = 0, location */
+/* <Field> Assign.value = 1, expr */
+/* <Field> Assign.targets = 2, expr_list */
+
+/* <Field> AssignExpr.location = 0, location */
+/* <Field> AssignExpr.parenthesised = 1, bool */
+/* <Field> AssignExpr.value = 2, expr */
+/* <Field> AssignExpr.target = 3, expr */
+
+/* <Field> Attribute.location = 0, location */
+/* <Field> Attribute.parenthesised = 1, bool */
+/* <Field> Attribute.value = 2, expr */
+/* <Field> Attribute.attr = 3, str */
+/* <Field> Attribute.ctx = 4, expr_context */
+
+/* <Field> AugAssign.location = 0, location */
+/* <Field> AugAssign.operation = 1, BinOp */
+
+/* <Field> Await.location = 0, location */
+/* <Field> Await.parenthesised = 1, bool */
+/* <Field> Await.value = 2, expr */
+
+/* <Field> BinaryExpr.location = 0, location */
+/* <Field> BinaryExpr.parenthesised = 1, bool */
+/* <Field> BinaryExpr.left = 2, expr */
+/* <Field> BinaryExpr.op = 3, operator */
+/* <Field> BinaryExpr.right = 4, expr */
+/* <Parent> BinaryExpr = AugAssign  */
+
+/* <Field> BoolExpr.location = 0, location */
+/* <Field> BoolExpr.parenthesised = 1, bool */
+/* <Field> BoolExpr.op = 2, boolop */
+/* <Field> BoolExpr.values = 3, expr_list */
+
+/* <Field> Break.location = 0, location */
+
+/* <Field> Bytes.location = 0, location */
+/* <Field> Bytes.parenthesised = 1, bool */
+/* <Field> Bytes.s = 2, bytes */
+/* <Field> Bytes.prefix = 3, bytes */
+/* <Field> Bytes.implicitly_concatenated_parts = 4, StringPart_list */
+
+/* <Field> Call.location = 0, location */
+/* <Field> Call.parenthesised = 1, bool */
+/* <Field> Call.func = 2, expr */
+/* <Field> Call.positional_args = 3, expr_list */
+/* <Field> Call.named_args = 4, dict_item_list */
+
+/* <Field> Class.name = 0, str */
+/* <Field> Class.body = 1, stmt_list */
+/* <Parent> Class = ClassExpr  */
+
+/* <Field> ClassExpr.location = 0, location */
+/* <Field> ClassExpr.parenthesised = 1, bool */
+/* <Field> ClassExpr.name = 2, str */
+/* <Field> ClassExpr.bases = 3, expr_list */
+/* <Field> ClassExpr.keywords = 4, dict_item_list */
+/* <Field> ClassExpr.inner_scope = 5, Class */
+
+/* <Field> Compare.location = 0, location */
+/* <Field> Compare.parenthesised = 1, bool */
+/* <Field> Compare.left = 2, expr */
+/* <Field> Compare.ops = 3, cmpop_list */
+/* <Field> Compare.comparators = 4, expr_list */
+
+/* <Field> Continue.location = 0, location */
+
+/* <Field> Delete.location = 0, location */
+/* <Field> Delete.targets = 1, expr_list */
+
+/* <Field> Dict.location = 0, location */
+/* <Field> Dict.parenthesised = 1, bool */
+/* <Field> Dict.items = 2, dict_item_list */
+
+/* <Field> DictComp.location = 0, location */
+/* <Field> DictComp.parenthesised = 1, bool */
+/* <Field> DictComp.function = 2, Function */
+/* <Field> DictComp.iterable = 3, expr */
+
+/* <Field> DictUnpacking.location = 0, location */
+/* <Field> DictUnpacking.value = 1, expr */
+
+/* <Field> Ellipsis.location = 0, location */
+/* <Field> Ellipsis.parenthesised = 1, bool */
+
+/* <Field> ExceptStmt.location = 0, location */
+/* <Field> ExceptStmt.type = 1, expr */
+/* <Field> ExceptStmt.name = 2, expr */
+/* <Field> ExceptStmt.body = 3, stmt_list */
+
+/* <Field> Exec.location = 0, location */
+/* <Field> Exec.body = 1, expr */
+/* <Field> Exec.globals = 2, expr */
+/* <Field> Exec.locals = 3, expr */
+
+/* <Field> ExprStmt.location = 0, location */
+/* <Field> ExprStmt.value = 1, expr */
+
+/* <Field> Filter.location = 0, location */
+/* <Field> Filter.parenthesised = 1, bool */
+/* <Field> Filter.value = 2, expr */
+/* <Field> Filter.filter = 3, expr */
+
+/* <Field> For.location = 0, location */
+/* <Field> For.target = 1, expr */
+/* <Field> For.iter = 2, expr */
+/* <Field> For.body = 3, stmt_list */
+/* <Field> For.orelse = 4, stmt_list */
+/* <Field> For.is_async = 5, bool */
+
+/* <Field> FormattedValue.location = 0, location */
+/* <Field> FormattedValue.parenthesised = 1, bool */
+/* <Field> FormattedValue.value = 2, expr */
+/* <Field> FormattedValue.conversion = 3, str */
+/* <Field> FormattedValue.format_spec = 4, JoinedStr */
+
+/* <Field> Function.name = 0, str */
+/* <Field> Function.args = 1, parameter_list */
+/* <Field> Function.vararg = 2, expr */
+/* <Field> Function.kwonlyargs = 3, expr_list */
+/* <Field> Function.kwarg = 4, expr */
+/* <Field> Function.body = 5, stmt_list */
+/* <Field> Function.is_async = 6, bool */
+/* <Parent> Function = FunctionParent  */
+
+/* <Field> FunctionExpr.location = 0, location */
+/* <Field> FunctionExpr.parenthesised = 1, bool */
+/* <Field> FunctionExpr.name = 2, str */
+/* <Field> FunctionExpr.args = 3, arguments */
+/* <Field> FunctionExpr.returns = 4, expr */
+/* <Field> FunctionExpr.inner_scope = 5, Function */
+
+/* <Field> GeneratorExp.location = 0, location */
+/* <Field> GeneratorExp.parenthesised = 1, bool */
+/* <Field> GeneratorExp.function = 2, Function */
+/* <Field> GeneratorExp.iterable = 3, expr */
+
+/* <Field> Global.location = 0, location */
+/* <Field> Global.names = 1, str_list */
+
+/* <Field> If.location = 0, location */
+/* <Field> If.test = 1, expr */
+/* <Field> If.body = 2, stmt_list */
+/* <Field> If.orelse = 3, stmt_list */
+
+/* <Field> IfExp.location = 0, location */
+/* <Field> IfExp.parenthesised = 1, bool */
+/* <Field> IfExp.test = 2, expr */
+/* <Field> IfExp.body = 3, expr */
+/* <Field> IfExp.orelse = 4, expr */
+
+/* <Field> Import.location = 0, location */
+/* <Field> Import.names = 1, alias_list */
+
+/* <Field> ImportExpr.location = 0, location */
+/* <Field> ImportExpr.parenthesised = 1, bool */
+/* <Field> ImportExpr.level = 2, int */
+/* <Field> ImportExpr.name = 3, str */
+/* <Field> ImportExpr.top = 4, bool */
+
+/* <Field> ImportStar.location = 0, location */
+/* <Field> ImportStar.module = 1, expr */
+
+/* <Field> ImportMember.location = 0, location */
+/* <Field> ImportMember.parenthesised = 1, bool */
+/* <Field> ImportMember.module = 2, expr */
+/* <Field> ImportMember.name = 3, str */
+
+/* <Field> Fstring.location = 0, location */
+/* <Field> Fstring.parenthesised = 1, bool */
+/* <Field> Fstring.values = 2, expr_list */
+/* <Parent> Fstring = FormattedValue  */
+
+/* <Field> KeyValuePair.location = 0, location */
+/* <Field> KeyValuePair.value = 1, expr */
+/* <Field> KeyValuePair.key = 2, expr */
+
+/* <Field> Lambda.location = 0, location */
+/* <Field> Lambda.parenthesised = 1, bool */
+/* <Field> Lambda.args = 2, arguments */
+/* <Field> Lambda.inner_scope = 3, Function */
+
+/* <Field> List.location = 0, location */
+/* <Field> List.parenthesised = 1, bool */
+/* <Field> List.elts = 2, expr_list */
+/* <Field> List.ctx = 3, expr_context */
+
+/* <Field> ListComp.location = 0, location */
+/* <Field> ListComp.parenthesised = 1, bool */
+/* <Field> ListComp.function = 2, Function */
+/* <Field> ListComp.iterable = 3, expr */
+/* <Field> ListComp.generators = 4, comprehension_list */
+/* <Field> ListComp.elt = 5, expr */
+
+/* <Field> Module.name = 0, str */
+/* <Field> Module.hash = 1, str */
+/* <Field> Module.body = 2, stmt_list */
+/* <Field> Module.kind = 3, str */
+
+/* <Field> Name.location = 0, location */
+/* <Field> Name.parenthesised = 1, bool */
+/* <Field> Name.variable = 2, variable */
+/* <Field> Name.ctx = 3, expr_context */
+/* <Parent> Name = ParameterList  */
+
+/* <Field> Nonlocal.location = 0, location */
+/* <Field> Nonlocal.names = 1, str_list */
+
+/* <Field> Num.location = 0, location */
+/* <Field> Num.parenthesised = 1, bool */
+/* <Field> Num.n = 2, number */
+/* <Field> Num.text = 3, number */
+
+/* <Field> Pass.location = 0, location */
+
+/* <Field> PlaceHolder.location = 0, location */
+/* <Field> PlaceHolder.parenthesised = 1, bool */
+/* <Field> PlaceHolder.variable = 2, variable */
+/* <Field> PlaceHolder.ctx = 3, expr_context */
+
+/* <Field> Print.location = 0, location */
+/* <Field> Print.dest = 1, expr */
+/* <Field> Print.values = 2, expr_list */
+/* <Field> Print.nl = 3, bool */
+
+/* <Field> Raise.location = 0, location */
+/* <Field> Raise.exc = 1, expr */
+/* <Field> Raise.cause = 2, expr */
+/* <Field> Raise.type = 3, expr */
+/* <Field> Raise.inst = 4, expr */
+/* <Field> Raise.tback = 5, expr */
+
+/* <Field> Repr.location = 0, location */
+/* <Field> Repr.parenthesised = 1, bool */
+/* <Field> Repr.value = 2, expr */
+
+/* <Field> Return.location = 0, location */
+/* <Field> Return.value = 1, expr */
+
+/* <Field> Set.location = 0, location */
+/* <Field> Set.parenthesised = 1, bool */
+/* <Field> Set.elts = 2, expr_list */
+
+/* <Field> SetComp.location = 0, location */
+/* <Field> SetComp.parenthesised = 1, bool */
+/* <Field> SetComp.function = 2, Function */
+/* <Field> SetComp.iterable = 3, expr */
+
+/* <Field> Slice.location = 0, location */
+/* <Field> Slice.parenthesised = 1, bool */
+/* <Field> Slice.start = 2, expr */
+/* <Field> Slice.stop = 3, expr */
+/* <Field> Slice.step = 4, expr */
+
+/* <Field> SpecialOperation.location = 0, location */
+/* <Field> SpecialOperation.parenthesised = 1, bool */
+/* <Field> SpecialOperation.name = 2, str */
+/* <Field> SpecialOperation.arguments = 3, expr_list */
+
+/* <Field> Starred.location = 0, location */
+/* <Field> Starred.parenthesised = 1, bool */
+/* <Field> Starred.value = 2, expr */
+/* <Field> Starred.ctx = 3, expr_context */
+
+/* <Field> Str.location = 0, location */
+/* <Field> Str.parenthesised = 1, bool */
+/* <Field> Str.s = 2, str */
+/* <Field> Str.prefix = 3, str */
+/* <Field> Str.implicitly_concatenated_parts = 4, StringPart_list */
+
+/* <Field> StringPart.text = 0, str */
+/* <Field> StringPart.location = 1, location */
+/* <Parent> StringPart = StringPartList  */
+/* <Parent> StringPartList = BytesOrStr  */
+
+/* <Field> Subscript.location = 0, location */
+/* <Field> Subscript.parenthesised = 1, bool */
+/* <Field> Subscript.value = 2, expr */
+/* <Field> Subscript.index = 3, expr */
+/* <Field> Subscript.ctx = 4, expr_context */
+
+/* <Field> TemplateDottedNotation.location = 0, location */
+/* <Field> TemplateDottedNotation.parenthesised = 1, bool */
+/* <Field> TemplateDottedNotation.value = 2, expr */
+/* <Field> TemplateDottedNotation.attr = 3, str */
+/* <Field> TemplateDottedNotation.ctx = 4, expr_context */
+
+/* <Field> TemplateWrite.location = 0, location */
+/* <Field> TemplateWrite.value = 1, expr */
+
+/* <Field> Try.location = 0, location */
+/* <Field> Try.body = 1, stmt_list */
+/* <Field> Try.orelse = 2, stmt_list */
+/* <Field> Try.handlers = 3, stmt_list */
+/* <Field> Try.finalbody = 4, stmt_list */
+
+/* <Field> Tuple.location = 0, location */
+/* <Field> Tuple.parenthesised = 1, bool */
+/* <Field> Tuple.elts = 2, expr_list */
+/* <Field> Tuple.ctx = 3, expr_context */
+/* <Parent> Tuple = ParameterList  */
+
+/* <Field> UnaryExpr.location = 0, location */
+/* <Field> UnaryExpr.parenthesised = 1, bool */
+/* <Field> UnaryExpr.op = 2, unaryop */
+/* <Field> UnaryExpr.operand = 3, expr */
+
+/* <Field> While.location = 0, location */
+/* <Field> While.test = 1, expr */
+/* <Field> While.body = 2, stmt_list */
+/* <Field> While.orelse = 3, stmt_list */
+
+/* <Field> With.location = 0, location */
+/* <Field> With.context_expr = 1, expr */
+/* <Field> With.optional_vars = 2, expr */
+/* <Field> With.body = 3, stmt_list */
+/* <Field> With.is_async = 4, bool */
+
+/* <Field> Yield.location = 0, location */
+/* <Field> Yield.parenthesised = 1, bool */
+/* <Field> Yield.value = 2, expr */
+
+/* <Field> YieldFrom.location = 0, location */
+/* <Field> YieldFrom.parenthesised = 1, bool */
+/* <Field> YieldFrom.value = 2, expr */
+
+/* <Field> Alias.value = 0, expr */
+/* <Field> Alias.asname = 1, expr */
+/* <Parent> Alias = AliasList  */
+/* <Parent> AliasList = Import  */
+
+/* <Field> Arguments.kw_defaults = 0, expr_list */
+/* <Field> Arguments.defaults = 1, expr_list */
+/* <Field> Arguments.annotations = 2, expr_list */
+/* <Field> Arguments.varargannotation = 3, expr */
+/* <Field> Arguments.kwargannotation = 4, expr */
+/* <Field> Arguments.kw_annotations = 5, expr_list */
+/* <Parent> Arguments = ArgumentsParent  */
+/* <Parent> boolean = BoolParent  */
+/* <Parent> Boolop = BoolExpr  */
+/* <Parent> string = Bytes  */
+/* <Parent> Cmpop = CmpopList  */
+/* <Parent> CmpopList = Compare  */
+
+/* <Field> Comprehension.location = 0, location */
+/* <Field> Comprehension.iter = 1, expr */
+/* <Field> Comprehension.target = 2, expr */
+/* <Field> Comprehension.ifs = 3, expr_list */
+/* <Parent> Comprehension = ComprehensionList  */
+/* <Parent> ComprehensionList = ListComp  */
+/* <Parent> DictItem = DictItemList  */
+/* <Parent> DictItemList = DictItemListParent  */
+
+/* <Field> Expr.location = 0, location */
+/* <Field> Expr.parenthesised = 1, bool */
+/* <Parent> Expr = ExprParent  */
+/* <Parent> ExprContext = ExprContextParent  */
+/* <Parent> ExprList = ExprListParent  */
+/* <Parent> int = ImportExpr  */
+
+/* <Field> Keyword.location = 0, location */
+/* <Field> Keyword.value = 1, expr */
+/* <Field> Keyword.arg = 2, str */
+/* <Parent> Location = LocationParent  */
+/* <Parent> string = Num  */
+/* <Parent> Operator = BinaryExpr  */
+/* <Parent> ParameterList = Function  */
+
+/* <Field> Stmt.location = 0, location */
+/* <Parent> Stmt = StmtList  */
+/* <Parent> StmtList = StmtListParent  */
+/* <Parent> string = StrParent  */
+/* <Parent> StringList = StrListParent  */
+/* <Parent> Unaryop = UnaryExpr  */
+/* <Parent> Variable = VariableParent  */
+py_Classes(unique int id : @py_Class,
+    unique int parent : @py_ClassExpr ref);
+
+py_Functions(unique int id : @py_Function,
+    unique int parent : @py_Function_parent ref);
+
+py_Modules(unique int id : @py_Module);
+
+py_StringParts(unique int id : @py_StringPart,
+    int parent : @py_StringPart_list ref,
+    int idx : int ref);
+
+py_StringPart_lists(unique int id : @py_StringPart_list,
+    unique int parent : @py_Bytes_or_Str ref);
+
+py_aliases(unique int id : @py_alias,
+    int parent : @py_alias_list ref,
+    int idx : int ref);
+
+py_alias_lists(unique int id : @py_alias_list,
+    unique int parent : @py_Import ref);
+
+py_arguments(unique int id : @py_arguments,
+    unique int parent : @py_arguments_parent ref);
+
+py_bools(int parent : @py_bool_parent ref,
+    int idx : int ref);
+
+py_boolops(unique int id : @py_boolop,
+    int kind: int ref,
+    unique int parent : @py_BoolExpr ref);
+
+py_bytes(varchar(1) id : string ref,
+    int parent : @py_Bytes ref,
+    int idx : int ref);
+
+py_cmpops(unique int id : @py_cmpop,
+    int kind: int ref,
+    int parent : @py_cmpop_list ref,
+    int idx : int ref);
+
+py_cmpop_lists(unique int id : @py_cmpop_list,
+    unique int parent : @py_Compare ref);
+
+py_comprehensions(unique int id : @py_comprehension,
+    int parent : @py_comprehension_list ref,
+    int idx : int ref);
+
+py_comprehension_lists(unique int id : @py_comprehension_list,
+    unique int parent : @py_ListComp ref);
+
+py_dict_items(unique int id : @py_dict_item,
+    int kind: int ref,
+    int parent : @py_dict_item_list ref,
+    int idx : int ref);
+
+py_dict_item_lists(unique int id : @py_dict_item_list,
+    unique int parent : @py_dict_item_list_parent ref);
+
+py_exprs(unique int id : @py_expr,
+    int kind: int ref,
+    int parent : @py_expr_parent ref,
+    int idx : int ref);
+
+py_expr_contexts(unique int id : @py_expr_context,
+    int kind: int ref,
+    unique int parent : @py_expr_context_parent ref);
+
+py_expr_lists(unique int id : @py_expr_list,
+    int parent : @py_expr_list_parent ref,
+    int idx : int ref);
+
+py_ints(int id : int ref,
+    unique int parent : @py_ImportExpr ref);
+
+py_locations(unique int id : @location ref,
+    unique int parent : @py_location_parent ref);
+
+py_numbers(varchar(1) id : string ref,
+    int parent : @py_Num ref,
+    int idx : int ref);
+
+py_operators(unique int id : @py_operator,
+    int kind: int ref,
+    unique int parent : @py_BinaryExpr ref);
+
+py_parameter_lists(unique int id : @py_parameter_list,
+    unique int parent : @py_Function ref);
+
+py_stmts(unique int id : @py_stmt,
+    int kind: int ref,
+    int parent : @py_stmt_list ref,
+    int idx : int ref);
+
+py_stmt_lists(unique int id : @py_stmt_list,
+    int parent : @py_stmt_list_parent ref,
+    int idx : int ref);
+
+py_strs(varchar(1) id : string ref,
+    int parent : @py_str_parent ref,
+    int idx : int ref);
+
+py_str_lists(unique int id : @py_str_list,
+    unique int parent : @py_str_list_parent ref);
+
+py_unaryops(unique int id : @py_unaryop,
+    int kind: int ref,
+    unique int parent : @py_UnaryExpr ref);
+
+py_variables(int id : @py_variable ref,
+    unique int parent : @py_variable_parent ref);
+
+case @py_boolop.kind of
+    0 = @py_And
+|   1 = @py_Or;
+
+case @py_cmpop.kind of
+    0 = @py_Eq
+|   1 = @py_Gt
+|   2 = @py_GtE
+|   3 = @py_In
+|   4 = @py_Is
+|   5 = @py_IsNot
+|   6 = @py_Lt
+|   7 = @py_LtE
+|   8 = @py_NotEq
+|   9 = @py_NotIn;
+
+case @py_dict_item.kind of
+    0 = @py_DictUnpacking
+|   1 = @py_KeyValuePair
+|   2 = @py_keyword;
+
+case @py_expr.kind of
+    0 = @py_Attribute
+|   1 = @py_BinaryExpr
+|   2 = @py_BoolExpr
+|   3 = @py_Bytes
+|   4 = @py_Call
+|   5 = @py_ClassExpr
+|   6 = @py_Compare
+|   7 = @py_Dict
+|   8 = @py_DictComp
+|   9 = @py_Ellipsis
+|   10 = @py_FunctionExpr
+|   11 = @py_GeneratorExp
+|   12 = @py_IfExp
+|   13 = @py_ImportExpr
+|   14 = @py_ImportMember
+|   15 = @py_Lambda
+|   16 = @py_List
+|   17 = @py_ListComp
+|   18 = @py_Name
+|   19 = @py_Num
+|   20 = @py_Repr
+|   21 = @py_Set
+|   22 = @py_SetComp
+|   23 = @py_Slice
+|   24 = @py_Starred
+|   25 = @py_Str
+|   26 = @py_Subscript
+|   27 = @py_Tuple
+|   28 = @py_UnaryExpr
+|   29 = @py_Yield
+|   30 = @py_YieldFrom
+|   31 = @py_TemplateDottedNotation
+|   32 = @py_Filter
+|   33 = @py_PlaceHolder
+|   34 = @py_Await
+|   35 = @py_Fstring
+|   36 = @py_FormattedValue
+|   37 = @py_AssignExpr
+|   38 = @py_SpecialOperation;
+
+case @py_expr_context.kind of
+    0 = @py_AugLoad
+|   1 = @py_AugStore
+|   2 = @py_Del
+|   3 = @py_Load
+|   4 = @py_Param
+|   5 = @py_Store;
+
+case @py_operator.kind of
+    0 = @py_Add
+|   1 = @py_BitAnd
+|   2 = @py_BitOr
+|   3 = @py_BitXor
+|   4 = @py_Div
+|   5 = @py_FloorDiv
+|   6 = @py_LShift
+|   7 = @py_Mod
+|   8 = @py_Mult
+|   9 = @py_Pow
+|   10 = @py_RShift
+|   11 = @py_Sub
+|   12 = @py_MatMult;
+
+case @py_stmt.kind of
+    0 = @py_Assert
+|   1 = @py_Assign
+|   2 = @py_AugAssign
+|   3 = @py_Break
+|   4 = @py_Continue
+|   5 = @py_Delete
+|   6 = @py_ExceptStmt
+|   7 = @py_Exec
+|   8 = @py_Expr_stmt
+|   9 = @py_For
+|   10 = @py_Global
+|   11 = @py_If
+|   12 = @py_Import
+|   13 = @py_ImportStar
+|   14 = @py_Nonlocal
+|   15 = @py_Pass
+|   16 = @py_Print
+|   17 = @py_Raise
+|   18 = @py_Return
+|   19 = @py_Try
+|   20 = @py_While
+|   21 = @py_With
+|   22 = @py_TemplateWrite
+|   23 = @py_AnnAssign;
+
+case @py_unaryop.kind of
+    0 = @py_Invert
+|   1 = @py_Not
+|   2 = @py_UAdd
+|   3 = @py_USub;
+
+@py_Bytes_or_Str = @py_Bytes | @py_Str;
+
+@py_Function_parent = @py_DictComp | @py_FunctionExpr | @py_GeneratorExp | @py_Lambda | @py_ListComp | @py_SetComp;
+
+@py_arguments_parent = @py_FunctionExpr | @py_Lambda;
+
+@py_ast_node = @py_Class | @py_Function | @py_Module | @py_StringPart | @py_comprehension | @py_dict_item | @py_expr | @py_stmt;
+
+@py_bool_parent = @py_For | @py_Function | @py_Print | @py_With | @py_expr;
+
+@py_dict_item_list_parent = @py_Call | @py_ClassExpr | @py_Dict;
+
+@py_expr_context_parent = @py_Attribute | @py_List | @py_Name | @py_PlaceHolder | @py_Starred | @py_Subscript | @py_TemplateDottedNotation | @py_Tuple;
+
+@py_expr_list_parent = @py_Assign | @py_BoolExpr | @py_Call | @py_ClassExpr | @py_Compare | @py_Delete | @py_Fstring | @py_Function | @py_List | @py_Print | @py_Set | @py_SpecialOperation | @py_Tuple | @py_arguments | @py_comprehension;
+
+@py_expr_or_stmt = @py_expr | @py_stmt;
+
+@py_expr_parent = @py_AnnAssign | @py_Assert | @py_Assign | @py_AssignExpr | @py_Attribute | @py_AugAssign | @py_Await | @py_BinaryExpr | @py_Call | @py_Compare | @py_DictComp | @py_DictUnpacking | @py_ExceptStmt | @py_Exec | @py_Expr_stmt | @py_Filter | @py_For | @py_FormattedValue | @py_Function | @py_FunctionExpr | @py_GeneratorExp | @py_If | @py_IfExp | @py_ImportMember | @py_ImportStar | @py_KeyValuePair | @py_ListComp | @py_Print | @py_Raise | @py_Repr | @py_Return | @py_SetComp | @py_Slice | @py_Starred | @py_Subscript | @py_TemplateDottedNotation | @py_TemplateWrite | @py_UnaryExpr | @py_While | @py_With | @py_Yield | @py_YieldFrom | @py_alias | @py_arguments | @py_comprehension | @py_expr_list | @py_keyword | @py_parameter_list;
+
+@py_location_parent = @py_DictUnpacking | @py_KeyValuePair | @py_StringPart | @py_comprehension | @py_expr | @py_keyword | @py_stmt;
+
+@py_parameter = @py_Name | @py_Tuple;
+
+@py_scope = @py_Class | @py_Function | @py_Module;
+
+@py_stmt_list_parent = @py_Class | @py_ExceptStmt | @py_For | @py_Function | @py_If | @py_Module | @py_Try | @py_While | @py_With;
+
+@py_str_list_parent = @py_Global | @py_Nonlocal;
+
+@py_str_parent = @py_Attribute | @py_Class | @py_ClassExpr | @py_FormattedValue | @py_Function | @py_FunctionExpr | @py_ImportExpr | @py_ImportMember | @py_Module | @py_SpecialOperation | @py_Str | @py_StringPart | @py_TemplateDottedNotation | @py_keyword | @py_str_list;
+
+@py_variable_parent = @py_Name | @py_PlaceHolder;
+
+
+/*
+ * End of auto-generated part
+ */
+
+
+
+/* Map relative names to absolute names for imports */
+py_absolute_names(int module : @py_Module ref,
+                  varchar(1) relname : string ref,
+                  varchar(1) absname : string ref);
+
+py_exports(int id : @py_Module ref,
+           varchar(1) name : string ref);
+
+/* Successor information */
+py_successors(int predecessor : @py_flow_node ref,
+              int successor : @py_flow_node ref);
+
+py_true_successors(int predecessor : @py_flow_node ref,
+              int successor : @py_flow_node ref);
+
+py_exception_successors(int predecessor : @py_flow_node ref,
+              int successor : @py_flow_node ref);
+
+py_false_successors(int predecessor : @py_flow_node ref,
+              int successor : @py_flow_node ref);
+
+py_flow_bb_node(unique int flownode : @py_flow_node,
+             int realnode : @py_ast_node ref,
+             int basicblock : @py_flow_node ref,
+             int index : int ref);
+
+py_scope_flow(int flow : @py_flow_node ref,
+              int scope : @py_scope ref,
+              int kind : int ref);
+
+py_idoms(unique int node : @py_flow_node ref,
+         int immediate_dominator : @py_flow_node ref);
+
+py_ssa_phi(int phi : @py_ssa_var ref,
+           int arg: @py_ssa_var ref);
+
+py_ssa_var(unique int id : @py_ssa_var,
+            int var : @py_variable ref);
+
+py_ssa_use(int node: @py_flow_node ref,
+           int var : @py_ssa_var ref);
+
+py_ssa_defn(unique int id : @py_ssa_var ref,
+                  int node: @py_flow_node ref);
+
+@py_base_var = @py_variable | @py_ssa_var;
+
+py_scopes(unique int node : @py_expr_or_stmt ref,
+         int scope : @py_scope ref);
+
+py_scope_location(unique int id : @location ref,
+                   unique int scope : @py_scope ref);
+
+py_flags_versioned(varchar(1) name : string ref,
+                   varchar(1) value : string ref,
+                   varchar(1) version : string ref);
+
+py_syntax_error_versioned(unique int id : @location ref,
+                varchar(1) message : string ref,
+                varchar(1) version : string ref);
+
+py_comments(unique int id : @py_comment,
+           varchar(1) text : string ref,
+           unique int location : @location ref);
+
+/* Type information support */
+
+py_cobjects(unique int obj : @py_cobject);
+
+py_cobjecttypes(unique int obj : @py_cobject ref,
+                int typeof : @py_cobject ref);
+
+py_cobjectnames(unique int obj : @py_cobject ref,
+                varchar(1) name : string ref);
+
+/* Kind should be 0 for introspection, > 0 from source, as follows:
+   1 from C extension source
+ */
+py_cobject_sources(int obj : @py_cobject ref,
+                  int kind : int ref);
+
+py_cmembers_versioned(int object : @py_cobject ref,
+                      varchar(1) name : string ref,
+                      int member : @py_cobject ref,
+                      varchar(1) version : string ref);
+
+py_citems(int object : @py_cobject ref,
+            int index : int ref,
+            int member : @py_cobject ref);
+
+ext_argtype(int funcid : @py_object ref,
+            int arg : int ref,
+            int typeid : @py_object ref);
+
+ext_rettype(int funcid : @py_object ref,
+            int typeid : @py_object ref);
+
+ext_proptype(int propid : @py_object ref,
+             int typeid : @py_object ref);
+
+ext_argreturn(int funcid : @py_object ref,
+              int arg : int ref);
+
+py_special_objects(unique int obj : @py_cobject ref,
+                   unique varchar(1) name : string ref);
+
+py_decorated_object(int object : @py_object ref,
+                    int level: int ref);
+
+@py_object = @py_cobject | @py_flow_node;
+
+@py_source_element = @py_ast_node | @container;
+
+/* XML Files */
+
+xmlEncoding  (unique int id: @file ref, varchar(900) encoding: string ref);
+
+xmlDTDs       (unique int id: @xmldtd,
+              varchar(900) root: string ref,
+              varchar(900) publicId: string ref,
+              varchar(900) systemId: string ref,
+              int fileid: @file ref);
+
+xmlElements   (unique int id: @xmlelement,
+              varchar(900) name: string ref,
+              int parentid: @xmlparent ref,
+              int idx: int ref,
+              int fileid: @file ref);
+
+xmlAttrs      (unique int id: @xmlattribute,
+              int elementid: @xmlelement ref,
+              varchar(900) name: string ref,
+              varchar(3600) value: string ref,
+              int idx: int ref,
+              int fileid: @file ref);
+
+xmlNs         (int id: @xmlnamespace,
+              varchar(900) prefixName: string ref,
+              varchar(900) URI: string ref,
+              int fileid: @file ref);
+
+xmlHasNs      (int elementId: @xmlnamespaceable ref,
+              int nsId: @xmlnamespace ref,
+              int fileid: @file ref);
+
+xmlComments   (unique int id: @xmlcomment,
+              varchar(3600) text: string ref,
+              int parentid: @xmlparent ref,
+              int fileid: @file ref);
+
+xmlChars      (unique int id: @xmlcharacters,
+              varchar(3600) text: string ref,
+              int parentid: @xmlparent ref,
+              int idx: int ref,
+              int isCDATA: int ref,
+              int fileid: @file ref);
+
+@xmlparent = @file | @xmlelement;
+@xmlnamespaceable = @xmlelement | @xmlattribute;
+
+xmllocations(int xmlElement: @xmllocatable ref,
+             int location: @location_default ref);
+
+@xmllocatable = @xmlcharacters | @xmlelement | @xmlcomment | @xmlattribute | @xmldtd | @file | @xmlnamespace;

python/upgrades/4f1806347d7fafe2f78508da01c01e5aff5f7cbb/semmlecode.python.dbscheme

@@ -0,0 +1,994 @@
+/*
+ *      This dbscheme is auto-generated by 'semmle/dbscheme_gen.py'.
+ *      WARNING: Any modifications to this file will be lost.
+ *      Relations can be changed by modifying master.py or
+ *      by adding rules to dbscheme.template
+ */
+
+/* This is a dummy line to alter the dbscheme, so we can make a database upgrade
+ * without actually changing any of the dbscheme predicates. It contains a date
+ * to allow for such updates in the future as well.
+ *
+ * 2020-07-02
+ *
+ * DO NOT remove this comment carelessly, since it can revert the dbscheme back to a
+ * previously seen state (matching a previously seen SHA), which would make the upgrade
+ * mechanism not work properly.
+ */
+
+ /*
+ * External artifacts
+ */
+
+externalDefects(
+  unique int id : @externalDefect,
+  varchar(900) queryPath : string ref,
+  int location : @location ref,
+  varchar(900) message : string ref,
+  float severity : float ref
+);
+
+externalMetrics(
+  unique int id : @externalMetric,
+  varchar(900) queryPath : string ref,
+  int location : @location ref,
+  float value : float ref
+);
+
+externalData(
+  int id : @externalDataElement,
+  varchar(900) queryPath : string ref,
+  int column: int ref,
+  varchar(900) data : string ref
+);
+
+snapshotDate(unique date snapshotDate : date ref);
+
+sourceLocationPrefix(varchar(900) prefix : string ref);
+
+
+/*
+ * Duplicate code
+ */
+
+duplicateCode(
+  unique int id : @duplication,
+  varchar(900) relativePath : string ref,
+  int equivClass : int ref);
+
+similarCode(
+  unique int id : @similarity,
+  varchar(900) relativePath : string ref,
+  int equivClass : int ref);
+
+@duplication_or_similarity = @duplication | @similarity
+
+tokens(
+  int id : @duplication_or_similarity ref,
+  int offset : int ref,
+  int beginLine : int ref,
+  int beginColumn : int ref,
+  int endLine : int ref,
+  int endColumn : int ref);
+
+/*
+ * Line metrics
+ */
+py_codelines(int id : @py_scope ref,
+             int count : int ref);
+
+py_commentlines(int id : @py_scope ref,
+                int count : int ref);
+
+py_docstringlines(int id : @py_scope ref,
+                int count : int ref);
+
+py_alllines(int id : @py_scope ref,
+                int count : int ref);
+
+/*
+ * Version history
+ */
+
+svnentries(
+  int id : @svnentry,
+  varchar(500) revision : string ref,
+  varchar(500) author : string ref,
+  date revisionDate : date ref,
+  int changeSize : int ref
+)
+
+svnaffectedfiles(
+  int id : @svnentry ref,
+  int file : @file ref,
+  varchar(500) action : string ref
+)
+
+svnentrymsg(
+  int id : @svnentry ref,
+  varchar(500) message : string ref
+)
+
+svnchurn(
+  int commit : @svnentry ref,
+  int file : @file ref,
+  int addedLines : int ref,
+  int deletedLines : int ref
+)
+
+/****************************
+        Python dbscheme
+****************************/
+
+files(unique int id: @file,
+      varchar(900) name: string ref);
+
+folders(unique int id: @folder,
+        varchar(900) name: string ref);
+
+@container = @folder | @file;
+
+containerparent(int parent: @container ref,
+                unique int child: @container ref);
+
+@sourceline = @file | @py_Module | @xmllocatable;
+
+numlines(int element_id: @sourceline ref,
+         int num_lines: int ref,
+         int num_code: int ref,
+         int num_comment: int ref
+        );
+
+@location = @location_ast | @location_default ;
+
+locations_default(unique int id: @location_default,
+          int file: @file ref,
+          int beginLine: int ref,
+          int beginColumn: int ref,
+          int endLine: int ref,
+          int endColumn: int ref);
+
+locations_ast(unique int id: @location_ast,
+          int module: @py_Module ref,
+          int beginLine: int ref,
+          int beginColumn: int ref,
+          int endLine: int ref,
+          int endColumn: int ref);
+
+file_contents(unique int file: @file ref, string contents: string ref);
+
+py_module_path(int module: @py_Module ref, int file: @container ref);
+
+variable(unique int id : @py_variable,
+         int scope : @py_scope ref,
+         varchar(1) name : string ref);
+
+py_line_lengths(unique int id : @py_line,
+                int file: @py_Module ref,
+                int line : int ref,
+                int length : int ref);
+
+py_extracted_version(int module : @py_Module ref,
+                     varchar(1) version : string ref);
+
+/* AUTO GENERATED PART STARTS HERE */
+
+
+/* <Field> AnnAssign.location = 0, location */
+/* <Field> AnnAssign.value = 1, expr */
+/* <Field> AnnAssign.annotation = 2, expr */
+/* <Field> AnnAssign.target = 3, expr */
+
+/* <Field> Assert.location = 0, location */
+/* <Field> Assert.test = 1, expr */
+/* <Field> Assert.msg = 2, expr */
+
+/* <Field> Assign.location = 0, location */
+/* <Field> Assign.value = 1, expr */
+/* <Field> Assign.targets = 2, expr_list */
+
+/* <Field> AssignExpr.location = 0, location */
+/* <Field> AssignExpr.parenthesised = 1, bool */
+/* <Field> AssignExpr.value = 2, expr */
+/* <Field> AssignExpr.target = 3, expr */
+
+/* <Field> Attribute.location = 0, location */
+/* <Field> Attribute.parenthesised = 1, bool */
+/* <Field> Attribute.value = 2, expr */
+/* <Field> Attribute.attr = 3, str */
+/* <Field> Attribute.ctx = 4, expr_context */
+
+/* <Field> AugAssign.location = 0, location */
+/* <Field> AugAssign.operation = 1, BinOp */
+
+/* <Field> Await.location = 0, location */
+/* <Field> Await.parenthesised = 1, bool */
+/* <Field> Await.value = 2, expr */
+
+/* <Field> BinaryExpr.location = 0, location */
+/* <Field> BinaryExpr.parenthesised = 1, bool */
+/* <Field> BinaryExpr.left = 2, expr */
+/* <Field> BinaryExpr.op = 3, operator */
+/* <Field> BinaryExpr.right = 4, expr */
+/* <Parent> BinaryExpr = AugAssign  */
+
+/* <Field> BoolExpr.location = 0, location */
+/* <Field> BoolExpr.parenthesised = 1, bool */
+/* <Field> BoolExpr.op = 2, boolop */
+/* <Field> BoolExpr.values = 3, expr_list */
+
+/* <Field> Break.location = 0, location */
+
+/* <Field> Bytes.location = 0, location */
+/* <Field> Bytes.parenthesised = 1, bool */
+/* <Field> Bytes.s = 2, bytes */
+/* <Field> Bytes.prefix = 3, bytes */
+/* <Field> Bytes.implicitly_concatenated_parts = 4, StringPart_list */
+
+/* <Field> Call.location = 0, location */
+/* <Field> Call.parenthesised = 1, bool */
+/* <Field> Call.func = 2, expr */
+/* <Field> Call.positional_args = 3, expr_list */
+/* <Field> Call.named_args = 4, dict_item_list */
+
+/* <Field> Class.name = 0, str */
+/* <Field> Class.body = 1, stmt_list */
+/* <Parent> Class = ClassExpr  */
+
+/* <Field> ClassExpr.location = 0, location */
+/* <Field> ClassExpr.parenthesised = 1, bool */
+/* <Field> ClassExpr.name = 2, str */
+/* <Field> ClassExpr.bases = 3, expr_list */
+/* <Field> ClassExpr.keywords = 4, dict_item_list */
+/* <Field> ClassExpr.inner_scope = 5, Class */
+
+/* <Field> Compare.location = 0, location */
+/* <Field> Compare.parenthesised = 1, bool */
+/* <Field> Compare.left = 2, expr */
+/* <Field> Compare.ops = 3, cmpop_list */
+/* <Field> Compare.comparators = 4, expr_list */
+
+/* <Field> Continue.location = 0, location */
+
+/* <Field> Delete.location = 0, location */
+/* <Field> Delete.targets = 1, expr_list */
+
+/* <Field> Dict.location = 0, location */
+/* <Field> Dict.parenthesised = 1, bool */
+/* <Field> Dict.items = 2, dict_item_list */
+
+/* <Field> DictComp.location = 0, location */
+/* <Field> DictComp.parenthesised = 1, bool */
+/* <Field> DictComp.function = 2, Function */
+/* <Field> DictComp.iterable = 3, expr */
+
+/* <Field> DictUnpacking.location = 0, location */
+/* <Field> DictUnpacking.value = 1, expr */
+
+/* <Field> Ellipsis.location = 0, location */
+/* <Field> Ellipsis.parenthesised = 1, bool */
+
+/* <Field> ExceptStmt.location = 0, location */
+/* <Field> ExceptStmt.type = 1, expr */
+/* <Field> ExceptStmt.name = 2, expr */
+/* <Field> ExceptStmt.body = 3, stmt_list */
+
+/* <Field> Exec.location = 0, location */
+/* <Field> Exec.body = 1, expr */
+/* <Field> Exec.globals = 2, expr */
+/* <Field> Exec.locals = 3, expr */
+
+/* <Field> ExprStmt.location = 0, location */
+/* <Field> ExprStmt.value = 1, expr */
+
+/* <Field> Filter.location = 0, location */
+/* <Field> Filter.parenthesised = 1, bool */
+/* <Field> Filter.value = 2, expr */
+/* <Field> Filter.filter = 3, expr */
+
+/* <Field> For.location = 0, location */
+/* <Field> For.target = 1, expr */
+/* <Field> For.iter = 2, expr */
+/* <Field> For.body = 3, stmt_list */
+/* <Field> For.orelse = 4, stmt_list */
+/* <Field> For.is_async = 5, bool */
+
+/* <Field> FormattedValue.location = 0, location */
+/* <Field> FormattedValue.parenthesised = 1, bool */
+/* <Field> FormattedValue.value = 2, expr */
+/* <Field> FormattedValue.conversion = 3, str */
+/* <Field> FormattedValue.format_spec = 4, JoinedStr */
+
+/* <Field> Function.name = 0, str */
+/* <Field> Function.args = 1, parameter_list */
+/* <Field> Function.vararg = 2, expr */
+/* <Field> Function.kwonlyargs = 3, expr_list */
+/* <Field> Function.kwarg = 4, expr */
+/* <Field> Function.body = 5, stmt_list */
+/* <Field> Function.is_async = 6, bool */
+/* <Parent> Function = FunctionParent  */
+
+/* <Field> FunctionExpr.location = 0, location */
+/* <Field> FunctionExpr.parenthesised = 1, bool */
+/* <Field> FunctionExpr.name = 2, str */
+/* <Field> FunctionExpr.args = 3, arguments */
+/* <Field> FunctionExpr.returns = 4, expr */
+/* <Field> FunctionExpr.inner_scope = 5, Function */
+
+/* <Field> GeneratorExp.location = 0, location */
+/* <Field> GeneratorExp.parenthesised = 1, bool */
+/* <Field> GeneratorExp.function = 2, Function */
+/* <Field> GeneratorExp.iterable = 3, expr */
+
+/* <Field> Global.location = 0, location */
+/* <Field> Global.names = 1, str_list */
+
+/* <Field> If.location = 0, location */
+/* <Field> If.test = 1, expr */
+/* <Field> If.body = 2, stmt_list */
+/* <Field> If.orelse = 3, stmt_list */
+
+/* <Field> IfExp.location = 0, location */
+/* <Field> IfExp.parenthesised = 1, bool */
+/* <Field> IfExp.test = 2, expr */
+/* <Field> IfExp.body = 3, expr */
+/* <Field> IfExp.orelse = 4, expr */
+
+/* <Field> Import.location = 0, location */
+/* <Field> Import.names = 1, alias_list */
+
+/* <Field> ImportExpr.location = 0, location */
+/* <Field> ImportExpr.parenthesised = 1, bool */
+/* <Field> ImportExpr.level = 2, int */
+/* <Field> ImportExpr.name = 3, str */
+/* <Field> ImportExpr.top = 4, bool */
+
+/* <Field> ImportStar.location = 0, location */
+/* <Field> ImportStar.module = 1, expr */
+
+/* <Field> ImportMember.location = 0, location */
+/* <Field> ImportMember.parenthesised = 1, bool */
+/* <Field> ImportMember.module = 2, expr */
+/* <Field> ImportMember.name = 3, str */
+
+/* <Field> Fstring.location = 0, location */
+/* <Field> Fstring.parenthesised = 1, bool */
+/* <Field> Fstring.values = 2, expr_list */
+/* <Parent> Fstring = FormattedValue  */
+
+/* <Field> KeyValuePair.location = 0, location */
+/* <Field> KeyValuePair.value = 1, expr */
+/* <Field> KeyValuePair.key = 2, expr */
+
+/* <Field> Lambda.location = 0, location */
+/* <Field> Lambda.parenthesised = 1, bool */
+/* <Field> Lambda.args = 2, arguments */
+/* <Field> Lambda.inner_scope = 3, Function */
+
+/* <Field> List.location = 0, location */
+/* <Field> List.parenthesised = 1, bool */
+/* <Field> List.elts = 2, expr_list */
+/* <Field> List.ctx = 3, expr_context */
+
+/* <Field> ListComp.location = 0, location */
+/* <Field> ListComp.parenthesised = 1, bool */
+/* <Field> ListComp.function = 2, Function */
+/* <Field> ListComp.iterable = 3, expr */
+/* <Field> ListComp.generators = 4, comprehension_list */
+/* <Field> ListComp.elt = 5, expr */
+
+/* <Field> Module.name = 0, str */
+/* <Field> Module.hash = 1, str */
+/* <Field> Module.body = 2, stmt_list */
+/* <Field> Module.kind = 3, str */
+
+/* <Field> Name.location = 0, location */
+/* <Field> Name.parenthesised = 1, bool */
+/* <Field> Name.variable = 2, variable */
+/* <Field> Name.ctx = 3, expr_context */
+/* <Parent> Name = ParameterList  */
+
+/* <Field> Nonlocal.location = 0, location */
+/* <Field> Nonlocal.names = 1, str_list */
+
+/* <Field> Num.location = 0, location */
+/* <Field> Num.parenthesised = 1, bool */
+/* <Field> Num.n = 2, number */
+/* <Field> Num.text = 3, number */
+
+/* <Field> Pass.location = 0, location */
+
+/* <Field> PlaceHolder.location = 0, location */
+/* <Field> PlaceHolder.parenthesised = 1, bool */
+/* <Field> PlaceHolder.variable = 2, variable */
+/* <Field> PlaceHolder.ctx = 3, expr_context */
+
+/* <Field> Print.location = 0, location */
+/* <Field> Print.dest = 1, expr */
+/* <Field> Print.values = 2, expr_list */
+/* <Field> Print.nl = 3, bool */
+
+/* <Field> Raise.location = 0, location */
+/* <Field> Raise.exc = 1, expr */
+/* <Field> Raise.cause = 2, expr */
+/* <Field> Raise.type = 3, expr */
+/* <Field> Raise.inst = 4, expr */
+/* <Field> Raise.tback = 5, expr */
+
+/* <Field> Repr.location = 0, location */
+/* <Field> Repr.parenthesised = 1, bool */
+/* <Field> Repr.value = 2, expr */
+
+/* <Field> Return.location = 0, location */
+/* <Field> Return.value = 1, expr */
+
+/* <Field> Set.location = 0, location */
+/* <Field> Set.parenthesised = 1, bool */
+/* <Field> Set.elts = 2, expr_list */
+
+/* <Field> SetComp.location = 0, location */
+/* <Field> SetComp.parenthesised = 1, bool */
+/* <Field> SetComp.function = 2, Function */
+/* <Field> SetComp.iterable = 3, expr */
+
+/* <Field> Slice.location = 0, location */
+/* <Field> Slice.parenthesised = 1, bool */
+/* <Field> Slice.start = 2, expr */
+/* <Field> Slice.stop = 3, expr */
+/* <Field> Slice.step = 4, expr */
+
+/* <Field> SpecialOperation.location = 0, location */
+/* <Field> SpecialOperation.parenthesised = 1, bool */
+/* <Field> SpecialOperation.name = 2, str */
+/* <Field> SpecialOperation.arguments = 3, expr_list */
+
+/* <Field> Starred.location = 0, location */
+/* <Field> Starred.parenthesised = 1, bool */
+/* <Field> Starred.value = 2, expr */
+/* <Field> Starred.ctx = 3, expr_context */
+
+/* <Field> Str.location = 0, location */
+/* <Field> Str.parenthesised = 1, bool */
+/* <Field> Str.s = 2, str */
+/* <Field> Str.prefix = 3, str */
+/* <Field> Str.implicitly_concatenated_parts = 4, StringPart_list */
+
+/* <Field> StringPart.text = 0, str */
+/* <Field> StringPart.location = 1, location */
+/* <Parent> StringPart = StringPartList  */
+/* <Parent> StringPartList = BytesOrStr  */
+
+/* <Field> Subscript.location = 0, location */
+/* <Field> Subscript.parenthesised = 1, bool */
+/* <Field> Subscript.value = 2, expr */
+/* <Field> Subscript.index = 3, expr */
+/* <Field> Subscript.ctx = 4, expr_context */
+
+/* <Field> TemplateDottedNotation.location = 0, location */
+/* <Field> TemplateDottedNotation.parenthesised = 1, bool */
+/* <Field> TemplateDottedNotation.value = 2, expr */
+/* <Field> TemplateDottedNotation.attr = 3, str */
+/* <Field> TemplateDottedNotation.ctx = 4, expr_context */
+
+/* <Field> TemplateWrite.location = 0, location */
+/* <Field> TemplateWrite.value = 1, expr */
+
+/* <Field> Try.location = 0, location */
+/* <Field> Try.body = 1, stmt_list */
+/* <Field> Try.orelse = 2, stmt_list */
+/* <Field> Try.handlers = 3, stmt_list */
+/* <Field> Try.finalbody = 4, stmt_list */
+
+/* <Field> Tuple.location = 0, location */
+/* <Field> Tuple.parenthesised = 1, bool */
+/* <Field> Tuple.elts = 2, expr_list */
+/* <Field> Tuple.ctx = 3, expr_context */
+/* <Parent> Tuple = ParameterList  */
+
+/* <Field> UnaryExpr.location = 0, location */
+/* <Field> UnaryExpr.parenthesised = 1, bool */
+/* <Field> UnaryExpr.op = 2, unaryop */
+/* <Field> UnaryExpr.operand = 3, expr */
+
+/* <Field> While.location = 0, location */
+/* <Field> While.test = 1, expr */
+/* <Field> While.body = 2, stmt_list */
+/* <Field> While.orelse = 3, stmt_list */
+
+/* <Field> With.location = 0, location */
+/* <Field> With.context_expr = 1, expr */
+/* <Field> With.optional_vars = 2, expr */
+/* <Field> With.body = 3, stmt_list */
+/* <Field> With.is_async = 4, bool */
+
+/* <Field> Yield.location = 0, location */
+/* <Field> Yield.parenthesised = 1, bool */
+/* <Field> Yield.value = 2, expr */
+
+/* <Field> YieldFrom.location = 0, location */
+/* <Field> YieldFrom.parenthesised = 1, bool */
+/* <Field> YieldFrom.value = 2, expr */
+
+/* <Field> Alias.value = 0, expr */
+/* <Field> Alias.asname = 1, expr */
+/* <Parent> Alias = AliasList  */
+/* <Parent> AliasList = Import  */
+
+/* <Field> Arguments.kw_defaults = 0, expr_list */
+/* <Field> Arguments.defaults = 1, expr_list */
+/* <Field> Arguments.annotations = 2, expr_list */
+/* <Field> Arguments.varargannotation = 3, expr */
+/* <Field> Arguments.kwargannotation = 4, expr */
+/* <Field> Arguments.kw_annotations = 5, expr_list */
+/* <Parent> Arguments = ArgumentsParent  */
+/* <Parent> boolean = BoolParent  */
+/* <Parent> Boolop = BoolExpr  */
+/* <Parent> string = Bytes  */
+/* <Parent> Cmpop = CmpopList  */
+/* <Parent> CmpopList = Compare  */
+
+/* <Field> Comprehension.location = 0, location */
+/* <Field> Comprehension.iter = 1, expr */
+/* <Field> Comprehension.target = 2, expr */
+/* <Field> Comprehension.ifs = 3, expr_list */
+/* <Parent> Comprehension = ComprehensionList  */
+/* <Parent> ComprehensionList = ListComp  */
+/* <Parent> DictItem = DictItemList  */
+/* <Parent> DictItemList = DictItemListParent  */
+
+/* <Field> Expr.location = 0, location */
+/* <Field> Expr.parenthesised = 1, bool */
+/* <Parent> Expr = ExprParent  */
+/* <Parent> ExprContext = ExprContextParent  */
+/* <Parent> ExprList = ExprListParent  */
+/* <Parent> int = ImportExpr  */
+
+/* <Field> Keyword.location = 0, location */
+/* <Field> Keyword.value = 1, expr */
+/* <Field> Keyword.arg = 2, str */
+/* <Parent> Location = LocationParent  */
+/* <Parent> string = Num  */
+/* <Parent> Operator = BinaryExpr  */
+/* <Parent> ParameterList = Function  */
+
+/* <Field> Stmt.location = 0, location */
+/* <Parent> Stmt = StmtList  */
+/* <Parent> StmtList = StmtListParent  */
+/* <Parent> string = StrParent  */
+/* <Parent> StringList = StrListParent  */
+/* <Parent> Unaryop = UnaryExpr  */
+/* <Parent> Variable = VariableParent  */
+py_Classes(unique int id : @py_Class,
+    unique int parent : @py_ClassExpr ref);
+
+py_Functions(unique int id : @py_Function,
+    unique int parent : @py_Function_parent ref);
+
+py_Modules(unique int id : @py_Module);
+
+py_StringParts(unique int id : @py_StringPart,
+    int parent : @py_StringPart_list ref,
+    int idx : int ref);
+
+py_StringPart_lists(unique int id : @py_StringPart_list,
+    unique int parent : @py_Bytes_or_Str ref);
+
+py_aliases(unique int id : @py_alias,
+    int parent : @py_alias_list ref,
+    int idx : int ref);
+
+py_alias_lists(unique int id : @py_alias_list,
+    unique int parent : @py_Import ref);
+
+py_arguments(unique int id : @py_arguments,
+    unique int parent : @py_arguments_parent ref);
+
+py_bools(int parent : @py_bool_parent ref,
+    int idx : int ref);
+
+py_boolops(unique int id : @py_boolop,
+    int kind: int ref,
+    unique int parent : @py_BoolExpr ref);
+
+py_bytes(varchar(1) id : string ref,
+    int parent : @py_Bytes ref,
+    int idx : int ref);
+
+py_cmpops(unique int id : @py_cmpop,
+    int kind: int ref,
+    int parent : @py_cmpop_list ref,
+    int idx : int ref);
+
+py_cmpop_lists(unique int id : @py_cmpop_list,
+    unique int parent : @py_Compare ref);
+
+py_comprehensions(unique int id : @py_comprehension,
+    int parent : @py_comprehension_list ref,
+    int idx : int ref);
+
+py_comprehension_lists(unique int id : @py_comprehension_list,
+    unique int parent : @py_ListComp ref);
+
+py_dict_items(unique int id : @py_dict_item,
+    int kind: int ref,
+    int parent : @py_dict_item_list ref,
+    int idx : int ref);
+
+py_dict_item_lists(unique int id : @py_dict_item_list,
+    unique int parent : @py_dict_item_list_parent ref);
+
+py_exprs(unique int id : @py_expr,
+    int kind: int ref,
+    int parent : @py_expr_parent ref,
+    int idx : int ref);
+
+py_expr_contexts(unique int id : @py_expr_context,
+    int kind: int ref,
+    unique int parent : @py_expr_context_parent ref);
+
+py_expr_lists(unique int id : @py_expr_list,
+    int parent : @py_expr_list_parent ref,
+    int idx : int ref);
+
+py_ints(int id : int ref,
+    unique int parent : @py_ImportExpr ref);
+
+py_locations(unique int id : @location ref,
+    unique int parent : @py_location_parent ref);
+
+py_numbers(varchar(1) id : string ref,
+    int parent : @py_Num ref,
+    int idx : int ref);
+
+py_operators(unique int id : @py_operator,
+    int kind: int ref,
+    unique int parent : @py_BinaryExpr ref);
+
+py_parameter_lists(unique int id : @py_parameter_list,
+    unique int parent : @py_Function ref);
+
+py_stmts(unique int id : @py_stmt,
+    int kind: int ref,
+    int parent : @py_stmt_list ref,
+    int idx : int ref);
+
+py_stmt_lists(unique int id : @py_stmt_list,
+    int parent : @py_stmt_list_parent ref,
+    int idx : int ref);
+
+py_strs(varchar(1) id : string ref,
+    int parent : @py_str_parent ref,
+    int idx : int ref);
+
+py_str_lists(unique int id : @py_str_list,
+    unique int parent : @py_str_list_parent ref);
+
+py_unaryops(unique int id : @py_unaryop,
+    int kind: int ref,
+    unique int parent : @py_UnaryExpr ref);
+
+py_variables(int id : @py_variable ref,
+    unique int parent : @py_variable_parent ref);
+
+case @py_boolop.kind of
+    0 = @py_And
+|   1 = @py_Or;
+
+case @py_cmpop.kind of
+    0 = @py_Eq
+|   1 = @py_Gt
+|   2 = @py_GtE
+|   3 = @py_In
+|   4 = @py_Is
+|   5 = @py_IsNot
+|   6 = @py_Lt
+|   7 = @py_LtE
+|   8 = @py_NotEq
+|   9 = @py_NotIn;
+
+case @py_dict_item.kind of
+    0 = @py_DictUnpacking
+|   1 = @py_KeyValuePair
+|   2 = @py_keyword;
+
+case @py_expr.kind of
+    0 = @py_Attribute
+|   1 = @py_BinaryExpr
+|   2 = @py_BoolExpr
+|   3 = @py_Bytes
+|   4 = @py_Call
+|   5 = @py_ClassExpr
+|   6 = @py_Compare
+|   7 = @py_Dict
+|   8 = @py_DictComp
+|   9 = @py_Ellipsis
+|   10 = @py_FunctionExpr
+|   11 = @py_GeneratorExp
+|   12 = @py_IfExp
+|   13 = @py_ImportExpr
+|   14 = @py_ImportMember
+|   15 = @py_Lambda
+|   16 = @py_List
+|   17 = @py_ListComp
+|   18 = @py_Name
+|   19 = @py_Num
+|   20 = @py_Repr
+|   21 = @py_Set
+|   22 = @py_SetComp
+|   23 = @py_Slice
+|   24 = @py_Starred
+|   25 = @py_Str
+|   26 = @py_Subscript
+|   27 = @py_Tuple
+|   28 = @py_UnaryExpr
+|   29 = @py_Yield
+|   30 = @py_YieldFrom
+|   31 = @py_TemplateDottedNotation
+|   32 = @py_Filter
+|   33 = @py_PlaceHolder
+|   34 = @py_Await
+|   35 = @py_Fstring
+|   36 = @py_FormattedValue
+|   37 = @py_AssignExpr
+|   38 = @py_SpecialOperation;
+
+case @py_expr_context.kind of
+    0 = @py_AugLoad
+|   1 = @py_AugStore
+|   2 = @py_Del
+|   3 = @py_Load
+|   4 = @py_Param
+|   5 = @py_Store;
+
+case @py_operator.kind of
+    0 = @py_Add
+|   1 = @py_BitAnd
+|   2 = @py_BitOr
+|   3 = @py_BitXor
+|   4 = @py_Div
+|   5 = @py_FloorDiv
+|   6 = @py_LShift
+|   7 = @py_Mod
+|   8 = @py_Mult
+|   9 = @py_Pow
+|   10 = @py_RShift
+|   11 = @py_Sub
+|   12 = @py_MatMult;
+
+case @py_stmt.kind of
+    0 = @py_Assert
+|   1 = @py_Assign
+|   2 = @py_AugAssign
+|   3 = @py_Break
+|   4 = @py_Continue
+|   5 = @py_Delete
+|   6 = @py_ExceptStmt
+|   7 = @py_Exec
+|   8 = @py_Expr_stmt
+|   9 = @py_For
+|   10 = @py_Global
+|   11 = @py_If
+|   12 = @py_Import
+|   13 = @py_ImportStar
+|   14 = @py_Nonlocal
+|   15 = @py_Pass
+|   16 = @py_Print
+|   17 = @py_Raise
+|   18 = @py_Return
+|   19 = @py_Try
+|   20 = @py_While
+|   21 = @py_With
+|   22 = @py_TemplateWrite
+|   23 = @py_AnnAssign;
+
+case @py_unaryop.kind of
+    0 = @py_Invert
+|   1 = @py_Not
+|   2 = @py_UAdd
+|   3 = @py_USub;
+
+@py_Bytes_or_Str = @py_Bytes | @py_Str;
+
+@py_Function_parent = @py_DictComp | @py_FunctionExpr | @py_GeneratorExp | @py_Lambda | @py_ListComp | @py_SetComp;
+
+@py_arguments_parent = @py_FunctionExpr | @py_Lambda;
+
+@py_ast_node = @py_Class | @py_Function | @py_Module | @py_StringPart | @py_comprehension | @py_dict_item | @py_expr | @py_stmt;
+
+@py_bool_parent = @py_For | @py_Function | @py_Print | @py_With | @py_expr;
+
+@py_dict_item_list_parent = @py_Call | @py_ClassExpr | @py_Dict;
+
+@py_expr_context_parent = @py_Attribute | @py_List | @py_Name | @py_PlaceHolder | @py_Starred | @py_Subscript | @py_TemplateDottedNotation | @py_Tuple;
+
+@py_expr_list_parent = @py_Assign | @py_BoolExpr | @py_Call | @py_ClassExpr | @py_Compare | @py_Delete | @py_Fstring | @py_Function | @py_List | @py_Print | @py_Set | @py_SpecialOperation | @py_Tuple | @py_arguments | @py_comprehension;
+
+@py_expr_or_stmt = @py_expr | @py_stmt;
+
+@py_expr_parent = @py_AnnAssign | @py_Assert | @py_Assign | @py_AssignExpr | @py_Attribute | @py_AugAssign | @py_Await | @py_BinaryExpr | @py_Call | @py_Compare | @py_DictComp | @py_DictUnpacking | @py_ExceptStmt | @py_Exec | @py_Expr_stmt | @py_Filter | @py_For | @py_FormattedValue | @py_Function | @py_FunctionExpr | @py_GeneratorExp | @py_If | @py_IfExp | @py_ImportMember | @py_ImportStar | @py_KeyValuePair | @py_ListComp | @py_Print | @py_Raise | @py_Repr | @py_Return | @py_SetComp | @py_Slice | @py_Starred | @py_Subscript | @py_TemplateDottedNotation | @py_TemplateWrite | @py_UnaryExpr | @py_While | @py_With | @py_Yield | @py_YieldFrom | @py_alias | @py_arguments | @py_comprehension | @py_expr_list | @py_keyword | @py_parameter_list;
+
+@py_location_parent = @py_DictUnpacking | @py_KeyValuePair | @py_StringPart | @py_comprehension | @py_expr | @py_keyword | @py_stmt;
+
+@py_parameter = @py_Name | @py_Tuple;
+
+@py_scope = @py_Class | @py_Function | @py_Module;
+
+@py_stmt_list_parent = @py_Class | @py_ExceptStmt | @py_For | @py_Function | @py_If | @py_Module | @py_Try | @py_While | @py_With;
+
+@py_str_list_parent = @py_Global | @py_Nonlocal;
+
+@py_str_parent = @py_Attribute | @py_Class | @py_ClassExpr | @py_FormattedValue | @py_Function | @py_FunctionExpr | @py_ImportExpr | @py_ImportMember | @py_Module | @py_SpecialOperation | @py_Str | @py_StringPart | @py_TemplateDottedNotation | @py_keyword | @py_str_list;
+
+@py_variable_parent = @py_Name | @py_PlaceHolder;
+
+
+/*
+ * End of auto-generated part
+ */
+
+
+
+/* Map relative names to absolute names for imports */
+py_absolute_names(int module : @py_Module ref,
+                  varchar(1) relname : string ref,
+                  varchar(1) absname : string ref);
+
+py_exports(int id : @py_Module ref,
+           varchar(1) name : string ref);
+
+/* Successor information */
+py_successors(int predecessor : @py_flow_node ref,
+              int successor : @py_flow_node ref);
+
+py_true_successors(int predecessor : @py_flow_node ref,
+              int successor : @py_flow_node ref);
+
+py_exception_successors(int predecessor : @py_flow_node ref,
+              int successor : @py_flow_node ref);
+
+py_false_successors(int predecessor : @py_flow_node ref,
+              int successor : @py_flow_node ref);
+
+py_flow_bb_node(unique int flownode : @py_flow_node,
+             int realnode : @py_ast_node ref,
+             int basicblock : @py_flow_node ref,
+             int index : int ref);
+
+py_scope_flow(int flow : @py_flow_node ref,
+              int scope : @py_scope ref,
+              int kind : int ref);
+
+py_idoms(unique int node : @py_flow_node ref,
+         int immediate_dominator : @py_flow_node ref);
+
+py_ssa_phi(int phi : @py_ssa_var ref,
+           int arg: @py_ssa_var ref);
+
+py_ssa_var(unique int id : @py_ssa_var,
+            int var : @py_variable ref);
+
+py_ssa_use(int node: @py_flow_node ref,
+           int var : @py_ssa_var ref);
+
+py_ssa_defn(unique int id : @py_ssa_var ref,
+                  int node: @py_flow_node ref);
+
+@py_base_var = @py_variable | @py_ssa_var;
+
+py_scopes(unique int node : @py_expr_or_stmt ref,
+         int scope : @py_scope ref);
+
+py_scope_location(unique int id : @location ref,
+                   unique int scope : @py_scope ref);
+
+py_flags_versioned(varchar(1) name : string ref,
+                   varchar(1) value : string ref,
+                   varchar(1) version : string ref);
+
+py_syntax_error_versioned(unique int id : @location ref,
+                varchar(1) message : string ref,
+                varchar(1) version : string ref);
+
+py_comments(unique int id : @py_comment,
+           varchar(1) text : string ref,
+           unique int location : @location ref);
+
+/* Type information support */
+
+py_cobjects(unique int obj : @py_cobject);
+
+py_cobjecttypes(unique int obj : @py_cobject ref,
+                int typeof : @py_cobject ref);
+
+py_cobjectnames(unique int obj : @py_cobject ref,
+                varchar(1) name : string ref);
+
+/* Kind should be 0 for introspection, > 0 from source, as follows:
+   1 from C extension source
+ */
+py_cobject_sources(int obj : @py_cobject ref,
+                  int kind : int ref);
+
+py_cmembers_versioned(int object : @py_cobject ref,
+                      varchar(1) name : string ref,
+                      int member : @py_cobject ref,
+                      varchar(1) version : string ref);
+
+py_citems(int object : @py_cobject ref,
+            int index : int ref,
+            int member : @py_cobject ref);
+
+ext_argtype(int funcid : @py_object ref,
+            int arg : int ref,
+            int typeid : @py_object ref);
+
+ext_rettype(int funcid : @py_object ref,
+            int typeid : @py_object ref);
+
+ext_proptype(int propid : @py_object ref,
+             int typeid : @py_object ref);
+
+ext_argreturn(int funcid : @py_object ref,
+              int arg : int ref);
+
+py_special_objects(unique int obj : @py_cobject ref,
+                   unique varchar(1) name : string ref);
+
+py_decorated_object(int object : @py_object ref,
+                    int level: int ref);
+
+@py_object = @py_cobject | @py_flow_node;
+
+@py_source_element = @py_ast_node | @container;
+
+/* XML Files */
+
+xmlEncoding  (unique int id: @file ref, varchar(900) encoding: string ref);
+
+xmlDTDs       (unique int id: @xmldtd,
+              varchar(900) root: string ref,
+              varchar(900) publicId: string ref,
+              varchar(900) systemId: string ref,
+              int fileid: @file ref);
+
+xmlElements   (unique int id: @xmlelement,
+              varchar(900) name: string ref,
+              int parentid: @xmlparent ref,
+              int idx: int ref,
+              int fileid: @file ref);
+
+xmlAttrs      (unique int id: @xmlattribute,
+              int elementid: @xmlelement ref,
+              varchar(900) name: string ref,
+              varchar(3600) value: string ref,
+              int idx: int ref,
+              int fileid: @file ref);
+
+xmlNs         (int id: @xmlnamespace,
+              varchar(900) prefixName: string ref,
+              varchar(900) URI: string ref,
+              int fileid: @file ref);
+
+xmlHasNs      (int elementId: @xmlnamespaceable ref,
+              int nsId: @xmlnamespace ref,
+              int fileid: @file ref);
+
+xmlComments   (unique int id: @xmlcomment,
+              varchar(3600) text: string ref,
+              int parentid: @xmlparent ref,
+              int fileid: @file ref);
+
+xmlChars      (unique int id: @xmlcharacters,
+              varchar(3600) text: string ref,
+              int parentid: @xmlparent ref,
+              int idx: int ref,
+              int isCDATA: int ref,
+              int fileid: @file ref);
+
+@xmlparent = @file | @xmlelement;
+@xmlnamespaceable = @xmlelement | @xmlattribute;
+
+xmllocations(int xmlElement: @xmllocatable ref,
+             int location: @location_default ref);
+
+@xmllocatable = @xmlcharacters | @xmlelement | @xmlcomment | @xmlattribute | @xmldtd | @file | @xmlnamespace;

python/upgrades/4f1806347d7fafe2f78508da01c01e5aff5f7cbb/upgrade.properties

@@ -0,0 +1,4 @@
+description: Removed unused column from the `folders` and `files` relations
+compatibility: full
+files.rel: reorder files.rel (int id, string name, string simple, string ext, int fromSource) id name
+folders.rel: reorder folders.rel (int id, string name, string simple) id name