Merge remote-tracking branch 'origin/main' into nickrolfe/extractor-performance

This is the same idea as Java's LinkedHashMap: it gives the same O(1)
insertion and lookup as HashMap, but preserves insertion order for
iteration.

benjamin-button.md

@@ -1,51 +0,0 @@
-# benjamin-buttons.md
-
-This file describes the changes that have been applied to
-the library to make it behave as if it was younger.
-
-## TaintedPath.ql
-
-Sinks added between 2020-01-01 and 2020-10-06 have been removed. Found by looking at:
-
-- the commit titles of https://github.com/github/codeql/commits/main/javascript/ql/test/query-tests/Security/CWE-022/TaintedPath/TaintedPath.expected
-- the PR titles of https://github.com/github/codeql/pulls?page=2&q=is%3Apr+label%3AJS+is%3Aclosed+sink
-
-Sinks added between 2018-08-02 and 2020-01-01 have been removed. Found by looking at:
-
-- the commit titles of https://github.com/github/codeql/commits/main/javascript/ql/test/query-tests/Security/CWE-022/TaintedPath/TaintedPath.expected
-- the PR titles of https://github.com/github/codeql/pulls?page=2&q=is%3Apr+label%3AJS+is%3Aclosed+sink
-- the PR titles of https://github.com/github/codeql/pulls?page=2&q=is%3Apr+label%3AJS+is%3Aclosed+pathinjection
-- the PR titles of https://github.com/github/codeql/pulls?page=2&q=is%3Apr+label%3AJS+is%3Aclosed+tainted-path
-
-Sinks from the "graceful-fs" and "fs-extra" (added before the open-sourcing squash).
-
-## Xss.ql
-
-Sinks added between 2020-01-01 and 2020-10-06 have been removed. Found by looking at:
-
-- the commit titles of https://github.com/github/codeql/commits/main/javascript/ql/test/query-tests/Security/CWE-079/Xss.expected
-- the PR titles of https://github.com/github/codeql/pulls?page=2&q=is%3Apr+label%3AJS+is%3Aclosed+sink
-
-- recursive type tracking for `jQuery::dollar`, `DOM::domValueRef`.
-
-## SqlInjection.ql
-
-Sinks added between 2020-01-01 and 2020-10-06 have been removed. Found by looking at:
-
-- the commit titles of https://github.com/github/codeql/commits/main/javascript/ql/test/query-tests/Security/CWE-089
-- the PR titles of https://github.com/github/codeql/pulls?page=2&q=is%3Apr+label%3AJS+is%3Aclosed+sink
-
-Sinks added between 2018-08-02 and 2020-01-01 have been removed. Found by looking at:
-
-- the commit titles of https://github.com/github/codeql/commits/main/javascript/ql/test/query-tests/Security/CWE-089
-- the PR titles of https://github.com/github/codeql/pulls?page=2&q=is%3Apr+label%3AJS+is%3Aclosed+sink
-- the PR titles of https://github.com/github/codeql/pulls?page=2&q=is%3Apr+label%3AJS+is%3Aclosed+sql
-
-TypeTracking in SQL.qll (added before the open-sourcing squash)
-
-The model of `mssql` and `sequelize` (added before the open-sourcing squash)
-
-## PseudoProperties
-
-Pseudo-properties (`$name$`) used in type-tracking and global dataflow configurations have been disabled.
-Found by searching for `"\$.*\$"`.

cpp/ql/examples/qlpack.yml

@@ -1,6 +1,4 @@
 name: codeql/cpp-examples
-groups: 
-  - cpp
-  - examples
+version: 0.0.2
 dependencies:
   codeql/cpp-all: "*"

cpp/ql/src/qlpack.yml

@@ -1,8 +1,6 @@
 name: codeql/cpp-queries
 version: 0.0.8-dev
-groups: 
-  - cpp
-  - queries
+groups: cpp
 dependencies:
     codeql/cpp-all: "*"
     codeql/suite-helpers: "*"

csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TypeParameter.cs

@@ -1,5 +1,7 @@
 using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.CSharp.Syntax;
+using Semmle.Extraction.Entities;
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 
@@ -19,9 +21,33 @@ namespace Semmle.Extraction.CSharp.Entities
 
         public override void Populate(TextWriter trapFile)
         {
-            trapFile.types(this, Kinds.TypeKind.TYPE_PARAMETER, Symbol.Name);
+            var constraints = new TypeParameterConstraints(Context);
+            trapFile.type_parameter_constraints(constraints, this);
 
-            TypeParameterConstraints.Create(Context, this);
+            if (Symbol.HasReferenceTypeConstraint)
+                trapFile.general_type_parameter_constraints(constraints, 1);
+
+            if (Symbol.HasValueTypeConstraint)
+                trapFile.general_type_parameter_constraints(constraints, 2);
+
+            if (Symbol.HasConstructorConstraint)
+                trapFile.general_type_parameter_constraints(constraints, 3);
+
+            if (Symbol.HasUnmanagedTypeConstraint)
+                trapFile.general_type_parameter_constraints(constraints, 4);
+
+            if (Symbol.ReferenceTypeConstraintNullableAnnotation == NullableAnnotation.Annotated)
+                trapFile.general_type_parameter_constraints(constraints, 5);
+
+            foreach (var abase in Symbol.GetAnnotatedTypeConstraints())
+            {
+                var t = Create(Context, abase.Symbol);
+                trapFile.specific_type_parameter_constraints(constraints, t.TypeRef);
+                if (!abase.HasObliviousNullability())
+                    trapFile.specific_type_parameter_nullability(constraints, t.TypeRef, NullabilityEntity.Create(Context, Nullability.Create(abase)));
+            }
+
+            trapFile.types(this, Kinds.TypeKind.TYPE_PARAMETER, Symbol.Name);
 
             var parentNs = Namespace.Create(Context, Symbol.TypeParameterKind == TypeParameterKind.Method ? Context.Compilation.GlobalNamespace : Symbol.ContainingNamespace);
             trapFile.parent_namespace(this, parentNs);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TypeParameterConstraints.cs

@@ -1,65 +1,14 @@
-using Microsoft.CodeAnalysis;
 using System.IO;
 
 namespace Semmle.Extraction.CSharp.Entities
 {
-    internal class TypeParameterConstraints : CachedEntity<ITypeParameterSymbol>
+    internal class TypeParameterConstraints : FreshEntity
     {
-        private readonly TypeParameter parent;
+        public TypeParameterConstraints(Context cx)
+            : base(cx) { }
 
-        public TypeParameterConstraints(Context cx, TypeParameter parent)
-            : base(cx, parent.Symbol)
+        protected override void Populate(TextWriter trapFile)
         {
-            this.parent = parent;
-        }
-
-        public override void WriteId(EscapingTextWriter trapFile)
-        {
-            trapFile.WriteSubId(parent);
-            trapFile.Write(";typeparameterconstraints");
-        }
-
-        public override bool NeedsPopulation => true;
-
-        public override void Populate(TextWriter trapFile)
-        {
-            trapFile.type_parameter_constraints(this, parent);
-
-            if (Symbol.HasReferenceTypeConstraint)
-                trapFile.general_type_parameter_constraints(this, 1);
-
-            if (Symbol.HasValueTypeConstraint)
-                trapFile.general_type_parameter_constraints(this, 2);
-
-            if (Symbol.HasConstructorConstraint)
-                trapFile.general_type_parameter_constraints(this, 3);
-
-            if (Symbol.HasUnmanagedTypeConstraint)
-                trapFile.general_type_parameter_constraints(this, 4);
-
-            if (Symbol.ReferenceTypeConstraintNullableAnnotation == NullableAnnotation.Annotated)
-                trapFile.general_type_parameter_constraints(this, 5);
-
-            foreach (var abase in Symbol.GetAnnotatedTypeConstraints())
-            {
-                var t = Type.Create(Context, abase.Symbol);
-                trapFile.specific_type_parameter_constraints(this, t.TypeRef);
-                if (!abase.HasObliviousNullability())
-                    trapFile.specific_type_parameter_nullability(this, t.TypeRef, NullabilityEntity.Create(Context, Nullability.Create(abase)));
-            }
-        }
-
-        public override Location? ReportingLocation => null;
-
-        public static TypeParameterConstraints Create(Context cx, TypeParameter p) =>
-            TypeParameterConstraintsFactory.Instance.CreateEntity(cx, (typeof(TypeParameterConstraints), p), p);
-
-        private class TypeParameterConstraintsFactory : CachedEntityFactory<TypeParameter, TypeParameterConstraints>
-        {
-            public static TypeParameterConstraintsFactory Instance { get; } = new TypeParameterConstraintsFactory();
-
-            public override TypeParameterConstraints Create(Context cx, TypeParameter init) => new(cx, init);
         }
     }
 }
-

csharp/ql/examples/qlpack.yml

@@ -1,6 +1,4 @@
 name: codeql/csharp-examples
-groups: 
-  - csharp
-  - examples
+version: 0.0.2
 dependencies:
   codeql/csharp-all: "*"

csharp/ql/lib/semmle/code/csharp/controlflow/internal/ControlFlowGraphImplShared.qll

@@ -894,7 +894,7 @@ module TestOutput {
             p
             order by
               l.getFile().getBaseName(), l.getFile().getAbsolutePath(), l.getStartLine(),
-              l.getStartColumn(), l.getEndLine(), l.getEndColumn(), p.toString()
+              l.getStartColumn()
           )
       ).toString()
   }

csharp/ql/src/Metrics/internal/ExtractorDiagnostics.ql

@@ -1,24 +0,0 @@
-/**
- * @name Extractor diagnostics
- * @description This query is for internal use only and may change without notice.
- * @kind table
- * @id csharp/extractor-diagnostics
- */
-
-import csharp
-
-bindingset[i]
-private float getCompilationTimeSum(int i) {
-  result = sum(float f | compilation_time(_, _, i, f) | f)
-}
-
-select getCompilationTimeSum(0) as sum_frontend_cpu_seconds,
-  getCompilationTimeSum(1) as sum_frontend_elapsed_seconds,
-  getCompilationTimeSum(4) as sum_frontend_user_seconds,
-  getCompilationTimeSum(2) as sum_extractor_cpu_seconds,
-  getCompilationTimeSum(3) as sum_extractor_elapsed_seconds,
-  getCompilationTimeSum(5) as sum_extractor_user_seconds,
-  sum(float f | compilation_finished(_, f, _) | f) as sum_total_cpu_seconds,
-  sum(float f | compilation_finished(_, _, f) | f) as sum_total_elapsed_seconds,
-  getCompilationTimeSum(6) as sum_peak_working_set_mb,
-  max(float f | compilation_time(_, _, 6, f) | f) as max_peak_working_set_mb

csharp/ql/src/qlpack.yml

@@ -1,8 +1,6 @@
 name: codeql/csharp-queries
 version: 0.0.8-dev
-groups: 
-  - csharp
-  - queries
+groups: csharp
 suites: codeql-suites
 extractor: csharp
 defaultSuiteFile: codeql-suites/csharp-code-scanning.qls

docs/codeql/codeql-language-guides/codeql-library-for-javascript.rst

@@ -82,7 +82,7 @@ For example, the following query computes, for each folder, the number of JavaSc
 Locations
 ^^^^^^^^^
 
-Most entities in a CodeQL database have an associated source location. Locations are identified by five pieces of information: a file, a start line, a start column, an end line, and an end column. Line and column counts are 1-based (so the first character of a file is at line 1, column 1), and the end position is inclusive.
+Most entities in a CodeQL database have an associated source location. Locations are identified by four pieces of information: a file, a start line, a start column, an end line, and an end column. Line and column counts are 1-based (so the first character of a file is at line 1, column 1), and the end position is inclusive.
 
 All entities associated with a source location belong to the class `Locatable <https://codeql.github.com/codeql-standard-libraries/javascript/semmle/javascript/Locations.qll/type.Locations$Locatable.html>`__. The location itself is modeled by the class `Location <https://codeql.github.com/codeql-standard-libraries/javascript/semmle/javascript/Locations.qll/type.Locations$Location.html>`__ and can be accessed through the member predicate ``Locatable.getLocation()``. The `Location <https://codeql.github.com/codeql-standard-libraries/javascript/semmle/javascript/Locations.qll/type.Locations$Location.html>`__ class provides the following member predicates:
 

java/documentation/library-coverage/coverage.csv

@@ -33,7 +33,7 @@ java.lang,8,,58,,,,,,,,,,8,,,,,,,,,,,,,,,46,12
 java.net,10,3,7,,,,,,,,,,,,,10,,,,,,,,,,,3,7,
 java.nio,15,,6,,13,,,,,,,,,,,,,,,,,2,,,,,,6,
 java.sql,7,,,,,,,,,,,,,,,,,,7,,,,,,,,,,
-java.util,34,,438,,,,,,,,,,34,,,,,,,,,,,,,,,24,414
+java.util,34,,430,,,,,,,,,,34,,,,,,,,,,,,,,,16,414
 javax.faces.context,2,7,,,,,,,,,,,,,,,,,,,,,,,2,,7,,
 javax.json,,,123,,,,,,,,,,,,,,,,,,,,,,,,,100,23
 javax.management.remote,2,,,,,,,,,,2,,,,,,,,,,,,,,,,,,

java/documentation/library-coverage/coverage.rst

@@ -15,9 +15,9 @@ Java framework & library support
    `Apache HttpComponents <https://hc.apache.org/>`_,"``org.apache.hc.core5.*``, ``org.apache.http``",5,136,28,,,3,,,,25
    `Google Guava <https://guava.dev/>`_,``com.google.common.*``,,728,35,,6,,,,,
    `JSON-java <https://github.com/stleary/JSON-java>`_,``org.json``,,236,,,,,,,,
-   Java Standard Library,``java.*``,3,541,111,28,,,7,,,10
+   Java Standard Library,``java.*``,3,533,111,28,,,7,,,10
    Java extensions,"``javax.*``, ``jakarta.*``",54,552,32,,,4,,1,1,2
    `Spring <https://spring.io/>`_,``org.springframework.*``,29,472,96,,,,19,14,,29
    Others,"``androidx.slice``, ``cn.hutool.core.codec``, ``com.esotericsoftware.kryo.io``, ``com.esotericsoftware.kryo5.io``, ``com.fasterxml.jackson.core``, ``com.fasterxml.jackson.databind``, ``com.opensymphony.xwork2.ognl``, ``com.unboundid.ldap.sdk``, ``flexjson``, ``groovy.lang``, ``groovy.util``, ``jodd.json``, ``net.sf.saxon.s9api``, ``ognl``, ``org.apache.commons.codec``, ``org.apache.commons.jexl2``, ``org.apache.commons.jexl3``, ``org.apache.commons.logging``, ``org.apache.commons.ognl``, ``org.apache.directory.ldap.client.api``, ``org.apache.ibatis.jdbc``, ``org.apache.log4j``, ``org.apache.logging.log4j``, ``org.apache.shiro.codec``, ``org.apache.shiro.jndi``, ``org.codehaus.groovy.control``, ``org.dom4j``, ``org.hibernate``, ``org.jboss.logging``, ``org.jooq``, ``org.mvel2``, ``org.scijava.log``, ``org.slf4j``, ``org.xml.sax``, ``org.xmlpull.v1``, ``play.mvc``, ``ratpack.core.form``, ``ratpack.core.handling``, ``ratpack.core.http``, ``ratpack.exec``, ``ratpack.form``, ``ratpack.func``, ``ratpack.handling``, ``ratpack.http``, ``ratpack.util``",44,283,921,,,,14,18,,
-   Totals,,182,6220,1424,106,6,10,107,33,1,81
+   Totals,,182,6212,1424,106,6,10,107,33,1,81
 

java/ql/examples/qlpack.yml

@@ -1,6 +1,4 @@
 name: codeql/java-examples
-groups: 
-  - java
-  - examples
+version: 0.0.2
 dependencies:
-  codeql/java-all: "*"
+    codeql/java-all: "*"

java/ql/src/qlpack.yml

@@ -1,11 +1,9 @@
 name: codeql/java-queries
 version: 0.0.8-dev
-groups: 
-  - java
-  - queries
+groups: java
 suites: codeql-suites
 extractor: java
 defaultSuiteFile: codeql-suites/java-code-scanning.qls
 dependencies:
-  codeql/java-all: "*"
-  codeql/suite-helpers: "*"
+    codeql/java-all: "*"
+    codeql/suite-helpers: "*"

javascript/ql/examples/qlpack.yml

@@ -1,6 +1,4 @@
 name: codeql/javascript-examples
-groups: 
-  - javascript
-  - examples
+version: 0.0.3
 dependencies:
   codeql/javascript-all: "*"

javascript/ql/experimental/adaptivethreatmodeling/lib/experimental/adaptivethreatmodeling/FunctionBodyFeatures.qll

@@ -127,18 +127,6 @@ ASTNode getAnASTNodeWithAFeature(Function f) {
   result = getAnASTNodeToFeaturize(f)
 }
 
-int getNumCharsInFunction(Function f) {
-  result =
-    strictsum(ASTNode node | node = getAnASTNodeWithAFeature(f) | getTokenizedAstNode(node).length())
-}
-
-// Evaluator string limit is 5395415 characters. We choose a limit lower than this.
-private int getMaxChars() { result = 1000000 }
-
-Function getFeaturizableFunction(Function f) {
-  result = f and getNumCharsInFunction(f) <= getMaxChars()
-}
-
 /**
  * Returns a featurized representation of the function that can be used to populate the
  * `enclosingFunctionBody` feature for an endpoint.
@@ -147,13 +135,12 @@ string getBodyTokensFeature(Function function) {
   // Performance optimization: If a function has more than 256 body subtokens, then featurize it as
   // absent. This approximates the behavior of the classifer on non-generic body features where
   // large body features are replaced by the absent token.
+  //
+  // We count nodes instead of tokens because tokens are often not unique.
   strictcount(ASTNode node |
     node = getAnASTNodeToFeaturize(function) and
     exists(getTokenizedAstNode(node))
   ) <= 256 and
-  // Performance optimization: If a function has more than getMaxChars() characters in its body subtokens,
-  // then featurize it as absent.
-  function = getFeaturizableFunction(function) and
   result =
     strictconcat(Location l, string token |
       // The use of a nested exists here allows us to avoid duplicates due to two AST nodes in the

javascript/ql/lib/change-notes/2022-01-08-insecure-dependency-resolution.md

@@ -1,4 +0,0 @@
----
-category: newQuery
----
-* The `js/insecure-dependency` query has been added. It detects depedencies that are downloaded using an unencrypted connection.

javascript/ql/lib/semmle/javascript/DOM.qll

@@ -354,6 +354,35 @@ module DOM {
           call.getNumArgument() = 1 and
           unique(InferredType t | t = getArgumentTypeFromJQueryMethodGet(call)) = TTNumber()
         )
+        or
+        // A `this` node from a callback given to a `$().each(callback)` call.
+        // purposely not using JQuery::MethodCall to avoid `jquery.each()`.
+        exists(DataFlow::CallNode eachCall | eachCall = JQuery::objectRef().getAMethodCall("each") |
+          this = DataFlow::thisNode(eachCall.getCallback(0).getFunction()) or
+          this = eachCall.getABoundCallbackParameter(0, 1)
+        )
+        or
+        // A read of an array-element from a JQuery object. E.g. `$("#foo")[0]`
+        exists(DataFlow::PropRead read |
+          read = this and read = JQuery::objectRef().getAPropertyRead()
+        |
+          unique(InferredType t | t = read.getPropertyNameExpr().analyze().getAType()) = TTNumber()
+        )
+        or
+        // A receiver node of an event handler on a DOM node
+        exists(DataFlow::SourceNode domNode, DataFlow::FunctionNode eventHandler |
+          // NOTE: we do not use `getABoundFunctionValue()`, since bound functions tend to have
+          // a different receiver anyway
+          eventHandler = domNode.getAPropertySource(any(string n | n.matches("on%")))
+          or
+          eventHandler =
+            domNode.getAMethodCall("addEventListener").getArgument(1).getAFunctionValue()
+        |
+          domNode = domValueRef() and
+          this = eventHandler.getReceiver()
+        )
+        or
+        this = DataFlow::thisNode(any(EventHandlerCode evt))
       }
     }
   }
@@ -387,6 +416,11 @@ module DOM {
     or
     t.start() and
     result = domValueRef().getAMethodCall(["item", "namedItem"])
+    or
+    t.startInProp("target") and
+    result = domEventSource()
+    or
+    exists(DataFlow::TypeTracker t2 | result = domValueRef(t2).track(t2, t))
   }
 
   /** Gets a data flow node that may refer to a value from the DOM. */

javascript/ql/lib/semmle/javascript/Promises.qll

@@ -183,12 +183,12 @@ module Promises {
   /**
    * Gets the pseudo-field used to describe resolved values in a promise.
    */
-  string valueProp() { none() }
+  string valueProp() { result = "$PromiseResolveField$" }
 
   /**
    * Gets the pseudo-field used to describe rejected values in a promise.
    */
-  string errorProp() { none() }
+  string errorProp() { result = "$PromiseRejectField$" }
 }
 
 /**

javascript/ql/lib/semmle/javascript/dataflow/Configuration.qll

@@ -777,10 +777,10 @@ private class AdditionalFlowStepAsSharedStep extends SharedFlowStep {
  */
 module PseudoProperties {
   bindingset[s]
-  private string pseudoProperty(string s) { none() }
+  private string pseudoProperty(string s) { result = "$" + s + "$" }
 
   bindingset[s, v]
-  private string pseudoProperty(string s, string v) { none() }
+  private string pseudoProperty(string s, string v) { result = "$" + s + "|" + v + "$" }
 
   /**
    * Gets a pseudo-property for the location of elements in a `Set`

javascript/ql/lib/semmle/javascript/frameworks/Angular2.qll

@@ -136,7 +136,7 @@ module Angular2 {
 
   /** Gets a reference to a `DomSanitizer` object. */
   DataFlow::SourceNode domSanitizer() {
-    result.hasUnderlyingType("@angular/platform-browser", "DomSanitizer")
+    result.hasUnderlyingType(["@angular/platform-browser", "@angular/core"], "DomSanitizer")
   }
 
   /** A value that is about to be promoted to a trusted HTML or CSS value. */

javascript/ql/lib/semmle/javascript/frameworks/Express.qll

@@ -927,6 +927,28 @@ module Express {
     override string getCredentialsKind() { result = kind }
   }
 
+  /** A call to `response.sendFile`, considered as a file system access. */
+  private class ResponseSendFileAsFileSystemAccess extends FileSystemReadAccess,
+    DataFlow::MethodCallNode {
+    ResponseSendFileAsFileSystemAccess() {
+      exists(string name | name = "sendFile" or name = "sendfile" |
+        this.calls(any(ResponseExpr res).flow(), name)
+      )
+    }
+
+    override DataFlow::Node getADataNode() { none() }
+
+    override DataFlow::Node getAPathArgument() { result = this.getArgument(0) }
+
+    override DataFlow::Node getRootPathArgument() {
+      result = this.(DataFlow::CallNode).getOptionArgument(1, "root")
+    }
+
+    override predicate isUpwardNavigationRejected(DataFlow::Node argument) {
+      argument = this.getAPathArgument()
+    }
+  }
+
   /**
    * A function that flows to a route setup.
    */

javascript/ql/lib/semmle/javascript/frameworks/Files.qll

@@ -4,6 +4,23 @@
 
 import javascript
 
+/**
+ * A call that can produce a file name.
+ */
+abstract private class FileNameProducer extends DataFlow::Node {
+  /**
+   * Gets a file name produced by this producer.
+   */
+  abstract DataFlow::Node getAFileName();
+}
+
+/**
+ * A node that contains a file name, and is produced by a `ProducesFileNames`.
+ */
+private class ProducedFileName extends FileNameSource {
+  ProducedFileName() { this = any(FileNameProducer producer).getAFileName() }
+}
+
 /**
  * A file name from the `walk-sync` library.
  */
@@ -126,3 +143,341 @@ private DataFlow::Node fastGlobFileNameSource(DataFlow::TypeTracker t) {
 private class FastGlobFileNameSource extends FileNameSource {
   FastGlobFileNameSource() { this = fastGlobFileNameSource(DataFlow::TypeTracker::end()) }
 }
+
+/**
+ * Classes and predicates for modeling the `fstream` library (https://www.npmjs.com/package/fstream).
+ */
+private module FStream {
+  /**
+   * Gets a reference to a method in the `fstream` library.
+   */
+  private DataFlow::SourceNode getAnFStreamProperty(boolean writer) {
+    exists(DataFlow::SourceNode mod, string readOrWrite, string subMod |
+      mod = DataFlow::moduleImport("fstream") and
+      (
+        readOrWrite = "Reader" and writer = false
+        or
+        readOrWrite = "Writer" and writer = true
+      ) and
+      subMod = ["File", "Dir", "Link", "Proxy"]
+    |
+      result = mod.getAPropertyRead(readOrWrite) or
+      result = mod.getAPropertyRead(readOrWrite).getAPropertyRead(subMod) or
+      result = mod.getAPropertyRead(subMod).getAPropertyRead(readOrWrite)
+    )
+  }
+
+  /**
+   * An invocation of a method defined in the `fstream` library.
+   */
+  private class FStream extends FileSystemAccess, DataFlow::InvokeNode {
+    boolean writer;
+
+    FStream() { this = getAnFStreamProperty(writer).getAnInvocation() }
+
+    override DataFlow::Node getAPathArgument() {
+      result = this.getOptionArgument(0, "path")
+      or
+      not exists(this.getOptionArgument(0, "path")) and
+      result = this.getArgument(0)
+    }
+  }
+
+  /**
+   * An invocation of an `fstream` method that writes to a file.
+   */
+  private class FStreamWriter extends FileSystemWriteAccess, FStream {
+    FStreamWriter() { writer = true }
+
+    override DataFlow::Node getADataNode() { none() }
+  }
+
+  /**
+   * An invocation of an `fstream` method that reads a file.
+   */
+  private class FStreamReader extends FileSystemReadAccess, FStream {
+    FStreamReader() { writer = false }
+
+    override DataFlow::Node getADataNode() { none() }
+  }
+}
+
+/**
+ * A call to the library `write-file-atomic`.
+ */
+private class WriteFileAtomic extends FileSystemWriteAccess, DataFlow::CallNode {
+  WriteFileAtomic() {
+    this = DataFlow::moduleImport("write-file-atomic").getACall()
+    or
+    this = DataFlow::moduleMember("write-file-atomic", "sync").getACall()
+  }
+
+  override DataFlow::Node getAPathArgument() { result = this.getArgument(0) }
+
+  override DataFlow::Node getADataNode() { result = this.getArgument(1) }
+}
+
+/**
+ * A call to the library `recursive-readdir`.
+ */
+private class RecursiveReadDir extends FileSystemAccess, FileNameProducer, DataFlow::CallNode {
+  RecursiveReadDir() { this = DataFlow::moduleImport("recursive-readdir").getACall() }
+
+  override DataFlow::Node getAPathArgument() { result = this.getArgument(0) }
+
+  override DataFlow::Node getAFileName() {
+    result = this.trackFileSource(DataFlow::TypeTracker::end())
+  }
+
+  private DataFlow::SourceNode trackFileSource(DataFlow::TypeTracker t) {
+    t.start() and result = this.getCallback([1 .. 2]).getParameter(1)
+    or
+    t.startInPromise() and not exists(this.getCallback([1 .. 2])) and result = this
+    or
+    // Tracking out of a promise
+    exists(DataFlow::TypeTracker t2 |
+      result = PromiseTypeTracking::promiseStep(this.trackFileSource(t2), t, t2)
+    )
+  }
+}
+
+/**
+ * Classes and predicates for modeling the `jsonfile` library (https://www.npmjs.com/package/jsonfile).
+ */
+private module JSONFile {
+  /**
+   * A reader for JSON files.
+   */
+  class JSONFileReader extends FileSystemReadAccess, DataFlow::CallNode {
+    JSONFileReader() {
+      this =
+        DataFlow::moduleMember("jsonfile", any(string s | s = "readFile" or s = "readFileSync"))
+            .getACall()
+    }
+
+    override DataFlow::Node getAPathArgument() { result = this.getArgument(0) }
+
+    override DataFlow::Node getADataNode() { result = this.trackRead(DataFlow::TypeTracker::end()) }
+
+    private DataFlow::SourceNode trackRead(DataFlow::TypeTracker t) {
+      this.getCalleeName() = "readFile" and
+      (
+        t.start() and result = this.getCallback([1 .. 2]).getParameter(1)
+        or
+        t.startInPromise() and not exists(this.getCallback([1 .. 2])) and result = this
+      )
+      or
+      t.start() and
+      this.getCalleeName() = "readFileSync" and
+      result = this
+      or
+      // Tracking out of a promise
+      exists(DataFlow::TypeTracker t2 |
+        result = PromiseTypeTracking::promiseStep(this.trackRead(t2), t, t2)
+      )
+    }
+  }
+
+  /**
+   * A writer for JSON files.
+   */
+  class JSONFileWriter extends FileSystemWriteAccess, DataFlow::CallNode {
+    JSONFileWriter() {
+      this =
+        DataFlow::moduleMember("jsonfile", any(string s | s = "writeFile" or s = "writeFileSync"))
+            .getACall()
+    }
+
+    override DataFlow::Node getAPathArgument() { result = this.getArgument(0) }
+
+    override DataFlow::Node getADataNode() { result = this.getArgument(1) }
+  }
+}
+
+/**
+ * A call to the library `load-json-file`.
+ */
+private class LoadJsonFile extends FileSystemReadAccess, DataFlow::CallNode {
+  LoadJsonFile() {
+    this = DataFlow::moduleImport("load-json-file").getACall()
+    or
+    this = DataFlow::moduleMember("load-json-file", "sync").getACall()
+  }
+
+  override DataFlow::Node getAPathArgument() { result = this.getArgument(0) }
+
+  override DataFlow::Node getADataNode() { result = this.trackRead(DataFlow::TypeTracker::end()) }
+
+  private DataFlow::SourceNode trackRead(DataFlow::TypeTracker t) {
+    this.getCalleeName() = "sync" and t.start() and result = this
+    or
+    not this.getCalleeName() = "sync" and t.startInPromise() and result = this
+    or
+    // Tracking out of a promise
+    exists(DataFlow::TypeTracker t2 |
+      result = PromiseTypeTracking::promiseStep(this.trackRead(t2), t, t2)
+    )
+  }
+}
+
+/**
+ * A call to the library `write-json-file`.
+ */
+private class WriteJsonFile extends FileSystemWriteAccess, DataFlow::CallNode {
+  WriteJsonFile() {
+    this = DataFlow::moduleImport("write-json-file").getACall()
+    or
+    this = DataFlow::moduleMember("write-json-file", "sync").getACall()
+  }
+
+  override DataFlow::Node getAPathArgument() { result = this.getArgument(0) }
+
+  override DataFlow::Node getADataNode() { result = this.getArgument(1) }
+}
+
+/**
+ * A call to the library `walkdir`.
+ */
+private class WalkDir extends FileNameProducer, FileSystemAccess, DataFlow::CallNode {
+  WalkDir() {
+    this = DataFlow::moduleImport("walkdir").getACall()
+    or
+    this = DataFlow::moduleMember("walkdir", "sync").getACall()
+    or
+    this = DataFlow::moduleMember("walkdir", "async").getACall()
+  }
+
+  override DataFlow::Node getAPathArgument() { result = this.getArgument(0) }
+
+  override DataFlow::Node getAFileName() {
+    result = this.trackFileSource(DataFlow::TypeTracker::end())
+  }
+
+  private DataFlow::SourceNode trackFileSource(DataFlow::TypeTracker t) {
+    not this.getCalleeName() = any(string s | s = "sync" or s = "async") and
+    t.start() and
+    (
+      result = this.getCallback(this.getNumArgument() - 1).getParameter(0)
+      or
+      result = this.getAMethodCall(EventEmitter::on()).getCallback(1).getParameter(0)
+    )
+    or
+    t.start() and this.getCalleeName() = "sync" and result = this
+    or
+    t.startInPromise() and this.getCalleeName() = "async" and result = this
+    or
+    // Tracking out of a promise
+    exists(DataFlow::TypeTracker t2 |
+      result = PromiseTypeTracking::promiseStep(this.trackFileSource(t2), t, t2)
+    )
+  }
+}
+
+/**
+ * A call to the library `globule`.
+ */
+private class Globule extends FileNameProducer, FileSystemAccess, DataFlow::CallNode {
+  Globule() {
+    this = DataFlow::moduleMember("globule", "find").getACall()
+    or
+    this = DataFlow::moduleMember("globule", "match").getACall()
+    or
+    this = DataFlow::moduleMember("globule", "isMatch").getACall()
+    or
+    this = DataFlow::moduleMember("globule", "mapping").getACall()
+    or
+    this = DataFlow::moduleMember("globule", "findMapping").getACall()
+  }
+
+  override DataFlow::Node getAPathArgument() {
+    (this.getCalleeName() = "match" or this.getCalleeName() = "isMatch") and
+    result = this.getArgument(1)
+    or
+    this.getCalleeName() = "mapping" and
+    (
+      result = this.getAnArgument() and
+      not exists(result.getALocalSource().getAPropertyWrite("src"))
+      or
+      result = this.getAnArgument().getALocalSource().getAPropertyWrite("src").getRhs()
+    )
+  }
+
+  override DataFlow::Node getAFileName() {
+    result = this and
+    (
+      this.getCalleeName() = "find" or
+      this.getCalleeName() = "match" or
+      this.getCalleeName() = "findMapping" or
+      this.getCalleeName() = "mapping"
+    )
+  }
+}
+
+/**
+ * A file system access made by a NodeJS library.
+ * This class models multiple NodeJS libraries that access files.
+ */
+private class LibraryAccess extends FileSystemAccess, DataFlow::InvokeNode {
+  int pathArgument; // The index of the path argument.
+
+  LibraryAccess() {
+    pathArgument = 0 and
+    (
+      this = DataFlow::moduleImport("path-exists").getACall()
+      or
+      this = DataFlow::moduleImport("rimraf").getACall()
+      or
+      this = DataFlow::moduleImport("readdirp").getACall()
+      or
+      this = DataFlow::moduleImport("walker").getACall()
+      or
+      this =
+        DataFlow::moduleMember("node-dir",
+          ["readFiles", "readFilesStream", "files", "promiseFiles", "subdirs", "paths"]).getACall()
+    )
+    or
+    pathArgument = 0 and
+    this =
+      DataFlow::moduleMember("vinyl-fs", any(string s | s = "src" or s = "dest" or s = "symlink"))
+          .getACall()
+    or
+    pathArgument = [0 .. 1] and
+    (
+      this = DataFlow::moduleImport("ncp").getACall() or
+      this = DataFlow::moduleMember("ncp", "ncp").getACall()
+    )
+  }
+
+  override DataFlow::Node getAPathArgument() { result = this.getArgument(pathArgument) }
+}
+
+/**
+ * A call to the library [`chokidar`](https://www.npmjs.com/package/chokidar), where a call to `on` receives file names.
+ */
+class Chokidar extends FileNameProducer, FileSystemAccess, API::CallNode {
+  Chokidar() { this = API::moduleImport("chokidar").getMember("watch").getACall() }
+
+  override DataFlow::Node getAPathArgument() { result = this.getArgument(0) }
+
+  override DataFlow::Node getAFileName() {
+    exists(DataFlow::CallNode onCall, int pathIndex |
+      onCall = this.getAChainedMethodCall("on") and
+      if onCall.getArgument(0).mayHaveStringValue("all") then pathIndex = 1 else pathIndex = 0
+    |
+      result = onCall.getCallback(1).getParameter(pathIndex)
+    )
+  }
+}
+
+/**
+ * A call to the [`mkdirp`](https://www.npmjs.com/package/mkdirp) library.
+ */
+private class Mkdirp extends FileSystemAccess, API::CallNode {
+  Mkdirp() {
+    this = API::moduleImport("mkdirp").getACall()
+    or
+    this = API::moduleImport("mkdirp").getMember("sync").getACall()
+  }
+
+  override DataFlow::Node getAPathArgument() { result = this.getArgument(0) }
+}

javascript/ql/lib/semmle/javascript/frameworks/NoSQL.qll

@@ -7,7 +7,17 @@ import semmle.javascript.Promises
 
 module NoSQL {
   /** An expression that is interpreted as a NoSQL query. */
-  abstract class Query extends Expr { }
+  abstract class Query extends Expr {
+    /** Gets an expression that is interpreted as a code operator in this query. */
+    DataFlow::Node getACodeOperator() { none() }
+  }
+}
+
+/**
+ * Gets a value that has been assigned to the "$where" property of an object that flows to `queryArg`.
+ */
+private DataFlow::Node getADollarWhereProperty(API::Node queryArg) {
+  result = queryArg.getMember("$where").getARhs()
 }
 
 /**
@@ -15,112 +25,123 @@ module NoSQL {
  */
 private module MongoDB {
   /**
-   * Gets an import of MongoDB.
-   */
-  DataFlow::ModuleImportNode mongodb() { result.getPath() = "mongodb" }
-
-  /**
-   * Gets an access to `mongodb.MongoClient`.
-   */
-  private DataFlow::SourceNode getAMongoClient(DataFlow::TypeTracker t) {
-    t.start() and
-    result = mongodb().getAPropertyRead("MongoClient")
-    or
-    exists(DataFlow::TypeTracker t2 | result = getAMongoClient(t2).track(t2, t))
-  }
-
-  /**
-   * Gets an access to `mongodb.MongoClient`.
-   */
-  DataFlow::SourceNode getAMongoClient() { result = getAMongoClient(DataFlow::TypeTracker::end()) }
-
-  /** Gets a data flow node that leads to a `connect` callback. */
-  private DataFlow::SourceNode getAMongoDbCallback(DataFlow::TypeBackTracker t) {
-    t.start() and
-    result = getAMongoClient().getAMemberCall("connect").getArgument(1).getALocalSource()
-    or
-    exists(DataFlow::TypeBackTracker t2 | result = getAMongoDbCallback(t2).backtrack(t2, t))
-  }
-
-  /** Gets a data flow node that leads to a `connect` callback. */
-  private DataFlow::FunctionNode getAMongoDbCallback() {
-    result = getAMongoDbCallback(DataFlow::TypeBackTracker::end())
-  }
-
-  /**
-   * Gets an expression that may refer to a MongoDB database connection.
-   */
-  private DataFlow::SourceNode getAMongoDb(DataFlow::TypeTracker t) {
-    t.start() and
-    result = getAMongoDbCallback().getParameter(1)
-    or
-    exists(DataFlow::TypeTracker t2 | result = getAMongoDb(t2).track(t2, t))
-  }
-
-  /**
-   * Gets an expression that may refer to a MongoDB database connection.
-   */
-  DataFlow::SourceNode getAMongoDb() { result = getAMongoDb(DataFlow::TypeTracker::end()) }
-
-  /**
-   * A data flow node that may hold a MongoDB collection.
-   */
-  abstract class Collection extends DataFlow::SourceNode { }
-
-  /**
-   * A collection resulting from calling `Db.collection(...)`.
-   */
-  private class CollectionFromDb extends Collection {
-    CollectionFromDb() {
-      this = getAMongoDb().getAMethodCall("collection")
-      or
-      this = getAMongoDb().getAMethodCall("collection").getCallback(1).getParameter(0)
-    }
-  }
-
-  /**
-   * A collection based on the type `mongodb.Collection`.
+   * Gets an access to `mongodb.MongoClient` or a database.
    *
-   * Note that this also covers `mongoose` models since they are subtypes
-   * of `mongodb.Collection`.
+   * In Mongo version 2.x, a client and a database handle were the same concept, but in 3.x
+   * they were separated. To handle everything with a single model, we treat them as the same here.
    */
-  private class CollectionFromType extends Collection {
-    CollectionFromType() { hasUnderlyingType("mongodb", "Collection") }
-  }
-
-  /** Gets a data flow node referring to a MongoDB collection. */
-  private DataFlow::SourceNode getACollection(DataFlow::TypeTracker t) {
-    t.start() and
-    result instanceof Collection
+  private API::Node getAMongoClientOrDatabase() {
+    result = API::moduleImport("mongodb").getMember("MongoClient")
     or
-    exists(DataFlow::TypeTracker t2 | result = getACollection(t2).track(t2, t))
+    result = getAMongoClientOrDatabase().getMember("db").getReturn()
+    or
+    result = getAMongoClientOrDatabase().getMember("connect").getLastParameter().getParameter(1)
   }
 
   /** Gets a data flow node referring to a MongoDB collection. */
-  DataFlow::SourceNode getACollection() { result = getACollection(DataFlow::TypeTracker::end()) }
+  private API::Node getACollection() {
+    // A collection resulting from calling `Db.collection(...)`.
+    exists(API::Node collection |
+      collection = getAMongoClientOrDatabase().getMember("collection").getReturn()
+    |
+      result = collection
+      or
+      result = collection.getParameter(1).getParameter(0)
+    )
+    or
+    // note that this also covers `mongoose` models since they are subtypes of `mongodb.Collection`
+    result = API::Node::ofType("mongodb", "Collection")
+  }
 
   /** A call to a MongoDB query method. */
-  private class QueryCall extends DatabaseAccess, DataFlow::MethodCallNode {
+  private class QueryCall extends DatabaseAccess, API::CallNode {
     int queryArgIdx;
 
     QueryCall() {
-      exists(string m | this = getACollection().getAMethodCall(m) |
-        m = "count" and queryArgIdx = 0
-        or
-        m = "distinct" and queryArgIdx = 1
-        or
-        m = "find" and queryArgIdx = 0
+      exists(string method |
+        CollectionMethodSignatures::interpretsArgumentAsQuery(method, queryArgIdx) and
+        this = getACollection().getMember(method).getACall()
       )
     }
 
-    override DataFlow::Node getAQueryArgument() { result = getArgument(queryArgIdx) }
+    override DataFlow::Node getAQueryArgument() { result = this.getArgument(queryArgIdx) }
+
+    override DataFlow::Node getAResult() {
+      PromiseFlow::loadStep(this.getALocalUse(), result, Promises::valueProp())
+    }
+
+    DataFlow::Node getACodeOperator() {
+      result = getADollarWhereProperty(this.getParameter(queryArgIdx))
+    }
   }
 
   /**
    * An expression that is interpreted as a MongoDB query.
    */
   class Query extends NoSQL::Query {
-    Query() { this = any(QueryCall qc).getAQueryArgument().asExpr() }
+    QueryCall qc;
+
+    Query() { this = qc.getAQueryArgument().asExpr() }
+
+    override DataFlow::Node getACodeOperator() { result = qc.getACodeOperator() }
+  }
+
+  /**
+   * Provides signatures for the Collection methods.
+   */
+  module CollectionMethodSignatures {
+    /**
+     * Holds if Collection method `name` interprets parameter `n` as a query.
+     */
+    predicate interpretsArgumentAsQuery(string name, int n) {
+      // FilterQuery
+      (
+        name = "aggregate" and n = 0
+        or
+        name = "count" and n = 0
+        or
+        name = "countDocuments" and n = 0
+        or
+        name = "deleteMany" and n = 0
+        or
+        name = "deleteOne" and n = 0
+        or
+        name = "distinct" and n = 1
+        or
+        name = "find" and n = 0
+        or
+        name = "findOne" and n = 0
+        or
+        name = "findOneAndDelete" and n = 0
+        or
+        name = "findOneAndRemove" and n = 0
+        or
+        name = "findOneAndReplace" and n = 0
+        or
+        name = "findOneAndUpdate" and n = 0
+        or
+        name = "remove" and n = 0
+        or
+        name = "replaceOne" and n = 0
+        or
+        name = "update" and n = 0
+        or
+        name = "updateMany" and n = 0
+        or
+        name = "updateOne" and n = 0
+      )
+      or
+      // UpdateQuery
+      (
+        name = "findOneAndUpdate" and n = 1
+        or
+        name = "update" and n = 1
+        or
+        name = "updateMany" and n = 1
+        or
+        name = "updateOne" and n = 1
+      )
+    }
   }
 }
 
@@ -131,33 +152,354 @@ private module Mongoose {
   /**
    * Gets an import of Mongoose.
    */
-  DataFlow::ModuleImportNode getAMongooseInstance() { result.getPath() = "mongoose" }
+  API::Node getAMongooseInstance() { result = API::moduleImport("mongoose") }
 
   /**
-   * Gets a call to `mongoose.createConnection`.
+   * Gets a reference to `mongoose.createConnection`.
    */
-  DataFlow::CallNode createConnection() {
-    result = getAMongooseInstance().getAMemberCall("createConnection")
+  API::Node createConnection() { result = getAMongooseInstance().getMember("createConnection") }
+
+  /**
+   * A Mongoose function.
+   */
+  abstract private class MongooseFunction extends API::Node {
+    /**
+     * Gets the API-graph node for the result from this function (if the function returns a `Query`).
+     */
+    abstract API::Node getQueryReturn();
+
+    /**
+     * Holds if this function returns a `Query` that evaluates to one or
+     * more Documents (`asArray` is false if it evaluates to a single
+     * Document).
+     */
+    abstract predicate returnsDocumentQuery(boolean asArray);
+
+    /**
+     * Gets an argument that this function interprets as a query.
+     */
+    abstract API::Node getQueryArgument();
   }
 
   /**
-   * A Mongoose collection object.
+   * Provides classes modeling the Mongoose Model class
    */
-  class Model extends MongoDB::Collection {
-    Model() { this = getAMongooseInstance().getAMemberCall("model") }
+  module Model {
+    private class ModelFunction extends MongooseFunction {
+      string methodName;
+
+      ModelFunction() { this = getModelObject().getMember(methodName) }
+
+      override API::Node getQueryReturn() {
+        MethodSignatures::returnsQuery(methodName) and result = this.getReturn()
+      }
+
+      override predicate returnsDocumentQuery(boolean asArray) {
+        MethodSignatures::returnsDocumentQuery(methodName, asArray)
+      }
+
+      override API::Node getQueryArgument() {
+        exists(int n |
+          MethodSignatures::interpretsArgumentAsQuery(methodName, n) and
+          result = this.getParameter(n)
+        )
+      }
+    }
+
+    /**
+     * Gets a API-graph node referring to a Mongoose Model object.
+     */
+    private API::Node getModelObject() {
+      result = getAMongooseInstance().getMember("model").getReturn()
+      or
+      exists(API::Node conn | conn = createConnection().getReturn() |
+        result = conn.getMember("model").getReturn() or
+        result = conn.getMember("models").getAMember()
+      )
+      or
+      result = API::Node::ofType("mongoose", "Model")
+    }
+
+    /**
+     * Provides signatures for the Model methods.
+     */
+    module MethodSignatures {
+      /**
+       * Holds if Model method `name` interprets parameter `n` as a query.
+       */
+      predicate interpretsArgumentAsQuery(string name, int n) {
+        // implement lots of the MongoDB collection interface
+        MongoDB::CollectionMethodSignatures::interpretsArgumentAsQuery(name, n)
+        or
+        name = "find" + ["ById", "One"] + "AndUpdate" and n = 1
+        or
+        name in ["delete" + ["Many", "One"], "geoSearch", "remove", "replaceOne", "where"] and
+        n = 0
+        or
+        name in [
+            "find" + ["", "ById", "One"],
+            "find" + ["ById", "One"] + "And" + ["Delete", "Remove", "Update"],
+            "update" + ["", "Many", "One"]
+          ] and
+        n = 0
+      }
+
+      /**
+       * Holds if Model method `name` returns a Query.
+       */
+      predicate returnsQuery(string name) {
+        name =
+          [
+            "$where", "count", "findOne", "findOneAndDelete", "findOneAndRemove",
+            "findOneAndReplace", "findOneAndUpdate", "geosearch", "remove", "replaceOne", "update",
+            "updateMany", "countDocuments", "updateOne", "where", "deleteMany", "deleteOne", "find",
+            "findById", "findByIdAndDelete", "findByIdAndRemove", "findByIdAndUpdate"
+          ]
+      }
+
+      /**
+       * Holds if Document method `name` returns a query that results in
+       * one or more documents, the documents are wrapped in an array
+       * if `asArray` is true.
+       */
+      predicate returnsDocumentQuery(string name, boolean asArray) {
+        asArray = false and name = "findOne"
+        or
+        asArray = true and name = "find"
+      }
+    }
   }
 
   /**
-   * A collection based on the type `mongodb.Collection`.
-   *
-   * Note that this also covers `mongoose` models since they are subtypes
-   * of `mongodb.Collection`.
+   * Provides classes modeling the Mongoose Query class
+   */
+  module Query {
+    private class QueryFunction extends MongooseFunction {
+      string methodName;
+
+      QueryFunction() { this = getAMongooseQuery().getMember(methodName) }
+
+      override API::Node getQueryReturn() {
+        MethodSignatures::returnsQuery(methodName) and result = this.getReturn()
+      }
+
+      override predicate returnsDocumentQuery(boolean asArray) {
+        MethodSignatures::returnsDocumentQuery(methodName, asArray)
+      }
+
+      override API::Node getQueryArgument() {
+        exists(int n |
+          MethodSignatures::interpretsArgumentAsQuery(methodName, n) and
+          result = this.getParameter(n)
+        )
+      }
+    }
+
+    private class NewQueryFunction extends MongooseFunction {
+      NewQueryFunction() { this = getAMongooseInstance().getMember("Query") }
+
+      override API::Node getQueryReturn() { result = this.getInstance() }
+
+      override predicate returnsDocumentQuery(boolean asArray) { none() }
+
+      override API::Node getQueryArgument() { result = this.getParameter(2) }
+    }
+
+    /**
+     * Gets a data flow node referring to a Mongoose query object.
+     */
+    API::Node getAMongooseQuery() {
+      result = any(MongooseFunction f).getQueryReturn()
+      or
+      result = API::Node::ofType("mongoose", "Query")
+      or
+      result =
+        getAMongooseQuery()
+            .getMember(any(string name | MethodSignatures::returnsQuery(name)))
+            .getReturn()
+    }
+
+    /**
+     * Provides signatures for the Query methods.
+     */
+    module MethodSignatures {
+      /**
+       * Holds if Query method `name` interprets parameter `n` as a query.
+       */
+      predicate interpretsArgumentAsQuery(string name, int n) {
+        n = 0 and
+        name =
+          [
+            "and", "count", "findOneAndReplace", "findOneAndUpdate", "merge", "nor", "or", "remove",
+            "replaceOne", "setQuery", "setUpdate", "update", "countDocuments", "updateMany",
+            "updateOne", "where", "deleteMany", "deleteOne", "elemMatch", "find", "findOne",
+            "findOneAndDelete", "findOneAndRemove"
+          ]
+        or
+        n = 1 and
+        name = ["distinct", "findOneAndUpdate", "update", "updateMany", "updateOne"]
+      }
+
+      /**
+       * Holds if Query method `name` returns a Query.
+       */
+      predicate returnsQuery(string name) {
+        name =
+          [
+            "$where", "J", "comment", "count", "countDocuments", "distinct", "elemMatch", "equals",
+            "error", "estimatedDocumentCount", "exists", "explain", "all", "find", "findById",
+            "findOne", "findOneAndRemove", "findOneAndUpdate", "geometry", "get", "gt", "gte",
+            "hint", "and", "in", "intersects", "lean", "limit", "lt", "lte", "map", "map",
+            "maxDistance", "maxTimeMS", "batchsize", "maxscan", "mod", "ne", "near", "nearSphere",
+            "nin", "or", "orFail", "polygon", "populate", "box", "read", "readConcern", "regexp",
+            "remove", "select", "session", "set", "setOptions", "setQuery", "setUpdate", "center",
+            "size", "skip", "slaveOk", "slice", "snapshot", "sort", "update", "w", "where",
+            "within", "centerSphere", "wtimeout", "circle", "collation"
+          ]
+      }
+
+      /**
+       * Holds if Query method `name` returns a query that results in
+       * one or more documents, the documents are wrapped in an array
+       * if `asArray` is true.
+       */
+      predicate returnsDocumentQuery(string name, boolean asArray) {
+        asArray = false and name = "findOne"
+        or
+        asArray = true and name = "find"
+      }
+    }
+  }
+
+  /**
+   * Provides classes modeling the Mongoose Document class
+   */
+  module Document {
+    private class DocumentFunction extends MongooseFunction {
+      string methodName;
+
+      DocumentFunction() { this = getAMongooseDocument().getMember(methodName) }
+
+      override API::Node getQueryReturn() {
+        MethodSignatures::returnsQuery(methodName) and result = this.getReturn()
+      }
+
+      override predicate returnsDocumentQuery(boolean asArray) {
+        MethodSignatures::returnsDocumentQuery(methodName, asArray)
+      }
+
+      override API::Node getQueryArgument() {
+        exists(int n |
+          MethodSignatures::interpretsArgumentAsQuery(methodName, n) and
+          result = this.getParameter(n)
+        )
+      }
+    }
+
+    /**
+     * A Mongoose Document that is retrieved from the backing database.
+     */
+    class RetrievedDocument extends API::Node {
+      RetrievedDocument() {
+        exists(boolean asArray, API::Node param |
+          exists(MongooseFunction func |
+            func.returnsDocumentQuery(asArray) and
+            param = func.getLastParameter().getParameter(1)
+          )
+          or
+          exists(API::Node f |
+            f = Query::getAMongooseQuery().getMember("then") and
+            param = f.getParameter(0).getParameter(0)
+            or
+            f = Query::getAMongooseQuery().getMember("exec") and
+            param = f.getParameter(0).getParameter(1)
+          |
+            exists(DataFlow::MethodCallNode pred |
+              // limitation: look at the previous method call	
+              Query::MethodSignatures::returnsDocumentQuery(pred.getMethodName(), asArray) and
+              pred.getAMethodCall() = f.getACall()
+            )
+          )
+        |
+          asArray = false and this = param
+          or
+          asArray = true and
+          // limitation: look for direct accesses
+          this = param.getUnknownMember()
+        )
+      }
+    }
+
+    /**
+     * Gets a data flow node referring to a Mongoose Document object.
+     */
+    private API::Node getAMongooseDocument() {
+      result instanceof RetrievedDocument
+      or
+      result = API::Node::ofType("mongoose", "Document")
+      or
+      result =
+        getAMongooseDocument()
+            .getMember(any(string name | MethodSignatures::returnsDocument(name)))
+            .getReturn()
+    }
+
+    private module MethodSignatures {
+      /**
+       * Holds if Document method `name` returns a Query.
+       */
+      predicate returnsQuery(string name) {
+        // Documents are subtypes of Models
+        Model::MethodSignatures::returnsQuery(name) or
+        name = "replaceOne" or
+        name = "update" or
+        name = "updateOne"
+      }
+
+      /**
+       * Holds if Document method `name` interprets parameter `n` as a query.
+       */
+      predicate interpretsArgumentAsQuery(string name, int n) {
+        // Documents are subtypes of Models
+        Model::MethodSignatures::interpretsArgumentAsQuery(name, n)
+        or
+        n = 0 and
+        (
+          name = "replaceOne" or
+          name = "update" or
+          name = "updateOne"
+        )
+      }
+
+      /**
+       * Holds if Document method `name` returns a query that results in
+       * one or more documents, the documents are wrapped in an array
+       * if `asArray` is true.
+       */
+      predicate returnsDocumentQuery(string name, boolean asArray) {
+        // Documents are subtypes of Models
+        Model::MethodSignatures::returnsDocumentQuery(name, asArray)
+      }
+
+      /**
+       * Holds if Document method `name` returns a Document.
+       */
+      predicate returnsDocument(string name) {
+        name = ["depopulate", "init", "populate", "overwrite"]
+      }
+    }
+  }
+
+  /**
+   * An expression passed to `mongoose.createConnection` to supply credentials.
    */
   class Credentials extends CredentialsExpr {
     string kind;
 
     Credentials() {
-      exists(string prop | this = createConnection().getOptionArgument(3, prop).asExpr() |
+      exists(string prop |
+        this = createConnection().getParameter(3).getMember(prop).getARhs().asExpr()
+      |
         prop = "user" and kind = "user name"
         or
         prop = "pass" and kind = "password"
@@ -166,4 +508,308 @@ private module Mongoose {
 
     override string getCredentialsKind() { result = kind }
   }
+
+  /**
+   * An expression that is interpreted as a (part of a) MongoDB query.
+   */
+  class MongoDBQueryPart extends NoSQL::Query {
+    MongooseFunction f;
+
+    MongoDBQueryPart() { this = f.getQueryArgument().getARhs().asExpr() }
+
+    override DataFlow::Node getACodeOperator() {
+      result = getADollarWhereProperty(f.getQueryArgument())
+    }
+  }
+
+  /**
+   * An evaluation of a MongoDB query.
+   */
+  class ShorthandQueryEvaluation extends DatabaseAccess, DataFlow::InvokeNode {
+    MongooseFunction f;
+
+    ShorthandQueryEvaluation() {
+      this = f.getACall() and
+      // shorthand for execution: provide a callback
+      exists(f.getQueryReturn()) and
+      exists(this.getCallback(this.getNumArgument() - 1))
+    }
+
+    override DataFlow::Node getAQueryArgument() {
+      // NB: the complete information is not easily accessible for deeply chained calls
+      f.getQueryArgument().getARhs() = result
+    }
+
+    override DataFlow::Node getAResult() {
+      result = this.getCallback(this.getNumArgument() - 1).getParameter(1)
+    }
+  }
+
+  class ExplicitQueryEvaluation extends DatabaseAccess, DataFlow::CallNode {
+    string member;
+
+    ExplicitQueryEvaluation() {
+      // explicit execution using a Query method call
+      member = ["exec", "then", "catch"] and
+      Query::getAMongooseQuery().getMember(member).getACall() = this
+    }
+
+    private int resultParamIndex() {
+      member = "then" and result = 0
+      or
+      member = "exec" and result = 1
+    }
+
+    override DataFlow::Node getAResult() {
+      result = this.getCallback(_).getParameter(this.resultParamIndex())
+    }
+
+    override DataFlow::Node getAQueryArgument() {
+      // NB: the complete information is not easily accessible for deeply chained calls
+      none()
+    }
+  }
+}
+
+/**
+ * Provides classes modeling the Minimongo library.
+ */
+private module Minimongo {
+  /**
+   * Provides signatures for the Collection methods.
+   */
+  module CollectionMethodSignatures {
+    /**
+     * Holds if Collection method `name` interprets parameter `n` as a query.
+     */
+    predicate interpretsArgumentAsQuery(string m, int queryArgIdx) {
+      // implements most of the MongoDB interface
+      MongoDB::CollectionMethodSignatures::interpretsArgumentAsQuery(m, queryArgIdx)
+    }
+  }
+
+  /** A call to a Minimongo query method. */
+  private class QueryCall extends DatabaseAccess, API::CallNode {
+    int queryArgIdx;
+
+    QueryCall() {
+      exists(string m |
+        this =
+          API::moduleImport("minimongo")
+              .getAMember()
+              .getReturn()
+              .getAMember()
+              .getMember(m)
+              .getACall() and
+        CollectionMethodSignatures::interpretsArgumentAsQuery(m, queryArgIdx)
+      )
+    }
+
+    override DataFlow::Node getAQueryArgument() { result = this.getArgument(queryArgIdx) }
+
+    override DataFlow::Node getAResult() {
+      PromiseFlow::loadStep(this.getALocalUse(), result, Promises::valueProp())
+    }
+
+    DataFlow::Node getACodeOperator() {
+      result = getADollarWhereProperty(this.getParameter(queryArgIdx))
+    }
+  }
+
+  /**
+   * An expression that is interpreted as a Minimongo query.
+   */
+  class Query extends NoSQL::Query {
+    QueryCall qc;
+
+    Query() { this = qc.getAQueryArgument().asExpr() }
+
+    override DataFlow::Node getACodeOperator() { result = qc.getACodeOperator() }
+  }
+}
+
+/**
+ * Provides classes modeling the MarsDB library.
+ */
+private module MarsDB {
+  private class MarsDBAccess extends DatabaseAccess, DataFlow::CallNode {
+    string method;
+
+    MarsDBAccess() {
+      this =
+        API::moduleImport("marsdb")
+            .getMember("Collection")
+            .getInstance()
+            .getMember(method)
+            .getACall()
+    }
+
+    string getMethod() { result = method }
+
+    override DataFlow::Node getAResult() {
+      PromiseFlow::loadStep(this.getALocalUse(), result, Promises::valueProp())
+    }
+
+    override DataFlow::Node getAQueryArgument() { none() }
+  }
+
+  /** A call to a MarsDB query method. */
+  private class QueryCall extends MarsDBAccess, API::CallNode {
+    int queryArgIdx;
+
+    QueryCall() {
+      exists(string m |
+        this.getMethod() = m and
+        // implements parts of the Minimongo interface
+        Minimongo::CollectionMethodSignatures::interpretsArgumentAsQuery(m, queryArgIdx)
+      )
+    }
+
+    override DataFlow::Node getAResult() {
+      PromiseFlow::loadStep(this.getALocalUse(), result, Promises::valueProp())
+    }
+
+    override DataFlow::Node getAQueryArgument() { result = this.getArgument(queryArgIdx) }
+
+    DataFlow::Node getACodeOperator() {
+      result = getADollarWhereProperty(this.getParameter(queryArgIdx))
+    }
+  }
+
+  /**
+   * An expression that is interpreted as a MarsDB query.
+   */
+  class Query extends NoSQL::Query {
+    QueryCall qc;
+
+    Query() { this = qc.getAQueryArgument().asExpr() }
+
+    override DataFlow::Node getACodeOperator() { result = qc.getACodeOperator() }
+  }
+}
+
+/**
+ * Provides classes modeling the `Node Redis` library.
+ *
+ * Redis is an in-memory key-value store and not a database,
+ * but `Node Redis` can be exploited similarly to a NoSQL database by giving a method an array as argument instead of a string.
+ * As an example the below two invocations of `client.set` are equivalent:
+ *
+ * ```
+ * const redis = require("redis");
+ * const client = redis.createClient();
+ * client.set("key", "value");
+ * client.set(["key", "value"]);
+ * ```
+ *
+ * ioredis is a very similar library. However, ioredis does not support array arguments in the same way, and is therefore not vulnerable to the same kind of type confusion.
+ */
+private module Redis {
+  /**
+   * Gets a `Node Redis` client.
+   */
+  private API::Node client() {
+    result = API::moduleImport("redis").getMember("createClient").getReturn()
+    or
+    result = API::moduleImport("redis").getMember("RedisClient").getInstance()
+    or
+    result = client().getMember("duplicate").getReturn()
+    or
+    result = client().getMember("duplicate").getLastParameter().getParameter(1)
+  }
+
+  /**
+   * Gets a (possibly chained) reference to a batch operation object.
+   * These have the same API as a redis client, except the calls are chained, and the sequence is terminated with a `.exec` call.
+   */
+  private API::Node multi() {
+    result = client().getMember(["multi", "batch"]).getReturn()
+    or
+    result = multi().getAMember().getReturn()
+  }
+
+  /**
+   * Gets a `Node Redis` client instance. Either a client created using `createClient()`, or a batch operation object.
+   */
+  private API::Node redis() { result = [client(), multi()] }
+
+  /**
+   * Provides signatures for the query methods from Node Redis.
+   */
+  module QuerySignatures {
+    /**
+     * Holds if `method` interprets parameter `argIndex` as a key, and a later parameter determines a value/field.
+     * Thereby the method is vulnerable if parameter `argIndex` is unexpectedly an array instead of a string, as an attacker can control arguments to Redis that the attacker was not supposed to control.
+     *
+     * Only setters and similar methods are included.
+     * For getter-like methods it is not generally possible to gain access "outside" of where you are supposed to have access,
+     * it is at most possible to get a Redis call to return more results than expected (e.g. by adding more members to [`geohash`](https://redis.io/commands/geohash)).
+     */
+    predicate argumentIsAmbiguousKey(string method, int argIndex) {
+      method =
+        [
+          "set", "publish", "append", "bitfield", "decrby", "getset", "hincrby", "hincrbyfloat",
+          "hset", "hsetnx", "incrby", "incrbyfloat", "linsert", "lpush", "lpushx", "lset", "ltrim",
+          "rename", "renamenx", "rpushx", "setbit", "setex", "smove", "zincrby", "zinterstore",
+          "hdel", "lpush", "pfadd", "rpush", "sadd", "sdiffstore", "srem"
+        ] and
+      argIndex = 0
+      or
+      method = ["bitop", "hmset", "mset", "msetnx", "geoadd"] and
+      argIndex in [0 .. any(DataFlow::InvokeNode invk).getNumArgument() - 1]
+    }
+  }
+
+  /**
+   * An expression that is interpreted as a key in a Node Redis call.
+   */
+  class RedisKeyArgument extends NoSQL::Query {
+    RedisKeyArgument() {
+      exists(string method, int argIndex |
+        QuerySignatures::argumentIsAmbiguousKey(method, argIndex) and
+        this = redis().getMember(method).getParameter(argIndex).getARhs().asExpr()
+      )
+    }
+  }
+
+  /**
+   * An access to a database through redis
+   */
+  class RedisDatabaseAccess extends DatabaseAccess, DataFlow::CallNode {
+    RedisDatabaseAccess() { this = redis().getMember(_).getACall() }
+
+    override DataFlow::Node getAResult() {
+      PromiseFlow::loadStep(this.getALocalUse(), result, Promises::valueProp())
+    }
+
+    override DataFlow::Node getAQueryArgument() { none() }
+  }
+}
+
+/**
+ * Provides classes modeling the `ioredis` library.
+ *
+ * ```
+ * import Redis from 'ioredis'
+ * let client = new Redis(...)
+ * ```
+ */
+private module IoRedis {
+  /**
+   * Gets an `ioredis` client.
+   */
+  API::Node ioredis() { result = API::moduleImport("ioredis").getInstance() }
+
+  /**
+   * An access to a database through ioredis
+   */
+  class IoRedisDatabaseAccess extends DatabaseAccess, DataFlow::CallNode {
+    IoRedisDatabaseAccess() { this = ioredis().getMember(_).getACall() }
+
+    override DataFlow::Node getAResult() {
+      PromiseFlow::loadStep(this.getALocalUse(), result, Promises::valueProp())
+    }
+
+    override DataFlow::Node getAQueryArgument() { none() }
+  }
 }

javascript/ql/lib/semmle/javascript/frameworks/NodeJSLib.qll

@@ -493,11 +493,56 @@ module NodeJSLib {
    */
   module FS {
     /**
-     * A member `member` from module `fs`.
+     * A member `member` from module `fs` or its drop-in replacements `graceful-fs`, `fs-extra`, `original-fs`.
      */
     DataFlow::SourceNode moduleMember(string member) {
-      exists(string moduleName | moduleName = ["fs"] |
-        result = DataFlow::moduleMember(moduleName, member)
+      result = fsModule(DataFlow::TypeTracker::end()).getAPropertyRead(member)
+    }
+
+    private DataFlow::SourceNode fsModule(DataFlow::TypeTracker t) {
+      exists(string moduleName |
+        moduleName = ["mz/fs", "original-fs", "fs-extra", "graceful-fs", "fs"]
+      |
+        result = DataFlow::moduleImport(moduleName)
+        or
+        // extra support for flexible names
+        result.asExpr().(Require).getArgument(0).mayHaveStringValue(moduleName)
+      ) and
+      t.start()
+      or
+      t.start() and
+      result = DataFlow::moduleMember("fs", "promises")
+      or
+      exists(DataFlow::TypeTracker t2, DataFlow::SourceNode pred | pred = fsModule(t2) |
+        result = pred.track(t2, t)
+        or
+        t.continue() = t2 and
+        exists(Promisify::PromisifyAllCall promisifyAllCall |
+          result = promisifyAllCall and
+          pred.flowsTo(promisifyAllCall.getArgument(0))
+        )
+        or
+        // const fs = require('fs');
+        // let fs_copy = methods.reduce((obj, method) => {
+        //  obj[method] = fs[method];
+        //  return obj;
+        // }, {});
+        t.continue() = t2 and
+        exists(
+          DataFlow::MethodCallNode call, DataFlow::ParameterNode obj, DataFlow::SourceNode method
+        |
+          call.getMethodName() = "reduce" and
+          result = call and
+          obj = call.getABoundCallbackParameter(0, 0) and
+          obj.flowsTo(any(DataFlow::FunctionNode f).getAReturn()) and
+          exists(DataFlow::PropWrite write, DataFlow::PropRead read |
+            write = obj.getAPropertyWrite() and
+            method.flowsToExpr(write.getPropertyNameExpr()) and
+            method.flowsToExpr(read.getPropertyNameExpr()) and
+            read.getBase().getALocalSource() = fsModule(t2) and
+            write.getRhs() = maybePromisified(read)
+          )
+        )
       )
     }
   }
@@ -508,7 +553,7 @@ module NodeJSLib {
   private class NodeJSFileSystemAccess extends FileSystemAccess, DataFlow::CallNode {
     string methodName;
 
-    NodeJSFileSystemAccess() { this = FS::moduleMember(methodName).getACall() }
+    NodeJSFileSystemAccess() { this = maybePromisified(FS::moduleMember(methodName)).getACall() }
 
     /**
      * Gets the name of the called method.

javascript/ql/lib/semmle/javascript/frameworks/SQL.qll

@@ -33,38 +33,56 @@ module SQL {
  * Provides classes modeling the (API compatible) `mysql` and `mysql2` packages.
  */
 private module MySql {
-  private DataFlow::SourceNode mysql() { result = DataFlow::moduleImport(["mysql", "mysql2"]) }
+  private string moduleName() { result = ["mysql", "mysql2", "mysql2/promise"] }
 
-  private DataFlow::CallNode createPool() { result = mysql().getAMemberCall("createPool") }
+  /** Gets the package name `mysql` or `mysql2`. */
+  API::Node mysql() { result = API::moduleImport(moduleName()) }
 
-  /** Gets a reference to a MySQL pool. */
-  private DataFlow::SourceNode pool(DataFlow::TypeTracker t) {
-    t.start() and
-    result = createPool()
+  /** Gets a reference to `mysql.createConnection`. */
+  API::Node createConnection() {
+    result = mysql().getMember(["createConnection", "createConnectionPromise"])
   }
 
-  /** Gets a reference to a MySQL pool. */
-  private DataFlow::SourceNode pool() { result = pool(DataFlow::TypeTracker::end()) }
+  /** Gets a reference to `mysql.createPool`. */
+  API::Node createPool() { result = mysql().getMember(["createPool", "createPoolCluster"]) }
 
-  /** Gets a call to `mysql.createConnection`. */
-  DataFlow::CallNode createConnection() { result = mysql().getAMemberCall("createConnection") }
+  /** Gets a node that contains a MySQL pool created using `mysql.createPool()`. */
+  API::Node pool() {
+    result = createPool().getReturn()
+    or
+    result = pool().getMember("on").getReturn()
+    or
+    result = API::Node::ofType(moduleName(), ["Pool", "PoolCluster"])
+  }
 
-  /** Gets a reference to a MySQL connection instance. */
-  private DataFlow::SourceNode connection(DataFlow::TypeTracker t) {
-    t.start() and
-    (
-      result = createConnection()
-      or
-      result = pool().getAMethodCall("getConnection").getABoundCallbackParameter(0, 1)
+  /** Gets a data flow node that contains a freshly created MySQL connection instance. */
+  API::Node connection() {
+    result = createConnection().getReturn()
+    or
+    result = createConnection().getReturn().getPromised()
+    or
+    result = pool().getMember("getConnection").getParameter(0).getParameter(1)
+    or
+    result = pool().getMember("getConnection").getPromised()
+    or
+    exists(API::CallNode call |
+      call = pool().getMember("on").getACall() and
+      call.getArgument(0).getStringValue() = ["connection", "acquire", "release"] and
+      result = call.getParameter(1).getParameter(0)
     )
+    or
+    result = API::Node::ofType(moduleName(), ["Connection", "PoolConnection"])
   }
 
-  /** Gets a reference to a MySQL connection instance. */
-  DataFlow::SourceNode connection() { result = connection(DataFlow::TypeTracker::end()) }
-
   /** A call to the MySql `query` method. */
   private class QueryCall extends DatabaseAccess, DataFlow::MethodCallNode {
-    QueryCall() { this = [pool(), connection()].getAMethodCall("query") }
+    QueryCall() {
+      exists(API::Node recv | recv = pool() or recv = connection() |
+        this = recv.getMember(["query", "execute"]).getACall()
+      )
+    }
+
+    override DataFlow::Node getAResult() { result = this.getCallback(_).getParameter(1) }
 
     override DataFlow::Node getAQueryArgument() { result = this.getArgument(0) }
   }
@@ -77,7 +95,7 @@ private module MySql {
   /** A call to the `escape` or `escapeId` method that performs SQL sanitization. */
   class EscapingSanitizer extends SQL::SqlSanitizer, MethodCallExpr {
     EscapingSanitizer() {
-      this = [mysql(), pool(), connection()].getAMethodCall(["escape", "escapeId"]).asExpr() and
+      this = [mysql(), pool(), connection()].getMember(["escape", "escapeId"]).getACall().asExpr() and
       input = this.getArgument(0) and
       output = this
     }
@@ -88,8 +106,9 @@ private module MySql {
     string kind;
 
     Credentials() {
-      exists(string prop |
-        this = [createConnection(), createPool()].getOptionArgument(0, prop).asExpr() and
+      exists(API::Node callee, string prop |
+        callee in [createConnection(), createPool()] and
+        this = callee.getParameter(0).getMember(prop).getARhs().asExpr() and
         (
           prop = "user" and kind = "user name"
           or
@@ -106,23 +125,61 @@ private module MySql {
  * Provides classes modeling the PostgreSQL packages, such as `pg` and `pg-promise`.
  */
 private module Postgres {
-  /** Gets an expression that constructs a new connection pool. */
-  DataFlow::InvokeNode newPool() {
-    // new require('pg').Pool()
-    result = DataFlow::moduleImport("pg").getAConstructorInvocation("Pool")
+  API::Node pg() {
+    result = API::moduleImport("pg")
     or
-    // new require('pg-pool')
-    result = DataFlow::moduleImport("pg-pool").getAnInstantiation()
+    result = pgpMain().getMember("pg")
   }
 
-  /** Gets a creation of a Postgres client. */
-  DataFlow::InvokeNode newClient() {
-    result = DataFlow::moduleImport("pg").getAConstructorInvocation("Client")
+  /** Gets a reference to the `Client` constructor in the `pg` package, for example `require('pg').Client`. */
+  API::Node newClient() { result = pg().getMember("Client") }
+
+  /** Gets a freshly created Postgres client instance. */
+  API::Node client() {
+    result = newClient().getInstance()
+    or
+    // pool.connect(function(err, client) { ... })
+    result = pool().getMember("connect").getParameter(0).getParameter(1)
+    or
+    // await pool.connect()
+    result = pool().getMember("connect").getReturn().getPromised()
+    or
+    result = pgpConnection().getMember("client")
+    or
+    exists(API::CallNode call |
+      call = pool().getMember("on").getACall() and
+      call.getArgument(0).getStringValue() = ["connect", "acquire"] and
+      result = call.getParameter(1).getParameter(0)
+    )
+    or
+    result = client().getMember("on").getReturn()
+    or
+    result = API::Node::ofType("pg", ["Client", "PoolClient"])
+  }
+
+  /** Gets a constructor that when invoked constructs a new connection pool. */
+  API::Node newPool() {
+    // new require('pg').Pool()
+    result = pg().getMember("Pool")
+    or
+    // new require('pg-pool')
+    result = API::moduleImport("pg-pool")
+  }
+
+  /** Gets an API node that refers to a connection pool. */
+  API::Node pool() {
+    result = newPool().getInstance()
+    or
+    result = pgpDatabase().getMember("$pool")
+    or
+    result = pool().getMember("on").getReturn()
+    or
+    result = API::Node::ofType("pg", "Pool")
   }
 
   /** A call to the Postgres `query` method. */
   private class QueryCall extends DatabaseAccess, DataFlow::MethodCallNode {
-    QueryCall() { this = [newClient(), newPool()].getAMethodCall("query") }
+    QueryCall() { this = [client(), pool()].getMember("query").getACall() }
 
     override DataFlow::Node getAResult() {
       this.getNumArgument() = 2 and
@@ -151,7 +208,11 @@ private module Postgres {
     string kind;
 
     Credentials() {
-      exists(string prop | this = [newClient(), newPool()].getOptionArgument(0, prop).asExpr() |
+      exists(string prop |
+        this = [newClient(), newPool()].getParameter(0).getMember(prop).getARhs().asExpr()
+        or
+        this = pgPromise().getParameter(0).getMember(prop).getARhs().asExpr()
+      |
         prop = "user" and kind = "user name"
         or
         prop = "password" and kind = prop
@@ -296,30 +357,40 @@ private module Postgres {
  */
 private module Sqlite {
   /** Gets a reference to the `sqlite3` module. */
-  DataFlow::SourceNode sqlite() {
-    result = DataFlow::moduleImport("sqlite3")
+  API::Node sqlite() {
+    result = API::moduleImport("sqlite3")
     or
-    result = sqlite().getAMemberCall("verbose")
+    result = sqlite().getMember("verbose").getReturn()
   }
 
-  /** Gets an expression that constructs a Sqlite database instance. */
-  DataFlow::SourceNode newDb() {
+  /** Gets an expression that constructs or returns a Sqlite database instance. */
+  API::Node database() {
     // new require('sqlite3').Database()
-    result = sqlite().getAConstructorInvocation("Database")
+    result = sqlite().getMember("Database").getInstance()
+    or
+    // chained call
+    result = getAChainingQueryCall()
+    or
+    result = API::Node::ofType("sqlite3", "Database")
   }
 
-  /** Gets a data flow node referring to a Sqlite database instance. */
-  private DataFlow::SourceNode db(DataFlow::TypeTracker t) {
-    t.start() and
-    result = newDb()
+  /** A call to a query method on a Sqlite database instance that returns the same instance. */
+  private API::Node getAChainingQueryCall() {
+    result = database().getMember(["all", "each", "exec", "get", "run"]).getReturn()
   }
 
-  /** Gets a data flow node referring to a Sqlite database instance. */
-  DataFlow::SourceNode db() { result = db(DataFlow::TypeTracker::end()) }
-
   /** A call to a Sqlite query method. */
   private class QueryCall extends DatabaseAccess, DataFlow::MethodCallNode {
-    QueryCall() { this = db().getAMethodCall(["all", "each", "exec", "get", "prepare", "run"]) }
+    QueryCall() {
+      this = getAChainingQueryCall().getAnImmediateUse()
+      or
+      this = database().getMember("prepare").getACall()
+    }
+
+    override DataFlow::Node getAResult() {
+      result = this.getCallback(1).getParameter(1) or
+      PromiseFlow::loadStep(this.getALocalUse(), result, Promises::valueProp())
+    }
 
     override DataFlow::Node getAQueryArgument() { result = this.getArgument(0) }
   }
@@ -329,3 +400,203 @@ private module Sqlite {
     QueryString() { this = any(QueryCall qc).getAQueryArgument().asExpr() }
   }
 }
+
+/**
+ * Provides classes modeling the `mssql` package.
+ */
+private module MsSql {
+  /** Gets a reference to the `mssql` module. */
+  API::Node mssql() { result = API::moduleImport("mssql") }
+
+  /** Gets a node referring to an instance of the given class. */
+  API::Node mssqlClass(string name) {
+    result = mssql().getMember(name).getInstance()
+    or
+    result = API::Node::ofType("mssql", name)
+  }
+
+  /** Gets an API node referring to a Request object. */
+  API::Node request() {
+    result = mssqlClass("Request")
+    or
+    result = request().getMember(["input", "replaceInput", "output", "replaceOutput"]).getReturn()
+    or
+    result = [transaction(), pool()].getMember("request").getReturn()
+  }
+
+  /** Gets an API node referring to a Transaction object. */
+  API::Node transaction() {
+    result = mssqlClass("Transaction")
+    or
+    result = pool().getMember("transaction").getReturn()
+  }
+
+  /** Gets a API node referring to a ConnectionPool object. */
+  API::Node pool() { result = mssqlClass("ConnectionPool") }
+
+  /** A tagged template evaluated as a query. */
+  private class QueryTemplateExpr extends DatabaseAccess, DataFlow::ValueNode, DataFlow::SourceNode {
+    override TaggedTemplateExpr astNode;
+
+    QueryTemplateExpr() {
+      mssql().getMember("query").getAUse() = DataFlow::valueNode(astNode.getTag())
+    }
+
+    override DataFlow::Node getAResult() {
+      PromiseFlow::loadStep(this.getALocalUse(), result, Promises::valueProp())
+    }
+
+    override DataFlow::Node getAQueryArgument() {
+      result = DataFlow::valueNode(astNode.getTemplate().getAnElement())
+    }
+  }
+
+  /** A call to a MsSql query method. */
+  private class QueryCall extends DatabaseAccess, DataFlow::MethodCallNode {
+    QueryCall() { this = [mssql(), request()].getMember(["query", "batch"]).getACall() }
+
+    override DataFlow::Node getAResult() {
+      result = this.getCallback(1).getParameter(1)
+      or
+      PromiseFlow::loadStep(this.getALocalUse(), result, Promises::valueProp())
+    }
+
+    override DataFlow::Node getAQueryArgument() { result = this.getArgument(0) }
+  }
+
+  /** An expression that is passed to a method that interprets it as SQL. */
+  class QueryString extends SQL::SqlString {
+    QueryString() {
+      exists(DatabaseAccess dba | dba instanceof QueryTemplateExpr or dba instanceof QueryCall |
+        this = dba.getAQueryArgument().asExpr()
+      )
+    }
+  }
+
+  /** An element of a query template, which is automatically sanitized. */
+  class QueryTemplateSanitizer extends SQL::SqlSanitizer {
+    QueryTemplateSanitizer() {
+      this = any(QueryTemplateExpr qte).getAQueryArgument().asExpr() and
+      input = this and
+      output = this
+    }
+  }
+
+  /** An expression that is passed as user name or password when creating a client or a pool. */
+  class Credentials extends CredentialsExpr {
+    string kind;
+
+    Credentials() {
+      exists(API::Node callee, string prop |
+        (
+          callee = mssql().getMember("connect")
+          or
+          callee = mssql().getMember("ConnectionPool")
+        ) and
+        this = callee.getParameter(0).getMember(prop).getARhs().asExpr() and
+        (
+          prop = "user" and kind = "user name"
+          or
+          prop = "password" and kind = prop
+        )
+      )
+    }
+
+    override string getCredentialsKind() { result = kind }
+  }
+}
+
+/**
+ * Provides classes modeling the `sequelize` package.
+ */
+private module Sequelize {
+  class SequelizeModel extends ModelInput::TypeModelCsv {
+    override predicate row(string row) {
+      // package1;type1;package2;type2;path
+      row =
+        [
+          "sequelize;;sequelize-typescript;;", //
+          "sequelize;Sequelize;sequelize;default;", //
+          "sequelize;Sequelize;sequelize;;Instance",
+          "sequelize;Sequelize;sequelize;;Member[Sequelize].Instance",
+        ]
+    }
+  }
+
+  class SequelizeSink extends ModelInput::SinkModelCsv {
+    override predicate row(string row) {
+      row =
+        [
+          "sequelize;Sequelize;Member[query].Argument[0];sql-injection",
+          "sequelize;Sequelize;Member[query].Argument[0].Member[query];sql-injection",
+          "sequelize;;Member[literal,asIs].Argument[0];sql-injection",
+          "sequelize;;Argument[1];credentials[user name]",
+          "sequelize;;Argument[2];credentials[password]",
+          "sequelize;;Argument[0..].Member[username];credentials[user name]",
+          "sequelize;;Argument[0..].Member[password];credentials[password]"
+        ]
+    }
+  }
+
+  class SequelizeSource extends ModelInput::SourceModelCsv {
+    override predicate row(string row) {
+      row = "sequelize;Sequelize;Member[query].ReturnValue.Awaited;database-access-result"
+    }
+  }
+}
+
+private module SpannerCsv {
+  class SpannerTypes extends ModelInput::TypeModelCsv {
+    override predicate row(string row) {
+      // package1; type1; package2; type2; path
+      row =
+        [
+          "@google-cloud/spanner;;@google-cloud/spanner;;Member[Spanner]",
+          "@google-cloud/spanner;Database;@google-cloud/spanner;;ReturnValue.Member[instance].ReturnValue.Member[database].ReturnValue",
+          "@google-cloud/spanner;v1.SpannerClient;@google-cloud/spanner;;Member[v1].Member[SpannerClient].Instance",
+          "@google-cloud/spanner;Transaction;@google-cloud/spanner;Database;Member[runTransaction,runTransactionAsync,getTransaction].Argument[0..1].Parameter[1]",
+          "@google-cloud/spanner;Transaction;@google-cloud/spanner;Database;Member[getTransaction].ReturnValue.Awaited",
+          "@google-cloud/spanner;Snapshot;@google-cloud/spanner;Database;Member[getSnapshot].Argument[0..1].Parameter[1]",
+          "@google-cloud/spanner;Snapshot;@google-cloud/spanner;Database;Member[getSnapshot].ReturnValue.Awaited",
+          "@google-cloud/spanner;BatchTransaction;@google-cloud/spanner;Database;Member[batchTransaction].ReturnValue",
+          "@google-cloud/spanner;BatchTransaction;@google-cloud/spanner;Database;Member[createBatchTransaction].ReturnValue.Awaited",
+          "@google-cloud/spanner;~SqlExecutorDirect;@google-cloud/spanner;Database;Member[run,runPartitionedUpdate,runStream]",
+          "@google-cloud/spanner;~SqlExecutorDirect;@google-cloud/spanner;Transaction;Member[run,runStream,runUpdate]",
+          "@google-cloud/spanner;~SqlExecutorDirect;@google-cloud/spanner;BatchTransaction;Member[createQueryPartitions]",
+        ]
+    }
+  }
+
+  class SpannerSinks extends ModelInput::SinkModelCsv {
+    override predicate row(string row) {
+      // package; type; path; kind
+      row =
+        [
+          "@google-cloud/spanner;~SqlExecutorDirect;Argument[0];sql-injection",
+          "@google-cloud/spanner;~SqlExecutorDirect;Argument[0].Member[sql];sql-injection",
+          "@google-cloud/spanner;Transaction;Member[batchUpdate].Argument[0];sql-injection",
+          "@google-cloud/spanner;Transaction;Member[batchUpdate].Argument[0].ArrayElement.Member[sql];sql-injection",
+          "@google-cloud/spanner;v1.SpannerClient;Member[executeSql,executeStreamingSql].Argument[0].Member[sql];sql-injection",
+        ]
+    }
+  }
+
+  class SpannerSources extends ModelInput::SourceModelCsv {
+    string spannerClass() { result = ["v1.SpannerClient", "Database", "Transaction", "Snapshot",] }
+
+    string resultPath() {
+      result =
+        [
+          "Member[executeSql].Argument[0..].Parameter[1]",
+          "Member[executeSql].ReturnValue.Awaited.Member[0]", "Member[run].ReturnValue.Awaited",
+          "Member[run].Argument[0..].Parameter[1]",
+        ]
+    }
+
+    override predicate row(string row) {
+      row =
+        "@google-cloud/spanner;" + this.spannerClass() + ";" + this.resultPath() +
+          ";database-access-result"
+    }
+  }
+}

javascript/ql/lib/semmle/javascript/frameworks/TorrentLibraries.qll

@@ -36,7 +36,25 @@ module ParseTorrent {
    * An access to user-controlled torrent information.
    */
   class UserControlledTorrentInfo extends RemoteFlowSource {
-    UserControlledTorrentInfo() { none() }
+    UserControlledTorrentInfo() {
+      exists(DataFlow::SourceNode ref, DataFlow::PropRead read |
+        ref = parsedTorrentRef() and
+        read = ref.getAPropertyRead() and
+        this = read
+      |
+        exists(string prop |
+          not (
+            prop = "private" or
+            prop = "infoHash" or
+            prop = "length"
+            // "pieceLength" and "lastPieceLength" are not guaranteed to be numbers as of commit ae3ad15d
+          ) and
+          read.getPropertyName() = prop
+        )
+        or
+        not exists(read.getPropertyName())
+      )
+    }
 
     override string getSourceType() { result = "torrent information" }
   }

javascript/ql/lib/semmle/javascript/frameworks/jQuery.qll

@@ -480,6 +480,8 @@ module JQuery {
   private DataFlow::SourceNode dollar(DataFlow::TypeTracker t) {
     t.start() and
     result = dollarSource()
+    or
+    exists(DataFlow::TypeTracker t2 | result = dollar(t2).track(t2, t))
   }
 
   /**
@@ -513,6 +515,14 @@ module JQuery {
       }
     }
 
+    /**
+     * A `this` node in a JQuery plugin function, which is a JQuery object.
+     */
+    private class JQueryPluginThisObject extends Range {
+      JQueryPluginThisObject() {
+        this = DataFlow::thisNode(any(JQueryPluginMethod method).getFunction())
+      }
+    }
   }
 
   /** A source of jQuery objects from the AST-based `JQueryObject` class. */

javascript/ql/lib/semmle/javascript/security/dataflow/CodeInjectionCustomizations.qll

@@ -229,8 +229,14 @@ module CodeInjection {
   }
 
   /**
-   * The first argument to `Module.prototype._compile` from the Node.js built-in module `module`,
-   * considered as a code-injection sink.
+   * A code operator of a NoSQL query as a code injection sink.
+   */
+  class NoSQLCodeInjectionSink extends Sink {
+    NoSQLCodeInjectionSink() { any(NoSQL::Query q).getACodeOperator() = this }
+  }
+
+  /**
+   * The first argument to `Module.prototype._compile`, considered as a code-injection sink.
    */
   class ModuleCompileSink extends Sink {
     ModuleCompileSink() {

javascript/ql/lib/semmle/javascript/security/dataflow/TaintedPathCustomizations.qll

@@ -55,7 +55,7 @@ module TaintedPath {
      * There are currently four flow labels, representing the different combinations of
      * normalization and absoluteness.
      */
-    class PosixPath extends DataFlow::FlowLabel {
+    abstract class PosixPath extends DataFlow::FlowLabel {
       Normalization normalization;
       Relativeness relativeness;
 
@@ -113,7 +113,7 @@ module TaintedPath {
     /**
      * A flow label representing an array of path elements that may include "..".
      */
-    class SplitPath extends DataFlow::FlowLabel {
+    abstract class SplitPath extends DataFlow::FlowLabel {
       SplitPath() { this = "splitPath" }
     }
   }
@@ -218,12 +218,12 @@ module TaintedPath {
       output = this
       or
       // non-global replace or replace of something other than /\.\./g, /[/]/g, or /[\.]/g.
-      this.getCalleeName() = "replace" and
-      input = getReceiver() and
+      this instanceof StringReplaceCall and
+      input = this.getReceiver() and
       output = this and
       not exists(RegExpLiteral literal, RegExpTerm term |
-        getArgument(0).getALocalSource().asExpr() = literal and
-        literal.isGlobal() and
+        this.(StringReplaceCall).getRegExp().asExpr() = literal and
+        this.(StringReplaceCall).isGlobal() and
         literal.getRoot() = term
       |
         term.getAMatchedString() = "/" or
@@ -247,16 +247,15 @@ module TaintedPath {
   /**
    * A call that removes all instances of "../" in the prefix of the string.
    */
-  class DotDotSlashPrefixRemovingReplace extends DataFlow::CallNode {
+  class DotDotSlashPrefixRemovingReplace extends StringReplaceCall {
     DataFlow::Node input;
     DataFlow::Node output;
 
     DotDotSlashPrefixRemovingReplace() {
-      this.getCalleeName() = "replace" and
-      input = getReceiver() and
+      input = this.getReceiver() and
       output = this and
       exists(RegExpLiteral literal, RegExpTerm term |
-        getArgument(0).getALocalSource().asExpr() = literal and
+        this.getRegExp().asExpr() = literal and
         (term instanceof RegExpStar or term instanceof RegExpPlus) and
         term.getChild(0) = getADotDotSlashMatcher()
       |
@@ -298,17 +297,16 @@ module TaintedPath {
   /**
    * A call that removes all "." or ".." from a path, without also removing all forward slashes.
    */
-  class DotRemovingReplaceCall extends DataFlow::CallNode {
+  class DotRemovingReplaceCall extends StringReplaceCall {
     DataFlow::Node input;
     DataFlow::Node output;
 
     DotRemovingReplaceCall() {
-      this.getCalleeName() = "replace" and
-      input = getReceiver() and
+      input = this.getReceiver() and
       output = this and
+      this.isGlobal() and
       exists(RegExpLiteral literal, RegExpTerm term |
-        getArgument(0).getALocalSource().asExpr() = literal and
-        literal.isGlobal() and
+        this.getRegExp().asExpr() = literal and
         literal.getRoot() = term and
         not term.getAMatchedString() = "/"
       |
@@ -608,6 +606,8 @@ module TaintedPath {
       (
         this = fileSystemAccess.getAPathArgument() and
         not exists(fileSystemAccess.getRootPathArgument())
+        or
+        this = fileSystemAccess.getRootPathArgument()
       ) and
       not this = any(ResolvingPathCall call).getInput()
     }
@@ -648,6 +648,74 @@ module TaintedPath {
     AngularJSTemplateUrlSink() { this = any(AngularJS::CustomDirective d).getMember("templateUrl") }
   }
 
+  /**
+   * The path argument of a [send](https://www.npmjs.com/package/send) call, viewed as a sink.
+   */
+  class SendPathSink extends Sink, DataFlow::ValueNode {
+    SendPathSink() { this = DataFlow::moduleImport("send").getACall().getArgument(1) }
+  }
+
+  /**
+   * A path argument given to a `Page` in puppeteer, specifying where a pdf/screenshot should be saved.
+   */
+  private class PuppeteerPath extends TaintedPath::Sink {
+    PuppeteerPath() {
+      this =
+        Puppeteer::page()
+            .getMember(["pdf", "screenshot"])
+            .getParameter(0)
+            .getMember("path")
+            .getARhs()
+    }
+  }
+
+  /**
+   * An argument given to the `prettier` library specifying the location of a config file.
+   */
+  private class PrettierFileSink extends TaintedPath::Sink {
+    PrettierFileSink() {
+      this =
+        API::moduleImport("prettier")
+            .getMember(["resolveConfig", "resolveConfigFile", "getFileInfo"])
+            .getACall()
+            .getArgument(0)
+      or
+      this =
+        API::moduleImport("prettier")
+            .getMember("resolveConfig")
+            .getACall()
+            .getParameter(1)
+            .getMember("config")
+            .getARhs()
+    }
+  }
+
+  /**
+   * The `cwd` option for the `read-pkg` library.
+   */
+  private class ReadPkgCwdSink extends TaintedPath::Sink {
+    ReadPkgCwdSink() {
+      this =
+        API::moduleImport("read-pkg")
+            .getMember(["readPackageAsync", "readPackageSync"])
+            .getParameter(0)
+            .getMember("cwd")
+            .getARhs()
+    }
+  }
+
+  /**
+   * The `cwd` option to a shell execution.
+   */
+  private class ShellCwdSink extends TaintedPath::Sink {
+    ShellCwdSink() {
+      exists(SystemCommandExecution sys, API::Node opts |
+        opts.getARhs() = sys.getOptionsArg() and // assuming that an API::Node exists here.
+        this = opts.getMember("cwd").getARhs()
+      )
+    }
+  }
+
   /**
    * Holds if there is a step `src -> dst` mapping `srclabel` to `dstlabel` relevant for path traversal vulnerabilities.
    */

javascript/ql/lib/semmle/javascript/security/dataflow/Xss.qll

@@ -183,6 +183,17 @@ module DomBasedXss {
       this = any(Typeahead::TypeaheadSuggestionFunction f).getAReturn()
       or
       this = any(Handlebars::SafeString s).getAnArgument()
+      or
+      this = any(JQuery::MethodCall call | call.getMethodName() = "jGrowl").getArgument(0)
+      or
+      // A construction of a JSDOM object (server side DOM), where scripts are allowed.
+      exists(DataFlow::NewNode instance |
+        instance = API::moduleImport("jsdom").getMember("JSDOM").getInstance().getAnImmediateUse() and
+        this = instance.getArgument(0) and
+        instance.getOptionArgument(1, "runScripts").mayHaveStringValue("dangerously")
+      )
+      or
+      MooTools::interpretsNodeAsHtml(this)
     }
   }
 

javascript/ql/src/Security/CWE-020/IncompleteUrlSchemeCheck.ql

@@ -10,7 +10,6 @@
  * @tags security
  *       correctness
  *       external/cwe/cwe-020
- *       external/cwe/cwe-184
  */
 
 import javascript

javascript/ql/src/Security/CWE-116/BadTagFilter.ql

@@ -11,7 +11,6 @@
  *       external/cwe/cwe-020
  *       external/cwe/cwe-080
  *       external/cwe/cwe-116
- *       external/cwe/cwe-184
  *       external/cwe/cwe-185
  *       external/cwe/cwe-186
  */

javascript/ql/src/Security/CWE-300/InsecureDependencyResolution.qhelp

@@ -1,55 +0,0 @@
-<!DOCTYPE qhelp PUBLIC
-  "-//Semmle//qhelp//EN"
-  "qhelp.dtd">
-<qhelp>
-
-<overview>
-<p>
-Using an insecure protocol like HTTP or FTP to download build dependencies makes the build process vulnerable to a 
-man-in-the-middle (MITM) attack.
-</p>
-<p>
-This can allow attackers to inject malicious code into the downloaded dependencies, and thereby 
-infect the build artifacts and execute arbitrary code on the machine building the artifacts.
-</p>
-
-</overview>
-<recommendation>
-
-<p>Always use a secure protocol, such as HTTPS or SFTP, when downloading artifacts from an URL.</p>
-
-</recommendation>
-
-<example>
-<p>
-The below example shows a <code>package.json</code> file that downloads a dependency using the insecure HTTP protocol.
-</p>
-<sample src="examples/bad-package.json" />
-<p>
-The fix is to change the protocol to HTTPS.
-</p>
-<sample src="examples/good-package.json" />
-
-</example>
-
-<references>
-<li>
-  Jonathan Leitschuh: 
-  <a href="https://infosecwriteups.com/want-to-take-over-the-java-ecosystem-all-you-need-is-a-mitm-1fc329d898fb">
-    Want to take over the Java ecosystem? All you need is a MITM!
-  </a>
-</li>
-<li>
-  Max Veytsman: 
-  <a href="https://max.computer/blog/how-to-take-over-the-computer-of-any-java-or-clojure-or-scala-developer/">
-    How to take over the computer of any Java (or Closure or Scala) Developer.
-  </a>
-</li>
-<li>
-  Wikipedia: <a href="https://en.wikipedia.org/wiki/Supply_chain_attack">Supply chain attack.</a>
-</li>
-<li>
-  Wikipedia: <a href="https://en.wikipedia.org/wiki/Man-in-the-middle_attack">Man-in-the-middle attack.</a>
-</li>
-</references>
-</qhelp>

javascript/ql/src/Security/CWE-300/InsecureDependencyResolution.ql

@@ -1,23 +0,0 @@
-/**
- * @name Dependency download using unencrypted communication channel
- * @description Using unencrypted protocols to fetch dependencies can leave an application
- *              open to man-in-the-middle attacks.
- * @kind problem
- * @problem.severity warning
- * @security-severity 8.1
- * @precision high
- * @id js/insecure-dependency
- * @tags security
- *       external/cwe/cwe-300
- *       external/cwe/cwe-319
- *       external/cwe/cwe-494
- *       external/cwe/cwe-829
- */
-
-import javascript
-
-from PackageJSON pack, JSONString val
-where
-  [pack.getDependencies(), pack.getDevDependencies()].getPropValue(_) = val and
-  val.getValue().regexpMatch("(http|ftp)://.*")
-select val, "Dependency downloaded using unencrypted communication channel."

javascript/ql/src/Security/CWE-300/examples/bad-package.json

@@ -1,7 +0,0 @@
-{
-  "name": "example-project",
-  "dependencies": {
-    "unencrypted": "http://example.org/foo/tarball/release/0.0.1",
-    "lodash": "^4.0.0"
-  }
-}

javascript/ql/src/Security/CWE-300/examples/good-package.json

@@ -1,7 +0,0 @@
-{
-  "name": "example-project",
-  "dependencies": {
-    "unencrypted": "https://example.org/foo/tarball/release/0.0.1",
-    "lodash": "^4.0.0"
-  }
-}

javascript/ql/src/qlpack.yml

@@ -1,11 +1,9 @@
 name: codeql/javascript-queries
 version: 0.0.9-dev
-groups: 
-  - javascript
-  - queries
+groups: javascript
 suites: codeql-suites
 extractor: javascript
 defaultSuiteFile: codeql-suites/javascript-code-scanning.qls
 dependencies:
-  codeql/javascript-all: "*"
-  codeql/suite-helpers: "*"
+    codeql/javascript-all: "*"
+    codeql/suite-helpers: "*"

javascript/ql/test/query-tests/Security/CWE-300/InsecureDependencyResolution.expected

@@ -1,4 +0,0 @@
-| package.json:6:17:6:40 | "http:/ ... rg/foo" | Dependency downloaded using unencrypted communication channel. |
-| package.json:7:17:7:39 | "ftp:// ... rg/foo" | Dependency downloaded using unencrypted communication channel. |
-| package.json:12:17:12:40 | "http:/ ... rg/foo" | Dependency downloaded using unencrypted communication channel. |
-| package.json:13:17:13:39 | "ftp:// ... rg/foo" | Dependency downloaded using unencrypted communication channel. |

javascript/ql/test/query-tests/Security/CWE-300/InsecureDependencyResolution.qlref

@@ -1 +0,0 @@
-Security/CWE-300/InsecureDependencyResolution.ql

javascript/ql/test/query-tests/Security/CWE-300/foo.js

@@ -1 +0,0 @@
-console.log("foo");

javascript/ql/test/query-tests/Security/CWE-300/package.json

@@ -1,15 +0,0 @@
-{
-    "name": "insecure-dep-downloader",
-    "dependencies": {
-        "foo": "*",
-        "good1": "https://example.org/foo",
-        "bad1": "http://example.org/foo",
-        "bad2": "ftp://example.org/foo"
-    },
-    "devDependencies": {
-        "bar": "*",
-        "good2": "https://example.org/foo",
-        "bad3": "http://example.org/foo",
-        "bad4": "ftp://example.org/foo"
-    }
-}

python/ql/examples/qlpack.yml

@@ -1,6 +1,4 @@
 name: codeql/python-examples
-groups: 
-  - python
-  - examples
+version: 0.0.2
 dependencies:
-  codeql/python-all: "*"
+    codeql/python-all: "*"

python/ql/lib/downgrades/503c0516fba2e5da9570f00eb34ef43025ecb8fb/downgrade.properties

@@ -1,6 +0,0 @@
-description: Add new statements and expressions for the match syntax.
-compatibility: backwards
-py_exprs.rel: run py_exprs.qlo
-py_stmts.rel: run py_stmts.qlo
-py_patterns.rel: delete
-py_patterns_lists.rel: delete

python/ql/lib/downgrades/503c0516fba2e5da9570f00eb34ef43025ecb8fb/py_exprs.ql

@@ -1,42 +0,0 @@
-// First we need to wrap some database types
-class Location extends @location {
-  /** Gets the start line of this location */
-  int getStartLine() {
-    locations_default(this, _, result, _, _, _) or
-    locations_ast(this, _, result, _, _, _)
-  }
-
-  string toString() { result = "<some file>" + ":" + this.getStartLine().toString() }
-}
-
-class Expr_ extends @py_expr {
-  string toString() { result = "Expr" }
-
-  Location getLocation() { py_locations(result, this) }
-}
-
-class ExprParent_ extends @py_expr_parent {
-  string toString() { result = "ExprParent" }
-}
-
-/**
- * New kinds have been inserted such that
- * `@py_Name` which used to have index 18 now has index 19.
- * Entries with lower indices are unchanged.
- */
-bindingset[new_index]
-int old_index(int new_index) {
-  if new_index < 18 then result = new_index else result + (19 - 18) = new_index
-}
-
-// The schema for py_exprs is:
-//
-// py_exprs(unique int id : @py_expr,
-//     int kind: int ref,
-//     int parent : @py_expr_parent ref,
-//     int idx : int ref);
-from Expr_ expr, int new_kind, ExprParent_ parent, int idx, int old_kind
-where
-  py_exprs(expr, new_kind, parent, idx) and
-  old_kind = old_index(new_kind)
-select expr, old_kind, parent, idx

python/ql/lib/downgrades/503c0516fba2e5da9570f00eb34ef43025ecb8fb/py_stmts.ql

@@ -1,42 +0,0 @@
-// First we need to wrap some database types
-class Location extends @location {
-  /** Gets the start line of this location */
-  int getStartLine() {
-    locations_default(this, _, result, _, _, _) or
-    locations_ast(this, _, result, _, _, _)
-  }
-
-  string toString() { result = "<some file>" + ":" + this.getStartLine().toString() }
-}
-
-class Stmt_ extends @py_stmt {
-  string toString() { result = "Stmt" }
-
-  Location getLocation() { py_locations(result, this) }
-}
-
-class StmtList_ extends @py_stmt_list {
-  string toString() { result = "StmtList" }
-}
-
-/**
- * New kinds have been inserted such that
- * `@py_Nonlocal` which used to have index 14 now has index 16.
- * Entries with lower indices are unchanged.
- */
-bindingset[new_index]
-int old_index(int new_index) {
-  if new_index < 14 then result = new_index else result + (16 - 14) = new_index
-}
-
-// The schema for py_stmts is:
-//
-// py_stmts(unique int id : @py_stmt,
-//   int kind: int ref,
-//   int parent : @py_stmt_list ref,
-//   int idx : int ref);
-from Stmt_ expr, int new_kind, StmtList_ parent, int idx, int old_kind
-where
-  py_stmts(expr, new_kind, parent, idx) and
-  old_kind = old_index(new_kind)
-select expr, old_kind, parent, idx

python/ql/lib/downgrades/503c0516fba2e5da9570f00eb34ef43025ecb8fb/semmlecode.python.dbscheme

@@ -1,994 +0,0 @@
-/*
- *      This dbscheme is auto-generated by 'semmle/dbscheme_gen.py'.
- *      WARNING: Any modifications to this file will be lost.
- *      Relations can be changed by modifying master.py or
- *      by adding rules to dbscheme.template
- */
-
-/* This is a dummy line to alter the dbscheme, so we can make a database upgrade
- * without actually changing any of the dbscheme predicates. It contains a date
- * to allow for such updates in the future as well.
- *
- * 2020-07-02
- *
- * DO NOT remove this comment carelessly, since it can revert the dbscheme back to a
- * previously seen state (matching a previously seen SHA), which would make the upgrade
- * mechanism not work properly.
- */
-
- /*
- * External artifacts
- */
-
-externalDefects(
-  unique int id : @externalDefect,
-  varchar(900) queryPath : string ref,
-  int location : @location ref,
-  varchar(900) message : string ref,
-  float severity : float ref
-);
-
-externalMetrics(
-  unique int id : @externalMetric,
-  varchar(900) queryPath : string ref,
-  int location : @location ref,
-  float value : float ref
-);
-
-externalData(
-  int id : @externalDataElement,
-  varchar(900) queryPath : string ref,
-  int column: int ref,
-  varchar(900) data : string ref
-);
-
-snapshotDate(unique date snapshotDate : date ref);
-
-sourceLocationPrefix(varchar(900) prefix : string ref);
-
-
-/*
- * Duplicate code
- */
-
-duplicateCode(
-  unique int id : @duplication,
-  varchar(900) relativePath : string ref,
-  int equivClass : int ref);
-
-similarCode(
-  unique int id : @similarity,
-  varchar(900) relativePath : string ref,
-  int equivClass : int ref);
-
-@duplication_or_similarity = @duplication | @similarity
-
-tokens(
-  int id : @duplication_or_similarity ref,
-  int offset : int ref,
-  int beginLine : int ref,
-  int beginColumn : int ref,
-  int endLine : int ref,
-  int endColumn : int ref);
-
-/*
- * Line metrics
- */
-py_codelines(int id : @py_scope ref,
-             int count : int ref);
-
-py_commentlines(int id : @py_scope ref,
-                int count : int ref);
-
-py_docstringlines(int id : @py_scope ref,
-                int count : int ref);
-
-py_alllines(int id : @py_scope ref,
-                int count : int ref);
-
-/*
- * Version history
- */
-
-svnentries(
-  int id : @svnentry,
-  varchar(500) revision : string ref,
-  varchar(500) author : string ref,
-  date revisionDate : date ref,
-  int changeSize : int ref
-)
-
-svnaffectedfiles(
-  int id : @svnentry ref,
-  int file : @file ref,
-  varchar(500) action : string ref
-)
-
-svnentrymsg(
-  int id : @svnentry ref,
-  varchar(500) message : string ref
-)
-
-svnchurn(
-  int commit : @svnentry ref,
-  int file : @file ref,
-  int addedLines : int ref,
-  int deletedLines : int ref
-)
-
-/****************************
-        Python dbscheme
-****************************/
-
-files(unique int id: @file,
-      varchar(900) name: string ref);
-
-folders(unique int id: @folder,
-        varchar(900) name: string ref);
-
-@container = @folder | @file;
-
-containerparent(int parent: @container ref,
-                unique int child: @container ref);
-
-@sourceline = @file | @py_Module | @xmllocatable;
-
-numlines(int element_id: @sourceline ref,
-         int num_lines: int ref,
-         int num_code: int ref,
-         int num_comment: int ref
-        );
-
-@location = @location_ast | @location_default ;
-
-locations_default(unique int id: @location_default,
-          int file: @file ref,
-          int beginLine: int ref,
-          int beginColumn: int ref,
-          int endLine: int ref,
-          int endColumn: int ref);
-
-locations_ast(unique int id: @location_ast,
-          int module: @py_Module ref,
-          int beginLine: int ref,
-          int beginColumn: int ref,
-          int endLine: int ref,
-          int endColumn: int ref);
-
-file_contents(unique int file: @file ref, string contents: string ref);
-
-py_module_path(int module: @py_Module ref, int file: @container ref);
-
-variable(unique int id : @py_variable,
-         int scope : @py_scope ref,
-         varchar(1) name : string ref);
-
-py_line_lengths(unique int id : @py_line,
-                int file: @py_Module ref,
-                int line : int ref,
-                int length : int ref);
-
-py_extracted_version(int module : @py_Module ref,
-                     varchar(1) version : string ref);
-
-/* AUTO GENERATED PART STARTS HERE */
-
-
-/* <Field> AnnAssign.location = 0, location */
-/* <Field> AnnAssign.value = 1, expr */
-/* <Field> AnnAssign.annotation = 2, expr */
-/* <Field> AnnAssign.target = 3, expr */
-
-/* <Field> Assert.location = 0, location */
-/* <Field> Assert.test = 1, expr */
-/* <Field> Assert.msg = 2, expr */
-
-/* <Field> Assign.location = 0, location */
-/* <Field> Assign.value = 1, expr */
-/* <Field> Assign.targets = 2, expr_list */
-
-/* <Field> AssignExpr.location = 0, location */
-/* <Field> AssignExpr.parenthesised = 1, bool */
-/* <Field> AssignExpr.value = 2, expr */
-/* <Field> AssignExpr.target = 3, expr */
-
-/* <Field> Attribute.location = 0, location */
-/* <Field> Attribute.parenthesised = 1, bool */
-/* <Field> Attribute.value = 2, expr */
-/* <Field> Attribute.attr = 3, str */
-/* <Field> Attribute.ctx = 4, expr_context */
-
-/* <Field> AugAssign.location = 0, location */
-/* <Field> AugAssign.operation = 1, BinOp */
-
-/* <Field> Await.location = 0, location */
-/* <Field> Await.parenthesised = 1, bool */
-/* <Field> Await.value = 2, expr */
-
-/* <Field> BinaryExpr.location = 0, location */
-/* <Field> BinaryExpr.parenthesised = 1, bool */
-/* <Field> BinaryExpr.left = 2, expr */
-/* <Field> BinaryExpr.op = 3, operator */
-/* <Field> BinaryExpr.right = 4, expr */
-/* <Parent> BinaryExpr = AugAssign  */
-
-/* <Field> BoolExpr.location = 0, location */
-/* <Field> BoolExpr.parenthesised = 1, bool */
-/* <Field> BoolExpr.op = 2, boolop */
-/* <Field> BoolExpr.values = 3, expr_list */
-
-/* <Field> Break.location = 0, location */
-
-/* <Field> Bytes.location = 0, location */
-/* <Field> Bytes.parenthesised = 1, bool */
-/* <Field> Bytes.s = 2, bytes */
-/* <Field> Bytes.prefix = 3, bytes */
-/* <Field> Bytes.implicitly_concatenated_parts = 4, StringPart_list */
-
-/* <Field> Call.location = 0, location */
-/* <Field> Call.parenthesised = 1, bool */
-/* <Field> Call.func = 2, expr */
-/* <Field> Call.positional_args = 3, expr_list */
-/* <Field> Call.named_args = 4, dict_item_list */
-
-/* <Field> Class.name = 0, str */
-/* <Field> Class.body = 1, stmt_list */
-/* <Parent> Class = ClassExpr  */
-
-/* <Field> ClassExpr.location = 0, location */
-/* <Field> ClassExpr.parenthesised = 1, bool */
-/* <Field> ClassExpr.name = 2, str */
-/* <Field> ClassExpr.bases = 3, expr_list */
-/* <Field> ClassExpr.keywords = 4, dict_item_list */
-/* <Field> ClassExpr.inner_scope = 5, Class */
-
-/* <Field> Compare.location = 0, location */
-/* <Field> Compare.parenthesised = 1, bool */
-/* <Field> Compare.left = 2, expr */
-/* <Field> Compare.ops = 3, cmpop_list */
-/* <Field> Compare.comparators = 4, expr_list */
-
-/* <Field> Continue.location = 0, location */
-
-/* <Field> Delete.location = 0, location */
-/* <Field> Delete.targets = 1, expr_list */
-
-/* <Field> Dict.location = 0, location */
-/* <Field> Dict.parenthesised = 1, bool */
-/* <Field> Dict.items = 2, dict_item_list */
-
-/* <Field> DictComp.location = 0, location */
-/* <Field> DictComp.parenthesised = 1, bool */
-/* <Field> DictComp.function = 2, Function */
-/* <Field> DictComp.iterable = 3, expr */
-
-/* <Field> DictUnpacking.location = 0, location */
-/* <Field> DictUnpacking.value = 1, expr */
-
-/* <Field> Ellipsis.location = 0, location */
-/* <Field> Ellipsis.parenthesised = 1, bool */
-
-/* <Field> ExceptStmt.location = 0, location */
-/* <Field> ExceptStmt.type = 1, expr */
-/* <Field> ExceptStmt.name = 2, expr */
-/* <Field> ExceptStmt.body = 3, stmt_list */
-
-/* <Field> Exec.location = 0, location */
-/* <Field> Exec.body = 1, expr */
-/* <Field> Exec.globals = 2, expr */
-/* <Field> Exec.locals = 3, expr */
-
-/* <Field> ExprStmt.location = 0, location */
-/* <Field> ExprStmt.value = 1, expr */
-
-/* <Field> Filter.location = 0, location */
-/* <Field> Filter.parenthesised = 1, bool */
-/* <Field> Filter.value = 2, expr */
-/* <Field> Filter.filter = 3, expr */
-
-/* <Field> For.location = 0, location */
-/* <Field> For.target = 1, expr */
-/* <Field> For.iter = 2, expr */
-/* <Field> For.body = 3, stmt_list */
-/* <Field> For.orelse = 4, stmt_list */
-/* <Field> For.is_async = 5, bool */
-
-/* <Field> FormattedValue.location = 0, location */
-/* <Field> FormattedValue.parenthesised = 1, bool */
-/* <Field> FormattedValue.value = 2, expr */
-/* <Field> FormattedValue.conversion = 3, str */
-/* <Field> FormattedValue.format_spec = 4, JoinedStr */
-
-/* <Field> Function.name = 0, str */
-/* <Field> Function.args = 1, parameter_list */
-/* <Field> Function.vararg = 2, expr */
-/* <Field> Function.kwonlyargs = 3, expr_list */
-/* <Field> Function.kwarg = 4, expr */
-/* <Field> Function.body = 5, stmt_list */
-/* <Field> Function.is_async = 6, bool */
-/* <Parent> Function = FunctionParent  */
-
-/* <Field> FunctionExpr.location = 0, location */
-/* <Field> FunctionExpr.parenthesised = 1, bool */
-/* <Field> FunctionExpr.name = 2, str */
-/* <Field> FunctionExpr.args = 3, arguments */
-/* <Field> FunctionExpr.returns = 4, expr */
-/* <Field> FunctionExpr.inner_scope = 5, Function */
-
-/* <Field> GeneratorExp.location = 0, location */
-/* <Field> GeneratorExp.parenthesised = 1, bool */
-/* <Field> GeneratorExp.function = 2, Function */
-/* <Field> GeneratorExp.iterable = 3, expr */
-
-/* <Field> Global.location = 0, location */
-/* <Field> Global.names = 1, str_list */
-
-/* <Field> If.location = 0, location */
-/* <Field> If.test = 1, expr */
-/* <Field> If.body = 2, stmt_list */
-/* <Field> If.orelse = 3, stmt_list */
-
-/* <Field> IfExp.location = 0, location */
-/* <Field> IfExp.parenthesised = 1, bool */
-/* <Field> IfExp.test = 2, expr */
-/* <Field> IfExp.body = 3, expr */
-/* <Field> IfExp.orelse = 4, expr */
-
-/* <Field> Import.location = 0, location */
-/* <Field> Import.names = 1, alias_list */
-
-/* <Field> ImportExpr.location = 0, location */
-/* <Field> ImportExpr.parenthesised = 1, bool */
-/* <Field> ImportExpr.level = 2, int */
-/* <Field> ImportExpr.name = 3, str */
-/* <Field> ImportExpr.top = 4, bool */
-
-/* <Field> ImportStar.location = 0, location */
-/* <Field> ImportStar.module = 1, expr */
-
-/* <Field> ImportMember.location = 0, location */
-/* <Field> ImportMember.parenthesised = 1, bool */
-/* <Field> ImportMember.module = 2, expr */
-/* <Field> ImportMember.name = 3, str */
-
-/* <Field> Fstring.location = 0, location */
-/* <Field> Fstring.parenthesised = 1, bool */
-/* <Field> Fstring.values = 2, expr_list */
-/* <Parent> Fstring = FormattedValue  */
-
-/* <Field> KeyValuePair.location = 0, location */
-/* <Field> KeyValuePair.value = 1, expr */
-/* <Field> KeyValuePair.key = 2, expr */
-
-/* <Field> Lambda.location = 0, location */
-/* <Field> Lambda.parenthesised = 1, bool */
-/* <Field> Lambda.args = 2, arguments */
-/* <Field> Lambda.inner_scope = 3, Function */
-
-/* <Field> List.location = 0, location */
-/* <Field> List.parenthesised = 1, bool */
-/* <Field> List.elts = 2, expr_list */
-/* <Field> List.ctx = 3, expr_context */
-
-/* <Field> ListComp.location = 0, location */
-/* <Field> ListComp.parenthesised = 1, bool */
-/* <Field> ListComp.function = 2, Function */
-/* <Field> ListComp.iterable = 3, expr */
-/* <Field> ListComp.generators = 4, comprehension_list */
-/* <Field> ListComp.elt = 5, expr */
-
-/* <Field> Module.name = 0, str */
-/* <Field> Module.hash = 1, str */
-/* <Field> Module.body = 2, stmt_list */
-/* <Field> Module.kind = 3, str */
-
-/* <Field> Name.location = 0, location */
-/* <Field> Name.parenthesised = 1, bool */
-/* <Field> Name.variable = 2, variable */
-/* <Field> Name.ctx = 3, expr_context */
-/* <Parent> Name = ParameterList  */
-
-/* <Field> Nonlocal.location = 0, location */
-/* <Field> Nonlocal.names = 1, str_list */
-
-/* <Field> Num.location = 0, location */
-/* <Field> Num.parenthesised = 1, bool */
-/* <Field> Num.n = 2, number */
-/* <Field> Num.text = 3, number */
-
-/* <Field> Pass.location = 0, location */
-
-/* <Field> PlaceHolder.location = 0, location */
-/* <Field> PlaceHolder.parenthesised = 1, bool */
-/* <Field> PlaceHolder.variable = 2, variable */
-/* <Field> PlaceHolder.ctx = 3, expr_context */
-
-/* <Field> Print.location = 0, location */
-/* <Field> Print.dest = 1, expr */
-/* <Field> Print.values = 2, expr_list */
-/* <Field> Print.nl = 3, bool */
-
-/* <Field> Raise.location = 0, location */
-/* <Field> Raise.exc = 1, expr */
-/* <Field> Raise.cause = 2, expr */
-/* <Field> Raise.type = 3, expr */
-/* <Field> Raise.inst = 4, expr */
-/* <Field> Raise.tback = 5, expr */
-
-/* <Field> Repr.location = 0, location */
-/* <Field> Repr.parenthesised = 1, bool */
-/* <Field> Repr.value = 2, expr */
-
-/* <Field> Return.location = 0, location */
-/* <Field> Return.value = 1, expr */
-
-/* <Field> Set.location = 0, location */
-/* <Field> Set.parenthesised = 1, bool */
-/* <Field> Set.elts = 2, expr_list */
-
-/* <Field> SetComp.location = 0, location */
-/* <Field> SetComp.parenthesised = 1, bool */
-/* <Field> SetComp.function = 2, Function */
-/* <Field> SetComp.iterable = 3, expr */
-
-/* <Field> Slice.location = 0, location */
-/* <Field> Slice.parenthesised = 1, bool */
-/* <Field> Slice.start = 2, expr */
-/* <Field> Slice.stop = 3, expr */
-/* <Field> Slice.step = 4, expr */
-
-/* <Field> SpecialOperation.location = 0, location */
-/* <Field> SpecialOperation.parenthesised = 1, bool */
-/* <Field> SpecialOperation.name = 2, str */
-/* <Field> SpecialOperation.arguments = 3, expr_list */
-
-/* <Field> Starred.location = 0, location */
-/* <Field> Starred.parenthesised = 1, bool */
-/* <Field> Starred.value = 2, expr */
-/* <Field> Starred.ctx = 3, expr_context */
-
-/* <Field> Str.location = 0, location */
-/* <Field> Str.parenthesised = 1, bool */
-/* <Field> Str.s = 2, str */
-/* <Field> Str.prefix = 3, str */
-/* <Field> Str.implicitly_concatenated_parts = 4, StringPart_list */
-
-/* <Field> StringPart.text = 0, str */
-/* <Field> StringPart.location = 1, location */
-/* <Parent> StringPart = StringPartList  */
-/* <Parent> StringPartList = BytesOrStr  */
-
-/* <Field> Subscript.location = 0, location */
-/* <Field> Subscript.parenthesised = 1, bool */
-/* <Field> Subscript.value = 2, expr */
-/* <Field> Subscript.index = 3, expr */
-/* <Field> Subscript.ctx = 4, expr_context */
-
-/* <Field> TemplateDottedNotation.location = 0, location */
-/* <Field> TemplateDottedNotation.parenthesised = 1, bool */
-/* <Field> TemplateDottedNotation.value = 2, expr */
-/* <Field> TemplateDottedNotation.attr = 3, str */
-/* <Field> TemplateDottedNotation.ctx = 4, expr_context */
-
-/* <Field> TemplateWrite.location = 0, location */
-/* <Field> TemplateWrite.value = 1, expr */
-
-/* <Field> Try.location = 0, location */
-/* <Field> Try.body = 1, stmt_list */
-/* <Field> Try.orelse = 2, stmt_list */
-/* <Field> Try.handlers = 3, stmt_list */
-/* <Field> Try.finalbody = 4, stmt_list */
-
-/* <Field> Tuple.location = 0, location */
-/* <Field> Tuple.parenthesised = 1, bool */
-/* <Field> Tuple.elts = 2, expr_list */
-/* <Field> Tuple.ctx = 3, expr_context */
-/* <Parent> Tuple = ParameterList  */
-
-/* <Field> UnaryExpr.location = 0, location */
-/* <Field> UnaryExpr.parenthesised = 1, bool */
-/* <Field> UnaryExpr.op = 2, unaryop */
-/* <Field> UnaryExpr.operand = 3, expr */
-
-/* <Field> While.location = 0, location */
-/* <Field> While.test = 1, expr */
-/* <Field> While.body = 2, stmt_list */
-/* <Field> While.orelse = 3, stmt_list */
-
-/* <Field> With.location = 0, location */
-/* <Field> With.context_expr = 1, expr */
-/* <Field> With.optional_vars = 2, expr */
-/* <Field> With.body = 3, stmt_list */
-/* <Field> With.is_async = 4, bool */
-
-/* <Field> Yield.location = 0, location */
-/* <Field> Yield.parenthesised = 1, bool */
-/* <Field> Yield.value = 2, expr */
-
-/* <Field> YieldFrom.location = 0, location */
-/* <Field> YieldFrom.parenthesised = 1, bool */
-/* <Field> YieldFrom.value = 2, expr */
-
-/* <Field> Alias.value = 0, expr */
-/* <Field> Alias.asname = 1, expr */
-/* <Parent> Alias = AliasList  */
-/* <Parent> AliasList = Import  */
-
-/* <Field> Arguments.kw_defaults = 0, expr_list */
-/* <Field> Arguments.defaults = 1, expr_list */
-/* <Field> Arguments.annotations = 2, expr_list */
-/* <Field> Arguments.varargannotation = 3, expr */
-/* <Field> Arguments.kwargannotation = 4, expr */
-/* <Field> Arguments.kw_annotations = 5, expr_list */
-/* <Parent> Arguments = ArgumentsParent  */
-/* <Parent> boolean = BoolParent  */
-/* <Parent> Boolop = BoolExpr  */
-/* <Parent> string = Bytes  */
-/* <Parent> Cmpop = CmpopList  */
-/* <Parent> CmpopList = Compare  */
-
-/* <Field> Comprehension.location = 0, location */
-/* <Field> Comprehension.iter = 1, expr */
-/* <Field> Comprehension.target = 2, expr */
-/* <Field> Comprehension.ifs = 3, expr_list */
-/* <Parent> Comprehension = ComprehensionList  */
-/* <Parent> ComprehensionList = ListComp  */
-/* <Parent> DictItem = DictItemList  */
-/* <Parent> DictItemList = DictItemListParent  */
-
-/* <Field> Expr.location = 0, location */
-/* <Field> Expr.parenthesised = 1, bool */
-/* <Parent> Expr = ExprParent  */
-/* <Parent> ExprContext = ExprContextParent  */
-/* <Parent> ExprList = ExprListParent  */
-/* <Parent> int = ImportExpr  */
-
-/* <Field> Keyword.location = 0, location */
-/* <Field> Keyword.value = 1, expr */
-/* <Field> Keyword.arg = 2, str */
-/* <Parent> Location = LocationParent  */
-/* <Parent> string = Num  */
-/* <Parent> Operator = BinaryExpr  */
-/* <Parent> ParameterList = Function  */
-
-/* <Field> Stmt.location = 0, location */
-/* <Parent> Stmt = StmtList  */
-/* <Parent> StmtList = StmtListParent  */
-/* <Parent> string = StrParent  */
-/* <Parent> StringList = StrListParent  */
-/* <Parent> Unaryop = UnaryExpr  */
-/* <Parent> Variable = VariableParent  */
-py_Classes(unique int id : @py_Class,
-    unique int parent : @py_ClassExpr ref);
-
-py_Functions(unique int id : @py_Function,
-    unique int parent : @py_Function_parent ref);
-
-py_Modules(unique int id : @py_Module);
-
-py_StringParts(unique int id : @py_StringPart,
-    int parent : @py_StringPart_list ref,
-    int idx : int ref);
-
-py_StringPart_lists(unique int id : @py_StringPart_list,
-    unique int parent : @py_Bytes_or_Str ref);
-
-py_aliases(unique int id : @py_alias,
-    int parent : @py_alias_list ref,
-    int idx : int ref);
-
-py_alias_lists(unique int id : @py_alias_list,
-    unique int parent : @py_Import ref);
-
-py_arguments(unique int id : @py_arguments,
-    unique int parent : @py_arguments_parent ref);
-
-py_bools(int parent : @py_bool_parent ref,
-    int idx : int ref);
-
-py_boolops(unique int id : @py_boolop,
-    int kind: int ref,
-    unique int parent : @py_BoolExpr ref);
-
-py_bytes(varchar(1) id : string ref,
-    int parent : @py_Bytes ref,
-    int idx : int ref);
-
-py_cmpops(unique int id : @py_cmpop,
-    int kind: int ref,
-    int parent : @py_cmpop_list ref,
-    int idx : int ref);
-
-py_cmpop_lists(unique int id : @py_cmpop_list,
-    unique int parent : @py_Compare ref);
-
-py_comprehensions(unique int id : @py_comprehension,
-    int parent : @py_comprehension_list ref,
-    int idx : int ref);
-
-py_comprehension_lists(unique int id : @py_comprehension_list,
-    unique int parent : @py_ListComp ref);
-
-py_dict_items(unique int id : @py_dict_item,
-    int kind: int ref,
-    int parent : @py_dict_item_list ref,
-    int idx : int ref);
-
-py_dict_item_lists(unique int id : @py_dict_item_list,
-    unique int parent : @py_dict_item_list_parent ref);
-
-py_exprs(unique int id : @py_expr,
-    int kind: int ref,
-    int parent : @py_expr_parent ref,
-    int idx : int ref);
-
-py_expr_contexts(unique int id : @py_expr_context,
-    int kind: int ref,
-    unique int parent : @py_expr_context_parent ref);
-
-py_expr_lists(unique int id : @py_expr_list,
-    int parent : @py_expr_list_parent ref,
-    int idx : int ref);
-
-py_ints(int id : int ref,
-    unique int parent : @py_ImportExpr ref);
-
-py_locations(unique int id : @location ref,
-    unique int parent : @py_location_parent ref);
-
-py_numbers(varchar(1) id : string ref,
-    int parent : @py_Num ref,
-    int idx : int ref);
-
-py_operators(unique int id : @py_operator,
-    int kind: int ref,
-    unique int parent : @py_BinaryExpr ref);
-
-py_parameter_lists(unique int id : @py_parameter_list,
-    unique int parent : @py_Function ref);
-
-py_stmts(unique int id : @py_stmt,
-    int kind: int ref,
-    int parent : @py_stmt_list ref,
-    int idx : int ref);
-
-py_stmt_lists(unique int id : @py_stmt_list,
-    int parent : @py_stmt_list_parent ref,
-    int idx : int ref);
-
-py_strs(varchar(1) id : string ref,
-    int parent : @py_str_parent ref,
-    int idx : int ref);
-
-py_str_lists(unique int id : @py_str_list,
-    unique int parent : @py_str_list_parent ref);
-
-py_unaryops(unique int id : @py_unaryop,
-    int kind: int ref,
-    unique int parent : @py_UnaryExpr ref);
-
-py_variables(int id : @py_variable ref,
-    unique int parent : @py_variable_parent ref);
-
-case @py_boolop.kind of
-    0 = @py_And
-|   1 = @py_Or;
-
-case @py_cmpop.kind of
-    0 = @py_Eq
-|   1 = @py_Gt
-|   2 = @py_GtE
-|   3 = @py_In
-|   4 = @py_Is
-|   5 = @py_IsNot
-|   6 = @py_Lt
-|   7 = @py_LtE
-|   8 = @py_NotEq
-|   9 = @py_NotIn;
-
-case @py_dict_item.kind of
-    0 = @py_DictUnpacking
-|   1 = @py_KeyValuePair
-|   2 = @py_keyword;
-
-case @py_expr.kind of
-    0 = @py_Attribute
-|   1 = @py_BinaryExpr
-|   2 = @py_BoolExpr
-|   3 = @py_Bytes
-|   4 = @py_Call
-|   5 = @py_ClassExpr
-|   6 = @py_Compare
-|   7 = @py_Dict
-|   8 = @py_DictComp
-|   9 = @py_Ellipsis
-|   10 = @py_FunctionExpr
-|   11 = @py_GeneratorExp
-|   12 = @py_IfExp
-|   13 = @py_ImportExpr
-|   14 = @py_ImportMember
-|   15 = @py_Lambda
-|   16 = @py_List
-|   17 = @py_ListComp
-|   18 = @py_Name
-|   19 = @py_Num
-|   20 = @py_Repr
-|   21 = @py_Set
-|   22 = @py_SetComp
-|   23 = @py_Slice
-|   24 = @py_Starred
-|   25 = @py_Str
-|   26 = @py_Subscript
-|   27 = @py_Tuple
-|   28 = @py_UnaryExpr
-|   29 = @py_Yield
-|   30 = @py_YieldFrom
-|   31 = @py_TemplateDottedNotation
-|   32 = @py_Filter
-|   33 = @py_PlaceHolder
-|   34 = @py_Await
-|   35 = @py_Fstring
-|   36 = @py_FormattedValue
-|   37 = @py_AssignExpr
-|   38 = @py_SpecialOperation;
-
-case @py_expr_context.kind of
-    0 = @py_AugLoad
-|   1 = @py_AugStore
-|   2 = @py_Del
-|   3 = @py_Load
-|   4 = @py_Param
-|   5 = @py_Store;
-
-case @py_operator.kind of
-    0 = @py_Add
-|   1 = @py_BitAnd
-|   2 = @py_BitOr
-|   3 = @py_BitXor
-|   4 = @py_Div
-|   5 = @py_FloorDiv
-|   6 = @py_LShift
-|   7 = @py_Mod
-|   8 = @py_Mult
-|   9 = @py_Pow
-|   10 = @py_RShift
-|   11 = @py_Sub
-|   12 = @py_MatMult;
-
-case @py_stmt.kind of
-    0 = @py_Assert
-|   1 = @py_Assign
-|   2 = @py_AugAssign
-|   3 = @py_Break
-|   4 = @py_Continue
-|   5 = @py_Delete
-|   6 = @py_ExceptStmt
-|   7 = @py_Exec
-|   8 = @py_Expr_stmt
-|   9 = @py_For
-|   10 = @py_Global
-|   11 = @py_If
-|   12 = @py_Import
-|   13 = @py_ImportStar
-|   14 = @py_Nonlocal
-|   15 = @py_Pass
-|   16 = @py_Print
-|   17 = @py_Raise
-|   18 = @py_Return
-|   19 = @py_Try
-|   20 = @py_While
-|   21 = @py_With
-|   22 = @py_TemplateWrite
-|   23 = @py_AnnAssign;
-
-case @py_unaryop.kind of
-    0 = @py_Invert
-|   1 = @py_Not
-|   2 = @py_UAdd
-|   3 = @py_USub;
-
-@py_Bytes_or_Str = @py_Bytes | @py_Str;
-
-@py_Function_parent = @py_DictComp | @py_FunctionExpr | @py_GeneratorExp | @py_Lambda | @py_ListComp | @py_SetComp;
-
-@py_arguments_parent = @py_FunctionExpr | @py_Lambda;
-
-@py_ast_node = @py_Class | @py_Function | @py_Module | @py_StringPart | @py_comprehension | @py_dict_item | @py_expr | @py_stmt;
-
-@py_bool_parent = @py_For | @py_Function | @py_Print | @py_With | @py_expr;
-
-@py_dict_item_list_parent = @py_Call | @py_ClassExpr | @py_Dict;
-
-@py_expr_context_parent = @py_Attribute | @py_List | @py_Name | @py_PlaceHolder | @py_Starred | @py_Subscript | @py_TemplateDottedNotation | @py_Tuple;
-
-@py_expr_list_parent = @py_Assign | @py_BoolExpr | @py_Call | @py_ClassExpr | @py_Compare | @py_Delete | @py_Fstring | @py_Function | @py_List | @py_Print | @py_Set | @py_SpecialOperation | @py_Tuple | @py_arguments | @py_comprehension;
-
-@py_expr_or_stmt = @py_expr | @py_stmt;
-
-@py_expr_parent = @py_AnnAssign | @py_Assert | @py_Assign | @py_AssignExpr | @py_Attribute | @py_AugAssign | @py_Await | @py_BinaryExpr | @py_Call | @py_Compare | @py_DictComp | @py_DictUnpacking | @py_ExceptStmt | @py_Exec | @py_Expr_stmt | @py_Filter | @py_For | @py_FormattedValue | @py_Function | @py_FunctionExpr | @py_GeneratorExp | @py_If | @py_IfExp | @py_ImportMember | @py_ImportStar | @py_KeyValuePair | @py_ListComp | @py_Print | @py_Raise | @py_Repr | @py_Return | @py_SetComp | @py_Slice | @py_Starred | @py_Subscript | @py_TemplateDottedNotation | @py_TemplateWrite | @py_UnaryExpr | @py_While | @py_With | @py_Yield | @py_YieldFrom | @py_alias | @py_arguments | @py_comprehension | @py_expr_list | @py_keyword | @py_parameter_list;
-
-@py_location_parent = @py_DictUnpacking | @py_KeyValuePair | @py_StringPart | @py_comprehension | @py_expr | @py_keyword | @py_stmt;
-
-@py_parameter = @py_Name | @py_Tuple;
-
-@py_scope = @py_Class | @py_Function | @py_Module;
-
-@py_stmt_list_parent = @py_Class | @py_ExceptStmt | @py_For | @py_Function | @py_If | @py_Module | @py_Try | @py_While | @py_With;
-
-@py_str_list_parent = @py_Global | @py_Nonlocal;
-
-@py_str_parent = @py_Attribute | @py_Class | @py_ClassExpr | @py_FormattedValue | @py_Function | @py_FunctionExpr | @py_ImportExpr | @py_ImportMember | @py_Module | @py_SpecialOperation | @py_Str | @py_StringPart | @py_TemplateDottedNotation | @py_keyword | @py_str_list;
-
-@py_variable_parent = @py_Name | @py_PlaceHolder;
-
-
-/*
- * End of auto-generated part
- */
-
-
-
-/* Map relative names to absolute names for imports */
-py_absolute_names(int module : @py_Module ref,
-                  varchar(1) relname : string ref,
-                  varchar(1) absname : string ref);
-
-py_exports(int id : @py_Module ref,
-           varchar(1) name : string ref);
-
-/* Successor information */
-py_successors(int predecessor : @py_flow_node ref,
-              int successor : @py_flow_node ref);
-
-py_true_successors(int predecessor : @py_flow_node ref,
-              int successor : @py_flow_node ref);
-
-py_exception_successors(int predecessor : @py_flow_node ref,
-              int successor : @py_flow_node ref);
-
-py_false_successors(int predecessor : @py_flow_node ref,
-              int successor : @py_flow_node ref);
-
-py_flow_bb_node(unique int flownode : @py_flow_node,
-             int realnode : @py_ast_node ref,
-             int basicblock : @py_flow_node ref,
-             int index : int ref);
-
-py_scope_flow(int flow : @py_flow_node ref,
-              int scope : @py_scope ref,
-              int kind : int ref);
-
-py_idoms(unique int node : @py_flow_node ref,
-         int immediate_dominator : @py_flow_node ref);
-
-py_ssa_phi(int phi : @py_ssa_var ref,
-           int arg: @py_ssa_var ref);
-
-py_ssa_var(unique int id : @py_ssa_var,
-            int var : @py_variable ref);
-
-py_ssa_use(int node: @py_flow_node ref,
-           int var : @py_ssa_var ref);
-
-py_ssa_defn(unique int id : @py_ssa_var ref,
-                  int node: @py_flow_node ref);
-
-@py_base_var = @py_variable | @py_ssa_var;
-
-py_scopes(unique int node : @py_expr_or_stmt ref,
-         int scope : @py_scope ref);
-
-py_scope_location(unique int id : @location ref,
-                   unique int scope : @py_scope ref);
-
-py_flags_versioned(varchar(1) name : string ref,
-                   varchar(1) value : string ref,
-                   varchar(1) version : string ref);
-
-py_syntax_error_versioned(unique int id : @location ref,
-                varchar(1) message : string ref,
-                varchar(1) version : string ref);
-
-py_comments(unique int id : @py_comment,
-           varchar(1) text : string ref,
-           unique int location : @location ref);
-
-/* Type information support */
-
-py_cobjects(unique int obj : @py_cobject);
-
-py_cobjecttypes(unique int obj : @py_cobject ref,
-                int typeof : @py_cobject ref);
-
-py_cobjectnames(unique int obj : @py_cobject ref,
-                varchar(1) name : string ref);
-
-/* Kind should be 0 for introspection, > 0 from source, as follows:
-   1 from C extension source
- */
-py_cobject_sources(int obj : @py_cobject ref,
-                  int kind : int ref);
-
-py_cmembers_versioned(int object : @py_cobject ref,
-                      varchar(1) name : string ref,
-                      int member : @py_cobject ref,
-                      varchar(1) version : string ref);
-
-py_citems(int object : @py_cobject ref,
-            int index : int ref,
-            int member : @py_cobject ref);
-
-ext_argtype(int funcid : @py_object ref,
-            int arg : int ref,
-            int typeid : @py_object ref);
-
-ext_rettype(int funcid : @py_object ref,
-            int typeid : @py_object ref);
-
-ext_proptype(int propid : @py_object ref,
-             int typeid : @py_object ref);
-
-ext_argreturn(int funcid : @py_object ref,
-              int arg : int ref);
-
-py_special_objects(unique int obj : @py_cobject ref,
-                   unique varchar(1) name : string ref);
-
-py_decorated_object(int object : @py_object ref,
-                    int level: int ref);
-
-@py_object = @py_cobject | @py_flow_node;
-
-@py_source_element = @py_ast_node | @container;
-
-/* XML Files */
-
-xmlEncoding  (unique int id: @file ref, varchar(900) encoding: string ref);
-
-xmlDTDs       (unique int id: @xmldtd,
-              varchar(900) root: string ref,
-              varchar(900) publicId: string ref,
-              varchar(900) systemId: string ref,
-              int fileid: @file ref);
-
-xmlElements   (unique int id: @xmlelement,
-              varchar(900) name: string ref,
-              int parentid: @xmlparent ref,
-              int idx: int ref,
-              int fileid: @file ref);
-
-xmlAttrs      (unique int id: @xmlattribute,
-              int elementid: @xmlelement ref,
-              varchar(900) name: string ref,
-              varchar(3600) value: string ref,
-              int idx: int ref,
-              int fileid: @file ref);
-
-xmlNs         (int id: @xmlnamespace,
-              varchar(900) prefixName: string ref,
-              varchar(900) URI: string ref,
-              int fileid: @file ref);
-
-xmlHasNs      (int elementId: @xmlnamespaceable ref,
-              int nsId: @xmlnamespace ref,
-              int fileid: @file ref);
-
-xmlComments   (unique int id: @xmlcomment,
-              varchar(3600) text: string ref,
-              int parentid: @xmlparent ref,
-              int fileid: @file ref);
-
-xmlChars      (unique int id: @xmlcharacters,
-              varchar(3600) text: string ref,
-              int parentid: @xmlparent ref,
-              int idx: int ref,
-              int isCDATA: int ref,
-              int fileid: @file ref);
-
-@xmlparent = @file | @xmlelement;
-@xmlnamespaceable = @xmlelement | @xmlattribute;
-
-xmllocations(int xmlElement: @xmllocatable ref,
-             int location: @location_default ref);
-
-@xmllocatable = @xmlcharacters | @xmlelement | @xmlcomment | @xmlattribute | @xmldtd | @file | @xmlnamespace;

python/ql/lib/python.qll

@@ -10,7 +10,6 @@ import semmle.python.Class
 import semmle.python.Import
 import semmle.python.Stmts
 import semmle.python.Exprs
-import semmle.python.Patterns
 import semmle.python.Keywords
 import semmle.python.Comprehensions
 import semmle.python.Flow

python/ql/lib/semmle/python/AstExtended.qll

@@ -82,12 +82,6 @@ library class StrListParent extends StrListParent_ { }
 /** Internal implementation class */
 library class ExprParent extends ExprParent_ { }
 
-/** Internal implementation class */
-class PatternListParent extends PatternListParent_ { }
-
-/** Internal implementation class */
-library class PatternParent extends PatternParent_ { }
-
 library class DictItem extends DictItem_, AstNode {
   override string toString() { result = DictItem_.super.toString() }
 
@@ -168,9 +162,6 @@ class ExprList extends ExprList_ {
   /* syntax: Expr, ... */
 }
 
-/** A list of patterns */
-class PatternList extends PatternList_ { }
-
 library class DictItemList extends DictItemList_ { }
 
 library class DictItemListParent extends DictItemListParent_ { }

python/ql/lib/semmle/python/AstGenerated.qll

@@ -218,26 +218,6 @@ library class Call_ extends @py_Call, Expr {
   override string toString() { result = "Call" }
 }
 
-/** INTERNAL: See the class `Case` for further information. */
-library class Case_ extends @py_Case, Stmt {
-  /** Gets the pattern of this case statement. */
-  Pattern getPattern() { py_patterns(result, _, this, 1) }
-
-  /** Gets the guard of this case statement. */
-  Expr getGuard() { py_exprs(result, _, this, 2) }
-
-  /** Gets the body of this case statement. */
-  StmtList getBody() { py_stmt_lists(result, this, 3) }
-
-  /** Gets the nth statement of this case statement. */
-  Stmt getStmt(int index) { result = this.getBody().getItem(index) }
-
-  /** Gets a statement of this case statement. */
-  Stmt getAStmt() { result = this.getBody().getAnItem() }
-
-  override string toString() { result = "Case" }
-}
-
 /** INTERNAL: See the class `Class` for further information. */
 library class Class_ extends @py_Class {
   /** Gets the name of this class. */
@@ -252,7 +232,6 @@ library class Class_ extends @py_Class {
   /** Gets a statement of this class. */
   Stmt getAStmt() { result = this.getBody().getAnItem() }
 
-  /** Gets a parent of this class */
   ClassExpr getParent() { py_Classes(this, result) }
 
   /** Gets a textual representation of this element. */
@@ -534,7 +513,6 @@ library class Function_ extends @py_Function {
   /** Whether the async property of this function is true. */
   predicate isAsync() { py_bools(this, 6) }
 
-  /** Gets a parent of this function */
   FunctionParent getParent() { py_Functions(this, result) }
 
   /** Gets a textual representation of this element. */
@@ -599,14 +577,6 @@ library class GtE_ extends @py_GtE, Cmpop {
   override string toString() { result = "GtE" }
 }
 
-/** INTERNAL: See the class `Guard` for further information. */
-library class Guard_ extends @py_Guard, Expr {
-  /** Gets the test of this guard expression. */
-  Expr getTest() { py_exprs(result, _, this, 2) }
-
-  override string toString() { result = "Guard" }
-}
-
 /** INTERNAL: See the class `If` for further information. */
 library class If_ extends @py_If, Stmt {
   /** Gets the test of this if statement. */
@@ -820,172 +790,6 @@ library class MatMult_ extends @py_MatMult, Operator {
   override string toString() { result = "MatMult" }
 }
 
-/** INTERNAL: See the class `MatchStmt` for further information. */
-library class MatchStmt_ extends @py_MatchStmt, Stmt {
-  /** Gets the subject of this match statement. */
-  Expr getSubject() { py_exprs(result, _, this, 1) }
-
-  /** Gets the cases of this match statement. */
-  StmtList getCases() { py_stmt_lists(result, this, 2) }
-
-  /** Gets the nth case of this match statement. */
-  Stmt getCase(int index) { result = this.getCases().getItem(index) }
-
-  /** Gets a case of this match statement. */
-  Stmt getACase() { result = this.getCases().getAnItem() }
-
-  override string toString() { result = "MatchStmt" }
-}
-
-/** INTERNAL: See the class `MatchAsPattern` for further information. */
-library class MatchAsPattern_ extends @py_MatchAsPattern, Pattern {
-  /** Gets the pattern of this matchaspattern pattern. */
-  Pattern getPattern() { py_patterns(result, _, this, 2) }
-
-  /** Gets the alias of this matchaspattern pattern. */
-  Expr getAlias() { py_exprs(result, _, this, 3) }
-
-  override string toString() { result = "MatchAsPattern" }
-}
-
-/** INTERNAL: See the class `MatchCapturePattern` for further information. */
-library class MatchCapturePattern_ extends @py_MatchCapturePattern, Pattern {
-  /** Gets the variable of this matchcapturepattern pattern. */
-  Expr getVariable() { py_exprs(result, _, this, 2) }
-
-  override string toString() { result = "MatchCapturePattern" }
-}
-
-/** INTERNAL: See the class `MatchClassPattern` for further information. */
-library class MatchClassPattern_ extends @py_MatchClassPattern, Pattern {
-  /** Gets the class of this matchclasspattern pattern. */
-  Expr getClass() { py_exprs(result, _, this, 2) }
-
-  /** Gets the class_name of this matchclasspattern pattern. */
-  Expr getClassName() { py_exprs(result, _, this, 3) }
-
-  /** Gets the positional of this matchclasspattern pattern. */
-  PatternList getPositional() { py_pattern_lists(result, this, 4) }
-
-  /** Gets the nth positional of this matchclasspattern pattern. */
-  Pattern getPositional(int index) { result = this.getPositional().getItem(index) }
-
-  /** Gets a positional of this matchclasspattern pattern. */
-  Pattern getAPositional() { result = this.getPositional().getAnItem() }
-
-  /** Gets the keyword of this matchclasspattern pattern. */
-  PatternList getKeyword() { py_pattern_lists(result, this, 5) }
-
-  /** Gets the nth keyword of this matchclasspattern pattern. */
-  Pattern getKeyword(int index) { result = this.getKeyword().getItem(index) }
-
-  /** Gets a keyword of this matchclasspattern pattern. */
-  Pattern getAKeyword() { result = this.getKeyword().getAnItem() }
-
-  override string toString() { result = "MatchClassPattern" }
-}
-
-/** INTERNAL: See the class `MatchDoubleStarPattern` for further information. */
-library class MatchDoubleStarPattern_ extends @py_MatchDoubleStarPattern, Pattern {
-  /** Gets the target of this matchdoublestarpattern pattern. */
-  Pattern getTarget() { py_patterns(result, _, this, 2) }
-
-  override string toString() { result = "MatchDoubleStarPattern" }
-}
-
-/** INTERNAL: See the class `MatchKeyValuePattern` for further information. */
-library class MatchKeyValuePattern_ extends @py_MatchKeyValuePattern, Pattern {
-  /** Gets the key of this matchkeyvaluepattern pattern. */
-  Pattern getKey() { py_patterns(result, _, this, 2) }
-
-  /** Gets the value of this matchkeyvaluepattern pattern. */
-  Pattern getValue() { py_patterns(result, _, this, 3) }
-
-  override string toString() { result = "MatchKeyValuePattern" }
-}
-
-/** INTERNAL: See the class `MatchKeywordPattern` for further information. */
-library class MatchKeywordPattern_ extends @py_MatchKeywordPattern, Pattern {
-  /** Gets the attribute of this matchkeywordpattern pattern. */
-  Expr getAttribute() { py_exprs(result, _, this, 2) }
-
-  /** Gets the value of this matchkeywordpattern pattern. */
-  Pattern getValue() { py_patterns(result, _, this, 3) }
-
-  override string toString() { result = "MatchKeywordPattern" }
-}
-
-/** INTERNAL: See the class `MatchLiteralPattern` for further information. */
-library class MatchLiteralPattern_ extends @py_MatchLiteralPattern, Pattern {
-  /** Gets the literal of this matchliteralpattern pattern. */
-  Expr getLiteral() { py_exprs(result, _, this, 2) }
-
-  override string toString() { result = "MatchLiteralPattern" }
-}
-
-/** INTERNAL: See the class `MatchMappingPattern` for further information. */
-library class MatchMappingPattern_ extends @py_MatchMappingPattern, Pattern {
-  /** Gets the mappings of this matchmappingpattern pattern. */
-  PatternList getMappings() { py_pattern_lists(result, this, 2) }
-
-  /** Gets the nth mapping of this matchmappingpattern pattern. */
-  Pattern getMapping(int index) { result = this.getMappings().getItem(index) }
-
-  /** Gets a mapping of this matchmappingpattern pattern. */
-  Pattern getAMapping() { result = this.getMappings().getAnItem() }
-
-  override string toString() { result = "MatchMappingPattern" }
-}
-
-/** INTERNAL: See the class `MatchOrPattern` for further information. */
-library class MatchOrPattern_ extends @py_MatchOrPattern, Pattern {
-  /** Gets the patterns of this matchorpattern pattern. */
-  PatternList getPatterns() { py_pattern_lists(result, this, 2) }
-
-  /** Gets the nth pattern of this matchorpattern pattern. */
-  Pattern getPattern(int index) { result = this.getPatterns().getItem(index) }
-
-  /** Gets a pattern of this matchorpattern pattern. */
-  Pattern getAPattern() { result = this.getPatterns().getAnItem() }
-
-  override string toString() { result = "MatchOrPattern" }
-}
-
-/** INTERNAL: See the class `MatchSequencePattern` for further information. */
-library class MatchSequencePattern_ extends @py_MatchSequencePattern, Pattern {
-  /** Gets the patterns of this matchsequencepattern pattern. */
-  PatternList getPatterns() { py_pattern_lists(result, this, 2) }
-
-  /** Gets the nth pattern of this matchsequencepattern pattern. */
-  Pattern getPattern(int index) { result = this.getPatterns().getItem(index) }
-
-  /** Gets a pattern of this matchsequencepattern pattern. */
-  Pattern getAPattern() { result = this.getPatterns().getAnItem() }
-
-  override string toString() { result = "MatchSequencePattern" }
-}
-
-/** INTERNAL: See the class `MatchStarPattern` for further information. */
-library class MatchStarPattern_ extends @py_MatchStarPattern, Pattern {
-  /** Gets the target of this matchstarpattern pattern. */
-  Pattern getTarget() { py_patterns(result, _, this, 2) }
-
-  override string toString() { result = "MatchStarPattern" }
-}
-
-/** INTERNAL: See the class `MatchValuePattern` for further information. */
-library class MatchValuePattern_ extends @py_MatchValuePattern, Pattern {
-  /** Gets the value of this matchvaluepattern pattern. */
-  Expr getValue() { py_exprs(result, _, this, 2) }
-
-  override string toString() { result = "MatchValuePattern" }
-}
-
-/** INTERNAL: See the class `MatchWildcardPattern` for further information. */
-library class MatchWildcardPattern_ extends @py_MatchWildcardPattern, Pattern {
-  override string toString() { result = "MatchWildcardPattern" }
-}
-
 /** INTERNAL: See the class `Mod` for further information. */
 library class Mod_ extends @py_Mod, Operator {
   override string toString() { result = "Mod" }
@@ -1269,7 +1073,6 @@ library class StringPart_ extends @py_StringPart {
   /** Gets the location of this implicitly concatenated part. */
   Location getLocation() { py_locations(result, this) }
 
-  /** Gets a parent of this implicitly concatenated part */
   StringPartList getParent() { py_StringParts(this, result, _) }
 
   /** Gets a textual representation of this element. */
@@ -1278,7 +1081,6 @@ library class StringPart_ extends @py_StringPart {
 
 /** INTERNAL: See the class `StringPartList` for further information. */
 library class StringPartList_ extends @py_StringPart_list {
-  /** Gets a parent of this implicitly concatenated part list */
   BytesOrStr getParent() { py_StringPart_lists(this, result) }
 
   /** Gets an item of this implicitly concatenated part list */
@@ -1486,7 +1288,6 @@ library class Alias_ extends @py_alias {
   /** Gets the name of this alias. */
   Expr getAsname() { py_exprs(result, _, this, 1) }
 
-  /** Gets a parent of this alias */
   AliasList getParent() { py_aliases(this, result, _) }
 
   /** Gets a textual representation of this element. */
@@ -1495,7 +1296,6 @@ library class Alias_ extends @py_alias {
 
 /** INTERNAL: See the class `AliasList` for further information. */
 library class AliasList_ extends @py_alias_list {
-  /** Gets a parent of this alias list */
   Import getParent() { py_alias_lists(this, result) }
 
   /** Gets an item of this alias list */
@@ -1552,7 +1352,6 @@ library class Arguments_ extends @py_arguments {
   /** Gets a keyword-only annotation of this parameters definition. */
   Expr getAKwAnnotation() { result = this.getKwAnnotations().getAnItem() }
 
-  /** Gets a parent of this parameters definition */
   ArgumentsParent getParent() { py_arguments(this, result) }
 
   /** Gets a textual representation of this element. */
@@ -1579,7 +1378,6 @@ library class BoolParent_ extends @py_bool_parent {
 
 /** INTERNAL: See the class `Boolop` for further information. */
 library class Boolop_ extends @py_boolop {
-  /** Gets a parent of this boolean operator */
   BoolExpr getParent() { py_boolops(this, _, result) }
 
   /** Gets a textual representation of this element. */
@@ -1588,7 +1386,6 @@ library class Boolop_ extends @py_boolop {
 
 /** INTERNAL: See the class `Cmpop` for further information. */
 library class Cmpop_ extends @py_cmpop {
-  /** Gets a parent of this comparison operator */
   CmpopList getParent() { py_cmpops(this, _, result, _) }
 
   /** Gets a textual representation of this element. */
@@ -1597,7 +1394,6 @@ library class Cmpop_ extends @py_cmpop {
 
 /** INTERNAL: See the class `CmpopList` for further information. */
 library class CmpopList_ extends @py_cmpop_list {
-  /** Gets a parent of this comparison operator list */
   Compare getParent() { py_cmpop_lists(this, result) }
 
   /** Gets an item of this comparison operator list */
@@ -1630,7 +1426,6 @@ library class Comprehension_ extends @py_comprehension {
   /** Gets a condition of this comprehension. */
   Expr getAnIf() { result = this.getIfs().getAnItem() }
 
-  /** Gets a parent of this comprehension */
   ComprehensionList getParent() { py_comprehensions(this, result, _) }
 
   /** Gets a textual representation of this element. */
@@ -1639,7 +1434,6 @@ library class Comprehension_ extends @py_comprehension {
 
 /** INTERNAL: See the class `ComprehensionList` for further information. */
 library class ComprehensionList_ extends @py_comprehension_list {
-  /** Gets a parent of this comprehension list */
   ListComp getParent() { py_comprehension_lists(this, result) }
 
   /** Gets an item of this comprehension list */
@@ -1654,7 +1448,6 @@ library class ComprehensionList_ extends @py_comprehension_list {
 
 /** INTERNAL: See the class `DictItem` for further information. */
 library class DictItem_ extends @py_dict_item {
-  /** Gets a parent of this dict_item */
   DictItemList getParent() { py_dict_items(this, _, result, _) }
 
   /** Gets a textual representation of this element. */
@@ -1663,7 +1456,6 @@ library class DictItem_ extends @py_dict_item {
 
 /** INTERNAL: See the class `DictItemList` for further information. */
 library class DictItemList_ extends @py_dict_item_list {
-  /** Gets a parent of this dict_item list */
   DictItemListParent getParent() { py_dict_item_lists(this, result) }
 
   /** Gets an item of this dict_item list */
@@ -1690,7 +1482,6 @@ library class Expr_ extends @py_expr {
   /** Whether the parenthesised property of this expression is true. */
   predicate isParenthesised() { py_bools(this, 1) }
 
-  /** Gets a parent of this expression */
   ExprParent getParent() { py_exprs(this, _, result, _) }
 
   /** Gets a textual representation of this element. */
@@ -1699,7 +1490,6 @@ library class Expr_ extends @py_expr {
 
 /** INTERNAL: See the class `ExprContext` for further information. */
 library class ExprContext_ extends @py_expr_context {
-  /** Gets a parent of this expression context */
   ExprContextParent getParent() { py_expr_contexts(this, _, result) }
 
   /** Gets a textual representation of this element. */
@@ -1714,7 +1504,6 @@ library class ExprContextParent_ extends @py_expr_context_parent {
 
 /** INTERNAL: See the class `ExprList` for further information. */
 library class ExprList_ extends @py_expr_list {
-  /** Gets a parent of this expression list */
   ExprListParent getParent() { py_expr_lists(this, result, _) }
 
   /** Gets an item of this expression list */
@@ -1767,7 +1556,6 @@ library class LocationParent_ extends @py_location_parent {
 
 /** INTERNAL: See the class `Operator` for further information. */
 library class Operator_ extends @py_operator {
-  /** Gets a parent of this operator */
   BinaryExpr getParent() { py_operators(this, _, result) }
 
   /** Gets a textual representation of this element. */
@@ -1780,48 +1568,6 @@ library class Parameter_ extends @py_parameter {
   string toString() { result = "Parameter" }
 }
 
-/** INTERNAL: See the class `Pattern` for further information. */
-library class Pattern_ extends @py_pattern {
-  /** Gets the location of this pattern. */
-  Location getLocation() { py_locations(result, this) }
-
-  /** Whether the parenthesised property of this pattern is true. */
-  predicate isParenthesised() { py_bools(this, 1) }
-
-  /** Gets a parent of this pattern */
-  PatternParent getParent() { py_patterns(this, _, result, _) }
-
-  /** Gets a textual representation of this element. */
-  string toString() { result = "Pattern" }
-}
-
-/** INTERNAL: See the class `PatternList` for further information. */
-library class PatternList_ extends @py_pattern_list {
-  /** Gets a parent of this pattern list */
-  PatternListParent getParent() { py_pattern_lists(this, result, _) }
-
-  /** Gets an item of this pattern list */
-  Pattern getAnItem() { py_patterns(result, _, this, _) }
-
-  /** Gets the nth item of this pattern list */
-  Pattern getItem(int index) { py_patterns(result, _, this, index) }
-
-  /** Gets a textual representation of this element. */
-  string toString() { result = "PatternList" }
-}
-
-/** INTERNAL: See the class `PatternListParent` for further information. */
-library class PatternListParent_ extends @py_pattern_list_parent {
-  /** Gets a textual representation of this element. */
-  string toString() { result = "PatternListParent" }
-}
-
-/** INTERNAL: See the class `PatternParent` for further information. */
-library class PatternParent_ extends @py_pattern_parent {
-  /** Gets a textual representation of this element. */
-  string toString() { result = "PatternParent" }
-}
-
 /** INTERNAL: See the class `Scope` for further information. */
 library class Scope_ extends @py_scope {
   /** Gets a textual representation of this element. */
@@ -1833,7 +1579,6 @@ library class Stmt_ extends @py_stmt {
   /** Gets the location of this statement. */
   Location getLocation() { py_locations(result, this) }
 
-  /** Gets a parent of this statement */
   StmtList getParent() { py_stmts(this, _, result, _) }
 
   /** Gets a textual representation of this element. */
@@ -1842,7 +1587,6 @@ library class Stmt_ extends @py_stmt {
 
 /** INTERNAL: See the class `StmtList` for further information. */
 library class StmtList_ extends @py_stmt_list {
-  /** Gets a parent of this statement list */
   StmtListParent getParent() { py_stmt_lists(this, result, _) }
 
   /** Gets an item of this statement list */
@@ -1863,7 +1607,6 @@ library class StmtListParent_ extends @py_stmt_list_parent {
 
 /** INTERNAL: See the class `StringList` for further information. */
 library class StringList_ extends @py_str_list {
-  /** Gets a parent of this string list */
   StrListParent getParent() { py_str_lists(this, result) }
 
   /** Gets an item of this string list */
@@ -1890,7 +1633,6 @@ library class StrParent_ extends @py_str_parent {
 
 /** INTERNAL: See the class `Unaryop` for further information. */
 library class Unaryop_ extends @py_unaryop {
-  /** Gets a parent of this unary operation */
   UnaryExpr getParent() { py_unaryops(this, _, result) }
 
   /** Gets a textual representation of this element. */

python/ql/lib/semmle/python/Exprs.qll

@@ -718,12 +718,6 @@ class FormattedValue extends FormattedValue_ {
   }
 }
 
-/** A guard in a case statement */
-class Guard extends Guard_ {
-  /* syntax: if Expr */
-  override Expr getASubExpression() { result = this.getTest() }
-}
-
 /* Expression Contexts */
 /** A context in which an expression used */
 class ExprContext extends ExprContext_ { }

python/ql/lib/semmle/python/Patterns.qll

@@ -1,118 +0,0 @@
-/**
- * Wrapping generated AST classes: `Pattern_` and subclasses.
- */
-
-import python
-
-/** A pattern in a match statement */
-class Pattern extends Pattern_, AstNode {
-  /** Gets the scope of this pattern */
-  override Scope getScope() { result = this.getCase().getScope() }
-
-  /** Gets the case statement containing this pattern */
-  Case getCase() { result.contains(this) }
-
-  override string toString() { result = "Pattern" }
-
-  /** Gets the module enclosing this pattern */
-  Module getEnclosingModule() { result = this.getScope().getEnclosingModule() }
-
-  /** Whether the parenthesized property of this expression is true. */
-  predicate isParenthesized() { Pattern_.super.isParenthesised() }
-
-  override Location getLocation() { result = Pattern_.super.getLocation() }
-
-  /** Gets an immediate (non-nested) sub-expression of this pattern */
-  Expr getASubExpression() { none() }
-
-  /** Gets an immediate (non-nested) sub-statement of this pattern */
-  Stmt getASubStatement() { none() }
-
-  /** Gets an immediate (non-nested) sub-pattern of this pattern */
-  Pattern getASubPattern() { none() }
-
-  override AstNode getAChildNode() {
-    result = this.getASubExpression()
-    or
-    result = this.getASubStatement()
-    or
-    result = this.getASubPattern()
-  }
-}
-
-/** An as-pattern in a match statement: `<subpattern> as alias` */
-class MatchAsPattern extends MatchAsPattern_ {
-  override Pattern getASubPattern() { result = this.getPattern() }
-
-  override Expr getASubExpression() { result = this.getAlias() }
-
-  override Name getAlias() { result = super.getAlias() }
-}
-
-/** An or-pattern in a match statement: `(<pattern1>|<pattern2>)` */
-class MatchOrPattern extends MatchOrPattern_ {
-  override Pattern getASubPattern() { result = this.getAPattern() }
-}
-
-/** A literal pattern in a match statement: `42` */
-class MatchLiteralPattern extends MatchLiteralPattern_ {
-  override Expr getASubExpression() { result = this.getLiteral() }
-}
-
-/** A capture pattern in a match statement: `var` */
-class MatchCapturePattern extends MatchCapturePattern_ {
-  /* syntax: varname */
-  override Expr getASubExpression() { result = this.getVariable() }
-
-  /** Gets the variable that is bound by this capture pattern */
-  override Name getVariable() { result = super.getVariable() }
-}
-
-/** A wildcard pattern in a match statement: `_` */
-class MatchWildcardPattern extends MatchWildcardPattern_ { }
-
-/** A value pattern in a match statement: `Http.OK` */
-class MatchValuePattern extends MatchValuePattern_ {
-  override Expr getASubExpression() { result = this.getValue() }
-}
-
-/** A sequence pattern in a match statement `<p1>, <p2>` */
-class MatchSequencePattern extends MatchSequencePattern_ {
-  override Pattern getASubPattern() { result = this.getAPattern() }
-}
-
-/** A star pattern in a match statement: `(..., *)` */
-class MatchStarPattern extends MatchStarPattern_ {
-  override Pattern getASubPattern() { result = this.getTarget() }
-}
-
-/** A mapping pattern in a match statement: `{'a': var}` */
-class MatchMappingPattern extends MatchMappingPattern_ {
-  override Pattern getASubPattern() { result = this.getAMapping() }
-}
-
-/** A double star pattern in a match statement: `{..., **}` */
-class MatchDoubleStarPattern extends MatchDoubleStarPattern_ {
-  override Pattern getASubPattern() { result = this.getTarget() }
-}
-
-/** A key-value pattern inside a mapping pattern: `a: var` */
-class MatchKeyValuePattern extends MatchKeyValuePattern_ {
-  override Pattern getASubPattern() { result = this.getKey() or result = this.getValue() }
-}
-
-/** A class pattern in a match statement: `Circle(radius = 3)` */
-class MatchClassPattern extends MatchClassPattern_ {
-  override Expr getASubExpression() { result = this.getClassName() }
-
-  override Pattern getASubPattern() {
-    result = this.getAPositional() or result = this.getAKeyword()
-  }
-}
-
-/** A keyword pattern inside a class pattern: `radius = 3` */
-class MatchKeywordPattern extends MatchKeywordPattern_ {
-  override Expr getASubExpression() { result = this.getAttribute() }
-
-  override Pattern getASubPattern() { result = this.getValue() }
-}

python/ql/lib/semmle/python/Stmts.qll

@@ -18,15 +18,10 @@ class Stmt extends Stmt_, AstNode {
   /** Gets an immediate (non-nested) sub-statement of this statement */
   Stmt getASubStatement() { none() }
 
-  /** Gets an immediate (non-nested) sub-pattern of this statement */
-  Pattern getASubPattern() { none() }
-
   override AstNode getAChildNode() {
     result = this.getASubExpression()
     or
     result = this.getASubStatement()
-    or
-    result = this.getASubPattern()
   }
 
   private ControlFlowNode possibleEntryNode() {
@@ -417,24 +412,6 @@ class With extends With_ {
   override Stmt getLastStatement() { result = this.getBody().getLastItem().getLastStatement() }
 }
 
-/** A match statement */
-class MatchStmt extends MatchStmt_ {
-  /* syntax: match subject: */
-  override Expr getASubExpression() { result = this.getSubject() }
-
-  override Stmt getASubStatement() { result = this.getCase(_) }
-}
-
-/** A case statement */
-class Case extends Case_ {
-  /* syntax: case pattern if guard: */
-  override Expr getASubExpression() { result = this.getGuard() }
-
-  override Stmt getASubStatement() { result = this.getStmt(_) }
-
-  override Pattern getASubPattern() { result = this.getPattern() }
-}
-
 /** A plain text used in a template is wrapped in a TemplateWrite statement */
 class TemplateWrite extends TemplateWrite_ {
   override Expr getASubExpression() { result = this.getValue() }

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPrivate.qll

@@ -249,8 +249,6 @@ module EssaFlow {
     // Flow inside an unpacking assignment
     iterableUnpackingFlowStep(nodeFrom, nodeTo)
     or
-    matchFlowStep(nodeFrom, nodeTo)
-    or
     // Overflow keyword argument
     exists(CallNode call, CallableValue callable |
       call = callable.getACall() and
@@ -984,8 +982,6 @@ predicate storeStep(Node nodeFrom, Content c, Node nodeTo) {
   posOverflowStoreStep(nodeFrom, c, nodeTo)
   or
   kwOverflowStoreStep(nodeFrom, c, nodeTo)
-  or
-  matchStoreStep(nodeFrom, c, nodeTo)
 }
 
 /** Data flows from an element of a list to the list. */
@@ -1128,8 +1124,6 @@ predicate readStep(Node nodeFrom, Content c, Node nodeTo) {
   or
   iterableUnpackingReadStep(nodeFrom, c, nodeTo)
   or
-  matchReadStep(nodeFrom, c, nodeTo)
-  or
   popReadStep(nodeFrom, c, nodeTo)
   or
   forReadStep(nodeFrom, c, nodeTo)
@@ -1559,318 +1553,6 @@ module IterableUnpacking {
 
 import IterableUnpacking
 
-/**
- * There are a number of patterns available for the match statement.
- * Each one transfers data and content differently to its parts.
- *
- * Furthermore, given a successful match, we can infer some data about
- * the subject. Consider the example:
- * ```python
- * match choice:
- *   case 'Y':
- *     ...body
- * ```
- * Inside `body`, we know that `choice` has the value `'Y'`.
- *
- * A similar thing happens with the "as pattern". Consider the example:
- * ```python
- * match choice:
- *  case ('y'|'Y') as c:
- *   ...body
- * ```
- * By the binding rules, there is data flow from `choice` to `c`. But we
- * can infer the value of `c` to be either `'y'` or `'Y'` if the match succeeds.
- *
- * We will treat such inferences separately as guards. First we will model the data flow
- * stemming from the bindings and the matching of shape. Below, 'subject' is not necessarily the
- * top-level subject of the match, but rather the part recursively matched by the current pattern.
- * For instance, in the example:
- * ```python
- * match command:
- *  case ('quit' as c) | ('go', ('up'|'down') as c):
- *   ...body
- * ```
- * `command` is the subject of first the as-pattern, while the second component of `command`
- * is the subject of the second as-pattern. As such, 'subject' refers to the pattern under evaluation.
- *
- * - as pattern: subject flows to alias as well as to the interior pattern
- * - or pattern: subject flows to each alternative
- * - literal pattern: flow from the literal to the pattern, to add information
- * - capture pattern: subject flows to the variable
- * - wildcard pattern: no flow
- * - value pattern: flow from the value to the pattern, to add information
- * - sequence pattern: each element reads from subject at the associated index
- * - star pattern: subject flows to the variable, possibly via a conversion
- * - mapping pattern: each value reads from subject at the associated key
- * - double star pattern: subject flows to the variable, possibly via a conversion
- * - key-value pattern: the value reads from the subject at the key (see mapping pattern)
- * - class pattern: all keywords read the appropriate attribute from the subject
- * - keyword pattern: the appropriate attribute is read from the subject (see class pattern)
- *
- * Inside the class pattern, we also find positional arguments. They are converted to
- * keyword arguments using the `__match_args__` attribute on the class. We do not
- * currently model this.
- */
-module MatchUnpacking {
-  /**
-   * The subject of a match flows to each top-level pattern
-   * (a pattern directly under a `case` statement).
-   *
-   * We could consider a model closer to use-use-flow, where the subject
-   * only flows to the first top-level pattern and from there to the
-   * following ones.
-   */
-  predicate matchSubjectFlowStep(Node nodeFrom, Node nodeTo) {
-    exists(MatchStmt match, Expr subject, Pattern target |
-      subject = match.getSubject() and
-      target = match.getCase(_).(Case).getPattern()
-    |
-      nodeFrom.asExpr() = subject and
-      nodeTo.asCfgNode().getNode() = target
-    )
-  }
-
-  /**
-   * as pattern: subject flows to alias as well as to the interior pattern
-   * syntax (toplevel): `case pattern as alias:`
-   */
-  predicate matchAsFlowStep(Node nodeFrom, Node nodeTo) {
-    exists(MatchAsPattern subject, Name alias | alias = subject.getAlias() |
-      // We make the subject flow to the interior pattern via the alias.
-      // That way, information can propagate from the interior pattern to the alias.
-      //
-      // the subject flows to the interior pattern
-      nodeFrom.asCfgNode().getNode() = subject and
-      nodeTo.asCfgNode().getNode() = subject.getPattern()
-      or
-      // the interior pattern flows to the alias
-      nodeFrom.asCfgNode().getNode() = subject.getPattern() and
-      nodeTo.asVar().getDefinition().(PatternAliasDefinition).getDefiningNode().getNode() = alias
-    )
-  }
-
-  /**
-   * or pattern: subject flows to each alternative
-   * syntax (toplevel): `case alt1 | alt2:`
-   */
-  predicate matchOrFlowStep(Node nodeFrom, Node nodeTo) {
-    exists(MatchOrPattern subject, Pattern pattern | pattern = subject.getAPattern() |
-      nodeFrom.asCfgNode().getNode() = subject and
-      nodeTo.asCfgNode().getNode() = pattern
-    )
-  }
-
-  /**
-   * literal pattern: flow from the literal to the pattern, to add information
-   * syntax (toplevel): `case literal:`
-   */
-  predicate matchLiteralFlowStep(Node nodeFrom, Node nodeTo) {
-    exists(MatchLiteralPattern pattern, Expr literal | literal = pattern.getLiteral() |
-      nodeFrom.asExpr() = literal and
-      nodeTo.asCfgNode().getNode() = pattern
-    )
-  }
-
-  /**
-   * capture pattern: subject flows to the variable
-   * syntax (toplevel): `case var:`
-   */
-  predicate matchCaptureFlowStep(Node nodeFrom, Node nodeTo) {
-    exists(MatchCapturePattern capture, Name var | capture.getVariable() = var |
-      nodeFrom.asCfgNode().getNode() = capture and
-      nodeTo.asVar().getDefinition().(PatternCaptureDefinition).getDefiningNode().getNode() = var
-    )
-  }
-
-  /**
-   * value pattern: flow from the value to the pattern, to add information
-   * syntax (toplevel): `case Dotted.value:`
-   */
-  predicate matchValueFlowStep(Node nodeFrom, Node nodeTo) {
-    exists(MatchValuePattern pattern, Expr value | value = pattern.getValue() |
-      nodeFrom.asExpr() = value and
-      nodeTo.asCfgNode().getNode() = pattern
-    )
-  }
-
-  /**
-   * sequence pattern: each element reads from subject at the associated index
-   * syntax (toplevel): `case [a, b]:`
-   */
-  predicate matchSequenceReadStep(Node nodeFrom, Content c, Node nodeTo) {
-    exists(MatchSequencePattern subject, int index, Pattern element |
-      element = subject.getPattern(index)
-    |
-      nodeFrom.asCfgNode().getNode() = subject and
-      nodeTo.asCfgNode().getNode() = element and
-      (
-        // tuple content
-        c.(TupleElementContent).getIndex() = index
-        or
-        // list content
-        c instanceof ListElementContent
-        // set content is excluded from sequence patterns,
-        // see https://www.python.org/dev/peps/pep-0635/#sequence-patterns
-      )
-    )
-  }
-
-  /**
-   * star pattern: subject flows to the variable, possibly via a conversion
-   * syntax (toplevel): `case *var:`
-   *
-   * We decompose this flow into a read step and a store step. The read step
-   * reads both tuple and list content, the store step only stores list content.
-   * This way, we convert all content to list content.
-   *
-   * This is the read step.
-   */
-  predicate matchStarReadStep(Node nodeFrom, Content c, Node nodeTo) {
-    exists(MatchSequencePattern subject, int index, MatchStarPattern star |
-      star = subject.getPattern(index)
-    |
-      nodeFrom.asCfgNode().getNode() = subject and
-      nodeTo = TStarPatternElementNode(star) and
-      (
-        // tuple content
-        c.(TupleElementContent).getIndex() >= index
-        or
-        // list content
-        c instanceof ListElementContent
-        // set content is excluded from sequence patterns,
-        // see https://www.python.org/dev/peps/pep-0635/#sequence-patterns
-      )
-    )
-  }
-
-  /**
-   * star pattern: subject flows to the variable, possibly via a conversion
-   * syntax (toplevel): `case *var:`
-   *
-   * We decompose this flow into a read step and a store step. The read step
-   * reads both tuple and list content, the store step only stores list content.
-   * This way, we convert all content to list content.
-   *
-   * This is the store step.
-   */
-  predicate matchStarStoreStep(Node nodeFrom, Content c, Node nodeTo) {
-    exists(MatchStarPattern star |
-      nodeFrom = TStarPatternElementNode(star) and
-      nodeTo.asCfgNode().getNode() = star.getTarget() and
-      c instanceof ListElementContent
-    )
-  }
-
-  /**
-   * mapping pattern: each value reads from subject at the associated key
-   * syntax (toplevel): `case {"color": c, "height": x}:`
-   */
-  predicate matchMappingReadStep(Node nodeFrom, Content c, Node nodeTo) {
-    exists(
-      MatchMappingPattern subject, MatchKeyValuePattern keyValue, MatchLiteralPattern key,
-      Pattern value
-    |
-      keyValue = subject.getAMapping() and
-      key = keyValue.getKey() and
-      value = keyValue.getValue()
-    |
-      nodeFrom.asCfgNode().getNode() = subject and
-      nodeTo.asCfgNode().getNode() = value and
-      c.(DictionaryElementContent).getKey() = key.getLiteral().(StrConst).getText()
-    )
-  }
-
-  /**
-   * double star pattern: subject flows to the variable, possibly via a conversion
-   * syntax (toplevel): `case {**var}:`
-   *
-   * Dictionary content flows to the double star, but all mentioned keys in the
-   * mapping pattern should be cleared.
-   */
-  predicate matchMappingFlowStep(Node nodeFrom, Node nodeTo) {
-    exists(MatchMappingPattern subject, MatchDoubleStarPattern dstar |
-      dstar = subject.getAMapping()
-    |
-      nodeFrom.asCfgNode().getNode() = subject and
-      nodeTo.asCfgNode().getNode() = dstar.getTarget()
-    )
-  }
-
-  /**
-   * Bindings that are mentioned in a mapping pattern will not be available
-   * to a double star pattern in the same mapping pattern.
-   */
-  predicate matchMappingClearStep(Node n, Content c) {
-    exists(
-      MatchMappingPattern subject, MatchKeyValuePattern keyValue, MatchLiteralPattern key,
-      MatchDoubleStarPattern dstar
-    |
-      keyValue = subject.getAMapping() and
-      key = keyValue.getKey() and
-      dstar = subject.getAMapping()
-    |
-      n.asCfgNode().getNode() = dstar.getTarget() and
-      c.(DictionaryElementContent).getKey() = key.getLiteral().(StrConst).getText()
-    )
-  }
-
-  /**
-   * class pattern: all keywords read the appropriate attribute from the subject
-   * syntax (toplevel): `case ClassName(attr = val):`
-   */
-  predicate matchClassReadStep(Node nodeFrom, Content c, Node nodeTo) {
-    exists(MatchClassPattern subject, MatchKeywordPattern keyword, Name attr, Pattern value |
-      keyword = subject.getKeyword(_) and
-      attr = keyword.getAttribute() and
-      value = keyword.getValue()
-    |
-      nodeFrom.asCfgNode().getNode() = subject and
-      nodeTo.asCfgNode().getNode() = value and
-      c.(AttributeContent).getAttribute() = attr.getId()
-    )
-  }
-
-  /** All flow steps associated with match. */
-  predicate matchFlowStep(Node nodeFrom, Node nodeTo) {
-    matchSubjectFlowStep(nodeFrom, nodeTo)
-    or
-    matchAsFlowStep(nodeFrom, nodeTo)
-    or
-    matchOrFlowStep(nodeFrom, nodeTo)
-    or
-    matchLiteralFlowStep(nodeFrom, nodeTo)
-    or
-    matchCaptureFlowStep(nodeFrom, nodeTo)
-    or
-    matchValueFlowStep(nodeFrom, nodeTo)
-    or
-    matchMappingFlowStep(nodeFrom, nodeTo)
-  }
-
-  /** All read steps associated with match. */
-  predicate matchReadStep(Node nodeFrom, Content c, Node nodeTo) {
-    matchClassReadStep(nodeFrom, c, nodeTo)
-    or
-    matchSequenceReadStep(nodeFrom, c, nodeTo)
-    or
-    matchMappingReadStep(nodeFrom, c, nodeTo)
-    or
-    matchStarReadStep(nodeFrom, c, nodeTo)
-  }
-
-  /** All store steps associated with match. */
-  predicate matchStoreStep(Node nodeFrom, Content c, Node nodeTo) {
-    matchStarStoreStep(nodeFrom, c, nodeTo)
-  }
-
-  /**
-   * All clear steps associated with match
-   */
-  predicate matchClearStep(Node n, Content c) { matchMappingClearStep(n, c) }
-}
-
-import MatchUnpacking
-
 /** Data flows from a sequence to a call to `pop` on the sequence. */
 predicate popReadStep(CfgNode nodeFrom, Content c, CfgNode nodeTo) {
   // set.pop or list.pop
@@ -1952,27 +1634,17 @@ predicate kwUnpackReadStep(CfgNode nodeFrom, DictionaryElementContent c, Node no
   )
 }
 
-/**
- * Clear content at key `name` of the synthesized dictionary `TKwOverflowNode(call, callable)`,
- * whenever `call` unpacks `name`.
- */
-predicate kwOverflowClearStep(Node n, Content c) {
-  exists(CallNode call, CallableValue callable, string name |
-    call_unpacks(call, _, callable, name, _) and
-    n = TKwOverflowNode(call, callable) and
-    c.(DictionaryElementContent).getKey() = name
-  )
-}
-
 /**
  * Holds if values stored inside content `c` are cleared at node `n`. For example,
  * any value stored inside `f` is cleared at the pre-update node associated with `x`
  * in `x.f = newValue`.
  */
 predicate clearsContent(Node n, Content c) {
-  kwOverflowClearStep(n, c)
-  or
-  matchClearStep(n, c)
+  exists(CallNode call, CallableValue callable, string name |
+    call_unpacks(call, _, callable, name, _) and
+    n = TKwOverflowNode(call, callable) and
+    c.(DictionaryElementContent).getKey() = name
+  )
 }
 
 //--------

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPublic.qll

@@ -25,11 +25,7 @@ newtype TNode =
   /** A node corresponding to an SSA variable. */
   TEssaNode(EssaVariable var) or
   /** A node corresponding to a control flow node. */
-  TCfgNode(ControlFlowNode node) {
-    isExpressionNode(node)
-    or
-    node.getNode() instanceof Pattern
-  } or
+  TCfgNode(ControlFlowNode node) { isExpressionNode(node) } or
   /** A synthetic node representing the value of an object before a state change */
   TSyntheticPreUpdateNode(NeedsSyntheticPreUpdateNode post) or
   /** A synthetic node representing the value of an object after a state change. */
@@ -83,11 +79,7 @@ newtype TNode =
    * A synthetic node representing that there may be an iterable element
    * for `consumer` to consume.
    */
-  TIterableElementNode(UnpackingAssignmentTarget consumer) or
-  /**
-   * A synthetic node representing element content in a star pattern.
-   */
-  TStarPatternElementNode(MatchStarPattern target)
+  TIterableElementNode(UnpackingAssignmentTarget consumer)
 
 /** Helper for `Node::getEnclosingCallable`. */
 private DataFlowCallable getCallableScope(Scope s) {
@@ -484,21 +476,6 @@ class IterableElementNode extends Node, TIterableElementNode {
   override Location getLocation() { result = consumer.getLocation() }
 }
 
-/**
- * A synthetic node representing element content of a star pattern.
- */
-class StarPatternElementNode extends Node, TStarPatternElementNode {
-  CfgNode consumer;
-
-  StarPatternElementNode() { this = TStarPatternElementNode(consumer.getNode().getNode()) }
-
-  override string toString() { result = "StarPatternElement" }
-
-  override DataFlowCallable getEnclosingCallable() { result = consumer.getEnclosingCallable() }
-
-  override Location getLocation() { result = consumer.getLocation() }
-}
-
 /**
  * A node that controls whether other nodes are evaluated.
  */

python/ql/lib/semmle/python/essa/Definitions.qll

@@ -70,10 +70,6 @@ abstract class SsaSourceVariable extends @py_variable {
     SsaSource::exception_capture(this, def)
     or
     SsaSource::with_definition(this, def)
-    or
-    SsaSource::pattern_capture_definition(this, def)
-    or
-    SsaSource::pattern_alias_definition(this, def)
   }
 
   /**

python/ql/lib/semmle/python/essa/Essa.qll

@@ -545,24 +545,6 @@ class WithDefinition extends EssaNodeDefinition {
   override string getRepresentation() { result = "with" }
 }
 
-/** A definition of a variable via a capture pattern */
-class PatternCaptureDefinition extends EssaNodeDefinition {
-  PatternCaptureDefinition() {
-    SsaSource::pattern_capture_definition(this.getSourceVariable(), this.getDefiningNode())
-  }
-
-  override string getRepresentation() { result = "pattern capture" }
-}
-
-/** A definition of a variable via a pattern alias */
-class PatternAliasDefinition extends EssaNodeDefinition {
-  PatternAliasDefinition() {
-    SsaSource::pattern_alias_definition(this.getSourceVariable(), this.getDefiningNode())
-  }
-
-  override string getRepresentation() { result = "pattern alias" }
-}
-
 /** A definition of a variable by declaring it as a parameter */
 class ParameterDefinition extends EssaNodeDefinition {
   ParameterDefinition() {

python/ql/lib/semmle/python/essa/SsaDefinitions.qll

@@ -40,28 +40,6 @@ module SsaSource {
     )
   }
 
-  /** Holds if `v` is defined by a capture pattern. */
-  cached
-  predicate pattern_capture_definition(Variable v, ControlFlowNode defn) {
-    exists(MatchCapturePattern capture, Name var |
-      capture.getVariable() = var and
-      var.getAFlowNode() = defn
-    |
-      var = v.getAStore()
-    )
-  }
-
-  /** Holds if `v` is defined by as the alias of an as-pattern. */
-  cached
-  predicate pattern_alias_definition(Variable v, ControlFlowNode defn) {
-    exists(MatchAsPattern pattern, Name var |
-      pattern.getAlias() = var and
-      var.getAFlowNode() = defn
-    |
-      var = v.getAStore()
-    )
-  }
-
   /** Holds if `v` is defined by multiple assignment at `defn`. */
   cached
   predicate multi_assignment_definition(Variable v, ControlFlowNode defn, int n, SequenceNode lhs) {

python/ql/lib/semmlecode.python.dbscheme

@@ -231,11 +231,6 @@ py_extracted_version(int module : @py_Module ref,
 /* <Field> Call.positional_args = 3, expr_list */
 /* <Field> Call.named_args = 4, dict_item_list */
 
-/* <Field> Case.location = 0, location */
-/* <Field> Case.pattern = 1, pattern */
-/* <Field> Case.guard = 2, expr */
-/* <Field> Case.body = 3, stmt_list */
-
 /* <Field> Class.name = 0, str */
 /* <Field> Class.body = 1, stmt_list */
 /* <Parent> Class = ClassExpr  */
@@ -328,10 +323,6 @@ py_extracted_version(int module : @py_Module ref,
 /* <Field> Global.location = 0, location */
 /* <Field> Global.names = 1, str_list */
 
-/* <Field> Guard.location = 0, location */
-/* <Field> Guard.parenthesised = 1, bool */
-/* <Field> Guard.test = 2, expr */
-
 /* <Field> If.location = 0, location */
 /* <Field> If.test = 1, expr */
 /* <Field> If.body = 2, stmt_list */
@@ -386,67 +377,6 @@ py_extracted_version(int module : @py_Module ref,
 /* <Field> ListComp.generators = 4, comprehension_list */
 /* <Field> ListComp.elt = 5, expr */
 
-/* <Field> MatchStmt.location = 0, location */
-/* <Field> MatchStmt.subject = 1, expr */
-/* <Field> MatchStmt.cases = 2, stmt_list */
-
-/* <Field> MatchAsPattern.location = 0, location */
-/* <Field> MatchAsPattern.parenthesised = 1, bool */
-/* <Field> MatchAsPattern.pattern = 2, pattern */
-/* <Field> MatchAsPattern.alias = 3, expr */
-
-/* <Field> MatchCapturePattern.location = 0, location */
-/* <Field> MatchCapturePattern.parenthesised = 1, bool */
-/* <Field> MatchCapturePattern.variable = 2, expr */
-
-/* <Field> MatchClassPattern.location = 0, location */
-/* <Field> MatchClassPattern.parenthesised = 1, bool */
-/* <Field> MatchClassPattern.class = 2, expr */
-/* <Field> MatchClassPattern.class_name = 3, expr */
-/* <Field> MatchClassPattern.positional = 4, pattern_list */
-/* <Field> MatchClassPattern.keyword = 5, pattern_list */
-
-/* <Field> MatchDoubleStarPattern.location = 0, location */
-/* <Field> MatchDoubleStarPattern.parenthesised = 1, bool */
-/* <Field> MatchDoubleStarPattern.target = 2, pattern */
-
-/* <Field> MatchKeyValuePattern.location = 0, location */
-/* <Field> MatchKeyValuePattern.parenthesised = 1, bool */
-/* <Field> MatchKeyValuePattern.key = 2, pattern */
-/* <Field> MatchKeyValuePattern.value = 3, pattern */
-
-/* <Field> MatchKeywordPattern.location = 0, location */
-/* <Field> MatchKeywordPattern.parenthesised = 1, bool */
-/* <Field> MatchKeywordPattern.attribute = 2, expr */
-/* <Field> MatchKeywordPattern.value = 3, pattern */
-
-/* <Field> MatchLiteralPattern.location = 0, location */
-/* <Field> MatchLiteralPattern.parenthesised = 1, bool */
-/* <Field> MatchLiteralPattern.literal = 2, expr */
-
-/* <Field> MatchMappingPattern.location = 0, location */
-/* <Field> MatchMappingPattern.parenthesised = 1, bool */
-/* <Field> MatchMappingPattern.mappings = 2, pattern_list */
-
-/* <Field> MatchOrPattern.location = 0, location */
-/* <Field> MatchOrPattern.parenthesised = 1, bool */
-/* <Field> MatchOrPattern.patterns = 2, pattern_list */
-
-/* <Field> MatchSequencePattern.location = 0, location */
-/* <Field> MatchSequencePattern.parenthesised = 1, bool */
-/* <Field> MatchSequencePattern.patterns = 2, pattern_list */
-
-/* <Field> MatchStarPattern.location = 0, location */
-/* <Field> MatchStarPattern.parenthesised = 1, bool */
-/* <Field> MatchStarPattern.target = 2, pattern */
-
-/* <Field> MatchValuePattern.location = 0, location */
-/* <Field> MatchValuePattern.parenthesised = 1, bool */
-/* <Field> MatchValuePattern.value = 2, expr */
-
-/* <Field> MatchWildcardPattern.location = 0, location */
-/* <Field> MatchWildcardPattern.parenthesised = 1, bool */
-
 /* <Field> Module.name = 0, str */
 /* <Field> Module.hash = 1, str */
 /* <Field> Module.body = 2, stmt_list */
@@ -621,11 +551,6 @@ py_extracted_version(int module : @py_Module ref,
 /* <Parent> Operator = BinaryExpr  */
 /* <Parent> ParameterList = Function  */
 
-/* <Field> Pattern.location = 0, location */
-/* <Field> Pattern.parenthesised = 1, bool */
-/* <Parent> Pattern = PatternParent  */
-/* <Parent> PatternList = PatternListParent  */
-
 /* <Field> Stmt.location = 0, location */
 /* <Parent> Stmt = StmtList  */
 /* <Parent> StmtList = StmtListParent  */
@@ -722,15 +647,6 @@ py_operators(unique int id : @py_operator,
 py_parameter_lists(unique int id : @py_parameter_list,
     unique int parent : @py_Function ref);
 
-py_patterns(unique int id : @py_pattern,
-    int kind: int ref,
-    int parent : @py_pattern_parent ref,
-    int idx : int ref);
-
-py_pattern_lists(unique int id : @py_pattern_list,
-    int parent : @py_pattern_list_parent ref,
-    int idx : int ref);
-
 py_stmts(unique int id : @py_stmt,
     int kind: int ref,
     int parent : @py_stmt_list ref,
@@ -794,28 +710,27 @@ case @py_expr.kind of
 |   15 = @py_Lambda
 |   16 = @py_List
 |   17 = @py_ListComp
-|   18 = @py_Guard
-|   19 = @py_Name
-|   20 = @py_Num
-|   21 = @py_Repr
-|   22 = @py_Set
-|   23 = @py_SetComp
-|   24 = @py_Slice
-|   25 = @py_Starred
-|   26 = @py_Str
-|   27 = @py_Subscript
-|   28 = @py_Tuple
-|   29 = @py_UnaryExpr
-|   30 = @py_Yield
-|   31 = @py_YieldFrom
-|   32 = @py_TemplateDottedNotation
-|   33 = @py_Filter
-|   34 = @py_PlaceHolder
-|   35 = @py_Await
-|   36 = @py_Fstring
-|   37 = @py_FormattedValue
-|   38 = @py_AssignExpr
-|   39 = @py_SpecialOperation;
+|   18 = @py_Name
+|   19 = @py_Num
+|   20 = @py_Repr
+|   21 = @py_Set
+|   22 = @py_SetComp
+|   23 = @py_Slice
+|   24 = @py_Starred
+|   25 = @py_Str
+|   26 = @py_Subscript
+|   27 = @py_Tuple
+|   28 = @py_UnaryExpr
+|   29 = @py_Yield
+|   30 = @py_YieldFrom
+|   31 = @py_TemplateDottedNotation
+|   32 = @py_Filter
+|   33 = @py_PlaceHolder
+|   34 = @py_Await
+|   35 = @py_Fstring
+|   36 = @py_FormattedValue
+|   37 = @py_AssignExpr
+|   38 = @py_SpecialOperation;
 
 case @py_expr_context.kind of
     0 = @py_AugLoad
@@ -840,21 +755,6 @@ case @py_operator.kind of
 |   11 = @py_Sub
 |   12 = @py_MatMult;
 
-case @py_pattern.kind of
-    0 = @py_MatchAsPattern
-|   1 = @py_MatchOrPattern
-|   2 = @py_MatchLiteralPattern
-|   3 = @py_MatchCapturePattern
-|   4 = @py_MatchWildcardPattern
-|   5 = @py_MatchValuePattern
-|   6 = @py_MatchSequencePattern
-|   7 = @py_MatchStarPattern
-|   8 = @py_MatchMappingPattern
-|   9 = @py_MatchDoubleStarPattern
-|   10 = @py_MatchKeyValuePattern
-|   11 = @py_MatchClassPattern
-|   12 = @py_MatchKeywordPattern;
-
 case @py_stmt.kind of
     0 = @py_Assert
 |   1 = @py_Assign
@@ -870,18 +770,16 @@ case @py_stmt.kind of
 |   11 = @py_If
 |   12 = @py_Import
 |   13 = @py_ImportStar
-|   14 = @py_MatchStmt
-|   15 = @py_Case
-|   16 = @py_Nonlocal
-|   17 = @py_Pass
-|   18 = @py_Print
-|   19 = @py_Raise
-|   20 = @py_Return
-|   21 = @py_Try
-|   22 = @py_While
-|   23 = @py_With
-|   24 = @py_TemplateWrite
-|   25 = @py_AnnAssign;
+|   14 = @py_Nonlocal
+|   15 = @py_Pass
+|   16 = @py_Print
+|   17 = @py_Raise
+|   18 = @py_Return
+|   19 = @py_Try
+|   20 = @py_While
+|   21 = @py_With
+|   22 = @py_TemplateWrite
+|   23 = @py_AnnAssign;
 
 case @py_unaryop.kind of
     0 = @py_Invert
@@ -895,9 +793,9 @@ case @py_unaryop.kind of
 
 @py_arguments_parent = @py_FunctionExpr | @py_Lambda;
 
-@py_ast_node = @py_Class | @py_Function | @py_Module | @py_StringPart | @py_comprehension | @py_dict_item | @py_expr | @py_pattern | @py_stmt;
+@py_ast_node = @py_Class | @py_Function | @py_Module | @py_StringPart | @py_comprehension | @py_dict_item | @py_expr | @py_stmt;
 
-@py_bool_parent = @py_For | @py_Function | @py_Print | @py_With | @py_expr | @py_pattern;
+@py_bool_parent = @py_For | @py_Function | @py_Print | @py_With | @py_expr;
 
 @py_dict_item_list_parent = @py_Call | @py_ClassExpr | @py_Dict;
 
@@ -907,19 +805,15 @@ case @py_unaryop.kind of
 
 @py_expr_or_stmt = @py_expr | @py_stmt;
 
-@py_expr_parent = @py_AnnAssign | @py_Assert | @py_Assign | @py_AssignExpr | @py_Attribute | @py_AugAssign | @py_Await | @py_BinaryExpr | @py_Call | @py_Case | @py_Compare | @py_DictComp | @py_DictUnpacking | @py_ExceptStmt | @py_Exec | @py_Expr_stmt | @py_Filter | @py_For | @py_FormattedValue | @py_Function | @py_FunctionExpr | @py_GeneratorExp | @py_Guard | @py_If | @py_IfExp | @py_ImportMember | @py_ImportStar | @py_KeyValuePair | @py_ListComp | @py_MatchAsPattern | @py_MatchCapturePattern | @py_MatchClassPattern | @py_MatchKeywordPattern | @py_MatchLiteralPattern | @py_MatchStmt | @py_MatchValuePattern | @py_Print | @py_Raise | @py_Repr | @py_Return | @py_SetComp | @py_Slice | @py_Starred | @py_Subscript | @py_TemplateDottedNotation | @py_TemplateWrite | @py_UnaryExpr | @py_While | @py_With | @py_Yield | @py_YieldFrom | @py_alias | @py_arguments | @py_comprehension | @py_expr_list | @py_keyword | @py_parameter_list;
+@py_expr_parent = @py_AnnAssign | @py_Assert | @py_Assign | @py_AssignExpr | @py_Attribute | @py_AugAssign | @py_Await | @py_BinaryExpr | @py_Call | @py_Compare | @py_DictComp | @py_DictUnpacking | @py_ExceptStmt | @py_Exec | @py_Expr_stmt | @py_Filter | @py_For | @py_FormattedValue | @py_Function | @py_FunctionExpr | @py_GeneratorExp | @py_If | @py_IfExp | @py_ImportMember | @py_ImportStar | @py_KeyValuePair | @py_ListComp | @py_Print | @py_Raise | @py_Repr | @py_Return | @py_SetComp | @py_Slice | @py_Starred | @py_Subscript | @py_TemplateDottedNotation | @py_TemplateWrite | @py_UnaryExpr | @py_While | @py_With | @py_Yield | @py_YieldFrom | @py_alias | @py_arguments | @py_comprehension | @py_expr_list | @py_keyword | @py_parameter_list;
 
-@py_location_parent = @py_DictUnpacking | @py_KeyValuePair | @py_StringPart | @py_comprehension | @py_expr | @py_keyword | @py_pattern | @py_stmt;
+@py_location_parent = @py_DictUnpacking | @py_KeyValuePair | @py_StringPart | @py_comprehension | @py_expr | @py_keyword | @py_stmt;
 
 @py_parameter = @py_Name | @py_Tuple;
 
-@py_pattern_list_parent = @py_MatchClassPattern | @py_MatchMappingPattern | @py_MatchOrPattern | @py_MatchSequencePattern;
-
-@py_pattern_parent = @py_Case | @py_MatchAsPattern | @py_MatchDoubleStarPattern | @py_MatchKeyValuePattern | @py_MatchKeywordPattern | @py_MatchStarPattern | @py_pattern_list;
-
 @py_scope = @py_Class | @py_Function | @py_Module;
 
-@py_stmt_list_parent = @py_Case | @py_Class | @py_ExceptStmt | @py_For | @py_Function | @py_If | @py_MatchStmt | @py_Module | @py_Try | @py_While | @py_With;
+@py_stmt_list_parent = @py_Class | @py_ExceptStmt | @py_For | @py_Function | @py_If | @py_Module | @py_Try | @py_While | @py_With;
 
 @py_str_list_parent = @py_Global | @py_Nonlocal;
 

python/ql/lib/semmlecode.python.dbscheme.stats

@@ -1,22 +1,5 @@
 <dbstats>
   <typesizes><e>
-<k>@py_Guard</k><v>100</v></e><e>
-<k>@py_MatchAsPattern</k><v>100</v></e><e>
-<k>@py_MatchOrPattern</k><v>100</v></e><e>
-<k>@py_MatchLiteralPattern</k><v>100</v></e><e>
-<k>@py_MatchCapturePattern</k><v>100</v></e><e>
-<k>@py_MatchWildcardPattern</k><v>100</v></e><e>
-<k>@py_MatchValuePattern</k><v>100</v></e><e>
-<k>@py_MatchSequencePattern</k><v>100</v></e><e>
-<k>@py_MatchStarPattern</k><v>100</v></e><e>
-<k>@py_MatchMappingPattern</k><v>100</v></e><e>
-<k>@py_MatchDoubleStarPattern</k><v>100</v></e><e>
-<k>@py_MatchKeyValuePattern</k><v>100</v></e><e>
-<k>@py_MatchClassPattern</k><v>100</v></e><e>
-<k>@py_MatchKeywordPattern</k><v>100</v></e><e>
-<k>@py_Case</k><v>100</v></e><e>
-<k>@py_MatchStmt</k><v>100</v></e><e>
-<k>@py_pattern_list</k><v>100</v></e><e>
 <k>@externalDefect</k>
 <v>100</v>
 </e>
@@ -7497,48 +7480,6 @@
 <dependencies/>
 </relation>
 <relation>
-<name>py_patterns</name>
-<cardinality>1000</cardinality>
-<columnsizes>
-<e>
-<k>id</k>
-<v>1000</v>
-</e>
-<e>
-<k>kind</k>
-<v>13</v>
-</e>
-<e>
-<k>parent</k>
-<v>1000</v>
-</e>
-<e>
-<k>idx</k>
-<v>100</v>
-</e>
-</columnsizes>
-<dependencies/>
-</relation>
-<relation>
-<name>py_pattern_lists</name>
-<cardinality>1000</cardinality>
-<columnsizes>
-<e>
-<k>id</k>
-<v>1000</v>
-</e>
-<e>
-<k>parent</k>
-<v>1000</v>
-</e>
-<e>
-<k>idx</k>
-<v>100</v>
-</e>
-</columnsizes>
-<dependencies/>
-</relation>
-<relation>
 <name>py_extracted_version</name>
 <cardinality>3337</cardinality>
 <columnsizes>

python/ql/lib/upgrades/ff5327c074ca235c6b2fa4ea9ed29f4c60aa46cb/old.dbscheme

@@ -1,994 +0,0 @@
-/*
- *      This dbscheme is auto-generated by 'semmle/dbscheme_gen.py'.
- *      WARNING: Any modifications to this file will be lost.
- *      Relations can be changed by modifying master.py or
- *      by adding rules to dbscheme.template
- */
-
-/* This is a dummy line to alter the dbscheme, so we can make a database upgrade
- * without actually changing any of the dbscheme predicates. It contains a date
- * to allow for such updates in the future as well.
- *
- * 2020-07-02
- *
- * DO NOT remove this comment carelessly, since it can revert the dbscheme back to a
- * previously seen state (matching a previously seen SHA), which would make the upgrade
- * mechanism not work properly.
- */
-
- /*
- * External artifacts
- */
-
-externalDefects(
-  unique int id : @externalDefect,
-  varchar(900) queryPath : string ref,
-  int location : @location ref,
-  varchar(900) message : string ref,
-  float severity : float ref
-);
-
-externalMetrics(
-  unique int id : @externalMetric,
-  varchar(900) queryPath : string ref,
-  int location : @location ref,
-  float value : float ref
-);
-
-externalData(
-  int id : @externalDataElement,
-  varchar(900) queryPath : string ref,
-  int column: int ref,
-  varchar(900) data : string ref
-);
-
-snapshotDate(unique date snapshotDate : date ref);
-
-sourceLocationPrefix(varchar(900) prefix : string ref);
-
-
-/*
- * Duplicate code
- */
-
-duplicateCode(
-  unique int id : @duplication,
-  varchar(900) relativePath : string ref,
-  int equivClass : int ref);
-
-similarCode(
-  unique int id : @similarity,
-  varchar(900) relativePath : string ref,
-  int equivClass : int ref);
-
-@duplication_or_similarity = @duplication | @similarity
-
-tokens(
-  int id : @duplication_or_similarity ref,
-  int offset : int ref,
-  int beginLine : int ref,
-  int beginColumn : int ref,
-  int endLine : int ref,
-  int endColumn : int ref);
-
-/*
- * Line metrics
- */
-py_codelines(int id : @py_scope ref,
-             int count : int ref);
-
-py_commentlines(int id : @py_scope ref,
-                int count : int ref);
-
-py_docstringlines(int id : @py_scope ref,
-                int count : int ref);
-
-py_alllines(int id : @py_scope ref,
-                int count : int ref);
-
-/*
- * Version history
- */
-
-svnentries(
-  int id : @svnentry,
-  varchar(500) revision : string ref,
-  varchar(500) author : string ref,
-  date revisionDate : date ref,
-  int changeSize : int ref
-)
-
-svnaffectedfiles(
-  int id : @svnentry ref,
-  int file : @file ref,
-  varchar(500) action : string ref
-)
-
-svnentrymsg(
-  int id : @svnentry ref,
-  varchar(500) message : string ref
-)
-
-svnchurn(
-  int commit : @svnentry ref,
-  int file : @file ref,
-  int addedLines : int ref,
-  int deletedLines : int ref
-)
-
-/****************************
-        Python dbscheme
-****************************/
-
-files(unique int id: @file,
-      varchar(900) name: string ref);
-
-folders(unique int id: @folder,
-        varchar(900) name: string ref);
-
-@container = @folder | @file;
-
-containerparent(int parent: @container ref,
-                unique int child: @container ref);
-
-@sourceline = @file | @py_Module | @xmllocatable;
-
-numlines(int element_id: @sourceline ref,
-         int num_lines: int ref,
-         int num_code: int ref,
-         int num_comment: int ref
-        );
-
-@location = @location_ast | @location_default ;
-
-locations_default(unique int id: @location_default,
-          int file: @file ref,
-          int beginLine: int ref,
-          int beginColumn: int ref,
-          int endLine: int ref,
-          int endColumn: int ref);
-
-locations_ast(unique int id: @location_ast,
-          int module: @py_Module ref,
-          int beginLine: int ref,
-          int beginColumn: int ref,
-          int endLine: int ref,
-          int endColumn: int ref);
-
-file_contents(unique int file: @file ref, string contents: string ref);
-
-py_module_path(int module: @py_Module ref, int file: @container ref);
-
-variable(unique int id : @py_variable,
-         int scope : @py_scope ref,
-         varchar(1) name : string ref);
-
-py_line_lengths(unique int id : @py_line,
-                int file: @py_Module ref,
-                int line : int ref,
-                int length : int ref);
-
-py_extracted_version(int module : @py_Module ref,
-                     varchar(1) version : string ref);
-
-/* AUTO GENERATED PART STARTS HERE */
-
-
-/* <Field> AnnAssign.location = 0, location */
-/* <Field> AnnAssign.value = 1, expr */
-/* <Field> AnnAssign.annotation = 2, expr */
-/* <Field> AnnAssign.target = 3, expr */
-
-/* <Field> Assert.location = 0, location */
-/* <Field> Assert.test = 1, expr */
-/* <Field> Assert.msg = 2, expr */
-
-/* <Field> Assign.location = 0, location */
-/* <Field> Assign.value = 1, expr */
-/* <Field> Assign.targets = 2, expr_list */
-
-/* <Field> AssignExpr.location = 0, location */
-/* <Field> AssignExpr.parenthesised = 1, bool */
-/* <Field> AssignExpr.value = 2, expr */
-/* <Field> AssignExpr.target = 3, expr */
-
-/* <Field> Attribute.location = 0, location */
-/* <Field> Attribute.parenthesised = 1, bool */
-/* <Field> Attribute.value = 2, expr */
-/* <Field> Attribute.attr = 3, str */
-/* <Field> Attribute.ctx = 4, expr_context */
-
-/* <Field> AugAssign.location = 0, location */
-/* <Field> AugAssign.operation = 1, BinOp */
-
-/* <Field> Await.location = 0, location */
-/* <Field> Await.parenthesised = 1, bool */
-/* <Field> Await.value = 2, expr */
-
-/* <Field> BinaryExpr.location = 0, location */
-/* <Field> BinaryExpr.parenthesised = 1, bool */
-/* <Field> BinaryExpr.left = 2, expr */
-/* <Field> BinaryExpr.op = 3, operator */
-/* <Field> BinaryExpr.right = 4, expr */
-/* <Parent> BinaryExpr = AugAssign  */
-
-/* <Field> BoolExpr.location = 0, location */
-/* <Field> BoolExpr.parenthesised = 1, bool */
-/* <Field> BoolExpr.op = 2, boolop */
-/* <Field> BoolExpr.values = 3, expr_list */
-
-/* <Field> Break.location = 0, location */
-
-/* <Field> Bytes.location = 0, location */
-/* <Field> Bytes.parenthesised = 1, bool */
-/* <Field> Bytes.s = 2, bytes */
-/* <Field> Bytes.prefix = 3, bytes */
-/* <Field> Bytes.implicitly_concatenated_parts = 4, StringPart_list */
-
-/* <Field> Call.location = 0, location */
-/* <Field> Call.parenthesised = 1, bool */
-/* <Field> Call.func = 2, expr */
-/* <Field> Call.positional_args = 3, expr_list */
-/* <Field> Call.named_args = 4, dict_item_list */
-
-/* <Field> Class.name = 0, str */
-/* <Field> Class.body = 1, stmt_list */
-/* <Parent> Class = ClassExpr  */
-
-/* <Field> ClassExpr.location = 0, location */
-/* <Field> ClassExpr.parenthesised = 1, bool */
-/* <Field> ClassExpr.name = 2, str */
-/* <Field> ClassExpr.bases = 3, expr_list */
-/* <Field> ClassExpr.keywords = 4, dict_item_list */
-/* <Field> ClassExpr.inner_scope = 5, Class */
-
-/* <Field> Compare.location = 0, location */
-/* <Field> Compare.parenthesised = 1, bool */
-/* <Field> Compare.left = 2, expr */
-/* <Field> Compare.ops = 3, cmpop_list */
-/* <Field> Compare.comparators = 4, expr_list */
-
-/* <Field> Continue.location = 0, location */
-
-/* <Field> Delete.location = 0, location */
-/* <Field> Delete.targets = 1, expr_list */
-
-/* <Field> Dict.location = 0, location */
-/* <Field> Dict.parenthesised = 1, bool */
-/* <Field> Dict.items = 2, dict_item_list */
-
-/* <Field> DictComp.location = 0, location */
-/* <Field> DictComp.parenthesised = 1, bool */
-/* <Field> DictComp.function = 2, Function */
-/* <Field> DictComp.iterable = 3, expr */
-
-/* <Field> DictUnpacking.location = 0, location */
-/* <Field> DictUnpacking.value = 1, expr */
-
-/* <Field> Ellipsis.location = 0, location */
-/* <Field> Ellipsis.parenthesised = 1, bool */
-
-/* <Field> ExceptStmt.location = 0, location */
-/* <Field> ExceptStmt.type = 1, expr */
-/* <Field> ExceptStmt.name = 2, expr */
-/* <Field> ExceptStmt.body = 3, stmt_list */
-
-/* <Field> Exec.location = 0, location */
-/* <Field> Exec.body = 1, expr */
-/* <Field> Exec.globals = 2, expr */
-/* <Field> Exec.locals = 3, expr */
-
-/* <Field> ExprStmt.location = 0, location */
-/* <Field> ExprStmt.value = 1, expr */
-
-/* <Field> Filter.location = 0, location */
-/* <Field> Filter.parenthesised = 1, bool */
-/* <Field> Filter.value = 2, expr */
-/* <Field> Filter.filter = 3, expr */
-
-/* <Field> For.location = 0, location */
-/* <Field> For.target = 1, expr */
-/* <Field> For.iter = 2, expr */
-/* <Field> For.body = 3, stmt_list */
-/* <Field> For.orelse = 4, stmt_list */
-/* <Field> For.is_async = 5, bool */
-
-/* <Field> FormattedValue.location = 0, location */
-/* <Field> FormattedValue.parenthesised = 1, bool */
-/* <Field> FormattedValue.value = 2, expr */
-/* <Field> FormattedValue.conversion = 3, str */
-/* <Field> FormattedValue.format_spec = 4, JoinedStr */
-
-/* <Field> Function.name = 0, str */
-/* <Field> Function.args = 1, parameter_list */
-/* <Field> Function.vararg = 2, expr */
-/* <Field> Function.kwonlyargs = 3, expr_list */
-/* <Field> Function.kwarg = 4, expr */
-/* <Field> Function.body = 5, stmt_list */
-/* <Field> Function.is_async = 6, bool */
-/* <Parent> Function = FunctionParent  */
-
-/* <Field> FunctionExpr.location = 0, location */
-/* <Field> FunctionExpr.parenthesised = 1, bool */
-/* <Field> FunctionExpr.name = 2, str */
-/* <Field> FunctionExpr.args = 3, arguments */
-/* <Field> FunctionExpr.returns = 4, expr */
-/* <Field> FunctionExpr.inner_scope = 5, Function */
-
-/* <Field> GeneratorExp.location = 0, location */
-/* <Field> GeneratorExp.parenthesised = 1, bool */
-/* <Field> GeneratorExp.function = 2, Function */
-/* <Field> GeneratorExp.iterable = 3, expr */
-
-/* <Field> Global.location = 0, location */
-/* <Field> Global.names = 1, str_list */
-
-/* <Field> If.location = 0, location */
-/* <Field> If.test = 1, expr */
-/* <Field> If.body = 2, stmt_list */
-/* <Field> If.orelse = 3, stmt_list */
-
-/* <Field> IfExp.location = 0, location */
-/* <Field> IfExp.parenthesised = 1, bool */
-/* <Field> IfExp.test = 2, expr */
-/* <Field> IfExp.body = 3, expr */
-/* <Field> IfExp.orelse = 4, expr */
-
-/* <Field> Import.location = 0, location */
-/* <Field> Import.names = 1, alias_list */
-
-/* <Field> ImportExpr.location = 0, location */
-/* <Field> ImportExpr.parenthesised = 1, bool */
-/* <Field> ImportExpr.level = 2, int */
-/* <Field> ImportExpr.name = 3, str */
-/* <Field> ImportExpr.top = 4, bool */
-
-/* <Field> ImportStar.location = 0, location */
-/* <Field> ImportStar.module = 1, expr */
-
-/* <Field> ImportMember.location = 0, location */
-/* <Field> ImportMember.parenthesised = 1, bool */
-/* <Field> ImportMember.module = 2, expr */
-/* <Field> ImportMember.name = 3, str */
-
-/* <Field> Fstring.location = 0, location */
-/* <Field> Fstring.parenthesised = 1, bool */
-/* <Field> Fstring.values = 2, expr_list */
-/* <Parent> Fstring = FormattedValue  */
-
-/* <Field> KeyValuePair.location = 0, location */
-/* <Field> KeyValuePair.value = 1, expr */
-/* <Field> KeyValuePair.key = 2, expr */
-
-/* <Field> Lambda.location = 0, location */
-/* <Field> Lambda.parenthesised = 1, bool */
-/* <Field> Lambda.args = 2, arguments */
-/* <Field> Lambda.inner_scope = 3, Function */
-
-/* <Field> List.location = 0, location */
-/* <Field> List.parenthesised = 1, bool */
-/* <Field> List.elts = 2, expr_list */
-/* <Field> List.ctx = 3, expr_context */
-
-/* <Field> ListComp.location = 0, location */
-/* <Field> ListComp.parenthesised = 1, bool */
-/* <Field> ListComp.function = 2, Function */
-/* <Field> ListComp.iterable = 3, expr */
-/* <Field> ListComp.generators = 4, comprehension_list */
-/* <Field> ListComp.elt = 5, expr */
-
-/* <Field> Module.name = 0, str */
-/* <Field> Module.hash = 1, str */
-/* <Field> Module.body = 2, stmt_list */
-/* <Field> Module.kind = 3, str */
-
-/* <Field> Name.location = 0, location */
-/* <Field> Name.parenthesised = 1, bool */
-/* <Field> Name.variable = 2, variable */
-/* <Field> Name.ctx = 3, expr_context */
-/* <Parent> Name = ParameterList  */
-
-/* <Field> Nonlocal.location = 0, location */
-/* <Field> Nonlocal.names = 1, str_list */
-
-/* <Field> Num.location = 0, location */
-/* <Field> Num.parenthesised = 1, bool */
-/* <Field> Num.n = 2, number */
-/* <Field> Num.text = 3, number */
-
-/* <Field> Pass.location = 0, location */
-
-/* <Field> PlaceHolder.location = 0, location */
-/* <Field> PlaceHolder.parenthesised = 1, bool */
-/* <Field> PlaceHolder.variable = 2, variable */
-/* <Field> PlaceHolder.ctx = 3, expr_context */
-
-/* <Field> Print.location = 0, location */
-/* <Field> Print.dest = 1, expr */
-/* <Field> Print.values = 2, expr_list */
-/* <Field> Print.nl = 3, bool */
-
-/* <Field> Raise.location = 0, location */
-/* <Field> Raise.exc = 1, expr */
-/* <Field> Raise.cause = 2, expr */
-/* <Field> Raise.type = 3, expr */
-/* <Field> Raise.inst = 4, expr */
-/* <Field> Raise.tback = 5, expr */
-
-/* <Field> Repr.location = 0, location */
-/* <Field> Repr.parenthesised = 1, bool */
-/* <Field> Repr.value = 2, expr */
-
-/* <Field> Return.location = 0, location */
-/* <Field> Return.value = 1, expr */
-
-/* <Field> Set.location = 0, location */
-/* <Field> Set.parenthesised = 1, bool */
-/* <Field> Set.elts = 2, expr_list */
-
-/* <Field> SetComp.location = 0, location */
-/* <Field> SetComp.parenthesised = 1, bool */
-/* <Field> SetComp.function = 2, Function */
-/* <Field> SetComp.iterable = 3, expr */
-
-/* <Field> Slice.location = 0, location */
-/* <Field> Slice.parenthesised = 1, bool */
-/* <Field> Slice.start = 2, expr */
-/* <Field> Slice.stop = 3, expr */
-/* <Field> Slice.step = 4, expr */
-
-/* <Field> SpecialOperation.location = 0, location */
-/* <Field> SpecialOperation.parenthesised = 1, bool */
-/* <Field> SpecialOperation.name = 2, str */
-/* <Field> SpecialOperation.arguments = 3, expr_list */
-
-/* <Field> Starred.location = 0, location */
-/* <Field> Starred.parenthesised = 1, bool */
-/* <Field> Starred.value = 2, expr */
-/* <Field> Starred.ctx = 3, expr_context */
-
-/* <Field> Str.location = 0, location */
-/* <Field> Str.parenthesised = 1, bool */
-/* <Field> Str.s = 2, str */
-/* <Field> Str.prefix = 3, str */
-/* <Field> Str.implicitly_concatenated_parts = 4, StringPart_list */
-
-/* <Field> StringPart.text = 0, str */
-/* <Field> StringPart.location = 1, location */
-/* <Parent> StringPart = StringPartList  */
-/* <Parent> StringPartList = BytesOrStr  */
-
-/* <Field> Subscript.location = 0, location */
-/* <Field> Subscript.parenthesised = 1, bool */
-/* <Field> Subscript.value = 2, expr */
-/* <Field> Subscript.index = 3, expr */
-/* <Field> Subscript.ctx = 4, expr_context */
-
-/* <Field> TemplateDottedNotation.location = 0, location */
-/* <Field> TemplateDottedNotation.parenthesised = 1, bool */
-/* <Field> TemplateDottedNotation.value = 2, expr */
-/* <Field> TemplateDottedNotation.attr = 3, str */
-/* <Field> TemplateDottedNotation.ctx = 4, expr_context */
-
-/* <Field> TemplateWrite.location = 0, location */
-/* <Field> TemplateWrite.value = 1, expr */
-
-/* <Field> Try.location = 0, location */
-/* <Field> Try.body = 1, stmt_list */
-/* <Field> Try.orelse = 2, stmt_list */
-/* <Field> Try.handlers = 3, stmt_list */
-/* <Field> Try.finalbody = 4, stmt_list */
-
-/* <Field> Tuple.location = 0, location */
-/* <Field> Tuple.parenthesised = 1, bool */
-/* <Field> Tuple.elts = 2, expr_list */
-/* <Field> Tuple.ctx = 3, expr_context */
-/* <Parent> Tuple = ParameterList  */
-
-/* <Field> UnaryExpr.location = 0, location */
-/* <Field> UnaryExpr.parenthesised = 1, bool */
-/* <Field> UnaryExpr.op = 2, unaryop */
-/* <Field> UnaryExpr.operand = 3, expr */
-
-/* <Field> While.location = 0, location */
-/* <Field> While.test = 1, expr */
-/* <Field> While.body = 2, stmt_list */
-/* <Field> While.orelse = 3, stmt_list */
-
-/* <Field> With.location = 0, location */
-/* <Field> With.context_expr = 1, expr */
-/* <Field> With.optional_vars = 2, expr */
-/* <Field> With.body = 3, stmt_list */
-/* <Field> With.is_async = 4, bool */
-
-/* <Field> Yield.location = 0, location */
-/* <Field> Yield.parenthesised = 1, bool */
-/* <Field> Yield.value = 2, expr */
-
-/* <Field> YieldFrom.location = 0, location */
-/* <Field> YieldFrom.parenthesised = 1, bool */
-/* <Field> YieldFrom.value = 2, expr */
-
-/* <Field> Alias.value = 0, expr */
-/* <Field> Alias.asname = 1, expr */
-/* <Parent> Alias = AliasList  */
-/* <Parent> AliasList = Import  */
-
-/* <Field> Arguments.kw_defaults = 0, expr_list */
-/* <Field> Arguments.defaults = 1, expr_list */
-/* <Field> Arguments.annotations = 2, expr_list */
-/* <Field> Arguments.varargannotation = 3, expr */
-/* <Field> Arguments.kwargannotation = 4, expr */
-/* <Field> Arguments.kw_annotations = 5, expr_list */
-/* <Parent> Arguments = ArgumentsParent  */
-/* <Parent> boolean = BoolParent  */
-/* <Parent> Boolop = BoolExpr  */
-/* <Parent> string = Bytes  */
-/* <Parent> Cmpop = CmpopList  */
-/* <Parent> CmpopList = Compare  */
-
-/* <Field> Comprehension.location = 0, location */
-/* <Field> Comprehension.iter = 1, expr */
-/* <Field> Comprehension.target = 2, expr */
-/* <Field> Comprehension.ifs = 3, expr_list */
-/* <Parent> Comprehension = ComprehensionList  */
-/* <Parent> ComprehensionList = ListComp  */
-/* <Parent> DictItem = DictItemList  */
-/* <Parent> DictItemList = DictItemListParent  */
-
-/* <Field> Expr.location = 0, location */
-/* <Field> Expr.parenthesised = 1, bool */
-/* <Parent> Expr = ExprParent  */
-/* <Parent> ExprContext = ExprContextParent  */
-/* <Parent> ExprList = ExprListParent  */
-/* <Parent> int = ImportExpr  */
-
-/* <Field> Keyword.location = 0, location */
-/* <Field> Keyword.value = 1, expr */
-/* <Field> Keyword.arg = 2, str */
-/* <Parent> Location = LocationParent  */
-/* <Parent> string = Num  */
-/* <Parent> Operator = BinaryExpr  */
-/* <Parent> ParameterList = Function  */
-
-/* <Field> Stmt.location = 0, location */
-/* <Parent> Stmt = StmtList  */
-/* <Parent> StmtList = StmtListParent  */
-/* <Parent> string = StrParent  */
-/* <Parent> StringList = StrListParent  */
-/* <Parent> Unaryop = UnaryExpr  */
-/* <Parent> Variable = VariableParent  */
-py_Classes(unique int id : @py_Class,
-    unique int parent : @py_ClassExpr ref);
-
-py_Functions(unique int id : @py_Function,
-    unique int parent : @py_Function_parent ref);
-
-py_Modules(unique int id : @py_Module);
-
-py_StringParts(unique int id : @py_StringPart,
-    int parent : @py_StringPart_list ref,
-    int idx : int ref);
-
-py_StringPart_lists(unique int id : @py_StringPart_list,
-    unique int parent : @py_Bytes_or_Str ref);
-
-py_aliases(unique int id : @py_alias,
-    int parent : @py_alias_list ref,
-    int idx : int ref);
-
-py_alias_lists(unique int id : @py_alias_list,
-    unique int parent : @py_Import ref);
-
-py_arguments(unique int id : @py_arguments,
-    unique int parent : @py_arguments_parent ref);
-
-py_bools(int parent : @py_bool_parent ref,
-    int idx : int ref);
-
-py_boolops(unique int id : @py_boolop,
-    int kind: int ref,
-    unique int parent : @py_BoolExpr ref);
-
-py_bytes(varchar(1) id : string ref,
-    int parent : @py_Bytes ref,
-    int idx : int ref);
-
-py_cmpops(unique int id : @py_cmpop,
-    int kind: int ref,
-    int parent : @py_cmpop_list ref,
-    int idx : int ref);
-
-py_cmpop_lists(unique int id : @py_cmpop_list,
-    unique int parent : @py_Compare ref);
-
-py_comprehensions(unique int id : @py_comprehension,
-    int parent : @py_comprehension_list ref,
-    int idx : int ref);
-
-py_comprehension_lists(unique int id : @py_comprehension_list,
-    unique int parent : @py_ListComp ref);
-
-py_dict_items(unique int id : @py_dict_item,
-    int kind: int ref,
-    int parent : @py_dict_item_list ref,
-    int idx : int ref);
-
-py_dict_item_lists(unique int id : @py_dict_item_list,
-    unique int parent : @py_dict_item_list_parent ref);
-
-py_exprs(unique int id : @py_expr,
-    int kind: int ref,
-    int parent : @py_expr_parent ref,
-    int idx : int ref);
-
-py_expr_contexts(unique int id : @py_expr_context,
-    int kind: int ref,
-    unique int parent : @py_expr_context_parent ref);
-
-py_expr_lists(unique int id : @py_expr_list,
-    int parent : @py_expr_list_parent ref,
-    int idx : int ref);
-
-py_ints(int id : int ref,
-    unique int parent : @py_ImportExpr ref);
-
-py_locations(unique int id : @location ref,
-    unique int parent : @py_location_parent ref);
-
-py_numbers(varchar(1) id : string ref,
-    int parent : @py_Num ref,
-    int idx : int ref);
-
-py_operators(unique int id : @py_operator,
-    int kind: int ref,
-    unique int parent : @py_BinaryExpr ref);
-
-py_parameter_lists(unique int id : @py_parameter_list,
-    unique int parent : @py_Function ref);
-
-py_stmts(unique int id : @py_stmt,
-    int kind: int ref,
-    int parent : @py_stmt_list ref,
-    int idx : int ref);
-
-py_stmt_lists(unique int id : @py_stmt_list,
-    int parent : @py_stmt_list_parent ref,
-    int idx : int ref);
-
-py_strs(varchar(1) id : string ref,
-    int parent : @py_str_parent ref,
-    int idx : int ref);
-
-py_str_lists(unique int id : @py_str_list,
-    unique int parent : @py_str_list_parent ref);
-
-py_unaryops(unique int id : @py_unaryop,
-    int kind: int ref,
-    unique int parent : @py_UnaryExpr ref);
-
-py_variables(int id : @py_variable ref,
-    unique int parent : @py_variable_parent ref);
-
-case @py_boolop.kind of
-    0 = @py_And
-|   1 = @py_Or;
-
-case @py_cmpop.kind of
-    0 = @py_Eq
-|   1 = @py_Gt
-|   2 = @py_GtE
-|   3 = @py_In
-|   4 = @py_Is
-|   5 = @py_IsNot
-|   6 = @py_Lt
-|   7 = @py_LtE
-|   8 = @py_NotEq
-|   9 = @py_NotIn;
-
-case @py_dict_item.kind of
-    0 = @py_DictUnpacking
-|   1 = @py_KeyValuePair
-|   2 = @py_keyword;
-
-case @py_expr.kind of
-    0 = @py_Attribute
-|   1 = @py_BinaryExpr
-|   2 = @py_BoolExpr
-|   3 = @py_Bytes
-|   4 = @py_Call
-|   5 = @py_ClassExpr
-|   6 = @py_Compare
-|   7 = @py_Dict
-|   8 = @py_DictComp
-|   9 = @py_Ellipsis
-|   10 = @py_FunctionExpr
-|   11 = @py_GeneratorExp
-|   12 = @py_IfExp
-|   13 = @py_ImportExpr
-|   14 = @py_ImportMember
-|   15 = @py_Lambda
-|   16 = @py_List
-|   17 = @py_ListComp
-|   18 = @py_Name
-|   19 = @py_Num
-|   20 = @py_Repr
-|   21 = @py_Set
-|   22 = @py_SetComp
-|   23 = @py_Slice
-|   24 = @py_Starred
-|   25 = @py_Str
-|   26 = @py_Subscript
-|   27 = @py_Tuple
-|   28 = @py_UnaryExpr
-|   29 = @py_Yield
-|   30 = @py_YieldFrom
-|   31 = @py_TemplateDottedNotation
-|   32 = @py_Filter
-|   33 = @py_PlaceHolder
-|   34 = @py_Await
-|   35 = @py_Fstring
-|   36 = @py_FormattedValue
-|   37 = @py_AssignExpr
-|   38 = @py_SpecialOperation;
-
-case @py_expr_context.kind of
-    0 = @py_AugLoad
-|   1 = @py_AugStore
-|   2 = @py_Del
-|   3 = @py_Load
-|   4 = @py_Param
-|   5 = @py_Store;
-
-case @py_operator.kind of
-    0 = @py_Add
-|   1 = @py_BitAnd
-|   2 = @py_BitOr
-|   3 = @py_BitXor
-|   4 = @py_Div
-|   5 = @py_FloorDiv
-|   6 = @py_LShift
-|   7 = @py_Mod
-|   8 = @py_Mult
-|   9 = @py_Pow
-|   10 = @py_RShift
-|   11 = @py_Sub
-|   12 = @py_MatMult;
-
-case @py_stmt.kind of
-    0 = @py_Assert
-|   1 = @py_Assign
-|   2 = @py_AugAssign
-|   3 = @py_Break
-|   4 = @py_Continue
-|   5 = @py_Delete
-|   6 = @py_ExceptStmt
-|   7 = @py_Exec
-|   8 = @py_Expr_stmt
-|   9 = @py_For
-|   10 = @py_Global
-|   11 = @py_If
-|   12 = @py_Import
-|   13 = @py_ImportStar
-|   14 = @py_Nonlocal
-|   15 = @py_Pass
-|   16 = @py_Print
-|   17 = @py_Raise
-|   18 = @py_Return
-|   19 = @py_Try
-|   20 = @py_While
-|   21 = @py_With
-|   22 = @py_TemplateWrite
-|   23 = @py_AnnAssign;
-
-case @py_unaryop.kind of
-    0 = @py_Invert
-|   1 = @py_Not
-|   2 = @py_UAdd
-|   3 = @py_USub;
-
-@py_Bytes_or_Str = @py_Bytes | @py_Str;
-
-@py_Function_parent = @py_DictComp | @py_FunctionExpr | @py_GeneratorExp | @py_Lambda | @py_ListComp | @py_SetComp;
-
-@py_arguments_parent = @py_FunctionExpr | @py_Lambda;
-
-@py_ast_node = @py_Class | @py_Function | @py_Module | @py_StringPart | @py_comprehension | @py_dict_item | @py_expr | @py_stmt;
-
-@py_bool_parent = @py_For | @py_Function | @py_Print | @py_With | @py_expr;
-
-@py_dict_item_list_parent = @py_Call | @py_ClassExpr | @py_Dict;
-
-@py_expr_context_parent = @py_Attribute | @py_List | @py_Name | @py_PlaceHolder | @py_Starred | @py_Subscript | @py_TemplateDottedNotation | @py_Tuple;
-
-@py_expr_list_parent = @py_Assign | @py_BoolExpr | @py_Call | @py_ClassExpr | @py_Compare | @py_Delete | @py_Fstring | @py_Function | @py_List | @py_Print | @py_Set | @py_SpecialOperation | @py_Tuple | @py_arguments | @py_comprehension;
-
-@py_expr_or_stmt = @py_expr | @py_stmt;
-
-@py_expr_parent = @py_AnnAssign | @py_Assert | @py_Assign | @py_AssignExpr | @py_Attribute | @py_AugAssign | @py_Await | @py_BinaryExpr | @py_Call | @py_Compare | @py_DictComp | @py_DictUnpacking | @py_ExceptStmt | @py_Exec | @py_Expr_stmt | @py_Filter | @py_For | @py_FormattedValue | @py_Function | @py_FunctionExpr | @py_GeneratorExp | @py_If | @py_IfExp | @py_ImportMember | @py_ImportStar | @py_KeyValuePair | @py_ListComp | @py_Print | @py_Raise | @py_Repr | @py_Return | @py_SetComp | @py_Slice | @py_Starred | @py_Subscript | @py_TemplateDottedNotation | @py_TemplateWrite | @py_UnaryExpr | @py_While | @py_With | @py_Yield | @py_YieldFrom | @py_alias | @py_arguments | @py_comprehension | @py_expr_list | @py_keyword | @py_parameter_list;
-
-@py_location_parent = @py_DictUnpacking | @py_KeyValuePair | @py_StringPart | @py_comprehension | @py_expr | @py_keyword | @py_stmt;
-
-@py_parameter = @py_Name | @py_Tuple;
-
-@py_scope = @py_Class | @py_Function | @py_Module;
-
-@py_stmt_list_parent = @py_Class | @py_ExceptStmt | @py_For | @py_Function | @py_If | @py_Module | @py_Try | @py_While | @py_With;
-
-@py_str_list_parent = @py_Global | @py_Nonlocal;
-
-@py_str_parent = @py_Attribute | @py_Class | @py_ClassExpr | @py_FormattedValue | @py_Function | @py_FunctionExpr | @py_ImportExpr | @py_ImportMember | @py_Module | @py_SpecialOperation | @py_Str | @py_StringPart | @py_TemplateDottedNotation | @py_keyword | @py_str_list;
-
-@py_variable_parent = @py_Name | @py_PlaceHolder;
-
-
-/*
- * End of auto-generated part
- */
-
-
-
-/* Map relative names to absolute names for imports */
-py_absolute_names(int module : @py_Module ref,
-                  varchar(1) relname : string ref,
-                  varchar(1) absname : string ref);
-
-py_exports(int id : @py_Module ref,
-           varchar(1) name : string ref);
-
-/* Successor information */
-py_successors(int predecessor : @py_flow_node ref,
-              int successor : @py_flow_node ref);
-
-py_true_successors(int predecessor : @py_flow_node ref,
-              int successor : @py_flow_node ref);
-
-py_exception_successors(int predecessor : @py_flow_node ref,
-              int successor : @py_flow_node ref);
-
-py_false_successors(int predecessor : @py_flow_node ref,
-              int successor : @py_flow_node ref);
-
-py_flow_bb_node(unique int flownode : @py_flow_node,
-             int realnode : @py_ast_node ref,
-             int basicblock : @py_flow_node ref,
-             int index : int ref);
-
-py_scope_flow(int flow : @py_flow_node ref,
-              int scope : @py_scope ref,
-              int kind : int ref);
-
-py_idoms(unique int node : @py_flow_node ref,
-         int immediate_dominator : @py_flow_node ref);
-
-py_ssa_phi(int phi : @py_ssa_var ref,
-           int arg: @py_ssa_var ref);
-
-py_ssa_var(unique int id : @py_ssa_var,
-            int var : @py_variable ref);
-
-py_ssa_use(int node: @py_flow_node ref,
-           int var : @py_ssa_var ref);
-
-py_ssa_defn(unique int id : @py_ssa_var ref,
-                  int node: @py_flow_node ref);
-
-@py_base_var = @py_variable | @py_ssa_var;
-
-py_scopes(unique int node : @py_expr_or_stmt ref,
-         int scope : @py_scope ref);
-
-py_scope_location(unique int id : @location ref,
-                   unique int scope : @py_scope ref);
-
-py_flags_versioned(varchar(1) name : string ref,
-                   varchar(1) value : string ref,
-                   varchar(1) version : string ref);
-
-py_syntax_error_versioned(unique int id : @location ref,
-                varchar(1) message : string ref,
-                varchar(1) version : string ref);
-
-py_comments(unique int id : @py_comment,
-           varchar(1) text : string ref,
-           unique int location : @location ref);
-
-/* Type information support */
-
-py_cobjects(unique int obj : @py_cobject);
-
-py_cobjecttypes(unique int obj : @py_cobject ref,
-                int typeof : @py_cobject ref);
-
-py_cobjectnames(unique int obj : @py_cobject ref,
-                varchar(1) name : string ref);
-
-/* Kind should be 0 for introspection, > 0 from source, as follows:
-   1 from C extension source
- */
-py_cobject_sources(int obj : @py_cobject ref,
-                  int kind : int ref);
-
-py_cmembers_versioned(int object : @py_cobject ref,
-                      varchar(1) name : string ref,
-                      int member : @py_cobject ref,
-                      varchar(1) version : string ref);
-
-py_citems(int object : @py_cobject ref,
-            int index : int ref,
-            int member : @py_cobject ref);
-
-ext_argtype(int funcid : @py_object ref,
-            int arg : int ref,
-            int typeid : @py_object ref);
-
-ext_rettype(int funcid : @py_object ref,
-            int typeid : @py_object ref);
-
-ext_proptype(int propid : @py_object ref,
-             int typeid : @py_object ref);
-
-ext_argreturn(int funcid : @py_object ref,
-              int arg : int ref);
-
-py_special_objects(unique int obj : @py_cobject ref,
-                   unique varchar(1) name : string ref);
-
-py_decorated_object(int object : @py_object ref,
-                    int level: int ref);
-
-@py_object = @py_cobject | @py_flow_node;
-
-@py_source_element = @py_ast_node | @container;
-
-/* XML Files */
-
-xmlEncoding  (unique int id: @file ref, varchar(900) encoding: string ref);
-
-xmlDTDs       (unique int id: @xmldtd,
-              varchar(900) root: string ref,
-              varchar(900) publicId: string ref,
-              varchar(900) systemId: string ref,
-              int fileid: @file ref);
-
-xmlElements   (unique int id: @xmlelement,
-              varchar(900) name: string ref,
-              int parentid: @xmlparent ref,
-              int idx: int ref,
-              int fileid: @file ref);
-
-xmlAttrs      (unique int id: @xmlattribute,
-              int elementid: @xmlelement ref,
-              varchar(900) name: string ref,
-              varchar(3600) value: string ref,
-              int idx: int ref,
-              int fileid: @file ref);
-
-xmlNs         (int id: @xmlnamespace,
-              varchar(900) prefixName: string ref,
-              varchar(900) URI: string ref,
-              int fileid: @file ref);
-
-xmlHasNs      (int elementId: @xmlnamespaceable ref,
-              int nsId: @xmlnamespace ref,
-              int fileid: @file ref);
-
-xmlComments   (unique int id: @xmlcomment,
-              varchar(3600) text: string ref,
-              int parentid: @xmlparent ref,
-              int fileid: @file ref);
-
-xmlChars      (unique int id: @xmlcharacters,
-              varchar(3600) text: string ref,
-              int parentid: @xmlparent ref,
-              int idx: int ref,
-              int isCDATA: int ref,
-              int fileid: @file ref);
-
-@xmlparent = @file | @xmlelement;
-@xmlnamespaceable = @xmlelement | @xmlattribute;
-
-xmllocations(int xmlElement: @xmllocatable ref,
-             int location: @location_default ref);
-
-@xmllocatable = @xmlcharacters | @xmlelement | @xmlcomment | @xmlattribute | @xmldtd | @file | @xmlnamespace;

python/ql/lib/upgrades/ff5327c074ca235c6b2fa4ea9ed29f4c60aa46cb/py_exprs.ql

@@ -1,42 +0,0 @@
-// First we need to wrap some database types
-class Location extends @location {
-  /** Gets the start line of this location */
-  int getStartLine() {
-    locations_default(this, _, result, _, _, _) or
-    locations_ast(this, _, result, _, _, _)
-  }
-
-  string toString() { result = "<some file>" + ":" + this.getStartLine().toString() }
-}
-
-class Expr_ extends @py_expr {
-  string toString() { result = "Expr" }
-
-  Location getLocation() { py_locations(result, this) }
-}
-
-class ExprParent_ extends @py_expr_parent {
-  string toString() { result = "ExprParent" }
-}
-
-/**
- * New kinds have been inserted such that
- * `@py_Name` which used to have index 18 now has index 19.
- * Entries with lower indices are unchanged.
- */
-bindingset[old_index]
-int new_index(int old_index) {
-  if old_index < 18 then result = old_index else result = (19 - 18) + old_index
-}
-
-// The schema for py_exprs is:
-//
-// py_exprs(unique int id : @py_expr,
-//     int kind: int ref,
-//     int parent : @py_expr_parent ref,
-//     int idx : int ref);
-from Expr_ expr, int old_kind, ExprParent_ parent, int idx, int new_kind
-where
-  py_exprs(expr, old_kind, parent, idx) and
-  new_kind = new_index(old_kind)
-select expr, new_kind, parent, idx

python/ql/lib/upgrades/ff5327c074ca235c6b2fa4ea9ed29f4c60aa46cb/py_stmts.ql

@@ -1,42 +0,0 @@
-// First we need to wrap some database types
-class Location extends @location {
-  /** Gets the start line of this location */
-  int getStartLine() {
-    locations_default(this, _, result, _, _, _) or
-    locations_ast(this, _, result, _, _, _)
-  }
-
-  string toString() { result = "<some file>" + ":" + this.getStartLine().toString() }
-}
-
-class Stmt_ extends @py_stmt {
-  string toString() { result = "Stmt" }
-
-  Location getLocation() { py_locations(result, this) }
-}
-
-class StmtList_ extends @py_stmt_list {
-  string toString() { result = "StmtList" }
-}
-
-/**
- * New kinds have been inserted such that
- * `@py_Nonlocal` which used to have index 14 now has index 16.
- * Entries with lower indices are unchanged.
- */
-bindingset[old_index]
-int new_index(int old_index) {
-  if old_index < 14 then result = old_index else result = (16 - 14) + old_index
-}
-
-// The schema for py_stmts is:
-//
-// py_stmts(unique int id : @py_stmt,
-//   int kind: int ref,
-//   int parent : @py_stmt_list ref,
-//   int idx : int ref);
-from Stmt_ expr, int old_kind, StmtList_ parent, int idx, int new_kind
-where
-  py_stmts(expr, old_kind, parent, idx) and
-  new_kind = new_index(old_kind)
-select expr, new_kind, parent, idx

python/ql/lib/upgrades/ff5327c074ca235c6b2fa4ea9ed29f4c60aa46cb/upgrade.properties

@@ -1,4 +0,0 @@
-description: Add new statements and expressions for the match syntax.
-compatibility: backwards
-py_exprs.rel: run py_exprs.qlo
-py_stmts.rel: run py_stmts.qlo

python/ql/src/experimental/Security-new-dataflow/promote.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -Eeuo pipefail # see https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
+
+# Promotes new dataflow queries to be the real ones
+
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+cd $SCRIPTDIR
+for file in $(find . -mindepth 2); do
+    echo "Promoting $file"
+    mkdir -p "../../Security/$(dirname $file)"
+    mv "$file" "../../Security/${file}"
+done

python/ql/src/experimental/Security-old-dataflow/CVE-2018-1281/BindToAllInterfaces.ql

@@ -0,0 +1,35 @@
+/**
+ * @name Binding a socket to all network interfaces
+ * @description Binding a socket to all interfaces opens it up to traffic from any IPv4 address
+ * and is therefore associated with security risks.
+ * @kind problem
+ * @id py/old/bind-socket-all-network-interfaces
+ * @problem.severity error
+ */
+
+import python
+
+Value aSocket() { result.getClass() = Value::named("socket.socket") }
+
+CallNode socketBindCall() {
+  result = aSocket().attr("bind").(CallableValue).getACall() and major_version() = 3
+  or
+  result.getFunction().(AttrNode).getObject("bind").pointsTo(aSocket()) and
+  major_version() = 2
+}
+
+string allInterfaces() { result = "0.0.0.0" or result = "" }
+
+Value getTextValue(string address) {
+  result = Value::forUnicode(address) and major_version() = 3
+  or
+  result = Value::forString(address) and major_version() = 2
+}
+
+from CallNode call, TupleValue args, string address
+where
+  call = socketBindCall() and
+  call.getArg(0).pointsTo(args) and
+  args.getItem(0) = getTextValue(address) and
+  address = allInterfaces()
+select call.getNode(), "'" + address + "' binds a socket to all interfaces."

python/ql/src/experimental/Security-old-dataflow/CWE-022/PathInjection.ql

@@ -0,0 +1,38 @@
+/**
+ * @name OLD QUERY: Uncontrolled data used in path expression
+ * @description Accessing paths influenced by users can allow an attacker to access unexpected resources.
+ * @kind path-problem
+ * @problem.severity error
+ * @id py/old/path-injection
+ */
+
+import python
+import semmle.python.security.Paths
+/* Sources */
+import semmle.python.web.HttpRequest
+/* Sinks */
+import semmle.python.security.injection.Path
+
+class PathInjectionConfiguration extends TaintTracking::Configuration {
+  PathInjectionConfiguration() { this = "Path injection configuration" }
+
+  override predicate isSource(TaintTracking::Source source) {
+    source instanceof HttpRequestTaintSource
+  }
+
+  override predicate isSink(TaintTracking::Sink sink) { sink instanceof OpenNode }
+
+  override predicate isSanitizer(Sanitizer sanitizer) {
+    sanitizer instanceof PathSanitizer or
+    sanitizer instanceof NormalizedPathSanitizer
+  }
+
+  override predicate isExtension(TaintTracking::Extension extension) {
+    extension instanceof AbsPath
+  }
+}
+
+from PathInjectionConfiguration config, TaintedPathSource src, TaintedPathSink sink
+where config.hasFlowPath(src, sink)
+select sink.getSink(), src, sink, "This path depends on $@.", src.getSource(),
+  "a user-provided value"

python/ql/src/experimental/Security-old-dataflow/CWE-078/CommandInjection.ql

@@ -0,0 +1,36 @@
+/**
+ * @name OLD QUERY: Uncontrolled command line
+ * @description Using externally controlled strings in a command line may allow a malicious
+ *              user to change the meaning of the command.
+ * @kind path-problem
+ * @problem.severity error
+ * @id py/old/command-line-injection
+ */
+
+import python
+import semmle.python.security.Paths
+/* Sources */
+import semmle.python.web.HttpRequest
+/* Sinks */
+import semmle.python.security.injection.Command
+
+class CommandInjectionConfiguration extends TaintTracking::Configuration {
+  CommandInjectionConfiguration() { this = "Command injection configuration" }
+
+  override predicate isSource(TaintTracking::Source source) {
+    source instanceof HttpRequestTaintSource
+  }
+
+  override predicate isSink(TaintTracking::Sink sink) { sink instanceof CommandSink }
+
+  override predicate isExtension(TaintTracking::Extension extension) {
+    extension instanceof FirstElementFlow
+    or
+    extension instanceof FabricExecuteExtension
+  }
+}
+
+from CommandInjectionConfiguration config, TaintedPathSource src, TaintedPathSink sink
+where config.hasFlowPath(src, sink)
+select sink.getSink(), src, sink, "This command depends on $@.", src.getSource(),
+  "a user-provided value"

python/ql/src/experimental/Security-old-dataflow/CWE-079/ReflectedXss.ql

@@ -0,0 +1,37 @@
+/**
+ * @name OLD QUERY: Reflected server-side cross-site scripting
+ * @description Writing user input directly to a web page
+ *              allows for a cross-site scripting vulnerability.
+ * @kind path-problem
+ * @problem.severity error
+ * @id py/old/reflective-xss
+ */
+
+import python
+import semmle.python.security.Paths
+/* Sources */
+import semmle.python.web.HttpRequest
+/* Sinks */
+import semmle.python.web.HttpResponse
+/* Flow */
+import semmle.python.security.strings.Untrusted
+
+class ReflectedXssConfiguration extends TaintTracking::Configuration {
+  ReflectedXssConfiguration() { this = "Reflected XSS configuration" }
+
+  override predicate isSource(TaintTracking::Source source) {
+    source instanceof HttpRequestTaintSource
+  }
+
+  override predicate isSink(TaintTracking::Sink sink) {
+    sink instanceof HttpResponseTaintSink and
+    not sink instanceof DjangoResponseContent
+    or
+    sink instanceof DjangoResponseContentXSSVulnerable
+  }
+}
+
+from ReflectedXssConfiguration config, TaintedPathSource src, TaintedPathSink sink
+where config.hasFlowPath(src, sink)
+select sink.getSink(), src, sink, "Cross-site scripting vulnerability due to $@.", src.getSource(),
+  "a user-provided value"

python/ql/src/experimental/Security-old-dataflow/CWE-089/SqlInjection.ql

@@ -0,0 +1,47 @@
+/**
+ * @name OLD QUERY: SQL query built from user-controlled sources
+ * @description Building a SQL query from user-controlled sources is vulnerable to insertion of
+ *              malicious SQL code by the user.
+ * @kind path-problem
+ * @problem.severity error
+ * @id py/old/sql-injection
+ */
+
+import python
+import semmle.python.security.Paths
+/* Sources */
+import semmle.python.web.HttpRequest
+/* Sinks */
+import semmle.python.security.injection.Sql
+import semmle.python.web.django.Db
+import semmle.python.web.django.Model
+
+class SQLInjectionConfiguration extends TaintTracking::Configuration {
+  SQLInjectionConfiguration() { this = "SQL injection configuration" }
+
+  override predicate isSource(TaintTracking::Source source) {
+    source instanceof HttpRequestTaintSource
+  }
+
+  override predicate isSink(TaintTracking::Sink sink) { sink instanceof SqlInjectionSink }
+}
+
+/*
+ * Additional configuration to support tracking of DB objects. Connections, cursors, etc.
+ * Without this configuration (or the LegacyConfiguration), the pattern of
+ * `any(MyTaintKind k).taints(control_flow_node)` used in DbConnectionExecuteArgument would not work.
+ */
+
+class DbConfiguration extends TaintTracking::Configuration {
+  DbConfiguration() { this = "DB configuration" }
+
+  override predicate isSource(TaintTracking::Source source) {
+    source instanceof DjangoModelObjects or
+    source instanceof DbConnectionSource
+  }
+}
+
+from SQLInjectionConfiguration config, TaintedPathSource src, TaintedPathSink sink
+where config.hasFlowPath(src, sink)
+select sink.getSink(), src, sink, "This SQL query depends on $@.", src.getSource(),
+  "a user-provided value"

python/ql/src/experimental/Security-old-dataflow/CWE-094/CodeInjection.ql

@@ -0,0 +1,30 @@
+/**
+ * @name Code injection
+ * @description OLD QUERY: Interpreting unsanitized user input as code allows a malicious user arbitrary
+ *              code execution.
+ * @kind path-problem
+ * @problem.severity error
+ * @id py/old/code-injection
+ */
+
+import python
+import semmle.python.security.Paths
+/* Sources */
+import semmle.python.web.HttpRequest
+/* Sinks */
+import semmle.python.security.injection.Exec
+
+class CodeInjectionConfiguration extends TaintTracking::Configuration {
+  CodeInjectionConfiguration() { this = "Code injection configuration" }
+
+  override predicate isSource(TaintTracking::Source source) {
+    source instanceof HttpRequestTaintSource
+  }
+
+  override predicate isSink(TaintTracking::Sink sink) { sink instanceof StringEvaluationNode }
+}
+
+from CodeInjectionConfiguration config, TaintedPathSource src, TaintedPathSink sink
+where config.hasFlowPath(src, sink)
+select sink.getSink(), src, sink, "$@ flows to here and is interpreted as code.", src.getSource(),
+  "A user-provided value"

python/ql/src/experimental/Security-old-dataflow/CWE-312/CleartextLogging.ql

@@ -0,0 +1,33 @@
+/**
+ * @name Clear-text logging of sensitive information
+ * @description OLD QUERY: Logging sensitive information without encryption or hashing can
+ *              expose it to an attacker.
+ * @kind path-problem
+ * @problem.severity error
+ * @id py/old/clear-text-logging-sensitive-data
+ * @deprecated
+ */
+
+import python
+import semmle.python.security.Paths
+import semmle.python.dataflow.TaintTracking
+import semmle.python.security.SensitiveData
+import semmle.python.security.ClearText
+
+class CleartextLoggingConfiguration extends TaintTracking::Configuration {
+  CleartextLoggingConfiguration() { this = "ClearTextLogging" }
+
+  override predicate isSource(DataFlow::Node src, TaintKind kind) {
+    src.asCfgNode().(SensitiveData::Source).isSourceOf(kind)
+  }
+
+  override predicate isSink(DataFlow::Node sink, TaintKind kind) {
+    sink.asCfgNode() instanceof ClearTextLogging::Sink and
+    kind instanceof SensitiveData
+  }
+}
+
+from CleartextLoggingConfiguration config, TaintedPathSource source, TaintedPathSink sink
+where config.hasFlowPath(source, sink)
+select sink.getSink(), source, sink, "Sensitive data returned by $@ is logged here.",
+  source.getSource(), source.getCfgNode().(SensitiveData::Source).repr()

python/ql/src/experimental/Security-old-dataflow/CWE-312/CleartextStorage.ql

@@ -0,0 +1,33 @@
+/**
+ * @name Clear-text storage of sensitive information
+ * @description OLD QUERY: Sensitive information stored without encryption or hashing can expose it to an
+ *              attacker.
+ * @kind path-problem
+ * @problem.severity error
+ * @id py/old/clear-text-storage-sensitive-data
+ * @deprecated
+ */
+
+import python
+import semmle.python.security.Paths
+import semmle.python.dataflow.TaintTracking
+import semmle.python.security.SensitiveData
+import semmle.python.security.ClearText
+
+class CleartextStorageConfiguration extends TaintTracking::Configuration {
+  CleartextStorageConfiguration() { this = "ClearTextStorage" }
+
+  override predicate isSource(DataFlow::Node src, TaintKind kind) {
+    src.asCfgNode().(SensitiveData::Source).isSourceOf(kind)
+  }
+
+  override predicate isSink(DataFlow::Node sink, TaintKind kind) {
+    sink.asCfgNode() instanceof ClearTextStorage::Sink and
+    kind instanceof SensitiveData
+  }
+}
+
+from CleartextStorageConfiguration config, TaintedPathSource source, TaintedPathSink sink
+where config.hasFlowPath(source, sink)
+select sink.getSink(), source, sink, "Sensitive data from $@ is stored here.", source.getSource(),
+  source.getCfgNode().(SensitiveData::Source).repr()

python/ql/src/experimental/Security-old-dataflow/CWE-326/WeakCrypto.ql

@@ -0,0 +1,78 @@
+/**
+ * @name OLD QUERY: Use of weak cryptographic key
+ * @description Use of a cryptographic key that is too small may allow the encryption to be broken.
+ * @kind problem
+ * @problem.severity error
+ * @id py/old/weak-crypto-key
+ */
+
+import python
+
+int minimumSecureKeySize(string algo) {
+  algo = "DSA" and result = 2048
+  or
+  algo = "RSA" and result = 2048
+  or
+  algo = "ECC" and result = 224
+}
+
+predicate dsaRsaKeySizeArg(FunctionValue func, string algorithm, string arg) {
+  exists(ModuleValue mod | func = mod.attr(_) |
+    algorithm = "DSA" and
+    (
+      mod = Module::named("cryptography.hazmat.primitives.asymmetric.dsa") and arg = "key_size"
+      or
+      mod = Module::named("Crypto.PublicKey.DSA") and arg = "bits"
+      or
+      mod = Module::named("Cryptodome.PublicKey.DSA") and arg = "bits"
+    )
+    or
+    algorithm = "RSA" and
+    (
+      mod = Module::named("cryptography.hazmat.primitives.asymmetric.rsa") and arg = "key_size"
+      or
+      mod = Module::named("Crypto.PublicKey.RSA") and arg = "bits"
+      or
+      mod = Module::named("Cryptodome.PublicKey.RSA") and arg = "bits"
+    )
+  )
+}
+
+predicate ecKeySizeArg(FunctionValue func, string arg) {
+  exists(ModuleValue mod | func = mod.attr(_) |
+    mod = Module::named("cryptography.hazmat.primitives.asymmetric.ec") and arg = "curve"
+  )
+}
+
+int keySizeFromCurve(ClassValue curveClass) {
+  result = curveClass.declaredAttribute("key_size").(NumericValue).getIntValue()
+}
+
+predicate algorithmAndKeysizeForCall(
+  CallNode call, string algorithm, int keySize, ControlFlowNode keyOrigin
+) {
+  exists(FunctionValue func, string argname, ControlFlowNode arg |
+    arg = func.getNamedArgumentForCall(call, argname)
+  |
+    exists(NumericValue key |
+      arg.pointsTo(key, keyOrigin) and
+      dsaRsaKeySizeArg(func, algorithm, argname) and
+      keySize = key.getIntValue()
+    )
+    or
+    exists(Value curveClassInstance |
+      algorithm = "ECC" and
+      ecKeySizeArg(func, argname) and
+      arg.pointsTo(_, curveClassInstance, keyOrigin) and
+      keySize = keySizeFromCurve(curveClassInstance.getClass())
+    )
+  )
+}
+
+from CallNode call, string algo, int keySize, ControlFlowNode origin
+where
+  algorithmAndKeysizeForCall(call, algo, keySize, origin) and
+  keySize < minimumSecureKeySize(algo)
+select call,
+  "Creation of an " + algo + " key uses $@ bits, which is below " + minimumSecureKeySize(algo) +
+    " and considered breakable.", origin, keySize.toString()

python/ql/src/experimental/Security-old-dataflow/CWE-327/BrokenCryptoAlgorithm.ql

@@ -0,0 +1,28 @@
+/**
+ * @name OLD QUERY: Use of a broken or weak cryptographic algorithm
+ * @description Using broken or weak cryptographic algorithms can compromise security.
+ * @kind path-problem
+ * @problem.severity warning
+ * @id py/old/weak-cryptographic-algorithm
+ * @deprecated
+ */
+
+import python
+import semmle.python.security.Paths
+import semmle.python.security.SensitiveData
+import semmle.python.security.Crypto
+
+class BrokenCryptoConfiguration extends TaintTracking::Configuration {
+  BrokenCryptoConfiguration() { this = "Broken crypto configuration" }
+
+  override predicate isSource(TaintTracking::Source source) {
+    source instanceof SensitiveDataSource
+  }
+
+  override predicate isSink(TaintTracking::Sink sink) { sink instanceof WeakCryptoSink }
+}
+
+from BrokenCryptoConfiguration config, TaintedPathSource src, TaintedPathSink sink
+where config.hasFlowPath(src, sink)
+select sink.getSink(), src, sink, "$@ is used in a broken or weak cryptographic algorithm.",
+  src.getSource(), "Sensitive data"

python/ql/src/experimental/Security-old-dataflow/CWE-502/UnsafeDeserialization.ql

@@ -0,0 +1,32 @@
+/**
+ * @name OLD QUERY: Deserializing untrusted input
+ * @description Deserializing user-controlled data may allow attackers to execute arbitrary code.
+ * @kind path-problem
+ * @id py/old/unsafe-deserialization
+ * @problem.severity error
+ */
+
+import python
+import semmle.python.security.Paths
+// Sources -- Any untrusted input
+import semmle.python.web.HttpRequest
+// Flow -- untrusted string
+import semmle.python.security.strings.Untrusted
+// Sink -- Unpickling and other deserialization formats.
+import semmle.python.security.injection.Pickle
+import semmle.python.security.injection.Marshal
+import semmle.python.security.injection.Yaml
+
+class UnsafeDeserializationConfiguration extends TaintTracking::Configuration {
+  UnsafeDeserializationConfiguration() { this = "Unsafe deserialization configuration" }
+
+  override predicate isSource(TaintTracking::Source source) {
+    source instanceof HttpRequestTaintSource
+  }
+
+  override predicate isSink(TaintTracking::Sink sink) { sink instanceof DeserializationSink }
+}
+
+from UnsafeDeserializationConfiguration config, TaintedPathSource src, TaintedPathSink sink
+where config.hasFlowPath(src, sink)
+select sink.getSink(), src, sink, "Deserializing of $@.", src.getSource(), "untrusted input"

python/ql/src/experimental/Security-old-dataflow/CWE-601/UrlRedirect.ql

@@ -0,0 +1,37 @@
+/**
+ * @name OLD QUERY: URL redirection from remote source
+ * @description URL redirection based on unvalidated user input
+ *              may cause redirection to malicious web sites.
+ * @kind path-problem
+ * @problem.severity error
+ * @id py/old/url-redirection
+ */
+
+import python
+import semmle.python.security.Paths
+import semmle.python.web.HttpRedirect
+import semmle.python.web.HttpRequest
+import semmle.python.security.strings.Untrusted
+
+/** Url redirection is a problem only if the user controls the prefix of the URL */
+class UntrustedPrefixStringKind extends UntrustedStringKind {
+  override TaintKind getTaintForFlowStep(ControlFlowNode fromnode, ControlFlowNode tonode) {
+    result = UntrustedStringKind.super.getTaintForFlowStep(fromnode, tonode) and
+    not tonode.(BinaryExprNode).getRight() = fromnode
+  }
+}
+
+class UrlRedirectConfiguration extends TaintTracking::Configuration {
+  UrlRedirectConfiguration() { this = "URL redirect configuration" }
+
+  override predicate isSource(TaintTracking::Source source) {
+    source instanceof HttpRequestTaintSource
+  }
+
+  override predicate isSink(TaintTracking::Sink sink) { sink instanceof HttpRedirectTaintSink }
+}
+
+from UrlRedirectConfiguration config, TaintedPathSource src, TaintedPathSink sink
+where config.hasFlowPath(src, sink)
+select sink.getSink(), src, sink, "Untrusted URL redirection due to $@.", src.getSource(),
+  "a user-provided value"

python/ql/src/qlpack.yml

@@ -1,11 +1,9 @@
 name: codeql/python-queries
 version: 0.0.8-dev
-groups: 
-  - python
-  - queries
+groups: python
 dependencies:
-  codeql/python-all: "*"
-  codeql/suite-helpers: "*"
+    codeql/python-all: "*"
+    codeql/suite-helpers: "*"
 suites: codeql-suites
 extractor: python
 defaultSuiteFile: codeql-suites/python-code-scanning.qls

python/ql/test/experimental/dataflow/match/dataflowTest.ql

@@ -1,13 +0,0 @@
-import python
-import experimental.dataflow.TestUtil.FlowTest
-import experimental.dataflow.testConfig
-
-class DataFlowTest extends FlowTest {
-  DataFlowTest() { this = "DataFlowTest" }
-
-  override string flowTag() { result = "flow" }
-
-  override predicate relevantFlow(DataFlow::Node source, DataFlow::Node sink) {
-    exists(TestConfiguration cfg | cfg.hasFlow(source, sink))
-  }
-}

python/ql/test/experimental/dataflow/match/test.py

@@ -1,156 +0,0 @@
-
-import sys
-import os
-
-sys.path.append(os.path.dirname(os.path.dirname((__file__))))
-from testlib import *
-
-# These are defined so that we can evaluate the test code.
-NONSOURCE = "not a source"
-SOURCE = "source"
-
-
-def is_source(x):
-    return x == "source" or x == b"source" or x == 42 or x == 42.0 or x == 42j
-
-
-def SINK(x):
-    if is_source(x):
-        print("OK")
-    else:
-        print("Unexpected flow", x)
-
-
-def SINK_F(x):
-    if is_source(x):
-        print("Unexpected flow", x)
-    else:
-        print("OK")
-
-def test_guard():
-    match SOURCE:
-        case x if SINK(x): #$ flow="SOURCE, l:-1 -> x"
-            pass
-
-@expects(2)
-def test_as_pattern():
-    match SOURCE:
-        case x as y:
-            SINK(x) #$ flow="SOURCE, l:-2 -> x"
-            SINK(y) #$ flow="SOURCE, l:-3 -> y"
-
-def test_or_pattern():
-    match SOURCE:
-        # We cannot use NONSOURCE in place of "" below, since it would be seen as a variable.
-        case ("" as x) | x:
-            SINK(x) #$ flow="SOURCE, l:-3 -> x"
-
-# No flow for literal pattern
-def test_literal_pattern():
-    match SOURCE:
-        case 42 as x:
-            SINK(x) #$ flow="SOURCE, l:-2 -> x" flow="42, l:-1 -> x"
-
-def test_capture_pattern():
-    match SOURCE:
-        case x:
-            SINK(x) #$ flow="SOURCE, l:-2 -> x"
-
-# No flow for wildcard pattern
-
-class Unsafe:
-    VALUE = SOURCE
-
-def test_value_pattern():
-    match SOURCE:
-        case Unsafe.VALUE as x:
-            SINK(x) #$ flow="SOURCE, l:-2 -> x" MISSING: flow="SOURCE, l:-5 -> x"
-
-@expects(2)
-def test_sequence_pattern_tuple():
-    match (NONSOURCE, SOURCE):
-        case (x, y):
-            SINK_F(x)
-            SINK(y) #$ flow="SOURCE, l:-3 -> y"
-
-@expects(2)
-def test_sequence_pattern_list():
-    match [NONSOURCE, SOURCE]:
-        case [x, y]:
-            SINK_F(x) #$ SPURIOUS: flow="SOURCE, l:-2 -> x"
-            SINK(y) #$ flow="SOURCE, l:-3 -> y"
-
-# Sets are excluded from sequence patterns,
-#   see https://www.python.org/dev/peps/pep-0635/#sequence-patterns
-
-@expects(2)
-def test_star_pattern_tuple():
-    match (NONSOURCE, SOURCE):
-        case (x, *y):
-            SINK_F(x)
-            SINK(y[0]) #$ flow="SOURCE, l:-3 -> y[0]"
-
-@expects(2)
-def test_star_pattern_tuple_exclusion():
-    match (SOURCE, NONSOURCE):
-        case (x, *y):
-            SINK(x) #$ flow="SOURCE, l:-2 -> x"
-            SINK_F(y[0])
-
-@expects(2)
-def test_star_pattern_list():
-    match [NONSOURCE, SOURCE]:
-        case [x, *y]:
-            SINK_F(x) #$ SPURIOUS: flow="SOURCE, l:-2 -> x"
-            SINK(y[0]) #$ flow="SOURCE, l:-3 -> y[0]"
-
-@expects(2)
-def test_star_pattern_list_exclusion():
-    match [SOURCE, NONSOURCE]:
-        case [x, *y]:
-            SINK(x) #$ flow="SOURCE, l:-2 -> x"
-            SINK_F(y[0]) #$ SPURIOUS: flow="SOURCE, l:-3 -> y[0]"
-
-@expects(2)
-def test_mapping_pattern():
-    match {"a": NONSOURCE, "b": SOURCE}:
-        case {"a": x, "b": y}:
-            SINK_F(x)
-            SINK(y) #$ flow="SOURCE, l:-3 -> y"
-
-# also tests the key value pattern
-@expects(2)
-def test_double_star_pattern():
-    match {"a": NONSOURCE, "b": SOURCE}:
-        case {"a": x, **y}:
-            SINK_F(x)
-            SINK(y["b"]) #$ flow="SOURCE, l:-3 -> y['b']"
-
-@expects(2)
-def test_double_star_pattern_exclusion():
-    match {"a": SOURCE, "b": NONSOURCE}:
-        case {"a": x, **y}:
-            SINK(x) #$ flow="SOURCE, l:-2 -> x"
-            SINK_F(y["b"])
-            try:
-                SINK_F(y["a"])
-            except KeyError:
-                pass
-
-class Cell:
-    def __init__(self, value):
-        self.value = value
-
-# also tests the keyword pattern
-@expects(2)
-def test_class_pattern():
-    bad_cell = Cell(SOURCE)
-    good_cell = Cell(NONSOURCE)
-
-    match bad_cell:
-        case Cell(value = x):
-            SINK(x) #$ flow="SOURCE, l:-5 -> x"
-
-    match good_cell:
-        case Cell(value = x):
-            SINK_F(x)

python/ql/test/experimental/dataflow/validTest.py

@@ -57,10 +57,6 @@ if __name__ == "__main__":
     check_tests_valid("variable-capture.nonlocal")
     check_tests_valid("variable-capture.dict")
     check_tests_valid("module-initialization.multiphase")
-
-    # The below will fail unless we use Python 3.10 or newer.
-    # check_tests_valid("match.test")
-
     # The below fails when trying to import modules
     # check_tests_valid("module-initialization.test")
     # check_tests_valid("module-initialization.testOnce")

python/ql/test/experimental/meta/debug/dataflowTestPaths.expected

@@ -1,4 +0,0 @@
-edges
-nodes
-subpaths
-#select

python/ql/test/experimental/meta/debug/dataflowTestPaths.ql

@@ -1,25 +0,0 @@
-/**
- * @kind path-problem
- */
-
-// This query is for debugging InlineTaintTestFailures.
-// The intended usage is
-// 1. load the database of the failing test
-// 2. run this query to see actual paths
-// 3. if necessary, look at partial paths by (un)commenting appropriate lines
-import python
-import semmle.python.dataflow.new.DataFlow
-import experimental.dataflow.testConfig
-// import DataFlow::PartialPathGraph
-import DataFlow::PathGraph
-
-class Conf extends TestConfiguration {
-  override int explorationLimit() { result = 5 }
-}
-
-// from Conf config, DataFlow::PartialPathNode source, DataFlow::PartialPathNode sink
-// where config.hasPartialFlow(source, sink, _)
-from Conf config, DataFlow::PathNode source, DataFlow::PathNode sink
-where config.hasFlowPath(source, sink)
-select sink.getNode(), source, sink, "This node receives taint from $@.", source.getNode(),
-  "this source"

python/tools/prepare-db-downgrade.sh

@@ -1,107 +0,0 @@
-#!/bin/sh
-#
-# Prepare the downgrade script directory for a Python database schema downgrade.
-
-set -e
-set -u
-
-app_name="$(basename "$0")"
-
-usage()
-{
-  exit_code="$1"
-  shift
-
-  cat >&2 <<EOF
-${app_name}: $@
-${app_name}: Generate skeleton downgrade script.
-Usage: ${app_name} [--prev_hash <COMMITISH>]"
-
---prev-hash <COMMITISH>
-        Hash/branch to use to get SHA1 for previous DB scheme.
-        Default: origin/main
-
-Must be run within the git repo needing an update.
-EOF
-  exit "${exit_code}"
-}
-
-prev_hash="origin/main"
-
-while [ $# -gt 0 ]; do
-  case "$1" in
-    -x)
-      set -x
-      ;;
-    -h | --help)
-      usage 0
-      ;;
-    --prev-hash)
-      if [ $# -eq 1 ]; then
-        usage 2 "--prev-hash requires Commit/Branch option"
-      fi
-      shift
-      prev_hash="$1"
-      ;;
-    --)
-      shift
-      break
-      ;;
-    -*)
-      usage 2 "Unrecognised option: $1"
-      ;;
-    *)
-      break
-      ;;
-  esac
-  shift
-done
-
-if [ $# -gt 0 ]; then
-  usage 2 "Unrecognised operand: $1"
-fi
-
-scheme_file_name="semmlecode.python.dbscheme"
-scheme_file="ql/lib/${scheme_file_name}"
-downgrade_root="ql/downgrades"
-
-check_hash_valid()
-{
-  if [ ${#2} -ne 40 ]; then
-    echo "Did not get expected $1 hash: $2" >&2
-    exit 2
-  fi
-}
-
-# Get the hash of the previous and current DB Schema files
-prev_hash="$(git show "${prev_hash}:python/${scheme_file}" | git hash-object --stdin)"
-check_hash_valid previous "${prev_hash}"
-current_hash="$(git hash-object "${scheme_file}")"
-check_hash_valid current "${current_hash}"
-if [ "${current_hash}" = "${prev_hash}" ]; then
-  echo "No work to be done."
-  exit
-fi
-
-# Copy current and new dbscheme into the downgrade dir
-downgradedir="${downgrade_root}/${current_hash}"
-mkdir -p "${downgradedir}"
-
-cp "${scheme_file}" "${downgradedir}/old.dbscheme"
-git cat-file blob "${prev_hash}" > "${downgradedir}/${scheme_file_name}"
-
-# Create the template downgrade.properties file.
-cat <<EOF > "${downgradedir}/downgrade.properties"
-description: <INSERT DESCRIPTION HERE>
-compatibility: full|backwards|partial|breaking
-EOF
-
-# Tell user what we've done
-cat <<EOF
-Created downgrade directory here:
-  ${downgradedir}
-
-Please update:
-  ${downgradedir}/downgrade.properties
-with appropriate downgrade instructions
-EOF

python/tools/prepare-db-upgrade.sh

@@ -1,106 +0,0 @@
-#!/bin/sh
-#
-# Prepare the upgrade script directory for a Python database schema upgrade.
-
-set -e
-set -u
-
-app_name="$(basename "$0")"
-
-usage()
-{
-  exit_code="$1"
-  shift
-
-  cat >&2 <<EOF
-${app_name}: $@
-${app_name}: Generate skeleton upgrade script.
-Usage: ${app_name} [--prev_hash <COMMITISH>]"
-
---prev-hash <COMMITISH>
-        Hash/branch to use to get SHA1 for previous DB scheme.
-        Default: origin/main
-
-Must be run within the git repo needing an update.
-EOF
-  exit "${exit_code}"
-}
-
-prev_hash="origin/main"
-
-while [ $# -gt 0 ]; do
-  case "$1" in
-    -x)
-      set -x
-      ;;
-    -h | --help)
-      usage 0
-      ;;
-    --prev-hash)
-      if [ $# -eq 1 ]; then
-        usage 2 "--prev-hash requires Commit/Branch option"
-      fi
-      shift
-      prev_hash="$1"
-      ;;
-    --)
-      shift
-      break
-      ;;
-    -*)
-      usage 2 "Unrecognised option: $1"
-      ;;
-    *)
-      break
-      ;;
-  esac
-  shift
-done
-
-if [ $# -gt 0 ]; then
-  usage 2 "Unrecognised operand: $1"
-fi
-
-scheme_file="ql/lib/semmlecode.python.dbscheme"
-upgrade_root="ql/upgrades"
-
-check_hash_valid()
-{
-  if [ ${#2} -ne 40 ]; then
-    echo "Did not get expected $1 hash: $2" >&2
-    exit 2
-  fi
-}
-
-# Get the hash of the previous and current DB Schema files
-prev_hash="$(git show "${prev_hash}:python/${scheme_file}" | git hash-object --stdin)"
-check_hash_valid previous "${prev_hash}"
-current_hash="$(git hash-object "${scheme_file}")"
-check_hash_valid current "${current_hash}"
-if [ "${current_hash}" = "${prev_hash}" ]; then
-  echo "No work to be done."
-  exit
-fi
-
-# Copy current and new dbscheme into the upgrade dir
-upgradedir="${upgrade_root}/${prev_hash}"
-mkdir -p "${upgradedir}"
-
-cp "${scheme_file}" "${upgradedir}"
-git cat-file blob "${prev_hash}" > "${upgradedir}/old.dbscheme"
-
-# Create the template upgrade.properties file.
-cat <<EOF > "${upgradedir}/upgrade.properties"
-description: <INSERT DESCRIPTION HERE>
-compatibility: full|backwards|partial|breaking
-EOF
-
-# Tell user what we've done
-cat <<EOF
-Created upgrade directory here:
-  ${upgradedir}
-
-Please update:
-  ${upgradedir}/upgrade.properties
-with appropriate upgrade instructions
-EOF

ruby/extractor/Cargo.toml

@@ -18,3 +18,4 @@ tracing-subscriber = { version = "0.3.3", features = ["env-filter"] }
 rayon = "1.5.0"
 num_cpus = "1.13.0"
 regex = "1.4.3"
+indexmap = "1.7.0"

ruby/extractor/src/extractor.rs

@@ -1,161 +1,111 @@
+use crate::trap;
+use indexmap::IndexMap;
 use node_types::{EntryKind, Field, NodeTypeMap, Storage, TypeName};
-use std::borrow::Cow;
-use std::collections::BTreeMap as Map;
-use std::collections::BTreeSet as Set;
 use std::fmt;
-use std::io::Write;
 use std::path::Path;
 
 use tracing::{error, info, span, Level};
 use tree_sitter::{Language, Node, Parser, Range, Tree};
 
-pub struct TrapWriter {
-    /// The accumulated trap entries
-    trap_output: Vec<TrapEntry>,
-    /// A counter for generating fresh labels
-    counter: u32,
-    /// cache of global keys
-    global_keys: std::collections::HashMap<String, Label>,
+pub fn populate_file(writer: &mut trap::Writer, absolute_path: &Path) -> trap::Label {
+    let (file_label, fresh) =
+        writer.global_id(trap::full_id_for_file(&normalize_path(absolute_path)));
+    if fresh {
+        writer.add_tuple(
+            "files",
+            vec![
+                trap::Arg::Label(file_label),
+                trap::Arg::String(normalize_path(absolute_path)),
+            ],
+        );
+        populate_parent_folders(writer, file_label, absolute_path.parent());
+    }
+    file_label
 }
 
-pub fn new_trap_writer() -> TrapWriter {
-    TrapWriter {
-        counter: 0,
-        trap_output: Vec::new(),
-        global_keys: std::collections::HashMap::new(),
+fn populate_empty_file(writer: &mut trap::Writer) -> trap::Label {
+    let (file_label, fresh) = writer.global_id("empty;sourcefile".to_owned());
+    if fresh {
+        writer.add_tuple(
+            "files",
+            vec![
+                trap::Arg::Label(file_label),
+                trap::Arg::String("".to_string()),
+            ],
+        );
     }
+    file_label
 }
 
-impl TrapWriter {
-    ///  Gets a label that will hold the unique ID of the passed string at import time.
-    ///  This can be used for incrementally importable TRAP files -- use globally unique
-    ///  strings to compute a unique ID for table tuples.
-    ///
-    ///  Note: You probably want to make sure that the key strings that you use are disjoint
-    ///  for disjoint column types; the standard way of doing this is to prefix (or append)
-    ///  the column type name to the ID. Thus, you might identify methods in Java by the
-    ///  full ID "methods_com.method.package.DeclaringClass.method(argumentList)".
+pub fn populate_empty_location(writer: &mut trap::Writer) {
+    let file_label = populate_empty_file(writer);
+    location(writer, file_label, 0, 0, 0, 0);
+}
 
-    fn fresh_id(&mut self) -> Label {
-        let label = Label(self.counter);
-        self.counter += 1;
-        self.trap_output.push(TrapEntry::FreshId(label));
-        label
-    }
-
-    fn global_id(&mut self, key: &str) -> (Label, bool) {
-        if let Some(label) = self.global_keys.get(key) {
-            return (*label, false);
-        }
-        let label = Label(self.counter);
-        self.counter += 1;
-        self.global_keys.insert(key.to_owned(), label);
-        self.trap_output
-            .push(TrapEntry::MapLabelToKey(label, key.to_owned()));
-        (label, true)
-    }
-
-    fn add_tuple(&mut self, table_name: &str, args: Vec<Arg>) {
-        self.trap_output
-            .push(TrapEntry::GenericTuple(table_name.to_owned(), args))
-    }
-
-    fn populate_file(&mut self, absolute_path: &Path) -> Label {
-        let (file_label, fresh) = self.global_id(&full_id_for_file(absolute_path));
-        if fresh {
-            self.add_tuple(
-                "files",
-                vec![
-                    Arg::Label(file_label),
-                    Arg::String(normalize_path(absolute_path)),
-                ],
-            );
-            self.populate_parent_folders(file_label, absolute_path.parent());
-        }
-        file_label
-    }
-
-    fn populate_empty_file(&mut self) -> Label {
-        let (file_label, fresh) = self.global_id("empty;sourcefile");
-        if fresh {
-            self.add_tuple(
-                "files",
-                vec![Arg::Label(file_label), Arg::String("".to_string())],
-            );
-        }
-        file_label
-    }
-
-    pub fn populate_empty_location(&mut self) {
-        let file_label = self.populate_empty_file();
-        self.location(file_label, 0, 0, 0, 0);
-    }
-
-    fn populate_parent_folders(&mut self, child_label: Label, path: Option<&Path>) {
-        let mut path = path;
-        let mut child_label = child_label;
-        loop {
-            match path {
-                None => break,
-                Some(folder) => {
-                    let (folder_label, fresh) = self.global_id(&full_id_for_folder(folder));
-                    self.add_tuple(
-                        "containerparent",
-                        vec![Arg::Label(folder_label), Arg::Label(child_label)],
+pub fn populate_parent_folders(
+    writer: &mut trap::Writer,
+    child_label: trap::Label,
+    path: Option<&Path>,
+) {
+    let mut path = path;
+    let mut child_label = child_label;
+    loop {
+        match path {
+            None => break,
+            Some(folder) => {
+                let (folder_label, fresh) =
+                    writer.global_id(trap::full_id_for_folder(&normalize_path(folder)));
+                writer.add_tuple(
+                    "containerparent",
+                    vec![
+                        trap::Arg::Label(folder_label),
+                        trap::Arg::Label(child_label),
+                    ],
+                );
+                if fresh {
+                    writer.add_tuple(
+                        "folders",
+                        vec![
+                            trap::Arg::Label(folder_label),
+                            trap::Arg::String(normalize_path(folder)),
+                        ],
                     );
-                    if fresh {
-                        self.add_tuple(
-                            "folders",
-                            vec![
-                                Arg::Label(folder_label),
-                                Arg::String(normalize_path(folder)),
-                            ],
-                        );
-                        path = folder.parent();
-                        child_label = folder_label;
-                    } else {
-                        break;
-                    }
+                    path = folder.parent();
+                    child_label = folder_label;
+                } else {
+                    break;
                 }
             }
         }
     }
+}
 
-    fn location(
-        &mut self,
-        file_label: Label,
-        start_line: usize,
-        start_column: usize,
-        end_line: usize,
-        end_column: usize,
-    ) -> Label {
-        let (loc_label, fresh) = self.global_id(&format!(
-            "loc,{{{}}},{},{},{},{}",
-            file_label, start_line, start_column, end_line, end_column
-        ));
-        if fresh {
-            self.add_tuple(
-                "locations_default",
-                vec![
-                    Arg::Label(loc_label),
-                    Arg::Label(file_label),
-                    Arg::Int(start_line),
-                    Arg::Int(start_column),
-                    Arg::Int(end_line),
-                    Arg::Int(end_column),
-                ],
-            );
-        }
-        loc_label
-    }
-
-    fn comment(&mut self, text: String) {
-        self.trap_output.push(TrapEntry::Comment(text));
-    }
-
-    pub fn output(self, writer: &mut dyn Write) -> std::io::Result<()> {
-        write!(writer, "{}", Program(self.trap_output))
+fn location(
+    writer: &mut trap::Writer,
+    file_label: trap::Label,
+    start_line: usize,
+    start_column: usize,
+    end_line: usize,
+    end_column: usize,
+) -> trap::Label {
+    let (loc_label, fresh) = writer.global_id(format!(
+        "loc,{{{}}},{},{},{},{}",
+        file_label, start_line, start_column, end_line, end_column
+    ));
+    if fresh {
+        writer.add_tuple(
+            "locations_default",
+            vec![
+                trap::Arg::Label(loc_label),
+                trap::Arg::Label(file_label),
+                trap::Arg::Int(start_line),
+                trap::Arg::Int(start_column),
+                trap::Arg::Int(end_line),
+                trap::Arg::Int(end_column),
+            ],
+        );
     }
+    loc_label
 }
 
 /// Extracts the source file at `path`, which is assumed to be canonicalized.
@@ -163,71 +113,42 @@ pub fn extract(
     language: Language,
     language_prefix: &str,
     schema: &NodeTypeMap,
-    trap_writer: &mut TrapWriter,
+    trap_writer: &mut trap::Writer,
     path: &Path,
     source: &[u8],
     ranges: &[Range],
 ) -> std::io::Result<()> {
+    let path_str = format!("{}", path.display());
     let span = span!(
         Level::TRACE,
         "extract",
-        file = %path.display()
+        file = %path_str
     );
 
     let _enter = span.enter();
 
-    info!("extracting: {}", path.display());
+    info!("extracting: {}", path_str);
 
     let mut parser = Parser::new();
     parser.set_language(language).unwrap();
     parser.set_included_ranges(ranges).unwrap();
     let tree = parser.parse(&source, None).expect("Failed to parse file");
-    trap_writer.comment(format!("Auto-generated TRAP file for {}", path.display()));
-    let file_label = &trap_writer.populate_file(path);
-    let mut visitor = Visitor {
+    let file_label = populate_file(trap_writer, path);
+    let mut visitor = Visitor::new(
         source,
         trap_writer,
         // TODO: should we handle path strings that are not valid UTF8 better?
-        path: format!("{}", path.display()),
-        file_label: *file_label,
-        toplevel_child_counter: 0,
-        stack: Vec::new(),
+        &path_str,
+        file_label,
         language_prefix,
         schema,
-    };
+    );
     traverse(&tree, &mut visitor);
 
     parser.reset();
     Ok(())
 }
 
-/// Escapes a string for use in a TRAP key, by replacing special characters with
-/// HTML entities.
-fn escape_key<'a, S: Into<Cow<'a, str>>>(key: S) -> Cow<'a, str> {
-    fn needs_escaping(c: char) -> bool {
-        matches!(c, '&' | '{' | '}' | '"' | '@' | '#')
-    }
-
-    let key = key.into();
-    if key.contains(needs_escaping) {
-        let mut escaped = String::with_capacity(2 * key.len());
-        for c in key.chars() {
-            match c {
-                '&' => escaped.push_str("&amp;"),
-                '{' => escaped.push_str("&lbrace;"),
-                '}' => escaped.push_str("&rbrace;"),
-                '"' => escaped.push_str("&quot;"),
-                '@' => escaped.push_str("&commat;"),
-                '#' => escaped.push_str("&num;"),
-                _ => escaped.push(c),
-            }
-        }
-        Cow::Owned(escaped)
-    } else {
-        key
-    }
-}
-
 /// Normalizes the path according the common CodeQL specification. Assumes that
 /// `path` has already been canonicalized using `std::fs::canonicalize`.
 fn normalize_path(path: &Path) -> String {
@@ -267,34 +188,28 @@ fn normalize_path(path: &Path) -> String {
     }
 }
 
-fn full_id_for_file(path: &Path) -> String {
-    format!("{};sourcefile", escape_key(&normalize_path(path)))
-}
-
-fn full_id_for_folder(path: &Path) -> String {
-    format!("{};folder", escape_key(&normalize_path(path)))
-}
-
 struct ChildNode {
     field_name: Option<&'static str>,
-    label: Label,
+    label: trap::Label,
     type_name: TypeName,
 }
 
 struct Visitor<'a> {
     /// The file path of the source code (as string)
-    path: String,
+    path: &'a str,
     /// The label to use whenever we need to refer to the `@file` entity of this
     /// source file.
-    file_label: Label,
+    file_label: trap::Label,
     /// The source code as a UTF-8 byte array
     source: &'a [u8],
-    /// A TrapWriter to accumulate trap entries
-    trap_writer: &'a mut TrapWriter,
+    /// A trap::Writer to accumulate trap entries
+    trap_writer: &'a mut trap::Writer,
     /// A counter for top-level child nodes
     toplevel_child_counter: usize,
-    /// Language prefix
-    language_prefix: &'a str,
+    /// Language-specific name of the AST parent table
+    ast_node_parent_table_name: String,
+    /// Language-specific name of the tokeninfo table
+    tokeninfo_table_name: String,
     /// A lookup table from type name to node types
     schema: &'a NodeTypeMap,
     /// A stack for gathering information from child nodes. Whenever a node is
@@ -303,39 +218,62 @@ struct Visitor<'a> {
     /// node the list containing the child data is popped from the stack and
     /// matched against the dbscheme for the node. If the expectations are met
     /// the corresponding row definitions are added to the trap_output.
-    stack: Vec<(Label, usize, Vec<ChildNode>)>,
+    stack: Vec<(trap::Label, usize, Vec<ChildNode>)>,
 }
 
-impl Visitor<'_> {
+impl<'a> Visitor<'a> {
+    fn new(
+        source: &'a [u8],
+        trap_writer: &'a mut trap::Writer,
+        path: &'a str,
+        file_label: trap::Label,
+        language_prefix: &str,
+        schema: &'a NodeTypeMap,
+    ) -> Visitor<'a> {
+        Visitor {
+            path,
+            file_label,
+            source,
+            trap_writer,
+            toplevel_child_counter: 0,
+            ast_node_parent_table_name: format!("{}_ast_node_parent", language_prefix),
+            tokeninfo_table_name: format!("{}_tokeninfo", language_prefix),
+            schema,
+            stack: Vec::new(),
+        }
+    }
+
     fn record_parse_error(
         &mut self,
         error_message: String,
         full_error_message: String,
-        loc: Label,
+        loc: trap::Label,
     ) {
         error!("{}", full_error_message);
         let id = self.trap_writer.fresh_id();
         self.trap_writer.add_tuple(
             "diagnostics",
             vec![
-                Arg::Label(id),
-                Arg::Int(40), // severity 40 = error
-                Arg::String("parse_error".to_string()),
-                Arg::String(error_message),
-                Arg::String(full_error_message),
-                Arg::Label(loc),
+                trap::Arg::Label(id),
+                trap::Arg::Int(40), // severity 40 = error
+                trap::Arg::String("parse_error".to_string()),
+                trap::Arg::String(error_message),
+                trap::Arg::String(full_error_message),
+                trap::Arg::Label(loc),
             ],
         );
     }
 
-    fn record_parse_error_for_node(
-        &mut self,
-        error_message: String,
-        full_error_message: String,
-        node: Node,
-    ) {
+    fn record_parse_error_for_node(&mut self, error_message: String, node: &Node) {
+        let full_error_message = format!(
+            "{}:{}: {}",
+            &self.path,
+            node.start_position().row + 1,
+            &error_message
+        );
         let (start_line, start_column, end_line, end_column) = location_for(self.source, node);
-        let loc = self.trap_writer.location(
+        let loc = location(
+            self.trap_writer,
             self.file_label,
             start_line,
             start_column,
@@ -345,20 +283,14 @@ impl Visitor<'_> {
         self.record_parse_error(error_message, full_error_message, loc);
     }
 
-    fn enter_node(&mut self, node: Node) -> bool {
+    fn enter_node(&mut self, node: &Node) -> bool {
         if node.is_error() || node.is_missing() {
             let error_message = if node.is_missing() {
                 format!("parse error: expecting '{}'", node.kind())
             } else {
                 "parse error".to_string()
             };
-            let full_error_message = format!(
-                "{}:{}: {}",
-                &self.path,
-                node.start_position().row + 1,
-                error_message
-            );
-            self.record_parse_error_for_node(error_message, full_error_message, node);
+            self.record_parse_error_for_node(error_message, node);
             return false;
         }
 
@@ -368,13 +300,14 @@ impl Visitor<'_> {
         true
     }
 
-    fn leave_node(&mut self, field_name: Option<&'static str>, node: Node) {
+    fn leave_node(&mut self, field_name: Option<&'static str>, node: &Node) {
         if node.is_error() || node.is_missing() {
             return;
         }
         let (id, _, child_nodes) = self.stack.pop().expect("Vistor: empty stack");
         let (start_line, start_column, end_line, end_column) = location_for(self.source, node);
-        let loc = self.trap_writer.location(
+        let loc = location(
+            self.trap_writer,
             self.file_label,
             start_line,
             start_column,
@@ -402,20 +335,20 @@ impl Visitor<'_> {
         match &table.kind {
             EntryKind::Token { kind_id, .. } => {
                 self.trap_writer.add_tuple(
-                    &format!("{}_ast_node_parent", self.language_prefix),
+                    &self.ast_node_parent_table_name,
                     vec![
-                        Arg::Label(id),
-                        Arg::Label(parent_id),
-                        Arg::Int(parent_index),
+                        trap::Arg::Label(id),
+                        trap::Arg::Label(parent_id),
+                        trap::Arg::Int(parent_index),
                     ],
                 );
                 self.trap_writer.add_tuple(
-                    &format!("{}_tokeninfo", self.language_prefix),
+                    &self.tokeninfo_table_name,
                     vec![
-                        Arg::Label(id),
-                        Arg::Int(*kind_id),
+                        trap::Arg::Label(id),
+                        trap::Arg::Int(*kind_id),
                         sliced_source_arg(self.source, node),
-                        Arg::Label(loc),
+                        trap::Arg::Label(loc),
                     ],
                 );
             }
@@ -423,18 +356,18 @@ impl Visitor<'_> {
                 fields,
                 name: table_name,
             } => {
-                if let Some(args) = self.complex_node(&node, fields, &child_nodes, id) {
+                if let Some(args) = self.complex_node(node, fields, &child_nodes, id) {
                     self.trap_writer.add_tuple(
-                        &format!("{}_ast_node_parent", self.language_prefix),
+                        &self.ast_node_parent_table_name,
                         vec![
-                            Arg::Label(id),
-                            Arg::Label(parent_id),
-                            Arg::Int(parent_index),
+                            trap::Arg::Label(id),
+                            trap::Arg::Label(parent_id),
+                            trap::Arg::Int(parent_index),
                         ],
                     );
-                    let mut all_args = vec![Arg::Label(id)];
+                    let mut all_args = vec![trap::Arg::Label(id)];
                     all_args.extend(args);
-                    all_args.push(Arg::Label(loc));
+                    all_args.push(trap::Arg::Label(loc));
                     self.trap_writer.add_tuple(table_name, all_args);
                 }
             }
@@ -472,9 +405,9 @@ impl Visitor<'_> {
         node: &Node,
         fields: &[Field],
         child_nodes: &[ChildNode],
-        parent_id: Label,
-    ) -> Option<Vec<Arg>> {
-        let mut map: Map<&Option<String>, (&Field, Vec<Arg>)> = Map::new();
+        parent_id: trap::Label,
+    ) -> Option<Vec<trap::Arg>> {
+        let mut map: IndexMap<&Option<String>, (&Field, Vec<trap::Arg>)> = IndexMap::new();
         for field in fields {
             map.insert(&field.name, (field, Vec::new()));
         }
@@ -482,46 +415,47 @@ impl Visitor<'_> {
             if let Some((field, values)) = map.get_mut(&child_node.field_name.map(|x| x.to_owned()))
             {
                 //TODO: handle error and missing nodes
-                if self.type_matches(&child_node.type_name, &field.type_info) {
-                    if let node_types::FieldTypeInfo::ReservedWordInt(int_mapping) =
-                        &field.type_info
+                if field.type_info.valid_types.contains(&child_node.type_name) {
+                    if let node_types::FieldTypeKind::ReservedWordInt(int_mapping) =
+                        &field.type_info.kind
                     {
-                        // We can safely unwrap because type_matches checks the key is in the map.
-                        let (int_value, _) = int_mapping.get(&child_node.type_name.kind).unwrap();
-                        values.push(Arg::Int(*int_value));
+                        match int_mapping.get(&child_node.type_name.kind) {
+                            Some((int_value, _)) => values.push(trap::Arg::Int(*int_value)),
+                            None => self.record_parse_error_for_node(
+                                format!(
+                                    "could not map field {}::{} with type {:?} to an integer value",
+                                    node.kind(),
+                                    child_node.field_name.unwrap_or("child"),
+                                    child_node.type_name
+                                ),
+                                node,
+                            ),
+                        };
                     } else {
-                        values.push(Arg::Label(child_node.label));
+                        values.push(trap::Arg::Label(child_node.label));
                     }
                 } else if field.name.is_some() {
-                    let error_message = format!(
-                        "type mismatch for field {}::{} with type {:?} != {:?}",
-                        node.kind(),
-                        child_node.field_name.unwrap_or("child"),
-                        child_node.type_name,
-                        field.type_info
+                    self.record_parse_error_for_node(
+                        format!(
+                            "type mismatch for field {}::{} with type {:?} != {:?}",
+                            node.kind(),
+                            child_node.field_name.unwrap_or("child"),
+                            child_node.type_name,
+                            field.type_info
+                        ),
+                        node,
                     );
-                    let full_error_message = format!(
-                        "{}:{}: {}",
-                        &self.path,
-                        node.start_position().row + 1,
-                        error_message
-                    );
-                    self.record_parse_error_for_node(error_message, full_error_message, *node);
                 }
             } else if child_node.field_name.is_some() || child_node.type_name.named {
-                let error_message = format!(
-                    "value for unknown field: {}::{} and type {:?}",
-                    node.kind(),
-                    &child_node.field_name.unwrap_or("child"),
-                    &child_node.type_name
+                self.record_parse_error_for_node(
+                    format!(
+                        "value for unknown field: {}::{} and type {:?}",
+                        node.kind(),
+                        &child_node.field_name.unwrap_or("child"),
+                        &child_node.type_name
+                    ),
+                    node,
                 );
-                let full_error_message = format!(
-                    "{}:{}: {}",
-                    &self.path,
-                    node.start_position().row + 1,
-                    error_message
-                );
-                self.record_parse_error_for_node(error_message, full_error_message, *node);
             }
         }
         let mut args = Vec::new();
@@ -534,23 +468,19 @@ impl Visitor<'_> {
                         args.push(child_values.first().unwrap().clone());
                     } else {
                         is_valid = false;
-                        let error_message = format!(
-                            "{} for field: {}::{}",
-                            if child_values.is_empty() {
-                                "missing value"
-                            } else {
-                                "too many values"
-                            },
-                            node.kind(),
-                            column_name
+                        self.record_parse_error_for_node(
+                            format!(
+                                "{} for field: {}::{}",
+                                if child_values.is_empty() {
+                                    "missing value"
+                                } else {
+                                    "too many values"
+                                },
+                                node.kind(),
+                                column_name
+                            ),
+                            node,
                         );
-                        let full_error_message = format!(
-                            "{}:{}: {}",
-                            &self.path,
-                            node.start_position().row + 1,
-                            error_message
-                        );
-                        self.record_parse_error_for_node(error_message, full_error_message, *node);
                     }
                 }
                 Storage::Table {
@@ -569,9 +499,9 @@ impl Visitor<'_> {
                             );
                             break;
                         }
-                        let mut args = vec![Arg::Label(parent_id)];
+                        let mut args = vec![trap::Arg::Label(parent_id)];
                         if *has_index {
-                            args.push(Arg::Int(index))
+                            args.push(trap::Arg::Int(index))
                         }
                         args.push(child_value.clone());
                         self.trap_writer.add_tuple(table_name, args);
@@ -585,55 +515,18 @@ impl Visitor<'_> {
             None
         }
     }
-
-    fn type_matches(&self, tp: &TypeName, type_info: &node_types::FieldTypeInfo) -> bool {
-        match type_info {
-            node_types::FieldTypeInfo::Single(single_type) => {
-                if tp == single_type {
-                    return true;
-                }
-                if let EntryKind::Union { members } = &self.schema.get(single_type).unwrap().kind {
-                    if self.type_matches_set(tp, members) {
-                        return true;
-                    }
-                }
-            }
-            node_types::FieldTypeInfo::Multiple { types, .. } => {
-                return self.type_matches_set(tp, types);
-            }
-
-            node_types::FieldTypeInfo::ReservedWordInt(int_mapping) => {
-                return !tp.named && int_mapping.contains_key(&tp.kind)
-            }
-        }
-        false
-    }
-
-    fn type_matches_set(&self, tp: &TypeName, types: &Set<TypeName>) -> bool {
-        if types.contains(tp) {
-            return true;
-        }
-        for other in types.iter() {
-            if let EntryKind::Union { members } = &self.schema.get(other).unwrap().kind {
-                if self.type_matches_set(tp, members) {
-                    return true;
-                }
-            }
-        }
-        false
-    }
 }
 
-// Emit a slice of a source file as an Arg.
-fn sliced_source_arg(source: &[u8], n: Node) -> Arg {
+// Emit a slice of a source file as a trap::Arg.
+fn sliced_source_arg(source: &[u8], n: &Node) -> trap::Arg {
     let range = n.byte_range();
-    Arg::String(String::from_utf8_lossy(&source[range.start..range.end]).into_owned())
+    trap::Arg::String(String::from_utf8_lossy(&source[range.start..range.end]).into_owned())
 }
 
 // Emit a pair of `TrapEntry`s for the provided node, appropriately calibrated.
 // The first is the location and label definition, and the second is the
 // 'Located' entry.
-fn location_for(source: &[u8], n: Node) -> (usize, usize, usize, usize) {
+fn location_for(source: &[u8], n: &Node) -> (usize, usize, usize, usize) {
     // Tree-sitter row, column values are 0-based while CodeQL starts
     // counting at 1. In addition Tree-sitter's row and column for the
     // end position are exclusive while CodeQL's end positions are inclusive.
@@ -680,16 +573,16 @@ fn location_for(source: &[u8], n: Node) -> (usize, usize, usize, usize) {
 
 fn traverse(tree: &Tree, visitor: &mut Visitor) {
     let cursor = &mut tree.walk();
-    visitor.enter_node(cursor.node());
+    visitor.enter_node(&cursor.node());
     let mut recurse = true;
     loop {
         if recurse && cursor.goto_first_child() {
-            recurse = visitor.enter_node(cursor.node());
+            recurse = visitor.enter_node(&cursor.node());
         } else {
-            visitor.leave_node(cursor.field_name(), cursor.node());
+            visitor.leave_node(cursor.field_name(), &cursor.node());
 
             if cursor.goto_next_sibling() {
-                recurse = visitor.enter_node(cursor.node());
+                recurse = visitor.enter_node(&cursor.node());
             } else if cursor.goto_parent() {
                 recurse = false;
             } else {
@@ -699,59 +592,6 @@ fn traverse(tree: &Tree, visitor: &mut Visitor) {
     }
 }
 
-pub struct Program(Vec<TrapEntry>);
-
-impl fmt::Display for Program {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        let mut text = String::new();
-        for trap_entry in &self.0 {
-            text.push_str(&format!("{}\n", trap_entry));
-        }
-        write!(f, "{}", text)
-    }
-}
-
-enum TrapEntry {
-    /// Maps the label to a fresh id, e.g. `#123=*`.
-    FreshId(Label),
-    /// Maps the label to a key, e.g. `#7=@"foo"`.
-    MapLabelToKey(Label, String),
-    /// foo_bar(arg*)
-    GenericTuple(String, Vec<Arg>),
-    Comment(String),
-}
-impl fmt::Display for TrapEntry {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        match self {
-            TrapEntry::FreshId(label) => write!(f, "{}=*", label),
-            TrapEntry::MapLabelToKey(label, key) => {
-                write!(f, "{}=@\"{}\"", label, key.replace("\"", "\"\""))
-            }
-            TrapEntry::GenericTuple(name, args) => {
-                write!(f, "{}(", name)?;
-                for (index, arg) in args.iter().enumerate() {
-                    if index > 0 {
-                        write!(f, ",")?;
-                    }
-                    write!(f, "{}", arg)?;
-                }
-                write!(f, ")")
-            }
-            TrapEntry::Comment(line) => write!(f, "// {}", line),
-        }
-    }
-}
-
-#[derive(Debug, Copy, Clone)]
-// Identifiers of the form #0, #1...
-struct Label(u32);
-
-impl fmt::Display for Label {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "#{:x}", self.0)
-    }
-}
-
 // Numeric indices.
 #[derive(Debug, Copy, Clone)]
 struct Index(usize);
@@ -761,69 +601,3 @@ impl fmt::Display for Index {
         write!(f, "{}", self.0)
     }
 }
-
-// Some untyped argument to a TrapEntry.
-#[derive(Debug, Clone)]
-enum Arg {
-    Label(Label),
-    Int(usize),
-    String(String),
-}
-
-const MAX_STRLEN: usize = 1048576;
-
-impl fmt::Display for Arg {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        match self {
-            Arg::Label(x) => write!(f, "{}", x),
-            Arg::Int(x) => write!(f, "{}", x),
-            Arg::String(x) => write!(
-                f,
-                "\"{}\"",
-                limit_string(x, MAX_STRLEN).replace("\"", "\"\"")
-            ),
-        }
-    }
-}
-
-/// Limit the length (in bytes) of a string. If the string's length in bytes is
-/// less than or equal to the limit then the entire string is returned. Otherwise
-/// the string is sliced at the provided limit. If there is a multi-byte character
-/// at the limit then the returned slice will be slightly shorter than the limit to
-/// avoid splitting that multi-byte character.
-fn limit_string(string: &str, max_size: usize) -> &str {
-    if string.len() <= max_size {
-        return string;
-    }
-    let p = string.as_bytes();
-    let mut index = max_size;
-    // We want to clip the string at [max_size]; however, the character at that position
-    // may span several bytes. We need to find the first byte of the character. In UTF-8
-    // encoded data any byte that matches the bit pattern 10XXXXXX is not a start byte.
-    // Therefore we decrement the index as long as there are bytes matching this pattern.
-    // This ensures we cut the string at the border between one character and another.
-    while index > 0 && (p[index] & 0b11000000) == 0b10000000 {
-        index -= 1;
-    }
-    &string[0..index]
-}
-
-#[test]
-fn limit_string_test() {
-    assert_eq!("hello", limit_string(&"hello world".to_owned(), 5));
-    assert_eq!("hi ☹", limit_string(&"hi ☹☹".to_owned(), 6));
-    assert_eq!("hi ", limit_string(&"hi ☹☹".to_owned(), 5));
-}
-
-#[test]
-fn escape_key_test() {
-    assert_eq!("foo!", escape_key("foo!"));
-    assert_eq!("foo&lbrace;&rbrace;", escape_key("foo{}"));
-    assert_eq!("&lbrace;&rbrace;", escape_key("{}"));
-    assert_eq!("", escape_key(""));
-    assert_eq!("/path/to/foo.rb", escape_key("/path/to/foo.rb"));
-    assert_eq!(
-        "/path/to/foo&amp;&lbrace;&rbrace;&quot;&commat;&num;.rb",
-        escape_key("/path/to/foo&{}\"@#.rb")
-    );
-}

ruby/extractor/src/main.rs

@@ -1,51 +1,15 @@
 mod extractor;
+mod trap;
 
 extern crate num_cpus;
 
 use clap::arg;
-use flate2::write::GzEncoder;
 use rayon::prelude::*;
 use std::fs;
-use std::io::{BufRead, BufWriter};
+use std::io::BufRead;
 use std::path::{Path, PathBuf};
 use tree_sitter::{Language, Parser, Range};
 
-enum TrapCompression {
-    None,
-    Gzip,
-}
-
-impl TrapCompression {
-    fn from_env() -> TrapCompression {
-        match std::env::var("CODEQL_RUBY_TRAP_COMPRESSION") {
-            Ok(method) => match TrapCompression::from_string(&method) {
-                Some(c) => c,
-                None => {
-                    tracing::error!("Unknown compression method '{}'; using gzip.", &method);
-                    TrapCompression::Gzip
-                }
-            },
-            // Default compression method if the env var isn't set:
-            Err(_) => TrapCompression::Gzip,
-        }
-    }
-
-    fn from_string(s: &str) -> Option<TrapCompression> {
-        match s.to_lowercase().as_ref() {
-            "none" => Some(TrapCompression::None),
-            "gzip" => Some(TrapCompression::Gzip),
-            _ => None,
-        }
-    }
-
-    fn extension(&self) -> &str {
-        match self {
-            TrapCompression::None => "trap",
-            TrapCompression::Gzip => "trap.gz",
-        }
-    }
-}
-
 /**
  * Gets the number of threads the extractor should use, by reading the
  * CODEQL_THREADS environment variable and using it as described in the
@@ -118,7 +82,7 @@ fn main() -> std::io::Result<()> {
         .value_of("output-dir")
         .expect("missing --output-dir");
     let trap_dir = PathBuf::from(trap_dir);
-    let trap_compression = TrapCompression::from_env();
+    let trap_compression = trap::Compression::from_env("CODEQL_RUBY_TRAP_COMPRESSION");
 
     let file_list = matches.value_of("file-list").expect("missing --file-list");
     let file_list = fs::File::open(file_list)?;
@@ -141,7 +105,7 @@ fn main() -> std::io::Result<()> {
             let src_archive_file = path_for(&src_archive_dir, &path, "");
             let mut source = std::fs::read(&path)?;
             let code_ranges;
-            let mut trap_writer = extractor::new_trap_writer();
+            let mut trap_writer = trap::Writer::new();
             if path.extension().map_or(false, |x| x == "erb") {
                 tracing::info!("scanning: {}", path.display());
                 extractor::extract(
@@ -186,28 +150,20 @@ fn main() -> std::io::Result<()> {
         .expect("failed to extract files");
 
     let path = PathBuf::from("extras");
-    let mut trap_writer = extractor::new_trap_writer();
-    trap_writer.populate_empty_location();
+    let mut trap_writer = trap::Writer::new();
+    extractor::populate_empty_location(&mut trap_writer);
     write_trap(&trap_dir, path, trap_writer, &trap_compression)
 }
 
 fn write_trap(
     trap_dir: &Path,
     path: PathBuf,
-    trap_writer: extractor::TrapWriter,
-    trap_compression: &TrapCompression,
+    trap_writer: trap::Writer,
+    trap_compression: &trap::Compression,
 ) -> std::io::Result<()> {
     let trap_file = path_for(trap_dir, &path, trap_compression.extension());
     std::fs::create_dir_all(&trap_file.parent().unwrap())?;
-    let trap_file = std::fs::File::create(&trap_file)?;
-    let mut trap_file = BufWriter::new(trap_file);
-    match trap_compression {
-        TrapCompression::None => trap_writer.output(&mut trap_file),
-        TrapCompression::Gzip => {
-            let mut compressed_writer = GzEncoder::new(trap_file, flate2::Compression::fast());
-            trap_writer.output(&mut compressed_writer)
-        }
-    }
+    trap_writer.write_to_file(&trap_file, trap_compression)
 }
 
 fn scan_erb(

ruby/extractor/src/trap.rs

@@ -0,0 +1,247 @@
+use std::borrow::Cow;
+use std::fmt;
+use std::io::BufWriter;
+use std::path::Path;
+
+use flate2::write::GzEncoder;
+use indexmap::IndexMap;
+
+pub struct Writer {
+    /// Labels that should be assigned fresh ids, e.g. `#123=*`.
+    fresh_ids: Vec<Label>,
+
+    /// Labels that should be assigned trap keys, e.g. `#7=@"foo"`.
+    global_keys: IndexMap<String, Label>,
+
+    /// Database rows to emit. Each key is the tuple name, each value is a list.
+    /// Each member of *that* list represents an instance of that tuple,
+    /// containing a list of the arguments/column values.
+    tuples: IndexMap<String, Vec<Vec<Arg>>>,
+
+    /// A counter for generating fresh labels
+    counter: u32,
+}
+
+impl Writer {
+    pub fn new() -> Writer {
+        Writer {
+            fresh_ids: Vec::new(),
+            tuples: IndexMap::new(),
+            global_keys: IndexMap::new(),
+            counter: 0,
+        }
+    }
+
+    ///  Gets a label that will hold the unique ID of the passed string at import time.
+    ///  This can be used for incrementally importable TRAP files -- use globally unique
+    ///  strings to compute a unique ID for table tuples.
+    ///
+    ///  Note: You probably want to make sure that the key strings that you use are disjoint
+    ///  for disjoint column types; the standard way of doing this is to prefix (or append)
+    ///  the column type name to the ID. Thus, you might identify methods in Java by the
+    ///  full ID "methods_com.method.package.DeclaringClass.method(argumentList)".
+    pub fn fresh_id(&mut self) -> Label {
+        let label = Label(self.counter);
+        self.counter += 1;
+        self.fresh_ids.push(label);
+        label
+    }
+
+    pub fn global_id(&mut self, key: String) -> (Label, bool) {
+        if let Some(label) = self.global_keys.get(&key) {
+            return (*label, false);
+        }
+        let label = Label(self.counter);
+        self.counter += 1;
+        self.global_keys.insert(key, label);
+        (label, true)
+    }
+
+    pub fn add_tuple(&mut self, table_name: &str, args: Vec<Arg>) {
+        self.tuples
+            .entry(table_name.to_owned())
+            .or_insert_with(Vec::new)
+            .push(args);
+    }
+
+    fn write<T: std::io::Write>(&self, dest: &mut T) -> std::io::Result<()> {
+        for label in &self.fresh_ids {
+            writeln!(dest, "{}=*", label)?;
+        }
+        for (key, label) in &self.global_keys {
+            writeln!(dest, "{}=@\"{}\"", label, key.replace("\"", "\"\""))?;
+        }
+        for (name, instances) in &self.tuples {
+            for instance in instances {
+                write!(dest, "{}(", name)?;
+                for (index, arg) in instance.iter().enumerate() {
+                    if index > 0 {
+                        write!(dest, ",")?;
+                    }
+                    write!(dest, "{}", arg)?;
+                }
+                writeln!(dest, ")")?;
+            }
+        }
+        Ok(())
+    }
+
+    pub fn write_to_file(&self, path: &Path, compression: &Compression) -> std::io::Result<()> {
+        let trap_file = std::fs::File::create(path)?;
+        let mut trap_file = BufWriter::new(trap_file);
+        match compression {
+            Compression::None => self.write(&mut trap_file),
+            Compression::Gzip => {
+                let mut compressed_writer = GzEncoder::new(trap_file, flate2::Compression::fast());
+                self.write(&mut compressed_writer)
+            }
+        }
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+// Identifiers of the form #0, #1...
+pub struct Label(u32);
+
+impl fmt::Display for Label {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "#{:x}", self.0)
+    }
+}
+
+// Some untyped argument to a TrapEntry.
+#[derive(Debug, Clone)]
+pub enum Arg {
+    Label(Label),
+    Int(usize),
+    String(String),
+}
+
+const MAX_STRLEN: usize = 1048576;
+
+impl fmt::Display for Arg {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            Arg::Label(x) => write!(f, "{}", x),
+            Arg::Int(x) => write!(f, "{}", x),
+            Arg::String(x) => write!(
+                f,
+                "\"{}\"",
+                limit_string(x, MAX_STRLEN).replace("\"", "\"\"")
+            ),
+        }
+    }
+}
+
+pub fn full_id_for_file(normalized_path: &str) -> String {
+    format!("{};sourcefile", escape_key(normalized_path))
+}
+
+pub fn full_id_for_folder(normalized_path: &str) -> String {
+    format!("{};folder", escape_key(normalized_path))
+}
+
+/// Escapes a string for use in a TRAP key, by replacing special characters with
+/// HTML entities.
+fn escape_key<'a, S: Into<Cow<'a, str>>>(key: S) -> Cow<'a, str> {
+    fn needs_escaping(c: char) -> bool {
+        matches!(c, '&' | '{' | '}' | '"' | '@' | '#')
+    }
+
+    let key = key.into();
+    if key.contains(needs_escaping) {
+        let mut escaped = String::with_capacity(2 * key.len());
+        for c in key.chars() {
+            match c {
+                '&' => escaped.push_str("&amp;"),
+                '{' => escaped.push_str("&lbrace;"),
+                '}' => escaped.push_str("&rbrace;"),
+                '"' => escaped.push_str("&quot;"),
+                '@' => escaped.push_str("&commat;"),
+                '#' => escaped.push_str("&num;"),
+                _ => escaped.push(c),
+            }
+        }
+        Cow::Owned(escaped)
+    } else {
+        key
+    }
+}
+
+/// Limit the length (in bytes) of a string. If the string's length in bytes is
+/// less than or equal to the limit then the entire string is returned. Otherwise
+/// the string is sliced at the provided limit. If there is a multi-byte character
+/// at the limit then the returned slice will be slightly shorter than the limit to
+/// avoid splitting that multi-byte character.
+fn limit_string(string: &str, max_size: usize) -> &str {
+    if string.len() <= max_size {
+        return string;
+    }
+    let p = string.as_bytes();
+    let mut index = max_size;
+    // We want to clip the string at [max_size]; however, the character at that position
+    // may span several bytes. We need to find the first byte of the character. In UTF-8
+    // encoded data any byte that matches the bit pattern 10XXXXXX is not a start byte.
+    // Therefore we decrement the index as long as there are bytes matching this pattern.
+    // This ensures we cut the string at the border between one character and another.
+    while index > 0 && (p[index] & 0b11000000) == 0b10000000 {
+        index -= 1;
+    }
+    &string[0..index]
+}
+
+pub enum Compression {
+    None,
+    Gzip,
+}
+
+impl Compression {
+    pub fn from_env(var_name: &str) -> Compression {
+        match std::env::var(var_name) {
+            Ok(method) => match Compression::from_string(&method) {
+                Some(c) => c,
+                None => {
+                    tracing::error!("Unknown compression method '{}'; using gzip.", &method);
+                    Compression::Gzip
+                }
+            },
+            // Default compression method if the env var isn't set:
+            Err(_) => Compression::Gzip,
+        }
+    }
+
+    pub fn from_string(s: &str) -> Option<Compression> {
+        match s.to_lowercase().as_ref() {
+            "none" => Some(Compression::None),
+            "gzip" => Some(Compression::Gzip),
+            _ => None,
+        }
+    }
+
+    pub fn extension(&self) -> &str {
+        match self {
+            Compression::None => "trap",
+            Compression::Gzip => "trap.gz",
+        }
+    }
+}
+
+#[test]
+fn limit_string_test() {
+    assert_eq!("hello", limit_string(&"hello world".to_owned(), 5));
+    assert_eq!("hi ☹", limit_string(&"hi ☹☹".to_owned(), 6));
+    assert_eq!("hi ", limit_string(&"hi ☹☹".to_owned(), 5));
+}
+
+#[test]
+fn escape_key_test() {
+    assert_eq!("foo!", escape_key("foo!"));
+    assert_eq!("foo&lbrace;&rbrace;", escape_key("foo{}"));
+    assert_eq!("&lbrace;&rbrace;", escape_key("{}"));
+    assert_eq!("", escape_key(""));
+    assert_eq!("/path/to/foo.rb", escape_key("/path/to/foo.rb"));
+    assert_eq!(
+        "/path/to/foo&amp;&lbrace;&rbrace;&quot;&commat;&num;.rb",
+        escape_key("/path/to/foo&{}\"@#.rb")
+    );
+}

ruby/generator/src/main.rs

@@ -20,8 +20,8 @@ fn make_field_type<'a>(
     field: &'a node_types::Field,
     nodes: &'a node_types::NodeTypeMap,
 ) -> (ql::Type<'a>, Option<dbscheme::Entry<'a>>) {
-    match &field.type_info {
-        node_types::FieldTypeInfo::Multiple {
+    match &field.type_info.kind {
+        node_types::FieldTypeKind::Multiple {
             types,
             dbscheme_union,
             ql_class: _,
@@ -40,11 +40,11 @@ fn make_field_type<'a>(
                 })),
             )
         }
-        node_types::FieldTypeInfo::Single(t) => {
+        node_types::FieldTypeKind::Single(t) => {
             let dbscheme_name = &nodes.get(t).unwrap().dbscheme_name;
             (ql::Type::At(dbscheme_name), None)
         }
-        node_types::FieldTypeInfo::ReservedWordInt(int_mapping) => {
+        node_types::FieldTypeKind::ReservedWordInt(int_mapping) => {
             // The field will be an `int` in the db, and we add a case split to
             // create other db types for each integer value.
             let mut branches: Vec<(usize, &'a str)> = Vec::new();

ruby/generator/src/ql_gen.rs

@@ -345,16 +345,16 @@ fn create_field_getters<'a>(
     field: &'a node_types::Field,
     nodes: &'a node_types::NodeTypeMap,
 ) -> (ql::Predicate<'a>, Option<ql::Expression<'a>>) {
-    let return_type = match &field.type_info {
-        node_types::FieldTypeInfo::Single(t) => {
+    let return_type = match &field.type_info.kind {
+        node_types::FieldTypeKind::Single(t) => {
             Some(ql::Type::Normal(&nodes.get(t).unwrap().ql_class_name))
         }
-        node_types::FieldTypeInfo::Multiple {
+        node_types::FieldTypeKind::Multiple {
             types: _,
             dbscheme_union: _,
             ql_class,
         } => Some(ql::Type::Normal(ql_class)),
-        node_types::FieldTypeInfo::ReservedWordInt(_) => Some(ql::Type::String),
+        node_types::FieldTypeKind::ReservedWordInt(_) => Some(ql::Type::String),
     };
     let formal_parameters = match &field.storage {
         node_types::Storage::Column { .. } => vec![],
@@ -372,10 +372,10 @@ fn create_field_getters<'a>(
 
     // For the expression to get a value, what variable name should the result
     // be bound to?
-    let get_value_result_var_name = match &field.type_info {
-        node_types::FieldTypeInfo::ReservedWordInt(_) => "value",
-        node_types::FieldTypeInfo::Single(_) => "result",
-        node_types::FieldTypeInfo::Multiple { .. } => "result",
+    let get_value_result_var_name = match &field.type_info.kind {
+        node_types::FieldTypeKind::ReservedWordInt(_) => "value",
+        node_types::FieldTypeKind::Single(_) => "result",
+        node_types::FieldTypeKind::Multiple { .. } => "result",
     };
 
     // Two expressions for getting the value. One that's suitable use in the
@@ -418,8 +418,8 @@ fn create_field_getters<'a>(
             ),
         ),
     };
-    let (body, optional_expr) = match &field.type_info {
-        node_types::FieldTypeInfo::ReservedWordInt(int_mapping) => {
+    let (body, optional_expr) = match &field.type_info.kind {
+        node_types::FieldTypeKind::ReservedWordInt(int_mapping) => {
             // Create an expression that binds the corresponding string to `result` for each `value`, e.g.:
             //   result = "foo" and value = 0 or
             //   result = "bar" and value = 1 or
@@ -454,7 +454,7 @@ fn create_field_getters<'a>(
                 None,
             )
         }
-        node_types::FieldTypeInfo::Single(_) | node_types::FieldTypeInfo::Multiple { .. } => {
+        node_types::FieldTypeKind::Single(_) | node_types::FieldTypeKind::Multiple { .. } => {
             (get_value, Some(get_value_any_index))
         }
     };

ruby/node-types/src/lib.rs

@@ -1,5 +1,5 @@
 use serde::Deserialize;
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, HashMap, HashSet};
 use std::path::Path;
 
 use std::collections::BTreeSet as Set;
@@ -22,14 +22,23 @@ pub enum EntryKind {
     Token { kind_id: usize },
 }
 
-#[derive(Debug, Ord, PartialOrd, Eq, PartialEq)]
+#[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Hash, Clone)]
 pub struct TypeName {
     pub kind: String,
     pub named: bool,
 }
 
-#[derive(Debug)]
-pub enum FieldTypeInfo {
+#[derive(Debug, Eq, PartialEq)]
+pub struct FieldTypeInfo {
+    pub kind: FieldTypeKind,
+
+    /// The set of types this field is allowed to take, after recursively
+    /// expanding subtypes.
+    pub valid_types: HashSet<TypeName>,
+}
+
+#[derive(Debug, Eq, PartialEq)]
+pub enum FieldTypeKind {
     /// The field has a single type.
     Single(TypeName),
 
@@ -103,7 +112,49 @@ fn convert_types(node_types: &[NodeType]) -> Set<TypeName> {
     node_types.iter().map(convert_type).collect()
 }
 
+fn get_matching_types(
+    node: &NodeInfo,
+    type_map: &HashMap<NodeType, &NodeInfo>,
+) -> HashSet<TypeName> {
+    let mut result = HashSet::new();
+    let node_type_name = TypeName {
+        kind: node.kind.clone(),
+        named: node.named,
+    };
+    result.insert(node_type_name);
+    if let Some(subtypes) = &node.subtypes {
+        for subtype in subtypes {
+            let subtype = type_map.get(subtype).unwrap();
+            result.extend(get_matching_types(subtype, type_map));
+        }
+    }
+    result
+}
+
 pub fn convert_nodes(prefix: &str, nodes: &[NodeInfo]) -> NodeTypeMap {
+    // Since the nodes contain only weak references to their subtypes, build a
+    // map so we can resolve them.
+    let mut type_map: HashMap<NodeType, &NodeInfo> = HashMap::new();
+    for node in nodes {
+        let node_type = NodeType {
+            kind: node.kind.clone(),
+            named: node.named,
+        };
+        type_map.insert(node_type, node);
+    }
+
+    // Now recursively expand subtypes so that for each tree-sitter node type,
+    // we have a set of all matching types.
+    let mut transitive_type_map = HashMap::new();
+    for node in nodes {
+        let type_name = TypeName {
+            kind: node.kind.clone(),
+            named: node.named,
+        };
+        let matching_types = get_matching_types(node, &type_map);
+        transitive_type_map.insert(type_name, matching_types);
+    }
+
     let mut entries = NodeTypeMap::new();
     let mut token_kinds = Set::new();
 
@@ -166,6 +217,7 @@ pub fn convert_nodes(prefix: &str, nodes: &[NodeInfo]) -> NodeTypeMap {
                         field_info,
                         &mut fields,
                         &token_kinds,
+                        &transitive_type_map,
                     );
                 }
             }
@@ -178,6 +230,7 @@ pub fn convert_nodes(prefix: &str, nodes: &[NodeInfo]) -> NodeTypeMap {
                     children,
                     &mut fields,
                     &token_kinds,
+                    &transitive_type_map,
                 );
             }
             entries.insert(
@@ -222,6 +275,7 @@ fn add_field(
     field_info: &FieldInfo,
     fields: &mut Vec<Field>,
     token_kinds: &Set<TypeName>,
+    transitive_type_map: &HashMap<TypeName, HashSet<TypeName>>,
 ) {
     let parent_flattened_name = node_type_name(&parent_type_name.kind, parent_type_name.named);
     let column_name = escape_name(&name_for_field_or_child(&field_name));
@@ -245,6 +299,18 @@ fn add_field(
         }
     };
     let converted_types = convert_types(&field_info.types);
+
+    // Use the transitive type map we built earlier to create the set of all
+    // possible types this field could take.
+    let mut valid_types: HashSet<TypeName> = HashSet::new();
+    for type_name in &converted_types {
+        if let Some(types) = transitive_type_map.get(type_name) {
+            for t in types {
+                valid_types.insert(t.clone());
+            }
+        }
+    }
+
     let type_info = if field_info
         .types
         .iter()
@@ -259,20 +325,29 @@ fn add_field(
                 escape_name(&format!("{}_{}_{}", &prefix, parent_flattened_name, t.kind));
             field_token_ints.insert(t.kind.to_owned(), (counter, dbscheme_variant_name));
         }
-        FieldTypeInfo::ReservedWordInt(field_token_ints)
+        FieldTypeInfo {
+            kind: FieldTypeKind::ReservedWordInt(field_token_ints),
+            valid_types,
+        }
     } else if field_info.types.len() == 1 {
-        FieldTypeInfo::Single(converted_types.into_iter().next().unwrap())
+        FieldTypeInfo {
+            kind: FieldTypeKind::Single(converted_types.into_iter().next().unwrap()),
+            valid_types,
+        }
     } else {
         // The dbscheme type for this field will be a union. In QL, it'll just be AstNode.
-        FieldTypeInfo::Multiple {
-            types: converted_types,
-            dbscheme_union: format!(
-                "{}_{}_{}_type",
-                &prefix,
-                &parent_flattened_name,
-                &name_for_field_or_child(&field_name)
-            ),
-            ql_class: "AstNode".to_owned(),
+        FieldTypeInfo {
+            kind: FieldTypeKind::Multiple {
+                types: converted_types,
+                dbscheme_union: format!(
+                    "{}_{}_{}_type",
+                    &prefix,
+                    &parent_flattened_name,
+                    &name_for_field_or_child(&field_name)
+                ),
+                ql_class: "AstNode".to_owned(),
+            },
+            valid_types,
         }
     };
     let getter_name = format!(
@@ -303,7 +378,7 @@ pub struct NodeInfo {
     pub subtypes: Option<Vec<NodeType>>,
 }
 
-#[derive(Deserialize)]
+#[derive(Deserialize, Hash, Eq, PartialEq)]
 pub struct NodeType {
     #[serde(rename = "type")]
     pub kind: String,

ruby/ql/examples/qlpack.yml

@@ -1,6 +1,4 @@
 name: codeql/ruby-examples
-groups: 
-  - ruby
-  - examples
+version: 0.0.2
 dependencies:
   codeql/ruby-all: ^0.0.2

ruby/ql/lib/codeql/ruby/controlflow/internal/ControlFlowGraphImplShared.qll

@@ -894,7 +894,7 @@ module TestOutput {
             p
             order by
               l.getFile().getBaseName(), l.getFile().getAbsolutePath(), l.getStartLine(),
-              l.getStartColumn(), l.getEndLine(), l.getEndColumn(), p.toString()
+              l.getStartColumn()
           )
       ).toString()
   }