Ruby: update crate versions

JS: DB reads as taint sources

benjamin-button.md

@@ -1,51 +0,0 @@
-# benjamin-buttons.md
-
-This file describes the changes that have been applied to
-the library to make it behave as if it was younger.
-
-## TaintedPath.ql
-
-Sinks added between 2020-01-01 and 2020-10-06 have been removed. Found by looking at:
-
-- the commit titles of https://github.com/github/codeql/commits/main/javascript/ql/test/query-tests/Security/CWE-022/TaintedPath/TaintedPath.expected
-- the PR titles of https://github.com/github/codeql/pulls?page=2&q=is%3Apr+label%3AJS+is%3Aclosed+sink
-
-Sinks added between 2018-08-02 and 2020-01-01 have been removed. Found by looking at:
-
-- the commit titles of https://github.com/github/codeql/commits/main/javascript/ql/test/query-tests/Security/CWE-022/TaintedPath/TaintedPath.expected
-- the PR titles of https://github.com/github/codeql/pulls?page=2&q=is%3Apr+label%3AJS+is%3Aclosed+sink
-- the PR titles of https://github.com/github/codeql/pulls?page=2&q=is%3Apr+label%3AJS+is%3Aclosed+pathinjection
-- the PR titles of https://github.com/github/codeql/pulls?page=2&q=is%3Apr+label%3AJS+is%3Aclosed+tainted-path
-
-Sinks from the "graceful-fs" and "fs-extra" (added before the open-sourcing squash).
-
-## Xss.ql
-
-Sinks added between 2020-01-01 and 2020-10-06 have been removed. Found by looking at:
-
-- the commit titles of https://github.com/github/codeql/commits/main/javascript/ql/test/query-tests/Security/CWE-079/Xss.expected
-- the PR titles of https://github.com/github/codeql/pulls?page=2&q=is%3Apr+label%3AJS+is%3Aclosed+sink
-
-- recursive type tracking for `jQuery::dollar`, `DOM::domValueRef`.
-
-## SqlInjection.ql
-
-Sinks added between 2020-01-01 and 2020-10-06 have been removed. Found by looking at:
-
-- the commit titles of https://github.com/github/codeql/commits/main/javascript/ql/test/query-tests/Security/CWE-089
-- the PR titles of https://github.com/github/codeql/pulls?page=2&q=is%3Apr+label%3AJS+is%3Aclosed+sink
-
-Sinks added between 2018-08-02 and 2020-01-01 have been removed. Found by looking at:
-
-- the commit titles of https://github.com/github/codeql/commits/main/javascript/ql/test/query-tests/Security/CWE-089
-- the PR titles of https://github.com/github/codeql/pulls?page=2&q=is%3Apr+label%3AJS+is%3Aclosed+sink
-- the PR titles of https://github.com/github/codeql/pulls?page=2&q=is%3Apr+label%3AJS+is%3Aclosed+sql
-
-TypeTracking in SQL.qll (added before the open-sourcing squash)
-
-The model of `mssql` and `sequelize` (added before the open-sourcing squash)
-
-## PseudoProperties
-
-Pseudo-properties (`$name$`) used in type-tracking and global dataflow configurations have been disabled.
-Found by searching for `"\$.*\$"`.

cpp/ql/lib/semmle/code/cpp/dataflow/internal/FlowVar.qll

@@ -435,7 +435,7 @@ module FlowVar_internal {
       parameterIsNonConstReference(p) and
       p = v and
       // This definition reaches the exit node of the function CFG
-      getAReachedBlockVarSBB(this).getANode() = p.getFunction()
+      getAReachedBlockVarSBB(this).getEnd() = p.getFunction()
     }
 
     override predicate definedByInitialValue(StackVariable lsv) {

csharp/ql/test/library-tests/csharp10/constInterpolatedString.ql

@@ -1,7 +1,13 @@
 import csharp
 
-query predicate inserts(InterpolatedStringExpr expr, Expr e) { expr.getAnInsert() = e }
+private predicate inSpecificSource(Expr expr) {
+  expr.getFile().getBaseName() = "ConstInterpolatedString.cs"
+}
+
+query predicate inserts(InterpolatedStringExpr expr, Expr e) {
+  expr.getAnInsert() = e and inSpecificSource(expr)
+}
 
 query predicate texts(InterpolatedStringExpr expr, StringLiteral literal) {
-  expr.getAText() = literal
+  expr.getAText() = literal and inSpecificSource(expr)
 }

java/documentation/library-coverage/coverage.csv

@@ -29,7 +29,7 @@ jakarta.ws.rs.container,,9,,,,,,,,,,,,,,,,,,,,,,,9,,
 jakarta.ws.rs.core,2,,149,,,,,,,,,,,,,,,,,2,,,,,,94,55
 java.beans,,,1,,,,,,,,,,,,,,,,,,,,,,,1,
 java.io,3,,31,,3,,,,,,,,,,,,,,,,,,,,,30,1
-java.lang,8,,53,,,,,,,,,,8,,,,,,,,,,,,,42,11
+java.lang,8,,56,,,,,,,,,,8,,,,,,,,,,,,,45,11
 java.net,10,3,7,,,,,,,,,,,,,10,,,,,,,,,3,7,
 java.nio,10,,4,,10,,,,,,,,,,,,,,,,,,,,,4,
 java.sql,7,,,,,,,,,,,,,,,,,7,,,,,,,,,

java/documentation/library-coverage/coverage.rst

@@ -15,9 +15,9 @@ Java framework & library support
    `Apache HttpComponents <https://hc.apache.org/>`_,"``org.apache.hc.core5.*``, ``org.apache.http``",5,136,28,,,3,,,,25
    `Google Guava <https://guava.dev/>`_,``com.google.common.*``,,728,35,,6,,,,,
    `JSON-java <https://github.com/stleary/JSON-java>`_,``org.json``,,236,,,,,,,,
-   Java Standard Library,``java.*``,3,526,72,13,,,7,,,10
+   Java Standard Library,``java.*``,3,529,72,13,,,7,,,10
    Java extensions,"``javax.*``, ``jakarta.*``",54,552,32,,,4,,1,1,2
    `Spring <https://spring.io/>`_,``org.springframework.*``,29,469,91,,,,19,14,,29
    Others,"``androidx.slice``, ``cn.hutool.core.codec``, ``com.esotericsoftware.kryo.io``, ``com.esotericsoftware.kryo5.io``, ``com.fasterxml.jackson.core``, ``com.fasterxml.jackson.databind``, ``com.opensymphony.xwork2.ognl``, ``com.unboundid.ldap.sdk``, ``flexjson``, ``groovy.lang``, ``groovy.util``, ``jodd.json``, ``net.sf.saxon.s9api``, ``ognl``, ``org.apache.commons.codec``, ``org.apache.commons.jexl2``, ``org.apache.commons.jexl3``, ``org.apache.commons.logging``, ``org.apache.commons.ognl``, ``org.apache.directory.ldap.client.api``, ``org.apache.ibatis.jdbc``, ``org.apache.log4j``, ``org.apache.logging.log4j``, ``org.apache.shiro.codec``, ``org.apache.shiro.jndi``, ``org.codehaus.groovy.control``, ``org.dom4j``, ``org.hibernate``, ``org.jboss.logging``, ``org.jooq``, ``org.mvel2``, ``org.scijava.log``, ``org.slf4j``, ``org.xml.sax``, ``org.xmlpull.v1``, ``play.mvc``, ``ratpack.core.form``, ``ratpack.core.handling``, ``ratpack.core.http``, ``ratpack.exec``, ``ratpack.form``, ``ratpack.func``, ``ratpack.handling``, ``ratpack.http``, ``ratpack.util``",44,283,919,,,,14,18,,
-   Totals,,180,5639,1276,13,6,10,107,33,1,66
+   Totals,,180,5642,1276,13,6,10,107,33,1,66
 

javascript/ql/experimental/adaptivethreatmodeling/lib/experimental/adaptivethreatmodeling/BaseScoring.qll

@@ -14,6 +14,73 @@ external predicate availableMlModels(
 /** Get the ATM configuration. */
 ATMConfig getCfg() { any() }
 
+/**
+ * This module provides functionality that takes an endpoint and provides an entity that encloses that
+ * endpoint and is suitable for similarity analysis.
+ */
+module EndpointToEntity {
+  private import CodeToFeatures
+
+  /**
+   * Get an entity enclosing the endpoint that is suitable for similarity analysis. In general,
+   * this may associate multiple entities to a single endpoint.
+   */
+  DatabaseFeatures::Entity getAnEntityForEndpoint(DataFlow::Node endpoint) {
+    DatabaseFeatures::entities(result, _, _, _, _, _, _, _, _) and
+    result.getDefinedFunction() = endpoint.getContainer().getEnclosingContainer*()
+  }
+}
+
+/**
+ * This module provides functionality that takes an entity and provides effective endpoints within
+ * that entity.
+ *
+ * We use the following terminology to describe endpoints:
+ *
+ * - The *candidate* endpoints are the set of data flow nodes that should be passed to the
+ *   appropriate endpoint filter to produce the set of effective endpoints.
+ *   When we have a model that beats the performance of the baseline, we will likely define the
+ *   candidate endpoints based on the most confident predictions of the model.
+ * - An *effective* endpoint is a candidate endpoint which passes through the endpoint filter.
+ *   In other words, it is a candidate endpoint for which the `isEffectiveSink` (or
+ *   `isEffectiveSource`) predicate defined in the `ATMConfig` instance in scope holds.
+ */
+module EntityToEffectiveEndpoint {
+  private import CodeToFeatures
+
+  /**
+   * Returns endpoint candidates within the specified entities.
+   *
+   * The baseline implementation of this is that a candidate endpoint is any data flow node that is
+   * enclosed within the specified entity.
+   */
+  private DataFlow::Node getABaselineEndpointCandidate(DatabaseFeatures::Entity entity) {
+    result.getContainer().getEnclosingContainer*() = entity.getDefinedFunction()
+  }
+
+  /**
+   * Get an effective source enclosed by the specified entity.
+   *
+   * N.B. This is _not_ an inverse of `EndpointToEntity::getAnEntityForEndpoint`: the effective
+   * source may occur in a function defined within the specified entity.
+   */
+  DataFlow::Node getAnEffectiveSource(DatabaseFeatures::Entity entity) {
+    result = getABaselineEndpointCandidate(entity) and
+    getCfg().isEffectiveSource(result)
+  }
+
+  /**
+   * Get an effective sink enclosed by the specified entity.
+   *
+   * N.B. This is _not_ an inverse of `EndpointToEntity::getAnEntityForEndpoint`: the effective
+   * sink may occur in a function defined within the specified entity.
+   */
+  DataFlow::Node getAnEffectiveSink(DatabaseFeatures::Entity entity) {
+    result = getABaselineEndpointCandidate(entity) and
+    getCfg().isEffectiveSink(result)
+  }
+}
+
 /**
  * Scoring information produced by a scoring model.
  *

javascript/ql/experimental/adaptivethreatmodeling/lib/experimental/adaptivethreatmodeling/CodeToFeatures.qll

@@ -0,0 +1,444 @@
+/*
+ * For internal use only.
+ *
+ * Extracts data about the functions in the database for use in adaptive threat modeling (ATM).
+ */
+
+module Raw {
+  private import javascript as raw
+
+  class RawAstNode = raw::ASTNode;
+
+  class Entity = raw::Function;
+
+  class Location = raw::Location;
+
+  /**
+   * Exposed as a tool for defining anchors for semantic search.
+   */
+  class UnderlyingFunction = raw::Function;
+
+  /**
+   * Determines whether an entity should be omitted from ATM.
+   */
+  predicate isEntityIgnored(Entity entity) {
+    // Ignore entities which don't have definitions, for example those in TypeScript
+    // declaration files.
+    not exists(entity.getBody())
+    or
+    // Ignore entities with an empty body, for example the JavaScript function () => {}.
+    entity.getNumBodyStmt() = 0 and not exists(entity.getAReturnedExpr())
+  }
+
+  newtype WrappedAstNode = TAstNode(RawAstNode rawNode)
+
+  /**
+   * This class represents nodes in the AST.
+   */
+  class AstNode extends TAstNode {
+    RawAstNode rawNode;
+
+    AstNode() { this = TAstNode(rawNode) }
+
+    AstNode getAChildNode() { result = TAstNode(rawNode.getAChild()) }
+
+    AstNode getParentNode() { result = TAstNode(rawNode.getParent()) }
+
+    /**
+     * Holds if the AST node has `result` as its `index`th attribute.
+     *
+     * The index is not intended to mean anything, and is only here for disambiguation.
+     * There are no guarantees about any particular index being used (or not being used).
+     */
+    string astNodeAttribute(int index) {
+      (
+        // NB: Unary and binary operator expressions e.g. -a, a + b and compound
+        // assignments e.g. a += b can be identified by the expression type.
+        result = rawNode.(raw::Identifier).getName()
+        or
+        // Computed property accesses for which we can predetermine the property being accessed.
+        // NB: May alias with operators e.g. could have '+' as a property name.
+        result = rawNode.(raw::IndexExpr).getPropertyName()
+        or
+        // We use `getRawValue` to give us distinct representations for `0xa`, `0xA`, and `10`.
+        result = rawNode.(raw::NumberLiteral).getRawValue()
+        or
+        // We use `getValue` rather than `getRawValue` so we assign `"a"` and `'a'` the same representation.
+        not rawNode instanceof raw::NumberLiteral and
+        result = rawNode.(raw::Literal).getValue()
+        or
+        result = rawNode.(raw::TemplateElement).getRawValue()
+      ) and
+      index = 0
+    }
+
+    /**
+     * Returns a string indicating the "type" of the AST node.
+     */
+    string astNodeType() {
+      // The definition of this method should correspond with that of the `@ast_node` entry in the
+      // dbscheme.
+      result = "js_exprs." + any(int kind | exprs(rawNode, kind, _, _, _))
+      or
+      result = "js_properties." + any(int kind | properties(rawNode, _, _, kind, _))
+      or
+      result = "js_stmts." + any(int kind | stmts(rawNode, kind, _, _, _))
+      or
+      result = "js_toplevel" and rawNode instanceof raw::TopLevel
+      or
+      result = "js_typeexprs." + any(int kind | typeexprs(rawNode, kind, _, _, _))
+    }
+
+    /**
+     * Holds if `result` is the `index`'th child of the AST node, for some arbitrary indexing.
+     * A root of the AST should be its own child, with an arbitrary (though conventionally
+     * 0) index.
+     *
+     * Notably, the order in which child nodes are visited is not required to be meaningful,
+     * and no particular index is required to be meaningful. However, `(parent, index)`
+     * should be a keyset.
+     */
+    pragma[nomagic]
+    AstNode astNodeChild(int index) {
+      result =
+        rank[index - 1](AstNode child, raw::Location l |
+          child = this.getAChildNode() and l = child.getLocation()
+        |
+          child
+          order by
+            l.getStartLine(), l.getStartColumn(), l.getEndLine(), l.getEndColumn(),
+            child.astNodeType()
+        )
+      or
+      not exists(result.getParentNode()) and this = result and index = 0
+    }
+
+    raw::Location getLocation() { result = rawNode.getLocation() }
+
+    string toString() { result = rawNode.toString() }
+
+    predicate isEntityNameNode(Entity entity) {
+      exists(int index |
+        TAstNode(entity) = this.getParentNode() and
+        this = this.getParentNode().astNodeChild(index) and
+        // An entity name node must be the first child of the entity.
+        index = min(int otherIndex | exists(this.getParentNode().astNodeChild(otherIndex))) and
+        entity.getName() = rawNode.(raw::VarDecl).getName()
+      )
+    }
+  }
+
+  /**
+   * Holds if `result` is the `index`'th child of the `parent` entity. Such
+   * a node is a root of an AST associated with this entity.
+   */
+  AstNode entityChild(AstNode parent, int index) {
+    // In JavaScript, entities appear in the AST parent/child relationship.
+    result = parent.astNodeChild(index)
+  }
+
+  /**
+   * Holds if `node` is contained in `entity`. Note that a single node may be contained
+   * in multiple entities, if they are nested. An entity, in particular, should be
+   * reported as contained within itself.
+   */
+  predicate entityContains(Entity entity, AstNode node) {
+    node.getParentNode*() = TAstNode(entity) and not node.isEntityNameNode(entity)
+  }
+
+  /**
+   * Get the name of the entity.
+   *
+   * We attempt to assign unnamed entities approximate names if they are passed to a likely
+   * external library function. If we can't assign them an approximate name, we give them the name
+   * `""`, so that these entities are included in `AdaptiveThreatModeling.qll`.
+   *
+   * For entities which have multiple names, we choose the lexically smallest name.
+   */
+  string getEntityName(Entity entity) {
+    if exists(entity.getName())
+    then
+      // https://github.com/github/ml-ql-adaptive-threat-modeling/issues/244 discusses making use
+      // of all the names during training.
+      result = min(entity.getName())
+    else
+      if exists(getApproximateNameForEntity(entity))
+      then result = getApproximateNameForEntity(entity)
+      else result = ""
+  }
+
+  /**
+   * Holds if the call `call` has `entity` is its `argumentIndex`th argument.
+   */
+  private predicate entityUsedAsArgumentToCall(
+    Entity entity, raw::DataFlow::CallNode call, int argumentIndex
+  ) {
+    raw::DataFlow::localFlowStep*(call.getArgument(argumentIndex), entity.flow())
+  }
+
+  /**
+   * Returns a generated name for the entity. This name is generated such that
+   * entities with the same names have similar behavior.
+   */
+  private string getApproximateNameForEntity(Entity entity) {
+    count(raw::DataFlow::CallNode call, int index | entityUsedAsArgumentToCall(entity, call, index)) =
+      1 and
+    exists(raw::DataFlow::CallNode call, int index, string basePart |
+      entityUsedAsArgumentToCall(entity, call, index) and
+      (
+        if count(getReceiverName(call)) = 1
+        then basePart = getReceiverName(call) + "."
+        else basePart = ""
+      ) and
+      result = basePart + call.getCalleeName() + "#functionalargument"
+    )
+  }
+
+  private string getReceiverName(raw::DataFlow::CallNode call) {
+    result = call.getReceiver().asExpr().(raw::VarAccess).getName()
+  }
+
+  /** Consistency checks: these predicates should each have no results */
+  module Consistency {
+    /** `getEntityName` should assign each entity a single name. */
+    query predicate entityWithManyNames(Entity entity, string name) {
+      name = getEntityName(entity) and
+      count(getEntityName(entity)) > 1
+    }
+
+    query predicate nodeWithNoType(AstNode node) { not exists(node.astNodeType()) }
+
+    query predicate nodeWithManyTypes(AstNode node, string type) {
+      type = node.astNodeType() and
+      count(node.astNodeType()) > 1
+    }
+
+    query predicate nodeWithNoParent(AstNode node, string type) {
+      not node = any(AstNode parent).astNodeChild(_) and
+      type = node.astNodeType() and
+      not exists(RawAstNode rawNode | node = TAstNode(rawNode) and rawNode instanceof raw::Module)
+    }
+
+    query predicate duplicateChildIndex(AstNode parent, int index, AstNode child) {
+      child = parent.astNodeChild(index) and
+      count(parent.astNodeChild(index)) > 1
+    }
+
+    query predicate duplicateAttributeIndex(AstNode node, int index) {
+      exists(node.astNodeAttribute(index)) and
+      count(node.astNodeAttribute(index)) > 1
+    }
+  }
+}
+
+module Wrapped {
+  /*
+   * We require any node with attributes to be a leaf. Where a non-leaf node
+   * has an attribute, we instead create a synthetic leaf node that has that
+   * attribute.
+   */
+
+  /**
+   * Holds if the AST node `e` is a leaf node.
+   */
+  private predicate isLeaf(Raw::AstNode e) { not exists(e.astNodeChild(_)) }
+
+  newtype WrappedEntity =
+    TEntity(Raw::Entity entity) {
+      exists(entity.getLocation().getFile().getRelativePath()) and
+      Raw::entityContains(entity, _)
+    }
+
+  /**
+   * A type ranging over the kinds of entities for which we want to consider embeddings.
+   */
+  class Entity extends WrappedEntity {
+    Raw::Entity rawEntity;
+
+    Entity() { this = TEntity(rawEntity) and not Raw::isEntityIgnored(rawEntity) }
+
+    string getName() { result = Raw::getEntityName(rawEntity) }
+
+    AstNode getAstRoot(int index) {
+      result = TAstNode(rawEntity, Raw::entityChild(Raw::TAstNode(rawEntity), index))
+    }
+
+    string toString() { result = rawEntity.toString() }
+
+    Raw::Location getLocation() { result = rawEntity.getLocation() }
+
+    Raw::UnderlyingFunction getDefinedFunction() { result = rawEntity }
+  }
+
+  newtype WrappedAstNode =
+    TAstNode(Raw::Entity enclosingEntity, Raw::AstNode node) {
+      Raw::entityContains(enclosingEntity, node)
+    } or
+    TSyntheticNode(
+      Raw::Entity enclosingEntity, Raw::AstNode node, int syntheticChildIndex, int attrIndex
+    ) {
+      Raw::entityContains(enclosingEntity, node) and
+      exists(node.astNodeAttribute(attrIndex)) and
+      not isLeaf(node) and
+      if exists(node.astNodeChild(_))
+      then
+        syntheticChildIndex =
+          attrIndex - min(int other | exists(node.astNodeAttribute(other))) +
+            max(int other | exists(node.astNodeChild(other))) + 1
+      else syntheticChildIndex = attrIndex
+    }
+
+  pragma[nomagic]
+  private AstNode injectedChild(Raw::Entity enclosingEntity, Raw::AstNode parent, int index) {
+    result = TAstNode(enclosingEntity, parent.astNodeChild(index)) or
+    result = TSyntheticNode(enclosingEntity, parent, index, _)
+  }
+
+  /**
+   * A type ranging over AST nodes. Ultimately, only nodes contained in entities will
+   * be considered.
+   */
+  class AstNode extends WrappedAstNode {
+    Raw::Entity enclosingEntity;
+    Raw::AstNode rawNode;
+
+    AstNode() {
+      (
+        this = TAstNode(enclosingEntity, rawNode) or
+        this = TSyntheticNode(enclosingEntity, rawNode, _, _)
+      ) and
+      not Raw::isEntityIgnored(enclosingEntity)
+    }
+
+    string getAttribute(int index) {
+      result = rawNode.astNodeAttribute(index) and
+      not exists(TSyntheticNode(enclosingEntity, rawNode, _, index))
+    }
+
+    string getType() { result = rawNode.astNodeType() }
+
+    AstNode getChild(int index) { result = injectedChild(enclosingEntity, rawNode, index) }
+
+    string toString() { result = this.getType() }
+
+    Raw::Location getLocation() { result = rawNode.getLocation() }
+  }
+
+  /**
+   * A synthetic AST node, created to be a leaf for an otherwise non-leaf attribute.
+   */
+  class SyntheticAstNode extends AstNode, TSyntheticNode {
+    int childIndex;
+    int attributeIndex;
+
+    SyntheticAstNode() {
+      this = TSyntheticNode(enclosingEntity, rawNode, childIndex, attributeIndex)
+    }
+
+    override string getAttribute(int index) {
+      result = rawNode.astNodeAttribute(attributeIndex) and index = attributeIndex
+    }
+
+    override string getType() {
+      result = rawNode.astNodeType() + "::<synthetic " + childIndex + ">"
+    }
+
+    override AstNode getChild(int index) { none() }
+  }
+}
+
+module DatabaseFeatures {
+  /**
+   * Exposed as a tool for defining anchors for semantic search.
+   */
+  class UnderlyingFunction = Raw::UnderlyingFunction;
+
+  private class Location = Raw::Location;
+
+  private newtype TEntityOrAstNode =
+    TEntity(Wrapped::Entity entity) or
+    TAstNode(Wrapped::AstNode astNode)
+
+  class EntityOrAstNode extends TEntityOrAstNode {
+    abstract string getType();
+
+    abstract string toString();
+
+    abstract Location getLocation();
+  }
+
+  class Entity extends EntityOrAstNode, TEntity {
+    Wrapped::Entity entity;
+
+    Entity() { this = TEntity(entity) }
+
+    string getName() { result = entity.getName() }
+
+    AstNode getAstRoot(int index) { result = TAstNode(entity.getAstRoot(index)) }
+
+    override string getType() { result = "javascript function" }
+
+    override string toString() { result = "Entity: " + this.getName() }
+
+    override Location getLocation() { result = entity.getLocation() }
+
+    UnderlyingFunction getDefinedFunction() { result = entity.getDefinedFunction() }
+  }
+
+  class AstNode extends EntityOrAstNode, TAstNode {
+    Wrapped::AstNode rawNode;
+
+    AstNode() { this = TAstNode(rawNode) }
+
+    AstNode getChild(int index) { result = TAstNode(rawNode.getChild(index)) }
+
+    string getAttribute(int index) { result = rawNode.getAttribute(index) }
+
+    override string getType() { result = rawNode.getType() }
+
+    override string toString() { result = this.getType() }
+
+    override Location getLocation() { result = rawNode.getLocation() }
+  }
+
+  /** Consistency checks: these predicates should each have no results */
+  module Consistency {
+    query predicate nonLeafAttribute(AstNode node, int index, string attribute) {
+      attribute = node.getAttribute(index) and
+      exists(node.getChild(_))
+    }
+  }
+
+  query predicate entities(
+    Entity entity, string entity_name, string entity_type, string path, int startLine,
+    int startColumn, int endLine, int endColumn, string absolutePath
+  ) {
+    entity_name = entity.getName() and
+    entity_type = entity.getType() and
+    exists(Location l | l = entity.getLocation() |
+      path = l.getFile().getRelativePath() and
+      absolutePath = l.getFile().getAbsolutePath() and
+      l.hasLocationInfo(_, startLine, startColumn, endLine, endColumn)
+    )
+  }
+
+  query predicate astNodes(
+    Entity enclosingEntity, EntityOrAstNode parent, int index, AstNode node, string node_type
+  ) {
+    node = enclosingEntity.getAstRoot(index) and
+    parent = enclosingEntity and
+    node_type = node.getType()
+    or
+    astNodes(enclosingEntity, _, _, parent, _) and
+    node = parent.(AstNode).getChild(index) and
+    node_type = node.getType()
+  }
+
+  query predicate nodeAttributes(AstNode node, string attr) {
+    // Only get attributes of AST nodes we extract.
+    // This excludes nodes in standard libraries since the standard library files
+    // are located outside the source root.
+    astNodes(_, _, _, node, _) and
+    attr = node.getAttribute(_)
+  }
+}

javascript/ql/experimental/adaptivethreatmodeling/lib/experimental/adaptivethreatmodeling/EndpointFeatures.qll

@@ -5,8 +5,21 @@
  */
 
 import javascript
-private import FeaturizationConfig
-private import FunctionBodyFeatures as FunctionBodyFeatures
+import CodeToFeatures
+private import EndpointScoring
+
+/**
+ * A configuration that defines which endpoints should be featurized.
+ *
+ * This is used as a performance optimization to ensure that we only featurize the endpoints we need
+ * to featurize.
+ */
+abstract class FeaturizationConfig extends string {
+  bindingset[this]
+  FeaturizationConfig() { any() }
+
+  abstract DataFlow::Node getAnEndpointToFeaturize();
+}
 
 /**
  * Gets the value of the token-based feature named `featureName` for the endpoint `endpoint`.
@@ -18,24 +31,14 @@ private string getTokenFeature(DataFlow::Node endpoint, string featureName) {
   endpoint = any(FeaturizationConfig cfg).getAnEndpointToFeaturize() and
   (
     // Features for endpoints that are contained within a function.
-    exists(Function function |
-      function = FunctionBodyFeatures::getRepresentativeFunctionForEndpoint(endpoint)
-    |
+    exists(DatabaseFeatures::Entity entity | entity = getRepresentativeEntityForEndpoint(endpoint) |
       // The name of the function that encloses the endpoint.
-      featureName = "enclosingFunctionName" and result = FunctionNames::getNameToFeaturize(function)
+      featureName = "enclosingFunctionName" and result = entity.getName()
       or
       // A feature containing natural language tokens from the function that encloses the endpoint in
       // the order that they appear in the source code.
       featureName = "enclosingFunctionBody" and
-      result =
-        strictconcat(string token, Location l |
-          FunctionBodyFeatures::bodyTokens(function, l, token)
-        |
-          token, " "
-          order by
-            l.getFile().getAbsolutePath(), l.getStartLine(), l.getStartColumn(), l.getEndLine(),
-            l.getEndColumn(), token
-        )
+      result = unique(string x | x = FunctionBodies::getBodyTokenFeatureForEntity(entity))
     )
     or
     result =
@@ -82,10 +85,11 @@ private string getTokenFeature(DataFlow::Node endpoint, string featureName) {
  *
  * This may in general report multiple strings, each containing a space-separated list of tokens.
  *
- * **Technical details:** This predicate can have multiple values per endpoint and feature name. As
- * a result, the results from this predicate must be concatenated together.  However concatenating
- * other features like the function body tokens is expensive, so for performance reasons we separate
- * out this predicate from those other features.
+ * **Technical details:** This predicate can have multiple values per endpoint and feature name.  As a
+ * result, the results from this predicate must be concatenated together.  However concatenating
+ * other features like the function body tokens is expensive, so we separate out this predicate
+ * from others like `FunctionBodies::getBodyTokenFeatureForEntity` to avoid having to perform this
+ * concatenation operation on other features like the function body tokens.
  */
 private string getACallBasedTokenFeatureComponent(
   DataFlow::Node endpoint, DataFlow::CallNode call, string featureName
@@ -122,6 +126,47 @@ private string getACallBasedTokenFeatureComponent(
   )
 }
 
+/** This module provides functionality for getting the function body feature associated with a particular entity. */
+module FunctionBodies {
+  /** Holds if `location` is the location of an AST node within the entity `entity` and `token` is a node attribute associated with that AST node. */
+  private predicate bodyTokens(DatabaseFeatures::Entity entity, Location location, string token) {
+    // Performance optimization: Restrict the set of entities to those containing an endpoint to featurize.
+    entity =
+      getRepresentativeEntityForEndpoint(any(FeaturizationConfig cfg).getAnEndpointToFeaturize()) and
+    // Performance optimization: If a function has more than 256 body tokens, then featurize it as
+    // absent. This approximates the behavior of the classifer on non-generic body features where
+    // large body features are replaced by the absent token.
+    //
+    // We count nodes instead of tokens because tokens are often not unique.
+    strictcount(DatabaseFeatures::AstNode node |
+      DatabaseFeatures::astNodes(entity, _, _, node, _) and
+      exists(string t | DatabaseFeatures::nodeAttributes(node, t))
+    ) <= 256 and
+    exists(DatabaseFeatures::AstNode node |
+      DatabaseFeatures::astNodes(entity, _, _, node, _) and
+      token = unique(string t | DatabaseFeatures::nodeAttributes(node, t)) and
+      location = node.getLocation()
+    )
+  }
+
+  /**
+   * Gets the body token feature for the specified entity.
+   *
+   * This is a string containing natural language tokens in the order that they appear in the source code for the entity.
+   */
+  string getBodyTokenFeatureForEntity(DatabaseFeatures::Entity entity) {
+    result =
+      strictconcat(string token, Location l |
+        bodyTokens(entity, l, token)
+      |
+        token, " "
+        order by
+          l.getFile().getAbsolutePath(), l.getStartLine(), l.getStartColumn(), l.getEndLine(),
+          l.getEndColumn(), token
+      )
+  }
+}
+
 /**
  * This module provides functionality for getting a representation of the access path of nodes
  * within the program.
@@ -240,59 +285,8 @@ private module AccessPaths {
   }
 }
 
-private module FunctionNames {
-  /**
-   * Get the name of the function.
-   *
-   * We attempt to assign unnamed entities approximate names if they are passed to a likely
-   * external library function. If we can't assign them an approximate name, we give them the name
-   * `""`, so that these entities are included in `AdaptiveThreatModeling.qll`.
-   *
-   * For entities which have multiple names, we choose the lexically smallest name.
-   */
-  string getNameToFeaturize(Function function) {
-    if exists(function.getName())
-    then result = min(function.getName())
-    else
-      if exists(getApproximateNameForFunction(function))
-      then result = getApproximateNameForFunction(function)
-      else result = ""
-  }
-
-  /**
-   * Holds if the call `call` has `function` is its `argumentIndex`th argument.
-   */
-  private predicate functionUsedAsArgumentToCall(
-    Function function, DataFlow::CallNode call, int argumentIndex
-  ) {
-    DataFlow::localFlowStep*(call.getArgument(argumentIndex), function.flow())
-  }
-
-  /**
-   * Returns a generated name for the function. This name is generated such that
-   * entities with the same names have similar behaviour.
-   */
-  private string getApproximateNameForFunction(Function function) {
-    count(DataFlow::CallNode call, int index | functionUsedAsArgumentToCall(function, call, index)) =
-      1 and
-    exists(DataFlow::CallNode call, int index, string basePart |
-      functionUsedAsArgumentToCall(function, call, index) and
-      (
-        if count(getReceiverName(call)) = 1
-        then basePart = getReceiverName(call) + "."
-        else basePart = ""
-      ) and
-      result = basePart + call.getCalleeName() + "#functionalargument"
-    )
-  }
-
-  private string getReceiverName(DataFlow::CallNode call) {
-    result = call.getReceiver().asExpr().(VarAccess).getName()
-  }
-}
-
 /** Get a name of a supported generic token-based feature. */
-string getASupportedFeatureName() {
+private string getASupportedFeatureName() {
   result =
     [
       "enclosingFunctionName", "calleeName", "receiverName", "argumentIndex", "calleeApiName",
@@ -309,5 +303,12 @@ string getASupportedFeatureName() {
 predicate tokenFeatures(DataFlow::Node endpoint, string featureName, string featureValue) {
   // Performance optimization: Restrict feature extraction to endpoints we've explicitly asked to featurize.
   endpoint = any(FeaturizationConfig cfg).getAnEndpointToFeaturize() and
-  featureValue = getTokenFeature(endpoint, featureName)
+  (
+    if strictcount(getTokenFeature(endpoint, featureName)) = 1
+    then featureValue = getTokenFeature(endpoint, featureName)
+    else (
+      // Performance note: this is a Cartesian product between all endpoints and feature names.
+      featureValue = "" and featureName = getASupportedFeatureName()
+    )
+  )
 }

javascript/ql/experimental/adaptivethreatmodeling/lib/experimental/adaptivethreatmodeling/EndpointScoring.qll

@@ -5,21 +5,86 @@
  */
 
 private import javascript
-private import BaseScoring
-private import EndpointFeatures as EndpointFeatures
-private import FeaturizationConfig
-private import EndpointTypes
+import BaseScoring
+import CodeToFeatures
+import EndpointFeatures as EndpointFeatures
+import EndpointTypes
 
 private string getACompatibleModelChecksum() {
   availableMlModels(result, "javascript", _, "atm-endpoint-scoring")
 }
 
+/**
+ * The maximum number of AST nodes an entity containing an endpoint should have before we should
+ * choose a smaller entity to represent the endpoint.
+ *
+ * This is intended to represent a balance in terms of the amount of context we provide to the
+ * model: we don't want the function to be too small, because then it doesn't contain very much
+ * context and miss useful information, but also we don't want it to be too large, because then
+ * there's likely to be a lot of irrelevant or very loosely related context.
+ */
+private int getMaxNumAstNodes() { result = 1024 }
+
+/**
+ * Returns the number of AST nodes contained within the specified entity.
+ */
+private int getNumAstNodesInEntity(DatabaseFeatures::Entity entity) {
+  // Restrict the values `entity` can take on
+  entity = EndpointToEntity::getAnEntityForEndpoint(_) and
+  result =
+    count(DatabaseFeatures::AstNode astNode | DatabaseFeatures::astNodes(entity, _, _, astNode, _))
+}
+
+/**
+ * Get a single entity to use as the representative entity for the endpoint.
+ *
+ * We try to use the largest entity containing the endpoint that's below the AST node limit defined
+ * in `getMaxNumAstNodes`. In the event of a tie, we use the entity that appears first within the
+ * source archive.
+ *
+ * If no entities are smaller than the AST node limit, then we use the smallest entity containing
+ * the endpoint.
+ */
+DatabaseFeatures::Entity getRepresentativeEntityForEndpoint(DataFlow::Node endpoint) {
+  // Check whether there's an entity containing the endpoint that's smaller than the AST node limit.
+  if
+    getNumAstNodesInEntity(EndpointToEntity::getAnEntityForEndpoint(endpoint)) <=
+      getMaxNumAstNodes()
+  then
+    // Use the largest entity smaller than the AST node limit, resolving ties using the entity that
+    // appears first in the source archive.
+    result =
+      min(DatabaseFeatures::Entity entity, int numAstNodes, Location l |
+        entity = EndpointToEntity::getAnEntityForEndpoint(endpoint) and
+        numAstNodes = getNumAstNodesInEntity(entity) and
+        numAstNodes <= getMaxNumAstNodes() and
+        l = entity.getLocation()
+      |
+        entity
+        order by
+          numAstNodes desc, l.getStartLine(), l.getStartColumn(), l.getEndLine(), l.getEndColumn()
+      )
+  else
+    // Use the smallest entity, resolving ties using the entity that
+    // appears first in the source archive.
+    result =
+      min(DatabaseFeatures::Entity entity, int numAstNodes, Location l |
+        entity = EndpointToEntity::getAnEntityForEndpoint(endpoint) and
+        numAstNodes = getNumAstNodesInEntity(entity) and
+        l = entity.getLocation()
+      |
+        entity
+        order by
+          numAstNodes, l.getStartLine(), l.getStartColumn(), l.getEndLine(), l.getEndColumn()
+      )
+}
+
 module ModelScoring {
   /**
    * A featurization config that only featurizes new candidate endpoints that are part of a flow
    * path.
    */
-  class RelevantFeaturizationConfig extends FeaturizationConfig {
+  class RelevantFeaturizationConfig extends EndpointFeatures::FeaturizationConfig {
     RelevantFeaturizationConfig() { this = "RelevantFeaturization" }
 
     override DataFlow::Node getAnEndpointToFeaturize() {
@@ -30,15 +95,15 @@ module ModelScoring {
   }
 
   DataFlow::Node getARequestedEndpoint() {
-    result = any(FeaturizationConfig cfg).getAnEndpointToFeaturize()
+    result = any(EndpointFeatures::FeaturizationConfig cfg).getAnEndpointToFeaturize()
   }
 
   private int getARequestedEndpointType() { result = any(EndpointType type).getEncoding() }
 
   predicate endpointScores(DataFlow::Node endpoint, int encodedEndpointType, float score) =
-    scoreEndpoints(getARequestedEndpoint/0, EndpointFeatures::tokenFeatures/3,
-      EndpointFeatures::getASupportedFeatureName/0, getARequestedEndpointType/0,
-      getACompatibleModelChecksum/0)(endpoint, encodedEndpointType, score)
+    scoreEndpoints(getARequestedEndpoint/0, getARequestedEndpointType/0,
+      EndpointFeatures::tokenFeatures/3, getACompatibleModelChecksum/0)(endpoint,
+      encodedEndpointType, score)
 }
 
 /**

javascript/ql/experimental/adaptivethreatmodeling/lib/experimental/adaptivethreatmodeling/FeaturizationConfig.qll

@@ -1,14 +0,0 @@
-import javascript
-
-/**
- * A configuration that defines which endpoints should be featurized.
- *
- * This is used as a performance optimization to ensure that we only featurize the endpoints we need
- * to featurize.
- */
-abstract class FeaturizationConfig extends string {
-  bindingset[this]
-  FeaturizationConfig() { any() }
-
-  abstract DataFlow::Node getAnEndpointToFeaturize();
-}

javascript/ql/experimental/adaptivethreatmodeling/lib/experimental/adaptivethreatmodeling/FunctionBodyFeatures.qll

@@ -1,126 +0,0 @@
-/*
- * FunctionBodyFeatures.qll
- *
- * Contains logic relating to the `enclosingFunctionBody` and `enclosingFunctionName` features.
- */
-
-import javascript
-private import FeaturizationConfig
-
-string getTokenizedAstNode(ASTNode node) {
-  // NB: Unary and binary operator expressions e.g. -a, a + b and compound
-  // assignments e.g. a += b can be identified by the expression type.
-  result = node.(Identifier).getName()
-  or
-  // Computed property accesses for which we can predetermine the property being accessed.
-  // NB: May alias with operators e.g. could have '+' as a property name.
-  result = node.(IndexExpr).getPropertyName()
-  or
-  // We use `getRawValue` to give us distinct representations for `0xa`, `0xA`, and `10`.
-  result = node.(NumberLiteral).getRawValue()
-  or
-  // We use `getValue` rather than `getRawValue` so we assign `"a"` and `'a'` the same representation.
-  not node instanceof NumberLiteral and
-  result = node.(Literal).getValue()
-  or
-  result = node.(TemplateElement).getRawValue()
-}
-
-/** Returns an AST node within the function `f` that we should featurize. */
-pragma[inline]
-ASTNode getAnASTNodeToFeaturize(Function f) {
-  result.getParent*() = f and
-  not result = f.getIdentifier() and
-  exists(getTokenizedAstNode(result))
-}
-
-/**
- * Get a function containing the endpoint that is suitable for featurization. In general, this
- * can associate an endpoint to multiple functions, since functions can be nested in JavaScript.
- */
-Function getAFunctionForEndpoint(DataFlow::Node endpoint) {
-  result = endpoint.getContainer().getEnclosingContainer*()
-}
-
-/**
- * The maximum number of AST nodes an function containing an endpoint should have before we should
- * choose a smaller function to represent the endpoint.
- *
- * This is intended to represent a balance in terms of the amount of context we provide to the
- * model: we don't want the function to be too small, because then it doesn't contain very much
- * context and miss useful information, but also we don't want it to be too large, because then
- * there's likely to be a lot of irrelevant or very loosely related context.
- */
-private int getMaxNumAstNodes() { result = 1024 }
-
-/**
- * Returns the number of AST nodes contained within the specified function.
- */
-private int getNumAstNodesInFunction(Function function) {
-  // Restrict the values `function` can take on
-  function = getAFunctionForEndpoint(_) and
-  result = count(getAnASTNodeToFeaturize(function))
-}
-
-/**
- * Get the enclosing function for an endpoint.
- *
- * This is used to compute the `enclosingFunctionBody` and `enclosingFunctionName` features.
- *
- * We try to use the largest function containing the endpoint that's below the AST node limit
- * defined in `getMaxNumAstNodes`. In the event of a tie, we use the function that appears first
- * within the source code.
- *
- * If no functions are smaller than the AST node limit, then we use the smallest function containing
- * the endpoint.
- */
-Function getRepresentativeFunctionForEndpoint(DataFlow::Node endpoint) {
-  // Check whether there's a function containing the endpoint that's smaller than the AST node
-  // limit.
-  if getNumAstNodesInFunction(getAFunctionForEndpoint(endpoint)) <= getMaxNumAstNodes()
-  then
-    // Use the largest function smaller than the AST node limit, resolving ties using the function
-    // that appears first in the source code.
-    result =
-      min(Function function, int numAstNodes, Location l |
-        function = getAFunctionForEndpoint(endpoint) and
-        numAstNodes = getNumAstNodesInFunction(function) and
-        numAstNodes <= getMaxNumAstNodes() and
-        l = function.getLocation()
-      |
-        function
-        order by
-          numAstNodes desc, l.getStartLine(), l.getStartColumn(), l.getEndLine(), l.getEndColumn()
-      )
-  else
-    // Use the smallest function, resolving ties using the function that appears first in the source
-    // code.
-    result =
-      min(Function function, int numAstNodes, Location l |
-        function = getAFunctionForEndpoint(endpoint) and
-        numAstNodes = getNumAstNodesInFunction(function) and
-        l = function.getLocation()
-      |
-        function
-        order by
-          numAstNodes, l.getStartLine(), l.getStartColumn(), l.getEndLine(), l.getEndColumn()
-      )
-}
-
-/** Holds if `location` is the location of an AST node within the function `function` and `token` is a node attribute associated with that AST node. */
-predicate bodyTokens(Function function, Location location, string token) {
-  // Performance optimization: Restrict the set of functions to those containing an endpoint to featurize.
-  function =
-    getRepresentativeFunctionForEndpoint(any(FeaturizationConfig cfg).getAnEndpointToFeaturize()) and
-  // Performance optimization: If a function has more than 256 body subtokens, then featurize it as absent. This
-  // approximates the behavior of the classifer on non-generic body features where large body
-  // features are replaced by the absent token.
-  //
-  // We count nodes instead of tokens because tokens are often not unique.
-  strictcount(getAnASTNodeToFeaturize(function)) <= 256 and
-  exists(ASTNode node |
-    node = getAnASTNodeToFeaturize(function) and
-    token = getTokenizedAstNode(node) and
-    location = node.getLocation()
-  )
-}

javascript/ql/experimental/adaptivethreatmodeling/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/javascript-experimental-atm-lib
-version: 0.0.2
+version: 0.0.4
 extractor: javascript
 library: true
 groups:

javascript/ql/experimental/adaptivethreatmodeling/src/qlpack.yml

@@ -1,6 +1,6 @@
 name: codeql/javascript-experimental-atm-queries
 language: javascript
-version: 0.0.2
+version: 0.0.4
 suites: codeql-suites
 defaultSuiteFile: codeql-suites/javascript-atm-code-scanning.qls
 groups:

javascript/ql/lib/semmle/javascript/Concepts.qll

@@ -84,6 +84,11 @@ abstract class FileNameSource extends DataFlow::Node { }
 abstract class DatabaseAccess extends DataFlow::Node {
   /** Gets an argument to this database access that is interpreted as a query. */
   abstract DataFlow::Node getAQueryArgument();
+
+  /** Gets a node to which a result of the access may flow. */
+  DataFlow::Node getAResult() {
+    none() // Overridden in subclass
+  }
 }
 
 /**

javascript/ql/lib/semmle/javascript/DOM.qll

@@ -359,6 +359,35 @@ module DOM {
           call.getNumArgument() = 1 and
           unique(InferredType t | t = getArgumentTypeFromJQueryMethodGet(call)) = TTNumber()
         )
+        or
+        // A `this` node from a callback given to a `$().each(callback)` call.
+        // purposely not using JQuery::MethodCall to avoid `jquery.each()`.
+        exists(DataFlow::CallNode eachCall | eachCall = JQuery::objectRef().getAMethodCall("each") |
+          this = DataFlow::thisNode(eachCall.getCallback(0).getFunction()) or
+          this = eachCall.getABoundCallbackParameter(0, 1)
+        )
+        or
+        // A read of an array-element from a JQuery object. E.g. `$("#foo")[0]`
+        exists(DataFlow::PropRead read |
+          read = this and read = JQuery::objectRef().getAPropertyRead()
+        |
+          unique(InferredType t | t = read.getPropertyNameExpr().analyze().getAType()) = TTNumber()
+        )
+        or
+        // A receiver node of an event handler on a DOM node
+        exists(DataFlow::SourceNode domNode, DataFlow::FunctionNode eventHandler |
+          // NOTE: we do not use `getABoundFunctionValue()`, since bound functions tend to have
+          // a different receiver anyway
+          eventHandler = domNode.getAPropertySource(any(string n | n.matches("on%")))
+          or
+          eventHandler =
+            domNode.getAMethodCall("addEventListener").getArgument(1).getAFunctionValue()
+        |
+          domNode = domValueRef() and
+          this = eventHandler.getReceiver()
+        )
+        or
+        this = DataFlow::thisNode(any(EventHandlerCode evt))
       }
     }
   }
@@ -392,6 +421,11 @@ module DOM {
     or
     t.start() and
     result = domValueRef().getAMethodCall(["item", "namedItem"])
+    or
+    t.startInProp("target") and
+    result = domEventSource()
+    or
+    exists(DataFlow::TypeTracker t2 | result = domValueRef(t2).track(t2, t))
   }
 
   /** Gets a data flow node that may refer to a value from the DOM. */

javascript/ql/lib/semmle/javascript/Promises.qll

@@ -185,12 +185,12 @@ module Promises {
   /**
    * Gets the pseudo-field used to describe resolved values in a promise.
    */
-  string valueProp() { none() }
+  string valueProp() { result = "$PromiseResolveField$" }
 
   /**
    * Gets the pseudo-field used to describe rejected values in a promise.
    */
-  string errorProp() { none() }
+  string errorProp() { result = "$PromiseRejectField$" }
 }
 
 /**

javascript/ql/lib/semmle/javascript/dataflow/Configuration.qll

@@ -777,10 +777,10 @@ private class AdditionalFlowStepAsSharedStep extends SharedFlowStep {
  */
 module PseudoProperties {
   bindingset[s]
-  private string pseudoProperty(string s) { none() }
+  private string pseudoProperty(string s) { result = "$" + s + "$" }
 
   bindingset[s, v]
-  private string pseudoProperty(string s, string v) { none() }
+  private string pseudoProperty(string s, string v) { result = "$" + s + "|" + v + "$" }
 
   /**
    * Gets a pseudo-property for the location of elements in a `Set`

javascript/ql/lib/semmle/javascript/frameworks/Angular2.qll

@@ -136,7 +136,7 @@ module Angular2 {
 
   /** Gets a reference to a `DomSanitizer` object. */
   DataFlow::SourceNode domSanitizer() {
-    result.hasUnderlyingType("@angular/platform-browser", "DomSanitizer")
+    result.hasUnderlyingType(["@angular/platform-browser", "@angular/core"], "DomSanitizer")
   }
 
   /** A value that is about to be promoted to a trusted HTML or CSS value. */

javascript/ql/lib/semmle/javascript/frameworks/Express.qll

@@ -927,6 +927,28 @@ module Express {
     override string getCredentialsKind() { result = kind }
   }
 
+  /** A call to `response.sendFile`, considered as a file system access. */
+  private class ResponseSendFileAsFileSystemAccess extends FileSystemReadAccess,
+    DataFlow::MethodCallNode {
+    ResponseSendFileAsFileSystemAccess() {
+      exists(string name | name = "sendFile" or name = "sendfile" |
+        this.calls(any(ResponseExpr res).flow(), name)
+      )
+    }
+
+    override DataFlow::Node getADataNode() { none() }
+
+    override DataFlow::Node getAPathArgument() { result = this.getArgument(0) }
+
+    override DataFlow::Node getRootPathArgument() {
+      result = this.(DataFlow::CallNode).getOptionArgument(1, "root")
+    }
+
+    override predicate isUpwardNavigationRejected(DataFlow::Node argument) {
+      argument = this.getAPathArgument()
+    }
+  }
+
   /**
    * A function that flows to a route setup.
    */

javascript/ql/lib/semmle/javascript/frameworks/Files.qll

@@ -4,6 +4,23 @@
 
 import javascript
 
+/**
+ * A call that can produce a file name.
+ */
+abstract private class FileNameProducer extends DataFlow::Node {
+  /**
+   * Gets a file name produced by this producer.
+   */
+  abstract DataFlow::Node getAFileName();
+}
+
+/**
+ * A node that contains a file name, and is produced by a `ProducesFileNames`.
+ */
+private class ProducedFileName extends FileNameSource {
+  ProducedFileName() { this = any(FileNameProducer producer).getAFileName() }
+}
+
 /**
  * A file name from the `walk-sync` library.
  */
@@ -126,3 +143,343 @@ private DataFlow::Node fastGlobFileNameSource(DataFlow::TypeTracker t) {
 private class FastGlobFileNameSource extends FileNameSource {
   FastGlobFileNameSource() { this = fastGlobFileNameSource(DataFlow::TypeTracker::end()) }
 }
+
+/**
+ * Classes and predicates for modeling the `fstream` library (https://www.npmjs.com/package/fstream).
+ */
+private module FStream {
+  /**
+   * Gets a reference to a method in the `fstream` library.
+   */
+  private DataFlow::SourceNode getAnFStreamProperty(boolean writer) {
+    exists(DataFlow::SourceNode mod, string readOrWrite, string subMod |
+      mod = DataFlow::moduleImport("fstream") and
+      (
+        readOrWrite = "Reader" and writer = false
+        or
+        readOrWrite = "Writer" and writer = true
+      ) and
+      (subMod = "File" or subMod = "Dir" or subMod = "Link" or subMod = "Proxy")
+    |
+      result = mod.getAPropertyRead(readOrWrite) or
+      result = mod.getAPropertyRead(readOrWrite).getAPropertyRead(subMod) or
+      result = mod.getAPropertyRead(subMod).getAPropertyRead(readOrWrite)
+    )
+  }
+
+  /**
+   * An invocation of a method defined in the `fstream` library.
+   */
+  private class FStream extends FileSystemAccess, DataFlow::InvokeNode {
+    boolean writer;
+
+    FStream() { this = getAnFStreamProperty(writer).getAnInvocation() }
+
+    override DataFlow::Node getAPathArgument() {
+      result = this.getOptionArgument(0, "path")
+      or
+      not exists(this.getOptionArgument(0, "path")) and
+      result = this.getArgument(0)
+    }
+  }
+
+  /**
+   * An invocation of an `fstream` method that writes to a file.
+   */
+  private class FStreamWriter extends FileSystemWriteAccess, FStream {
+    FStreamWriter() { writer = true }
+
+    override DataFlow::Node getADataNode() { none() }
+  }
+
+  /**
+   * An invocation of an `fstream` method that reads a file.
+   */
+  private class FStreamReader extends FileSystemReadAccess, FStream {
+    FStreamReader() { writer = false }
+
+    override DataFlow::Node getADataNode() { none() }
+  }
+}
+
+/**
+ * A call to the library `write-file-atomic`.
+ */
+private class WriteFileAtomic extends FileSystemWriteAccess, DataFlow::CallNode {
+  WriteFileAtomic() {
+    this = DataFlow::moduleImport("write-file-atomic").getACall()
+    or
+    this = DataFlow::moduleMember("write-file-atomic", "sync").getACall()
+  }
+
+  override DataFlow::Node getAPathArgument() { result = this.getArgument(0) }
+
+  override DataFlow::Node getADataNode() { result = this.getArgument(1) }
+}
+
+/**
+ * A call to the library `recursive-readdir`.
+ */
+private class RecursiveReadDir extends FileSystemAccess, FileNameProducer, DataFlow::CallNode {
+  RecursiveReadDir() { this = DataFlow::moduleImport("recursive-readdir").getACall() }
+
+  override DataFlow::Node getAPathArgument() { result = this.getArgument(0) }
+
+  override DataFlow::Node getAFileName() {
+    result = this.trackFileSource(DataFlow::TypeTracker::end())
+  }
+
+  private DataFlow::SourceNode trackFileSource(DataFlow::TypeTracker t) {
+    t.start() and result = this.getCallback([1 .. 2]).getParameter(1)
+    or
+    t.startInPromise() and not exists(this.getCallback([1 .. 2])) and result = this
+    or
+    // Tracking out of a promise
+    exists(DataFlow::TypeTracker t2 |
+      result = PromiseTypeTracking::promiseStep(this.trackFileSource(t2), t, t2)
+    )
+  }
+}
+
+/**
+ * Classes and predicates for modeling the `jsonfile` library (https://www.npmjs.com/package/jsonfile).
+ */
+private module JSONFile {
+  /**
+   * A reader for JSON files.
+   */
+  class JSONFileReader extends FileSystemReadAccess, DataFlow::CallNode {
+    JSONFileReader() {
+      this =
+        DataFlow::moduleMember("jsonfile", any(string s | s = "readFile" or s = "readFileSync"))
+            .getACall()
+    }
+
+    override DataFlow::Node getAPathArgument() { result = this.getArgument(0) }
+
+    override DataFlow::Node getADataNode() { result = this.trackRead(DataFlow::TypeTracker::end()) }
+
+    private DataFlow::SourceNode trackRead(DataFlow::TypeTracker t) {
+      this.getCalleeName() = "readFile" and
+      (
+        t.start() and result = this.getCallback([1 .. 2]).getParameter(1)
+        or
+        t.startInPromise() and not exists(this.getCallback([1 .. 2])) and result = this
+      )
+      or
+      t.start() and
+      this.getCalleeName() = "readFileSync" and
+      result = this
+      or
+      // Tracking out of a promise
+      exists(DataFlow::TypeTracker t2 |
+        result = PromiseTypeTracking::promiseStep(this.trackRead(t2), t, t2)
+      )
+    }
+  }
+
+  /**
+   * A writer for JSON files.
+   */
+  class JSONFileWriter extends FileSystemWriteAccess, DataFlow::CallNode {
+    JSONFileWriter() {
+      this =
+        DataFlow::moduleMember("jsonfile", any(string s | s = "writeFile" or s = "writeFileSync"))
+            .getACall()
+    }
+
+    override DataFlow::Node getAPathArgument() { result = this.getArgument(0) }
+
+    override DataFlow::Node getADataNode() { result = this.getArgument(1) }
+  }
+}
+
+/**
+ * A call to the library `load-json-file`.
+ */
+private class LoadJsonFile extends FileSystemReadAccess, DataFlow::CallNode {
+  LoadJsonFile() {
+    this = DataFlow::moduleImport("load-json-file").getACall()
+    or
+    this = DataFlow::moduleMember("load-json-file", "sync").getACall()
+  }
+
+  override DataFlow::Node getAPathArgument() { result = this.getArgument(0) }
+
+  override DataFlow::Node getADataNode() { result = this.trackRead(DataFlow::TypeTracker::end()) }
+
+  private DataFlow::SourceNode trackRead(DataFlow::TypeTracker t) {
+    this.getCalleeName() = "sync" and t.start() and result = this
+    or
+    not this.getCalleeName() = "sync" and t.startInPromise() and result = this
+    or
+    // Tracking out of a promise
+    exists(DataFlow::TypeTracker t2 |
+      result = PromiseTypeTracking::promiseStep(this.trackRead(t2), t, t2)
+    )
+  }
+}
+
+/**
+ * A call to the library `write-json-file`.
+ */
+private class WriteJsonFile extends FileSystemWriteAccess, DataFlow::CallNode {
+  WriteJsonFile() {
+    this = DataFlow::moduleImport("write-json-file").getACall()
+    or
+    this = DataFlow::moduleMember("write-json-file", "sync").getACall()
+  }
+
+  override DataFlow::Node getAPathArgument() { result = this.getArgument(0) }
+
+  override DataFlow::Node getADataNode() { result = this.getArgument(1) }
+}
+
+/**
+ * A call to the library `walkdir`.
+ */
+private class WalkDir extends FileNameProducer, FileSystemAccess, DataFlow::CallNode {
+  WalkDir() {
+    this = DataFlow::moduleImport("walkdir").getACall()
+    or
+    this = DataFlow::moduleMember("walkdir", "sync").getACall()
+    or
+    this = DataFlow::moduleMember("walkdir", "async").getACall()
+  }
+
+  override DataFlow::Node getAPathArgument() { result = this.getArgument(0) }
+
+  override DataFlow::Node getAFileName() {
+    result = this.trackFileSource(DataFlow::TypeTracker::end())
+  }
+
+  private DataFlow::SourceNode trackFileSource(DataFlow::TypeTracker t) {
+    not this.getCalleeName() = any(string s | s = "sync" or s = "async") and
+    t.start() and
+    (
+      result = this.getCallback(this.getNumArgument() - 1).getParameter(0)
+      or
+      result = this.getAMethodCall(EventEmitter::on()).getCallback(1).getParameter(0)
+    )
+    or
+    t.start() and this.getCalleeName() = "sync" and result = this
+    or
+    t.startInPromise() and this.getCalleeName() = "async" and result = this
+    or
+    // Tracking out of a promise
+    exists(DataFlow::TypeTracker t2 |
+      result = PromiseTypeTracking::promiseStep(this.trackFileSource(t2), t, t2)
+    )
+  }
+}
+
+/**
+ * A call to the library `globule`.
+ */
+private class Globule extends FileNameProducer, FileSystemAccess, DataFlow::CallNode {
+  Globule() {
+    this = DataFlow::moduleMember("globule", "find").getACall()
+    or
+    this = DataFlow::moduleMember("globule", "match").getACall()
+    or
+    this = DataFlow::moduleMember("globule", "isMatch").getACall()
+    or
+    this = DataFlow::moduleMember("globule", "mapping").getACall()
+    or
+    this = DataFlow::moduleMember("globule", "findMapping").getACall()
+  }
+
+  override DataFlow::Node getAPathArgument() {
+    (this.getCalleeName() = "match" or this.getCalleeName() = "isMatch") and
+    result = this.getArgument(1)
+    or
+    this.getCalleeName() = "mapping" and
+    (
+      result = this.getAnArgument() and
+      not exists(result.getALocalSource().getAPropertyWrite("src"))
+      or
+      result = this.getAnArgument().getALocalSource().getAPropertyWrite("src").getRhs()
+    )
+  }
+
+  override DataFlow::Node getAFileName() {
+    result = this and
+    (
+      this.getCalleeName() = "find" or
+      this.getCalleeName() = "match" or
+      this.getCalleeName() = "findMapping" or
+      this.getCalleeName() = "mapping"
+    )
+  }
+}
+
+/**
+ * A file system access made by a NodeJS library.
+ * This class models multiple NodeJS libraries that access files.
+ */
+private class LibraryAccess extends FileSystemAccess, DataFlow::InvokeNode {
+  int pathArgument; // The index of the path argument.
+
+  LibraryAccess() {
+    pathArgument = 0 and
+    (
+      this = DataFlow::moduleImport("path-exists").getACall()
+      or
+      this = DataFlow::moduleImport("rimraf").getACall()
+      or
+      this = DataFlow::moduleImport("readdirp").getACall()
+      or
+      this = DataFlow::moduleImport("walker").getACall()
+      or
+      this =
+        DataFlow::moduleMember("node-dir",
+          any(string s |
+            s = ["readFiles", "readFilesStream", "files", "promiseFiles", "subdirs", "paths"]
+          )).getACall()
+    )
+    or
+    pathArgument = 0 and
+    this =
+      DataFlow::moduleMember("vinyl-fs", any(string s | s = "src" or s = "dest" or s = "symlink"))
+          .getACall()
+    or
+    pathArgument = [0 .. 1] and
+    (
+      this = DataFlow::moduleImport("ncp").getACall() or
+      this = DataFlow::moduleMember("ncp", "ncp").getACall()
+    )
+  }
+
+  override DataFlow::Node getAPathArgument() { result = this.getArgument(pathArgument) }
+}
+
+/**
+ * A call to the library [`chokidar`](https://www.npmjs.com/package/chokidar), where a call to `on` receives file names.
+ */
+class Chokidar extends FileNameProducer, FileSystemAccess, API::CallNode {
+  Chokidar() { this = API::moduleImport("chokidar").getMember("watch").getACall() }
+
+  override DataFlow::Node getAPathArgument() { result = this.getArgument(0) }
+
+  override DataFlow::Node getAFileName() {
+    exists(DataFlow::CallNode onCall, int pathIndex |
+      onCall = this.getAChainedMethodCall("on") and
+      if onCall.getArgument(0).mayHaveStringValue("all") then pathIndex = 1 else pathIndex = 0
+    |
+      result = onCall.getCallback(1).getParameter(pathIndex)
+    )
+  }
+}
+
+/**
+ * A call to the [`mkdirp`](https://www.npmjs.com/package/mkdirp) library.
+ */
+private class Mkdirp extends FileSystemAccess, API::CallNode {
+  Mkdirp() {
+    this = API::moduleImport("mkdirp").getACall()
+    or
+    this = API::moduleImport("mkdirp").getMember("sync").getACall()
+  }
+
+  override DataFlow::Node getAPathArgument() { result = this.getArgument(0) }
+}

javascript/ql/lib/semmle/javascript/frameworks/Knex.qll

@@ -46,17 +46,35 @@ module Knex {
     RawKnexSqlString() { this = any(RawKnexCall call).getArgument(0).asExpr() }
   }
 
-  /** A call that triggers a SQL query submission. */
-  private class KnexDatabaseAccess extends DatabaseAccess {
-    KnexDatabaseAccess() {
-      this = knexObject().getMember(["then", "stream", "asCallback"]).getACall()
+  /** A call that triggers a SQL query submission by calling then/stream/asCallback. */
+  private class KnexDatabaseCallback extends DatabaseAccess, DataFlow::CallNode {
+    string member;
+
+    KnexDatabaseCallback() {
+      member = ["then", "stream", "asCallback"] and
+      this = knexObject().getMember(member).getACall()
+    }
+
+    override DataFlow::Node getAResult() {
+      member = "then" and
+      result = this.getCallback(0).getParameter(0)
       or
-      exists(AwaitExpr await |
-        this = await.flow() and
-        await.getOperand() = knexObject().getAUse().asExpr()
-      )
+      member = "asCallback" and
+      result = this.getCallback(0).getParameter(1)
     }
 
     override DataFlow::Node getAQueryArgument() { none() }
   }
+
+  private class KnexDatabaseAwait extends DatabaseAccess, DataFlow::ValueNode {
+    KnexDatabaseAwait() {
+      exists(AwaitExpr enclosingAwait | this = enclosingAwait.flow() |
+        enclosingAwait.getOperand() = knexObject().getAUse().asExpr()
+      )
+    }
+
+    override DataFlow::Node getAResult() { result = this }
+
+    override DataFlow::Node getAQueryArgument() { none() }
+  }
 }

javascript/ql/lib/semmle/javascript/frameworks/NoSQL.qll

@@ -3,10 +3,21 @@
  */
 
 import javascript
+import semmle.javascript.Promises
 
 module NoSQL {
   /** An expression that is interpreted as a NoSQL query. */
-  abstract class Query extends Expr { }
+  abstract class Query extends Expr {
+    /** Gets an expression that is interpreted as a code operator in this query. */
+    DataFlow::Node getACodeOperator() { none() }
+  }
+}
+
+/**
+ * Gets a value that has been assigned to the "$where" property of an object that flows to `queryArg`.
+ */
+private DataFlow::Node getADollarWhereProperty(API::Node queryArg) {
+  result = queryArg.getMember("$where").getARhs()
 }
 
 /**
@@ -14,112 +25,123 @@ module NoSQL {
  */
 private module MongoDB {
   /**
-   * Gets an import of MongoDB.
-   */
-  DataFlow::ModuleImportNode mongodb() { result.getPath() = "mongodb" }
-
-  /**
-   * Gets an access to `mongodb.MongoClient`.
-   */
-  private DataFlow::SourceNode getAMongoClient(DataFlow::TypeTracker t) {
-    t.start() and
-    result = mongodb().getAPropertyRead("MongoClient")
-    or
-    exists(DataFlow::TypeTracker t2 | result = getAMongoClient(t2).track(t2, t))
-  }
-
-  /**
-   * Gets an access to `mongodb.MongoClient`.
-   */
-  DataFlow::SourceNode getAMongoClient() { result = getAMongoClient(DataFlow::TypeTracker::end()) }
-
-  /** Gets a data flow node that leads to a `connect` callback. */
-  private DataFlow::SourceNode getAMongoDbCallback(DataFlow::TypeBackTracker t) {
-    t.start() and
-    result = getAMongoClient().getAMemberCall("connect").getArgument(1).getALocalSource()
-    or
-    exists(DataFlow::TypeBackTracker t2 | result = getAMongoDbCallback(t2).backtrack(t2, t))
-  }
-
-  /** Gets a data flow node that leads to a `connect` callback. */
-  private DataFlow::FunctionNode getAMongoDbCallback() {
-    result = getAMongoDbCallback(DataFlow::TypeBackTracker::end())
-  }
-
-  /**
-   * Gets an expression that may refer to a MongoDB database connection.
-   */
-  private DataFlow::SourceNode getAMongoDb(DataFlow::TypeTracker t) {
-    t.start() and
-    result = getAMongoDbCallback().getParameter(1)
-    or
-    exists(DataFlow::TypeTracker t2 | result = getAMongoDb(t2).track(t2, t))
-  }
-
-  /**
-   * Gets an expression that may refer to a MongoDB database connection.
-   */
-  DataFlow::SourceNode getAMongoDb() { result = getAMongoDb(DataFlow::TypeTracker::end()) }
-
-  /**
-   * A data flow node that may hold a MongoDB collection.
-   */
-  abstract class Collection extends DataFlow::SourceNode { }
-
-  /**
-   * A collection resulting from calling `Db.collection(...)`.
-   */
-  private class CollectionFromDb extends Collection {
-    CollectionFromDb() {
-      this = getAMongoDb().getAMethodCall("collection")
-      or
-      this = getAMongoDb().getAMethodCall("collection").getCallback(1).getParameter(0)
-    }
-  }
-
-  /**
-   * A collection based on the type `mongodb.Collection`.
+   * Gets an access to `mongodb.MongoClient` or a database.
    *
-   * Note that this also covers `mongoose` models since they are subtypes
-   * of `mongodb.Collection`.
+   * In Mongo version 2.x, a client and a database handle were the same concept, but in 3.x
+   * they were separated. To handle everything with a single model, we treat them as the same here.
    */
-  private class CollectionFromType extends Collection {
-    CollectionFromType() { hasUnderlyingType("mongodb", "Collection") }
-  }
-
-  /** Gets a data flow node referring to a MongoDB collection. */
-  private DataFlow::SourceNode getACollection(DataFlow::TypeTracker t) {
-    t.start() and
-    result instanceof Collection
+  private API::Node getAMongoClientOrDatabase() {
+    result = API::moduleImport("mongodb").getMember("MongoClient")
     or
-    exists(DataFlow::TypeTracker t2 | result = getACollection(t2).track(t2, t))
+    result = getAMongoClientOrDatabase().getMember("db").getReturn()
+    or
+    result = getAMongoClientOrDatabase().getMember("connect").getLastParameter().getParameter(1)
   }
 
   /** Gets a data flow node referring to a MongoDB collection. */
-  DataFlow::SourceNode getACollection() { result = getACollection(DataFlow::TypeTracker::end()) }
+  private API::Node getACollection() {
+    // A collection resulting from calling `Db.collection(...)`.
+    exists(API::Node collection |
+      collection = getAMongoClientOrDatabase().getMember("collection").getReturn()
+    |
+      result = collection
+      or
+      result = collection.getParameter(1).getParameter(0)
+    )
+    or
+    // note that this also covers `mongoose` models since they are subtypes of `mongodb.Collection`
+    result = API::Node::ofType("mongodb", "Collection")
+  }
 
   /** A call to a MongoDB query method. */
-  private class QueryCall extends DatabaseAccess, DataFlow::MethodCallNode {
+  private class QueryCall extends DatabaseAccess, API::CallNode {
     int queryArgIdx;
 
     QueryCall() {
-      exists(string m | this = getACollection().getAMethodCall(m) |
-        m = "count" and queryArgIdx = 0
-        or
-        m = "distinct" and queryArgIdx = 1
-        or
-        m = "find" and queryArgIdx = 0
+      exists(string method |
+        CollectionMethodSignatures::interpretsArgumentAsQuery(method, queryArgIdx) and
+        this = getACollection().getMember(method).getACall()
       )
     }
 
-    override DataFlow::Node getAQueryArgument() { result = getArgument(queryArgIdx) }
+    override DataFlow::Node getAQueryArgument() { result = this.getArgument(queryArgIdx) }
+
+    override DataFlow::Node getAResult() {
+      PromiseFlow::loadStep(this.getALocalUse(), result, Promises::valueProp())
+    }
+
+    DataFlow::Node getACodeOperator() {
+      result = getADollarWhereProperty(this.getParameter(queryArgIdx))
+    }
   }
 
   /**
    * An expression that is interpreted as a MongoDB query.
    */
   class Query extends NoSQL::Query {
-    Query() { this = any(QueryCall qc).getAQueryArgument().asExpr() }
+    QueryCall qc;
+
+    Query() { this = qc.getAQueryArgument().asExpr() }
+
+    override DataFlow::Node getACodeOperator() { result = qc.getACodeOperator() }
+  }
+
+  /**
+   * Provides signatures for the Collection methods.
+   */
+  module CollectionMethodSignatures {
+    /**
+     * Holds if Collection method `name` interprets parameter `n` as a query.
+     */
+    predicate interpretsArgumentAsQuery(string name, int n) {
+      // FilterQuery
+      (
+        name = "aggregate" and n = 0
+        or
+        name = "count" and n = 0
+        or
+        name = "countDocuments" and n = 0
+        or
+        name = "deleteMany" and n = 0
+        or
+        name = "deleteOne" and n = 0
+        or
+        name = "distinct" and n = 1
+        or
+        name = "find" and n = 0
+        or
+        name = "findOne" and n = 0
+        or
+        name = "findOneAndDelete" and n = 0
+        or
+        name = "findOneAndRemove" and n = 0
+        or
+        name = "findOneAndReplace" and n = 0
+        or
+        name = "findOneAndUpdate" and n = 0
+        or
+        name = "remove" and n = 0
+        or
+        name = "replaceOne" and n = 0
+        or
+        name = "update" and n = 0
+        or
+        name = "updateMany" and n = 0
+        or
+        name = "updateOne" and n = 0
+      )
+      or
+      // UpdateQuery
+      (
+        name = "findOneAndUpdate" and n = 1
+        or
+        name = "update" and n = 1
+        or
+        name = "updateMany" and n = 1
+        or
+        name = "updateOne" and n = 1
+      )
+    }
   }
 }
 
@@ -130,20 +152,345 @@ private module Mongoose {
   /**
    * Gets an import of Mongoose.
    */
-  DataFlow::ModuleImportNode getAMongooseInstance() { result.getPath() = "mongoose" }
+  API::Node getAMongooseInstance() { result = API::moduleImport("mongoose") }
 
   /**
-   * Gets a call to `mongoose.createConnection`.
+   * Gets a reference to `mongoose.createConnection`.
    */
-  DataFlow::CallNode createConnection() {
-    result = getAMongooseInstance().getAMemberCall("createConnection")
+  API::Node createConnection() { result = getAMongooseInstance().getMember("createConnection") }
+
+  /**
+   * A Mongoose function.
+   */
+  abstract private class MongooseFunction extends API::Node {
+    /**
+     * Gets the API-graph node for the result from this function (if the function returns a `Query`).
+     */
+    abstract API::Node getQueryReturn();
+
+    /**
+     * Holds if this function returns a `Query` that evaluates to one or
+     * more Documents (`asArray` is false if it evaluates to a single
+     * Document).
+     */
+    abstract predicate returnsDocumentQuery(boolean asArray);
+
+    /**
+     * Gets an argument that this function interprets as a query.
+     */
+    abstract API::Node getQueryArgument();
   }
 
   /**
-   * A Mongoose collection object.
+   * Provides classes modeling the Mongoose Model class
    */
-  class Model extends MongoDB::Collection {
-    Model() { this = getAMongooseInstance().getAMemberCall("model") }
+  module Model {
+    private class ModelFunction extends MongooseFunction {
+      string methodName;
+
+      ModelFunction() { this = getModelObject().getMember(methodName) }
+
+      override API::Node getQueryReturn() {
+        MethodSignatures::returnsQuery(methodName) and result = this.getReturn()
+      }
+
+      override predicate returnsDocumentQuery(boolean asArray) {
+        MethodSignatures::returnsDocumentQuery(methodName, asArray)
+      }
+
+      override API::Node getQueryArgument() {
+        exists(int n |
+          MethodSignatures::interpretsArgumentAsQuery(methodName, n) and
+          result = this.getParameter(n)
+        )
+      }
+    }
+
+    /**
+     * Gets a API-graph node referring to a Mongoose Model object.
+     */
+    private API::Node getModelObject() {
+      result = getAMongooseInstance().getMember("model").getReturn()
+      or
+      exists(API::Node conn | conn = createConnection().getReturn() |
+        result = conn.getMember("model").getReturn() or
+        result = conn.getMember("models").getAMember()
+      )
+      or
+      result = API::Node::ofType("mongoose", "Model")
+    }
+
+    /**
+     * Provides signatures for the Model methods.
+     */
+    module MethodSignatures {
+      /**
+       * Holds if Model method `name` interprets parameter `n` as a query.
+       */
+      predicate interpretsArgumentAsQuery(string name, int n) {
+        // implement lots of the MongoDB collection interface
+        MongoDB::CollectionMethodSignatures::interpretsArgumentAsQuery(name, n)
+        or
+        name = "find" + ["ById", "One"] + "AndUpdate" and n = 1
+        or
+        name in ["delete" + ["Many", "One"], "geoSearch", "remove", "replaceOne", "where"] and
+        n = 0
+        or
+        name in [
+            "find" + ["", "ById", "One"],
+            "find" + ["ById", "One"] + "And" + ["Delete", "Remove", "Update"],
+            "update" + ["", "Many", "One"]
+          ] and
+        n = 0
+      }
+
+      /**
+       * Holds if Model method `name` returns a Query.
+       */
+      predicate returnsQuery(string name) {
+        name =
+          [
+            "$where", "count", "findOne", "findOneAndDelete", "findOneAndRemove",
+            "findOneAndReplace", "findOneAndUpdate", "geosearch", "remove", "replaceOne", "update",
+            "updateMany", "countDocuments", "updateOne", "where", "deleteMany", "deleteOne", "find",
+            "findById", "findByIdAndDelete", "findByIdAndRemove", "findByIdAndUpdate"
+          ]
+      }
+
+      /**
+       * Holds if Document method `name` returns a query that results in
+       * one or more documents, the documents are wrapped in an array
+       * if `asArray` is true.
+       */
+      predicate returnsDocumentQuery(string name, boolean asArray) {
+        asArray = false and name = "findOne"
+        or
+        asArray = true and name = "find"
+      }
+    }
+  }
+
+  /**
+   * Provides classes modeling the Mongoose Query class
+   */
+  module Query {
+    private class QueryFunction extends MongooseFunction {
+      string methodName;
+
+      QueryFunction() { this = getAMongooseQuery().getMember(methodName) }
+
+      override API::Node getQueryReturn() {
+        MethodSignatures::returnsQuery(methodName) and result = this.getReturn()
+      }
+
+      override predicate returnsDocumentQuery(boolean asArray) {
+        MethodSignatures::returnsDocumentQuery(methodName, asArray)
+      }
+
+      override API::Node getQueryArgument() {
+        exists(int n |
+          MethodSignatures::interpretsArgumentAsQuery(methodName, n) and
+          result = this.getParameter(n)
+        )
+      }
+    }
+
+    private class NewQueryFunction extends MongooseFunction {
+      NewQueryFunction() { this = getAMongooseInstance().getMember("Query") }
+
+      override API::Node getQueryReturn() { result = this.getInstance() }
+
+      override predicate returnsDocumentQuery(boolean asArray) { none() }
+
+      override API::Node getQueryArgument() { result = this.getParameter(2) }
+    }
+
+    /**
+     * Gets a data flow node referring to a Mongoose query object.
+     */
+    API::Node getAMongooseQuery() {
+      result = any(MongooseFunction f).getQueryReturn()
+      or
+      result = API::Node::ofType("mongoose", "Query")
+      or
+      result =
+        getAMongooseQuery()
+            .getMember(any(string name | MethodSignatures::returnsQuery(name)))
+            .getReturn()
+    }
+
+    /**
+     * Provides signatures for the Query methods.
+     */
+    module MethodSignatures {
+      /**
+       * Holds if Query method `name` interprets parameter `n` as a query.
+       */
+      predicate interpretsArgumentAsQuery(string name, int n) {
+        n = 0 and
+        name =
+          [
+            "and", "count", "findOneAndReplace", "findOneAndUpdate", "merge", "nor", "or", "remove",
+            "replaceOne", "setQuery", "setUpdate", "update", "countDocuments", "updateMany",
+            "updateOne", "where", "deleteMany", "deleteOne", "elemMatch", "find", "findOne",
+            "findOneAndDelete", "findOneAndRemove"
+          ]
+        or
+        n = 1 and
+        name = ["distinct", "findOneAndUpdate", "update", "updateMany", "updateOne"]
+      }
+
+      /**
+       * Holds if Query method `name` returns a Query.
+       */
+      predicate returnsQuery(string name) {
+        name =
+          [
+            "$where", "J", "comment", "count", "countDocuments", "distinct", "elemMatch", "equals",
+            "error", "estimatedDocumentCount", "exists", "explain", "all", "find", "findById",
+            "findOne", "findOneAndRemove", "findOneAndUpdate", "geometry", "get", "gt", "gte",
+            "hint", "and", "in", "intersects", "lean", "limit", "lt", "lte", "map", "map",
+            "maxDistance", "maxTimeMS", "batchsize", "maxscan", "mod", "ne", "near", "nearSphere",
+            "nin", "or", "orFail", "polygon", "populate", "box", "read", "readConcern", "regexp",
+            "remove", "select", "session", "set", "setOptions", "setQuery", "setUpdate", "center",
+            "size", "skip", "slaveOk", "slice", "snapshot", "sort", "update", "w", "where",
+            "within", "centerSphere", "wtimeout", "circle", "collation"
+          ]
+      }
+
+      /**
+       * Holds if Query method `name` returns a query that results in
+       * one or more documents, the documents are wrapped in an array
+       * if `asArray` is true.
+       */
+      predicate returnsDocumentQuery(string name, boolean asArray) {
+        asArray = false and name = "findOne"
+        or
+        asArray = true and name = "find"
+      }
+    }
+  }
+
+  /**
+   * Provides classes modeling the Mongoose Document class
+   */
+  module Document {
+    private class DocumentFunction extends MongooseFunction {
+      string methodName;
+
+      DocumentFunction() { this = getAMongooseDocument().getMember(methodName) }
+
+      override API::Node getQueryReturn() {
+        MethodSignatures::returnsQuery(methodName) and result = this.getReturn()
+      }
+
+      override predicate returnsDocumentQuery(boolean asArray) {
+        MethodSignatures::returnsDocumentQuery(methodName, asArray)
+      }
+
+      override API::Node getQueryArgument() {
+        exists(int n |
+          MethodSignatures::interpretsArgumentAsQuery(methodName, n) and
+          result = this.getParameter(n)
+        )
+      }
+    }
+
+    /**
+     * A Mongoose Document that is retrieved from the backing database.
+     */
+    class RetrievedDocument extends API::Node {
+      RetrievedDocument() {
+        exists(boolean asArray, API::Node param |
+          exists(MongooseFunction func |
+            func.returnsDocumentQuery(asArray) and
+            param = func.getLastParameter().getParameter(1)
+          )
+          or
+          exists(API::Node f |
+            f = Query::getAMongooseQuery().getMember("then") and
+            param = f.getParameter(0).getParameter(0)
+            or
+            f = Query::getAMongooseQuery().getMember("exec") and
+            param = f.getParameter(0).getParameter(1)
+          |
+            exists(DataFlow::MethodCallNode pred |
+              // limitation: look at the previous method call	
+              Query::MethodSignatures::returnsDocumentQuery(pred.getMethodName(), asArray) and
+              pred.getAMethodCall() = f.getACall()
+            )
+          )
+        |
+          asArray = false and this = param
+          or
+          asArray = true and
+          // limitation: look for direct accesses
+          this = param.getUnknownMember()
+        )
+      }
+    }
+
+    /**
+     * Gets a data flow node referring to a Mongoose Document object.
+     */
+    private API::Node getAMongooseDocument() {
+      result instanceof RetrievedDocument
+      or
+      result = API::Node::ofType("mongoose", "Document")
+      or
+      result =
+        getAMongooseDocument()
+            .getMember(any(string name | MethodSignatures::returnsDocument(name)))
+            .getReturn()
+    }
+
+    private module MethodSignatures {
+      /**
+       * Holds if Document method `name` returns a Query.
+       */
+      predicate returnsQuery(string name) {
+        // Documents are subtypes of Models
+        Model::MethodSignatures::returnsQuery(name) or
+        name = "replaceOne" or
+        name = "update" or
+        name = "updateOne"
+      }
+
+      /**
+       * Holds if Document method `name` interprets parameter `n` as a query.
+       */
+      predicate interpretsArgumentAsQuery(string name, int n) {
+        // Documents are subtypes of Models
+        Model::MethodSignatures::interpretsArgumentAsQuery(name, n)
+        or
+        n = 0 and
+        (
+          name = "replaceOne" or
+          name = "update" or
+          name = "updateOne"
+        )
+      }
+
+      /**
+       * Holds if Document method `name` returns a query that results in
+       * one or more documents, the documents are wrapped in an array
+       * if `asArray` is true.
+       */
+      predicate returnsDocumentQuery(string name, boolean asArray) {
+        // Documents are subtypes of Models
+        Model::MethodSignatures::returnsDocumentQuery(name, asArray)
+      }
+
+      /**
+       * Holds if Document method `name` returns a Document.
+       */
+      predicate returnsDocument(string name) {
+        name = "depopulate" or
+        name = "init" or
+        name = "populate" or
+        name = "overwrite"
+      }
+    }
   }
 
   /**
@@ -153,7 +500,9 @@ private module Mongoose {
     string kind;
 
     Credentials() {
-      exists(string prop | this = createConnection().getOptionArgument(3, prop).asExpr() |
+      exists(string prop |
+        this = createConnection().getParameter(3).getMember(prop).getARhs().asExpr()
+      |
         prop = "user" and kind = "user name"
         or
         prop = "pass" and kind = "password"
@@ -162,4 +511,308 @@ private module Mongoose {
 
     override string getCredentialsKind() { result = kind }
   }
+
+  /**
+   * An expression that is interpreted as a (part of a) MongoDB query.
+   */
+  class MongoDBQueryPart extends NoSQL::Query {
+    MongooseFunction f;
+
+    MongoDBQueryPart() { this = f.getQueryArgument().getARhs().asExpr() }
+
+    override DataFlow::Node getACodeOperator() {
+      result = getADollarWhereProperty(f.getQueryArgument())
+    }
+  }
+
+  /**
+   * An evaluation of a MongoDB query.
+   */
+  class ShorthandQueryEvaluation extends DatabaseAccess, DataFlow::InvokeNode {
+    MongooseFunction f;
+
+    ShorthandQueryEvaluation() {
+      this = f.getACall() and
+      // shorthand for execution: provide a callback
+      exists(f.getQueryReturn()) and
+      exists(this.getCallback(this.getNumArgument() - 1))
+    }
+
+    override DataFlow::Node getAQueryArgument() {
+      // NB: the complete information is not easily accessible for deeply chained calls
+      f.getQueryArgument().getARhs() = result
+    }
+
+    override DataFlow::Node getAResult() {
+      result = this.getCallback(this.getNumArgument() - 1).getParameter(1)
+    }
+  }
+
+  class ExplicitQueryEvaluation extends DatabaseAccess, DataFlow::CallNode {
+    string member;
+
+    ExplicitQueryEvaluation() {
+      // explicit execution using a Query method call
+      member = ["exec", "then", "catch"] and
+      Query::getAMongooseQuery().getMember(member).getACall() = this
+    }
+
+    private int resultParamIndex() {
+      member = "then" and result = 0
+      or
+      member = "exec" and result = 1
+    }
+
+    override DataFlow::Node getAResult() {
+      result = this.getCallback(_).getParameter(this.resultParamIndex())
+    }
+
+    override DataFlow::Node getAQueryArgument() {
+      // NB: the complete information is not easily accessible for deeply chained calls
+      none()
+    }
+  }
+}
+
+/**
+ * Provides classes modeling the Minimongo library.
+ */
+private module Minimongo {
+  /**
+   * Provides signatures for the Collection methods.
+   */
+  module CollectionMethodSignatures {
+    /**
+     * Holds if Collection method `name` interprets parameter `n` as a query.
+     */
+    predicate interpretsArgumentAsQuery(string m, int queryArgIdx) {
+      // implements most of the MongoDB interface
+      MongoDB::CollectionMethodSignatures::interpretsArgumentAsQuery(m, queryArgIdx)
+    }
+  }
+
+  /** A call to a Minimongo query method. */
+  private class QueryCall extends DatabaseAccess, API::CallNode {
+    int queryArgIdx;
+
+    QueryCall() {
+      exists(string m |
+        this =
+          API::moduleImport("minimongo")
+              .getAMember()
+              .getReturn()
+              .getAMember()
+              .getMember(m)
+              .getACall() and
+        CollectionMethodSignatures::interpretsArgumentAsQuery(m, queryArgIdx)
+      )
+    }
+
+    override DataFlow::Node getAQueryArgument() { result = this.getArgument(queryArgIdx) }
+
+    override DataFlow::Node getAResult() {
+      PromiseFlow::loadStep(this.getALocalUse(), result, Promises::valueProp())
+    }
+
+    DataFlow::Node getACodeOperator() {
+      result = getADollarWhereProperty(this.getParameter(queryArgIdx))
+    }
+  }
+
+  /**
+   * An expression that is interpreted as a Minimongo query.
+   */
+  class Query extends NoSQL::Query {
+    QueryCall qc;
+
+    Query() { this = qc.getAQueryArgument().asExpr() }
+
+    override DataFlow::Node getACodeOperator() { result = qc.getACodeOperator() }
+  }
+}
+
+/**
+ * Provides classes modeling the MarsDB library.
+ */
+private module MarsDB {
+  private class MarsDBAccess extends DatabaseAccess, DataFlow::CallNode {
+    string method;
+
+    MarsDBAccess() {
+      this =
+        API::moduleImport("marsdb")
+            .getMember("Collection")
+            .getInstance()
+            .getMember(method)
+            .getACall()
+    }
+
+    string getMethod() { result = method }
+
+    override DataFlow::Node getAResult() {
+      PromiseFlow::loadStep(this.getALocalUse(), result, Promises::valueProp())
+    }
+
+    override DataFlow::Node getAQueryArgument() { none() }
+  }
+
+  /** A call to a MarsDB query method. */
+  private class QueryCall extends MarsDBAccess, API::CallNode {
+    int queryArgIdx;
+
+    QueryCall() {
+      exists(string m |
+        this.getMethod() = m and
+        // implements parts of the Minimongo interface
+        Minimongo::CollectionMethodSignatures::interpretsArgumentAsQuery(m, queryArgIdx)
+      )
+    }
+
+    override DataFlow::Node getAResult() {
+      PromiseFlow::loadStep(this.getALocalUse(), result, Promises::valueProp())
+    }
+
+    override DataFlow::Node getAQueryArgument() { result = this.getArgument(queryArgIdx) }
+
+    DataFlow::Node getACodeOperator() {
+      result = getADollarWhereProperty(this.getParameter(queryArgIdx))
+    }
+  }
+
+  /**
+   * An expression that is interpreted as a MarsDB query.
+   */
+  class Query extends NoSQL::Query {
+    QueryCall qc;
+
+    Query() { this = qc.getAQueryArgument().asExpr() }
+
+    override DataFlow::Node getACodeOperator() { result = qc.getACodeOperator() }
+  }
+}
+
+/**
+ * Provides classes modeling the `Node Redis` library.
+ *
+ * Redis is an in-memory key-value store and not a database,
+ * but `Node Redis` can be exploited similarly to a NoSQL database by giving a method an array as argument instead of a string.
+ * As an example the below two invocations of `client.set` are equivalent:
+ *
+ * ```
+ * const redis = require("redis");
+ * const client = redis.createClient();
+ * client.set("key", "value");
+ * client.set(["key", "value"]);
+ * ```
+ *
+ * ioredis is a very similar library. However, ioredis does not support array arguments in the same way, and is therefore not vulnerable to the same kind of type confusion.
+ */
+private module Redis {
+  /**
+   * Gets a `Node Redis` client.
+   */
+  private API::Node client() {
+    result = API::moduleImport("redis").getMember("createClient").getReturn()
+    or
+    result = API::moduleImport("redis").getMember("RedisClient").getInstance()
+    or
+    result = client().getMember("duplicate").getReturn()
+    or
+    result = client().getMember("duplicate").getLastParameter().getParameter(1)
+  }
+
+  /**
+   * Gets a (possibly chained) reference to a batch operation object.
+   * These have the same API as a redis client, except the calls are chained, and the sequence is terminated with a `.exec` call.
+   */
+  private API::Node multi() {
+    result = client().getMember(["multi", "batch"]).getReturn()
+    or
+    result = multi().getAMember().getReturn()
+  }
+
+  /**
+   * Gets a `Node Redis` client instance. Either a client created using `createClient()`, or a batch operation object.
+   */
+  private API::Node redis() { result = [client(), multi()] }
+
+  /**
+   * Provides signatures for the query methods from Node Redis.
+   */
+  module QuerySignatures {
+    /**
+     * Holds if `method` interprets parameter `argIndex` as a key, and a later parameter determines a value/field.
+     * Thereby the method is vulnerable if parameter `argIndex` is unexpectedly an array instead of a string, as an attacker can control arguments to Redis that the attacker was not supposed to control.
+     *
+     * Only setters and similar methods are included.
+     * For getter-like methods it is not generally possible to gain access "outside" of where you are supposed to have access,
+     * it is at most possible to get a Redis call to return more results than expected (e.g. by adding more members to [`geohash`](https://redis.io/commands/geohash)).
+     */
+    predicate argumentIsAmbiguousKey(string method, int argIndex) {
+      method =
+        [
+          "set", "publish", "append", "bitfield", "decrby", "getset", "hincrby", "hincrbyfloat",
+          "hset", "hsetnx", "incrby", "incrbyfloat", "linsert", "lpush", "lpushx", "lset", "ltrim",
+          "rename", "renamenx", "rpushx", "setbit", "setex", "smove", "zincrby", "zinterstore",
+          "hdel", "lpush", "pfadd", "rpush", "sadd", "sdiffstore", "srem"
+        ] and
+      argIndex = 0
+      or
+      method = ["bitop", "hmset", "mset", "msetnx", "geoadd"] and
+      argIndex in [0 .. any(DataFlow::InvokeNode invk).getNumArgument() - 1]
+    }
+  }
+
+  /**
+   * An expression that is interpreted as a key in a Node Redis call.
+   */
+  class RedisKeyArgument extends NoSQL::Query {
+    RedisKeyArgument() {
+      exists(string method, int argIndex |
+        QuerySignatures::argumentIsAmbiguousKey(method, argIndex) and
+        this = redis().getMember(method).getParameter(argIndex).getARhs().asExpr()
+      )
+    }
+  }
+
+  /**
+   * An access to a database through redis
+   */
+  class RedisDatabaseAccess extends DatabaseAccess, DataFlow::CallNode {
+    RedisDatabaseAccess() { this = redis().getMember(_).getACall() }
+
+    override DataFlow::Node getAResult() {
+      PromiseFlow::loadStep(this.getALocalUse(), result, Promises::valueProp())
+    }
+
+    override DataFlow::Node getAQueryArgument() { none() }
+  }
+}
+
+/**
+ * Provides classes modeling the `ioredis` library.
+ *
+ * ```
+ * import Redis from 'ioredis'
+ * let client = new Redis(...)
+ * ```
+ */
+private module IoRedis {
+  /**
+   * Gets an `ioredis` client.
+   */
+  API::Node ioredis() { result = API::moduleImport("ioredis").getInstance() }
+
+  /**
+   * An access to a database through ioredis
+   */
+  class IoRedisDatabaseAccess extends DatabaseAccess, DataFlow::CallNode {
+    IoRedisDatabaseAccess() { this = ioredis().getMember(_).getACall() }
+
+    override DataFlow::Node getAResult() {
+      PromiseFlow::loadStep(this.getALocalUse(), result, Promises::valueProp())
+    }
+
+    override DataFlow::Node getAQueryArgument() { none() }
+  }
 }

javascript/ql/lib/semmle/javascript/frameworks/NodeJSLib.qll

@@ -450,11 +450,56 @@ module NodeJSLib {
    */
   module FS {
     /**
-     * A member `member` from module `fs`.
+     * A member `member` from module `fs` or its drop-in replacements `graceful-fs`, `fs-extra`, `original-fs`.
      */
     DataFlow::SourceNode moduleMember(string member) {
-      exists(string moduleName | moduleName = ["fs"] |
-        result = DataFlow::moduleMember(moduleName, member)
+      result = fsModule(DataFlow::TypeTracker::end()).getAPropertyRead(member)
+    }
+
+    private DataFlow::SourceNode fsModule(DataFlow::TypeTracker t) {
+      exists(string moduleName |
+        moduleName = ["mz/fs", "original-fs", "fs-extra", "graceful-fs", "fs"]
+      |
+        result = DataFlow::moduleImport(moduleName)
+        or
+        // extra support for flexible names
+        result.asExpr().(Require).getArgument(0).mayHaveStringValue(moduleName)
+      ) and
+      t.start()
+      or
+      t.start() and
+      result = DataFlow::moduleMember("fs", "promises")
+      or
+      exists(DataFlow::TypeTracker t2, DataFlow::SourceNode pred | pred = fsModule(t2) |
+        result = pred.track(t2, t)
+        or
+        t.continue() = t2 and
+        exists(Promisify::PromisifyAllCall promisifyAllCall |
+          result = promisifyAllCall and
+          pred.flowsTo(promisifyAllCall.getArgument(0))
+        )
+        or
+        // const fs = require('fs');
+        // let fs_copy = methods.reduce((obj, method) => {
+        //  obj[method] = fs[method];
+        //  return obj;
+        // }, {});
+        t.continue() = t2 and
+        exists(
+          DataFlow::MethodCallNode call, DataFlow::ParameterNode obj, DataFlow::SourceNode method
+        |
+          call.getMethodName() = "reduce" and
+          result = call and
+          obj = call.getABoundCallbackParameter(0, 0) and
+          obj.flowsTo(any(DataFlow::FunctionNode f).getAReturn()) and
+          exists(DataFlow::PropWrite write, DataFlow::PropRead read |
+            write = obj.getAPropertyWrite() and
+            method.flowsToExpr(write.getPropertyNameExpr()) and
+            method.flowsToExpr(read.getPropertyNameExpr()) and
+            read.getBase().getALocalSource() = fsModule(t2) and
+            write.getRhs() = maybePromisified(read)
+          )
+        )
       )
     }
   }
@@ -465,7 +510,7 @@ module NodeJSLib {
   private class NodeJSFileSystemAccess extends FileSystemAccess, DataFlow::CallNode {
     string methodName;
 
-    NodeJSFileSystemAccess() { this = FS::moduleMember(methodName).getACall() }
+    NodeJSFileSystemAccess() { this = maybePromisified(FS::moduleMember(methodName)).getACall() }
 
     /**
      * Gets the name of the called method.

javascript/ql/lib/semmle/javascript/frameworks/SQL.qll

@@ -3,6 +3,7 @@
  */
 
 import javascript
+import semmle.javascript.Promises
 
 module SQL {
   /** A string-valued expression that is interpreted as a SQL command. */
@@ -32,38 +33,56 @@ module SQL {
  * Provides classes modeling the (API compatible) `mysql` and `mysql2` packages.
  */
 private module MySql {
-  private DataFlow::SourceNode mysql() { result = DataFlow::moduleImport(["mysql", "mysql2"]) }
+  private string moduleName() { result = ["mysql", "mysql2", "mysql2/promise"] }
 
-  private DataFlow::CallNode createPool() { result = mysql().getAMemberCall("createPool") }
+  /** Gets the package name `mysql` or `mysql2`. */
+  API::Node mysql() { result = API::moduleImport(moduleName()) }
 
-  /** Gets a reference to a MySQL pool. */
-  private DataFlow::SourceNode pool(DataFlow::TypeTracker t) {
-    t.start() and
-    result = createPool()
+  /** Gets a reference to `mysql.createConnection`. */
+  API::Node createConnection() {
+    result = mysql().getMember(["createConnection", "createConnectionPromise"])
   }
 
-  /** Gets a reference to a MySQL pool. */
-  private DataFlow::SourceNode pool() { result = pool(DataFlow::TypeTracker::end()) }
+  /** Gets a reference to `mysql.createPool`. */
+  API::Node createPool() { result = mysql().getMember(["createPool", "createPoolCluster"]) }
 
-  /** Gets a call to `mysql.createConnection`. */
-  DataFlow::CallNode createConnection() { result = mysql().getAMemberCall("createConnection") }
+  /** Gets a node that contains a MySQL pool created using `mysql.createPool()`. */
+  API::Node pool() {
+    result = createPool().getReturn()
+    or
+    result = pool().getMember("on").getReturn()
+    or
+    result = API::Node::ofType(moduleName(), ["Pool", "PoolCluster"])
+  }
 
-  /** Gets a reference to a MySQL connection instance. */
-  private DataFlow::SourceNode connection(DataFlow::TypeTracker t) {
-    t.start() and
-    (
-      result = createConnection()
-      or
-      result = pool().getAMethodCall("getConnection").getABoundCallbackParameter(0, 1)
+  /** Gets a data flow node that contains a freshly created MySQL connection instance. */
+  API::Node connection() {
+    result = createConnection().getReturn()
+    or
+    result = createConnection().getReturn().getPromised()
+    or
+    result = pool().getMember("getConnection").getParameter(0).getParameter(1)
+    or
+    result = pool().getMember("getConnection").getPromised()
+    or
+    exists(API::CallNode call |
+      call = pool().getMember("on").getACall() and
+      call.getArgument(0).getStringValue() = ["connection", "acquire", "release"] and
+      result = call.getParameter(1).getParameter(0)
     )
+    or
+    result = API::Node::ofType(moduleName(), ["Connection", "PoolConnection"])
   }
 
-  /** Gets a reference to a MySQL connection instance. */
-  DataFlow::SourceNode connection() { result = connection(DataFlow::TypeTracker::end()) }
-
   /** A call to the MySql `query` method. */
   private class QueryCall extends DatabaseAccess, DataFlow::MethodCallNode {
-    QueryCall() { this = [pool(), connection()].getAMethodCall("query") }
+    QueryCall() {
+      exists(API::Node recv | recv = pool() or recv = connection() |
+        this = recv.getMember(["query", "execute"]).getACall()
+      )
+    }
+
+    override DataFlow::Node getAResult() { result = this.getCallback(_).getParameter(1) }
 
     override DataFlow::Node getAQueryArgument() { result = this.getArgument(0) }
   }
@@ -76,7 +95,7 @@ private module MySql {
   /** A call to the `escape` or `escapeId` method that performs SQL sanitization. */
   class EscapingSanitizer extends SQL::SqlSanitizer, MethodCallExpr {
     EscapingSanitizer() {
-      this = [mysql(), pool(), connection()].getAMethodCall(["escape", "escapeId"]).asExpr() and
+      this = [mysql(), pool(), connection()].getMember(["escape", "escapeId"]).getACall().asExpr() and
       input = this.getArgument(0) and
       output = this
     }
@@ -87,8 +106,9 @@ private module MySql {
     string kind;
 
     Credentials() {
-      exists(string prop |
-        this = [createConnection(), createPool()].getOptionArgument(0, prop).asExpr() and
+      exists(API::Node callee, string prop |
+        callee in [createConnection(), createPool()] and
+        this = callee.getParameter(0).getMember(prop).getARhs().asExpr() and
         (
           prop = "user" and kind = "user name"
           or
@@ -105,23 +125,71 @@ private module MySql {
  * Provides classes modeling the PostgreSQL packages, such as `pg` and `pg-promise`.
  */
 private module Postgres {
-  /** Gets an expression that constructs a new connection pool. */
-  DataFlow::InvokeNode newPool() {
-    // new require('pg').Pool()
-    result = DataFlow::moduleImport("pg").getAConstructorInvocation("Pool")
+  API::Node pg() {
+    result = API::moduleImport("pg")
     or
-    // new require('pg-pool')
-    result = DataFlow::moduleImport("pg-pool").getAnInstantiation()
+    result = pgpMain().getMember("pg")
   }
 
-  /** Gets a creation of a Postgres client. */
-  DataFlow::InvokeNode newClient() {
-    result = DataFlow::moduleImport("pg").getAConstructorInvocation("Client")
+  /** Gets a reference to the `Client` constructor in the `pg` package, for example `require('pg').Client`. */
+  API::Node newClient() { result = pg().getMember("Client") }
+
+  /** Gets a freshly created Postgres client instance. */
+  API::Node client() {
+    result = newClient().getInstance()
+    or
+    // pool.connect(function(err, client) { ... })
+    result = pool().getMember("connect").getParameter(0).getParameter(1)
+    or
+    // await pool.connect()
+    result = pool().getMember("connect").getReturn().getPromised()
+    or
+    result = pgpConnection().getMember("client")
+    or
+    exists(API::CallNode call |
+      call = pool().getMember("on").getACall() and
+      call.getArgument(0).getStringValue() = ["connect", "acquire"] and
+      result = call.getParameter(1).getParameter(0)
+    )
+    or
+    result = client().getMember("on").getReturn()
+    or
+    result = API::Node::ofType("pg", ["Client", "PoolClient"])
+  }
+
+  /** Gets a constructor that when invoked constructs a new connection pool. */
+  API::Node newPool() {
+    // new require('pg').Pool()
+    result = pg().getMember("Pool")
+    or
+    // new require('pg-pool')
+    result = API::moduleImport("pg-pool")
+  }
+
+  /** Gets an API node that refers to a connection pool. */
+  API::Node pool() {
+    result = newPool().getInstance()
+    or
+    result = pgpDatabase().getMember("$pool")
+    or
+    result = pool().getMember("on").getReturn()
+    or
+    result = API::Node::ofType("pg", "Pool")
   }
 
   /** A call to the Postgres `query` method. */
   private class QueryCall extends DatabaseAccess, DataFlow::MethodCallNode {
-    QueryCall() { this = [newClient(), newPool()].getAMethodCall("query") }
+    QueryCall() { this = [client(), pool()].getMember("query").getACall() }
+
+    override DataFlow::Node getAResult() {
+      this.getNumArgument() = 2 and
+      result = this.getCallback(1).getParameter(1)
+      or
+      this.getNumArgument() = 1 and
+      result = this.getAMethodCall("then").getCallback(0).getParameter(0)
+      or
+      PromiseFlow::loadStep(this.getALocalUse(), result, Promises::valueProp())
+    }
 
     override DataFlow::Node getAQueryArgument() { result = this.getArgument(0) }
   }
@@ -140,7 +208,11 @@ private module Postgres {
     string kind;
 
     Credentials() {
-      exists(string prop | this = [newClient(), newPool()].getOptionArgument(0, prop).asExpr() |
+      exists(string prop |
+        this = [newClient(), newPool()].getParameter(0).getMember(prop).getARhs().asExpr()
+        or
+        this = pgPromise().getParameter(0).getMember(prop).getARhs().asExpr()
+      |
         prop = "user" and kind = "user name"
         or
         prop = "password" and kind = prop
@@ -263,6 +335,10 @@ private module Postgres {
       )
     }
 
+    override DataFlow::Node getAResult() {
+      PromiseFlow::loadStep(this.getALocalUse(), result, Promises::valueProp())
+    }
+
     override DataFlow::Node getAQueryArgument() {
       result = this.getADirectQueryArgument()
       or
@@ -281,30 +357,40 @@ private module Postgres {
  */
 private module Sqlite {
   /** Gets a reference to the `sqlite3` module. */
-  DataFlow::SourceNode sqlite() {
-    result = DataFlow::moduleImport("sqlite3")
+  API::Node sqlite() {
+    result = API::moduleImport("sqlite3")
     or
-    result = sqlite().getAMemberCall("verbose")
+    result = sqlite().getMember("verbose").getReturn()
   }
 
-  /** Gets an expression that constructs a Sqlite database instance. */
-  DataFlow::SourceNode newDb() {
+  /** Gets an expression that constructs or returns a Sqlite database instance. */
+  API::Node database() {
     // new require('sqlite3').Database()
-    result = sqlite().getAConstructorInvocation("Database")
+    result = sqlite().getMember("Database").getInstance()
+    or
+    // chained call
+    result = getAChainingQueryCall()
+    or
+    result = API::Node::ofType("sqlite3", "Database")
   }
 
-  /** Gets a data flow node referring to a Sqlite database instance. */
-  private DataFlow::SourceNode db(DataFlow::TypeTracker t) {
-    t.start() and
-    result = newDb()
+  /** A call to a query method on a Sqlite database instance that returns the same instance. */
+  private API::Node getAChainingQueryCall() {
+    result = database().getMember(["all", "each", "exec", "get", "run"]).getReturn()
   }
 
-  /** Gets a data flow node referring to a Sqlite database instance. */
-  DataFlow::SourceNode db() { result = db(DataFlow::TypeTracker::end()) }
-
   /** A call to a Sqlite query method. */
   private class QueryCall extends DatabaseAccess, DataFlow::MethodCallNode {
-    QueryCall() { this = db().getAMethodCall(["all", "each", "exec", "get", "prepare", "run"]) }
+    QueryCall() {
+      this = getAChainingQueryCall().getAnImmediateUse()
+      or
+      this = database().getMember("prepare").getACall()
+    }
+
+    override DataFlow::Node getAResult() {
+      result = this.getCallback(1).getParameter(1) or
+      PromiseFlow::loadStep(this.getALocalUse(), result, Promises::valueProp())
+    }
 
     override DataFlow::Node getAQueryArgument() { result = this.getArgument(0) }
   }
@@ -314,3 +400,203 @@ private module Sqlite {
     QueryString() { this = any(QueryCall qc).getAQueryArgument().asExpr() }
   }
 }
+
+/**
+ * Provides classes modeling the `mssql` package.
+ */
+private module MsSql {
+  /** Gets a reference to the `mssql` module. */
+  API::Node mssql() { result = API::moduleImport("mssql") }
+
+  /** Gets a node referring to an instance of the given class. */
+  API::Node mssqlClass(string name) {
+    result = mssql().getMember(name).getInstance()
+    or
+    result = API::Node::ofType("mssql", name)
+  }
+
+  /** Gets an API node referring to a Request object. */
+  API::Node request() {
+    result = mssqlClass("Request")
+    or
+    result = request().getMember(["input", "replaceInput", "output", "replaceOutput"]).getReturn()
+    or
+    result = [transaction(), pool()].getMember("request").getReturn()
+  }
+
+  /** Gets an API node referring to a Transaction object. */
+  API::Node transaction() {
+    result = mssqlClass("Transaction")
+    or
+    result = pool().getMember("transaction").getReturn()
+  }
+
+  /** Gets a API node referring to a ConnectionPool object. */
+  API::Node pool() { result = mssqlClass("ConnectionPool") }
+
+  /** A tagged template evaluated as a query. */
+  private class QueryTemplateExpr extends DatabaseAccess, DataFlow::ValueNode, DataFlow::SourceNode {
+    override TaggedTemplateExpr astNode;
+
+    QueryTemplateExpr() {
+      mssql().getMember("query").getAUse() = DataFlow::valueNode(astNode.getTag())
+    }
+
+    override DataFlow::Node getAResult() {
+      PromiseFlow::loadStep(this.getALocalUse(), result, Promises::valueProp())
+    }
+
+    override DataFlow::Node getAQueryArgument() {
+      result = DataFlow::valueNode(astNode.getTemplate().getAnElement())
+    }
+  }
+
+  /** A call to a MsSql query method. */
+  private class QueryCall extends DatabaseAccess, DataFlow::MethodCallNode {
+    QueryCall() { this = [mssql(), request()].getMember(["query", "batch"]).getACall() }
+
+    override DataFlow::Node getAResult() {
+      result = this.getCallback(1).getParameter(1)
+      or
+      PromiseFlow::loadStep(this.getALocalUse(), result, Promises::valueProp())
+    }
+
+    override DataFlow::Node getAQueryArgument() { result = this.getArgument(0) }
+  }
+
+  /** An expression that is passed to a method that interprets it as SQL. */
+  class QueryString extends SQL::SqlString {
+    QueryString() {
+      exists(DatabaseAccess dba | dba instanceof QueryTemplateExpr or dba instanceof QueryCall |
+        this = dba.getAQueryArgument().asExpr()
+      )
+    }
+  }
+
+  /** An element of a query template, which is automatically sanitized. */
+  class QueryTemplateSanitizer extends SQL::SqlSanitizer {
+    QueryTemplateSanitizer() {
+      this = any(QueryTemplateExpr qte).getAQueryArgument().asExpr() and
+      input = this and
+      output = this
+    }
+  }
+
+  /** An expression that is passed as user name or password when creating a client or a pool. */
+  class Credentials extends CredentialsExpr {
+    string kind;
+
+    Credentials() {
+      exists(API::Node callee, string prop |
+        (
+          callee = mssql().getMember("connect")
+          or
+          callee = mssql().getMember("ConnectionPool")
+        ) and
+        this = callee.getParameter(0).getMember(prop).getARhs().asExpr() and
+        (
+          prop = "user" and kind = "user name"
+          or
+          prop = "password" and kind = prop
+        )
+      )
+    }
+
+    override string getCredentialsKind() { result = kind }
+  }
+}
+
+/**
+ * Provides classes modeling the `sequelize` package.
+ */
+private module Sequelize {
+  class SequelizeModel extends ModelInput::TypeModelCsv {
+    override predicate row(string row) {
+      // package1;type1;package2;type2;path
+      row =
+        [
+          "sequelize;;sequelize-typescript;;", //
+          "sequelize;Sequelize;sequelize;default;", //
+          "sequelize;Sequelize;sequelize;;Instance",
+          "sequelize;Sequelize;sequelize;;Member[Sequelize].Instance",
+        ]
+    }
+  }
+
+  class SequelizeSink extends ModelInput::SinkModelCsv {
+    override predicate row(string row) {
+      row =
+        [
+          "sequelize;Sequelize;Member[query].Argument[0];sql-injection",
+          "sequelize;Sequelize;Member[query].Argument[0].Member[query];sql-injection",
+          "sequelize;;Member[literal,asIs].Argument[0];sql-injection",
+          "sequelize;;Argument[1];credentials[user name]",
+          "sequelize;;Argument[2];credentials[password]",
+          "sequelize;;Argument[0..].Member[username];credentials[user name]",
+          "sequelize;;Argument[0..].Member[password];credentials[password]"
+        ]
+    }
+  }
+
+  class SequelizeSource extends ModelInput::SourceModelCsv {
+    override predicate row(string row) {
+      row = "sequelize;Sequelize;Member[query].ReturnValue.Awaited;database-access-result"
+    }
+  }
+}
+
+private module SpannerCsv {
+  class SpannerTypes extends ModelInput::TypeModelCsv {
+    override predicate row(string row) {
+      // package1; type1; package2; type2; path
+      row =
+        [
+          "@google-cloud/spanner;;@google-cloud/spanner;;Member[Spanner]",
+          "@google-cloud/spanner;Database;@google-cloud/spanner;;ReturnValue.Member[instance].ReturnValue.Member[database].ReturnValue",
+          "@google-cloud/spanner;v1.SpannerClient;@google-cloud/spanner;;Member[v1].Member[SpannerClient].Instance",
+          "@google-cloud/spanner;Transaction;@google-cloud/spanner;Database;Member[runTransaction,runTransactionAsync,getTransaction].Argument[0..1].Parameter[1]",
+          "@google-cloud/spanner;Transaction;@google-cloud/spanner;Database;Member[getTransaction].ReturnValue.Awaited",
+          "@google-cloud/spanner;Snapshot;@google-cloud/spanner;Database;Member[getSnapshot].Argument[0..1].Parameter[1]",
+          "@google-cloud/spanner;Snapshot;@google-cloud/spanner;Database;Member[getSnapshot].ReturnValue.Awaited",
+          "@google-cloud/spanner;BatchTransaction;@google-cloud/spanner;Database;Member[batchTransaction].ReturnValue",
+          "@google-cloud/spanner;BatchTransaction;@google-cloud/spanner;Database;Member[createBatchTransaction].ReturnValue.Awaited",
+          "@google-cloud/spanner;~SqlExecutorDirect;@google-cloud/spanner;Database;Member[run,runPartitionedUpdate,runStream]",
+          "@google-cloud/spanner;~SqlExecutorDirect;@google-cloud/spanner;Transaction;Member[run,runStream,runUpdate]",
+          "@google-cloud/spanner;~SqlExecutorDirect;@google-cloud/spanner;BatchTransaction;Member[createQueryPartitions]",
+        ]
+    }
+  }
+
+  class SpannerSinks extends ModelInput::SinkModelCsv {
+    override predicate row(string row) {
+      // package; type; path; kind
+      row =
+        [
+          "@google-cloud/spanner;~SqlExecutorDirect;Argument[0];sql-injection",
+          "@google-cloud/spanner;~SqlExecutorDirect;Argument[0].Member[sql];sql-injection",
+          "@google-cloud/spanner;Transaction;Member[batchUpdate].Argument[0];sql-injection",
+          "@google-cloud/spanner;Transaction;Member[batchUpdate].Argument[0].ArrayElement.Member[sql];sql-injection",
+          "@google-cloud/spanner;v1.SpannerClient;Member[executeSql,executeStreamingSql].Argument[0].Member[sql];sql-injection",
+        ]
+    }
+  }
+
+  class SpannerSources extends ModelInput::SourceModelCsv {
+    string spannerClass() { result = ["v1.SpannerClient", "Database", "Transaction", "Snapshot",] }
+
+    string resultPath() {
+      result =
+        [
+          "Member[executeSql].Argument[0..].Parameter[1]",
+          "Member[executeSql].ReturnValue.Awaited.Member[0]", "Member[run].ReturnValue.Awaited",
+          "Member[run].Argument[0..].Parameter[1]",
+        ]
+    }
+
+    override predicate row(string row) {
+      row =
+        "@google-cloud/spanner;" + this.spannerClass() + ";" + this.resultPath() +
+          ";database-access-result"
+    }
+  }
+}

javascript/ql/lib/semmle/javascript/frameworks/TorrentLibraries.qll

@@ -36,7 +36,25 @@ module ParseTorrent {
    * An access to user-controlled torrent information.
    */
   class UserControlledTorrentInfo extends RemoteFlowSource {
-    UserControlledTorrentInfo() { none() }
+    UserControlledTorrentInfo() {
+      exists(DataFlow::SourceNode ref, DataFlow::PropRead read |
+        ref = parsedTorrentRef() and
+        read = ref.getAPropertyRead() and
+        this = read
+      |
+        exists(string prop |
+          not (
+            prop = "private" or
+            prop = "infoHash" or
+            prop = "length"
+            // "pieceLength" and "lastPieceLength" are not guaranteed to be numbers as of commit ae3ad15d
+          ) and
+          read.getPropertyName() = prop
+        )
+        or
+        not exists(read.getPropertyName())
+      )
+    }
 
     override string getSourceType() { result = "torrent information" }
   }

javascript/ql/lib/semmle/javascript/frameworks/data/internal/Shared.qll

@@ -31,7 +31,7 @@
  *     - Instance: the value returned by a constructor call
  *     - Awaited: the value from a resolved promise/future-like object
  *     - WithArity[n]: match a call with the given arity. May be a range of form `x..y` (inclusive) and/or a comma-separated list.
- *     - Other langauge-specific tokens mentioned in `ModelsAsData.qll`.
+ *     - Other language-specific tokens mentioned in `ModelsAsData.qll`.
  * 4. The `input` and `output` columns specify how data enters and leaves the element selected by the
  *    first `(package, type, path)` tuple. Both strings are `.`-separated access paths
  *    of the same syntax as the `path` column.

javascript/ql/lib/semmle/javascript/frameworks/jQuery.qll

@@ -485,6 +485,8 @@ module JQuery {
   private DataFlow::SourceNode dollar(DataFlow::TypeTracker t) {
     t.start() and
     result = dollarSource()
+    or
+    exists(DataFlow::TypeTracker t2 | result = dollar(t2).track(t2, t))
   }
 
   /**
@@ -518,6 +520,14 @@ module JQuery {
       }
     }
 
+    /**
+     * A `this` node in a JQuery plugin function, which is a JQuery object.
+     */
+    private class JQueryPluginThisObject extends Range {
+      JQueryPluginThisObject() {
+        this = DataFlow::thisNode(any(JQueryPluginMethod method).getFunction())
+      }
+    }
   }
 
   /** A source of jQuery objects from the AST-based `JQueryObject` class. */

javascript/ql/lib/semmle/javascript/heuristics/AdditionalSources.qll

@@ -14,7 +14,7 @@ private import semmle.javascript.security.dataflow.CommandInjectionCustomization
 abstract class HeuristicSource extends DataFlow::Node { }
 
 /**
- * An access to a password, viewed a source of remote flow.
+ * An access to a password, viewed as a source of remote flow.
  */
 private class RemoteFlowPassword extends HeuristicSource, RemoteFlowSource {
   RemoteFlowPassword() { isReadFrom(this, "(?is).*(password|passwd).*") }
@@ -52,3 +52,20 @@ class RemoteServerResponse extends HeuristicSource, RemoteFlowSource {
 
   override string getSourceType() { result = "a response from a remote server" }
 }
+
+/**
+ * A remote flow source originating from a database access.
+ */
+private class RemoteFlowSourceFromDBAccess extends RemoteFlowSource, HeuristicSource {
+  RemoteFlowSourceFromDBAccess() {
+    this = ModelOutput::getASourceNode("database-access-result").getAUse() or
+    exists(DatabaseAccess dba | this = dba.getAResult())
+  }
+
+  override string getSourceType() { result = "Database access" }
+
+  override predicate isUserControlledObject() {
+    // NB. supported databases all might return JSON.
+    any()
+  }
+}

javascript/ql/lib/semmle/javascript/security/dataflow/CodeInjectionCustomizations.qll

@@ -229,8 +229,14 @@ module CodeInjection {
   }
 
   /**
-   * The first argument to `Module.prototype._compile` from the Node.js built-in module `module`,
-   * considered as a code-injection sink.
+   * A code operator of a NoSQL query as a code injection sink.
+   */
+  class NoSQLCodeInjectionSink extends Sink {
+    NoSQLCodeInjectionSink() { any(NoSQL::Query q).getACodeOperator() = this }
+  }
+
+  /**
+   * The first argument to `Module.prototype._compile`, considered as a code-injection sink.
    */
   class ModuleCompileSink extends Sink {
     ModuleCompileSink() {

javascript/ql/lib/semmle/javascript/security/dataflow/TaintedPathCustomizations.qll

@@ -55,7 +55,7 @@ module TaintedPath {
      * There are currently four flow labels, representing the different combinations of
      * normalization and absoluteness.
      */
-    class PosixPath extends DataFlow::FlowLabel {
+    abstract class PosixPath extends DataFlow::FlowLabel {
       Normalization normalization;
       Relativeness relativeness;
 
@@ -113,7 +113,7 @@ module TaintedPath {
     /**
      * A flow label representing an array of path elements that may include "..".
      */
-    class SplitPath extends DataFlow::FlowLabel {
+    abstract class SplitPath extends DataFlow::FlowLabel {
       SplitPath() { this = "splitPath" }
     }
   }
@@ -218,12 +218,12 @@ module TaintedPath {
       output = this
       or
       // non-global replace or replace of something other than /\.\./g, /[/]/g, or /[\.]/g.
-      this.getCalleeName() = "replace" and
-      input = getReceiver() and
+      this instanceof StringReplaceCall and
+      input = this.getReceiver() and
       output = this and
       not exists(RegExpLiteral literal, RegExpTerm term |
-        getArgument(0).getALocalSource().asExpr() = literal and
-        literal.isGlobal() and
+        this.(StringReplaceCall).getRegExp().asExpr() = literal and
+        this.(StringReplaceCall).isGlobal() and
         literal.getRoot() = term
       |
         term.getAMatchedString() = "/" or
@@ -247,16 +247,15 @@ module TaintedPath {
   /**
    * A call that removes all instances of "../" in the prefix of the string.
    */
-  class DotDotSlashPrefixRemovingReplace extends DataFlow::CallNode {
+  class DotDotSlashPrefixRemovingReplace extends StringReplaceCall {
     DataFlow::Node input;
     DataFlow::Node output;
 
     DotDotSlashPrefixRemovingReplace() {
-      this.getCalleeName() = "replace" and
-      input = getReceiver() and
+      input = this.getReceiver() and
       output = this and
       exists(RegExpLiteral literal, RegExpTerm term |
-        getArgument(0).getALocalSource().asExpr() = literal and
+        this.getRegExp().asExpr() = literal and
         (term instanceof RegExpStar or term instanceof RegExpPlus) and
         term.getChild(0) = getADotDotSlashMatcher()
       |
@@ -298,17 +297,16 @@ module TaintedPath {
   /**
    * A call that removes all "." or ".." from a path, without also removing all forward slashes.
    */
-  class DotRemovingReplaceCall extends DataFlow::CallNode {
+  class DotRemovingReplaceCall extends StringReplaceCall {
     DataFlow::Node input;
     DataFlow::Node output;
 
     DotRemovingReplaceCall() {
-      this.getCalleeName() = "replace" and
-      input = getReceiver() and
+      input = this.getReceiver() and
       output = this and
+      this.isGlobal() and
       exists(RegExpLiteral literal, RegExpTerm term |
-        getArgument(0).getALocalSource().asExpr() = literal and
-        literal.isGlobal() and
+        this.getRegExp().asExpr() = literal and
         literal.getRoot() = term and
         not term.getAMatchedString() = "/"
       |
@@ -608,6 +606,8 @@ module TaintedPath {
       (
         this = fileSystemAccess.getAPathArgument() and
         not exists(fileSystemAccess.getRootPathArgument())
+        or
+        this = fileSystemAccess.getRootPathArgument()
       ) and
       not this = any(ResolvingPathCall call).getInput()
     }
@@ -648,6 +648,74 @@ module TaintedPath {
     AngularJSTemplateUrlSink() { this = any(AngularJS::CustomDirective d).getMember("templateUrl") }
   }
 
+  /**
+   * The path argument of a [send](https://www.npmjs.com/package/send) call, viewed as a sink.
+   */
+  class SendPathSink extends Sink, DataFlow::ValueNode {
+    SendPathSink() { this = DataFlow::moduleImport("send").getACall().getArgument(1) }
+  }
+
+  /**
+   * A path argument given to a `Page` in puppeteer, specifying where a pdf/screenshot should be saved.
+   */
+  private class PuppeteerPath extends TaintedPath::Sink {
+    PuppeteerPath() {
+      this =
+        Puppeteer::page()
+            .getMember(["pdf", "screenshot"])
+            .getParameter(0)
+            .getMember("path")
+            .getARhs()
+    }
+  }
+
+  /**
+   * An argument given to the `prettier` library specifying the location of a config file.
+   */
+  private class PrettierFileSink extends TaintedPath::Sink {
+    PrettierFileSink() {
+      this =
+        API::moduleImport("prettier")
+            .getMember(["resolveConfig", "resolveConfigFile", "getFileInfo"])
+            .getACall()
+            .getArgument(0)
+      or
+      this =
+        API::moduleImport("prettier")
+            .getMember("resolveConfig")
+            .getACall()
+            .getParameter(1)
+            .getMember("config")
+            .getARhs()
+    }
+  }
+
+  /**
+   * The `cwd` option for the `read-pkg` library.
+   */
+  private class ReadPkgCwdSink extends TaintedPath::Sink {
+    ReadPkgCwdSink() {
+      this =
+        API::moduleImport("read-pkg")
+            .getMember(["readPackageAsync", "readPackageSync"])
+            .getParameter(0)
+            .getMember("cwd")
+            .getARhs()
+    }
+  }
+
+  /**
+   * The `cwd` option to a shell execution.
+   */
+  private class ShellCwdSink extends TaintedPath::Sink {
+    ShellCwdSink() {
+      exists(SystemCommandExecution sys, API::Node opts |
+        opts.getARhs() = sys.getOptionsArg() and // assuming that an API::Node exists here.
+        this = opts.getMember("cwd").getARhs()
+      )
+    }
+  }
+
   /**
    * Holds if there is a step `src -> dst` mapping `srclabel` to `dstlabel` relevant for path traversal vulnerabilities.
    */

javascript/ql/lib/semmle/javascript/security/dataflow/Xss.qll

@@ -183,6 +183,17 @@ module DomBasedXss {
       this = any(Typeahead::TypeaheadSuggestionFunction f).getAReturn()
       or
       this = any(Handlebars::SafeString s).getAnArgument()
+      or
+      this = any(JQuery::MethodCall call | call.getMethodName() = "jGrowl").getArgument(0)
+      or
+      // A construction of a JSDOM object (server side DOM), where scripts are allowed.
+      exists(DataFlow::NewNode instance |
+        instance = API::moduleImport("jsdom").getMember("JSDOM").getInstance().getAnImmediateUse() and
+        this = instance.getArgument(0) and
+        instance.getOptionArgument(1, "runScripts").mayHaveStringValue("dangerously")
+      )
+      or
+      MooTools::interpretsNodeAsHtml(this)
     }
   }
 

javascript/ql/lib/semmle/javascript/security/performance/ReDoSUtil.qll

@@ -539,6 +539,55 @@ private class EdgeLabel extends TInputSymbol {
   }
 }
 
+/**
+ * A RegExp term that acts like a plus.
+ * Either it's a RegExpPlus, or it is a range {1,X} where X is >= 30.
+ * 30 has been chosen as a threshold because for exponential blowup 2^30 is enough to get a decent DOS attack.
+ */
+private class EffectivelyPlus extends RegExpTerm {
+  EffectivelyPlus() {
+    this instanceof RegExpPlus
+    or
+    exists(RegExpRange range |
+      range.getLowerBound() = 1 and
+      (range.getUpperBound() >= 30 or not exists(range.getUpperBound()))
+    |
+      this = range
+    )
+  }
+}
+
+/**
+ * A RegExp term that acts like a star.
+ * Either it's a RegExpStar, or it is a range {0,X} where X is >= 30.
+ */
+private class EffectivelyStar extends RegExpTerm {
+  EffectivelyStar() {
+    this instanceof RegExpStar
+    or
+    exists(RegExpRange range |
+      range.getLowerBound() = 0 and
+      (range.getUpperBound() >= 30 or not exists(range.getUpperBound()))
+    |
+      this = range
+    )
+  }
+}
+
+/**
+ * A RegExp term that acts like a question mark.
+ * Either it's a RegExpQuestion, or it is a range {0,1}.
+ */
+private class EffectivelyQuestion extends RegExpTerm {
+  EffectivelyQuestion() {
+    this instanceof RegExpOpt
+    or
+    exists(RegExpRange range | range.getLowerBound() = 0 and range.getUpperBound() = 1 |
+      this = range
+    )
+  }
+}
+
 /**
  * Gets the state before matching `t`.
  */
@@ -559,14 +608,14 @@ State after(RegExpTerm t) {
   or
   exists(RegExpGroup grp | t = grp.getAChild() | result = after(grp))
   or
-  exists(RegExpStar star | t = star.getAChild() | result = before(star))
+  exists(EffectivelyStar star | t = star.getAChild() | result = before(star))
   or
-  exists(RegExpPlus plus | t = plus.getAChild() |
+  exists(EffectivelyPlus plus | t = plus.getAChild() |
     result = before(plus) or
     result = after(plus)
   )
   or
-  exists(RegExpOpt opt | t = opt.getAChild() | result = after(opt))
+  exists(EffectivelyQuestion opt | t = opt.getAChild() | result = after(opt))
   or
   exists(RegExpRoot root | t = root | result = AcceptAnySuffix(root))
 }
@@ -617,15 +666,17 @@ predicate delta(State q1, EdgeLabel lbl, State q2) {
   or
   exists(RegExpGroup grp | lbl = Epsilon() | q1 = before(grp) and q2 = before(grp.getChild(0)))
   or
-  exists(RegExpStar star | lbl = Epsilon() |
+  exists(EffectivelyStar star | lbl = Epsilon() |
     q1 = before(star) and q2 = before(star.getChild(0))
     or
     q1 = before(star) and q2 = after(star)
   )
   or
-  exists(RegExpPlus plus | lbl = Epsilon() | q1 = before(plus) and q2 = before(plus.getChild(0)))
+  exists(EffectivelyPlus plus | lbl = Epsilon() |
+    q1 = before(plus) and q2 = before(plus.getChild(0))
+  )
   or
-  exists(RegExpOpt opt | lbl = Epsilon() |
+  exists(EffectivelyQuestion opt | lbl = Epsilon() |
     q1 = before(opt) and q2 = before(opt.getChild(0))
     or
     q1 = before(opt) and q2 = after(opt)

javascript/ql/test/library-tests/Security/heuristics/HeuristicSink.ql

@@ -1,4 +1,4 @@
 import javascript
 private import semmle.javascript.heuristics.AdditionalSinks
 
-select any(HeuristicSink s)
+select any(HeuristicSink s | s.getFile().getBaseName() = "sinks.js")

javascript/ql/test/library-tests/Security/heuristics/HeuristicSource.expected

@@ -1,2 +0,0 @@
-| sources.js:2:5:2:12 | password |
-| sources.js:3:5:3:20 | JSON.stringify() |

javascript/ql/test/library-tests/Security/heuristics/HeuristicSource.ql

@@ -1,4 +1,13 @@
 import javascript
 private import semmle.javascript.heuristics.AdditionalSources
+import testUtilities.ConsistencyChecking
 
-select any(HeuristicSource s)
+class Taint extends TaintTracking::Configuration {
+  Taint() { this = "Taint" }
+
+  override predicate isSource(DataFlow::Node node) { node instanceof HeuristicSource }
+
+  override predicate isSink(DataFlow::Node node) {
+    node = any(DataFlow::CallNode call | call.getCalleeName() = "sink").getAnArgument()
+  }
+}

javascript/ql/test/library-tests/Security/heuristics/sources.js

@@ -1,4 +1,236 @@
 (function() {
-    password;
-    JSON.stringify();
+    const password = '1234';
+    sink(password); // NOT OK
+    
+    const s = JSON.stringify();
+    sink(s); // NOT OK
+})();
+
+(async function() {
+    const knex = require('knex');
+    
+    const users = knex().select('*').from('users');
+    users.then(function (users) {
+        sink(users); // NOT OK
+    });
+
+    users.asCallback(function (err, users) {
+        sink(users); // NOT OK
+    });
+
+    sink(await users); // NOT OK
+})();
+
+(function() {
+    const pg = require('pg');
+    
+    const pool = new pg.Pool({});
+    pool.connect(async function (err, client, done) {
+        client.query('SELECT * FROM users', function (err, users) {
+            sink(users);
+        });
+        
+        const thenable = client.query('SELECT * FROM users')
+        thenable.then(function(users) {
+            sink(users); // NOT OK
+        });
+
+        const pgpromise = client.query('SELECT * FROM users');
+        sink(await pgpromise); // NOT OK
+    });
+})();
+
+(async function () {
+    const pgpromise = require('pg-promise')();
+    const db = pgpromise('postgres://username:password@localhost:1234/database');
+    const pgppromise = db.any('SELECT * FROM users');
+
+    pgppromise.then(function (users) {
+        sink(users);
+    });
+
+    sink(await pgppromise);
+})();
+
+(function () {
+    const mysql = require('mysql2');
+    const conn = mysql.createConnection({});
+
+    conn.query(
+        'SELECT * FROM `users`',
+        function(err, users, fields) {
+          sink(users); // NOT OK
+        }
+    );
+
+    conn.execute(
+        'SELECT * FROM `users` WHERE name = ?',
+        ['Alice'],
+        function(err, users) {
+            sink(users);
+        }
+    );
+})();
+
+(async function () {
+    const sqlite = require('sqlite3');
+    const db = new sqlite.Database(':memory:');
+
+    db.all('SELECT * FROM users', function (err, users) {
+        sink(users); // NOT OK
+    });
+    
+    const sqlitepromise = db.all('SELECT * FROM users');
+
+    sink(await sqlitepromise); // NOT OK
+})();
+
+(async function () {
+    const { Sequelize } = require('sequelize');
+    const sequelize = new Sequelize('sqlite::memory:');
+
+    class User extends sequelize.Model {}
+    User.init({ name: sequelize.DataTypes.String }, { sequelize, modelName: 'user' });
+
+    sequelize.query('SELECT * FROM users').then(function (users) {
+        sink(users); // NOT OK
+    });
+})();
+
+(async function () {
+    const sql = require('mssql');
+    await sql.connect('...');
+    
+    sql.query('SELECT * FROM users', function (err, users) {
+        sink(users); // NOT OK
+    });
+
+    const mssqlthenable = sql.query('SELECT * FROM users');
+    
+    mssqlthenable.then(function (users) {
+        sink(users); // NOT OK
+    });
+
+    const mssqlpromise = sql.query('SELECT * FROM users');
+    sink(await mssqlpromise); // NOT OK
+
+    const uname = 'Alice';
+    const mssqltaggedquery = sql.query`SELECT * FROM users where name=${uname}`
+    sink(await mssqltaggedquery); // NOT OK
+})();
+
+(async function () {
+    const {Spanner} = require('@google-cloud/spanner');
+    const db = new Spanner({projectId: 'test'})
+        .instance('instanceid')
+        .database('databaseid');
+
+    db.executeSql('SELECT * FROM users', {}, function (err, users) {
+        sink(users); // NOT OK
+    });
+
+    const [users] = (await db.executeSql('SELECT * FROM users', {}));
+    sink(users); // NOT OK
+    
+    const spannerpromise = db.run({
+        sql: 'SELECT * FROM users'
+    });
+
+    sink(await spannerpromise); // NOT OK
+
+    db.run({
+        sql: 'SELECT * FROM users'
+    }, function (err, rows, stats, meta) {
+        sink(rows); // NOT OK
+    });
+
+    const client = new Spanner.v1.SpannerClient({});
+    client.executeSql('SELECT * FROM users', {}, function (err, users) {
+        sink(users); // NOT OK
+    });
+
+    db.runTransaction(function(err, txn) {
+        txn.run('SELECT * FROM users', function (err, users) {
+            sink(users); // NOT OK
+        });
+        txn.commit(function () {});
+    });
+
+    db.getSnapshot(function (err, txn) {
+        txn.run('SELECT * FROM users', function (err, users) {
+            sink(users); // NOT OK
+        });
+        txn.end();
+    });
+})();
+
+(function () {
+    const { MongoClient } = require('mongodb');
+    
+    MongoClient.connect('mongodb://localhost:1234', async function (err, db) {
+        const collection = db.collection('users');
+        const users = await collection.find({});
+        sink(users); // NOT OK
+    });
+})();
+
+(async function () {
+    const mongoose = require('mongoose');
+    await mongoose.connect('mongodb://localhost:1234');
+
+    const User = mongoose.model('User', {
+        name: {
+            type: String,
+            unique: true
+        }
+    });
+
+    User.find({ name: 'Alice' }, function (err, alice) {
+        sink(alice); // NOT OK
+    });
+
+    User.find({ name: 'Bob' }).exec(function (err, bob) {
+        sink(bob); // NOT OK
+    });
+
+    const promise = User.find({ name: 'Claire' });
+    promise.then(c => sink(c)); // NOT OK
+})();
+
+(async function () {
+    const minimongo = require('minimongo');
+    const LocalDb = minimongo.MemoryDb;
+    const db = new LocalDb();
+    const doc = db.users;
+
+    const users = await doc.find({});
+    sink(users); // NOT OK
+})();
+
+(async function () {
+    const { Collection } = require('marsdb');
+    
+    const doc = new Collection('users');
+
+    const users = await doc.find({});
+    
+    sink(users); // NOT OK
+})();
+
+(async function () {
+    const redis = require("redis");
+    const client = redis.createClient();
+
+    const alice = await client.get('alice');
+
+    sink(alice); // NOT OK
+})();
+
+(async function () {
+    const Redis = require('ioredis');
+    const redis = new Redis();
+
+    const bob = await redis.get('bob');
+
+    sink(bob); // NOT OK
 })();

javascript/ql/test/query-tests/Performance/ReDoS/PolynomialBackTracking.expected

@@ -201,6 +201,7 @@
 | regexplib/markup.js:13:14:13:16 | .+? | Strings starting with '<' and with many repetitions of '!' can start matching anywhere after the start of the preceeding .*? |
 | regexplib/markup.js:14:13:14:14 | .* | Strings starting with '<' and with many repetitions of 'a' can start matching anywhere after the start of the preceeding .* |
 | regexplib/markup.js:14:24:14:25 | .* | Strings starting with '<>' and with many repetitions of '>a' can start matching anywhere after the start of the preceeding .* |
+| regexplib/markup.js:15:16:15:18 | .*? | Strings starting with '<img' and with many repetitions of '<imga' can start matching anywhere after the start of the preceeding <(\\/{0,1})img(.*?)(\\/{0,1})\\> |
 | regexplib/markup.js:16:5:16:9 | [^>]* | Strings starting with 'src' and with many repetitions of 'src' can start matching anywhere after the start of the preceeding src[^>]*[^/].(?:jpg\|bmp\|gif)(?:\\"\|\\') |
 | regexplib/markup.js:17:8:17:24 | (\\s(\\w*=".*?")?)* | Strings starting with '<a' and with many repetitions of ' =""' can start matching anywhere after the start of the preceeding .*? |
 | regexplib/markup.js:17:12:17:14 | \\w* | Strings starting with '<a ' and with many repetitions of '="" a' can start matching anywhere after the start of the preceeding .*? |
@@ -213,6 +214,10 @@
 | regexplib/markup.js:20:197:20:198 | "+ | Strings with many repetitions of '""' can start matching anywhere after the start of the preceeding "+ |
 | regexplib/markup.js:20:245:20:247 | .*? | Strings with many repetitions of 'color: # IF found THEN move ahead "" # single or double  # or no quotes\\t' can start matching anywhere after the start of the preceeding .*? |
 | regexplib/markup.js:20:274:20:276 | .*? | Strings starting with '<font # Match start of Font Tag ' and with many repetitions of '<font # Match start of Font Tag a' can start matching anywhere after the start of the preceeding <\\*?font # Match start of Font Tag (?(?=[^>]+color.*>) #IF\\/THEN lookahead color in tag (.*?color\\s*?[=\|:]\\s*?) # IF found THEN move ahead ('+\\#*?[\\w\\s]*'+ # CAPTURE ColorName\\/Hex \|"+\\#*?[\\w\\s]*"+ # single or double \|\\#*\\w*\\b) # or no quotes\t.*?> # & move to end of tag \|.*?> # ELSE move to end of Tag ) # Close the If\\/Then lookahead # Use Multiline and IgnoreCase # Replace the matches from RE with MatchEvaluator below: # if m.Groups(1).Value<>"" then # Return "<font color=" & m.Groups(1).Value & ">" # else # Return "<font>" # end if |
+| regexplib/markup.js:24:39:24:41 | \\s+ | Strings starting with '&lt;A' and with many repetitions of ' - != ' can start matching anywhere after the start of the preceeding \\s* |
+| regexplib/markup.js:24:43:24:45 | \\S+ | Strings starting with '&lt;A ' and with many repetitions of '- !=' can start matching anywhere after the start of the preceeding \\s* |
+| regexplib/markup.js:24:48:24:50 | \\s* | Strings starting with '&lt;A !' and with many repetitions of ' =- ! ' can start matching anywhere after the start of the preceeding \\s+ |
+| regexplib/markup.js:24:52:24:54 | \\s* | Strings starting with '&lt;A !=' and with many repetitions of '- !=' can start matching anywhere after the start of the preceeding \\S+ |
 | regexplib/markup.js:25:11:25:15 | [^>]* | Strings starting with '<A' and with many repetitions of '<A' can start matching anywhere after the start of the preceeding <[a-zA-Z][^>]*\\son\\w+=(\\w+\|'[^']*'\|"[^"]*")[^>]*> |
 | regexplib/markup.js:25:45:25:49 | [^>]* | Strings starting with '<A ona=a' and with many repetitions of '0' can start matching anywhere after the start of the preceeding \\w+ |
 | regexplib/markup.js:27:3:27:7 | [^>]* | Strings starting with '<' and with many repetitions of '<' can start matching anywhere after the start of the preceeding <[^>]*name[\\s]*=[\\s]*"?[^\\w_]*"?[^>]*> |
@@ -228,6 +233,10 @@
 | regexplib/markup.js:44:3:44:7 | [^>]* | Strings starting with '<' and with many repetitions of '<' can start matching anywhere after the start of the preceeding <[^>]*name[\\s]*=[\\s]*"?[^\\w_]*"?[^>]*> |
 | regexplib/markup.js:44:34:44:38 | [^>]* | Strings starting with '<name=' and with many repetitions of '\\t' can start matching anywhere after the start of the preceeding [\\s]* |
 | regexplib/markup.js:45:6:45:13 | [\\d\\D]*? | Strings starting with '/*' and with many repetitions of 'a/*' can start matching anywhere after the start of the preceeding \\/\\*[\\d\\D]*?\\*\\/ |
+| regexplib/markup.js:47:39:47:41 | \\s+ | Strings starting with '&lt;A' and with many repetitions of ' - != ' can start matching anywhere after the start of the preceeding \\s* |
+| regexplib/markup.js:47:43:47:45 | \\S+ | Strings starting with '&lt;A ' and with many repetitions of '- !=' can start matching anywhere after the start of the preceeding \\s* |
+| regexplib/markup.js:47:48:47:50 | \\s* | Strings starting with '&lt;A !' and with many repetitions of ' =- ! ' can start matching anywhere after the start of the preceeding \\s+ |
+| regexplib/markup.js:47:52:47:54 | \\s* | Strings starting with '&lt;A !=' and with many repetitions of '- !=' can start matching anywhere after the start of the preceeding \\S+ |
 | regexplib/markup.js:48:6:48:13 | [\\s\\S]*? | Strings starting with '<!--' and with many repetitions of '<!--' can start matching anywhere after the start of the preceeding <!--[\\s\\S]*?--> |
 | regexplib/markup.js:53:15:53:19 | [\\w]* | Strings starting with '[a' and with many repetitions of '0' can start matching anywhere after the start of the preceeding \\w+ |
 | regexplib/markup.js:56:23:56:25 | \\w+ | Strings with many repetitions of 'a' can start matching anywhere after the start of the preceeding (\\/?(?<step>\\w+))+ |
@@ -300,6 +309,7 @@
 | regexplib/strings.js:14:61:14:63 | \\w* | Strings starting with 'AA' and with many repetitions of 'A' can start matching anywhere after the start of the preceeding \\w* |
 | regexplib/strings.js:14:107:14:109 | \\w* | Strings starting with 'AAA' and with many repetitions of 'A' can start matching anywhere after the start of the preceeding \\w* |
 | regexplib/strings.js:19:31:19:57 | [a-z&#230;&#248;&#229;0-9]+ | Strings starting with '#@' and with many repetitions of '##' can start matching anywhere after the start of the preceeding [a-z&#230;&#248;&#229;0-9]+ |
+| regexplib/strings.js:19:69:19:95 | [a-z&#230;&#248;&#229;0-9]+ | Strings starting with '#@#' and with many repetitions of '##' can start matching anywhere after the start of the preceeding [a-z&#230;&#248;&#229;0-9]+ |
 | regexplib/strings.js:20:3:20:20 | ((\\\\")\|[^"(\\\\")])+ | Strings starting with '"' and with many repetitions of '\\\\"' can start matching anywhere after the start of the preceeding "((\\\\")\|[^"(\\\\")])+" |
 | regexplib/strings.js:21:3:21:7 | [^>]+ | Strings starting with '<' and with many repetitions of '<' can start matching anywhere after the start of the preceeding <[^>]+> |
 | regexplib/strings.js:23:3:23:20 | ((\\\\")\|[^"(\\\\")])+ | Strings starting with '"' and with many repetitions of '\\\\"' can start matching anywhere after the start of the preceeding "((\\\\")\|[^"(\\\\")])+" |
@@ -313,8 +323,10 @@
 | regexplib/strings.js:40:3:40:5 | \\w+ | Strings with many repetitions of 'a' can start matching anywhere after the start of the preceeding (\\w+)\\s+\\1 |
 | regexplib/strings.js:48:3:48:12 | [^\\.\\?\\!]* | Strings with many repetitions of ' ' can start matching anywhere after the start of the preceeding ([^\\.\\?\\!]*)[\\.\\?\\!] |
 | regexplib/strings.js:49:3:49:5 | \\S+ | Strings with many repetitions of '!' can start matching anywhere after the start of the preceeding (\\S+)\\x20{2,}(?=\\S+) |
-| regexplib/strings.js:53:25:53:33 | [a-z0-9]+ | Strings with many repetitions of '0' can start matching anywhere after the start of the preceeding [a-z0-9]+ |
-| regexplib/strings.js:53:65:53:73 | [a-z0-9]+ | Strings with many repetitions of '0' can start matching anywhere after the start of the preceeding [a-z0-9]+ |
+| regexplib/strings.js:53:4:53:12 | [a-z0-9]+ | Strings with many repetitions of '0.00' can start matching anywhere after the start of the preceeding [a-z0-9]+ |
+| regexplib/strings.js:53:25:53:33 | [a-z0-9]+ | Strings starting with '0' and with many repetitions of '0' can start matching anywhere after the start of the preceeding [a-z0-9]+ |
+| regexplib/strings.js:53:44:53:52 | [a-z0-9]+ | Strings with many repetitions of '00' can start matching anywhere after the start of the preceeding [a-z0-9]+ |
+| regexplib/strings.js:53:65:53:73 | [a-z0-9]+ | Strings starting with '0' and with many repetitions of '0' can start matching anywhere after the start of the preceeding [a-z0-9]+ |
 | regexplib/strings.js:54:20:54:22 | \\w+ | Strings with many repetitions of 'a' can start matching anywhere after the start of the preceeding (NOT)?(\\s*\\(*)\\s*(\\w+)\\s*(=\|<>\|<\|>\|LIKE\|IN)\\s*(\\(([^\\)]*)\\)\|'([^']*)'\|(-?\\d*\\.?\\d+))(\\s*\\)*\\s*)(AND\|OR)? |
 | regexplib/strings.js:56:52:56:53 | .+ | Strings starting with 'PRN.' and with many repetitions of '.' can start matching anywhere after the start of the preceeding .* |
 | regexplib/strings.js:57:36:57:38 | .*? | Strings starting with '?se[A' and with many repetitions of '?se[Aa' can start matching anywhere after the start of the preceeding (?s)(?:\\e\\[(?:(\\d+);?)*([A-Za-z])(.*?))(?=\\e\\[\|\\z) |
@@ -519,3 +531,6 @@
 | tst.js:399:6:399:12 | (d\|dd)* | Strings with many repetitions of 'd' can start matching anywhere after the start of the preceeding ((c\|cc)*\|(d\|dd)*\|(e\|ee)*)f$ |
 | tst.js:400:6:401:1 | (e\|ee)* | Strings with many repetitions of 'e' can start matching anywhere after the start of the preceeding ((c\|cc)*\|(d\|dd)*\|(e\|ee)*)f$ |
 | tst.js:404:6:405:7 | (g\|gg)* | Strings with many repetitions of 'g' can start matching anywhere after the start of the preceeding (g\|gg)*h$ |
+| tst.js:407:128:407:129 |  * | Strings starting with '0/*' and with many repetitions of ' ' can start matching anywhere after the start of the preceeding \\s* |
+| tst.js:409:23:409:29 | [\\w.-]* | Strings starting with '//' and with many repetitions of '//' can start matching anywhere after the start of the preceeding (\\/(?:\\/[\\w.-]*)*){0,1}:([\\w.-]+) |
+| tst.js:411:15:411:19 | a{1,} | Strings with many repetitions of 'a' can start matching anywhere after the start of the preceeding (a{1,})* |

javascript/ql/test/query-tests/Performance/ReDoS/ReDoS.expected

@@ -29,12 +29,14 @@
 | regexplib/email.js:5:24:5:35 | [a-zA-Z0-9]+ | This part of the regular expression may cause exponential backtracking on strings starting with '0' and containing many repetitions of '0'. |
 | regexplib/email.js:5:63:5:74 | [a-zA-Z0-9]+ | This part of the regular expression may cause exponential backtracking on strings starting with '0@0' and containing many repetitions of '0'. |
 | regexplib/email.js:6:10:6:35 | (?:[a-zA-Z0-9][\\.\\-\\+_]?)* | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of '0'. |
+| regexplib/email.js:6:60:6:88 | (?:[a-zA-Z0-9][\\.\\-_]?){0,62} | This part of the regular expression may cause exponential backtracking on strings starting with '0@' and containing many repetitions of '0'. |
 | regexplib/email.js:13:36:13:44 | [a-zA-Z]* | This part of the regular expression may cause exponential backtracking on strings starting with 'A' and containing many repetitions of 'A'. |
 | regexplib/email.js:25:67:25:78 | [a-zA-Z0-9]+ | This part of the regular expression may cause exponential backtracking on strings starting with '0' and containing many repetitions of '0'. |
 | regexplib/email.js:25:106:25:117 | [a-zA-Z0-9]+ | This part of the regular expression may cause exponential backtracking on strings starting with '0@0' and containing many repetitions of '0'. |
 | regexplib/email.js:25:212:25:223 | [a-zA-Z0-9]+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of '0'. |
 | regexplib/email.js:25:251:25:262 | [a-zA-Z0-9]+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of '0'. |
 | regexplib/email.js:32:10:32:25 | (?:\\w[\\.\\-\\+]?)* | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'a'. |
+| regexplib/email.js:32:41:32:61 | (?:\\w[\\.\\-\\+]?){0,62} | This part of the regular expression may cause exponential backtracking on strings starting with 'a@' and containing many repetitions of 'a'. |
 | regexplib/email.js:33:16:33:22 | [-.\\w]* | This part of the regular expression may cause exponential backtracking on strings starting with '0' and containing many repetitions of '0'. |
 | regexplib/email.js:33:38:33:51 | ([0-9a-zA-Z])+ | This part of the regular expression may cause exponential backtracking on strings starting with '0@' and containing many repetitions of '00.'. |
 | regexplib/email.js:33:53:33:58 | [-\\w]* | This part of the regular expression may cause exponential backtracking on strings starting with '0@0' and containing many repetitions of '0'. |
@@ -45,9 +47,13 @@
 | regexplib/markup.js:13:6:13:12 | [^"']+? | This part of the regular expression may cause exponential backtracking on strings starting with '<' and containing many repetitions of '!'. |
 | regexplib/markup.js:13:14:13:16 | .+? | This part of the regular expression may cause exponential backtracking on strings starting with '<' and containing many repetitions of 'a"'. |
 | regexplib/markup.js:17:17:17:19 | .*? | This part of the regular expression may cause exponential backtracking on strings starting with '<a ="' and containing many repetitions of '" ="'. |
+| regexplib/markup.js:24:43:24:45 | \\S+ | This part of the regular expression may cause exponential backtracking on strings starting with '&lt;A ' and containing many repetitions of '!=- '. |
+| regexplib/markup.js:24:47:24:118 | (\\s*=\\s*([-\\w\\.]{1,1024}\|&quot;[^&quot;]{0,1024}&quot;\|'[^']{0,1024}'))? | This part of the regular expression may cause exponential backtracking on strings starting with '&lt;A !' and containing many repetitions of ' =- !'. |
 | regexplib/markup.js:37:29:37:56 | [a-zA-Z0-9\|:\|\\/\|=\|-\|.\|\\?\|&]* | This part of the regular expression may cause exponential backtracking on strings starting with '[a=' and containing many repetitions of '='. |
 | regexplib/markup.js:40:23:40:25 | \\w+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'a'. |
 | regexplib/markup.js:40:132:40:134 | \\s* | This part of the regular expression may cause exponential backtracking on strings starting with 'a[@a=''' and containing many repetitions of ' @a<""'. |
+| regexplib/markup.js:47:43:47:45 | \\S+ | This part of the regular expression may cause exponential backtracking on strings starting with '&lt;A ' and containing many repetitions of '!=- '. |
+| regexplib/markup.js:47:47:47:118 | (\\s*=\\s*([-\\w\\.]{1,1024}\|&quot;[^&quot;]{0,1024}&quot;\|'[^']{0,1024}'))? | This part of the regular expression may cause exponential backtracking on strings starting with '&lt;A !' and containing many repetitions of ' =- !'. |
 | regexplib/markup.js:53:29:53:56 | [a-zA-Z0-9\|:\|\\/\|=\|-\|.\|\\?\|&]* | This part of the regular expression may cause exponential backtracking on strings starting with '[a=' and containing many repetitions of '='. |
 | regexplib/markup.js:56:23:56:25 | \\w+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'a'. |
 | regexplib/markup.js:56:132:56:134 | \\s* | This part of the regular expression may cause exponential backtracking on strings starting with 'a[@a=''' and containing many repetitions of ' @a<""'. |
@@ -62,7 +68,10 @@
 | regexplib/misc.js:148:23:148:29 | [^"'=]+ | This part of the regular expression may cause exponential backtracking on strings starting with '<! ' and containing many repetitions of '! '. |
 | regexplib/misc.js:173:4:173:11 | ([a-z])+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'aa'. |
 | regexplib/strings.js:19:31:19:57 | [a-z&#230;&#248;&#229;0-9]+ | This part of the regular expression may cause exponential backtracking on strings starting with '#@' and containing many repetitions of '#'. |
+| regexplib/strings.js:19:69:19:95 | [a-z&#230;&#248;&#229;0-9]+ | This part of the regular expression may cause exponential backtracking on strings starting with '#@#' and containing many repetitions of '##'. |
 | regexplib/strings.js:47:3:47:5 | \\S* | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of '!'. |
+| regexplib/strings.js:53:4:53:12 | [a-z0-9]+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of '00.'. |
+| regexplib/strings.js:53:14:53:24 | [\\-a-z0-9]* | This part of the regular expression may cause exponential backtracking on strings starting with '0' and containing many repetitions of '00.0'. |
 | regexplib/strings.js:57:17:57:19 | \\d+ | This part of the regular expression may cause exponential backtracking on strings starting with '?se[' and containing many repetitions of '9'. |
 | regexplib/strings.js:81:17:81:19 | \\d+ | This part of the regular expression may cause exponential backtracking on strings starting with '?se[' and containing many repetitions of '9'. |
 | regexplib/strings.js:91:3:91:5 | \\S* | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of '!'. |
@@ -188,3 +197,5 @@
 | tst.js:399:6:399:12 | (d\|dd)* | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'dd'. |
 | tst.js:400:6:401:1 | (e\|ee)* | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'ee'. |
 | tst.js:404:6:405:7 | (g\|gg)* | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'gg'. |
+| tst.js:407:125:407:127 | \\s* | This part of the regular expression may cause exponential backtracking on strings starting with '0/*' and containing many repetitions of ' ;0'. |
+| tst.js:411:15:411:19 | a{1,} | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'a'. |

javascript/ql/test/query-tests/Performance/ReDoS/tst.js

@@ -362,8 +362,8 @@ var bad84 = /^((?:a{0|-)|\w\{\d)+X$/;
 var bad85 = /^((?:a{0,|-)|\w\{\d,)+X$/;
 var bad86 = /^((?:a{0,2|-)|\w\{\d,\d)+X$/;
 
-// GOOD:
-var good42 = /^((?:a{0,2}|-)|\w\{\d,\d\})+X$/;
+// NOT GOOD - but not flagged
+var bad86AndAHalf = /^((?:a{0,2}|-)|\w\{\d,\d\})+X$/;
 
 // GOOD
 var good43 = /("[^"]*?"|[^"\s]+)+(?=\s*|\s*$)/g;
@@ -403,3 +403,9 @@ var bad96 = new RegExp("(" +
 var bad97 = new RegExp(
     "(g|gg" + 
     ")*h$");
+
+var bad98 = /^(?:\*\/\*|[a-zA-Z0-9][a-zA-Z0-9!\#\$&\-\^_\.\+]{0,126}\/(?:\*|[a-zA-Z0-9][a-zA-Z0-9!\#\$&\-\^_\.\+]{0,126})(?:\s* *; *[a-zA-Z0-9][a-zA-Z0-9!\#\$&\-\^_\.\+]{0,126}(?:="?[a-zA-Z0-9][a-zA-Z0-9!\#\$&\-\^_\.\+]{0,126}"?)?\s*)*)$/;
+
+var good48 = /(\/(?:\/[\w.-]*)*){0,1}:([\w.-]+)/;
+
+var bad99 = /(a{1,})*b/;

python/ql/lib/semmle/python/security/performance/ReDoSUtil.qll

@@ -539,6 +539,55 @@ private class EdgeLabel extends TInputSymbol {
   }
 }
 
+/**
+ * A RegExp term that acts like a plus.
+ * Either it's a RegExpPlus, or it is a range {1,X} where X is >= 30.
+ * 30 has been chosen as a threshold because for exponential blowup 2^30 is enough to get a decent DOS attack.
+ */
+private class EffectivelyPlus extends RegExpTerm {
+  EffectivelyPlus() {
+    this instanceof RegExpPlus
+    or
+    exists(RegExpRange range |
+      range.getLowerBound() = 1 and
+      (range.getUpperBound() >= 30 or not exists(range.getUpperBound()))
+    |
+      this = range
+    )
+  }
+}
+
+/**
+ * A RegExp term that acts like a star.
+ * Either it's a RegExpStar, or it is a range {0,X} where X is >= 30.
+ */
+private class EffectivelyStar extends RegExpTerm {
+  EffectivelyStar() {
+    this instanceof RegExpStar
+    or
+    exists(RegExpRange range |
+      range.getLowerBound() = 0 and
+      (range.getUpperBound() >= 30 or not exists(range.getUpperBound()))
+    |
+      this = range
+    )
+  }
+}
+
+/**
+ * A RegExp term that acts like a question mark.
+ * Either it's a RegExpQuestion, or it is a range {0,1}.
+ */
+private class EffectivelyQuestion extends RegExpTerm {
+  EffectivelyQuestion() {
+    this instanceof RegExpOpt
+    or
+    exists(RegExpRange range | range.getLowerBound() = 0 and range.getUpperBound() = 1 |
+      this = range
+    )
+  }
+}
+
 /**
  * Gets the state before matching `t`.
  */
@@ -559,14 +608,14 @@ State after(RegExpTerm t) {
   or
   exists(RegExpGroup grp | t = grp.getAChild() | result = after(grp))
   or
-  exists(RegExpStar star | t = star.getAChild() | result = before(star))
+  exists(EffectivelyStar star | t = star.getAChild() | result = before(star))
   or
-  exists(RegExpPlus plus | t = plus.getAChild() |
+  exists(EffectivelyPlus plus | t = plus.getAChild() |
     result = before(plus) or
     result = after(plus)
   )
   or
-  exists(RegExpOpt opt | t = opt.getAChild() | result = after(opt))
+  exists(EffectivelyQuestion opt | t = opt.getAChild() | result = after(opt))
   or
   exists(RegExpRoot root | t = root | result = AcceptAnySuffix(root))
 }
@@ -617,15 +666,17 @@ predicate delta(State q1, EdgeLabel lbl, State q2) {
   or
   exists(RegExpGroup grp | lbl = Epsilon() | q1 = before(grp) and q2 = before(grp.getChild(0)))
   or
-  exists(RegExpStar star | lbl = Epsilon() |
+  exists(EffectivelyStar star | lbl = Epsilon() |
     q1 = before(star) and q2 = before(star.getChild(0))
     or
     q1 = before(star) and q2 = after(star)
   )
   or
-  exists(RegExpPlus plus | lbl = Epsilon() | q1 = before(plus) and q2 = before(plus.getChild(0)))
+  exists(EffectivelyPlus plus | lbl = Epsilon() |
+    q1 = before(plus) and q2 = before(plus.getChild(0))
+  )
   or
-  exists(RegExpOpt opt | lbl = Epsilon() |
+  exists(EffectivelyQuestion opt | lbl = Epsilon() |
     q1 = before(opt) and q2 = before(opt.getChild(0))
     or
     q1 = before(opt) and q2 = after(opt)

ruby/extractor/Cargo.toml

@@ -12,7 +12,7 @@ node-types = { path = "../node-types" }
 tree-sitter = "0.19"
 tree-sitter-embedded-template = "0.19"
 tree-sitter-ruby = { git = "https://github.com/tree-sitter/tree-sitter-ruby.git", rev = "888e2e563ed3b43c417f17e57f7e29c39ce9aeea" }
-clap = "2.33"
+clap = "3.0"
 tracing = "0.1"
 tracing-subscriber = { version = "0.3.3", features = ["env-filter"] }
 rayon = "1.5.0"

ruby/generator/Cargo.toml

@@ -7,7 +7,7 @@ edition = "2018"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-clap = "2.33"
+clap = "3.0"
 node-types = { path = "../node-types" }
 tracing = "0.1"
 tracing-subscriber = { version = "0.3.3", features = ["env-filter"] }

ruby/node-types/Cargo.toml

@@ -8,4 +8,4 @@ edition = "2018"
 
 [dependencies]
 serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0"
+serde_json = "1.0"

ruby/ql/lib/codeql/ruby/security/performance/ReDoSUtil.qll

@@ -539,6 +539,55 @@ private class EdgeLabel extends TInputSymbol {
   }
 }
 
+/**
+ * A RegExp term that acts like a plus.
+ * Either it's a RegExpPlus, or it is a range {1,X} where X is >= 30.
+ * 30 has been chosen as a threshold because for exponential blowup 2^30 is enough to get a decent DOS attack.
+ */
+private class EffectivelyPlus extends RegExpTerm {
+  EffectivelyPlus() {
+    this instanceof RegExpPlus
+    or
+    exists(RegExpRange range |
+      range.getLowerBound() = 1 and
+      (range.getUpperBound() >= 30 or not exists(range.getUpperBound()))
+    |
+      this = range
+    )
+  }
+}
+
+/**
+ * A RegExp term that acts like a star.
+ * Either it's a RegExpStar, or it is a range {0,X} where X is >= 30.
+ */
+private class EffectivelyStar extends RegExpTerm {
+  EffectivelyStar() {
+    this instanceof RegExpStar
+    or
+    exists(RegExpRange range |
+      range.getLowerBound() = 0 and
+      (range.getUpperBound() >= 30 or not exists(range.getUpperBound()))
+    |
+      this = range
+    )
+  }
+}
+
+/**
+ * A RegExp term that acts like a question mark.
+ * Either it's a RegExpQuestion, or it is a range {0,1}.
+ */
+private class EffectivelyQuestion extends RegExpTerm {
+  EffectivelyQuestion() {
+    this instanceof RegExpOpt
+    or
+    exists(RegExpRange range | range.getLowerBound() = 0 and range.getUpperBound() = 1 |
+      this = range
+    )
+  }
+}
+
 /**
  * Gets the state before matching `t`.
  */
@@ -559,14 +608,14 @@ State after(RegExpTerm t) {
   or
   exists(RegExpGroup grp | t = grp.getAChild() | result = after(grp))
   or
-  exists(RegExpStar star | t = star.getAChild() | result = before(star))
+  exists(EffectivelyStar star | t = star.getAChild() | result = before(star))
   or
-  exists(RegExpPlus plus | t = plus.getAChild() |
+  exists(EffectivelyPlus plus | t = plus.getAChild() |
     result = before(plus) or
     result = after(plus)
   )
   or
-  exists(RegExpOpt opt | t = opt.getAChild() | result = after(opt))
+  exists(EffectivelyQuestion opt | t = opt.getAChild() | result = after(opt))
   or
   exists(RegExpRoot root | t = root | result = AcceptAnySuffix(root))
 }
@@ -617,15 +666,17 @@ predicate delta(State q1, EdgeLabel lbl, State q2) {
   or
   exists(RegExpGroup grp | lbl = Epsilon() | q1 = before(grp) and q2 = before(grp.getChild(0)))
   or
-  exists(RegExpStar star | lbl = Epsilon() |
+  exists(EffectivelyStar star | lbl = Epsilon() |
     q1 = before(star) and q2 = before(star.getChild(0))
     or
     q1 = before(star) and q2 = after(star)
   )
   or
-  exists(RegExpPlus plus | lbl = Epsilon() | q1 = before(plus) and q2 = before(plus.getChild(0)))
+  exists(EffectivelyPlus plus | lbl = Epsilon() |
+    q1 = before(plus) and q2 = before(plus.getChild(0))
+  )
   or
-  exists(RegExpOpt opt | lbl = Epsilon() |
+  exists(EffectivelyQuestion opt | lbl = Epsilon() |
     q1 = before(opt) and q2 = before(opt.getChild(0))
     or
     q1 = before(opt) and q2 = after(opt)

ruby/ql/test/query-tests/security/cwe-1333-exponential-redos/tst.rb

@@ -362,11 +362,11 @@ bad84 = /^((?:a{0|-)|\w\{\d)+X$/
 bad85 = /^((?:a{0,|-)|\w\{\d,)+X$/
 bad86 = /^((?:a{0,2|-)|\w\{\d,\d)+X$/
 
-# GOOD: 
-good42 = /^((?:a{0,2}|-)|\w\{\d,\d\})+X$/
+# NOT GOOD 
+bad87 = /^((?:a{0,2}|-)|\w\{\d,\d\})+X$/
 
 # NOT GOOD
-bad87 = /^X(\u0061|a)*Y$/
+bad88 = /^X(\u0061|a)*Y$/
 
 # GOOD
 good43 = /^X(\u0061|b)+Y$/