Merge branch 'main' into codeql-ci/atm/release-0.4.2

javascript/ql/experimental/adaptivethreatmodeling/README.md

@@ -1,6 +1,8 @@
-# [Internal only] Adaptive Threat Modeling for JavaScript
+# Adaptive Threat Modeling for JavaScript
 
 This directory contains CodeQL libraries and queries that power adaptive threat modeling for JavaScript.
 All APIs are experimental and may change in the future.
 
-These queries can only be run by internal users; for external users they will return no results.
+Only internal users can run these queries directly. External users can run these queries when performing
+JavaScript analysis on Code Scanning. For more information, see 
+[Code scanning finds more vulnerabilities using machine learning](https://github.blog/2022-02-17-code-scanning-finds-vulnerabilities-using-machine-learning/).

javascript/ql/experimental/adaptivethreatmodeling/lib/experimental/adaptivethreatmodeling/ATMConfig.qll

@@ -1,4 +1,4 @@
-/*
+/**
  * For internal use only.
  *
  * Configures boosting for adaptive threat modeling (ATM).
@@ -50,7 +50,8 @@ abstract class AtmConfig extends string {
     // known sink for the class.
     exists(EndpointCharacteristic characteristic |
       characteristic.getEndpoints(sink) and
-      characteristic.getImplications(this.getASinkEndpointType(), true, 1.0)
+      characteristic
+          .getImplications(this.getASinkEndpointType(), true, characteristic.maximalConfidence())
     )
   }
 

javascript/ql/experimental/adaptivethreatmodeling/lib/experimental/adaptivethreatmodeling/AdaptiveThreatModeling.qll

@@ -1,4 +1,4 @@
-/*
+/**
  * For internal use only.
  *
  * Provides information about the results of boosted queries for use in adaptive threat modeling (ATM).

javascript/ql/experimental/adaptivethreatmodeling/lib/experimental/adaptivethreatmodeling/BaseScoring.qll

@@ -1,4 +1,4 @@
-/*
+/**
  * For internal use only.
  *
  * Provides shared scoring functionality for use in adaptive threat modeling (ATM).

javascript/ql/experimental/adaptivethreatmodeling/lib/experimental/adaptivethreatmodeling/CoreKnowledge.qll

@@ -1,4 +1,4 @@
-/*
+/**
  * For internal use only.
  *
  * Provides predicates that expose the knowledge of models

javascript/ql/experimental/adaptivethreatmodeling/lib/experimental/adaptivethreatmodeling/EndpointCharacteristics.qll

@@ -30,18 +30,39 @@ abstract class EndpointCharacteristic extends string {
   /**
    * This predicate describes what the characteristic tells us about an endpoint.
    *
-   *  Params:
-   *  endpointClass: Class 0 is the negative class. Each positive int corresponds to a single sink type.
-   *  isPositiveIndicator: Does this characteristic indicate this endpoint _is_ a member of the class, or that it
-   *  _isn't_ a member of the class?
-   *  confidence: A number in [0, 1], which tells us how strong an indicator this characteristic is for the endpoint
-   *  belonging / not belonging to the given class.
+   * Params:
+   * endpointClass: The sink type. Each EndpointType has a predicate getEncoding, which specifies the classifier
+   * class for this sink type. Class 0 is the negative class (non-sink). Each positive int corresponds to a single
+   * sink type.
+   * isPositiveIndicator: If true, this characteristic indicates that this endpoint _is_ a member of the class; if
+   * false, it indicates that it _isn't_ a member of the class.
+   * confidence: A float in [0, 1], which tells us how strong an indicator this characteristic is for the endpoint
+   * belonging / not belonging to the given class. A confidence near zero means this characteristic is a very weak
+   * indicator of whether or not the endpoint belongs to the class. A confidence of 1 means that all endpoints with
+   * this characteristic definitively do/don't belong to the class.
    */
   abstract predicate getImplications(
     EndpointType endpointClass, boolean isPositiveIndicator, float confidence
   );
+
+  /** Indicators with confidence at or above this threshold are considered to be high-confidence indicators. */
+  final float getHighConfidenceThreshold() { result = 0.8 }
+
+  // The following are some confidence values that are used in practice by the subclasses. They are defined as named
+  // constants here to make it easier to change them in the future.
+  final float maximalConfidence() { result = 1.0 }
+
+  final float highConfidence() { result = 0.9 }
+
+  final float mediumConfidence() { result = 0.6 }
 }
 
+/*
+ * Characteristics that are indicative of a sink.
+ * NOTE: Initially each sink type has only one characteristic, which is that it's a sink of this type in the standard
+ * JavaScript libraries.
+ */
+
 /**
  * Endpoints identified as "DomBasedXssSink" by the standard JavaScript libraries are XSS sinks with maximal confidence.
  */
@@ -53,7 +74,9 @@ private class DomBasedXssSinkCharacteristic extends EndpointCharacteristic {
   override predicate getImplications(
     EndpointType endpointClass, boolean isPositiveIndicator, float confidence
   ) {
-    endpointClass instanceof XssSinkType and isPositiveIndicator = true and confidence = 1.0
+    endpointClass instanceof XssSinkType and
+    isPositiveIndicator = true and
+    confidence = maximalConfidence()
   }
 }
 
@@ -69,7 +92,9 @@ private class TaintedPathSinkCharacteristic extends EndpointCharacteristic {
   override predicate getImplications(
     EndpointType endpointClass, boolean isPositiveIndicator, float confidence
   ) {
-    endpointClass instanceof TaintedPathSinkType and isPositiveIndicator = true and confidence = 1.0
+    endpointClass instanceof TaintedPathSinkType and
+    isPositiveIndicator = true and
+    confidence = maximalConfidence()
   }
 }
 
@@ -87,7 +112,7 @@ private class SqlInjectionSinkCharacteristic extends EndpointCharacteristic {
   ) {
     endpointClass instanceof SqlInjectionSinkType and
     isPositiveIndicator = true and
-    confidence = 1.0
+    confidence = maximalConfidence()
   }
 }
 
@@ -105,6 +130,315 @@ private class NosqlInjectionSinkCharacteristic extends EndpointCharacteristic {
   ) {
     endpointClass instanceof NosqlInjectionSinkType and
     isPositiveIndicator = true and
-    confidence = 1.0
+    confidence = maximalConfidence()
+  }
+}
+
+/*
+ * Characteristics that are indicative of not being a sink of any type.
+ */
+
+/**
+ * A characteristic that is an indicator of not being a sink of any type, because it's an argument to a function of a
+ * builtin object.
+ */
+abstract private class ArgumentToBuiltinFunctionCharacteristic extends EndpointCharacteristic {
+  bindingset[this]
+  ArgumentToBuiltinFunctionCharacteristic() { any() }
+}
+
+/**
+ * A high-confidence characteristic that indicates that an endpoint is not a sink of any type.
+ */
+abstract private class NotASinkCharacteristic extends EndpointCharacteristic {
+  bindingset[this]
+  NotASinkCharacteristic() { any() }
+
+  override predicate getImplications(
+    EndpointType endpointClass, boolean isPositiveIndicator, float confidence
+  ) {
+    endpointClass instanceof NegativeType and
+    isPositiveIndicator = true and
+    confidence = highConfidence()
+  }
+}
+
+/**
+ * A medium-confidence characteristic that indicates that an endpoint is not a sink of any type.
+ *
+ * TODO: This class is currently not private, because the current extraction logic explicitly avoids including these
+ * endpoints in the training data. We might want to change this in the future.
+ */
+abstract class LikelyNotASinkCharacteristic extends EndpointCharacteristic {
+  bindingset[this]
+  LikelyNotASinkCharacteristic() { any() }
+
+  override predicate getImplications(
+    EndpointType endpointClass, boolean isPositiveIndicator, float confidence
+  ) {
+    endpointClass instanceof NegativeType and
+    isPositiveIndicator = true and
+    confidence = mediumConfidence()
+  }
+}
+
+private class LodashUnderscore extends NotASinkCharacteristic {
+  LodashUnderscore() { this = "LodashUnderscoreArgument" }
+
+  override predicate getEndpoints(DataFlow::Node n) {
+    any(LodashUnderscore::Member m).getACall().getAnArgument() = n
+  }
+}
+
+private class JQueryArgumentCharacteristic extends NotASinkCharacteristic {
+  JQueryArgumentCharacteristic() { this = "JQueryArgument" }
+
+  override predicate getEndpoints(DataFlow::Node n) {
+    any(JQuery::MethodCall m).getAnArgument() = n
+  }
+}
+
+private class ClientRequestCharacteristic extends NotASinkCharacteristic {
+  ClientRequestCharacteristic() { this = "ClientRequest" }
+
+  override predicate getEndpoints(DataFlow::Node n) {
+    exists(ClientRequest r |
+      r.getAnArgument() = n or n = r.getUrl() or n = r.getHost() or n = r.getADataNode()
+    )
+  }
+}
+
+private class PromiseDefinitionCharacteristic extends NotASinkCharacteristic {
+  PromiseDefinitionCharacteristic() { this = "PromiseDefinition" }
+
+  override predicate getEndpoints(DataFlow::Node n) {
+    exists(PromiseDefinition p |
+      n = [p.getResolveParameter(), p.getRejectParameter()].getACall().getAnArgument()
+    )
+  }
+}
+
+private class CryptographicKeyCharacteristic extends NotASinkCharacteristic {
+  CryptographicKeyCharacteristic() { this = "CryptographicKey" }
+
+  override predicate getEndpoints(DataFlow::Node n) { n instanceof CryptographicKey }
+}
+
+private class CryptographicOperationFlowCharacteristic extends NotASinkCharacteristic {
+  CryptographicOperationFlowCharacteristic() { this = "CryptographicOperationFlow" }
+
+  override predicate getEndpoints(DataFlow::Node n) {
+    any(CryptographicOperation op).getInput() = n
+  }
+}
+
+private class LoggerMethodCharacteristic extends NotASinkCharacteristic {
+  LoggerMethodCharacteristic() { this = "LoggerMethod" }
+
+  override predicate getEndpoints(DataFlow::Node n) {
+    exists(DataFlow::CallNode call | n = call.getAnArgument() |
+      call.getCalleeName() = getAStandardLoggerMethodName()
+    )
+  }
+}
+
+private class TimeoutCharacteristic extends NotASinkCharacteristic {
+  TimeoutCharacteristic() { this = "Timeout" }
+
+  override predicate getEndpoints(DataFlow::Node n) {
+    exists(DataFlow::CallNode call | n = call.getAnArgument() |
+      call.getCalleeName() = ["setTimeout", "clearTimeout"]
+    )
+  }
+}
+
+private class ReceiverStorageCharacteristic extends NotASinkCharacteristic {
+  ReceiverStorageCharacteristic() { this = "ReceiverStorage" }
+
+  override predicate getEndpoints(DataFlow::Node n) {
+    exists(DataFlow::CallNode call | n = call.getAnArgument() |
+      call.getReceiver() = DataFlow::globalVarRef(["localStorage", "sessionStorage"])
+    )
+  }
+}
+
+private class StringStartsWithCharacteristic extends NotASinkCharacteristic {
+  StringStartsWithCharacteristic() { this = "StringStartsWith" }
+
+  override predicate getEndpoints(DataFlow::Node n) {
+    exists(DataFlow::CallNode call | n = call.getAnArgument() |
+      call instanceof StringOps::StartsWith
+    )
+  }
+}
+
+private class StringEndsWithCharacteristic extends NotASinkCharacteristic {
+  StringEndsWithCharacteristic() { this = "StringEndsWith" }
+
+  override predicate getEndpoints(DataFlow::Node n) {
+    exists(DataFlow::CallNode call | n = call.getAnArgument() | call instanceof StringOps::EndsWith)
+  }
+}
+
+private class StringRegExpTestCharacteristic extends NotASinkCharacteristic {
+  StringRegExpTestCharacteristic() { this = "StringRegExpTest" }
+
+  override predicate getEndpoints(DataFlow::Node n) {
+    exists(DataFlow::CallNode call | n = call.getAnArgument() |
+      call instanceof StringOps::RegExpTest
+    )
+  }
+}
+
+private class EventRegistrationCharacteristic extends NotASinkCharacteristic {
+  EventRegistrationCharacteristic() { this = "EventRegistration" }
+
+  override predicate getEndpoints(DataFlow::Node n) {
+    exists(DataFlow::CallNode call | n = call.getAnArgument() | call instanceof EventRegistration)
+  }
+}
+
+private class EventDispatchCharacteristic extends NotASinkCharacteristic {
+  EventDispatchCharacteristic() { this = "EventDispatch" }
+
+  override predicate getEndpoints(DataFlow::Node n) {
+    exists(DataFlow::CallNode call | n = call.getAnArgument() | call instanceof EventDispatch)
+  }
+}
+
+private class MembershipCandidateTestCharacteristic extends NotASinkCharacteristic {
+  MembershipCandidateTestCharacteristic() { this = "MembershipCandidateTest" }
+
+  override predicate getEndpoints(DataFlow::Node n) {
+    exists(DataFlow::CallNode call | n = call.getAnArgument() |
+      call = any(MembershipCandidate c).getTest()
+    )
+  }
+}
+
+private class FileSystemAccessCharacteristic extends NotASinkCharacteristic {
+  FileSystemAccessCharacteristic() { this = "FileSystemAccess" }
+
+  override predicate getEndpoints(DataFlow::Node n) {
+    exists(DataFlow::CallNode call | n = call.getAnArgument() | call instanceof FileSystemAccess)
+  }
+}
+
+private class DatabaseAccessCharacteristic extends NotASinkCharacteristic {
+  DatabaseAccessCharacteristic() { this = "DatabaseAccess" }
+
+  override predicate getEndpoints(DataFlow::Node n) {
+    // TODO database accesses are less well defined than database query sinks, so this may cover unmodeled sinks on
+    // existing database models
+    exists(DataFlow::CallNode call | n = call.getAnArgument() |
+      [
+        call, call.getAMethodCall()
+      /* command pattern where the query is built, and then exec'ed later */ ] instanceof
+        DatabaseAccess
+    )
+  }
+}
+
+private class DomCharacteristic extends NotASinkCharacteristic {
+  DomCharacteristic() { this = "DOM" }
+
+  override predicate getEndpoints(DataFlow::Node n) {
+    exists(DataFlow::CallNode call | n = call.getAnArgument() | call = DOM::domValueRef())
+  }
+}
+
+private class NextFunctionCallCharacteristic extends NotASinkCharacteristic {
+  NextFunctionCallCharacteristic() { this = "NextFunctionCall" }
+
+  override predicate getEndpoints(DataFlow::Node n) {
+    exists(DataFlow::CallNode call | n = call.getAnArgument() |
+      call.getCalleeName() = "next" and
+      exists(DataFlow::FunctionNode f | call = f.getLastParameter().getACall())
+    )
+  }
+}
+
+private class DojoRequireCharacteristic extends NotASinkCharacteristic {
+  DojoRequireCharacteristic() { this = "DojoRequire" }
+
+  override predicate getEndpoints(DataFlow::Node n) {
+    exists(DataFlow::CallNode call | n = call.getAnArgument() |
+      call = DataFlow::globalVarRef("dojo").getAPropertyRead("require").getACall()
+    )
+  }
+}
+
+private class Base64ManipulationCharacteristic extends NotASinkCharacteristic {
+  Base64ManipulationCharacteristic() { this = "Base64Manipulation" }
+
+  override predicate getEndpoints(DataFlow::Node n) {
+    exists(Base64::Decode d | n = d.getInput()) or
+    exists(Base64::Encode d | n = d.getInput())
+  }
+}
+
+private class ArgumentToArrayCharacteristic extends ArgumentToBuiltinFunctionCharacteristic,
+  LikelyNotASinkCharacteristic {
+  ArgumentToArrayCharacteristic() { this = "ArgumentToArray" }
+
+  override predicate getEndpoints(DataFlow::Node n) {
+    exists(DataFlow::SourceNode builtin, DataFlow::SourceNode receiver, DataFlow::InvokeNode invk |
+      builtin instanceof DataFlow::ArrayCreationNode
+    |
+      receiver = [builtin.getAnInvocation(), builtin] and
+      invk = [receiver, receiver.getAPropertyRead()].getAnInvocation() and
+      invk.getAnArgument() = n
+    )
+  }
+}
+
+private class ArgumentToBuiltinGlobalVarRefCharacteristic extends ArgumentToBuiltinFunctionCharacteristic,
+  LikelyNotASinkCharacteristic {
+  ArgumentToBuiltinGlobalVarRefCharacteristic() { this = "ArgumentToBuiltinGlobalVarRef" }
+
+  override predicate getEndpoints(DataFlow::Node n) {
+    exists(DataFlow::SourceNode builtin, DataFlow::SourceNode receiver, DataFlow::InvokeNode invk |
+      builtin =
+        DataFlow::globalVarRef([
+            "Map", "Set", "WeakMap", "WeakSet", "Number", "Object", "String", "Array", "Error",
+            "Math", "Boolean"
+          ])
+    |
+      receiver = [builtin.getAnInvocation(), builtin] and
+      invk = [receiver, receiver.getAPropertyRead()].getAnInvocation() and
+      invk.getAnArgument() = n
+    )
+  }
+}
+
+private class ConstantReceiverCharacteristic extends ArgumentToBuiltinFunctionCharacteristic,
+  NotASinkCharacteristic {
+  ConstantReceiverCharacteristic() { this = "ConstantReceiver" }
+
+  override predicate getEndpoints(DataFlow::Node n) {
+    exists(Expr primitive, MethodCallExpr c |
+      primitive instanceof ConstantString or
+      primitive instanceof NumberLiteral or
+      primitive instanceof BooleanLiteral
+    |
+      c.calls(primitive, _) and
+      c.getAnArgument() = n.asExpr()
+    )
+  }
+}
+
+private class BuiltinCallNameCharacteristic extends ArgumentToBuiltinFunctionCharacteristic,
+  NotASinkCharacteristic {
+  BuiltinCallNameCharacteristic() { this = "BuiltinCallName" }
+
+  override predicate getEndpoints(DataFlow::Node n) {
+    exists(DataFlow::CallNode call |
+      call.getAnArgument() = n and
+      call.getCalleeName() =
+        [
+          "indexOf", "hasOwnProperty", "substring", "isDecimal", "decode", "encode", "keys",
+          "shift", "values", "forEach", "toString", "slice", "splice", "push", "isArray", "sort"
+        ]
+    )
   }
 }

javascript/ql/experimental/adaptivethreatmodeling/lib/experimental/adaptivethreatmodeling/EndpointFeatures.qll

@@ -1,4 +1,4 @@
-/*
+/**
  * For internal use only.
  *
  * Extracts data about the database for use in adaptive threat modeling (ATM).

javascript/ql/experimental/adaptivethreatmodeling/lib/experimental/adaptivethreatmodeling/EndpointScoring.qll

@@ -1,4 +1,4 @@
-/*
+/**
  * For internal use only.
  *
  * Provides an implementation of scoring alerts for use in adaptive threat modeling (ATM).

javascript/ql/experimental/adaptivethreatmodeling/lib/experimental/adaptivethreatmodeling/EndpointTypes.qll

@@ -16,6 +16,11 @@ newtype TEndpointType =
 abstract class EndpointType extends TEndpointType {
   abstract string getDescription();
 
+  /**
+   * Gets the integer representation of this endpoint type. This integer representation specifies the class number
+   * used by the endpoint scoring model (the classifier) to represent this endpoint type. Class 0 is the negative
+   * class (non-sink). Each positive int corresponds to a single sink type.
+   */
   abstract int getEncoding();
 
   string toString() { result = getDescription() }

javascript/ql/experimental/adaptivethreatmodeling/lib/experimental/adaptivethreatmodeling/FunctionBodyFeatures.qll

@@ -1,4 +1,4 @@
-/*
+/**
  * FunctionBodyFeatures.qll
  *
  * Contains logic relating to the `enclosingFunctionBody` and `enclosingFunctionName` features.

javascript/ql/experimental/adaptivethreatmodeling/lib/qlpack.yml

@@ -1,4 +1,5 @@
 name: codeql/javascript-experimental-atm-lib
+description: CodeQL libraries for the experimental ML-powered queries
 version: 0.4.3
 extractor: javascript
 library: true

javascript/ql/experimental/adaptivethreatmodeling/model/qlpack.yml

@@ -1,4 +1,5 @@
 name: codeql/javascript-experimental-atm-model
+description: Machine learning model supporting the experimental ML-powered queries
 version: 0.3.1
 groups:
     - javascript

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/DebugResultInclusion.ql

@@ -11,7 +11,7 @@
 
 import javascript
 import experimental.adaptivethreatmodeling.ATMConfig
-import extraction.ExtractEndpointData
+import extraction.ExtractEndpointDataTraining
 
 string getAReasonSinkExcluded(DataFlow::Node sinkCandidate, Query query) {
   query instanceof NosqlInjectionQuery and
@@ -33,7 +33,7 @@ string getDescriptionForAlertCandidate(
 ) {
   result = "excluded[reason=" + getAReasonSinkExcluded(sinkCandidate, query) + "]"
   or
-  getAtmCfg(query).isKnownSink(sinkCandidate) and
+  getDataFlowCfg(query).(AtmConfig).isKnownSink(sinkCandidate) and
   result = "excluded[reason=known-sink]"
   or
   not exists(getAReasonSinkExcluded(sinkCandidate, query)) and

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/counting/CountAlertsAndSinks.qll

@@ -1,4 +1,4 @@
-/*
+/**
  * For internal use only.
  *
  *

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/extraction/Exclusions.qll

@@ -1,4 +1,4 @@
-/*
+/**
  * For internal use only.
  *
  * Defines files that should be excluded from the evaluation of ML models.

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/extraction/ExtractEndpointData.ql

@@ -1,11 +0,0 @@
-/*
- * For internal use only.
- *
- * Extracts training and evaluation data we can use to train ML models for ML-powered queries.
- */
-
-import ExtractEndpointData as ExtractEndpointData
-
-query predicate endpoints = ExtractEndpointData::endpoints/5;
-
-query predicate tokenFeatures = ExtractEndpointData::tokenFeatures/3;

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/extraction/ExtractEndpointData.qll

@@ -1,215 +0,0 @@
-/*
- * For internal use only.
- *
- * Library code for training and evaluation data we can use to train ML models for ML-powered
- * queries.
- */
-
-import javascript
-import Exclusions as Exclusions
-import evaluation.EndToEndEvaluation as EndToEndEvaluation
-import experimental.adaptivethreatmodeling.ATMConfig
-import experimental.adaptivethreatmodeling.CoreKnowledge as CoreKnowledge
-import experimental.adaptivethreatmodeling.EndpointFeatures as EndpointFeatures
-import experimental.adaptivethreatmodeling.EndpointScoring as EndpointScoring
-import experimental.adaptivethreatmodeling.EndpointTypes
-import experimental.adaptivethreatmodeling.FilteringReasons
-import experimental.adaptivethreatmodeling.NosqlInjectionATM as NosqlInjectionAtm
-
-/** DEPRECATED: Alias for NosqlInjectionAtm */
-deprecated module NosqlInjectionATM = NosqlInjectionAtm;
-
-import experimental.adaptivethreatmodeling.SqlInjectionATM as SqlInjectionAtm
-
-/** DEPRECATED: Alias for SqlInjectionAtm */
-deprecated module SqlInjectionATM = SqlInjectionAtm;
-
-import experimental.adaptivethreatmodeling.TaintedPathATM as TaintedPathAtm
-
-/** DEPRECATED: Alias for TaintedPathAtm */
-deprecated module TaintedPathATM = TaintedPathAtm;
-
-import experimental.adaptivethreatmodeling.XssATM as XssAtm
-
-/** DEPRECATED: Alias for XssAtm */
-deprecated module XssATM = XssAtm;
-
-import Labels
-import NoFeaturizationRestrictionsConfig
-import Queries
-
-/** Gets the ATM configuration object for the specified query. */
-AtmConfig getAtmCfg(Query query) {
-  query instanceof NosqlInjectionQuery and
-  result instanceof NosqlInjectionAtm::NosqlInjectionAtmConfig
-  or
-  query instanceof SqlInjectionQuery and result instanceof SqlInjectionAtm::SqlInjectionAtmConfig
-  or
-  query instanceof TaintedPathQuery and result instanceof TaintedPathAtm::TaintedPathAtmConfig
-  or
-  query instanceof XssQuery and result instanceof XssAtm::DomBasedXssAtmConfig
-}
-
-/** DEPRECATED: Alias for getAtmCfg */
-deprecated ATMConfig getATMCfg(Query query) { result = getAtmCfg(query) }
-
-/** Gets the ATM data flow configuration for the specified query. */
-DataFlow::Configuration getDataFlowCfg(Query query) {
-  query instanceof NosqlInjectionQuery and result instanceof NosqlInjectionAtm::Configuration
-  or
-  query instanceof SqlInjectionQuery and result instanceof SqlInjectionAtm::Configuration
-  or
-  query instanceof TaintedPathQuery and result instanceof TaintedPathAtm::Configuration
-  or
-  query instanceof XssQuery and result instanceof XssAtm::Configuration
-}
-
-/** Gets a known sink for the specified query. */
-private DataFlow::Node getASink(Query query) {
-  getAtmCfg(query).isKnownSink(result) and
-  // Only consider the source code for the project being analyzed.
-  exists(result.getFile().getRelativePath())
-}
-
-/** Gets a data flow node that is known not to be a sink for the specified query. */
-private DataFlow::Node getANotASink(NotASinkReason reason) {
-  CoreKnowledge::isOtherModeledArgument(result, reason) and
-  // Some endpoints can be assigned both a `NotASinkReason` and a `LikelyNotASinkReason`. We
-  // consider these endpoints to be `LikelyNotASink`, therefore this line excludes them from the
-  // definition of `NotASink`.
-  not CoreKnowledge::isOtherModeledArgument(result, any(LikelyNotASinkReason t)) and
-  not result = getASink(_) and
-  // Only consider the source code for the project being analyzed.
-  exists(result.getFile().getRelativePath())
-}
-
-/**
- * Gets a data flow node whose label is unknown for the specified query.
- *
- * In other words, this is an endpoint that is not `Sink`, `NotASink`, or `LikelyNotASink` for the
- * specified query.
- */
-private DataFlow::Node getAnUnknown(Query query) {
-  getAtmCfg(query).isEffectiveSink(result) and
-  // Effective sinks should exclude sinks but this is a defensive requirement
-  not result = getASink(query) and
-  // Effective sinks should exclude NotASink but for some queries (e.g. Xss) this is currently not always the case and
-  // so this is a defensive requirement
-  not result = getANotASink(_) and
-  // Only consider the source code for the project being analyzed.
-  exists(result.getFile().getRelativePath())
-}
-
-/** Gets the query-specific sink label for the given endpoint, if such a label exists. */
-private EndpointLabel getSinkLabelForEndpoint(DataFlow::Node endpoint, Query query) {
-  endpoint = getASink(query) and result instanceof SinkLabel
-  or
-  endpoint = getANotASink(_) and result instanceof NotASinkLabel
-  or
-  endpoint = getAnUnknown(query) and result instanceof UnknownLabel
-}
-
-/** Gets an endpoint that should be extracted. */
-DataFlow::Node getAnEndpoint(Query query) { exists(getSinkLabelForEndpoint(result, query)) }
-
-/**
- * Endpoints and associated metadata.
- *
- * Note that we draw a distinction between _features_, that are provided to the model at training
- * and query time, and _metadata_, that is only provided to the model at training time.
- *
- * Internal: See the design document for
- * [extensible extraction queries](https://docs.google.com/document/d/1g3ci2Nf1hGMG6ZUP0Y4PqCy_8elcoC_dhBvgTxdAWpg)
- * for technical information about the design of this predicate.
- */
-predicate endpoints(
-  DataFlow::Node endpoint, string queryName, string key, string value, string valueType
-) {
-  exists(Query query |
-    // Only provide metadata for labelled endpoints, since we do not extract all endpoints.
-    endpoint = getAnEndpoint(query) and
-    queryName = query.getName() and
-    (
-      // Holds if there is a taint flow path from a known source to the endpoint
-      key = "hasFlowFromSource" and
-      (
-        if FlowFromSource::hasFlowFromSource(endpoint, query)
-        then value = "true"
-        else value = "false"
-      ) and
-      valueType = "boolean"
-      or
-      // Constant expressions always evaluate to a constant primitive value. Therefore they can't ever
-      // appear in an alert, making them less interesting training examples.
-      key = "isConstantExpression" and
-      (if endpoint.asExpr() instanceof ConstantExpr then value = "true" else value = "false") and
-      valueType = "boolean"
-      or
-      // Holds if alerts involving the endpoint are excluded from the end-to-end evaluation.
-      key = "isExcludedFromEndToEndEvaluation" and
-      (if Exclusions::isFileExcluded(endpoint.getFile()) then value = "true" else value = "false") and
-      valueType = "boolean"
-      or
-      // The label for this query, considering the endpoint as a sink.
-      key = "sinkLabel" and
-      value = getSinkLabelForEndpoint(endpoint, query).getEncoding() and
-      valueType = "string"
-      or
-      // The reason, or reasons, why the endpoint was labeled NotASink for this query.
-      key = "notASinkReason" and
-      exists(FilteringReason reason |
-        endpoint = getANotASink(reason) and
-        value = reason.getDescription()
-      ) and
-      valueType = "string"
-    )
-  )
-}
-
-/**
- * `EndpointFeatures::tokenFeatures` has no results when `featureName` is absent for the endpoint
- * `endpoint`. To preserve compatibility with the data pipeline, this relation will instead set
- * `featureValue` to the empty string in this case.
- */
-predicate tokenFeatures(DataFlow::Node endpoint, string featureName, string featureValue) {
-  endpoints(endpoint, _, _, _, _) and
-  (
-    EndpointFeatures::tokenFeatures(endpoint, featureName, featureValue)
-    or
-    // Performance note: this creates a Cartesian product between `endpoint` and `featureName`.
-    featureName = EndpointFeatures::getASupportedFeatureName() and
-    not exists(string value | EndpointFeatures::tokenFeatures(endpoint, featureName, value)) and
-    featureValue = ""
-  )
-}
-
-module FlowFromSource {
-  predicate hasFlowFromSource(DataFlow::Node endpoint, Query q) {
-    exists(Configuration cfg | cfg.getQuery() = q | cfg.hasFlow(_, endpoint))
-  }
-
-  /**
-   * A data flow configuration that replicates the data flow configuration for a specific query, but
-   * replaces the set of sinks with the set of endpoints we're extracting.
-   *
-   * We use this to find out when there is flow to a particular endpoint from a known source.
-   *
-   * This configuration behaves in a very similar way to the `ForwardExploringConfiguration` class
-   * from the CodeQL standard libraries for JavaScript.
-   */
-  private class Configuration extends DataFlow::Configuration {
-    Query q;
-
-    Configuration() { this = getDataFlowCfg(q) }
-
-    Query getQuery() { result = q }
-
-    /** Holds if `sink` is an endpoint we're extracting. */
-    override predicate isSink(DataFlow::Node sink) { sink = getAnEndpoint(q) }
-
-    /** Holds if `sink` is an endpoint we're extracting. */
-    override predicate isSink(DataFlow::Node sink, DataFlow::FlowLabel lbl) {
-      sink = getAnEndpoint(q) and exists(lbl)
-    }
-  }
-}

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/extraction/ExtractEndpointDataTraining.ql

@@ -4,23 +4,8 @@
  * Extracts training data we can use to train ML models for ML-powered queries.
  */
 
-import javascript
-import ExtractEndpointData as ExtractEndpointData
+private import ExtractEndpointDataTraining as ExtractEndpointDataTraining
 
-query predicate endpoints(
-  DataFlow::Node endpoint, string queryName, string key, string value, string valueType
-) {
-  ExtractEndpointData::endpoints(endpoint, queryName, key, value, valueType) and
-  // only select endpoints that are either Sink or NotASink
-  ExtractEndpointData::endpoints(endpoint, queryName, "sinkLabel", ["Sink", "NotASink"], "string") and
-  // do not select endpoints filtered out by end-to-end evaluation
-  ExtractEndpointData::endpoints(endpoint, queryName, "isExcludedFromEndToEndEvaluation", "false",
-    "boolean") and
-  // only select endpoints that can be part of a tainted flow
-  ExtractEndpointData::endpoints(endpoint, queryName, "isConstantExpression", "false", "boolean")
-}
+query predicate endpoints = ExtractEndpointDataTraining::reformattedTrainingEndpoints/5;
 
-query predicate tokenFeatures(DataFlow::Node endpoint, string featureName, string featureValue) {
-  endpoints(endpoint, _, _, _, _) and
-  ExtractEndpointData::tokenFeatures(endpoint, featureName, featureValue)
-}
+query predicate tokenFeatures = ExtractEndpointDataTraining::tokenFeatures/3;

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/extraction/ExtractEndpointDataTraining.qll

@@ -0,0 +1,238 @@
+/**
+ * For internal use only.
+ *
+ * Extracts training data we can use to train ML models for ML-powered queries.
+ */
+
+import javascript
+import experimental.adaptivethreatmodeling.EndpointCharacteristics
+import experimental.adaptivethreatmodeling.EndpointFeatures as EndpointFeatures
+import NoFeaturizationRestrictionsConfig
+private import Exclusions as Exclusions
+import Queries
+import experimental.adaptivethreatmodeling.NosqlInjectionATM as NosqlInjectionAtm
+import experimental.adaptivethreatmodeling.SqlInjectionATM as SqlInjectionAtm
+import experimental.adaptivethreatmodeling.TaintedPathATM as TaintedPathAtm
+import experimental.adaptivethreatmodeling.XssATM as XssAtm
+
+/**
+ * Gets the set of featureName-featureValue pairs for each endpoint in the training set.
+ *
+ * `EndpointFeatures::tokenFeatures` has no results when `featureName` is absent for the endpoint
+ * `endpoint`. To preserve compatibility with the data pipeline, this relation will instead set
+ * `featureValue` to the empty string in this case.
+ */
+predicate tokenFeatures(DataFlow::Node endpoint, string featureName, string featureValue) {
+  trainingEndpoints(endpoint, _, _) and
+  (
+    EndpointFeatures::tokenFeatures(endpoint, featureName, featureValue)
+    or
+    // Performance note: this creates a Cartesian product between `endpoint` and `featureName`.
+    featureName = EndpointFeatures::getASupportedFeatureName() and
+    not exists(string value | EndpointFeatures::tokenFeatures(endpoint, featureName, value)) and
+    featureValue = ""
+  )
+}
+
+/**
+ * Holds if the given endpoint should be included in the training set as a sample belonging to endpointClass, and has
+ * the given characteristic. This query uses the endpoint characteristics to select and label endpoints for the training
+ * set, and provides a list of characteristics for each endpoint in the training set, which is used in the modeling
+ * code.
+ *
+ * Params:
+ * endpoint: The endpoint to include / exclude.
+ * endpointClass: The sink type. See the documentation of EndpointType.getEncoding for details about the relationship
+ * between an EndpointType and a class in the classifier.
+ * characteristic: Provides the list of characteristics that apply to the endpoint, which the modeling code currently
+ * uses for type balancing.
+ *
+ * Note: This predicate will produce multiple tuples for endpoints that have multiple characteristics, which we must
+ * then group together into a list of characteristics.
+ */
+query predicate trainingEndpoints(
+  DataFlow::Node endpoint, EndpointType endpointClass, EndpointCharacteristic characteristic
+) {
+  characteristic.getEndpoints(endpoint) and
+  // Only consider the source code for the project being analyzed.
+  exists(endpoint.getFile().getRelativePath()) and
+  // Only select endpoints that can be part of a tainted flow: Constant expressions always evaluate to a constant
+  // primitive value. Therefore they can't ever appear in an alert, making them less interesting training examples.
+  // TODO: Experiment with removing this requirement.
+  not endpoint.asExpr() instanceof ConstantExpr and
+  // Do not select endpoints filtered out by end-to-end evaluation.
+  // TODO: Experiment with removing this requirement.
+  not Exclusions::isFileExcluded(endpoint.getFile()) and
+  // Filter out negative examples that also have a LikelyNotASinkReason, because this is currently done here
+  // https://github.com/github/codeql/blob/387e57546bf7352f7c1cfe781daa1a3799b7063e/javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/extraction/ExtractEndpointData.qll#L77
+  // TODO: Experiment with removing this requirement.
+  not (
+    endpointClass instanceof NegativeType and
+    exists(EndpointCharacteristic c |
+      c.getEndpoints(endpoint) and
+      c instanceof LikelyNotASinkCharacteristic
+    )
+  ) and
+  (
+    // If the list of characteristics includes positive indicators with high confidence for this class, select this as a
+    // training sample belonging to the class.
+    exists(EndpointCharacteristic characteristic2, float confidence |
+      characteristic2.getEndpoints(endpoint) and
+      characteristic2.getImplications(endpointClass, true, confidence) and
+      confidence >= characteristic2.getHighConfidenceThreshold()
+    ) and
+    (
+      // Temporarily limit this only to positive classes. For negative classes, additionally select only endpoints that
+      // have no high confidence indicators that they are sinks, because this is what was previously done.
+      // TODO: Experiment with removing this requirement, and instead ensuring that an endpoint never has both a high
+      // confidence indicator that it _is_ a sink and a high confidence indicator that it is _not_ a sink.
+      not endpointClass instanceof NegativeType
+      or
+      not exists(EndpointCharacteristic characteristic3, float confidence3, EndpointType posClass |
+        characteristic3.getEndpoints(endpoint) and
+        characteristic3.getImplications(posClass, true, confidence3) and
+        confidence3 >= characteristic3.getHighConfidenceThreshold() and
+        not posClass instanceof NegativeType
+      )
+    )
+    or
+    // If the list of characteristics includes negative indicators with high confidence for all classes other than 0,
+    // select this as a training sample of class 0 (this means we had query-specific characteristics to decide this
+    // endpoint isn't a sink for each of our sink types).
+    endpointClass instanceof NegativeType and
+    forall(EndpointType otherClass | not otherClass instanceof NegativeType |
+      exists(EndpointCharacteristic characteristic2, float confidence |
+        characteristic2.getEndpoints(endpoint) and
+        characteristic2.getImplications(otherClass, false, confidence) and
+        confidence >= characteristic2.getHighConfidenceThreshold()
+      )
+    )
+  )
+}
+
+/**
+ * Temporary:
+ * Reformat the training data that was extracted with the new logic to match the format produced by the old predicate.
+ * This is the format expected by the endpoint pipeline.
+ */
+query predicate reformattedTrainingEndpoints(
+  DataFlow::Node endpoint, string queryName, string key, string value, string valueType
+) {
+  trainingEndpoints(endpoint, _, _) and
+  exists(Query query |
+    queryName = query.getName() and
+    // For sinks, only list that sink type, but for non-sinks, list all sink types.
+    (
+      exists(EndpointType endpointClass |
+        endpointClass.getDescription().matches(queryName + "%") and
+        not endpointClass instanceof NegativeType and
+        trainingEndpoints(endpoint, endpointClass, _)
+      )
+      or
+      exists(EndpointType endpointClass |
+        endpointClass instanceof NegativeType and
+        trainingEndpoints(endpoint, endpointClass, _)
+      )
+    ) and
+    (
+      // NOTE: We don't use hasFlowFromSource in training, so we could just hardcode it to be false.
+      key = "hasFlowFromSource" and
+      (
+        if FlowFromSource::hasFlowFromSource(endpoint, query)
+        then value = "true"
+        else value = "false"
+      ) and
+      valueType = "boolean"
+      or
+      // Constant expressions always evaluate to a constant primitive value. Therefore they can't ever
+      // appear in an alert, making them less interesting training examples.
+      key = "isConstantExpression" and
+      (if endpoint.asExpr() instanceof ConstantExpr then value = "true" else value = "false") and
+      valueType = "boolean"
+      or
+      // Holds if alerts involving the endpoint are excluded from the end-to-end evaluation.
+      key = "isExcludedFromEndToEndEvaluation" and
+      (if Exclusions::isFileExcluded(endpoint.getFile()) then value = "true" else value = "false") and
+      valueType = "boolean"
+      or
+      // The label for this query, considering the endpoint as a sink.
+      key = "sinkLabel" and
+      valueType = "string" and
+      value = "Sink" and
+      exists(EndpointType endpointClass |
+        endpointClass.getDescription().matches(queryName + "%") and
+        not endpointClass instanceof NegativeType and
+        trainingEndpoints(endpoint, endpointClass, _)
+      )
+      or
+      key = "sinkLabel" and
+      valueType = "string" and
+      value = "NotASink" and
+      exists(EndpointType endpointClass |
+        endpointClass instanceof NegativeType and
+        trainingEndpoints(endpoint, endpointClass, _)
+      )
+      or
+      // The reason, or reasons, why the endpoint was labeled NotASink for this query, only for negative examples.
+      key = "notASinkReason" and
+      exists(EndpointCharacteristic characteristic, EndpointType endpointClass |
+        characteristic.getEndpoints(endpoint) and
+        characteristic.getImplications(endpointClass, true, _) and
+        endpointClass instanceof NegativeType and
+        value = characteristic
+      ) and
+      // Don't include a notASinkReason for endpoints that are also known sinks.
+      not exists(EndpointCharacteristic characteristic3, float confidence3, EndpointType posClass |
+        characteristic3.getEndpoints(endpoint) and
+        characteristic3.getImplications(posClass, true, confidence3) and
+        confidence3 >= characteristic3.getHighConfidenceThreshold() and
+        not posClass instanceof NegativeType
+      ) and
+      valueType = "string"
+    )
+  )
+}
+
+/**
+ * Gets the ATM data flow configuration for the specified query.
+ * TODO: Delete this once we are no longer surfacing `hasFlowFromSource`.
+ */
+DataFlow::Configuration getDataFlowCfg(Query query) {
+  query instanceof NosqlInjectionQuery and result instanceof NosqlInjectionAtm::Configuration
+  or
+  query instanceof SqlInjectionQuery and result instanceof SqlInjectionAtm::Configuration
+  or
+  query instanceof TaintedPathQuery and result instanceof TaintedPathAtm::Configuration
+  or
+  query instanceof XssQuery and result instanceof XssAtm::Configuration
+}
+
+// TODO: Delete this once we are no longer surfacing `hasFlowFromSource`.
+private module FlowFromSource {
+  predicate hasFlowFromSource(DataFlow::Node endpoint, Query q) {
+    exists(Configuration cfg | cfg.getQuery() = q | cfg.hasFlow(_, endpoint))
+  }
+
+  /**
+   * A data flow configuration that replicates the data flow configuration for a specific query, but
+   * replaces the set of sinks with the set of endpoints we're extracting.
+   *
+   * We use this to find out when there is flow to a particular endpoint from a known source.
+   *
+   * This configuration behaves in a very similar way to the `ForwardExploringConfiguration` class
+   * from the CodeQL standard libraries for JavaScript.
+   */
+  private class Configuration extends DataFlow::Configuration {
+    Query q;
+
+    Configuration() { this = getDataFlowCfg(q) }
+
+    Query getQuery() { result = q }
+
+    /** Holds if `sink` is an endpoint we're extracting. */
+    override predicate isSink(DataFlow::Node sink) { any() }
+
+    /** Holds if `sink` is an endpoint we're extracting. */
+    override predicate isSink(DataFlow::Node sink, DataFlow::FlowLabel lbl) { exists(lbl) }
+  }
+}

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/qlpack.yml

@@ -1,4 +1,5 @@
 name: codeql/javascript-experimental-atm-model-building
+description: CodeQL libraries for building machine learning models for the experimental ML-powered queries
 extractor: javascript
 library: false
 groups:

javascript/ql/experimental/adaptivethreatmodeling/src/qlpack.yml

@@ -1,4 +1,5 @@
 name: codeql/javascript-experimental-atm-queries
+description: Experimental ML-powered queries for JavaScript
 language: javascript
 version: 0.4.3
 suites: codeql-suites

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/ExtractEndpointData.qlref

@@ -1 +0,0 @@
-extraction/ExtractEndpointData.ql

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_unit_tests/ExtractEndpointData.qlref

@@ -1 +0,0 @@
-extraction/ExtractEndpointData.ql

javascript/ql/lib/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.3.4
+
+### Major Analysis Improvements
+
+* Added support for TypeScript 4.9.
+
 ## 0.3.3
 
 No user-facing changes.

javascript/ql/lib/change-notes/2022-11-17-deleted-deps.md

@@ -0,0 +1,6 @@
+---
+category: minorAnalysis
+---
+* Deleted the deprecated `Instance` class from the `Vue` module.
+* Deleted the deprecated `VHtmlSourceWrite` class from `DomBasedXssQuery.qll`.
+* Deleted all the deprecated `[QueryName].qll` files from the `javascript/ql/lib/semmle/javascript/security/dataflow` folder, use the corresponding `[QueryName]Query.qll` files instead.

javascript/ql/lib/change-notes/released/0.3.4.md

@@ -0,0 +1,5 @@
+## 0.3.4
+
+### Major Analysis Improvements
+
+* Added support for TypeScript 4.9.

javascript/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.3.3
+lastReleaseVersion: 0.3.4

javascript/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/javascript-all
-version: 0.3.4-dev
+version: 0.3.5-dev
 groups: javascript
 dbscheme: semmlecode.javascript.dbscheme
 extractor: javascript

javascript/ql/lib/semmle/javascript/GlobalAccessPaths.qll

@@ -233,6 +233,8 @@ module AccessPath {
       baseName = fromReference(write.getBase(), root)
       or
       baseName = fromRhs(write.getBase(), root)
+      or
+      baseName = fromRhs(GetLaterAccess::getLaterBaseAccess(write), root)
     )
     or
     exists(GlobalVariable var |
@@ -266,6 +268,100 @@ module AccessPath {
     )
   }
 
+  /** A module for computing an access to a variable that happens after a property has been written onto it */
+  private module GetLaterAccess {
+    /**
+     * Gets an access to a variable that is written to in `write`, where the access is after the write.
+     *
+     * This allows `fromRhs` to compute an access path for e.g. the below example:
+     * ```JavaScript
+     * function foo(x) {
+     *  var obj = {
+     *    bar: x // `x` has the access path "foo.bar" starting from the root `this`.
+     *  };
+     *  this.foo = obj;
+     * }
+     * ```
+     */
+    pragma[noopt]
+    DataFlow::Node getLaterBaseAccess(DataFlow::PropWrite write) {
+      exists(
+        ControlFlowNode writeNode, BindingPattern access, VarRef otherAccess, Variable variable,
+        StmtContainer container
+      |
+        access = getBaseVar(write) and
+        writeNode = write.getWriteNode() and
+        access = getAnAccessInContainer(variable, container, true) and
+        variable = getARelevantVariable() and // manual magic
+        otherAccess = getAnAccessInContainer(variable, container, false) and
+        access != otherAccess and
+        result.asExpr() = otherAccess
+      |
+        exists(BasicBlock bb, int i, int j |
+          bb.getNode(i) = writeNode and
+          bb.getNode(j) = otherAccess and
+          i < j
+        )
+        or
+        otherAccess.getBasicBlock() = getASuccessorBBThatReadsVar(write) // more manual magic - outlined into a helper predicate.
+      )
+    }
+
+    /** Gets a variable ref that `write` writes a property to. */
+    VarRef getBaseVar(DataFlow::PropWrite write) {
+      result = write.getBase().asExpr()
+      or
+      exists(Assignment assign |
+        write.getBase().asExpr() = assign.getRhs() and
+        result = assign.getLhs()
+      )
+      or
+      exists(VariableDeclarator decl |
+        write.getBase().asExpr() = decl.getInit() and
+        result = decl.getBindingPattern()
+      )
+    }
+
+    /** Gets an access to `var` inside `container` where `usedInWrite` indicates whether the access is the base of a property write. */
+    private VarRef getAnAccessInContainer(Variable var, StmtContainer container, boolean usedInWrite) {
+      result.getVariable() = var and
+      result.getContainer() = container and
+      var.isLocal() and
+      if result = getBaseVar(_) then usedInWrite = true else usedInWrite = false
+    }
+
+    /** Gets a variable that is relevant for the computations in the `GetLaterAccess` module. */
+    private Variable getARelevantVariable() {
+      // The variable might be used where `getLaterBaseAccess()` is called.
+      exists(DataFlow::Node node |
+        exists(fromRhs(node, _)) and
+        node.asExpr().(VarAccess).getVariable() = result
+      ) and
+      // There is a write that writes to the variable.
+      getBaseVar(_).getVariable() = result and
+      // It's local.
+      result.isLocal() and // we skip global variables, because that turns messy quick.
+      // There is both a "write" and "read" in the same container of the variable.
+      exists(StmtContainer container |
+        exists(getAnAccessInContainer(result, container, true)) and // a "write", an access to the variable that is the base of a property reference.
+        exists(getAnAccessInContainer(result, container, false)) // a "read", an access to the variable that is not the base of a property reference.
+      )
+    }
+
+    /** Gets a basic-block that has a read of the variable that is written to by `write`, where the basicblock occurs after `start`. */
+    private ReachableBasicBlock getASuccessorBBThatReadsVar(DataFlow::PropWrite write) {
+      exists(VarAccess baseExpr, Variable var, ControlFlowNode writeNode |
+        baseExpr = getBaseVar(write) and
+        var = baseExpr.getVariable() and
+        var = getARelevantVariable() and
+        writeNode = write.getWriteNode() and
+        writeNode.getBasicBlock().(ReachableBasicBlock).strictlyDominates(result) and
+        // manual magic.
+        result = getAnAccessInContainer(getARelevantVariable(), _, false).getBasicBlock()
+      )
+    }
+  }
+
   /**
    * Gets a node that refers to the given access path relative to the given `root` node,
    * or `root` itself if the access path is empty.

javascript/ql/lib/semmle/javascript/PackageExports.qll

@@ -75,6 +75,16 @@ private DataFlow::Node getAValueExportedByPackage() {
     result = getAnExportFromModule(mod)
   )
   or
+  // re-export of a value from another module
+  // `module.exports.foo = require("./other").bar;`
+  // other.js:
+  // `module.exports.bar = function () { ... };`
+  exists(DataFlow::PropRead read, Import imp |
+    read = getAValueExportedByPackage() and
+    read.getBase().getALocalSource() = imp.getImportedModuleNode() and
+    result = imp.getImportedModule().getAnExportedValue(read.getPropertyName())
+  )
+  or
   // require("./other-module.js"); inside an AMD module.
   exists(Module mod, CallExpr call |
     call = getAValueExportedByPackage().asExpr() and

javascript/ql/lib/semmle/javascript/TypeScript.qll

@@ -1375,6 +1375,27 @@ class AsTypeAssertion extends TypeAssertion, @as_type_assertion { }
  */
 class PrefixTypeAssertion extends TypeAssertion, @prefix_type_assertion { }
 
+/**
+ * A satisfies type asserion of the form `E satisfies T` where `E` is an expression and `T` is a type.
+ */
+class SatisfiesExpr extends Expr, @satisfies_expr {
+  /** Gets the expression whose type to assert, that is, the `E` in `E as T` or `<T> E`. */
+  Expr getExpression() { result = this.getChildExpr(0) }
+
+  /** Gets the type to cast to, that is, the `T` in `E as T` or `<T> E`. */
+  TypeExpr getTypeAnnotation() { result = this.getChildTypeExpr(1) }
+
+  override ControlFlowNode getFirstControlFlowNode() {
+    result = this.getExpression().getFirstControlFlowNode()
+  }
+
+  override Expr getUnderlyingValue() { result = this.getExpression().getUnderlyingValue() }
+
+  override Expr getUnderlyingReference() { result = this.getExpression().getUnderlyingReference() }
+
+  override string getAPrimaryQlClass() { result = "SatisfiesExpr" }
+}
+
 /**
  * A TypeScript expression of form `E!`, asserting that `E` is not null.
  */

javascript/ql/lib/semmle/javascript/dataflow/DataFlow.qll

@@ -1580,6 +1580,8 @@ module DataFlow {
       or
       predExpr = succExpr.(TypeAssertion).getExpression()
       or
+      predExpr = succExpr.(SatisfiesExpr).getExpression()
+      or
       predExpr = succExpr.(NonNullAssertion).getExpression()
       or
       predExpr = succExpr.(ExpressionWithTypeArguments).getExpression()
@@ -1692,10 +1694,10 @@ module DataFlow {
    */
   predicate localFieldStep(DataFlow::Node pred, DataFlow::Node succ) {
     exists(ClassNode cls, string prop |
-      pred = cls.getADirectSuperClass*().getAReceiverNode().getAPropertyWrite(prop).getRhs() or
+      pred = AccessPath::getAnAssignmentTo(cls.getADirectSuperClass*().getAReceiverNode(), prop) or
       pred = cls.getInstanceMethod(prop)
     |
-      succ = cls.getAReceiverNode().getAPropertyRead(prop)
+      succ = AccessPath::getAReferenceTo(cls.getAReceiverNode(), prop)
     )
   }
 

javascript/ql/lib/semmle/javascript/frameworks/Hapi.qll

@@ -9,10 +9,28 @@ module Hapi {
   /**
    * An expression that creates a new Hapi server.
    */
-  class ServerDefinition extends Http::Servers::StandardServerDefinition, DataFlow::NewNode {
+  class ServerDefinition extends Http::Servers::StandardServerDefinition, DataFlow::Node {
     ServerDefinition() {
       // `server = new Hapi.Server()`
       this = DataFlow::moduleMember("hapi", "Server").getAnInstantiation()
+      or
+      // `server = Glue.compose(manifest, composeOptions)`
+      this = DataFlow::moduleMember("@hapi/glue", "compose").getAnInvocation()
+      or
+      // `register (server, options)`
+      // `module.exports.plugin = {register, pkg};`
+      this =
+        any(Module m)
+            .getAnExportedValue("plugin")
+            .getALocalSource()
+            .getAPropertySource("register")
+            .getAFunctionValue()
+            .getParameter(0)
+      or
+      // `const after = function (server) {...};`
+      // `server.dependency('name', after);`
+      this =
+        any(ServerDefinition s).ref().getAMethodCall("dependency").getABoundCallbackParameter(1, 0)
     }
   }
 
@@ -123,7 +141,7 @@ module Hapi {
         kind = "parameter" and
         exists(DataFlow::PropRead query |
           // `request.query.name`
-          query.accesses(request, "query") and
+          query.accesses(request, ["query", "params"]) and
           this.(DataFlow::PropRead).accesses(query, _)
         )
         or
@@ -131,7 +149,7 @@ module Hapi {
           // `request.url.path`
           kind = "url" and
           url.accesses(request, "url") and
-          this.(DataFlow::PropRead).accesses(url, "path")
+          this.(DataFlow::PropRead).accesses(url, ["path", "origin"])
         )
         or
         exists(DataFlow::PropRead state |
@@ -209,6 +227,17 @@ module Hapi {
         // server.ext('/', fun)
         this.getMethodName() = "ext" and
         handler = this.getArgument(1)
+        or
+        // server.route([{ handler(request){}])
+        this.getMethodName() = "route" and
+        handler =
+          this.getArgument(0)
+              .getALocalSource()
+              .(DataFlow::ArrayCreationNode)
+              .getAnElement()
+              .getALocalSource()
+              .getAPropertySource("handler")
+              .getAFunctionValue()
       )
     }
 
@@ -240,7 +269,7 @@ module Hapi {
     RouteHandlerCandidate() {
       exists(string request, string responseToolkit |
         (request = "request" or request = "req") and
-        responseToolkit = "h" and
+        responseToolkit = ["h", "hapi"] and
         // heuristic: parameter names match the Hapi documentation
         astNode.getNumParameter() = 2 and
         astNode.getParameter(0).getName() = request and

javascript/ql/lib/semmle/javascript/frameworks/NoSQL.qll

@@ -251,7 +251,7 @@ private module Redis {
           "set", "publish", "append", "bitfield", "decrby", "getset", "hincrby", "hincrbyfloat",
           "hset", "hsetnx", "incrby", "incrbyfloat", "linsert", "lpush", "lpushx", "lset", "ltrim",
           "rename", "renamenx", "rpushx", "setbit", "setex", "smove", "zincrby", "zinterstore",
-          "hdel", "lpush", "pfadd", "rpush", "sadd", "sdiffstore", "srem"
+          "hdel", "pfadd", "rpush", "sadd", "sdiffstore", "srem"
         ] and
       argIndex = 0
       or

javascript/ql/lib/semmle/javascript/frameworks/Vue.qll

@@ -115,11 +115,6 @@ module Vue {
     kind = DataFlow::MemberKind::setter() and result = "set"
   }
 
-  /**
-   * DEPRECATED. This class has been renamed to `Vue::Component`.
-   */
-  deprecated class Instance = Component;
-
   /**
    * A Vue component, such as a `new Vue({ ... })` call or a `.vue` file.
    *
@@ -383,23 +378,6 @@ module Vue {
     }
   }
 
-  /**
-   * DEPRECATED. Use `Vue::Component` instead.
-   *
-   * A Vue component from `new Vue({...})`.
-   */
-  deprecated class VueInstance extends Component {
-    VueInstance() {
-      // restrict charpred to match original behavior
-      this = MkComponentInstantiation(vueLibrary().getAnInstantiation())
-    }
-  }
-
-  /**
-   * DEPRECATED. Use `Vue::ComponentExtension` or `Vue::Component` instead.
-   */
-  deprecated class ExtendedVue = ComponentExtension;
-
   /**
    * A component created via an explicit call to `Vue.extend({...})` or `CustomComponent.extend({...})`.
    */
@@ -429,19 +407,6 @@ module Vue {
     }
   }
 
-  /**
-   * DEPRECATED. Use `Vue::Component` instead.
-   *
-   * An instance of an extended Vue, for example `instance` of `var Ext = Vue.extend({...}); var instance = new Ext({...})`.
-   */
-  deprecated class ExtendedInstance extends Component {
-    ExtendedInstance() {
-      // restrict charpred to match original behavior
-      this =
-        MkComponentInstantiation(vueLibrary().getMember("extend").getReturn().getAnInstantiation())
-    }
-  }
-
   /**
    * A Vue component from `Vue.component("my-component", { ... })`.
    */
@@ -568,9 +533,6 @@ module Vue {
     }
   }
 
-  /** DEPRECATED. Do not use. */
-  deprecated class InstanceHeapStep = PropStep;
-
   /**
    * A Vue `v-html` attribute.
    */
@@ -609,11 +571,6 @@ module Vue {
     }
   }
 
-  /**
-   * DEPRECATED. Do not use.
-   */
-  deprecated class VHtmlSourceWrite = VHtmlAttributeStep;
-
   /*
    * Provides classes for working with Vue templates.
    */

javascript/ql/lib/semmle/javascript/frameworks/data/internal/AccessPathSyntax.qll

@@ -1,5 +1,5 @@
 /**
- * Module for parsing access paths from CSV models, both the identifying access path used
+ * Module for parsing access paths from MaD models, both the identifying access path used
  * by dynamic languages, and the input/output specifications for summary steps.
  *
  * This file is used by the shared data flow library and by the JavaScript libraries

javascript/ql/lib/semmle/javascript/security/dataflow/BrokenCryptoAlgorithm.qll

@@ -1,7 +0,0 @@
-/** DEPRECATED. Import `BrokenCryptoAlgorithmQuery` instead. */
-
-import javascript
-private import BrokenCryptoAlgorithmQuery as BrokenCryptoAlgorithmQuery // ignore-query-import
-
-/** DEPRECATED. Import `BrokenCryptoAlgorithmQuery` instead. */
-deprecated module BrokenCryptoAlgorithm = BrokenCryptoAlgorithmQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/BuildArtifactLeak.qll

@@ -1,7 +0,0 @@
-/** DEPRECATED. Import `BuildArtifactLeakQuery` instead. */
-
-import javascript
-private import BuildArtifactLeakQuery as BuildArtifactLeakQuery // ignore-query-import
-
-/** DEPRECATED. Import `BuildArtifactLeakQuery` instead. */
-deprecated module BuildArtifactLeak = BuildArtifactLeakQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/CleartextLogging.qll

@@ -1,7 +0,0 @@
-/** DEPRECATED. Import `CleartextLoggingQuery` instead. */
-
-import javascript
-private import CleartextLoggingQuery as CleartextLoggingQuery // ignore-query-import
-
-/** DEPRECATED. Import `CleartextLoggingQuery` instead. */
-deprecated module CleartextLogging = CleartextLoggingQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/CleartextStorage.qll

@@ -1,7 +0,0 @@
-/** DEPRECATED. Import `CleartextStorageQuery` instead. */
-
-import javascript
-private import CleartextStorageQuery as CleartextStorageQuery // ignore-query-import
-
-/** DEPRECATED. Import `CleartextStorageQuery` instead. */
-deprecated module CleartextStorage = CleartextStorageQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/ClientSideUrlRedirect.qll

@@ -1,8 +0,0 @@
-/** DEPRECATED. Import `ClientSideUrlRedirectQuery` instead. */
-
-import javascript
-import UrlConcatenation
-private import ClientSideUrlRedirectQuery as ClientSideUrlRedirectQuery // ignore-query-import
-
-/** DEPRECATED. Import `ClientSideUrlRedirectQuery` instead. */
-deprecated module ClientSideUrlRedirect = ClientSideUrlRedirectQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/CodeInjection.qll

@@ -1,7 +0,0 @@
-/** DEPRECATED. Import `CodeInjectionQuery` instead. */
-
-import javascript
-private import CodeInjectionQuery as CodeInjectionQuery // ignore-query-import
-
-/** DEPRECATED. Import `CodeInjectionQuery` instead. */
-deprecated module CodeInjection = CodeInjectionQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/CommandInjection.qll

@@ -1,7 +0,0 @@
-/** DEPRECATED. Import `CommandInjectionQuery` instead. */
-
-import javascript
-private import CommandInjectionQuery as CommandInjectionQuery // ignore-query-import
-
-/** DEPRECATED. Import `CommandInjectionQuery` instead. */
-deprecated module CommandInjection = CommandInjectionQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/ConditionalBypass.qll

@@ -1,7 +0,0 @@
-/** DEPRECATED. Import `ConditionalBypassQuery` instead. */
-
-import javascript
-private import ConditionalBypassQuery as ConditionalBypassQuery // ignore-query-import
-
-/** DEPRECATED. Import `ConditionalBypassQuery` instead. */
-deprecated module ConditionalBypass = ConditionalBypassQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/CorsMisconfigurationForCredentials.qll

@@ -1,7 +0,0 @@
-/** DEPRECATED. Import `CorsMisconfigurationForCredentialsQuery` instead. */
-
-import javascript
-private import CorsMisconfigurationForCredentialsQuery as CorsMisconfigurationForCredentialsQuery // ignore-query-import
-
-/** DEPRECATED. Import `CorsMisconfigurationForCredentialsQuery` instead. */
-deprecated module CorsMisconfigurationForCredentials = CorsMisconfigurationForCredentialsQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/DeepObjectResourceExhaustion.qll

@@ -1,8 +0,0 @@
-/** DEPRECATED. Import `DeepObjectResourceExhaustionQuery` instead. */
-
-import javascript
-import semmle.javascript.security.TaintedObject
-private import DeepObjectResourceExhaustionQuery as DeepObjectResourceExhaustionQuery // ignore-query-import
-
-/** DEPRECATED. Import `DeepObjectResourceExhaustionQuery` instead. */
-deprecated module DeepObjectResourceExhaustion = DeepObjectResourceExhaustionQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/DifferentKindsComparisonBypass.qll

@@ -1,7 +0,0 @@
-/** DEPRECATED. Import `DifferentKindsComparisonBypassQuery` instead. */
-
-import javascript
-private import DifferentKindsComparisonBypassQuery as DifferentKindsComparisonBypassQuery // ignore-query-import
-
-/** DEPRECATED. Import `DifferentKindsComparisonBypassQuery` instead. */
-deprecated module DifferentKindsComparisonBypass = DifferentKindsComparisonBypassQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/DomBasedXss.qll

@@ -1,7 +0,0 @@
-/** DEPRECATED. Import `DomBasedXssQuery` instead. */
-
-import javascript
-private import DomBasedXssQuery as DomBasedXssQuery // ignore-query-import
-
-/** DEPRECATED. Import `DomBasedXssQuery` instead. */
-deprecated module DomBasedXss = DomBasedXssQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/DomBasedXssQuery.qll

@@ -8,11 +8,6 @@ private import semmle.javascript.security.TaintedUrlSuffix
 import DomBasedXssCustomizations::DomBasedXss
 private import Xss::Shared as Shared
 
-/**
- * DEPRECATED. Use `Vue::VHtmlSourceWrite` instead.
- */
-deprecated class VHtmlSourceWrite = Vue::VHtmlSourceWrite;
-
 /** DEPRECATED. Use `Configuration`. */
 deprecated class HtmlInjectionConfiguration = Configuration;
 

javascript/ql/lib/semmle/javascript/security/dataflow/ExceptionXss.qll

@@ -1,7 +0,0 @@
-/** DEPRECATED. Import `ExceptionXssQuery` instead. */
-
-import javascript
-private import ExceptionXssQuery as ExceptionXssQuery // ignore-query-import
-
-/** DEPRECATED. Import `ExceptionXssQuery` instead. */
-deprecated module ExceptionXss = ExceptionXssQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/FileAccessToHttp.qll

@@ -1,7 +0,0 @@
-/** DEPRECATED. Import `FileAccessToHttpQuery` instead. */
-
-import javascript
-private import FileAccessToHttpQuery as FileAccessToHttpQuery // ignore-query-import
-
-/** DEPRECATED. Import `FileAccessToHttpQuery` instead. */
-deprecated module FileAccessToHttp = FileAccessToHttpQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/HardcodedCredentials.qll

@@ -1,7 +0,0 @@
-/** DEPRECATED. Import `HardcodedCredentialsQuery` instead. */
-
-import javascript
-private import HardcodedCredentialsQuery as HardcodedCredentialsQuery // ignore-query-import
-
-/** DEPRECATED. Import `HardcodedCredentialsQuery` instead. */
-deprecated module HardcodedCredentials = HardcodedCredentialsQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/HardcodedDataInterpretedAsCode.qll

@@ -1,7 +0,0 @@
-/** DEPRECATED. Import `HardcodedDataInterpretedAsCodeQuery` instead. */
-
-import javascript
-private import HardcodedDataInterpretedAsCodeQuery as HardcodedDataInterpretedAsCodeQuery // ignore-query-import
-
-/** DEPRECATED. Import `HardcodedDataInterpretedAsCodeQuery` instead. */
-deprecated module HardcodedDataInterpretedAsCode = HardcodedDataInterpretedAsCodeQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/HostHeaderPoisoningInEmailGeneration.qll

@@ -1,7 +0,0 @@
-/** DEPRECATED. Import `HostHeaderPoisoningInEmailGenerationQuery` instead. */
-
-import javascript
-private import HostHeaderPoisoningInEmailGenerationQuery as HostHeaderPoisoningInEmailGenerationQuery // ignore-query-import
-
-/** DEPRECATED. Import `HostHeaderPoisoningInEmailGenerationQuery` instead. */
-deprecated module HostHeaderPoisoningInEmailGeneration = HostHeaderPoisoningInEmailGenerationQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/HttpToFileAccess.qll

@@ -1,7 +0,0 @@
-/** DEPRECATED. Import `HttpToFileAccessQuery` instead. */
-
-import javascript
-private import HttpToFileAccessQuery as HttpToFileAccessQuery // ignore-query-import
-
-/** DEPRECATED. Import `HttpToFileAccessQuery` instead. */
-deprecated module HttpToFileAccess = HttpToFileAccessQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/ImproperCodeSanitization.qll

@@ -1,7 +0,0 @@
-/** DEPRECATED. Import `ImproperCodeSanitizationQuery` instead. */
-
-import javascript
-private import ImproperCodeSanitizationQuery as ImproperCodeSanitizationQuery // ignore-query-import
-
-/** DEPRECATED. Import `ImproperCodeSanitizationQuery` instead. */
-deprecated module ImproperCodeSanitization = ImproperCodeSanitizationQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/IncompleteHtmlAttributeSanitization.qll

@@ -1,7 +0,0 @@
-/** DEPRECATED. Import `IncompleteHtmlAttributeSanitizationQuery` instead. */
-
-import javascript
-private import IncompleteHtmlAttributeSanitizationQuery as IncompleteHtmlAttributeSanitizationQuery // ignore-query-import
-
-/** DEPRECATED. Import `IncompleteHtmlAttributeSanitizationQuery` instead. */
-deprecated module IncompleteHtmlAttributeSanitization = IncompleteHtmlAttributeSanitizationQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/IndirectCommandInjection.qll

@@ -1,7 +0,0 @@
-/** DEPRECATED. Import `IndirectCommandInjectionQuery` instead. */
-
-import javascript
-private import IndirectCommandInjectionQuery as IndirectCommandInjectionQuery // ignore-query-import
-
-/** DEPRECATED. Import `IndirectCommandInjectionQuery` instead. */
-deprecated module IndirectCommandInjection = IndirectCommandInjectionQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/InsecureDownload.qll

@@ -1,7 +0,0 @@
-/** DEPRECATED. Import `InsecureDownloadQuery` instead. */
-
-import javascript
-private import InsecureDownloadQuery as InsecureDownloadQuery // ignore-query-import
-
-/** DEPRECATED. Import `InsecureDownloadQuery` instead. */
-deprecated module InsecureDownload = InsecureDownloadQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/InsecureRandomness.qll

@@ -1,7 +0,0 @@
-/** DEPRECATED. Import `InsecureRandomnessQuery` instead. */
-
-import javascript
-private import InsecureRandomnessQuery as InsecureRandomnessQuery // ignore-query-import
-
-/** DEPRECATED. Import `InsecureRandomnessQuery` instead. */
-deprecated module InsecureRandomness = InsecureRandomnessQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/InsufficientPasswordHash.qll

@@ -1,7 +0,0 @@
-/** DEPRECATED. Import `InsufficientPasswordHashQuery` instead. */
-
-import javascript
-private import InsufficientPasswordHashQuery as InsufficientPasswordHashQuery // ignore-query-import
-
-/** DEPRECATED. Import `InsufficientPasswordHashQuery` instead. */
-deprecated module InsufficientPasswordHash = InsufficientPasswordHashQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/LogInjection.qll

@@ -1,7 +0,0 @@
-/** DEPRECATED. Import `LogInjectionQuery` instead. */
-
-import javascript
-private import LogInjectionQuery as LogInjectionQuery // ignore-query-import
-
-/** DEPRECATED. Import `LogInjectionQuery` instead. */
-deprecated module LogInjection = LogInjectionQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/LoopBoundInjection.qll

@@ -1,8 +0,0 @@
-/** DEPRECATED. Import `LoopBoundInjectionQuery` instead. */
-
-import javascript
-import semmle.javascript.security.TaintedObject
-private import LoopBoundInjectionQuery as LoopBoundInjectionQuery // ignore-query-import
-
-/** DEPRECATED. Import `LoopBoundInjectionQuery` instead. */
-deprecated module LoopBoundInjection = LoopBoundInjectionQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/LoopBoundInjectionCustomizations.qll

@@ -122,10 +122,10 @@ module LoopBoundInjection {
         "flattenDeep", "flattenDepth", "initial", "intersection", "intersectionBy",
         "intersectionWith", "join", "remove", "reverse", "slice", "sortedUniq", "sortedUniqBy",
         "tail", "union", "unionBy", "unionWith", "uniqBy", "unzip", "unzipWith", "without", "zip",
-        "zipObject", "zipObjectDeep", "zipWith", "countBy", "each", "forEach", "eachRight",
-        "forEachRight", "filter", "find", "findLast", "flatMap", "flatMapDeep", "flatMapDepth",
-        "forEach", "forEachRight", "groupBy", "invokeMap", "keyBy", "map", "orderBy", "partition",
-        "reduce", "reduceRight", "reject", "sortBy"
+        "zipObject", "zipObjectDeep", "zipWith", "countBy", "each", "eachRight", "forEachRight",
+        "filter", "find", "findLast", "flatMap", "flatMapDeep", "flatMapDepth", "forEach",
+        "groupBy", "invokeMap", "keyBy", "map", "orderBy", "partition", "reduce", "reduceRight",
+        "reject", "sortBy"
       ]
   }
 

javascript/ql/lib/semmle/javascript/security/dataflow/NosqlInjection.qll

@@ -1,8 +0,0 @@
-/** DEPRECATED. Import `NosqlInjectionQuery` instead. */
-
-import javascript
-import semmle.javascript.security.TaintedObject
-private import NosqlInjectionQuery as NosqlInjectionQuery // ignore-query-import
-
-/** DEPRECATED. Import `NosqlInjectionQuery` instead. */
-deprecated module NosqlInjection = NosqlInjectionQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/PostMessageStar.qll

@@ -1,7 +0,0 @@
-/** DEPRECATED. Import `PostMessageStarQuery` instead. */
-
-import javascript
-private import PostMessageStarQuery as PostMessageStarQuery // ignore-query-import
-
-/** DEPRECATED. Import `PostMessageStarQuery` instead. */
-deprecated module PostMessageStar = PostMessageStarQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/PrototypePollutingAssignment.qll

@@ -1,6 +0,0 @@
-/** DEPRECATED. Import `PrototypePollutingAssignmentQuery` instead. */
-
-private import PrototypePollutingAssignmentQuery as PrototypePollutingAssignmentQuery // ignore-query-import
-
-/** DEPRECATED. Import `PrototypePollutingAssignmentQuery` instead. */
-deprecated module PrototypePollutingAssignment = PrototypePollutingAssignmentQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/PrototypePollution.qll

@@ -1,10 +0,0 @@
-/** DEPRECATED. Import `PrototypePollutionQuery` instead. */
-
-import javascript
-import semmle.javascript.security.TaintedObject
-import semmle.javascript.dependencies.Dependencies
-import semmle.javascript.dependencies.SemVer
-private import PrototypePollutionQuery as PrototypePollutionQuery // ignore-query-import
-
-/** DEPRECATED. Import `PrototypePollutionQuery` instead. */
-deprecated module PrototypePollution = PrototypePollutionQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/ReflectedXss.qll

@@ -1,7 +0,0 @@
-/** DEPRECATED. Import `ReflectedXssQuery` instead. */
-
-import javascript
-private import ReflectedXssQuery as ReflectedXssQuery // ignore-query-import
-
-/** DEPRECATED. Import `ReflectedXssQuery` instead. */
-deprecated module ReflectedXss = ReflectedXssQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/RegExpInjection.qll

@@ -1,7 +0,0 @@
-/** DEPRECATED. Import `RegExpInjectionQuery` instead. */
-
-import javascript
-private import RegExpInjectionQuery as RegExpInjectionQuery // ignore-query-import
-
-/** DEPRECATED. Import `RegExpInjectionQuery` instead. */
-deprecated module RegExpInjection = RegExpInjectionQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/RemotePropertyInjection.qll

@@ -1,7 +0,0 @@
-/** DEPRECATED. Import `RemotePropertyInjectionQuery` instead. */
-
-import javascript
-private import RemotePropertyInjectionQuery as RemotePropertyInjectionQuery // ignore-query-import
-
-/** DEPRECATED. Import `RemotePropertyInjectionQuery` instead. */
-deprecated module RemotePropertyInjection = RemotePropertyInjectionQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/RequestForgery.qll

@@ -1,8 +0,0 @@
-/** DEPRECATED. Import `RequestForgeryQuery` instead. */
-
-import javascript
-import UrlConcatenation
-private import RequestForgeryQuery as RequestForgeryQuery // ignore-query-import
-
-/** DEPRECATED. Import `RequestForgeryQuery` instead. */
-deprecated module RequestForgery = RequestForgeryQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/SecondOrderCommandInjectionQuery.qll

@@ -12,7 +12,7 @@ import SecondOrderCommandInjectionCustomizations::SecondOrderCommandInjection
 private import semmle.javascript.security.TaintedObject
 
 /**
- * A taint-tracking configuration for reasoning about command-injection vulnerabilities.
+ * A taint-tracking configuration for reasoning about second order command-injection vulnerabilities.
  */
 class Configuration extends TaintTracking::Configuration {
   Configuration() { this = "SecondOrderCommandInjection" }

javascript/ql/lib/semmle/javascript/security/dataflow/ServerSideUrlRedirect.qll

@@ -1,9 +0,0 @@
-/** DEPRECATED. Import `ServerSideUrlRedirectQuery` instead. */
-
-import javascript
-import RemoteFlowSources
-import UrlConcatenation
-private import ServerSideUrlRedirectQuery as ServerSideUrlRedirectQuery // ignore-query-import
-
-/** DEPRECATED. Import `ServerSideUrlRedirectQuery` instead. */
-deprecated module ServerSideUrlRedirect = ServerSideUrlRedirectQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/ShellCommandInjectionFromEnvironment.qll

@@ -1,7 +0,0 @@
-/** DEPRECATED. Import `ShellCommandInjectionFromEnvironmentQuery` instead. */
-
-import javascript
-private import ShellCommandInjectionFromEnvironmentQuery as ShellCommandInjectionFromEnvironmentQuery // ignore-query-import
-
-/** DEPRECATED. Import `ShellCommandInjectionFromEnvironmentQuery` instead. */
-deprecated module ShellCommandInjectionFromEnvironment = ShellCommandInjectionFromEnvironmentQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/SqlInjection.qll

@@ -1,7 +0,0 @@
-/** DEPRECATED. Import `SqlInjectionQuery` instead. */
-
-import javascript
-private import SqlInjectionQuery as SqlInjectionQuery // ignore-query-import
-
-/** DEPRECATED. Import `SqlInjectionQuery` instead. */
-deprecated module SqlInjection = SqlInjectionQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/StackTraceExposure.qll

@@ -1,7 +0,0 @@
-/** DEPRECATED. Import `StackTraceExposureQuery` instead. */
-
-import javascript
-private import StackTraceExposureQuery as StackTraceExposureQuery // ignore-query-import
-
-/** DEPRECATED. Import `StackTraceExposureQuery` instead. */
-deprecated module StackTraceExposure = StackTraceExposureQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/StoredXss.qll

@@ -1,7 +0,0 @@
-/** DEPRECATED. Import `StoredXssQuery` instead. */
-
-import javascript
-private import StoredXssQuery as StoredXssQuery // ignore-query-import
-
-/** DEPRECATED. Import `StoredXssQuery` instead. */
-deprecated module StoredXss = StoredXssQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/TaintedFormatString.qll

@@ -1,8 +0,0 @@
-/** DEPRECATED. Import `TaintedFormatStringQuery` instead. */
-
-import javascript
-import semmle.javascript.security.dataflow.DOM
-private import TaintedFormatStringQuery as TaintedFormatStringQuery // ignore-query-import
-
-/** DEPRECATED. Import `TaintedFormatStringQuery` instead. */
-deprecated module TaintedFormatString = TaintedFormatStringQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/TaintedPath.qll

@@ -1,7 +0,0 @@
-/** DEPRECATED. Import `TaintedPathQuery` instead. */
-
-import javascript
-private import TaintedPathQuery as TaintedPathQuery // ignore-query-import
-
-/** DEPRECATED. Import `TaintedPathQuery` instead. */
-deprecated module TaintedPath = TaintedPathQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/TemplateObjectInjection.qll

@@ -1,7 +0,0 @@
-/** DEPRECATED. Import `TemplateObjectInjectionQuery` instead. */
-
-import javascript
-private import TemplateObjectInjectionQuery as TemplateObjectInjectionQuery // ignore-query-import
-
-/** DEPRECATED. Import `TemplateObjectInjectionQuery` instead. */
-deprecated module TemplateObjectInjection = TemplateObjectInjectionQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/TypeConfusionThroughParameterTampering.qll

@@ -1,8 +0,0 @@
-/** DEPRECATED. Import `TypeConfusionThroughParameterTamperingQuery` instead. */
-
-import javascript
-private import TypeConfusionThroughParameterTamperingQuery as TypeConfusionThroughParameterTamperingQuery // ignore-query-import
-
-/** DEPRECATED. Import `TypeConfusionThroughParameterTamperingQuery` instead. */
-deprecated module TypeConfusionThroughParameterTampering =
-  TypeConfusionThroughParameterTamperingQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/UnsafeDeserialization.qll

@@ -1,7 +0,0 @@
-/** DEPRECATED. Import `UnsafeDeserializationQuery` instead. */
-
-import javascript
-private import UnsafeDeserializationQuery as UnsafeDeserializationQuery // ignore-query-import
-
-/** DEPRECATED. Import `UnsafeDeserializationQuery` instead. */
-deprecated module UnsafeDeserialization = UnsafeDeserializationQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/UnsafeDynamicMethodAccess.qll

@@ -1,8 +0,0 @@
-/** DEPRECATED. Import `UnsafeDynamicMethodAccessQuery` instead. */
-
-import javascript
-import PropertyInjectionShared
-private import UnsafeDynamicMethodAccessQuery as UnsafeDynamicMethodAccessQuery // ignore-query-import
-
-/** DEPRECATED. Import `UnsafeDynamicMethodAccessQuery` instead. */
-deprecated module UnsafeDynamicMethodAccess = UnsafeDynamicMethodAccessQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/UnsafeHtmlConstruction.qll

@@ -1,7 +0,0 @@
-/** DEPRECATED. Import `UnsafeHtmlConstructionQuery` instead. */
-
-import javascript
-private import UnsafeHtmlConstructionQuery as UnsafeHtmlConstructionQuery // ignore-query-import
-
-/** DEPRECATED. Import `UnsafeHtmlConstructionQuery` instead. */
-deprecated module UnsafeHtmlConstruction = UnsafeHtmlConstructionQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/UnsafeJQueryPlugin.qll

@@ -1,8 +0,0 @@
-/** DEPRECATED. Import `UnsafeJQueryPluginQuery` instead. */
-
-import javascript
-import semmle.javascript.security.dataflow.Xss
-private import UnsafeJQueryPluginQuery as UnsafeJQueryPluginQuery // ignore-query-import
-
-/** DEPRECATED. Import `UnsafeJQueryPluginQuery` instead. */
-deprecated module UnsafeJQueryPlugin = UnsafeJQueryPluginQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/UnsafeShellCommandConstruction.qll

@@ -1,7 +0,0 @@
-/** DEPRECATED. Import `UnsafeShellCommandConstructionQuery` instead. */
-
-import javascript
-private import UnsafeShellCommandConstructionQuery as UnsafeShellCommandConstructionQuery // ignore-query-import
-
-/** DEPRECATED. Import `UnsafeShellCommandConstructionQuery` instead. */
-deprecated module UnsafeShellCommandConstruction = UnsafeShellCommandConstructionQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/UnsafeShellCommandConstructionCustomizations.qll

@@ -156,14 +156,9 @@ module UnsafeShellCommandConstruction {
   }
 
   /**
-   * Gets a node that ends up in an array that is ultimately executed as a shell script by `sys`.
+   * Holds if the arguments array given to `sys` is joined as a string because `shell` is set to true.
    */
-  private DataFlow::SourceNode endsInShellExecutedArray(
-    DataFlow::TypeBackTracker t, SystemCommandExecution sys
-  ) {
-    t.start() and
-    result = sys.getArgumentList().getALocalSource() and
-    // the array gets joined to a string when `shell` is set to true.
+  predicate executesArrayAsShell(SystemCommandExecution sys) {
     sys.getOptionsArg()
         .getALocalSource()
         .getAPropertyWrite("shell")
@@ -171,6 +166,17 @@ module UnsafeShellCommandConstruction {
         .asExpr()
         .(BooleanLiteral)
         .getValue() = "true"
+  }
+
+  /**
+   * Gets a node that ends up in an array that is ultimately executed as a shell script by `sys`.
+   */
+  private DataFlow::SourceNode endsInShellExecutedArray(
+    DataFlow::TypeBackTracker t, SystemCommandExecution sys
+  ) {
+    t.start() and
+    result = sys.getArgumentList().getALocalSource() and
+    executesArrayAsShell(sys)
     or
     exists(DataFlow::TypeBackTracker t2 |
       result = endsInShellExecutedArray(t2, sys).backtrack(t2, t)
@@ -193,6 +199,10 @@ module UnsafeShellCommandConstruction {
         or
         this = arr.getAMethodCall(["push", "unshift"]).getAnArgument()
       )
+      or
+      this = sys.getArgumentList() and
+      not this instanceof DataFlow::ArrayCreationNode and
+      executesArrayAsShell(sys)
     }
 
     override string getSinkType() { result = "shell argument" }

javascript/ql/lib/semmle/javascript/security/dataflow/UnvalidatedDynamicMethodCall.qll

@@ -1,9 +0,0 @@
-/** DEPRECATED. Import `UnvalidatedDynamicMethodCallQuery` instead. */
-
-import javascript
-import semmle.javascript.frameworks.Express
-import PropertyInjectionShared
-private import UnvalidatedDynamicMethodCallQuery as UnvalidatedDynamicMethodCallQuery // ignore-query-import
-
-/** DEPRECATED. Import `UnvalidatedDynamicMethodCallQuery` instead. */
-deprecated module UnvalidatedDynamicMethodCall = UnvalidatedDynamicMethodCallQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/XmlBomb.qll

@@ -1,7 +0,0 @@
-/** DEPRECATED. Import `XmlBombQuery` instead. */
-
-import javascript
-private import XmlBombQuery as XmlBombQuery // ignore-query-import
-
-/** DEPRECATED. Import `XmlBombQuery` instead. */
-deprecated module XmlBomb = XmlBombQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/XpathInjection.qll

@@ -1,8 +0,0 @@
-/** DEPRECATED. Import `XpathInjectionQuery` instead. */
-
-import javascript
-import semmle.javascript.security.dataflow.DOM
-private import XpathInjectionQuery as XpathInjectionQuery // ignore-query-import
-
-/** DEPRECATED. Import `XpathInjectionQuery` instead. */
-deprecated module XpathInjection = XpathInjectionQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/XssThroughDom.qll

@@ -1,7 +0,0 @@
-/** DEPRECATED. Import `XssThroughDomQuery` instead. */
-
-import javascript
-private import XssThroughDomQuery as XssThroughDomQuery // ignore-query-import
-
-/** DEPRECATED. Import `XssThroughDomQuery` instead. */
-deprecated module XssThroughDom = XssThroughDomQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/Xxe.qll

@@ -1,7 +0,0 @@
-/** DEPRECATED. Import `XxeQuery` instead. */
-
-import javascript
-private import XxeQuery as XxeQuery // ignore-query-import
-
-/** DEPRECATED. Import `XxeQuery` instead. */
-deprecated module Xxe = XxeQuery;

javascript/ql/lib/semmle/javascript/security/dataflow/ZipSlip.qll

@@ -1,7 +0,0 @@
-/** DEPRECATED. Import `ZipSlipQuery` instead. */
-
-import javascript
-private import ZipSlipQuery as ZipSlipQuery // ignore-query-import
-
-/** DEPRECATED. Import `ZipSlipQuery` instead. */
-deprecated module ZipSlip = ZipSlipQuery;

javascript/ql/lib/semmle/javascript/security/internal/SensitiveDataHeuristics.qll

@@ -103,7 +103,7 @@ module HeuristicNames {
    */
   string notSensitiveRegexp() {
     result =
-      "(?is).*([^\\w$.-]|redact|censor|obfuscate|hash|md5|sha|random|((?<!un)(en))?(crypt|code)|certain|concert|secretar|accountant|accountab).*"
+      "(?is).*([^\\w$.-]|redact|censor|obfuscate|hash|md5|sha|random|((?<!un)(en))?(crypt|(?<!pass)code)|certain|concert|secretar|accountant|accountab).*"
   }
 
   /**

javascript/ql/lib/semmle/javascript/security/regexp/NfaUtils.qll

@@ -129,19 +129,20 @@ private predicate isCanonicalTerm(RelevantRegExpTerm term, string str) {
     min(RelevantRegExpTerm t, Location loc, File file |
       loc = t.getLocation() and
       file = t.getFile() and
-      str = t.getRawValue() + "|" + getCanonicalizationFlags(t.getRootTerm())
+      str = getCanonicalizationString(t)
     |
       t order by t.getFile().getRelativePath(), loc.getStartLine(), loc.getStartColumn()
     )
 }
 
 /**
- * Gets a string representation of the flags used with the regular expression.
- * Only the flags that are relevant for the canonicalization are included.
+ * Gets a string representation of `term` that is used for canonicalization.
  */
-string getCanonicalizationFlags(RegExpTerm root) {
-  root.isRootTerm() and
-  (if RegExpFlags::isIgnoreCase(root) then result = "i" else result = "")
+private string getCanonicalizationString(RelevantRegExpTerm term) {
+  exists(string ignoreCase |
+    (if RegExpFlags::isIgnoreCase(term.getRootTerm()) then ignoreCase = "i" else ignoreCase = "") and
+    result = term.getRawValue() + "|" + ignoreCase
+  )
 }
 
 /**
@@ -186,12 +187,19 @@ private newtype TInputSymbol =
   Epsilon()
 
 /**
- * Gets the canonical CharClass for `term`.
+ * Gets the the CharClass corresponding to the canonical representative `term`.
  */
-CharClass getCanonicalCharClass(RegExpTerm term) {
+private CharClass getCharClassForCanonicalTerm(RegExpTerm term) {
   exists(string str | isCanonicalTerm(term, str) | result = CharClass(str))
 }
 
+/**
+ * Gets a char class that represents `term`, even when `term` is not the canonical representative.
+ */
+CharacterClass getCanonicalCharClass(RegExpTerm term) {
+  exists(string str | str = getCanonicalizationString(term) and result = CharClass(str))
+}
+
 /**
  * Holds if `a` and `b` are input symbols from the same regexp.
  */
@@ -284,7 +292,7 @@ private module CharacterClasses {
    */
   pragma[noinline]
   predicate hasChildThatMatchesIgnoringCasingFlags(RegExpCharacterClass cc, string char) {
-    exists(getCanonicalCharClass(cc)) and
+    exists(getCharClassForCanonicalTerm(cc)) and
     exists(RegExpTerm child | child = cc.getAChild() |
       char = child.(RegexpCharacterConstant).getValue()
       or
@@ -387,7 +395,7 @@ private module CharacterClasses {
   private class PositiveCharacterClass extends CharacterClass {
     RegExpCharacterClass cc;
 
-    PositiveCharacterClass() { this = getCanonicalCharClass(cc) and not cc.isInverted() }
+    PositiveCharacterClass() { this = getCharClassForCanonicalTerm(cc) and not cc.isInverted() }
 
     override string getARelevantChar() { result = caseNormalize(getAMentionedChar(cc), cc) }
 
@@ -400,7 +408,7 @@ private module CharacterClasses {
   private class InvertedCharacterClass extends CharacterClass {
     RegExpCharacterClass cc;
 
-    InvertedCharacterClass() { this = getCanonicalCharClass(cc) and cc.isInverted() }
+    InvertedCharacterClass() { this = getCharClassForCanonicalTerm(cc) and cc.isInverted() }
 
     override string getARelevantChar() {
       result = nextChar(caseNormalize(getAMentionedChar(cc), cc)) or
@@ -435,7 +443,7 @@ private module CharacterClasses {
 
     PositiveCharacterClassEscape() {
       isEscapeClass(cc, charClass) and
-      this = getCanonicalCharClass(cc) and
+      this = getCharClassForCanonicalTerm(cc) and
       charClass = ["d", "s", "w"]
     }
 
@@ -475,7 +483,7 @@ private module CharacterClasses {
     NegativeCharacterClassEscape() {
       exists(RegExpTerm cc |
         isEscapeClass(cc, charClass) and
-        this = getCanonicalCharClass(cc) and
+        this = getCharClassForCanonicalTerm(cc) and
         charClass = ["D", "S", "W"]
       )
     }
@@ -652,17 +660,13 @@ predicate delta(State q1, EdgeLabel lbl, State q2) {
     cc.isUniversalClass() and q1 = before(cc) and lbl = Any() and q2 = after(cc)
     or
     q1 = before(cc) and
-    lbl =
-      CharacterClasses::normalize(CharClass(cc.getRawValue() + "|" +
-            getCanonicalizationFlags(cc.getRootTerm()))) and
+    lbl = CharacterClasses::normalize(CharClass(getCanonicalizationString(cc))) and
     q2 = after(cc)
   )
   or
   exists(RegExpTerm cc | isEscapeClass(cc, _) |
     q1 = before(cc) and
-    lbl =
-      CharacterClasses::normalize(CharClass(cc.getRawValue() + "|" +
-            getCanonicalizationFlags(cc.getRootTerm()))) and
+    lbl = CharacterClasses::normalize(CharClass(getCanonicalizationString(cc))) and
     q2 = after(cc)
   )
   or

javascript/ql/lib/semmle/javascript/security/regexp/PolynomialReDoSCustomizations.qll

@@ -90,7 +90,8 @@ module PolynomialReDoS {
         isCharClassLike(root)
       )
       or
-      this.(DataFlow::MethodCallNode).getMethodName() = StringOps::substringMethodName()
+      this.(DataFlow::MethodCallNode).getMethodName() = StringOps::substringMethodName() and
+      not this.(DataFlow::MethodCallNode).getNumArgument() = 1 // with one argument it just slices off the beginning
     }
   }
 

javascript/ql/lib/semmlecode.javascript.dbscheme

@@ -357,6 +357,7 @@ case @expr.kind of
 | 118 = @assignnullishcoalescingexpr
 | 119 = @template_pipe_ref
 | 120 = @generated_code_expr
+| 121 = @satisfies_expr
 ;
 
 @varaccess = @proper_varaccess | @export_varaccess;

javascript/ql/lib/semmlecode.javascript.dbscheme.stats

@@ -450,6 +450,10 @@
 <v>100</v>
 </e>
 <e>
+<k>@satisfies_expr</k>
+<v>100</v>
+</e>
+<e>
 <k>@preinc_expr</k>
 <v>1792</v>
 </e>