Merge branch 'main' into post-release-prep/codeql-cli-2.8.0

javascript/ql/examples/qlpack.yml

@@ -1,4 +1,6 @@
 name: codeql/javascript-examples
-version: 0.0.3
+groups: 
+  - javascript
+  - examples
 dependencies:
   codeql/javascript-all: "*"

javascript/ql/experimental/adaptivethreatmodeling/lib/experimental/adaptivethreatmodeling/FunctionBodyFeatures.qll

@@ -127,6 +127,18 @@ ASTNode getAnASTNodeWithAFeature(Function f) {
   result = getAnASTNodeToFeaturize(f)
 }
 
+/** Returns the number of source-code characters in a function. */
+int getNumCharsInFunction(Function f) {
+  result =
+    strictsum(ASTNode node | node = getAnASTNodeWithAFeature(f) | getTokenizedAstNode(node).length())
+}
+
+/**
+ * The maximum number of characters a feature can be.
+ * The evaluator string limit is 5395415 characters. We choose a limit lower than this.
+ */
+private int getMaxChars() { result = 1000000 }
+
 /**
  * Returns a featurized representation of the function that can be used to populate the
  * `enclosingFunctionBody` feature for an endpoint.
@@ -141,6 +153,9 @@ string getBodyTokensFeature(Function function) {
     node = getAnASTNodeToFeaturize(function) and
     exists(getTokenizedAstNode(node))
   ) <= 256 and
+  // Performance optimization: If a function has more than getMaxChars() characters in its body subtokens,
+  // then featurize it as absent.
+  getNumCharsInFunction(function) <= getMaxChars() and
   result =
     strictconcat(Location l, string token |
       // The use of a nested exists here allows us to avoid duplicates due to two AST nodes in the

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/DebugResultInclusion.ql

@@ -0,0 +1,67 @@
+/**
+ * @name Debug result inclusion
+ * @description Use this query to understand why some alerts are included or excluded from the
+ *              results of boosted queries. The results for this query are the union of the alerts
+ *              generated by each boosted query. Each alert includes an explanation why it was
+ *              included or excluded for each of the four security queries.
+ * @kind problem
+ * @problem.severity error
+ * @id adaptive-threat-modeling/js/debug-result-inclusion
+ */
+
+import javascript
+import experimental.adaptivethreatmodeling.ATMConfig
+import extraction.ExtractEndpointData
+
+string getAReasonSinkExcluded(DataFlow::Node sinkCandidate, Query query) {
+  query instanceof NosqlInjectionQuery and
+  result = NosqlInjectionATM::SinkEndpointFilter::getAReasonSinkExcluded(sinkCandidate)
+  or
+  query instanceof SqlInjectionQuery and
+  result = SqlInjectionATM::SinkEndpointFilter::getAReasonSinkExcluded(sinkCandidate)
+  or
+  query instanceof TaintedPathQuery and
+  result = TaintedPathATM::SinkEndpointFilter::getAReasonSinkExcluded(sinkCandidate)
+  or
+  query instanceof XssQuery and
+  result = XssATM::SinkEndpointFilter::getAReasonSinkExcluded(sinkCandidate)
+}
+
+pragma[inline]
+string getDescriptionForAlertCandidate(
+  DataFlow::Node sourceCandidate, DataFlow::Node sinkCandidate, Query query
+) {
+  result = "excluded[reason=" + getAReasonSinkExcluded(sinkCandidate, query) + "]"
+  or
+  getATMCfg(query).isKnownSink(sinkCandidate) and
+  result = "excluded[reason=known-sink]"
+  or
+  not exists(getAReasonSinkExcluded(sinkCandidate, query)) and
+  not getDataFlowCfg(query).hasFlow(sourceCandidate, sinkCandidate) and
+  (
+    if
+      getDataFlowCfg(query).isSource(sourceCandidate) or
+      getDataFlowCfg(query).isSource(sourceCandidate, _)
+    then result = "no flow"
+    else result = "not a known source"
+  )
+  or
+  getDataFlowCfg(query).hasFlow(sourceCandidate, sinkCandidate) and
+  result = "included"
+}
+
+pragma[inline]
+string getDescriptionForAlert(DataFlow::Node sourceCandidate, DataFlow::Node sinkCandidate) {
+  result =
+    concat(Query query |
+      |
+      query.getName() + ": " +
+          getDescriptionForAlertCandidate(sourceCandidate, sinkCandidate, query), ", "
+    )
+}
+
+from DataFlow::Configuration cfg, DataFlow::Node source, DataFlow::Node sink
+where cfg.hasFlow(source, sink)
+select sink,
+  "This is an ATM result that may depend on $@ [" + getDescriptionForAlert(source, sink) + "]",
+  source, "a user-provided value"

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/evaluation/EndToEndEvaluation.qll

@@ -0,0 +1,11 @@
+private import javascript
+private import extraction.Exclusions as Exclusions
+
+/**
+ * Holds if the flow from `source` to `sink` should be excluded from the results of an end-to-end
+ * evaluation query.
+ */
+pragma[inline]
+predicate isFlowExcluded(DataFlow::Node source, DataFlow::Node sink) {
+  Exclusions::isFileExcluded([source.getFile(), sink.getFile()])
+}

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/evaluation/EndpointScoresIntegrationTest.ql

@@ -0,0 +1,27 @@
+/**
+ * EndpointScoresIntegrationTest.ql
+ *
+ * Extract scores for each test endpoint that is an argument to a function call in the database.
+ * This is used by integration tests to verify that QL and the modeling codebase agree on the scores
+ * of a set of test endpoints.
+ */
+
+import javascript
+import experimental.adaptivethreatmodeling.ATMConfig
+import experimental.adaptivethreatmodeling.FeaturizationConfig
+import experimental.adaptivethreatmodeling.EndpointScoring::ModelScoring as ModelScoring
+
+/**
+ * A featurization config that featurizes endpoints that are arguments to function calls.
+ *
+ * This should only be used in extraction queries and tests.
+ */
+class FunctionArgumentFeaturizationConfig extends FeaturizationConfig {
+  FunctionArgumentFeaturizationConfig() { this = "FunctionArgumentFeaturization" }
+
+  override DataFlow::Node getAnEndpointToFeaturize() {
+    exists(DataFlow::CallNode call | result = call.getAnArgument())
+  }
+}
+
+query predicate endpointScores = ModelScoring::endpointScores/3;

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/evaluation/ModelCheck.ql

@@ -0,0 +1,16 @@
+/**
+ * ModelCheck.ql
+ *
+ * Returns checksums of ATM models.
+ */
+
+/**
+ * The `availableMlModels` template predicate.
+ *
+ * This is populated by the evaluator with metadata for the available machine learning models.
+ */
+external predicate availableMlModels(
+  string modelChecksum, string modelLanguage, string modelName, string modelType
+);
+
+select any(string checksum | availableMlModels(checksum, "javascript", _, _))

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/evaluation/NosqlInjection.ql

@@ -0,0 +1,24 @@
+/**
+ * NosqlInjection.ql
+ *
+ * Version of the standard NoSQL injection query with an output relation ready to plug into the
+ * evaluation pipeline.
+ */
+
+import semmle.javascript.security.dataflow.NosqlInjection
+import EndToEndEvaluation as EndToEndEvaluation
+
+from
+  DataFlow::Configuration cfg, DataFlow::Node source, DataFlow::Node sink, string filePathSink,
+  int startLineSink, int endLineSink, int startColumnSink, int endColumnSink, string filePathSource,
+  int startLineSource, int endLineSource, int startColumnSource, int endColumnSource
+where
+  cfg instanceof NosqlInjection::Configuration and
+  cfg.hasFlow(source, sink) and
+  not EndToEndEvaluation::isFlowExcluded(source, sink) and
+  sink.hasLocationInfo(filePathSink, startLineSink, startColumnSink, endLineSink, endColumnSink) and
+  source
+      .hasLocationInfo(filePathSource, startLineSource, startColumnSource, endLineSource,
+        endColumnSource)
+select source, startLineSource, startColumnSource, endLineSource, endColumnSource, filePathSource,
+  sink, startLineSink, startColumnSink, endLineSink, endColumnSink, filePathSink

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/evaluation/NosqlInjectionATM.ql

@@ -0,0 +1,28 @@
+/**
+ * NosqlInjectionATM.ql
+ *
+ * Version of the boosted NoSQL injection query with an output relation ready to plug into the
+ * evaluation pipeline.
+ */
+
+import ATM::ResultsInfo
+import EndToEndEvaluation as EndToEndEvaluation
+import experimental.adaptivethreatmodeling.NosqlInjectionATM
+
+from
+  DataFlow::Configuration cfg, DataFlow::Node source, DataFlow::Node sink, string filePathSink,
+  int startLineSink, int endLineSink, int startColumnSink, int endColumnSink, string filePathSource,
+  int startLineSource, int endLineSource, int startColumnSource, int endColumnSource, float score
+where
+  cfg.hasFlow(source, sink) and
+  not EndToEndEvaluation::isFlowExcluded(source, sink) and
+  not isFlowLikelyInBaseQuery(source, sink) and
+  sink.hasLocationInfo(filePathSink, startLineSink, startColumnSink, endLineSink, endColumnSink) and
+  source
+      .hasLocationInfo(filePathSource, startLineSource, startColumnSource, endLineSource,
+        endColumnSource) and
+  getScoreForFlow(source, sink) = score
+select source, startLineSource, startColumnSource, endLineSource, endColumnSource, filePathSource,
+  sink, startLineSink, startColumnSink, endLineSink, endColumnSink, filePathSink, score order by
+    score desc, startLineSource, startColumnSource, endLineSource, endColumnSource, filePathSource,
+    startLineSink, startColumnSink, endLineSink, endColumnSink, filePathSink

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/evaluation/NosqlInjectionATMLite.ql

@@ -0,0 +1,29 @@
+/**
+ * NosqlInjectionATMLite.ql
+ *
+ * Arbitrarily ranked version of the boosted NoSQL injection query with an output relation ready to
+ * plug into the evaluation pipeline. This is useful (a) for evaluating the performance of endpoint
+ * filters, and (b) as a baseline to compare the model against.
+ */
+
+import ATM::ResultsInfo
+import EndToEndEvaluation as EndToEndEvaluation
+import experimental.adaptivethreatmodeling.NosqlInjectionATM
+
+from
+  DataFlow::Configuration cfg, DataFlow::Node source, DataFlow::Node sink, string filePathSink,
+  int startLineSink, int endLineSink, int startColumnSink, int endColumnSink, string filePathSource,
+  int startLineSource, int endLineSource, int startColumnSource, int endColumnSource, float score
+where
+  cfg.hasFlow(source, sink) and
+  not EndToEndEvaluation::isFlowExcluded(source, sink) and
+  not isFlowLikelyInBaseQuery(source, sink) and
+  sink.hasLocationInfo(filePathSink, startLineSink, startColumnSink, endLineSink, endColumnSink) and
+  source
+      .hasLocationInfo(filePathSource, startLineSource, startColumnSource, endLineSource,
+        endColumnSource) and
+  score = 0
+select source, startLineSource, startColumnSource, endLineSource, endColumnSource, filePathSource,
+  sink, startLineSink, startColumnSink, endLineSink, endColumnSink, filePathSink, score order by
+    score desc, startLineSource, startColumnSource, endLineSource, endColumnSource, filePathSource,
+    startLineSink, startColumnSink, endLineSink, endColumnSink, filePathSink

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/evaluation/SqlInjection.ql

@@ -0,0 +1,24 @@
+/**
+ * SqlInjection.ql
+ *
+ * Version of the standard SQL injection query with an output relation ready to plug into the
+ * evaluation pipeline.
+ */
+
+import semmle.javascript.security.dataflow.SqlInjection
+import EndToEndEvaluation as EndToEndEvaluation
+
+from
+  DataFlow::Configuration cfg, DataFlow::Node source, DataFlow::Node sink, string filePathSink,
+  int startLineSink, int endLineSink, int startColumnSink, int endColumnSink, string filePathSource,
+  int startLineSource, int endLineSource, int startColumnSource, int endColumnSource
+where
+  cfg instanceof SqlInjection::Configuration and
+  cfg.hasFlow(source, sink) and
+  not EndToEndEvaluation::isFlowExcluded(source, sink) and
+  sink.hasLocationInfo(filePathSink, startLineSink, startColumnSink, endLineSink, endColumnSink) and
+  source
+      .hasLocationInfo(filePathSource, startLineSource, startColumnSource, endLineSource,
+        endColumnSource)
+select source, startLineSource, startColumnSource, endLineSource, endColumnSource, filePathSource,
+  sink, startLineSink, startColumnSink, endLineSink, endColumnSink, filePathSink

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/evaluation/SqlInjectionATM.ql

@@ -0,0 +1,28 @@
+/**
+ * SqlInjectionATM.ql
+ *
+ * Version of the boosted SQL injection query with an output relation ready to plug into the
+ * evaluation pipeline.
+ */
+
+import ATM::ResultsInfo
+import EndToEndEvaluation as EndToEndEvaluation
+import experimental.adaptivethreatmodeling.SqlInjectionATM
+
+from
+  DataFlow::Configuration cfg, DataFlow::Node source, DataFlow::Node sink, string filePathSink,
+  int startLineSink, int endLineSink, int startColumnSink, int endColumnSink, string filePathSource,
+  int startLineSource, int endLineSource, int startColumnSource, int endColumnSource, float score
+where
+  cfg.hasFlow(source, sink) and
+  not EndToEndEvaluation::isFlowExcluded(source, sink) and
+  not isFlowLikelyInBaseQuery(source, sink) and
+  sink.hasLocationInfo(filePathSink, startLineSink, startColumnSink, endLineSink, endColumnSink) and
+  source
+      .hasLocationInfo(filePathSource, startLineSource, startColumnSource, endLineSource,
+        endColumnSource) and
+  getScoreForFlow(source, sink) = score
+select source, startLineSource, startColumnSource, endLineSource, endColumnSource, filePathSource,
+  sink, startLineSink, startColumnSink, endLineSink, endColumnSink, filePathSink, score order by
+    score desc, startLineSource, startColumnSource, endLineSource, endColumnSource, filePathSource,
+    startLineSink, startColumnSink, endLineSink, endColumnSink, filePathSink

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/evaluation/SqlInjectionATMLite.ql

@@ -0,0 +1,29 @@
+/**
+ * SqlInjectionATMLite.ql
+ *
+ * Arbitrarily ranked version of the boosted SQL injection query with an output relation ready to
+ * plug into the evaluation pipeline. This is useful (a) for evaluating the performance of endpoint
+ * filters, and (b) as a baseline to compare the model against.
+ */
+
+import ATM::ResultsInfo
+import EndToEndEvaluation as EndToEndEvaluation
+import experimental.adaptivethreatmodeling.SqlInjectionATM
+
+from
+  DataFlow::Configuration cfg, DataFlow::Node source, DataFlow::Node sink, string filePathSink,
+  int startLineSink, int endLineSink, int startColumnSink, int endColumnSink, string filePathSource,
+  int startLineSource, int endLineSource, int startColumnSource, int endColumnSource, float score
+where
+  cfg.hasFlow(source, sink) and
+  not EndToEndEvaluation::isFlowExcluded(source, sink) and
+  not isFlowLikelyInBaseQuery(source, sink) and
+  sink.hasLocationInfo(filePathSink, startLineSink, startColumnSink, endLineSink, endColumnSink) and
+  source
+      .hasLocationInfo(filePathSource, startLineSource, startColumnSource, endLineSource,
+        endColumnSource) and
+  score = 0
+select source, startLineSource, startColumnSource, endLineSource, endColumnSource, filePathSource,
+  sink, startLineSink, startColumnSink, endLineSink, endColumnSink, filePathSink, score order by
+    score desc, startLineSource, startColumnSource, endLineSource, endColumnSource, filePathSource,
+    startLineSink, startColumnSink, endLineSink, endColumnSink, filePathSink

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/evaluation/TaintedPath.ql

@@ -0,0 +1,24 @@
+/**
+ * TaintedPath.ql
+ *
+ * Version of the standard path injection query with an output relation ready to plug into the
+ * evaluation pipeline.
+ */
+
+import semmle.javascript.security.dataflow.TaintedPath
+import EndToEndEvaluation as EndToEndEvaluation
+
+from
+  DataFlow::Configuration cfg, DataFlow::Node source, DataFlow::Node sink, string filePathSink,
+  int startLineSink, int endLineSink, int startColumnSink, int endColumnSink, string filePathSource,
+  int startLineSource, int endLineSource, int startColumnSource, int endColumnSource
+where
+  cfg instanceof TaintedPath::Configuration and
+  cfg.hasFlow(source, sink) and
+  not EndToEndEvaluation::isFlowExcluded(source, sink) and
+  sink.hasLocationInfo(filePathSink, startLineSink, startColumnSink, endLineSink, endColumnSink) and
+  source
+      .hasLocationInfo(filePathSource, startLineSource, startColumnSource, endLineSource,
+        endColumnSource)
+select source, startLineSource, startColumnSource, endLineSource, endColumnSource, filePathSource,
+  sink, startLineSink, startColumnSink, endLineSink, endColumnSink, filePathSink

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/evaluation/TaintedPathATM.ql

@@ -0,0 +1,28 @@
+/**
+ * TaintedPathATM.ql
+ *
+ * Version of the boosted path injection query with an output relation ready to plug into the
+ * evaluation pipeline.
+ */
+
+import ATM::ResultsInfo
+import EndToEndEvaluation as EndToEndEvaluation
+import experimental.adaptivethreatmodeling.TaintedPathATM
+
+from
+  DataFlow::Configuration cfg, DataFlow::Node source, DataFlow::Node sink, string filePathSink,
+  int startLineSink, int endLineSink, int startColumnSink, int endColumnSink, string filePathSource,
+  int startLineSource, int endLineSource, int startColumnSource, int endColumnSource, float score
+where
+  cfg.hasFlow(source, sink) and
+  not EndToEndEvaluation::isFlowExcluded(source, sink) and
+  not isFlowLikelyInBaseQuery(source, sink) and
+  sink.hasLocationInfo(filePathSink, startLineSink, startColumnSink, endLineSink, endColumnSink) and
+  source
+      .hasLocationInfo(filePathSource, startLineSource, startColumnSource, endLineSource,
+        endColumnSource) and
+  getScoreForFlow(source, sink) = score
+select source, startLineSource, startColumnSource, endLineSource, endColumnSource, filePathSource,
+  sink, startLineSink, startColumnSink, endLineSink, endColumnSink, filePathSink, score order by
+    score desc, startLineSource, startColumnSource, endLineSource, endColumnSource, filePathSource,
+    startLineSink, startColumnSink, endLineSink, endColumnSink, filePathSink

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/evaluation/TaintedPathATMLite.ql

@@ -0,0 +1,29 @@
+/**
+ * TaintedPathATMLite.ql
+ *
+ * Arbitrarily ranked version of the boosted path injection query with an output relation ready to
+ * plug into the evaluation pipeline. This is useful (a) for evaluating the performance of endpoint
+ * filters, and (b) as a baseline to compare the model against.
+ */
+
+import ATM::ResultsInfo
+import EndToEndEvaluation as EndToEndEvaluation
+import experimental.adaptivethreatmodeling.TaintedPathATM
+
+from
+  DataFlow::Configuration cfg, DataFlow::Node source, DataFlow::Node sink, string filePathSink,
+  int startLineSink, int endLineSink, int startColumnSink, int endColumnSink, string filePathSource,
+  int startLineSource, int endLineSource, int startColumnSource, int endColumnSource, float score
+where
+  cfg.hasFlow(source, sink) and
+  not EndToEndEvaluation::isFlowExcluded(source, sink) and
+  not isFlowLikelyInBaseQuery(source, sink) and
+  sink.hasLocationInfo(filePathSink, startLineSink, startColumnSink, endLineSink, endColumnSink) and
+  source
+      .hasLocationInfo(filePathSource, startLineSource, startColumnSource, endLineSource,
+        endColumnSource) and
+  score = 0
+select source, startLineSource, startColumnSource, endLineSource, endColumnSource, filePathSource,
+  sink, startLineSink, startColumnSink, endLineSink, endColumnSink, filePathSink, score order by
+    score desc, startLineSource, startColumnSource, endLineSource, endColumnSource, filePathSource,
+    startLineSink, startColumnSink, endLineSink, endColumnSink, filePathSink

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/evaluation/Xss.ql

@@ -0,0 +1,24 @@
+/**
+ * Xss.ql
+ *
+ * Version of the standard XSS query with an output relation ready to plug into the evaluation
+ * pipeline.
+ */
+
+import semmle.javascript.security.dataflow.DomBasedXss
+import EndToEndEvaluation as EndToEndEvaluation
+
+from
+  DataFlow::Configuration cfg, DataFlow::Node source, DataFlow::Node sink, string filePathSink,
+  int startLineSink, int endLineSink, int startColumnSink, int endColumnSink, string filePathSource,
+  int startLineSource, int endLineSource, int startColumnSource, int endColumnSource
+where
+  cfg instanceof DomBasedXss::Configuration and
+  cfg.hasFlow(source, sink) and
+  not EndToEndEvaluation::isFlowExcluded(source, sink) and
+  sink.hasLocationInfo(filePathSink, startLineSink, startColumnSink, endLineSink, endColumnSink) and
+  source
+      .hasLocationInfo(filePathSource, startLineSource, startColumnSource, endLineSource,
+        endColumnSource)
+select source, startLineSource, startColumnSource, endLineSource, endColumnSource, filePathSource,
+  sink, startLineSink, startColumnSink, endLineSink, endColumnSink, filePathSink

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/evaluation/XssATM.ql

@@ -0,0 +1,29 @@
+/**
+ * XssATM.ql
+ *
+ * Version of the boosted XSS query with an output relation ready to plug into the evaluation
+ * pipeline.
+ */
+
+import javascript
+import ATM::ResultsInfo
+import EndToEndEvaluation as EndToEndEvaluation
+import experimental.adaptivethreatmodeling.XssATM
+
+from
+  DataFlow::Configuration cfg, DataFlow::Node source, DataFlow::Node sink, string filePathSink,
+  int startLineSink, int endLineSink, int startColumnSink, int endColumnSink, string filePathSource,
+  int startLineSource, int endLineSource, int startColumnSource, int endColumnSource, float score
+where
+  cfg.hasFlow(source, sink) and
+  not EndToEndEvaluation::isFlowExcluded(source, sink) and
+  not isFlowLikelyInBaseQuery(source, sink) and
+  sink.hasLocationInfo(filePathSink, startLineSink, startColumnSink, endLineSink, endColumnSink) and
+  source
+      .hasLocationInfo(filePathSource, startLineSource, startColumnSource, endLineSource,
+        endColumnSource) and
+  getScoreForFlow(source, sink) = score
+select source, startLineSource, startColumnSource, endLineSource, endColumnSource, filePathSource,
+  sink, startLineSink, startColumnSink, endLineSink, endColumnSink, filePathSink, score order by
+    score desc, startLineSource, startColumnSource, endLineSource, endColumnSource, filePathSource,
+    startLineSink, startColumnSink, endLineSink, endColumnSink, filePathSink

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/evaluation/XssATMLite.ql

@@ -0,0 +1,30 @@
+/**
+ * XssATMLite.ql
+ *
+ * Arbitrarily ranked version of the boosted XSS query with an output relation ready to plug into
+ * the evaluation pipeline. This is useful (a) for evaluating the performance of endpoint filters,
+ * and (b) as a baseline to compare the model against.
+ */
+
+import javascript
+import ATM::ResultsInfo
+import EndToEndEvaluation as EndToEndEvaluation
+import experimental.adaptivethreatmodeling.XssATM
+
+from
+  DataFlow::Configuration cfg, DataFlow::Node source, DataFlow::Node sink, string filePathSink,
+  int startLineSink, int endLineSink, int startColumnSink, int endColumnSink, string filePathSource,
+  int startLineSource, int endLineSource, int startColumnSource, int endColumnSource, float score
+where
+  cfg.hasFlow(source, sink) and
+  not EndToEndEvaluation::isFlowExcluded(source, sink) and
+  not isFlowLikelyInBaseQuery(source, sink) and
+  sink.hasLocationInfo(filePathSink, startLineSink, startColumnSink, endLineSink, endColumnSink) and
+  source
+      .hasLocationInfo(filePathSource, startLineSource, startColumnSource, endLineSource,
+        endColumnSource) and
+  score = 0
+select source, startLineSource, startColumnSource, endLineSource, endColumnSource, filePathSource,
+  sink, startLineSink, startColumnSink, endLineSink, endColumnSink, filePathSink, score order by
+    score desc, startLineSource, startColumnSource, endLineSource, endColumnSource, filePathSource,
+    startLineSink, startColumnSink, endLineSink, endColumnSink, filePathSink

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/extraction/CountAlertsAndEndpoints.ql

@@ -0,0 +1,26 @@
+/*
+ * For internal use only.
+ *
+ * [DEPRECATED] Counts alerts and sinks for JavaScript security queries.
+ *
+ * This query is deprecated due to the performance implications of bringing in data flow
+ * configurations from multiple queries. Instead use `CountSourcesAndSinks.ql` to count sinks for
+ * JavaScript security queries, and count alerts by running the standard or evaluation queries for
+ * each security vulnerability.
+ */
+
+import semmle.javascript.security.dataflow.NosqlInjection
+import semmle.javascript.security.dataflow.SqlInjection
+import semmle.javascript.security.dataflow.TaintedPath
+import semmle.javascript.security.dataflow.DomBasedXss
+
+int numAlerts(DataFlow::Configuration cfg) {
+  result = count(DataFlow::Node source, DataFlow::Node sink | cfg.hasFlow(source, sink))
+}
+
+select numAlerts(any(NosqlInjection::Configuration cfg)) as numNosqlAlerts,
+  numAlerts(any(SqlInjection::Configuration cfg)) as numSqlAlerts,
+  numAlerts(any(TaintedPath::Configuration cfg)) as numTaintedPathAlerts,
+  numAlerts(any(DomBasedXss::Configuration cfg)) as numXssAlerts,
+  count(NosqlInjection::Sink sink) as numNosqlSinks, count(SqlInjection::Sink sink) as numSqlSinks,
+  count(TaintedPath::Sink sink) as numTaintedPathSinks, count(DomBasedXss::Sink sink) as numXssSinks

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/extraction/CountSourcesAndSinks.ql

@@ -0,0 +1,72 @@
+/*
+ * For internal use only.
+ *
+ * Counts sources and sinks for JavaScript security queries.
+ */
+
+import javascript
+import semmle.javascript.dataflow.Configuration
+// javascript/ql/lib/semmle/javascript/security/dataflow$ ls *Query.qll | sed -e 's/\(.*\)Query.qll/import semmle.javascript.security.dataflow.\1Query as \1/'
+import semmle.javascript.security.dataflow.BrokenCryptoAlgorithmQuery as BrokenCryptoAlgorithm
+import semmle.javascript.security.dataflow.BuildArtifactLeakQuery as BuildArtifactLeak
+import semmle.javascript.security.dataflow.CleartextLoggingQuery as CleartextLogging
+import semmle.javascript.security.dataflow.CleartextStorageQuery as CleartextStorage
+import semmle.javascript.security.dataflow.ClientSideUrlRedirectQuery as ClientSideUrlRedirect
+import semmle.javascript.security.dataflow.CodeInjectionQuery as CodeInjection
+import semmle.javascript.security.dataflow.CommandInjectionQuery as CommandInjection
+import semmle.javascript.security.dataflow.ConditionalBypassQuery as ConditionalBypass
+import semmle.javascript.security.dataflow.CorsMisconfigurationForCredentialsQuery as CorsMisconfigurationForCredentials
+import semmle.javascript.security.dataflow.DeepObjectResourceExhaustionQuery as DeepObjectResourceExhaustion
+import semmle.javascript.security.dataflow.DifferentKindsComparisonBypassQuery as DifferentKindsComparisonBypass
+import semmle.javascript.security.dataflow.DomBasedXssQuery as DomBasedXss
+import semmle.javascript.security.dataflow.ExceptionXssQuery as ExceptionXss
+import semmle.javascript.security.dataflow.ExternalAPIUsedWithUntrustedDataQuery as ExternalAPIUsedWithUntrustedData
+import semmle.javascript.security.dataflow.FileAccessToHttpQuery as FileAccessToHttp
+import semmle.javascript.security.dataflow.HardcodedCredentialsQuery as HardcodedCredentials
+import semmle.javascript.security.dataflow.HardcodedDataInterpretedAsCodeQuery as HardcodedDataInterpretedAsCode
+import semmle.javascript.security.dataflow.HostHeaderPoisoningInEmailGenerationQuery as HostHeaderPoisoningInEmailGeneration
+import semmle.javascript.security.dataflow.HttpToFileAccessQuery as HttpToFileAccess
+import semmle.javascript.security.dataflow.ImproperCodeSanitizationQuery as ImproperCodeSanitization
+import semmle.javascript.security.dataflow.IncompleteHtmlAttributeSanitizationQuery as IncompleteHtmlAttributeSanitization
+import semmle.javascript.security.dataflow.IndirectCommandInjectionQuery as IndirectCommandInjection
+import semmle.javascript.security.dataflow.InsecureDownloadQuery as InsecureDownload
+import semmle.javascript.security.dataflow.InsecureRandomnessQuery as InsecureRandomness
+import semmle.javascript.security.dataflow.InsufficientPasswordHashQuery as InsufficientPasswordHash
+import semmle.javascript.security.dataflow.LogInjectionQuery as LogInjection
+import semmle.javascript.security.dataflow.LoopBoundInjectionQuery as LoopBoundInjection
+import semmle.javascript.security.dataflow.NosqlInjectionQuery as NosqlInjection
+import semmle.javascript.security.dataflow.PostMessageStarQuery as PostMessageStar
+import semmle.javascript.security.dataflow.PrototypePollutingAssignmentQuery as PrototypePollutingAssignment
+import semmle.javascript.security.dataflow.PrototypePollutionQuery as PrototypePollution
+import semmle.javascript.security.dataflow.ReflectedXssQuery as ReflectedXss
+import semmle.javascript.security.dataflow.RegExpInjectionQuery as RegExpInjection
+import semmle.javascript.security.dataflow.RemotePropertyInjectionQuery as RemotePropertyInjection
+import semmle.javascript.security.dataflow.RequestForgeryQuery as RequestForgery
+import semmle.javascript.security.dataflow.ServerSideUrlRedirectQuery as ServerSideUrlRedirect
+import semmle.javascript.security.dataflow.ShellCommandInjectionFromEnvironmentQuery as ShellCommandInjectionFromEnvironment
+import semmle.javascript.security.dataflow.SqlInjectionQuery as SqlInjection
+import semmle.javascript.security.dataflow.StackTraceExposureQuery as StackTraceExposure
+import semmle.javascript.security.dataflow.StoredXssQuery as StoredXss
+import semmle.javascript.security.dataflow.TaintedFormatStringQuery as TaintedFormatString
+import semmle.javascript.security.dataflow.TaintedPathQuery as TaintedPath
+import semmle.javascript.security.dataflow.TemplateObjectInjectionQuery as TemplateObjectInjection
+import semmle.javascript.security.dataflow.TypeConfusionThroughParameterTamperingQuery as TypeConfusionThroughParameterTampering
+import semmle.javascript.security.dataflow.UnsafeDeserializationQuery as UnsafeDeserialization
+import semmle.javascript.security.dataflow.UnsafeDynamicMethodAccessQuery as UnsafeDynamicMethodAccess
+import semmle.javascript.security.dataflow.UnsafeHtmlConstructionQuery as UnsafeHtmlConstruction
+import semmle.javascript.security.dataflow.UnsafeJQueryPluginQuery as UnsafeJQueryPlugin
+import semmle.javascript.security.dataflow.UnsafeShellCommandConstructionQuery as UnsafeShellCommandConstruction
+import semmle.javascript.security.dataflow.UnvalidatedDynamicMethodCallQuery as UnvalidatedDynamicMethodCall
+import semmle.javascript.security.dataflow.XmlBombQuery as XmlBomb
+import semmle.javascript.security.dataflow.XpathInjectionQuery as XpathInjection
+import semmle.javascript.security.dataflow.XssThroughDomQuery as XssThroughDom
+import semmle.javascript.security.dataflow.XxeQuery as Xxe
+import semmle.javascript.security.dataflow.ZipSlipQuery as ZipSlip
+
+DataFlow::Node getASink(Configuration cfg) { cfg.isSink(result) or cfg.isSink(result, _) }
+
+DataFlow::Node getASource(Configuration cfg) { cfg.isSource(result) or cfg.isSource(result, _) }
+
+from Configuration cfg, int sources, int sinks
+where count(getASource(cfg)) = sources and count(getASink(cfg)) = sinks
+select cfg, sources, sinks

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/extraction/Exclusions.qll

@@ -0,0 +1,49 @@
+/*
+ * For internal use only.
+ *
+ * Defines files that should be excluded from the evaluation of ML models.
+ */
+
+private import javascript
+private import semmle.javascript.filters.ClassifyFiles as ClassifyFiles
+
+/** Holds if the file should be excluded from end-to-end evaluation. */
+predicate isFileExcluded(File file) {
+  // Ignore files that are outside the root folder of the analyzed source location.
+  //
+  // If the file doesn't have a relative path, then the source file is located outside the root
+  // folder of the analyzed source location, meaning that the files are additional files added to
+  // the database like standard library files that we would like to ignore.
+  not exists(file.getRelativePath())
+  or
+  // Ignore files based on their path.
+  exists(string ignorePattern, string separator |
+    ignorePattern =
+      // Exclude test files
+      "(tests?|test[_-]?case|" +
+        // Exclude library files
+        //
+        // - The Bower and npm package managers store packages in bower_components and node_modules
+        // folders respectively.
+        // - Specific exclusion for end-to-end: `applications/examples/static/epydoc` contains
+        //   library code from Epydoc.
+        "3rd[_-]?party|bower_components|extern(s|al)?|node_modules|resources|third[_-]?party|_?vendor|"
+        + "applications" + separator + "examples" + separator + "static" + separator + "epydoc|" +
+        // Exclude generated code
+        "gen|\\.?generated|" +
+        // Exclude benchmarks
+        "benchmarks?|" +
+        // Exclude documentation
+        "docs?|documentation)" and
+    separator = "(\\/|\\.)" and
+    exists(
+      file.getRelativePath()
+          .toLowerCase()
+          .regexpFind(separator + ignorePattern + separator + "|" + "^" + ignorePattern + separator +
+              "|" + separator + ignorePattern + "$", _, _)
+    )
+  )
+  or
+  // Ignore externs, generated, library, and test files.
+  ClassifyFiles::classify(file, ["externs", "generated", "library", "test"])
+}

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/extraction/ExtractEndpointData.ql

@@ -0,0 +1,11 @@
+/*
+ * For internal use only.
+ *
+ * Extracts training and evaluation data we can use to train ML models for ML-powered queries.
+ */
+
+import ExtractEndpointData as ExtractEndpointData
+
+query predicate endpoints = ExtractEndpointData::endpoints/5;
+
+query predicate tokenFeatures = ExtractEndpointData::tokenFeatures/3;

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/extraction/ExtractEndpointData.qll

@@ -0,0 +1,195 @@
+/*
+ * For internal use only.
+ *
+ * Library code for training and evaluation data we can use to train ML models for ML-powered
+ * queries.
+ */
+
+import javascript
+import Exclusions as Exclusions
+import evaluation.EndToEndEvaluation as EndToEndEvaluation
+import experimental.adaptivethreatmodeling.ATMConfig
+import experimental.adaptivethreatmodeling.CoreKnowledge as CoreKnowledge
+import experimental.adaptivethreatmodeling.EndpointFeatures as EndpointFeatures
+import experimental.adaptivethreatmodeling.EndpointScoring as EndpointScoring
+import experimental.adaptivethreatmodeling.EndpointTypes
+import experimental.adaptivethreatmodeling.FilteringReasons
+import experimental.adaptivethreatmodeling.NosqlInjectionATM as NosqlInjectionATM
+import experimental.adaptivethreatmodeling.SqlInjectionATM as SqlInjectionATM
+import experimental.adaptivethreatmodeling.TaintedPathATM as TaintedPathATM
+import experimental.adaptivethreatmodeling.XssATM as XssATM
+import Labels
+import NoFeaturizationRestrictionsConfig
+import Queries
+
+/** Gets the ATM configuration object for the specified query. */
+ATMConfig getATMCfg(Query query) {
+  query instanceof NosqlInjectionQuery and
+  result instanceof NosqlInjectionATM::NosqlInjectionATMConfig
+  or
+  query instanceof SqlInjectionQuery and result instanceof SqlInjectionATM::SqlInjectionATMConfig
+  or
+  query instanceof TaintedPathQuery and result instanceof TaintedPathATM::TaintedPathATMConfig
+  or
+  query instanceof XssQuery and result instanceof XssATM::DomBasedXssATMConfig
+}
+
+/** Gets the ATM data flow configuration for the specified query. */
+DataFlow::Configuration getDataFlowCfg(Query query) {
+  query instanceof NosqlInjectionQuery and result instanceof NosqlInjectionATM::Configuration
+  or
+  query instanceof SqlInjectionQuery and result instanceof SqlInjectionATM::Configuration
+  or
+  query instanceof TaintedPathQuery and result instanceof TaintedPathATM::Configuration
+  or
+  query instanceof XssQuery and result instanceof XssATM::Configuration
+}
+
+/** Gets a known sink for the specified query. */
+private DataFlow::Node getASink(Query query) {
+  getATMCfg(query).isKnownSink(result) and
+  // Only consider the source code for the project being analyzed.
+  exists(result.getFile().getRelativePath())
+}
+
+/** Gets a data flow node that is known not to be a sink for the specified query. */
+private DataFlow::Node getANotASink(NotASinkReason reason) {
+  CoreKnowledge::isOtherModeledArgument(result, reason) and
+  // Some endpoints can be assigned both a `NotASinkReason` and a `LikelyNotASinkReason`. We
+  // consider these endpoints to be `LikelyNotASink`, therefore this line excludes them from the
+  // definition of `NotASink`.
+  not CoreKnowledge::isOtherModeledArgument(result, any(LikelyNotASinkReason t)) and
+  not result = getASink(_) and
+  // Only consider the source code for the project being analyzed.
+  exists(result.getFile().getRelativePath())
+}
+
+/**
+ * Gets a data flow node whose label is unknown for the specified query.
+ *
+ * In other words, this is an endpoint that is not `Sink`, `NotASink`, or `LikelyNotASink` for the
+ * specified query.
+ */
+private DataFlow::Node getAnUnknown(Query query) {
+  (
+    getATMCfg(query).isEffectiveSink(result) or
+    getATMCfg(query).isEffectiveSinkWithOverridingScore(result, _, _)
+  ) and
+  not result = getASink(query) and
+  // Only consider the source code for the project being analyzed.
+  exists(result.getFile().getRelativePath())
+}
+
+/** Gets the query-specific sink label for the given endpoint, if such a label exists. */
+private EndpointLabel getSinkLabelForEndpoint(DataFlow::Node endpoint, Query query) {
+  endpoint = getASink(query) and result instanceof SinkLabel
+  or
+  endpoint = getANotASink(_) and result instanceof NotASinkLabel
+  or
+  endpoint = getAnUnknown(query) and result instanceof UnknownLabel
+}
+
+/** Gets an endpoint that should be extracted. */
+DataFlow::Node getAnEndpoint(Query query) { exists(getSinkLabelForEndpoint(result, query)) }
+
+/**
+ * Endpoints and associated metadata.
+ *
+ * Note that we draw a distinction between _features_, that are provided to the model at training
+ * and query time, and _metadata_, that is only provided to the model at training time.
+ *
+ * Internal: See the design document for
+ * [extensible extraction queries](https://docs.google.com/document/d/1g3ci2Nf1hGMG6ZUP0Y4PqCy_8elcoC_dhBvgTxdAWpg)
+ * for technical information about the design of this predicate.
+ */
+predicate endpoints(
+  DataFlow::Node endpoint, string queryName, string key, string value, string valueType
+) {
+  exists(Query query |
+    // Only provide metadata for labelled endpoints, since we do not extract all endpoints.
+    endpoint = getAnEndpoint(query) and
+    queryName = query.getName() and
+    (
+      // Holds if there is a taint flow path from a known source to the endpoint
+      key = "hasFlowFromSource" and
+      (
+        if FlowFromSource::hasFlowFromSource(endpoint, query)
+        then value = "true"
+        else value = "false"
+      ) and
+      valueType = "boolean"
+      or
+      // Constant expressions always evaluate to a constant primitive value. Therefore they can't ever
+      // appear in an alert, making them less interesting training examples.
+      key = "isConstantExpression" and
+      (if endpoint.asExpr() instanceof ConstantExpr then value = "true" else value = "false") and
+      valueType = "boolean"
+      or
+      // Holds if alerts involving the endpoint are excluded from the end-to-end evaluation.
+      key = "isExcludedFromEndToEndEvaluation" and
+      (if Exclusions::isFileExcluded(endpoint.getFile()) then value = "true" else value = "false") and
+      valueType = "boolean"
+      or
+      // The label for this query, considering the endpoint as a sink.
+      key = "sinkLabel" and
+      value = getSinkLabelForEndpoint(endpoint, query).getEncoding() and
+      valueType = "string"
+      or
+      // The reason, or reasons, why the endpoint was labeled NotASink for this query.
+      key = "notASinkReason" and
+      exists(FilteringReason reason |
+        endpoint = getANotASink(reason) and
+        value = reason.getDescription()
+      ) and
+      valueType = "string"
+    )
+  )
+}
+
+/**
+ * `EndpointFeatures::tokenFeatures` has no results when `featureName` is absent for the endpoint
+ * `endpoint`. To preserve compatibility with the data pipeline, this relation will instead set
+ * `featureValue` to the empty string in this case.
+ */
+predicate tokenFeatures(DataFlow::Node endpoint, string featureName, string featureValue) {
+  endpoints(endpoint, _, _, _, _) and
+  (
+    EndpointFeatures::tokenFeatures(endpoint, featureName, featureValue)
+    or
+    // Performance note: this creates a Cartesian product between `endpoint` and `featureName`.
+    featureName = EndpointFeatures::getASupportedFeatureName() and
+    not exists(string value | EndpointFeatures::tokenFeatures(endpoint, featureName, value)) and
+    featureValue = ""
+  )
+}
+
+module FlowFromSource {
+  predicate hasFlowFromSource(DataFlow::Node endpoint, Query q) {
+    exists(Configuration cfg | cfg.getQuery() = q | cfg.hasFlow(_, endpoint))
+  }
+
+  /**
+   * A data flow configuration that replicates the data flow configuration for a specific query, but
+   * replaces the set of sinks with the set of endpoints we're extracting.
+   *
+   * We use this to find out when there is flow to a particular endpoint from a known source.
+   *
+   * This configuration behaves in a very similar way to the `ForwardExploringConfiguration` class
+   * from the CodeQL standard libraries for JavaScript.
+   */
+  private class Configuration extends DataFlow::Configuration {
+    Query q;
+
+    Configuration() { this = getDataFlowCfg(q) }
+
+    Query getQuery() { result = q }
+
+    /** The sinks are the endpoints we're extracting. */
+    override predicate isSink(DataFlow::Node sink) { sink = getAnEndpoint(q) }
+
+    /** The sinks are the endpoints we're extracting. */
+    override predicate isSink(DataFlow::Node sink, DataFlow::FlowLabel lbl) {
+      sink = getAnEndpoint(q)
+    }
+  }
+}

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/extraction/ExtractEndpointDataEvaluation.ql

@@ -0,0 +1,25 @@
+/*
+ * For internal use only.
+ *
+ * Extracts evaluation data we can use to evaluate ML models for ML-powered queries.
+ */
+
+import javascript
+import ExtractEndpointData as ExtractEndpointData
+
+query predicate endpoints(
+  DataFlow::Node endpoint, string queryName, string key, string value, string valueType
+) {
+  ExtractEndpointData::endpoints(endpoint, queryName, key, value, valueType) and
+  // only select endpoints that are either Sink, NotASink or Unknown
+  ExtractEndpointData::endpoints(endpoint, queryName, "sinkLabel", ["Sink", "NotASink", "Unknown"],
+    "string") and
+  // do not select endpoints filtered out by end-to-end evaluation
+  ExtractEndpointData::endpoints(endpoint, queryName, "isExcludedFromEndToEndEvaluation", "false",
+    "boolean")
+}
+
+query predicate tokenFeatures(DataFlow::Node endpoint, string featureName, string featureValue) {
+  endpoints(endpoint, _, _, _, _) and
+  ExtractEndpointData::tokenFeatures(endpoint, featureName, featureValue)
+}

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/extraction/ExtractEndpointDataTraining.ql

@@ -0,0 +1,26 @@
+/*
+ * For internal use only.
+ *
+ * Extracts training data we can use to train ML models for ML-powered queries.
+ */
+
+import javascript
+import ExtractEndpointData as ExtractEndpointData
+
+query predicate endpoints(
+  DataFlow::Node endpoint, string queryName, string key, string value, string valueType
+) {
+  ExtractEndpointData::endpoints(endpoint, queryName, key, value, valueType) and
+  // only select endpoints that are either Sink or NotASink
+  ExtractEndpointData::endpoints(endpoint, queryName, "sinkLabel", ["Sink", "NotASink"], "string") and
+  // do not select endpoints filtered out by end-to-end evaluation
+  ExtractEndpointData::endpoints(endpoint, queryName, "isExcludedFromEndToEndEvaluation", "false",
+    "boolean") and
+  // only select endpoints that can be part of a tainted flow
+  ExtractEndpointData::endpoints(endpoint, queryName, "isConstantExpression", "false", "boolean")
+}
+
+query predicate tokenFeatures(DataFlow::Node endpoint, string featureName, string featureValue) {
+  endpoints(endpoint, _, _, _, _) and
+  ExtractEndpointData::tokenFeatures(endpoint, featureName, featureValue)
+}

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/extraction/ExtractEndpointLabelEncoding.ql

@@ -0,0 +1,11 @@
+/**
+ * @name Endpoint types
+ * @description Maps endpoint type encodings to human-readable descriptions.
+ * @kind table
+ * @id js/ml-powered/model-building/endpoint-type-encodings
+ */
+
+import experimental.adaptivethreatmodeling.EndpointTypes
+
+from EndpointType type
+select type.getEncoding() as encoding, type.getDescription() as description order by encoding

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/extraction/ExtractMisclassifiedEndpointFeatures.ql

@@ -0,0 +1,44 @@
+/*
+ * For internal use only.
+ *
+ * Query for finding misclassified endpoints which we can use to debug ML-powered queries.
+ */
+
+import javascript
+import experimental.adaptivethreatmodeling.AdaptiveThreatModeling
+import experimental.adaptivethreatmodeling.ATMConfig
+import experimental.adaptivethreatmodeling.BaseScoring
+import experimental.adaptivethreatmodeling.EndpointFeatures as EndpointFeatures
+import experimental.adaptivethreatmodeling.EndpointTypes
+import semmle.javascript.security.dataflow.NosqlInjectionCustomizations
+
+/** The positive endpoint type for which you wish to find misclassified examples. */
+EndpointType getEndpointType() { result instanceof NosqlInjectionSinkType }
+
+/** Get a positive endpoint. This will be run through the classifier to determine whether it is misclassified. */
+DataFlow::Node getAPositiveEndpoint() { result instanceof NosqlInjection::Sink }
+
+/** An ATM configuration to find misclassified endpoints of type `getEndpointType()`. */
+class ExtractMisclassifiedEndpointsATMConfig extends ATMConfig {
+  ExtractMisclassifiedEndpointsATMConfig() { this = "ExtractMisclassifiedEndpointsATMConfig" }
+
+  override predicate isEffectiveSink(DataFlow::Node sinkCandidate) {
+    sinkCandidate = getAPositiveEndpoint()
+  }
+
+  override EndpointType getASinkEndpointType() { result = getEndpointType() }
+}
+
+/** Get an endpoint from `getAPositiveEndpoint()` that is incorrectly excluded from the results. */
+DataFlow::Node getAMisclassifedEndpoint() {
+  any(ExtractMisclassifiedEndpointsATMConfig config).isEffectiveSink(result) and
+  not any(ScoringResults results).shouldResultBeIncluded(_, result)
+}
+
+/** The token features for each misclassified endpoint. */
+query predicate tokenFeaturesForMisclassifiedEndpoints(
+  DataFlow::Node endpoint, string featureName, string featureValue
+) {
+  endpoint = getAMisclassifedEndpoint() and
+  EndpointFeatures::tokenFeatures(endpoint, featureName, featureValue)
+}

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/extraction/Labels.qll

@@ -0,0 +1,29 @@
+/*
+ * For internal use only.
+ *
+ * Labels used in training and evaluation data to indicate knowledge about whether an endpoint is a
+ * sink for a particular security query.
+ */
+
+newtype TEndpointLabel =
+  TSinkLabel() or
+  TNotASinkLabel() or
+  TUnknownLabel()
+
+abstract class EndpointLabel extends TEndpointLabel {
+  abstract string getEncoding();
+
+  string toString() { result = getEncoding() }
+}
+
+class SinkLabel extends EndpointLabel, TSinkLabel {
+  override string getEncoding() { result = "Sink" }
+}
+
+class NotASinkLabel extends EndpointLabel, TNotASinkLabel {
+  override string getEncoding() { result = "NotASink" }
+}
+
+class UnknownLabel extends EndpointLabel, TUnknownLabel {
+  override string getEncoding() { result = "Unknown" }
+}

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/extraction/NoFeaturizationRestrictionsConfig.qll

@@ -0,0 +1,16 @@
+/*
+ * For internal use only.
+ */
+
+private import experimental.adaptivethreatmodeling.FeaturizationConfig
+
+/**
+ * A featurization config that featurizes all endpoints.
+ *
+ * This should only be used in extraction queries and tests.
+ */
+class NoRestrictionsFeaturizationConfig extends FeaturizationConfig {
+  NoRestrictionsFeaturizationConfig() { this = "NoRestrictionsFeaturization" }
+
+  override DataFlow::Node getAnEndpointToFeaturize() { any() }
+}

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/extraction/Queries.qll

@@ -0,0 +1,33 @@
+/*
+ * For internal use only.
+ *
+ * Represents the security queries for which we currently have ML-powered versions.
+ */
+
+newtype TQuery =
+  TNosqlInjectionQuery() or
+  TSqlInjectionQuery() or
+  TTaintedPathQuery() or
+  TXssQuery()
+
+abstract class Query extends TQuery {
+  abstract string getName();
+
+  string toString() { result = getName() }
+}
+
+class NosqlInjectionQuery extends Query, TNosqlInjectionQuery {
+  override string getName() { result = "NosqlInjection" }
+}
+
+class SqlInjectionQuery extends Query, TSqlInjectionQuery {
+  override string getName() { result = "SqlInjection" }
+}
+
+class TaintedPathQuery extends Query, TTaintedPathQuery {
+  override string getName() { result = "TaintedPath" }
+}
+
+class XssQuery extends Query, TXssQuery {
+  override string getName() { result = "Xss" }
+}

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/qlpack.yml

@@ -0,0 +1,8 @@
+name: codeql/javascript-experimental-atm-model-building
+extractor: javascript
+library: false
+groups:
+    - javascript
+    - experimental
+dependencies:
+    codeql/javascript-experimental-atm-lib: "*"

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/EndpointFeatures.ql

@@ -0,0 +1,34 @@
+/**
+ * EndpointFeatures.ql
+ *
+ * This tests generic token-based featurization of all endpoint candidates for all of the security
+ * queries we support. This is in comparison to the `ExtractEndpointData.qlref` test, which tests
+ * just the endpoints we extract in the training data.
+ */
+
+import javascript
+import experimental.adaptivethreatmodeling.NosqlInjectionATM as NosqlInjectionATM
+import experimental.adaptivethreatmodeling.SqlInjectionATM as SqlInjectionATM
+import experimental.adaptivethreatmodeling.TaintedPathATM as TaintedPathATM
+import experimental.adaptivethreatmodeling.XssATM as XssATM
+import experimental.adaptivethreatmodeling.EndpointFeatures as EndpointFeatures
+import experimental.adaptivethreatmodeling.StandardEndpointFilters as StandardEndpointFilters
+import extraction.NoFeaturizationRestrictionsConfig
+
+query predicate tokenFeatures(DataFlow::Node endpoint, string featureName, string featureValue) {
+  (
+    not exists(NosqlInjectionATM::SinkEndpointFilter::getAReasonSinkExcluded(endpoint)) or
+    not exists(SqlInjectionATM::SinkEndpointFilter::getAReasonSinkExcluded(endpoint)) or
+    not exists(TaintedPathATM::SinkEndpointFilter::getAReasonSinkExcluded(endpoint)) or
+    not exists(XssATM::SinkEndpointFilter::getAReasonSinkExcluded(endpoint)) or
+    StandardEndpointFilters::isArgumentToModeledFunction(endpoint)
+  ) and
+  EndpointFeatures::tokenFeatures(endpoint, featureName, featureValue)
+}
+
+query predicate invalidTokenFeatures(
+  DataFlow::Node endpoint, string featureName, string featureValue
+) {
+  strictcount(string value | EndpointFeatures::tokenFeatures(endpoint, featureName, value)) > 1 and
+  EndpointFeatures::tokenFeatures(endpoint, featureName, featureValue)
+}

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/ExtractEndpointData.qlref

@@ -0,0 +1 @@
+extraction/ExtractEndpointData.ql

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/ExtractEndpointDataEvaluation.qlref

@@ -0,0 +1 @@
+extraction/ExtractEndpointDataEvaluation.ql

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/ExtractEndpointDataTraining.qlref

@@ -0,0 +1 @@
+extraction/ExtractEndpointDataTraining.ql

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/FilteredTruePositives.expected

@@ -0,0 +1,16 @@
+nosqlFilteredTruePositives
+| autogenerated/NosqlAndSqlInjection/untyped/mongoose.js:111:14:111:18 | query | not a direct argument to a likely external library call or a heuristic sink |
+sqlFilteredTruePositives
+| autogenerated/NosqlAndSqlInjection/untyped/tst2.js:7:13:7:45 | select  ... e id =  | not an argument to a likely external library call or a heuristic sink |
+| autogenerated/NosqlAndSqlInjection/untyped/tst2.js:7:48:7:60 | req.params.id | not an argument to a likely external library call or a heuristic sink |
+taintedPathFilteredTruePositives
+| autogenerated/TaintedPath/TaintedPath.js:66:26:66:31 | "SAFE" | not a direct argument to a likely external library call or a heuristic sink |
+| autogenerated/TaintedPath/TaintedPath.js:71:26:71:45 | Cookie.get("unsafe") | not a direct argument to a likely external library call or a heuristic sink |
+xssFilteredTruePositives
+| autogenerated/Xss/DomBasedXss/d3.js:12:20:12:29 | getTaint() | not a direct argument to a likely external library call or a heuristic sink |
+| autogenerated/Xss/DomBasedXss/d3.js:14:20:14:29 | getTaint() | not a direct argument to a likely external library call or a heuristic sink |
+| autogenerated/Xss/DomBasedXss/express.js:7:15:7:33 | req.param("wobble") | not a direct argument to a likely external library call or a heuristic sink |
+| autogenerated/Xss/DomBasedXss/jwt-server.js:11:19:11:29 | decoded.foo | not a direct argument to a likely external library call or a heuristic sink |
+| autogenerated/Xss/DomBasedXss/tst.js:316:35:316:42 | location | not a direct argument to a likely external library call or a heuristic sink |
+| autogenerated/Xss/DomBasedXss/typeahead.js:10:16:10:18 | loc | not a direct argument to a likely external library call or a heuristic sink |
+| autogenerated/Xss/DomBasedXss/typeahead.js:25:18:25:20 | val | not a direct argument to a likely external library call or a heuristic sink |

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/FilteredTruePositives.ql

@@ -0,0 +1,47 @@
+/*
+ * FilteredTruePositives.ql
+ *
+ * This test checks several components of the endpoint filters for each query to see whether they
+ * filter out any known sinks. It explicitly does not check the endpoint filtering step that's based
+ * on whether the endpoint is an argument to a modelled function, since this necessarily filters out
+ * all known sinks. However, we can test all the other filtering steps against the set of known
+ * sinks.
+ *
+ * Ideally, the sink endpoint filters would have perfect recall and therefore each of the predicates
+ * in this test would have zero results. However, in some cases we have chosen to sacrifice recall
+ * when we perceive the improved precision of the results to be worth the drop in recall.
+ */
+
+import semmle.javascript.security.dataflow.NosqlInjectionCustomizations
+import semmle.javascript.security.dataflow.SqlInjectionCustomizations
+import semmle.javascript.security.dataflow.TaintedPathCustomizations
+import semmle.javascript.security.dataflow.DomBasedXssCustomizations
+import experimental.adaptivethreatmodeling.StandardEndpointFilters as StandardEndpointFilters
+import experimental.adaptivethreatmodeling.NosqlInjectionATM as NosqlInjectionATM
+import experimental.adaptivethreatmodeling.SqlInjectionATM as SqlInjectionATM
+import experimental.adaptivethreatmodeling.TaintedPathATM as TaintedPathATM
+import experimental.adaptivethreatmodeling.XssATM as XssATM
+
+query predicate nosqlFilteredTruePositives(DataFlow::Node endpoint, string reason) {
+  endpoint instanceof NosqlInjection::Sink and
+  reason = NosqlInjectionATM::SinkEndpointFilter::getAReasonSinkExcluded(endpoint) and
+  not reason = ["argument to modeled function", "modeled sink", "modeled database access"]
+}
+
+query predicate sqlFilteredTruePositives(DataFlow::Node endpoint, string reason) {
+  endpoint instanceof SqlInjection::Sink and
+  reason = SqlInjectionATM::SinkEndpointFilter::getAReasonSinkExcluded(endpoint) and
+  reason != "argument to modeled function"
+}
+
+query predicate taintedPathFilteredTruePositives(DataFlow::Node endpoint, string reason) {
+  endpoint instanceof TaintedPath::Sink and
+  reason = TaintedPathATM::SinkEndpointFilter::getAReasonSinkExcluded(endpoint) and
+  reason != "argument to modeled function"
+}
+
+query predicate xssFilteredTruePositives(DataFlow::Node endpoint, string reason) {
+  endpoint instanceof DomBasedXss::Sink and
+  reason = XssATM::SinkEndpointFilter::getAReasonSinkExcluded(endpoint) and
+  reason != "argument to modeled function"
+}

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/NosqlAndSqlInjection/typed/shim.d.ts

@@ -0,0 +1,13 @@
+declare module "mongodb" {
+  interface Collection {
+    find(query: any): any;
+  }
+}
+declare module "mongoose" {
+  interface Model {
+    find(query: any): any;
+  }
+  interface Query {
+    find(query: any): any;
+  }
+}

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/NosqlAndSqlInjection/typed/typedClient.ts

@@ -0,0 +1,24 @@
+import * as mongodb from "mongodb";
+
+const express = require("express") as any;
+const bodyParser = require("body-parser") as any;
+
+declare function getCollection(): mongodb.Collection;
+
+let app = express();
+
+app.use(bodyParser.json());
+
+app.post("/find", (req, res) => {
+  let v = JSON.parse(req.body.x);
+  getCollection().find({ id: v }); // NOT OK
+});
+
+import * as mongoose from "mongoose";
+declare function getMongooseModel(): mongoose.Model;
+declare function getMongooseQuery(): mongoose.Query;
+app.post("/find", (req, res) => {
+  let v = JSON.parse(req.body.x);
+  getMongooseModel().find({ id: v }); // NOT OK
+  getMongooseQuery().find({ id: v }); // NOT OK
+});

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/NosqlAndSqlInjection/untyped/dbo.js

@@ -0,0 +1,13 @@
+let dbClient = require("mongodb").MongoClient,
+  db = null;
+module.exports = {
+  db: () => {
+    return db;
+  },
+  connect: fn => {
+    dbClient.connect(process.env.DB_URL, {}, (err, client) => {
+      db = client.db(process.env.DB_NAME);
+      return fn(err);
+    });
+  }
+};

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/NosqlAndSqlInjection/untyped/json-schema-validator.js

@@ -0,0 +1,37 @@
+import Ajv from 'ajv';
+import express from 'express';
+import { MongoClient } from 'mongodb';
+
+const app = express();
+
+const schema = {
+    type: 'object',
+    properties: {
+        date: { type: 'string' },
+        title: { type: 'string' },
+    },
+};
+const ajv = new Ajv();
+const checkSchema = ajv.compile(schema);
+
+function validate(x) {
+    return x != null;
+}
+
+app.post('/documents/find', (req, res) => {
+    MongoClient.connect('mongodb://localhost:27017/test', (err, db) => {
+        let doc = db.collection('doc');
+
+        const query = JSON.parse(req.query.data);
+        if (checkSchema(query)) {
+            doc.find(query); // OK
+        }
+        if (ajv.validate(schema, query)) {
+            doc.find(query); // OK
+        }
+        if (validate(query)) {
+            doc.find(query); // NOT OK - validate() doesn't sanitize
+        }
+        doc.find(query); // NOT OK
+    });
+});

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/NosqlAndSqlInjection/untyped/marsdb-flow-from.js

@@ -0,0 +1,9 @@
+const MarsDB = require("marsdb");
+
+const myDoc = new MarsDB.Collection("myDoc");
+
+const db = {
+  myDoc
+};
+
+module.exports = db;

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/NosqlAndSqlInjection/untyped/marsdb-flow-to.js

@@ -0,0 +1,15 @@
+const express = require("express"),
+      bodyParser = require("body-parser"),
+      db = require('./marsdb-flow-from');
+
+const app = express();
+
+app.use(bodyParser.urlencoded({ extended: true }));
+
+app.post("/documents/find", (req, res) => {
+  const query = {};
+  query.title = req.body.title;
+
+  // NOT OK: query is tainted by user-provided object value
+  db.myDoc.find(query);
+});

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/NosqlAndSqlInjection/untyped/marsdb.js

@@ -0,0 +1,17 @@
+const express = require("express"),
+  MarsDB = require("marsdb"),
+  bodyParser = require("body-parser");
+
+let doc = new MarsDB.Collection("myDoc");
+
+const app = express();
+
+app.use(bodyParser.urlencoded({ extended: true }));
+
+app.post("/documents/find", (req, res) => {
+  const query = {};
+  query.title = req.body.title;
+
+  // NOT OK: query is tainted by user-provided object value
+  doc.find(query);
+});

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/NosqlAndSqlInjection/untyped/minimongo.js

@@ -0,0 +1,19 @@
+const express = require("express"),
+  minimongo = require("minimongo"),
+  bodyParser = require("body-parser");
+
+var LocalDb = minimongo.MemoryDb,
+  db = new LocalDb(),
+  doc = db.myDocs;
+
+const app = express();
+
+app.use(bodyParser.urlencoded({ extended: true }));
+
+app.post("/documents/find", (req, res) => {
+  const query = {};
+  query.title = req.body.title;
+
+  // NOT OK: query is tainted by user-provided object value
+  doc.find(query);
+});

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/NosqlAndSqlInjection/untyped/mongodb.js

@@ -0,0 +1,114 @@
+const express = require('express'),
+      mongodb = require('mongodb'),
+      bodyParser = require('body-parser');
+
+const MongoClient = mongodb.MongoClient;
+
+const app = express();
+
+app.use(bodyParser.urlencoded({ extended: true }));
+
+app.post('/documents/find', (req, res) => {
+    const query = {};
+    query.title = req.body.title;
+    MongoClient.connect('mongodb://localhost:27017/test', (err, db) => {
+      let doc = db.collection('doc');
+
+      // NOT OK: query is tainted by user-provided object value
+      doc.find(query);
+
+      // OK: user-data is coerced to a string
+      doc.find({ title: '' + query.body.title });
+
+      // OK: throws unless user-data is a string
+      doc.find({ title: query.body.title.substr(1) });
+
+      let title = req.body.title;
+      if (typeof title === "string") {
+        // OK: input checked to be a string
+        doc.find({ title: title });
+
+        // NOT OK: input is parsed as JSON after string check
+        doc.find({ title: JSON.parse(title) });
+      }
+    });
+});
+
+app.get('/:id', (req, res) => {
+    let query = { id: req.param.id };
+    MongoClient.connect('mongodb://localhost:27017/test', (err, db) => {
+      let doc = db.collection('doc');
+
+      // OK: query is tainted, but only by string value
+      doc.find(query);
+    });
+});
+
+app.post('/documents/find', (req, res) => {
+    const query = {};
+    query.title = req.query.title;
+    MongoClient.connect('mongodb://localhost:27017/test', (err, db) => {
+      let doc = db.collection('doc');
+
+      // NOT OK: query is tainted by user-provided object value
+      doc.find(query);
+    });
+});
+
+app.post('/documents/find', (req, res) => {
+	const query = {};
+	query.title = req.query.title;
+	MongoClient.connect('mongodb://localhost:27017/test', (err, client) => {
+		let doc = client.db("MASTER").collection('doc');
+
+		// NOT OK: query is tainted by user-provided object value
+		doc.find(query);
+	});
+});
+
+app.post("/logs/count-by-tag", (req, res) => {
+  let tag = req.query.tag;
+
+  MongoClient.connect(process.env.DB_URL, {}, (err, client) => {
+    client
+      .db(process.env.DB_NAME)
+      .collection("logs")
+      // NOT OK: query is tainted by user-provided object value
+      .count({ tags: tag });
+  });
+
+  let importedDbo = require("./dbo.js");
+  importedDbo
+    .db()
+    .collection("logs")
+    // NOT OK: query is tainted by user-provided object value
+    .count({ tags: tag });
+});
+
+
+app.get('/:id', (req, res) => {
+  useParams(req.param);
+});
+function useParams(params) {
+  let query = { id: params.id };
+  MongoClient.connect('mongodb://localhost:27017/test', (err, db) => {
+    let doc = db.collection('doc');
+
+    // OK: query is tainted, but only by string value
+    doc.find(query);
+  });
+}
+
+app.post('/documents/find', (req, res) => {
+  useQuery(req.query);
+});
+function useQuery(queries) {
+  const query = {};
+  query.title = queries.title;
+  MongoClient.connect('mongodb://localhost:27017/test', (err, db) => {
+    let doc = db.collection('doc');
+
+    // NOT OK: query is tainted by user-provided object value
+    doc.find(query);
+  });
+}

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/NosqlAndSqlInjection/untyped/mongodb_bodySafe.js

@@ -0,0 +1,31 @@
+const express = require('express'),
+      mongodb = require('mongodb'),
+      bodyParser = require('body-parser');
+
+const MongoClient = mongodb.MongoClient;
+
+const app = express();
+
+app.use(bodyParser.urlencoded({ extended: false }));
+
+app.post('/documents/find', (req, res) => {
+    const query = {};
+    query.title = req.body.title;
+    MongoClient.connect('mongodb://localhost:27017/test', (err, db) => {
+      let doc = db.collection('doc');
+
+      // OK: req.body is safe
+      doc.find(query);
+    });
+});
+
+app.post('/documents/find', (req, res) => {
+    const query = {};
+    query.title = req.query.title;
+    MongoClient.connect('mongodb://localhost:27017/test', (err, db) => {
+      let doc = db.collection('doc');
+
+      // NOT OK: regardless of body parser, query value is still tainted
+      doc.find(query);
+    });
+});

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/NosqlAndSqlInjection/untyped/mongoose.js

@@ -0,0 +1,132 @@
+'use strict';
+const Express = require('express');
+const BodyParser = require('body-parser');
+const Mongoose = require('mongoose');
+Mongoose.Promise = global.Promise;
+Mongoose.connect('mongodb://localhost/injectable1');
+
+const app = Express();
+app.use(BodyParser.json());
+
+const Document = Mongoose.model('Document', {
+    title: {
+        type: String,
+        unique: true
+    },
+    type: String
+});
+
+app.post('/documents/find', (req, res) => {
+    const query = {};
+    query.title = req.body.title;
+
+    // NOT OK: query is tainted by user-provided object value
+    Document.aggregate([query]);
+
+    // NOT OK: query is tainted by user-provided object value
+    Document.count(query);
+
+    // NOT OK: query is tainted by user-provided object value
+    Document.deleteMany(query);
+
+    // NOT OK: query is tainted by user-provided object value
+    Document.deleteOne(query);
+
+    // NOT OK: query is tainted by user-provided object value
+    Document.distinct('type', query);
+
+    // NOT OK: query is tainted by user-provided object value
+    Document.find(query);
+
+    // NOT OK: query is tainted by user-provided object value
+    Document.findOne(query);
+
+    // NOT OK: query is tainted by user-provided object value
+    Document.findOneAndDelete(query);
+
+    // NOT OK: query is tainted by user-provided object value
+    Document.findOneAndRemove(query);
+
+    // NOT OK: query is tainted by user-provided object value
+    Document.findOneAndUpdate(query);
+
+    // NOT OK: query is tainted by user-provided object value
+    Document.replaceOne(query);
+
+    // NOT OK: query is tainted by user-provided object value
+    Document.update(query);
+
+    // NOT OK: query is tainted by user-provided object value
+    Document.updateMany(query);
+
+    // NOT OK: query is tainted by user-provided object value
+	Document.updateOne(query).then(X);
+
+	Document.findByIdAndUpdate(X, query, function(){}); // NOT OK
+
+	new Mongoose.Query(X, Y, query)	// NOT OK
+		.and(query, function(){}) // NOT OK
+	;
+
+    Document.where(query)	// NOT OK - `.where()` on a Model. 
+        .where(query)	// NOT OK - `.where()` on a Query. 
+		.and(query) // NOT OK
+		.or(query) // NOT OK
+		.distinct(X, query) // NOT OK
+		.comment(query) // OK
+		.count(query) // NOT OK
+		.exec()
+	;
+
+	Mongoose.createConnection(X).count(query); // OK (invalid program)
+	Mongoose.createConnection(X).model(Y).count(query); // NOT OK
+	Mongoose.createConnection(X).models[Y].count(query); // NOT OK
+
+	Document.findOne(X, (err, res) => res.count(query)); // NOT OK
+	Document.findOne(X, (err, res) => err.count(query)); // OK
+	Document.findOne(X).exec((err, res) => res.count(query)); // NOT OK
+	Document.findOne(X).exec((err, res) => err.count(query)); // OK
+	Document.findOne(X).then((res) => res.count(query)); // NOT OK
+	Document.findOne(X).then(Y, (err) => err.count(query)); // OK
+
+	Document.find(X, (err, res) => res[i].count(query)); // NOT OK
+	Document.find(X, (err, res) => err.count(query)); // OK
+	Document.find(X).exec((err, res) => res[i].count(query)); // NOT OK
+	Document.find(X).exec((err, res) => err.count(query)); // OK
+	Document.find(X).then((res) => res[i].count(query)); // NOT OK
+	Document.find(X).then(Y, (err) => err.count(query)); // OK
+
+	Document.count(X, (err, res) => res.count(query)); // OK (res is a number)
+    
+	function innocent(X, Y, query) { // To detect if API-graphs were used incorrectly.
+		return new Mongoose.Query("constant", "constant", "constant");
+	}
+	new innocent(X, Y, query);
+
+	function getQueryConstructor() {
+		return Mongoose.Query;
+	}
+
+	var C = getQueryConstructor();
+	new C(X, Y, query); // NOT OK
+
+	Document.findOneAndUpdate(X, query, function () { }); // NOT OK
+
+	let id = req.query.id, cond = req.query.cond;
+	Document.deleteMany(cond); // NOT OK
+	Document.deleteOne(cond); // NOT OK
+	Document.geoSearch(cond); // NOT OK
+	Document.remove(cond); // NOT OK
+	Document.replaceOne(cond, Y); // NOT OK
+	Document.find(cond); // NOT OK
+	Document.findOne(cond); // NOT OK
+	Document.findById(id); // NOT OK
+	Document.findOneAndDelete(cond); // NOT OK
+	Document.findOneAndRemove(cond); // NOT OK
+	Document.findOneAndUpdate(cond, Y); // NOT OK
+	Document.update(cond, Y); // NOT OK
+	Document.updateMany(cond, Y); // NOT OK
+	Document.updateOne(cond, Y); // NOT OK
+	Document.find({ _id: id }); // NOT OK
+	Document.find({ _id: { $eq: id } }); // OK
+});

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/NosqlAndSqlInjection/untyped/mongooseJsonParse.js

@@ -0,0 +1,25 @@
+'use strict';
+const Express = require('express');
+const BodyParser = require('body-parser');
+const Mongoose = require('mongoose');
+Mongoose.Promise = global.Promise;
+Mongoose.connect('mongodb://localhost/injectable1');
+
+const app = Express();
+
+const Document = Mongoose.model('Document', {
+    title: {
+        type: String,
+        unique: true
+    },
+    type: String
+});
+
+app.get('/documents/find', (req, res) => {
+    const query = {};
+    query.title = JSON.parse(req.query.data).title;
+
+    // NOT OK: query is tainted by user-provided object value
+    Document.find(query);
+});
+

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/NosqlAndSqlInjection/untyped/mongooseModel.js

@@ -0,0 +1,3 @@
+import mongoose from 'mongoose';
+
+export const MyModel = mongoose.model('MyModel', getSchema());

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/NosqlAndSqlInjection/untyped/mongooseModelClient.js

@@ -0,0 +1,14 @@
+import { MyModel } from './mongooseModel';
+import express from 'express';
+import bodyParser from 'body-parser';
+
+let app = express();
+
+app.use(bodyParser.json());
+
+app.post('/find', (req, res) => {
+  let v = JSON.parse(req.body.x);
+  MyModel.find({ id: v }); // NOT OK
+  MyModel.find({ id: req.body.id }); // NOT OK
+  MyModel.find({ id: `${req.body.id}` }); // OK
+});

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/NosqlAndSqlInjection/untyped/pg-promise-types.ts

@@ -0,0 +1,13 @@
+import { IDatabase } from "pg-promise";
+
+export class Foo {
+  db: IDatabase;
+
+  onRequest(req, res) {
+    let taint = req.params.x;
+    this.db.one(taint); // NOT OK
+    res.end();
+  }
+}
+
+require('express')().get('/foo', (req, res) => new Foo().onRequest(req, res));

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/NosqlAndSqlInjection/untyped/pg-promise.js

@@ -0,0 +1,66 @@
+const pgp = require('pg-promise')();
+
+require('express')().get('/foo', (req, res) => {
+  const db = pgp(process.env['DB_CONNECTION_STRING']);
+
+  var query = "SELECT ITEM,PRICE FROM PRODUCT WHERE ITEM_CATEGORY='"
+             + req.params.category + "' ORDER BY PRICE";
+  
+  db.any(query); // NOT OK
+  db.many(query); // NOT OK
+  db.manyOrNone(query); // NOT OK
+  db.map(query); // NOT OK
+  db.multi(query); // NOT OK
+  db.multiResult(query); // NOT OK
+  db.none(query); // NOT OK
+  db.one(query); // NOT OK
+  db.oneOrNone(query); // NOT OK
+  db.query(query); // NOT OK
+  db.result(query); // NOT OK
+  
+  db.one({
+    text: query // NOT OK
+  });
+  db.one({
+    text: 'SELECT * FROM news where id = $1', // OK
+    values: req.params.id, // OK
+  });
+  db.one({
+    text: 'SELECT * FROM news where id = $1:raw',
+    values: req.params.id, // NOT OK - interpreted as raw parameter
+  });
+  db.one({
+    text: 'SELECT * FROM news where id = $1^',
+    values: req.params.id, // NOT OK
+  });
+  db.one({
+    text: 'SELECT * FROM news where id = $1:raw AND name = $2:raw AND foo = $3',
+    values: [
+      req.params.id, // NOT OK
+      req.params.name, // NOT OK
+      req.params.foo, // OK - not using raw interpolation
+    ]
+  });
+  db.one({
+    text: 'SELECT * FROM news where id = ${id}:raw AND name = ${name}',
+    values: {
+      id: req.params.id, // NOT OK
+      name: req.params.name, // OK - not using raw interpolation
+    }
+  });
+  db.one({
+    text: "SELECT * FROM news where id = ${id}:value AND name LIKE '%${name}:value%' AND title LIKE \"%${title}:value%\"",
+    values: {
+      id: req.params.id, // NOT OK
+      name: req.params.name, // OK - :value cannot break out of single quotes
+      title: req.params.title, // NOT OK - enclosed by wrong type of quote
+    }
+  });
+  db.task(t => {
+      return t.one(query); // NOT OK
+  });
+  db.task(
+    { cnd: t => t.one(query) }, // NOT OK
+    t => t.one(query) // NOT OK
+  );
+});

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/NosqlAndSqlInjection/untyped/redis.js

@@ -0,0 +1,53 @@
+
+const redis = require("redis");
+const client = redis.createClient();
+
+const Express = require('express');
+const app = Express();
+app.use(require('body-parser').json());
+
+app.post('/documents/find', (req, res) => {
+    client.set(req.body.key, "value"); // NOT OK
+
+    var key = req.body.key;
+    if (typeof key === "string") {
+        client.set(key, "value"); // OK
+        client.set(["key", "value"]);
+    }  
+
+    client.set(key, "value"); // NOT OK
+    client.hmset("key", "field", "value", key, "value2"); // NOT OK
+
+    // chain commands
+    client
+        .multi()
+        .set("constant", "value")
+        .set(key, "value") // NOT OK
+        .get(key) // OK
+        .exec(function (err, replies) { });
+
+    client.duplicate((err, newClient) => {
+        newClient.set(key, "value"); // NOT OK
+    });
+    client.duplicate().set(key, "value"); // NOT OK
+});
+
+
+import { promisify } from 'util';
+app.post('/documents/find', (req, res) => {
+    const key = req.body.key;
+    client.set(key, "value"); // NOT OK
+
+    const setAsync = promisify(client.set).bind(client);
+
+    const foo1 = setAsync(key, "value"); // NOT OK
+
+    client.setAsync = promisify(client.set);
+    const foo2 = client.setAsync(key, "value"); // NOT OK
+
+    client.unrelated = promisify(() => {});
+    const foo3 = client.unrelated(key, "value"); // OK
+
+    const unrelated = promisify(client.foobar).bind(client);
+    const foo4 = unrelated(key, "value"); // OK
+});

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/NosqlAndSqlInjection/untyped/socketio.js

@@ -0,0 +1,13 @@
+// Adapted from https://github.com/mapbox/node-sqlite3/wiki/API, which is
+// part of the node-sqlite3 project, which is licensed under the BSD 3-Clause
+// License; see file node-sqlite3-LICENSE.
+var express = require('express');
+var sqlite3 = require('sqlite3').verbose();
+var db = new sqlite3.Database(':memory:');
+
+var io = require('socket.io')();
+io.on('connection', (socket) => {
+  socket.on('newuser', (handle) => {
+    db.run(`INSERT INTO users(name) VALUES ${handle}`);
+  });
+});

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/NosqlAndSqlInjection/untyped/tst.js

@@ -0,0 +1,11 @@
+// Adapted from https://github.com/mapbox/node-sqlite3/wiki/API, which is
+// part of the node-sqlite3 project, which is licensed under the BSD 3-Clause
+// License; see file node-sqlite3-LICENSE.
+var express = require('express');
+var sqlite3 = require('sqlite3').verbose();
+var db = new sqlite3.Database(':memory:');
+
+var app = express();
+app.get('/post/:id', function(req, res) {
+  db.get('SELECT * FROM Post WHERE id = "' + req.params.id + '"');
+});

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/NosqlAndSqlInjection/untyped/tst2.js

@@ -0,0 +1,10 @@
+var express = require('express');
+const sql = require('mssql');
+
+var app = express();
+app.get('/post/:id', async function(req, res) {
+  // OK
+  sql.query`select * from mytable where id = ${req.params.id}`;
+  // NOT OK
+  new sql.Request().query("select * from mytable where id = '" + req.params.id + "'");
+});

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/NosqlAndSqlInjection/untyped/tst3.js

@@ -0,0 +1,21 @@
+// Adapted from the documentation of https://github.com/brianc/node-postgres,
+// which is licensed under the MIT license; see file node-postgres-LICENSE.
+const pg = require('pg');
+const pool = new pg.Pool(config);
+
+function handler(req, res) {
+  var query1 = "SELECT ITEM,PRICE FROM PRODUCT WHERE ITEM_CATEGORY='"
+             + req.params.category + "' ORDER BY PRICE";
+  pool.query(query1, [], function(err, results) { // BAD: the category might have SQL special characters in it
+    // process results
+  });
+
+  // GOOD: use parameters
+  var query2 = "SELECT ITEM,PRICE FROM PRODUCT WHERE ITEM_CATEGORY=$1"
+             + " ORDER BY PRICE";
+  pool.query(query2, [req.params.category], function(err, results) {
+    // process results
+  });
+}
+
+require('express')().get('/foo', handler);

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/NosqlAndSqlInjection/untyped/tst4.js

@@ -0,0 +1,9 @@
+// Adapted from https://github.com/mapbox/node-sqlite3/wiki/API, which is
+// part of the node-sqlite3 project, which is licensed under the BSD 3-Clause
+var sqlite3 = require('sqlite3').verbose();
+var db = new sqlite3.Database(':memory:');
+
+angular.module('myApp', ['ngRoute'])
+.controller('FindPost', function($routeParams) {
+  db.get('SELECT * FROM Post WHERE id = "' + $routeParams.id + '"');
+});

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/README.md

@@ -0,0 +1,7 @@
+# autogenerated
+
+This folder contains test data for the ATM endpoint CodeQL tests that has been autogenerated from the standard JS CodeQL libraries.
+
+It is helpful, but not required, to periodically update this test data to incorporate new test data introduced in the standard JS CodeQL libraries.
+To update this test data, run `python /path/to/codeql-lib/ql/javascript/test/update_endpoint_test_files.py --codeql-lib-path /path/to/codeql-lib`.
+For more information, run `python /path/to/codeql-lib/ql/javascript/test/update_endpoint_test_files.py --help` or view the source code of [`update_endpoint_test_files.py`](../../update_endpoint_test_files.py).

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/TaintedPath/TaintedPath-es6.js

@@ -0,0 +1,11 @@
+import { readFileSync } from 'fs';
+import { createServer } from 'http';
+import { parse } from 'url';
+import { join } from 'path';
+
+var server = createServer(function(req, res) {
+  let path = parse(req.url, true).query.path;
+
+  // BAD: This could read any file on the file system
+  res.write(readFileSync(join("public", path)));
+});

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/TaintedPath/TaintedPath.js

@@ -0,0 +1,196 @@
+var fs = require('fs'),
+    http = require('http'),
+    url = require('url'),
+    sanitize = require('sanitize-filename'),
+    pathModule = require('path')
+    ;
+
+var server = http.createServer(function(req, res) {
+  let path = url.parse(req.url, true).query.path;
+
+  // BAD: This could read any file on the file system
+  res.write(fs.readFileSync(path));
+
+  // BAD: This could still read any file on the file system
+  res.write(fs.readFileSync("/home/user/" + path));
+
+  if (path.startsWith("/home/user/"))
+      res.write(fs.readFileSync(path)); // BAD: Insufficient sanitisation
+
+  if (path.indexOf("secret") == -1)
+      res.write(fs.readFileSync(path)); // BAD: Insufficient sanitisation
+
+  if (fs.existsSync(path))
+      res.write(fs.readFileSync(path)); // BAD: Insufficient sanitisation
+
+  if (path === 'foo.txt')
+    res.write(fs.readFileSync(path)); // GOOD: Path is compared to white-list
+
+  if (path === 'foo.txt' || path === 'bar.txt')
+    res.write(fs.readFileSync(path)); // GOOD: Path is compared to white-list
+
+  if (path === 'foo.txt' || path === 'bar.txt' || someOpaqueCondition())
+    res.write(fs.readFileSync(path)); // BAD: Path is incompletely compared to white-list
+
+  path = sanitize(path);
+  res.write(fs.readFileSync(path)); // GOOD: Path is sanitized
+
+  path = url.parse(req.url, true).query.path;
+  // GOOD: basename is safe
+  res.write(fs.readFileSync(pathModule.basename(path)));
+  // BAD: taint is preserved
+  res.write(fs.readFileSync(pathModule.dirname(path)));
+  // GOOD: extname is safe
+  res.write(fs.readFileSync(pathModule.extname(path)));
+  // BAD: taint is preserved
+  res.write(fs.readFileSync(pathModule.join(path)));
+  // BAD: taint is preserved
+  res.write(fs.readFileSync(pathModule.join(x, y, path, z)));
+  // BAD: taint is preserved
+  res.write(fs.readFileSync(pathModule.normalize(path)));
+  // BAD: taint is preserved
+  res.write(fs.readFileSync(pathModule.relative(x, path)));
+  // BAD: taint is preserved
+  res.write(fs.readFileSync(pathModule.relative(path, x)));
+  // BAD: taint is preserved
+  res.write(fs.readFileSync(pathModule.resolve(path)));
+  // BAD: taint is preserved
+  res.write(fs.readFileSync(pathModule.resolve(x, y, path, z)));
+  // BAD: taint is preserved
+  res.write(fs.readFileSync(pathModule.toNamespacedPath(path)));
+});
+
+angular.module('myApp', [])
+    .directive('myCustomer', function() {
+        return {
+            templateUrl: "SAFE" // OK
+        }
+    })
+    .directive('myCustomer', function() {
+        return {
+            templateUrl: Cookie.get("unsafe") // NOT OK
+        }
+    })
+
+var server = http.createServer(function(req, res) {
+    // tests for a few uri-libraries
+    res.write(fs.readFileSync(require("querystringify").parse(req.url).query)); // NOT OK
+    res.write(fs.readFileSync(require("query-string").parse(req.url).query)); // NOT OK
+    res.write(fs.readFileSync(require("querystring").parse(req.url).query)); // NOT OK
+});
+
+(function(){
+
+    var express = require('express');
+    var application = express();
+
+    var views_local = (req, res) => res.render(req.params[0]);
+    application.get('/views/*', views_local);
+
+    var views_imported = require("./views");
+    application.get('/views/*', views_imported);
+
+})();
+
+addEventListener('message', (ev) => {
+  Cookie.set("unsafe", ev.data);
+});
+
+var server = http.createServer(function(req, res) {
+	let path = url.parse(req.url, true).query.path;
+
+	res.write(fs.readFileSync(fs.realpathSync(path)));
+	fs.realpath(path,
+	                function(err, realpath){
+		                res.write(fs.readFileSync(realpath));
+	                }
+	               );
+
+});
+
+var server = http.createServer(function(req, res) {
+  let path = url.parse(req.url, true).query.path;
+  
+  if (path) { // sanitization
+    path = path.replace(/[\]\[*,;'"`<>\\?\/]/g, ''); // remove all invalid characters from states plus slashes
+    path = path.replace(/\.\./g, ''); // remove all ".."
+  }
+  
+  res.write(fs.readFileSync(path));  // OK. Is sanitized above.
+});
+
+var server = http.createServer(function(req, res) {
+  let path = url.parse(req.url, true).query.path;
+  
+  if (!path) {
+    
+  } else { // sanitization
+	  path = path.replace(/[\]\[*,;'"`<>\\?\/]/g, ''); // remove all invalid characters from states plus slashes
+    path = path.replace(/\.\./g, ''); // remove all ".."
+  }
+  
+  res.write(fs.readFileSync(path));  // OK. Is sanitized above.
+});
+
+var server = http.createServer(function(req, res) {
+	let path = url.parse(req.url, true).query.path;
+
+	require('send')(req, path); // NOT OK
+});
+
+var server = http.createServer(function(req, res) {
+  let path = url.parse(req.url, true).query.path;
+
+  fs.readFileSync(path); // NOT OK
+  
+  var split = path.split("/");
+  
+  fs.readFileSync(split.join("/")); // NOT OK
+
+  fs.readFileSync(prefix + split[split.length - 1]) // OK
+
+  fs.readFileSync(split[x]) // NOT OK
+  fs.readFileSync(prefix + split[x]) // NOT OK 
+
+  var concatted = prefix.concat(split);
+  fs.readFileSync(concatted.join("/")); // NOT OK
+
+  var concatted2 = split.concat(prefix);
+  fs.readFileSync(concatted2.join("/")); // NOT OK
+
+  fs.readFileSync(split.pop()); // NOT OK 
+
+});
+
+var server = http.createServer(function(req, res) {
+  let path = url.parse(req.url, true).query.path;
+  
+  // Removal of forward-slash or dots.
+  res.write(fs.readFileSync(path.replace(/[\]\[*,;'"`<>\\?\/]/g, ''))); // OK.
+  res.write(fs.readFileSync(path.replace(/[abcd]/g, ''))); // NOT OK
+  res.write(fs.readFileSync(path.replace(/[./]/g, ''))); // OK
+  res.write(fs.readFileSync(path.replace(/[foobar/foobar]/g, ''))); // OK
+  res.write(fs.readFileSync(path.replace(/\//g, ''))); // OK
+  res.write(fs.readFileSync(path.replace(/\.|\//g, ''))); // OK
+
+  res.write(fs.readFileSync(path.replace(/[.]/g, ''))); // NOT OK (can be absolute)
+  res.write(fs.readFileSync(path.replace(/[..]/g, ''))); // NOT OK (can be absolute)
+  res.write(fs.readFileSync(path.replace(/\./g, ''))); // NOT OK (can be absolute)
+  res.write(fs.readFileSync(path.replace(/\.\.|BLA/g, ''))); // NOT OK (can be absolute)
+
+  if (!pathModule.isAbsolute(path)) {
+    res.write(fs.readFileSync(path.replace(/[.]/g, ''))); // OK
+  	res.write(fs.readFileSync(path.replace(/[..]/g, ''))); // OK
+    res.write(fs.readFileSync(path.replace(/\./g, ''))); // OK
+  	res.write(fs.readFileSync(path.replace(/\.\.|BLA/g, ''))); // OK
+  }
+
+  // removing of "../" from prefix.
+  res.write(fs.readFileSync("prefix" + pathModule.normalize(path).replace(/^(\.\.[\/\\])+/, ''))); // OK
+  res.write(fs.readFileSync("prefix" + pathModule.normalize(path).replace(/(\.\.[\/\\])+/, ''))); // OK
+  res.write(fs.readFileSync("prefix" + pathModule.normalize(path).replace(/(\.\.\/)+/, ''))); // OK
+  res.write(fs.readFileSync("prefix" + pathModule.normalize(path).replace(/(\.\.\/)*/, ''))); // OK
+
+  res.write(fs.readFileSync("prefix" + path.replace(/^(\.\.[\/\\])+/, ''))); // NOT OK - not normalized
+  res.write(fs.readFileSync(pathModule.normalize(path).replace(/^(\.\.[\/\\])+/, ''))); // NOT OK (can be absolute)
+});

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/TaintedPath/fs.js

@@ -0,0 +1,48 @@
+// Adapted from externs generated from TypeScript type definitions provided
+// by DefinitelyTyped, which is licensed under the MIT license; see file
+// DefinitelyTyped-LICENSE.
+// Type definitions for Node.js 10.5.x
+// Project: http://nodejs.org/
+// Definitions by: Microsoft TypeScript <http://typescriptlang.org>
+//                 DefinitelyTyped <https://github.com/DefinitelyTyped/DefinitelyTyped>
+//                 Parambir Singh <https://github.com/parambirs>
+//                 Christian Vaagland Tellnes <https://github.com/tellnes>
+//                 Wilco Bakker <https://github.com/WilcoBakker>
+//                 Nicolas Voigt <https://github.com/octo-sniffle>
+//                 Chigozirim C. <https://github.com/smac89>
+//                 Flarna <https://github.com/Flarna>
+//                 Mariusz Wiktorczyk <https://github.com/mwiktorczyk>
+//                 wwwy3y3 <https://github.com/wwwy3y3>
+//                 Deividas Bakanas <https://github.com/DeividasBakanas>
+//                 Kelvin Jin <https://github.com/kjin>
+//                 Alvis HT Tang <https://github.com/alvis>
+//                 Sebastian Silbermann <https://github.com/eps1lon>
+//                 Hannes Magnusson <https://github.com/Hannes-Magnusson-CK>
+//                 Alberto Schiabel <https://github.com/jkomyno>
+//                 Klaus Meinhardt <https://github.com/ajafff>
+//                 Huw <https://github.com/hoo29>
+//                 Nicolas Even <https://github.com/n-e>
+//                 Bruno Scheufler <https://github.com/brunoscheufler>
+//                 Mohsen Azimi <https://github.com/mohsen1>
+//                 Hoàng Văn Khải <https://github.com/KSXGitHub>
+//                 Alexander T. <https://github.com/a-tarasyuk>
+//                 Lishude <https://github.com/islishude>
+//                 Andrew Makarov <https://github.com/r3nya>
+//                 Zane Hannan AU <https://github.com/ZaneHannanAU>
+// Definitions: https://github.com/DefinitelyTyped/DefinitelyTyped
+
+
+/**
+ * @externs
+ */
+
+var fs = {};
+
+/**
+ * @param {string} filename
+ * @param {string} encoding
+ * @return {string}
+ */
+fs.readFileSync = function(filename, encoding) {};
+
+module.exports = fs;

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/TaintedPath/my-async-fs-module.js

@@ -0,0 +1,14 @@
+const fs = require('fs');
+const {promisify} = require('bluebird');
+
+const methods = [
+  'readFile',
+  'writeFile',
+  'readFileSync',
+  'writeFileSync'
+];
+
+module.exports = methods.reduce((obj, method) => {
+  obj[method] = promisify(fs[method]);
+  return obj;
+}, {});

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/TaintedPath/my-fs-module.js

@@ -0,0 +1,7 @@
+exports.require = function(special) {
+	if (special) {
+		return require("fs");
+	} else {
+		return require("original-fs");
+	}
+};

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/TaintedPath/normalizedPaths.js

@@ -0,0 +1,373 @@
+var fs = require('fs'),
+    express = require('express'),
+    url = require('url'),
+    sanitize = require('sanitize-filename'),
+    pathModule = require('path')
+    ;
+
+let app = express();
+
+app.get('/basic', (req, res) => {
+  let path = req.query.path;
+
+  fs.readFileSync(path); // NOT OK
+  fs.readFileSync('./' + path); // NOT OK
+  fs.readFileSync(path + '/index.html'); // NOT OK
+  fs.readFileSync(pathModule.join(path, 'index.html')); // NOT OK
+  fs.readFileSync(pathModule.join('/home/user/www', path)); // NOT OK
+});
+
+app.get('/normalize', (req, res) => {
+  let path = pathModule.normalize(req.query.path);
+
+  fs.readFileSync(path); // NOT OK
+  fs.readFileSync('./' + path); // NOT OK
+  fs.readFileSync(path + '/index.html'); // NOT OK
+  fs.readFileSync(pathModule.join(path, 'index.html')); // NOT OK
+  fs.readFileSync(pathModule.join('/home/user/www', path)); // NOT OK
+});
+
+app.get('/normalize-notAbsolute', (req, res) => {
+  let path = pathModule.normalize(req.query.path);
+
+  if (pathModule.isAbsolute(path))
+    return;
+   
+  fs.readFileSync(path); // NOT OK
+
+  if (!path.startsWith("."))
+    fs.readFileSync(path); // OK
+  else
+    fs.readFileSync(path); // NOT OK - wrong polarity
+  
+  if (!path.startsWith(".."))
+    fs.readFileSync(path); // OK
+  
+  if (!path.startsWith("../"))
+    fs.readFileSync(path); // OK
+
+  if (!path.startsWith(".." + pathModule.sep))
+    fs.readFileSync(path); // OK
+});
+
+app.get('/normalize-noInitialDotDot', (req, res) => {
+  let path = pathModule.normalize(req.query.path);
+  
+  if (path.startsWith(".."))
+    return;
+
+  fs.readFileSync(path); // NOT OK - could be absolute
+
+  fs.readFileSync("./" + path); // OK - coerced to relative
+
+  fs.readFileSync(path + "/index.html"); // NOT OK - not coerced
+
+  if (!pathModule.isAbsolute(path))
+    fs.readFileSync(path); // OK
+  else
+    fs.readFileSync(path); // NOT OK
+});
+
+app.get('/prepend-normalize', (req, res) => {
+  // Coerce to relative prior to normalization
+  let path = pathModule.normalize('./' + req.query.path);
+
+  if (!path.startsWith(".."))
+    fs.readFileSync(path); // OK
+  else
+     fs.readFileSync(path); // NOT OK
+});
+
+app.get('/absolute', (req, res) => {
+  let path = req.query.path;
+  
+  if (!pathModule.isAbsolute(path))
+    return;
+
+  res.write(fs.readFileSync(path)); // NOT OK
+
+  if (path.startsWith('/home/user/www'))
+    res.write(fs.readFileSync(path)); // NOT OK - can still contain '../'
+});
+
+app.get('/normalized-absolute', (req, res) => {
+  let path = pathModule.normalize(req.query.path);
+  
+  if (!pathModule.isAbsolute(path))
+    return;
+  
+  res.write(fs.readFileSync(path)); // NOT OK
+
+  if (path.startsWith('/home/user/www'))
+    res.write(fs.readFileSync(path)); // OK
+});
+
+app.get('/combined-check', (req, res) => {
+  let path = pathModule.normalize(req.query.path);
+  
+  // Combined absoluteness and folder check in one startsWith call
+  if (path.startsWith("/home/user/www"))
+    fs.readFileSync(path); // OK
+
+  if (path[0] !== "/" && path[0] !== ".")
+    fs.readFileSync(path); // OK
+});
+
+app.get('/realpath', (req, res) => {
+  let path = fs.realpathSync(req.query.path);
+
+  fs.readFileSync(path); // NOT OK
+  fs.readFileSync(pathModule.join(path, 'index.html')); // NOT OK
+
+  if (path.startsWith("/home/user/www"))
+    fs.readFileSync(path); // OK - both absolute and normalized before check
+    
+  fs.readFileSync(pathModule.join('.', path)); // OK - normalized and coerced to relative
+  fs.readFileSync(pathModule.join('/home/user/www', path)); // OK
+});
+
+app.get('/coerce-relative', (req, res) => {
+  let path = pathModule.join('.', req.query.path);
+
+  if (!path.startsWith('..'))
+    fs.readFileSync(path); // OK
+  else
+    fs.readFileSync(path); // NOT OK
+});
+
+app.get('/coerce-absolute', (req, res) => {
+  let path = pathModule.join('/home/user/www', req.query.path);
+
+  if (path.startsWith('/home/user/www'))
+    fs.readFileSync(path); // OK
+  else
+    fs.readFileSync(path); // NOT OK
+});
+
+app.get('/concat-after-normalization', (req, res) => {
+  let path = 'foo/' + pathModule.normalize(req.query.path);
+
+  if (!path.startsWith('..'))
+    fs.readFileSync(path); // NOT OK - prefixing foo/ invalidates check
+  else
+    fs.readFileSync(path); // NOT OK
+
+  if (!path.includes('..'))
+    fs.readFileSync(path); // OK
+});
+
+app.get('/noDotDot', (req, res) => {
+  let path = pathModule.normalize(req.query.path);
+
+  if (path.includes('..'))
+    return;
+
+  fs.readFileSync(path); // NOT OK - can still be absolute
+
+  if (!pathModule.isAbsolute(path))
+    fs.readFileSync(path); // OK
+  else
+    fs.readFileSync(path); // NOT OK
+});
+
+app.get('/join-regression', (req, res) => {
+  let path = req.query.path;
+
+  // Regression test for a specific corner case:
+  // Some guard nodes sanitize both branches, but for a different set of flow labels.
+  // Verify that this does not break anything.
+  if (pathModule.isAbsolute(path)) {path;} else {path;}
+  if (path.startsWith('/')) {path;} else {path;}
+  if (path.startsWith('/x')) {path;} else {path;}
+  if (path.startsWith('.')) {path;} else {path;}
+
+  fs.readFileSync(path); // NOT OK
+
+  if (pathModule.isAbsolute(path))
+    fs.readFileSync(path); // NOT OK
+  else
+    fs.readFileSync(path); // NOT OK
+
+  if (path.includes('..'))
+    fs.readFileSync(path); // NOT OK
+  else
+    fs.readFileSync(path); // NOT OK
+
+  if (!path.includes('..') && !pathModule.isAbsolute(path))
+    fs.readFileSync(path); // OK
+  else
+    fs.readFileSync(path); // NOT OK
+
+  let normalizedPath = pathModule.normalize(path);
+  if (normalizedPath.startsWith('/home/user/www'))
+    fs.readFileSync(normalizedPath); // OK
+  else
+    fs.readFileSync(normalizedPath); // NOT OK
+
+  if (normalizedPath.startsWith('/home/user/www') || normalizedPath.startsWith('/home/user/public'))
+    fs.readFileSync(normalizedPath); // OK - but flagged anyway [INCONSISTENCY]
+  else
+    fs.readFileSync(normalizedPath); // NOT OK
+});
+
+app.get('/decode-after-normalization', (req, res) => {
+  let path = pathModule.normalize(req.query.path);
+  
+  if (!pathModule.isAbsolute(path) && !path.startsWith('..'))
+    fs.readFileSync(path); // OK
+
+  path = decodeURIComponent(path);
+
+  if (!pathModule.isAbsolute(path) && !path.startsWith('..'))
+    fs.readFileSync(path); // NOT OK - not normalized
+});
+
+app.get('/replace', (req, res) => {
+  let path = pathModule.normalize(req.query.path).replace(/%20/g, ' ');
+  if (!pathModule.isAbsolute(path)) {
+    fs.readFileSync(path); // NOT OK
+
+    path = path.replace(/\.\./g, '');
+    fs.readFileSync(path); // OK
+  }
+});
+
+app.get('/resolve-path', (req, res) => {
+  let path = pathModule.resolve(req.query.path);
+
+  fs.readFileSync(path); // NOT OK
+
+  var self = something();
+	
+  if (path.substring(0, self.dir.length) === self.dir)
+    fs.readFileSync(path); // OK
+  else
+    fs.readFileSync(path); // NOT OK - wrong polarity
+
+  if (path.slice(0, self.dir.length) === self.dir)
+    fs.readFileSync(path); // OK
+  else
+    fs.readFileSync(path); // NOT OK - wrong polarity
+});
+
+app.get('/relative-startswith', (req, res) => {
+  let path = pathModule.resolve(req.query.path);
+
+  fs.readFileSync(path); // NOT OK
+
+  var self = something();
+	
+  var relative = pathModule.relative(self.webroot, path);
+  if(relative.startsWith(".." + pathModule.sep) || relative == "..") {
+    fs.readFileSync(path); // NOT OK! 
+  } else {
+    fs.readFileSync(path); // OK! 
+  }
+
+  let newpath = pathModule.normalize(path);
+  var relativePath = pathModule.relative(pathModule.normalize(workspaceDir), newpath);
+  if (relativePath.indexOf('..' + pathModule.sep) === 0) {
+    fs.readFileSync(newpath); // NOT OK!
+  } else {
+    fs.readFileSync(newpath); // OK!
+  }
+
+  let newpath = pathModule.normalize(path);
+  var relativePath = pathModule.relative(pathModule.normalize(workspaceDir), newpath);
+  if (relativePath.indexOf('../') === 0) {
+    fs.readFileSync(newpath); // NOT OK!
+  } else {
+    fs.readFileSync(newpath); // OK! 
+  }
+
+  let newpath = pathModule.normalize(path);
+  var relativePath = pathModule.relative(pathModule.normalize(workspaceDir), newpath);
+  if (pathModule.normalize(relativePath).indexOf('../') === 0) {
+    fs.readFileSync(newpath); // NOT OK!
+  } else {
+    fs.readFileSync(newpath); // OK! 
+  }
+
+  let newpath = pathModule.normalize(path);
+  var relativePath = pathModule.relative(pathModule.normalize(workspaceDir), newpath);
+  if (pathModule.normalize(relativePath).indexOf('../')) {
+    fs.readFileSync(newpath); // OK!
+  } else {
+    fs.readFileSync(newpath); // NOT OK! 
+  }
+});
+
+var isPathInside = require("is-path-inside"),
+    pathIsInside = require("path-is-inside");
+app.get('/pseudo-normalizations', (req, res) => {
+	let path = req.query.path;
+	fs.readFileSync(path); // NOT OK
+	if (isPathInside(path, SAFE)) {
+		fs.readFileSync(path); // OK
+		return;
+	} else {
+		fs.readFileSync(path); // NOT OK
+
+	}
+	if (pathIsInside(path, SAFE)) {
+		fs.readFileSync(path); // NOT OK - can be of the form 'safe/directory/../../../etc/passwd'
+		return;
+	} else {
+		fs.readFileSync(path); // NOT OK
+
+	}
+
+	let normalizedPath = pathModule.join(SAFE, path);
+	if (pathIsInside(normalizedPath, SAFE)) {
+		fs.readFileSync(normalizedPath); // OK
+		return;
+	} else {
+		fs.readFileSync(normalizedPath); // NOT OK
+	}
+
+	if (pathIsInside(normalizedPath, SAFE)) {
+		fs.readFileSync(normalizedPath); // OK
+		return;
+	} else {
+		fs.readFileSync(normalizedPath); // NOT OK
+
+	}
+
+});
+
+app.get('/yet-another-prefix', (req, res) => {
+	let path = pathModule.resolve(req.query.path);
+
+	fs.readFileSync(path); // NOT OK
+
+	var abs = pathModule.resolve(path); 
+
+	if (abs.indexOf(root) !== 0) {
+		fs.readFileSync(path); // NOT OK
+		return;
+    }
+	fs.readFileSync(path); // OK
+});
+
+var rootPath = process.cwd();
+app.get('/yet-another-prefix2', (req, res) => {
+  let path = req.query.path;
+
+  fs.readFileSync(path); // NOT OK
+
+  var requestPath = pathModule.join(rootPath, path);
+
+  var targetPath;
+  if (!allowPath(requestPath, rootPath)) {
+    targetPath = rootPath;
+    fs.readFileSync(requestPath); // NOT OK
+  } else {
+    targetPath = requestPath;
+    fs.readFileSync(requestPath); // OK
+  }
+  fs.readFileSync(targetPath); // OK
+
+  function allowPath(requestPath, rootPath) {
+    return requestPath.indexOf(rootPath) === 0;
+  }
+});

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/TaintedPath/other-fs-libraries.js

@@ -0,0 +1,53 @@
+var http = require("http"),
+  url = require("url"),
+  fs = require("fs"),
+  gracefulFs = require("graceful-fs"),
+  fsExtra = require("fs-extra"),
+  originalFs = require("original-fs");
+
+var server = http.createServer(function(req, res) {
+  var path = url.parse(req.url, true).query.path;
+
+  fs.readFileSync(path); // NOT OK
+  gracefulFs.readFileSync(path); // NOT OK
+  fsExtra.readFileSync(path); // NOT OK
+  originalFs.readFileSync(path); // NOT OK
+
+  getFsModule(true).readFileSync(path); // NOT OK
+  getFsModule(false).readFileSync(path); // NOT OK
+
+  require("./my-fs-module").require(true).readFileSync(path); // NOT OK
+
+  let flexibleModuleName = require(process.versions["electron"]
+    ? "original-fs"
+    : "fs");
+  flexibleModuleName.readFileSync(path); // NOT OK
+});
+
+function getFsModule(special) {
+  if (special) {
+    return require("fs");
+  } else {
+    return require("original-fs");
+  }
+}
+
+var util = require("util");
+
+http.createServer(function(req, res) {
+  var path = url.parse(req.url, true).query.path;
+
+  util.promisify(fs.readFileSync)(path); // NOT OK
+  require("bluebird").promisify(fs.readFileSync)(path); // NOT OK
+  require("bluebird").promisifyAll(fs).readFileSync(path); // NOT OK
+});
+
+
+const asyncFS = require("./my-async-fs-module");
+
+http.createServer(function(req, res) {
+  var path = url.parse(req.url, true).query.path;
+
+  fs.readFileSync(path); // NOT OK
+  asyncFS.readFileSync(path); // NOT OK
+});

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/TaintedPath/pupeteer.js

@@ -0,0 +1,18 @@
+const puppeteer = require('puppeteer');
+const parseTorrent = require('parse-torrent');
+
+(async () => {
+    let tainted = "dir/" + parseTorrent(torrent).name + ".torrent.data";
+
+    const browser = await puppeteer.launch();
+    const page = await browser.newPage();
+    await page.pdf({ path: tainted, format: 'a4' });
+
+    const pages = await browser.pages();
+    for (let i = 0; i < something(); i++) {
+        pages[i].screenshot({ path: tainted });
+    }
+
+    await browser.close();
+})();
+

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/TaintedPath/tainted-access-paths.js

@@ -0,0 +1,34 @@
+var fs = require('fs'),
+    http = require('http'),
+    url = require('url');
+
+var server = http.createServer(function(req, res) {
+  let path = url.parse(req.url, true).query.path;
+  
+  fs.readFileSync(path); // NOT OK
+
+  var obj = bla ? something() : path;
+
+  fs.readFileSync(obj.sub); // NOT OK
+
+  obj.sub = "safe";
+
+  fs.readFileSync(obj.sub); // OK
+
+  obj.sub2 = "safe";
+  if (random()) {
+    fs.readFileSync(obj.sub2); // OK
+  }
+
+  if (random()) {
+    obj.sub3 = "safe"
+  }
+  fs.readFileSync(obj.sub3); // NOT OK
+
+  obj.sub4 = 
+    fs.readFileSync(obj.sub4) ? // NOT OK
+      fs.readFileSync(obj.sub4) : // NOT OK
+      fs.readFileSync(obj.sub4); // NOT OK
+});
+
+server.listen();

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/TaintedPath/tainted-array-steps.js

@@ -0,0 +1,17 @@
+var fs = require('fs'),
+    http = require('http'),
+    url = require('url'),
+    sanitize = require('sanitize-filename'),
+    pathModule = require('path')
+    ;
+
+var server = http.createServer(function(req, res) {
+  let path = url.parse(req.url, true).query.path;
+  res.write(fs.readFileSync(['public', path].join('/'))); // BAD - but not flagged because we have no array-steps [INCONSISTENCY]
+
+  let parts = ['public', path];
+  parts = parts.map(x => x.toLowerCase());
+  res.write(fs.readFileSync(parts.join('/'))); // BAD - but not flagged because we have no array-steps [INCONSISTENCY]
+});
+
+server.listen();

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/TaintedPath/tainted-require.js

@@ -0,0 +1,8 @@
+var express = require('express');
+
+var app = express();
+
+app.get('/some/path', function(req, res) {
+  // BAD: loading a module based on un-sanitized query parameters
+  var m = require(req.param("module"));
+});

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/TaintedPath/tainted-sendFile.js

@@ -0,0 +1,28 @@
+var express = require('express');
+let path = require('path');
+
+var app = express();
+
+app.get('/some/path/:x', function(req, res) {
+  // BAD: sending a file based on un-sanitized query parameters
+  res.sendFile(req.param("gimme"));
+  // BAD: same as above
+  res.sendfile(req.param("gimme"));
+
+  // GOOD: ensures files cannot be accessed outside of root folder
+  res.sendFile(req.param("gimme"), { root: process.cwd() });
+  // GOOD: ensures files cannot be accessed outside of root folder
+  res.sendfile(req.param("gimme"), { root: process.cwd() });
+
+  // BAD: doesn't help if user controls root
+  res.sendFile(req.param("file"), { root: req.param("dir") });
+
+  let homeDir = path.resolve('.');
+  res.sendFile(homeDir + '/data/' + req.params.x); // OK: sendFile disallows ../
+  res.sendfile('data/' + req.params.x); // OK: sendfile disallows ../
+
+  res.sendFile(path.resolve('data', req.params.x)); // NOT OK
+  res.sendfile(path.join('data', req.params.x)); // NOT OK
+
+  res.sendFile(homeDir + path.join('data', req.params.x)); // kinda OK - can only escape from 'data/'
+});

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/TaintedPath/tainted-string-steps.js

@@ -0,0 +1,31 @@
+var fs = require('fs'),
+    http = require('http'),
+    url = require('url');
+
+var server = http.createServer(function(req, res) {
+  let path = url.parse(req.url, true).query.path;
+	fs.readFileSync(path.substring(i, j)); // OK
+	fs.readFileSync(path.substring(4)); // NOT OK
+	fs.readFileSync(path.substring(0, i)); // NOT OK
+	fs.readFileSync(path.substr(4)); // NOT OK
+	fs.readFileSync(path.slice(4)); // NOT OK
+
+	fs.readFileSync(path.concat(unknown)); // NOT OK
+	fs.readFileSync(unknown.concat(path)); // NOT OK
+	fs.readFileSync(unknown.concat(unknown, path)); // NOT OK
+
+	fs.readFileSync(path.trim()); // NOT OK
+	fs.readFileSync(path.toLowerCase()); // NOT OK
+
+	fs.readFileSync(path.split('/')); // OK (readFile throws an exception when the filename is an array)
+	fs.readFileSync(path.split('/')[0]); // OK -- for now
+	fs.readFileSync(path.split('/')[i]); // NOT OK
+	fs.readFileSync(path.split(/\//)[i]); // NOT OK
+	fs.readFileSync(path.split("?")[0]); // NOT OK
+	fs.readFileSync(path.split(unknown)[i]); // NOT OK -- but not yet flagged [INCONSISTENCY]
+	fs.readFileSync(path.split(unknown).whatever); // OK -- but still flagged [INCONSISTENCY]
+	fs.readFileSync(path.split(unknown)); // NOT OK
+	fs.readFileSync(path.split("?")[i]); // NOT OK -- but not yet flagged [INCONSISTENCY]
+});
+
+server.listen();

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/TaintedPath/torrents.js

@@ -0,0 +1,8 @@
+const parseTorrent = require('parse-torrent'),
+      fs = require('fs');
+
+function getTorrentData(dir, torrent){
+	let name = parseTorrent(torrent).name,
+	    loc = dir + "/" + name + ".torrent.data";
+	return fs.readFileSync(loc); // NOT OK
+}

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/TaintedPath/typescript.ts

@@ -0,0 +1,34 @@
+var fs = require('fs'),
+    http = require('http'),
+    url = require('url'),
+    sanitize = require('sanitize-filename'),
+    pathModule = require('path')
+    ;
+
+var server = http.createServer(function(req, res) {
+  let path = url.parse(req.url, true).query.path;
+
+  // BAD: This could read any file on the file system
+  res.write(fs.readFileSync(path));
+
+  if (path === 'foo.txt')
+    res.write(fs.readFileSync(path)); // GOOD: Path is compared to white-list
+
+  let path2 = path;
+  path2 ||= res.write(fs.readFileSync(path2)); // GOOD: path is falsy
+
+  let path3 = path;
+  path3 &&= res.write(fs.readFileSync(path3)); // BAD: path is truthy
+
+  let path4 = path;
+  path4 ??= res.write(fs.readFileSync(path4)); // GOOD - path is null or undefined - but we don't capture that. [INCONSISTENCY]
+
+  let path5 = path;
+  path5 &&= "clean";
+  res.write(fs.readFileSync(path5)); // GOOD: path is either falsy or "clean";
+
+  let path6 = path;
+  path6 ||= "clean";
+  res.write(fs.readFileSync(path6)); // BAD: path can still be tainted
+
+});

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/TaintedPath/views.js

@@ -0,0 +1 @@
+module.exports = (req, res) => res.render(req.params[0]);

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/Xss/DomBasedXss/addEventListener.js

@@ -0,0 +1,17 @@
+this.addEventListener('message', function(event) {
+    document.write(event.data); // NOT OK
+})
+
+this.addEventListener('message', function({data}) {
+    document.write(data); // NOT OK
+})
+
+function test() {
+    function foo(x, event, y) {
+        document.write(x.data);     // OK
+        document.write(event.data); // NOT OK
+        document.write(y.data);     // OK
+    }
+
+    window.addEventListener("message", foo.bind(null, {data: 'items'}));
+}

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/Xss/DomBasedXss/angular2-client.ts

@@ -0,0 +1,46 @@
+import { Component, OnInit, DomSanitizer as DomSanitizer2 } from '@angular/core';
+import { ɵgetDOM } from '@angular/common';
+import { ActivatedRoute, ActivatedRouteSnapshot, Router } from '@angular/router';
+import { DomSanitizer } from '@angular/platform-browser';
+
+@Component({
+  selector: 'app-root',
+  templateUrl: './app.component.html',
+  styleUrls: ['./app.component.css']
+})
+export class AppComponent implements OnInit {
+  title = 'my-app';
+
+  constructor(
+    private route: ActivatedRoute,
+    private sanitizer: DomSanitizer,
+    private router: Router,
+    private sanitizer2: DomSanitizer2
+  ) {}
+
+  ngOnInit() {
+    this.sanitizer.bypassSecurityTrustHtml(ɵgetDOM().getLocation().href); // NOT OK
+
+    this.sanitizer.bypassSecurityTrustHtml(this.route.snapshot.params.foo); // NOT OK
+    this.sanitizer.bypassSecurityTrustHtml(this.route.snapshot.queryParams.foo); // NOT OK
+    this.sanitizer.bypassSecurityTrustHtml(this.route.snapshot.fragment); // NOT OK
+    this.sanitizer.bypassSecurityTrustHtml(this.route.snapshot.paramMap.get('foo')); // NOT OK
+    this.sanitizer.bypassSecurityTrustHtml(this.route.snapshot.queryParamMap.get('foo')); // NOT OK
+    this.route.paramMap.subscribe(map => {
+      this.sanitizer.bypassSecurityTrustHtml(map.get('foo')); // NOT OK
+    });
+
+    this.sanitizer.bypassSecurityTrustHtml(this.route.snapshot.url[1].path); // NOT OK - though depends on route config
+    this.sanitizer.bypassSecurityTrustHtml(this.route.snapshot.url[1].parameters.x); // NOT OK
+    this.sanitizer.bypassSecurityTrustHtml(this.route.snapshot.url[1].parameterMap.get('x')); // NOT OK
+    this.sanitizer.bypassSecurityTrustHtml(this.route.snapshot.url[1].parameterMap.params.x); // NOT OK
+
+    this.sanitizer.bypassSecurityTrustHtml(this.router.url); // NOT OK
+
+    this.sanitizer2.bypassSecurityTrustHtml(this.router.url); // NOT OK
+  }
+
+  someMethod(routeSnapshot: ActivatedRouteSnapshot) {
+    this.sanitizer.bypassSecurityTrustHtml(routeSnapshot.paramMap.get('foo')); // NOT OK
+  }
+}

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/Xss/DomBasedXss/classnames.js

@@ -0,0 +1,16 @@
+import classNames from 'classnames';
+import classNamesD from 'classnames/dedupe';
+import classNamesB from 'classnames/bind';
+import clsx from 'clsx';
+
+function main() {
+    document.body.innerHTML = `<span class="${classNames(window.name)}">Hello<span>`; // NOT OK
+    document.body.innerHTML = `<span class="${classNamesD(window.name)}">Hello<span>`; // NOT OK
+    document.body.innerHTML = `<span class="${classNamesB(window.name)}">Hello<span>`; // NOT OK
+    let unsafeStyle = classNames.bind({foo: window.name});
+    document.body.innerHTML = `<span class="${unsafeStyle('foo')}">Hello<span>`; // NOT OK
+    let safeStyle = classNames.bind({});
+    document.body.innerHTML = `<span class="${safeStyle(window.name)}">Hello<span>`; // NOT OK
+    document.body.innerHTML = `<span class="${safeStyle('foo')}">Hello<span>`; // OK
+    document.body.innerHTML = `<span class="${clsx(window.name)}">Hello<span>`; // NOT OK
+}

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/Xss/DomBasedXss/d3.js

@@ -0,0 +1,22 @@
+const d3 = require('d3');
+
+function getTaint() {
+    return window.name;
+}
+
+function doSomething() {
+    d3.select('#main')
+        .attr('width', 100)
+        .style('color', 'red')
+        .html(getTaint()) // NOT OK
+        .html(d => getTaint()) // NOT OK
+        .call(otherFunction)
+        .html(d => getTaint()); // NOT OK
+}
+
+
+function otherFunction(selection) {
+    selection
+        .attr('foo', 'bar')
+        .html(getTaint()); // NOT OK
+}

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/Xss/DomBasedXss/dates.js

@@ -0,0 +1,19 @@
+import dateFns from 'date-fns';
+import dateFnsFp from 'date-fns/fp';
+import dateFnsEsm from 'date-fns/esm';
+import moment from 'moment';
+import dateformat from 'dateformat';
+
+function main() {
+    let time = new Date();
+    let taint = decodeURIComponent(window.location.hash.substring(1));
+
+    document.body.innerHTML = `Time is ${dateFns.format(time, taint)}`; // NOT OK
+    document.body.innerHTML = `Time is ${dateFnsEsm.format(time, taint)}`; // NOT OK
+    document.body.innerHTML = `Time is ${dateFnsFp.format(taint)(time)}`; // NOT OK
+    document.body.innerHTML = `Time is ${dateFns.format(taint, time)}`; // OK - time arg is safe
+    document.body.innerHTML = `Time is ${dateFnsFp.format(time)(taint)}`; // OK - time arg is safe
+    document.body.innerHTML = `Time is ${moment(time).format(taint)}`; // NOT OK
+    document.body.innerHTML = `Time is ${moment(taint).format()}`; // OK
+    document.body.innerHTML = `Time is ${dateformat(time, taint)}`; // NOT OK
+}

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/Xss/DomBasedXss/encodeuri.js

@@ -0,0 +1,4 @@
+function test() {
+  let loc = window.location.href;
+  $('<a href="' + encodeURIComponent(loc) + '">click</a>'); // OK
+}

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/Xss/DomBasedXss/event-handler-receiver.js

@@ -0,0 +1,3 @@
+document.getElementById('my-id').onclick = function() {
+  this.parentNode.innerHTML = '<h2><a href="' + location.href + '">A link</a></h2>'; // NOT OK
+};

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/Xss/DomBasedXss/express.js

@@ -0,0 +1,11 @@
+var express = require('express');
+var app = express();
+
+import { JSDOM } from "jsdom";
+app.get('/some/path', function (req, res) {
+    // NOT OK
+    new JSDOM(req.param("wobble"), { runScripts: "dangerously" });
+
+    // OK
+    new JSDOM(req.param("wobble"), { runScripts: "outside-only" });
+});

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/Xss/DomBasedXss/externs.js

@@ -0,0 +1,59 @@
+// Adapted from the Google Closure externs; original copyright header included below.
+/*
+ * Copyright 2008 The Closure Compiler Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * @externs
+ */
+
+/**
+ * @interface
+ */
+function EventTarget() {}
+
+/**
+ * Stub for the DOM hierarchy.
+ *
+ * @constructor
+ * @extends {EventTarget}
+ */
+function DomObjectStub() {}
+
+/**
+ * @type {!DomObjectStub}
+ */
+DomObjectStub.prototype.body;
+
+/**
+ * @type {!DomObjectStub}
+ */
+DomObjectStub.prototype.value;
+
+/**
+ * @type {!DomObjectStub}
+ */
+var document;
+
+/**
+ * @constructor
+ * @implements {EventTarget}
+ */
+function Node() {}
+
+/**
+ * @type {Node}
+ */
+Node.prototype.parentNode;

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/Xss/DomBasedXss/jquery.js

@@ -0,0 +1,17 @@
+function test() {
+  var tainted = document.location.search
+
+  $(tainted);                        // OK - location.search starts with '?'
+  $("body", tainted);                // OK
+  $("." + tainted);                  // OK
+  $("<div id=\"" + tainted + "\">"); // NOT OK
+  $("body").html("XSS: " + tainted); // NOT OK
+  $(window.location.hash);           // OK - location.hash starts with '#'
+  $("<b>" + location.toString() + "</b>"); // NOT OK
+
+  // Not related to jQuery, but the handling of $() should not affect this sink
+  let elm = document.getElementById('x');
+  elm.innerHTML = decodeURIComponent(window.location.hash); // NOT OK
+  elm.innerHTML = decodeURIComponent(window.location.search); // NOT OK
+  elm.innerHTML = decodeURIComponent(window.location.toString()); // NOT OK
+}

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/Xss/DomBasedXss/jwt-server.js

@@ -0,0 +1,13 @@
+var express = require('express');
+var app = express();
+import jwt from "jsonwebtoken";
+
+import { JSDOM } from "jsdom";
+app.get('/some/path', function (req, res) {
+    var taint = req.param("wobble");
+
+    jwt.verify(taint, 'my-secret-key', function (err, decoded) {
+        // NOT OK
+        new JSDOM(decoded.foo, { runScripts: "dangerously" });
+    });
+});

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/Xss/DomBasedXss/jwt.js

@@ -0,0 +1,7 @@
+import jwt_decode from "jwt-decode";
+import $ from "jquery"
+
+$.post(loginUrl(), {data: "foo"}, (data, xhr) => {
+    var decoded = jwt_decode(data);
+    $.jGrowl(decoded); // NOT OK - but only flagged with additional sources [INCONSISTENCY]
+});

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/Xss/DomBasedXss/nodemailer.js

@@ -0,0 +1,15 @@
+let nodemailer = require('nodemailer');
+let express = require('express');
+let app = express();
+let backend = require('./backend');
+
+app.post('/private_message', (req, res) => {
+  let transport = nodemailer.createTransport({});
+  transport.sendMail({
+    from: 'webmaster@example.com',
+    to: backend.getUserEmail(req.query.receiver),
+    subject: 'Private message',
+    text: `Hi, you got a message from someone. ${req.query.message}.`, // OK
+    html: `Hi, you got a message from someone. ${req.query.message}.`, // NOT OK
+  });
+});

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/Xss/DomBasedXss/optionalSanitizer.js

@@ -0,0 +1,46 @@
+function test() {
+  var target = document.location.search
+
+  $('myId').html(sanitize ? DOMPurify.sanitize(target) : target); // OK
+
+  $('myId').html(target); // NOT OK
+
+  var tainted = target;
+  $('myId').html(tainted); // NOT OK
+  if (sanitize) {
+    tainted = DOMPurify.sanitize(tainted);
+  }
+  $('myId').html(tainted); // OK
+
+  inner(target);
+  function inner(x) {
+    $('myId').html(x); // NOT OK
+    if (sanitize) {
+      x = DOMPurify.sanitize(x);
+    }
+    $('myId').html(x); // OK
+  }
+}
+
+function badSanitizer() {
+  var target = document.location.search
+
+  function sanitizeBad(x) {
+    return x; // No sanitization;
+  }
+  var tainted2 = target;
+  $('myId').html(tainted2); // NOT OK
+  if (sanitize) {
+    tainted2 = sanitizeBad(tainted2);
+  }
+  $('myId').html(tainted2); // NOT OK
+
+  var tainted3 = target;
+  $('myId').html(tainted3); // NOT OK
+  if (sanitize) {
+    tainted3 = sanitizeBad(tainted3);
+  }
+  $('myId').html(tainted3); // NOT OK
+
+  $('myId').html(sanitize ? sanitizeBad(target) : target); // NOT OK
+}

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/Xss/DomBasedXss/react-create-context.js

@@ -0,0 +1,3 @@
+import { createContext } from 'react';
+
+export let MyContext = createContext({root: null});

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/Xss/DomBasedXss/react-native.js

@@ -0,0 +1,10 @@
+import express from 'express';
+import { WebView } from 'react-native';
+
+var app = express();
+
+app.get('/some/path', function(req, res) {
+  let tainted = req.param("code");
+  <WebView html={tainted}/>;            // NOT OK
+  <WebView source={{html: tainted}}/>;  // NOT OK
+});

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/Xss/DomBasedXss/react-provide-context.js

@@ -0,0 +1,5 @@
+import { MyContext } from './react-create-context';
+
+export function renderMain() {
+    return <MyContext.Provider value={{root: document.body}}></MyContext.Provider>
+}

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/Xss/DomBasedXss/react-use-context.js

@@ -0,0 +1,20 @@
+import { useContext, Component } from 'react';
+import { MyContext } from './react-create-context';
+
+function useMyContext() {
+    return useContext(MyContext);
+}
+
+export function useDoc1() {
+    let { root } = useMyContext();
+    root.appendChild(window.name); // NOT OK
+}
+
+class C extends Component {
+    foo() {
+        let { root } = this.context;
+        root.appendChild(window.name); // NOT OK
+    }
+}
+
+C.contextType = MyContext;

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/Xss/DomBasedXss/react-use-state.js

@@ -0,0 +1,33 @@
+import { useState } from 'react';
+
+function initialState() {
+    let [state, setState] = useState(window.name);
+    return <div dangerouslySetInnerHTML={{__html: state}}></div>; // NOT OK
+}
+
+function setStateValue() {
+    let [state, setState] = useState('foo');
+    setState(window.name);
+    return <div dangerouslySetInnerHTML={{__html: state}}></div>; // NOT OK
+}
+
+function setStateValueLazy() {
+    let [state, setState] = useState('foo');
+    setState(() => window.name);
+    return <div dangerouslySetInnerHTML={{__html: state}}></div>; // NOT OK
+}
+
+function setStateValueLazy() {
+    let [state, setState] = useState('foo');
+    setState(prev => {
+        document.body.innerHTML = prev; // NOT OK
+    })
+    setState(() => window.name);
+}
+
+function setStateValueSafe() {
+    let [state, setState] = useState('foo');
+    setState('safe');
+    setState(() => 'also safe');
+    return <div dangerouslySetInnerHTML={{__html: state}}></div>; // OK
+}

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/Xss/DomBasedXss/sanitiser.js

@@ -0,0 +1,49 @@
+function escapeHtml(s) {
+    var amp = /&/g, lt = /</g, gt = />/g;
+    return s.toString()
+        .replace(amp, '&amp;')
+        .replace(lt, '&lt;')
+        .replace(gt, '&gt;');
+}
+
+function escapeAttr(s) {
+    return s.toString()
+         .replace(/'/g, '%22')
+         .replace(/"/g, '%27');
+}
+
+function test() {
+  var tainted = window.name;
+  var elt = document.createElement();
+  elt.innerHTML = "<a href=\"" + escapeAttr(tainted) + "\">" + escapeHtml(tainted) + "</a>"; // OK
+  elt.innerHTML = "<div>" + escapeAttr(tainted) + "</div>"; // NOT OK, but not flagged - [INCONSISTENCY]
+
+  const regex = /[<>'"&]/;
+  if (regex.test(tainted)) {
+    elt.innerHTML = '<b>' + tainted + '</b>'; // NOT OK
+  } else {
+    elt.innerHTML = '<b>' + tainted + '</b>'; // OK
+  }
+  if (!regex.test(tainted)) {
+    elt.innerHTML = '<b>' + tainted + '</b>'; // OK
+  } else {
+    elt.innerHTML = '<b>' + tainted + '</b>'; // NOT OK
+  }
+  if (regex.exec(tainted)) {
+    elt.innerHTML = '<b>' + tainted + '</b>'; // NOT OK
+  } else {
+    elt.innerHTML = '<b>' + tainted + '</b>'; // OK
+  }
+  if (regex.exec(tainted) != null) {
+    elt.innerHTML = '<b>' + tainted + '</b>'; // NOT OK
+  } else {
+    elt.innerHTML = '<b>' + tainted + '</b>'; // OK
+  }
+  if (regex.exec(tainted) == null) {
+    elt.innerHTML = '<b>' + tainted + '</b>'; // OK
+  } else {
+    elt.innerHTML = '<b>' + tainted + '</b>'; // NOT OK
+  }
+
+  elt.innerHTML = tainted.replace(/<\w+/g, ''); // NOT OK
+}