add debug queries

Co-authored-by: Henry Mercer <henrymercer@github.com>

javascript/ql/experimental/adaptivethreatmodeling/lib/experimental/adaptivethreatmodeling/EndpointFeatures.qll

@@ -16,102 +16,8 @@ private import FunctionBodyFeatures as FunctionBodyFeatures
 private string getTokenFeature(DataFlow::Node endpoint, string featureName) {
   // Performance optimization: Restrict feature extraction to endpoints we've explicitly asked to featurize.
   endpoint = any(FeaturizationConfig cfg).getAnEndpointToFeaturize() and
-  (
+  exists(EndpointFeature f | f.getName() = featureName and result = f.getValue(endpoint)) and
-    // Features for endpoints that are contained within a function.
+  featureName = getASupportedFeatureName()
-    exists(Function function |
-      function = FunctionBodyFeatures::getRepresentativeFunctionForEndpoint(endpoint)
-    |
-      // The name of the function that encloses the endpoint.
-      featureName = "enclosingFunctionName" and result = FunctionNames::getNameToFeaturize(function)
-      or
-      // A feature containing natural language tokens from the function that encloses the endpoint in
-      // the order that they appear in the source code.
-      featureName = "enclosingFunctionBody" and
-      result = FunctionBodyFeatures::getBodyTokensFeature(function)
-    )
-    or
-    result =
-      strictconcat(DataFlow::CallNode call, string component |
-        component = getACallBasedTokenFeatureComponent(endpoint, call, featureName)
-      |
-        component, " "
-      )
-    or
-    // The access path of the function being called, both with and without structural info, if the
-    // function being called originates from an external API. For example, the endpoint here:
-    //
-    // ```js
-    // const mongoose = require('mongoose'),
-    //   User = mongoose.model('User', null);
-    // User.findOne(ENDPOINT);
-    // ```
-    //
-    // would have a callee access path with structural info of
-    // `mongoose member model instanceorreturn member findOne instanceorreturn`, and a callee access
-    // path without structural info of `mongoose model findOne`.
-    //
-    // These features indicate that the callee comes from (reading the access path backwards) an
-    // instance of the `findOne` member of an instance of the `model` member of the `mongoose`
-    // external library.
-    exists(AccessPaths::Boolean includeStructuralInfo |
-      featureName =
-        "calleeAccessPath" +
-          any(string x | if includeStructuralInfo = true then x = "WithStructuralInfo" else x = "") and
-      result =
-        concat(API::Node node, string accessPath |
-          node.getInducingNode().(DataFlow::CallNode).getAnArgument() = endpoint and
-          AccessPaths::accessPaths(node, includeStructuralInfo, accessPath, _)
-        |
-          accessPath, " "
-        )
-    )
-  )
-}
-
-/**
- * Gets a value of the function-call-related token-based feature named `featureName` associated
- * with the function call `call` and the endpoint `endpoint`.
- *
- * This may in general report multiple strings, each containing a space-separated list of tokens.
- *
- * **Technical details:** This predicate can have multiple values per endpoint and feature name. As
- * a result, the results from this predicate must be concatenated together.  However concatenating
- * other features like the function body tokens is expensive, so for performance reasons we separate
- * out this predicate from those other features.
- */
-private string getACallBasedTokenFeatureComponent(
-  DataFlow::Node endpoint, DataFlow::CallNode call, string featureName
-) {
-  // Performance optimization: Restrict feature extraction to endpoints we've explicitly asked to featurize.
-  endpoint = any(FeaturizationConfig cfg).getAnEndpointToFeaturize() and
-  // Features for endpoints that are an argument to a function call.
-  endpoint = call.getAnArgument() and
-  (
-    // The name of the function being called, e.g. in a call `Artist.findOne(...)`, this is `findOne`.
-    featureName = "calleeName" and result = call.getCalleeName()
-    or
-    // The name of the receiver of the call, e.g. in a call `Artist.findOne(...)`, this is `Artist`.
-    featureName = "receiverName" and result = call.getReceiver().asExpr().(VarRef).getName()
-    or
-    // The argument index of the endpoint, e.g. in `f(a, endpoint, b)`, this is 1.
-    featureName = "argumentIndex" and
-    result = any(int argIndex | call.getArgument(argIndex) = endpoint).toString()
-    or
-    // The name of the API that the function being called originates from, if the function being
-    // called originates from an external API. For example, the endpoint here:
-    //
-    // ```js
-    // const mongoose = require('mongoose'),
-    //   User = mongoose.model('User', null);
-    // User.findOne(ENDPOINT);
-    // ```
-    //
-    // would have a callee API name of `mongoose`.
-    featureName = "calleeApiName" and
-    exists(API::Node apiNode |
-      AccessPaths::accessPaths(apiNode, false, _, result) and call = apiNode.getInducingNode()
-    )
-  )
 }
 
 /**
@@ -163,7 +69,8 @@ private module AccessPaths {
     API::Node node, Boolean includeStructuralInfo, string accessPath, string apiName
   ) {
     //node = API::moduleImport(result)
-    node = API::moduleImport(apiName) and accessPath = apiName
+    node = API::moduleImport(apiName) and
+    accessPath = apiName
     or
     exists(API::Node previousNode, string previousAccessPath |
       previousNode.getDepth() < node.getDepth() and
@@ -285,11 +192,27 @@ private module FunctionNames {
 
 /** Get a name of a supported generic token-based feature. */
 string getASupportedFeatureName() {
+  // allowlist of vetted features that are permitted in production
   result =
-    [
+    any(EndpointFeature f |
-      "enclosingFunctionName", "calleeName", "receiverName", "argumentIndex", "calleeApiName",
+      f instanceof EnclosingFunctionName or
-      "calleeAccessPath", "calleeAccessPathWithStructuralInfo", "enclosingFunctionBody"
+      f instanceof CalleeName or
-    ]
+      f instanceof ReceiverName or
+      f instanceof ArgumentIndex or
+      f instanceof CalleeApiName or
+      f instanceof CalleeAccessPath or
+      f instanceof CalleeAccessPathWithStructuralInfo or
+      f instanceof EnclosingFunctionBody or
+      f instanceof ContextFunctionInterfaces or
+      f instanceof ContextSurroundingFunctionParameters or
+      f instanceof FileImports or
+      f instanceof CalleeImports or
+      f instanceof CalleeFlexibleAccessPath or
+      f instanceof InputAccessPathFromCallee or
+      f instanceof InputArgumentIndex or
+      f instanceof AssignedToPropName or
+      f instanceof StringConcatenatedWith
+    ).getName()
 }
 
 /**
@@ -303,3 +226,700 @@ predicate tokenFeatures(DataFlow::Node endpoint, string featureName, string feat
   endpoint = any(FeaturizationConfig cfg).getAnEndpointToFeaturize() and
   featureValue = getTokenFeature(endpoint, featureName)
 }
+
+/**
+ * See EndpointFeature
+ */
+private newtype TEndpointFeature =
+  TEnclosingFunctionName() or
+  TCalleeName() or
+  TReceiverName() or
+  TArgumentIndex() or
+  TCalleeApiName() or
+  TCalleeAccessPath() or
+  TCalleeAccessPathWithStructuralInfo() or
+  TEnclosingFunctionBody() or
+  TFileImports() or
+  TCalleeImports() or
+  TCalleeFlexibleAccessPath() or
+  TInputAccessPathFromCallee() or
+  TInputArgumentIndex() or
+  TContextFunctionInterfaces() or
+  TContextSurroundingFunctionParameters() or
+  TAssignedToPropName() or
+  TStringConcatenatedWith()
+
+/**
+ * An implementation of an endpoint feature: produces feature names and values for used in ML.
+ */
+abstract class EndpointFeature extends TEndpointFeature {
+  /**
+   * Gets the name of the feature. Used by the ML model.
+   * Changes to the name of a feature requires training the model again.
+   */
+  abstract string getName();
+
+  /**
+   * Gets the value of the feature. Used by the ML model.
+   * Changes to the value of a feature requires training the model again.
+   */
+  abstract string getValue(DataFlow::Node endpoint);
+
+  string toString() { result = this.getName() }
+}
+
+/**
+ * The feature for the name of the function that encloses the endpoint.
+ */
+class EnclosingFunctionName extends EndpointFeature, TEnclosingFunctionName {
+  override string getName() { result = "enclosingFunctionName" }
+
+  override string getValue(DataFlow::Node endpoint) {
+    result =
+      FunctionNames::getNameToFeaturize(FunctionBodyFeatures::getRepresentativeFunctionForEndpoint(endpoint))
+  }
+}
+
+/**
+ * The feature for the name of the function being called, e.g. in a call `Artist.findOne(...)`, this is `findOne`.
+ */
+class CalleeName extends EndpointFeature, TCalleeName {
+  override string getName() { result = "calleeName" }
+
+  override string getValue(DataFlow::Node endpoint) {
+    result =
+      strictconcat(DataFlow::CallNode call, string component |
+        endpoint = call.getAnArgument() and component = call.getCalleeName()
+      |
+        component, " "
+      )
+  }
+}
+
+/**
+ * The feature for the name of the receiver of the call, e.g. in a call `Artist.findOne(...)`, this is `Artist`.
+ */
+class ReceiverName extends EndpointFeature, TReceiverName {
+  override string getName() { result = "receiverName" }
+
+  override string getValue(DataFlow::Node endpoint) {
+    result =
+      strictconcat(DataFlow::CallNode call, string component |
+        endpoint = call.getAnArgument() and
+        component = call.getReceiver().asExpr().(VarRef).getName()
+      |
+        component, " "
+      )
+  }
+}
+
+/**
+ * The feature for the argument index of the endpoint, e.g. in `f(a, endpoint, b)`, this is 1.
+ */
+class ArgumentIndex extends EndpointFeature, TArgumentIndex {
+  override string getName() { result = "argumentIndex" }
+
+  override string getValue(DataFlow::Node endpoint) {
+    result =
+      strictconcat(DataFlow::CallNode call, string component |
+        endpoint = call.getAnArgument() and
+        component = any(int argIndex | call.getArgument(argIndex) = endpoint).toString()
+      |
+        component, " "
+      )
+  }
+}
+
+/**
+ * The feature for the name of the API that the function being called originates from, if the function being
+ * called originates from an external API. For example, the endpoint here:
+ *
+ * ```js
+ * const mongoose = require('mongoose'),
+ *   User = mongoose.model('User', null);
+ * User.findOne(ENDPOINT);
+ * ```
+ */
+class CalleeApiName extends EndpointFeature, TCalleeApiName {
+  override string getName() { result = "calleeApiName" }
+
+  override string getValue(DataFlow::Node endpoint) {
+    result =
+      strictconcat(API::Node apiNode, string component |
+        endpoint = apiNode.getInducingNode().(DataFlow::CallNode).getAnArgument() and
+        AccessPaths::accessPaths(apiNode, false, _, component)
+      |
+        component, " "
+      )
+  }
+}
+
+/**
+ * The access path of the function being called, both without structural info, if the
+ * function being called originates from an external API. For example, the endpoint here:
+ *
+ * ```js
+ * const mongoose = require('mongoose'),
+ *   User = mongoose.model('User', null);
+ * User.findOne(ENDPOINT);
+ * ```
+ *
+ * would have a callee access path without structural info of `mongoose model findOne`.
+ */
+class CalleeAccessPath extends EndpointFeature, TCalleeAccessPath {
+  override string getName() { result = "calleeAccessPath" }
+
+  override string getValue(DataFlow::Node endpoint) {
+    result =
+      concat(API::Node node, string accessPath |
+        node.getInducingNode().(DataFlow::CallNode).getAnArgument() = endpoint and
+        AccessPaths::accessPaths(node, false, accessPath, _)
+      |
+        accessPath, " "
+      )
+  }
+}
+
+/**
+ * The access path of the function being called, both with structural info, if the
+ * function being called originates from an external API. For example, the endpoint here:
+ *
+ * ```js
+ * const mongoose = require('mongoose'),
+ *   User = mongoose.model('User', null);
+ * User.findOne(ENDPOINT);
+ * ```
+ *
+ * would have a callee access path with structural info of
+ * `mongoose member model instanceorreturn member findOne instanceorreturn`
+ *
+ * These features indicate that the callee comes from (reading the access path backwards) an
+ * instance of the `findOne` member of an instance of the `model` member of the `mongoose`
+ * external library.
+ */
+class CalleeAccessPathWithStructuralInfo extends EndpointFeature,
+  TCalleeAccessPathWithStructuralInfo {
+  override string getName() { result = "calleeAccessPathWithStructuralInfo" }
+
+  override string getValue(DataFlow::Node endpoint) {
+    result =
+      concat(API::Node node, string accessPath |
+        node.getInducingNode().(DataFlow::CallNode).getAnArgument() = endpoint and
+        AccessPaths::accessPaths(node, true, accessPath, _)
+      |
+        accessPath, " "
+      )
+  }
+}
+
+/**
+ * The feature for the natural language tokens from the function that encloses the endpoint in
+ *    the order that they appear in the source code.
+ */
+class EnclosingFunctionBody extends EndpointFeature, TEnclosingFunctionBody {
+  override string getName() { result = "enclosingFunctionBody" }
+
+  override string getValue(DataFlow::Node endpoint) {
+    endpoint = any(FeaturizationConfig cfg).getAnEndpointToFeaturize() and
+    result =
+      FunctionBodyFeatures::getBodyTokensFeature(FunctionBodyFeatures::getRepresentativeFunctionForEndpoint(endpoint))
+  }
+}
+
+/**
+ * The feature for the imports defined in the file containing an endpoint.
+ *
+ * ### Example
+ *
+ * ```javascript
+ * import { findOne } from 'mongoose';
+ * import * as _ from 'lodash';
+ * const pg = require('pg');
+ *
+ * // ...
+ * ```
+ *
+ * In this file, all endpoints will have the value `lodash mongoose pg` for the feature `fileImports`.
+ */
+class FileImports extends EndpointFeature, TFileImports {
+  override string getName() { result = "fileImports" }
+
+  override string getValue(DataFlow::Node endpoint) {
+    result = SyntacticUtilities::getImportPathsForFile(endpoint.getFile())
+  }
+}
+
+/**
+ * The feature for the function parameters of the functions that enclose an endpoint.
+ *
+ * ### Example
+ * ```javascript
+ * function f(a, b) {
+ *   // ...
+ *   const g = (c, d) => x.foo(endpoint);
+ * //                          ^^^^^^^^
+ * }
+ * ```
+ * In the above example, the feature for the marked endpoint has value '(a, b)\n(c, d)'.
+ */
+class ContextSurroundingFunctionParameters extends EndpointFeature,
+  TContextSurroundingFunctionParameters {
+  override string getName() { result = "contextSurroundingFunctionParameters" }
+
+  Function getRelevantFunction(DataFlow::Node endpoint) {
+    result = endpoint.asExpr().getEnclosingFunction*()
+  }
+
+  override string getValue(DataFlow::Node endpoint) {
+    result =
+      concat(string functionParameterLine, Function f |
+        f = getRelevantFunction(endpoint) and
+        functionParameterLine = SyntacticUtilities::getFunctionParametersFeatureComponent(f)
+      |
+        functionParameterLine, "\n"
+        order by
+          f.getLocation().getStartLine(), f.getLocation().getStartColumn()
+      )
+  }
+}
+
+/**
+ * The feature that gives the name an endpoint is assigned to (if any).
+ *
+ * ### Example
+ * ```javascript
+ * const div = document.createElement('div');
+ * div.innerHTML = endpoint; // feature value is 'innerHTML'
+ * ```
+ */
+class AssignedToPropName extends EndpointFeature, TAssignedToPropName {
+  override string getName() { result = "assignedToPropName" }
+
+  override string getValue(DataFlow::Node endpoint) {
+    exists(DataFlow::PropWrite w | w.getRhs().asExpr().getUnderlyingValue().flow() = endpoint |
+      result = w.getPropertyName()
+    )
+  }
+}
+
+/**
+ * The feature that shows the text an endpoint is being concatenated with.class
+ *
+ * ### Example
+ *
+ * ```javascript
+ * const x = 'foo' + endpoint + 'bar'; // feature value is `'foo' -endpoint- 'bar'`
+ */
+class StringConcatenatedWith extends EndpointFeature, TStringConcatenatedWith {
+  override string getName() { result = "stringConcatenatedWith" }
+
+  override string getValue(DataFlow::Node endpoint) {
+    exists(StringOps::ConcatenationRoot root |
+      root.getALeaf() = endpoint and
+      result =
+        concat(StringOps::ConcatenationLeaf p |
+            p.getRoot() = root and
+            (
+              p.getStartLine() < endpoint.getStartLine()
+              or
+              p.getStartLine() = endpoint.getStartLine() and
+              p.getStartColumn() < endpoint.getStartColumn()
+            )
+          |
+            SyntacticUtilities::renderStringConcatOperand(p), " + "
+            order by
+              p.getStartLine(), p.getStartColumn()
+          ) + " -endpoint- " +
+          concat(StringOps::ConcatenationLeaf p |
+            p.getRoot() = root and
+            (
+              p.getStartLine() > endpoint.getStartLine()
+              or
+              p.getStartLine() = endpoint.getStartLine() and
+              p.getStartColumn() > endpoint.getStartColumn()
+            )
+          |
+            SyntacticUtilities::renderStringConcatOperand(p), " + "
+            order by
+              p.getStartLine(), p.getStartColumn()
+          )
+    )
+  }
+}
+
+/**
+ * The feature for the imports used in the callee of an invocation.
+ *
+ * ### Example
+ *
+ * ```javascript
+ * import * as _ from 'lodash';
+ *
+ * // ...
+ * _.deepClone(someObject);
+ * //          ^^^^^^^^^^ will have the value `lodash` for the feature `calleeImports`.
+ * ```
+ */
+class CalleeImports extends EndpointFeature, TCalleeImports {
+  override string getName() { result = "calleeImports" }
+
+  override string getValue(DataFlow::Node endpoint) {
+    not result = SyntacticUtilities::getUnknownSymbol() and
+    exists(DataFlow::InvokeNode invk |
+      (
+        invk.getAnArgument() = endpoint or
+        SyntacticUtilities::getANestedInitializerValue(invk.getAnArgument()
+              .asExpr()
+              .getUnderlyingValue()).flow() = endpoint
+      ) and
+      result =
+        concat(string importPath |
+          importPath = SyntacticUtilities::getCalleeImportPath(invk.getCalleeNode())
+        |
+          importPath, " " order by importPath
+        )
+    )
+  }
+}
+
+/**
+ * The feature for the interfaces of all named functions in the same file as the endpoint.
+ *
+ * ### Example
+ * ```javascript
+ * // Will return: "f(a, b, c)\ng(x, y, z)\nh(u, v)" for this file.
+ * function f(a, b, c) { ... }
+ *
+ * function g(x, y, z) {
+ *   function h(u, v) { ... }
+ *   ...
+ * }
+ * ```
+ *
+ * The feature value for the marked endpoint will be `f(a, b, c)\ng(x, y, z)\nh(u, v)`.
+ */
+class ContextFunctionInterfaces extends EndpointFeature, TContextFunctionInterfaces {
+  override string getName() { result = "contextFunctionInterfaces" }
+
+  override string getValue(DataFlow::Node endpoint) {
+    result = SyntacticUtilities::getFunctionInterfacesForFile(endpoint.getFile())
+  }
+}
+
+/**
+ * Syntactic utilities for feature value computation.
+ */
+private module SyntacticUtilities {
+  bindingset[start, end]
+  string renderStringConcatOperands(DataFlow::Node root, int start, int end) {
+    result =
+      concat(int i, string operand |
+        i = [start .. end] and
+        operand = renderStringConcatOperand(StringConcatenation::getOperand(root, i))
+      |
+        operand, " + " order by i
+      )
+  }
+
+  string renderStringConcatOperand(DataFlow::Node operand) {
+    if exists(unique(string v | operand.mayHaveStringValue(v)))
+    then result = "'" + any(string v | operand.mayHaveStringValue(v)) + "'"
+    else result = getSimpleAccessPath(operand)
+  }
+
+  /** Gets all the imports defined in the file containing the endpoint. */
+  string getImportPathsForFile(File file) {
+    result =
+      concat(string importPath |
+        importPath = SyntacticUtilities::getImportPathForFile(file)
+      |
+        importPath, " " order by importPath
+      )
+  }
+
+  /** Gets an import located in `file`. */
+  string getImportPathForFile(File file) {
+    result = any(Import imp | imp.getFile() = file).getImportedPath().getValue()
+  }
+
+  /**
+   * Gets the feature component for the parameters of a function.
+   *
+   * ```javascript
+   * function f(a, b, c) { // will return "(a, b, c)" for this function
+   *  return a + b + c;
+   * }
+   *
+   * async function g(a) { // will return "(a)" for this function
+   *   return 2*a
+   * };
+   *
+   * const h = (b) => 3*b; // will return "(b)" for this function
+   * ```
+   */
+  string getFunctionParametersFeatureComponent(Function f) {
+    result =
+      "(" +
+        concat(string parameter, int i |
+          parameter = getParameterNameOrUnknown(f.getParameter(i))
+        |
+          parameter, ", " order by i
+        ) + ")"
+  }
+
+  /**
+   * Gets the function interfaces of all named functions in a file, concatenated together.
+   *
+   * ```javascript
+   * // Will return: "f(a, b, c)\ng(x, y, z)\nh(u, v)" for this file.
+   * function f(a, b, c) { ... }
+   *
+   * function g(x, y, z) {
+   *   function h(u, v) { ... }
+   *   ...
+   * }
+   */
+  string getFunctionInterfacesForFile(File file) {
+    result =
+      concat(Function func, string line |
+        func.getFile() = file and
+        exists(func.getName()) and
+        line = func.getName() + getFunctionParametersFeatureComponent(func)
+      |
+        line, "\n" order by line
+      )
+  }
+
+  /**
+   * Gets a property initializer value in an object literal or one of its nested object literals.
+   */
+  Expr getANestedInitializerValue(ObjectExpr o) {
+    exists(Expr init | init = o.getAProperty().getInit().getUnderlyingValue() |
+      result = [init, getANestedInitializerValue(init)]
+    )
+  }
+
+  /**
+   * Computes a simple access path for how a callee can refer to a value that appears in an argument to a call.
+   *
+   * Supports:
+   * - direct arguments
+   * - properties of (nested) objects that are arguments
+   *
+   * Unknown cases and property names results in `?`.
+   */
+  string getSimpleParameterAccessPath(DataFlow::Node node) {
+    if exists(DataFlow::CallNode call | node = call.getArgument(_))
+    then exists(DataFlow::CallNode call, int i | node = call.getArgument(i) | result = i + "")
+    else result = getSimplePropertyAccessPath(node)
+  }
+
+  /**
+   * Computes a simple access path for how a user can refer to a value that appears in an (nested) object.
+   *
+   * Supports:
+   * - properties of (nested) objects
+   *
+   * Unknown cases and property names results in `?`.
+   */
+  string getSimplePropertyAccessPath(DataFlow::Node node) {
+    if exists(ObjectExpr o | o.getAProperty().getInit().getUnderlyingValue() = node.asExpr())
+    then
+      exists(DataFlow::PropWrite w |
+        w.getRhs() = node and
+        result = getSimpleParameterAccessPath(w.getBase()) + "." + getPropertyNameOrUnknown(w)
+      )
+    else result = getUnknownSymbol()
+  }
+
+  /**
+   * Gets the imported package path that this node depends on, if any.
+   *
+   * Otherwise, returns '?'.
+   *
+   * XXX Be careful with using this in your features, as it might teach the model
+   * a fixed list of "dangerous" libraries that could lead to bad generalization.
+   */
+  string getCalleeImportPath(DataFlow::Node node) {
+    exists(DataFlow::Node src | src = node.getALocalSource() |
+      if src instanceof DataFlow::ModuleImportNode
+      then result = src.(DataFlow::ModuleImportNode).getPath()
+      else
+        if src instanceof DataFlow::PropRead
+        then result = getCalleeImportPath(src.(DataFlow::PropRead).getBase())
+        else
+          if src instanceof DataFlow::InvokeNode
+          then result = getCalleeImportPath(src.(DataFlow::InvokeNode).getCalleeNode())
+          else
+            if src.asExpr() instanceof AwaitExpr
+            then result = getCalleeImportPath(src.asExpr().(AwaitExpr).getOperand().flow())
+            else result = getUnknownSymbol()
+    )
+  }
+
+  /**
+   * Computes a simple access path for a node.
+   *
+   * Supports:
+   * - variable reads (including `this` and `super`)
+   * - imports
+   * - await
+   * - property reads
+   * - invocations
+   *
+   * Unknown cases and property names results in `?`.
+   */
+  string getSimpleAccessPath(DataFlow::Node node) {
+    exists(Expr e | e = node.asExpr().getUnderlyingValue() |
+      if e instanceof SuperAccess
+      then result = "super"
+      else
+        if e instanceof ThisAccess
+        then result = "this"
+        else
+          if e instanceof VarAccess
+          then result = e.(VarAccess).getName()
+          else
+            if e instanceof Import
+            then result = "import(" + getSimpleImportPath(e) + ")"
+            else
+              if e instanceof AwaitExpr
+              then result = "(await " + getSimpleAccessPath(e.(AwaitExpr).getOperand().flow()) + ")"
+              else
+                if node instanceof DataFlow::PropRead
+                then
+                  result =
+                    getSimpleAccessPath(node.(DataFlow::PropRead).getBase()) + "." +
+                      getPropertyNameOrUnknown(node)
+                else
+                  if node instanceof DataFlow::InvokeNode
+                  then
+                    result = getSimpleAccessPath(node.(DataFlow::InvokeNode).getCalleeNode()) + "()"
+                  else result = getUnknownSymbol()
+    )
+  }
+
+  string getUnknownSymbol() { result = "?" }
+
+  /**
+   * Gets the imported path.
+   *
+   * XXX To avoid teaching the ML model about npm packages, only relative paths are supported
+   *
+   * Unknown paths result in `?`.
+   */
+  string getSimpleImportPath(Import i) {
+    if exists(i.getImportedPath().getValue())
+    then
+      exists(string p | p = i.getImportedPath().getValue() |
+        if p.matches(".%") then result = "\"p\"" else result = "!" // hide absolute imports from the ML training
+      )
+    else result = getUnknownSymbol()
+  }
+
+  /**
+   * Gets the property name of a property reference or `?` if it is unknown.
+   */
+  string getPropertyNameOrUnknown(DataFlow::PropRef ref) {
+    if exists(ref.getPropertyName())
+    then result = ref.getPropertyName()
+    else result = getUnknownSymbol()
+  }
+
+  /**
+   * Gets the parameter name if it exists, or `?` if it is unknown.
+   */
+  string getParameterNameOrUnknown(Parameter p) {
+    if exists(p.getName()) then result = p.getName() else result = getUnknownSymbol()
+  }
+}
+
+/**
+ * The feature for the access path of the callee node of a call that has an argument that "contains" the endpoint.
+ *
+ * "Containment" is syntactic, and currently means that the endpoint is an argument to the call, or that the endpoint is a (nested) property value of an argument.
+ *
+ * This feature is intended as a superior version of the many `Callee*` features.
+ *
+ * Examples:
+ * ```
+ * foo(endpoint); // -> foo
+ * foo.bar(endpoint); // -> foo.bar
+ * foo.bar({ baz: endpoint }); // -> foo.bar
+ * this.foo.bar(endpoint); // -> this.foo.bar
+ * foo[complex()].bar(endpoint); // -> foo.?.bar
+ * ```
+ */
+class CalleeFlexibleAccessPath extends EndpointFeature, TCalleeFlexibleAccessPath {
+  override string getName() { result = "CalleeFlexibleAccessPath" }
+
+  override string getValue(DataFlow::Node endpoint) {
+    exists(DataFlow::InvokeNode invk |
+      result = SyntacticUtilities::getSimpleAccessPath(invk.getCalleeNode()) and
+      // ignore the unknown path
+      not result = SyntacticUtilities::getUnknownSymbol() and
+      (
+        invk.getAnArgument() = endpoint or
+        SyntacticUtilities::getANestedInitializerValue(invk.getAnArgument()
+              .asExpr()
+              .getUnderlyingValue()).flow() = endpoint
+      )
+    )
+  }
+}
+
+/**
+ * The feature for how a callee can refer to a the endpoint that is "contained" in some argument to a call
+ *
+ * "Containment" is syntactic, and currently means that the endpoint is an argument to the call, or that the endpoint is a (nested) property value of an argument.
+ *
+ * This feature, together with `InputArgumentIndex` is intended as a far superior version of the `ArgumentIndexFeature`.
+ *
+ * Examples:
+ * ```
+ * foo({ bar: endpoint }); // -> bar
+ * foo(x, { bar: { baz: endpoint } }); // -> bar.baz
+ * ```
+ */
+class InputAccessPathFromCallee extends EndpointFeature, TInputAccessPathFromCallee {
+  override string getName() { result = "InputAccessPathFromCallee" }
+
+  override string getValue(DataFlow::Node endpoint) {
+    exists(DataFlow::InvokeNode invk |
+      result = SyntacticUtilities::getSimpleParameterAccessPath(endpoint) and
+      SyntacticUtilities::getANestedInitializerValue(invk.getAnArgument()
+            .asExpr()
+            .getUnderlyingValue()).flow() = endpoint
+    )
+  }
+}
+
+/**
+ * The feature for how the index of an argument that "contains" and endpoint.
+ *
+ * "Containment" is syntactic, and currently means that the endpoint is an argument to the call, or that the endpoint is a (nested) property value of an argument.
+ *
+ * This feature is intended as a superior version of the `ArgumentIndexFeature`.
+ *
+ * Examples:
+ * ```
+ * foo(endpoint); // -> 0
+ * foo({ bar: endpoint }); // -> 0
+ * foo(x, { bar: { baz: endpoint } }); // -> 1
+ * ```
+ */
+class InputArgumentIndex extends EndpointFeature, TInputArgumentIndex {
+  override string getName() { result = "InputArgumentIndex" }
+
+  override string getValue(DataFlow::Node endpoint) {
+    exists(DataFlow::InvokeNode invk, DataFlow::Node arg, int i | arg = invk.getArgument(i) |
+      result = i + "" and
+      (
+        invk.getArgument(i) = endpoint
+        or
+        SyntacticUtilities::getANestedInitializerValue(arg.asExpr().getUnderlyingValue()).flow() =
+          endpoint
+      )
+    )
+  }
+}

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/debug/CompareFeatures.ql

@@ -0,0 +1,23 @@
+import experimental.adaptivethreatmodeling.TaintedPathATM
+import experimental.adaptivethreatmodeling.EndpointFeatures
+import experimental.adaptivethreatmodeling.EndpointScoring
+
+string getValueOrNone(EndpointFeature feature, DataFlow::Node endpoint) {
+  if exists(feature.getValue(endpoint)) then feature.getValue(endpoint) = result else isNone(result)
+}
+
+predicate isNone(string value) { value = "" }
+
+// query for comparing feature values
+from
+  DataFlow::Node endpoint, EndpointFeature feature1, EndpointFeature feature2, string featureValue1,
+  string featureValue2
+where
+  feature1 instanceof Input_ArgumentIndex and
+  feature2 instanceof ArgumentIndex and
+  featureValue1 = getValueOrNone(feature1, endpoint) and
+  featureValue2 = getValueOrNone(feature2, endpoint) and
+  featureValue1 != featureValue2 and
+  isNone([featureValue1, featureValue2])
+select endpoint, endpoint.getFile().getBaseName() as file, endpoint.getStartLine() as line,
+  featureValue1, featureValue2

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/debug/NosqlInjectionATMAllEndpoints.ql

@@ -0,0 +1,66 @@
+import experimental.adaptivethreatmodeling.FeaturizationConfig
+import experimental.adaptivethreatmodeling.NosqlInjectionATM
+import experimental.adaptivethreatmodeling.EndpointFeatures
+import experimental.adaptivethreatmodeling.EndpointScoring
+private import javascript
+private import semmle.javascript.filters.ClassifyFiles as ClassifyFiles
+
+/** Holds if the file should be excluded from end-to-end evaluation. */
+predicate isFileExcluded(File file) {
+  // Ignore files that are outside the root folder of the analyzed source location.
+  //
+  // If the file doesn't have a relative path, then the source file is located outside the root
+  // folder of the analyzed source location, meaning that the files are additional files added to
+  // the database like standard library files that we would like to ignore.
+  not exists(file.getRelativePath())
+  or
+  // Ignore files based on their path.
+  exists(string ignorePattern, string separator |
+    ignorePattern =
+      // Exclude test files
+      "(tests?|test[_-]?case|" +
+        // Exclude library files
+        //
+        // - The Bower and npm package managers store packages in bower_components and node_modules
+        // folders respectively.
+        // - Specific exclusion for end-to-end: `applications/examples/static/epydoc` contains
+        //   library code from Epydoc.
+        "3rd[_-]?party|bower_components|extern(s|al)?|node_modules|resources|third[_-]?party|_?vendor|"
+        + "applications" + separator + "examples" + separator + "static" + separator + "epydoc|" +
+        // Exclude generated code
+        "gen|\\.?generated|" +
+        // Exclude benchmarks
+        "benchmarks?|" +
+        // Exclude documentation
+        "docs?|documentation)" and
+    separator = "(\\/|\\.)" and
+    exists(
+      file.getRelativePath()
+          .toLowerCase()
+          .regexpFind(separator + ignorePattern + separator + "|" + "^" + ignorePattern + separator +
+              "|" + separator + ignorePattern + "$", _, _)
+    )
+  )
+  or
+  // Ignore externs, generated, library, and test files.
+  ClassifyFiles::classify(file, ["externs", "generated", "library", "test"])
+}
+
+class FunctionArgumentFeaturizationConfig extends FeaturizationConfig {
+  FunctionArgumentFeaturizationConfig() { this = "FunctionArgumentFeaturization" }
+
+  override DataFlow::Node getAnEndpointToFeaturize() {
+    exists(DataFlow::CallNode call | result = call.getAnArgument())
+  }
+}
+
+from
+  DataFlow::Node sink, float score, int type, string filePathSink, int startLineSink,
+  int endLineSink, int startColumnSink, int endColumnSink
+where
+  ModelScoring::endpointScores(sink, type, score) and
+  not isFileExcluded(sink.getFile()) and
+  sink.hasLocationInfo(filePathSink, startLineSink, startColumnSink, endLineSink, endColumnSink) and
+  type = 2
+select sink, startLineSink, startColumnSink, endLineSink, endColumnSink, filePathSink, type, score
+  order by score desc

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/debug/SqlInjectionATMAllEndpoints.ql

@@ -0,0 +1,66 @@
+import experimental.adaptivethreatmodeling.FeaturizationConfig
+import experimental.adaptivethreatmodeling.SqlInjectionATM
+import experimental.adaptivethreatmodeling.EndpointFeatures
+import experimental.adaptivethreatmodeling.EndpointScoring
+private import javascript
+private import semmle.javascript.filters.ClassifyFiles as ClassifyFiles
+
+/** Holds if the file should be excluded from end-to-end evaluation. */
+predicate isFileExcluded(File file) {
+  // Ignore files that are outside the root folder of the analyzed source location.
+  //
+  // If the file doesn't have a relative path, then the source file is located outside the root
+  // folder of the analyzed source location, meaning that the files are additional files added to
+  // the database like standard library files that we would like to ignore.
+  not exists(file.getRelativePath())
+  or
+  // Ignore files based on their path.
+  exists(string ignorePattern, string separator |
+    ignorePattern =
+      // Exclude test files
+      "(tests?|test[_-]?case|" +
+        // Exclude library files
+        //
+        // - The Bower and npm package managers store packages in bower_components and node_modules
+        // folders respectively.
+        // - Specific exclusion for end-to-end: `applications/examples/static/epydoc` contains
+        //   library code from Epydoc.
+        "3rd[_-]?party|bower_components|extern(s|al)?|node_modules|resources|third[_-]?party|_?vendor|"
+        + "applications" + separator + "examples" + separator + "static" + separator + "epydoc|" +
+        // Exclude generated code
+        "gen|\\.?generated|" +
+        // Exclude benchmarks
+        "benchmarks?|" +
+        // Exclude documentation
+        "docs?|documentation)" and
+    separator = "(\\/|\\.)" and
+    exists(
+      file.getRelativePath()
+          .toLowerCase()
+          .regexpFind(separator + ignorePattern + separator + "|" + "^" + ignorePattern + separator +
+              "|" + separator + ignorePattern + "$", _, _)
+    )
+  )
+  or
+  // Ignore externs, generated, library, and test files.
+  ClassifyFiles::classify(file, ["externs", "generated", "library", "test"])
+}
+
+class FunctionArgumentFeaturizationConfig extends FeaturizationConfig {
+  FunctionArgumentFeaturizationConfig() { this = "FunctionArgumentFeaturization" }
+
+  override DataFlow::Node getAnEndpointToFeaturize() {
+    exists(DataFlow::CallNode call | result = call.getAnArgument())
+  }
+}
+
+from
+  DataFlow::Node sink, float score, int type, string filePathSink, int startLineSink,
+  int endLineSink, int startColumnSink, int endColumnSink
+where
+  ModelScoring::endpointScores(sink, type, score) and
+  not isFileExcluded(sink.getFile()) and
+  sink.hasLocationInfo(filePathSink, startLineSink, startColumnSink, endLineSink, endColumnSink) and
+  type = 3
+select sink, startLineSink, startColumnSink, endLineSink, endColumnSink, filePathSink, type, score
+  order by score desc

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/debug/TaintedPathATMAllEndpoints.ql

@@ -0,0 +1,66 @@
+import experimental.adaptivethreatmodeling.FeaturizationConfig
+import experimental.adaptivethreatmodeling.TaintedPathATM
+import experimental.adaptivethreatmodeling.EndpointFeatures
+import experimental.adaptivethreatmodeling.EndpointScoring
+private import javascript
+private import semmle.javascript.filters.ClassifyFiles as ClassifyFiles
+
+/** Holds if the file should be excluded from end-to-end evaluation. */
+predicate isFileExcluded(File file) {
+  // Ignore files that are outside the root folder of the analyzed source location.
+  //
+  // If the file doesn't have a relative path, then the source file is located outside the root
+  // folder of the analyzed source location, meaning that the files are additional files added to
+  // the database like standard library files that we would like to ignore.
+  not exists(file.getRelativePath())
+  or
+  // Ignore files based on their path.
+  exists(string ignorePattern, string separator |
+    ignorePattern =
+      // Exclude test files
+      "(tests?|test[_-]?case|" +
+        // Exclude library files
+        //
+        // - The Bower and npm package managers store packages in bower_components and node_modules
+        // folders respectively.
+        // - Specific exclusion for end-to-end: `applications/examples/static/epydoc` contains
+        //   library code from Epydoc.
+        "3rd[_-]?party|bower_components|extern(s|al)?|node_modules|resources|third[_-]?party|_?vendor|"
+        + "applications" + separator + "examples" + separator + "static" + separator + "epydoc|" +
+        // Exclude generated code
+        "gen|\\.?generated|" +
+        // Exclude benchmarks
+        "benchmarks?|" +
+        // Exclude documentation
+        "docs?|documentation)" and
+    separator = "(\\/|\\.)" and
+    exists(
+      file.getRelativePath()
+          .toLowerCase()
+          .regexpFind(separator + ignorePattern + separator + "|" + "^" + ignorePattern + separator +
+              "|" + separator + ignorePattern + "$", _, _)
+    )
+  )
+  or
+  // Ignore externs, generated, library, and test files.
+  ClassifyFiles::classify(file, ["externs", "generated", "library", "test"])
+}
+
+class FunctionArgumentFeaturizationConfig extends FeaturizationConfig {
+  FunctionArgumentFeaturizationConfig() { this = "FunctionArgumentFeaturization" }
+
+  override DataFlow::Node getAnEndpointToFeaturize() {
+    exists(DataFlow::CallNode call | result = call.getAnArgument())
+  }
+}
+
+from
+  DataFlow::Node sink, float score, int type, string filePathSink, int startLineSink,
+  int endLineSink, int startColumnSink, int endColumnSink
+where
+  ModelScoring::endpointScores(sink, type, score) and
+  not isFileExcluded(sink.getFile()) and
+  sink.hasLocationInfo(filePathSink, startLineSink, startColumnSink, endLineSink, endColumnSink) and
+  type = 4
+select sink, startLineSink, startColumnSink, endLineSink, endColumnSink, filePathSink, type, score
+  order by score desc

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/debug/XssATMAllEndpoints.ql

@@ -0,0 +1,66 @@
+import experimental.adaptivethreatmodeling.FeaturizationConfig
+import experimental.adaptivethreatmodeling.XssATM
+import experimental.adaptivethreatmodeling.EndpointFeatures
+import experimental.adaptivethreatmodeling.EndpointScoring
+private import javascript
+private import semmle.javascript.filters.ClassifyFiles as ClassifyFiles
+
+/** Holds if the file should be excluded from end-to-end evaluation. */
+predicate isFileExcluded(File file) {
+  // Ignore files that are outside the root folder of the analyzed source location.
+  //
+  // If the file doesn't have a relative path, then the source file is located outside the root
+  // folder of the analyzed source location, meaning that the files are additional files added to
+  // the database like standard library files that we would like to ignore.
+  not exists(file.getRelativePath())
+  or
+  // Ignore files based on their path.
+  exists(string ignorePattern, string separator |
+    ignorePattern =
+      // Exclude test files
+      "(tests?|test[_-]?case|" +
+        // Exclude library files
+        //
+        // - The Bower and npm package managers store packages in bower_components and node_modules
+        // folders respectively.
+        // - Specific exclusion for end-to-end: `applications/examples/static/epydoc` contains
+        //   library code from Epydoc.
+        "3rd[_-]?party|bower_components|extern(s|al)?|node_modules|resources|third[_-]?party|_?vendor|"
+        + "applications" + separator + "examples" + separator + "static" + separator + "epydoc|" +
+        // Exclude generated code
+        "gen|\\.?generated|" +
+        // Exclude benchmarks
+        "benchmarks?|" +
+        // Exclude documentation
+        "docs?|documentation)" and
+    separator = "(\\/|\\.)" and
+    exists(
+      file.getRelativePath()
+          .toLowerCase()
+          .regexpFind(separator + ignorePattern + separator + "|" + "^" + ignorePattern + separator +
+              "|" + separator + ignorePattern + "$", _, _)
+    )
+  )
+  or
+  // Ignore externs, generated, library, and test files.
+  ClassifyFiles::classify(file, ["externs", "generated", "library", "test"])
+}
+
+class FunctionArgumentFeaturizationConfig extends FeaturizationConfig {
+  FunctionArgumentFeaturizationConfig() { this = "FunctionArgumentFeaturization" }
+
+  override DataFlow::Node getAnEndpointToFeaturize() {
+    exists(DataFlow::CallNode call | result = call.getAnArgument())
+  }
+}
+
+from
+  DataFlow::Node sink, float score, int type, string filePathSink, int startLineSink,
+  int endLineSink, int startColumnSink, int endColumnSink
+where
+  ModelScoring::endpointScores(sink, type, score) and
+  not isFileExcluded(sink.getFile()) and
+  sink.hasLocationInfo(filePathSink, startLineSink, startColumnSink, endLineSink, endColumnSink) and
+  type = 1
+select sink, startLineSink, startColumnSink, endLineSink, endColumnSink, filePathSink, type, score
+  order by score desc

javascript/ql/experimental/adaptivethreatmodeling/test/generic_feature_testing/EmptyFeature.ql

@@ -0,0 +1,9 @@
+import javascript
+import experimental.adaptivethreatmodeling.EndpointFeatures
+import experimental.adaptivethreatmodeling.FeaturizationConfig
+import TestUtil
+
+// every feature must produce a value for at least one endpoint, otherwise the feature is completely broken, or a relevant test example is missing
+from EndpointFeature feature
+where forall(Endpoint endpoint | not exists(feature.getValue(endpoint)))
+select feature.getName()

javascript/ql/experimental/adaptivethreatmodeling/test/generic_feature_testing/FeatureValue.expected

@@ -0,0 +1,202 @@
+| test.html:2:61:2:68 | endpoint | CalleeFlexibleAccessPath | $event.target.files.item |
+| test.html:2:61:2:68 | endpoint | InputArgumentIndex | 0 |
+| test.html:2:61:2:68 | endpoint | argumentIndex | 0 |
+| test.html:2:61:2:68 | endpoint | calleeAccessPath |  |
+| test.html:2:61:2:68 | endpoint | calleeAccessPathWithStructuralInfo |  |
+| test.html:2:61:2:68 | endpoint | calleeName | item |
+| test.html:2:61:2:68 | endpoint | contextFunctionInterfaces |  |
+| test.html:2:61:2:68 | endpoint | contextSurroundingFunctionParameters |  |
+| test.html:2:61:2:68 | endpoint | fileImports |  |
+| test.js:6:7:6:14 | endpoint | CalleeFlexibleAccessPath | f |
+| test.js:6:7:6:14 | endpoint | InputArgumentIndex | 0 |
+| test.js:6:7:6:14 | endpoint | argumentIndex | 0 |
+| test.js:6:7:6:14 | endpoint | calleeAccessPath | lib3 |
+| test.js:6:7:6:14 | endpoint | calleeAccessPathWithStructuralInfo | lib3 instanceorreturn |
+| test.js:6:7:6:14 | endpoint | calleeApiName | lib3 |
+| test.js:6:7:6:14 | endpoint | calleeImports | ? lib3 |
+| test.js:6:7:6:14 | endpoint | calleeName | f |
+| test.js:6:7:6:14 | endpoint | contextFunctionInterfaces | f(?)\nfoo()\ng()\nm() |
+| test.js:6:7:6:14 | endpoint | contextSurroundingFunctionParameters | () |
+| test.js:6:7:6:14 | endpoint | enclosingFunctionBody | f endpoint 12 f p endpoint f p q endpoint o m endpoint o m p endpoint o m p q endpoint F endpoint o m m m endpoint f endpoint o x m endpoint o m x p m endpoint p endpoint foo bar baz endpoint foo bar endpoint f f o m endpoint |
+| test.js:6:7:6:14 | endpoint | enclosingFunctionName |  |
+| test.js:6:7:6:14 | endpoint | fileImports | foo lib1 lib2 lib3 |
+| test.js:7:11:7:18 | endpoint | CalleeFlexibleAccessPath | f |
+| test.js:7:11:7:18 | endpoint | InputAccessPathFromCallee | 0.p |
+| test.js:7:11:7:18 | endpoint | InputArgumentIndex | 0 |
+| test.js:7:11:7:18 | endpoint | calleeAccessPath |  |
+| test.js:7:11:7:18 | endpoint | calleeAccessPathWithStructuralInfo |  |
+| test.js:7:11:7:18 | endpoint | calleeImports | ? lib3 |
+| test.js:7:11:7:18 | endpoint | contextFunctionInterfaces | f(?)\nfoo()\ng()\nm() |
+| test.js:7:11:7:18 | endpoint | contextSurroundingFunctionParameters | () |
+| test.js:7:11:7:18 | endpoint | enclosingFunctionBody | f endpoint 12 f p endpoint f p q endpoint o m endpoint o m p endpoint o m p q endpoint F endpoint o m m m endpoint f endpoint o x m endpoint o m x p m endpoint p endpoint foo bar baz endpoint foo bar endpoint f f o m endpoint |
+| test.js:7:11:7:18 | endpoint | enclosingFunctionName |  |
+| test.js:7:11:7:18 | endpoint | fileImports | foo lib1 lib2 lib3 |
+| test.js:8:15:8:22 | endpoint | CalleeFlexibleAccessPath | f |
+| test.js:8:15:8:22 | endpoint | InputAccessPathFromCallee | 0.p.q |
+| test.js:8:15:8:22 | endpoint | InputArgumentIndex | 0 |
+| test.js:8:15:8:22 | endpoint | calleeAccessPath |  |
+| test.js:8:15:8:22 | endpoint | calleeAccessPathWithStructuralInfo |  |
+| test.js:8:15:8:22 | endpoint | calleeImports | ? lib3 |
+| test.js:8:15:8:22 | endpoint | contextFunctionInterfaces | f(?)\nfoo()\ng()\nm() |
+| test.js:8:15:8:22 | endpoint | contextSurroundingFunctionParameters | () |
+| test.js:8:15:8:22 | endpoint | enclosingFunctionBody | f endpoint 12 f p endpoint f p q endpoint o m endpoint o m p endpoint o m p q endpoint F endpoint o m m m endpoint f endpoint o x m endpoint o m x p m endpoint p endpoint foo bar baz endpoint foo bar endpoint f f o m endpoint |
+| test.js:8:15:8:22 | endpoint | enclosingFunctionName |  |
+| test.js:8:15:8:22 | endpoint | fileImports | foo lib1 lib2 lib3 |
+| test.js:9:9:9:16 | endpoint | CalleeFlexibleAccessPath | o.m |
+| test.js:9:9:9:16 | endpoint | InputArgumentIndex | 0 |
+| test.js:9:9:9:16 | endpoint | argumentIndex | 0 |
+| test.js:9:9:9:16 | endpoint | calleeAccessPath | lib2 m |
+| test.js:9:9:9:16 | endpoint | calleeAccessPathWithStructuralInfo | lib2 member m instanceorreturn |
+| test.js:9:9:9:16 | endpoint | calleeApiName | lib2 |
+| test.js:9:9:9:16 | endpoint | calleeImports | ? lib2 |
+| test.js:9:9:9:16 | endpoint | calleeName | m |
+| test.js:9:9:9:16 | endpoint | contextFunctionInterfaces | f(?)\nfoo()\ng()\nm() |
+| test.js:9:9:9:16 | endpoint | contextSurroundingFunctionParameters | () |
+| test.js:9:9:9:16 | endpoint | enclosingFunctionBody | f endpoint 12 f p endpoint f p q endpoint o m endpoint o m p endpoint o m p q endpoint F endpoint o m m m endpoint f endpoint o x m endpoint o m x p m endpoint p endpoint foo bar baz endpoint foo bar endpoint f f o m endpoint |
+| test.js:9:9:9:16 | endpoint | enclosingFunctionName |  |
+| test.js:9:9:9:16 | endpoint | fileImports | foo lib1 lib2 lib3 |
+| test.js:9:9:9:16 | endpoint | receiverName | o |
+| test.js:10:13:10:20 | endpoint | CalleeFlexibleAccessPath | o.m |
+| test.js:10:13:10:20 | endpoint | InputAccessPathFromCallee | 0.p |
+| test.js:10:13:10:20 | endpoint | InputArgumentIndex | 0 |
+| test.js:10:13:10:20 | endpoint | calleeAccessPath |  |
+| test.js:10:13:10:20 | endpoint | calleeAccessPathWithStructuralInfo |  |
+| test.js:10:13:10:20 | endpoint | calleeImports | ? lib2 |
+| test.js:10:13:10:20 | endpoint | contextFunctionInterfaces | f(?)\nfoo()\ng()\nm() |
+| test.js:10:13:10:20 | endpoint | contextSurroundingFunctionParameters | () |
+| test.js:10:13:10:20 | endpoint | enclosingFunctionBody | f endpoint 12 f p endpoint f p q endpoint o m endpoint o m p endpoint o m p q endpoint F endpoint o m m m endpoint f endpoint o x m endpoint o m x p m endpoint p endpoint foo bar baz endpoint foo bar endpoint f f o m endpoint |
+| test.js:10:13:10:20 | endpoint | enclosingFunctionName |  |
+| test.js:10:13:10:20 | endpoint | fileImports | foo lib1 lib2 lib3 |
+| test.js:11:17:11:24 | endpoint | CalleeFlexibleAccessPath | o.m |
+| test.js:11:17:11:24 | endpoint | InputAccessPathFromCallee | 0.p.q |
+| test.js:11:17:11:24 | endpoint | InputArgumentIndex | 0 |
+| test.js:11:17:11:24 | endpoint | calleeAccessPath |  |
+| test.js:11:17:11:24 | endpoint | calleeAccessPathWithStructuralInfo |  |
+| test.js:11:17:11:24 | endpoint | calleeImports | ? lib2 |
+| test.js:11:17:11:24 | endpoint | contextFunctionInterfaces | f(?)\nfoo()\ng()\nm() |
+| test.js:11:17:11:24 | endpoint | contextSurroundingFunctionParameters | () |
+| test.js:11:17:11:24 | endpoint | enclosingFunctionBody | f endpoint 12 f p endpoint f p q endpoint o m endpoint o m p endpoint o m p q endpoint F endpoint o m m m endpoint f endpoint o x m endpoint o m x p m endpoint p endpoint foo bar baz endpoint foo bar endpoint f f o m endpoint |
+| test.js:11:17:11:24 | endpoint | enclosingFunctionName |  |
+| test.js:11:17:11:24 | endpoint | fileImports | foo lib1 lib2 lib3 |
+| test.js:12:11:12:18 | endpoint | CalleeFlexibleAccessPath | F |
+| test.js:12:11:12:18 | endpoint | InputArgumentIndex | 0 |
+| test.js:12:11:12:18 | endpoint | calleeAccessPath |  |
+| test.js:12:11:12:18 | endpoint | calleeAccessPathWithStructuralInfo |  |
+| test.js:12:11:12:18 | endpoint | calleeImports | lib1 |
+| test.js:12:11:12:18 | endpoint | contextFunctionInterfaces | f(?)\nfoo()\ng()\nm() |
+| test.js:12:11:12:18 | endpoint | contextSurroundingFunctionParameters | () |
+| test.js:12:11:12:18 | endpoint | enclosingFunctionBody | f endpoint 12 f p endpoint f p q endpoint o m endpoint o m p endpoint o m p q endpoint F endpoint o m m m endpoint f endpoint o x m endpoint o m x p m endpoint p endpoint foo bar baz endpoint foo bar endpoint f f o m endpoint |
+| test.js:12:11:12:18 | endpoint | enclosingFunctionName |  |
+| test.js:12:11:12:18 | endpoint | fileImports | foo lib1 lib2 lib3 |
+| test.js:13:17:13:24 | endpoint | CalleeFlexibleAccessPath | o.m().m().m |
+| test.js:13:17:13:24 | endpoint | InputArgumentIndex | 0 |
+| test.js:13:17:13:24 | endpoint | argumentIndex | 0 |
+| test.js:13:17:13:24 | endpoint | calleeAccessPath | lib2 m m m |
+| test.js:13:17:13:24 | endpoint | calleeAccessPathWithStructuralInfo | lib2 member m instanceorreturn member m instanceorreturn member m instanceorreturn |
+| test.js:13:17:13:24 | endpoint | calleeApiName | lib2 |
+| test.js:13:17:13:24 | endpoint | calleeImports | ? lib2 |
+| test.js:13:17:13:24 | endpoint | calleeName | m |
+| test.js:13:17:13:24 | endpoint | contextFunctionInterfaces | f(?)\nfoo()\ng()\nm() |
+| test.js:13:17:13:24 | endpoint | contextSurroundingFunctionParameters | () |
+| test.js:13:17:13:24 | endpoint | enclosingFunctionBody | f endpoint 12 f p endpoint f p q endpoint o m endpoint o m p endpoint o m p q endpoint F endpoint o m m m endpoint f endpoint o x m endpoint o m x p m endpoint p endpoint foo bar baz endpoint foo bar endpoint f f o m endpoint |
+| test.js:13:17:13:24 | endpoint | enclosingFunctionName |  |
+| test.js:13:17:13:24 | endpoint | fileImports | foo lib1 lib2 lib3 |
+| test.js:14:9:14:16 | endpoint | CalleeFlexibleAccessPath | f() |
+| test.js:14:9:14:16 | endpoint | InputArgumentIndex | 0 |
+| test.js:14:9:14:16 | endpoint | argumentIndex | 0 |
+| test.js:14:9:14:16 | endpoint | calleeAccessPath | lib3 |
+| test.js:14:9:14:16 | endpoint | calleeAccessPathWithStructuralInfo | lib3 instanceorreturn instanceorreturn |
+| test.js:14:9:14:16 | endpoint | calleeApiName | lib3 |
+| test.js:14:9:14:16 | endpoint | calleeImports | ? lib3 |
+| test.js:14:9:14:16 | endpoint | contextFunctionInterfaces | f(?)\nfoo()\ng()\nm() |
+| test.js:14:9:14:16 | endpoint | contextSurroundingFunctionParameters | () |
+| test.js:14:9:14:16 | endpoint | enclosingFunctionBody | f endpoint 12 f p endpoint f p q endpoint o m endpoint o m p endpoint o m p q endpoint F endpoint o m m m endpoint f endpoint o x m endpoint o m x p m endpoint p endpoint foo bar baz endpoint foo bar endpoint f f o m endpoint |
+| test.js:14:9:14:16 | endpoint | enclosingFunctionName |  |
+| test.js:14:9:14:16 | endpoint | fileImports | foo lib1 lib2 lib3 |
+| test.js:15:12:15:19 | endpoint | CalleeFlexibleAccessPath | o.?.m |
+| test.js:15:12:15:19 | endpoint | InputArgumentIndex | 0 |
+| test.js:15:12:15:19 | endpoint | argumentIndex | 0 |
+| test.js:15:12:15:19 | endpoint | calleeAccessPath | lib2 m |
+| test.js:15:12:15:19 | endpoint | calleeAccessPathWithStructuralInfo | lib2 member member m instanceorreturn |
+| test.js:15:12:15:19 | endpoint | calleeApiName | lib2 |
+| test.js:15:12:15:19 | endpoint | calleeImports | ? lib2 |
+| test.js:15:12:15:19 | endpoint | calleeName | m |
+| test.js:15:12:15:19 | endpoint | contextFunctionInterfaces | f(?)\nfoo()\ng()\nm() |
+| test.js:15:12:15:19 | endpoint | contextSurroundingFunctionParameters | () |
+| test.js:15:12:15:19 | endpoint | enclosingFunctionBody | f endpoint 12 f p endpoint f p q endpoint o m endpoint o m p endpoint o m p q endpoint F endpoint o m m m endpoint f endpoint o x m endpoint o m x p m endpoint p endpoint foo bar baz endpoint foo bar endpoint f f o m endpoint |
+| test.js:15:12:15:19 | endpoint | enclosingFunctionName |  |
+| test.js:15:12:15:19 | endpoint | fileImports | foo lib1 lib2 lib3 |
+| test.js:16:16:16:23 | endpoint | CalleeFlexibleAccessPath | o.m.?.p.m |
+| test.js:16:16:16:23 | endpoint | InputArgumentIndex | 0 |
+| test.js:16:16:16:23 | endpoint | argumentIndex | 0 |
+| test.js:16:16:16:23 | endpoint | calleeAccessPath | lib2 m p m |
+| test.js:16:16:16:23 | endpoint | calleeAccessPathWithStructuralInfo | lib2 member m member member p member m instanceorreturn |
+| test.js:16:16:16:23 | endpoint | calleeApiName | lib2 |
+| test.js:16:16:16:23 | endpoint | calleeImports | ? lib2 |
+| test.js:16:16:16:23 | endpoint | calleeName | m |
+| test.js:16:16:16:23 | endpoint | contextFunctionInterfaces | f(?)\nfoo()\ng()\nm() |
+| test.js:16:16:16:23 | endpoint | contextSurroundingFunctionParameters | () |
+| test.js:16:16:16:23 | endpoint | enclosingFunctionBody | f endpoint 12 f p endpoint f p q endpoint o m endpoint o m p endpoint o m p q endpoint F endpoint o m m m endpoint f endpoint o x m endpoint o m x p m endpoint p endpoint foo bar baz endpoint foo bar endpoint f f o m endpoint |
+| test.js:16:16:16:23 | endpoint | enclosingFunctionName |  |
+| test.js:16:16:16:23 | endpoint | fileImports | foo lib1 lib2 lib3 |
+| test.js:17:15:17:22 | endpoint | CalleeFlexibleAccessPath | (await p) |
+| test.js:17:15:17:22 | endpoint | InputArgumentIndex | 0 |
+| test.js:17:15:17:22 | endpoint | argumentIndex | 0 |
+| test.js:17:15:17:22 | endpoint | calleeAccessPath | lib1 p |
+| test.js:17:15:17:22 | endpoint | calleeAccessPathWithStructuralInfo | lib1 member p instanceorreturn |
+| test.js:17:15:17:22 | endpoint | calleeApiName | lib1 |
+| test.js:17:15:17:22 | endpoint | calleeImports | lib1 |
+| test.js:17:15:17:22 | endpoint | contextFunctionInterfaces | f(?)\nfoo()\ng()\nm() |
+| test.js:17:15:17:22 | endpoint | contextSurroundingFunctionParameters | () |
+| test.js:17:15:17:22 | endpoint | enclosingFunctionBody | f endpoint 12 f p endpoint f p q endpoint o m endpoint o m p endpoint o m p q endpoint F endpoint o m m m endpoint f endpoint o x m endpoint o m x p m endpoint p endpoint foo bar baz endpoint foo bar endpoint f f o m endpoint |
+| test.js:17:15:17:22 | endpoint | enclosingFunctionName |  |
+| test.js:17:15:17:22 | endpoint | fileImports | foo lib1 lib2 lib3 |
+| test.js:18:27:18:34 | endpoint | CalleeFlexibleAccessPath | import(!).bar.baz |
+| test.js:18:27:18:34 | endpoint | InputArgumentIndex | 0 |
+| test.js:18:27:18:34 | endpoint | argumentIndex | 0 |
+| test.js:18:27:18:34 | endpoint | calleeAccessPath | foo bar baz |
+| test.js:18:27:18:34 | endpoint | calleeAccessPathWithStructuralInfo | foo member bar member baz instanceorreturn |
+| test.js:18:27:18:34 | endpoint | calleeApiName | foo |
+| test.js:18:27:18:34 | endpoint | calleeImports | foo |
+| test.js:18:27:18:34 | endpoint | calleeName | baz |
+| test.js:18:27:18:34 | endpoint | contextFunctionInterfaces | f(?)\nfoo()\ng()\nm() |
+| test.js:18:27:18:34 | endpoint | contextSurroundingFunctionParameters | () |
+| test.js:18:27:18:34 | endpoint | enclosingFunctionBody | f endpoint 12 f p endpoint f p q endpoint o m endpoint o m p endpoint o m p q endpoint F endpoint o m m m endpoint f endpoint o x m endpoint o m x p m endpoint p endpoint foo bar baz endpoint foo bar endpoint f f o m endpoint |
+| test.js:18:27:18:34 | endpoint | enclosingFunctionName |  |
+| test.js:18:27:18:34 | endpoint | fileImports | foo lib1 lib2 lib3 |
+| test.js:20:13:20:20 | endpoint | CalleeFlexibleAccessPath | bar |
+| test.js:20:13:20:20 | endpoint | InputArgumentIndex | 0 |
+| test.js:20:13:20:20 | endpoint | argumentIndex | 0 |
+| test.js:20:13:20:20 | endpoint | calleeAccessPath | lib1 bar |
+| test.js:20:13:20:20 | endpoint | calleeAccessPathWithStructuralInfo | lib1 member bar instanceorreturn |
+| test.js:20:13:20:20 | endpoint | calleeApiName | lib1 |
+| test.js:20:13:20:20 | endpoint | calleeImports | lib1 |
+| test.js:20:13:20:20 | endpoint | calleeName | bar |
+| test.js:20:13:20:20 | endpoint | contextFunctionInterfaces | f(?)\nfoo()\ng()\nm() |
+| test.js:20:13:20:20 | endpoint | contextSurroundingFunctionParameters | () |
+| test.js:20:13:20:20 | endpoint | enclosingFunctionBody | f endpoint 12 f p endpoint f p q endpoint o m endpoint o m p endpoint o m p q endpoint F endpoint o m m m endpoint f endpoint o x m endpoint o m x p m endpoint p endpoint foo bar baz endpoint foo bar endpoint f f o m endpoint |
+| test.js:20:13:20:20 | endpoint | enclosingFunctionName |  |
+| test.js:20:13:20:20 | endpoint | fileImports | foo lib1 lib2 lib3 |
+| test.js:22:21:22:28 | endpoint | InputArgumentIndex | 0 |
+| test.js:22:21:22:28 | endpoint | argumentIndex | 0 |
+| test.js:22:21:22:28 | endpoint | calleeAccessPath | lib3 |
+| test.js:22:21:22:28 | endpoint | calleeAccessPathWithStructuralInfo | lib3 instanceorreturn |
+| test.js:22:21:22:28 | endpoint | calleeApiName | lib3 |
+| test.js:22:21:22:28 | endpoint | calleeImports | ? lib2 lib3 |
+| test.js:22:21:22:28 | endpoint | contextFunctionInterfaces | f(?)\nfoo()\ng()\nm() |
+| test.js:22:21:22:28 | endpoint | contextSurroundingFunctionParameters | () |
+| test.js:22:21:22:28 | endpoint | enclosingFunctionBody | f endpoint 12 f p endpoint f p q endpoint o m endpoint o m p endpoint o m p q endpoint F endpoint o m m m endpoint f endpoint o x m endpoint o m x p m endpoint p endpoint foo bar baz endpoint foo bar endpoint f f o m endpoint |
+| test.js:22:21:22:28 | endpoint | enclosingFunctionName |  |
+| test.js:22:21:22:28 | endpoint | fileImports | foo lib1 lib2 lib3 |
+| test.js:33:50:33:57 | endpoint | calleeAccessPath |  |
+| test.js:33:50:33:57 | endpoint | calleeAccessPathWithStructuralInfo |  |
+| test.js:33:50:33:57 | endpoint | contextFunctionInterfaces | f(?)\nfoo()\ng()\nm() |
+| test.js:33:50:33:57 | endpoint | contextSurroundingFunctionParameters |  |
+| test.js:33:50:33:57 | endpoint | fileImports | foo lib1 lib2 lib3 |
+| test.js:33:50:33:57 | endpoint | stringConcatenatedWith | f() + '<a target="_blank" href="' -endpoint- '"></a>' |
+| test.js:35:18:35:25 | endpoint | calleeAccessPath |  |
+| test.js:35:18:35:25 | endpoint | calleeAccessPathWithStructuralInfo |  |
+| test.js:35:18:35:25 | endpoint | contextFunctionInterfaces | f(?)\nfoo()\ng()\nm() |
+| test.js:35:18:35:25 | endpoint | contextSurroundingFunctionParameters |  |
+| test.js:35:18:35:25 | endpoint | fileImports | foo lib1 lib2 lib3 |
+| test.js:35:18:35:25 | endpoint | stringConcatenatedWith | 'foo' -endpoint- 'bar' |

javascript/ql/experimental/adaptivethreatmodeling/test/generic_feature_testing/FeatureValue.ql

@@ -0,0 +1,7 @@
+import javascript
+import experimental.adaptivethreatmodeling.EndpointFeatures
+import TestUtil
+
+// detailed output for the nearby tests
+from Endpoint endpoint, EndpointFeature feature
+select endpoint, feature.getName(), feature.getValue(endpoint)

javascript/ql/experimental/adaptivethreatmodeling/test/generic_feature_testing/NonFeaturizedEndpoint.ql

@@ -0,0 +1,8 @@
+import javascript
+import experimental.adaptivethreatmodeling.EndpointFeatures
+import TestUtil
+
+// every endpoint should have at least one feature value, otherwise the test source is likely malformed
+from Endpoint endpoint
+where not exists(EndpointFeature f | exists(f.getValue(endpoint)))
+select endpoint

javascript/ql/experimental/adaptivethreatmodeling/test/generic_feature_testing/NonFunctionalFeature.ql

@@ -0,0 +1,8 @@
+import javascript
+import experimental.adaptivethreatmodeling.EndpointFeatures
+import TestUtil
+
+// every feature must produce a single value for each endpoint that it computes a value for, per the contract of the `scoreEndpoints` HOP
+from Endpoint endpoint, EndpointFeature feature, int arity
+where arity = count(feature.getValue(endpoint)) and arity > 1
+select endpoint, feature.getName(), arity

javascript/ql/experimental/adaptivethreatmodeling/test/generic_feature_testing/TestUtil.qll

@@ -0,0 +1,6 @@
+import javascript
+import extraction.NoFeaturizationRestrictionsConfig
+
+class Endpoint extends DataFlow::Node {
+  Endpoint() { this.asExpr().(VarAccess).getName() = "endpoint" }
+}

javascript/ql/experimental/adaptivethreatmodeling/test/generic_feature_testing/test.html

@@ -0,0 +1,3 @@
+<div class="form-group">
+    <input (change)="restoreBackup($event.target.files.item(endpoint))" />
+</div>

javascript/ql/experimental/adaptivethreatmodeling/test/generic_feature_testing/test.js

@@ -0,0 +1,35 @@
+import { bar, F, p } from 'lib1';
+import * as o from 'lib2';
+const f = require('lib3');
+
+(async function () {
+    f(endpoint, 12);
+    f({p: endpoint});
+    f({p: {q: endpoint}});
+    o.m(endpoint);
+    o.m({p: endpoint});
+    o.m({p: {q: endpoint}});
+    new F(endpoint);
+    o.m().m().m(endpoint);
+    f()(endpoint);
+    o[x].m(endpoint);
+    o.m[x].p.m(endpoint);
+    (await p)(endpoint);
+    import("foo").bar.baz(endpoint);
+    function foo() {
+        bar(endpoint);
+    }
+    (f() ? f : o.m)(endpoint);
+});
+
+function f({ endpoint }) {}
+
+const g = async () => undefined;
+
+const o = { m: () => undefined }
+
+const url = f();
+
+const x = f() + "<a target=\"_blank\" href=\"" + endpoint + "\"></a>";
+
+const y = "foo"+ endpoint + "bar";