enable new features

Co-authored-by: Henry Mercer <henrymercer@github.com>

javascript/ql/experimental/adaptivethreatmodeling/lib/experimental/adaptivethreatmodeling/EndpointFeatures.qll

@@ -16,102 +16,8 @@ private import FunctionBodyFeatures as FunctionBodyFeatures
 private string getTokenFeature(DataFlow::Node endpoint, string featureName) {
   // Performance optimization: Restrict feature extraction to endpoints we've explicitly asked to featurize.
   endpoint = any(FeaturizationConfig cfg).getAnEndpointToFeaturize() and
-  (
+  exists(EndpointFeature f | f.getName() = featureName and result = f.getValue(endpoint)) and
-    // Features for endpoints that are contained within a function.
+  featureName = getASupportedFeatureName()
-    exists(Function function |
-      function = FunctionBodyFeatures::getRepresentativeFunctionForEndpoint(endpoint)
-    |
-      // The name of the function that encloses the endpoint.
-      featureName = "enclosingFunctionName" and result = FunctionNames::getNameToFeaturize(function)
-      or
-      // A feature containing natural language tokens from the function that encloses the endpoint in
-      // the order that they appear in the source code.
-      featureName = "enclosingFunctionBody" and
-      result = FunctionBodyFeatures::getBodyTokensFeature(function)
-    )
-    or
-    result =
-      strictconcat(DataFlow::CallNode call, string component |
-        component = getACallBasedTokenFeatureComponent(endpoint, call, featureName)
-      |
-        component, " "
-      )
-    or
-    // The access path of the function being called, both with and without structural info, if the
-    // function being called originates from an external API. For example, the endpoint here:
-    //
-    // ```js
-    // const mongoose = require('mongoose'),
-    //   User = mongoose.model('User', null);
-    // User.findOne(ENDPOINT);
-    // ```
-    //
-    // would have a callee access path with structural info of
-    // `mongoose member model instanceorreturn member findOne instanceorreturn`, and a callee access
-    // path without structural info of `mongoose model findOne`.
-    //
-    // These features indicate that the callee comes from (reading the access path backwards) an
-    // instance of the `findOne` member of an instance of the `model` member of the `mongoose`
-    // external library.
-    exists(AccessPaths::Boolean includeStructuralInfo |
-      featureName =
-        "calleeAccessPath" +
-          any(string x | if includeStructuralInfo = true then x = "WithStructuralInfo" else x = "") and
-      result =
-        concat(API::Node node, string accessPath |
-          node.getInducingNode().(DataFlow::CallNode).getAnArgument() = endpoint and
-          AccessPaths::accessPaths(node, includeStructuralInfo, accessPath, _)
-        |
-          accessPath, " "
-        )
-    )
-  )
-}
-
-/**
- * Gets a value of the function-call-related token-based feature named `featureName` associated
- * with the function call `call` and the endpoint `endpoint`.
- *
- * This may in general report multiple strings, each containing a space-separated list of tokens.
- *
- * **Technical details:** This predicate can have multiple values per endpoint and feature name. As
- * a result, the results from this predicate must be concatenated together.  However concatenating
- * other features like the function body tokens is expensive, so for performance reasons we separate
- * out this predicate from those other features.
- */
-private string getACallBasedTokenFeatureComponent(
-  DataFlow::Node endpoint, DataFlow::CallNode call, string featureName
-) {
-  // Performance optimization: Restrict feature extraction to endpoints we've explicitly asked to featurize.
-  endpoint = any(FeaturizationConfig cfg).getAnEndpointToFeaturize() and
-  // Features for endpoints that are an argument to a function call.
-  endpoint = call.getAnArgument() and
-  (
-    // The name of the function being called, e.g. in a call `Artist.findOne(...)`, this is `findOne`.
-    featureName = "calleeName" and result = call.getCalleeName()
-    or
-    // The name of the receiver of the call, e.g. in a call `Artist.findOne(...)`, this is `Artist`.
-    featureName = "receiverName" and result = call.getReceiver().asExpr().(VarRef).getName()
-    or
-    // The argument index of the endpoint, e.g. in `f(a, endpoint, b)`, this is 1.
-    featureName = "argumentIndex" and
-    result = any(int argIndex | call.getArgument(argIndex) = endpoint).toString()
-    or
-    // The name of the API that the function being called originates from, if the function being
-    // called originates from an external API. For example, the endpoint here:
-    //
-    // ```js
-    // const mongoose = require('mongoose'),
-    //   User = mongoose.model('User', null);
-    // User.findOne(ENDPOINT);
-    // ```
-    //
-    // would have a callee API name of `mongoose`.
-    featureName = "calleeApiName" and
-    exists(API::Node apiNode |
-      AccessPaths::accessPaths(apiNode, false, _, result) and call = apiNode.getInducingNode()
-    )
-  )
 }
 
 /**
@@ -163,7 +69,8 @@ private module AccessPaths {
     API::Node node, Boolean includeStructuralInfo, string accessPath, string apiName
   ) {
     //node = API::moduleImport(result)
-    node = API::moduleImport(apiName) and accessPath = apiName
+    node = API::moduleImport(apiName) and
+    accessPath = apiName
     or
     exists(API::Node previousNode, string previousAccessPath |
       previousNode.getDepth() < node.getDepth() and
@@ -285,11 +192,22 @@ private module FunctionNames {
 
 /** Get a name of a supported generic token-based feature. */
 string getASupportedFeatureName() {
+  // allowlist of vetted features that are permitted in production
   result =
-    [
+    any(EndpointFeature f |
-      "enclosingFunctionName", "calleeName", "receiverName", "argumentIndex", "calleeApiName",
+      f instanceof EnclosingFunctionName or
-      "calleeAccessPath", "calleeAccessPathWithStructuralInfo", "enclosingFunctionBody"
+      f instanceof CalleeName or
-    ]
+      f instanceof ReceiverName or
+      f instanceof ArgumentIndex or
+      f instanceof CalleeApiName or
+      f instanceof CalleeAccessPath or
+      f instanceof CalleeAccessPathWithStructuralInfo or
+      f instanceof EnclosingFunctionBody or
+      f instanceof Callee_AccessPath or
+      f instanceof Input_ArgumentIndexAndAccessPathFromCallee or
+      f instanceof Input_AccessPathFromCallee or
+      f instanceof Input_ArgumentIndex
+    ).getName()
 }
 
 /**
@@ -303,3 +221,433 @@ predicate tokenFeatures(DataFlow::Node endpoint, string featureName, string feat
   endpoint = any(FeaturizationConfig cfg).getAnEndpointToFeaturize() and
   featureValue = getTokenFeature(endpoint, featureName)
 }
+
+/**
+ * See EndpointFeauture
+ */
+private newtype TEndpointFeature =
+  TEnclosingFunctionName() or
+  TCalleeName() or
+  TReceiverName() or
+  TArgumentIndex() or
+  TCalleeApiName() or
+  TCalleeAccessPath() or
+  TCalleeAccessPathWithStructuralInfo() or
+  TEnclosingFunctionBody() or
+  TCallee_AccessPath() or
+  TInput_ArgumentIndexAndAccessPathFromCallee() or
+  TInput_AccessPathFromCallee() or
+  TInput_ArgumentIndex()
+
+/**
+ * An implementation of an endpoint feature: produces feature names and values for used in ML.
+ */
+abstract class EndpointFeature extends TEndpointFeature {
+  /**
+   * Gets the name of the feature. Used by the ML model.
+   * Changes to the name of a feature requires training the model again.
+   */
+  abstract string getName();
+
+  /**
+   * Gets the value of the feature. Used by the ML model.
+   * Changes to the value of a feature requires training the model again.
+   */
+  abstract string getValue(DataFlow::Node endpoint);
+
+  string toString() { result = this.getName() }
+}
+
+/**
+ * The feature for the name of the function that encloses the endpoint.
+ */
+class EnclosingFunctionName extends EndpointFeature, TEnclosingFunctionName {
+  override string getName() { result = "enclosingFunctionName" }
+
+  override string getValue(DataFlow::Node endpoint) {
+    result =
+      FunctionNames::getNameToFeaturize(FunctionBodyFeatures::getRepresentativeFunctionForEndpoint(endpoint))
+  }
+}
+
+/**
+ * The feature for the name of the function being called, e.g. in a call `Artist.findOne(...)`, this is `findOne`.
+ */
+class CalleeName extends EndpointFeature, TCalleeName {
+  override string getName() { result = "calleeName" }
+
+  override string getValue(DataFlow::Node endpoint) {
+    result =
+      strictconcat(DataFlow::CallNode call, string component |
+        endpoint = call.getAnArgument() and component = call.getCalleeName()
+      |
+        component, " "
+      )
+  }
+}
+
+/**
+ * The feature for the name of the receiver of the call, e.g. in a call `Artist.findOne(...)`, this is `Artist`.
+ */
+class ReceiverName extends EndpointFeature, TReceiverName {
+  override string getName() { result = "receiverName" }
+
+  override string getValue(DataFlow::Node endpoint) {
+    result =
+      strictconcat(DataFlow::CallNode call, string component |
+        endpoint = call.getAnArgument() and
+        component = call.getReceiver().asExpr().(VarRef).getName()
+      |
+        component, " "
+      )
+  }
+}
+
+/**
+ * The feature for the argument index of the endpoint, e.g. in `f(a, endpoint, b)`, this is 1.
+ */
+class ArgumentIndex extends EndpointFeature, TArgumentIndex {
+  override string getName() { result = "argumentIndex" }
+
+  override string getValue(DataFlow::Node endpoint) {
+    result =
+      strictconcat(DataFlow::CallNode call, string component |
+        endpoint = call.getAnArgument() and
+        component = any(int argIndex | call.getArgument(argIndex) = endpoint).toString()
+      |
+        component, " "
+      )
+  }
+}
+
+/**
+ * The feature for the name of the API that the function being called originates from, if the function being
+ * called originates from an external API. For example, the endpoint here:
+ *
+ * ```js
+ * const mongoose = require('mongoose'),
+ *   User = mongoose.model('User', null);
+ * User.findOne(ENDPOINT);
+ * ```
+ */
+class CalleeApiName extends EndpointFeature, TCalleeApiName {
+  override string getName() { result = "calleeApiName" }
+
+  override string getValue(DataFlow::Node endpoint) {
+    result =
+      strictconcat(API::Node apiNode, string component |
+        endpoint = apiNode.getInducingNode().(DataFlow::CallNode).getAnArgument() and
+        AccessPaths::accessPaths(apiNode, false, _, component)
+      |
+        component, " "
+      )
+  }
+}
+
+/**
+ * The access path of the function being called, both without structural info, if the
+ * function being called originates from an external API. For example, the endpoint here:
+ *
+ * ```js
+ * const mongoose = require('mongoose'),
+ *   User = mongoose.model('User', null);
+ * User.findOne(ENDPOINT);
+ * ```
+ *
+ * would have a callee access path without structural info of `mongoose model findOne`.
+ */
+class CalleeAccessPath extends EndpointFeature, TCalleeAccessPath {
+  override string getName() { result = "calleeAccessPath" }
+
+  override string getValue(DataFlow::Node endpoint) {
+    result =
+      concat(API::Node node, string accessPath |
+        node.getInducingNode().(DataFlow::CallNode).getAnArgument() = endpoint and
+        AccessPaths::accessPaths(node, false, accessPath, _)
+      |
+        accessPath, " "
+      )
+  }
+}
+
+/**
+ * The access path of the function being called, both with structural info, if the
+ * function being called originates from an external API. For example, the endpoint here:
+ *
+ * ```js
+ * const mongoose = require('mongoose'),
+ *   User = mongoose.model('User', null);
+ * User.findOne(ENDPOINT);
+ * ```
+ *
+ * would have a callee access path with structural info of
+ * `mongoose member model instanceorreturn member findOne instanceorreturn`
+ *
+ * These features indicate that the callee comes from (reading the access path backwards) an
+ * instance of the `findOne` member of an instance of the `model` member of the `mongoose`
+ * external library.
+ */
+class CalleeAccessPathWithStructuralInfo extends EndpointFeature,
+  TCalleeAccessPathWithStructuralInfo {
+  override string getName() { result = "calleeAccessPathWithStructuralInfo" }
+
+  override string getValue(DataFlow::Node endpoint) {
+    result =
+      concat(API::Node node, string accessPath |
+        node.getInducingNode().(DataFlow::CallNode).getAnArgument() = endpoint and
+        AccessPaths::accessPaths(node, true, accessPath, _)
+      |
+        accessPath, " "
+      )
+  }
+}
+
+/**
+ * The feature for the natural language tokens from the function that encloses the endpoint in
+ *    the order that they appear in the source code.
+ */
+class EnclosingFunctionBody extends EndpointFeature, TEnclosingFunctionBody {
+  override string getName() { result = "enclosingFunctionBody" }
+
+  override string getValue(DataFlow::Node endpoint) {
+    result =
+      FunctionBodyFeatures::getBodyTokensFeature(FunctionBodyFeatures::getRepresentativeFunctionForEndpoint(endpoint))
+  }
+}
+
+/**
+ * Syntactic utilities for feature value computation.
+ */
+private module SyntacticUtilities {
+  /**
+   * Gets a property initializer value in a an object literal or one of its nested object literals.
+   */
+  Expr getANestedInitializerValue(ObjectExpr o) {
+    exists(Expr init | init = o.getAProperty().getInit().getUnderlyingValue() |
+      result = [init, getANestedInitializerValue(init)]
+    )
+  }
+
+  /**
+   * Computes a simple access path for how a callee can refer to a value that appears in an argument to a call.
+   *
+   * Supports:
+   * - direct arguments
+   * - properties of (nested) objects that are arguments
+   *
+   * Unknown cases and property names results in `?`.
+   */
+  string getSimpleParameterAccessPath(DataFlow::Node node) {
+    if exists(DataFlow::CallNode call | node = call.getArgument(_))
+    then exists(DataFlow::CallNode call, int i | node = call.getArgument(i) | result = i + "")
+    else result = getSimplePropertyAccessPath(node)
+  }
+
+  /**
+   * Computes a simple access path for how a user can refer to a value that appears in an (nested) object.
+   *
+   * Supports:
+   * - properties of (nested) objects
+   *
+   * Unknown cases and property names results in `?`.
+   */
+  string getSimplePropertyAccessPath(DataFlow::Node node) {
+    if exists(ObjectExpr o | o.getAProperty().getInit().getUnderlyingValue() = node.asExpr())
+    then
+      exists(DataFlow::PropWrite w |
+        w.getRhs() = node and
+        result = getSimpleParameterAccessPath(w.getBase()) + "." + getPropertyNameOrUnknown(w)
+      )
+    else result = getUnknownSymbol()
+  }
+
+  /**
+   * Computes a simple access path for a node.
+   *
+   * Supports:
+   * - variable reads (including `this` and `super`)
+   * - imports
+   * - await
+   * - property reads
+   * - invocations
+   *
+   * Unknown cases and property names results in `?`.
+   */
+  string getSimpleAccessPath(DataFlow::Node node) {
+    exists(Expr e | e = node.asExpr().getUnderlyingValue() |
+      if e instanceof SuperAccess
+      then result = "super"
+      else
+        if e instanceof ThisAccess
+        then result = "this"
+        else
+          if e instanceof VarAccess
+          then result = e.(VarAccess).getName()
+          else
+            if e instanceof Import
+            then result = "import(" + getSimpleImportPath(e) + ")"
+            else
+              if e instanceof AwaitExpr
+              then result = "(await " + getSimpleAccessPath(e.(AwaitExpr).getOperand().flow()) + ")"
+              else
+                if node instanceof DataFlow::PropRead
+                then
+                  result =
+                    getSimpleAccessPath(node.(DataFlow::PropRead).getBase()) + "." +
+                      getPropertyNameOrUnknown(node)
+                else
+                  if node instanceof DataFlow::InvokeNode
+                  then
+                    result = getSimpleAccessPath(node.(DataFlow::InvokeNode).getCalleeNode()) + "()"
+                  else result = getUnknownSymbol()
+    )
+  }
+
+  string getUnknownSymbol() { result = "?" }
+
+  /**
+   * Gets the imported path.
+   *
+   * XXX To avoid teaching the ML model about npm packages, only relative paths are supported
+   *
+   * Unknown paths result in `?`.
+   */
+  string getSimpleImportPath(Import i) {
+    if exists(i.getImportedPath().getValue())
+    then
+      exists(string p | p = i.getImportedPath().getValue() |
+        if p.matches(".%") then result = "\"p\"" else result = "!" // hide absolute imports from the ML training
+      )
+    else result = getUnknownSymbol()
+  }
+
+  /**
+   * Gets the property name of a property reference or `?` if it is unknown.
+   */
+  string getPropertyNameOrUnknown(DataFlow::PropRef ref) {
+    if exists(ref.getPropertyName())
+    then result = ref.getPropertyName()
+    else result = getUnknownSymbol()
+  }
+}
+
+/**
+ * The feature for the access path of the callee node of a call that has an argument that "contains" the endpoint.
+ *
+ * "Containment" is syntactic, and currently means that the endpoint is an argument to the call, or that the endpoint is a (nested) property value of an argument.
+ *
+ * This feature is intended as a superior version of the many `Callee*` features.
+ *
+ * Examples:
+ * ```
+ * foo(endpoint); // -> foo
+ * foo.bar(endpoint); // -> foo.bar
+ * foo.bar({ baz: endpoint }); // -> foo.bar
+ * this.foo.bar(endpoint); // -> this.foo.bar
+ * foo[complex()].bar(endpoint); // -> foo.?.bar
+ * ```
+ */
+class Callee_AccessPath extends EndpointFeature, TCallee_AccessPath {
+  override string getName() { result = "Callee_AccessPath" }
+
+  override string getValue(DataFlow::Node endpoint) {
+    exists(DataFlow::InvokeNode invk |
+      result = SyntacticUtilities::getSimpleAccessPath(invk.getCalleeNode()) and
+      // ignore the unknown path
+      not result = SyntacticUtilities::getUnknownSymbol() and
+      (
+        invk.getAnArgument() = endpoint or
+        SyntacticUtilities::getANestedInitializerValue(invk.getAnArgument()
+              .asExpr()
+              .getUnderlyingValue()).flow() = endpoint
+      )
+    )
+  }
+}
+
+/**
+ * The feature for how a callee can refer to a the endpoint that is "contained" in an argument to a call
+ *
+ * "Containment" is syntactic, and currently means that the endpoint is an argument to the call, or that the endpoint is a (nested) property value of an argument.
+ *
+ * This feature is intended as a superior version of the `ArgumentIndexFeature`.
+ *
+ * Examples:
+ * ```
+ * foo(endpoint); // -> 0
+ * foo({ bar: endpoint }); // -> 0.bar
+ * foo(x, { bar: { baz: endpoint } }); // -> 1.bar.baz
+ * ```
+ */
+class Input_ArgumentIndexAndAccessPathFromCallee extends EndpointFeature,
+  TInput_ArgumentIndexAndAccessPathFromCallee {
+  override string getName() { result = "Input_ArgumentIndexAndAccessPathFromCallee" }
+
+  override string getValue(DataFlow::Node endpoint) {
+    exists(DataFlow::InvokeNode invk |
+      result = SyntacticUtilities::getSimpleParameterAccessPath(endpoint) and
+      (
+        invk.getAnArgument() = endpoint or
+        SyntacticUtilities::getANestedInitializerValue(invk.getAnArgument()
+              .asExpr()
+              .getUnderlyingValue()).flow() = endpoint
+      )
+    )
+  }
+}
+
+/**
+ * The feature for how a callee can refer to a the endpoint that is "contained" in some argument to a call
+ *
+ * "Containment" is syntactic, and currently means that the endpoint is an argument to the call, or that the endpoint is a (nested) property value of an argument.
+ *
+ * This feature is intended as a superior version of the `ArgumentIndexFeature`.
+ *
+ * Examples:
+ * ```
+ * foo({ bar: endpoint }); // -> bar
+ * foo(x, { bar: { baz: endpoint } }); // -> bar.baz
+ * ```
+ */
+class Input_AccessPathFromCallee extends EndpointFeature, TInput_AccessPathFromCallee {
+  override string getName() { result = "Input_AccessPathFromCallee" }
+
+  override string getValue(DataFlow::Node endpoint) {
+    exists(DataFlow::InvokeNode invk |
+      result = SyntacticUtilities::getSimpleParameterAccessPath(endpoint) and
+      SyntacticUtilities::getANestedInitializerValue(invk.getAnArgument()
+            .asExpr()
+            .getUnderlyingValue()).flow() = endpoint
+    )
+  }
+}
+
+/**
+ * The feature for how the index of an argument that "contains" and endpoint.
+ *
+ * "Containment" is syntactic, and currently means that the endpoint is an argument to the call, or that the endpoint is a (nested) property value of an argument.
+ *
+ * This feature is intended as a superior version of the `ArgumentIndexFeature`.
+ *
+ * Examples:
+ * ```
+ * foo(endpoint); // -> 0
+ * foo({ bar: endpoint }); // -> 0
+ * foo(x, { bar: { baz: endpoint } }); // -> 1
+ * ```
+ */
+class Input_ArgumentIndex extends EndpointFeature, TInput_ArgumentIndex {
+  override string getName() { result = "Input_ArgumentIndex" }
+
+  override string getValue(DataFlow::Node endpoint) {
+    exists(DataFlow::InvokeNode invk, DataFlow::Node arg, int i | arg = invk.getArgument(i) |
+      result = i + "" and
+      (
+        invk.getAnArgument() = endpoint
+        or
+        SyntacticUtilities::getANestedInitializerValue(arg.asExpr().getUnderlyingValue()).flow() =
+          endpoint
+      )
+    )
+  }
+}

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/debug/CompareFeatures.ql

@@ -0,0 +1,23 @@
+import experimental.adaptivethreatmodeling.TaintedPathATM
+import experimental.adaptivethreatmodeling.EndpointFeatures
+import experimental.adaptivethreatmodeling.EndpointScoring
+
+string getValueOrNone(EndpointFeature feature, DataFlow::Node endpoint) {
+  if exists(feature.getValue(endpoint)) then feature.getValue(endpoint) = result else isNone(result)
+}
+
+predicate isNone(string value) { value = "" }
+
+// query for comparing feature values
+from
+  DataFlow::Node endpoint, EndpointFeature feature1, EndpointFeature feature2, string featureValue1,
+  string featureValue2
+where
+  feature1 instanceof Input_ArgumentIndex and
+  feature2 instanceof ArgumentIndex and
+  featureValue1 = getValueOrNone(feature1, endpoint) and
+  featureValue2 = getValueOrNone(feature2, endpoint) and
+  featureValue1 != featureValue2 and
+  isNone([featureValue1, featureValue2])
+select endpoint, endpoint.getFile().getBaseName() as file, endpoint.getStartLine() as line,
+  featureValue1, featureValue2

javascript/ql/experimental/adaptivethreatmodeling/test/generic_feature_testing/EmptyFeature.ql

@@ -0,0 +1,9 @@
+import javascript
+import experimental.adaptivethreatmodeling.EndpointFeatures
+import experimental.adaptivethreatmodeling.FeaturizationConfig
+import TestUtil
+
+// every feature must produce a value for at least one endpoint, otherwise the feature is completely broken, or a relevant test example is missing
+from EndpointFeature feature
+where forall(Endpoint endpoint | not exists(feature.getValue(endpoint)))
+select feature.getName()

javascript/ql/experimental/adaptivethreatmodeling/test/generic_feature_testing/FeatureValue.expected

@@ -0,0 +1,127 @@
+| test.html:2:61:2:68 | endpoint | Callee_AccessPath | $event.target.files.item |
+| test.html:2:61:2:68 | endpoint | Input_ArgumentIndex | 0 |
+| test.html:2:61:2:68 | endpoint | Input_ArgumentIndexAndAccessPathFromCallee | 0 |
+| test.html:2:61:2:68 | endpoint | argumentIndex | 0 |
+| test.html:2:61:2:68 | endpoint | calleeAccessPath |  |
+| test.html:2:61:2:68 | endpoint | calleeAccessPathWithStructuralInfo |  |
+| test.html:2:61:2:68 | endpoint | calleeName | item |
+| test.js:2:7:2:14 | endpoint | Callee_AccessPath | f |
+| test.js:2:7:2:14 | endpoint | Input_ArgumentIndex | 0 |
+| test.js:2:7:2:14 | endpoint | Input_ArgumentIndexAndAccessPathFromCallee | 0 |
+| test.js:2:7:2:14 | endpoint | argumentIndex | 0 |
+| test.js:2:7:2:14 | endpoint | calleeAccessPath |  |
+| test.js:2:7:2:14 | endpoint | calleeAccessPathWithStructuralInfo |  |
+| test.js:2:7:2:14 | endpoint | calleeName | f |
+| test.js:2:7:2:14 | endpoint | enclosingFunctionBody | f endpoint f p endpoint f p q endpoint o m endpoint o m p endpoint o m p q endpoint F endpoint o m m m endpoint f endpoint o x m endpoint o m x p m endpoint p endpoint foo bar baz endpoint foo bar endpoint |
+| test.js:2:7:2:14 | endpoint | enclosingFunctionName |  |
+| test.js:3:11:3:18 | endpoint | Callee_AccessPath | f |
+| test.js:3:11:3:18 | endpoint | Input_AccessPathFromCallee | 0.p |
+| test.js:3:11:3:18 | endpoint | Input_ArgumentIndex | 0 |
+| test.js:3:11:3:18 | endpoint | Input_ArgumentIndexAndAccessPathFromCallee | 0.p |
+| test.js:3:11:3:18 | endpoint | calleeAccessPath |  |
+| test.js:3:11:3:18 | endpoint | calleeAccessPathWithStructuralInfo |  |
+| test.js:3:11:3:18 | endpoint | enclosingFunctionBody | f endpoint f p endpoint f p q endpoint o m endpoint o m p endpoint o m p q endpoint F endpoint o m m m endpoint f endpoint o x m endpoint o m x p m endpoint p endpoint foo bar baz endpoint foo bar endpoint |
+| test.js:3:11:3:18 | endpoint | enclosingFunctionName |  |
+| test.js:4:15:4:22 | endpoint | Callee_AccessPath | f |
+| test.js:4:15:4:22 | endpoint | Input_AccessPathFromCallee | 0.p.q |
+| test.js:4:15:4:22 | endpoint | Input_ArgumentIndex | 0 |
+| test.js:4:15:4:22 | endpoint | Input_ArgumentIndexAndAccessPathFromCallee | 0.p.q |
+| test.js:4:15:4:22 | endpoint | calleeAccessPath |  |
+| test.js:4:15:4:22 | endpoint | calleeAccessPathWithStructuralInfo |  |
+| test.js:4:15:4:22 | endpoint | enclosingFunctionBody | f endpoint f p endpoint f p q endpoint o m endpoint o m p endpoint o m p q endpoint F endpoint o m m m endpoint f endpoint o x m endpoint o m x p m endpoint p endpoint foo bar baz endpoint foo bar endpoint |
+| test.js:4:15:4:22 | endpoint | enclosingFunctionName |  |
+| test.js:5:9:5:16 | endpoint | Callee_AccessPath | o.m |
+| test.js:5:9:5:16 | endpoint | Input_ArgumentIndex | 0 |
+| test.js:5:9:5:16 | endpoint | Input_ArgumentIndexAndAccessPathFromCallee | 0 |
+| test.js:5:9:5:16 | endpoint | argumentIndex | 0 |
+| test.js:5:9:5:16 | endpoint | calleeAccessPath |  |
+| test.js:5:9:5:16 | endpoint | calleeAccessPathWithStructuralInfo |  |
+| test.js:5:9:5:16 | endpoint | calleeName | m |
+| test.js:5:9:5:16 | endpoint | enclosingFunctionBody | f endpoint f p endpoint f p q endpoint o m endpoint o m p endpoint o m p q endpoint F endpoint o m m m endpoint f endpoint o x m endpoint o m x p m endpoint p endpoint foo bar baz endpoint foo bar endpoint |
+| test.js:5:9:5:16 | endpoint | enclosingFunctionName |  |
+| test.js:5:9:5:16 | endpoint | receiverName | o |
+| test.js:6:13:6:20 | endpoint | Callee_AccessPath | o.m |
+| test.js:6:13:6:20 | endpoint | Input_AccessPathFromCallee | 0.p |
+| test.js:6:13:6:20 | endpoint | Input_ArgumentIndex | 0 |
+| test.js:6:13:6:20 | endpoint | Input_ArgumentIndexAndAccessPathFromCallee | 0.p |
+| test.js:6:13:6:20 | endpoint | calleeAccessPath |  |
+| test.js:6:13:6:20 | endpoint | calleeAccessPathWithStructuralInfo |  |
+| test.js:6:13:6:20 | endpoint | enclosingFunctionBody | f endpoint f p endpoint f p q endpoint o m endpoint o m p endpoint o m p q endpoint F endpoint o m m m endpoint f endpoint o x m endpoint o m x p m endpoint p endpoint foo bar baz endpoint foo bar endpoint |
+| test.js:6:13:6:20 | endpoint | enclosingFunctionName |  |
+| test.js:7:17:7:24 | endpoint | Callee_AccessPath | o.m |
+| test.js:7:17:7:24 | endpoint | Input_AccessPathFromCallee | 0.p.q |
+| test.js:7:17:7:24 | endpoint | Input_ArgumentIndex | 0 |
+| test.js:7:17:7:24 | endpoint | Input_ArgumentIndexAndAccessPathFromCallee | 0.p.q |
+| test.js:7:17:7:24 | endpoint | calleeAccessPath |  |
+| test.js:7:17:7:24 | endpoint | calleeAccessPathWithStructuralInfo |  |
+| test.js:7:17:7:24 | endpoint | enclosingFunctionBody | f endpoint f p endpoint f p q endpoint o m endpoint o m p endpoint o m p q endpoint F endpoint o m m m endpoint f endpoint o x m endpoint o m x p m endpoint p endpoint foo bar baz endpoint foo bar endpoint |
+| test.js:7:17:7:24 | endpoint | enclosingFunctionName |  |
+| test.js:8:11:8:18 | endpoint | Callee_AccessPath | F |
+| test.js:8:11:8:18 | endpoint | Input_ArgumentIndex | 0 |
+| test.js:8:11:8:18 | endpoint | Input_ArgumentIndexAndAccessPathFromCallee | ? |
+| test.js:8:11:8:18 | endpoint | calleeAccessPath |  |
+| test.js:8:11:8:18 | endpoint | calleeAccessPathWithStructuralInfo |  |
+| test.js:8:11:8:18 | endpoint | enclosingFunctionBody | f endpoint f p endpoint f p q endpoint o m endpoint o m p endpoint o m p q endpoint F endpoint o m m m endpoint f endpoint o x m endpoint o m x p m endpoint p endpoint foo bar baz endpoint foo bar endpoint |
+| test.js:8:11:8:18 | endpoint | enclosingFunctionName |  |
+| test.js:9:17:9:24 | endpoint | Callee_AccessPath | o.m().m().m |
+| test.js:9:17:9:24 | endpoint | Input_ArgumentIndex | 0 |
+| test.js:9:17:9:24 | endpoint | Input_ArgumentIndexAndAccessPathFromCallee | 0 |
+| test.js:9:17:9:24 | endpoint | argumentIndex | 0 |
+| test.js:9:17:9:24 | endpoint | calleeAccessPath |  |
+| test.js:9:17:9:24 | endpoint | calleeAccessPathWithStructuralInfo |  |
+| test.js:9:17:9:24 | endpoint | calleeName | m |
+| test.js:9:17:9:24 | endpoint | enclosingFunctionBody | f endpoint f p endpoint f p q endpoint o m endpoint o m p endpoint o m p q endpoint F endpoint o m m m endpoint f endpoint o x m endpoint o m x p m endpoint p endpoint foo bar baz endpoint foo bar endpoint |
+| test.js:9:17:9:24 | endpoint | enclosingFunctionName |  |
+| test.js:10:9:10:16 | endpoint | Callee_AccessPath | f() |
+| test.js:10:9:10:16 | endpoint | Input_ArgumentIndex | 0 |
+| test.js:10:9:10:16 | endpoint | Input_ArgumentIndexAndAccessPathFromCallee | 0 |
+| test.js:10:9:10:16 | endpoint | argumentIndex | 0 |
+| test.js:10:9:10:16 | endpoint | calleeAccessPath |  |
+| test.js:10:9:10:16 | endpoint | calleeAccessPathWithStructuralInfo |  |
+| test.js:10:9:10:16 | endpoint | enclosingFunctionBody | f endpoint f p endpoint f p q endpoint o m endpoint o m p endpoint o m p q endpoint F endpoint o m m m endpoint f endpoint o x m endpoint o m x p m endpoint p endpoint foo bar baz endpoint foo bar endpoint |
+| test.js:10:9:10:16 | endpoint | enclosingFunctionName |  |
+| test.js:11:12:11:19 | endpoint | Callee_AccessPath | o.?.m |
+| test.js:11:12:11:19 | endpoint | Input_ArgumentIndex | 0 |
+| test.js:11:12:11:19 | endpoint | Input_ArgumentIndexAndAccessPathFromCallee | 0 |
+| test.js:11:12:11:19 | endpoint | argumentIndex | 0 |
+| test.js:11:12:11:19 | endpoint | calleeAccessPath |  |
+| test.js:11:12:11:19 | endpoint | calleeAccessPathWithStructuralInfo |  |
+| test.js:11:12:11:19 | endpoint | calleeName | m |
+| test.js:11:12:11:19 | endpoint | enclosingFunctionBody | f endpoint f p endpoint f p q endpoint o m endpoint o m p endpoint o m p q endpoint F endpoint o m m m endpoint f endpoint o x m endpoint o m x p m endpoint p endpoint foo bar baz endpoint foo bar endpoint |
+| test.js:11:12:11:19 | endpoint | enclosingFunctionName |  |
+| test.js:12:16:12:23 | endpoint | Callee_AccessPath | o.m.?.p.m |
+| test.js:12:16:12:23 | endpoint | Input_ArgumentIndex | 0 |
+| test.js:12:16:12:23 | endpoint | Input_ArgumentIndexAndAccessPathFromCallee | 0 |
+| test.js:12:16:12:23 | endpoint | argumentIndex | 0 |
+| test.js:12:16:12:23 | endpoint | calleeAccessPath |  |
+| test.js:12:16:12:23 | endpoint | calleeAccessPathWithStructuralInfo |  |
+| test.js:12:16:12:23 | endpoint | calleeName | m |
+| test.js:12:16:12:23 | endpoint | enclosingFunctionBody | f endpoint f p endpoint f p q endpoint o m endpoint o m p endpoint o m p q endpoint F endpoint o m m m endpoint f endpoint o x m endpoint o m x p m endpoint p endpoint foo bar baz endpoint foo bar endpoint |
+| test.js:12:16:12:23 | endpoint | enclosingFunctionName |  |
+| test.js:13:15:13:22 | endpoint | Callee_AccessPath | (await p) |
+| test.js:13:15:13:22 | endpoint | Input_ArgumentIndex | 0 |
+| test.js:13:15:13:22 | endpoint | Input_ArgumentIndexAndAccessPathFromCallee | 0 |
+| test.js:13:15:13:22 | endpoint | argumentIndex | 0 |
+| test.js:13:15:13:22 | endpoint | calleeAccessPath |  |
+| test.js:13:15:13:22 | endpoint | calleeAccessPathWithStructuralInfo |  |
+| test.js:13:15:13:22 | endpoint | enclosingFunctionBody | f endpoint f p endpoint f p q endpoint o m endpoint o m p endpoint o m p q endpoint F endpoint o m m m endpoint f endpoint o x m endpoint o m x p m endpoint p endpoint foo bar baz endpoint foo bar endpoint |
+| test.js:13:15:13:22 | endpoint | enclosingFunctionName |  |
+| test.js:14:27:14:34 | endpoint | Callee_AccessPath | import(!).bar.baz |
+| test.js:14:27:14:34 | endpoint | Input_ArgumentIndex | 0 |
+| test.js:14:27:14:34 | endpoint | Input_ArgumentIndexAndAccessPathFromCallee | 0 |
+| test.js:14:27:14:34 | endpoint | argumentIndex | 0 |
+| test.js:14:27:14:34 | endpoint | calleeAccessPath | foo bar baz |
+| test.js:14:27:14:34 | endpoint | calleeAccessPathWithStructuralInfo | foo member bar member baz instanceorreturn |
+| test.js:14:27:14:34 | endpoint | calleeApiName | foo |
+| test.js:14:27:14:34 | endpoint | calleeName | baz |
+| test.js:14:27:14:34 | endpoint | enclosingFunctionBody | f endpoint f p endpoint f p q endpoint o m endpoint o m p endpoint o m p q endpoint F endpoint o m m m endpoint f endpoint o x m endpoint o m x p m endpoint p endpoint foo bar baz endpoint foo bar endpoint |
+| test.js:14:27:14:34 | endpoint | enclosingFunctionName |  |
+| test.js:16:13:16:20 | endpoint | Callee_AccessPath | bar |
+| test.js:16:13:16:20 | endpoint | Input_ArgumentIndex | 0 |
+| test.js:16:13:16:20 | endpoint | Input_ArgumentIndexAndAccessPathFromCallee | 0 |
+| test.js:16:13:16:20 | endpoint | argumentIndex | 0 |
+| test.js:16:13:16:20 | endpoint | calleeAccessPath |  |
+| test.js:16:13:16:20 | endpoint | calleeAccessPathWithStructuralInfo |  |
+| test.js:16:13:16:20 | endpoint | calleeName | bar |
+| test.js:16:13:16:20 | endpoint | enclosingFunctionBody | f endpoint f p endpoint f p q endpoint o m endpoint o m p endpoint o m p q endpoint F endpoint o m m m endpoint f endpoint o x m endpoint o m x p m endpoint p endpoint foo bar baz endpoint foo bar endpoint |
+| test.js:16:13:16:20 | endpoint | enclosingFunctionName |  |

javascript/ql/experimental/adaptivethreatmodeling/test/generic_feature_testing/FeatureValue.ql

@@ -0,0 +1,7 @@
+import javascript
+import experimental.adaptivethreatmodeling.EndpointFeatures
+import TestUtil
+
+// detailed output for the nearby tests
+from Endpoint endpoint, EndpointFeature feature
+select endpoint, feature.getName(), feature.getValue(endpoint)

javascript/ql/experimental/adaptivethreatmodeling/test/generic_feature_testing/NonFeaturizedEndpoint.ql

@@ -0,0 +1,8 @@
+import javascript
+import experimental.adaptivethreatmodeling.EndpointFeatures
+import TestUtil
+
+// every endpoint should have at least one feature value, otherwise the test source is likely malformed
+from Endpoint endpoint
+where not exists(EndpointFeature f | exists(f.getValue(endpoint)))
+select endpoint

javascript/ql/experimental/adaptivethreatmodeling/test/generic_feature_testing/NonFunctionalFeature.ql

@@ -0,0 +1,8 @@
+import javascript
+import experimental.adaptivethreatmodeling.EndpointFeatures
+import TestUtil
+
+// every feature must produce a single value for each endpoint that it computes a value for, per the contract of the `scoreEndpoints` HOP
+from Endpoint endpoint, EndpointFeature feature, int arity
+where arity = count(feature.getValue(endpoint)) and arity > 1
+select endpoint, feature.getName(), arity

javascript/ql/experimental/adaptivethreatmodeling/test/generic_feature_testing/TestUtil.qll

@@ -0,0 +1,17 @@
+import javascript
+import experimental.adaptivethreatmodeling.FeaturizationConfig
+
+/**
+ * A featurization config that featurizes all endpoints.
+ *
+ * Ideally this should not be in here, but it is needed for EnclosingFunctionName and EnclosingFunctionBody due to performance considerations :(.
+ */
+class NoRestrictionsFeaturizationConfig extends FeaturizationConfig {
+  NoRestrictionsFeaturizationConfig() { this = "NoRestrictionsFeaturization" }
+
+  override DataFlow::Node getAnEndpointToFeaturize() { any() }
+}
+
+class Endpoint extends DataFlow::Node {
+  Endpoint() { this.asExpr().(VarAccess).getName() = "endpoint" }
+}

javascript/ql/experimental/adaptivethreatmodeling/test/generic_feature_testing/test.html

@@ -0,0 +1,3 @@
+<div class="form-group">
+    <input (change)="restoreBackup($event.target.files.item(endpoint))" />
+</div>

javascript/ql/experimental/adaptivethreatmodeling/test/generic_feature_testing/test.js

@@ -0,0 +1,18 @@
+(async function(){
+    f(endpoint);
+    f({p: endpoint});
+    f({p: {q: endpoint}});
+    o.m(endpoint);
+    o.m({p: endpoint});
+    o.m({p: {q: endpoint}});
+    new F(endpoint);
+    o.m().m().m(endpoint);
+    f()(endpoint);
+    o[x].m(endpoint);
+    o.m[x].p.m(endpoint);
+    (await p)(endpoint);
+    import("foo").bar.baz(endpoint);
+    function foo() {
+        bar(endpoint);
+    }
+});