Merge pull request #13239 from github/tausbn/automodel-application-mode

Java: Add QL support for automodel application mode

java/ql/src/Telemetry/AutomodelApplicationModeCharacteristics.qll

@@ -0,0 +1,444 @@
+/**
+ * For internal use only.
+ */
+
+private import java
+private import semmle.code.Location as Location
+private import semmle.code.java.dataflow.DataFlow
+private import semmle.code.java.dataflow.TaintTracking
+private import semmle.code.java.security.PathCreation
+private import semmle.code.java.dataflow.ExternalFlow as ExternalFlow
+private import semmle.code.java.dataflow.internal.FlowSummaryImpl as FlowSummaryImpl
+private import semmle.code.java.security.ExternalAPIs as ExternalAPIs
+private import semmle.code.java.Expr as Expr
+private import semmle.code.java.security.QueryInjection
+private import semmle.code.java.security.RequestForgery
+private import semmle.code.java.dataflow.internal.ModelExclusions as ModelExclusions
+private import AutomodelJavaUtil as AutomodelJavaUtil
+private import semmle.code.java.security.PathSanitizer as PathSanitizer
+private import AutomodelSharedGetCallable as AutomodelSharedGetCallable
+import AutomodelSharedCharacteristics as SharedCharacteristics
+import AutomodelEndpointTypes as AutomodelEndpointTypes
+
+newtype JavaRelatedLocationType = CallContext()
+
+/**
+ * A class representing nodes that are arguments to calls.
+ */
+private class ArgumentNode extends DataFlow::Node {
+  ArgumentNode() { this.asExpr() = [any(Call c).getAnArgument(), any(Call c).getQualifier()] }
+}
+
+/**
+ * A candidates implementation.
+ *
+ * Some important notes:
+ *  - This mode is using arguments as endpoints.
+ *  - We use the `CallContext` (the surrounding call expression) as related location.
+ */
+module ApplicationCandidatesImpl implements SharedCharacteristics::CandidateSig {
+  // for documentation of the implementations here, see the QLDoc in the CandidateSig signature module.
+  class Endpoint = ArgumentNode;
+
+  class EndpointType = AutomodelEndpointTypes::EndpointType;
+
+  class NegativeEndpointType = AutomodelEndpointTypes::NegativeSinkType;
+
+  class RelatedLocation = Location::Top;
+
+  class RelatedLocationType = JavaRelatedLocationType;
+
+  // Sanitizers are currently not modeled in MaD. TODO: check if this has large negative impact.
+  predicate isSanitizer(Endpoint e, EndpointType t) {
+    exists(t) and
+    (
+      e.getType() instanceof BoxedType
+      or
+      e.getType() instanceof PrimitiveType
+      or
+      e.getType() instanceof NumberType
+    )
+    or
+    t instanceof AutomodelEndpointTypes::TaintedPathSinkType and
+    e instanceof PathSanitizer::PathInjectionSanitizer
+  }
+
+  RelatedLocation asLocation(Endpoint e) { result = e.asExpr() }
+
+  predicate isKnownKind = AutomodelJavaUtil::isKnownKind/3;
+
+  predicate isSink(Endpoint e, string kind) {
+    exists(string package, string type, string name, string signature, string ext, string input |
+      sinkSpec(e, package, type, name, signature, ext, input) and
+      ExternalFlow::sinkModel(package, type, _, name, [signature, ""], ext, input, kind, _)
+    )
+    or
+    isCustomSink(e, kind)
+  }
+
+  predicate isNeutral(Endpoint e) {
+    exists(string package, string type, string name, string signature |
+      sinkSpec(e, package, type, name, signature, _, _) and
+      ExternalFlow::neutralModel(package, type, name, [signature, ""], _, _)
+    )
+  }
+
+  additional predicate sinkSpec(
+    Endpoint e, string package, string type, string name, string signature, string ext, string input
+  ) {
+    ApplicationModeGetCallable::getCallable(e).hasQualifiedName(package, type, name) and
+    signature = ExternalFlow::paramsString(ApplicationModeGetCallable::getCallable(e)) and
+    ext = "" and
+    (
+      exists(Call c, int argIdx |
+        e.asExpr() = c.getArgument(argIdx) and
+        input = AutomodelJavaUtil::getArgumentForIndex(argIdx)
+      )
+      or
+      exists(Call c |
+        e.asExpr() = c.getQualifier() and input = AutomodelJavaUtil::getArgumentForIndex(-1)
+      )
+    )
+  }
+
+  /**
+   * Gets the related location for the given endpoint.
+   *
+   * The only related location we model is the the call expression surrounding to
+   * which the endpoint is either argument or qualifier (known as the call context).
+   */
+  RelatedLocation getRelatedLocation(Endpoint e, RelatedLocationType type) {
+    type = CallContext() and
+    result = any(Call c | e.asExpr() = [c.getAnArgument(), c.getQualifier()])
+  }
+}
+
+private class JavaCallable = Callable;
+
+private module ApplicationModeGetCallable implements AutomodelSharedGetCallable::GetCallableSig {
+  class Callable = JavaCallable;
+
+  class Endpoint = ApplicationCandidatesImpl::Endpoint;
+
+  /**
+   * Returns the API callable being modeled.
+   */
+  Callable getCallable(Endpoint e) {
+    exists(Call c |
+      e.asExpr() = [c.getAnArgument(), c.getQualifier()] and
+      result = c.getCallee()
+    )
+  }
+}
+
+/**
+ * Contains endpoints that are defined in QL code rather than as a MaD model. Ideally this predicate
+ * should be empty.
+ */
+private predicate isCustomSink(Endpoint e, string kind) {
+  e.asExpr() instanceof ArgumentToExec and kind = "command injection"
+  or
+  e instanceof RequestForgerySink and kind = "request forgery"
+  or
+  e instanceof QueryInjectionSink and kind = "sql"
+}
+
+module CharacteristicsImpl =
+  SharedCharacteristics::SharedCharacteristics<ApplicationCandidatesImpl>;
+
+class EndpointCharacteristic = CharacteristicsImpl::EndpointCharacteristic;
+
+class Endpoint = ApplicationCandidatesImpl::Endpoint;
+
+/*
+ * Predicates that are used to surface prompt examples and candidates for classification with an ML model.
+ */
+
+/**
+ * A MetadataExtractor that extracts metadata for application mode.
+ */
+class ApplicationModeMetadataExtractor extends string {
+  ApplicationModeMetadataExtractor() { this = "ApplicationModeMetadataExtractor" }
+
+  predicate hasMetadata(
+    Endpoint e, string package, string type, string subtypes, string name, string signature,
+    string input
+  ) {
+    exists(Call call, Callable callable, int argIdx |
+      call.getCallee() = callable and
+      (
+        e.asExpr() = call.getArgument(argIdx)
+        or
+        e.asExpr() = call.getQualifier() and argIdx = -1
+      ) and
+      input = AutomodelJavaUtil::getArgumentForIndex(argIdx) and
+      package = callable.getDeclaringType().getPackage().getName() and
+      // we're using the erased types because the MaD convention is to not specify type parameters.
+      // Whether something is or isn't a sink doesn't usually depend on the type parameters.
+      type = callable.getDeclaringType().getErasure().(RefType).nestedName() and
+      subtypes = AutomodelJavaUtil::considerSubtypes(callable).toString() and
+      name = callable.getName() and
+      signature = ExternalFlow::paramsString(callable)
+    )
+  }
+}
+
+/*
+ * EndpointCharacteristic classes that are specific to Automodel for Java.
+ */
+
+/**
+ * A negative characteristic that indicates that an is-style boolean method is unexploitable even if it is a sink.
+ *
+ * A sink is highly unlikely to be exploitable if its callable's name starts with `is` and the callable has a boolean return
+ * type (e.g. `isDirectory`). These kinds of calls normally do only checks, and appear before the proper call that does
+ * the dangerous/interesting thing, so we want the latter to be modeled as the sink.
+ *
+ * TODO: this might filter too much, it's possible that methods with more than one parameter contain interesting sinks
+ */
+private class UnexploitableIsCharacteristic extends CharacteristicsImpl::NotASinkCharacteristic {
+  UnexploitableIsCharacteristic() { this = "unexploitable (is-style boolean method)" }
+
+  override predicate appliesToEndpoint(Endpoint e) {
+    not ApplicationCandidatesImpl::isSink(e, _) and
+    ApplicationModeGetCallable::getCallable(e).getName().matches("is%") and
+    ApplicationModeGetCallable::getCallable(e).getReturnType() instanceof BooleanType
+  }
+}
+
+/**
+ * A negative characteristic that indicates that an existence-checking boolean method is unexploitable even if it is a
+ * sink.
+ *
+ * A sink is highly unlikely to be exploitable if its callable's name is `exists` or `notExists` and the callable has a
+ * boolean return type. These kinds of calls normally do only checks, and appear before the proper call that does the
+ * dangerous/interesting thing, so we want the latter to be modeled as the sink.
+ */
+private class UnexploitableExistsCharacteristic extends CharacteristicsImpl::NotASinkCharacteristic {
+  UnexploitableExistsCharacteristic() { this = "unexploitable (existence-checking boolean method)" }
+
+  override predicate appliesToEndpoint(Endpoint e) {
+    not ApplicationCandidatesImpl::isSink(e, _) and
+    exists(Callable callable |
+      callable = ApplicationModeGetCallable::getCallable(e) and
+      callable.getName().toLowerCase() = ["exists", "notexists"] and
+      callable.getReturnType() instanceof BooleanType
+    )
+  }
+}
+
+/**
+ * A negative characteristic that indicates that an endpoint is an argument to an exception, which is not a sink.
+ */
+private class ExceptionCharacteristic extends CharacteristicsImpl::NotASinkCharacteristic {
+  ExceptionCharacteristic() { this = "exception" }
+
+  override predicate appliesToEndpoint(Endpoint e) {
+    ApplicationModeGetCallable::getCallable(e).getDeclaringType().getASupertype*() instanceof
+      TypeThrowable
+  }
+}
+
+/**
+ * A negative characteristic that indicates that an endpoint is a MaD taint step. MaD modeled taint steps are global,
+ * so they are not sinks for any query. Non-MaD taint steps might be specific to a particular query, so we don't
+ * filter those out.
+ */
+private class IsMaDTaintStepCharacteristic extends CharacteristicsImpl::NotASinkCharacteristic {
+  IsMaDTaintStepCharacteristic() { this = "taint step" }
+
+  override predicate appliesToEndpoint(Endpoint e) {
+    FlowSummaryImpl::Private::Steps::summaryThroughStepValue(e, _, _) or
+    FlowSummaryImpl::Private::Steps::summaryThroughStepTaint(e, _, _) or
+    FlowSummaryImpl::Private::Steps::summaryGetterStep(e, _, _, _) or
+    FlowSummaryImpl::Private::Steps::summarySetterStep(e, _, _, _)
+  }
+}
+
+/**
+ * A negative characteristic that filters out qualifiers that are classes (i.e. static calls). These
+ * are unlikely to have any non-trivial flow going into them.
+ *
+ * Technically, an accessed type _could_ come from outside of the source code, but there's not
+ * much likelihood of that being user-controlled.
+ */
+private class ClassQualifierCharacteristic extends CharacteristicsImpl::NotASinkCharacteristic {
+  ClassQualifierCharacteristic() { this = "class qualifier" }
+
+  override predicate appliesToEndpoint(Endpoint e) {
+    exists(Call c |
+      e.asExpr() = c.getQualifier() and
+      c.getQualifier() instanceof TypeAccess
+    )
+  }
+}
+
+/**
+ * A call to a method that's known locally will not be considered as a candidate to model.
+ *
+ * The reason is that we would expect data/taint flow into the method implementation to uncover
+ * any sinks that are present there.
+ */
+private class ArgumentToLocalCall extends CharacteristicsImpl::UninterestingToModelCharacteristic {
+  ArgumentToLocalCall() { this = "argument to local call" }
+
+  override predicate appliesToEndpoint(Endpoint e) {
+    ApplicationModeGetCallable::getCallable(e).fromSource()
+  }
+}
+
+/**
+ * A Characteristic that marks endpoints as uninteresting to model, according to the Java ModelExclusions module.
+ */
+private class ExcludedFromModeling extends CharacteristicsImpl::UninterestingToModelCharacteristic {
+  ExcludedFromModeling() { this = "excluded from modeling" }
+
+  override predicate appliesToEndpoint(Endpoint e) {
+    ModelExclusions::isUninterestingForModels(ApplicationModeGetCallable::getCallable(e))
+  }
+}
+
+/**
+ * A negative characteristic that filters out non-public methods. Non-public methods are not interesting to include in
+ * the standard Java modeling, because they cannot be called from outside the package.
+ */
+private class NonPublicMethodCharacteristic extends CharacteristicsImpl::UninterestingToModelCharacteristic
+{
+  NonPublicMethodCharacteristic() { this = "non-public method" }
+
+  override predicate appliesToEndpoint(Endpoint e) {
+    not ApplicationModeGetCallable::getCallable(e).isPublic()
+  }
+}
+
+/**
+ * A negative characteristic that indicates that an endpoint is a non-sink argument to a method whose sinks have already
+ * been modeled.
+ *
+ * WARNING: These endpoints should not be used as negative samples for training, because some sinks may have been missed
+ * when the method was modeled. Specifically, as we start using ATM to merge in new declarations, we can be less sure
+ * that a method with one argument modeled as a MaD sink has also had its remaining arguments manually reviewed. The
+ * ML model might have predicted argument 0 of some method to be a sink but not argument 1, when in fact argument 1 is
+ * also a sink.
+ */
+private class OtherArgumentToModeledMethodCharacteristic extends CharacteristicsImpl::LikelyNotASinkCharacteristic
+{
+  OtherArgumentToModeledMethodCharacteristic() {
+    this = "other argument to a method that has already been modeled"
+  }
+
+  override predicate appliesToEndpoint(Endpoint e) {
+    not ApplicationCandidatesImpl::isSink(e, _) and
+    exists(DataFlow::Node otherSink |
+      ApplicationCandidatesImpl::isSink(otherSink, _) and
+      e.asExpr() = otherSink.asExpr().(Argument).getCall().getAnArgument() and
+      e != otherSink
+    )
+  }
+}
+
+/**
+ * A characteristic that marks functional expression as likely not sinks.
+ *
+ * These expressions may well _contain_ sinks, but rarely are sinks themselves.
+ */
+private class FunctionValueCharacteristic extends CharacteristicsImpl::LikelyNotASinkCharacteristic {
+  FunctionValueCharacteristic() { this = "function value" }
+
+  override predicate appliesToEndpoint(Endpoint e) { e.asExpr() instanceof FunctionalExpr }
+}
+
+/**
+ * A negative characteristic that indicates that an endpoint is not a `to` node for any known taint step. Such a node
+ * cannot be tainted, because taint can't flow into it.
+ *
+ * WARNING: These endpoints should not be used as negative samples for training, because they may include sinks for
+ * which our taint tracking modeling is incomplete.
+ */
+private class CannotBeTaintedCharacteristic extends CharacteristicsImpl::LikelyNotASinkCharacteristic
+{
+  CannotBeTaintedCharacteristic() { this = "cannot be tainted" }
+
+  override predicate appliesToEndpoint(Endpoint e) { not this.isKnownOutNodeForStep(e) }
+
+  /**
+   * Holds if the node `n` is known as the predecessor in a modeled flow step.
+   */
+  private predicate isKnownOutNodeForStep(Endpoint e) {
+    e.asExpr() instanceof Call or // we just assume flow in that case
+    TaintTracking::localTaintStep(_, e) or
+    FlowSummaryImpl::Private::Steps::summaryThroughStepValue(_, e, _) or
+    FlowSummaryImpl::Private::Steps::summaryThroughStepTaint(_, e, _) or
+    FlowSummaryImpl::Private::Steps::summaryGetterStep(_, _, e, _) or
+    FlowSummaryImpl::Private::Steps::summarySetterStep(_, _, e, _)
+  }
+}
+
+/**
+ * Holds if the given endpoint has a self-contradictory combination of characteristics. Detects errors in our endpoint
+ * characteristics. Lists the problematic characteristics and their implications for all such endpoints, together with
+ * an error message indicating why this combination is problematic.
+ *
+ * Copied from
+ *   javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/ContradictoryEndpointCharacteristics.ql
+ */
+predicate erroneousEndpoints(
+  Endpoint endpoint, EndpointCharacteristic characteristic,
+  AutomodelEndpointTypes::EndpointType endpointType, float confidence, string errorMessage,
+  boolean ignoreKnownModelingErrors
+) {
+  // An endpoint's characteristics should not include positive indicators with medium/high confidence for more than one
+  // sink/source type (including the negative type).
+  exists(
+    EndpointCharacteristic characteristic2, AutomodelEndpointTypes::EndpointType endpointClass2,
+    float confidence2
+  |
+    endpointType != endpointClass2 and
+    (
+      endpointType instanceof AutomodelEndpointTypes::SinkType and
+      endpointClass2 instanceof AutomodelEndpointTypes::SinkType
+      or
+      endpointType instanceof AutomodelEndpointTypes::SourceType and
+      endpointClass2 instanceof AutomodelEndpointTypes::SourceType
+    ) and
+    characteristic.appliesToEndpoint(endpoint) and
+    characteristic2.appliesToEndpoint(endpoint) and
+    characteristic.hasImplications(endpointType, true, confidence) and
+    characteristic2.hasImplications(endpointClass2, true, confidence2) and
+    confidence > SharedCharacteristics::mediumConfidence() and
+    confidence2 > SharedCharacteristics::mediumConfidence() and
+    (
+      ignoreKnownModelingErrors = true and
+      not knownOverlappingCharacteristics(characteristic, characteristic2)
+      or
+      ignoreKnownModelingErrors = false
+    )
+  ) and
+  errorMessage = "Endpoint has high-confidence positive indicators for multiple classes"
+  or
+  // An endpoint's characteristics should not include positive indicators with medium/high confidence for some class and
+  // also include negative indicators with medium/high confidence for this same class.
+  exists(EndpointCharacteristic characteristic2, float confidence2 |
+    characteristic.appliesToEndpoint(endpoint) and
+    characteristic2.appliesToEndpoint(endpoint) and
+    characteristic.hasImplications(endpointType, true, confidence) and
+    characteristic2.hasImplications(endpointType, false, confidence2) and
+    confidence > SharedCharacteristics::mediumConfidence() and
+    confidence2 > SharedCharacteristics::mediumConfidence()
+  ) and
+  ignoreKnownModelingErrors = false and
+  errorMessage = "Endpoint has high-confidence positive and negative indicators for the same class"
+}
+
+/**
+ * Holds if `characteristic1` and `characteristic2` are among the pairs of currently known positive characteristics that
+ * have some overlap in their results. This indicates a problem with the underlying Java modeling. Specifically,
+ * `PathCreation` is prone to FPs.
+ */
+private predicate knownOverlappingCharacteristics(
+  EndpointCharacteristic characteristic1, EndpointCharacteristic characteristic2
+) {
+  characteristic1 != characteristic2 and
+  characteristic1 = ["mad taint step", "create path", "read file", "known non-sink"] and
+  characteristic2 = ["mad taint step", "create path", "read file", "known non-sink"]
+}

java/ql/src/Telemetry/AutomodelApplicationModeExtractCandidates.ql

@@ -0,0 +1,49 @@
+/**
+ * Surfaces the endpoints that are not already known to be sinks, and are therefore used as candidates for
+ * classification with an ML model.
+ *
+ * Note: This query does not actually classify the endpoints using the model.
+ *
+ * @name Automodel candidates (application mode)
+ * @description A query to extract automodel candidates in application mode.
+ * @kind problem
+ * @problem.severity recommendation
+ * @id java/ml/extract-automodel-application-candidates
+ * @tags internal extract automodel application-mode candidates
+ */
+
+private import AutomodelApplicationModeCharacteristics
+private import AutomodelJavaUtil
+
+from
+  Endpoint endpoint, string message, ApplicationModeMetadataExtractor meta, DollarAtString package,
+  DollarAtString type, DollarAtString subtypes, DollarAtString name, DollarAtString signature,
+  DollarAtString input
+where
+  not exists(CharacteristicsImpl::UninterestingToModelCharacteristic u |
+    u.appliesToEndpoint(endpoint)
+  ) and
+  // If a node is already a known sink for any of our existing ATM queries and is already modeled as a MaD sink, we
+  // don't include it as a candidate. Otherwise, we might include it as a candidate for query A, but the model will
+  // label it as a sink for one of the sink types of query B, for which it's already a known sink. This would result in
+  // overlap between our detected sinks and the pre-existing modeling. We assume that, if a sink has already been
+  // modeled in a MaD model, then it doesn't belong to any additional sink types, and we don't need to reexamine it.
+  not CharacteristicsImpl::isSink(endpoint, _) and
+  meta.hasMetadata(endpoint, package, type, subtypes, name, signature, input) and
+  // The message is the concatenation of all sink types for which this endpoint is known neither to be a sink nor to be
+  // a non-sink, and we surface only endpoints that have at least one such sink type.
+  message =
+    strictconcat(AutomodelEndpointTypes::SinkType sinkType |
+      not CharacteristicsImpl::isKnownSink(endpoint, sinkType) and
+      CharacteristicsImpl::isSinkCandidate(endpoint, sinkType)
+    |
+      sinkType, ", "
+    )
+select endpoint, message + "\nrelated locations: $@." + "\nmetadata: $@, $@, $@, $@, $@, $@.", //
+  CharacteristicsImpl::getRelatedLocationOrCandidate(endpoint, CallContext()), "CallContext", //
+  package, "package", //
+  type, "type", //
+  subtypes, "subtypes", //
+  name, "name", // method name
+  signature, "signature", //
+  input, "input" //

java/ql/src/Telemetry/AutomodelApplicationModeExtractNegativeExamples.ql

@@ -0,0 +1,73 @@
+/**
+ * Surfaces endpoints that are non-sinks with high confidence, for use as negative examples in the prompt.
+ *
+ * @name Negative examples (application mode)
+ * @kind problem
+ * @problem.severity recommendation
+ * @id java/ml/extract-automodel-application-negative-examples
+ * @tags internal extract automodel application-mode negative examples
+ */
+
+private import java
+private import AutomodelApplicationModeCharacteristics
+private import AutomodelEndpointTypes
+private import AutomodelJavaUtil
+
+/**
+ * Gets a sample of endpoints (of at most `limit` samples) for which the given characteristic applies.
+ *
+ * The main purpose of this helper predicate is to avoid selecting too many samples, as this may
+ * cause the SARIF file to exceed the maximum size limit.
+ */
+bindingset[limit]
+Endpoint getSampleForCharacteristic(EndpointCharacteristic c, int limit) {
+  exists(int n, int num_endpoints | num_endpoints = count(Endpoint e | c.appliesToEndpoint(e)) |
+    result =
+      rank[n](Endpoint e, Location loc |
+        loc = e.getLocation() and c.appliesToEndpoint(e)
+      |
+        e
+        order by
+          loc.getFile().getAbsolutePath(), loc.getStartLine(), loc.getStartColumn(),
+          loc.getEndLine(), loc.getEndColumn()
+      ) and
+    // To avoid selecting samples that are too close together (as the ranking above goes by file
+    // path first), we select `limit` evenly spaced samples from the ranked list of endpoints. By
+    // default this would always include the first sample, so we add a random-chosen prime offset
+    // to the first sample index, and reduce modulo the number of endpoints.
+    // Finally, we add 1 to the result, as ranking results in a 1-indexed relation.
+    n = 1 + (([0 .. limit - 1] * (num_endpoints / limit).floor() + 46337) % num_endpoints)
+  )
+}
+
+from
+  Endpoint endpoint, EndpointCharacteristic characteristic, float confidence, string message,
+  ApplicationModeMetadataExtractor meta, DollarAtString package, DollarAtString type,
+  DollarAtString subtypes, DollarAtString name, DollarAtString signature, DollarAtString input
+where
+  endpoint = getSampleForCharacteristic(characteristic, 100) and
+  confidence >= SharedCharacteristics::highConfidence() and
+  characteristic.hasImplications(any(NegativeSinkType negative), true, confidence) and
+  // Exclude endpoints that have contradictory endpoint characteristics, because we only want examples we're highly
+  // certain about in the prompt.
+  not erroneousEndpoints(endpoint, _, _, _, _, false) and
+  meta.hasMetadata(endpoint, package, type, subtypes, name, signature, input) and
+  // It's valid for a node to satisfy the logic for both `isSink` and `isSanitizer`, but in that case it will be
+  // treated by the actual query as a sanitizer, since the final logic is something like
+  // `isSink(n) and not isSanitizer(n)`. We don't want to include such nodes as negative examples in the prompt, because
+  // they're ambiguous and might confuse the model, so we explicitly exclude all known sinks from the negative examples.
+  not exists(EndpointCharacteristic characteristic2, float confidence2, SinkType positiveType |
+    not positiveType instanceof NegativeSinkType and
+    characteristic2.appliesToEndpoint(endpoint) and
+    confidence2 >= SharedCharacteristics::maximalConfidence() and
+    characteristic2.hasImplications(positiveType, true, confidence2)
+  ) and
+  message = characteristic
+select endpoint, message + "\nrelated locations: $@." + "\nmetadata: $@, $@, $@, $@, $@, $@.", //
+  CharacteristicsImpl::getRelatedLocationOrCandidate(endpoint, CallContext()), "CallContext", //
+  package, "package", //
+  type, "type", //
+  subtypes, "subtypes", //
+  name, "name", //
+  signature, "signature", //
+  input, "input" //

java/ql/src/Telemetry/AutomodelApplicationModeExtractPositiveExamples.ql

@@ -0,0 +1,33 @@
+/**
+ * Surfaces endpoints that are sinks with high confidence, for use as positive examples in the prompt.
+ *
+ * @name Positive examples (application mode)
+ * @kind problem
+ * @problem.severity recommendation
+ * @id java/ml/extract-automodel-application-positive-examples
+ * @tags internal extract automodel application-mode positive examples
+ */
+
+private import AutomodelApplicationModeCharacteristics
+private import AutomodelEndpointTypes
+private import AutomodelJavaUtil
+
+from
+  Endpoint endpoint, SinkType sinkType, ApplicationModeMetadataExtractor meta,
+  DollarAtString package, DollarAtString type, DollarAtString subtypes, DollarAtString name,
+  DollarAtString signature, DollarAtString input
+where
+  // Exclude endpoints that have contradictory endpoint characteristics, because we only want examples we're highly
+  // certain about in the prompt.
+  not erroneousEndpoints(endpoint, _, _, _, _, false) and
+  meta.hasMetadata(endpoint, package, type, subtypes, name, signature, input) and
+  // Extract positive examples of sinks belonging to the existing ATM query configurations.
+  CharacteristicsImpl::isKnownSink(endpoint, sinkType)
+select endpoint, sinkType + "\nrelated locations: $@." + "\nmetadata: $@, $@, $@, $@, $@, $@.", //
+  CharacteristicsImpl::getRelatedLocationOrCandidate(endpoint, CallContext()), "CallContext", //
+  package, "package", //
+  type, "type", //
+  subtypes, "subtypes", //
+  name, "name", //
+  signature, "signature", //
+  input, "input" //

java/ql/src/Telemetry/AutomodelFrameworkModeCharacteristics.qll

@@ -14,23 +14,11 @@ private import semmle.code.java.Expr as Expr
 private import semmle.code.java.security.QueryInjection
 private import semmle.code.java.security.RequestForgery
 private import semmle.code.java.dataflow.internal.ModelExclusions as ModelExclusions
+private import AutomodelJavaUtil as AutomodelJavaUtil
+private import AutomodelSharedGetCallable as AutomodelSharedGetCallable
 import AutomodelSharedCharacteristics as SharedCharacteristics
 import AutomodelEndpointTypes as AutomodelEndpointTypes
 
-/**
- * A meta data extractor. Any Java extraction mode needs to implement exactly
- * one instance of this class.
- */
-abstract class MetadataExtractor extends string {
-  bindingset[this]
-  MetadataExtractor() { any() }
-
-  abstract predicate hasMetadata(
-    DataFlow::ParameterNode e, string package, string type, boolean subtypes, string name,
-    string signature, int input, string parameterName
-  );
-}
-
 newtype JavaRelatedLocationType =
   MethodDoc() or
   ClassDoc()
@@ -60,31 +48,7 @@ module FrameworkCandidatesImpl implements SharedCharacteristics::CandidateSig {
 
   RelatedLocation asLocation(Endpoint e) { result = e.asParameter() }
 
-  predicate isKnownKind(string kind, string humanReadableKind, EndpointType type) {
-    kind = "read-file" and
-    humanReadableKind = "read file" and
-    type instanceof AutomodelEndpointTypes::TaintedPathSinkType
-    or
-    kind = "create-file" and
-    humanReadableKind = "create file" and
-    type instanceof AutomodelEndpointTypes::TaintedPathSinkType
-    or
-    kind = "sql" and
-    humanReadableKind = "mad modeled sql" and
-    type instanceof AutomodelEndpointTypes::SqlSinkType
-    or
-    kind = "open-url" and
-    humanReadableKind = "open url" and
-    type instanceof AutomodelEndpointTypes::RequestForgerySinkType
-    or
-    kind = "jdbc-url" and
-    humanReadableKind = "jdbc url" and
-    type instanceof AutomodelEndpointTypes::RequestForgerySinkType
-    or
-    kind = "command-injection" and
-    humanReadableKind = "command injection" and
-    type instanceof AutomodelEndpointTypes::CommandInjectionSinkType
-  }
+  predicate isKnownKind = AutomodelJavaUtil::isKnownKind/3;
 
   predicate isSink(Endpoint e, string kind) {
     exists(string package, string type, string name, string signature, string ext, string input |
@@ -103,33 +67,41 @@ module FrameworkCandidatesImpl implements SharedCharacteristics::CandidateSig {
   additional predicate sinkSpec(
     Endpoint e, string package, string type, string name, string signature, string ext, string input
   ) {
-    FrameworkCandidatesImpl::getCallable(e).hasQualifiedName(package, type, name) and
-    signature = ExternalFlow::paramsString(getCallable(e)) and
+    FrameworkModeGetCallable::getCallable(e).hasQualifiedName(package, type, name) and
+    signature = ExternalFlow::paramsString(FrameworkModeGetCallable::getCallable(e)) and
     ext = "" and
     exists(int paramIdx | e.isParameterOf(_, paramIdx) |
-      if paramIdx = -1 then input = "Argument[this]" else input = "Argument[" + paramIdx + "]"
+      input = AutomodelJavaUtil::getArgumentForIndex(paramIdx)
     )
   }
 
   /**
-   * Returns the related location for the given endpoint.
+   * Gets the related location for the given endpoint.
    *
    * Related locations can be JavaDoc comments of the class or the method.
    */
   RelatedLocation getRelatedLocation(Endpoint e, RelatedLocationType type) {
     type = MethodDoc() and
-    result = FrameworkCandidatesImpl::getCallable(e).(Documentable).getJavadoc()
+    result = FrameworkModeGetCallable::getCallable(e).(Documentable).getJavadoc()
     or
     type = ClassDoc() and
-    result = FrameworkCandidatesImpl::getCallable(e).getDeclaringType().(Documentable).getJavadoc()
+    result = FrameworkModeGetCallable::getCallable(e).getDeclaringType().(Documentable).getJavadoc()
   }
+}
+
+private class JavaCallable = Callable;
+
+private module FrameworkModeGetCallable implements AutomodelSharedGetCallable::GetCallableSig {
+  class Callable = JavaCallable;
+
+  class Endpoint = FrameworkCandidatesImpl::Endpoint;
 
   /**
    * Returns the callable that contains the given endpoint.
    *
    * Each Java mode should implement this predicate.
    */
-  additional Callable getCallable(Endpoint e) { result = e.getEnclosingCallable() }
+  Callable getCallable(Endpoint e) { result = e.getEnclosingCallable() }
 }
 
 module CharacteristicsImpl = SharedCharacteristics::SharedCharacteristics<FrameworkCandidatesImpl>;
@@ -145,35 +117,19 @@ class Endpoint = FrameworkCandidatesImpl::Endpoint;
 /**
  * A MetadataExtractor that extracts metadata for framework mode.
  */
-class FrameworkModeMetadataExtractor extends MetadataExtractor {
+class FrameworkModeMetadataExtractor extends string {
   FrameworkModeMetadataExtractor() { this = "FrameworkModeMetadataExtractor" }
 
-  /**
-   * By convention, the subtypes property of the MaD declaration should only be
-   * true when there _can_ exist any subtypes with a different implementation.
-   *
-   * It would technically be ok to always use the value 'true', but this would
-   * break convention.
-   */
-  boolean considerSubtypes(Callable callable) {
-    if
-      callable.isStatic() or
-      callable.getDeclaringType().isStatic() or
-      callable.isFinal() or
-      callable.getDeclaringType().isFinal()
-    then result = false
-    else result = true
-  }
-
-  override predicate hasMetadata(
-    Endpoint e, string package, string type, boolean subtypes, string name, string signature,
-    int input, string parameterName
+  predicate hasMetadata(
+    Endpoint e, string package, string type, string subtypes, string name, string signature,
+    string input, string parameterName
   ) {
-    exists(Callable callable |
-      e.asParameter() = callable.getParameter(input) and
+    exists(Callable callable, int paramIdx |
+      e.asParameter() = callable.getParameter(paramIdx) and
+      input = AutomodelJavaUtil::getArgumentForIndex(paramIdx) and
       package = callable.getDeclaringType().getPackage().getName() and
       type = callable.getDeclaringType().getErasure().(RefType).nestedName() and
-      subtypes = this.considerSubtypes(callable) and
+      subtypes = AutomodelJavaUtil::considerSubtypes(callable).toString() and
       name = callable.getName() and
       parameterName = e.asParameter().getName() and
       signature = ExternalFlow::paramsString(callable)
@@ -199,8 +155,8 @@ private class UnexploitableIsCharacteristic extends CharacteristicsImpl::NotASin
 
   override predicate appliesToEndpoint(Endpoint e) {
     not FrameworkCandidatesImpl::isSink(e, _) and
-    FrameworkCandidatesImpl::getCallable(e).getName().matches("is%") and
-    FrameworkCandidatesImpl::getCallable(e).getReturnType() instanceof BooleanType
+    FrameworkModeGetCallable::getCallable(e).getName().matches("is%") and
+    FrameworkModeGetCallable::getCallable(e).getReturnType() instanceof BooleanType
   }
 }
 
@@ -218,7 +174,7 @@ private class UnexploitableExistsCharacteristic extends CharacteristicsImpl::Not
   override predicate appliesToEndpoint(Endpoint e) {
     not FrameworkCandidatesImpl::isSink(e, _) and
     exists(Callable callable |
-      callable = FrameworkCandidatesImpl::getCallable(e) and
+      callable = FrameworkModeGetCallable::getCallable(e) and
       callable.getName().toLowerCase() = ["exists", "notexists"] and
       callable.getReturnType() instanceof BooleanType
     )
@@ -232,7 +188,7 @@ private class ExceptionCharacteristic extends CharacteristicsImpl::NotASinkChara
   ExceptionCharacteristic() { this = "exception" }
 
   override predicate appliesToEndpoint(Endpoint e) {
-    FrameworkCandidatesImpl::getCallable(e).getDeclaringType().getASupertype*() instanceof
+    FrameworkModeGetCallable::getCallable(e).getDeclaringType().getASupertype*() instanceof
       TypeThrowable
   }
 }
@@ -258,7 +214,7 @@ private class NonPublicMethodCharacteristic extends CharacteristicsImpl::Uninter
   NonPublicMethodCharacteristic() { this = "non-public method" }
 
   override predicate appliesToEndpoint(Endpoint e) {
-    not FrameworkCandidatesImpl::getCallable(e).isPublic()
+    not FrameworkModeGetCallable::getCallable(e).isPublic()
   }
 }
 

java/ql/src/Telemetry/AutomodelFrameworkModeExtractCandidates.ql

@@ -4,20 +4,21 @@
  *
  * Note: This query does not actually classify the endpoints using the model.
  *
- * @name Automodel candidates
- * @description A query to extract automodel candidates.
+ * @name Automodel candidates (framework mode)
+ * @description A query to extract automodel candidates in framework mode.
  * @kind problem
- * @severity info
- * @id java/ml/extract-automodel-candidates
- * @tags internal automodel extract candidates
+ * @problem.severity recommendation
+ * @id java/ml/extract-automodel-framework-candidates
+ * @tags internal extract automodel framework-mode candidates
  */
 
 private import AutomodelFrameworkModeCharacteristics
-private import AutomodelSharedUtil
+private import AutomodelJavaUtil
 
 from
-  Endpoint endpoint, string message, MetadataExtractor meta, string package, string type,
-  boolean subtypes, string name, string signature, int input, string parameterName
+  Endpoint endpoint, string message, FrameworkModeMetadataExtractor meta, DollarAtString package,
+  DollarAtString type, DollarAtString subtypes, DollarAtString name, DollarAtString signature,
+  DollarAtString input, DollarAtString parameterName
 where
   not exists(CharacteristicsImpl::UninterestingToModelCharacteristic u |
     u.appliesToEndpoint(endpoint)
@@ -42,10 +43,10 @@ select endpoint,
   message + "\nrelated locations: $@, $@." + "\nmetadata: $@, $@, $@, $@, $@, $@, $@.", //
   CharacteristicsImpl::getRelatedLocationOrCandidate(endpoint, MethodDoc()), "MethodDoc", //
   CharacteristicsImpl::getRelatedLocationOrCandidate(endpoint, ClassDoc()), "ClassDoc", //
-  package.(DollarAtString), "package", //
-  type.(DollarAtString), "type", //
-  subtypes.toString().(DollarAtString), "subtypes", //
-  name.(DollarAtString), "name", //
-  signature.(DollarAtString), "signature", //
-  input.toString().(DollarAtString), "input", //
-  parameterName.(DollarAtString), "parameterName" //
+  package, "package", //
+  type, "type", //
+  subtypes, "subtypes", //
+  name, "name", //
+  signature, "signature", //
+  input, "input", //
+  parameterName, "parameterName" //

java/ql/src/Telemetry/AutomodelFrameworkModeExtractNegativeExamples.ql

@@ -1,21 +1,22 @@
 /**
  * Surfaces endpoints that are non-sinks with high confidence, for use as negative examples in the prompt.
  *
- * @name Negative examples (experimental)
+ * @name Negative examples (framework mode)
  * @kind problem
- * @severity info
- * @id java/ml/non-sink
- * @tags internal automodel extract examples negative
+ * @problem.severity recommendation
+ * @id java/ml/extract-automodel-framework-negative-examples
+ * @tags internal extract automodel framework-mode negative examples
  */
 
 private import AutomodelFrameworkModeCharacteristics
 private import AutomodelEndpointTypes
-private import AutomodelSharedUtil
+private import AutomodelJavaUtil
 
 from
-  Endpoint endpoint, EndpointCharacteristic characteristic, float confidence, string message,
-  MetadataExtractor meta, string package, string type, boolean subtypes, string name,
-  string signature, int input, string parameterName
+  Endpoint endpoint, EndpointCharacteristic characteristic, float confidence,
+  DollarAtString message, FrameworkModeMetadataExtractor meta, DollarAtString package,
+  DollarAtString type, DollarAtString subtypes, DollarAtString name, DollarAtString signature,
+  DollarAtString input, DollarAtString parameterName
 where
   characteristic.appliesToEndpoint(endpoint) and
   confidence >= SharedCharacteristics::highConfidence() and
@@ -39,10 +40,10 @@ select endpoint,
   message + "\nrelated locations: $@, $@." + "\nmetadata: $@, $@, $@, $@, $@, $@, $@.", //
   CharacteristicsImpl::getRelatedLocationOrCandidate(endpoint, MethodDoc()), "MethodDoc", //
   CharacteristicsImpl::getRelatedLocationOrCandidate(endpoint, ClassDoc()), "ClassDoc", //
-  package.(DollarAtString), "package", //
-  type.(DollarAtString), "type", //
-  subtypes.toString().(DollarAtString), "subtypes", //
-  name.(DollarAtString), "name", //
-  signature.(DollarAtString), "signature", //
-  input.toString().(DollarAtString), "input", //
-  parameterName.(DollarAtString), "parameterName" //
+  package, "package", //
+  type, "type", //
+  subtypes, "subtypes", //
+  name, "name", //
+  signature, "signature", //
+  input, "input", //
+  parameterName, "parameterName" //

java/ql/src/Telemetry/AutomodelFrameworkModeExtractPositiveExamples.ql

@@ -1,20 +1,21 @@
 /**
  * Surfaces endpoints that are sinks with high confidence, for use as positive examples in the prompt.
  *
- * @name Positive examples (experimental)
+ * @name Positive examples (framework mode)
  * @kind problem
- * @severity info
- * @id java/ml/known-sink
- * @tags internal automodel extract examples positive
+ * @problem.severity recommendation
+ * @id java/ml/extract-automodel-framework-positive-examples
+ * @tags internal extract automodel framework-mode positive examples
  */
 
 private import AutomodelFrameworkModeCharacteristics
 private import AutomodelEndpointTypes
-private import AutomodelSharedUtil
+private import AutomodelJavaUtil
 
 from
-  Endpoint endpoint, SinkType sinkType, MetadataExtractor meta, string package, string type,
-  boolean subtypes, string name, string signature, int input, string parameterName
+  Endpoint endpoint, SinkType sinkType, FrameworkModeMetadataExtractor meta, DollarAtString package,
+  DollarAtString type, DollarAtString subtypes, DollarAtString name, DollarAtString signature,
+  DollarAtString input, DollarAtString parameterName
 where
   // Exclude endpoints that have contradictory endpoint characteristics, because we only want examples we're highly
   // certain about in the prompt.
@@ -26,10 +27,10 @@ select endpoint,
   sinkType + "\nrelated locations: $@, $@." + "\nmetadata: $@, $@, $@, $@, $@, $@, $@.", //
   CharacteristicsImpl::getRelatedLocationOrCandidate(endpoint, MethodDoc()), "MethodDoc", //
   CharacteristicsImpl::getRelatedLocationOrCandidate(endpoint, ClassDoc()), "ClassDoc", //
-  package.(DollarAtString), "package", //
-  type.(DollarAtString), "type", //
-  subtypes.toString().(DollarAtString), "subtypes", //
-  name.(DollarAtString), "name", //
-  signature.(DollarAtString), "signature", //
-  input.toString().(DollarAtString), "input", //
-  parameterName.(DollarAtString), "parameterName" //
+  package, "package", //
+  type, "type", //
+  subtypes, "subtypes", //
+  name, "name", //
+  signature, "signature", //
+  input, "input", //
+  parameterName, "parameterName" //

java/ql/src/Telemetry/AutomodelJavaUtil.qll

@@ -0,0 +1,81 @@
+private import java
+private import AutomodelEndpointTypes as AutomodelEndpointTypes
+
+/**
+ * A helper class to represent a string value that can be returned by a query using $@ notation.
+ *
+ * It extends `string`, but adds a mock `hasLocationInfo` method that returns the string itself as the file name.
+ *
+ * Use this, when you want to return a string value from a query using $@ notation - the string value
+ * will be included in the sarif file.
+ *
+ *
+ * Background information on `hasLocationInfo`:
+ * https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/#providing-location-information
+ */
+class DollarAtString extends string {
+  bindingset[this]
+  DollarAtString() { any() }
+
+  bindingset[this]
+  predicate hasLocationInfo(string path, int sl, int sc, int el, int ec) {
+    path = this and sl = 1 and sc = 1 and el = 1 and ec = 1
+  }
+}
+
+/**
+ * Holds for all combinations of MaD kinds (`kind`) and their human readable
+ * descriptions.
+ */
+predicate isKnownKind(
+  string kind, string humanReadableKind, AutomodelEndpointTypes::EndpointType type
+) {
+  kind = "read-file" and
+  humanReadableKind = "read file" and
+  type instanceof AutomodelEndpointTypes::TaintedPathSinkType
+  or
+  kind = "create-file" and
+  humanReadableKind = "create file" and
+  type instanceof AutomodelEndpointTypes::TaintedPathSinkType
+  or
+  kind = "sql" and
+  humanReadableKind = "mad modeled sql" and
+  type instanceof AutomodelEndpointTypes::SqlSinkType
+  or
+  kind = "open-url" and
+  humanReadableKind = "open url" and
+  type instanceof AutomodelEndpointTypes::RequestForgerySinkType
+  or
+  kind = "jdbc-url" and
+  humanReadableKind = "jdbc url" and
+  type instanceof AutomodelEndpointTypes::RequestForgerySinkType
+  or
+  kind = "command-injection" and
+  humanReadableKind = "command injection" and
+  type instanceof AutomodelEndpointTypes::CommandInjectionSinkType
+}
+
+/** Gets the models-as-data description for the method argument with the index `index`. */
+bindingset[index]
+string getArgumentForIndex(int index) {
+  index = -1 and result = "Argument[this]"
+  or
+  index >= 0 and result = "Argument[" + index + "]"
+}
+
+/**
+ * By convention, the subtypes property of the MaD declaration should only be
+ * true when there _can_ exist any subtypes with a different implementation.
+ *
+ * It would technically be ok to always use the value 'true', but this would
+ * break convention.
+ */
+boolean considerSubtypes(Callable callable) {
+  if
+    callable.isStatic() or
+    callable.getDeclaringType().isStatic() or
+    callable.isFinal() or
+    callable.getDeclaringType().isFinal()
+  then result = false
+  else result = true
+}

java/ql/src/Telemetry/AutomodelSharedGetCallable.qll

@@ -0,0 +1,21 @@
+/**
+ * An automodel extraction mode instantiates this interface to define how to access
+ * the callable that's associated with an endpoint.
+ */
+signature module GetCallableSig {
+  /**
+   * A callable is the definition of a method, function, etc. - something that can be called.
+   */
+  class Callable;
+
+  /**
+   * An endpoint is a potential candidate for modeling. This will typically be bound to the language's
+   * DataFlow node class, or a subtype thereof.
+   */
+  class Endpoint;
+
+  /**
+   * Gets the callable that's associated with the given endpoint.
+   */
+  Callable getCallable(Endpoint endpoint);
+}

java/ql/src/Telemetry/AutomodelSharedUtil.qll

@@ -1,21 +0,0 @@
-/**
- * A helper class to represent a string value that can be returned by a query using $@ notation.
- *
- * It extends `string`, but adds a mock `hasLocationInfo` method that returns the string itself as the file name.
- *
- * Use this, when you want to return a string value from a query using $@ notation - the string value
- * will be included in the sarif file.
- *
- *
- * Background information on `hasLocationInfo`:
- * https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/#providing-location-information
- */
-class DollarAtString extends string {
-  bindingset[this]
-  DollarAtString() { any() }
-
-  bindingset[this]
-  predicate hasLocationInfo(string path, int sl, int sc, int el, int ec) {
-    path = this and sl = 1 and sc = 1 and el = 1 and ec = 1
-  }
-}