Merge pull request #5136 from aschackmull/java/csv-models

Java: Add support for framework modelling through csv data.

java/ql/src/semmle/code/java/dataflow/ExternalFlow.qll

@@ -0,0 +1,519 @@
+/**
+ * INTERNAL use only. This is an experimental API subject to change without notice.
+ *
+ * Provides classes and predicates for dealing with flow models specified in CSV format.
+ *
+ * The CSV specification has the following columns:
+ * - Sources:
+ *   `namespace; type; subtypes; name; signature; ext; output; kind`
+ * - Sinks:
+ *   `namespace; type; subtypes; name; signature; ext; input; kind`
+ * - Summaries:
+ *   `namespace; type; subtypes; name; signature; ext; input; output; kind`
+ *
+ * The interpretation of a row is similar to API-graphs with a left-to-right
+ * reading.
+ * 1. The `namespace` column selects a package.
+ * 2. The `type` column selects a type within that package.
+ * 3. The `subtypes` is a boolean that indicates whether to jump to an
+ *    arbitrary subtype of that type.
+ * 4. The `name` column optionally selects a specific named member of the type.
+ * 5. The `signature` column optionally restricts the named member. If
+ *    `signature` is blank then no such filtering is done. The format of the
+ *    signature is a comma-separated list of types enclosed in parentheses. The
+ *    types can be short names or fully qualified names (mixing these two options
+ *    is not allowed within a single signature).
+ * 6. The `ext` column specifies additional API-graph-like edges. Currently
+ *    there are only two valid values: "" and "Annotated". The empty string has no
+ *    effect. "Annotated" applies if `name` and `signature` were left blank and
+ *    acts by selecting an element that is annotated by the annotation type
+ *    selected by the first 4 columns. This can be another member such as a field
+ *    or method, or a parameter.
+ * 7. The `input` column specifies how data enters the element selected by the
+ *    first 6 columns, and the `output` column specifies how data leaves the
+ *    element selected by the first 6 columns. An `input` can be either "",
+ *    "Argument", "Argument[n]", "ReturnValue":
+ *    - "": Selects a write to the selected element in case this is a field.
+ *    - "Argument": Selects any argument in a call to the selected element.
+ *    - "Argument[n]": Similar to "Argument" but restricted to a specific numbered
+ *      argument (zero-indexed, and `-1` specifies the qualifier).
+ *    - "ReturnValue": Selects a value being returned by the selected element.
+ *      This requires that the selected element is a method with a body.
+ *
+ *    An `output` can be either "", "Argument", "Argument[n]", "Parameter",
+ *    "Parameter[n]", or "ReturnValue":
+ *    - "": Selects a read of a selected field, or a selected parameter.
+ *    - "Argument": Selects the post-update value of an argument in a call to the
+ *      selected element. That is, the value of the argument after the call returns.
+ *    - "Argument[n]": Similar to "Argument" but restricted to a specific numbered
+ *      argument (zero-indexed, and `-1` specifies the qualifier).
+ *    - "Parameter": Selects the value of a parameter of the selected element.
+ *      "Parameter" is also allowed in case the selected element is already a
+ *      parameter itself.
+ *    - "Parameter[n]": Similar to "Parameter" but restricted to a specific
+ *      numbered parameter (zero-indexed, and `-1` specifies the value of `this`).
+ *    - "ReturnValue": Selects the return value of a call to the selected element.
+ * 8. The `kind` column is a tag that can be referenced from QL to determine to
+ *    which classes the interpreted elements should be added. For example, for
+ *    sources "remote" indicates a default remote flow source, and for summaries
+ *    "taint" indicates a default additional taint step and "value" indicates a
+ *    globally applicable value-preserving step.
+ */
+
+import java
+private import semmle.code.java.dataflow.DataFlow::DataFlow
+private import internal.DataFlowPrivate
+
+private predicate sourceModelCsv(string row) {
+  row =
+    [
+      // ServletRequestGetParameterMethod
+      "javax.servlet;ServletRequest;false;getParameter;(String);;ReturnValue;remote",
+      "javax.servlet;ServletRequest;false;getParameterValues;(String);;ReturnValue;remote",
+      "javax.servlet.http;HttpServletRequest;false;getParameter;(String);;ReturnValue;remote",
+      "javax.servlet.http;HttpServletRequest;false;getParameterValues;(String);;ReturnValue;remote",
+      // ServletRequestGetParameterMapMethod
+      "javax.servlet;ServletRequest;false;getParameterMap;();;ReturnValue;remote",
+      "javax.servlet.http;HttpServletRequest;false;getParameterMap;();;ReturnValue;remote",
+      // ServletRequestGetParameterNamesMethod
+      "javax.servlet;ServletRequest;false;getParameterNames;();;ReturnValue;remote",
+      "javax.servlet.http;HttpServletRequest;false;getParameterNames;();;ReturnValue;remote",
+      // HttpServletRequestGetQueryStringMethod
+      "javax.servlet.http;HttpServletRequest;false;getQueryString;();;ReturnValue;remote",
+      //
+      // URLConnectionGetInputStreamMethod
+      "java.net;URLConnection;false;getInputStream;();;ReturnValue;remote",
+      // SocketGetInputStreamMethod
+      "java.net;Socket;false;getInputStream;();;ReturnValue;remote",
+      // BeanValidationSource
+      "javax.validation;ConstraintValidator;true;isValid;;;Parameter[0];remote"
+    ]
+}
+
+private predicate sinkModelCsv(string row) { none() }
+
+private predicate summaryModelCsv(string row) { none() }
+
+/**
+ * A unit class for adding additional source model rows.
+ *
+ * Extend this class to add additional source definitions.
+ */
+class SourceModelCsv extends Unit {
+  /** Holds if `row` specifies a source definition. */
+  abstract predicate row(string row);
+}
+
+/**
+ * A unit class for adding additional sink model rows.
+ *
+ * Extend this class to add additional sink definitions.
+ */
+class SinkModelCsv extends Unit {
+  /** Holds if `row` specifies a sink definition. */
+  abstract predicate row(string row);
+}
+
+/**
+ * A unit class for adding additional summary model rows.
+ *
+ * Extend this class to add additional flow summary definitions.
+ */
+class SummaryModelCsv extends Unit {
+  /** Holds if `row` specifies a summary definition. */
+  abstract predicate row(string row);
+}
+
+private predicate sourceModel(string row) {
+  sourceModelCsv(row) or
+  any(SourceModelCsv s).row(row)
+}
+
+private predicate sinkModel(string row) {
+  sinkModelCsv(row) or
+  any(SinkModelCsv s).row(row)
+}
+
+private predicate summaryModel(string row) {
+  summaryModelCsv(row) or
+  any(SummaryModelCsv s).row(row)
+}
+
+private predicate sourceModel(
+  string namespace, string type, boolean subtypes, string name, string signature, string ext,
+  string output, string kind
+) {
+  exists(string row |
+    sourceModel(row) and
+    row.splitAt(";", 0) = namespace and
+    row.splitAt(";", 1) = type and
+    row.splitAt(";", 2) = subtypes.toString() and
+    subtypes = [true, false] and
+    row.splitAt(";", 3) = name and
+    row.splitAt(";", 4) = signature and
+    row.splitAt(";", 5) = ext and
+    row.splitAt(";", 6) = output and
+    row.splitAt(";", 7) = kind
+  )
+}
+
+private predicate sinkModel(
+  string namespace, string type, boolean subtypes, string name, string signature, string ext,
+  string input, string kind
+) {
+  exists(string row |
+    sinkModel(row) and
+    row.splitAt(";", 0) = namespace and
+    row.splitAt(";", 1) = type and
+    row.splitAt(";", 2) = subtypes.toString() and
+    subtypes = [true, false] and
+    row.splitAt(";", 3) = name and
+    row.splitAt(";", 4) = signature and
+    row.splitAt(";", 5) = ext and
+    row.splitAt(";", 6) = input and
+    row.splitAt(";", 7) = kind
+  )
+}
+
+private predicate summaryModel(
+  string namespace, string type, boolean subtypes, string name, string signature, string ext,
+  string input, string output, string kind
+) {
+  exists(string row |
+    summaryModel(row) and
+    row.splitAt(";", 0) = namespace and
+    row.splitAt(";", 1) = type and
+    row.splitAt(";", 2) = subtypes.toString() and
+    subtypes = [true, false] and
+    row.splitAt(";", 3) = name and
+    row.splitAt(";", 4) = signature and
+    row.splitAt(";", 5) = ext and
+    row.splitAt(";", 6) = input and
+    row.splitAt(";", 7) = output and
+    row.splitAt(";", 8) = kind
+  )
+}
+
+/** Provides a query predicate to check the CSV data for validation errors. */
+module CsvValidation {
+  /** Holds if some row in a CSV-based flow model appears to contain typos. */
+  query predicate invalidModelRow(string msg) {
+    exists(string pred, string namespace, string type, string name, string signature, string ext |
+      sourceModel(namespace, type, _, name, signature, ext, _, _) and pred = "source"
+      or
+      sinkModel(namespace, type, _, name, signature, ext, _, _) and pred = "sink"
+      or
+      summaryModel(namespace, type, _, name, signature, ext, _, _, _) and pred = "summary"
+    |
+      not namespace.regexpMatch("[a-zA-Z0-9_\\.]+") and
+      msg = "Dubious namespace \"" + namespace + "\" in " + pred + " model."
+      or
+      not type.regexpMatch("[a-zA-Z0-9_\\$]+") and
+      msg = "Dubious type \"" + type + "\" in " + pred + " model."
+      or
+      not name.regexpMatch("[a-zA-Z0-9_]*") and
+      msg = "Dubious name \"" + name + "\" in " + pred + " model."
+      or
+      not signature.regexpMatch("|\\([a-zA-Z0-9_\\.\\$<>,]*\\)") and
+      msg = "Dubious signature \"" + signature + "\" in " + pred + " model."
+      or
+      not ext.regexpMatch("|Annotated") and
+      msg = "Unrecognized extra API graph element \"" + ext + "\" in " + pred + " model."
+    )
+    or
+    exists(string pred, string input, string part |
+      sinkModel(_, _, _, _, _, _, input, _) and pred = "sink"
+      or
+      summaryModel(_, _, _, _, _, _, input, _, _) and pred = "summary"
+    |
+      specSplit(input, part, _) and
+      not part.regexpMatch("|Argument|ReturnValue") and
+      not parseArg(part, _) and
+      msg = "Unrecognized input specification \"" + part + "\" in " + pred + " model."
+    )
+    or
+    exists(string pred, string output, string part |
+      sourceModel(_, _, _, _, _, _, output, _) and pred = "source"
+      or
+      summaryModel(_, _, _, _, _, _, _, output, _) and pred = "summary"
+    |
+      specSplit(output, part, _) and
+      not part.regexpMatch("|Argument|Parameter|ReturnValue") and
+      not parseArg(part, _) and
+      not parseParam(part, _) and
+      msg = "Unrecognized output specification \"" + part + "\" in " + pred + " model."
+    )
+    or
+    exists(string pred, string row, int expect |
+      sourceModel(row) and expect = 8 and pred = "source"
+      or
+      sinkModel(row) and expect = 8 and pred = "sink"
+      or
+      summaryModel(row) and expect = 9 and pred = "summary"
+    |
+      exists(int cols |
+        cols = 1 + max(int n | exists(row.splitAt(";", n))) and
+        cols != expect and
+        msg =
+          "Wrong number of columns in " + pred + " model row, expected " + expect + ", got " + cols +
+            "."
+      )
+      or
+      exists(string b |
+        b = row.splitAt(";", 2) and
+        not b = ["true", "false"] and
+        msg = "Invalid boolean \"" + b + "\" in " + pred + " model."
+      )
+    )
+  }
+}
+
+private predicate elementSpec(
+  string namespace, string type, boolean subtypes, string name, string signature, string ext
+) {
+  sourceModel(namespace, type, subtypes, name, signature, ext, _, _) or
+  sinkModel(namespace, type, subtypes, name, signature, ext, _, _) or
+  summaryModel(namespace, type, subtypes, name, signature, ext, _, _, _)
+}
+
+bindingset[namespace, type, subtypes]
+private RefType interpretType(string namespace, string type, boolean subtypes) {
+  exists(RefType t |
+    t.hasQualifiedName(namespace, type) and
+    if subtypes = true then result.getASourceSupertype*() = t else result = t
+  )
+}
+
+private string paramsStringPart(Callable c, int i) {
+  i = -1 and result = "("
+  or
+  exists(int n, string p | c.getParameterType(n).toString() = p |
+    i = 2 * n and result = p
+    or
+    i = 2 * n - 1 and result = "," and n != 0
+  )
+  or
+  i = 2 * c.getNumberOfParameters() and result = ")"
+}
+
+private string paramsString(Callable c) {
+  result = concat(int i | | paramsStringPart(c, i) order by i)
+}
+
+private Element interpretElement0(
+  string namespace, string type, boolean subtypes, string name, string signature
+) {
+  elementSpec(namespace, type, subtypes, name, signature, _) and
+  exists(RefType t | t = interpretType(namespace, type, subtypes) |
+    exists(Member m |
+      result = m and
+      m.getDeclaringType() = t and
+      m.hasName(name)
+    |
+      signature = "" or
+      m.(Callable).getSignature() = any(string nameprefix) + signature or
+      paramsString(m) = signature
+    )
+    or
+    result = t and
+    name = "" and
+    signature = ""
+  )
+}
+
+private Element interpretElement(
+  string namespace, string type, boolean subtypes, string name, string signature, string ext
+) {
+  elementSpec(namespace, type, subtypes, name, signature, ext) and
+  exists(Element e | e = interpretElement0(namespace, type, subtypes, name, signature) |
+    ext = "" and result = e
+    or
+    ext = "Annotated" and result.(Annotatable).getAnAnnotation().getType() = e
+  )
+}
+
+private predicate sourceElement(Element e, string output, string kind) {
+  exists(
+    string namespace, string type, boolean subtypes, string name, string signature, string ext
+  |
+    sourceModel(namespace, type, subtypes, name, signature, ext, output, kind) and
+    e = interpretElement(namespace, type, subtypes, name, signature, ext)
+  )
+}
+
+private predicate sinkElement(Element e, string input, string kind) {
+  exists(
+    string namespace, string type, boolean subtypes, string name, string signature, string ext
+  |
+    sinkModel(namespace, type, subtypes, name, signature, ext, input, kind) and
+    e = interpretElement(namespace, type, subtypes, name, signature, ext)
+  )
+}
+
+private predicate summaryElement(Element e, string input, string output, string kind) {
+  exists(
+    string namespace, string type, boolean subtypes, string name, string signature, string ext
+  |
+    summaryModel(namespace, type, subtypes, name, signature, ext, input, output, kind) and
+    e = interpretElement(namespace, type, subtypes, name, signature, ext)
+  )
+}
+
+private string inOutSpec() {
+  sourceModel(_, _, _, _, _, _, result, _) or
+  sinkModel(_, _, _, _, _, _, result, _) or
+  summaryModel(_, _, _, _, _, _, result, _, _) or
+  summaryModel(_, _, _, _, _, _, _, result, _)
+}
+
+private predicate specSplit(string s, string c, int n) {
+  inOutSpec() = s and s.splitAt(" of ", n) = c
+}
+
+private predicate len(string s, int len) { len = 1 + max(int n | specSplit(s, _, n)) }
+
+private string getLast(string s) {
+  exists(int len |
+    len(s, len) and
+    specSplit(s, result, len - 1)
+  )
+}
+
+private predicate parseParam(string c, int n) {
+  specSplit(_, c, _) and c.regexpCapture("Parameter\\[([-0-9]+)\\]", 1).toInt() = n
+}
+
+private predicate parseArg(string c, int n) {
+  specSplit(_, c, _) and c.regexpCapture("Argument\\[([-0-9]+)\\]", 1).toInt() = n
+}
+
+private predicate inputNeedsReference(string c) {
+  c = "Argument" or
+  parseArg(c, _)
+}
+
+private predicate outputNeedsReference(string c) {
+  c = "Argument" or
+  parseArg(c, _) or
+  c = "ReturnValue"
+}
+
+private predicate sourceElementRef(Top ref, string output, string kind) {
+  exists(Element e |
+    sourceElement(e, output, kind) and
+    if outputNeedsReference(getLast(output)) then ref.(Call).getCallee() = e else ref = e
+  )
+}
+
+private predicate sinkElementRef(Top ref, string input, string kind) {
+  exists(Element e |
+    sinkElement(e, input, kind) and
+    if inputNeedsReference(getLast(input)) then ref.(Call).getCallee() = e else ref = e
+  )
+}
+
+private predicate summaryElementRef(Top ref, string input, string output, string kind) {
+  exists(Element e |
+    summaryElement(e, input, output, kind) and
+    if inputNeedsReference(getLast(input)) then ref.(Call).getCallee() = e else ref = e
+  )
+}
+
+private newtype TAstOrNode =
+  TAst(Top t) or
+  TNode(Node n)
+
+private predicate interpretOutput(string output, int idx, Top ref, TAstOrNode node) {
+  (
+    sourceElementRef(ref, output, _) or
+    summaryElementRef(ref, _, output, _)
+  ) and
+  len(output, idx) and
+  node = TAst(ref)
+  or
+  exists(Top mid, string c, Node n |
+    interpretOutput(output, idx + 1, ref, TAst(mid)) and
+    specSplit(output, c, idx) and
+    node = TNode(n)
+  |
+    exists(int pos | n.(PostUpdateNode).getPreUpdateNode().(ArgumentNode).argumentOf(mid, pos) |
+      c = "Argument" or parseArg(c, pos)
+    )
+    or
+    exists(int pos | n.(ParameterNode).isParameterOf(mid, pos) |
+      c = "Parameter" or parseParam(c, pos)
+    )
+    or
+    (c = "Parameter" or c = "") and
+    n.asParameter() = mid
+    or
+    c = "ReturnValue" and
+    n.asExpr().(Call) = mid
+    or
+    c = "" and
+    n.asExpr().(FieldRead).getField() = mid
+  )
+}
+
+private predicate interpretInput(string input, int idx, Top ref, TAstOrNode node) {
+  (
+    sinkElementRef(ref, input, _) or
+    summaryElementRef(ref, input, _, _)
+  ) and
+  len(input, idx) and
+  node = TAst(ref)
+  or
+  exists(Top mid, string c, Node n |
+    interpretInput(input, idx + 1, ref, TAst(mid)) and
+    specSplit(input, c, idx) and
+    node = TNode(n)
+  |
+    exists(int pos | n.(ArgumentNode).argumentOf(mid, pos) | c = "Argument" or parseArg(c, pos))
+    or
+    exists(ReturnStmt ret |
+      c = "ReturnValue" and
+      n.asExpr() = ret.getResult() and
+      mid = ret.getEnclosingCallable()
+    )
+    or
+    exists(FieldWrite fw |
+      c = "" and
+      fw.getField() = mid and
+      n.asExpr() = fw.getRHS()
+    )
+  )
+}
+
+/**
+ * Holds if `node` is specified as a source with the given kind in a CSV flow
+ * model.
+ */
+predicate sourceNode(Node node, string kind) {
+  exists(Top ref, string output |
+    sourceElementRef(ref, output, kind) and
+    interpretOutput(output, 0, ref, TNode(node))
+  )
+}
+
+/**
+ * Holds if `node` is specified as a sink with the given kind in a CSV flow
+ * model.
+ */
+predicate sinkNode(Node node, string kind) {
+  exists(Top ref, string input |
+    sinkElementRef(ref, input, kind) and
+    interpretInput(input, 0, ref, TNode(node))
+  )
+}
+
+/**
+ * Holds if `node1` to `node2` is specified as a flow step with the given kind
+ * in a CSV flow model.
+ */
+predicate summaryStep(Node node1, Node node2, string kind) {
+  exists(Top ref, string input, string output |
+    summaryElementRef(ref, input, output, kind) and
+    interpretInput(input, 0, ref, TNode(node1)) and
+    interpretOutput(output, 0, ref, TNode(node2))
+  )
+}

java/ql/src/semmle/code/java/dataflow/FlowSources.qll

@@ -24,6 +24,7 @@ import semmle.code.java.frameworks.spring.SpringWebClient
 import semmle.code.java.frameworks.Guice
 import semmle.code.java.frameworks.struts.StrutsActions
 import semmle.code.java.frameworks.Thrift
+private import semmle.code.java.dataflow.ExternalFlow
 
 /** A data flow source of remote user input. */
 abstract class RemoteFlowSource extends DataFlow::Node {
@@ -31,6 +32,12 @@ abstract class RemoteFlowSource extends DataFlow::Node {
   abstract string getSourceType();
 }
 
+private class ExternalRemoteFlowSource extends RemoteFlowSource {
+  ExternalRemoteFlowSource() { sourceNode(this, "remote") }
+
+  override string getSourceType() { result = "external" }
+}
+
 private class RemoteTaintedMethodAccessSource extends RemoteFlowSource {
   RemoteTaintedMethodAccessSource() {
     this.asExpr().(MethodAccess).getMethod() instanceof RemoteTaintedMethod

java/ql/src/semmle/code/java/dataflow/internal/DataFlowUtil.qll

@@ -7,6 +7,7 @@ private import DataFlowPrivate
 private import semmle.code.java.dataflow.SSA
 private import semmle.code.java.dataflow.TypeFlow
 private import semmle.code.java.controlflow.Guards
+private import semmle.code.java.dataflow.ExternalFlow
 import semmle.code.java.dataflow.InstanceAccess
 
 cached
@@ -405,6 +406,8 @@ predicate simpleLocalFlowStep(Node node1, Node node2) {
   or
   node2.asExpr().(AssignExpr).getSource() = node1.asExpr()
   or
+  summaryStep(node1, node2, "value")
+  or
   exists(MethodAccess ma, Method m |
     ma = node2.asExpr() and
     m = ma.getMethod() and

java/ql/src/semmle/code/java/dataflow/internal/TaintTrackingUtil.qll

@@ -10,6 +10,7 @@ private import semmle.code.java.dataflow.internal.ContainerFlow
 private import semmle.code.java.frameworks.spring.SpringController
 private import semmle.code.java.frameworks.spring.SpringHttp
 private import semmle.code.java.frameworks.Networking
+private import semmle.code.java.dataflow.ExternalFlow
 import semmle.code.java.dataflow.FlowSteps
 
 /**
@@ -45,6 +46,8 @@ predicate localAdditionalTaintStep(DataFlow::Node src, DataFlow::Node sink) {
   localAdditionalTaintUpdateStep(src.asExpr(),
     sink.(DataFlow::PostUpdateNode).getPreUpdateNode().asExpr())
   or
+  summaryStep(src, sink, "taint")
+  or
   exists(Argument arg |
     src.asExpr() = arg and
     arg.isVararg() and

java/ql/test/library-tests/dataflow/external-models/A.java

@@ -0,0 +1,62 @@
+package my.qltest;
+
+public class A {
+  void foo() {
+    Object x;
+    x = src1();
+    x = src1("");
+
+    Sub sub = new Sub();
+    x = sub.src2();
+    x = sub.src3();
+
+    srcArg(x);
+
+    Handler h = srcparam1 -> { };
+
+    Handler h2 = new Handler() {
+      @Override public void handle(Object srcparam2) { }
+    };
+
+    x = taggedSrcMethod();
+    x = taggedSrcField;
+
+    x = srcTwoArg("", "");
+  }
+
+  @Tag
+  void tagged1(Object taggedMethodParam) {
+  }
+
+  void tagged2(@Tag Object taggedSrcParam) {
+  }
+
+  Object src1() { return null; }
+
+  Object src1(String s) { return null; }
+
+  Object src2() { return null; }
+
+  Object src3() { return null; }
+
+  static class Sub extends A {
+    // inherit src2
+    @Override Object src3() { return null; }
+  }
+
+  void srcArg(Object src) { }
+
+  interface Handler {
+    void handle(Object src);
+  }
+
+  @interface Tag { }
+
+  @Tag
+  Object taggedSrcMethod() { return null; }
+
+  @Tag
+  Object taggedSrcField;
+
+  Object srcTwoArg(String s1, String s2) { return null; }
+}

java/ql/test/library-tests/dataflow/external-models/B.java

@@ -0,0 +1,35 @@
+package my.qltest;
+
+public class B {
+  void foo() {
+    Object arg1 = new Object();
+    sink1(arg1);
+
+    Object argToTagged = new Object();
+    taggedSinkMethod(argToTagged);
+
+    Object fieldWrite = new Object();
+    taggedField = fieldWrite;
+  }
+
+  Object sinkMethod() {
+    Object res = new Object();
+    return res;
+  }
+
+  @Tag
+  Object taggedSinkMethod() {
+    Object resTag = new Object();
+    return resTag;
+  }
+
+  void sink1(Object x) { }
+
+  @interface Tag { }
+
+  @Tag
+  void taggedSinkMethod(Object x) { }
+
+  @Tag
+  Object taggedField;
+}

java/ql/test/library-tests/dataflow/external-models/C.java

@@ -0,0 +1,36 @@
+package my.qltest;
+
+public class C {
+  void foo() {
+    Object arg1 = new Object();
+    stepArgRes(arg1);
+
+    Object argIn1 = new Object();
+    Object argOut1 = new Object();
+    stepArgArg(argIn1, argOut1);
+    Object argIn2 = new Object();
+    Object argOut2 = new Object();
+    stepArgArg(argIn2, argOut2);
+
+    Object arg2 = new Object();
+    stepArgQual(arg2);
+    Object arg3 = new Object();
+    this.stepArgQual(arg3);
+
+    this.stepQualRes();
+    stepQualRes();
+
+    Object argOut = new Object();
+    stepQualArg(argOut);
+  }
+
+  Object stepArgRes(Object x) { return null; }
+
+  void stepArgArg(Object in, Object out) { }
+
+  void stepArgQual(Object x) { }
+
+  Object stepQualRes() { return null; }
+
+  void stepQualArg(Object out) { }
+}

java/ql/test/library-tests/dataflow/external-models/sinks.expected

@@ -0,0 +1,8 @@
+invalidModelRow
+#select
+| B.java:6:11:6:14 | arg1 | qltest |
+| B.java:9:5:9:33 | this <.method> | qltest-arg |
+| B.java:9:22:9:32 | argToTagged | qltest-arg |
+| B.java:12:19:12:28 | fieldWrite | qltest-nospec |
+| B.java:17:12:17:14 | res | qltest |
+| B.java:23:12:23:17 | resTag | qltest-retval |

java/ql/test/library-tests/dataflow/external-models/sinks.ql

@@ -0,0 +1,22 @@
+import java
+import semmle.code.java.dataflow.DataFlow
+import semmle.code.java.dataflow.ExternalFlow
+import CsvValidation
+
+class SinkModelTest extends SinkModelCsv {
+  override predicate row(string row) {
+    row =
+      [
+        //"package;type;overrides;name;signature;ext;spec;kind",
+        "my.qltest;B;false;sink1;(Object);;Argument[0];qltest",
+        "my.qltest;B;false;sinkMethod;();;ReturnValue;qltest",
+        "my.qltest;B$Tag;false;;;Annotated;ReturnValue;qltest-retval",
+        "my.qltest;B$Tag;false;;;Annotated;Argument;qltest-arg",
+        "my.qltest;B$Tag;false;;;Annotated;;qltest-nospec"
+      ]
+  }
+}
+
+from DataFlow::Node node, string kind
+where sinkNode(node, kind)
+select node, kind

java/ql/test/library-tests/dataflow/external-models/srcs.expected

@@ -0,0 +1,24 @@
+invalidModelRow
+#select
+| A.java:6:9:6:14 | src1(...) | qltest |
+| A.java:6:9:6:14 | src1(...) | qltest-all-overloads |
+| A.java:7:9:7:16 | src1(...) | qltest |
+| A.java:7:9:7:16 | src1(...) | qltest-all-overloads |
+| A.java:7:9:7:16 | src1(...) | qltest-alt |
+| A.java:10:9:10:18 | src2(...) | qltest |
+| A.java:10:9:10:18 | src2(...) | qltest-w-subtypes |
+| A.java:11:9:11:18 | src3(...) | qltest-w-subtypes |
+| A.java:13:5:13:13 | this <.method> [post update] | qltest-argany |
+| A.java:13:12:13:12 | x [post update] | qltest-argany |
+| A.java:13:12:13:12 | x [post update] | qltest-argnum |
+| A.java:15:17:15:25 | srcparam1 | qltest-param-override |
+| A.java:18:36:18:51 | srcparam2 | qltest-param-override |
+| A.java:21:9:21:25 | taggedSrcMethod(...) | qltest-retval |
+| A.java:22:9:22:22 | taggedSrcField | qltest-nospec |
+| A.java:24:9:24:25 | srcTwoArg(...) | qltest-longsig |
+| A.java:24:9:24:25 | srcTwoArg(...) | qltest-shortsig |
+| A.java:28:8:28:14 | parameter this | qltest-param |
+| A.java:28:16:28:39 | taggedMethodParam | qltest-param |
+| A.java:31:16:31:41 | taggedSrcParam | qltest-nospec |
+| A.java:31:16:31:41 | taggedSrcParam | qltest-param |
+| A.java:56:10:56:24 | parameter this | qltest-param |

java/ql/test/library-tests/dataflow/external-models/srcs.ql

@@ -0,0 +1,33 @@
+import java
+import semmle.code.java.dataflow.DataFlow
+import semmle.code.java.dataflow.ExternalFlow
+import CsvValidation
+
+class SourceModelTest extends SourceModelCsv {
+  override predicate row(string row) {
+    row =
+      [
+        //"package;type;overrides;name;signature;ext;spec;kind",
+        "my.qltest;A;false;src1;();;ReturnValue;qltest",
+        "my.qltest;A;false;src1;(String);;ReturnValue;qltest",
+        "my.qltest;A;false;src1;(java.lang.String);;ReturnValue;qltest-alt",
+        "my.qltest;A;false;src1;;;ReturnValue;qltest-all-overloads",
+        "my.qltest;A;false;src2;();;ReturnValue;qltest",
+        "my.qltest;A;false;src3;();;ReturnValue;qltest",
+        "my.qltest;A;true;src2;();;ReturnValue;qltest-w-subtypes",
+        "my.qltest;A;true;src3;();;ReturnValue;qltest-w-subtypes",
+        "my.qltest;A;false;srcArg;(Object);;Argument[0];qltest-argnum",
+        "my.qltest;A;false;srcArg;(Object);;Argument;qltest-argany",
+        "my.qltest;A$Handler;true;handle;(Object);;Parameter[0];qltest-param-override",
+        "my.qltest;A$Tag;false;;;Annotated;ReturnValue;qltest-retval",
+        "my.qltest;A$Tag;false;;;Annotated;Parameter;qltest-param",
+        "my.qltest;A$Tag;false;;;Annotated;;qltest-nospec",
+        "my.qltest;A;false;srcTwoArg;(String,String);;ReturnValue;qltest-shortsig",
+        "my.qltest;A;false;srcTwoArg;(java.lang.String,java.lang.String);;ReturnValue;qltest-longsig"
+      ]
+  }
+}
+
+from DataFlow::Node node, string kind
+where sourceNode(node, kind)
+select node, kind

java/ql/test/library-tests/dataflow/external-models/steps.expected

@@ -0,0 +1,10 @@
+invalidModelRow
+#select
+| C.java:6:16:6:19 | arg1 | C.java:6:5:6:20 | stepArgRes(...) | qltest |
+| C.java:10:16:10:21 | argIn1 | C.java:10:24:10:30 | argOut1 [post update] | qltest |
+| C.java:13:16:13:21 | argIn2 | C.java:13:24:13:30 | argOut2 [post update] | qltest |
+| C.java:16:17:16:20 | arg2 | C.java:16:5:16:21 | this <.method> [post update] | qltest |
+| C.java:18:22:18:25 | arg3 | C.java:18:5:18:8 | this [post update] | qltest |
+| C.java:20:5:20:8 | this | C.java:20:5:20:22 | stepQualRes(...) | qltest |
+| C.java:21:5:21:17 | this <.method> | C.java:21:5:21:17 | stepQualRes(...) | qltest |
+| C.java:24:5:24:23 | this <.method> | C.java:24:17:24:22 | argOut [post update] | qltest |

java/ql/test/library-tests/dataflow/external-models/steps.ql

@@ -0,0 +1,22 @@
+import java
+import semmle.code.java.dataflow.DataFlow
+import semmle.code.java.dataflow.ExternalFlow
+import CsvValidation
+
+class SummaryModelTest extends SummaryModelCsv {
+  override predicate row(string row) {
+    row =
+      [
+        //"package;type;overrides;name;signature;ext;inputspec;outputspec;kind",
+        "my.qltest;C;false;stepArgRes;(Object);;Argument[0];ReturnValue;qltest",
+        "my.qltest;C;false;stepArgArg;(Object,Object);;Argument[0];Argument[1];qltest",
+        "my.qltest;C;false;stepArgQual;(Object);;Argument[0];Argument[-1];qltest",
+        "my.qltest;C;false;stepQualRes;();;Argument[-1];ReturnValue;qltest",
+        "my.qltest;C;false;stepQualArg;(Object);;Argument[-1];Argument[0];qltest"
+      ]
+  }
+}
+
+from DataFlow::Node node1, DataFlow::Node node2, string kind
+where summaryStep(node1, node2, kind)
+select node1, node2, kind