Merge pull request #5979 from hvitved/java/shared-external-summaries

Java: Move some CSV flow summary code into shared library

java/ql/src/semmle/code/java/dataflow/ExternalFlow.qll

@@ -67,6 +67,7 @@
 import java
 private import semmle.code.java.dataflow.DataFlow::DataFlow
 private import internal.DataFlowPrivate
+private import internal.FlowSummaryImpl::Private::External
 private import FlowSummary
 
 /**
@@ -498,10 +499,15 @@ module CsvValidation {
       or
       summaryModel(_, _, _, _, _, _, input, _, _) and pred = "summary"
     |
-      specSplit(input, part, _) and
-      not part.regexpMatch("|ReturnValue|ArrayElement|Element|MapKey|MapValue") and
-      not (part = "Argument" and pred = "sink") and
-      not parseArg(part, _) and
+      (
+        invalidSpecComponent(input, part) and
+        not part = "" and
+        not (part = "Argument" and pred = "sink") and
+        not parseArg(part, _)
+        or
+        specSplit(input, part, _) and
+        parseParam(part, _)
+      ) and
       msg = "Unrecognized input specification \"" + part + "\" in " + pred + " model."
     )
     or
@@ -510,11 +516,9 @@ module CsvValidation {
       or
       summaryModel(_, _, _, _, _, _, _, output, _) and pred = "summary"
     |
-      specSplit(output, part, _) and
-      not part.regexpMatch("|ReturnValue|ArrayElement|Element|MapKey|MapValue") and
+      invalidSpecComponent(output, part) and
+      not part = "" and
       not (part = ["Argument", "Parameter"] and pred = "source") and
-      not parseArg(part, _) and
-      not parseParam(part, _) and
       msg = "Unrecognized output specification \"" + part + "\" in " + pred + " model."
     )
     or
@@ -624,7 +628,11 @@ private predicate sinkElement(Element e, string input, string kind) {
   )
 }
 
-private predicate summaryElement(Element e, string input, string output, string kind) {
+/**
+ * Holds if an external flow summary exists for `e` with input specification
+ * `input`, output specification `output`, and kind `kind`.
+ */
+predicate summaryElement(Element e, string input, string output, string kind) {
   exists(
     string namespace, string type, boolean subtypes, string name, string signature, string ext
   |
@@ -633,52 +641,14 @@ private predicate summaryElement(Element e, string input, string output, string
   )
 }
 
-private string inOutSpec() {
+/** Gets a specification used in a source model, sink model, or summary model. */
+string inOutSpec() {
   sourceModel(_, _, _, _, _, _, result, _) or
   sinkModel(_, _, _, _, _, _, result, _) or
   summaryModel(_, _, _, _, _, _, result, _, _) or
   summaryModel(_, _, _, _, _, _, _, result, _)
 }
 
-private predicate specSplit(string s, string c, int n) {
-  inOutSpec() = s and s.splitAt(" of ", n) = c
-}
-
-private predicate len(string s, int len) { len = 1 + max(int n | specSplit(s, _, n)) }
-
-private string getLast(string s) {
-  exists(int len |
-    len(s, len) and
-    specSplit(s, result, len - 1)
-  )
-}
-
-private predicate parseParam(string c, int n) {
-  specSplit(_, c, _) and
-  (
-    c.regexpCapture("Parameter\\[([-0-9]+)\\]", 1).toInt() = n
-    or
-    exists(int n1, int n2 |
-      c.regexpCapture("Parameter\\[([-0-9]+)\\.\\.([0-9]+)\\]", 1).toInt() = n1 and
-      c.regexpCapture("Parameter\\[([-0-9]+)\\.\\.([0-9]+)\\]", 2).toInt() = n2 and
-      n = [n1 .. n2]
-    )
-  )
-}
-
-private predicate parseArg(string c, int n) {
-  specSplit(_, c, _) and
-  (
-    c.regexpCapture("Argument\\[([-0-9]+)\\]", 1).toInt() = n
-    or
-    exists(int n1, int n2 |
-      c.regexpCapture("Argument\\[([-0-9]+)\\.\\.([0-9]+)\\]", 1).toInt() = n1 and
-      c.regexpCapture("Argument\\[([-0-9]+)\\.\\.([0-9]+)\\]", 2).toInt() = n2 and
-      n = [n1 .. n2]
-    )
-  )
-}
-
 private predicate inputNeedsReference(string c) {
   c = "Argument" or
   parseArg(c, _)
@@ -693,7 +663,7 @@ private predicate outputNeedsReference(string c) {
 private predicate sourceElementRef(Top ref, string output, string kind) {
   exists(Element e |
     sourceElement(e, output, kind) and
-    if outputNeedsReference(getLast(output))
+    if outputNeedsReference(specLast(output))
     then ref.(Call).getCallee().getSourceDeclaration() = e
     else ref = e
   )
@@ -702,7 +672,7 @@ private predicate sourceElementRef(Top ref, string output, string kind) {
 private predicate sinkElementRef(Top ref, string input, string kind) {
   exists(Element e |
     sinkElement(e, input, kind) and
-    if inputNeedsReference(getLast(input))
+    if inputNeedsReference(specLast(input))
     then ref.(Call).getCallee().getSourceDeclaration() = e
     else ref = e
   )
@@ -711,81 +681,12 @@ private predicate sinkElementRef(Top ref, string input, string kind) {
 private predicate summaryElementRef(Top ref, string input, string output, string kind) {
   exists(Element e |
     summaryElement(e, input, output, kind) and
-    if inputNeedsReference(getLast(input))
+    if inputNeedsReference(specLast(input))
     then ref.(Call).getCallee().getSourceDeclaration() = e
     else ref = e
   )
 }
 
-private SummaryComponent interpretComponent(string c) {
-  specSplit(_, c, _) and
-  (
-    exists(int pos | parseArg(c, pos) and result = SummaryComponent::argument(pos))
-    or
-    exists(int pos | parseParam(c, pos) and result = SummaryComponent::parameter(pos))
-    or
-    c = "ReturnValue" and result = SummaryComponent::return()
-    or
-    c = "ArrayElement" and result = SummaryComponent::content(any(ArrayContent c0))
-    or
-    c = "Element" and result = SummaryComponent::content(any(CollectionContent c0))
-    or
-    c = "MapKey" and result = SummaryComponent::content(any(MapKeyContent c0))
-    or
-    c = "MapValue" and result = SummaryComponent::content(any(MapValueContent c0))
-  )
-}
-
-private predicate interpretSpec(string spec, int idx, SummaryComponentStack stack) {
-  exists(string c |
-    summaryElement(_, spec, _, _) or
-    summaryElement(_, _, spec, _)
-  |
-    len(spec, idx + 1) and
-    specSplit(spec, c, idx) and
-    stack = SummaryComponentStack::singleton(interpretComponent(c))
-  )
-  or
-  exists(SummaryComponent head, SummaryComponentStack tail |
-    interpretSpec(spec, idx, head, tail) and
-    stack = SummaryComponentStack::push(head, tail)
-  )
-}
-
-private predicate interpretSpec(
-  string output, int idx, SummaryComponent head, SummaryComponentStack tail
-) {
-  exists(string c |
-    interpretSpec(output, idx + 1, tail) and
-    specSplit(output, c, idx) and
-    head = interpretComponent(c)
-  )
-}
-
-private class MkStack extends RequiredSummaryComponentStack {
-  MkStack() { interpretSpec(_, _, _, this) }
-
-  override predicate required(SummaryComponent c) { interpretSpec(_, _, c, this) }
-}
-
-private class SummarizedCallableExternal extends SummarizedCallable {
-  SummarizedCallableExternal() { summaryElement(this, _, _, _) }
-
-  override predicate propagatesFlow(
-    SummaryComponentStack input, SummaryComponentStack output, boolean preservesValue
-  ) {
-    exists(string inSpec, string outSpec, string kind |
-      summaryElement(this, inSpec, outSpec, kind) and
-      interpretSpec(inSpec, 0, input) and
-      interpretSpec(outSpec, 0, output)
-    |
-      kind = "value" and preservesValue = true
-      or
-      kind = "taint" and preservesValue = false
-    )
-  }
-}
-
 private newtype TAstOrNode =
   TAst(Top t) or
   TNode(Node n)
@@ -795,7 +696,7 @@ private predicate interpretOutput(string output, int idx, Top ref, TAstOrNode no
     sourceElementRef(ref, output, _) or
     summaryElementRef(ref, _, output, _)
   ) and
-  len(output, idx) and
+  specLength(output, idx) and
   node = TAst(ref)
   or
   exists(Top mid, string c, Node n |
@@ -827,7 +728,7 @@ private predicate interpretInput(string input, int idx, Top ref, TAstOrNode node
     sinkElementRef(ref, input, _) or
     summaryElementRef(ref, input, _, _)
   ) and
-  len(input, idx) and
+  specLength(input, idx) and
   node = TAst(ref)
   or
   exists(Top mid, string c, Node n |

java/ql/src/semmle/code/java/dataflow/internal/FlowSummaryImpl.qll

@@ -580,88 +580,112 @@ module Private {
    * summaries into a `SummarizedCallable`s.
    */
   module External {
-    /**
-     * Provides a means of translating an externally (e.g., CSV) defined flow
-     * summary into a `SummarizedCallable`.
-     */
-    abstract class ExternalSummaryCompilation extends string {
-      bindingset[this]
-      ExternalSummaryCompilation() { any() }
+    /** Holds if the `n`th component of specification `s` is `c`. */
+    predicate specSplit(string s, string c, int n) { relevantSpec(s) and s.splitAt(" of ", n) = c }
 
-      /** Holds if this flow summary is for callable `c`. */
-      abstract predicate callable(DataFlowCallable c, boolean preservesValue);
+    /** Holds if specification `s` has length `len`. */
+    predicate specLength(string s, int len) { len = 1 + max(int n | specSplit(s, _, n)) }
 
-      /** Holds if the `i`th input component is `c`. */
-      abstract predicate input(int i, SummaryComponent c);
-
-      /** Holds if the `i`th output component is `c`. */
-      abstract predicate output(int i, SummaryComponent c);
-
-      /**
-       * Holds if the input components starting from index `i` translate into `suffix`.
-       */
-      final predicate translateInput(int i, SummaryComponentStack suffix) {
-        exists(SummaryComponent comp | this.input(i, comp) |
-          i = max(int j | this.input(j, _)) and
-          suffix = TSingletonSummaryComponentStack(comp)
-          or
-          exists(TSummaryComponent head, SummaryComponentStack tail |
-            this.translateInputCons(i, head, tail) and
-            suffix = TConsSummaryComponentStack(head, tail)
-          )
-        )
-      }
-
-      final predicate translateInputCons(int i, SummaryComponent head, SummaryComponentStack tail) {
-        this.input(i, head) and
-        this.translateInput(i + 1, tail)
-      }
-
-      /**
-       * Holds if the output components starting from index `i` translate into `suffix`.
-       */
-      predicate translateOutput(int i, SummaryComponentStack suffix) {
-        exists(SummaryComponent comp | this.output(i, comp) |
-          i = max(int j | this.output(j, _)) and
-          suffix = TSingletonSummaryComponentStack(comp)
-          or
-          exists(TSummaryComponent head, SummaryComponentStack tail |
-            this.translateOutputCons(i, head, tail) and
-            suffix = TConsSummaryComponentStack(head, tail)
-          )
-        )
-      }
-
-      predicate translateOutputCons(int i, SummaryComponent head, SummaryComponentStack tail) {
-        this.output(i, head) and
-        this.translateOutput(i + 1, tail)
-      }
+    /** Gets the last component of specification `s`. */
+    string specLast(string s) {
+      exists(int len |
+        specLength(s, len) and
+        specSplit(s, result, len - 1)
+      )
     }
 
-    private class ExternalRequiredSummaryComponentStack extends RequiredSummaryComponentStack {
-      private SummaryComponent head;
-
-      ExternalRequiredSummaryComponentStack() {
-        any(ExternalSummaryCompilation s).translateInputCons(_, head, this) or
-        any(ExternalSummaryCompilation s).translateOutputCons(_, head, this)
-      }
-
-      override predicate required(SummaryComponent c) { c = head }
+    /** Holds if specification component `c` parses as parameter `n`. */
+    predicate parseParam(string c, int n) {
+      specSplit(_, c, _) and
+      (
+        c.regexpCapture("Parameter\\[([-0-9]+)\\]", 1).toInt() = n
+        or
+        exists(int n1, int n2 |
+          c.regexpCapture("Parameter\\[([-0-9]+)\\.\\.([0-9]+)\\]", 1).toInt() = n1 and
+          c.regexpCapture("Parameter\\[([-0-9]+)\\.\\.([0-9]+)\\]", 2).toInt() = n2 and
+          n = [n1 .. n2]
+        )
+      )
     }
 
-    class ExternalSummarizedCallableAdaptor extends SummarizedCallable {
-      ExternalSummarizedCallableAdaptor() { any(ExternalSummaryCompilation s).callable(this, _) }
+    /** Holds if specification component `c` parses as argument `n`. */
+    predicate parseArg(string c, int n) {
+      specSplit(_, c, _) and
+      (
+        c.regexpCapture("Argument\\[([-0-9]+)\\]", 1).toInt() = n
+        or
+        exists(int n1, int n2 |
+          c.regexpCapture("Argument\\[([-0-9]+)\\.\\.([0-9]+)\\]", 1).toInt() = n1 and
+          c.regexpCapture("Argument\\[([-0-9]+)\\.\\.([0-9]+)\\]", 2).toInt() = n2 and
+          n = [n1 .. n2]
+        )
+      )
+    }
+
+    private SummaryComponent interpretComponent(string c) {
+      specSplit(_, c, _) and
+      (
+        exists(int pos | parseArg(c, pos) and result = SummaryComponent::argument(pos))
+        or
+        exists(int pos | parseParam(c, pos) and result = SummaryComponent::parameter(pos))
+        or
+        result = interpretComponentSpecific(c)
+      )
+    }
+
+    private predicate interpretSpec(string spec, int idx, SummaryComponentStack stack) {
+      exists(string c |
+        relevantSpec(spec) and
+        specLength(spec, idx + 1) and
+        specSplit(spec, c, idx) and
+        stack = SummaryComponentStack::singleton(interpretComponent(c))
+      )
+      or
+      exists(SummaryComponent head, SummaryComponentStack tail |
+        interpretSpec(spec, idx, head, tail) and
+        stack = SummaryComponentStack::push(head, tail)
+      )
+    }
+
+    private predicate interpretSpec(
+      string output, int idx, SummaryComponent head, SummaryComponentStack tail
+    ) {
+      exists(string c |
+        interpretSpec(output, idx + 1, tail) and
+        specSplit(output, c, idx) and
+        head = interpretComponent(c)
+      )
+    }
+
+    private class MkStack extends RequiredSummaryComponentStack {
+      MkStack() { interpretSpec(_, _, _, this) }
+
+      override predicate required(SummaryComponent c) { interpretSpec(_, _, c, this) }
+    }
+
+    private class SummarizedCallableExternal extends SummarizedCallable {
+      SummarizedCallableExternal() { externalSummary(this, _, _, _) }
 
       override predicate propagatesFlow(
         SummaryComponentStack input, SummaryComponentStack output, boolean preservesValue
       ) {
-        exists(ExternalSummaryCompilation s |
-          s.callable(this, preservesValue) and
-          s.translateInput(0, input) and
-          s.translateOutput(0, output)
+        exists(string inSpec, string outSpec, string kind |
+          externalSummary(this, inSpec, outSpec, kind) and
+          interpretSpec(inSpec, 0, input) and
+          interpretSpec(outSpec, 0, output)
+        |
+          kind = "value" and preservesValue = true
+          or
+          kind = "taint" and preservesValue = false
         )
       }
     }
+
+    /** Holds if component `c` of specification `spec` cannot be parsed. */
+    predicate invalidSpecComponent(string spec, string c) {
+      specSplit(spec, c, _) and
+      not exists(interpretComponent(c))
+    }
   }
 
   /** Provides a query predicate for outputting a set of relevant flow summaries. */

java/ql/src/semmle/code/java/dataflow/internal/FlowSummaryImplSpecific.qll

@@ -7,6 +7,7 @@ private import DataFlowPrivate
 private import DataFlowUtil
 private import FlowSummaryImpl::Private
 private import FlowSummaryImpl::Public
+private import semmle.code.java.dataflow.ExternalFlow
 
 private module FlowSummaries {
   private import semmle.code.java.dataflow.FlowSummary as F
@@ -49,3 +50,28 @@ DataFlowType getCallbackParameterType(DataFlowType t, int i) { none() }
  * callback of type `t`.
  */
 DataFlowType getCallbackReturnType(DataFlowType t, ReturnKind rk) { none() }
+
+/** Holds if `spec` is a relevant external specification. */
+predicate relevantSpec(string spec) { spec = inOutSpec() }
+
+/**
+ * Holds if an external flow summary exists for `c` with input specification
+ * `input`, output specification `output`, and kind `kind`.
+ */
+predicate externalSummary(DataFlowCallable c, string input, string output, string kind) {
+  summaryElement(c, input, output, kind)
+}
+
+/** Gets the summary component for specification component `c`, if any. */
+bindingset[c]
+SummaryComponent interpretComponentSpecific(string c) {
+  c = "ReturnValue" and result = SummaryComponent::return(_)
+  or
+  c = "ArrayElement" and result = SummaryComponent::content(any(ArrayContent c0))
+  or
+  c = "Element" and result = SummaryComponent::content(any(CollectionContent c0))
+  or
+  c = "MapKey" and result = SummaryComponent::content(any(MapKeyContent c0))
+  or
+  c = "MapValue" and result = SummaryComponent::content(any(MapValueContent c0))
+}