Merge pull request #12302 from MathiasVP/recursive-join-order-metric

QL: Extend the join-order badness query to recursive predicates
2025-12-17 01:03:14 +01:00 · 2023-02-23 22:35:35 +00:00
parent bb692a7e4a f0fe6fba88
commit a86a8ced54
4 changed files with 450 additions and 44 deletions
--- a/ql/ql/src/codeql_ql/StructuredLogs.qll
+++ b/ql/ql/src/codeql_ql/StructuredLogs.qll
@@ -77,7 +77,8 @@ class Array extends JSON::Array {
 *
 * This is needed because the evaluator log is padded with -1s in some cases.
 */
-private float getRanked(Array a, int i) {
+pragma[nomagic]
+private float getRankedFloat(Array a, int i) {
  result = rank[i + 1](int j, float f | f = a.getFloat(j) and f >= 0 | f order by j)
 }

@@ -137,10 +138,10 @@ module EvaluatorLog {

    string getRAReference() { result = this.getString("raReference") }

-    float getCount(int i) { result = getRanked(this.getArray("counts"), i) }
+    float getCount(int i) { result = getRankedFloat(this.getArray("counts"), i) }

    float getDuplicationPercentage(int i) {
-      result = getRanked(this.getArray("duplicationPercentages"), i)
+      result = getRankedFloat(this.getArray("duplicationPercentages"), i)
    }

    float getResultSize() { result = this.getFloat("resultSize") }
@@ -250,6 +251,11 @@ module KindPredicatesLog {

    int getMillis() { result = this.getNumber("millis") }

+    PipeLineRuns getPipelineRuns() { result = this.getArray("pipelineRuns") }
+
+    pragma[nomagic]
+    float getDeltaSize(int i) { result = getRankedFloat(this.getArray("deltaSizes"), i) }
+
    predicate hasCompletionTime(
      int year, string month, int day, int hours, int minute, int second, int millisecond
    ) {
@@ -271,9 +277,7 @@ module KindPredicatesLog {

    float getResultSize() { result = this.getFloat("resultSize") }

-    Array getRA(string ordering) { result = this.getObject("ra").getArray(ordering) }
-
-    string getAnOrdering() { exists(this.getRA(result)) }
+    string getAnOrdering() { exists(this.getRA().getPipeLine(result)) }

    override string toString() {
      if exists(this.getPredicateName())
@@ -307,6 +311,8 @@ module KindPredicatesLog {
    RA() { evt.getObject("ra") = this }

    PipeLine getPipeLine(string name) { result = this.getArray(name) }
+
+    PipeLine getPipeLine() { result = this.getPipeLine("pipeline") }
  }

  class SentinelEmpty extends SummaryEvent {
@@ -341,12 +347,12 @@ module KindPredicatesLog {

    Array getCounts() { result = this.getArray("counts") }

-    float getCount(int i) { result = getRanked(this.getArray("counts"), i) }
+    float getCount(int i) { result = getRankedFloat(this.getArray("counts"), i) }

    Array getDuplicationPercentage() { result = this.getArray("duplicationPercentages") }

    float getDuplicationPercentage(int i) {
-      result = getRanked(this.getArray("duplicationPercentages"), i)
+      result = getRankedFloat(this.getArray("duplicationPercentages"), i)
    }
  }

@@ -391,10 +397,14 @@ module KindPredicatesLog {
  }

  /** Gets the `index`'th event that's evaluated by `recursive`. */
-  private InLayer layerEventRank(ComputeRecursive recursive, int index) {
+  private SummaryEvent layerEventRank(ComputeRecursive recursive, int index) {
    result =
-      rank[index + 1](InLayer cand, int startline, string filepath |
-        cand.getComputeRecursiveEvent() = recursive and
+      rank[index + 1](SummaryEvent cand, int startline, string filepath |
+        (
+          cand = recursive
+          or
+          cand.(InLayer).getComputeRecursiveEvent() = recursive
+        ) and
        cand.hasLocationInfo(filepath, startline, _, _, _)
      |
        cand order by filepath, startline
@@ -405,15 +415,13 @@ module KindPredicatesLog {
   * Gets the first predicate that's evaluated in an iteration
   * of the SCC computation rooted at `recursive`.
   */
-  private InLayer firstPredicate(ComputeRecursive recursive) {
-    result = layerEventRank(recursive, 0)
-  }
+  SummaryEvent firstPredicate(ComputeRecursive recursive) { result = layerEventRank(recursive, 0) }

  /**
   * Gets the last predicate that's evaluated in an iteration
   * of the SCC computation rooted at `recursive`.
   */
-  private InLayer lastPredicate(ComputeRecursive recursive) {
+  SummaryEvent lastPredicate(ComputeRecursive recursive) {
    exists(int n |
      result = layerEventRank(recursive, n) and
      not exists(layerEventRank(recursive, n + 1))
@@ -424,7 +432,7 @@ module KindPredicatesLog {
   * Holds if the predicate represented by `next` was evaluated after the
   * predicate represented by `prev` in the SCC computation rooted at `recursive`.
   */
-  predicate successor(ComputeRecursive recursive, InLayer prev, InLayer next) {
+  predicate successor(ComputeRecursive recursive, SummaryEvent prev, InLayer next) {
    exists(int index |
      layerEventRank(recursive, index) = prev and
      layerEventRank(recursive, index + 1) = next
@@ -503,6 +511,8 @@ module KindPredicatesLog {

  class ComputeRecursive extends SummaryEvent {
    ComputeRecursive() { evaluationStrategy = "COMPUTE_RECURSIVE" }
+
+    Depencencies getDependencies() { result = this.getObject("dependencies") }
  }

  class InLayer extends SummaryEvent {
@@ -515,12 +525,8 @@ module KindPredicatesLog {
    Array getPredicateIterationMillis() { result = this.getArray("predicateIterationMillis") }

    float getPredicateIterationMillis(int i) {
-      result = getRanked(this.getArray("predicateIterationMillis"), i)
+      result = getRankedFloat(this.getArray("predicateIterationMillis"), i)
    }
-
-    PipeLineRuns getPipelineRuns() { result = this.getArray("pipelineRuns") }
-
-    float getDeltaSize(int i) { result = getRanked(this.getArray("deltaSizes"), i) }
  }

  class ComputedExtensional extends SummaryEvent {
--- a/ql/ql/src/experimental/RA.qll
+++ b/ql/ql/src/experimental/RA.qll
@@ -26,8 +26,71 @@ module RAParser<RApredicate Predicate> {
    result = str.trim().regexpCapture("return r([0-9]+)", 1).toInt()
  }

+  bindingset[str]
+  private predicate parseScan(string str, int arity, int lhs, string rhs) {
+    exists(string r, string trimmed |
+      r = "\\{(\\d+)\\}\\s+r(\\d+)\\s+=\\s+SCAN\\s+([0-9a-zA-Z:#_]+)\\s.*" and
+      trimmed = str.trim()
+    |
+      arity = trimmed.regexpCapture(r, 1).toInt() and
+      lhs = trimmed.regexpCapture(r, 2).toInt() and
+      rhs = trimmed.regexpCapture(r, 3)
+    )
+  }
+
+  bindingset[str]
+  private predicate parseJoin(string str, int arity, int lhs, string left, string right) {
+    exists(string r, string trimmed |
+      r =
+        "\\{(\\d+)\\}\\s+r(\\d+)\\s+=\\s+JOIN\\s+([0-9a-zA-Z:#_]+)\\s+WITH\\s+([0-9a-zA-Z:#_]+)\\s.*" and
+      trimmed = str.trim()
+    |
+      arity = trimmed.regexpCapture(r, 1).toInt() and
+      lhs = trimmed.regexpCapture(r, 2).toInt() and
+      left = trimmed.regexpCapture(r, 3) and
+      right = trimmed.regexpCapture(r, 4)
+    )
+  }
+
+  bindingset[str]
+  private predicate parseSelect(string str, int arity, int lhs, string rhs) {
+    exists(string r, string trimmed |
+      r = "\\{(\\d+)\\}\\s+r(\\d+)\\s+=\\s+SELECT\\s+([0-9a-zA-Z:#_]+).*" and
+      trimmed = str.trim()
+    |
+      arity = trimmed.regexpCapture(r, 1).toInt() and
+      lhs = trimmed.regexpCapture(r, 2).toInt() and
+      rhs = trimmed.regexpCapture(r, 3)
+    )
+  }
+
+  bindingset[str]
+  private predicate parseAntiJoin(string str, int arity, int lhs, string left, string right) {
+    exists(string r, string trimmed |
+      r = "\\{(\\d+)\\}\\s+r(\\d+)\\s+=\\s+([0-9a-zA-Z:#_]+)\\s+AND\\s+NOT\\s+([0-9a-zA-Z:#_]+).*" and
+      trimmed = str.trim()
+    |
+      arity = trimmed.regexpCapture(r, 1).toInt() and
+      lhs = trimmed.regexpCapture(r, 2).toInt() and
+      left = trimmed.regexpCapture(r, 3) and
+      right = trimmed.regexpCapture(r, 4)
+    )
+  }
+
  private newtype TRA =
    TReturn(Predicate p, int line, int v) { v = parseReturn(p.getLineOfRA(line)) } or
+    TScan(Predicate p, int line, int arity, int lhs, string rhs) {
+      parseScan(p.getLineOfRA(line), arity, lhs, rhs)
+    } or
+    TJoin(Predicate p, int line, int arity, int lhs, string left, string right) {
+      parseJoin(p.getLineOfRA(line), arity, lhs, left, right)
+    } or
+    TSelect(Predicate p, int line, int arity, int lhs, string rhs) {
+      parseSelect(p.getLineOfRA(line), arity, lhs, rhs)
+    } or
+    TAntiJoin(Predicate p, int line, int arity, int lhs, string left, string right) {
+      parseAntiJoin(p.getLineOfRA(line), arity, lhs, left, right)
+    } or
    TUnknown(Predicate p, int line, int lhs, int arity, string rhs) {
      rhs = parseRaExpr(p, line, arity, lhs)
    }
@@ -90,12 +153,12 @@ module RAParser<RApredicate Predicate> {
  }

  class RAReturnExpr extends RAExpr, TReturn {
-    RAReturnExpr() { this = TReturn(p, line, res) }
-
    Predicate p;
    int line;
    int res;

+    RAReturnExpr() { this = TReturn(p, line, res) }
+
    override Predicate getPredicate() { result = p }

    override int getLine() { result = line }
@@ -108,4 +171,115 @@ module RAParser<RApredicate Predicate> {

    override string getARhsPredicate() { none() }
  }
+
+  class RAScanExpr extends RAExpr, TScan {
+    Predicate p;
+    int line;
+    int arity;
+    int lhs;
+    string rhs;
+
+    RAScanExpr() { this = TScan(p, line, arity, lhs, rhs) }
+
+    override Predicate getPredicate() { result = p }
+
+    override int getLine() { result = line }
+
+    override int getLhs() { result = lhs }
+
+    override int getArity() { result = arity }
+
+    override int getARhsVariable() { isVariable(rhs, result) }
+
+    override string getARhsPredicate() {
+      result = rhs and
+      not isVariable(result, _)
+    }
+  }
+
+  bindingset[s]
+  private predicate isVariable(string s, int n) { n = s.regexpCapture("r(\\d+)", 1).toInt() }
+
+  class RAJoinExpr extends RAExpr, TJoin {
+    Predicate p;
+    int line;
+    int arity;
+    int lhs;
+    string left;
+    string right;
+
+    RAJoinExpr() { this = TJoin(p, line, arity, lhs, left, right) }
+
+    override Predicate getPredicate() { result = p }
+
+    override int getLine() { result = line }
+
+    override int getLhs() { result = lhs }
+
+    override int getArity() { result = arity }
+
+    // Note: We could return reasonable values here sometimes.
+    override int getARhsVariable() { isVariable([left, right], result) }
+
+    // Note: We could return reasonable values here sometimes.
+    override string getARhsPredicate() {
+      result = [left, right] and
+      not isVariable(result, _)
+    }
+  }
+
+  class RaSelectExpr extends RAExpr, TSelect {
+    Predicate p;
+    int line;
+    int arity;
+    int lhs;
+    string rhs;
+
+    RaSelectExpr() { this = TSelect(p, line, arity, lhs, rhs) }
+
+    override Predicate getPredicate() { result = p }
+
+    override int getLine() { result = line }
+
+    override int getLhs() { result = lhs }
+
+    override int getArity() { result = arity }
+
+    // Note: We could return reasonable values here sometimes.
+    override int getARhsVariable() { isVariable(rhs, result) }
+
+    // Note: We could return reasonable values here sometimes.
+    override string getARhsPredicate() {
+      result = rhs and
+      not isVariable(result, _)
+    }
+  }
+
+  class RaAntiJoinExpr extends RAExpr, TAntiJoin {
+    Predicate p;
+    int line;
+    int arity;
+    int lhs;
+    string left;
+    string right;
+
+    RaAntiJoinExpr() { this = TAntiJoin(p, line, arity, lhs, left, right) }
+
+    override Predicate getPredicate() { result = p }
+
+    override int getLine() { result = line }
+
+    override int getLhs() { result = lhs }
+
+    override int getArity() { result = arity }
+
+    // Note: We could return reasonable values here sometimes.
+    override int getARhsVariable() { isVariable([left, right], result) }
+
+    // Note: We could return reasonable values here sometimes.
+    override string getARhsPredicate() {
+      result = [left, right] and
+      not isVariable(result, _)
+    }
+  }
 }
--- a/ql/ql/src/queries/performance/LargeJoinOrder.ql
+++ b/ql/ql/src/queries/performance/LargeJoinOrder.ql
@@ -6,6 +6,7 @@
 import ql
 import codeql_ql.StructuredLogs
 import KindPredicatesLog
+import experimental.RA

 /**
 * Gets the badness of a non-recursive predicate evaluation.
@@ -13,10 +14,10 @@ import KindPredicatesLog
 * The badness is the maximum number of tuples in the pipeline divided by the
 * maximum of two numbers: the size of the result and the size of the largest dependency.
 */
-float getBadness(ComputeSimple simple) {
+float getNonRecursiveBadness(ComputeSimple simple) {
  exists(float maxTupleCount, float resultSize, float largestDependency, float denom |
    resultSize = simple.getResultSize() and
-    extractInformation(simple, maxTupleCount, _, _, _) and
+    maxTupleCount = max(simple.getPipelineRun().getCount(_)) and
    largestDependency = max(simple.getDependencies().getADependency().getResultSize()) and
    denom = resultSize.maximum(largestDependency) and
    denom > 0 and // avoid division by zero (which would create a NaN result).
@@ -24,29 +25,245 @@ float getBadness(ComputeSimple simple) {
  )
 }

-pragma[nomagic]
-predicate hasPipelineRun(ComputeSimple simple, PipeLineRun run) { run = simple.getPipelineRun() }
-
-predicate extractInformation(
-  ComputeSimple simple, float maxTupleCount, Array tuples, Array duplicationPercentages, Array ra
+predicate hasTupleCount(
+  ComputeRecursive recursive, string ordering, SummaryEvent inLayer, int iteration, int i,
+  float tupleCount
 ) {
-  exists(PipeLineRun run |
-    hasPipelineRun(simple, run) and
-    maxTupleCount = max(run.getCount(_)) and
-    tuples = run.getCounts() and
-    duplicationPercentages = run.getDuplicationPercentage() and
-    ra = simple.getPipeLine()
+  inLayer = firstPredicate(recursive) and
+  exists(PipeLineRun run | run = inLayer.getPipelineRuns().getRun(iteration) |
+    ordering = run.getRAReference() and
+    tupleCount = run.getCount(i)
+  )
+  or
+  exists(SummaryEvent inLayer0, float tupleCount0, PipeLineRun run |
+    run = inLayer.getPipelineRuns().getRun(pragma[only_bind_into](iteration)) and
+    successor(recursive, inLayer0, inLayer) and
+    hasTupleCount(recursive, ordering, inLayer0, pragma[only_bind_into](iteration), i, tupleCount0) and
+    tupleCount = run.getCount(i) + tupleCount0
  )
 }

+predicate hasTupleCount(ComputeRecursive recursive, string ordering, int i, float tupleCount) {
+  tupleCount =
+    strictsum(SummaryEvent inLayer, int iteration, int tc |
+      inLayer = getInLayerOrRecursive(recursive) and
+      hasTupleCount(recursive, ordering, inLayer, iteration, i, tc)
+    |
+      tc
+    )
+}
+
+pragma[nomagic]
+predicate hasDuplication(
+  ComputeRecursive recursive, string ordering, SummaryEvent inLayer, int iteration, int i,
+  float duplication
+) {
+  inLayer = firstPredicate(recursive) and
+  exists(PipeLineRun run | run = inLayer.getPipelineRuns().getRun(iteration) |
+    ordering = run.getRAReference() and
+    duplication = run.getDuplicationPercentage(i)
+  )
+  or
+  exists(SummaryEvent inLayer0, float duplication0, PipeLineRun run |
+    run = inLayer.getPipelineRuns().getRun(pragma[only_bind_into](iteration)) and
+    successor(recursive, inLayer0, inLayer) and
+    hasDuplication(recursive, ordering, inLayer0, pragma[only_bind_into](iteration), i, duplication0) and
+    duplication = run.getDuplicationPercentage(i).maximum(duplication0)
+  )
+}
+
+predicate hasDuplication(ComputeRecursive recursive, string ordering, int i, float duplication) {
+  duplication =
+    max(SummaryEvent inLayer, int iteration, int dup |
+      inLayer = getInLayerOrRecursive(recursive) and
+      hasDuplication(recursive, ordering, inLayer, iteration, i, dup)
+    |
+      dup
+    )
+}
+
+// -----
+/**
+ * Holds if the bucket `bucket` has `resultSize` resultSize in the `iteration`'th iteration.
+ *
+ * For example, the "base" bucket in iteration 0 has size 42.
+ */
+private predicate hasResultSize(
+  ComputeRecursive recursive, string ordering, SummaryEvent inLayer, int iteration, float resultSize
+) {
+  inLayer = firstPredicate(recursive) and
+  ordering = inLayer.getPipelineRuns().getRun(iteration).getRAReference() and
+  resultSize = inLayer.getDeltaSize(iteration)
+  or
+  exists(SummaryEvent inLayer0, int resultSize0 |
+    successor(recursive, inLayer0, inLayer) and
+    hasResultSize(recursive, ordering, inLayer0, iteration, resultSize0) and
+    resultSize = inLayer.getDeltaSize(iteration) + resultSize0
+  )
+}
+
+predicate hasResultSize(ComputeRecursive recursive, string ordering, float resultSize) {
+  resultSize =
+    strictsum(InLayer inLayer, int iteration, float r |
+      inLayer.getComputeRecursiveEvent() = recursive and
+      hasResultSize(recursive, ordering, inLayer, iteration, r)
+    |
+      r
+    )
+}
+
+RAParser<PipeLine>::RAExpr getAnRaOperation(SummaryEvent inLayer, string ordering) {
+  inLayer.getRA().getPipeLine(ordering) = result.getPredicate()
+}
+
+SummaryEvent getInLayerEventWithName(ComputeRecursive recursive, string predicateName) {
+  result = getInLayerOrRecursive(recursive) and
+  result.getPredicateName() = predicateName
+}
+
+bindingset[predicateName, iteration]
+int getSize(ComputeRecursive recursive, string predicateName, int iteration, TDeltaKind kind) {
+  exists(int i |
+    kind = TPrevious() and
+    i = iteration - 1
+    or
+    kind = TCurrent() and
+    i = iteration
+  |
+    result = getInLayerEventWithName(recursive, predicateName).getDeltaSize(iteration - 1)
+    or
+    not exists(getInLayerEventWithName(recursive, predicateName).getDeltaSize(iteration - 1)) and
+    result = 0
+  )
+}
+
+SummaryEvent getDependencyWithName(Depencencies dependency, string predicateName) {
+  result.getPredicateName() = predicateName and
+  dependency.getADependency() = result
+}
+
+newtype TDeltaKind =
+  TCurrent() or
+  TPrevious()
+
+bindingset[predicateName]
+private predicate isDelta(string predicateName, TDeltaKind kind, string withoutSuffix) {
+  kind = TPrevious() and
+  withoutSuffix = predicateName.regexpCapture("(.+)#prev_delta", 1)
+  or
+  kind = TCurrent() and
+  withoutSuffix = predicateName.regexpCapture("(.+)#cur_delta", 1)
+}
+
+predicate hasDependentPredicateSizeInBucket(
+  ComputeRecursive recursive, string bucket, SummaryEvent inLayer, int iteration,
+  string predicateName, float size
+) {
+  exists( |
+    inLayer = firstPredicate(recursive) and
+    bucket = inLayer.getPipelineRuns().getRun(iteration).getRAReference()
+  |
+    // We treat iteration 0 as a non-recursive case
+    if bucket = "base"
+    then size = getDependencyWithName(recursive.getDependencies(), predicateName).getResultSize()
+    else
+      exists(TDeltaKind kind |
+        size = getSize(recursive, predicateName, iteration, kind) and
+        isDelta(getAnRaOperation(inLayer, bucket).getARhsPredicate(), kind, predicateName)
+      )
+  )
+  or
+  exists(SummaryEvent inLayer0, float size0 |
+    successor(recursive, inLayer0, inLayer) and
+    hasDependentPredicateSizeInBucket(recursive, bucket, inLayer0, iteration, predicateName, size0)
+  |
+    // We treat iteration 0 as a non-recursive case
+    if bucket = "base"
+    then size = getDependencyWithName(recursive.getDependencies(), predicateName).getResultSize()
+    else
+      exists(TDeltaKind kind |
+        size = getSize(recursive, predicateName, iteration, kind) + size0 and
+        isDelta(getAnRaOperation(inLayer, bucket).getARhsPredicate(), kind, predicateName)
+      )
+  )
+}
+
+SummaryEvent getInLayerOrRecursive(ComputeRecursive recursive) {
+  result = recursive or result.(InLayer).getComputeRecursiveEvent() = recursive
+}
+
+predicate hasDependentPredicateSizeInBucket(
+  ComputeRecursive recursive, string bucket, string predicateName, float size
+) {
+  size =
+    strictsum(SummaryEvent inLayer, int iteration, int s |
+      inLayer = getInLayerOrRecursive(recursive) and
+      hasDependentPredicateSizeInBucket(recursive, bucket, inLayer, iteration, predicateName, s)
+    |
+      s
+    )
+}
+
+/**
+ * Gets the badness of a recursive predicate evaluation.
+ *
+ * The badness is the maximum number of tuples in the pipeline divided by the
+ * maximum of two numbers: the size of the result and the size of the largest dependency.
+ *
+ * A dependency of a recursive predicate is defined as follows:
+ * - For a "base" ordering, it is identical to the definition of a dependency for a
+ *   non-recursive predicate
+ * - For a non-"base" ordering, it's defined as any `#prev_delta` or `#cur_delta` predicates
+ *   that appear in the pipeline.
+ */
+float getRecursiveBadness(ComputeRecursive recursive, string bucket) {
+  exists(float maxTupleCount, float resultSize, float maxDependentPredicateSize |
+    maxTupleCount = max(float tc | hasTupleCount(recursive, bucket, _, tc) | tc) and
+    hasResultSize(recursive, bucket, resultSize) and
+    maxDependentPredicateSize =
+      max(float size | hasDependentPredicateSizeInBucket(recursive, bucket, _, size) | size) and
+    resultSize.maximum(maxDependentPredicateSize) > 0 and
+    result = maxTupleCount / resultSize.maximum(maxDependentPredicateSize)
+  )
+}
+
+predicate extractSimpleInformation(
+  ComputeSimple simple, string predicateName, int index, float tupleCount,
+  float duplicationPercentage, string operation
+) {
+  exists(PipeLineRun run |
+    run = simple.getPipelineRun() and
+    tupleCount = run.getCounts().getFloat(pragma[only_bind_into](index)) and
+    duplicationPercentage = run.getDuplicationPercentage().getFloat(pragma[only_bind_into](index)) and
+    operation = simple.getRA().getPipeLine().getLineOfRA(pragma[only_bind_into](index)) and
+    predicateName = simple.getPredicateName()
+  )
+}
+
+predicate extractRecursiveInformation(
+  ComputeRecursive recursive, string predicateName, string ordering, int index, float tupleCount,
+  float duplicationPercentage, string operation
+) {
+  hasTupleCount(recursive, ordering, index, tupleCount) and
+  hasDuplication(recursive, ordering, index, duplicationPercentage) and
+  operation = recursive.getRA().getPipeLine(ordering).getLineOfRA(index) and
+  predicateName = recursive.getPredicateName() + " (" + ordering + ")"
+}
+
 from
-  ComputeSimple evt, float badness, float maxTupleCount, Array tuples, Array duplicationPercentages,
-  Array ra, int index
+  SummaryEvent evt, float badness, float tupleCount, float duplicationPercentage, string operation,
+  int index, string predicateName
 where
-  badness = getBadness(evt) and
  badness > 1.5 and
-  extractInformation(evt, maxTupleCount, tuples, duplicationPercentages, ra)
-select evt.getPredicateName() as predicate_name, badness, index,
-  tuples.getFloat(index) as tuple_count,
-  duplicationPercentages.getFloat(index) as duplication_percentage, ra.getString(index) order by
-    badness desc
+  (
+    badness = getNonRecursiveBadness(evt) and
+    extractSimpleInformation(evt, predicateName, index, tupleCount, duplicationPercentage, operation)
+    or
+    exists(string bucket |
+      badness = getRecursiveBadness(evt, bucket) and
+      extractRecursiveInformation(evt, predicateName, bucket, index, tupleCount,
+        duplicationPercentage, operation)
+    )
+  )
+select predicateName as predicate_name, badness, index, tupleCount as tuple_count,
+  duplicationPercentage as duplication_percentage, operation order by badness desc
--- a/ql/ql/test/experimental/raparser.expected
+++ b/ql/ql/test/experimental/raparser.expected
@@ -1,17 +1,26 @@
 children
 |     return r7 |     {4} r7 = r3 UNION r6 |
 |     {4} r2 = JOIN r1 WITH raparser#d14bb8db::TestPredicate#b ON FIRST 1 OUTPUT "p1", toString("p1"), 123, "  r1 = SCAN fubar\\n   r1" |     {1} r1 = CONSTANT(unique string)["p1"] |
+|     {4} r2 = JOIN r1 WITH raparser#d14bb8db::TestPredicate#b ON FIRST 1 OUTPUT "p1", toString("p1"), 123, "  r1 = SCAN fubar\\n   r1" |     {1} r1 = CONSTANT(unique string)["p1"] |
+|     {4} r3 = STREAM DEDUP r2 |     {4} r2 = JOIN r1 WITH raparser#d14bb8db::TestPredicate#b ON FIRST 1 OUTPUT "p1", toString("p1"), 123, "  r1 = SCAN fubar\\n   r1" |
 |     {4} r3 = STREAM DEDUP r2 |     {4} r2 = JOIN r1 WITH raparser#d14bb8db::TestPredicate#b ON FIRST 1 OUTPUT "p1", toString("p1"), 123, "  r1 = SCAN fubar\\n   r1" |
 |     {4} r4 = JOIN r1 WITH raparser#d14bb8db::TestPredicate#b ON FIRST 1 OUTPUT "p1", "(no string representation)", 123, "  r1 = SCAN fubar\\n   r1" |     {1} r1 = CONSTANT(unique string)["p1"] |
+|     {4} r4 = JOIN r1 WITH raparser#d14bb8db::TestPredicate#b ON FIRST 1 OUTPUT "p1", "(no string representation)", 123, "  r1 = SCAN fubar\\n   r1" |     {1} r1 = CONSTANT(unique string)["p1"] |
+|     {4} r5 = STREAM DEDUP r4 |     {4} r4 = JOIN r1 WITH raparser#d14bb8db::TestPredicate#b ON FIRST 1 OUTPUT "p1", "(no string representation)", 123, "  r1 = SCAN fubar\\n   r1" |
 |     {4} r5 = STREAM DEDUP r4 |     {4} r4 = JOIN r1 WITH raparser#d14bb8db::TestPredicate#b ON FIRST 1 OUTPUT "p1", "(no string representation)", 123, "  r1 = SCAN fubar\\n   r1" |
 |     {4} r6 = r5 AND NOT project##select#query#ffff#nullary({}) |     {4} r5 = STREAM DEDUP r4 |
+|     {4} r6 = r5 AND NOT project##select#query#ffff#nullary({}) |     {4} r5 = STREAM DEDUP r4 |
 |     {4} r7 = r3 UNION r6 |     {4} r3 = STREAM DEDUP r2 |
 |     {4} r7 = r3 UNION r6 |     {4} r6 = r5 AND NOT project##select#query#ffff#nullary({}) |
+|     {4} r7 = r3 UNION r6 |     {4} r6 = r5 AND NOT project##select#query#ffff#nullary({}) |
 #select
 | p1 | 1 |     {1} r1 = CONSTANT(unique string)["p1"] | 1 | 1 | 0 | 0 |
 | p1 | 3 |     {4} r2 = JOIN r1 WITH raparser#d14bb8db::TestPredicate#b ON FIRST 1 OUTPUT "p1", toString("p1"), 123, "  r1 = SCAN fubar\\n   r1" | 2 | 4 | 1 | 1 |
+| p1 | 3 |     {4} r2 = JOIN r1 WITH raparser#d14bb8db::TestPredicate#b ON FIRST 1 OUTPUT "p1", toString("p1"), 123, "  r1 = SCAN fubar\\n   r1" | 2 | 4 | 1 | 1 |
 | p1 | 5 |     {4} r3 = STREAM DEDUP r2 | 3 | 4 | 0 | 1 |
 | p1 | 6 |     {4} r4 = JOIN r1 WITH raparser#d14bb8db::TestPredicate#b ON FIRST 1 OUTPUT "p1", "(no string representation)", 123, "  r1 = SCAN fubar\\n   r1" | 4 | 4 | 1 | 1 |
+| p1 | 6 |     {4} r4 = JOIN r1 WITH raparser#d14bb8db::TestPredicate#b ON FIRST 1 OUTPUT "p1", "(no string representation)", 123, "  r1 = SCAN fubar\\n   r1" | 4 | 4 | 1 | 1 |
 | p1 | 7 |     {4} r5 = STREAM DEDUP r4 | 5 | 4 | 0 | 1 |
 | p1 | 8 |     {4} r6 = r5 AND NOT project##select#query#ffff#nullary({}) | 6 | 4 | 1 | 1 |
+| p1 | 8 |     {4} r6 = r5 AND NOT project##select#query#ffff#nullary({}) | 6 | 4 | 1 | 1 |
 | p1 | 9 |     {4} r7 = r3 UNION r6 | 7 | 4 | 0 | 2 |