From 7595c1c306704e3bf07ef226bfb04bc2d494cc03 Mon Sep 17 00:00:00 2001 From: Mathias Vorreiter Pedersen Date: Thu, 23 Feb 2023 12:59:58 +0000 Subject: [PATCH 1/2] QL: Add a visitor for traversing recursive evaluations. --- ql/ql/src/codeql_ql/StructuredLogs.qll | 154 ++++++++++++++++++ .../src/codeql_ql/ast/internal/TreeSitter.qll | 6 + 2 files changed, 160 insertions(+) diff --git a/ql/ql/src/codeql_ql/StructuredLogs.qll b/ql/ql/src/codeql_ql/StructuredLogs.qll index e763e5bbc85..1637b07c301 100644 --- a/ql/ql/src/codeql_ql/StructuredLogs.qll +++ b/ql/ql/src/codeql_ql/StructuredLogs.qll @@ -261,6 +261,10 @@ module KindPredicatesLog { float getCompletionTime() { result = stringToTimestamp(this.getCompletionTimeString()) } float getResultSize() { result = this.getFloat("resultSize") } + + Array getRA(string ordering) { result = this.getObject("ra").getArray(ordering) } + + string getAnOrdering() { exists(this.getRA(result)) } } class SentinelEmpty extends SummaryEvent { @@ -317,12 +321,162 @@ module KindPredicatesLog { Predicate getPredicate() { result = getPredicateFromPosition(this.getPosition()) } } + /** Gets the `index`'th event that's evaluated by `recursive`. */ + private InLayer layerEventRank(ComputeRecursive recursive, int index) { + result = + rank[index + 1](InLayer cand, int startline, string filepath | + cand.getComputeRecursiveEvent() = recursive and + cand.hasLocationInfo(filepath, startline, _, _, _) + | + cand order by filepath, startline + ) + } + + /** + * Gets the first predicate that's evaluated in an iteration + * of the SCC computation rooted at `recursive`. + */ + private InLayer firstPredicate(ComputeRecursive recursive) { + result = layerEventRank(recursive, 0) + } + + /** + * Gets the last predicate that's evaluated in an iteration + * of the SCC computation rooted at `recursive`. + */ + private InLayer lastPredicate(ComputeRecursive recursive) { + exists(int n | + result = layerEventRank(recursive, n) and + not exists(layerEventRank(recursive, n + 1)) + ) + } + + /** + * Holds if the predicate represented by `next` was evaluated after the + * predicate represented by `prev` in the SCC computation rooted at `recursive`. + */ + predicate successor(ComputeRecursive recursive, InLayer prev, InLayer next) { + exists(int index | + layerEventRank(recursive, index) = prev and + layerEventRank(recursive, index + 1) = next + ) + } + + /** + * Holds if the predicate represented by `inLayer` was run in the `iteration`'iteration + * of the SCC computation rooted at `recursive`. + */ + private predicate ran(ComputeRecursive recursive, int iteration, InLayer inLayer) { + exists(int index | + inLayer = layerEventRank(recursive, index) and + inLayer.getPredicateIterationMillis().getNumber(iteration) >= 0 + ) + } + + /** + * Gets the next iteration in which the predicate `pred` in the `iteration`'th iteration + * of a recursive SCC rooted at `recursive` should be evaluated. + */ + int nextPipeline(ComputeRecursive recursive, int iteration, InLayer inLayer) { + iteration = 0 and + if ran(recursive, iteration, inLayer) then result = 1 else result = 0 + or + iteration > 1 and + exists(int n | n = nextPipeline(recursive, iteration - 1, inLayer) | + if ran(recursive, iteration, inLayer) then result = n + 1 else result = n + ) + } + + bindingset[this] + signature class ResultSig; + + /** + * A signature for generically traversing a SCC computation. + */ + signature module Fold { + /** + * Gets the base case for the fold. That is, the initial value that + * is produced from the first evaluation of the first IN_LAYER event + * in the recursive evaluation. + */ + bindingset[run] + R base(PipeLineRun run); + + /** + * Gets the recursive case for the fold. That is, `r` is the accumulation + * of the previous evaluations, and `run` is the pipeline of the next IN_LAYER + * event that is evaluated. + */ + bindingset[run, r] + R fold(PipeLineRun run, R r); + } + + module Iterate F> { + private R iterate(ComputeRecursive recursive, int iteration, InLayer pred) { + // Case: The first iteration + iteration = 0 and + ( + // Subcase: The first predicate in the first iteration + pred = firstPredicate(recursive) and + result = F::base(pred.getPipelineRuns().getRun(0)) + or + // Subcase: The predicate has a predecessor + exists(InLayer pred0, R r | + successor(recursive, pred0, pred) and + r = iterate(recursive, 0, pred0) and + result = F::fold(pred.getPipelineRuns().getRun(0), r) + ) + ) + or + // Case: Not the first iteration + iteration > 0 and + ( + // Subcase: The first predicate in the iteration + pred = firstPredicate(recursive) and + exists(InLayer last, R r | + last = lastPredicate(recursive) and + r = iterate(recursive, iteration - 1, last) and + result = F::fold(pred.getPipelineRuns().getRun(iteration), r) + ) + or + // Subcase: The predicate has a predecessor in the same iteration + exists(InLayer pred0, R r | + successor(recursive, pred0, pred) and + r = iterate(recursive, iteration, pred0) and + result = F::fold(pred.getPipelineRuns().getRun(iteration), r) + ) + ) + } + + R iterate(ComputeRecursive recursive) { + exists(int iteration, InLayer pred | + pred = lastPredicate(recursive) and + result = iterate(recursive, iteration, pred) and + not exists(iterate(recursive, iteration + 1, pred)) + ) + } + } + class ComputeRecursive extends SummaryEvent { ComputeRecursive() { evaluationStrategy = "COMPUTE_RECURSIVE" } } class InLayer extends SummaryEvent { InLayer() { evaluationStrategy = "IN_LAYER" } + + string getMainHash() { result = this.getString("mainHash") } + + ComputeRecursive getComputeRecursiveEvent() { result.getRAHash() = this.getMainHash() } + + Array getPredicateIterationMillis() { result = this.getArray("predicateIterationMillis") } + + float getPredicateIterationMillis(int i) { + result = getRanked(this.getArray("predicateIterationMillis"), i) + } + + PipeLineRuns getPipelineRuns() { result = this.getArray("pipelineRuns") } + + float getDeltaSize(int i) { result = getRanked(this.getArray("deltaSizes"), i) } } class ComputedExtensional extends SummaryEvent { diff --git a/ql/ql/src/codeql_ql/ast/internal/TreeSitter.qll b/ql/ql/src/codeql_ql/ast/internal/TreeSitter.qll index d15da37b4f2..1e7bfa71352 100644 --- a/ql/ql/src/codeql_ql/ast/internal/TreeSitter.qll +++ b/ql/ql/src/codeql_ql/ast/internal/TreeSitter.qll @@ -1866,6 +1866,12 @@ module JSON { /** Gets the location of this element. */ final L::Location getLocation() { json_ast_node_info(this, _, _, result) } + predicate hasLocationInfo( + string filepath, int startline, int startcolumn, int endline, int endcolumn + ) { + this.getLocation().hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn) + } + /** Gets the parent of this element. */ final AstNode getParent() { json_ast_node_info(this, result, _, _) } From 23b9abcbbfce60b70d50847176a3a51069a22a0d Mon Sep 17 00:00:00 2001 From: Mathias Vorreiter Pedersen Date: Thu, 23 Feb 2023 13:00:22 +0000 Subject: [PATCH 2/2] QL: Add a query for finding the predicates with the highest tuple sums. --- ql/ql/src/codeql_ql/StructuredLogs.qll | 25 ----------------- .../src/queries/performance/LargeTupleSum.ql | 28 +++++++++++++++++++ 2 files changed, 28 insertions(+), 25 deletions(-) create mode 100644 ql/ql/src/queries/performance/LargeTupleSum.ql diff --git a/ql/ql/src/codeql_ql/StructuredLogs.qll b/ql/ql/src/codeql_ql/StructuredLogs.qll index 1637b07c301..26925241ce5 100644 --- a/ql/ql/src/codeql_ql/StructuredLogs.qll +++ b/ql/ql/src/codeql_ql/StructuredLogs.qll @@ -362,31 +362,6 @@ module KindPredicatesLog { ) } - /** - * Holds if the predicate represented by `inLayer` was run in the `iteration`'iteration - * of the SCC computation rooted at `recursive`. - */ - private predicate ran(ComputeRecursive recursive, int iteration, InLayer inLayer) { - exists(int index | - inLayer = layerEventRank(recursive, index) and - inLayer.getPredicateIterationMillis().getNumber(iteration) >= 0 - ) - } - - /** - * Gets the next iteration in which the predicate `pred` in the `iteration`'th iteration - * of a recursive SCC rooted at `recursive` should be evaluated. - */ - int nextPipeline(ComputeRecursive recursive, int iteration, InLayer inLayer) { - iteration = 0 and - if ran(recursive, iteration, inLayer) then result = 1 else result = 0 - or - iteration > 1 and - exists(int n | n = nextPipeline(recursive, iteration - 1, inLayer) | - if ran(recursive, iteration, inLayer) then result = n + 1 else result = n - ) - } - bindingset[this] signature class ResultSig; diff --git a/ql/ql/src/queries/performance/LargeTupleSum.ql b/ql/ql/src/queries/performance/LargeTupleSum.ql new file mode 100644 index 00000000000..e5bbed79bbd --- /dev/null +++ b/ql/ql/src/queries/performance/LargeTupleSum.ql @@ -0,0 +1,28 @@ +/** + * Shows a list of the "slow" predicates by tuple sum. + */ + +import ql +import codeql_ql.StructuredLogs +import KindPredicatesLog + +module SumCounts implements Fold { + int base(PipeLineRun run) { result = sum(int i | | run.getCount(i)) } + + bindingset[s] + int fold(PipeLineRun run, int s) { result = sum(int i | | run.getCount(i)) + s } +} + +int sumTuples(SummaryEvent event) { + result = strictsum(int i | | event.(ComputeSimple).getPipelineRun().getCount(i)) + or + result = Iterate::iterate(event) +} + +int predicateRank(SummaryEvent evt) { + evt = rank[result](SummaryEvent y, int s | s = sumTuples(y) | y order by s desc) +} + +from SummaryEvent evt, int s +where predicateRank(evt) < 50 and s = sumTuples(evt) +select evt, s order by s desc