Merge pull request #5323 from erik-krogh/staging

Approved by asgerf

javascript/ql/src/semmle/javascript/AMD.qll

@@ -4,6 +4,7 @@
  */
 
 import javascript
+private import semmle.javascript.internal.CachedStages
 
 /**
  * An AMD `define` call.
@@ -298,7 +299,10 @@ private class AmdDependencyImport extends Import {
  */
 class AmdModule extends Module {
   cached
-  AmdModule() { exists(unique(AmdModuleDefinition def | amdModuleTopLevel(def, this))) }
+  AmdModule() {
+    Stages::DataFlowStage::ref() and
+    exists(unique(AmdModuleDefinition def | amdModuleTopLevel(def, this)))
+  }
 
   /** Gets the definition of this module. */
   AmdModuleDefinition getDefine() { amdModuleTopLevel(result, this) }

javascript/ql/src/semmle/javascript/AST.qll

@@ -4,6 +4,7 @@
 
 import javascript
 private import internal.StmtContainers
+private import semmle.javascript.internal.CachedStages
 
 /**
  * A program element corresponding to JavaScript code, such as an expression
@@ -75,7 +76,7 @@ class ASTNode extends @ast_node, NodeInStmtContainer {
 
   /** Gets the toplevel syntactic unit to which this element belongs. */
   cached
-  TopLevel getTopLevel() { result = getParent().getTopLevel() }
+  TopLevel getTopLevel() { Stages::Ast::ref() and result = getParent().getTopLevel() }
 
   /**
    * Gets the `i`th child node of this node.
@@ -119,7 +120,7 @@ class ASTNode extends @ast_node, NodeInStmtContainer {
 
   /** Gets the parent node of this node, if any. */
   cached
-  ASTNode getParent() { this = result.getAChild() }
+  ASTNode getParent() { Stages::Ast::ref() and this = result.getAChild() }
 
   /** Gets the first control flow node belonging to this syntactic entity. */
   ControlFlowNode getFirstControlFlowNode() { result = this }
@@ -135,6 +136,7 @@ class ASTNode extends @ast_node, NodeInStmtContainer {
    */
   cached
   private predicate isAmbientInternal() {
+    Stages::Ast::ref() and
     getParent().isAmbientInternal()
     or
     not isAmbientTopLevel(getTopLevel()) and
@@ -186,7 +188,9 @@ class ASTNode extends @ast_node, NodeInStmtContainer {
  * Holds if the given file is a `.d.ts` file.
  */
 cached
-private predicate isAmbientTopLevel(TopLevel tl) { tl.getFile().getBaseName().matches("%.d.ts") }
+private predicate isAmbientTopLevel(TopLevel tl) {
+  Stages::Ast::ref() and tl.getFile().getBaseName().matches("%.d.ts")
+}
 
 /**
  * A toplevel syntactic unit; that is, a stand-alone script, an inline script

javascript/ql/src/semmle/javascript/BasicBlocks.qll

@@ -5,6 +5,7 @@
 
 import javascript
 private import internal.StmtContainers
+private import semmle.javascript.internal.CachedStages
 
 /**
  * Holds if `nd` starts a new basic block.
@@ -60,7 +61,9 @@ private module Internal {
 
   cached
   predicate useAt(BasicBlock bb, int i, Variable v, VarUse u) {
-    v = u.getVariable() and bbIndex(bb, u, i)
+    Stages::BasicBlocks::ref() and
+    v = u.getVariable() and
+    bbIndex(bb, u, i)
   }
 
   cached

javascript/ql/src/semmle/javascript/Expr.qll

@@ -3,6 +3,7 @@
  */
 
 import javascript
+private import semmle.javascript.internal.CachedStages
 
 /**
  * A program element that is either an expression or a type annotation.
@@ -109,7 +110,7 @@ class Expr extends @expr, ExprOrStmt, ExprOrType, AST::ValueNode {
 
   /** Gets the constant string value this expression evaluates to, if any. */
   cached
-  string getStringValue() { result = getStringValue(this) }
+  string getStringValue() { Stages::Ast::ref() and result = getStringValue(this) }
 
   /** Holds if this expression is impure, that is, its evaluation could have side effects. */
   predicate isImpure() { any() }
@@ -257,6 +258,7 @@ class Expr extends @expr, ExprOrStmt, ExprOrType, AST::ValueNode {
 
 cached
 private DataFlow::Node getCatchParameterFromStmt(Stmt stmt) {
+  Stages::DataFlowStage::ref() and
   result =
     DataFlow::parameterNode(stmt.getEnclosingTryCatchStmt().getACatchClause().getAParameter())
 }
@@ -806,7 +808,9 @@ class FunctionExpr extends @function_expr, Expr, Function {
   /** Gets the statement in which this function expression appears. */
   override Stmt getEnclosingStmt() { result = Expr.super.getEnclosingStmt() }
 
-  override StmtContainer getEnclosingContainer() { result = Expr.super.getContainer() }
+  override StmtContainer getEnclosingContainer() {
+    Stages::Ast::ref() and result = Expr.super.getContainer()
+  }
 
   override predicate isImpure() { none() }
 

javascript/ql/src/semmle/javascript/GlobalAccessPaths.qll

@@ -5,6 +5,7 @@
 import javascript
 private import semmle.javascript.dataflow.InferredTypes
 private import semmle.javascript.dataflow.internal.FlowSteps as FlowSteps
+private import semmle.javascript.internal.CachedStages
 
 deprecated module GlobalAccessPath {
   /**
@@ -429,7 +430,6 @@ module AccessPath {
     /**
      * A classification of acccess paths into reads and writes.
      */
-    cached
     private newtype AccessPathKind =
       AccessPathRead() or
       AccessPathWrite()
@@ -440,6 +440,7 @@ module AccessPath {
      *
      * Only has a result if there exists both a read and write of the access-path within `bb`.
      */
+    pragma[nomagic]
     private ControlFlowNode rankedAccessPath(
       ReachableBasicBlock bb, Root root, string path, int ranking, AccessPathKind type
     ) {
@@ -539,6 +540,7 @@ module AccessPath {
      */
     cached
     predicate hasDominatingWrite(DataFlow::PropRead read) {
+      Stages::FlowSteps::ref() and
       // within the same basic block.
       exists(ReachableBasicBlock bb, Root root, string path, int ranking |
         read.asExpr() = rankedAccessPath(bb, root, path, ranking, AccessPathRead()) and

javascript/ql/src/semmle/javascript/JSDoc.qll

@@ -1,6 +1,7 @@
 /** Provides classes for working with JSDoc comments. */
 
 import javascript
+private import semmle.javascript.internal.CachedStages
 
 /**
  * A JSDoc comment.
@@ -57,7 +58,9 @@ class JSDoc extends @jsdoc, Locatable {
 abstract class Documentable extends ASTNode {
   /** Gets the JSDoc comment for this element, if any. */
   cached
-  JSDoc getDocumentation() { result.getComment().getNextToken() = getFirstToken() }
+  JSDoc getDocumentation() {
+    Stages::Ast::ref() and result.getComment().getNextToken() = getFirstToken()
+  }
 }
 
 /**

javascript/ql/src/semmle/javascript/Modules.qll

@@ -5,6 +5,7 @@
  */
 
 import javascript
+private import semmle.javascript.internal.CachedStages
 
 /**
  * A module, which may either be an ECMAScript 2015-style module,
@@ -239,6 +240,7 @@ abstract class Import extends ASTNode {
    */
   cached
   Module getImportedModule() {
+    Stages::Imports::ref() and
     if exists(resolveExternsImport())
     then result = resolveExternsImport()
     else (

javascript/ql/src/semmle/javascript/SSA.qll

@@ -76,6 +76,7 @@
 
 import javascript
 private import semmle.javascript.dataflow.Refinements
+private import semmle.javascript.internal.CachedStages
 
 /**
  * A variable that can be SSA converted, that is, a local variable.
@@ -354,6 +355,7 @@ private module Internal {
    */
   cached
   SsaDefinition getDefReachingEndOf(ReachableBasicBlock bb, SsaSourceVariable v) {
+    Stages::DataFlowStage::ref() and
     exists(int lastRef | lastRef = max(int i | ssaRef(bb, i, v, _)) |
       result = getLocalDefinition(bb, lastRef, v)
       or

javascript/ql/src/semmle/javascript/dataflow/Configuration.qll

@@ -72,6 +72,7 @@ private import javascript
 private import internal.FlowSteps
 private import internal.AccessPaths
 private import internal.CallGraphs
+private import semmle.javascript.internal.CachedStages
 
 /**
  * A data flow tracking configuration for finding inter-procedural paths from
@@ -733,6 +734,7 @@ private class FlowStepThroughImport extends AdditionalFlowStep, DataFlow::ValueN
   override ImportSpecifier astNode;
 
   override predicate step(DataFlow::Node pred, DataFlow::Node succ) {
+    Stages::FlowSteps::ref() and
     pred = this and
     succ = DataFlow::ssaDefinitionNode(SSA::definition(astNode))
   }

javascript/ql/src/semmle/javascript/dataflow/DataFlow.qll

@@ -24,6 +24,7 @@ private import internal.FlowSteps as FlowSteps
 private import internal.DataFlowNode
 private import internal.AnalyzedParameters
 private import internal.PreCallGraphStep
+private import semmle.javascript.internal.CachedStages
 
 module DataFlow {
   /**
@@ -1493,6 +1494,7 @@ module DataFlow {
    */
   cached
   predicate localFlowStep(Node pred, Node succ) {
+    Stages::DataFlowStage::ref() and
     // flow from RHS into LHS
     lvalueFlowStep(pred, succ)
     or

javascript/ql/src/semmle/javascript/dataflow/Nodes.qll

@@ -7,6 +7,7 @@
 private import javascript
 private import semmle.javascript.dependencies.Dependencies
 private import internal.CallGraphs
+private import semmle.javascript.internal.CachedStages
 
 /**
  * A data flow node corresponding to an expression.
@@ -734,7 +735,7 @@ module ModuleImportNode {
  * This predicate can be extended by subclassing `ModuleImportNode::Range`.
  */
 cached
-ModuleImportNode moduleImport(string path) { result.getPath() = path }
+ModuleImportNode moduleImport(string path) { Stages::Imports::ref() and result.getPath() = path }
 
 /**
  * Gets a (default) import of the given dependency `dep`, such as

javascript/ql/src/semmle/javascript/dataflow/Sources.qll

@@ -8,6 +8,7 @@
 
 private import javascript
 private import semmle.javascript.dataflow.TypeTracking
+private import semmle.javascript.internal.CachedStages
 
 /**
  * A source node for local data flow, that is, a node from which local data flow is tracked.
@@ -220,6 +221,7 @@ private module Cached {
    */
   cached
   predicate namedPropRef(DataFlow::SourceNode base, string prop, DataFlow::PropRef ref) {
+    Stages::DataFlowStage::ref() and
     hasLocalSource(ref.getBase(), base) and
     ref.getPropertyName() = prop
   }

javascript/ql/src/semmle/javascript/dataflow/TaintTracking.qll

@@ -16,6 +16,7 @@
 import javascript
 private import semmle.javascript.dataflow.internal.FlowSteps as FlowSteps
 private import semmle.javascript.dataflow.InferredTypes
+private import semmle.javascript.internal.CachedStages
 
 /**
  * Provides classes for modelling taint propagation.
@@ -232,7 +233,9 @@ module TaintTracking {
     HeapTaintStep() { heapStep(_, this) }
 
     override predicate step(DataFlow::Node pred, DataFlow::Node succ) {
-      heapStep(pred, succ) and succ = this
+      Stages::Taint::ref() and
+      heapStep(pred, succ) and
+      succ = this
     }
   }
 

javascript/ql/src/semmle/javascript/dataflow/internal/FlowSteps.qll

@@ -7,6 +7,7 @@
 import javascript
 import semmle.javascript.dataflow.Configuration
 import semmle.javascript.dataflow.internal.CallGraphs
+private import semmle.javascript.internal.CachedStages
 
 /**
  * Holds if flow should be tracked through properties of `obj`.
@@ -378,6 +379,7 @@ private module CachedSteps {
    */
   cached
   predicate basicLoadStep(DataFlow::Node pred, DataFlow::PropRead succ, string prop) {
+    Stages::TypeTracking::ref() and
     succ.accesses(pred, prop)
   }
 
@@ -403,6 +405,7 @@ private module CachedSteps {
    */
   cached
   predicate callback(DataFlow::Node arg, DataFlow::SourceNode cb) {
+    Stages::TypeTracking::ref() and
     exists(DataFlow::InvokeNode invk, DataFlow::ParameterNode cbParm, DataFlow::Node cbArg |
       arg = invk.getAnArgument() and
       cbParm.flowsTo(invk.getCalleeNode()) and

javascript/ql/src/semmle/javascript/dataflow/internal/PreCallGraphStep.qll

@@ -4,6 +4,7 @@
  */
 
 private import javascript
+private import semmle.javascript.internal.CachedStages
 
 private newtype TUnit = MkUnit()
 
@@ -63,6 +64,7 @@ module PreCallGraphStep {
    */
   cached
   predicate loadStep(DataFlow::Node pred, DataFlow::Node succ, string prop) {
+    Stages::TypeTracking::ref() and
     any(PreCallGraphStep s).loadStep(pred, succ, prop)
   }
 

javascript/ql/src/semmle/javascript/internal/CachedStages.qll

@@ -0,0 +1,238 @@
+/**
+ * INTERNAL: Do not use.
+ *
+ * The purpose of this file is to control which cached predicates belong to the same stage.
+ *
+ * Combining stages can improve performance as we are more likely to reuse shared, non-cached predicates.
+ *
+ * To make a predicate `p` belong to a stage `A`:
+ * - make `p` depend on `A::ref()`, and
+ * - make `A::backref()` depend on `p`.
+ *
+ * Since `A` is a cached module, `ref` and `backref` must be in the same stage, and the dependency
+ * chain above thus forces `p` to be in that stage as well.
+ *
+ * With these two predicates in a `cached module` we ensure that all the cached predicates will be in a single stage at runtime.
+ *
+ * Grouping stages can cause unnecessary computation, as a concrete query might not depend on
+ * all the cached predicates in a stage.
+ * Care should therefore be taken not to combine two stages, if it is likely that a query only depend
+ * on some but not all the cached predicates in the combined stage.
+ */
+
+import javascript
+private import StmtContainers
+private import semmle.javascript.dataflow.internal.PreCallGraphStep
+private import semmle.javascript.dataflow.internal.FlowSteps
+
+/**
+ * Contains a `cached module` for each stage.
+ * Each `cached module` ensures that predicates that are supposed to be in the same stage, are in the same stage.
+ *
+ * Each `cached module` contain two predicates:
+ * The first, `ref`, always holds, and is referenced from `cached` predicates.
+ * The second, `backref`, contains references to the same `cached` predicates.
+ * The `backref` predicate starts with `1 = 1 or` to ensure that the predicate will be optimized down to a constant by the optimizer.
+ */
+module Stages {
+  /**
+   * The `ast` stage.
+   */
+  cached
+  module Ast {
+    /**
+     * Always holds.
+     * Ensures that a predicate is evaluated as part of the Ast stage.
+     */
+    cached
+    predicate ref() { 1 = 1 }
+
+    /**
+     * DONT USE!
+     * Contains references to each predicate that use the above `ref` predicate.
+     */
+    cached
+    predicate backref() {
+      1 = 1
+      or
+      exists(any(ASTNode a).getTopLevel())
+      or
+      exists(any(ASTNode a).getParent())
+      or
+      exists(any(StmtContainer c).getEnclosingContainer())
+      or
+      exists(any(Documentable d).getDocumentation())
+      or
+      exists(any(NodeInStmtContainer n).getContainer())
+      or
+      exists(any(Expr e).getStringValue())
+      or
+      any(ASTNode node).isAmbient()
+    }
+  }
+
+  /**
+   * The `basicblocks` stage.
+   */
+  cached
+  module BasicBlocks {
+    /**
+     * Always holds.
+     * Ensures that a predicate is evaluated as part of the BasicBlocks stage.
+     */
+    cached
+    predicate ref() { 1 = 1 }
+
+    /**
+     * DONT USE!
+     * Contains references to each predicate that use the above `ref` predicate.
+     */
+    cached
+    predicate backref() {
+      1 = 1
+      or
+      any(ReachableBasicBlock bb).dominates(_)
+      or
+      exists(any(BasicBlock bb).getNode(_))
+    }
+  }
+
+  /**
+   * The `dataflow` stage.
+   */
+  cached
+  module DataFlowStage {
+    /**
+     * Always holds.
+     * Ensures that a predicate is evaluated as part of the DataFlow stage.
+     */
+    cached
+    predicate ref() { 1 = 1 }
+
+    /**
+     * DONT USE!
+     * Contains references to each predicate that use the above `ref` predicate.
+     */
+    cached
+    predicate backref() {
+      1 = 1
+      or
+      exists(AmdModule a)
+      or
+      DataFlow::localFlowStep(_, _)
+      or
+      exists(any(DataFlow::SourceNode s).getAPropertyReference("foo"))
+      or
+      exists(any(Expr e).getExceptionTarget())
+      or
+      exists(DataFlow::ssaDefinitionNode(_))
+    }
+  }
+
+  /**
+   * The `imports` stage.
+   *
+   * It would have been preferable to include these predicates in the dataflow or typetracking stage.
+   * But that trips the BDD limit.
+   */
+  cached
+  module Imports {
+    /**
+     * Always holds.
+     * Ensures that a predicate is evaluated as part of the Imports stage.
+     */
+    cached
+    predicate ref() { 1 = 1 }
+
+    /**
+     * DONT USE!
+     * Contains references to each predicate that use the above `ref` predicate.
+     */
+    cached
+    predicate backrefs() {
+      1 = 1
+      or
+      exists(any(Import i).getImportedModule())
+      or
+      exists(DataFlow::moduleImport(_))
+    }
+  }
+
+  /**
+   * The `typetracking` stage.
+   */
+  cached
+  module TypeTracking {
+    /**
+     * Always holds.
+     * Ensures that a predicate is evaluated as part of the TypeTracking stage.
+     */
+    cached
+    predicate ref() { 1 = 1 }
+
+    /**
+     * DONT USE!
+     * Contains references to each predicate that use the above `ref` predicate.
+     */
+    cached
+    predicate backref() {
+      1 = 1
+      or
+      PreCallGraphStep::loadStep(_, _, _)
+      or
+      basicLoadStep(_, _, _)
+    }
+  }
+
+  /**
+   * The `flowsteps` stage.
+   */
+  cached
+  module FlowSteps {
+    /**
+     * Always holds.
+     * Ensures that a predicate is evaluated as part of the FlowSteps stage.
+     */
+    cached
+    predicate ref() { 1 = 1 }
+
+    /**
+     * DONT USE!
+     * Contains references to each predicate that use the above `ref` predicate.
+     */
+    cached
+    predicate backref() {
+      1 = 1
+      or
+      AccessPath::DominatingPaths::hasDominatingWrite(_)
+      or
+      any(DataFlow::AdditionalFlowStep s).step(_, _)
+    }
+  }
+
+  /**
+   * The `taint` stage.
+   */
+  cached
+  module Taint {
+    /**
+     * Always holds.
+     * Ensures that a predicate is evaluated as part of the Taint stage.
+     */
+    cached
+    predicate ref() { 1 = 1 }
+
+    /**
+     * DONT USE!
+     * Contains references to each predicate that use the above `ref` predicate.
+     */
+    cached
+    predicate backref() {
+      1 = 1
+      or
+      any(TaintTracking::AdditionalTaintStep step).step(_, _)
+      or
+      exists(RemoteFlowSource r)
+    }
+  }
+}

javascript/ql/src/semmle/javascript/internal/StmtContainers.qll

@@ -6,9 +6,11 @@
  */
 
 private import javascript
+private import semmle.javascript.internal.CachedStages
 
 cached
 private StmtContainer getStmtContainer(NodeInStmtContainer node) {
+  Stages::Ast::ref() and
   expr_containers(node, result)
   or
   stmt_containers(node, result)

javascript/ql/src/semmle/javascript/security/dataflow/RemoteFlowSources.qll

@@ -5,6 +5,7 @@
 import javascript
 import semmle.javascript.frameworks.HTTP
 import semmle.javascript.security.dataflow.DOM
+private import semmle.javascript.internal.CachedStages
 
 /** A data flow source of remote user input. */
 cached
@@ -110,7 +111,7 @@ private class ExternalRemoteFlowSourceSpecEntryPoint extends API::EntryPoint {
 private class ExternalRemoteFlowSource extends RemoteFlowSource {
   RemoteFlowSourceAccessPath ap;
 
-  ExternalRemoteFlowSource() { this = ap.resolve().getAnImmediateUse() }
+  ExternalRemoteFlowSource() { Stages::Taint::ref() and this = ap.resolve().getAnImmediateUse() }
 
   override string getSourceType() { result = ap.getSourceType() }
 }