Merge remote-tracking branch 'upstream/main' into python-dataflow/flow-summaries-from-scratch

.devcontainer/devcontainer.json

@@ -3,6 +3,8 @@
     "rust-lang.rust",
     "bungcip.better-toml",
     "github.vscode-codeql",
+    "hbenl.vscode-test-explorer",
+    "ms-vscode.test-adapter-converter",
     "slevesque.vscode-zipexplorer"
   ],
   "settings": {

.github/workflows/swift-codegen.yml

@@ -18,8 +18,16 @@ jobs:
       - name: Run unit tests
         run: |
           bazel test //swift/codegen/test --test_output=errors
-      - name: Check that code was generated
+      - name: Check that QL generated code was checked in
         run: |
           bazel run //swift/codegen
           git add swift
           git diff --exit-code --stat HEAD
+      - name: Generate C++ files
+        run: |
+          bazel run //swift/codegen:trapgen -- --cpp-output=$PWD/swift-generated-headers
+          bazel run //swift/codegen:cppgen -- --cpp-output=$PWD/swift-generated-headers
+      - uses: actions/upload-artifact@v3
+        with:
+          name: swift-generated-headers
+          path: swift-generated-headers/*.h

CODEOWNERS

@@ -5,6 +5,8 @@
 /python/ @github/codeql-python
 /ruby/ @github/codeql-ruby
 /swift/ @github/codeql-c
+/java/kotlin-extractor/ @github/codeql-kotlin
+/java/kotlin-explorer/ @github/codeql-kotlin
 
 # ML-powered queries
 /javascript/ql/experimental/adaptivethreatmodeling/ @github/codeql-ml-powered-queries-reviewers

config/identical-files.json

@@ -476,20 +476,23 @@
     "python/ql/lib/semmle/python/security/internal/SensitiveDataHeuristics.qll",
     "ruby/ql/lib/codeql/ruby/security/internal/SensitiveDataHeuristics.qll"
   ],
-  "ReDoS Util Python/JS/Ruby": [
+  "ReDoS Util Python/JS/Ruby/Java": [
     "javascript/ql/lib/semmle/javascript/security/performance/ReDoSUtil.qll",
     "python/ql/lib/semmle/python/security/performance/ReDoSUtil.qll",
-    "ruby/ql/lib/codeql/ruby/security/performance/ReDoSUtil.qll"
+    "ruby/ql/lib/codeql/ruby/security/performance/ReDoSUtil.qll",
+    "java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll"
   ],
-  "ReDoS Exponential Python/JS/Ruby": [
+  "ReDoS Exponential Python/JS/Ruby/Java": [
     "javascript/ql/lib/semmle/javascript/security/performance/ExponentialBackTracking.qll",
     "python/ql/lib/semmle/python/security/performance/ExponentialBackTracking.qll",
-    "ruby/ql/lib/codeql/ruby/security/performance/ExponentialBackTracking.qll"
+    "ruby/ql/lib/codeql/ruby/security/performance/ExponentialBackTracking.qll",
+    "java/ql/lib/semmle/code/java/security/performance/ExponentialBackTracking.qll"
   ],
-  "ReDoS Polynomial Python/JS/Ruby": [
+  "ReDoS Polynomial Python/JS/Ruby/Java": [
     "javascript/ql/lib/semmle/javascript/security/performance/SuperlinearBackTracking.qll",
     "python/ql/lib/semmle/python/security/performance/SuperlinearBackTracking.qll",
-    "ruby/ql/lib/codeql/ruby/security/performance/SuperlinearBackTracking.qll"
+    "ruby/ql/lib/codeql/ruby/security/performance/SuperlinearBackTracking.qll",
+    "java/ql/lib/semmle/code/java/security/performance/SuperlinearBackTracking.qll"
   ],
   "BadTagFilterQuery Python/JS/Ruby": [
     "javascript/ql/lib/semmle/javascript/security/BadTagFilterQuery.qll",

cpp/downgrades/cf72c8898d19eb1b3374432cf79d8276cb07ad43/exprparents.ql

@@ -0,0 +1,21 @@
+class Element extends @element {
+  string toString() { none() }
+}
+
+class Expr extends @expr {
+  string toString() { none() }
+}
+
+class Stmt extends @stmt {
+  string toString() { none() }
+}
+
+predicate isStmtWithInitializer(Stmt stmt) {
+  exists(int kind | stmts(stmt, kind, _) | kind = 2 or kind = 11 or kind = 35)
+}
+
+from Expr child, int index, int index_new, Element parent
+where
+  exprparents(child, index, parent) and
+  if isStmtWithInitializer(parent) then index_new = index - 1 else index_new = index
+select child, index_new, parent

cpp/downgrades/cf72c8898d19eb1b3374432cf79d8276cb07ad43/stmtparents.ql

@@ -0,0 +1,22 @@
+class Element extends @element {
+  string toString() { none() }
+}
+
+class Stmt extends @stmt {
+  string toString() { none() }
+}
+
+predicate isStmtWithInitializer(Stmt stmt) {
+  exists(int kind | stmts(stmt, kind, _) | kind = 2 or kind = 11 or kind = 35)
+}
+
+from Stmt child, int index, int index_new, Element parent
+where
+  stmtparents(child, index, parent) and
+  (
+    not isStmtWithInitializer(parent)
+    or
+    index > 0
+  ) and
+  if isStmtWithInitializer(parent) then index_new = index - 1 else index_new = index
+select child, index_new, parent

cpp/downgrades/cf72c8898d19eb1b3374432cf79d8276cb07ad43/upgrade.properties

@@ -0,0 +1,6 @@
+description: Support C++17 if and switch initializers
+compatibility: partial
+if_initialization.rel: delete
+switch_initialization.rel: delete
+exprparents.rel: run exprparents.qlo
+stmtparents.rel: run stmtparents.qlo

cpp/ql/lib/change-notes/2022-04-12-if-and-switch-initializers.md

@@ -0,0 +1,4 @@
+---
+category: feature
+---
+* A `getInitialization` predicate was added to the `ConstexprIfStmt`, `IfStmt`, and `SwitchStmt` classes that yields the C++17-style initializer of the `if` or `switch` statement when it exists.

cpp/ql/lib/change-notes/2022-05-11-deprecated-analysed-string.md

@@ -0,0 +1,4 @@
+---
+ category: deprecated
+---
+ * The `AnalysedString` class in the `StringAnalysis` module has been replaced with `AnalyzedString`, to follow our style guide. The old name still exists as a deprecated alias.

cpp/ql/lib/semmle/code/cpp/PrintAST.qll

@@ -663,18 +663,24 @@ private predicate namedStmtChildPredicates(Locatable s, Element e, string pred)
     or
     s.(ComputedGotoStmt).getExpr() = e and pred = "getExpr()"
     or
+    s.(ConstexprIfStmt).getInitialization() = e and pred = "getInitialization()"
+    or
     s.(ConstexprIfStmt).getCondition() = e and pred = "getCondition()"
     or
     s.(ConstexprIfStmt).getThen() = e and pred = "getThen()"
     or
     s.(ConstexprIfStmt).getElse() = e and pred = "getElse()"
     or
+    s.(IfStmt).getInitialization() = e and pred = "getInitialization()"
+    or
     s.(IfStmt).getCondition() = e and pred = "getCondition()"
     or
     s.(IfStmt).getThen() = e and pred = "getThen()"
     or
     s.(IfStmt).getElse() = e and pred = "getElse()"
     or
+    s.(SwitchStmt).getInitialization() = e and pred = "getInitialization()"
+    or
     s.(SwitchStmt).getExpr() = e and pred = "getExpr()"
     or
     s.(SwitchStmt).getStmt() = e and pred = "getStmt()"

cpp/ql/lib/semmle/code/cpp/commons/Printf.qll

@@ -1312,7 +1312,7 @@ class FormatLiteral extends Literal {
         len =
           min(int v |
             v = this.getPrecision(n) or
-            v = this.getUse().getFormatArgument(n).(AnalysedString).getMaxLength() - 1 // (don't count null terminator)
+            v = this.getUse().getFormatArgument(n).(AnalyzedString).getMaxLength() - 1 // (don't count null terminator)
           ) and
         reason = TValueFlowAnalysis()
       )

cpp/ql/lib/semmle/code/cpp/commons/StringAnalysis.qll

@@ -27,11 +27,14 @@ predicate canValueFlow(Expr fromExpr, Expr toExpr) {
   fromExpr = toExpr.(ConditionalExpr).getElse()
 }
 
+/** DEPRECATED: Alias for AnalyzedString */
+deprecated class AnalysedString = AnalyzedString;
+
 /**
- * An analysed null terminated string.
+ * An analyzed null terminated string.
  */
-class AnalysedString extends Expr {
-  AnalysedString() {
+class AnalyzedString extends Expr {
+  AnalyzedString() {
     this.getUnspecifiedType() instanceof ArrayType or
     this.getUnspecifiedType() instanceof PointerType
   }
@@ -41,15 +44,15 @@ class AnalysedString extends Expr {
    * can be calculated.
    */
   int getMaxLength() {
-    // take the longest AnalysedString it's value could 'flow' from; however if even one doesn't
+    // take the longest AnalyzedString its value could 'flow' from; however if even one doesn't
     // return a value (this essentially means 'infinity') we can't return a value either.
     result =
-      max(AnalysedString expr, int toMax |
+      max(AnalyzedString expr, int toMax |
         canValueFlow*(expr, this) and toMax = expr.(StringLiteral).getOriginalLength()
       |
         toMax
       ) and // maximum length
-    forall(AnalysedString expr | canValueFlow(expr, this) | exists(expr.getMaxLength())) // all sources return a value (recursive)
+    forall(AnalyzedString expr | canValueFlow(expr, this) | exists(expr.getMaxLength())) // all sources return a value (recursive)
   }
 }
 

cpp/ql/lib/semmle/code/cpp/controlflow/internal/CFG.qll

@@ -708,30 +708,33 @@ private predicate straightLineSparse(Node scope, int i, Node ni, Spec spec) {
   or
   scope =
     any(SwitchStmt s |
+      // SwitchStmt [-> init] -> expr
       i = -1 and ni = s and spec.isAt()
       or
-      i = 0 and ni = s.getExpr() and spec.isAround()
+      i = 0 and ni = s.getInitialization() and spec.isAround()
+      or
+      i = 1 and ni = s.getExpr() and spec.isAround()
       or
       // If the switch body is not a block then this step is skipped, and the
       // expression jumps directly to the cases.
-      i = 1 and ni = s.getStmt().(BlockStmt) and spec.isAt()
+      i = 2 and ni = s.getStmt().(BlockStmt) and spec.isAt()
       or
-      i = 2 and ni = s.getASwitchCase() and spec.isBefore()
+      i = 3 and ni = s.getASwitchCase() and spec.isBefore()
       or
       // If there is no default case, we can jump to after the block. Note: `i`
       // is same value as above.
       not s.getASwitchCase() instanceof DefaultCase and
-      i = 2 and
+      i = 3 and
       ni = s.getStmt() and
       spec.isAfter()
       or
-      i = 3 and /* BARRIER */ ni = s and spec.isBarrier()
+      i = 4 and /* BARRIER */ ni = s and spec.isBarrier()
       or
-      i = 4 and ni = s.getStmt() and spec.isAfter()
+      i = 5 and ni = s.getStmt() and spec.isAfter()
       or
-      i = 5 and ni = s and spec.isAroundDestructors()
+      i = 6 and ni = s and spec.isAroundDestructors()
       or
-      i = 6 and ni = s and spec.isAfter()
+      i = 7 and ni = s and spec.isAfter()
     )
   or
   scope =
@@ -836,8 +839,15 @@ private predicate subEdge(Pos p1, Node n1, Node n2, Pos p2) {
     p2.nodeAt(n2, f)
   )
   or
-  // IfStmt -> condition ; { then, else } ->
+  // IfStmt -> [ init -> ] condition ; { then, else } ->
   exists(IfStmt s |
+    p1.nodeAt(n1, s) and
+    p2.nodeBefore(n2, s.getInitialization())
+    or
+    p1.nodeAfter(n1, s.getInitialization()) and
+    p2.nodeBefore(n2, s.getCondition())
+    or
+    not exists(s.getInitialization()) and
     p1.nodeAt(n1, s) and
     p2.nodeBefore(n2, s.getCondition())
     or
@@ -851,8 +861,15 @@ private predicate subEdge(Pos p1, Node n1, Node n2, Pos p2) {
     p2.nodeAfter(n2, s)
   )
   or
-  // ConstexprIfStmt -> condition ; { then, else } -> // same as IfStmt
+  // ConstexprIfStmt -> [ init -> ] condition ; { then, else } -> // same as IfStmt
   exists(ConstexprIfStmt s |
+    p1.nodeAt(n1, s) and
+    p2.nodeBefore(n2, s.getInitialization())
+    or
+    p1.nodeAfter(n1, s.getInitialization()) and
+    p2.nodeBefore(n2, s.getCondition())
+    or
+    not exists(s.getInitialization()) and
     p1.nodeAt(n1, s) and
     p2.nodeBefore(n2, s.getCondition())
     or

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl.qll

@@ -170,6 +170,14 @@ abstract class Configuration extends string {
    */
   int explorationLimit() { none() }
 
+  /**
+   * Holds if hidden nodes should be included in the data flow graph.
+   *
+   * This feature should only be used for debugging or when the data flow graph
+   * is not visualized (for example in a `path-problem` query).
+   */
+  predicate includeHiddenNodes() { none() }
+
   /**
    * Holds if there is a partial data flow path from `source` to `node`. The
    * approximate distance between `node` and the closest source is `dist` and
@@ -1673,10 +1681,24 @@ private module Stage2 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -2495,10 +2517,24 @@ private module Stage3 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -3322,10 +3358,24 @@ private module Stage4 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -3394,17 +3444,28 @@ private Configuration unbindConf(Configuration conf) {
   exists(Configuration c | result = pragma[only_bind_into](c) and conf = pragma[only_bind_into](c))
 }
 
-private predicate nodeMayUseSummary(
-  NodeEx n, FlowState state, AccessPathApprox apa, Configuration config
+pragma[nomagic]
+private predicate nodeMayUseSummary0(
+  NodeEx n, DataFlowCallable c, FlowState state, AccessPathApprox apa, Configuration config
 ) {
-  exists(DataFlowCallable c, AccessPathApprox apa0 |
-    Stage4::parameterMayFlowThrough(_, c, apa, _) and
+  exists(AccessPathApprox apa0 |
+    Stage4::parameterMayFlowThrough(_, c, _, _) and
     Stage4::revFlow(n, state, true, _, apa0, config) and
     Stage4::fwdFlow(n, state, any(CallContextCall ccc), TAccessPathApproxSome(apa), apa0, config) and
     n.getEnclosingCallable() = c
   )
 }
 
+pragma[nomagic]
+private predicate nodeMayUseSummary(
+  NodeEx n, FlowState state, AccessPathApprox apa, Configuration config
+) {
+  exists(DataFlowCallable c |
+    Stage4::parameterMayFlowThrough(_, c, apa, config) and
+    nodeMayUseSummary0(n, c, state, apa, config)
+  )
+}
+
 private newtype TSummaryCtx =
   TSummaryCtxNone() or
   TSummaryCtxSome(ParamNodeEx p, FlowState state, AccessPath ap) {
@@ -3600,7 +3661,7 @@ private newtype TPathNode =
  * of dereference operations needed to get from the value in the node to the
  * tracked object. The final type indicates the type of the tracked object.
  */
-abstract private class AccessPath extends TAccessPath {
+private class AccessPath extends TAccessPath {
   /** Gets the head of this access path, if any. */
   abstract TypedContent getHead();
 
@@ -3815,11 +3876,14 @@ abstract private class PathNodeImpl extends PathNode {
   abstract NodeEx getNodeEx();
 
   predicate isHidden() {
-    hiddenNode(this.getNodeEx().asNode()) and
-    not this.isSource() and
-    not this instanceof PathNodeSink
-    or
-    this.getNodeEx() instanceof TNodeImplicitRead
+    not this.getConfiguration().includeHiddenNodes() and
+    (
+      hiddenNode(this.getNodeEx().asNode()) and
+      not this.isSource() and
+      not this instanceof PathNodeSink
+      or
+      this.getNodeEx() instanceof TNodeImplicitRead
+    )
   }
 
   private string ppAp() {

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl2.qll

@@ -170,6 +170,14 @@ abstract class Configuration extends string {
    */
   int explorationLimit() { none() }
 
+  /**
+   * Holds if hidden nodes should be included in the data flow graph.
+   *
+   * This feature should only be used for debugging or when the data flow graph
+   * is not visualized (for example in a `path-problem` query).
+   */
+  predicate includeHiddenNodes() { none() }
+
   /**
    * Holds if there is a partial data flow path from `source` to `node`. The
    * approximate distance between `node` and the closest source is `dist` and
@@ -1673,10 +1681,24 @@ private module Stage2 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -2495,10 +2517,24 @@ private module Stage3 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -3322,10 +3358,24 @@ private module Stage4 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -3394,17 +3444,28 @@ private Configuration unbindConf(Configuration conf) {
   exists(Configuration c | result = pragma[only_bind_into](c) and conf = pragma[only_bind_into](c))
 }
 
-private predicate nodeMayUseSummary(
-  NodeEx n, FlowState state, AccessPathApprox apa, Configuration config
+pragma[nomagic]
+private predicate nodeMayUseSummary0(
+  NodeEx n, DataFlowCallable c, FlowState state, AccessPathApprox apa, Configuration config
 ) {
-  exists(DataFlowCallable c, AccessPathApprox apa0 |
-    Stage4::parameterMayFlowThrough(_, c, apa, _) and
+  exists(AccessPathApprox apa0 |
+    Stage4::parameterMayFlowThrough(_, c, _, _) and
     Stage4::revFlow(n, state, true, _, apa0, config) and
     Stage4::fwdFlow(n, state, any(CallContextCall ccc), TAccessPathApproxSome(apa), apa0, config) and
     n.getEnclosingCallable() = c
   )
 }
 
+pragma[nomagic]
+private predicate nodeMayUseSummary(
+  NodeEx n, FlowState state, AccessPathApprox apa, Configuration config
+) {
+  exists(DataFlowCallable c |
+    Stage4::parameterMayFlowThrough(_, c, apa, config) and
+    nodeMayUseSummary0(n, c, state, apa, config)
+  )
+}
+
 private newtype TSummaryCtx =
   TSummaryCtxNone() or
   TSummaryCtxSome(ParamNodeEx p, FlowState state, AccessPath ap) {
@@ -3600,7 +3661,7 @@ private newtype TPathNode =
  * of dereference operations needed to get from the value in the node to the
  * tracked object. The final type indicates the type of the tracked object.
  */
-abstract private class AccessPath extends TAccessPath {
+private class AccessPath extends TAccessPath {
   /** Gets the head of this access path, if any. */
   abstract TypedContent getHead();
 
@@ -3815,11 +3876,14 @@ abstract private class PathNodeImpl extends PathNode {
   abstract NodeEx getNodeEx();
 
   predicate isHidden() {
-    hiddenNode(this.getNodeEx().asNode()) and
-    not this.isSource() and
-    not this instanceof PathNodeSink
-    or
-    this.getNodeEx() instanceof TNodeImplicitRead
+    not this.getConfiguration().includeHiddenNodes() and
+    (
+      hiddenNode(this.getNodeEx().asNode()) and
+      not this.isSource() and
+      not this instanceof PathNodeSink
+      or
+      this.getNodeEx() instanceof TNodeImplicitRead
+    )
   }
 
   private string ppAp() {

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl3.qll

@@ -170,6 +170,14 @@ abstract class Configuration extends string {
    */
   int explorationLimit() { none() }
 
+  /**
+   * Holds if hidden nodes should be included in the data flow graph.
+   *
+   * This feature should only be used for debugging or when the data flow graph
+   * is not visualized (for example in a `path-problem` query).
+   */
+  predicate includeHiddenNodes() { none() }
+
   /**
    * Holds if there is a partial data flow path from `source` to `node`. The
    * approximate distance between `node` and the closest source is `dist` and
@@ -1673,10 +1681,24 @@ private module Stage2 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -2495,10 +2517,24 @@ private module Stage3 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -3322,10 +3358,24 @@ private module Stage4 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -3394,17 +3444,28 @@ private Configuration unbindConf(Configuration conf) {
   exists(Configuration c | result = pragma[only_bind_into](c) and conf = pragma[only_bind_into](c))
 }
 
-private predicate nodeMayUseSummary(
-  NodeEx n, FlowState state, AccessPathApprox apa, Configuration config
+pragma[nomagic]
+private predicate nodeMayUseSummary0(
+  NodeEx n, DataFlowCallable c, FlowState state, AccessPathApprox apa, Configuration config
 ) {
-  exists(DataFlowCallable c, AccessPathApprox apa0 |
-    Stage4::parameterMayFlowThrough(_, c, apa, _) and
+  exists(AccessPathApprox apa0 |
+    Stage4::parameterMayFlowThrough(_, c, _, _) and
     Stage4::revFlow(n, state, true, _, apa0, config) and
     Stage4::fwdFlow(n, state, any(CallContextCall ccc), TAccessPathApproxSome(apa), apa0, config) and
     n.getEnclosingCallable() = c
   )
 }
 
+pragma[nomagic]
+private predicate nodeMayUseSummary(
+  NodeEx n, FlowState state, AccessPathApprox apa, Configuration config
+) {
+  exists(DataFlowCallable c |
+    Stage4::parameterMayFlowThrough(_, c, apa, config) and
+    nodeMayUseSummary0(n, c, state, apa, config)
+  )
+}
+
 private newtype TSummaryCtx =
   TSummaryCtxNone() or
   TSummaryCtxSome(ParamNodeEx p, FlowState state, AccessPath ap) {
@@ -3600,7 +3661,7 @@ private newtype TPathNode =
  * of dereference operations needed to get from the value in the node to the
  * tracked object. The final type indicates the type of the tracked object.
  */
-abstract private class AccessPath extends TAccessPath {
+private class AccessPath extends TAccessPath {
   /** Gets the head of this access path, if any. */
   abstract TypedContent getHead();
 
@@ -3815,11 +3876,14 @@ abstract private class PathNodeImpl extends PathNode {
   abstract NodeEx getNodeEx();
 
   predicate isHidden() {
-    hiddenNode(this.getNodeEx().asNode()) and
-    not this.isSource() and
-    not this instanceof PathNodeSink
-    or
-    this.getNodeEx() instanceof TNodeImplicitRead
+    not this.getConfiguration().includeHiddenNodes() and
+    (
+      hiddenNode(this.getNodeEx().asNode()) and
+      not this.isSource() and
+      not this instanceof PathNodeSink
+      or
+      this.getNodeEx() instanceof TNodeImplicitRead
+    )
   }
 
   private string ppAp() {

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl4.qll

@@ -170,6 +170,14 @@ abstract class Configuration extends string {
    */
   int explorationLimit() { none() }
 
+  /**
+   * Holds if hidden nodes should be included in the data flow graph.
+   *
+   * This feature should only be used for debugging or when the data flow graph
+   * is not visualized (for example in a `path-problem` query).
+   */
+  predicate includeHiddenNodes() { none() }
+
   /**
    * Holds if there is a partial data flow path from `source` to `node`. The
    * approximate distance between `node` and the closest source is `dist` and
@@ -1673,10 +1681,24 @@ private module Stage2 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -2495,10 +2517,24 @@ private module Stage3 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -3322,10 +3358,24 @@ private module Stage4 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -3394,17 +3444,28 @@ private Configuration unbindConf(Configuration conf) {
   exists(Configuration c | result = pragma[only_bind_into](c) and conf = pragma[only_bind_into](c))
 }
 
-private predicate nodeMayUseSummary(
-  NodeEx n, FlowState state, AccessPathApprox apa, Configuration config
+pragma[nomagic]
+private predicate nodeMayUseSummary0(
+  NodeEx n, DataFlowCallable c, FlowState state, AccessPathApprox apa, Configuration config
 ) {
-  exists(DataFlowCallable c, AccessPathApprox apa0 |
-    Stage4::parameterMayFlowThrough(_, c, apa, _) and
+  exists(AccessPathApprox apa0 |
+    Stage4::parameterMayFlowThrough(_, c, _, _) and
     Stage4::revFlow(n, state, true, _, apa0, config) and
     Stage4::fwdFlow(n, state, any(CallContextCall ccc), TAccessPathApproxSome(apa), apa0, config) and
     n.getEnclosingCallable() = c
   )
 }
 
+pragma[nomagic]
+private predicate nodeMayUseSummary(
+  NodeEx n, FlowState state, AccessPathApprox apa, Configuration config
+) {
+  exists(DataFlowCallable c |
+    Stage4::parameterMayFlowThrough(_, c, apa, config) and
+    nodeMayUseSummary0(n, c, state, apa, config)
+  )
+}
+
 private newtype TSummaryCtx =
   TSummaryCtxNone() or
   TSummaryCtxSome(ParamNodeEx p, FlowState state, AccessPath ap) {
@@ -3600,7 +3661,7 @@ private newtype TPathNode =
  * of dereference operations needed to get from the value in the node to the
  * tracked object. The final type indicates the type of the tracked object.
  */
-abstract private class AccessPath extends TAccessPath {
+private class AccessPath extends TAccessPath {
   /** Gets the head of this access path, if any. */
   abstract TypedContent getHead();
 
@@ -3815,11 +3876,14 @@ abstract private class PathNodeImpl extends PathNode {
   abstract NodeEx getNodeEx();
 
   predicate isHidden() {
-    hiddenNode(this.getNodeEx().asNode()) and
-    not this.isSource() and
-    not this instanceof PathNodeSink
-    or
-    this.getNodeEx() instanceof TNodeImplicitRead
+    not this.getConfiguration().includeHiddenNodes() and
+    (
+      hiddenNode(this.getNodeEx().asNode()) and
+      not this.isSource() and
+      not this instanceof PathNodeSink
+      or
+      this.getNodeEx() instanceof TNodeImplicitRead
+    )
   }
 
   private string ppAp() {

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplLocal.qll

@@ -170,6 +170,14 @@ abstract class Configuration extends string {
    */
   int explorationLimit() { none() }
 
+  /**
+   * Holds if hidden nodes should be included in the data flow graph.
+   *
+   * This feature should only be used for debugging or when the data flow graph
+   * is not visualized (for example in a `path-problem` query).
+   */
+  predicate includeHiddenNodes() { none() }
+
   /**
    * Holds if there is a partial data flow path from `source` to `node`. The
    * approximate distance between `node` and the closest source is `dist` and
@@ -1673,10 +1681,24 @@ private module Stage2 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -2495,10 +2517,24 @@ private module Stage3 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -3322,10 +3358,24 @@ private module Stage4 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -3394,17 +3444,28 @@ private Configuration unbindConf(Configuration conf) {
   exists(Configuration c | result = pragma[only_bind_into](c) and conf = pragma[only_bind_into](c))
 }
 
-private predicate nodeMayUseSummary(
-  NodeEx n, FlowState state, AccessPathApprox apa, Configuration config
+pragma[nomagic]
+private predicate nodeMayUseSummary0(
+  NodeEx n, DataFlowCallable c, FlowState state, AccessPathApprox apa, Configuration config
 ) {
-  exists(DataFlowCallable c, AccessPathApprox apa0 |
-    Stage4::parameterMayFlowThrough(_, c, apa, _) and
+  exists(AccessPathApprox apa0 |
+    Stage4::parameterMayFlowThrough(_, c, _, _) and
     Stage4::revFlow(n, state, true, _, apa0, config) and
     Stage4::fwdFlow(n, state, any(CallContextCall ccc), TAccessPathApproxSome(apa), apa0, config) and
     n.getEnclosingCallable() = c
   )
 }
 
+pragma[nomagic]
+private predicate nodeMayUseSummary(
+  NodeEx n, FlowState state, AccessPathApprox apa, Configuration config
+) {
+  exists(DataFlowCallable c |
+    Stage4::parameterMayFlowThrough(_, c, apa, config) and
+    nodeMayUseSummary0(n, c, state, apa, config)
+  )
+}
+
 private newtype TSummaryCtx =
   TSummaryCtxNone() or
   TSummaryCtxSome(ParamNodeEx p, FlowState state, AccessPath ap) {
@@ -3600,7 +3661,7 @@ private newtype TPathNode =
  * of dereference operations needed to get from the value in the node to the
  * tracked object. The final type indicates the type of the tracked object.
  */
-abstract private class AccessPath extends TAccessPath {
+private class AccessPath extends TAccessPath {
   /** Gets the head of this access path, if any. */
   abstract TypedContent getHead();
 
@@ -3815,11 +3876,14 @@ abstract private class PathNodeImpl extends PathNode {
   abstract NodeEx getNodeEx();
 
   predicate isHidden() {
-    hiddenNode(this.getNodeEx().asNode()) and
-    not this.isSource() and
-    not this instanceof PathNodeSink
-    or
-    this.getNodeEx() instanceof TNodeImplicitRead
+    not this.getConfiguration().includeHiddenNodes() and
+    (
+      hiddenNode(this.getNodeEx().asNode()) and
+      not this.isSource() and
+      not this instanceof PathNodeSink
+      or
+      this.getNodeEx() instanceof TNodeImplicitRead
+    )
   }
 
   private string ppAp() {

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl.qll

@@ -170,6 +170,14 @@ abstract class Configuration extends string {
    */
   int explorationLimit() { none() }
 
+  /**
+   * Holds if hidden nodes should be included in the data flow graph.
+   *
+   * This feature should only be used for debugging or when the data flow graph
+   * is not visualized (for example in a `path-problem` query).
+   */
+  predicate includeHiddenNodes() { none() }
+
   /**
    * Holds if there is a partial data flow path from `source` to `node`. The
    * approximate distance between `node` and the closest source is `dist` and
@@ -1673,10 +1681,24 @@ private module Stage2 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -2495,10 +2517,24 @@ private module Stage3 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -3322,10 +3358,24 @@ private module Stage4 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -3394,17 +3444,28 @@ private Configuration unbindConf(Configuration conf) {
   exists(Configuration c | result = pragma[only_bind_into](c) and conf = pragma[only_bind_into](c))
 }
 
-private predicate nodeMayUseSummary(
-  NodeEx n, FlowState state, AccessPathApprox apa, Configuration config
+pragma[nomagic]
+private predicate nodeMayUseSummary0(
+  NodeEx n, DataFlowCallable c, FlowState state, AccessPathApprox apa, Configuration config
 ) {
-  exists(DataFlowCallable c, AccessPathApprox apa0 |
-    Stage4::parameterMayFlowThrough(_, c, apa, _) and
+  exists(AccessPathApprox apa0 |
+    Stage4::parameterMayFlowThrough(_, c, _, _) and
     Stage4::revFlow(n, state, true, _, apa0, config) and
     Stage4::fwdFlow(n, state, any(CallContextCall ccc), TAccessPathApproxSome(apa), apa0, config) and
     n.getEnclosingCallable() = c
   )
 }
 
+pragma[nomagic]
+private predicate nodeMayUseSummary(
+  NodeEx n, FlowState state, AccessPathApprox apa, Configuration config
+) {
+  exists(DataFlowCallable c |
+    Stage4::parameterMayFlowThrough(_, c, apa, config) and
+    nodeMayUseSummary0(n, c, state, apa, config)
+  )
+}
+
 private newtype TSummaryCtx =
   TSummaryCtxNone() or
   TSummaryCtxSome(ParamNodeEx p, FlowState state, AccessPath ap) {
@@ -3600,7 +3661,7 @@ private newtype TPathNode =
  * of dereference operations needed to get from the value in the node to the
  * tracked object. The final type indicates the type of the tracked object.
  */
-abstract private class AccessPath extends TAccessPath {
+private class AccessPath extends TAccessPath {
   /** Gets the head of this access path, if any. */
   abstract TypedContent getHead();
 
@@ -3815,11 +3876,14 @@ abstract private class PathNodeImpl extends PathNode {
   abstract NodeEx getNodeEx();
 
   predicate isHidden() {
-    hiddenNode(this.getNodeEx().asNode()) and
-    not this.isSource() and
-    not this instanceof PathNodeSink
-    or
-    this.getNodeEx() instanceof TNodeImplicitRead
+    not this.getConfiguration().includeHiddenNodes() and
+    (
+      hiddenNode(this.getNodeEx().asNode()) and
+      not this.isSource() and
+      not this instanceof PathNodeSink
+      or
+      this.getNodeEx() instanceof TNodeImplicitRead
+    )
   }
 
   private string ppAp() {

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl2.qll

@@ -170,6 +170,14 @@ abstract class Configuration extends string {
    */
   int explorationLimit() { none() }
 
+  /**
+   * Holds if hidden nodes should be included in the data flow graph.
+   *
+   * This feature should only be used for debugging or when the data flow graph
+   * is not visualized (for example in a `path-problem` query).
+   */
+  predicate includeHiddenNodes() { none() }
+
   /**
    * Holds if there is a partial data flow path from `source` to `node`. The
    * approximate distance between `node` and the closest source is `dist` and
@@ -1673,10 +1681,24 @@ private module Stage2 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -2495,10 +2517,24 @@ private module Stage3 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -3322,10 +3358,24 @@ private module Stage4 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -3394,17 +3444,28 @@ private Configuration unbindConf(Configuration conf) {
   exists(Configuration c | result = pragma[only_bind_into](c) and conf = pragma[only_bind_into](c))
 }
 
-private predicate nodeMayUseSummary(
-  NodeEx n, FlowState state, AccessPathApprox apa, Configuration config
+pragma[nomagic]
+private predicate nodeMayUseSummary0(
+  NodeEx n, DataFlowCallable c, FlowState state, AccessPathApprox apa, Configuration config
 ) {
-  exists(DataFlowCallable c, AccessPathApprox apa0 |
-    Stage4::parameterMayFlowThrough(_, c, apa, _) and
+  exists(AccessPathApprox apa0 |
+    Stage4::parameterMayFlowThrough(_, c, _, _) and
     Stage4::revFlow(n, state, true, _, apa0, config) and
     Stage4::fwdFlow(n, state, any(CallContextCall ccc), TAccessPathApproxSome(apa), apa0, config) and
     n.getEnclosingCallable() = c
   )
 }
 
+pragma[nomagic]
+private predicate nodeMayUseSummary(
+  NodeEx n, FlowState state, AccessPathApprox apa, Configuration config
+) {
+  exists(DataFlowCallable c |
+    Stage4::parameterMayFlowThrough(_, c, apa, config) and
+    nodeMayUseSummary0(n, c, state, apa, config)
+  )
+}
+
 private newtype TSummaryCtx =
   TSummaryCtxNone() or
   TSummaryCtxSome(ParamNodeEx p, FlowState state, AccessPath ap) {
@@ -3600,7 +3661,7 @@ private newtype TPathNode =
  * of dereference operations needed to get from the value in the node to the
  * tracked object. The final type indicates the type of the tracked object.
  */
-abstract private class AccessPath extends TAccessPath {
+private class AccessPath extends TAccessPath {
   /** Gets the head of this access path, if any. */
   abstract TypedContent getHead();
 
@@ -3815,11 +3876,14 @@ abstract private class PathNodeImpl extends PathNode {
   abstract NodeEx getNodeEx();
 
   predicate isHidden() {
-    hiddenNode(this.getNodeEx().asNode()) and
-    not this.isSource() and
-    not this instanceof PathNodeSink
-    or
-    this.getNodeEx() instanceof TNodeImplicitRead
+    not this.getConfiguration().includeHiddenNodes() and
+    (
+      hiddenNode(this.getNodeEx().asNode()) and
+      not this.isSource() and
+      not this instanceof PathNodeSink
+      or
+      this.getNodeEx() instanceof TNodeImplicitRead
+    )
   }
 
   private string ppAp() {

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl3.qll

@@ -170,6 +170,14 @@ abstract class Configuration extends string {
    */
   int explorationLimit() { none() }
 
+  /**
+   * Holds if hidden nodes should be included in the data flow graph.
+   *
+   * This feature should only be used for debugging or when the data flow graph
+   * is not visualized (for example in a `path-problem` query).
+   */
+  predicate includeHiddenNodes() { none() }
+
   /**
    * Holds if there is a partial data flow path from `source` to `node`. The
    * approximate distance between `node` and the closest source is `dist` and
@@ -1673,10 +1681,24 @@ private module Stage2 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -2495,10 +2517,24 @@ private module Stage3 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -3322,10 +3358,24 @@ private module Stage4 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -3394,17 +3444,28 @@ private Configuration unbindConf(Configuration conf) {
   exists(Configuration c | result = pragma[only_bind_into](c) and conf = pragma[only_bind_into](c))
 }
 
-private predicate nodeMayUseSummary(
-  NodeEx n, FlowState state, AccessPathApprox apa, Configuration config
+pragma[nomagic]
+private predicate nodeMayUseSummary0(
+  NodeEx n, DataFlowCallable c, FlowState state, AccessPathApprox apa, Configuration config
 ) {
-  exists(DataFlowCallable c, AccessPathApprox apa0 |
-    Stage4::parameterMayFlowThrough(_, c, apa, _) and
+  exists(AccessPathApprox apa0 |
+    Stage4::parameterMayFlowThrough(_, c, _, _) and
     Stage4::revFlow(n, state, true, _, apa0, config) and
     Stage4::fwdFlow(n, state, any(CallContextCall ccc), TAccessPathApproxSome(apa), apa0, config) and
     n.getEnclosingCallable() = c
   )
 }
 
+pragma[nomagic]
+private predicate nodeMayUseSummary(
+  NodeEx n, FlowState state, AccessPathApprox apa, Configuration config
+) {
+  exists(DataFlowCallable c |
+    Stage4::parameterMayFlowThrough(_, c, apa, config) and
+    nodeMayUseSummary0(n, c, state, apa, config)
+  )
+}
+
 private newtype TSummaryCtx =
   TSummaryCtxNone() or
   TSummaryCtxSome(ParamNodeEx p, FlowState state, AccessPath ap) {
@@ -3600,7 +3661,7 @@ private newtype TPathNode =
  * of dereference operations needed to get from the value in the node to the
  * tracked object. The final type indicates the type of the tracked object.
  */
-abstract private class AccessPath extends TAccessPath {
+private class AccessPath extends TAccessPath {
   /** Gets the head of this access path, if any. */
   abstract TypedContent getHead();
 
@@ -3815,11 +3876,14 @@ abstract private class PathNodeImpl extends PathNode {
   abstract NodeEx getNodeEx();
 
   predicate isHidden() {
-    hiddenNode(this.getNodeEx().asNode()) and
-    not this.isSource() and
-    not this instanceof PathNodeSink
-    or
-    this.getNodeEx() instanceof TNodeImplicitRead
+    not this.getConfiguration().includeHiddenNodes() and
+    (
+      hiddenNode(this.getNodeEx().asNode()) and
+      not this.isSource() and
+      not this instanceof PathNodeSink
+      or
+      this.getNodeEx() instanceof TNodeImplicitRead
+    )
   }
 
   private string ppAp() {

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl4.qll

@@ -170,6 +170,14 @@ abstract class Configuration extends string {
    */
   int explorationLimit() { none() }
 
+  /**
+   * Holds if hidden nodes should be included in the data flow graph.
+   *
+   * This feature should only be used for debugging or when the data flow graph
+   * is not visualized (for example in a `path-problem` query).
+   */
+  predicate includeHiddenNodes() { none() }
+
   /**
    * Holds if there is a partial data flow path from `source` to `node`. The
    * approximate distance between `node` and the closest source is `dist` and
@@ -1673,10 +1681,24 @@ private module Stage2 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -2495,10 +2517,24 @@ private module Stage3 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -3322,10 +3358,24 @@ private module Stage4 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -3394,17 +3444,28 @@ private Configuration unbindConf(Configuration conf) {
   exists(Configuration c | result = pragma[only_bind_into](c) and conf = pragma[only_bind_into](c))
 }
 
-private predicate nodeMayUseSummary(
-  NodeEx n, FlowState state, AccessPathApprox apa, Configuration config
+pragma[nomagic]
+private predicate nodeMayUseSummary0(
+  NodeEx n, DataFlowCallable c, FlowState state, AccessPathApprox apa, Configuration config
 ) {
-  exists(DataFlowCallable c, AccessPathApprox apa0 |
-    Stage4::parameterMayFlowThrough(_, c, apa, _) and
+  exists(AccessPathApprox apa0 |
+    Stage4::parameterMayFlowThrough(_, c, _, _) and
     Stage4::revFlow(n, state, true, _, apa0, config) and
     Stage4::fwdFlow(n, state, any(CallContextCall ccc), TAccessPathApproxSome(apa), apa0, config) and
     n.getEnclosingCallable() = c
   )
 }
 
+pragma[nomagic]
+private predicate nodeMayUseSummary(
+  NodeEx n, FlowState state, AccessPathApprox apa, Configuration config
+) {
+  exists(DataFlowCallable c |
+    Stage4::parameterMayFlowThrough(_, c, apa, config) and
+    nodeMayUseSummary0(n, c, state, apa, config)
+  )
+}
+
 private newtype TSummaryCtx =
   TSummaryCtxNone() or
   TSummaryCtxSome(ParamNodeEx p, FlowState state, AccessPath ap) {
@@ -3600,7 +3661,7 @@ private newtype TPathNode =
  * of dereference operations needed to get from the value in the node to the
  * tracked object. The final type indicates the type of the tracked object.
  */
-abstract private class AccessPath extends TAccessPath {
+private class AccessPath extends TAccessPath {
   /** Gets the head of this access path, if any. */
   abstract TypedContent getHead();
 
@@ -3815,11 +3876,14 @@ abstract private class PathNodeImpl extends PathNode {
   abstract NodeEx getNodeEx();
 
   predicate isHidden() {
-    hiddenNode(this.getNodeEx().asNode()) and
-    not this.isSource() and
-    not this instanceof PathNodeSink
-    or
-    this.getNodeEx() instanceof TNodeImplicitRead
+    not this.getConfiguration().includeHiddenNodes() and
+    (
+      hiddenNode(this.getNodeEx().asNode()) and
+      not this.isSource() and
+      not this instanceof PathNodeSink
+      or
+      this.getNodeEx() instanceof TNodeImplicitRead
+    )
   }
 
   private string ppAp() {

cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/TranslatedStmt.qll

@@ -421,20 +421,36 @@ class TranslatedCatchAnyHandler extends TranslatedHandler {
 class TranslatedIfStmt extends TranslatedStmt, ConditionContext {
   override IfStmt stmt;
 
-  override Instruction getFirstInstruction() { result = getCondition().getFirstInstruction() }
+  override Instruction getFirstInstruction() {
+    if hasInitialization()
+    then result = getInitialization().getFirstInstruction()
+    else result = getFirstConditionInstruction()
+  }
 
   override TranslatedElement getChild(int id) {
-    id = 0 and result = getCondition()
+    id = 0 and result = getInitialization()
     or
-    id = 1 and result = getThen()
+    id = 1 and result = getCondition()
     or
-    id = 2 and result = getElse()
+    id = 2 and result = getThen()
+    or
+    id = 3 and result = getElse()
+  }
+
+  private predicate hasInitialization() { exists(stmt.getInitialization()) }
+
+  private TranslatedStmt getInitialization() {
+    result = getTranslatedStmt(stmt.getInitialization())
   }
 
   private TranslatedCondition getCondition() {
     result = getTranslatedCondition(stmt.getCondition().getFullyConverted())
   }
 
+  private Instruction getFirstConditionInstruction() {
+    result = getCondition().getFirstInstruction()
+  }
+
   private TranslatedStmt getThen() { result = getTranslatedStmt(stmt.getThen()) }
 
   private TranslatedStmt getElse() { result = getTranslatedStmt(stmt.getElse()) }
@@ -456,6 +472,9 @@ class TranslatedIfStmt extends TranslatedStmt, ConditionContext {
   }
 
   override Instruction getChildSuccessor(TranslatedElement child) {
+    child = getInitialization() and
+    result = getFirstConditionInstruction()
+    or
     (child = getThen() or child = getElse()) and
     result = getParent().getChildSuccessor(this)
   }
@@ -698,14 +717,28 @@ class TranslatedSwitchStmt extends TranslatedStmt {
     result = getTranslatedExpr(stmt.getExpr().getFullyConverted())
   }
 
+  private Instruction getFirstExprInstruction() { result = getExpr().getFirstInstruction() }
+
   private TranslatedStmt getBody() { result = getTranslatedStmt(stmt.getStmt()) }
 
-  override Instruction getFirstInstruction() { result = getExpr().getFirstInstruction() }
+  override Instruction getFirstInstruction() {
+    if hasInitialization()
+    then result = getInitialization().getFirstInstruction()
+    else result = getFirstExprInstruction()
+  }
 
   override TranslatedElement getChild(int id) {
-    id = 0 and result = getExpr()
+    id = 0 and result = getInitialization()
     or
-    id = 1 and result = getBody()
+    id = 1 and result = getExpr()
+    or
+    id = 2 and result = getBody()
+  }
+
+  private predicate hasInitialization() { exists(stmt.getInitialization()) }
+
+  private TranslatedStmt getInitialization() {
+    result = getTranslatedStmt(stmt.getInitialization())
   }
 
   override predicate hasInstruction(Opcode opcode, InstructionTag tag, CppType resultType) {
@@ -735,6 +768,8 @@ class TranslatedSwitchStmt extends TranslatedStmt {
   }
 
   override Instruction getChildSuccessor(TranslatedElement child) {
+    child = getInitialization() and result = getFirstExprInstruction()
+    or
     child = getExpr() and result = getInstruction(SwitchBranchTag())
     or
     child = getBody() and result = getParent().getChildSuccessor(this)

cpp/ql/lib/semmle/code/cpp/security/BufferWrite.qll

@@ -155,7 +155,7 @@ class StrCopyBW extends BufferWriteCall {
     // when result exists, it is an exact flow analysis
     reason instanceof ValueFlowAnalysis and
     result =
-      this.getArgument(this.getParamSrc()).(AnalysedString).getMaxLength() * this.getCharSize()
+      this.getArgument(this.getParamSrc()).(AnalyzedString).getMaxLength() * this.getCharSize()
   }
 
   override int getMaxData(BufferWriteEstimationReason reason) {
@@ -201,7 +201,7 @@ class StrCatBW extends BufferWriteCall {
     // when result exists, it is an exact flow analysis
     reason instanceof ValueFlowAnalysis and
     result =
-      this.getArgument(this.getParamSrc()).(AnalysedString).getMaxLength() * this.getCharSize()
+      this.getArgument(this.getParamSrc()).(AnalyzedString).getMaxLength() * this.getCharSize()
   }
 
   override int getMaxData(BufferWriteEstimationReason reason) {

cpp/ql/lib/semmle/code/cpp/stmts/Stmt.qll

@@ -213,6 +213,26 @@ class ConditionalStmt extends ControlStructure, TConditionalStmt { }
 class IfStmt extends ConditionalStmt, @stmt_if {
   override string getAPrimaryQlClass() { result = "IfStmt" }
 
+  /**
+   * Gets the initialization statement of this 'if' statement, if any.
+   *
+   * For example, for
+   * ```
+   * if (int x = y; b) { f(); }
+   * ```
+   * the result is `int x = y;`.
+   *
+   * Does not hold if the initialization statement is missing or an empty statement, as in
+   * ```
+   * if (b) { f(); }
+   * ```
+   * or
+   * ```
+   * if (; b) { f(); }
+   * ```
+   */
+  Stmt getInitialization() { if_initialization(underlyingElement(this), unresolveElement(result)) }
+
   /**
    * Gets the condition expression of this 'if' statement.
    *
@@ -222,7 +242,7 @@ class IfStmt extends ConditionalStmt, @stmt_if {
    * ```
    * the result is `b`.
    */
-  Expr getCondition() { result = this.getChild(0) }
+  Expr getCondition() { result = this.getChild(1) }
 
   override Expr getControllingExpr() { result = this.getCondition() }
 
@@ -299,6 +319,28 @@ class IfStmt extends ConditionalStmt, @stmt_if {
 class ConstexprIfStmt extends ConditionalStmt, @stmt_constexpr_if {
   override string getAPrimaryQlClass() { result = "ConstexprIfStmt" }
 
+  /**
+   * Gets the initialization statement of this 'constexpr if' statement, if any.
+   *
+   * For example, for
+   * ```
+   * if constexpr (int x = y; b) { f(); }
+   * ```
+   * the result is `int x = y;`.
+   *
+   * Does not hold if the initialization statement is missing or an empty statement, as in
+   * ```
+   * if constexpr (b) { f(); }
+   * ```
+   * or
+   * ```
+   * if constexpr (; b) { f(); }
+   * ```
+   */
+  Stmt getInitialization() {
+    constexpr_if_initialization(underlyingElement(this), unresolveElement(result))
+  }
+
   /**
    * Gets the condition expression of this 'constexpr if' statement.
    *
@@ -308,7 +350,7 @@ class ConstexprIfStmt extends ConditionalStmt, @stmt_constexpr_if {
    * ```
    * the result is `b`.
    */
-  Expr getCondition() { result = this.getChild(0) }
+  Expr getCondition() { result = this.getChild(1) }
 
   override Expr getControllingExpr() { result = this.getCondition() }
 
@@ -926,7 +968,7 @@ class ForStmt extends Loop, @stmt_for {
    *
    * Does not hold if the initialization statement is an empty statement, as in
    * ```
-   * for (; i < 10; i++) { j++ }
+   * for (; i < 10; i++) { j++; }
    * ```
    */
   Stmt getInitialization() { for_initialization(underlyingElement(this), unresolveElement(result)) }
@@ -1470,6 +1512,28 @@ class DefaultCase extends SwitchCase {
 class SwitchStmt extends ConditionalStmt, @stmt_switch {
   override string getAPrimaryQlClass() { result = "SwitchStmt" }
 
+  /**
+   * Gets the initialization statement of this 'switch' statement, if any.
+   *
+   * For example, for
+   * ```
+   * switch (int x = y; b) { }
+   * ```
+   * the result is `int x = y;`.
+   *
+   * Does not hold if the initialization statement is missing or an empty statement, as in
+   * ```
+   * switch (b) { }
+   * ```
+   * or
+   * ```
+   * switch (; b) { }
+   * ```
+   */
+  Stmt getInitialization() {
+    switch_initialization(underlyingElement(this), unresolveElement(result))
+  }
+
   /**
    * Gets the expression that this 'switch' statement switches on.
    *
@@ -1485,7 +1549,7 @@ class SwitchStmt extends ConditionalStmt, @stmt_switch {
    * ```
    * the result is `i`.
    */
-  Expr getExpr() { result = this.getChild(0) }
+  Expr getExpr() { result = this.getChild(1) }
 
   override Expr getControllingExpr() { result = this.getExpr() }
 

cpp/ql/lib/semmlecode.cpp.dbscheme

@@ -1863,6 +1863,11 @@ variable_vla(
     int decl: @stmt_vla_decl ref
 );
 
+if_initialization(
+    unique int if_stmt: @stmt_if ref,
+    int init_id: @stmt ref
+);
+
 if_then(
     unique int if_stmt: @stmt_if ref,
     int then_id: @stmt ref
@@ -1873,6 +1878,11 @@ if_else(
     int else_id: @stmt ref
 );
 
+constexpr_if_initialization(
+    unique int constexpr_if_stmt: @stmt_constexpr_if ref,
+    int init_id: @stmt ref
+);
+
 constexpr_if_then(
     unique int constexpr_if_stmt: @stmt_constexpr_if ref,
     int then_id: @stmt ref
@@ -1893,6 +1903,11 @@ do_body(
     int body_id: @stmt ref
 );
 
+switch_initialization(
+    unique int switch_stmt: @stmt_switch ref,
+    int init_id: @stmt ref
+);
+
 #keyset[switch_stmt, index]
 switch_case(
     int switch_stmt: @stmt_switch ref,

cpp/ql/lib/upgrades/e9a518baf14f4322ac243578a8e1391386ff030f/exprparents.ql

@@ -0,0 +1,21 @@
+class Element extends @element {
+  string toString() { none() }
+}
+
+class Expr extends @expr {
+  string toString() { none() }
+}
+
+class Stmt extends @stmt {
+  string toString() { none() }
+}
+
+predicate isStmtWithInitializer(Stmt stmt) {
+  exists(int kind | stmts(stmt, kind, _) | kind = 2 or kind = 11 or kind = 35)
+}
+
+from Expr child, int index, int index_new, Element parent
+where
+  exprparents(child, index, parent) and
+  if isStmtWithInitializer(parent) then index_new = index + 1 else index_new = index
+select child, index_new, parent

cpp/ql/lib/upgrades/e9a518baf14f4322ac243578a8e1391386ff030f/stmtparents.ql

@@ -0,0 +1,17 @@
+class Element extends @element {
+  string toString() { none() }
+}
+
+class Stmt extends @stmt {
+  string toString() { none() }
+}
+
+predicate isStmtWithInitializer(Stmt stmt) {
+  exists(int kind | stmts(stmt, kind, _) | kind = 2 or kind = 11 or kind = 35)
+}
+
+from Stmt child, int index, int index_new, Element parent
+where
+  stmtparents(child, index, parent) and
+  if isStmtWithInitializer(parent) then index_new = index + 1 else index_new = index
+select child, index_new, parent

cpp/ql/lib/upgrades/e9a518baf14f4322ac243578a8e1391386ff030f/upgrade.properties

@@ -0,0 +1,4 @@
+description: Support C++17 if and switch initializers
+compatibility: partial
+exprparents.rel: run exprparents.qlo
+stmtparents.rel: run stmtparents.qlo

cpp/ql/src/Best Practices/Unused Entities/UnusedLocals.ql

@@ -57,6 +57,5 @@ where
   not declarationHasSideEffects(v) and
   not exists(AsmStmt s | f = s.getEnclosingFunction()) and
   not v.getAnAttribute().getName() = "unused" and
-  not any(ErrorExpr e).getEnclosingFunction() = f and // unextracted expr may use `v`
-  not any(ConditionDeclExpr cde).getEnclosingFunction() = f // this case can be removed when the `if (a = b; a)` and `switch (a = b; a)` test cases don't depend on this exclusion
+  not any(ErrorExpr e).getEnclosingFunction() = f // unextracted expr may use `v`
 select v, "Variable " + v.getName() + " is not used"

cpp/ql/src/Security/CWE/CWE-611/XXE.ql

@@ -57,11 +57,25 @@ class XercesDOMParserClass extends Class {
   XercesDOMParserClass() { this.hasName("XercesDOMParser") }
 }
 
+/**
+ * The `DOMLSParser` class.
+ */
+class DomLSParserClass extends Class {
+  DomLSParserClass() { this.hasName("DOMLSParser") }
+}
+
 /**
  * The `SAXParser` class.
  */
-class SAXParserClass extends Class {
-  SAXParserClass() { this.hasName("SAXParser") }
+class SaxParserClass extends Class {
+  SaxParserClass() { this.hasName("SAXParser") }
+}
+
+/**
+ * The `SAX2XMLReader` class.
+ */
+class Sax2XmlReader extends Class {
+  Sax2XmlReader() { this.hasName("SAX2XMLReader") }
 }
 
 /**
@@ -113,7 +127,7 @@ class DisableDefaultEntityResolutionTranformer extends XXEFlowStateTranformer {
       call.getTarget() = f and
       (
         f.getDeclaringType() instanceof AbstractDOMParserClass or
-        f.getDeclaringType() instanceof SAXParserClass
+        f.getDeclaringType() instanceof SaxParserClass
       ) and
       f.hasName("setDisableDefaultEntityResolution") and
       this = call.getQualifier() and
@@ -146,8 +160,7 @@ class CreateEntityReferenceNodesTranformer extends XXEFlowStateTranformer {
   CreateEntityReferenceNodesTranformer() {
     exists(Call call, Function f |
       call.getTarget() = f and
-      f.getDeclaringType() instanceof AbstractDOMParserClass and
-      f.hasName("setCreateEntityReferenceNodes") and
+      f.getClassAndName("setCreateEntityReferenceNodes") instanceof AbstractDOMParserClass and
       this = call.getQualifier() and
       newValue = call.getArgument(0)
     )
@@ -168,12 +181,116 @@ class CreateEntityReferenceNodesTranformer extends XXEFlowStateTranformer {
 }
 
 /**
- * The `AbstractDOMParser.parse` or `SAXParser.parse` method.
+ * The `XMLUni.fgXercesDisableDefaultEntityResolution` constant.
+ */
+class FeatureDisableDefaultEntityResolution extends Variable {
+  FeatureDisableDefaultEntityResolution() {
+    this.getName() = "fgXercesDisableDefaultEntityResolution" and
+    this.getDeclaringType().getName() = "XMLUni"
+  }
+}
+
+/**
+ * A flow state transformer for a call to `SAX2XMLReader.setFeature`
+ * specifying the feature `XMLUni::fgXercesDisableDefaultEntityResolution`.
+ * Transforms the flow state through the qualifier according to this setting.
+ */
+class SetFeatureTranformer extends XXEFlowStateTranformer {
+  Expr newValue;
+
+  SetFeatureTranformer() {
+    exists(Call call, Function f |
+      call.getTarget() = f and
+      f.getClassAndName("setFeature") instanceof Sax2XmlReader and
+      this = call.getQualifier() and
+      globalValueNumber(call.getArgument(0)).getAnExpr().(VariableAccess).getTarget() instanceof
+        FeatureDisableDefaultEntityResolution and
+      newValue = call.getArgument(1)
+    )
+  }
+
+  final override XXEFlowState transform(XXEFlowState flowstate) {
+    exists(int createEntityReferenceNodes |
+      encodeXercesFlowState(flowstate, _, createEntityReferenceNodes) and
+      (
+        globalValueNumber(newValue).getAnExpr().getValue().toInt() = 1 and // true
+        encodeXercesFlowState(result, 1, createEntityReferenceNodes)
+        or
+        not globalValueNumber(newValue).getAnExpr().getValue().toInt() = 1 and // false or unknown
+        encodeXercesFlowState(result, 0, createEntityReferenceNodes)
+      )
+    )
+  }
+}
+
+/**
+ * The `DOMLSParser.getDomConfig` function.
+ */
+class GetDomConfig extends Function {
+  GetDomConfig() { this.getClassAndName("getDomConfig") instanceof DomLSParserClass }
+}
+
+/**
+ * The `DOMConfiguration.setParameter` function.
+ */
+class DomConfigurationSetParameter extends Function {
+  DomConfigurationSetParameter() {
+    this.getClassAndName("setParameter").getName() = "DOMConfiguration"
+  }
+}
+
+/**
+ * A flow state transformer for a call to `DOMConfiguration.setParameter`
+ * specifying the feature `XMLUni::fgXercesDisableDefaultEntityResolution`.
+ * This is a slightly more complex transformer because the qualifier is a
+ * `DOMConfiguration` pointer returned by `DOMLSParser.getDomConfig` - and it
+ * is *that* qualifier we want to transform the flow state of.
+ */
+class DomConfigurationSetParameterTranformer extends XXEFlowStateTranformer {
+  Expr newValue;
+
+  DomConfigurationSetParameterTranformer() {
+    exists(FunctionCall getDomConfigCall, FunctionCall setParameterCall |
+      // this is the qualifier of a call to `DOMLSParser.getDomConfig`.
+      getDomConfigCall.getTarget() instanceof GetDomConfig and
+      this = getDomConfigCall.getQualifier() and
+      // `setParameterCall` is a call to `setParameter` on the return value of
+      // the same call to `DOMLSParser.getDomConfig`.
+      setParameterCall.getTarget() instanceof DomConfigurationSetParameter and
+      globalValueNumber(setParameterCall.getQualifier()).getAnExpr() = getDomConfigCall and
+      // the parameter being set is
+      // `XMLUni::fgXercesDisableDefaultEntityResolution`.
+      globalValueNumber(setParameterCall.getArgument(0)).getAnExpr().(VariableAccess).getTarget()
+        instanceof FeatureDisableDefaultEntityResolution and
+      // the value being set is `newValue`.
+      newValue = setParameterCall.getArgument(1)
+    )
+  }
+
+  final override XXEFlowState transform(XXEFlowState flowstate) {
+    exists(int createEntityReferenceNodes |
+      encodeXercesFlowState(flowstate, _, createEntityReferenceNodes) and
+      (
+        globalValueNumber(newValue).getAnExpr().getValue().toInt() = 1 and // true
+        encodeXercesFlowState(result, 1, createEntityReferenceNodes)
+        or
+        not globalValueNumber(newValue).getAnExpr().getValue().toInt() = 1 and // false or unknown
+        encodeXercesFlowState(result, 0, createEntityReferenceNodes)
+      )
+    )
+  }
+}
+
+/**
+ * The `AbstractDOMParser.parse`, `DOMLSParserClass.parse`, `SAXParser.parse`
+ * or `SAX2XMLReader.parse` method.
  */
 class ParseFunction extends Function {
   ParseFunction() {
     this.getClassAndName("parse") instanceof AbstractDOMParserClass or
-    this.getClassAndName("parse") instanceof SAXParserClass
+    this.getClassAndName("parse") instanceof DomLSParserClass or
+    this.getClassAndName("parse") instanceof SaxParserClass or
+    this.getClassAndName("parse") instanceof Sax2XmlReader
   }
 }
 
@@ -184,7 +301,18 @@ class ParseFunction extends Function {
 class CreateLSParser extends Function {
   CreateLSParser() {
     this.hasName("createLSParser") and
-    this.getUnspecifiedType().(PointerType).getBaseType().getName() = "DOMLSParser" // returns a `DOMLSParser *`.
+    this.getUnspecifiedType().(PointerType).getBaseType() instanceof DomLSParserClass // returns a `DOMLSParser *`.
+  }
+}
+
+/**
+ * The `createXMLReader` function that returns a newly created `SAX2XMLReader`
+ * object.
+ */
+class CreateXmlReader extends Function {
+  CreateXmlReader() {
+    this.hasName("createXMLReader") and
+    this.getUnspecifiedType().(PointerType).getBaseType() instanceof Sax2XmlReader // returns a `SAX2XMLReader *`.
   }
 }
 
@@ -250,12 +378,19 @@ class XXEConfiguration extends DataFlow::Configuration {
     // source is the write on `this` of a call to the `SAXParser`
     // constructor.
     exists(CallInstruction call |
-      call.getStaticCallTarget() = any(SAXParserClass c).getAConstructor() and
+      call.getStaticCallTarget() = any(SaxParserClass c).getAConstructor() and
       node.asInstruction().(WriteSideEffectInstruction).getDestinationAddress() =
         call.getThisArgument() and
       encodeXercesFlowState(flowstate, 0, 1) // default configuration
     )
     or
+    // source is the result of a call to `createXMLReader`.
+    exists(Call call |
+      call.getTarget() instanceof CreateXmlReader and
+      call = node.asExpr() and
+      encodeXercesFlowState(flowstate, 0, 1) // default configuration
+    )
+    or
     // source is an `options` argument on a `libxml2` parse call that specifies
     // at least one unsafe option.
     //

cpp/ql/src/change-notes/2022-04-12-unused-local-variable.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* The `cpp/unused-local-variable` no longer ignores functions that include `if` and `switch` statements with C++17-style initializers.

cpp/ql/test/library-tests/ir/ir/PrintAST.expected

@@ -13559,6 +13559,422 @@ ir.cpp:
 # 1754|                   Type = [SpecifiedType] const CopyConstructorTestVirtualClass
 # 1754|                   ValueCategory = lvalue
 # 1755|     getStmt(2): [ReturnStmt] return ...
+# 1757| [TopLevelFunction] void if_initialization(int)
+# 1757|   <params>: 
+# 1757|     getParameter(0): [Parameter] x
+# 1757|         Type = [IntType] int
+# 1757|   getEntryPoint(): [BlockStmt] { ... }
+# 1758|     getStmt(0): [IfStmt] if (...) ... 
+# 1758|       getInitialization(): [DeclStmt] declaration
+# 1758|         getDeclarationEntry(0): [VariableDeclarationEntry] definition of y
+# 1758|             Type = [IntType] int
+# 1758|           getVariable().getInitializer(): [Initializer] initializer for y
+# 1758|             getExpr(): [VariableAccess] x
+# 1758|                 Type = [IntType] int
+# 1758|                 ValueCategory = prvalue(load)
+# 1758|       getCondition(): [AddExpr] ... + ...
+# 1758|           Type = [IntType] int
+# 1758|           ValueCategory = prvalue
+# 1758|         getLeftOperand(): [VariableAccess] x
+# 1758|             Type = [IntType] int
+# 1758|             ValueCategory = prvalue(load)
+# 1758|         getRightOperand(): [Literal] 1
+# 1758|             Type = [IntType] int
+# 1758|             Value = [Literal] 1
+# 1758|             ValueCategory = prvalue
+# 1758|       getThen(): [BlockStmt] { ... }
+# 1759|         getStmt(0): [ExprStmt] ExprStmt
+# 1759|           getExpr(): [AssignExpr] ... = ...
+# 1759|               Type = [IntType] int
+# 1759|               ValueCategory = lvalue
+# 1759|             getLValue(): [VariableAccess] x
+# 1759|                 Type = [IntType] int
+# 1759|                 ValueCategory = lvalue
+# 1759|             getRValue(): [AddExpr] ... + ...
+# 1759|                 Type = [IntType] int
+# 1759|                 ValueCategory = prvalue
+# 1759|               getLeftOperand(): [VariableAccess] x
+# 1759|                   Type = [IntType] int
+# 1759|                   ValueCategory = prvalue(load)
+# 1759|               getRightOperand(): [VariableAccess] y
+# 1759|                   Type = [IntType] int
+# 1759|                   ValueCategory = prvalue(load)
+# 1758|       getCondition().getFullyConverted(): [CStyleCast] (bool)...
+# 1758|           Conversion = [BoolConversion] conversion to bool
+# 1758|           Type = [BoolType] bool
+# 1758|           ValueCategory = prvalue
+# 1762|     getStmt(1): [DeclStmt] declaration
+# 1762|       getDeclarationEntry(0): [VariableDeclarationEntry] definition of w
+# 1762|           Type = [IntType] int
+# 1763|     getStmt(2): [IfStmt] if (...) ... 
+# 1763|       getInitialization(): [ExprStmt] ExprStmt
+# 1763|         getExpr(): [AssignExpr] ... = ...
+# 1763|             Type = [IntType] int
+# 1763|             ValueCategory = lvalue
+# 1763|           getLValue(): [VariableAccess] w
+# 1763|               Type = [IntType] int
+# 1763|               ValueCategory = lvalue
+# 1763|           getRValue(): [VariableAccess] x
+# 1763|               Type = [IntType] int
+# 1763|               ValueCategory = prvalue(load)
+# 1763|       getCondition(): [AddExpr] ... + ...
+# 1763|           Type = [IntType] int
+# 1763|           ValueCategory = prvalue
+# 1763|         getLeftOperand(): [VariableAccess] x
+# 1763|             Type = [IntType] int
+# 1763|             ValueCategory = prvalue(load)
+# 1763|         getRightOperand(): [Literal] 1
+# 1763|             Type = [IntType] int
+# 1763|             Value = [Literal] 1
+# 1763|             ValueCategory = prvalue
+# 1763|       getThen(): [BlockStmt] { ... }
+# 1764|         getStmt(0): [ExprStmt] ExprStmt
+# 1764|           getExpr(): [AssignExpr] ... = ...
+# 1764|               Type = [IntType] int
+# 1764|               ValueCategory = lvalue
+# 1764|             getLValue(): [VariableAccess] x
+# 1764|                 Type = [IntType] int
+# 1764|                 ValueCategory = lvalue
+# 1764|             getRValue(): [AddExpr] ... + ...
+# 1764|                 Type = [IntType] int
+# 1764|                 ValueCategory = prvalue
+# 1764|               getLeftOperand(): [VariableAccess] x
+# 1764|                   Type = [IntType] int
+# 1764|                   ValueCategory = prvalue(load)
+# 1764|               getRightOperand(): [VariableAccess] w
+# 1764|                   Type = [IntType] int
+# 1764|                   ValueCategory = prvalue(load)
+# 1763|       getCondition().getFullyConverted(): [CStyleCast] (bool)...
+# 1763|           Conversion = [BoolConversion] conversion to bool
+# 1763|           Type = [BoolType] bool
+# 1763|           ValueCategory = prvalue
+# 1767|     getStmt(3): [IfStmt] if (...) ... 
+# 1767|       getInitialization(): [ExprStmt] ExprStmt
+# 1767|         getExpr(): [AssignExpr] ... = ...
+# 1767|             Type = [IntType] int
+# 1767|             ValueCategory = lvalue
+# 1767|           getLValue(): [VariableAccess] w
+# 1767|               Type = [IntType] int
+# 1767|               ValueCategory = lvalue
+# 1767|           getRValue(): [VariableAccess] x
+# 1767|               Type = [IntType] int
+# 1767|               ValueCategory = prvalue(load)
+# 1767|       getCondition(): [ConditionDeclExpr] (condition decl)
+# 1767|           Type = [BoolType] bool
+# 1767|           ValueCategory = prvalue
+# 1767|         getVariableAccess(): [VariableAccess] w2
+# 1767|             Type = [IntType] int
+# 1767|             ValueCategory = prvalue(load)
+# 1767|         getVariableAccess().getFullyConverted(): [CStyleCast] (bool)...
+# 1767|             Conversion = [BoolConversion] conversion to bool
+# 1767|             Type = [BoolType] bool
+# 1767|             ValueCategory = prvalue
+# 1767|       getThen(): [BlockStmt] { ... }
+# 1768|         getStmt(0): [ExprStmt] ExprStmt
+# 1768|           getExpr(): [AssignExpr] ... = ...
+# 1768|               Type = [IntType] int
+# 1768|               ValueCategory = lvalue
+# 1768|             getLValue(): [VariableAccess] x
+# 1768|                 Type = [IntType] int
+# 1768|                 ValueCategory = lvalue
+# 1768|             getRValue(): [AddExpr] ... + ...
+# 1768|                 Type = [IntType] int
+# 1768|                 ValueCategory = prvalue
+# 1768|               getLeftOperand(): [VariableAccess] x
+# 1768|                   Type = [IntType] int
+# 1768|                   ValueCategory = prvalue(load)
+# 1768|               getRightOperand(): [VariableAccess] w
+# 1768|                   Type = [IntType] int
+# 1768|                   ValueCategory = prvalue(load)
+# 1771|     getStmt(4): [IfStmt] if (...) ... 
+# 1771|       getInitialization(): [DeclStmt] declaration
+# 1771|         getDeclarationEntry(0): [VariableDeclarationEntry] definition of v
+# 1771|             Type = [IntType] int
+# 1771|           getVariable().getInitializer(): [Initializer] initializer for v
+# 1771|             getExpr(): [VariableAccess] x
+# 1771|                 Type = [IntType] int
+# 1771|                 ValueCategory = prvalue(load)
+# 1771|       getCondition(): [ConditionDeclExpr] (condition decl)
+# 1771|           Type = [BoolType] bool
+# 1771|           ValueCategory = prvalue
+# 1771|         getVariableAccess(): [VariableAccess] v2
+# 1771|             Type = [IntType] int
+# 1771|             ValueCategory = prvalue(load)
+# 1771|         getVariableAccess().getFullyConverted(): [CStyleCast] (bool)...
+# 1771|             Conversion = [BoolConversion] conversion to bool
+# 1771|             Type = [BoolType] bool
+# 1771|             ValueCategory = prvalue
+# 1771|       getThen(): [BlockStmt] { ... }
+# 1772|         getStmt(0): [ExprStmt] ExprStmt
+# 1772|           getExpr(): [AssignExpr] ... = ...
+# 1772|               Type = [IntType] int
+# 1772|               ValueCategory = lvalue
+# 1772|             getLValue(): [VariableAccess] x
+# 1772|                 Type = [IntType] int
+# 1772|                 ValueCategory = lvalue
+# 1772|             getRValue(): [AddExpr] ... + ...
+# 1772|                 Type = [IntType] int
+# 1772|                 ValueCategory = prvalue
+# 1772|               getLeftOperand(): [VariableAccess] x
+# 1772|                   Type = [IntType] int
+# 1772|                   ValueCategory = prvalue(load)
+# 1772|               getRightOperand(): [VariableAccess] v
+# 1772|                   Type = [IntType] int
+# 1772|                   ValueCategory = prvalue(load)
+# 1775|     getStmt(5): [DeclStmt] declaration
+# 1775|       getDeclarationEntry(0): [VariableDeclarationEntry] definition of z
+# 1775|           Type = [IntType] int
+# 1775|         getVariable().getInitializer(): [Initializer] initializer for z
+# 1775|           getExpr(): [VariableAccess] x
+# 1775|               Type = [IntType] int
+# 1775|               ValueCategory = prvalue(load)
+# 1776|     getStmt(6): [IfStmt] if (...) ... 
+# 1776|       getCondition(): [VariableAccess] z
+# 1776|           Type = [IntType] int
+# 1776|           ValueCategory = prvalue(load)
+# 1776|       getThen(): [BlockStmt] { ... }
+# 1777|         getStmt(0): [ExprStmt] ExprStmt
+# 1777|           getExpr(): [AssignExpr] ... = ...
+# 1777|               Type = [IntType] int
+# 1777|               ValueCategory = lvalue
+# 1777|             getLValue(): [VariableAccess] x
+# 1777|                 Type = [IntType] int
+# 1777|                 ValueCategory = lvalue
+# 1777|             getRValue(): [AddExpr] ... + ...
+# 1777|                 Type = [IntType] int
+# 1777|                 ValueCategory = prvalue
+# 1777|               getLeftOperand(): [VariableAccess] x
+# 1777|                   Type = [IntType] int
+# 1777|                   ValueCategory = prvalue(load)
+# 1777|               getRightOperand(): [VariableAccess] z
+# 1777|                   Type = [IntType] int
+# 1777|                   ValueCategory = prvalue(load)
+# 1776|       getCondition().getFullyConverted(): [CStyleCast] (bool)...
+# 1776|           Conversion = [BoolConversion] conversion to bool
+# 1776|           Type = [BoolType] bool
+# 1776|           ValueCategory = prvalue
+# 1780|     getStmt(7): [IfStmt] if (...) ... 
+# 1780|       getCondition(): [ConditionDeclExpr] (condition decl)
+# 1780|           Type = [BoolType] bool
+# 1780|           ValueCategory = prvalue
+# 1780|         getVariableAccess(): [VariableAccess] z2
+# 1780|             Type = [IntType] int
+# 1780|             ValueCategory = prvalue(load)
+# 1780|         getVariableAccess().getFullyConverted(): [CStyleCast] (bool)...
+# 1780|             Conversion = [BoolConversion] conversion to bool
+# 1780|             Type = [BoolType] bool
+# 1780|             ValueCategory = prvalue
+# 1780|       getThen(): [BlockStmt] { ... }
+# 1781|         getStmt(0): [ExprStmt] ExprStmt
+# 1781|           getExpr(): [AssignAddExpr] ... += ...
+# 1781|               Type = [IntType] int
+# 1781|               ValueCategory = lvalue
+# 1781|             getLValue(): [VariableAccess] x
+# 1781|                 Type = [IntType] int
+# 1781|                 ValueCategory = lvalue
+# 1781|             getRValue(): [VariableAccess] z2
+# 1781|                 Type = [IntType] int
+# 1781|                 ValueCategory = prvalue(load)
+# 1783|     getStmt(8): [ReturnStmt] return ...
+# 1785| [TopLevelFunction] void switch_initialization(int)
+# 1785|   <params>: 
+# 1785|     getParameter(0): [Parameter] x
+# 1785|         Type = [IntType] int
+# 1785|   getEntryPoint(): [BlockStmt] { ... }
+# 1786|     getStmt(0): [SwitchStmt] switch (...) ... 
+# 1786|       getInitialization(): [DeclStmt] declaration
+# 1786|         getDeclarationEntry(0): [VariableDeclarationEntry] definition of y
+# 1786|             Type = [IntType] int
+# 1786|           getVariable().getInitializer(): [Initializer] initializer for y
+# 1786|             getExpr(): [VariableAccess] x
+# 1786|                 Type = [IntType] int
+# 1786|                 ValueCategory = prvalue(load)
+# 1786|       getExpr(): [AddExpr] ... + ...
+# 1786|           Type = [IntType] int
+# 1786|           ValueCategory = prvalue
+# 1786|         getLeftOperand(): [VariableAccess] x
+# 1786|             Type = [IntType] int
+# 1786|             ValueCategory = prvalue(load)
+# 1786|         getRightOperand(): [Literal] 1
+# 1786|             Type = [IntType] int
+# 1786|             Value = [Literal] 1
+# 1786|             ValueCategory = prvalue
+# 1786|       getStmt(): [BlockStmt] { ... }
+# 1787|         getStmt(0): [SwitchCase] default: 
+# 1788|         getStmt(1): [ExprStmt] ExprStmt
+# 1788|           getExpr(): [AssignExpr] ... = ...
+# 1788|               Type = [IntType] int
+# 1788|               ValueCategory = lvalue
+# 1788|             getLValue(): [VariableAccess] x
+# 1788|                 Type = [IntType] int
+# 1788|                 ValueCategory = lvalue
+# 1788|             getRValue(): [AddExpr] ... + ...
+# 1788|                 Type = [IntType] int
+# 1788|                 ValueCategory = prvalue
+# 1788|               getLeftOperand(): [VariableAccess] x
+# 1788|                   Type = [IntType] int
+# 1788|                   ValueCategory = prvalue(load)
+# 1788|               getRightOperand(): [VariableAccess] y
+# 1788|                   Type = [IntType] int
+# 1788|                   ValueCategory = prvalue(load)
+# 1791|     getStmt(1): [DeclStmt] declaration
+# 1791|       getDeclarationEntry(0): [VariableDeclarationEntry] definition of w
+# 1791|           Type = [IntType] int
+# 1792|     getStmt(2): [SwitchStmt] switch (...) ... 
+# 1792|       getInitialization(): [ExprStmt] ExprStmt
+# 1792|         getExpr(): [AssignExpr] ... = ...
+# 1792|             Type = [IntType] int
+# 1792|             ValueCategory = lvalue
+# 1792|           getLValue(): [VariableAccess] w
+# 1792|               Type = [IntType] int
+# 1792|               ValueCategory = lvalue
+# 1792|           getRValue(): [VariableAccess] x
+# 1792|               Type = [IntType] int
+# 1792|               ValueCategory = prvalue(load)
+# 1792|       getExpr(): [AddExpr] ... + ...
+# 1792|           Type = [IntType] int
+# 1792|           ValueCategory = prvalue
+# 1792|         getLeftOperand(): [VariableAccess] x
+# 1792|             Type = [IntType] int
+# 1792|             ValueCategory = prvalue(load)
+# 1792|         getRightOperand(): [Literal] 1
+# 1792|             Type = [IntType] int
+# 1792|             Value = [Literal] 1
+# 1792|             ValueCategory = prvalue
+# 1792|       getStmt(): [BlockStmt] { ... }
+# 1793|         getStmt(0): [SwitchCase] default: 
+# 1794|         getStmt(1): [ExprStmt] ExprStmt
+# 1794|           getExpr(): [AssignExpr] ... = ...
+# 1794|               Type = [IntType] int
+# 1794|               ValueCategory = lvalue
+# 1794|             getLValue(): [VariableAccess] x
+# 1794|                 Type = [IntType] int
+# 1794|                 ValueCategory = lvalue
+# 1794|             getRValue(): [AddExpr] ... + ...
+# 1794|                 Type = [IntType] int
+# 1794|                 ValueCategory = prvalue
+# 1794|               getLeftOperand(): [VariableAccess] x
+# 1794|                   Type = [IntType] int
+# 1794|                   ValueCategory = prvalue(load)
+# 1794|               getRightOperand(): [VariableAccess] w
+# 1794|                   Type = [IntType] int
+# 1794|                   ValueCategory = prvalue(load)
+# 1797|     getStmt(3): [SwitchStmt] switch (...) ... 
+# 1797|       getInitialization(): [ExprStmt] ExprStmt
+# 1797|         getExpr(): [AssignExpr] ... = ...
+# 1797|             Type = [IntType] int
+# 1797|             ValueCategory = lvalue
+# 1797|           getLValue(): [VariableAccess] w
+# 1797|               Type = [IntType] int
+# 1797|               ValueCategory = lvalue
+# 1797|           getRValue(): [VariableAccess] x
+# 1797|               Type = [IntType] int
+# 1797|               ValueCategory = prvalue(load)
+# 1797|       getExpr(): [ConditionDeclExpr] (condition decl)
+# 1797|           Type = [IntType] int
+# 1797|           ValueCategory = prvalue
+# 1797|         getVariableAccess(): [VariableAccess] w2
+# 1797|             Type = [IntType] int
+# 1797|             ValueCategory = prvalue(load)
+# 1797|       getStmt(): [BlockStmt] { ... }
+# 1798|         getStmt(0): [SwitchCase] default: 
+# 1799|         getStmt(1): [ExprStmt] ExprStmt
+# 1799|           getExpr(): [AssignExpr] ... = ...
+# 1799|               Type = [IntType] int
+# 1799|               ValueCategory = lvalue
+# 1799|             getLValue(): [VariableAccess] x
+# 1799|                 Type = [IntType] int
+# 1799|                 ValueCategory = lvalue
+# 1799|             getRValue(): [AddExpr] ... + ...
+# 1799|                 Type = [IntType] int
+# 1799|                 ValueCategory = prvalue
+# 1799|               getLeftOperand(): [VariableAccess] x
+# 1799|                   Type = [IntType] int
+# 1799|                   ValueCategory = prvalue(load)
+# 1799|               getRightOperand(): [VariableAccess] w
+# 1799|                   Type = [IntType] int
+# 1799|                   ValueCategory = prvalue(load)
+# 1802|     getStmt(4): [SwitchStmt] switch (...) ... 
+# 1802|       getInitialization(): [DeclStmt] declaration
+# 1802|         getDeclarationEntry(0): [VariableDeclarationEntry] definition of v
+# 1802|             Type = [IntType] int
+# 1802|           getVariable().getInitializer(): [Initializer] initializer for v
+# 1802|             getExpr(): [VariableAccess] x
+# 1802|                 Type = [IntType] int
+# 1802|                 ValueCategory = prvalue(load)
+# 1802|       getExpr(): [ConditionDeclExpr] (condition decl)
+# 1802|           Type = [IntType] int
+# 1802|           ValueCategory = prvalue
+# 1802|         getVariableAccess(): [VariableAccess] v2
+# 1802|             Type = [IntType] int
+# 1802|             ValueCategory = prvalue(load)
+# 1802|       getStmt(): [BlockStmt] { ... }
+# 1803|         getStmt(0): [SwitchCase] default: 
+# 1804|         getStmt(1): [ExprStmt] ExprStmt
+# 1804|           getExpr(): [AssignExpr] ... = ...
+# 1804|               Type = [IntType] int
+# 1804|               ValueCategory = lvalue
+# 1804|             getLValue(): [VariableAccess] x
+# 1804|                 Type = [IntType] int
+# 1804|                 ValueCategory = lvalue
+# 1804|             getRValue(): [AddExpr] ... + ...
+# 1804|                 Type = [IntType] int
+# 1804|                 ValueCategory = prvalue
+# 1804|               getLeftOperand(): [VariableAccess] x
+# 1804|                   Type = [IntType] int
+# 1804|                   ValueCategory = prvalue(load)
+# 1804|               getRightOperand(): [VariableAccess] v
+# 1804|                   Type = [IntType] int
+# 1804|                   ValueCategory = prvalue(load)
+# 1807|     getStmt(5): [DeclStmt] declaration
+# 1807|       getDeclarationEntry(0): [VariableDeclarationEntry] definition of z
+# 1807|           Type = [IntType] int
+# 1807|         getVariable().getInitializer(): [Initializer] initializer for z
+# 1807|           getExpr(): [VariableAccess] x
+# 1807|               Type = [IntType] int
+# 1807|               ValueCategory = prvalue(load)
+# 1808|     getStmt(6): [SwitchStmt] switch (...) ... 
+# 1808|       getExpr(): [VariableAccess] z
+# 1808|           Type = [IntType] int
+# 1808|           ValueCategory = prvalue(load)
+# 1808|       getStmt(): [BlockStmt] { ... }
+# 1809|         getStmt(0): [SwitchCase] default: 
+# 1810|         getStmt(1): [ExprStmt] ExprStmt
+# 1810|           getExpr(): [AssignExpr] ... = ...
+# 1810|               Type = [IntType] int
+# 1810|               ValueCategory = lvalue
+# 1810|             getLValue(): [VariableAccess] x
+# 1810|                 Type = [IntType] int
+# 1810|                 ValueCategory = lvalue
+# 1810|             getRValue(): [AddExpr] ... + ...
+# 1810|                 Type = [IntType] int
+# 1810|                 ValueCategory = prvalue
+# 1810|               getLeftOperand(): [VariableAccess] x
+# 1810|                   Type = [IntType] int
+# 1810|                   ValueCategory = prvalue(load)
+# 1810|               getRightOperand(): [VariableAccess] z
+# 1810|                   Type = [IntType] int
+# 1810|                   ValueCategory = prvalue(load)
+# 1813|     getStmt(7): [SwitchStmt] switch (...) ... 
+# 1813|       getExpr(): [ConditionDeclExpr] (condition decl)
+# 1813|           Type = [IntType] int
+# 1813|           ValueCategory = prvalue
+# 1813|         getVariableAccess(): [VariableAccess] z2
+# 1813|             Type = [IntType] int
+# 1813|             ValueCategory = prvalue(load)
+# 1813|       getStmt(): [BlockStmt] { ... }
+# 1814|         getStmt(0): [SwitchCase] default: 
+# 1815|         getStmt(1): [ExprStmt] ExprStmt
+# 1815|           getExpr(): [AssignAddExpr] ... += ...
+# 1815|               Type = [IntType] int
+# 1815|               ValueCategory = lvalue
+# 1815|             getLValue(): [VariableAccess] x
+# 1815|                 Type = [IntType] int
+# 1815|                 ValueCategory = lvalue
+# 1815|             getRValue(): [VariableAccess] z2
+# 1815|                 Type = [IntType] int
+# 1815|                 ValueCategory = prvalue(load)
+# 1817|     getStmt(8): [ReturnStmt] return ...
 perf-regression.cpp:
 #    4| [CopyAssignmentOperator] Big& Big::operator=(Big const&)
 #    4|   <params>: 

cpp/ql/test/library-tests/ir/ir/ir.cpp

@@ -1754,4 +1754,66 @@ int implicit_copy_constructor_test(
     CopyConstructorTestVirtualClass cy = y;
 }
 
+void if_initialization(int x) {
+    if (int y = x; x + 1) {
+        x = x + y;
+    }
+
+    int w;
+    if (w = x; x + 1) {
+        x = x + w;
+    }
+
+    if (w = x; int w2 = w) {
+        x = x + w;
+    }
+
+    if (int v = x; int v2 = v) {
+        x = x + v;
+    }
+
+    int z = x;
+    if (z) {
+        x = x + z;
+    }
+
+    if (int z2 = z) {
+        x += z2;
+    }
+}
+
+void switch_initialization(int x) {
+    switch (int y = x; x + 1) {
+    default:
+        x = x + y;
+    }
+
+    int w;
+    switch (w = x; x + 1) {
+    default:
+        x = x + w;
+    }
+
+    switch (w = x; int w2 = w) {
+    default:
+        x = x + w;
+    }
+
+    switch (int v = x; int v2 = v) {
+    default:
+        x = x + v;
+    }
+
+    int z = x;
+    switch (z) {
+    default:
+        x = x + z;
+    }
+
+    switch (int z2 = z) {
+    default:
+        x += z2;
+    }
+}
+
 // semmle-extractor-options: -std=c++17 --clang

cpp/ql/test/library-tests/ir/ir/operand_locations.expected

@@ -8215,6 +8215,248 @@
 | ir.cpp:1754:42:1754:42 | SideEffect | ~m1752_4 |
 | ir.cpp:1754:42:1754:42 | Unary | r1754_5 |
 | ir.cpp:1754:42:1754:42 | Unary | r1754_6 |
+| ir.cpp:1757:6:1757:22 | ChiPartial | partial:m1757_3 |
+| ir.cpp:1757:6:1757:22 | ChiTotal | total:m1757_2 |
+| ir.cpp:1757:6:1757:22 | SideEffect | m1757_3 |
+| ir.cpp:1757:28:1757:28 | Address | &:r1757_5 |
+| ir.cpp:1758:13:1758:13 | Address | &:r1758_1 |
+| ir.cpp:1758:17:1758:17 | Address | &:r1758_2 |
+| ir.cpp:1758:17:1758:17 | Load | m1757_6 |
+| ir.cpp:1758:17:1758:17 | StoreValue | r1758_3 |
+| ir.cpp:1758:20:1758:20 | Address | &:r1758_5 |
+| ir.cpp:1758:20:1758:20 | Left | r1758_6 |
+| ir.cpp:1758:20:1758:20 | Load | m1757_6 |
+| ir.cpp:1758:20:1758:24 | Condition | r1758_10 |
+| ir.cpp:1758:20:1758:24 | Left | r1758_8 |
+| ir.cpp:1758:20:1758:24 | Right | r1758_9 |
+| ir.cpp:1758:24:1758:24 | Right | r1758_7 |
+| ir.cpp:1759:9:1759:9 | Address | &:r1759_6 |
+| ir.cpp:1759:13:1759:13 | Address | &:r1759_1 |
+| ir.cpp:1759:13:1759:13 | Left | r1759_2 |
+| ir.cpp:1759:13:1759:13 | Load | m1757_6 |
+| ir.cpp:1759:13:1759:17 | StoreValue | r1759_5 |
+| ir.cpp:1759:17:1759:17 | Address | &:r1759_3 |
+| ir.cpp:1759:17:1759:17 | Load | m1758_4 |
+| ir.cpp:1759:17:1759:17 | Right | r1759_4 |
+| ir.cpp:1762:9:1762:9 | Address | &:r1762_2 |
+| ir.cpp:1762:9:1762:9 | Phi | from 0:m1757_6 |
+| ir.cpp:1762:9:1762:9 | Phi | from 1:m1759_7 |
+| ir.cpp:1763:9:1763:9 | Address | &:r1763_3 |
+| ir.cpp:1763:13:1763:13 | Address | &:r1763_1 |
+| ir.cpp:1763:13:1763:13 | Load | m1762_1 |
+| ir.cpp:1763:13:1763:13 | StoreValue | r1763_2 |
+| ir.cpp:1763:16:1763:16 | Address | &:r1763_5 |
+| ir.cpp:1763:16:1763:16 | Left | r1763_6 |
+| ir.cpp:1763:16:1763:16 | Load | m1762_1 |
+| ir.cpp:1763:16:1763:20 | Condition | r1763_10 |
+| ir.cpp:1763:16:1763:20 | Left | r1763_8 |
+| ir.cpp:1763:16:1763:20 | Right | r1763_9 |
+| ir.cpp:1763:20:1763:20 | Right | r1763_7 |
+| ir.cpp:1764:9:1764:9 | Address | &:r1764_6 |
+| ir.cpp:1764:13:1764:13 | Address | &:r1764_1 |
+| ir.cpp:1764:13:1764:13 | Left | r1764_2 |
+| ir.cpp:1764:13:1764:13 | Load | m1762_1 |
+| ir.cpp:1764:13:1764:17 | StoreValue | r1764_5 |
+| ir.cpp:1764:17:1764:17 | Address | &:r1764_3 |
+| ir.cpp:1764:17:1764:17 | Load | m1763_4 |
+| ir.cpp:1764:17:1764:17 | Right | r1764_4 |
+| ir.cpp:1767:9:1767:9 | Address | &:r1767_4 |
+| ir.cpp:1767:13:1767:13 | Address | &:r1767_2 |
+| ir.cpp:1767:13:1767:13 | Load | m1767_1 |
+| ir.cpp:1767:13:1767:13 | Phi | from 2:m1762_1 |
+| ir.cpp:1767:13:1767:13 | Phi | from 3:m1764_7 |
+| ir.cpp:1767:13:1767:13 | StoreValue | r1767_3 |
+| ir.cpp:1767:14:1767:25 | Address | &:r1767_6 |
+| ir.cpp:1767:14:1767:25 | Condition | r1767_14 |
+| ir.cpp:1767:20:1767:21 | Address | &:r1767_10 |
+| ir.cpp:1767:20:1767:21 | Left | r1767_11 |
+| ir.cpp:1767:20:1767:21 | Load | m1767_9 |
+| ir.cpp:1767:20:1767:21 | Right | r1767_12 |
+| ir.cpp:1767:20:1767:21 | Unary | r1767_13 |
+| ir.cpp:1767:25:1767:25 | Address | &:r1767_7 |
+| ir.cpp:1767:25:1767:25 | Load | m1767_5 |
+| ir.cpp:1767:25:1767:25 | StoreValue | r1767_8 |
+| ir.cpp:1768:9:1768:9 | Address | &:r1768_6 |
+| ir.cpp:1768:13:1768:13 | Address | &:r1768_1 |
+| ir.cpp:1768:13:1768:13 | Left | r1768_2 |
+| ir.cpp:1768:13:1768:13 | Load | m1767_1 |
+| ir.cpp:1768:13:1768:17 | StoreValue | r1768_5 |
+| ir.cpp:1768:17:1768:17 | Address | &:r1768_3 |
+| ir.cpp:1768:17:1768:17 | Load | m1767_5 |
+| ir.cpp:1768:17:1768:17 | Right | r1768_4 |
+| ir.cpp:1771:9:1771:29 | Address | &:r1771_6 |
+| ir.cpp:1771:9:1771:29 | Condition | r1771_14 |
+| ir.cpp:1771:13:1771:13 | Address | &:r1771_2 |
+| ir.cpp:1771:13:1771:13 | Phi | from 4:m1767_1 |
+| ir.cpp:1771:13:1771:13 | Phi | from 5:m1768_7 |
+| ir.cpp:1771:17:1771:17 | Address | &:r1771_3 |
+| ir.cpp:1771:17:1771:17 | Load | m1771_1 |
+| ir.cpp:1771:17:1771:17 | StoreValue | r1771_4 |
+| ir.cpp:1771:24:1771:25 | Address | &:r1771_10 |
+| ir.cpp:1771:24:1771:25 | Left | r1771_11 |
+| ir.cpp:1771:24:1771:25 | Load | m1771_9 |
+| ir.cpp:1771:24:1771:25 | Right | r1771_12 |
+| ir.cpp:1771:24:1771:25 | Unary | r1771_13 |
+| ir.cpp:1771:29:1771:29 | Address | &:r1771_7 |
+| ir.cpp:1771:29:1771:29 | Load | m1771_5 |
+| ir.cpp:1771:29:1771:29 | StoreValue | r1771_8 |
+| ir.cpp:1772:9:1772:9 | Address | &:r1772_6 |
+| ir.cpp:1772:13:1772:13 | Address | &:r1772_1 |
+| ir.cpp:1772:13:1772:13 | Left | r1772_2 |
+| ir.cpp:1772:13:1772:13 | Load | m1771_1 |
+| ir.cpp:1772:13:1772:17 | StoreValue | r1772_5 |
+| ir.cpp:1772:17:1772:17 | Address | &:r1772_3 |
+| ir.cpp:1772:17:1772:17 | Load | m1771_5 |
+| ir.cpp:1772:17:1772:17 | Right | r1772_4 |
+| ir.cpp:1775:9:1775:9 | Address | &:r1775_2 |
+| ir.cpp:1775:9:1775:9 | Phi | from 6:m1771_1 |
+| ir.cpp:1775:9:1775:9 | Phi | from 7:m1772_7 |
+| ir.cpp:1775:13:1775:13 | Address | &:r1775_3 |
+| ir.cpp:1775:13:1775:13 | Load | m1775_1 |
+| ir.cpp:1775:13:1775:13 | StoreValue | r1775_4 |
+| ir.cpp:1776:9:1776:9 | Address | &:r1776_1 |
+| ir.cpp:1776:9:1776:9 | Condition | r1776_4 |
+| ir.cpp:1776:9:1776:9 | Left | r1776_2 |
+| ir.cpp:1776:9:1776:9 | Load | m1775_5 |
+| ir.cpp:1776:9:1776:9 | Right | r1776_3 |
+| ir.cpp:1777:9:1777:9 | Address | &:r1777_6 |
+| ir.cpp:1777:13:1777:13 | Address | &:r1777_1 |
+| ir.cpp:1777:13:1777:13 | Left | r1777_2 |
+| ir.cpp:1777:13:1777:13 | Load | m1775_1 |
+| ir.cpp:1777:13:1777:17 | StoreValue | r1777_5 |
+| ir.cpp:1777:17:1777:17 | Address | &:r1777_3 |
+| ir.cpp:1777:17:1777:17 | Load | m1775_5 |
+| ir.cpp:1777:17:1777:17 | Right | r1777_4 |
+| ir.cpp:1780:9:1780:18 | Address | &:r1780_2 |
+| ir.cpp:1780:9:1780:18 | Condition | r1780_10 |
+| ir.cpp:1780:9:1780:18 | Phi | from 8:m1775_1 |
+| ir.cpp:1780:9:1780:18 | Phi | from 9:m1777_7 |
+| ir.cpp:1780:13:1780:14 | Address | &:r1780_6 |
+| ir.cpp:1780:13:1780:14 | Left | r1780_7 |
+| ir.cpp:1780:13:1780:14 | Load | m1780_5 |
+| ir.cpp:1780:13:1780:14 | Right | r1780_8 |
+| ir.cpp:1780:13:1780:14 | Unary | r1780_9 |
+| ir.cpp:1780:18:1780:18 | Address | &:r1780_3 |
+| ir.cpp:1780:18:1780:18 | Load | m1775_5 |
+| ir.cpp:1780:18:1780:18 | StoreValue | r1780_4 |
+| ir.cpp:1781:9:1781:9 | Address | &:r1781_3 |
+| ir.cpp:1781:9:1781:9 | Address | &:r1781_3 |
+| ir.cpp:1781:9:1781:9 | Left | r1781_4 |
+| ir.cpp:1781:9:1781:9 | Load | m1780_1 |
+| ir.cpp:1781:9:1781:15 | StoreValue | r1781_5 |
+| ir.cpp:1781:14:1781:15 | Address | &:r1781_1 |
+| ir.cpp:1781:14:1781:15 | Load | m1780_5 |
+| ir.cpp:1781:14:1781:15 | Right | r1781_2 |
+| ir.cpp:1785:6:1785:26 | ChiPartial | partial:m1785_3 |
+| ir.cpp:1785:6:1785:26 | ChiTotal | total:m1785_2 |
+| ir.cpp:1785:6:1785:26 | SideEffect | m1785_3 |
+| ir.cpp:1785:32:1785:32 | Address | &:r1785_5 |
+| ir.cpp:1786:17:1786:17 | Address | &:r1786_1 |
+| ir.cpp:1786:21:1786:21 | Address | &:r1786_2 |
+| ir.cpp:1786:21:1786:21 | Load | m1785_6 |
+| ir.cpp:1786:21:1786:21 | StoreValue | r1786_3 |
+| ir.cpp:1786:24:1786:24 | Address | &:r1786_5 |
+| ir.cpp:1786:24:1786:24 | Left | r1786_6 |
+| ir.cpp:1786:24:1786:24 | Load | m1785_6 |
+| ir.cpp:1786:24:1786:28 | Condition | r1786_8 |
+| ir.cpp:1786:28:1786:28 | Right | r1786_7 |
+| ir.cpp:1788:9:1788:9 | Address | &:r1788_6 |
+| ir.cpp:1788:13:1788:13 | Address | &:r1788_1 |
+| ir.cpp:1788:13:1788:13 | Left | r1788_2 |
+| ir.cpp:1788:13:1788:13 | Load | m1785_6 |
+| ir.cpp:1788:13:1788:17 | StoreValue | r1788_5 |
+| ir.cpp:1788:17:1788:17 | Address | &:r1788_3 |
+| ir.cpp:1788:17:1788:17 | Load | m1786_4 |
+| ir.cpp:1788:17:1788:17 | Right | r1788_4 |
+| ir.cpp:1791:9:1791:9 | Address | &:r1791_1 |
+| ir.cpp:1792:13:1792:13 | Address | &:r1792_3 |
+| ir.cpp:1792:17:1792:17 | Address | &:r1792_1 |
+| ir.cpp:1792:17:1792:17 | Load | m1788_7 |
+| ir.cpp:1792:17:1792:17 | StoreValue | r1792_2 |
+| ir.cpp:1792:20:1792:20 | Address | &:r1792_5 |
+| ir.cpp:1792:20:1792:20 | Left | r1792_6 |
+| ir.cpp:1792:20:1792:20 | Load | m1788_7 |
+| ir.cpp:1792:20:1792:24 | Condition | r1792_8 |
+| ir.cpp:1792:24:1792:24 | Right | r1792_7 |
+| ir.cpp:1794:9:1794:9 | Address | &:r1794_6 |
+| ir.cpp:1794:13:1794:13 | Address | &:r1794_1 |
+| ir.cpp:1794:13:1794:13 | Left | r1794_2 |
+| ir.cpp:1794:13:1794:13 | Load | m1788_7 |
+| ir.cpp:1794:13:1794:17 | StoreValue | r1794_5 |
+| ir.cpp:1794:17:1794:17 | Address | &:r1794_3 |
+| ir.cpp:1794:17:1794:17 | Load | m1792_4 |
+| ir.cpp:1794:17:1794:17 | Right | r1794_4 |
+| ir.cpp:1797:13:1797:13 | Address | &:r1797_3 |
+| ir.cpp:1797:17:1797:17 | Address | &:r1797_1 |
+| ir.cpp:1797:17:1797:17 | Load | m1794_7 |
+| ir.cpp:1797:17:1797:17 | StoreValue | r1797_2 |
+| ir.cpp:1797:18:1797:29 | Address | &:r1797_5 |
+| ir.cpp:1797:18:1797:29 | Condition | r1797_11 |
+| ir.cpp:1797:24:1797:25 | Address | &:r1797_9 |
+| ir.cpp:1797:24:1797:25 | Load | m1797_8 |
+| ir.cpp:1797:24:1797:25 | Unary | r1797_10 |
+| ir.cpp:1797:29:1797:29 | Address | &:r1797_6 |
+| ir.cpp:1797:29:1797:29 | Load | m1797_4 |
+| ir.cpp:1797:29:1797:29 | StoreValue | r1797_7 |
+| ir.cpp:1799:9:1799:9 | Address | &:r1799_6 |
+| ir.cpp:1799:13:1799:13 | Address | &:r1799_1 |
+| ir.cpp:1799:13:1799:13 | Left | r1799_2 |
+| ir.cpp:1799:13:1799:13 | Load | m1794_7 |
+| ir.cpp:1799:13:1799:17 | StoreValue | r1799_5 |
+| ir.cpp:1799:17:1799:17 | Address | &:r1799_3 |
+| ir.cpp:1799:17:1799:17 | Load | m1797_4 |
+| ir.cpp:1799:17:1799:17 | Right | r1799_4 |
+| ir.cpp:1802:13:1802:33 | Address | &:r1802_5 |
+| ir.cpp:1802:13:1802:33 | Condition | r1802_11 |
+| ir.cpp:1802:17:1802:17 | Address | &:r1802_1 |
+| ir.cpp:1802:21:1802:21 | Address | &:r1802_2 |
+| ir.cpp:1802:21:1802:21 | Load | m1799_7 |
+| ir.cpp:1802:21:1802:21 | StoreValue | r1802_3 |
+| ir.cpp:1802:28:1802:29 | Address | &:r1802_9 |
+| ir.cpp:1802:28:1802:29 | Load | m1802_8 |
+| ir.cpp:1802:28:1802:29 | Unary | r1802_10 |
+| ir.cpp:1802:33:1802:33 | Address | &:r1802_6 |
+| ir.cpp:1802:33:1802:33 | Load | m1802_4 |
+| ir.cpp:1802:33:1802:33 | StoreValue | r1802_7 |
+| ir.cpp:1804:9:1804:9 | Address | &:r1804_6 |
+| ir.cpp:1804:13:1804:13 | Address | &:r1804_1 |
+| ir.cpp:1804:13:1804:13 | Left | r1804_2 |
+| ir.cpp:1804:13:1804:13 | Load | m1799_7 |
+| ir.cpp:1804:13:1804:17 | StoreValue | r1804_5 |
+| ir.cpp:1804:17:1804:17 | Address | &:r1804_3 |
+| ir.cpp:1804:17:1804:17 | Load | m1802_4 |
+| ir.cpp:1804:17:1804:17 | Right | r1804_4 |
+| ir.cpp:1807:9:1807:9 | Address | &:r1807_1 |
+| ir.cpp:1807:13:1807:13 | Address | &:r1807_2 |
+| ir.cpp:1807:13:1807:13 | Load | m1804_7 |
+| ir.cpp:1807:13:1807:13 | StoreValue | r1807_3 |
+| ir.cpp:1808:13:1808:13 | Address | &:r1808_1 |
+| ir.cpp:1808:13:1808:13 | Condition | r1808_2 |
+| ir.cpp:1808:13:1808:13 | Load | m1807_4 |
+| ir.cpp:1810:9:1810:9 | Address | &:r1810_6 |
+| ir.cpp:1810:13:1810:13 | Address | &:r1810_1 |
+| ir.cpp:1810:13:1810:13 | Left | r1810_2 |
+| ir.cpp:1810:13:1810:13 | Load | m1804_7 |
+| ir.cpp:1810:13:1810:17 | StoreValue | r1810_5 |
+| ir.cpp:1810:17:1810:17 | Address | &:r1810_3 |
+| ir.cpp:1810:17:1810:17 | Load | m1807_4 |
+| ir.cpp:1810:17:1810:17 | Right | r1810_4 |
+| ir.cpp:1813:13:1813:22 | Address | &:r1813_1 |
+| ir.cpp:1813:13:1813:22 | Condition | r1813_7 |
+| ir.cpp:1813:17:1813:18 | Address | &:r1813_5 |
+| ir.cpp:1813:17:1813:18 | Load | m1813_4 |
+| ir.cpp:1813:17:1813:18 | Unary | r1813_6 |
+| ir.cpp:1813:22:1813:22 | Address | &:r1813_2 |
+| ir.cpp:1813:22:1813:22 | Load | m1807_4 |
+| ir.cpp:1813:22:1813:22 | StoreValue | r1813_3 |
+| ir.cpp:1815:9:1815:9 | Address | &:r1815_3 |
+| ir.cpp:1815:9:1815:9 | Address | &:r1815_3 |
+| ir.cpp:1815:9:1815:9 | Left | r1815_4 |
+| ir.cpp:1815:9:1815:9 | Load | m1810_7 |
+| ir.cpp:1815:9:1815:15 | StoreValue | r1815_5 |
+| ir.cpp:1815:14:1815:15 | Address | &:r1815_1 |
+| ir.cpp:1815:14:1815:15 | Load | m1813_4 |
+| ir.cpp:1815:14:1815:15 | Right | r1815_2 |
 | perf-regression.cpp:6:3:6:5 | Address | &:r6_5 |
 | perf-regression.cpp:6:3:6:5 | Address | &:r6_5 |
 | perf-regression.cpp:6:3:6:5 | Address | &:r6_7 |

cpp/ql/test/library-tests/ir/ir/raw_ir.expected

@@ -9418,6 +9418,308 @@ ir.cpp:
 # 1750|     v1750_6(void)       = AliasedUse               : ~m?
 # 1750|     v1750_7(void)       = ExitFunction             : 
 
+# 1757| void if_initialization(int)
+# 1757|   Block 0
+# 1757|     v1757_1(void)       = EnterFunction          : 
+# 1757|     mu1757_2(unknown)   = AliasedDefinition      : 
+# 1757|     mu1757_3(unknown)   = InitializeNonLocal     : 
+# 1757|     r1757_4(glval<int>) = VariableAddress[x]     : 
+# 1757|     mu1757_5(int)       = InitializeParameter[x] : &:r1757_4
+# 1758|     r1758_1(glval<int>) = VariableAddress[y]     : 
+# 1758|     r1758_2(glval<int>) = VariableAddress[x]     : 
+# 1758|     r1758_3(int)        = Load[x]                : &:r1758_2, ~m?
+# 1758|     mu1758_4(int)       = Store[y]               : &:r1758_1, r1758_3
+# 1758|     r1758_5(glval<int>) = VariableAddress[x]     : 
+# 1758|     r1758_6(int)        = Load[x]                : &:r1758_5, ~m?
+# 1758|     r1758_7(int)        = Constant[1]            : 
+# 1758|     r1758_8(int)        = Add                    : r1758_6, r1758_7
+# 1758|     r1758_9(int)        = Constant[0]            : 
+# 1758|     r1758_10(bool)      = CompareNE              : r1758_8, r1758_9
+# 1758|     v1758_11(void)      = ConditionalBranch      : r1758_10
+#-----|   False -> Block 2
+#-----|   True -> Block 1
+
+# 1759|   Block 1
+# 1759|     r1759_1(glval<int>) = VariableAddress[x] : 
+# 1759|     r1759_2(int)        = Load[x]            : &:r1759_1, ~m?
+# 1759|     r1759_3(glval<int>) = VariableAddress[y] : 
+# 1759|     r1759_4(int)        = Load[y]            : &:r1759_3, ~m?
+# 1759|     r1759_5(int)        = Add                : r1759_2, r1759_4
+# 1759|     r1759_6(glval<int>) = VariableAddress[x] : 
+# 1759|     mu1759_7(int)       = Store[x]           : &:r1759_6, r1759_5
+#-----|   Goto -> Block 2
+
+# 1762|   Block 2
+# 1762|     r1762_1(glval<int>) = VariableAddress[w] : 
+# 1762|     mu1762_2(int)       = Uninitialized[w]   : &:r1762_1
+# 1763|     r1763_1(glval<int>) = VariableAddress[x] : 
+# 1763|     r1763_2(int)        = Load[x]            : &:r1763_1, ~m?
+# 1763|     r1763_3(glval<int>) = VariableAddress[w] : 
+# 1763|     mu1763_4(int)       = Store[w]           : &:r1763_3, r1763_2
+# 1763|     r1763_5(glval<int>) = VariableAddress[x] : 
+# 1763|     r1763_6(int)        = Load[x]            : &:r1763_5, ~m?
+# 1763|     r1763_7(int)        = Constant[1]        : 
+# 1763|     r1763_8(int)        = Add                : r1763_6, r1763_7
+# 1763|     r1763_9(int)        = Constant[0]        : 
+# 1763|     r1763_10(bool)      = CompareNE          : r1763_8, r1763_9
+# 1763|     v1763_11(void)      = ConditionalBranch  : r1763_10
+#-----|   False -> Block 4
+#-----|   True -> Block 3
+
+# 1764|   Block 3
+# 1764|     r1764_1(glval<int>) = VariableAddress[x] : 
+# 1764|     r1764_2(int)        = Load[x]            : &:r1764_1, ~m?
+# 1764|     r1764_3(glval<int>) = VariableAddress[w] : 
+# 1764|     r1764_4(int)        = Load[w]            : &:r1764_3, ~m?
+# 1764|     r1764_5(int)        = Add                : r1764_2, r1764_4
+# 1764|     r1764_6(glval<int>) = VariableAddress[x] : 
+# 1764|     mu1764_7(int)       = Store[x]           : &:r1764_6, r1764_5
+#-----|   Goto -> Block 4
+
+# 1767|   Block 4
+# 1767|     r1767_1(glval<int>) = VariableAddress[x]  : 
+# 1767|     r1767_2(int)        = Load[x]             : &:r1767_1, ~m?
+# 1767|     r1767_3(glval<int>) = VariableAddress[w]  : 
+# 1767|     mu1767_4(int)       = Store[w]            : &:r1767_3, r1767_2
+# 1767|     r1767_5(glval<int>) = VariableAddress[w2] : 
+# 1767|     r1767_6(glval<int>) = VariableAddress[w]  : 
+# 1767|     r1767_7(int)        = Load[w]             : &:r1767_6, ~m?
+# 1767|     mu1767_8(int)       = Store[w2]           : &:r1767_5, r1767_7
+# 1767|     r1767_9(glval<int>) = VariableAddress[w2] : 
+# 1767|     r1767_10(int)       = Load[w2]            : &:r1767_9, ~m?
+# 1767|     r1767_11(int)       = Constant[0]         : 
+# 1767|     r1767_12(bool)      = CompareNE           : r1767_10, r1767_11
+# 1767|     r1767_13(bool)      = CopyValue           : r1767_12
+# 1767|     v1767_14(void)      = ConditionalBranch   : r1767_13
+#-----|   False -> Block 6
+#-----|   True -> Block 5
+
+# 1768|   Block 5
+# 1768|     r1768_1(glval<int>) = VariableAddress[x] : 
+# 1768|     r1768_2(int)        = Load[x]            : &:r1768_1, ~m?
+# 1768|     r1768_3(glval<int>) = VariableAddress[w] : 
+# 1768|     r1768_4(int)        = Load[w]            : &:r1768_3, ~m?
+# 1768|     r1768_5(int)        = Add                : r1768_2, r1768_4
+# 1768|     r1768_6(glval<int>) = VariableAddress[x] : 
+# 1768|     mu1768_7(int)       = Store[x]           : &:r1768_6, r1768_5
+#-----|   Goto -> Block 6
+
+# 1771|   Block 6
+# 1771|     r1771_1(glval<int>) = VariableAddress[v]  : 
+# 1771|     r1771_2(glval<int>) = VariableAddress[x]  : 
+# 1771|     r1771_3(int)        = Load[x]             : &:r1771_2, ~m?
+# 1771|     mu1771_4(int)       = Store[v]            : &:r1771_1, r1771_3
+# 1771|     r1771_5(glval<int>) = VariableAddress[v2] : 
+# 1771|     r1771_6(glval<int>) = VariableAddress[v]  : 
+# 1771|     r1771_7(int)        = Load[v]             : &:r1771_6, ~m?
+# 1771|     mu1771_8(int)       = Store[v2]           : &:r1771_5, r1771_7
+# 1771|     r1771_9(glval<int>) = VariableAddress[v2] : 
+# 1771|     r1771_10(int)       = Load[v2]            : &:r1771_9, ~m?
+# 1771|     r1771_11(int)       = Constant[0]         : 
+# 1771|     r1771_12(bool)      = CompareNE           : r1771_10, r1771_11
+# 1771|     r1771_13(bool)      = CopyValue           : r1771_12
+# 1771|     v1771_14(void)      = ConditionalBranch   : r1771_13
+#-----|   False -> Block 8
+#-----|   True -> Block 7
+
+# 1772|   Block 7
+# 1772|     r1772_1(glval<int>) = VariableAddress[x] : 
+# 1772|     r1772_2(int)        = Load[x]            : &:r1772_1, ~m?
+# 1772|     r1772_3(glval<int>) = VariableAddress[v] : 
+# 1772|     r1772_4(int)        = Load[v]            : &:r1772_3, ~m?
+# 1772|     r1772_5(int)        = Add                : r1772_2, r1772_4
+# 1772|     r1772_6(glval<int>) = VariableAddress[x] : 
+# 1772|     mu1772_7(int)       = Store[x]           : &:r1772_6, r1772_5
+#-----|   Goto -> Block 8
+
+# 1775|   Block 8
+# 1775|     r1775_1(glval<int>) = VariableAddress[z] : 
+# 1775|     r1775_2(glval<int>) = VariableAddress[x] : 
+# 1775|     r1775_3(int)        = Load[x]            : &:r1775_2, ~m?
+# 1775|     mu1775_4(int)       = Store[z]           : &:r1775_1, r1775_3
+# 1776|     r1776_1(glval<int>) = VariableAddress[z] : 
+# 1776|     r1776_2(int)        = Load[z]            : &:r1776_1, ~m?
+# 1776|     r1776_3(int)        = Constant[0]        : 
+# 1776|     r1776_4(bool)       = CompareNE          : r1776_2, r1776_3
+# 1776|     v1776_5(void)       = ConditionalBranch  : r1776_4
+#-----|   False -> Block 10
+#-----|   True -> Block 9
+
+# 1777|   Block 9
+# 1777|     r1777_1(glval<int>) = VariableAddress[x] : 
+# 1777|     r1777_2(int)        = Load[x]            : &:r1777_1, ~m?
+# 1777|     r1777_3(glval<int>) = VariableAddress[z] : 
+# 1777|     r1777_4(int)        = Load[z]            : &:r1777_3, ~m?
+# 1777|     r1777_5(int)        = Add                : r1777_2, r1777_4
+# 1777|     r1777_6(glval<int>) = VariableAddress[x] : 
+# 1777|     mu1777_7(int)       = Store[x]           : &:r1777_6, r1777_5
+#-----|   Goto -> Block 10
+
+# 1780|   Block 10
+# 1780|     r1780_1(glval<int>) = VariableAddress[z2] : 
+# 1780|     r1780_2(glval<int>) = VariableAddress[z]  : 
+# 1780|     r1780_3(int)        = Load[z]             : &:r1780_2, ~m?
+# 1780|     mu1780_4(int)       = Store[z2]           : &:r1780_1, r1780_3
+# 1780|     r1780_5(glval<int>) = VariableAddress[z2] : 
+# 1780|     r1780_6(int)        = Load[z2]            : &:r1780_5, ~m?
+# 1780|     r1780_7(int)        = Constant[0]         : 
+# 1780|     r1780_8(bool)       = CompareNE           : r1780_6, r1780_7
+# 1780|     r1780_9(bool)       = CopyValue           : r1780_8
+# 1780|     v1780_10(void)      = ConditionalBranch   : r1780_9
+#-----|   False -> Block 12
+#-----|   True -> Block 11
+
+# 1781|   Block 11
+# 1781|     r1781_1(glval<int>) = VariableAddress[z2] : 
+# 1781|     r1781_2(int)        = Load[z2]            : &:r1781_1, ~m?
+# 1781|     r1781_3(glval<int>) = VariableAddress[x]  : 
+# 1781|     r1781_4(int)        = Load[x]             : &:r1781_3, ~m?
+# 1781|     r1781_5(int)        = Add                 : r1781_4, r1781_2
+# 1781|     mu1781_6(int)       = Store[x]            : &:r1781_3, r1781_5
+#-----|   Goto -> Block 12
+
+# 1783|   Block 12
+# 1783|     v1783_1(void) = NoOp         : 
+# 1757|     v1757_6(void) = ReturnVoid   : 
+# 1757|     v1757_7(void) = AliasedUse   : ~m?
+# 1757|     v1757_8(void) = ExitFunction : 
+
+# 1785| void switch_initialization(int)
+# 1785|   Block 0
+# 1785|     v1785_1(void)       = EnterFunction          : 
+# 1785|     mu1785_2(unknown)   = AliasedDefinition      : 
+# 1785|     mu1785_3(unknown)   = InitializeNonLocal     : 
+# 1785|     r1785_4(glval<int>) = VariableAddress[x]     : 
+# 1785|     mu1785_5(int)       = InitializeParameter[x] : &:r1785_4
+# 1786|     r1786_1(glval<int>) = VariableAddress[y]     : 
+# 1786|     r1786_2(glval<int>) = VariableAddress[x]     : 
+# 1786|     r1786_3(int)        = Load[x]                : &:r1786_2, ~m?
+# 1786|     mu1786_4(int)       = Store[y]               : &:r1786_1, r1786_3
+# 1786|     r1786_5(glval<int>) = VariableAddress[x]     : 
+# 1786|     r1786_6(int)        = Load[x]                : &:r1786_5, ~m?
+# 1786|     r1786_7(int)        = Constant[1]            : 
+# 1786|     r1786_8(int)        = Add                    : r1786_6, r1786_7
+# 1786|     v1786_9(void)       = Switch                 : r1786_8
+#-----|   Default -> Block 1
+
+# 1787|   Block 1
+# 1787|     v1787_1(void)       = NoOp               : 
+# 1788|     r1788_1(glval<int>) = VariableAddress[x] : 
+# 1788|     r1788_2(int)        = Load[x]            : &:r1788_1, ~m?
+# 1788|     r1788_3(glval<int>) = VariableAddress[y] : 
+# 1788|     r1788_4(int)        = Load[y]            : &:r1788_3, ~m?
+# 1788|     r1788_5(int)        = Add                : r1788_2, r1788_4
+# 1788|     r1788_6(glval<int>) = VariableAddress[x] : 
+# 1788|     mu1788_7(int)       = Store[x]           : &:r1788_6, r1788_5
+# 1791|     r1791_1(glval<int>) = VariableAddress[w] : 
+# 1791|     mu1791_2(int)       = Uninitialized[w]   : &:r1791_1
+# 1792|     r1792_1(glval<int>) = VariableAddress[x] : 
+# 1792|     r1792_2(int)        = Load[x]            : &:r1792_1, ~m?
+# 1792|     r1792_3(glval<int>) = VariableAddress[w] : 
+# 1792|     mu1792_4(int)       = Store[w]           : &:r1792_3, r1792_2
+# 1792|     r1792_5(glval<int>) = VariableAddress[x] : 
+# 1792|     r1792_6(int)        = Load[x]            : &:r1792_5, ~m?
+# 1792|     r1792_7(int)        = Constant[1]        : 
+# 1792|     r1792_8(int)        = Add                : r1792_6, r1792_7
+# 1792|     v1792_9(void)       = Switch             : r1792_8
+#-----|   Default -> Block 2
+
+# 1793|   Block 2
+# 1793|     v1793_1(void)       = NoOp                : 
+# 1794|     r1794_1(glval<int>) = VariableAddress[x]  : 
+# 1794|     r1794_2(int)        = Load[x]             : &:r1794_1, ~m?
+# 1794|     r1794_3(glval<int>) = VariableAddress[w]  : 
+# 1794|     r1794_4(int)        = Load[w]             : &:r1794_3, ~m?
+# 1794|     r1794_5(int)        = Add                 : r1794_2, r1794_4
+# 1794|     r1794_6(glval<int>) = VariableAddress[x]  : 
+# 1794|     mu1794_7(int)       = Store[x]            : &:r1794_6, r1794_5
+# 1797|     r1797_1(glval<int>) = VariableAddress[x]  : 
+# 1797|     r1797_2(int)        = Load[x]             : &:r1797_1, ~m?
+# 1797|     r1797_3(glval<int>) = VariableAddress[w]  : 
+# 1797|     mu1797_4(int)       = Store[w]            : &:r1797_3, r1797_2
+# 1797|     r1797_5(glval<int>) = VariableAddress[w2] : 
+# 1797|     r1797_6(glval<int>) = VariableAddress[w]  : 
+# 1797|     r1797_7(int)        = Load[w]             : &:r1797_6, ~m?
+# 1797|     mu1797_8(int)       = Store[w2]           : &:r1797_5, r1797_7
+# 1797|     r1797_9(glval<int>) = VariableAddress[w2] : 
+# 1797|     r1797_10(int)       = Load[w2]            : &:r1797_9, ~m?
+# 1797|     r1797_11(int)       = CopyValue           : r1797_10
+# 1797|     v1797_12(void)      = Switch              : r1797_11
+#-----|   Default -> Block 3
+
+# 1798|   Block 3
+# 1798|     v1798_1(void)       = NoOp                : 
+# 1799|     r1799_1(glval<int>) = VariableAddress[x]  : 
+# 1799|     r1799_2(int)        = Load[x]             : &:r1799_1, ~m?
+# 1799|     r1799_3(glval<int>) = VariableAddress[w]  : 
+# 1799|     r1799_4(int)        = Load[w]             : &:r1799_3, ~m?
+# 1799|     r1799_5(int)        = Add                 : r1799_2, r1799_4
+# 1799|     r1799_6(glval<int>) = VariableAddress[x]  : 
+# 1799|     mu1799_7(int)       = Store[x]            : &:r1799_6, r1799_5
+# 1802|     r1802_1(glval<int>) = VariableAddress[v]  : 
+# 1802|     r1802_2(glval<int>) = VariableAddress[x]  : 
+# 1802|     r1802_3(int)        = Load[x]             : &:r1802_2, ~m?
+# 1802|     mu1802_4(int)       = Store[v]            : &:r1802_1, r1802_3
+# 1802|     r1802_5(glval<int>) = VariableAddress[v2] : 
+# 1802|     r1802_6(glval<int>) = VariableAddress[v]  : 
+# 1802|     r1802_7(int)        = Load[v]             : &:r1802_6, ~m?
+# 1802|     mu1802_8(int)       = Store[v2]           : &:r1802_5, r1802_7
+# 1802|     r1802_9(glval<int>) = VariableAddress[v2] : 
+# 1802|     r1802_10(int)       = Load[v2]            : &:r1802_9, ~m?
+# 1802|     r1802_11(int)       = CopyValue           : r1802_10
+# 1802|     v1802_12(void)      = Switch              : r1802_11
+#-----|   Default -> Block 4
+
+# 1803|   Block 4
+# 1803|     v1803_1(void)       = NoOp               : 
+# 1804|     r1804_1(glval<int>) = VariableAddress[x] : 
+# 1804|     r1804_2(int)        = Load[x]            : &:r1804_1, ~m?
+# 1804|     r1804_3(glval<int>) = VariableAddress[v] : 
+# 1804|     r1804_4(int)        = Load[v]            : &:r1804_3, ~m?
+# 1804|     r1804_5(int)        = Add                : r1804_2, r1804_4
+# 1804|     r1804_6(glval<int>) = VariableAddress[x] : 
+# 1804|     mu1804_7(int)       = Store[x]           : &:r1804_6, r1804_5
+# 1807|     r1807_1(glval<int>) = VariableAddress[z] : 
+# 1807|     r1807_2(glval<int>) = VariableAddress[x] : 
+# 1807|     r1807_3(int)        = Load[x]            : &:r1807_2, ~m?
+# 1807|     mu1807_4(int)       = Store[z]           : &:r1807_1, r1807_3
+# 1808|     r1808_1(glval<int>) = VariableAddress[z] : 
+# 1808|     r1808_2(int)        = Load[z]            : &:r1808_1, ~m?
+# 1808|     v1808_3(void)       = Switch             : r1808_2
+#-----|   Default -> Block 5
+
+# 1809|   Block 5
+# 1809|     v1809_1(void)       = NoOp                : 
+# 1810|     r1810_1(glval<int>) = VariableAddress[x]  : 
+# 1810|     r1810_2(int)        = Load[x]             : &:r1810_1, ~m?
+# 1810|     r1810_3(glval<int>) = VariableAddress[z]  : 
+# 1810|     r1810_4(int)        = Load[z]             : &:r1810_3, ~m?
+# 1810|     r1810_5(int)        = Add                 : r1810_2, r1810_4
+# 1810|     r1810_6(glval<int>) = VariableAddress[x]  : 
+# 1810|     mu1810_7(int)       = Store[x]            : &:r1810_6, r1810_5
+# 1813|     r1813_1(glval<int>) = VariableAddress[z2] : 
+# 1813|     r1813_2(glval<int>) = VariableAddress[z]  : 
+# 1813|     r1813_3(int)        = Load[z]             : &:r1813_2, ~m?
+# 1813|     mu1813_4(int)       = Store[z2]           : &:r1813_1, r1813_3
+# 1813|     r1813_5(glval<int>) = VariableAddress[z2] : 
+# 1813|     r1813_6(int)        = Load[z2]            : &:r1813_5, ~m?
+# 1813|     r1813_7(int)        = CopyValue           : r1813_6
+# 1813|     v1813_8(void)       = Switch              : r1813_7
+#-----|   Default -> Block 6
+
+# 1814|   Block 6
+# 1814|     v1814_1(void)       = NoOp                : 
+# 1815|     r1815_1(glval<int>) = VariableAddress[z2] : 
+# 1815|     r1815_2(int)        = Load[z2]            : &:r1815_1, ~m?
+# 1815|     r1815_3(glval<int>) = VariableAddress[x]  : 
+# 1815|     r1815_4(int)        = Load[x]             : &:r1815_3, ~m?
+# 1815|     r1815_5(int)        = Add                 : r1815_4, r1815_2
+# 1815|     mu1815_6(int)       = Store[x]            : &:r1815_3, r1815_5
+# 1817|     v1817_1(void)       = NoOp                : 
+# 1785|     v1785_6(void)       = ReturnVoid          : 
+# 1785|     v1785_7(void)       = AliasedUse          : ~m?
+# 1785|     v1785_8(void)       = ExitFunction        : 
+
 perf-regression.cpp:
 #    6| void Big::Big()
 #    6|   Block 0

cpp/ql/test/library-tests/string_analysis/StringAnalysis.ql

@@ -5,7 +5,7 @@
 
 import cpp
 
-from AnalysedString s, string str
+from AnalyzedString s, string str
 where
   if s.(StringLiteral).getUnspecifiedType().(DerivedType).getBaseType() instanceof Wchar_t
   then str = "[?]"

cpp/ql/test/query-tests/Security/CWE/CWE-611/XXE.expected

@@ -1,6 +1,18 @@
 edges
 | tests2.cpp:20:17:20:31 | SAXParser output argument | tests2.cpp:22:2:22:2 | p |
 | tests2.cpp:33:17:33:31 | SAXParser output argument | tests2.cpp:37:2:37:2 | p |
+| tests3.cpp:23:21:23:53 | call to createXMLReader | tests3.cpp:25:2:25:2 | p |
+| tests3.cpp:60:21:60:53 | call to createXMLReader | tests3.cpp:63:2:63:2 | p |
+| tests3.cpp:67:21:67:53 | call to createXMLReader | tests3.cpp:70:2:70:2 | p |
+| tests5.cpp:27:25:27:38 | call to createLSParser | tests5.cpp:29:2:29:2 | p |
+| tests5.cpp:40:25:40:38 | call to createLSParser | tests5.cpp:43:2:43:2 | p |
+| tests5.cpp:55:25:55:38 | call to createLSParser | tests5.cpp:59:2:59:2 | p |
+| tests5.cpp:81:25:81:38 | call to createLSParser | tests5.cpp:83:2:83:2 | p |
+| tests5.cpp:81:25:81:38 | call to createLSParser | tests5.cpp:83:2:83:2 | p |
+| tests5.cpp:83:2:83:2 | p | tests5.cpp:85:2:85:2 | p |
+| tests5.cpp:85:2:85:2 | p | tests5.cpp:86:2:86:2 | p |
+| tests5.cpp:86:2:86:2 | p | tests5.cpp:88:2:88:2 | p |
+| tests5.cpp:88:2:88:2 | p | tests5.cpp:89:2:89:2 | p |
 | tests.cpp:15:23:15:43 | XercesDOMParser output argument | tests.cpp:17:2:17:2 | p |
 | tests.cpp:28:23:28:43 | XercesDOMParser output argument | tests.cpp:31:2:31:2 | p |
 | tests.cpp:35:19:35:19 | VariableAddress [post update] | tests.cpp:37:2:37:2 | p |
@@ -32,11 +44,30 @@ nodes
 | tests2.cpp:22:2:22:2 | p | semmle.label | p |
 | tests2.cpp:33:17:33:31 | SAXParser output argument | semmle.label | SAXParser output argument |
 | tests2.cpp:37:2:37:2 | p | semmle.label | p |
+| tests3.cpp:23:21:23:53 | call to createXMLReader | semmle.label | call to createXMLReader |
+| tests3.cpp:25:2:25:2 | p | semmle.label | p |
+| tests3.cpp:60:21:60:53 | call to createXMLReader | semmle.label | call to createXMLReader |
+| tests3.cpp:63:2:63:2 | p | semmle.label | p |
+| tests3.cpp:67:21:67:53 | call to createXMLReader | semmle.label | call to createXMLReader |
+| tests3.cpp:70:2:70:2 | p | semmle.label | p |
 | tests4.cpp:26:34:26:48 | (int)... | semmle.label | (int)... |
 | tests4.cpp:36:34:36:50 | (int)... | semmle.label | (int)... |
 | tests4.cpp:46:34:46:68 | ... \| ... | semmle.label | ... \| ... |
 | tests4.cpp:77:34:77:38 | flags | semmle.label | flags |
 | tests4.cpp:130:39:130:55 | (int)... | semmle.label | (int)... |
+| tests5.cpp:27:25:27:38 | call to createLSParser | semmle.label | call to createLSParser |
+| tests5.cpp:29:2:29:2 | p | semmle.label | p |
+| tests5.cpp:40:25:40:38 | call to createLSParser | semmle.label | call to createLSParser |
+| tests5.cpp:43:2:43:2 | p | semmle.label | p |
+| tests5.cpp:55:25:55:38 | call to createLSParser | semmle.label | call to createLSParser |
+| tests5.cpp:59:2:59:2 | p | semmle.label | p |
+| tests5.cpp:81:25:81:38 | call to createLSParser | semmle.label | call to createLSParser |
+| tests5.cpp:83:2:83:2 | p | semmle.label | p |
+| tests5.cpp:83:2:83:2 | p | semmle.label | p |
+| tests5.cpp:85:2:85:2 | p | semmle.label | p |
+| tests5.cpp:86:2:86:2 | p | semmle.label | p |
+| tests5.cpp:88:2:88:2 | p | semmle.label | p |
+| tests5.cpp:89:2:89:2 | p | semmle.label | p |
 | tests.cpp:15:23:15:43 | XercesDOMParser output argument | semmle.label | XercesDOMParser output argument |
 | tests.cpp:17:2:17:2 | p | semmle.label | p |
 | tests.cpp:28:23:28:43 | XercesDOMParser output argument | semmle.label | XercesDOMParser output argument |
@@ -76,11 +107,19 @@ subpaths
 #select
 | tests2.cpp:22:2:22:2 | p | tests2.cpp:20:17:20:31 | SAXParser output argument | tests2.cpp:22:2:22:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests2.cpp:20:17:20:31 | SAXParser output argument | XML parser |
 | tests2.cpp:37:2:37:2 | p | tests2.cpp:33:17:33:31 | SAXParser output argument | tests2.cpp:37:2:37:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests2.cpp:33:17:33:31 | SAXParser output argument | XML parser |
+| tests3.cpp:25:2:25:2 | p | tests3.cpp:23:21:23:53 | call to createXMLReader | tests3.cpp:25:2:25:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests3.cpp:23:21:23:53 | call to createXMLReader | XML parser |
+| tests3.cpp:63:2:63:2 | p | tests3.cpp:60:21:60:53 | call to createXMLReader | tests3.cpp:63:2:63:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests3.cpp:60:21:60:53 | call to createXMLReader | XML parser |
+| tests3.cpp:70:2:70:2 | p | tests3.cpp:67:21:67:53 | call to createXMLReader | tests3.cpp:70:2:70:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests3.cpp:67:21:67:53 | call to createXMLReader | XML parser |
 | tests4.cpp:26:34:26:48 | (int)... | tests4.cpp:26:34:26:48 | (int)... | tests4.cpp:26:34:26:48 | (int)... | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests4.cpp:26:34:26:48 | (int)... | XML parser |
 | tests4.cpp:36:34:36:50 | (int)... | tests4.cpp:36:34:36:50 | (int)... | tests4.cpp:36:34:36:50 | (int)... | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests4.cpp:36:34:36:50 | (int)... | XML parser |
 | tests4.cpp:46:34:46:68 | ... \| ... | tests4.cpp:46:34:46:68 | ... \| ... | tests4.cpp:46:34:46:68 | ... \| ... | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests4.cpp:46:34:46:68 | ... \| ... | XML parser |
 | tests4.cpp:77:34:77:38 | flags | tests4.cpp:77:34:77:38 | flags | tests4.cpp:77:34:77:38 | flags | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests4.cpp:77:34:77:38 | flags | XML parser |
 | tests4.cpp:130:39:130:55 | (int)... | tests4.cpp:130:39:130:55 | (int)... | tests4.cpp:130:39:130:55 | (int)... | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests4.cpp:130:39:130:55 | (int)... | XML parser |
+| tests5.cpp:29:2:29:2 | p | tests5.cpp:27:25:27:38 | call to createLSParser | tests5.cpp:29:2:29:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests5.cpp:27:25:27:38 | call to createLSParser | XML parser |
+| tests5.cpp:43:2:43:2 | p | tests5.cpp:40:25:40:38 | call to createLSParser | tests5.cpp:43:2:43:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests5.cpp:40:25:40:38 | call to createLSParser | XML parser |
+| tests5.cpp:59:2:59:2 | p | tests5.cpp:55:25:55:38 | call to createLSParser | tests5.cpp:59:2:59:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests5.cpp:55:25:55:38 | call to createLSParser | XML parser |
+| tests5.cpp:83:2:83:2 | p | tests5.cpp:81:25:81:38 | call to createLSParser | tests5.cpp:83:2:83:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests5.cpp:81:25:81:38 | call to createLSParser | XML parser |
+| tests5.cpp:89:2:89:2 | p | tests5.cpp:81:25:81:38 | call to createLSParser | tests5.cpp:89:2:89:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests5.cpp:81:25:81:38 | call to createLSParser | XML parser |
 | tests.cpp:17:2:17:2 | p | tests.cpp:15:23:15:43 | XercesDOMParser output argument | tests.cpp:17:2:17:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests.cpp:15:23:15:43 | XercesDOMParser output argument | XML parser |
 | tests.cpp:31:2:31:2 | p | tests.cpp:28:23:28:43 | XercesDOMParser output argument | tests.cpp:31:2:31:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests.cpp:28:23:28:43 | XercesDOMParser output argument | XML parser |
 | tests.cpp:39:2:39:2 | p | tests.cpp:35:23:35:43 | XercesDOMParser output argument | tests.cpp:39:2:39:2 | p | This $@ is not configured to prevent an XML external entity (XXE) attack. | tests.cpp:35:23:35:43 | XercesDOMParser output argument | XML parser |

cpp/ql/test/query-tests/Security/CWE/CWE-611/tests.h

@@ -21,5 +21,6 @@ class XMLUni
 {
 public:
 	static const XMLCh fgXercesDisableDefaultEntityResolution[];
+	static const XMLCh fgXercesHarmlessOption[];
 };
 

cpp/ql/test/query-tests/Security/CWE/CWE-611/tests3.cpp

@@ -55,3 +55,28 @@ void test3_5(InputSource &data) {
 	test3_5_init();
 	p_3_5->parse(data); // GOOD
 }
+
+void test3_6(InputSource &data) {
+	SAX2XMLReader *p = XMLReaderFactory::createXMLReader();
+
+	p->setFeature(XMLUni::fgXercesDisableDefaultEntityResolution, false);
+	p->parse(data); // BAD (parser not correctly configured)
+}
+
+void test3_7(InputSource &data) {
+	SAX2XMLReader *p = XMLReaderFactory::createXMLReader();
+
+	p->setFeature(XMLUni::fgXercesHarmlessOption, true);
+	p->parse(data); // BAD (parser not correctly configured)
+}
+
+void test3_8(InputSource &data) {
+	SAX2XMLReader *p = XMLReaderFactory::createXMLReader();
+	const XMLCh *feature = XMLUni::fgXercesDisableDefaultEntityResolution;
+
+	p->setFeature(feature, true);
+	p->parse(data); // GOOD
+}
+
+
+

cpp/ql/test/query-tests/Security/CWE/CWE-611/tests5.cpp

@@ -26,7 +26,7 @@ public:
 void test5_1(DOMImplementationLS *impl, InputSource &data) {
 	DOMLSParser *p = impl->createLSParser();
 
-	p->parse(data); // BAD (parser not correctly configured) [NOT DETECTED]
+	p->parse(data); // BAD (parser not correctly configured)
 }
 
 void test5_2(DOMImplementationLS *impl, InputSource &data) {
@@ -40,7 +40,7 @@ void test5_3(DOMImplementationLS *impl, InputSource &data) {
 	DOMLSParser *p = impl->createLSParser();
 
 	p->getDomConfig()->setParameter(XMLUni::fgXercesDisableDefaultEntityResolution, false);
-	p->parse(data); // BAD (parser not correctly configured) [NOT DETECTED]
+	p->parse(data); // BAD (parser not correctly configured)
 }
 
 void test5_4(DOMImplementationLS *impl, InputSource &data) {
@@ -56,7 +56,7 @@ void test5_5(DOMImplementationLS *impl, InputSource &data) {
 	DOMConfiguration *cfg = p->getDomConfig();
 
 	cfg->setParameter(XMLUni::fgXercesDisableDefaultEntityResolution, false);
-	p->parse(data); // BAD (parser not correctly configured) [NOT DETECTED]
+	p->parse(data); // BAD (parser not correctly configured)
 }
 
 DOMImplementationLS *g_impl;
@@ -76,3 +76,28 @@ void test5_6() {
 	g_p1->parse(*g_data); // GOOD
 	g_p2->parse(*g_data); // BAD (parser not correctly configured) [NOT DETECTED]
 }
+
+void test5_7(DOMImplementationLS *impl, InputSource &data) {
+	DOMLSParser *p = impl->createLSParser();
+
+	p->parse(data); // BAD (parser not correctly configured)
+
+	p->getDomConfig()->setParameter(XMLUni::fgXercesDisableDefaultEntityResolution, true);
+	p->parse(data); // GOOD
+
+	p->getDomConfig()->setParameter(XMLUni::fgXercesDisableDefaultEntityResolution, false);
+	p->parse(data); // BAD (parser not correctly configured)
+}
+
+void test5_8(DOMImplementationLS *impl, InputSource &data) {
+	DOMLSParser *p = impl->createLSParser();
+	DOMConfiguration *cfg = p->getDomConfig();
+
+	p->parse(data); // BAD (parser not correctly configured) [NOT DETECTED]
+
+	cfg->setParameter(XMLUni::fgXercesDisableDefaultEntityResolution, true);
+	p->parse(data); // GOOD
+
+	cfg->setParameter(XMLUni::fgXercesDisableDefaultEntityResolution, false);
+	p->parse(data); // BAD (parser not correctly configured) [NOT DETECTED]
+}

cpp/ql/test/successor-tests/ifstmt/ifstmt/ifstmt.cpp

@@ -0,0 +1,8 @@
+void normal(int x, int y) {
+  if(int z = y; x == z) {
+    l1:;
+  }
+  l2:;
+}
+
+// semmle-extractor-options: -std=c++17

cpp/ql/test/successor-tests/ifstmt/ifstmt/ifstmt01.expected

@@ -1 +1,2 @@
 | ifstmt.c:28:6:28:11 | ... == ... | l2 |
+| ifstmt.cpp:2:17:2:22 | ... == ... | l2 |

cpp/ql/test/successor-tests/ifstmt/ifstmt/ifstmt02.expected

@@ -1 +1,2 @@
 | ifstmt.c:28:6:28:11 | ... == ... | l1 |
+| ifstmt.cpp:2:17:2:22 | ... == ... | l1 |

cpp/ql/test/successor-tests/ifstmt/ifstmt/ifstmt04.expected

@@ -1 +1,2 @@
 | ifstmt.c:29:8:29:8 | ; | l2 |
+| ifstmt.cpp:3:8:3:8 | ; | l2 |

cpp/ql/test/successor-tests/ifstmt/ifstmt/ifstmt05.ql

@@ -1,6 +1,9 @@
 /**
  * @name ifstmt05
- * @description Every if statement has its condition or one of the condition's descendants as its unique successor.
+ * @description Every if statement with an initialization has the initialization or one of the
+ *              initialization's descendants as its unique successor. Every if statement without
+ *              and initialization has its condition or one of the condition's descendants as
+ *              its unique successor.
  */
 
 import cpp
@@ -8,7 +11,11 @@ import cpp
 from IfStmt is
 where
   not (
-    is.getASuccessor() = is.getCondition().getAChild*() and
+    (
+      if exists(is.getInitialization())
+      then is.getASuccessor() = is.getInitialization().getAChild*()
+      else is.getASuccessor() = is.getCondition().getAChild*()
+    ) and
     count(is.getASuccessor()) = 1
   )
 select is

cpp/ql/test/successor-tests/ifstmt/ifstmt/ifstmt10.expected

@@ -1,12 +1,27 @@
 | 0 | ifstmt.c:27:27:32:1 | { ... } | 1 | ifstmt.c:28:3:30:3 | if (...) ...  |
+| 0 | ifstmt.cpp:1:27:6:1 | { ... } | 1 | ifstmt.cpp:2:3:4:3 | if (...) ...  |
 | 1 | ifstmt.c:28:3:30:3 | if (...) ...  | 1 | ifstmt.c:28:6:28:6 | x |
 | 1 | ifstmt.c:28:6:28:6 | x | 1 | ifstmt.c:28:11:28:11 | y |
 | 1 | ifstmt.c:28:6:28:11 | ... == ... | 1 | ifstmt.c:28:14:30:3 | { ... } |
 | 1 | ifstmt.c:28:6:28:11 | ... == ... | 4 | ifstmt.c:31:3:31:5 | label ...: |
 | 1 | ifstmt.c:28:11:28:11 | y | 1 | ifstmt.c:28:6:28:11 | ... == ... |
 | 1 | ifstmt.c:28:14:30:3 | { ... } | 2 | ifstmt.c:29:5:29:7 | label ...: |
+| 1 | ifstmt.cpp:2:3:4:3 | if (...) ...  | 1 | ifstmt.cpp:2:6:2:6 | declaration |
+| 1 | ifstmt.cpp:2:6:2:6 | declaration | 1 | ifstmt.cpp:2:13:2:14 | initializer for z |
+| 1 | ifstmt.cpp:2:13:2:14 | initializer for z | 1 | ifstmt.cpp:2:14:2:14 | y |
+| 1 | ifstmt.cpp:2:14:2:14 | y | 1 | ifstmt.cpp:2:17:2:17 | x |
+| 1 | ifstmt.cpp:2:17:2:17 | x | 1 | ifstmt.cpp:2:22:2:22 | z |
+| 1 | ifstmt.cpp:2:17:2:22 | ... == ... | 1 | ifstmt.cpp:2:25:4:3 | { ... } |
+| 1 | ifstmt.cpp:2:17:2:22 | ... == ... | 4 | ifstmt.cpp:5:3:5:5 | label ...: |
+| 1 | ifstmt.cpp:2:22:2:22 | z | 1 | ifstmt.cpp:2:17:2:22 | ... == ... |
+| 1 | ifstmt.cpp:2:25:4:3 | { ... } | 2 | ifstmt.cpp:3:5:3:7 | label ...: |
 | 2 | ifstmt.c:29:5:29:7 | label ...: | 2 | ifstmt.c:29:8:29:8 | ; |
 | 2 | ifstmt.c:29:8:29:8 | ; | 4 | ifstmt.c:31:3:31:5 | label ...: |
+| 2 | ifstmt.cpp:3:5:3:7 | label ...: | 2 | ifstmt.cpp:3:8:3:8 | ; |
+| 2 | ifstmt.cpp:3:8:3:8 | ; | 4 | ifstmt.cpp:5:3:5:5 | label ...: |
 | 4 | ifstmt.c:31:3:31:5 | label ...: | 4 | ifstmt.c:31:6:31:6 | ; |
 | 4 | ifstmt.c:31:6:31:6 | ; | 5 | ifstmt.c:32:1:32:1 | return ... |
+| 4 | ifstmt.cpp:5:3:5:5 | label ...: | 4 | ifstmt.cpp:5:6:5:6 | ; |
+| 4 | ifstmt.cpp:5:6:5:6 | ; | 5 | ifstmt.cpp:6:1:6:1 | return ... |
 | 5 | ifstmt.c:32:1:32:1 | return ... | 0 | ifstmt.c:27:6:27:11 | normal |
+| 5 | ifstmt.cpp:6:1:6:1 | return ... | 0 | ifstmt.cpp:1:6:1:11 | normal |

cpp/ql/test/successor-tests/ifstmt/ifstmt/ifstmt11.ql

@@ -0,0 +1,15 @@
+/**
+ * @name ifstmt11
+ * @description If an initialization exists, then the condition is a successor of the initialization.
+ */
+
+import cpp
+
+from IfStmt is, Expr e, Stmt s, ControlFlowNode n
+where
+  s = is.getInitialization() and
+  e = is.getCondition() and
+  n = s.getASuccessor*() and
+  not exists(ControlFlowNode m | m = e.getASuccessor*() | m = n) and
+  not exists(ControlFlowNode m | m = e.getAPredecessor*() | m = n)
+select is

cpp/ql/test/successor-tests/switchstmt/switchstmt/cfg.expected

@@ -12,3 +12,20 @@
 | switchstmt | switchstmt.c:1:6:1:6 | f | 7 | 8 | switchstmt.c:7:5:7:5 | switchstmt.c:7:5:7:5 | switchstmt.c:7:5:7:5 | ; | 8: return ... |
 | switchstmt | switchstmt.c:1:6:1:6 | f | 8 | 9 | switchstmt.c:8:1:8:1 | switchstmt.c:8:1:8:1 | switchstmt.c:8:1:8:1 | return ... | 8: f |
 | switchstmt | switchstmt.c:1:6:1:6 | f | 8 | 10 | switchstmt.c:1:6:1:6 | switchstmt.c:1:6:1:6 | switchstmt.c:1:6:1:6 | f | <none> |
+| switchstmt | switchstmt.cpp:3:6:3:6 | g | 3 | 1 | switchstmt.cpp:3:15:12:1 | switchstmt.cpp:3:15:12:1 | switchstmt.cpp:3:15:12:1 | { ... } | 4: switch (...) ...  |
+| switchstmt | switchstmt.cpp:3:6:3:6 | g | 4 | 2 | switchstmt.cpp:4:6:10:5 | switchstmt.cpp:4:6:10:5 | switchstmt.cpp:4:6:10:5 | switch (...) ...  | 5: declaration |
+| switchstmt | switchstmt.cpp:3:6:3:6 | g | 5 | 3 | switchstmt.cpp:5:10:5:10 | switchstmt.cpp:5:10:5:10 | switchstmt.cpp:5:10:5:10 | declaration | 5: initializer for y |
+| switchstmt | switchstmt.cpp:3:6:3:6 | g | 5 | 4 | switchstmt.cpp:5:17:5:18 | switchstmt.cpp:5:17:5:18 | switchstmt.cpp:5:17:5:18 | initializer for y | 5: x |
+| switchstmt | switchstmt.cpp:3:6:3:6 | g | 5 | 5 | switchstmt.cpp:5:18:5:18 | switchstmt.cpp:5:18:5:18 | switchstmt.cpp:5:18:5:18 | x | 6: y |
+| switchstmt | switchstmt.cpp:3:6:3:6 | g | 6 | 6 | switchstmt.cpp:6:10:6:10 | switchstmt.cpp:6:10:6:10 | switchstmt.cpp:6:10:6:10 | y | 6: { ... } |
+| switchstmt | switchstmt.cpp:3:6:3:6 | g | 6 | 7 | switchstmt.cpp:6:13:10:5 | switchstmt.cpp:6:13:10:5 | switchstmt.cpp:6:13:10:5 | { ... } | 7: case ...: |
+| switchstmt | switchstmt.cpp:3:6:3:6 | g | 6 | 7 | switchstmt.cpp:6:13:10:5 | switchstmt.cpp:6:13:10:5 | switchstmt.cpp:6:13:10:5 | { ... } | 8: case ...: |
+| switchstmt | switchstmt.cpp:3:6:3:6 | g | 6 | 7 | switchstmt.cpp:6:13:10:5 | switchstmt.cpp:6:13:10:5 | switchstmt.cpp:6:13:10:5 | { ... } | 9: default:  |
+| switchstmt | switchstmt.cpp:3:6:3:6 | g | 7 | 1 | switchstmt.cpp:7:14:7:14 | switchstmt.cpp:7:14:7:14 | switchstmt.cpp:7:14:7:14 | 1 | <none> |
+| switchstmt | switchstmt.cpp:3:6:3:6 | g | 7 | 8 | switchstmt.cpp:7:9:7:15 | switchstmt.cpp:7:9:7:15 | switchstmt.cpp:7:9:7:15 | case ...: | 8: case ...: |
+| switchstmt | switchstmt.cpp:3:6:3:6 | g | 8 | 1 | switchstmt.cpp:8:14:8:14 | switchstmt.cpp:8:14:8:14 | switchstmt.cpp:8:14:8:14 | 2 | <none> |
+| switchstmt | switchstmt.cpp:3:6:3:6 | g | 8 | 9 | switchstmt.cpp:8:9:8:15 | switchstmt.cpp:8:9:8:15 | switchstmt.cpp:8:9:8:15 | case ...: | 9: default:  |
+| switchstmt | switchstmt.cpp:3:6:3:6 | g | 9 | 10 | switchstmt.cpp:9:9:9:16 | switchstmt.cpp:9:9:9:16 | switchstmt.cpp:9:9:9:16 | default:  | 11: ; |
+| switchstmt | switchstmt.cpp:3:6:3:6 | g | 11 | 11 | switchstmt.cpp:11:5:11:5 | switchstmt.cpp:11:5:11:5 | switchstmt.cpp:11:5:11:5 | ; | 12: return ... |
+| switchstmt | switchstmt.cpp:3:6:3:6 | g | 12 | 12 | switchstmt.cpp:12:1:12:1 | switchstmt.cpp:12:1:12:1 | switchstmt.cpp:12:1:12:1 | return ... | 12: g |
+| switchstmt | switchstmt.cpp:3:6:3:6 | g | 12 | 13 | switchstmt.cpp:3:6:3:6 | switchstmt.cpp:3:6:3:6 | switchstmt.cpp:3:6:3:6 | g | <none> |

cpp/ql/test/successor-tests/switchstmt/switchstmt/switchstmt.cpp

@@ -0,0 +1,12 @@
+// semmle-extractor-options: -std=c++17
+
+void g(int x) {
+     switch (
+         int y = x;
+         y) {
+        case 1:
+        case 2:
+        default:
+    }
+    ;
+}

csharp/ql/lib/semmle/code/csharp/dataflow/internal/AccessPathSyntax.qll

@@ -167,9 +167,16 @@ class AccessPathToken extends string {
   /** Gets the `n`th argument to this token, such as `x` or `y` from `Member[x,y]`. */
   string getArgument(int n) { result = this.getArgumentList().splitAt(",", n).trim() }
 
+  /** Gets the `n`th argument to this `name` token, such as `x` or `y` from `Member[x,y]`. */
+  pragma[nomagic]
+  string getArgument(string name, int n) { name = this.getName() and result = this.getArgument(n) }
+
   /** Gets an argument to this token, such as `x` or `y` from `Member[x,y]`. */
   string getAnArgument() { result = this.getArgument(_) }
 
+  /** Gets an argument to this `name` token, such as `x` or `y` from `Member[x,y]`. */
+  string getAnArgument(string name) { result = this.getArgument(name, _) }
+
   /** Gets the number of arguments to this token, such as 2 for `Member[x,y]` or zero for `ReturnValue`. */
   int getNumArgument() { result = count(int n | exists(this.getArgument(n))) }
 }

csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl.qll

@@ -170,6 +170,14 @@ abstract class Configuration extends string {
    */
   int explorationLimit() { none() }
 
+  /**
+   * Holds if hidden nodes should be included in the data flow graph.
+   *
+   * This feature should only be used for debugging or when the data flow graph
+   * is not visualized (for example in a `path-problem` query).
+   */
+  predicate includeHiddenNodes() { none() }
+
   /**
    * Holds if there is a partial data flow path from `source` to `node`. The
    * approximate distance between `node` and the closest source is `dist` and
@@ -1673,10 +1681,24 @@ private module Stage2 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -2495,10 +2517,24 @@ private module Stage3 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -3322,10 +3358,24 @@ private module Stage4 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -3394,17 +3444,28 @@ private Configuration unbindConf(Configuration conf) {
   exists(Configuration c | result = pragma[only_bind_into](c) and conf = pragma[only_bind_into](c))
 }
 
-private predicate nodeMayUseSummary(
-  NodeEx n, FlowState state, AccessPathApprox apa, Configuration config
+pragma[nomagic]
+private predicate nodeMayUseSummary0(
+  NodeEx n, DataFlowCallable c, FlowState state, AccessPathApprox apa, Configuration config
 ) {
-  exists(DataFlowCallable c, AccessPathApprox apa0 |
-    Stage4::parameterMayFlowThrough(_, c, apa, _) and
+  exists(AccessPathApprox apa0 |
+    Stage4::parameterMayFlowThrough(_, c, _, _) and
     Stage4::revFlow(n, state, true, _, apa0, config) and
     Stage4::fwdFlow(n, state, any(CallContextCall ccc), TAccessPathApproxSome(apa), apa0, config) and
     n.getEnclosingCallable() = c
   )
 }
 
+pragma[nomagic]
+private predicate nodeMayUseSummary(
+  NodeEx n, FlowState state, AccessPathApprox apa, Configuration config
+) {
+  exists(DataFlowCallable c |
+    Stage4::parameterMayFlowThrough(_, c, apa, config) and
+    nodeMayUseSummary0(n, c, state, apa, config)
+  )
+}
+
 private newtype TSummaryCtx =
   TSummaryCtxNone() or
   TSummaryCtxSome(ParamNodeEx p, FlowState state, AccessPath ap) {
@@ -3600,7 +3661,7 @@ private newtype TPathNode =
  * of dereference operations needed to get from the value in the node to the
  * tracked object. The final type indicates the type of the tracked object.
  */
-abstract private class AccessPath extends TAccessPath {
+private class AccessPath extends TAccessPath {
   /** Gets the head of this access path, if any. */
   abstract TypedContent getHead();
 
@@ -3815,11 +3876,14 @@ abstract private class PathNodeImpl extends PathNode {
   abstract NodeEx getNodeEx();
 
   predicate isHidden() {
-    hiddenNode(this.getNodeEx().asNode()) and
-    not this.isSource() and
-    not this instanceof PathNodeSink
-    or
-    this.getNodeEx() instanceof TNodeImplicitRead
+    not this.getConfiguration().includeHiddenNodes() and
+    (
+      hiddenNode(this.getNodeEx().asNode()) and
+      not this.isSource() and
+      not this instanceof PathNodeSink
+      or
+      this.getNodeEx() instanceof TNodeImplicitRead
+    )
   }
 
   private string ppAp() {

csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl2.qll

@@ -170,6 +170,14 @@ abstract class Configuration extends string {
    */
   int explorationLimit() { none() }
 
+  /**
+   * Holds if hidden nodes should be included in the data flow graph.
+   *
+   * This feature should only be used for debugging or when the data flow graph
+   * is not visualized (for example in a `path-problem` query).
+   */
+  predicate includeHiddenNodes() { none() }
+
   /**
    * Holds if there is a partial data flow path from `source` to `node`. The
    * approximate distance between `node` and the closest source is `dist` and
@@ -1673,10 +1681,24 @@ private module Stage2 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -2495,10 +2517,24 @@ private module Stage3 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -3322,10 +3358,24 @@ private module Stage4 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -3394,17 +3444,28 @@ private Configuration unbindConf(Configuration conf) {
   exists(Configuration c | result = pragma[only_bind_into](c) and conf = pragma[only_bind_into](c))
 }
 
-private predicate nodeMayUseSummary(
-  NodeEx n, FlowState state, AccessPathApprox apa, Configuration config
+pragma[nomagic]
+private predicate nodeMayUseSummary0(
+  NodeEx n, DataFlowCallable c, FlowState state, AccessPathApprox apa, Configuration config
 ) {
-  exists(DataFlowCallable c, AccessPathApprox apa0 |
-    Stage4::parameterMayFlowThrough(_, c, apa, _) and
+  exists(AccessPathApprox apa0 |
+    Stage4::parameterMayFlowThrough(_, c, _, _) and
     Stage4::revFlow(n, state, true, _, apa0, config) and
     Stage4::fwdFlow(n, state, any(CallContextCall ccc), TAccessPathApproxSome(apa), apa0, config) and
     n.getEnclosingCallable() = c
   )
 }
 
+pragma[nomagic]
+private predicate nodeMayUseSummary(
+  NodeEx n, FlowState state, AccessPathApprox apa, Configuration config
+) {
+  exists(DataFlowCallable c |
+    Stage4::parameterMayFlowThrough(_, c, apa, config) and
+    nodeMayUseSummary0(n, c, state, apa, config)
+  )
+}
+
 private newtype TSummaryCtx =
   TSummaryCtxNone() or
   TSummaryCtxSome(ParamNodeEx p, FlowState state, AccessPath ap) {
@@ -3600,7 +3661,7 @@ private newtype TPathNode =
  * of dereference operations needed to get from the value in the node to the
  * tracked object. The final type indicates the type of the tracked object.
  */
-abstract private class AccessPath extends TAccessPath {
+private class AccessPath extends TAccessPath {
   /** Gets the head of this access path, if any. */
   abstract TypedContent getHead();
 
@@ -3815,11 +3876,14 @@ abstract private class PathNodeImpl extends PathNode {
   abstract NodeEx getNodeEx();
 
   predicate isHidden() {
-    hiddenNode(this.getNodeEx().asNode()) and
-    not this.isSource() and
-    not this instanceof PathNodeSink
-    or
-    this.getNodeEx() instanceof TNodeImplicitRead
+    not this.getConfiguration().includeHiddenNodes() and
+    (
+      hiddenNode(this.getNodeEx().asNode()) and
+      not this.isSource() and
+      not this instanceof PathNodeSink
+      or
+      this.getNodeEx() instanceof TNodeImplicitRead
+    )
   }
 
   private string ppAp() {

csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl3.qll

@@ -170,6 +170,14 @@ abstract class Configuration extends string {
    */
   int explorationLimit() { none() }
 
+  /**
+   * Holds if hidden nodes should be included in the data flow graph.
+   *
+   * This feature should only be used for debugging or when the data flow graph
+   * is not visualized (for example in a `path-problem` query).
+   */
+  predicate includeHiddenNodes() { none() }
+
   /**
    * Holds if there is a partial data flow path from `source` to `node`. The
    * approximate distance between `node` and the closest source is `dist` and
@@ -1673,10 +1681,24 @@ private module Stage2 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -2495,10 +2517,24 @@ private module Stage3 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -3322,10 +3358,24 @@ private module Stage4 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -3394,17 +3444,28 @@ private Configuration unbindConf(Configuration conf) {
   exists(Configuration c | result = pragma[only_bind_into](c) and conf = pragma[only_bind_into](c))
 }
 
-private predicate nodeMayUseSummary(
-  NodeEx n, FlowState state, AccessPathApprox apa, Configuration config
+pragma[nomagic]
+private predicate nodeMayUseSummary0(
+  NodeEx n, DataFlowCallable c, FlowState state, AccessPathApprox apa, Configuration config
 ) {
-  exists(DataFlowCallable c, AccessPathApprox apa0 |
-    Stage4::parameterMayFlowThrough(_, c, apa, _) and
+  exists(AccessPathApprox apa0 |
+    Stage4::parameterMayFlowThrough(_, c, _, _) and
     Stage4::revFlow(n, state, true, _, apa0, config) and
     Stage4::fwdFlow(n, state, any(CallContextCall ccc), TAccessPathApproxSome(apa), apa0, config) and
     n.getEnclosingCallable() = c
   )
 }
 
+pragma[nomagic]
+private predicate nodeMayUseSummary(
+  NodeEx n, FlowState state, AccessPathApprox apa, Configuration config
+) {
+  exists(DataFlowCallable c |
+    Stage4::parameterMayFlowThrough(_, c, apa, config) and
+    nodeMayUseSummary0(n, c, state, apa, config)
+  )
+}
+
 private newtype TSummaryCtx =
   TSummaryCtxNone() or
   TSummaryCtxSome(ParamNodeEx p, FlowState state, AccessPath ap) {
@@ -3600,7 +3661,7 @@ private newtype TPathNode =
  * of dereference operations needed to get from the value in the node to the
  * tracked object. The final type indicates the type of the tracked object.
  */
-abstract private class AccessPath extends TAccessPath {
+private class AccessPath extends TAccessPath {
   /** Gets the head of this access path, if any. */
   abstract TypedContent getHead();
 
@@ -3815,11 +3876,14 @@ abstract private class PathNodeImpl extends PathNode {
   abstract NodeEx getNodeEx();
 
   predicate isHidden() {
-    hiddenNode(this.getNodeEx().asNode()) and
-    not this.isSource() and
-    not this instanceof PathNodeSink
-    or
-    this.getNodeEx() instanceof TNodeImplicitRead
+    not this.getConfiguration().includeHiddenNodes() and
+    (
+      hiddenNode(this.getNodeEx().asNode()) and
+      not this.isSource() and
+      not this instanceof PathNodeSink
+      or
+      this.getNodeEx() instanceof TNodeImplicitRead
+    )
   }
 
   private string ppAp() {

csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl4.qll

@@ -170,6 +170,14 @@ abstract class Configuration extends string {
    */
   int explorationLimit() { none() }
 
+  /**
+   * Holds if hidden nodes should be included in the data flow graph.
+   *
+   * This feature should only be used for debugging or when the data flow graph
+   * is not visualized (for example in a `path-problem` query).
+   */
+  predicate includeHiddenNodes() { none() }
+
   /**
    * Holds if there is a partial data flow path from `source` to `node`. The
    * approximate distance between `node` and the closest source is `dist` and
@@ -1673,10 +1681,24 @@ private module Stage2 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -2495,10 +2517,24 @@ private module Stage3 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -3322,10 +3358,24 @@ private module Stage4 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -3394,17 +3444,28 @@ private Configuration unbindConf(Configuration conf) {
   exists(Configuration c | result = pragma[only_bind_into](c) and conf = pragma[only_bind_into](c))
 }
 
-private predicate nodeMayUseSummary(
-  NodeEx n, FlowState state, AccessPathApprox apa, Configuration config
+pragma[nomagic]
+private predicate nodeMayUseSummary0(
+  NodeEx n, DataFlowCallable c, FlowState state, AccessPathApprox apa, Configuration config
 ) {
-  exists(DataFlowCallable c, AccessPathApprox apa0 |
-    Stage4::parameterMayFlowThrough(_, c, apa, _) and
+  exists(AccessPathApprox apa0 |
+    Stage4::parameterMayFlowThrough(_, c, _, _) and
     Stage4::revFlow(n, state, true, _, apa0, config) and
     Stage4::fwdFlow(n, state, any(CallContextCall ccc), TAccessPathApproxSome(apa), apa0, config) and
     n.getEnclosingCallable() = c
   )
 }
 
+pragma[nomagic]
+private predicate nodeMayUseSummary(
+  NodeEx n, FlowState state, AccessPathApprox apa, Configuration config
+) {
+  exists(DataFlowCallable c |
+    Stage4::parameterMayFlowThrough(_, c, apa, config) and
+    nodeMayUseSummary0(n, c, state, apa, config)
+  )
+}
+
 private newtype TSummaryCtx =
   TSummaryCtxNone() or
   TSummaryCtxSome(ParamNodeEx p, FlowState state, AccessPath ap) {
@@ -3600,7 +3661,7 @@ private newtype TPathNode =
  * of dereference operations needed to get from the value in the node to the
  * tracked object. The final type indicates the type of the tracked object.
  */
-abstract private class AccessPath extends TAccessPath {
+private class AccessPath extends TAccessPath {
   /** Gets the head of this access path, if any. */
   abstract TypedContent getHead();
 
@@ -3815,11 +3876,14 @@ abstract private class PathNodeImpl extends PathNode {
   abstract NodeEx getNodeEx();
 
   predicate isHidden() {
-    hiddenNode(this.getNodeEx().asNode()) and
-    not this.isSource() and
-    not this instanceof PathNodeSink
-    or
-    this.getNodeEx() instanceof TNodeImplicitRead
+    not this.getConfiguration().includeHiddenNodes() and
+    (
+      hiddenNode(this.getNodeEx().asNode()) and
+      not this.isSource() and
+      not this instanceof PathNodeSink
+      or
+      this.getNodeEx() instanceof TNodeImplicitRead
+    )
   }
 
   private string ppAp() {

csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl5.qll

@@ -170,6 +170,14 @@ abstract class Configuration extends string {
    */
   int explorationLimit() { none() }
 
+  /**
+   * Holds if hidden nodes should be included in the data flow graph.
+   *
+   * This feature should only be used for debugging or when the data flow graph
+   * is not visualized (for example in a `path-problem` query).
+   */
+  predicate includeHiddenNodes() { none() }
+
   /**
    * Holds if there is a partial data flow path from `source` to `node`. The
    * approximate distance between `node` and the closest source is `dist` and
@@ -1673,10 +1681,24 @@ private module Stage2 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -2495,10 +2517,24 @@ private module Stage3 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -3322,10 +3358,24 @@ private module Stage4 {
     storeStepFwd(_, ap, tc, _, _, config)
   }
 
-  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+  private predicate revConsCand(TypedContent tc, Ap ap, Configuration config) {
     storeStepCand(_, ap, tc, _, _, config)
   }
 
+  private predicate validAp(Ap ap, Configuration config) {
+    revFlow(_, _, _, _, ap, config) and ap instanceof ApNil
+    or
+    exists(TypedContent head, Ap tail |
+      consCand(head, tail, config) and
+      ap = apCons(head, tail)
+    )
+  }
+
+  predicate consCand(TypedContent tc, Ap ap, Configuration config) {
+    revConsCand(tc, ap, config) and
+    validAp(ap, config)
+  }
+
   pragma[noinline]
   private predicate parameterFlow(
     ParamNodeEx p, Ap ap, Ap ap0, DataFlowCallable c, Configuration config
@@ -3394,17 +3444,28 @@ private Configuration unbindConf(Configuration conf) {
   exists(Configuration c | result = pragma[only_bind_into](c) and conf = pragma[only_bind_into](c))
 }
 
-private predicate nodeMayUseSummary(
-  NodeEx n, FlowState state, AccessPathApprox apa, Configuration config
+pragma[nomagic]
+private predicate nodeMayUseSummary0(
+  NodeEx n, DataFlowCallable c, FlowState state, AccessPathApprox apa, Configuration config
 ) {
-  exists(DataFlowCallable c, AccessPathApprox apa0 |
-    Stage4::parameterMayFlowThrough(_, c, apa, _) and
+  exists(AccessPathApprox apa0 |
+    Stage4::parameterMayFlowThrough(_, c, _, _) and
     Stage4::revFlow(n, state, true, _, apa0, config) and
     Stage4::fwdFlow(n, state, any(CallContextCall ccc), TAccessPathApproxSome(apa), apa0, config) and
     n.getEnclosingCallable() = c
   )
 }
 
+pragma[nomagic]
+private predicate nodeMayUseSummary(
+  NodeEx n, FlowState state, AccessPathApprox apa, Configuration config
+) {
+  exists(DataFlowCallable c |
+    Stage4::parameterMayFlowThrough(_, c, apa, config) and
+    nodeMayUseSummary0(n, c, state, apa, config)
+  )
+}
+
 private newtype TSummaryCtx =
   TSummaryCtxNone() or
   TSummaryCtxSome(ParamNodeEx p, FlowState state, AccessPath ap) {
@@ -3600,7 +3661,7 @@ private newtype TPathNode =
  * of dereference operations needed to get from the value in the node to the
  * tracked object. The final type indicates the type of the tracked object.
  */
-abstract private class AccessPath extends TAccessPath {
+private class AccessPath extends TAccessPath {
   /** Gets the head of this access path, if any. */
   abstract TypedContent getHead();
 
@@ -3815,11 +3876,14 @@ abstract private class PathNodeImpl extends PathNode {
   abstract NodeEx getNodeEx();
 
   predicate isHidden() {
-    hiddenNode(this.getNodeEx().asNode()) and
-    not this.isSource() and
-    not this instanceof PathNodeSink
-    or
-    this.getNodeEx() instanceof TNodeImplicitRead
+    not this.getConfiguration().includeHiddenNodes() and
+    (
+      hiddenNode(this.getNodeEx().asNode()) and
+      not this.isSource() and
+      not this instanceof PathNodeSink
+      or
+      this.getNodeEx() instanceof TNodeImplicitRead
+    )
   }
 
   private string ppAp() {

csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowPrivate.qll

@@ -421,7 +421,7 @@ predicate simpleLocalFlowStep(Node nodeFrom, Node nodeTo) {
   or
   exists(Ssa::Definition def |
     LocalFlow::localSsaFlowStepUseUse(def, nodeFrom, nodeTo) and
-    not FlowSummaryImpl::Private::Steps::summaryClearsContentArg(nodeFrom, _) and
+    not FlowSummaryImpl::Private::Steps::prohibitsUseUseFlow(nodeFrom) and
     not LocalFlow::usesInstanceField(def)
   )
   or
@@ -1718,7 +1718,9 @@ predicate clearsContent(Node n, Content c) {
  * Holds if the value that is being tracked is expected to be stored inside content `c`
  * at node `n`.
  */
-predicate expectsContent(Node n, ContentSet c) { none() }
+predicate expectsContent(Node n, ContentSet c) {
+  FlowSummaryImpl::Private::Steps::summaryExpectsContent(n, c)
+}
 
 /**
  * Holds if the node `n` is unreachable when the call context is `call`.

csharp/ql/lib/semmle/code/csharp/dataflow/internal/FlowSummaryImpl.qll

@@ -26,6 +26,10 @@ module Public {
     string toString() {
       exists(ContentSet c | this = TContentSummaryComponent(c) and result = c.toString())
       or
+      exists(ContentSet c | this = TWithoutContentSummaryComponent(c) and result = "without " + c)
+      or
+      exists(ContentSet c | this = TWithContentSummaryComponent(c) and result = "with " + c)
+      or
       exists(ArgumentPosition pos |
         this = TParameterSummaryComponent(pos) and result = "parameter " + pos
       )
@@ -43,6 +47,12 @@ module Public {
     /** Gets a summary component for content `c`. */
     SummaryComponent content(ContentSet c) { result = TContentSummaryComponent(c) }
 
+    /** Gets a summary component where data is not allowed to be stored in `c`. */
+    SummaryComponent withoutContent(ContentSet c) { result = TWithoutContentSummaryComponent(c) }
+
+    /** Gets a summary component where data must be stored in `c`. */
+    SummaryComponent withContent(ContentSet c) { result = TWithContentSummaryComponent(c) }
+
     /** Gets a summary component for a parameter at position `pos`. */
     SummaryComponent parameter(ArgumentPosition pos) { result = TParameterSummaryComponent(pos) }
 
@@ -216,6 +226,8 @@ module Public {
     /**
      * Holds if values stored inside `content` are cleared on objects passed as
      * arguments at position `pos` to this callable.
+     *
+     * TODO: Remove once all languages support `WithoutContent` tokens.
      */
     pragma[nomagic]
     predicate clearsContent(ParameterPosition pos, ContentSet content) { none() }
@@ -239,7 +251,9 @@ module Private {
     TContentSummaryComponent(ContentSet c) or
     TParameterSummaryComponent(ArgumentPosition pos) or
     TArgumentSummaryComponent(ParameterPosition pos) or
-    TReturnSummaryComponent(ReturnKind rk)
+    TReturnSummaryComponent(ReturnKind rk) or
+    TWithoutContentSummaryComponent(ContentSet c) or
+    TWithContentSummaryComponent(ContentSet c)
 
   private TParameterSummaryComponent thisParam() {
     result = TParameterSummaryComponent(instanceParameterPosition())
@@ -301,6 +315,23 @@ module Private {
           SummaryComponentStack::singleton(TArgumentSummaryComponent(_))) and
       preservesValue = preservesValue1.booleanAnd(preservesValue2)
     )
+    or
+    exists(ParameterPosition ppos, ContentSet cs |
+      c.clearsContent(ppos, cs) and
+      input = SummaryComponentStack::push(SummaryComponent::withoutContent(cs), output) and
+      output = SummaryComponentStack::argument(ppos) and
+      preservesValue = true
+    )
+  }
+
+  private class MkClearStack extends RequiredSummaryComponentStack {
+    override predicate required(SummaryComponent head, SummaryComponentStack tail) {
+      exists(SummarizedCallable sc, ParameterPosition ppos, ContentSet cs |
+        sc.clearsContent(ppos, cs) and
+        head = SummaryComponent::withoutContent(cs) and
+        tail = SummaryComponentStack::argument(ppos)
+      )
+    }
   }
 
   /**
@@ -383,10 +414,7 @@ module Private {
 
   private newtype TSummaryNodeState =
     TSummaryNodeInputState(SummaryComponentStack s) { inputState(_, s) } or
-    TSummaryNodeOutputState(SummaryComponentStack s) { outputState(_, s) } or
-    TSummaryNodeClearsContentState(ParameterPosition pos, boolean post) {
-      any(SummarizedCallable sc).clearsContent(pos, _) and post in [false, true]
-    }
+    TSummaryNodeOutputState(SummaryComponentStack s) { outputState(_, s) }
 
   /**
    * A state used to break up (complex) flow summaries into atomic flow steps.
@@ -433,12 +461,6 @@ module Private {
         this = TSummaryNodeOutputState(s) and
         result = "to write: " + s
       )
-      or
-      exists(ParameterPosition pos, boolean post, string postStr |
-        this = TSummaryNodeClearsContentState(pos, post) and
-        (if post = true then postStr = " (post)" else postStr = "") and
-        result = "clear: " + pos + postStr
-      )
     }
   }
 
@@ -462,11 +484,6 @@ module Private {
     not parameterReadState(c, state, _)
     or
     state.isOutputState(c, _)
-    or
-    exists(ParameterPosition pos |
-      c.clearsContent(pos, _) and
-      state = TSummaryNodeClearsContentState(pos, _)
-    )
   }
 
   pragma[noinline]
@@ -502,8 +519,6 @@ module Private {
     parameterReadState(c, _, pos)
     or
     isParameterPostUpdate(_, c, pos)
-    or
-    c.clearsContent(pos, _)
   }
 
   private predicate callbackOutput(
@@ -511,7 +526,7 @@ module Private {
   ) {
     any(SummaryNodeState state).isInputState(c, s) and
     s.head() = TReturnSummaryComponent(rk) and
-    receiver = summaryNodeInputState(c, s.drop(1))
+    receiver = summaryNodeInputState(c, s.tail())
   }
 
   private predicate callbackInput(
@@ -519,7 +534,7 @@ module Private {
   ) {
     any(SummaryNodeState state).isOutputState(c, s) and
     s.head() = TParameterSummaryComponent(pos) and
-    receiver = summaryNodeInputState(c, s.drop(1))
+    receiver = summaryNodeInputState(c, s.tail())
   }
 
   /** Holds if a call targeting `receiver` should be synthesized inside `c`. */
@@ -545,15 +560,21 @@ module Private {
     exists(SummarizedCallable c, SummaryComponentStack s, SummaryComponent head | head = s.head() |
       n = summaryNodeInputState(c, s) and
       (
+        exists(ContentSet cont | result = getContentType(cont) |
+          head = TContentSummaryComponent(cont) or
+          head = TWithContentSummaryComponent(cont)
+        )
+        or
         exists(ContentSet cont |
-          head = TContentSummaryComponent(cont) and result = getContentType(cont)
+          head = TWithoutContentSummaryComponent(cont) and
+          result = getNodeType(summaryNodeInputState(c, s.tail()))
         )
         or
         exists(ReturnKind rk |
           head = TReturnSummaryComponent(rk) and
           result =
             getCallbackReturnType(getNodeType(summaryNodeInputState(pragma[only_bind_out](c),
-                  s.drop(1))), rk)
+                  s.tail())), rk)
         )
       )
       or
@@ -572,16 +593,10 @@ module Private {
         exists(ArgumentPosition pos | head = TParameterSummaryComponent(pos) |
           result =
             getCallbackParameterType(getNodeType(summaryNodeInputState(pragma[only_bind_out](c),
-                  s.drop(1))), pos)
+                  s.tail())), pos)
         )
       )
     )
-    or
-    exists(SummarizedCallable c, ParameterPosition pos, ParamNode p |
-      n = summaryNode(c, TSummaryNodeClearsContentState(pos, false)) and
-      p.isParameterOf(c, pos) and
-      result = getNodeType(p)
-    )
   }
 
   /** Holds if summary node `out` contains output of kind `rk` from call `c`. */
@@ -607,9 +622,6 @@ module Private {
     exists(SummarizedCallable c, ParameterPosition pos |
       isParameterPostUpdate(post, c, pos) and
       pre.(ParamNode).isParameterOf(c, pos)
-      or
-      pre = summaryNode(c, TSummaryNodeClearsContentState(pos, false)) and
-      post = summaryNode(c, TSummaryNodeClearsContentState(pos, true))
     )
     or
     exists(SummarizedCallable callable, SummaryComponentStack s |
@@ -633,8 +645,6 @@ module Private {
    */
   predicate summaryAllowParameterReturnInSelf(ParamNode p) {
     exists(SummarizedCallable c, ParameterPosition ppos | p.isParameterOf(c, ppos) |
-      c.clearsContent(ppos, _)
-      or
       exists(SummaryComponentStack inputContents, SummaryComponentStack outputContents |
         summary(c, inputContents, outputContents, _) and
         inputContents.bottom() = pragma[only_bind_into](TArgumentSummaryComponent(ppos)) and
@@ -663,9 +673,10 @@ module Private {
         preservesValue = false and not summary(c, inputContents, outputContents, true)
       )
       or
-      exists(SummarizedCallable c, ParameterPosition pos |
-        pred.(ParamNode).isParameterOf(c, pos) and
-        succ = summaryNode(c, TSummaryNodeClearsContentState(pos, _)) and
+      exists(SummarizedCallable c, SummaryComponentStack s |
+        pred = summaryNodeInputState(c, s.tail()) and
+        succ = summaryNodeInputState(c, s) and
+        s.head() = [SummaryComponent::withContent(_), SummaryComponent::withoutContent(_)] and
         preservesValue = true
       )
     }
@@ -676,7 +687,7 @@ module Private {
      */
     predicate summaryReadStep(Node pred, ContentSet c, Node succ) {
       exists(SummarizedCallable sc, SummaryComponentStack s |
-        pred = summaryNodeInputState(sc, s.drop(1)) and
+        pred = summaryNodeInputState(sc, s.tail()) and
         succ = summaryNodeInputState(sc, s) and
         SummaryComponent::content(c) = s.head()
       )
@@ -689,7 +700,7 @@ module Private {
     predicate summaryStoreStep(Node pred, ContentSet c, Node succ) {
       exists(SummarizedCallable sc, SummaryComponentStack s |
         pred = summaryNodeOutputState(sc, s) and
-        succ = summaryNodeOutputState(sc, s.drop(1)) and
+        succ = summaryNodeOutputState(sc, s.tail()) and
         SummaryComponent::content(c) = s.head()
       )
     }
@@ -714,9 +725,22 @@ module Private {
      * node where field `b` is cleared).
      */
     predicate summaryClearsContent(Node n, ContentSet c) {
-      exists(SummarizedCallable sc, ParameterPosition pos |
-        n = summaryNode(sc, TSummaryNodeClearsContentState(pos, true)) and
-        sc.clearsContent(pos, c)
+      exists(SummarizedCallable sc, SummaryNodeState state, SummaryComponentStack stack |
+        n = summaryNode(sc, state) and
+        state.isInputState(sc, stack) and
+        stack.head() = SummaryComponent::withoutContent(c)
+      )
+    }
+
+    /**
+     * Holds if the value that is being tracked is expected to be stored inside
+     * content `c` at `n`.
+     */
+    predicate summaryExpectsContent(Node n, ContentSet c) {
+      exists(SummarizedCallable sc, SummaryNodeState state, SummaryComponentStack stack |
+        n = summaryNode(sc, state) and
+        state.isInputState(sc, stack) and
+        stack.head() = SummaryComponent::withContent(c)
       )
     }
 
@@ -728,22 +752,6 @@ module Private {
       sc = viableCallable(call)
     }
 
-    /**
-     * Holds if values stored inside content `c` are cleared inside a
-     * callable to which `arg` is an argument.
-     *
-     * In such cases, it is important to prevent use-use flow out of
-     * `arg` (see comment for `summaryClearsContent`).
-     */
-    pragma[nomagic]
-    predicate summaryClearsContentArg(ArgNode arg, ContentSet c) {
-      exists(DataFlowCall call, SummarizedCallable sc, ParameterPosition ppos |
-        argumentPositionMatch(call, arg, ppos) and
-        viableParam(call, sc, ppos, _) and
-        sc.clearsContent(ppos, c)
-      )
-    }
-
     pragma[nomagic]
     private ParamNode summaryArgParam0(DataFlowCall call, ArgNode arg) {
       exists(ParameterPosition ppos, SummarizedCallable sc |
@@ -752,6 +760,27 @@ module Private {
       )
     }
 
+    /**
+     * Holds if use-use flow starting from `arg` should be prohibited.
+     *
+     * This is the case when `arg` is the argument of a call that targets a
+     * flow summary where the corresponding parameter either clears contents
+     * or expects contents.
+     */
+    pragma[nomagic]
+    predicate prohibitsUseUseFlow(ArgNode arg) {
+      exists(ParamNode p, Node mid, ParameterPosition ppos, Node ret |
+        p = summaryArgParam0(_, arg) and
+        p.isParameterOf(_, ppos) and
+        summaryLocalStep(p, mid, true) and
+        summaryLocalStep(mid, ret, true) and
+        isParameterPostUpdate(ret, _, ppos)
+      |
+        summaryClearsContent(mid, _) or
+        summaryExpectsContent(mid, _)
+      )
+    }
+
     pragma[nomagic]
     private ParamNode summaryArgParam(ArgNode arg, ReturnKindExt rk, OutNodeExt out) {
       exists(DataFlowCall call |
@@ -1152,6 +1181,10 @@ module Private {
         Private::Steps::summaryClearsContent(a.asNode(), c) and
         b = a and
         value = "clear (" + c + ")"
+        or
+        Private::Steps::summaryExpectsContent(a.asNode(), c) and
+        b = a and
+        value = "expect (" + c + ")"
       )
       or
       summaryPostUpdateNode(b.asNode(), a.asNode()) and

csharp/ql/lib/semmle/code/csharp/dataflow/internal/rangeanalysis/SignAnalysisCommon.qll

@@ -49,7 +49,7 @@ private predicate unknownSign(Expr e) {
     or
     exists(LongLiteral lit | lit = e and not exists(lit.getValue().toFloat()))
     or
-    exists(CastExpr cast, Type fromtyp |
+    exists(CastingExpr cast, Type fromtyp |
       cast = e and
       fromtyp = cast.getSourceType() and
       not fromtyp instanceof NumericOrCharType

csharp/ql/lib/semmle/code/csharp/dataflow/internal/rangeanalysis/SignAnalysisSpecific.qll

@@ -29,6 +29,8 @@ module Private {
 
   class CastExpr = RU::ExprNode::CastExpr;
 
+  class CastingExpr = CastExpr;
+
   class Type = CS::Type;
 
   class Expr = CS::ControlFlow::Nodes::ExprNode;

csharp/ql/lib/semmle/code/csharp/frameworks/system/Collections.qll

@@ -55,7 +55,7 @@ private class SystemCollectionsIEnumerableClearFlow extends SummarizedCallable {
   }
 
   override predicate clearsContent(ParameterPosition pos, DataFlow::ContentSet content) {
-    pos.isThisParameter() and
+    (if this.(Modifiable).isStatic() then pos.getPosition() = 0 else pos.isThisParameter()) and
     content instanceof DataFlow::ElementContent
   }
 }

csharp/ql/src/Telemetry/ExternalLibraryUsage.ql

@@ -2,7 +2,7 @@
  * @name External libraries
  * @description A list of external libraries used in the code
  * @kind metric
- * @tags summary
+ * @tags summary telemetry
  * @id csharp/telemetry/external-libs
  */
 

csharp/ql/src/Telemetry/SupportedExternalSinks.ql

@@ -2,7 +2,7 @@
  * @name Supported sinks in external libraries
  * @description A list of 3rd party APIs detected as sinks. Excludes APIs exposed by test libraries.
  * @kind metric
- * @tags summary
+ * @tags summary telemetry
  * @id csharp/telemetry/supported-external-api-sinks
  */
 

csharp/ql/src/Telemetry/SupportedExternalSources.ql

@@ -2,7 +2,7 @@
  * @name Supported sources in external libraries
  * @description A list of 3rd party APIs detected as sources. Excludes APIs exposed by test libraries.
  * @kind metric
- * @tags summary
+ * @tags summary telemetry
  * @id csharp/telemetry/supported-external-api-sources
  */
 

csharp/ql/src/Telemetry/SupportedExternalTaint.ql

@@ -2,7 +2,7 @@
  * @name Supported flow steps in external libraries
  * @description A list of 3rd party APIs detected as flow steps. Excludes APIs exposed by test libraries.
  * @kind metric
- * @tags summary
+ * @tags summary telemetry
  * @id csharp/telemetry/supported-external-api-taint
  */
 

csharp/ql/src/Telemetry/UnsupportedExternalAPIs.ql

@@ -2,7 +2,7 @@
  * @name Usage of unsupported APIs coming from external libraries
  * @description A list of 3rd party APIs used in the codebase. Excludes APIs exposed by test libraries.
  * @kind metric
- * @tags summary
+ * @tags summary telemetry
  * @id csharp/telemetry/unsupported-external-api
  */
 

csharp/ql/test/library-tests/dataflow/external-models/steps.expected

@@ -8,8 +8,6 @@ summaryThroughStep
 | Steps.cs:22:13:22:16 | this access | Steps.cs:22:13:22:30 | call to method StepQualRes | false |
 | Steps.cs:23:13:23:25 | this access | Steps.cs:23:13:23:25 | call to method StepQualRes | false |
 | Steps.cs:26:13:26:31 | this access | Steps.cs:26:25:26:30 | [post] access to local variable argOut | false |
-| Steps.cs:30:13:30:16 | this access | Steps.cs:30:13:30:16 | [post] this access | true |
-| Steps.cs:34:13:34:16 | this access | Steps.cs:34:13:34:16 | [post] this access | true |
 | Steps.cs:41:29:41:29 | 0 | Steps.cs:41:13:41:30 | call to method StepGeneric | true |
 | Steps.cs:42:30:42:34 | false | Steps.cs:42:13:42:35 | call to method StepGeneric2<Boolean> | true |
 | Steps.cs:44:36:44:43 | "string" | Steps.cs:44:13:44:44 | call to method StepOverride | true |

csharp/tools/tracing-config.lua

@@ -1,4 +1,4 @@
-function RegisterExtractorPack()
+function RegisterExtractorPack(id)
     local extractor = GetPlatformToolsDirectory() ..
                           'Semmle.Extraction.CSharp.Driver'
     if OperatingSystem == 'windows' then extractor = extractor .. '.exe' end
@@ -26,14 +26,10 @@ function RegisterExtractorPack()
                 return {
                     replace = true,
                     invocations = {
-                        {
-                            path = compilerPath,
-                            transformedArguments = {
-                                nativeArgumentPointer = compilerArguments['nativeArgumentPointer'],
-                                append = {'/p:UseSharedCompilation=false'},
-                                prepend = {}
-                            }
-                        }
+                        BuildExtractorInvocation(id, compilerPath, compilerPath,
+                                                 compilerArguments, nil, {
+                            '/p:UseSharedCompilation=false'
+                        })
                     }
                 }
             end

docs/codeql/support/reusables/versions-compilers.rst

@@ -16,7 +16,7 @@
    .NET Core up to 3.1
 
    .NET 5, .NET 6","``.sln``, ``.csproj``, ``.cs``, ``.cshtml``, ``.xaml``"
-   Go (aka Golang), "Go up to 1.17", "Go 1.11 or more recent", ``.go``
+   Go (aka Golang), "Go up to 1.18", "Go 1.11 or more recent", ``.go``
    Java,"Java 7 to 18 [4]_","javac (OpenJDK and Oracle JDK),
 
    Eclipse compiler for Java (ECJ) [5]_",``.java``

java/documentation/library-coverage/coverage.csv

@@ -1,118 +1,121 @@
-package,sink,source,summary,sink:bean-validation,sink:create-file,sink:groovy,sink:header-splitting,sink:information-leak,sink:intent-start,sink:jdbc-url,sink:jexl,sink:jndi-injection,sink:ldap,sink:logging,sink:mvel,sink:ognl-injection,sink:open-url,sink:pending-intent-sent,sink:set-hostname-verifier,sink:sql,sink:url-open-stream,sink:url-redirect,sink:write-file,sink:xpath,sink:xslt,sink:xss,source:android-widget,source:contentprovider,source:remote,summary:taint,summary:value
-android.app,16,,103,,,,,,7,,,,,,,,,9,,,,,,,,,,,,18,85
-android.content,24,27,108,,,,,,16,,,,,,,,,,,8,,,,,,,,27,,31,77
-android.database,59,,30,,,,,,,,,,,,,,,,,59,,,,,,,,,,30,
-android.net,,,60,,,,,,,,,,,,,,,,,,,,,,,,,,,45,15
-android.os,,,122,,,,,,,,,,,,,,,,,,,,,,,,,,,41,81
-android.util,6,16,,,,,,,,,,,,6,,,,,,,,,,,,,,,16,,
-android.webkit,3,2,,,,,,,,,,,,,,,,,,,,,,,,3,,,2,,
-android.widget,,1,1,,,,,,,,,,,,,,,,,,,,,,,,1,,,1,
-androidx.slice,2,5,88,,,,,,,,,,,,,,,2,,,,,,,,,,5,,27,61
-cn.hutool.core.codec,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,1,
-com.esotericsoftware.kryo.io,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,1,
-com.esotericsoftware.kryo5.io,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,1,
-com.fasterxml.jackson.core,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,1,
-com.fasterxml.jackson.databind,,,6,,,,,,,,,,,,,,,,,,,,,,,,,,,6,
-com.google.common.base,,,85,,,,,,,,,,,,,,,,,,,,,,,,,,,62,23
-com.google.common.cache,,,17,,,,,,,,,,,,,,,,,,,,,,,,,,,,17
-com.google.common.collect,,,553,,,,,,,,,,,,,,,,,,,,,,,,,,,2,551
-com.google.common.flogger,29,,,,,,,,,,,,,29,,,,,,,,,,,,,,,,,
-com.google.common.io,6,,73,,,,,,,,,,,,,,,,,,6,,,,,,,,,72,1
-com.opensymphony.xwork2.ognl,3,,,,,,,,,,,,,,,3,,,,,,,,,,,,,,,
-com.rabbitmq.client,,21,7,,,,,,,,,,,,,,,,,,,,,,,,,,21,7,
-com.unboundid.ldap.sdk,17,,,,,,,,,,,,17,,,,,,,,,,,,,,,,,,
-com.zaxxer.hikari,2,,,,,,,,,2,,,,,,,,,,,,,,,,,,,,,
-flexjson,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,1
-groovy.lang,26,,,,,26,,,,,,,,,,,,,,,,,,,,,,,,,
-groovy.util,5,,,,,5,,,,,,,,,,,,,,,,,,,,,,,,,
-jakarta.faces.context,2,7,,,,,,,,,,,,,,,,,,,,,,,,2,,,7,,
-jakarta.json,,,123,,,,,,,,,,,,,,,,,,,,,,,,,,,100,23
-jakarta.ws.rs.client,1,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,
-jakarta.ws.rs.container,,9,,,,,,,,,,,,,,,,,,,,,,,,,,,9,,
-jakarta.ws.rs.core,2,,149,,,,,,,,,,,,,,,,,,,2,,,,,,,,94,55
-java.beans,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,1,
-java.io,37,,39,,15,,,,,,,,,,,,,,,,,,22,,,,,,,39,
-java.lang,8,,58,,,,,,,,,,,8,,,,,,,,,,,,,,,,46,12
-java.net,10,3,7,,,,,,,,,,,,,,10,,,,,,,,,,,,3,7,
-java.nio,15,,6,,13,,,,,,,,,,,,,,,,,,2,,,,,,,6,
-java.sql,11,,,,,,,,,4,,,,,,,,,,7,,,,,,,,,,,
-java.util,34,,438,,,,,,,,,,,34,,,,,,,,,,,,,,,,24,414
-javax.faces.context,2,7,,,,,,,,,,,,,,,,,,,,,,,,2,,,7,,
-javax.jms,,9,57,,,,,,,,,,,,,,,,,,,,,,,,,,9,57,
-javax.json,,,123,,,,,,,,,,,,,,,,,,,,,,,,,,,100,23
-javax.management.remote,2,,,,,,,,,,,2,,,,,,,,,,,,,,,,,,,
-javax.naming,7,,,,,,,,,,,6,1,,,,,,,,,,,,,,,,,,
-javax.net.ssl,2,,,,,,,,,,,,,,,,,,2,,,,,,,,,,,,
-javax.script,1,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,
-javax.servlet,4,21,2,,,,3,1,,,,,,,,,,,,,,,,,,,,,21,2,
-javax.validation,1,1,,1,,,,,,,,,,,,,,,,,,,,,,,,,1,,
-javax.ws.rs.client,1,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,
-javax.ws.rs.container,,9,,,,,,,,,,,,,,,,,,,,,,,,,,,9,,
-javax.ws.rs.core,3,,149,,,,1,,,,,,,,,,,,,,,2,,,,,,,,94,55
-javax.xml.transform,1,,6,,,,,,,,,,,,,,,,,,,,,,1,,,,,6,
-javax.xml.xpath,3,,,,,,,,,,,,,,,,,,,,,,,3,,,,,,,
-jodd.json,,,10,,,,,,,,,,,,,,,,,,,,,,,,,,,,10
-net.sf.saxon.s9api,5,,,,,,,,,,,,,,,,,,,,,,,,5,,,,,,
-ognl,6,,,,,,,,,,,,,,,6,,,,,,,,,,,,,,,
-org.apache.commons.codec,,,6,,,,,,,,,,,,,,,,,,,,,,,,,,,6,
-org.apache.commons.collections,,,800,,,,,,,,,,,,,,,,,,,,,,,,,,,17,783
-org.apache.commons.collections4,,,800,,,,,,,,,,,,,,,,,,,,,,,,,,,17,783
-org.apache.commons.io,93,2,565,,78,,,,,,,,,,,,15,,,,,,,,,,,,2,551,14
-org.apache.commons.jexl2,15,,,,,,,,,,15,,,,,,,,,,,,,,,,,,,,
-org.apache.commons.jexl3,15,,,,,,,,,,15,,,,,,,,,,,,,,,,,,,,
-org.apache.commons.lang3,,,424,,,,,,,,,,,,,,,,,,,,,,,,,,,293,131
-org.apache.commons.logging,6,,,,,,,,,,,,,6,,,,,,,,,,,,,,,,,
-org.apache.commons.ognl,6,,,,,,,,,,,,,,,6,,,,,,,,,,,,,,,
-org.apache.commons.text,,,272,,,,,,,,,,,,,,,,,,,,,,,,,,,220,52
-org.apache.directory.ldap.client.api,1,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,
-org.apache.hc.core5.function,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,1,
-org.apache.hc.core5.http,1,2,39,,,,,,,,,,,,,,,,,,,,,,,1,,,2,39,
-org.apache.hc.core5.net,,,2,,,,,,,,,,,,,,,,,,,,,,,,,,,2,
-org.apache.hc.core5.util,,,24,,,,,,,,,,,,,,,,,,,,,,,,,,,18,6
-org.apache.http,27,3,70,,,,,,,,,,,,,,25,,,,,,,,,2,,,3,62,8
-org.apache.ibatis.jdbc,6,,57,,,,,,,,,,,,,,,,,6,,,,,,,,,,57,
-org.apache.log4j,11,,,,,,,,,,,,,11,,,,,,,,,,,,,,,,,
-org.apache.logging.log4j,359,,8,,,,,,,,,,,359,,,,,,,,,,,,,,,,4,4
-org.apache.shiro.codec,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,1,
-org.apache.shiro.jndi,1,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,
-org.codehaus.groovy.control,1,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,
-org.dom4j,20,,,,,,,,,,,,,,,,,,,,,,,20,,,,,,,
-org.hibernate,7,,,,,,,,,,,,,,,,,,,7,,,,,,,,,,,
-org.jboss.logging,324,,,,,,,,,,,,,324,,,,,,,,,,,,,,,,,
-org.jdbi.v3.core,6,,,,,,,,,6,,,,,,,,,,,,,,,,,,,,,
-org.jooq,1,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,
-org.json,,,236,,,,,,,,,,,,,,,,,,,,,,,,,,,198,38
-org.mvel2,16,,,,,,,,,,,,,,16,,,,,,,,,,,,,,,,
-org.scijava.log,13,,,,,,,,,,,,,13,,,,,,,,,,,,,,,,,
-org.slf4j,55,,6,,,,,,,,,,,55,,,,,,,,,,,,,,,,2,4
-org.springframework.beans,,,30,,,,,,,,,,,,,,,,,,,,,,,,,,,,30
-org.springframework.boot.jdbc,1,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,
-org.springframework.cache,,,13,,,,,,,,,,,,,,,,,,,,,,,,,,,,13
-org.springframework.context,,,3,,,,,,,,,,,,,,,,,,,,,,,,,,,3,
-org.springframework.http,14,,70,,,,,,,,,,,,,,14,,,,,,,,,,,,,60,10
-org.springframework.jdbc.core,10,,,,,,,,,,,,,,,,,,,10,,,,,,,,,,,
-org.springframework.jdbc.datasource,4,,,,,,,,,4,,,,,,,,,,,,,,,,,,,,,
-org.springframework.jdbc.object,9,,,,,,,,,,,,,,,,,,,9,,,,,,,,,,,
-org.springframework.jndi,1,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,
-org.springframework.ldap,47,,,,,,,,,,,33,14,,,,,,,,,,,,,,,,,,
-org.springframework.security.web.savedrequest,,6,,,,,,,,,,,,,,,,,,,,,,,,,,,6,,
-org.springframework.ui,,,32,,,,,,,,,,,,,,,,,,,,,,,,,,,,32
-org.springframework.util,,,139,,,,,,,,,,,,,,,,,,,,,,,,,,,87,52
-org.springframework.validation,,,13,,,,,,,,,,,,,,,,,,,,,,,,,,,13,
-org.springframework.web.client,13,3,,,,,,,,,,,,,,,13,,,,,,,,,,,,3,,
-org.springframework.web.context.request,,8,,,,,,,,,,,,,,,,,,,,,,,,,,,8,,
-org.springframework.web.multipart,,12,13,,,,,,,,,,,,,,,,,,,,,,,,,,12,13,
-org.springframework.web.reactive.function.client,2,,,,,,,,,,,,,,,,2,,,,,,,,,,,,,,
-org.springframework.web.util,,,163,,,,,,,,,,,,,,,,,,,,,,,,,,,138,25
-org.xml.sax,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,1,
-org.xmlpull.v1,,3,,,,,,,,,,,,,,,,,,,,,,,,,,,3,,
-play.mvc,,4,,,,,,,,,,,,,,,,,,,,,,,,,,,4,,
-ratpack.core.form,,,3,,,,,,,,,,,,,,,,,,,,,,,,,,,3,
-ratpack.core.handling,,6,4,,,,,,,,,,,,,,,,,,,,,,,,,,6,4,
-ratpack.core.http,,10,10,,,,,,,,,,,,,,,,,,,,,,,,,,10,10,
-ratpack.exec,,,48,,,,,,,,,,,,,,,,,,,,,,,,,,,,48
-ratpack.form,,,3,,,,,,,,,,,,,,,,,,,,,,,,,,,3,
-ratpack.func,,,35,,,,,,,,,,,,,,,,,,,,,,,,,,,,35
-ratpack.handling,,6,4,,,,,,,,,,,,,,,,,,,,,,,,,,6,4,
-ratpack.http,,10,10,,,,,,,,,,,,,,,,,,,,,,,,,,10,10,
-ratpack.util,,,35,,,,,,,,,,,,,,,,,,,,,,,,,,,,35
+package,sink,source,summary,sink:bean-validation,sink:create-file,sink:groovy,sink:header-splitting,sink:information-leak,sink:intent-start,sink:jdbc-url,sink:jexl,sink:jndi-injection,sink:ldap,sink:logging,sink:mvel,sink:ognl-injection,sink:open-url,sink:pending-intent-sent,sink:regex-use[-1],sink:regex-use[0],sink:regex-use[],sink:regex-use[f-1],sink:regex-use[f1],sink:regex-use[f],sink:set-hostname-verifier,sink:sql,sink:url-open-stream,sink:url-redirect,sink:write-file,sink:xpath,sink:xslt,sink:xss,source:android-widget,source:contentprovider,source:remote,summary:taint,summary:value
+android.app,16,,103,,,,,,7,,,,,,,,,9,,,,,,,,,,,,,,,,,,18,85
+android.content,24,27,108,,,,,,16,,,,,,,,,,,,,,,,,8,,,,,,,,27,,31,77
+android.database,59,,30,,,,,,,,,,,,,,,,,,,,,,,59,,,,,,,,,,30,
+android.net,,,60,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,45,15
+android.os,,,122,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,41,81
+android.util,6,16,,,,,,,,,,,,6,,,,,,,,,,,,,,,,,,,,,16,,
+android.webkit,3,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,,,2,,
+android.widget,,1,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,1,
+androidx.slice,2,5,88,,,,,,,,,,,,,,,2,,,,,,,,,,,,,,,,5,,27,61
+cn.hutool.core.codec,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,
+com.esotericsoftware.kryo.io,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,
+com.esotericsoftware.kryo5.io,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,
+com.fasterxml.jackson.core,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,
+com.fasterxml.jackson.databind,,,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,6,
+com.google.common.base,4,,85,,,,,,,,,,,,,,,,,3,1,,,,,,,,,,,,,,,62,23
+com.google.common.cache,,,17,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,17
+com.google.common.collect,,,553,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,551
+com.google.common.flogger,29,,,,,,,,,,,,,29,,,,,,,,,,,,,,,,,,,,,,,
+com.google.common.io,6,,73,,,,,,,,,,,,,,,,,,,,,,,,6,,,,,,,,,72,1
+com.opensymphony.xwork2.ognl,3,,,,,,,,,,,,,,,3,,,,,,,,,,,,,,,,,,,,,
+com.rabbitmq.client,,21,7,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,21,7,
+com.unboundid.ldap.sdk,17,,,,,,,,,,,,17,,,,,,,,,,,,,,,,,,,,,,,,
+com.zaxxer.hikari,2,,,,,,,,,2,,,,,,,,,,,,,,,,,,,,,,,,,,,
+flexjson,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1
+groovy.lang,26,,,,,26,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+groovy.util,5,,,,,5,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+jakarta.faces.context,2,7,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,7,,
+jakarta.json,,,123,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,100,23
+jakarta.ws.rs.client,1,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,
+jakarta.ws.rs.container,,9,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,9,,
+jakarta.ws.rs.core,2,,149,,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,94,55
+java.beans,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,
+java.io,37,,39,,15,,,,,,,,,,,,,,,,,,,,,,,,22,,,,,,,39,
+java.lang,13,,58,,,,,,,,,,,8,,,,,4,,,1,,,,,,,,,,,,,,46,12
+java.net,10,3,7,,,,,,,,,,,,,,10,,,,,,,,,,,,,,,,,,3,7,
+java.nio,15,,6,,13,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,6,
+java.sql,11,,,,,,,,,4,,,,,,,,,,,,,,,,7,,,,,,,,,,,
+java.util,44,,438,,,,,,,,,,,34,,,,,,5,2,,1,2,,,,,,,,,,,,24,414
+javax.faces.context,2,7,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,7,,
+javax.jms,,9,57,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,9,57,
+javax.json,,,123,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,100,23
+javax.management.remote,2,,,,,,,,,,,2,,,,,,,,,,,,,,,,,,,,,,,,,
+javax.naming,7,,,,,,,,,,,6,1,,,,,,,,,,,,,,,,,,,,,,,,
+javax.net.ssl,2,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,,,,,
+javax.script,1,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,
+javax.servlet,4,21,2,,,,3,1,,,,,,,,,,,,,,,,,,,,,,,,,,,21,2,
+javax.validation,1,1,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,
+javax.ws.rs.client,1,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,
+javax.ws.rs.container,,9,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,9,,
+javax.ws.rs.core,3,,149,,,,1,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,94,55
+javax.xml.transform,1,,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,6,
+javax.xml.xpath,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,,,,,,,
+jodd.json,,,10,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,10
+kotlin.jvm.internal,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1
+net.sf.saxon.s9api,5,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,5,,,,,,
+ognl,6,,,,,,,,,,,,,,,6,,,,,,,,,,,,,,,,,,,,,
+okhttp3,2,,47,,,,,,,,,,,,,,2,,,,,,,,,,,,,,,,,,,22,25
+org.apache.commons.codec,,,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,6,
+org.apache.commons.collections,,,800,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,17,783
+org.apache.commons.collections4,,,800,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,17,783
+org.apache.commons.io,93,2,565,,78,,,,,,,,,,,,15,,,,,,,,,,,,,,,,,,2,551,14
+org.apache.commons.jexl2,15,,,,,,,,,,15,,,,,,,,,,,,,,,,,,,,,,,,,,
+org.apache.commons.jexl3,15,,,,,,,,,,15,,,,,,,,,,,,,,,,,,,,,,,,,,
+org.apache.commons.lang3,,,424,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,293,131
+org.apache.commons.logging,6,,,,,,,,,,,,,6,,,,,,,,,,,,,,,,,,,,,,,
+org.apache.commons.ognl,6,,,,,,,,,,,,,,,6,,,,,,,,,,,,,,,,,,,,,
+org.apache.commons.text,,,272,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,220,52
+org.apache.directory.ldap.client.api,1,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,
+org.apache.hc.core5.function,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,
+org.apache.hc.core5.http,1,2,39,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,2,39,
+org.apache.hc.core5.net,,,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,
+org.apache.hc.core5.util,,,24,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,18,6
+org.apache.http,27,3,70,,,,,,,,,,,,,,25,,,,,,,,,,,,,,,2,,,3,62,8
+org.apache.ibatis.jdbc,6,,57,,,,,,,,,,,,,,,,,,,,,,,6,,,,,,,,,,57,
+org.apache.log4j,11,,,,,,,,,,,,,11,,,,,,,,,,,,,,,,,,,,,,,
+org.apache.logging.log4j,359,,8,,,,,,,,,,,359,,,,,,,,,,,,,,,,,,,,,,4,4
+org.apache.shiro.codec,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,
+org.apache.shiro.jndi,1,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,
+org.codehaus.groovy.control,1,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+org.dom4j,20,,,,,,,,,,,,,,,,,,,,,,,,,,,,,20,,,,,,,
+org.hibernate,7,,,,,,,,,,,,,,,,,,,,,,,,,7,,,,,,,,,,,
+org.jboss.logging,324,,,,,,,,,,,,,324,,,,,,,,,,,,,,,,,,,,,,,
+org.jdbi.v3.core,6,,,,,,,,,6,,,,,,,,,,,,,,,,,,,,,,,,,,,
+org.jooq,1,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,
+org.json,,,236,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,198,38
+org.mvel2,16,,,,,,,,,,,,,,16,,,,,,,,,,,,,,,,,,,,,,
+org.scijava.log,13,,,,,,,,,,,,,13,,,,,,,,,,,,,,,,,,,,,,,
+org.slf4j,55,,6,,,,,,,,,,,55,,,,,,,,,,,,,,,,,,,,,,2,4
+org.springframework.beans,,,30,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,30
+org.springframework.boot.jdbc,1,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,
+org.springframework.cache,,,13,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,13
+org.springframework.context,,,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,
+org.springframework.http,14,,70,,,,,,,,,,,,,,14,,,,,,,,,,,,,,,,,,,60,10
+org.springframework.jdbc.core,10,,,,,,,,,,,,,,,,,,,,,,,,,10,,,,,,,,,,,
+org.springframework.jdbc.datasource,4,,,,,,,,,4,,,,,,,,,,,,,,,,,,,,,,,,,,,
+org.springframework.jdbc.object,9,,,,,,,,,,,,,,,,,,,,,,,,,9,,,,,,,,,,,
+org.springframework.jndi,1,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,
+org.springframework.ldap,47,,,,,,,,,,,33,14,,,,,,,,,,,,,,,,,,,,,,,,
+org.springframework.security.web.savedrequest,,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,6,,
+org.springframework.ui,,,32,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,32
+org.springframework.util,,,139,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,87,52
+org.springframework.validation,,,13,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,13,
+org.springframework.web.client,13,3,,,,,,,,,,,,,,,13,,,,,,,,,,,,,,,,,,3,,
+org.springframework.web.context.request,,8,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,8,,
+org.springframework.web.multipart,,12,13,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,12,13,
+org.springframework.web.reactive.function.client,2,,,,,,,,,,,,,,,,2,,,,,,,,,,,,,,,,,,,,
+org.springframework.web.util,,,163,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,138,25
+org.xml.sax,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,
+org.xmlpull.v1,,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,,
+play.mvc,,4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,4,,
+ratpack.core.form,,,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,
+ratpack.core.handling,,6,4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,6,4,
+ratpack.core.http,,10,10,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,10,10,
+ratpack.exec,,,48,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,48
+ratpack.form,,,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,
+ratpack.func,,,35,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,35
+ratpack.handling,,6,4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,6,4,
+ratpack.http,,10,10,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,10,10,
+ratpack.util,,,35,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,35
+retrofit2,1,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,

java/documentation/library-coverage/coverage.rst

@@ -13,11 +13,11 @@ Java framework & library support
    `Apache Commons Lang <https://commons.apache.org/proper/commons-lang/>`_,``org.apache.commons.lang3``,,424,,,,,,,,
    `Apache Commons Text <https://commons.apache.org/proper/commons-text/>`_,``org.apache.commons.text``,,272,,,,,,,,
    `Apache HttpComponents <https://hc.apache.org/>`_,"``org.apache.hc.core5.*``, ``org.apache.http``",5,136,28,,,3,,,,25
-   `Google Guava <https://guava.dev/>`_,``com.google.common.*``,,728,35,,6,,,,,
+   `Google Guava <https://guava.dev/>`_,``com.google.common.*``,,728,39,,6,,,,,
    `JSON-java <https://github.com/stleary/JSON-java>`_,``org.json``,,236,,,,,,,,
-   Java Standard Library,``java.*``,3,549,115,28,,,7,,,10
+   Java Standard Library,``java.*``,3,549,130,28,,,7,,,10
    Java extensions,"``javax.*``, ``jakarta.*``",63,609,32,,,4,,1,1,2
    `Spring <https://spring.io/>`_,``org.springframework.*``,29,476,101,,,,19,14,,29
-   Others,"``androidx.slice``, ``cn.hutool.core.codec``, ``com.esotericsoftware.kryo.io``, ``com.esotericsoftware.kryo5.io``, ``com.fasterxml.jackson.core``, ``com.fasterxml.jackson.databind``, ``com.opensymphony.xwork2.ognl``, ``com.rabbitmq.client``, ``com.unboundid.ldap.sdk``, ``com.zaxxer.hikari``, ``flexjson``, ``groovy.lang``, ``groovy.util``, ``jodd.json``, ``net.sf.saxon.s9api``, ``ognl``, ``org.apache.commons.codec``, ``org.apache.commons.jexl2``, ``org.apache.commons.jexl3``, ``org.apache.commons.logging``, ``org.apache.commons.ognl``, ``org.apache.directory.ldap.client.api``, ``org.apache.ibatis.jdbc``, ``org.apache.log4j``, ``org.apache.logging.log4j``, ``org.apache.shiro.codec``, ``org.apache.shiro.jndi``, ``org.codehaus.groovy.control``, ``org.dom4j``, ``org.hibernate``, ``org.jboss.logging``, ``org.jdbi.v3.core``, ``org.jooq``, ``org.mvel2``, ``org.scijava.log``, ``org.slf4j``, ``org.xml.sax``, ``org.xmlpull.v1``, ``play.mvc``, ``ratpack.core.form``, ``ratpack.core.handling``, ``ratpack.core.http``, ``ratpack.exec``, ``ratpack.form``, ``ratpack.func``, ``ratpack.handling``, ``ratpack.http``, ``ratpack.util``",65,347,929,,,,14,18,,
-   Totals,,213,6366,1441,106,6,10,107,33,1,81
+   Others,"``androidx.slice``, ``cn.hutool.core.codec``, ``com.esotericsoftware.kryo.io``, ``com.esotericsoftware.kryo5.io``, ``com.fasterxml.jackson.core``, ``com.fasterxml.jackson.databind``, ``com.opensymphony.xwork2.ognl``, ``com.rabbitmq.client``, ``com.unboundid.ldap.sdk``, ``com.zaxxer.hikari``, ``flexjson``, ``groovy.lang``, ``groovy.util``, ``jodd.json``, ``kotlin.jvm.internal``, ``net.sf.saxon.s9api``, ``ognl``, ``okhttp3``, ``org.apache.commons.codec``, ``org.apache.commons.jexl2``, ``org.apache.commons.jexl3``, ``org.apache.commons.logging``, ``org.apache.commons.ognl``, ``org.apache.directory.ldap.client.api``, ``org.apache.ibatis.jdbc``, ``org.apache.log4j``, ``org.apache.logging.log4j``, ``org.apache.shiro.codec``, ``org.apache.shiro.jndi``, ``org.codehaus.groovy.control``, ``org.dom4j``, ``org.hibernate``, ``org.jboss.logging``, ``org.jdbi.v3.core``, ``org.jooq``, ``org.mvel2``, ``org.scijava.log``, ``org.slf4j``, ``org.xml.sax``, ``org.xmlpull.v1``, ``play.mvc``, ``ratpack.core.form``, ``ratpack.core.handling``, ``ratpack.core.http``, ``ratpack.exec``, ``ratpack.form``, ``ratpack.func``, ``ratpack.handling``, ``ratpack.http``, ``ratpack.util``, ``retrofit2``",65,395,932,,,,14,18,,3
+   Totals,,213,6414,1463,106,6,10,107,33,1,84
 

java/kotlin-explorer/.gitignore

@@ -0,0 +1,10 @@
+.classpath
+.gradle
+.idea
+.project
+.settings
+bin/
+build/
+gradle/
+gradlew
+gradlew.bat

java/kotlin-explorer/README

@@ -0,0 +1,9 @@
+
+This shows what is encoded in the kotlin.Metadata section shown in the
+output of `javap -v SomeKotlinClass`.
+
+It is not currently able to extract the information from .class files
+itself; the values are hard coded in src/main/kotlin/Explorer.kt
+
+Run `gradle run` in this directory to run it.
+

java/kotlin-explorer/build.gradle

@@ -0,0 +1,28 @@
+plugins {
+    id 'org.jetbrains.kotlin.jvm' version "${kotlinVersion}"
+    id 'org.jetbrains.dokka' version '1.4.32'
+    id "com.vanniktech.maven.publish" version '0.15.1'
+    id 'application'
+}
+
+group 'com.github.codeql'
+version '0.0.1'
+
+dependencies {
+    implementation "org.jetbrains.kotlin:kotlin-stdlib"
+    implementation "org.jetbrains.kotlinx:kotlinx-metadata-jvm:0.3.0"
+}
+
+repositories {
+    mavenCentral()
+}
+
+tasks.withType(org.jetbrains.kotlin.gradle.tasks.KotlinCompile).configureEach {
+    kotlinOptions {
+        jvmTarget = "1.8"
+    }
+}
+
+application {
+    mainClass = 'com.github.codeql.ExplorerKt'
+}

java/kotlin-explorer/gradle.properties

@@ -0,0 +1,7 @@
+kotlin.code.style=official
+kotlinVersion=1.5.21
+
+GROUP=com.github.codeql
+VERSION_NAME=0.0.1
+POM_DESCRIPTION=CodeQL Kotlin explorer
+

java/kotlin-explorer/settings.gradle

@@ -0,0 +1,8 @@
+pluginManagement {
+    repositories {
+        mavenCentral()
+        gradlePluginPortal()
+    }
+}
+
+rootProject.name = 'codeql-kotlin-explorer'

java/kotlin-explorer/src/main/kotlin/Explorer.kt

@@ -0,0 +1,217 @@
+package com.github.codeql
+import kotlinx.metadata.internal.metadata.jvm.deserialization.JvmMetadataVersion
+import kotlinx.metadata.jvm.*
+import kotlinx.metadata.*
+
+fun main(args : Array<String>) {
+    /*
+    Values from `javap -v` on TestKt.class from:
+
+    class MyClass {}
+
+    class MyParamClass<T> {}
+
+    fun f(x: MyClass, y: MyClass?,
+          l1: MyParamClass<MyClass>,
+          l2: MyParamClass<MyClass?>,
+          l3: MyParamClass<MyClass>?,
+          l4: MyParamClass<MyClass?>?) {
+    }
+    */
+    val kind = 2
+    val metadataVersion = intArrayOf(1, 5, 1)
+    val data1 = arrayOf("\u0000\u0018\n\u0000\n\u0002\u0010\u0002\n\u0000\n\u0002\u0018\u0002\n\u0002\b\u0002\n\u0002\u0018\u0002\n\u0002\b\u0003\u001aX\u0010\u0000\u001a\u00020\u00012\u0006\u0010\u0002\u001a\u00020\u00032\b\u0010\u0004\u001a\u0004\u0018\u00010\u00032\u000c\u0010\u0005\u001a\b\u0012\u0004\u0012\u00020\u00030\u00062\u000e\u0010\u0007\u001a\n\u0012\u0006\u0012\u0004\u0018\u00010\u00030\u00062\u000e\u0010\b\u001a\n\u0012\u0004\u0012\u00020\u0003\u0018\u00010\u00062\u0010\u0010\t\u001a\u000c\u0012\u0006\u0012\u0004\u0018\u00010\u0003\u0018\u00010\u0006")
+    val data2 = arrayOf("f","","x","LMyClass;","y","l1","LMyParamClass;","l2","l3","l4")
+    val extraString = null
+    val packageName = null
+    val extraInt = 48
+    val kch = KotlinClassHeader(kind, metadataVersion, data1, data2, extraString, packageName, extraInt)
+
+    val md = KotlinClassMetadata.read(kch)
+    when (md) {
+        is KotlinClassMetadata.Class -> println("Metadata for Class not yet supported")
+        is KotlinClassMetadata.FileFacade -> {
+            println("Metadata for FileFacade:")
+            val kmp = md.toKmPackage()
+            kmp.accept(MyPackageVisitor(0))
+        }
+        is KotlinClassMetadata.SyntheticClass -> println("Metadata for SyntheticClass not yet supported")
+        is KotlinClassMetadata.MultiFileClassFacade -> println("Metadata for MultiFileClassFacade not yet supported")
+        is KotlinClassMetadata.MultiFileClassPart -> println("Metadata for MultiFileClassPart not yet supported")
+        is KotlinClassMetadata.Unknown -> println("Unknown kind")
+        else -> println("Unexpected kind")
+    }
+}
+
+fun pr(indent: Int, s: String) {
+    println("    ".repeat(indent) + s)
+}
+
+class MyPackageVisitor(val indent: Int): KmPackageVisitor() {
+    override fun visitFunction(flags: Flags, name: String): KmFunctionVisitor? {
+        pr(indent, "=> Function; flags:$flags, name:$name")
+        return MyFunctionVisitor(indent + 1)
+    }
+
+    override fun visitProperty(flags: Flags, name: String, getterFlags: Flags, setterFlags: Flags): KmPropertyVisitor? {
+        pr(indent, "=> Properties not yet handled")
+        return null
+    }
+
+    override fun visitTypeAlias(flags: Flags, name: String): KmTypeAliasVisitor? {
+        pr(indent, "=> Type aliases not yet handled")
+        return null
+    }
+
+    override fun visitExtensions(type: KmExtensionType): KmPackageExtensionVisitor? {
+        pr(indent, "=> Package extensions; type:$type")
+        when (type) {
+            JvmPackageExtensionVisitor.TYPE -> return MyJvmPackageExtensionVisitor(indent + 1)
+            else -> {
+                pr(indent, "- Not yet handled")
+                return null
+            }
+        }
+    }
+}
+
+class MyFunctionVisitor(val indent: Int): KmFunctionVisitor() {
+    override fun visitTypeParameter(flags: Flags, name: String, id: Int, variance: KmVariance): KmTypeParameterVisitor? {
+        pr(indent, "=> Type parameter; flags:$flags, name:$name, id:$id, variance:$variance")
+        pr(indent, "    -> Not yet handled")
+        return null
+    }
+    override fun visitReceiverParameterType(flags: Flags): KmTypeVisitor? {
+        pr(indent, "=> Receiver parameter type; flags:$flags")
+        pr(indent, "    -> Not yet handled")
+        return null
+    }
+
+    override fun visitValueParameter(flags: Flags, name: String): KmValueParameterVisitor? {
+        pr(indent, "=> Value parameter; flags:$flags, name:$name")
+        return MyValueParameterVisitor(indent + 1)
+    }
+
+    override fun visitReturnType(flags: Flags): KmTypeVisitor? {
+        pr(indent, "=> Return type; flags:$flags")
+        return MyTypeVisitor(indent + 1)
+    }
+
+    override fun visitVersionRequirement(): KmVersionRequirementVisitor? {
+        pr(indent, "=> VersionRequirement not yet handled")
+        return null
+    }
+
+    override fun visitContract(): KmContractVisitor? {
+        pr(indent, "=> Contract not yet handled")
+        return null
+    }
+
+    override fun visitExtensions(type: KmExtensionType): KmFunctionExtensionVisitor? {
+        pr(indent, "=> Function extensions; type:$type")
+        when (type) {
+            JvmFunctionExtensionVisitor.TYPE -> return MyJvmFunctionExtensionVisitor(indent + 1)
+            else -> {
+                pr(indent, "- Not yet handled")
+                return null
+            }
+        }
+    }
+}
+
+class MyValueParameterVisitor(val indent: Int): KmValueParameterVisitor() {
+    override fun visitType(flags: Flags): KmTypeVisitor? {
+        pr(indent, "=> Type; flags:$flags")
+        return MyTypeVisitor(indent + 1)
+    }
+
+    override fun visitVarargElementType(flags: Flags): KmTypeVisitor? {
+        pr(indent, "=> VarargElementType not yet handled")
+        return null
+    }
+
+    override fun visitExtensions(type: KmExtensionType): KmValueParameterExtensionVisitor? {
+        pr(indent, "=> Value parameter extensions; type:$type; not yet handled")
+        return null
+    }
+}
+
+class MyTypeVisitor(val indent: Int): KmTypeVisitor() {
+    override fun visitClass(name: ClassName) {
+        pr(indent, "=> Class; name:$name")
+    }
+
+    override fun visitTypeAlias(name: ClassName) {
+        pr(indent, "=> Type alias; name:$name")
+    }
+
+    override fun visitTypeParameter(id: Int) {
+        pr(indent, "=> Type parameter; id:$id")
+    }
+
+    override fun visitArgument(flags: Flags, variance: KmVariance): KmTypeVisitor? {
+        pr(indent, "=> Argument; flags:$flags, variance:$variance")
+        return MyTypeVisitor(indent + 1)
+    }
+
+    override fun visitStarProjection() {
+        pr(indent, "=> Star projection")
+    }
+
+    override fun visitAbbreviatedType(flags: Flags): KmTypeVisitor? {
+        pr(indent, "=> AbbreviatedType not yet handled")
+        return null
+    }
+
+    override fun visitOuterType(flags: Flags): KmTypeVisitor? {
+        pr(indent, "=> OuterType not yet handled")
+        return null
+    }
+
+    override fun visitFlexibleTypeUpperBound(flags: Flags, typeFlexibilityId: String?): KmTypeVisitor? {
+        pr(indent, "=> FlexibleTypeUpperBound not yet handled")
+        return null
+    }
+
+    override fun visitExtensions(type: KmExtensionType): KmTypeExtensionVisitor? {
+        pr(indent, "=> Type extensions; type:$type")
+        when (type) {
+            JvmTypeExtensionVisitor.TYPE -> return MyJvmTypeExtensionVisitor(indent + 1)
+            else -> {
+                pr(indent, "- Not yet handled")
+                return null
+            }
+        }
+    }
+}
+
+class MyJvmTypeExtensionVisitor(val indent: Int): JvmTypeExtensionVisitor() {
+    override fun visit(isRaw: Boolean) {
+        pr(indent, "=> isRaw:$isRaw")
+    }
+
+    override fun visitAnnotation(annotation: KmAnnotation) {
+        pr(indent, "=> Annotation; annotation:$annotation")
+    }
+}
+
+class MyJvmPackageExtensionVisitor(val indent: Int): JvmPackageExtensionVisitor() {
+    override fun visitLocalDelegatedProperty(flags: Flags, name: String, getterFlags: Flags, setterFlags: Flags): KmPropertyVisitor? {
+        pr(indent, "=> Local delegate not yet handled")
+        return null
+    }
+
+    override fun visitModuleName(name: String) {
+        pr(indent, "=> Module name; name:$name")
+    }
+}
+
+class MyJvmFunctionExtensionVisitor(val indent: Int): JvmFunctionExtensionVisitor() {
+    override fun visit(signature: JvmMethodSignature?) {
+        pr(indent, "=> signature:$signature")
+    }
+
+    override fun visitLambdaClassOriginName(internalName: String) {
+        pr(indent, "=> LambdaClassOriginName; internalName:$internalName")
+    }
+}

java/kotlin-extractor/.gitignore

@@ -0,0 +1,12 @@
+.classpath
+.gradle
+.idea
+.project
+.settings
+bin/
+build/
+gradle/
+gradlew
+gradlew.bat
+
+src/main/kotlin/KotlinExtractorDbScheme.kt

java/kotlin-extractor/build.gradle

@@ -0,0 +1,53 @@
+plugins {
+    id 'org.jetbrains.kotlin.jvm' version "${kotlinVersion}"
+    id 'org.jetbrains.dokka' version '1.4.32'
+}
+
+group 'com.github.codeql'
+version '0.0.1'
+
+dependencies {
+    implementation "org.jetbrains.kotlin:kotlin-stdlib"
+    compileOnly("org.jetbrains.kotlin:kotlin-compiler")
+}
+
+repositories {
+    mavenCentral()
+}
+
+tasks.withType(org.jetbrains.kotlin.gradle.tasks.KotlinCompile).configureEach {
+    kotlinOptions {
+        jvmTarget = "1.8"
+        freeCompilerArgs += "-Xopt-in=kotlin.RequiresOptIn"
+        // enable the below for building with kotlinVersion=1.4.32:
+        // languageVersion = "1.5"
+    }
+}
+
+sourceSets {
+    main {
+        kotlin {
+            // change the excludes for building with other versions:
+            excludes = [
+                    "utils/versions/v_1_4_32/*.kt",
+                    "utils/versions/v_1_5_31/*.kt",
+                    "utils/versions/v_1_6_10/*.kt"]
+        }
+    }
+}
+
+jar {
+    archiveName = "${OUTPUT_JAR_NAME}"
+}
+
+task getHomeDir {
+    doLast {
+        println gradle.gradleHomeDir
+    }
+}
+
+java {
+  toolchain {
+    languageVersion.set(JavaLanguageVersion.of(8))
+  }
+}

java/kotlin-extractor/build.py

@@ -0,0 +1,209 @@
+#!/usr/bin/env python3
+
+import argparse
+import kotlin_plugin_versions
+import glob
+import platform
+import re
+import subprocess
+import shutil
+import os
+import os.path
+import sys
+import shlex
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--dependencies', default='../../../resources/kotlin-dependencies',
+                        help='Folder containing the dependencies')
+    parser.add_argument('--many', action='store_true',
+                        help='Build for all versions/kinds')
+    parser.add_argument('--single', action='store_false',
+                        dest='many', help='Build for a single version/kind')
+    return parser.parse_args()
+
+
+args = parse_args()
+
+
+def is_windows():
+    '''Whether we appear to be running on Windows'''
+    if platform.system() == 'Windows':
+        return True
+    if platform.system().startswith('CYGWIN'):
+        return True
+    return False
+
+
+kotlinc = 'kotlinc.bat' if is_windows() else 'kotlinc'
+javac = 'javac'
+kotlin_dependency_folder = args.dependencies
+
+
+def quote_for_batch(arg):
+    if ';' in arg or '=' in arg:
+        if '"' in arg:
+            raise Exception('Need to quote something containing a quote')
+        return '"' + arg + '"'
+    else:
+        return arg
+
+
+def run_process(cmd, capture_output=False):
+    print("Running command: " + shlex.join(cmd))
+    if is_windows():
+        cmd = ' '.join(map(quote_for_batch, cmd))
+        print("Converted to Windows command: " + cmd)
+    try:
+        if capture_output:
+            return subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        else:
+            return subprocess.run(cmd, check=True)
+    except subprocess.CalledProcessError as e:
+        print("In: " + os.getcwd(), file=sys.stderr)
+        shell_cmd = cmd if is_windows() else shlex.join(cmd)
+        print("Command failed: " + shell_cmd, file=sys.stderr)
+        if capture_output:
+            print("stdout output:\n" + e.stdout.decode(encoding='UTF-8',
+                  errors='replace'), file=sys.stderr)
+            print("stderr output:\n" + e.stderr.decode(encoding='UTF-8',
+                  errors='replace'), file=sys.stderr)
+        raise e
+
+
+def compile_to_dir(srcs, classpath, java_classpath, output):
+    # Use kotlinc to compile .kt files:
+    run_process([kotlinc,
+                 # kotlinc can default to 256M, which isn't enough when we are extracting the build
+                 '-J-Xmx2G',
+                 '-Xopt-in=kotlin.RequiresOptIn',
+                 '-d', output,
+                 '-module-name', 'codeql-kotlin-extractor',
+                 '-no-reflect', '-no-stdlib',
+                 '-jvm-target', '1.8',
+                 '-classpath', classpath] + srcs)
+
+    # Use javac to compile .java files, referencing the Kotlin class files:
+    run_process([javac,
+                 '-d', output,
+                 '-source', '8', '-target', '8',
+                 '-classpath', os.path.pathsep.join([output, classpath, java_classpath])] + [s for s in srcs if s.endswith(".java")])
+
+
+def compile_to_jar(srcs, classpath, java_classpath, output):
+    builddir = 'build/classes'
+
+    if os.path.exists(builddir):
+        shutil.rmtree(builddir)
+    os.makedirs(builddir)
+
+    compile_to_dir(srcs, classpath, java_classpath, builddir)
+
+    run_process(['jar', 'cf', output,
+                 '-C', builddir, '.',
+                 '-C', 'src/main/resources', 'META-INF'])
+    shutil.rmtree(builddir)
+
+
+def find_sources(path):
+    return glob.glob(path + '/**/*.kt', recursive=True) + glob.glob(path + '/**/*.java', recursive=True)
+
+
+def get_kotlin_lib_folder():
+    x = run_process([kotlinc, '-version', '-verbose'], capture_output=True)
+    output = x.stderr.decode(encoding='UTF-8', errors='strict')
+    m = re.match(
+        r'.*\nlogging: using Kotlin home directory ([^\n]+)\n.*', output)
+    if m is None:
+        raise Exception('Cannot determine kotlinc home directory')
+    kotlin_home = m.group(1)
+    print("Kotlin home directory: " + kotlin_home)
+    return kotlin_home + '/lib'
+
+
+def get_gradle_lib_folder():
+    x = run_process(['gradle', 'getHomeDir'], capture_output=True)
+    output = x.stdout.decode(encoding='UTF-8', errors='strict')
+    m = re.search(r'(?m)^> Task :getHomeDir\n([^\n]+)$', output)
+    if m is None:
+        print("gradle getHomeDir output:\n" + output, file=sys.stderr)
+        raise Exception('Cannot determine gradle home directory')
+    gradle_home = m.group(1)
+    print("Gradle home directory: " + gradle_home)
+    return gradle_home + '/lib'
+
+
+def find_jar(path, pattern):
+    result = glob.glob(path + '/' + pattern + '*.jar')
+    if len(result) == 0:
+        raise Exception('Cannot find jar file %s under path %s' %
+                        (pattern, path))
+    return result
+
+
+def patterns_to_classpath(path, patterns):
+    result = []
+    for pattern in patterns:
+        result += find_jar(path, pattern)
+    return os.path.pathsep.join(result)
+
+
+def transform_to_embeddable(srcs):
+    # replace imports in files:
+    for src in srcs:
+        with open(src, 'r') as f:
+            content = f.read()
+        content = content.replace('import com.intellij',
+                                  'import org.jetbrains.kotlin.com.intellij')
+        with open(src, 'w') as f:
+            f.write(content)
+
+
+def compile(jars, java_jars, dependency_folder, transform_to_embeddable, output, tmp_dir, version):
+    classpath = patterns_to_classpath(dependency_folder, jars)
+    java_classpath = patterns_to_classpath(dependency_folder, java_jars)
+
+    if os.path.exists(tmp_dir):
+        shutil.rmtree(tmp_dir)
+    shutil.copytree('src', tmp_dir)
+
+    for v in kotlin_plugin_versions.many_versions:
+        if v != version:
+            shutil.rmtree(
+                tmp_dir + '/main/kotlin/utils/versions/v_' + v.replace('.', '_'))
+
+    srcs = find_sources(tmp_dir)
+
+    transform_to_embeddable(srcs)
+
+    compile_to_jar(srcs, classpath, java_classpath, output)
+
+    shutil.rmtree(tmp_dir)
+
+
+def compile_embeddable(version):
+    compile(['kotlin-stdlib-' + version, 'kotlin-compiler-embeddable-' + version],
+            ['kotlin-stdlib-' + version],
+            kotlin_dependency_folder,
+            transform_to_embeddable,
+            'codeql-extractor-kotlin-embeddable-%s.jar' % (version),
+            'build/temp_src',
+            version)
+
+
+def compile_standalone(version):
+    compile(['kotlin-stdlib-' + version, 'kotlin-compiler-' + version],
+            ['kotlin-stdlib-' + version],
+            kotlin_dependency_folder,
+            lambda srcs: None,
+            'codeql-extractor-kotlin-standalone-%s.jar' % (version),
+            'build/temp_src',
+            version)
+
+if args.many:
+    for version in kotlin_plugin_versions.many_versions:
+        compile_standalone(version)
+        compile_embeddable(version)
+else:
+    compile_standalone(kotlin_plugin_versions.get_single_version())

java/kotlin-extractor/generate_dbscheme.py

@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+
+import re
+import sys
+
+enums = {}
+unions = {}
+tables = {}
+
+dbscheme = sys.argv[1] if len(sys.argv) >= 2 else '../ql/lib/config/semmlecode.dbscheme'
+
+def parse_dbscheme(filename):
+    with open(filename, 'r') as f:
+        dbscheme = f.read()
+
+    # Remove comments
+    dbscheme = re.sub(r'/\*.*?\*/', '', dbscheme, flags=re.DOTALL)
+    dbscheme = re.sub(r'//[^\r\n]*', '', dbscheme)
+
+    # kind enums
+    for name, kind, body in re.findall(r'case\s+@([^.\s]*)\.([^.\s]*)\s+of\b(.*?);',
+                                       dbscheme,
+                                       flags=re.DOTALL):
+        mapping = []
+        for num, typ in re.findall(r'(\d+)\s*=\s*@(\S+)', body):
+            mapping.append((int(num), typ))
+        enums[name] = (kind, mapping)
+
+    # unions
+    for name, rhs in re.findall(r'@(\w+)\s*=\s*(@\w+(?:\s*\|\s*@\w+)*)',
+                                dbscheme,
+                                flags=re.DOTALL):
+        typs = re.findall(r'@(\w+)', rhs)
+        unions[name] = typs
+
+    # tables
+    for relname, body in re.findall('\n([\w_]+)(\([^)]*\))',
+                                    dbscheme,
+                                    flags=re.DOTALL):
+        columns = list(re.findall('(\S+)\s*:\s*([^\s,]+)(?:\s+(ref)|)', body))
+        tables[relname] = columns
+
+parse_dbscheme(dbscheme)
+
+type_aliases = {}
+
+for alias, typs in unions.items():
+    if len(typs) == 1:
+        real = typs[0]
+        if real in type_aliases:
+            real = type_aliases[real]
+        type_aliases[alias] = real
+
+def unalias(t):
+    return type_aliases.get(t, t)
+
+type_leaf = set()
+type_union = {}
+
+for name, (kind, mapping) in enums.items():
+    s = set()
+    for num, typ in mapping:
+        s.add(typ)
+        type_leaf.add(typ)
+    type_union[name] = s
+
+for name, typs in unions.items():
+    if name not in type_aliases:
+        type_union[name] = set(map(unalias, typs))
+
+for relname, columns in tables.items():
+    for _, db_type, ref in columns:
+        if db_type[0] == '@' and ref == '':
+            db_type_name = db_type[1:]
+            if db_type_name not in enums:
+                type_leaf.add(db_type_name)
+
+type_union_of_leaves = {}
+
+def to_leaves(t):
+    if t not in type_union_of_leaves:
+        xs = type_union[t]
+        leaves = set()
+        for x in xs:
+            if x in type_leaf:
+                leaves.add(x)
+            else:
+                to_leaves(x)
+                leaves.update(type_union_of_leaves[x])
+        type_union_of_leaves[t] = leaves
+
+for t in type_union:
+    to_leaves(t)
+
+supertypes = {}
+for t in type_leaf:
+    supers = set()
+    for sup, s in type_union_of_leaves.items():
+        if t in s:
+            supers.add(sup)
+    supertypes[t] = supers
+for t, leaves in type_union_of_leaves.items():
+    supers = set()
+    for sup, s in type_union_of_leaves.items():
+        if t != sup and leaves.issubset(s):
+            supers.add(sup)
+    supertypes[t] = supers
+
+def upperFirst(string):
+    return string[0].upper() + string[1:]
+
+def genTable(kt, relname, columns, enum = None, kind = None, num = None, typ = None):
+    kt.write('fun TrapWriter.write' + upperFirst(relname))
+    if kind is not None:
+        kt.write('_' + typ)
+    kt.write('(')
+    for colname, db_type, _ in columns:
+        if colname != kind:
+            kt.write(colname + ': ')
+            if db_type == 'int':
+                kt.write('Int')
+            elif db_type == 'float':
+                kt.write('Double')
+            elif db_type == 'string':
+                kt.write('String')
+            elif db_type == 'date':
+                kt.write('Date')
+            elif db_type == 'boolean':
+                kt.write('Boolean')
+            elif db_type[0] == '@':
+                label = db_type[1:]
+                if label == enum:
+                    label = typ
+                kt.write('Label<out Db' + upperFirst(label) + '>')
+            else:
+                raise Exception('Bad db_type: ' + db_type)
+            kt.write(', ')
+    kt.write(') {\n')
+    kt.write('    this.writeTrap("' + relname + '(')
+    comma = ''
+    for colname, db_type, _ in columns:
+        kt.write(comma)
+        if colname == kind:
+            kt.write(str(num))
+        elif db_type == 'string':
+            kt.write('\\"${this.escapeTrapString(this.truncateString(' + colname + '))}\\"')
+        elif db_type == 'date':
+            kt.write('D\\"${' + colname + '}\\"')
+        else:
+            kt.write('$' + colname)
+        comma = ', '
+    kt.write(')\\n")\n')
+    kt.write('}\n')
+
+with open('src/main/kotlin/KotlinExtractorDbScheme.kt', 'w') as kt:
+    kt.write('/* Generated by ' + sys.argv[0] + ': Do not edit manually. */\n')
+    kt.write('package com.github.codeql\n')
+    kt.write('import java.util.Date\n')
+
+    for relname, columns in tables.items():
+        enum = None
+        for _, db_type, ref in columns:
+            if db_type[0] == '@' and ref == '':
+                db_type_name = db_type[1:]
+                if db_type_name in enums:
+                    enum = db_type_name
+        if enum is None:
+            genTable(kt, relname, columns)
+        else:
+            (kind, mapping) = enums[enum]
+            for num, typ in mapping:
+                genTable(kt, relname, columns, enum, kind, num, typ)
+
+    for typ in sorted(supertypes):
+        kt.write('sealed interface Db' + upperFirst(typ))
+        # Sorting makes the output deterministic.
+        names = sorted(supertypes[typ])
+        if names:
+            kt.write(': ')
+            kt.write(', '.join(map(lambda name: 'Db' + upperFirst(name), names)))
+        kt.write('\n')
+    for alias in sorted(type_aliases):
+        kt.write('typealias Db' + upperFirst(alias) + ' = Db' + upperFirst(type_aliases[alias]) + '\n')

java/kotlin-extractor/gradle.properties

@@ -0,0 +1,8 @@
+kotlin.code.style=official
+kotlinVersion=1.6.20
+
+GROUP=com.github.codeql
+VERSION_NAME=0.0.1
+POM_DESCRIPTION=CodeQL Kotlin extractor
+OUTPUT_JAR_NAME=codeql-extractor-kotlin.jar
+

java/kotlin-extractor/kotlin_plugin_versions.py

@@ -0,0 +1,62 @@
+import platform
+import re
+import subprocess
+import sys
+
+def is_windows():
+    '''Whether we appear to be running on Windows'''
+    if platform.system() == 'Windows':
+        return True
+    if platform.system().startswith('CYGWIN'):
+        return True
+    return False
+
+def version_tuple_to_string(version):
+    return f'{version[0]}.{version[1]}.{version[2]}'
+
+def version_string_to_tuple(version):
+    m = re.match(r'([0-9]+)\.([0-9]+)\.([0-9]+)', version)
+    return tuple([int(m.group(i)) for i in range(1, 4)])
+
+many_versions = [ '1.4.32', '1.5.31', '1.6.10', '1.6.20' ]
+
+many_versions_tuples = [version_string_to_tuple(v) for v in many_versions]
+
+def get_single_version(fakeVersionOutput = None):
+    # TODO: `shell=True` is a workaround to get CI working on Windows. It breaks the build on Linux.
+    versionOutput = subprocess.run(['kotlinc', '-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, shell=is_windows()).stderr if fakeVersionOutput is None else fakeVersionOutput
+    m = re.match(r'.* kotlinc-jvm ([0-9]+\.[0-9]+\.[0-9]+) .*', versionOutput)
+    if m is None:
+        raise Exception('Cannot detect version of kotlinc (got ' + str(versionOutput) + ')')
+    current_version = version_string_to_tuple(m.group(1))
+    matching_minor_versions = [ version for version in many_versions_tuples if version[0:2] == current_version[0:2] ]
+    if len(matching_minor_versions) == 0:
+        raise Exception(f'Cannot find a matching minor version for kotlinc version {current_version} (got {versionOutput}; know about {str(many_versions)})')
+
+    matching_minor_versions.sort()
+
+    for version in matching_minor_versions:
+        if version >= current_version:
+            return version_tuple_to_string(version)
+
+    return version_tuple_to_string(matching_minor_versions[-1])
+
+    raise Exception(f'No suitable kotlinc version found for {current_version} (got {versionOutput}; know about {str(many_versions)})')
+
+def get_latest_url():
+    version = many_versions[-1]
+    url = 'https://github.com/JetBrains/kotlin/releases/download/v' + version + '/kotlin-compiler-' + version + '.zip'
+    return url
+
+if __name__ == "__main__":
+    args = sys.argv
+    if len(args) < 2:
+        raise Exception("Bad arguments")
+    command = args[1]
+    if command == 'latest-url':
+        print(get_latest_url())
+    elif command == 'single-version':
+        print(get_single_version(*args[2:]))
+    else:
+        raise Exception("Unknown command: " + command)
+

java/kotlin-extractor/settings.gradle

@@ -0,0 +1,8 @@
+pluginManagement {
+    repositories {
+        mavenCentral()
+        gradlePluginPortal()
+    }
+}
+
+rootProject.name = 'codeql-kotlin-extractor'

java/kotlin-extractor/src/main/java/com/semmle/extractor/java/OdasaOutput.java

@@ -0,0 +1,628 @@
+package com.semmle.extractor.java;
+
+import java.lang.reflect.*;
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.regex.Pattern;
+
+import com.github.codeql.Logger;
+import static com.github.codeql.ClassNamesKt.getIrDeclBinaryName;
+import static com.github.codeql.ClassNamesKt.getIrClassVirtualFile;
+
+import org.jetbrains.kotlin.ir.declarations.IrClass;
+
+import com.intellij.openapi.vfs.VirtualFile;
+
+import org.jetbrains.kotlin.ir.declarations.IrDeclaration;
+import org.jetbrains.kotlin.ir.declarations.IrDeclarationWithName;
+import org.jetbrains.org.objectweb.asm.ClassVisitor;
+import org.jetbrains.org.objectweb.asm.ClassReader;
+import org.jetbrains.org.objectweb.asm.Opcodes;
+
+import com.semmle.util.concurrent.LockDirectory;
+import com.semmle.util.concurrent.LockDirectory.LockingMode;
+import com.semmle.util.exception.CatastrophicError;
+import com.semmle.util.exception.NestedError;
+import com.semmle.util.exception.ResourceError;
+import com.semmle.util.extraction.PopulationSpecFile;
+import com.semmle.util.extraction.SpecFileEntry;
+import com.semmle.util.files.FileUtil;
+import com.semmle.util.io.WholeIO;
+import com.semmle.util.process.Env;
+import com.semmle.util.process.Env.Var;
+import com.semmle.util.trap.dependencies.TrapDependencies;
+import com.semmle.util.trap.dependencies.TrapSet;
+import com.semmle.util.trap.pathtransformers.PathTransformer;
+
+public class OdasaOutput {
+
+	// either these are set ...
+	private final File trapFolder;
+	private final File sourceArchiveFolder;
+
+	// ... or this one is set
+	private final PopulationSpecFile specFile;
+
+	private File currentSourceFile;
+	private TrapSet trapsCreated;
+	private TrapDependencies trapDependenciesForSource;
+
+	private SpecFileEntry currentSpecFileEntry;
+
+	// should origin tracking be used?
+	private final boolean trackClassOrigins;
+
+	private final Logger log;
+
+	/** DEBUG only: just use the given file as the root for TRAP, source archive etc */
+	OdasaOutput(File outputRoot, Logger log) {
+		this.trapFolder = new File(outputRoot, "trap");
+		this.sourceArchiveFolder = new File(outputRoot, "src_archive");
+		this.specFile = null;
+		this.trackClassOrigins = false;
+		this.log = log;
+	}
+
+	public OdasaOutput(boolean trackClassOrigins, Logger log) {
+		String trapFolderVar = Env.systemEnv().getFirstNonEmpty("CODEQL_EXTRACTOR_JAVA_TRAP_DIR", Var.TRAP_FOLDER.name());
+		if (trapFolderVar != null) {
+			String sourceArchiveVar = Env.systemEnv().getFirstNonEmpty("CODEQL_EXTRACTOR_JAVA_SOURCE_ARCHIVE_DIR", Var.SOURCE_ARCHIVE.name());
+			if (sourceArchiveVar == null)
+				throw new ResourceError(Var.TRAP_FOLDER + " was set to '" + trapFolderVar + "', but "
+						+ Var.SOURCE_ARCHIVE + " was not set");
+			this.trapFolder = new File(trapFolderVar);
+			this.sourceArchiveFolder = new File(sourceArchiveVar);
+			this.specFile = null;
+		} else {
+			this.trapFolder = null;
+			this.sourceArchiveFolder = null;
+			String specFileVar = Env.systemEnv().get(Var.ODASA_JAVA_LAYOUT);
+			if (specFileVar == null)
+				throw new ResourceError("Neither " + Var.TRAP_FOLDER + " nor " + Var.ODASA_JAVA_LAYOUT + " was set");
+			this.specFile = new PopulationSpecFile(new File(specFileVar));
+		}
+		this.trackClassOrigins = trackClassOrigins;
+		this.log = log;
+	}
+
+	public File getTrapFolder() {
+		return trapFolder;
+	}
+
+	public boolean getTrackClassOrigins() {
+		return trackClassOrigins;
+	}
+
+	/**
+	 * Set the source file that is currently being processed. This may affect
+	 * things like trap and source archive directories, and persists as a
+	 * setting until this method is called again.
+	 * @param f the current source file
+	 */
+	public void setCurrentSourceFile(File f) {
+		currentSourceFile = f;
+		currentSpecFileEntry = entryFor();
+		trapsCreated = new TrapSet();
+		trapsCreated.addSource(PathTransformer.std().fileAsDatabaseString(f));
+		trapDependenciesForSource = null;
+	}
+
+	/** The output paths for that file, or null if it shouldn't be included */
+	private SpecFileEntry entryFor() {
+		if (specFile != null)
+			return specFile.getEntryFor(currentSourceFile);
+		else
+			return new SpecFileEntry(trapFolder, sourceArchiveFolder,
+					Arrays.asList(PathTransformer.std().fileAsDatabaseString(currentSourceFile)));
+	}
+
+	/*
+	 * Trap sets and dependencies.
+	 */
+
+	public void writeTrapSet() {
+		trapsCreated.save(trapSetFor(currentSourceFile).toPath());
+	}
+
+	private File trapSetFor(File file) {
+		return FileUtil.appendAbsolutePath(
+			currentSpecFileEntry.getTrapFolder(), PathTransformer.std().fileAsDatabaseString(file) + ".set");
+	}
+
+	public void addDependency(IrDeclaration sym, String signature) {
+		String path = trapFilePathForDecl(sym, signature);
+		trapDependenciesForSource.addDependency(path);
+	}
+
+	/*
+	 * Source archive.
+	 */
+
+	/**
+	 * Write the given source file to the right source archive, encoded in UTF-8,
+	 * or do nothing if the file shouldn't be populated.
+	 */
+	public void writeCurrentSourceFileToSourceArchive(String contents) {
+		if (currentSpecFileEntry != null && currentSpecFileEntry.getSourceArchivePath() != null) {
+			File target = sourceArchiveFileFor(currentSourceFile);
+			target.getParentFile().mkdirs();
+			new WholeIO().write(target, contents);
+		}
+	}
+
+	public void writeFileToSourceArchive(File srcFile) {
+		File target = sourceArchiveFileFor(srcFile);
+		target.getParentFile().mkdirs();
+		String contents = new WholeIO().strictread(srcFile);
+		new WholeIO().write(target, contents);
+	}
+
+	private File sourceArchiveFileFor(File file) {
+		return FileUtil.appendAbsolutePath(currentSpecFileEntry.getSourceArchivePath(),
+				PathTransformer.std().fileAsDatabaseString(file));
+	}
+
+	/*
+	 * Trap file names and paths.
+	 */
+
+	private static final String CLASSES_DIR = "classes";
+	private static final String JARS_DIR = "jars";
+	private static final String MODULES_DIR = "modules";
+
+	private File getTrapFileForCurrentSourceFile() {
+		if (currentSpecFileEntry == null)
+			return null;
+		return trapFileFor(currentSourceFile);
+	}
+
+	private File getTrapFileForJarFile(File jarFile) {
+		if (!jarFile.getAbsolutePath().endsWith(".jar"))
+			return null;
+		return FileUtil.appendAbsolutePath(
+				currentSpecFileEntry.getTrapFolder(),
+				JARS_DIR + "/" + PathTransformer.std().fileAsDatabaseString(jarFile) + ".trap.gz");
+	}
+
+	private File getTrapFileForModule(String moduleName) {
+		return FileUtil.appendAbsolutePath(
+				currentSpecFileEntry.getTrapFolder(),
+				MODULES_DIR + "/" + moduleName + ".trap.gz");
+	}
+
+	private File trapFileFor(File file) {
+		return FileUtil.appendAbsolutePath(currentSpecFileEntry.getTrapFolder(),
+				PathTransformer.std().fileAsDatabaseString(file) + ".trap.gz");
+	}
+
+	private File getTrapFileForDecl(IrDeclaration sym, String signature) {
+		if (currentSpecFileEntry == null)
+			return null;
+		return trapFileForDecl(sym, signature);
+	}
+
+	private File trapFileForDecl(IrDeclaration sym, String signature) {
+		return FileUtil.fileRelativeTo(currentSpecFileEntry.getTrapFolder(),
+				trapFilePathForDecl(sym, signature));
+	}
+
+	private final Map<String, String> memberTrapPaths = new LinkedHashMap<String, String>();
+	private static final Pattern dots = Pattern.compile(".", Pattern.LITERAL);
+	private String trapFilePathForDecl(IrDeclaration sym, String signature) {
+		String binaryName = getIrDeclBinaryName(sym);
+		String binaryNameWithSignature = binaryName + signature;
+		// TODO: Reinstate this?
+		//if (getTrackClassOrigins())
+		//  classId += "-" + StringDigestor.digest(sym.getSourceFileId());
+		String result = memberTrapPaths.get(binaryNameWithSignature);
+		if (result == null) {
+			result = CLASSES_DIR + "/" +
+					dots.matcher(binaryName).replaceAll("/") +
+					signature +
+					".members" +
+					".trap.gz";
+			memberTrapPaths.put(binaryNameWithSignature, result);
+		}
+		return result;
+	}
+
+	/*
+	 * Deletion of existing trap files.
+	 */
+
+	private void deleteTrapFileAndDependencies(IrDeclaration sym, String signature) {
+		File trap = trapFileForDecl(sym, signature);
+		if (trap.exists()) {
+			trap.delete();
+			File depFile = new File(trap.getParentFile(), trap.getName().replace(".trap.gz", ".dep"));
+			if (depFile.exists())
+				depFile.delete();
+			File metadataFile = new File(trap.getParentFile(), trap.getName().replace(".trap.gz", ".metadata"));
+			if (metadataFile.exists())
+				metadataFile.delete();
+		}
+	}
+
+	/*
+	 * Trap writers.
+	 */
+
+	/**
+	 * A {@link TrapFileManager} to output facts for the given source file,
+	 * or <code>null</code> if the source file should not be populated.
+	 */
+	private TrapFileManager getTrapWriterForCurrentSourceFile() {
+		File trapFile = getTrapFileForCurrentSourceFile();
+		if (trapFile==null)
+			return null;
+		return trapWriter(trapFile, null, null);
+	}
+
+	/**
+	 * Get a {@link TrapFileManager} to write members
+	 * about a declaration, or <code>null</code> if the declaration shouldn't be populated.
+	 *
+	 * @param sym
+	 * 		The declaration's symbol, including, in particular, its fully qualified
+	 * 		binary class name.
+	 * @param signature
+	 * 		Any unique suffix needed to distinguish `sym` from other declarations with the same name.
+	 * 		For functions for example, this means its parameter signature.
+	 */
+	private TrapFileManager getMembersWriterForDecl(IrDeclaration sym, String signature) {
+		File trap = getTrapFileForDecl(sym, signature);
+		if (trap==null)
+			return null;
+		TrapClassVersion currVersion = TrapClassVersion.fromSymbol(sym, log);
+		String shortName = sym instanceof IrDeclarationWithName ? ((IrDeclarationWithName)sym).getName().asString() : "(name unknown)";
+		if (trap.exists()) {
+			// Only re-write an existing trap file if we encountered a newer version of the same class.
+			TrapClassVersion trapVersion = readVersionInfo(trap);
+			if (!currVersion.isValid()) {
+				log.warn("Not rewriting trap file for: " + shortName + " " + trapVersion + " " + currVersion + " " + trap);
+			} else if (currVersion.newerThan(trapVersion)) {
+				log.trace("Rewriting trap file for: " + shortName + " " + trapVersion + " " + currVersion + " " + trap);
+				deleteTrapFileAndDependencies(sym, signature);
+			} else {
+				return null;
+			}
+		} else {
+			log.trace("Writing trap file for: " + shortName + " " + currVersion + " " + trap);
+		}
+		return trapWriter(trap, sym, signature);
+	}
+
+	private TrapFileManager trapWriter(File trapFile, IrDeclaration sym, String signature) {
+		if (!trapFile.getName().endsWith(".trap.gz"))
+			throw new CatastrophicError("OdasaOutput only supports writing to compressed trap files");
+		String relative = FileUtil.relativePath(trapFile, currentSpecFileEntry.getTrapFolder());
+		trapFile.getParentFile().mkdirs();
+		trapsCreated.addTrap(relative);
+		return concurrentWriter(trapFile, relative, log, sym, signature);
+	}
+
+	private TrapFileManager concurrentWriter(File trapFile, String relative, Logger log, IrDeclaration sym, String signature) {
+		if (trapFile.exists())
+			return null;
+		return new TrapFileManager(trapFile, relative, true, log, sym, signature);
+	}
+
+	public class TrapFileManager implements AutoCloseable {
+
+		private TrapDependencies trapDependenciesForClass;
+		private File trapFile;
+		private IrDeclaration sym;
+		private String signature;
+		private boolean hasError = false;
+
+		private TrapFileManager(File trapFile, String relative, boolean concurrentCreation, Logger log, IrDeclaration sym, String signature) {
+			trapDependenciesForClass = new TrapDependencies(relative);
+			this.trapFile = trapFile;
+			this.sym = sym;
+			this.signature = signature;
+		}
+
+		public File getFile() {
+			return trapFile;
+		}
+
+		public void addDependency(IrDeclaration dep, String signature) {
+			trapDependenciesForClass.addDependency(trapFilePathForDecl(dep, signature));
+		}
+
+		public void addDependency(IrClass c) {
+			addDependency(c, "");
+		}
+
+		public void close() {
+			if (hasError) {
+				return;
+			}
+
+			writeTrapDependencies(trapDependenciesForClass);
+			// Record major/minor version information for extracted class files.
+			// This is subsequently used to determine whether to re-extract (a newer version of) the same class.
+			File metadataFile = new File(trapFile.getAbsolutePath().replace(".trap.gz", ".metadata"));
+			try {
+				Map<String, String> versionMap = new LinkedHashMap<>();
+				TrapClassVersion tcv = TrapClassVersion.fromSymbol(sym, log);
+				versionMap.put(MAJOR_VERSION, String.valueOf(tcv.getMajorVersion()));
+				versionMap.put(MINOR_VERSION, String.valueOf(tcv.getMinorVersion()));
+				versionMap.put(LAST_MODIFIED, String.valueOf(tcv.getLastModified()));
+				versionMap.put(EXTRACTOR_NAME, tcv.getExtractorName());
+				FileUtil.writePropertiesCSV(metadataFile, versionMap);
+			} catch (IOException e) {
+				log.warn("Could not save trap metadata file: " + metadataFile.getAbsolutePath(), e);
+			}
+		}
+		private void writeTrapDependencies(TrapDependencies trapDependencies) {
+			String dep = trapDependencies.trapFile().replace(".trap.gz", ".dep");
+			trapDependencies.save(
+					currentSpecFileEntry.getTrapFolder().toPath().resolve(dep));
+		}
+
+		public void setHasError() {
+			hasError = true;
+		}
+	}
+
+	/*
+	 * Trap file locking.
+	 */
+
+	/**
+	 * <b>CAUTION</b>: to avoid the potential for deadlock between multiple concurrent extractor processes,
+	 * only one source file {@link TrapLocker} may be open at any time, and the lock must be obtained
+	 * <b>before</b> any <b>class</b> file lock.
+	 *
+	 * Trap file extensions (and paths) ensure that source and class file locks are distinct.
+	 *
+	 * @return  a {@link TrapLocker} for the currently processed source file, which must have been
+	 * previously set by a call to {@link OdasaOutput#setCurrentSourceFile(File)}.
+	 */
+	public TrapLocker getTrapLockerForCurrentSourceFile() {
+		return new TrapLocker((IrClass)null, null);
+	}
+
+	/**
+	 * <b>CAUTION</b>: to avoid the potential for deadlock between multiple concurrent extractor processes,
+	 * only one jar file {@link TrapLocker} may be open at any time, and the lock must be obtained
+	 * <b>after</b> any <b>source</b> file lock. Only one jar or class file lock may be open at any time.
+	 *
+	 * Trap file extensions (and paths) ensure that source and jar file locks are distinct.
+	 *
+	 * @return  a {@link TrapLocker} for the trap file corresponding to the given jar file.
+	 */
+	public TrapLocker getTrapLockerForJarFile(File jarFile) {
+		return new TrapLocker(jarFile);
+	}
+
+	/**
+	 * <b>CAUTION</b>: to avoid the potential for deadlock between multiple concurrent extractor processes,
+	 * only one module {@link TrapLocker} may be open at any time, and the lock must be obtained
+	 * <b>after</b> any <b>source</b> file lock. Only one jar or class file or module lock may be open at any time.
+	 *
+	 * Trap file extensions (and paths) ensure that source and module file locks are distinct.
+	 *
+	 * @return  a {@link TrapLocker} for the trap file corresponding to the given module.
+	 */
+	public TrapLocker getTrapLockerForModule(String moduleName) {
+		return new TrapLocker(moduleName);
+	}
+
+	/**
+	 * <b>CAUTION</b>: to avoid the potential for deadlock between multiple concurrent extractor processes,
+	 * only one class file {@link TrapLocker} may be open at any time, and the lock must be obtained
+	 * <b>after</b> any <b>source</b> file lock. Only one jar or class file lock may be open at any time.
+	 *
+	 * Trap file extensions (and paths) ensure that source and class file locks are distinct.
+	 *
+	 * @return  a {@link TrapLocker} for the trap file corresponding to the given class symbol.
+	 */
+	public TrapLocker getTrapLockerForDecl(IrDeclaration sym, String signature) {
+		return new TrapLocker(sym, signature);
+	}
+
+	public class TrapLocker implements AutoCloseable {
+		private final IrDeclaration sym;
+		private final File trapFile;
+		private final String signature;
+		private final boolean isNonSourceTrapFile;
+		private TrapLocker(IrDeclaration decl, String signature) {
+			this.sym = decl;
+			this.signature = signature;
+			if (sym==null) {
+				trapFile = getTrapFileForCurrentSourceFile();
+			} else {
+				trapFile = getTrapFileForDecl(sym, signature);
+			}
+			isNonSourceTrapFile = false;
+		}
+		private TrapLocker(File jarFile) {
+			sym = null;
+			signature = null;
+			trapFile = getTrapFileForJarFile(jarFile);
+			isNonSourceTrapFile = true;
+		}
+		private TrapLocker(String moduleName) {
+			sym = null;
+			signature = null;
+			trapFile = getTrapFileForModule(moduleName);
+			isNonSourceTrapFile = true;
+		}
+		public TrapFileManager getTrapFileManager() {
+			if (trapFile!=null) {
+				lockTrapFile(trapFile);
+				return getMembersWriterForDecl(sym, signature);
+			} else {
+				return null;
+			}
+		}
+		@Override
+		public void close() {
+			if (trapFile!=null) {
+				try {
+					unlockTrapFile(trapFile);
+				} catch (NestedError e) {
+					log.warn("Error unlocking trap file " + trapFile.getAbsolutePath(), e);
+				}
+			}
+		}
+
+		private LockDirectory getExtractorLockDir() {
+			return LockDirectory.instance(currentSpecFileEntry.getTrapFolder(), log);
+		}
+
+		private void lockTrapFile(File trapFile) {
+			getExtractorLockDir().blockingLock(LockingMode.Exclusive, trapFile, "Java extractor lock");
+		}
+
+		private void unlockTrapFile(File trapFile) {
+			boolean success = getExtractorLockDir().maybeUnlock(LockingMode.Exclusive, trapFile);
+			if (!success) {
+				log.warn("Trap file was not locked: " + trapFile);
+			}
+		}
+	}
+
+	/*
+	 * Class version tracking.
+	 */
+
+	private static final String MAJOR_VERSION = "majorVersion";
+	private static final String MINOR_VERSION = "minorVersion";
+	private static final String LAST_MODIFIED = "lastModified";
+	private static final String EXTRACTOR_NAME = "extractorName";
+
+	private static class TrapClassVersion {
+		private int majorVersion;
+		private int minorVersion;
+		private long lastModified;
+		private String extractorName; // May be null if not given
+
+		public int getMajorVersion() {
+			return majorVersion;
+		}
+
+		public int getMinorVersion() {
+			return minorVersion;
+		}
+
+		public long getLastModified() {
+			return lastModified;
+		}
+
+		public String getExtractorName() { return extractorName; }
+
+		private TrapClassVersion(int majorVersion, int minorVersion, long lastModified, String extractorName) {
+			this.majorVersion = majorVersion;
+			this.minorVersion = minorVersion;
+			this.lastModified = lastModified;
+			this.extractorName = extractorName;
+		}
+		private boolean newerThan(TrapClassVersion tcv) {
+			// Classes being compiled from source have major version 0 but should take precedence
+			// over any classes with the same qualified name loaded from the classpath
+			// in previous or subsequent extractor invocations.
+			if (tcv.majorVersion==0)
+				return false;
+			else if (majorVersion==0)
+				return true;
+			// Always consider the Kotlin extractor superior to the Java extractor, because we may decode and extract
+			// Kotlin metadata that the Java extractor can't understand:
+			if(!Objects.equals(tcv.extractorName, extractorName)) {
+				if (Objects.equals(tcv.extractorName, "kotlin"))
+					return false;
+				if (Objects.equals(extractorName, "kotlin"))
+					return true;
+			}
+			// Otherwise, determine precedence in the following order:
+			// majorVersion, minorVersion, lastModified.
+			return tcv.majorVersion < majorVersion ||
+					(tcv.majorVersion == majorVersion && tcv.minorVersion < minorVersion) ||
+					(tcv.majorVersion == majorVersion && tcv.minorVersion == minorVersion &&
+					tcv.lastModified < lastModified);
+		}
+		private static TrapClassVersion fromSymbol(IrDeclaration sym, Logger log) {
+			VirtualFile vf = sym instanceof IrClass ? getIrClassVirtualFile((IrClass)sym) :
+					sym.getParent() instanceof IrClass ? getIrClassVirtualFile((IrClass)sym.getParent()) :
+					null;
+			if(vf == null)
+				return new TrapClassVersion(-1, 0, 0, null);
+
+			final int[] versionStore = new int[1];
+
+			try {
+				// Opcodes has fields called ASM4, ASM5, ...
+				// We want to use the latest one that there is.
+				Field asmField = null;
+				int asmNum = -1;
+				for(Field f : Opcodes.class.getDeclaredFields()) {
+					String name = f.getName();
+					if(name.startsWith("ASM")) {
+						try {
+							int i = Integer.parseInt(name.substring(3));
+							if(i > asmNum) {
+								asmNum = i;
+								asmField = f;
+							}
+						} catch (NumberFormatException ex) {
+							// Do nothing; this field doesn't have a name of the right format
+						}
+					}
+				}
+				int asm = asmField.getInt(null);
+				ClassVisitor versionGetter = new ClassVisitor(asm) {
+					public void visit(int version, int access, java.lang.String name, java.lang.String signature, java.lang.String superName, java.lang.String[] interfaces) {
+						versionStore[0] = version;
+					}
+				};
+				(new ClassReader(vf.contentsToByteArray())).accept(versionGetter, ClassReader.SKIP_CODE | ClassReader.SKIP_DEBUG | ClassReader.SKIP_FRAMES);
+
+				return new TrapClassVersion(versionStore[0] & 0xffff, versionStore[0] >> 16, vf.getTimeStamp(), "kotlin");
+			}
+			catch(IllegalAccessException e) {
+				log.warn("Failed to read class file version information", e);
+				return new TrapClassVersion(-1, 0, 0, null);
+			}
+			catch(IOException e) {
+				log.warn("Failed to read class file version information", e);
+				return new TrapClassVersion(-1, 0, 0, null);
+			}
+		}
+		private boolean isValid() {
+			return majorVersion>=0 && minorVersion>=0;
+		}
+		@Override
+		public String toString() {
+			return majorVersion + "." + minorVersion + "-" + lastModified + "-" + extractorName;
+		}
+	}
+
+	private TrapClassVersion readVersionInfo(File trap) {
+		int majorVersion = 0;
+		int minorVersion = 0;
+		long lastModified = 0;
+		String extractorName = null;
+		File metadataFile = new File(trap.getAbsolutePath().replace(".trap.gz", ".metadata"));
+		if (metadataFile.exists()) {
+			Map<String,String> metadataMap = FileUtil.readPropertiesCSV(metadataFile);
+			try {
+				majorVersion = Integer.parseInt(metadataMap.get(MAJOR_VERSION));
+				minorVersion = Integer.parseInt(metadataMap.get(MINOR_VERSION));
+				lastModified = Long.parseLong(metadataMap.get(LAST_MODIFIED));
+				extractorName = metadataMap.get(EXTRACTOR_NAME);
+			} catch (NumberFormatException e) {
+				log.warn("Invalid class file version for " + trap.getAbsolutePath(), e);
+			}
+		} else {
+			log.warn("Trap metadata file does not exist: " + metadataFile.getAbsolutePath());
+		}
+		return new TrapClassVersion(majorVersion, minorVersion, lastModified, extractorName);
+	}
+
+}

java/kotlin-extractor/src/main/java/com/semmle/extractor/java/PopulateFile.java

@@ -0,0 +1,152 @@
+package com.semmle.extractor.java;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.Map;
+
+import com.github.codeql.Label;
+import com.github.codeql.DbFile;
+import com.github.codeql.TrapWriter;
+import com.github.codeql.KotlinExtractorDbSchemeKt;
+import com.semmle.util.exception.CatastrophicError;
+import com.semmle.util.files.FileUtil;
+import com.semmle.util.trap.pathtransformers.PathTransformer;
+import kotlin.Unit;
+
+public class PopulateFile {
+
+  private TrapWriter tw;
+  private PathTransformer transformer;
+  public PopulateFile(TrapWriter tw) {
+    this.tw = tw;
+    this.transformer = PathTransformer.std();
+  }
+
+	private static final String[] keyReplacementMap = new String[127];
+	static {
+		keyReplacementMap['&'] = "&";
+		keyReplacementMap['{'] = "{";
+		keyReplacementMap['}'] = "}";
+		keyReplacementMap['"'] = """;
+		keyReplacementMap['@'] = "@";
+		keyReplacementMap['#'] = "#";
+	}
+
+	/**
+	 * Escape a string for use in a TRAP key, by replacing special characters with HTML entities.
+	 * <p>
+	 * The given string cannot contain any sub-keys, as the delimiters <code>{</code> and <code>}</code>
+	 * are escaped.
+	 * <p>
+	 * To construct a key containing both sub-keys and arbitrary input data, escape the individual parts of
+	 * the key rather than the key as a whole, for example:
+	 * <pre>
+	 * "foo;{" + label.toString() + "};" + escapeKey(data)
+	 * </pre>
+	 */
+	public static String escapeKey(String s) {
+		StringBuilder sb = null;
+		int lastIndex = 0;
+		for (int i = 0; i < s.length(); ++i) {
+			char ch = s.charAt(i);
+			switch (ch) {
+			case '&':
+			case '{':
+			case '}':
+			case '"':
+			case '@':
+			case '#':
+				if (sb == null) {
+					sb = new StringBuilder();
+				}
+				sb.append(s, lastIndex, i);
+				sb.append(keyReplacementMap[ch]);
+				lastIndex = i + 1;
+				break;
+			}
+		}
+		if (sb != null) {
+			sb.append(s, lastIndex, s.length());
+			return sb.toString();
+		} else {
+			return s;
+		}
+	}
+
+    public Label populateFile(File absoluteFile) {
+        return getFileLabel(absoluteFile, true);
+    }
+
+    public Label<DbFile> getFileLabel(File absoluteFile, boolean populateTables) {
+        String databasePath = transformer.fileAsDatabaseString(absoluteFile);
+        Label result = tw.<DbFile>getLabelFor("@\"" + escapeKey(databasePath) + ";sourcefile" + "\"", label -> {
+            if(populateTables) {
+                KotlinExtractorDbSchemeKt.writeFiles(tw, label, databasePath);
+                populateParents(new File(databasePath), label);
+            }
+            return Unit.INSTANCE;
+        });
+        return result;
+    }
+
+	private Label addFolderTuple(String databasePath) {
+		Label result = tw.getLabelFor("@\"" + escapeKey(databasePath) + ";folder" + "\"");
+		KotlinExtractorDbSchemeKt.writeFolders(tw, result, databasePath);
+		return result;
+	}
+
+	/**
+	 * Populate the parents of an already-normalised file. The path transformers
+	 * and canonicalisation of {@link PathTransformer#fileAsDatabaseString(File)} will not be
+	 * re-applied to this, so it should only be called after proper normalisation
+	 * has happened. It will fill in all parent folders in the current TRAP file.
+	 */
+	private void populateParents(File normalisedFile, Label label) {
+		File parent = normalisedFile.getParentFile();
+		if (parent == null) return;
+
+    Label parentLabel = addFolderTuple(FileUtil.normalisePath(parent.getPath()));
+    populateParents(parent, parentLabel);
+		KotlinExtractorDbSchemeKt.writeContainerparent(tw, parentLabel, label);
+	}
+
+    public Label relativeFileId(File jarFile, String pathWithinJar) {
+        return getFileInJarLabel(jarFile, pathWithinJar, true);
+    }
+
+	public Label<DbFile> getFileInJarLabel(File jarFile, String pathWithinJar, boolean populateTables) {
+		if (pathWithinJar.contains("\\"))
+			throw new CatastrophicError("Invalid jar path: '" + pathWithinJar + "' should not contain '\\'.");
+
+        String databasePath = transformer.fileAsDatabaseString(jarFile);
+        if(!populateTables)
+            return tw.getLabelFor("@\"" + databasePath + "/" + pathWithinJar + ";jarFile\"");
+
+		Label jarFileId = this.populateFile(jarFile);
+        Label jarFileLocation = tw.getLocation(jarFileId, 0, 0, 0, 0);
+        KotlinExtractorDbSchemeKt.writeHasLocation(tw, jarFileId, jarFileLocation);
+
+		StringBuilder fullName = new StringBuilder(databasePath);
+		String[] split = pathWithinJar.split("/");
+		Label current = jarFileId;
+		for (int i = 0; i < split.length; i++) {
+			String shortName = split[i];
+
+			fullName.append("/");
+			fullName.append(shortName);
+			Label fileId = tw.getLabelFor("@\"" + fullName + ";jarFile" + "\"");
+
+			boolean file = i == split.length - 1;
+			if (file) {
+				KotlinExtractorDbSchemeKt.writeFiles(tw, fileId, fullName.toString());
+			} else {
+				KotlinExtractorDbSchemeKt.writeFolders(tw, fileId, fullName.toString());
+			}
+			KotlinExtractorDbSchemeKt.writeContainerparent(tw, current, fileId);
+			current = fileId;
+		}
+
+		return current;
+	}
+
+}

java/kotlin-extractor/src/main/java/com/semmle/util/array/ArrayUtil.java

@@ -0,0 +1,246 @@
+package com.semmle.util.array;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+import com.semmle.util.basic.ObjectUtil;
+
+/**
+ * Convenience methods for manipulating arrays.
+ */
+public class ArrayUtil
+{
+	
+	/**
+	 * A number slightly smaller than the maximum length of an array on most vms. 
+	 * This matches the constant in ArrayList.
+	 */
+	public static final int MAX_ARRAY_LENGTH = Integer.MAX_VALUE - 8;
+
+	/**
+	 * Comparator for primitive int values.
+	 */
+	public static interface IntComparator
+	{
+		/**
+		 * Compare ints {@code a} and {@code b}, returning a negative value if {@code a} is 'less' than
+		 * {@code b}, zero if they are equal, otherwise a positive value.
+		 */
+		public int compare (int a, int b);
+	}
+
+	/**
+	 * Find the index of the first occurrence of the given {@code value} in the given {@code array},
+	 * returning -1 if there is no such element.
+	 */
+	public static int findFirst(boolean[] array, boolean value)
+	{
+		for(int i=0; i<array.length; ++i) {
+			if (value == array[i])
+				return i;
+		}
+		return -1;
+	}
+
+	/**
+	 * Find the index of the first occurrence of the given {@code value} in the given {@code array},
+	 * returning -1 if there is no such element.
+	 */
+	public static int findFirst(byte[] array, byte value)
+	{
+		for(int i=0; i<array.length; ++i) {
+			if (value == array[i])
+				return i;
+		}
+		return -1;
+	}
+
+	/**
+	 * Find the index of the first occurrence of the given {@code value} in the given {@code array},
+	 * returning -1 if there is no such element.
+	 */
+	public static int findFirst(char[] array, char value)
+	{
+		for(int i=0; i<array.length; ++i) {
+			if (value == array[i])
+				return i;
+		}
+		return -1;
+	}
+
+	/**
+	 * Find the index of the first occurrence of the given {@code value} in the given {@code array},
+	 * returning -1 if there is no such element.
+	 */
+	public static int findFirst(double[] array, double value)
+	{
+		for(int i=0; i<array.length; ++i) {
+			if (value == array[i])
+				return i;
+		}
+		return -1;
+	}
+
+	/**
+	 * Find the index of the first occurrence of the given {@code value} in the given {@code array},
+	 * returning -1 if there is no such element.
+	 */
+	public static int findFirst(float[] array, float value)
+	{
+		for(int i=0; i<array.length; ++i) {
+			if (value == array[i])
+				return i;
+		}
+		return -1;
+	}
+
+
+	/**
+	 * Find the index of the first occurrence of the given {@code value} in the given {@code array},
+	 * returning -1 if there is no such element.
+	 */
+	public static int findFirst(int[] array, int value)
+	{
+		for(int i=0; i<array.length; ++i) {
+			if (value == array[i])
+				return i;
+		}
+		return -1;
+	}
+
+	/**
+	 * Find the index of the first occurrence of the given {@code value} in the given {@code array},
+	 * returning -1 if there is no element for which {@code value.equals(element)} is true.
+	 *
+	 * @see #findFirstSame(Object[], Object)
+	 */
+	public static <T> int findFirst(T[] array, T value)
+	{
+		for(int i=0; i<array.length; ++i) {
+			if (ObjectUtil.equals(value, array[i])) {
+				return i;
+			}
+		}
+		return -1;
+	}
+
+	/**
+	 * Find the index of the first occurrence of the given {@code value} in the given {@code array},
+	 * returning -1 if there is no element for which {@code value == element}.
+	 *
+	 * @see #findFirstSame(Object[], Object)
+	 */
+	public static <T> int findFirstSame(T[] array, T value)
+	{
+		for(int i=0; i<array.length; ++i) {
+			if (value == array[i])
+				return i;
+		}
+		return -1;
+	}
+
+	/**
+	 * Query whether the given {@code array} contains any element equal to the given {@code element}.
+	 */
+	public static boolean contains (int element, int ... array)
+	{
+		return findFirst(array, element) != -1;
+	}
+
+	
+
+	/**
+	 * Query whether the given {@code array} contains any element equal to the given {@code element}.
+	 */
+	@SafeVarargs
+	public static <T> boolean contains (T element, T ... array)
+	{
+		return findFirst(array, element) != -1;
+	}
+
+	/**
+	 * Construct a new array with length increased by one, containing all elements of a given array
+	 * followed by an additional element.
+	 */
+	public static <T> T[] append (T[] array, T element)
+	{
+		array = Arrays.copyOf(array, array.length + 1);
+		array[array.length-1] = element;
+		return array;
+	}
+
+	/**
+	 * Construct a new array containing the concatenation of the elements in a number of arrays.
+	 *
+	 * @param arrays The arrays to concatenate; may be null (in which case the result will be null).
+	 *          Null elements will be treated as empty arrays.
+	 * @return If {@code arrays} is null, a null array, otherwise a newly allocated array containing
+	 *         the elements of every non-null array in {@code arrays} concatenated consecutively.
+	 */
+	public static byte[] concatenate (byte[] ... arrays)
+	{
+		// Quick break-out if arrays is null
+		if (arrays == null) {
+			return null;
+		}
+		// Find the total length that will be required
+		int totalLength = 0;
+		for(byte[] array : arrays) {
+			totalLength += array == null ? 0 : array.length;
+		}
+		// Allocate a new array for the concatenation
+		byte[] concatenation = new byte[totalLength];
+		// Copy each non-null array into the concatenation
+		int offset = 0;
+		for(byte[] array : arrays) {
+			if (array != null) {
+				System.arraycopy(array, 0, concatenation, offset, array.length);
+				offset += array.length;
+			}
+		}
+
+		return concatenation;
+	}
+
+	/** Trivial short-hand for building an array (returns {@code elements} unchanged). */
+	public static <T> T[] toArray (T ... elements)
+	{
+		return elements;
+	}
+	
+	/**
+	 * Swap two elements in an array.
+	 *
+	 * @param array The array containing the elements to be swapped; must be non-null.
+	 * @param index1 The index of the first element to swap; must be in-bounds.
+	 * @param index2 The index of the second element to swap; must be in-bounds.
+	 * @return The given {@code array}.
+	 */
+	public static int[] swap (int[] array, int index1, int index2)
+	{
+		int value     = array[index1];
+		array[index1] = array[index2];
+		array[index2] = value;
+
+		return array;
+	}
+
+	/**
+	 * Returns a fresh Set containing all the elements in the array.
+	 *
+	 * @param <T>
+	 *            the class of the objects in the array
+	 * @param array
+	 *            the array containing the elements
+	 * @return a Set containing all the elements in the array.
+	 */
+	@SafeVarargs
+	public static <T> Set<T> asSet (T ... array)
+	{
+		Set<T> ts = new LinkedHashSet<>();
+		Collections.addAll(ts, array);
+		return ts;
+	}
+}

java/kotlin-extractor/src/main/java/com/semmle/util/basic/ObjectUtil.java

@@ -0,0 +1,73 @@
+package com.semmle.util.basic;
+
+/**
+ * Trivial utility methods.
+ */
+public class ObjectUtil {
+
+	/** Query if {@code object1} and {@code object2} are reference-equal, or both null. */
+	public static boolean isSame (Object object1, Object object2)
+	{
+		return object1 == object2; // Reference equality comparison is deliberate
+	}
+
+	/**
+	 * Query if {@code object1} and {@code object2} are both null, or both non-null and equal
+	 * according to {@link Object#equals(Object)} (applied as {@code object1.equals(object2)}).
+	 */
+	public static boolean equals (Object object1, Object object2)
+	{
+		return object1 == null ? object2 == null : object1.equals(object2);
+	}
+
+	/**
+	 * Query whether {@code object} is equal to any element in {@code objects}, short-circuiting
+	 * the evaluation if possible.
+	 */
+	public static boolean equalsAny (Object object, Object ... objects)
+	{
+		// Quick break-out if there are no objects to be equal to
+		if (objects == null || objects.length == 0) {
+			return false;
+		}
+		// Compare against each object in turn
+		for(Object other : objects) {
+			if (equals(object, other)) {
+				return true;
+			}
+		}
+
+		return false;
+	}
+
+	/**
+	 * Return {@code object1.compareTo(object2)}, but handle the case of null input by returning 0 if
+	 * both objects are null, or 1 if only {@code object1} is null (implying that null is always
+	 * 'greater' than non-null).
+	 */
+	public static <T1, T2 extends T1> int compareTo (Comparable<T1> object1, T2 object2)
+	{
+		if (object1 == null) {
+			return object2 == null ? 0 : 1;
+		}
+		return object1.compareTo(object2);
+	}
+
+	/**
+	 * Return {@code value} if non-null, otherwise {@code replacement}.
+	 */
+	public static <T> T replaceNull (T value, T replacement)
+	{
+		return value == null ? replacement : value;
+	}
+	
+	@SafeVarargs
+	public static <T> T nullCoalesce(T... values) {
+		for(T value : values) {
+			if (value != null) {
+				return value;
+			}
+		}
+		return null;
+	}
+}

java/kotlin-extractor/src/main/java/com/semmle/util/concurrent/LockDirectory.java

@@ -0,0 +1,395 @@
+package com.semmle.util.concurrent;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import com.semmle.util.data.StringDigestor;
+import com.semmle.util.exception.CatastrophicError;
+import com.semmle.util.exception.ResourceError;
+import com.semmle.util.files.FileUtil;
+import com.semmle.util.io.WholeIO;
+
+import com.github.codeql.Logger;
+import com.github.codeql.Severity;
+
+/**
+ * Helper class to simplify handling of file-system-based inter-process
+ * locking and mutual exclusion.
+ *
+ * Both files and directories can be locked; locks are provided in the
+ * usual flavours of "shared" and "exclusive", plus a no-op variety to
+ * help unify code -- see the {@link LockingMode} enum.
+ *
+ * Note that each locked file requires one file descriptor to be held open.
+ * It is vital for clients to avoid creating too many locks, and to release
+ * locks when possible.
+ *
+ * The locks obtained by this class are VM-wide, and cannot be used to
+ * ensure mutual exclusion between threads of the same VM. Rather, they
+ * can enforce mutual exclusion between separate VMs trying to acquire
+ * locks for the same paths.
+ */
+public class LockDirectory {
+	private final Logger logger;
+
+	private final File lockDir;
+
+	/**
+	 * An enum describing the possible locking modes.
+	 */
+	public enum LockingMode {
+		/**
+		 * Shared mode: A shared lock can be taken any number of times, but only
+		 * if no exclusive lock is in place.
+		 */
+		Shared(true),
+		/**
+		 * An exclusive lock can only be taken if no other lock is in place; it
+		 * prevents all other locks.
+		 */
+		Exclusive(false),
+		/**
+		 * A dummy mode: Lock and unlock operations are no-ops.
+		 */
+		None(true),
+		;
+
+		private boolean shared;
+
+		private LockingMode(boolean shared) {
+			this.shared = shared;
+		}
+
+		public boolean isShared() { return shared; }
+	}
+
+	/**
+	 * An internal representation of a locked path. Contains some immutable state: The canonical
+	 * path being locked, and the (derived) lock and status files. When the {@link #lock(LockDirectory.LockingMode, String)}
+	 * method is called, a file descriptor to the lock file is opened; {@link #unlock(LockDirectory.LockingMode)} must be
+	 * called to release it when the lock is no longer required.
+	 *
+	 * This class is not thread-safe, but it is expected that its clients ({@link LockDirectory})
+	 * enforce thread-safe access to instances.
+	 */
+	private class LockFile {
+		private final String lockedPath;
+		private final File lockFile;
+		private final File statusFile;
+
+		private LockingMode mode = null;
+		private RandomAccessFile lockStream = null;
+		private FileChannel lockChannel = null;
+		private FileLock lock = null;
+
+		public LockFile(File f) {
+			try {
+				lockedPath = f.getCanonicalPath();
+			} catch (IOException e) {
+				throw new ResourceError("Failed to canonicalise path for locking: " + f, e);
+			}
+			String sha = StringDigestor.digest(lockedPath);
+			lockFile = new File(lockDir, sha);
+			statusFile = new File(lockDir, sha + ".log");
+		}
+
+		/**
+		 * Get the (canonical) path associated with this lock file -- this is the
+		 * path that is being locked.
+		 */
+		public String getLockedPath() {
+			return lockedPath;
+		}
+
+		/**
+		 * Acquire a lock with the given mode. If this method returns normally,
+		 * the lock has been acquired -- an exception is thrown otherwise. This
+		 * method does not block.
+		 *
+		 * If no exception is thrown, a file descriptor is kept open until
+		 * {@link #unlock(LockDirectory.LockingMode)} is called.
+		 * @param mode The desired locking mode. If {@link LockingMode#None}, this
+		 * 	operation is a no-op (and does not in fact open a file descriptor).
+		 * @param message A message to be recorded alongside the lock file. This
+		 * is included in the exception message of other processes using this
+		 * 	infrastructure when the lock acquisition fails.
+		 * @throws CatastrophicError if a lock has already been obtained and not released.
+		 * @throws ResourceError if an exception occurs while obtaining the lock, including
+		 * 	if it cannot be acquired because another process holds it.
+		 */
+		public void lock(LockingMode mode, String message) {
+			if (mode == LockingMode.None) return;
+			if (lock != null)
+				throw new CatastrophicError("Trying to re-lock existing lock for " + lockedPath);
+			this.mode = mode;
+			try {
+				lockStream = new RandomAccessFile(lockFile, "rw");
+				lockChannel = lockStream.getChannel();
+				tryLock(mode);
+				new WholeIO().strictwrite(statusFile, mode + " lock acquired for " + lockedPath + ": " + message);
+			} catch (IOException e) {
+				throw new ResourceError("Failed to obtain lock for " + lockedPath + " at " + lockFile, e);
+			}
+		}
+
+		/**
+		 * Acquire a lock with the given mode. If this method returns normally,
+		 * the lock has been acquired -- an exception is thrown otherwise. This
+		 * method blocks indefinitely while waiting to acquire the lock.
+		 *
+		 * If no exception is thrown, a file descriptor is kept open until
+		 * {@link #unlock(LockDirectory.LockingMode)} is called.
+		 * @param mode The desired locking mode. If {@link LockingMode#None}, this
+		 * 	operation is a no-op (and does not in fact open a file descriptor).
+		 * @param message A message to be recorded alongside the lock file. This
+		 * is included in the exception message of other processes using this
+		 * 	infrastructure when the lock acquisition fails.
+		 * @throws ResourceError if an exception occurs while obtaining the lock,.
+		 */
+		public void blockingLock(LockingMode mode, String message) {
+			if (mode == LockingMode.None) return;
+			if (lock != null)
+				throw new CatastrophicError("Trying to re-lock existing lock for " + lockedPath);
+			this.mode = mode;
+			try {
+				lockStream = new RandomAccessFile(lockFile, "rw");
+				lockChannel = lockStream.getChannel();
+				lock = lockChannel.tryLock(0, Long.MAX_VALUE, mode.isShared());
+				while (lock == null) {
+					ThreadUtil.sleep(500, true);
+					lock = lockChannel.tryLock(0, Long.MAX_VALUE, mode.isShared());
+				}
+				new WholeIO().strictwrite(statusFile, mode + " lock acquired for " + lockedPath + ": " + message);
+			} catch (IOException e) {
+				throw new ResourceError("Failed to obtain lock for " + lockedPath + " at " + lockFile, e);
+			}
+		}
+
+		/**
+		 * Internal helper method: Try to acquire a particular kind of lock, assuming the
+		 * {@link #lockChannel} has been set up. Throws if acquisition fails, rather than
+		 * blocking.
+		 * @param mode The desired lock mode -- exclusive or shared.
+		 * @throws IOException if acquisition of the lock fails for reasons other than
+		 * 	an incompatible lock already being held by another process.
+		 * @throws ResourceError if the lock is already held by another process. The exception
+		 * 	message includes the status string, if it can be determined.
+		 */
+		private void tryLock(LockingMode mode) throws IOException {
+			lock = lockChannel.tryLock(0, Long.MAX_VALUE, mode.isShared());
+			if (lock == null) {
+				String status = new WholeIO().read(statusFile);
+				throw new ResourceError("Failed to acquire " + mode + " lock for " + lockedPath + "." +
+						(status == null ? "" : "\nExisting lock message: " + status));
+			}
+		}
+
+		/**
+		 * Release this lock. This will close the file descriptor opened by {@link #lock(LockDirectory.LockingMode, String)}.
+		 * @param mode A mode, which must match the mode passed into {@link #lock(LockDirectory.LockingMode, String)}
+		 * 	(unless it is {@link LockingMode#None}, in which case the method is a no-op).
+		 * @throws CatastrophicError if the passed mode does not match the one used for locking.
+		 * @throws ResourceError if releasing the lock or clearing up temporary files fails.
+		 */
+		public void unlock(LockingMode mode) {
+			if (mode == LockingMode.None)
+				return;
+			if (mode != this.mode)
+				throw new CatastrophicError("Attempting to unlock " + lockedPath + " with incompatible mode: " +
+						this.mode + " lock was obtained, but " + mode + " lock is being released.");
+			release(mode);
+		}
+
+		private void release(LockingMode mode) {
+			try {
+				if (lock != null)
+					try {
+						// On Windows, the lockChannel/lockStream prevents the lockFile from being
+						// deleted. The statusFile should only be written after the lock is held,
+						// so deleting it before releasing the lock is not expected to fail if the
+						// lock is exclusive.
+						// Deleting the lock file may fail, if another process just acquires it
+						// after we release it.
+						try {
+							if (statusFile.exists() && !statusFile.delete()) {
+									if (!mode.isShared()) throw new ResourceError("Could not clear status file " + statusFile);
+							}
+						} finally {
+							lock.release();
+							FileUtil.close(lockStream);
+							FileUtil.close(lockChannel);
+							if (!lockFile.delete())
+								logger.error("Could not clear lock file " + lockFile + " (it might have been locked by another process).");
+						}
+					} catch (IOException e) {
+						throw new ResourceError("Couldn't release lock for " + lockedPath, e);
+					}
+			} finally {
+				mode = null;
+				lockStream = null;
+				lockChannel = null;
+				lock = null;
+			}
+		}
+	}
+
+	private static final Map<File, LockDirectory> instances = new LinkedHashMap<File, LockDirectory>();
+
+	/**
+	 * Obtain the {@link LockDirectory} instance for a given lock directory. The directory
+	 * in question will be created if it doesn't exist.
+	 * @param lockDir A directory -- must be writable, and will be created if it doesn't
+	 * 	already exist.
+	 * @return The {@link LockDirectory} instance responsible for the specified lock directory.
+	 * @throws ResourceError if the directory cannot be created, exists as a non-directory
+	 * 	or cannot be canonicalised.
+	 */
+	public static synchronized LockDirectory instance(File lockDir) {
+		return instance(lockDir, null);
+	}
+
+	/**
+	 * See {@link #instance(File)}.
+	 * Use this method only if log output should be directed to a custom {@link Logger}.
+	 */
+	public static synchronized LockDirectory instance(File lockDir, Logger logger) {
+		// First try to create the directory -- canonicalisation will fail if it doesn't exist.
+		try {
+			FileUtil.mkdirs(lockDir);
+		} catch(ResourceError e) {
+			throw new ResourceError("Couldn't ensure lock directory " + lockDir + " exists.", e);
+		}
+
+		// Canonicalise.
+		try {
+			lockDir = lockDir.getCanonicalFile();
+		} catch (IOException e) {
+			throw new ResourceError("Couldn't canonicalise requested lock directory " + lockDir, e);
+		}
+
+		// Find and return the right instance.
+		LockDirectory instance = instances.get(lockDir);
+		if (instance == null) {
+			instance = new LockDirectory(lockDir, logger);
+			instances.put(lockDir, instance);
+		}
+		return instance;
+	}
+
+	/**
+	 * A map from canonical locked paths to the associated {@link LockFile} instances.
+	 */
+	private final Map<String, LockFile> locks = new LinkedHashMap<String, LockFile>();
+
+	/**
+	 * Create a new instance of {@link LockDirectory}, holding all locks in the
+	 * specified log directory.
+	 * @param lockDir A writable directory in which locks will be stored.
+	 * @param logger The {@link Logger} to use, if non-null.
+	 */
+	private LockDirectory(File lockDir, Logger logger) {
+		this.lockDir = lockDir;
+		this.logger = logger;
+	}
+
+	/**
+	 * Acquire a lock of the specified kind for the path represented by the given file.
+	 * The file should exist, and its path should be canonicalisable.
+	 *
+	 * Calling this method keeps one file descriptor open
+	 * @param mode The desired locking mode. If {@link LockingMode#None} is passed, this is a no-op,
+	 * 	otherwise it determines whether a shared or exclusive lock is acquired.
+	 * @param f The path that should be locked -- does not need to be writable, and will not
+	 * 	be opened.
+	 * @param message A message describing the purpose of the lock acquisition. This is
+	 * 	potentially displayed when other processes fail to acquire the lock for the given
+	 * 	path.
+	 * @throws CatastrophicError if an attempt is made to lock an already locked path.
+	 */
+	public synchronized void lock(LockingMode mode, File f, String message) {
+		if (mode == LockingMode.None) return;
+		LockFile lock = new LockFile(f);
+		if (locks.containsKey(lock.getLockedPath()))
+				throw new CatastrophicError("Trying to lock already locked path " + lock.getLockedPath() + ".");
+		lock.lock(mode, message);
+		locks.put(lock.getLockedPath(), lock);
+	}
+
+	/**
+	 * Acquire a lock of the specified kind for the path represented by the given file.
+	 * The file should exist, and its path should be canonicalisable. This method waits
+	 * indefinitely for the lock to become available. There is no ordering on processes
+	 * that are waiting to acquire the lock in this manner.
+	 *
+	 * Calling this method keeps one file descriptor open
+	 * @param mode The desired locking mode. If {@link LockingMode#None} is passed, this is a no-op,
+	 * 	otherwise it determines whether a shared or exclusive lock is acquired.
+	 * @param f The path that should be locked -- does not need to be writable, and will not
+	 * 	be opened.
+	 * @param message A message describing the purpose of the lock acquisition. This is
+	 * 	potentially displayed when other processes fail to acquire the lock for the given
+	 * 	path.
+	 */
+	public synchronized void blockingLock(LockingMode mode, File f, String message) {
+		if (mode == LockingMode.None) return;
+		LockFile lock = new LockFile(f);
+		if (locks.containsKey(lock.getLockedPath()))
+			throw new CatastrophicError("Trying to lock already locked path " + lock.getLockedPath() + ".");
+		lock.blockingLock(mode, message);
+		locks.put(lock.getLockedPath(), lock);
+	}
+
+	/**
+	 * Release a lock held on a particular path.
+	 *
+	 * This method closes the file descriptor associated with the lock, freeing related
+	 * resources.
+	 * @param mode the mode of the lock. If it equals {@link LockingMode#None}, this is a no-op; otherwise
+	 * 	it is expected to match the mode passed to the corresponding {@link #lock(LockingMode, File, String)}
+	 * 	call.
+	 * @param f The path which should be unlocked. As with {@link #lock(LockingMode, File, String)}, it is
+	 *	expected to exist and be canonicalisable. It also must be currently locked.
+	 * @throws CatastrophicError on API contract violation: The path isn't currently locked, or the
+	 * 	mode doesn't correspond to the mode specified when it was locked.
+	 * @throws ResourceError if something goes wrong while releasing resources.
+	 */
+	public synchronized void unlock(LockingMode mode, File f) {
+		if (!maybeUnlock(mode, f))
+			throw new CatastrophicError("Trying to unlock " + new LockFile(f).getLockedPath() + ", but it is not locked.");
+	}
+
+	/**
+	 * Release a lock that may be held on a particular path.
+	 *
+	 * This method closes the file descriptor associated with the lock, freeing related
+	 * resources. Unlike {@link #unlock(LockingMode, File)}, this method will not throw
+	 * if the specified {@link File} is not locked, making it more suitable for post-exception
+	 * cleanup -- <code>false</code> will be returned in that case.
+	 * @param mode the mode of the lock. If it equals {@link LockingMode#None}, this is a no-op; otherwise
+	 * 	it is expected to match the mode passed to the corresponding {@link #lock(LockingMode, File, String)}
+	 * 	call.
+	 * @param f The path which should be unlocked. As with {@link #lock(LockingMode, File, String)}, it is
+	 *	expected to exist and be canonicalisable.
+	 * @return <code>true</code> if <code>mode == LockingMode.None</code>, or the unlock operation completed
+	 * 		successfully; <code>false</code> if the path <code>f</code> isn't currently locked.
+	 * @throws ResourceError if something goes wrong while releasing resources.
+	 */
+	public synchronized boolean maybeUnlock(LockingMode mode, File f) {
+		if (mode == LockingMode.None) return true;
+		// New instance constructed just to share the logic of computing the canonical path.
+		LockFile key = new LockFile(f);
+		LockFile existing = locks.get(key.getLockedPath());
+		if (existing == null)
+			return false;
+		locks.remove(key.getLockedPath());
+		existing.unlock(mode);
+		return true;
+	}
+
+	public File getDir(){ return lockDir; }
+}

java/kotlin-extractor/src/main/java/com/semmle/util/concurrent/ThreadUtil.java

@@ -0,0 +1,43 @@
+package com.semmle.util.concurrent;
+
+import com.semmle.util.exception.CatastrophicError;
+import com.semmle.util.exception.Exceptions;
+
+
+/**
+ * Utility methods related to Threads.
+ */
+public enum ThreadUtil
+{
+	/** Singleton instance of {@link ThreadUtil}. */
+	SINGLETON;
+
+	/**
+	 * Sleep for {@code millis} milliseconds.
+	 * <p>
+	 * Unlike {@link Thread#sleep(long)} (which is wrapped), this method does not throw an
+	 * {@link InterruptedException}, rather in the event of interruption it either throws an
+	 * {@link CatastrophicError} (if {@code allowInterrupt} is false), or accepts the interruption and
+	 * returns false.
+	 * </p>
+	 *
+	 * @return true if a sleep of {@code millis} milliseconds was performed without interruption, or
+	 *         false if an interruption occurred.
+	 */
+	public static boolean sleep(long millis, boolean allowInterrupt)
+	{
+		try {
+			Thread.sleep(millis);
+		}
+		catch (InterruptedException ie) {
+			if (allowInterrupt) {
+				Exceptions.ignore(ie, "explicitly permitted interruption");
+				return false;
+			}
+			else {
+				throw new CatastrophicError("Interrupted", ie);
+			}
+		}
+		return true;
+	}
+}

java/kotlin-extractor/src/main/java/com/semmle/util/data/IntRef.java

@@ -0,0 +1,19 @@
+package com.semmle.util.data;
+
+/**
+ * A mutable reference to a primitive int. Specialised to avoid
+ * boxing.
+ *
+ */
+public class IntRef {
+	private int value;
+	
+	public IntRef(int value) {
+		this.value = value;
+	}
+	
+	public int get() { return value; }
+	public void set(int value) { this.value = value; }
+	public void inc() { value++; }
+	public void add(int val) { value += val; };
+}

java/kotlin-extractor/src/main/java/com/semmle/util/data/Pair.java

@@ -0,0 +1,62 @@
+package com.semmle.util.data;
+
+
+/**
+ * An (immutable) ordered pair of values.
+ * <p>
+ * Pairs are compared with structural equality: <code>(x,y) = (x', y')</code> iff <code>x=x'</code>
+ * and <code>y=y'</code>.
+ * </p>
+ *
+ * @param <X> the type of the first component of the pair
+ * @param <Y> the type of the second component of the pair
+ */
+public class Pair<X,Y> extends Tuple2<X, Y>
+{
+	private static final long serialVersionUID = -2871892357006076659L;
+	
+	/*
+	 * Constructor and factory
+	 */
+
+
+	/**
+	 * Create a new pair of values
+	 * @param x the first component of the pair
+	 * @param y the second component of the pair
+	 */
+	public Pair(X x, Y y) {
+		super(x, y);
+	}
+
+	/**
+	 * Create a new pair of values. This behaves identically
+	 * to the constructor, but benefits from type inference
+	 * @param x the first component of the pair
+	 * @param y the second component of the pair
+	 */
+	public static <X,Y> Pair<X,Y> make(X x, Y y) {
+		return new Pair<X,Y>(x, y);
+	}
+
+	/*
+	 * Getters
+	 */
+
+	/**
+	 * Get the first component of this pair
+	 * @return the first component of the pair
+	 */
+	public X fst() {
+		return value0();
+	}
+
+	/**
+	 * Get the second component of this pair
+	 * @return the second component of the pair
+	 */
+	public Y snd() {
+		return value1();
+	}
+
+}

java/kotlin-extractor/src/main/java/com/semmle/util/data/StringDigestor.java

@@ -0,0 +1,173 @@
+package com.semmle.util.data;
+
+import java.nio.charset.Charset;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
+import com.semmle.util.exception.CatastrophicError;
+
+/**
+ * Encapsulate the creation of message digests from strings.
+ * 
+ * <p>
+ * This class acts as a (partial) output stream, until the <code>getDigest()</code> method is
+ * called. After this the class can no longer be used, except to repeatedly call
+ * {@link #getDigest()}.
+ * 
+ * <p>
+ * UTF-8 is used internally as the {@link Charset} for this class when converting Strings to bytes.
+ */
+public class StringDigestor {
+	private static final Charset UTF8 = Charset.forName("UTF-8");
+	private static final String NULL_STRING = "<null>";
+	private static final int CHUNK_SIZE = 32;
+
+	private MessageDigest digest;
+	private byte[] digestBytes;
+	private final byte[] buf = new byte[CHUNK_SIZE * 3]; // A Java char becomes at most 3 bytes of UTF-8
+
+	/**
+	 * Create a StringDigestor using SHA-1, ready to accept data
+	 */
+	public StringDigestor() {
+		this("SHA-1");
+	}
+
+	/**
+	 * @param digestAlgorithm the algorithm to use in the internal {@link MessageDigest}.
+	 */
+	public StringDigestor(String digestAlgorithm) {
+		try {
+			digest = MessageDigest.getInstance(digestAlgorithm);
+		} catch (NoSuchAlgorithmException e) {
+			throw new CatastrophicError("StringDigestor failed to find the required digest algorithm: " + digestAlgorithm, e);
+		}
+	}
+
+	public void reset() {
+		if (digestBytes == null) throw new CatastrophicError("API violation: Digestor is not finished.");
+		digest.reset();
+		digestBytes = null;
+	}
+
+	/**
+	 * Write an object into this digestor. This converts the object to a
+	 * string using toString(), writes the length, and then writes the
+	 * string itself.
+	 */
+	public StringDigestor write(Object toAppend) {
+		String str;
+		if (toAppend == null) {
+			str = NULL_STRING;
+		} else {
+			str = toAppend.toString();
+		}
+		writeBinaryInt(str.length());
+		writeNoLength(str);
+		return this;
+	}
+
+	/**
+	 * Write the given string without prefixing it by its length. 
+	 */
+	public StringDigestor writeNoLength(Object toAppend) {
+		String s = toAppend.toString();
+		int len = s.length();
+		int i = 0;
+		while(i + CHUNK_SIZE < len) {
+			i = writeUTF8(s, i, i + CHUNK_SIZE);
+		}
+		writeUTF8(s, i, len);
+		return this;
+	}
+
+	private int writeUTF8(String s, int begin, int end) {
+		if (digestBytes != null) throw new CatastrophicError("API violation: Digestor is finished.");
+		byte[] buf = this.buf;
+		int len = 0;
+		for(int i = begin; i < end; ++i) {
+			int c = s.charAt(i);
+			if (c <= 0x7f) {
+				buf[len++] = (byte)c;
+			} else if (c <= 0x7ff) {
+				buf[len] = (byte)(0xc0 | (c >> 6));
+				buf[len+1] = (byte)(0x80 | (c & 0x3f));
+				len += 2;
+			} else if (c < 0xd800 || c > 0xdfff) {
+				buf[len] = (byte)(0xe0 | (c >> 12));
+				buf[len+1] = (byte)(0x80 | ((c >> 6) & 0x3f));
+				buf[len+2] = (byte)(0x80 | (c & 0x3f));
+				len += 3;
+			} else if (i + 1 < end) {
+				int c2 = s.charAt(i + 1);
+				if (c > 0xdbff || c2 < 0xdc00 || c2 > 0xdfff) {
+					// Invalid UTF-16
+				} else {
+					c = 0x10000 + ((c - 0xd800) << 10) + (c2 - 0xdc00);
+					buf[len] = (byte)(0xf0 | (c >> 18));
+					buf[len+1] = (byte)(0x80 | ((c >> 12) & 0x3f));
+					buf[len+2] = (byte)(0x80 | ((c >> 6) & 0x3f));
+					buf[len+3] = (byte)(0x80 | (c & 0x3f));
+					len += 4;
+					++i;
+				}
+			} else {
+				--end;
+				break;
+			}
+		}
+		digest.update(buf, 0, len);
+		return end;
+	}
+
+	/**
+	 * Write an array of raw bytes to the digestor. This appends the contents
+	 * of the array to the accumulated data used for the digest.
+	 */
+	public StringDigestor writeBytes(byte[] data) {
+		if (digestBytes != null) throw new CatastrophicError("API violation: Digestor is finished.");
+		digest.update(data);
+		return this;
+	}
+
+	/**
+	 * Return the hex-encoded digest as a {@link String}.
+	 * 
+	 * Get the digest from the data previously appended using <code>write(Object)</code>.
+	 * After this is called, the instance's {@link #write(Object)} and {@link #writeBytes(byte[])}
+	 * methods may no longer be used.
+	 */
+	public String getDigest() {
+		if (digestBytes == null) {
+			digestBytes = digest.digest();
+		}
+		return StringUtil.toHex(digestBytes);
+	}
+
+	public static String digest(Object o) {
+		StringDigestor digestor = new StringDigestor();
+		digestor.writeNoLength(o);
+		return digestor.getDigest();
+	}
+
+	/** Compute a git-style SHA for the given string. */
+	public static String gitBlobSha(String content) {
+		byte[] bytes = content.getBytes(UTF8);
+		return digest("blob " + bytes.length + "\0" + content);
+	}
+
+	/**
+	 * Convert an int to a byte[4] using its little-endian 32bit representation, and append the
+	 * resulting bytes to the accumulated data used for the digest.
+	 */
+	public StringDigestor writeBinaryInt(int i) {
+		if (digestBytes != null) throw new CatastrophicError("API violation: Digestor is finished.");
+		byte[] buf = this.buf;
+		buf[0] = (byte)(i & 0xff);
+		buf[1] = (byte)((i >>> 8) & 0xff);
+		buf[2] = (byte)((i >>> 16) & 0xff);
+		buf[3] = (byte)((i >>> 24) & 0xff);
+		digest.update(buf, 0, 4);
+		return this;
+	}
+}