Merge remote-tracking branch 'upstream/main' into csharp/rework-summaries

.github/workflows/docs-review.yml

@@ -1,29 +0,0 @@
-# When a PR is labelled with 'ready-for-docs-review',
-# this workflow comments on the PR to notify the GitHub CodeQL docs team.
-name: Request docs review
-on:
-  # Runs in the context of the base repo.
-  # This gives the workflow write access to comment on PRs.
-  # The workflow should not check out or build the given ref,
-  # or use untrusted data from the event payload in a command line.
-  pull_request_target:
-    types: [labeled]
-
-jobs:
-  request-docs-review:
-    name: Request docs review
-    # Run only on labelled PRs to the main repository.
-    # Do not run on PRs to forks.
-    if:
-      github.event.label.name == 'ready-for-docs-review'
-      && github.event.pull_request.draft == false
-      && github.event.pull_request.base.repo.full_name == 'github/codeql'
-    runs-on: ubuntu-latest
-    steps:
-      - name: Comment to request docs review
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          PR_NUMBER: ${{ github.event.pull_request.number }}
-        run: |
-          gh pr comment "$PR_NUMBER" --repo "github/codeql" \
-            --body "Hello @github/docs-content-codeql - this PR is ready for docs review."

config/identical-files.json

@@ -429,10 +429,11 @@
   "SSA C#": [
     "csharp/ql/src/semmle/code/csharp/dataflow/internal/SsaImplCommon.qll",
     "csharp/ql/src/semmle/code/csharp/controlflow/internal/pressa/SsaImplCommon.qll",
-    "csharp/ql/src/semmle/code/csharp/dataflow/internal/basessa/SsaImplCommon.qll"
+    "csharp/ql/src/semmle/code/csharp/dataflow/internal/basessa/SsaImplCommon.qll",
+    "csharp/ql/src/semmle/code/cil/internal/SsaImplCommon.qll"
   ],
   "CryptoAlgorithms Python/JS": [
     "javascript/ql/src/semmle/javascript/security/CryptoAlgorithms.qll",
     "python/ql/src/semmle/crypto/Crypto.qll"
   ]
-}
+}

cpp/ql/test/query-tests/Security/CWE/CWE-114/semmle/UncontrolledProcessOperation/UncontrolledProcessOperation.expected

@@ -35,6 +35,14 @@ edges
 | test.cpp:76:12:76:17 | fgets output argument | test.cpp:78:10:78:15 | buffer |
 | test.cpp:76:12:76:17 | fgets output argument | test.cpp:79:10:79:13 | (const char *)... |
 | test.cpp:76:12:76:17 | fgets output argument | test.cpp:79:10:79:13 | data |
+| test.cpp:98:17:98:22 | buffer | test.cpp:99:15:99:20 | (const char *)... |
+| test.cpp:98:17:98:22 | buffer | test.cpp:99:15:99:20 | buffer |
+| test.cpp:98:17:98:22 | recv output argument | test.cpp:99:15:99:20 | (const char *)... |
+| test.cpp:98:17:98:22 | recv output argument | test.cpp:99:15:99:20 | buffer |
+| test.cpp:106:17:106:22 | buffer | test.cpp:107:15:107:20 | (const char *)... |
+| test.cpp:106:17:106:22 | buffer | test.cpp:107:15:107:20 | buffer |
+| test.cpp:106:17:106:22 | recv output argument | test.cpp:107:15:107:20 | (const char *)... |
+| test.cpp:106:17:106:22 | recv output argument | test.cpp:107:15:107:20 | buffer |
 nodes
 | test.cpp:24:30:24:36 | *command | semmle.label | *command |
 | test.cpp:24:30:24:36 | command | semmle.label | command |
@@ -70,6 +78,16 @@ nodes
 | test.cpp:79:10:79:13 | (const char *)... | semmle.label | (const char *)... |
 | test.cpp:79:10:79:13 | (const char *)... | semmle.label | (const char *)... |
 | test.cpp:79:10:79:13 | data | semmle.label | data |
+| test.cpp:98:17:98:22 | buffer | semmle.label | buffer |
+| test.cpp:98:17:98:22 | recv output argument | semmle.label | recv output argument |
+| test.cpp:99:15:99:20 | (const char *)... | semmle.label | (const char *)... |
+| test.cpp:99:15:99:20 | (const char *)... | semmle.label | (const char *)... |
+| test.cpp:99:15:99:20 | buffer | semmle.label | buffer |
+| test.cpp:106:17:106:22 | buffer | semmle.label | buffer |
+| test.cpp:106:17:106:22 | recv output argument | semmle.label | recv output argument |
+| test.cpp:107:15:107:20 | (const char *)... | semmle.label | (const char *)... |
+| test.cpp:107:15:107:20 | (const char *)... | semmle.label | (const char *)... |
+| test.cpp:107:15:107:20 | buffer | semmle.label | buffer |
 #select
 | test.cpp:26:10:26:16 | command | test.cpp:42:18:42:23 | call to getenv | test.cpp:26:10:26:16 | command | The value of this argument may come from $@ and is being passed to system | test.cpp:42:18:42:23 | call to getenv | call to getenv |
 | test.cpp:31:10:31:16 | command | test.cpp:43:18:43:23 | call to getenv | test.cpp:31:10:31:16 | command | The value of this argument may come from $@ and is being passed to system | test.cpp:43:18:43:23 | call to getenv | call to getenv |
@@ -77,3 +95,5 @@ nodes
 | test.cpp:63:10:63:13 | data | test.cpp:56:12:56:17 | buffer | test.cpp:63:10:63:13 | data | The value of this argument may come from $@ and is being passed to system | test.cpp:56:12:56:17 | buffer | buffer |
 | test.cpp:78:10:78:15 | buffer | test.cpp:76:12:76:17 | buffer | test.cpp:78:10:78:15 | buffer | The value of this argument may come from $@ and is being passed to system | test.cpp:76:12:76:17 | buffer | buffer |
 | test.cpp:79:10:79:13 | data | test.cpp:76:12:76:17 | buffer | test.cpp:79:10:79:13 | data | The value of this argument may come from $@ and is being passed to system | test.cpp:76:12:76:17 | buffer | buffer |
+| test.cpp:99:15:99:20 | buffer | test.cpp:98:17:98:22 | buffer | test.cpp:99:15:99:20 | buffer | The value of this argument may come from $@ and is being passed to LoadLibrary | test.cpp:98:17:98:22 | buffer | buffer |
+| test.cpp:107:15:107:20 | buffer | test.cpp:106:17:106:22 | buffer | test.cpp:107:15:107:20 | buffer | The value of this argument may come from $@ and is being passed to LoadLibrary | test.cpp:106:17:106:22 | buffer | buffer |

cpp/ql/test/query-tests/Security/CWE/CWE-114/semmle/UncontrolledProcessOperation/test.cpp

@@ -81,3 +81,29 @@ void testReferencePointer2()
 		system(data2); // BAD [NOT DETECTED]
 	}
 }
+
+// ---
+
+typedef unsigned long size_t;
+
+void accept(int arg, char *buf, size_t *bufSize);
+void recv(int arg, char *buf, size_t bufSize);
+void LoadLibrary(const char *arg);
+
+void testAcceptRecv(int socket1, int socket2)
+{
+	{
+		char buffer[1024];
+
+		recv(socket1, buffer, 1024);
+		LoadLibrary(buffer); // BAD: using data from recv
+	}
+	
+	{
+		char buffer[1024];
+
+		accept(socket2, 0, 0);
+		recv(socket2, buffer, 1024);
+		LoadLibrary(buffer); // BAD: using data from recv
+	}
+}

csharp/change-notes/2021-03-24-cil-ssa.md

@@ -0,0 +1,2 @@
+lgtm,codescanning
+* A static single assignment (SSA) library has been added to the CIL analysis library. The SSA library replaces the existing `DefUse` module, which has been deprecated.

csharp/ql/src/semmle/code/cil/BasicBlock.qll

@@ -240,6 +240,9 @@ class BasicBlock extends Cached::TBasicBlockStart {
 
   /** Gets a textual representation of this basic block. */
   string toString() { result = getFirstNode().toString() }
+
+  /** Gets the location of this basic block. */
+  Location getLocation() { result = this.getFirstNode().getLocation() }
 }
 
 /**
@@ -305,7 +308,7 @@ class EntryBasicBlock extends BasicBlock {
 }
 
 /** Holds if `bb` is an entry basic block. */
-private predicate entryBB(BasicBlock bb) { bb.getFirstNode() instanceof EntryPoint }
+private predicate entryBB(BasicBlock bb) { bb.getFirstNode() instanceof MethodImplementation }
 
 /**
  * An exit basic block, that is, a basic block whose last node is

csharp/ql/src/semmle/code/cil/CIL.qll

@@ -20,3 +20,4 @@ import Attribute
 import Stubs
 import CustomModifierReceiver
 import Parameterizable
+import semmle.code.cil.Ssa

csharp/ql/src/semmle/code/cil/CallableReturns.qll

@@ -42,7 +42,11 @@ private predicate alwaysNullExpr(Expr expr) {
   or
   alwaysNullMethod(expr.(StaticCall).getTarget())
   or
-  forex(VariableUpdate vu | DefUse::variableUpdateUse(_, vu, expr) | alwaysNullVariableUpdate(vu))
+  forex(Ssa::Definition def |
+    expr = any(Ssa::Definition def0 | def = def0.getAnUltimateDefinition()).getARead()
+  |
+    alwaysNullVariableUpdate(def.getVariableUpdate())
+  )
 }
 
 pragma[noinline]
@@ -58,7 +62,9 @@ private predicate alwaysNotNullExpr(Expr expr) {
   or
   alwaysNotNullMethod(expr.(StaticCall).getTarget())
   or
-  forex(VariableUpdate vu | DefUse::variableUpdateUse(_, vu, expr) |
-    alwaysNotNullVariableUpdate(vu)
+  forex(Ssa::Definition def |
+    expr = any(Ssa::Definition def0 | def = def0.getAnUltimateDefinition()).getARead()
+  |
+    alwaysNotNullVariableUpdate(def.getVariableUpdate())
   )
 }

csharp/ql/src/semmle/code/cil/ControlFlow.qll

@@ -9,6 +9,9 @@ class ControlFlowNode extends @cil_controlflow_node {
   /** Gets a textual representation of this control flow node. */
   string toString() { none() }
 
+  /** Gets the location of this control flow node. */
+  Location getLocation() { none() }
+
   /**
    * Gets the number of items this node pushes onto the stack.
    * This value is either 0 or 1, except for the instruction `dup`

csharp/ql/src/semmle/code/cil/DataFlow.qll

@@ -57,12 +57,40 @@ class Untainted extends TaintType, TExactValue { }
 /** A taint type where the data is tainted. */
 class Tainted extends TaintType, TTaintedValue { }
 
+private predicate localFlowPhiInput(DataFlowNode input, Ssa::PhiNode phi) {
+  exists(Ssa::Definition def, BasicBlock bb, int i | phi.hasLastInputRef(def, bb, i) |
+    def.definesAt(_, bb, i) and
+    input = def.getVariableUpdate().getSource()
+    or
+    input =
+      any(ReadAccess ra |
+        bb.getNode(i) = ra and
+        ra.getTarget() = def.getSourceVariable()
+      )
+  )
+  or
+  exists(Ssa::PhiNode mid, BasicBlock bb, int i |
+    localFlowPhiInput(input, mid) and
+    phi.hasLastInputRef(mid, bb, i) and
+    mid.definesAt(_, bb, i)
+  )
+}
+
 private predicate localExactStep(DataFlowNode src, DataFlowNode sink) {
   src = sink.(Opcodes::Dup).getAnOperand()
   or
-  defUse(_, src, sink)
+  exists(Ssa::Definition def, VariableUpdate vu |
+    vu = def.getVariableUpdate() and
+    src = vu.getSource() and
+    sink = def.getAFirstRead()
+  )
   or
-  src = sink.(ParameterReadAccess).getTarget()
+  any(Ssa::Definition def).hasAdjacentReads(src, sink)
+  or
+  exists(Ssa::PhiNode phi |
+    localFlowPhiInput(src, phi) and
+    sink = phi.getAFirstRead()
+  )
   or
   src = sink.(Conversion).getExpr()
   or
@@ -73,12 +101,6 @@ private predicate localExactStep(DataFlowNode src, DataFlowNode sink) {
   src = sink.(Return).getExpr()
   or
   src = sink.(ConditionalBranch).getAnOperand()
-  or
-  src = sink.(MethodParameter).getAWrite()
-  or
-  exists(VariableUpdate update |
-    update.getVariable().(Parameter) = sink and src = update.getSource()
-  )
 }
 
 private predicate localTaintStep(DataFlowNode src, DataFlowNode sink) {
@@ -87,8 +109,7 @@ private predicate localTaintStep(DataFlowNode src, DataFlowNode sink) {
   src = sink.(UnaryBitwiseOperation).getOperand()
 }
 
-cached
-module DefUse {
+deprecated module DefUse {
   /**
    * A classification of variable references into reads and writes.
    */
@@ -189,7 +210,7 @@ module DefUse {
 
   /** Holds if the variable update `vu` can be used at the read `use`. */
   cached
-  predicate variableUpdateUse(StackVariable target, VariableUpdate vu, ReadAccess use) {
+  deprecated predicate variableUpdateUse(StackVariable target, VariableUpdate vu, ReadAccess use) {
     defReachesReadWithinBlock(target, vu, use)
     or
     exists(BasicBlock bb, int i |
@@ -202,23 +223,40 @@ module DefUse {
 
   /** Holds if the update `def` can be used at the read `use`. */
   cached
-  predicate defUse(StackVariable target, Expr def, ReadAccess use) {
+  deprecated predicate defUse(StackVariable target, Expr def, ReadAccess use) {
     exists(VariableUpdate vu | def = vu.getSource() | variableUpdateUse(target, vu, use))
   }
 }
 
-private import DefUse
-
-abstract library class VariableUpdate extends Instruction {
-  abstract Expr getSource();
+/** A node that updates a variable. */
+abstract class VariableUpdate extends DataFlowNode {
+  /** Gets the value assigned, if any. */
+  abstract DataFlowNode getSource();
 
+  /** Gets the variable that is updated. */
   abstract Variable getVariable();
+
+  /** Holds if this variable update happens at index `i` in basic block `bb`. */
+  abstract predicate updatesAt(BasicBlock bb, int i);
+}
+
+private class MethodParameterDef extends VariableUpdate, MethodParameter {
+  override MethodParameter getSource() { result = this }
+
+  override MethodParameter getVariable() { result = this }
+
+  override predicate updatesAt(BasicBlock bb, int i) {
+    bb.(EntryBasicBlock).getANode().getImplementation().getMethod() = this.getMethod() and
+    i = -1
+  }
 }
 
 private class VariableWrite extends VariableUpdate, WriteAccess {
-  override Expr getSource() { result = getExpr() }
+  override Expr getSource() { result = this.getExpr() }
 
-  override Variable getVariable() { result = getTarget() }
+  override Variable getVariable() { result = this.getTarget() }
+
+  override predicate updatesAt(BasicBlock bb, int i) { this = bb.getNode(i) }
 }
 
 private class MethodOutOrRefTarget extends VariableUpdate, Call {
@@ -230,5 +268,7 @@ private class MethodOutOrRefTarget extends VariableUpdate, Call {
     result = this.getRawArgument(parameterIndex).(ReadAccess).getTarget()
   }
 
-  override Expr getSource() { result = this }
+  override Expr getSource() { none() }
+
+  override predicate updatesAt(BasicBlock bb, int i) { this = bb.getNode(i) }
 }

csharp/ql/src/semmle/code/cil/Method.qll

@@ -19,7 +19,7 @@ class MethodImplementation extends EntryPoint, @cil_method_implementation {
   override MethodImplementation getImplementation() { result = this }
 
   /** Gets the location of this implementation. */
-  Assembly getLocation() { cil_method_implementation(this, _, result) }
+  override Assembly getLocation() { cil_method_implementation(this, _, result) }
 
   /** Gets the instruction at index `index`. */
   Instruction getInstruction(int index) { cil_instruction(result, _, index, this) }

csharp/ql/src/semmle/code/cil/Ssa.qll

@@ -0,0 +1,66 @@
+/**
+ * Provides the module `Ssa` for working with static single assignment (SSA) form.
+ */
+
+private import CIL
+
+/**
+ * Provides classes for working with static single assignment (SSA) form.
+ */
+module Ssa {
+  private import internal.SsaImplCommon as SsaImpl
+  private import internal.SsaImpl
+
+  /** An SSA definition. */
+  class Definition extends SsaImpl::Definition {
+    /** Gets a read of this SSA definition. */
+    final ReadAccess getARead() { result = getARead(this) }
+
+    /** Gets the underlying variable update, if any. */
+    final VariableUpdate getVariableUpdate() {
+      exists(BasicBlock bb, int i |
+        result.updatesAt(bb, i) and
+        this.definesAt(result.getVariable(), bb, i)
+      )
+    }
+
+    /** Gets a first read of this SSA definition. */
+    final ReadAccess getAFirstRead() { result = getAFirstRead(this) }
+
+    /** Holds if `first` and `second` are adjacent reads of this SSA definition. */
+    final predicate hasAdjacentReads(ReadAccess first, ReadAccess second) {
+      hasAdjacentReads(this, first, second)
+    }
+
+    private Definition getAPhiInput() { result = this.(PhiNode).getAnInput() }
+
+    /**
+     * Gets a definition that ultimately defines this SSA definition and is
+     * not itself a phi node.
+     */
+    final Definition getAnUltimateDefinition() {
+      result = this.getAPhiInput*() and
+      not result instanceof PhiNode
+    }
+
+    /** Gets the location of this SSA definition. */
+    Location getLocation() { result = this.getVariableUpdate().getLocation() }
+  }
+
+  /** A phi node. */
+  class PhiNode extends SsaImpl::PhiNode, Definition {
+    final override Location getLocation() { result = this.getBasicBlock().getLocation() }
+
+    /** Gets an input to this phi node. */
+    final Definition getAnInput() { result = getAPhiInput(this) }
+
+    /**
+     * Holds if if `def` is an input to this phi node, and a reference to `def` at
+     * index `i` in basic block `bb` can reach this phi node without going through
+     * other references.
+     */
+    final predicate hasLastInputRef(Definition def, BasicBlock bb, int i) {
+      hasLastInputRef(this, def, bb, i)
+    }
+  }
+}

csharp/ql/src/semmle/code/cil/internal/SsaImpl.qll

@@ -0,0 +1,45 @@
+private import semmle.code.cil.CIL
+private import SsaImplCommon
+
+cached
+private module Cached {
+  cached
+  predicate forceCachingInSameStage() { any() }
+
+  cached
+  ReadAccess getARead(Definition def) {
+    exists(BasicBlock bb, int i |
+      ssaDefReachesRead(_, def, bb, i) and
+      result = bb.getNode(i)
+    )
+  }
+
+  cached
+  ReadAccess getAFirstRead(Definition def) {
+    exists(BasicBlock bb1, int i1, BasicBlock bb2, int i2 |
+      def.definesAt(_, bb1, i1) and
+      adjacentDefRead(def, bb1, i1, bb2, i2) and
+      result = bb2.getNode(i2)
+    )
+  }
+
+  cached
+  predicate hasAdjacentReads(Definition def, ReadAccess first, ReadAccess second) {
+    exists(BasicBlock bb1, int i1, BasicBlock bb2, int i2 |
+      first = bb1.getNode(i1) and
+      adjacentDefRead(def, bb1, i1, bb2, i2) and
+      second = bb2.getNode(i2)
+    )
+  }
+
+  cached
+  Definition getAPhiInput(PhiNode phi) { phiHasInputFromBlock(phi, result, _) }
+
+  cached
+  predicate hasLastInputRef(Definition phi, Definition def, BasicBlock bb, int i) {
+    lastRefRedef(def, bb, i, phi) and
+    def = getAPhiInput(phi)
+  }
+}
+
+import Cached

csharp/ql/src/semmle/code/cil/internal/SsaImplCommon.qll

@@ -0,0 +1,619 @@
+/**
+ * Provides a language-independant implementation of static single assignment
+ * (SSA) form.
+ */
+
+private import SsaImplSpecific
+
+private BasicBlock getABasicBlockPredecessor(BasicBlock bb) { getABasicBlockSuccessor(result) = bb }
+
+/**
+ * Liveness analysis (based on source variables) to restrict the size of the
+ * SSA representation.
+ */
+private module Liveness {
+  /**
+   * A classification of variable references into reads (of a given kind) and
+   * (certain or uncertain) writes.
+   */
+  private newtype TRefKind =
+    Read(boolean certain) { certain in [false, true] } or
+    Write(boolean certain) { certain in [false, true] }
+
+  private class RefKind extends TRefKind {
+    string toString() {
+      exists(boolean certain | this = Read(certain) and result = "read (" + certain + ")")
+      or
+      exists(boolean certain | this = Write(certain) and result = "write (" + certain + ")")
+    }
+
+    int getOrder() {
+      this = Read(_) and
+      result = 0
+      or
+      this = Write(_) and
+      result = 1
+    }
+  }
+
+  /**
+   * Holds if the `i`th node of basic block `bb` is a reference to `v` of kind `k`.
+   */
+  private predicate ref(BasicBlock bb, int i, SourceVariable v, RefKind k) {
+    exists(boolean certain | variableRead(bb, i, v, certain) | k = Read(certain))
+    or
+    exists(boolean certain | variableWrite(bb, i, v, certain) | k = Write(certain))
+  }
+
+  private newtype OrderedRefIndex =
+    MkOrderedRefIndex(int i, int tag) {
+      exists(RefKind rk | ref(_, i, _, rk) | tag = rk.getOrder())
+    }
+
+  private OrderedRefIndex refOrd(BasicBlock bb, int i, SourceVariable v, RefKind k, int ord) {
+    ref(bb, i, v, k) and
+    result = MkOrderedRefIndex(i, ord) and
+    ord = k.getOrder()
+  }
+
+  /**
+   * Gets the (1-based) rank of the reference to `v` at the `i`th node of
+   * basic block `bb`, which has the given reference kind `k`.
+   *
+   * Reads are considered before writes when they happen at the same index.
+   */
+  private int refRank(BasicBlock bb, int i, SourceVariable v, RefKind k) {
+    refOrd(bb, i, v, k, _) =
+      rank[result](int j, int ord, OrderedRefIndex res |
+        res = refOrd(bb, j, v, _, ord)
+      |
+        res order by j, ord
+      )
+  }
+
+  private int maxRefRank(BasicBlock bb, SourceVariable v) {
+    result = refRank(bb, _, v, _) and
+    not result + 1 = refRank(bb, _, v, _)
+  }
+
+  /**
+   * Gets the (1-based) rank of the first reference to `v` inside basic block `bb`
+   * that is either a read or a certain write.
+   */
+  private int firstReadOrCertainWrite(BasicBlock bb, SourceVariable v) {
+    result =
+      min(int r, RefKind k |
+        r = refRank(bb, _, v, k) and
+        k != Write(false)
+      |
+        r
+      )
+  }
+
+  /**
+   * Holds if source variable `v` is live at the beginning of basic block `bb`.
+   */
+  predicate liveAtEntry(BasicBlock bb, SourceVariable v) {
+    // The first read or certain write to `v` inside `bb` is a read
+    refRank(bb, _, v, Read(_)) = firstReadOrCertainWrite(bb, v)
+    or
+    // There is no certain write to `v` inside `bb`, but `v` is live at entry
+    // to a successor basic block of `bb`
+    not exists(firstReadOrCertainWrite(bb, v)) and
+    liveAtExit(bb, v)
+  }
+
+  /**
+   * Holds if source variable `v` is live at the end of basic block `bb`.
+   */
+  predicate liveAtExit(BasicBlock bb, SourceVariable v) {
+    liveAtEntry(getABasicBlockSuccessor(bb), v)
+  }
+
+  /**
+   * Holds if variable `v` is live in basic block `bb` at index `i`.
+   * The rank of `i` is `rnk` as defined by `refRank()`.
+   */
+  private predicate liveAtRank(BasicBlock bb, int i, SourceVariable v, int rnk) {
+    exists(RefKind kind | rnk = refRank(bb, i, v, kind) |
+      rnk = maxRefRank(bb, v) and
+      liveAtExit(bb, v)
+      or
+      ref(bb, i, v, kind) and
+      kind = Read(_)
+      or
+      exists(RefKind nextKind |
+        liveAtRank(bb, _, v, rnk + 1) and
+        rnk + 1 = refRank(bb, _, v, nextKind) and
+        nextKind != Write(true)
+      )
+    )
+  }
+
+  /**
+   * Holds if variable `v` is live after the (certain or uncertain) write at
+   * index `i` inside basic block `bb`.
+   */
+  predicate liveAfterWrite(BasicBlock bb, int i, SourceVariable v) {
+    exists(int rnk | rnk = refRank(bb, i, v, Write(_)) | liveAtRank(bb, i, v, rnk))
+  }
+}
+
+private import Liveness
+
+/** Holds if `bb1` strictly dominates `bb2`. */
+private predicate strictlyDominates(BasicBlock bb1, BasicBlock bb2) {
+  bb1 = getImmediateBasicBlockDominator+(bb2)
+}
+
+/** Holds if `bb1` dominates a predecessor of `bb2`. */
+private predicate dominatesPredecessor(BasicBlock bb1, BasicBlock bb2) {
+  exists(BasicBlock pred | pred = getABasicBlockPredecessor(bb2) |
+    bb1 = pred
+    or
+    strictlyDominates(bb1, pred)
+  )
+}
+
+/** Holds if `df` is in the dominance frontier of `bb`. */
+private predicate inDominanceFrontier(BasicBlock bb, BasicBlock df) {
+  dominatesPredecessor(bb, df) and
+  not strictlyDominates(bb, df)
+}
+
+/**
+ * Holds if `bb` is in the dominance frontier of a block containing a
+ * definition of `v`.
+ */
+pragma[noinline]
+private predicate inDefDominanceFrontier(BasicBlock bb, SourceVariable v) {
+  exists(BasicBlock defbb, Definition def |
+    def.definesAt(v, defbb, _) and
+    inDominanceFrontier(defbb, bb)
+  )
+}
+
+cached
+newtype TDefinition =
+  TWriteDef(SourceVariable v, BasicBlock bb, int i) {
+    variableWrite(bb, i, v, _) and
+    liveAfterWrite(bb, i, v)
+  } or
+  TPhiNode(SourceVariable v, BasicBlock bb) {
+    inDefDominanceFrontier(bb, v) and
+    liveAtEntry(bb, v)
+  }
+
+private module SsaDefReaches {
+  newtype TSsaRefKind =
+    SsaRead() or
+    SsaDef()
+
+  /**
+   * A classification of SSA variable references into reads and definitions.
+   */
+  class SsaRefKind extends TSsaRefKind {
+    string toString() {
+      this = SsaRead() and
+      result = "SsaRead"
+      or
+      this = SsaDef() and
+      result = "SsaDef"
+    }
+
+    int getOrder() {
+      this = SsaRead() and
+      result = 0
+      or
+      this = SsaDef() and
+      result = 1
+    }
+  }
+
+  /**
+   * Holds if the `i`th node of basic block `bb` is a reference to `v`,
+   * either a read (when `k` is `SsaRead()`) or an SSA definition (when `k`
+   * is `SsaDef()`).
+   *
+   * Unlike `Liveness::ref`, this includes `phi` nodes.
+   */
+  predicate ssaRef(BasicBlock bb, int i, SourceVariable v, SsaRefKind k) {
+    variableRead(bb, i, v, _) and
+    k = SsaRead()
+    or
+    exists(Definition def | def.definesAt(v, bb, i)) and
+    k = SsaDef()
+  }
+
+  private newtype OrderedSsaRefIndex =
+    MkOrderedSsaRefIndex(int i, SsaRefKind k) { ssaRef(_, i, _, k) }
+
+  private OrderedSsaRefIndex ssaRefOrd(BasicBlock bb, int i, SourceVariable v, SsaRefKind k, int ord) {
+    ssaRef(bb, i, v, k) and
+    result = MkOrderedSsaRefIndex(i, k) and
+    ord = k.getOrder()
+  }
+
+  /**
+   * Gets the (1-based) rank of the reference to `v` at the `i`th node of basic
+   * block `bb`, which has the given reference kind `k`.
+   *
+   * For example, if `bb` is a basic block with a phi node for `v` (considered
+   * to be at index -1), reads `v` at node 2, and defines it at node 5, we have:
+   *
+   * ```ql
+   * ssaRefRank(bb, -1, v, SsaDef()) = 1    // phi node
+   * ssaRefRank(bb,  2, v, Read())   = 2    // read at node 2
+   * ssaRefRank(bb,  5, v, SsaDef()) = 3    // definition at node 5
+   * ```
+   *
+   * Reads are considered before writes when they happen at the same index.
+   */
+  int ssaRefRank(BasicBlock bb, int i, SourceVariable v, SsaRefKind k) {
+    ssaRefOrd(bb, i, v, k, _) =
+      rank[result](int j, int ord, OrderedSsaRefIndex res |
+        res = ssaRefOrd(bb, j, v, _, ord)
+      |
+        res order by j, ord
+      )
+  }
+
+  int maxSsaRefRank(BasicBlock bb, SourceVariable v) {
+    result = ssaRefRank(bb, _, v, _) and
+    not result + 1 = ssaRefRank(bb, _, v, _)
+  }
+
+  /**
+   * Holds if the SSA definition `def` reaches rank index `rnk` in its own
+   * basic block `bb`.
+   */
+  predicate ssaDefReachesRank(BasicBlock bb, Definition def, int rnk, SourceVariable v) {
+    exists(int i |
+      rnk = ssaRefRank(bb, i, v, SsaDef()) and
+      def.definesAt(v, bb, i)
+    )
+    or
+    ssaDefReachesRank(bb, def, rnk - 1, v) and
+    rnk = ssaRefRank(bb, _, v, SsaRead())
+  }
+
+  /**
+   * Holds if the SSA definition of `v` at `def` reaches index `i` in the same
+   * basic block `bb`, without crossing another SSA definition of `v`.
+   */
+  predicate ssaDefReachesReadWithinBlock(SourceVariable v, Definition def, BasicBlock bb, int i) {
+    exists(int rnk |
+      ssaDefReachesRank(bb, def, rnk, v) and
+      rnk = ssaRefRank(bb, i, v, SsaRead()) and
+      variableRead(bb, i, v, _)
+    )
+  }
+
+  /**
+   * Holds if the SSA definition of `v` at `def` reaches uncertain SSA definition
+   * `redef` in the same basic block, without crossing another SSA definition of `v`.
+   */
+  predicate ssaDefReachesUncertainDefWithinBlock(
+    SourceVariable v, Definition def, UncertainWriteDefinition redef
+  ) {
+    exists(BasicBlock bb, int rnk, int i |
+      ssaDefReachesRank(bb, def, rnk, v) and
+      rnk = ssaRefRank(bb, i, v, SsaDef()) - 1 and
+      redef.definesAt(v, bb, i)
+    )
+  }
+
+  /**
+   * Same as `ssaRefRank()`, but restricted to a particular SSA definition `def`.
+   */
+  int ssaDefRank(Definition def, SourceVariable v, BasicBlock bb, int i, SsaRefKind k) {
+    v = def.getSourceVariable() and
+    result = ssaRefRank(bb, i, v, k) and
+    (
+      ssaDefReachesRead(_, def, bb, i)
+      or
+      def.definesAt(_, bb, i)
+    )
+  }
+
+  predicate defOccursInBlock(Definition def, BasicBlock bb, SourceVariable v) {
+    exists(ssaDefRank(def, v, bb, _, _))
+  }
+
+  pragma[noinline]
+  private BasicBlock getAMaybeLiveSuccessor(Definition def, BasicBlock bb) {
+    result = getABasicBlockSuccessor(bb) and
+    not defOccursInBlock(_, bb, def.getSourceVariable()) and
+    ssaDefReachesEndOfBlock(bb, def, _)
+  }
+
+  /**
+   * Holds if `def` is accessed in basic block `bb1` (either a read or a write),
+   * `bb2` is a transitive successor of `bb1`, `def` is live at the end of `bb1`,
+   * and the underlying variable for `def` is neither read nor written in any block
+   * on the path between `bb1` and `bb2`.
+   */
+  predicate varBlockReaches(Definition def, BasicBlock bb1, BasicBlock bb2) {
+    defOccursInBlock(def, bb1, _) and
+    bb2 = getABasicBlockSuccessor(bb1)
+    or
+    exists(BasicBlock mid | varBlockReaches(def, bb1, mid) | bb2 = getAMaybeLiveSuccessor(def, mid))
+  }
+
+  /**
+   * Holds if `def` is accessed in basic block `bb1` (either a read or a write),
+   * `def` is read at index `i2` in basic block `bb2`, `bb2` is in a transitive
+   * successor block of `bb1`, and `def` is neither read nor written in any block
+   * on a path between `bb1` and `bb2`.
+   */
+  predicate defAdjacentRead(Definition def, BasicBlock bb1, BasicBlock bb2, int i2) {
+    varBlockReaches(def, bb1, bb2) and
+    ssaRefRank(bb2, i2, def.getSourceVariable(), SsaRead()) = 1 and
+    variableRead(bb2, i2, _, _)
+  }
+}
+
+private import SsaDefReaches
+
+pragma[noinline]
+private predicate ssaDefReachesEndOfBlockRec(BasicBlock bb, Definition def, SourceVariable v) {
+  exists(BasicBlock idom | ssaDefReachesEndOfBlock(idom, def, v) |
+    // The construction of SSA form ensures that each read of a variable is
+    // dominated by its definition. An SSA definition therefore reaches a
+    // control flow node if it is the _closest_ SSA definition that dominates
+    // the node. If two definitions dominate a node then one must dominate the
+    // other, so therefore the definition of _closest_ is given by the dominator
+    // tree. Thus, reaching definitions can be calculated in terms of dominance.
+    idom = getImmediateBasicBlockDominator(bb)
+  )
+}
+
+/**
+ * NB: If this predicate is exposed, it should be cached.
+ *
+ * Holds if the SSA definition of `v` at `def` reaches the end of basic
+ * block `bb`, at which point it is still live, without crossing another
+ * SSA definition of `v`.
+ */
+pragma[nomagic]
+predicate ssaDefReachesEndOfBlock(BasicBlock bb, Definition def, SourceVariable v) {
+  exists(int last | last = maxSsaRefRank(bb, v) |
+    ssaDefReachesRank(bb, def, last, v) and
+    liveAtExit(bb, v)
+  )
+  or
+  ssaDefReachesEndOfBlockRec(bb, def, v) and
+  liveAtExit(bb, v) and
+  not ssaRef(bb, _, v, SsaDef())
+}
+
+/**
+ * NB: If this predicate is exposed, it should be cached.
+ *
+ * Holds if `inp` is an input to the phi node `phi` along the edge originating in `bb`.
+ */
+pragma[nomagic]
+predicate phiHasInputFromBlock(PhiNode phi, Definition inp, BasicBlock bb) {
+  exists(SourceVariable v, BasicBlock bbDef |
+    phi.definesAt(v, bbDef, _) and
+    getABasicBlockPredecessor(bbDef) = bb and
+    ssaDefReachesEndOfBlock(bb, inp, v)
+  )
+}
+
+/**
+ * NB: If this predicate is exposed, it should be cached.
+ *
+ * Holds if the SSA definition of `v` at `def` reaches a read at index `i` in
+ * basic block `bb`, without crossing another SSA definition of `v`. The read
+ * is of kind `rk`.
+ */
+pragma[nomagic]
+predicate ssaDefReachesRead(SourceVariable v, Definition def, BasicBlock bb, int i) {
+  ssaDefReachesReadWithinBlock(v, def, bb, i)
+  or
+  variableRead(bb, i, v, _) and
+  ssaDefReachesEndOfBlock(getABasicBlockPredecessor(bb), def, v) and
+  not ssaDefReachesReadWithinBlock(v, _, bb, i)
+}
+
+/**
+ * NB: If this predicate is exposed, it should be cached.
+ *
+ * Holds if `def` is accessed at index `i1` in basic block `bb1` (either a read
+ * or a write), `def` is read at index `i2` in basic block `bb2`, and there is a
+ * path between them without any read of `def`.
+ */
+pragma[nomagic]
+predicate adjacentDefRead(Definition def, BasicBlock bb1, int i1, BasicBlock bb2, int i2) {
+  exists(int rnk |
+    rnk = ssaDefRank(def, _, bb1, i1, _) and
+    rnk + 1 = ssaDefRank(def, _, bb1, i2, SsaRead()) and
+    variableRead(bb1, i2, _, _) and
+    bb2 = bb1
+  )
+  or
+  exists(SourceVariable v | ssaDefRank(def, v, bb1, i1, _) = maxSsaRefRank(bb1, v)) and
+  defAdjacentRead(def, bb1, bb2, i2)
+}
+
+private predicate adjacentDefReachesRead(
+  Definition def, BasicBlock bb1, int i1, BasicBlock bb2, int i2
+) {
+  adjacentDefRead(def, bb1, i1, bb2, i2) and
+  exists(SourceVariable v | v = def.getSourceVariable() |
+    ssaRef(bb1, i1, v, SsaDef())
+    or
+    variableRead(bb1, i1, v, true)
+  )
+  or
+  exists(BasicBlock bb3, int i3 |
+    adjacentDefReachesRead(def, bb1, i1, bb3, i3) and
+    variableRead(bb3, i3, _, false) and
+    adjacentDefRead(def, bb3, i3, bb2, i2)
+  )
+}
+
+/**
+ * NB: If this predicate is exposed, it should be cached.
+ *
+ * Same as `adjacentDefRead`, but ignores uncertain reads.
+ */
+pragma[nomagic]
+predicate adjacentDefNoUncertainReads(Definition def, BasicBlock bb1, int i1, BasicBlock bb2, int i2) {
+  adjacentDefReachesRead(def, bb1, i1, bb2, i2) and
+  variableRead(bb2, i2, _, true)
+}
+
+/**
+ * NB: If this predicate is exposed, it should be cached.
+ *
+ * Holds if the node at index `i` in `bb` is a last reference to SSA definition
+ * `def`. The reference is last because it can reach another write `next`,
+ * without passing through another read or write.
+ */
+pragma[nomagic]
+predicate lastRefRedef(Definition def, BasicBlock bb, int i, Definition next) {
+  exists(int rnk, SourceVariable v, int j | rnk = ssaDefRank(def, v, bb, i, _) |
+    // Next reference to `v` inside `bb` is a write
+    next.definesAt(v, bb, j) and
+    rnk + 1 = ssaRefRank(bb, j, v, SsaDef())
+    or
+    // Can reach a write using one or more steps
+    rnk = maxSsaRefRank(bb, v) and
+    exists(BasicBlock bb2 |
+      varBlockReaches(def, bb, bb2) and
+      next.definesAt(v, bb2, j) and
+      1 = ssaRefRank(bb2, j, v, SsaDef())
+    )
+  )
+}
+
+/**
+ * NB: If this predicate is exposed, it should be cached.
+ *
+ * Holds if `inp` is an immediately preceding definition of uncertain definition
+ * `def`. Since `def` is uncertain, the value from the preceding definition might
+ * still be valid.
+ */
+pragma[nomagic]
+predicate uncertainWriteDefinitionInput(UncertainWriteDefinition def, Definition inp) {
+  lastRefRedef(inp, _, _, def)
+}
+
+private predicate adjacentDefReachesUncertainRead(
+  Definition def, BasicBlock bb1, int i1, BasicBlock bb2, int i2
+) {
+  adjacentDefReachesRead(def, bb1, i1, bb2, i2) and
+  variableRead(bb2, i2, _, false)
+}
+
+/**
+ * NB: If this predicate is exposed, it should be cached.
+ *
+ * Same as `lastRefRedef`, but ignores uncertain reads.
+ */
+pragma[nomagic]
+predicate lastRefRedefNoUncertainReads(Definition def, BasicBlock bb, int i, Definition next) {
+  lastRefRedef(def, bb, i, next) and
+  not variableRead(bb, i, def.getSourceVariable(), false)
+  or
+  exists(BasicBlock bb0, int i0 |
+    lastRefRedef(def, bb0, i0, next) and
+    adjacentDefReachesUncertainRead(def, bb, i, bb0, i0)
+  )
+}
+
+/**
+ * NB: If this predicate is exposed, it should be cached.
+ *
+ * Holds if the node at index `i` in `bb` is a last reference to SSA
+ * definition `def`.
+ *
+ * That is, the node can reach the end of the enclosing callable, or another
+ * SSA definition for the underlying source variable, without passing through
+ * another read.
+ */
+pragma[nomagic]
+predicate lastRef(Definition def, BasicBlock bb, int i) {
+  lastRefRedef(def, bb, i, _)
+  or
+  exists(SourceVariable v | ssaDefRank(def, v, bb, i, _) = maxSsaRefRank(bb, v) |
+    // Can reach exit directly
+    bb instanceof ExitBasicBlock
+    or
+    // Can reach a block using one or more steps, where `def` is no longer live
+    exists(BasicBlock bb2 | varBlockReaches(def, bb, bb2) |
+      not defOccursInBlock(def, bb2, _) and
+      not ssaDefReachesEndOfBlock(bb2, def, _)
+    )
+  )
+}
+
+/**
+ * NB: If this predicate is exposed, it should be cached.
+ *
+ * Same as `lastRefRedef`, but ignores uncertain reads.
+ */
+pragma[nomagic]
+predicate lastRefNoUncertainReads(Definition def, BasicBlock bb, int i) {
+  lastRef(def, bb, i) and
+  not variableRead(bb, i, def.getSourceVariable(), false)
+  or
+  exists(BasicBlock bb0, int i0 |
+    lastRef(def, bb0, i0) and
+    adjacentDefReachesUncertainRead(def, bb, i, bb0, i0)
+  )
+}
+
+/** A static single assignment (SSA) definition. */
+class Definition extends TDefinition {
+  /** Gets the source variable underlying this SSA definition. */
+  SourceVariable getSourceVariable() { this.definesAt(result, _, _) }
+
+  /**
+   * Holds if this SSA definition defines `v` at index `i` in basic block `bb`.
+   * Phi nodes are considered to be at index `-1`, while normal variable writes
+   * are at the index of the control flow node they wrap.
+   */
+  final predicate definesAt(SourceVariable v, BasicBlock bb, int i) {
+    this = TWriteDef(v, bb, i)
+    or
+    this = TPhiNode(v, bb) and i = -1
+  }
+
+  /** Gets the basic block to which this SSA definition belongs. */
+  final BasicBlock getBasicBlock() { this.definesAt(_, result, _) }
+
+  /** Gets a textual representation of this SSA definition. */
+  string toString() { none() }
+}
+
+/** An SSA definition that corresponds to a write. */
+class WriteDefinition extends Definition, TWriteDef {
+  private SourceVariable v;
+  private BasicBlock bb;
+  private int i;
+
+  WriteDefinition() { this = TWriteDef(v, bb, i) }
+
+  override string toString() { result = "WriteDef" }
+}
+
+/** A phi node. */
+class PhiNode extends Definition, TPhiNode {
+  override string toString() { result = "Phi" }
+}
+
+/**
+ * An SSA definition that represents an uncertain update of the underlying
+ * source variable.
+ */
+class UncertainWriteDefinition extends WriteDefinition {
+  UncertainWriteDefinition() {
+    exists(SourceVariable v, BasicBlock bb, int i |
+      this.definesAt(v, bb, i) and
+      variableWrite(bb, i, v, false)
+    )
+  }
+}

csharp/ql/src/semmle/code/cil/internal/SsaImplSpecific.qll

@@ -0,0 +1,30 @@
+/** Provides the CIL specific parameters for `SsaImplCommon.qll`. */
+
+private import cil
+private import SsaImpl
+
+class BasicBlock = CIL::BasicBlock;
+
+BasicBlock getImmediateBasicBlockDominator(BasicBlock bb) { result = bb.getImmediateDominator() }
+
+BasicBlock getABasicBlockSuccessor(BasicBlock bb) { result = bb.getASuccessor() }
+
+class ExitBasicBlock = CIL::ExitBasicBlock;
+
+class SourceVariable = CIL::StackVariable;
+
+predicate variableWrite(BasicBlock bb, int i, SourceVariable v, boolean certain) {
+  forceCachingInSameStage() and
+  exists(CIL::VariableUpdate vu |
+    vu.updatesAt(bb, i) and
+    v = vu.getVariable() and
+    certain = true
+  )
+}
+
+predicate variableRead(BasicBlock bb, int i, SourceVariable v, boolean certain) {
+  exists(CIL::ReadAccess ra | bb.getNode(i) = ra |
+    ra.getTarget() = v and
+    certain = true
+  )
+}

csharp/ql/src/semmle/code/csharp/commons/Disposal.qll

@@ -11,14 +11,22 @@ private predicate isDisposeMethod(DotNet::Callable method) {
   method.getNumberOfParameters() = 0
 }
 
+private predicate cilVariableReadFlowsTo(CIL::Variable variable, CIL::DataFlowNode n) {
+  n = variable.getARead()
+  or
+  exists(CIL::DataFlowNode mid |
+    cilVariableReadFlowsTo(variable, mid) and
+    mid.getALocalFlowSucc(n, any(CIL::Untainted u))
+  )
+}
+
 private predicate disposedCilVariable(CIL::Variable variable) {
   // `variable` is the `this` parameter on a dispose method.
   isDisposeMethod(variable.(CIL::ThisParameter).getMethod())
   or
   // `variable` is passed to a method that disposes it.
-  exists(CIL::Call call, CIL::Parameter param, CIL::ReadAccess read |
-    read.getTarget() = variable and
-    read.flowsTo(call.getArgumentForParameter(param)) and
+  exists(CIL::Call call, CIL::Parameter param |
+    cilVariableReadFlowsTo(variable, call.getArgumentForParameter(param)) and
     disposedCilVariable(param)
   )
   or
@@ -27,9 +35,8 @@ private predicate disposedCilVariable(CIL::Variable variable) {
   or
   // A variable is disposed if it's assigned to another variable
   // that may be disposed.
-  exists(CIL::ReadAccess read, CIL::WriteAccess write |
-    read.flowsTo(write.getExpr()) and
-    read.getTarget() = variable and
+  exists(CIL::WriteAccess write |
+    cilVariableReadFlowsTo(variable, write.getExpr()) and
     disposedCilVariable(write.getTarget())
   )
 }

java/change-notes/2021-03-25-remove-legacy-code-duplication-library.md

@@ -0,0 +1,2 @@
+lgtm,codescanning
+* The legacy code duplication library has been removed.

java/change-notes/2021-03-25-remove-legacy-filter-queries.md

@@ -0,0 +1,2 @@
+lgtm,codescanning
+* Legacy filter queries have been removed.

java/ql/src/Metrics/Dependencies/ExternalDependencies.ql

@@ -5,7 +5,6 @@
  * @kind treemap
  * @treemap.warnOn highValues
  * @metricType externalDependency
- * @precision medium
  * @id java/external-dependencies
  */
 

java/ql/src/Metrics/Files/FLinesOfCode.ql

@@ -5,7 +5,6 @@
  * @treemap.warnOn highValues
  * @metricType file
  * @metricAggregate avg sum max
- * @precision very-high
  * @id java/lines-of-code-in-files
  * @tags maintainability
  *       complexity

java/ql/src/Metrics/Files/FLinesOfComment.ql

@@ -5,7 +5,6 @@
  * @treemap.warnOn lowValues
  * @metricType file
  * @metricAggregate avg sum max
- * @precision very-high
  * @id java/lines-of-comments-in-files
  * @tags maintainability
  *       documentation

java/ql/src/Metrics/Files/FLinesOfCommentedCode.ql

@@ -5,7 +5,6 @@
  * @treemap.warnOn highValues
  * @metricType file
  * @metricAggregate avg sum max
- * @precision high
  * @id java/lines-of-commented-out-code-in-files
  * @tags maintainability
  *       documentation

java/ql/src/Metrics/Files/FLinesOfDuplicatedCode.ql

@@ -7,21 +7,13 @@
  * @treemap.warnOn highValues
  * @metricType file
  * @metricAggregate avg sum max
- * @precision high
  * @id java/duplicated-lines-in-files
  * @tags testability
  *       modularity
  */
 
-import external.CodeDuplication
+import java
 
 from File f, int n
-where
-  n =
-    count(int line |
-      exists(DuplicateBlock d | d.sourceFile() = f |
-        line in [d.sourceStartLine() .. d.sourceEndLine()] and
-        not whitelistedLineForDuplication(f, line)
-      )
-    )
+where none()
 select f, n order by n desc

java/ql/src/Metrics/Files/FLinesOfSimilarCode.ql

@@ -11,15 +11,8 @@
  * @tags testability
  */
 
-import external.CodeDuplication
+import java
 
 from File f, int n
-where
-  n =
-    count(int line |
-      exists(SimilarBlock d | d.sourceFile() = f |
-        line in [d.sourceStartLine() .. d.sourceEndLine()] and
-        not whitelistedLineForDuplication(f, line)
-      )
-    )
+where none()
 select f, n order by n desc

java/ql/src/Metrics/Files/FNumberOfTests.ql

@@ -5,7 +5,6 @@
  * @treemap.warnOn lowValues
  * @metricType file
  * @metricAggregate avg sum max
- * @precision medium
  * @id java/tests-in-files
  * @tags maintainability
  */

java/ql/src/experimental/Security/CWE/CWE-036/OpenStream.ql

@@ -1,8 +1,13 @@
 /**
  * @name openStream called on URLs created from remote source
  * @description Calling openStream on URLs created from remote source
- * can lead to local file disclosure.
+ *              can lead to local file disclosure.
  * @kind path-problem
+ * @problem.severity warning
+ * @precision medium
+ * @id java/openstream-called-on-tainted-url
+ * @tags security
+ *       external/cwe/cwe-036
  */
 
 import java

java/ql/src/experimental/Security/CWE/CWE-117/LogInjection.qhelp

@@ -0,0 +1,49 @@
+
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+
+<overview>
+
+<p>If unsanitized user input is written to a log entry, a malicious user may be able to forge new log entries.</p>
+
+<p>Forgery can occur if a user provides some input creating the appearance of multiple
+  log entries. This can include unescaped new-line characters, or HTML or other markup.</p>
+</overview>
+
+<recommendation>
+<p>
+User input should be suitably sanitized before it is logged.
+</p>
+<p>
+If the log entries are plain text then line breaks should be removed from user input, using for example
+<code>String replace(char oldChar, char newChar)</code> or similar. Care should also be taken that user input is clearly marked
+in log entries, and that a malicious user cannot cause confusion in other ways.
+</p>
+<p>
+For log entries that will be displayed in HTML, user input should be HTML encoded before being logged, to prevent forgery and
+other forms of HTML injection.
+</p>
+
+</recommendation>
+
+<example>
+  <p>In the example, a username, provided by the user, is logged using <code>logger.warn</code> (from  <code>org.slf4j.Logger</code>). 
+    In the first case (<code>/bad</code> endpoint), the username is logged without any sanitization.
+    If a malicious user provides <code>Guest'%0AUser:'Admin</code> as a username parameter, 
+    the log entry will be split into two separate lines, where the first line will be <code>User:'Guest'</code> and the second one will be <code>User:'Admin'</code>.
+</p>
+<sample src="LogInjectionBad.java" />
+
+<p> In the second case (<code>/good</code> endpoint), <code>matches()</code> is used to ensure the user input only has alphanumeric characters.
+  If a malicious user provides `Guest'%0AUser:'Admin` as a username parameter, 
+  the log entry will not be split into two separate lines, resulting in a single line <code>User:'Guest'User:'Admin'</code>.</p>
+
+<sample src="LogInjectionGood.java" />
+</example>
+
+<references>
+<li>OWASP: <a href="https://owasp.org/www-community/attacks/Log_Injection">Log Injection</a>.</li>
+</references>
+</qhelp>

java/ql/src/experimental/Security/CWE/CWE-117/LogInjection.ql

@@ -0,0 +1,38 @@
+/**
+ * @name Log Injection
+ * @description Building log entries from user-controlled data is vulnerable to
+ *              insertion of forged log entries by a malicious user.
+ * @kind path-problem
+ * @problem.severity error
+ * @precision high
+ * @id java/log-injection
+ * @tags security
+ *       external/cwe/cwe-117
+ */
+
+import java
+import DataFlow::PathGraph
+import experimental.semmle.code.java.Logging
+import semmle.code.java.dataflow.FlowSources
+
+/**
+ * A taint-tracking configuration for tracking untrusted user input used in log entries.
+ */
+private class LogInjectionConfiguration extends TaintTracking::Configuration {
+  LogInjectionConfiguration() { this = "Log Injection" }
+
+  override predicate isSource(DataFlow::Node source) { source instanceof RemoteFlowSource }
+
+  override predicate isSink(DataFlow::Node sink) {
+    sink.asExpr() = any(LoggingCall c).getALogArgument()
+  }
+
+  override predicate isSanitizer(DataFlow::Node node) {
+    node.getType() instanceof BoxedType or node.getType() instanceof PrimitiveType
+  }
+}
+
+from LogInjectionConfiguration cfg, DataFlow::PathNode source, DataFlow::PathNode sink
+where cfg.hasFlowPath(source, sink)
+select sink.getNode(), source, sink, "$@ flows to log entry.", source.getNode(),
+  "User-provided value"

java/ql/src/experimental/Security/CWE/CWE-117/LogInjectionBad.java

@@ -0,0 +1,24 @@
+package com.example.restservice;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+@RestController
+public class LogInjection {
+
+    private final Logger log = LoggerFactory.getLogger(LogInjection.class);
+
+    // /bad?username=Guest'%0AUser:'Admin
+    @GetMapping("/bad")
+    public String bad(@RequestParam(value = "username", defaultValue = "name") String username) {
+        log.warn("User:'{}'", username);
+        // The logging call above would result in multiple log entries as shown below:
+        // User:'Guest'
+        // User:'Admin'
+        return username;
+    }
+}
+

java/ql/src/experimental/Security/CWE/CWE-117/LogInjectionGood.java

@@ -0,0 +1,25 @@
+package com.example.restservice;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+@RestController
+public class LogInjection {
+
+    private final Logger log = LoggerFactory.getLogger(LogInjection.class);
+
+    // /good?username=Guest'%0AUser:'Admin
+    @GetMapping("/good")
+    public String good(@RequestParam(value = "username", defaultValue = "name") String username) {
+        // The regex check here, allows only alphanumeric characters to pass.
+        // Hence, does not result in log injection
+        if (username.matches("\w*")) {
+            log.warn("User:'{}'", username);
+            
+            return username;
+        }
+    }
+}

java/ql/src/experimental/Security/CWE/CWE-273/UnsafeCertTrust.ql

@@ -1,7 +1,12 @@
 /**
  * @name Unsafe certificate trust
- * @description Unsafe implementation of the interface X509TrustManager and SSLSocket/SSLEngine ignores all SSL certificate validation errors when establishing an HTTPS connection, thereby making the app vulnerable to man-in-the-middle attacks.
+ * @description Unsafe implementation of the interface X509TrustManager and
+ *              SSLSocket/SSLEngine ignores all SSL certificate validation
+ *              errors when establishing an HTTPS connection, thereby making
+ *              the app vulnerable to man-in-the-middle attacks.
  * @kind problem
+ * @problem.severity warning
+ * @precision medium
  * @id java/unsafe-cert-trust
  * @tags security
  *       external/cwe-273

java/ql/src/experimental/Security/CWE/CWE-295/JxBrowserWithoutCertValidation.ql

@@ -1,7 +1,11 @@
 /**
  * @name JxBrowser with disabled certificate validation
- * @description Insecure configuration of JxBrowser disables certificate validation making the app vulnerable to man-in-the-middle attacks.
+ * @description Insecure configuration of JxBrowser disables certificate
+ *              validation making the app vulnerable to man-in-the-middle
+ *              attacks.
  * @kind problem
+ * @problem.severity warning
+ * @precision medium
  * @id java/jxbrowser/disabled-certificate-validation
  * @tags security
  *       external/cwe/cwe-295

java/ql/src/experimental/Security/CWE/CWE-297/InsecureJavaMail.ql

@@ -1,8 +1,12 @@
 /**
- * @id java/insecure-smtp-ssl
  * @name Insecure JavaMail SSL Configuration
- * @description Java application configured to use authenticated mail session over SSL does not validate the SSL certificate to properly ensure that it is actually associated with that host.
+ * @description Java application configured to use authenticated mail session
+ *              over SSL does not validate the SSL certificate to properly
+ *              ensure that it is actually associated with that host.
  * @kind problem
+ * @problem.severity warning
+ * @precision medium
+ * @id java/insecure-smtp-ssl
  * @tags security
  *       external/cwe-297
  */

java/ql/src/experimental/Security/CWE/CWE-297/InsecureLdapEndpoint.ql

@@ -1,8 +1,11 @@
 /**
  * @name Insecure LDAPS Endpoint Configuration
- * @description Java application configured to disable LDAPS endpoint identification does not validate
- *              the SSL certificate to properly ensure that it is actually associated with that host.
+ * @description Java application configured to disable LDAPS endpoint
+ *              identification does not validate the SSL certificate to
+ *              properly ensure that it is actually associated with that host.
  * @kind problem
+ * @problem.severity warning
+ * @precision medium
  * @id java/insecure-ldaps-endpoint
  * @tags security
  *       external/cwe-297

java/ql/src/experimental/Security/CWE/CWE-312/CleartextStorageSharedPrefs.ql

@@ -1,7 +1,11 @@
 /**
  * @name Cleartext storage of sensitive information using `SharedPreferences` on Android
- * @description Cleartext Storage of Sensitive Information using SharedPreferences on Android allows access for users with root privileges or unexpected exposure from chained vulnerabilities.
+ * @description Cleartext Storage of Sensitive Information using
+ *              SharedPreferences on Android allows access for users with root
+ *              privileges or unexpected exposure from chained vulnerabilities.
  * @kind problem
+ * @problem.severity warning
+ * @precision medium
  * @id java/android/cleartext-storage-shared-prefs
  * @tags security
  *       external/cwe/cwe-312

java/ql/src/experimental/Security/CWE/CWE-326/InsufficientKeySize.ql

@@ -2,6 +2,8 @@
  * @name Weak encryption: Insufficient key size
  * @description Finds uses of encryption algorithms with too small a key size
  * @kind problem
+ * @problem.severity warning
+ * @precision medium
  * @id java/insufficient-key-size
  * @tags security
  *       external/cwe/cwe-326

java/ql/src/experimental/Security/CWE/CWE-489/EJBMain.ql

@@ -2,6 +2,8 @@
  * @name Main Method in Enterprise Java Bean
  * @description Java EE applications with a main method.
  * @kind problem
+ * @problem.severity warning
+ * @precision medium
  * @id java/main-method-in-enterprise-bean
  * @tags security
  *       external/cwe-489

java/ql/src/experimental/Security/CWE/CWE-489/WebComponentMain.ql

@@ -2,6 +2,8 @@
  * @name Main Method in Java EE Web Components
  * @description Java EE web applications with a main method.
  * @kind problem
+ * @problem.severity warning
+ * @precision medium
  * @id java/main-method-in-web-components
  * @tags security
  *       external/cwe-489

java/ql/src/experimental/Security/CWE/CWE-502/SpringExporterUnsafeDeserialization.java

@@ -0,0 +1,24 @@
+@Configuration
+public class Server {
+
+    @Bean(name = "/account")
+    HttpInvokerServiceExporter accountService() {
+        HttpInvokerServiceExporter exporter = new HttpInvokerServiceExporter();
+        exporter.setService(new AccountServiceImpl());
+        exporter.setServiceInterface(AccountService.class);
+        return exporter;
+    }
+
+}
+
+class AccountServiceImpl implements AccountService {
+
+    @Override
+    public String echo(String data) {
+        return data;
+    }
+}
+
+interface AccountService {
+    String echo(String data);
+}

java/ql/src/experimental/Security/CWE/CWE-502/SpringExporterUnsafeDeserialization.xml

@@ -0,0 +1,4 @@
+<bean name="/account" class="org.springframework.remoting.httpinvoker.HttpInvokerServiceExporter">
+    <property name="service" ref="accountService"/>
+    <property name="serviceInterface" value="AccountService"/>
+</bean>

java/ql/src/experimental/Security/CWE/CWE-502/UnsafeSpringExporterInConfigurationClass.qhelp

@@ -0,0 +1,8 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+  <include src="UnsafeSpringExporterQuery.inc.qhelp" />
+  <include src="UnsafeSpringExporterInConfigurationClassExample.inc.qhelp" />
+  <include src="UnsafeSpringExporterReferences.inc.qhelp" />
+</qhelp>

java/ql/src/experimental/Security/CWE/CWE-502/UnsafeSpringExporterInConfigurationClass.ql

@@ -0,0 +1,58 @@
+/**
+ * @name Unsafe deserialization with Spring's remote service exporters.
+ * @description A Spring bean, which is based on RemoteInvocationSerializingExporter,
+ *              initializes an endpoint that uses ObjectInputStream to deserialize
+ *              incoming data. In the worst case, that may lead to remote code execution.
+ * @kind problem
+ * @problem.severity error
+ * @precision high
+ * @id java/unsafe-deserialization-spring-exporter-in-configuration-class
+ * @tags security
+ *       external/cwe/cwe-502
+ */
+
+import java
+import UnsafeSpringExporterLib
+
+/**
+ * Holds if `type` is a Spring configuration that declares beans.
+ */
+private predicate isConfiguration(RefType type) {
+  type.hasAnnotation("org.springframework.context.annotation", "Configuration") or
+  isConfigurationAnnotation(type.getAnAnnotation())
+}
+
+/**
+ * Holds if `annotation` is a Java annotations that declares a Spring configuration.
+ */
+private predicate isConfigurationAnnotation(Annotation annotation) {
+  isConfiguration(annotation.getType()) or
+  isConfigurationAnnotation(annotation.getType().getAnAnnotation())
+}
+
+/**
+ * A method that initializes a unsafe bean based on `RemoteInvocationSerializingExporter`.
+ */
+private class UnsafeBeanInitMethod extends Method {
+  string identifier;
+
+  UnsafeBeanInitMethod() {
+    isRemoteInvocationSerializingExporter(this.getReturnType()) and
+    isConfiguration(this.getDeclaringType()) and
+    exists(Annotation a | this.getAnAnnotation() = a |
+      a.getType().hasQualifiedName("org.springframework.context.annotation", "Bean") and
+      if a.getValue("name") instanceof StringLiteral
+      then identifier = a.getValue("name").(StringLiteral).getRepresentedString()
+      else identifier = this.getName()
+    )
+  }
+
+  /**
+   * Gets this bean's name if given by the `Bean` annotation, or this method's identifier otherwise.
+   */
+  string getBeanIdentifier() { result = identifier }
+}
+
+from UnsafeBeanInitMethod method
+select method,
+  "Unsafe deserialization in a Spring exporter bean '" + method.getBeanIdentifier() + "'"

java/ql/src/experimental/Security/CWE/CWE-502/UnsafeSpringExporterInConfigurationClassExample.inc.qhelp

@@ -0,0 +1,14 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+
+<example>
+<p>
+The following example shows how a vulnerable HTTP endpoint can be defined
+using <code>HttpInvokerServiceExporter</code> and Spring annotations:
+</p>
+<sample src="SpringExporterUnsafeDeserialization.java" />
+</example>
+
+</qhelp>

java/ql/src/experimental/Security/CWE/CWE-502/UnsafeSpringExporterInXMLConfiguration.qhelp

@@ -0,0 +1,8 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+  <include src="UnsafeSpringExporterQuery.inc.qhelp" />
+  <include src="UnsafeSpringExporterInXMLConfigurationExample.inc.qhelp" />
+  <include src="UnsafeSpringExporterReferences.inc.qhelp" />
+</qhelp>

java/ql/src/experimental/Security/CWE/CWE-502/UnsafeSpringExporterInXMLConfiguration.ql

@@ -0,0 +1,20 @@
+/**
+ * @name Unsafe deserialization with Spring's remote service exporters.
+ * @description A Spring bean, which is based on RemoteInvocationSerializingExporter,
+ *              initializes an endpoint that uses ObjectInputStream to deserialize
+ *              incoming data. In the worst case, that may lead to remote code execution.
+ * @kind problem
+ * @problem.severity error
+ * @precision high
+ * @id java/unsafe-deserialization-spring-exporter-in-xml-configuration
+ * @tags security
+ *       external/cwe/cwe-502
+ */
+
+import java
+import semmle.code.java.frameworks.spring.SpringBean
+import UnsafeSpringExporterLib
+
+from SpringBean bean
+where isRemoteInvocationSerializingExporter(bean.getClass())
+select bean, "Unsafe deserialization in a Spring exporter bean '" + bean.getBeanIdentifier() + "'"

java/ql/src/experimental/Security/CWE/CWE-502/UnsafeSpringExporterInXMLConfigurationExample.inc.qhelp

@@ -0,0 +1,13 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+
+<example>
+<p>
+The following examples shows how a vulnerable HTTP endpoint can be defined in a Spring XML config:
+</p>
+<sample src="SpringExporterUnsafeDeserialization.xml" />
+</example>
+
+</qhelp>

java/ql/src/experimental/Security/CWE/CWE-502/UnsafeSpringExporterLib.qll

@@ -0,0 +1,9 @@
+import java
+
+/**
+ * Holds if `type` is `RemoteInvocationSerializingExporter`.
+ */
+predicate isRemoteInvocationSerializingExporter(RefType type) {
+  type.getASupertype*()
+      .hasQualifiedName("org.springframework.remoting.rmi", "RemoteInvocationSerializingExporter")
+}

java/ql/src/experimental/Security/CWE/CWE-502/UnsafeSpringExporterQuery.inc.qhelp

@@ -0,0 +1,41 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+
+<overview>
+<p>
+The Spring Framework provides an abstract base class <code>RemoteInvocationSerializingExporter</code>
+for creating remote service exporters.
+A Spring exporter, which is based on this class, deserializes incoming data using <code>ObjectInputStream</code>.
+Deserializing untrusted data is easily exploitable and in many cases allows an attacker
+to execute arbitrary code.
+</p>
+<p>
+The Spring Framework also provides <code>HttpInvokerServiceExporter</code>
+and <code>SimpleHttpInvokerServiceExporter</code> classes
+that extend <code>RemoteInvocationSerializingExporter</code>.
+</p>
+<p>
+These classes export specified beans as HTTP endpoints that deserialize data from an HTTP request
+using unsafe <code>ObjectInputStream</code>. If a remote attacker can reach such endpoints,
+it results in remote code execution in the worst case.
+</p>
+<p>
+CVE-2016-1000027 has been assigned to this issue in the Spring Framework.
+It is regarded as a design limitation, and can be mitigated but not fixed outright.
+</p>
+</overview>
+
+<recommendation>
+<p>
+Avoid using <code>HttpInvokerServiceExporter</code>, <code>SimpleHttpInvokerServiceExporter</code>
+and any other exporter that is based on <code>RemoteInvocationSerializingExporter</code>.
+Instead, use other message formats for API endpoints (for example, JSON),
+but make sure that the underlying deserialization mechanism is properly configured
+so that deserialization attacks are not possible. If the vulnerable exporters can not be replaced,
+consider using global deserialization filters introduced in JEP 290.
+</p>
+</recommendation>
+
+</qhelp>

java/ql/src/experimental/Security/CWE/CWE-502/UnsafeSpringExporterReferences.inc.qhelp

@@ -0,0 +1,37 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+
+<references>
+<li>
+OWASP:
+<a href="https://www.owasp.org/index.php/Deserialization_of_untrusted_data">Deserialization of untrusted data</a>.
+</li>
+<li>
+Spring Framework API documentation:
+<a href="https://docs.spring.io/spring-framework/docs/current/javadoc-api/org/springframework/remoting/rmi/RemoteInvocationSerializingExporter.html">RemoteInvocationSerializingExporter class</a>
+</li>
+<li>
+Spring Framework API documentation:
+<a href="https://docs.spring.io/spring-framework/docs/current/javadoc-api/org/springframework/remoting/httpinvoker/HttpInvokerServiceExporter.html">HttpInvokerServiceExporter class</a>
+</li>
+<li>
+National Vulnerability Database:
+<a href="https://nvd.nist.gov/vuln/detail/CVE-2016-1000027">CVE-2016-1000027</a>
+</li>
+<li>
+Tenable Research Advisory:
+<a href="https://www.tenable.com/security/research/tra-2016-20">[R2] Pivotal Spring Framework HttpInvokerServiceExporter readRemoteInvocation Method Untrusted Java Deserialization</a>
+</li>
+<li>
+Spring Framework bug tracker:
+<a href="https://github.com/spring-projects/spring-framework/issues/24434">Sonatype vulnerability CVE-2016-1000027 in Spring-web project</a>  
+</li>
+<li>
+OpenJDK:
+<a href="https://openjdk.java.net/jeps/290">JEP 290: Filter Incoming Serialization Data</a>
+</li>
+</references>
+
+</qhelp>

java/ql/src/experimental/Security/CWE/CWE-522/InsecureBasicAuth.ql

@@ -1,7 +1,12 @@
 /**
  * @name Insecure basic authentication
- * @description Basic authentication only obfuscates username/password in Base64 encoding, which can be easily recognized and reversed. Transmission of sensitive information not over HTTPS is vulnerable to packet sniffing.
+ * @description Basic authentication only obfuscates username/password in
+ *              Base64 encoding, which can be easily recognized and reversed.
+ *              Transmission of sensitive information not over HTTPS is
+ *              vulnerable to packet sniffing.
  * @kind path-problem
+ * @problem.severity warning
+ * @precision medium
  * @id java/insecure-basic-auth
  * @tags security
  *       external/cwe-522

java/ql/src/experimental/Security/CWE/CWE-522/InsecureLdapAuth.ql

@@ -2,6 +2,8 @@
  * @name Insecure LDAP authentication
  * @description LDAP authentication with credentials sent in cleartext.
  * @kind path-problem
+ * @problem.severity warning
+ * @precision medium
  * @id java/insecure-ldap-auth
  * @tags security
  *       external/cwe-522

java/ql/src/experimental/Security/CWE/CWE-532/SensitiveInfoLog.ql

@@ -1,8 +1,11 @@
 /**
- * @id java/sensitiveinfo-in-logfile
  * @name Insertion of sensitive information into log files
- * @description Writing sensitive information to log files can give valuable guidance to an attacker or expose sensitive user information.
+ * @description Writing sensitive information to log files can give valuable
+ *              guidance to an attacker or expose sensitive user information.
  * @kind path-problem
+ * @problem.severity warning
+ * @precision medium
+ * @id java/sensitiveinfo-in-logfile
  * @tags security
  *       external/cwe-532
  */
@@ -10,6 +13,7 @@
 import java
 import semmle.code.java.dataflow.TaintTracking
 import semmle.code.java.security.SensitiveActions
+import experimental.semmle.code.java.Logging
 import DataFlow
 import PathGraph
 
@@ -27,38 +31,14 @@ class CredentialExpr extends Expr {
   }
 }
 
-/** Class of popular logging utilities * */
-class LoggerType extends RefType {
-  LoggerType() {
-    this.hasQualifiedName("org.apache.log4j", "Category") or //Log4J
-    this.hasQualifiedName("org.apache.logging.log4j", "Logger") or //Log4J 2
-    this.hasQualifiedName("org.slf4j", "Logger") or //SLF4j and Gradle Logging
-    this.hasQualifiedName("org.jboss.logging", "BasicLogger") or //JBoss Logging
-    this.hasQualifiedName("org.jboss.logging", "Logger") or //JBoss Logging (`org.jboss.logging.Logger` in some implementations like JBoss Application Server 4.0.4 did not implement `BasicLogger`)
-    this.hasQualifiedName("org.apache.commons.logging", "Log") or //Apache Commons Logging
-    this.hasQualifiedName("org.scijava.log", "Logger") //SciJava Logging
-  }
-}
-
-predicate isSensitiveLoggingSink(DataFlow::Node sink) {
-  exists(MethodAccess ma |
-    ma.getMethod().getDeclaringType() instanceof LoggerType and
-    (
-      ma.getMethod().hasName("debug") or
-      ma.getMethod().hasName("trace") or
-      ma.getMethod().hasName("debugf") or
-      ma.getMethod().hasName("debugv")
-    ) and //Check low priority log levels which are more likely to be real issues to reduce false positives
-    sink.asExpr() = ma.getAnArgument()
-  )
-}
-
 class LoggerConfiguration extends DataFlow::Configuration {
   LoggerConfiguration() { this = "Logger Configuration" }
 
   override predicate isSource(DataFlow::Node source) { source.asExpr() instanceof CredentialExpr }
 
-  override predicate isSink(DataFlow::Node sink) { isSensitiveLoggingSink(sink) }
+  override predicate isSink(DataFlow::Node sink) {
+    exists(LoggingCall c | sink.asExpr() = c.getALogArgument())
+  }
 
   override predicate isAdditionalFlowStep(DataFlow::Node node1, DataFlow::Node node2) {
     TaintTracking::localTaintStep(node1, node2)

java/ql/src/experimental/Security/CWE/CWE-548/InsecureDirectoryConfig.ql

@@ -1,7 +1,12 @@
 /**
  * @name Directories and files exposure
- * @description A directory listing provides an attacker with the complete index of all the resources located inside of the complete web directory, which could yield files containing sensitive information like source code and credentials to the attacker.
+ * @description A directory listing provides an attacker with the complete
+ *              index of all the resources located inside of the complete web
+ *              directory, which could yield files containing sensitive
+ *              information like source code and credentials to the attacker.
  * @kind problem
+ * @problem.severity warning
+ * @precision medium
  * @id java/server-directory-listing
  * @tags security
  *       external/cwe-548

java/ql/src/experimental/Security/CWE/CWE-555/PasswordInConfigurationFile.ql

@@ -2,6 +2,8 @@
  * @name Password in configuration file
  * @description Finds passwords in configuration files.
  * @kind problem
+ * @problem.severity warning
+ * @precision medium
  * @id java/password-in-configuration
  * @tags security
  *       external/cwe/cwe-555

java/ql/src/experimental/Security/CWE/CWE-598/SensitiveGetQuery.ql

@@ -2,6 +2,8 @@
  * @name Sensitive GET Query
  * @description Use of GET request method with sensitive query strings.
  * @kind path-problem
+ * @problem.severity warning
+ * @precision medium
  * @id java/sensitive-query-with-get
  * @tags security
  *       external/cwe-598

java/ql/src/experimental/Security/CWE/CWE-600/UncaughtServletException.ql

@@ -1,7 +1,12 @@
 /**
  * @name Uncaught Servlet Exception
- * @description Uncaught exceptions in a servlet could leave a system in an unexpected state, possibly resulting in denial-of-service attacks or the exposure of sensitive information disclosed in stack traces.
+ * @description Uncaught exceptions in a servlet could leave a system in an
+ *              unexpected state, possibly resulting in denial-of-service
+ *              attacks or the exposure of sensitive information disclosed in
+ *              stack traces.
  * @kind path-problem
+ * @problem.severity warning
+ * @precision medium
  * @id java/uncaught-servlet-exception
  * @tags security
  *       external/cwe-600

java/ql/src/experimental/Security/CWE/CWE-749/UnsafeAndroidAccess.ql

@@ -1,8 +1,11 @@
 /**
  * @name Unsafe resource fetching in Android webview
- * @id java/android/unsafe-android-webview-fetch
- * @description JavaScript rendered inside WebViews can access any protected application file and web resource from any origin
+ * @description JavaScript rendered inside WebViews can access any protected
+ *              application file and web resource from any origin
  * @kind path-problem
+ * @problem.severity warning
+ * @precision medium
+ * @id java/android/unsafe-android-webview-fetch
  * @tags security
  *       external/cwe/cwe-749
  *       external/cwe/cwe-079

java/ql/src/experimental/Security/CWE/CWE-755/NFEAndroidDoS.ql

@@ -1,8 +1,13 @@
 /**
  * @name Local Android DoS Caused By NumberFormatException
- * @id java/android/nfe-local-android-dos
- * @description NumberFormatException thrown but not caught by an Android application that allows external inputs can crash the application, constituting a local Denial of Service (DoS) attack.
+ * @description NumberFormatException thrown but not caught by an Android
+ *              application that allows external inputs can crash the
+ *              application, constituting a local Denial of Service (DoS)
+ *              attack.
  * @kind path-problem
+ * @problem.severity warning
+ * @precision medium
+ * @id java/android/nfe-local-android-dos
  * @tags security
  *       external/cwe/cwe-755
  */

java/ql/src/experimental/Security/CWE/CWE-927/SensitiveBroadcast.ql

@@ -1,8 +1,12 @@
 /**
  * @name Broadcasting sensitive data to all Android applications
- * @id java/sensitive-broadcast
- * @description An Android application uses implicit intents to broadcast sensitive data to all applications without specifying any receiver permission.
+ * @description An Android application uses implicit intents to broadcast
+ *              sensitive data to all applications without specifying any
+ *              receiver permission.
  * @kind path-problem
+ * @problem.severity warning
+ * @precision medium
+ * @id java/sensitive-broadcast
  * @tags security
  *       external/cwe-927
  */

java/ql/src/experimental/Security/CWE/CWE-939/IncorrectURLVerification.ql

@@ -1,8 +1,12 @@
 /**
- * @id java/incorrect-url-verification
  * @name Incorrect URL verification
- * @description Apps that rely on URL parsing to verify that a given URL is pointing to a trusted server are susceptible to wrong ways of URL parsing and verification.
+ * @description Apps that rely on URL parsing to verify that a given URL is
+ *              pointing to a trusted server are susceptible to wrong ways of
+ *              URL parsing and verification.
  * @kind problem
+ * @problem.severity warning
+ * @precision medium
+ * @id java/incorrect-url-verification
  * @tags security
  *       external/cwe-939
  */

java/ql/src/experimental/semmle/code/java/Logging.qll

@@ -0,0 +1,35 @@
+/**
+ * Provides classes and predicates for working with loggers.
+ */
+
+import java
+
+/** Models a call to a logging method. */
+class LoggingCall extends MethodAccess {
+  LoggingCall() {
+    exists(RefType t, Method m |
+      t.hasQualifiedName("org.apache.log4j", "Category") or // Log4j 1
+      t.hasQualifiedName("org.apache.logging.log4j", ["Logger", "LogBuilder"]) or // Log4j 2
+      t.hasQualifiedName("org.apache.commons.logging", "Log") or
+      // JBoss Logging (`org.jboss.logging.Logger` in some implementations like JBoss Application Server 4.0.4 did not implement `BasicLogger`)
+      t.hasQualifiedName("org.jboss.logging", ["BasicLogger", "Logger"]) or
+      t.hasQualifiedName("org.slf4j.spi", "LoggingEventBuilder") or
+      t.hasQualifiedName("org.slf4j", "Logger") or
+      t.hasQualifiedName("org.scijava.log", "Logger") or
+      t.hasQualifiedName("com.google.common.flogger", "LoggingApi") or
+      t.hasQualifiedName("java.lang", "System$Logger") or
+      t.hasQualifiedName("java.util.logging", "Logger") or
+      t.hasQualifiedName("android.util", "Log")
+    |
+      (
+        m.getDeclaringType().getASourceSupertype*() = t or
+        m.getDeclaringType().extendsOrImplements*(t)
+      ) and
+      m.getReturnType() instanceof VoidType and
+      this = m.getAReference()
+    )
+  }
+
+  /** Returns an argument which would be logged by this call. */
+  Argument getALogArgument() { result = this.getArgument(_) }
+}

java/ql/src/external/CodeDuplication.qll

@@ -1,268 +0,0 @@
-import java
-
-private string relativePath(File file) { result = file.getRelativePath().replaceAll("\\", "/") }
-
-cached
-private predicate tokenLocation(File file, int sl, int sc, int ec, int el, Copy copy, int index) {
-  file = copy.sourceFile() and
-  tokens(copy, index, sl, sc, ec, el)
-}
-
-class Copy extends @duplication_or_similarity {
-  private int lastToken() { result = max(int i | tokens(this, i, _, _, _, _) | i) }
-
-  int tokenStartingAt(Location loc) {
-    tokenLocation(loc.getFile(), loc.getStartLine(), loc.getStartColumn(), _, _, this, result)
-  }
-
-  int tokenEndingAt(Location loc) {
-    tokenLocation(loc.getFile(), _, _, loc.getEndLine(), loc.getEndColumn(), this, result)
-  }
-
-  int sourceStartLine() { tokens(this, 0, result, _, _, _) }
-
-  int sourceStartColumn() { tokens(this, 0, _, result, _, _) }
-
-  int sourceEndLine() { tokens(this, lastToken(), _, _, result, _) }
-
-  int sourceEndColumn() { tokens(this, lastToken(), _, _, _, result) }
-
-  int sourceLines() { result = this.sourceEndLine() + 1 - this.sourceStartLine() }
-
-  int getEquivalenceClass() { duplicateCode(this, _, result) or similarCode(this, _, result) }
-
-  File sourceFile() {
-    exists(string name | duplicateCode(this, name, _) or similarCode(this, name, _) |
-      name.replaceAll("\\", "/") = relativePath(result)
-    )
-  }
-
-  predicate hasLocationInfo(
-    string filepath, int startline, int startcolumn, int endline, int endcolumn
-  ) {
-    sourceFile().getAbsolutePath() = filepath and
-    startline = sourceStartLine() and
-    startcolumn = sourceStartColumn() and
-    endline = sourceEndLine() and
-    endcolumn = sourceEndColumn()
-  }
-
-  string toString() { none() }
-}
-
-class DuplicateBlock extends Copy, @duplication {
-  override string toString() { result = "Duplicate code: " + sourceLines() + " duplicated lines." }
-}
-
-class SimilarBlock extends Copy, @similarity {
-  override string toString() {
-    result = "Similar code: " + sourceLines() + " almost duplicated lines."
-  }
-}
-
-Method sourceMethod() { hasLocation(result, _) and numlines(result, _, _, _) }
-
-int numberOfSourceMethods(Class c) {
-  result = count(Method m | m = sourceMethod() and m.getDeclaringType() = c)
-}
-
-private predicate blockCoversStatement(int equivClass, int first, int last, Stmt stmt) {
-  exists(DuplicateBlock b, Location loc |
-    stmt.getLocation() = loc and
-    first = b.tokenStartingAt(loc) and
-    last = b.tokenEndingAt(loc) and
-    b.getEquivalenceClass() = equivClass
-  )
-}
-
-private Stmt statementInMethod(Method m) {
-  result.getEnclosingCallable() = m and
-  not result instanceof BlockStmt
-}
-
-private predicate duplicateStatement(Method m1, Method m2, Stmt s1, Stmt s2) {
-  exists(int equivClass, int first, int last |
-    s1 = statementInMethod(m1) and
-    s2 = statementInMethod(m2) and
-    blockCoversStatement(equivClass, first, last, s1) and
-    blockCoversStatement(equivClass, first, last, s2) and
-    s1 != s2 and
-    m1 != m2
-  )
-}
-
-predicate duplicateStatements(Method m1, Method m2, int duplicate, int total) {
-  duplicate = strictcount(Stmt s | duplicateStatement(m1, m2, s, _)) and
-  total = strictcount(statementInMethod(m1))
-}
-
-/**
- * Pairs of methods that are identical.
- */
-predicate duplicateMethod(Method m, Method other) {
-  exists(int total | duplicateStatements(m, other, total, total))
-}
-
-predicate similarLines(File f, int line) {
-  exists(SimilarBlock b | b.sourceFile() = f and line in [b.sourceStartLine() .. b.sourceEndLine()])
-}
-
-private predicate similarLinesPerEquivalenceClass(int equivClass, int lines, File f) {
-  lines =
-    strictsum(SimilarBlock b, int toSum |
-      (b.sourceFile() = f and b.getEquivalenceClass() = equivClass) and
-      toSum = b.sourceLines()
-    |
-      toSum
-    )
-}
-
-pragma[noopt]
-private predicate similarLinesCovered(File f, int coveredLines, File otherFile) {
-  exists(int numLines | numLines = f.getTotalNumberOfLines() |
-    exists(int coveredApprox |
-      coveredApprox =
-        strictsum(int num |
-          exists(int equivClass |
-            similarLinesPerEquivalenceClass(equivClass, num, f) and
-            similarLinesPerEquivalenceClass(equivClass, num, otherFile) and
-            f != otherFile
-          )
-        ) and
-      exists(int n, int product | product = coveredApprox * 100 and n = product / numLines | n > 75)
-    ) and
-    exists(int notCovered |
-      notCovered = count(int j | j in [1 .. numLines] and not similarLines(f, j)) and
-      coveredLines = numLines - notCovered
-    )
-  )
-}
-
-predicate duplicateLines(File f, int line) {
-  exists(DuplicateBlock b |
-    b.sourceFile() = f and line in [b.sourceStartLine() .. b.sourceEndLine()]
-  )
-}
-
-private predicate duplicateLinesPerEquivalenceClass(int equivClass, int lines, File f) {
-  lines =
-    strictsum(DuplicateBlock b, int toSum |
-      (b.sourceFile() = f and b.getEquivalenceClass() = equivClass) and
-      toSum = b.sourceLines()
-    |
-      toSum
-    )
-}
-
-pragma[noopt]
-private predicate duplicateLinesCovered(File f, int coveredLines, File otherFile) {
-  exists(int numLines | numLines = f.getTotalNumberOfLines() |
-    exists(int coveredApprox |
-      coveredApprox =
-        strictsum(int num |
-          exists(int equivClass |
-            duplicateLinesPerEquivalenceClass(equivClass, num, f) and
-            duplicateLinesPerEquivalenceClass(equivClass, num, otherFile) and
-            f != otherFile
-          )
-        ) and
-      exists(int n, int product | product = coveredApprox * 100 and n = product / numLines | n > 75)
-    ) and
-    exists(int notCovered |
-      notCovered = count(int j | j in [1 .. numLines] and not duplicateLines(f, j)) and
-      coveredLines = numLines - notCovered
-    )
-  )
-}
-
-predicate similarFiles(File f, File other, int percent) {
-  exists(int covered, int total |
-    similarLinesCovered(f, covered, other) and
-    total = f.getTotalNumberOfLines() and
-    covered * 100 / total = percent and
-    percent > 80
-  ) and
-  not duplicateFiles(f, other, _)
-}
-
-predicate duplicateFiles(File f, File other, int percent) {
-  exists(int covered, int total |
-    duplicateLinesCovered(f, covered, other) and
-    total = f.getTotalNumberOfLines() and
-    covered * 100 / total = percent and
-    percent > 70
-  )
-}
-
-predicate duplicateAnonymousClass(AnonymousClass c, AnonymousClass other) {
-  exists(int numDup |
-    numDup =
-      strictcount(Method m1 |
-        exists(Method m2 |
-          duplicateMethod(m1, m2) and
-          m1 = sourceMethod() and
-          m1.getDeclaringType() = c and
-          m2.getDeclaringType() = other and
-          c != other
-        )
-      ) and
-    numDup = numberOfSourceMethods(c) and
-    numDup = numberOfSourceMethods(other) and
-    forall(Type t | c.getASupertype() = t | t = other.getASupertype())
-  )
-}
-
-pragma[noopt]
-predicate mostlyDuplicateClassBase(Class c, Class other, int numDup, int total) {
-  numDup =
-    strictcount(Method m1 |
-      exists(Method m2 |
-        duplicateMethod(m1, m2) and
-        m1 = sourceMethod() and
-        m1.getDeclaringType() = c and
-        m2.getDeclaringType() = other and
-        other instanceof Class and
-        c != other
-      )
-    ) and
-  total = numberOfSourceMethods(c) and
-  exists(int n, int product | product = 100 * numDup and n = product / total | n > 80)
-}
-
-predicate mostlyDuplicateClass(Class c, Class other, string message) {
-  exists(int numDup, int total |
-    mostlyDuplicateClassBase(c, other, numDup, total) and
-    not c instanceof AnonymousClass and
-    not other instanceof AnonymousClass and
-    (
-      total != numDup and
-      exists(string s1, string s2, string s3, string name |
-        s1 = " out of " and
-        s2 = " methods in " and
-        s3 = " are duplicated in $@." and
-        name = c.getName()
-      |
-        message = numDup + s1 + total + s2 + name + s3
-      )
-      or
-      total = numDup and
-      exists(string s1, string s2, string name |
-        s1 = "All methods in " and s2 = " are identical in $@." and name = c.getName()
-      |
-        message = s1 + name + s2
-      )
-    )
-  )
-}
-
-predicate fileLevelDuplication(File f, File other) {
-  similarFiles(f, other, _) or duplicateFiles(f, other, _)
-}
-
-predicate classLevelDuplication(Class c, Class other) {
-  duplicateAnonymousClass(c, other) or mostlyDuplicateClass(c, other, _)
-}
-
-predicate whitelistedLineForDuplication(File f, int line) {
-  exists(Import i | i.getFile() = f and i.getLocation().getStartLine() = line)
-}

java/ql/src/external/DefectFilter.qll

@@ -1,55 +0,0 @@
-/** Provides a class for working with defect query results stored in dashboard databases. */
-
-import java
-
-/**
- * Holds if `id` in the opaque identifier of a result reported by query `queryPath`,
- * such that `message` is the associated message and the location of the result spans
- * column `startcolumn` of line `startline` to column `endcolumn` of line `endline`
- * in file `filepath`.
- *
- * For more information, see [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
- */
-external predicate defectResults(
-  int id, string queryPath, string file, int startline, int startcol, int endline, int endcol,
-  string message
-);
-
-/**
- * A defect query result stored in a dashboard database.
- */
-class DefectResult extends int {
-  DefectResult() { defectResults(this, _, _, _, _, _, _, _) }
-
-  /** Gets the path of the query that reported the result. */
-  string getQueryPath() { defectResults(this, result, _, _, _, _, _, _) }
-
-  /** Gets the file in which this query result was reported. */
-  File getFile() {
-    exists(string path | defectResults(this, _, path, _, _, _, _, _) |
-      result.getAbsolutePath() = path
-    )
-  }
-
-  /** Gets the line on which the location of this query result starts. */
-  int getStartLine() { defectResults(this, _, _, result, _, _, _, _) }
-
-  /** Gets the column on which the location of this query result starts. */
-  int getStartColumn() { defectResults(this, _, _, _, result, _, _, _) }
-
-  /** Gets the line on which the location of this query result ends. */
-  int getEndLine() { defectResults(this, _, _, _, _, result, _, _) }
-
-  /** Gets the column on which the location of this query result ends. */
-  int getEndColumn() { defectResults(this, _, _, _, _, _, result, _) }
-
-  /** Gets the message associated with this query result. */
-  string getMessage() { defectResults(this, _, _, _, _, _, _, result) }
-
-  /** Gets the URL corresponding to the location of this query result. */
-  string getURL() {
-    result =
-      "file://" + getFile().getAbsolutePath() + ":" + getStartLine() + ":" + getStartColumn() + ":" +
-        getEndLine() + ":" + getEndColumn()
-  }
-}

java/ql/src/external/DuplicateAnonymous.ql

@@ -15,11 +15,8 @@
  */
 
 import java
-import CodeDuplication
 
 from AnonymousClass c, AnonymousClass other
-where
-  duplicateAnonymousClass(c, other) and
-  not fileLevelDuplication(c.getCompilationUnit(), other.getCompilationUnit())
+where none()
 select c, "Anonymous class is identical to $@.", other,
   "another anonymous class in " + other.getFile().getStem()

java/ql/src/external/DuplicateBlock.ql

@@ -8,16 +8,10 @@
  * @id java/duplicate-block
  */
 
-import CodeDuplication
+import java
 
-from DuplicateBlock d, DuplicateBlock other, int lines, File otherFile, int otherLine
-where
-  lines = d.sourceLines() and
-  lines > 10 and
-  other.getEquivalenceClass() = d.getEquivalenceClass() and
-  other != d and
-  otherFile = other.sourceFile() and
-  otherLine = other.sourceStartLine()
+from BlockStmt d, int lines, File otherFile, int otherLine
+where none()
 select d,
   "Duplicate code: " + lines + " lines are duplicated at " + otherFile.getStem() + ":" + otherLine +
     "."

java/ql/src/external/DuplicateMethod.ql

@@ -16,19 +16,8 @@
  */
 
 import java
-import CodeDuplication
-
-predicate relevant(Method m) {
-  m.getNumberOfLinesOfCode() > 5 and not m.getName().matches("get%")
-  or
-  m.getNumberOfLinesOfCode() > 10
-}
 
 from Method m, Method other
-where
-  duplicateMethod(m, other) and
-  relevant(m) and
-  not fileLevelDuplication(m.getCompilationUnit(), other.getCompilationUnit()) and
-  not classLevelDuplication(m.getDeclaringType(), other.getDeclaringType())
+where none()
 select m, "Method " + m.getName() + " is duplicated in $@.", other,
   other.getDeclaringType().getQualifiedName()

java/ql/src/external/MetricFilter.qll

@@ -1,44 +0,0 @@
-import java
-
-external predicate metricResults(
-  int id, string queryPath, string file, int startline, int startcol, int endline, int endcol,
-  float value
-);
-
-class MetricResult extends int {
-  MetricResult() { metricResults(this, _, _, _, _, _, _, _) }
-
-  string getQueryPath() { metricResults(this, result, _, _, _, _, _, _) }
-
-  File getFile() {
-    exists(string path |
-      metricResults(this, _, path, _, _, _, _, _) and result.getAbsolutePath() = path
-    )
-  }
-
-  int getStartLine() { metricResults(this, _, _, result, _, _, _, _) }
-
-  int getStartColumn() { metricResults(this, _, _, _, result, _, _, _) }
-
-  int getEndLine() { metricResults(this, _, _, _, _, result, _, _) }
-
-  int getEndColumn() { metricResults(this, _, _, _, _, _, result, _) }
-
-  predicate hasMatchingLocation() { exists(this.getMatchingLocation()) }
-
-  Location getMatchingLocation() {
-    result.getFile() = this.getFile() and
-    result.getStartLine() = this.getStartLine() and
-    result.getEndLine() = this.getEndLine() and
-    result.getStartColumn() = this.getStartColumn() and
-    result.getEndColumn() = this.getEndColumn()
-  }
-
-  float getValue() { metricResults(this, _, _, _, _, _, _, result) }
-
-  string getURL() {
-    result =
-      "file://" + getFile().getAbsolutePath() + ":" + getStartLine() + ":" + getStartColumn() + ":" +
-        getEndLine() + ":" + getEndColumn()
-  }
-}

java/ql/src/external/MostlyDuplicateClass.ql

@@ -16,10 +16,7 @@
  */
 
 import java
-import CodeDuplication
 
 from Class c, string message, Class link
-where
-  mostlyDuplicateClass(c, link, message) and
-  not fileLevelDuplication(c.getCompilationUnit(), _)
+where none()
 select c, message, link, link.getQualifiedName()

java/ql/src/external/MostlyDuplicateFile.ql

@@ -16,9 +16,8 @@
  */
 
 import java
-import CodeDuplication
 
 from File f, File other, int percent
-where duplicateFiles(f, other, percent)
+where none()
 select f, percent + "% of the lines in " + f.getStem() + " are copies of lines in $@.", other,
   other.getStem()

java/ql/src/external/MostlyDuplicateMethod.ql

@@ -16,17 +16,8 @@
  */
 
 import java
-import CodeDuplication
 
-from Method m, int covered, int total, Method other, int percent
-where
-  duplicateStatements(m, other, covered, total) and
-  covered != total and
-  m.getMetrics().getNumberOfLinesOfCode() > 5 and
-  covered * 100 / total = percent and
-  percent > 80 and
-  not duplicateMethod(m, other) and
-  not classLevelDuplication(m.getDeclaringType(), other.getDeclaringType()) and
-  not fileLevelDuplication(m.getCompilationUnit(), other.getCompilationUnit())
+from Method m, Method other, int percent
+where none()
 select m, percent + "% of the statements in " + m.getName() + " are duplicated in $@.", other,
   other.getDeclaringType().getName() + "." + other.getStringSignature()

java/ql/src/external/MostlySimilarFile.ql

@@ -16,9 +16,8 @@
  */
 
 import java
-import CodeDuplication
 
 from File f, File other, int percent
-where similarFiles(f, other, percent)
+where none()
 select f, percent + "% of the lines in " + f.getStem() + " are similar to lines in $@.", other,
   other.getStem()

java/ql/src/filters/FromSource.ql

@@ -1,15 +0,0 @@
-/**
- * @name Filter: only keep results from source
- * @description Shows how to filter for only certain files
- * @kind problem
- * @id java/source-filter
- */
-
-import java
-import external.DefectFilter
-
-from DefectResult res, CompilationUnit cu
-where
-  cu = res.getFile() and
-  cu.fromSource()
-select res, res.getMessage()

java/ql/src/filters/NotGenerated.ql

@@ -1,13 +0,0 @@
-/**
- * @name Filter: non-generated files
- * @description Only keep results that aren't in generated files
- * @kind problem
- * @id java/not-generated-file-filter
- */
-
-import java
-import external.DefectFilter
-
-from DefectResult res
-where not res.getFile() instanceof GeneratedFile
-select res, res.getMessage()

java/ql/src/filters/NotGeneratedForMetric.ql

@@ -1,13 +0,0 @@
-/**
- * @name Metric Filter: non-generated files
- * @description Only keep metric results that aren't in generated files
- * @kind treemap
- * @id java/not-generated-file-metric-filter
- */
-
-import java
-import external.MetricFilter
-
-from MetricResult res
-where not res.getFile() instanceof GeneratedFile
-select res, res.getValue()

java/ql/src/filters/SuppressionComment.ql

@@ -1,52 +0,0 @@
-/**
- * @name Filter: Suppression comments
- * @description Recognise comments containing `NOSEMMLE` as suppression comments
- *              when they appear on a line containing an alert or the
- *              immediately preceding line. As further customisations,
- *              `NOSEMMLE(some text)` will only suppress alerts where the
- *              message contains "some text", and `NOSEMMLE/some regex/` will
- *              only suppress alerts where the message contains a match of the
- *              regex. No special way of escaping `)` or `/` in the suppression
- *              comment argument is provided.
- * @kind problem
- * @id java/nosemmle-suppression-comment-filter
- */
-
-import java
-import external.DefectFilter
-
-class SuppressionComment extends Javadoc {
-  SuppressionComment() { this.getAChild*().getText().matches("%NOSEMMLE%") }
-
-  private string getASuppressionDirective() {
-    result = this.getAChild*().getText().regexpFind("\\bNOSEMMLE\\b(\\([^)]+?\\)|/[^/]+?/|)", _, 0)
-  }
-
-  private string getAnActualSubstringArg() {
-    result = this.getASuppressionDirective().regexpCapture("NOSEMMLE\\((.*)\\)", 1)
-  }
-
-  private string getAnActualRegexArg() {
-    result = ".*" + this.getASuppressionDirective().regexpCapture("NOSEMMLE/(.*)/", 1) + ".*"
-  }
-
-  private string getASuppressionRegex() {
-    result = getAnActualRegexArg()
-    or
-    exists(string substring | substring = getAnActualSubstringArg() |
-      result = "\\Q" + substring.replaceAll("\\E", "\\E\\\\E\\Q") + "\\E"
-    )
-    or
-    result = ".*" and getASuppressionDirective() = "NOSEMMLE"
-  }
-
-  predicate suppresses(DefectResult res) {
-    this.getFile() = res.getFile() and
-    res.getEndLine() - this.getLocation().getEndLine() in [0 .. 2] and
-    res.getMessage().regexpMatch(this.getASuppressionRegex())
-  }
-}
-
-from DefectResult res
-where not exists(SuppressionComment s | s.suppresses(res))
-select res, res.getMessage()

java/ql/src/meta/frameworks/Coverage.ql

@@ -0,0 +1,14 @@
+/**
+ * @name Framework coverage
+ * @description The number of API endpoints covered by CSV models sorted by
+ *              package and source-, sink-, and summary-kind.
+ * @kind table
+ * @id java/meta/framework-coverage
+ */
+
+import java
+import semmle.code.java.dataflow.ExternalFlow
+
+from string package, int pkgs, string kind, string part, int n
+where modelCoverage(package, pkgs, kind, part, n)
+select package, pkgs, kind, part, n

java/ql/src/semmle/code/java/dataflow/ExternalFlow.qll

@@ -32,26 +32,30 @@
  * 7. The `input` column specifies how data enters the element selected by the
  *    first 6 columns, and the `output` column specifies how data leaves the
  *    element selected by the first 6 columns. An `input` can be either "",
- *    "Argument", "Argument[n]", "ReturnValue":
+ *    "Argument[n]", "Argument[n1..n2]", "ReturnValue":
  *    - "": Selects a write to the selected element in case this is a field.
- *    - "Argument": Selects any argument in a call to the selected element.
- *    - "Argument[n]": Similar to "Argument" but restricted to a specific numbered
- *      argument (zero-indexed, and `-1` specifies the qualifier).
+ *    - "Argument[n]": Selects an argument in a call to the selected element.
+ *      The arguments are zero-indexed, and `-1` specifies the qualifier.
+ *    - "Argument[n1..n2]": Similar to "Argument[n]" but select any argument in
+ *      the given range. The range is inclusive at both ends.
  *    - "ReturnValue": Selects a value being returned by the selected element.
  *      This requires that the selected element is a method with a body.
  *
- *    An `output` can be either "", "Argument", "Argument[n]", "Parameter",
- *    "Parameter[n]", or "ReturnValue":
+ *    An `output` can be either "", "Argument[n]", "Argument[n1..n2]", "Parameter",
+ *    "Parameter[n]", "Parameter[n1..n2]", or "ReturnValue":
  *    - "": Selects a read of a selected field, or a selected parameter.
- *    - "Argument": Selects the post-update value of an argument in a call to the
+ *    - "Argument[n]": Selects the post-update value of an argument in a call to the
  *      selected element. That is, the value of the argument after the call returns.
- *    - "Argument[n]": Similar to "Argument" but restricted to a specific numbered
- *      argument (zero-indexed, and `-1` specifies the qualifier).
+ *      The arguments are zero-indexed, and `-1` specifies the qualifier.
+ *    - "Argument[n1..n2]": Similar to "Argument[n]" but select any argument in
+ *      the given range. The range is inclusive at both ends.
  *    - "Parameter": Selects the value of a parameter of the selected element.
  *      "Parameter" is also allowed in case the selected element is already a
  *      parameter itself.
  *    - "Parameter[n]": Similar to "Parameter" but restricted to a specific
  *      numbered parameter (zero-indexed, and `-1` specifies the value of `this`).
+ *    - "Parameter[n1..n2]": Similar to "Parameter[n]" but selects any parameter
+ *      in the given range. The range is inclusive at both ends.
  *    - "ReturnValue": Selects the return value of a call to the selected element.
  * 8. The `kind` column is a tag that can be referenced from QL to determine to
  *    which classes the interpreted elements should be added. For example, for
@@ -368,6 +372,60 @@ private predicate summaryModel(
   )
 }
 
+private predicate relevantPackage(string package) {
+  sourceModel(package, _, _, _, _, _, _, _) or
+  sinkModel(package, _, _, _, _, _, _, _) or
+  summaryModel(package, _, _, _, _, _, _, _, _)
+}
+
+private predicate packageLink(string shortpkg, string longpkg) {
+  relevantPackage(shortpkg) and
+  relevantPackage(longpkg) and
+  longpkg.prefix(longpkg.indexOf(".")) = shortpkg
+}
+
+private predicate canonicalPackage(string package) {
+  relevantPackage(package) and not packageLink(_, package)
+}
+
+private predicate canonicalPkgLink(string package, string subpkg) {
+  canonicalPackage(package) and
+  (subpkg = package or packageLink(package, subpkg))
+}
+
+/**
+ * Holds if CSV framework coverage of `package` is `n` api endpoints of the
+ * kind `(kind, part)`.
+ */
+predicate modelCoverage(string package, int pkgs, string kind, string part, int n) {
+  pkgs = strictcount(string subpkg | canonicalPkgLink(package, subpkg)) and
+  (
+    part = "source" and
+    n =
+      strictcount(string subpkg, string type, boolean subtypes, string name, string signature,
+        string ext, string output |
+        canonicalPkgLink(package, subpkg) and
+        sourceModel(subpkg, type, subtypes, name, signature, ext, output, kind)
+      )
+    or
+    part = "sink" and
+    n =
+      strictcount(string subpkg, string type, boolean subtypes, string name, string signature,
+        string ext, string input |
+        canonicalPkgLink(package, subpkg) and
+        sinkModel(subpkg, type, subtypes, name, signature, ext, input, kind)
+      )
+    or
+    part = "summary" and
+    n =
+      strictcount(string subpkg, string type, boolean subtypes, string name, string signature,
+        string ext, string input, string output |
+        canonicalPkgLink(package, subpkg) and
+        summaryModel(subpkg, type, subtypes, name, signature, ext, input, output, kind)
+      )
+  )
+}
+
 /** Provides a query predicate to check the CSV data for validation errors. */
 module CsvValidation {
   /** Holds if some row in a CSV-based flow model appears to contain typos. */
@@ -554,11 +612,29 @@ private string getLast(string s) {
 }
 
 private predicate parseParam(string c, int n) {
-  specSplit(_, c, _) and c.regexpCapture("Parameter\\[([-0-9]+)\\]", 1).toInt() = n
+  specSplit(_, c, _) and
+  (
+    c.regexpCapture("Parameter\\[([-0-9]+)\\]", 1).toInt() = n
+    or
+    exists(int n1, int n2 |
+      c.regexpCapture("Parameter\\[([-0-9]+)\\.\\.([0-9]+)\\]", 1).toInt() = n1 and
+      c.regexpCapture("Parameter\\[([-0-9]+)\\.\\.([0-9]+)\\]", 2).toInt() = n2 and
+      n = [n1 .. n2]
+    )
+  )
 }
 
 private predicate parseArg(string c, int n) {
-  specSplit(_, c, _) and c.regexpCapture("Argument\\[([-0-9]+)\\]", 1).toInt() = n
+  specSplit(_, c, _) and
+  (
+    c.regexpCapture("Argument\\[([-0-9]+)\\]", 1).toInt() = n
+    or
+    exists(int n1, int n2 |
+      c.regexpCapture("Argument\\[([-0-9]+)\\.\\.([0-9]+)\\]", 1).toInt() = n1 and
+      c.regexpCapture("Argument\\[([-0-9]+)\\.\\.([0-9]+)\\]", 2).toInt() = n2 and
+      n = [n1 .. n2]
+    )
+  )
 }
 
 private predicate inputNeedsReference(string c) {

java/ql/src/semmle/code/java/dataflow/internal/TaintTrackingUtil.qll

@@ -67,8 +67,8 @@ private predicate localAdditionalBasicTaintStep(DataFlow::Node src, DataFlow::No
  * Holds if an additional step from `src` to `sink` through a call can be inferred from the
  * combination of a value-preserving step providing an alias between an input and the output
  * and a taint step from `src` to one the aliased nodes. For example, if we know that `f(a, b)` returns
- * the exact value of `a` and also propagates taint from `b` to its result, then we also know that
- * `a` is tainted after `f` completes, and vice versa.
+ * the exact value of `a` and also propagates taint from `b` to `a`, then we also know that
+ * the return value is tainted after `f` completes.
  */
 private predicate composedValueAndTaintModelStep(ArgumentNode src, DataFlow::Node sink) {
   exists(Call call, ArgumentNode valueSource, DataFlow::PostUpdateNode valueSourcePost |
@@ -76,16 +76,10 @@ private predicate composedValueAndTaintModelStep(ArgumentNode src, DataFlow::Nod
     valueSource.argumentOf(call, _) and
     src != valueSource and
     valueSourcePost.getPreUpdateNode() = valueSource and
+    // in-x -value-> out-y and in-z -taint-> in-x ==> in-z -taint-> out-y
+    localAdditionalBasicTaintStep(src, valueSourcePost) and
     DataFlow::localFlowStep(valueSource, DataFlow::exprNode(call)) and
-    (
-      // in-x -value-> out-y and in-z -taint-> out-y ==> in-z -taint-> in-x
-      localAdditionalBasicTaintStep(src, DataFlow::exprNode(call)) and
-      sink = valueSourcePost
-      or
-      // in-x -value-> out-y and in-z -taint-> in-x ==> in-z -taint-> out-y
-      localAdditionalBasicTaintStep(src, valueSourcePost) and
-      sink = DataFlow::exprNode(call)
-    )
+    sink = DataFlow::exprNode(call)
   )
 }
 

java/ql/test/experimental/query-tests/security/CWE-502/SpringExporterUnsafeDeserialization.java

@@ -0,0 +1,79 @@
+import org.springframework.boot.SpringBootConfiguration;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.remoting.httpinvoker.HttpInvokerServiceExporter;
+import org.springframework.remoting.rmi.RemoteInvocationSerializingExporter;
+
+@Configuration
+public class SpringExporterUnsafeDeserialization {
+
+    @Bean(name = "/unsafeHttpInvokerServiceExporter")
+    HttpInvokerServiceExporter unsafeHttpInvokerServiceExporter() {
+        HttpInvokerServiceExporter exporter = new HttpInvokerServiceExporter();
+        exporter.setService(new AccountServiceImpl());
+        exporter.setServiceInterface(AccountService.class);
+        return exporter;
+    }
+
+    @Bean(name = "/unsafeCustomeRemoteInvocationSerializingExporter")
+    RemoteInvocationSerializingExporter unsafeCustomeRemoteInvocationSerializingExporter() {
+        return new CustomeRemoteInvocationSerializingExporter();
+    }
+
+    HttpInvokerServiceExporter notABean() {
+        HttpInvokerServiceExporter exporter = new HttpInvokerServiceExporter();
+        exporter.setService(new AccountServiceImpl());
+        exporter.setServiceInterface(AccountService.class);
+        return exporter;
+    }
+}
+
+@SpringBootApplication
+class SpringBootTestApplication {
+
+    @Bean(name = "/unsafeHttpInvokerServiceExporter")
+    HttpInvokerServiceExporter unsafeHttpInvokerServiceExporter() {
+        HttpInvokerServiceExporter exporter = new HttpInvokerServiceExporter();
+        exporter.setService(new AccountServiceImpl());
+        exporter.setServiceInterface(AccountService.class);
+        return exporter;
+    }
+}
+
+@SpringBootConfiguration
+class SpringBootTestConfiguration {
+
+    @Bean(name = "/unsafeHttpInvokerServiceExporter")
+    HttpInvokerServiceExporter unsafeHttpInvokerServiceExporter() {
+        HttpInvokerServiceExporter exporter = new HttpInvokerServiceExporter();
+        exporter.setService(new AccountServiceImpl());
+        exporter.setServiceInterface(AccountService.class);
+        return exporter;
+    }
+}
+
+class CustomeRemoteInvocationSerializingExporter extends RemoteInvocationSerializingExporter {}
+
+class NotAConfiguration {
+
+    @Bean(name = "/notAnEndpoint")
+    HttpInvokerServiceExporter notAnEndpoint() {
+        HttpInvokerServiceExporter exporter = new HttpInvokerServiceExporter();
+        exporter.setService(new AccountServiceImpl());
+        exporter.setServiceInterface(AccountService.class);
+        return exporter;
+    }
+}
+
+class AccountServiceImpl implements AccountService {
+
+    @Override
+    public String echo(String data) {
+        return data;
+    }
+}
+
+interface AccountService {
+    String echo(String data);
+}

java/ql/test/experimental/query-tests/security/CWE-502/UnsafeSpringExporterInConfigurationClass.expected

@@ -0,0 +1,4 @@
+| SpringExporterUnsafeDeserialization.java:12:32:12:63 | unsafeHttpInvokerServiceExporter | Unsafe deserialization in a Spring exporter bean '/unsafeHttpInvokerServiceExporter' |
+| SpringExporterUnsafeDeserialization.java:20:41:20:88 | unsafeCustomeRemoteInvocationSerializingExporter | Unsafe deserialization in a Spring exporter bean '/unsafeCustomeRemoteInvocationSerializingExporter' |
+| SpringExporterUnsafeDeserialization.java:36:32:36:63 | unsafeHttpInvokerServiceExporter | Unsafe deserialization in a Spring exporter bean '/unsafeHttpInvokerServiceExporter' |
+| SpringExporterUnsafeDeserialization.java:48:32:48:63 | unsafeHttpInvokerServiceExporter | Unsafe deserialization in a Spring exporter bean '/unsafeHttpInvokerServiceExporter' |

java/ql/test/experimental/query-tests/security/CWE-502/UnsafeSpringExporterInConfigurationClass.qlref

@@ -0,0 +1 @@
+experimental/Security/CWE/CWE-502/UnsafeSpringExporterInConfigurationClass.ql

java/ql/test/experimental/query-tests/security/CWE-502/UnsafeSpringExporterInXMLConfiguration.expected

@@ -0,0 +1,2 @@
+| beans.xml:10:5:13:12 | /unsafeBooking | Unsafe deserialization in a Spring exporter bean '/unsafeBooking' |
+| beans.xml:15:5:18:12 | org.springframework.remoting.httpinvoker.HttpInvokerServiceExporter | Unsafe deserialization in a Spring exporter bean 'org.springframework.remoting.httpinvoker.HttpInvokerServiceExporter' |

java/ql/test/experimental/query-tests/security/CWE-502/UnsafeSpringExporterInXMLConfiguration.qlref

@@ -0,0 +1 @@
+experimental/Security/CWE/CWE-502/UnsafeSpringExporterInXMLConfiguration.ql

java/ql/test/experimental/query-tests/security/CWE-502/beans.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<beans xmlns="http://www.springframework.org/schema/beans"
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xmlns:context="http://www.springframework.org/schema/context"
+       xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
+	http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd">
+
+    <bean id="anotherBookingService" class="com.gypsyengineer.server.CabBookingServiceImpl"/>
+
+    <bean name="/unsafeBooking" class="org.springframework.remoting.httpinvoker.HttpInvokerServiceExporter">
+        <property name="service" ref="anotherBookingService"/>
+        <property name="serviceInterface" value="com.gypsyengineer.api.CabBookingService"/>
+    </bean>
+
+    <bean class="org.springframework.remoting.httpinvoker.HttpInvokerServiceExporter">
+        <property name="service" ref="anotherBookingService"/>
+        <property name="serviceInterface" value="com.gypsyengineer.api.CabBookingService"/>
+    </bean>
+</beans>

java/ql/test/experimental/query-tests/security/CWE-502/options

@@ -0,0 +1 @@
+//semmle-extractor-options: --javac-args -cp ${testdir}/../../../../stubs/springframework-5.2.3

java/ql/test/stubs/springframework-5.2.3/org/springframework/boot/SpringBootConfiguration.java

@@ -0,0 +1,10 @@
+package org.springframework.boot;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Target;
+
+import org.springframework.context.annotation.Configuration;
+
+@Target(ElementType.TYPE)
+@Configuration
+public @interface SpringBootConfiguration {}

java/ql/test/stubs/springframework-5.2.3/org/springframework/boot/autoconfigure/SpringBootApplication.java

@@ -0,0 +1,12 @@
+package org.springframework.boot.autoconfigure;
+
+import java.lang.annotation.Target;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+
+import org.springframework.boot.SpringBootConfiguration;
+
+@Target(ElementType.TYPE)
+@Inherited
+@SpringBootConfiguration
+public @interface SpringBootApplication {}

java/ql/test/stubs/springframework-5.2.3/org/springframework/context/annotation/Bean.java

@@ -0,0 +1,10 @@
+package org.springframework.context.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Target;
+
+@Target({ElementType.METHOD, ElementType.ANNOTATION_TYPE})
+public @interface Bean {
+
+	String[] name() default {};
+}

java/ql/test/stubs/springframework-5.2.3/org/springframework/context/annotation/Configuration.java

@@ -0,0 +1,7 @@
+package org.springframework.context.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Target;
+
+@Target(ElementType.TYPE)
+public @interface Configuration {}

java/ql/test/stubs/springframework-5.2.3/org/springframework/remoting/httpinvoker/HttpInvokerServiceExporter.java

@@ -0,0 +1,8 @@
+package org.springframework.remoting.httpinvoker;
+
+public class HttpInvokerServiceExporter extends org.springframework.remoting.rmi.RemoteInvocationSerializingExporter {
+
+    public void setService(Object service) {}
+
+    public void setServiceInterface(Class clazz) {}
+}

java/ql/test/stubs/springframework-5.2.3/org/springframework/remoting/rmi/RemoteInvocationSerializingExporter.java

@@ -0,0 +1,3 @@
+package org.springframework.remoting.rmi;
+
+public abstract class RemoteInvocationSerializingExporter {}

javascript/change-notes/2021-03-19-async-execute.md

@@ -0,0 +1,4 @@
+lgtm,codescanning
+* The command injection security queries now recognize additional sinks.
+  Affected packages are
+    [async-execute](https://npmjs.com/package/async-execute)

javascript/change-notes/2021-03-23-accessor-calls.md

@@ -0,0 +1,3 @@
+lgtm,codescanning
+* Calls to property accessors are now analyzed on par with regular function calls,
+  leading to more results from queries that rely on data flow.

javascript/ql/src/Security/CWE-078/UnsafeShellCommandConstruction.ql

@@ -18,6 +18,6 @@ import DataFlow::PathGraph
 
 from Configuration cfg, DataFlow::PathNode source, DataFlow::PathNode sink, Sink sinkNode
 where cfg.hasFlowPath(source, sink) and sinkNode = sink.getNode()
-select sinkNode.getAlertLocation(), source, sink, "$@ based on library input is later used in $@.",
-  sinkNode.getAlertLocation(), sinkNode.getSinkType(), sinkNode.getCommandExecution(),
-  "shell command"
+select sinkNode.getAlertLocation(), source, sink, "$@ based on $@ is later used in $@.",
+  sinkNode.getAlertLocation(), sinkNode.getSinkType(), source.getNode(), "library input",
+  sinkNode.getCommandExecution(), "shell command"

javascript/ql/src/Security/CWE-327/BadRandomness.ql

@@ -87,11 +87,6 @@ private DataFlow::Node goodRandom(DataFlow::TypeTracker t, DataFlow::SourceNode
   or
   exists(DataFlow::TypeTracker t2 | t = t2.smallstep(goodRandom(t2, source), result))
   or
-  // re-using the collection steps for `Set`.
-  exists(DataFlow::TypeTracker t2 |
-    result = CollectionsTypeTracking::collectionStep(goodRandom(t2, source), t, t2)
-  )
-  or
   InsecureRandomness::isAdditionalTaintStep(goodRandom(t.continue(), source), result) and
   // bit shifts and multiplication by powers of two are generally used for constructing larger numbers from smaller numbers.
   not exists(BinaryExpr binop | binop = result.asExpr() |

javascript/ql/src/meta/Consistency.ql

@@ -4,7 +4,6 @@
  *              in the documentation, queries that rely on the contract may yield unexpected
  *              results.
  * @kind table
- * @problem.severity error
  * @id js/consistency/api-contracts
  * @tags consistency
  */

javascript/ql/src/meta/alerts/CallGraph.ql

@@ -10,6 +10,9 @@
 
 import javascript
 
-from DataFlow::InvokeNode invoke, Function f
-where invoke.getACallee() = f
-select invoke, "Call to $@", f, f.describe()
+from DataFlow::Node invoke, Function f, string kind
+where
+  invoke.(DataFlow::InvokeNode).getACallee() = f and kind = "Call"
+  or
+  invoke.(DataFlow::PropRef).getAnAccessorCallee().getFunction() = f and kind = "Accessor call"
+select invoke, kind + " to $@", f, f.describe()

javascript/ql/src/meta/analysis-quality/TaintSteps.ql

@@ -15,17 +15,17 @@ predicate relevantStep(DataFlow::Node pred, DataFlow::Node succ) {
   (
     TaintTracking::sharedTaintStep(pred, succ)
     or
-    any(DataFlow::AdditionalFlowStep cfg).step(pred, succ)
+    DataFlow::SharedFlowStep::step(pred, succ)
     or
-    any(DataFlow::AdditionalFlowStep cfg).step(pred, succ, _, _)
+    DataFlow::SharedFlowStep::step(pred, succ, _, _)
     or
-    any(DataFlow::AdditionalFlowStep cfg).loadStep(pred, succ, _)
+    DataFlow::SharedFlowStep::loadStep(pred, succ, _)
     or
-    any(DataFlow::AdditionalFlowStep cfg).storeStep(pred, succ, _)
+    DataFlow::SharedFlowStep::storeStep(pred, succ, _)
     or
-    any(DataFlow::AdditionalFlowStep cfg).loadStoreStep(pred, succ, _, _)
+    DataFlow::SharedFlowStep::loadStoreStep(pred, succ, _, _)
     or
-    any(DataFlow::AdditionalFlowStep cfg).loadStoreStep(pred, succ, _)
+    DataFlow::SharedFlowStep::loadStoreStep(pred, succ, _)
   ) and
   not pred.getFile() instanceof IgnoredFile and
   not succ.getFile() instanceof IgnoredFile

javascript/ql/src/semmle/javascript/ApiGraphs.qll

@@ -619,11 +619,11 @@ module API {
     cached
     predicate use(TApiNode nd, DataFlow::Node ref) {
       exists(string m, Module mod | nd = MkModuleDef(m) and mod = importableModule(m) |
-        ref.(ModuleAsSourceNode).getModule() = mod
+        ref.(ModuleVarNode).getModule() = mod
       )
       or
       exists(string m, Module mod | nd = MkModuleExport(m) and mod = importableModule(m) |
-        ref.(ExportsAsSourceNode).getModule() = mod
+        ref.(ExportsVarNode).getModule() = mod
         or
         exists(DataFlow::Node base | use(MkModuleDef(m), base) |
           ref = trackUseNode(base).getAPropertyRead("exports")
@@ -746,9 +746,9 @@ module API {
       or
       // additional backwards step from `require('m')` to `exports` or `module.exports` in m
       exists(Import imp | imp.getImportedModuleNode() = trackDefNode(nd, t.continue()) |
-        result.(ExportsAsSourceNode).getModule() = imp.getImportedModule()
+        result.(ExportsVarNode).getModule() = imp.getImportedModule()
         or
-        exists(ModuleAsSourceNode mod |
+        exists(ModuleVarNode mod |
           mod.getModule() = imp.getImportedModule() and
           result = mod.(DataFlow::SourceNode).getAPropertyRead("exports")
         )
@@ -983,31 +983,44 @@ private module Label {
   string promised() { result = "promised" }
 }
 
+private class NodeModuleSourcesNodes extends DataFlow::SourceNode::Range {
+  Variable v;
+
+  NodeModuleSourcesNodes() {
+    exists(NodeModule m |
+      this = DataFlow::ssaDefinitionNode(SSA::implicitInit(v)) and
+      v = [m.getModuleVariable(), m.getExportsVariable()]
+    )
+  }
+
+  Variable getVariable() { result = v }
+}
+
 /**
- * A CommonJS/AMD `module` variable, considered as a source node.
+ * A CommonJS/AMD `module` variable.
  */
-private class ModuleAsSourceNode extends DataFlow::SourceNode::Range {
+private class ModuleVarNode extends DataFlow::Node {
   Module m;
 
-  ModuleAsSourceNode() {
-    this = DataFlow::ssaDefinitionNode(SSA::implicitInit(m.(NodeModule).getModuleVariable()))
+  ModuleVarNode() {
+    this.(NodeModuleSourcesNodes).getVariable() = m.(NodeModule).getModuleVariable()
     or
-    this = DataFlow::parameterNode(m.(AmdModule).getDefine().getModuleParameter())
+    DataFlow::parameterNode(this, m.(AmdModule).getDefine().getModuleParameter())
   }
 
   Module getModule() { result = m }
 }
 
 /**
- * A CommonJS/AMD `exports` variable, considered as a source node.
+ * A CommonJS/AMD `exports` variable.
  */
-private class ExportsAsSourceNode extends DataFlow::SourceNode::Range {
+private class ExportsVarNode extends DataFlow::Node {
   Module m;
 
-  ExportsAsSourceNode() {
-    this = DataFlow::ssaDefinitionNode(SSA::implicitInit(m.(NodeModule).getExportsVariable()))
+  ExportsVarNode() {
+    this.(NodeModuleSourcesNodes).getVariable() = m.(NodeModule).getExportsVariable()
     or
-    this = DataFlow::parameterNode(m.(AmdModule).getDefine().getExportsParameter())
+    DataFlow::parameterNode(this, m.(AmdModule).getDefine().getExportsParameter())
   }
 
   Module getModule() { result = m }

javascript/ql/src/semmle/javascript/Arrays.qll

@@ -84,16 +84,17 @@ private module ArrayDataFlow {
    * A step modelling the creation of an Array using the `Array.from(x)` method.
    * The step copies the elements of the argument (set, array, or iterator elements) into the resulting array.
    */
-  private class ArrayFrom extends DataFlow::AdditionalFlowStep, DataFlow::CallNode {
-    ArrayFrom() { this = DataFlow::globalVarRef("Array").getAMemberCall("from") }
-
+  private class ArrayFrom extends DataFlow::SharedFlowStep {
     override predicate loadStoreStep(
       DataFlow::Node pred, DataFlow::Node succ, string fromProp, string toProp
     ) {
-      pred = this.getArgument(0) and
-      succ = this and
-      fromProp = arrayLikeElement() and
-      toProp = arrayElement()
+      exists(DataFlow::CallNode call |
+        call = DataFlow::globalVarRef("Array").getAMemberCall("from") and
+        pred = call.getArgument(0) and
+        succ = call and
+        fromProp = arrayLikeElement() and
+        toProp = arrayElement()
+      )
     }
   }
 
@@ -103,55 +104,45 @@ private module ArrayDataFlow {
    *
    * Such a step can occur both with the `push` and `unshift` methods, or when creating a new array.
    */
-  private class ArrayCopySpread extends DataFlow::AdditionalFlowStep {
-    DataFlow::Node spreadArgument; // the spread argument containing the elements to be copied.
-    DataFlow::Node base; // the object where the elements should be copied to.
-
-    ArrayCopySpread() {
-      exists(DataFlow::MethodCallNode mcn | mcn = this |
-        mcn.getMethodName() = ["push", "unshift"] and
-        spreadArgument = mcn.getASpreadArgument() and
-        base = mcn.getReceiver().getALocalSource()
-      )
-      or
-      spreadArgument = this.(DataFlow::ArrayCreationNode).getASpreadArgument() and
-      base = this
-    }
-
+  private class ArrayCopySpread extends DataFlow::SharedFlowStep {
     override predicate loadStoreStep(
       DataFlow::Node pred, DataFlow::Node succ, string fromProp, string toProp
     ) {
-      pred = spreadArgument and
-      succ = base and
       fromProp = arrayLikeElement() and
-      toProp = arrayElement()
+      toProp = arrayElement() and
+      (
+        exists(DataFlow::MethodCallNode mcn |
+          mcn.getMethodName() = ["push", "unshift"] and
+          pred = mcn.getASpreadArgument() and
+          succ = mcn.getReceiver().getALocalSource()
+        )
+        or
+        pred = succ.(DataFlow::ArrayCreationNode).getASpreadArgument()
+      )
     }
   }
 
   /**
    * A step for storing an element on an array using `arr.push(e)` or `arr.unshift(e)`.
    */
-  private class ArrayAppendStep extends DataFlow::AdditionalFlowStep, DataFlow::MethodCallNode {
-    ArrayAppendStep() {
-      this.getMethodName() = "push" or
-      this.getMethodName() = "unshift"
-    }
-
+  private class ArrayAppendStep extends DataFlow::SharedFlowStep {
     override predicate storeStep(DataFlow::Node element, DataFlow::SourceNode obj, string prop) {
       prop = arrayElement() and
-      element = this.getAnArgument() and
-      obj.getAMethodCall() = this
+      exists(DataFlow::MethodCallNode call |
+        call.getMethodName() = ["push", "unshift"] and
+        element = call.getAnArgument() and
+        obj.getAMethodCall() = call
+      )
     }
   }
 
   /**
-   * A step for reading/writing an element from an array inside a for-loop.
-   * E.g. a read from `foo[i]` to `bar` in `for(var i = 0; i < arr.length; i++) {bar = foo[i]}`.
+   * A node that reads or writes an element from an array inside a for-loop.
    */
-  private class ArrayIndexingStep extends DataFlow::AdditionalFlowStep, DataFlow::Node {
+  private class ArrayIndexingAccess extends DataFlow::Node {
     DataFlow::PropRef read;
 
-    ArrayIndexingStep() {
+    ArrayIndexingAccess() {
       read = this and
       TTNumber() =
         unique(InferredType type | type = read.getPropertyNameExpr().flow().analyze().getAType()) and
@@ -162,17 +153,27 @@ private module ArrayDataFlow {
         i.getVariable().getADefinition().(VariableDeclarator).getDeclStmt() = init
       )
     }
+  }
 
+  /**
+   * A step for reading/writing an element from an array inside a for-loop.
+   * E.g. a read from `foo[i]` to `bar` in `for(var i = 0; i < arr.length; i++) {bar = foo[i]}`.
+   */
+  private class ArrayIndexingStep extends DataFlow::SharedFlowStep {
     override predicate loadStep(DataFlow::Node obj, DataFlow::Node element, string prop) {
-      prop = arrayElement() and
-      obj = this.(DataFlow::PropRead).getBase() and
-      element = this
+      exists(ArrayIndexingAccess access |
+        prop = arrayElement() and
+        obj = access.(DataFlow::PropRead).getBase() and
+        element = access
+      )
     }
 
     override predicate storeStep(DataFlow::Node element, DataFlow::SourceNode obj, string prop) {
-      prop = arrayElement() and
-      element = this.(DataFlow::PropWrite).getRhs() and
-      this = obj.getAPropertyWrite()
+      exists(ArrayIndexingAccess access |
+        prop = arrayElement() and
+        element = access.(DataFlow::PropWrite).getRhs() and
+        access = obj.getAPropertyWrite()
+      )
     }
   }
 
@@ -180,16 +181,14 @@ private module ArrayDataFlow {
    * A step for retrieving an element from an array using `.pop()` or `.shift()`.
    * E.g. `array.pop()`.
    */
-  private class ArrayPopStep extends DataFlow::AdditionalFlowStep, DataFlow::MethodCallNode {
-    ArrayPopStep() {
-      getMethodName() = "pop" or
-      getMethodName() = "shift"
-    }
-
+  private class ArrayPopStep extends DataFlow::SharedFlowStep {
     override predicate loadStep(DataFlow::Node obj, DataFlow::Node element, string prop) {
-      prop = arrayElement() and
-      obj = this.getReceiver() and
-      element = this
+      exists(DataFlow::MethodCallNode call |
+        call.getMethodName() = ["pop", "shift"] and
+        prop = arrayElement() and
+        obj = call.getReceiver() and
+        element = call
+      )
     }
   }
 
@@ -235,12 +234,12 @@ private module ArrayDataFlow {
   /**
    * A step for creating an array and storing the elements in the array.
    */
-  private class ArrayCreationStep extends DataFlow::AdditionalFlowStep, DataFlow::ArrayCreationNode {
+  private class ArrayCreationStep extends DataFlow::SharedFlowStep {
     override predicate storeStep(DataFlow::Node element, DataFlow::SourceNode obj, string prop) {
-      exists(int i |
-        element = this.getElement(i) and
-        obj = this and
-        if this = any(PromiseAllCreation c).getArrayNode()
+      exists(DataFlow::ArrayCreationNode array, int i |
+        element = array.getElement(i) and
+        obj = array and
+        if array = any(PromiseAllCreation c).getArrayNode()
         then prop = arrayElement(i)
         else prop = arrayElement()
       )
@@ -251,13 +250,14 @@ private module ArrayDataFlow {
    * A step modelling that `splice` can insert elements into an array.
    * For example in `array.splice(i, del, e)`: if `e` is tainted, then so is `array
    */
-  private class ArraySpliceStep extends DataFlow::AdditionalFlowStep, DataFlow::MethodCallNode {
-    ArraySpliceStep() { this.getMethodName() = "splice" }
-
+  private class ArraySpliceStep extends DataFlow::SharedFlowStep {
     override predicate storeStep(DataFlow::Node element, DataFlow::SourceNode obj, string prop) {
-      prop = arrayElement() and
-      element = getArgument(2) and
-      this = obj.getAMethodCall()
+      exists(DataFlow::MethodCallNode call |
+        call.getMethodName() = "splice" and
+        prop = arrayElement() and
+        element = call.getArgument(2) and
+        call = obj.getAMethodCall()
+      )
     }
   }
 
@@ -265,42 +265,27 @@ private module ArrayDataFlow {
    * A step for modelling `concat`.
    * For example in `e = arr1.concat(arr2, arr3)`: if any of the `arr` is tainted, then so is `e`.
    */
-  private class ArrayConcatStep extends DataFlow::AdditionalFlowStep, DataFlow::MethodCallNode {
-    ArrayConcatStep() { this.getMethodName() = "concat" }
-
+  private class ArrayConcatStep extends DataFlow::SharedFlowStep {
     override predicate loadStoreStep(DataFlow::Node pred, DataFlow::Node succ, string prop) {
-      prop = arrayElement() and
-      (pred = this.getReceiver() or pred = this.getAnArgument()) and
-      succ = this
+      exists(DataFlow::MethodCallNode call |
+        call.getMethodName() = "concat" and
+        prop = arrayElement() and
+        (pred = call.getReceiver() or pred = call.getAnArgument()) and
+        succ = call
+      )
     }
   }
 
   /**
    * A step for modelling that elements from an array `arr` also appear in the result from calling `slice`/`splice`/`filter`.
    */
-  private class ArraySliceStep extends DataFlow::AdditionalFlowStep, DataFlow::MethodCallNode {
-    ArraySliceStep() {
-      this.getMethodName() = "slice" or
-      this.getMethodName() = "splice" or
-      this.getMethodName() = "filter"
-    }
-
+  private class ArraySliceStep extends DataFlow::SharedFlowStep {
     override predicate loadStoreStep(DataFlow::Node pred, DataFlow::Node succ, string prop) {
-      prop = arrayElement() and
-      pred = this.getReceiver() and
-      succ = this
-    }
-  }
-
-  /**
-   * A step for modelling `for of` iteration on arrays.
-   */
-  private class ForOfStep extends PreCallGraphStep {
-    override predicate loadStep(DataFlow::Node obj, DataFlow::Node e, string prop) {
-      exists(ForOfStmt forOf |
-        obj = forOf.getIterationDomain().flow() and
-        e = DataFlow::lvalueNode(forOf.getLValue()) and
-        prop = arrayElement()
+      exists(DataFlow::MethodCallNode call |
+        call.getMethodName() = ["slice", "splice", "filter"] and
+        prop = arrayElement() and
+        pred = call.getReceiver() and
+        succ = call
       )
     }
   }