Merge branch 'main' into mathiasvp/replace-ast-with-ir-use-usedataflow

.github/workflows/ql-for-ql-build.yml

@@ -27,7 +27,7 @@ jobs:
         uses: ./.github/actions/find-latest-bundle
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@45955cb1830b640e2c1603ad72ad542a49d47b96
+        uses: github/codeql-action/init@beae46e6b1da530ed5e9fc6a756f92433ca47ae1
         with:
           languages: javascript # does not matter
           tools: ${{ steps.find-latest-bundle.outputs.url }}
@@ -139,7 +139,7 @@ jobs:
         env:
           CONF: ./ql-for-ql-config.yml
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@45955cb1830b640e2c1603ad72ad542a49d47b96
+        uses: github/codeql-action/init@beae46e6b1da530ed5e9fc6a756f92433ca47ae1
         with:
           languages: ql
           db-location: ${{ runner.temp }}/db
@@ -152,7 +152,7 @@ jobs:
           PACK: ${{ runner.temp }}/pack
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@45955cb1830b640e2c1603ad72ad542a49d47b96
+        uses: github/codeql-action/analyze@beae46e6b1da530ed5e9fc6a756f92433ca47ae1
         with:
           category: "ql-for-ql"
       - name: Copy sarif file to CWD

.github/workflows/ql-for-ql-dataset_measure.yml

@@ -25,7 +25,7 @@ jobs:
 
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@45955cb1830b640e2c1603ad72ad542a49d47b96
+        uses: github/codeql-action/init@beae46e6b1da530ed5e9fc6a756f92433ca47ae1
         with:
           languages: javascript # does not matter
       - uses: ./.github/actions/os-version

.github/workflows/ql-for-ql-tests.yml

@@ -6,11 +6,13 @@ on:
     paths:
       - "ql/**"
       - codeql-workspace.yml
+      - .github/workflows/ql-for-ql-tests.yml
   pull_request:
     branches: [main]
     paths:
       - "ql/**"
       - codeql-workspace.yml
+      - .github/workflows/ql-for-ql-tests.yml
 
 env:
   CARGO_TERM_COLOR: always
@@ -22,7 +24,7 @@ jobs:
       - uses: actions/checkout@v3
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@45955cb1830b640e2c1603ad72ad542a49d47b96
+        uses: github/codeql-action/init@beae46e6b1da530ed5e9fc6a756f92433ca47ae1
         with:
           languages: javascript # does not matter
       - uses: ./.github/actions/os-version
@@ -65,7 +67,7 @@ jobs:
           echo "/usr/local/opt/gnu-tar/libexec/gnubin" >> $GITHUB_PATH
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@77a8d2d10c0b403a8b4aadbd223dc489ecd22683
+        uses: github/codeql-action/init@beae46e6b1da530ed5e9fc6a756f92433ca47ae1
         with:
           languages: javascript # does not matter
       - uses: ./.github/actions/os-version

cpp/ql/lib/experimental/semmle/code/cpp/rangeanalysis/extensions/ConstantShiftExprRange.qll

@@ -0,0 +1,263 @@
+private import cpp
+private import experimental.semmle.code.cpp.models.interfaces.SimpleRangeAnalysisExpr
+private import semmle.code.cpp.rangeanalysis.RangeAnalysisUtils
+
+float evaluateConstantExpr(Expr e) {
+  result = e.getValue().toFloat()
+  or
+  // This handles when a constant value is put into a variable
+  // and the variable is used later
+  exists(SsaDefinition defn, StackVariable sv |
+    defn.getAUse(sv) = e and
+    result = defn.getDefiningValue(sv).getValue().toFloat()
+  )
+}
+
+// If the constant right operand is negative or is greater than or equal to the number of
+// bits in the left operands type, then the result is undefined (except on the IA-32
+// architecture where the shift value is masked with 0b00011111, but we can't
+// assume the architecture).
+bindingset[val]
+private predicate isValidShiftExprShift(float val, Expr l) {
+  val >= 0 and
+  // We use getFullyConverted because the spec says to use the *promoted* left operand
+  val < (l.getFullyConverted().getUnderlyingType().getSize() * 8)
+}
+
+bindingset[val, shift, max_val]
+private predicate canLShiftOverflow(int val, int shift, int max_val) {
+  // val << shift = val * 2^shift > max_val => val > max_val/2^shift = max_val >> b
+  val > max_val.bitShiftRight(shift)
+}
+
+/**
+ * A range analysis expression consisting of the `>>` or `>>=` operator when at least
+ * one operand is a constant (and if the right operand is a constant, it must be "valid"
+ * (see `isValidShiftExprShift`)). When handling any undefined behavior, it leaves the
+ * values unconstrained. From the C++ standard: "The behavior is undefined if the right
+ * operand is negative, or greater than or equal to the length in bits of the promoted
+ * left operand. The value of E1 >> E2 is E1 right-shifted E2 bit positions. If E1 has an
+ * unsigned type or if E1 has a signed type and a non-negative value, the value of the
+ * result is the integral part of the quotient of E1/2^E2. If E1 has a signed type and a
+ * negative value, the resulting value is implementation-defined."
+ */
+class ConstantRShiftExprRange extends SimpleRangeAnalysisExpr {
+  /**
+   * Holds for `a >> b` or `a >>= b` in one of the following two cases:
+   * 1. `a` is a constant and `b` is not
+   * 2. `b` is constant
+   *
+   * We don't handle the case where `a` and `b` are both non-constant values.
+   */
+  ConstantRShiftExprRange() {
+    getUnspecifiedType() instanceof IntegralType and
+    exists(Expr l, Expr r |
+      l = this.(RShiftExpr).getLeftOperand() and
+      r = this.(RShiftExpr).getRightOperand()
+      or
+      l = this.(AssignRShiftExpr).getLValue() and
+      r = this.(AssignRShiftExpr).getRValue()
+    |
+      l.getUnspecifiedType() instanceof IntegralType and
+      r.getUnspecifiedType() instanceof IntegralType and
+      (
+        // If the left operand is a constant, verify that the right operand is not a constant
+        exists(evaluateConstantExpr(l)) and not exists(evaluateConstantExpr(r))
+        or
+        // If the right operand is a constant, check if it is a valid shift expression
+        exists(float constROp |
+          constROp = evaluateConstantExpr(r) and isValidShiftExprShift(constROp, l)
+        )
+      )
+    )
+  }
+
+  Expr getLeftOperand() {
+    result = this.(RShiftExpr).getLeftOperand() or
+    result = this.(AssignRShiftExpr).getLValue()
+  }
+
+  Expr getRightOperand() {
+    result = this.(RShiftExpr).getRightOperand() or
+    result = this.(AssignRShiftExpr).getRValue()
+  }
+
+  override float getLowerBounds() {
+    exists(int lLower, int lUpper, int rLower, int rUpper |
+      lLower = getFullyConvertedLowerBounds(getLeftOperand()) and
+      lUpper = getFullyConvertedUpperBounds(getLeftOperand()) and
+      rLower = getFullyConvertedLowerBounds(getRightOperand()) and
+      rUpper = getFullyConvertedUpperBounds(getRightOperand()) and
+      lLower <= lUpper and
+      rLower <= rUpper
+    |
+      if
+        lLower < 0
+        or
+        not (
+          isValidShiftExprShift(rLower, getLeftOperand()) and
+          isValidShiftExprShift(rUpper, getLeftOperand())
+        )
+      then
+        // We don't want to deal with shifting negative numbers at the moment,
+        // and a negative shift is implementation defined, so we set the result
+        // to the minimum value
+        result = exprMinVal(this)
+      else
+        // We can get the smallest value by shifting the smallest bound by the largest bound
+        result = lLower.bitShiftRight(rUpper)
+    )
+  }
+
+  override float getUpperBounds() {
+    exists(int lLower, int lUpper, int rLower, int rUpper |
+      lLower = getFullyConvertedLowerBounds(getLeftOperand()) and
+      lUpper = getFullyConvertedUpperBounds(getLeftOperand()) and
+      rLower = getFullyConvertedLowerBounds(getRightOperand()) and
+      rUpper = getFullyConvertedUpperBounds(getRightOperand()) and
+      lLower <= lUpper and
+      rLower <= rUpper
+    |
+      if
+        lLower < 0
+        or
+        not (
+          isValidShiftExprShift(rLower, getLeftOperand()) and
+          isValidShiftExprShift(rUpper, getLeftOperand())
+        )
+      then
+        // We don't want to deal with shifting negative numbers at the moment,
+        // and a negative shift is implementation defined, so we set the result
+        // to the maximum value
+        result = exprMaxVal(this)
+      else
+        // We can get the largest value by shifting the largest bound by the smallest bound
+        result = lUpper.bitShiftRight(rLower)
+    )
+  }
+
+  override predicate dependsOnChild(Expr child) {
+    child = getLeftOperand() or child = getRightOperand()
+  }
+}
+
+/**
+ * A range analysis expression consisting of the `<<` or `<<=` operator when at least
+ * one operand is a constant (and if the right operand is a constant, it must be "valid"
+ * (see `isValidShiftExprShift`)). When handling any undefined behavior, it leaves the
+ * values unconstrained. From the C++ standard: "The behavior is undefined if the right
+ * operand is negative, or greater than or equal to the length in bits of the promoted left operand.
+ * The value of E1 << E2 is E1 left-shifted E2 bit positions; vacated bits are zero-filled. If E1
+ * has an unsigned type, the value of the result is E1 x 2 E2, reduced modulo one more than the
+ * maximum value representable in the result type. Otherwise, if E1 has a signed type and
+ * non-negative value, and E1 x 2 E2 is representable in the corresponding unsigned type of the
+ * result type, then that value, converted to the result type, is the resulting value; otherwise,
+ * the behavior is undefined."
+ */
+class ConstantLShiftExprRange extends SimpleRangeAnalysisExpr {
+  /**
+   * Holds for `a << b` or `a <<= b` in one of the following two cases:
+   * 1. `a` is a constant and `b` is not
+   * 2. `b` is constant
+   *
+   * We don't handle the case where `a` and `b` are both non-constant values.
+   */
+  ConstantLShiftExprRange() {
+    getUnspecifiedType() instanceof IntegralType and
+    exists(Expr l, Expr r |
+      l = this.(LShiftExpr).getLeftOperand() and
+      r = this.(LShiftExpr).getRightOperand()
+      or
+      l = this.(AssignLShiftExpr).getLValue() and
+      r = this.(AssignLShiftExpr).getRValue()
+    |
+      l.getUnspecifiedType() instanceof IntegralType and
+      r.getUnspecifiedType() instanceof IntegralType and
+      (
+        // If the left operand is a constant, verify that the right operand is not a constant
+        exists(evaluateConstantExpr(l)) and not exists(evaluateConstantExpr(r))
+        or
+        // If the right operand is a constant, check if it is a valid shift expression
+        exists(float constROp |
+          constROp = evaluateConstantExpr(r) and isValidShiftExprShift(constROp, l)
+        )
+      )
+    )
+  }
+
+  Expr getLeftOperand() {
+    result = this.(LShiftExpr).getLeftOperand() or
+    result = this.(AssignLShiftExpr).getLValue()
+  }
+
+  Expr getRightOperand() {
+    result = this.(LShiftExpr).getRightOperand() or
+    result = this.(AssignLShiftExpr).getRValue()
+  }
+
+  override float getLowerBounds() {
+    exists(int lLower, int lUpper, int rLower, int rUpper |
+      lLower = getFullyConvertedLowerBounds(getLeftOperand()) and
+      lUpper = getFullyConvertedUpperBounds(getLeftOperand()) and
+      rLower = getFullyConvertedLowerBounds(getRightOperand()) and
+      rUpper = getFullyConvertedUpperBounds(getRightOperand()) and
+      lLower <= lUpper and
+      rLower <= rUpper
+    |
+      if
+        lLower < 0
+        or
+        not (
+          isValidShiftExprShift(rLower, getLeftOperand()) and
+          isValidShiftExprShift(rUpper, getLeftOperand())
+        )
+      then
+        // We don't want to deal with shifting negative numbers at the moment,
+        // and a negative shift is undefined, so we set to the minimum value
+        result = exprMinVal(this)
+      else
+        // If we have `0b01010000 << [0, 2]`, the max value for 8 bits is 0b10100000
+        // (a shift of 1) but doing a shift by the upper bound would give 0b01000000.
+        // So if the left shift operation causes an overflow, we just assume the max value
+        // If necessary, we may be able to improve this bound in the future
+        if canLShiftOverflow(lUpper, rUpper, exprMaxVal(this))
+        then result = exprMinVal(this)
+        else result = lLower.bitShiftLeft(rLower)
+    )
+  }
+
+  override float getUpperBounds() {
+    exists(int lLower, int lUpper, int rLower, int rUpper |
+      lLower = getFullyConvertedLowerBounds(getLeftOperand()) and
+      lUpper = getFullyConvertedUpperBounds(getLeftOperand()) and
+      rLower = getFullyConvertedLowerBounds(getRightOperand()) and
+      rUpper = getFullyConvertedUpperBounds(getRightOperand()) and
+      lLower <= lUpper and
+      rLower <= rUpper
+    |
+      if
+        lLower < 0
+        or
+        not (
+          isValidShiftExprShift(rLower, getLeftOperand()) and
+          isValidShiftExprShift(rUpper, getLeftOperand())
+        )
+      then
+        // We don't want to deal with shifting negative numbers at the moment,
+        // and a negative shift is undefined, so we set it to the maximum value
+        result = exprMaxVal(this)
+      else
+        // If we have `0b01010000 << [0, 2]`, the max value for 8 bits is 0b10100000
+        // (a shift of 1) but doing a shift by the upper bound would give 0b01000000.
+        // So if the left shift operation causes an overflow, we just assume the max value
+        // If necessary, we may be able to improve this bound in the future
+        if canLShiftOverflow(lUpper, rUpper, exprMaxVal(this))
+        then result = exprMaxVal(this)
+        else result = lUpper.bitShiftLeft(rUpper)
+    )
+  }
+
+  override predicate dependsOnChild(Expr child) {
+    child = getLeftOperand() or child = getRightOperand()
+  }
+}

cpp/ql/lib/semmle/code/cpp/models/implementations/Iterator.qll

@@ -107,130 +107,34 @@ private FunctionInput getIteratorArgumentInput(Operator op, int index) {
   )
 }
 
-/**
- * A non-member prefix `operator*` function for an iterator type.
- */
-private class IteratorPointerDereferenceOperator extends Operator, TaintFunction,
-  IteratorReferenceFunction {
-  FunctionInput iteratorInput;
-
-  IteratorPointerDereferenceOperator() {
-    this.hasName("operator*") and
-    iteratorInput = getIteratorArgumentInput(this, 0)
-  }
-
-  override predicate hasTaintFlow(FunctionInput input, FunctionOutput output) {
-    input = iteratorInput and
-    output.isReturnValue()
-    or
-    input.isReturnValueDeref() and
-    output.isParameterDeref(0)
-  }
-}
-
 /**
  * A non-member `operator++` or `operator--` function for an iterator type.
+ *
+ * Note that this class _only_ matches non-member functions. To find both
+ * non-member and versions, use `IteratorCrementOperator`.
  */
-private class IteratorCrementOperator extends Operator, DataFlowFunction {
+class IteratorCrementNonMemberOperator extends Operator {
-  FunctionInput iteratorInput;
+  IteratorCrementNonMemberOperator() {
-
-  IteratorCrementOperator() {
     this.hasName(["operator++", "operator--"]) and
-    iteratorInput = getIteratorArgumentInput(this, 0)
-  }
-
-  override predicate hasDataFlow(FunctionInput input, FunctionOutput output) {
-    input = iteratorInput and
-    output.isReturnValue()
-    or
-    input.isParameterDeref(0) and output.isReturnValueDeref()
-  }
-}
-
-/**
- * A non-member `operator+` function for an iterator type.
- */
-private class IteratorAddOperator extends Operator, TaintFunction {
-  FunctionInput iteratorInput;
-
-  IteratorAddOperator() {
-    this.hasName("operator+") and
-    iteratorInput = getIteratorArgumentInput(this, [0, 1])
-  }
-
-  override predicate hasTaintFlow(FunctionInput input, FunctionOutput output) {
-    input = iteratorInput and
-    output.isReturnValue()
-  }
-}
-
-/**
- * A non-member `operator-` function that takes a pointer difference type as its second argument.
- */
-private class IteratorSubOperator extends Operator, TaintFunction {
-  FunctionInput iteratorInput;
-
-  IteratorSubOperator() {
-    this.hasName("operator-") and
-    iteratorInput = getIteratorArgumentInput(this, 0) and
-    this.getParameter(1).getUnspecifiedType() instanceof IntegralType // not an iterator difference
-  }
-
-  override predicate hasTaintFlow(FunctionInput input, FunctionOutput output) {
-    input = iteratorInput and
-    output.isReturnValue()
-  }
-}
-
-/**
- * A non-member `operator+=` or `operator-=` function for an iterator type.
- */
-class IteratorAssignArithmeticOperator extends Operator {
-  IteratorAssignArithmeticOperator() {
-    this.hasName(["operator+=", "operator-="]) and
     exists(getIteratorArgumentInput(this, 0))
   }
 }
 
-private class IteratorAssignArithmeticOperatorModel extends IteratorAssignArithmeticOperator,
+private class IteratorCrementNonMemberOperatorModel extends IteratorCrementNonMemberOperator,
-  DataFlowFunction, TaintFunction {
+  DataFlowFunction {
   override predicate hasDataFlow(FunctionInput input, FunctionOutput output) {
-    input.isParameter(0) and
+    input = getIteratorArgumentInput(this, 0) and
     output.isReturnValue()
-  }
+    or
-
-  override predicate hasTaintFlow(FunctionInput input, FunctionOutput output) {
     input.isParameterDeref(0) and output.isReturnValueDeref()
-    or
-    // reverse flow from returned reference to the object referenced by the first parameter
-    input.isReturnValueDeref() and
-    output.isParameterDeref(0)
-    or
-    (input.isParameter(1) or input.isParameterDeref(1)) and
-    output.isParameterDeref(0)
-  }
-}
-
-/**
- * A prefix `operator*` member function for an iterator type.
- */
-class IteratorPointerDereferenceMemberOperator extends MemberFunction, TaintFunction,
-  IteratorReferenceFunction {
-  IteratorPointerDereferenceMemberOperator() {
-    this.getClassAndName("operator*") instanceof Iterator
-  }
-
-  override predicate hasTaintFlow(FunctionInput input, FunctionOutput output) {
-    input.isQualifierObject() and
-    output.isReturnValue()
-    or
-    input.isReturnValueDeref() and
-    output.isQualifierObject()
   }
 }
 
 /**
  * An `operator++` or `operator--` member function for an iterator type.
+ *
+ * Note that this class _only_ matches member functions. To find both
+ * non-member and member versions, use `IteratorCrementOperator`.
  */
 class IteratorCrementMemberOperator extends MemberFunction {
   IteratorCrementMemberOperator() {
@@ -258,25 +162,49 @@ private class IteratorCrementMemberOperatorModel extends IteratorCrementMemberOp
 }
 
 /**
- * A member `operator->` function for an iterator type.
+ * A (member or non-member) `operator++` or `operator--` function for an iterator type.
  */
-private class IteratorFieldMemberOperator extends Operator, TaintFunction {
+class IteratorCrementOperator extends Function {
-  IteratorFieldMemberOperator() { this.getClassAndName("operator->") instanceof Iterator }
+  IteratorCrementOperator() {
+    this instanceof IteratorCrementNonMemberOperator or
+    this instanceof IteratorCrementMemberOperator
+  }
+}
 
+/**
+ * A non-member `operator+` function for an iterator type.
+ *
+ * Note that this class _only_ matches non-member functions. To find both
+ * non-member and member versions, use `IteratorBinaryAddOperator`.
+ */
+class IteratorAddNonMemberOperator extends Operator {
+  IteratorAddNonMemberOperator() {
+    this.hasName("operator+") and
+    exists(getIteratorArgumentInput(this, [0, 1]))
+  }
+}
+
+private class IteratorAddNonMemberOperatorModel extends IteratorAddNonMemberOperator, TaintFunction {
   override predicate hasTaintFlow(FunctionInput input, FunctionOutput output) {
-    input.isQualifierObject() and
+    input = getIteratorArgumentInput(this, [0, 1]) and
     output.isReturnValue()
   }
 }
 
 /**
  * An `operator+` or `operator-` member function of an iterator class.
+ *
+ * Note that this class _only_ matches member functions. To find both
+ * non-member and member versions, use `IteratorBinaryAddOperator`.
  */
-private class IteratorBinaryArithmeticMemberOperator extends MemberFunction, TaintFunction {
+class IteratorBinaryArithmeticMemberOperator extends MemberFunction {
   IteratorBinaryArithmeticMemberOperator() {
     this.getClassAndName(["operator+", "operator-"]) instanceof Iterator
   }
+}
 
+private class IteratorBinaryArithmeticMemberOperatorModel extends IteratorBinaryArithmeticMemberOperator,
+  TaintFunction {
   override predicate hasTaintFlow(FunctionInput input, FunctionOutput output) {
     input.isQualifierObject() and
     output.isReturnValue()
@@ -284,14 +212,84 @@ private class IteratorBinaryArithmeticMemberOperator extends MemberFunction, Tai
 }
 
 /**
- * An `operator+=` or `operator-=` member function of an iterator class.
+ * A (member or non-member) `operator+` or `operator-` function for an iterator type.
  */
-private class IteratorAssignArithmeticMemberOperator extends MemberFunction, DataFlowFunction,
+class IteratorBinaryArithmeticOperator extends Function {
-  TaintFunction {
+  IteratorBinaryArithmeticOperator() {
+    this instanceof IteratorAddNonMemberOperator or
+    this instanceof IteratorSubNonMemberOperator or
+    this instanceof IteratorBinaryArithmeticMemberOperator
+  }
+}
+
+/**
+ * A non-member `operator-` function that takes a pointer difference type as its second argument.
+ *
+ * Note that this class _only_ matches non-member functions. To find both
+ * non-member and member versions, use `IteratorBinaryArithmeticOperator` (which also
+ * includes `operator+` versions).
+ */
+class IteratorSubNonMemberOperator extends Operator {
+  IteratorSubNonMemberOperator() {
+    this.hasName("operator-") and
+    exists(getIteratorArgumentInput(this, 0)) and
+    this.getParameter(1).getUnspecifiedType() instanceof IntegralType // not an iterator difference
+  }
+}
+
+private class IteratorSubOperatorModel extends IteratorSubNonMemberOperator, TaintFunction {
+  override predicate hasTaintFlow(FunctionInput input, FunctionOutput output) {
+    input = getIteratorArgumentInput(this, 0) and
+    output.isReturnValue()
+  }
+}
+
+/**
+ * A non-member `operator+=` or `operator-=` function for an iterator type.
+ *
+ * Note that this class _only_ matches non-member functions. To find both
+ * non-member and member versions, use `IteratorAssignArithmeticOperator`.
+ */
+class IteratorAssignArithmeticNonMemberOperator extends Operator {
+  IteratorAssignArithmeticNonMemberOperator() {
+    this.hasName(["operator+=", "operator-="]) and
+    exists(getIteratorArgumentInput(this, 0))
+  }
+}
+
+private class IteratorAssignArithmeticNonMemberOperatorModel extends IteratorAssignArithmeticNonMemberOperator,
+  DataFlowFunction, TaintFunction {
+  override predicate hasDataFlow(FunctionInput input, FunctionOutput output) {
+    input.isParameter(0) and
+    output.isReturnValue()
+  }
+
+  override predicate hasTaintFlow(FunctionInput input, FunctionOutput output) {
+    input.isParameterDeref(0) and output.isReturnValueDeref()
+    or
+    // reverse flow from returned reference to the object referenced by the first parameter
+    input.isReturnValueDeref() and
+    output.isParameterDeref(0)
+    or
+    (input.isParameter(1) or input.isParameterDeref(1)) and
+    output.isParameterDeref(0)
+  }
+}
+
+/**
+ * An `operator+=` or `operator-=` member function of an iterator class.
+ *
+ * Note that this class _only_ matches member functions. To find both
+ * non-member and member versions, use `IteratorAssignArithmeticOperator`.
+ */
+class IteratorAssignArithmeticMemberOperator extends MemberFunction {
   IteratorAssignArithmeticMemberOperator() {
     this.getClassAndName(["operator+=", "operator-="]) instanceof Iterator
   }
+}
 
+private class IteratorAssignArithmeticMemberOperatorModel extends IteratorAssignArithmeticMemberOperator,
+  DataFlowFunction, TaintFunction {
   override predicate hasDataFlow(FunctionInput input, FunctionOutput output) {
     input.isQualifierAddress() and
     output.isReturnValue()
@@ -310,6 +308,83 @@ private class IteratorAssignArithmeticMemberOperator extends MemberFunction, Dat
   }
 }
 
+/**
+ * A (member or non-member) `operator+=` or `operator-=` function for an iterator type.
+ */
+class IteratorAssignArithmeticOperator extends Function {
+  IteratorAssignArithmeticOperator() {
+    this instanceof IteratorAssignArithmeticNonMemberOperator or
+    this instanceof IteratorAssignArithmeticMemberOperator
+  }
+}
+
+/**
+ * A prefix `operator*` member function for an iterator type.
+ *
+ * Note that this class _only_ matches member functions. To find both
+ * non-member and member versions, use `IteratorPointerDereferenceOperator`.
+ */
+class IteratorPointerDereferenceMemberOperator extends MemberFunction, TaintFunction,
+  IteratorReferenceFunction {
+  IteratorPointerDereferenceMemberOperator() {
+    this.getClassAndName("operator*") instanceof Iterator
+  }
+
+  override predicate hasTaintFlow(FunctionInput input, FunctionOutput output) {
+    input.isQualifierObject() and
+    output.isReturnValue()
+    or
+    input.isReturnValueDeref() and
+    output.isQualifierObject()
+  }
+}
+
+/**
+ * A non-member prefix `operator*` function for an iterator type.
+ *
+ * Note that this class _only_ matches non-member functions. To find both
+ * non-member and member versions, use `IteratorPointerDereferenceOperator`.
+ */
+class IteratorPointerDereferenceNonMemberOperator extends Operator, IteratorReferenceFunction {
+  IteratorPointerDereferenceNonMemberOperator() {
+    this.hasName("operator*") and
+    exists(getIteratorArgumentInput(this, 0))
+  }
+}
+
+private class IteratorPointerDereferenceNonMemberOperatorModel extends IteratorPointerDereferenceNonMemberOperator,
+  TaintFunction {
+  override predicate hasTaintFlow(FunctionInput input, FunctionOutput output) {
+    input = getIteratorArgumentInput(this, 0) and
+    output.isReturnValue()
+    or
+    input.isReturnValueDeref() and
+    output.isParameterDeref(0)
+  }
+}
+
+/**
+ * A (member or non-member) prefix `operator*` function for an iterator type.
+ */
+class IteratorPointerDereferenceOperator extends Function {
+  IteratorPointerDereferenceOperator() {
+    this instanceof IteratorPointerDereferenceNonMemberOperator or
+    this instanceof IteratorPointerDereferenceMemberOperator
+  }
+}
+
+/**
+ * A member `operator->` function for an iterator type.
+ */
+private class IteratorFieldMemberOperator extends Operator, TaintFunction {
+  IteratorFieldMemberOperator() { this.getClassAndName("operator->") instanceof Iterator }
+
+  override predicate hasTaintFlow(FunctionInput input, FunctionOutput output) {
+    input.isQualifierObject() and
+    output.isReturnValue()
+  }
+}
+
 /**
  * An `operator[]` member function of an iterator class.
  */
@@ -326,17 +401,24 @@ private class IteratorArrayMemberOperator extends MemberFunction, TaintFunction,
 /**
  * An `operator=` member function of an iterator class that is not a copy or move assignment
  * operator.
- *
- * The `hasTaintFlow` override provides flow through output iterators that return themselves with
- * `operator*` and use their own `operator=` to assign to the container.
  */
-private class IteratorAssignmentMemberOperator extends MemberFunction, TaintFunction {
+class IteratorAssignmentMemberOperator extends MemberFunction {
   IteratorAssignmentMemberOperator() {
     this.getClassAndName("operator=") instanceof Iterator and
     not this instanceof CopyAssignmentOperator and
     not this instanceof MoveAssignmentOperator
   }
+}
 
+/**
+ * An `operator=` member function of an iterator class that is not a copy or move assignment
+ * operator.
+ *
+ * The `hasTaintFlow` override provides flow through output iterators that return themselves with
+ * `operator*` and use their own `operator=` to assign to the container.
+ */
+private class IteratorAssignmentMemberOperatorModel extends IteratorAssignmentMemberOperator,
+  TaintFunction {
   override predicate hasTaintFlow(FunctionInput input, FunctionOutput output) {
     input.isParameterDeref(0) and
     output.isQualifierObject()

cpp/ql/test/experimental/library-tests/rangeanalysis/bitshift/BitShiftRange.expected

@@ -0,0 +1,44 @@
+| bitshift.cpp:23:3:23:9 | ... <<= ... | 0.0 | 255.0 | file://:0:0:0:0 | unsigned char | file://:0:0:0:0 | int | file://:0:0:0:0 | unsigned char | file://:0:0:0:0 | int |
+| bitshift.cpp:25:5:25:11 | ... <<= ... | 0.0 | 240.0 | file://:0:0:0:0 | unsigned char | file://:0:0:0:0 | int | file://:0:0:0:0 | unsigned char | file://:0:0:0:0 | int |
+| bitshift.cpp:29:3:29:8 | ... << ... | 0.0 | 1020.0 | file://:0:0:0:0 | unsigned char | file://:0:0:0:0 | int | file://:0:0:0:0 | int | file://:0:0:0:0 | int |
+| bitshift.cpp:32:3:32:9 | ... << ... | -2.147483648E9 | 2.147483647E9 | file://:0:0:0:0 | unsigned char | file://:0:0:0:0 | int | file://:0:0:0:0 | int | file://:0:0:0:0 | int |
+| bitshift.cpp:35:3:35:9 | ... << ... | -2.147483648E9 | 2.147483647E9 | file://:0:0:0:0 | unsigned char | file://:0:0:0:0 | int | file://:0:0:0:0 | int | file://:0:0:0:0 | int |
+| bitshift.cpp:38:3:38:22 | ... << ... | 0.0 | 32640.0 | file://:0:0:0:0 | unsigned char | file://:0:0:0:0 | unsigned char | file://:0:0:0:0 | int | file://:0:0:0:0 | int |
+| bitshift.cpp:39:3:39:22 | ... << ... | 0.0 | 32640.0 | file://:0:0:0:0 | unsigned char | file://:0:0:0:0 | unsigned char | file://:0:0:0:0 | int | file://:0:0:0:0 | int |
+| bitshift.cpp:40:3:40:22 | ... << ... | 0.0 | 32640.0 | file://:0:0:0:0 | unsigned char | file://:0:0:0:0 | unsigned char | file://:0:0:0:0 | int | file://:0:0:0:0 | int |
+| bitshift.cpp:43:3:43:19 | ... << ... | -2.147483648E9 | 2.147483647E9 | file://:0:0:0:0 | unsigned char | file://:0:0:0:0 | signed char | file://:0:0:0:0 | int | file://:0:0:0:0 | int |
+| bitshift.cpp:46:3:46:22 | ... << ... | 128.0 | 128.0 | file://:0:0:0:0 | int | file://:0:0:0:0 | unsigned char | file://:0:0:0:0 | int | file://:0:0:0:0 | int |
+| bitshift.cpp:49:3:49:8 | ... << ... | -2.147483648E9 | 2.147483647E9 | file://:0:0:0:0 | int | file://:0:0:0:0 | unsigned char | file://:0:0:0:0 | int | file://:0:0:0:0 | int |
+| bitshift.cpp:52:5:52:10 | ... << ... | 1.0 | 128.0 | file://:0:0:0:0 | int | file://:0:0:0:0 | unsigned char | file://:0:0:0:0 | int | file://:0:0:0:0 | int |
+| bitshift.cpp:57:3:57:8 | ... << ... | -2.147483648E9 | 2.147483647E9 | file://:0:0:0:0 | signed char | file://:0:0:0:0 | int | file://:0:0:0:0 | int | file://:0:0:0:0 | int |
+| bitshift.cpp:58:3:58:9 | ... << ... | -2.147483648E9 | 2.147483647E9 | file://:0:0:0:0 | signed char | file://:0:0:0:0 | int | file://:0:0:0:0 | int | file://:0:0:0:0 | int |
+| bitshift.cpp:59:3:59:9 | ... << ... | -2.147483648E9 | 2.147483647E9 | file://:0:0:0:0 | signed char | file://:0:0:0:0 | int | file://:0:0:0:0 | int | file://:0:0:0:0 | int |
+| bitshift.cpp:60:3:60:22 | ... << ... | -2.147483648E9 | 2.147483647E9 | file://:0:0:0:0 | signed char | file://:0:0:0:0 | unsigned char | file://:0:0:0:0 | int | file://:0:0:0:0 | int |
+| bitshift.cpp:61:3:61:19 | ... << ... | -2.147483648E9 | 2.147483647E9 | file://:0:0:0:0 | signed char | file://:0:0:0:0 | signed char | file://:0:0:0:0 | int | file://:0:0:0:0 | int |
+| bitshift.cpp:64:3:64:19 | ... << ... | -2.147483648E9 | 2.147483647E9 | file://:0:0:0:0 | int | file://:0:0:0:0 | signed char | file://:0:0:0:0 | int | file://:0:0:0:0 | int |
+| bitshift.cpp:67:3:67:8 | ... << ... | -2.147483648E9 | 2.147483647E9 | file://:0:0:0:0 | int | file://:0:0:0:0 | signed char | file://:0:0:0:0 | int | file://:0:0:0:0 | int |
+| bitshift.cpp:70:5:70:10 | ... << ... | 1.0 | 128.0 | file://:0:0:0:0 | int | file://:0:0:0:0 | signed char | file://:0:0:0:0 | int | file://:0:0:0:0 | int |
+| bitshift.cpp:75:5:75:10 | ... << ... | -2.147483648E9 | 2.147483647E9 | file://:0:0:0:0 | unsigned char | file://:0:0:0:0 | signed char | file://:0:0:0:0 | int | file://:0:0:0:0 | int |
+| bitshift.cpp:76:5:76:10 | ... << ... | -2.147483648E9 | 2.147483647E9 | file://:0:0:0:0 | signed char | file://:0:0:0:0 | unsigned char | file://:0:0:0:0 | int | file://:0:0:0:0 | int |
+| bitshift.cpp:90:3:90:9 | ... >>= ... | 0.0 | 63.0 | file://:0:0:0:0 | unsigned char | file://:0:0:0:0 | int | file://:0:0:0:0 | unsigned char | file://:0:0:0:0 | int |
+| bitshift.cpp:92:5:92:11 | ... >>= ... | 0.0 | 15.0 | file://:0:0:0:0 | unsigned char | file://:0:0:0:0 | int | file://:0:0:0:0 | unsigned char | file://:0:0:0:0 | int |
+| bitshift.cpp:96:3:96:8 | ... >> ... | 0.0 | 63.0 | file://:0:0:0:0 | unsigned char | file://:0:0:0:0 | int | file://:0:0:0:0 | int | file://:0:0:0:0 | int |
+| bitshift.cpp:99:3:99:9 | ... >> ... | 0.0 | 0.0 | file://:0:0:0:0 | unsigned char | file://:0:0:0:0 | int | file://:0:0:0:0 | int | file://:0:0:0:0 | int |
+| bitshift.cpp:103:3:103:9 | ... >> ... | 0.0 | 0.0 | file://:0:0:0:0 | unsigned char | file://:0:0:0:0 | int | file://:0:0:0:0 | int | file://:0:0:0:0 | int |
+| bitshift.cpp:106:3:106:22 | ... >> ... | 0.0 | 63.0 | file://:0:0:0:0 | unsigned char | file://:0:0:0:0 | unsigned char | file://:0:0:0:0 | int | file://:0:0:0:0 | int |
+| bitshift.cpp:107:3:107:22 | ... >> ... | 0.0 | 63.0 | file://:0:0:0:0 | unsigned char | file://:0:0:0:0 | unsigned char | file://:0:0:0:0 | int | file://:0:0:0:0 | int |
+| bitshift.cpp:108:3:108:22 | ... >> ... | 0.0 | 63.0 | file://:0:0:0:0 | unsigned char | file://:0:0:0:0 | unsigned char | file://:0:0:0:0 | int | file://:0:0:0:0 | int |
+| bitshift.cpp:111:3:111:19 | ... >> ... | -2.147483648E9 | 2.147483647E9 | file://:0:0:0:0 | unsigned char | file://:0:0:0:0 | signed char | file://:0:0:0:0 | int | file://:0:0:0:0 | int |
+| bitshift.cpp:114:3:114:24 | ... >> ... | 32.0 | 32.0 | file://:0:0:0:0 | int | file://:0:0:0:0 | unsigned char | file://:0:0:0:0 | int | file://:0:0:0:0 | int |
+| bitshift.cpp:117:3:117:10 | ... >> ... | -2.147483648E9 | 2.147483647E9 | file://:0:0:0:0 | int | file://:0:0:0:0 | unsigned char | file://:0:0:0:0 | int | file://:0:0:0:0 | int |
+| bitshift.cpp:120:5:120:12 | ... >> ... | 32.0 | 128.0 | file://:0:0:0:0 | int | file://:0:0:0:0 | unsigned char | file://:0:0:0:0 | int | file://:0:0:0:0 | int |
+| bitshift.cpp:126:3:126:8 | ... >> ... | -2.147483648E9 | 2.147483647E9 | file://:0:0:0:0 | signed char | file://:0:0:0:0 | int | file://:0:0:0:0 | int | file://:0:0:0:0 | int |
+| bitshift.cpp:127:3:127:9 | ... >> ... | -2.147483648E9 | 2.147483647E9 | file://:0:0:0:0 | signed char | file://:0:0:0:0 | int | file://:0:0:0:0 | int | file://:0:0:0:0 | int |
+| bitshift.cpp:128:3:128:9 | ... >> ... | -1.0 | 0.0 | file://:0:0:0:0 | signed char | file://:0:0:0:0 | int | file://:0:0:0:0 | int | file://:0:0:0:0 | int |
+| bitshift.cpp:129:3:129:22 | ... >> ... | -2.147483648E9 | 2.147483647E9 | file://:0:0:0:0 | signed char | file://:0:0:0:0 | unsigned char | file://:0:0:0:0 | int | file://:0:0:0:0 | int |
+| bitshift.cpp:130:3:130:19 | ... >> ... | -2.147483648E9 | 2.147483647E9 | file://:0:0:0:0 | signed char | file://:0:0:0:0 | signed char | file://:0:0:0:0 | int | file://:0:0:0:0 | int |
+| bitshift.cpp:133:3:133:21 | ... >> ... | -2.147483648E9 | 2.147483647E9 | file://:0:0:0:0 | int | file://:0:0:0:0 | signed char | file://:0:0:0:0 | int | file://:0:0:0:0 | int |
+| bitshift.cpp:136:3:136:10 | ... >> ... | -2.147483648E9 | 2.147483647E9 | file://:0:0:0:0 | int | file://:0:0:0:0 | signed char | file://:0:0:0:0 | int | file://:0:0:0:0 | int |
+| bitshift.cpp:139:5:139:12 | ... >> ... | 32.0 | 128.0 | file://:0:0:0:0 | int | file://:0:0:0:0 | signed char | file://:0:0:0:0 | int | file://:0:0:0:0 | int |
+| bitshift.cpp:144:5:144:10 | ... >> ... | -2.147483648E9 | 2.147483647E9 | file://:0:0:0:0 | unsigned char | file://:0:0:0:0 | signed char | file://:0:0:0:0 | int | file://:0:0:0:0 | int |
+| bitshift.cpp:145:5:145:10 | ... >> ... | -2.147483648E9 | 2.147483647E9 | file://:0:0:0:0 | signed char | file://:0:0:0:0 | unsigned char | file://:0:0:0:0 | int | file://:0:0:0:0 | int |

cpp/ql/test/experimental/library-tests/rangeanalysis/bitshift/BitShiftRange.ql

@@ -0,0 +1,24 @@
+import cpp
+import semmle.code.cpp.rangeanalysis.SimpleRangeAnalysis
+import experimental.semmle.code.cpp.rangeanalysis.extensions.ConstantShiftExprRange
+
+Expr getLOp(Operation o) {
+  result = o.(BinaryOperation).getLeftOperand() or
+  result = o.(Assignment).getLValue()
+}
+
+Expr getROp(Operation o) {
+  result = o.(BinaryOperation).getRightOperand() or
+  result = o.(Assignment).getRValue()
+}
+
+from Operation o
+where
+  (
+    o instanceof BinaryBitwiseOperation
+    or
+    o instanceof AssignBitwiseOperation
+  )
+select o, lowerBound(o), upperBound(o), getLOp(o).getUnderlyingType(),
+  getROp(o).getUnderlyingType(), getLOp(o).getFullyConverted().getUnderlyingType(),
+  getROp(o).getFullyConverted().getUnderlyingType()

cpp/ql/test/experimental/library-tests/rangeanalysis/bitshift/bitshift.cpp

@@ -0,0 +1,147 @@
+typedef signed char int8_t;
+typedef short int16_t;
+typedef int int32_t;
+typedef long int64_t;
+
+typedef unsigned char uint8_t;
+typedef unsigned short uint16_t;
+typedef unsigned int uint32_t;
+typedef unsigned long uint64_t;
+
+extern uint8_t value_known_at_runtime8();
+
+void testLShiftOperator() {
+  uint8_t unsigned_const1 = 7;
+  uint8_t unsigned_const2(7);
+  uint8_t unsigned_const3{7};
+  int8_t signed_const = -7;
+  uint8_t x = value_known_at_runtime8();
+  int8_t y = (int8_t)value_known_at_runtime8();
+  uint8_t z = value_known_at_runtime8();
+
+  // An assign left shift operator. Note that no promotion occurs here
+  z <<= 2;  // [0, 255]
+  if (z <= 60) {
+    z <<= 2;  // [0, 240]
+  }
+
+  // A normal shift
+  x << 2;  // [0, 1020]
+
+  // Possible to exceed the maximum size
+  x << 25;  // [-2147483648, 2147483648]
+
+  // Undefined behavior
+  x << 34;  // [-2147483648, 2147483648]
+
+  // A normal shift by a constant in a variable
+  x << unsigned_const1;  // [0, 32640]
+  x << unsigned_const2;  // [0, 32640]
+  x << unsigned_const3;  // [0, 32640]
+
+  // Negative shifts are undefined
+  x << signed_const;  // [-2147483648, 2147483648]
+
+  // Now the left operand is a constant
+  1 << unsigned_const1;  // [128, 128]
+
+  // x could be large enough to cause undefined behavior
+  1 << x;  // [-2147483648, 2147483647]
+  if (x < 8) {
+    // x is now constrained so the shift is defined
+    1 << x;  // [1, 128]
+  }
+
+  // We don't support shifting negative values (and some of these are undefined
+  // anyway)
+  y << 2;                // [-2147483648, 2147483647]
+  y << 25;               // [-2147483648, 2147483648]
+  y << 34;               // [-2147483648, 2147483648]
+  y << unsigned_const1;  // [-2147483648, 2147483647]
+  y << signed_const;     // [-2147483648, 2147483648]
+
+  // Negative shifts are undefined
+  1 << signed_const;  // [-2147483648, 2147483648]
+
+  // We don't handle cases where the shift range could be negative
+  1 << y;  // [-2147483648, 2147483648]
+  if (y >= 0 && y < 8) {
+    // The shift range is now positive
+    1 << y;  // [1, 128]
+  }
+
+  if (x > 0 and x < 2 and y > 0 and x < 2) {
+    // We don't support shifts where neither operand is a constant at the moment
+    x << y;  // [-2147483648, 2147483648]
+    y << x;  // [-2147483648, 2147483648]
+  }
+}
+
+void testRShiftOperator() {
+  uint8_t unsigned_const1 = 2;
+  uint8_t unsigned_const2(2);
+  uint8_t unsigned_const3{2};
+  int8_t signed_const = -2;
+  uint8_t x = value_known_at_runtime8();
+  int8_t y = (int8_t)value_known_at_runtime8();
+  uint8_t z = value_known_at_runtime8();
+
+  // An assign right shift operator. Note that no promotion occurs here
+  z >>= 2;  // [0, 63]
+  if (z <= 60) {
+    z >>= 2;  // [0, 15]
+  }
+
+  // A normal shift
+  x >> 2;  // [0, 63]
+
+  // Possible to exceed the maximum size
+  x >> 25;  // [0, 0]
+
+  // Undefined behavior, but this case is handled by the SimpleRangeAnalysis
+  // library and sets the the bounds to [0, 0], which is fine
+  x >> 34;  // [0, 0]
+
+  // A normal shift by a constant in a variable
+  x >> unsigned_const1;  // [0, 63]
+  x >> unsigned_const2;  // [0, 63]
+  x >> unsigned_const3;  // [0, 63]
+
+  // Negative shifts are undefined
+  x >> signed_const;  // [-2147483648, 2147483648]
+
+  // Now the left operand is a constant
+  128 >> unsigned_const1;  // [32, 32]
+
+  // x could be large enough to cause undefined behavior
+  128 >> x;  // [-2147483648, 2147483647]
+  if (x < 3) {
+    // x is now constrained so the shift is defined
+    128 >> x;  // [32, 128]
+  }
+
+  // We don't support shifting negative values, but the SimpleRangeAnalysis
+  // library handles the first three cases even though they're implementation
+  // defined or undefined behavior (TODO: Check ideone)
+  y >> 2;   // [-2147483648, 2147483647] (Default is [-32, 31])
+  y >> 25;  // -2147483648, 2147483647] (Default is [-1, 0])
+  y >> 34;  // [-1, 0] (My code doesn't touch this, so default code is used)
+  y >> unsigned_const1;  // [-2147483648, 2147483647]
+  y >> signed_const;     // [-2147483648, 2147483648]
+
+  // Negative shifts are undefined
+  128 >> signed_const;  // [-2147483648, 2147483648]
+
+  // We don't handle cases where the shift range could be negative
+  128 >> y;  // [-2147483648, 2147483648]
+  if (y >= 0 && y < 3) {
+    // The shift range is now positive
+    128 >> y;  // [32, 128]
+  }
+
+  if (x > 0 and x < 2 and y > 0 and x < 2) {
+    // We don't support shifts where neither operand is a constant at the moment
+    x >> y;  // [-2147483648, 2147483648]
+    y >> x;  // [-2147483648, 2147483648]
+  }
+}

csharp/downgrades/97da4e8c5750e07c4f7c8a366b21060e05ecd5ed/cil_type_annotation.ql

@@ -0,0 +1,11 @@
+class AnnotatedElement extends @cil_has_type_annotation {
+  string toString() { none() }
+}
+
+class Field extends @cil_field {
+  string toString() { none() }
+}
+
+from AnnotatedElement element, int annotation
+where cil_type_annotation(element, annotation) and not element instanceof Field
+select element, annotation

csharp/downgrades/97da4e8c5750e07c4f7c8a366b21060e05ecd5ed/type_annotation.ql

@@ -0,0 +1,11 @@
+class AnnotatedElement extends @has_type_annotation {
+  string toString() { none() }
+}
+
+class Field extends @field {
+  string toString() { none() }
+}
+
+from AnnotatedElement element, int annotation
+where type_annotation(element, annotation) and not element instanceof Field
+select element, annotation

csharp/downgrades/97da4e8c5750e07c4f7c8a366b21060e05ecd5ed/upgrade.properties

@@ -0,0 +1,4 @@
+description: Remove CIL fields as entities that can have type annotations and remove field type annotations.
+compatibility: backwards
+cil_type_annotation.rel: run cil_type_annotation.qlo
+type_annotation.rel: run type_annotation.qlo

csharp/extractor/Semmle.Extraction.CIL/Entities/Field.cs

@@ -1,7 +1,4 @@
-using System;
+using System.Collections.Generic;
-using System.Collections.Generic;
-using Microsoft.CodeAnalysis;
-using System.IO;
 
 namespace Semmle.Extraction.CIL.Entities
 {
@@ -38,6 +35,11 @@ namespace Semmle.Extraction.CIL.Entities
                     t = mt.Unmodified;
                     yield return Tuples.cil_custom_modifiers(this, mt.Modifier, mt.IsRequired);
                 }
+                if (t is ByRefType brt)
+                {
+                    t = brt.ElementType;
+                    yield return Tuples.cil_type_annotation(this, TypeAnnotation.Ref);
+                }
                 yield return Tuples.cil_field(this, DeclaringType, Name, t);
             }
         }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Field.cs

@@ -30,6 +30,7 @@ namespace Semmle.Extraction.CSharp.Entities
             PopulateAttributes();
             ContainingType!.PopulateGenerics();
             PopulateNullability(trapFile, Symbol.GetAnnotatedType());
+            PopulateRefKind(trapFile, Symbol.RefKind);
 
             var unboundFieldKey = Field.Create(Context, Symbol.OriginalDefinition);
             trapFile.fields(this, (Symbol.IsConst ? 2 : 1), Symbol.Name, ContainingType, Type.TypeRef, unboundFieldKey);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Modifier.cs

@@ -25,19 +25,14 @@ namespace Semmle.Extraction.CSharp.Entities
 
         public static string AccessibilityModifier(Accessibility access)
         {
-            switch (access)
+            return access switch
             {
-                case Accessibility.Private:
+                Accessibility.Private => Modifiers.Private,
-                    return "private";
+                Accessibility.Protected => Modifiers.Protected,
-                case Accessibility.Protected:
+                Accessibility.Public => Modifiers.Public,
-                    return "protected";
+                Accessibility.Internal => Modifiers.Internal,
-                case Accessibility.Public:
+                _ => throw new InternalError("Unavailable modifier combination"),
-                    return "public";
+            };
-                case Accessibility.Internal:
-                    return "internal";
-                default:
-                    throw new InternalError("Unavailable modifier combination");
-            }
         }
 
         public static void HasAccessibility(Context cx, TextWriter trapFile, IEntity type, Accessibility access)
@@ -48,17 +43,17 @@ namespace Semmle.Extraction.CSharp.Entities
                 case Accessibility.Public:
                 case Accessibility.Protected:
                 case Accessibility.Internal:
-                    HasModifier(cx, trapFile, type, Modifier.AccessibilityModifier(access));
+                    HasModifier(cx, trapFile, type, AccessibilityModifier(access));
                     break;
                 case Accessibility.NotApplicable:
                     break;
                 case Accessibility.ProtectedOrInternal:
-                    HasModifier(cx, trapFile, type, "protected");
+                    HasModifier(cx, trapFile, type, Modifiers.Protected);
-                    HasModifier(cx, trapFile, type, "internal");
+                    HasModifier(cx, trapFile, type, Modifiers.Internal);
                     break;
                 case Accessibility.ProtectedAndInternal:
-                    HasModifier(cx, trapFile, type, "protected");
+                    HasModifier(cx, trapFile, type, Modifiers.Protected);
-                    HasModifier(cx, trapFile, type, "private");
+                    HasModifier(cx, trapFile, type, Modifiers.Private);
                     break;
                 default:
                     throw new InternalError($"Unhandled Microsoft.CodeAnalysis.Accessibility value: {access}");
@@ -70,6 +65,27 @@ namespace Semmle.Extraction.CSharp.Entities
             trapFile.has_modifiers(target, Modifier.Create(cx, modifier));
         }
 
+        private static void ExtractNamedTypeModifiers(Context cx, TextWriter trapFile, IEntity key, ISymbol symbol)
+        {
+            if (symbol.Kind != SymbolKind.NamedType)
+                return;
+
+            if (symbol is not INamedTypeSymbol nt)
+                throw new InternalError(symbol, "Symbol kind is inconsistent with its type");
+
+            if (nt.IsRecord)
+                HasModifier(cx, trapFile, key, Modifiers.Record);
+
+            if (nt.TypeKind == TypeKind.Struct)
+            {
+                if (nt.IsReadOnly)
+                    HasModifier(cx, trapFile, key, Modifiers.Readonly);
+
+                if (nt.IsRefLikeType)
+                    HasModifier(cx, trapFile, key, Modifiers.Ref);
+            }
+        }
+
         public static void ExtractModifiers(Context cx, TextWriter trapFile, IEntity key, ISymbol symbol)
         {
             HasAccessibility(cx, trapFile, key, symbol.DeclaredAccessibility);
@@ -77,51 +93,35 @@ namespace Semmle.Extraction.CSharp.Entities
                 trapFile.has_modifiers(key, Modifier.Create(cx, Accessibility.Public));
 
             if (symbol.IsAbstract && (symbol.Kind != SymbolKind.NamedType || ((INamedTypeSymbol)symbol).TypeKind != TypeKind.Interface))
-                HasModifier(cx, trapFile, key, "abstract");
+                HasModifier(cx, trapFile, key, Modifiers.Abstract);
 
             if (symbol.IsSealed)
-                HasModifier(cx, trapFile, key, "sealed");
+                HasModifier(cx, trapFile, key, Modifiers.Sealed);
 
             var fromSource = symbol.DeclaringSyntaxReferences.Length > 0;
 
             if (symbol.IsStatic && !(symbol.Kind == SymbolKind.Field && ((IFieldSymbol)symbol).IsConst && !fromSource))
-                HasModifier(cx, trapFile, key, "static");
+                HasModifier(cx, trapFile, key, Modifiers.Static);
 
             if (symbol.IsVirtual)
-                HasModifier(cx, trapFile, key, "virtual");
+                HasModifier(cx, trapFile, key, Modifiers.Virtual);
 
             if (symbol.Kind == SymbolKind.Field && ((IFieldSymbol)symbol).IsReadOnly)
-                HasModifier(cx, trapFile, key, "readonly");
+                HasModifier(cx, trapFile, key, Modifiers.Readonly);
 
             if (symbol.IsOverride)
-                HasModifier(cx, trapFile, key, "override");
+                HasModifier(cx, trapFile, key, Modifiers.Override);
 
             if (symbol.Kind == SymbolKind.Method && ((IMethodSymbol)symbol).IsAsync)
-                HasModifier(cx, trapFile, key, "async");
+                HasModifier(cx, trapFile, key, Modifiers.Async);
 
             if (symbol.IsExtern)
-                HasModifier(cx, trapFile, key, "extern");
+                HasModifier(cx, trapFile, key, Modifiers.Extern);
 
             foreach (var modifier in symbol.GetSourceLevelModifiers())
                 HasModifier(cx, trapFile, key, modifier);
 
-            if (symbol.Kind == SymbolKind.NamedType)
+            ExtractNamedTypeModifiers(cx, trapFile, key, symbol);
-            {
-                var nt = symbol as INamedTypeSymbol;
-                if (nt is null)
-                    throw new InternalError(symbol, "Symbol kind is inconsistent with its type");
-
-                if (nt.IsRecord)
-                    HasModifier(cx, trapFile, key, "record");
-
-                if (nt.TypeKind == TypeKind.Struct)
-                {
-                    if (nt.IsReadOnly)
-                        HasModifier(cx, trapFile, key, "readonly");
-                    if (nt.IsRefLikeType)
-                        HasModifier(cx, trapFile, key, "ref");
-                }
-            }
         }
 
         public static Modifier Create(Context cx, string modifier)

csharp/extractor/Semmle.Extraction.CSharp/Entities/Modifiers.cs

@@ -0,0 +1,20 @@
+internal static class Modifiers
+{
+    public const string Abstract = "abstract";
+    public const string Async = "async";
+    public const string Const = "const";
+    public const string Extern = "extern";
+    public const string Internal = "internal";
+    public const string New = "new";
+    public const string Override = "override";
+    public const string Partial = "partial";
+    public const string Private = "private";
+    public const string Protected = "protected";
+    public const string Public = "public";
+    public const string Readonly = "readonly";
+    public const string Record = "record";
+    public const string Ref = "ref";
+    public const string Sealed = "sealed";
+    public const string Static = "static";
+    public const string Virtual = "virtual";
+}

csharp/ql/lib/change-notes/2023-01-26-reffields.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* C# 11: Added extractor support for `ref` fields in `ref struct` declarations.

csharp/ql/lib/semmle/code/cil/Variable.qll

@@ -152,4 +152,7 @@ class Field extends DotNet::Field, Variable, Member, CustomModifierReceiver, @ci
   override ValueOrRefType getDeclaringType() { cil_field(this, result, _, _) }
 
   override Location getLocation() { result = this.getDeclaringType().getLocation() }
+
+  /** Holds if this declaration is `ref`. */
+  predicate isRef() { cil_type_annotation(this, 32) }
 }

csharp/ql/lib/semmle/code/csharp/Variable.qll

@@ -399,6 +399,12 @@ class Field extends Variable, AssignableMember, Attributable, TopLevelExprParent
   /** Holds if this field is `volatile`. */
   predicate isVolatile() { this.hasModifier("volatile") }
 
+  /** Holds if this is a `ref` field. */
+  predicate isRef() { this.getAnnotatedType().isRef() }
+
+  /** Holds if this is a `ref readonly` field. */
+  predicate isReadonlyRef() { this.getAnnotatedType().isReadonlyRef() }
+
   /** Holds if this field is `readonly`. */
   predicate isReadOnly() { this.hasModifier("readonly") }
 

csharp/ql/lib/semmlecode.csharp.dbscheme

@@ -1868,7 +1868,7 @@ cil_field(
 @cil_member = @cil_method | @cil_type | @cil_field | @cil_property | @cil_event;
 @cil_custom_modifier_receiver = @cil_method | @cil_property | @cil_parameter | @cil_field | @cil_function_pointer_type;
 @cil_parameterizable = @cil_method | @cil_function_pointer_type;
-@cil_has_type_annotation = @cil_stack_variable | @cil_property | @cil_method | @cil_function_pointer_type;
+@cil_has_type_annotation = @cil_stack_variable | @cil_property | @cil_field | @cil_method | @cil_function_pointer_type;
 
 #keyset[parameterizable, index]
 cil_parameter(

csharp/ql/lib/upgrades/81a394bedda3d4747e36496c7be4d958424636d6/upgrade.properties

@@ -0,0 +1,2 @@
+description: Add CIL fields as entities that can have type annotations.
+compatibility: backwards

csharp/ql/test/library-tests/csharp11/Struct.cs_

@@ -0,0 +1,15 @@
+// TODO: Test needs to be enabled when .NET 7 is used as the runtime.
+namespace structassembly;
+
+public class MyEmptyClass { }
+
+public ref struct RefStruct
+{
+    public int MyInt;
+    public ref byte MyByte;
+    public ref object MyObject;
+    internal ref MyEmptyClass MyEmptyClass;
+    public ref readonly byte MyReadonlyByte;
+    public readonly ref object MyReadonlyObject;
+    public readonly ref readonly string MyReadonlyString;
+}

csharp/ql/test/library-tests/csharp11/cil/Struct.cs_

@@ -0,0 +1,14 @@
+namespace structassembly;
+
+public class MyEmptyClass { }
+
+public ref struct RefStruct
+{
+    public int MyInt;
+    public ref byte MyByte;
+    public ref object MyObject;
+    internal ref MyEmptyClass MyEmptyClass;
+    public ref readonly byte MyReadonlyByte;
+    public readonly ref object MyReadonlyObject;
+    public readonly ref readonly string MyReadonlyString;
+}

csharp/ql/test/library-tests/csharp11/cil/refField.expected

@@ -0,0 +1,6 @@
+| file://:0:0:0:0 | MyByte | Byte |
+| file://:0:0:0:0 | MyEmptyClass | MyEmptyClass |
+| file://:0:0:0:0 | MyObject | Object |
+| file://:0:0:0:0 | MyReadonlyByte | Byte |
+| file://:0:0:0:0 | MyReadonlyObject | Object |
+| file://:0:0:0:0 | MyReadonlyString | String |

csharp/ql/test/library-tests/csharp11/cil/refField.ql

@@ -0,0 +1,5 @@
+import cil
+
+query predicate cilfields(CIL::Field f, string type) {
+  f.isRef() and type = f.getType().toString() and f.getDeclaringType().getName() = "RefStruct"
+}

csharp/ql/test/library-tests/csharp11/refField.expected

@@ -0,0 +1,3 @@
+reffields
+readonlyreffields
+readonlyfield

csharp/ql/test/library-tests/csharp11/refField.ql

@@ -0,0 +1,16 @@
+import csharp
+
+query predicate reffields(Field f) {
+  f.getFile().getStem() = "Struct" and
+  f.isRef()
+}
+
+query predicate readonlyreffields(Field f) {
+  f.getFile().getStem() = "Struct" and
+  f.isReadonlyRef()
+}
+
+query predicate readonlyfield(Field f) {
+  f.getFile().getStem() = "Struct" and
+  f.isReadOnly()
+}

javascript/ql/lib/change-notes/2023-01-31-node-pty.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Added sinks from the [`node-pty`](https://www.npmjs.com/package/node-pty) library to the `js/code-injection` query.

javascript/ql/lib/semmle/javascript/security/dataflow/CodeInjectionCustomizations.qll

@@ -294,6 +294,27 @@ module CodeInjection {
     }
   }
 
+  /**
+   * An execution of a terminal command via the `node-pty` library, seen as a code injection sink.
+   * Example:
+   * ```JS
+   * var pty = require('node-pty');
+   * var ptyProcess = pty.spawn("bash", [], {...});
+   * ptyProcess.write('ls\r');
+   * ```
+   */
+  class NodePty extends Sink {
+    NodePty() {
+      this =
+        API::moduleImport("node-pty")
+            .getMember("spawn")
+            .getReturn()
+            .getMember("write")
+            .getACall()
+            .getArgument(0)
+    }
+  }
+
   /** A sink for code injection via template injection. */
   abstract private class TemplateSink extends Sink {
     deprecated override string getMessageSuffix() {

javascript/ql/test/query-tests/Security/CWE-094/CodeInjection/CodeInjection.expected

@@ -84,6 +84,11 @@ nodes
 | express.js:26:17:26:35 | req.param("wobble") |
 | express.js:27:34:27:38 | taint |
 | express.js:27:34:27:38 | taint |
+| express.js:34:9:34:35 | taint |
+| express.js:34:17:34:35 | req.param("wobble") |
+| express.js:34:17:34:35 | req.param("wobble") |
+| express.js:43:15:43:19 | taint |
+| express.js:43:15:43:19 | taint |
 | module.js:9:16:9:29 | req.query.code |
 | module.js:9:16:9:29 | req.query.code |
 | module.js:9:16:9:29 | req.query.code |
@@ -216,6 +221,10 @@ edges
 | express.js:26:9:26:35 | taint | express.js:27:34:27:38 | taint |
 | express.js:26:17:26:35 | req.param("wobble") | express.js:26:9:26:35 | taint |
 | express.js:26:17:26:35 | req.param("wobble") | express.js:26:9:26:35 | taint |
+| express.js:34:9:34:35 | taint | express.js:43:15:43:19 | taint |
+| express.js:34:9:34:35 | taint | express.js:43:15:43:19 | taint |
+| express.js:34:17:34:35 | req.param("wobble") | express.js:34:9:34:35 | taint |
+| express.js:34:17:34:35 | req.param("wobble") | express.js:34:9:34:35 | taint |
 | module.js:9:16:9:29 | req.query.code | module.js:9:16:9:29 | req.query.code |
 | module.js:11:17:11:30 | req.query.code | module.js:11:17:11:30 | req.query.code |
 | react-native.js:7:7:7:33 | tainted | react-native.js:8:32:8:38 | tainted |
@@ -311,6 +320,7 @@ edges
 | express.js:19:37:19:70 | req.par ... odule") | express.js:19:37:19:70 | req.par ... odule") | express.js:19:37:19:70 | req.par ... odule") | This code execution depends on a $@. | express.js:19:37:19:70 | req.par ... odule") | user-provided value |
 | express.js:21:19:21:48 | req.par ... ntext") | express.js:21:19:21:48 | req.par ... ntext") | express.js:21:19:21:48 | req.par ... ntext") | This code execution depends on a $@. | express.js:21:19:21:48 | req.par ... ntext") | user-provided value |
 | express.js:27:34:27:38 | taint | express.js:26:17:26:35 | req.param("wobble") | express.js:27:34:27:38 | taint | This code execution depends on a $@. | express.js:26:17:26:35 | req.param("wobble") | user-provided value |
+| express.js:43:15:43:19 | taint | express.js:34:17:34:35 | req.param("wobble") | express.js:43:15:43:19 | taint | This code execution depends on a $@. | express.js:34:17:34:35 | req.param("wobble") | user-provided value |
 | module.js:9:16:9:29 | req.query.code | module.js:9:16:9:29 | req.query.code | module.js:9:16:9:29 | req.query.code | This code execution depends on a $@. | module.js:9:16:9:29 | req.query.code | user-provided value |
 | module.js:11:17:11:30 | req.query.code | module.js:11:17:11:30 | req.query.code | module.js:11:17:11:30 | req.query.code | This code execution depends on a $@. | module.js:11:17:11:30 | req.query.code | user-provided value |
 | react-native.js:8:32:8:38 | tainted | react-native.js:7:17:7:33 | req.param("code") | react-native.js:8:32:8:38 | tainted | This code execution depends on a $@. | react-native.js:7:17:7:33 | req.param("code") | user-provided value |

javascript/ql/test/query-tests/Security/CWE-094/CodeInjection/HeuristicSourceCodeInjection.expected

@@ -88,6 +88,11 @@ nodes
 | express.js:26:17:26:35 | req.param("wobble") |
 | express.js:27:34:27:38 | taint |
 | express.js:27:34:27:38 | taint |
+| express.js:34:9:34:35 | taint |
+| express.js:34:17:34:35 | req.param("wobble") |
+| express.js:34:17:34:35 | req.param("wobble") |
+| express.js:43:15:43:19 | taint |
+| express.js:43:15:43:19 | taint |
 | module.js:9:16:9:29 | req.query.code |
 | module.js:9:16:9:29 | req.query.code |
 | module.js:9:16:9:29 | req.query.code |
@@ -224,6 +229,10 @@ edges
 | express.js:26:9:26:35 | taint | express.js:27:34:27:38 | taint |
 | express.js:26:17:26:35 | req.param("wobble") | express.js:26:9:26:35 | taint |
 | express.js:26:17:26:35 | req.param("wobble") | express.js:26:9:26:35 | taint |
+| express.js:34:9:34:35 | taint | express.js:43:15:43:19 | taint |
+| express.js:34:9:34:35 | taint | express.js:43:15:43:19 | taint |
+| express.js:34:17:34:35 | req.param("wobble") | express.js:34:9:34:35 | taint |
+| express.js:34:17:34:35 | req.param("wobble") | express.js:34:9:34:35 | taint |
 | module.js:9:16:9:29 | req.query.code | module.js:9:16:9:29 | req.query.code |
 | module.js:11:17:11:30 | req.query.code | module.js:11:17:11:30 | req.query.code |
 | react-native.js:7:7:7:33 | tainted | react-native.js:8:32:8:38 | tainted |

javascript/ql/test/query-tests/Security/CWE-094/CodeInjection/express.js

@@ -28,3 +28,18 @@ app.get('/other/path', function(req, res) {
 
   cp.execFileSync('node', ['-e', `console.log(${JSON.stringify(taint)})`]); // OK
 });
+
+const pty = require('node-pty');
+app.get('/terminal', function(req, res) {
+  const taint = req.param("wobble");
+  const shell = pty.spawn('bash', [], {
+    name: 'xterm-color',
+    cols: 80,
+    rows: 30,
+    cwd: process.env.HOME,
+    env: process.env
+  });
+
+  shell.write(taint); // NOT OK
+});
+  

ql/extractor/Cargo.toml

@@ -15,7 +15,7 @@ tree-sitter-ql-dbscheme = { git = "https://github.com/erik-krogh/tree-sitter-ql-
 tree-sitter-ql-yaml = {git = "https://github.com/erik-krogh/tree-sitter-ql.git", rev = "cf704bf3671e1ae148e173464fb65a4d2bbf5f99"}
 clap = "2.33"
 tracing = "0.1"
-tracing-subscriber = { version = "0.3.3", features = ["env-filter"] }
+tracing-subscriber = { version = "0.3.16", features = ["env-filter"] }
 rayon = "1.5.0"
-num_cpus = "1.13.0"
+num_cpus = "1.14.0"
 regex = "1.7.1"

ql/generator/Cargo.toml

@@ -10,7 +10,7 @@ edition = "2018"
 clap = "2.33"
 node-types = { path = "../node-types" }
 tracing = "0.1"
-tracing-subscriber = { version = "0.3.3", features = ["env-filter"] }
+tracing-subscriber = { version = "0.3.16", features = ["env-filter"] }
 tree-sitter-ql = { git = "https://github.com/tree-sitter/tree-sitter-ql.git", rev = "d08db734f8dc52f6bc04db53a966603122bc6985"}
 tree-sitter-ql-dbscheme = { git = "https://github.com/erik-krogh/tree-sitter-ql-dbscheme.git", rev = "63e1344353f63931e88bfbc2faa2e78e1421b213"}
 tree-sitter-ql-yaml = {git = "https://github.com/erik-krogh/tree-sitter-ql.git", rev = "cf704bf3671e1ae148e173464fb65a4d2bbf5f99"}