boost StoredXssATM (automatically with tb boost)

Fix typo in FileWritable

.codeqlmanifest.json

@@ -4,10 +4,12 @@
         "*/ql/lib/qlpack.yml",
         "*/ql/test/qlpack.yml",
         "*/ql/examples/qlpack.yml",
-        "*/upgrades/qlpack.yml",
         "cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml",
         "javascript/ql/experimental/adaptivethreatmodeling/lib/qlpack.yml",
         "javascript/ql/experimental/adaptivethreatmodeling/src/qlpack.yml",
+        "csharp/ql/campaigns/Solorigate/lib/qlpack.yml",
+        "csharp/ql/campaigns/Solorigate/src/qlpack.yml",
+        "csharp/ql/campaigns/Solorigate/test/qlpack.yml",
         "misc/legacy-support/*/qlpack.yml",
         "misc/suite-helpers/qlpack.yml",
         "ruby/extractor-pack/codeql-extractor.yml",

.github/labeler.yml

@@ -26,3 +26,6 @@ documentation:
   - "**/*.qhelp"
   - "**/*.md"
   - docs/**/*
+
+"QL-for-QL": 
+  - ql/**/*

.github/workflows/check-change-note.yml

@@ -7,6 +7,7 @@ on:
       - "*/ql/src/**/*.ql"
       - "*/ql/src/**/*.qll"
       - "!**/experimental/**"
+      - "!ql/**"
 
 jobs:
   check-change-note:

.github/workflows/ql-for-ql-build.yml

@@ -16,7 +16,7 @@ jobs:
       - uses: actions/checkout@v2
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@esbena/ql
+        uses: github/codeql-action/init@erik-krogh/ql
         with:
           languages: javascript # does not matter
       - name: Get CodeQL version
@@ -56,27 +56,46 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/cache@v2
+      - name: Cache entire extractor
+        id: cache-extractor
+        uses: actions/cache@v2
+        with:
+          path: |
+            ql/target/release/ql-autobuilder
+            ql/target/release/ql-autobuilder.exe
+            ql/target/release/ql-extractor
+            ql/target/release/ql-extractor.exe
+          key: ${{ runner.os }}-extractor-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('ql/**/*.rs') }}
+      - name: Cache cargo
+        if: steps.cache-extractor.outputs.cache-hit != 'true'
+        uses: actions/cache@v2
         with:
           path: |
             ~/.cargo/registry
             ~/.cargo/git
             ql/target
-          key: ${{ runner.os }}-rust-cargo-${{ hashFiles('**/Cargo.lock') }}
+          key: ${{ runner.os }}-rust-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
       - name: Check formatting
+        if: steps.cache-extractor.outputs.cache-hit != 'true'
         run: cd ql; cargo fmt --all -- --check
       - name: Build
+        if: steps.cache-extractor.outputs.cache-hit != 'true'
         run: cd ql; cargo build --verbose
       - name: Run tests
+        if: steps.cache-extractor.outputs.cache-hit != 'true'
         run: cd ql; cargo test --verbose
       - name: Release build
+        if: steps.cache-extractor.outputs.cache-hit != 'true'
         run: cd ql; cargo build --release
       - name: Generate dbscheme
+        if: steps.cache-extractor.outputs.cache-hit != 'true'
         run: ql/target/release/ql-generator --dbscheme ql/ql/src/ql.dbscheme --library ql/ql/src/codeql_ql/ast/internal/TreeSitter.qll
       - uses: actions/upload-artifact@v2
         with:
           name: extractor-ubuntu-latest
           path: |
+            ql/target/release/ql-autobuilder
+            ql/target/release/ql-autobuilder.exe
             ql/target/release/ql-extractor
             ql/target/release/ql-extractor.exe
           retention-days: 1
@@ -101,6 +120,10 @@ jobs:
           unzip query-pack-zip/*.zip -d pack
           cp -r ql/codeql-extractor.yml ql/tools ql/ql/src/ql.dbscheme.stats pack/
           mkdir -p pack/tools/linux64
+          if [[ -f linux64/ql-autobuilder ]]; then
+            cp linux64/ql-autobuilder pack/tools/linux64/autobuilder
+            chmod +x pack/tools/linux64/autobuilder
+          fi
           if [[ -f linux64/ql-extractor ]]; then
             cp linux64/ql-extractor pack/tools/linux64/extractor
             chmod +x pack/tools/linux64/extractor
@@ -112,13 +135,16 @@ jobs:
           name: codeql-ql-pack
           path: codeql-ql.zip
           retention-days: 1
-  analyze: 
+  analyze:
     runs-on: ubuntu-latest
-    
-    needs: 
+    strategy:
+      matrix:
+        folder: [cpp, csharp, java, javascript, python, ql, ruby]
+
+    needs:
       - package
-    
-    steps: 
+
+    steps:
       - name: Download pack
         uses: actions/download-artifact@v2
         with:
@@ -140,13 +166,27 @@ jobs:
 
       - name: Checkout repository
         uses: actions/checkout@v2
+      - name: Create CodeQL config file
+        run: |
+          echo "paths:" > ${CONF}
+          echo "  - ${FOLDER}" >> ${CONF}
+          echo "paths-ignore:" >> ${CONF}
+          echo "  - ql/ql/test" >> ${CONF}
+          echo "Config file: "
+          cat ${CONF}
+        env: 
+          CONF: ./ql-for-ql-config.yml
+          FOLDER: ${{ matrix.folder }}
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@esbena/ql
+        uses: github/codeql-action/init@erik-krogh/ql
         with:
           languages: ql
           db-location: ${{ runner.temp }}/db
+          config-file: ./ql-for-ql-config.yml
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@esbena/ql
+        uses: github/codeql-action/analyze@erik-krogh/ql
+        with: 
+          category: "ql-for-ql-${{ matrix.folder }}"
 

.github/workflows/ql-for-ql-dataset_measure.yml

@@ -26,7 +26,7 @@ jobs:
 
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@esbena/ql
+        uses: github/codeql-action/init@erik-krogh/ql
         with:
           languages: javascript # does not matter
       - uses: actions/cache@v2

.github/workflows/ql-for-ql-tests.yml

@@ -4,11 +4,11 @@ on:
   push:
     branches: [main]
     paths:
-      - ql/*
+      - "ql/**"
   pull_request:
     branches: [main]
     paths:
-      - ql/*
+      - "ql/**"
 
 env:
   CARGO_TERM_COLOR: always
@@ -20,7 +20,7 @@ jobs:
       - uses: actions/checkout@v2
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@esbena/ql
+        uses: github/codeql-action/init@erik-krogh/ql
         with:
           languages: javascript # does not matter
       - uses: actions/cache@v2

CODEOWNERS

@@ -27,4 +27,4 @@
 /docs/query-*-style-guide.md @github/codeql-analysis-reviewers
 
 # QL for QL reviewers
-/ql/ @erik-krogh @tausbn
+/ql/ @github/codeql-ql-for-ql-reviewers

CONTRIBUTING.md

@@ -4,6 +4,9 @@ We welcome contributions to our CodeQL libraries and queries. Got an idea for a
 
 There is lots of useful documentation to help you write queries, ranging from information about query file structure to tutorials for specific target languages. For more information on the documentation available, see [CodeQL queries](https://help.semmle.com/QL/learn-ql/writing-queries/writing-queries.html) on [help.semmle.com](https://help.semmle.com).
 
+## Change notes
+
+Any nontrivial user-visible change to a query pack or library pack should have a change note. For details on how to add a change note for your change, see [this guide](docs/change-notes.md).
 
 ## Submitting a new experimental query
 

README.md

@@ -1,11 +1,11 @@
 # CodeQL
 
-This open source repository contains the standard CodeQL libraries and queries that power [LGTM](https://lgtm.com) and the other CodeQL products that [GitHub](https://github.com) makes available to its customers worldwide. For the queries, libraries, and extractor that power Go analysis, visit the [CodeQL for Go repository](https://github.com/github/codeql-go).
+This open source repository contains the standard CodeQL libraries and queries that power [GitHub Advanced Security](https://github.com/features/security/code) and the other application security products that [GitHub](https://github.com/features/security/) makes available to its customers worldwide. For the queries, libraries, and extractor that power Go analysis, visit the [CodeQL for Go repository](https://github.com/github/codeql-go).
 
 ## How do I learn CodeQL and run queries?
 
 There is [extensive documentation](https://codeql.github.com/docs/) on getting started with writing CodeQL.
-You can use the [interactive query console](https://lgtm.com/help/lgtm/using-query-console) on LGTM.com or the [CodeQL for Visual Studio Code](https://codeql.github.com/docs/codeql-for-visual-studio-code/) extension to try out your queries on any open source project that's currently being analyzed.
+You can use the [CodeQL for Visual Studio Code](https://codeql.github.com/docs/codeql-for-visual-studio-code/) extension or the [interactive query console](https://lgtm.com/help/lgtm/using-query-console) on LGTM.com (Semmle Legacy product) to try out your queries on any open source project that's currently being analyzed.
 
 ## Contributing
 
@@ -13,7 +13,7 @@ We welcome contributions to our standard library and standard checks. Do you hav
 
 ## License
 
-The code in this repository is licensed under the [MIT License](LICENSE) by [GitHub](https://github.com).
+The code in this repository is licensed under the [MIT License](LICENSE) by [GitHub](https://github.com). The use of CodeQL on open source code is licensed under specific [Terms & Conditions](https://securitylab.github.com/tools/codeql/license/) UNLESS you have a commercial license in place. If you'd like to use CodeQL with a commercial codebase, please [contact us](https://github.com/enterprise/contact) for further help.
 
 ## Visual Studio Code integration
 

config/identical-files.json

@@ -452,9 +452,15 @@
     "ruby/ql/lib/codeql/ruby/dataflow/internal/SsaImplCommon.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/SsaImplCommon.qll"
   ],
-  "CryptoAlgorithms Python/JS": [
+  "CryptoAlgorithms Python/JS/Ruby": [
     "javascript/ql/lib/semmle/javascript/security/CryptoAlgorithms.qll",
-    "python/ql/lib/semmle/python/concepts/CryptoAlgorithms.qll"
+    "python/ql/lib/semmle/python/concepts/CryptoAlgorithms.qll",
+    "ruby/ql/lib/codeql/ruby/security/CryptoAlgorithms.qll"
+  ],
+  "CryptoAlgorithmNames Python/JS/Ruby": [
+    "javascript/ql/lib/semmle/javascript/security/internal/CryptoAlgorithmNames.qll",
+    "python/ql/lib/semmle/python/concepts/internal/CryptoAlgorithmNames.qll",
+    "ruby/ql/lib/codeql/ruby/security/internal/CryptoAlgorithmNames.qll"
   ],
   "SensitiveDataHeuristics Python/JS": [
     "javascript/ql/lib/semmle/javascript/security/internal/SensitiveDataHeuristics.qll",

cpp/change-notes/2021-11-01-isFromSystemMacroDefinition.md

@@ -1,4 +0,0 @@
-lgtm,codescanning
-* The QL library `semmle.code.cpp.commons.Exclusions` now contains a predicate
-  `isFromSystemMacroDefinition` for identifying code that originates from a
-  macro outside the project being analyzed.

cpp/change-notes/2021-11-09-use-of-http.md

@@ -1,2 +0,0 @@
-lgtm,codescanning
-* A new query `cpp/non-https-url` has been added for C/C++. The query flags uses of `http` URLs that might be better replaced with `https`.

cpp/change-notes/2021-11-25-certificate-not-checked.md

@@ -1,2 +0,0 @@
-lgtm,codescanning
-* A new query `cpp/certificate-not-checked` has been added for C/C++. The query flags unsafe use of OpenSSL and similar libraries.

cpp/change-notes/2021-11-25-certificate-result-conflation.md

@@ -1,2 +0,0 @@
-lgtm,codescanning
-* A new query `cpp/certificate-result-conflation` has been added for C/C++. The query flags unsafe use of OpenSSL and similar libraries.

cpp/config/suites/security/cwe-120

@@ -5,9 +5,11 @@
   @name Badly bounded write (CWE-120)
 + semmlecode-cpp-queries/Security/CWE/CWE-120/OverrunWrite.ql: /CWE/CWE-120
   @name Potentially overrunning write (CWE-120)
++ semmlecode-cpp-queries/Security/CWE/CWE-120/VeryLikelyOverrunWrite.ql: /CWE/CWE-120
+  @name Likely overrunning write
 + semmlecode-cpp-queries/Security/CWE/CWE-120/OverrunWriteFloat.ql: /CWE/CWE-120
   @name Potentially overrunning write with float to string conversion (CWE-120)
 + semmlecode-cpp-queries/Best Practices/Likely Errors/OffsetUseBeforeRangeCheck.ql: /CWE/CWE-120
   @name Array offset used before range check (CWE-120)
 + semmlecode-cpp-queries/Likely Bugs/Memory Management/UnsafeUseOfStrcat.ql: /CWE/CWE-120
-    @name Potentially unsafe use of strcat (CWE-120)
+  @name Potentially unsafe use of strcat (CWE-120)

cpp/old-change-notes/2021-10-07-cleartext-transmission.md

@@ -0,0 +1,2 @@
+lgtm,codescanning
+* The "Cleartext transmission of sensitive information" (`cpp/cleartext-transmission`) query has been improved, reducing the number of false positive results when encryption is present.

cpp/ql/lib/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.0.6
+
+## 0.0.5
+
 ## 0.0.4
 
 ### New Features

cpp/ql/lib/DefaultOptions.qll

@@ -73,7 +73,7 @@ class Options extends string {
    *   __assume(0);
    * ```
    * (note that in this case if the hint is wrong and the expression is reached at
-   * runtime, the program's behaviour is undefined)
+   * runtime, the program's behavior is undefined)
    */
   predicate exprExits(Expr e) {
     e.(AssumeExpr).getChild(0).(CompileTimeConstantInt).getIntValue() = 0 or

cpp/ql/lib/Options.qll

@@ -50,7 +50,7 @@ class CustomOptions extends Options {
    *   __assume(0);
    * ```
    * (note that in this case if the hint is wrong and the expression is reached at
-   * runtime, the program's behaviour is undefined)
+   * runtime, the program's behavior is undefined)
    */
   override predicate exprExits(Expr e) { Options.super.exprExits(e) }
 

cpp/ql/lib/change-notes/2022-01-11-remove-upgrades-packs.md

@@ -0,0 +1,4 @@
+---
+category: deprecated
+---
+* The `codeql/cpp-upgrades` CodeQL pack has been removed. All upgrades scripts have been merged into the `codeql/cpp-all` CodeQL pack.

cpp/ql/lib/change-notes/2022-01-14-hex-format-range-analysis.md

@@ -0,0 +1,5 @@
+---
+category: minorAnalysis
+---
+* `FormatLiteral::getMaxConvertedLength` now uses range analysis to provide a
+  more accurate length for integers formatted with `%x`

cpp/ql/lib/change-notes/released/0.0.6.md

@@ -0,0 +1 @@
+## 0.0.6

cpp/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.0.4
+lastReleaseVersion: 0.0.6

cpp/ql/lib/experimental/semmle/code/cpp/models/interfaces/SimpleRangeAnalysisDefinition.qll

@@ -37,7 +37,7 @@ abstract class SimpleRangeAnalysisDefinition extends RangeSsaDefinition {
    * dependencies. Without this information, range analysis might work for
    * simple cases but will go into infinite loops on complex code.
    *
-   * For example, when modelling the definition by reference in a call to an
+   * For example, when modeling the definition by reference in a call to an
    * overloaded `operator=`, written as `v = e`, the definition of `(this, v)`
    * depends on `e`.
    */

cpp/ql/lib/experimental/semmle/code/cpp/rangeanalysis/InBoundsPointerDeref.qll

@@ -5,7 +5,7 @@
  * `Instruction` level), and then using the array length analysis and the range
  * analysis together to prove that some of these pointer dereferences are safe.
  *
- * The analysis is soundy, i.e. it is sound if no undefined behaviour is present
+ * The analysis is soundy, i.e. it is sound if no undefined behavior is present
  * in the program.
  * Furthermore, it crucially depends on the soundiness of the range analysis and
  * the array length analysis.

cpp/ql/lib/qlpack.yml

@@ -1,8 +1,7 @@
 name: codeql/cpp-all
-version: 0.0.5-dev
+version: 0.0.7-dev
 groups: cpp
 dbscheme: semmlecode.cpp.dbscheme
 extractor: cpp
 library: true
-dependencies:
-  codeql/cpp-upgrades: ^0.0.3
+upgrades: upgrades

cpp/ql/lib/semmle/code/cpp/commons/NullTermination.qll

@@ -101,6 +101,21 @@ predicate functionArgumentMustBeNullTerminated(Function f, int i) {
   f instanceof StrcatFunction and i = 0
 }
 
+/**
+ * Holds if `arg` is a string format argument to a formatting function call
+ * `ffc`.
+ */
+predicate formatArgumentMustBeNullTerminated(FormattingFunctionCall ffc, Expr arg) {
+  // String argument to a formatting function (such as `printf`)
+  exists(int n, FormatLiteral fl |
+    ffc.getConversionArgument(n) = arg and
+    fl = ffc.getFormat() and
+    fl.getConversionType(n) instanceof PointerType and // `%s`, `%ws` etc
+    not fl.getConversionType(n) instanceof VoidPointerType and // exclude: `%p`
+    not fl.hasPrecision(n) // exclude: `%.*s`
+  )
+}
+
 /**
  * Holds if `va` is a variable access where the contents must be null terminated.
  */
@@ -113,13 +128,7 @@ predicate variableMustBeNullTerminated(VariableAccess va) {
     )
     or
     // String argument to a formatting function (such as `printf`)
-    exists(int n, FormatLiteral fl |
-      fc.(FormattingFunctionCall).getConversionArgument(n) = va and
-      fl = fc.(FormattingFunctionCall).getFormat() and
-      fl.getConversionType(n) instanceof PointerType and // `%s`, `%ws` etc
-      not fl.getConversionType(n) instanceof VoidPointerType and // exclude: `%p`
-      not fl.hasPrecision(n) // exclude: `%.*s`
-    )
+    formatArgumentMustBeNullTerminated(fc, va)
     or
     // Call to a wrapper function that requires null termination
     // (not itself adding a null terminator)

cpp/ql/lib/semmle/code/cpp/commons/Printf.qll

@@ -10,10 +10,22 @@ private import semmle.code.cpp.rangeanalysis.SimpleRangeAnalysis
 private import semmle.code.cpp.rangeanalysis.RangeAnalysisUtils
 
 private newtype TBufferWriteEstimationReason =
-  TNoSpecifiedEstimateReason() or
+  TUnspecifiedEstimateReason() or
   TTypeBoundsAnalysis() or
+  TWidenedValueFlowAnalysis() or
   TValueFlowAnalysis()
 
+private predicate gradeToReason(int grade, TBufferWriteEstimationReason reason) {
+  // when combining reasons, lower grade takes precedence
+  grade = 0 and reason = TUnspecifiedEstimateReason()
+  or
+  grade = 1 and reason = TTypeBoundsAnalysis()
+  or
+  grade = 2 and reason = TWidenedValueFlowAnalysis()
+  or
+  grade = 3 and reason = TValueFlowAnalysis()
+}
+
 /**
  * A reason for a specific buffer write size estimate.
  */
@@ -32,7 +44,13 @@ abstract class BufferWriteEstimationReason extends TBufferWriteEstimationReason
    * Combine estimate reasons. Used to give a reason for the size of a format string
    * conversion given reasons coming from its individual specifiers.
    */
-  abstract BufferWriteEstimationReason combineWith(BufferWriteEstimationReason other);
+  BufferWriteEstimationReason combineWith(BufferWriteEstimationReason other) {
+    exists(int grade, int otherGrade |
+      gradeToReason(grade, this) and gradeToReason(otherGrade, other)
+    |
+      if otherGrade < grade then result = other else result = this
+    )
+  }
 }
 
 /**
@@ -40,16 +58,10 @@ abstract class BufferWriteEstimationReason extends TBufferWriteEstimationReason
  * classes derived from BufferWrite and overriding `getMaxData/0` still work with the
  * queries as intended.
  */
-class NoSpecifiedEstimateReason extends BufferWriteEstimationReason, TNoSpecifiedEstimateReason {
-  override string toString() { result = "NoSpecifiedEstimateReason" }
+class UnspecifiedEstimateReason extends BufferWriteEstimationReason, TUnspecifiedEstimateReason {
+  override string toString() { result = "UnspecifiedEstimateReason" }
 
   override string getDescription() { result = "no reason specified" }
-
-  override BufferWriteEstimationReason combineWith(BufferWriteEstimationReason other) {
-    // this reason should not be used in format specifiers, so it should not be combined
-    // with other reasons
-    none()
-  }
 }
 
 /**
@@ -60,9 +72,24 @@ class TypeBoundsAnalysis extends BufferWriteEstimationReason, TTypeBoundsAnalysi
   override string toString() { result = "TypeBoundsAnalysis" }
 
   override string getDescription() { result = "based on type bounds" }
+}
 
-  override BufferWriteEstimationReason combineWith(BufferWriteEstimationReason other) {
-    other != TNoSpecifiedEstimateReason() and result = TTypeBoundsAnalysis()
+/**
+ * The estimation comes from non trivial bounds found via actual flow analysis,
+ * but a widening aproximation might have been used for variables in loops.
+ * For example
+ * ```
+ * for (int i = 0; i < 10; ++i) {
+ *    int j = i + i;
+ *    //...  <- estimation done here based on j
+ * }
+ * ```
+ */
+class WidenedValueFlowAnalysis extends BufferWriteEstimationReason, TWidenedValueFlowAnalysis {
+  override string toString() { result = "WidenedValueFlowAnalysis" }
+
+  override string getDescription() {
+    result = "based on flow analysis of value bounds with a widening approximation"
   }
 }
 
@@ -80,10 +107,6 @@ class ValueFlowAnalysis extends BufferWriteEstimationReason, TValueFlowAnalysis
   override string toString() { result = "ValueFlowAnalysis" }
 
   override string getDescription() { result = "based on flow analysis of value bounds" }
-
-  override BufferWriteEstimationReason combineWith(BufferWriteEstimationReason other) {
-    other != TNoSpecifiedEstimateReason() and result = other
-  }
 }
 
 class PrintfFormatAttribute extends FormatAttribute {
@@ -359,6 +382,38 @@ private int lengthInBase10(float f) {
   result = f.log10().floor() + 1
 }
 
+pragma[nomagic]
+private predicate isPointerTypeWithBase(Type base, PointerType pt) { base = pt.getBaseType() }
+
+bindingset[expr]
+private BufferWriteEstimationReason getEstimationReasonForIntegralExpression(Expr expr) {
+  // we consider the range analysis non trivial if it
+  // * constrained non-trivially both sides of a signed value, or
+  // * constrained non-trivially the positive side of an unsigned value
+  // expr should already be given as getFullyConverted
+  if
+    upperBound(expr) < exprMaxVal(expr) and
+    (exprMinVal(expr) >= 0 or lowerBound(expr) > exprMinVal(expr))
+  then
+    // next we check whether the estimate may have been widened
+    if upperBoundMayBeWidened(expr)
+    then result = TWidenedValueFlowAnalysis()
+    else result = TValueFlowAnalysis()
+  else result = TTypeBoundsAnalysis()
+}
+
+/**
+ * Gets the number of hex digits required to represent the integer represented by `f`.
+ *
+ * `f` is assumed to be nonnegative.
+ */
+bindingset[f]
+private int lengthInBase16(float f) {
+  f = 0 and result = 1
+  or
+  result = (f.log2() / 4.0).floor() + 1
+}
+
 /**
  * A class to represent format strings that occur as arguments to invocations of formatting functions.
  */
@@ -910,19 +965,19 @@ class FormatLiteral extends Literal {
       (
         conv = ["s", "S"] and
         len = "h" and
-        result.(PointerType).getBaseType() instanceof PlainCharType
+        isPointerTypeWithBase(any(PlainCharType plainCharType), result)
         or
         conv = ["s", "S"] and
         len = ["l", "w"] and
-        result.(PointerType).getBaseType() = this.getWideCharType()
+        isPointerTypeWithBase(this.getWideCharType(), result)
         or
         conv = "s" and
         (len != "l" and len != "w" and len != "h") and
-        result.(PointerType).getBaseType() = this.getDefaultCharType()
+        isPointerTypeWithBase(this.getDefaultCharType(), result)
         or
         conv = "S" and
         (len != "l" and len != "w" and len != "h") and
-        result.(PointerType).getBaseType() = this.getNonDefaultCharType()
+        isPointerTypeWithBase(this.getNonDefaultCharType(), result)
       )
     )
   }
@@ -1157,12 +1212,10 @@ class FormatLiteral extends Literal {
             1 + lengthInBase10(2.pow(this.getIntegralDisplayType(n).getSize() * 8 - 1)) and
           // The second case uses range analysis to deduce a length that's shorter than the length
           // of the number -2^31.
-          exists(Expr arg, float lower, float upper, float typeLower, float typeUpper |
+          exists(Expr arg, float lower, float upper |
             arg = this.getUse().getConversionArgument(n) and
             lower = lowerBound(arg.getFullyConverted()) and
-            upper = upperBound(arg.getFullyConverted()) and
-            typeLower = exprMinVal(arg.getFullyConverted()) and
-            typeUpper = exprMaxVal(arg.getFullyConverted())
+            upper = upperBound(arg.getFullyConverted())
           |
             valueBasedBound =
               max(int cand |
@@ -1179,11 +1232,9 @@ class FormatLiteral extends Literal {
                   else cand = lengthInBase10(upper)
                 )
               ) and
-            (
-              if lower > typeLower or upper < typeUpper
-              then reason = TValueFlowAnalysis()
-              else reason = TTypeBoundsAnalysis()
-            )
+            // we don't want to call this on `arg.getFullyConverted()` as we want
+            // to detect non-trivial range analysis without taking into account up-casting
+            reason = getEstimationReasonForIntegralExpression(arg)
           ) and
           len = valueBasedBound.minimum(typeBasedBound)
         )
@@ -1195,6 +1246,40 @@ class FormatLiteral extends Literal {
           typeBasedBound = lengthInBase10(2.pow(this.getIntegralDisplayType(n).getSize() * 8) - 1) and
           // The second case uses range analysis to deduce a length that's shorter than
           // the length of the number 2^31 - 1.
+          exists(Expr arg, float lower, float upper |
+            arg = this.getUse().getConversionArgument(n) and
+            lower = lowerBound(arg.getFullyConverted()) and
+            upper = upperBound(arg.getFullyConverted())
+          |
+            valueBasedBound =
+              lengthInBase10(max(float cand |
+                  // If lower can be negative we use `(unsigned)-1` as the candidate value.
+                  lower < 0 and
+                  cand = 2.pow(any(IntType t | t.isUnsigned()).getSize() * 8)
+                  or
+                  cand = upper
+                )) and
+            // we don't want to call this on `arg.getFullyConverted()` as we want
+            // to detect non-trivial range analysis without taking into account up-casting
+            reason = getEstimationReasonForIntegralExpression(arg)
+          ) and
+          len = valueBasedBound.minimum(typeBasedBound)
+        )
+        or
+        this.getConversionChar(n).toLowerCase() = "x" and
+        // e.g. "12345678"
+        exists(int baseLen, int typeBasedBound, int valueBasedBound |
+          typeBasedBound =
+            min(int digits |
+              digits = 2 * this.getIntegralDisplayType(n).getSize()
+              or
+              exists(IntegralType t |
+                t = this.getUse().getConversionArgument(n).getType().getUnderlyingType()
+              |
+                t.isUnsigned() and
+                digits = 2 * t.getSize()
+              )
+            ) and
           exists(Expr arg, float lower, float upper, float typeLower, float typeUpper |
             arg = this.getUse().getConversionArgument(n) and
             lower = lowerBound(arg.getFullyConverted()) and
@@ -1203,7 +1288,7 @@ class FormatLiteral extends Literal {
             typeUpper = exprMaxVal(arg.getFullyConverted())
           |
             valueBasedBound =
-              lengthInBase10(max(float cand |
+              lengthInBase16(max(float cand |
                   // If lower can be negative we use `(unsigned)-1` as the candidate value.
                   lower < 0 and
                   cand = 2.pow(any(IntType t | t.isUnsigned()).getSize() * 8)
@@ -1216,29 +1301,10 @@ class FormatLiteral extends Literal {
               else reason = TTypeBoundsAnalysis()
             )
           ) and
-          len = valueBasedBound.minimum(typeBasedBound)
+          baseLen = valueBasedBound.minimum(typeBasedBound) and
+          if this.hasAlternateFlag(n) then len = 2 + baseLen else len = baseLen // "0x"
         )
         or
-        this.getConversionChar(n).toLowerCase() = "x" and
-        // e.g. "12345678"
-        exists(int sizeBytes, int baseLen |
-          sizeBytes =
-            min(int bytes |
-              bytes = this.getIntegralDisplayType(n).getSize()
-              or
-              exists(IntegralType t |
-                t = this.getUse().getConversionArgument(n).getType().getUnderlyingType()
-              |
-                t.isUnsigned() and bytes = t.getSize()
-              )
-            ) and
-          baseLen = sizeBytes * 2 and
-          (
-            if this.hasAlternateFlag(n) then len = 2 + baseLen else len = baseLen // "0x"
-          )
-        ) and
-        reason = TTypeBoundsAnalysis()
-        or
         this.getConversionChar(n).toLowerCase() = "p" and
         exists(PointerType ptrType, int baseLen |
           ptrType = this.getFullyConverted().getType() and

cpp/ql/lib/semmle/code/cpp/controlflow/SSAUtils.qll

@@ -153,9 +153,11 @@ library class SSAHelper extends int {
    * Modern Compiler Implementation by Andrew Appel.
    */
   private predicate frontier_phi_node(StackVariable v, BasicBlock b) {
-    exists(BasicBlock x | dominanceFrontier(x, b) and ssa_defn_rec(v, x)) and
+    exists(BasicBlock x |
+      dominanceFrontier(x, b) and ssa_defn_rec(pragma[only_bind_into](v), pragma[only_bind_into](x))
+    ) and
     /* We can also eliminate those nodes where the variable is not live on any incoming edge */
-    live_at_start_of_bb(v, b)
+    live_at_start_of_bb(pragma[only_bind_into](v), b)
   }
 
   private predicate ssa_defn_rec(StackVariable v, BasicBlock b) {

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplCommon.qll

@@ -3,6 +3,17 @@ private import DataFlowImplSpecific::Public
 import Cached
 
 module DataFlowImplCommonPublic {
+  /** A state value to track during data flow. */
+  class FlowState = string;
+
+  /**
+   * The default state, which is used when the state is unspecified for a source
+   * or a sink.
+   */
+  class FlowStateEmpty extends FlowState {
+    FlowStateEmpty() { this = "" }
+  }
+
   private newtype TFlowFeature =
     TFeatureHasSourceCallContext() or
     TFeatureHasSinkCallContext() or

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplConsistency.qll

@@ -15,11 +15,23 @@ module Consistency {
   class ConsistencyConfiguration extends TConsistencyConfiguration {
     string toString() { none() }
 
+    /** Holds if `n` should be excluded from the consistency test `uniqueEnclosingCallable`. */
+    predicate uniqueEnclosingCallableExclude(Node n) { none() }
+
+    /** Holds if `n` should be excluded from the consistency test `uniqueNodeLocation`. */
+    predicate uniqueNodeLocationExclude(Node n) { none() }
+
+    /** Holds if `n` should be excluded from the consistency test `missingLocation`. */
+    predicate missingLocationExclude(Node n) { none() }
+
     /** Holds if `n` should be excluded from the consistency test `postWithInFlow`. */
     predicate postWithInFlowExclude(Node n) { none() }
 
     /** Holds if `n` should be excluded from the consistency test `argHasPostUpdate`. */
     predicate argHasPostUpdateExclude(ArgumentNode n) { none() }
+
+    /** Holds if `n` should be excluded from the consistency test `reverseRead`. */
+    predicate reverseReadExclude(Node n) { none() }
   }
 
   private class RelevantNode extends Node {
@@ -46,6 +58,7 @@ module Consistency {
       n instanceof RelevantNode and
       c = count(nodeGetEnclosingCallable(n)) and
       c != 1 and
+      not any(ConsistencyConfiguration conf).uniqueEnclosingCallableExclude(n) and
       msg = "Node should have one enclosing callable but has " + c + "."
     )
   }
@@ -66,6 +79,7 @@ module Consistency {
           n.hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn)
         ) and
       c != 1 and
+      not any(ConsistencyConfiguration conf).uniqueNodeLocationExclude(n) and
       msg = "Node should have one location but has " + c + "."
     )
   }
@@ -76,7 +90,8 @@ module Consistency {
         strictcount(Node n |
           not exists(string filepath, int startline, int startcolumn, int endline, int endcolumn |
             n.hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn)
-          )
+          ) and
+          not any(ConsistencyConfiguration conf).missingLocationExclude(n)
         ) and
       msg = "Nodes without location: " + c
     )
@@ -172,6 +187,7 @@ module Consistency {
 
   query predicate reverseRead(Node n, string msg) {
     exists(Node n2 | readStep(n, _, n2) and hasPost(n2) and not hasPost(n)) and
+    not any(ConsistencyConfiguration conf).reverseReadExclude(n) and
     msg = "Origin of readStep is missing a PostUpdateNode."
   }
 

cpp/ql/lib/semmle/code/cpp/dataflow/internal/FlowVar.qll

@@ -435,7 +435,7 @@ module FlowVar_internal {
       parameterIsNonConstReference(p) and
       p = v and
       // This definition reaches the exit node of the function CFG
-      getAReachedBlockVarSBB(this).getANode() = p.getFunction()
+      getAReachedBlockVarSBB(this).getEnd() = p.getFunction()
     }
 
     override predicate definedByInitialValue(StackVariable lsv) {

cpp/ql/lib/semmle/code/cpp/dataflow/internal/TaintTrackingUtil.qll

@@ -47,6 +47,12 @@ predicate defaultImplicitTaintRead(DataFlow::Node node, DataFlow::Content c) { n
  */
 predicate defaultTaintSanitizer(DataFlow::Node node) { none() }
 
+/**
+ * Holds if `guard` should be a sanitizer guard in all global taint flow configurations
+ * but not in local taint.
+ */
+predicate defaultTaintSanitizerGuard(DataFlow::BarrierGuard guard) { none() }
+
 /**
  * Holds if taint can flow in one local step from `nodeFrom` to `nodeTo` excluding
  * local data flow steps. That is, `nodeFrom` and `nodeTo` are likely to represent

cpp/ql/lib/semmle/code/cpp/dataflow/internal/tainttracking1/TaintTrackingImpl.qll

@@ -61,7 +61,7 @@ abstract class Configuration extends DataFlow::Configuration {
    * The smaller this predicate is, the faster `hasFlow()` will converge.
    */
   // overridden to provide taint-tracking specific qldoc
-  abstract override predicate isSource(DataFlow::Node source);
+  override predicate isSource(DataFlow::Node source) { none() }
 
   /**
    * Holds if `sink` is a relevant taint sink.
@@ -69,7 +69,7 @@ abstract class Configuration extends DataFlow::Configuration {
    * The smaller this predicate is, the faster `hasFlow()` will converge.
    */
   // overridden to provide taint-tracking specific qldoc
-  abstract override predicate isSink(DataFlow::Node sink);
+  override predicate isSink(DataFlow::Node sink) { none() }
 
   /** Holds if the node `node` is a taint sanitizer. */
   predicate isSanitizer(DataFlow::Node node) { none() }
@@ -93,7 +93,7 @@ abstract class Configuration extends DataFlow::Configuration {
   predicate isSanitizerGuard(DataFlow::BarrierGuard guard) { none() }
 
   final override predicate isBarrierGuard(DataFlow::BarrierGuard guard) {
-    this.isSanitizerGuard(guard)
+    this.isSanitizerGuard(guard) or defaultTaintSanitizerGuard(guard)
   }
 
   /**

cpp/ql/lib/semmle/code/cpp/dataflow/internal/tainttracking2/TaintTrackingImpl.qll

@@ -61,7 +61,7 @@ abstract class Configuration extends DataFlow::Configuration {
    * The smaller this predicate is, the faster `hasFlow()` will converge.
    */
   // overridden to provide taint-tracking specific qldoc
-  abstract override predicate isSource(DataFlow::Node source);
+  override predicate isSource(DataFlow::Node source) { none() }
 
   /**
    * Holds if `sink` is a relevant taint sink.
@@ -69,7 +69,7 @@ abstract class Configuration extends DataFlow::Configuration {
    * The smaller this predicate is, the faster `hasFlow()` will converge.
    */
   // overridden to provide taint-tracking specific qldoc
-  abstract override predicate isSink(DataFlow::Node sink);
+  override predicate isSink(DataFlow::Node sink) { none() }
 
   /** Holds if the node `node` is a taint sanitizer. */
   predicate isSanitizer(DataFlow::Node node) { none() }
@@ -93,7 +93,7 @@ abstract class Configuration extends DataFlow::Configuration {
   predicate isSanitizerGuard(DataFlow::BarrierGuard guard) { none() }
 
   final override predicate isBarrierGuard(DataFlow::BarrierGuard guard) {
-    this.isSanitizerGuard(guard)
+    this.isSanitizerGuard(guard) or defaultTaintSanitizerGuard(guard)
   }
 
   /**

cpp/ql/lib/semmle/code/cpp/exprs/Lambda.qll

@@ -118,7 +118,7 @@ class LambdaCapture extends Locatable, @lambdacapture {
    * An identifier is captured by reference if:
    *   - It is explicitly captured by reference.
    *   - It is implicitly captured, and the lambda's default capture mode is by-reference.
-   *   - The identifier is "this". [Said behaviour is dictated by the C++11 standard, but it
+   *   - The identifier is "this". [Said behavior is dictated by the C++11 standard, but it
    *                                is actually "*this" being captured rather than "this".]
    */
   predicate isCapturedByReference() { lambda_capture(this, _, _, _, true, _, _) }

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowDispatch.qll

@@ -63,7 +63,7 @@ private module VirtualDispatch {
         this.flowsFrom(other, allowOtherFromArg)
       |
         // Call argument
-        exists(DataFlowCall call, int i |
+        exists(DataFlowCall call, Position i |
           other
               .(DataFlow::ParameterNode)
               .isParameterOf(pragma[only_bind_into](call).getStaticCallTarget(), i) and
@@ -268,16 +268,6 @@ Function viableImplInCallContext(CallInstruction call, CallInstruction ctx) {
   )
 }
 
-/** A parameter position represented by an integer. */
-class ParameterPosition extends int {
-  ParameterPosition() { any(ParameterNode p).isParameterOf(_, this) }
-}
-
-/** An argument position represented by an integer. */
-class ArgumentPosition extends int {
-  ArgumentPosition() { any(ArgumentNode a).argumentOf(_, this) }
-}
-
 /** Holds if arguments at position `apos` match parameters at position `ppos`. */
 pragma[inline]
 predicate parameterMatch(ParameterPosition ppos, ArgumentPosition apos) { ppos = apos }

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplCommon.qll

@@ -3,6 +3,17 @@ private import DataFlowImplSpecific::Public
 import Cached
 
 module DataFlowImplCommonPublic {
+  /** A state value to track during data flow. */
+  class FlowState = string;
+
+  /**
+   * The default state, which is used when the state is unspecified for a source
+   * or a sink.
+   */
+  class FlowStateEmpty extends FlowState {
+    FlowStateEmpty() { this = "" }
+  }
+
   private newtype TFlowFeature =
     TFeatureHasSourceCallContext() or
     TFeatureHasSinkCallContext() or

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplConsistency.qll

@@ -15,11 +15,23 @@ module Consistency {
   class ConsistencyConfiguration extends TConsistencyConfiguration {
     string toString() { none() }
 
+    /** Holds if `n` should be excluded from the consistency test `uniqueEnclosingCallable`. */
+    predicate uniqueEnclosingCallableExclude(Node n) { none() }
+
+    /** Holds if `n` should be excluded from the consistency test `uniqueNodeLocation`. */
+    predicate uniqueNodeLocationExclude(Node n) { none() }
+
+    /** Holds if `n` should be excluded from the consistency test `missingLocation`. */
+    predicate missingLocationExclude(Node n) { none() }
+
     /** Holds if `n` should be excluded from the consistency test `postWithInFlow`. */
     predicate postWithInFlowExclude(Node n) { none() }
 
     /** Holds if `n` should be excluded from the consistency test `argHasPostUpdate`. */
     predicate argHasPostUpdateExclude(ArgumentNode n) { none() }
+
+    /** Holds if `n` should be excluded from the consistency test `reverseRead`. */
+    predicate reverseReadExclude(Node n) { none() }
   }
 
   private class RelevantNode extends Node {
@@ -46,6 +58,7 @@ module Consistency {
       n instanceof RelevantNode and
       c = count(nodeGetEnclosingCallable(n)) and
       c != 1 and
+      not any(ConsistencyConfiguration conf).uniqueEnclosingCallableExclude(n) and
       msg = "Node should have one enclosing callable but has " + c + "."
     )
   }
@@ -66,6 +79,7 @@ module Consistency {
           n.hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn)
         ) and
       c != 1 and
+      not any(ConsistencyConfiguration conf).uniqueNodeLocationExclude(n) and
       msg = "Node should have one location but has " + c + "."
     )
   }
@@ -76,7 +90,8 @@ module Consistency {
         strictcount(Node n |
           not exists(string filepath, int startline, int startcolumn, int endline, int endcolumn |
             n.hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn)
-          )
+          ) and
+          not any(ConsistencyConfiguration conf).missingLocationExclude(n)
         ) and
       msg = "Nodes without location: " + c
     )
@@ -172,6 +187,7 @@ module Consistency {
 
   query predicate reverseRead(Node n, string msg) {
     exists(Node n2 | readStep(n, _, n2) and hasPost(n2) and not hasPost(n)) and
+    not any(ConsistencyConfiguration conf).reverseReadExclude(n) and
     msg = "Origin of readStep is missing a PostUpdateNode."
   }
 

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowPrivate.qll

@@ -27,7 +27,7 @@ abstract class ArgumentNode extends OperandNode {
    * Holds if this argument occurs at the given position in the given call.
    * The instance argument is considered to have index `-1`.
    */
-  abstract predicate argumentOf(DataFlowCall call, int pos);
+  abstract predicate argumentOf(DataFlowCall call, ArgumentPosition pos);
 
   /** Gets the call in which this node is an argument. */
   DataFlowCall getCall() { this.argumentOf(result, _) }
@@ -42,7 +42,9 @@ private class PrimaryArgumentNode extends ArgumentNode {
 
   PrimaryArgumentNode() { exists(CallInstruction call | op = call.getAnArgumentOperand()) }
 
-  override predicate argumentOf(DataFlowCall call, int pos) { op = call.getArgumentOperand(pos) }
+  override predicate argumentOf(DataFlowCall call, ArgumentPosition pos) {
+    op = call.getArgumentOperand(pos.(DirectPosition).getIndex())
+  }
 
   override string toString() {
     exists(Expr unconverted |
@@ -71,9 +73,9 @@ private class SideEffectArgumentNode extends ArgumentNode {
 
   SideEffectArgumentNode() { op = read.getSideEffectOperand() }
 
-  override predicate argumentOf(DataFlowCall call, int pos) {
+  override predicate argumentOf(DataFlowCall call, ArgumentPosition pos) {
     read.getPrimaryInstruction() = call and
-    pos = getArgumentPosOfSideEffect(read.getIndex())
+    pos.(IndirectionPosition).getIndex() = read.getIndex()
   }
 
   override string toString() {
@@ -90,6 +92,54 @@ private class SideEffectArgumentNode extends ArgumentNode {
   }
 }
 
+/** A parameter position represented by an integer. */
+class ParameterPosition = Position;
+
+/** An argument position represented by an integer. */
+class ArgumentPosition = Position;
+
+class Position extends TPosition {
+  abstract string toString();
+}
+
+class DirectPosition extends TDirectPosition {
+  int index;
+
+  DirectPosition() { this = TDirectPosition(index) }
+
+  string toString() {
+    index = -1 and
+    result = "this"
+    or
+    index != -1 and
+    result = index.toString()
+  }
+
+  int getIndex() { result = index }
+}
+
+class IndirectionPosition extends TIndirectionPosition {
+  int index;
+
+  IndirectionPosition() { this = TIndirectionPosition(index) }
+
+  string toString() {
+    index = -1 and
+    result = "this"
+    or
+    index != -1 and
+    result = index.toString()
+  }
+
+  int getIndex() { result = index }
+}
+
+newtype TPosition =
+  TDirectPosition(int index) { exists(any(CallInstruction c).getArgument(index)) } or
+  TIndirectionPosition(int index) {
+    exists(ReadSideEffectInstruction instr | instr.getIndex() = index)
+  }
+
 private newtype TReturnKind =
   TNormalReturnKind() or
   TIndirectReturnKind(ParameterIndex index)

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowUtil.qll

@@ -490,19 +490,6 @@ class ExprNode extends InstructionNode {
   override string toString() { result = this.asConvertedExpr().toString() }
 }
 
-/**
- * INTERNAL: do not use. Translates a parameter/argument index into a negative
- * number that denotes the index of its side effect (pointer indirection).
- */
-bindingset[index]
-int getArgumentPosOfSideEffect(int index) {
-  // -1 -> -2
-  //  0 -> -3
-  //  1 -> -4
-  // ...
-  result = -3 - index
-}
-
 /**
  * The value of a parameter at function entry, viewed as a node in a data
  * flow graph. This includes both explicit parameters such as `x` in `f(x)`
@@ -525,7 +512,7 @@ class ParameterNode extends InstructionNode {
    * implicit `this` parameter is considered to have position `-1`, and
    * pointer-indirection parameters are at further negative positions.
    */
-  predicate isParameterOf(Function f, int pos) { none() } // overridden by subclasses
+  predicate isParameterOf(Function f, ParameterPosition pos) { none() } // overridden by subclasses
 }
 
 /** An explicit positional parameter, not including `this` or `...`. */
@@ -534,8 +521,8 @@ private class ExplicitParameterNode extends ParameterNode {
 
   ExplicitParameterNode() { exists(instr.getParameter()) }
 
-  override predicate isParameterOf(Function f, int pos) {
-    f.getParameter(pos) = instr.getParameter()
+  override predicate isParameterOf(Function f, ParameterPosition pos) {
+    f.getParameter(pos.(DirectPosition).getIndex()) = instr.getParameter()
   }
 
   /** Gets the `Parameter` associated with this node. */
@@ -550,8 +537,8 @@ class ThisParameterNode extends ParameterNode {
 
   ThisParameterNode() { instr.getIRVariable() instanceof IRThisVariable }
 
-  override predicate isParameterOf(Function f, int pos) {
-    pos = -1 and instr.getEnclosingFunction() = f
+  override predicate isParameterOf(Function f, ParameterPosition pos) {
+    pos.(DirectPosition).getIndex() = -1 and instr.getEnclosingFunction() = f
   }
 
   override string toString() { result = "this" }
@@ -561,12 +548,12 @@ class ThisParameterNode extends ParameterNode {
 class ParameterIndirectionNode extends ParameterNode {
   override InitializeIndirectionInstruction instr;
 
-  override predicate isParameterOf(Function f, int pos) {
+  override predicate isParameterOf(Function f, ParameterPosition pos) {
     exists(int index |
       instr.getEnclosingFunction() = f and
       instr.hasIndex(index)
     |
-      pos = getArgumentPosOfSideEffect(index)
+      pos.(IndirectionPosition).getIndex() = index
     )
   }