Post-release preparation for 2.7.4

Release preparation for version 2.7.4

.codeqlmanifest.json

@@ -11,9 +11,7 @@
         "misc/legacy-support/*/qlpack.yml",
         "misc/suite-helpers/qlpack.yml",
         "ruby/extractor-pack/codeql-extractor.yml",
-        "ruby/ql/consistency-queries/qlpack.yml",
-        "ql/ql/consistency-queries/qlpack.yml",
-        "ql/extractor-pack/codeql-extractor.yml"
+        "ruby/ql/consistency-queries/qlpack.yml"
     ],
     "versionPolicies": {
       "default": {

.github/workflows/ql-for-ql-build.yml

@@ -1,152 +0,0 @@
-name: Run QL for QL
-
-on:
-  push:
-    branches: [main]
-  pull_request:
-    branches: [main]
-
-env:
-  CARGO_TERM_COLOR: always
-
-jobs:
-  queries:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Find codeql
-        id: find-codeql
-        uses: github/codeql-action/init@esbena/ql
-        with:
-          languages: javascript # does not matter
-      - name: Get CodeQL version
-        id: get-codeql-version
-        run: |
-          echo "::set-output name=version::$("${CODEQL}" --version | head -n 1 | rev | cut -d " " -f 1 | rev)"
-        shell: bash
-        env:
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-      - name: Cache queries
-        id: cache-queries
-        uses: actions/cache@v2
-        with:
-          path: ${{ runner.temp }}/query-pack.zip
-          key: queries-${{ hashFiles('ql/**/*.ql*') }}-${{ hashFiles('ql/ql/src/ql.dbscheme*') }}-${{ steps.get-codeql-version.outputs.version }}
-      - name: Build query pack
-        if: steps.cache-queries.outputs.cache-hit != 'true'
-        run: |
-          cd ql/ql/src
-          "${CODEQL}" pack create
-          cd .codeql/pack/codeql/ql-all/0.0.0
-          zip "${PACKZIP}" -r .
-        env:
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-          PACKZIP: ${{ runner.temp }}/query-pack.zip
-      - name: Upload query pack
-        uses: actions/upload-artifact@v2
-        with:
-          name: query-pack-zip
-          path: ${{ runner.temp }}/query-pack.zip
-
-  extractors:
-    strategy:
-      fail-fast: false
-
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
-        with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            ql/target
-          key: ${{ runner.os }}-rust-cargo-${{ hashFiles('**/Cargo.lock') }}
-      - name: Check formatting
-        run: cd ql; cargo fmt --all -- --check
-      - name: Build
-        run: cd ql; cargo build --verbose
-      - name: Run tests
-        run: cd ql; cargo test --verbose
-      - name: Release build
-        run: cd ql; cargo build --release
-      - name: Generate dbscheme
-        run: ql/target/release/ql-generator --dbscheme ql/ql/src/ql.dbscheme --library ql/ql/src/codeql_ql/ast/internal/TreeSitter.qll
-      - uses: actions/upload-artifact@v2
-        with:
-          name: extractor-ubuntu-latest
-          path: |
-            ql/target/release/ql-extractor
-            ql/target/release/ql-extractor.exe
-          retention-days: 1
-  package:
-    runs-on: ubuntu-latest
-
-    needs:
-      - extractors
-      - queries
-
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/download-artifact@v2
-        with:
-          name: query-pack-zip
-          path: query-pack-zip
-      - uses: actions/download-artifact@v2
-        with:
-          name: extractor-ubuntu-latest
-          path: linux64
-      - run: |
-          unzip query-pack-zip/*.zip -d pack
-          cp -r ql/codeql-extractor.yml ql/tools ql/ql/src/ql.dbscheme.stats pack/
-          mkdir -p pack/tools/linux64
-          if [[ -f linux64/ql-extractor ]]; then
-            cp linux64/ql-extractor pack/tools/linux64/extractor
-            chmod +x pack/tools/linux64/extractor
-          fi
-          cd pack
-          zip -rq ../codeql-ql.zip .
-      - uses: actions/upload-artifact@v2
-        with:
-          name: codeql-ql-pack
-          path: codeql-ql.zip
-          retention-days: 1
-  analyze: 
-    runs-on: ubuntu-latest
-    
-    needs: 
-      - package
-    
-    steps: 
-      - name: Download pack
-        uses: actions/download-artifact@v2
-        with:
-          name: codeql-ql-pack
-          path: ${{ runner.temp }}/codeql-ql-pack-artifact
-
-      - name: Prepare pack
-        run: |
-          unzip "${PACK_ARTIFACT}/*.zip" -d "${PACK}"
-        env:
-          PACK_ARTIFACT: ${{ runner.temp }}/codeql-ql-pack-artifact
-          PACK: ${{ runner.temp }}/pack
-      - name: Hack codeql-action options
-        run: |
-          JSON=$(jq -nc --arg pack "${PACK}" '.resolve.queries=["--search-path", $pack] | .resolve.extractor=["--search-path", $pack] | .database.init=["--search-path", $pack]')
-          echo "CODEQL_ACTION_EXTRA_OPTIONS=${JSON}" >> ${GITHUB_ENV}
-        env:
-          PACK: ${{ runner.temp }}/pack
-
-      - name: Checkout repository
-        uses: actions/checkout@v2
-
-      - name: Initialize CodeQL
-        uses: github/codeql-action/init@esbena/ql
-        with:
-          languages: ql
-          db-location: ${{ runner.temp }}/db
-
-      - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@esbena/ql
-

.github/workflows/ql-for-ql-dataset_measure.yml

@@ -1,84 +0,0 @@
-name: Collect database stats for QL for QL
-
-on:
-  push:
-    branches: [main]
-    paths:
-      - ql/ql/src/ql.dbscheme
-  pull_request:
-    branches: [main]
-    paths:
-      - ql/ql/src/ql.dbscheme
-  workflow_dispatch:
-
-jobs:
-  measure:
-    env:
-      CODEQL_THREADS: 4 # TODO: remove this once it's set by the CLI
-    strategy:
-      matrix:
-        repo: 
-          - github/codeql
-          - github/codeql-go
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Find codeql
-        id: find-codeql
-        uses: github/codeql-action/init@esbena/ql
-        with:
-          languages: javascript # does not matter
-      - uses: actions/cache@v2
-        with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            ql/target
-          key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('**/Cargo.lock') }}
-      - name: Build Extractor
-        run: cd ql; env "PATH=$PATH:`dirname ${CODEQL}`" ./create-extractor-pack.sh
-        env:
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-      - name: Checkout ${{ matrix.repo }}
-        uses: actions/checkout@v2
-        with:
-          repository: ${{ matrix.repo }}
-          path: ${{ github.workspace }}/repo
-      - name: Create database
-        run: |
-          "${CODEQL}" database create \
-            --search-path "ql/extractor-pack" \
-            --threads 4 \
-            --language ql --source-root "${{ github.workspace }}/repo" \
-            "${{ runner.temp }}/database"
-        env:
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-      - name: Measure database
-        run: |
-          mkdir -p "stats/${{ matrix.repo }}"
-          "${CODEQL}" dataset measure --threads 4 --output "stats/${{ matrix.repo }}/stats.xml" "${{ runner.temp }}/database/db-ql"
-        env:
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-      - uses: actions/upload-artifact@v2
-        with:
-          name: measurements
-          path: stats
-          retention-days: 1
-
-  merge:
-    runs-on: ubuntu-latest
-    needs: measure
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/download-artifact@v2
-        with:
-          name: measurements
-          path: stats
-      - run: |
-          python -m pip install --user lxml
-          find stats -name 'stats.xml' -print0 | sort -z | xargs -0 python ql/scripts/merge_stats.py --output ql/ql/src/ql.dbscheme.stats --normalise ql_tokeninfo
-      - uses: actions/upload-artifact@v2
-        with:
-          name: ql.dbscheme.stats
-          path: ql/ql/src/ql.dbscheme.stats

.github/workflows/ql-for-ql-tests.yml

@@ -1,52 +0,0 @@
-name: Run QL for QL Tests
-
-on:
-  push:
-    branches: [main]
-    paths:
-      - ql/*
-  pull_request:
-    branches: [main]
-    paths:
-      - ql/*
-
-env:
-  CARGO_TERM_COLOR: always
-
-jobs:
-  qltest:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Find codeql
-        id: find-codeql
-        uses: github/codeql-action/init@esbena/ql
-        with:
-          languages: javascript # does not matter
-      - uses: actions/cache@v2
-        with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            ql/target
-          key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('**/Cargo.lock') }}
-      - name: Build extractor
-        run: |
-          cd ql;
-          codeqlpath=$(dirname ${{ steps.find-codeql.outputs.codeql-path }});
-          env "PATH=$PATH:$codeqlpath" ./create-extractor-pack.sh
-      - name: Run QL tests
-        run: | 
-          "${CODEQL}" test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}/ql/extractor-pack" --consistency-queries ql/ql/consistency-queries ql/ql/test
-        env:
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-      - name: Check QL formatting
-        run: | 
-          find ql/ql "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 "${CODEQL}" query format --check-only
-        env:
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-      - name: Check QL compilation
-        run: | 
-          "${CODEQL}" query compile --check-only --threads=4 --warnings=error --search-path "${{ github.workspace }}/ql/extractor-pack" "ql/ql/src" "ql/ql/examples"
-        env:
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}

CODEOWNERS

@@ -25,6 +25,3 @@
 /docs/codeql-for-visual-studio-code/ @github/codeql-vscode-reviewers
 /docs/ql-language-reference/ @github/codeql-frontend-reviewers
 /docs/query-*-style-guide.md @github/codeql-analysis-reviewers
-
-# QL for QL reviewers
-/ql/ @erik-krogh @tausbn

cpp/autobuilder/Semmle.Autobuild.Cpp/Semmle.Autobuild.Cpp.csproj

@@ -17,7 +17,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Build" Version="16.11.0" />
+    <PackageReference Include="Microsoft.Build" Version="16.9.0" />
   </ItemGroup>
 
   <ItemGroup>

cpp/change-notes/2021-11-01-isFromSystemMacroDefinition.md

@@ -1,4 +0,0 @@
-lgtm,codescanning
-* The QL library `semmle.code.cpp.commons.Exclusions` now contains a predicate
-  `isFromSystemMacroDefinition` for identifying code that originates from a
-  macro outside the project being analyzed.

cpp/change-notes/2021-11-09-use-of-http.md

@@ -1,2 +0,0 @@
-lgtm,codescanning
-* A new query `cpp/non-https-url` has been added for C/C++. The query flags uses of `http` URLs that might be better replaced with `https`.

cpp/change-notes/2021-11-25-certificate-not-checked.md

@@ -1,2 +0,0 @@
-lgtm,codescanning
-* A new query `cpp/certificate-not-checked` has been added for C/C++. The query flags unsafe use of OpenSSL and similar libraries.

cpp/change-notes/2021-11-25-certificate-result-conflation.md

@@ -1,2 +0,0 @@
-lgtm,codescanning
-* A new query `cpp/certificate-result-conflation` has been added for C/C++. The query flags unsafe use of OpenSSL and similar libraries.

cpp/ql/lib/CHANGELOG.md

@@ -1,3 +1,5 @@
+## 0.0.5
+
 ## 0.0.4
 
 ### New Features

cpp/ql/lib/change-notes/released/0.0.5.md

@@ -0,0 +1 @@
+## 0.0.5

cpp/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.0.4
+lastReleaseVersion: 0.0.5

cpp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-all
-version: 0.0.5-dev
+version: 0.0.6-dev
 groups: cpp
 dbscheme: semmlecode.cpp.dbscheme
 extractor: cpp

cpp/ql/lib/semmle/code/cpp/commons/Printf.qll

@@ -9,83 +9,6 @@ import semmle.code.cpp.models.interfaces.FormattingFunction
 private import semmle.code.cpp.rangeanalysis.SimpleRangeAnalysis
 private import semmle.code.cpp.rangeanalysis.RangeAnalysisUtils
 
-private newtype TBufferWriteEstimationReason =
-  TNoSpecifiedEstimateReason() or
-  TTypeBoundsAnalysis() or
-  TValueFlowAnalysis()
-
-/**
- * A reason for a specific buffer write size estimate.
- */
-abstract class BufferWriteEstimationReason extends TBufferWriteEstimationReason {
-  /**
-   * Returns the name of the concrete class.
-   */
-  abstract string toString();
-
-  /**
-   * Returns a human readable representation of this reason.
-   */
-  abstract string getDescription();
-
-  /**
-   * Combine estimate reasons. Used to give a reason for the size of a format string
-   * conversion given reasons coming from its individual specifiers.
-   */
-  abstract BufferWriteEstimationReason combineWith(BufferWriteEstimationReason other);
-}
-
-/**
- * No particular reason given. This is currently used for backward compatibility so that
- * classes derived from BufferWrite and overriding `getMaxData/0` still work with the
- * queries as intended.
- */
-class NoSpecifiedEstimateReason extends BufferWriteEstimationReason, TNoSpecifiedEstimateReason {
-  override string toString() { result = "NoSpecifiedEstimateReason" }
-
-  override string getDescription() { result = "no reason specified" }
-
-  override BufferWriteEstimationReason combineWith(BufferWriteEstimationReason other) {
-    // this reason should not be used in format specifiers, so it should not be combined
-    // with other reasons
-    none()
-  }
-}
-
-/**
- * The estimation comes from rough bounds just based on the type (e.g.
- * `0 <= x < 2^32` for an unsigned 32 bit integer).
- */
-class TypeBoundsAnalysis extends BufferWriteEstimationReason, TTypeBoundsAnalysis {
-  override string toString() { result = "TypeBoundsAnalysis" }
-
-  override string getDescription() { result = "based on type bounds" }
-
-  override BufferWriteEstimationReason combineWith(BufferWriteEstimationReason other) {
-    other != TNoSpecifiedEstimateReason() and result = TTypeBoundsAnalysis()
-  }
-}
-
-/**
- * The estimation comes from non trivial bounds found via actual flow analysis.
- * For example
- * ```
- * unsigned u = x;
- * if (u < 1000) {
- *    //...  <- estimation done here based on u
- * }
- * ```
- */
-class ValueFlowAnalysis extends BufferWriteEstimationReason, TValueFlowAnalysis {
-  override string toString() { result = "ValueFlowAnalysis" }
-
-  override string getDescription() { result = "based on flow analysis of value bounds" }
-
-  override BufferWriteEstimationReason combineWith(BufferWriteEstimationReason other) {
-    other != TNoSpecifiedEstimateReason() and result = other
-  }
-}
-
 class PrintfFormatAttribute extends FormatAttribute {
   PrintfFormatAttribute() { this.getArchetype() = ["printf", "__printf__"] }
 }
@@ -1067,14 +990,7 @@ class FormatLiteral extends Literal {
    * conversion specifier of this format string; has no result if this cannot
    * be determined.
    */
-  int getMaxConvertedLength(int n) { result = max(getMaxConvertedLength(n, _)) }
-
-  /**
-   * Gets the maximum length of the string that can be produced by the nth
-   * conversion specifier of this format string, specifying the estimation reason;
-   * has no result if this cannot be determined.
-   */
-  int getMaxConvertedLength(int n, BufferWriteEstimationReason reason) {
+  int getMaxConvertedLength(int n) {
     exists(int len |
       (
         (
@@ -1086,12 +1002,10 @@ class FormatLiteral extends Literal {
       ) and
       (
         this.getConversionChar(n) = "%" and
-        len = 1 and
-        reason = TValueFlowAnalysis()
+        len = 1
         or
         this.getConversionChar(n).toLowerCase() = "c" and
-        len = 1 and
-        reason = TValueFlowAnalysis() // e.g. 'a'
+        len = 1 // e.g. 'a'
         or
         this.getConversionChar(n).toLowerCase() = "f" and
         exists(int dot, int afterdot |
@@ -1105,8 +1019,7 @@ class FormatLiteral extends Literal {
             afterdot = 6
           ) and
           len = 1 + 309 + dot + afterdot
-        ) and
-        reason = TTypeBoundsAnalysis() // e.g. -1e308="-100000"...
+        ) // e.g. -1e308="-100000"...
         or
         this.getConversionChar(n).toLowerCase() = "e" and
         exists(int dot, int afterdot |
@@ -1120,8 +1033,7 @@ class FormatLiteral extends Literal {
             afterdot = 6
           ) and
           len = 1 + 1 + dot + afterdot + 1 + 1 + 3
-        ) and
-        reason = TTypeBoundsAnalysis() // -1e308="-1.000000e+308"
+        ) // -1e308="-1.000000e+308"
         or
         this.getConversionChar(n).toLowerCase() = "g" and
         exists(int dot, int afterdot |
@@ -1144,80 +1056,67 @@ class FormatLiteral extends Literal {
           //       (e.g. 123456, 0.000123456 are just OK)
           //       so case %f can be at most P characters + 4 zeroes, sign, dot = P + 6
           len = (afterdot.maximum(1) + 6).maximum(1 + 1 + dot + afterdot + 1 + 1 + 3)
-        ) and
-        reason = TTypeBoundsAnalysis() // (e.g. "-1.59203e-319")
+        ) // (e.g. "-1.59203e-319")
         or
         this.getConversionChar(n).toLowerCase() = ["d", "i"] and
         // e.g. -2^31 = "-2147483648"
-        exists(float typeBasedBound, float valueBasedBound |
-          // The first case handles length sub-specifiers
-          // Subtract one in the exponent because one bit is for the sign.
-          // Add 1 to account for the possible sign in the output.
-          typeBasedBound =
-            1 + lengthInBase10(2.pow(this.getIntegralDisplayType(n).getSize() * 8 - 1)) and
-          // The second case uses range analysis to deduce a length that's shorter than the length
-          // of the number -2^31.
-          exists(Expr arg, float lower, float upper, float typeLower, float typeUpper |
-            arg = this.getUse().getConversionArgument(n) and
-            lower = lowerBound(arg.getFullyConverted()) and
-            upper = upperBound(arg.getFullyConverted()) and
-            typeLower = exprMinVal(arg.getFullyConverted()) and
-            typeUpper = exprMaxVal(arg.getFullyConverted())
-          |
-            valueBasedBound =
-              max(int cand |
-                // Include the sign bit in the length if it can be negative
-                (
-                  if lower < 0
-                  then cand = 1 + lengthInBase10(lower.abs())
-                  else cand = lengthInBase10(lower)
+        len =
+          min(float cand |
+            // The first case handles length sub-specifiers
+            // Subtract one in the exponent because one bit is for the sign.
+            // Add 1 to account for the possible sign in the output.
+            cand = 1 + lengthInBase10(2.pow(this.getIntegralDisplayType(n).getSize() * 8 - 1))
+            or
+            // The second case uses range analysis to deduce a length that's shorter than the length
+            // of the number -2^31.
+            exists(Expr arg, float lower, float upper |
+              arg = this.getUse().getConversionArgument(n) and
+              lower = lowerBound(arg.getFullyConverted()) and
+              upper = upperBound(arg.getFullyConverted())
+            |
+              cand =
+                max(int cand0 |
+                  // Include the sign bit in the length if it can be negative
+                  (
+                    if lower < 0
+                    then cand0 = 1 + lengthInBase10(lower.abs())
+                    else cand0 = lengthInBase10(lower)
+                  )
+                  or
+                  (
+                    if upper < 0
+                    then cand0 = 1 + lengthInBase10(upper.abs())
+                    else cand0 = lengthInBase10(upper)
+                  )
                 )
-                or
-                (
-                  if upper < 0
-                  then cand = 1 + lengthInBase10(upper.abs())
-                  else cand = lengthInBase10(upper)
-                )
-              ) and
-            (
-              if lower > typeLower or upper < typeUpper
-              then reason = TValueFlowAnalysis()
-              else reason = TTypeBoundsAnalysis()
             )
-          ) and
-          len = valueBasedBound.minimum(typeBasedBound)
-        )
+          )
         or
         this.getConversionChar(n).toLowerCase() = "u" and
         // e.g. 2^32 - 1 = "4294967295"
-        exists(float typeBasedBound, float valueBasedBound |
-          // The first case handles length sub-specifiers
-          typeBasedBound = lengthInBase10(2.pow(this.getIntegralDisplayType(n).getSize() * 8) - 1) and
-          // The second case uses range analysis to deduce a length that's shorter than
-          // the length of the number 2^31 - 1.
-          exists(Expr arg, float lower, float upper, float typeLower, float typeUpper |
-            arg = this.getUse().getConversionArgument(n) and
-            lower = lowerBound(arg.getFullyConverted()) and
-            upper = upperBound(arg.getFullyConverted()) and
-            typeLower = exprMinVal(arg.getFullyConverted()) and
-            typeUpper = exprMaxVal(arg.getFullyConverted())
-          |
-            valueBasedBound =
-              lengthInBase10(max(float cand |
+        len =
+          min(float cand |
+            // The first case handles length sub-specifiers
+            cand = 2.pow(this.getIntegralDisplayType(n).getSize() * 8)
+            or
+            // The second case uses range analysis to deduce a length that's shorter than
+            // the length of the number 2^31 - 1.
+            exists(Expr arg, float lower |
+              arg = this.getUse().getConversionArgument(n) and
+              lower = lowerBound(arg.getFullyConverted())
+            |
+              cand =
+                max(float cand0 |
                   // If lower can be negative we use `(unsigned)-1` as the candidate value.
                   lower < 0 and
-                  cand = 2.pow(any(IntType t | t.isUnsigned()).getSize() * 8)
+                  cand0 = 2.pow(any(IntType t | t.isUnsigned()).getSize() * 8)
                   or
-                  cand = upper
-                )) and
-            (
-              if lower > typeLower or upper < typeUpper
-              then reason = TValueFlowAnalysis()
-              else reason = TTypeBoundsAnalysis()
+                  cand0 = upperBound(arg.getFullyConverted())
+                )
             )
-          ) and
-          len = valueBasedBound.minimum(typeBasedBound)
-        )
+          |
+            lengthInBase10(cand)
+          )
         or
         this.getConversionChar(n).toLowerCase() = "x" and
         // e.g. "12345678"
@@ -1236,8 +1135,7 @@ class FormatLiteral extends Literal {
           (
             if this.hasAlternateFlag(n) then len = 2 + baseLen else len = baseLen // "0x"
           )
-        ) and
-        reason = TTypeBoundsAnalysis()
+        )
         or
         this.getConversionChar(n).toLowerCase() = "p" and
         exists(PointerType ptrType, int baseLen |
@@ -1246,8 +1144,7 @@ class FormatLiteral extends Literal {
           (
             if this.hasAlternateFlag(n) then len = 2 + baseLen else len = baseLen // "0x"
           )
-        ) and
-        reason = TValueFlowAnalysis()
+        )
         or
         this.getConversionChar(n).toLowerCase() = "o" and
         // e.g. 2^32 - 1 = "37777777777"
@@ -1266,16 +1163,14 @@ class FormatLiteral extends Literal {
           (
             if this.hasAlternateFlag(n) then len = 1 + baseLen else len = baseLen // "0"
           )
-        ) and
-        reason = TTypeBoundsAnalysis()
+        )
         or
         this.getConversionChar(n).toLowerCase() = "s" and
         len =
           min(int v |
             v = this.getPrecision(n) or
             v = this.getUse().getFormatArgument(n).(AnalysedString).getMaxLength() - 1 // (don't count null terminator)
-          ) and
-        reason = TValueFlowAnalysis()
+          )
       )
     )
   }
@@ -1287,19 +1182,10 @@ class FormatLiteral extends Literal {
    * determining whether a buffer overflow is caused by long float to string
    * conversions.
    */
-  int getMaxConvertedLengthLimited(int n) { result = max(getMaxConvertedLengthLimited(n, _)) }
-
-  /**
-   * Gets the maximum length of the string that can be produced by the nth
-   * conversion specifier of this format string, specifying the reason for the
-   * estimation, except that float to string conversions are assumed to be 8
-   * characters.  This is helpful for determining whether a buffer overflow is
-   * caused by long float to string conversions.
-   */
-  int getMaxConvertedLengthLimited(int n, BufferWriteEstimationReason reason) {
+  int getMaxConvertedLengthLimited(int n) {
     if this.getConversionChar(n).toLowerCase() = "f"
-    then result = this.getMaxConvertedLength(n, reason).minimum(8)
-    else result = this.getMaxConvertedLength(n, reason)
+    then result = this.getMaxConvertedLength(n).minimum(8)
+    else result = this.getMaxConvertedLength(n)
   }
 
   /**
@@ -1339,35 +1225,29 @@ class FormatLiteral extends Literal {
     )
   }
 
-  private int getMaxConvertedLengthAfter(int n, BufferWriteEstimationReason reason) {
+  private int getMaxConvertedLengthAfter(int n) {
     if n = this.getNumConvSpec()
-    then result = this.getConstantSuffix().length() + 1 and reason = TValueFlowAnalysis()
+    then result = this.getConstantSuffix().length() + 1
     else
-      exists(BufferWriteEstimationReason headReason, BufferWriteEstimationReason tailReason |
-        result =
-          this.getConstantPart(n).length() + this.getMaxConvertedLength(n, headReason) +
-            this.getMaxConvertedLengthAfter(n + 1, tailReason) and
-        reason = headReason.combineWith(tailReason)
-      )
+      result =
+        this.getConstantPart(n).length() + this.getMaxConvertedLength(n) +
+          this.getMaxConvertedLengthAfter(n + 1)
   }
 
-  private int getMaxConvertedLengthAfterLimited(int n, BufferWriteEstimationReason reason) {
+  private int getMaxConvertedLengthAfterLimited(int n) {
     if n = this.getNumConvSpec()
-    then result = this.getConstantSuffix().length() + 1 and reason = TValueFlowAnalysis()
+    then result = this.getConstantSuffix().length() + 1
     else
-      exists(BufferWriteEstimationReason headReason, BufferWriteEstimationReason tailReason |
-        result =
-          this.getConstantPart(n).length() + this.getMaxConvertedLengthLimited(n, headReason) +
-            this.getMaxConvertedLengthAfterLimited(n + 1, tailReason) and
-        reason = headReason.combineWith(tailReason)
-      )
+      result =
+        this.getConstantPart(n).length() + this.getMaxConvertedLengthLimited(n) +
+          this.getMaxConvertedLengthAfterLimited(n + 1)
   }
 
   /**
    * Gets the maximum length of the string that can be produced by this format
    * string.  Has no result if this cannot be determined.
    */
-  int getMaxConvertedLength() { result = this.getMaxConvertedLengthAfter(0, _) }
+  int getMaxConvertedLength() { result = this.getMaxConvertedLengthAfter(0) }
 
   /**
    * Gets the maximum length of the string that can be produced by this format
@@ -1375,24 +1255,5 @@ class FormatLiteral extends Literal {
    * characters.  This is helpful for determining whether a buffer overflow
    * is caused by long float to string conversions.
    */
-  int getMaxConvertedLengthLimited() { result = this.getMaxConvertedLengthAfterLimited(0, _) }
-
-  /**
-   * Gets the maximum length of the string that can be produced by this format
-   * string, specifying the reason for the estimate. Has no result if no estimate
-   * can be found.
-   */
-  int getMaxConvertedLengthWithReason(BufferWriteEstimationReason reason) {
-    result = this.getMaxConvertedLengthAfter(0, reason)
-  }
-
-  /**
-   * Gets the maximum length of the string that can be produced by this format
-   * string, specifying the reason for the estimate, except that float to string
-   * conversions are assumed to be 8 characters.  This is helpful for determining
-   * whether a buffer overflow is caused by long float to string conversions.
-   */
-  int getMaxConvertedLengthLimitedWithReason(BufferWriteEstimationReason reason) {
-    result = this.getMaxConvertedLengthAfterLimited(0, reason)
-  }
+  int getMaxConvertedLengthLimited() { result = this.getMaxConvertedLengthAfterLimited(0) }
 }

cpp/ql/lib/semmle/code/cpp/security/BufferWrite.qll

@@ -71,30 +71,13 @@ abstract class BufferWrite extends Expr {
    */
   int getMaxData() { none() }
 
-  /**
-   * Gets an upper bound to the amount of data that's being written (if one
-   * can be found), specifying the reason for the estimation.
-   */
-  int getMaxData(BufferWriteEstimationReason reason) {
-    reason instanceof NoSpecifiedEstimateReason and result = getMaxData()
-  }
-
   /**
    * Gets an upper bound to the amount of data that's being written (if one
    * can be found), except that float to string conversions are assumed to be
    * much smaller (8 bytes) than their true maximum length.  This can be
    * helpful in determining the cause of a buffer overflow issue.
    */
-  int getMaxDataLimited() { result = getMaxData() }
-
-  /**
-   * Gets an upper bound to the amount of data that's being written (if one
-   * can be found), specifying the reason for the estimation, except that
-   * float to string conversions are assumed to be much smaller (8 bytes)
-   * than their true maximum length.  This can be helpful in determining the
-   * cause of a buffer overflow issue.
-   */
-  int getMaxDataLimited(BufferWriteEstimationReason reason) { result = getMaxData(reason) }
+  int getMaxDataLimited() { result = this.getMaxData() }
 
   /**
    * Gets the size of a single character of the type this
@@ -152,16 +135,10 @@ class StrCopyBW extends BufferWriteCall {
     result = this.getArgument(this.getParamSize()).getValue().toInt() * this.getCharSize()
   }
 
-  private int getMaxDataImpl(BufferWriteEstimationReason reason) {
-    // when result exists, it is an exact flow analysis
-    reason instanceof ValueFlowAnalysis and
+  override int getMaxData() {
     result =
       this.getArgument(this.getParamSrc()).(AnalysedString).getMaxLength() * this.getCharSize()
   }
-
-  override int getMaxData(BufferWriteEstimationReason reason) { result = getMaxDataImpl(reason) }
-
-  override int getMaxData() { result = max(getMaxDataImpl(_)) }
 }
 
 /**
@@ -196,16 +173,10 @@ class StrCatBW extends BufferWriteCall {
     result = this.getArgument(this.getParamSize()).getValue().toInt() * this.getCharSize()
   }
 
-  private int getMaxDataImpl(BufferWriteEstimationReason reason) {
-    // when result exists, it is an exact flow analysis
-    reason instanceof ValueFlowAnalysis and
+  override int getMaxData() {
     result =
       this.getArgument(this.getParamSrc()).(AnalysedString).getMaxLength() * this.getCharSize()
   }
-
-  override int getMaxData(BufferWriteEstimationReason reason) { result = getMaxDataImpl(reason) }
-
-  override int getMaxData() { result = max(getMaxDataImpl(_)) }
 }
 
 /**
@@ -262,29 +233,19 @@ class SprintfBW extends BufferWriteCall {
 
   override Expr getDest() { result = this.getArgument(f.getOutputParameterIndex(false)) }
 
-  private int getMaxDataImpl(BufferWriteEstimationReason reason) {
+  override int getMaxData() {
     exists(FormatLiteral fl |
       fl = this.(FormattingFunctionCall).getFormat() and
-      result = fl.getMaxConvertedLengthWithReason(reason) * this.getCharSize()
+      result = fl.getMaxConvertedLength() * this.getCharSize()
     )
   }
 
-  override int getMaxData(BufferWriteEstimationReason reason) { result = getMaxDataImpl(reason) }
-
-  override int getMaxData() { result = max(getMaxDataImpl(_)) }
-
-  private int getMaxDataLimitedImpl(BufferWriteEstimationReason reason) {
+  override int getMaxDataLimited() {
     exists(FormatLiteral fl |
       fl = this.(FormattingFunctionCall).getFormat() and
-      result = fl.getMaxConvertedLengthLimitedWithReason(reason) * this.getCharSize()
+      result = fl.getMaxConvertedLengthLimited() * this.getCharSize()
     )
   }
-
-  override int getMaxDataLimited(BufferWriteEstimationReason reason) {
-    result = getMaxDataLimitedImpl(reason)
-  }
-
-  override int getMaxDataLimited() { result = max(getMaxDataLimitedImpl(_)) }
 }
 
 /**
@@ -375,29 +336,19 @@ class SnprintfBW extends BufferWriteCall {
     result = this.getArgument(this.getParamSize()).getValue().toInt() * this.getCharSize()
   }
 
-  private int getMaxDataImpl(BufferWriteEstimationReason reason) {
+  override int getMaxData() {
     exists(FormatLiteral fl |
       fl = this.(FormattingFunctionCall).getFormat() and
-      result = fl.getMaxConvertedLengthWithReason(reason) * this.getCharSize()
+      result = fl.getMaxConvertedLength() * this.getCharSize()
     )
   }
 
-  override int getMaxData(BufferWriteEstimationReason reason) { result = getMaxDataImpl(reason) }
-
-  override int getMaxData() { result = max(getMaxDataImpl(_)) }
-
-  private int getMaxDataLimitedImpl(BufferWriteEstimationReason reason) {
+  override int getMaxDataLimited() {
     exists(FormatLiteral fl |
       fl = this.(FormattingFunctionCall).getFormat() and
-      result = fl.getMaxConvertedLengthLimitedWithReason(reason) * this.getCharSize()
+      result = fl.getMaxConvertedLengthLimited() * this.getCharSize()
     )
   }
-
-  override int getMaxDataLimited(BufferWriteEstimationReason reason) {
-    result = getMaxDataLimitedImpl(reason)
-  }
-
-  override int getMaxDataLimited() { result = max(getMaxDataLimitedImpl(_)) }
 }
 
 /**
@@ -485,9 +436,7 @@ class ScanfBW extends BufferWrite {
 
   override Expr getDest() { result = this }
 
-  private int getMaxDataImpl(BufferWriteEstimationReason reason) {
-    // when this returns, it is based on exact flow analysis
-    reason instanceof ValueFlowAnalysis and
+  override int getMaxData() {
     exists(ScanfFunctionCall fc, ScanfFormatLiteral fl, int arg |
       this = fc.getArgument(arg) and
       fl = fc.getFormat() and
@@ -495,10 +444,6 @@ class ScanfBW extends BufferWrite {
     )
   }
 
-  override int getMaxData(BufferWriteEstimationReason reason) { result = getMaxDataImpl(reason) }
-
-  override int getMaxData() { result = max(getMaxDataImpl(_)) }
-
   override string getBWDesc() {
     exists(FunctionCall fc |
       this = fc.getArgument(_) and
@@ -529,14 +474,8 @@ class RealpathBW extends BufferWriteCall {
 
   override Expr getASource() { result = this.getArgument(0) }
 
-  private int getMaxDataImpl(BufferWriteEstimationReason reason) {
-    // although there may be some unknown invariants guaranteeing that a real path is shorter than PATH_MAX, we can consider providing less than PATH_MAX a problem with high precision
-    reason instanceof ValueFlowAnalysis and
+  override int getMaxData() {
     result = path_max() and
     this = this // Suppress a compiler warning
   }
-
-  override int getMaxData(BufferWriteEstimationReason reason) { result = getMaxDataImpl(reason) }
-
-  override int getMaxData() { result = max(getMaxDataImpl(_)) }
 }

cpp/ql/src/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 0.0.5
+
+### New Queries
+
+* A new query `cpp/certificate-not-checked` has been added for C/C++. The query flags unsafe use of OpenSSL and similar libraries.
+* A new query `cpp/certificate-result-conflation` has been added for C/C++. The query flags unsafe use of OpenSSL and similar libraries.
+
 ## 0.0.4
 
 ### New Queries

cpp/ql/src/Likely Bugs/Arithmetic/ComparisonWithCancelingSubExpr.ql

@@ -85,8 +85,7 @@ private predicate cancelingSubExprs(ComparisonOperation cmp, VariableAccess a1,
   exists(Variable v |
     exists(float m | m < 0 and cmpLinearSubVariable(cmp, v, a1, m)) and
     exists(float m | m > 0 and cmpLinearSubVariable(cmp, v, a2, m))
-  ) and
-  not any(ClassTemplateInstantiation inst).getATemplateArgument() = cmp.getParent*()
+  )
 }
 
 from ComparisonOperation cmp, VariableAccess a1, VariableAccess a2

cpp/ql/src/Likely Bugs/Arithmetic/PointlessSelfComparison.qll

@@ -29,9 +29,7 @@ predicate pointlessSelfComparison(ComparisonOperation cmp) {
     not exists(lhs.getQualifier()) and // Avoid structure fields
     not exists(rhs.getQualifier()) and // Avoid structure fields
     not convertedExprMightOverflow(lhs) and
-    not convertedExprMightOverflow(rhs) and
-    // Don't warn if the comparison is part of a template argument.
-    not any(ClassTemplateInstantiation inst).getATemplateArgument() = cmp.getParent*()
+    not convertedExprMightOverflow(rhs)
   )
 }
 

cpp/ql/src/Security/CWE/CWE-120/OverrunWrite.ql

@@ -21,15 +21,14 @@ import semmle.code.cpp.commons.Alloc
  * See CWE-120/UnboundedWrite.ql for a summary of CWE-120 alert cases.
  */
 
-from BufferWrite bw, Expr dest, int destSize, int estimated
+from BufferWrite bw, Expr dest, int destSize
 where
   not bw.hasExplicitLimit() and // has no explicit size limit
   dest = bw.getDest() and
   destSize = getBufferSize(dest, _) and
-  estimated = bw.getMaxDataLimited(_) and
   // we can deduce that too much data may be copied (even without
   // long '%f' conversions)
-  estimated > destSize
+  bw.getMaxDataLimited() > destSize
 select bw,
-  "This '" + bw.getBWDesc() + "' operation requires " + estimated +
+  "This '" + bw.getBWDesc() + "' operation requires " + bw.getMaxData() +
     " bytes but the destination is only " + destSize + " bytes."

cpp/ql/src/Security/CWE/CWE-120/OverrunWriteFloat.ql

@@ -21,15 +21,14 @@ import semmle.code.cpp.security.BufferWrite
  * See CWE-120/UnboundedWrite.ql for a summary of CWE-120 alert cases.
  */
 
-from BufferWrite bw, int destSize, int estimated, BufferWriteEstimationReason reason
+from BufferWrite bw, int destSize
 where
   not bw.hasExplicitLimit() and
   // has no explicit size limit
   destSize = getBufferSize(bw.getDest(), _) and
-  estimated = bw.getMaxData(reason) and
-  estimated > destSize and
+  bw.getMaxData() > destSize and
   // and we can deduce that too much data may be copied
-  bw.getMaxDataLimited(reason) <= destSize // but it would fit without long '%f' conversions
+  bw.getMaxDataLimited() <= destSize // but it would fit without long '%f' conversions
 select bw,
-  "This '" + bw.getBWDesc() + "' operation may require " + estimated +
+  "This '" + bw.getBWDesc() + "' operation may require " + bw.getMaxData() +
     " bytes because of float conversions, but the target is only " + destSize + " bytes."

cpp/ql/src/Security/CWE/CWE-120/UnboundedWrite.ql

@@ -44,7 +44,7 @@ import TaintedWithPath
 
 predicate isUnboundedWrite(BufferWrite bw) {
   not bw.hasExplicitLimit() and // has no explicit size limit
-  not exists(bw.getMaxData(_)) // and we can't deduce an upper bound to the amount copied
+  not exists(bw.getMaxData()) // and we can't deduce an upper bound to the amount copied
 }
 
 /*

cpp/ql/src/Security/CWE/CWE-319/UseOfHttp.ql

@@ -28,11 +28,6 @@ class PrivateHostName extends string {
   }
 }
 
-pragma[nomagic]
-predicate privateHostNameFlowsToExpr(Expr e) {
-  TaintTracking::localExprTaint(any(StringLiteral p | p.getValue() instanceof PrivateHostName), e)
-}
-
 /**
  * A string containing an HTTP URL not in a private domain.
  */
@@ -43,9 +38,11 @@ class HttpStringLiteral extends StringLiteral {
       or
       exists(string tail |
         tail = s.regexpCapture("http://(.*)", 1) and not tail instanceof PrivateHostName
-      )
-    ) and
-    not privateHostNameFlowsToExpr(this.getParent*())
+      ) and
+      not TaintTracking::localExprTaint(any(StringLiteral p |
+          p.getValue() instanceof PrivateHostName
+        ), this.getParent*())
+    )
   }
 }
 

cpp/ql/src/change-notes/released/0.0.5.md

@@ -0,0 +1,6 @@
+## 0.0.5
+
+### New Queries
+
+* A new query `cpp/certificate-not-checked` has been added for C/C++. The query flags unsafe use of OpenSSL and similar libraries.
+* A new query `cpp/certificate-result-conflation` has been added for C/C++. The query flags unsafe use of OpenSSL and similar libraries.

cpp/ql/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.0.4
+lastReleaseVersion: 0.0.5

cpp/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-queries
-version: 0.0.5-dev
+version: 0.0.6-dev
 groups: cpp
 dependencies:
     codeql/cpp-all: "*"

cpp/ql/test/query-tests/Likely Bugs/Arithmetic/BadAdditionOverflowCheck/templates.cpp

@@ -20,22 +20,3 @@ bool compareValues() {
 bool callCompareValues() {
   return compareValues<C1, C2> || compareValues<C1, C1>();
 }
-
-template <bool C, typename T = void>
-struct enable_if {};
-
-template <typename T>
-struct enable_if<true, T> { typedef T type; };
-
-template<typename T1, typename T2>
-typename enable_if<T1::value <= T2::value, bool>::type constant_comparison() {
-  return true;
-}
-
-struct Value0 {
-  const static int value = 0;
-};
-
-void instantiation_with_pointless_comparison() {
-  constant_comparison<Value0, Value0>(); // GOOD
-}

cpp/upgrades/CHANGELOG.md

@@ -1 +1,3 @@
+## 0.0.5
+
 ## 0.0.4

cpp/upgrades/change-notes/released/0.0.5.md

@@ -0,0 +1 @@
+## 0.0.5

cpp/upgrades/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.0.4
+lastReleaseVersion: 0.0.5

cpp/upgrades/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-upgrades
 groups: cpp
 upgrades: .
-version: 0.0.5-dev
+version: 0.0.6-dev
 library: true

csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/Semmle.Autobuild.CSharp.Tests.csproj

@@ -1,22 +1,25 @@
 <Project Sdk="Microsoft.NET.Sdk">
-	<PropertyGroup>
-		<TargetFramework>net5.0</TargetFramework>
-		<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
-		<RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
-		<Nullable>enable</Nullable>
-	</PropertyGroup>
-	<ItemGroup>
-		<PackageReference Include="System.IO.FileSystem" Version="4.3.0"/>
-		<PackageReference Include="System.IO.FileSystem.Primitives" Version="4.3.0"/>
-		<PackageReference Include="xunit" Version="2.4.1"/>
-		<PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
-			<PrivateAssets>all</PrivateAssets>
-			<IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
-		</PackageReference>
-		<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.0.0"/>
-	</ItemGroup>
-	<ItemGroup>
-		<ProjectReference Include="..\Semmle.Autobuild.CSharp\Semmle.Autobuild.CSharp.csproj"/>
-		<ProjectReference Include="..\Semmle.Autobuild.Shared\Semmle.Autobuild.Shared.csproj"/>
-	</ItemGroup>
-</Project>
+
+  <PropertyGroup>
+    <TargetFramework>net5.0</TargetFramework>
+    <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
+    <RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="System.IO.FileSystem" Version="4.3.0" />
+    <PackageReference Include="System.IO.FileSystem.Primitives" Version="4.3.0" />
+    <PackageReference Include="xunit" Version="2.4.1" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
+    </PackageReference>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.9.1" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Semmle.Autobuild.CSharp\Semmle.Autobuild.CSharp.csproj" />
+    <ProjectReference Include="..\Semmle.Autobuild.Shared\Semmle.Autobuild.Shared.csproj" />
+  </ItemGroup>
+</Project>

csharp/autobuilder/Semmle.Autobuild.CSharp/Semmle.Autobuild.CSharp.csproj

@@ -1,25 +1,30 @@
 <Project Sdk="Microsoft.NET.Sdk">
-	<PropertyGroup>
-		<TargetFramework>net5.0</TargetFramework>
-		<AssemblyName>Semmle.Autobuild.CSharp</AssemblyName>
-		<RootNamespace>Semmle.Autobuild.CSharp</RootNamespace>
-		<ApplicationIcon/>
-		<OutputType>Exe</OutputType>
-		<StartupObject/>
-		<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
-		<RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
-		<Nullable>enable</Nullable>
-	</PropertyGroup>
-	<ItemGroup>
-		<Folder Include="Properties\"/>
-	</ItemGroup>
-	<ItemGroup>
-		<PackageReference Include="Microsoft.Build" Version="16.11.0"/>
-		<PackageReference Include="Newtonsoft.Json" Version="13.0.1"/>
-	</ItemGroup>
-	<ItemGroup>
-		<ProjectReference Include="..\..\extractor\Semmle.Util\Semmle.Util.csproj"/>
-		<ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp\Semmle.Extraction.CSharp.csproj"/>
-		<ProjectReference Include="..\Semmle.Autobuild.Shared\Semmle.Autobuild.Shared.csproj"/>
-	</ItemGroup>
-</Project>
+
+  <PropertyGroup>
+    <TargetFramework>net5.0</TargetFramework>
+    <AssemblyName>Semmle.Autobuild.CSharp</AssemblyName>
+    <RootNamespace>Semmle.Autobuild.CSharp</RootNamespace>
+    <ApplicationIcon />
+    <OutputType>Exe</OutputType>
+    <StartupObject />
+    <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
+    <RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Folder Include="Properties\" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Build" Version="16.9.0" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\extractor\Semmle.Util\Semmle.Util.csproj" />
+    <ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp\Semmle.Extraction.CSharp.csproj" />
+    <ProjectReference Include="..\Semmle.Autobuild.Shared\Semmle.Autobuild.Shared.csproj" />
+  </ItemGroup>
+
+</Project>

csharp/autobuilder/Semmle.Autobuild.Shared/Semmle.Autobuild.Shared.csproj

@@ -1,19 +1,24 @@
 <Project Sdk="Microsoft.NET.Sdk">
-	<PropertyGroup>
-		<TargetFramework>net5.0</TargetFramework>
-		<AssemblyName>Semmle.Autobuild.Shared</AssemblyName>
-		<RootNamespace>Semmle.Autobuild.Shared</RootNamespace>
-		<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
-		<RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
-		<Nullable>enable</Nullable>
-	</PropertyGroup>
-	<ItemGroup>
-		<Folder Include="Properties\"/>
-	</ItemGroup>
-	<ItemGroup>
-		<PackageReference Include="Microsoft.Build" Version="16.11.0"/>
-	</ItemGroup>
-	<ItemGroup>
-		<ProjectReference Include="..\..\extractor\Semmle.Util\Semmle.Util.csproj"/>
-	</ItemGroup>
-</Project>
+
+  <PropertyGroup>
+    <TargetFramework>net5.0</TargetFramework>
+    <AssemblyName>Semmle.Autobuild.Shared</AssemblyName>
+    <RootNamespace>Semmle.Autobuild.Shared</RootNamespace>
+    <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
+    <RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Folder Include="Properties\" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Build" Version="16.9.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\extractor\Semmle.Util\Semmle.Util.csproj" />
+  </ItemGroup>
+
+</Project>

csharp/documentation/library-coverage/coverage.csv

@@ -1,7 +1,6 @@
-package,sink,source,summary,sink:code,sink:html,sink:remote,sink:sql,sink:xss,source:local,summary:taint,summary:value
-Dapper,55,,,,,,55,,,,
-Microsoft.ApplicationBlocks.Data,28,,,,,,28,,,,
-MySql.Data.MySqlClient,48,,,,,,48,,,,
-Newtonsoft.Json,,,73,,,,,,,73,
-ServiceStack,194,,7,27,,75,92,,,7,
-System,28,3,1221,,4,,23,1,3,611,610
+package,sink,source,summary,sink:code,sink:html,sink:remote,sink:sql,sink:xss,source:local,summary:taint
+Dapper,55,,,,,,55,,,
+Microsoft.ApplicationBlocks.Data,28,,,,,,28,,,
+MySql.Data.MySqlClient,48,,,,,,48,,,
+ServiceStack,194,,7,27,,75,92,,,7
+System,28,3,25,,4,,23,1,3,25

csharp/documentation/library-coverage/coverage.rst

@@ -8,7 +8,7 @@ C# framework & library support
 
    Framework / library,Package,Flow sources,Taint & value steps,Sinks (total),`CWE-079` :sub:`Cross-site scripting`
    `ServiceStack <https://servicestack.net/>`_,"``ServiceStack.*``, ``ServiceStack``",,7,194,
-   System,"``System.*``, ``System``",3,1221,28,5
-   Others,"``Dapper``, ``Microsoft.ApplicationBlocks.Data``, ``MySql.Data.MySqlClient``, ``Newtonsoft.Json``",,73,131,
-   Totals,,3,1301,353,5
+   System,"``System.*``, ``System``",3,25,28,5
+   Others,"``Dapper``, ``Microsoft.ApplicationBlocks.Data``, ``MySql.Data.MySqlClient``",,,131,
+   Totals,,3,32,353,5
 

csharp/extractor/Semmle.Extraction.CIL/Semmle.Extraction.CIL.csproj

@@ -26,9 +26,7 @@
   <ItemGroup>
     <PackageReference Include="Microsoft.DiaSymReader" Version="1.3.0" />
     <PackageReference Include="Microsoft.DiaSymReader.Native" Version="1.7.0" />
-    <PackageReference Include="Microsoft.DiaSymReader.PortablePdb" Version="1.6.0"><IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
-<PrivateAssets>all</PrivateAssets>
-</PackageReference>
+    <PackageReference Include="Microsoft.DiaSymReader.PortablePdb" Version="1.5.0" />
   </ItemGroup>
 
 </Project>

csharp/extractor/Semmle.Extraction.CSharp.Standalone/Semmle.Extraction.CSharp.Standalone.csproj

@@ -1,28 +1,33 @@
-<Project Sdk="Microsoft.NET.Sdk">
-	<PropertyGroup>
-		<OutputType>Exe</OutputType>
-		<TargetFramework>net5.0</TargetFramework>
-		<AssemblyName>Semmle.Extraction.CSharp.Standalone</AssemblyName>
-		<RootNamespace>Semmle.Extraction.CSharp.Standalone</RootNamespace>
-		<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
-		<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
-		<TreatWarningsAsErrors>false</TreatWarningsAsErrors>
-		<WarningsAsErrors/>
-		<RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
-		<Nullable>enable</Nullable>
-	</PropertyGroup>
-	<ItemGroup>
-		<ProjectReference Include="..\Semmle.Extraction.CSharp\Semmle.Extraction.CSharp.csproj"/>
-		<ProjectReference Include="..\Semmle.Util\Semmle.Util.csproj"/>
-	</ItemGroup>
-	<ItemGroup>
-		<Folder Include="Properties\"/>
-	</ItemGroup>
-	<ItemGroup>
-		<PackageReference Include="Microsoft.Build" Version="16.11.0"/>
-		<PackageReference Include="Microsoft.Win32.Primitives" Version="4.3.0"/>
-		<PackageReference Include="System.Net.Primitives" Version="4.3.1"/>
-		<PackageReference Include="System.Security.Principal" Version="4.3.0"/>
-		<PackageReference Include="System.Threading.ThreadPool" Version="4.3.0"/>
-	</ItemGroup>
-</Project>
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net5.0</TargetFramework>
+    <AssemblyName>Semmle.Extraction.CSharp.Standalone</AssemblyName>
+    <RootNamespace>Semmle.Extraction.CSharp.Standalone</RootNamespace>
+    <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+    <TreatWarningsAsErrors>false</TreatWarningsAsErrors>
+    <WarningsAsErrors />
+    <RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Semmle.Extraction.CSharp\Semmle.Extraction.CSharp.csproj" />
+    <ProjectReference Include="..\Semmle.Util\Semmle.Util.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Folder Include="Properties\" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Build" Version="16.9.0" />
+    <PackageReference Include="Microsoft.Win32.Primitives" Version="4.3.0" />
+    <PackageReference Include="System.Net.Primitives" Version="4.3.1" />
+    <PackageReference Include="System.Security.Principal" Version="4.3.0" />
+    <PackageReference Include="System.Threading.ThreadPool" Version="4.3.0" />
+  </ItemGroup>
+
+</Project>

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Lambda.cs

@@ -18,7 +18,6 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
         private void VisitParameter(ParameterSyntax p)
         {
             var symbol = Context.GetModel(p).GetDeclaredSymbol(p)!;
-            Context.CacheLambdaParameterSymbol(symbol, p);
             Parameter.Create(Context, symbol, this);
         }
 

csharp/extractor/Semmle.Extraction.CSharp/Entities/Parameter.cs

@@ -55,17 +55,11 @@ namespace Semmle.Extraction.CSharp.Entities
             }
         }
 
-        public static Parameter Create(Context cx, IParameterSymbol param, IEntity parent, Parameter? original = null)
-        {
-            var cachedSymbol = cx.GetPossiblyCachedParameterSymbol(param);
-            return ParameterFactory.Instance.CreateEntity(cx, cachedSymbol, (cachedSymbol, parent, original));
-        }
+        public static Parameter Create(Context cx, IParameterSymbol param, IEntity parent, Parameter? original = null) =>
+            ParameterFactory.Instance.CreateEntity(cx, param, (param, parent, original));
 
-        public static Parameter Create(Context cx, IParameterSymbol param)
-        {
-            var cachedSymbol = cx.GetPossiblyCachedParameterSymbol(param);
-            return ParameterFactory.Instance.CreateEntity(cx, cachedSymbol, (cachedSymbol, null, null));
-        }
+        public static Parameter Create(Context cx, IParameterSymbol param) =>
+            ParameterFactory.Instance.CreateEntity(cx, param, (param, null, null));
 
         public override void WriteId(EscapingTextWriter trapFile)
         {

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Context.cs

@@ -18,56 +18,18 @@ namespace Semmle.Extraction.CSharp
         /// </summary>
         public SemanticModel GetModel(SyntaxNode node)
         {
-            if (node.SyntaxTree == SourceTree)
+            // todo: when this context belongs to a SourceScope, the syntax tree can be retrieved from the scope, and
+            // the node parameter could be removed. Is there any case when we pass in a node that's not from the current
+            // tree?
+            if (cachedModel is null || node.SyntaxTree != cachedModel.SyntaxTree)
             {
-                if (cachedModelForTree is null)
-                {
-                    cachedModelForTree = Compilation.GetSemanticModel(node.SyntaxTree);
-                }
-
-                return cachedModelForTree;
+                cachedModel = Compilation.GetSemanticModel(node.SyntaxTree);
             }
 
-            if (cachedModelForOtherTrees is null || node.SyntaxTree != cachedModelForOtherTrees.SyntaxTree)
-            {
-                cachedModelForOtherTrees = Compilation.GetSemanticModel(node.SyntaxTree);
-            }
-
-            return cachedModelForOtherTrees;
+            return cachedModel;
         }
 
-        private SemanticModel? cachedModelForTree;
-        private SemanticModel? cachedModelForOtherTrees;
-
-        // The below is a workaround to the bug reported in https://github.com/dotnet/roslyn/issues/58226
-        // Lambda parameters that are equal according to `SymbolEqualityComparer.Default`, might have different
-        // hash-codes, and as a result might not be found in `symbolEntityCache` by hash-code lookup.
-        internal IParameterSymbol GetPossiblyCachedParameterSymbol(IParameterSymbol param)
-        {
-            if ((param.ContainingSymbol as IMethodSymbol)?.MethodKind != MethodKind.AnonymousFunction)
-            {
-                return param;
-            }
-
-            foreach (var sr in param.DeclaringSyntaxReferences)
-            {
-                var syntax = sr.GetSyntax();
-                if (lambdaParameterCache.TryGetValue(syntax, out var cached) &&
-                    SymbolEqualityComparer.Default.Equals(param, cached))
-                {
-                    return cached;
-                }
-            }
-
-            return param;
-        }
-
-        internal void CacheLambdaParameterSymbol(IParameterSymbol param, SyntaxNode syntax)
-        {
-            lambdaParameterCache[syntax] = param;
-        }
-
-        private readonly Dictionary<SyntaxNode, IParameterSymbol> lambdaParameterCache = new Dictionary<SyntaxNode, IParameterSymbol>();
+        private SemanticModel? cachedModel;
 
         /// <summary>
         /// The current compilation unit.

csharp/extractor/Semmle.Extraction.CSharp/Semmle.Extraction.CSharp.csproj

@@ -1,24 +1,29 @@
 <Project Sdk="Microsoft.NET.Sdk">
-	<PropertyGroup>
-		<TargetFramework>net5.0</TargetFramework>
-		<AssemblyName>Semmle.Extraction.CSharp</AssemblyName>
-		<RootNamespace>Semmle.Extraction.CSharp</RootNamespace>
-		<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
-		<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
-		<RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
-		<RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
-		<Nullable>enable</Nullable>
-	</PropertyGroup>
-	<ItemGroup>
-		<ProjectReference Include="..\Semmle.Extraction.CIL\Semmle.Extraction.CIL.csproj"/>
-		<ProjectReference Include="..\Semmle.Extraction\Semmle.Extraction.csproj"/>
-		<ProjectReference Include="..\Semmle.Util\Semmle.Util.csproj"/>
-	</ItemGroup>
-	<ItemGroup>
-		<Folder Include="Properties\"/>
-	</ItemGroup>
-	<ItemGroup>
-		<PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="4.0.1"/>
-		<PackageReference Include="Microsoft.Build" Version="16.11.0"/>
-	</ItemGroup>
-</Project>
+
+  <PropertyGroup>
+    <TargetFramework>net5.0</TargetFramework>
+    <AssemblyName>Semmle.Extraction.CSharp</AssemblyName>
+    <RootNamespace>Semmle.Extraction.CSharp</RootNamespace>
+    <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+    <RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
+    <RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Semmle.Extraction.CIL\Semmle.Extraction.CIL.csproj" />
+    <ProjectReference Include="..\Semmle.Extraction\Semmle.Extraction.csproj" />
+    <ProjectReference Include="..\Semmle.Util\Semmle.Util.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Folder Include="Properties\" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="3.9.0" />
+    <PackageReference Include="Microsoft.Build" Version="16.9.0" />
+  </ItemGroup>
+
+</Project>

csharp/extractor/Semmle.Extraction.Tests/Semmle.Extraction.Tests.csproj

@@ -1,24 +1,28 @@
 <Project Sdk="Microsoft.NET.Sdk">
-	<PropertyGroup>
-		<TargetFramework>net5.0</TargetFramework>
-		<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
-		<RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
-		<Nullable>enable</Nullable>
-	</PropertyGroup>
-	<ItemGroup>
-		<PackageReference Include="System.IO.FileSystem" Version="4.3.0"/>
-		<PackageReference Include="System.IO.FileSystem.Primitives" Version="4.3.0"/>
-		<PackageReference Include="xunit" Version="2.4.1"/>
-		<PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
-			<PrivateAssets>all</PrivateAssets>
-			<IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
-		</PackageReference>
-		<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.0.0"/>
-	</ItemGroup>
-	<ItemGroup>
-		<ProjectReference Include="..\Semmle.Extraction.CSharp.Standalone\Semmle.Extraction.CSharp.Standalone.csproj"/>
-		<ProjectReference Include="..\Semmle.Extraction.CSharp\Semmle.Extraction.CSharp.csproj"/>
-		<ProjectReference Include="..\Semmle.Extraction\Semmle.Extraction.csproj"/>
-		<ProjectReference Include="..\Semmle.Util\Semmle.Util.csproj"/>
-	</ItemGroup>
-</Project>
+
+  <PropertyGroup>
+    <TargetFramework>net5.0</TargetFramework>
+    <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
+    <RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="System.IO.FileSystem" Version="4.3.0" />
+    <PackageReference Include="System.IO.FileSystem.Primitives" Version="4.3.0" />
+    <PackageReference Include="xunit" Version="2.4.1" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
+    </PackageReference>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.9.1" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Semmle.Extraction.CSharp.Standalone\Semmle.Extraction.CSharp.Standalone.csproj" />
+    <ProjectReference Include="..\Semmle.Extraction.CSharp\Semmle.Extraction.CSharp.csproj" />
+    <ProjectReference Include="..\Semmle.Extraction\Semmle.Extraction.csproj" />
+    <ProjectReference Include="..\Semmle.Util\Semmle.Util.csproj" />
+  </ItemGroup>
+
+</Project>

csharp/extractor/Semmle.Extraction/Semmle.Extraction.csproj

@@ -1,24 +1,29 @@
-<Project Sdk="Microsoft.NET.Sdk">
-	<PropertyGroup>
-		<TargetFramework>net5.0</TargetFramework>
-		<AssemblyName>Semmle.Extraction</AssemblyName>
-		<RootNamespace>Semmle.Extraction</RootNamespace>
-		<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
-		<CodeAnalysisRuleSet>Semmle.Extraction.ruleset</CodeAnalysisRuleSet>
-		<RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
-		<Nullable>enable</Nullable>
-	</PropertyGroup>
-	<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
-		<DefineConstants>TRACE;DEBUG;DEBUG_LABELS</DefineConstants>
-	</PropertyGroup>
-	<ItemGroup>
-		<PackageReference Include="Microsoft.CodeAnalysis" Version="4.0.1"/>
-		<PackageReference Include="GitInfo" Version="2.2.0">
-			<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
-			<PrivateAssets>all</PrivateAssets>
-		</PackageReference>
-	</ItemGroup>
-	<ItemGroup>
-		<ProjectReference Include="..\Semmle.Util\Semmle.Util.csproj"/>
-	</ItemGroup>
-</Project>
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net5.0</TargetFramework>
+    <AssemblyName>Semmle.Extraction</AssemblyName>
+    <RootNamespace>Semmle.Extraction</RootNamespace>
+    <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
+    <CodeAnalysisRuleSet>Semmle.Extraction.ruleset</CodeAnalysisRuleSet>
+    <RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
+    <DefineConstants>TRACE;DEBUG;DEBUG_LABELS</DefineConstants>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.CodeAnalysis" Version="3.9.0" />
+    <PackageReference Include="GitInfo" Version="2.1.2">
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageReference>
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Semmle.Util\Semmle.Util.csproj" />
+  </ItemGroup>
+
+</Project>

csharp/extractor/Semmle.Util.Tests/Semmle.Util.Tests.csproj

@@ -1,19 +1,23 @@
 <Project Sdk="Microsoft.NET.Sdk">
-	<PropertyGroup>
-		<TargetFramework>net5.0</TargetFramework>
-		<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
-		<RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
-		<Nullable>enable</Nullable>
-	</PropertyGroup>
-	<ItemGroup>
-		<PackageReference Include="xunit" Version="2.4.1"/>
-		<PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
-			<PrivateAssets>all</PrivateAssets>
-			<IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
-		</PackageReference>
-		<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.0.0"/>
-	</ItemGroup>
-	<ItemGroup>
-		<ProjectReference Include="..\Semmle.Util\Semmle.Util.csproj"/>
-	</ItemGroup>
-</Project>
+
+  <PropertyGroup>
+    <TargetFramework>net5.0</TargetFramework>
+    <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
+    <RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="xunit" Version="2.4.1" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
+    </PackageReference>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.9.1" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Semmle.Util\Semmle.Util.csproj" />
+  </ItemGroup>
+
+</Project>