Merge branch 'main' into python/promote-log-injection

.codeqlmanifest.json

@@ -4,8 +4,10 @@
         "*/ql/lib/qlpack.yml",
         "*/ql/test/qlpack.yml",
         "*/ql/examples/qlpack.yml",
+        "*/ql/consistency-queries/qlpack.yml",
         "cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml",
         "javascript/ql/experimental/adaptivethreatmodeling/lib/qlpack.yml",
+        "javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/qlpack.yml",
         "javascript/ql/experimental/adaptivethreatmodeling/src/qlpack.yml",
         "csharp/ql/campaigns/Solorigate/lib/qlpack.yml",
         "csharp/ql/campaigns/Solorigate/src/qlpack.yml",
@@ -13,8 +15,6 @@
         "misc/legacy-support/*/qlpack.yml",
         "misc/suite-helpers/qlpack.yml",
         "ruby/extractor-pack/codeql-extractor.yml",
-        "ruby/ql/consistency-queries/qlpack.yml",
-        "ql/ql/consistency-queries/qlpack.yml",
         "ql/extractor-pack/codeql-extractor.yml"
     ],
     "versionPolicies": {

.gitattributes

@@ -50,4 +50,15 @@
 *.pdb -text
 
 java/ql/test/stubs/**/*.java linguist-generated=true
-java/ql/test/experimental/stubs/**/*.java linguist-generated=true
+java/ql/test/experimental/stubs/**/*.java linguist-generated=true
+
+# For some languages, upgrade script testing references really old dbscheme
+# files from legacy upgrades that have CRLF line endings. Since upgrade
+# resolution relies on object hashes, we must suppress line ending conversion
+# for those testing dbscheme files.
+*/ql/lib/upgrades/initial/*.dbscheme -text
+
+# Generated test files - these are synced from the standard JavaScript libraries using
+# `javascript/ql/experimental/adaptivethreatmodeling/test/update_endpoint_test_files.py`.
+javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.js linguist-generated=true -merge
+javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.ts linguist-generated=true -merge

.github/workflows/check-change-note.yml

@@ -6,8 +6,11 @@ on:
     paths:
       - "*/ql/src/**/*.ql"
       - "*/ql/src/**/*.qll"
+      - "*/ql/lib/**/*.ql"
+      - "*/ql/lib/**/*.qll"
       - "!**/experimental/**"
       - "!ql/**"
+      - ".github/workflows/check-change-note.yml"
 
 jobs:
   check-change-note:

.github/workflows/codeql-analysis.yml

@@ -27,6 +27,11 @@ jobs:
       pull-requests: read
 
     steps:
+    - name: Setup dotnet
+      uses: actions/setup-dotnet@v1
+      with:
+        dotnet-version: 6.0.101
+
     - name: Checkout repository
       uses: actions/checkout@v2
 
@@ -51,7 +56,7 @@ jobs:
     #    uses a compiled language
 
     - run: |
-       dotnet build csharp
+       dotnet build csharp /p:UseSharedCompilation=false
 
     - name: Perform CodeQL Analysis
       uses: github/codeql-action/analyze@main

.github/workflows/csv-coverage-metrics.yml

@@ -0,0 +1,43 @@
+name: "Publish framework coverage as metrics"
+
+on:
+  schedule:
+    - cron: '5 0 * * *'
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - ".github/workflows/csv-coverage-metrics.yml"
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - name: Setup CodeQL
+        uses: ./.github/actions/fetch-codeql
+      - name: Create empty database
+        run: |
+          DATABASE="${{ runner.temp }}/java-database"
+          PROJECT="${{ runner.temp }}/java-project"
+          mkdir -p "$PROJECT/src/tmp/empty"
+          echo "class Empty {}" >> "$PROJECT/src/tmp/empty/Empty.java"
+          codeql database create "$DATABASE" --language=java --source-root="$PROJECT" --command 'javac src/tmp/empty/Empty.java'
+      - name: Capture coverage information
+        run: |
+          DATABASE="${{ runner.temp }}/java-database"
+          codeql database analyze --format=sarif-latest --output=metrics.sarif -- "$DATABASE" ./java/ql/src/Metrics/Summaries/FrameworkCoverage.ql
+      - uses: actions/upload-artifact@v2
+        with:
+          name: metrics.sarif
+          path: metrics.sarif
+          retention-days: 20
+      - name: Upload SARIF file
+        uses: github/codeql-action/upload-sarif@v1
+        with:
+          sarif_file: metrics.sarif

.github/workflows/js-ml-tests.yml

@@ -0,0 +1,67 @@
+name: JS ML-powered queries tests
+
+on:
+  push:
+    paths:
+      - "javascript/ql/experimental/adaptivethreatmodeling/**"
+      - .github/workflows/js-ml-tests.yml
+    branches:
+      - main
+      - "rc/*"
+  pull_request:
+    paths:
+      - "javascript/ql/experimental/adaptivethreatmodeling/**"
+      - .github/workflows/js-ml-tests.yml
+
+defaults:
+  run:
+    working-directory: javascript/ql/experimental/adaptivethreatmodeling
+
+jobs:
+  qlformat:
+    name: Check QL formatting
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: ./.github/actions/fetch-codeql
+
+      - name: Check QL formatting
+        run: |
+          find . "(" -name "*.ql" -or -name "*.qll" ")" -print0 | \
+            xargs -0 codeql query format --check-only
+
+  qlcompile:
+    name: Check QL compilation
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: ./.github/actions/fetch-codeql
+
+      - name: Check QL compilation
+        run: |
+          codeql query compile \
+            --check-only \
+            --ram 5120 \
+            --additional-packs "${{ github.workspace }}" \
+            --threads=0 \
+            -- \
+            lib modelbuilding src
+
+  qltest:
+    name: Run QL tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: ./.github/actions/fetch-codeql
+
+      - name: Run QL tests
+        run: |
+          codeql test run \
+            --threads=0 \
+            --ram 5120 \
+            --additional-packs "${{ github.workspace }}" \
+            -- \
+            test

.github/workflows/mad_modelDiff.yml

@@ -0,0 +1,103 @@
+name: Models as Data - Diff
+
+on:
+  workflow_dispatch:
+    inputs:
+      projects:
+        description: "The projects to generate models for"
+        required: true
+        default: '["netty/netty"]'
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "java/ql/src/utils/model-generator/**/*.*"
+      - ".github/workflows/mad_modelDiff.yml"
+
+permissions:
+  contents: read
+
+jobs:
+  model-diff:
+    name: Model Difference
+    runs-on: ubuntu-latest
+    if: github.repository == 'github/codeql'
+    strategy:
+      matrix:
+        slug: ${{fromJson(github.event.inputs.projects || '["apache/commons-codec", "apache/commons-io", "apache/commons-beanutils", "apache/commons-logging", "apache/commons-fileupload", "apache/commons-lang", "apache/commons-validator", "apache/commons-csv", "apache/dubbo"]' )}}
+    steps:
+      - name: Clone github/codeql from PR
+        uses: actions/checkout@v2
+        if: github.event.pull_request
+        with:
+          path: codeql-pr
+      - name: Clone github/codeql from main
+        uses: actions/checkout@v2
+        with:
+          path: codeql-main
+          ref: main
+      - uses: ./codeql-main/.github/actions/fetch-codeql
+      - name: Download database
+        env:
+          SLUG: ${{ matrix.slug }}
+        run: |
+          set -x
+          mkdir lib-dbs
+          SHORTNAME=${SLUG//[^a-zA-Z0-9_]/}
+          projectId=`curl -s https://lgtm.com/api/v1.0/projects/g/${SLUG} | jq .id`
+          curl -L "https://lgtm.com/api/v1.0/snapshots/$projectId/java" -o "$SHORTNAME.zip"
+          unzip -q -d "${SHORTNAME}-db" "${SHORTNAME}.zip"
+          mkdir "lib-dbs/$SHORTNAME/"
+          mv "${SHORTNAME}-db/"$(ls -1 "${SHORTNAME}"-db)/* "lib-dbs/${SHORTNAME}/"
+      - name: Generate Models (PR and main)
+        run: |
+          set -x
+          mkdir tmp-models
+          MODELS=`pwd`/tmp-models
+          DATABASES=`pwd`/lib-dbs
+
+          analyzeDatabaseWithCheckout() {
+            QL_VARIANT=$1
+            DATABASE=$2
+            cd codeql-$QL_VARIANT
+            SHORTNAME=`basename $DATABASE`
+            python java/ql/src/utils/model-generator/GenerateFlowModel.py $DATABASE $MODELS/${SHORTNAME}.qll
+            mv $MODELS/${SHORTNAME}.qll $MODELS/${SHORTNAME}Generated_${QL_VARIANT}.qll
+            cd ..
+          }
+
+          for d in $DATABASES/*/ ; do
+            ls -1 "$d"
+
+            analyzeDatabaseWithCheckout "main" $d
+            if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]
+            then
+              analyzeDatabaseWithCheckout "pr" $d
+            fi
+          done
+      - name: Install diff2html
+        if: github.event.pull_request
+        run: |
+          npm install -g diff2html-cli
+      - name: Generate Model Diff
+        if: github.event.pull_request
+        run: |
+          set -x
+          MODELS=`pwd`/tmp-models
+          ls -1 tmp-models/
+          for m in $MODELS/*_main.qll ; do
+            t="${m/main/"pr"}"
+            basename=`basename $m`
+            name="diff_${basename/_main.qll/""}"
+            (diff -w -u $m $t | diff2html -i stdin -F $MODELS/$name.html) || true
+          done
+      - uses: actions/upload-artifact@v2
+        with:
+          name: models
+          path: tmp-models/*.qll
+          retention-days: 20
+      - uses: actions/upload-artifact@v2
+        with:
+          name: diffs
+          path: tmp-models/*.html
+          retention-days: 20

.github/workflows/mad_regenerate-models.yml

@@ -0,0 +1,62 @@
+name: Regenerate framework models
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "30 2 * * *"
+  pull_request:
+    branches:
+      - main
+    paths:
+      - ".github/workflows/mad_regenerate-models.yml"
+
+jobs:
+  regenerate-models:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        # placeholder required for each axis, excluded below, replaced by the actual combinations (see include)
+        slug: ["placeholder"]
+        ref: ["placeholder"]
+        include:
+          - slug: "apache/commons-io"
+            ref: "8985de8fe74f6622a419b37a6eed0dbc484dc128"
+        exclude:
+          - slug: "placeholder"
+            ref: "placeholder"
+    steps:
+      - name: Clone self (github/codeql)
+        uses: actions/checkout@v2
+      - name: Setup CodeQL binaries
+        uses: ./.github/actions/fetch-codeql
+      - name: Clone repositories
+        uses: actions/checkout@v2
+        with:
+          path: repos/${{ matrix.ref }}
+          ref: ${{ matrix.ref }}
+          repository: ${{ matrix.slug }}
+      - name: Build database
+        env:
+          SLUG: ${{ matrix.slug }}
+          REF: ${{ matrix.ref }}
+        run: |
+          mkdir dbs
+          cd repos/${REF}
+          SHORTNAME=${SLUG//[^a-zA-Z0-9_]/}
+          codeql database create --language=java ../../dbs/${SHORTNAME}
+      - name: Regenerate models in-place
+        env:
+          SLUG: ${{ matrix.slug }}
+        run: |
+          SHORTNAME=${SLUG//[^a-zA-Z0-9_]/}
+          java/ql/src/utils/model-generator/RegenerateModels.py "${SLUG}" dbs/${SHORTNAME}
+      - name: Stage changes
+        run: |
+          find java -name "*.qll" -print0 | xargs -0 git add
+          git status
+          git diff --cached > models.patch
+      - uses: actions/upload-artifact@v2
+        with:
+          name: patch
+          path: models.patch
+          retention-days: 7

.github/workflows/ql-for-ql-build.yml

@@ -31,13 +31,13 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ${{ runner.temp }}/query-pack.zip
-          key: queries-${{ hashFiles('ql/**/*.ql*') }}-${{ hashFiles('ql/ql/src/ql.dbscheme*') }}-${{ steps.get-codeql-version.outputs.version }}
+          key: queries-${{ hashFiles('ql/**/*.ql*') }}-${{ hashFiles('ql/**/qlpack.yml') }}-${{ hashFiles('ql/ql/src/ql.dbscheme*') }}-${{ steps.get-codeql-version.outputs.version }}
       - name: Build query pack
         if: steps.cache-queries.outputs.cache-hit != 'true'
         run: |
           cd ql/ql/src
           "${CODEQL}" pack create
-          cd .codeql/pack/codeql/ql-all/0.0.0
+          cd .codeql/pack/codeql/ql/0.0.0
           zip "${PACKZIP}" -r .
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
@@ -189,4 +189,11 @@ jobs:
         uses: github/codeql-action/analyze@erik-krogh/ql
         with: 
           category: "ql-for-ql-${{ matrix.folder }}"
+      - name: Copy sarif file to CWD
+        run: cp ../results/ql.sarif ./${{ matrix.folder }}.sarif
+      - name: Sarif as artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ matrix.folder }}.sarif
+          path: ${{ matrix.folder }}.sarif
 

.github/workflows/ql-for-ql-dataset_measure.yml

@@ -17,7 +17,7 @@ jobs:
       CODEQL_THREADS: 4 # TODO: remove this once it's set by the CLI
     strategy:
       matrix:
-        repo: 
+        repo:
           - github/codeql
           - github/codeql-go
     runs-on: ubuntu-latest
@@ -35,7 +35,7 @@ jobs:
             ~/.cargo/registry
             ~/.cargo/git
             ql/target
-          key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('**/Cargo.lock') }}
+          key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
       - name: Build Extractor
         run: cd ql; env "PATH=$PATH:`dirname ${CODEQL}`" ./create-extractor-pack.sh
         env:

.github/workflows/ql-for-ql-tests.yml

@@ -29,24 +29,24 @@ jobs:
             ~/.cargo/registry
             ~/.cargo/git
             ql/target
-          key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('**/Cargo.lock') }}
+          key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
       - name: Build extractor
         run: |
           cd ql;
           codeqlpath=$(dirname ${{ steps.find-codeql.outputs.codeql-path }});
           env "PATH=$PATH:$codeqlpath" ./create-extractor-pack.sh
       - name: Run QL tests
-        run: | 
+        run: |
           "${CODEQL}" test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}/ql/extractor-pack" --consistency-queries ql/ql/consistency-queries ql/ql/test
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
       - name: Check QL formatting
-        run: | 
+        run: |
           find ql/ql "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 "${CODEQL}" query format --check-only
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
       - name: Check QL compilation
-        run: | 
+        run: |
           "${CODEQL}" query compile --check-only --threads=4 --warnings=error --search-path "${{ github.workspace }}/ql/extractor-pack" "ql/ql/src" "ql/ql/examples"
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}

.github/workflows/ruby-build.yml

@@ -50,7 +50,7 @@ jobs:
             ~/.cargo/registry
             ~/.cargo/git
             ruby/target
-          key: ${{ runner.os }}-rust-cargo-${{ hashFiles('**/Cargo.lock') }}
+          key: ${{ runner.os }}-ruby-rust-cargo-${{ hashFiles('ruby/rust-toolchain.toml', 'ruby/**/Cargo.lock') }}
       - name: Check formatting
         run: cargo fmt --all -- --check
       - name: Build

.github/workflows/ruby-qltest.yml

@@ -24,27 +24,53 @@ defaults:
     working-directory: ruby
 
 jobs:
-  qltest:
+  qlformat:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: ./.github/actions/fetch-codeql
-      - uses: ./ruby/actions/create-extractor-pack
-      - name: Run QL tests
-        run: |
-          codeql test run --threads=0 --ram 5000 --search-path "${{ github.workspace }}/ruby/extractor-pack" --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
       - name: Check QL formatting
         run: find ql "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 codeql query format --check-only
+  qlcompile:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ./.github/actions/fetch-codeql
       - name: Check QL compilation
         run: |
           codeql query compile --check-only --threads=0 --ram 5000 --warnings=error "ql/src" "ql/examples"
         env:
           GITHUB_TOKEN: ${{ github.token }}
+  qlupgrade:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ./.github/actions/fetch-codeql
       - name: Check DB upgrade scripts
         run: |
           echo >empty.trap
           codeql dataset import -S ql/lib/upgrades/initial/ruby.dbscheme testdb empty.trap
           codeql dataset upgrade testdb --additional-packs ql/lib
           diff -q testdb/ruby.dbscheme ql/lib/ruby.dbscheme
+      - name: Check DB downgrade scripts
+        run: |
+          echo >empty.trap
+          rm -rf testdb; codeql dataset import -S ql/lib/ruby.dbscheme testdb empty.trap
+          codeql resolve upgrades --format=lines --allow-downgrades --additional-packs downgrades \
+           --dbscheme=ql/lib/ruby.dbscheme --target-dbscheme=downgrades/initial/ruby.dbscheme |
+           xargs codeql execute upgrades testdb
+          diff -q testdb/ruby.dbscheme downgrades/initial/ruby.dbscheme
+  qltest:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        slice: ["1/2", "2/2"]
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ./.github/actions/fetch-codeql
+      - uses: ./ruby/actions/create-extractor-pack
+      - name: Run QL tests
+        run: |
+          codeql test run --threads=0 --ram 5000 --slice ${{ matrix.slice }} --search-path "${{ github.workspace }}/ruby/extractor-pack" --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test
+        env:
+          GITHUB_TOKEN: ${{ github.token }}

.github/workflows/validate-change-notes.yml

@@ -0,0 +1,29 @@
+name: Validate change notes
+
+on:
+  push:
+    paths:
+      - "*/ql/*/change-notes/**/*"
+      - ".github/workflows/validate-change-notes.yml"
+    branches:
+      - main
+      - "rc/*"
+  pull_request:
+    paths:
+      - "*/ql/*/change-notes/**/*"
+      - ".github/workflows/validate-change-notes.yml"
+
+jobs:
+  check-change-note:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Setup CodeQL
+        uses: ./.github/actions/fetch-codeql
+
+      - name: Fail if there are any errors with existing change notes
+
+        run: |
+          codeql pack release --groups cpp,csharp,java,javascript,python,ruby,-examples,-test,-experimental

CODEOWNERS

@@ -13,6 +13,9 @@
 /python/**/experimental/**/* @github/codeql-python @xcorail
 /ruby/**/experimental/**/* @github/codeql-ruby @xcorail
 
+# ML-powered queries
+/javascript/ql/experimental/adaptivethreatmodeling/ @github/codeql-ml-powered-queries-reviewers
+
 # Notify members of codeql-go about PRs to the shared data-flow library files
 /java/ql/src/semmle/code/java/dataflow/internal/DataFlowImpl.qll @github/codeql-java @github/codeql-go
 /java/ql/src/semmle/code/java/dataflow/internal/DataFlowImpl2.qll @github/codeql-java @github/codeql-go
@@ -27,4 +30,4 @@
 /docs/query-*-style-guide.md @github/codeql-analysis-reviewers
 
 # QL for QL reviewers
-/ql/ @github/codeql-ql-for-ql-reviewers
+/ql/ @github/codeql-ql-for-ql-reviewers

cpp/autobuilder/Semmle.Autobuild.Cpp.Tests/Semmle.Autobuild.Cpp.Tests.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net5.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
     <RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
     <Nullable>enable</Nullable>

cpp/autobuilder/Semmle.Autobuild.Cpp/Semmle.Autobuild.Cpp.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net5.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
     <AssemblyName>Semmle.Autobuild.Cpp</AssemblyName>
     <RootNamespace>Semmle.Autobuild.Cpp</RootNamespace>
     <ApplicationIcon />

cpp/downgrades/2cd420191e5f782589b4e4efb70127de265390dd/upgrade.properties

@@ -0,0 +1,2 @@
+description: Remove unused legacy relations
+compatibility: backwards

cpp/downgrades/qlpack.yml

@@ -0,0 +1,4 @@
+name: codeql/cpp-downgrades
+groups: cpp
+downgrades: .
+library: true

cpp/ql/lib/CHANGELOG.md

@@ -1,3 +1,16 @@
+## 0.0.9
+
+## 0.0.8
+
+### Deprecated APIs
+
+* The `codeql/cpp-upgrades` CodeQL pack has been removed. All upgrades scripts have been merged into the `codeql/cpp-all` CodeQL pack.
+
+### Minor Analysis Improvements
+
+* `FormatLiteral::getMaxConvertedLength` now uses range analysis to provide a
+  more accurate length for integers formatted with `%x`
+
 ## 0.0.7
 
 ## 0.0.6

cpp/ql/lib/change-notes/2022-01-11-remove-upgrades-packs.md

@@ -1,4 +0,0 @@
----
-category: deprecated
----
-* The `codeql/cpp-upgrades` CodeQL pack has been removed. All upgrades scripts have been merged into the `codeql/cpp-all` CodeQL pack.

cpp/ql/lib/change-notes/2022-01-14-hex-format-range-analysis.md

@@ -1,5 +0,0 @@
----
-category: minorAnalysis
----
-* `FormatLiteral::getMaxConvertedLength` now uses range analysis to provide a
-  more accurate length for integers formatted with `%x`

cpp/ql/lib/change-notes/released/0.0.8.md

@@ -0,0 +1,10 @@
+## 0.0.8
+
+### Deprecated APIs
+
+* The `codeql/cpp-upgrades` CodeQL pack has been removed. All upgrades scripts have been merged into the `codeql/cpp-all` CodeQL pack.
+
+### Minor Analysis Improvements
+
+* `FormatLiteral::getMaxConvertedLength` now uses range analysis to provide a
+  more accurate length for integers formatted with `%x`

cpp/ql/lib/change-notes/released/0.0.9.md

@@ -0,0 +1,2 @@
+## 0.0.9
+

cpp/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.0.7
+lastReleaseVersion: 0.0.9

cpp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-all
-version: 0.0.8-dev
+version: 0.0.10-dev
 groups: cpp
 dbscheme: semmlecode.cpp.dbscheme
 extractor: cpp

cpp/ql/lib/semmle/code/cpp/commons/unix/Constants.qll

@@ -11,10 +11,10 @@ import cpp
  */
 bindingset[input]
 int parseOctal(string input) {
-  input.charAt(0) = "0" and
+  input.regexpMatch("0[0-7]+") and
   result =
     strictsum(int ix |
-      ix in [0 .. input.length()]
+      ix in [1 .. input.length()]
     |
       8.pow(input.length() - (ix + 1)) * input.charAt(ix).toInt()
     )

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplCommon.qll

@@ -1290,7 +1290,7 @@ class DataFlowCallOption extends TDataFlowCallOption {
   }
 }
 
-/** Content tagged with the type of a containing object. */
+/** A `Content` tagged with the type of a containing object. */
 class TypedContent extends MkTypedContent {
   private Content c;
   private DataFlowType t;

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowUtil.qll

@@ -592,12 +592,14 @@ predicate simpleLocalFlowStep(Node nodeFrom, Node nodeTo) {
  * Holds if data flows from `source` to `sink` in zero or more local
  * (intra-procedural) steps.
  */
+pragma[inline]
 predicate localFlow(Node source, Node sink) { localFlowStep*(source, sink) }
 
 /**
  * Holds if data can flow from `e1` to `e2` in zero or more
  * local (intra-procedural) steps.
  */
+pragma[inline]
 predicate localExprFlow(Expr e1, Expr e2) { localFlow(exprNode(e1), exprNode(e2)) }
 
 /**

cpp/ql/lib/semmle/code/cpp/dataflow/internal/FlowVar.qll

@@ -353,9 +353,9 @@ module FlowVar_internal {
         // indirection.
         result = def.getAUse(v)
         or
-        exists(SsaDefinition descendentDef |
-          this.getASuccessorSsaVar+() = TSsaVar(descendentDef, _) and
-          result = descendentDef.getAUse(v)
+        exists(SsaDefinition descendantDef |
+          this.getASuccessorSsaVar+() = TSsaVar(descendantDef, _) and
+          result = descendantDef.getAUse(v)
         )
       )
       or

cpp/ql/lib/semmle/code/cpp/dataflow/internal/TaintTrackingUtil.qll

@@ -124,12 +124,14 @@ predicate localAdditionalTaintStep(DataFlow::Node nodeFrom, DataFlow::Node nodeT
  * Holds if taint may propagate from `source` to `sink` in zero or more local
  * (intra-procedural) steps.
  */
+pragma[inline]
 predicate localTaint(DataFlow::Node source, DataFlow::Node sink) { localTaintStep*(source, sink) }
 
 /**
  * Holds if taint can flow from `e1` to `e2` in zero or more
  * local (intra-procedural) steps.
  */
+pragma[inline]
 predicate localExprTaint(Expr e1, Expr e2) {
   localTaint(DataFlow::exprNode(e1), DataFlow::exprNode(e2))
 }

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplCommon.qll

@@ -1290,7 +1290,7 @@ class DataFlowCallOption extends TDataFlowCallOption {
   }
 }
 
-/** Content tagged with the type of a containing object. */
+/** A `Content` tagged with the type of a containing object. */
 class TypedContent extends MkTypedContent {
   private Content c;
   private DataFlowType t;

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowUtil.qll

@@ -1032,12 +1032,14 @@ SideEffectInstruction getSideEffectFor(CallInstruction call, int argument) {
  * Holds if data flows from `source` to `sink` in zero or more local
  * (intra-procedural) steps.
  */
+pragma[inline]
 predicate localFlow(Node source, Node sink) { localFlowStep*(source, sink) }
 
 /**
  * Holds if data can flow from `i1` to `i2` in zero or more
  * local (intra-procedural) steps.
  */
+pragma[inline]
 predicate localInstructionFlow(Instruction e1, Instruction e2) {
   localFlow(instructionNode(e1), instructionNode(e2))
 }
@@ -1046,6 +1048,7 @@ predicate localInstructionFlow(Instruction e1, Instruction e2) {
  * Holds if data can flow from `e1` to `e2` in zero or more
  * local (intra-procedural) steps.
  */
+pragma[inline]
 predicate localExprFlow(Expr e1, Expr e2) { localFlow(exprNode(e1), exprNode(e2)) }
 
 private newtype TContent =

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TaintTrackingUtil.qll

@@ -121,12 +121,14 @@ private predicate operandToInstructionTaintStep(Operand opFrom, Instruction inst
  * Holds if taint may propagate from `source` to `sink` in zero or more local
  * (intra-procedural) steps.
  */
+pragma[inline]
 predicate localTaint(DataFlow::Node source, DataFlow::Node sink) { localTaintStep*(source, sink) }
 
 /**
  * Holds if taint can flow from `i1` to `i2` in zero or more
  * local (intra-procedural) steps.
  */
+pragma[inline]
 predicate localInstructionTaint(Instruction i1, Instruction i2) {
   localTaint(DataFlow::instructionNode(i1), DataFlow::instructionNode(i2))
 }
@@ -135,6 +137,7 @@ predicate localInstructionTaint(Instruction i1, Instruction i2) {
  * Holds if taint can flow from `e1` to `e2` in zero or more
  * local (intra-procedural) steps.
  */
+pragma[inline]
 predicate localExprTaint(Expr e1, Expr e2) {
   localTaint(DataFlow::exprNode(e1), DataFlow::exprNode(e2))
 }

cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/TranslatedElement.qll

@@ -567,6 +567,13 @@ newtype TTranslatedElement =
   } or
   // The initialization of a base class from within a constructor.
   TTranslatedConstructorBaseInit(ConstructorBaseInit init) { not ignoreExpr(init) } or
+  // Workaround for a case where no base constructor is generated but a targetless base
+  // constructor call is present.
+  TTranslatedConstructorBareInit(ConstructorInit init) {
+    not ignoreExpr(init) and
+    not init instanceof ConstructorBaseInit and
+    not init instanceof ConstructorFieldInit
+  } or
   // The destruction of a base class from within a destructor.
   TTranslatedDestructorBaseDestruction(DestructorBaseDestruction destruction) {
     not ignoreExpr(destruction)

cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/TranslatedFunction.qll

@@ -573,6 +573,11 @@ class TranslatedConstructorInitList extends TranslatedElement, InitializationCon
       baseInit = func.(Constructor).getInitializer(id) and
       result = getTranslatedConstructorBaseInit(baseInit)
     )
+    or
+    exists(ConstructorInit bareInit |
+      bareInit = func.(Constructor).getInitializer(id) and
+      result = getTranslatedConstructorBareInit(bareInit)
+    )
   }
 
   override Instruction getFirstInstruction() {

cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/TranslatedInitialization.qll

@@ -917,3 +917,36 @@ class TranslatedDestructorBaseDestruction extends TranslatedBaseStructorCall,
 
   final override string toString() { result = "destroy base: " + call.toString() }
 }
+
+/**
+ * A constructor base init call where no base constructor has been generated.
+ *
+ * Workaround for an extractor issue.
+ */
+class TranslatedConstructorBareInit extends TranslatedElement, TTranslatedConstructorBareInit {
+  ConstructorInit init;
+
+  TranslatedConstructorBareInit() { this = TTranslatedConstructorBareInit(init) }
+
+  override Locatable getAST() { result = init }
+
+  final override string toString() { result = "construct base (no constructor)" }
+
+  override Instruction getFirstInstruction() { result = getParent().getChildSuccessor(this) }
+
+  override predicate hasInstruction(Opcode opcode, InstructionTag tag, CppType resultType) {
+    none()
+  }
+
+  override TranslatedElement getChild(int id) { none() }
+
+  override Function getFunction() { result = getParent().getFunction() }
+
+  override Instruction getInstructionSuccessor(InstructionTag tag, EdgeKind kind) { none() }
+
+  override Instruction getChildSuccessor(TranslatedElement child) { none() }
+}
+
+TranslatedConstructorBareInit getTranslatedConstructorBareInit(ConstructorInit init) {
+  result.getAST() = init
+}

cpp/ql/lib/semmlecode.cpp.dbscheme

@@ -135,52 +135,11 @@ externalData(
     string value : string ref
 );
 
-/**
- * The date of the snapshot.
- */
-snapshotDate(unique date snapshotDate : date ref);
-
 /**
  * The source location of the snapshot.
  */
 sourceLocationPrefix(string prefix : string ref);
 
-/**
- * Data used by the 'duplicate code' detection.
- */
-duplicateCode(
-    unique int id : @duplication,
-    string relativePath : string ref,
-    int equivClass : int ref
-);
-
-/**
- * Data used by the 'similar code' detection.
- */
-similarCode(
-    unique int id : @similarity,
-    string relativePath : string ref,
-    int equivClass : int ref
-);
-
-/**
- * Data used by the 'duplicate code' and 'similar code' detection.
- */
-@duplication_or_similarity = @duplication | @similarity
-
-/**
- * Data used by the 'duplicate code' and 'similar code' detection.
- */
-#keyset[id, offset]
-tokens(
-    int id : @duplication_or_similarity ref,
-    int offset : int ref,
-    int beginLine : int ref,
-    int beginColumn : int ref,
-    int endLine : int ref,
-    int endColumn : int ref
-);
-
 /**
  * Information about packages that provide code used during compilation.
  * The `id` is just a unique identifier.

cpp/ql/lib/tutorial.qll

@@ -6,122 +6,22 @@
  */
 class Person extends string {
   Person() {
-    this = "Ronil" or
-    this = "Dina" or
-    this = "Ravi" or
-    this = "Bruce" or
-    this = "Jo" or
-    this = "Aida" or
-    this = "Esme" or
-    this = "Charlie" or
-    this = "Fred" or
-    this = "Meera" or
-    this = "Maya" or
-    this = "Chad" or
-    this = "Tiana" or
-    this = "Laura" or
-    this = "George" or
-    this = "Will" or
-    this = "Mary" or
-    this = "Almira" or
-    this = "Susannah" or
-    this = "Rhoda" or
-    this = "Cynthia" or
-    this = "Eunice" or
-    this = "Olive" or
-    this = "Virginia" or
-    this = "Angeline" or
-    this = "Helen" or
-    this = "Cornelia" or
-    this = "Harriet" or
-    this = "Mahala" or
-    this = "Abby" or
-    this = "Margaret" or
-    this = "Deb" or
-    this = "Minerva" or
-    this = "Severus" or
-    this = "Lavina" or
-    this = "Adeline" or
-    this = "Cath" or
-    this = "Elisa" or
-    this = "Lucretia" or
-    this = "Anne" or
-    this = "Eleanor" or
-    this = "Joanna" or
-    this = "Adam" or
-    this = "Agnes" or
-    this = "Rosanna" or
-    this = "Clara" or
-    this = "Melissa" or
-    this = "Amy" or
-    this = "Isabel" or
-    this = "Jemima" or
-    this = "Cordelia" or
-    this = "Melinda" or
-    this = "Delila" or
-    this = "Jeremiah" or
-    this = "Elijah" or
-    this = "Hester" or
-    this = "Walter" or
-    this = "Oliver" or
-    this = "Hugh" or
-    this = "Aaron" or
-    this = "Reuben" or
-    this = "Eli" or
-    this = "Amos" or
-    this = "Augustus" or
-    this = "Theodore" or
-    this = "Ira" or
-    this = "Timothy" or
-    this = "Cyrus" or
-    this = "Horace" or
-    this = "Simon" or
-    this = "Asa" or
-    this = "Frank" or
-    this = "Nelson" or
-    this = "Leonard" or
-    this = "Harrison" or
-    this = "Anthony" or
-    this = "Louis" or
-    this = "Milton" or
-    this = "Noah" or
-    this = "Cornelius" or
-    this = "Abdul" or
-    this = "Warren" or
-    this = "Harvey" or
-    this = "Dennis" or
-    this = "Wesley" or
-    this = "Sylvester" or
-    this = "Gilbert" or
-    this = "Sullivan" or
-    this = "Edmund" or
-    this = "Wilson" or
-    this = "Perry" or
-    this = "Matthew" or
-    this = "Simba" or
-    this = "Nala" or
-    this = "Rafiki" or
-    this = "Shenzi" or
-    this = "Ernest" or
-    this = "Gertrude" or
-    this = "Oscar" or
-    this = "Lilian" or
-    this = "Raymond" or
-    this = "Elgar" or
-    this = "Elmer" or
-    this = "Herbert" or
-    this = "Maude" or
-    this = "Mae" or
-    this = "Otto" or
-    this = "Edwin" or
-    this = "Ophelia" or
-    this = "Parsley" or
-    this = "Sage" or
-    this = "Rosemary" or
-    this = "Thyme" or
-    this = "Garfunkel" or
-    this = "King Basil" or
-    this = "Stephen"
+    this =
+      [
+        "Ronil", "Dina", "Ravi", "Bruce", "Jo", "Aida", "Esme", "Charlie", "Fred", "Meera", "Maya",
+        "Chad", "Tiana", "Laura", "George", "Will", "Mary", "Almira", "Susannah", "Rhoda",
+        "Cynthia", "Eunice", "Olive", "Virginia", "Angeline", "Helen", "Cornelia", "Harriet",
+        "Mahala", "Abby", "Margaret", "Deb", "Minerva", "Severus", "Lavina", "Adeline", "Cath",
+        "Elisa", "Lucretia", "Anne", "Eleanor", "Joanna", "Adam", "Agnes", "Rosanna", "Clara",
+        "Melissa", "Amy", "Isabel", "Jemima", "Cordelia", "Melinda", "Delila", "Jeremiah", "Elijah",
+        "Hester", "Walter", "Oliver", "Hugh", "Aaron", "Reuben", "Eli", "Amos", "Augustus",
+        "Theodore", "Ira", "Timothy", "Cyrus", "Horace", "Simon", "Asa", "Frank", "Nelson",
+        "Leonard", "Harrison", "Anthony", "Louis", "Milton", "Noah", "Cornelius", "Abdul", "Warren",
+        "Harvey", "Dennis", "Wesley", "Sylvester", "Gilbert", "Sullivan", "Edmund", "Wilson",
+        "Perry", "Matthew", "Simba", "Nala", "Rafiki", "Shenzi", "Ernest", "Gertrude", "Oscar",
+        "Lilian", "Raymond", "Elgar", "Elmer", "Herbert", "Maude", "Mae", "Otto", "Edwin",
+        "Ophelia", "Parsley", "Sage", "Rosemary", "Thyme", "Garfunkel", "King Basil", "Stephen"
+      ]
   }
 
   /** Gets the hair color of the person. If the person is bald, there is no result. */
@@ -936,25 +836,12 @@ class Person extends string {
 
   /** Holds if the person is deceased. */
   predicate isDeceased() {
-    this = "Ernest" or
-    this = "Gertrude" or
-    this = "Oscar" or
-    this = "Lilian" or
-    this = "Edwin" or
-    this = "Raymond" or
-    this = "Elgar" or
-    this = "Elmer" or
-    this = "Herbert" or
-    this = "Maude" or
-    this = "Mae" or
-    this = "Otto" or
-    this = "Ophelia" or
-    this = "Parsley" or
-    this = "Sage" or
-    this = "Rosemary" or
-    this = "Thyme" or
-    this = "Garfunkel" or
-    this = "King Basil"
+    this =
+      [
+        "Ernest", "Gertrude", "Oscar", "Lilian", "Edwin", "Raymond", "Elgar", "Elmer", "Herbert",
+        "Maude", "Mae", "Otto", "Ophelia", "Parsley", "Sage", "Rosemary", "Thyme", "Garfunkel",
+        "King Basil"
+      ]
   }
 
   /** Gets a parent of the person (alive or deceased). */
@@ -1195,12 +1082,7 @@ class Person extends string {
   }
 
   /** Holds if the person is allowed in the region. Initially, all villagers are allowed in every region. */
-  predicate isAllowedIn(string region) {
-    region = "north" or
-    region = "south" or
-    region = "east" or
-    region = "west"
-  }
+  predicate isAllowedIn(string region) { region = ["north", "south", "east", "west"] }
 }
 
 /** Returns a parent of the person. */

cpp/ql/lib/upgrades/018f430097e80bcf4b2786c989dae94b7d82b819/upgrade.properties

@@ -0,0 +1,6 @@
+description: Remove unused legacy relations
+compatibility: full
+snapshotDate.rel: delete
+duplicateCode.rel: delete
+similarCode.rel: delete
+tokens.rel: delete

cpp/ql/src/CHANGELOG.md

@@ -1,3 +1,36 @@
+## 0.0.9
+
+### New Queries
+
+* Added a new query, `cpp/open-call-with-mode-argument`, to detect when `open` or `openat` is called with the `O_CREAT` or `O_TMPFILE` flag but when the `mode` argument is omitted.
+
+### Minor Analysis Improvements
+
+* The "Cleartext transmission of sensitive information" (`cpp/cleartext-transmission`) query has been further improved to reduce false positive results, and upgraded from `medium` to `high` precision.
+* The "Cleartext transmission of sensitive information" (`cpp/cleartext-transmission`) query now finds more results, where a password is stored in a struct field or class member variable.
+* The `cpp/cleartext-storage-file` query has been improved, removing false positives where data is written to a standard output stream.
+* The `cpp/cleartext-storage-buffer` query has been updated to use the `semmle.code.cpp.dataflow.TaintTracking` library.
+* The `cpp/world-writable-file-creation` query now only detects `open` and `openat` calls with the `O_CREAT` or `O_TMPFILE` flag.
+
+## 0.0.8
+
+### New Queries
+
+* The `security` tag has been added to the `cpp/return-stack-allocated-memory` query. As a result, its results will now appear by default.
+* The "Uncontrolled data in arithmetic expression" (cpp/uncontrolled-arithmetic) query has been enhanced to reduce false positive results and its @precision increased to high.
+* A new `cpp/very-likely-overruning-write` query has been added to the default query suite for C/C++. The query reports some results that were formerly flagged by `cpp/overruning-write`.
+
+### Minor Analysis Improvements
+
+* Fix an issue with the `cpp/declaration-hides-variable` query where it would report variables that are unnamed in a database.
+* The `cpp/cleartext-storage-file` query has been upgraded with non-local taint flow and has been converted to a `path-problem` query.
+* The `cpp/return-stack-allocated-memory` query has been improved to produce fewer false positives. The
+  query has also been converted to a `path-problem` query.
+* The "Cleartext transmission of sensitive information" (`cpp/cleartext-transmission`) query has been improved in several ways to reduce false positive results.
+* The "Potential improper null termination" (`cpp/improper-null-termination`) query now produces fewer false positive results around control flow branches and loops.
+* Added exception for GLib's gboolean to cpp/ambiguously-signed-bit-field.
+  This change reduces the number of false positives in the query.
+
 ## 0.0.7
 
 ## 0.0.6

cpp/ql/src/Likely Bugs/Memory Management/ReturnStackAllocatedMemory.ql

@@ -14,6 +14,9 @@
  */
 
 import cpp
+// We don't actually use the global value numbering library in this query, but without it we end up
+// recomputing the IR.
+private import semmle.code.cpp.valuenumbering.GlobalValueNumbering
 import semmle.code.cpp.ir.IR
 import semmle.code.cpp.ir.dataflow.DataFlow::DataFlow
 

cpp/ql/src/Security/CWE/CWE-311/CleartextTransmission.ql

@@ -5,7 +5,7 @@
  * @kind path-problem
  * @problem.severity warning
  * @security-severity 7.5
- * @precision medium
+ * @precision high
  * @id cpp/cleartext-transmission
  * @tags security
  *       external/cwe/cwe-319
@@ -14,8 +14,8 @@
 import cpp
 import semmle.code.cpp.security.SensitiveExprs
 import semmle.code.cpp.dataflow.TaintTracking
-import semmle.code.cpp.valuenumbering.GlobalValueNumbering
 import semmle.code.cpp.models.interfaces.FlowSource
+import semmle.code.cpp.commons.File
 import DataFlow::PathGraph
 
 /**
@@ -27,6 +27,7 @@ class SensitiveNode extends DataFlow::Node {
     this.asExpr() = any(SensitiveVariable sv).getInitializer().getExpr() or
     this.asExpr().(VariableAccess).getTarget() =
       any(SensitiveVariable sv).(GlobalOrNamespaceVariable) or
+    this.asExpr().(VariableAccess).getTarget() = any(SensitiveVariable v | v instanceof Field) or
     this.asUninitialized() instanceof SensitiveVariable or
     this.asParameter() instanceof SensitiveVariable or
     this.asExpr().(FunctionCall).getTarget() instanceof SensitiveFunction
@@ -120,24 +121,32 @@ abstract class NetworkSendRecv extends FunctionCall {
   NetworkSendRecv() {
     this.getTarget() = target and
     // exclude calls based on the socket...
-    not exists(GVN g |
-      g = globalValueNumber(target.getSocketExpr(this)) and
+    not exists(DataFlow::Node src, DataFlow::Node dest |
+      DataFlow::localFlow(src, dest) and
+      dest.asExpr() = target.getSocketExpr(this) and
       (
         // literal constant
-        globalValueNumber(any(Literal l)) = g
+        src.asExpr() instanceof Literal
         or
         // variable (such as a global) initialized to a literal constant
         exists(Variable v |
           v.getInitializer().getExpr() instanceof Literal and
-          g = globalValueNumber(v.getAnAccess())
+          src.asExpr() = v.getAnAccess()
         )
         or
         // result of a function call with literal inputs (likely constant)
+        forex(Expr arg | arg = src.asExpr().(FunctionCall).getAnArgument() | arg instanceof Literal)
+        or
+        // variable called `stdin`, `stdout` or `stderr`
+        src.asExpr().(VariableAccess).getTarget().getName() = ["stdin", "stdout", "stderr"]
+        or
+        // open of `"/dev/tty"`
         exists(FunctionCall fc |
-          forex(Expr arg | arg = fc.getAnArgument() | arg instanceof Literal) and
-          g = globalValueNumber(fc)
+          fopenCall(fc) and
+          fc.getAnArgument().getValue() = "/dev/tty" and
+          src.asExpr() = fc
         )
-        // (this is far from exhaustive)
+        // (this is not exhaustive)
       )
     )
   }
@@ -159,6 +168,16 @@ class NetworkRecv extends NetworkSendRecv {
   override Recv target;
 }
 
+pragma[noinline]
+predicate encryptionFunction(Function f) {
+  f.getName().toLowerCase().regexpMatch(".*(crypt|encode|decode|hash|securezero).*")
+}
+
+pragma[noinline]
+predicate encryptionType(UserType t) {
+  t.getName().toLowerCase().regexpMatch(".*(crypt|encode|decode|hash|securezero).*")
+}
+
 /**
  * An expression that is an argument or return value from an encryption /
  * decryption call. This is quite inclusive to minimize false positives, for
@@ -168,10 +187,7 @@ class NetworkRecv extends NetworkSendRecv {
 class Encrypted extends Expr {
   Encrypted() {
     exists(FunctionCall fc |
-      fc.getTarget()
-          .getName()
-          .toLowerCase()
-          .regexpMatch(".*(crypt|encode|decode|hash|securezero).*") and
+      encryptionFunction(fc.getTarget()) and
       (
         this = fc or
         this = fc.getAnArgument()
@@ -180,7 +196,7 @@ class Encrypted extends Expr {
     or
     exists(Type t |
       this.getType().refersTo(t) and
-      t.getName().toLowerCase().regexpMatch(".*(crypt|encode|decode|hash|securezero).*")
+      encryptionType(t)
     )
   }
 }

cpp/ql/src/Security/CWE/CWE-732/DoNotCreateWorldWritable.ql

@@ -12,17 +12,16 @@
 
 import cpp
 import FilePermissions
-import semmle.code.cpp.commons.unix.Constants
 
 predicate worldWritableCreation(FileCreationExpr fc, int mode) {
   mode = localUmask(fc).mask(fc.getMode()) and
-  sets(mode, s_iwoth())
+  setsAnyBits(mode, UnixConstants::s_iwoth())
 }
 
 predicate setWorldWritable(FunctionCall fc, int mode) {
   fc.getTarget().getName() = ["chmod", "fchmod", "_chmod", "_wchmod"] and
   mode = fc.getArgument(1).getValue().toInt() and
-  sets(mode, s_iwoth())
+  setsAnyBits(mode, UnixConstants::s_iwoth())
 }
 
 from Expr fc, int mode, string message

cpp/ql/src/Security/CWE/CWE-732/FilePermissions.qll

@@ -1,5 +1,49 @@
 import cpp
-import semmle.code.cpp.commons.unix.Constants
+import semmle.code.cpp.commons.unix.Constants as UnixConstants
+
+/**
+ * Gets the number corresponding to the contents of `input` in base-16.
+ * Note: the first two characters of `input` must be `0x`. For example:
+ * `parseHex("0x123abc") = 1194684`.
+ */
+bindingset[input]
+int parseHex(string input) {
+  exists(string lowerCaseInput | lowerCaseInput = input.toLowerCase() |
+    lowerCaseInput.regexpMatch("0x[0-9a-f]+") and
+    result =
+      strictsum(int ix |
+        ix in [2 .. input.length()]
+      |
+        16.pow(input.length() - (ix + 1)) * "0123456789abcdef".indexOf(lowerCaseInput.charAt(ix))
+      )
+  )
+}
+
+/**
+ * Gets the value defined by the `O_CREAT` macro if the macro
+ * exists and if every definition defines the same value.
+ */
+int o_creat() {
+  result =
+    unique(int v |
+      exists(Macro m | m.getName() = "O_CREAT" |
+        v = parseHex(m.getBody()) or v = UnixConstants::parseOctal(m.getBody())
+      )
+    )
+}
+
+/**
+ * Gets the value defined by the `O_TMPFILE` macro if the macro
+ * exists and if every definition defines the same value.
+ */
+int o_tmpfile() {
+  result =
+    unique(int v |
+      exists(Macro m | m.getName() = "O_TMPFILE" |
+        v = parseHex(m.getBody()) or v = UnixConstants::parseOctal(m.getBody())
+      )
+    )
+}
 
 bindingset[n, digit]
 private string octalDigit(int n, int digit) {
@@ -20,11 +64,17 @@ string octalFileMode(int mode) {
   else result = "[non-standard mode: decimal " + mode + "]"
 }
 
+/**
+ * Holds if the bitmask `value` sets the bits in `flag`.
+ */
+bindingset[value, flag]
+predicate setsFlag(int value, int flag) { value.bitAnd(flag) = flag }
+
 /**
  * Holds if the bitmask `mask` sets any of the bit fields in `fields`.
  */
 bindingset[mask, fields]
-predicate sets(int mask, int fields) { mask.bitAnd(fields) != 0 }
+predicate setsAnyBits(int mask, int fields) { mask.bitAnd(fields) != 0 }
 
 /**
  * Gets the value that `fc` sets the umask to, if `fc` is a call to
@@ -83,16 +133,24 @@ abstract class FileCreationExpr extends FunctionCall {
   abstract int getMode();
 }
 
-class OpenCreationExpr extends FileCreationExpr {
+abstract class FileCreationWithOptionalModeExpr extends FileCreationExpr {
+  abstract predicate hasModeArgument();
+}
+
+class OpenCreationExpr extends FileCreationWithOptionalModeExpr {
   OpenCreationExpr() {
-    this.getTarget().getName() = ["open", "_open", "_wopen"] and
-    sets(this.getArgument(1).getValue().toInt(), o_creat())
+    this.getTarget().hasGlobalOrStdName(["open", "_open", "_wopen"]) and
+    exists(int flag | flag = this.getArgument(1).getValue().toInt() |
+      setsFlag(flag, o_creat()) or setsFlag(flag, o_tmpfile())
+    )
   }
 
   override Expr getPath() { result = this.getArgument(0) }
 
+  override predicate hasModeArgument() { exists(this.getArgument(2)) }
+
   override int getMode() {
-    if exists(this.getArgument(2))
+    if this.hasModeArgument()
     then result = this.getArgument(2).getValue().toInt()
     else
       // assume anything is permitted
@@ -108,20 +166,35 @@ class CreatCreationExpr extends FileCreationExpr {
   override int getMode() { result = this.getArgument(1).getValue().toInt() }
 }
 
-class OpenatCreationExpr extends FileCreationExpr {
+class OpenatCreationExpr extends FileCreationWithOptionalModeExpr {
   OpenatCreationExpr() {
-    this.getTarget().getName() = "openat" and
-    this.getNumberOfArguments() = 4
+    this.getTarget().hasGlobalOrStdName("openat") and
+    exists(int flag | flag = this.getArgument(2).getValue().toInt() |
+      setsFlag(flag, o_creat()) or setsFlag(flag, o_tmpfile())
+    )
   }
 
   override Expr getPath() { result = this.getArgument(1) }
 
-  override int getMode() { result = this.getArgument(3).getValue().toInt() }
+  override predicate hasModeArgument() { exists(this.getArgument(3)) }
+
+  override int getMode() {
+    if this.hasModeArgument()
+    then result = this.getArgument(3).getValue().toInt()
+    else
+      // assume anything is permitted
+      result = 0.bitNot()
+  }
 }
 
 private int fopenMode() {
   result =
-    s_irusr().bitOr(s_irgrp()).bitOr(s_iroth()).bitOr(s_iwusr()).bitOr(s_iwgrp()).bitOr(s_iwoth())
+    UnixConstants::s_irusr()
+        .bitOr(UnixConstants::s_irgrp())
+        .bitOr(UnixConstants::s_iroth())
+        .bitOr(UnixConstants::s_iwusr())
+        .bitOr(UnixConstants::s_iwgrp())
+        .bitOr(UnixConstants::s_iwoth())
 }
 
 class FopenCreationExpr extends FileCreationExpr {
@@ -153,6 +226,6 @@ class FopensCreationExpr extends FileCreationExpr {
     // fopen_s has restrictive permissions unless you have "u" in the mode
     if this.getArgument(2).getValue().charAt(_) = "u"
     then result = fopenMode()
-    else result = s_irusr().bitOr(s_iwusr())
+    else result = UnixConstants::s_irusr().bitOr(UnixConstants::s_iwusr())
   }
 }

cpp/ql/src/Security/CWE/CWE-732/OpenCallMissingModeArgument.c

@@ -0,0 +1,9 @@
+int open_file_bad() {
+	// BAD - this uses arbitrary bytes from the stack as mode argument
+        return open(FILE, O_CREAT)
+}
+
+int open_file_good() {
+	// GOOD - the mode argument is supplied
+        return open(FILE, O_CREAT, S_IRUSR | S_IWUSR)
+}

cpp/ql/src/Security/CWE/CWE-732/OpenCallMissingModeArgument.qhelp

@@ -0,0 +1,31 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+
+<overview>
+<p>
+When opening a file with the <code>O_CREAT</code> or <code>O_TMPFILE</code> flag, the <code>mode</code> must
+be supplied. If the <code>mode</code> argument is omitted, some arbitrary bytes from the stack will be used
+as the file mode. This leaks some bits from the stack into the permissions of the file.
+</p>
+</overview>
+
+<recommendation>
+<p>
+The <code>mode</code> must be supplied when <code>O_CREAT</code> or <code>O_TMPFILE</code> is specified.
+</p>
+</recommendation>
+
+<example>
+<p>
+The first example opens a file with the <code>O_CREAT</code> flag without supplying the <code>mode</code>
+argument. In this case arbitrary bytes from the stack will be used as <code>mode</code> argument. The
+second example correctly supplies the <code>mode</code> argument and creates a file that is user readable
+and writable.
+</p>
+
+<sample src="OpenCallMissingModeArgument.c" />
+
+</example>
+</qhelp>

cpp/ql/src/Security/CWE/CWE-732/OpenCallMissingModeArgument.ql

@@ -0,0 +1,19 @@
+/**
+ * @name File opened with O_CREAT flag but without mode argument
+ * @description Opening a file with the O_CREAT flag but without mode argument reads arbitrary bytes from the stack.
+ * @kind problem
+ * @problem.severity error
+ * @security-severity 7.8
+ * @precision high
+ * @id cpp/open-call-with-mode-argument
+ * @tags security
+ *       external/cwe/cwe-732
+ */
+
+import cpp
+import FilePermissions
+
+from FileCreationWithOptionalModeExpr fc
+where not fc.hasModeArgument()
+select fc,
+  "A file is created here without providing a mode argument, which may leak bits from the stack."

cpp/ql/src/change-notes/2021-12-14-overruning-write-split.md

@@ -1,4 +0,0 @@
----
-category: newQuery
----
-* A new `cpp/very-likely-overruning-write` query has been added to the default query suite for C/C++. The query reports some results that were formerly flagged by `cpp/overruning-write`.

cpp/ql/src/change-notes/2021-12-30-ambiguously-signed-bit-field.md

@@ -1,5 +0,0 @@
----
-category: minorAnalysis
----
-* Added exception for GLib's gboolean to cpp/ambiguously-signed-bit-field.
-  This change reduces the number of false positives in the query.

cpp/ql/src/change-notes/2022-01-05-promote-uncontrolled-arithmetic.md

@@ -1,4 +0,0 @@
----
-category: newQuery
----
-* The "Uncontrolled data in arithmetic expression" (cpp/uncontrolled-arithmetic) query has been enhanced to reduce false positive results and its @precision increased to high.

cpp/ql/src/change-notes/2022-01-18-improper-null-termination.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* The "Potential improper null termination" (`cpp/improper-null-termination`) query now produces fewer false positive results around control flow branches and loops.

cpp/ql/src/change-notes/2022-01-19-cleartext-transmission.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* The "Cleartext transmission of sensitive information" (`cpp/cleartext-transmission`) query has been improved in several ways to reduce false positive results.

cpp/ql/src/change-notes/2022-01-20-return-stack-allocated-memory.md

@@ -1,5 +0,0 @@
----
-category: minorAnalysis
----
-* The `cpp/return-stack-allocated-memory` query has been improved to produce fewer false positives. The
-  query has also been converted to a `path-problem` query.

cpp/ql/src/change-notes/2022-01-24-cleartext-storage-file.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* The `cpp/cleartext-storage-file` query has been upgraded with non-local taint flow and has been converted to a `path-problem` query.

cpp/ql/src/change-notes/2022-01-24-return-stack-allocated-memory.md

@@ -1,4 +0,0 @@
----
-category: newQuery
----
-* The `security` tag has been added to the `cpp/return-stack-allocated-memory` query. As a result, its results will now appear by default.

cpp/ql/src/change-notes/2022-01-25-cleartext-storage-buffer.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* The `cpp/cleartext-storage-buffer` query has been updated to use the `semmle.code.cpp.dataflow.TaintTracking` library.

cpp/ql/src/change-notes/2022-01-26-unnamed-variable-declaration-hiding.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* Fix an issue with the `cpp/declaration-hides-variable` query where it would report variables that are unnamed in a database.

cpp/ql/src/change-notes/2022-01-28-cleartext-storage-file.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* The `cpp/cleartext-storage-file` query has been improved, removing false positives where data is written to a standard output stream.

cpp/ql/src/change-notes/2022-02-11-code-duplication.md

@@ -0,0 +1,4 @@
+---
+category: deprecated
+---
+* The `CodeDuplication.Copy`, `CodeDuplication.DuplicateBlock`, and `CodeDuplication.SimilarBlock` classes have been deprecated.

cpp/ql/src/change-notes/released/0.0.8.md

@@ -0,0 +1,18 @@
+## 0.0.8
+
+### New Queries
+
+* The `security` tag has been added to the `cpp/return-stack-allocated-memory` query. As a result, its results will now appear by default.
+* The "Uncontrolled data in arithmetic expression" (cpp/uncontrolled-arithmetic) query has been enhanced to reduce false positive results and its @precision increased to high.
+* A new `cpp/very-likely-overruning-write` query has been added to the default query suite for C/C++. The query reports some results that were formerly flagged by `cpp/overruning-write`.
+
+### Minor Analysis Improvements
+
+* Fix an issue with the `cpp/declaration-hides-variable` query where it would report variables that are unnamed in a database.
+* The `cpp/cleartext-storage-file` query has been upgraded with non-local taint flow and has been converted to a `path-problem` query.
+* The `cpp/return-stack-allocated-memory` query has been improved to produce fewer false positives. The
+  query has also been converted to a `path-problem` query.
+* The "Cleartext transmission of sensitive information" (`cpp/cleartext-transmission`) query has been improved in several ways to reduce false positive results.
+* The "Potential improper null termination" (`cpp/improper-null-termination`) query now produces fewer false positive results around control flow branches and loops.
+* Added exception for GLib's gboolean to cpp/ambiguously-signed-bit-field.
+  This change reduces the number of false positives in the query.

cpp/ql/src/change-notes/released/0.0.9.md

@@ -0,0 +1,13 @@
+## 0.0.9
+
+### New Queries
+
+* Added a new query, `cpp/open-call-with-mode-argument`, to detect when `open` or `openat` is called with the `O_CREAT` or `O_TMPFILE` flag but when the `mode` argument is omitted.
+
+### Minor Analysis Improvements
+
+* The "Cleartext transmission of sensitive information" (`cpp/cleartext-transmission`) query has been further improved to reduce false positive results, and upgraded from `medium` to `high` precision.
+* The "Cleartext transmission of sensitive information" (`cpp/cleartext-transmission`) query now finds more results, where a password is stored in a struct field or class member variable.
+* The `cpp/cleartext-storage-file` query has been improved, removing false positives where data is written to a standard output stream.
+* The `cpp/cleartext-storage-buffer` query has been updated to use the `semmle.code.cpp.dataflow.TaintTracking` library.
+* The `cpp/world-writable-file-creation` query now only detects `open` and `openat` calls with the `O_CREAT` or `O_TMPFILE` flag.

cpp/ql/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.0.7
+lastReleaseVersion: 0.0.9

cpp/ql/src/experimental/Best Practices/UselessTest.cpp

@@ -0,0 +1,9 @@
+void test(){
+    int a = 8;
+    int b = 9;
+
+    //Useless NonEquals
+    if(a==8 && a != 7) {}
+
+    while(a==8 && a!=7){}
+}

cpp/ql/src/experimental/Best Practices/UselessTest.qhelp

@@ -0,0 +1,18 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+
+<overview>
+<p>Comparison operations like <code>a==8 &amp;&amp; a!=7</code> contain a useless part : the non-equal part. This rule finds tests of this kind within an <code>if</code> or a <code>while</code> statement</p>
+</overview>
+
+<recommendation>
+<p>Remove the useless comparisons</p>
+</recommendation>
+
+<example>
+<sample src="UselessTest.cpp" />
+</example>
+
+</qhelp>

cpp/ql/src/experimental/Best Practices/UselessTest.ql

@@ -0,0 +1,45 @@
+/**
+ * @name Useless Test
+ * @description A boolean condition that is guaranteed to never be evaluated should be deleted.
+ * @kind problem
+ * @problem.severity warning
+ * @id cpp/uselesstest
+ * @tags reliability
+ *       readability
+ */
+
+import cpp
+import semmle.code.cpp.valuenumbering.GlobalValueNumbering
+
+predicate sameExpr(Expr e1, Expr e2) { globalValueNumber(e1).getAnExpr() = e2 }
+
+Element nearestParent(Expr e) {
+  if
+    e.getParent().(Expr).getConversion*() instanceof ParenthesisExpr or
+    e.getParent() instanceof IfStmt or
+    e.getParent() instanceof WhileStmt
+  then result = e.getParent()
+  else result = nearestParent(e.getParent())
+}
+
+from LogicalAndExpr b, EQExpr eq, NEExpr ne
+where
+  (
+    b.getAChild*() = eq and
+    b.getAChild*() = ne and
+    eq.getParent() instanceof LogicalAndExpr and
+    ne.getParent() instanceof LogicalAndExpr
+  ) and
+  (
+    eq.getLeftOperand() instanceof VariableAccess and ne.getLeftOperand() instanceof VariableAccess
+    or
+    eq.getLeftOperand() instanceof PointerDereferenceExpr and
+    ne.getLeftOperand() instanceof PointerDereferenceExpr
+  ) and
+  eq.getRightOperand() instanceof Literal and
+  ne.getRightOperand() instanceof Literal and
+  eq.getLeftOperand().getFullyConverted().getUnspecifiedType() =
+    ne.getLeftOperand().getFullyConverted().getUnspecifiedType() and
+  nearestParent(eq) = nearestParent(ne) and
+  sameExpr(eq.getLeftOperand(), ne.getLeftOperand())
+select ne, "Useless Test"

cpp/ql/src/external/CodeDuplication.qll

@@ -2,59 +2,40 @@
 
 import cpp
 
-private string relativePath(File file) { result = file.getRelativePath().replaceAll("\\", "/") }
-
-cached
-private predicate tokenLocation(string path, int sl, int sc, int ec, int el, Copy copy, int index) {
-  path = copy.sourceFile().getAbsolutePath() and
-  tokens(copy, index, sl, sc, ec, el)
-}
-
-/** A token block used for detection of duplicate and similar code. */
-class Copy extends @duplication_or_similarity {
-  /** Gets the index of the last token in this block. */
-  private int lastToken() { result = max(int i | tokens(this, i, _, _, _, _) | i) }
+private newtype TDuplicationOrSimilarity = MKDuplicationOrSimilarity()
 
+/**
+ * DEPRECATED: This class is no longer used.
+ *
+ * A token block used for detection of duplicate and similar code.
+ */
+class Copy extends TDuplicationOrSimilarity {
   /** Gets the index of the token in this block starting at the location `loc`, if any. */
-  int tokenStartingAt(Location loc) {
-    exists(string filepath, int startline, int startcol |
-      loc.hasLocationInfo(filepath, startline, startcol, _, _) and
-      tokenLocation(filepath, startline, startcol, _, _, this, result)
-    )
-  }
+  int tokenStartingAt(Location loc) { none() }
 
   /** Gets the index of the token in this block ending at the location `loc`, if any. */
-  int tokenEndingAt(Location loc) {
-    exists(string filepath, int endline, int endcol |
-      loc.hasLocationInfo(filepath, _, _, endline, endcol) and
-      tokenLocation(filepath, _, _, endline, endcol, this, result)
-    )
-  }
+  int tokenEndingAt(Location loc) { none() }
 
   /** Gets the line on which the first token in this block starts. */
-  int sourceStartLine() { tokens(this, 0, result, _, _, _) }
+  int sourceStartLine() { none() }
 
   /** Gets the column on which the first token in this block starts. */
-  int sourceStartColumn() { tokens(this, 0, _, result, _, _) }
+  int sourceStartColumn() { none() }
 
   /** Gets the line on which the last token in this block ends. */
-  int sourceEndLine() { tokens(this, this.lastToken(), _, _, result, _) }
+  int sourceEndLine() { none() }
 
   /** Gets the column on which the last token in this block ends. */
-  int sourceEndColumn() { tokens(this, this.lastToken(), _, _, _, result) }
+  int sourceEndColumn() { none() }
 
   /** Gets the number of lines containing at least (part of) one token in this block. */
   int sourceLines() { result = this.sourceEndLine() + 1 - this.sourceStartLine() }
 
   /** Gets an opaque identifier for the equivalence class of this block. */
-  int getEquivalenceClass() { duplicateCode(this, _, result) or similarCode(this, _, result) }
+  int getEquivalenceClass() { none() }
 
   /** Gets the source file in which this block appears. */
-  File sourceFile() {
-    exists(string name | duplicateCode(this, name, _) or similarCode(this, name, _) |
-      name.replaceAll("\\", "/") = relativePath(result)
-    )
-  }
+  File sourceFile() { none() }
 
   /**
    * Holds if this element is at the specified location.
@@ -77,15 +58,23 @@ class Copy extends @duplication_or_similarity {
   string toString() { none() }
 }
 
-/** A block of duplicated code. */
-class DuplicateBlock extends Copy, @duplication {
+/**
+ * DEPRECATED: This class is no longer used.
+ *
+ * A block of duplicated code.
+ */
+class DuplicateBlock extends Copy {
   override string toString() {
     result = "Duplicate code: " + this.sourceLines() + " duplicated lines."
   }
 }
 
-/** A block of similar code. */
-class SimilarBlock extends Copy, @similarity {
+/**
+ * DEPRECATED: This class is no longer used.
+ *
+ * A block of similar code.
+ */
+class SimilarBlock extends Copy {
   override string toString() {
     result = "Similar code: " + this.sourceLines() + " almost duplicated lines."
   }

cpp/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-queries
-version: 0.0.8-dev
+version: 0.0.10-dev
 groups: 
   - cpp
   - queries

cpp/ql/test/TestUtilities/InlineExpectationsTest.qll

@@ -93,7 +93,7 @@
 private import InlineExpectationsTestPrivate
 
 /**
- * Base class for tests with inline expectations. The test extends this class to provide the actual
+ * The base class for tests with inline expectations. The test extends this class to provide the actual
  * results of the query, which are then compared with the expected results in comments to produce a
  * list of failure messages that point out where the actual results differ from the expected
  * results.
@@ -123,6 +123,15 @@ abstract class InlineExpectationsTest extends string {
    */
   abstract predicate hasActualResult(Location location, string element, string tag, string value);
 
+  /**
+   * Like `hasActualResult`, but returns results that do not require a matching annotation.
+   * A failure will still arise if there is an annotation that does not match any results, but not vice versa.
+   * Override this predicate to specify optional results.
+   */
+  predicate hasOptionalResult(Location location, string element, string tag, string value) {
+    none()
+  }
+
   final predicate hasFailureMessage(FailureLocatable element, string message) {
     exists(ActualResult actualResult |
       actualResult.getTest() = this and
@@ -134,7 +143,8 @@ abstract class InlineExpectationsTest extends string {
         )
         or
         not exists(ValidExpectation expectation | expectation.matchesActualResult(actualResult)) and
-        message = "Unexpected result: " + actualResult.getExpectationText()
+        message = "Unexpected result: " + actualResult.getExpectationText() and
+        not actualResult.isOptional()
       )
     )
     or
@@ -243,9 +253,13 @@ private string expectationPattern() {
 
 private newtype TFailureLocatable =
   TActualResult(
-    InlineExpectationsTest test, Location location, string element, string tag, string value
+    InlineExpectationsTest test, Location location, string element, string tag, string value,
+    boolean optional
   ) {
-    test.hasActualResult(location, element, tag, value)
+    test.hasActualResult(location, element, tag, value) and
+    optional = false
+    or
+    test.hasOptionalResult(location, element, tag, value) and optional = true
   } or
   TValidExpectation(ExpectationComment comment, string tag, string value, string knownFailure) {
     exists(TColumn column, string tags |
@@ -277,8 +291,9 @@ class ActualResult extends FailureLocatable, TActualResult {
   string element;
   string tag;
   string value;
+  boolean optional;
 
-  ActualResult() { this = TActualResult(test, location, element, tag, value) }
+  ActualResult() { this = TActualResult(test, location, element, tag, value, optional) }
 
   override string toString() { result = element }
 
@@ -289,6 +304,8 @@ class ActualResult extends FailureLocatable, TActualResult {
   override string getTag() { result = tag }
 
   override string getValue() { result = value }
+
+  predicate isOptional() { optional = true }
 }
 
 abstract private class Expectation extends FailureLocatable {

cpp/ql/test/library-tests/ir/ir/PrintAST.expected

@@ -11332,6 +11332,64 @@ ir.cpp:
 # 1447|                         Type = [Struct] POD_Derived
 # 1447|                         ValueCategory = prvalue
 # 1448|     getStmt(4): [ReturnStmt] return ...
+# 1450| [CopyAssignmentOperator] Inheritance_Test_B& Inheritance_Test_B::operator=(Inheritance_Test_B const&)
+# 1450|   <params>: 
+#-----|     getParameter(0): [Parameter] (unnamed parameter 0)
+#-----|         Type = [LValueReferenceType] const Inheritance_Test_B &
+# 1450| [Constructor] void Inheritance_Test_B::Inheritance_Test_B()
+# 1450|   <params>: 
+# 1451| [Destructor] void Inheritance_Test_B::~Inheritance_Test_B()
+# 1451|   <params>: 
+# 1451|   getEntryPoint(): [BlockStmt] { ... }
+# 1451|     getStmt(0): [ReturnStmt] return ...
+# 1451|   <destructions>: 
+# 1454| [CopyAssignmentOperator] Inheritance_Test_A& Inheritance_Test_A::operator=(Inheritance_Test_A const&)
+# 1454|   <params>: 
+#-----|     getParameter(0): [Parameter] (unnamed parameter 0)
+#-----|         Type = [LValueReferenceType] const Inheritance_Test_A &
+# 1454| [MoveAssignmentOperator] Inheritance_Test_A& Inheritance_Test_A::operator=(Inheritance_Test_A&&)
+# 1454|   <params>: 
+#-----|     getParameter(0): [Parameter] (unnamed parameter 0)
+#-----|         Type = [RValueReferenceType] Inheritance_Test_A &&
+# 1454| [CopyConstructor] void Inheritance_Test_A::Inheritance_Test_A(Inheritance_Test_A const&)
+# 1454|   <params>: 
+#-----|     getParameter(0): [Parameter] (unnamed parameter 0)
+#-----|         Type = [LValueReferenceType] const Inheritance_Test_A &
+# 1454| [MoveConstructor] void Inheritance_Test_A::Inheritance_Test_A(Inheritance_Test_A&&)
+# 1454|   <params>: 
+#-----|     getParameter(0): [Parameter] (unnamed parameter 0)
+#-----|         Type = [RValueReferenceType] Inheritance_Test_A &&
+# 1454| [Destructor] void Inheritance_Test_A::~Inheritance_Test_A()
+# 1454|   <params>: 
+# 1457| [Constructor] void Inheritance_Test_A::Inheritance_Test_A()
+# 1457|   <params>: 
+# 1457|   <initializations>: 
+# 1457|     getInitializer(0): (no string representation)
+# 1457|         Type = [Struct] Inheritance_Test_B
+# 1457|         ValueCategory = prvalue
+# 1457|     getInitializer(1): [ConstructorFieldInit] constructor init of field x
+# 1457|         Type = [IntType] int
+# 1457|         ValueCategory = prvalue
+# 1457|       getExpr(): [Literal] 42
+# 1457|           Type = [IntType] int
+# 1457|           Value = [Literal] 42
+# 1457|           ValueCategory = prvalue
+# 1457|   getEntryPoint(): [BlockStmt] { ... }
+# 1458|     getStmt(0): [ExprStmt] ExprStmt
+# 1458|       getExpr(): [AssignExpr] ... = ...
+# 1458|           Type = [IntType] int
+# 1458|           ValueCategory = lvalue
+# 1458|         getLValue(): [PointerFieldAccess] y
+# 1458|             Type = [IntType] int
+# 1458|             ValueCategory = lvalue
+# 1458|           getQualifier(): [ThisExpr] this
+# 1458|               Type = [PointerType] Inheritance_Test_A *
+# 1458|               ValueCategory = prvalue(load)
+# 1458|         getRValue(): [Literal] 3
+# 1458|             Type = [IntType] int
+# 1458|             Value = [Literal] 3
+# 1458|             ValueCategory = prvalue
+# 1459|     getStmt(1): [ReturnStmt] return ...
 perf-regression.cpp:
 #    4| [CopyAssignmentOperator] Big& Big::operator=(Big const&)
 #    4|   <params>: 

cpp/ql/test/library-tests/ir/ir/ir.cpp

@@ -1447,4 +1447,15 @@ void temporary_hierarchy() {
     float f = (returnValue<POD_Derived>()).f();
 }
 
+struct Inheritance_Test_B {
+  ~Inheritance_Test_B() {}
+};
+
+struct Inheritance_Test_A : public Inheritance_Test_B {
+  int x;
+  int y;
+  Inheritance_Test_A() : x(42) {
+    y = 3;
+  }
+};
 // semmle-extractor-options: -std=c++17 --clang

cpp/ql/test/library-tests/ir/ir/operand_locations.expected

@@ -6626,6 +6626,36 @@
 | ir.cpp:1447:44:1447:44 | ChiTotal | total:m1447_5 |
 | ir.cpp:1447:44:1447:44 | SideEffect | ~m1447_5 |
 | ir.cpp:1447:44:1447:44 | StoreValue | r1447_8 |
+| ir.cpp:1451:3:1451:21 | Address | &:r1451_5 |
+| ir.cpp:1451:3:1451:21 | Address | &:r1451_5 |
+| ir.cpp:1451:3:1451:21 | Address | &:r1451_7 |
+| ir.cpp:1451:3:1451:21 | Address | &:r1451_7 |
+| ir.cpp:1451:3:1451:21 | ChiPartial | partial:m1451_3 |
+| ir.cpp:1451:3:1451:21 | ChiTotal | total:m1451_2 |
+| ir.cpp:1451:3:1451:21 | Load | m1451_6 |
+| ir.cpp:1451:3:1451:21 | SideEffect | m1451_3 |
+| ir.cpp:1451:3:1451:21 | SideEffect | m1451_8 |
+| ir.cpp:1457:3:1457:20 | Address | &:r1457_5 |
+| ir.cpp:1457:3:1457:20 | Address | &:r1457_5 |
+| ir.cpp:1457:3:1457:20 | Address | &:r1457_7 |
+| ir.cpp:1457:3:1457:20 | Address | &:r1457_7 |
+| ir.cpp:1457:3:1457:20 | ChiPartial | partial:m1457_3 |
+| ir.cpp:1457:3:1457:20 | ChiTotal | total:m1457_2 |
+| ir.cpp:1457:3:1457:20 | Load | m1457_6 |
+| ir.cpp:1457:3:1457:20 | SideEffect | m1457_3 |
+| ir.cpp:1457:3:1457:20 | SideEffect | m1458_6 |
+| ir.cpp:1457:3:1457:20 | Unary | m1457_6 |
+| ir.cpp:1457:26:1457:30 | Address | &:r1457_9 |
+| ir.cpp:1457:26:1457:30 | ChiPartial | partial:m1457_11 |
+| ir.cpp:1457:26:1457:30 | ChiTotal | total:m1457_8 |
+| ir.cpp:1457:26:1457:30 | StoreValue | r1457_10 |
+| ir.cpp:1458:5:1458:5 | Address | &:r1458_2 |
+| ir.cpp:1458:5:1458:5 | Address | &:r1458_4 |
+| ir.cpp:1458:5:1458:5 | Load | m1457_6 |
+| ir.cpp:1458:5:1458:5 | Unary | r1458_3 |
+| ir.cpp:1458:5:1458:9 | ChiPartial | partial:m1458_5 |
+| ir.cpp:1458:5:1458:9 | ChiTotal | total:m1457_12 |
+| ir.cpp:1458:9:1458:9 | StoreValue | r1458_1 |
 | perf-regression.cpp:6:3:6:5 | Address | &:r6_5 |
 | perf-regression.cpp:6:3:6:5 | Address | &:r6_5 |
 | perf-regression.cpp:6:3:6:5 | Address | &:r6_7 |

cpp/ql/test/library-tests/ir/ir/raw_ir.expected

@@ -7858,6 +7858,44 @@ ir.cpp:
 # 1443|     v1443_5(void)               = AliasedUse                                        : ~m?
 # 1443|     v1443_6(void)               = ExitFunction                                      : 
 
+# 1451| void Inheritance_Test_B::~Inheritance_Test_B()
+# 1451|   Block 0
+# 1451|     v1451_1(void)                       = EnterFunction                : 
+# 1451|     mu1451_2(unknown)                   = AliasedDefinition            : 
+# 1451|     mu1451_3(unknown)                   = InitializeNonLocal           : 
+# 1451|     r1451_4(glval<unknown>)             = VariableAddress[#this]       : 
+# 1451|     mu1451_5(glval<Inheritance_Test_B>) = InitializeParameter[#this]   : &:r1451_4
+# 1451|     r1451_6(glval<Inheritance_Test_B>)  = Load[#this]                  : &:r1451_4, ~m?
+# 1451|     mu1451_7(Inheritance_Test_B)        = InitializeIndirection[#this] : &:r1451_6
+# 1451|     v1451_8(void)                       = NoOp                         : 
+# 1451|     v1451_9(void)                       = ReturnIndirection[#this]     : &:r1451_6, ~m?
+# 1451|     v1451_10(void)                      = ReturnVoid                   : 
+# 1451|     v1451_11(void)                      = AliasedUse                   : ~m?
+# 1451|     v1451_12(void)                      = ExitFunction                 : 
+
+# 1457| void Inheritance_Test_A::Inheritance_Test_A()
+# 1457|   Block 0
+# 1457|     v1457_1(void)                       = EnterFunction                : 
+# 1457|     mu1457_2(unknown)                   = AliasedDefinition            : 
+# 1457|     mu1457_3(unknown)                   = InitializeNonLocal           : 
+# 1457|     r1457_4(glval<unknown>)             = VariableAddress[#this]       : 
+# 1457|     mu1457_5(glval<Inheritance_Test_A>) = InitializeParameter[#this]   : &:r1457_4
+# 1457|     r1457_6(glval<Inheritance_Test_A>)  = Load[#this]                  : &:r1457_4, ~m?
+# 1457|     mu1457_7(Inheritance_Test_A)        = InitializeIndirection[#this] : &:r1457_6
+# 1457|     r1457_8(glval<int>)                 = FieldAddress[x]              : mu1457_5
+# 1457|     r1457_9(int)                        = Constant[42]                 : 
+# 1457|     mu1457_10(int)                      = Store[?]                     : &:r1457_8, r1457_9
+# 1458|     r1458_1(int)                        = Constant[3]                  : 
+# 1458|     r1458_2(glval<unknown>)             = VariableAddress[#this]       : 
+# 1458|     r1458_3(Inheritance_Test_A *)       = Load[#this]                  : &:r1458_2, ~m?
+# 1458|     r1458_4(glval<int>)                 = FieldAddress[y]              : r1458_3
+# 1458|     mu1458_5(int)                       = Store[?]                     : &:r1458_4, r1458_1
+# 1459|     v1459_1(void)                       = NoOp                         : 
+# 1457|     v1457_11(void)                      = ReturnIndirection[#this]     : &:r1457_6, ~m?
+# 1457|     v1457_12(void)                      = ReturnVoid                   : 
+# 1457|     v1457_13(void)                      = AliasedUse                   : ~m?
+# 1457|     v1457_14(void)                      = ExitFunction                 : 
+
 perf-regression.cpp:
 #    6| void Big::Big()
 #    6|   Block 0

cpp/ql/test/query-tests/Best Practices/Hiding/DeclarationHidesVariable/DeclarationHidesVariable.expected

@@ -1,2 +1,3 @@
 | hiding.cpp:6:17:6:17 | i | Variable i hides another variable of the same name (on $@). | hiding.cpp:4:13:4:13 | i | line 4 |
 | hiding.cpp:18:15:18:15 | k | Variable k hides another variable of the same name (on $@). | hiding.cpp:15:11:15:11 | k | line 15 |
+| hiding.cpp:38:11:38:11 | x | Variable x hides another variable of the same name (on $@). | hiding.cpp:36:9:36:9 | x | line 36 |

cpp/ql/test/query-tests/Best Practices/Hiding/DeclarationHidesVariable/hiding.cpp

@@ -35,7 +35,7 @@ void structuredBinding() {
   int xs[1] = {1};
   auto [x] = xs;
   {
-    auto [x] = xs; // BAD [NOT DETECTED]
+    auto [x] = xs; // BAD
     auto [y] = xs; // GOOD
   }
 }

cpp/ql/test/query-tests/Security/CWE/CWE-311/semmle/tests/CleartextTransmission.expected

@@ -1,4 +1,5 @@
 edges
+| test2.cpp:63:24:63:31 | password | test2.cpp:63:16:63:20 | call to crypt |
 | test3.cpp:17:28:17:36 | password1 | test3.cpp:22:15:22:23 | password1 |
 | test3.cpp:17:51:17:59 | password2 | test3.cpp:26:15:26:23 | password2 |
 | test3.cpp:45:8:45:15 | password | test3.cpp:47:15:47:22 | password |
@@ -89,11 +90,16 @@ edges
 | test3.cpp:398:18:398:25 | password | test3.cpp:400:15:400:23 | & ... |
 | test3.cpp:398:18:398:25 | password | test3.cpp:400:16:400:23 | password |
 | test3.cpp:398:18:398:25 | password | test3.cpp:400:33:400:40 | password |
+| test3.cpp:421:21:421:28 | password | test3.cpp:421:3:421:17 | call to decrypt_inplace |
+| test3.cpp:429:7:429:14 | password | test3.cpp:431:8:431:15 | password |
 | test.cpp:41:23:41:43 | cleartext password! | test.cpp:48:21:48:27 | call to encrypt |
 | test.cpp:41:23:41:43 | cleartext password! | test.cpp:48:29:48:39 | thePassword |
 | test.cpp:66:23:66:43 | cleartext password! | test.cpp:76:21:76:27 | call to encrypt |
 | test.cpp:66:23:66:43 | cleartext password! | test.cpp:76:29:76:39 | thePassword |
 nodes
+| test2.cpp:63:16:63:20 | call to crypt | semmle.label | call to crypt |
+| test2.cpp:63:24:63:31 | password | semmle.label | password |
+| test2.cpp:63:24:63:31 | password | semmle.label | password |
 | test3.cpp:17:28:17:36 | password1 | semmle.label | password1 |
 | test3.cpp:17:51:17:59 | password2 | semmle.label | password2 |
 | test3.cpp:22:15:22:23 | password1 | semmle.label | password1 |
@@ -208,6 +214,13 @@ nodes
 | test3.cpp:400:15:400:23 | & ... | semmle.label | & ... |
 | test3.cpp:400:16:400:23 | password | semmle.label | password |
 | test3.cpp:400:33:400:40 | password | semmle.label | password |
+| test3.cpp:414:17:414:24 | password | semmle.label | password |
+| test3.cpp:420:17:420:24 | password | semmle.label | password |
+| test3.cpp:421:3:421:17 | call to decrypt_inplace | semmle.label | call to decrypt_inplace |
+| test3.cpp:421:21:421:28 | password | semmle.label | password |
+| test3.cpp:421:21:421:28 | password | semmle.label | password |
+| test3.cpp:429:7:429:14 | password | semmle.label | password |
+| test3.cpp:431:8:431:15 | password | semmle.label | password |
 | test.cpp:41:23:41:43 | cleartext password! | semmle.label | cleartext password! |
 | test.cpp:48:21:48:27 | call to encrypt | semmle.label | call to encrypt |
 | test.cpp:48:29:48:39 | thePassword | semmle.label | thePassword |
@@ -238,3 +251,6 @@ subpaths
 | test3.cpp:300:2:300:5 | call to send | test3.cpp:308:58:308:66 | password2 | test3.cpp:300:14:300:17 | data | This operation transmits 'data', which may contain unencrypted sensitive data from $@ | test3.cpp:308:58:308:66 | password2 | password2 |
 | test3.cpp:341:4:341:7 | call to recv | test3.cpp:339:9:339:16 | password | test3.cpp:341:16:341:23 | password | This operation receives into 'password', which may put unencrypted sensitive data into $@ | test3.cpp:339:9:339:16 | password | password |
 | test3.cpp:388:3:388:6 | call to recv | test3.cpp:386:8:386:15 | password | test3.cpp:388:15:388:22 | password | This operation receives into 'password', which may put unencrypted sensitive data into $@ | test3.cpp:386:8:386:15 | password | password |
+| test3.cpp:414:3:414:6 | call to recv | test3.cpp:414:17:414:24 | password | test3.cpp:414:17:414:24 | password | This operation receives into 'password', which may put unencrypted sensitive data into $@ | test3.cpp:414:17:414:24 | password | password |
+| test3.cpp:420:3:420:6 | call to recv | test3.cpp:420:17:420:24 | password | test3.cpp:420:17:420:24 | password | This operation receives into 'password', which may put unencrypted sensitive data into $@ | test3.cpp:420:17:420:24 | password | password |
+| test3.cpp:431:2:431:6 | call to fgets | test3.cpp:429:7:429:14 | password | test3.cpp:431:8:431:15 | password | This operation receives into 'password', which may put unencrypted sensitive data into $@ | test3.cpp:429:7:429:14 | password | password |

cpp/ql/test/query-tests/Security/CWE/CWE-311/semmle/tests/test3.cpp

@@ -411,13 +411,66 @@ void test_member_password()
 	{
 		packet p;
 
-		recv(val(), p.password, 256, val()); // BAD: not encrypted [NOT DETECTED]
+		recv(val(), p.password, 256, val()); // BAD: not encrypted
 	}
 
 	{
 		packet p;
 
-		recv(val(), p.password, 256, val()); // GOOD: password is encrypted
+		recv(val(), p.password, 256, val()); // GOOD: password is encrypted [FALSE POSITIVE]
 		decrypt_inplace(p.password); // proof that `password` was in fact encrypted
 	}
 }
+
+extern FILE *stdin;
+
+void test_stdin_param(FILE *stream)
+{
+	char password[128];
+
+	fgets(password, 128, stream); // GOOD: from standard input (see call below) [FALSE POSITIVE]
+}
+
+void test_stdin()
+{
+	char password[128];
+	FILE *f = stdin;
+
+	fgets(password, 128, stdin); // GOOD: from standard input
+	fgets(password, 128, f); // GOOD: from standard input
+	test_stdin_param(stdin);
+}
+
+int open(const char *filename, int b);
+
+void test_tty()
+{
+	{
+		char password[256];
+		int f;
+
+		f = open("/dev/tty", val());
+		recv(f, password, 256, val()); // GOOD: from terminal
+	}
+
+	{
+		char password[256];
+		int f;
+
+		f = STDIN_FILENO;
+		recv(f, password, 256, val()); // GOOD: from stdin
+	}
+
+	{
+		char password[256];
+		int f;
+
+		f = open("/dev/tty", val());
+		if (f == -1)
+		{
+			f = STDIN_FILENO;
+		}
+
+		recv(f, password, 256, val()); // GOOD: from terminal or stdin
+	}
+}

cpp/ql/test/query-tests/Security/CWE/CWE-732/OpenCallMissingModeArgument.c

@@ -0,0 +1,29 @@
+typedef unsigned int mode_t;
+
+#define O_RDWR     0x0002
+#define O_CLOEXEC  0x0040
+#define O_NONBLOCK 0x0080
+#define O_CREAT    0x0200
+#define O_APPEND   0x0800
+#define O_TMPFILE  0x2000
+
+int open(const char *pathname, int flags, ...);
+
+int openat(int dirfd, const char *pathname, int flags, ...);
+
+const char *a_file = "/a_file";
+
+void test_open() {
+  open(a_file, O_NONBLOCK); // GOOD
+  open(a_file, O_RDWR | O_CLOEXEC); // GOOD
+  open(a_file, O_APPEND); // GOOD
+  open(a_file, O_CREAT); // BAD
+  open(a_file, O_CREAT, 0); // GOOD
+  open(a_file, O_TMPFILE); // BAD
+  open(a_file, O_TMPFILE, 0); // GOOD
+  openat(0, a_file, O_APPEND); // GOOD
+  openat(0, a_file, O_CREAT); // BAD
+  openat(0, a_file, O_CREAT, 0); // GOOD
+  openat(0, a_file, O_TMPFILE); // BAD
+  openat(0, a_file, O_TMPFILE, 0); // GOOD
+}

cpp/ql/test/query-tests/Security/CWE/CWE-732/OpenCallMissingModeArgument.expected

@@ -0,0 +1,4 @@
+| OpenCallMissingModeArgument.c:20:3:20:6 | call to open | A file is created here without providing a mode argument, which may leak bits from the stack. |
+| OpenCallMissingModeArgument.c:22:3:22:6 | call to open | A file is created here without providing a mode argument, which may leak bits from the stack. |
+| OpenCallMissingModeArgument.c:25:3:25:8 | call to openat | A file is created here without providing a mode argument, which may leak bits from the stack. |
+| OpenCallMissingModeArgument.c:27:3:27:8 | call to openat | A file is created here without providing a mode argument, which may leak bits from the stack. |

cpp/ql/test/query-tests/Security/CWE/CWE-732/OpenCallMissingModeArgument.qlref

@@ -0,0 +1 @@
+Security/CWE/CWE-732/OpenCallMissingModeArgument.ql

csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/Semmle.Autobuild.CSharp.Tests.csproj

@@ -1,6 +1,6 @@
 <Project Sdk="Microsoft.NET.Sdk">
 	<PropertyGroup>
-		<TargetFramework>net5.0</TargetFramework>
+		<TargetFramework>net6.0</TargetFramework>
 		<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
 		<RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
 		<Nullable>enable</Nullable>

csharp/autobuilder/Semmle.Autobuild.CSharp/Semmle.Autobuild.CSharp.csproj

@@ -1,6 +1,6 @@
 <Project Sdk="Microsoft.NET.Sdk">
 	<PropertyGroup>
-		<TargetFramework>net5.0</TargetFramework>
+		<TargetFramework>net6.0</TargetFramework>
 		<AssemblyName>Semmle.Autobuild.CSharp</AssemblyName>
 		<RootNamespace>Semmle.Autobuild.CSharp</RootNamespace>
 		<ApplicationIcon/>

csharp/autobuilder/Semmle.Autobuild.Shared/Semmle.Autobuild.Shared.csproj

@@ -1,6 +1,6 @@
 <Project Sdk="Microsoft.NET.Sdk">
 	<PropertyGroup>
-		<TargetFramework>net5.0</TargetFramework>
+		<TargetFramework>net6.0</TargetFramework>
 		<AssemblyName>Semmle.Autobuild.Shared</AssemblyName>
 		<RootNamespace>Semmle.Autobuild.Shared</RootNamespace>
 		<GenerateAssemblyInfo>false</GenerateAssemblyInfo>

csharp/downgrades/4b9c288ece4218779b6113b17af42795f4bbbee1/upgrade.properties

@@ -0,0 +1,3 @@
+description: Remove 'lambda_expr_return_type' relation.
+compatibility: backwards
+lambda_expr_return_type.rel: delete

csharp/downgrades/a696c8bae067f69ab3208e98ce35f4fdf7efb68b/upgrade.properties

@@ -0,0 +1,3 @@
+description: Add externalData relation
+compatibility: full
+externalData.rel: delete

csharp/downgrades/ab09ac8287516082b7a7367f8fda1862b1be47c5/upgrade.properties

@@ -0,0 +1,3 @@
+description: Remove 'kind' from 'attributes'.
+compatability: full
+attributes.rel: reorder attributes.rel (int id, int kind, int type_id, int target) id type_id target