Merge pull request #10022 from github/release-prep/2.10.3

Release preparation for version 2.10.3

.github/actions/fetch-codeql/action.yml

@@ -3,22 +3,12 @@ description: Fetches the latest version of CodeQL
 runs:
   using: composite
   steps:
-    - name: Select platform - Linux
-      if: runner.os == 'Linux'
-      shell: bash
-      run: echo "GA_CODEQL_CLI_PLATFORM=linux64" >> $GITHUB_ENV
-
-    - name: Select platform - MacOS
-      if: runner.os == 'MacOS'
-      shell: bash
-      run: echo "GA_CODEQL_CLI_PLATFORM=osx64" >> $GITHUB_ENV
-
     - name: Fetch CodeQL
       shell: bash
       run: |
-        LATEST=$(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | grep -v beta | sort --version-sort | tail -1)
-        gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql-$GA_CODEQL_CLI_PLATFORM.zip "$LATEST"
-        unzip -q -d "${RUNNER_TEMP}" codeql-$GA_CODEQL_CLI_PLATFORM.zip
-        echo "${RUNNER_TEMP}/codeql" >> "${GITHUB_PATH}"
+        gh extension install github/gh-codeql
+        gh codeql set-channel nightly
+        gh codeql version
+        gh codeql version --format=json | jq -r .unpackedLocation >> "${GITHUB_PATH}"
       env:
         GITHUB_TOKEN: ${{ github.token }}

.github/workflows/check-change-note.yml

@@ -10,6 +10,7 @@ on:
       - "*/ql/lib/**/*.qll"
       - "!**/experimental/**"
       - "!ql/**"
+      - "!swift/**"
       - ".github/workflows/check-change-note.yml"
 
 jobs:

.github/workflows/check-qldoc.yml

@@ -5,6 +5,7 @@ on:
     paths:
       - "*/ql/lib/**"
       - .github/workflows/check-qldoc.yml
+      - .github/actions/fetch-codeql/action.yml
     branches:
       - main
       - "rc/*"
@@ -14,18 +15,13 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - name: Install CodeQL
-        run: |
-          gh extension install github/gh-codeql
-          gh codeql set-channel nightly
-          gh codeql version
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-
       - uses: actions/checkout@v3
         with:
           fetch-depth: 2
 
+      - name: Install CodeQL
+        uses: ./.github/actions/fetch-codeql
+
       - name: Check QLdoc coverage
         shell: bash
         run: |
@@ -34,7 +30,7 @@ jobs:
           changed_lib_packs="$(git diff --name-only --diff-filter=ACMRT HEAD^ HEAD | { grep -Po '^(?!swift)[a-z]*/ql/lib' || true; } | sort -u)"
           for pack_dir in ${changed_lib_packs}; do
             lang="${pack_dir%/ql/lib}"
-            gh codeql generate library-doc-coverage --output="${RUNNER_TEMP}/${lang}-current.txt" --dir="${pack_dir}"
+            codeql generate library-doc-coverage --output="${RUNNER_TEMP}/${lang}-current.txt" --dir="${pack_dir}"
           done
           git checkout HEAD^
           for pack_dir in ${changed_lib_packs}; do
@@ -42,7 +38,7 @@ jobs:
             # In this case the right thing to do is to skip the check.
             [[ ! -d "${pack_dir}" ]] && continue
             lang="${pack_dir%/ql/lib}"
-            gh codeql generate library-doc-coverage --output="${RUNNER_TEMP}/${lang}-baseline.txt" --dir="${pack_dir}"
+            codeql generate library-doc-coverage --output="${RUNNER_TEMP}/${lang}-baseline.txt" --dir="${pack_dir}"
             awk -F, '{gsub(/"/,""); if ($4==0 && $6=="public") print "\""$3"\"" }' "${RUNNER_TEMP}/${lang}-current.txt" | sort -u > "${RUNNER_TEMP}/current-undocumented.txt"
             awk -F, '{gsub(/"/,""); if ($4==0 && $6=="public") print "\""$3"\"" }' "${RUNNER_TEMP}/${lang}-baseline.txt" | sort -u > "${RUNNER_TEMP}/baseline-undocumented.txt"
             UNDOCUMENTED="$(grep -f <(comm -13 "${RUNNER_TEMP}/baseline-undocumented.txt" "${RUNNER_TEMP}/current-undocumented.txt") "${RUNNER_TEMP}/${lang}-current.txt" || true)"

.github/workflows/csv-coverage-metrics.yml

@@ -12,6 +12,7 @@ on:
       - main
     paths:
       - ".github/workflows/csv-coverage-metrics.yml"
+      - ".github/actions/fetch-codeql/action.yml"
 
 jobs:
   publish-java:

.github/workflows/csv-coverage-pr-artifacts.yml

@@ -3,18 +3,20 @@ name: Check framework coverage changes
 on:
   pull_request:
     paths:
-      - '.github/workflows/csv-coverage-pr-comment.yml'
-      - '*/ql/src/**/*.ql'
-      - '*/ql/src/**/*.qll'
-      - '*/ql/lib/**/*.ql'
-      - '*/ql/lib/**/*.qll'
-      - 'misc/scripts/library-coverage/*.py'
+      - ".github/workflows/csv-coverage-pr-comment.yml"
+      - ".github/workflows/csv-coverage-pr-artifacts.yml"
+      - ".github/actions/fetch-codeql/action.yml"
+      - "*/ql/src/**/*.ql"
+      - "*/ql/src/**/*.qll"
+      - "*/ql/lib/**/*.ql"
+      - "*/ql/lib/**/*.qll"
+      - "misc/scripts/library-coverage/*.py"
       # input data files
-      - '*/documentation/library-coverage/cwe-sink.csv'
-      - '*/documentation/library-coverage/frameworks.csv'
+      - "*/documentation/library-coverage/cwe-sink.csv"
+      - "*/documentation/library-coverage/frameworks.csv"
     branches:
       - main
-      - 'rc/*'
+      - "rc/*"
 
 jobs:
   generate:
@@ -23,77 +25,72 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - name: Dump GitHub context
-      env:
-        GITHUB_CONTEXT: ${{ toJSON(github.event) }}
-      run: echo "$GITHUB_CONTEXT"
-    - name: Clone self (github/codeql) - MERGE
-      uses: actions/checkout@v3
-      with:
-        path: merge
-    - name: Clone self (github/codeql) - BASE
-      uses: actions/checkout@v3
-      with:
-        fetch-depth: 2
-        path: base
-    - run: |
-        git checkout HEAD^1
-        git log -1 --format='%H'
-      working-directory: base
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v4
-      with:
-        python-version: 3.8
-    - name: Download CodeQL CLI
-      env:
-         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: |
-         gh release download --repo "github/codeql-cli-binaries" --pattern "codeql-linux64.zip"
-    - name: Unzip CodeQL CLI
-      run: unzip -d codeql-cli codeql-linux64.zip
-    - name: Generate CSV files on merge commit of the PR
-      run: |
-        echo "Running generator on merge"
-        PATH="$PATH:codeql-cli/codeql" python merge/misc/scripts/library-coverage/generate-report.py ci merge merge
-        mkdir out_merge
-        cp framework-coverage-*.csv out_merge/
-        cp framework-coverage-*.rst out_merge/
-    - name: Generate CSV files on base commit of the PR
-      run: |
-        echo "Running generator on base"
-        PATH="$PATH:codeql-cli/codeql" python base/misc/scripts/library-coverage/generate-report.py ci base base
-        mkdir out_base
-        cp framework-coverage-*.csv out_base/
-        cp framework-coverage-*.rst out_base/
-    - name: Generate diff of coverage reports
-      run: |
-        python base/misc/scripts/library-coverage/compare-folders.py out_base out_merge comparison.md
-    - name: Upload CSV package list
-      uses: actions/upload-artifact@v3
-      with:
-        name: csv-framework-coverage-merge
-        path: |
-          out_merge/framework-coverage-*.csv
-          out_merge/framework-coverage-*.rst
-    - name: Upload CSV package list
-      uses: actions/upload-artifact@v3
-      with:
-        name: csv-framework-coverage-base
-        path: |
-          out_base/framework-coverage-*.csv
-          out_base/framework-coverage-*.rst
-    - name: Upload comparison results
-      uses: actions/upload-artifact@v3
-      with:
-        name: comparison
-        path: |
-          comparison.md
-    - name: Save PR number
-      run: |
-        mkdir -p pr
-        echo ${{ github.event.pull_request.number }} > pr/NR
-    - name: Upload PR number
-      uses: actions/upload-artifact@v3
-      with:
-        name: pr
-        path: pr/
+      - name: Dump GitHub context
+        env:
+          GITHUB_CONTEXT: ${{ toJSON(github.event) }}
+        run: echo "$GITHUB_CONTEXT"
+      - name: Clone self (github/codeql) - MERGE
+        uses: actions/checkout@v3
+        with:
+          path: merge
+      - name: Clone self (github/codeql) - BASE
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+          path: base
+      - run: |
+          git checkout HEAD^1
+          git log -1 --format='%H'
+        working-directory: base
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+      - name: Download CodeQL CLI
+        uses: ./merge/.github/actions/fetch-codeql
+      - name: Generate CSV files on merge commit of the PR
+        run: |
+          echo "Running generator on merge"
+          python merge/misc/scripts/library-coverage/generate-report.py ci merge merge
+          mkdir out_merge
+          cp framework-coverage-*.csv out_merge/
+          cp framework-coverage-*.rst out_merge/
+      - name: Generate CSV files on base commit of the PR
+        run: |
+          echo "Running generator on base"
+          python base/misc/scripts/library-coverage/generate-report.py ci base base
+          mkdir out_base
+          cp framework-coverage-*.csv out_base/
+          cp framework-coverage-*.rst out_base/
+      - name: Generate diff of coverage reports
+        run: |
+          python base/misc/scripts/library-coverage/compare-folders.py out_base out_merge comparison.md
+      - name: Upload CSV package list
+        uses: actions/upload-artifact@v3
+        with:
+          name: csv-framework-coverage-merge
+          path: |
+            out_merge/framework-coverage-*.csv
+            out_merge/framework-coverage-*.rst
+      - name: Upload CSV package list
+        uses: actions/upload-artifact@v3
+        with:
+          name: csv-framework-coverage-base
+          path: |
+            out_base/framework-coverage-*.csv
+            out_base/framework-coverage-*.rst
+      - name: Upload comparison results
+        uses: actions/upload-artifact@v3
+        with:
+          name: comparison
+          path: |
+            comparison.md
+      - name: Save PR number
+        run: |
+          mkdir -p pr
+          echo ${{ github.event.pull_request.number }} > pr/NR
+      - name: Upload PR number
+        uses: actions/upload-artifact@v3
+        with:
+          name: pr
+          path: pr/

.github/workflows/csv-coverage-timeseries.yml

@@ -5,38 +5,29 @@ on:
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
 
     steps:
-    - name: Clone self (github/codeql)
-      uses: actions/checkout@v3
-      with:
-        path: script
-    - name: Clone self (github/codeql) for analysis
-      uses: actions/checkout@v3
-      with:
-        path: codeqlModels
-        fetch-depth: 0
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v4
-      with:
-        python-version: 3.8
-    - name: Download CodeQL CLI
-      env:
-         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: |
-         gh release download --repo "github/codeql-cli-binaries" --pattern "codeql-linux64.zip"
-    - name: Unzip CodeQL CLI
-      run: unzip -d codeql-cli codeql-linux64.zip
-    - name: Build modeled package list
-      run: |
-        CLI=$(realpath "codeql-cli/codeql")
-        echo $CLI
-        PATH="$PATH:$CLI" python script/misc/scripts/library-coverage/generate-timeseries.py codeqlModels
-    - name: Upload timeseries CSV
-      uses: actions/upload-artifact@v3
-      with:
-        name: framework-coverage-timeseries
-        path: framework-coverage-timeseries-*.csv
-
+      - name: Clone self (github/codeql)
+        uses: actions/checkout@v3
+        with:
+          path: script
+      - name: Clone self (github/codeql) for analysis
+        uses: actions/checkout@v3
+        with:
+          path: codeqlModels
+          fetch-depth: 0
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+      - name: Download CodeQL CLI
+        uses: ./script/.github/actions/fetch-codeql
+      - name: Build modeled package list
+        run: |
+          python script/misc/scripts/library-coverage/generate-timeseries.py codeqlModels
+      - name: Upload timeseries CSV
+        uses: actions/upload-artifact@v3
+        with:
+          name: framework-coverage-timeseries
+          path: framework-coverage-timeseries-*.csv

.github/workflows/csv-coverage-update.yml

@@ -12,33 +12,27 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - name: Dump GitHub context
-      env:
-        GITHUB_CONTEXT: ${{ toJSON(github.event) }}
-      run: echo "$GITHUB_CONTEXT"
-    - name: Clone self (github/codeql)
-      uses: actions/checkout@v3
-      with:
-        path: ql
-        fetch-depth: 0
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v4
-      with:
-        python-version: 3.8
-    - name: Download CodeQL CLI
-      env:
-         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: |
-         gh release download --repo "github/codeql-cli-binaries" --pattern "codeql-linux64.zip"
-    - name: Unzip CodeQL CLI
-      run: unzip -d codeql-cli codeql-linux64.zip
+      - name: Dump GitHub context
+        env:
+          GITHUB_CONTEXT: ${{ toJSON(github.event) }}
+        run: echo "$GITHUB_CONTEXT"
+      - name: Clone self (github/codeql)
+        uses: actions/checkout@v3
+        with:
+          path: ql
+          fetch-depth: 0
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+      - name: Download CodeQL CLI
+        uses: ./ql/.github/actions/fetch-codeql
+      - name: Generate coverage files
+        run: |
+          python ql/misc/scripts/library-coverage/generate-report.py ci ql ql
 
-    - name: Generate coverage files
-      run: |
-        PATH="$PATH:codeql-cli/codeql" python ql/misc/scripts/library-coverage/generate-report.py ci ql ql
-
-    - name: Create pull request with changes
-      env:
-         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: |
-        python ql/misc/scripts/library-coverage/create-pr.py ql "$GITHUB_REPOSITORY"
+      - name: Create pull request with changes
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          python ql/misc/scripts/library-coverage/create-pr.py ql "$GITHUB_REPOSITORY"

.github/workflows/csv-coverage.yml

@@ -4,46 +4,39 @@ on:
   workflow_dispatch:
     inputs:
       qlModelShaOverride:
-        description: 'github/codeql repo SHA used for looking up the CSV models'
+        description: "github/codeql repo SHA used for looking up the CSV models"
         required: false
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
 
     steps:
-    - name: Clone self (github/codeql)
-      uses: actions/checkout@v3
-      with:
-        path: script
-    - name: Clone self (github/codeql) for analysis
-      uses: actions/checkout@v3
-      with:
-        path: codeqlModels
-        ref: ${{ github.event.inputs.qlModelShaOverride || github.ref }}
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v4
-      with:
-        python-version: 3.8
-    - name: Download CodeQL CLI
-      env:
-         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: |
-         gh release download --repo "github/codeql-cli-binaries" --pattern "codeql-linux64.zip"
-    - name: Unzip CodeQL CLI
-      run: unzip -d codeql-cli codeql-linux64.zip
-    - name: Build modeled package list
-      run: |
-        PATH="$PATH:codeql-cli/codeql" python script/misc/scripts/library-coverage/generate-report.py ci codeqlModels script
-    - name: Upload CSV package list
-      uses: actions/upload-artifact@v3
-      with:
-        name: framework-coverage-csv
-        path: framework-coverage-*.csv
-    - name: Upload RST package list
-      uses: actions/upload-artifact@v3
-      with:
-        name: framework-coverage-rst
-        path: framework-coverage-*.rst
-
+      - name: Clone self (github/codeql)
+        uses: actions/checkout@v3
+        with:
+          path: script
+      - name: Clone self (github/codeql) for analysis
+        uses: actions/checkout@v3
+        with:
+          path: codeqlModels
+          ref: ${{ github.event.inputs.qlModelShaOverride || github.ref }}
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+      - name: Download CodeQL CLI
+        uses: ./script/.github/actions/fetch-codeql
+      - name: Build modeled package list
+        run: |
+          python script/misc/scripts/library-coverage/generate-report.py ci codeqlModels script
+      - name: Upload CSV package list
+        uses: actions/upload-artifact@v3
+        with:
+          name: framework-coverage-csv
+          path: framework-coverage-*.csv
+      - name: Upload RST package list
+        uses: actions/upload-artifact@v3
+        with:
+          name: framework-coverage-rst
+          path: framework-coverage-*.rst

.github/workflows/go-tests.yml

@@ -4,159 +4,111 @@ on:
     paths:
       - "go/**"
       - .github/workflows/go-tests.yml
+      - .github/actions/fetch-codeql/action.yml
       - codeql-workspace.yml
 jobs:
-
   test-linux:
     name: Test Linux (Ubuntu)
     runs-on: ubuntu-latest
     steps:
+      - name: Set up Go 1.18.1
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.18.1
+        id: go
 
-    - name: Set up Go 1.18.1
-      uses: actions/setup-go@v3
-      with:
-        go-version: 1.18.1
-      id: go
+      - name: Check out code
+        uses: actions/checkout@v2
 
-    - name: Set up CodeQL CLI
-      run: |
-        echo "Removing old CodeQL Directory..."
-        rm -rf $HOME/codeql
-        echo "Done"
-        cd $HOME
-        echo "Downloading CodeQL CLI..."
-        LATEST=$(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | sort --version-sort | grep -v beta | tail -1)
-        gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql-linux64.zip "$LATEST"
-        echo "Done"
-        echo "Unpacking CodeQL CLI..."
-        unzip -q codeql-linux64.zip
-        rm -f codeql-linux64.zip
-        echo "Done"
-      env:
-        GITHUB_TOKEN: ${{ github.token }}
+      - name: Set up CodeQL CLI
+        uses: ./.github/actions/fetch-codeql
 
-    - name: Check out code
-      uses: actions/checkout@v2
+      - name: Enable problem matchers in repository
+        shell: bash
+        run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
 
-    - name: Enable problem matchers in repository
-      shell: bash
-      run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
+      - name: Build
+        run: |
+          cd go
+          make
 
-    - name: Build
-      run: |
-        cd go
-        env PATH=$PATH:$HOME/codeql make
+      - name: Check that all QL and Go code is autoformatted
+        run: |
+          cd go
+          make check-formatting
 
-    - name: Check that all QL and Go code is autoformatted
-      run: |
-        cd go
-        env PATH=$PATH:$HOME/codeql make check-formatting
+      - name: Compile qhelp files to markdown
+        run: |
+          cd go
+          env QHELP_OUT_DIR=qhelp-out make qhelp-to-markdown
 
-    - name: Compile qhelp files to markdown
-      run: |
-        cd go
-        env PATH=$PATH:$HOME/codeql QHELP_OUT_DIR=qhelp-out make qhelp-to-markdown
+      - name: Upload qhelp markdown
+        uses: actions/upload-artifact@v2
+        with:
+          name: qhelp-markdown
+          path: go/qhelp-out/**/*.md
 
-    - name: Upload qhelp markdown
-      uses: actions/upload-artifact@v2
-      with:
-        name: qhelp-markdown
-        path: go/qhelp-out/**/*.md
-
-    - name: Test
-      run: |
-        cd go
-        env PATH=$PATH:$HOME/codeql make test
+      - name: Test
+        run: |
+          cd go
+          make test
 
   test-mac:
     name: Test MacOS
-    runs-on: macOS-latest
+    runs-on: macos-latest
     steps:
-    - name: Set up Go 1.18.1
-      uses: actions/setup-go@v3
-      with:
-        go-version: 1.18.1
-      id: go
+      - name: Set up Go 1.18.1
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.18.1
+        id: go
 
-    - name: Set up CodeQL CLI
-      run: |
-        echo "Removing old CodeQL Directory..."
-        rm -rf $HOME/codeql
-        echo "Done"
-        cd $HOME
-        echo "Downloading CodeQL CLI..."
-        LATEST=$(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | sort --version-sort | grep -v beta | tail -1)
-        gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql-osx64.zip "$LATEST"
-        echo "Done"
-        echo "Unpacking CodeQL CLI..."
-        unzip -q codeql-osx64.zip
-        rm -f codeql-osx64.zip
-        echo "Done"
-      env:
-        GITHUB_TOKEN: ${{ github.token }}
+      - name: Check out code
+        uses: actions/checkout@v2
 
-    - name: Check out code
-      uses: actions/checkout@v2
+      - name: Set up CodeQL CLI
+        uses: ./.github/actions/fetch-codeql
 
-    - name: Enable problem matchers in repository
-      shell: bash
-      run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
+      - name: Enable problem matchers in repository
+        shell: bash
+        run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
 
-    - name: Build
-      run: |
-        cd go
-        env PATH=$PATH:$HOME/codeql make
+      - name: Build
+        run: |
+          cd go
+          make
 
-    - name: Test
-      run: |
-        cd go
-        env PATH=$PATH:$HOME/codeql make test
+      - name: Test
+        run: |
+          cd go
+          make test
 
   test-win:
     name: Test Windows
     runs-on: windows-2019
     steps:
-    - name: Set up Go 1.18.1
-      uses: actions/setup-go@v3
-      with:
-        go-version: 1.18.1
-      id: go
+      - name: Set up Go 1.18.1
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.18.1
+        id: go
 
-    - name: Set up CodeQL CLI
-      run: |
-        echo "Removing old CodeQL Directory..."
-        rm -rf $HOME/codeql
-        echo "Done"
-        cd "$HOME"
-        echo "Downloading CodeQL CLI..."
-        LATEST=$(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | sort --version-sort | grep -v beta | tail -1)
-        gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql-win64.zip "$LATEST"
-        echo "Done"
-        echo "Unpacking CodeQL CLI..."
-        unzip -q -o codeql-win64.zip
-        unzip -q -o codeql-win64.zip codeql/codeql.exe
-        rm -f codeql-win64.zip
-        echo "Done"
-      env:
-        GITHUB_TOKEN: ${{ github.token }}
-      shell:
-        bash
+      - name: Check out code
+        uses: actions/checkout@v2
 
-    - name: Check out code
-      uses: actions/checkout@v2
+      - name: Set up CodeQL CLI
+        uses: ./.github/actions/fetch-codeql
 
-    - name: Enable problem matchers in repository
-      shell: bash
-      run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
+      - name: Enable problem matchers in repository
+        shell: bash
+        run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
 
-    - name: Build
-      run: |
-        $Env:Path += ";$HOME\codeql"
-        cd go
-        make
+      - name: Build
+        run: |
+          cd go
+          make
 
-    - name: Test
-      run: |
-        $Env:Path += ";$HOME\codeql"
-        cd go
-        make test
+      - name: Test
+        run: |
+          cd go
+          make test

.github/workflows/js-ml-tests.yml

@@ -5,6 +5,7 @@ on:
     paths:
       - "javascript/ql/experimental/adaptivethreatmodeling/**"
       - .github/workflows/js-ml-tests.yml
+      - .github/actions/fetch-codeql/action.yml
       - codeql-workspace.yml
     branches:
       - main
@@ -13,6 +14,7 @@ on:
     paths:
       - "javascript/ql/experimental/adaptivethreatmodeling/**"
       - .github/workflows/js-ml-tests.yml
+      - .github/actions/fetch-codeql/action.yml
       - codeql-workspace.yml
   workflow_dispatch:
 

.github/workflows/mad_regenerate-models.yml

@@ -9,6 +9,7 @@ on:
       - main
     paths:
       - ".github/workflows/mad_regenerate-models.yml"
+      - ".github/actions/fetch-codeql/action.yml"
 
 jobs:
   regenerate-models:

.github/workflows/ql-for-ql-build.yml

@@ -10,16 +10,16 @@ env:
   CARGO_TERM_COLOR: always
 
 jobs:
-  queries:
-    runs-on: ubuntu-latest
+  analyze:
+    runs-on: ubuntu-latest-xl
     steps:
+      ### Build the queries ###
       - uses: actions/checkout@v3
       - name: Find codeql
         id: find-codeql
         uses: github/codeql-action/init@aa93aea877e5fb8841bcb1193f672abf6e9f2980
         with:
           languages: javascript # does not matter
-          tools: latest
       - name: Get CodeQL version
         id: get-codeql-version
         run: |
@@ -49,14 +49,7 @@ jobs:
           name: query-pack-zip
           path: ${{ runner.temp }}/query-pack.zip
 
-  extractors:
-    strategy:
-      fail-fast: false
-
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v3
+      ### Build the extractor ###
       - name: Cache entire extractor
         id: cache-extractor
         uses: actions/cache@v3
@@ -100,15 +93,8 @@ jobs:
             ql/target/release/ql-extractor
             ql/target/release/ql-extractor.exe
           retention-days: 1
-  package:
-    runs-on: ubuntu-latest
 
-    needs:
-      - extractors
-      - queries
-
-    steps:
-      - uses: actions/checkout@v3
+      ### Package the queries and extractor ###
       - uses: actions/download-artifact@v3
         with:
           name: query-pack-zip
@@ -136,16 +122,8 @@ jobs:
           name: codeql-ql-pack
           path: codeql-ql.zip
           retention-days: 1
-  analyze:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        folder: [cpp, csharp, java, javascript, python, ql, ruby, swift, go]
 
-    needs:
-      - package
-
-    steps:
+      ### Run the analysis ###
       - name: Download pack
         uses: actions/download-artifact@v3
         with:
@@ -165,14 +143,11 @@ jobs:
         env:
           PACK: ${{ runner.temp }}/pack
 
-      - name: Checkout repository
-        uses: actions/checkout@v3
       - name: Create CodeQL config file
         run: |
-          echo "paths:" > ${CONF}
-          echo "  - ${FOLDER}" >> ${CONF}
           echo "paths-ignore:" >> ${CONF}
           echo "  - ql/ql/test" >> ${CONF} 
+          echo "  - \"*/ql/lib/upgrades/\"" >> ${CONF} 
           echo "disable-default-queries: true" >> ${CONF}
           echo "packs:" >> ${CONF}
           echo "  - codeql/ql" >> ${CONF}
@@ -180,24 +155,34 @@ jobs:
           cat ${CONF}
         env: 
           CONF: ./ql-for-ql-config.yml
-          FOLDER: ${{ matrix.folder }}
       - name: Initialize CodeQL
         uses: github/codeql-action/init@aa93aea877e5fb8841bcb1193f672abf6e9f2980
         with:
           languages: ql
           db-location: ${{ runner.temp }}/db
           config-file: ./ql-for-ql-config.yml
-          tools: latest
 
       - name: Perform CodeQL Analysis
         uses: github/codeql-action/analyze@aa93aea877e5fb8841bcb1193f672abf6e9f2980
         with: 
-          category: "ql-for-ql-${{ matrix.folder }}"
+          category: "ql-for-ql"
       - name: Copy sarif file to CWD
-        run: cp ../results/ql.sarif ./${{ matrix.folder }}.sarif
+        run: cp ../results/ql.sarif ./ql-for-ql.sarif
+      - name: Fixup the $scema in sarif  # Until https://github.com/microsoft/sarif-vscode-extension/pull/436/ is part in a stable release
+        run: |
+          sed -i 's/\$schema.*/\$schema": "https:\/\/raw.githubusercontent.com\/oasis-tcs\/sarif-spec\/master\/Schemata\/sarif-schema-2.1.0",/' ql-for-ql.sarif 
       - name: Sarif as artifact
         uses: actions/upload-artifact@v3
         with:
-          name: ${{ matrix.folder }}.sarif
-          path: ${{ matrix.folder }}.sarif
-
+          name: ql-for-ql.sarif
+          path: ql-for-ql.sarif
+      - name: Split out the sarif file into langs
+        run: |
+          mkdir split-sarif
+          node ./ql/scripts/split-sarif.js ql-for-ql.sarif split-sarif
+      - name: Upload langs as artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: ql-for-ql-langs
+          path: split-sarif
+          retention-days: 1

.github/workflows/ql-for-ql-dataset_measure.yml

@@ -36,7 +36,7 @@ jobs:
             ql/target
           key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
       - name: Build Extractor
-        run: cd ql; env "PATH=$PATH:`dirname ${CODEQL}`" ./create-extractor-pack.sh
+        run: cd ql; env "PATH=$PATH:`dirname ${CODEQL}`" ./scripts/create-extractor-pack.sh
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
       - name: Checkout ${{ matrix.repo }}

.github/workflows/ql-for-ql-tests.yml

@@ -36,7 +36,7 @@ jobs:
         run: |
           cd ql;
           codeqlpath=$(dirname ${{ steps.find-codeql.outputs.codeql-path }});
-          env "PATH=$PATH:$codeqlpath" ./create-extractor-pack.sh
+          env "PATH=$PATH:$codeqlpath" ./scripts/create-extractor-pack.sh
       - name: Run QL tests
         run: |
           "${CODEQL}" test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}/ql/extractor-pack" --consistency-queries ql/ql/consistency-queries ql/ql/test

.github/workflows/query-list.yml

@@ -10,6 +10,7 @@ on:
   pull_request:
     paths:
       - '.github/workflows/query-list.yml'
+      - '.github/actions/fetch-codeql/action.yml'
       - 'misc/scripts/generate-code-scanning-query-list.py'
 
 jobs:
@@ -29,8 +30,6 @@ jobs:
     - name: Download CodeQL CLI
       # Look under the `codeql` directory, as this is where we checked out the `github/codeql` repo
       uses: ./codeql/.github/actions/fetch-codeql
-    - name: Unzip CodeQL CLI
-      run: unzip -d codeql-cli codeql-linux64.zip
     - name: Build code scanning query list
       run: |
         python codeql/misc/scripts/generate-code-scanning-query-list.py > code-scanning-query-list.csv

.github/workflows/ruby-build.yml

@@ -5,6 +5,7 @@ on:
     paths:
       - "ruby/**"
       - .github/workflows/ruby-build.yml
+      - .github/actions/fetch-codeql/action.yml
       - codeql-workspace.yml
     branches:
       - main
@@ -13,6 +14,7 @@ on:
     paths:
       - "ruby/**"
       - .github/workflows/ruby-build.yml
+      - .github/actions/fetch-codeql/action.yml
       - codeql-workspace.yml
     branches:
       - main
@@ -90,19 +92,14 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Fetch CodeQL
-        run: |
-          LATEST=$(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | grep -v beta | sort --version-sort | tail -1)
-          gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql-linux64.zip "$LATEST"
-          unzip -q codeql-linux64.zip
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
+        uses: ./.github/actions/fetch-codeql
       - name: Build Query Pack
         run: |
-          codeql/codeql pack create ql/lib --output target/packs
-          codeql/codeql pack install ql/src
-          codeql/codeql pack create ql/src --output target/packs
+          codeql pack create ql/lib --output target/packs
+          codeql pack install ql/src
+          codeql pack create ql/src --output target/packs
           PACK_FOLDER=$(readlink -f target/packs/codeql/ruby-queries/*)
-          codeql/codeql generate query-help --format=sarifv2.1.0 --output="${PACK_FOLDER}/rules.sarif" ql/src
+          codeql generate query-help --format=sarifv2.1.0 --output="${PACK_FOLDER}/rules.sarif" ql/src
           (cd ql/src; find queries \( -name '*.qhelp' -o -name '*.rb' -o -name '*.erb' \) -exec bash -c 'mkdir -p "'"${PACK_FOLDER}"'/$(dirname "{}")"' \; -exec cp "{}" "${PACK_FOLDER}/{}" \;)
       - uses: actions/upload-artifact@v3
         with:
@@ -179,19 +176,15 @@ jobs:
     runs-on: ${{ matrix.os }}
     needs: [package]
     steps:
+      - uses: actions/checkout@v3
+      - name: Fetch CodeQL
+        uses: ./.github/actions/fetch-codeql
+
       - uses: actions/checkout@v3
         with:
           repository: Shopify/example-ruby-app
           ref: 67a0decc5eb550f3a9228eda53925c3afd40dfe9
-      - name: Fetch CodeQL
-        shell: bash
-        run: |
-          LATEST=$(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | grep -v beta | sort --version-sort | tail -1)
-          gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql.zip "$LATEST"
-          unzip -q codeql.zip
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-        working-directory: ${{ runner.temp }}
+
       - name: Download Ruby bundle
         uses: actions/download-artifact@v3
         with:
@@ -215,12 +208,12 @@ jobs:
       - name: Run QL test
         shell: bash
         run: |
-          "${{ runner.temp }}/codeql/codeql" test run --search-path "${{ runner.temp }}/ruby-bundle" --additional-packs "${{ runner.temp }}/ruby-bundle" .
+          codeql test run --search-path "${{ runner.temp }}/ruby-bundle" --additional-packs "${{ runner.temp }}/ruby-bundle" .
       - name: Create database
         shell: bash
         run: |
-          "${{ runner.temp }}/codeql/codeql" database create --search-path "${{ runner.temp }}/ruby-bundle" --language ruby --source-root . ../database
+          codeql database create --search-path "${{ runner.temp }}/ruby-bundle" --language ruby --source-root . ../database
       - name: Analyze database
         shell: bash
         run: |
-          "${{ runner.temp }}/codeql/codeql" database analyze --search-path "${{ runner.temp }}/ruby-bundle" --format=sarifv2.1.0 --output=out.sarif ../database ruby-code-scanning.qls
+          codeql database analyze --search-path "${{ runner.temp }}/ruby-bundle" --format=sarifv2.1.0 --output=out.sarif ../database ruby-code-scanning.qls

.github/workflows/ruby-qltest.yml

@@ -5,6 +5,7 @@ on:
     paths:
       - "ruby/**"
       - .github/workflows/ruby-qltest.yml
+      - .github/actions/fetch-codeql/action.yml
       - codeql-workspace.yml
     branches:
       - main
@@ -13,6 +14,7 @@ on:
     paths:
       - "ruby/**"
       - .github/workflows/ruby-qltest.yml
+      - .github/actions/fetch-codeql/action.yml
       - codeql-workspace.yml
     branches:
       - main

.github/workflows/swift-codegen.yml

@@ -5,6 +5,7 @@ on:
     paths:
       - "swift/**"
       - .github/workflows/swift-codegen.yml
+      - .github/actions/fetch-codeql/action.yml
     branches:
       - main
 
@@ -15,18 +16,22 @@ jobs:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/fetch-codeql
       - uses: bazelbuild/setup-bazelisk@v2
+      - uses: actions/setup-python@v3
+      - uses: pre-commit/action@v3.0.0
+        name: Check that python code is properly formatted
+        with:
+          extra_args: autopep8 --all-files
       - name: Run unit tests
         run: |
           bazel test //swift/codegen/test --test_output=errors
-      - name: Check that QL generated code was checked in
-        run: |
-          bazel run //swift/codegen
-          git add swift
-          git diff --exit-code HEAD
+      - uses: pre-commit/action@v3.0.0
+        name: Check that QL generated code was checked in
+        with:
+          extra_args: swift-codegen --all-files
       - name: Generate C++ files
         run: |
-          bazel run //swift/codegen:codegen -- --generate=trap,cpp --cpp-output=$PWD/swift-generated-headers
+          bazel run //swift/codegen:codegen -- --generate=trap,cpp --cpp-output=$PWD/swift-generated-cpp-files
       - uses: actions/upload-artifact@v3
         with:
-          name: swift-generated-headers
-          path: swift-generated-headers/*.h
+          name: swift-generated-cpp-files
+          path: swift-generated-cpp-files/**

.github/workflows/swift-integration-tests.yml

@@ -0,0 +1,35 @@
+name: "Swift: Run Integration Tests"
+
+on:
+  pull_request:
+    paths:
+      - "swift/**"
+      - .github/workflows/swift-integration-tests.yml
+      - .github/actions/fetch-codeql/action.yml
+      - codeql-workspace.yml
+    branches:
+      - main
+defaults:
+  run:
+    working-directory: swift
+
+jobs:
+  integration-tests:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - ubuntu-20.04
+#          - macos-latest  TODO
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/fetch-codeql
+      - uses: bazelbuild/setup-bazelisk@v2
+      - uses: actions/setup-python@v3
+      - name: Build Swift extractor
+        run: |
+          bazel run //swift:create-extractor-pack
+      - name: Run integration tests
+        run: |
+          python integration-tests/runner.py

.github/workflows/swift-qltest.yml

@@ -5,6 +5,7 @@ on:
     paths:
       - "swift/**"
       - .github/workflows/swift-qltest.yml
+      - .github/actions/fetch-codeql/action.yml
       - codeql-workspace.yml
     branches:
       - main

.github/workflows/validate-change-notes.yml

@@ -5,6 +5,7 @@ on:
     paths:
       - "*/ql/*/change-notes/**/*"
       - ".github/workflows/validate-change-notes.yml"
+      - ".github/actions/fetch-codeql/action.yml"
     branches:
       - main
       - "rc/*"
@@ -12,6 +13,7 @@ on:
     paths:
       - "*/ql/*/change-notes/**/*"
       - ".github/workflows/validate-change-notes.yml"
+      - ".github/actions/fetch-codeql/action.yml"
 
 jobs:
   check-change-note:

.pre-commit-config.yaml

@@ -15,6 +15,12 @@ repos:
       - id: clang-format
         files: ^swift/.*\.(h|c|cpp)$
 
+  - repo: https://github.com/pre-commit/mirrors-autopep8
+    rev: v1.6.0
+    hooks:
+      - id: autopep8
+        files: ^swift/codegen/.*\.py
+
   - repo: local
     hooks:
       - id: codeql-format

CODEOWNERS

@@ -42,3 +42,4 @@ WORKSPACE.bazel @github/codeql-ci-reviewers
 /.github/workflows/js-ml-tests.yml @github/codeql-ml-powered-queries-reviewers
 /.github/workflows/ql-for-ql-* @github/codeql-ql-for-ql-reviewers
 /.github/workflows/ruby-* @github/codeql-ruby
+/.github/workflows/swift-* @github/codeql-c

config/identical-files.json

@@ -392,7 +392,8 @@
     "python/ql/test/TestUtilities/InlineExpectationsTest.qll",
     "ruby/ql/test/TestUtilities/InlineExpectationsTest.qll",
     "ql/ql/test/TestUtilities/InlineExpectationsTest.qll",
-    "go/ql/test/TestUtilities/InlineExpectationsTest.qll"
+    "go/ql/test/TestUtilities/InlineExpectationsTest.qll",
+    "swift/ql/test/TestUtilities/InlineExpectationsTest.qll"
   ],
   "C++ ExternalAPIs": [
     "cpp/ql/src/Security/CWE/CWE-020/ExternalAPIs.qll",
@@ -453,11 +454,11 @@
     "python/ql/src/Lexical/CommentedOutCodeReferences.inc.qhelp"
   ],
   "IDE Contextual Queries": [
-    "cpp/ql/src/IDEContextual.qll",
-    "csharp/ql/src/IDEContextual.qll",
-    "java/ql/src/IDEContextual.qll",
-    "javascript/ql/src/IDEContextual.qll",
-    "python/ql/src/analysis/IDEContextual.qll"
+    "cpp/ql/lib/IDEContextual.qll",
+    "csharp/ql/lib/IDEContextual.qll",
+    "java/ql/lib/IDEContextual.qll",
+    "javascript/ql/lib/IDEContextual.qll",
+    "python/ql/lib/analysis/IDEContextual.qll"
   ],
   "SSA C#": [
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/SsaImplCommon.qll",

cpp/downgrades/23f7cbb88a4eb29f30c3490363dc201bc054c5ff/exprs.ql

@@ -0,0 +1,17 @@
+class Expr extends @expr {
+  string toString() { none() }
+}
+
+class Location extends @location_expr {
+  string toString() { none() }
+}
+
+predicate isExprWithNewBuiltin(Expr expr) {
+  exists(int kind | exprs(expr, kind, _) | 330 <= kind and kind <= 334)
+}
+
+from Expr expr, int kind, int kind_new, Location location
+where
+  exprs(expr, kind, location) and
+  if isExprWithNewBuiltin(expr) then kind_new = 0 else kind_new = kind
+select expr, kind_new, location

cpp/downgrades/23f7cbb88a4eb29f30c3490363dc201bc054c5ff/upgrade.properties

@@ -0,0 +1,3 @@
+description: Add new builtin operations
+compatibility: partial
+exprs.rel: run exprs.qlo

cpp/downgrades/34549c3b0937002f11037d01822ebe99442c1402/attribute_args.ql

@@ -0,0 +1,17 @@
+class AttributeArgument extends @attribute_arg {
+  string toString() { none() }
+}
+
+class Attribute extends @attribute {
+  string toString() { none() }
+}
+
+class LocationDefault extends @location_default {
+  string toString() { none() }
+}
+
+from AttributeArgument arg, int kind, Attribute attr, int index, LocationDefault location
+where
+  attribute_args(arg, kind, attr, index, location) and
+  not arg instanceof @attribute_arg_constant_expr
+select arg, kind, attr, index, location

cpp/downgrades/34549c3b0937002f11037d01822ebe99442c1402/upgrade.properties

@@ -0,0 +1,4 @@
+description: Support all constant attribute arguments
+compatibility: backwards
+attribute_arg_constant.rel: delete
+attribute_args.rel: run attribute_args.qlo

cpp/ql/lib/CHANGELOG.md

@@ -1,3 +1,27 @@
+## 0.3.3
+
+### New Features
+
+* Added a predicate `getValueConstant` to `AttributeArgument` that yields the argument value as an `Expr` when the value is a constant expression.
+* A new class predicate `MustFlowConfiguration::allowInterproceduralFlow` has been added to the `semmle.code.cpp.ir.dataflow.MustFlow` library. The new predicate can be overridden to disable interprocedural flow.
+* Added subclasses of `BuiltInOperations` for `__builtin_bit_cast`, `__builtin_shuffle`, `__has_unique_object_representations`, `__is_aggregate`, and `__is_assignable`.
+
+### Major Analysis Improvements
+
+* The IR dataflow library now includes flow through global variables. This enables new findings in many scenarios.
+
+## 0.3.2
+
+### Bug Fixes
+
+* Under certain circumstances a variable declaration that is not also a definition could be associated with a `Variable` that did not have the definition as a `VariableDeclarationEntry`. This is now fixed, and a unique `Variable` will exist that has both the declaration and the definition as a `VariableDeclarationEntry`.
+
+## 0.3.1
+
+### Minor Analysis Improvements
+
+* `AnalysedExpr::isNullCheck` and `AnalysedExpr::isValidCheck` have been updated to handle variable accesses on the left-hand side of the C++ logical "and", and variable declarations in conditions.
+
 ## 0.3.0
 
 ### Deprecated APIs

cpp/ql/lib/change-notes/released/0.3.1.md

@@ -0,0 +1,5 @@
+## 0.3.1
+
+### Minor Analysis Improvements
+
+* `AnalysedExpr::isNullCheck` and `AnalysedExpr::isValidCheck` have been updated to handle variable accesses on the left-hand side of the C++ logical "and", and variable declarations in conditions.

cpp/ql/lib/change-notes/released/0.3.2.md

@@ -0,0 +1,5 @@
+## 0.3.2
+
+### Bug Fixes
+
+* Under certain circumstances a variable declaration that is not also a definition could be associated with a `Variable` that did not have the definition as a `VariableDeclarationEntry`. This is now fixed, and a unique `Variable` will exist that has both the declaration and the definition as a `VariableDeclarationEntry`.

cpp/ql/lib/change-notes/released/0.3.3.md

@@ -0,0 +1,11 @@
+## 0.3.3
+
+### New Features
+
+* Added a predicate `getValueConstant` to `AttributeArgument` that yields the argument value as an `Expr` when the value is a constant expression.
+* A new class predicate `MustFlowConfiguration::allowInterproceduralFlow` has been added to the `semmle.code.cpp.ir.dataflow.MustFlow` library. The new predicate can be overridden to disable interprocedural flow.
+* Added subclasses of `BuiltInOperations` for `__builtin_bit_cast`, `__builtin_shuffle`, `__has_unique_object_representations`, `__is_aggregate`, and `__is_assignable`.
+
+### Major Analysis Improvements
+
+* The IR dataflow library now includes flow through global variables. This enables new findings in many scenarios.

cpp/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.3.0
+lastReleaseVersion: 0.3.3

cpp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-all
-version: 0.3.0
+version: 0.3.3
 groups: cpp
 dbscheme: semmlecode.cpp.dbscheme
 extractor: cpp

cpp/ql/lib/semmle/code/cpp/Element.qll

@@ -6,6 +6,7 @@
 import semmle.code.cpp.Location
 private import semmle.code.cpp.Enclosing
 private import semmle.code.cpp.internal.ResolveClass
+private import semmle.code.cpp.internal.ResolveGlobalVariable
 
 /**
  * Get the `Element` that represents this `@element`.
@@ -28,9 +29,12 @@ Element mkElement(@element e) { unresolveElement(result) = e }
 pragma[inline]
 @element unresolveElement(Element e) {
   not result instanceof @usertype and
+  not result instanceof @variable and
   result = e
   or
   e = resolveClass(result)
+  or
+  e = resolveGlobalVariable(result)
 }
 
 /**

cpp/ql/lib/semmle/code/cpp/Specifier.qll

@@ -12,7 +12,7 @@ private import semmle.code.cpp.internal.ResolveClass
 class Specifier extends Element, @specifier {
   /** Gets a dummy location for the specifier. */
   override Location getLocation() {
-    suppressUnusedThis(this) and
+    exists(this) and
     result instanceof UnknownDefaultLocation
   }
 
@@ -256,9 +256,13 @@ class AttributeArgument extends Element, @attribute_arg {
 
   /**
    * Gets the text for the value of this argument, if its value is
-   * a string or a number.
+   * a constant or token.
    */
-  string getValueText() { attribute_arg_value(underlyingElement(this), result) }
+  string getValueText() {
+    if underlyingElement(this) instanceof @attribute_arg_constant_expr
+    then result = this.getValueConstant().getValue()
+    else attribute_arg_value(underlyingElement(this), result)
+  }
 
   /**
    * Gets the value of this argument, if its value is integral.
@@ -270,6 +274,13 @@ class AttributeArgument extends Element, @attribute_arg {
    */
   Type getValueType() { attribute_arg_type(underlyingElement(this), unresolveElement(result)) }
 
+  /**
+   * Gets the value of this argument, if its value is a constant.
+   */
+  Expr getValueConstant() {
+    attribute_arg_constant(underlyingElement(this), unresolveElement(result))
+  }
+
   /**
    * Gets the attribute to which this is an argument.
    */
@@ -294,11 +305,12 @@ class AttributeArgument extends Element, @attribute_arg {
         (
           if underlyingElement(this) instanceof @attribute_arg_type
           then tail = this.getValueType().getName()
-          else tail = this.getValueText()
+          else
+            if underlyingElement(this) instanceof @attribute_arg_constant_expr
+            then tail = this.getValueConstant().toString()
+            else tail = this.getValueText()
         ) and
         result = prefix + tail
       )
   }
 }
-
-private predicate suppressUnusedThis(Specifier s) { any() }

cpp/ql/lib/semmle/code/cpp/Variable.qll

@@ -6,6 +6,7 @@ import semmle.code.cpp.Element
 import semmle.code.cpp.exprs.Access
 import semmle.code.cpp.Initializer
 private import semmle.code.cpp.internal.ResolveClass
+private import semmle.code.cpp.internal.ResolveGlobalVariable
 
 /**
  * A C/C++ variable. For example, in the following code there are four
@@ -32,6 +33,8 @@ private import semmle.code.cpp.internal.ResolveClass
  * can have multiple declarations.
  */
 class Variable extends Declaration, @variable {
+  Variable() { isVariable(underlyingElement(this)) }
+
   override string getAPrimaryQlClass() { result = "Variable" }
 
   /** Gets the initializer of this variable, if any. */

cpp/ql/lib/semmle/code/cpp/controlflow/Nullness.qll

@@ -46,7 +46,7 @@ predicate nullCheckExpr(Expr checkExpr, Variable var) {
     or
     exists(LogicalAndExpr op, AnalysedExpr child |
       expr = op and
-      op.getRightOperand() = child and
+      op.getAnOperand() = child and
       nullCheckExpr(child, v)
     )
     or
@@ -99,7 +99,7 @@ predicate validCheckExpr(Expr checkExpr, Variable var) {
     or
     exists(LogicalAndExpr op, AnalysedExpr child |
       expr = op and
-      op.getRightOperand() = child and
+      op.getAnOperand() = child and
       validCheckExpr(child, v)
     )
     or
@@ -169,7 +169,10 @@ class AnalysedExpr extends Expr {
    */
   predicate isDef(LocalScopeVariable v) {
     this.inCondition() and
-    this.(Assignment).getLValue() = v.getAnAccess()
+    (
+      this.(Assignment).getLValue() = v.getAnAccess() or
+      this.(ConditionDeclExpr).getVariableAccess() = v.getAnAccess()
+    )
   }
 
   /**

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowUtil.qll

@@ -699,7 +699,7 @@ private predicate exprToExprStep_nocfg(Expr fromExpr, Expr toExpr) {
         call.getTarget() = f and
         // AST dataflow treats a reference as if it were the referred-to object, while the dataflow
         // models treat references as pointers. If the return type of the call is a reference, then
-        // look for data flow the the referred-to object, rather than the reference itself.
+        // look for data flow to the referred-to object, rather than the reference itself.
         if call.getType().getUnspecifiedType() instanceof ReferenceType
         then outModel.isReturnValueDeref()
         else outModel.isReturnValue()

cpp/ql/lib/semmle/code/cpp/exprs/BuiltInOperations.qll

@@ -1,5 +1,5 @@
 /**
- * Provides classes for modeling built-in operations.  Built-in operations are
+ * Provides classes for modeling built-in operations. Built-in operations are
  * typically compiler specific and are used by libraries and generated code.
  */
 
@@ -120,8 +120,8 @@ class BuiltInNoOp extends BuiltInOperation, @noopexpr {
 
 /**
  * A C/C++ `__builtin_offsetof` built-in operation (used by some implementations
- * of `offsetof`).  The operation retains its semantics even in the presence
- * of an overloaded `operator &`).  This is a GNU/Clang extension.
+ * of `offsetof`). The operation retains its semantics even in the presence
+ * of an overloaded `operator &`). This is a gcc/clang extension.
  * ```
  * struct S {
  *   int a, b;
@@ -137,8 +137,8 @@ class BuiltInOperationBuiltInOffsetOf extends BuiltInOperation, @offsetofexpr {
 
 /**
  * A C/C++ `__INTADDR__` built-in operation (used by some implementations
- * of `offsetof`).  The operation retains its semantics even in the presence
- * of an overloaded `operator &`).  This is an EDG extension.
+ * of `offsetof`). The operation retains its semantics even in the presence
+ * of an overloaded `operator &`). This is an EDG extension.
  * ```
  * struct S {
  *   int a, b;
@@ -173,7 +173,7 @@ class BuiltInOperationHasAssign extends BuiltInOperation, @hasassignexpr {
  *
  * Returns `true` if the type has a copy constructor.
  * ```
- * std::integral_constant< bool, __has_copy(_Tp)> hc;
+ * std::integral_constant<bool, __has_copy(_Tp)> hc;
  * ```
  */
 class BuiltInOperationHasCopy extends BuiltInOperation, @hascopyexpr {
@@ -189,7 +189,7 @@ class BuiltInOperationHasCopy extends BuiltInOperation, @hascopyexpr {
  * Returns `true` if a copy assignment operator has an empty exception
  * specification.
  * ```
- * std::integral_constant< bool, __has_nothrow_assign(_Tp)> hnta;
+ * std::integral_constant<bool, __has_nothrow_assign(_Tp)> hnta;
  * ```
  */
 class BuiltInOperationHasNoThrowAssign extends BuiltInOperation, @hasnothrowassign {
@@ -220,7 +220,7 @@ class BuiltInOperationHasNoThrowConstructor extends BuiltInOperation, @hasnothro
  *
  * Returns `true` if the copy constructor has an empty exception specification.
  * ```
- * std::integral_constant< bool, __has_nothrow_copy(MyType) >;
+ * std::integral_constant<bool, __has_nothrow_copy(MyType) >;
  * ```
  */
 class BuiltInOperationHasNoThrowCopy extends BuiltInOperation, @hasnothrowcopy {
@@ -266,7 +266,7 @@ class BuiltInOperationHasTrivialConstructor extends BuiltInOperation, @hastrivia
  *
  * Returns true if the type has a trivial copy constructor.
  * ```
- * std::integral_constant< bool, __has_trivial_copy(MyType) > htc;
+ * std::integral_constant<bool, __has_trivial_copy(MyType)> htc;
  * ```
  */
 class BuiltInOperationHasTrivialCopy extends BuiltInOperation, @hastrivialcopy {
@@ -468,7 +468,7 @@ class BuiltInOperationIsUnion extends BuiltInOperation, @isunionexpr {
  * ```
  * template<typename _Tp1, typename _Tp2>
  *   struct types_compatible
- *   : public integral_constant<bool, __builtin_types_compatible_p(_Tp1, _Tp2) >
+ *   : public integral_constant<bool, __builtin_types_compatible_p(_Tp1, _Tp2)>
  *   { };
  * ```
  */
@@ -479,8 +479,7 @@ class BuiltInOperationBuiltInTypesCompatibleP extends BuiltInOperation, @typesco
 /**
  * A clang `__builtin_shufflevector` expression.
  *
- * It outputs a permutation of elements from one or two input vectors.
- * Please see
+ * It outputs a permutation of elements from one or two input vectors. See
  * https://releases.llvm.org/3.7.0/tools/clang/docs/LanguageExtensions.html#langext-builtin-shufflevector
  * for more information.
  * ```
@@ -494,11 +493,29 @@ class BuiltInOperationBuiltInShuffleVector extends BuiltInOperation, @builtinshu
   override string getAPrimaryQlClass() { result = "BuiltInOperationBuiltInShuffleVector" }
 }
 
+/**
+ * A gcc `__builtin_shuffle` expression.
+ *
+ * It outputs a permutation of elements from one or two input vectors.
+ * See https://gcc.gnu.org/onlinedocs/gcc/Vector-Extensions.html
+ * for more information.
+ * ```
+ * // Concatenate every other element of 4-element vectors V1 and V2.
+ * M = {0, 2, 4, 6};
+ * V3 = __builtin_shuffle(V1, V2, M);
+ * ```
+ */
+class BuiltInOperationBuiltInShuffle extends BuiltInOperation, @builtinshuffle {
+  override string toString() { result = "__builtin_shuffle" }
+
+  override string getAPrimaryQlClass() { result = "BuiltInOperationBuiltInShuffle" }
+}
+
 /**
  * A clang `__builtin_convertvector` expression.
  *
  * Allows for conversion of vectors of equal element count and compatible
- * element types. Please see
+ * element types. See
  * https://releases.llvm.org/3.7.0/tools/clang/docs/LanguageExtensions.html#builtin-convertvector
  * for more information.
  * ```
@@ -547,7 +564,7 @@ class BuiltInOperationBuiltInAddressOf extends UnaryOperation, BuiltInOperation,
  * ```
  * template<typename T, typename... Args>
  *   struct is_trivially_constructible
- *   : public integral_constant<bool, __is_trivially_constructible(T, Args...) >
+ *   : public integral_constant<bool, __is_trivially_constructible(T, Args...)>
  *   { };
  * ```
  */
@@ -612,13 +629,10 @@ class BuiltInOperationIsTriviallyDestructible extends BuiltInOperation, @istrivi
  * The `__is_trivially_assignable` built-in operation (used by some
  * implementations of the `<type_traits>` header).
  *
- * Returns `true` if the assignment operator `C::operator =(const C& c)` is
- * trivial.
+ * Returns `true` if the assignment operator `C::operator =(const D& d)` is
+ * trivial (i.e., it will not call any operation that is non-trivial).
  * ```
- * template<typename T>
- *   struct is_trivially_assignable
- *   : public integral_constant<bool, __is_trivially_assignable(T) >
- *   { };
+ * bool v = __is_trivially_assignable(MyType1, MyType2);
  * ```
  */
 class BuiltInOperationIsTriviallyAssignable extends BuiltInOperation, @istriviallyassignableexpr {
@@ -631,10 +645,10 @@ class BuiltInOperationIsTriviallyAssignable extends BuiltInOperation, @istrivial
  * The `__is_nothrow_assignable` built-in operation (used by some
  * implementations of the `<type_traits>` header).
  *
- * Returns true if there exists a `C::operator =(const C& c) nothrow`
+ * Returns true if there exists a `C::operator =(const D& d) nothrow`
  * assignment operator (i.e, with an empty exception specification).
  * ```
- * bool v = __is_nothrow_assignable(MyType);
+ * bool v = __is_nothrow_assignable(MyType1, MyType2);
  * ```
  */
 class BuiltInOperationIsNothrowAssignable extends BuiltInOperation, @isnothrowassignableexpr {
@@ -643,15 +657,30 @@ class BuiltInOperationIsNothrowAssignable extends BuiltInOperation, @isnothrowas
   override string getAPrimaryQlClass() { result = "BuiltInOperationIsNothrowAssignable" }
 }
 
+/**
+ * The `__is_assignable` built-in operation (used by some implementations
+ * of the `<type_traits>` header).
+ *
+ * Returns true if there exists a `C::operator =(const D& d)` assignment
+ * operator.
+ * ```
+ * bool v = __is_assignable(MyType1, MyType2);
+ * ```
+ */
+class BuiltInOperationIsAssignable extends BuiltInOperation, @isassignable {
+  override string toString() { result = "__is_assignable" }
+
+  override string getAPrimaryQlClass() { result = "BuiltInOperationIsAssignable" }
+}
+
 /**
  * The `__is_standard_layout` built-in operation (used by some implementations
  * of the `<type_traits>` header).
  *
  * Returns `true` if the type is a primitive type, or a `class`, `struct` or
- * `union` WITHOUT (1) virtual functions or base classes, (2) reference member
- * variable or (3) multiple occurrences of base `class` objects, among other
- * restrictions.  Please see
- * https://en.cppreference.com/w/cpp/named_req/StandardLayoutType
+ * `union` without (1) virtual functions or base classes, (2) reference member
+ * variable, or (3) multiple occurrences of base `class` objects, among other
+ * restrictions. See https://en.cppreference.com/w/cpp/named_req/StandardLayoutType
  * for more information.
  * ```
  * bool v = __is_standard_layout(MyType);
@@ -668,7 +697,7 @@ class BuiltInOperationIsStandardLayout extends BuiltInOperation, @isstandardlayo
  * implementations of the `<type_traits>` header).
  *
  * Returns `true` if instances of this type can be copied by trivial
- * means.  The copying is done in a manner similar to the `memcpy`
+ * means. The copying is done in a manner similar to the `memcpy`
  * function.
  */
 class BuiltInOperationIsTriviallyCopyable extends BuiltInOperation, @istriviallycopyableexpr {
@@ -682,13 +711,13 @@ class BuiltInOperationIsTriviallyCopyable extends BuiltInOperation, @istrivially
  * the `<type_traits>` header).
  *
  * Returns `true` if the type is a scalar type, a reference type or an array of
- * literal types, among others. Please see
+ * literal types, among others. See
  * https://en.cppreference.com/w/cpp/named_req/LiteralType
  * for more information.
  *
  * ```
  * template <typename _Tp>
- * std::integral_constant< bool, __is_literal_type(_Tp)> ilt;
+ * std::integral_constant<bool, __is_literal_type(_Tp)> ilt;
  * ```
  */
 class BuiltInOperationIsLiteralType extends BuiltInOperation, @isliteraltypeexpr {
@@ -705,7 +734,7 @@ class BuiltInOperationIsLiteralType extends BuiltInOperation, @isliteraltypeexpr
  * compiler, with semantics of the `memcpy` operation.
  * ```
  * template <typename _Tp>
- * std::integral_constant< bool, __has_trivial_move_constructor(_Tp)> htmc;
+ * std::integral_constant<bool, __has_trivial_move_constructor(_Tp)> htmc;
  * ```
  */
 class BuiltInOperationHasTrivialMoveConstructor extends BuiltInOperation,
@@ -723,7 +752,7 @@ class BuiltInOperationHasTrivialMoveConstructor extends BuiltInOperation,
  * ```
  * template<typename T>
  *   struct has_trivial_move_assign
- *   : public integral_constant<bool, __has_trivial_move_assign(T) >
+ *   : public integral_constant<bool, __has_trivial_move_assign(T)>
  *   { };
  * ```
  */
@@ -758,7 +787,7 @@ class BuiltInOperationHasNothrowMoveAssign extends BuiltInOperation, @hasnothrow
  * ```
  * template<typename T, typename... Args>
  *   struct is_constructible
- *   : public integral_constant<bool, __is_constructible(T, Args...) >
+ *   : public integral_constant<bool, __is_constructible(T, Args...)>
  *   { };
  * ```
  */
@@ -785,7 +814,7 @@ class BuiltInOperationIsNothrowConstructible extends BuiltInOperation, @isnothro
 }
 
 /**
- * The `__has_finalizer` built-in operation.  This is a Microsoft extension.
+ * The `__has_finalizer` built-in operation. This is a Microsoft extension.
  *
  * Returns `true` if the type defines a _finalizer_ `C::!C(void)`, to be called
  * from either the regular destructor or the garbage collector.
@@ -800,10 +829,10 @@ class BuiltInOperationHasFinalizer extends BuiltInOperation, @hasfinalizerexpr {
 }
 
 /**
- * The `__is_delegate` built-in operation.  This is a Microsoft extension.
+ * The `__is_delegate` built-in operation. This is a Microsoft extension.
  *
  * Returns `true` if the function has been declared as a `delegate`, used in
- * message forwarding.  Please see
+ * message forwarding. See
  * https://docs.microsoft.com/en-us/cpp/extensions/delegate-cpp-component-extensions
  * for more information.
  */
@@ -814,9 +843,9 @@ class BuiltInOperationIsDelegate extends BuiltInOperation, @isdelegateexpr {
 }
 
 /**
- * The `__is_interface_class` built-in operation.  This is a Microsoft extension.
+ * The `__is_interface_class` built-in operation. This is a Microsoft extension.
  *
- * Returns `true` if the type has been declared as an `interface`.  Please see
+ * Returns `true` if the type has been declared as an `interface`. See
  * https://docs.microsoft.com/en-us/cpp/extensions/interface-class-cpp-component-extensions
  * for more information.
  */
@@ -827,9 +856,9 @@ class BuiltInOperationIsInterfaceClass extends BuiltInOperation, @isinterfacecla
 }
 
 /**
- * The `__is_ref_array` built-in operation.  This is a Microsoft extension.
+ * The `__is_ref_array` built-in operation. This is a Microsoft extension.
  *
- * Returns `true` if the object passed in is a _platform array_.  Please see
+ * Returns `true` if the object passed in is a _platform array_. See
  * https://docs.microsoft.com/en-us/cpp/extensions/arrays-cpp-component-extensions
  * for more information.
  * ```
@@ -844,9 +873,9 @@ class BuiltInOperationIsRefArray extends BuiltInOperation, @isrefarrayexpr {
 }
 
 /**
- * The `__is_ref_class` built-in operation.  This is a Microsoft extension.
+ * The `__is_ref_class` built-in operation. This is a Microsoft extension.
  *
- * Returns `true` if the type is a _reference class_.  Please see
+ * Returns `true` if the type is a _reference class_. See
  * https://docs.microsoft.com/en-us/cpp/extensions/classes-and-structs-cpp-component-extensions
  * for more information.
  * ```
@@ -861,10 +890,10 @@ class BuiltInOperationIsRefClass extends BuiltInOperation, @isrefclassexpr {
 }
 
 /**
- * The `__is_sealed` built-in operation.  This is a Microsoft extension.
+ * The `__is_sealed` built-in operation. This is a Microsoft extension.
  *
  * Returns `true` if a given class or virtual function is marked as `sealed`,
- * meaning that it cannot be extended or overridden.  The `sealed` keyword
+ * meaning that it cannot be extended or overridden. The `sealed` keyword
  * is similar to the C++11 `final` keyword.
  * ```
  * ref class X sealed {
@@ -879,7 +908,7 @@ class BuiltInOperationIsSealed extends BuiltInOperation, @issealedexpr {
 }
 
 /**
- * The `__is_simple_value_class` built-in operation.  This is a Microsoft extension.
+ * The `__is_simple_value_class` built-in operation. This is a Microsoft extension.
  *
  * Returns `true` if passed a value type that contains no references to the
  * garbage-collected heap.
@@ -898,9 +927,9 @@ class BuiltInOperationIsSimpleValueClass extends BuiltInOperation, @issimplevalu
 }
 
 /**
- * The `__is_value_class` built-in operation.  This is a Microsoft extension.
+ * The `__is_value_class` built-in operation. This is a Microsoft extension.
  *
- * Returns `true` if passed a value type.  Please see
+ * Returns `true` if passed a value type. See
  * https://docs.microsoft.com/en-us/cpp/extensions/classes-and-structs-cpp-component-extensions
  * For more information.
  * ```
@@ -922,7 +951,7 @@ class BuiltInOperationIsValueClass extends BuiltInOperation, @isvalueclassexpr {
  * ```
  * template<typename T>
  *   struct is_final
- *   : public integral_constant<bool, __is_final(T) >
+ *   : public integral_constant<bool, __is_final(T)>
  *   { };
  * ```
  */
@@ -933,7 +962,7 @@ class BuiltInOperationIsFinal extends BuiltInOperation, @isfinalexpr {
 }
 
 /**
- * The `__builtin_choose_expr` expression.  This is a GNU/Clang extension.
+ * The `__builtin_choose_expr` expression. This is a gcc/clang extension.
  *
  * The expression functions similarly to the ternary `?:` operator, except
  * that it is evaluated at compile-time.
@@ -978,3 +1007,50 @@ class BuiltInComplexOperation extends BuiltInOperation, @builtincomplex {
   /** Gets the operand corresponding to the imaginary part of the complex number. */
   Expr getImaginaryOperand() { this.hasChild(result, 1) }
 }
+
+/**
+ * A C++ `__is_aggregate` built-in operation (used by some implementations of the
+ * `<type_traits>` header).
+ *
+ * Returns `true` if the type has is an aggregate type.
+ * ```
+ * std::integral_constant<bool, __is_aggregate(_Tp)> ia;
+ * ```
+ */
+class BuiltInOperationIsAggregate extends BuiltInOperation, @isaggregate {
+  override string toString() { result = "__is_aggregate" }
+
+  override string getAPrimaryQlClass() { result = "BuiltInOperationIsAggregate" }
+}
+
+/**
+ * A C++ `__has_unique_object_representations` built-in operation (used by some
+ * implementations of the `<type_traits>` header).
+ *
+ * Returns `true` if the type is trivially copyable and if the object representation
+ * is unique for two objects with the same value.
+ * ```
+ * bool v = __has_unique_object_representations(MyType);
+ * ```
+ */
+class BuiltInOperationHasUniqueObjectRepresentations extends BuiltInOperation,
+  @hasuniqueobjectrepresentations {
+  override string toString() { result = "__has_unique_object_representations" }
+
+  override string getAPrimaryQlClass() { result = "BuiltInOperationHasUniqueObjectRepresentations" }
+}
+
+/**
+ * A C/C++ `__builtin_bit_cast` built-in operation (used by some implementations
+ * of `std::bit_cast`).
+ *
+ * Performs a bit cast from a value to a type.
+ * ```
+ * __builtin_bit_cast(Type, value);
+ * ```
+ */
+class BuiltInBitCast extends BuiltInOperation, @builtinbitcast {
+  override string toString() { result = "__builtin_bit_cast" }
+
+  override string getAPrimaryQlClass() { result = "BuiltInBitCast" }
+}

cpp/ql/lib/semmle/code/cpp/exprs/Call.qll

@@ -255,8 +255,10 @@ class FunctionCall extends Call, @funbindexpr {
   /**
    * Gets the function called by this call.
    *
-   * In the case of virtual function calls, the result is the most-specific function in the override tree (as
-   * determined by the compiler) such that the target at runtime will be one of `result.getAnOverridingFunction*()`.
+   * In the case of virtual function calls, the result is the most-specific function in the override tree
+   * such that the target at runtime will be one of `result.getAnOverridingFunction*()`. The most-specific
+   * function is determined by the compiler based on the compile time type of the object the function is a
+   * member of.
    */
   override Function getTarget() { funbind(underlyingElement(this), unresolveElement(result)) }
 

cpp/ql/lib/semmle/code/cpp/exprs/Expr.qll

@@ -596,9 +596,12 @@ class ParenthesisExpr extends Conversion, @parexpr {
 }
 
 /**
- * A C/C++ expression that has not been resolved.
+ * A C/C++ expression that could not be resolved, or that can no longer be
+ * represented due to a database upgrade or downgrade.
  *
- * It is assigned `ErroneousType` as its type.
+ * If the expression could not be resolved, it has type `ErroneousType`. In the
+ * case of a database upgrade or downgrade, the original type from before the
+ * upgrade or downgrade is kept if that type can be represented.
  */
 class ErrorExpr extends Expr, @errorexpr {
   override string toString() { result = "<error expr>" }

cpp/ql/lib/semmle/code/cpp/internal/ResolveGlobalVariable.qll

@@ -0,0 +1,57 @@
+private predicate hasDefinition(@globalvariable g) {
+  exists(@var_decl vd | var_decls(vd, g, _, _, _) | var_def(vd))
+}
+
+private predicate onlyOneCompleteGlobalVariableExistsWithMangledName(@mangledname name) {
+  strictcount(@globalvariable g | hasDefinition(g) and mangled_name(g, name)) = 1
+}
+
+/** Holds if `g` is a unique global variable with a definition named `name`. */
+private predicate isGlobalWithMangledNameAndWithDefinition(@mangledname name, @globalvariable g) {
+  hasDefinition(g) and
+  mangled_name(g, name) and
+  onlyOneCompleteGlobalVariableExistsWithMangledName(name)
+}
+
+/** Holds if `g` is a global variable without a definition named `name`. */
+private predicate isGlobalWithMangledNameAndWithoutDefinition(@mangledname name, @globalvariable g) {
+  not hasDefinition(g) and
+  mangled_name(g, name)
+}
+
+/**
+ * Holds if `incomplete` is a global variable without a definition, and there exists
+ * a unique global variable `complete` with the same name that does have a definition.
+ */
+private predicate hasTwinWithDefinition(@globalvariable incomplete, @globalvariable complete) {
+  exists(@mangledname name |
+    not variable_instantiation(incomplete, complete) and
+    isGlobalWithMangledNameAndWithoutDefinition(name, incomplete) and
+    isGlobalWithMangledNameAndWithDefinition(name, complete)
+  )
+}
+
+import Cached
+
+cached
+private module Cached {
+  /**
+   * If `v` is a global variable without a definition, and there exists a unique
+   * global variable with the same name that does have a definition, then the
+   * result is that unique global variable. Otherwise, the result is `v`.
+   */
+  cached
+  @variable resolveGlobalVariable(@variable v) {
+    hasTwinWithDefinition(v, result)
+    or
+    not hasTwinWithDefinition(v, _) and
+    result = v
+  }
+
+  cached
+  predicate isVariable(@variable v) {
+    not v instanceof @globalvariable
+    or
+    v = resolveGlobalVariable(_)
+  }
+}

cpp/ql/lib/semmle/code/cpp/ir/dataflow/MustFlow.qll

@@ -38,6 +38,9 @@ abstract class MustFlowConfiguration extends string {
    */
   predicate isAdditionalFlowStep(DataFlow::Node node1, DataFlow::Node node2) { none() }
 
+  /** Holds if this configuration allows flow from arguments to parameters. */
+  predicate allowInterproceduralFlow() { any() }
+
   /**
    * Holds if data must flow from `source` to `sink` for this configuration.
    *
@@ -204,10 +207,25 @@ private module Cached {
   }
 }
 
+/**
+ * Gets the enclosing callable of `n`. Unlike `n.getEnclosingCallable()`, this
+ * predicate ensures that joins go from `n` to the result instead of the other
+ * way around.
+ */
+pragma[inline]
+private Declaration getEnclosingCallable(DataFlow::Node n) {
+  pragma[only_bind_into](result) = pragma[only_bind_out](n).getEnclosingCallable()
+}
+
 /** Holds if `nodeFrom` flows to `nodeTo`. */
 private predicate step(DataFlow::Node nodeFrom, DataFlow::Node nodeTo, MustFlowConfiguration config) {
   exists(config) and
-  Cached::step(nodeFrom, nodeTo)
+  Cached::step(pragma[only_bind_into](nodeFrom), pragma[only_bind_into](nodeTo)) and
+  (
+    config.allowInterproceduralFlow()
+    or
+    getEnclosingCallable(nodeFrom) = getEnclosingCallable(nodeTo)
+  )
   or
   config.isAdditionalFlowStep(nodeFrom, nodeTo)
 }

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowPrivate.qll

@@ -244,7 +244,25 @@ OutNode getAnOutNode(DataFlowCall call, ReturnKind kind) {
  * calling context. For example, this would happen with flow through a
  * global or static variable.
  */
-predicate jumpStep(Node n1, Node n2) { none() }
+predicate jumpStep(Node n1, Node n2) {
+  exists(GlobalOrNamespaceVariable v |
+    v =
+      n1.asInstruction()
+          .(StoreInstruction)
+          .getResultAddress()
+          .(VariableAddressInstruction)
+          .getAstVariable() and
+    v = n2.asVariable()
+    or
+    v =
+      n2.asInstruction()
+          .(LoadInstruction)
+          .getSourceAddress()
+          .(VariableAddressInstruction)
+          .getAstVariable() and
+    v = n1.asVariable()
+  )
+}
 
 /**
  * Holds if data can flow from `node1` to `node2` via an assignment to `f`.

cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/TranslatedElement.qll

@@ -947,7 +947,7 @@ abstract class TranslatedElement extends TTranslatedElement {
 }
 
 /**
- * Represents the IR translation of a root element, either a function or a global variable.
+ * The IR translation of a root element, either a function or a global variable.
  */
 abstract class TranslatedRootElement extends TranslatedElement {
   TranslatedRootElement() {

cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/TranslatedGlobalVar.qll

@@ -65,7 +65,7 @@ class TranslatedGlobalOrNamespaceVarInit extends TranslatedRootElement,
       result = this.getInstruction(InitializerVariableAddressTag())
       or
       tag = InitializerVariableAddressTag() and
-      result = getChild(1).getFirstInstruction()
+      result = this.getChild(1).getFirstInstruction()
       or
       tag = ReturnTag() and
       result = this.getInstruction(AliasedUseTag())

cpp/ql/lib/semmle/code/cpp/rangeanalysis/SimpleRangeAnalysis.qll

@@ -1573,7 +1573,7 @@ private module SimpleRangeAnalysisCached {
     result = min([max(getTruncatedUpperBounds(expr)), getGuardedUpperBound(expr)])
   }
 
-  /** Holds if the upper bound of `expr` may have been widened. This means the the upper bound is in practice likely to be overly wide. */
+  /** Holds if the upper bound of `expr` may have been widened. This means the upper bound is in practice likely to be overly wide. */
   cached
   predicate upperBoundMayBeWidened(Expr e) {
     isRecursiveExpr(e) and

cpp/ql/lib/semmlecode.cpp.dbscheme

@@ -899,6 +899,7 @@ case @attribute_arg.kind of
 | 1 = @attribute_arg_token
 | 2 = @attribute_arg_constant
 | 3 = @attribute_arg_type
+| 4 = @attribute_arg_constant_expr
 ;
 
 attribute_arg_value(
@@ -909,6 +910,10 @@ attribute_arg_type(
     unique int arg: @attribute_arg ref,
     int type_id: @type ref
 );
+attribute_arg_constant(
+    unique int arg: @attribute_arg ref,
+    int constant: @expr ref
+)
 attribute_arg_name(
     unique int arg: @attribute_arg ref,
     string name: string ref
@@ -1650,6 +1655,11 @@ case @expr.kind of
 | 327 = @co_await
 | 328 = @co_yield
 | 329 = @temp_init
+| 330 = @isassignable
+| 331 = @isaggregate
+| 332 = @hasuniqueobjectrepresentations
+| 333 = @builtinbitcast
+| 334 = @builtinshuffle
 ;
 
 @var_args_expr = @vastartexpr
@@ -1711,6 +1721,11 @@ case @expr.kind of
             | @isfinalexpr
             | @builtinchooseexpr
             | @builtincomplex
+            | @isassignable
+            | @isaggregate
+            | @hasuniqueobjectrepresentations
+            | @builtinbitcast
+            | @builtinshuffle
             ;
 
 new_allocated_type(

cpp/ql/lib/upgrades/19e31bf071f588bb7efd1e4d5a185ce4f6fbbd84/upgrade.properties

@@ -0,0 +1,2 @@
+description: Add new builtin operations
+compatibility: backwards

cpp/ql/lib/upgrades/23f7cbb88a4eb29f30c3490363dc201bc054c5ff/upgrade.properties

@@ -0,0 +1,2 @@
+description: Support all constant attribute arguments
+compatibility: partial

cpp/ql/src/CHANGELOG.md

@@ -1,3 +1,17 @@
+## 0.3.2
+
+### Minor Analysis Improvements
+
+* The query `cpp/bad-strncpy-size` now covers more `strncpy`-like functions than before, including `strxfrm`(`_l`), `wcsxfrm`(`_l`), and `stpncpy`. Users of this query may see an increase in results.
+
+## 0.3.1
+
+## 0.3.0
+
+### Breaking Changes
+
+* Contextual queries and the query libraries they depend on have been moved to the `codeql/cpp-all` package.
+
 ## 0.2.0
 
 ## 0.1.4

cpp/ql/src/Likely Bugs/Conversion/LossyFunctionResultCast.ql

@@ -44,7 +44,7 @@ predicate whiteListWrapped(FunctionCall fc) {
 
 from FunctionCall c, FloatingPointType t1, IntegralType t2
 where
-  t1 = c.getTarget().getType().getUnderlyingType() and
+  pragma[only_bind_into](t1) = c.getTarget().getType().getUnderlyingType() and
   t2 = c.getActualType() and
   c.hasImplicitConversion() and
   not whiteListWrapped(c)

cpp/ql/src/Likely Bugs/Likely Typos/IncorrectNotOperatorUsage.ql

@@ -10,7 +10,6 @@
  * @precision medium
  * @tags security
  *       external/cwe/cwe-480
- *       external/microsoft/c6317
  */
 
 import cpp

cpp/ql/src/Likely Bugs/Likely Typos/UsingStrcpyAsBoolean.ql

@@ -7,8 +7,7 @@
  * @problem.severity error
  * @precision high
  * @id cpp/string-copy-return-value-as-boolean
- * @tags external/microsoft/C6324
- *       correctness
+ * @tags correctness
  */
 
 import cpp

cpp/ql/src/Likely Bugs/Likely Typos/inconsistentLoopDirection.ql

@@ -7,7 +7,6 @@
  * @id cpp/inconsistent-loop-direction
  * @tags correctness
  *       external/cwe/cwe-835
- *       external/microsoft/6293
  * @msrc.severity important
  */
 

cpp/ql/src/Likely Bugs/Memory Management/ReturnStackAllocatedMemory.ql

@@ -18,7 +18,7 @@ import semmle.code.cpp.ir.IR
 import semmle.code.cpp.ir.dataflow.MustFlow
 import PathGraph
 
-/** Holds if `f` has a name that we intrepret as evidence of intentionally returning the value of the stack pointer. */
+/** Holds if `f` has a name that we interpret as evidence of intentionally returning the value of the stack pointer. */
 predicate intentionallyReturnsStackPointer(Function f) {
   f.getName().toLowerCase().matches(["%stack%", "%sp%"])
 }
@@ -52,6 +52,18 @@ class ReturnStackAllocatedMemoryConfig extends MustFlowConfiguration {
     )
   }
 
+  //  We disable flow into callables in this query as we'd otherwise get a result on this piece of code:
+  //   ```cpp
+  //  int* id(int* px) {
+  //   return px; // this returns the local variable `x`, but it's fine as the local variable isn't declared in this scope.
+  //  }
+  //  void f() {
+  //   int x;
+  //   int* px = id(&x);
+  //  }
+  //   ```
+  override predicate allowInterproceduralFlow() { none() }
+
   /**
    * This configuration intentionally conflates addresses of fields and their object, and pointer offsets
    * with their base pointer as this allows us to detect cases where an object's address flows to a
@@ -74,13 +86,9 @@ class ReturnStackAllocatedMemoryConfig extends MustFlowConfiguration {
 
 from
   MustFlowPathNode source, MustFlowPathNode sink, VariableAddressInstruction var,
-  ReturnStackAllocatedMemoryConfig conf, Function f
+  ReturnStackAllocatedMemoryConfig conf
 where
-  conf.hasFlowPath(source, sink) and
-  source.getNode().asInstruction() = var and
-  // Only raise an alert if we're returning from the _same_ callable as the on that
-  // declared the stack variable.
-  var.getEnclosingFunction() = pragma[only_bind_into](f) and
-  sink.getNode().getEnclosingCallable() = pragma[only_bind_into](f)
+  conf.hasFlowPath(pragma[only_bind_into](source), pragma[only_bind_into](sink)) and
+  source.getNode().asInstruction() = var
 select sink.getNode(), source, sink, "May return stack-allocated memory from $@.", var.getAst(),
   var.getAst().toString()

cpp/ql/src/Likely Bugs/Memory Management/StrncpyFlippedArgs.ql

@@ -18,6 +18,7 @@
 import cpp
 import Buffer
 private import semmle.code.cpp.valuenumbering.GlobalValueNumbering
+private import semmle.code.cpp.models.implementations.Strcpy
 
 predicate isSizePlus(Expr e, BufferSizeExpr baseSize, int plus) {
   // baseSize
@@ -41,33 +42,6 @@ predicate isSizePlus(Expr e, BufferSizeExpr baseSize, int plus) {
   )
 }
 
-predicate strncpyFunction(Function f, int argDest, int argSrc, int argLimit) {
-  exists(string name | name = f.getName() |
-    name =
-      [
-        "strcpy_s", // strcpy_s(dst, max_amount, src)
-        "wcscpy_s", // wcscpy_s(dst, max_amount, src)
-        "_mbscpy_s" // _mbscpy_s(dst, max_amount, src)
-      ] and
-    argDest = 0 and
-    argSrc = 2 and
-    argLimit = 1
-    or
-    name =
-      [
-        "strncpy", // strncpy(dst, src, max_amount)
-        "strncpy_l", // strncpy_l(dst, src, max_amount, locale)
-        "wcsncpy", // wcsncpy(dst, src, max_amount)
-        "_wcsncpy_l", // _wcsncpy_l(dst, src, max_amount, locale)
-        "_mbsncpy", // _mbsncpy(dst, src, max_amount)
-        "_mbsncpy_l" // _mbsncpy_l(dst, src, max_amount, locale)
-      ] and
-    argDest = 0 and
-    argSrc = 1 and
-    argLimit = 2
-  )
-}
-
 string nthString(int num) {
   num = 0 and
   result = "first"
@@ -96,11 +70,13 @@ int arrayExprFixedSize(Expr e) {
 }
 
 from
-  Function f, FunctionCall fc, int argDest, int argSrc, int argLimit, int charSize, Access copyDest,
-  Access copySource, string name, string nth
+  StrcpyFunction f, FunctionCall fc, int argDest, int argSrc, int argLimit, int charSize,
+  Access copyDest, Access copySource, string name, string nth
 where
   f = fc.getTarget() and
-  strncpyFunction(f, argDest, argSrc, argLimit) and
+  argDest = f.getParamDest() and
+  argSrc = f.getParamSrc() and
+  argLimit = f.getParamSize() and
   copyDest = fc.getArgument(argDest) and
   copySource = fc.getArgument(argSrc) and
   // Some of the functions operate on a larger char type, like `wchar_t`, so we

cpp/ql/src/Likely Bugs/Memory Management/UsingExpiredStackAddress.ql

@@ -133,7 +133,9 @@ TGlobalAddress globalAddress(Instruction instr) {
   )
   or
   exists(FieldAddressInstruction fai | instr = fai |
-    result = TFieldAddress(globalAddress(fai.getObjectAddress()), fai.getField())
+    result =
+      TFieldAddress(globalAddress(pragma[only_bind_into](fai.getObjectAddress())),
+        pragma[only_bind_out](fai.getField()))
   )
   or
   result = globalAddress(instr.(PointerOffsetInstruction).getLeft())

cpp/ql/src/Likely Bugs/UseInOwnInitializer.ql

@@ -15,6 +15,7 @@ class VariableAccessInInitializer extends VariableAccess {
   Variable var;
   Initializer init;
 
+  pragma[nomagic]
   VariableAccessInInitializer() {
     init.getDeclaration() = var and
     init.getExpr().getAChild*() = this

cpp/ql/src/Security/CWE/CWE-078/ExecTainted.ql

@@ -77,7 +77,7 @@ class ExecState extends DataFlow::FlowState {
   ExecState() {
     this =
       "ExecState (" + fst.getLocation() + " | " + fst + ", " + snd.getLocation() + " | " + snd + ")" and
-    interestingConcatenation(fst, snd)
+    interestingConcatenation(pragma[only_bind_into](fst), pragma[only_bind_into](snd))
   }
 
   DataFlow::Node getFstNode() { result = fst }

cpp/ql/src/Security/CWE/CWE-253/HResultBooleanConversion.ql

@@ -8,11 +8,6 @@
  * @precision high
  * @tags security
  *       external/cwe/cwe-253
- *       external/microsoft/C6214
- *       external/microsoft/C6215
- *       external/microsoft/C6216
- *       external/microsoft/C6217
- *       external/microsoft/C6230
  */
 
 import cpp

cpp/ql/src/Security/CWE/CWE-428/UnsafeCreateProcessCall.ql

@@ -9,7 +9,6 @@
  * @msrc.severity important
  * @tags security
  *       external/cwe/cwe-428
- *       external/microsoft/C6277
  */
 
 import cpp

cpp/ql/src/Security/CWE/CWE-704/WcharCharConversion.ql

@@ -10,7 +10,6 @@
  * @precision high
  * @tags security
  *       external/cwe/cwe-704
- *       external/microsoft/c/c6276
  */
 
 import cpp

cpp/ql/src/Security/CWE/CWE-732/UnsafeDaclSecurityDescriptor.ql

@@ -11,7 +11,6 @@
  * @precision high
  * @tags security
  *       external/cwe/cwe-732
- *       external/microsoft/C6248
  */
 
 import cpp

cpp/ql/src/change-notes/released/0.3.0.md

@@ -0,0 +1,5 @@
+## 0.3.0
+
+### Breaking Changes
+
+* Contextual queries and the query libraries they depend on have been moved to the `codeql/cpp-all` package.

cpp/ql/src/change-notes/released/0.3.1.md

@@ -0,0 +1 @@
+## 0.3.1

cpp/ql/src/change-notes/released/0.3.2.md

@@ -0,0 +1,5 @@
+## 0.3.2
+
+### Minor Analysis Improvements
+
+* The query `cpp/bad-strncpy-size` now covers more `strncpy`-like functions than before, including `strxfrm`(`_l`), `wcsxfrm`(`_l`), and `stpncpy`. Users of this query may see an increase in results.

cpp/ql/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.2.0
+lastReleaseVersion: 0.3.2

cpp/ql/src/experimental/Security/CWE/CWE-125/DangerousWorksWithMultibyteOrWideCharacters.cpp

@@ -0,0 +1,6 @@
+
+...
+mbtowc(&wc, ptr, 4)); // BAD:we can get unpredictable results
+...
+mbtowc(&wc, ptr, MB_LEN_MAX); // GOOD
+...

cpp/ql/src/experimental/Security/CWE/CWE-125/DangerousWorksWithMultibyteOrWideCharacters.qhelp

@@ -0,0 +1,23 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+<overview>
+<p> Using a function to convert multibyte or wide characters with an invalid length argument may result in an out-of-range access error or unexpected results.</p>
+
+</overview>
+
+<example>
+<p>The following example shows the erroneous and corrected method of using function mbtowc.</p>
+<sample src="DangerousWorksWithMultibyteOrWideCharacters.cpp" />
+
+</example>
+<references>
+
+<li>
+  CERT Coding Standard:
+  <a href="https://wiki.sei.cmu.edu/confluence/display/c/ARR30-C.+Do+not+form+or+use+out-of-bounds+pointers+or+array+subscripts">ARR30-C. Do not form or use out-of-bounds pointers or array subscripts - SEI CERT C Coding Standard - Confluence</a>.
+</li>
+
+</references>
+</qhelp>

cpp/ql/src/experimental/Security/CWE/CWE-125/DangerousWorksWithMultibyteOrWideCharacters.ql

@@ -0,0 +1,236 @@
+/**
+ * @name Dangerous use convert function.
+ * @description Using convert function with an invalid length argument can result in an out-of-bounds access error or unexpected result.
+ * @kind problem
+ * @id cpp/dangerous-use-convert-function
+ * @problem.severity warning
+ * @precision medium
+ * @tags correctness
+ *       security
+ *       external/cwe/cwe-125
+ */
+
+import cpp
+import semmle.code.cpp.valuenumbering.GlobalValueNumbering
+
+/** Holds if there are indications that the variable is treated as a string. */
+predicate exprMayBeString(Expr exp) {
+  (
+    exists(StringLiteral sl | globalValueNumber(exp) = globalValueNumber(sl))
+    or
+    exists(FunctionCall fctmp |
+      (
+        fctmp.getAnArgument().(VariableAccess).getTarget() = exp.(VariableAccess).getTarget() or
+        globalValueNumber(fctmp.getAnArgument()) = globalValueNumber(exp)
+      ) and
+      fctmp.getTarget().hasName(["strlen", "strcat", "strncat", "strcpy", "sptintf", "printf"])
+    )
+    or
+    exists(AssignExpr astmp |
+      astmp.getRValue().getValue() = "0" and
+      astmp.getLValue().(ArrayExpr).getArrayBase().(VariableAccess).getTarget() =
+        exp.(VariableAccess).getTarget()
+    )
+    or
+    exists(ComparisonOperation cotmp, Expr exptmp1, Expr exptmp2 |
+      exptmp1.getValue() = "0" and
+      (
+        exptmp2.(PointerDereferenceExpr).getOperand().(VariableAccess).getTarget() =
+          exp.(VariableAccess).getTarget() or
+        exptmp2.(ArrayExpr).getArrayBase().(VariableAccess).getTarget() =
+          exp.getAChild().(VariableAccess).getTarget()
+      ) and
+      cotmp.hasOperands(exptmp1, exptmp2)
+    )
+  )
+}
+
+/** Holds if expression is constant or operator call `sizeof`. */
+predicate argConstOrSizeof(Expr exp) {
+  exp.getValue().toInt() > 1 or
+  exp.(SizeofTypeOperator).getTypeOperand().getSize() > 1
+}
+
+/** Holds if expression is macro. */
+predicate argMacro(Expr exp) {
+  exists(MacroInvocation matmp |
+    exp = matmp.getExpr() and
+    (
+      matmp.getMacroName() = "MB_LEN_MAX" or
+      matmp.getMacroName() = "MB_CUR_MAX"
+    )
+  )
+}
+
+/** Holds if erroneous situations of using functions `mbtowc` and `mbrtowc` are detected. */
+predicate findUseCharacterConversion(Expr exp, string msg) {
+  exists(FunctionCall fc |
+    fc = exp and
+    (
+      exists(Loop lptmp | lptmp = fc.getEnclosingStmt().getParentStmt*()) and
+      fc.getTarget().hasName(["mbtowc", "mbrtowc", "_mbtowc_l"]) and
+      not fc.getArgument(0).isConstant() and
+      not fc.getArgument(1).isConstant() and
+      (
+        exprMayBeString(fc.getArgument(1)) and
+        argConstOrSizeof(fc.getArgument(2)) and
+        fc.getArgument(2).getValue().toInt() < 5 and
+        not argMacro(fc.getArgument(2)) and
+        msg = "Size can be less than maximum character length, use macro MB_CUR_MAX."
+        or
+        not exprMayBeString(fc.getArgument(1)) and
+        (
+          argConstOrSizeof(fc.getArgument(2))
+          or
+          argMacro(fc.getArgument(2))
+          or
+          exists(DecrementOperation dotmp |
+            globalValueNumber(dotmp.getAnOperand()) = globalValueNumber(fc.getArgument(2)) and
+            not exists(AssignSubExpr aetmp |
+              (
+                aetmp.getLValue().(VariableAccess).getTarget() =
+                  fc.getArgument(2).(VariableAccess).getTarget() or
+                globalValueNumber(aetmp.getLValue()) = globalValueNumber(fc.getArgument(2))
+              ) and
+              globalValueNumber(aetmp.getRValue()) = globalValueNumber(fc)
+            )
+          )
+        ) and
+        msg =
+          "Access beyond the allocated memory is possible, the length can change without changing the pointer."
+        or
+        exists(AssignPointerAddExpr aetmp |
+          (
+            aetmp.getLValue().(VariableAccess).getTarget() =
+              fc.getArgument(0).(VariableAccess).getTarget() or
+            globalValueNumber(aetmp.getLValue()) = globalValueNumber(fc.getArgument(0))
+          ) and
+          globalValueNumber(aetmp.getRValue()) = globalValueNumber(fc)
+        ) and
+        msg = "Maybe you're using the function's return value incorrectly."
+      )
+    )
+  )
+}
+
+/** Holds if detecting erroneous situations of working with multibyte characters. */
+predicate findUseMultibyteCharacter(Expr exp, string msg) {
+  exists(ArrayType arrayType, ArrayExpr arrayExpr |
+    arrayExpr = exp and
+    arrayExpr.getArrayBase().getType() = arrayType and
+    (
+      exists(AssignExpr assZero, SizeofExprOperator sizeofArray, Expr oneValue |
+        oneValue.getValue() = "1" and
+        sizeofArray.getExprOperand().getType() = arrayType and
+        assZero.getLValue() = arrayExpr and
+        arrayExpr.getArrayOffset().(SubExpr).hasOperands(sizeofArray, oneValue) and
+        assZero.getRValue().getValue() = "0"
+      ) and
+      arrayType.getArraySize() != arrayType.getByteSize() and
+      msg =
+        "The size of the array element is greater than one byte, so the offset will point outside the array."
+      or
+      exists(FunctionCall mbFunction |
+        (
+          mbFunction.getTarget().getName().matches("_mbs%") or
+          mbFunction.getTarget().getName().matches("mbs%") or
+          mbFunction.getTarget().getName().matches("_mbc%") or
+          mbFunction.getTarget().getName().matches("mbc%")
+        ) and
+        mbFunction.getAnArgument().(VariableAccess).getTarget().getADeclarationEntry().getType() =
+          arrayType
+      ) and
+      exists(Loop loop, SizeofExprOperator sizeofArray, AssignExpr assignExpr |
+        arrayExpr.getEnclosingStmt().getParentStmt*() = loop and
+        sizeofArray.getExprOperand().getType() = arrayType and
+        assignExpr.getLValue() = arrayExpr and
+        loop.getCondition().(LTExpr).getLeftOperand().(VariableAccess).getTarget() =
+          arrayExpr.getArrayOffset().getAChild*().(VariableAccess).getTarget() and
+        loop.getCondition().(LTExpr).getRightOperand() = sizeofArray
+      ) and
+      msg =
+        "This buffer may contain multibyte characters, so attempting to copy may result in part of the last character being lost."
+    )
+  )
+  or
+  exists(FunctionCall mbccpy, Loop loop, SizeofExprOperator sizeofOp |
+    mbccpy.getTarget().hasName("_mbccpy") and
+    mbccpy.getArgument(0) = exp and
+    exp.getEnclosingStmt().getParentStmt*() = loop and
+    sizeofOp.getExprOperand().getType() =
+      exp.getAChild*().(VariableAccess).getTarget().getADeclarationEntry().getType() and
+    loop.getCondition().(LTExpr).getLeftOperand().(VariableAccess).getTarget() =
+      exp.getAChild*().(VariableAccess).getTarget() and
+    loop.getCondition().(LTExpr).getRightOperand() = sizeofOp and
+    msg =
+      "This buffer may contain multibyte characters, so an attempt to copy may result in an overflow."
+  )
+}
+
+/** Holds if erroneous situations of using functions `MultiByteToWideChar` and `WideCharToMultiByte` or `mbstowcs` and `_mbstowcs_l` and `mbsrtowcs` are detected. */
+predicate findUseStringConversion(
+  Expr exp, string msg, int posBufSrc, int posBufDst, int posSizeDst, string nameCalls
+) {
+  exists(FunctionCall fc |
+    fc = exp and
+    posBufSrc in [0 .. fc.getNumberOfArguments() - 1] and
+    posSizeDst in [0 .. fc.getNumberOfArguments() - 1] and
+    (
+      fc.getTarget().hasName(nameCalls) and
+      (
+        globalValueNumber(fc.getArgument(posBufDst)) = globalValueNumber(fc.getArgument(posBufSrc)) and
+        msg =
+          "According to the definition of the functions, if the source buffer and the destination buffer are the same, undefined behavior is possible."
+        or
+        exists(ArrayType arrayDst |
+          fc.getArgument(posBufDst).(VariableAccess).getTarget().getADeclarationEntry().getType() =
+            arrayDst and
+          fc.getArgument(posSizeDst).getValue().toInt() >= arrayDst.getArraySize() and
+          not exists(AssignExpr assZero |
+            assZero.getLValue().(ArrayExpr).getArrayBase().(VariableAccess).getTarget() =
+              fc.getArgument(posBufDst).(VariableAccess).getTarget() and
+            assZero.getRValue().getValue() = "0"
+          ) and
+          not exists(Expr someExp, FunctionCall checkSize |
+            checkSize.getASuccessor*() = fc and
+            checkSize.getTarget().hasName(nameCalls) and
+            checkSize.getArgument(posSizeDst).getValue() = "0" and
+            globalValueNumber(checkSize) = globalValueNumber(someExp) and
+            someExp.getEnclosingStmt().getParentStmt*() instanceof IfStmt
+          ) and
+          exprMayBeString(fc.getArgument(posBufDst)) and
+          msg =
+            "According to the definition of the functions, it is not guaranteed to write a null character at the end of the string, so access beyond the bounds of the destination buffer is possible."
+        )
+        or
+        exists(FunctionCall allocMem |
+          allocMem.getTarget().hasName(["calloc", "malloc"]) and
+          globalValueNumber(fc.getArgument(posBufDst)) = globalValueNumber(allocMem) and
+          (
+            allocMem.getArgument(allocMem.getNumberOfArguments() - 1).getValue() = "1" or
+            not exists(SizeofOperator sizeofOperator |
+              globalValueNumber(allocMem
+                    .getArgument(allocMem.getNumberOfArguments() - 1)
+                    .getAChild*()) = globalValueNumber(sizeofOperator)
+            )
+          ) and
+          msg =
+            "The buffer destination has a type other than char, you need to take this into account when allocating memory."
+        )
+        or
+        fc.getArgument(posBufDst).getValue() = "0" and
+        fc.getArgument(posSizeDst).getValue() != "0" and
+        msg =
+          "If the destination buffer is NULL and its size is not 0, then undefined behavior is possible."
+      )
+    )
+  )
+}
+
+from Expr exp, string msg
+where
+  findUseCharacterConversion(exp, msg) or
+  findUseMultibyteCharacter(exp, msg) or
+  findUseStringConversion(exp, msg, 1, 0, 2, ["mbstowcs", "_mbstowcs_l", "mbsrtowcs"]) or
+  findUseStringConversion(exp, msg, 2, 4, 5, ["MultiByteToWideChar", "WideCharToMultiByte"])
+select exp, msg

cpp/ql/src/experimental/Security/CWE/CWE-273/PrivilegeDroppingOutoforder.qhelp

@@ -27,6 +27,9 @@ groups, and finally set the target user.</p>
 </example>
 <references>
 
+<li>CERT C Coding Standard:
+<a href="https://wiki.sei.cmu.edu/confluence/display/c/POS36-C.+Observe+correct+revocation+order+while+relinquishing+privileges">POS36-C. Observe correct revocation order while relinquishing privileges</a>.
+</li>
 <li>CERT C Coding Standard:
 <a href="https://wiki.sei.cmu.edu/confluence/display/c/POS37-C.+Ensure+that+privilege+relinquishment+is+successful">POS37-C. Ensure that privilege relinquishment is successful</a>.
 </li>

cpp/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-queries
-version: 0.2.0
+version: 0.3.2
 groups: 
   - cpp
   - queries

cpp/ql/test/experimental/query-tests/Security/CWE/CWE-125/semmle/tests/DangerousWorksWithMultibyteOrWideCharacters.expected

@@ -0,0 +1,26 @@
+| test1.cpp:28:5:28:23 | call to WideCharToMultiByte | According to the definition of the functions, if the source buffer and the destination buffer are the same, undefined behavior is possible. |
+| test1.cpp:29:5:29:23 | call to MultiByteToWideChar | According to the definition of the functions, if the source buffer and the destination buffer are the same, undefined behavior is possible. |
+| test1.cpp:45:3:45:21 | call to WideCharToMultiByte | According to the definition of the functions, it is not guaranteed to write a null character at the end of the string, so access beyond the bounds of the destination buffer is possible. |
+| test1.cpp:58:3:58:21 | call to MultiByteToWideChar | The buffer destination has a type other than char, you need to take this into account when allocating memory. |
+| test1.cpp:70:3:70:21 | call to MultiByteToWideChar | The buffer destination has a type other than char, you need to take this into account when allocating memory. |
+| test1.cpp:76:10:76:28 | call to WideCharToMultiByte | If the destination buffer is NULL and its size is not 0, then undefined behavior is possible. |
+| test1.cpp:93:5:93:23 | call to WideCharToMultiByte | According to the definition of the functions, it is not guaranteed to write a null character at the end of the string, so access beyond the bounds of the destination buffer is possible. |
+| test2.cpp:15:5:15:12 | call to mbstowcs | According to the definition of the functions, if the source buffer and the destination buffer are the same, undefined behavior is possible. |
+| test2.cpp:17:5:17:15 | call to _mbstowcs_l | According to the definition of the functions, if the source buffer and the destination buffer are the same, undefined behavior is possible. |
+| test2.cpp:19:5:19:13 | call to mbsrtowcs | According to the definition of the functions, if the source buffer and the destination buffer are the same, undefined behavior is possible. |
+| test2.cpp:35:3:35:10 | call to mbstowcs | According to the definition of the functions, it is not guaranteed to write a null character at the end of the string, so access beyond the bounds of the destination buffer is possible. |
+| test2.cpp:48:3:48:10 | call to mbstowcs | The buffer destination has a type other than char, you need to take this into account when allocating memory. |
+| test2.cpp:60:3:60:10 | call to mbstowcs | The buffer destination has a type other than char, you need to take this into account when allocating memory. |
+| test2.cpp:66:10:66:17 | call to mbstowcs | If the destination buffer is NULL and its size is not 0, then undefined behavior is possible. |
+| test2.cpp:80:3:80:10 | call to mbstowcs | According to the definition of the functions, it is not guaranteed to write a null character at the end of the string, so access beyond the bounds of the destination buffer is possible. |
+| test3.cpp:16:5:16:13 | access to array | This buffer may contain multibyte characters, so attempting to copy may result in part of the last character being lost. |
+| test3.cpp:36:13:36:18 | ... + ... | This buffer may contain multibyte characters, so an attempt to copy may result in an overflow. |
+| test3.cpp:47:3:47:24 | access to array | The size of the array element is greater than one byte, so the offset will point outside the array. |
+| test.cpp:66:27:66:32 | call to mbtowc | Size can be less than maximum character length, use macro MB_CUR_MAX. |
+| test.cpp:76:27:76:32 | call to mbtowc | Size can be less than maximum character length, use macro MB_CUR_MAX. |
+| test.cpp:106:11:106:16 | call to mbtowc | Access beyond the allocated memory is possible, the length can change without changing the pointer. |
+| test.cpp:123:11:123:16 | call to mbtowc | Access beyond the allocated memory is possible, the length can change without changing the pointer. |
+| test.cpp:140:11:140:16 | call to mbtowc | Access beyond the allocated memory is possible, the length can change without changing the pointer. |
+| test.cpp:158:11:158:16 | call to mbtowc | Access beyond the allocated memory is possible, the length can change without changing the pointer. |
+| test.cpp:181:11:181:16 | call to mbtowc | Access beyond the allocated memory is possible, the length can change without changing the pointer. |
+| test.cpp:197:11:197:16 | call to mbtowc | Maybe you're using the function's return value incorrectly. |

cpp/ql/test/experimental/query-tests/Security/CWE/CWE-125/semmle/tests/DangerousWorksWithMultibyteOrWideCharacters.qlref

@@ -0,0 +1 @@
+experimental/Security/CWE/CWE-125/DangerousWorksWithMultibyteOrWideCharacters.ql

cpp/ql/test/experimental/query-tests/Security/CWE/CWE-125/semmle/tests/test.cpp

@@ -0,0 +1,206 @@
+typedef unsigned long size_t;
+#define MB_CUR_MAX 6
+#define MB_LEN_MAX 16
+int mbtowc(wchar_t *out, const char *in, size_t size);
+int wprintf (const wchar_t* format, ...);
+int strlen( const char * string );
+int checkErrors();
+
+void goodTest0()
+{
+  char * ptr = "123456789";
+  int ret;
+  int len;
+  len = 9;
+  for (wchar_t wc; (ret = mbtowc(&wc, ptr, len)) > 0; len-=ret) { // GOOD
+    wprintf(L"%lc", wc);
+    ptr += ret;
+  }
+}
+void goodTest1(const char* ptr)
+{
+  int ret;
+  int len;
+  len = strlen(ptr);
+  for (wchar_t wc; (ret = mbtowc(&wc, ptr, len)) > 0; len-=ret) { // GOOD
+    wprintf(L"%lc", wc);
+    ptr += ret;
+  }
+}
+void goodTest2(char* ptr)
+{
+  int ret;
+  ptr[10]=0;
+  int len = 9;
+  for (wchar_t wc; (ret = mbtowc(&wc, ptr, 16)) > 0; len-=ret) { // GOOD
+    wprintf(L"%lc", wc);
+    ptr += ret;
+  }
+}
+
+void goodTest3(const char* ptr)
+{
+  int ret;
+  int len;
+  len = strlen(ptr);
+  for (wchar_t wc; (ret = mbtowc(&wc, ptr, MB_CUR_MAX)) > 0; len-=ret) { // GOOD
+    wprintf(L"%lc", wc);
+    ptr += ret;
+  }
+}
+void goodTest4(const char* ptr)
+{
+  int ret;
+  int len;
+  len = strlen(ptr);
+  for (wchar_t wc; (ret = mbtowc(&wc, ptr, MB_LEN_MAX)) > 0; len-=ret) { // GOOD
+    wprintf(L"%lc", wc);
+    ptr += ret;
+  }
+}
+void badTest1(const char* ptr)
+{
+  int ret;
+  int len;
+  len = strlen(ptr);
+  for (wchar_t wc; (ret = mbtowc(&wc, ptr, 4)) > 0; len-=ret) { // BAD:we can get unpredictable results
+    wprintf(L"%lc", wc);
+    ptr += ret;
+  }
+}
+void badTest2(const char* ptr)
+{
+  int ret;
+  int len;
+  len = strlen(ptr);
+  for (wchar_t wc; (ret = mbtowc(&wc, ptr, sizeof(wchar_t))) > 0; len-=ret) { // BAD:we can get unpredictable results
+    wprintf(L"%lc", wc);
+    ptr += ret;
+  }
+}
+
+void goodTest5(const char* ptr,wchar_t *wc,int wc_len)
+{
+  int ret;
+  int len;
+  len = wc_len;
+  while (*ptr && len > 0) {
+    ret = mbtowc(wc, ptr, len); // GOOD
+    if (ret <0)
+      break;
+    if (ret == 0 || ret > len)
+      break;
+    len-=ret;
+    ptr+=ret;
+    wc++;
+  }
+}
+
+void badTest3(const char* ptr,int wc_len)
+{
+  int ret;
+  int len;
+  len = wc_len;
+  wchar_t *wc = new wchar_t[wc_len];
+  while (*ptr && len > 0) {
+    ret = mbtowc(wc, ptr, MB_CUR_MAX); // BAD
+    if (ret <0)
+      break;
+    if (ret == 0 || ret > len)
+      break;
+    len-=ret;
+    ptr+=ret;
+    wc++;
+  }
+}
+void badTest4(const char* ptr,int wc_len)
+{
+  int ret;
+  int len;
+  len = wc_len;
+  wchar_t *wc = new wchar_t[wc_len];
+  while (*ptr && len > 0) {
+    ret = mbtowc(wc, ptr, 16); // BAD
+    if (ret <0)
+      break;
+    if (ret == 0 || ret > len)
+      break;
+    len-=ret;
+    ptr+=ret;
+    wc++;
+  }
+}
+void badTest5(const char* ptr,int wc_len)
+{
+  int ret;
+  int len;
+  len = wc_len;
+  wchar_t *wc = new wchar_t[wc_len];
+  while (*ptr && len > 0) {
+    ret = mbtowc(wc, ptr, sizeof(wchar_t)); // BAD
+    if (ret <0)
+      break;
+    if (ret == 0 || ret > len)
+      break;
+    len-=ret;
+    ptr+=ret;
+    wc++;
+  }
+}
+
+void badTest6(const char* ptr,int wc_len)
+{
+  int ret;
+  int len;
+  len = wc_len;
+  wchar_t *wc = new wchar_t[wc_len];
+  while (*ptr && wc_len > 0) {
+    ret = mbtowc(wc, ptr, wc_len); // BAD
+    if (ret <0)
+      if (checkErrors()) {
+        ++ptr;
+        --len;
+        continue;
+      } else
+        break;
+    if (ret == 0 || ret > len)
+      break;
+    wc_len--;
+    len-=ret;
+    wc++;
+    ptr+=ret;
+  }
+}
+void badTest7(const char* ptr,int wc_len)
+{
+  int ret;
+  int len;
+  len = wc_len;
+  wchar_t *wc = new wchar_t[wc_len];
+  while (*ptr && wc_len > 0) {
+    ret = mbtowc(wc, ptr, len); // BAD
+    if (ret <0)
+        break;
+    if (ret == 0 || ret > len)
+      break;
+    len--;
+    wc++;
+    ptr+=ret;
+  }
+}
+void badTest8(const char* ptr,wchar_t *wc)
+{
+  int ret;
+  int len;
+  len = strlen(ptr);
+  while (*ptr && len > 0) {
+    ret = mbtowc(wc, ptr, len); // BAD
+    if (ret <0)
+      break;
+    if (ret == 0 || ret > len)
+      break;
+    len-=ret;
+    ptr++;
+    wc+=ret;
+  }
+}