repatch

This reverts commit a0f479d503.

.codeqlmanifest.json

@@ -4,17 +4,15 @@
         "*/ql/lib/qlpack.yml",
         "*/ql/test/qlpack.yml",
         "*/ql/examples/qlpack.yml",
-        "*/ql/consistency-queries/qlpack.yml",
+        "*/upgrades/qlpack.yml",
         "cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml",
         "javascript/ql/experimental/adaptivethreatmodeling/lib/qlpack.yml",
-        "javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/qlpack.yml",
         "javascript/ql/experimental/adaptivethreatmodeling/src/qlpack.yml",
-        "csharp/ql/campaigns/Solorigate/lib/qlpack.yml",
-        "csharp/ql/campaigns/Solorigate/src/qlpack.yml",
-        "csharp/ql/campaigns/Solorigate/test/qlpack.yml",
         "misc/legacy-support/*/qlpack.yml",
         "misc/suite-helpers/qlpack.yml",
         "ruby/extractor-pack/codeql-extractor.yml",
+        "ruby/ql/consistency-queries/qlpack.yml",
+        "ql/ql/consistency-queries/qlpack.yml",
         "ql/extractor-pack/codeql-extractor.yml"
     ],
     "versionPolicies": {

.gitattributes

@@ -51,14 +51,3 @@
 
 java/ql/test/stubs/**/*.java linguist-generated=true
 java/ql/test/experimental/stubs/**/*.java linguist-generated=true
-
-# For some languages, upgrade script testing references really old dbscheme
-# files from legacy upgrades that have CRLF line endings. Since upgrade
-# resolution relies on object hashes, we must suppress line ending conversion
-# for those testing dbscheme files.
-*/ql/lib/upgrades/initial/*.dbscheme -text
-
-# Generated test files - these are synced from the standard JavaScript libraries using
-# `javascript/ql/experimental/adaptivethreatmodeling/test/update_endpoint_test_files.py`.
-javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.js linguist-generated=true -merge
-javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.ts linguist-generated=true -merge

.github/labeler.yml

@@ -26,6 +26,3 @@ documentation:
   - "**/*.qhelp"
   - "**/*.md"
   - docs/**/*
-
-"QL-for-QL": 
-  - ql/**/*

.github/workflows/check-change-note.yml

@@ -6,11 +6,7 @@ on:
     paths:
       - "*/ql/src/**/*.ql"
       - "*/ql/src/**/*.qll"
-      - "*/ql/lib/**/*.ql"
-      - "*/ql/lib/**/*.qll"
       - "!**/experimental/**"
-      - "!ql/**"
-      - ".github/workflows/check-change-note.yml"
 
 jobs:
   check-change-note:

.github/workflows/codeql-analysis.yml

@@ -27,11 +27,6 @@ jobs:
       pull-requests: read
 
     steps:
-    - name: Setup dotnet
-      uses: actions/setup-dotnet@v1
-      with:
-        dotnet-version: 6.0.101
-
     - name: Checkout repository
       uses: actions/checkout@v2
 
@@ -56,7 +51,7 @@ jobs:
     #    uses a compiled language
 
     - run: |
-       dotnet build csharp /p:UseSharedCompilation=false
+       dotnet build csharp
 
     - name: Perform CodeQL Analysis
       uses: github/codeql-action/analyze@main

.github/workflows/csv-coverage-metrics.yml

@@ -1,43 +0,0 @@
-name: "Publish framework coverage as metrics"
-
-on:
-  schedule:
-    - cron: '5 0 * * *'
-  push:
-    branches:
-      - main
-  workflow_dispatch:
-  pull_request:
-    branches:
-      - main
-    paths:
-      - ".github/workflows/csv-coverage-metrics.yml"
-
-jobs:
-  publish:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v2
-      - name: Setup CodeQL
-        uses: ./.github/actions/fetch-codeql
-      - name: Create empty database
-        run: |
-          DATABASE="${{ runner.temp }}/java-database"
-          PROJECT="${{ runner.temp }}/java-project"
-          mkdir -p "$PROJECT/src/tmp/empty"
-          echo "class Empty {}" >> "$PROJECT/src/tmp/empty/Empty.java"
-          codeql database create "$DATABASE" --language=java --source-root="$PROJECT" --command 'javac src/tmp/empty/Empty.java'
-      - name: Capture coverage information
-        run: |
-          DATABASE="${{ runner.temp }}/java-database"
-          codeql database analyze --format=sarif-latest --output=metrics.sarif -- "$DATABASE" ./java/ql/src/Metrics/Summaries/FrameworkCoverage.ql
-      - uses: actions/upload-artifact@v2
-        with:
-          name: metrics.sarif
-          path: metrics.sarif
-          retention-days: 20
-      - name: Upload SARIF file
-        uses: github/codeql-action/upload-sarif@v1
-        with:
-          sarif_file: metrics.sarif

.github/workflows/js-ml-tests.yml

@@ -1,76 +0,0 @@
-name: JS ML-powered queries tests
-
-on:
-  push:
-    paths:
-      - "javascript/ql/experimental/adaptivethreatmodeling/**"
-      - .github/workflows/js-ml-tests.yml
-    branches:
-      - main
-      - "rc/*"
-  pull_request:
-    paths:
-      - "javascript/ql/experimental/adaptivethreatmodeling/**"
-      - .github/workflows/js-ml-tests.yml
-
-defaults:
-  run:
-    working-directory: javascript/ql/experimental/adaptivethreatmodeling
-
-jobs:
-  qlformat:
-    name: Check QL formatting
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-
-      - uses: ./.github/actions/fetch-codeql
-
-      - name: Check QL formatting
-        run: |
-          find . "(" -name "*.ql" -or -name "*.qll" ")" -print0 | \
-            xargs -0 codeql query format --check-only
-
-  qlcompile:
-    name: Check QL compilation
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-
-      - uses: ./.github/actions/fetch-codeql
-
-      - name: Install pack dependencies
-        run: |
-          for pack in modelbuilding src; do
-            codeql pack install --mode verify -- "${pack}"
-          done
-
-      - name: Check QL compilation
-        run: |
-          codeql query compile \
-            --check-only \
-            --ram 5120 \
-            --additional-packs "${{ github.workspace }}" \
-            --threads=0 \
-            -- \
-            lib modelbuilding src
-
-  qltest:
-    name: Run QL tests
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-
-      - uses: ./.github/actions/fetch-codeql
-
-      - name: Install pack dependencies
-        run: codeql pack install -- test
-
-      - name: Run QL tests
-        run: |
-          codeql test run \
-            --threads=0 \
-            --ram 5120 \
-            --additional-packs "${{ github.workspace }}" \
-            -- \
-            test

.github/workflows/mad_modelDiff.yml

@@ -1,103 +0,0 @@
-name: Models as Data - Diff
-
-on:
-  workflow_dispatch:
-    inputs:
-      projects:
-        description: "The projects to generate models for"
-        required: true
-        default: '["netty/netty"]'
-  pull_request:
-    branches:
-      - main
-    paths:
-      - "java/ql/src/utils/model-generator/**/*.*"
-      - ".github/workflows/mad_modelDiff.yml"
-
-permissions:
-  contents: read
-
-jobs:
-  model-diff:
-    name: Model Difference
-    runs-on: ubuntu-latest
-    if: github.repository == 'github/codeql'
-    strategy:
-      matrix:
-        slug: ${{fromJson(github.event.inputs.projects || '["apache/commons-codec", "apache/commons-io", "apache/commons-beanutils", "apache/commons-logging", "apache/commons-fileupload", "apache/commons-lang", "apache/commons-validator", "apache/commons-csv", "apache/dubbo"]' )}}
-    steps:
-      - name: Clone github/codeql from PR
-        uses: actions/checkout@v2
-        if: github.event.pull_request
-        with:
-          path: codeql-pr
-      - name: Clone github/codeql from main
-        uses: actions/checkout@v2
-        with:
-          path: codeql-main
-          ref: main
-      - uses: ./codeql-main/.github/actions/fetch-codeql
-      - name: Download database
-        env:
-          SLUG: ${{ matrix.slug }}
-        run: |
-          set -x
-          mkdir lib-dbs
-          SHORTNAME=${SLUG//[^a-zA-Z0-9_]/}
-          projectId=`curl -s https://lgtm.com/api/v1.0/projects/g/${SLUG} | jq .id`
-          curl -L "https://lgtm.com/api/v1.0/snapshots/$projectId/java" -o "$SHORTNAME.zip"
-          unzip -q -d "${SHORTNAME}-db" "${SHORTNAME}.zip"
-          mkdir "lib-dbs/$SHORTNAME/"
-          mv "${SHORTNAME}-db/"$(ls -1 "${SHORTNAME}"-db)/* "lib-dbs/${SHORTNAME}/"
-      - name: Generate Models (PR and main)
-        run: |
-          set -x
-          mkdir tmp-models
-          MODELS=`pwd`/tmp-models
-          DATABASES=`pwd`/lib-dbs
-
-          analyzeDatabaseWithCheckout() {
-            QL_VARIANT=$1
-            DATABASE=$2
-            cd codeql-$QL_VARIANT
-            SHORTNAME=`basename $DATABASE`
-            python java/ql/src/utils/model-generator/GenerateFlowModel.py $DATABASE $MODELS/${SHORTNAME}.qll
-            mv $MODELS/${SHORTNAME}.qll $MODELS/${SHORTNAME}Generated_${QL_VARIANT}.qll
-            cd ..
-          }
-
-          for d in $DATABASES/*/ ; do
-            ls -1 "$d"
-
-            analyzeDatabaseWithCheckout "main" $d
-            if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]
-            then
-              analyzeDatabaseWithCheckout "pr" $d
-            fi
-          done
-      - name: Install diff2html
-        if: github.event.pull_request
-        run: |
-          npm install -g diff2html-cli
-      - name: Generate Model Diff
-        if: github.event.pull_request
-        run: |
-          set -x
-          MODELS=`pwd`/tmp-models
-          ls -1 tmp-models/
-          for m in $MODELS/*_main.qll ; do
-            t="${m/main/"pr"}"
-            basename=`basename $m`
-            name="diff_${basename/_main.qll/""}"
-            (diff -w -u $m $t | diff2html -i stdin -F $MODELS/$name.html) || true
-          done
-      - uses: actions/upload-artifact@v2
-        with:
-          name: models
-          path: tmp-models/*.qll
-          retention-days: 20
-      - uses: actions/upload-artifact@v2
-        with:
-          name: diffs
-          path: tmp-models/*.html
-          retention-days: 20

.github/workflows/mad_regenerate-models.yml

@@ -1,62 +0,0 @@
-name: Regenerate framework models
-
-on:
-  workflow_dispatch:
-  schedule:
-    - cron: "30 2 * * *"
-  pull_request:
-    branches:
-      - main
-    paths:
-      - ".github/workflows/mad_regenerate-models.yml"
-
-jobs:
-  regenerate-models:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        # placeholder required for each axis, excluded below, replaced by the actual combinations (see include)
-        slug: ["placeholder"]
-        ref: ["placeholder"]
-        include:
-          - slug: "apache/commons-io"
-            ref: "8985de8fe74f6622a419b37a6eed0dbc484dc128"
-        exclude:
-          - slug: "placeholder"
-            ref: "placeholder"
-    steps:
-      - name: Clone self (github/codeql)
-        uses: actions/checkout@v2
-      - name: Setup CodeQL binaries
-        uses: ./.github/actions/fetch-codeql
-      - name: Clone repositories
-        uses: actions/checkout@v2
-        with:
-          path: repos/${{ matrix.ref }}
-          ref: ${{ matrix.ref }}
-          repository: ${{ matrix.slug }}
-      - name: Build database
-        env:
-          SLUG: ${{ matrix.slug }}
-          REF: ${{ matrix.ref }}
-        run: |
-          mkdir dbs
-          cd repos/${REF}
-          SHORTNAME=${SLUG//[^a-zA-Z0-9_]/}
-          codeql database create --language=java ../../dbs/${SHORTNAME}
-      - name: Regenerate models in-place
-        env:
-          SLUG: ${{ matrix.slug }}
-        run: |
-          SHORTNAME=${SLUG//[^a-zA-Z0-9_]/}
-          java/ql/src/utils/model-generator/RegenerateModels.py "${SLUG}" dbs/${SHORTNAME}
-      - name: Stage changes
-        run: |
-          find java -name "*.qll" -print0 | xargs -0 git add
-          git status
-          git diff --cached > models.patch
-      - uses: actions/upload-artifact@v2
-        with:
-          name: patch
-          path: models.patch
-          retention-days: 7

.github/workflows/ql-for-ql-build.yml

@@ -16,7 +16,7 @@ jobs:
       - uses: actions/checkout@v2
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@erik-krogh/ql
+        uses: github/codeql-action/init@esbena/ql
         with:
           languages: javascript # does not matter
       - name: Get CodeQL version
@@ -31,13 +31,13 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ${{ runner.temp }}/query-pack.zip
-          key: queries-${{ hashFiles('ql/**/*.ql*') }}-${{ hashFiles('ql/**/qlpack.yml') }}-${{ hashFiles('ql/ql/src/ql.dbscheme*') }}-${{ steps.get-codeql-version.outputs.version }}
+          key: queries-${{ hashFiles('ql/**/*.ql*') }}-${{ hashFiles('ql/ql/src/ql.dbscheme*') }}-${{ steps.get-codeql-version.outputs.version }}
       - name: Build query pack
         if: steps.cache-queries.outputs.cache-hit != 'true'
         run: |
           cd ql/ql/src
           "${CODEQL}" pack create
-          cd .codeql/pack/codeql/ql/0.0.0
+          cd .codeql/pack/codeql/ql-all/0.0.0
           zip "${PACKZIP}" -r .
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
@@ -56,46 +56,27 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
-      - name: Cache entire extractor
+      - uses: actions/cache@v2
-        id: cache-extractor
-        uses: actions/cache@v2
-        with:
-          path: |
-            ql/target/release/ql-autobuilder
-            ql/target/release/ql-autobuilder.exe
-            ql/target/release/ql-extractor
-            ql/target/release/ql-extractor.exe
-          key: ${{ runner.os }}-extractor-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('ql/**/*.rs') }}
-      - name: Cache cargo
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
-        uses: actions/cache@v2
         with:
           path: |
             ~/.cargo/registry
             ~/.cargo/git
             ql/target
-          key: ${{ runner.os }}-rust-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
+          key: ${{ runner.os }}-rust-cargo-${{ hashFiles('**/Cargo.lock') }}
       - name: Check formatting
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
         run: cd ql; cargo fmt --all -- --check
       - name: Build
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
         run: cd ql; cargo build --verbose
       - name: Run tests
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
         run: cd ql; cargo test --verbose
       - name: Release build
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
         run: cd ql; cargo build --release
       - name: Generate dbscheme
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
         run: ql/target/release/ql-generator --dbscheme ql/ql/src/ql.dbscheme --library ql/ql/src/codeql_ql/ast/internal/TreeSitter.qll
       - uses: actions/upload-artifact@v2
         with:
           name: extractor-ubuntu-latest
           path: |
-            ql/target/release/ql-autobuilder
-            ql/target/release/ql-autobuilder.exe
             ql/target/release/ql-extractor
             ql/target/release/ql-extractor.exe
           retention-days: 1
@@ -120,10 +101,6 @@ jobs:
           unzip query-pack-zip/*.zip -d pack
           cp -r ql/codeql-extractor.yml ql/tools ql/ql/src/ql.dbscheme.stats pack/
           mkdir -p pack/tools/linux64
-          if [[ -f linux64/ql-autobuilder ]]; then
-            cp linux64/ql-autobuilder pack/tools/linux64/autobuilder
-            chmod +x pack/tools/linux64/autobuilder
-          fi
           if [[ -f linux64/ql-extractor ]]; then
             cp linux64/ql-extractor pack/tools/linux64/extractor
             chmod +x pack/tools/linux64/extractor
@@ -137,9 +114,6 @@ jobs:
           retention-days: 1
   analyze: 
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        folder: [cpp, csharp, java, javascript, python, ql, ruby]
     
     needs: 
       - package
@@ -166,34 +140,13 @@ jobs:
 
       - name: Checkout repository
         uses: actions/checkout@v2
-      - name: Create CodeQL config file
-        run: |
-          echo "paths:" > ${CONF}
-          echo "  - ${FOLDER}" >> ${CONF}
-          echo "paths-ignore:" >> ${CONF}
-          echo "  - ql/ql/test" >> ${CONF}
-          echo "Config file: "
-          cat ${CONF}
-        env: 
-          CONF: ./ql-for-ql-config.yml
-          FOLDER: ${{ matrix.folder }}
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@erik-krogh/ql
+        uses: github/codeql-action/init@esbena/ql
         with:
           languages: ql
           db-location: ${{ runner.temp }}/db
-          config-file: ./ql-for-ql-config.yml
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@erik-krogh/ql
+        uses: github/codeql-action/analyze@esbena/ql
-        with: 
-          category: "ql-for-ql-${{ matrix.folder }}"
-      - name: Copy sarif file to CWD
-        run: cp ../results/ql.sarif ./${{ matrix.folder }}.sarif
-      - name: Sarif as artifact
-        uses: actions/upload-artifact@v2
-        with:
-          name: ${{ matrix.folder }}.sarif
-          path: ${{ matrix.folder }}.sarif
 

.github/workflows/ql-for-ql-dataset_measure.yml

@@ -26,7 +26,7 @@ jobs:
 
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@erik-krogh/ql
+        uses: github/codeql-action/init@esbena/ql
         with:
           languages: javascript # does not matter
       - uses: actions/cache@v2
@@ -35,7 +35,7 @@ jobs:
             ~/.cargo/registry
             ~/.cargo/git
             ql/target
-          key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
+          key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('**/Cargo.lock') }}
       - name: Build Extractor
         run: cd ql; env "PATH=$PATH:`dirname ${CODEQL}`" ./create-extractor-pack.sh
         env:

.github/workflows/ql-for-ql-tests.yml

@@ -4,11 +4,11 @@ on:
   push:
     branches: [main]
     paths:
-      - "ql/**"
+      - ql/*
   pull_request:
     branches: [main]
     paths:
-      - "ql/**"
+      - ql/*
 
 env:
   CARGO_TERM_COLOR: always
@@ -20,7 +20,7 @@ jobs:
       - uses: actions/checkout@v2
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@erik-krogh/ql
+        uses: github/codeql-action/init@esbena/ql
         with:
           languages: javascript # does not matter
       - uses: actions/cache@v2
@@ -29,7 +29,7 @@ jobs:
             ~/.cargo/registry
             ~/.cargo/git
             ql/target
-          key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
+          key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('**/Cargo.lock') }}
       - name: Build extractor
         run: |
           cd ql;

.github/workflows/ruby-build.yml

@@ -50,7 +50,7 @@ jobs:
             ~/.cargo/registry
             ~/.cargo/git
             ruby/target
-          key: ${{ runner.os }}-ruby-rust-cargo-${{ hashFiles('ruby/rust-toolchain.toml', 'ruby/**/Cargo.lock') }}
+          key: ${{ runner.os }}-rust-cargo-${{ hashFiles('**/Cargo.lock') }}
       - name: Check formatting
         run: cargo fmt --all -- --check
       - name: Build

.github/workflows/ruby-qltest.yml

@@ -24,54 +24,27 @@ defaults:
     working-directory: ruby
 
 jobs:
-  qlformat:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: ./.github/actions/fetch-codeql
-      - name: Check QL formatting
-        run: find ql "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 codeql query format --check-only
-  qlcompile:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: ./.github/actions/fetch-codeql
-      - name: Check QL compilation
-        run: |
-          codeql query compile --check-only --threads=0 --ram 5000 --warnings=error "ql/src" "ql/examples"
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-  qlupgrade:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: ./.github/actions/fetch-codeql
-      - name: Check DB upgrade scripts
-        run: |
-          echo >empty.trap
-          codeql dataset import -S ql/lib/upgrades/initial/ruby.dbscheme testdb empty.trap
-          codeql dataset upgrade testdb --additional-packs ql/lib
-          diff -q testdb/ruby.dbscheme ql/lib/ruby.dbscheme
-      - name: Check DB downgrade scripts
-        run: |
-          echo >empty.trap
-          rm -rf testdb; codeql dataset import -S ql/lib/ruby.dbscheme testdb empty.trap
-          codeql resolve upgrades --format=lines --allow-downgrades --additional-packs downgrades \
-           --dbscheme=ql/lib/ruby.dbscheme --target-dbscheme=downgrades/initial/ruby.dbscheme |
-           xargs codeql execute upgrades testdb
-          diff -q testdb/ruby.dbscheme downgrades/initial/ruby.dbscheme
   qltest:
     runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        slice: ["1/2", "2/2"]
     steps:
       - uses: actions/checkout@v2
       - uses: ./.github/actions/fetch-codeql
       - uses: ./ruby/actions/create-extractor-pack
       - name: Run QL tests
         run: |
-          codeql test run --threads=0 --ram 5000 --slice ${{ matrix.slice }} --search-path "${{ github.workspace }}/ruby/extractor-pack" --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test
+          codeql test run --search-path "${{ github.workspace }}/ruby/extractor-pack" --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test
         env:
           GITHUB_TOKEN: ${{ github.token }}
+      - name: Check QL formatting
+        run: find ql "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 codeql query format --check-only
+      - name: Check QL compilation
+        run: |
+          codeql query compile --check-only --threads=4 --warnings=error "ql/src" "ql/examples"
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+      - name: Check DB upgrade scripts
+        run: |
+          echo >empty.trap
+          codeql dataset import -S ql/lib/upgrades/initial/ruby.dbscheme testdb empty.trap
+          codeql dataset upgrade testdb --additional-packs ql/lib
+          diff -q testdb/ruby.dbscheme ql/lib/ruby.dbscheme

.github/workflows/validate-change-notes.yml

@@ -1,29 +0,0 @@
-name: Validate change notes
-
-on:
-  push:
-    paths:
-      - "*/ql/*/change-notes/**/*"
-      - ".github/workflows/validate-change-notes.yml"
-    branches:
-      - main
-      - "rc/*"
-  pull_request:
-    paths:
-      - "*/ql/*/change-notes/**/*"
-      - ".github/workflows/validate-change-notes.yml"
-
-jobs:
-  check-change-note:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v2
-
-      - name: Setup CodeQL
-        uses: ./.github/actions/fetch-codeql
-
-      - name: Fail if there are any errors with existing change notes
-
-        run: |
-          codeql pack release --groups cpp,csharp,java,javascript,python,ruby,-examples,-test,-experimental

.pre-commit-config.yaml

@@ -1,29 +0,0 @@
-# See https://pre-commit.com for more information
-# See https://pre-commit.com/hooks.html for more hooks
-exclude: /test/.*$(?<!\.ql)(?<!\.qll)(?<!\.qlref)
-repos:
--   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.2.0
-    hooks:
-    -   id: trailing-whitespace
-    -   id: end-of-file-fixer
-
--   repo: local
-    hooks:
-    -   id: codeql-format
-        name: Fix QL file formatting
-        files: \.qll?$
-        language: system
-        entry: codeql query format --in-place
-
-    -   id: sync-files
-        name: Fix files required to be identical
-        language: system
-        entry: python3 config/sync-files.py --latest
-        pass_filenames: false
-
-    -   id: qhelp
-        name: Check query help generation
-        files: \.qhelp$
-        language: system
-        entry: python3 misc/scripts/check-qhelp.py

CODEOWNERS

@@ -13,9 +13,6 @@
 /python/**/experimental/**/* @github/codeql-python @xcorail
 /ruby/**/experimental/**/* @github/codeql-ruby @xcorail
 
-# ML-powered queries
-/javascript/ql/experimental/adaptivethreatmodeling/ @github/codeql-ml-powered-queries-reviewers
-
 # Notify members of codeql-go about PRs to the shared data-flow library files
 /java/ql/src/semmle/code/java/dataflow/internal/DataFlowImpl.qll @github/codeql-java @github/codeql-go
 /java/ql/src/semmle/code/java/dataflow/internal/DataFlowImpl2.qll @github/codeql-java @github/codeql-go
@@ -30,4 +27,4 @@
 /docs/query-*-style-guide.md @github/codeql-analysis-reviewers
 
 # QL for QL reviewers
-/ql/ @github/codeql-ql-for-ql-reviewers
+/ql/ @erik-krogh @tausbn

CONTRIBUTING.md

@@ -4,9 +4,6 @@ We welcome contributions to our CodeQL libraries and queries. Got an idea for a
 
 There is lots of useful documentation to help you write queries, ranging from information about query file structure to tutorials for specific target languages. For more information on the documentation available, see [CodeQL queries](https://help.semmle.com/QL/learn-ql/writing-queries/writing-queries.html) on [help.semmle.com](https://help.semmle.com).
 
-## Change notes
-
-Any nontrivial user-visible change to a query pack or library pack should have a change note. For details on how to add a change note for your change, see [this guide](docs/change-notes.md).
 
 ## Submitting a new experimental query
 
@@ -42,11 +39,7 @@ If you have an idea for a query that you would like to share with other CodeQL u
 
     - The queries and libraries must be autoformatted, for example using the "Format Document" command in [CodeQL for Visual Studio Code](https://help.semmle.com/codeql/codeql-for-vscode/procedures/about-codeql-for-vscode.html).
 
-    If you prefer, you can either:
+    If you prefer, you can use this [pre-commit hook](misc/scripts/pre-commit) that automatically checks whether your files are correctly formatted. See the [pre-commit hook installation guide](docs/pre-commit-hook-setup.md) for instructions on how to install the hook.
-    1. install the [pre-commit framework](https://pre-commit.com/) and install the configured hooks on this repo via `pre-commit install`, or
-    2. use this [pre-commit hook](misc/scripts/pre-commit) that automatically checks whether your files are correctly formatted.
-
-    See the [pre-commit hook installation guide](docs/pre-commit-hook-setup.md) for instructions on the two approaches.
 
 4. **Compilation**
 

README.md

@@ -1,11 +1,11 @@
 # CodeQL
 
-This open source repository contains the standard CodeQL libraries and queries that power [GitHub Advanced Security](https://github.com/features/security/code) and the other application security products that [GitHub](https://github.com/features/security/) makes available to its customers worldwide. For the queries, libraries, and extractor that power Go analysis, visit the [CodeQL for Go repository](https://github.com/github/codeql-go).
+This open source repository contains the standard CodeQL libraries and queries that power [LGTM](https://lgtm.com) and the other CodeQL products that [GitHub](https://github.com) makes available to its customers worldwide. For the queries, libraries, and extractor that power Go analysis, visit the [CodeQL for Go repository](https://github.com/github/codeql-go).
 
 ## How do I learn CodeQL and run queries?
 
 There is [extensive documentation](https://codeql.github.com/docs/) on getting started with writing CodeQL.
-You can use the [CodeQL for Visual Studio Code](https://codeql.github.com/docs/codeql-for-visual-studio-code/) extension or the [interactive query console](https://lgtm.com/help/lgtm/using-query-console) on LGTM.com (Semmle Legacy product) to try out your queries on any open source project that's currently being analyzed.
+You can use the [interactive query console](https://lgtm.com/help/lgtm/using-query-console) on LGTM.com or the [CodeQL for Visual Studio Code](https://codeql.github.com/docs/codeql-for-visual-studio-code/) extension to try out your queries on any open source project that's currently being analyzed.
 
 ## Contributing
 
@@ -13,7 +13,7 @@ We welcome contributions to our standard library and standard checks. Do you hav
 
 ## License
 
-The code in this repository is licensed under the [MIT License](LICENSE) by [GitHub](https://github.com). The use of CodeQL on open source code is licensed under specific [Terms & Conditions](https://securitylab.github.com/tools/codeql/license/) UNLESS you have a commercial license in place. If you'd like to use CodeQL with a commercial codebase, please [contact us](https://github.com/enterprise/contact) for further help.
+The code in this repository is licensed under the [MIT License](LICENSE) by [GitHub](https://github.com).
 
 ## Visual Studio Code integration
 

config/identical-files.json

@@ -7,7 +7,6 @@
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl5.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl6.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplForSerializability.qll",
-    "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplForOnActivityResult.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl2.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl3.qll",
@@ -426,6 +425,7 @@
     "python/ql/src/Lexical/CommentedOutCodeMetricOverview.inc.qhelp"
   ],
   "FLinesOfDuplicatedCodeCommon.inc.qhelp": [
+    "cpp/ql/src/Metrics/Files/FLinesOfDuplicatedCodeCommon.inc.qhelp",
     "java/ql/src/Metrics/Files/FLinesOfDuplicatedCodeCommon.inc.qhelp",
     "javascript/ql/src/Metrics/FLinesOfDuplicatedCodeCommon.inc.qhelp",
     "python/ql/src/Metrics/FLinesOfDuplicatedCodeCommon.inc.qhelp"
@@ -452,20 +452,13 @@
     "ruby/ql/lib/codeql/ruby/dataflow/internal/SsaImplCommon.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/SsaImplCommon.qll"
   ],
-  "CryptoAlgorithms Python/JS/Ruby": [
+  "CryptoAlgorithms Python/JS": [
     "javascript/ql/lib/semmle/javascript/security/CryptoAlgorithms.qll",
-    "python/ql/lib/semmle/python/concepts/CryptoAlgorithms.qll",
+    "python/ql/lib/semmle/python/concepts/CryptoAlgorithms.qll"
-    "ruby/ql/lib/codeql/ruby/security/CryptoAlgorithms.qll"
-  ],
-  "CryptoAlgorithmNames Python/JS/Ruby": [
-    "javascript/ql/lib/semmle/javascript/security/internal/CryptoAlgorithmNames.qll",
-    "python/ql/lib/semmle/python/concepts/internal/CryptoAlgorithmNames.qll",
-    "ruby/ql/lib/codeql/ruby/security/internal/CryptoAlgorithmNames.qll"
   ],
   "SensitiveDataHeuristics Python/JS": [
     "javascript/ql/lib/semmle/javascript/security/internal/SensitiveDataHeuristics.qll",
-    "python/ql/lib/semmle/python/security/internal/SensitiveDataHeuristics.qll",
+    "python/ql/lib/semmle/python/security/internal/SensitiveDataHeuristics.qll"
-    "ruby/ql/lib/codeql/ruby/security/internal/SensitiveDataHeuristics.qll"
   ],
   "ReDoS Util Python/JS/Ruby": [
     "javascript/ql/lib/semmle/javascript/security/performance/ReDoSUtil.qll",
@@ -501,11 +494,5 @@
     "javascript/ql/lib/tutorial.qll",
     "python/ql/lib/tutorial.qll",
     "ruby/ql/lib/tutorial.qll"
-  ],
-  "AccessPathSyntax": [
-    "csharp/ql/lib/semmle/code/csharp/dataflow/internal/AccessPathSyntax.qll",
-    "java/ql/lib/semmle/code/java/dataflow/internal/AccessPathSyntax.qll",
-    "javascript/ql/lib/semmle/javascript/frameworks/data/internal/AccessPathSyntax.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/AccessPathSyntax.qll"
   ]
 }

cpp/autobuilder/Semmle.Autobuild.Cpp.Tests/Semmle.Autobuild.Cpp.Tests.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net5.0</TargetFramework>
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
     <RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
     <Nullable>enable</Nullable>

cpp/autobuilder/Semmle.Autobuild.Cpp/Semmle.Autobuild.Cpp.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net5.0</TargetFramework>
     <AssemblyName>Semmle.Autobuild.Cpp</AssemblyName>
     <RootNamespace>Semmle.Autobuild.Cpp</RootNamespace>
     <ApplicationIcon />

cpp/change-notes/2021-11-01-isFromSystemMacroDefinition.md

@@ -0,0 +1,4 @@
+lgtm,codescanning
+* The QL library `semmle.code.cpp.commons.Exclusions` now contains a predicate
+  `isFromSystemMacroDefinition` for identifying code that originates from a
+  macro outside the project being analyzed.

cpp/change-notes/2021-11-09-use-of-http.md

@@ -0,0 +1,2 @@
+lgtm,codescanning
+* A new query `cpp/non-https-url` has been added for C/C++. The query flags uses of `http` URLs that might be better replaced with `https`.

cpp/change-notes/2021-11-25-certificate-not-checked.md

@@ -0,0 +1,2 @@
+lgtm,codescanning
+* A new query `cpp/certificate-not-checked` has been added for C/C++. The query flags unsafe use of OpenSSL and similar libraries.

cpp/change-notes/2021-11-25-certificate-result-conflation.md

@@ -0,0 +1,2 @@
+lgtm,codescanning
+* A new query `cpp/certificate-result-conflation` has been added for C/C++. The query flags unsafe use of OpenSSL and similar libraries.

cpp/config/suites/c/correctness

@@ -31,7 +31,6 @@
 + semmlecode-cpp-queries/Critical/NewArrayDeleteMismatch.ql: /Correctness/Common Errors
 + semmlecode-cpp-queries/Critical/NewDeleteArrayMismatch.ql: /Correctness/Common Errors
 + semmlecode-cpp-queries/Critical/NewFreeMismatch.ql: /Correctness/Common Errors
-+ semmlecode-cpp-queries/Likely Bugs/Memory Management/UsingExpiredStackAddress.ql: /Correctness/Common Errors
   # Use of Libraries
 + semmlecode-cpp-queries/Likely Bugs/Memory Management/SuspiciousCallToMemset.ql: /Correctness/Use of Libraries
 + semmlecode-cpp-queries/Likely Bugs/Memory Management/SuspiciousSizeof.ql: /Correctness/Use of Libraries

cpp/config/suites/cpp/correctness

@@ -34,7 +34,6 @@
 + semmlecode-cpp-queries/Critical/NewArrayDeleteMismatch.ql: /Correctness/Common Errors
 + semmlecode-cpp-queries/Critical/NewDeleteArrayMismatch.ql: /Correctness/Common Errors
 + semmlecode-cpp-queries/Critical/NewFreeMismatch.ql: /Correctness/Common Errors
-+ semmlecode-cpp-queries/Likely Bugs/Memory Management/UsingExpiredStackAddress.ql: /Correctness/Common Errors
   # Exceptions
 + semmlecode-cpp-queries/Best Practices/Exceptions/AccidentalRethrow.ql: /Correctness/Exceptions
 + semmlecode-cpp-queries/Best Practices/Exceptions/CatchingByValue.ql: /Correctness/Exceptions

cpp/config/suites/security/cwe-120

@@ -5,11 +5,9 @@
   @name Badly bounded write (CWE-120)
 + semmlecode-cpp-queries/Security/CWE/CWE-120/OverrunWrite.ql: /CWE/CWE-120
   @name Potentially overrunning write (CWE-120)
-+ semmlecode-cpp-queries/Security/CWE/CWE-120/VeryLikelyOverrunWrite.ql: /CWE/CWE-120
-  @name Likely overrunning write
 + semmlecode-cpp-queries/Security/CWE/CWE-120/OverrunWriteFloat.ql: /CWE/CWE-120
   @name Potentially overrunning write with float to string conversion (CWE-120)
 + semmlecode-cpp-queries/Best Practices/Likely Errors/OffsetUseBeforeRangeCheck.ql: /CWE/CWE-120
   @name Array offset used before range check (CWE-120)
 + semmlecode-cpp-queries/Likely Bugs/Memory Management/UnsafeUseOfStrcat.ql: /CWE/CWE-120
-  @name Potentially unsafe use of strcat (CWE-120)
+    @name Potentially unsafe use of strcat (CWE-120)

cpp/downgrades/2cd420191e5f782589b4e4efb70127de265390dd/upgrade.properties

@@ -1,2 +0,0 @@
-description: Remove unused legacy relations
-compatibility: backwards

cpp/downgrades/bb0f279f2acd793105a347d589b5afc8715d94c4/upgrade.properties

@@ -1,3 +0,0 @@
-description: Add relation for tracking variables from structured binding declarations
-compatibility: full
-is_structured_binding.rel: delete

cpp/downgrades/qlpack.yml

@@ -1,4 +0,0 @@
-name: codeql/cpp-downgrades
-groups: cpp
-downgrades: .
-library: true

cpp/old-change-notes/2021-10-07-cleartext-transmission.md

@@ -1,2 +0,0 @@
-lgtm,codescanning
-* The "Cleartext transmission of sensitive information" (`cpp/cleartext-transmission`) query has been improved, reducing the number of false positive results when encryption is present.

cpp/ql/examples/qlpack.yml

@@ -1,6 +1,4 @@
 name: codeql/cpp-examples
-groups: 
+version: 0.0.2
-  - cpp
-  - examples
 dependencies:
   codeql/cpp-all: "*"

cpp/ql/lib/CHANGELOG.md

@@ -1,35 +1,3 @@
-## 0.0.11
-
-### Minor Analysis Improvements
-
-* Many queries now support structured bindings, as structured bindings are now handled in the IR translation.
-
-## 0.0.10
-
-### New Features
-
-* Added a `isStructuredBinding` predicate to the `Variable` class which holds when the variable is declared as part of a structured binding declaration.
-
-## 0.0.9
-
-
-## 0.0.8
-
-### Deprecated APIs
-
-* The `codeql/cpp-upgrades` CodeQL pack has been removed. All upgrades scripts have been merged into the `codeql/cpp-all` CodeQL pack.
-
-### Minor Analysis Improvements
-
-* `FormatLiteral::getMaxConvertedLength` now uses range analysis to provide a
-  more accurate length for integers formatted with `%x`
-
-## 0.0.7
-
-## 0.0.6
-
-## 0.0.5
-
 ## 0.0.4
 
 ### New Features

cpp/ql/lib/DefaultOptions.qll

@@ -73,7 +73,7 @@ class Options extends string {
    *   __assume(0);
    * ```
    * (note that in this case if the hint is wrong and the expression is reached at
-   * runtime, the program's behavior is undefined)
+   * runtime, the program's behaviour is undefined)
    */
   predicate exprExits(Expr e) {
     e.(AssumeExpr).getChild(0).(CompileTimeConstantInt).getIntValue() = 0 or

cpp/ql/lib/Options.qll

@@ -50,7 +50,7 @@ class CustomOptions extends Options {
    *   __assume(0);
    * ```
    * (note that in this case if the hint is wrong and the expression is reached at
-   * runtime, the program's behavior is undefined)
+   * runtime, the program's behaviour is undefined)
    */
   override predicate exprExits(Expr e) { Options.super.exprExits(e) }
 

cpp/ql/lib/change-notes/released/0.0.10.md

@@ -1,5 +0,0 @@
-## 0.0.10
-
-### New Features
-
-* Added a `isStructuredBinding` predicate to the `Variable` class which holds when the variable is declared as part of a structured binding declaration.

cpp/ql/lib/change-notes/released/0.0.11.md

@@ -1,5 +0,0 @@
-## 0.0.11
-
-### Minor Analysis Improvements
-
-* Many queries now support structured bindings, as structured bindings are now handled in the IR translation.

cpp/ql/lib/change-notes/released/0.0.6.md

@@ -1 +0,0 @@
-## 0.0.6

cpp/ql/lib/change-notes/released/0.0.7.md

@@ -1 +0,0 @@
-## 0.0.7

cpp/ql/lib/change-notes/released/0.0.8.md

@@ -1,10 +0,0 @@
-## 0.0.8
-
-### Deprecated APIs
-
-* The `codeql/cpp-upgrades` CodeQL pack has been removed. All upgrades scripts have been merged into the `codeql/cpp-all` CodeQL pack.
-
-### Minor Analysis Improvements
-
-* `FormatLiteral::getMaxConvertedLength` now uses range analysis to provide a
-  more accurate length for integers formatted with `%x`

cpp/ql/lib/change-notes/released/0.0.9.md

@@ -1,2 +0,0 @@
-## 0.0.9
-

cpp/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.0.11
+lastReleaseVersion: 0.0.4

cpp/ql/lib/experimental/semmle/code/cpp/models/interfaces/SimpleRangeAnalysisDefinition.qll

@@ -37,7 +37,7 @@ abstract class SimpleRangeAnalysisDefinition extends RangeSsaDefinition {
    * dependencies. Without this information, range analysis might work for
    * simple cases but will go into infinite loops on complex code.
    *
-   * For example, when modeling the definition by reference in a call to an
+   * For example, when modelling the definition by reference in a call to an
    * overloaded `operator=`, written as `v = e`, the definition of `(this, v)`
    * depends on `e`.
    */

cpp/ql/lib/experimental/semmle/code/cpp/rangeanalysis/InBoundsPointerDeref.qll

@@ -5,7 +5,7 @@
  * `Instruction` level), and then using the array length analysis and the range
  * analysis together to prove that some of these pointer dereferences are safe.
  *
- * The analysis is soundy, i.e. it is sound if no undefined behavior is present
+ * The analysis is soundy, i.e. it is sound if no undefined behaviour is present
  * in the program.
  * Furthermore, it crucially depends on the soundiness of the range analysis and
  * the array length analysis.

cpp/ql/lib/qlpack.yml

@@ -1,7 +1,8 @@
 name: codeql/cpp-all
-version: 0.0.11
+version: 0.0.5-dev
 groups: cpp
 dbscheme: semmlecode.cpp.dbscheme
 extractor: cpp
 library: true
-upgrades: upgrades
+dependencies:
+  codeql/cpp-upgrades: ^0.0.3