C++: Set stack_referenced_include_directories on some tests

Ruby: Bump tree-sitter grammar

.bazelrc

@@ -10,16 +10,19 @@ common --override_module=semmle_code=%workspace%/misc/bazel/semmle_code_stub
 
 build --repo_env=CC=clang --repo_env=CXX=clang++
 
-build:linux --cxxopt=-std=c++20
-# we currently cannot built the swift extractor for ARM
-build:macos --cxxopt=-std=c++20 --copt=-arch --copt=x86_64 --linkopt=-arch --linkopt=x86_64
-build:windows --cxxopt=/std:c++20 --cxxopt=/Zc:preprocessor
+# we use transitions that break builds of `...`, so for `test` to work with that we need the following
+test --build_tests_only
 
 # this requires developer mode, but is required to have pack installer functioning
 startup --windows_enable_symlinks
 common --enable_runfiles
 
+# with the above, we can avoid building python zips which is the default on windows as that's expensive
+build --nobuild_python_zip
+
 common --registry=file:///%workspace%/misc/bazel/registry
 common --registry=https://bcr.bazel.build
 
+common --@rules_dotnet//dotnet/settings:strict_deps=false
+
 try-import %workspace%/local.bazelrc

.bazelrc.internal

@@ -2,3 +2,9 @@
 
 common --registry=file:///%workspace%/ql/misc/bazel/registry
 common --registry=https://bcr.bazel.build
+
+# See bazelbuild/rules_dotnet#413: strict_deps in C# also appliy to 3rd-party deps, and when we pull
+# in (for example) the xunit package, there's no code in this at all, it just depends transitively on
+# its implementation packages without providing any code itself.
+# We either can depend on internal implementation details, or turn of strict deps.
+common --@rules_dotnet//dotnet/settings:strict_deps=false

.gitattributes

@@ -50,31 +50,40 @@
 *.dll -text
 *.pdb -text
 
-java/ql/test/stubs/**/*.java linguist-generated=true
-java/ql/test/experimental/stubs/**/*.java linguist-generated=true
+/java/ql/test/stubs/**/*.java linguist-generated=true
+/java/ql/test/experimental/stubs/**/*.java linguist-generated=true
+/java/kotlin-extractor/deps/*.jar filter=lfs diff=lfs merge=lfs -text
 
 # Force git not to modify line endings for go or html files under the go/ql directory
-go/ql/**/*.go -text
-go/ql/**/*.html -text
+/go/ql/**/*.go -text
+/go/ql/**/*.html -text
 # Force git not to modify line endings for go dbschemes
-go/*.dbscheme -text
+/go/*.dbscheme -text
 # Preserve unusual line ending from codeql-go merge
-go/extractor/opencsv/CSVReader.java -text
+/go/extractor/opencsv/CSVReader.java -text
 
 # For some languages, upgrade script testing references really old dbscheme
 # files from legacy upgrades that have CRLF line endings. Since upgrade
 # resolution relies on object hashes, we must suppress line ending conversion
 # for those testing dbscheme files.
-*/ql/lib/upgrades/initial/*.dbscheme -text
+/*/ql/lib/upgrades/initial/*.dbscheme -text
 
 # Auto-generated modeling for Python
-python/ql/lib/semmle/python/frameworks/data/internal/subclass-capture/*.yml linguist-generated=true
+/python/ql/lib/semmle/python/frameworks/data/internal/subclass-capture/*.yml linguist-generated=true
 
 # auto-generated bazel lock file
-ruby/extractor/cargo-bazel-lock.json linguist-generated=true
-ruby/extractor/cargo-bazel-lock.json -merge
+/ruby/extractor/cargo-bazel-lock.json linguist-generated=true
+/ruby/extractor/cargo-bazel-lock.json -merge
 
 # auto-generated files for the C# build
-csharp/paket.lock linguist-generated=true
-# needs eol=crlf, as `paket` touches this file and saves it als crlf
-csharp/.paket/Paket.Restore.targets linguist-generated=true eol=crlf
+/csharp/paket.lock linguist-generated=true
+# needs eol=crlf, as `paket` touches this file and saves it as crlf
+/csharp/.paket/Paket.Restore.targets linguist-generated=true eol=crlf
+/csharp/paket.main.bzl linguist-generated=true
+/csharp/paket.main_extension.bzl linguist-generated=true
+
+# ripunzip tool
+/misc/ripunzip/ripunzip-* filter=lfs diff=lfs merge=lfs -text
+
+# swift prebuilt resources
+/swift/third_party/resource-dir/*.zip filter=lfs diff=lfs merge=lfs -text

.github/workflows/build-ripunzip.yml

@@ -0,0 +1,74 @@
+name: Build runzip
+
+on:
+  workflow_dispatch:
+    inputs:
+      ripunzip-version:
+        description: "what reference to checktout from google/runzip"
+        required: false
+        default: v1.2.1
+      openssl-version:
+        description: "what reference to checkout from openssl/openssl for Linux"
+        required: false
+        default: openssl-3.3.0
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-20.04, macos-12, windows-2019]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          repository: google/ripunzip
+          ref: ${{ inputs.ripunzip-version }}
+      # we need to avoid ripunzip dynamically linking into libssl
+      # see https://github.com/sfackler/rust-openssl/issues/183
+      - if: runner.os == 'Linux'
+        name: checkout openssl
+        uses: actions/checkout@v4
+        with:
+          repository: openssl/openssl
+          path: openssl
+          ref: ${{ inputs.openssl-version }}
+      - if: runner.os == 'Linux'
+        name: build and install openssl with fPIC
+        shell: bash
+        working-directory: openssl
+        run: |
+          ./config -fPIC --prefix=$HOME/.local --openssldir=$HOME/.local/ssl
+          make -j $(nproc)
+          make install_sw -j $(nproc)
+      - if: runner.os == 'Linux'
+        name: build (linux)
+        shell: bash
+        run: |
+          env OPENSSL_LIB_DIR=$HOME/.local/lib64 OPENSSL_INCLUDE_DIR=$HOME/.local/include OPENSSL_STATIC=yes cargo build --release
+          mv target/release/ripunzip ripunzip-linux
+      - if: runner.os == 'Windows'
+        name: build (windows)
+        shell: bash
+        run: |
+          cargo build --release
+          mv target/release/ripunzip ripunzip-windows
+      - name: build (macOS)
+        if: runner.os == 'macOS'
+        shell: bash
+        run: |
+          rustup target install x86_64-apple-darwin
+          rustup target install aarch64-apple-darwin
+          cargo build --target x86_64-apple-darwin --release
+          cargo build --target aarch64-apple-darwin --release
+          lipo -create -output ripunzip-macos \
+            -arch x86_64 target/x86_64-apple-darwin/release/ripunzip \
+            -arch arm64 target/aarch64-apple-darwin/release/ripunzip
+      - uses: actions/upload-artifact@v4
+        with:
+          name: ripunzip-${{ runner.os }}
+          path: ripunzip-*
+      - name: Check built binary
+        shell: bash
+        run: |
+          ./ripunzip-* --version

.github/workflows/csharp-qltest.yml

@@ -65,7 +65,7 @@ jobs:
           key: csharp-qltest-${{ matrix.slice }}
       - name: Run QL tests
         run: |
-          codeql test run --threads=0 --ram 50000 --slice ${{ matrix.slice }} --search-path extractor-pack --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+          codeql test run --threads=0 --ram 50000 --slice ${{ matrix.slice }} --search-path "${{ github.workspace }}" --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
         env:
           GITHUB_TOKEN: ${{ github.token }}
   unit-tests:
@@ -101,6 +101,6 @@ jobs:
           # Update existing stubs in the repo with the freshly generated ones
           mv "$STUBS_PATH/output/stubs/_frameworks" ql/test/resources/stubs/
           git status
-          codeql test run --threads=0 --search-path extractor-pack --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries -- ql/test/library-tests/dataflow/flowsources/aspremote
+          codeql test run --threads=0 --search-path "${{ github.workspace }}" --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries -- ql/test/library-tests/dataflow/flowsources/aspremote
         env:
           GITHUB_TOKEN: ${{ github.token }}

.github/workflows/go-tests-other-os.yml

@@ -7,6 +7,9 @@ on:
       - .github/workflows/go-tests-other-os.yml
       - .github/actions/**
       - codeql-workspace.yml
+      - MODULE.bazel
+      - .bazelrc
+      - misc/bazel/**
 
 permissions:
   contents: read

.github/workflows/go-tests.yml

@@ -15,6 +15,9 @@ on:
       - .github/workflows/go-tests.yml
       - .github/actions/**
       - codeql-workspace.yml
+      - MODULE.bazel
+      - .bazelrc
+      - misc/bazel/**
 
 permissions:
   contents: read

.github/workflows/kotlin-build.yml

@@ -0,0 +1,28 @@
+name: "Kotlin Build"
+
+on:
+  pull_request:
+    paths:
+      - "java/kotlin-extractor/**"
+      - "misc/bazel/**"
+      - "misc/codegen/**"
+      - "*.bazel*"
+      - .github/workflows/kotlin-build.yml
+    branches:
+      - main
+      - rc/*
+      - codeql-cli-*
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: |
+          bazel query //java/kotlin-extractor/...
+          # only build the default version as a quick check that we can build from `codeql`
+          # the full official build will be checked by QLucie
+          bazel build //java/kotlin-extractor

.github/workflows/ql-for-ql-build.yml

@@ -49,20 +49,20 @@ jobs:
           key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-rust-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
       - name: Release build
         if: steps.cache-extractor.outputs.cache-hit != 'true'
-        run: cd ql; ./scripts/create-extractor-pack.sh       
+        run: cd ql; ./scripts/create-extractor-pack.sh
         env:
-          GH_TOKEN: ${{ github.token }}   
+          GH_TOKEN: ${{ github.token }}
       - name: Cache compilation cache
         id: query-cache
         uses: ./.github/actions/cache-query-compilation
-        with: 
+        with:
           key: run-ql-for-ql
       - name: Make database and analyze
         run: |
           ./ql/target/release/buramu | tee deprecated.blame # Add a blame file for the extractor to parse.
-          ${CODEQL} database create -l=ql --search-path ql/extractor-pack ${DB}
+          ${CODEQL} database create -l=ql ${DB} --search-path "${{ github.workspace }}"
           ${CODEQL} database analyze -j0 --format=sarif-latest --output=ql-for-ql.sarif ${DB} ql/ql/src/codeql-suites/ql-code-scanning.qls  --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
-        env: 
+        env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
           DB: ${{ runner.temp }}/DB
           LGTM_INDEX_FILTERS: |

.github/workflows/ql-for-ql-dataset_measure.yml

@@ -53,8 +53,8 @@ jobs:
       - name: Create database
         run: |
           "${CODEQL}" database create \
-            --search-path "ql/extractor-pack" \
-            --threads 4 \
+          --search-path "${{ github.workspace }}"
+          --threads 4 \
             --language ql --source-root "${{ github.workspace }}/repo" \
             "${{ runner.temp }}/database"
         env:

.github/workflows/ql-for-ql-tests.yml

@@ -49,15 +49,15 @@ jobs:
       - name: Cache compilation cache
         id: query-cache
         uses: ./.github/actions/cache-query-compilation
-        with: 
+        with:
           key: ql-for-ql-tests
       - name: Run QL tests
         run: |
-          "${CODEQL}" test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}/ql/extractor-pack" --consistency-queries ql/ql/consistency-queries --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" ql/ql/test
+          "${CODEQL}" test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}" --consistency-queries ql/ql/consistency-queries --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" ql/ql/test
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
 
-  other-os: 
+  other-os:
     strategy:
       matrix:
         os: [macos-latest, windows-latest]
@@ -65,7 +65,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
-      - name: Install GNU tar 
+      - name: Install GNU tar
         if: runner.os == 'macOS'
         run: |
           brew install gnu-tar
@@ -100,7 +100,7 @@ jobs:
       - name: Run a single QL tests - Unix
         if: runner.os != 'Windows'
         run: |
-          "${CODEQL}" test run --check-databases --search-path "${{ github.workspace }}/ql/extractor-pack" ql/ql/test/queries/style/DeadCode/DeadCode.qlref
+          "${CODEQL}" test run --check-databases --search-path "${{ github.workspace }}" ql/ql/test/queries/style/DeadCode/DeadCode.qlref
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
       - name: Run a single QL tests - Windows
@@ -108,5 +108,4 @@ jobs:
         shell: pwsh
         run: |
           $Env:PATH += ";$(dirname ${{ steps.find-codeql.outputs.codeql-path }})"
-          codeql test run --check-databases --search-path "${{ github.workspace }}/ql/extractor-pack" ql/ql/test/queries/style/DeadCode/DeadCode.qlref
-      
+          codeql test run --check-databases --search-path "${{ github.workspace }}" ql/ql/test/queries/style/DeadCode/DeadCode.qlref

.github/workflows/ruby-dataset-measure.yml

@@ -44,7 +44,7 @@ jobs:
       - name: Create database
         run: |
           codeql database create \
-            --search-path "${{ github.workspace }}/ruby/extractor-pack" \
+            --search-path "${{ github.workspace }}" \
             --threads 4 \
             --language ruby --source-root "${{ github.workspace }}/repo" \
             "${{ runner.temp }}/database"

.github/workflows/ruby-qltest.yml

@@ -64,10 +64,10 @@ jobs:
       - name: Cache compilation cache
         id: query-cache
         uses: ./.github/actions/cache-query-compilation
-        with: 
+        with:
           key: ruby-qltest
       - name: Run QL tests
         run: |
-          codeql test run --threads=0 --ram 50000 --search-path "${{ github.workspace }}/ruby/extractor-pack" --check-databases --check-undefined-labels --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+          codeql test run --threads=0 --ram 50000 --search-path "${{ github.workspace }}" --check-databases --check-undefined-labels --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
         env:
           GITHUB_TOKEN: ${{ github.token }}

.github/workflows/swift.yml

@@ -68,21 +68,6 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: ./swift/actions/run-ql-tests
-  integration-tests-linux:
-    if: github.repository_owner == 'github'
-    needs: build-and-test-linux
-    runs-on: ubuntu-latest-xl
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./swift/actions/run-integration-tests
-  integration-tests-macos:
-    if: ${{ github.repository_owner == 'github' && github.event_name == 'pull_request' }}
-    needs: build-and-test-macos
-    runs-on: macos-12-xl
-    timeout-minutes: 60
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./swift/actions/run-integration-tests
   clang-format:
     if : ${{ github.event_name == 'pull_request' }}
     runs-on: ubuntu-latest

.github/workflows/zipmerge-test.yml

@@ -0,0 +1,23 @@
+name: "Test zipmerge code"
+
+on:
+  pull_request:
+    paths:
+      - "misc/bazel/internal/zipmerge/**"
+      - "MODULE.bazel"
+      - ".bazelrc*"
+    branches:
+      - main
+      - "rc/*"
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - run: |
+          bazel test //misc/bazel/internal/zipmerge:test --test_output=all

.gitignore

@@ -62,3 +62,6 @@ node_modules/
 
 # Temporary folders for working with generated models
 .model-temp
+
+# bazel-built in-tree extractor packs
+/*/extractor-pack

.lfsconfig

@@ -2,4 +2,6 @@
 # codeql is publicly forked by many users, and we don't want any LFS file polluting their working
 # copies. We therefore exclude everything by default.
 # For files required by bazel builds, use rules in `misc/bazel/lfs.bzl` to download them on demand.
+# we go for `fetchinclude` to something not exsiting rather than `fetchexclude = *` because the
+# former is easier to override (with `git -c` or a local git config) to fetch something specific
 fetchinclude = /nothing

BUILD.bazel

@@ -0,0 +1 @@
+exports_files(["LICENSE"])

MODULE.bazel

@@ -22,10 +22,22 @@ bazel_dep(name = "bazel_skylib", version = "1.5.0")
 bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "absl")
 bazel_dep(name = "nlohmann_json", version = "3.11.3", repo_name = "json")
 bazel_dep(name = "fmt", version = "10.0.0")
+bazel_dep(name = "rules_kotlin", version = "1.9.4-codeql.1")
 bazel_dep(name = "gazelle", version = "0.36.0")
+bazel_dep(name = "rules_dotnet", version = "0.15.1")
+bazel_dep(name = "googletest", version = "1.14.0.bcr.1")
 
 bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True)
 
+dotnet = use_extension("@rules_dotnet//dotnet:extensions.bzl", "dotnet")
+dotnet.toolchain(dotnet_version = "8.0.101")
+use_repo(dotnet, "dotnet_toolchains")
+
+register_toolchains("@dotnet_toolchains//:all")
+
+csharp_main_extension = use_extension("//csharp:paket.main_extension.bzl", "main_extension")
+use_repo(csharp_main_extension, "paket.main")
+
 pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
 pip.parse(
     hub_name = "codegen_deps",
@@ -54,9 +66,84 @@ node.toolchain(
 )
 use_repo(node, "nodejs", "nodejs_toolchains")
 
+kotlin_extractor_deps = use_extension("//java/kotlin-extractor:deps.bzl", "kotlin_extractor_deps")
+
+# following list can be kept in sync by running `bazel mod tidy` in `codeql`
+use_repo(
+    kotlin_extractor_deps,
+    "codeql_kotlin_defaults",
+    "codeql_kotlin_embeddable",
+    "kotlin-compiler-1.5.0",
+    "kotlin-compiler-1.5.10",
+    "kotlin-compiler-1.5.20",
+    "kotlin-compiler-1.5.30",
+    "kotlin-compiler-1.6.0",
+    "kotlin-compiler-1.6.20",
+    "kotlin-compiler-1.7.0",
+    "kotlin-compiler-1.7.20",
+    "kotlin-compiler-1.8.0",
+    "kotlin-compiler-1.9.0-Beta",
+    "kotlin-compiler-1.9.20-Beta",
+    "kotlin-compiler-2.0.0-RC1",
+    "kotlin-compiler-embeddable-1.5.0",
+    "kotlin-compiler-embeddable-1.5.10",
+    "kotlin-compiler-embeddable-1.5.20",
+    "kotlin-compiler-embeddable-1.5.30",
+    "kotlin-compiler-embeddable-1.6.0",
+    "kotlin-compiler-embeddable-1.6.20",
+    "kotlin-compiler-embeddable-1.7.0",
+    "kotlin-compiler-embeddable-1.7.20",
+    "kotlin-compiler-embeddable-1.8.0",
+    "kotlin-compiler-embeddable-1.9.0-Beta",
+    "kotlin-compiler-embeddable-1.9.20-Beta",
+    "kotlin-compiler-embeddable-2.0.0-RC1",
+    "kotlin-stdlib-1.5.0",
+    "kotlin-stdlib-1.5.10",
+    "kotlin-stdlib-1.5.20",
+    "kotlin-stdlib-1.5.30",
+    "kotlin-stdlib-1.6.0",
+    "kotlin-stdlib-1.6.20",
+    "kotlin-stdlib-1.7.0",
+    "kotlin-stdlib-1.7.20",
+    "kotlin-stdlib-1.8.0",
+    "kotlin-stdlib-1.9.0-Beta",
+    "kotlin-stdlib-1.9.20-Beta",
+    "kotlin-stdlib-2.0.0-RC1",
+)
+
 go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk")
 go_sdk.download(version = "1.22.2")
 
+lfs_files = use_repo_rule("//misc/bazel:lfs.bzl", "lfs_files")
+
+lfs_files(
+    name = "ripunzip-linux",
+    srcs = ["//misc/ripunzip:ripunzip-linux"],
+    executable = True,
+)
+
+lfs_files(
+    name = "ripunzip-windows",
+    srcs = ["//misc/ripunzip:ripunzip-windows.exe"],
+    executable = True,
+)
+
+lfs_files(
+    name = "ripunzip-macos",
+    srcs = ["//misc/ripunzip:ripunzip-macos"],
+    executable = True,
+)
+
+lfs_files(
+    name = "swift-resource-dir-linux",
+    srcs = ["//swift/third_party/resource-dir:resource-dir-linux.zip"],
+)
+
+lfs_files(
+    name = "swift-resource-dir-macos",
+    srcs = ["//swift/third_party/resource-dir:resource-dir-macos.zip"],
+)
+
 register_toolchains(
     "@nodejs_toolchains//:all",
 )

codeql-workspace.yml

@@ -6,19 +6,16 @@ provide:
   - "*/ql/consistency-queries/qlpack.yml"
   - "*/ql/automodel/src/qlpack.yml"
   - "*/ql/automodel/test/qlpack.yml"
+  - "*/extractor-pack/codeql-extractor.yml"
   - "python/extractor/qlpack.yml"
   - "shared/**/qlpack.yml"
   - "cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml"
   - "go/ql/config/legacy-support/qlpack.yml"
-  - "go/build/codeql-extractor-go/codeql-extractor.yml"
   - "csharp/ql/campaigns/Solorigate/lib/qlpack.yml"
   - "csharp/ql/campaigns/Solorigate/src/qlpack.yml"
   - "csharp/ql/campaigns/Solorigate/test/qlpack.yml"
   - "misc/legacy-support/*/qlpack.yml"
   - "misc/suite-helpers/qlpack.yml"
-  - "ruby/extractor-pack/codeql-extractor.yml"
-  - "swift/extractor-pack/codeql-extractor.yml"
-  - "ql/extractor-pack/codeql-extractor.yml"
   - ".github/codeql/extensions/**/codeql-pack.yml"
 
 versionPolicies:

config/dbscheme-fragments.json

@@ -28,6 +28,7 @@
     "/*- Yaml dbscheme -*/",
     "/*- Blame dbscheme -*/",
     "/*- JSON dbscheme -*/",
-    "/*- Python dbscheme -*/"
+    "/*- Python dbscheme -*/",
+    "/*- Empty location -*/"
   ]
 }

config/identical-files.json

@@ -364,5 +364,9 @@
   "Python model summaries test extension": [
     "python/ql/test/library-tests/dataflow/model-summaries/InlineTaintTest.ext.yml",
     "python/ql/test/library-tests/dataflow/model-summaries/NormalDataflowTest.ext.yml"
+  ],
+  "shared tree-sitter extractor cargo.toml": [
+    "shared/tree-sitter-extractor/Cargo.toml",
+    "ruby/extractor/codeql-extractor-fake-crate/Cargo.toml"
   ]
 }

cpp/downgrades/BUILD.bazel

@@ -6,7 +6,7 @@ pkg_files(
         ["**"],
         exclude = ["BUILD.bazel"],
     ),
-    prefix = "cpp/downgrades",
+    prefix = "downgrades",
     strip_prefix = strip_prefix.from_pkg(),
     visibility = ["//cpp:__pkg__"],
 )

cpp/ql/lib/BUILD.bazel

@@ -5,11 +5,9 @@ package(default_visibility = ["//cpp:__pkg__"])
 pkg_files(
     name = "dbscheme",
     srcs = ["semmlecode.cpp.dbscheme"],
-    prefix = "cpp",
 )
 
 pkg_files(
     name = "dbscheme-stats",
     srcs = ["semmlecode.cpp.dbscheme.stats"],
-    prefix = "cpp",
 )

cpp/ql/lib/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 1.0.0
+
+### Breaking Changes
+
+* CodeQL package management is now generally available, and all GitHub-produced CodeQL packages have had their version numbers increased to 1.0.0.
+
 ## 0.13.1
 
 No user-facing changes.

cpp/ql/lib/change-notes/2024-06-14-boost-asio.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* A partial model for the `Boost.Asio` network library has been added. This includes sources, sinks and summaries for certain functions in `Boost.Asio`, such as `read_until` and `write`.

cpp/ql/lib/change-notes/2024-06-14-models-as-data-yml-extensions.md

@@ -0,0 +1,4 @@
+---
+category: feature
+---
+* Data models can now be added with data extensions. In this way source, sink and summary models can be added in extension `.model.yml` files, rather than by writing classes in QL code. New models should be added in the `lib/ext` folder.

cpp/ql/lib/change-notes/released/1.0.0.md

@@ -0,0 +1,5 @@
+## 1.0.0
+
+### Breaking Changes
+
+* CodeQL package management is now generally available, and all GitHub-produced CodeQL packages have had their version numbers increased to 1.0.0.

cpp/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.13.1
+lastReleaseVersion: 1.0.0

cpp/ql/lib/ext/Boost.Asio.model.yml

@@ -0,0 +1,26 @@
+extensions:
+  # partial model of the Boost::Asio network library
+extensions:
+  - addsTo:
+      pack: codeql/cpp-all
+      extensible: sourceModel
+    data: # namespace, type, subtypes, name, signature, ext, output, kind, provenance
+      - ["boost::asio", "", False, "read", "", "", "Argument[*1]", "remote", "manual"]
+      - ["boost::asio", "", False, "read_at", "", "", "Argument[*2]", "remote", "manual"]
+      - ["boost::asio", "", False, "read_until", "", "", "Argument[*1]", "remote", "manual"]
+      - ["boost::asio", "", False, "async_read", "", "", "Argument[*1]", "remote", "manual"]
+      - ["boost::asio", "", False, "async_read_at", "", "", "Argument[*2]", "remote", "manual"]
+      - ["boost::asio", "", False, "async_read_until", "", "", "Argument[*1]", "remote", "manual"]
+  - addsTo:
+      pack: codeql/cpp-all
+      extensible: sinkModel
+    data: # namespace, type, subtypes, name, signature, ext, input, kind, provenance
+      - ["boost::asio", "", False, "write", "", "", "Argument[*1]", "remote-sink", "manual"]
+      - ["boost::asio", "", False, "write_at", "", "", "Argument[*2]", "remote-sink", "manual"]
+      - ["boost::asio", "", False, "async_write", "", "", "Argument[*1]", "remote-sink", "manual"]
+      - ["boost::asio", "", False, "async_write_at", "", "", "Argument[*2]", "remote-sink", "manual"]
+  - addsTo:
+      pack: codeql/cpp-all
+      extensible: summaryModel
+    data: # namespace, type, subtypes, name, signature, ext, input, output, kind, provenance
+      - ["boost::asio", "", False, "buffer", "", "", "Argument[*0]", "ReturnValue", "taint", "manual"]

cpp/ql/lib/ext/empty.model.yml

@@ -0,0 +1,15 @@
+extensions:
+  # Make sure that the extensible model predicates have at least one definition
+  # to avoid errors about undefined extensionals.
+  - addsTo:
+      pack: codeql/cpp-all
+      extensible: sourceModel
+    data: []
+  - addsTo:
+      pack: codeql/cpp-all
+      extensible: sinkModel
+    data: []
+  - addsTo:
+      pack: codeql/cpp-all
+      extensible: summaryModel
+    data: []

cpp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-all
-version: 0.13.2-dev
+version: 1.0.1-dev
 groups: cpp
 dbscheme: semmlecode.cpp.dbscheme
 extractor: cpp
@@ -14,4 +14,6 @@ dependencies:
   codeql/tutorial: ${workspace}
   codeql/util: ${workspace}
   codeql/xml: ${workspace}
+dataExtensions:
+  - ext/*.model.yml
 warnOnImplicitThis: true

cpp/ql/lib/semmle/code/cpp/dataflow/ExternalFlow.qll

@@ -78,6 +78,7 @@ private import internal.FlowSummaryImpl
 private import internal.FlowSummaryImpl::Public
 private import internal.FlowSummaryImpl::Private
 private import internal.FlowSummaryImpl::Private::External
+private import internal.ExternalFlowExtensions as Extensions
 private import codeql.mad.ModelValidation as SharedModelVal
 private import codeql.util.Unit
 
@@ -138,6 +139,9 @@ predicate sourceModel(
     row.splitAt(";", 7) = kind
   ) and
   provenance = "manual"
+  or
+  Extensions::sourceModel(namespace, type, subtypes, name, signature, ext, output, kind, provenance,
+    _)
 }
 
 /** Holds if a sink model exists for the given parameters. */
@@ -158,6 +162,8 @@ predicate sinkModel(
     row.splitAt(";", 7) = kind
   ) and
   provenance = "manual"
+  or
+  Extensions::sinkModel(namespace, type, subtypes, name, signature, ext, input, kind, provenance, _)
 }
 
 /** Holds if a summary model exists for the given parameters. */
@@ -179,6 +185,9 @@ predicate summaryModel(
     row.splitAt(";", 8) = kind
   ) and
   provenance = "manual"
+  or
+  Extensions::summaryModel(namespace, type, subtypes, name, signature, ext, input, output, kind,
+    provenance, _)
 }
 
 private predicate relevantNamespace(string namespace) {
@@ -203,8 +212,10 @@ private predicate canonicalNamespaceLink(string namespace, string subns) {
 }
 
 /**
- * Holds if CSV framework coverage of `namespace` is `n` api endpoints of the
- * kind `(kind, part)`.
+ * Holds if MaD framework coverage of `namespace` is `n` api endpoints of the
+ * kind `(kind, part)`, and `namespaces` is the number of subnamespaces of
+ * `namespace` which have MaD framework coverage (including `namespace`
+ * itself).
  */
 predicate modelCoverage(string namespace, int namespaces, string kind, string part, int n) {
   namespaces = strictcount(string subns | canonicalNamespaceLink(namespace, subns)) and
@@ -321,10 +332,10 @@ module CsvValidation {
       or
       summaryModel(namespace, type, _, name, signature, ext, _, _, _, _) and pred = "summary"
     |
-      not namespace.regexpMatch("[a-zA-Z0-9_\\.]+") and
+      not namespace.regexpMatch("[a-zA-Z0-9_\\.:]*") and
       result = "Dubious namespace \"" + namespace + "\" in " + pred + " model."
       or
-      not type.regexpMatch("[a-zA-Z0-9_<>,\\+]+") and
+      not type.regexpMatch("[a-zA-Z0-9_<>,\\+]*") and
       result = "Dubious type \"" + type + "\" in " + pred + " model."
       or
       not name.regexpMatch("[a-zA-Z0-9_<>,]*") and

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowDispatch.qll

@@ -9,7 +9,7 @@ private import DataFlowUtil
 /**
  * Gets a function that might be called by `call`.
  */
-Function viableCallable(DataFlowCall call) {
+DataFlowCallable viableCallable(DataFlowCall call) {
   result = call.(Call).getTarget()
   or
   // If the target of the call does not have a body in the snapshot, it might

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowPrivate.qll

@@ -242,7 +242,17 @@ class CastNode extends Node {
   CastNode() { none() } // stub implementation
 }
 
-class DataFlowCallable = Function;
+class DataFlowCallable extends Function {
+  /** Gets a best-effort total ordering. */
+  int totalorder() {
+    this =
+      rank[result](DataFlowCallable c, string file, int startline, int startcolumn |
+        c.getLocation().hasLocationInfo(file, startline, startcolumn, _, _)
+      |
+        c order by file, startline, startcolumn
+      )
+  }
+}
 
 class DataFlowExpr = Expr;
 
@@ -261,10 +271,28 @@ class DataFlowCall extends Expr instanceof Call {
   ExprNode getNode() { result.getExpr() = this }
 
   /** Gets the enclosing callable of this call. */
-  Function getEnclosingCallable() { result = this.getEnclosingFunction() }
+  DataFlowCallable getEnclosingCallable() { result = this.getEnclosingFunction() }
+
+  /** Gets a best-effort total ordering. */
+  int totalorder() {
+    this =
+      rank[result](DataFlowCall c, int startline, int startcolumn |
+        c.getLocation().hasLocationInfo(_, startline, startcolumn, _, _)
+      |
+        c order by startline, startcolumn
+      )
+  }
 }
 
-predicate isUnreachableInCall(Node n, DataFlowCall call) { none() } // stub implementation
+class NodeRegion instanceof Unit {
+  string toString() { result = "NodeRegion" }
+
+  predicate contains(Node n) { none() }
+
+  int totalOrder() { result = 1 }
+}
+
+predicate isUnreachableInCall(NodeRegion nr, DataFlowCall call) { none() } // stub implementation
 
 /**
  * Holds if access paths with `c` at their head always should be tracked at high

cpp/ql/lib/semmle/code/cpp/dataflow/internal/ExternalFlowExtensions.qll

@@ -0,0 +1,27 @@
+/**
+ * This module provides extensible predicates for defining MaD models.
+ */
+
+/**
+ * Holds if an external source model exists for the given parameters.
+ */
+extensible predicate sourceModel(
+  string namespace, string type, boolean subtypes, string name, string signature, string ext,
+  string output, string kind, string provenance, QlBuiltins::ExtensionId madId
+);
+
+/**
+ * Holds if an external sink model exists for the given parameters.
+ */
+extensible predicate sinkModel(
+  string namespace, string type, boolean subtypes, string name, string signature, string ext,
+  string input, string kind, string provenance, QlBuiltins::ExtensionId madId
+);
+
+/**
+ * Holds if an external summary model exists for the given parameters.
+ */
+extensible predicate summaryModel(
+  string namespace, string type, boolean subtypes, string name, string signature, string ext,
+  string input, string output, string kind, string provenance, QlBuiltins::ExtensionId madId
+);

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowPrivate.qll

@@ -1062,6 +1062,16 @@ class DataFlowCallable extends TDataFlowCallable {
     result = this.asSummarizedCallable() or // SummarizedCallable = Function (in CPP)
     result = this.asSourceCallable()
   }
+
+  /** Gets a best-effort total ordering. */
+  int totalorder() {
+    this =
+      rank[result](DataFlowCallable c, string file, int startline, int startcolumn |
+        c.getLocation().hasLocationInfo(file, startline, startcolumn, _, _)
+      |
+        c order by file, startline, startcolumn
+      )
+  }
 }
 
 /**
@@ -1159,6 +1169,16 @@ class DataFlowCall extends TDataFlowCall {
    * Gets the location of this call.
    */
   Location getLocation() { none() }
+
+  /** Gets a best-effort total ordering. */
+  int totalorder() {
+    this =
+      rank[result](DataFlowCall c, int startline, int startcolumn |
+        c.getLocation().hasLocationInfo(_, startline, startcolumn, _, _)
+      |
+        c order by startline, startcolumn
+      )
+  }
 }
 
 /**
@@ -1247,43 +1267,53 @@ module IsUnreachableInCall {
     any(G::IRGuardCondition guard).ensuresLt(left, right, k, block, areEqual)
   }
 
-  predicate isUnreachableInCall(Node n, DataFlowCall call) {
+  class NodeRegion instanceof IRBlock {
+    string toString() { result = "NodeRegion" }
+
+    predicate contains(Node n) { this = n.getBasicBlock() }
+
+    int totalOrder() {
+      this =
+        rank[result](IRBlock b, int startline, int startcolumn |
+          b.getLocation().hasLocationInfo(_, startline, startcolumn, _, _)
+        |
+          b order by startline, startcolumn
+        )
+    }
+  }
+
+  predicate isUnreachableInCall(NodeRegion block, DataFlowCall call) {
     exists(
       InstructionDirectParameterNode paramNode, ConstantIntegralTypeArgumentNode arg,
-      IntegerConstantInstruction constant, int k, Operand left, Operand right, IRBlock block
+      IntegerConstantInstruction constant, int k, Operand left, Operand right, int argval
     |
       // arg flows into `paramNode`
-      DataFlowImplCommon::viableParamArg(call, paramNode, arg) and
+      DataFlowImplCommon::viableParamArg(call, pragma[only_bind_into](paramNode),
+        pragma[only_bind_into](arg)) and
       left = constant.getAUse() and
       right = valueNumber(paramNode.getInstruction()).getAUse() and
-      block = n.getBasicBlock()
+      argval = arg.getValue()
     |
       // and there's a guard condition which ensures that the result of `left == right + k` is `areEqual`
-      exists(boolean areEqual |
-        ensuresEq(pragma[only_bind_into](left), pragma[only_bind_into](right),
-          pragma[only_bind_into](k), pragma[only_bind_into](block), areEqual)
-      |
+      exists(boolean areEqual | ensuresEq(left, right, k, block, areEqual) |
         // this block ensures that left = right + k, but it holds that `left != right + k`
         areEqual = true and
-        constant.getValue().toInt() != arg.getValue() + k
+        constant.getValue().toInt() != argval + k
         or
         // this block ensures that or `left != right + k`, but it holds that `left = right + k`
         areEqual = false and
-        constant.getValue().toInt() = arg.getValue() + k
+        constant.getValue().toInt() = argval + k
       )
       or
       // or there's a guard condition which ensures that the result of `left < right + k` is `isLessThan`
-      exists(boolean isLessThan |
-        ensuresLt(pragma[only_bind_into](left), pragma[only_bind_into](right),
-          pragma[only_bind_into](k), pragma[only_bind_into](block), isLessThan)
-      |
+      exists(boolean isLessThan | ensuresLt(left, right, k, block, isLessThan) |
         isLessThan = true and
         // this block ensures that `left < right + k`, but it holds that `left >= right + k`
-        constant.getValue().toInt() >= arg.getValue() + k
+        constant.getValue().toInt() >= argval + k
         or
         // this block ensures that `left >= right + k`, but it holds that `left < right + k`
         isLessThan = false and
-        constant.getValue().toInt() < arg.getValue() + k
+        constant.getValue().toInt() < argval + k
       )
     )
   }

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowUtil.qll

@@ -114,6 +114,13 @@ predicate conversionFlow(
       instrTo.(CheckedConvertOrNullInstruction).getUnaryOperand() = opFrom
       or
       instrTo.(InheritanceConversionInstruction).getUnaryOperand() = opFrom
+      or
+      exists(BuiltInInstruction builtIn |
+        builtIn = instrTo and
+        // __builtin_bit_cast
+        builtIn.getBuiltInOperation() instanceof BuiltInBitCast and
+        opFrom = builtIn.getAnOperand()
+      )
     )
     or
     additional = true and

cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/TranslatedExpr.qll

@@ -3208,9 +3208,20 @@ class TranslatedBuiltInOperation extends TranslatedNonConstantExpr {
 
   final override Instruction getResult() { result = this.getInstruction(OnlyInstructionTag()) }
 
+  /**
+   * Gets the rnk'th (0-indexed) child for which a `TranslatedElement` exists.
+   *
+   * We use this predicate to filter out `TypeName` expressions that sometimes
+   * occur in builtin operations since the IR doesn't have an instruction to
+   * represent a reference to a type.
+   */
+  private TranslatedElement getRankedChild(int rnk) {
+    result = rank[rnk + 1](int id, TranslatedElement te | te = this.getChild(id) | te order by id)
+  }
+
   final override Instruction getFirstInstruction(EdgeKind kind) {
-    if exists(this.getChild(0))
-    then result = this.getChild(0).getFirstInstruction(kind)
+    if exists(this.getRankedChild(0))
+    then result = this.getRankedChild(0).getFirstInstruction(kind)
     else (
       kind instanceof GotoEdge and result = this.getInstruction(OnlyInstructionTag())
     )
@@ -3230,11 +3241,11 @@ class TranslatedBuiltInOperation extends TranslatedNonConstantExpr {
   }
 
   final override Instruction getChildSuccessorInternal(TranslatedElement child, EdgeKind kind) {
-    exists(int id | child = this.getChild(id) |
-      result = this.getChild(id + 1).getFirstInstruction(kind)
+    exists(int id | child = this.getRankedChild(id) |
+      result = this.getRankedChild(id + 1).getFirstInstruction(kind)
       or
       kind instanceof GotoEdge and
-      not exists(this.getChild(id + 1)) and
+      not exists(this.getRankedChild(id + 1)) and
       result = this.getInstruction(OnlyInstructionTag())
     )
   }
@@ -3249,7 +3260,7 @@ class TranslatedBuiltInOperation extends TranslatedNonConstantExpr {
     tag = OnlyInstructionTag() and
     exists(int index |
       operandTag = positionalArgumentOperand(index) and
-      result = this.getChild(index).(TranslatedExpr).getResult()
+      result = this.getRankedChild(index).(TranslatedExpr).getResult()
     )
   }
 

cpp/ql/src/CHANGELOG.md

@@ -1,3 +1,14 @@
+## 1.0.0
+
+### Breaking Changes
+
+* CodeQL package management is now generally available, and all GitHub-produced CodeQL packages have had their version numbers increased to 1.0.0.
+
+### Minor Analysis Improvements
+
+* The "Use of unique pointer after lifetime ends" query (`cpp/use-of-unique-pointer-after-lifetime-ends`) no longer reports an alert when the pointer is converted to a boolean
+* The "Variable not initialized before use" query (`cpp/not-initialised`) no longer reports an alert on static variables.
+
 ## 0.9.12
 
 ### New Queries

cpp/ql/src/Security/CWE/CWE-131/NoSpaceForZeroTerminator.qhelp

@@ -12,8 +12,8 @@ the required buffer size, but do not allocate space for the zero terminator.
 </overview>
 <recommendation>
 <p>
-The expression highlighted by this rule creates a buffer that is of insufficient size to contain
-the data being copied. This makes the code vulnerable to buffer overflow which can result in anything from a segmentation fault to a security vulnerability (particularly if the array is on stack-allocated memory).
+The highlighted code segment creates a buffer without ensuring it's large enough to accommodate the copied data.
+This leaves the code susceptible to a buffer overflow attack, which could lead to anything from program crashes to malicious code execution.
 </p>
 
 <p>

cpp/ql/src/Security/CWE/CWE-676/DangerousFunctionOverflow.ql

@@ -17,5 +17,6 @@ import cpp
 from FunctionCall call, Function target
 where
   call.getTarget() = target and
-  target.hasGlobalOrStdName("gets")
+  target.hasGlobalOrStdName("gets") and
+  target.getNumberOfParameters() = 1
 select call, "'gets' does not guard against buffer overflow."

cpp/ql/src/change-notes/2014-06-05-gets-parameter.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* The `cpp/dangerous-function-overflow` no longer produces a false positive alert when the `gets` function does not have exactly one parameter.

cpp/ql/src/change-notes/2024-05-19-avoid-reporting-static-variable.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* The "Variable not initialized before use" query (`cpp/not-initialised`) no longer reports an alert on static variables.

cpp/ql/src/change-notes/2024-05-22-use-of-unique-pointer-after-lifetime-ends-fp.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* The "Use of unique pointer after lifetime ends" query (`cpp/use-of-unique-pointer-after-lifetime-ends`) no longer reports an alert when the pointer is converted to a boolean

cpp/ql/src/change-notes/released/1.0.0.md

@@ -0,0 +1,10 @@
+## 1.0.0
+
+### Breaking Changes
+
+* CodeQL package management is now generally available, and all GitHub-produced CodeQL packages have had their version numbers increased to 1.0.0.
+
+### Minor Analysis Improvements
+
+* The "Use of unique pointer after lifetime ends" query (`cpp/use-of-unique-pointer-after-lifetime-ends`) no longer reports an alert when the pointer is converted to a boolean
+* The "Variable not initialized before use" query (`cpp/not-initialised`) no longer reports an alert on static variables.

cpp/ql/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.9.12
+lastReleaseVersion: 1.0.0

cpp/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-queries
-version: 0.9.13-dev
+version: 1.0.1-dev
 groups:
   - cpp
   - queries

cpp/ql/test/experimental/query-tests/Security/CWE/CWE-193/constant-size/ConstantSizeArrayOffByOne.expected

@@ -61,7 +61,6 @@ edges
 | test.cpp:289:19:289:25 | buffer3 | test.cpp:277:35:277:35 | p | provenance |  |
 | test.cpp:289:19:289:25 | buffer3 | test.cpp:289:19:289:25 | buffer3 | provenance |  |
 | test.cpp:292:25:292:27 | arr | test.cpp:299:16:299:21 | access to array | provenance | Config |
-| test.cpp:292:25:292:27 | arr | test.cpp:299:16:299:21 | access to array | provenance | Config |
 | test.cpp:306:20:306:23 | arr1 | test.cpp:292:25:292:27 | arr | provenance |  |
 | test.cpp:306:20:306:23 | arr1 | test.cpp:306:20:306:23 | arr1 | provenance |  |
 | test.cpp:309:20:309:23 | arr2 | test.cpp:292:25:292:27 | arr | provenance |  |
@@ -159,7 +158,6 @@ nodes
 | test.cpp:289:19:289:25 | buffer3 | semmle.label | buffer3 |
 | test.cpp:289:19:289:25 | buffer3 | semmle.label | buffer3 |
 | test.cpp:292:25:292:27 | arr | semmle.label | arr |
-| test.cpp:292:25:292:27 | arr | semmle.label | arr |
 | test.cpp:299:16:299:21 | access to array | semmle.label | access to array |
 | test.cpp:306:20:306:23 | arr1 | semmle.label | arr1 |
 | test.cpp:306:20:306:23 | arr1 | semmle.label | arr1 |

cpp/ql/test/header-variant-tests/multi-target-includes/main1.cpp

@@ -1,4 +1,4 @@
-// semmle-extractor-options: -I${testdir}/subdir1
+// semmle-extractor-options: -I${testdir}/subdir1 --edg --set_flag  --edg stack_referenced_include_directories
 // main1.cpp
 #include "common.h"
 

cpp/ql/test/header-variant-tests/multi-target-includes/main2.cpp

@@ -1,4 +1,4 @@
-// semmle-extractor-options: -I${testdir}/subdir2
+// semmle-extractor-options: -I${testdir}/subdir2 --edg --set_flag --edg stack_referenced_include_directories
 // main2.cpp
 #include "common.h"
 

cpp/ql/test/library-tests/dataflow/dataflow-tests/clang.cpp

@@ -52,3 +52,9 @@ void following_pointers( // $ ast-def=sourceStruct1_ptr ir-def=*cleanArray1 ir-d
   sink(stackArray); // $ ast,ir
   indirect_sink(stackArray); // $ ast ir=50:25 ir=50:35 ir=51:19
 }
+
+void test_bitcast() {
+  unsigned long x = source();
+  double d = __builtin_bit_cast(double, x);
+  sink(d); // $ ir MISSING: ast
+}

cpp/ql/test/library-tests/dataflow/dataflow-tests/test-source-sink.expected

@@ -153,6 +153,7 @@ irFlow
 | clang.cpp:50:25:50:30 | call to source | clang.cpp:53:17:53:26 | *stackArray |
 | clang.cpp:50:35:50:40 | call to source | clang.cpp:53:17:53:26 | *stackArray |
 | clang.cpp:51:19:51:24 | call to source | clang.cpp:53:17:53:26 | *stackArray |
+| clang.cpp:57:21:57:28 | call to source | clang.cpp:59:8:59:8 | d |
 | dispatch.cpp:9:37:9:42 | call to source | dispatch.cpp:35:16:35:25 | call to notSource1 |
 | dispatch.cpp:9:37:9:42 | call to source | dispatch.cpp:43:15:43:24 | call to notSource1 |
 | dispatch.cpp:10:37:10:42 | call to source | dispatch.cpp:36:16:36:25 | call to notSource2 |

cpp/ql/test/library-tests/dataflow/external-models/asio_streams.cpp

@@ -0,0 +1,107 @@
+
+// --- stub library headers ---
+
+namespace std {
+	typedef unsigned long size_t;
+	#define SIZE_MAX 0xFFFFFFFF
+
+	template <class T> class allocator {
+	};
+
+	template<class charT> struct char_traits {
+	};
+
+	template<class charT, class traits = char_traits<charT>, class Allocator = allocator<charT> >
+	class basic_string {
+	public:
+		basic_string(const charT* s, const Allocator& a = Allocator());
+	};
+
+	typedef basic_string<char> string;
+};
+
+namespace boost {
+	namespace system {
+		class error_code {
+		public:
+			operator bool() const;
+		};
+	};
+	
+	namespace asio {
+		template<typename Protocol/*, typename Executor*/>
+		class basic_stream_socket /*: public basic_socket<Protocol, Executor>*/ {
+		};
+
+		namespace ip {
+			class tcp {
+			public:
+				typedef basic_stream_socket<tcp> socket;
+			};
+		};
+
+		template<typename Allocator = std::allocator<char>> class basic_streambuf {
+		public:
+			basic_streambuf(
+				std::size_t maximum_size = SIZE_MAX,
+				const Allocator &allocator = Allocator());
+		};
+
+		typedef basic_streambuf<> streambuf;
+
+		class mutable_buffer {
+		};
+
+		template<typename Elem, typename Traits, typename Allocator>
+		mutable_buffer buffer(std::basic_string<Elem, Traits, Allocator> & data);
+
+		template<typename SyncReadStream, typename Allocator> std::size_t read_until(
+			SyncReadStream &s,
+			asio::basic_streambuf<Allocator> &b,
+			char delim,
+			boost::system::error_code &ec);
+
+		template<typename SyncWriteStream, typename ConstBufferSequence> std::size_t write(
+			SyncWriteStream &s,
+			const ConstBufferSequence &buffers,
+			boost::system::error_code &ec,
+			int constraint = 0); // simplified
+	};
+};
+
+// --- test code ---
+
+char *source();
+void sink(char *);
+void sink(std::string);
+void sink(boost::asio::streambuf);
+void sink(boost::asio::mutable_buffer);
+
+char *getenv(const char *name);
+int send(int, const void*, int, int);
+
+void test(boost::asio::ip::tcp::socket &socket) {
+	boost::asio::streambuf recv_buffer;
+	boost::system::error_code error;
+
+	boost::asio::read_until(socket, recv_buffer, '\0', error);
+	if (error) {
+		// ...
+	}
+	sink(recv_buffer); // $ ir
+
+	boost::asio::write(socket, recv_buffer, error); // $ ir
+
+	// ---
+
+	std::string send_str = std::string(source());
+	sink(send_str); // $ ir
+
+	boost::asio::mutable_buffer send_buffer = boost::asio::buffer(send_str);
+	sink(send_buffer); // $ ir
+
+	boost::asio::write(socket, send_buffer, error); // $ ir
+	if (error) {
+		// ...
+	}
+}

cpp/ql/test/library-tests/dataflow/external-models/flow.expected

@@ -0,0 +1,2 @@
+testFailures
+failures

cpp/ql/test/library-tests/dataflow/external-models/flow.ext.yml

@@ -0,0 +1,16 @@
+extensions:
+  - addsTo:
+      pack: codeql/cpp-all
+      extensible: sourceModel
+    data: # namespace, type, subtypes, name, signature, ext, output, kind, provenance
+      - ["", "", False, "ymlSource", "", "", "ReturnValue", "local", "manual"]
+  - addsTo:
+      pack: codeql/cpp-all
+      extensible: sinkModel
+    data: # namespace, type, subtypes, name, signature, ext, input, kind, provenance
+      - ["", "", False, "ymlSink", "", "", "Argument[0]", "test-sink", "manual"]
+  - addsTo:
+      pack: codeql/cpp-all
+      extensible: summaryModel
+    data: # namespace, type, subtypes, name, signature, ext, input, output, kind, provenance
+      - ["", "", False, "ymlStep", "", "", "Argument[0]", "ReturnValue", "taint", "manual"]

cpp/ql/test/library-tests/dataflow/external-models/flow.ql

@@ -0,0 +1,34 @@
+import TestUtilities.dataflow.FlowTestCommon
+import cpp
+import semmle.code.cpp.security.FlowSources
+
+module IRTest {
+  private import semmle.code.cpp.ir.IR
+  private import semmle.code.cpp.ir.dataflow.TaintTracking
+
+  /** Common data flow configuration to be used by tests. */
+  module TestAllocationConfig implements DataFlow::ConfigSig {
+    predicate isSource(DataFlow::Node source) {
+      // external flow source node
+      sourceNode(source, _)
+      or
+      // test source function
+      source.asExpr().(FunctionCall).getTarget().getName() = "source"
+    }
+
+    predicate isSink(DataFlow::Node sink) {
+      // external flow sink node
+      sinkNode(sink, _)
+      or
+      // test sink function
+      exists(FunctionCall call |
+        call.getTarget().getName() = "sink" and
+        sink.asExpr() = call.getAnArgument()
+      )
+    }
+  }
+
+  module IRFlow = TaintTracking::Global<TestAllocationConfig>;
+}
+
+import MakeTest<IRFlowTest<IRTest::IRFlow>>

cpp/ql/test/library-tests/dataflow/external-models/sinks.expected

@@ -0,0 +1,5 @@
+| asio_streams.cpp:93:29:93:39 | *recv_buffer | remote-sink |
+| asio_streams.cpp:103:29:103:39 | *send_buffer | remote-sink |
+| test.cpp:9:10:9:10 | 0 | test-sink |
+| test.cpp:11:10:11:10 | x | test-sink |
+| test.cpp:15:10:15:10 | y | test-sink |

cpp/ql/test/library-tests/dataflow/external-models/sinks.ext.yml

@@ -0,0 +1,6 @@
+extensions:
+  - addsTo:
+      pack: codeql/cpp-all
+      extensible: sinkModel
+    data: # namespace, type, subtypes, name, signature, ext, input, kind, provenance
+      - ["", "", False, "ymlSink", "", "", "Argument[0]", "test-sink", "manual"]

cpp/ql/test/library-tests/dataflow/external-models/sinks.ql

@@ -0,0 +1,7 @@
+import cpp
+import semmle.code.cpp.ir.dataflow.DataFlow
+import semmle.code.cpp.dataflow.ExternalFlow
+
+from DataFlow::Node node, string kind
+where sinkNode(node, kind)
+select node, kind

cpp/ql/test/library-tests/dataflow/external-models/sources.expected

@@ -0,0 +1,2 @@
+| asio_streams.cpp:87:34:87:44 | read_until output argument | remote |
+| test.cpp:7:10:7:18 | call to ymlSource | local |

cpp/ql/test/library-tests/dataflow/external-models/sources.ext.yml

@@ -0,0 +1,6 @@
+extensions:
+  - addsTo:
+      pack: codeql/cpp-all
+      extensible: sourceModel
+    data: # namespace, type, subtypes, name, signature, ext, output, kind, provenance
+      - ["", "", False, "ymlSource", "", "", "ReturnValue", "local", "manual"]

cpp/ql/test/library-tests/dataflow/external-models/sources.ql

@@ -0,0 +1,7 @@
+import cpp
+import semmle.code.cpp.ir.dataflow.DataFlow
+import semmle.code.cpp.dataflow.ExternalFlow
+
+from DataFlow::Node node, string kind
+where sourceNode(node, kind)
+select node, kind

cpp/ql/test/library-tests/dataflow/external-models/steps.expected

@@ -0,0 +1,2 @@
+| asio_streams.cpp:100:64:100:71 | *send_str | asio_streams.cpp:100:44:100:62 | call to buffer |
+| test.cpp:13:18:13:18 | x | test.cpp:13:10:13:16 | call to ymlStep |

cpp/ql/test/library-tests/dataflow/external-models/steps.ext.yml

@@ -0,0 +1,6 @@
+extensions:
+  - addsTo:
+      pack: codeql/cpp-all
+      extensible: summaryModel
+    data: # namespace, type, subtypes, name, signature, ext, input, output, kind, provenance
+      - ["", "", False, "ymlStep", "", "", "Argument[0]", "ReturnValue", "taint", "manual"]

cpp/ql/test/library-tests/dataflow/external-models/steps.ql

@@ -0,0 +1,8 @@
+import cpp
+import semmle.code.cpp.ir.dataflow.DataFlow
+import semmle.code.cpp.dataflow.ExternalFlow
+import semmle.code.cpp.dataflow.internal.FlowSummaryImpl as FlowSummaryImpl
+
+from DataFlow::Node node1, DataFlow::Node node2
+where FlowSummaryImpl::Private::Steps::summaryThroughStepTaint(node1, node2, _)
+select node1, node2

cpp/ql/test/library-tests/dataflow/external-models/test.cpp

@@ -0,0 +1,16 @@
+
+int ymlSource();
+void ymlSink(int value);
+int ymlStep(int value);
+
+void test() {
+	int x = ymlSource();
+
+	ymlSink(0);
+
+	ymlSink(x); // $ ir
+
+	int y = ymlStep(x);
+
+	ymlSink(y); // $ ir
+}

cpp/ql/test/library-tests/dataflow/external-models/validatemodels.ql

@@ -0,0 +1,2 @@
+import cpp
+import semmle.code.cpp.dataflow.ExternalFlow::CsvValidation

cpp/ql/test/library-tests/dataflow/source-sink-tests/asio_streams.cpp

@@ -0,0 +1,89 @@
+
+// --- stub library headers ---
+
+namespace std {
+	typedef unsigned long size_t;
+	#define SIZE_MAX 0xFFFFFFFF
+
+	template <class T> class allocator {
+	};
+
+	template<class charT> struct char_traits {
+	};
+
+	template<class charT, class traits = char_traits<charT>, class Allocator = allocator<charT> >
+	class basic_string {
+	public:
+		basic_string(const charT* s, const Allocator& a = Allocator());
+	};
+
+	typedef basic_string<char> string;
+};
+
+namespace boost {
+	namespace system {
+		class error_code {
+		public:
+			operator bool() const;
+		};
+	};
+	
+	namespace asio {
+		template<typename Protocol/*, typename Executor*/>
+		class basic_stream_socket /*: public basic_socket<Protocol, Executor>*/ {
+		};
+
+		namespace ip {
+			class tcp {
+			public:
+				typedef basic_stream_socket<tcp> socket;
+			};
+		};
+
+		template<typename Allocator = std::allocator<char>> class basic_streambuf {
+		public:
+			basic_streambuf(
+				std::size_t maximum_size = SIZE_MAX,
+				const Allocator &allocator = Allocator());
+		};
+
+		typedef basic_streambuf<> streambuf;
+
+		class mutable_buffer {
+		};
+
+		template<typename Elem, typename Traits, typename Allocator>
+		mutable_buffer buffer(std::basic_string<Elem, Traits, Allocator> & data);
+
+		template<typename SyncReadStream, typename Allocator> std::size_t read_until(
+			SyncReadStream &s,
+			asio::basic_streambuf<Allocator> &b,
+			char delim,
+			boost::system::error_code &ec);
+
+		template<typename SyncWriteStream, typename ConstBufferSequence> std::size_t write(
+			SyncWriteStream &s,
+			const ConstBufferSequence &buffers,
+			boost::system::error_code &ec,
+			int constraint = 0); // simplified
+	};
+};
+
+// --- test code ---
+
+void test(boost::asio::ip::tcp::socket &socket) {
+	boost::asio::streambuf recv_buffer;
+	boost::system::error_code error;
+
+	boost::asio::read_until(socket, recv_buffer, '\0', error); // $ remote_source
+	if (error) {
+		// ...
+	}
+
+	std::string send_str = std::string("message");
+	boost::asio::mutable_buffer send_buffer = boost::asio::buffer(send_str);
+	boost::asio::write(socket, send_buffer, error); // $ remote_sink
+	if (error) {
+		// ...
+	}
+}

cpp/ql/test/library-tests/includes/include_next/foo.c

@@ -1,3 +1,3 @@
-// semmle-extractor-options: -I${testdir}/a -I${testdir}/b
+// semmle-extractor-options: -I${testdir}/a -I${testdir}/b --edg --set_flag --edg stack_referenced_include_directories
 #include <test.h>
 

cpp/ql/test/library-tests/ir/ir/PrintAST.expected

@@ -22734,6 +22734,25 @@ ir.cpp:
 # 2552|             Type = [Class] ClassWithDestructor
 # 2552|             ValueCategory = xvalue
 # 2553|     getStmt(1): [ReturnStmt] return ...
+# 2555| [TopLevelFunction] void builtin_bitcast(unsigned long)
+# 2555|   <params>: 
+# 2555|     getParameter(0): [Parameter] ul
+# 2555|         Type = [LongType] unsigned long
+# 2555|   getEntryPoint(): [BlockStmt] { ... }
+# 2556|     getStmt(0): [DeclStmt] declaration
+# 2556|       getDeclarationEntry(0): [VariableDeclarationEntry] definition of d
+# 2556|           Type = [DoubleType] double
+# 2556|         getVariable().getInitializer(): [Initializer] initializer for d
+# 2556|           getExpr(): [BuiltInBitCast] __builtin_bit_cast
+# 2556|               Type = [DoubleType] double
+# 2556|               ValueCategory = prvalue
+# 2556|             getChild(0): [TypeName] double
+# 2556|                 Type = [DoubleType] double
+# 2556|                 ValueCategory = prvalue
+# 2556|             getChild(1): [VariableAccess] ul
+# 2556|                 Type = [LongType] unsigned long
+# 2556|                 ValueCategory = prvalue(load)
+# 2557|     getStmt(1): [ReturnStmt] return ...
 perf-regression.cpp:
 #    4| [CopyAssignmentOperator] Big& Big::operator=(Big const&)
 #    4|   <params>: 

cpp/ql/test/library-tests/ir/ir/aliased_ir.expected

@@ -18351,6 +18351,24 @@ ir.cpp:
 # 2550|   Block 2
 # 2550|     v2550_10(void) = Unreached : 
 
+# 2555| void builtin_bitcast(unsigned long)
+# 2555|   Block 0
+# 2555|     v2555_1(void)                 = EnterFunction               : 
+# 2555|     m2555_2(unknown)              = AliasedDefinition           : 
+# 2555|     m2555_3(unknown)              = InitializeNonLocal          : 
+# 2555|     m2555_4(unknown)              = Chi                         : total:m2555_2, partial:m2555_3
+# 2555|     r2555_5(glval<unsigned long>) = VariableAddress[ul]         : 
+# 2555|     m2555_6(unsigned long)        = InitializeParameter[ul]     : &:r2555_5
+# 2556|     r2556_1(glval<double>)        = VariableAddress[d]          : 
+# 2556|     r2556_2(glval<unsigned long>) = VariableAddress[ul]         : 
+# 2556|     r2556_3(unsigned long)        = Load[ul]                    : &:r2556_2, m2555_6
+# 2556|     r2556_4(double)               = BuiltIn[__builtin_bit_cast] : 0:r2556_3
+# 2556|     m2556_5(double)               = Store[d]                    : &:r2556_1, r2556_4
+# 2557|     v2557_1(void)                 = NoOp                        : 
+# 2555|     v2555_7(void)                 = ReturnVoid                  : 
+# 2555|     v2555_8(void)                 = AliasedUse                  : m2555_3
+# 2555|     v2555_9(void)                 = ExitFunction                : 
+
 perf-regression.cpp:
 #    6| void Big::Big()
 #    6|   Block 0

cpp/ql/test/library-tests/ir/ir/ir.cpp

@@ -2552,4 +2552,8 @@ void constexpr_inconsistency(bool b) {
     ;
 }
 
+void builtin_bitcast(unsigned long ul) {
+    double d = __builtin_bit_cast(double, ul);
+}
+
 // semmle-extractor-options: -std=c++20 --clang

cpp/ql/test/library-tests/ir/ir/raw_ir.expected

@@ -16693,6 +16693,23 @@ ir.cpp:
 # 2550|     v2550_7(void)                       = AliasedUse                            : ~m?
 # 2550|     v2550_8(void)                       = ExitFunction                          : 
 
+# 2555| void builtin_bitcast(unsigned long)
+# 2555|   Block 0
+# 2555|     v2555_1(void)                 = EnterFunction               : 
+# 2555|     mu2555_2(unknown)             = AliasedDefinition           : 
+# 2555|     mu2555_3(unknown)             = InitializeNonLocal          : 
+# 2555|     r2555_4(glval<unsigned long>) = VariableAddress[ul]         : 
+# 2555|     mu2555_5(unsigned long)       = InitializeParameter[ul]     : &:r2555_4
+# 2556|     r2556_1(glval<double>)        = VariableAddress[d]          : 
+# 2556|     r2556_2(glval<unsigned long>) = VariableAddress[ul]         : 
+# 2556|     r2556_3(unsigned long)        = Load[ul]                    : &:r2556_2, ~m?
+# 2556|     r2556_4(double)               = BuiltIn[__builtin_bit_cast] : 0:r2556_3
+# 2556|     mu2556_5(double)              = Store[d]                    : &:r2556_1, r2556_4
+# 2557|     v2557_1(void)                 = NoOp                        : 
+# 2555|     v2555_6(void)                 = ReturnVoid                  : 
+# 2555|     v2555_7(void)                 = AliasedUse                  : ~m?
+# 2555|     v2555_8(void)                 = ExitFunction                : 
+
 perf-regression.cpp:
 #    6| void Big::Big()
 #    6|   Block 0

cpp/ql/test/library-tests/padding/options

@@ -0,0 +1 @@
+semmle-extractor-options: -D__x86_64=1

cpp/ql/test/query-tests/Security/CWE/CWE-676/semmle/PotentiallyDangerousFunction/test.c

@@ -36,7 +36,7 @@ char *gets(char *s);
 
 void testGets() {
 	char buf1[1024];
-	char buf2 = malloc(1024);
+	char *buf2 = malloc(1024);
 	char *s;
 
 	gets(buf1); // BAD: use of gets

cpp/ql/test/query-tests/Security/CWE/CWE-676/semmle/PotentiallyDangerousFunction/test.cpp

@@ -0,0 +1,7 @@
+char *gets();
+
+void testOtherGets() {
+	char *s;
+
+	s = gets(); // GOOD: this is not the gets from stdio.h
+}

csharp/.gitignore

@@ -14,5 +14,4 @@ csharp.log
 .vscode/launch.json
 
 extractor/Semmle.Extraction.CSharp.Driver/Properties/launchSettings.json
-extractor-pack
 paket-files/

csharp/BUILD.bazel

@@ -1,3 +1,6 @@
+load("@rules_pkg//pkg:mappings.bzl", "pkg_filegroup", "pkg_files")
+load("//misc/bazel:pkg.bzl", "codeql_pack", "codeql_pkg_files_overlay")
+
 package(default_visibility = ["//visibility:public"])
 
 alias(
@@ -9,3 +12,70 @@ alias(
     name = "dbscheme-stats",
     actual = "//csharp/ql/lib:dbscheme-stats",
 )
+
+pkg_files(
+    name = "dbscheme-group",
+    srcs = [
+        ":dbscheme",
+        ":dbscheme-stats",
+    ],
+    strip_prefix = None,
+)
+
+pkg_files(
+    name = "extractor-asp",
+    srcs = [
+        "@semmle_code//extractor-asp:extractor-asp-fat",
+    ],
+    prefix = "tools",
+    renames = {
+        "@semmle_code//extractor-asp:extractor-asp-fat": "extractor-asp.jar",
+    },
+)
+
+pkg_filegroup(
+    name = "db-files",
+    srcs = [
+        ":dbscheme-group",
+        "//csharp/downgrades",
+    ],
+)
+
+pkg_files(
+    name = "extra-files",
+    srcs = [
+        ":codeql-extractor.yml",
+        "//:LICENSE",
+    ],
+)
+
+codeql_pkg_files_overlay(
+    name = "extractor-arch-overlay",
+    srcs = [
+        "//csharp/autobuilder/Semmle.Autobuild.CSharp",
+        "//csharp/extractor/Semmle.Extraction.CSharp.Driver",
+        "//csharp/extractor/Semmle.Extraction.CSharp.Standalone",
+    ],
+)
+
+codeql_pack(
+    name = "csharp",
+    srcs = [
+        ":dbscheme-group",
+        ":extra-files",
+        ":extractor-arch-overlay",
+        ":extractor-asp",
+        "//csharp/downgrades",
+        "//csharp/tools",
+    ],
+)
+
+test_suite(
+    name = "unit-tests",
+    tags = ["csharp"],
+    tests = [
+        "//csharp/autobuilder/Semmle.Autobuild.CSharp.Tests",
+        "//csharp/autobuilder/Semmle.Autobuild.Cpp.Tests",
+        "//csharp/extractor/Semmle.Extraction.Tests",
+    ],
+)

csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BUILD.bazel

@@ -0,0 +1,18 @@
+load(
+    "//misc/bazel:csharp.bzl",
+    "codeql_xunit_test",
+)
+
+codeql_xunit_test(
+    name = "Semmle.Autobuild.CSharp.Tests",
+    srcs = glob([
+        "*.cs",
+        "Properties/*.cs",
+    ]),
+    deps = [
+        "//csharp/autobuilder/Semmle.Autobuild.CSharp:bin/Semmle.Autobuild.CSharp",
+        "//csharp/autobuilder/Semmle.Autobuild.Shared",
+        "@paket.main//microsoft.net.test.sdk",
+        "@paket.main//system.io.filesystem",
+    ],
+)

csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs

@@ -215,9 +215,9 @@ namespace Semmle.Autobuild.CSharp.Tests
 
     internal class TestDiagnosticWriter : IDiagnosticsWriter
     {
-        public IList<DiagnosticMessage> Diagnostics { get; } = new List<DiagnosticMessage>();
+        public IList<Semmle.Util.DiagnosticMessage> Diagnostics { get; } = new List<Semmle.Util.DiagnosticMessage>();
 
-        public void AddEntry(DiagnosticMessage message) => this.Diagnostics.Add(message);
+        public void AddEntry(Semmle.Util.DiagnosticMessage message) => this.Diagnostics.Add(message);
 
         public void Dispose() { }
     }

csharp/autobuilder/Semmle.Autobuild.CSharp/BUILD.bazel

@@ -0,0 +1,22 @@
+load(
+    "//misc/bazel:csharp.bzl",
+    "codeql_csharp_binary",
+)
+
+codeql_csharp_binary(
+    name = "Semmle.Autobuild.CSharp",
+    srcs = glob([
+        "*.cs",
+        "Properties/*.cs",
+    ]),
+    visibility = ["//csharp:__subpackages__"],
+    deps = [
+        "//csharp/autobuilder/Semmle.Autobuild.Shared",
+        "//csharp/extractor/Semmle.Extraction.CSharp",
+        "//csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching",
+        "//csharp/extractor/Semmle.Extraction.CSharp.Standalone:bin/Semmle.Extraction.CSharp.Standalone",
+        "//csharp/extractor/Semmle.Util",
+        "@paket.main//microsoft.build",
+        "@paket.main//newtonsoft.json",
+    ],
+)

csharp/autobuilder/Semmle.Autobuild.Cpp.Tests/BUILD.bazel

@@ -0,0 +1,18 @@
+load(
+    "//misc/bazel:csharp.bzl",
+    "codeql_xunit_test",
+)
+
+codeql_xunit_test(
+    name = "Semmle.Autobuild.Cpp.Tests",
+    srcs = glob([
+        "*.cs",
+        "Properties/*.cs",
+    ]),
+    deps = [
+        "//csharp/autobuilder/Semmle.Autobuild.Cpp:bin/Semmle.Autobuild.Cpp",
+        "//csharp/autobuilder/Semmle.Autobuild.Shared",
+        "@paket.main//microsoft.net.test.sdk",
+        "@paket.main//system.io.filesystem",
+    ],
+)

csharp/autobuilder/Semmle.Autobuild.Cpp.Tests/BuildScripts.cs

@@ -200,9 +200,9 @@ namespace Semmle.Autobuild.Cpp.Tests
 
     internal class TestDiagnosticWriter : IDiagnosticsWriter
     {
-        public IList<DiagnosticMessage> Diagnostics { get; } = new List<DiagnosticMessage>();
+        public IList<Semmle.Util.DiagnosticMessage> Diagnostics { get; } = new List<Semmle.Util.DiagnosticMessage>();
 
-        public void AddEntry(DiagnosticMessage message) => this.Diagnostics.Add(message);
+        public void AddEntry(Semmle.Util.DiagnosticMessage message) => this.Diagnostics.Add(message);
 
         public void Dispose() { }
     }

csharp/autobuilder/Semmle.Autobuild.Cpp/BUILD.bazel

@@ -0,0 +1,18 @@
+load(
+    "//misc/bazel:csharp.bzl",
+    "codeql_csharp_binary",
+)
+
+codeql_csharp_binary(
+    name = "Semmle.Autobuild.Cpp",
+    srcs = glob([
+        "*.cs",
+        "Properties/*.cs",
+    ]),
+    visibility = ["//visibility:public"],
+    deps = [
+        "//csharp/autobuilder/Semmle.Autobuild.Shared",
+        "//csharp/extractor/Semmle.Util",
+        "@paket.main//microsoft.build",
+    ],
+)

csharp/autobuilder/Semmle.Autobuild.Shared/BUILD.bazel

@@ -0,0 +1,17 @@
+load(
+    "//misc/bazel:csharp.bzl",
+    "codeql_csharp_library",
+)
+
+codeql_csharp_library(
+    name = "Semmle.Autobuild.Shared",
+    srcs = glob([
+        "*.cs",
+        "Properties/*.cs",
+    ]),
+    visibility = ["//visibility:public"],
+    deps = [
+        "//csharp/extractor/Semmle.Util",
+        "@paket.main//microsoft.build",
+    ],
+)

csharp/documentation/library-coverage/coverage.csv

@@ -4,41 +4,41 @@ Amazon.Lambda.Core,10,,,,,,,,,,,10,,,,,,,,,,,
 Dapper,55,42,1,,,,,,,,,,55,,42,,,,,,,,1
 ILCompiler,,,81,,,,,,,,,,,,,,,,,,,81,
 ILLink.RoslynAnalyzer,,,63,,,,,,,,,,,,,,,,,,,63,
-ILLink.Shared,,,32,,,,,,,,,,,,,,,,,,,29,3
+ILLink.Shared,,,32,,,,,,,,,,,,,,,,,,,30,2
 ILLink.Tasks,,,3,,,,,,,,,,,,,,,,,,,3,
-Internal.IL,,,69,,,,,,,,,,,,,,,,,,,67,2
+Internal.IL,,,46,,,,,,,,,,,,,,,,,,,44,2
 Internal.Pgo,,,9,,,,,,,,,,,,,,,,,,,8,1
-Internal.TypeSystem,,,367,,,,,,,,,,,,,,,,,,,331,36
+Internal.TypeSystem,,,291,,,,,,,,,,,,,,,,,,,275,16
 JsonToItemsTaskFactory,,,5,,,,,,,,,,,,,,,,,,,5,
 Microsoft.Android.Build,,,14,,,,,,,,,,,,,,,,,,,14,
 Microsoft.Apple.Build,,,5,,,,,,,,,,,,,,,,,,,5,
 Microsoft.ApplicationBlocks.Data,28,,,,,,,,,,,,28,,,,,,,,,,
-Microsoft.CSharp,,,24,,,,,,,,,,,,,,,,,,,24,
-Microsoft.Diagnostics.Tools.Pgo,,,13,,,,,,,,,,,,,,,,,,,13,
+Microsoft.CSharp,,,10,,,,,,,,,,,,,,,,,,,10,
+Microsoft.Diagnostics.Tools.Pgo,,,12,,,,,,,,,,,,,,,,,,,12,
 Microsoft.EntityFrameworkCore,6,,12,,,,,,,,,,6,,,,,,,,,,12
 Microsoft.Extensions.Caching.Distributed,,,9,,,,,,,,,,,,,,,,,,,9,
 Microsoft.Extensions.Caching.Memory,,,30,,,,,,,,,,,,,,,,,,,29,1
-Microsoft.Extensions.Configuration,,2,83,,,,,,,,,,,,,2,,,,,,81,2
-Microsoft.Extensions.DependencyInjection,,,120,,,,,,,,,,,,,,,,,,,120,
-Microsoft.Extensions.DependencyModel,,,12,,,,,,,,,,,,,,,,,,,12,
-Microsoft.Extensions.Diagnostics.Metrics,,,13,,,,,,,,,,,,,,,,,,,13,
+Microsoft.Extensions.Configuration,,2,77,,,,,,,,,,,,,2,,,,,,76,1
+Microsoft.Extensions.DependencyInjection,,,96,,,,,,,,,,,,,,,,,,,95,1
+Microsoft.Extensions.DependencyModel,,,9,,,,,,,,,,,,,,,,,,,9,
+Microsoft.Extensions.Diagnostics.Metrics,,,15,,,,,,,,,,,,,,,,,,,15,
 Microsoft.Extensions.FileProviders,,,15,,,,,,,,,,,,,,,,,,,15,
 Microsoft.Extensions.FileSystemGlobbing,,,16,,,,,,,,,,,,,,,,,,,14,2
-Microsoft.Extensions.Hosting,,,23,,,,,,,,,,,,,,,,,,,22,1
+Microsoft.Extensions.Hosting,,,26,,,,,,,,,,,,,,,,,,,25,1
 Microsoft.Extensions.Http,,,8,,,,,,,,,,,,,,,,,,,8,
-Microsoft.Extensions.Logging,,,60,,,,,,,,,,,,,,,,,,,59,1
+Microsoft.Extensions.Logging,,,53,,,,,,,,,,,,,,,,,,,52,1
 Microsoft.Extensions.Options,,,8,,,,,,,,,,,,,,,,,,,8,
 Microsoft.Extensions.Primitives,,,64,,,,,,,,,,,,,,,,,,,64,
-Microsoft.Interop,,,78,,,,,,,,,,,,,,,,,,,78,
+Microsoft.Interop,,,73,,,,,,,,,,,,,,,,,,,73,
 Microsoft.NET.Build.Tasks,,,1,,,,,,,,,,,,,,,,,,,1,
 Microsoft.NET.WebAssembly.Webcil,,,7,,,,,,,,,,,,,,,,,,,7,
-Microsoft.VisualBasic,,,10,,,,,,,,,,,,,,,,,,,5,5
+Microsoft.VisualBasic,,,6,,,,,,,,,,,,,,,,,,,1,5
 Microsoft.WebAssembly.Build.Tasks,,,3,,,,,,,,,,,,,,,,,,,3,
 Microsoft.Win32,,4,4,,,,,,,,,,,,,,,,,,4,4,
-Mono.Linker,,,161,,,,,,,,,,,,,,,,,,,161,
+Mono.Linker,,,158,,,,,,,,,,,,,,,,,,,158,
 MySql.Data.MySqlClient,48,,,,,,,,,,,,48,,,,,,,,,,
 Newtonsoft.Json,,,91,,,,,,,,,,,,,,,,,,,73,18
 ServiceStack,194,,7,27,,,,,75,,,,92,,,,,,,,,7,
 SourceGenerators,,,4,,,,,,,,,,,,,,,,,,,4,
-System,59,44,10429,,8,8,1,,,4,5,,33,2,,3,15,17,3,4,,8460,1969
+System,49,44,9873,,3,3,1,,,4,5,,33,2,,3,15,17,3,4,,7968,1905
 Windows.Security.Cryptography.Core,1,,,,,,,1,,,,,,,,,,,,,,,

csharp/documentation/library-coverage/coverage.rst

@@ -8,7 +8,7 @@ C# framework & library support
 
    Framework / library,Package,Flow sources,Taint & value steps,Sinks (total),`CWE-079` :sub:`Cross-site scripting`
    `ServiceStack <https://servicestack.net/>`_,"``ServiceStack.*``, ``ServiceStack``",,7,194,
-   System,"``System.*``, ``System``",44,10429,59,9
-   Others,"``Amazon.Lambda.APIGatewayEvents``, ``Amazon.Lambda.Core``, ``Dapper``, ``ILCompiler``, ``ILLink.RoslynAnalyzer``, ``ILLink.Shared``, ``ILLink.Tasks``, ``Internal.IL``, ``Internal.Pgo``, ``Internal.TypeSystem``, ``JsonToItemsTaskFactory``, ``Microsoft.Android.Build``, ``Microsoft.Apple.Build``, ``Microsoft.ApplicationBlocks.Data``, ``Microsoft.CSharp``, ``Microsoft.Diagnostics.Tools.Pgo``, ``Microsoft.EntityFrameworkCore``, ``Microsoft.Extensions.Caching.Distributed``, ``Microsoft.Extensions.Caching.Memory``, ``Microsoft.Extensions.Configuration``, ``Microsoft.Extensions.DependencyInjection``, ``Microsoft.Extensions.DependencyModel``, ``Microsoft.Extensions.Diagnostics.Metrics``, ``Microsoft.Extensions.FileProviders``, ``Microsoft.Extensions.FileSystemGlobbing``, ``Microsoft.Extensions.Hosting``, ``Microsoft.Extensions.Http``, ``Microsoft.Extensions.Logging``, ``Microsoft.Extensions.Options``, ``Microsoft.Extensions.Primitives``, ``Microsoft.Interop``, ``Microsoft.NET.Build.Tasks``, ``Microsoft.NET.WebAssembly.Webcil``, ``Microsoft.VisualBasic``, ``Microsoft.WebAssembly.Build.Tasks``, ``Microsoft.Win32``, ``Mono.Linker``, ``MySql.Data.MySqlClient``, ``Newtonsoft.Json``, ``SourceGenerators``, ``Windows.Security.Cryptography.Core``",54,1518,148,
-   Totals,,98,11954,401,9
+   System,"``System.*``, ``System``",44,9873,49,9
+   Others,"``Amazon.Lambda.APIGatewayEvents``, ``Amazon.Lambda.Core``, ``Dapper``, ``ILCompiler``, ``ILLink.RoslynAnalyzer``, ``ILLink.Shared``, ``ILLink.Tasks``, ``Internal.IL``, ``Internal.Pgo``, ``Internal.TypeSystem``, ``JsonToItemsTaskFactory``, ``Microsoft.Android.Build``, ``Microsoft.Apple.Build``, ``Microsoft.ApplicationBlocks.Data``, ``Microsoft.CSharp``, ``Microsoft.Diagnostics.Tools.Pgo``, ``Microsoft.EntityFrameworkCore``, ``Microsoft.Extensions.Caching.Distributed``, ``Microsoft.Extensions.Caching.Memory``, ``Microsoft.Extensions.Configuration``, ``Microsoft.Extensions.DependencyInjection``, ``Microsoft.Extensions.DependencyModel``, ``Microsoft.Extensions.Diagnostics.Metrics``, ``Microsoft.Extensions.FileProviders``, ``Microsoft.Extensions.FileSystemGlobbing``, ``Microsoft.Extensions.Hosting``, ``Microsoft.Extensions.Http``, ``Microsoft.Extensions.Logging``, ``Microsoft.Extensions.Options``, ``Microsoft.Extensions.Primitives``, ``Microsoft.Interop``, ``Microsoft.NET.Build.Tasks``, ``Microsoft.NET.WebAssembly.Webcil``, ``Microsoft.VisualBasic``, ``Microsoft.WebAssembly.Build.Tasks``, ``Microsoft.Win32``, ``Mono.Linker``, ``MySql.Data.MySqlClient``, ``Newtonsoft.Json``, ``SourceGenerators``, ``Windows.Security.Cryptography.Core``",54,1357,148,
+   Totals,,98,11237,391,9
 

csharp/downgrades/BUILD.bazel

@@ -0,0 +1,12 @@
+load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix")
+
+pkg_files(
+    name = "downgrades",
+    srcs = glob(
+        ["**"],
+        exclude = ["BUILD.bazel"],
+    ),
+    prefix = "downgrades",
+    strip_prefix = strip_prefix.from_pkg(),
+    visibility = ["//csharp:__pkg__"],
+)

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/BUILD.bazel

@@ -0,0 +1,21 @@
+load(
+    "//misc/bazel:csharp.bzl",
+    "codeql_csharp_library",
+)
+
+codeql_csharp_library(
+    name = "Semmle.Extraction.CSharp.DependencyFetching",
+    srcs = glob([
+        "*.cs",
+        "Properties/*.cs",
+        "SourceGenerators/**/*.cs",
+    ]),
+    allow_unsafe_blocks = True,
+    internals_visible_to = ["Semmle.Extraction.Tests"],
+    nowarn = ["CA1822"],
+    visibility = ["//csharp:__subpackages__"],
+    deps = [
+        "//csharp/extractor/Semmle.Extraction",
+        "//csharp/extractor/Semmle.Util",
+    ],
+)

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs

@@ -97,6 +97,15 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                 return envVarPath;
             }
 
+            try
+            {
+                return DownloadNugetExe(fileProvider.SourceDir.FullName);
+            }
+            catch (Exception exc)
+            {
+                logger.LogInfo($"Download of nuget.exe failed: {exc.Message}");
+            }
+
             var nugetExesInRepo = fileProvider.NugetExes;
             if (nugetExesInRepo.Count > 1)
             {
@@ -119,7 +128,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                 return nugetPath;
             }
 
-            return DownloadNugetExe(fileProvider.SourceDir.FullName);
+            throw new Exception("Could not find or download nuget.exe.");
         }
 
         private string DownloadNugetExe(string sourceDir)
@@ -136,28 +145,20 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 
             Directory.CreateDirectory(directory);
             logger.LogInfo("Attempting to download nuget.exe");
-            try
-            {
-                FileUtils.DownloadFile(FileUtils.NugetExeUrl, nuget);
-                logger.LogInfo($"Downloaded nuget.exe to {nuget}");
-                return nuget;
-            }
-            catch
-            {
-                // Download failed.
-                throw new FileNotFoundException("Download of nuget.exe failed.");
-            }
+            FileUtils.DownloadFile(FileUtils.NugetExeUrl, nuget);
+            logger.LogInfo($"Downloaded nuget.exe to {nuget}");
+            return nuget;
         }
 
         private bool RunWithMono => !Win32.IsWindows() && !string.IsNullOrEmpty(Path.GetExtension(nugetExe));
 
         /// <summary>
-        /// Restore all files in a specified package.
+        /// Restore all packages in the specified packages.config file.
         /// </summary>
-        /// <param name="package">The package file.</param>
-        private bool TryRestoreNugetPackage(string package)
+        /// <param name="packagesConfig">The packages.config file.</param>
+        private bool TryRestoreNugetPackage(string packagesConfig)
         {
-            logger.LogInfo($"Restoring file {package}...");
+            logger.LogInfo($"Restoring file \"{packagesConfig}\"...");
 
             /* Use nuget.exe to install a package.
              * Note that there is a clutch of NuGet assemblies which could be used to
@@ -169,12 +170,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             if (RunWithMono)
             {
                 exe = "mono";
-                args = $"{nugetExe} install -OutputDirectory {packageDirectory} {package}";
+                args = $"\"{nugetExe}\" install -OutputDirectory \"{packageDirectory}\" \"{packagesConfig}\"";
             }
             else
             {
                 exe = nugetExe!;
-                args = $"install -OutputDirectory {packageDirectory} {package}";
+                args = $"install -OutputDirectory \"{packageDirectory}\" \"{packagesConfig}\"";
             }
 
             var pi = new ProcessStartInfo(exe, args)
@@ -195,7 +196,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             }
             else
             {
-                logger.LogInfo($"Restored file {package}");
+                logger.LogInfo($"Restored file \"{packagesConfig}\"");
                 return true;
             }
         }
@@ -205,7 +206,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         /// </summary>
         public int InstallPackages()
         {
-            return fileProvider.PackagesConfigs.Count(package => TryRestoreNugetPackage(package));
+            return fileProvider.PackagesConfigs.Count(TryRestoreNugetPackage);
         }
 
         private bool HasNoPackageSource()
@@ -239,7 +240,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             if (RunWithMono)
             {
                 exe = "mono";
-                args = $"{nugetExe} {command}";
+                args = $"\"{nugetExe}\" {command}";
             }
             else
             {

csharp/extractor/Semmle.Extraction.CSharp.DependencyStubGenerator/BUILD.bazel

@@ -0,0 +1,18 @@
+load(
+    "//misc/bazel:csharp.bzl",
+    "codeql_csharp_binary",
+)
+
+codeql_csharp_binary(
+    name = "Semmle.Extraction.CSharp.DependencyStubGenerator",
+    srcs = glob([
+        "*.cs",
+        "Properties/*.cs",
+    ]),
+    visibility = ["//csharp:__pkg__"],
+    deps = [
+        "//csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching",
+        "//csharp/extractor/Semmle.Extraction.CSharp.StubGenerator",
+        "//csharp/extractor/Semmle.Util",
+    ],
+)

csharp/extractor/Semmle.Extraction.CSharp.Driver/BUILD.bazel

@@ -0,0 +1,16 @@
+load(
+    "//misc/bazel:csharp.bzl",
+    "codeql_csharp_binary",
+)
+
+codeql_csharp_binary(
+    name = "Semmle.Extraction.CSharp.Driver",
+    srcs = glob([
+        "*.cs",
+        "Properties/*.cs",
+    ]),
+    visibility = ["//csharp:__pkg__"],
+    deps = [
+        "//csharp/extractor/Semmle.Extraction.CSharp",
+    ],
+)

csharp/extractor/Semmle.Extraction.CSharp.Standalone/BUILD.bazel

@@ -0,0 +1,18 @@
+load(
+    "//misc/bazel:csharp.bzl",
+    "codeql_csharp_binary",
+)
+
+codeql_csharp_binary(
+    name = "Semmle.Extraction.CSharp.Standalone",
+    srcs = glob([
+        "*.cs",
+        "Properties/*.cs",
+    ]),
+    visibility = ["//csharp:__subpackages__"],
+    deps = [
+        "//csharp/extractor/Semmle.Extraction.CSharp",
+        "//csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching",
+        "//csharp/extractor/Semmle.Util",
+    ],
+)

csharp/extractor/Semmle.Extraction.CSharp.StubGenerator/BUILD.bazel

@@ -0,0 +1,20 @@
+load(
+    "//misc/bazel:csharp.bzl",
+    "codeql_csharp_library",
+)
+
+codeql_csharp_library(
+    name = "Semmle.Extraction.CSharp.StubGenerator",
+    srcs = glob([
+        "*.cs",
+        "Properties/*.cs",
+    ]),
+    internals_visible_to = ["Semmle.Extraction.Tests"],
+    visibility = ["//csharp:__subpackages__"],
+    deps = [
+        "//csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching",
+        "//csharp/extractor/Semmle.Extraction.CSharp.Util",
+        "//csharp/extractor/Semmle.Util",
+        "@paket.main//microsoft.codeanalysis.csharp",
+    ],
+)

csharp/extractor/Semmle.Extraction.CSharp.Util/BUILD.bazel

@@ -0,0 +1,17 @@
+load(
+    "//misc/bazel:csharp.bzl",
+    "codeql_csharp_library",
+)
+
+codeql_csharp_library(
+    name = "Semmle.Extraction.CSharp.Util",
+    srcs = glob([
+        "Properties/*.cs",
+        "*.cs",
+    ]),
+    visibility = ["//csharp:__subpackages__"],
+    deps = [
+        "//csharp/extractor/Semmle.Util",
+        "@paket.main//microsoft.codeanalysis.csharp",
+    ],
+)

csharp/extractor/Semmle.Extraction.CSharp/BUILD.bazel

@@ -0,0 +1,26 @@
+load(
+    "//misc/bazel:csharp.bzl",
+    "codeql_csharp_library",
+)
+
+codeql_csharp_library(
+    name = "Semmle.Extraction.CSharp",
+    srcs = glob([
+        "Comments/**/*.cs",
+        "Entities/**/*.cs",
+        "Extractor/**/*.cs",
+        "Kinds/**/*.cs",
+        "Populators/**/*.cs",
+        "Properties/**/*.cs",
+        "*.cs",
+    ]),
+    allow_unsafe_blocks = True,
+    visibility = ["//csharp:__subpackages__"],
+    deps = [
+        "//csharp/extractor/Semmle.Extraction",
+        "//csharp/extractor/Semmle.Extraction.CSharp.Util",
+        "//csharp/extractor/Semmle.Util",
+        "@paket.main//microsoft.build",
+        "@paket.main//microsoft.codeanalysis.csharp",
+    ],
+)

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs

@@ -312,8 +312,6 @@ namespace Semmle.Extraction.CSharp
             else
                 Logger.Log(Severity.Info, "EXTRACTION SUCCEEDED in {0}", stopWatch.Elapsed);
 
-            Logger.Dispose();
-
             compilationTrapFile?.Dispose();
         }