Merge pull request #20988 from github/release-prep/2.23.8

Release preparation for version 2.23.8

.gitattributes

@@ -82,9 +82,6 @@
 /csharp/paket.main.bzl linguist-generated=true
 /csharp/paket.main_extension.bzl linguist-generated=true
 
-# ripunzip tool
-/misc/ripunzip/ripunzip-* filter=lfs diff=lfs merge=lfs -text
-
 # swift prebuilt resources
 /swift/third_party/resources/*.zip filter=lfs diff=lfs merge=lfs -text
 /swift/third_party/resources/*.tar.zst filter=lfs diff=lfs merge=lfs -text

.github/workflows/build-ripunzip.yml

@@ -1,166 +0,0 @@
-name: Build ripunzip
-
-on:
-  workflow_dispatch:
-    inputs:
-      ripunzip-version:
-        description: What reference to checkout from google/ripunzip. Latest by default
-        required: false
-      openssl-version:
-        description: What reference to checkout from openssl/openssl for Linux. Latest by default
-        required: false
-      open-pr:
-        description: Open a pull request updating the ripunzip versions committed to lfs
-        required: false
-        default: true  # will be false on PRs
-  pull_request:
-    paths:
-      - .github/workflows/build-ripunzip.yml
-
-permissions: {}
-  
-jobs:
-  versions:
-    runs-on: ubuntu-slim
-    outputs:
-      ripunzip-version: ${{ inputs.ripunzip-version || steps.fetch-ripunzip-version.outputs.version }}
-      openssl-version: ${{ inputs.openssl-version || steps.fetch-openssl-version.outputs.version }}
-    steps:
-      - name: Fetch latest ripunzip version
-        id: fetch-ripunzip-version
-        if: "!inputs.ripunzip-version"
-        run: &fetch-version
-          echo "version=$(gh release view --repo $REPO --json tagName --jq .tagName)" | tee -a $GITHUB_OUTPUT
-        env:
-          REPO: "google/ripunzip"
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - name: Fetch latest openssl version
-        id: fetch-openssl-version
-        if: "!inputs.openssl-version"
-        run: *fetch-version
-        env:
-          REPO: "openssl/openssl"
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-  build:
-    needs: versions
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-24.04, macos-15, windows-2025]
-    runs-on: ${{ matrix.os }}
-    steps:
-      - uses: actions/checkout@v5
-        with:
-          repository: google/ripunzip
-          ref: ${{ needs.versions.outputs.ripunzip-version }}
-      # we need to avoid ripunzip dynamically linking into libssl
-      # see https://github.com/sfackler/rust-openssl/issues/183
-      - if: runner.os == 'Linux'
-        name: checkout openssl
-        uses: actions/checkout@v5
-        with:
-          repository: openssl/openssl
-          path: openssl
-          ref: ${{ needs.versions.outputs.openssl-version }}
-      - if: runner.os == 'Linux'
-        name: build and install openssl with fPIC
-        shell: bash
-        working-directory: openssl
-        run: |
-          ./config -fPIC --prefix=$HOME/.local --openssldir=$HOME/.local/ssl
-          make -j $(nproc)
-          make install_sw -j $(nproc)
-      - if: runner.os == 'Linux'
-        name: build (linux)
-        shell: bash
-        run: |
-          env OPENSSL_LIB_DIR=$HOME/.local/lib64 OPENSSL_INCLUDE_DIR=$HOME/.local/include OPENSSL_STATIC=yes cargo build --release
-          mv target/release/ripunzip ripunzip-linux
-      - if: runner.os == 'Windows'
-        name: build (windows)
-        shell: bash
-        run: |
-          cargo build --release
-          mv target/release/ripunzip ripunzip-windows
-      - name: build (macOS)
-        if: runner.os == 'macOS'
-        shell: bash
-        run: |
-          rustup target install x86_64-apple-darwin
-          rustup target install aarch64-apple-darwin
-          cargo build --target x86_64-apple-darwin --release
-          cargo build --target aarch64-apple-darwin --release
-          lipo -create -output ripunzip-macos \
-            -arch x86_64 target/x86_64-apple-darwin/release/ripunzip \
-            -arch arm64 target/aarch64-apple-darwin/release/ripunzip
-      - name: Archive
-        shell: bash
-        run: |
-          tar acf ripunzip-$RUNNER_OS.tar.zst ripunzip-$(echo $RUNNER_OS | tr '[:upper:]' '[:lower:]')
-      - name: Upload built binary
-        uses: actions/upload-artifact@v4
-        with:
-          name: ripunzip-${{ runner.os }}
-          path: ripunzip-${{ runner.os }}.tar.zst
-          retention-days: 5
-          compression: 0
-      - name: Check built binary
-        shell: bash
-        run: |
-          rm -f ripunzip-*.tar.zst
-          ./ripunzip-* --version
-  publish:
-    needs: [versions, build]
-    if: inputs.open-pr == 'true'
-    permissions:
-      contents: write
-      pull-requests: write
-    runs-on: ubuntu-slim
-    steps:
-      # workaround for git-lfs not being installed yet on ubuntu-slim runners
-      - name: Ensure git-lfs is installed
-        shell: bash
-        run: |
-          if which git-lfs &>/dev/null; then
-            echo "git-lfs is already installed"
-            exit 0
-          fi
-          cd $TMP
-          gh release download --repo git-lfs/git-lfs --pattern "git-lfs-linux-amd64-*.tar.gz" --clobber
-          tar xzf git-lfs-linux-amd64-*.tar.gz
-          rm git-lfs-linux-amd64-*.tar.gz
-          cd git-lfs-*
-          pwd | tee -a $GITHUB_PATH
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - uses: actions/checkout@v5
-        with:
-          sparse-checkout: |
-            .github
-            misc/ripunzip
-          lfs: true
-      - name: Download built binaries
-        uses: actions/download-artifact@v4
-        with:
-          merge-multiple: true
-          path: misc/ripunzip
-      - name: Open PR
-        shell: bash
-        run: |
-          git config --global user.name "github-actions[bot]"
-          git config --global user.email "github-actions[bot]@users.noreply.github.com"
-          git switch -c update-ripunzip
-          git add misc/ripunzip
-          git commit -m "Update ripunzip binaries to version $VERSION"
-          git push --set-upstream origin update-ripunzip --force
-          TITLE="Update ripunzip binaries to version $VERSION"
-          gh pr create \
-            --draft \
-            --title "$TITLE" \
-            --body "Automated update of ripunzip binaries." \
-            --assignee "$ACTOR" ||
-              (gh pr edit --title "$TITLE" --add-assignee "$ACTOR" && gh pr ready --undo)
-        env:
-          ACTOR: ${{ github.actor }}
-          VERSION: ${{ needs.versions.outputs.ripunzip-version }}
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

MODULE.bazel

@@ -23,7 +23,7 @@ bazel_dep(name = "rules_shell", version = "0.5.0")
 bazel_dep(name = "bazel_skylib", version = "1.8.1")
 bazel_dep(name = "abseil-cpp", version = "20240116.1", repo_name = "absl")
 bazel_dep(name = "nlohmann_json", version = "3.11.3", repo_name = "json")
-bazel_dep(name = "fmt", version = "10.0.0")
+bazel_dep(name = "fmt", version = "12.1.0-codeql.1")
 bazel_dep(name = "rules_kotlin", version = "2.1.3-codeql.1")
 bazel_dep(name = "gazelle", version = "0.40.0")
 bazel_dep(name = "rules_dotnet", version = "0.19.2-codeql.1")
@@ -269,24 +269,16 @@ go_deps = use_extension("@gazelle//:extensions.bzl", "go_deps")
 go_deps.from_file(go_mod = "//go/extractor:go.mod")
 use_repo(go_deps, "org_golang_x_mod", "org_golang_x_tools")
 
-lfs_archive = use_repo_rule("//misc/bazel:lfs.bzl", "lfs_archive")
+ripunzip_archive = use_repo_rule("//misc/ripunzip:ripunzip.bzl", "ripunzip_archive")
 
-lfs_archive(
-    name = "ripunzip-linux",
-    src = "//misc/ripunzip:ripunzip-Linux.tar.zst",
-    build_file = "//misc/ripunzip:BUILD.ripunzip.bazel",
-)
-
-lfs_archive(
-    name = "ripunzip-windows",
-    src = "//misc/ripunzip:ripunzip-Windows.tar.zst",
-    build_file = "//misc/ripunzip:BUILD.ripunzip.bazel",
-)
-
-lfs_archive(
-    name = "ripunzip-macos",
-    src = "//misc/ripunzip:ripunzip-macOS.tar.zst",
-    build_file = "//misc/ripunzip:BUILD.ripunzip.bazel",
+# go to https://github.com/GoogleChrome/ripunzip/releases to find latest version and corresponding sha256s
+ripunzip_archive(
+    name = "ripunzip",
+    sha256_linux = "ee0e8a957687a5dc3a66b2a4b25883bf762df4c9c07f0651af527a32a405054b",
+    sha256_macos_arm = "8a88eea54eac232d162a72a42065e0429b82dbf4f05e9642915dff9d7a81f846",
+    sha256_macos_intel = "4457a18bfcc5feabe09f5ea3d1157128e07b4873392cb404a870e611924abf64",
+    sha256_windows = "66d0c1375301bf5ab815348048f43b110631d3fa7200acd50d50a8ed8655ca62",
+    version = "2.0.3",
 )
 
 register_toolchains(

actions/ql/lib/CHANGELOG.md

@@ -1,3 +1,15 @@
+## 0.4.24
+
+No user-facing changes.
+
+## 0.4.23
+
+No user-facing changes.
+
+## 0.4.22
+
+No user-facing changes.
+
 ## 0.4.21
 
 No user-facing changes.

actions/ql/lib/change-notes/released/0.4.22.md

@@ -0,0 +1,3 @@
+## 0.4.22
+
+No user-facing changes.

actions/ql/lib/change-notes/released/0.4.23.md

@@ -0,0 +1,3 @@
+## 0.4.23
+
+No user-facing changes.

actions/ql/lib/change-notes/released/0.4.24.md

@@ -0,0 +1,3 @@
+## 0.4.24
+
+No user-facing changes.

actions/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.4.21
+lastReleaseVersion: 0.4.24

actions/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/actions-all
-version: 0.4.21
+version: 0.4.24
 library: true
 warnOnImplicitThis: true
 dependencies:

actions/ql/src/CHANGELOG.md

@@ -1,3 +1,15 @@
+## 0.6.16
+
+No user-facing changes.
+
+## 0.6.15
+
+No user-facing changes.
+
+## 0.6.14
+
+No user-facing changes.
+
 ## 0.6.13
 
 No user-facing changes.

actions/ql/src/change-notes/released/0.6.14.md

@@ -0,0 +1,3 @@
+## 0.6.14
+
+No user-facing changes.

actions/ql/src/change-notes/released/0.6.15.md

@@ -0,0 +1,3 @@
+## 0.6.15
+
+No user-facing changes.

actions/ql/src/change-notes/released/0.6.16.md

@@ -0,0 +1,3 @@
+## 0.6.16
+
+No user-facing changes.

actions/ql/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.6.13
+lastReleaseVersion: 0.6.16

actions/ql/src/experimental/Security/CWE-829/ArtifactPoisoningPathTraversal.ql

@@ -1,5 +1,5 @@
 /**
- * @name Artifact Poisoning (Path Traversal).
+ * @name Artifact Poisoning (Path Traversal)
  * @description An attacker may be able to poison the workflow's artifacts and influence on consequent steps.
  * @kind problem
  * @problem.severity error

actions/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/actions-queries
-version: 0.6.13
+version: 0.6.16
 library: false
 warnOnImplicitThis: true
 groups: [actions, queries]

config/identical-files.json

@@ -276,5 +276,12 @@
   "Python model summaries test extension": [
     "python/ql/test/library-tests/dataflow/model-summaries/InlineTaintTest.ext.yml",
     "python/ql/test/library-tests/dataflow/model-summaries/NormalDataflowTest.ext.yml"
+  ],
+  "XML discard predicates": [
+    "javascript/ql/lib/semmle/javascript/internal/OverlayXml.qll",
+    "java/ql/lib/semmle/code/java/internal/OverlayXml.qll",
+    "go/ql/lib/semmle/go/internal/OverlayXml.qll",
+    "python/ql/lib/semmle/python/internal/OverlayXml.qll",
+    "csharp/ql/lib/semmle/code/csharp/internal/OverlayXml.qll"
   ]
 }

cpp/downgrades/a42ce5fc943254097f85471b94ae2247e819104a/upgrade.properties

@@ -0,0 +1,4 @@
+description: Add databaseMetadata and overlayChangedFiles relations
+compatibility: full
+databaseMetadata.rel: delete
+overlayChangedFiles.rel: delete

cpp/ql/lib/CHANGELOG.md

@@ -1,3 +1,17 @@
+## 6.1.3
+
+No user-facing changes.
+
+## 6.1.2
+
+No user-facing changes.
+
+## 6.1.1
+
+### Minor Analysis Improvements
+
+* The class `DataFlow::FieldContent` now covers both `union` and `struct`/`class` types. A new predicate `FieldContent.getAField` has been added to access the union members associated with the `FieldContent`. The old `FieldContent` has been renamed to `NonUnionFieldContent`.
+
 ## 6.1.0
 
 ### New Features

cpp/ql/lib/change-notes/released/6.1.1.md

@@ -0,0 +1,5 @@
+## 6.1.1
+
+### Minor Analysis Improvements
+
+* The class `DataFlow::FieldContent` now covers both `union` and `struct`/`class` types. A new predicate `FieldContent.getAField` has been added to access the union members associated with the `FieldContent`. The old `FieldContent` has been renamed to `NonUnionFieldContent`.

cpp/ql/lib/change-notes/released/6.1.2.md

@@ -0,0 +1,3 @@
+## 6.1.2
+
+No user-facing changes.

cpp/ql/lib/change-notes/released/6.1.3.md

@@ -0,0 +1,3 @@
+## 6.1.3
+
+No user-facing changes.

cpp/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 6.1.0
+lastReleaseVersion: 6.1.3

cpp/ql/lib/cpp.qll

@@ -74,3 +74,4 @@ import semmle.code.cpp.Preprocessor
 import semmle.code.cpp.Iteration
 import semmle.code.cpp.NameQualifiers
 import DefaultOptions
+private import semmle.code.cpp.internal.Overlay

cpp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-all
-version: 6.1.0
+version: 6.1.3
 groups: cpp
 dbscheme: semmlecode.cpp.dbscheme
 extractor: cpp
@@ -21,3 +21,4 @@ dataExtensions:
   - ext/deallocation/*.model.yml
   - ext/allocation/*.model.yml
 warnOnImplicitThis: true
+compileForOverlayEval: true

cpp/ql/lib/semmle/code/cpp/Compilation.qll

@@ -97,7 +97,7 @@ class Compilation extends @compilation {
   /**
    * Gets an expanded argument passed to the extractor on this invocation.
    */
-  string getAnExpandedArgument() { result = this.getArgument(_) }
+  string getAnExpandedArgument() { result = this.getExpandedArgument(_) }
 
   /**
    * Gets the `i`th expanded argument passed to the extractor on this
@@ -107,7 +107,11 @@ class Compilation extends @compilation {
    * includes the arguments from that file, rather than just taking the
    * argument literally.
    */
-  string getExpandedArgument(int i) { compilation_expanded_args(this, i, result) }
+  string getExpandedArgument(int i) {
+    if exists(string arg | compilation_expanded_args(this, _, arg))
+    then compilation_expanded_args(this, i, result)
+    else result = this.getArgument(i)
+  }
 
   /**
    * Gets the total amount of CPU time spent processing all the files in the

cpp/ql/lib/semmle/code/cpp/Function.qll

@@ -171,12 +171,14 @@ class Function extends Declaration, ControlFlowNode, AccessHolder, @function {
    * Gets the nth parameter of this function. There is no result for the
    * implicit `this` parameter, and there is no `...` varargs pseudo-parameter.
    */
+  pragma[nomagic]
   Parameter getParameter(int n) { params(unresolveElement(result), underlyingElement(this), n, _) }
 
   /**
    * Gets a parameter of this function. There is no result for the implicit
    * `this` parameter, and there is no `...` varargs pseudo-parameter.
    */
+  pragma[nomagic]
   Parameter getAParameter() { params(unresolveElement(result), underlyingElement(this), _, _) }
 
   /**

cpp/ql/lib/semmle/code/cpp/dataflow/ExternalFlow.qll

@@ -15,16 +15,17 @@
  * reading.
  * 1. The `namespace` column selects a namespace.
  * 2. The `type` column selects a type within that namespace. This column can
- *    introduce template names that can be mentioned in the `signature` column.
+ *    introduce template type names that can be mentioned in the `signature` column.
  *    For example, `vector<T,Allocator>` introduces the template names `T` and
- *    `Allocator`.
+ *    `Allocator`. Non-type template parameters cannot be specified.
  * 3. The `subtypes` is a boolean that indicates whether to jump to an
  *    arbitrary subtype of that type. Set this to `false` if leaving the `type`
  *    blank (for example, a free function).
  * 4. The `name` column optionally selects a specific named member of the type.
- *    Like the `type` column, this column can introduce template names that can
- *    be mentioned in the `signature` column. For example, `insert<InputIt>`
- *    introduces the template name `InputIt`.
+ *    Like the `type` column, this column can introduce template type names
+ *    that can be mentioned in the `signature` column. For example,
+ *    `insert<InputIt>` introduces the template name `InputIt`. Non-type
+ *    template parameters cannot be specified.
  * 5. The `signature` column optionally restricts the named member. If
  *    `signature` is blank then no such filtering is done. The format of the
  *    signature is a comma-separated list of types enclosed in parentheses. The
@@ -633,6 +634,28 @@ string getParameterTypeWithoutTemplateArguments(Function f, int n, boolean canon
   canonical = true
 }
 
+/**
+ * Gets the largest index of a template parameter of `templateFunction` that
+ * is a type template parameter.
+ */
+private int getLastTypeTemplateFunctionParameterIndex(Function templateFunction) {
+  result =
+    max(int index | templateFunction.getTemplateArgument(index) instanceof TypeTemplateParameter)
+}
+
+/** Gets the number of supported template parameters for `templateFunction`. */
+private int getNumberOfSupportedFunctionTemplateArguments(Function templateFunction) {
+  result = count(int i | exists(getSupportedFunctionTemplateArgument(templateFunction, i)) | i)
+}
+
+/** Gets the `i`'th supported template parameter for `templateFunction`. */
+private Locatable getSupportedFunctionTemplateArgument(Function templateFunction, int i) {
+  result = templateFunction.getTemplateArgument(i) and
+  // We don't yet support non-type template parameters in the middle of a
+  // template parameter list
+  i <= getLastTypeTemplateFunctionParameterIndex(templateFunction)
+}
+
 /**
  * Normalize the `n`'th parameter of `f` by replacing template names
  * with `func:N` (where `N` is the index of the template).
@@ -640,18 +663,41 @@ string getParameterTypeWithoutTemplateArguments(Function f, int n, boolean canon
 private string getTypeNameWithoutFunctionTemplates(Function f, int n, int remaining) {
   exists(Function templateFunction |
     templateFunction = getFullyTemplatedFunction(f) and
-    remaining = templateFunction.getNumberOfTemplateArguments() and
+    remaining = getNumberOfSupportedFunctionTemplateArguments(templateFunction) and
     result = getParameterTypeWithoutTemplateArguments(templateFunction, n, _)
   )
   or
   exists(string mid, TypeTemplateParameter tp, Function templateFunction |
     mid = getTypeNameWithoutFunctionTemplates(f, n, remaining + 1) and
     templateFunction = getFullyTemplatedFunction(f) and
-    tp = templateFunction.getTemplateArgument(remaining) and
+    tp = getSupportedFunctionTemplateArgument(templateFunction, remaining)
+  |
     result = mid.replaceAll(tp.getName(), "func:" + remaining.toString())
   )
 }
 
+/**
+ * Gets the largest index of a template parameter of `templateClass` that
+ * is a type template parameter.
+ */
+private int getLastTypeTemplateClassParameterIndex(Class templateClass) {
+  result =
+    max(int index | templateClass.getTemplateArgument(index) instanceof TypeTemplateParameter)
+}
+
+/** Gets the `i`'th supported template parameter for `templateClass`. */
+private Locatable getSupportedClassTemplateArgument(Class templateClass, int i) {
+  result = templateClass.getTemplateArgument(i) and
+  // We don't yet support non-type template parameters in the middle of a
+  // template parameter list
+  i <= getLastTypeTemplateClassParameterIndex(templateClass)
+}
+
+/** Gets the number of supported template parameters for `templateClass`. */
+private int getNumberOfSupportedClassTemplateArguments(Class templateClass) {
+  result = count(int i | exists(getSupportedClassTemplateArgument(templateClass, i)) | i)
+}
+
 /**
  * Normalize the `n`'th parameter of `f` by replacing template names
  * with `class:N` (where `N` is the index of the template).
@@ -661,7 +707,7 @@ private string getTypeNameWithoutClassTemplates(Function f, int n, int remaining
   // If there is a declaring type then we start by expanding the function templates
   exists(Class template |
     isClassConstructedFrom(f.getDeclaringType(), template) and
-    remaining = template.getNumberOfTemplateArguments() and
+    remaining = getNumberOfSupportedClassTemplateArguments(template) and
     result = getTypeNameWithoutFunctionTemplates(f, n, 0)
   )
   or
@@ -673,7 +719,8 @@ private string getTypeNameWithoutClassTemplates(Function f, int n, int remaining
   exists(string mid, TypeTemplateParameter tp, Class template |
     mid = getTypeNameWithoutClassTemplates(f, n, remaining + 1) and
     isClassConstructedFrom(f.getDeclaringType(), template) and
-    tp = template.getTemplateArgument(remaining) and
+    tp = getSupportedClassTemplateArgument(template, remaining)
+  |
     result = mid.replaceAll(tp.getName(), "class:" + remaining.toString())
   )
 }

cpp/ql/lib/semmle/code/cpp/exprs/Cast.qll

@@ -750,6 +750,16 @@ class SizeofPackTypeOperator extends SizeofPackOperator {
  */
 class SizeofOperator extends Expr, @runtime_sizeof {
   override int getPrecedence() { result = 16 }
+
+  /**
+   * Gets the contained type of this `sizeof`. For example,
+   * the result is `int` in both cases below:
+   * ```
+   * sizeof(int);
+   * sizeof(42);
+   * ```
+   */
+  Type getTypeOperand() { none() } // overridden in subclasses
 }
 
 /**
@@ -766,6 +776,8 @@ class SizeofExprOperator extends SizeofOperator {
   /** Gets the contained expression. */
   Expr getExprOperand() { result = this.getChild(0) }
 
+  override Type getTypeOperand() { result = this.getExprOperand().getType() }
+
   override string toString() { result = "sizeof(<expr>)" }
 
   override predicate mayBeImpure() { this.getExprOperand().mayBeImpure() }
@@ -784,8 +796,7 @@ class SizeofTypeOperator extends SizeofOperator {
 
   override string getAPrimaryQlClass() { result = "SizeofTypeOperator" }
 
-  /** Gets the contained type. */
-  Type getTypeOperand() { sizeof_bind(underlyingElement(this), unresolveElement(result)) }
+  override Type getTypeOperand() { sizeof_bind(underlyingElement(this), unresolveElement(result)) }
 
   override string toString() { result = "sizeof(" + this.getTypeOperand().getName() + ")" }
 
@@ -842,6 +853,16 @@ class AlignofTypeOperator extends AlignofOperator {
  */
 class DatasizeofOperator extends Expr, @datasizeof {
   override int getPrecedence() { result = 16 }
+
+  /**
+   * Gets the contained type of this `__datasizeof`. For example,
+   * the result is `int` in both cases below:
+   * ```
+   * __datasizeof(int);
+   * __datasizeof(42);
+   * ```
+   */
+  Type getTypeOperand() { none() }
 }
 
 /**
@@ -855,6 +876,8 @@ class DatasizeofExprOperator extends DatasizeofOperator {
   /** Gets the contained expression. */
   Expr getExprOperand() { result = this.getChild(0) }
 
+  override Type getTypeOperand() { result = this.getExprOperand().getType() }
+
   override string toString() { result = "__datasizeof(<expr>)" }
 
   override predicate mayBeImpure() { this.getExprOperand().mayBeImpure() }
@@ -870,8 +893,7 @@ class DatasizeofTypeOperator extends DatasizeofOperator {
 
   override string getAPrimaryQlClass() { result = "DatasizeofTypeOperator" }
 
-  /** Gets the contained type. */
-  Type getTypeOperand() { sizeof_bind(underlyingElement(this), unresolveElement(result)) }
+  override Type getTypeOperand() { sizeof_bind(underlyingElement(this), unresolveElement(result)) }
 
   override string toString() { result = "__datasizeof(" + this.getTypeOperand().getName() + ")" }
 

cpp/ql/lib/semmle/code/cpp/internal/Overlay.qll

@@ -0,0 +1,60 @@
+/**
+ * Defines entity discard predicates for C++ overlay analysis.
+ */
+
+/**
+ * Holds always for the overlay variant and never for the base variant.
+ * This local predicate is used to define local predicates that behave
+ * differently for the base and overlay variant.
+ */
+overlay[local]
+predicate isOverlay() { databaseMetadata("isOverlay", "true") }
+
+overlay[local]
+private string getLocationFilePath(@location_default loc) {
+  exists(@file file | locations_default(loc, file, _, _, _, _) | files(file, result))
+}
+
+/**
+ * Gets the file path for an element with a single location.
+ */
+overlay[local]
+private string getSingleLocationFilePath(@element e) {
+  // @var_decl has a direct location in the var_decls relation
+  exists(@location_default loc | var_decls(e, _, _, _, loc) | result = getLocationFilePath(loc))
+  //TODO: add other kinds of elements with single locations
+}
+
+/**
+ * Gets the file path for an element with potentially multiple locations.
+ */
+overlay[local]
+private string getMultiLocationFilePath(@element e) {
+  // @variable gets its location(s) from its @var_decl(s)
+  exists(@var_decl vd, @location_default loc | var_decls(vd, e, _, _, loc) |
+    result = getLocationFilePath(loc)
+  )
+  //TODO: add other kinds of elements with multiple locations
+}
+
+/**
+ * A local helper predicate that holds in the base variant and never in the
+ * overlay variant.
+ */
+overlay[local]
+private predicate holdsInBase() { not isOverlay() }
+
+/**
+ * Discards an element from the base variant if:
+ * - It has a single location in a changed file, or
+ * - All of its locations are in changed files.
+ */
+overlay[discard_entity]
+private predicate discardElement(@element e) {
+  holdsInBase() and
+  (
+    overlayChangedFiles(getSingleLocationFilePath(e))
+    or
+    forex(string path | path = getMultiLocationFilePath(e) | overlayChangedFiles(path))
+  )
+}

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowPrivate.qll

@@ -861,6 +861,10 @@ predicate jumpStep(Node n1, Node n2) {
     n2.(FlowSummaryNode).getSummaryNode())
 }
 
+bindingset[c]
+pragma[inline_late]
+private int getIndirectionIndexLate(Content c) { result = c.getIndirectionIndex() }
+
 /**
  * Holds if data can flow from `node1` to `node2` via an assignment to `f`.
  * Thus, `node2` references an object with a field `f` that contains the
@@ -873,23 +877,17 @@ predicate jumpStep(Node n1, Node n2) {
 predicate storeStepImpl(Node node1, Content c, Node node2, boolean certain) {
   exists(
     PostFieldUpdateNode postFieldUpdate, int indirectionIndex1, int numberOfLoads,
-    StoreInstruction store
+    StoreInstruction store, FieldContent fc
   |
     postFieldUpdate = node2 and
-    nodeHasInstruction(node1, store, pragma[only_bind_into](indirectionIndex1)) and
+    fc = c and
+    nodeHasInstruction(node1, pragma[only_bind_into](store),
+      pragma[only_bind_into](indirectionIndex1)) and
     postFieldUpdate.getIndirectionIndex() = 1 and
     numberOfLoadsFromOperand(postFieldUpdate.getFieldAddress(),
-      store.getDestinationAddressOperand(), numberOfLoads, certain)
-  |
-    exists(FieldContent fc | fc = c |
-      fc.getField() = postFieldUpdate.getUpdatedField() and
-      fc.getIndirectionIndex() = 1 + indirectionIndex1 + numberOfLoads
-    )
-    or
-    exists(UnionContent uc | uc = c |
-      uc.getAField() = postFieldUpdate.getUpdatedField() and
-      uc.getIndirectionIndex() = 1 + indirectionIndex1 + numberOfLoads
-    )
+      store.getDestinationAddressOperand(), numberOfLoads, certain) and
+    fc.getAField() = postFieldUpdate.getUpdatedField() and
+    getIndirectionIndexLate(fc) = 1 + indirectionIndex1 + numberOfLoads
   )
   or
   // models-as-data summarized flow
@@ -965,22 +963,17 @@ predicate nodeHasInstruction(Node node, Instruction instr, int indirectionIndex)
  * `node2`.
  */
 predicate readStep(Node node1, ContentSet c, Node node2) {
-  exists(FieldAddress fa1, Operand operand, int numberOfLoads, int indirectionIndex2 |
+  exists(
+    FieldAddress fa1, Operand operand, int numberOfLoads, int indirectionIndex2, FieldContent fc
+  |
+    fc = c and
     nodeHasOperand(node2, operand, indirectionIndex2) and
     // The `1` here matches the `node2.getIndirectionIndex() = 1` conjunct
     // in `storeStep`.
     nodeHasOperand(node1, fa1.getObjectAddressOperand(), 1) and
-    numberOfLoadsFromOperand(fa1, operand, numberOfLoads, _)
-  |
-    exists(FieldContent fc | fc = c |
-      fc.getField() = fa1.getField() and
-      fc.getIndirectionIndex() = indirectionIndex2 + numberOfLoads
-    )
-    or
-    exists(UnionContent uc | uc = c |
-      uc.getAField() = fa1.getField() and
-      uc.getIndirectionIndex() = indirectionIndex2 + numberOfLoads
-    )
+    numberOfLoadsFromOperand(fa1, operand, numberOfLoads, _) and
+    fc.getAField() = fa1.getField() and
+    getIndirectionIndexLate(fc) = indirectionIndex2 + numberOfLoads
   )
   or
   // models-as-data summarized flow
@@ -1574,7 +1567,7 @@ pragma[inline]
 ContentApprox getContentApprox(Content c) {
   exists(string prefix, Field f |
     prefix = result.(FieldApproxContent).getPrefix() and
-    f = c.(FieldContent).getField() and
+    f = c.(NonUnionFieldContent).getField() and
     fieldHasApproxName(f, prefix)
   )
   or

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowUtil.qll

@@ -2078,38 +2078,151 @@ predicate localExprFlow(Expr e1, Expr e2) {
   localExprFlowPlus(e1, e2)
 }
 
+/**
+ * A canonical representation of a field.
+ *
+ * For performance reasons we want a unique `Content` that represents
+ * a given field across any template instantiation of a class.
+ *
+ * This is possible in _almost_ all cases, but there are cases where it is
+ * not possible to map between a field in the uninstantiated template to a
+ * field in the instantiated template. This happens in the case of local class
+ * definitions (because the local class is not the template that constructs
+ * the instantiation - it is the enclosing function). So this abstract class
+ * has two implementations: a non-local case (where we can represent a
+ * canonical field as the field declaration from an uninstantiated class
+ * template or a non-templated class), and a local case (where we simply use
+ * the field from the instantiated class).
+ */
+abstract private class CanonicalField extends Field {
+  /** Gets a field represented by this canonical field. */
+  abstract Field getAField();
+
+  /**
+   * Gets a class that declares a field represented by this canonical field.
+   */
+  abstract Class getADeclaringType();
+
+  /**
+   * Gets a type that this canonical field may have. Note that this may
+   * not be a unique type. For example, consider this case:
+   * ```
+   * template<typename T>
+   * struct S { T x; };
+   *
+   * S<int> s1;
+   * S<char> s2;
+   * ```
+   * In this case the canonical field corresponding to `S::x` has two types:
+   * `int` and `char`.
+   */
+  Type getAType() { result = this.getAField().getType() }
+
+  Type getAnUnspecifiedType() { result = this.getAType().getUnspecifiedType() }
+}
+
+private class NonLocalCanonicalField extends CanonicalField {
+  Class declaringType;
+
+  NonLocalCanonicalField() {
+    declaringType = this.getDeclaringType() and
+    not declaringType.isFromTemplateInstantiation(_) and
+    not declaringType.isLocal() // handled in LocalCanonicalField
+  }
+
+  override Field getAField() {
+    exists(Class c | result.getDeclaringType() = c |
+      // Either the declaring class of the field is a template instantiation
+      // that has been constructed from this canonical declaration
+      c.isConstructedFrom(declaringType) and
+      pragma[only_bind_out](result.getName()) = pragma[only_bind_out](this.getName())
+      or
+      // or this canonical declaration is not a template.
+      not c.isConstructedFrom(_) and
+      result = this
+    )
+  }
+
+  override Class getADeclaringType() {
+    result = this.getDeclaringType()
+    or
+    result.isConstructedFrom(this.getDeclaringType())
+  }
+}
+
+private class LocalCanonicalField extends CanonicalField {
+  Class declaringType;
+
+  LocalCanonicalField() {
+    declaringType = this.getDeclaringType() and
+    declaringType.isLocal()
+  }
+
+  override Field getAField() { result = this }
+
+  override Class getADeclaringType() { result = declaringType }
+}
+
+/**
+ * A canonical representation of a `Union`. See `CanonicalField` for the explanation for
+ * why we need a canonical representation.
+ */
+abstract private class CanonicalUnion extends Union {
+  /** Gets a union represented by this canonical union. */
+  abstract Union getAUnion();
+
+  /** Gets a canonical field of this canonical union. */
+  CanonicalField getACanonicalField() { result.getDeclaringType() = this }
+}
+
+private class NonLocalCanonicalUnion extends CanonicalUnion {
+  NonLocalCanonicalUnion() { not this.isFromTemplateInstantiation(_) and not this.isLocal() }
+
+  override Union getAUnion() {
+    result = this
+    or
+    result.isConstructedFrom(this)
+  }
+}
+
+private class LocalCanonicalUnion extends CanonicalUnion {
+  LocalCanonicalUnion() { this.isLocal() }
+
+  override Union getAUnion() { result = this }
+}
+
 bindingset[f]
 pragma[inline_late]
-private int getFieldSize(Field f) { result = f.getType().getSize() }
+private int getFieldSize(CanonicalField f) { result = max(f.getAType().getSize()) }
 
 /**
  * Gets a field in the union `u` whose size
  * is `bytes` number of bytes.
  */
-private Field getAFieldWithSize(Union u, int bytes) {
-  result = u.getAField() and
+private CanonicalField getAFieldWithSize(CanonicalUnion u, int bytes) {
+  result = u.getACanonicalField() and
   bytes = getFieldSize(result)
 }
 
 cached
 private newtype TContent =
-  TFieldContent(Field f, int indirectionIndex) {
-    // the indirection index for field content starts at 1 (because `TFieldContent` is thought of as
+  TNonUnionContent(CanonicalField f, int indirectionIndex) {
+    // the indirection index for field content starts at 1 (because `TNonUnionContent` is thought of as
     // the address of the field, `FieldAddress` in the IR).
-    indirectionIndex = [1 .. SsaImpl::getMaxIndirectionsForType(f.getUnspecifiedType())] and
+    indirectionIndex = [1 .. max(SsaImpl::getMaxIndirectionsForType(f.getAnUnspecifiedType()))] and
     // Reads and writes of union fields are tracked using `UnionContent`.
     not f.getDeclaringType() instanceof Union
   } or
-  TUnionContent(Union u, int bytes, int indirectionIndex) {
-    exists(Field f |
-      f = u.getAField() and
+  TUnionContent(CanonicalUnion u, int bytes, int indirectionIndex) {
+    exists(CanonicalField f |
+      f = u.getACanonicalField() and
       bytes = getFieldSize(f) and
       // We key `UnionContent` by the union instead of its fields since a write to one
       // field can be read by any read of the union's fields. Again, the indirection index
       // is 1-based (because 0 is considered the address).
       indirectionIndex =
         [1 .. max(SsaImpl::getMaxIndirectionsForType(getAFieldWithSize(u, bytes)
-                    .getUnspecifiedType())
+                    .getAnUnspecifiedType())
           )]
     )
   } or
@@ -2124,14 +2237,14 @@ private newtype TContent =
  */
 class Content extends TContent {
   /** Gets a textual representation of this element. */
-  abstract string toString();
+  string toString() { none() } // overridden in subclasses
 
   predicate hasLocationInfo(string path, int sl, int sc, int el, int ec) {
     path = "" and sl = 0 and sc = 0 and el = 0 and ec = 0
   }
 
   /** Gets the indirection index of this `Content`. */
-  abstract int getIndirectionIndex();
+  int getIndirectionIndex() { none() } // overridden in subclasses
 
   /**
    * INTERNAL: Do not use.
@@ -2142,7 +2255,7 @@ class Content extends TContent {
    * For example, a write to a field `f` implies that any content of
    * the form `*f` is also cleared.
    */
-  abstract predicate impliesClearOf(Content c);
+  predicate impliesClearOf(Content c) { none() } // overridden in subclasses
 }
 
 /**
@@ -2162,37 +2275,62 @@ private module ContentStars {
 
 private import ContentStars
 
-/** A reference through a non-union instance field. */
+private class TFieldContent = TNonUnionContent or TUnionContent;
+
+/**
+ * A `Content` that references a `Field`. This may be a field of a `struct`,
+ * `class`, or `union`. In the case of a `union` there may be multiple fields
+ * associated with the same `Content`.
+ */
 class FieldContent extends Content, TFieldContent {
-  private Field f;
+  /** Gets a `Field` of this `Content`. */
+  Field getAField() { none() }
+
+  /**
+   * Gets the field associated with this `Content`, if a unique one exists.
+   *
+   * For fields from template instantiations this predicate may still return
+   * more than one field, but all the fields will be constructed from the same
+   * template.
+   */
+  Field getField() { none() } // overridden in subclasses
+
+  override int getIndirectionIndex() { none() } // overridden in subclasses
+
+  override string toString() { none() } // overridden in subclasses
+
+  override predicate impliesClearOf(Content c) { none() } // overridden in subclasses
+}
+
+/** A reference through a non-union instance field. */
+class NonUnionFieldContent extends FieldContent, TNonUnionContent {
+  private CanonicalField f;
   private int indirectionIndex;
 
-  FieldContent() { this = TFieldContent(f, indirectionIndex) }
+  NonUnionFieldContent() { this = TNonUnionContent(f, indirectionIndex) }
 
   override string toString() { result = contentStars(this) + f.toString() }
 
-  Field getField() { result = f }
+  final override Field getField() { result = f.getAField() }
+
+  override Field getAField() { result = this.getField() }
 
   /** Gets the indirection index of this `FieldContent`. */
-  pragma[inline]
-  override int getIndirectionIndex() {
-    pragma[only_bind_into](result) = pragma[only_bind_out](indirectionIndex)
-  }
+  override int getIndirectionIndex() { result = indirectionIndex }
 
   override predicate impliesClearOf(Content c) {
-    exists(FieldContent fc |
-      fc = c and
-      fc.getField() = f and
+    exists(int i |
+      c = TNonUnionContent(f, i) and
       // If `this` is `f` then `c` is cleared if it's of the
       // form `*f`, `**f`, etc.
-      fc.getIndirectionIndex() >= indirectionIndex
+      i >= indirectionIndex
     )
   }
 }
 
 /** A reference through an instance field of a union. */
-class UnionContent extends Content, TUnionContent {
-  private Union u;
+class UnionContent extends FieldContent, TUnionContent {
+  private CanonicalUnion u;
   private int indirectionIndex;
   private int bytes;
 
@@ -2200,27 +2338,31 @@ class UnionContent extends Content, TUnionContent {
 
   override string toString() { result = contentStars(this) + u.toString() }
 
+  final override Field getField() { result = unique( | | u.getACanonicalField()).getAField() }
+
   /** Gets a field of the underlying union of this `UnionContent`, if any. */
-  Field getAField() { result = u.getAField() and getFieldSize(result) = bytes }
-
-  /** Gets the underlying union of this `UnionContent`. */
-  Union getUnion() { result = u }
-
-  /** Gets the indirection index of this `UnionContent`. */
-  pragma[inline]
-  override int getIndirectionIndex() {
-    pragma[only_bind_into](result) = pragma[only_bind_out](indirectionIndex)
+  override Field getAField() {
+    exists(CanonicalField cf |
+      cf = u.getACanonicalField() and
+      result = cf.getAField() and
+      getFieldSize(cf) = bytes
+    )
   }
 
+  /** Gets the underlying union of this `UnionContent`. */
+  Union getUnion() { result = u.getAUnion() }
+
+  /** Gets the indirection index of this `UnionContent`. */
+  override int getIndirectionIndex() { result = indirectionIndex }
+
   override predicate impliesClearOf(Content c) {
-    exists(UnionContent uc |
-      uc = c and
-      uc.getUnion() = u and
+    exists(int i |
+      c = TUnionContent(u, _, i) and
       // If `this` is `u` then `c` is cleared if it's of the
       // form `*u`, `**u`, etc. (and we ignore `bytes` because
       // we know the entire union is overwritten because it's a
       // union).
-      uc.getIndirectionIndex() >= indirectionIndex
+      i >= indirectionIndex
     )
   }
 }
@@ -2234,10 +2376,7 @@ class ElementContent extends Content, TElementContent {
 
   ElementContent() { this = TElementContent(indirectionIndex) }
 
-  pragma[inline]
-  override int getIndirectionIndex() {
-    pragma[only_bind_into](result) = pragma[only_bind_out](indirectionIndex)
-  }
+  override int getIndirectionIndex() { result = indirectionIndex }
 
   override predicate impliesClearOf(Content c) { none() }
 

cpp/ql/lib/semmlecode.cpp.dbscheme

@@ -1,3 +1,4 @@
+
 /*- Compilations -*/
 
 /**
@@ -2378,6 +2379,24 @@ link_parent(
     int link_target : @link_target ref
 );
 
+/**
+ * The CLI will automatically emit applicable tuples for this table,
+ * such as `databaseMetadata("isOverlay", "true")` when building an
+ * overlay database.
+ */
+databaseMetadata(
+  string metadataKey: string ref,
+  string value: string ref
+);
+
+/**
+ * The CLI will automatically emit tuples for each new/modified/deleted file
+ * when building an overlay database.
+ */
+overlayChangedFiles(
+  string path: string ref
+);
+
 /*- XML Files -*/
 
 xmlEncoding(

cpp/ql/lib/upgrades/1a6854060d5d3ada16c580a29f8c5ce21f3367f8/upgrade.properties

@@ -0,0 +1,2 @@
+description: Add databaseMetadata and overlayChangedFiles relations
+compatibility: full

cpp/ql/src/CHANGELOG.md

@@ -1,3 +1,15 @@
+## 1.5.7
+
+No user-facing changes.
+
+## 1.5.6
+
+No user-facing changes.
+
+## 1.5.5
+
+No user-facing changes.
+
 ## 1.5.4
 
 No user-facing changes.

cpp/ql/src/change-notes/released/1.5.5.md

@@ -0,0 +1,3 @@
+## 1.5.5
+
+No user-facing changes.

cpp/ql/src/change-notes/released/1.5.6.md

@@ -0,0 +1,3 @@
+## 1.5.6
+
+No user-facing changes.

cpp/ql/src/change-notes/released/1.5.7.md

@@ -0,0 +1,3 @@
+## 1.5.7
+
+No user-facing changes.

cpp/ql/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 1.5.4
+lastReleaseVersion: 1.5.7

cpp/ql/src/experimental/Security/CWE/CWE-125/DangerousWorksWithMultibyteOrWideCharacters.ql

@@ -1,5 +1,5 @@
 /**
- * @name Dangerous use convert function.
+ * @name Dangerous use convert function
  * @description Using convert function with an invalid length argument can result in an out-of-bounds access error or unexpected result.
  * @kind problem
  * @id cpp/dangerous-use-convert-function

cpp/ql/src/experimental/Security/CWE/CWE-190/DangerousUseOfTransformationAfterOperation.ql

@@ -1,5 +1,5 @@
 /**
- * @name Dangerous use of transformation after operation.
+ * @name Dangerous use of transformation after operation
  * @description By using the transformation after the operation, you are doing a pointless and dangerous action.
  * @kind problem
  * @id cpp/dangerous-use-of-transformation-after-operation

cpp/ql/src/experimental/Security/CWE/CWE-200/ExposureSensitiveInformationUnauthorizedActor.ql

@@ -1,5 +1,5 @@
 /**
- * @name Writing to a file without setting permissions.
+ * @name Writing to a file without setting permissions
  * @description Lack of restriction on file access rights can be unsafe.
  * @kind problem
  * @id cpp/work-with-file-without-permissions-rights

cpp/ql/src/experimental/Security/CWE/CWE-243/IncorrectChangingWorkingDirectory.ql

@@ -1,5 +1,5 @@
 /**
- * @name Find work with changing working directories, with security errors.
+ * @name Find work with changing working directories, with security errors
  * @description Not validating the return value or pinning the directory can be unsafe.
  * @kind problem
  * @id cpp/work-with-changing-working-directories

cpp/ql/src/experimental/Security/CWE/CWE-266/IncorrectPrivilegeAssignment.ql

@@ -1,5 +1,5 @@
 /**
- * @name Find the wrong use of the umask function.
+ * @name Find the wrong use of the umask function
  * @description Incorrectly evaluated argument to the umask function may have security implications.
  * @kind problem
  * @id cpp/wrong-use-of-the-umask

cpp/ql/src/experimental/Security/CWE/CWE-377/InsecureTemporaryFile.ql

@@ -1,5 +1,5 @@
 /**
- * @name Insecure generation of filenames.
+ * @name Insecure generation of filenames
  * @description Using a predictable filename when creating a temporary file can lead to an attacker-controlled input.
  * @kind problem
  * @id cpp/insecure-generation-of-filename

cpp/ql/src/experimental/Security/CWE/CWE-476/DangerousUseOfExceptionBlocks.ql

@@ -1,5 +1,5 @@
 /**
- * @name Dangerous use of exception blocks.
+ * @name Dangerous use of exception blocks
  * @description When clearing the data in the catch block, you must be sure that the memory was allocated before the exception.
  * @kind problem
  * @id cpp/dangerous-use-of-exception-blocks

cpp/ql/src/experimental/Security/CWE/CWE-670/DangerousUseSSL_shutdown.ql

@@ -1,5 +1,5 @@
 /**
- * @name Dangerous use SSL_shutdown.
+ * @name Dangerous use SSL_shutdown
  * @description Incorrect closing of the connection leads to the creation of different states for the server and client, which can be exploited by an attacker.
  * @kind problem
  * @id cpp/dangerous-use-of-ssl-shutdown

cpp/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-queries
-version: 1.5.4
+version: 1.5.7
 groups:
   - cpp
   - queries

cpp/ql/src/utils/modelgenerator/CaptureContentSummaryModels.ql

@@ -1,5 +1,5 @@
 /**
- * @name Capture content based summary models.
+ * @name Capture content based summary models
  * @description Finds applicable content based summary models to be used by other queries.
  * @kind diagnostic
  * @id cpp/utils/modelgenerator/contentbased-summary-models

cpp/ql/src/utils/modelgenerator/CaptureNeutralModels.ql

@@ -1,5 +1,5 @@
 /**
- * @name Capture neutral models.
+ * @name Capture neutral models
  * @description Finds neutral models to be used by other queries.
  * @kind diagnostic
  * @id cpp/utils/modelgenerator/neutral-models

cpp/ql/src/utils/modelgenerator/CaptureSinkModels.ql

@@ -1,5 +1,5 @@
 /**
- * @name Capture sink models.
+ * @name Capture sink models
  * @description Finds public methods that act as sinks as they flow into a known sink.
  * @kind diagnostic
  * @id cpp/utils/modelgenerator/sink-models

cpp/ql/src/utils/modelgenerator/CaptureSourceModels.ql

@@ -1,5 +1,5 @@
 /**
- * @name Capture source models.
+ * @name Capture source models
  * @description Finds APIs that act as sources as they expose already known sources.
  * @kind diagnostic
  * @id cpp/utils/modelgenerator/source-models

cpp/ql/src/utils/modelgenerator/CaptureSummaryModels.ql

@@ -1,5 +1,5 @@
 /**
- * @name Capture summary models.
+ * @name Capture summary models
  * @description Finds applicable summary models to be used by other queries.
  * @kind diagnostic
  * @id cpp/utils/modelgenerator/summary-models

cpp/ql/src/utils/modelgenerator/internal/CaptureModels.qll

@@ -190,7 +190,7 @@ module ModelGeneratorCommonInput implements ModelGeneratorCommonInputSig<Cpp::Lo
   predicate isRelevantType(Type t) { any() }
 
   Type getUnderlyingContentType(DataFlow::ContentSet c) {
-    result = c.(DataFlow::FieldContent).getField().getUnspecifiedType() or
+    result = c.(DataFlow::NonUnionFieldContent).getField().getUnspecifiedType() or
     result = c.(DataFlow::UnionContent).getUnion().getUnspecifiedType()
   }
 
@@ -340,12 +340,7 @@ private module SummaryModelGeneratorInput implements SummaryModelGeneratorInputS
     )
   }
 
-  predicate isField(DataFlow::ContentSet cs) {
-    exists(DataFlow::Content c | cs.isSingleton(c) |
-      c instanceof DataFlow::FieldContent or
-      c instanceof DataFlow::UnionContent
-    )
-  }
+  predicate isField(DataFlow::ContentSet cs) { cs.isSingleton(any(DataFlow::FieldContent fc)) }
 
   predicate isCallback(DataFlow::ContentSet c) { none() }
 

cpp/ql/test/library-tests/attributes/deprecated_with_msg/deprecated_with_msg.expected

@@ -1,4 +1,2 @@
 | clang421.c:1:12:1:19 | clang421 | 0 |
 | clang450.c:1:12:1:19 | clang450 | 1 |
-| gcc421.c:1:12:1:17 | gcc421 | 0 |
-| gcc450.c:1:12:1:17 | gcc450 | 1 |

cpp/ql/test/library-tests/attributes/deprecated_with_msg/gcc421.c

@@ -1,2 +0,0 @@
-static int gcc421 = __has_feature(attribute_deprecated_with_message);
-// semmle-extractor-options: --gnu_version 40201

cpp/ql/test/library-tests/attributes/deprecated_with_msg/gcc450.c

@@ -1,2 +0,0 @@
-static int gcc450 = __has_feature(attribute_deprecated_with_message);
-// semmle-extractor-options: --gnu_version 40500

cpp/ql/test/library-tests/dataflow/external-models/flow.expected

@@ -30,13 +30,14 @@ models
 | 29 | Summary: ; ; false; RtlMoveMemory; ; ; Argument[*@1]; Argument[*@0]; value; manual |
 | 30 | Summary: ; ; false; RtlMoveVolatileMemory; ; ; Argument[*@1]; Argument[*@0]; value; manual |
 | 31 | Summary: ; ; false; callWithArgument; ; ; Argument[1]; Argument[0].Parameter[0]; value; manual |
-| 32 | Summary: ; ; false; pthread_create; ; ; Argument[@3]; Argument[2].Parameter[@0]; value; manual |
-| 33 | Summary: ; ; false; ymlStepGenerated; ; ; Argument[0]; ReturnValue; taint; df-generated |
-| 34 | Summary: ; ; false; ymlStepManual; ; ; Argument[0]; ReturnValue; taint; manual |
-| 35 | Summary: ; ; false; ymlStepManual_with_body; ; ; Argument[0]; ReturnValue; taint; manual |
-| 36 | Summary: boost::asio; ; false; buffer; ; ; Argument[*0]; ReturnValue; taint; manual |
+| 32 | Summary: ; ; false; callWithNonTypeTemplate<T>; (const T &); ; Argument[*0]; ReturnValue; value; manual |
+| 33 | Summary: ; ; false; pthread_create; ; ; Argument[@3]; Argument[2].Parameter[@0]; value; manual |
+| 34 | Summary: ; ; false; ymlStepGenerated; ; ; Argument[0]; ReturnValue; taint; df-generated |
+| 35 | Summary: ; ; false; ymlStepManual; ; ; Argument[0]; ReturnValue; taint; manual |
+| 36 | Summary: ; ; false; ymlStepManual_with_body; ; ; Argument[0]; ReturnValue; taint; manual |
+| 37 | Summary: boost::asio; ; false; buffer; ; ; Argument[*0]; ReturnValue; taint; manual |
 edges
-| asio_streams.cpp:56:18:56:23 | [summary param] *0 in buffer | asio_streams.cpp:56:18:56:23 | [summary] to write: ReturnValue in buffer | provenance | MaD:36 |
+| asio_streams.cpp:56:18:56:23 | [summary param] *0 in buffer | asio_streams.cpp:56:18:56:23 | [summary] to write: ReturnValue in buffer | provenance | MaD:37 |
 | asio_streams.cpp:87:34:87:44 | read_until output argument | asio_streams.cpp:91:7:91:17 | recv_buffer | provenance | Src:MaD:17  |
 | asio_streams.cpp:87:34:87:44 | read_until output argument | asio_streams.cpp:93:29:93:39 | *recv_buffer | provenance | Src:MaD:17 Sink:MaD:2 |
 | asio_streams.cpp:97:37:97:44 | call to source | asio_streams.cpp:98:7:98:14 | send_str | provenance | TaintFunction |
@@ -45,10 +46,10 @@ edges
 | asio_streams.cpp:100:44:100:62 | call to buffer | asio_streams.cpp:101:7:101:17 | send_buffer | provenance |  |
 | asio_streams.cpp:100:44:100:62 | call to buffer | asio_streams.cpp:103:29:103:39 | *send_buffer | provenance | Sink:MaD:2 |
 | asio_streams.cpp:100:64:100:71 | *send_str | asio_streams.cpp:56:18:56:23 | [summary param] *0 in buffer | provenance |  |
-| asio_streams.cpp:100:64:100:71 | *send_str | asio_streams.cpp:100:44:100:62 | call to buffer | provenance | MaD:36 |
-| test.cpp:4:5:4:17 | [summary param] 0 in ymlStepManual | test.cpp:4:5:4:17 | [summary] to write: ReturnValue in ymlStepManual | provenance | MaD:34 |
-| test.cpp:5:5:5:20 | [summary param] 0 in ymlStepGenerated | test.cpp:5:5:5:20 | [summary] to write: ReturnValue in ymlStepGenerated | provenance | MaD:33 |
-| test.cpp:6:5:6:27 | [summary param] 0 in ymlStepManual_with_body | test.cpp:6:5:6:27 | [summary] to write: ReturnValue in ymlStepManual_with_body | provenance | MaD:35 |
+| asio_streams.cpp:100:64:100:71 | *send_str | asio_streams.cpp:100:44:100:62 | call to buffer | provenance | MaD:37 |
+| test.cpp:4:5:4:17 | [summary param] 0 in ymlStepManual | test.cpp:4:5:4:17 | [summary] to write: ReturnValue in ymlStepManual | provenance | MaD:35 |
+| test.cpp:5:5:5:20 | [summary param] 0 in ymlStepGenerated | test.cpp:5:5:5:20 | [summary] to write: ReturnValue in ymlStepGenerated | provenance | MaD:34 |
+| test.cpp:6:5:6:27 | [summary param] 0 in ymlStepManual_with_body | test.cpp:6:5:6:27 | [summary] to write: ReturnValue in ymlStepManual_with_body | provenance | MaD:36 |
 | test.cpp:7:47:7:52 | value2 | test.cpp:7:64:7:69 | value2 | provenance |  |
 | test.cpp:7:64:7:69 | value2 | test.cpp:7:5:7:30 | *ymlStepGenerated_with_body | provenance |  |
 | test.cpp:10:10:10:18 | call to ymlSource | test.cpp:10:10:10:18 | call to ymlSource | provenance | Src:MaD:16  |
@@ -60,15 +61,15 @@ edges
 | test.cpp:17:10:17:22 | call to ymlStepManual | test.cpp:17:10:17:22 | call to ymlStepManual | provenance |  |
 | test.cpp:17:10:17:22 | call to ymlStepManual | test.cpp:18:10:18:10 | y | provenance | Sink:MaD:1 |
 | test.cpp:17:24:17:24 | x | test.cpp:4:5:4:17 | [summary param] 0 in ymlStepManual | provenance |  |
-| test.cpp:17:24:17:24 | x | test.cpp:17:10:17:22 | call to ymlStepManual | provenance | MaD:34 |
+| test.cpp:17:24:17:24 | x | test.cpp:17:10:17:22 | call to ymlStepManual | provenance | MaD:35 |
 | test.cpp:21:10:21:25 | call to ymlStepGenerated | test.cpp:21:10:21:25 | call to ymlStepGenerated | provenance |  |
 | test.cpp:21:10:21:25 | call to ymlStepGenerated | test.cpp:22:10:22:10 | z | provenance | Sink:MaD:1 |
 | test.cpp:21:27:21:27 | x | test.cpp:5:5:5:20 | [summary param] 0 in ymlStepGenerated | provenance |  |
-| test.cpp:21:27:21:27 | x | test.cpp:21:10:21:25 | call to ymlStepGenerated | provenance | MaD:33 |
+| test.cpp:21:27:21:27 | x | test.cpp:21:10:21:25 | call to ymlStepGenerated | provenance | MaD:34 |
 | test.cpp:25:11:25:33 | call to ymlStepManual_with_body | test.cpp:25:11:25:33 | call to ymlStepManual_with_body | provenance |  |
 | test.cpp:25:11:25:33 | call to ymlStepManual_with_body | test.cpp:26:10:26:11 | y2 | provenance | Sink:MaD:1 |
 | test.cpp:25:35:25:35 | x | test.cpp:6:5:6:27 | [summary param] 0 in ymlStepManual_with_body | provenance |  |
-| test.cpp:25:35:25:35 | x | test.cpp:25:11:25:33 | call to ymlStepManual_with_body | provenance | MaD:35 |
+| test.cpp:25:35:25:35 | x | test.cpp:25:11:25:33 | call to ymlStepManual_with_body | provenance | MaD:36 |
 | test.cpp:32:11:32:36 | call to ymlStepGenerated_with_body | test.cpp:32:11:32:36 | call to ymlStepGenerated_with_body | provenance |  |
 | test.cpp:32:11:32:36 | call to ymlStepGenerated_with_body | test.cpp:33:10:33:11 | z2 | provenance | Sink:MaD:1 |
 | test.cpp:32:41:32:41 | x | test.cpp:7:47:7:52 | value2 | provenance |  |
@@ -76,7 +77,7 @@ edges
 | test.cpp:46:30:46:32 | *arg [x] | test.cpp:47:12:47:19 | *arg [x] | provenance |  |
 | test.cpp:47:12:47:19 | *arg [x] | test.cpp:48:13:48:13 | *s [x] | provenance |  |
 | test.cpp:48:13:48:13 | *s [x] | test.cpp:48:16:48:16 | x | provenance | Sink:MaD:1 |
-| test.cpp:52:5:52:18 | [summary param] *3 in pthread_create [x] | test.cpp:52:5:52:18 | [summary] to write: Argument[2].Parameter[*0] in pthread_create [x] | provenance | MaD:32 |
+| test.cpp:52:5:52:18 | [summary param] *3 in pthread_create [x] | test.cpp:52:5:52:18 | [summary] to write: Argument[2].Parameter[*0] in pthread_create [x] | provenance | MaD:33 |
 | test.cpp:52:5:52:18 | [summary] to write: Argument[2].Parameter[*0] in pthread_create [x] | test.cpp:46:30:46:32 | *arg [x] | provenance |  |
 | test.cpp:56:2:56:2 | *s [post update] [x] | test.cpp:59:55:59:64 | *& ... [x] | provenance |  |
 | test.cpp:56:2:56:18 | ... = ... | test.cpp:56:2:56:2 | *s [post update] [x] | provenance |  |
@@ -103,6 +104,13 @@ edges
 | test.cpp:101:26:101:26 | x | test.cpp:63:6:63:21 | [summary param] 1 in callWithArgument | provenance |  |
 | test.cpp:103:63:103:63 | x | test.cpp:63:6:63:21 | [summary param] 1 in callWithArgument | provenance |  |
 | test.cpp:104:62:104:62 | x | test.cpp:63:6:63:21 | [summary param] 1 in callWithArgument | provenance |  |
+| test.cpp:111:3:111:25 | [summary param] *0 in callWithNonTypeTemplate | test.cpp:111:3:111:25 | [summary] to write: ReturnValue in callWithNonTypeTemplate | provenance | MaD:32 |
+| test.cpp:114:10:114:18 | call to ymlSource | test.cpp:114:10:114:18 | call to ymlSource | provenance | Src:MaD:16  |
+| test.cpp:114:10:114:18 | call to ymlSource | test.cpp:118:44:118:44 | *x | provenance |  |
+| test.cpp:118:11:118:42 | call to callWithNonTypeTemplate | test.cpp:118:11:118:42 | call to callWithNonTypeTemplate | provenance |  |
+| test.cpp:118:11:118:42 | call to callWithNonTypeTemplate | test.cpp:119:10:119:11 | y2 | provenance | Sink:MaD:1 |
+| test.cpp:118:44:118:44 | *x | test.cpp:111:3:111:25 | [summary param] *0 in callWithNonTypeTemplate | provenance |  |
+| test.cpp:118:44:118:44 | *x | test.cpp:118:11:118:42 | call to callWithNonTypeTemplate | provenance | MaD:32 |
 | windows.cpp:17:8:17:25 | [summary param] *0 in CommandLineToArgvA | windows.cpp:17:8:17:25 | [summary] to write: ReturnValue[**] in CommandLineToArgvA | provenance | MaD:18 |
 | windows.cpp:22:15:22:29 | *call to GetCommandLineA | windows.cpp:22:15:22:29 | *call to GetCommandLineA | provenance | Src:MaD:3  |
 | windows.cpp:22:15:22:29 | *call to GetCommandLineA | windows.cpp:24:8:24:11 | * ... | provenance |  |
@@ -314,6 +322,14 @@ nodes
 | test.cpp:101:26:101:26 | x | semmle.label | x |
 | test.cpp:103:63:103:63 | x | semmle.label | x |
 | test.cpp:104:62:104:62 | x | semmle.label | x |
+| test.cpp:111:3:111:25 | [summary param] *0 in callWithNonTypeTemplate | semmle.label | [summary param] *0 in callWithNonTypeTemplate |
+| test.cpp:111:3:111:25 | [summary] to write: ReturnValue in callWithNonTypeTemplate | semmle.label | [summary] to write: ReturnValue in callWithNonTypeTemplate |
+| test.cpp:114:10:114:18 | call to ymlSource | semmle.label | call to ymlSource |
+| test.cpp:114:10:114:18 | call to ymlSource | semmle.label | call to ymlSource |
+| test.cpp:118:11:118:42 | call to callWithNonTypeTemplate | semmle.label | call to callWithNonTypeTemplate |
+| test.cpp:118:11:118:42 | call to callWithNonTypeTemplate | semmle.label | call to callWithNonTypeTemplate |
+| test.cpp:118:44:118:44 | *x | semmle.label | *x |
+| test.cpp:119:10:119:11 | y2 | semmle.label | y2 |
 | windows.cpp:17:8:17:25 | [summary param] *0 in CommandLineToArgvA | semmle.label | [summary param] *0 in CommandLineToArgvA |
 | windows.cpp:17:8:17:25 | [summary] to write: ReturnValue[**] in CommandLineToArgvA | semmle.label | [summary] to write: ReturnValue[**] in CommandLineToArgvA |
 | windows.cpp:22:15:22:29 | *call to GetCommandLineA | semmle.label | *call to GetCommandLineA |
@@ -472,6 +488,7 @@ subpaths
 | test.cpp:21:27:21:27 | x | test.cpp:5:5:5:20 | [summary param] 0 in ymlStepGenerated | test.cpp:5:5:5:20 | [summary] to write: ReturnValue in ymlStepGenerated | test.cpp:21:10:21:25 | call to ymlStepGenerated |
 | test.cpp:25:35:25:35 | x | test.cpp:6:5:6:27 | [summary param] 0 in ymlStepManual_with_body | test.cpp:6:5:6:27 | [summary] to write: ReturnValue in ymlStepManual_with_body | test.cpp:25:11:25:33 | call to ymlStepManual_with_body |
 | test.cpp:32:41:32:41 | x | test.cpp:7:47:7:52 | value2 | test.cpp:7:5:7:30 | *ymlStepGenerated_with_body | test.cpp:32:11:32:36 | call to ymlStepGenerated_with_body |
+| test.cpp:118:44:118:44 | *x | test.cpp:111:3:111:25 | [summary param] *0 in callWithNonTypeTemplate | test.cpp:111:3:111:25 | [summary] to write: ReturnValue in callWithNonTypeTemplate | test.cpp:118:11:118:42 | call to callWithNonTypeTemplate |
 | windows.cpp:27:36:27:38 | *cmd | windows.cpp:17:8:17:25 | [summary param] *0 in CommandLineToArgvA | windows.cpp:17:8:17:25 | [summary] to write: ReturnValue[**] in CommandLineToArgvA | windows.cpp:27:17:27:34 | **call to CommandLineToArgvA |
 | windows.cpp:537:40:537:41 | *& ... | windows.cpp:473:17:473:37 | [summary param] *1 in RtlCopyVolatileMemory | windows.cpp:473:17:473:37 | [summary param] *0 in RtlCopyVolatileMemory [Return] | windows.cpp:537:27:537:37 | RtlCopyVolatileMemory output argument |
 | windows.cpp:542:38:542:39 | *& ... | windows.cpp:479:17:479:35 | [summary param] *1 in RtlCopyDeviceMemory | windows.cpp:479:17:479:35 | [summary param] *0 in RtlCopyDeviceMemory [Return] | windows.cpp:542:25:542:35 | RtlCopyDeviceMemory output argument |

cpp/ql/test/library-tests/dataflow/external-models/flow.ext.yml

@@ -17,4 +17,5 @@ extensions:
       - ["", "", False, "ymlStepGenerated", "", "", "Argument[0]", "ReturnValue", "taint", "df-generated"]
       - ["", "", False, "ymlStepManual_with_body", "", "", "Argument[0]", "ReturnValue", "taint", "manual"]
       - ["", "", False, "ymlStepGenerated_with_body", "", "", "Argument[0]", "ReturnValue", "taint", "df-generated"]
-      - ["", "", False, "callWithArgument", "", "", "Argument[1]", "Argument[0].Parameter[0]", "value", "manual"]
+      - ["", "", False, "callWithArgument", "", "", "Argument[1]", "Argument[0].Parameter[0]", "value", "manual"]
+      - ["", "", False, "callWithNonTypeTemplate<T>", "(const T &)", "", "Argument[*0]", "ReturnValue", "value", "manual"]

cpp/ql/test/library-tests/dataflow/external-models/sinks.expected

@@ -13,3 +13,5 @@
 | test.cpp:75:11:75:11 | y | test-sink |
 | test.cpp:83:11:83:11 | y | test-sink |
 | test.cpp:89:11:89:11 | y | test-sink |
+| test.cpp:116:10:116:11 | y1 | test-sink |
+| test.cpp:119:10:119:11 | y2 | test-sink |

cpp/ql/test/library-tests/dataflow/external-models/sources.expected

@@ -2,6 +2,7 @@
 | test.cpp:10:10:10:18 | call to ymlSource | local |
 | test.cpp:56:8:56:16 | call to ymlSource | local |
 | test.cpp:94:10:94:18 | call to ymlSource | local |
+| test.cpp:114:10:114:18 | call to ymlSource | local |
 | windows.cpp:22:15:22:29 | *call to GetCommandLineA | local |
 | windows.cpp:34:17:34:38 | *call to GetEnvironmentStringsA | local |
 | windows.cpp:39:36:39:38 | GetEnvironmentVariableA output argument | local |

cpp/ql/test/library-tests/dataflow/external-models/test.cpp

@@ -102,4 +102,19 @@ void test_callWithArgument() {
 	}
 	callWithArgument(StructWithOperatorCall_has_constructor_2(), x);
 	callWithArgument(StructWithOperatorCall_no_constructor_2(), x);
-}
+}
+
+template<int N, typename T>
+T callWithNonTypeTemplate(const T&);
+
+template<typename T, int N>
+T callWithNonTypeTemplate(const T&);
+
+void test_callWithNonTypeTemplate() {
+	int x = ymlSource();
+	int y1 = callWithNonTypeTemplate<10, int>(x);
+	ymlSink(y1); // $ MISSING: ir
+
+	int y2 = callWithNonTypeTemplate<int, 10>(x);
+	ymlSink(y2); // $ ir
+}

cpp/ql/test/library-tests/dataflow/fields/dataflow-consistency.expected

@@ -142,6 +142,7 @@ postWithInFlow
 | simple.cpp:92:7:92:7 | i [post update] | PostUpdateNode should not be the target of local flow. |
 | simple.cpp:118:7:118:7 | i [post update] | PostUpdateNode should not be the target of local flow. |
 | simple.cpp:124:5:124:6 | * ... [post update] | PostUpdateNode should not be the target of local flow. |
+| simple.cpp:167:9:167:9 | x [post update] | PostUpdateNode should not be the target of local flow. |
 viableImplInCallContextTooLarge
 uniqueParameterNodeAtPosition
 uniqueParameterNodePosition

cpp/ql/test/library-tests/dataflow/fields/partial-definition-diff.expected

@@ -308,3 +308,5 @@ WARNING: module 'DataFlow' has been deprecated and may be removed in future (par
 | simple.cpp:124:5:124:6 | * ... | AST only |
 | simple.cpp:131:14:131:14 | a | IR only |
 | simple.cpp:136:10:136:10 | a | IR only |
+| simple.cpp:167:9:167:9 | x | AST only |
+| simple.cpp:168:8:168:12 | u_int | IR only |

cpp/ql/test/library-tests/dataflow/fields/partial-definition-ir.expected

@@ -670,6 +670,8 @@
 | simple.cpp:131:14:131:14 | a |
 | simple.cpp:135:20:135:20 | q |
 | simple.cpp:136:10:136:10 | a |
+| simple.cpp:167:3:167:7 | u_int |
+| simple.cpp:168:8:168:12 | u_int |
 | struct_init.c:15:8:15:9 | ab |
 | struct_init.c:15:12:15:12 | a |
 | struct_init.c:16:8:16:9 | ab |

cpp/ql/test/library-tests/dataflow/fields/partial-definition.expected

@@ -597,6 +597,8 @@ WARNING: module 'DataFlow' has been deprecated and may be removed in future (par
 | simple.cpp:118:7:118:7 | i |
 | simple.cpp:124:5:124:6 | * ... |
 | simple.cpp:135:20:135:20 | q |
+| simple.cpp:167:3:167:7 | u_int |
+| simple.cpp:167:9:167:9 | x |
 | struct_init.c:15:8:15:9 | ab |
 | struct_init.c:15:12:15:12 | a |
 | struct_init.c:16:8:16:9 | ab |

cpp/ql/test/library-tests/dataflow/fields/simple.cpp

@@ -136,4 +136,36 @@ void alias_with_fields(bool b) {
     sink(a.i); // $ MISSING: ast,ir
 }
 
+template<typename T>
+union U_with_two_instantiations_of_different_size {
+  int x;
+  T y;
+};
+
+struct LargeStruct {
+  int data[64];
+};
+
+void test_union_with_two_instantiations_of_different_sizes() {
+  // A union's fields is partitioned into "chunks" for field-flow in order to
+  // improve performance (so that a write to a field of a union does not flow
+  // to too many reads that don't happen at runtime). The partitioning is based
+  // the size of the types in the union. So a write to a field of size k only
+  // flows to a read of size k.
+  // Since field-flow is based on uninstantiated types a field can have
+  // multiple sizes if the union is instantiated with types of
+  // different sizes. So to compute the partition we pick the maximum size.
+  // Because of this there are `Content`s corresponding to the union
+  // `U_with_two_instantiations_of_different_size<T>`: The one for size
+  // `sizeof(int)`, and the one for size `sizeof(LargeStruct)` (because
+  // `LargeStruct` is larger than `int`). So the write to `x` writes to the
+  // `Content` for size `sizeof(int)`, and the read of `y` reads from the
+  // `Content` for size `sizeof(LargeStruct)`.
+  U_with_two_instantiations_of_different_size<int> u_int;
+  U_with_two_instantiations_of_different_size<LargeStruct> u_very_large;
+
+  u_int.x = user_input();
+  sink(u_int.y); // $ MISSING: ir
+}
+
 } // namespace Simple

cpp/ql/test/library-tests/dataflow/taint-tests/test_mad-signatures.expected

@@ -26843,6 +26843,24 @@ getParameterTypeName
 | atl.cpp:71:5:71:17 | _U_STRINGorID | 0 | unsigned int |
 | atl.cpp:72:5:72:17 | _U_STRINGorID | 0 | LPCTSTR |
 | atl.cpp:72:5:72:17 | _U_STRINGorID | 0 | const char * |
+| atl.cpp:96:5:96:10 | CA2AEX | 0 | LPCSTR |
+| atl.cpp:96:5:96:10 | CA2AEX | 0 | const char * |
+| atl.cpp:96:5:96:10 | CA2AEX | 1 | UINT |
+| atl.cpp:96:5:96:10 | CA2AEX | 1 | unsigned int |
+| atl.cpp:97:5:97:10 | CA2AEX | 0 | LPCSTR |
+| atl.cpp:97:5:97:10 | CA2AEX | 0 | const char * |
+| atl.cpp:124:5:124:11 | CA2CAEX | 0 | LPCSTR |
+| atl.cpp:124:5:124:11 | CA2CAEX | 0 | const char * |
+| atl.cpp:124:5:124:11 | CA2CAEX | 1 | UINT |
+| atl.cpp:124:5:124:11 | CA2CAEX | 1 | unsigned int |
+| atl.cpp:125:5:125:11 | CA2CAEX | 0 | LPCSTR |
+| atl.cpp:125:5:125:11 | CA2CAEX | 0 | const char * |
+| atl.cpp:149:5:149:10 | CA2WEX | 0 | LPCSTR |
+| atl.cpp:149:5:149:10 | CA2WEX | 0 | const char * |
+| atl.cpp:149:5:149:10 | CA2WEX | 1 | UINT |
+| atl.cpp:149:5:149:10 | CA2WEX | 1 | unsigned int |
+| atl.cpp:150:5:150:10 | CA2WEX | 0 | LPCSTR |
+| atl.cpp:150:5:150:10 | CA2WEX | 0 | const char * |
 | atl.cpp:196:12:196:14 | Add | 0 | INARGTYPclass:0 |
 | atl.cpp:198:12:198:17 | Append | 0 | const CAtlArray & |
 | atl.cpp:199:10:199:13 | Copy | 0 | const CAtlArray & |
@@ -27083,6 +27101,10 @@ getParameterTypeName
 | atl.cpp:940:10:940:18 | SetString | 0 | PCXSTR |
 | atl.cpp:940:10:940:18 | SetString | 0 | const class:0 * |
 | atl.cpp:942:11:942:20 | operator[] | 0 | int |
+| atl.cpp:1018:10:1018:10 | operator= | 0 | MakeOther && |
+| atl.cpp:1018:10:1018:10 | operator= | 0 | const MakeOther & |
+| atl.cpp:1023:10:1023:10 | operator= | 0 | MakeOther && |
+| atl.cpp:1023:10:1023:10 | operator= | 0 | const MakeOther & |
 | atl.cpp:1036:5:1036:12 | CStringT | 0 | const VARIANT & |
 | atl.cpp:1036:5:1036:12 | CStringT | 0 | const tagVARIANT & |
 | atl.cpp:1037:5:1037:12 | CStringT | 0 | const VARIANT & |
@@ -27286,6 +27308,8 @@ getParameterTypeName
 | standalone_iterators.cpp:20:7:20:7 | operator= | 0 | const int_iterator_by_trait & |
 | standalone_iterators.cpp:20:7:20:7 | operator= | 0 | int_iterator_by_trait && |
 | standalone_iterators.cpp:23:27:23:36 | operator++ | 0 | int |
+| standalone_iterators.cpp:28:13:28:13 | operator= | 0 | const iterator_traits & |
+| standalone_iterators.cpp:28:13:28:13 | operator= | 0 | iterator_traits && |
 | standalone_iterators.cpp:36:7:36:7 | operator= | 0 | const non_iterator & |
 | standalone_iterators.cpp:36:7:36:7 | operator= | 0 | non_iterator && |
 | standalone_iterators.cpp:39:18:39:27 | operator++ | 0 | int |
@@ -27297,6 +27321,8 @@ getParameterTypeName
 | standalone_iterators.cpp:66:30:66:39 | operator++ | 0 | int |
 | standalone_iterators.cpp:68:30:68:39 | operator-- | 0 | int |
 | standalone_iterators.cpp:70:31:70:39 | operator= | 0 | int |
+| standalone_iterators.cpp:74:13:74:13 | operator= | 0 | const iterator_traits & |
+| standalone_iterators.cpp:74:13:74:13 | operator= | 0 | iterator_traits && |
 | standalone_iterators.cpp:82:7:82:7 | container | 0 | const container & |
 | standalone_iterators.cpp:82:7:82:7 | container | 0 | container && |
 | standalone_iterators.cpp:82:7:82:7 | operator= | 0 | const container & |

cpp/ql/test/library-tests/types/sizeof/sizeof.expected

@@ -1,10 +1,16 @@
-| sizeof.cpp:19:15:19:25 | sizeof(int) | 4 | SizeofTypeOperator.getTypeOperand() | file://:0:0:0:0 | int |
-| sizeof.cpp:20:15:20:26 | sizeof(char) | 1 | SizeofTypeOperator.getTypeOperand() | file://:0:0:0:0 | char |
-| sizeof.cpp:21:15:21:27 | sizeof(int *) | 8 | SizeofTypeOperator.getTypeOperand() | file://:0:0:0:0 | int * |
-| sizeof.cpp:22:15:22:29 | sizeof(MyClass) | 16 | SizeofTypeOperator.getTypeOperand() | sizeof.cpp:4:7:4:13 | MyClass |
+| sizeof.cpp:19:15:19:25 | sizeof(int) | 4 | SizeofOperator.getTypeOperand() | file://:0:0:0:0 | int |
+| sizeof.cpp:20:15:20:26 | sizeof(char) | 1 | SizeofOperator.getTypeOperand() | file://:0:0:0:0 | char |
+| sizeof.cpp:21:15:21:27 | sizeof(int *) | 8 | SizeofOperator.getTypeOperand() | file://:0:0:0:0 | int * |
+| sizeof.cpp:22:15:22:29 | sizeof(MyClass) | 16 | SizeofOperator.getTypeOperand() | sizeof.cpp:4:7:4:13 | MyClass |
 | sizeof.cpp:23:15:23:23 | sizeof(<expr>) | 4 | SizeofExprOperator.getExprOperand() | sizeof.cpp:23:22:23:22 | i |
+| sizeof.cpp:23:15:23:23 | sizeof(<expr>) | 4 | SizeofOperator.getTypeOperand() | file://:0:0:0:0 | int |
 | sizeof.cpp:24:15:24:23 | sizeof(<expr>) | 1 | SizeofExprOperator.getExprOperand() | sizeof.cpp:24:22:24:22 | c |
+| sizeof.cpp:24:15:24:23 | sizeof(<expr>) | 1 | SizeofOperator.getTypeOperand() | file://:0:0:0:0 | char |
 | sizeof.cpp:25:15:25:25 | sizeof(<expr>) | 8 | SizeofExprOperator.getExprOperand() | sizeof.cpp:25:22:25:24 | ptr |
+| sizeof.cpp:25:15:25:25 | sizeof(<expr>) | 8 | SizeofOperator.getTypeOperand() | file://:0:0:0:0 | int * |
 | sizeof.cpp:26:15:26:24 | sizeof(<expr>) | 16 | SizeofExprOperator.getExprOperand() | sizeof.cpp:26:22:26:23 | mc |
+| sizeof.cpp:26:15:26:24 | sizeof(<expr>) | 16 | SizeofOperator.getTypeOperand() | sizeof.cpp:4:7:4:13 | MyClass |
 | sizeof.cpp:27:15:27:25 | sizeof(<expr>) | 40 | SizeofExprOperator.getExprOperand() | sizeof.cpp:27:22:27:24 | arr |
+| sizeof.cpp:27:15:27:25 | sizeof(<expr>) | 40 | SizeofOperator.getTypeOperand() | file://:0:0:0:0 | int[10] |
 | sizeof.cpp:28:16:28:29 | sizeof(<expr>) | 4 | SizeofExprOperator.getExprOperand() | sizeof.cpp:28:23:28:28 | access to array |
+| sizeof.cpp:28:16:28:29 | sizeof(<expr>) | 4 | SizeofOperator.getTypeOperand() | file://:0:0:0:0 | int |

cpp/ql/test/library-tests/types/sizeof/sizeof.ql

@@ -2,8 +2,8 @@ import cpp
 
 from SizeofOperator sto, string elemDesc, Element e
 where
-  elemDesc = "SizeofTypeOperator.getTypeOperand()" and
-  e = sto.(SizeofTypeOperator).getTypeOperand()
+  elemDesc = "SizeofOperator.getTypeOperand()" and
+  e = sto.getTypeOperand()
   or
   elemDesc = "SizeofExprOperator.getExprOperand()" and
   e = sto.(SizeofExprOperator).getExprOperand()

cpp/ql/test/library-tests/variables/variables/variable.expected

@@ -2,10 +2,10 @@
 | file://:0:0:0:0 | (unnamed parameter 0) | file://:0:0:0:0 | address && | SemanticStackVariable |  |  |
 | file://:0:0:0:0 | (unnamed parameter 0) | file://:0:0:0:0 | const __va_list_tag & | SemanticStackVariable |  |  |
 | file://:0:0:0:0 | (unnamed parameter 0) | file://:0:0:0:0 | const address & | SemanticStackVariable |  |  |
-| file://:0:0:0:0 | fp_offset | file://:0:0:0:0 | unsigned int | Field |  |  |
-| file://:0:0:0:0 | gp_offset | file://:0:0:0:0 | unsigned int | Field |  |  |
-| file://:0:0:0:0 | overflow_arg_area | file://:0:0:0:0 | void * | Field |  |  |
-| file://:0:0:0:0 | reg_save_area | file://:0:0:0:0 | void * | Field |  |  |
+| file://:0:0:0:0 | fp_offset | file://:0:0:0:0 | unsigned int | NonLocalCanonicalField |  |  |
+| file://:0:0:0:0 | gp_offset | file://:0:0:0:0 | unsigned int | NonLocalCanonicalField |  |  |
+| file://:0:0:0:0 | overflow_arg_area | file://:0:0:0:0 | void * | NonLocalCanonicalField |  |  |
+| file://:0:0:0:0 | reg_save_area | file://:0:0:0:0 | void * | NonLocalCanonicalField |  |  |
 | variables.cpp:1:12:1:12 | i | file://:0:0:0:0 | int | GlobalLikeVariable, GlobalVariable, StaticStorageDurationVariable |  |  |
 | variables.cpp:2:12:2:12 | i | file://:0:0:0:0 | int | GlobalLikeVariable, GlobalVariable, StaticStorageDurationVariable |  |  |
 | variables.cpp:3:12:3:12 | i | file://:0:0:0:0 | int | GlobalLikeVariable, GlobalVariable, StaticStorageDurationVariable |  |  |
@@ -33,10 +33,10 @@
 | variables.cpp:37:6:37:8 | ap3 | file://:0:0:0:0 | int * | GlobalLikeVariable, GlobalVariable, StaticStorageDurationVariable |  |  |
 | variables.cpp:41:7:41:11 | local | file://:0:0:0:0 | char[] | LocalVariable, SemanticStackVariable |  |  |
 | variables.cpp:43:14:43:18 | local | file://:0:0:0:0 | int | GlobalLikeVariable, StaticLocalVariable |  | static |
-| variables.cpp:48:9:48:12 | name | file://:0:0:0:0 | char * | Field |  |  |
-| variables.cpp:49:12:49:17 | number | file://:0:0:0:0 | long | Field |  |  |
-| variables.cpp:50:9:50:14 | street | file://:0:0:0:0 | char * | Field |  |  |
-| variables.cpp:51:9:51:12 | town | file://:0:0:0:0 | char * | Field |  |  |
+| variables.cpp:48:9:48:12 | name | file://:0:0:0:0 | char * | NonLocalCanonicalField |  |  |
+| variables.cpp:49:12:49:17 | number | file://:0:0:0:0 | long | NonLocalCanonicalField |  |  |
+| variables.cpp:50:9:50:14 | street | file://:0:0:0:0 | char * | NonLocalCanonicalField |  |  |
+| variables.cpp:51:9:51:12 | town | file://:0:0:0:0 | char * | NonLocalCanonicalField |  |  |
 | variables.cpp:52:16:52:22 | country | file://:0:0:0:0 | char * | MemberVariable, StaticStorageDurationVariable |  | static |
 | variables.cpp:56:14:56:29 | externInFunction | file://:0:0:0:0 | int | GlobalLikeVariable, GlobalVariable, StaticStorageDurationVariable |  |  |
 | variables.cpp:60:10:60:17 | __func__ | file://:0:0:0:0 | const char[9] | GlobalLikeVariable, StaticInitializedStaticLocalVariable |  | static |

cpp/ql/test/query-tests/Security/CWE/CWE-119/semmle/tests/OverflowBuffer.expected

@@ -1,6 +1,10 @@
 | overflowdestination.cpp:46:2:46:7 | call to memcpy | This 'memcpy' operation accesses 128 bytes but the $@ is only 64 bytes. | overflowdestination.cpp:40:7:40:10 | dest | destination buffer |
 | tests.cpp:23:2:23:7 | call to memcpy | This 'memcpy' operation accesses 20 bytes but the $@ is only 10 bytes. | tests.cpp:19:7:19:17 | smallbuffer | source buffer |
 | tests.cpp:25:2:25:7 | call to memcpy | This 'memcpy' operation accesses 20 bytes but the $@ is only 10 bytes. | tests.cpp:19:7:19:17 | smallbuffer | destination buffer |
+| tests.cpp:34:2:34:7 | call to memcpy | This 'memcpy' operation accesses 20 bytes but the $@ is only 10 bytes. | tests.cpp:30:30:30:35 | call to malloc | source buffer |
+| tests.cpp:36:2:36:7 | call to memcpy | This 'memcpy' operation accesses 20 bytes but the $@ is only 10 bytes. | tests.cpp:30:30:30:35 | call to malloc | destination buffer |
+| tests.cpp:50:2:50:7 | call to memcpy | This 'memcpy' operation accesses 20 bytes but the $@ is only 10 bytes. | tests.cpp:46:16:46:27 | new[] | source buffer |
+| tests.cpp:52:2:52:7 | call to memcpy | This 'memcpy' operation accesses 20 bytes but the $@ is only 10 bytes. | tests.cpp:46:16:46:27 | new[] | destination buffer |
 | tests.cpp:172:23:172:31 | access to array | This array indexing operation accesses a negative index -1 on the $@. | tests.cpp:170:17:170:41 | {...} | array |
 | tests.cpp:176:23:176:30 | access to array | This array indexing operation accesses byte offset 31 but the $@ is only 24 bytes. | tests.cpp:170:17:170:41 | {...} | array |
 | tests.cpp:222:3:222:8 | call to memset | This 'memset' operation accesses 33 bytes but the $@ is only 32 bytes. | tests.cpp:214:8:214:14 | buffer1 | destination buffer |

cpp/ql/test/query-tests/Security/CWE/CWE-119/semmle/tests/tests.cpp

@@ -30,10 +30,10 @@ void test2()
 	char *smallbuffer = (char *)malloc(sizeof(char) * 10);
 	char *bigbuffer = (char *)malloc(sizeof(char) * 20);
 
-	memcpy(bigbuffer, smallbuffer, sizeof(smallbuffer)); // GOOD
-	memcpy(bigbuffer, smallbuffer, sizeof(bigbuffer)); // BAD: over-read [NOT DETECTED]
-	memcpy(smallbuffer, bigbuffer, sizeof(smallbuffer)); // GOOD
-	memcpy(smallbuffer, bigbuffer, sizeof(bigbuffer)); // BAD: over-write [NOT DETECTED]
+	memcpy(bigbuffer, smallbuffer, sizeof(char) * 10); // GOOD
+	memcpy(bigbuffer, smallbuffer, sizeof(char) * 20); // BAD: over-read
+	memcpy(smallbuffer, bigbuffer, sizeof(char) * 10); // GOOD
+	memcpy(smallbuffer, bigbuffer, sizeof(char) * 20); // BAD: over-write
 
 	free(bigbuffer);
 	free(smallbuffer);
@@ -46,10 +46,10 @@ void test3()
 	smallbuffer = new char[10];
 	bigbuffer = new char[20];
 
-	memcpy(bigbuffer, smallbuffer, sizeof(smallbuffer)); // GOOD
-	memcpy(bigbuffer, smallbuffer, sizeof(bigbuffer)); // BAD: over-read [NOT DETECTED]
-	memcpy(smallbuffer, bigbuffer, sizeof(smallbuffer)); // GOOD
-	memcpy(smallbuffer, bigbuffer, sizeof(bigbuffer)); // BAD: over-write [NOT DETECTED]
+	memcpy(bigbuffer, smallbuffer, sizeof(char[10])); // GOOD
+	memcpy(bigbuffer, smallbuffer, sizeof(char[20])); // BAD: over-read
+	memcpy(smallbuffer, bigbuffer, sizeof(char[10])); // GOOD
+	memcpy(smallbuffer, bigbuffer, sizeof(char[20])); // BAD: over-write
 
 	delete [] bigbuffer;
 	delete [] smallbuffer;

csharp/autobuilder/Semmle.Autobuild.CSharp/DotNetRule.cs

@@ -84,11 +84,7 @@ namespace Semmle.Autobuild.CSharp
             var temp = FileUtils.GetTemporaryWorkingDirectory(builder.Actions.GetEnvironmentVariable, builder.Options.Language.UpperCaseName, out var shouldCleanUp);
             return DotNet.WithDotNet(builder.Actions, builder.Logger, builder.Paths.Select(x => x.Item1), temp, shouldCleanUp, ensureDotNetAvailable, builder.Options.DotNetVersion, installDir =>
             {
-                var env = new Dictionary<string, string>
-                {
-                    { "DOTNET_SKIP_FIRST_TIME_EXPERIENCE", "true" },
-                    { "MSBUILDDISABLENODEREUSE", "1" }
-                };
+                var env = DotNet.MinimalEnvironment.ToDictionary();
                 if (installDir is not null)
                 {
                     // The installation succeeded, so use the newly installed .NET

csharp/codeql-extractor.yml

@@ -74,3 +74,8 @@ options:
       [EXPERIMENTAL] The value is a path to the MsBuild binary log file that should be extracted.
       This option only works when `--build-mode none` is also specified.
     type: array
+  buildless_dependency_dir:
+    title: The path where buildless (standalone) extraction should keep dependencies.
+    description: >
+      If set, the buildless (standalone) extractor will store dependencies in this directory.
+    type: string

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyDirectory.cs

@@ -0,0 +1,62 @@
+using System;
+using System.IO;
+using Semmle.Util;
+using Semmle.Util.Logging;
+
+namespace Semmle.Extraction.CSharp.DependencyFetching
+{
+    /// <summary>
+    /// A directory used for storing fetched dependencies.
+    /// When a specific directory is set via the dependency directory extractor option,
+    /// we store dependencies in that directory for caching purposes.
+    /// Otherwise, we create a temporary directory that is deleted upon disposal.
+    /// </summary>
+    public sealed class DependencyDirectory : IDisposable
+    {
+        private readonly string userReportedDirectoryPurpose;
+        private readonly ILogger logger;
+        private readonly bool attemptCleanup;
+
+        public DirectoryInfo DirInfo { get; }
+
+        public DependencyDirectory(string subfolderName, string userReportedDirectoryPurpose, ILogger logger)
+        {
+            this.logger = logger;
+            this.userReportedDirectoryPurpose = userReportedDirectoryPurpose;
+
+            string path;
+            if (EnvironmentVariables.GetBuildlessDependencyDir() is string dir)
+            {
+                path = dir;
+                attemptCleanup = false;
+            }
+            else
+            {
+                path = FileUtils.GetTemporaryWorkingDirectory(out _);
+                attemptCleanup = true;
+            }
+            DirInfo = new DirectoryInfo(Path.Join(path, subfolderName));
+            DirInfo.Create();
+        }
+
+        public void Dispose()
+        {
+            if (!attemptCleanup)
+            {
+                logger.LogInfo($"Keeping {userReportedDirectoryPurpose} directory {DirInfo.FullName} for possible caching purposes.");
+                return;
+            }
+
+            try
+            {
+                DirInfo.Delete(true);
+            }
+            catch (Exception exc)
+            {
+                logger.LogInfo($"Couldn't delete {userReportedDirectoryPurpose} directory {exc.Message}");
+            }
+        }
+
+        public override string ToString() => DirInfo.FullName;
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Collections.ObjectModel;
 using System.IO;
 using System.Linq;
 using Newtonsoft.Json.Linq;
@@ -140,6 +141,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         // The version number should be kept in sync with the version .NET version used for building the application.
         public const string LatestDotNetSdkVersion = "9.0.300";
 
+        public static ReadOnlyDictionary<string, string> MinimalEnvironment => IDotNetCliInvoker.MinimalEnvironment;
+
         /// <summary>
         /// Returns a script for downloading relevant versions of the
         /// .NET SDK. The SDK(s) will be installed at <code>installDir</code>
@@ -254,7 +257,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                 else
                 {
                     var dotnetInstallPath = actions.PathCombine(tempWorkingDirectory, ".dotnet", "dotnet-install.sh");
-
                     var downloadDotNetInstallSh = BuildScript.DownloadFile(
                         "https://dot.net/v1/dotnet-install.sh",
                         dotnetInstallPath,
@@ -269,17 +271,28 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                     prelude = downloadDotNetInstallSh & chmod.Script;
                     postlude = shouldCleanUp ? BuildScript.DeleteFile(dotnetInstallPath) : BuildScript.Success;
 
-                    getInstall = version => new CommandBuilder(actions).
-                        RunCommand(dotnetInstallPath).
-                        Argument("--channel").
-                        Argument("release").
-                        Argument("--version").
-                        Argument(version).
-                        Argument("--install-dir").
-                        Argument(path).Script;
+                    getInstall = version =>
+                    {
+                        var cb = new CommandBuilder(actions).
+                            RunCommand(dotnetInstallPath).
+                            Argument("--channel").
+                            Argument("release").
+                            Argument("--version").
+                            Argument(version);
+
+                        // Request ARM64 architecture on Apple Silicon machines
+                        if (actions.IsRunningOnAppleSilicon())
+                        {
+                            cb.Argument("--architecture").
+                                Argument("arm64");
+                        }
+
+                        return cb.Argument("--install-dir").
+                            Argument(path).Script;
+                    };
                 }
 
-                var dotnetInfo = new CommandBuilder(actions).
+                var dotnetInfo = new CommandBuilder(actions, environment: MinimalEnvironment).
                     RunCommand(actions.PathCombine(path, "dotnet")).
                     Argument("--info").Script;
 
@@ -311,7 +324,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 
         private static BuildScript GetInstalledSdksScript(IBuildActions actions)
         {
-            var listSdks = new CommandBuilder(actions, silent: true).
+            var listSdks = new CommandBuilder(actions, silent: true, environment: MinimalEnvironment).
                 RunCommand("dotnet").
                 Argument("--list-sdks");
             return listSdks.Script;

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Collections.ObjectModel;
 using System.Diagnostics;
 using Semmle.Util;
 using Semmle.Util.Logging;
@@ -36,10 +37,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             {
                 startInfo.WorkingDirectory = workingDirectory;
             }
-            // Set the .NET CLI language to English to avoid localized output.
-            startInfo.EnvironmentVariables["DOTNET_CLI_UI_LANGUAGE"] = "en";
-            startInfo.EnvironmentVariables["MSBUILDDISABLENODEREUSE"] = "1";
-            startInfo.EnvironmentVariables["DOTNET_SKIP_FIRST_TIME_EXPERIENCE"] = "true";
+
+            // Set minimal environment variables.
+            foreach (var kvp in IDotNetCliInvoker.MinimalEnvironment)
+            {
+                startInfo.EnvironmentVariables[kvp.Key] = kvp.Value;
+            }
 
             // Configure the proxy settings, if applicable.
             if (this.proxy != null)

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNetCliInvoker.cs

@@ -1,4 +1,5 @@
 using System.Collections.Generic;
+using System.Collections.ObjectModel;
 
 namespace Semmle.Extraction.CSharp.DependencyFetching
 {
@@ -9,6 +10,20 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         /// </summary>
         string Exec { get; }
 
+        /// <summary>
+        /// A minimal environment for running the .NET CLI.
+        /// 
+        /// DOTNET_CLI_UI_LANGUAGE: The .NET CLI language is set to English to avoid localized output.
+        /// MSBUILDDISABLENODEREUSE: To ensure clean environment for each build.
+        /// DOTNET_SKIP_FIRST_TIME_EXPERIENCE: To skip first time experience messages.
+        /// </summary>
+        static ReadOnlyDictionary<string, string> MinimalEnvironment { get; } = new(new Dictionary<string, string>
+        {
+            {"DOTNET_CLI_UI_LANGUAGE", "en"},
+            {"MSBUILDDISABLENODEREUSE", "1"},
+            {"DOTNET_SKIP_FIRST_TIME_EXPERIENCE", "true"}
+        });
+
         /// <summary>
         /// Execute `dotnet <paramref name="args"/>` and return true if the command succeeded, otherwise false.
         /// If `silent` is true the output of the command is logged as `debug` otherwise as `info`.

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs

@@ -24,16 +24,16 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         private readonly FileProvider fileProvider;
 
         /// <summary>
-        /// The computed packages directory.
-        /// This will be in the Temp location
+        /// The packages directory.
+        /// This will be in the user-specified or computed Temp location
         /// so as to not trample the source tree.
         /// </summary>
-        private readonly TemporaryDirectory packageDirectory;
+        private readonly DependencyDirectory packageDirectory;
 
         /// <summary>
         /// Create the package manager for a specified source tree.
         /// </summary>
-        public NugetExeWrapper(FileProvider fileProvider, TemporaryDirectory packageDirectory, Semmle.Util.Logging.ILogger logger)
+        public NugetExeWrapper(FileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger)
         {
             this.fileProvider = fileProvider;
             this.packageDirectory = packageDirectory;

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs

@@ -24,12 +24,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         private readonly IDotNet dotnet;
         private readonly DependabotProxy? dependabotProxy;
         private readonly IDiagnosticsWriter diagnosticsWriter;
-        private readonly TemporaryDirectory legacyPackageDirectory;
-        private readonly TemporaryDirectory missingPackageDirectory;
+        private readonly DependencyDirectory legacyPackageDirectory;
+        private readonly DependencyDirectory missingPackageDirectory;
         private readonly ILogger logger;
         private readonly ICompilationInfoContainer compilationInfoContainer;
 
-        public TemporaryDirectory PackageDirectory { get; }
+        public DependencyDirectory PackageDirectory { get; }
 
         public NugetPackageRestorer(
             FileProvider fileProvider,
@@ -48,9 +48,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             this.logger = logger;
             this.compilationInfoContainer = compilationInfoContainer;
 
-            PackageDirectory = new TemporaryDirectory(ComputeTempDirectoryPath("packages"), "package", logger);
-            legacyPackageDirectory = new TemporaryDirectory(ComputeTempDirectoryPath("legacypackages"), "legacy package", logger);
-            missingPackageDirectory = new TemporaryDirectory(ComputeTempDirectoryPath("missingpackages"), "missing package", logger);
+            PackageDirectory = new DependencyDirectory("packages", "package", logger);
+            legacyPackageDirectory = new DependencyDirectory("legacypackages", "legacy package", logger);
+            missingPackageDirectory = new DependencyDirectory("missingpackages", "missing package", logger);
         }
 
         public string? TryRestore(string package)

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs

@@ -360,5 +360,22 @@ namespace Semmle.Extraction.CSharp
                 return versionString.InformationalVersion;
             }
         }
+
+        private static readonly HashSet<string> errorsToIgnore = new HashSet<string>
+        {
+            "CS7027",   // Code signing failure
+            "CS1589",   // XML referencing not supported
+            "CS1569"    // Error writing XML documentation
+        };
+
+        /// <summary>
+        /// Retrieves the diagnostics from the compilation, filtering out those that should be ignored.
+        /// </summary>
+        protected List<Diagnostic> GetFilteredDiagnostics() =>
+            compilation is not null
+                ? compilation.GetDiagnostics()
+                    .Where(e => e.Severity >= DiagnosticSeverity.Error && !errorsToIgnore.Contains(e.Id))
+                    .ToList()
+                : [];
     }
 }

csharp/extractor/Semmle.Extraction.CSharp/Extractor/StandaloneAnalyser.cs

@@ -13,6 +13,14 @@ namespace Semmle.Extraction.CSharp
         {
         }
 
+        private void LogDiagnostics()
+        {
+            foreach (var error in GetFilteredDiagnostics())
+            {
+                Logger.LogDebug($"  Compilation error: {error}");
+            }
+        }
+
         public void Initialize(string outputPath, IEnumerable<(string, string)> compilationInfos, CSharpCompilation compilationIn, CommonOptions options)
         {
             compilation = compilationIn;
@@ -20,6 +28,7 @@ namespace Semmle.Extraction.CSharp
             this.options = options;
             LogExtractorInfo();
             SetReferencePaths();
+            LogDiagnostics();
         }
     }
 }

csharp/extractor/Semmle.Extraction.CSharp/Extractor/TracingAnalyser.cs

@@ -136,11 +136,7 @@ namespace Semmle.Extraction.CSharp
 
         private int LogDiagnostics()
         {
-            var filteredDiagnostics = compilation!
-                .GetDiagnostics()
-                .Where(e => e.Severity >= DiagnosticSeverity.Error && !errorsToIgnore.Contains(e.Id))
-                .ToList();
-
+            var filteredDiagnostics = GetFilteredDiagnostics();
             foreach (var error in filteredDiagnostics)
             {
                 Logger.LogError($"  Compilation error: {error}");
@@ -148,7 +144,7 @@ namespace Semmle.Extraction.CSharp
 
             if (filteredDiagnostics.Count != 0)
             {
-                foreach (var reference in compilation.References)
+                foreach (var reference in compilation!.References)
                 {
                     Logger.LogInfo($"  Resolved reference {reference.Display}");
                 }
@@ -156,12 +152,5 @@ namespace Semmle.Extraction.CSharp
 
             return filteredDiagnostics.Count;
         }
-
-        private static readonly HashSet<string> errorsToIgnore = new HashSet<string>
-        {
-            "CS7027",   // Code signing failure
-            "CS1589",   // XML referencing not supported
-            "CS1569"    // Error writing XML documentation
-        };
     }
 }

csharp/extractor/Semmle.Util/EnvironmentVariables.cs

@@ -76,5 +76,14 @@ namespace Semmle.Util
         {
             return Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_OVERLAY_BASE_METADATA_OUT");
         }
+
+        /// <summary>
+        /// If set, returns the directory where buildless dependencies should be stored.
+        /// This can be used for caching dependencies.
+        /// </summary>
+        public static string? GetBuildlessDependencyDir()
+        {
+            return Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_OPTION_BUILDLESS_DEPENDENCY_DIR");
+        }
     }
 }

csharp/ql/campaigns/Solorigate/lib/CHANGELOG.md

@@ -1,3 +1,15 @@
+## 1.7.55
+
+No user-facing changes.
+
+## 1.7.54
+
+No user-facing changes.
+
+## 1.7.53
+
+No user-facing changes.
+
 ## 1.7.52
 
 No user-facing changes.

csharp/ql/campaigns/Solorigate/lib/change-notes/released/1.7.53.md

@@ -0,0 +1,3 @@
+## 1.7.53
+
+No user-facing changes.

csharp/ql/campaigns/Solorigate/lib/change-notes/released/1.7.54.md

@@ -0,0 +1,3 @@
+## 1.7.54
+
+No user-facing changes.

csharp/ql/campaigns/Solorigate/lib/change-notes/released/1.7.55.md

@@ -0,0 +1,3 @@
+## 1.7.55
+
+No user-facing changes.

csharp/ql/campaigns/Solorigate/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 1.7.52
+lastReleaseVersion: 1.7.55

csharp/ql/campaigns/Solorigate/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-solorigate-all
-version: 1.7.52
+version: 1.7.55
 groups:
   - csharp
   - solorigate

csharp/ql/campaigns/Solorigate/src/CHANGELOG.md

@@ -1,3 +1,15 @@
+## 1.7.55
+
+No user-facing changes.
+
+## 1.7.54
+
+No user-facing changes.
+
+## 1.7.53
+
+No user-facing changes.
+
 ## 1.7.52
 
 No user-facing changes.

csharp/ql/campaigns/Solorigate/src/change-notes/released/1.7.53.md

@@ -0,0 +1,3 @@
+## 1.7.53
+
+No user-facing changes.

csharp/ql/campaigns/Solorigate/src/change-notes/released/1.7.54.md

@@ -0,0 +1,3 @@
+## 1.7.54
+
+No user-facing changes.

csharp/ql/campaigns/Solorigate/src/change-notes/released/1.7.55.md

@@ -0,0 +1,3 @@
+## 1.7.55
+
+No user-facing changes.