Merge branch 'main' into jb1/16-cryptography-models-libraries-and-queries-migration

.github/workflows/csharp-qltest.yml

@@ -53,7 +53,6 @@ jobs:
         slice: ["1/2", "2/2"]
     steps:
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/fetch-codeql
       - uses: ./csharp/actions/create-extractor-pack
       - name: Cache compilation cache
         id: query-cache
@@ -62,16 +61,14 @@ jobs:
           key: csharp-qltest-${{ matrix.slice }}
       - name: Run QL tests
         run: |
-          CODEQL_PATH=$(gh codeql version --format=json | jq -r .unpackedLocation)
-          # The legacy ASP extractor is not in this repo, so take the one from the nightly build
-          mv "$CODEQL_PATH/csharp/tools/extractor-asp.jar" "${{ github.workspace }}/csharp/extractor-pack/tools"
-          # Safe guard against using the bundled extractor
-          rm -rf "$CODEQL_PATH/csharp"
-          codeql test run --threads=0 --ram 50000 --slice ${{ matrix.slice }} --search-path "${{ github.workspace }}/csharp/extractor-pack" --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+          codeql test run --threads=0 --ram 50000 --slice ${{ matrix.slice }} --search-path extractor-pack --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
         env:
           GITHUB_TOKEN: ${{ github.token }}
   unit-tests:
-    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-2019]
+    runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
       - name: Setup dotnet
@@ -80,7 +77,25 @@ jobs:
           dotnet-version: 7.0.102
       - name: Extractor unit tests
         run: |
-          dotnet test -p:RuntimeFrameworkVersion=7.0.2 "${{ github.workspace }}/csharp/extractor/Semmle.Util.Tests"
-          dotnet test -p:RuntimeFrameworkVersion=7.0.2 "${{ github.workspace }}/csharp/extractor/Semmle.Extraction.Tests"
-          dotnet test -p:RuntimeFrameworkVersion=7.0.2 "${{ github.workspace }}/csharp/autobuilder/Semmle.Autobuild.CSharp.Tests"
+          dotnet test -p:RuntimeFrameworkVersion=7.0.2 extractor/Semmle.Util.Tests
+          dotnet test -p:RuntimeFrameworkVersion=7.0.2 extractor/Semmle.Extraction.Tests
+          dotnet test -p:RuntimeFrameworkVersion=7.0.2 autobuilder/Semmle.Autobuild.CSharp.Tests
           dotnet test -p:RuntimeFrameworkVersion=7.0.2 "${{ github.workspace }}/cpp/autobuilder/Semmle.Autobuild.Cpp.Tests"
+        shell: bash
+  stubgentest:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./csharp/actions/create-extractor-pack
+      - name: Run stub generator tests
+        run: |
+          # Generate (Asp)NetCore stubs
+          STUBS_PATH=stubs_output
+          python3 ql/src/Stubs/make_stubs_nuget.py webapp Swashbuckle.AspNetCore.Swagger latest "$STUBS_PATH"
+          rm -rf ql/test/resources/stubs/_frameworks
+          # Update existing stubs in the repo with the freshly generated ones
+          mv "$STUBS_PATH/output/stubs/_frameworks" ql/test/resources/stubs/
+          git status
+          codeql test run --threads=0 --search-path extractor-pack --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries -- ql/test/library-tests/dataflow/flowsources/aspremote
+        env:
+          GITHUB_TOKEN: ${{ github.token }}

.github/workflows/js-ml-tests.yml

@@ -1,65 +0,0 @@
-name: JS ML-powered queries tests
-
-on:
-  push:
-    paths:
-      - "javascript/ql/experimental/adaptivethreatmodeling/**"
-      - .github/workflows/js-ml-tests.yml
-      - .github/actions/fetch-codeql/action.yml
-      - codeql-workspace.yml
-    branches:
-      - main
-      - "rc/*"
-  pull_request:
-    paths:
-      - "javascript/ql/experimental/adaptivethreatmodeling/**"
-      - .github/workflows/js-ml-tests.yml
-      - .github/actions/fetch-codeql/action.yml
-      - codeql-workspace.yml
-  workflow_dispatch:
-
-defaults:
-  run:
-    working-directory: javascript/ql/experimental/adaptivethreatmodeling
-
-jobs:
-  qltest:
-    name: Test QL
-    runs-on: ubuntu-latest-xl
-    steps:
-      - uses: actions/checkout@v4
-
-      - uses: ./.github/actions/fetch-codeql
-
-      - name: Install pack dependencies
-        run: |
-          for pack in modelbuilding src test; do
-            codeql pack install --mode verify -- "${pack}"
-          done
-      
-      - name: Cache compilation cache
-        id: query-cache
-        uses: ./.github/actions/cache-query-compilation
-        with: 
-          key: js-ml-test
-
-      - name: Check QL compilation
-        run: |
-          codeql query compile \
-            --check-only \
-            --ram 50000 \
-            --additional-packs "${{ github.workspace }}" \
-            --threads=0 \
-            --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" \
-            -- \
-            lib modelbuilding src
-
-      - name: Run QL tests
-        run: |
-          codeql test run \
-            --threads=0 \
-            --ram 50000 \
-            --additional-packs "${{ github.workspace }}" \
-            --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" \
-            -- \
-            test

cpp/ql/lib/semmle/code/cpp/File.qll

@@ -5,155 +5,35 @@
 import semmle.code.cpp.Element
 import semmle.code.cpp.Declaration
 import semmle.code.cpp.metrics.MetricFile
+private import codeql.util.FileSystem
+
+private module Input implements InputSig {
+  abstract class ContainerBase extends @container {
+    abstract string getAbsolutePath();
+
+    ContainerBase getParentContainer() {
+      containerparent(unresolveElement(result), underlyingElement(this))
+    }
+
+    string toString() { result = this.getAbsolutePath() }
+  }
+
+  class FolderBase extends ContainerBase, @folder {
+    override string getAbsolutePath() { folders(underlyingElement(this), result) }
+  }
+
+  class FileBase extends ContainerBase, @file {
+    override string getAbsolutePath() { files(underlyingElement(this), result) }
+  }
+
+  predicate hasSourceLocationPrefix = sourceLocationPrefix/1;
+}
+
+private module Impl = Make<Input>;
 
 /** A file or folder. */
-class Container extends Locatable, @container {
-  /**
-   * Gets the absolute, canonical path of this container, using forward slashes
-   * as path separator.
-   *
-   * The path starts with a _root prefix_ followed by zero or more _path
-   * segments_ separated by forward slashes.
-   *
-   * The root prefix is of one of the following forms:
-   *
-   *   1. A single forward slash `/` (Unix-style)
-   *   2. An upper-case drive letter followed by a colon and a forward slash,
-   *      such as `C:/` (Windows-style)
-   *   3. Two forward slashes, a computer name, and then another forward slash,
-   *      such as `//FileServer/` (UNC-style)
-   *
-   * Path segments are never empty (that is, absolute paths never contain two
-   * contiguous slashes, except as part of a UNC-style root prefix). Also, path
-   * segments never contain forward slashes, and no path segment is of the
-   * form `.` (one dot) or `..` (two dots).
-   *
-   * Note that an absolute path never ends with a forward slash, except if it is
-   * a bare root prefix, that is, the path has no path segments. A container
-   * whose absolute path has no segments is always a `Folder`, not a `File`.
-   */
-  string getAbsolutePath() { none() } // overridden by subclasses
-
-  /**
-   * Gets the relative path of this file or folder from the root folder of the
-   * analyzed source location. The relative path of the root folder itself is
-   * the empty string.
-   *
-   * This has no result if the container is outside the source root, that is,
-   * if the root folder is not a reflexive, transitive parent of this container.
-   */
-  string getRelativePath() {
-    exists(string absPath, string pref |
-      absPath = this.getAbsolutePath() and sourceLocationPrefix(pref)
-    |
-      absPath = pref and result = ""
-      or
-      absPath = pref.regexpReplaceAll("/$", "") + "/" + result and
-      not result.matches("/%")
-    )
-  }
-
-  /**
-   * Gets the base name of this container including extension, that is, the last
-   * segment of its absolute path, or the empty string if it has no segments.
-   *
-   * Here are some examples of absolute paths and the corresponding base names
-   * (surrounded with quotes to avoid ambiguity):
-   *
-   * <table border="1">
-   * <tr><th>Absolute path</th><th>Base name</th></tr>
-   * <tr><td>"/tmp/tst.js"</td><td>"tst.js"</td></tr>
-   * <tr><td>"C:/Program Files (x86)"</td><td>"Program Files (x86)"</td></tr>
-   * <tr><td>"/"</td><td>""</td></tr>
-   * <tr><td>"C:/"</td><td>""</td></tr>
-   * <tr><td>"D:/"</td><td>""</td></tr>
-   * <tr><td>"//FileServer/"</td><td>""</td></tr>
-   * </table>
-   */
-  string getBaseName() {
-    result = this.getAbsolutePath().regexpCapture(".*/(([^/]*?)(?:\\.([^.]*))?)", 1)
-  }
-
-  /**
-   * Gets the extension of this container, that is, the suffix of its base name
-   * after the last dot character, if any.
-   *
-   * In particular,
-   *
-   *  - if the name does not include a dot, there is no extension, so this
-   *    predicate has no result;
-   *  - if the name ends in a dot, the extension is the empty string;
-   *  - if the name contains multiple dots, the extension follows the last dot.
-   *
-   * Here are some examples of absolute paths and the corresponding extensions
-   * (surrounded with quotes to avoid ambiguity):
-   *
-   * <table border="1">
-   * <tr><th>Absolute path</th><th>Extension</th></tr>
-   * <tr><td>"/tmp/tst.js"</td><td>"js"</td></tr>
-   * <tr><td>"/tmp/.classpath"</td><td>"classpath"</td></tr>
-   * <tr><td>"/bin/bash"</td><td>not defined</td></tr>
-   * <tr><td>"/tmp/tst2."</td><td>""</td></tr>
-   * <tr><td>"/tmp/x.tar.gz"</td><td>"gz"</td></tr>
-   * </table>
-   */
-  string getExtension() {
-    result = this.getAbsolutePath().regexpCapture(".*/([^/]*?)(\\.([^.]*))?", 3)
-  }
-
-  /**
-   * Gets the stem of this container, that is, the prefix of its base name up to
-   * (but not including) the last dot character if there is one, or the entire
-   * base name if there is not.
-   *
-   * Here are some examples of absolute paths and the corresponding stems
-   * (surrounded with quotes to avoid ambiguity):
-   *
-   * <table border="1">
-   * <tr><th>Absolute path</th><th>Stem</th></tr>
-   * <tr><td>"/tmp/tst.js"</td><td>"tst"</td></tr>
-   * <tr><td>"/tmp/.classpath"</td><td>""</td></tr>
-   * <tr><td>"/bin/bash"</td><td>"bash"</td></tr>
-   * <tr><td>"/tmp/tst2."</td><td>"tst2"</td></tr>
-   * <tr><td>"/tmp/x.tar.gz"</td><td>"x.tar"</td></tr>
-   * </table>
-   */
-  string getStem() {
-    result = this.getAbsolutePath().regexpCapture(".*/([^/]*?)(?:\\.([^.]*))?", 1)
-  }
-
-  /** Gets the parent container of this file or folder, if any. */
-  Container getParentContainer() {
-    containerparent(unresolveElement(result), underlyingElement(this))
-  }
-
-  /** Gets a file or sub-folder in this container. */
-  Container getAChildContainer() { this = result.getParentContainer() }
-
-  /** Gets a file in this container. */
-  File getAFile() { result = this.getAChildContainer() }
-
-  /** Gets the file in this container that has the given `baseName`, if any. */
-  File getFile(string baseName) {
-    result = this.getAFile() and
-    result.getBaseName() = baseName
-  }
-
-  /** Gets a sub-folder in this container. */
-  Folder getAFolder() { result = this.getAChildContainer() }
-
-  /** Gets the sub-folder in this container that has the given `baseName`, if any. */
-  Folder getFolder(string baseName) {
-    result = this.getAFolder() and
-    result.getBaseName() = baseName
-  }
-
-  /**
-   * Gets a textual representation of the path of this container.
-   *
-   * This is the absolute path of the container.
-   */
-  override string toString() { result = this.getAbsolutePath() }
+class Container extends Locatable, Impl::Container {
+  override string toString() { result = Impl::Container.super.toString() }
 }
 
 /**
@@ -166,9 +46,7 @@ class Container extends Locatable, @container {
  *
  * To get the full path, use `getAbsolutePath`.
  */
-class Folder extends Container, @folder {
-  override string getAbsolutePath() { folders(underlyingElement(this), result) }
-
+class Folder extends Container, Impl::Folder {
   override Location getLocation() {
     result.getContainer() = this and
     result.hasLocationInfo(_, 0, 0, 0, 0)
@@ -189,9 +67,7 @@ class Folder extends Container, @folder {
  * The base name further decomposes into the _stem_ and _extension_ -- see
  * `getStem` and `getExtension`. To get the full path, use `getAbsolutePath`.
  */
-class File extends Container, @file {
-  override string getAbsolutePath() { files(underlyingElement(this), result) }
-
+class File extends Container, Impl::File {
   override string getAPrimaryQlClass() { result = "File" }
 
   override Location getLocation() {

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowDispatch.qll

@@ -79,13 +79,3 @@ class ArgumentPosition extends int {
 /** Holds if arguments at position `apos` match parameters at position `ppos`. */
 pragma[inline]
 predicate parameterMatch(ParameterPosition ppos, ArgumentPosition apos) { ppos = apos }
-
-/**
- * Holds if flow from `call`'s argument `arg` to parameter `p` is permissible.
- *
- * This is a temporary hook to support technical debt in the Go language; do not use.
- */
-pragma[inline]
-predicate golangSpecificParamArgFilter(DataFlowCall call, ParameterNode p, ArgumentNode arg) {
-  any()
-}

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl1.qll

@@ -297,6 +297,10 @@ private module Config implements FullStateConfigSig {
 
   predicate isBarrierOut(Node node) { any(Configuration config).isBarrierOut(node) }
 
+  predicate isBarrierIn(Node node, FlowState state) { none() }
+
+  predicate isBarrierOut(Node node, FlowState state) { none() }
+
   predicate isAdditionalFlowStep(Node node1, Node node2) {
     singleConfiguration() and
     any(Configuration config).isAdditionalFlowStep(node1, node2)

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl2.qll

@@ -297,6 +297,10 @@ private module Config implements FullStateConfigSig {
 
   predicate isBarrierOut(Node node) { any(Configuration config).isBarrierOut(node) }
 
+  predicate isBarrierIn(Node node, FlowState state) { none() }
+
+  predicate isBarrierOut(Node node, FlowState state) { none() }
+
   predicate isAdditionalFlowStep(Node node1, Node node2) {
     singleConfiguration() and
     any(Configuration config).isAdditionalFlowStep(node1, node2)

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl3.qll

@@ -297,6 +297,10 @@ private module Config implements FullStateConfigSig {
 
   predicate isBarrierOut(Node node) { any(Configuration config).isBarrierOut(node) }
 
+  predicate isBarrierIn(Node node, FlowState state) { none() }
+
+  predicate isBarrierOut(Node node, FlowState state) { none() }
+
   predicate isAdditionalFlowStep(Node node1, Node node2) {
     singleConfiguration() and
     any(Configuration config).isAdditionalFlowStep(node1, node2)

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl4.qll

@@ -297,6 +297,10 @@ private module Config implements FullStateConfigSig {
 
   predicate isBarrierOut(Node node) { any(Configuration config).isBarrierOut(node) }
 
+  predicate isBarrierIn(Node node, FlowState state) { none() }
+
+  predicate isBarrierOut(Node node, FlowState state) { none() }
+
   predicate isAdditionalFlowStep(Node node1, Node node2) {
     singleConfiguration() and
     any(Configuration config).isAdditionalFlowStep(node1, node2)

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplLocal.qll

@@ -297,6 +297,10 @@ private module Config implements FullStateConfigSig {
 
   predicate isBarrierOut(Node node) { any(Configuration config).isBarrierOut(node) }
 
+  predicate isBarrierIn(Node node, FlowState state) { none() }
+
+  predicate isBarrierOut(Node node, FlowState state) { none() }
+
   predicate isAdditionalFlowStep(Node node1, Node node2) {
     singleConfiguration() and
     any(Configuration config).isAdditionalFlowStep(node1, node2)

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowPrivate.qll

@@ -297,12 +297,3 @@ class ContentApprox = Unit;
 /** Gets an approximated value for content `c`. */
 pragma[inline]
 ContentApprox getContentApprox(Content c) { any() }
-
-/**
- * Gets an additional term that is added to the `join` and `branch` computations to reflect
- * an additional forward or backwards branching factor that is not taken into account
- * when calculating the (virtual) dispatch cost.
- *
- * Argument `arg` is part of a path from a source to a sink, and `p` is the target parameter.
- */
-int getAdditionalFlowIntoCallNodeTerm(ArgumentNode arg, ParameterNode p) { none() }

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowDispatch.qll

@@ -271,13 +271,3 @@ DataFlowCallable viableImplInCallContext(DataFlowCall call, DataFlowCall ctx) {
 /** Holds if arguments at position `apos` match parameters at position `ppos`. */
 pragma[inline]
 predicate parameterMatch(ParameterPosition ppos, ArgumentPosition apos) { ppos = apos }
-
-/**
- * Holds if flow from `call`'s argument `arg` to parameter `p` is permissible.
- *
- * This is a temporary hook to support technical debt in the Go language; do not use.
- */
-pragma[inline]
-predicate golangSpecificParamArgFilter(DataFlowCall call, ParameterNode p, ArgumentNode arg) {
-  any()
-}

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl1.qll

@@ -297,6 +297,10 @@ private module Config implements FullStateConfigSig {
 
   predicate isBarrierOut(Node node) { any(Configuration config).isBarrierOut(node) }
 
+  predicate isBarrierIn(Node node, FlowState state) { none() }
+
+  predicate isBarrierOut(Node node, FlowState state) { none() }
+
   predicate isAdditionalFlowStep(Node node1, Node node2) {
     singleConfiguration() and
     any(Configuration config).isAdditionalFlowStep(node1, node2)

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl2.qll

@@ -297,6 +297,10 @@ private module Config implements FullStateConfigSig {
 
   predicate isBarrierOut(Node node) { any(Configuration config).isBarrierOut(node) }
 
+  predicate isBarrierIn(Node node, FlowState state) { none() }
+
+  predicate isBarrierOut(Node node, FlowState state) { none() }
+
   predicate isAdditionalFlowStep(Node node1, Node node2) {
     singleConfiguration() and
     any(Configuration config).isAdditionalFlowStep(node1, node2)

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl3.qll

@@ -297,6 +297,10 @@ private module Config implements FullStateConfigSig {
 
   predicate isBarrierOut(Node node) { any(Configuration config).isBarrierOut(node) }
 
+  predicate isBarrierIn(Node node, FlowState state) { none() }
+
+  predicate isBarrierOut(Node node, FlowState state) { none() }
+
   predicate isAdditionalFlowStep(Node node1, Node node2) {
     singleConfiguration() and
     any(Configuration config).isAdditionalFlowStep(node1, node2)

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl4.qll

@@ -297,6 +297,10 @@ private module Config implements FullStateConfigSig {
 
   predicate isBarrierOut(Node node) { any(Configuration config).isBarrierOut(node) }
 
+  predicate isBarrierIn(Node node, FlowState state) { none() }
+
+  predicate isBarrierOut(Node node, FlowState state) { none() }
+
   predicate isAdditionalFlowStep(Node node1, Node node2) {
     singleConfiguration() and
     any(Configuration config).isAdditionalFlowStep(node1, node2)

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplSpecific.qll

@@ -18,4 +18,6 @@ module CppDataFlow implements InputSig {
   import Public
 
   Node exprNode(DataFlowExpr e) { result = Public::exprNode(e) }
+
+  predicate getAdditionalFlowIntoCallNodeTerm = Private::getAdditionalFlowIntoCallNodeTerm/2;
 }

cpp/ql/src/Critical/DoubleFree.ql

@@ -2,7 +2,7 @@
  * @name Potential double free
  * @description Freeing a resource more than once can lead to undefined behavior and cause memory corruption.
  * @kind path-problem
- * @precision medium
+ * @precision high
  * @id cpp/double-free
  * @problem.severity warning
  * @security-severity 9.3

cpp/ql/src/Critical/UseAfterFree.ql

@@ -2,7 +2,7 @@
  * @name Potential use after free
  * @description An allocated memory block is used after it has been freed. Behavior in such cases is undefined and can cause memory corruption.
  * @kind path-problem
- * @precision medium
+ * @precision high
  * @id cpp/use-after-free
  * @problem.severity warning
  * @security-severity 9.3
@@ -29,8 +29,7 @@ private predicate externalCallNeverDereferences(FormattingFunctionCall call, int
   )
 }
 
-predicate isUse0(DataFlow::Node n, Expr e) {
-  e = n.asExpr() and
+predicate isUse0(Expr e) {
   not isFree(_, e, _) and
   (
     e = any(PointerDereferenceExpr pde).getOperand()
@@ -43,7 +42,7 @@ predicate isUse0(DataFlow::Node n, Expr e) {
     or
     // Assume any function without a body will dereference the pointer
     exists(int i, Call call, Function f |
-      n.asExpr() = call.getArgument(i) and
+      e = call.getArgument(i) and
       f = call.getTarget() and
       not f.hasEntryPoint() and
       // Exclude known functions we know won't dereference the pointer.
@@ -57,7 +56,7 @@ module ParameterSinks {
   import semmle.code.cpp.ir.ValueNumbering
 
   predicate flowsToUse(DataFlow::Node n) {
-    isUse0(n, _)
+    isUse0(n.asExpr())
     or
     exists(DataFlow::Node succ |
       flowsToUse(succ) and
@@ -90,7 +89,7 @@ module ParameterSinks {
   ) {
     pragma[only_bind_out](source.asParameter()) = pragma[only_bind_out](init.getParameter()) and
     paramToUse(source, sink) and
-    isUse0(sink, _)
+    isUse0(sink.asExpr())
   }
 
   private InitializeParameterInstruction getAnAlwaysDereferencedParameter0() {
@@ -139,7 +138,7 @@ module IsUse {
   private import semmle.code.cpp.ir.dataflow.internal.DataFlowImplCommon
 
   predicate isUse(DataFlow::Node n, Expr e) {
-    isUse0(n, e)
+    isUse0(e) and n.asExpr() = e
     or
     exists(CallInstruction call, InitializeParameterInstruction init |
       n.asOperand().getDef().getUnconvertedResultExpression() = e and

cpp/ql/src/change-notes/2023-10-03-double-free.md

@@ -0,0 +1,4 @@
+---
+category: queryMetadata
+---
+* The `cpp/double-free` query has been further improved to reduce false positives and its precision has been increased from `medium` to `high`.

cpp/ql/src/change-notes/2023-10-03-use-after-free.md

@@ -0,0 +1,4 @@
+---
+category: queryMetadata
+---
+* The `cpp/use-after-free` query has been further improved to reduce false positives and its precision has been increased from `medium` to `high`.

cpp/ql/test/query-tests/Critical/MemoryFreed/MemoryFreed.expected

@@ -96,6 +96,7 @@
 | test_free.cpp:255:10:255:10 | p |
 | test_free.cpp:260:9:260:9 | p |
 | test_free.cpp:263:12:263:12 | p |
+| test_free.cpp:269:7:269:11 | ... = ... |
 | virtual.cpp:18:10:18:10 | a |
 | virtual.cpp:19:10:19:10 | c |
 | virtual.cpp:38:10:38:10 | b |

cpp/ql/test/query-tests/Critical/MemoryFreed/MemoryMayNotBeFreed.expected

@@ -1 +1,2 @@
 | test_free.cpp:36:22:36:35 | ... = ... | This memory allocation may not be released at $@. | test_free.cpp:38:1:38:1 | return ... | this exit point |
+| test_free.cpp:267:12:267:17 | call to malloc | This memory allocation may not be released at $@. | test_free.cpp:270:1:270:1 | return ... | this exit point |

cpp/ql/test/query-tests/Critical/MemoryFreed/test_free.cpp

@@ -261,4 +261,10 @@ void test_ref_delete(int *&p) {
 	p = new int;
 	use(p);  // GOOD
     delete p;  // GOOD
+}
+
+void test_free_assign() {
+	void *a = malloc(10); 
+	void *b;
+	free(b = a); // GOOD 
 }

csharp/CSharp.sln

@@ -1,5 +1,4 @@
-
-Microsoft Visual Studio Solution File, Format Version 12.00
+Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio 15
 VisualStudioVersion = 15.0.27130.2036
 MinimumVisualStudioVersion = 10.0.40219.1
@@ -15,6 +14,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CSharp.De
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CSharp.Standalone", "extractor\Semmle.Extraction.CSharp.Standalone\Semmle.Extraction.CSharp.Standalone.csproj", "{D00E7D25-0FA0-48EC-B048-CD60CE1B30D8}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CSharp.StubGenerator", "extractor\Semmle.Extraction.CSharp.StubGenerator\Semmle.Extraction.CSharp.StubGenerator.csproj", "{B7C9FD47-A78C-4C20-AC29-B0AE638ADE9D}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CSharp.Util", "extractor\Semmle.Extraction.CSharp.Util\Semmle.Extraction.CSharp.Util.csproj", "{998A0D4C-8BFC-4513-A28D-4816AFB89882}"
+EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CIL.Driver", "extractor\Semmle.Extraction.CIL.Driver\Semmle.Extraction.CIL.Driver.csproj", "{EFA400B3-C1CE-446F-A4E2-8B44E61EF47C}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CSharp.Driver", "extractor\Semmle.Extraction.CSharp.Driver\Semmle.Extraction.CSharp.Driver.csproj", "{C36453BF-0C82-448A-B15D-26947503A2D3}"
@@ -29,6 +32,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Semmle.Autobuild.CSharp", "
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Semmle.Autobuild.CSharp.Tests", "autobuilder\Semmle.Autobuild.CSharp.Tests\Semmle.Autobuild.CSharp.Tests.csproj", "{34256E8F-866A-46C1-800E-3DF69FD1DCB7}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Semmle.Extraction.CSharp.DependencyStubGenerator", "extractor\Semmle.Extraction.CSharp.DependencyStubGenerator\Semmle.Extraction.CSharp.DependencyStubGenerator.csproj", "{0EDA21A3-ADD8-4C10-B494-58B12B526B76}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -85,6 +90,18 @@ Global
 		{34256E8F-866A-46C1-800E-3DF69FD1DCB7}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{34256E8F-866A-46C1-800E-3DF69FD1DCB7}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{34256E8F-866A-46C1-800E-3DF69FD1DCB7}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B7C9FD47-A78C-4C20-AC29-B0AE638ADE9D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B7C9FD47-A78C-4C20-AC29-B0AE638ADE9D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B7C9FD47-A78C-4C20-AC29-B0AE638ADE9D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B7C9FD47-A78C-4C20-AC29-B0AE638ADE9D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{998A0D4C-8BFC-4513-A28D-4816AFB89882}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{998A0D4C-8BFC-4513-A28D-4816AFB89882}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{998A0D4C-8BFC-4513-A28D-4816AFB89882}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{998A0D4C-8BFC-4513-A28D-4816AFB89882}.Release|Any CPU.Build.0 = Release|Any CPU
+		{0EDA21A3-ADD8-4C10-B494-58B12B526B76}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{0EDA21A3-ADD8-4C10-B494-58B12B526B76}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{0EDA21A3-ADD8-4C10-B494-58B12B526B76}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{0EDA21A3-ADD8-4C10-B494-58B12B526B76}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

csharp/actions/create-extractor-pack/action.yml

@@ -3,6 +3,7 @@ description: Builds the C# CodeQL pack
 runs:
   using: composite
   steps:
+    - uses: ./.github/actions/fetch-codeql
     - name: Setup dotnet
       uses: actions/setup-dotnet@v3
       with:
@@ -11,3 +12,13 @@ runs:
       shell: bash
       run: scripts/create-extractor-pack.sh
       working-directory: csharp
+    - name: Patch bundle to include ASP extractor
+      shell: bash
+      run: |
+        CODEQL_PATH=$(gh codeql version --format=json | jq -r .unpackedLocation)
+        # The legacy ASP extractor is not in this repo, so take the one from the nightly build
+        mv "$CODEQL_PATH/csharp/tools/extractor-asp.jar" "${{ github.workspace }}/csharp/extractor-pack/tools"
+        # Safe guard against using the bundled extractor
+        rm -rf "$CODEQL_PATH/csharp"
+      env:
+        GITHUB_TOKEN: ${{ github.token }}

csharp/documentation/library-coverage/coverage.csv

@@ -24,5 +24,5 @@ Microsoft.Win32,,,8,,,,,,,,,,,,,,8,
 MySql.Data.MySqlClient,48,,,,,,,,,,,48,,,,,,
 Newtonsoft.Json,,,91,,,,,,,,,,,,,,73,18
 ServiceStack,194,,7,27,,,,,75,,,92,,,,,7,
-System,65,25,12148,,8,8,9,,,4,3,33,1,17,3,4,10163,1985
+System,65,25,12149,,8,8,9,,,4,3,33,1,17,3,4,10163,1986
 Windows.Security.Cryptography.Core,1,,,,,,,1,,,,,,,,,,

csharp/documentation/library-coverage/coverage.rst

@@ -8,7 +8,7 @@ C# framework & library support
 
    Framework / library,Package,Flow sources,Taint & value steps,Sinks (total),`CWE-079` :sub:`Cross-site scripting`
    `ServiceStack <https://servicestack.net/>`_,"``ServiceStack.*``, ``ServiceStack``",,7,194,
-   System,"``System.*``, ``System``",25,12148,65,7
+   System,"``System.*``, ``System``",25,12149,65,7
    Others,"``Dapper``, ``JsonToItemsTaskFactory``, ``Microsoft.ApplicationBlocks.Data``, ``Microsoft.CSharp``, ``Microsoft.EntityFrameworkCore``, ``Microsoft.Extensions.Caching.Distributed``, ``Microsoft.Extensions.Caching.Memory``, ``Microsoft.Extensions.Configuration``, ``Microsoft.Extensions.DependencyInjection``, ``Microsoft.Extensions.DependencyModel``, ``Microsoft.Extensions.FileProviders``, ``Microsoft.Extensions.FileSystemGlobbing``, ``Microsoft.Extensions.Hosting``, ``Microsoft.Extensions.Http``, ``Microsoft.Extensions.Logging``, ``Microsoft.Extensions.Options``, ``Microsoft.Extensions.Primitives``, ``Microsoft.Interop``, ``Microsoft.NET.Build.Tasks``, ``Microsoft.NETCore.Platforms.BuildTasks``, ``Microsoft.VisualBasic``, ``Microsoft.Win32``, ``MySql.Data.MySqlClient``, ``Newtonsoft.Json``, ``Windows.Security.Cryptography.Core``",,568,138,
-   Totals,,25,12723,397,7
+   Totals,,25,12724,397,7
 

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -23,7 +23,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         private readonly IDictionary<string, string> unresolvedReferences = new ConcurrentDictionary<string, string>();
         private int failedProjects;
         private int succeededProjects;
-        private readonly List<string> allSources;
+        private readonly List<string> nonGeneratedSources;
+        private readonly List<string> generatedSources;
         private int conflictedReferences = 0;
         private readonly IDependencyOptions options;
         private readonly DirectoryInfo sourceDir;
@@ -31,6 +32,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         private readonly FileContent fileContent;
         private readonly TemporaryDirectory packageDirectory;
         private readonly TemporaryDirectory tempWorkingDirectory;
+        private readonly bool cleanupTempWorkingDirectory;
 
         /// <summary>
         /// Performs C# dependency fetching.
@@ -58,14 +60,15 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             this.progressMonitor.FindingFiles(srcDir);
 
             packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName));
-            tempWorkingDirectory = new TemporaryDirectory(GetTemporaryWorkingDirectory());
+            tempWorkingDirectory = new TemporaryDirectory(FileUtils.GetTemporaryWorkingDirectory(out cleanupTempWorkingDirectory));
 
             var allFiles = GetAllFiles();
             var binaryFileExtensions = new HashSet<string>(new[] { ".dll", ".exe" }); // TODO: add more binary file extensions.
             var allNonBinaryFiles = allFiles.Where(f => !binaryFileExtensions.Contains(f.Extension.ToLowerInvariant())).ToList();
             var smallNonBinaryFiles = allNonBinaryFiles.SelectSmallFiles(progressMonitor).SelectFileNames();
             this.fileContent = new FileContent(progressMonitor, smallNonBinaryFiles);
-            this.allSources = allNonBinaryFiles.SelectFileNamesByExtension(".cs").ToList();
+            this.nonGeneratedSources = allNonBinaryFiles.SelectFileNamesByExtension(".cs").ToList();
+            this.generatedSources = new();
             var allProjects = allNonBinaryFiles.SelectFileNamesByExtension(".csproj");
             var solutions = options.SolutionFile is not null
                 ? new[] { options.SolutionFile }
@@ -164,6 +167,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                 Path.Combine(packageFolder, "microsoft.netcore.app.runtime"),
                 Path.Combine(packageFolder, "microsoft.aspnetcore.app.runtime"),
                 Path.Combine(packageFolder, "microsoft.windowsdesktop.app.runtime"),
+
+                // legacy runtime packages:
+                Path.Combine(packageFolder, "runtime.linux-x64.microsoft.netcore.app"),
+                Path.Combine(packageFolder, "runtime.osx-x64.microsoft.netcore.app"),
+                Path.Combine(packageFolder, "runtime.win-x64.microsoft.netcore.app"),
             };
 
             foreach (var filename in usedReferences.Keys)
@@ -214,7 +222,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                     }
                 }
 
-                this.allSources.Add(path);
+                this.generatedSources.Add(path);
             }
         }
 
@@ -236,7 +244,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                         var razor = new Razor(sdk, dotnet, progressMonitor);
                         var targetDir = GetTemporaryWorkingDirectory("razor");
                         var generatedFiles = razor.GenerateFiles(views, usedReferences.Keys, targetDir);
-                        this.allSources.AddRange(generatedFiles);
+                        this.generatedSources.AddRange(generatedFiles);
                     }
                     catch (Exception ex)
                     {
@@ -278,20 +286,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             return Path.Combine(Path.GetTempPath(), "GitHub", "packages", sb.ToString());
         }
 
-        private static string GetTemporaryWorkingDirectory()
-        {
-            var tempFolder = EnvironmentVariables.GetScratchDirectory();
-
-            if (string.IsNullOrEmpty(tempFolder))
-            {
-                var tempPath = Path.GetTempPath();
-                var name = Guid.NewGuid().ToString("N").ToUpper();
-                tempFolder = Path.Combine(tempPath, "GitHub", name);
-            }
-
-            return tempFolder;
-        }
-
         /// <summary>
         /// Creates a temporary directory with the given subfolder name.
         /// The created directory might be inside the repo folder, and it is deleted when the object is disposed.
@@ -379,10 +373,15 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         /// </summary>
         public IEnumerable<string> ProjectSourceFiles => sources.Where(s => s.Value).Select(s => s.Key);
 
+        /// <summary>
+        /// All of the generated source files in the source directory.
+        /// </summary>
+        public IEnumerable<string> GeneratedSourceFiles => generatedSources;
+
         /// <summary>
         /// All of the source files in the source directory.
         /// </summary>
-        public IEnumerable<string> AllSourceFiles => allSources;
+        public IEnumerable<string> AllSourceFiles => generatedSources.Concat(nonGeneratedSources);
 
         /// <summary>
         /// List of assembly IDs which couldn't be resolved.
@@ -562,7 +561,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         public void Dispose()
         {
             packageDirectory?.Dispose();
-            tempWorkingDirectory?.Dispose();
+            if (cleanupTempWorkingDirectory)
+                tempWorkingDirectory?.Dispose();
         }
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyStubGenerator/Program.cs

@@ -0,0 +1,9 @@
+using Semmle.Extraction.CSharp.DependencyFetching;
+using Semmle.Extraction.CSharp.StubGenerator;
+using Semmle.Util.Logging;
+
+var logger = new ConsoleLogger(Verbosity.Info, logThreadId: false);
+using var dependencyManager = new DependencyManager(".", DependencyOptions.Default, logger);
+StubGenerator.GenerateStubs(logger, dependencyManager.ReferenceFiles, "codeql_csharp_stubs");
+
+return 0;

csharp/extractor/Semmle.Extraction.CSharp.DependencyStubGenerator/Semmle.Extraction.CSharp.DependencyStubGenerator.csproj

@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net7.0</TargetFramework>
+    <AssemblyName>Semmle.Extraction.CSharp.DependencyStubGenerator</AssemblyName>
+    <RootNamespace>Semmle.Extraction.CSharp.DependencyStubGenerator</RootNamespace>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+	<ItemGroup>
+		<ProjectReference Include="..\Semmle.Util\Semmle.Util.csproj" />
+		<ProjectReference Include="..\Semmle.Extraction.CSharp.DependencyFetching\Semmle.Extraction.CSharp.DependencyFetching.csproj" />
+		<ProjectReference Include="..\Semmle.Extraction.CSharp.StubGenerator\Semmle.Extraction.CSharp.StubGenerator.csproj" />
+	</ItemGroup>
+
+</Project>

csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs

@@ -30,29 +30,46 @@ namespace Semmle.Extraction.CSharp.Standalone
             IProgressMonitor progressMonitor,
             Stopwatch stopwatch)
         {
-            CSharp.Extractor.Analyse(stopwatch, analyser, options,
-                references => GetResolvedReferencesStandalone(referencePaths, references),
-                (analyser, syntaxTrees) => CSharp.Extractor.ReadSyntaxTrees(sources, analyser, null, null, syntaxTrees),
-                (syntaxTrees, references) => CSharpCompilation.Create(
-                    "csharp.dll", syntaxTrees, references, new CSharpCompilationOptions(OutputKind.ConsoleApplication, allowUnsafe: true)
-                    ),
-                (compilation, options) => analyser.Initialize(compilation, options),
-                () => { },
-                _ => { },
-                () =>
+            var output = FileUtils.CreateTemporaryFile(".dll", out var shouldCleanUpContainingFolder);
+
+            try
+            {
+                CSharp.Extractor.Analyse(stopwatch, analyser, options,
+                    references => GetResolvedReferencesStandalone(referencePaths, references),
+                    (analyser, syntaxTrees) => CSharp.Extractor.ReadSyntaxTrees(sources, analyser, null, null, syntaxTrees),
+                    (syntaxTrees, references) => CSharpCompilation.Create(
+                        output.Name, syntaxTrees, references, new CSharpCompilationOptions(OutputKind.ConsoleApplication, allowUnsafe: true)
+                        ),
+                    (compilation, options) => analyser.Initialize(output.FullName, compilation, options),
+                    _ => { },
+                    () =>
+                    {
+                        foreach (var type in analyser.MissingNamespaces)
+                        {
+                            progressMonitor.MissingNamespace(type);
+                        }
+
+                        foreach (var type in analyser.MissingTypes)
+                        {
+                            progressMonitor.MissingType(type);
+                        }
+
+                        progressMonitor.MissingSummary(analyser.MissingTypes.Count(), analyser.MissingNamespaces.Count());
+                    });
+            }
+            finally
+            {
+                try
                 {
-                    foreach (var type in analyser.MissingNamespaces)
+                    FileUtils.TryDelete(output.FullName);
+                    if (shouldCleanUpContainingFolder)
                     {
-                        progressMonitor.MissingNamespace(type);
+                        output.Directory?.Delete(true);
                     }
-
-                    foreach (var type in analyser.MissingTypes)
-                    {
-                        progressMonitor.MissingType(type);
-                    }
-
-                    progressMonitor.MissingSummary(analyser.MissingTypes.Count(), analyser.MissingNamespaces.Count());
-                });
+                }
+                catch
+                { }
+            }
         }
 
         private static void ExtractStandalone(

csharp/extractor/Semmle.Extraction.CSharp.StubGenerator/IRelevantSymbol.cs

@@ -0,0 +1,9 @@
+using Microsoft.CodeAnalysis;
+
+namespace Semmle.Extraction.CSharp.StubGenerator;
+
+internal interface IRelevantSymbol
+{
+    bool IsRelevantNamedType(INamedTypeSymbol symbol);
+    bool IsRelevantNamespace(INamespaceSymbol symbol);
+}

csharp/extractor/Semmle.Extraction.CSharp.StubGenerator/Properties/AssemblyInfo.cs

@@ -0,0 +1,39 @@
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+// General Information about an assembly is controlled through the following
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+[assembly: AssemblyTitle("Semmle.Extraction.CSharp.StubGenerator")]
+[assembly: AssemblyDescription("Stub generator for C#")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("")]
+[assembly: AssemblyProduct("Semmle.Extraction.CSharp.StubGenerator")]
+[assembly: AssemblyCopyright("Copyright ©  2023")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// Setting ComVisible to false makes the types in this assembly not visible
+// to COM components.  If you need to access a type in this assembly from
+// COM, set the ComVisible attribute to true on that type.
+[assembly: ComVisible(false)]
+
+// The following GUID is for the ID of the typelib if this project is exposed to COM
+[assembly: Guid("6d55ca39-615a-4c1b-a648-a4010995e1ea")]
+
+// Expose internals for testing purposes.
+[assembly: InternalsVisibleTo("Semmle.Extraction.Tests")]
+
+// Version information for an assembly consists of the following four values:
+//
+//      Major Version
+//      Minor Version
+//      Build Number
+//      Revision
+//
+// You can specify all the values or you can default the Build and Revision Numbers
+// by using the '*' as shown below:
+// [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]

csharp/extractor/Semmle.Extraction.CSharp.StubGenerator/RelevantSymbol.cs

@@ -0,0 +1,30 @@
+using System.Linq;
+using Microsoft.CodeAnalysis;
+using Semmle.Util;
+
+namespace Semmle.Extraction.CSharp.StubGenerator;
+
+internal class RelevantSymbol : IRelevantSymbol
+{
+    private readonly IAssemblySymbol assembly;
+    private readonly MemoizedFunc<INamedTypeSymbol, bool> isRelevantNamedType;
+    private readonly MemoizedFunc<INamespaceSymbol, bool> isRelevantNamespace;
+
+    public RelevantSymbol(IAssemblySymbol assembly)
+    {
+        this.assembly = assembly;
+
+        isRelevantNamedType = new(symbol =>
+            StubVisitor.IsRelevantBaseType(symbol) &&
+            SymbolEqualityComparer.Default.Equals(symbol.ContainingAssembly, assembly));
+
+        isRelevantNamespace = new(symbol =>
+            symbol.GetTypeMembers().Any(IsRelevantNamedType) ||
+            symbol.GetNamespaceMembers().Any(IsRelevantNamespace));
+    }
+
+    public bool IsRelevantNamedType(INamedTypeSymbol symbol) => isRelevantNamedType.Invoke(symbol);
+
+    public bool IsRelevantNamespace(INamespaceSymbol symbol) => isRelevantNamespace.Invoke(symbol);
+}
+

csharp/extractor/Semmle.Extraction.CSharp.StubGenerator/Semmle.Extraction.CSharp.StubGenerator.csproj

@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+	<PropertyGroup>
+		<TargetFramework>net7.0</TargetFramework>
+		<AssemblyName>Semmle.Extraction.CSharp.StubGenerator</AssemblyName>
+		<RootNamespace>Semmle.Extraction.CSharp.StubGenerator</RootNamespace>
+		<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
+		<RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
+		<Nullable>enable</Nullable>
+	</PropertyGroup>
+	<ItemGroup>
+		<ProjectReference Include="..\Semmle.Util\Semmle.Util.csproj" />
+		<ProjectReference Include="..\Semmle.Extraction.CSharp.DependencyFetching\Semmle.Extraction.CSharp.DependencyFetching.csproj" />
+		<ProjectReference Include="..\Semmle.Extraction.CSharp.Util\Semmle.Extraction.CSharp.Util.csproj" />
+	</ItemGroup>
+	<ItemGroup>
+		<PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="4.4.0" />
+	</ItemGroup>
+</Project>

csharp/extractor/Semmle.Extraction.CSharp.StubGenerator/StubGenerator.cs

@@ -0,0 +1,85 @@
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.CSharp;
+
+using Semmle.Util;
+using Semmle.Util.Logging;
+
+namespace Semmle.Extraction.CSharp.StubGenerator;
+
+public static class StubGenerator
+{
+    /// <summary>
+    /// Generates stubs for all the provided assembly paths.
+    /// </summary>
+    /// <param name="referencesPaths">The paths of the assemblies to generate stubs for.</param>
+    /// <param name="outputPath">The path in which to store the stubs.</param>
+    public static string[] GenerateStubs(ILogger logger, IEnumerable<string> referencesPaths, string outputPath)
+    {
+        var stopWatch = new System.Diagnostics.Stopwatch();
+        stopWatch.Start();
+
+        var threads = EnvironmentVariables.GetDefaultNumberOfThreads();
+
+        using var stubPaths = new BlockingCollection<string>();
+        using var references = new BlockingCollection<(MetadataReference Reference, string Path)>();
+
+        Parallel.ForEach(referencesPaths, new ParallelOptions { MaxDegreeOfParallelism = threads }, path =>
+        {
+            var reference = MetadataReference.CreateFromFile(path);
+            references.Add((reference, path));
+        });
+
+        logger.Log(Severity.Info, $"Generating stubs for {references.Count} assemblies.");
+
+        var compilation = CSharpCompilation.Create(
+            "stubgenerator.dll",
+            null,
+            references.Select(tuple => tuple.Item1),
+            new CSharpCompilationOptions(OutputKind.ConsoleApplication, allowUnsafe: true));
+
+        Parallel.ForEach(references, new ParallelOptions { MaxDegreeOfParallelism = threads }, @ref =>
+        {
+            StubReference(logger, compilation, outputPath, @ref.Reference, @ref.Path, stubPaths);
+        });
+
+        stopWatch.Stop();
+        logger.Log(Severity.Info, $"Stub generation took {stopWatch.Elapsed}.");
+
+        return stubPaths.ToArray();
+    }
+
+    private static void StubReference(ILogger logger, CSharpCompilation compilation, string outputPath, MetadataReference reference, string path, BlockingCollection<string> stubPaths)
+    {
+        if (compilation.GetAssemblyOrModuleSymbol(reference) is not IAssemblySymbol assembly)
+            return;
+
+        var relevantSymbol = new RelevantSymbol(assembly);
+
+        if (!assembly.Modules.Any(m => relevantSymbol.IsRelevantNamespace(m.GlobalNamespace)))
+            return;
+
+        var stubPath = FileUtils.NestPaths(logger, outputPath, path.Replace(".dll", ".cs"));
+        stubPaths.Add(stubPath);
+        using var fileStream = new FileStream(stubPath, FileMode.Create, FileAccess.Write);
+        using var writer = new StreamWriter(fileStream, new UTF8Encoding(false)) { NewLine = "\n" };
+
+        var visitor = new StubVisitor(writer, relevantSymbol);
+
+        writer.WriteLine("// This file contains auto-generated code.");
+        writer.WriteLine($"// Generated from `{assembly.Identity}`.");
+
+        visitor.StubAttributes(assembly.GetAttributes(), "assembly: ");
+
+        foreach (var module in assembly.Modules)
+        {
+            module.GlobalNamespace.Accept(visitor);
+        }
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp.StubGenerator/StubVisitor.cs

@@ -0,0 +1,868 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+
+using Microsoft.CodeAnalysis;
+
+using Semmle.Extraction.CSharp.Util;
+
+namespace Semmle.Extraction.CSharp.StubGenerator;
+
+internal sealed class StubVisitor : SymbolVisitor
+{
+    private readonly TextWriter stubWriter;
+    private readonly IRelevantSymbol relevantSymbol;
+
+    public StubVisitor(TextWriter stubWriter, IRelevantSymbol relevantSymbol)
+    {
+        this.stubWriter = stubWriter;
+        this.relevantSymbol = relevantSymbol;
+    }
+
+    private static bool IsNotPublic(Accessibility accessibility) =>
+        accessibility == Accessibility.Private ||
+        accessibility == Accessibility.Internal ||
+        accessibility == Accessibility.ProtectedAndInternal;
+
+    public static bool IsRelevantBaseType(INamedTypeSymbol symbol) =>
+        !IsNotPublic(symbol.DeclaredAccessibility) &&
+        symbol.CanBeReferencedByName;
+
+    private void StubExplicitInterface(ISymbol symbol, ISymbol? explicitInterfaceSymbol, bool writeName = true)
+    {
+        static bool ContainsTupleType(ITypeSymbol type) =>
+            type is INamedTypeSymbol named && (named.IsTupleType || named.TypeArguments.Any(ContainsTupleType)) ||
+            type is IArrayTypeSymbol array && ContainsTupleType(array.ElementType) ||
+            type is IPointerTypeSymbol pointer && ContainsTupleType(pointer.PointedAtType);
+
+        static bool EqualsModuloTupleElementNames(ITypeSymbol t1, ITypeSymbol t2) =>
+           SymbolEqualityComparer.Default.Equals(t1, t2) ||
+            (
+                t1 is INamedTypeSymbol named1 &&
+                t2 is INamedTypeSymbol named2 &&
+                EqualsModuloTupleElementNames(named1.ConstructedFrom, named2.ConstructedFrom) &&
+                named1.TypeArguments.Length == named2.TypeArguments.Length &&
+                named1.TypeArguments.Zip(named2.TypeArguments).All(p => EqualsModuloTupleElementNames(p.First, p.Second))
+            ) ||
+            (
+                t1 is IArrayTypeSymbol array1 &&
+                t2 is IArrayTypeSymbol array2 &&
+                EqualsModuloTupleElementNames(array1.ElementType, array2.ElementType)
+            ) ||
+            (
+                t1 is IPointerTypeSymbol pointer1 &&
+                t2 is IPointerTypeSymbol pointer2 &&
+                EqualsModuloTupleElementNames(pointer1.PointedAtType, pointer2.PointedAtType)
+            );
+
+        if (explicitInterfaceSymbol is not null)
+        {
+            var explicitInterfaceType = explicitInterfaceSymbol.ContainingType;
+
+            // Workaround for when the explicit interface type contains named tuple types,
+            // in which case Roslyn may incorrectly forget the names of the tuple elements.
+            //
+            // For example, without this workaround we would incorrectly generate the following stub:
+            //
+            // ```csharp
+            // public sealed class UnorderedItemsCollection : System.Collections.Generic.IEnumerable<(TElement Element, TPriority Priority)>, ...
+            // {
+            //     System.Collections.Generic.IEnumerator<(TElement Element, TPriority Priority)> System.Collections.Generic.IEnumerable<(TElement, TPriority)>.GetEnumerator() => throw null;
+            // }
+            // ```
+            if (ContainsTupleType(explicitInterfaceType))
+            {
+                explicitInterfaceType = symbol.ContainingType.Interfaces.First(i => ContainsTupleType(i) && EqualsModuloTupleElementNames(i, explicitInterfaceSymbol.ContainingType));
+            }
+
+            stubWriter.Write(explicitInterfaceType.GetQualifiedName());
+            stubWriter.Write('.');
+            if (writeName)
+                stubWriter.Write(explicitInterfaceSymbol.GetName());
+        }
+        else if (writeName)
+        {
+            stubWriter.Write(symbol.GetName());
+        }
+    }
+
+    private void StubAccessibility(Accessibility accessibility)
+    {
+        switch (accessibility)
+        {
+            case Accessibility.Public:
+                stubWriter.Write("public ");
+                break;
+            case Accessibility.Protected or Accessibility.ProtectedOrInternal:
+                stubWriter.Write("protected ");
+                break;
+            case Accessibility.Internal:
+                stubWriter.Write("internal ");
+                break;
+            case Accessibility.ProtectedAndInternal:
+                stubWriter.Write("protected internal ");
+                break;
+            default:
+                stubWriter.Write($"/* TODO: {accessibility} */");
+                break;
+        }
+    }
+
+    private void StubModifiers(ISymbol symbol, bool skipAccessibility = false)
+    {
+        if (symbol.ContainingType is ITypeSymbol containing && containing.TypeKind == TypeKind.Interface)
+            skipAccessibility = true;
+
+        if (symbol is IMethodSymbol method && method.MethodKind == MethodKind.Constructor && symbol.IsStatic)
+            skipAccessibility = true;
+
+        if (!skipAccessibility)
+            StubAccessibility(symbol.DeclaredAccessibility);
+
+        if (symbol.IsAbstract)
+        {
+            if (
+                // exclude interface declarations
+                (symbol is not INamedTypeSymbol type || type.TypeKind != TypeKind.Interface) &&
+                // exclude non-static interface members
+                (symbol.ContainingType is not INamedTypeSymbol containingType || containingType.TypeKind != TypeKind.Interface || symbol.IsStatic))
+            {
+                stubWriter.Write("abstract ");
+            }
+        }
+
+        if (symbol.IsStatic && !(symbol is IFieldSymbol field && field.IsConst))
+            stubWriter.Write("static ");
+        if (symbol.IsVirtual)
+            stubWriter.Write("virtual ");
+        if (symbol.IsOverride)
+            stubWriter.Write("override ");
+        if (symbol.IsSealed)
+        {
+            if (!(symbol is INamedTypeSymbol type && (type.TypeKind == TypeKind.Enum || type.TypeKind == TypeKind.Delegate || type.TypeKind == TypeKind.Struct)))
+                stubWriter.Write("sealed ");
+        }
+        if (symbol.IsExtern)
+            stubWriter.Write("extern ");
+    }
+
+    private void StubTypedConstant(TypedConstant c)
+    {
+        switch (c.Kind)
+        {
+            case TypedConstantKind.Primitive:
+                if (c.Value is string s)
+                {
+                    stubWriter.Write($"\"{s}\"");
+                }
+                else if (c.Value is char ch)
+                {
+                    stubWriter.Write($"'{ch}'");
+                }
+                else if (c.Value is bool b)
+                {
+                    stubWriter.Write(b ? "true" : "false");
+                }
+                else if (c.Value is int i)
+                {
+                    stubWriter.Write(i);
+                }
+                else if (c.Value is long l)
+                {
+                    stubWriter.Write(l);
+                }
+                else if (c.Value is float f)
+                {
+                    stubWriter.Write(f);
+                }
+                else if (c.Value is double d)
+                {
+                    stubWriter.Write(d);
+                }
+                else
+                {
+                    stubWriter.Write("throw null");
+                }
+                break;
+            case TypedConstantKind.Enum:
+                stubWriter.Write("throw null");
+                break;
+            case TypedConstantKind.Array:
+                stubWriter.Write("new []{");
+                WriteCommaSep(c.Values, StubTypedConstant);
+                stubWriter.Write("}");
+                break;
+            default:
+                stubWriter.Write($"/* TODO: {c.Kind} */ throw null");
+                break;
+        }
+    }
+
+    private static readonly HashSet<string> attributeAllowList = new() {
+        "System.FlagsAttribute"
+    };
+
+    private void StubAttribute(AttributeData a, string prefix)
+    {
+        if (a.AttributeClass is not INamedTypeSymbol @class)
+            return;
+
+        var qualifiedName = @class.GetQualifiedName();
+        if (!attributeAllowList.Contains(qualifiedName))
+            return;
+
+        if (qualifiedName.EndsWith("Attribute"))
+            qualifiedName = qualifiedName[..^9];
+        stubWriter.Write($"[{prefix}{qualifiedName}");
+        if (a.ConstructorArguments.Any())
+        {
+            stubWriter.Write("(");
+            WriteCommaSep(a.ConstructorArguments, StubTypedConstant);
+            stubWriter.Write(")");
+        }
+        stubWriter.WriteLine("]");
+    }
+
+    public void StubAttributes(IEnumerable<AttributeData> a, string prefix = "")
+    {
+        foreach (var attribute in a)
+        {
+            StubAttribute(attribute, prefix);
+        }
+    }
+
+    private void StubEvent(IEventSymbol symbol, IEventSymbol? explicitInterfaceSymbol)
+    {
+        StubAttributes(symbol.GetAttributes());
+
+        StubModifiers(symbol, explicitInterfaceSymbol is not null);
+        stubWriter.Write("event ");
+        stubWriter.Write(symbol.Type.GetQualifiedName());
+        stubWriter.Write(" ");
+
+        StubExplicitInterface(symbol, explicitInterfaceSymbol);
+
+        if (explicitInterfaceSymbol is null)
+        {
+            stubWriter.WriteLine(";");
+        }
+        else
+        {
+            stubWriter.Write(" { ");
+            stubWriter.Write("add {} ");
+            stubWriter.Write("remove {} ");
+            stubWriter.WriteLine("}");
+        }
+    }
+
+    private static T[] FilterExplicitInterfaceImplementations<T>(IEnumerable<T> explicitInterfaceImplementations) where T : ISymbol =>
+        explicitInterfaceImplementations.Where(i => IsRelevantBaseType(i.ContainingType)).ToArray();
+
+    public override void VisitEvent(IEventSymbol symbol)
+    {
+        var explicitInterfaceImplementations = FilterExplicitInterfaceImplementations(symbol.ExplicitInterfaceImplementations);
+
+        if (IsNotPublic(symbol.DeclaredAccessibility) && explicitInterfaceImplementations.Length == 0)
+            return;
+
+        foreach (var explicitInterfaceSymbol in explicitInterfaceImplementations)
+        {
+            StubEvent(symbol, explicitInterfaceSymbol);
+        }
+
+        if (explicitInterfaceImplementations.Length == 0)
+            StubEvent(symbol, null);
+    }
+
+    private static bool IsUnsafe(ITypeSymbol symbol) =>
+        symbol.TypeKind == TypeKind.Pointer ||
+        symbol.TypeKind == TypeKind.FunctionPointer ||
+        (symbol is INamedTypeSymbol named && named.TypeArguments.Any(IsUnsafe)) ||
+        (symbol is IArrayTypeSymbol at && IsUnsafe(at.ElementType));
+
+    private static readonly HashSet<string> keywords = new() {
+        "abstract", "as", "base", "bool", "break", "byte", "case", "catch", "char", "checked",
+        "class", "const", "continue", "decimal", "default", "delegate", "do", "double", "else",
+        "enum", "event", "explicit", "extern", "false", "finally", "fixed", "float", "for", "foreach",
+        "goto", "if", "implicit", "in", "int", "interface", "internal", "is", "lock", "long",
+        "namespace", "new", "null", "object", "operator", "out", "override", "params", "private",
+        "protected", "public", "readonly", "ref", "return", "sbyte", "sealed", "short", "sizeof",
+        "stackalloc", "static", "string", "struct", "switch", "this", "throw", "true", "try",
+        "typeof", "uint", "ulong", "unchecked", "unsafe", "ushort", "using", "virtual", "void",
+        "volatile", "while"
+    };
+
+    private static string EscapeIdentifier(string identifier) =>
+        keywords.Contains(identifier) ? "@" + identifier : identifier;
+
+    private static bool TryGetConstantValue(IFieldSymbol symbol, out string value)
+    {
+        value = "";
+        if (!symbol.HasConstantValue)
+        {
+            return false;
+        }
+
+        var v = symbol.ConstantValue;
+        switch (symbol.Type.SpecialType)
+        {
+            case SpecialType.System_Boolean:
+                value = (bool)v ? "true" : "false";
+                return true;
+            case SpecialType.System_Byte:
+            case SpecialType.System_SByte:
+            case SpecialType.System_UInt16:
+            case SpecialType.System_UInt32:
+            case SpecialType.System_UInt64:
+            case SpecialType.System_Int16:
+            case SpecialType.System_Int32:
+            case SpecialType.System_Int64:
+                value = v.ToString()!;
+                return true;
+            default:
+                return false;
+        }
+    }
+
+    public override void VisitField(IFieldSymbol symbol)
+    {
+        if (IsNotPublic(symbol.DeclaredAccessibility))
+            return;
+
+        StubAttributes(symbol.GetAttributes());
+
+        StubModifiers(symbol);
+
+        if (symbol.IsConst)
+        {
+            stubWriter.Write("const ");
+        }
+
+        if (!symbol.IsConst && symbol.IsReadOnly)
+        {
+            stubWriter.Write("readonly ");
+        }
+
+        if (IsUnsafe(symbol.Type))
+        {
+            stubWriter.Write("unsafe ");
+        }
+
+        stubWriter.Write(symbol.Type.GetQualifiedName());
+        stubWriter.Write(" ");
+        stubWriter.Write(EscapeIdentifier(symbol.Name));
+        if (symbol.IsConst)
+        {
+            var v = TryGetConstantValue(symbol, out var value) ? value : "default";
+            stubWriter.Write($" = {v}");
+        }
+        stubWriter.WriteLine(";");
+    }
+
+    private void WriteCommaSep<T>(IEnumerable<T> items, Action<T> writeItem)
+    {
+        var first = true;
+        foreach (var item in items)
+        {
+            if (!first)
+            {
+                stubWriter.Write(", ");
+            }
+            writeItem(item);
+            first = false;
+        }
+    }
+
+    private void WriteStringCommaSep<T>(IEnumerable<T> items, Func<T, string> writeItem)
+    {
+        WriteCommaSep(items, item => stubWriter.Write(writeItem(item)));
+    }
+
+    private void StubTypeParameters(IEnumerable<ITypeParameterSymbol> typeParameters)
+    {
+        if (!typeParameters.Any())
+            return;
+
+        stubWriter.Write('<');
+        WriteStringCommaSep(typeParameters, typeParameter => typeParameter.Name);
+        stubWriter.Write('>');
+    }
+
+    private void StubTypeParameterConstraints(IEnumerable<ITypeParameterSymbol> typeParameters)
+    {
+        if (!typeParameters.Any())
+            return;
+
+        var inheritsConstraints = typeParameters.Any(tp =>
+            tp.DeclaringMethod is IMethodSymbol method &&
+            (method.IsOverride || method.ExplicitInterfaceImplementations.Any()));
+
+        foreach (var typeParameter in typeParameters)
+        {
+            var firstTypeParameterConstraint = true;
+
+            void WriteTypeParameterConstraint(Action a)
+            {
+                if (firstTypeParameterConstraint)
+                {
+                    stubWriter.Write($" where {typeParameter.Name} : ");
+                }
+                else
+                {
+                    stubWriter.Write(", ");
+                }
+                a();
+                firstTypeParameterConstraint = false;
+            }
+
+            if (typeParameter.HasReferenceTypeConstraint)
+            {
+                WriteTypeParameterConstraint(() => stubWriter.Write("class"));
+            }
+
+            if (typeParameter.HasValueTypeConstraint &&
+                !typeParameter.HasUnmanagedTypeConstraint &&
+                !typeParameter.ConstraintTypes.Any(t => t.GetQualifiedName() is "System.Enum"))
+            {
+                WriteTypeParameterConstraint(() => stubWriter.Write("struct"));
+            }
+
+            if (inheritsConstraints)
+                continue;
+
+            if (typeParameter.HasUnmanagedTypeConstraint)
+            {
+                WriteTypeParameterConstraint(() => stubWriter.Write("unmanaged"));
+            }
+
+            var constraintTypes = typeParameter.ConstraintTypes.Select(t => t.GetQualifiedName()).Where(s => s is not "").ToArray();
+            if (constraintTypes.Any())
+            {
+                WriteTypeParameterConstraint(() =>
+                {
+                    WriteStringCommaSep(constraintTypes, constraintType => constraintType);
+                });
+            }
+
+            if (typeParameter.HasConstructorConstraint)
+            {
+                WriteTypeParameterConstraint(() => stubWriter.Write("new()"));
+            }
+        }
+    }
+
+    private static INamedTypeSymbol? GetBaseType(INamedTypeSymbol symbol)
+    {
+        if (symbol.BaseType is INamedTypeSymbol @base &&
+            @base.SpecialType != SpecialType.System_Object &&
+            @base.SpecialType != SpecialType.System_ValueType)
+        {
+            return @base;
+        }
+
+        return null;
+    }
+
+    private static IMethodSymbol? GetBaseConstructor(INamedTypeSymbol symbol)
+    {
+        if (GetBaseType(symbol) is not INamedTypeSymbol @base)
+            return null;
+
+        var containingTypes = new HashSet<INamedTypeSymbol>(SymbolEqualityComparer.Default);
+        var current = symbol;
+        while (current is not null)
+        {
+            containingTypes.Add(current);
+            current = current.ContainingType;
+        }
+
+        var baseCtor = @base.Constructors.
+            Where(c => !c.IsStatic).
+            Where(c =>
+                c.DeclaredAccessibility == Accessibility.Public ||
+                c.DeclaredAccessibility == Accessibility.Protected ||
+                c.DeclaredAccessibility == Accessibility.ProtectedOrInternal ||
+                containingTypes.Contains(c.ContainingType)
+            ).
+            MinBy(c => c.Parameters.Length);
+
+        return baseCtor?.Parameters.Length > 0 ? baseCtor : null;
+    }
+
+    private static IMethodSymbol? GetBaseConstructor(IMethodSymbol ctor)
+    {
+        if (ctor.MethodKind != MethodKind.Constructor)
+            return null;
+
+        return GetBaseConstructor(ctor.ContainingType);
+    }
+
+    private void StubParameters(ICollection<IParameterSymbol> parameters)
+    {
+        WriteCommaSep(parameters, parameter =>
+        {
+            switch (parameter.RefKind)
+            {
+                case RefKind.None:
+                    break;
+                case RefKind.Ref:
+                    stubWriter.Write("ref ");
+                    break;
+                case RefKind.Out:
+                    stubWriter.Write("out ");
+                    break;
+                case RefKind.In:
+                    stubWriter.Write("in ");
+                    break;
+                default:
+                    stubWriter.Write($"/* TODO: {parameter.RefKind} */");
+                    break;
+            }
+
+            if (parameter.IsParams)
+                stubWriter.Write("params ");
+
+            stubWriter.Write(parameter.Type.GetQualifiedName());
+            stubWriter.Write(" ");
+            stubWriter.Write(EscapeIdentifier(parameter.Name));
+
+            if (parameter.HasExplicitDefaultValue)
+            {
+                stubWriter.Write(" = ");
+                stubWriter.Write($"default({parameter.Type.GetQualifiedName()})");
+            }
+        });
+    }
+
+    private void StubMethod(IMethodSymbol symbol, IMethodSymbol? explicitInterfaceSymbol, IMethodSymbol? baseCtor)
+    {
+        var methodKind = explicitInterfaceSymbol is null ? symbol.MethodKind : explicitInterfaceSymbol.MethodKind;
+
+        var relevantMethods = new[] {
+                MethodKind.Constructor,
+                MethodKind.Conversion,
+                MethodKind.UserDefinedOperator,
+                MethodKind.Ordinary
+            };
+
+        if (!relevantMethods.Contains(methodKind))
+            return;
+
+        StubAttributes(symbol.GetAttributes());
+
+        StubModifiers(symbol, explicitInterfaceSymbol is not null);
+
+        if (IsUnsafe(symbol.ReturnType) || symbol.Parameters.Any(p => IsUnsafe(p.Type)))
+        {
+            stubWriter.Write("unsafe ");
+        }
+
+        if (methodKind == MethodKind.Constructor)
+        {
+            stubWriter.Write(symbol.ContainingType.Name);
+        }
+        else if (methodKind == MethodKind.Conversion)
+        {
+            if (!symbol.TryGetOperatorSymbol(out var operatorName))
+            {
+                stubWriter.WriteLine($"/* TODO: {symbol.Name} */");
+                return;
+            }
+
+            switch (operatorName)
+            {
+                case "explicit conversion":
+                    stubWriter.Write("explicit operator ");
+                    break;
+                case "checked explicit conversion":
+                    stubWriter.Write("explicit operator checked ");
+                    break;
+                case "implicit conversion":
+                    stubWriter.Write("implicit operator ");
+                    break;
+                case "checked implicit conversion":
+                    stubWriter.Write("implicit operator checked ");
+                    break;
+                default:
+                    stubWriter.Write($"/* TODO: {symbol.Name} */");
+                    break;
+            }
+
+            stubWriter.Write(symbol.ReturnType.GetQualifiedName());
+        }
+        else if (methodKind == MethodKind.UserDefinedOperator)
+        {
+            if (!symbol.TryGetOperatorSymbol(out var operatorName))
+            {
+                stubWriter.WriteLine($"/* TODO: {symbol.Name} */");
+                return;
+            }
+
+            stubWriter.Write(symbol.ReturnType.GetQualifiedName());
+            stubWriter.Write(" ");
+            StubExplicitInterface(symbol, explicitInterfaceSymbol, writeName: false);
+            stubWriter.Write("operator ");
+            stubWriter.Write(operatorName);
+        }
+        else
+        {
+            stubWriter.Write(symbol.ReturnType.GetQualifiedName());
+            stubWriter.Write(" ");
+            StubExplicitInterface(symbol, explicitInterfaceSymbol);
+            StubTypeParameters(symbol.TypeParameters);
+        }
+
+        stubWriter.Write("(");
+
+        if (symbol.IsExtensionMethod)
+        {
+            stubWriter.Write("this ");
+        }
+
+        StubParameters(symbol.Parameters);
+
+        stubWriter.Write(")");
+
+        if (baseCtor is not null)
+        {
+            stubWriter.Write(" : base(");
+            WriteStringCommaSep(baseCtor.Parameters, parameter => $"default({parameter.Type.GetQualifiedName()})");
+            stubWriter.Write(")");
+        }
+
+        StubTypeParameterConstraints(symbol.TypeParameters);
+
+        if (symbol.IsAbstract)
+            stubWriter.WriteLine(";");
+        else
+            stubWriter.WriteLine(" => throw null;");
+    }
+
+    public override void VisitMethod(IMethodSymbol symbol)
+    {
+        var baseCtor = GetBaseConstructor(symbol);
+        var explicitInterfaceImplementations = FilterExplicitInterfaceImplementations(symbol.ExplicitInterfaceImplementations);
+
+        if (baseCtor is null &&
+            ((IsNotPublic(symbol.DeclaredAccessibility) && explicitInterfaceImplementations.Length == 0) ||
+            symbol.IsImplicitlyDeclared))
+        {
+            return;
+        }
+
+        foreach (var explicitInterfaceSymbol in explicitInterfaceImplementations)
+        {
+            StubMethod(symbol, explicitInterfaceSymbol, baseCtor);
+        }
+
+        // Roslyn reports certain methods to be only explicit interface methods, such as
+        //  `System.Numerics.INumberBase<int>.TryParse(string s, System.Globalization.NumberStyles style, System.IFormatProvider provider, out int result)`
+        // in the `System.Int32` struct. However, we also need a non-explicit implementation
+        // in order for things to compile.
+        var roslynExplicitInterfaceWorkaround =
+            symbol.ContainingType.GetQualifiedName() is "int" &&
+            explicitInterfaceImplementations.Any(i => i.ContainingType.GetQualifiedName() is "System.Numerics.INumberBase<int>");
+
+        if (explicitInterfaceImplementations.Length == 0 || roslynExplicitInterfaceWorkaround)
+            StubMethod(symbol, null, baseCtor);
+    }
+
+    public override void VisitNamedType(INamedTypeSymbol symbol)
+    {
+        if (!relevantSymbol.IsRelevantNamedType(symbol))
+        {
+            return;
+        }
+
+        if (symbol.TypeKind == TypeKind.Delegate)
+        {
+            var invokeMethod = symbol.DelegateInvokeMethod!;
+            StubAttributes(symbol.GetAttributes());
+            StubModifiers(symbol);
+
+            if (IsUnsafe(invokeMethod.ReturnType) || invokeMethod.Parameters.Any(p => IsUnsafe(p.Type)))
+            {
+                stubWriter.Write("unsafe ");
+            }
+
+            stubWriter.Write("delegate ");
+            stubWriter.Write(invokeMethod.ReturnType.GetQualifiedName());
+            stubWriter.Write($" {symbol.Name}");
+            StubTypeParameters(symbol.TypeParameters);
+            stubWriter.Write("(");
+            StubParameters(invokeMethod.Parameters);
+            stubWriter.Write(")");
+            StubTypeParameterConstraints(symbol.TypeParameters);
+            stubWriter.WriteLine(";");
+            return;
+        }
+
+        switch (symbol.TypeKind)
+        {
+            case TypeKind.Class:
+                StubAttributes(symbol.GetAttributes());
+                StubModifiers(symbol);
+                // certain classes, such as `Microsoft.Extensions.Logging.LoggingBuilderExtensions`
+                // exist in multiple assemblies, so make them partial
+                if (symbol.IsStatic && symbol.Name.EndsWith("Extensions"))
+                    stubWriter.Write("partial ");
+                stubWriter.Write("class ");
+                break;
+            case TypeKind.Enum:
+                StubAttributes(symbol.GetAttributes());
+                StubModifiers(symbol);
+                stubWriter.Write("enum ");
+                break;
+            case TypeKind.Interface:
+                StubAttributes(symbol.GetAttributes());
+                StubModifiers(symbol);
+                stubWriter.Write("interface ");
+                break;
+            case TypeKind.Struct:
+                StubAttributes(symbol.GetAttributes());
+                StubModifiers(symbol);
+                stubWriter.Write("struct ");
+                break;
+            default:
+                return;
+        }
+
+        stubWriter.Write(symbol.Name);
+
+        StubTypeParameters(symbol.TypeParameters);
+
+        if (symbol.TypeKind == TypeKind.Enum)
+        {
+            if (symbol.EnumUnderlyingType is INamedTypeSymbol enumBase && enumBase.SpecialType != SpecialType.System_Int32)
+            {
+                stubWriter.Write(" : ");
+                stubWriter.Write(enumBase.GetQualifiedName());
+            }
+        }
+        else
+        {
+            var bases = symbol.Interfaces.Where(IsRelevantBaseType).OrderBy(i => i.GetName()).ToList();
+            if (GetBaseType(symbol) is INamedTypeSymbol @base && IsRelevantBaseType(@base))
+            {
+                bases.Insert(0, @base);
+            }
+
+            if (bases.Any())
+            {
+                stubWriter.Write(" : ");
+                WriteStringCommaSep(bases, b => b.GetQualifiedName());
+            }
+        }
+
+        StubTypeParameterConstraints(symbol.TypeParameters);
+
+        stubWriter.WriteLine(" {");
+
+        if (symbol.TypeKind == TypeKind.Enum)
+        {
+            foreach (var field in symbol.GetMembers().OfType<IFieldSymbol>().Where(field => field.ConstantValue is not null))
+            {
+                stubWriter.Write(field.Name);
+                stubWriter.Write(" = ");
+                stubWriter.Write(field.ConstantValue);
+                stubWriter.WriteLine(",");
+            }
+        }
+        else
+        {
+            var seenCtor = false;
+            foreach (var childSymbol in symbol.GetMembers().OrderBy(m => m.GetName()))
+            {
+                seenCtor |= childSymbol is IMethodSymbol method && method.MethodKind == MethodKind.Constructor;
+                childSymbol.Accept(this);
+            }
+
+            if (!seenCtor && GetBaseConstructor(symbol) is IMethodSymbol baseCtor)
+            {
+                stubWriter.Write($"internal {symbol.Name}() : base(");
+                WriteStringCommaSep(baseCtor.Parameters, parameter => $"default({parameter.Type.GetQualifiedName()})");
+                stubWriter.WriteLine(") {}");
+            }
+        }
+
+        stubWriter.WriteLine("}");
+    }
+
+    public override void VisitNamespace(INamespaceSymbol symbol)
+    {
+        if (!relevantSymbol.IsRelevantNamespace(symbol))
+        {
+            return;
+        }
+
+        var isGlobal = symbol.IsGlobalNamespace;
+
+        if (!isGlobal)
+            stubWriter.WriteLine($"namespace {symbol.Name} {{");
+
+        foreach (var childSymbol in symbol.GetMembers().OrderBy(m => m.GetName()))
+        {
+            childSymbol.Accept(this);
+        }
+
+        if (!isGlobal)
+            stubWriter.WriteLine("}");
+    }
+
+    private void StubProperty(IPropertySymbol symbol, IPropertySymbol? explicitInterfaceSymbol)
+    {
+        if (symbol.Parameters.Any())
+        {
+            var name = symbol.GetName(useMetadataName: true);
+            if (name is not "Item" && explicitInterfaceSymbol is null)
+                stubWriter.WriteLine($"[System.Runtime.CompilerServices.IndexerName(\"{name}\")]");
+        }
+
+        StubAttributes(symbol.GetAttributes());
+        StubModifiers(symbol, explicitInterfaceSymbol is not null);
+
+        if (IsUnsafe(symbol.Type) || symbol.Parameters.Any(p => IsUnsafe(p.Type)))
+        {
+            stubWriter.Write("unsafe ");
+        }
+
+        stubWriter.Write(symbol.Type.GetQualifiedName());
+        stubWriter.Write(" ");
+
+        if (symbol.Parameters.Any())
+        {
+            StubExplicitInterface(symbol, explicitInterfaceSymbol, writeName: false);
+            stubWriter.Write("this[");
+            StubParameters(symbol.Parameters);
+            stubWriter.Write("]");
+        }
+        else
+        {
+            StubExplicitInterface(symbol, explicitInterfaceSymbol);
+        }
+
+        stubWriter.Write(" { ");
+        if (symbol.GetMethod is not null)
+            stubWriter.Write(symbol.IsAbstract ? "get; " : "get => throw null; ");
+        if (symbol.SetMethod is not null)
+            stubWriter.Write(symbol.IsAbstract ? "set; " : "set {} ");
+        stubWriter.WriteLine("}");
+    }
+
+    public override void VisitProperty(IPropertySymbol symbol)
+    {
+        var explicitInterfaceImplementations = FilterExplicitInterfaceImplementations(symbol.ExplicitInterfaceImplementations);
+
+        if (IsNotPublic(symbol.DeclaredAccessibility) && explicitInterfaceImplementations.Length == 0)
+            return;
+
+        foreach (var explicitInterfaceImplementation in explicitInterfaceImplementations)
+        {
+            StubProperty(symbol, explicitInterfaceImplementation);
+        }
+
+        if (explicitInterfaceImplementations.Length == 0)
+            StubProperty(symbol, null);
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp.StubGenerator/SymbolExtensions.cs

@@ -0,0 +1,9 @@
+using Microsoft.CodeAnalysis;
+
+namespace Semmle.Extraction.CSharp.StubGenerator;
+
+public static class SymbolExtensions
+{
+    public static string GetQualifiedName(this ISymbol symbol) =>
+        symbol.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat.WithGlobalNamespaceStyle(SymbolDisplayGlobalNamespaceStyle.Omitted));
+}

csharp/extractor/Semmle.Extraction.CSharp.Util/Semmle.Extraction.CSharp.Util.csproj

@@ -0,0 +1,17 @@
+<Project Sdk="Microsoft.NET.Sdk">
+	<PropertyGroup>
+		<TargetFramework>net7.0</TargetFramework>
+		<AssemblyName>Semmle.Extraction.CSharp.Util</AssemblyName>
+		<RootNamespace>Semmle.Extraction.CSharp.Util</RootNamespace>
+		<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
+		<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+		<RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
+		<Nullable>enable</Nullable>
+	</PropertyGroup>
+	<ItemGroup>
+		<ProjectReference Include="..\Semmle.Util\Semmle.Util.csproj" />
+	</ItemGroup>
+	<ItemGroup>
+		<PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="4.4.0" />
+	</ItemGroup>
+</Project>

csharp/extractor/Semmle.Extraction.CSharp.Util/SymbolExtensions.cs

@@ -0,0 +1,131 @@
+using System.Text.RegularExpressions;
+using Microsoft.CodeAnalysis;
+
+namespace Semmle.Extraction.CSharp.Util
+{
+    public static partial class SymbolExtensions
+    {
+        /// <summary>
+        /// Gets the name of this symbol.
+        ///
+        /// If the symbol implements an explicit interface, only the
+        /// name of the member being implemented is included, not the
+        /// explicit prefix.
+        /// </summary>
+        public static string GetName(this ISymbol symbol, bool useMetadataName = false)
+        {
+            var name = useMetadataName ? symbol.MetadataName : symbol.Name;
+            return symbol.CanBeReferencedByName ? name : name.Substring(symbol.Name.LastIndexOf('.') + 1);
+        }
+
+        /// <summary>
+        /// Convert an operator method name in to a symbolic name.
+        /// A return value indicates whether the conversion succeeded.
+        /// </summary>
+        public static bool TryGetOperatorSymbol(this ISymbol symbol, out string operatorName)
+        {
+            static bool TryGetOperatorSymbolFromName(string methodName, out string operatorName)
+            {
+                var success = true;
+                switch (methodName)
+                {
+                    case "op_LogicalNot":
+                        operatorName = "!";
+                        break;
+                    case "op_BitwiseAnd":
+                        operatorName = "&";
+                        break;
+                    case "op_Equality":
+                        operatorName = "==";
+                        break;
+                    case "op_Inequality":
+                        operatorName = "!=";
+                        break;
+                    case "op_UnaryPlus":
+                    case "op_Addition":
+                        operatorName = "+";
+                        break;
+                    case "op_UnaryNegation":
+                    case "op_Subtraction":
+                        operatorName = "-";
+                        break;
+                    case "op_Multiply":
+                        operatorName = "*";
+                        break;
+                    case "op_Division":
+                        operatorName = "/";
+                        break;
+                    case "op_Modulus":
+                        operatorName = "%";
+                        break;
+                    case "op_GreaterThan":
+                        operatorName = ">";
+                        break;
+                    case "op_GreaterThanOrEqual":
+                        operatorName = ">=";
+                        break;
+                    case "op_LessThan":
+                        operatorName = "<";
+                        break;
+                    case "op_LessThanOrEqual":
+                        operatorName = "<=";
+                        break;
+                    case "op_Decrement":
+                        operatorName = "--";
+                        break;
+                    case "op_Increment":
+                        operatorName = "++";
+                        break;
+                    case "op_Implicit":
+                        operatorName = "implicit conversion";
+                        break;
+                    case "op_Explicit":
+                        operatorName = "explicit conversion";
+                        break;
+                    case "op_OnesComplement":
+                        operatorName = "~";
+                        break;
+                    case "op_RightShift":
+                        operatorName = ">>";
+                        break;
+                    case "op_UnsignedRightShift":
+                        operatorName = ">>>";
+                        break;
+                    case "op_LeftShift":
+                        operatorName = "<<";
+                        break;
+                    case "op_BitwiseOr":
+                        operatorName = "|";
+                        break;
+                    case "op_ExclusiveOr":
+                        operatorName = "^";
+                        break;
+                    case "op_True":
+                        operatorName = "true";
+                        break;
+                    case "op_False":
+                        operatorName = "false";
+                        break;
+                    default:
+                        var match = CheckedRegex().Match(methodName);
+                        if (match.Success)
+                        {
+                            TryGetOperatorSymbolFromName("op_" + match.Groups[1], out var uncheckedName);
+                            operatorName = "checked " + uncheckedName;
+                            break;
+                        }
+                        operatorName = methodName;
+                        success = false;
+                        break;
+                }
+                return success;
+            }
+
+            var methodName = symbol.GetName(useMetadataName: false);
+            return TryGetOperatorSymbolFromName(methodName, out operatorName);
+        }
+
+        [GeneratedRegex("^op_Checked(.*)$")]
+        private static partial Regex CheckedRegex();
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp/Entities/Assembly.cs

@@ -16,7 +16,7 @@ namespace Semmle.Extraction.CSharp.Entities
             if (init is null)
             {
                 // This is the output assembly
-                assemblyPath = ((TracingExtractor)cx.Extractor).OutputPath;
+                assemblyPath = cx.Extractor.OutputPath;
                 assembly = cx.Compilation.Assembly;
             }
             else
@@ -63,8 +63,6 @@ namespace Semmle.Extraction.CSharp.Entities
 
         public static Assembly CreateOutputAssembly(Context cx)
         {
-            if (cx.Extractor.Mode.HasFlag(ExtractorMode.Standalone))
-                throw new InternalError("Attempting to create the output assembly in standalone extraction mode");
             return AssemblyConstructorFactory.Instance.CreateEntity(cx, outputAssemblyCacheKey, null);
         }
 

csharp/extractor/Semmle.Extraction.CSharp/Entities/Attribute.cs

@@ -63,10 +63,7 @@ namespace Semmle.Extraction.CSharp.Entities
 
             if (attributeSyntax is not null)
             {
-                if (!Context.Extractor.Mode.HasFlag(ExtractorMode.Standalone))
-                {
-                    trapFile.attribute_location(this, Assembly.CreateOutputAssembly(Context));
-                }
+                trapFile.attribute_location(this, Assembly.CreateOutputAssembly(Context));
 
                 TypeMention.Create(Context, attributeSyntax.Name, this, type);
             }

csharp/extractor/Semmle.Extraction.CSharp/Entities/CachedSymbol.cs

@@ -97,7 +97,7 @@ namespace Semmle.Extraction.CSharp.Entities
                 {
                     // Some built in operators lack locations, so loc is null.
                     yield return Context.CreateLocation(ReportingLocation);
-                    if (!Context.Extractor.Mode.HasFlag(ExtractorMode.Standalone) && loc.Kind == LocationKind.SourceFile)
+                    if (loc.Kind == LocationKind.SourceFile)
                         yield return Assembly.CreateOutputAssembly(Context);
                 }
             }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Event.cs

@@ -2,6 +2,7 @@ using System.IO;
 using System.Linq;
 using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.CSharp.Syntax;
+using Semmle.Extraction.CSharp.Util;
 
 namespace Semmle.Extraction.CSharp.Entities
 {

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expression.cs

@@ -6,6 +6,7 @@ using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.CSharp;
 using Microsoft.CodeAnalysis.CSharp.Syntax;
 using Semmle.Extraction.CSharp.Entities.Expressions;
+using Semmle.Extraction.CSharp.Util;
 using Semmle.Extraction.Kinds;
 
 namespace Semmle.Extraction.CSharp.Entities
@@ -205,7 +206,7 @@ namespace Semmle.Extraction.CSharp.Entities
             {
                 // this can happen in VB.NET
                 cx.ExtractionError($"Extracting default argument value 'object {parameter.Name} = default' instead of 'object {parameter.Name} = {defaultValue}'. The latter is not supported in C#.",
-                    null, null, severity: Util.Logging.Severity.Warning);
+                    null, null, severity: Semmle.Util.Logging.Severity.Warning);
 
                 // we're generating a default expression:
                 return Default.CreateGenerated(cx, parent, childIndex, location, ValueAsString(null));
@@ -250,7 +251,7 @@ namespace Semmle.Extraction.CSharp.Entities
                 var callType = GetCallType(Context, node);
                 if (callType == CallType.Dynamic)
                 {
-                    UserOperator.TryGetOperatorSymbol(method.Name, out var operatorName);
+                    method.TryGetOperatorSymbol(out var operatorName);
                     trapFile.dynamic_member_name(this, operatorName);
                     return;
                 }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ImplicitCast.cs

@@ -1,5 +1,6 @@
 using System.Linq;
 using Microsoft.CodeAnalysis;
+using Semmle.Extraction.CSharp.Util;
 using Semmle.Extraction.Kinds;
 
 namespace Semmle.Extraction.CSharp.Entities.Expressions

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ObjectCreation/DateTimeObjectCreation.cs

@@ -1,6 +1,7 @@
 using System.IO;
 using System.Linq;
 using Microsoft.CodeAnalysis;
+using Semmle.Extraction.CSharp.Util;
 using Semmle.Extraction.Kinds;
 
 namespace Semmle.Extraction.CSharp.Entities.Expressions

csharp/extractor/Semmle.Extraction.CSharp/Entities/Indexer.cs

@@ -2,6 +2,7 @@ using System.IO;
 using System.Linq;
 using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.CSharp.Syntax;
+using Semmle.Extraction.CSharp.Util;
 
 namespace Semmle.Extraction.CSharp.Entities
 {

csharp/extractor/Semmle.Extraction.CSharp/Entities/OrdinaryMethod.cs

@@ -3,7 +3,7 @@ using System.Linq;
 using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.CSharp.Syntax;
 using Semmle.Extraction.CSharp.Populators;
-
+using Semmle.Extraction.CSharp.Util;
 
 namespace Semmle.Extraction.CSharp.Entities
 {
@@ -51,7 +51,7 @@ namespace Semmle.Extraction.CSharp.Entities
         {
             if (method.MethodKind == MethodKind.ReducedExtension)
             {
-                cx.Extractor.Logger.Log(Util.Logging.Severity.Warning, "Reduced extension method symbols should not be directly extracted.");
+                cx.Extractor.Logger.Log(Semmle.Util.Logging.Severity.Warning, "Reduced extension method symbols should not be directly extracted.");
             }
 
             return OrdinaryMethodFactory.Instance.CreateEntityFromSymbol(cx, method);

csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/PreprocessorDirective.cs

@@ -15,11 +15,8 @@ namespace Semmle.Extraction.CSharp.Entities
             trapFile.preprocessor_directive_active(this, Symbol.IsActive);
             trapFile.preprocessor_directive_location(this, Context.CreateLocation(ReportingLocation));
 
-            if (!Context.Extractor.Mode.HasFlag(ExtractorMode.Standalone))
-            {
-                var compilation = Compilation.Create(Context);
-                trapFile.preprocessor_directive_compilation(this, compilation);
-            }
+            var compilation = Compilation.Create(Context);
+            trapFile.preprocessor_directive_compilation(this, compilation);
         }
 
         protected abstract void PopulatePreprocessor(TextWriter trapFile);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Property.cs

@@ -5,6 +5,7 @@ using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.CSharp.Syntax;
 using Semmle.Extraction.CSharp.Entities.Expressions;
 using Semmle.Extraction.Kinds;
+using Semmle.Extraction.CSharp.Util;
 
 namespace Semmle.Extraction.CSharp.Entities
 {

csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/ForEach.cs

@@ -44,7 +44,7 @@ namespace Semmle.Extraction.CSharp.Entities.Statements
 
             if (info.Equals(default))
             {
-                Context.ExtractionError("Could not get foreach statement info", null, Context.CreateLocation(this.ReportingLocation), severity: Util.Logging.Severity.Info);
+                Context.ExtractionError("Could not get foreach statement info", null, Context.CreateLocation(this.ReportingLocation), severity: Semmle.Util.Logging.Severity.Info);
                 return;
             }
 

csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/LocalFunction.cs

@@ -34,7 +34,7 @@ namespace Semmle.Extraction.CSharp.Entities.Statements
         {
             if (Symbol is null)
             {
-                Context.ExtractionError("Could not get local function symbol", null, Context.CreateLocation(this.ReportingLocation), severity: Util.Logging.Severity.Warning);
+                Context.ExtractionError("Could not get local function symbol", null, Context.CreateLocation(this.ReportingLocation), severity: Semmle.Util.Logging.Severity.Warning);
                 return;
             }
 

csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NamedType.cs

@@ -108,7 +108,7 @@ namespace Semmle.Extraction.CSharp.Entities
                 foreach (var l in GetLocations(Symbol))
                     yield return Context.CreateLocation(l);
 
-                if (!Context.Extractor.Mode.HasFlag(ExtractorMode.Standalone) && Symbol.DeclaringSyntaxReferences.Any())
+                if (Symbol.DeclaringSyntaxReferences.Any())
                     yield return Assembly.CreateOutputAssembly(Context);
             }
         }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/Type.cs

@@ -87,7 +87,7 @@ namespace Semmle.Extraction.CSharp.Entities
             var hasExpandingCycle = GenericsRecursionGraph.HasExpandingCycle(Symbol);
             if (hasExpandingCycle)
             {
-                Context.ExtractionError("Found recursive generic inheritance hierarchy. Base class of type is not extracted", Symbol.ToDisplayString(), Context.CreateLocation(ReportingLocation), severity: Util.Logging.Severity.Warning);
+                Context.ExtractionError("Found recursive generic inheritance hierarchy. Base class of type is not extracted", Symbol.ToDisplayString(), Context.CreateLocation(ReportingLocation), severity: Semmle.Util.Logging.Severity.Warning);
             }
 
             // Visit base types

csharp/extractor/Semmle.Extraction.CSharp/Entities/UserOperator.cs

@@ -1,8 +1,8 @@
 using System.IO;
 using System.Linq;
-using System.Text.RegularExpressions;
 using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.CSharp.Syntax;
+using Semmle.Extraction.CSharp.Util;
 
 namespace Semmle.Extraction.CSharp.Entities
 {
@@ -79,108 +79,7 @@ namespace Semmle.Extraction.CSharp.Entities
             return true;
         }
 
-        /// <summary>
-        /// Convert an operator method name in to a symbolic name.
-        /// A return value indicates whether the conversion succeeded.
-        /// </summary>
-        /// <param name="methodName">The method name.</param>
-        /// <param name="operatorName">The converted operator name.</param>
-        public static bool TryGetOperatorSymbol(string methodName, out string operatorName)
-        {
-            var success = true;
-            switch (methodName)
-            {
-                case "op_LogicalNot":
-                    operatorName = "!";
-                    break;
-                case "op_BitwiseAnd":
-                    operatorName = "&";
-                    break;
-                case "op_Equality":
-                    operatorName = "==";
-                    break;
-                case "op_Inequality":
-                    operatorName = "!=";
-                    break;
-                case "op_UnaryPlus":
-                case "op_Addition":
-                    operatorName = "+";
-                    break;
-                case "op_UnaryNegation":
-                case "op_Subtraction":
-                    operatorName = "-";
-                    break;
-                case "op_Multiply":
-                    operatorName = "*";
-                    break;
-                case "op_Division":
-                    operatorName = "/";
-                    break;
-                case "op_Modulus":
-                    operatorName = "%";
-                    break;
-                case "op_GreaterThan":
-                    operatorName = ">";
-                    break;
-                case "op_GreaterThanOrEqual":
-                    operatorName = ">=";
-                    break;
-                case "op_LessThan":
-                    operatorName = "<";
-                    break;
-                case "op_LessThanOrEqual":
-                    operatorName = "<=";
-                    break;
-                case "op_Decrement":
-                    operatorName = "--";
-                    break;
-                case "op_Increment":
-                    operatorName = "++";
-                    break;
-                case "op_Implicit":
-                    operatorName = "implicit conversion";
-                    break;
-                case "op_Explicit":
-                    operatorName = "explicit conversion";
-                    break;
-                case "op_OnesComplement":
-                    operatorName = "~";
-                    break;
-                case "op_RightShift":
-                    operatorName = ">>";
-                    break;
-                case "op_UnsignedRightShift":
-                    operatorName = ">>>";
-                    break;
-                case "op_LeftShift":
-                    operatorName = "<<";
-                    break;
-                case "op_BitwiseOr":
-                    operatorName = "|";
-                    break;
-                case "op_ExclusiveOr":
-                    operatorName = "^";
-                    break;
-                case "op_True":
-                    operatorName = "true";
-                    break;
-                case "op_False":
-                    operatorName = "false";
-                    break;
-                default:
-                    var match = Regex.Match(methodName, "^op_Checked(.*)$");
-                    if (match.Success)
-                    {
-                        TryGetOperatorSymbol("op_" + match.Groups[1], out var uncheckedName);
-                        operatorName = "checked " + uncheckedName;
-                        break;
-                    }
-                    operatorName = methodName;
-                    success = false;
-                    break;
-            }
-            return success;
-        }
+
 
         /// <summary>
         /// Converts a method name into a symbolic name.
@@ -191,12 +90,8 @@ namespace Semmle.Extraction.CSharp.Entities
         /// <returns>The converted name.</returns>
         private static string OperatorSymbol(Context cx, IMethodSymbol method)
         {
-            if (method.ExplicitInterfaceImplementations.Any())
-                return OperatorSymbol(cx, method.ExplicitInterfaceImplementations.First());
-
-            var methodName = method.Name;
-            if (!TryGetOperatorSymbol(methodName, out var result))
-                cx.ModelError(method, $"Unhandled operator name in OperatorSymbol(): '{methodName}'");
+            if (!method.TryGetOperatorSymbol(out var result))
+                cx.ModelError(method, $"Unhandled operator name in OperatorSymbol(): '{method.Name}'");
             return result;
         }
 

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs

@@ -19,6 +19,8 @@ namespace Semmle.Extraction.CSharp
         protected Extraction.Extractor? extractor;
         protected CSharpCompilation? compilation;
         protected CommonOptions? options;
+        private protected Entities.Compilation? compilationEntity;
+        private IDisposable? compilationTrapFile;
 
         private readonly object progressMutex = new object();
 
@@ -226,8 +228,35 @@ namespace Semmle.Extraction.CSharp
             }
         }
 
+        private void DoAnalyseCompilation()
+        {
+            try
+            {
+                var assemblyPath = extractor.OutputPath;
+                var transformedAssemblyPath = PathTransformer.Transform(assemblyPath);
+                var assembly = compilation.Assembly;
+                var trapWriter = transformedAssemblyPath.CreateTrapWriter(Logger, options.TrapCompression, discardDuplicates: false);
+                compilationTrapFile = trapWriter;  // Dispose later
+                var cx = new Context(extractor, compilation.Clone(), trapWriter, new AssemblyScope(assembly, assemblyPath), addAssemblyTrapPrefix);
+
+                compilationEntity = Entities.Compilation.Create(cx);
+            }
+            catch (Exception ex)  // lgtm[cs/catch-of-all-exceptions]
+            {
+                Logger.Log(Severity.Error, "  Unhandled exception analyzing {0}: {1}", "compilation", ex);
+            }
+        }
+
 #nullable restore warnings
 
+        /// <summary>
+        /// Extracts compilation-wide entities, such as compilations and compiler diagnostics.
+        /// </summary>
+        public void AnalyseCompilation()
+        {
+            extractionTasks.Add(() => DoAnalyseCompilation());
+        }
+
         private static bool FileIsUpToDate(string src, string dest)
         {
             return File.Exists(dest) &&
@@ -275,6 +304,8 @@ namespace Semmle.Extraction.CSharp
                 Logger.Log(Severity.Info, "EXTRACTION SUCCEEDED in {0}", stopWatch.Elapsed);
 
             Logger.Dispose();
+
+            compilationTrapFile?.Dispose();
         }
 
         /// <summary>

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs

@@ -302,7 +302,6 @@ namespace Semmle.Extraction.CSharp
             Func<Analyser, List<SyntaxTree>, IEnumerable<Action>> getSyntaxTreeTasks,
             Func<IEnumerable<SyntaxTree>, IEnumerable<MetadataReference>, CSharpCompilation> getCompilation,
             Action<CSharpCompilation, CommonOptions> initializeAnalyser,
-            Action analyseCompilation,
             Action<Entities.PerformanceMetrics> logPerformance,
             Action postProcess)
         {
@@ -332,7 +331,7 @@ namespace Semmle.Extraction.CSharp
             var compilation = getCompilation(syntaxTrees, references);
 
             initializeAnalyser(compilation, options);
-            analyseCompilation();
+            analyser.AnalyseCompilation();
             analyser.AnalyseReferences();
 
             foreach (var tree in compilation.SyntaxTrees)
@@ -416,7 +415,6 @@ namespace Semmle.Extraction.CSharp
                         );
                 },
                 (compilation, options) => analyser.EndInitialize(compilerArguments, options, compilation),
-                () => analyser.AnalyseCompilation(),
                 performance => analyser.LogPerformance(performance),
                 () => { });
         }

csharp/extractor/Semmle.Extraction.CSharp/Extractor/StandaloneAnalyser.cs

@@ -1,4 +1,6 @@
+using System;
 using System.Collections.Generic;
+using System.IO;
 using Microsoft.CodeAnalysis.CSharp;
 using Semmle.Util.Logging;
 
@@ -11,13 +13,15 @@ namespace Semmle.Extraction.CSharp
         {
         }
 
-        public void Initialize(CSharpCompilation compilationIn, CommonOptions options)
+        public void Initialize(string outputPath, CSharpCompilation compilationIn, CommonOptions options)
         {
             compilation = compilationIn;
-            extractor = new StandaloneExtractor(Logger, PathTransformer, options);
+            extractor = new StandaloneExtractor(outputPath, Logger, PathTransformer, options);
             this.options = options;
             LogExtractorInfo(Extraction.Extractor.Version);
             SetReferencePaths();
+
+            Entities.Compilation.Settings = (Directory.GetCurrentDirectory(), Array.Empty<string>());
         }
 
 #nullable disable warnings

csharp/extractor/Semmle.Extraction.CSharp/Extractor/TracingAnalyser.cs

@@ -9,11 +9,8 @@ using Semmle.Util.Logging;
 
 namespace Semmle.Extraction.CSharp
 {
-    public class TracingAnalyser : Analyser, IDisposable
+    public class TracingAnalyser : Analyser
     {
-        private Entities.Compilation? compilationEntity;
-        private IDisposable? compilationTrapFile;
-
         private bool init;
 
         public TracingAnalyser(IProgressMonitor pm, ILogger logger, bool addAssemblyTrapPrefix, PathTransformer pathTransformer)
@@ -55,20 +52,6 @@ namespace Semmle.Extraction.CSharp
             CompilationErrors += FilteredDiagnostics.Count();
         }
 
-        public override void Dispose()
-        {
-            compilationTrapFile?.Dispose();
-            base.Dispose();
-        }
-
-        /// <summary>
-        /// Extracts compilation-wide entities, such as compilations and compiler diagnostics.
-        /// </summary>
-        public void AnalyseCompilation()
-        {
-            extractionTasks.Add(() => DoAnalyseCompilation());
-        }
-
         /// <summary>
         /// Logs information about the extractor, as well as the arguments to Roslyn.
         /// </summary>
@@ -193,25 +176,6 @@ namespace Semmle.Extraction.CSharp
             }
         }
 
-        private void DoAnalyseCompilation()
-        {
-            try
-            {
-                var assemblyPath = ((TracingExtractor?)extractor).OutputPath;
-                var transformedAssemblyPath = PathTransformer.Transform(assemblyPath);
-                var assembly = compilation.Assembly;
-                var trapWriter = transformedAssemblyPath.CreateTrapWriter(Logger, options.TrapCompression, discardDuplicates: false);
-                compilationTrapFile = trapWriter;  // Dispose later
-                var cx = new Context(extractor, compilation.Clone(), trapWriter, new AssemblyScope(assembly, assemblyPath), addAssemblyTrapPrefix);
-
-                compilationEntity = Entities.Compilation.Create(cx);
-            }
-            catch (Exception ex)  // lgtm[cs/catch-of-all-exceptions]
-            {
-                Logger.Log(Severity.Error, "  Unhandled exception analyzing {0}: {1}", "compilation", ex);
-            }
-        }
-
         public void LogPerformance(Entities.PerformanceMetrics p) => compilationEntity.PopulatePerformance(p);
 
 #nullable restore warnings

csharp/extractor/Semmle.Extraction.CSharp/Populators/DirectiveVisitor.cs

@@ -59,7 +59,7 @@ namespace Semmle.Extraction.CSharp.Populators
             if (regionStarts.Count == 0)
             {
                 cx.ExtractionError("Couldn't find start region", null,
-                    cx.CreateLocation(node.GetLocation()), null, Util.Logging.Severity.Warning);
+                    cx.CreateLocation(node.GetLocation()), null, Semmle.Util.Logging.Severity.Warning);
                 return;
             }
 
@@ -94,7 +94,7 @@ namespace Semmle.Extraction.CSharp.Populators
             if (ifStarts.Count == 0)
             {
                 cx.ExtractionError("Couldn't find start if", null,
-                    cx.CreateLocation(node.GetLocation()), null, Util.Logging.Severity.Warning);
+                    cx.CreateLocation(node.GetLocation()), null, Semmle.Util.Logging.Severity.Warning);
                 return;
             }
 
@@ -107,7 +107,7 @@ namespace Semmle.Extraction.CSharp.Populators
             if (ifStarts.Count == 0)
             {
                 cx.ExtractionError("Couldn't find start if", null,
-                    cx.CreateLocation(node.GetLocation()), null, Util.Logging.Severity.Warning);
+                    cx.CreateLocation(node.GetLocation()), null, Semmle.Util.Logging.Severity.Warning);
                 return;
             }
 
@@ -122,7 +122,7 @@ namespace Semmle.Extraction.CSharp.Populators
             if (ifStarts.Count == 0)
             {
                 cx.ExtractionError("Couldn't find start if", null,
-                    cx.CreateLocation(node.GetLocation()), null, Util.Logging.Severity.Warning);
+                    cx.CreateLocation(node.GetLocation()), null, Semmle.Util.Logging.Severity.Warning);
                 return;
             }
 

csharp/extractor/Semmle.Extraction.CSharp/Populators/TypeContainerVisitor.cs

@@ -82,9 +82,6 @@ namespace Semmle.Extraction.CSharp.Populators
 
         public override void VisitAttributeList(AttributeListSyntax node)
         {
-            if (Cx.Extractor.Mode.HasFlag(ExtractorMode.Standalone))
-                return;
-
             var outputAssembly = Assembly.CreateOutputAssembly(Cx);
             var kind = node.Target?.Identifier.Kind() switch
             {

csharp/extractor/Semmle.Extraction.CSharp/Semmle.Extraction.CSharp.csproj

@@ -6,12 +6,12 @@
 		<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
 		<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
 		<RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
-		<RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
 		<Nullable>enable</Nullable>
 	</PropertyGroup>
 	<ItemGroup>
 		<ProjectReference Include="..\Semmle.Extraction.CIL\Semmle.Extraction.CIL.csproj" />
 		<ProjectReference Include="..\Semmle.Extraction\Semmle.Extraction.csproj" />
+		<ProjectReference Include="..\Semmle.Extraction.CSharp.Util\Semmle.Extraction.CSharp.Util.csproj" />
 		<ProjectReference Include="..\Semmle.Util\Semmle.Util.csproj" />
 	</ItemGroup>
 	<ItemGroup>

csharp/extractor/Semmle.Extraction.CSharp/SymbolExtensions.cs

@@ -55,19 +55,6 @@ namespace Semmle.Extraction.CSharp
                 : type;
         }
 
-        /// <summary>
-        /// Gets the name of this symbol.
-        ///
-        /// If the symbol implements an explicit interface, only the
-        /// name of the member being implemented is included, not the
-        /// explicit prefix.
-        /// </summary>
-        public static string GetName(this ISymbol symbol, bool useMetadataName = false)
-        {
-            var name = useMetadataName ? symbol.MetadataName : symbol.Name;
-            return symbol.CanBeReferencedByName ? name : name.Substring(symbol.Name.LastIndexOf('.') + 1);
-        }
-
         private static IEnumerable<SyntaxToken> GetModifiers<T>(this ISymbol symbol, Func<T, IEnumerable<SyntaxToken>> getModifierTokens) =>
             symbol.DeclaringSyntaxReferences
                 .Select(r => r.GetSyntax())

csharp/extractor/Semmle.Extraction.Tests/Semmle.Extraction.Tests.csproj

@@ -16,6 +16,7 @@
 		<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.0" />
 	</ItemGroup>
 	<ItemGroup>
+		<ProjectReference Include="..\Semmle.Extraction.CSharp.StubGenerator\Semmle.Extraction.CSharp.StubGenerator.csproj" />
 		<ProjectReference Include="..\Semmle.Extraction.CSharp.Standalone\Semmle.Extraction.CSharp.Standalone.csproj" />
 		<ProjectReference Include="..\Semmle.Extraction.CSharp\Semmle.Extraction.CSharp.csproj" />
 		<ProjectReference Include="..\Semmle.Extraction\Semmle.Extraction.csproj" />

csharp/extractor/Semmle.Extraction.Tests/StubGenerator.cs

@@ -0,0 +1,74 @@
+using Xunit;
+using System.IO;
+using System.Text;
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.CSharp;
+using Semmle.Extraction.CSharp.StubGenerator;
+
+namespace Semmle.Extraction.Tests;
+/// <summary>
+/// Tests for the stub generator.
+/// 
+/// These tests can be used to more easily step debug the stub generator SymbolVisitor.
+/// </summary>
+public class StubGeneratorTests
+{
+    [Fact]
+    public void StubGeneratorFieldTest()
+    {
+        // Setup
+        const string source = @"
+public class MyTest {
+    public static readonly int MyField1;
+    public const string MyField2 = ""hello"";
+    }
+";
+
+        // Execute
+        var stub = GenerateStub(source);
+
+        // Verify
+        const string expected = @"public class MyTest {
+public static readonly int MyField1;
+public const string MyField2 = default;
+}
+";
+        Assert.Equal(expected, stub);
+    }
+
+    [Fact]
+    public void StubGeneratorMethodTest()
+    {
+        // Setup
+        const string source = @"
+public class MyTest {
+    public int M1(string arg1) { return 0;}
+}";
+
+        // Execute
+        var stub = GenerateStub(source);
+
+        // Verify
+        const string expected = @"public class MyTest {
+public int M1(string arg1) => throw null;
+}
+";
+        Assert.Equal(expected, stub);
+    }
+
+    private static string GenerateStub(string source)
+    {
+        var st = CSharpSyntaxTree.ParseText(source);
+        var compilation = CSharpCompilation.Create(null, new[] { st });
+        var sb = new StringBuilder();
+        var visitor = new StubVisitor(new StringWriter(sb) { NewLine = "\n" }, new RelevantSymbolStub());
+        compilation.GlobalNamespace.Accept(visitor);
+        return sb.ToString();
+    }
+
+    private class RelevantSymbolStub : IRelevantSymbol
+    {
+        public bool IsRelevantNamedType(INamedTypeSymbol symbol) => true;
+        public bool IsRelevantNamespace(INamespaceSymbol symbol) => true;
+    }
+}

csharp/extractor/Semmle.Extraction.Tests/TrapWriter.cs

@@ -1,52 +0,0 @@
-using Xunit;
-using Semmle.Util.Logging;
-using Semmle.Util;
-
-namespace Semmle.Extraction.Tests
-{
-    public class TrapWriterTests
-    {
-        [Fact]
-        public void NestedPaths()
-        {
-            string tempDir = System.IO.Path.GetTempPath();
-            string root1, root2, root3;
-
-            if (Win32.IsWindows())
-            {
-                root1 = "E:";
-                root2 = "e:";
-                root3 = @"\";
-            }
-            else
-            {
-                root1 = "/E_";
-                root2 = "/e_";
-                root3 = "/";
-            }
-
-            using var logger = new LoggerMock();
-
-            Assert.Equal($@"C:\Temp\source_archive\def.cs", TrapWriter.NestPaths(logger, @"C:\Temp\source_archive", "def.cs").Replace('/', '\\'));
-
-            Assert.Equal(@"C:\Temp\source_archive\def.cs", TrapWriter.NestPaths(logger, @"C:\Temp\source_archive", "def.cs").Replace('/', '\\'));
-
-            Assert.Equal(@"C:\Temp\source_archive\E_\source\def.cs", TrapWriter.NestPaths(logger, @"C:\Temp\source_archive", $@"{root1}\source\def.cs").Replace('/', '\\'));
-
-            Assert.Equal(@"C:\Temp\source_archive\e_\source\def.cs", TrapWriter.NestPaths(logger, @"C:\Temp\source_archive", $@"{root2}\source\def.cs").Replace('/', '\\'));
-
-            Assert.Equal(@"C:\Temp\source_archive\source\def.cs", TrapWriter.NestPaths(logger, @"C:\Temp\source_archive", $@"{root3}source\def.cs").Replace('/', '\\'));
-
-            Assert.Equal(@"C:\Temp\source_archive\source\def.cs", TrapWriter.NestPaths(logger, @"C:\Temp\source_archive", $@"{root3}source\def.cs").Replace('/', '\\'));
-
-            Assert.Equal(@"C:\Temp\source_archive\diskstation\share\source\def.cs", TrapWriter.NestPaths(logger, @"C:\Temp\source_archive", $@"{root3}{root3}diskstation\share\source\def.cs").Replace('/', '\\'));
-        }
-
-        private sealed class LoggerMock : ILogger
-        {
-            public void Dispose() { }
-
-            public void Log(Severity s, string text) { }
-        }
-    }
-}

csharp/extractor/Semmle.Extraction/Extractor/Extractor.cs

@@ -9,14 +9,16 @@ namespace Semmle.Extraction
     public abstract class Extractor
     {
         public abstract ExtractorMode Mode { get; }
+        public string OutputPath { get; }
 
         /// <summary>
         /// Creates a new extractor instance for one compilation unit.
         /// </summary>
         /// <param name="logger">The object used for logging.</param>
         /// <param name="pathTransformer">The object used for path transformations.</param>
-        protected Extractor(ILogger logger, PathTransformer pathTransformer)
+        protected Extractor(string outputPath, ILogger logger, PathTransformer pathTransformer)
         {
+            OutputPath = outputPath;
             Logger = logger;
             PathTransformer = pathTransformer;
         }

csharp/extractor/Semmle.Extraction/Extractor/StandaloneExtractor.cs

@@ -11,7 +11,7 @@ namespace Semmle.Extraction
         /// </summary>
         /// <param name="logger">The object used for logging.</param>
         /// <param name="pathTransformer">The object used for path transformations.</param>
-        public StandaloneExtractor(ILogger logger, PathTransformer pathTransformer, CommonOptions options) : base(logger, pathTransformer)
+        public StandaloneExtractor(string outputPath, ILogger logger, PathTransformer pathTransformer, CommonOptions options) : base(outputPath, logger, pathTransformer)
         {
             Mode = ExtractorMode.Standalone;
             if (options.QlTest)

csharp/extractor/Semmle.Extraction/Extractor/TracingExtractor.cs

@@ -5,7 +5,6 @@ namespace Semmle.Extraction
     public class TracingExtractor : Extractor
     {
         public override ExtractorMode Mode { get; }
-        public string OutputPath { get; }
 
         /// <summary>
         /// Creates a new extractor instance for one compilation unit.
@@ -13,9 +12,8 @@ namespace Semmle.Extraction
         /// <param name="outputPath">The name of the output DLL/EXE, or null if not specified (standalone extraction).</param>
         /// <param name="logger">The object used for logging.</param>
         /// <param name="pathTransformer">The object used for path transformations.</param>
-        public TracingExtractor(string outputPath, ILogger logger, PathTransformer pathTransformer, CommonOptions options) : base(logger, pathTransformer)
+        public TracingExtractor(string outputPath, ILogger logger, PathTransformer pathTransformer, CommonOptions options) : base(outputPath, logger, pathTransformer)
         {
-            OutputPath = outputPath;
             Mode = ExtractorMode.None;
             if (options.QlTest)
             {

csharp/extractor/Semmle.Extraction/TrapWriter.cs

@@ -216,7 +216,7 @@ namespace Semmle.Extraction
 
         private void ArchiveContents(PathTransformer.ITransformedPath transformedPath, string contents)
         {
-            var dest = NestPaths(logger, archive, transformedPath.Value);
+            var dest = FileUtils.NestPaths(logger, archive, transformedPath.Value);
             var tmpSrcFile = Path.GetTempFileName();
             File.WriteAllText(tmpSrcFile, contents, utf8);
             try
@@ -231,38 +231,6 @@ namespace Semmle.Extraction
             }
         }
 
-        public static string NestPaths(ILogger logger, string? outerpath, string innerpath)
-        {
-            var nested = innerpath;
-            if (!string.IsNullOrEmpty(outerpath))
-            {
-                // Remove all leading path separators / or \
-                // For example, UNC paths have two leading \\
-                innerpath = innerpath.TrimStart(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
-
-                if (innerpath.Length > 1 && innerpath[1] == ':')
-                    innerpath = innerpath[0] + "_" + innerpath.Substring(2);
-
-                nested = Path.Combine(outerpath, innerpath);
-            }
-            try
-            {
-                var directoryName = Path.GetDirectoryName(nested);
-                if (directoryName is null)
-                {
-                    logger.Log(Severity.Warning, "Failed to get directory name from path '" + nested + "'.");
-                    throw new InvalidOperationException();
-                }
-                Directory.CreateDirectory(directoryName);
-            }
-            catch (PathTooLongException)
-            {
-                logger.Log(Severity.Warning, "Failed to create parent directory of '" + nested + "': Path too long.");
-                throw;
-            }
-            return nested;
-        }
-
         private static string TrapExtension(CompressionMode compression)
         {
             switch (compression)
@@ -280,7 +248,7 @@ namespace Semmle.Extraction
             if (string.IsNullOrEmpty(folder))
                 folder = Directory.GetCurrentDirectory();
 
-            return NestPaths(logger, folder, filename);
+            return FileUtils.NestPaths(logger, folder, filename);
         }
     }
 }

csharp/extractor/Semmle.Util.Tests/FileUtils.cs

@@ -1,5 +1,6 @@
 using Xunit;
 using Semmle.Util;
+using Semmle.Util.Logging;
 
 namespace SemmleTests.Semmle.Util
 {
@@ -16,5 +17,47 @@ namespace SemmleTests.Semmle.Util
 
             Assert.Equal(Win32.IsWindows() ? @"foo\bar" : "foo/bar", FileUtils.ConvertToNative("foo/bar"));
         }
+
+        [Fact]
+        public void NestedPaths()
+        {
+            string root1, root2, root3;
+
+            if (Win32.IsWindows())
+            {
+                root1 = "E:";
+                root2 = "e:";
+                root3 = @"\";
+            }
+            else
+            {
+                root1 = "/E_";
+                root2 = "/e_";
+                root3 = "/";
+            }
+
+            using var logger = new LoggerMock();
+
+            Assert.Equal($@"C:\Temp\source_archive\def.cs", FileUtils.NestPaths(logger, @"C:\Temp\source_archive", "def.cs").Replace('/', '\\'));
+
+            Assert.Equal(@"C:\Temp\source_archive\def.cs", FileUtils.NestPaths(logger, @"C:\Temp\source_archive", "def.cs").Replace('/', '\\'));
+
+            Assert.Equal(@"C:\Temp\source_archive\E_\source\def.cs", FileUtils.NestPaths(logger, @"C:\Temp\source_archive", $@"{root1}\source\def.cs").Replace('/', '\\'));
+
+            Assert.Equal(@"C:\Temp\source_archive\e_\source\def.cs", FileUtils.NestPaths(logger, @"C:\Temp\source_archive", $@"{root2}\source\def.cs").Replace('/', '\\'));
+
+            Assert.Equal(@"C:\Temp\source_archive\source\def.cs", FileUtils.NestPaths(logger, @"C:\Temp\source_archive", $@"{root3}source\def.cs").Replace('/', '\\'));
+
+            Assert.Equal(@"C:\Temp\source_archive\source\def.cs", FileUtils.NestPaths(logger, @"C:\Temp\source_archive", $@"{root3}source\def.cs").Replace('/', '\\'));
+
+            Assert.Equal(@"C:\Temp\source_archive\diskstation\share\source\def.cs", FileUtils.NestPaths(logger, @"C:\Temp\source_archive", $@"{root3}{root3}diskstation\share\source\def.cs").Replace('/', '\\'));
+        }
+
+        private sealed class LoggerMock : ILogger
+        {
+            public void Dispose() { }
+
+            public void Log(Severity s, string text) { }
+        }
     }
 }

csharp/extractor/Semmle.Util/FileUtils.cs

@@ -5,6 +5,7 @@ using System.Net.Http;
 using System.Security.Cryptography;
 using System.Text;
 using System.Threading.Tasks;
+using Semmle.Util.Logging;
 
 namespace Semmle.Util
 {
@@ -110,5 +111,69 @@ namespace Semmle.Util
         /// </summary>
         public static void DownloadFile(string address, string fileName) =>
            DownloadFileAsync(address, fileName).Wait();
+
+        public static string NestPaths(ILogger logger, string? outerpath, string innerpath)
+        {
+            var nested = innerpath;
+            if (!string.IsNullOrEmpty(outerpath))
+            {
+                // Remove all leading path separators / or \
+                // For example, UNC paths have two leading \\
+                innerpath = innerpath.TrimStart(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
+
+                if (innerpath.Length > 1 && innerpath[1] == ':')
+                    innerpath = innerpath[0] + "_" + innerpath.Substring(2);
+
+                nested = Path.Combine(outerpath, innerpath);
+            }
+            try
+            {
+                var directoryName = Path.GetDirectoryName(nested);
+                if (directoryName is null)
+                {
+                    logger.Log(Severity.Warning, "Failed to get directory name from path '" + nested + "'.");
+                    throw new InvalidOperationException();
+                }
+                Directory.CreateDirectory(directoryName);
+            }
+            catch (PathTooLongException)
+            {
+                logger.Log(Severity.Warning, "Failed to create parent directory of '" + nested + "': Path too long.");
+                throw;
+            }
+            return nested;
+        }
+
+        public static string GetTemporaryWorkingDirectory(out bool shouldCleanUp)
+        {
+            shouldCleanUp = false;
+            var tempFolder = EnvironmentVariables.GetScratchDirectory();
+
+            if (string.IsNullOrEmpty(tempFolder))
+            {
+                var tempPath = Path.GetTempPath();
+                var name = Guid.NewGuid().ToString("N").ToUpper();
+                tempFolder = Path.Combine(tempPath, "GitHub", name);
+                shouldCleanUp = true;
+            }
+
+            return tempFolder;
+        }
+
+        public static FileInfo CreateTemporaryFile(string extension, out bool shouldCleanUpContainingFolder)
+        {
+            var tempFolder = GetTemporaryWorkingDirectory(out shouldCleanUpContainingFolder);
+            Directory.CreateDirectory(tempFolder);
+            string outputPath;
+            do
+            {
+                outputPath = Path.Combine(tempFolder, Path.GetRandomFileName() + extension);
+            }
+            while (File.Exists(outputPath));
+
+            File.Create(outputPath);
+
+            return new FileInfo(outputPath);
+        }
     }
 }

csharp/extractor/Semmle.Util/MemoizedFunc.cs

@@ -0,0 +1,39 @@
+using System;
+using System.Collections.Generic;
+using System.Collections.Concurrent;
+
+namespace Semmle.Util;
+
+public class MemoizedFunc<T1, T2> where T1 : notnull
+{
+    private readonly Func<T1, T2> f;
+    private readonly Dictionary<T1, T2> cache = new();
+
+    public MemoizedFunc(Func<T1, T2> f)
+    {
+        this.f = f;
+    }
+
+    public T2 Invoke(T1 s)
+    {
+        if (!cache.TryGetValue(s, out var t))
+        {
+            t = f(s);
+            cache[s] = t;
+        }
+        return t;
+    }
+}
+
+public class ConcurrentMemoizedFunc<T1, T2> where T1 : notnull
+{
+    private readonly Func<T1, T2> f;
+    private readonly ConcurrentDictionary<T1, T2> cache = new();
+
+    public ConcurrentMemoizedFunc(Func<T1, T2> f)
+    {
+        this.f = f;
+    }
+
+    public T2 Invoke(T1 s) => cache.GetOrAdd(s, f);
+}

csharp/ql/lib/ext/System.Collections.Generic.model.yml

@@ -47,6 +47,7 @@ extensions:
       - ["System.Collections.Generic", "List<>", False, "FindAll", "(System.Predicate<T>)", "", "Argument[this].Element", "ReturnValue", "value", "manual"]
       - ["System.Collections.Generic", "List<>", False, "FindLast", "(System.Predicate<T>)", "", "Argument[this].Element", "Argument[0].Parameter[0]", "value", "manual"]
       - ["System.Collections.Generic", "List<>", False, "FindLast", "(System.Predicate<T>)", "", "Argument[this].Element", "ReturnValue", "value", "manual"]
+      - ["System.Collections.Generic", "List<>", False, "ForEach", "(System.Action<T>)", "", "Argument[this].Element", "Argument[0].Parameter[0]", "value", "manual"]
       - ["System.Collections.Generic", "List<>", False, "GetEnumerator", "()", "", "Argument[this].Element", "ReturnValue.Property[System.Collections.Generic.List<>+Enumerator.Current]", "value", "manual"]
       - ["System.Collections.Generic", "List<>", False, "GetRange", "(System.Int32,System.Int32)", "", "Argument[this].Element", "ReturnValue.Element", "value", "manual"]
       - ["System.Collections.Generic", "List<>", False, "InsertRange", "(System.Int32,System.Collections.Generic.IEnumerable<T>)", "", "Argument[1].Element", "Argument[this].Element", "value", "manual"]

csharp/ql/lib/semmle/code/csharp/File.qll

@@ -29,8 +29,7 @@ private module Impl = Make<Input>;
 
 class Container = Impl::Container;
 
-/** A folder. */
-class Folder extends Container, Impl::Folder { }
+class Folder = Impl::Folder;
 
 bindingset[flag]
 private predicate fileHasExtractionFlag(File f, int flag) {

csharp/ql/lib/semmle/code/csharp/dataflow/ExternalFlow.qll

@@ -418,6 +418,20 @@ Element interpretElement(
   )
 }
 
+/**
+ * A callable where there exists a MaD sink model that applies to it.
+ */
+class SinkCallable extends Callable {
+  SinkCallable() { sinkElement(this, _, _, _) }
+}
+
+/**
+ * A callable where there exists a MaD source model that applies to it.
+ */
+class SourceCallable extends Callable {
+  SourceCallable() { sourceElement(this, _, _, _) }
+}
+
 cached
 private module Cached {
   /**

csharp/ql/lib/semmle/code/csharp/dataflow/FlowSummary.qll

@@ -4,6 +4,7 @@ import csharp
 private import dotnet
 private import internal.FlowSummaryImpl as Impl
 private import internal.DataFlowDispatch as DataFlowDispatch
+private import Impl::Public::SummaryComponent as SummaryComponentInternal
 
 class ParameterPosition = DataFlowDispatch::ParameterPosition;
 
@@ -18,8 +19,6 @@ class SummaryComponent = Impl::Public::SummaryComponent;
 
 /** Provides predicates for constructing summary components. */
 module SummaryComponent {
-  private import Impl::Public::SummaryComponent as SummaryComponentInternal
-
   predicate content = SummaryComponentInternal::content/1;
 
   /** Gets a summary component for parameter `i`. */
@@ -155,3 +154,45 @@ private class RecordConstructorFlowRequiredSummaryComponentStack extends Require
     )
   }
 }
+
+class Provenance = Impl::Public::Provenance;
+
+private import semmle.code.csharp.frameworks.system.linq.Expressions
+
+private SummaryComponent delegateSelf() {
+  exists(ArgumentPosition pos |
+    result = SummaryComponentInternal::parameter(pos) and
+    pos.isDelegateSelf()
+  )
+}
+
+private predicate mayInvokeCallback(Callable c, int n) {
+  c.getParameter(n).getType() instanceof SystemLinqExpressions::DelegateExtType and
+  not c.fromSource()
+}
+
+private class SummarizedCallableWithCallback extends SummarizedCallable {
+  private int pos;
+
+  SummarizedCallableWithCallback() { mayInvokeCallback(this, pos) }
+
+  override predicate propagatesFlow(
+    SummaryComponentStack input, SummaryComponentStack output, boolean preservesValue
+  ) {
+    input = SummaryComponentStack::argument(pos) and
+    output = SummaryComponentStack::push(delegateSelf(), input) and
+    preservesValue = true
+  }
+
+  override predicate hasProvenance(Provenance provenance) { provenance = "hq-generated" }
+}
+
+private class RequiredComponentStackForCallback extends RequiredSummaryComponentStack {
+  override predicate required(SummaryComponent head, SummaryComponentStack tail) {
+    exists(int pos |
+      mayInvokeCallback(_, pos) and
+      head = delegateSelf() and
+      tail = SummaryComponentStack::argument(pos)
+    )
+  }
+}

csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowDispatch.qll

@@ -136,13 +136,15 @@ private module Cached {
   newtype TParameterPosition =
     TPositionalParameterPosition(int i) { i = any(Parameter p).getPosition() } or
     TThisParameterPosition() or
-    TImplicitCapturedParameterPosition(LocalScopeVariable v) { capturedWithFlowIn(v) }
+    TImplicitCapturedParameterPosition(LocalScopeVariable v) { capturedWithFlowIn(v) } or
+    TDelegateSelfParameterPosition()
 
   cached
   newtype TArgumentPosition =
     TPositionalArgumentPosition(int i) { i = any(Parameter p).getPosition() } or
     TQualifierArgumentPosition() or
-    TImplicitCapturedArgumentPosition(LocalScopeVariable v) { capturedWithFlowIn(v) }
+    TImplicitCapturedArgumentPosition(LocalScopeVariable v) { capturedWithFlowIn(v) } or
+    TDelegateSelfArgumentPosition()
 }
 
 import Cached
@@ -480,6 +482,14 @@ class ParameterPosition extends TParameterPosition {
     this = TImplicitCapturedParameterPosition(v)
   }
 
+  /**
+   * Holds if this position represents a reference to a delegate itself.
+   *
+   * Used for tracking flow through captured variables and for improving
+   * delegate dispatch.
+   */
+  predicate isDelegateSelf() { this = TDelegateSelfParameterPosition() }
+
   /** Gets a textual representation of this position. */
   string toString() {
     result = "position " + this.getPosition()
@@ -489,6 +499,9 @@ class ParameterPosition extends TParameterPosition {
     exists(LocalScopeVariable v |
       this.isImplicitCapturedParameterPosition(v) and result = "captured " + v
     )
+    or
+    this.isDelegateSelf() and
+    result = "delegate self"
   }
 }
 
@@ -505,6 +518,14 @@ class ArgumentPosition extends TArgumentPosition {
     this = TImplicitCapturedArgumentPosition(v)
   }
 
+  /**
+   * Holds if this position represents a reference to a delegate itself.
+   *
+   * Used for tracking flow through captured variables and for improving
+   * delegate dispatch.
+   */
+  predicate isDelegateSelf() { this = TDelegateSelfArgumentPosition() }
+
   /** Gets a textual representation of this position. */
   string toString() {
     result = "position " + this.getPosition()
@@ -514,6 +535,9 @@ class ArgumentPosition extends TArgumentPosition {
     exists(LocalScopeVariable v |
       this.isImplicitCapturedArgumentPosition(v) and result = "captured " + v
     )
+    or
+    this.isDelegateSelf() and
+    result = "delegate self"
   }
 }
 
@@ -527,14 +551,6 @@ predicate parameterMatch(ParameterPosition ppos, ArgumentPosition apos) {
     ppos.isImplicitCapturedParameterPosition(v) and
     apos.isImplicitCapturedArgumentPosition(v)
   )
-}
-
-/**
- * Holds if flow from `call`'s argument `arg` to parameter `p` is permissible.
- *
- * This is a temporary hook to support technical debt in the Go language; do not use.
- */
-pragma[inline]
-predicate golangSpecificParamArgFilter(DataFlowCall call, ParameterNode p, ArgumentNode arg) {
-  any()
+  or
+  ppos.isDelegateSelf() and apos.isDelegateSelf()
 }

csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl1.qll

@@ -297,6 +297,10 @@ private module Config implements FullStateConfigSig {
 
   predicate isBarrierOut(Node node) { any(Configuration config).isBarrierOut(node) }
 
+  predicate isBarrierIn(Node node, FlowState state) { none() }
+
+  predicate isBarrierOut(Node node, FlowState state) { none() }
+
   predicate isAdditionalFlowStep(Node node1, Node node2) {
     singleConfiguration() and
     any(Configuration config).isAdditionalFlowStep(node1, node2)

csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl2.qll

@@ -297,6 +297,10 @@ private module Config implements FullStateConfigSig {
 
   predicate isBarrierOut(Node node) { any(Configuration config).isBarrierOut(node) }
 
+  predicate isBarrierIn(Node node, FlowState state) { none() }
+
+  predicate isBarrierOut(Node node, FlowState state) { none() }
+
   predicate isAdditionalFlowStep(Node node1, Node node2) {
     singleConfiguration() and
     any(Configuration config).isAdditionalFlowStep(node1, node2)

csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl3.qll

@@ -297,6 +297,10 @@ private module Config implements FullStateConfigSig {
 
   predicate isBarrierOut(Node node) { any(Configuration config).isBarrierOut(node) }
 
+  predicate isBarrierIn(Node node, FlowState state) { none() }
+
+  predicate isBarrierOut(Node node, FlowState state) { none() }
+
   predicate isAdditionalFlowStep(Node node1, Node node2) {
     singleConfiguration() and
     any(Configuration config).isAdditionalFlowStep(node1, node2)

csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl4.qll

@@ -297,6 +297,10 @@ private module Config implements FullStateConfigSig {
 
   predicate isBarrierOut(Node node) { any(Configuration config).isBarrierOut(node) }
 
+  predicate isBarrierIn(Node node, FlowState state) { none() }
+
+  predicate isBarrierOut(Node node, FlowState state) { none() }
+
   predicate isAdditionalFlowStep(Node node1, Node node2) {
     singleConfiguration() and
     any(Configuration config).isAdditionalFlowStep(node1, node2)

csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl5.qll

@@ -297,6 +297,10 @@ private module Config implements FullStateConfigSig {
 
   predicate isBarrierOut(Node node) { any(Configuration config).isBarrierOut(node) }
 
+  predicate isBarrierIn(Node node, FlowState state) { none() }
+
+  predicate isBarrierOut(Node node, FlowState state) { none() }
+
   predicate isAdditionalFlowStep(Node node1, Node node2) {
     singleConfiguration() and
     any(Configuration config).isAdditionalFlowStep(node1, node2)

csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowPrivate.qll

@@ -45,9 +45,9 @@ abstract class NodeImpl extends Node {
   abstract DotNet::Type getTypeImpl();
 
   /** Gets the type of this node used for type pruning. */
-  Gvn::GvnType getDataFlowType() {
+  DataFlowType getDataFlowType() {
     forceCachingInSameStage() and
-    exists(Type t0 | result = Gvn::getGlobalValueNumber(t0) |
+    exists(Type t0 | result.asGvnType() = Gvn::getGlobalValueNumber(t0) |
       t0 = getCSharpType(this.getType())
       or
       not exists(getCSharpType(this.getType())) and
@@ -557,6 +557,8 @@ module LocalFlow {
       exists(SsaImpl::getAReadAtNode(def, node2.(ExprNode).getControlFlowNode()))
     )
     or
+    delegateCreationStep(node1, node2)
+    or
     node1 =
       unique(FlowSummaryNode n1 |
         FlowSummaryImpl::Private::Steps::summaryLocalStep(n1.getSummaryNode(),
@@ -793,16 +795,16 @@ private Type getCSharpType(DotNet::Type t) {
   result.matchesHandle(t)
 }
 
-private class RelevantDataFlowType extends DataFlowType {
-  RelevantDataFlowType() { this = any(NodeImpl n).getDataFlowType() }
+private class RelevantGvnType extends Gvn::GvnType {
+  RelevantGvnType() { this = any(NodeImpl n).getDataFlowType().asGvnType() }
 }
 
 /** A GVN type that is either a `DataFlowType` or unifiable with a `DataFlowType`. */
 private class DataFlowTypeOrUnifiable extends Gvn::GvnType {
   pragma[nomagic]
   DataFlowTypeOrUnifiable() {
-    this instanceof RelevantDataFlowType or
-    Gvn::unifiable(any(RelevantDataFlowType t), this)
+    this instanceof RelevantGvnType or
+    Gvn::unifiable(any(RelevantGvnType t), this)
   }
 }
 
@@ -813,7 +815,7 @@ private TypeParameter getATypeParameterSubType(DataFlowTypeOrUnifiable t) {
 }
 
 pragma[noinline]
-private TypeParameter getATypeParameterSubTypeRestricted(RelevantDataFlowType t) {
+private TypeParameter getATypeParameterSubTypeRestricted(RelevantGvnType t) {
   result = getATypeParameterSubType(t)
 }
 
@@ -829,7 +831,7 @@ private Gvn::GvnType getANonTypeParameterSubType(DataFlowTypeOrUnifiable t) {
 }
 
 pragma[noinline]
-private Gvn::GvnType getANonTypeParameterSubTypeRestricted(RelevantDataFlowType t) {
+private Gvn::GvnType getANonTypeParameterSubTypeRestricted(RelevantGvnType t) {
   result = getANonTypeParameterSubType(t)
 }
 
@@ -866,6 +868,7 @@ private module Cached {
       c = any(DataFlowCallable dfc).asCallable() and
       not c.(Modifiable).isStatic()
     } or
+    TDelegateSelfReferenceNode(Callable c) { lambdaCreationExpr(_, c) } or
     TYieldReturnNode(ControlFlow::Nodes::ElementNode cfn) {
       any(Callable c).canYieldReturn(cfn.getAstNode())
     } or
@@ -949,7 +952,7 @@ private module Cached {
     TSyntheticFieldApproxContent()
 
   pragma[nomagic]
-  private predicate commonSubTypeGeneral(DataFlowTypeOrUnifiable t1, RelevantDataFlowType t2) {
+  private predicate commonSubTypeGeneral(DataFlowTypeOrUnifiable t1, RelevantGvnType t2) {
     not t1 instanceof Gvn::TypeParameterGvnType and
     t1 = t2
     or
@@ -963,17 +966,20 @@ private module Cached {
    * `t2` are allowed to be type parameters.
    */
   cached
-  predicate commonSubType(RelevantDataFlowType t1, RelevantDataFlowType t2) {
-    commonSubTypeGeneral(t1, t2)
-  }
+  predicate commonSubType(RelevantGvnType t1, RelevantGvnType t2) { commonSubTypeGeneral(t1, t2) }
 
   cached
-  predicate commonSubTypeUnifiableLeft(RelevantDataFlowType t1, RelevantDataFlowType t2) {
+  predicate commonSubTypeUnifiableLeft(RelevantGvnType t1, RelevantGvnType t2) {
     exists(Gvn::GvnType t |
       Gvn::unifiable(t1, t) and
       commonSubTypeGeneral(t, t2)
     )
   }
+
+  cached
+  newtype TDataFlowType =
+    TGvnDataFlowType(Gvn::GvnType t) or
+    TDelegateDataFlowType(Callable lambda) { lambdaCreationExpr(_, lambda) }
 }
 
 import Cached
@@ -1119,6 +1125,37 @@ private module ParameterNodes {
     override string toStringImpl() { result = "this" }
   }
 
+  /**
+   * The value of a delegate itself at function entry, viewed as a node in a data
+   * flow graph.
+   *
+   * This is used for improving lambda dispatch, and will eventually also be
+   * used for tracking flow through captured variables.
+   */
+  private class DelegateSelfReferenceNode extends ParameterNodeImpl, TDelegateSelfReferenceNode {
+    private Callable callable;
+
+    DelegateSelfReferenceNode() { this = TDelegateSelfReferenceNode(callable) }
+
+    final Callable getCallable() { result = callable }
+
+    override predicate isParameterOf(DataFlowCallable c, ParameterPosition pos) {
+      callable = c.asCallable() and pos.isDelegateSelf()
+    }
+
+    override ControlFlow::Node getControlFlowNodeImpl() { none() }
+
+    override DataFlowCallable getEnclosingCallableImpl() { result.asCallable() = callable }
+
+    override Location getLocationImpl() { result = callable.getLocation() }
+
+    override DotNet::Type getTypeImpl() { none() }
+
+    override DataFlowType getDataFlowType() { callable = result.asDelegate() }
+
+    override string toStringImpl() { result = "delegate self in " + callable }
+  }
+
   /** An implicit entry definition for a captured variable. */
   class SsaCapturedEntryDefinition extends Ssa::ImplicitEntryDefinition {
     private LocalScopeVariable v;
@@ -1232,6 +1269,18 @@ private module ArgumentNodes {
     }
   }
 
+  /** A data-flow node that represents a delegate passed into itself. */
+  class DelegateSelfArgumentNode extends ArgumentNodeImpl {
+    private DataFlowCall call_;
+
+    DelegateSelfArgumentNode() { lambdaCallExpr(call_, this) }
+
+    override predicate argumentOf(DataFlowCall call, ArgumentPosition pos) {
+      call = call_ and
+      pos.isDelegateSelf()
+    }
+  }
+
   /**
    * The value of a captured variable as an implicit argument of a call, viewed
    * as a node in a data flow graph.
@@ -1985,41 +2034,92 @@ predicate isUnreachableInCall(Node n, DataFlowCall call) {
  * For example, `Func<T, int>` and `Func<S, int>` are mapped to the same
  * `DataFlowType`, while `Func<T, int>` and `Func<string, int>` are not, because
  * `string` is not a type parameter.
+ *
+ * For delegates, we use the delegate itself instead of its type, in order to
+ * improve dispatch.
  */
-class DataFlowType = Gvn::GvnType;
+class DataFlowType extends TDataFlowType {
+  Gvn::GvnType asGvnType() { this = TGvnDataFlowType(result) }
+
+  Callable asDelegate() { this = TDelegateDataFlowType(result) }
+
+  /**
+   * Gets an expression that creates a delegate of this type.
+   *
+   * For methods used as method groups in calls there can be multiple
+   * creations associated with the same type.
+   */
+  Expr getADelegateCreation() {
+    exists(Callable callable |
+      lambdaCreationExpr(result, callable) and
+      this = TDelegateDataFlowType(callable)
+    )
+  }
+
+  final string toString() {
+    result = this.asGvnType().toString()
+    or
+    result = this.asDelegate().toString()
+  }
+}
 
 /** Gets the type of `n` used for type pruning. */
-Gvn::GvnType getNodeType(Node n) { result = n.(NodeImpl).getDataFlowType() }
+DataFlowType getNodeType(Node n) {
+  result = n.(NodeImpl).getDataFlowType() and
+  not lambdaCreation(n, _, _) and
+  not delegateCreationStep(_, n)
+  or
+  exists(Node arg |
+    delegateCreationStep(arg, n) and
+    result = getNodeType(arg)
+  )
+  or
+  n.asExpr() = result.getADelegateCreation()
+}
 
 /** Gets a string representation of a `DataFlowType`. */
 string ppReprType(DataFlowType t) { result = t.toString() }
 
-private class DataFlowNullType extends DataFlowType {
+private class DataFlowNullType extends Gvn::GvnType {
   DataFlowNullType() { this = Gvn::getGlobalValueNumber(any(NullType nt)) }
 
   pragma[noinline]
-  predicate isConvertibleTo(DataFlowType t) {
+  predicate isConvertibleTo(Gvn::GvnType t) {
     defaultNullConversion(_, any(Type t0 | t = Gvn::getGlobalValueNumber(t0)))
   }
 }
 
-private class DataFlowUnknownType extends DataFlowType {
-  DataFlowUnknownType() { this = Gvn::getGlobalValueNumber(any(UnknownType ut)) }
-}
-
-private predicate uselessTypebound(DataFlowType t) {
-  t instanceof DataFlowUnknownType or
-  t instanceof Gvn::TypeParameterGvnType
+private class GvnUnknownType extends Gvn::GvnType {
+  GvnUnknownType() { this = Gvn::getGlobalValueNumber(any(UnknownType ut)) }
 }
 
 pragma[nomagic]
-predicate typeStrongerThan(DataFlowType t1, DataFlowType t2) {
-  t1 != t2 and
-  t1 = getANonTypeParameterSubTypeRestricted(t2)
-  or
-  t1 instanceof RelevantDataFlowType and
-  not uselessTypebound(t1) and
-  uselessTypebound(t2)
+private predicate uselessTypebound(DataFlowType dt) {
+  dt.asGvnType() =
+    any(Gvn::GvnType t |
+      t instanceof GvnUnknownType or
+      t instanceof Gvn::TypeParameterGvnType
+    )
+}
+
+pragma[inline]
+private predicate compatibleTypesDelegateLeft(DataFlowType dt1, DataFlowType dt2) {
+  exists(Gvn::GvnType t1, Gvn::GvnType t2 |
+    t1 = exprNode(dt1.getADelegateCreation()).(NodeImpl).getDataFlowType().asGvnType() and
+    t2 = dt2.asGvnType()
+  |
+    commonSubType(t1, t2)
+    or
+    commonSubTypeUnifiableLeft(t1, t2)
+    or
+    commonSubTypeUnifiableLeft(t2, t1)
+    or
+    t2.(DataFlowNullType).isConvertibleTo(t1)
+    or
+    t2 instanceof Gvn::TypeParameterGvnType
+    or
+    t2 instanceof GvnUnknownType
+  )
 }
 
 /**
@@ -2027,24 +2127,47 @@ predicate typeStrongerThan(DataFlowType t1, DataFlowType t2) {
  * a node of type `t1` to a node of type `t2`.
  */
 pragma[inline]
-predicate compatibleTypes(DataFlowType t1, DataFlowType t2) {
-  commonSubType(t1, t2)
+predicate compatibleTypes(DataFlowType dt1, DataFlowType dt2) {
+  exists(Gvn::GvnType t1, Gvn::GvnType t2 |
+    t1 = dt1.asGvnType() and
+    t2 = dt2.asGvnType()
+  |
+    commonSubType(t1, t2)
+    or
+    commonSubTypeUnifiableLeft(t1, t2)
+    or
+    commonSubTypeUnifiableLeft(t2, t1)
+    or
+    t1.(DataFlowNullType).isConvertibleTo(t2)
+    or
+    t2.(DataFlowNullType).isConvertibleTo(t1)
+    or
+    t1 instanceof Gvn::TypeParameterGvnType
+    or
+    t2 instanceof Gvn::TypeParameterGvnType
+    or
+    t1 instanceof GvnUnknownType
+    or
+    t2 instanceof GvnUnknownType
+  )
   or
-  commonSubTypeUnifiableLeft(t1, t2)
+  compatibleTypesDelegateLeft(dt1, dt2)
   or
-  commonSubTypeUnifiableLeft(t2, t1)
+  compatibleTypesDelegateLeft(dt2, dt1)
   or
-  t1.(DataFlowNullType).isConvertibleTo(t2)
+  dt1.asDelegate() = dt2.asDelegate()
+}
+
+pragma[nomagic]
+predicate typeStrongerThan(DataFlowType t1, DataFlowType t2) {
+  t1 != t2 and
+  t1.asGvnType() = getANonTypeParameterSubTypeRestricted(t2.asGvnType())
   or
-  t2.(DataFlowNullType).isConvertibleTo(t1)
+  t1.asGvnType() instanceof RelevantGvnType and
+  not uselessTypebound(t1) and
+  uselessTypebound(t2)
   or
-  t1 instanceof Gvn::TypeParameterGvnType
-  or
-  t2 instanceof Gvn::TypeParameterGvnType
-  or
-  t1 instanceof DataFlowUnknownType
-  or
-  t2 instanceof DataFlowUnknownType
+  compatibleTypesDelegateLeft(t1, t2)
 }
 
 /**
@@ -2216,17 +2339,20 @@ int accessPathLimit() { result = 5 }
  */
 predicate forceHighPrecision(Content c) { c instanceof ElementContent }
 
+private predicate lambdaCreationExpr(Expr creation, Callable c) {
+  c =
+    [
+      creation.(AnonymousFunctionExpr),
+      creation.(CallableAccess).getTarget().getUnboundDeclaration(),
+      creation.(AddressOfExpr).getOperand().(CallableAccess).getTarget().getUnboundDeclaration()
+    ]
+}
+
 class LambdaCallKind = Unit;
 
 /** Holds if `creation` is an expression that creates a delegate for `c`. */
 predicate lambdaCreation(Node creation, LambdaCallKind kind, DataFlowCallable c) {
-  exists(Expr e | e = creation.asExpr() |
-    c.asCallable() =
-      [
-        e.(AnonymousFunctionExpr), e.(CallableAccess).getTarget().getUnboundDeclaration(),
-        e.(AddressOfExpr).getOperand().(CallableAccess).getTarget().getUnboundDeclaration()
-      ]
-  ) and
+  lambdaCreationExpr(creation.asExpr(), c.asCallable()) and
   exists(kind)
 }
 
@@ -2248,19 +2374,29 @@ private class LambdaConfiguration extends ControlFlowReachabilityConfiguration {
   }
 }
 
+private predicate lambdaCallExpr(DataFlowCall call, ExprNode receiver) {
+  exists(LambdaConfiguration x, DelegateLikeCall dc |
+    x.hasExprPath(dc.getExpr(), receiver.getControlFlowNode(), dc, call.getControlFlowNode())
+  )
+}
+
 /** Holds if `call` is a lambda call where `receiver` is the lambda expression. */
 predicate lambdaCall(DataFlowCall call, LambdaCallKind kind, Node receiver) {
   (
-    exists(LambdaConfiguration x, DelegateLikeCall dc |
-      x.hasExprPath(dc.getExpr(), receiver.(ExprNode).getControlFlowNode(), dc,
-        call.getControlFlowNode())
-    )
+    lambdaCallExpr(call, receiver)
     or
     receiver.(FlowSummaryNode).getSummaryNode() = call.(SummaryCall).getReceiver()
   ) and
   exists(kind)
 }
 
+private predicate delegateCreationStep(Node nodeFrom, Node nodeTo) {
+  exists(LambdaConfiguration x, DelegateCreation dc |
+    x.hasExprPath(dc.getArgument(), nodeFrom.(ExprNode).getControlFlowNode(), dc,
+      nodeTo.(ExprNode).getControlFlowNode())
+  )
+}
+
 /** Extra data-flow steps needed for lambda flow analysis. */
 predicate additionalLambdaFlowStep(Node nodeFrom, Node nodeTo, boolean preservesValue) {
   exists(SsaImpl::DefinitionExt def |
@@ -2269,11 +2405,8 @@ predicate additionalLambdaFlowStep(Node nodeFrom, Node nodeTo, boolean preserves
     preservesValue = true
   )
   or
-  exists(LambdaConfiguration x, DelegateCreation dc |
-    x.hasExprPath(dc.getArgument(), nodeFrom.(ExprNode).getControlFlowNode(), dc,
-      nodeTo.(ExprNode).getControlFlowNode()) and
-    preservesValue = false
-  )
+  delegateCreationStep(nodeFrom, nodeTo) and
+  preservesValue = true
   or
   exists(AddEventExpr aee |
     nodeFrom.asExpr() = aee.getRValue() and
@@ -2420,12 +2553,3 @@ module Csv {
     )
   }
 }
-
-/**
- * Gets an additional term that is added to the `join` and `branch` computations to reflect
- * an additional forward or backwards branching factor that is not taken into account
- * when calculating the (virtual) dispatch cost.
- *
- * Argument `arg` is part of a path from a source to a sink, and `p` is the target parameter.
- */
-int getAdditionalFlowIntoCallNodeTerm(ArgumentNode arg, ParameterNode p) { none() }

csharp/ql/lib/semmle/code/csharp/dataflow/internal/FlowSummaryImplSpecific.qll

@@ -35,14 +35,14 @@ private module SyntheticGlobals {
 DataFlowCallable inject(SummarizedCallable c) { result.asSummarizedCallable() = c }
 
 /** Gets the parameter position of the instance parameter. */
-ArgumentPosition callbackSelfParameterPosition() { none() } // disables implicit summary flow to `this` for callbacks
+ArgumentPosition callbackSelfParameterPosition() { result.isDelegateSelf() }
 
 /** Gets the synthesized data-flow call for `receiver`. */
 SummaryCall summaryDataFlowCall(SummaryNode receiver) { receiver = result.getReceiver() }
 
 /** Gets the type of content `c`. */
 DataFlowType getContentType(Content c) {
-  exists(Type t | result = Gvn::getGlobalValueNumber(t) |
+  exists(Type t | result.asGvnType() = Gvn::getGlobalValueNumber(t) |
     t = c.(FieldContent).getField().getType()
     or
     t = c.(PropertyContent).getProperty().getType()
@@ -56,7 +56,7 @@ DataFlowType getContentType(Content c) {
 
 /** Gets the type of the parameter at the given position. */
 DataFlowType getParameterType(SummarizedCallable c, ParameterPosition pos) {
-  exists(Type t | result = Gvn::getGlobalValueNumber(t) |
+  exists(Type t | result.asGvnType() = Gvn::getGlobalValueNumber(t) |
     exists(int i |
       pos.getPosition() = i and
       t = c.getParameter(i).getType()
@@ -69,7 +69,7 @@ DataFlowType getParameterType(SummarizedCallable c, ParameterPosition pos) {
 
 /** Gets the return type of kind `rk` for callable `c`. */
 DataFlowType getReturnType(DotNet::Callable c, ReturnKind rk) {
-  exists(Type t | result = Gvn::getGlobalValueNumber(t) |
+  exists(Type t | result.asGvnType() = Gvn::getGlobalValueNumber(t) |
     rk instanceof NormalReturnKind and
     (
       t = c.(Constructor).getDeclaringType()
@@ -88,10 +88,13 @@ DataFlowType getReturnType(DotNet::Callable c, ReturnKind rk) {
  */
 DataFlowType getCallbackParameterType(DataFlowType t, ArgumentPosition pos) {
   exists(SystemLinqExpressions::DelegateExtType dt |
-    t = Gvn::getGlobalValueNumber(dt) and
-    result =
+    t.asGvnType() = Gvn::getGlobalValueNumber(dt) and
+    result.asGvnType() =
       Gvn::getGlobalValueNumber(dt.getDelegateType().getParameter(pos.getPosition()).getType())
   )
+  or
+  pos.isDelegateSelf() and
+  result = t
 }
 
 /**
@@ -101,15 +104,15 @@ DataFlowType getCallbackParameterType(DataFlowType t, ArgumentPosition pos) {
 DataFlowType getCallbackReturnType(DataFlowType t, ReturnKind rk) {
   rk instanceof NormalReturnKind and
   exists(SystemLinqExpressions::DelegateExtType dt |
-    t = Gvn::getGlobalValueNumber(dt) and
-    result = Gvn::getGlobalValueNumber(dt.getDelegateType().getReturnType())
+    t.asGvnType() = Gvn::getGlobalValueNumber(dt) and
+    result.asGvnType() = Gvn::getGlobalValueNumber(dt.getDelegateType().getReturnType())
   )
 }
 
 /** Gets the type of synthetic global `sg`. */
 DataFlowType getSyntheticGlobalType(SummaryComponent::SyntheticGlobal sg) {
   exists(sg) and
-  result = Gvn::getGlobalValueNumber(any(ObjectType t))
+  result.asGvnType() = Gvn::getGlobalValueNumber(any(ObjectType t))
 }
 
 /**
@@ -223,6 +226,9 @@ string getParameterPosition(ParameterPosition pos) {
   or
   pos.isThisParameter() and
   result = "this"
+  or
+  pos.isDelegateSelf() and
+  result = "delegate-self"
 }
 
 /** Gets the textual representation of an argument position in the format used for flow summaries. */
@@ -231,6 +237,9 @@ string getArgumentPosition(ArgumentPosition pos) {
   or
   pos.isQualifier() and
   result = "this"
+  or
+  pos.isDelegateSelf() and
+  result = "delegate-self"
 }
 
 /** Holds if input specification component `c` needs a reference. */
@@ -312,6 +321,9 @@ ArgumentPosition parseParamBody(string s) {
   or
   s = "this" and
   result.isQualifier()
+  or
+  s = "delegate-self" and
+  result.isDelegateSelf()
 }
 
 /** Gets the parameter position obtained by parsing `X` in `Argument[X]`. */
@@ -321,4 +333,7 @@ ParameterPosition parseArgBody(string s) {
   or
   s = "this" and
   result.isThisParameter()
+  or
+  s = "delegate-self" and
+  result.isDelegateSelf()
 }

csharp/ql/lib/semmle/code/csharp/security/auth/ActionMethods.qll

@@ -0,0 +1,107 @@
+/** Common definitions for queries checking for access control measures on action methods. */
+
+import csharp
+import semmle.code.csharp.frameworks.microsoft.AspNetCore
+import semmle.code.csharp.frameworks.system.web.UI
+
+/** A method representing an action for a web endpoint. */
+abstract class ActionMethod extends Method {
+  /**
+   * Gets a string that can indicate what this method does to determine if it should have an auth check;
+   * such as its method name, class name, or file path.
+   */
+  string getADescription() {
+    result = [this.getName(), this.getDeclaringType().getBaseClass*().getName(), this.getARoute()]
+  }
+
+  /** Holds if this method may represent a stateful action such as editing or deleting */
+  predicate isEdit() {
+    exists(string str |
+      str =
+        this.getADescription()
+            // separate camelCase words
+            .regexpReplaceAll("([a-z])([A-Z])", "$1_$2") and
+      str.regexpMatch("(?i).*(edit|delete|modify|change).*") and
+      not str.regexpMatch("(?i).*(on_?change|changed).*")
+    )
+  }
+
+  /** Holds if this method may be intended to be restricted to admin users */
+  predicate isAdmin() {
+    this.getADescription()
+        // separate camelCase words
+        .regexpReplaceAll("([a-z])([A-Z])", "$1_$2")
+        .regexpMatch("(?i).*(admin|superuser).*")
+  }
+
+  /** Gets a callable for which if it contains an auth check, this method should be considered authenticated. */
+  Callable getAnAuthorizingCallable() { result = this }
+
+  /**
+   * Gets a possible url route that could refer to this action,
+   * which would be covered by `<location>` configurations specifying a prefix of it.
+   */
+  string getARoute() { result = this.getDeclaringType().getFile().getRelativePath() }
+}
+
+/** An action method in the MVC framework. */
+private class MvcActionMethod extends ActionMethod {
+  MvcActionMethod() { this = any(MicrosoftAspNetCoreMvcController c).getAnActionMethod() }
+}
+
+/** An action method on a subclass of `System.Web.UI.Page`. */
+private class WebFormActionMethod extends ActionMethod {
+  WebFormActionMethod() {
+    this.getDeclaringType().getBaseClass+() instanceof SystemWebUIPageClass and
+    this.getAParameter().getType().getName().matches("%EventArgs")
+  }
+
+  override Callable getAnAuthorizingCallable() {
+    result = super.getAnAuthorizingCallable()
+    or
+    pageLoad(result, this.getDeclaringType())
+  }
+
+  override string getARoute() {
+    exists(string physicalRoute | physicalRoute = super.getARoute() |
+      result = physicalRoute
+      or
+      exists(string absolutePhysical |
+        virtualRouteMapping(result, absolutePhysical) and
+        physicalRouteMatches(absolutePhysical, physicalRoute)
+      )
+    )
+  }
+}
+
+pragma[nomagic]
+private predicate pageLoad(Callable c, Type decl) {
+  c.getName() = "Page_Load" and
+  decl = c.getDeclaringType()
+}
+
+/**
+ * Holds if `virtualRoute` is a URL path
+ * that can map to the corresponding `physicalRoute` filepath
+ * through a call to `MapPageRoute`
+ */
+private predicate virtualRouteMapping(string virtualRoute, string physicalRoute) {
+  exists(MethodCall mapPageRouteCall, StringLiteral virtualLit, StringLiteral physicalLit |
+    mapPageRouteCall
+        .getTarget()
+        .hasQualifiedName("System.Web.Routing", "RouteCollection", "MapPageRoute") and
+    virtualLit = mapPageRouteCall.getArgument(1) and
+    physicalLit = mapPageRouteCall.getArgument(2) and
+    virtualLit.getValue() = virtualRoute and
+    physicalLit.getValue() = physicalRoute
+  )
+}
+
+/** Holds if the filepath `route` can refer to `actual` after expanding a '~". */
+bindingset[route, actual]
+private predicate physicalRouteMatches(string route, string actual) {
+  route = actual
+  or
+  route.charAt(0) = "~" and
+  exists(string dir | actual = dir + route.suffix(1) + ".cs")
+}

csharp/ql/lib/semmle/code/csharp/security/auth/InsecureDirectObjectReferenceQuery.qll

@@ -0,0 +1,96 @@
+/** Definitions for the Insecure Direct Object Reference query */
+
+import csharp
+import semmle.code.csharp.dataflow.flowsources.Remote
+import ActionMethods
+
+/**
+ * Holds if `m` is a method that may require checks
+ * against the current user to modify a particular resource.
+ */
+// We exclude admin methods as it may be expected that an admin user should be able to modify any resource.
+// Other queries check that there are authorization checks in place for admin methods.
+private predicate needsChecks(ActionMethod m) { m.isEdit() and not m.isAdmin() }
+
+/**
+ * Holds if `m` has a parameter or access a remote flow source
+ * that may indicate that it's used as the ID for some resource
+ */
+private predicate hasIdParameter(ActionMethod m) {
+  exists(RemoteFlowSource src | src.getEnclosingCallable() = m |
+    src.asParameter().getName().toLowerCase().matches(["%id", "%idx"])
+    or
+    // handle cases like `Request.QueryString["Id"]`
+    exists(StringLiteral idStr, IndexerCall idx |
+      idStr.getValue().toLowerCase().matches(["%id", "%idx"]) and
+      TaintTracking::localTaint(src, DataFlow::exprNode(idx.getQualifier())) and
+      DataFlow::localExprFlow(idStr, idx.getArgument(0))
+    )
+  )
+}
+
+private predicate authorizingCallable(Callable c) {
+  exists(string name | name = c.getName().toLowerCase() |
+    name.matches(["%user%", "%session%"]) and
+    not name.matches("%get%by%") // methods like `getUserById` or `getXByUsername` aren't likely to be referring to the current user
+  )
+}
+
+/** Holds if `m` at some point in its call graph may make some kind of check against the current user. */
+private predicate checksUser(ActionMethod m) {
+  exists(Callable c |
+    authorizingCallable(c) and
+    callsPlus(m, c)
+  )
+}
+
+private predicate calls(Callable c1, Callable c2) { c1.calls(c2) }
+
+private predicate callsPlus(Callable c1, Callable c2) = fastTC(calls/2)(c1, c2)
+
+/** Holds if `m`, its containing class, or a parent class has an attribute that extends `AuthorizeAttribute` */
+private predicate hasAuthorizeAttribute(ActionMethod m) {
+  exists(Attribute attr |
+    attr.getType()
+        .getABaseType*()
+        .hasQualifiedName([
+            "Microsoft.AspNetCore.Authorization", "System.Web.Mvc", "System.Web.Http"
+          ], "AuthorizeAttribute")
+  |
+    attr = m.getOverridee*().getAnAttribute() or
+    attr = m.getDeclaringType().getABaseType*().getAnAttribute()
+  )
+}
+
+/** Holds if `m`, its containing class, or a parent class has an attribute that extends `AllowAnonymousAttribute` */
+private predicate hasAllowAnonymousAttribute(ActionMethod m) {
+  exists(Attribute attr |
+    attr.getType()
+        .getABaseType*()
+        .hasQualifiedName([
+            "Microsoft.AspNetCore.Authorization", "System.Web.Mvc", "System.Web.Http"
+          ], "AllowAnonymousAttribute")
+  |
+    attr = m.getOverridee*().getAnAttribute() or
+    attr = m.getDeclaringType().getABaseType*().getAnAttribute()
+  )
+}
+
+/** Holds if `m` is authorized via an `Authorize` attribute */
+private predicate isAuthorizedViaAttribute(ActionMethod m) {
+  hasAuthorizeAttribute(m) and
+  not hasAllowAnonymousAttribute(m)
+}
+
+/**
+ * Holds if `m` is a method that modifies a particular resource based on
+ * an ID provided by user input, but does not check anything based on the current user
+ * to determine if they should modify this resource.
+ */
+predicate hasInsecureDirectObjectReference(ActionMethod m) {
+  needsChecks(m) and
+  hasIdParameter(m) and
+  not checksUser(m) and
+  not isAuthorizedViaAttribute(m) and
+  exists(m.getBody().getAChildStmt())
+}

csharp/ql/lib/semmle/code/csharp/security/auth/MissingFunctionLevelAccessControlQuery.qll

@@ -4,96 +4,10 @@ import csharp
 import semmle.code.csharp.frameworks.microsoft.AspNetCore
 import semmle.code.csharp.frameworks.system.web.UI
 import semmle.code.asp.WebConfig
+import ActionMethods
 
-/** A method representing an action for a web endpoint. */
-abstract class ActionMethod extends Method {
-  /**
-   * Gets a string that can indicate what this method does to determine if it should have an auth check;
-   * such as its method name, class name, or file path.
-   */
-  string getADescription() {
-    result =
-      [
-        this.getName(), this.getDeclaringType().getBaseClass*().getName(),
-        this.getDeclaringType().getFile().getRelativePath()
-      ]
-  }
-
-  /** Holds if this method may need an authorization check. */
-  predicate needsAuth() {
-    this.getADescription()
-        .regexpReplaceAll("([a-z])([A-Z])", "$1_$2")
-        // separate camelCase words
-        .toLowerCase()
-        .regexpMatch(".*(edit|delete|modify|admin|superuser).*")
-  }
-
-  /** Gets a callable for which if it contains an auth check, this method should be considered authenticated. */
-  Callable getAnAuthorizingCallable() { result = this }
-
-  /**
-   * Gets a possible url route that could refer to this action,
-   * which would be covered by `<location>` configurations specifying a prefix of it.
-   */
-  string getARoute() { result = this.getDeclaringType().getFile().getRelativePath() }
-}
-
-/** An action method in the MVC framework. */
-private class MvcActionMethod extends ActionMethod {
-  MvcActionMethod() { this = any(MicrosoftAspNetCoreMvcController c).getAnActionMethod() }
-}
-
-/** An action method on a subclass of `System.Web.UI.Page`. */
-private class WebFormActionMethod extends ActionMethod {
-  WebFormActionMethod() {
-    this.getDeclaringType().getBaseClass+() instanceof SystemWebUIPageClass and
-    this.getAParameter().getType().getName().matches("%EventArgs")
-  }
-
-  override Callable getAnAuthorizingCallable() {
-    result = super.getAnAuthorizingCallable()
-    or
-    result.getDeclaringType() = this.getDeclaringType() and
-    result.getName() = "Page_Load"
-  }
-
-  override string getARoute() {
-    exists(string physicalRoute | physicalRoute = super.getARoute() |
-      result = physicalRoute
-      or
-      exists(string absolutePhysical |
-        virtualRouteMapping(result, absolutePhysical) and
-        physicalRouteMatches(absolutePhysical, physicalRoute)
-      )
-    )
-  }
-}
-
-/**
- * Holds if `virtualRoute` is a URL path
- * that can map to the corresponding `physicalRoute` filepath
- * through a call to `MapPageRoute`
- */
-private predicate virtualRouteMapping(string virtualRoute, string physicalRoute) {
-  exists(MethodCall mapPageRouteCall, StringLiteral virtualLit, StringLiteral physicalLit |
-    mapPageRouteCall
-        .getTarget()
-        .hasQualifiedName("System.Web.Routing", "RouteCollection", "MapPageRoute") and
-    virtualLit = mapPageRouteCall.getArgument(1) and
-    physicalLit = mapPageRouteCall.getArgument(2) and
-    virtualLit.getValue() = virtualRoute and
-    physicalLit.getValue() = physicalRoute
-  )
-}
-
-/** Holds if the filepath `route` can refer to `actual` after expanding a '~". */
-bindingset[route, actual]
-private predicate physicalRouteMatches(string route, string actual) {
-  route = actual
-  or
-  route.charAt(0) = "~" and
-  exists(string dir | actual = dir + route.suffix(1) + ".cs")
-}
+/** Holds if the method `m` may need an authorization check. */
+predicate needsAuth(ActionMethod m) { m.isEdit() or m.isAdmin() }
 
 /** An expression that indicates that some authorization/authentication check is being performed. */
 class AuthExpr extends Expr {
@@ -114,7 +28,7 @@ class AuthExpr extends Expr {
 
 /** Holds if `m` is a method that should have an auth check, and does indeed have one. */
 predicate hasAuthViaCode(ActionMethod m) {
-  m.needsAuth() and
+  needsAuth(m) and
   exists(Callable caller, AuthExpr auth |
     m.getAnAuthorizingCallable().calls*(caller) and
     auth.getEnclosingCallable() = caller
@@ -175,7 +89,7 @@ predicate hasAuthViaAttribute(ActionMethod m) {
 
 /** Holds if `m` is a method that should have an auth check, but is missing it. */
 predicate missingAuth(ActionMethod m) {
-  m.needsAuth() and
+  needsAuth(m) and
   not hasAuthViaCode(m) and
   not hasAuthViaXml(m) and
   not hasAuthViaAttribute(m) and

csharp/ql/src/Security Features/CWE-639/InsecureDirectObjectReference.qhelp

@@ -0,0 +1,30 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+<overview>
+<p>Resources like comments or user profiles can be accessed and modified through an action method. To target a certain resource, the action method accepts an ID parameter pointing to that specific resource. If the methods do not check that the current user is authorized to access the specified resource, an attacker can access a resource by guessing or otherwise determining the linked ID parameter.</p>
+
+</overview>
+<recommendation>
+<p>
+Ensure that the current user is authorized to access the resource of the provided ID.
+</p>
+
+</recommendation>
+<example>
+<p>In the following example, in the "BAD" case, there is no authorization check, so any user can edit any comment for which they guess or determine the ID parameter.
+The "GOOD" case includes a check that the current user matches the author of the comment, preventing unauthorized access.</p>
+<sample src="WebFormsExample.cs" />
+<p>The following example shows a similar scenario for the ASP.NET Core framework. As above, the "BAD" case provides an example with no authorization check, and the first "GOOD" case provides an example with a check that the current user authored the specified comment. Additionally, in the second "GOOD" case, the `Authorize` attribute is used to restrict the method to administrators, who are expected to be able to access arbitrary resources.</p>
+<sample src="MVCExample.cs" />
+
+</example>
+<references>
+
+  <li>OWASP: <a href="https://wiki.owasp.org/index.php/Top_10_2013-A4-Insecure_Direct_Object_References">Insecure Direct Object Refrences</a>.</li>
+  <li>OWASP: <a href="https://owasp.org/www-project-web-security-testing-guide/latest/4-Web_Application_Security_Testing/05-Authorization_Testing/04-Testing_for_Insecure_Direct_Object_References">Testing for Insecure Direct Object References</a>.</li>
+  <li>Microsoft Learn: <a href="https://learn.microsoft.com/en-us/aspnet/core/security/authorization/resourcebased?view=aspnetcore-7.0">Resource-based authorization in ASP.NET Core</a>.</li>
+
+</references>
+</qhelp>

csharp/ql/src/Security Features/CWE-639/InsecureDirectObjectReference.ql

@@ -0,0 +1,20 @@
+/**
+ * @name Insecure Direct Object Reference
+ * @description Using user input to control which object is modified without
+ *              proper authorization checks allows an attacker to modify arbitrary objects.
+ * @kind problem
+ * @problem.severity warning
+ * @security-severity 7.5
+ * @precision medium
+ * @id cs/web/insecure-direct-object-reference
+ * @tags security
+ *       external/cwe-639
+ */
+
+import csharp
+import semmle.code.csharp.security.auth.InsecureDirectObjectReferenceQuery
+
+from ActionMethod m
+where hasInsecureDirectObjectReference(m)
+select m,
+  "This method may be missing authorization checks for which users can access the resource of the provided ID."

csharp/ql/src/Security Features/CWE-639/MVCExample.cs

@@ -0,0 +1,45 @@
+public class CommentController : Controller {
+    private readonly IAuthorizationService _authorizationService;
+    private readonly IDocumentRepository _commentRepository;
+
+    public CommentController(IAuthorizationService authorizationService,
+                              ICommentRepository commentRepository)
+    {
+        _authorizationService = authorizationService;
+        _commentRepository = commentRepository;
+    }
+
+    // BAD: Any user can access this.
+    public async Task<IActionResult> Edit1(int commentId, string text) {
+        Comment comment = _commentRepository.Find(commentId);
+        
+        comment.Text = text;
+
+        return View();
+    }
+
+    // GOOD: An authorization check is made.
+    public async Task<IActionResult> Edit2(int commentId, string text) {
+        Comment comment = _commentRepository.Find(commentId);
+        
+        var authResult = await _authorizationService.AuthorizeAsync(User, Comment, "EditPolicy");
+
+        if (authResult.Succeeded) {
+            comment.Text = text;
+            return View();
+        }
+        else {
+            return ForbidResult();
+        }
+    }
+
+    // GOOD: Only users with the `admin` role can access this method.
+    [Authorize(Roles="admin")]
+    public async Task<IActionResult> Edit3(int commentId, string text) {
+        Comment comment = _commentRepository.Find(commentId);
+        
+        comment.Text = text;
+
+        return View();
+    }
+}

csharp/ql/src/Security Features/CWE-639/WebFormsExample.cs

@@ -0,0 +1,15 @@
+    // BAD - Any user can access this method.
+    protected void btn1_Click(object sender, EventArgs e) {
+        string commentId = Request.QueryString["Id"];
+        Comment comment = getCommentById(commentId);
+        comment.Body = inputCommentBody.Text;
+    }
+
+    // GOOD - The user ID is verified.
+    protected void btn2_Click(object sender, EventArgs e) {
+        string commentId = Request.QueryString["Id"];
+        Comment comment = getCommentById(commentId);
+        if (comment.AuthorName == User.Identity.Name){
+            comment.Body = inputCommentBody.Text;
+        }
+    }

csharp/ql/src/Stubs/helpers.py

@@ -9,7 +9,7 @@ def run_cmd(cmd, msg="Failed to run command"):
         exit(1)
 
 
-def run_cmd_cwd(cmd, cwd, msg):
+def run_cmd_cwd(cmd, cwd, msg="Failed to run command"):
     print('Change working directory to: ' + cwd)
     print('Running ' + ' '.join(cmd))
     if subprocess.check_call(cmd, cwd=cwd):

csharp/ql/src/Stubs/make_stubs_nuget.py

@@ -18,19 +18,25 @@ def write_csproj_prefix(ioWrapper):
 
 print('Script to generate stub file from a nuget package')
 print(' Usage: python3 ' + sys.argv[0] +
-      ' NUGET_PACKAGE_NAME [VERSION=latest] [WORK_DIR=tempDir]')
+      ' TEMPLATE NUGET_PACKAGE_NAME [VERSION=latest] [WORK_DIR=tempDir]')
 print(' The script uses the dotnet cli, codeql cli, and dotnet format global tool')
+print(' TEMPLATE should be either classlib or webapp, depending on the nuget package. For example, `Swashbuckle.AspNetCore.Swagger` should use `webapp` while `newtonsoft.json` should use `classlib`.')
 
 if len(sys.argv) < 2:
+    print("\nPlease supply a template name.")
+    exit(1)
+
+if len(sys.argv) < 3:
     print("\nPlease supply a nuget package name.")
     exit(1)
 
 thisScript = sys.argv[0]
 thisDir = os.path.abspath(os.path.dirname(thisScript))
-nuget = sys.argv[1]
+template = sys.argv[1]
+nuget = sys.argv[2]
 
 # /input contains a dotnet project that's being extracted
-workDir = os.path.abspath(helpers.get_argv(3, "tempDir"))
+workDir = os.path.abspath(helpers.get_argv(4, "tempDir"))
 projectNameIn = "input"
 projectDirIn = os.path.join(workDir, projectNameIn)
 
@@ -57,10 +63,10 @@ outputName = "stub"
 outputFile = os.path.join(projectDirOut, outputName + '.cs')
 bqrsFile = os.path.join(rawOutputDir, outputName + '.bqrs')
 jsonFile = os.path.join(rawOutputDir, outputName + '.json')
-version = helpers.get_argv(2, "latest")
+version = helpers.get_argv(3, "latest")
 
 print("\n* Creating new input project")
-run_cmd(['dotnet', 'new', 'classlib', "-f", "net7.0", "--language", "C#", '--name',
+run_cmd(['dotnet', 'new', template, "-f", "net7.0", "--language", "C#", '--name',
                  projectNameIn, '--output', projectDirIn])
 helpers.remove_files(projectDirIn, '.cs')
 
@@ -75,36 +81,27 @@ sdk_version = '7.0.102'
 print("\n* Creating new global.json file and setting SDK to " + sdk_version)
 run_cmd(['dotnet', 'new', 'globaljson', '--force', '--sdk-version', sdk_version, '--output', workDir])
 
-print("\n* Creating DB")
-run_cmd(['codeql', 'database', 'create', dbDir, '--language=csharp',
-                 '--command', 'dotnet build /t:rebuild ' + projectDirIn])
-
-if not os.path.isdir(dbDir):
-    print("Expected database directory " + dbDir + " not found.")
-    exit(1)
-
-print("\n* Running stubbing CodeQL query")
-run_cmd(['codeql', 'query', 'run', os.path.join(
-    thisDir, 'AllStubsFromReference.ql'), '--database', dbDir, '--output', bqrsFile])
-
-run_cmd(['codeql', 'bqrs', 'decode', bqrsFile, '--output',
-                 jsonFile, '--format=json'])
+print("\n* Running stub generator")
+helpers.run_cmd_cwd(['dotnet', 'run', '--project', thisDir + '/../../../extractor/Semmle.Extraction.CSharp.DependencyStubGenerator/Semmle.Extraction.CSharp.DependencyStubGenerator.csproj'], projectDirIn)
 
 print("\n* Creating new raw output project")
 rawSrcOutputDirName = 'src'
 rawSrcOutputDir = os.path.join(rawOutputDir, rawSrcOutputDirName)
-run_cmd(['dotnet', 'new', 'classlib', "--language", "C#",
+run_cmd(['dotnet', 'new', template, "--language", "C#",
                 '--name', rawSrcOutputDirName, '--output', rawSrcOutputDir])
 helpers.remove_files(rawSrcOutputDir, '.cs')
 
-# load json from query result file and split it into separate .cs files
+# copy each file from projectDirIn to rawSrcOutputDir
 pathInfos = {}
-with open(jsonFile) as json_data:
-    data = json.load(json_data)
-    for row in data['#select']['tuples']:
-        pathInfos[row[3]] = os.path.join(rawSrcOutputDir, row[1] + '.cs')
-        with open(pathInfos[row[3]], 'a') as f:
-            f.write(row[4])
+codeqlStubsDir = os.path.join(projectDirIn, 'codeql_csharp_stubs')
+for root, dirs, files in os.walk(codeqlStubsDir):
+    for file in files:
+        if file.endswith('.cs'):
+            path = os.path.join(root, file)
+            relPath, _ = os.path.splitext(os.path.relpath(path, codeqlStubsDir))
+            origDllPath = "/" + relPath + ".dll"
+            pathInfos[origDllPath] = os.path.join(rawSrcOutputDir, file)
+            shutil.copy2(path, rawSrcOutputDir)
 
 print("\n --> Generated stub files: " + rawSrcOutputDir)
 

csharp/ql/src/Telemetry/ExternalApi.qll

@@ -1,34 +1,18 @@
 /** Provides classes and predicates related to handling APIs from external libraries. */
 
 private import csharp
-private import dotnet
 private import semmle.code.csharp.dispatch.Dispatch
 private import semmle.code.csharp.dataflow.ExternalFlow
 private import semmle.code.csharp.dataflow.FlowSummary
-private import semmle.code.csharp.dataflow.internal.DataFlowImplCommon as DataFlowImplCommon
 private import semmle.code.csharp.dataflow.internal.DataFlowPrivate
 private import semmle.code.csharp.dataflow.internal.DataFlowDispatch as DataFlowDispatch
 private import semmle.code.csharp.dataflow.internal.FlowSummaryImpl as FlowSummaryImpl
 private import semmle.code.csharp.dataflow.internal.TaintTrackingPrivate
 private import semmle.code.csharp.security.dataflow.flowsources.Remote
-
-pragma[nomagic]
-private predicate isTestNamespace(Namespace ns) {
-  ns.getFullName()
-      .matches([
-          "NUnit.Framework%", "Xunit%", "Microsoft.VisualStudio.TestTools.UnitTesting%", "Moq%"
-        ])
-}
-
-/**
- * A test library.
- */
-class TestLibrary extends RefType {
-  TestLibrary() { isTestNamespace(this.getNamespace()) }
-}
+private import TestLibrary
 
 /** Holds if the given callable is not worth supporting. */
-private predicate isUninteresting(DotNet::Callable c) {
+private predicate isUninteresting(Callable c) {
   c.getDeclaringType() instanceof TestLibrary or
   c.(Constructor).isParameterless()
 }
@@ -36,7 +20,7 @@ private predicate isUninteresting(DotNet::Callable c) {
 /**
  * An external API from either the C# Standard Library or a 3rd party library.
  */
-class ExternalApi extends DotNet::Callable {
+class ExternalApi extends Callable {
   ExternalApi() {
     this.isUnboundDeclaration() and
     this.fromLibrary() and
@@ -77,13 +61,11 @@ class ExternalApi extends DotNet::Callable {
 
   /** Gets a node that is an output from a call to this API. */
   private DataFlow::Node getAnOutput() {
-    exists(
-      Call c, DataFlowDispatch::NonDelegateDataFlowCall dc, DataFlowImplCommon::ReturnKindExt ret
-    |
+    exists(Call c, DataFlowDispatch::NonDelegateDataFlowCall dc |
       dc.getDispatchCall().getCall() = c and
       c.getTarget().getUnboundDeclaration() = this
     |
-      result = ret.getAnOutNode(dc)
+      result = DataFlowDispatch::getAnOutNode(dc, _)
     )
   }
 
@@ -119,18 +101,17 @@ class ExternalApi extends DotNet::Callable {
 }
 
 /**
- * Gets the nested name of the declaration.
+ * Gets the nested name of the type `t`.
  *
- * If the declaration is not a nested type, the result is the same as \`getName()\`.
+ * If the type is not a nested type, the result is the same as \`getName()\`.
  * Otherwise the name of the nested type is prefixed with a \`+\` and appended to
  * the name of the enclosing type, which might be a nested type as well.
  */
-private string nestedName(Declaration declaration) {
-  not exists(declaration.getDeclaringType().getUnboundDeclaration()) and
-  result = declaration.getName()
+private string nestedName(Type t) {
+  not exists(t.getDeclaringType().getUnboundDeclaration()) and
+  result = t.getName()
   or
-  nestedName(declaration.getDeclaringType().getUnboundDeclaration()) + "+" + declaration.getName() =
-    result
+  nestedName(t.getDeclaringType().getUnboundDeclaration()) + "+" + t.getName() = result
 }
 
 /**

csharp/ql/src/Telemetry/TestLibrary.qll

@@ -0,0 +1,17 @@
+private import csharp
+private import dotnet
+
+pragma[nomagic]
+private predicate isTestNamespace(Namespace ns) {
+  ns.getFullName()
+      .matches([
+          "NUnit.Framework%", "Xunit%", "Microsoft.VisualStudio.TestTools.UnitTesting%", "Moq%"
+        ])
+}
+
+/**
+ * A test library.
+ */
+class TestLibrary extends RefType {
+  TestLibrary() { isTestNamespace(this.getNamespace()) }
+}

csharp/ql/src/change-notes/2023-08-21-insecure-direct-object-reference.md

@@ -0,0 +1,4 @@
+---
+category: newQuery
+---
+* Added a new query, `cs/web/insecure-direct-object-reference`, to find instances of missing authorization checks for resources selected by an ID parameter.

csharp/ql/src/utils/modeleditor/ApplicationModeEndpoints.ql

@@ -0,0 +1,23 @@
+/**
+ * @name Fetch endpoints for use in the model editor (application mode)
+ * @description A list of 3rd party endpoints (methods and attributes) used in the codebase. Excludes test and generated code.
+ * @kind table
+ * @id csharp/utils/modeleditor/application-mode-endpoints
+ * @tags modeleditor endpoints application-mode
+ */
+
+import csharp
+import ApplicationModeEndpointsQuery
+import ModelEditor
+
+private Call aUsage(ExternalEndpoint api) { result.getTarget().getUnboundDeclaration() = api }
+
+from ExternalEndpoint endpoint, boolean supported, Call usage, string type, string classification
+where
+  supported = isSupported(endpoint) and
+  usage = aUsage(endpoint) and
+  type = supportedType(endpoint) and
+  classification = methodClassification(usage)
+select usage, endpoint.getNamespace(), endpoint.getTypeName(), endpoint.getName(),
+  endpoint.getParameterTypes(), supported, endpoint.dllName(), endpoint.dllVersion(), type,
+  classification

csharp/ql/src/utils/modeleditor/ApplicationModeEndpointsQuery.qll

@@ -0,0 +1,45 @@
+private import csharp
+private import semmle.code.csharp.dataflow.ExternalFlow
+private import semmle.code.csharp.dataflow.internal.DataFlowDispatch as DataFlowDispatch
+private import semmle.code.csharp.dataflow.internal.DataFlowPrivate
+private import semmle.code.csharp.dataflow.internal.TaintTrackingPrivate
+private import semmle.code.csharp.security.dataflow.flowsources.Remote
+private import ModelEditor
+
+/**
+ * A class of effectively public callables in library code.
+ */
+class ExternalEndpoint extends Endpoint {
+  ExternalEndpoint() { this.fromLibrary() }
+
+  /** Gets a node that is an input to a call to this API. */
+  private ArgumentNode getAnInput() {
+    result
+        .getCall()
+        .(DataFlowDispatch::NonDelegateDataFlowCall)
+        .getATarget(_)
+        .getUnboundDeclaration() = this
+  }
+
+  /** Gets a node that is an output from a call to this API. */
+  private DataFlow::Node getAnOutput() {
+    exists(Call c, DataFlowDispatch::NonDelegateDataFlowCall dc |
+      dc.getDispatchCall().getCall() = c and
+      c.getTarget().getUnboundDeclaration() = this
+    |
+      result = DataFlowDispatch::getAnOutNode(dc, _)
+    )
+  }
+
+  override predicate hasSummary() {
+    Endpoint.super.hasSummary()
+    or
+    defaultAdditionalTaintStep(this.getAnInput(), _)
+  }
+
+  override predicate isSource() {
+    this.getAnOutput() instanceof RemoteFlowSource or sourceNode(this.getAnOutput(), _)
+  }
+
+  override predicate isSink() { sinkNode(this.getAnInput(), _) }
+}