JS: Address review comments

C#: Fully qualified name.

.bazelrc

@@ -11,7 +11,7 @@ common --override_module=semmle_code=%workspace%/misc/bazel/semmle_code_stub
 build --repo_env=CC=clang --repo_env=CXX=clang++
 
 build:linux --cxxopt=-std=c++20
-build:macos --cxxopt=-std=c++20 --platforms=@apple_support//platforms:macos_x86_64
+build:macos --cxxopt=-std=c++20 --cpu=darwin_x86_64
 build:windows --cxxopt=/std:c++20 --cxxopt=/Zc:preprocessor
 
 try-import %workspace%/local.bazelrc

.github/workflows/swift.yml

@@ -17,28 +17,34 @@ on:
       - main
       - rc/*
       - codeql-cli-*
+  push:
+    paths:
+      - "swift/**"
+      - "misc/bazel/**"
+      - "misc/codegen/**"
+      - "*.bazel*"
+      - .github/workflows/swift.yml
+      - .github/actions/**
+      - codeql-workspace.yml
+      - "!**/*.md"
+      - "!**/*.qhelp"
+    branches:
+      - main
+      - rc/*
+      - codeql-cli-*
 
 permissions:
   contents: read
 
 jobs:
-  # not putting using a matrix as you cannot depend on a specific job in a matrix, and we want to start qltest and
-  # integration tests as soon as the corresponding build required has finished, without waiting for the slowest macOS
-  # build
-  build-and-test-macos-intel:
+  # not using a matrix as you cannot depend on a specific job in a matrix, and we want to start linux checks
+  # without waiting for the macOS build
+  build-and-test-macos:
     if: github.repository_owner == 'github'
-    runs-on: macos-14-large
+    runs-on: macos-12-xl
     steps:
       - uses: actions/checkout@v4
       - uses: ./swift/actions/build-and-test
-  build-and-test-macos-arm:
-    if: github.repository_owner == 'github'
-    runs-on: macos-14-xlarge
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./swift/actions/build-and-test
-        with:
-          upload: false  # use intel build for further tests
   build-and-test-linux:
     if: github.repository_owner == 'github'
     runs-on: ubuntu-latest-xl
@@ -54,11 +60,26 @@ jobs:
       - uses: ./swift/actions/run-ql-tests
   qltests-macos:
     if: ${{ github.repository_owner == 'github' && github.event_name == 'pull_request' }}
-    needs: build-and-test-macos-intel
-    runs-on: macos-14-large
+    needs: build-and-test-macos
+    runs-on: macos-12-xl
     steps:
       - uses: actions/checkout@v4
       - uses: ./swift/actions/run-ql-tests
+  integration-tests-linux:
+    if: github.repository_owner == 'github'
+    needs: build-and-test-linux
+    runs-on: ubuntu-latest-xl
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./swift/actions/run-integration-tests
+  integration-tests-macos:
+    if: ${{ github.repository_owner == 'github' && github.event_name == 'pull_request' }}
+    needs: build-and-test-macos
+    runs-on: macos-12-xl
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./swift/actions/run-integration-tests
   clang-format:
     if : ${{ github.event_name == 'pull_request' }}
     runs-on: ubuntu-latest

MODULE.bazel

@@ -12,7 +12,7 @@ local_path_override(
 )
 
 # see https://registry.bazel.build/ for a list of available packages
-bazel_dep(name = "apple_support", version = "1.14.0")
+
 bazel_dep(name = "platforms", version = "0.0.8")
 bazel_dep(name = "rules_pkg", version = "0.9.1")
 bazel_dep(name = "rules_nodejs", version = "6.0.3")

config/identical-files.json

@@ -251,12 +251,6 @@
     "cpp/ql/src/Security/CWE/CWE-020/SafeExternalAPIFunction.qll",
     "cpp/ql/src/Security/CWE/CWE-020/ir/SafeExternalAPIFunction.qll"
   ],
-  "XML": [
-    "cpp/ql/lib/semmle/code/cpp/XML.qll",
-    "csharp/ql/lib/semmle/code/csharp/XML.qll",
-    "java/ql/lib/semmle/code/xml/XML.qll",
-    "python/ql/lib/semmle/python/xml/XML.qll"
-  ],
   "DuplicationProblems.inc.qhelp": [
     "cpp/ql/src/Metrics/Files/DuplicationProblems.inc.qhelp",
     "javascript/ql/src/Metrics/DuplicationProblems.inc.qhelp",

cpp/autobuilder/Semmle.Autobuild.Cpp.Tests/BuildScripts.cs

@@ -203,6 +203,8 @@ namespace Semmle.Autobuild.Cpp.Tests
         public IList<DiagnosticMessage> Diagnostics { get; } = new List<DiagnosticMessage>();
 
         public void AddEntry(DiagnosticMessage message) => this.Diagnostics.Add(message);
+
+        public void Dispose() { }
     }
 
     /// <summary>

cpp/autobuilder/Semmle.Autobuild.Cpp/Program.cs

@@ -17,7 +17,7 @@ namespace Semmle.Autobuild.Cpp
                 try
                 {
                     Console.WriteLine("CodeQL C++ autobuilder");
-                    var builder = new CppAutobuilder(actions, options);
+                    using var builder = new CppAutobuilder(actions, options);
                     return builder.AttemptBuild();
                 }
                 catch (InvalidEnvironmentException ex)

cpp/ql/lib/qlpack.yml

@@ -11,4 +11,5 @@ dependencies:
   codeql/ssa: ${workspace}
   codeql/tutorial: ${workspace}
   codeql/util: ${workspace}
+  codeql/xml: ${workspace}
 warnOnImplicitThis: true

cpp/ql/lib/semmle/code/cpp/XML.qll

@@ -3,305 +3,67 @@
  */
 
 import semmle.files.FileSystem
+private import codeql.xml.Xml
 
-private class TXmlLocatable =
-  @xmldtd or @xmlelement or @xmlattribute or @xmlnamespace or @xmlcomment or @xmlcharacters;
+private module Input implements InputSig<File, Location> {
+  class XmlLocatableBase = @xmllocatable or @xmlnamespaceable;
 
-/** An XML element that has a location. */
-class XmlLocatable extends @xmllocatable, TXmlLocatable {
-  /** Gets the source location for this element. */
-  Location getLocation() { xmllocations(this, result) }
+  predicate xmllocations_(XmlLocatableBase e, Location loc) { xmllocations(e, loc) }
 
-  /**
-   * Holds if this element is at the specified location.
-   * The location spans column `startcolumn` of line `startline` to
-   * column `endcolumn` of line `endline` in file `filepath`.
-   * For more information, see
-   * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
-   */
-  predicate hasLocationInfo(
-    string filepath, int startline, int startcolumn, int endline, int endcolumn
+  class XmlParentBase = @xmlparent;
+
+  class XmlNamespaceableBase = @xmlnamespaceable;
+
+  class XmlElementBase = @xmlelement;
+
+  class XmlFileBase = File;
+
+  predicate xmlEncoding_(XmlFileBase f, string enc) { xmlEncoding(f, enc) }
+
+  class XmlDtdBase = @xmldtd;
+
+  predicate xmlDTDs_(XmlDtdBase e, string root, string publicId, string systemId, XmlFileBase file) {
+    xmlDTDs(e, root, publicId, systemId, file)
+  }
+
+  predicate xmlElements_(
+    XmlElementBase e, string name, XmlParentBase parent, int idx, XmlFileBase file
   ) {
-    exists(File f, Location l | l = this.getLocation() |
-      locations_default(l, f, startline, startcolumn, endline, endcolumn) and
-      filepath = f.getAbsolutePath()
-    )
+    xmlElements(e, name, parent, idx, file)
   }
 
-  /** Gets a textual representation of this element. */
-  string toString() { none() } // overridden in subclasses
-}
+  class XmlAttributeBase = @xmlattribute;
 
-/**
- * An `XmlParent` is either an `XmlElement` or an `XmlFile`,
- * both of which can contain other elements.
- */
-class XmlParent extends @xmlparent {
-  XmlParent() {
-    // explicitly restrict `this` to be either an `XmlElement` or an `XmlFile`;
-    // the type `@xmlparent` currently also includes non-XML files
-    this instanceof @xmlelement or xmlEncoding(this, _)
+  predicate xmlAttrs_(
+    XmlAttributeBase e, XmlElementBase elementid, string name, string value, int idx,
+    XmlFileBase file
+  ) {
+    xmlAttrs(e, elementid, name, value, idx, file)
   }
 
-  /**
-   * Gets a printable representation of this XML parent.
-   * (Intended to be overridden in subclasses.)
-   */
-  string getName() { none() } // overridden in subclasses
+  class XmlNamespaceBase = @xmlnamespace;
 
-  /** Gets the file to which this XML parent belongs. */
-  XmlFile getFile() { result = this or xmlElements(this, _, _, _, result) }
-
-  /** Gets the child element at a specified index of this XML parent. */
-  XmlElement getChild(int index) { xmlElements(result, _, this, index, _) }
-
-  /** Gets a child element of this XML parent. */
-  XmlElement getAChild() { xmlElements(result, _, this, _, _) }
-
-  /** Gets a child element of this XML parent with the given `name`. */
-  XmlElement getAChild(string name) { xmlElements(result, _, this, _, _) and result.hasName(name) }
-
-  /** Gets a comment that is a child of this XML parent. */
-  XmlComment getAComment() { xmlComments(result, _, this, _) }
-
-  /** Gets a character sequence that is a child of this XML parent. */
-  XmlCharacters getACharactersSet() { xmlChars(result, _, this, _, _, _) }
-
-  /** Gets the depth in the tree. (Overridden in XmlElement.) */
-  int getDepth() { result = 0 }
-
-  /** Gets the number of child XML elements of this XML parent. */
-  int getNumberOfChildren() { result = count(XmlElement e | xmlElements(e, _, this, _, _)) }
-
-  /** Gets the number of places in the body of this XML parent where text occurs. */
-  int getNumberOfCharacterSets() { result = count(int pos | xmlChars(_, _, this, pos, _, _)) }
-
-  /**
-   * Gets the result of appending all the character sequences of this XML parent from
-   * left to right, separated by a space.
-   */
-  string allCharactersString() {
-    result =
-      concat(string chars, int pos | xmlChars(_, chars, this, pos, _, _) | chars, " " order by pos)
+  predicate xmlNs_(XmlNamespaceBase e, string prefixName, string uri, XmlFileBase file) {
+    xmlNs(e, prefixName, uri, file)
   }
 
-  /** Gets the text value contained in this XML parent. */
-  string getTextValue() { result = this.allCharactersString() }
+  predicate xmlHasNs_(XmlNamespaceableBase e, XmlNamespaceBase ns, XmlFileBase file) {
+    xmlHasNs(e, ns, file)
+  }
 
-  /** Gets a printable representation of this XML parent. */
-  string toString() { result = this.getName() }
-}
+  class XmlCommentBase = @xmlcomment;
 
-/** An XML file. */
-class XmlFile extends XmlParent, File {
-  XmlFile() { xmlEncoding(this, _) }
+  predicate xmlComments_(XmlCommentBase e, string text, XmlParentBase parent, XmlFileBase file) {
+    xmlComments(e, text, parent, file)
+  }
 
-  /** Gets a printable representation of this XML file. */
-  override string toString() { result = this.getName() }
+  class XmlCharactersBase = @xmlcharacters;
 
-  /** Gets the name of this XML file. */
-  override string getName() { result = File.super.getAbsolutePath() }
-
-  /** Gets the encoding of this XML file. */
-  string getEncoding() { xmlEncoding(this, result) }
-
-  /** Gets the XML file itself. */
-  override XmlFile getFile() { result = this }
-
-  /** Gets a top-most element in an XML file. */
-  XmlElement getARootElement() { result = this.getAChild() }
-
-  /** Gets a DTD associated with this XML file. */
-  XmlDtd getADtd() { xmlDTDs(result, _, _, _, this) }
-}
-
-/**
- * An XML document type definition (DTD).
- *
- * Example:
- *
- * ```
- * <!ELEMENT person (firstName, lastName?)>
- * <!ELEMENT firstName (#PCDATA)>
- * <!ELEMENT lastName (#PCDATA)>
- * ```
- */
-class XmlDtd extends XmlLocatable, @xmldtd {
-  /** Gets the name of the root element of this DTD. */
-  string getRoot() { xmlDTDs(this, result, _, _, _) }
-
-  /** Gets the public ID of this DTD. */
-  string getPublicId() { xmlDTDs(this, _, result, _, _) }
-
-  /** Gets the system ID of this DTD. */
-  string getSystemId() { xmlDTDs(this, _, _, result, _) }
-
-  /** Holds if this DTD is public. */
-  predicate isPublic() { not xmlDTDs(this, _, "", _, _) }
-
-  /** Gets the parent of this DTD. */
-  XmlParent getParent() { xmlDTDs(this, _, _, _, result) }
-
-  override string toString() {
-    this.isPublic() and
-    result = this.getRoot() + " PUBLIC '" + this.getPublicId() + "' '" + this.getSystemId() + "'"
-    or
-    not this.isPublic() and
-    result = this.getRoot() + " SYSTEM '" + this.getSystemId() + "'"
+  predicate xmlChars_(
+    XmlCharactersBase e, string text, XmlParentBase parent, int idx, int isCDATA, XmlFileBase file
+  ) {
+    xmlChars(e, text, parent, idx, isCDATA, file)
   }
 }
 
-/**
- * An XML element in an XML file.
- *
- * Example:
- *
- * ```
- * <manifest xmlns:android="http://schemas.android.com/apk/res/android"
- *           package="com.example.exampleapp" android:versionCode="1">
- * </manifest>
- * ```
- */
-class XmlElement extends @xmlelement, XmlParent, XmlLocatable {
-  /** Holds if this XML element has the given `name`. */
-  predicate hasName(string name) { name = this.getName() }
-
-  /** Gets the name of this XML element. */
-  override string getName() { xmlElements(this, result, _, _, _) }
-
-  /** Gets the XML file in which this XML element occurs. */
-  override XmlFile getFile() { xmlElements(this, _, _, _, result) }
-
-  /** Gets the parent of this XML element. */
-  XmlParent getParent() { xmlElements(this, _, result, _, _) }
-
-  /** Gets the index of this XML element among its parent's children. */
-  int getIndex() { xmlElements(this, _, _, result, _) }
-
-  /** Holds if this XML element has a namespace. */
-  predicate hasNamespace() { xmlHasNs(this, _, _) }
-
-  /** Gets the namespace of this XML element, if any. */
-  XmlNamespace getNamespace() { xmlHasNs(this, result, _) }
-
-  /** Gets the index of this XML element among its parent's children. */
-  int getElementPositionIndex() { xmlElements(this, _, _, result, _) }
-
-  /** Gets the depth of this element within the XML file tree structure. */
-  override int getDepth() { result = this.getParent().getDepth() + 1 }
-
-  /** Gets an XML attribute of this XML element. */
-  XmlAttribute getAnAttribute() { result.getElement() = this }
-
-  /** Gets the attribute with the specified `name`, if any. */
-  XmlAttribute getAttribute(string name) { result.getElement() = this and result.getName() = name }
-
-  /** Holds if this XML element has an attribute with the specified `name`. */
-  predicate hasAttribute(string name) { exists(this.getAttribute(name)) }
-
-  /** Gets the value of the attribute with the specified `name`, if any. */
-  string getAttributeValue(string name) { result = this.getAttribute(name).getValue() }
-
-  /** Gets a printable representation of this XML element. */
-  override string toString() { result = this.getName() }
-}
-
-/**
- * An attribute that occurs inside an XML element.
- *
- * Examples:
- *
- * ```
- * package="com.example.exampleapp"
- * android:versionCode="1"
- * ```
- */
-class XmlAttribute extends @xmlattribute, XmlLocatable {
-  /** Gets the name of this attribute. */
-  string getName() { xmlAttrs(this, _, result, _, _, _) }
-
-  /** Gets the XML element to which this attribute belongs. */
-  XmlElement getElement() { xmlAttrs(this, result, _, _, _, _) }
-
-  /** Holds if this attribute has a namespace. */
-  predicate hasNamespace() { xmlHasNs(this, _, _) }
-
-  /** Gets the namespace of this attribute, if any. */
-  XmlNamespace getNamespace() { xmlHasNs(this, result, _) }
-
-  /** Gets the value of this attribute. */
-  string getValue() { xmlAttrs(this, _, _, result, _, _) }
-
-  /** Gets a printable representation of this XML attribute. */
-  override string toString() { result = this.getName() + "=" + this.getValue() }
-}
-
-/**
- * A namespace used in an XML file.
- *
- * Example:
- *
- * ```
- * xmlns:android="http://schemas.android.com/apk/res/android"
- * ```
- */
-class XmlNamespace extends XmlLocatable, @xmlnamespace {
-  /** Gets the prefix of this namespace. */
-  string getPrefix() { xmlNs(this, result, _, _) }
-
-  /** Gets the URI of this namespace. */
-  string getUri() { xmlNs(this, _, result, _) }
-
-  /** Holds if this namespace has no prefix. */
-  predicate isDefault() { this.getPrefix() = "" }
-
-  override string toString() {
-    this.isDefault() and result = this.getUri()
-    or
-    not this.isDefault() and result = this.getPrefix() + ":" + this.getUri()
-  }
-}
-
-/**
- * A comment in an XML file.
- *
- * Example:
- *
- * ```
- * <!-- This is a comment. -->
- * ```
- */
-class XmlComment extends @xmlcomment, XmlLocatable {
-  /** Gets the text content of this XML comment. */
-  string getText() { xmlComments(this, result, _, _) }
-
-  /** Gets the parent of this XML comment. */
-  XmlParent getParent() { xmlComments(this, _, result, _) }
-
-  /** Gets a printable representation of this XML comment. */
-  override string toString() { result = this.getText() }
-}
-
-/**
- * A sequence of characters that occurs between opening and
- * closing tags of an XML element, excluding other elements.
- *
- * Example:
- *
- * ```
- * <content>This is a sequence of characters.</content>
- * ```
- */
-class XmlCharacters extends @xmlcharacters, XmlLocatable {
-  /** Gets the content of this character sequence. */
-  string getCharacters() { xmlChars(this, result, _, _, _, _) }
-
-  /** Gets the parent of this character sequence. */
-  XmlParent getParent() { xmlChars(this, _, result, _, _, _) }
-
-  /** Holds if this character sequence is CDATA. */
-  predicate isCDATA() { xmlChars(this, _, _, _, 1, _) }
-
-  /** Gets a printable representation of this XML character sequence. */
-  override string toString() { result = this.getCharacters() }
-}
+import Make<File, Location, Input>

cpp/ql/lib/semmle/code/cpp/models/Models.qll

@@ -41,3 +41,4 @@ private import implementations.SqLite3
 private import implementations.PostgreSql
 private import implementations.System
 private import implementations.StructuredExceptionHandling
+private import implementations.Fopen

cpp/ql/lib/semmle/code/cpp/models/implementations/Fopen.qll

@@ -0,0 +1,50 @@
+/**
+ * Provides implementation classes modeling `fopen` and various similar
+ * functions. See `semmle.code.cpp.models.Models` for usage information.
+ */
+
+import semmle.code.cpp.models.interfaces.Alias
+import semmle.code.cpp.models.interfaces.SideEffect
+
+/** The function `fopen` and friends. */
+private class Fopen extends Function, AliasFunction, SideEffectFunction {
+  Fopen() {
+    this.hasGlobalOrStdName(["fopen", "fopen_s", "freopen"])
+    or
+    this.hasGlobalName(["_open", "_wfopen", "_fsopen", "_wfsopen", "_wopen"])
+  }
+
+  override predicate hasOnlySpecificWriteSideEffects() { any() }
+
+  override predicate hasOnlySpecificReadSideEffects() { any() }
+
+  override predicate parameterEscapesOnlyViaReturn(int i) { none() }
+
+  override predicate parameterNeverEscapes(int index) {
+    // None of the parameters escape
+    this.getParameter(index).getUnspecifiedType() instanceof PointerType
+  }
+
+  override predicate hasSpecificReadSideEffect(ParameterIndex i, boolean buffer) {
+    (
+      this.hasGlobalOrStdName(["fopen", "fopen_s"])
+      or
+      this.hasGlobalName(["_wfopen", "_fsopen", "_wfsopen"])
+    ) and
+    i = [0, 1] and
+    buffer = true
+    or
+    this.hasGlobalOrStdName("freopen") and
+    (
+      i = [0, 1] and
+      buffer = true
+      or
+      i = 2 and
+      buffer = false
+    )
+    or
+    this.hasGlobalName(["_open", "_wopen"]) and
+    i = 0 and
+    buffer = true
+  }
+}

cpp/ql/test/library-tests/ir/ir/ir.cpp

@@ -1933,6 +1933,20 @@ namespace missing_declaration_entries {
         Bar2<int> b;
         b.two_missing_variable_declaration_entries();
     }
+
+    template<typename T> struct Bar3 {
+
+        int two_more_missing_variable_declaration_entries() {
+            extern int g;
+            int z(float);
+            return g;
+        }
+    };
+
+    void test3() {
+        Bar3<int> b;
+        b.two_more_missing_variable_declaration_entries();
+    }
 }
 
 template<typename T> T global_template = 42;
@@ -2417,4 +2431,20 @@ void initialization_with_temp_destructor() {
         y += x;
 }
 
+void param_with_destructor_by_value(ClassWithDestructor c) {
+    // The call to ~ClassWithDestructor::ClassWithDestructor() seems to be missing here.
+}
+
+void param_with_destructor_by_pointer(ClassWithDestructor* c) {
+    // No destructor call should be here
+}
+
+void param_with_destructor_by_ref(ClassWithDestructor& c) {
+    // No destructor call should be here
+}
+
+void param_with_destructor_by_rref(ClassWithDestructor&& c) {
+    // No destructor call should be here
+}
+
 // semmle-extractor-options: -std=c++20 --clang

csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs

@@ -218,6 +218,8 @@ namespace Semmle.Autobuild.CSharp.Tests
         public IList<DiagnosticMessage> Diagnostics { get; } = new List<DiagnosticMessage>();
 
         public void AddEntry(DiagnosticMessage message) => this.Diagnostics.Add(message);
+
+        public void Dispose() { }
     }
 
     /// <summary>

csharp/autobuilder/Semmle.Autobuild.CSharp/DotNetRule.cs

@@ -32,7 +32,7 @@ namespace Semmle.Autobuild.CSharp
             if (auto)
             {
                 NotDotNetProjects = builder.ProjectsOrSolutionsToBuild
-                    .SelectMany(p => Enumerators.Singleton(p).Concat(p.IncludedProjects))
+                    .SelectMany(p => new[] { p }.Concat(p.IncludedProjects))
                     .OfType<Project<CSharpAutobuildOptions>>()
                     .Where(p => !p.DotNetProject);
                 var notDotNetProject = NotDotNetProjects.FirstOrDefault();

csharp/autobuilder/Semmle.Autobuild.CSharp/Program.cs

@@ -17,7 +17,7 @@ namespace Semmle.Autobuild.CSharp
                 try
                 {
                     Console.WriteLine("CodeQL C# autobuilder");
-                    var builder = new CSharpAutobuilder(actions, options);
+                    using var builder = new CSharpAutobuilder(actions, options);
                     return builder.AttemptBuild();
                 }
                 catch (InvalidEnvironmentException ex)

csharp/autobuilder/Semmle.Autobuild.Shared/Autobuilder.cs

@@ -92,7 +92,7 @@ namespace Semmle.Autobuild.Shared
     /// The overall design is intended to be extensible so that in theory,
     /// it should be possible to add new build rules without touching this code.
     /// </summary>
-    public abstract class Autobuilder<TAutobuildOptions> : IAutobuilder<TAutobuildOptions> where TAutobuildOptions : AutobuildOptionsShared
+    public abstract class Autobuilder<TAutobuildOptions> : IDisposable, IAutobuilder<TAutobuildOptions> where TAutobuildOptions : AutobuildOptionsShared
     {
         /// <summary>
         /// Full file paths of files found in the project directory, as well as
@@ -351,6 +351,20 @@ namespace Semmle.Autobuild.Shared
             }
         });
 
+        public void Dispose()
+        {
+            Dispose(true);
+            GC.SuppressFinalize(this);
+        }
+
+        protected virtual void Dispose(bool disposing)
+        {
+            if (disposing)
+            {
+                diagnostics.Dispose();
+            }
+        }
+
         /// <summary>
         /// Value of CODEQL_EXTRACTOR_<LANG>_ROOT environment variable.
         /// </summary>

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs

@@ -0,0 +1,514 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Net.Http;
+using System.Text.RegularExpressions;
+using System.Threading;
+using System.Threading.Tasks;
+using Semmle.Util;
+
+namespace Semmle.Extraction.CSharp.DependencyFetching
+{
+    public sealed partial class DependencyManager
+    {
+        private void RestoreNugetPackages(List<FileInfo> allNonBinaryFiles, IEnumerable<string> allProjects, IEnumerable<string> allSolutions, HashSet<string> dllPaths)
+        {
+            try
+            {
+                var checkNugetFeedResponsiveness = EnvironmentVariables.GetBoolean(EnvironmentVariableNames.CheckNugetFeedResponsiveness);
+                if (checkNugetFeedResponsiveness && !CheckFeeds(allNonBinaryFiles))
+                {
+                    DownloadMissingPackages(allNonBinaryFiles, dllPaths, withNugetConfig: false);
+                    return;
+                }
+
+                using (var nuget = new NugetPackages(sourceDir.FullName, legacyPackageDirectory, logger))
+                {
+                    var count = nuget.InstallPackages();
+
+                    if (nuget.PackageCount > 0)
+                    {
+                        CompilationInfos.Add(("packages.config files", nuget.PackageCount.ToString()));
+                        CompilationInfos.Add(("Successfully restored packages.config files", count.ToString()));
+                    }
+                }
+
+                var nugetPackageDlls = legacyPackageDirectory.DirInfo.GetFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = true });
+                var nugetPackageDllPaths = nugetPackageDlls.Select(f => f.FullName).ToHashSet();
+                var excludedPaths = nugetPackageDllPaths
+                    .Where(path => IsPathInSubfolder(path, legacyPackageDirectory.DirInfo.FullName, "tools"))
+                    .ToList();
+
+                if (nugetPackageDllPaths.Count > 0)
+                {
+                    logger.LogInfo($"Restored {nugetPackageDllPaths.Count} Nuget DLLs.");
+                }
+                if (excludedPaths.Count > 0)
+                {
+                    logger.LogInfo($"Excluding {excludedPaths.Count} Nuget DLLs.");
+                }
+
+                foreach (var excludedPath in excludedPaths)
+                {
+                    logger.LogInfo($"Excluded Nuget DLL: {excludedPath}");
+                }
+
+                nugetPackageDllPaths.ExceptWith(excludedPaths);
+                dllPaths.UnionWith(nugetPackageDllPaths);
+            }
+            catch (Exception exc)
+            {
+                logger.LogError($"Failed to restore Nuget packages with nuget.exe: {exc.Message}");
+            }
+
+            var restoredProjects = RestoreSolutions(allSolutions, out var assets1);
+            var projects = allProjects.Except(restoredProjects);
+            RestoreProjects(projects, out var assets2);
+
+            var dependencies = Assets.GetCompilationDependencies(logger, assets1.Union(assets2));
+
+            var paths = dependencies
+                .Paths
+                .Select(d => Path.Combine(packageDirectory.DirInfo.FullName, d))
+                .ToList();
+            dllPaths.UnionWith(paths);
+
+            LogAllUnusedPackages(dependencies);
+            DownloadMissingPackages(allNonBinaryFiles, dllPaths);
+        }
+
+        /// <summary>
+        /// Executes `dotnet restore` on all solution files in solutions.
+        /// As opposed to RestoreProjects this is not run in parallel using PLINQ
+        /// as `dotnet restore` on a solution already uses multiple threads for restoring
+        /// the projects (this can be disabled with the `--disable-parallel` flag).
+        /// Populates assets with the relative paths to the assets files generated by the restore.
+        /// Returns a list of projects that are up to date with respect to restore.
+        /// </summary>
+        /// <param name="solutions">A list of paths to solution files.</param>
+        private IEnumerable<string> RestoreSolutions(IEnumerable<string> solutions, out IEnumerable<string> assets)
+        {
+            var successCount = 0;
+            var nugetSourceFailures = 0;
+            var assetFiles = new List<string>();
+            var projects = solutions.SelectMany(solution =>
+                {
+                    logger.LogInfo($"Restoring solution {solution}...");
+                    var res = dotnet.Restore(new(solution, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
+                    if (res.Success)
+                    {
+                        successCount++;
+                    }
+                    if (res.HasNugetPackageSourceError)
+                    {
+                        nugetSourceFailures++;
+                    }
+                    assetFiles.AddRange(res.AssetsFilePaths);
+                    return res.RestoredProjects;
+                }).ToList();
+            assets = assetFiles;
+            CompilationInfos.Add(("Successfully restored solution files", successCount.ToString()));
+            CompilationInfos.Add(("Failed solution restore with package source error", nugetSourceFailures.ToString()));
+            CompilationInfos.Add(("Restored projects through solution files", projects.Count.ToString()));
+            return projects;
+        }
+
+        /// <summary>
+        /// Executes `dotnet restore` on all projects in projects.
+        /// This is done in parallel for performance reasons.
+        /// Populates assets with the relative paths to the assets files generated by the restore.
+        /// </summary>
+        /// <param name="projects">A list of paths to project files.</param>
+        private void RestoreProjects(IEnumerable<string> projects, out IEnumerable<string> assets)
+        {
+            var successCount = 0;
+            var nugetSourceFailures = 0;
+            var assetFiles = new List<string>();
+            var sync = new object();
+            Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = threads }, project =>
+            {
+                logger.LogInfo($"Restoring project {project}...");
+                var res = dotnet.Restore(new(project, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
+                lock (sync)
+                {
+                    if (res.Success)
+                    {
+                        successCount++;
+                    }
+                    if (res.HasNugetPackageSourceError)
+                    {
+                        nugetSourceFailures++;
+                    }
+                    assetFiles.AddRange(res.AssetsFilePaths);
+                }
+            });
+            assets = assetFiles;
+            CompilationInfos.Add(("Successfully restored project files", successCount.ToString()));
+            CompilationInfos.Add(("Failed project restore with package source error", nugetSourceFailures.ToString()));
+        }
+
+        private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPaths, bool withNugetConfig = true)
+        {
+            var alreadyDownloadedPackages = GetRestoredPackageDirectoryNames(packageDirectory.DirInfo);
+            var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames();
+
+            var notYetDownloadedPackages = new HashSet<PackageReference>(fileContent.AllPackages);
+            foreach (var alreadyDownloadedPackage in alreadyDownloadedPackages)
+            {
+                notYetDownloadedPackages.Remove(new(alreadyDownloadedPackage, PackageReferenceSource.SdkCsProj));
+            }
+            foreach (var alreadyDownloadedLegacyPackage in alreadyDownloadedLegacyPackages)
+            {
+                notYetDownloadedPackages.Remove(new(alreadyDownloadedLegacyPackage, PackageReferenceSource.PackagesConfig));
+            }
+
+            if (notYetDownloadedPackages.Count == 0)
+            {
+                return;
+            }
+
+            var multipleVersions = notYetDownloadedPackages
+                .GroupBy(p => p.Name)
+                .Where(g => g.Count() > 1)
+                .Select(g => g.Key)
+                .ToList();
+
+            foreach (var package in multipleVersions)
+            {
+                logger.LogWarning($"Found multiple not yet restored packages with name '{package}'.");
+                notYetDownloadedPackages.Remove(new(package, PackageReferenceSource.PackagesConfig));
+            }
+
+            logger.LogInfo($"Found {notYetDownloadedPackages.Count} packages that are not yet restored");
+            var nugetConfig = withNugetConfig
+                ? GetNugetConfig(allFiles)
+                : null;
+
+            CompilationInfos.Add(("Fallback nuget restore", notYetDownloadedPackages.Count.ToString()));
+
+            var successCount = 0;
+            var sync = new object();
+
+            Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = threads }, package =>
+            {
+                var success = TryRestorePackageManually(package.Name, nugetConfig, package.PackageReferenceSource);
+                if (!success)
+                {
+                    return;
+                }
+
+                lock (sync)
+                {
+                    successCount++;
+                }
+            });
+
+            CompilationInfos.Add(("Successfully ran fallback nuget restore", successCount.ToString()));
+
+            dllPaths.Add(missingPackageDirectory.DirInfo.FullName);
+        }
+
+        private string[] GetAllNugetConfigs(List<FileInfo> allFiles) => allFiles.SelectFileNamesByName("nuget.config").ToArray();
+
+        private string? GetNugetConfig(List<FileInfo> allFiles)
+        {
+            var nugetConfigs = GetAllNugetConfigs(allFiles);
+            string? nugetConfig;
+            if (nugetConfigs.Length > 1)
+            {
+                logger.LogInfo($"Found multiple nuget.config files: {string.Join(", ", nugetConfigs)}.");
+                nugetConfig = allFiles
+                    .SelectRootFiles(sourceDir)
+                    .SelectFileNamesByName("nuget.config")
+                    .FirstOrDefault();
+                if (nugetConfig == null)
+                {
+                    logger.LogInfo("Could not find a top-level nuget.config file.");
+                }
+            }
+            else
+            {
+                nugetConfig = nugetConfigs.FirstOrDefault();
+            }
+
+            if (nugetConfig != null)
+            {
+                logger.LogInfo($"Using nuget.config file {nugetConfig}.");
+            }
+
+            return nugetConfig;
+        }
+
+        private void LogAllUnusedPackages(DependencyContainer dependencies)
+        {
+            var allPackageDirectories = GetAllPackageDirectories();
+
+            logger.LogInfo($"Restored {allPackageDirectories.Count} packages");
+            logger.LogInfo($"Found {dependencies.Packages.Count} packages in project.assets.json files");
+
+            allPackageDirectories
+                .Where(package => !dependencies.Packages.Contains(package))
+                .Order()
+                .ForEach(package => logger.LogInfo($"Unused package: {package}"));
+        }
+
+
+        private ICollection<string> GetAllPackageDirectories()
+        {
+            return new DirectoryInfo(packageDirectory.DirInfo.FullName)
+                .EnumerateDirectories("*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
+                .Select(d => d.Name)
+                .ToList();
+        }
+
+        private static bool IsPathInSubfolder(string path, string rootFolder, string subFolder)
+        {
+            return path.IndexOf(
+                $"{Path.DirectorySeparatorChar}{subFolder}{Path.DirectorySeparatorChar}",
+                rootFolder.Length,
+                StringComparison.InvariantCultureIgnoreCase) >= 0;
+        }
+
+        private IEnumerable<string> GetRestoredLegacyPackageNames()
+        {
+            var oldPackageDirectories = GetRestoredPackageDirectoryNames(legacyPackageDirectory.DirInfo);
+            foreach (var oldPackageDirectory in oldPackageDirectories)
+            {
+                // nuget install restores packages to 'packagename.version' folders (dotnet restore to 'packagename/version' folders)
+                // typical folder names look like:
+                //   newtonsoft.json.13.0.3
+                // there are more complex ones too, such as:
+                //   runtime.tizen.4.0.0-armel.Microsoft.NETCore.DotNetHostResolver.2.0.0-preview2-25407-01
+
+                var match = LegacyNugetPackage().Match(oldPackageDirectory);
+                if (!match.Success)
+                {
+                    logger.LogWarning($"Package directory '{oldPackageDirectory}' doesn't match the expected pattern.");
+                    continue;
+                }
+
+                yield return match.Groups[1].Value.ToLowerInvariant();
+            }
+        }
+
+        private static IEnumerable<string> GetRestoredPackageDirectoryNames(DirectoryInfo root)
+        {
+            return Directory.GetDirectories(root.FullName)
+                .Select(d => Path.GetFileName(d).ToLowerInvariant());
+        }
+
+        private bool TryRestorePackageManually(string package, string? nugetConfig, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj)
+        {
+            logger.LogInfo($"Restoring package {package}...");
+            using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package, "missingpackages_workingdir"));
+            var success = dotnet.New(tempDir.DirInfo.FullName);
+            if (!success)
+            {
+                return false;
+            }
+
+            if (packageReferenceSource == PackageReferenceSource.PackagesConfig)
+            {
+                TryChangeTargetFrameworkMoniker(tempDir.DirInfo);
+            }
+
+            success = dotnet.AddPackage(tempDir.DirInfo.FullName, package);
+            if (!success)
+            {
+                return false;
+            }
+
+            var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig));
+            if (!res.Success)
+            {
+                if (res.HasNugetPackageSourceError && nugetConfig is not null)
+                {
+                    // Restore could not be completed because the listed source is unavailable. Try without the nuget.config:
+                    res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: null, ForceReevaluation: true));
+                }
+
+                // TODO: the restore might fail, we could retry with
+                // - a prerelease (*-* instead of *) version of the package,
+                // - a different target framework moniker.
+
+                if (!res.Success)
+                {
+                    logger.LogInfo($"Failed to restore nuget package {package}");
+                    return false;
+                }
+            }
+
+            return true;
+        }
+
+        private void TryChangeTargetFrameworkMoniker(DirectoryInfo tempDir)
+        {
+            try
+            {
+                logger.LogInfo($"Changing the target framework moniker in {tempDir.FullName}...");
+
+                var csprojs = tempDir.GetFiles("*.csproj", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive });
+                if (csprojs.Length != 1)
+                {
+                    logger.LogError($"Could not find the .csproj file in {tempDir.FullName}, count = {csprojs.Length}");
+                    return;
+                }
+
+                var csproj = csprojs[0];
+                var content = File.ReadAllText(csproj.FullName);
+                var matches = TargetFramework().Matches(content);
+                if (matches.Count == 0)
+                {
+                    logger.LogError($"Could not find target framework in {csproj.FullName}");
+                }
+                else
+                {
+                    content = TargetFramework().Replace(content, $"<TargetFramework>{FrameworkPackageNames.LatestNetFrameworkMoniker}</TargetFramework>", 1);
+                    File.WriteAllText(csproj.FullName, content);
+                }
+            }
+            catch (Exception exc)
+            {
+                logger.LogError($"Failed to update target framework in {tempDir.FullName}: {exc}");
+            }
+        }
+
+        private static async Task ExecuteGetRequest(string address, HttpClient httpClient, CancellationToken cancellationToken)
+        {
+            using var stream = await httpClient.GetStreamAsync(address, cancellationToken);
+            var buffer = new byte[1024];
+            int bytesRead;
+            while ((bytesRead = stream.Read(buffer, 0, buffer.Length)) > 0)
+            {
+                // do nothing
+            }
+        }
+
+        private bool IsFeedReachable(string feed)
+        {
+            logger.LogInfo($"Checking if Nuget feed '{feed}' is reachable...");
+            using HttpClient client = new();
+            int timeoutMilliSeconds = int.TryParse(Environment.GetEnvironmentVariable(EnvironmentVariableNames.NugetFeedResponsivenessInitialTimeout), out timeoutMilliSeconds)
+                ? timeoutMilliSeconds
+                : 1000;
+            int tryCount = int.TryParse(Environment.GetEnvironmentVariable(EnvironmentVariableNames.NugetFeedResponsivenessRequestCount), out tryCount)
+                ? tryCount
+                : 4;
+
+            for (var i = 0; i < tryCount; i++)
+            {
+                using var cts = new CancellationTokenSource();
+                cts.CancelAfter(timeoutMilliSeconds);
+                try
+                {
+                    ExecuteGetRequest(feed, client, cts.Token).GetAwaiter().GetResult();
+                    return true;
+                }
+                catch (Exception exc)
+                {
+                    if (exc is TaskCanceledException tce &&
+                        tce.CancellationToken == cts.Token &&
+                        cts.Token.IsCancellationRequested)
+                    {
+                        logger.LogWarning($"Didn't receive answer from Nuget feed '{feed}' in {timeoutMilliSeconds}ms.");
+                        timeoutMilliSeconds *= 2;
+                        continue;
+                    }
+
+                    // We're only interested in timeouts.
+                    logger.LogWarning($"Querying Nuget feed '{feed}' failed: {exc}");
+                    return true;
+                }
+            }
+
+            logger.LogWarning($"Didn't receive answer from Nuget feed '{feed}'. Tried it {tryCount} times.");
+            return false;
+        }
+
+        private bool CheckFeeds(List<FileInfo> allFiles)
+        {
+            logger.LogInfo("Checking Nuget feeds...");
+            var feeds = GetAllFeeds(allFiles);
+
+            var excludedFeeds = Environment.GetEnvironmentVariable(EnvironmentVariableNames.ExcludedNugetFeedsFromResponsivenessCheck)
+                ?.Split(" ", StringSplitOptions.RemoveEmptyEntries)
+                .ToHashSet() ?? [];
+
+            if (excludedFeeds.Count > 0)
+            {
+                logger.LogInfo($"Excluded Nuget feeds from responsiveness check: {string.Join(", ", excludedFeeds.OrderBy(f => f))}");
+            }
+
+            var allFeedsReachable = feeds.All(feed => excludedFeeds.Contains(feed) || IsFeedReachable(feed));
+            if (!allFeedsReachable)
+            {
+                logger.LogWarning("Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.");
+                diagnosticsWriter.AddEntry(new DiagnosticMessage(
+                    Language.CSharp,
+                    "buildless/unreachable-feed",
+                    "Found unreachable Nuget feed in C# analysis with build-mode 'none'",
+                    visibility: new DiagnosticMessage.TspVisibility(statusPage: true, cliSummaryTable: true, telemetry: true),
+                    markdownMessage: "Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.",
+                    severity: DiagnosticMessage.TspSeverity.Warning
+                ));
+            }
+            CompilationInfos.Add(("All Nuget feeds reachable", allFeedsReachable ? "1" : "0"));
+            return allFeedsReachable;
+        }
+
+        private IEnumerable<string> GetFeeds(string nugetConfig)
+        {
+            logger.LogInfo($"Getting Nuget feeds from '{nugetConfig}'...");
+            var results = dotnet.GetNugetFeeds(nugetConfig);
+            var regex = EnabledNugetFeed();
+            foreach (var result in results)
+            {
+                var match = regex.Match(result);
+                if (!match.Success)
+                {
+                    logger.LogError($"Failed to parse feed from '{result}'");
+                    continue;
+                }
+
+                var url = match.Groups[1].Value;
+                if (!url.StartsWith("https://", StringComparison.InvariantCultureIgnoreCase) &&
+                    !url.StartsWith("http://", StringComparison.InvariantCultureIgnoreCase))
+                {
+                    logger.LogInfo($"Skipping feed '{url}' as it is not a valid URL.");
+                    continue;
+                }
+
+                yield return url;
+            }
+        }
+
+        private HashSet<string> GetAllFeeds(List<FileInfo> allFiles)
+        {
+            var nugetConfigs = GetAllNugetConfigs(allFiles);
+            var feeds = nugetConfigs
+                .SelectMany(GetFeeds)
+                .Where(str => !string.IsNullOrWhiteSpace(str))
+                .ToHashSet();
+
+            if (feeds.Count > 0)
+            {
+                logger.LogInfo($"Found {feeds.Count} Nuget feeds in nuget.config files: {string.Join(", ", feeds.OrderBy(f => f))}");
+            }
+            else
+            {
+                logger.LogDebug("No Nuget feeds found in nuget.config files.");
+            }
+            return feeds;
+        }
+
+        [GeneratedRegex(@"<TargetFramework>.*</TargetFramework>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
+        private static partial Regex TargetFramework();
+
+        [GeneratedRegex(@"^(.+)\.(\d+\.\d+\.\d+(-(.+))?)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
+        private static partial Regex LegacyNugetPackage();
+
+        [GeneratedRegex(@"^E\s(.*)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
+        private static partial Regex EnabledNugetFeed();
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -20,6 +20,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
     {
         private readonly AssemblyCache assemblyCache;
         private readonly ILogger logger;
+        private readonly IDiagnosticsWriter diagnosticsWriter;
 
         // Only used as a set, but ConcurrentDictionary is the only concurrent set in .NET.
         private readonly IDictionary<string, bool> usedReferences = new ConcurrentDictionary<string, bool>();
@@ -52,6 +53,25 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             var startTime = DateTime.Now;
 
             this.logger = logger;
+
+            var diagDirEnv = Environment.GetEnvironmentVariable(EnvironmentVariableNames.DiagnosticDir);
+            if (!string.IsNullOrWhiteSpace(diagDirEnv) &&
+                !Directory.Exists(diagDirEnv))
+            {
+                try
+                {
+                    Directory.CreateDirectory(diagDirEnv);
+                }
+                catch (Exception e)
+                {
+                    logger.LogError($"Failed to create diagnostic directory {diagDirEnv}: {e.Message}");
+                    diagDirEnv = null;
+                }
+            }
+
+            this.diagnosticsWriter = new DiagnosticsStream(Path.Combine(
+                diagDirEnv ?? "",
+                $"dependency-manager-{DateTime.UtcNow:yyyyMMddHHmm}-{Environment.ProcessId}.jsonc"));
             this.sourceDir = new DirectoryInfo(srcDir);
 
             packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName, "packages"));
@@ -177,8 +197,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             var frameworkLocations = new HashSet<string>();
 
             var frameworkReferences = Environment.GetEnvironmentVariable(EnvironmentVariableNames.DotnetFrameworkReferences);
-            var frameworkReferencesUseSubfolders = Environment.GetEnvironmentVariable(EnvironmentVariableNames.DotnetFrameworkReferencesUseSubfolders);
-            _ = bool.TryParse(frameworkReferencesUseSubfolders, out var useSubfolders);
+            var useSubfolders = EnvironmentVariables.GetBoolean(EnvironmentVariableNames.DotnetFrameworkReferencesUseSubfolders);
             if (!string.IsNullOrWhiteSpace(frameworkReferences))
             {
                 RemoveFrameworkNugetPackages(dllPaths);
@@ -230,73 +249,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             return frameworkLocations;
         }
 
-        private void RestoreNugetPackages(List<FileInfo> allNonBinaryFiles, IEnumerable<string> allProjects, IEnumerable<string> allSolutions, HashSet<string> dllPaths)
-        {
-            try
-            {
-                using (var nuget = new NugetPackages(sourceDir.FullName, legacyPackageDirectory, logger))
-                {
-                    var count = nuget.InstallPackages();
-
-                    if (nuget.PackageCount > 0)
-                    {
-                        CompilationInfos.Add(("packages.config files", nuget.PackageCount.ToString()));
-                        CompilationInfos.Add(("Successfully restored packages.config files", count.ToString()));
-                    }
-                }
-
-                var nugetPackageDlls = legacyPackageDirectory.DirInfo.GetFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = true });
-                var nugetPackageDllPaths = nugetPackageDlls.Select(f => f.FullName).ToHashSet();
-                var excludedPaths = nugetPackageDllPaths
-                    .Where(path => IsPathInSubfolder(path, legacyPackageDirectory.DirInfo.FullName, "tools"))
-                    .ToList();
-
-                if (nugetPackageDllPaths.Count > 0)
-                {
-                    logger.LogInfo($"Restored {nugetPackageDllPaths.Count} Nuget DLLs.");
-                }
-                if (excludedPaths.Count > 0)
-                {
-                    logger.LogInfo($"Excluding {excludedPaths.Count} Nuget DLLs.");
-                }
-
-                foreach (var excludedPath in excludedPaths)
-                {
-                    logger.LogInfo($"Excluded Nuget DLL: {excludedPath}");
-                }
-
-                nugetPackageDllPaths.ExceptWith(excludedPaths);
-                dllPaths.UnionWith(nugetPackageDllPaths);
-            }
-            catch (Exception exc)
-            {
-                logger.LogError($"Failed to restore Nuget packages with nuget.exe: {exc.Message}");
-            }
-
-            var restoredProjects = RestoreSolutions(allSolutions, out var assets1);
-            var projects = allProjects.Except(restoredProjects);
-            RestoreProjects(projects, out var assets2);
-
-            var dependencies = Assets.GetCompilationDependencies(logger, assets1.Union(assets2));
-
-            var paths = dependencies
-                .Paths
-                .Select(d => Path.Combine(packageDirectory.DirInfo.FullName, d))
-                .ToList();
-            dllPaths.UnionWith(paths);
-
-            LogAllUnusedPackages(dependencies);
-            DownloadMissingPackages(allNonBinaryFiles, dllPaths);
-        }
-
-        private static bool IsPathInSubfolder(string path, string rootFolder, string subFolder)
-        {
-            return path.IndexOf(
-                $"{Path.DirectorySeparatorChar}{subFolder}{Path.DirectorySeparatorChar}",
-                rootFolder.Length,
-                StringComparison.InvariantCultureIgnoreCase) >= 0;
-        }
-
         private void RemoveNugetAnalyzerReferences()
         {
             var packageFolder = packageDirectory.DirInfo.FullName.ToLowerInvariant();
@@ -483,27 +435,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                 .FullName;
         }
 
-        private ICollection<string> GetAllPackageDirectories()
-        {
-            return new DirectoryInfo(packageDirectory.DirInfo.FullName)
-                .EnumerateDirectories("*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
-                .Select(d => d.Name)
-                .ToList();
-        }
-
-        private void LogAllUnusedPackages(DependencyContainer dependencies)
-        {
-            var allPackageDirectories = GetAllPackageDirectories();
-
-            logger.LogInfo($"Restored {allPackageDirectories.Count} packages");
-            logger.LogInfo($"Found {dependencies.Packages.Count} packages in project.assets.json files");
-
-            allPackageDirectories
-                .Where(package => !dependencies.Packages.Contains(package))
-                .Order()
-                .ForEach(package => logger.LogInfo($"Unused package: {package}"));
-        }
-
         private void GenerateSourceFileFromImplicitUsings()
         {
             var usings = new HashSet<string>();
@@ -807,269 +738,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             }
         }
 
-        /// <summary>
-        /// Executes `dotnet restore` on all solution files in solutions.
-        /// As opposed to RestoreProjects this is not run in parallel using PLINQ
-        /// as `dotnet restore` on a solution already uses multiple threads for restoring
-        /// the projects (this can be disabled with the `--disable-parallel` flag).
-        /// Populates assets with the relative paths to the assets files generated by the restore.
-        /// Returns a list of projects that are up to date with respect to restore.
-        /// </summary>
-        /// <param name="solutions">A list of paths to solution files.</param>
-        private IEnumerable<string> RestoreSolutions(IEnumerable<string> solutions, out IEnumerable<string> assets)
-        {
-            var successCount = 0;
-            var nugetSourceFailures = 0;
-            var assetFiles = new List<string>();
-            var projects = solutions.SelectMany(solution =>
-                {
-                    logger.LogInfo($"Restoring solution {solution}...");
-                    var res = dotnet.Restore(new(solution, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
-                    if (res.Success)
-                    {
-                        successCount++;
-                    }
-                    if (res.HasNugetPackageSourceError)
-                    {
-                        nugetSourceFailures++;
-                    }
-                    assetFiles.AddRange(res.AssetsFilePaths);
-                    return res.RestoredProjects;
-                }).ToList();
-            assets = assetFiles;
-            CompilationInfos.Add(("Successfully restored solution files", successCount.ToString()));
-            CompilationInfos.Add(("Failed solution restore with package source error", nugetSourceFailures.ToString()));
-            CompilationInfos.Add(("Restored projects through solution files", projects.Count.ToString()));
-            return projects;
-        }
-
-        /// <summary>
-        /// Executes `dotnet restore` on all projects in projects.
-        /// This is done in parallel for performance reasons.
-        /// Populates assets with the relative paths to the assets files generated by the restore.
-        /// </summary>
-        /// <param name="projects">A list of paths to project files.</param>
-        private void RestoreProjects(IEnumerable<string> projects, out IEnumerable<string> assets)
-        {
-            var successCount = 0;
-            var nugetSourceFailures = 0;
-            var assetFiles = new List<string>();
-            var sync = new object();
-            Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = threads }, project =>
-            {
-                logger.LogInfo($"Restoring project {project}...");
-                var res = dotnet.Restore(new(project, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
-                lock (sync)
-                {
-                    if (res.Success)
-                    {
-                        successCount++;
-                    }
-                    if (res.HasNugetPackageSourceError)
-                    {
-                        nugetSourceFailures++;
-                    }
-                    assetFiles.AddRange(res.AssetsFilePaths);
-                }
-            });
-            assets = assetFiles;
-            CompilationInfos.Add(("Successfully restored project files", successCount.ToString()));
-            CompilationInfos.Add(("Failed project restore with package source error", nugetSourceFailures.ToString()));
-        }
-
-        [GeneratedRegex(@"^(.+)\.(\d+\.\d+\.\d+(-(.+))?)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
-        private static partial Regex LegacyNugetPackage();
-
-
-        private static IEnumerable<string> GetRestoredPackageDirectoryNames(DirectoryInfo root)
-        {
-            return Directory.GetDirectories(root.FullName)
-                .Select(d => Path.GetFileName(d).ToLowerInvariant());
-        }
-
-        private IEnumerable<string> GetRestoredLegacyPackageNames()
-        {
-            var oldPackageDirectories = GetRestoredPackageDirectoryNames(legacyPackageDirectory.DirInfo);
-            foreach (var oldPackageDirectory in oldPackageDirectories)
-            {
-                // nuget install restores packages to 'packagename.version' folders (dotnet restore to 'packagename/version' folders)
-                // typical folder names look like:
-                //   newtonsoft.json.13.0.3
-                // there are more complex ones too, such as:
-                //   runtime.tizen.4.0.0-armel.Microsoft.NETCore.DotNetHostResolver.2.0.0-preview2-25407-01
-
-                var match = LegacyNugetPackage().Match(oldPackageDirectory);
-                if (!match.Success)
-                {
-                    logger.LogWarning($"Package directory '{oldPackageDirectory}' doesn't match the expected pattern.");
-                    continue;
-                }
-
-                yield return match.Groups[1].Value.ToLowerInvariant();
-            }
-        }
-
-        private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPaths)
-        {
-            var alreadyDownloadedPackages = GetRestoredPackageDirectoryNames(packageDirectory.DirInfo);
-            var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames();
-
-            var notYetDownloadedPackages = new HashSet<PackageReference>(fileContent.AllPackages);
-            foreach (var alreadyDownloadedPackage in alreadyDownloadedPackages)
-            {
-                notYetDownloadedPackages.Remove(new(alreadyDownloadedPackage, PackageReferenceSource.SdkCsProj));
-            }
-            foreach (var alreadyDownloadedLegacyPackage in alreadyDownloadedLegacyPackages)
-            {
-                notYetDownloadedPackages.Remove(new(alreadyDownloadedLegacyPackage, PackageReferenceSource.PackagesConfig));
-            }
-
-            if (notYetDownloadedPackages.Count == 0)
-            {
-                return;
-            }
-
-            var multipleVersions = notYetDownloadedPackages
-                .GroupBy(p => p.Name)
-                .Where(g => g.Count() > 1)
-                .Select(g => g.Key)
-                .ToList();
-
-            foreach (var package in multipleVersions)
-            {
-                logger.LogWarning($"Found multiple not yet restored packages with name '{package}'.");
-                notYetDownloadedPackages.Remove(new(package, PackageReferenceSource.PackagesConfig));
-            }
-
-            logger.LogInfo($"Found {notYetDownloadedPackages.Count} packages that are not yet restored");
-
-            var nugetConfigs = allFiles.SelectFileNamesByName("nuget.config").ToArray();
-            string? nugetConfig = null;
-            if (nugetConfigs.Length > 1)
-            {
-                logger.LogInfo($"Found multiple nuget.config files: {string.Join(", ", nugetConfigs)}.");
-                nugetConfig = allFiles
-                    .SelectRootFiles(sourceDir)
-                    .SelectFileNamesByName("nuget.config")
-                    .FirstOrDefault();
-                if (nugetConfig == null)
-                {
-                    logger.LogInfo("Could not find a top-level nuget.config file.");
-                }
-            }
-            else
-            {
-                nugetConfig = nugetConfigs.FirstOrDefault();
-            }
-
-            if (nugetConfig != null)
-            {
-                logger.LogInfo($"Using nuget.config file {nugetConfig}.");
-            }
-
-            CompilationInfos.Add(("Fallback nuget restore", notYetDownloadedPackages.Count.ToString()));
-
-            var successCount = 0;
-            var sync = new object();
-
-            Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = threads }, package =>
-            {
-                var success = TryRestorePackageManually(package.Name, nugetConfig, package.PackageReferenceSource);
-                if (!success)
-                {
-                    return;
-                }
-
-                lock (sync)
-                {
-                    successCount++;
-                }
-            });
-
-            CompilationInfos.Add(("Successfully ran fallback nuget restore", successCount.ToString()));
-
-            dllPaths.Add(missingPackageDirectory.DirInfo.FullName);
-        }
-
-        [GeneratedRegex(@"<TargetFramework>.*</TargetFramework>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
-        private static partial Regex TargetFramework();
-
-        private bool TryRestorePackageManually(string package, string? nugetConfig, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj)
-        {
-            logger.LogInfo($"Restoring package {package}...");
-            using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package, "missingpackages_workingdir"));
-            var success = dotnet.New(tempDir.DirInfo.FullName);
-            if (!success)
-            {
-                return false;
-            }
-
-            if (packageReferenceSource == PackageReferenceSource.PackagesConfig)
-            {
-                TryChangeTargetFrameworkMoniker(tempDir.DirInfo);
-            }
-
-            success = dotnet.AddPackage(tempDir.DirInfo.FullName, package);
-            if (!success)
-            {
-                return false;
-            }
-
-            var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig));
-            if (!res.Success)
-            {
-                if (res.HasNugetPackageSourceError && nugetConfig is not null)
-                {
-                    // Restore could not be completed because the listed source is unavailable. Try without the nuget.config:
-                    res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: null, ForceReevaluation: true));
-                }
-
-                // TODO: the restore might fail, we could retry with
-                // - a prerelease (*-* instead of *) version of the package,
-                // - a different target framework moniker.
-
-                if (!res.Success)
-                {
-                    logger.LogInfo($"Failed to restore nuget package {package}");
-                    return false;
-                }
-            }
-
-            return true;
-        }
-
-        private void TryChangeTargetFrameworkMoniker(DirectoryInfo tempDir)
-        {
-            try
-            {
-                logger.LogInfo($"Changing the target framework moniker in {tempDir.FullName}...");
-
-                var csprojs = tempDir.GetFiles("*.csproj", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive });
-                if (csprojs.Length != 1)
-                {
-                    logger.LogError($"Could not find the .csproj file in {tempDir.FullName}, count = {csprojs.Length}");
-                    return;
-                }
-
-                var csproj = csprojs[0];
-                var content = File.ReadAllText(csproj.FullName);
-                var matches = TargetFramework().Matches(content);
-                if (matches.Count == 0)
-                {
-                    logger.LogError($"Could not find target framework in {csproj.FullName}");
-                }
-                else
-                {
-                    content = TargetFramework().Replace(content, $"<TargetFramework>{FrameworkPackageNames.LatestNetFrameworkMoniker}</TargetFramework>", 1);
-                    File.WriteAllText(csproj.FullName, content);
-                }
-            }
-            catch (Exception exc)
-            {
-                logger.LogError($"Failed to update target framework in {tempDir.FullName}: {exc}");
-            }
-        }
-
         public void Dispose(TemporaryDirectory? dir, string name)
         {
             try
@@ -1091,6 +759,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             {
                 Dispose(tempWorkingDirectory, "temporary working");
             }
+
+            diagnosticsWriter?.Dispose();
         }
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs

@@ -16,12 +16,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
     public partial class DotNet : IDotNet
     {
         private readonly IDotNetCliInvoker dotnetCliInvoker;
+        private readonly ILogger logger;
         private readonly TemporaryDirectory? tempWorkingDirectory;
 
         private DotNet(IDotNetCliInvoker dotnetCliInvoker, ILogger logger, TemporaryDirectory? tempWorkingDirectory = null)
         {
             this.tempWorkingDirectory = tempWorkingDirectory;
             this.dotnetCliInvoker = dotnetCliInvoker;
+            this.logger = logger;
             Info();
         }
 
@@ -89,17 +91,18 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             return dotnetCliInvoker.RunCommand(args);
         }
 
-        public IList<string> GetListedRuntimes() => GetListed("--list-runtimes");
+        public IList<string> GetListedRuntimes() => GetResultList("--list-runtimes");
 
-        public IList<string> GetListedSdks() => GetListed("--list-sdks");
+        public IList<string> GetListedSdks() => GetResultList("--list-sdks");
 
-        private IList<string> GetListed(string args)
+        private IList<string> GetResultList(string args)
         {
-            if (dotnetCliInvoker.RunCommand(args, out var artifacts))
+            if (dotnetCliInvoker.RunCommand(args, out var results))
             {
-                return artifacts;
+                return results;
             }
-            return new List<string>();
+            logger.LogWarning($"Running 'dotnet {args}' failed.");
+            return [];
         }
 
         public bool Exec(string execArgs)
@@ -108,6 +111,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             return dotnetCliInvoker.RunCommand(args);
         }
 
+        public IList<string> GetNugetFeeds(string nugetConfig) => GetResultList($"nuget list source --format Short --configfile \"{nugetConfig}\"");
+
         // The version number should be kept in sync with the version .NET version used for building the application.
         public const string LatestDotNetSdkVersion = "8.0.101";
 

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs

@@ -16,5 +16,30 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         /// Controls whether to use framework dependencies from subfolders.
         /// </summary>
         public const string DotnetFrameworkReferencesUseSubfolders = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_DOTNET_FRAMEWORK_REFERENCES_USE_SUBFOLDERS";
+
+        /// <summary>
+        /// Controls whether to check the responsiveness of NuGet feeds.
+        /// </summary>
+        public const string CheckNugetFeedResponsiveness = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK";
+
+        /// <summary>
+        /// Specifies the NuGet feeds to exclude from the responsiveness check. The value is a space-separated list of feed URLs.
+        /// </summary>
+        public const string ExcludedNugetFeedsFromResponsivenessCheck = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_EXCLUDED";
+
+        /// <summary>
+        /// Specifies the timeout (as an integer) in milliseconds for the initial check of NuGet feeds responsiveness. The value is then doubled for each subsequent check.
+        /// </summary>
+        public const string NugetFeedResponsivenessInitialTimeout = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_TIMEOUT";
+
+        /// <summary>
+        /// Specifies how many requests to make to the NuGet feed to check its responsiveness.
+        /// </summary>
+        public const string NugetFeedResponsivenessRequestCount = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_LIMIT";
+
+        /// <summary>
+        /// Specifies the location of the diagnostic directory.
+        /// </summary>
+        public const string DiagnosticDir = "CODEQL_EXTRACTOR_CSHARP_DIAGNOSTIC_DIR";
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs

@@ -13,6 +13,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         IList<string> GetListedRuntimes();
         IList<string> GetListedSdks();
         bool Exec(string execArgs);
+        IList<string> GetNugetFeeds(string nugetConfig);
     }
 
     public record class RestoreSettings(string File, string PackageDirectory, bool ForceDotnetRefAssemblyFetching, string? PathToNugetConfig = null, bool ForceReevaluation = false);

csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs

@@ -143,7 +143,7 @@ namespace Semmle.Extraction.CSharp.Standalone
             stopwatch.Start();
 
             using var logger = new ConsoleLogger(options.Verbosity, logThreadId: true);
-            logger.Log(Severity.Info, "Extracting C# in buildless mode");
+            logger.Log(Severity.Info, "Extracting C# with build-mode set to 'none'");
             using var dependencyManager = new DependencyManager(options.SrcDir, logger);
 
             if (!dependencyManager.NonGeneratedSourcesFiles.Any())

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expression.cs

@@ -100,8 +100,14 @@ namespace Semmle.Extraction.CSharp.Entities
         /// <param name="child">The child index.</param>
         /// <param name="type">A type hint.</param>
         /// <returns>The new expression.</returns>
-        public static Expression Create(Context cx, ExpressionSyntax node, IExpressionParentEntity parent, int child) =>
-            CreateFromNode(new ExpressionNodeInfo(cx, node, parent, child));
+        public static Expression Create(Context cx, ExpressionSyntax node, IExpressionParentEntity parent, int child, Boolean isCompilerGenerated = false)
+        {
+            var info = new ExpressionNodeInfo(cx, node, parent, child)
+            {
+                IsCompilerGenerated = isCompilerGenerated
+            };
+            return CreateFromNode(info);
+        }
 
         public static Expression CreateFromNode(ExpressionNodeInfo info) => Expressions.ImplicitCast.Create(info);
 

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ArrayCreation.cs

@@ -97,7 +97,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 ExprKind.ARRAY_CREATION,
                 parent,
                 childIndex,
-                true,
+                isCompilerGenerated: true,
                 null);
 
             var arrayCreation = new Expression(info);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Assignment.cs

@@ -26,10 +26,10 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
             if (operatorKind.HasValue)
             {
                 // Convert assignment such as `a += b` into `a = a + b`.
-                var simpleAssignExpr = new Expression(new ExpressionInfo(Context, Type, Location, ExprKind.SIMPLE_ASSIGN, this, 2, false, null));
+                var simpleAssignExpr = new Expression(new ExpressionInfo(Context, Type, Location, ExprKind.SIMPLE_ASSIGN, this, 2, isCompilerGenerated: true, null));
                 Create(Context, Syntax.Left, simpleAssignExpr, 1);
-                var opexpr = new Expression(new ExpressionInfo(Context, Type, Location, operatorKind.Value, simpleAssignExpr, 0, false, null));
-                Create(Context, Syntax.Left, opexpr, 0);
+                var opexpr = new Expression(new ExpressionInfo(Context, Type, Location, operatorKind.Value, simpleAssignExpr, 0, isCompilerGenerated: true, null));
+                Create(Context, Syntax.Left, opexpr, 0, isCompilerGenerated: true);
                 Create(Context, Syntax.Right, opexpr, 1);
                 opexpr.OperatorCall(trapFile, Syntax);
             }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Cast.cs

@@ -41,7 +41,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 ExprKind.CAST,
                 parent,
                 childIndex,
-                true,
+                isCompilerGenerated: true,
                 ValueAsString(value));
 
             var ret = new Expression(info);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Collections/Spread.cs

@@ -6,7 +6,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
     internal class Spread : Expression
     {
         public Spread(Context cx, SpreadElementSyntax syntax, IExpressionParentEntity parent, int child) :
-            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), ExprKind.SPREAD_ELEMENT, parent, child, false, null))
+            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), ExprKind.SPREAD_ELEMENT, parent, child, isCompilerGenerated: false, null))
         {
             Create(cx, syntax.Expression, this, 0);
         }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Default.cs

@@ -24,7 +24,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 ExprKind.DEFAULT,
                 parent,
                 childIndex,
-                true,
+                isCompilerGenerated: true,
                 value);
 
             return new Expression(info);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Discard.cs

@@ -11,7 +11,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
         }
 
         private Discard(Context cx, CSharpSyntaxNode syntax, IExpressionParentEntity parent, int child) :
-            base(new ExpressionInfo(cx, cx.GetType(syntax), cx.CreateLocation(syntax.GetLocation()), ExprKind.DISCARD, parent, child, false, null))
+            base(new ExpressionInfo(cx, cx.GetType(syntax), cx.CreateLocation(syntax.GetLocation()), ExprKind.DISCARD, parent, child, isCompilerGenerated: false, null))
         {
         }
 

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ElementAccess.cs

@@ -22,7 +22,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
             if (Kind == ExprKind.POINTER_INDIRECTION)
             {
                 var qualifierInfo = new ExpressionNodeInfo(Context, qualifier, this, 0);
-                var add = new Expression(new ExpressionInfo(Context, qualifierInfo.Type, Location, ExprKind.ADD, this, 0, false, null));
+                var add = new Expression(new ExpressionInfo(Context, qualifierInfo.Type, Location, ExprKind.ADD, this, 0, isCompilerGenerated: false, null));
                 qualifierInfo.SetParent(add, 0);
                 CreateFromNode(qualifierInfo);
                 PopulateArguments(trapFile, argumentList, 1);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ImplicitCast.cs

@@ -14,13 +14,13 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
         }
 
         private ImplicitCast(ExpressionNodeInfo info)
-            : base(new ExpressionInfo(info.Context, info.ConvertedType, info.Location, ExprKind.CAST, info.Parent, info.Child, true, info.ExprValue))
+            : base(new ExpressionInfo(info.Context, info.ConvertedType, info.Location, ExprKind.CAST, info.Parent, info.Child, isCompilerGenerated: true, info.ExprValue))
         {
             Expr = Factory.Create(new ExpressionNodeInfo(Context, info.Node, this, 0));
         }
 
         private ImplicitCast(ExpressionNodeInfo info, IMethodSymbol method)
-            : base(new ExpressionInfo(info.Context, info.ConvertedType, info.Location, ExprKind.OPERATOR_INVOCATION, info.Parent, info.Child, true, info.ExprValue))
+            : base(new ExpressionInfo(info.Context, info.ConvertedType, info.Location, ExprKind.OPERATOR_INVOCATION, info.Parent, info.Child, isCompilerGenerated: true, info.ExprValue))
         {
             Expr = Factory.Create(info.SetParent(this, 0));
 
@@ -65,7 +65,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                     kind,
                     parent,
                     childIndex,
-                    true,
+                    isCompilerGenerated: true,
                     v);
 
             var method = GetImplicitConversionMethod(type, value);
@@ -93,7 +93,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                     ExprKind.CAST,
                     parent,
                     childIndex,
-                    true,
+                    isCompilerGenerated: true,
                     ValueAsString(value));
 
             return new Expression(info);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Initializer.cs

@@ -45,7 +45,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 ExprKind.ARRAY_INIT,
                 parent,
                 index,
-                true,
+                isCompilerGenerated: true,
                 null);
 
             return new Expression(info);
@@ -132,7 +132,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 var addMethod = Method.Create(Context, collectionInfo.Symbol as IMethodSymbol);
                 var voidType = AnnotatedTypeSymbol.CreateNotAnnotated(Context.Compilation.GetSpecialType(SpecialType.System_Void));
 
-                var invocation = new Expression(new ExpressionInfo(Context, voidType, Context.CreateLocation(i.GetLocation()), ExprKind.METHOD_INVOCATION, this, child++, false, null));
+                var invocation = new Expression(new ExpressionInfo(Context, voidType, Context.CreateLocation(i.GetLocation()), ExprKind.METHOD_INVOCATION, this, child++, isCompilerGenerated: true, null));
 
                 if (addMethod is not null)
                     trapFile.expr_call(invocation, addMethod);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/InterpolatedString.cs

@@ -25,7 +25,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                     case SyntaxKind.InterpolatedStringText:
                         // Create a string literal
                         var interpolatedText = (InterpolatedStringTextSyntax)c;
-                        new Expression(new ExpressionInfo(Context, Type, Context.CreateLocation(c.GetLocation()), ExprKind.UTF16_STRING_LITERAL, this, child++, false, interpolatedText.TextToken.ValueText));
+                        new Expression(new ExpressionInfo(Context, Type, Context.CreateLocation(c.GetLocation()), ExprKind.UTF16_STRING_LITERAL, this, child++, isCompilerGenerated: false, interpolatedText.TextToken.ValueText));
                         break;
                     default:
                         throw new InternalError(c, $"Unhandled interpolation kind {c.Kind()}");

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Lambda.cs

@@ -63,7 +63,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
         public static Lambda Create(ExpressionNodeInfo info, ParenthesizedLambdaExpressionSyntax node) => new Lambda(info, node);
 
         private Lambda(ExpressionNodeInfo info, SimpleLambdaExpressionSyntax node)
-            : this(info.SetKind(ExprKind.LAMBDA), node.Body, Enumerators.Singleton(node.Parameter), null) { }
+            : this(info.SetKind(ExprKind.LAMBDA), node.Body, [node.Parameter], null) { }
 
         public static Lambda Create(ExpressionNodeInfo info, SimpleLambdaExpressionSyntax node) => new Lambda(info, node);
 

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Literal.cs

@@ -97,7 +97,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 kind,
                 parent,
                 childIndex,
-                true,
+                isCompilerGenerated: true,
                 ValueAsString(value));
 
             return new Expression(info);
@@ -112,7 +112,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 ExprKind.NULL_LITERAL,
                 parent,
                 childIndex,
-                true,
+                isCompilerGenerated: true,
                 ValueAsString(null));
 
             return new Expression(info);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ObjectCreation/AnonymousObjectCreation.cs

@@ -30,7 +30,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 return;
             }
 
-            var objectInitializer = new Expression(new ExpressionInfo(Context, Type, Location, ExprKind.OBJECT_INIT, this, -1, false, null));
+            var objectInitializer = new Expression(new ExpressionInfo(Context, Type, Location, ExprKind.OBJECT_INIT, this, -1, isCompilerGenerated: false, null));
 
             foreach (var init in Syntax.Initializers)
             {
@@ -40,11 +40,11 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 var type = property.GetAnnotatedType();
                 var loc = Context.CreateLocation(init.GetLocation());
 
-                var assignment = new Expression(new ExpressionInfo(Context, type, loc, ExprKind.SIMPLE_ASSIGN, objectInitializer, child++, false, null));
+                var assignment = new Expression(new ExpressionInfo(Context, type, loc, ExprKind.SIMPLE_ASSIGN, objectInitializer, child++, isCompilerGenerated: false, null));
                 Create(Context, init.Expression, assignment, 0);
                 Property.Create(Context, property);
 
-                var access = new Expression(new ExpressionInfo(Context, type, loc, ExprKind.PROPERTY_ACCESS, assignment, 1, false, null));
+                var access = new Expression(new ExpressionInfo(Context, type, loc, ExprKind.PROPERTY_ACCESS, assignment, 1, isCompilerGenerated: false, null));
                 trapFile.expr_access(access, propEntity);
             }
         }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ObjectCreation/DateTimeObjectCreation.cs

@@ -59,7 +59,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 ExprKind.OBJECT_CREATION,
                 parent,
                 childIndex,
-                true,
+                isCompilerGenerated: true,
                 null));
 
             var longTypeSymbol = constructorSymbol.Parameters[0].Type;

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Patterns/BinaryPattern.cs

@@ -8,7 +8,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
     internal class BinaryPattern : Expression
     {
         public BinaryPattern(Context cx, BinaryPatternSyntax syntax, IExpressionParentEntity parent, int child) :
-            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), GetKind(syntax.OperatorToken, syntax), parent, child, false, null))
+            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), GetKind(syntax.OperatorToken, syntax), parent, child, isCompilerGenerated: false, null))
         {
             Pattern.Create(cx, syntax.Left, this, 0);
             Pattern.Create(cx, syntax.Right, this, 1);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Patterns/ListPattern.cs

@@ -7,7 +7,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
     internal class ListPattern : Expression
     {
         internal ListPattern(Context cx, ListPatternSyntax syntax, IExpressionParentEntity parent, int child) :
-            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), ExprKind.LIST_PATTERN, parent, child, false, null))
+            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), ExprKind.LIST_PATTERN, parent, child, isCompilerGenerated: false, null))
         {
             syntax.Patterns.ForEach((p, i) => Pattern.Create(cx, p, this, i));
         }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Patterns/PositionalPattern.cs

@@ -7,7 +7,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
     internal class PositionalPattern : Expression
     {
         internal PositionalPattern(Context cx, PositionalPatternClauseSyntax posPc, IExpressionParentEntity parent, int child) :
-            base(new ExpressionInfo(cx, null, cx.CreateLocation(posPc.GetLocation()), ExprKind.POSITIONAL_PATTERN, parent, child, false, null))
+            base(new ExpressionInfo(cx, null, cx.CreateLocation(posPc.GetLocation()), ExprKind.POSITIONAL_PATTERN, parent, child, isCompilerGenerated: false, null))
         {
             posPc.Subpatterns.ForEach((p, i) => Pattern.Create(cx, p.Pattern, this, i));
         }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Patterns/PropertyPattern.cs

@@ -7,7 +7,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
     internal class PropertyPattern : Expression
     {
         internal PropertyPattern(Context cx, PropertyPatternClauseSyntax pp, IExpressionParentEntity parent, int child) :
-            base(new ExpressionInfo(cx, null, cx.CreateLocation(pp.GetLocation()), ExprKind.PROPERTY_PATTERN, parent, child, false, null))
+            base(new ExpressionInfo(cx, null, cx.CreateLocation(pp.GetLocation()), ExprKind.PROPERTY_PATTERN, parent, child, isCompilerGenerated: false, null))
         {
             child = 0;
             foreach (var sub in pp.Subpatterns)
@@ -56,7 +56,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
             };
 
         private static Expression CreateSyntheticExp(Context cx, Microsoft.CodeAnalysis.Location location, IExpressionParentEntity parent, int child) =>
-            new Expression(new ExpressionInfo(cx, null, cx.CreateLocation(location), ExprKind.PROPERTY_PATTERN, parent, child, false, null));
+            new Expression(new ExpressionInfo(cx, null, cx.CreateLocation(location), ExprKind.PROPERTY_PATTERN, parent, child, isCompilerGenerated: false, null));
 
         private static void MakeExpressions(Context cx, IExpressionParentEntity parent, SubpatternSyntax syntax, int child)
         {

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Patterns/RecursivePattern.cs

@@ -15,7 +15,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
         /// <param name="parent">The parent pattern/expression.</param>
         /// <param name="child">The child index of this pattern.</param>
         public RecursivePattern(Context cx, RecursivePatternSyntax syntax, IExpressionParentEntity parent, int child) :
-            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), ExprKind.RECURSIVE_PATTERN, parent, child, false, null))
+            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), ExprKind.RECURSIVE_PATTERN, parent, child, isCompilerGenerated: false, null))
         {
             // Extract the type access
             if (syntax.Type is TypeSyntax t)

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Patterns/RelationalPattern.cs

@@ -8,7 +8,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
     internal class RelationalPattern : Expression
     {
         public RelationalPattern(Context cx, RelationalPatternSyntax syntax, IExpressionParentEntity parent, int child) :
-             base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), GetKind(syntax.OperatorToken), parent, child, false, null))
+             base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), GetKind(syntax.OperatorToken), parent, child, isCompilerGenerated: false, null))
         {
             Expression.Create(cx, syntax.Expression, this, 0);
         }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Patterns/SlicePattern.cs

@@ -6,7 +6,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
     internal class SlicePattern : Expression
     {
         public SlicePattern(Context cx, SlicePatternSyntax syntax, IExpressionParentEntity parent, int child) :
-            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), ExprKind.SLICE_PATTERN, parent, child, false, null))
+            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), ExprKind.SLICE_PATTERN, parent, child, isCompilerGenerated: false, null))
         {
             if (syntax.Pattern is not null)
             {

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Patterns/UnaryPattern.cs

@@ -6,7 +6,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
     internal class UnaryPattern : Expression
     {
         public UnaryPattern(Context cx, UnaryPatternSyntax syntax, IExpressionParentEntity parent, int child) :
-            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), ExprKind.NOT_PATTERN, parent, child, false, null))
+            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), ExprKind.NOT_PATTERN, parent, child, isCompilerGenerated: false, null))
         {
             Pattern.Create(cx, syntax.Pattern, this, 0);
         }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Query.cs

@@ -23,7 +23,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
             public QueryCall(Context cx, IMethodSymbol? method, SyntaxNode clause, IExpressionParentEntity parent, int child)
                 : base(new ExpressionInfo(cx, method?.GetAnnotatedReturnType(),
                         cx.CreateLocation(clause.GetLocation()),
-                        ExprKind.METHOD_INVOCATION, parent, child, false, null))
+                        ExprKind.METHOD_INVOCATION, parent, child, isCompilerGenerated: false, null))
             {
                 if (method is not null)
                     cx.TrapWriter.Writer.expr_call(this, Method.Create(cx, method));
@@ -97,7 +97,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 Expression.Create(cx, Expr, decl, 0);
 
                 var nameLoc = cx.CreateLocation(name.GetLocation());
-                var access = new Expression(new ExpressionInfo(cx, type, nameLoc, ExprKind.LOCAL_VARIABLE_ACCESS, decl, 1, false, null));
+                var access = new Expression(new ExpressionInfo(cx, type, nameLoc, ExprKind.LOCAL_VARIABLE_ACCESS, decl, 1, isCompilerGenerated: false, null));
                 cx.TrapWriter.Writer.expr_access(access, LocalVariable.Create(cx, variableSymbol));
 
                 return decl;

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Switch.cs

@@ -27,7 +27,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
         internal SwitchCase(Context cx, SwitchExpressionArmSyntax arm, Switch parent, int child) :
             base(new ExpressionInfo(
                 cx, cx.GetType(arm.Expression), cx.CreateLocation(arm.GetLocation()),
-                ExprKind.SWITCH_CASE, parent, child, false, null))
+                ExprKind.SWITCH_CASE, parent, child, isCompilerGenerated: false, null))
         {
             Expressions.Pattern.Create(cx, arm.Pattern, this, 0);
             if (arm.WhenClause is WhenClauseSyntax when)

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/This.cs

@@ -8,7 +8,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
         private This(IExpressionInfo info) : base(info) { }
 
         public static This CreateImplicit(Context cx, ITypeSymbol @class, Extraction.Entities.Location loc, IExpressionParentEntity parent, int child) =>
-            new This(new ExpressionInfo(cx, AnnotatedTypeSymbol.CreateNotAnnotated(@class), loc, Kinds.ExprKind.THIS_ACCESS, parent, child, true, null));
+            new This(new ExpressionInfo(cx, AnnotatedTypeSymbol.CreateNotAnnotated(@class), loc, Kinds.ExprKind.THIS_ACCESS, parent, child, isCompilerGenerated: true, null));
 
         public static This CreateExplicit(ExpressionNodeInfo info) => new This(info.SetKind(ExprKind.THIS_ACCESS));
     }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/TypeAccess.cs

@@ -44,7 +44,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 ExprKind.TYPE_ACCESS,
                 parent,
                 childIndex,
-                true,
+                isCompilerGenerated: true,
                 null);
 
             return new Expression(typeAccessInfo);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/TypeOf.cs

@@ -26,7 +26,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 ExprKind.TYPEOF,
                 parent,
                 childIndex,
-                true,
+                isCompilerGenerated: true,
                 null);
 
             var ret = new Expression(info);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/VariableDeclaration.cs

@@ -15,7 +15,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
 
         public static VariableDeclaration Create(Context cx, ISymbol symbol, AnnotatedTypeSymbol? type, TypeSyntax? optionalSyntax, Extraction.Entities.Location exprLocation, bool isVar, IExpressionParentEntity parent, int child)
         {
-            var ret = new VariableDeclaration(new ExpressionInfo(cx, type, exprLocation, ExprKind.LOCAL_VAR_DECL, parent, child, false, null));
+            var ret = new VariableDeclaration(new ExpressionInfo(cx, type, exprLocation, ExprKind.LOCAL_VAR_DECL, parent, child, isCompilerGenerated: false, null));
             cx.Try(null, null, () =>
             {
                 var l = LocalVariable.Create(cx, symbol);
@@ -52,7 +52,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
         public static Expression CreateParenthesized(Context cx, DeclarationExpressionSyntax node, ParenthesizedVariableDesignationSyntax designation, IExpressionParentEntity parent, int child, INamedTypeSymbol? t)
         {
             var type = t is null ? (AnnotatedTypeSymbol?)null : new AnnotatedTypeSymbol(t, t.NullableAnnotation);
-            var tuple = new Expression(new ExpressionInfo(cx, type, cx.CreateLocation(node.GetLocation()), ExprKind.TUPLE, parent, child, false, null));
+            var tuple = new Expression(new ExpressionInfo(cx, type, cx.CreateLocation(node.GetLocation()), ExprKind.TUPLE, parent, child, isCompilerGenerated: false, null));
 
             cx.Try(null, null, () =>
             {
@@ -68,7 +68,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
         public static Expression CreateParenthesized(Context cx, VarPatternSyntax varPattern, ParenthesizedVariableDesignationSyntax designation, IExpressionParentEntity parent, int child)
         {
             var tuple = new Expression(
-                new ExpressionInfo(cx, null, cx.CreateLocation(varPattern.GetLocation()), ExprKind.TUPLE, parent, child, false, null),
+                new ExpressionInfo(cx, null, cx.CreateLocation(varPattern.GetLocation()), ExprKind.TUPLE, parent, child, isCompilerGenerated: false, null),
                 shouldPopulate: false);
 
             var elementTypes = new List<ITypeSymbol?>();
@@ -148,7 +148,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
             Create(cx, node, node.Designation, parent, child, cx.GetTypeInfo(node).Type.DisambiguateType() as INamedTypeSymbol);
 
         public static VariableDeclaration Create(Context cx, CSharpSyntaxNode c, AnnotatedTypeSymbol? type, IExpressionParentEntity parent, int child) =>
-            new VariableDeclaration(new ExpressionInfo(cx, type, cx.CreateLocation(c.FixedLocation()), ExprKind.LOCAL_VAR_DECL, parent, child, false, null));
+            new VariableDeclaration(new ExpressionInfo(cx, type, cx.CreateLocation(c.FixedLocation()), ExprKind.LOCAL_VAR_DECL, parent, child, isCompilerGenerated: false, null));
 
         public static VariableDeclaration Create(Context cx, CatchDeclarationSyntax d, bool isVar, IExpressionParentEntity parent, int child)
         {
@@ -179,7 +179,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                     Create(cx, d.Initializer.Value, ret, 0);
 
                     // Create an access
-                    var access = new Expression(new ExpressionInfo(cx, type, localVar.Location, ExprKind.LOCAL_VARIABLE_ACCESS, ret, 1, false, null));
+                    var access = new Expression(new ExpressionInfo(cx, type, localVar.Location, ExprKind.LOCAL_VARIABLE_ACCESS, ret, 1, isCompilerGenerated: false, null));
                     cx.TrapWriter.Writer.expr_access(access, localVar);
                 }
 

csharp/extractor/Semmle.Extraction.CSharp/Entities/Field.cs

@@ -110,9 +110,9 @@ namespace Semmle.Extraction.CSharp.Entities
             string? constValue, ref int child)
         {
             var type = Symbol.GetAnnotatedType();
-            var simpleAssignExpr = new Expression(new ExpressionInfo(Context, type, loc, ExprKind.SIMPLE_ASSIGN, this, child++, false, constValue));
+            var simpleAssignExpr = new Expression(new ExpressionInfo(Context, type, loc, ExprKind.SIMPLE_ASSIGN, this, child++, isCompilerGenerated: true, constValue));
             Expression.CreateFromNode(new ExpressionNodeInfo(Context, initializer, simpleAssignExpr, 0));
-            var access = new Expression(new ExpressionInfo(Context, type, Location, ExprKind.FIELD_ACCESS, simpleAssignExpr, 1, false, constValue));
+            var access = new Expression(new ExpressionInfo(Context, type, Location, ExprKind.FIELD_ACCESS, simpleAssignExpr, 1, isCompilerGenerated: true, constValue));
             trapFile.expr_access(access, this);
             return access;
         }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Property.cs

@@ -86,9 +86,9 @@ namespace Semmle.Extraction.CSharp.Entities
                     {
                         var loc = Context.CreateLocation(initializer!.GetLocation());
                         var annotatedType = AnnotatedTypeSymbol.CreateNotAnnotated(Symbol.Type);
-                        var simpleAssignExpr = new Expression(new ExpressionInfo(Context, annotatedType, loc, ExprKind.SIMPLE_ASSIGN, this, child++, false, null));
+                        var simpleAssignExpr = new Expression(new ExpressionInfo(Context, annotatedType, loc, ExprKind.SIMPLE_ASSIGN, this, child++, isCompilerGenerated: true, null));
                         Expression.CreateFromNode(new ExpressionNodeInfo(Context, initializer.Value, simpleAssignExpr, 0));
-                        var access = new Expression(new ExpressionInfo(Context, annotatedType, Location, ExprKind.PROPERTY_ACCESS, simpleAssignExpr, 1, false, null));
+                        var access = new Expression(new ExpressionInfo(Context, annotatedType, Location, ExprKind.PROPERTY_ACCESS, simpleAssignExpr, 1, isCompilerGenerated: true, null));
                         trapFile.expr_access(access, this);
                         if (!Symbol.IsStatic)
                         {

csharp/extractor/Semmle.Extraction.Tests/Runtime.cs

@@ -26,6 +26,8 @@ namespace Semmle.Extraction.Tests
         public IList<string> GetListedSdks() => sdks;
 
         public bool Exec(string execArgs) => true;
+
+        public IList<string> GetNugetFeeds(string nugetConfig) => [];
     }
 
     public class RuntimeTests

csharp/extractor/Semmle.Util.Tests/ActionMap.cs

@@ -1,54 +0,0 @@
-using Xunit;
-using Semmle.Util;
-
-using Assert = Xunit.Assert;
-
-namespace SemmleTests.Semmle.Util
-{
-
-    public class ActionMapTests
-    {
-        [Fact]
-        public void TestAddthenOnAdd()
-        {
-            var am = new ActionMap<int, int>();
-            am.Add(1, 2);
-            int value = 0;
-            am.OnAdd(1, x => value = x);
-            Assert.Equal(2, value);
-        }
-
-        [Fact]
-        public void TestOnAddthenAdd()
-        {
-            var am = new ActionMap<int, int>();
-            int value = 0;
-            am.OnAdd(1, x => value = x);
-            am.Add(1, 2);
-            Assert.Equal(2, value);
-        }
-
-        [Fact]
-        public void TestNotAdded()
-        {
-            var am = new ActionMap<int, int>();
-            int value = 0;
-            am.OnAdd(1, x => value = x);
-            am.Add(2, 2);
-            Assert.Equal(0, value);
-        }
-
-        [Fact]
-        public void TestMultipleActions()
-        {
-            var am = new ActionMap<int, int>();
-            int value1 = 0, value2 = 0;
-            am.OnAdd(1, x => value1 = x);
-            am.OnAdd(1, x => value2 = x);
-            am.Add(1, 2);
-            Assert.Equal(2, value1);
-            Assert.Equal(2, value2);
-        }
-
-    }
-}

csharp/extractor/Semmle.Util.Tests/TextTest.cs

@@ -1,78 +0,0 @@
-using Xunit;
-using System;
-using Semmle.Util;
-
-using Assert = Xunit.Assert;
-
-namespace SemmleTests
-{
-    public class TextTest
-    {
-        //#################### PRIVATE VARIABLES ####################
-        #region
-
-        /// <summary>
-        /// A shorter way of writing Environment.NewLine (it gets used repeatedly).
-        /// </summary>
-        private static readonly string NL = Environment.NewLine;
-
-        #endregion
-
-        //#################### TEST METHODS ####################
-        #region
-
-        [Fact]
-        public void GetAllTest()
-        {
-            var input = new string[]
-            {
-                "Said once a young coder from Crewe,",
-                "'I like to write tests, so I do!",
-                "They help me confirm",
-                "That I don't need to squirm -",
-                "My code might look nice, but works too!'"
-            };
-
-            var text = new Text(input);
-
-            Assert.Equal(string.Join(NL, input) + NL, text.GetAll());
-        }
-
-        [Fact]
-        public void GetPortionTest()
-        {
-            var input = new string[]
-            {
-                "There once was a jolly young tester",
-                "Who couldn't leave software to fester -",
-                "He'd prod and he'd poke",
-                "Until something bad broke,",
-                "And then he'd find someone to pester."
-            };
-
-            var text = new Text(input);
-
-            // A single-line range (to test the special case).
-            Assert.Equal("jolly" + NL, text.GetPortion(0, 17, 0, 22));
-
-            // A two-line range.
-            Assert.Equal("prod and he'd poke" + NL + "Until" + NL, text.GetPortion(2, 5, 3, 5));
-
-            // A three-line range (to test that the middle line is included in full).
-            Assert.Equal("poke" + NL + "Until something bad broke," + NL + "And then" + NL, text.GetPortion(2, 19, 4, 8));
-
-            // An invalid but recoverable range (to test that a best effort is made rather than crashing).
-            Assert.Equal(NL + "Who couldn't leave software to fester -" + NL, text.GetPortion(0, int.MaxValue, 1, int.MaxValue));
-
-            // Some quite definitely dodgy ranges (to test that exceptions are thrown).
-            Assert.Throws<Exception>(() => text.GetPortion(-1, 0, 0, 0));
-            Assert.Throws<Exception>(() => text.GetPortion(0, -1, 0, 0));
-            Assert.Throws<Exception>(() => text.GetPortion(0, 0, -1, 0));
-            Assert.Throws<Exception>(() => text.GetPortion(0, 0, 0, -1));
-            Assert.Throws<Exception>(() => text.GetPortion(3, 5, 2, 5));
-            Assert.Throws<Exception>(() => text.GetPortion(2, 5, int.MaxValue, 5));
-        }
-
-        #endregion
-    }
-}

csharp/extractor/Semmle.Util/ActionMap.cs

@@ -1,45 +0,0 @@
-using System;
-using System.Collections.Generic;
-
-namespace Semmle.Util
-{
-    /// <summary>
-    /// A dictionary which performs an action when items are added to the dictionary.
-    /// The order in which keys and actions are added does not matter.
-    /// </summary>
-    /// <typeparam name="TKey"></typeparam>
-    /// <typeparam name="TValue"></typeparam>
-    public class ActionMap<TKey, TValue> where TKey : notnull
-    {
-        public void Add(TKey key, TValue value)
-        {
-
-            if (actions.TryGetValue(key, out var a))
-                a(value);
-            values[key] = value;
-        }
-
-        public void OnAdd(TKey key, Action<TValue> action)
-        {
-            if (actions.TryGetValue(key, out var a))
-            {
-                actions[key] = a + action;
-            }
-            else
-            {
-                actions.Add(key, action);
-            }
-
-            if (values.TryGetValue(key, out var val))
-            {
-                action(val);
-            }
-        }
-
-        // Action associated with each key.
-        private readonly Dictionary<TKey, Action<TValue>> actions = new Dictionary<TKey, Action<TValue>>();
-
-        // Values associated with each key.
-        private readonly Dictionary<TKey, TValue> values = new Dictionary<TKey, TValue>();
-    }
-}

csharp/extractor/Semmle.Util/Enumerators.cs

@@ -1,19 +0,0 @@
-using System.Collections.Generic;
-
-namespace Semmle.Util
-{
-    public static class Enumerators
-    {
-        /// <summary>
-        /// Create an enumerable with a single element.
-        /// </summary>
-        ///
-        /// <typeparam name="T">The type of the enumerable/element.</typeparam>
-        /// <param name="t">The element.</param>
-        /// <returns>An enumerable containing a single element.</returns>
-        public static IEnumerable<T> Singleton<T>(T t)
-        {
-            yield return t;
-        }
-    }
-}

csharp/extractor/Semmle.Util/EnvironmentVariables.cs

@@ -27,5 +27,12 @@ namespace Semmle.Util
             }
             return threads;
         }
+
+        public static bool GetBoolean(string name)
+        {
+            var env = Environment.GetEnvironmentVariable(name);
+            var _ = bool.TryParse(env, out var value);
+            return value;
+        }
     }
 }

csharp/extractor/Semmle.Util/FileRenamer.cs

@@ -1,34 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-
-namespace Semmle.Util
-{
-    /// <summary>
-    /// Utility to temporarily rename a set of files.
-    /// </summary>
-    public sealed class FileRenamer : IDisposable
-    {
-        private readonly string[] files;
-        private const string suffix = ".codeqlhidden";
-
-        public FileRenamer(IEnumerable<FileInfo> oldFiles)
-        {
-            files = oldFiles.Select(f => f.FullName).ToArray();
-
-            foreach (var file in files)
-            {
-                File.Move(file, file + suffix);
-            }
-        }
-
-        public void Dispose()
-        {
-            foreach (var file in files)
-            {
-                File.Move(file + suffix, file);
-            }
-        }
-    }
-}

csharp/extractor/Semmle.Util/FileUtils.cs

@@ -102,8 +102,7 @@ namespace Semmle.Util
         private static async Task DownloadFileAsync(string address, string filename)
         {
             using var httpClient = new HttpClient();
-            using var request = new HttpRequestMessage(HttpMethod.Get, address);
-            using var contentStream = await (await httpClient.SendAsync(request)).Content.ReadAsStreamAsync();
+            using var contentStream = await httpClient.GetStreamAsync(address);
             using var stream = new FileStream(filename, FileMode.Create, FileAccess.Write, FileShare.None, 4096, true);
             await contentStream.CopyToAsync(stream);
         }
@@ -112,7 +111,7 @@ namespace Semmle.Util
         /// Downloads the file at <paramref name="address"/> to <paramref name="fileName"/>.
         /// </summary>
         public static void DownloadFile(string address, string fileName) =>
-           DownloadFileAsync(address, fileName).Wait();
+           DownloadFileAsync(address, fileName).GetAwaiter().GetResult();
 
         public static string NestPaths(ILogger logger, string? outerpath, string innerpath)
         {

csharp/extractor/Semmle.Util/FuzzyDictionary.cs

@@ -1,165 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-
-namespace Semmle.Util
-{
-    /// <summary>
-    /// A dictionary from strings to elements of type T.
-    /// </summary>
-    ///
-    /// <remarks>
-    /// This data structure is able to locate items based on an "approximate match"
-    /// of the key. This is used for example when attempting to identify two terms
-    /// in different trap files which are similar but not identical.
-    ///
-    /// The algorithm locates the closest match to a string based on a "distance function".
-    ///
-    /// Whilst many distance functions are possible, a bespoke algorithm is used here,
-    /// for efficiency and suitability for the domain.
-    ///
-    /// The distance is defined as the Hamming Distance of the numbers in the string.
-    /// Each string is split into the base "form" (stripped of numbers) and a vector of
-    /// numbers. (Numbers are non-negative integers in this context).
-    ///
-    /// Strings with a different "form" are considered different and have a distance
-    /// of infinity.
-    ///
-    /// This distance function is reflexive, symmetric and obeys the triangle inequality.
-    ///
-    /// E.g. foo(bar,1,2) has form "foo(bar,,)" and integers {1,2}
-    ///
-    /// distance(foo(bar,1,2), foo(bar,1,2)) = 0
-    /// distance(foo(bar,1,2), foo(bar,1,3)) = 1
-    /// distance(foo(bar,2,1), foo(bar,1,2)) = 2
-    /// distance(foo(bar,1,2), foo(baz,1,2)) = infinity
-    /// </remarks>
-    ///
-    /// <typeparam name="T">The value type.</typeparam>
-    public class FuzzyDictionary<T> where T : class
-    {
-        // All data items indexed by the "base string" (stripped of numbers)
-        private readonly Dictionary<string, List<KeyValuePair<string, T>>> index = new Dictionary<string, List<KeyValuePair<string, T>>>();
-
-        /// <summary>
-        /// Stores a new KeyValuePair in the data structure.
-        /// </summary>
-        /// <param name="k">The key.</param>
-        /// <param name="v">The value.</param>
-        public void Add(string k, T v)
-        {
-            var kv = new KeyValuePair<string, T>(k, v);
-
-            var root = StripDigits(k);
-            index.AddAnother(root, kv);
-        }
-
-        /// <summary>
-        /// Computes the Hamming Distance between two sequences of the same length.
-        /// </summary>
-        /// <param name="v1">Vector 1</param>
-        /// <param name="v2">Vector 2</param>
-        /// <returns>The Hamming Distance.</returns>
-        private static int HammingDistance<TElement>(IEnumerable<TElement> v1, IEnumerable<TElement> v2) where TElement : notnull
-        {
-            return v1.Zip(v2, (x, y) => x.Equals(y) ? 0 : 1).Sum();
-        }
-
-        /// <summary>
-        /// Locates the value with the smallest Hamming Distance from the query.
-        /// </summary>
-        /// <param name="query">The query string.</param>
-        /// <param name="distance">The distance between the query string and the stored string.</param>
-        /// <returns>The best match, or null (default).</returns>
-        public T? FindMatch(string query, out int distance)
-        {
-            var root = StripDigits(query);
-            if (!index.TryGetValue(root, out var list))
-            {
-                distance = 0;
-                return default(T);
-            }
-
-            return BestMatch(query, list, (a, b) => HammingDistance(ExtractIntegers(a), ExtractIntegers(b)), out distance);
-        }
-
-        /// <summary>
-        /// Returns the best match (with the smallest distance) for a query.
-        /// </summary>
-        /// <param name="query">The query string.</param>
-        /// <param name="candidates">The list of candidate matches.</param>
-        /// <param name="distance">The distance function.</param>
-        /// <param name="bestDistance">The distance between the query and the stored string.</param>
-        /// <returns>The stored value.</returns>
-        private static T? BestMatch(string query, IEnumerable<KeyValuePair<string, T>> candidates, Func<string, string, int> distance, out int bestDistance)
-        {
-            var bestMatch = default(T);
-            bestDistance = 0;
-            var first = true;
-
-            foreach (var candidate in candidates)
-            {
-                var d = distance(query, candidate.Key);
-                if (d == 0)
-                    return candidate.Value;
-
-                if (first || d < bestDistance)
-                {
-                    bestDistance = d;
-                    bestMatch = candidate.Value;
-                    first = false;
-                }
-            }
-
-            return bestMatch;
-        }
-
-        /// <summary>
-        /// Removes all digits from a string.
-        /// </summary>
-        /// <param name="input">The input string.</param>
-        /// <returns>String with digits removed.</returns>
-        private static string StripDigits(string input)
-        {
-            var result = new StringBuilder();
-            foreach (var c in input.Where(c => !char.IsDigit(c)))
-                result.Append(c);
-            return result.ToString();
-        }
-
-        /// <summary>
-        /// Extracts and enumerates all non-negative integers in a string.
-        /// </summary>
-        /// <param name="input">The string to enumerate.</param>
-        /// <returns>The sequence of integers.</returns>
-        private static IEnumerable<int> ExtractIntegers(string input)
-        {
-            var inNumber = false;
-            var value = 0;
-            foreach (var c in input)
-            {
-                if (char.IsDigit(c))
-                {
-                    if (inNumber)
-                    {
-                        value = value * 10 + (c - '0');
-                    }
-                    else
-                    {
-                        inNumber = true;
-                        value = c - '0';
-                    }
-                }
-                else
-                {
-                    if (inNumber)
-                    {
-                        yield return value;
-                        inNumber = false;
-                    }
-                }
-            }
-        }
-    }
-}

csharp/extractor/Semmle.Util/SharedReference.cs

@@ -1,18 +0,0 @@
-namespace Semmle.Util
-{
-    /// <summary>
-    /// An instance of this class maintains a shared reference to an object.
-    /// This makes it possible for several different parts of the code to
-    /// share access to an object that can change (that is, they all want
-    /// to refer to the same object, but the object to which they jointly
-    /// refer may vary over time).
-    /// </summary>
-    /// <typeparam name="T">The type of the shared object.</typeparam>
-    public sealed class SharedReference<T> where T : class
-    {
-        /// <summary>
-        /// The shared object to which different parts of the code want to refer.
-        /// </summary>
-        public T? Obj { get; set; }
-    }
-}

csharp/extractor/Semmle.Util/StringExtensions.cs

@@ -1,41 +0,0 @@
-using System.Collections.Generic;
-using System.Linq;
-
-namespace Semmle.Util
-{
-    public static class StringExtensions
-    {
-        public static (string, string) Split(this string self, int index0)
-        {
-            var split = self.Split(new[] { index0 });
-            return (split[0], split[1]);
-        }
-
-        public static (string, string, string) Split(this string self, int index0, int index1)
-        {
-            var split = self.Split(new[] { index0, index1 });
-            return (split[0], split[1], split[2]);
-        }
-
-        public static (string, string, string, string) Split(this string self, int index0, int index1, int index2)
-        {
-            var split = self.Split(new[] { index0, index1, index2 });
-            return (split[0], split[1], split[2], split[3]);
-        }
-
-        private static List<string> Split(this string self, params int[] indices)
-        {
-            var ret = new List<string>();
-            var previousIndex = 0;
-            foreach (var index in indices.OrderBy(i => i))
-            {
-                ret.Add(self.Substring(previousIndex, index - previousIndex));
-                previousIndex = index;
-            }
-
-            ret.Add(self.Substring(previousIndex));
-
-            return ret;
-        }
-    }
-}

csharp/extractor/Semmle.Util/Text.cs

@@ -1,105 +0,0 @@
-using System;
-using System.IO;
-
-namespace Semmle.Util
-{
-    /// <summary>
-    /// An instance of this class represents a piece of text, e.g. the text of a C# source file.
-    /// </summary>
-    public sealed class Text
-    {
-        //#################### PRIVATE VARIABLES ####################
-        #region
-
-        /// <summary>
-        /// The text, stored line-by-line.
-        /// </summary>
-        private readonly string[] lines;
-
-        #endregion
-
-        //#################### CONSTRUCTORS ####################
-        #region
-
-        /// <summary>
-        /// Constructs a text object from an array of lines.
-        /// </summary>
-        /// <param name="lines">The lines of text.</param>
-        public Text(string[] lines)
-        {
-            this.lines = lines;
-        }
-
-        #endregion
-
-        //#################### PUBLIC METHODS ####################
-        #region
-
-        /// <summary>
-        /// Gets the whole text.
-        /// </summary>
-        /// <returns>The whole text.</returns>
-        public string GetAll()
-        {
-            using var sw = new StringWriter();
-            foreach (var s in lines)
-            {
-                sw.WriteLine(s);
-            }
-            return sw.ToString();
-        }
-
-        /// <summary>
-        /// Gets the portion of text that lies in the specified location range.
-        /// </summary>
-        /// <param name="startRow">The row at which the portion starts.</param>
-        /// <param name="startColumn">The column in the start row at which the portion starts.</param>
-        /// <param name="endRow">The row at which the portion ends.</param>
-        /// <param name="endColumn">The column in the end row at which the portion ends.</param>
-        /// <returns>The portion of text that lies in the specified location range.</returns>
-        public string GetPortion(int startRow, int startColumn, int endRow, int endColumn)
-        {
-            // Perform some basic validation on the range bounds.
-            if (startRow < 0 || endRow < 0 || startColumn < 0 || endColumn < 0 || endRow >= lines.Length || startRow > endRow)
-            {
-                throw new Exception
-                (
-                    string.Format("Bad range ({0},{1}):({2},{3}) in a piece of text with {4} lines", startRow, startColumn, endRow, endColumn, lines.Length)
-                );
-            }
-
-            using var sw = new StringWriter();
-            string line;
-
-            for (var i = startRow; i <= endRow; ++i)
-            {
-                if (i == startRow && i == endRow)
-                {
-                    // This is a single-line range, so take the bit between "startColumn" and "endColumn".
-                    line = startColumn <= lines[i].Length ? lines[i].Substring(startColumn, endColumn - startColumn) : "";
-                }
-                else if (i == startRow)
-                {
-                    // This is the first line of a multi-line range, so take the bit from "startColumn" onwards.
-                    line = startColumn <= lines[i].Length ? lines[i].Substring(startColumn) : "";
-                }
-                else if (i == endRow)
-                {
-                    // This is the last line of a multi-line range, so take the bit up to "endColumn".
-                    line = endColumn <= lines[i].Length ? lines[i].Substring(0, endColumn) : lines[i];
-                }
-                else
-                {
-                    // This is a line in the middle of a multi-line range, so take the whole line.
-                    line = lines[i];
-                }
-
-                sw.WriteLine(line);
-            }
-
-            return sw.ToString();
-        }
-
-        #endregion
-    }
-}

csharp/extractor/Semmle.Util/ToolStatusPage/DiagnosticMessage.cs

@@ -1,7 +1,6 @@
 using System;
 using System.Collections.Generic;
 using System.Globalization;
-using System.IO;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Converters;
 using Newtonsoft.Json.Serialization;
@@ -179,64 +178,4 @@ namespace Semmle.Util
             this.PlaintextMessage = plaintextMessage;
         }
     }
-
-    /// <summary>
-    /// Provides the ability to write diagnostic messages to some output.
-    /// </summary>
-    public interface IDiagnosticsWriter
-    {
-        /// <summary>
-        /// Adds <paramref name="message" /> as a new diagnostics entry.
-        /// </summary>
-        /// <param name="message">The diagnostics entry to add.</param>
-        void AddEntry(DiagnosticMessage message);
-    }
-
-    /// <summary>
-    /// A wrapper around an underlying <see cref="StreamWriter" /> which allows
-    /// <see cref="DiagnosticMessage" /> objects to be serialized to it.
-    /// </summary>
-    public sealed class DiagnosticsStream : IDiagnosticsWriter, IDisposable
-    {
-        private readonly JsonSerializer serializer;
-        private readonly StreamWriter writer;
-
-        /// <summary>
-        /// Initialises a new <see cref="DiagnosticsStream" /> for a file at <paramref name="path" />.
-        /// </summary>
-        /// <param name="path">The path to the file that should be created.</param>
-        public DiagnosticsStream(string path)
-        {
-            this.writer = File.CreateText(path);
-
-            var contractResolver = new DefaultContractResolver
-            {
-                NamingStrategy = new CamelCaseNamingStrategy()
-            };
-
-            serializer = new JsonSerializer
-            {
-                ContractResolver = contractResolver,
-                NullValueHandling = NullValueHandling.Ignore
-            };
-        }
-
-        /// <summary>
-        /// Adds <paramref name="message" /> as a new diagnostics entry.
-        /// </summary>
-        /// <param name="message">The diagnostics entry to add.</param>
-        public void AddEntry(DiagnosticMessage message)
-        {
-            serializer.Serialize(writer, message);
-            writer.Flush();
-        }
-
-        /// <summary>
-        /// Releases all resources used by the <see cref="DiagnosticsStream" /> object.
-        /// </summary>
-        public void Dispose()
-        {
-            writer.Dispose();
-        }
-    }
 }

csharp/extractor/Semmle.Util/ToolStatusPage/DiagnosticsStream.cs

@@ -0,0 +1,55 @@
+using System;
+using System.IO;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Serialization;
+
+namespace Semmle.Util
+{
+    /// <summary>
+    /// A wrapper around an underlying <see cref="StreamWriter" /> which allows
+    /// <see cref="DiagnosticMessage" /> objects to be serialized to it.
+    /// </summary>
+    public sealed class DiagnosticsStream : IDiagnosticsWriter
+    {
+        private readonly JsonSerializer serializer;
+        private readonly StreamWriter writer;
+
+        /// <summary>
+        /// Initialises a new <see cref="DiagnosticsStream" /> for a file at <paramref name="path" />.
+        /// </summary>
+        /// <param name="path">The path to the file that should be created.</param>
+        public DiagnosticsStream(string path)
+        {
+            this.writer = File.CreateText(path);
+
+            var contractResolver = new DefaultContractResolver
+            {
+                NamingStrategy = new CamelCaseNamingStrategy()
+            };
+
+            serializer = new JsonSerializer
+            {
+                ContractResolver = contractResolver,
+                NullValueHandling = NullValueHandling.Ignore
+            };
+        }
+
+        /// <summary>
+        /// Adds <paramref name="message" /> as a new diagnostics entry.
+        /// </summary>
+        /// <param name="message">The diagnostics entry to add.</param>
+        public void AddEntry(DiagnosticMessage message)
+        {
+            serializer.Serialize(writer, message);
+            writer.Flush();
+        }
+
+        /// <summary>
+        /// Releases all resources used by the <see cref="DiagnosticsStream" /> object.
+        /// </summary>
+        public void Dispose()
+        {
+            writer.Dispose();
+        }
+    }
+}

csharp/extractor/Semmle.Util/ToolStatusPage/IDiagnosticsWriter.cs

@@ -0,0 +1,16 @@
+using System;
+
+namespace Semmle.Util
+{
+    /// <summary>
+    /// Provides the ability to write diagnostic messages to some output.
+    /// </summary>
+    public interface IDiagnosticsWriter : IDisposable
+    {
+        /// <summary>
+        /// Adds <paramref name="message" /> as a new diagnostics entry.
+        /// </summary>
+        /// <param name="message">The diagnostics entry to add.</param>
+        void AddEntry(DiagnosticMessage message);
+    }
+}

csharp/extractor/Semmle.Util/Worklist.cs

@@ -1,57 +0,0 @@
-using System.Collections.Generic;
-
-namespace Semmle.Util
-{
-    /// <summary>
-    /// A worklist of items, providing the operations of adding an item, checking
-    /// whether there are new items and iterating a chunk of unprocessed items.
-    /// Any one item will only be accepted into the worklist once.
-    /// </summary>
-    public class Worklist<T>
-    {
-        private readonly HashSet<T> internalSet = new HashSet<T>();
-        private LinkedList<T> internalList = new LinkedList<T>();
-        private bool hasNewElements = false;
-
-        /// <summary>
-        /// Gets a value indicating whether this instance has had any new elements added
-        /// since the last time <c>GetUnprocessedElements()</c> was called.
-        /// </summary>
-        /// <value>
-        /// <c>true</c> if this instance has new elements; otherwise, <c>false</c>.
-        /// </value>
-        public bool HasNewElements => hasNewElements;
-
-        /// <summary>
-        /// Add the specified element to the worklist.
-        /// </summary>
-        /// <param name='element'>
-        /// If set to <c>true</c> element.
-        /// </param>
-        public bool Add(T element)
-        {
-            if (internalSet.Contains(element))
-                return false;
-            internalSet.Add(element);
-            internalList.AddLast(element);
-            hasNewElements = true;
-            return true;
-        }
-
-        /// <summary>
-        /// Gets the unprocessed elements that have been accumulated since the last time
-        /// this method was called. If <c>HasNewElements == true</c>, the resulting list
-        /// will be non-empty.
-        /// </summary>
-        /// <returns>
-        /// The unprocessed elements.
-        /// </returns>
-        public LinkedList<T> GetUnprocessedElements()
-        {
-            var result = internalList;
-            internalList = new LinkedList<T>();
-            hasNewElements = false;
-            return result;
-        }
-    }
-}

csharp/ql/integration-tests/all-platforms/diag_recursive_generics/Types.ql

@@ -2,4 +2,4 @@ import csharp
 
 from Class c
 where c.fromSource()
-select c, c.getBaseClass().getFullyQualifiedName()
+select c, c.getBaseClass().getFullyQualifiedNameDebug()

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/Assemblies.expected

@@ -0,0 +1 @@
+| [...]/newtonsoft.json/13.0.3/lib/net6.0/Newtonsoft.Json.dll |

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/Assemblies.ql

@@ -0,0 +1,11 @@
+import csharp
+
+private string getPath(Assembly a) {
+  not a.getCompilation().getOutputAssembly() = a and
+  exists(string s | s = a.getFile().getAbsolutePath() |
+    result = "[...]/" + s.substring(s.indexOf("newtonsoft.json"), s.length())
+  )
+}
+
+from Assembly a
+select getPath(a)

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/CompilationInfo.expected

@@ -0,0 +1,13 @@
+| All Nuget feeds reachable | 0.0 |
+| Fallback nuget restore | 1.0 |
+| Project files on filesystem | 1.0 |
+| Resolved assembly conflicts | 7.0 |
+| Restored .NET framework variants | 0.0 |
+| Solution files on filesystem | 1.0 |
+| Source files generated | 0.0 |
+| Source files on filesystem | 1.0 |
+| Successfully ran fallback nuget restore | 1.0 |
+| Unresolved references | 0.0 |
+| UseWPF set | 0.0 |
+| UseWindowsForms set | 0.0 |
+| WebView extraction enabled | 1.0 |

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/CompilationInfo.ql

@@ -0,0 +1,15 @@
+import csharp
+import semmle.code.csharp.commons.Diagnostics
+
+query predicate compilationInfo(string key, float value) {
+  key != "Resolved references" and
+  not key.matches("Compiler diagnostic count for%") and
+  exists(Compilation c, string infoKey, string infoValue | infoValue = c.getInfo(infoKey) |
+    key = infoKey and
+    value = infoValue.toFloat()
+    or
+    not exists(infoValue.toFloat()) and
+    key = infoKey + ": " + infoValue and
+    value = 1
+  )
+}

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/diagnostics.expected

@@ -0,0 +1,42 @@
+{
+  "markdownMessage": "C# analysis with build-mode 'none' completed.",
+  "severity": "unknown",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/buildless/complete",
+    "name": "C# analysis with build-mode 'none' completed"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": false,
+    "telemetry": true
+  }
+}
+{
+  "markdownMessage": "C# with build-mode set to 'none'. This means that all C# source in the working directory will be scanned, with build tools, such as Nuget and Dotnet CLIs, only contributing information about external dependencies.",
+  "severity": "note",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/buildless/mode-active",
+    "name": "C# with build-mode set to 'none'"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}
+{
+  "markdownMessage": "Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.",
+  "severity": "warning",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/buildless/unreachable-feed",
+    "name": "Found unreachable Nuget feed in C# analysis with build-mode 'none'"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/proj/Program.cs

@@ -0,0 +1,6 @@
+class Program
+{
+    static void Main(string[] args)
+    {
+    }
+}

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/proj/nuget.config

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <packageSources>
+    <clear />
+    <add key="x" value="https://localhost:53/packages/" />
+    <add key="y" value="https://localhost:80/packages/" />
+  </packageSources>
+</configuration>

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/proj/proj.csproj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFrameworks>net8.0</TargetFrameworks>
+  </PropertyGroup>
+
+  <Target Name="DeleteBinObjFolders" BeforeTargets="Clean">
+    <RemoveDir Directories=".\bin" />
+    <RemoveDir Directories=".\obj" />
+  </Target>
+
+  <ItemGroup>
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+  </ItemGroup>
+</Project>

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/standalone.sln

@@ -0,0 +1,19 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.5.002.0
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "proj", "proj\proj.csproj", "{6ED00460-7666-4AE9-A405-4B6C8B02279A}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {4ED55A1C-066C-43DF-B32E-7EAA035985EE}
+	EndGlobalSection
+EndGlobal

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/test.py

@@ -0,0 +1,10 @@
+from create_database_utils import *
+from diagnostics_test_utils import *
+import os
+
+os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK"] = "true"          # Enable NuGet feed check
+os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_TIMEOUT"] = "1"     # 1ms, the GET request should fail with such short timeout
+os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_LIMIT"] = "1"       # Limit the count of checks to 1
+os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_EXCLUDED"] = "https://abc.de:8000/packages/"   # Exclude this feed from check
+run_codeql_database_create([], lang="csharp", extra_args=["--build-mode=none"])
+check_diagnostics()

csharp/ql/lib/ext/System.Diagnostics.model.yml

@@ -19,3 +19,12 @@ extensions:
       - ["System.Diagnostics", "TraceListenerCollection", False, "get_Item", "(System.Int32)", "", "Argument[this].Element", "ReturnValue", "value", "manual"]
       - ["System.Diagnostics", "TraceListenerCollection", False, "get_Item", "(System.String)", "", "Argument[this].Element", "ReturnValue", "value", "manual"]
       - ["System.Diagnostics", "TraceListenerCollection", False, "set_Item", "(System.Int32,System.Diagnostics.TraceListener)", "", "Argument[1]", "Argument[this].Element", "value", "manual"]
+  - addsTo:
+      pack: codeql/csharp-all
+      extensible: neutralModel
+    data:
+      - ["System.Diagnostics", "ProcessStartInfo", "set_Arguments", "(System.String)", "summary", "manual"]
+      - ["System.Diagnostics", "ProcessStartInfo", "set_FileName", "(System.String)", "summary", "manual"]
+      - ["System.Diagnostics", "ProcessStartInfo", "set_UserName", "(System.String)", "summary", "manual"]
+      - ["System.Diagnostics", "ProcessStartInfo", "set_Verb", "(System.String)", "summary", "manual"]
+      - ["System.Diagnostics", "ProcessStartInfo", "set_WorkingDirectory", "(System.String)", "summary", "manual"]

csharp/ql/lib/qlpack.yml

@@ -13,6 +13,7 @@ dependencies:
   codeql/threat-models: ${workspace}
   codeql/tutorial: ${workspace}
   codeql/util: ${workspace}
+  codeql/xml: ${workspace}
 dataExtensions:
   - ext/*.model.yml
   - ext/generated/*.model.yml

csharp/ql/lib/semmle/code/csharp/Element.qll

@@ -102,25 +102,8 @@ class NamedElement extends Element, @named_element {
   final predicate hasName(string name) { name = this.getName() }
 
   /**
-   * Gets the fully qualified name of this element, for example the
-   * fully qualified name of `M` on line 3 is `N.C.M` in
+   * DEPRECATED: Use `hasFullyQualifiedName` instead.
    *
-   * ```csharp
-   * namespace N {
-   *   class C {
-   *     void M(int i, string s) { }
-   *   }
-   * }
-   * ```
-   */
-  cached
-  deprecated final string getQualifiedName() {
-    exists(string qualifier, string name | this.hasQualifiedName(qualifier, name) |
-      if qualifier = "" then result = name else result = qualifier + "." + name
-    )
-  }
-
-  /**
    * Gets the fully qualified name of this element, for example the
    * fully qualified name of `M` on line 3 is `N.C.M` in
    *
@@ -135,21 +118,39 @@ class NamedElement extends Element, @named_element {
    * Unbound generic types, such as `IList<T>`, are represented as
    * ``System.Collections.Generic.IList`1``.
    */
-  cached
-  final string getFullyQualifiedName() {
+  deprecated final string getFullyQualifiedName() {
     exists(string qualifier, string name | this.hasFullyQualifiedName(qualifier, name) |
       if qualifier = "" then result = name else result = qualifier + "." + name
     )
   }
 
   /**
-   * DEPRECATED: Use `hasFullyQualifiedName` instead.
+   * INTERNAL: Do not use.
    *
-   * Holds if this element has the qualified name `qualifier`.`name`.
+   * This is intended for DEBUG ONLY.
+   * Constructing the fully qualified name for all elements in a large codebase
+   * puts severe stress on the string pool.
+   *
+   * Gets the fully qualified name of this element, for example the
+   * fully qualified name of `M` on line 3 is `N.C.M` in
+   *
+   * ```csharp
+   * namespace N {
+   *   class C {
+   *     void M(int i, string s) { }
+   *   }
+   * }
+   * ```
+   *
+   * Unbound generic types, such as `IList<T>`, are represented as
+   * ``System.Collections.Generic.IList`1``.
    */
-  cached
-  deprecated predicate hasQualifiedName(string qualifier, string name) {
-    qualifier = "" and name = this.getName()
+  bindingset[this]
+  pragma[inline_late]
+  final string getFullyQualifiedNameDebug() {
+    exists(string qualifier, string name | this.hasFullyQualifiedName(qualifier, name) |
+      if qualifier = "" then result = name else result = qualifier + "." + name
+    )
   }
 
   /** Holds if this element has the fully qualified name `qualifier`.`name`. */

csharp/ql/lib/semmle/code/csharp/Member.qll

@@ -71,17 +71,11 @@ class Declaration extends NamedElement, @declaration {
 
   override string toString() { result = this.getName() }
 
-  deprecated override predicate hasQualifiedName(string qualifier, string name) {
-    QualifiedName<QualifiedNameInput>::hasQualifiedName(this, qualifier, name)
-  }
-
   override predicate hasFullyQualifiedName(string qualifier, string name) {
     QualifiedName<FullyQualifiedNameInput>::hasQualifiedName(this, qualifier, name)
   }
 
   /**
-   * DEPRECATED: Use `getFullyQualifiedNameWithTypes` instead.
-   *
    * Gets the fully qualified name of this declaration, including types, for example
    * the fully qualified name with types of `M` on line 3 is `N.C.M(int, string)` in
    *
@@ -93,33 +87,13 @@ class Declaration extends NamedElement, @declaration {
    * }
    * ```
    */
-  deprecated string getQualifiedNameWithTypes() {
-    exists(string qual |
-      qual = this.getDeclaringType().getQualifiedName() and
+  deprecated string getFullyQualifiedNameWithTypes() {
+    exists(string fullqual, string qual, string name |
+      this.getDeclaringType().hasFullyQualifiedName(qual, name) and
+      fullqual = getQualifiedName(qual, name) and
       if this instanceof NestedType
-      then result = qual + "+" + this.toStringWithTypes()
-      else result = qual + "." + this.toStringWithTypes()
-    )
-  }
-
-  /**
-   * Gets the fully qualified name of this declaration, including types, for example
-   * the fully qualified name with types of `M` on line 3 is `N.C.M(int, string)` in
-   *
-   * ```csharp
-   * namespace N {
-   *   class C {
-   *     void M(int i, string s) { }
-   *   }
-   * }
-   * ```
-   */
-  string getFullyQualifiedNameWithTypes() {
-    exists(string qual |
-      qual = this.getDeclaringType().getFullyQualifiedName() and
-      if this instanceof NestedType
-      then result = qual + "+" + this.toStringWithTypes()
-      else result = qual + "." + this.toStringWithTypes()
+      then result = fullqual + "+" + this.toStringWithTypes()
+      else result = fullqual + "." + this.toStringWithTypes()
     )
   }
 
@@ -263,17 +237,6 @@ class Member extends Modifiable, @member {
   /** Gets an access to this member. */
   MemberAccess getAnAccess() { result.getTarget() = this }
 
-  /**
-   * DEPRECATED: Use `hasFullyQualifiedName` instead.
-   *
-   * Holds if this member has name `name` and is defined in type `type`
-   * with namespace `namespace`.
-   */
-  cached
-  deprecated final predicate hasQualifiedName(string namespace, string type, string name) {
-    QualifiedName<QualifiedNameInput>::hasQualifiedName(this, namespace, type, name)
-  }
-
   /**
    * Holds if this member has name `name` and is defined in type `type`
    * with namespace `namespace`.

csharp/ql/lib/semmle/code/csharp/Namespace.qll

@@ -38,16 +38,6 @@ class Namespace extends TypeContainer, Declaration, @namespace {
     parent_namespace(result, this)
   }
 
-  /**
-   * Holds if this namespace has the qualified name `qualifier`.`name`.
-   *
-   * For example if the qualified name is `System.Collections.Generic`, then
-   * `qualifier`=`System.Collections` and `name`=`Generic`.
-   */
-  deprecated override predicate hasQualifiedName(string qualifier, string name) {
-    namespaceHasQualifiedName(this, qualifier, name)
-  }
-
   /**
    * Holds if this namespace has the qualified name `qualifier`.`name`.
    *

csharp/ql/lib/semmle/code/csharp/PrintAst.qll

@@ -32,7 +32,10 @@ private predicate shouldPrint(Element e, Location l) {
 }
 
 private predicate isImplicitExpression(ControlFlowElement element) {
-  element.(Expr).isImplicit() and not exists(element.getAChild())
+  element.(Expr).isImplicit() and
+  not element instanceof CastExpr and
+  not element.(OperatorCall).getTarget() instanceof ImplicitConversionOperator and
+  not element instanceof ElementInitializer
 }
 
 private predicate isFilteredCompilerGenerated(Declaration d) {
@@ -291,18 +294,6 @@ class ControlFlowElementNode extends ElementNode {
     controlFlowElement = element and
     // Removing implicit expressions
     not isImplicitExpression(element) and
-    // Removing extra nodes that are generated for an `AssignOperation`
-    not exists(AssignOperation ao |
-      ao.hasExpandedAssignment() and
-      (
-        ao.getExpandedAssignment() = controlFlowElement or
-        ao.getExpandedAssignment().getRValue() = controlFlowElement or
-        ao.getExpandedAssignment().getRValue().(BinaryOperation).getLeftOperand() =
-          controlFlowElement.getParent*() or
-        ao.getExpandedAssignment().getRValue().(OperatorCall).getChild(0) =
-          controlFlowElement.getParent*()
-      )
-    ) and
     not isNotNeeded(element.getParent+()) and
     // LambdaExpr is both a Callable and a ControlFlowElement,
     // print it with the more specific CallableNode
@@ -429,7 +420,7 @@ final class DeclarationWithAccessorsNode extends ElementNode {
     result.(ParametersNode).getParameterizable() = declaration
     or
     childIndex = 2 and
-    result.(ElementNode).getElement() = declaration.(Property).getInitializer().getParent()
+    result.(ElementNode).getElement() = declaration.(Property).getInitializer()
     or
     result.(ElementNode).getElement() =
       rank[childIndex - 2](Element a, string file, int line, int column, string name |
@@ -462,12 +453,7 @@ final class FieldNode extends ElementNode {
     result.(AttributesNode).getAttributable() = field
     or
     childIndex = 1 and
-    field.hasInitializer() and
-    (
-      if field.getDeclaringType() instanceof Enum
-      then result.(ElementNode).getElement() = field.getInitializer()
-      else result.(ElementNode).getElement() = field.getInitializer().getParent()
-    )
+    result.(ElementNode).getElement() = field.getInitializer()
   }
 }
 

csharp/ql/lib/semmle/code/csharp/Variable.qll

@@ -408,7 +408,7 @@ class Field extends Variable, AssignableMember, Attributable, TopLevelExprParent
    * }
    * ```
    */
-  override Expr getInitializer() { result = this.getChildExpr(0).getChildExpr(0) }
+  final override Expr getInitializer() { result = this.getChildExpr(0).getChildExpr(0) }
 
   /**
    * Holds if this field has an initial value. For example, the initial
@@ -515,6 +515,4 @@ class EnumConstant extends MemberConstant {
    * ```
    */
   predicate hasExplicitValue() { exists(this.getInitializer()) }
-
-  override Expr getInitializer() { result = this.getChildExpr(0) }
 }

csharp/ql/lib/semmle/code/csharp/XML.qll

@@ -3,305 +3,67 @@
  */
 
 import semmle.files.FileSystem
+private import codeql.xml.Xml
 
-private class TXmlLocatable =
-  @xmldtd or @xmlelement or @xmlattribute or @xmlnamespace or @xmlcomment or @xmlcharacters;
+private module Input implements InputSig<File, Location> {
+  class XmlLocatableBase = @xmllocatable or @xmlnamespaceable;
 
-/** An XML element that has a location. */
-class XmlLocatable extends @xmllocatable, TXmlLocatable {
-  /** Gets the source location for this element. */
-  Location getLocation() { xmllocations(this, result) }
+  predicate xmllocations_(XmlLocatableBase e, Location loc) { xmllocations(e, loc) }
 
-  /**
-   * Holds if this element is at the specified location.
-   * The location spans column `startcolumn` of line `startline` to
-   * column `endcolumn` of line `endline` in file `filepath`.
-   * For more information, see
-   * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
-   */
-  predicate hasLocationInfo(
-    string filepath, int startline, int startcolumn, int endline, int endcolumn
+  class XmlParentBase = @xmlparent;
+
+  class XmlNamespaceableBase = @xmlnamespaceable;
+
+  class XmlElementBase = @xmlelement;
+
+  class XmlFileBase = File;
+
+  predicate xmlEncoding_(XmlFileBase f, string enc) { xmlEncoding(f, enc) }
+
+  class XmlDtdBase = @xmldtd;
+
+  predicate xmlDTDs_(XmlDtdBase e, string root, string publicId, string systemId, XmlFileBase file) {
+    xmlDTDs(e, root, publicId, systemId, file)
+  }
+
+  predicate xmlElements_(
+    XmlElementBase e, string name, XmlParentBase parent, int idx, XmlFileBase file
   ) {
-    exists(File f, Location l | l = this.getLocation() |
-      locations_default(l, f, startline, startcolumn, endline, endcolumn) and
-      filepath = f.getAbsolutePath()
-    )
+    xmlElements(e, name, parent, idx, file)
   }
 
-  /** Gets a textual representation of this element. */
-  string toString() { none() } // overridden in subclasses
-}
+  class XmlAttributeBase = @xmlattribute;
 
-/**
- * An `XmlParent` is either an `XmlElement` or an `XmlFile`,
- * both of which can contain other elements.
- */
-class XmlParent extends @xmlparent {
-  XmlParent() {
-    // explicitly restrict `this` to be either an `XmlElement` or an `XmlFile`;
-    // the type `@xmlparent` currently also includes non-XML files
-    this instanceof @xmlelement or xmlEncoding(this, _)
+  predicate xmlAttrs_(
+    XmlAttributeBase e, XmlElementBase elementid, string name, string value, int idx,
+    XmlFileBase file
+  ) {
+    xmlAttrs(e, elementid, name, value, idx, file)
   }
 
-  /**
-   * Gets a printable representation of this XML parent.
-   * (Intended to be overridden in subclasses.)
-   */
-  string getName() { none() } // overridden in subclasses
+  class XmlNamespaceBase = @xmlnamespace;
 
-  /** Gets the file to which this XML parent belongs. */
-  XmlFile getFile() { result = this or xmlElements(this, _, _, _, result) }
-
-  /** Gets the child element at a specified index of this XML parent. */
-  XmlElement getChild(int index) { xmlElements(result, _, this, index, _) }
-
-  /** Gets a child element of this XML parent. */
-  XmlElement getAChild() { xmlElements(result, _, this, _, _) }
-
-  /** Gets a child element of this XML parent with the given `name`. */
-  XmlElement getAChild(string name) { xmlElements(result, _, this, _, _) and result.hasName(name) }
-
-  /** Gets a comment that is a child of this XML parent. */
-  XmlComment getAComment() { xmlComments(result, _, this, _) }
-
-  /** Gets a character sequence that is a child of this XML parent. */
-  XmlCharacters getACharactersSet() { xmlChars(result, _, this, _, _, _) }
-
-  /** Gets the depth in the tree. (Overridden in XmlElement.) */
-  int getDepth() { result = 0 }
-
-  /** Gets the number of child XML elements of this XML parent. */
-  int getNumberOfChildren() { result = count(XmlElement e | xmlElements(e, _, this, _, _)) }
-
-  /** Gets the number of places in the body of this XML parent where text occurs. */
-  int getNumberOfCharacterSets() { result = count(int pos | xmlChars(_, _, this, pos, _, _)) }
-
-  /**
-   * Gets the result of appending all the character sequences of this XML parent from
-   * left to right, separated by a space.
-   */
-  string allCharactersString() {
-    result =
-      concat(string chars, int pos | xmlChars(_, chars, this, pos, _, _) | chars, " " order by pos)
+  predicate xmlNs_(XmlNamespaceBase e, string prefixName, string uri, XmlFileBase file) {
+    xmlNs(e, prefixName, uri, file)
   }
 
-  /** Gets the text value contained in this XML parent. */
-  string getTextValue() { result = this.allCharactersString() }
+  predicate xmlHasNs_(XmlNamespaceableBase e, XmlNamespaceBase ns, XmlFileBase file) {
+    xmlHasNs(e, ns, file)
+  }
 
-  /** Gets a printable representation of this XML parent. */
-  string toString() { result = this.getName() }
-}
+  class XmlCommentBase = @xmlcomment;
 
-/** An XML file. */
-class XmlFile extends XmlParent, File {
-  XmlFile() { xmlEncoding(this, _) }
+  predicate xmlComments_(XmlCommentBase e, string text, XmlParentBase parent, XmlFileBase file) {
+    xmlComments(e, text, parent, file)
+  }
 
-  /** Gets a printable representation of this XML file. */
-  override string toString() { result = this.getName() }
+  class XmlCharactersBase = @xmlcharacters;
 
-  /** Gets the name of this XML file. */
-  override string getName() { result = File.super.getAbsolutePath() }
-
-  /** Gets the encoding of this XML file. */
-  string getEncoding() { xmlEncoding(this, result) }
-
-  /** Gets the XML file itself. */
-  override XmlFile getFile() { result = this }
-
-  /** Gets a top-most element in an XML file. */
-  XmlElement getARootElement() { result = this.getAChild() }
-
-  /** Gets a DTD associated with this XML file. */
-  XmlDtd getADtd() { xmlDTDs(result, _, _, _, this) }
-}
-
-/**
- * An XML document type definition (DTD).
- *
- * Example:
- *
- * ```
- * <!ELEMENT person (firstName, lastName?)>
- * <!ELEMENT firstName (#PCDATA)>
- * <!ELEMENT lastName (#PCDATA)>
- * ```
- */
-class XmlDtd extends XmlLocatable, @xmldtd {
-  /** Gets the name of the root element of this DTD. */
-  string getRoot() { xmlDTDs(this, result, _, _, _) }
-
-  /** Gets the public ID of this DTD. */
-  string getPublicId() { xmlDTDs(this, _, result, _, _) }
-
-  /** Gets the system ID of this DTD. */
-  string getSystemId() { xmlDTDs(this, _, _, result, _) }
-
-  /** Holds if this DTD is public. */
-  predicate isPublic() { not xmlDTDs(this, _, "", _, _) }
-
-  /** Gets the parent of this DTD. */
-  XmlParent getParent() { xmlDTDs(this, _, _, _, result) }
-
-  override string toString() {
-    this.isPublic() and
-    result = this.getRoot() + " PUBLIC '" + this.getPublicId() + "' '" + this.getSystemId() + "'"
-    or
-    not this.isPublic() and
-    result = this.getRoot() + " SYSTEM '" + this.getSystemId() + "'"
+  predicate xmlChars_(
+    XmlCharactersBase e, string text, XmlParentBase parent, int idx, int isCDATA, XmlFileBase file
+  ) {
+    xmlChars(e, text, parent, idx, isCDATA, file)
   }
 }
 
-/**
- * An XML element in an XML file.
- *
- * Example:
- *
- * ```
- * <manifest xmlns:android="http://schemas.android.com/apk/res/android"
- *           package="com.example.exampleapp" android:versionCode="1">
- * </manifest>
- * ```
- */
-class XmlElement extends @xmlelement, XmlParent, XmlLocatable {
-  /** Holds if this XML element has the given `name`. */
-  predicate hasName(string name) { name = this.getName() }
-
-  /** Gets the name of this XML element. */
-  override string getName() { xmlElements(this, result, _, _, _) }
-
-  /** Gets the XML file in which this XML element occurs. */
-  override XmlFile getFile() { xmlElements(this, _, _, _, result) }
-
-  /** Gets the parent of this XML element. */
-  XmlParent getParent() { xmlElements(this, _, result, _, _) }
-
-  /** Gets the index of this XML element among its parent's children. */
-  int getIndex() { xmlElements(this, _, _, result, _) }
-
-  /** Holds if this XML element has a namespace. */
-  predicate hasNamespace() { xmlHasNs(this, _, _) }
-
-  /** Gets the namespace of this XML element, if any. */
-  XmlNamespace getNamespace() { xmlHasNs(this, result, _) }
-
-  /** Gets the index of this XML element among its parent's children. */
-  int getElementPositionIndex() { xmlElements(this, _, _, result, _) }
-
-  /** Gets the depth of this element within the XML file tree structure. */
-  override int getDepth() { result = this.getParent().getDepth() + 1 }
-
-  /** Gets an XML attribute of this XML element. */
-  XmlAttribute getAnAttribute() { result.getElement() = this }
-
-  /** Gets the attribute with the specified `name`, if any. */
-  XmlAttribute getAttribute(string name) { result.getElement() = this and result.getName() = name }
-
-  /** Holds if this XML element has an attribute with the specified `name`. */
-  predicate hasAttribute(string name) { exists(this.getAttribute(name)) }
-
-  /** Gets the value of the attribute with the specified `name`, if any. */
-  string getAttributeValue(string name) { result = this.getAttribute(name).getValue() }
-
-  /** Gets a printable representation of this XML element. */
-  override string toString() { result = this.getName() }
-}
-
-/**
- * An attribute that occurs inside an XML element.
- *
- * Examples:
- *
- * ```
- * package="com.example.exampleapp"
- * android:versionCode="1"
- * ```
- */
-class XmlAttribute extends @xmlattribute, XmlLocatable {
-  /** Gets the name of this attribute. */
-  string getName() { xmlAttrs(this, _, result, _, _, _) }
-
-  /** Gets the XML element to which this attribute belongs. */
-  XmlElement getElement() { xmlAttrs(this, result, _, _, _, _) }
-
-  /** Holds if this attribute has a namespace. */
-  predicate hasNamespace() { xmlHasNs(this, _, _) }
-
-  /** Gets the namespace of this attribute, if any. */
-  XmlNamespace getNamespace() { xmlHasNs(this, result, _) }
-
-  /** Gets the value of this attribute. */
-  string getValue() { xmlAttrs(this, _, _, result, _, _) }
-
-  /** Gets a printable representation of this XML attribute. */
-  override string toString() { result = this.getName() + "=" + this.getValue() }
-}
-
-/**
- * A namespace used in an XML file.
- *
- * Example:
- *
- * ```
- * xmlns:android="http://schemas.android.com/apk/res/android"
- * ```
- */
-class XmlNamespace extends XmlLocatable, @xmlnamespace {
-  /** Gets the prefix of this namespace. */
-  string getPrefix() { xmlNs(this, result, _, _) }
-
-  /** Gets the URI of this namespace. */
-  string getUri() { xmlNs(this, _, result, _) }
-
-  /** Holds if this namespace has no prefix. */
-  predicate isDefault() { this.getPrefix() = "" }
-
-  override string toString() {
-    this.isDefault() and result = this.getUri()
-    or
-    not this.isDefault() and result = this.getPrefix() + ":" + this.getUri()
-  }
-}
-
-/**
- * A comment in an XML file.
- *
- * Example:
- *
- * ```
- * <!-- This is a comment. -->
- * ```
- */
-class XmlComment extends @xmlcomment, XmlLocatable {
-  /** Gets the text content of this XML comment. */
-  string getText() { xmlComments(this, result, _, _) }
-
-  /** Gets the parent of this XML comment. */
-  XmlParent getParent() { xmlComments(this, _, result, _) }
-
-  /** Gets a printable representation of this XML comment. */
-  override string toString() { result = this.getText() }
-}
-
-/**
- * A sequence of characters that occurs between opening and
- * closing tags of an XML element, excluding other elements.
- *
- * Example:
- *
- * ```
- * <content>This is a sequence of characters.</content>
- * ```
- */
-class XmlCharacters extends @xmlcharacters, XmlLocatable {
-  /** Gets the content of this character sequence. */
-  string getCharacters() { xmlChars(this, result, _, _, _, _) }
-
-  /** Gets the parent of this character sequence. */
-  XmlParent getParent() { xmlChars(this, _, result, _, _, _) }
-
-  /** Holds if this character sequence is CDATA. */
-  predicate isCDATA() { xmlChars(this, _, _, _, 1, _) }
-
-  /** Gets a printable representation of this XML character sequence. */
-  override string toString() { result = this.getCharacters() }
-}
+import Make<File, Location, Input>

csharp/ql/lib/semmle/code/csharp/commons/QualifiedName.qll

@@ -219,3 +219,28 @@ predicate splitQualifiedName(string qualifiedName, string qualifier, string name
     name = qualifiedName
   )
 }
+
+/**
+ * INTERNAL: Do not use.
+ *
+ * Gets the fully qualified name of this declaration, including types, for example
+ * the fully qualified name with types of `M` on line 3 is `N.C.M(int, string)` in
+ *
+ * ```csharp
+ * namespace N {
+ *   class C {
+ *     void M(int i, string s) { }
+ *   }
+ * }
+ * ```
+ */
+bindingset[d]
+string getFullyQualifiedNameWithTypes(Declaration d) {
+  exists(string fullqual, string qual, string name |
+    d.getDeclaringType().hasFullyQualifiedName(qual, name) and
+    fullqual = getQualifiedName(qual, name) and
+    if d instanceof NestedType
+    then result = fullqual + "+" + d.toStringWithTypes()
+    else result = fullqual + "." + d.toStringWithTypes()
+  )
+}

csharp/ql/lib/semmle/code/csharp/dataflow/SSA.qll

@@ -120,7 +120,7 @@ module Ssa {
           result = prefix + "." + this.getAssignable()
         |
           if f.(Modifiable).isStatic()
-          then prefix = f.getDeclaringType().getFullyQualifiedName()
+          then prefix = f.getDeclaringType().getName()
           else prefix = "this"
         )
       }

csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowPrivate.qll

@@ -1997,6 +1997,8 @@ class FieldOrProperty extends Assignable, Modifiable {
           or
           p.getDeclaringType() instanceof AnonymousClass
         )
+        or
+        p.fromLibrary()
       )
   }
 

csharp/ql/lib/semmle/code/csharp/dataflow/internal/FlowSummaryImpl.qll

@@ -3,6 +3,7 @@
  */
 
 private import csharp
+private import semmle.code.csharp.commons.QualifiedName
 private import semmle.code.csharp.frameworks.system.linq.Expressions
 private import codeql.dataflow.internal.FlowSummaryImpl
 private import codeql.dataflow.internal.AccessPathSyntax as AccessPath
@@ -42,10 +43,18 @@ module Input implements InputSig<Location, DataFlowImplSpecific::CsharpDataFlow>
   string encodeContent(ContentSet c, string arg) {
     c = TElementContent() and result = "Element" and arg = ""
     or
-    exists(Field f | c = TFieldContent(f) and result = "Field" and arg = f.getFullyQualifiedName())
+    exists(Field f, string qualifier, string name |
+      c = TFieldContent(f) and
+      f.hasFullyQualifiedName(qualifier, name) and
+      arg = getQualifiedName(qualifier, name) and
+      result = "Field"
+    )
     or
-    exists(Property p |
-      c = TPropertyContent(p) and result = "Property" and arg = p.getFullyQualifiedName()
+    exists(Property p, string qualifier, string name |
+      c = TPropertyContent(p) and
+      p.hasFullyQualifiedName(qualifier, name) and
+      arg = getQualifiedName(qualifier, name) and
+      result = "Property"
     )
     or
     exists(SyntheticField f |

csharp/ql/lib/semmle/code/csharp/exprs/Expr.qll

@@ -67,7 +67,11 @@ class Expr extends ControlFlowElement, @expr {
    * Holds if this expression is generated by the compiler and does not appear
    * explicitly in the source code.
    */
-  predicate isImplicit() { compiler_generated(this) }
+  final predicate isImplicit() {
+    compiler_generated(this) or
+    this =
+      any(AssignOperation op).getExpandedAssignment().getRValue().getChildExpr(0).getAChildExpr+()
+  }
 
   /**
    * Gets an expression that is the result of stripping (recursively) all

csharp/ql/lib/semmle/code/csharp/security/dataflow/ExternalAPIsQuery.qll

@@ -139,13 +139,13 @@ class ExternalApiUsedWithUntrustedData extends TExternalApi {
 
   /** Gets a textual representation of this element. */
   string toString() {
-    exists(Callable m, int index, string indexString |
+    exists(Callable m, int index, string indexString, string qualifier, string name |
       if index = -1 then indexString = "qualifier" else indexString = "param " + index
     |
       this = TExternalApiParameter(m, index) and
+      m.getDeclaringType().hasFullyQualifiedName(qualifier, name) and
       result =
-        m.getDeclaringType().getFullyQualifiedName() + "." + m.toStringWithTypes() + " [" +
-          indexString + "]"
+        getQualifiedName(qualifier, name) + "." + m.toStringWithTypes() + " [" + indexString + "]"
     )
   }
 }

csharp/ql/src/Telemetry/ExtractorInformation.ql

@@ -87,7 +87,7 @@ predicate extractionIsStandalone(string key, int value) {
     value = 0 and
     not extractionIsStandalone()
   ) and
-  key = "Is buildless extraction"
+  key = "Is extracted with build-mode set to 'none'"
 }
 
 signature module StatsSig {

csharp/ql/test/library-tests/attributes/AttributeElements.ql

@@ -1,9 +1,7 @@
 import csharp
-import semmle.code.csharp.commons.QualifiedName
 
-from Attributable element, Attribute attribute, string qualifier, string name
+from Attributable element, Attribute attribute
 where
   attribute = element.getAnAttribute() and
-  (attribute.fromSource() or element.(Assembly).getName() in ["attributes", "Assembly1"]) and
-  attribute.getType().hasFullyQualifiedName(qualifier, name)
-select element, attribute, getQualifiedName(qualifier, name)
+  (attribute.fromSource() or element.(Assembly).getName() in ["attributes", "Assembly1"])
+select element, attribute, attribute.getType().getFullyQualifiedNameDebug()