DataFlow: use doublyBoundedFastTC in flowPath

.github/workflows/cpp-swift-analysis.yml

@@ -1,55 +0,0 @@
-name: "Code scanning - C++"
-
-on:
-  push:
-    branches:
-      - main
-      - 'rc/*'
-  pull_request:
-    branches:
-      - main
-      - 'rc/*'
-    paths:
-      - 'swift/**'
-      - '.github/codeql/**'
-      - '.github/workflows/cpp-swift-analysis.yml'
-  schedule:
-    - cron: '0 9 * * 1'
-
-jobs:
-  CodeQL-Build:
-
-    runs-on: ubuntu-latest
-
-    permissions:
-      contents: read
-      security-events: write
-      pull-requests: read
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v4
-
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@main
-      # Override language selection by uncommenting this and choosing your languages
-      with:
-        languages: cpp
-        config-file: ./.github/codeql/codeql-config.yml
-  
-    - name: "[Ubuntu] Remove GCC 13 from runner image"
-      shell: bash
-      run: |
-        sudo rm -f /etc/apt/sources.list.d/ubuntu-toolchain-r-ubuntu-test-jammy.list
-        sudo apt-get update
-        sudo apt-get install -y --allow-downgrades libc6=2.35-* libc6-dev=2.35-* libstdc++6=12.3.0-* libgcc-s1=12.3.0-*
-
-    - name: "Build Swift extractor using Bazel"
-      run: |
-        bazel clean --expunge
-        bazel run //swift:create-extractor-pack --nouse_action_cache --noremote_accept_cached --noremote_upload_local_results --spawn_strategy=local --features=-layering_check
-        bazel shutdown
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@main

CODEOWNERS

@@ -1,5 +1,4 @@
 /cpp/ @github/codeql-c-analysis
-/cpp/autobuilder/ @github/codeql-c-extractor
 /csharp/ @github/codeql-csharp
 /go/ @github/codeql-go
 /java/ @github/codeql-java

config/identical-files.json

@@ -251,6 +251,12 @@
     "cpp/ql/src/Security/CWE/CWE-020/SafeExternalAPIFunction.qll",
     "cpp/ql/src/Security/CWE/CWE-020/ir/SafeExternalAPIFunction.qll"
   ],
+  "XML": [
+    "cpp/ql/lib/semmle/code/cpp/XML.qll",
+    "csharp/ql/lib/semmle/code/csharp/XML.qll",
+    "java/ql/lib/semmle/code/xml/XML.qll",
+    "python/ql/lib/semmle/python/xml/XML.qll"
+  ],
   "DuplicationProblems.inc.qhelp": [
     "cpp/ql/src/Metrics/Files/DuplicationProblems.inc.qhelp",
     "javascript/ql/src/Metrics/DuplicationProblems.inc.qhelp",

cpp/autobuilder/Semmle.Autobuild.Cpp.Tests/BuildScripts.cs

@@ -203,8 +203,6 @@ namespace Semmle.Autobuild.Cpp.Tests
         public IList<DiagnosticMessage> Diagnostics { get; } = new List<DiagnosticMessage>();
 
         public void AddEntry(DiagnosticMessage message) => this.Diagnostics.Add(message);
-
-        public void Dispose() { }
     }
 
     /// <summary>
@@ -252,7 +250,12 @@ namespace Semmle.Autobuild.Cpp.Tests
             EndCallbackIn.Add(s);
         }
 
-        CppAutobuilder CreateAutoBuilder(bool isWindows, string? dotnetVersion = null, string cwd = @"C:\Project")
+        CppAutobuilder CreateAutoBuilder(bool isWindows,
+            string? buildless = null, string? solution = null, string? buildCommand = null, string? ignoreErrors = null,
+            string? msBuildArguments = null, string? msBuildPlatform = null, string? msBuildConfiguration = null, string? msBuildTarget = null,
+            string? dotnetArguments = null, string? dotnetVersion = null, string? vsToolsVersion = null,
+            string? nugetRestore = null, string? allSolutions = null,
+            string cwd = @"C:\Project")
         {
             string codeqlUpperLanguage = Language.Cpp.UpperCaseName;
             Actions.GetEnvironmentVariable[$"CODEQL_AUTOBUILDER_{codeqlUpperLanguage}_NO_INDEXING"] = "false";
@@ -262,7 +265,22 @@ namespace Semmle.Autobuild.Cpp.Tests
             Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_DIAGNOSTIC_DIR"] = "";
             Actions.GetEnvironmentVariable["CODEQL_JAVA_HOME"] = @"C:\codeql\tools\java";
             Actions.GetEnvironmentVariable["CODEQL_PLATFORM"] = "win64";
-            Actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_OPTION_DOTNET_VERSION"] = dotnetVersion;
+            Actions.GetEnvironmentVariable["SEMMLE_DIST"] = @"C:\odasa";
+            Actions.GetEnvironmentVariable["SEMMLE_JAVA_HOME"] = @"C:\odasa\tools\java";
+            Actions.GetEnvironmentVariable["SEMMLE_PLATFORM_TOOLS"] = @"C:\odasa\tools";
+            Actions.GetEnvironmentVariable["LGTM_INDEX_VSTOOLS_VERSION"] = vsToolsVersion;
+            Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_ARGUMENTS"] = msBuildArguments;
+            Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_PLATFORM"] = msBuildPlatform;
+            Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_CONFIGURATION"] = msBuildConfiguration;
+            Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_TARGET"] = msBuildTarget;
+            Actions.GetEnvironmentVariable["LGTM_INDEX_DOTNET_ARGUMENTS"] = dotnetArguments;
+            Actions.GetEnvironmentVariable["LGTM_INDEX_DOTNET_VERSION"] = dotnetVersion;
+            Actions.GetEnvironmentVariable["LGTM_INDEX_BUILD_COMMAND"] = buildCommand;
+            Actions.GetEnvironmentVariable["LGTM_INDEX_SOLUTION"] = solution;
+            Actions.GetEnvironmentVariable["LGTM_INDEX_IGNORE_ERRORS"] = ignoreErrors;
+            Actions.GetEnvironmentVariable["LGTM_INDEX_BUILDLESS"] = buildless;
+            Actions.GetEnvironmentVariable["LGTM_INDEX_ALL_SOLUTIONS"] = allSolutions;
+            Actions.GetEnvironmentVariable["LGTM_INDEX_NUGET_RESTORE"] = nugetRestore;
             Actions.GetEnvironmentVariable["ProgramFiles(x86)"] = isWindows ? @"C:\Program Files (x86)" : null;
             Actions.GetCurrentDirectory = cwd;
             Actions.IsWindows = isWindows;

cpp/autobuilder/Semmle.Autobuild.Cpp/CppAutobuilder.cs

@@ -26,6 +26,9 @@ namespace Semmle.Autobuild.Cpp
 
         public override BuildScript GetBuildScript()
         {
+            if (Options.BuildCommand != null)
+                return new BuildCommandRule((_, f) => f(null)).Analyse(this, false);
+
             return
                 // First try MSBuild
                 new MsBuildRule().Analyse(this, true) |

cpp/autobuilder/Semmle.Autobuild.Cpp/Program.cs

@@ -17,7 +17,7 @@ namespace Semmle.Autobuild.Cpp
                 try
                 {
                     Console.WriteLine("CodeQL C++ autobuilder");
-                    using var builder = new CppAutobuilder(actions, options);
+                    var builder = new CppAutobuilder(actions, options);
                     return builder.AttemptBuild();
                 }
                 catch (InvalidEnvironmentException ex)

cpp/downgrades/abfce5c170f93e281948f7689ece373464fdaf87/expr_reuse.ql

@@ -1,7 +0,0 @@
-class Expr extends @expr {
-  string toString() { none() }
-}
-
-from Expr reuse, Expr original
-where expr_reuse(reuse, original, _)
-select reuse, original

cpp/downgrades/abfce5c170f93e281948f7689ece373464fdaf87/expr_types.ql

@@ -1,22 +0,0 @@
-class Expr extends @expr {
-  string toString() { none() }
-}
-
-class Type extends @type {
-  string toString() { none() }
-}
-
-predicate existingType(Expr expr, Type type, int value_category) {
-  expr_types(expr, type, value_category)
-}
-
-predicate reuseType(Expr reuse, Type type, int value_category) {
-  exists(Expr original |
-    expr_reuse(reuse, original, value_category) and
-    expr_types(original, type, _)
-  )
-}
-
-from Expr expr, Type type, int value_category
-where existingType(expr, type, value_category) or reuseType(expr, type, value_category)
-select expr, type, value_category

cpp/downgrades/abfce5c170f93e281948f7689ece373464fdaf87/upgrade.properties

@@ -1,4 +0,0 @@
-description: Add value category to expr_reuse table
-compatibility: full
-expr_reuse.rel: run expr_reuse.qlo
-expr_types.rel: run expr_types.qlo

cpp/ql/lib/CHANGELOG.md

@@ -1,22 +1,3 @@
-## 0.12.10
-
-### New Features
-
-* Added a `TaintInheritingContent` class that can be extended to model taint flowing from a qualifier to a field.
-* Added a predicate `GuardCondition.comparesEq/4` to query whether an expression is compared to a constant. 
-* Added a predicate `GuardCondition.ensuresEq/4` to query whether a basic block is guarded by an expression being equal to a constant.
-* Added a predicate `GuardCondition.comparesLt/4` to query whether an expression is compared to a constant. 
-* Added a predicate `GuardCondition.ensuresLt/4` to query whether a basic block is guarded by an expression being less than a constant.
-* Added a predicate `GuardCondition.valueControls` to query whether a basic block is guarded by a particular `case` of a `switch` statement.
-
-### Minor Analysis Improvements
-
-* Added destructors for temporary objects with extended lifetimes to the intermediate representation.
-
-## 0.12.9
-
-No user-facing changes.
-
 ## 0.12.8
 
 No user-facing changes.

cpp/ql/lib/change-notes/2024-03-15-switches-in-guard-conditions.md

@@ -0,0 +1,4 @@
+---
+category: feature
+---
+* Added a predicate `GuardCondition.valueControls` to query whether a basic block is guarded by a particular `case` of a `switch` statement.

cpp/ql/lib/change-notes/2024-03-19-predicates-for-switches-as-guards-2.md

@@ -0,0 +1,5 @@
+---
+category: feature
+---
+* Added a predicate `GuardCondition.comparesLt/4` to query whether an expression is compared to a constant. 
+* Added a predicate `GuardCondition.ensuresLt/4` to query whether a basic block is guarded by an expression being less than a constant.

cpp/ql/lib/change-notes/2024-03-19-predicates-for-switches-as-guards.md

@@ -0,0 +1,5 @@
+---
+category: feature
+---
+* Added a predicate `GuardCondition.comparesEq/4` to query whether an expression is compared to a constant. 
+* Added a predicate `GuardCondition.ensuresEq/4` to query whether a basic block is guarded by an expression being equal to a constant.

cpp/ql/lib/change-notes/released/0.12.10.md

@@ -1,14 +0,0 @@
-## 0.12.10
-
-### New Features
-
-* Added a `TaintInheritingContent` class that can be extended to model taint flowing from a qualifier to a field.
-* Added a predicate `GuardCondition.comparesEq/4` to query whether an expression is compared to a constant. 
-* Added a predicate `GuardCondition.ensuresEq/4` to query whether a basic block is guarded by an expression being equal to a constant.
-* Added a predicate `GuardCondition.comparesLt/4` to query whether an expression is compared to a constant. 
-* Added a predicate `GuardCondition.ensuresLt/4` to query whether a basic block is guarded by an expression being less than a constant.
-* Added a predicate `GuardCondition.valueControls` to query whether a basic block is guarded by a particular `case` of a `switch` statement.
-
-### Minor Analysis Improvements
-
-* Added destructors for temporary objects with extended lifetimes to the intermediate representation.

cpp/ql/lib/change-notes/released/0.12.9.md

@@ -1,3 +0,0 @@
-## 0.12.9
-
-No user-facing changes.

cpp/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.12.10
+lastReleaseVersion: 0.12.8

cpp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-all
-version: 0.12.11-dev
+version: 0.12.9-dev
 groups: cpp
 dbscheme: semmlecode.cpp.dbscheme
 extractor: cpp
@@ -11,5 +11,4 @@ dependencies:
   codeql/ssa: ${workspace}
   codeql/tutorial: ${workspace}
   codeql/util: ${workspace}
-  codeql/xml: ${workspace}
 warnOnImplicitThis: true

cpp/ql/lib/semmle/code/cpp/PrintAST.qll

@@ -862,7 +862,7 @@ private predicate namedExprChildPredicates(Expr expr, Element ele, string pred)
     or
     expr.(DeleteOrDeleteArrayExpr).getDestructorCall() = ele and pred = "getDestructorCall()"
     or
-    expr.(DeleteOrDeleteArrayExpr).getExprWithReuse() = ele and pred = "getExprWithReuse()"
+    expr.(DeleteOrDeleteArrayExpr).getExpr() = ele and pred = "getExpr()"
     or
     expr.(DestructorFieldDestruction).getExpr() = ele and pred = "getExpr()"
     or

cpp/ql/lib/semmle/code/cpp/Variable.qll

@@ -590,33 +590,6 @@ class TemplateVariable extends Variable {
   Variable getAnInstantiation() { result.isConstructedFrom(this) }
 }
 
-/**
- * A variable that is an instantiation of a template. For example
- * the instantiation `myTemplateVariable<int>` in the following code:
- * ```
- * template<class T>
- * T myTemplateVariable;
- *
- * void caller(int i) {
- *   myTemplateVariable<int> = i;
- * }
- * ```
- */
-class VariableTemplateInstantiation extends Variable {
-  TemplateVariable tv;
-
-  VariableTemplateInstantiation() { tv.getAnInstantiation() = this }
-
-  override string getAPrimaryQlClass() { result = "VariableTemplateInstantiation" }
-
-  /**
-   * Gets the variable template from which this instantiation was instantiated.
-   *
-   * Example: For `int x<int>`, returns `T x`.
-   */
-  TemplateVariable getTemplate() { result = tv }
-}
-
 /**
  * A non-static local variable or parameter that is not part of an
  * uninstantiated template. Uninstantiated templates are purely syntax, and

cpp/ql/lib/semmle/code/cpp/XML.qll

@@ -3,67 +3,305 @@
  */
 
 import semmle.files.FileSystem
-private import codeql.xml.Xml
 
-private module Input implements InputSig<File, Location> {
-  class XmlLocatableBase = @xmllocatable or @xmlnamespaceable;
+private class TXmlLocatable =
+  @xmldtd or @xmlelement or @xmlattribute or @xmlnamespace or @xmlcomment or @xmlcharacters;
 
-  predicate xmllocations_(XmlLocatableBase e, Location loc) { xmllocations(e, loc) }
+/** An XML element that has a location. */
+class XmlLocatable extends @xmllocatable, TXmlLocatable {
+  /** Gets the source location for this element. */
+  Location getLocation() { xmllocations(this, result) }
 
-  class XmlParentBase = @xmlparent;
-
-  class XmlNamespaceableBase = @xmlnamespaceable;
-
-  class XmlElementBase = @xmlelement;
-
-  class XmlFileBase = File;
-
-  predicate xmlEncoding_(XmlFileBase f, string enc) { xmlEncoding(f, enc) }
-
-  class XmlDtdBase = @xmldtd;
-
-  predicate xmlDTDs_(XmlDtdBase e, string root, string publicId, string systemId, XmlFileBase file) {
-    xmlDTDs(e, root, publicId, systemId, file)
-  }
-
-  predicate xmlElements_(
-    XmlElementBase e, string name, XmlParentBase parent, int idx, XmlFileBase file
+  /**
+   * Holds if this element is at the specified location.
+   * The location spans column `startcolumn` of line `startline` to
+   * column `endcolumn` of line `endline` in file `filepath`.
+   * For more information, see
+   * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
+   */
+  predicate hasLocationInfo(
+    string filepath, int startline, int startcolumn, int endline, int endcolumn
   ) {
-    xmlElements(e, name, parent, idx, file)
+    exists(File f, Location l | l = this.getLocation() |
+      locations_default(l, f, startline, startcolumn, endline, endcolumn) and
+      filepath = f.getAbsolutePath()
+    )
   }
 
-  class XmlAttributeBase = @xmlattribute;
+  /** Gets a textual representation of this element. */
+  string toString() { none() } // overridden in subclasses
+}
 
-  predicate xmlAttrs_(
-    XmlAttributeBase e, XmlElementBase elementid, string name, string value, int idx,
-    XmlFileBase file
-  ) {
-    xmlAttrs(e, elementid, name, value, idx, file)
+/**
+ * An `XmlParent` is either an `XmlElement` or an `XmlFile`,
+ * both of which can contain other elements.
+ */
+class XmlParent extends @xmlparent {
+  XmlParent() {
+    // explicitly restrict `this` to be either an `XmlElement` or an `XmlFile`;
+    // the type `@xmlparent` currently also includes non-XML files
+    this instanceof @xmlelement or xmlEncoding(this, _)
   }
 
-  class XmlNamespaceBase = @xmlnamespace;
+  /**
+   * Gets a printable representation of this XML parent.
+   * (Intended to be overridden in subclasses.)
+   */
+  string getName() { none() } // overridden in subclasses
 
-  predicate xmlNs_(XmlNamespaceBase e, string prefixName, string uri, XmlFileBase file) {
-    xmlNs(e, prefixName, uri, file)
+  /** Gets the file to which this XML parent belongs. */
+  XmlFile getFile() { result = this or xmlElements(this, _, _, _, result) }
+
+  /** Gets the child element at a specified index of this XML parent. */
+  XmlElement getChild(int index) { xmlElements(result, _, this, index, _) }
+
+  /** Gets a child element of this XML parent. */
+  XmlElement getAChild() { xmlElements(result, _, this, _, _) }
+
+  /** Gets a child element of this XML parent with the given `name`. */
+  XmlElement getAChild(string name) { xmlElements(result, _, this, _, _) and result.hasName(name) }
+
+  /** Gets a comment that is a child of this XML parent. */
+  XmlComment getAComment() { xmlComments(result, _, this, _) }
+
+  /** Gets a character sequence that is a child of this XML parent. */
+  XmlCharacters getACharactersSet() { xmlChars(result, _, this, _, _, _) }
+
+  /** Gets the depth in the tree. (Overridden in XmlElement.) */
+  int getDepth() { result = 0 }
+
+  /** Gets the number of child XML elements of this XML parent. */
+  int getNumberOfChildren() { result = count(XmlElement e | xmlElements(e, _, this, _, _)) }
+
+  /** Gets the number of places in the body of this XML parent where text occurs. */
+  int getNumberOfCharacterSets() { result = count(int pos | xmlChars(_, _, this, pos, _, _)) }
+
+  /**
+   * Gets the result of appending all the character sequences of this XML parent from
+   * left to right, separated by a space.
+   */
+  string allCharactersString() {
+    result =
+      concat(string chars, int pos | xmlChars(_, chars, this, pos, _, _) | chars, " " order by pos)
   }
 
-  predicate xmlHasNs_(XmlNamespaceableBase e, XmlNamespaceBase ns, XmlFileBase file) {
-    xmlHasNs(e, ns, file)
-  }
+  /** Gets the text value contained in this XML parent. */
+  string getTextValue() { result = this.allCharactersString() }
 
-  class XmlCommentBase = @xmlcomment;
+  /** Gets a printable representation of this XML parent. */
+  string toString() { result = this.getName() }
+}
 
-  predicate xmlComments_(XmlCommentBase e, string text, XmlParentBase parent, XmlFileBase file) {
-    xmlComments(e, text, parent, file)
-  }
+/** An XML file. */
+class XmlFile extends XmlParent, File {
+  XmlFile() { xmlEncoding(this, _) }
 
-  class XmlCharactersBase = @xmlcharacters;
+  /** Gets a printable representation of this XML file. */
+  override string toString() { result = this.getName() }
 
-  predicate xmlChars_(
-    XmlCharactersBase e, string text, XmlParentBase parent, int idx, int isCDATA, XmlFileBase file
-  ) {
-    xmlChars(e, text, parent, idx, isCDATA, file)
+  /** Gets the name of this XML file. */
+  override string getName() { result = File.super.getAbsolutePath() }
+
+  /** Gets the encoding of this XML file. */
+  string getEncoding() { xmlEncoding(this, result) }
+
+  /** Gets the XML file itself. */
+  override XmlFile getFile() { result = this }
+
+  /** Gets a top-most element in an XML file. */
+  XmlElement getARootElement() { result = this.getAChild() }
+
+  /** Gets a DTD associated with this XML file. */
+  XmlDtd getADtd() { xmlDTDs(result, _, _, _, this) }
+}
+
+/**
+ * An XML document type definition (DTD).
+ *
+ * Example:
+ *
+ * ```
+ * <!ELEMENT person (firstName, lastName?)>
+ * <!ELEMENT firstName (#PCDATA)>
+ * <!ELEMENT lastName (#PCDATA)>
+ * ```
+ */
+class XmlDtd extends XmlLocatable, @xmldtd {
+  /** Gets the name of the root element of this DTD. */
+  string getRoot() { xmlDTDs(this, result, _, _, _) }
+
+  /** Gets the public ID of this DTD. */
+  string getPublicId() { xmlDTDs(this, _, result, _, _) }
+
+  /** Gets the system ID of this DTD. */
+  string getSystemId() { xmlDTDs(this, _, _, result, _) }
+
+  /** Holds if this DTD is public. */
+  predicate isPublic() { not xmlDTDs(this, _, "", _, _) }
+
+  /** Gets the parent of this DTD. */
+  XmlParent getParent() { xmlDTDs(this, _, _, _, result) }
+
+  override string toString() {
+    this.isPublic() and
+    result = this.getRoot() + " PUBLIC '" + this.getPublicId() + "' '" + this.getSystemId() + "'"
+    or
+    not this.isPublic() and
+    result = this.getRoot() + " SYSTEM '" + this.getSystemId() + "'"
   }
 }
 
-import Make<File, Location, Input>
+/**
+ * An XML element in an XML file.
+ *
+ * Example:
+ *
+ * ```
+ * <manifest xmlns:android="http://schemas.android.com/apk/res/android"
+ *           package="com.example.exampleapp" android:versionCode="1">
+ * </manifest>
+ * ```
+ */
+class XmlElement extends @xmlelement, XmlParent, XmlLocatable {
+  /** Holds if this XML element has the given `name`. */
+  predicate hasName(string name) { name = this.getName() }
+
+  /** Gets the name of this XML element. */
+  override string getName() { xmlElements(this, result, _, _, _) }
+
+  /** Gets the XML file in which this XML element occurs. */
+  override XmlFile getFile() { xmlElements(this, _, _, _, result) }
+
+  /** Gets the parent of this XML element. */
+  XmlParent getParent() { xmlElements(this, _, result, _, _) }
+
+  /** Gets the index of this XML element among its parent's children. */
+  int getIndex() { xmlElements(this, _, _, result, _) }
+
+  /** Holds if this XML element has a namespace. */
+  predicate hasNamespace() { xmlHasNs(this, _, _) }
+
+  /** Gets the namespace of this XML element, if any. */
+  XmlNamespace getNamespace() { xmlHasNs(this, result, _) }
+
+  /** Gets the index of this XML element among its parent's children. */
+  int getElementPositionIndex() { xmlElements(this, _, _, result, _) }
+
+  /** Gets the depth of this element within the XML file tree structure. */
+  override int getDepth() { result = this.getParent().getDepth() + 1 }
+
+  /** Gets an XML attribute of this XML element. */
+  XmlAttribute getAnAttribute() { result.getElement() = this }
+
+  /** Gets the attribute with the specified `name`, if any. */
+  XmlAttribute getAttribute(string name) { result.getElement() = this and result.getName() = name }
+
+  /** Holds if this XML element has an attribute with the specified `name`. */
+  predicate hasAttribute(string name) { exists(this.getAttribute(name)) }
+
+  /** Gets the value of the attribute with the specified `name`, if any. */
+  string getAttributeValue(string name) { result = this.getAttribute(name).getValue() }
+
+  /** Gets a printable representation of this XML element. */
+  override string toString() { result = this.getName() }
+}
+
+/**
+ * An attribute that occurs inside an XML element.
+ *
+ * Examples:
+ *
+ * ```
+ * package="com.example.exampleapp"
+ * android:versionCode="1"
+ * ```
+ */
+class XmlAttribute extends @xmlattribute, XmlLocatable {
+  /** Gets the name of this attribute. */
+  string getName() { xmlAttrs(this, _, result, _, _, _) }
+
+  /** Gets the XML element to which this attribute belongs. */
+  XmlElement getElement() { xmlAttrs(this, result, _, _, _, _) }
+
+  /** Holds if this attribute has a namespace. */
+  predicate hasNamespace() { xmlHasNs(this, _, _) }
+
+  /** Gets the namespace of this attribute, if any. */
+  XmlNamespace getNamespace() { xmlHasNs(this, result, _) }
+
+  /** Gets the value of this attribute. */
+  string getValue() { xmlAttrs(this, _, _, result, _, _) }
+
+  /** Gets a printable representation of this XML attribute. */
+  override string toString() { result = this.getName() + "=" + this.getValue() }
+}
+
+/**
+ * A namespace used in an XML file.
+ *
+ * Example:
+ *
+ * ```
+ * xmlns:android="http://schemas.android.com/apk/res/android"
+ * ```
+ */
+class XmlNamespace extends XmlLocatable, @xmlnamespace {
+  /** Gets the prefix of this namespace. */
+  string getPrefix() { xmlNs(this, result, _, _) }
+
+  /** Gets the URI of this namespace. */
+  string getUri() { xmlNs(this, _, result, _) }
+
+  /** Holds if this namespace has no prefix. */
+  predicate isDefault() { this.getPrefix() = "" }
+
+  override string toString() {
+    this.isDefault() and result = this.getUri()
+    or
+    not this.isDefault() and result = this.getPrefix() + ":" + this.getUri()
+  }
+}
+
+/**
+ * A comment in an XML file.
+ *
+ * Example:
+ *
+ * ```
+ * <!-- This is a comment. -->
+ * ```
+ */
+class XmlComment extends @xmlcomment, XmlLocatable {
+  /** Gets the text content of this XML comment. */
+  string getText() { xmlComments(this, result, _, _) }
+
+  /** Gets the parent of this XML comment. */
+  XmlParent getParent() { xmlComments(this, _, result, _) }
+
+  /** Gets a printable representation of this XML comment. */
+  override string toString() { result = this.getText() }
+}
+
+/**
+ * A sequence of characters that occurs between opening and
+ * closing tags of an XML element, excluding other elements.
+ *
+ * Example:
+ *
+ * ```
+ * <content>This is a sequence of characters.</content>
+ * ```
+ */
+class XmlCharacters extends @xmlcharacters, XmlLocatable {
+  /** Gets the content of this character sequence. */
+  string getCharacters() { xmlChars(this, result, _, _, _, _) }
+
+  /** Gets the parent of this character sequence. */
+  XmlParent getParent() { xmlChars(this, _, result, _, _, _) }
+
+  /** Holds if this character sequence is CDATA. */
+  predicate isCDATA() { xmlChars(this, _, _, _, 1, _) }
+
+  /** Gets a printable representation of this XML character sequence. */
+  override string toString() { result = this.getCharacters() }
+}

cpp/ql/lib/semmle/code/cpp/exprs/Expr.qll

@@ -1015,33 +1015,8 @@ class DeleteOrDeleteArrayExpr extends Expr, TDeleteOrDeleteArrayExpr {
   Expr getExpr() {
     // If there is a destructor call, the object being deleted is the qualifier
     // otherwise it is the third child.
-    exists(Expr exprWithReuse | exprWithReuse = this.getExprWithReuse() |
-      if not exprWithReuse instanceof ReuseExpr
-      then result = exprWithReuse
-      else result = this.getDestructorCall().getQualifier()
-    )
+    result = this.getChild(3) or result = this.getDestructorCall().getQualifier()
   }
-
-  /**
-   * Gets the object or array being deleted, and gets a `ReuseExpr` when there
-   * is a destructor call and the object is also the qualifier of the call.
-   *
-   * For example, given:
-   * ```
-   * struct HasDestructor { ~HasDestructor(); };
-   * struct PlainOldData { int x, char y; };
-   *
-   * void f(HasDestructor* hasDestructor, PlainOldData* pod) {
-   *   delete hasDestructor;
-   *   delete pod;
-   * }
-   * ```
-   * This predicate yields a `ReuseExpr` for `delete hasDestructor`, as the
-   * the deleted expression has a destructor, and that expression is also
-   * the qualifier of the destructor call. In the case of `delete pod` the
-   * predicate does not yield a `ReuseExpr`, as there is no destructor call.
-   */
-  Expr getExprWithReuse() { result = this.getChild(3) }
 }
 
 /**
@@ -1365,23 +1340,5 @@ class ReuseExpr extends Expr, @reuseexpr {
   /**
    * Gets the expression that is being re-used.
    */
-  Expr getReusedExpr() {
-    // In the case of a prvalue, the extractor outputs the expression
-    // before conversion, but the converted expression is intended.
-    if this.isPRValueCategory()
-    then result = this.getBaseReusedExpr().getFullyConverted()
-    else result = this.getBaseReusedExpr()
-  }
-
-  private Expr getBaseReusedExpr() {
-    expr_reuse(underlyingElement(this), unresolveElement(result), _)
-  }
-
-  override Type getType() { result = this.getReusedExpr().getType() }
-
-  override predicate isLValueCategory() { expr_reuse(underlyingElement(this), _, 3) }
-
-  override predicate isXValueCategory() { expr_reuse(underlyingElement(this), _, 2) }
-
-  override predicate isPRValueCategory() { expr_reuse(underlyingElement(this), _, 1) }
+  Expr getReusedExpr() { expr_reuse(underlyingElement(this), unresolveElement(result)) }
 }

cpp/ql/lib/semmle/code/cpp/ir/dataflow/FlowSteps.qll

@@ -1,20 +0,0 @@
-/**
- * This file provides an abstract class that can be used to model additional
- * object-to-field taint-flow.
- */
-
-private import codeql.util.Unit
-private import semmle.code.cpp.dataflow.new.DataFlow
-
-/**
- * A `Content` that should be implicitly regarded as tainted whenever an object with such `Content`
- * is itself tainted.
- *
- * For example, if we had a type `struct Container { int field; }`, then by default a tainted
- * `Container` and a `Container` with a tainted `int` stored in its `field` are distinct.
- *
- * If `any(DataFlow::FieldContent fc | fc.getField().hasQualifiedName("Container", "field"))` was
- * included in this type however, then a tainted `Container` would imply that its `field` is also
- * tainted (but not vice versa).
- */
-abstract class TaintInheritingContent extends DataFlow::Content { }

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowUtil.qll

@@ -2301,8 +2301,8 @@ private import ContentStars
 
 /** A reference through a non-union instance field. */
 class FieldContent extends Content, TFieldContent {
-  private Field f;
-  private int indirectionIndex;
+  Field f;
+  int indirectionIndex;
 
   FieldContent() { this = TFieldContent(f, indirectionIndex) }
 
@@ -2329,9 +2329,9 @@ class FieldContent extends Content, TFieldContent {
 
 /** A reference through an instance field of a union. */
 class UnionContent extends Content, TUnionContent {
-  private Union u;
-  private int indirectionIndex;
-  private int bytes;
+  Union u;
+  int indirectionIndex;
+  int bytes;
 
   UnionContent() { this = TUnionContent(u, bytes, indirectionIndex) }
 

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TaintTrackingUtil.qll

@@ -6,7 +6,6 @@ private import semmle.code.cpp.models.interfaces.SideEffect
 private import DataFlowUtil
 private import DataFlowPrivate
 private import SsaInternals as Ssa
-private import semmle.code.cpp.ir.dataflow.FlowSteps
 
 /**
  * Holds if taint propagates from `nodeFrom` to `nodeTo` in exactly one local
@@ -38,12 +37,6 @@ predicate localAdditionalTaintStep(DataFlow::Node nodeFrom, DataFlow::Node nodeT
   )
   or
   any(Ssa::Indirection ind).isAdditionalTaintStep(nodeFrom, nodeTo)
-  or
-  // object->field conflation for content that is a `TaintInheritingContent`.
-  exists(DataFlow::ContentSet f |
-    readStep(nodeFrom, f, nodeTo) and
-    f.getAReadContent() instanceof TaintInheritingContent
-  )
 }
 
 /**

cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/TranslatedElement.qll

@@ -130,6 +130,11 @@ private predicate ignoreExprAndDescendants(Expr expr) {
   or
   // suppress destructors of temporary variables until proper support is added for them.
   exists(Expr parent | parent.getAnImplicitDestructorCall() = expr)
+  or
+  exists(Stmt parent |
+    parent.getAnImplicitDestructorCall() = expr and
+    expr.(DestructorCall).getQualifier() instanceof ReuseExpr
+  )
 }
 
 /**
@@ -150,6 +155,11 @@ private predicate ignoreExprOnly(Expr expr) {
   or
   not translateFunction(getEnclosingFunction(expr)) and
   not Raw::varHasIRFunc(getEnclosingVariable(expr))
+  or
+  exists(DeleteOrDeleteArrayExpr deleteExpr |
+    // Ignore the destructor call, because the duplicated qualifier breaks control flow.
+    deleteExpr.getDestructorCall() = expr
+  )
 }
 
 /**

cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/TranslatedExpr.qll

@@ -2245,11 +2245,7 @@ class TranslatedDeleteOrDeleteArrayExpr extends TranslatedNonConstantExpr, Trans
 
   final override Type getCallResultType() { result = expr.getType() }
 
-  final override TranslatedExpr getQualifier() {
-    result = getTranslatedExpr(expr.getDestructorCall())
-  }
-
-  final override Instruction getQualifierResult() { none() }
+  final override TranslatedExpr getQualifier() { none() }
 
   final override predicate hasArguments() {
     // All deallocator calls have at least one argument.
@@ -2264,7 +2260,7 @@ class TranslatedDeleteOrDeleteArrayExpr extends TranslatedNonConstantExpr, Trans
   final override TranslatedExpr getArgument(int index) {
     // The only argument we define is the pointer to be deallocated.
     index = 0 and
-    result = getTranslatedExpr(expr.getExprWithReuse().getFullyConverted())
+    result = getTranslatedExpr(expr.getExpr().getFullyConverted())
   }
 
   final override predicate mayThrowException() {
@@ -2773,50 +2769,6 @@ class TranslatedTemporaryObjectExpr extends TranslatedNonConstantExpr,
   final override Instruction getResult() { result = this.getTargetAddress() }
 }
 
-/**
- * IR translation of a `ReuseExpr`.
- *
- * This translation produces a copy of the glvalue instruction holding the (unconverted) result
- * of the reused expression. In the case where the original expression was a prvalue, the
- * result will be a copy of the glvalue operand of a `TranslatedLoad`.
- */
-class TranslatedReuseExpr extends TranslatedNonConstantExpr {
-  override ReuseExpr expr;
-
-  override Instruction getFirstInstruction(EdgeKind kind) {
-    result = this.getInstruction(OnlyInstructionTag()) and
-    kind instanceof GotoEdge
-  }
-
-  override predicate hasInstruction(Opcode opcode, InstructionTag tag, CppType resultType) {
-    opcode instanceof Opcode::CopyValue and
-    tag instanceof OnlyInstructionTag and
-    resultType = this.getResultType()
-  }
-
-  override Instruction getResult() { result = this.getInstruction(OnlyInstructionTag()) }
-
-  override Instruction getInstructionSuccessorInternal(InstructionTag tag, EdgeKind kind) {
-    tag = OnlyInstructionTag() and
-    kind instanceof GotoEdge and
-    result = this.getParent().getChildSuccessor(this, kind)
-  }
-
-  override TranslatedElement getChildInternal(int id) { none() }
-
-  override Instruction getALastInstructionInternal() {
-    result = this.getInstruction(OnlyInstructionTag())
-  }
-
-  override Instruction getInstructionRegisterOperand(InstructionTag tag, OperandTag operandTag) {
-    tag = OnlyInstructionTag() and
-    operandTag instanceof UnaryOperandTag and
-    if getTranslatedExpr(expr.getReusedExpr()) instanceof TranslatedLoad
-    then result = getTranslatedExpr(expr.getReusedExpr()).(TranslatedLoad).getOperand().getResult()
-    else result = getTranslatedExpr(expr.getReusedExpr()).getResult()
-  }
-}
-
 /**
  * IR translation of a `throw` expression.
  */

cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/TranslatedStmt.qll

@@ -248,9 +248,19 @@ abstract class TranslatedStmt extends TranslatedElement, TTranslatedStmt {
   final override TranslatedElement getChild(int id) {
     result = this.getChildInternal(id)
     or
-    exists(int destructorIndex |
+    exists(int destructorIndex, int tempDestructorCount |
       result.(TranslatedExpr).getExpr() = stmt.getImplicitDestructorCall(destructorIndex) and
-      id = this.getFirstDestructorCallIndex() + destructorIndex
+      id = this.getFirstDestructorCallIndex() + destructorIndex - tempDestructorCount and
+      // suppress destructors of temporary variables until proper support is added for them.
+      tempDestructorCount =
+        count(DestructorCall call, int tempIndex |
+          stmt.getImplicitDestructorCall(tempIndex) = call and
+          tempIndex < destructorIndex and
+          call.getQualifier() instanceof ReuseExpr
+        |
+          call
+        ) and
+      not stmt.getImplicitDestructorCall(destructorIndex).getQualifier() instanceof ReuseExpr
     )
   }
 
@@ -261,7 +271,11 @@ abstract class TranslatedStmt extends TranslatedElement, TTranslatedStmt {
   }
 
   final override predicate hasAnImplicitDestructorCall() {
-    exists(stmt.getAnImplicitDestructorCall())
+    exists(stmt.getAnImplicitDestructorCall()) and
+    // suppress destructors of temporary variables until proper support is added for them.
+    exists(Expr expr | stmt.getAnImplicitDestructorCall().getQualifier() = expr |
+      not expr instanceof ReuseExpr
+    )
   }
 
   final override string toString() { result = stmt.toString() }

cpp/ql/lib/semmle/code/cpp/models/Models.qll

@@ -41,4 +41,3 @@ private import implementations.SqLite3
 private import implementations.PostgreSql
 private import implementations.System
 private import implementations.StructuredExceptionHandling
-private import implementations.Fopen

cpp/ql/lib/semmle/code/cpp/models/implementations/Fopen.qll

@@ -1,50 +0,0 @@
-/**
- * Provides implementation classes modeling `fopen` and various similar
- * functions. See `semmle.code.cpp.models.Models` for usage information.
- */
-
-import semmle.code.cpp.models.interfaces.Alias
-import semmle.code.cpp.models.interfaces.SideEffect
-
-/** The function `fopen` and friends. */
-private class Fopen extends Function, AliasFunction, SideEffectFunction {
-  Fopen() {
-    this.hasGlobalOrStdName(["fopen", "fopen_s", "freopen"])
-    or
-    this.hasGlobalName(["_open", "_wfopen", "_fsopen", "_wfsopen", "_wopen"])
-  }
-
-  override predicate hasOnlySpecificWriteSideEffects() { any() }
-
-  override predicate hasOnlySpecificReadSideEffects() { any() }
-
-  override predicate parameterEscapesOnlyViaReturn(int i) { none() }
-
-  override predicate parameterNeverEscapes(int index) {
-    // None of the parameters escape
-    this.getParameter(index).getUnspecifiedType() instanceof PointerType
-  }
-
-  override predicate hasSpecificReadSideEffect(ParameterIndex i, boolean buffer) {
-    (
-      this.hasGlobalOrStdName(["fopen", "fopen_s"])
-      or
-      this.hasGlobalName(["_wfopen", "_fsopen", "_wfsopen"])
-    ) and
-    i = [0, 1] and
-    buffer = true
-    or
-    this.hasGlobalOrStdName("freopen") and
-    (
-      i = [0, 1] and
-      buffer = true
-      or
-      i = 2 and
-      buffer = false
-    )
-    or
-    this.hasGlobalName(["_open", "_wopen"]) and
-    i = 0 and
-    buffer = true
-  }
-}

cpp/ql/lib/semmlecode.cpp.dbscheme

@@ -1515,8 +1515,7 @@ exprs(
 
 expr_reuse(
     int reuse: @expr ref,
-    int original: @expr ref,
-    int value_category: int ref
+    int original: @expr ref
 )
 
 /*

cpp/ql/lib/upgrades/aa7ff0ab32cd4674f6ab731d32fea64116997b05/expr_reuse.ql

@@ -1,7 +0,0 @@
-class Expr extends @expr {
-  string toString() { none() }
-}
-
-from Expr reuse, Expr original, int value_category
-where expr_reuse(reuse, original) and expr_types(original, _, value_category)
-select reuse, original, value_category

cpp/ql/lib/upgrades/aa7ff0ab32cd4674f6ab731d32fea64116997b05/upgrade.properties

@@ -1,3 +0,0 @@
-description: Add value category to expr_reuse table
-compatibility: full
-expr_reuse.rel: run expr_reuse.qlo

cpp/ql/src/CHANGELOG.md

@@ -1,23 +1,3 @@
-## 0.9.9
-
-### New Queries
-
-* Added a new query, `cpp/type-confusion`, to detect casts to invalid types.
-
-### Query Metadata Changes
-
-* `@precision medium` metadata was added to the `cpp/boost/tls-settings-misconfiguration` and `cpp/boost/use-of-deprecated-hardcoded-security-protocol` queries, and these queries are now included in the security-extended suite. The `@name` metadata of these queries were also updated.
-
-### Minor Analysis Improvements
-
-* The "Missing return-value check for a 'scanf'-like function" query (`cpp/missing-check-scanf`) has been converted to a `path-problem` query.
-* The "Potentially uninitialized local variable" query (`cpp/uninitialized-local`) has been converted to a `path-problem` query.
-* Added models for `GLib` allocation and deallocation functions.
-
-## 0.9.8
-
-No user-facing changes.
-
 ## 0.9.7
 
 No user-facing changes.

cpp/ql/src/Critical/DoubleFree.ql

@@ -37,5 +37,6 @@ where
   DoubleFree::flowPath(source, sink) and
   isFree(source.getNode(), _, _, dealloc) and
   isFree(sink.getNode(), e2)
-select sink.getNode(), source, sink, "Memory pointed to by $@ may already have been freed by $@.",
-  e2, e2.toString(), dealloc, dealloc.toString()
+select sink.getNode(), source, sink,
+  "Memory pointed to by '" + e2.toString() + "' may already have been freed by $@.", dealloc,
+  dealloc.toString()

cpp/ql/src/change-notes/2024-03-05-type-confusion-query.md

@@ -0,0 +1,4 @@
+---
+category: newQuery
+---
+* Added a new query, `cpp/type-confusion`, to detect casts to invalid types.

cpp/ql/src/change-notes/2024-03-13-glib-alloc-and-dealloc.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Added models for `GLib` allocation and deallocation functions.

cpp/ql/src/change-notes/2024-03-18-uninitialized-local-path-problem.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* The "Potentially uninitialized local variable" query (`cpp/uninitialized-local`) has been converted to a `path-problem` query.

cpp/ql/src/change-notes/2024-03-20-missing-check-scanf-path-problem.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* The "Missing return-value check for a 'scanf'-like function" query (`cpp/missing-check-scanf`) has been converted to a `path-problem` query.

cpp/ql/src/change-notes/2024-03-22-boost-ssl.md

@@ -0,0 +1,4 @@
+---
+category: queryMetadata
+---
+* `@precision medium` metadata was added to the `cpp/boost/tls-settings-misconfiguration` and `cpp/boost/use-of-deprecated-hardcoded-security-protocol` queries, and these queries are now included in the security-extended suite. The `@name` metadata of these queries were also updated.

cpp/ql/src/change-notes/released/0.9.8.md

@@ -1,3 +0,0 @@
-## 0.9.8
-
-No user-facing changes.

cpp/ql/src/change-notes/released/0.9.9.md

@@ -1,15 +0,0 @@
-## 0.9.9
-
-### New Queries
-
-* Added a new query, `cpp/type-confusion`, to detect casts to invalid types.
-
-### Query Metadata Changes
-
-* `@precision medium` metadata was added to the `cpp/boost/tls-settings-misconfiguration` and `cpp/boost/use-of-deprecated-hardcoded-security-protocol` queries, and these queries are now included in the security-extended suite. The `@name` metadata of these queries were also updated.
-
-### Minor Analysis Improvements
-
-* The "Missing return-value check for a 'scanf'-like function" query (`cpp/missing-check-scanf`) has been converted to a `path-problem` query.
-* The "Potentially uninitialized local variable" query (`cpp/uninitialized-local`) has been converted to a `path-problem` query.
-* Added models for `GLib` allocation and deallocation functions.

cpp/ql/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.9.9
+lastReleaseVersion: 0.9.7

cpp/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-queries
-version: 0.9.10-dev
+version: 0.9.8-dev
 groups:
   - cpp
   - queries

cpp/ql/test/examples/expressions/PrintAST.expected

@@ -426,14 +426,11 @@ DestructorCall.cpp:
 #   12|           getQualifier(): [VariableAccess] c
 #   12|               Type = [PointerType] C *
 #   12|               ValueCategory = prvalue(load)
-#   12|         getExprWithReuse(): [ReuseExpr] reuse of c
-#   12|             Type = [PointerType] C *
-#   12|             ValueCategory = prvalue
 #   13|     getStmt(1): [ExprStmt] ExprStmt
 #   13|       getExpr(): [DeleteExpr] delete
 #   13|           Type = [VoidType] void
 #   13|           ValueCategory = prvalue
-#   13|         getExprWithReuse(): [VariableAccess] d
+#   13|         getExpr(): [VariableAccess] d
 #   13|             Type = [PointerType] D *
 #   13|             ValueCategory = prvalue(load)
 #   14|     getStmt(2): [ReturnStmt] return ...

cpp/ql/test/library-tests/compiler_generated/compilerGenerated.expected

@@ -14,7 +14,6 @@
 | cpp.cpp:10:7:10:7 | operator= | Function |
 | cpp.cpp:10:7:10:7 | ~MyClass | Function |
 | cpp.cpp:15:5:15:12 | call to ~MyClass | Expr |
-| cpp.cpp:15:12:15:12 | reuse of m | Expr |
 | cpp.cpp:16:1:16:1 | return ... | Stmt |
 | file://:0:0:0:0 | operator delete | Function |
 | file://:0:0:0:0 | operator new | Function |

cpp/ql/test/library-tests/dataflow/taint-tests/localTaint.expected

@@ -6676,7 +6676,6 @@ WARNING: Module TaintTracking has been deprecated and may be removed in future (
 | taint.cpp:757:7:757:10 | path | taint.cpp:759:8:759:11 | path |  |
 | taint.cpp:758:21:758:24 | ref arg path | taint.cpp:759:8:759:11 | path |  |
 | taint.cpp:759:8:759:11 | path | taint.cpp:759:7:759:11 | * ... |  |
-| taint.cpp:769:37:769:42 | call to source | taint.cpp:770:7:770:9 | obj |  |
 | vector.cpp:16:43:16:49 | source1 | vector.cpp:17:26:17:32 | source1 |  |
 | vector.cpp:16:43:16:49 | source1 | vector.cpp:31:38:31:44 | source1 |  |
 | vector.cpp:17:21:17:33 | call to vector | vector.cpp:19:14:19:14 | v |  |

cpp/ql/test/library-tests/dataflow/taint-tests/taint.cpp

@@ -757,15 +757,4 @@ void test_call_sprintf() {
 	char path[10];
 	call_sprintf_twice(path, indirect_source());
 	sink(*path); // $ ast,ir
-}
-
-struct TaintInheritingContentObject {
-	int flowFromObject;
-};
-
-TaintInheritingContentObject source(bool);
-
-void test_TaintInheritingContent() {
-	TaintInheritingContentObject obj = source(true);
-	sink(obj.flowFromObject); // $ ir MISSING: ast
 }

cpp/ql/test/library-tests/dataflow/taint-tests/taint.ql

@@ -76,24 +76,6 @@ module AstTest {
 module IRTest {
   private import semmle.code.cpp.ir.IR
   private import semmle.code.cpp.ir.dataflow.TaintTracking
-  private import semmle.code.cpp.ir.dataflow.FlowSteps
-
-  /**
-   * Object->field flow when the object is of type
-   * TaintInheritingContentObject and the field is named
-   * flowFromObject
-   */
-  class TaintInheritingContentTest extends TaintInheritingContent, DataFlow::FieldContent {
-    TaintInheritingContentTest() {
-      exists(Struct o, Field f |
-        this.getField() = f and
-        f = o.getAField() and
-        o.hasGlobalName("TaintInheritingContentObject") and
-        f.hasName("flowFromObject") and
-        this.getIndirectionIndex() = 1
-      )
-    }
-  }
 
   /** Common data flow configuration to be used by tests. */
   module TestAllocationConfig implements DataFlow::ConfigSig {

cpp/ql/test/library-tests/ir/ir/ir.cpp

@@ -1933,20 +1933,6 @@ namespace missing_declaration_entries {
         Bar2<int> b;
         b.two_missing_variable_declaration_entries();
     }
-
-    template<typename T> struct Bar3 {
-
-        int two_more_missing_variable_declaration_entries() {
-            extern int g;
-            int z(float);
-            return g;
-        }
-    };
-
-    void test3() {
-        Bar3<int> b;
-        b.two_more_missing_variable_declaration_entries();
-    }
 }
 
 template<typename T> T global_template = 42;
@@ -2431,20 +2417,4 @@ void initialization_with_temp_destructor() {
         y += x;
 }
 
-void param_with_destructor_by_value(ClassWithDestructor c) {
-    // The call to ~ClassWithDestructor::ClassWithDestructor() seems to be missing here.
-}
-
-void param_with_destructor_by_pointer(ClassWithDestructor* c) {
-    // No destructor call should be here
-}
-
-void param_with_destructor_by_ref(ClassWithDestructor& c) {
-    // No destructor call should be here
-}
-
-void param_with_destructor_by_rref(ClassWithDestructor&& c) {
-    // No destructor call should be here
-}
-
 // semmle-extractor-options: -std=c++20 --clang

cpp/ql/test/query-tests/Critical/MemoryFreed/DoubleFree.expected

@@ -41,16 +41,16 @@ nodes
 | test_free.cpp:302:12:302:14 | buf | semmle.label | buf |
 subpaths
 #select
-| test_free.cpp:14:10:14:10 | a | test_free.cpp:11:10:11:10 | pointer to free output argument | test_free.cpp:14:10:14:10 | a | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:14:10:14:10 | a | a | test_free.cpp:11:5:11:8 | call to free | call to free |
-| test_free.cpp:31:27:31:27 | a | test_free.cpp:30:10:30:10 | pointer to free output argument | test_free.cpp:31:27:31:27 | a | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:31:27:31:27 | a | a | test_free.cpp:30:5:30:8 | call to free | call to free |
-| test_free.cpp:37:27:37:27 | a | test_free.cpp:35:10:35:10 | pointer to free output argument | test_free.cpp:37:27:37:27 | a | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:37:27:37:27 | a | a | test_free.cpp:35:5:35:8 | call to free | call to free |
-| test_free.cpp:46:10:46:10 | a | test_free.cpp:42:27:42:27 | pointer to free output argument | test_free.cpp:46:10:46:10 | a | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:46:10:46:10 | a | a | test_free.cpp:42:22:42:25 | call to free | call to free |
-| test_free.cpp:46:10:46:10 | a | test_free.cpp:44:27:44:27 | pointer to free output argument | test_free.cpp:46:10:46:10 | a | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:46:10:46:10 | a | a | test_free.cpp:44:22:44:25 | call to free | call to free |
-| test_free.cpp:51:10:51:10 | a | test_free.cpp:50:27:50:27 | pointer to free output argument | test_free.cpp:51:10:51:10 | a | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:51:10:51:10 | a | a | test_free.cpp:50:22:50:25 | call to free | call to free |
-| test_free.cpp:72:14:72:14 | a | test_free.cpp:69:10:69:10 | pointer to free output argument | test_free.cpp:72:14:72:14 | a | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:72:14:72:14 | a | a | test_free.cpp:69:5:69:8 | call to free | call to free |
-| test_free.cpp:85:12:85:12 | a | test_free.cpp:83:12:83:12 | pointer to operator delete output argument | test_free.cpp:85:12:85:12 | a | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:85:12:85:12 | a | a | test_free.cpp:83:5:83:13 | delete | delete |
-| test_free.cpp:103:10:103:10 | a | test_free.cpp:101:10:101:10 | pointer to free output argument | test_free.cpp:103:10:103:10 | a | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:103:10:103:10 | a | a | test_free.cpp:101:5:101:8 | call to free | call to free |
-| test_free.cpp:129:10:129:11 | * ... | test_free.cpp:128:10:128:11 | pointer to free output argument | test_free.cpp:129:10:129:11 | * ... | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:129:10:129:11 | * ... | * ... | test_free.cpp:128:5:128:8 | call to free | call to free |
-| test_free.cpp:154:10:154:10 | a | test_free.cpp:152:27:152:27 | pointer to free output argument | test_free.cpp:154:10:154:10 | a | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:154:10:154:10 | a | a | test_free.cpp:152:22:152:25 | call to free | call to free |
-| test_free.cpp:209:10:209:10 | a | test_free.cpp:207:10:207:10 | pointer to free output argument | test_free.cpp:209:10:209:10 | a | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:209:10:209:10 | a | a | test_free.cpp:207:5:207:8 | call to free | call to free |
-| test_free.cpp:302:12:302:14 | buf | test_free.cpp:301:12:301:14 | pointer to g_free output argument | test_free.cpp:302:12:302:14 | buf | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:302:12:302:14 | buf | buf | test_free.cpp:301:5:301:10 | call to g_free | call to g_free |
+| test_free.cpp:14:10:14:10 | a | test_free.cpp:11:10:11:10 | pointer to free output argument | test_free.cpp:14:10:14:10 | a | Memory pointed to by 'a' may already have been freed by $@. | test_free.cpp:11:5:11:8 | call to free | call to free |
+| test_free.cpp:31:27:31:27 | a | test_free.cpp:30:10:30:10 | pointer to free output argument | test_free.cpp:31:27:31:27 | a | Memory pointed to by 'a' may already have been freed by $@. | test_free.cpp:30:5:30:8 | call to free | call to free |
+| test_free.cpp:37:27:37:27 | a | test_free.cpp:35:10:35:10 | pointer to free output argument | test_free.cpp:37:27:37:27 | a | Memory pointed to by 'a' may already have been freed by $@. | test_free.cpp:35:5:35:8 | call to free | call to free |
+| test_free.cpp:46:10:46:10 | a | test_free.cpp:42:27:42:27 | pointer to free output argument | test_free.cpp:46:10:46:10 | a | Memory pointed to by 'a' may already have been freed by $@. | test_free.cpp:42:22:42:25 | call to free | call to free |
+| test_free.cpp:46:10:46:10 | a | test_free.cpp:44:27:44:27 | pointer to free output argument | test_free.cpp:46:10:46:10 | a | Memory pointed to by 'a' may already have been freed by $@. | test_free.cpp:44:22:44:25 | call to free | call to free |
+| test_free.cpp:51:10:51:10 | a | test_free.cpp:50:27:50:27 | pointer to free output argument | test_free.cpp:51:10:51:10 | a | Memory pointed to by 'a' may already have been freed by $@. | test_free.cpp:50:22:50:25 | call to free | call to free |
+| test_free.cpp:72:14:72:14 | a | test_free.cpp:69:10:69:10 | pointer to free output argument | test_free.cpp:72:14:72:14 | a | Memory pointed to by 'a' may already have been freed by $@. | test_free.cpp:69:5:69:8 | call to free | call to free |
+| test_free.cpp:85:12:85:12 | a | test_free.cpp:83:12:83:12 | pointer to operator delete output argument | test_free.cpp:85:12:85:12 | a | Memory pointed to by 'a' may already have been freed by $@. | test_free.cpp:83:5:83:13 | delete | delete |
+| test_free.cpp:103:10:103:10 | a | test_free.cpp:101:10:101:10 | pointer to free output argument | test_free.cpp:103:10:103:10 | a | Memory pointed to by 'a' may already have been freed by $@. | test_free.cpp:101:5:101:8 | call to free | call to free |
+| test_free.cpp:129:10:129:11 | * ... | test_free.cpp:128:10:128:11 | pointer to free output argument | test_free.cpp:129:10:129:11 | * ... | Memory pointed to by '* ...' may already have been freed by $@. | test_free.cpp:128:5:128:8 | call to free | call to free |
+| test_free.cpp:154:10:154:10 | a | test_free.cpp:152:27:152:27 | pointer to free output argument | test_free.cpp:154:10:154:10 | a | Memory pointed to by 'a' may already have been freed by $@. | test_free.cpp:152:22:152:25 | call to free | call to free |
+| test_free.cpp:209:10:209:10 | a | test_free.cpp:207:10:207:10 | pointer to free output argument | test_free.cpp:209:10:209:10 | a | Memory pointed to by 'a' may already have been freed by $@. | test_free.cpp:207:5:207:8 | call to free | call to free |
+| test_free.cpp:302:12:302:14 | buf | test_free.cpp:301:12:301:14 | pointer to g_free output argument | test_free.cpp:302:12:302:14 | buf | Memory pointed to by 'buf' may already have been freed by $@. | test_free.cpp:301:5:301:10 | call to g_free | call to g_free |

csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs

@@ -218,8 +218,6 @@ namespace Semmle.Autobuild.CSharp.Tests
         public IList<DiagnosticMessage> Diagnostics { get; } = new List<DiagnosticMessage>();
 
         public void AddEntry(DiagnosticMessage message) => this.Diagnostics.Add(message);
-
-        public void Dispose() { }
     }
 
     /// <summary>
@@ -401,8 +399,10 @@ namespace Semmle.Autobuild.CSharp.Tests
         }
 
         private CSharpAutobuilder CreateAutoBuilder(bool isWindows,
-            string? buildless = null,
-            string? dotnetVersion = null,
+            string? buildless = null, string? solution = null, string? buildCommand = null, string? ignoreErrors = null,
+            string? msBuildArguments = null, string? msBuildPlatform = null, string? msBuildConfiguration = null, string? msBuildTarget = null,
+            string? dotnetArguments = null, string? dotnetVersion = null, string? vsToolsVersion = null,
+            string? nugetRestore = null, string? allSolutions = null,
             string cwd = @"C:\Project")
         {
             var codeqlUpperLanguage = Language.CSharp.UpperCaseName;
@@ -412,9 +412,20 @@ namespace Semmle.Autobuild.CSharp.Tests
             actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_DIAGNOSTIC_DIR"] = "";
             actions.GetEnvironmentVariable["CODEQL_JAVA_HOME"] = @"C:\codeql\tools\java";
             actions.GetEnvironmentVariable["CODEQL_PLATFORM"] = isWindows ? "win64" : "linux64";
+            actions.GetEnvironmentVariable["LGTM_INDEX_VSTOOLS_VERSION"] = vsToolsVersion;
+            actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_ARGUMENTS"] = msBuildArguments;
+            actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_PLATFORM"] = msBuildPlatform;
+            actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_CONFIGURATION"] = msBuildConfiguration;
+            actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_TARGET"] = msBuildTarget;
+            actions.GetEnvironmentVariable["LGTM_INDEX_DOTNET_ARGUMENTS"] = dotnetArguments;
+            actions.GetEnvironmentVariable["LGTM_INDEX_DOTNET_VERSION"] = dotnetVersion;
+            actions.GetEnvironmentVariable["LGTM_INDEX_BUILD_COMMAND"] = buildCommand;
+            actions.GetEnvironmentVariable["LGTM_INDEX_SOLUTION"] = solution;
+            actions.GetEnvironmentVariable["LGTM_INDEX_IGNORE_ERRORS"] = ignoreErrors;
+            actions.GetEnvironmentVariable["LGTM_INDEX_BUILDLESS"] = buildless;
             actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_OPTION_BUILDLESS"] = buildless;
-            if (dotnetVersion is not null)
-                actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_OPTION_DOTNET_VERSION"] = dotnetVersion;
+            actions.GetEnvironmentVariable["LGTM_INDEX_ALL_SOLUTIONS"] = allSolutions;
+            actions.GetEnvironmentVariable["LGTM_INDEX_NUGET_RESTORE"] = nugetRestore;
             actions.GetEnvironmentVariable["ProgramFiles(x86)"] = isWindows ? @"C:\Program Files (x86)" : null;
             actions.GetCurrentDirectory = cwd;
             actions.IsWindows = isWindows;
@@ -589,6 +600,15 @@ namespace Semmle.Autobuild.CSharp.Tests
             TestAutobuilderScript(autobuilder, 0, 1);
         }
 
+        private void SkipVsWhere()
+        {
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = false;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"] = false;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat"] = false;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\vcvarsall.bat"] = false;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat"] = false;
+        }
+
         private void TestAutobuilderScript(CSharpAutobuilder autobuilder, int expectedOutput, int commandsRun)
         {
             Assert.Equal(expectedOutput, autobuilder.GetBuildScript().Run(actions, StartCallback, EndCallback));
@@ -608,6 +628,23 @@ namespace Semmle.Autobuild.CSharp.Tests
             }
         }
 
+        [Fact]
+        public void TestLinuxBuildCommand()
+        {
+            actions.RunProcess["./build.sh --skip-tests"] = 0;
+            actions.FileExists["csharp.log"] = true;
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
+            actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln";
+            actions.EnumerateDirectories[@"C:\Project"] = "";
+
+            SkipVsWhere();
+
+            var autobuilder = CreateAutoBuilder(false, buildCommand: "./build.sh --skip-tests");
+            TestAutobuilderScript(autobuilder, 0, 1);
+        }
+
         [Fact]
         public void TestLinuxBuildSh()
         {
@@ -677,6 +714,177 @@ namespace Semmle.Autobuild.CSharp.Tests
             TestAutobuilderScript(autobuilder, 0, 1);
         }
 
+        [Fact]
+        public void TestWindowsBuildBatIgnoreErrors()
+        {
+            actions.EnumerateFiles[@"C:\Project"] = "foo.cs\nbuild.bat";
+            actions.EnumerateDirectories[@"C:\Project"] = "";
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
+            actions.RunProcess[@"cmd.exe /C C:\Project\build.bat"] = 1;
+            actions.RunProcessWorkingDirectory[@"cmd.exe /C C:\Project\build.bat"] = @"C:\Project";
+            actions.RunProcess[@"cmd.exe /C C:\codeql\tools\java\bin\java -jar C:\codeql\csharp\tools\extractor-asp.jar ."] = 0;
+            actions.RunProcess[@"cmd.exe /C C:\codeql\tools\codeql index --xml --extensions config"] = 0;
+            actions.FileExists["csharp.log"] = true;
+
+            var autobuilder = CreateAutoBuilder(true, ignoreErrors: "true");
+            TestAutobuilderScript(autobuilder, 1, 1);
+        }
+
+        [Fact]
+        public void TestWindowsCmdIgnoreErrors()
+        {
+            actions.RunProcess["cmd.exe /C ^\"build.cmd^ --skip-tests^\""] = 3;
+            actions.RunProcess[@"cmd.exe /C C:\codeql\tools\java\bin\java -jar C:\codeql\csharp\tools\extractor-asp.jar ."] = 0;
+            actions.RunProcess[@"cmd.exe /C C:\codeql\tools\codeql index --xml --extensions config"] = 0;
+            actions.FileExists["csharp.log"] = true;
+            SkipVsWhere();
+
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
+            actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln";
+            actions.EnumerateDirectories[@"C:\Project"] = "";
+
+            var autobuilder = CreateAutoBuilder(true, buildCommand: "build.cmd --skip-tests", ignoreErrors: "true");
+            TestAutobuilderScript(autobuilder, 3, 1);
+        }
+
+        [Fact]
+        public void TestWindowCSharpMsBuild()
+        {
+            actions.RunProcess[@"cmd.exe /C C:\Project\.nuget\nuget.exe restore C:\Project\test1.sln -DisableParallelProcessing"] = 0;
+            actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program^ Files^ ^(x86^)\\Microsoft^ Visual^ Studio^ 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test1.sln /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /P:Fu=Bar"] = 0;
+            actions.RunProcess[@"cmd.exe /C C:\Project\.nuget\nuget.exe restore C:\Project\test2.sln -DisableParallelProcessing"] = 0;
+            actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program^ Files^ ^(x86^)\\Microsoft^ Visual^ Studio^ 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test2.sln /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /P:Fu=Bar"] = 0;
+            actions.FileExists["csharp.log"] = true;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = false;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"] = false;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat"] = true;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\vcvarsall.bat"] = false;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat"] = true;
+
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
+            actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest1.cs\ntest2.cs";
+            actions.EnumerateFiles[@"C:\Project\.nuget"] = "nuget.exe";
+            actions.EnumerateDirectories[@"C:\Project"] = @".nuget";
+            actions.EnumerateDirectories[@"C:\Project\.nuget"] = "";
+
+            var autobuilder = CreateAutoBuilder(true, msBuildArguments: "/P:Fu=Bar", msBuildTarget: "Windows", msBuildPlatform: "x86", msBuildConfiguration: "Debug",
+                vsToolsVersion: "12", allSolutions: "true");
+            var testSolution1 = new TestSolution(@"C:\Project\test1.sln");
+            var testSolution2 = new TestSolution(@"C:\Project\test2.sln");
+            autobuilder.ProjectsOrSolutionsToBuild.Add(testSolution1);
+            autobuilder.ProjectsOrSolutionsToBuild.Add(testSolution2);
+
+            TestAutobuilderScript(autobuilder, 0, 4);
+        }
+
+        [Fact]
+        public void TestWindowCSharpMsBuildMultipleSolutions()
+        {
+            actions.RunProcess[@"cmd.exe /C nuget restore C:\Project\test1.csproj -DisableParallelProcessing"] = 0;
+            actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program^ Files^ ^(x86^)\\Microsoft^ Visual^ Studio^ 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test1.csproj /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /P:Fu=Bar"] = 0;
+            actions.RunProcess[@"cmd.exe /C nuget restore C:\Project\test2.csproj -DisableParallelProcessing"] = 0;
+            actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program^ Files^ ^(x86^)\\Microsoft^ Visual^ Studio^ 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test2.csproj /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /P:Fu=Bar"] = 0;
+            actions.FileExists["csharp.log"] = true;
+            actions.FileExists[@"C:\Project\test1.csproj"] = true;
+            actions.FileExists[@"C:\Project\test2.csproj"] = true;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = false;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"] = false;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat"] = true;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\vcvarsall.bat"] = false;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat"] = true;
+
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
+            actions.EnumerateFiles[@"C:\Project"] = "test1.csproj\ntest2.csproj\ntest1.cs\ntest2.cs";
+            actions.EnumerateDirectories[@"C:\Project"] = "";
+
+            var csproj1 = new XmlDocument();
+            csproj1.LoadXml(@"<?xml version=""1.0"" encoding=""utf - 8""?>
+  <Project ToolsVersion=""15.0"" xmlns=""http://schemas.microsoft.com/developer/msbuild/2003"">
+    <ItemGroup>
+      <Compile Include=""test1.cs"" />
+    </ItemGroup>
+  </Project>");
+            actions.LoadXml[@"C:\Project\test1.csproj"] = csproj1;
+
+            var csproj2 = new XmlDocument();
+            csproj2.LoadXml(@"<?xml version=""1.0"" encoding=""utf - 8""?>
+  <Project ToolsVersion=""15.0"" xmlns=""http://schemas.microsoft.com/developer/msbuild/2003"">
+    <ItemGroup>
+      <Compile Include=""test1.cs"" />
+    </ItemGroup>
+  </Project>");
+            actions.LoadXml[@"C:\Project\test2.csproj"] = csproj2;
+
+            var autobuilder = CreateAutoBuilder(true, msBuildArguments: "/P:Fu=Bar", msBuildTarget: "Windows", msBuildPlatform: "x86", msBuildConfiguration: "Debug",
+                vsToolsVersion: "12");
+
+            TestAutobuilderScript(autobuilder, 0, 4);
+        }
+
+        [Fact]
+        public void TestWindowCSharpMsBuildFailed()
+        {
+            actions.RunProcess[@"cmd.exe /C nuget restore C:\Project\test1.sln -DisableParallelProcessing"] = 0;
+            actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program^ Files^ ^(x86^)\\Microsoft^ Visual^ Studio^ 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test1.sln /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /P:Fu=Bar"] = 1;
+            actions.FileExists["csharp.log"] = true;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = false;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"] = false;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat"] = true;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\vcvarsall.bat"] = false;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat"] = true;
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
+            actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest1.cs\ntest2.cs";
+            actions.EnumerateDirectories[@"C:\Project"] = "";
+
+            var autobuilder = CreateAutoBuilder(true, msBuildArguments: "/P:Fu=Bar", msBuildTarget: "Windows", msBuildPlatform: "x86", msBuildConfiguration: "Debug",
+                vsToolsVersion: "12", allSolutions: "true");
+            var testSolution1 = new TestSolution(@"C:\Project\test1.sln");
+            var testSolution2 = new TestSolution(@"C:\Project\test2.sln");
+            autobuilder.ProjectsOrSolutionsToBuild.Add(testSolution1);
+            autobuilder.ProjectsOrSolutionsToBuild.Add(testSolution2);
+
+            TestAutobuilderScript(autobuilder, 1, 2);
+        }
+
+
+        [Fact]
+        public void TestSkipNugetMsBuild()
+        {
+            actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program^ Files^ ^(x86^)\\Microsoft^ Visual^ Studio^ 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test1.sln /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /P:Fu=Bar"] = 0;
+            actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program^ Files^ ^(x86^)\\Microsoft^ Visual^ Studio^ 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test2.sln /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /P:Fu=Bar"] = 0;
+            actions.FileExists["csharp.log"] = true;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = false;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"] = false;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat"] = true;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\vcvarsall.bat"] = false;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat"] = true;
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
+            actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest1.cs\ntest2.cs";
+            actions.EnumerateDirectories[@"C:\Project"] = "";
+
+            var autobuilder = CreateAutoBuilder(true, msBuildArguments: "/P:Fu=Bar", msBuildTarget: "Windows",
+                msBuildPlatform: "x86", msBuildConfiguration: "Debug", vsToolsVersion: "12",
+                allSolutions: "true", nugetRestore: "false");
+            var testSolution1 = new TestSolution(@"C:\Project\test1.sln");
+            var testSolution2 = new TestSolution(@"C:\Project\test2.sln");
+            autobuilder.ProjectsOrSolutionsToBuild.Add(testSolution1);
+            autobuilder.ProjectsOrSolutionsToBuild.Add(testSolution2);
+
+            TestAutobuilderScript(autobuilder, 0, 2);
+        }
+
         [Fact]
         public void TestSkipNugetBuildless()
         {
@@ -692,6 +900,35 @@ namespace Semmle.Autobuild.CSharp.Tests
             TestAutobuilderScript(autobuilder, 0, 1);
         }
 
+
+        [Fact]
+        public void TestSkipNugetDotnet()
+        {
+            actions.RunProcess["dotnet --info"] = 0;
+            actions.RunProcess[@"dotnet clean C:\Project/test.csproj"] = 0;
+            actions.RunProcess[@"dotnet restore C:\Project/test.csproj"] = 0;
+            actions.RunProcess[@"dotnet build --no-incremental --no-restore C:\Project/test.csproj"] = 0;
+            actions.FileExists["csharp.log"] = true;
+            actions.FileExists[@"C:\Project/test.csproj"] = true;
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
+            actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.cs\ntest.csproj";
+            actions.EnumerateDirectories[@"C:\Project"] = "";
+            var xml = new XmlDocument();
+            xml.LoadXml(@"<Project Sdk=""Microsoft.NET.Sdk"">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>netcoreapp2.1</TargetFramework>
+  </PropertyGroup>
+
+</Project>");
+            actions.LoadXml[@"C:\Project/test.csproj"] = xml;
+
+            var autobuilder = CreateAutoBuilder(false, dotnetArguments: "--no-restore");  // nugetRestore=false does not work for now.
+            TestAutobuilderScript(autobuilder, 0, 4);
+        }
+
         [Fact]
         public void TestDotnetVersionNotInstalled()
         {
@@ -816,7 +1053,7 @@ namespace Semmle.Autobuild.CSharp.Tests
         {
             actions.RunProcess[@"cmd.exe /C nuget restore C:\Project\dirs.proj -DisableParallelProcessing"] = 1;
             actions.RunProcess[@"cmd.exe /C scratch\.nuget\nuget.exe restore C:\Project\dirs.proj -DisableParallelProcessing"] = 0;
-            actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program^ Files^ ^(x86^)\\Microsoft^ Visual^ Studio^ 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\dirs.proj /t:rebuild"] = 0;
+            actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program^ Files^ ^(x86^)\\Microsoft^ Visual^ Studio^ 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\dirs.proj /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /P:Fu=Bar"] = 0;
             actions.FileExists["csharp.log"] = true;
             actions.FileExists[@"C:\Project\a\test.csproj"] = true;
             actions.FileExists[@"C:\Project\dirs.proj"] = true;
@@ -851,7 +1088,8 @@ namespace Semmle.Autobuild.CSharp.Tests
 </Project>");
             actions.LoadXml[@"C:\Project\dirs.proj"] = dirsproj;
 
-            var autobuilder = CreateAutoBuilder(true);
+            var autobuilder = CreateAutoBuilder(true, msBuildArguments: "/P:Fu=Bar", msBuildTarget: "Windows", msBuildPlatform: "x86", msBuildConfiguration: "Debug",
+                vsToolsVersion: "12", allSolutions: "true");
             TestAutobuilderScript(autobuilder, 0, 3);
         }
 

csharp/autobuilder/Semmle.Autobuild.CSharp/CSharpAutobuilder.cs

@@ -25,11 +25,9 @@ namespace Semmle.Autobuild.CSharp
         /// </summary>
         public CSharpAutobuildOptions(IBuildActions actions) : base(actions)
         {
-            Buildless =
+            Buildless = actions.GetEnvironmentVariable(lgtmPrefix + "BUILDLESS").AsBool("buildless", false) ||
                 actions.GetEnvironmentVariable(extractorOptionPrefix + "BUILDLESS").AsBool("buildless", false) ||
                 actions.GetEnvironmentVariable(buildModeEnvironmentVariable)?.ToLower() == "none";
-
-
         }
     }
 
@@ -48,11 +46,18 @@ namespace Semmle.Autobuild.CSharp
             var attempt = BuildScript.Failure;
             switch (GetCSharpBuildStrategy())
             {
+                case CSharpBuildStrategy.CustomBuildCommand:
+                    attempt = new BuildCommandRule(DotNetRule.WithDotNet).Analyse(this, false) & CheckExtractorRun(true);
+                    break;
                 case CSharpBuildStrategy.Buildless:
                     // No need to check that the extractor has been executed in buildless mode
-                    attempt = BuildScript.Bind(
-                        AddBuildlessStartedDiagnostic() & new StandaloneBuildRule().Analyse(this, false),
-                        AddBuildlessEndedDiagnostic);
+                    attempt = new StandaloneBuildRule().Analyse(this, false);
+                    break;
+                case CSharpBuildStrategy.MSBuild:
+                    attempt = new MsBuildRule().Analyse(this, false) & CheckExtractorRun(true);
+                    break;
+                case CSharpBuildStrategy.DotNet:
+                    attempt = new DotNetRule().Analyse(this, false) & CheckExtractorRun(true);
                     break;
                 case CSharpBuildStrategy.Auto:
                     attempt =
@@ -81,52 +86,6 @@ namespace Semmle.Autobuild.CSharp
                 return 1;
             });
 
-        private BuildScript AddBuildlessStartedDiagnostic()
-        {
-            return BuildScript.Create(actions =>
-            {
-                AddDiagnostic(new DiagnosticMessage(
-                    Options.Language,
-                    "buildless/mode-active",
-                    "C# with build-mode set to 'none'",
-                    visibility: new DiagnosticMessage.TspVisibility(statusPage: true, cliSummaryTable: true, telemetry: true),
-                    markdownMessage: "C# with build-mode set to 'none'. This means that all C# source in the working directory will be scanned, with build tools, such as Nuget and Dotnet CLIs, only contributing information about external dependencies.",
-                    severity: DiagnosticMessage.TspSeverity.Note
-                ));
-                return 0;
-            });
-        }
-
-        private BuildScript AddBuildlessEndedDiagnostic(int buildResult)
-        {
-            return BuildScript.Create(actions =>
-            {
-                if (buildResult == 0)
-                {
-                    AddDiagnostic(new DiagnosticMessage(
-                        Options.Language,
-                        "buildless/complete",
-                        "C# analysis with build-mode 'none' completed",
-                        visibility: new DiagnosticMessage.TspVisibility(statusPage: false, cliSummaryTable: true, telemetry: true),
-                        markdownMessage: "C# analysis with build-mode 'none' completed.",
-                        severity: DiagnosticMessage.TspSeverity.Unknown
-                    ));
-                }
-                else
-                {
-                    AddDiagnostic(new DiagnosticMessage(
-                        Options.Language,
-                        "buildless/failed",
-                        "C# analysis with build-mode 'none' failed",
-                        visibility: new DiagnosticMessage.TspVisibility(statusPage: true, cliSummaryTable: true, telemetry: true),
-                        markdownMessage: "C# analysis with build-mode 'none' failed.",
-                        severity: DiagnosticMessage.TspSeverity.Error
-                    ));
-                }
-                return buildResult;
-            });
-        }
-
         protected override void AutobuildFailureDiagnostic()
         {
             // if `ScriptPath` is not null here, the `BuildCommandAuto` rule was
@@ -239,15 +198,32 @@ namespace Semmle.Autobuild.CSharp
         /// </summary>
         private CSharpBuildStrategy GetCSharpBuildStrategy()
         {
+            if (Options.BuildCommand is not null)
+                return CSharpBuildStrategy.CustomBuildCommand;
+
             if (Options.Buildless)
                 return CSharpBuildStrategy.Buildless;
 
+            if (Options.MsBuildArguments is not null
+                || Options.MsBuildConfiguration is not null
+                || Options.MsBuildPlatform is not null
+                || Options.MsBuildTarget is not null)
+            {
+                return CSharpBuildStrategy.MSBuild;
+            }
+
+            if (Options.DotNetArguments is not null || Options.DotNetVersion is not null)
+                return CSharpBuildStrategy.DotNet;
+
             return CSharpBuildStrategy.Auto;
         }
 
         private enum CSharpBuildStrategy
         {
+            CustomBuildCommand,
             Buildless,
+            MSBuild,
+            DotNet,
             Auto
         }
     }

csharp/autobuilder/Semmle.Autobuild.CSharp/DotNetRule.cs

@@ -32,7 +32,7 @@ namespace Semmle.Autobuild.CSharp
             if (auto)
             {
                 NotDotNetProjects = builder.ProjectsOrSolutionsToBuild
-                    .SelectMany(p => new[] { p }.Concat(p.IncludedProjects))
+                    .SelectMany(p => Enumerators.Singleton(p).Concat(p.IncludedProjects))
                     .OfType<Project<CSharpAutobuildOptions>>()
                     .Where(p => !p.DotNetProject);
                 var notDotNetProject = NotDotNetProjects.FirstOrDefault();
@@ -150,7 +150,8 @@ namespace Semmle.Autobuild.CSharp
                 Argument("--no-incremental");
 
             return
-                script.QuoteArgument(projOrSln).
+                script.Argument(builder.Options.DotNetArguments).
+                    QuoteArgument(projOrSln).
                     Script;
         }
     }

csharp/autobuilder/Semmle.Autobuild.CSharp/Program.cs

@@ -17,7 +17,7 @@ namespace Semmle.Autobuild.CSharp
                 try
                 {
                     Console.WriteLine("CodeQL C# autobuilder");
-                    using var builder = new CSharpAutobuilder(actions, options);
+                    var builder = new CSharpAutobuilder(actions, options);
                     return builder.AttemptBuild();
                 }
                 catch (InvalidEnvironmentException ex)

csharp/autobuilder/Semmle.Autobuild.Shared/AutobuildOptions.cs

@@ -11,9 +11,24 @@ namespace Semmle.Autobuild.Shared
     /// </summary>
     public abstract class AutobuildOptionsShared
     {
+        protected const string lgtmPrefix = "LGTM_INDEX_";
+
+
         public int SearchDepth { get; } = 3;
         public string RootDirectory { get; }
+        public string? VsToolsVersion { get; }
+        public string? MsBuildArguments { get; }
+        public string? MsBuildPlatform { get; }
+        public string? MsBuildConfiguration { get; }
+        public string? MsBuildTarget { get; }
+        public string? DotNetArguments { get; }
         public string? DotNetVersion { get; }
+        public string? BuildCommand { get; }
+        public IEnumerable<string> Solution { get; }
+        public bool IgnoreErrors { get; }
+
+        public bool AllSolutions { get; }
+        public bool NugetRestore { get; }
         public abstract Language Language { get; }
 
         /// <summary>
@@ -23,7 +38,19 @@ namespace Semmle.Autobuild.Shared
         public AutobuildOptionsShared(IBuildActions actions)
         {
             RootDirectory = actions.GetCurrentDirectory();
-            DotNetVersion = actions.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_OPTION_DOTNET_VERSION");
+            VsToolsVersion = actions.GetEnvironmentVariable(lgtmPrefix + "VSTOOLS_VERSION");
+            MsBuildArguments = actions.GetEnvironmentVariable(lgtmPrefix + "MSBUILD_ARGUMENTS")?.AsStringWithExpandedEnvVars(actions);
+            MsBuildPlatform = actions.GetEnvironmentVariable(lgtmPrefix + "MSBUILD_PLATFORM");
+            MsBuildConfiguration = actions.GetEnvironmentVariable(lgtmPrefix + "MSBUILD_CONFIGURATION");
+            MsBuildTarget = actions.GetEnvironmentVariable(lgtmPrefix + "MSBUILD_TARGET");
+            DotNetArguments = actions.GetEnvironmentVariable(lgtmPrefix + "DOTNET_ARGUMENTS")?.AsStringWithExpandedEnvVars(actions);
+            DotNetVersion = actions.GetEnvironmentVariable(lgtmPrefix + "DOTNET_VERSION");
+            BuildCommand = actions.GetEnvironmentVariable(lgtmPrefix + "BUILD_COMMAND");
+            Solution = actions.GetEnvironmentVariable(lgtmPrefix + "SOLUTION").AsListWithExpandedEnvVars(actions, Array.Empty<string>());
+
+            IgnoreErrors = actions.GetEnvironmentVariable(lgtmPrefix + "IGNORE_ERRORS").AsBool("ignore_errors", false);
+            AllSolutions = actions.GetEnvironmentVariable(lgtmPrefix + "ALL_SOLUTIONS").AsBool("all_solutions", false);
+            NugetRestore = actions.GetEnvironmentVariable(lgtmPrefix + "NUGET_RESTORE").AsBool("nuget_restore", true);
         }
     }
 

csharp/autobuilder/Semmle.Autobuild.Shared/Autobuilder.cs

@@ -92,7 +92,7 @@ namespace Semmle.Autobuild.Shared
     /// The overall design is intended to be extensible so that in theory,
     /// it should be possible to add new build rules without touching this code.
     /// </summary>
-    public abstract class Autobuilder<TAutobuildOptions> : IDisposable, IAutobuilder<TAutobuildOptions> where TAutobuildOptions : AutobuildOptionsShared
+    public abstract class Autobuilder<TAutobuildOptions> : IAutobuilder<TAutobuildOptions> where TAutobuildOptions : AutobuildOptionsShared
     {
         /// <summary>
         /// Full file paths of files found in the project directory, as well as
@@ -161,6 +161,9 @@ namespace Semmle.Autobuild.Shared
             if (matchingFiles.Length == 0)
                 return null;
 
+            if (Options.AllSolutions)
+                return matchingFiles.Select(p => p.ProjectOrSolution);
+
             return matchingFiles
                 .Where(f => f.DistanceFromRoot == matchingFiles[0].DistanceFromRoot)
                 .Select(f => f.ProjectOrSolution);
@@ -182,6 +185,19 @@ namespace Semmle.Autobuild.Shared
             projectsOrSolutionsToBuildLazy = new Lazy<IList<IProjectOrSolution>>(() =>
             {
                 List<IProjectOrSolution>? ret;
+                if (options.Solution.Any())
+                {
+                    ret = new List<IProjectOrSolution>();
+                    foreach (var solution in options.Solution)
+                    {
+                        if (actions.FileExists(solution))
+                            ret.Add(new Solution<TAutobuildOptions>(this, solution, true));
+                        else
+                            logger.LogError($"The specified project or solution file {solution} was not found");
+                    }
+                    return ret;
+                }
+
                 // First look for `.proj` files
                 ret = FindFiles(".proj", f => new Project<TAutobuildOptions>(this, f))?.ToList();
                 if (ret is not null)
@@ -269,6 +285,9 @@ namespace Semmle.Autobuild.Shared
 
             var script = GetBuildScript();
 
+            if (Options.IgnoreErrors)
+                script |= BuildScript.Success;
+
             void startCallback(string s, bool silent)
             {
                 logger.Log(silent ? Severity.Debug : Severity.Info, $"\nRunning {s}");
@@ -351,20 +370,6 @@ namespace Semmle.Autobuild.Shared
             }
         });
 
-        public void Dispose()
-        {
-            Dispose(true);
-            GC.SuppressFinalize(this);
-        }
-
-        protected virtual void Dispose(bool disposing)
-        {
-            if (disposing)
-            {
-                diagnostics.Dispose();
-            }
-        }
-
         /// <summary>
         /// Value of CODEQL_EXTRACTOR_<LANG>_ROOT environment variable.
         /// </summary>

csharp/autobuilder/Semmle.Autobuild.Shared/BuildCommandAutoRule.cs

@@ -82,6 +82,11 @@ namespace Semmle.Autobuild.Shared
             {
                 var command = new CommandBuilder(builder.Actions, dir, environment);
 
+                // A specific Visual Studio version may be required
+                var vsTools = MsBuildRule.GetVcVarsBatFile(builder);
+                if (vsTools is not null)
+                    command.CallBatFile(vsTools.Path);
+
                 command.RunCommand(this.ScriptPath);
                 return command.Script;
             });

csharp/autobuilder/Semmle.Autobuild.Shared/BuildCommandRule.cs

@@ -0,0 +1,37 @@
+using Semmle.Util;
+
+namespace Semmle.Autobuild.Shared
+{
+    /// <summary>
+    /// Execute the build_command rule.
+    /// </summary>
+    public class BuildCommandRule : IBuildRule<AutobuildOptionsShared>
+    {
+        private readonly WithDotNet<AutobuildOptionsShared> withDotNet;
+
+        public BuildCommandRule(WithDotNet<AutobuildOptionsShared> withDotNet)
+        {
+            this.withDotNet = withDotNet;
+        }
+
+        public BuildScript Analyse(IAutobuilder<AutobuildOptionsShared> builder, bool auto)
+        {
+            if (builder.Options.BuildCommand is null)
+                return BuildScript.Failure;
+
+            // Custom build commands may require a specific .NET Core version
+            return withDotNet(builder, environment =>
+                {
+                    var command = new CommandBuilder(builder.Actions, null, environment);
+
+                    // Custom build commands may require a specific Visual Studio version
+                    var vsTools = MsBuildRule.GetVcVarsBatFile(builder);
+                    if (vsTools is not null)
+                        command.CallBatFile(vsTools.Path);
+                    command.RunCommand(builder.Options.BuildCommand);
+
+                    return command.Script;
+                });
+        }
+    }
+}

csharp/autobuilder/Semmle.Autobuild.Shared/MsBuildRule.cs

@@ -42,9 +42,9 @@ namespace Semmle.Autobuild.Shared
             if (auto)
                 builder.Logger.LogInfo("Attempting to build using MSBuild");
 
-            VcVarsBatFile? vsTools = null;
+            var vsTools = GetVcVarsBatFile(builder);
 
-            if (builder.ProjectsOrSolutionsToBuild.Any())
+            if (vsTools is null && builder.ProjectsOrSolutionsToBuild.Any())
             {
                 var firstSolution = builder.ProjectsOrSolutionsToBuild.OfType<ISolution>().FirstOrDefault();
                 vsTools = firstSolution is not null
@@ -67,44 +67,46 @@ namespace Semmle.Autobuild.Shared
 
             foreach (var projectOrSolution in builder.ProjectsOrSolutionsToBuild)
             {
-
-                BuildScript GetNugetRestoreScript() =>
-                    new CommandBuilder(builder.Actions).
-                        RunCommand(nuget).
-                        Argument("restore").
-                        QuoteArgument(projectOrSolution.FullPath).
-                        Argument("-DisableParallelProcessing").
-                        Script;
-                var nugetRestore = GetNugetRestoreScript();
-                var msbuildRestoreCommand = new CommandBuilder(builder.Actions).
-                    MsBuildCommand(builder).
-                    Argument("/t:restore").
-                    QuoteArgument(projectOrSolution.FullPath);
-
-                if (builder.Actions.IsRunningOnAppleSilicon())
+                if (builder.Options.NugetRestore)
                 {
-                    // On Apple Silicon, only try package restore with `dotnet msbuild /t:restore`
-                    ret &= BuildScript.Try(msbuildRestoreCommand.Script);
-                }
-                else if (nugetDownloaded)
-                {
-                    ret &= BuildScript.Try(nugetRestore | msbuildRestoreCommand.Script);
-                }
-                else
-                {
-                    // If `nuget restore` fails, and we have not already attempted to download `nuget.exe`,
-                    // download it and reattempt `nuget restore`.
-                    var nugetDownloadAndRestore =
-                        BuildScript.Bind(DownloadNugetExe(builder, nugetDownloadPath), exitCode =>
-                        {
-                            nugetDownloaded = true;
-                            if (exitCode != 0)
-                                return BuildScript.Failure;
+                    BuildScript GetNugetRestoreScript() =>
+                        new CommandBuilder(builder.Actions).
+                            RunCommand(nuget).
+                            Argument("restore").
+                            QuoteArgument(projectOrSolution.FullPath).
+                            Argument("-DisableParallelProcessing").
+                            Script;
+                    var nugetRestore = GetNugetRestoreScript();
+                    var msbuildRestoreCommand = new CommandBuilder(builder.Actions).
+                        MsBuildCommand(builder).
+                        Argument("/t:restore").
+                        QuoteArgument(projectOrSolution.FullPath);
 
-                            nuget = nugetDownloadPath;
-                            return GetNugetRestoreScript();
-                        });
-                    ret &= BuildScript.Try(nugetRestore | nugetDownloadAndRestore | msbuildRestoreCommand.Script);
+                    if (builder.Actions.IsRunningOnAppleSilicon())
+                    {
+                        // On Apple Silicon, only try package restore with `dotnet msbuild /t:restore`
+                        ret &= BuildScript.Try(msbuildRestoreCommand.Script);
+                    }
+                    else if (nugetDownloaded)
+                    {
+                        ret &= BuildScript.Try(nugetRestore | msbuildRestoreCommand.Script);
+                    }
+                    else
+                    {
+                        // If `nuget restore` fails, and we have not already attempted to download `nuget.exe`,
+                        // download it and reattempt `nuget restore`.
+                        var nugetDownloadAndRestore =
+                            BuildScript.Bind(DownloadNugetExe(builder, nugetDownloadPath), exitCode =>
+                            {
+                                nugetDownloaded = true;
+                                if (exitCode != 0)
+                                    return BuildScript.Failure;
+
+                                nuget = nugetDownloadPath;
+                                return GetNugetRestoreScript();
+                            });
+                        ret &= BuildScript.Try(nugetRestore | nugetDownloadAndRestore | msbuildRestoreCommand.Script);
+                    }
                 }
 
                 var command = new CommandBuilder(builder.Actions);
@@ -122,9 +124,9 @@ namespace Semmle.Autobuild.Shared
                 command.MsBuildCommand(builder);
                 command.QuoteArgument(projectOrSolution.FullPath);
 
-                var target = "rebuild";
-                var platform = projectOrSolution is ISolution s1 ? s1.DefaultPlatformName : null;
-                var configuration = projectOrSolution is ISolution s2 ? s2.DefaultConfigurationName : null;
+                var target = builder.Options.MsBuildTarget ?? "rebuild";
+                var platform = builder.Options.MsBuildPlatform ?? (projectOrSolution is ISolution s1 ? s1.DefaultPlatformName : null);
+                var configuration = builder.Options.MsBuildConfiguration ?? (projectOrSolution is ISolution s2 ? s2.DefaultConfigurationName : null);
 
                 command.Argument("/t:" + target);
                 if (platform is not null)
@@ -132,6 +134,8 @@ namespace Semmle.Autobuild.Shared
                 if (configuration is not null)
                     command.Argument(string.Format("/p:Configuration=\"{0}\"", configuration));
 
+                command.Argument(builder.Options.MsBuildArguments);
+
                 // append the build script which invokes msbuild to the overall build script `ret`;
                 // we insert a check that building the current project or solution was successful:
                 // if it was not successful, we add it to `FailedProjectsOrSolutions`
@@ -144,6 +148,41 @@ namespace Semmle.Autobuild.Shared
             return ret;
         }
 
+        /// <summary>
+        /// Gets the BAT file used to initialize the appropriate Visual Studio
+        /// version/platform, as specified by the `vstools_version` property in
+        /// lgtm.yml.
+        ///
+        /// Returns <code>null</code> when no version is specified.
+        /// </summary>
+        public static VcVarsBatFile? GetVcVarsBatFile<TAutobuildOptions>(IAutobuilder<TAutobuildOptions> builder) where TAutobuildOptions : AutobuildOptionsShared
+        {
+            VcVarsBatFile? vsTools = null;
+
+            if (builder.Options.VsToolsVersion is not null)
+            {
+                if (int.TryParse(builder.Options.VsToolsVersion, out var msToolsVersion))
+                {
+                    foreach (var b in BuildTools.VcVarsAllBatFiles(builder.Actions))
+                    {
+                        builder.Logger.Log(Severity.Info, "Found {0} version {1}", b.Path, b.ToolsVersion);
+                    }
+
+                    vsTools = BuildTools.FindCompatibleVcVars(builder.Actions, msToolsVersion);
+                    if (vsTools is null)
+                        builder.Logger.LogWarning("Could not find build tools matching version {0}", msToolsVersion);
+                    else
+                        builder.Logger.Log(Severity.Info, "Setting Visual Studio tools to {0}", vsTools.Path);
+                }
+                else
+                {
+                    builder.Logger.LogError("The format of vstools_version is incorrect. Please specify an integer.");
+                }
+            }
+
+            return vsTools;
+        }
+
         /// <summary>
         /// Returns a script for downloading `nuget.exe` from nuget.org.
         /// </summary>

csharp/documentation/library-coverage/coverage.csv

@@ -1,44 +1,44 @@
-package,sink,source,summary,sink:code-injection,sink:encryption-decryptor,sink:encryption-encryptor,sink:encryption-keyprop,sink:encryption-symmetrickey,sink:file-content-store,sink:html-injection,sink:js-injection,sink:log-injection,sink:sql-injection,source:commandargs,source:database,source:environment,source:file,source:file-write,source:local,source:remote,source:windows-registry,summary:taint,summary:value
-Amazon.Lambda.APIGatewayEvents,,6,,,,,,,,,,,,,,,,,,6,,,
-Amazon.Lambda.Core,10,,,,,,,,,,,10,,,,,,,,,,,
-Dapper,55,42,1,,,,,,,,,,55,,42,,,,,,,,1
-ILCompiler,,,81,,,,,,,,,,,,,,,,,,,81,
-ILLink.RoslynAnalyzer,,,63,,,,,,,,,,,,,,,,,,,63,
-ILLink.Shared,,,32,,,,,,,,,,,,,,,,,,,29,3
-ILLink.Tasks,,,5,,,,,,,,,,,,,,,,,,,5,
-Internal.IL,,,69,,,,,,,,,,,,,,,,,,,67,2
-Internal.Pgo,,,9,,,,,,,,,,,,,,,,,,,8,1
-Internal.TypeSystem,,,367,,,,,,,,,,,,,,,,,,,331,36
-JsonToItemsTaskFactory,,,7,,,,,,,,,,,,,,,,,,,7,
-Microsoft.Android.Build,,,14,,,,,,,,,,,,,,,,,,,14,
-Microsoft.Apple.Build,,,7,,,,,,,,,,,,,,,,,,,7,
-Microsoft.ApplicationBlocks.Data,28,,,,,,,,,,,,28,,,,,,,,,,
-Microsoft.CSharp,,,24,,,,,,,,,,,,,,,,,,,24,
-Microsoft.Diagnostics.Tools.Pgo,,,13,,,,,,,,,,,,,,,,,,,13,
-Microsoft.EntityFrameworkCore,6,,12,,,,,,,,,,6,,,,,,,,,,12
-Microsoft.Extensions.Caching.Distributed,,,15,,,,,,,,,,,,,,,,,,,15,
-Microsoft.Extensions.Caching.Memory,,,38,,,,,,,,,,,,,,,,,,,37,1
-Microsoft.Extensions.Configuration,,2,89,,,,,,,,,,,,,2,,,,,,86,3
-Microsoft.Extensions.DependencyInjection,,,120,,,,,,,,,,,,,,,,,,,120,
-Microsoft.Extensions.DependencyModel,,,12,,,,,,,,,,,,,,,,,,,12,
-Microsoft.Extensions.Diagnostics.Metrics,,,13,,,,,,,,,,,,,,,,,,,13,
-Microsoft.Extensions.FileProviders,,,15,,,,,,,,,,,,,,,,,,,15,
-Microsoft.Extensions.FileSystemGlobbing,,,16,,,,,,,,,,,,,,,,,,,14,2
-Microsoft.Extensions.Hosting,,,23,,,,,,,,,,,,,,,,,,,22,1
-Microsoft.Extensions.Http,,,10,,,,,,,,,,,,,,,,,,,10,
-Microsoft.Extensions.Logging,,,60,,,,,,,,,,,,,,,,,,,59,1
-Microsoft.Extensions.Options,,,8,,,,,,,,,,,,,,,,,,,8,
-Microsoft.Extensions.Primitives,,,64,,,,,,,,,,,,,,,,,,,64,
-Microsoft.Interop,,,78,,,,,,,,,,,,,,,,,,,78,
-Microsoft.NET.Build.Tasks,,,1,,,,,,,,,,,,,,,,,,,1,
-Microsoft.NET.WebAssembly.Webcil,,,7,,,,,,,,,,,,,,,,,,,7,
-Microsoft.VisualBasic,,,10,,,,,,,,,,,,,,,,,,,5,5
-Microsoft.WebAssembly.Build.Tasks,,,3,,,,,,,,,,,,,,,,,,,3,
-Microsoft.Win32,,4,4,,,,,,,,,,,,,,,,,,4,4,
-Mono.Linker,,,163,,,,,,,,,,,,,,,,,,,163,
-MySql.Data.MySqlClient,48,,,,,,,,,,,,48,,,,,,,,,,
-Newtonsoft.Json,,,91,,,,,,,,,,,,,,,,,,,73,18
-ServiceStack,194,,7,27,,,,,75,,,,92,,,,,,,,,7,
-SourceGenerators,,,4,,,,,,,,,,,,,,,,,,,4,
-System,67,44,11872,,8,8,9,,,4,5,,33,2,,3,15,17,3,4,,9906,1966
-Windows.Security.Cryptography.Core,1,,,,,,,1,,,,,,,,,,,,,,,
+package,sink,source,summary,sink:code-injection,sink:encryption-decryptor,sink:encryption-encryptor,sink:encryption-keyprop,sink:encryption-symmetrickey,sink:file-content-store,sink:html-injection,sink:js-injection,sink:log-injection,sink:sql-injection,source:commandargs,source:environment,source:file,source:file-write,source:local,source:remote,source:windows-registry,summary:taint,summary:value
+Amazon.Lambda.APIGatewayEvents,,6,,,,,,,,,,,,,,,,,6,,,
+Amazon.Lambda.Core,10,,,,,,,,,,,10,,,,,,,,,,
+Dapper,55,,,,,,,,,,,,55,,,,,,,,,
+ILCompiler,,,81,,,,,,,,,,,,,,,,,,81,
+ILLink.RoslynAnalyzer,,,63,,,,,,,,,,,,,,,,,,63,
+ILLink.Shared,,,32,,,,,,,,,,,,,,,,,,29,3
+ILLink.Tasks,,,5,,,,,,,,,,,,,,,,,,5,
+Internal.IL,,,69,,,,,,,,,,,,,,,,,,67,2
+Internal.Pgo,,,9,,,,,,,,,,,,,,,,,,8,1
+Internal.TypeSystem,,,367,,,,,,,,,,,,,,,,,,331,36
+JsonToItemsTaskFactory,,,7,,,,,,,,,,,,,,,,,,7,
+Microsoft.Android.Build,,,14,,,,,,,,,,,,,,,,,,14,
+Microsoft.Apple.Build,,,7,,,,,,,,,,,,,,,,,,7,
+Microsoft.ApplicationBlocks.Data,28,,,,,,,,,,,,28,,,,,,,,,
+Microsoft.CSharp,,,24,,,,,,,,,,,,,,,,,,24,
+Microsoft.Diagnostics.Tools.Pgo,,,13,,,,,,,,,,,,,,,,,,13,
+Microsoft.EntityFrameworkCore,6,,12,,,,,,,,,,6,,,,,,,,,12
+Microsoft.Extensions.Caching.Distributed,,,15,,,,,,,,,,,,,,,,,,15,
+Microsoft.Extensions.Caching.Memory,,,38,,,,,,,,,,,,,,,,,,37,1
+Microsoft.Extensions.Configuration,,2,89,,,,,,,,,,,,2,,,,,,86,3
+Microsoft.Extensions.DependencyInjection,,,120,,,,,,,,,,,,,,,,,,120,
+Microsoft.Extensions.DependencyModel,,,12,,,,,,,,,,,,,,,,,,12,
+Microsoft.Extensions.Diagnostics.Metrics,,,13,,,,,,,,,,,,,,,,,,13,
+Microsoft.Extensions.FileProviders,,,15,,,,,,,,,,,,,,,,,,15,
+Microsoft.Extensions.FileSystemGlobbing,,,16,,,,,,,,,,,,,,,,,,14,2
+Microsoft.Extensions.Hosting,,,23,,,,,,,,,,,,,,,,,,22,1
+Microsoft.Extensions.Http,,,10,,,,,,,,,,,,,,,,,,10,
+Microsoft.Extensions.Logging,,,60,,,,,,,,,,,,,,,,,,59,1
+Microsoft.Extensions.Options,,,8,,,,,,,,,,,,,,,,,,8,
+Microsoft.Extensions.Primitives,,,64,,,,,,,,,,,,,,,,,,64,
+Microsoft.Interop,,,78,,,,,,,,,,,,,,,,,,78,
+Microsoft.NET.Build.Tasks,,,1,,,,,,,,,,,,,,,,,,1,
+Microsoft.NET.WebAssembly.Webcil,,,7,,,,,,,,,,,,,,,,,,7,
+Microsoft.VisualBasic,,,10,,,,,,,,,,,,,,,,,,5,5
+Microsoft.WebAssembly.Build.Tasks,,,3,,,,,,,,,,,,,,,,,,3,
+Microsoft.Win32,,4,4,,,,,,,,,,,,,,,,,4,4,
+Mono.Linker,,,163,,,,,,,,,,,,,,,,,,163,
+MySql.Data.MySqlClient,48,,,,,,,,,,,,48,,,,,,,,,
+Newtonsoft.Json,,,91,,,,,,,,,,,,,,,,,,73,18
+ServiceStack,194,,7,27,,,,,75,,,,92,,,,,,,,7,
+SourceGenerators,,,4,,,,,,,,,,,,,,,,,,4,
+System,67,30,11864,,8,8,9,,,4,5,,33,2,3,1,17,3,4,,9898,1966
+Windows.Security.Cryptography.Core,1,,,,,,,1,,,,,,,,,,,,,,

csharp/documentation/library-coverage/coverage.rst

@@ -8,7 +8,7 @@ C# framework & library support
 
    Framework / library,Package,Flow sources,Taint & value steps,Sinks (total),`CWE-079` :sub:`Cross-site scripting`
    `ServiceStack <https://servicestack.net/>`_,"``ServiceStack.*``, ``ServiceStack``",,7,194,
-   System,"``System.*``, ``System``",44,11872,67,9
-   Others,"``Amazon.Lambda.APIGatewayEvents``, ``Amazon.Lambda.Core``, ``Dapper``, ``ILCompiler``, ``ILLink.RoslynAnalyzer``, ``ILLink.Shared``, ``ILLink.Tasks``, ``Internal.IL``, ``Internal.Pgo``, ``Internal.TypeSystem``, ``JsonToItemsTaskFactory``, ``Microsoft.Android.Build``, ``Microsoft.Apple.Build``, ``Microsoft.ApplicationBlocks.Data``, ``Microsoft.CSharp``, ``Microsoft.Diagnostics.Tools.Pgo``, ``Microsoft.EntityFrameworkCore``, ``Microsoft.Extensions.Caching.Distributed``, ``Microsoft.Extensions.Caching.Memory``, ``Microsoft.Extensions.Configuration``, ``Microsoft.Extensions.DependencyInjection``, ``Microsoft.Extensions.DependencyModel``, ``Microsoft.Extensions.Diagnostics.Metrics``, ``Microsoft.Extensions.FileProviders``, ``Microsoft.Extensions.FileSystemGlobbing``, ``Microsoft.Extensions.Hosting``, ``Microsoft.Extensions.Http``, ``Microsoft.Extensions.Logging``, ``Microsoft.Extensions.Options``, ``Microsoft.Extensions.Primitives``, ``Microsoft.Interop``, ``Microsoft.NET.Build.Tasks``, ``Microsoft.NET.WebAssembly.Webcil``, ``Microsoft.VisualBasic``, ``Microsoft.WebAssembly.Build.Tasks``, ``Microsoft.Win32``, ``Mono.Linker``, ``MySql.Data.MySqlClient``, ``Newtonsoft.Json``, ``SourceGenerators``, ``Windows.Security.Cryptography.Core``",54,1548,148,
-   Totals,,98,13427,409,9
+   System,"``System.*``, ``System``",30,11864,67,9
+   Others,"``Amazon.Lambda.APIGatewayEvents``, ``Amazon.Lambda.Core``, ``Dapper``, ``ILCompiler``, ``ILLink.RoslynAnalyzer``, ``ILLink.Shared``, ``ILLink.Tasks``, ``Internal.IL``, ``Internal.Pgo``, ``Internal.TypeSystem``, ``JsonToItemsTaskFactory``, ``Microsoft.Android.Build``, ``Microsoft.Apple.Build``, ``Microsoft.ApplicationBlocks.Data``, ``Microsoft.CSharp``, ``Microsoft.Diagnostics.Tools.Pgo``, ``Microsoft.EntityFrameworkCore``, ``Microsoft.Extensions.Caching.Distributed``, ``Microsoft.Extensions.Caching.Memory``, ``Microsoft.Extensions.Configuration``, ``Microsoft.Extensions.DependencyInjection``, ``Microsoft.Extensions.DependencyModel``, ``Microsoft.Extensions.Diagnostics.Metrics``, ``Microsoft.Extensions.FileProviders``, ``Microsoft.Extensions.FileSystemGlobbing``, ``Microsoft.Extensions.Hosting``, ``Microsoft.Extensions.Http``, ``Microsoft.Extensions.Logging``, ``Microsoft.Extensions.Options``, ``Microsoft.Extensions.Primitives``, ``Microsoft.Interop``, ``Microsoft.NET.Build.Tasks``, ``Microsoft.NET.WebAssembly.Webcil``, ``Microsoft.VisualBasic``, ``Microsoft.WebAssembly.Build.Tasks``, ``Microsoft.Win32``, ``Mono.Linker``, ``MySql.Data.MySqlClient``, ``Newtonsoft.Json``, ``SourceGenerators``, ``Windows.Security.Cryptography.Core``",12,1547,148,
+   Totals,,42,13418,409,9
 

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs

@@ -1,514 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using System.Net.Http;
-using System.Text.RegularExpressions;
-using System.Threading;
-using System.Threading.Tasks;
-using Semmle.Util;
-
-namespace Semmle.Extraction.CSharp.DependencyFetching
-{
-    public sealed partial class DependencyManager
-    {
-        private void RestoreNugetPackages(List<FileInfo> allNonBinaryFiles, IEnumerable<string> allProjects, IEnumerable<string> allSolutions, HashSet<string> dllPaths)
-        {
-            try
-            {
-                var checkNugetFeedResponsiveness = EnvironmentVariables.GetBoolean(EnvironmentVariableNames.CheckNugetFeedResponsiveness);
-                if (checkNugetFeedResponsiveness && !CheckFeeds(allNonBinaryFiles))
-                {
-                    DownloadMissingPackages(allNonBinaryFiles, dllPaths, withNugetConfig: false);
-                    return;
-                }
-
-                using (var nuget = new NugetPackages(sourceDir.FullName, legacyPackageDirectory, logger))
-                {
-                    var count = nuget.InstallPackages();
-
-                    if (nuget.PackageCount > 0)
-                    {
-                        CompilationInfos.Add(("packages.config files", nuget.PackageCount.ToString()));
-                        CompilationInfos.Add(("Successfully restored packages.config files", count.ToString()));
-                    }
-                }
-
-                var nugetPackageDlls = legacyPackageDirectory.DirInfo.GetFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = true });
-                var nugetPackageDllPaths = nugetPackageDlls.Select(f => f.FullName).ToHashSet();
-                var excludedPaths = nugetPackageDllPaths
-                    .Where(path => IsPathInSubfolder(path, legacyPackageDirectory.DirInfo.FullName, "tools"))
-                    .ToList();
-
-                if (nugetPackageDllPaths.Count > 0)
-                {
-                    logger.LogInfo($"Restored {nugetPackageDllPaths.Count} Nuget DLLs.");
-                }
-                if (excludedPaths.Count > 0)
-                {
-                    logger.LogInfo($"Excluding {excludedPaths.Count} Nuget DLLs.");
-                }
-
-                foreach (var excludedPath in excludedPaths)
-                {
-                    logger.LogInfo($"Excluded Nuget DLL: {excludedPath}");
-                }
-
-                nugetPackageDllPaths.ExceptWith(excludedPaths);
-                dllPaths.UnionWith(nugetPackageDllPaths);
-            }
-            catch (Exception exc)
-            {
-                logger.LogError($"Failed to restore Nuget packages with nuget.exe: {exc.Message}");
-            }
-
-            var restoredProjects = RestoreSolutions(allSolutions, out var assets1);
-            var projects = allProjects.Except(restoredProjects);
-            RestoreProjects(projects, out var assets2);
-
-            var dependencies = Assets.GetCompilationDependencies(logger, assets1.Union(assets2));
-
-            var paths = dependencies
-                .Paths
-                .Select(d => Path.Combine(packageDirectory.DirInfo.FullName, d))
-                .ToList();
-            dllPaths.UnionWith(paths);
-
-            LogAllUnusedPackages(dependencies);
-            DownloadMissingPackages(allNonBinaryFiles, dllPaths);
-        }
-
-        /// <summary>
-        /// Executes `dotnet restore` on all solution files in solutions.
-        /// As opposed to RestoreProjects this is not run in parallel using PLINQ
-        /// as `dotnet restore` on a solution already uses multiple threads for restoring
-        /// the projects (this can be disabled with the `--disable-parallel` flag).
-        /// Populates assets with the relative paths to the assets files generated by the restore.
-        /// Returns a list of projects that are up to date with respect to restore.
-        /// </summary>
-        /// <param name="solutions">A list of paths to solution files.</param>
-        private IEnumerable<string> RestoreSolutions(IEnumerable<string> solutions, out IEnumerable<string> assets)
-        {
-            var successCount = 0;
-            var nugetSourceFailures = 0;
-            var assetFiles = new List<string>();
-            var projects = solutions.SelectMany(solution =>
-                {
-                    logger.LogInfo($"Restoring solution {solution}...");
-                    var res = dotnet.Restore(new(solution, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
-                    if (res.Success)
-                    {
-                        successCount++;
-                    }
-                    if (res.HasNugetPackageSourceError)
-                    {
-                        nugetSourceFailures++;
-                    }
-                    assetFiles.AddRange(res.AssetsFilePaths);
-                    return res.RestoredProjects;
-                }).ToList();
-            assets = assetFiles;
-            CompilationInfos.Add(("Successfully restored solution files", successCount.ToString()));
-            CompilationInfos.Add(("Failed solution restore with package source error", nugetSourceFailures.ToString()));
-            CompilationInfos.Add(("Restored projects through solution files", projects.Count.ToString()));
-            return projects;
-        }
-
-        /// <summary>
-        /// Executes `dotnet restore` on all projects in projects.
-        /// This is done in parallel for performance reasons.
-        /// Populates assets with the relative paths to the assets files generated by the restore.
-        /// </summary>
-        /// <param name="projects">A list of paths to project files.</param>
-        private void RestoreProjects(IEnumerable<string> projects, out IEnumerable<string> assets)
-        {
-            var successCount = 0;
-            var nugetSourceFailures = 0;
-            var assetFiles = new List<string>();
-            var sync = new object();
-            Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = threads }, project =>
-            {
-                logger.LogInfo($"Restoring project {project}...");
-                var res = dotnet.Restore(new(project, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
-                lock (sync)
-                {
-                    if (res.Success)
-                    {
-                        successCount++;
-                    }
-                    if (res.HasNugetPackageSourceError)
-                    {
-                        nugetSourceFailures++;
-                    }
-                    assetFiles.AddRange(res.AssetsFilePaths);
-                }
-            });
-            assets = assetFiles;
-            CompilationInfos.Add(("Successfully restored project files", successCount.ToString()));
-            CompilationInfos.Add(("Failed project restore with package source error", nugetSourceFailures.ToString()));
-        }
-
-        private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPaths, bool withNugetConfig = true)
-        {
-            var alreadyDownloadedPackages = GetRestoredPackageDirectoryNames(packageDirectory.DirInfo);
-            var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames();
-
-            var notYetDownloadedPackages = new HashSet<PackageReference>(fileContent.AllPackages);
-            foreach (var alreadyDownloadedPackage in alreadyDownloadedPackages)
-            {
-                notYetDownloadedPackages.Remove(new(alreadyDownloadedPackage, PackageReferenceSource.SdkCsProj));
-            }
-            foreach (var alreadyDownloadedLegacyPackage in alreadyDownloadedLegacyPackages)
-            {
-                notYetDownloadedPackages.Remove(new(alreadyDownloadedLegacyPackage, PackageReferenceSource.PackagesConfig));
-            }
-
-            if (notYetDownloadedPackages.Count == 0)
-            {
-                return;
-            }
-
-            var multipleVersions = notYetDownloadedPackages
-                .GroupBy(p => p.Name)
-                .Where(g => g.Count() > 1)
-                .Select(g => g.Key)
-                .ToList();
-
-            foreach (var package in multipleVersions)
-            {
-                logger.LogWarning($"Found multiple not yet restored packages with name '{package}'.");
-                notYetDownloadedPackages.Remove(new(package, PackageReferenceSource.PackagesConfig));
-            }
-
-            logger.LogInfo($"Found {notYetDownloadedPackages.Count} packages that are not yet restored");
-            var nugetConfig = withNugetConfig
-                ? GetNugetConfig(allFiles)
-                : null;
-
-            CompilationInfos.Add(("Fallback nuget restore", notYetDownloadedPackages.Count.ToString()));
-
-            var successCount = 0;
-            var sync = new object();
-
-            Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = threads }, package =>
-            {
-                var success = TryRestorePackageManually(package.Name, nugetConfig, package.PackageReferenceSource);
-                if (!success)
-                {
-                    return;
-                }
-
-                lock (sync)
-                {
-                    successCount++;
-                }
-            });
-
-            CompilationInfos.Add(("Successfully ran fallback nuget restore", successCount.ToString()));
-
-            dllPaths.Add(missingPackageDirectory.DirInfo.FullName);
-        }
-
-        private string[] GetAllNugetConfigs(List<FileInfo> allFiles) => allFiles.SelectFileNamesByName("nuget.config").ToArray();
-
-        private string? GetNugetConfig(List<FileInfo> allFiles)
-        {
-            var nugetConfigs = GetAllNugetConfigs(allFiles);
-            string? nugetConfig;
-            if (nugetConfigs.Length > 1)
-            {
-                logger.LogInfo($"Found multiple nuget.config files: {string.Join(", ", nugetConfigs)}.");
-                nugetConfig = allFiles
-                    .SelectRootFiles(sourceDir)
-                    .SelectFileNamesByName("nuget.config")
-                    .FirstOrDefault();
-                if (nugetConfig == null)
-                {
-                    logger.LogInfo("Could not find a top-level nuget.config file.");
-                }
-            }
-            else
-            {
-                nugetConfig = nugetConfigs.FirstOrDefault();
-            }
-
-            if (nugetConfig != null)
-            {
-                logger.LogInfo($"Using nuget.config file {nugetConfig}.");
-            }
-
-            return nugetConfig;
-        }
-
-        private void LogAllUnusedPackages(DependencyContainer dependencies)
-        {
-            var allPackageDirectories = GetAllPackageDirectories();
-
-            logger.LogInfo($"Restored {allPackageDirectories.Count} packages");
-            logger.LogInfo($"Found {dependencies.Packages.Count} packages in project.assets.json files");
-
-            allPackageDirectories
-                .Where(package => !dependencies.Packages.Contains(package))
-                .Order()
-                .ForEach(package => logger.LogInfo($"Unused package: {package}"));
-        }
-
-
-        private ICollection<string> GetAllPackageDirectories()
-        {
-            return new DirectoryInfo(packageDirectory.DirInfo.FullName)
-                .EnumerateDirectories("*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
-                .Select(d => d.Name)
-                .ToList();
-        }
-
-        private static bool IsPathInSubfolder(string path, string rootFolder, string subFolder)
-        {
-            return path.IndexOf(
-                $"{Path.DirectorySeparatorChar}{subFolder}{Path.DirectorySeparatorChar}",
-                rootFolder.Length,
-                StringComparison.InvariantCultureIgnoreCase) >= 0;
-        }
-
-        private IEnumerable<string> GetRestoredLegacyPackageNames()
-        {
-            var oldPackageDirectories = GetRestoredPackageDirectoryNames(legacyPackageDirectory.DirInfo);
-            foreach (var oldPackageDirectory in oldPackageDirectories)
-            {
-                // nuget install restores packages to 'packagename.version' folders (dotnet restore to 'packagename/version' folders)
-                // typical folder names look like:
-                //   newtonsoft.json.13.0.3
-                // there are more complex ones too, such as:
-                //   runtime.tizen.4.0.0-armel.Microsoft.NETCore.DotNetHostResolver.2.0.0-preview2-25407-01
-
-                var match = LegacyNugetPackage().Match(oldPackageDirectory);
-                if (!match.Success)
-                {
-                    logger.LogWarning($"Package directory '{oldPackageDirectory}' doesn't match the expected pattern.");
-                    continue;
-                }
-
-                yield return match.Groups[1].Value.ToLowerInvariant();
-            }
-        }
-
-        private static IEnumerable<string> GetRestoredPackageDirectoryNames(DirectoryInfo root)
-        {
-            return Directory.GetDirectories(root.FullName)
-                .Select(d => Path.GetFileName(d).ToLowerInvariant());
-        }
-
-        private bool TryRestorePackageManually(string package, string? nugetConfig, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj)
-        {
-            logger.LogInfo($"Restoring package {package}...");
-            using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package, "missingpackages_workingdir"));
-            var success = dotnet.New(tempDir.DirInfo.FullName);
-            if (!success)
-            {
-                return false;
-            }
-
-            if (packageReferenceSource == PackageReferenceSource.PackagesConfig)
-            {
-                TryChangeTargetFrameworkMoniker(tempDir.DirInfo);
-            }
-
-            success = dotnet.AddPackage(tempDir.DirInfo.FullName, package);
-            if (!success)
-            {
-                return false;
-            }
-
-            var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig));
-            if (!res.Success)
-            {
-                if (res.HasNugetPackageSourceError && nugetConfig is not null)
-                {
-                    // Restore could not be completed because the listed source is unavailable. Try without the nuget.config:
-                    res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: null, ForceReevaluation: true));
-                }
-
-                // TODO: the restore might fail, we could retry with
-                // - a prerelease (*-* instead of *) version of the package,
-                // - a different target framework moniker.
-
-                if (!res.Success)
-                {
-                    logger.LogInfo($"Failed to restore nuget package {package}");
-                    return false;
-                }
-            }
-
-            return true;
-        }
-
-        private void TryChangeTargetFrameworkMoniker(DirectoryInfo tempDir)
-        {
-            try
-            {
-                logger.LogInfo($"Changing the target framework moniker in {tempDir.FullName}...");
-
-                var csprojs = tempDir.GetFiles("*.csproj", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive });
-                if (csprojs.Length != 1)
-                {
-                    logger.LogError($"Could not find the .csproj file in {tempDir.FullName}, count = {csprojs.Length}");
-                    return;
-                }
-
-                var csproj = csprojs[0];
-                var content = File.ReadAllText(csproj.FullName);
-                var matches = TargetFramework().Matches(content);
-                if (matches.Count == 0)
-                {
-                    logger.LogError($"Could not find target framework in {csproj.FullName}");
-                }
-                else
-                {
-                    content = TargetFramework().Replace(content, $"<TargetFramework>{FrameworkPackageNames.LatestNetFrameworkMoniker}</TargetFramework>", 1);
-                    File.WriteAllText(csproj.FullName, content);
-                }
-            }
-            catch (Exception exc)
-            {
-                logger.LogError($"Failed to update target framework in {tempDir.FullName}: {exc}");
-            }
-        }
-
-        private static async Task ExecuteGetRequest(string address, HttpClient httpClient, CancellationToken cancellationToken)
-        {
-            using var stream = await httpClient.GetStreamAsync(address, cancellationToken);
-            var buffer = new byte[1024];
-            int bytesRead;
-            while ((bytesRead = stream.Read(buffer, 0, buffer.Length)) > 0)
-            {
-                // do nothing
-            }
-        }
-
-        private bool IsFeedReachable(string feed)
-        {
-            logger.LogInfo($"Checking if Nuget feed '{feed}' is reachable...");
-            using HttpClient client = new();
-            int timeoutMilliSeconds = int.TryParse(Environment.GetEnvironmentVariable(EnvironmentVariableNames.NugetFeedResponsivenessInitialTimeout), out timeoutMilliSeconds)
-                ? timeoutMilliSeconds
-                : 1000;
-            int tryCount = int.TryParse(Environment.GetEnvironmentVariable(EnvironmentVariableNames.NugetFeedResponsivenessRequestCount), out tryCount)
-                ? tryCount
-                : 4;
-
-            for (var i = 0; i < tryCount; i++)
-            {
-                using var cts = new CancellationTokenSource();
-                cts.CancelAfter(timeoutMilliSeconds);
-                try
-                {
-                    ExecuteGetRequest(feed, client, cts.Token).GetAwaiter().GetResult();
-                    return true;
-                }
-                catch (Exception exc)
-                {
-                    if (exc is TaskCanceledException tce &&
-                        tce.CancellationToken == cts.Token &&
-                        cts.Token.IsCancellationRequested)
-                    {
-                        logger.LogWarning($"Didn't receive answer from Nuget feed '{feed}' in {timeoutMilliSeconds}ms.");
-                        timeoutMilliSeconds *= 2;
-                        continue;
-                    }
-
-                    // We're only interested in timeouts.
-                    logger.LogWarning($"Querying Nuget feed '{feed}' failed: {exc}");
-                    return true;
-                }
-            }
-
-            logger.LogWarning($"Didn't receive answer from Nuget feed '{feed}'. Tried it {tryCount} times.");
-            return false;
-        }
-
-        private bool CheckFeeds(List<FileInfo> allFiles)
-        {
-            logger.LogInfo("Checking Nuget feeds...");
-            var feeds = GetAllFeeds(allFiles);
-
-            var excludedFeeds = Environment.GetEnvironmentVariable(EnvironmentVariableNames.ExcludedNugetFeedsFromResponsivenessCheck)
-                ?.Split(" ", StringSplitOptions.RemoveEmptyEntries)
-                .ToHashSet() ?? [];
-
-            if (excludedFeeds.Count > 0)
-            {
-                logger.LogInfo($"Excluded Nuget feeds from responsiveness check: {string.Join(", ", excludedFeeds.OrderBy(f => f))}");
-            }
-
-            var allFeedsReachable = feeds.All(feed => excludedFeeds.Contains(feed) || IsFeedReachable(feed));
-            if (!allFeedsReachable)
-            {
-                logger.LogWarning("Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.");
-                diagnosticsWriter.AddEntry(new DiagnosticMessage(
-                    Language.CSharp,
-                    "buildless/unreachable-feed",
-                    "Found unreachable Nuget feed in C# analysis with build-mode 'none'",
-                    visibility: new DiagnosticMessage.TspVisibility(statusPage: true, cliSummaryTable: true, telemetry: true),
-                    markdownMessage: "Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.",
-                    severity: DiagnosticMessage.TspSeverity.Warning
-                ));
-            }
-            CompilationInfos.Add(("All Nuget feeds reachable", allFeedsReachable ? "1" : "0"));
-            return allFeedsReachable;
-        }
-
-        private IEnumerable<string> GetFeeds(string nugetConfig)
-        {
-            logger.LogInfo($"Getting Nuget feeds from '{nugetConfig}'...");
-            var results = dotnet.GetNugetFeeds(nugetConfig);
-            var regex = EnabledNugetFeed();
-            foreach (var result in results)
-            {
-                var match = regex.Match(result);
-                if (!match.Success)
-                {
-                    logger.LogError($"Failed to parse feed from '{result}'");
-                    continue;
-                }
-
-                var url = match.Groups[1].Value;
-                if (!url.StartsWith("https://", StringComparison.InvariantCultureIgnoreCase) &&
-                    !url.StartsWith("http://", StringComparison.InvariantCultureIgnoreCase))
-                {
-                    logger.LogInfo($"Skipping feed '{url}' as it is not a valid URL.");
-                    continue;
-                }
-
-                yield return url;
-            }
-        }
-
-        private HashSet<string> GetAllFeeds(List<FileInfo> allFiles)
-        {
-            var nugetConfigs = GetAllNugetConfigs(allFiles);
-            var feeds = nugetConfigs
-                .SelectMany(GetFeeds)
-                .Where(str => !string.IsNullOrWhiteSpace(str))
-                .ToHashSet();
-
-            if (feeds.Count > 0)
-            {
-                logger.LogInfo($"Found {feeds.Count} Nuget feeds in nuget.config files: {string.Join(", ", feeds.OrderBy(f => f))}");
-            }
-            else
-            {
-                logger.LogDebug("No Nuget feeds found in nuget.config files.");
-            }
-            return feeds;
-        }
-
-        [GeneratedRegex(@"<TargetFramework>.*</TargetFramework>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
-        private static partial Regex TargetFramework();
-
-        [GeneratedRegex(@"^(.+)\.(\d+\.\d+\.\d+(-(.+))?)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
-        private static partial Regex LegacyNugetPackage();
-
-        [GeneratedRegex(@"^E\s(.*)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
-        private static partial Regex EnabledNugetFeed();
-    }
-}

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -20,7 +20,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
     {
         private readonly AssemblyCache assemblyCache;
         private readonly ILogger logger;
-        private readonly IDiagnosticsWriter diagnosticsWriter;
 
         // Only used as a set, but ConcurrentDictionary is the only concurrent set in .NET.
         private readonly IDictionary<string, bool> usedReferences = new ConcurrentDictionary<string, bool>();
@@ -53,25 +52,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             var startTime = DateTime.Now;
 
             this.logger = logger;
-
-            var diagDirEnv = Environment.GetEnvironmentVariable(EnvironmentVariableNames.DiagnosticDir);
-            if (!string.IsNullOrWhiteSpace(diagDirEnv) &&
-                !Directory.Exists(diagDirEnv))
-            {
-                try
-                {
-                    Directory.CreateDirectory(diagDirEnv);
-                }
-                catch (Exception e)
-                {
-                    logger.LogError($"Failed to create diagnostic directory {diagDirEnv}: {e.Message}");
-                    diagDirEnv = null;
-                }
-            }
-
-            this.diagnosticsWriter = new DiagnosticsStream(Path.Combine(
-                diagDirEnv ?? "",
-                $"dependency-manager-{DateTime.UtcNow:yyyyMMddHHmm}-{Environment.ProcessId}.jsonc"));
             this.sourceDir = new DirectoryInfo(srcDir);
 
             packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName, "packages"));
@@ -197,7 +177,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             var frameworkLocations = new HashSet<string>();
 
             var frameworkReferences = Environment.GetEnvironmentVariable(EnvironmentVariableNames.DotnetFrameworkReferences);
-            var useSubfolders = EnvironmentVariables.GetBoolean(EnvironmentVariableNames.DotnetFrameworkReferencesUseSubfolders);
+            var frameworkReferencesUseSubfolders = Environment.GetEnvironmentVariable(EnvironmentVariableNames.DotnetFrameworkReferencesUseSubfolders);
+            _ = bool.TryParse(frameworkReferencesUseSubfolders, out var useSubfolders);
             if (!string.IsNullOrWhiteSpace(frameworkReferences))
             {
                 RemoveFrameworkNugetPackages(dllPaths);
@@ -249,6 +230,73 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             return frameworkLocations;
         }
 
+        private void RestoreNugetPackages(List<FileInfo> allNonBinaryFiles, IEnumerable<string> allProjects, IEnumerable<string> allSolutions, HashSet<string> dllPaths)
+        {
+            try
+            {
+                using (var nuget = new NugetPackages(sourceDir.FullName, legacyPackageDirectory, logger))
+                {
+                    var count = nuget.InstallPackages();
+
+                    if (nuget.PackageCount > 0)
+                    {
+                        CompilationInfos.Add(("packages.config files", nuget.PackageCount.ToString()));
+                        CompilationInfos.Add(("Successfully restored packages.config files", count.ToString()));
+                    }
+                }
+
+                var nugetPackageDlls = legacyPackageDirectory.DirInfo.GetFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = true });
+                var nugetPackageDllPaths = nugetPackageDlls.Select(f => f.FullName).ToHashSet();
+                var excludedPaths = nugetPackageDllPaths
+                    .Where(path => IsPathInSubfolder(path, legacyPackageDirectory.DirInfo.FullName, "tools"))
+                    .ToList();
+
+                if (nugetPackageDllPaths.Count > 0)
+                {
+                    logger.LogInfo($"Restored {nugetPackageDllPaths.Count} Nuget DLLs.");
+                }
+                if (excludedPaths.Count > 0)
+                {
+                    logger.LogInfo($"Excluding {excludedPaths.Count} Nuget DLLs.");
+                }
+
+                foreach (var excludedPath in excludedPaths)
+                {
+                    logger.LogInfo($"Excluded Nuget DLL: {excludedPath}");
+                }
+
+                nugetPackageDllPaths.ExceptWith(excludedPaths);
+                dllPaths.UnionWith(nugetPackageDllPaths);
+            }
+            catch (Exception exc)
+            {
+                logger.LogError($"Failed to restore Nuget packages with nuget.exe: {exc.Message}");
+            }
+
+            var restoredProjects = RestoreSolutions(allSolutions, out var assets1);
+            var projects = allProjects.Except(restoredProjects);
+            RestoreProjects(projects, out var assets2);
+
+            var dependencies = Assets.GetCompilationDependencies(logger, assets1.Union(assets2));
+
+            var paths = dependencies
+                .Paths
+                .Select(d => Path.Combine(packageDirectory.DirInfo.FullName, d))
+                .ToList();
+            dllPaths.UnionWith(paths);
+
+            LogAllUnusedPackages(dependencies);
+            DownloadMissingPackages(allNonBinaryFiles, dllPaths);
+        }
+
+        private static bool IsPathInSubfolder(string path, string rootFolder, string subFolder)
+        {
+            return path.IndexOf(
+                $"{Path.DirectorySeparatorChar}{subFolder}{Path.DirectorySeparatorChar}",
+                rootFolder.Length,
+                StringComparison.InvariantCultureIgnoreCase) >= 0;
+        }
+
         private void RemoveNugetAnalyzerReferences()
         {
             var packageFolder = packageDirectory.DirInfo.FullName.ToLowerInvariant();
@@ -435,6 +483,27 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                 .FullName;
         }
 
+        private ICollection<string> GetAllPackageDirectories()
+        {
+            return new DirectoryInfo(packageDirectory.DirInfo.FullName)
+                .EnumerateDirectories("*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
+                .Select(d => d.Name)
+                .ToList();
+        }
+
+        private void LogAllUnusedPackages(DependencyContainer dependencies)
+        {
+            var allPackageDirectories = GetAllPackageDirectories();
+
+            logger.LogInfo($"Restored {allPackageDirectories.Count} packages");
+            logger.LogInfo($"Found {dependencies.Packages.Count} packages in project.assets.json files");
+
+            allPackageDirectories
+                .Where(package => !dependencies.Packages.Contains(package))
+                .Order()
+                .ForEach(package => logger.LogInfo($"Unused package: {package}"));
+        }
+
         private void GenerateSourceFileFromImplicitUsings()
         {
             var usings = new HashSet<string>();
@@ -659,11 +728,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         /// </summary>
         public IEnumerable<string> GeneratedSourceFiles => generatedSources;
 
-        /// <summary>
-        /// All of the non-generated source files in the source directory.
-        /// </summary>
-        public IEnumerable<string> NonGeneratedSourcesFiles => nonGeneratedSources;
-
         /// <summary>
         /// All of the source files in the source directory.
         /// </summary>
@@ -738,6 +802,269 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             }
         }
 
+        /// <summary>
+        /// Executes `dotnet restore` on all solution files in solutions.
+        /// As opposed to RestoreProjects this is not run in parallel using PLINQ
+        /// as `dotnet restore` on a solution already uses multiple threads for restoring
+        /// the projects (this can be disabled with the `--disable-parallel` flag).
+        /// Populates assets with the relative paths to the assets files generated by the restore.
+        /// Returns a list of projects that are up to date with respect to restore.
+        /// </summary>
+        /// <param name="solutions">A list of paths to solution files.</param>
+        private IEnumerable<string> RestoreSolutions(IEnumerable<string> solutions, out IEnumerable<string> assets)
+        {
+            var successCount = 0;
+            var nugetSourceFailures = 0;
+            var assetFiles = new List<string>();
+            var projects = solutions.SelectMany(solution =>
+                {
+                    logger.LogInfo($"Restoring solution {solution}...");
+                    var res = dotnet.Restore(new(solution, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
+                    if (res.Success)
+                    {
+                        successCount++;
+                    }
+                    if (res.HasNugetPackageSourceError)
+                    {
+                        nugetSourceFailures++;
+                    }
+                    assetFiles.AddRange(res.AssetsFilePaths);
+                    return res.RestoredProjects;
+                }).ToList();
+            assets = assetFiles;
+            CompilationInfos.Add(("Successfully restored solution files", successCount.ToString()));
+            CompilationInfos.Add(("Failed solution restore with package source error", nugetSourceFailures.ToString()));
+            CompilationInfos.Add(("Restored projects through solution files", projects.Count.ToString()));
+            return projects;
+        }
+
+        /// <summary>
+        /// Executes `dotnet restore` on all projects in projects.
+        /// This is done in parallel for performance reasons.
+        /// Populates assets with the relative paths to the assets files generated by the restore.
+        /// </summary>
+        /// <param name="projects">A list of paths to project files.</param>
+        private void RestoreProjects(IEnumerable<string> projects, out IEnumerable<string> assets)
+        {
+            var successCount = 0;
+            var nugetSourceFailures = 0;
+            var assetFiles = new List<string>();
+            var sync = new object();
+            Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = threads }, project =>
+            {
+                logger.LogInfo($"Restoring project {project}...");
+                var res = dotnet.Restore(new(project, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
+                lock (sync)
+                {
+                    if (res.Success)
+                    {
+                        successCount++;
+                    }
+                    if (res.HasNugetPackageSourceError)
+                    {
+                        nugetSourceFailures++;
+                    }
+                    assetFiles.AddRange(res.AssetsFilePaths);
+                }
+            });
+            assets = assetFiles;
+            CompilationInfos.Add(("Successfully restored project files", successCount.ToString()));
+            CompilationInfos.Add(("Failed project restore with package source error", nugetSourceFailures.ToString()));
+        }
+
+        [GeneratedRegex(@"^(.+)\.(\d+\.\d+\.\d+(-(.+))?)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
+        private static partial Regex LegacyNugetPackage();
+
+
+        private static IEnumerable<string> GetRestoredPackageDirectoryNames(DirectoryInfo root)
+        {
+            return Directory.GetDirectories(root.FullName)
+                .Select(d => Path.GetFileName(d).ToLowerInvariant());
+        }
+
+        private IEnumerable<string> GetRestoredLegacyPackageNames()
+        {
+            var oldPackageDirectories = GetRestoredPackageDirectoryNames(legacyPackageDirectory.DirInfo);
+            foreach (var oldPackageDirectory in oldPackageDirectories)
+            {
+                // nuget install restores packages to 'packagename.version' folders (dotnet restore to 'packagename/version' folders)
+                // typical folder names look like:
+                //   newtonsoft.json.13.0.3
+                // there are more complex ones too, such as:
+                //   runtime.tizen.4.0.0-armel.Microsoft.NETCore.DotNetHostResolver.2.0.0-preview2-25407-01
+
+                var match = LegacyNugetPackage().Match(oldPackageDirectory);
+                if (!match.Success)
+                {
+                    logger.LogWarning($"Package directory '{oldPackageDirectory}' doesn't match the expected pattern.");
+                    continue;
+                }
+
+                yield return match.Groups[1].Value.ToLowerInvariant();
+            }
+        }
+
+        private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPaths)
+        {
+            var alreadyDownloadedPackages = GetRestoredPackageDirectoryNames(packageDirectory.DirInfo);
+            var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames();
+
+            var notYetDownloadedPackages = new HashSet<PackageReference>(fileContent.AllPackages);
+            foreach (var alreadyDownloadedPackage in alreadyDownloadedPackages)
+            {
+                notYetDownloadedPackages.Remove(new(alreadyDownloadedPackage, PackageReferenceSource.SdkCsProj));
+            }
+            foreach (var alreadyDownloadedLegacyPackage in alreadyDownloadedLegacyPackages)
+            {
+                notYetDownloadedPackages.Remove(new(alreadyDownloadedLegacyPackage, PackageReferenceSource.PackagesConfig));
+            }
+
+            if (notYetDownloadedPackages.Count == 0)
+            {
+                return;
+            }
+
+            var multipleVersions = notYetDownloadedPackages
+                .GroupBy(p => p.Name)
+                .Where(g => g.Count() > 1)
+                .Select(g => g.Key)
+                .ToList();
+
+            foreach (var package in multipleVersions)
+            {
+                logger.LogWarning($"Found multiple not yet restored packages with name '{package}'.");
+                notYetDownloadedPackages.Remove(new(package, PackageReferenceSource.PackagesConfig));
+            }
+
+            logger.LogInfo($"Found {notYetDownloadedPackages.Count} packages that are not yet restored");
+
+            var nugetConfigs = allFiles.SelectFileNamesByName("nuget.config").ToArray();
+            string? nugetConfig = null;
+            if (nugetConfigs.Length > 1)
+            {
+                logger.LogInfo($"Found multiple nuget.config files: {string.Join(", ", nugetConfigs)}.");
+                nugetConfig = allFiles
+                    .SelectRootFiles(sourceDir)
+                    .SelectFileNamesByName("nuget.config")
+                    .FirstOrDefault();
+                if (nugetConfig == null)
+                {
+                    logger.LogInfo("Could not find a top-level nuget.config file.");
+                }
+            }
+            else
+            {
+                nugetConfig = nugetConfigs.FirstOrDefault();
+            }
+
+            if (nugetConfig != null)
+            {
+                logger.LogInfo($"Using nuget.config file {nugetConfig}.");
+            }
+
+            CompilationInfos.Add(("Fallback nuget restore", notYetDownloadedPackages.Count.ToString()));
+
+            var successCount = 0;
+            var sync = new object();
+
+            Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = threads }, package =>
+            {
+                var success = TryRestorePackageManually(package.Name, nugetConfig, package.PackageReferenceSource);
+                if (!success)
+                {
+                    return;
+                }
+
+                lock (sync)
+                {
+                    successCount++;
+                }
+            });
+
+            CompilationInfos.Add(("Successfully ran fallback nuget restore", successCount.ToString()));
+
+            dllPaths.Add(missingPackageDirectory.DirInfo.FullName);
+        }
+
+        [GeneratedRegex(@"<TargetFramework>.*</TargetFramework>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
+        private static partial Regex TargetFramework();
+
+        private bool TryRestorePackageManually(string package, string? nugetConfig, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj)
+        {
+            logger.LogInfo($"Restoring package {package}...");
+            using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package, "missingpackages_workingdir"));
+            var success = dotnet.New(tempDir.DirInfo.FullName);
+            if (!success)
+            {
+                return false;
+            }
+
+            if (packageReferenceSource == PackageReferenceSource.PackagesConfig)
+            {
+                TryChangeTargetFrameworkMoniker(tempDir.DirInfo);
+            }
+
+            success = dotnet.AddPackage(tempDir.DirInfo.FullName, package);
+            if (!success)
+            {
+                return false;
+            }
+
+            var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig));
+            if (!res.Success)
+            {
+                if (res.HasNugetPackageSourceError && nugetConfig is not null)
+                {
+                    // Restore could not be completed because the listed source is unavailable. Try without the nuget.config:
+                    res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: null, ForceReevaluation: true));
+                }
+
+                // TODO: the restore might fail, we could retry with
+                // - a prerelease (*-* instead of *) version of the package,
+                // - a different target framework moniker.
+
+                if (!res.Success)
+                {
+                    logger.LogInfo($"Failed to restore nuget package {package}");
+                    return false;
+                }
+            }
+
+            return true;
+        }
+
+        private void TryChangeTargetFrameworkMoniker(DirectoryInfo tempDir)
+        {
+            try
+            {
+                logger.LogInfo($"Changing the target framework moniker in {tempDir.FullName}...");
+
+                var csprojs = tempDir.GetFiles("*.csproj", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive });
+                if (csprojs.Length != 1)
+                {
+                    logger.LogError($"Could not find the .csproj file in {tempDir.FullName}, count = {csprojs.Length}");
+                    return;
+                }
+
+                var csproj = csprojs[0];
+                var content = File.ReadAllText(csproj.FullName);
+                var matches = TargetFramework().Matches(content);
+                if (matches.Count == 0)
+                {
+                    logger.LogError($"Could not find target framework in {csproj.FullName}");
+                }
+                else
+                {
+                    content = TargetFramework().Replace(content, $"<TargetFramework>{FrameworkPackageNames.LatestNetFrameworkMoniker}</TargetFramework>", 1);
+                    File.WriteAllText(csproj.FullName, content);
+                }
+            }
+            catch (Exception exc)
+            {
+                logger.LogError($"Failed to update target framework in {tempDir.FullName}: {exc}");
+            }
+        }
+
         public void Dispose(TemporaryDirectory? dir, string name)
         {
             try
@@ -759,8 +1086,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             {
                 Dispose(tempWorkingDirectory, "temporary working");
             }
-
-            diagnosticsWriter?.Dispose();
         }
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs

@@ -16,14 +16,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
     public partial class DotNet : IDotNet
     {
         private readonly IDotNetCliInvoker dotnetCliInvoker;
-        private readonly ILogger logger;
         private readonly TemporaryDirectory? tempWorkingDirectory;
 
         private DotNet(IDotNetCliInvoker dotnetCliInvoker, ILogger logger, TemporaryDirectory? tempWorkingDirectory = null)
         {
             this.tempWorkingDirectory = tempWorkingDirectory;
             this.dotnetCliInvoker = dotnetCliInvoker;
-            this.logger = logger;
             Info();
         }
 
@@ -91,18 +89,17 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             return dotnetCliInvoker.RunCommand(args);
         }
 
-        public IList<string> GetListedRuntimes() => GetResultList("--list-runtimes");
+        public IList<string> GetListedRuntimes() => GetListed("--list-runtimes", "runtime");
 
-        public IList<string> GetListedSdks() => GetResultList("--list-sdks");
+        public IList<string> GetListedSdks() => GetListed("--list-sdks", "SDK");
 
-        private IList<string> GetResultList(string args)
+        private IList<string> GetListed(string args, string artifact)
         {
-            if (dotnetCliInvoker.RunCommand(args, out var results))
+            if (dotnetCliInvoker.RunCommand(args, out var artifacts))
             {
-                return results;
+                return artifacts;
             }
-            logger.LogWarning($"Running 'dotnet {args}' failed.");
-            return [];
+            return new List<string>();
         }
 
         public bool Exec(string execArgs)
@@ -111,8 +108,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             return dotnetCliInvoker.RunCommand(args);
         }
 
-        public IList<string> GetNugetFeeds(string nugetConfig) => GetResultList($"nuget list source --format Short --configfile \"{nugetConfig}\"");
-
         // The version number should be kept in sync with the version .NET version used for building the application.
         public const string LatestDotNetSdkVersion = "8.0.101";
 

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs

@@ -16,30 +16,5 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         /// Controls whether to use framework dependencies from subfolders.
         /// </summary>
         public const string DotnetFrameworkReferencesUseSubfolders = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_DOTNET_FRAMEWORK_REFERENCES_USE_SUBFOLDERS";
-
-        /// <summary>
-        /// Controls whether to check the responsiveness of NuGet feeds.
-        /// </summary>
-        public const string CheckNugetFeedResponsiveness = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK";
-
-        /// <summary>
-        /// Specifies the NuGet feeds to exclude from the responsiveness check. The value is a space-separated list of feed URLs.
-        /// </summary>
-        public const string ExcludedNugetFeedsFromResponsivenessCheck = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_EXCLUDED";
-
-        /// <summary>
-        /// Specifies the timeout (as an integer) in milliseconds for the initial check of NuGet feeds responsiveness. The value is then doubled for each subsequent check.
-        /// </summary>
-        public const string NugetFeedResponsivenessInitialTimeout = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_TIMEOUT";
-
-        /// <summary>
-        /// Specifies how many requests to make to the NuGet feed to check its responsiveness.
-        /// </summary>
-        public const string NugetFeedResponsivenessRequestCount = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_LIMIT";
-
-        /// <summary>
-        /// Specifies the location of the diagnostic directory.
-        /// </summary>
-        public const string DiagnosticDir = "CODEQL_EXTRACTOR_CSHARP_DIAGNOSTIC_DIR";
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs

@@ -13,7 +13,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         IList<string> GetListedRuntimes();
         IList<string> GetListedSdks();
         bool Exec(string execArgs);
-        IList<string> GetNugetFeeds(string nugetConfig);
     }
 
     public record class RestoreSettings(string File, string PackageDirectory, bool ForceDotnetRefAssemblyFetching, string? PathToNugetConfig = null, bool ForceReevaluation = false);

csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs

@@ -143,10 +143,10 @@ namespace Semmle.Extraction.CSharp.Standalone
             stopwatch.Start();
 
             using var logger = new ConsoleLogger(options.Verbosity, logThreadId: true);
-            logger.Log(Severity.Info, "Extracting C# with build-mode set to 'none'");
+            logger.Log(Severity.Info, "Extracting C# in buildless mode");
             using var dependencyManager = new DependencyManager(options.SrcDir, logger);
 
-            if (!dependencyManager.NonGeneratedSourcesFiles.Any())
+            if (!dependencyManager.AllSourceFiles.Any())
             {
                 logger.Log(Severity.Error, "No source files found");
                 return ExitCode.Errors;

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expression.cs

@@ -100,14 +100,8 @@ namespace Semmle.Extraction.CSharp.Entities
         /// <param name="child">The child index.</param>
         /// <param name="type">A type hint.</param>
         /// <returns>The new expression.</returns>
-        public static Expression Create(Context cx, ExpressionSyntax node, IExpressionParentEntity parent, int child, Boolean isCompilerGenerated = false)
-        {
-            var info = new ExpressionNodeInfo(cx, node, parent, child)
-            {
-                IsCompilerGenerated = isCompilerGenerated
-            };
-            return CreateFromNode(info);
-        }
+        public static Expression Create(Context cx, ExpressionSyntax node, IExpressionParentEntity parent, int child) =>
+            CreateFromNode(new ExpressionNodeInfo(cx, node, parent, child));
 
         public static Expression CreateFromNode(ExpressionNodeInfo info) => Expressions.ImplicitCast.Create(info);
 

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ArrayCreation.cs

@@ -97,7 +97,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 ExprKind.ARRAY_CREATION,
                 parent,
                 childIndex,
-                isCompilerGenerated: true,
+                true,
                 null);
 
             var arrayCreation = new Expression(info);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Assignment.cs

@@ -26,10 +26,10 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
             if (operatorKind.HasValue)
             {
                 // Convert assignment such as `a += b` into `a = a + b`.
-                var simpleAssignExpr = new Expression(new ExpressionInfo(Context, Type, Location, ExprKind.SIMPLE_ASSIGN, this, 2, isCompilerGenerated: true, null));
+                var simpleAssignExpr = new Expression(new ExpressionInfo(Context, Type, Location, ExprKind.SIMPLE_ASSIGN, this, 2, false, null));
                 Create(Context, Syntax.Left, simpleAssignExpr, 1);
-                var opexpr = new Expression(new ExpressionInfo(Context, Type, Location, operatorKind.Value, simpleAssignExpr, 0, isCompilerGenerated: true, null));
-                Create(Context, Syntax.Left, opexpr, 0, isCompilerGenerated: true);
+                var opexpr = new Expression(new ExpressionInfo(Context, Type, Location, operatorKind.Value, simpleAssignExpr, 0, false, null));
+                Create(Context, Syntax.Left, opexpr, 0);
                 Create(Context, Syntax.Right, opexpr, 1);
                 opexpr.OperatorCall(trapFile, Syntax);
             }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Cast.cs

@@ -41,7 +41,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 ExprKind.CAST,
                 parent,
                 childIndex,
-                isCompilerGenerated: true,
+                true,
                 ValueAsString(value));
 
             var ret = new Expression(info);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Collections/Spread.cs

@@ -6,7 +6,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
     internal class Spread : Expression
     {
         public Spread(Context cx, SpreadElementSyntax syntax, IExpressionParentEntity parent, int child) :
-            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), ExprKind.SPREAD_ELEMENT, parent, child, isCompilerGenerated: false, null))
+            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), ExprKind.SPREAD_ELEMENT, parent, child, false, null))
         {
             Create(cx, syntax.Expression, this, 0);
         }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Default.cs

@@ -24,7 +24,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 ExprKind.DEFAULT,
                 parent,
                 childIndex,
-                isCompilerGenerated: true,
+                true,
                 value);
 
             return new Expression(info);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Discard.cs

@@ -11,7 +11,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
         }
 
         private Discard(Context cx, CSharpSyntaxNode syntax, IExpressionParentEntity parent, int child) :
-            base(new ExpressionInfo(cx, cx.GetType(syntax), cx.CreateLocation(syntax.GetLocation()), ExprKind.DISCARD, parent, child, isCompilerGenerated: false, null))
+            base(new ExpressionInfo(cx, cx.GetType(syntax), cx.CreateLocation(syntax.GetLocation()), ExprKind.DISCARD, parent, child, false, null))
         {
         }
 

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ElementAccess.cs

@@ -22,7 +22,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
             if (Kind == ExprKind.POINTER_INDIRECTION)
             {
                 var qualifierInfo = new ExpressionNodeInfo(Context, qualifier, this, 0);
-                var add = new Expression(new ExpressionInfo(Context, qualifierInfo.Type, Location, ExprKind.ADD, this, 0, isCompilerGenerated: false, null));
+                var add = new Expression(new ExpressionInfo(Context, qualifierInfo.Type, Location, ExprKind.ADD, this, 0, false, null));
                 qualifierInfo.SetParent(add, 0);
                 CreateFromNode(qualifierInfo);
                 PopulateArguments(trapFile, argumentList, 1);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ImplicitCast.cs

@@ -14,13 +14,13 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
         }
 
         private ImplicitCast(ExpressionNodeInfo info)
-            : base(new ExpressionInfo(info.Context, info.ConvertedType, info.Location, ExprKind.CAST, info.Parent, info.Child, isCompilerGenerated: true, info.ExprValue))
+            : base(new ExpressionInfo(info.Context, info.ConvertedType, info.Location, ExprKind.CAST, info.Parent, info.Child, true, info.ExprValue))
         {
             Expr = Factory.Create(new ExpressionNodeInfo(Context, info.Node, this, 0));
         }
 
         private ImplicitCast(ExpressionNodeInfo info, IMethodSymbol method)
-            : base(new ExpressionInfo(info.Context, info.ConvertedType, info.Location, ExprKind.OPERATOR_INVOCATION, info.Parent, info.Child, isCompilerGenerated: true, info.ExprValue))
+            : base(new ExpressionInfo(info.Context, info.ConvertedType, info.Location, ExprKind.OPERATOR_INVOCATION, info.Parent, info.Child, true, info.ExprValue))
         {
             Expr = Factory.Create(info.SetParent(this, 0));
 
@@ -65,7 +65,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                     kind,
                     parent,
                     childIndex,
-                    isCompilerGenerated: true,
+                    true,
                     v);
 
             var method = GetImplicitConversionMethod(type, value);
@@ -93,7 +93,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                     ExprKind.CAST,
                     parent,
                     childIndex,
-                    isCompilerGenerated: true,
+                    true,
                     ValueAsString(value));
 
             return new Expression(info);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Initializer.cs

@@ -45,7 +45,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 ExprKind.ARRAY_INIT,
                 parent,
                 index,
-                isCompilerGenerated: true,
+                true,
                 null);
 
             return new Expression(info);
@@ -132,7 +132,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 var addMethod = Method.Create(Context, collectionInfo.Symbol as IMethodSymbol);
                 var voidType = AnnotatedTypeSymbol.CreateNotAnnotated(Context.Compilation.GetSpecialType(SpecialType.System_Void));
 
-                var invocation = new Expression(new ExpressionInfo(Context, voidType, Context.CreateLocation(i.GetLocation()), ExprKind.METHOD_INVOCATION, this, child++, isCompilerGenerated: true, null));
+                var invocation = new Expression(new ExpressionInfo(Context, voidType, Context.CreateLocation(i.GetLocation()), ExprKind.METHOD_INVOCATION, this, child++, false, null));
 
                 if (addMethod is not null)
                     trapFile.expr_call(invocation, addMethod);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/InterpolatedString.cs

@@ -25,7 +25,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                     case SyntaxKind.InterpolatedStringText:
                         // Create a string literal
                         var interpolatedText = (InterpolatedStringTextSyntax)c;
-                        new Expression(new ExpressionInfo(Context, Type, Context.CreateLocation(c.GetLocation()), ExprKind.UTF16_STRING_LITERAL, this, child++, isCompilerGenerated: false, interpolatedText.TextToken.ValueText));
+                        new Expression(new ExpressionInfo(Context, Type, Context.CreateLocation(c.GetLocation()), ExprKind.UTF16_STRING_LITERAL, this, child++, false, interpolatedText.TextToken.ValueText));
                         break;
                     default:
                         throw new InternalError(c, $"Unhandled interpolation kind {c.Kind()}");

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Lambda.cs

@@ -63,7 +63,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
         public static Lambda Create(ExpressionNodeInfo info, ParenthesizedLambdaExpressionSyntax node) => new Lambda(info, node);
 
         private Lambda(ExpressionNodeInfo info, SimpleLambdaExpressionSyntax node)
-            : this(info.SetKind(ExprKind.LAMBDA), node.Body, [node.Parameter], null) { }
+            : this(info.SetKind(ExprKind.LAMBDA), node.Body, Enumerators.Singleton(node.Parameter), null) { }
 
         public static Lambda Create(ExpressionNodeInfo info, SimpleLambdaExpressionSyntax node) => new Lambda(info, node);
 

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Literal.cs

@@ -97,7 +97,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 kind,
                 parent,
                 childIndex,
-                isCompilerGenerated: true,
+                true,
                 ValueAsString(value));
 
             return new Expression(info);
@@ -112,7 +112,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 ExprKind.NULL_LITERAL,
                 parent,
                 childIndex,
-                isCompilerGenerated: true,
+                true,
                 ValueAsString(null));
 
             return new Expression(info);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ObjectCreation/AnonymousObjectCreation.cs

@@ -30,7 +30,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 return;
             }
 
-            var objectInitializer = new Expression(new ExpressionInfo(Context, Type, Location, ExprKind.OBJECT_INIT, this, -1, isCompilerGenerated: false, null));
+            var objectInitializer = new Expression(new ExpressionInfo(Context, Type, Location, ExprKind.OBJECT_INIT, this, -1, false, null));
 
             foreach (var init in Syntax.Initializers)
             {
@@ -40,11 +40,11 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 var type = property.GetAnnotatedType();
                 var loc = Context.CreateLocation(init.GetLocation());
 
-                var assignment = new Expression(new ExpressionInfo(Context, type, loc, ExprKind.SIMPLE_ASSIGN, objectInitializer, child++, isCompilerGenerated: false, null));
+                var assignment = new Expression(new ExpressionInfo(Context, type, loc, ExprKind.SIMPLE_ASSIGN, objectInitializer, child++, false, null));
                 Create(Context, init.Expression, assignment, 0);
                 Property.Create(Context, property);
 
-                var access = new Expression(new ExpressionInfo(Context, type, loc, ExprKind.PROPERTY_ACCESS, assignment, 1, isCompilerGenerated: false, null));
+                var access = new Expression(new ExpressionInfo(Context, type, loc, ExprKind.PROPERTY_ACCESS, assignment, 1, false, null));
                 trapFile.expr_access(access, propEntity);
             }
         }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ObjectCreation/DateTimeObjectCreation.cs

@@ -59,7 +59,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 ExprKind.OBJECT_CREATION,
                 parent,
                 childIndex,
-                isCompilerGenerated: true,
+                true,
                 null));
 
             var longTypeSymbol = constructorSymbol.Parameters[0].Type;

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Patterns/BinaryPattern.cs

@@ -8,7 +8,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
     internal class BinaryPattern : Expression
     {
         public BinaryPattern(Context cx, BinaryPatternSyntax syntax, IExpressionParentEntity parent, int child) :
-            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), GetKind(syntax.OperatorToken, syntax), parent, child, isCompilerGenerated: false, null))
+            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), GetKind(syntax.OperatorToken, syntax), parent, child, false, null))
         {
             Pattern.Create(cx, syntax.Left, this, 0);
             Pattern.Create(cx, syntax.Right, this, 1);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Patterns/ListPattern.cs

@@ -7,7 +7,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
     internal class ListPattern : Expression
     {
         internal ListPattern(Context cx, ListPatternSyntax syntax, IExpressionParentEntity parent, int child) :
-            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), ExprKind.LIST_PATTERN, parent, child, isCompilerGenerated: false, null))
+            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), ExprKind.LIST_PATTERN, parent, child, false, null))
         {
             syntax.Patterns.ForEach((p, i) => Pattern.Create(cx, p, this, i));
         }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Patterns/PositionalPattern.cs

@@ -7,7 +7,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
     internal class PositionalPattern : Expression
     {
         internal PositionalPattern(Context cx, PositionalPatternClauseSyntax posPc, IExpressionParentEntity parent, int child) :
-            base(new ExpressionInfo(cx, null, cx.CreateLocation(posPc.GetLocation()), ExprKind.POSITIONAL_PATTERN, parent, child, isCompilerGenerated: false, null))
+            base(new ExpressionInfo(cx, null, cx.CreateLocation(posPc.GetLocation()), ExprKind.POSITIONAL_PATTERN, parent, child, false, null))
         {
             posPc.Subpatterns.ForEach((p, i) => Pattern.Create(cx, p.Pattern, this, i));
         }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Patterns/PropertyPattern.cs

@@ -7,7 +7,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
     internal class PropertyPattern : Expression
     {
         internal PropertyPattern(Context cx, PropertyPatternClauseSyntax pp, IExpressionParentEntity parent, int child) :
-            base(new ExpressionInfo(cx, null, cx.CreateLocation(pp.GetLocation()), ExprKind.PROPERTY_PATTERN, parent, child, isCompilerGenerated: false, null))
+            base(new ExpressionInfo(cx, null, cx.CreateLocation(pp.GetLocation()), ExprKind.PROPERTY_PATTERN, parent, child, false, null))
         {
             child = 0;
             foreach (var sub in pp.Subpatterns)
@@ -56,7 +56,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
             };
 
         private static Expression CreateSyntheticExp(Context cx, Microsoft.CodeAnalysis.Location location, IExpressionParentEntity parent, int child) =>
-            new Expression(new ExpressionInfo(cx, null, cx.CreateLocation(location), ExprKind.PROPERTY_PATTERN, parent, child, isCompilerGenerated: false, null));
+            new Expression(new ExpressionInfo(cx, null, cx.CreateLocation(location), ExprKind.PROPERTY_PATTERN, parent, child, false, null));
 
         private static void MakeExpressions(Context cx, IExpressionParentEntity parent, SubpatternSyntax syntax, int child)
         {

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Patterns/RecursivePattern.cs

@@ -15,7 +15,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
         /// <param name="parent">The parent pattern/expression.</param>
         /// <param name="child">The child index of this pattern.</param>
         public RecursivePattern(Context cx, RecursivePatternSyntax syntax, IExpressionParentEntity parent, int child) :
-            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), ExprKind.RECURSIVE_PATTERN, parent, child, isCompilerGenerated: false, null))
+            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), ExprKind.RECURSIVE_PATTERN, parent, child, false, null))
         {
             // Extract the type access
             if (syntax.Type is TypeSyntax t)

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Patterns/RelationalPattern.cs

@@ -8,7 +8,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
     internal class RelationalPattern : Expression
     {
         public RelationalPattern(Context cx, RelationalPatternSyntax syntax, IExpressionParentEntity parent, int child) :
-             base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), GetKind(syntax.OperatorToken), parent, child, isCompilerGenerated: false, null))
+             base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), GetKind(syntax.OperatorToken), parent, child, false, null))
         {
             Expression.Create(cx, syntax.Expression, this, 0);
         }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Patterns/SlicePattern.cs

@@ -6,7 +6,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
     internal class SlicePattern : Expression
     {
         public SlicePattern(Context cx, SlicePatternSyntax syntax, IExpressionParentEntity parent, int child) :
-            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), ExprKind.SLICE_PATTERN, parent, child, isCompilerGenerated: false, null))
+            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), ExprKind.SLICE_PATTERN, parent, child, false, null))
         {
             if (syntax.Pattern is not null)
             {

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Patterns/UnaryPattern.cs

@@ -6,7 +6,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
     internal class UnaryPattern : Expression
     {
         public UnaryPattern(Context cx, UnaryPatternSyntax syntax, IExpressionParentEntity parent, int child) :
-            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), ExprKind.NOT_PATTERN, parent, child, isCompilerGenerated: false, null))
+            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), ExprKind.NOT_PATTERN, parent, child, false, null))
         {
             Pattern.Create(cx, syntax.Pattern, this, 0);
         }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Query.cs

@@ -23,7 +23,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
             public QueryCall(Context cx, IMethodSymbol? method, SyntaxNode clause, IExpressionParentEntity parent, int child)
                 : base(new ExpressionInfo(cx, method?.GetAnnotatedReturnType(),
                         cx.CreateLocation(clause.GetLocation()),
-                        ExprKind.METHOD_INVOCATION, parent, child, isCompilerGenerated: false, null))
+                        ExprKind.METHOD_INVOCATION, parent, child, false, null))
             {
                 if (method is not null)
                     cx.TrapWriter.Writer.expr_call(this, Method.Create(cx, method));
@@ -97,7 +97,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 Expression.Create(cx, Expr, decl, 0);
 
                 var nameLoc = cx.CreateLocation(name.GetLocation());
-                var access = new Expression(new ExpressionInfo(cx, type, nameLoc, ExprKind.LOCAL_VARIABLE_ACCESS, decl, 1, isCompilerGenerated: false, null));
+                var access = new Expression(new ExpressionInfo(cx, type, nameLoc, ExprKind.LOCAL_VARIABLE_ACCESS, decl, 1, false, null));
                 cx.TrapWriter.Writer.expr_access(access, LocalVariable.Create(cx, variableSymbol));
 
                 return decl;