Java: Temporarily replace model pack for jsonwebtoken with automodel-generated one.

config/identical-files.json

@@ -251,6 +251,12 @@
     "cpp/ql/src/Security/CWE/CWE-020/SafeExternalAPIFunction.qll",
     "cpp/ql/src/Security/CWE/CWE-020/ir/SafeExternalAPIFunction.qll"
   ],
+  "XML": [
+    "cpp/ql/lib/semmle/code/cpp/XML.qll",
+    "csharp/ql/lib/semmle/code/csharp/XML.qll",
+    "java/ql/lib/semmle/code/xml/XML.qll",
+    "python/ql/lib/semmle/python/xml/XML.qll"
+  ],
   "DuplicationProblems.inc.qhelp": [
     "cpp/ql/src/Metrics/Files/DuplicationProblems.inc.qhelp",
     "javascript/ql/src/Metrics/DuplicationProblems.inc.qhelp",

cpp/autobuilder/Semmle.Autobuild.Cpp.Tests/BuildScripts.cs

@@ -203,8 +203,6 @@ namespace Semmle.Autobuild.Cpp.Tests
         public IList<DiagnosticMessage> Diagnostics { get; } = new List<DiagnosticMessage>();
 
         public void AddEntry(DiagnosticMessage message) => this.Diagnostics.Add(message);
-
-        public void Dispose() { }
     }
 
     /// <summary>
@@ -252,7 +250,12 @@ namespace Semmle.Autobuild.Cpp.Tests
             EndCallbackIn.Add(s);
         }
 
-        CppAutobuilder CreateAutoBuilder(bool isWindows, string? dotnetVersion = null, string cwd = @"C:\Project")
+        CppAutobuilder CreateAutoBuilder(bool isWindows,
+            string? buildless = null, string? solution = null, string? buildCommand = null, string? ignoreErrors = null,
+            string? msBuildArguments = null, string? msBuildPlatform = null, string? msBuildConfiguration = null, string? msBuildTarget = null,
+            string? dotnetArguments = null, string? dotnetVersion = null, string? vsToolsVersion = null,
+            string? nugetRestore = null, string? allSolutions = null,
+            string cwd = @"C:\Project")
         {
             string codeqlUpperLanguage = Language.Cpp.UpperCaseName;
             Actions.GetEnvironmentVariable[$"CODEQL_AUTOBUILDER_{codeqlUpperLanguage}_NO_INDEXING"] = "false";
@@ -262,7 +265,22 @@ namespace Semmle.Autobuild.Cpp.Tests
             Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_DIAGNOSTIC_DIR"] = "";
             Actions.GetEnvironmentVariable["CODEQL_JAVA_HOME"] = @"C:\codeql\tools\java";
             Actions.GetEnvironmentVariable["CODEQL_PLATFORM"] = "win64";
-            Actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_OPTION_DOTNET_VERSION"] = dotnetVersion;
+            Actions.GetEnvironmentVariable["SEMMLE_DIST"] = @"C:\odasa";
+            Actions.GetEnvironmentVariable["SEMMLE_JAVA_HOME"] = @"C:\odasa\tools\java";
+            Actions.GetEnvironmentVariable["SEMMLE_PLATFORM_TOOLS"] = @"C:\odasa\tools";
+            Actions.GetEnvironmentVariable["LGTM_INDEX_VSTOOLS_VERSION"] = vsToolsVersion;
+            Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_ARGUMENTS"] = msBuildArguments;
+            Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_PLATFORM"] = msBuildPlatform;
+            Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_CONFIGURATION"] = msBuildConfiguration;
+            Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_TARGET"] = msBuildTarget;
+            Actions.GetEnvironmentVariable["LGTM_INDEX_DOTNET_ARGUMENTS"] = dotnetArguments;
+            Actions.GetEnvironmentVariable["LGTM_INDEX_DOTNET_VERSION"] = dotnetVersion;
+            Actions.GetEnvironmentVariable["LGTM_INDEX_BUILD_COMMAND"] = buildCommand;
+            Actions.GetEnvironmentVariable["LGTM_INDEX_SOLUTION"] = solution;
+            Actions.GetEnvironmentVariable["LGTM_INDEX_IGNORE_ERRORS"] = ignoreErrors;
+            Actions.GetEnvironmentVariable["LGTM_INDEX_BUILDLESS"] = buildless;
+            Actions.GetEnvironmentVariable["LGTM_INDEX_ALL_SOLUTIONS"] = allSolutions;
+            Actions.GetEnvironmentVariable["LGTM_INDEX_NUGET_RESTORE"] = nugetRestore;
             Actions.GetEnvironmentVariable["ProgramFiles(x86)"] = isWindows ? @"C:\Program Files (x86)" : null;
             Actions.GetCurrentDirectory = cwd;
             Actions.IsWindows = isWindows;

cpp/autobuilder/Semmle.Autobuild.Cpp/CppAutobuilder.cs

@@ -26,6 +26,9 @@ namespace Semmle.Autobuild.Cpp
 
         public override BuildScript GetBuildScript()
         {
+            if (Options.BuildCommand != null)
+                return new BuildCommandRule((_, f) => f(null)).Analyse(this, false);
+
             return
                 // First try MSBuild
                 new MsBuildRule().Analyse(this, true) |

cpp/autobuilder/Semmle.Autobuild.Cpp/Program.cs

@@ -17,7 +17,7 @@ namespace Semmle.Autobuild.Cpp
                 try
                 {
                     Console.WriteLine("CodeQL C++ autobuilder");
-                    using var builder = new CppAutobuilder(actions, options);
+                    var builder = new CppAutobuilder(actions, options);
                     return builder.AttemptBuild();
                 }
                 catch (InvalidEnvironmentException ex)

cpp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-all
-version: 0.12.11-dev
+version: 0.12.10
 groups: cpp
 dbscheme: semmlecode.cpp.dbscheme
 extractor: cpp
@@ -11,5 +11,4 @@ dependencies:
   codeql/ssa: ${workspace}
   codeql/tutorial: ${workspace}
   codeql/util: ${workspace}
-  codeql/xml: ${workspace}
 warnOnImplicitThis: true

cpp/ql/lib/semmle/code/cpp/PrintAST.qll

@@ -862,7 +862,7 @@ private predicate namedExprChildPredicates(Expr expr, Element ele, string pred)
     or
     expr.(DeleteOrDeleteArrayExpr).getDestructorCall() = ele and pred = "getDestructorCall()"
     or
-    expr.(DeleteOrDeleteArrayExpr).getExprWithReuse() = ele and pred = "getExprWithReuse()"
+    expr.(DeleteOrDeleteArrayExpr).getExpr() = ele and pred = "getExpr()"
     or
     expr.(DestructorFieldDestruction).getExpr() = ele and pred = "getExpr()"
     or

cpp/ql/lib/semmle/code/cpp/XML.qll

@@ -3,67 +3,305 @@
  */
 
 import semmle.files.FileSystem
-private import codeql.xml.Xml
 
-private module Input implements InputSig<File, Location> {
-  class XmlLocatableBase = @xmllocatable or @xmlnamespaceable;
+private class TXmlLocatable =
+  @xmldtd or @xmlelement or @xmlattribute or @xmlnamespace or @xmlcomment or @xmlcharacters;
 
-  predicate xmllocations_(XmlLocatableBase e, Location loc) { xmllocations(e, loc) }
+/** An XML element that has a location. */
+class XmlLocatable extends @xmllocatable, TXmlLocatable {
+  /** Gets the source location for this element. */
+  Location getLocation() { xmllocations(this, result) }
 
-  class XmlParentBase = @xmlparent;
-
-  class XmlNamespaceableBase = @xmlnamespaceable;
-
-  class XmlElementBase = @xmlelement;
-
-  class XmlFileBase = File;
-
-  predicate xmlEncoding_(XmlFileBase f, string enc) { xmlEncoding(f, enc) }
-
-  class XmlDtdBase = @xmldtd;
-
-  predicate xmlDTDs_(XmlDtdBase e, string root, string publicId, string systemId, XmlFileBase file) {
-    xmlDTDs(e, root, publicId, systemId, file)
-  }
-
-  predicate xmlElements_(
-    XmlElementBase e, string name, XmlParentBase parent, int idx, XmlFileBase file
+  /**
+   * Holds if this element is at the specified location.
+   * The location spans column `startcolumn` of line `startline` to
+   * column `endcolumn` of line `endline` in file `filepath`.
+   * For more information, see
+   * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
+   */
+  predicate hasLocationInfo(
+    string filepath, int startline, int startcolumn, int endline, int endcolumn
   ) {
-    xmlElements(e, name, parent, idx, file)
+    exists(File f, Location l | l = this.getLocation() |
+      locations_default(l, f, startline, startcolumn, endline, endcolumn) and
+      filepath = f.getAbsolutePath()
+    )
   }
 
-  class XmlAttributeBase = @xmlattribute;
+  /** Gets a textual representation of this element. */
+  string toString() { none() } // overridden in subclasses
+}
 
-  predicate xmlAttrs_(
-    XmlAttributeBase e, XmlElementBase elementid, string name, string value, int idx,
-    XmlFileBase file
-  ) {
-    xmlAttrs(e, elementid, name, value, idx, file)
+/**
+ * An `XmlParent` is either an `XmlElement` or an `XmlFile`,
+ * both of which can contain other elements.
+ */
+class XmlParent extends @xmlparent {
+  XmlParent() {
+    // explicitly restrict `this` to be either an `XmlElement` or an `XmlFile`;
+    // the type `@xmlparent` currently also includes non-XML files
+    this instanceof @xmlelement or xmlEncoding(this, _)
   }
 
-  class XmlNamespaceBase = @xmlnamespace;
+  /**
+   * Gets a printable representation of this XML parent.
+   * (Intended to be overridden in subclasses.)
+   */
+  string getName() { none() } // overridden in subclasses
 
-  predicate xmlNs_(XmlNamespaceBase e, string prefixName, string uri, XmlFileBase file) {
-    xmlNs(e, prefixName, uri, file)
+  /** Gets the file to which this XML parent belongs. */
+  XmlFile getFile() { result = this or xmlElements(this, _, _, _, result) }
+
+  /** Gets the child element at a specified index of this XML parent. */
+  XmlElement getChild(int index) { xmlElements(result, _, this, index, _) }
+
+  /** Gets a child element of this XML parent. */
+  XmlElement getAChild() { xmlElements(result, _, this, _, _) }
+
+  /** Gets a child element of this XML parent with the given `name`. */
+  XmlElement getAChild(string name) { xmlElements(result, _, this, _, _) and result.hasName(name) }
+
+  /** Gets a comment that is a child of this XML parent. */
+  XmlComment getAComment() { xmlComments(result, _, this, _) }
+
+  /** Gets a character sequence that is a child of this XML parent. */
+  XmlCharacters getACharactersSet() { xmlChars(result, _, this, _, _, _) }
+
+  /** Gets the depth in the tree. (Overridden in XmlElement.) */
+  int getDepth() { result = 0 }
+
+  /** Gets the number of child XML elements of this XML parent. */
+  int getNumberOfChildren() { result = count(XmlElement e | xmlElements(e, _, this, _, _)) }
+
+  /** Gets the number of places in the body of this XML parent where text occurs. */
+  int getNumberOfCharacterSets() { result = count(int pos | xmlChars(_, _, this, pos, _, _)) }
+
+  /**
+   * Gets the result of appending all the character sequences of this XML parent from
+   * left to right, separated by a space.
+   */
+  string allCharactersString() {
+    result =
+      concat(string chars, int pos | xmlChars(_, chars, this, pos, _, _) | chars, " " order by pos)
   }
 
-  predicate xmlHasNs_(XmlNamespaceableBase e, XmlNamespaceBase ns, XmlFileBase file) {
-    xmlHasNs(e, ns, file)
-  }
+  /** Gets the text value contained in this XML parent. */
+  string getTextValue() { result = this.allCharactersString() }
 
-  class XmlCommentBase = @xmlcomment;
+  /** Gets a printable representation of this XML parent. */
+  string toString() { result = this.getName() }
+}
 
-  predicate xmlComments_(XmlCommentBase e, string text, XmlParentBase parent, XmlFileBase file) {
-    xmlComments(e, text, parent, file)
-  }
+/** An XML file. */
+class XmlFile extends XmlParent, File {
+  XmlFile() { xmlEncoding(this, _) }
 
-  class XmlCharactersBase = @xmlcharacters;
+  /** Gets a printable representation of this XML file. */
+  override string toString() { result = this.getName() }
 
-  predicate xmlChars_(
-    XmlCharactersBase e, string text, XmlParentBase parent, int idx, int isCDATA, XmlFileBase file
-  ) {
-    xmlChars(e, text, parent, idx, isCDATA, file)
+  /** Gets the name of this XML file. */
+  override string getName() { result = File.super.getAbsolutePath() }
+
+  /** Gets the encoding of this XML file. */
+  string getEncoding() { xmlEncoding(this, result) }
+
+  /** Gets the XML file itself. */
+  override XmlFile getFile() { result = this }
+
+  /** Gets a top-most element in an XML file. */
+  XmlElement getARootElement() { result = this.getAChild() }
+
+  /** Gets a DTD associated with this XML file. */
+  XmlDtd getADtd() { xmlDTDs(result, _, _, _, this) }
+}
+
+/**
+ * An XML document type definition (DTD).
+ *
+ * Example:
+ *
+ * ```
+ * <!ELEMENT person (firstName, lastName?)>
+ * <!ELEMENT firstName (#PCDATA)>
+ * <!ELEMENT lastName (#PCDATA)>
+ * ```
+ */
+class XmlDtd extends XmlLocatable, @xmldtd {
+  /** Gets the name of the root element of this DTD. */
+  string getRoot() { xmlDTDs(this, result, _, _, _) }
+
+  /** Gets the public ID of this DTD. */
+  string getPublicId() { xmlDTDs(this, _, result, _, _) }
+
+  /** Gets the system ID of this DTD. */
+  string getSystemId() { xmlDTDs(this, _, _, result, _) }
+
+  /** Holds if this DTD is public. */
+  predicate isPublic() { not xmlDTDs(this, _, "", _, _) }
+
+  /** Gets the parent of this DTD. */
+  XmlParent getParent() { xmlDTDs(this, _, _, _, result) }
+
+  override string toString() {
+    this.isPublic() and
+    result = this.getRoot() + " PUBLIC '" + this.getPublicId() + "' '" + this.getSystemId() + "'"
+    or
+    not this.isPublic() and
+    result = this.getRoot() + " SYSTEM '" + this.getSystemId() + "'"
   }
 }
 
-import Make<File, Location, Input>
+/**
+ * An XML element in an XML file.
+ *
+ * Example:
+ *
+ * ```
+ * <manifest xmlns:android="http://schemas.android.com/apk/res/android"
+ *           package="com.example.exampleapp" android:versionCode="1">
+ * </manifest>
+ * ```
+ */
+class XmlElement extends @xmlelement, XmlParent, XmlLocatable {
+  /** Holds if this XML element has the given `name`. */
+  predicate hasName(string name) { name = this.getName() }
+
+  /** Gets the name of this XML element. */
+  override string getName() { xmlElements(this, result, _, _, _) }
+
+  /** Gets the XML file in which this XML element occurs. */
+  override XmlFile getFile() { xmlElements(this, _, _, _, result) }
+
+  /** Gets the parent of this XML element. */
+  XmlParent getParent() { xmlElements(this, _, result, _, _) }
+
+  /** Gets the index of this XML element among its parent's children. */
+  int getIndex() { xmlElements(this, _, _, result, _) }
+
+  /** Holds if this XML element has a namespace. */
+  predicate hasNamespace() { xmlHasNs(this, _, _) }
+
+  /** Gets the namespace of this XML element, if any. */
+  XmlNamespace getNamespace() { xmlHasNs(this, result, _) }
+
+  /** Gets the index of this XML element among its parent's children. */
+  int getElementPositionIndex() { xmlElements(this, _, _, result, _) }
+
+  /** Gets the depth of this element within the XML file tree structure. */
+  override int getDepth() { result = this.getParent().getDepth() + 1 }
+
+  /** Gets an XML attribute of this XML element. */
+  XmlAttribute getAnAttribute() { result.getElement() = this }
+
+  /** Gets the attribute with the specified `name`, if any. */
+  XmlAttribute getAttribute(string name) { result.getElement() = this and result.getName() = name }
+
+  /** Holds if this XML element has an attribute with the specified `name`. */
+  predicate hasAttribute(string name) { exists(this.getAttribute(name)) }
+
+  /** Gets the value of the attribute with the specified `name`, if any. */
+  string getAttributeValue(string name) { result = this.getAttribute(name).getValue() }
+
+  /** Gets a printable representation of this XML element. */
+  override string toString() { result = this.getName() }
+}
+
+/**
+ * An attribute that occurs inside an XML element.
+ *
+ * Examples:
+ *
+ * ```
+ * package="com.example.exampleapp"
+ * android:versionCode="1"
+ * ```
+ */
+class XmlAttribute extends @xmlattribute, XmlLocatable {
+  /** Gets the name of this attribute. */
+  string getName() { xmlAttrs(this, _, result, _, _, _) }
+
+  /** Gets the XML element to which this attribute belongs. */
+  XmlElement getElement() { xmlAttrs(this, result, _, _, _, _) }
+
+  /** Holds if this attribute has a namespace. */
+  predicate hasNamespace() { xmlHasNs(this, _, _) }
+
+  /** Gets the namespace of this attribute, if any. */
+  XmlNamespace getNamespace() { xmlHasNs(this, result, _) }
+
+  /** Gets the value of this attribute. */
+  string getValue() { xmlAttrs(this, _, _, result, _, _) }
+
+  /** Gets a printable representation of this XML attribute. */
+  override string toString() { result = this.getName() + "=" + this.getValue() }
+}
+
+/**
+ * A namespace used in an XML file.
+ *
+ * Example:
+ *
+ * ```
+ * xmlns:android="http://schemas.android.com/apk/res/android"
+ * ```
+ */
+class XmlNamespace extends XmlLocatable, @xmlnamespace {
+  /** Gets the prefix of this namespace. */
+  string getPrefix() { xmlNs(this, result, _, _) }
+
+  /** Gets the URI of this namespace. */
+  string getUri() { xmlNs(this, _, result, _) }
+
+  /** Holds if this namespace has no prefix. */
+  predicate isDefault() { this.getPrefix() = "" }
+
+  override string toString() {
+    this.isDefault() and result = this.getUri()
+    or
+    not this.isDefault() and result = this.getPrefix() + ":" + this.getUri()
+  }
+}
+
+/**
+ * A comment in an XML file.
+ *
+ * Example:
+ *
+ * ```
+ * <!-- This is a comment. -->
+ * ```
+ */
+class XmlComment extends @xmlcomment, XmlLocatable {
+  /** Gets the text content of this XML comment. */
+  string getText() { xmlComments(this, result, _, _) }
+
+  /** Gets the parent of this XML comment. */
+  XmlParent getParent() { xmlComments(this, _, result, _) }
+
+  /** Gets a printable representation of this XML comment. */
+  override string toString() { result = this.getText() }
+}
+
+/**
+ * A sequence of characters that occurs between opening and
+ * closing tags of an XML element, excluding other elements.
+ *
+ * Example:
+ *
+ * ```
+ * <content>This is a sequence of characters.</content>
+ * ```
+ */
+class XmlCharacters extends @xmlcharacters, XmlLocatable {
+  /** Gets the content of this character sequence. */
+  string getCharacters() { xmlChars(this, result, _, _, _, _) }
+
+  /** Gets the parent of this character sequence. */
+  XmlParent getParent() { xmlChars(this, _, result, _, _, _) }
+
+  /** Holds if this character sequence is CDATA. */
+  predicate isCDATA() { xmlChars(this, _, _, _, 1, _) }
+
+  /** Gets a printable representation of this XML character sequence. */
+  override string toString() { result = this.getCharacters() }
+}

cpp/ql/lib/semmle/code/cpp/exprs/Expr.qll

@@ -1015,33 +1015,8 @@ class DeleteOrDeleteArrayExpr extends Expr, TDeleteOrDeleteArrayExpr {
   Expr getExpr() {
     // If there is a destructor call, the object being deleted is the qualifier
     // otherwise it is the third child.
-    exists(Expr exprWithReuse | exprWithReuse = this.getExprWithReuse() |
-      if not exprWithReuse instanceof ReuseExpr
-      then result = exprWithReuse
-      else result = this.getDestructorCall().getQualifier()
-    )
+    result = this.getChild(3) or result = this.getDestructorCall().getQualifier()
   }
-
-  /**
-   * Gets the object or array being deleted, and gets a `ReuseExpr` when there
-   * is a destructor call and the object is also the qualifier of the call.
-   *
-   * For example, given:
-   * ```
-   * struct HasDestructor { ~HasDestructor(); };
-   * struct PlainOldData { int x, char y; };
-   *
-   * void f(HasDestructor* hasDestructor, PlainOldData* pod) {
-   *   delete hasDestructor;
-   *   delete pod;
-   * }
-   * ```
-   * This predicate yields a `ReuseExpr` for `delete hasDestructor`, as the
-   * the deleted expression has a destructor, and that expression is also
-   * the qualifier of the destructor call. In the case of `delete pod` the
-   * predicate does not yield a `ReuseExpr`, as there is no destructor call.
-   */
-  Expr getExprWithReuse() { result = this.getChild(3) }
 }
 
 /**
@@ -1365,17 +1340,7 @@ class ReuseExpr extends Expr, @reuseexpr {
   /**
    * Gets the expression that is being re-used.
    */
-  Expr getReusedExpr() {
-    // In the case of a prvalue, the extractor outputs the expression
-    // before conversion, but the converted expression is intended.
-    if this.isPRValueCategory()
-    then result = this.getBaseReusedExpr().getFullyConverted()
-    else result = this.getBaseReusedExpr()
-  }
-
-  private Expr getBaseReusedExpr() {
-    expr_reuse(underlyingElement(this), unresolveElement(result), _)
-  }
+  Expr getReusedExpr() { expr_reuse(underlyingElement(this), unresolveElement(result), _) }
 
   override Type getType() { result = this.getReusedExpr().getType() }
 

cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/TranslatedElement.qll

@@ -150,6 +150,11 @@ private predicate ignoreExprOnly(Expr expr) {
   or
   not translateFunction(getEnclosingFunction(expr)) and
   not Raw::varHasIRFunc(getEnclosingVariable(expr))
+  or
+  exists(DeleteOrDeleteArrayExpr deleteExpr |
+    // Ignore the destructor call, because the duplicated qualifier breaks control flow.
+    deleteExpr.getDestructorCall() = expr
+  )
 }
 
 /**

cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/TranslatedExpr.qll

@@ -2245,11 +2245,7 @@ class TranslatedDeleteOrDeleteArrayExpr extends TranslatedNonConstantExpr, Trans
 
   final override Type getCallResultType() { result = expr.getType() }
 
-  final override TranslatedExpr getQualifier() {
-    result = getTranslatedExpr(expr.getDestructorCall())
-  }
-
-  final override Instruction getQualifierResult() { none() }
+  final override TranslatedExpr getQualifier() { none() }
 
   final override predicate hasArguments() {
     // All deallocator calls have at least one argument.
@@ -2264,7 +2260,7 @@ class TranslatedDeleteOrDeleteArrayExpr extends TranslatedNonConstantExpr, Trans
   final override TranslatedExpr getArgument(int index) {
     // The only argument we define is the pointer to be deallocated.
     index = 0 and
-    result = getTranslatedExpr(expr.getExprWithReuse().getFullyConverted())
+    result = getTranslatedExpr(expr.getExpr().getFullyConverted())
   }
 
   final override predicate mayThrowException() {

cpp/ql/lib/semmle/code/cpp/models/Models.qll

@@ -41,4 +41,3 @@ private import implementations.SqLite3
 private import implementations.PostgreSql
 private import implementations.System
 private import implementations.StructuredExceptionHandling
-private import implementations.Fopen

cpp/ql/lib/semmle/code/cpp/models/implementations/Fopen.qll

@@ -1,50 +0,0 @@
-/**
- * Provides implementation classes modeling `fopen` and various similar
- * functions. See `semmle.code.cpp.models.Models` for usage information.
- */
-
-import semmle.code.cpp.models.interfaces.Alias
-import semmle.code.cpp.models.interfaces.SideEffect
-
-/** The function `fopen` and friends. */
-private class Fopen extends Function, AliasFunction, SideEffectFunction {
-  Fopen() {
-    this.hasGlobalOrStdName(["fopen", "fopen_s", "freopen"])
-    or
-    this.hasGlobalName(["_open", "_wfopen", "_fsopen", "_wfsopen", "_wopen"])
-  }
-
-  override predicate hasOnlySpecificWriteSideEffects() { any() }
-
-  override predicate hasOnlySpecificReadSideEffects() { any() }
-
-  override predicate parameterEscapesOnlyViaReturn(int i) { none() }
-
-  override predicate parameterNeverEscapes(int index) {
-    // None of the parameters escape
-    this.getParameter(index).getUnspecifiedType() instanceof PointerType
-  }
-
-  override predicate hasSpecificReadSideEffect(ParameterIndex i, boolean buffer) {
-    (
-      this.hasGlobalOrStdName(["fopen", "fopen_s"])
-      or
-      this.hasGlobalName(["_wfopen", "_fsopen", "_wfsopen"])
-    ) and
-    i = [0, 1] and
-    buffer = true
-    or
-    this.hasGlobalOrStdName("freopen") and
-    (
-      i = [0, 1] and
-      buffer = true
-      or
-      i = 2 and
-      buffer = false
-    )
-    or
-    this.hasGlobalName(["_open", "_wopen"]) and
-    i = 0 and
-    buffer = true
-  }
-}

cpp/ql/src/Critical/DoubleFree.ql

@@ -37,5 +37,6 @@ where
   DoubleFree::flowPath(source, sink) and
   isFree(source.getNode(), _, _, dealloc) and
   isFree(sink.getNode(), e2)
-select sink.getNode(), source, sink, "Memory pointed to by $@ may already have been freed by $@.",
-  e2, e2.toString(), dealloc, dealloc.toString()
+select sink.getNode(), source, sink,
+  "Memory pointed to by '" + e2.toString() + "' may already have been freed by $@.", dealloc,
+  dealloc.toString()

cpp/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-queries
-version: 0.9.10-dev
+version: 0.9.9
 groups:
   - cpp
   - queries

cpp/ql/test/examples/expressions/PrintAST.expected

@@ -426,14 +426,11 @@ DestructorCall.cpp:
 #   12|           getQualifier(): [VariableAccess] c
 #   12|               Type = [PointerType] C *
 #   12|               ValueCategory = prvalue(load)
-#   12|         getExprWithReuse(): [ReuseExpr] reuse of c
-#   12|             Type = [PointerType] C *
-#   12|             ValueCategory = prvalue
 #   13|     getStmt(1): [ExprStmt] ExprStmt
 #   13|       getExpr(): [DeleteExpr] delete
 #   13|           Type = [VoidType] void
 #   13|           ValueCategory = prvalue
-#   13|         getExprWithReuse(): [VariableAccess] d
+#   13|         getExpr(): [VariableAccess] d
 #   13|             Type = [PointerType] D *
 #   13|             ValueCategory = prvalue(load)
 #   14|     getStmt(2): [ReturnStmt] return ...

cpp/ql/test/library-tests/compiler_generated/compilerGenerated.expected

@@ -14,7 +14,6 @@
 | cpp.cpp:10:7:10:7 | operator= | Function |
 | cpp.cpp:10:7:10:7 | ~MyClass | Function |
 | cpp.cpp:15:5:15:12 | call to ~MyClass | Expr |
-| cpp.cpp:15:12:15:12 | reuse of m | Expr |
 | cpp.cpp:16:1:16:1 | return ... | Stmt |
 | file://:0:0:0:0 | operator delete | Function |
 | file://:0:0:0:0 | operator new | Function |

cpp/ql/test/library-tests/ir/ir/ir.cpp

@@ -1933,20 +1933,6 @@ namespace missing_declaration_entries {
         Bar2<int> b;
         b.two_missing_variable_declaration_entries();
     }
-
-    template<typename T> struct Bar3 {
-
-        int two_more_missing_variable_declaration_entries() {
-            extern int g;
-            int z(float);
-            return g;
-        }
-    };
-
-    void test3() {
-        Bar3<int> b;
-        b.two_more_missing_variable_declaration_entries();
-    }
 }
 
 template<typename T> T global_template = 42;
@@ -2431,20 +2417,4 @@ void initialization_with_temp_destructor() {
         y += x;
 }
 
-void param_with_destructor_by_value(ClassWithDestructor c) {
-    // The call to ~ClassWithDestructor::ClassWithDestructor() seems to be missing here.
-}
-
-void param_with_destructor_by_pointer(ClassWithDestructor* c) {
-    // No destructor call should be here
-}
-
-void param_with_destructor_by_ref(ClassWithDestructor& c) {
-    // No destructor call should be here
-}
-
-void param_with_destructor_by_rref(ClassWithDestructor&& c) {
-    // No destructor call should be here
-}
-
 // semmle-extractor-options: -std=c++20 --clang

cpp/ql/test/query-tests/Critical/MemoryFreed/DoubleFree.expected

@@ -41,16 +41,16 @@ nodes
 | test_free.cpp:302:12:302:14 | buf | semmle.label | buf |
 subpaths
 #select
-| test_free.cpp:14:10:14:10 | a | test_free.cpp:11:10:11:10 | pointer to free output argument | test_free.cpp:14:10:14:10 | a | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:14:10:14:10 | a | a | test_free.cpp:11:5:11:8 | call to free | call to free |
-| test_free.cpp:31:27:31:27 | a | test_free.cpp:30:10:30:10 | pointer to free output argument | test_free.cpp:31:27:31:27 | a | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:31:27:31:27 | a | a | test_free.cpp:30:5:30:8 | call to free | call to free |
-| test_free.cpp:37:27:37:27 | a | test_free.cpp:35:10:35:10 | pointer to free output argument | test_free.cpp:37:27:37:27 | a | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:37:27:37:27 | a | a | test_free.cpp:35:5:35:8 | call to free | call to free |
-| test_free.cpp:46:10:46:10 | a | test_free.cpp:42:27:42:27 | pointer to free output argument | test_free.cpp:46:10:46:10 | a | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:46:10:46:10 | a | a | test_free.cpp:42:22:42:25 | call to free | call to free |
-| test_free.cpp:46:10:46:10 | a | test_free.cpp:44:27:44:27 | pointer to free output argument | test_free.cpp:46:10:46:10 | a | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:46:10:46:10 | a | a | test_free.cpp:44:22:44:25 | call to free | call to free |
-| test_free.cpp:51:10:51:10 | a | test_free.cpp:50:27:50:27 | pointer to free output argument | test_free.cpp:51:10:51:10 | a | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:51:10:51:10 | a | a | test_free.cpp:50:22:50:25 | call to free | call to free |
-| test_free.cpp:72:14:72:14 | a | test_free.cpp:69:10:69:10 | pointer to free output argument | test_free.cpp:72:14:72:14 | a | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:72:14:72:14 | a | a | test_free.cpp:69:5:69:8 | call to free | call to free |
-| test_free.cpp:85:12:85:12 | a | test_free.cpp:83:12:83:12 | pointer to operator delete output argument | test_free.cpp:85:12:85:12 | a | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:85:12:85:12 | a | a | test_free.cpp:83:5:83:13 | delete | delete |
-| test_free.cpp:103:10:103:10 | a | test_free.cpp:101:10:101:10 | pointer to free output argument | test_free.cpp:103:10:103:10 | a | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:103:10:103:10 | a | a | test_free.cpp:101:5:101:8 | call to free | call to free |
-| test_free.cpp:129:10:129:11 | * ... | test_free.cpp:128:10:128:11 | pointer to free output argument | test_free.cpp:129:10:129:11 | * ... | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:129:10:129:11 | * ... | * ... | test_free.cpp:128:5:128:8 | call to free | call to free |
-| test_free.cpp:154:10:154:10 | a | test_free.cpp:152:27:152:27 | pointer to free output argument | test_free.cpp:154:10:154:10 | a | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:154:10:154:10 | a | a | test_free.cpp:152:22:152:25 | call to free | call to free |
-| test_free.cpp:209:10:209:10 | a | test_free.cpp:207:10:207:10 | pointer to free output argument | test_free.cpp:209:10:209:10 | a | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:209:10:209:10 | a | a | test_free.cpp:207:5:207:8 | call to free | call to free |
-| test_free.cpp:302:12:302:14 | buf | test_free.cpp:301:12:301:14 | pointer to g_free output argument | test_free.cpp:302:12:302:14 | buf | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:302:12:302:14 | buf | buf | test_free.cpp:301:5:301:10 | call to g_free | call to g_free |
+| test_free.cpp:14:10:14:10 | a | test_free.cpp:11:10:11:10 | pointer to free output argument | test_free.cpp:14:10:14:10 | a | Memory pointed to by 'a' may already have been freed by $@. | test_free.cpp:11:5:11:8 | call to free | call to free |
+| test_free.cpp:31:27:31:27 | a | test_free.cpp:30:10:30:10 | pointer to free output argument | test_free.cpp:31:27:31:27 | a | Memory pointed to by 'a' may already have been freed by $@. | test_free.cpp:30:5:30:8 | call to free | call to free |
+| test_free.cpp:37:27:37:27 | a | test_free.cpp:35:10:35:10 | pointer to free output argument | test_free.cpp:37:27:37:27 | a | Memory pointed to by 'a' may already have been freed by $@. | test_free.cpp:35:5:35:8 | call to free | call to free |
+| test_free.cpp:46:10:46:10 | a | test_free.cpp:42:27:42:27 | pointer to free output argument | test_free.cpp:46:10:46:10 | a | Memory pointed to by 'a' may already have been freed by $@. | test_free.cpp:42:22:42:25 | call to free | call to free |
+| test_free.cpp:46:10:46:10 | a | test_free.cpp:44:27:44:27 | pointer to free output argument | test_free.cpp:46:10:46:10 | a | Memory pointed to by 'a' may already have been freed by $@. | test_free.cpp:44:22:44:25 | call to free | call to free |
+| test_free.cpp:51:10:51:10 | a | test_free.cpp:50:27:50:27 | pointer to free output argument | test_free.cpp:51:10:51:10 | a | Memory pointed to by 'a' may already have been freed by $@. | test_free.cpp:50:22:50:25 | call to free | call to free |
+| test_free.cpp:72:14:72:14 | a | test_free.cpp:69:10:69:10 | pointer to free output argument | test_free.cpp:72:14:72:14 | a | Memory pointed to by 'a' may already have been freed by $@. | test_free.cpp:69:5:69:8 | call to free | call to free |
+| test_free.cpp:85:12:85:12 | a | test_free.cpp:83:12:83:12 | pointer to operator delete output argument | test_free.cpp:85:12:85:12 | a | Memory pointed to by 'a' may already have been freed by $@. | test_free.cpp:83:5:83:13 | delete | delete |
+| test_free.cpp:103:10:103:10 | a | test_free.cpp:101:10:101:10 | pointer to free output argument | test_free.cpp:103:10:103:10 | a | Memory pointed to by 'a' may already have been freed by $@. | test_free.cpp:101:5:101:8 | call to free | call to free |
+| test_free.cpp:129:10:129:11 | * ... | test_free.cpp:128:10:128:11 | pointer to free output argument | test_free.cpp:129:10:129:11 | * ... | Memory pointed to by '* ...' may already have been freed by $@. | test_free.cpp:128:5:128:8 | call to free | call to free |
+| test_free.cpp:154:10:154:10 | a | test_free.cpp:152:27:152:27 | pointer to free output argument | test_free.cpp:154:10:154:10 | a | Memory pointed to by 'a' may already have been freed by $@. | test_free.cpp:152:22:152:25 | call to free | call to free |
+| test_free.cpp:209:10:209:10 | a | test_free.cpp:207:10:207:10 | pointer to free output argument | test_free.cpp:209:10:209:10 | a | Memory pointed to by 'a' may already have been freed by $@. | test_free.cpp:207:5:207:8 | call to free | call to free |
+| test_free.cpp:302:12:302:14 | buf | test_free.cpp:301:12:301:14 | pointer to g_free output argument | test_free.cpp:302:12:302:14 | buf | Memory pointed to by 'buf' may already have been freed by $@. | test_free.cpp:301:5:301:10 | call to g_free | call to g_free |

csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs

@@ -218,8 +218,6 @@ namespace Semmle.Autobuild.CSharp.Tests
         public IList<DiagnosticMessage> Diagnostics { get; } = new List<DiagnosticMessage>();
 
         public void AddEntry(DiagnosticMessage message) => this.Diagnostics.Add(message);
-
-        public void Dispose() { }
     }
 
     /// <summary>
@@ -401,8 +399,10 @@ namespace Semmle.Autobuild.CSharp.Tests
         }
 
         private CSharpAutobuilder CreateAutoBuilder(bool isWindows,
-            string? buildless = null,
-            string? dotnetVersion = null,
+            string? buildless = null, string? solution = null, string? buildCommand = null, string? ignoreErrors = null,
+            string? msBuildArguments = null, string? msBuildPlatform = null, string? msBuildConfiguration = null, string? msBuildTarget = null,
+            string? dotnetArguments = null, string? dotnetVersion = null, string? vsToolsVersion = null,
+            string? nugetRestore = null, string? allSolutions = null,
             string cwd = @"C:\Project")
         {
             var codeqlUpperLanguage = Language.CSharp.UpperCaseName;
@@ -412,9 +412,20 @@ namespace Semmle.Autobuild.CSharp.Tests
             actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_DIAGNOSTIC_DIR"] = "";
             actions.GetEnvironmentVariable["CODEQL_JAVA_HOME"] = @"C:\codeql\tools\java";
             actions.GetEnvironmentVariable["CODEQL_PLATFORM"] = isWindows ? "win64" : "linux64";
+            actions.GetEnvironmentVariable["LGTM_INDEX_VSTOOLS_VERSION"] = vsToolsVersion;
+            actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_ARGUMENTS"] = msBuildArguments;
+            actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_PLATFORM"] = msBuildPlatform;
+            actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_CONFIGURATION"] = msBuildConfiguration;
+            actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_TARGET"] = msBuildTarget;
+            actions.GetEnvironmentVariable["LGTM_INDEX_DOTNET_ARGUMENTS"] = dotnetArguments;
+            actions.GetEnvironmentVariable["LGTM_INDEX_DOTNET_VERSION"] = dotnetVersion;
+            actions.GetEnvironmentVariable["LGTM_INDEX_BUILD_COMMAND"] = buildCommand;
+            actions.GetEnvironmentVariable["LGTM_INDEX_SOLUTION"] = solution;
+            actions.GetEnvironmentVariable["LGTM_INDEX_IGNORE_ERRORS"] = ignoreErrors;
+            actions.GetEnvironmentVariable["LGTM_INDEX_BUILDLESS"] = buildless;
             actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_OPTION_BUILDLESS"] = buildless;
-            if (dotnetVersion is not null)
-                actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_OPTION_DOTNET_VERSION"] = dotnetVersion;
+            actions.GetEnvironmentVariable["LGTM_INDEX_ALL_SOLUTIONS"] = allSolutions;
+            actions.GetEnvironmentVariable["LGTM_INDEX_NUGET_RESTORE"] = nugetRestore;
             actions.GetEnvironmentVariable["ProgramFiles(x86)"] = isWindows ? @"C:\Program Files (x86)" : null;
             actions.GetCurrentDirectory = cwd;
             actions.IsWindows = isWindows;
@@ -589,6 +600,15 @@ namespace Semmle.Autobuild.CSharp.Tests
             TestAutobuilderScript(autobuilder, 0, 1);
         }
 
+        private void SkipVsWhere()
+        {
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = false;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"] = false;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat"] = false;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\vcvarsall.bat"] = false;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat"] = false;
+        }
+
         private void TestAutobuilderScript(CSharpAutobuilder autobuilder, int expectedOutput, int commandsRun)
         {
             Assert.Equal(expectedOutput, autobuilder.GetBuildScript().Run(actions, StartCallback, EndCallback));
@@ -608,6 +628,23 @@ namespace Semmle.Autobuild.CSharp.Tests
             }
         }
 
+        [Fact]
+        public void TestLinuxBuildCommand()
+        {
+            actions.RunProcess["./build.sh --skip-tests"] = 0;
+            actions.FileExists["csharp.log"] = true;
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
+            actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln";
+            actions.EnumerateDirectories[@"C:\Project"] = "";
+
+            SkipVsWhere();
+
+            var autobuilder = CreateAutoBuilder(false, buildCommand: "./build.sh --skip-tests");
+            TestAutobuilderScript(autobuilder, 0, 1);
+        }
+
         [Fact]
         public void TestLinuxBuildSh()
         {
@@ -677,6 +714,177 @@ namespace Semmle.Autobuild.CSharp.Tests
             TestAutobuilderScript(autobuilder, 0, 1);
         }
 
+        [Fact]
+        public void TestWindowsBuildBatIgnoreErrors()
+        {
+            actions.EnumerateFiles[@"C:\Project"] = "foo.cs\nbuild.bat";
+            actions.EnumerateDirectories[@"C:\Project"] = "";
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
+            actions.RunProcess[@"cmd.exe /C C:\Project\build.bat"] = 1;
+            actions.RunProcessWorkingDirectory[@"cmd.exe /C C:\Project\build.bat"] = @"C:\Project";
+            actions.RunProcess[@"cmd.exe /C C:\codeql\tools\java\bin\java -jar C:\codeql\csharp\tools\extractor-asp.jar ."] = 0;
+            actions.RunProcess[@"cmd.exe /C C:\codeql\tools\codeql index --xml --extensions config"] = 0;
+            actions.FileExists["csharp.log"] = true;
+
+            var autobuilder = CreateAutoBuilder(true, ignoreErrors: "true");
+            TestAutobuilderScript(autobuilder, 1, 1);
+        }
+
+        [Fact]
+        public void TestWindowsCmdIgnoreErrors()
+        {
+            actions.RunProcess["cmd.exe /C ^\"build.cmd^ --skip-tests^\""] = 3;
+            actions.RunProcess[@"cmd.exe /C C:\codeql\tools\java\bin\java -jar C:\codeql\csharp\tools\extractor-asp.jar ."] = 0;
+            actions.RunProcess[@"cmd.exe /C C:\codeql\tools\codeql index --xml --extensions config"] = 0;
+            actions.FileExists["csharp.log"] = true;
+            SkipVsWhere();
+
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
+            actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln";
+            actions.EnumerateDirectories[@"C:\Project"] = "";
+
+            var autobuilder = CreateAutoBuilder(true, buildCommand: "build.cmd --skip-tests", ignoreErrors: "true");
+            TestAutobuilderScript(autobuilder, 3, 1);
+        }
+
+        [Fact]
+        public void TestWindowCSharpMsBuild()
+        {
+            actions.RunProcess[@"cmd.exe /C C:\Project\.nuget\nuget.exe restore C:\Project\test1.sln -DisableParallelProcessing"] = 0;
+            actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program^ Files^ ^(x86^)\\Microsoft^ Visual^ Studio^ 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test1.sln /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /P:Fu=Bar"] = 0;
+            actions.RunProcess[@"cmd.exe /C C:\Project\.nuget\nuget.exe restore C:\Project\test2.sln -DisableParallelProcessing"] = 0;
+            actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program^ Files^ ^(x86^)\\Microsoft^ Visual^ Studio^ 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test2.sln /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /P:Fu=Bar"] = 0;
+            actions.FileExists["csharp.log"] = true;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = false;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"] = false;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat"] = true;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\vcvarsall.bat"] = false;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat"] = true;
+
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
+            actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest1.cs\ntest2.cs";
+            actions.EnumerateFiles[@"C:\Project\.nuget"] = "nuget.exe";
+            actions.EnumerateDirectories[@"C:\Project"] = @".nuget";
+            actions.EnumerateDirectories[@"C:\Project\.nuget"] = "";
+
+            var autobuilder = CreateAutoBuilder(true, msBuildArguments: "/P:Fu=Bar", msBuildTarget: "Windows", msBuildPlatform: "x86", msBuildConfiguration: "Debug",
+                vsToolsVersion: "12", allSolutions: "true");
+            var testSolution1 = new TestSolution(@"C:\Project\test1.sln");
+            var testSolution2 = new TestSolution(@"C:\Project\test2.sln");
+            autobuilder.ProjectsOrSolutionsToBuild.Add(testSolution1);
+            autobuilder.ProjectsOrSolutionsToBuild.Add(testSolution2);
+
+            TestAutobuilderScript(autobuilder, 0, 4);
+        }
+
+        [Fact]
+        public void TestWindowCSharpMsBuildMultipleSolutions()
+        {
+            actions.RunProcess[@"cmd.exe /C nuget restore C:\Project\test1.csproj -DisableParallelProcessing"] = 0;
+            actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program^ Files^ ^(x86^)\\Microsoft^ Visual^ Studio^ 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test1.csproj /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /P:Fu=Bar"] = 0;
+            actions.RunProcess[@"cmd.exe /C nuget restore C:\Project\test2.csproj -DisableParallelProcessing"] = 0;
+            actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program^ Files^ ^(x86^)\\Microsoft^ Visual^ Studio^ 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test2.csproj /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /P:Fu=Bar"] = 0;
+            actions.FileExists["csharp.log"] = true;
+            actions.FileExists[@"C:\Project\test1.csproj"] = true;
+            actions.FileExists[@"C:\Project\test2.csproj"] = true;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = false;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"] = false;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat"] = true;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\vcvarsall.bat"] = false;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat"] = true;
+
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
+            actions.EnumerateFiles[@"C:\Project"] = "test1.csproj\ntest2.csproj\ntest1.cs\ntest2.cs";
+            actions.EnumerateDirectories[@"C:\Project"] = "";
+
+            var csproj1 = new XmlDocument();
+            csproj1.LoadXml(@"<?xml version=""1.0"" encoding=""utf - 8""?>
+  <Project ToolsVersion=""15.0"" xmlns=""http://schemas.microsoft.com/developer/msbuild/2003"">
+    <ItemGroup>
+      <Compile Include=""test1.cs"" />
+    </ItemGroup>
+  </Project>");
+            actions.LoadXml[@"C:\Project\test1.csproj"] = csproj1;
+
+            var csproj2 = new XmlDocument();
+            csproj2.LoadXml(@"<?xml version=""1.0"" encoding=""utf - 8""?>
+  <Project ToolsVersion=""15.0"" xmlns=""http://schemas.microsoft.com/developer/msbuild/2003"">
+    <ItemGroup>
+      <Compile Include=""test1.cs"" />
+    </ItemGroup>
+  </Project>");
+            actions.LoadXml[@"C:\Project\test2.csproj"] = csproj2;
+
+            var autobuilder = CreateAutoBuilder(true, msBuildArguments: "/P:Fu=Bar", msBuildTarget: "Windows", msBuildPlatform: "x86", msBuildConfiguration: "Debug",
+                vsToolsVersion: "12");
+
+            TestAutobuilderScript(autobuilder, 0, 4);
+        }
+
+        [Fact]
+        public void TestWindowCSharpMsBuildFailed()
+        {
+            actions.RunProcess[@"cmd.exe /C nuget restore C:\Project\test1.sln -DisableParallelProcessing"] = 0;
+            actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program^ Files^ ^(x86^)\\Microsoft^ Visual^ Studio^ 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test1.sln /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /P:Fu=Bar"] = 1;
+            actions.FileExists["csharp.log"] = true;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = false;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"] = false;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat"] = true;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\vcvarsall.bat"] = false;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat"] = true;
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
+            actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest1.cs\ntest2.cs";
+            actions.EnumerateDirectories[@"C:\Project"] = "";
+
+            var autobuilder = CreateAutoBuilder(true, msBuildArguments: "/P:Fu=Bar", msBuildTarget: "Windows", msBuildPlatform: "x86", msBuildConfiguration: "Debug",
+                vsToolsVersion: "12", allSolutions: "true");
+            var testSolution1 = new TestSolution(@"C:\Project\test1.sln");
+            var testSolution2 = new TestSolution(@"C:\Project\test2.sln");
+            autobuilder.ProjectsOrSolutionsToBuild.Add(testSolution1);
+            autobuilder.ProjectsOrSolutionsToBuild.Add(testSolution2);
+
+            TestAutobuilderScript(autobuilder, 1, 2);
+        }
+
+
+        [Fact]
+        public void TestSkipNugetMsBuild()
+        {
+            actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program^ Files^ ^(x86^)\\Microsoft^ Visual^ Studio^ 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test1.sln /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /P:Fu=Bar"] = 0;
+            actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program^ Files^ ^(x86^)\\Microsoft^ Visual^ Studio^ 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test2.sln /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /P:Fu=Bar"] = 0;
+            actions.FileExists["csharp.log"] = true;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = false;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"] = false;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat"] = true;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\vcvarsall.bat"] = false;
+            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat"] = true;
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
+            actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest1.cs\ntest2.cs";
+            actions.EnumerateDirectories[@"C:\Project"] = "";
+
+            var autobuilder = CreateAutoBuilder(true, msBuildArguments: "/P:Fu=Bar", msBuildTarget: "Windows",
+                msBuildPlatform: "x86", msBuildConfiguration: "Debug", vsToolsVersion: "12",
+                allSolutions: "true", nugetRestore: "false");
+            var testSolution1 = new TestSolution(@"C:\Project\test1.sln");
+            var testSolution2 = new TestSolution(@"C:\Project\test2.sln");
+            autobuilder.ProjectsOrSolutionsToBuild.Add(testSolution1);
+            autobuilder.ProjectsOrSolutionsToBuild.Add(testSolution2);
+
+            TestAutobuilderScript(autobuilder, 0, 2);
+        }
+
         [Fact]
         public void TestSkipNugetBuildless()
         {
@@ -692,6 +900,35 @@ namespace Semmle.Autobuild.CSharp.Tests
             TestAutobuilderScript(autobuilder, 0, 1);
         }
 
+
+        [Fact]
+        public void TestSkipNugetDotnet()
+        {
+            actions.RunProcess["dotnet --info"] = 0;
+            actions.RunProcess[@"dotnet clean C:\Project/test.csproj"] = 0;
+            actions.RunProcess[@"dotnet restore C:\Project/test.csproj"] = 0;
+            actions.RunProcess[@"dotnet build --no-incremental --no-restore C:\Project/test.csproj"] = 0;
+            actions.FileExists["csharp.log"] = true;
+            actions.FileExists[@"C:\Project/test.csproj"] = true;
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
+            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
+            actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.cs\ntest.csproj";
+            actions.EnumerateDirectories[@"C:\Project"] = "";
+            var xml = new XmlDocument();
+            xml.LoadXml(@"<Project Sdk=""Microsoft.NET.Sdk"">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>netcoreapp2.1</TargetFramework>
+  </PropertyGroup>
+
+</Project>");
+            actions.LoadXml[@"C:\Project/test.csproj"] = xml;
+
+            var autobuilder = CreateAutoBuilder(false, dotnetArguments: "--no-restore");  // nugetRestore=false does not work for now.
+            TestAutobuilderScript(autobuilder, 0, 4);
+        }
+
         [Fact]
         public void TestDotnetVersionNotInstalled()
         {
@@ -816,7 +1053,7 @@ namespace Semmle.Autobuild.CSharp.Tests
         {
             actions.RunProcess[@"cmd.exe /C nuget restore C:\Project\dirs.proj -DisableParallelProcessing"] = 1;
             actions.RunProcess[@"cmd.exe /C scratch\.nuget\nuget.exe restore C:\Project\dirs.proj -DisableParallelProcessing"] = 0;
-            actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program^ Files^ ^(x86^)\\Microsoft^ Visual^ Studio^ 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\dirs.proj /t:rebuild"] = 0;
+            actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program^ Files^ ^(x86^)\\Microsoft^ Visual^ Studio^ 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\dirs.proj /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /P:Fu=Bar"] = 0;
             actions.FileExists["csharp.log"] = true;
             actions.FileExists[@"C:\Project\a\test.csproj"] = true;
             actions.FileExists[@"C:\Project\dirs.proj"] = true;
@@ -851,7 +1088,8 @@ namespace Semmle.Autobuild.CSharp.Tests
 </Project>");
             actions.LoadXml[@"C:\Project\dirs.proj"] = dirsproj;
 
-            var autobuilder = CreateAutoBuilder(true);
+            var autobuilder = CreateAutoBuilder(true, msBuildArguments: "/P:Fu=Bar", msBuildTarget: "Windows", msBuildPlatform: "x86", msBuildConfiguration: "Debug",
+                vsToolsVersion: "12", allSolutions: "true");
             TestAutobuilderScript(autobuilder, 0, 3);
         }
 

csharp/autobuilder/Semmle.Autobuild.CSharp/CSharpAutobuilder.cs

@@ -25,11 +25,9 @@ namespace Semmle.Autobuild.CSharp
         /// </summary>
         public CSharpAutobuildOptions(IBuildActions actions) : base(actions)
         {
-            Buildless =
+            Buildless = actions.GetEnvironmentVariable(lgtmPrefix + "BUILDLESS").AsBool("buildless", false) ||
                 actions.GetEnvironmentVariable(extractorOptionPrefix + "BUILDLESS").AsBool("buildless", false) ||
                 actions.GetEnvironmentVariable(buildModeEnvironmentVariable)?.ToLower() == "none";
-
-
         }
     }
 
@@ -48,12 +46,21 @@ namespace Semmle.Autobuild.CSharp
             var attempt = BuildScript.Failure;
             switch (GetCSharpBuildStrategy())
             {
+                case CSharpBuildStrategy.CustomBuildCommand:
+                    attempt = new BuildCommandRule(DotNetRule.WithDotNet).Analyse(this, false) & CheckExtractorRun(true);
+                    break;
                 case CSharpBuildStrategy.Buildless:
                     // No need to check that the extractor has been executed in buildless mode
                     attempt = BuildScript.Bind(
                         AddBuildlessStartedDiagnostic() & new StandaloneBuildRule().Analyse(this, false),
                         AddBuildlessEndedDiagnostic);
                     break;
+                case CSharpBuildStrategy.MSBuild:
+                    attempt = new MsBuildRule().Analyse(this, false) & CheckExtractorRun(true);
+                    break;
+                case CSharpBuildStrategy.DotNet:
+                    attempt = new DotNetRule().Analyse(this, false) & CheckExtractorRun(true);
+                    break;
                 case CSharpBuildStrategy.Auto:
                     attempt =
                         // Attempt a few different build strategies to see if one works
@@ -239,15 +246,32 @@ namespace Semmle.Autobuild.CSharp
         /// </summary>
         private CSharpBuildStrategy GetCSharpBuildStrategy()
         {
+            if (Options.BuildCommand is not null)
+                return CSharpBuildStrategy.CustomBuildCommand;
+
             if (Options.Buildless)
                 return CSharpBuildStrategy.Buildless;
 
+            if (Options.MsBuildArguments is not null
+                || Options.MsBuildConfiguration is not null
+                || Options.MsBuildPlatform is not null
+                || Options.MsBuildTarget is not null)
+            {
+                return CSharpBuildStrategy.MSBuild;
+            }
+
+            if (Options.DotNetArguments is not null || Options.DotNetVersion is not null)
+                return CSharpBuildStrategy.DotNet;
+
             return CSharpBuildStrategy.Auto;
         }
 
         private enum CSharpBuildStrategy
         {
+            CustomBuildCommand,
             Buildless,
+            MSBuild,
+            DotNet,
             Auto
         }
     }

csharp/autobuilder/Semmle.Autobuild.CSharp/DotNetRule.cs

@@ -32,7 +32,7 @@ namespace Semmle.Autobuild.CSharp
             if (auto)
             {
                 NotDotNetProjects = builder.ProjectsOrSolutionsToBuild
-                    .SelectMany(p => new[] { p }.Concat(p.IncludedProjects))
+                    .SelectMany(p => Enumerators.Singleton(p).Concat(p.IncludedProjects))
                     .OfType<Project<CSharpAutobuildOptions>>()
                     .Where(p => !p.DotNetProject);
                 var notDotNetProject = NotDotNetProjects.FirstOrDefault();
@@ -150,7 +150,8 @@ namespace Semmle.Autobuild.CSharp
                 Argument("--no-incremental");
 
             return
-                script.QuoteArgument(projOrSln).
+                script.Argument(builder.Options.DotNetArguments).
+                    QuoteArgument(projOrSln).
                     Script;
         }
     }

csharp/autobuilder/Semmle.Autobuild.CSharp/Program.cs

@@ -17,7 +17,7 @@ namespace Semmle.Autobuild.CSharp
                 try
                 {
                     Console.WriteLine("CodeQL C# autobuilder");
-                    using var builder = new CSharpAutobuilder(actions, options);
+                    var builder = new CSharpAutobuilder(actions, options);
                     return builder.AttemptBuild();
                 }
                 catch (InvalidEnvironmentException ex)

csharp/autobuilder/Semmle.Autobuild.Shared/AutobuildOptions.cs

@@ -11,9 +11,24 @@ namespace Semmle.Autobuild.Shared
     /// </summary>
     public abstract class AutobuildOptionsShared
     {
+        protected const string lgtmPrefix = "LGTM_INDEX_";
+
+
         public int SearchDepth { get; } = 3;
         public string RootDirectory { get; }
+        public string? VsToolsVersion { get; }
+        public string? MsBuildArguments { get; }
+        public string? MsBuildPlatform { get; }
+        public string? MsBuildConfiguration { get; }
+        public string? MsBuildTarget { get; }
+        public string? DotNetArguments { get; }
         public string? DotNetVersion { get; }
+        public string? BuildCommand { get; }
+        public IEnumerable<string> Solution { get; }
+        public bool IgnoreErrors { get; }
+
+        public bool AllSolutions { get; }
+        public bool NugetRestore { get; }
         public abstract Language Language { get; }
 
         /// <summary>
@@ -23,7 +38,19 @@ namespace Semmle.Autobuild.Shared
         public AutobuildOptionsShared(IBuildActions actions)
         {
             RootDirectory = actions.GetCurrentDirectory();
-            DotNetVersion = actions.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_OPTION_DOTNET_VERSION");
+            VsToolsVersion = actions.GetEnvironmentVariable(lgtmPrefix + "VSTOOLS_VERSION");
+            MsBuildArguments = actions.GetEnvironmentVariable(lgtmPrefix + "MSBUILD_ARGUMENTS")?.AsStringWithExpandedEnvVars(actions);
+            MsBuildPlatform = actions.GetEnvironmentVariable(lgtmPrefix + "MSBUILD_PLATFORM");
+            MsBuildConfiguration = actions.GetEnvironmentVariable(lgtmPrefix + "MSBUILD_CONFIGURATION");
+            MsBuildTarget = actions.GetEnvironmentVariable(lgtmPrefix + "MSBUILD_TARGET");
+            DotNetArguments = actions.GetEnvironmentVariable(lgtmPrefix + "DOTNET_ARGUMENTS")?.AsStringWithExpandedEnvVars(actions);
+            DotNetVersion = actions.GetEnvironmentVariable(lgtmPrefix + "DOTNET_VERSION");
+            BuildCommand = actions.GetEnvironmentVariable(lgtmPrefix + "BUILD_COMMAND");
+            Solution = actions.GetEnvironmentVariable(lgtmPrefix + "SOLUTION").AsListWithExpandedEnvVars(actions, Array.Empty<string>());
+
+            IgnoreErrors = actions.GetEnvironmentVariable(lgtmPrefix + "IGNORE_ERRORS").AsBool("ignore_errors", false);
+            AllSolutions = actions.GetEnvironmentVariable(lgtmPrefix + "ALL_SOLUTIONS").AsBool("all_solutions", false);
+            NugetRestore = actions.GetEnvironmentVariable(lgtmPrefix + "NUGET_RESTORE").AsBool("nuget_restore", true);
         }
     }
 

csharp/autobuilder/Semmle.Autobuild.Shared/Autobuilder.cs

@@ -92,7 +92,7 @@ namespace Semmle.Autobuild.Shared
     /// The overall design is intended to be extensible so that in theory,
     /// it should be possible to add new build rules without touching this code.
     /// </summary>
-    public abstract class Autobuilder<TAutobuildOptions> : IDisposable, IAutobuilder<TAutobuildOptions> where TAutobuildOptions : AutobuildOptionsShared
+    public abstract class Autobuilder<TAutobuildOptions> : IAutobuilder<TAutobuildOptions> where TAutobuildOptions : AutobuildOptionsShared
     {
         /// <summary>
         /// Full file paths of files found in the project directory, as well as
@@ -161,6 +161,9 @@ namespace Semmle.Autobuild.Shared
             if (matchingFiles.Length == 0)
                 return null;
 
+            if (Options.AllSolutions)
+                return matchingFiles.Select(p => p.ProjectOrSolution);
+
             return matchingFiles
                 .Where(f => f.DistanceFromRoot == matchingFiles[0].DistanceFromRoot)
                 .Select(f => f.ProjectOrSolution);
@@ -182,6 +185,19 @@ namespace Semmle.Autobuild.Shared
             projectsOrSolutionsToBuildLazy = new Lazy<IList<IProjectOrSolution>>(() =>
             {
                 List<IProjectOrSolution>? ret;
+                if (options.Solution.Any())
+                {
+                    ret = new List<IProjectOrSolution>();
+                    foreach (var solution in options.Solution)
+                    {
+                        if (actions.FileExists(solution))
+                            ret.Add(new Solution<TAutobuildOptions>(this, solution, true));
+                        else
+                            logger.LogError($"The specified project or solution file {solution} was not found");
+                    }
+                    return ret;
+                }
+
                 // First look for `.proj` files
                 ret = FindFiles(".proj", f => new Project<TAutobuildOptions>(this, f))?.ToList();
                 if (ret is not null)
@@ -269,6 +285,9 @@ namespace Semmle.Autobuild.Shared
 
             var script = GetBuildScript();
 
+            if (Options.IgnoreErrors)
+                script |= BuildScript.Success;
+
             void startCallback(string s, bool silent)
             {
                 logger.Log(silent ? Severity.Debug : Severity.Info, $"\nRunning {s}");
@@ -351,20 +370,6 @@ namespace Semmle.Autobuild.Shared
             }
         });
 
-        public void Dispose()
-        {
-            Dispose(true);
-            GC.SuppressFinalize(this);
-        }
-
-        protected virtual void Dispose(bool disposing)
-        {
-            if (disposing)
-            {
-                diagnostics.Dispose();
-            }
-        }
-
         /// <summary>
         /// Value of CODEQL_EXTRACTOR_<LANG>_ROOT environment variable.
         /// </summary>

csharp/autobuilder/Semmle.Autobuild.Shared/BuildCommandAutoRule.cs

@@ -82,6 +82,11 @@ namespace Semmle.Autobuild.Shared
             {
                 var command = new CommandBuilder(builder.Actions, dir, environment);
 
+                // A specific Visual Studio version may be required
+                var vsTools = MsBuildRule.GetVcVarsBatFile(builder);
+                if (vsTools is not null)
+                    command.CallBatFile(vsTools.Path);
+
                 command.RunCommand(this.ScriptPath);
                 return command.Script;
             });

csharp/autobuilder/Semmle.Autobuild.Shared/BuildCommandRule.cs

@@ -0,0 +1,37 @@
+using Semmle.Util;
+
+namespace Semmle.Autobuild.Shared
+{
+    /// <summary>
+    /// Execute the build_command rule.
+    /// </summary>
+    public class BuildCommandRule : IBuildRule<AutobuildOptionsShared>
+    {
+        private readonly WithDotNet<AutobuildOptionsShared> withDotNet;
+
+        public BuildCommandRule(WithDotNet<AutobuildOptionsShared> withDotNet)
+        {
+            this.withDotNet = withDotNet;
+        }
+
+        public BuildScript Analyse(IAutobuilder<AutobuildOptionsShared> builder, bool auto)
+        {
+            if (builder.Options.BuildCommand is null)
+                return BuildScript.Failure;
+
+            // Custom build commands may require a specific .NET Core version
+            return withDotNet(builder, environment =>
+                {
+                    var command = new CommandBuilder(builder.Actions, null, environment);
+
+                    // Custom build commands may require a specific Visual Studio version
+                    var vsTools = MsBuildRule.GetVcVarsBatFile(builder);
+                    if (vsTools is not null)
+                        command.CallBatFile(vsTools.Path);
+                    command.RunCommand(builder.Options.BuildCommand);
+
+                    return command.Script;
+                });
+        }
+    }
+}

csharp/autobuilder/Semmle.Autobuild.Shared/MsBuildRule.cs

@@ -42,9 +42,9 @@ namespace Semmle.Autobuild.Shared
             if (auto)
                 builder.Logger.LogInfo("Attempting to build using MSBuild");
 
-            VcVarsBatFile? vsTools = null;
+            var vsTools = GetVcVarsBatFile(builder);
 
-            if (builder.ProjectsOrSolutionsToBuild.Any())
+            if (vsTools is null && builder.ProjectsOrSolutionsToBuild.Any())
             {
                 var firstSolution = builder.ProjectsOrSolutionsToBuild.OfType<ISolution>().FirstOrDefault();
                 vsTools = firstSolution is not null
@@ -67,44 +67,46 @@ namespace Semmle.Autobuild.Shared
 
             foreach (var projectOrSolution in builder.ProjectsOrSolutionsToBuild)
             {
-
-                BuildScript GetNugetRestoreScript() =>
-                    new CommandBuilder(builder.Actions).
-                        RunCommand(nuget).
-                        Argument("restore").
-                        QuoteArgument(projectOrSolution.FullPath).
-                        Argument("-DisableParallelProcessing").
-                        Script;
-                var nugetRestore = GetNugetRestoreScript();
-                var msbuildRestoreCommand = new CommandBuilder(builder.Actions).
-                    MsBuildCommand(builder).
-                    Argument("/t:restore").
-                    QuoteArgument(projectOrSolution.FullPath);
-
-                if (builder.Actions.IsRunningOnAppleSilicon())
+                if (builder.Options.NugetRestore)
                 {
-                    // On Apple Silicon, only try package restore with `dotnet msbuild /t:restore`
-                    ret &= BuildScript.Try(msbuildRestoreCommand.Script);
-                }
-                else if (nugetDownloaded)
-                {
-                    ret &= BuildScript.Try(nugetRestore | msbuildRestoreCommand.Script);
-                }
-                else
-                {
-                    // If `nuget restore` fails, and we have not already attempted to download `nuget.exe`,
-                    // download it and reattempt `nuget restore`.
-                    var nugetDownloadAndRestore =
-                        BuildScript.Bind(DownloadNugetExe(builder, nugetDownloadPath), exitCode =>
-                        {
-                            nugetDownloaded = true;
-                            if (exitCode != 0)
-                                return BuildScript.Failure;
+                    BuildScript GetNugetRestoreScript() =>
+                        new CommandBuilder(builder.Actions).
+                            RunCommand(nuget).
+                            Argument("restore").
+                            QuoteArgument(projectOrSolution.FullPath).
+                            Argument("-DisableParallelProcessing").
+                            Script;
+                    var nugetRestore = GetNugetRestoreScript();
+                    var msbuildRestoreCommand = new CommandBuilder(builder.Actions).
+                        MsBuildCommand(builder).
+                        Argument("/t:restore").
+                        QuoteArgument(projectOrSolution.FullPath);
 
-                            nuget = nugetDownloadPath;
-                            return GetNugetRestoreScript();
-                        });
-                    ret &= BuildScript.Try(nugetRestore | nugetDownloadAndRestore | msbuildRestoreCommand.Script);
+                    if (builder.Actions.IsRunningOnAppleSilicon())
+                    {
+                        // On Apple Silicon, only try package restore with `dotnet msbuild /t:restore`
+                        ret &= BuildScript.Try(msbuildRestoreCommand.Script);
+                    }
+                    else if (nugetDownloaded)
+                    {
+                        ret &= BuildScript.Try(nugetRestore | msbuildRestoreCommand.Script);
+                    }
+                    else
+                    {
+                        // If `nuget restore` fails, and we have not already attempted to download `nuget.exe`,
+                        // download it and reattempt `nuget restore`.
+                        var nugetDownloadAndRestore =
+                            BuildScript.Bind(DownloadNugetExe(builder, nugetDownloadPath), exitCode =>
+                            {
+                                nugetDownloaded = true;
+                                if (exitCode != 0)
+                                    return BuildScript.Failure;
+
+                                nuget = nugetDownloadPath;
+                                return GetNugetRestoreScript();
+                            });
+                        ret &= BuildScript.Try(nugetRestore | nugetDownloadAndRestore | msbuildRestoreCommand.Script);
+                    }
                 }
 
                 var command = new CommandBuilder(builder.Actions);
@@ -122,9 +124,9 @@ namespace Semmle.Autobuild.Shared
                 command.MsBuildCommand(builder);
                 command.QuoteArgument(projectOrSolution.FullPath);
 
-                var target = "rebuild";
-                var platform = projectOrSolution is ISolution s1 ? s1.DefaultPlatformName : null;
-                var configuration = projectOrSolution is ISolution s2 ? s2.DefaultConfigurationName : null;
+                var target = builder.Options.MsBuildTarget ?? "rebuild";
+                var platform = builder.Options.MsBuildPlatform ?? (projectOrSolution is ISolution s1 ? s1.DefaultPlatformName : null);
+                var configuration = builder.Options.MsBuildConfiguration ?? (projectOrSolution is ISolution s2 ? s2.DefaultConfigurationName : null);
 
                 command.Argument("/t:" + target);
                 if (platform is not null)
@@ -132,6 +134,8 @@ namespace Semmle.Autobuild.Shared
                 if (configuration is not null)
                     command.Argument(string.Format("/p:Configuration=\"{0}\"", configuration));
 
+                command.Argument(builder.Options.MsBuildArguments);
+
                 // append the build script which invokes msbuild to the overall build script `ret`;
                 // we insert a check that building the current project or solution was successful:
                 // if it was not successful, we add it to `FailedProjectsOrSolutions`
@@ -144,6 +148,41 @@ namespace Semmle.Autobuild.Shared
             return ret;
         }
 
+        /// <summary>
+        /// Gets the BAT file used to initialize the appropriate Visual Studio
+        /// version/platform, as specified by the `vstools_version` property in
+        /// lgtm.yml.
+        ///
+        /// Returns <code>null</code> when no version is specified.
+        /// </summary>
+        public static VcVarsBatFile? GetVcVarsBatFile<TAutobuildOptions>(IAutobuilder<TAutobuildOptions> builder) where TAutobuildOptions : AutobuildOptionsShared
+        {
+            VcVarsBatFile? vsTools = null;
+
+            if (builder.Options.VsToolsVersion is not null)
+            {
+                if (int.TryParse(builder.Options.VsToolsVersion, out var msToolsVersion))
+                {
+                    foreach (var b in BuildTools.VcVarsAllBatFiles(builder.Actions))
+                    {
+                        builder.Logger.Log(Severity.Info, "Found {0} version {1}", b.Path, b.ToolsVersion);
+                    }
+
+                    vsTools = BuildTools.FindCompatibleVcVars(builder.Actions, msToolsVersion);
+                    if (vsTools is null)
+                        builder.Logger.LogWarning("Could not find build tools matching version {0}", msToolsVersion);
+                    else
+                        builder.Logger.Log(Severity.Info, "Setting Visual Studio tools to {0}", vsTools.Path);
+                }
+                else
+                {
+                    builder.Logger.LogError("The format of vstools_version is incorrect. Please specify an integer.");
+                }
+            }
+
+            return vsTools;
+        }
+
         /// <summary>
         /// Returns a script for downloading `nuget.exe` from nuget.org.
         /// </summary>

csharp/documentation/library-coverage/coverage.csv

@@ -1,44 +1,44 @@
-package,sink,source,summary,sink:code-injection,sink:encryption-decryptor,sink:encryption-encryptor,sink:encryption-keyprop,sink:encryption-symmetrickey,sink:file-content-store,sink:html-injection,sink:js-injection,sink:log-injection,sink:sql-injection,source:commandargs,source:database,source:environment,source:file,source:file-write,source:local,source:remote,source:windows-registry,summary:taint,summary:value
-Amazon.Lambda.APIGatewayEvents,,6,,,,,,,,,,,,,,,,,,6,,,
-Amazon.Lambda.Core,10,,,,,,,,,,,10,,,,,,,,,,,
-Dapper,55,42,1,,,,,,,,,,55,,42,,,,,,,,1
-ILCompiler,,,81,,,,,,,,,,,,,,,,,,,81,
-ILLink.RoslynAnalyzer,,,63,,,,,,,,,,,,,,,,,,,63,
-ILLink.Shared,,,32,,,,,,,,,,,,,,,,,,,29,3
-ILLink.Tasks,,,5,,,,,,,,,,,,,,,,,,,5,
-Internal.IL,,,69,,,,,,,,,,,,,,,,,,,67,2
-Internal.Pgo,,,9,,,,,,,,,,,,,,,,,,,8,1
-Internal.TypeSystem,,,367,,,,,,,,,,,,,,,,,,,331,36
-JsonToItemsTaskFactory,,,7,,,,,,,,,,,,,,,,,,,7,
-Microsoft.Android.Build,,,14,,,,,,,,,,,,,,,,,,,14,
-Microsoft.Apple.Build,,,7,,,,,,,,,,,,,,,,,,,7,
-Microsoft.ApplicationBlocks.Data,28,,,,,,,,,,,,28,,,,,,,,,,
-Microsoft.CSharp,,,24,,,,,,,,,,,,,,,,,,,24,
-Microsoft.Diagnostics.Tools.Pgo,,,13,,,,,,,,,,,,,,,,,,,13,
-Microsoft.EntityFrameworkCore,6,,12,,,,,,,,,,6,,,,,,,,,,12
-Microsoft.Extensions.Caching.Distributed,,,15,,,,,,,,,,,,,,,,,,,15,
-Microsoft.Extensions.Caching.Memory,,,38,,,,,,,,,,,,,,,,,,,37,1
-Microsoft.Extensions.Configuration,,2,89,,,,,,,,,,,,,2,,,,,,86,3
-Microsoft.Extensions.DependencyInjection,,,120,,,,,,,,,,,,,,,,,,,120,
-Microsoft.Extensions.DependencyModel,,,12,,,,,,,,,,,,,,,,,,,12,
-Microsoft.Extensions.Diagnostics.Metrics,,,13,,,,,,,,,,,,,,,,,,,13,
-Microsoft.Extensions.FileProviders,,,15,,,,,,,,,,,,,,,,,,,15,
-Microsoft.Extensions.FileSystemGlobbing,,,16,,,,,,,,,,,,,,,,,,,14,2
-Microsoft.Extensions.Hosting,,,23,,,,,,,,,,,,,,,,,,,22,1
-Microsoft.Extensions.Http,,,10,,,,,,,,,,,,,,,,,,,10,
-Microsoft.Extensions.Logging,,,60,,,,,,,,,,,,,,,,,,,59,1
-Microsoft.Extensions.Options,,,8,,,,,,,,,,,,,,,,,,,8,
-Microsoft.Extensions.Primitives,,,64,,,,,,,,,,,,,,,,,,,64,
-Microsoft.Interop,,,78,,,,,,,,,,,,,,,,,,,78,
-Microsoft.NET.Build.Tasks,,,1,,,,,,,,,,,,,,,,,,,1,
-Microsoft.NET.WebAssembly.Webcil,,,7,,,,,,,,,,,,,,,,,,,7,
-Microsoft.VisualBasic,,,10,,,,,,,,,,,,,,,,,,,5,5
-Microsoft.WebAssembly.Build.Tasks,,,3,,,,,,,,,,,,,,,,,,,3,
-Microsoft.Win32,,4,4,,,,,,,,,,,,,,,,,,4,4,
-Mono.Linker,,,163,,,,,,,,,,,,,,,,,,,163,
-MySql.Data.MySqlClient,48,,,,,,,,,,,,48,,,,,,,,,,
-Newtonsoft.Json,,,91,,,,,,,,,,,,,,,,,,,73,18
-ServiceStack,194,,7,27,,,,,75,,,,92,,,,,,,,,7,
-SourceGenerators,,,4,,,,,,,,,,,,,,,,,,,4,
-System,67,44,11872,,8,8,9,,,4,5,,33,2,,3,15,17,3,4,,9906,1966
-Windows.Security.Cryptography.Core,1,,,,,,,1,,,,,,,,,,,,,,,
+package,sink,source,summary,sink:code-injection,sink:encryption-decryptor,sink:encryption-encryptor,sink:encryption-keyprop,sink:encryption-symmetrickey,sink:file-content-store,sink:html-injection,sink:js-injection,sink:log-injection,sink:sql-injection,source:commandargs,source:environment,source:file,source:file-write,source:local,source:remote,source:windows-registry,summary:taint,summary:value
+Amazon.Lambda.APIGatewayEvents,,6,,,,,,,,,,,,,,,,,6,,,
+Amazon.Lambda.Core,10,,,,,,,,,,,10,,,,,,,,,,
+Dapper,55,,,,,,,,,,,,55,,,,,,,,,
+ILCompiler,,,81,,,,,,,,,,,,,,,,,,81,
+ILLink.RoslynAnalyzer,,,63,,,,,,,,,,,,,,,,,,63,
+ILLink.Shared,,,32,,,,,,,,,,,,,,,,,,29,3
+ILLink.Tasks,,,5,,,,,,,,,,,,,,,,,,5,
+Internal.IL,,,69,,,,,,,,,,,,,,,,,,67,2
+Internal.Pgo,,,9,,,,,,,,,,,,,,,,,,8,1
+Internal.TypeSystem,,,367,,,,,,,,,,,,,,,,,,331,36
+JsonToItemsTaskFactory,,,7,,,,,,,,,,,,,,,,,,7,
+Microsoft.Android.Build,,,14,,,,,,,,,,,,,,,,,,14,
+Microsoft.Apple.Build,,,7,,,,,,,,,,,,,,,,,,7,
+Microsoft.ApplicationBlocks.Data,28,,,,,,,,,,,,28,,,,,,,,,
+Microsoft.CSharp,,,24,,,,,,,,,,,,,,,,,,24,
+Microsoft.Diagnostics.Tools.Pgo,,,13,,,,,,,,,,,,,,,,,,13,
+Microsoft.EntityFrameworkCore,6,,12,,,,,,,,,,6,,,,,,,,,12
+Microsoft.Extensions.Caching.Distributed,,,15,,,,,,,,,,,,,,,,,,15,
+Microsoft.Extensions.Caching.Memory,,,38,,,,,,,,,,,,,,,,,,37,1
+Microsoft.Extensions.Configuration,,2,89,,,,,,,,,,,,2,,,,,,86,3
+Microsoft.Extensions.DependencyInjection,,,120,,,,,,,,,,,,,,,,,,120,
+Microsoft.Extensions.DependencyModel,,,12,,,,,,,,,,,,,,,,,,12,
+Microsoft.Extensions.Diagnostics.Metrics,,,13,,,,,,,,,,,,,,,,,,13,
+Microsoft.Extensions.FileProviders,,,15,,,,,,,,,,,,,,,,,,15,
+Microsoft.Extensions.FileSystemGlobbing,,,16,,,,,,,,,,,,,,,,,,14,2
+Microsoft.Extensions.Hosting,,,23,,,,,,,,,,,,,,,,,,22,1
+Microsoft.Extensions.Http,,,10,,,,,,,,,,,,,,,,,,10,
+Microsoft.Extensions.Logging,,,60,,,,,,,,,,,,,,,,,,59,1
+Microsoft.Extensions.Options,,,8,,,,,,,,,,,,,,,,,,8,
+Microsoft.Extensions.Primitives,,,64,,,,,,,,,,,,,,,,,,64,
+Microsoft.Interop,,,78,,,,,,,,,,,,,,,,,,78,
+Microsoft.NET.Build.Tasks,,,1,,,,,,,,,,,,,,,,,,1,
+Microsoft.NET.WebAssembly.Webcil,,,7,,,,,,,,,,,,,,,,,,7,
+Microsoft.VisualBasic,,,10,,,,,,,,,,,,,,,,,,5,5
+Microsoft.WebAssembly.Build.Tasks,,,3,,,,,,,,,,,,,,,,,,3,
+Microsoft.Win32,,4,4,,,,,,,,,,,,,,,,,4,4,
+Mono.Linker,,,163,,,,,,,,,,,,,,,,,,163,
+MySql.Data.MySqlClient,48,,,,,,,,,,,,48,,,,,,,,,
+Newtonsoft.Json,,,91,,,,,,,,,,,,,,,,,,73,18
+ServiceStack,194,,7,27,,,,,75,,,,92,,,,,,,,7,
+SourceGenerators,,,4,,,,,,,,,,,,,,,,,,4,
+System,67,30,11864,,8,8,9,,,4,5,,33,2,3,1,17,3,4,,9898,1966
+Windows.Security.Cryptography.Core,1,,,,,,,1,,,,,,,,,,,,,,

csharp/documentation/library-coverage/coverage.rst

@@ -8,7 +8,7 @@ C# framework & library support
 
    Framework / library,Package,Flow sources,Taint & value steps,Sinks (total),`CWE-079` :sub:`Cross-site scripting`
    `ServiceStack <https://servicestack.net/>`_,"``ServiceStack.*``, ``ServiceStack``",,7,194,
-   System,"``System.*``, ``System``",44,11872,67,9
-   Others,"``Amazon.Lambda.APIGatewayEvents``, ``Amazon.Lambda.Core``, ``Dapper``, ``ILCompiler``, ``ILLink.RoslynAnalyzer``, ``ILLink.Shared``, ``ILLink.Tasks``, ``Internal.IL``, ``Internal.Pgo``, ``Internal.TypeSystem``, ``JsonToItemsTaskFactory``, ``Microsoft.Android.Build``, ``Microsoft.Apple.Build``, ``Microsoft.ApplicationBlocks.Data``, ``Microsoft.CSharp``, ``Microsoft.Diagnostics.Tools.Pgo``, ``Microsoft.EntityFrameworkCore``, ``Microsoft.Extensions.Caching.Distributed``, ``Microsoft.Extensions.Caching.Memory``, ``Microsoft.Extensions.Configuration``, ``Microsoft.Extensions.DependencyInjection``, ``Microsoft.Extensions.DependencyModel``, ``Microsoft.Extensions.Diagnostics.Metrics``, ``Microsoft.Extensions.FileProviders``, ``Microsoft.Extensions.FileSystemGlobbing``, ``Microsoft.Extensions.Hosting``, ``Microsoft.Extensions.Http``, ``Microsoft.Extensions.Logging``, ``Microsoft.Extensions.Options``, ``Microsoft.Extensions.Primitives``, ``Microsoft.Interop``, ``Microsoft.NET.Build.Tasks``, ``Microsoft.NET.WebAssembly.Webcil``, ``Microsoft.VisualBasic``, ``Microsoft.WebAssembly.Build.Tasks``, ``Microsoft.Win32``, ``Mono.Linker``, ``MySql.Data.MySqlClient``, ``Newtonsoft.Json``, ``SourceGenerators``, ``Windows.Security.Cryptography.Core``",54,1548,148,
-   Totals,,98,13427,409,9
+   System,"``System.*``, ``System``",30,11864,67,9
+   Others,"``Amazon.Lambda.APIGatewayEvents``, ``Amazon.Lambda.Core``, ``Dapper``, ``ILCompiler``, ``ILLink.RoslynAnalyzer``, ``ILLink.Shared``, ``ILLink.Tasks``, ``Internal.IL``, ``Internal.Pgo``, ``Internal.TypeSystem``, ``JsonToItemsTaskFactory``, ``Microsoft.Android.Build``, ``Microsoft.Apple.Build``, ``Microsoft.ApplicationBlocks.Data``, ``Microsoft.CSharp``, ``Microsoft.Diagnostics.Tools.Pgo``, ``Microsoft.EntityFrameworkCore``, ``Microsoft.Extensions.Caching.Distributed``, ``Microsoft.Extensions.Caching.Memory``, ``Microsoft.Extensions.Configuration``, ``Microsoft.Extensions.DependencyInjection``, ``Microsoft.Extensions.DependencyModel``, ``Microsoft.Extensions.Diagnostics.Metrics``, ``Microsoft.Extensions.FileProviders``, ``Microsoft.Extensions.FileSystemGlobbing``, ``Microsoft.Extensions.Hosting``, ``Microsoft.Extensions.Http``, ``Microsoft.Extensions.Logging``, ``Microsoft.Extensions.Options``, ``Microsoft.Extensions.Primitives``, ``Microsoft.Interop``, ``Microsoft.NET.Build.Tasks``, ``Microsoft.NET.WebAssembly.Webcil``, ``Microsoft.VisualBasic``, ``Microsoft.WebAssembly.Build.Tasks``, ``Microsoft.Win32``, ``Mono.Linker``, ``MySql.Data.MySqlClient``, ``Newtonsoft.Json``, ``SourceGenerators``, ``Windows.Security.Cryptography.Core``",12,1547,148,
+   Totals,,42,13418,409,9
 

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs

@@ -1,514 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using System.Net.Http;
-using System.Text.RegularExpressions;
-using System.Threading;
-using System.Threading.Tasks;
-using Semmle.Util;
-
-namespace Semmle.Extraction.CSharp.DependencyFetching
-{
-    public sealed partial class DependencyManager
-    {
-        private void RestoreNugetPackages(List<FileInfo> allNonBinaryFiles, IEnumerable<string> allProjects, IEnumerable<string> allSolutions, HashSet<string> dllPaths)
-        {
-            try
-            {
-                var checkNugetFeedResponsiveness = EnvironmentVariables.GetBoolean(EnvironmentVariableNames.CheckNugetFeedResponsiveness);
-                if (checkNugetFeedResponsiveness && !CheckFeeds(allNonBinaryFiles))
-                {
-                    DownloadMissingPackages(allNonBinaryFiles, dllPaths, withNugetConfig: false);
-                    return;
-                }
-
-                using (var nuget = new NugetPackages(sourceDir.FullName, legacyPackageDirectory, logger))
-                {
-                    var count = nuget.InstallPackages();
-
-                    if (nuget.PackageCount > 0)
-                    {
-                        CompilationInfos.Add(("packages.config files", nuget.PackageCount.ToString()));
-                        CompilationInfos.Add(("Successfully restored packages.config files", count.ToString()));
-                    }
-                }
-
-                var nugetPackageDlls = legacyPackageDirectory.DirInfo.GetFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = true });
-                var nugetPackageDllPaths = nugetPackageDlls.Select(f => f.FullName).ToHashSet();
-                var excludedPaths = nugetPackageDllPaths
-                    .Where(path => IsPathInSubfolder(path, legacyPackageDirectory.DirInfo.FullName, "tools"))
-                    .ToList();
-
-                if (nugetPackageDllPaths.Count > 0)
-                {
-                    logger.LogInfo($"Restored {nugetPackageDllPaths.Count} Nuget DLLs.");
-                }
-                if (excludedPaths.Count > 0)
-                {
-                    logger.LogInfo($"Excluding {excludedPaths.Count} Nuget DLLs.");
-                }
-
-                foreach (var excludedPath in excludedPaths)
-                {
-                    logger.LogInfo($"Excluded Nuget DLL: {excludedPath}");
-                }
-
-                nugetPackageDllPaths.ExceptWith(excludedPaths);
-                dllPaths.UnionWith(nugetPackageDllPaths);
-            }
-            catch (Exception exc)
-            {
-                logger.LogError($"Failed to restore Nuget packages with nuget.exe: {exc.Message}");
-            }
-
-            var restoredProjects = RestoreSolutions(allSolutions, out var assets1);
-            var projects = allProjects.Except(restoredProjects);
-            RestoreProjects(projects, out var assets2);
-
-            var dependencies = Assets.GetCompilationDependencies(logger, assets1.Union(assets2));
-
-            var paths = dependencies
-                .Paths
-                .Select(d => Path.Combine(packageDirectory.DirInfo.FullName, d))
-                .ToList();
-            dllPaths.UnionWith(paths);
-
-            LogAllUnusedPackages(dependencies);
-            DownloadMissingPackages(allNonBinaryFiles, dllPaths);
-        }
-
-        /// <summary>
-        /// Executes `dotnet restore` on all solution files in solutions.
-        /// As opposed to RestoreProjects this is not run in parallel using PLINQ
-        /// as `dotnet restore` on a solution already uses multiple threads for restoring
-        /// the projects (this can be disabled with the `--disable-parallel` flag).
-        /// Populates assets with the relative paths to the assets files generated by the restore.
-        /// Returns a list of projects that are up to date with respect to restore.
-        /// </summary>
-        /// <param name="solutions">A list of paths to solution files.</param>
-        private IEnumerable<string> RestoreSolutions(IEnumerable<string> solutions, out IEnumerable<string> assets)
-        {
-            var successCount = 0;
-            var nugetSourceFailures = 0;
-            var assetFiles = new List<string>();
-            var projects = solutions.SelectMany(solution =>
-                {
-                    logger.LogInfo($"Restoring solution {solution}...");
-                    var res = dotnet.Restore(new(solution, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
-                    if (res.Success)
-                    {
-                        successCount++;
-                    }
-                    if (res.HasNugetPackageSourceError)
-                    {
-                        nugetSourceFailures++;
-                    }
-                    assetFiles.AddRange(res.AssetsFilePaths);
-                    return res.RestoredProjects;
-                }).ToList();
-            assets = assetFiles;
-            CompilationInfos.Add(("Successfully restored solution files", successCount.ToString()));
-            CompilationInfos.Add(("Failed solution restore with package source error", nugetSourceFailures.ToString()));
-            CompilationInfos.Add(("Restored projects through solution files", projects.Count.ToString()));
-            return projects;
-        }
-
-        /// <summary>
-        /// Executes `dotnet restore` on all projects in projects.
-        /// This is done in parallel for performance reasons.
-        /// Populates assets with the relative paths to the assets files generated by the restore.
-        /// </summary>
-        /// <param name="projects">A list of paths to project files.</param>
-        private void RestoreProjects(IEnumerable<string> projects, out IEnumerable<string> assets)
-        {
-            var successCount = 0;
-            var nugetSourceFailures = 0;
-            var assetFiles = new List<string>();
-            var sync = new object();
-            Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = threads }, project =>
-            {
-                logger.LogInfo($"Restoring project {project}...");
-                var res = dotnet.Restore(new(project, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
-                lock (sync)
-                {
-                    if (res.Success)
-                    {
-                        successCount++;
-                    }
-                    if (res.HasNugetPackageSourceError)
-                    {
-                        nugetSourceFailures++;
-                    }
-                    assetFiles.AddRange(res.AssetsFilePaths);
-                }
-            });
-            assets = assetFiles;
-            CompilationInfos.Add(("Successfully restored project files", successCount.ToString()));
-            CompilationInfos.Add(("Failed project restore with package source error", nugetSourceFailures.ToString()));
-        }
-
-        private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPaths, bool withNugetConfig = true)
-        {
-            var alreadyDownloadedPackages = GetRestoredPackageDirectoryNames(packageDirectory.DirInfo);
-            var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames();
-
-            var notYetDownloadedPackages = new HashSet<PackageReference>(fileContent.AllPackages);
-            foreach (var alreadyDownloadedPackage in alreadyDownloadedPackages)
-            {
-                notYetDownloadedPackages.Remove(new(alreadyDownloadedPackage, PackageReferenceSource.SdkCsProj));
-            }
-            foreach (var alreadyDownloadedLegacyPackage in alreadyDownloadedLegacyPackages)
-            {
-                notYetDownloadedPackages.Remove(new(alreadyDownloadedLegacyPackage, PackageReferenceSource.PackagesConfig));
-            }
-
-            if (notYetDownloadedPackages.Count == 0)
-            {
-                return;
-            }
-
-            var multipleVersions = notYetDownloadedPackages
-                .GroupBy(p => p.Name)
-                .Where(g => g.Count() > 1)
-                .Select(g => g.Key)
-                .ToList();
-
-            foreach (var package in multipleVersions)
-            {
-                logger.LogWarning($"Found multiple not yet restored packages with name '{package}'.");
-                notYetDownloadedPackages.Remove(new(package, PackageReferenceSource.PackagesConfig));
-            }
-
-            logger.LogInfo($"Found {notYetDownloadedPackages.Count} packages that are not yet restored");
-            var nugetConfig = withNugetConfig
-                ? GetNugetConfig(allFiles)
-                : null;
-
-            CompilationInfos.Add(("Fallback nuget restore", notYetDownloadedPackages.Count.ToString()));
-
-            var successCount = 0;
-            var sync = new object();
-
-            Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = threads }, package =>
-            {
-                var success = TryRestorePackageManually(package.Name, nugetConfig, package.PackageReferenceSource);
-                if (!success)
-                {
-                    return;
-                }
-
-                lock (sync)
-                {
-                    successCount++;
-                }
-            });
-
-            CompilationInfos.Add(("Successfully ran fallback nuget restore", successCount.ToString()));
-
-            dllPaths.Add(missingPackageDirectory.DirInfo.FullName);
-        }
-
-        private string[] GetAllNugetConfigs(List<FileInfo> allFiles) => allFiles.SelectFileNamesByName("nuget.config").ToArray();
-
-        private string? GetNugetConfig(List<FileInfo> allFiles)
-        {
-            var nugetConfigs = GetAllNugetConfigs(allFiles);
-            string? nugetConfig;
-            if (nugetConfigs.Length > 1)
-            {
-                logger.LogInfo($"Found multiple nuget.config files: {string.Join(", ", nugetConfigs)}.");
-                nugetConfig = allFiles
-                    .SelectRootFiles(sourceDir)
-                    .SelectFileNamesByName("nuget.config")
-                    .FirstOrDefault();
-                if (nugetConfig == null)
-                {
-                    logger.LogInfo("Could not find a top-level nuget.config file.");
-                }
-            }
-            else
-            {
-                nugetConfig = nugetConfigs.FirstOrDefault();
-            }
-
-            if (nugetConfig != null)
-            {
-                logger.LogInfo($"Using nuget.config file {nugetConfig}.");
-            }
-
-            return nugetConfig;
-        }
-
-        private void LogAllUnusedPackages(DependencyContainer dependencies)
-        {
-            var allPackageDirectories = GetAllPackageDirectories();
-
-            logger.LogInfo($"Restored {allPackageDirectories.Count} packages");
-            logger.LogInfo($"Found {dependencies.Packages.Count} packages in project.assets.json files");
-
-            allPackageDirectories
-                .Where(package => !dependencies.Packages.Contains(package))
-                .Order()
-                .ForEach(package => logger.LogInfo($"Unused package: {package}"));
-        }
-
-
-        private ICollection<string> GetAllPackageDirectories()
-        {
-            return new DirectoryInfo(packageDirectory.DirInfo.FullName)
-                .EnumerateDirectories("*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
-                .Select(d => d.Name)
-                .ToList();
-        }
-
-        private static bool IsPathInSubfolder(string path, string rootFolder, string subFolder)
-        {
-            return path.IndexOf(
-                $"{Path.DirectorySeparatorChar}{subFolder}{Path.DirectorySeparatorChar}",
-                rootFolder.Length,
-                StringComparison.InvariantCultureIgnoreCase) >= 0;
-        }
-
-        private IEnumerable<string> GetRestoredLegacyPackageNames()
-        {
-            var oldPackageDirectories = GetRestoredPackageDirectoryNames(legacyPackageDirectory.DirInfo);
-            foreach (var oldPackageDirectory in oldPackageDirectories)
-            {
-                // nuget install restores packages to 'packagename.version' folders (dotnet restore to 'packagename/version' folders)
-                // typical folder names look like:
-                //   newtonsoft.json.13.0.3
-                // there are more complex ones too, such as:
-                //   runtime.tizen.4.0.0-armel.Microsoft.NETCore.DotNetHostResolver.2.0.0-preview2-25407-01
-
-                var match = LegacyNugetPackage().Match(oldPackageDirectory);
-                if (!match.Success)
-                {
-                    logger.LogWarning($"Package directory '{oldPackageDirectory}' doesn't match the expected pattern.");
-                    continue;
-                }
-
-                yield return match.Groups[1].Value.ToLowerInvariant();
-            }
-        }
-
-        private static IEnumerable<string> GetRestoredPackageDirectoryNames(DirectoryInfo root)
-        {
-            return Directory.GetDirectories(root.FullName)
-                .Select(d => Path.GetFileName(d).ToLowerInvariant());
-        }
-
-        private bool TryRestorePackageManually(string package, string? nugetConfig, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj)
-        {
-            logger.LogInfo($"Restoring package {package}...");
-            using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package, "missingpackages_workingdir"));
-            var success = dotnet.New(tempDir.DirInfo.FullName);
-            if (!success)
-            {
-                return false;
-            }
-
-            if (packageReferenceSource == PackageReferenceSource.PackagesConfig)
-            {
-                TryChangeTargetFrameworkMoniker(tempDir.DirInfo);
-            }
-
-            success = dotnet.AddPackage(tempDir.DirInfo.FullName, package);
-            if (!success)
-            {
-                return false;
-            }
-
-            var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig));
-            if (!res.Success)
-            {
-                if (res.HasNugetPackageSourceError && nugetConfig is not null)
-                {
-                    // Restore could not be completed because the listed source is unavailable. Try without the nuget.config:
-                    res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: null, ForceReevaluation: true));
-                }
-
-                // TODO: the restore might fail, we could retry with
-                // - a prerelease (*-* instead of *) version of the package,
-                // - a different target framework moniker.
-
-                if (!res.Success)
-                {
-                    logger.LogInfo($"Failed to restore nuget package {package}");
-                    return false;
-                }
-            }
-
-            return true;
-        }
-
-        private void TryChangeTargetFrameworkMoniker(DirectoryInfo tempDir)
-        {
-            try
-            {
-                logger.LogInfo($"Changing the target framework moniker in {tempDir.FullName}...");
-
-                var csprojs = tempDir.GetFiles("*.csproj", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive });
-                if (csprojs.Length != 1)
-                {
-                    logger.LogError($"Could not find the .csproj file in {tempDir.FullName}, count = {csprojs.Length}");
-                    return;
-                }
-
-                var csproj = csprojs[0];
-                var content = File.ReadAllText(csproj.FullName);
-                var matches = TargetFramework().Matches(content);
-                if (matches.Count == 0)
-                {
-                    logger.LogError($"Could not find target framework in {csproj.FullName}");
-                }
-                else
-                {
-                    content = TargetFramework().Replace(content, $"<TargetFramework>{FrameworkPackageNames.LatestNetFrameworkMoniker}</TargetFramework>", 1);
-                    File.WriteAllText(csproj.FullName, content);
-                }
-            }
-            catch (Exception exc)
-            {
-                logger.LogError($"Failed to update target framework in {tempDir.FullName}: {exc}");
-            }
-        }
-
-        private static async Task ExecuteGetRequest(string address, HttpClient httpClient, CancellationToken cancellationToken)
-        {
-            using var stream = await httpClient.GetStreamAsync(address, cancellationToken);
-            var buffer = new byte[1024];
-            int bytesRead;
-            while ((bytesRead = stream.Read(buffer, 0, buffer.Length)) > 0)
-            {
-                // do nothing
-            }
-        }
-
-        private bool IsFeedReachable(string feed)
-        {
-            logger.LogInfo($"Checking if Nuget feed '{feed}' is reachable...");
-            using HttpClient client = new();
-            int timeoutMilliSeconds = int.TryParse(Environment.GetEnvironmentVariable(EnvironmentVariableNames.NugetFeedResponsivenessInitialTimeout), out timeoutMilliSeconds)
-                ? timeoutMilliSeconds
-                : 1000;
-            int tryCount = int.TryParse(Environment.GetEnvironmentVariable(EnvironmentVariableNames.NugetFeedResponsivenessRequestCount), out tryCount)
-                ? tryCount
-                : 4;
-
-            for (var i = 0; i < tryCount; i++)
-            {
-                using var cts = new CancellationTokenSource();
-                cts.CancelAfter(timeoutMilliSeconds);
-                try
-                {
-                    ExecuteGetRequest(feed, client, cts.Token).GetAwaiter().GetResult();
-                    return true;
-                }
-                catch (Exception exc)
-                {
-                    if (exc is TaskCanceledException tce &&
-                        tce.CancellationToken == cts.Token &&
-                        cts.Token.IsCancellationRequested)
-                    {
-                        logger.LogWarning($"Didn't receive answer from Nuget feed '{feed}' in {timeoutMilliSeconds}ms.");
-                        timeoutMilliSeconds *= 2;
-                        continue;
-                    }
-
-                    // We're only interested in timeouts.
-                    logger.LogWarning($"Querying Nuget feed '{feed}' failed: {exc}");
-                    return true;
-                }
-            }
-
-            logger.LogWarning($"Didn't receive answer from Nuget feed '{feed}'. Tried it {tryCount} times.");
-            return false;
-        }
-
-        private bool CheckFeeds(List<FileInfo> allFiles)
-        {
-            logger.LogInfo("Checking Nuget feeds...");
-            var feeds = GetAllFeeds(allFiles);
-
-            var excludedFeeds = Environment.GetEnvironmentVariable(EnvironmentVariableNames.ExcludedNugetFeedsFromResponsivenessCheck)
-                ?.Split(" ", StringSplitOptions.RemoveEmptyEntries)
-                .ToHashSet() ?? [];
-
-            if (excludedFeeds.Count > 0)
-            {
-                logger.LogInfo($"Excluded Nuget feeds from responsiveness check: {string.Join(", ", excludedFeeds.OrderBy(f => f))}");
-            }
-
-            var allFeedsReachable = feeds.All(feed => excludedFeeds.Contains(feed) || IsFeedReachable(feed));
-            if (!allFeedsReachable)
-            {
-                logger.LogWarning("Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.");
-                diagnosticsWriter.AddEntry(new DiagnosticMessage(
-                    Language.CSharp,
-                    "buildless/unreachable-feed",
-                    "Found unreachable Nuget feed in C# analysis with build-mode 'none'",
-                    visibility: new DiagnosticMessage.TspVisibility(statusPage: true, cliSummaryTable: true, telemetry: true),
-                    markdownMessage: "Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.",
-                    severity: DiagnosticMessage.TspSeverity.Warning
-                ));
-            }
-            CompilationInfos.Add(("All Nuget feeds reachable", allFeedsReachable ? "1" : "0"));
-            return allFeedsReachable;
-        }
-
-        private IEnumerable<string> GetFeeds(string nugetConfig)
-        {
-            logger.LogInfo($"Getting Nuget feeds from '{nugetConfig}'...");
-            var results = dotnet.GetNugetFeeds(nugetConfig);
-            var regex = EnabledNugetFeed();
-            foreach (var result in results)
-            {
-                var match = regex.Match(result);
-                if (!match.Success)
-                {
-                    logger.LogError($"Failed to parse feed from '{result}'");
-                    continue;
-                }
-
-                var url = match.Groups[1].Value;
-                if (!url.StartsWith("https://", StringComparison.InvariantCultureIgnoreCase) &&
-                    !url.StartsWith("http://", StringComparison.InvariantCultureIgnoreCase))
-                {
-                    logger.LogInfo($"Skipping feed '{url}' as it is not a valid URL.");
-                    continue;
-                }
-
-                yield return url;
-            }
-        }
-
-        private HashSet<string> GetAllFeeds(List<FileInfo> allFiles)
-        {
-            var nugetConfigs = GetAllNugetConfigs(allFiles);
-            var feeds = nugetConfigs
-                .SelectMany(GetFeeds)
-                .Where(str => !string.IsNullOrWhiteSpace(str))
-                .ToHashSet();
-
-            if (feeds.Count > 0)
-            {
-                logger.LogInfo($"Found {feeds.Count} Nuget feeds in nuget.config files: {string.Join(", ", feeds.OrderBy(f => f))}");
-            }
-            else
-            {
-                logger.LogDebug("No Nuget feeds found in nuget.config files.");
-            }
-            return feeds;
-        }
-
-        [GeneratedRegex(@"<TargetFramework>.*</TargetFramework>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
-        private static partial Regex TargetFramework();
-
-        [GeneratedRegex(@"^(.+)\.(\d+\.\d+\.\d+(-(.+))?)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
-        private static partial Regex LegacyNugetPackage();
-
-        [GeneratedRegex(@"^E\s(.*)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
-        private static partial Regex EnabledNugetFeed();
-    }
-}

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -20,7 +20,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
     {
         private readonly AssemblyCache assemblyCache;
         private readonly ILogger logger;
-        private readonly IDiagnosticsWriter diagnosticsWriter;
 
         // Only used as a set, but ConcurrentDictionary is the only concurrent set in .NET.
         private readonly IDictionary<string, bool> usedReferences = new ConcurrentDictionary<string, bool>();
@@ -53,25 +52,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             var startTime = DateTime.Now;
 
             this.logger = logger;
-
-            var diagDirEnv = Environment.GetEnvironmentVariable(EnvironmentVariableNames.DiagnosticDir);
-            if (!string.IsNullOrWhiteSpace(diagDirEnv) &&
-                !Directory.Exists(diagDirEnv))
-            {
-                try
-                {
-                    Directory.CreateDirectory(diagDirEnv);
-                }
-                catch (Exception e)
-                {
-                    logger.LogError($"Failed to create diagnostic directory {diagDirEnv}: {e.Message}");
-                    diagDirEnv = null;
-                }
-            }
-
-            this.diagnosticsWriter = new DiagnosticsStream(Path.Combine(
-                diagDirEnv ?? "",
-                $"dependency-manager-{DateTime.UtcNow:yyyyMMddHHmm}-{Environment.ProcessId}.jsonc"));
             this.sourceDir = new DirectoryInfo(srcDir);
 
             packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName, "packages"));
@@ -197,7 +177,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             var frameworkLocations = new HashSet<string>();
 
             var frameworkReferences = Environment.GetEnvironmentVariable(EnvironmentVariableNames.DotnetFrameworkReferences);
-            var useSubfolders = EnvironmentVariables.GetBoolean(EnvironmentVariableNames.DotnetFrameworkReferencesUseSubfolders);
+            var frameworkReferencesUseSubfolders = Environment.GetEnvironmentVariable(EnvironmentVariableNames.DotnetFrameworkReferencesUseSubfolders);
+            _ = bool.TryParse(frameworkReferencesUseSubfolders, out var useSubfolders);
             if (!string.IsNullOrWhiteSpace(frameworkReferences))
             {
                 RemoveFrameworkNugetPackages(dllPaths);
@@ -249,6 +230,73 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             return frameworkLocations;
         }
 
+        private void RestoreNugetPackages(List<FileInfo> allNonBinaryFiles, IEnumerable<string> allProjects, IEnumerable<string> allSolutions, HashSet<string> dllPaths)
+        {
+            try
+            {
+                using (var nuget = new NugetPackages(sourceDir.FullName, legacyPackageDirectory, logger))
+                {
+                    var count = nuget.InstallPackages();
+
+                    if (nuget.PackageCount > 0)
+                    {
+                        CompilationInfos.Add(("packages.config files", nuget.PackageCount.ToString()));
+                        CompilationInfos.Add(("Successfully restored packages.config files", count.ToString()));
+                    }
+                }
+
+                var nugetPackageDlls = legacyPackageDirectory.DirInfo.GetFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = true });
+                var nugetPackageDllPaths = nugetPackageDlls.Select(f => f.FullName).ToHashSet();
+                var excludedPaths = nugetPackageDllPaths
+                    .Where(path => IsPathInSubfolder(path, legacyPackageDirectory.DirInfo.FullName, "tools"))
+                    .ToList();
+
+                if (nugetPackageDllPaths.Count > 0)
+                {
+                    logger.LogInfo($"Restored {nugetPackageDllPaths.Count} Nuget DLLs.");
+                }
+                if (excludedPaths.Count > 0)
+                {
+                    logger.LogInfo($"Excluding {excludedPaths.Count} Nuget DLLs.");
+                }
+
+                foreach (var excludedPath in excludedPaths)
+                {
+                    logger.LogInfo($"Excluded Nuget DLL: {excludedPath}");
+                }
+
+                nugetPackageDllPaths.ExceptWith(excludedPaths);
+                dllPaths.UnionWith(nugetPackageDllPaths);
+            }
+            catch (Exception exc)
+            {
+                logger.LogError($"Failed to restore Nuget packages with nuget.exe: {exc.Message}");
+            }
+
+            var restoredProjects = RestoreSolutions(allSolutions, out var assets1);
+            var projects = allProjects.Except(restoredProjects);
+            RestoreProjects(projects, out var assets2);
+
+            var dependencies = Assets.GetCompilationDependencies(logger, assets1.Union(assets2));
+
+            var paths = dependencies
+                .Paths
+                .Select(d => Path.Combine(packageDirectory.DirInfo.FullName, d))
+                .ToList();
+            dllPaths.UnionWith(paths);
+
+            LogAllUnusedPackages(dependencies);
+            DownloadMissingPackages(allNonBinaryFiles, dllPaths);
+        }
+
+        private static bool IsPathInSubfolder(string path, string rootFolder, string subFolder)
+        {
+            return path.IndexOf(
+                $"{Path.DirectorySeparatorChar}{subFolder}{Path.DirectorySeparatorChar}",
+                rootFolder.Length,
+                StringComparison.InvariantCultureIgnoreCase) >= 0;
+        }
+
         private void RemoveNugetAnalyzerReferences()
         {
             var packageFolder = packageDirectory.DirInfo.FullName.ToLowerInvariant();
@@ -435,6 +483,27 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                 .FullName;
         }
 
+        private ICollection<string> GetAllPackageDirectories()
+        {
+            return new DirectoryInfo(packageDirectory.DirInfo.FullName)
+                .EnumerateDirectories("*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
+                .Select(d => d.Name)
+                .ToList();
+        }
+
+        private void LogAllUnusedPackages(DependencyContainer dependencies)
+        {
+            var allPackageDirectories = GetAllPackageDirectories();
+
+            logger.LogInfo($"Restored {allPackageDirectories.Count} packages");
+            logger.LogInfo($"Found {dependencies.Packages.Count} packages in project.assets.json files");
+
+            allPackageDirectories
+                .Where(package => !dependencies.Packages.Contains(package))
+                .Order()
+                .ForEach(package => logger.LogInfo($"Unused package: {package}"));
+        }
+
         private void GenerateSourceFileFromImplicitUsings()
         {
             var usings = new HashSet<string>();
@@ -738,6 +807,269 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             }
         }
 
+        /// <summary>
+        /// Executes `dotnet restore` on all solution files in solutions.
+        /// As opposed to RestoreProjects this is not run in parallel using PLINQ
+        /// as `dotnet restore` on a solution already uses multiple threads for restoring
+        /// the projects (this can be disabled with the `--disable-parallel` flag).
+        /// Populates assets with the relative paths to the assets files generated by the restore.
+        /// Returns a list of projects that are up to date with respect to restore.
+        /// </summary>
+        /// <param name="solutions">A list of paths to solution files.</param>
+        private IEnumerable<string> RestoreSolutions(IEnumerable<string> solutions, out IEnumerable<string> assets)
+        {
+            var successCount = 0;
+            var nugetSourceFailures = 0;
+            var assetFiles = new List<string>();
+            var projects = solutions.SelectMany(solution =>
+                {
+                    logger.LogInfo($"Restoring solution {solution}...");
+                    var res = dotnet.Restore(new(solution, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
+                    if (res.Success)
+                    {
+                        successCount++;
+                    }
+                    if (res.HasNugetPackageSourceError)
+                    {
+                        nugetSourceFailures++;
+                    }
+                    assetFiles.AddRange(res.AssetsFilePaths);
+                    return res.RestoredProjects;
+                }).ToList();
+            assets = assetFiles;
+            CompilationInfos.Add(("Successfully restored solution files", successCount.ToString()));
+            CompilationInfos.Add(("Failed solution restore with package source error", nugetSourceFailures.ToString()));
+            CompilationInfos.Add(("Restored projects through solution files", projects.Count.ToString()));
+            return projects;
+        }
+
+        /// <summary>
+        /// Executes `dotnet restore` on all projects in projects.
+        /// This is done in parallel for performance reasons.
+        /// Populates assets with the relative paths to the assets files generated by the restore.
+        /// </summary>
+        /// <param name="projects">A list of paths to project files.</param>
+        private void RestoreProjects(IEnumerable<string> projects, out IEnumerable<string> assets)
+        {
+            var successCount = 0;
+            var nugetSourceFailures = 0;
+            var assetFiles = new List<string>();
+            var sync = new object();
+            Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = threads }, project =>
+            {
+                logger.LogInfo($"Restoring project {project}...");
+                var res = dotnet.Restore(new(project, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
+                lock (sync)
+                {
+                    if (res.Success)
+                    {
+                        successCount++;
+                    }
+                    if (res.HasNugetPackageSourceError)
+                    {
+                        nugetSourceFailures++;
+                    }
+                    assetFiles.AddRange(res.AssetsFilePaths);
+                }
+            });
+            assets = assetFiles;
+            CompilationInfos.Add(("Successfully restored project files", successCount.ToString()));
+            CompilationInfos.Add(("Failed project restore with package source error", nugetSourceFailures.ToString()));
+        }
+
+        [GeneratedRegex(@"^(.+)\.(\d+\.\d+\.\d+(-(.+))?)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
+        private static partial Regex LegacyNugetPackage();
+
+
+        private static IEnumerable<string> GetRestoredPackageDirectoryNames(DirectoryInfo root)
+        {
+            return Directory.GetDirectories(root.FullName)
+                .Select(d => Path.GetFileName(d).ToLowerInvariant());
+        }
+
+        private IEnumerable<string> GetRestoredLegacyPackageNames()
+        {
+            var oldPackageDirectories = GetRestoredPackageDirectoryNames(legacyPackageDirectory.DirInfo);
+            foreach (var oldPackageDirectory in oldPackageDirectories)
+            {
+                // nuget install restores packages to 'packagename.version' folders (dotnet restore to 'packagename/version' folders)
+                // typical folder names look like:
+                //   newtonsoft.json.13.0.3
+                // there are more complex ones too, such as:
+                //   runtime.tizen.4.0.0-armel.Microsoft.NETCore.DotNetHostResolver.2.0.0-preview2-25407-01
+
+                var match = LegacyNugetPackage().Match(oldPackageDirectory);
+                if (!match.Success)
+                {
+                    logger.LogWarning($"Package directory '{oldPackageDirectory}' doesn't match the expected pattern.");
+                    continue;
+                }
+
+                yield return match.Groups[1].Value.ToLowerInvariant();
+            }
+        }
+
+        private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPaths)
+        {
+            var alreadyDownloadedPackages = GetRestoredPackageDirectoryNames(packageDirectory.DirInfo);
+            var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames();
+
+            var notYetDownloadedPackages = new HashSet<PackageReference>(fileContent.AllPackages);
+            foreach (var alreadyDownloadedPackage in alreadyDownloadedPackages)
+            {
+                notYetDownloadedPackages.Remove(new(alreadyDownloadedPackage, PackageReferenceSource.SdkCsProj));
+            }
+            foreach (var alreadyDownloadedLegacyPackage in alreadyDownloadedLegacyPackages)
+            {
+                notYetDownloadedPackages.Remove(new(alreadyDownloadedLegacyPackage, PackageReferenceSource.PackagesConfig));
+            }
+
+            if (notYetDownloadedPackages.Count == 0)
+            {
+                return;
+            }
+
+            var multipleVersions = notYetDownloadedPackages
+                .GroupBy(p => p.Name)
+                .Where(g => g.Count() > 1)
+                .Select(g => g.Key)
+                .ToList();
+
+            foreach (var package in multipleVersions)
+            {
+                logger.LogWarning($"Found multiple not yet restored packages with name '{package}'.");
+                notYetDownloadedPackages.Remove(new(package, PackageReferenceSource.PackagesConfig));
+            }
+
+            logger.LogInfo($"Found {notYetDownloadedPackages.Count} packages that are not yet restored");
+
+            var nugetConfigs = allFiles.SelectFileNamesByName("nuget.config").ToArray();
+            string? nugetConfig = null;
+            if (nugetConfigs.Length > 1)
+            {
+                logger.LogInfo($"Found multiple nuget.config files: {string.Join(", ", nugetConfigs)}.");
+                nugetConfig = allFiles
+                    .SelectRootFiles(sourceDir)
+                    .SelectFileNamesByName("nuget.config")
+                    .FirstOrDefault();
+                if (nugetConfig == null)
+                {
+                    logger.LogInfo("Could not find a top-level nuget.config file.");
+                }
+            }
+            else
+            {
+                nugetConfig = nugetConfigs.FirstOrDefault();
+            }
+
+            if (nugetConfig != null)
+            {
+                logger.LogInfo($"Using nuget.config file {nugetConfig}.");
+            }
+
+            CompilationInfos.Add(("Fallback nuget restore", notYetDownloadedPackages.Count.ToString()));
+
+            var successCount = 0;
+            var sync = new object();
+
+            Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = threads }, package =>
+            {
+                var success = TryRestorePackageManually(package.Name, nugetConfig, package.PackageReferenceSource);
+                if (!success)
+                {
+                    return;
+                }
+
+                lock (sync)
+                {
+                    successCount++;
+                }
+            });
+
+            CompilationInfos.Add(("Successfully ran fallback nuget restore", successCount.ToString()));
+
+            dllPaths.Add(missingPackageDirectory.DirInfo.FullName);
+        }
+
+        [GeneratedRegex(@"<TargetFramework>.*</TargetFramework>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
+        private static partial Regex TargetFramework();
+
+        private bool TryRestorePackageManually(string package, string? nugetConfig, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj)
+        {
+            logger.LogInfo($"Restoring package {package}...");
+            using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package, "missingpackages_workingdir"));
+            var success = dotnet.New(tempDir.DirInfo.FullName);
+            if (!success)
+            {
+                return false;
+            }
+
+            if (packageReferenceSource == PackageReferenceSource.PackagesConfig)
+            {
+                TryChangeTargetFrameworkMoniker(tempDir.DirInfo);
+            }
+
+            success = dotnet.AddPackage(tempDir.DirInfo.FullName, package);
+            if (!success)
+            {
+                return false;
+            }
+
+            var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig));
+            if (!res.Success)
+            {
+                if (res.HasNugetPackageSourceError && nugetConfig is not null)
+                {
+                    // Restore could not be completed because the listed source is unavailable. Try without the nuget.config:
+                    res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: null, ForceReevaluation: true));
+                }
+
+                // TODO: the restore might fail, we could retry with
+                // - a prerelease (*-* instead of *) version of the package,
+                // - a different target framework moniker.
+
+                if (!res.Success)
+                {
+                    logger.LogInfo($"Failed to restore nuget package {package}");
+                    return false;
+                }
+            }
+
+            return true;
+        }
+
+        private void TryChangeTargetFrameworkMoniker(DirectoryInfo tempDir)
+        {
+            try
+            {
+                logger.LogInfo($"Changing the target framework moniker in {tempDir.FullName}...");
+
+                var csprojs = tempDir.GetFiles("*.csproj", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive });
+                if (csprojs.Length != 1)
+                {
+                    logger.LogError($"Could not find the .csproj file in {tempDir.FullName}, count = {csprojs.Length}");
+                    return;
+                }
+
+                var csproj = csprojs[0];
+                var content = File.ReadAllText(csproj.FullName);
+                var matches = TargetFramework().Matches(content);
+                if (matches.Count == 0)
+                {
+                    logger.LogError($"Could not find target framework in {csproj.FullName}");
+                }
+                else
+                {
+                    content = TargetFramework().Replace(content, $"<TargetFramework>{FrameworkPackageNames.LatestNetFrameworkMoniker}</TargetFramework>", 1);
+                    File.WriteAllText(csproj.FullName, content);
+                }
+            }
+            catch (Exception exc)
+            {
+                logger.LogError($"Failed to update target framework in {tempDir.FullName}: {exc}");
+            }
+        }
+
         public void Dispose(TemporaryDirectory? dir, string name)
         {
             try
@@ -759,8 +1091,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             {
                 Dispose(tempWorkingDirectory, "temporary working");
             }
-
-            diagnosticsWriter?.Dispose();
         }
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs

@@ -16,14 +16,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
     public partial class DotNet : IDotNet
     {
         private readonly IDotNetCliInvoker dotnetCliInvoker;
-        private readonly ILogger logger;
         private readonly TemporaryDirectory? tempWorkingDirectory;
 
         private DotNet(IDotNetCliInvoker dotnetCliInvoker, ILogger logger, TemporaryDirectory? tempWorkingDirectory = null)
         {
             this.tempWorkingDirectory = tempWorkingDirectory;
             this.dotnetCliInvoker = dotnetCliInvoker;
-            this.logger = logger;
             Info();
         }
 
@@ -91,18 +89,17 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             return dotnetCliInvoker.RunCommand(args);
         }
 
-        public IList<string> GetListedRuntimes() => GetResultList("--list-runtimes");
+        public IList<string> GetListedRuntimes() => GetListed("--list-runtimes", "runtime");
 
-        public IList<string> GetListedSdks() => GetResultList("--list-sdks");
+        public IList<string> GetListedSdks() => GetListed("--list-sdks", "SDK");
 
-        private IList<string> GetResultList(string args)
+        private IList<string> GetListed(string args, string artifact)
         {
-            if (dotnetCliInvoker.RunCommand(args, out var results))
+            if (dotnetCliInvoker.RunCommand(args, out var artifacts))
             {
-                return results;
+                return artifacts;
             }
-            logger.LogWarning($"Running 'dotnet {args}' failed.");
-            return [];
+            return new List<string>();
         }
 
         public bool Exec(string execArgs)
@@ -111,8 +108,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             return dotnetCliInvoker.RunCommand(args);
         }
 
-        public IList<string> GetNugetFeeds(string nugetConfig) => GetResultList($"nuget list source --format Short --configfile \"{nugetConfig}\"");
-
         // The version number should be kept in sync with the version .NET version used for building the application.
         public const string LatestDotNetSdkVersion = "8.0.101";
 

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs

@@ -16,30 +16,5 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         /// Controls whether to use framework dependencies from subfolders.
         /// </summary>
         public const string DotnetFrameworkReferencesUseSubfolders = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_DOTNET_FRAMEWORK_REFERENCES_USE_SUBFOLDERS";
-
-        /// <summary>
-        /// Controls whether to check the responsiveness of NuGet feeds.
-        /// </summary>
-        public const string CheckNugetFeedResponsiveness = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK";
-
-        /// <summary>
-        /// Specifies the NuGet feeds to exclude from the responsiveness check. The value is a space-separated list of feed URLs.
-        /// </summary>
-        public const string ExcludedNugetFeedsFromResponsivenessCheck = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_EXCLUDED";
-
-        /// <summary>
-        /// Specifies the timeout (as an integer) in milliseconds for the initial check of NuGet feeds responsiveness. The value is then doubled for each subsequent check.
-        /// </summary>
-        public const string NugetFeedResponsivenessInitialTimeout = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_TIMEOUT";
-
-        /// <summary>
-        /// Specifies how many requests to make to the NuGet feed to check its responsiveness.
-        /// </summary>
-        public const string NugetFeedResponsivenessRequestCount = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_LIMIT";
-
-        /// <summary>
-        /// Specifies the location of the diagnostic directory.
-        /// </summary>
-        public const string DiagnosticDir = "CODEQL_EXTRACTOR_CSHARP_DIAGNOSTIC_DIR";
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs

@@ -13,7 +13,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         IList<string> GetListedRuntimes();
         IList<string> GetListedSdks();
         bool Exec(string execArgs);
-        IList<string> GetNugetFeeds(string nugetConfig);
     }
 
     public record class RestoreSettings(string File, string PackageDirectory, bool ForceDotnetRefAssemblyFetching, string? PathToNugetConfig = null, bool ForceReevaluation = false);

csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs

@@ -143,7 +143,7 @@ namespace Semmle.Extraction.CSharp.Standalone
             stopwatch.Start();
 
             using var logger = new ConsoleLogger(options.Verbosity, logThreadId: true);
-            logger.Log(Severity.Info, "Extracting C# with build-mode set to 'none'");
+            logger.Log(Severity.Info, "Extracting C# in buildless mode");
             using var dependencyManager = new DependencyManager(options.SrcDir, logger);
 
             if (!dependencyManager.NonGeneratedSourcesFiles.Any())

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expression.cs

@@ -100,14 +100,8 @@ namespace Semmle.Extraction.CSharp.Entities
         /// <param name="child">The child index.</param>
         /// <param name="type">A type hint.</param>
         /// <returns>The new expression.</returns>
-        public static Expression Create(Context cx, ExpressionSyntax node, IExpressionParentEntity parent, int child, Boolean isCompilerGenerated = false)
-        {
-            var info = new ExpressionNodeInfo(cx, node, parent, child)
-            {
-                IsCompilerGenerated = isCompilerGenerated
-            };
-            return CreateFromNode(info);
-        }
+        public static Expression Create(Context cx, ExpressionSyntax node, IExpressionParentEntity parent, int child) =>
+            CreateFromNode(new ExpressionNodeInfo(cx, node, parent, child));
 
         public static Expression CreateFromNode(ExpressionNodeInfo info) => Expressions.ImplicitCast.Create(info);
 

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ArrayCreation.cs

@@ -97,7 +97,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 ExprKind.ARRAY_CREATION,
                 parent,
                 childIndex,
-                isCompilerGenerated: true,
+                true,
                 null);
 
             var arrayCreation = new Expression(info);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Assignment.cs

@@ -26,10 +26,10 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
             if (operatorKind.HasValue)
             {
                 // Convert assignment such as `a += b` into `a = a + b`.
-                var simpleAssignExpr = new Expression(new ExpressionInfo(Context, Type, Location, ExprKind.SIMPLE_ASSIGN, this, 2, isCompilerGenerated: true, null));
+                var simpleAssignExpr = new Expression(new ExpressionInfo(Context, Type, Location, ExprKind.SIMPLE_ASSIGN, this, 2, false, null));
                 Create(Context, Syntax.Left, simpleAssignExpr, 1);
-                var opexpr = new Expression(new ExpressionInfo(Context, Type, Location, operatorKind.Value, simpleAssignExpr, 0, isCompilerGenerated: true, null));
-                Create(Context, Syntax.Left, opexpr, 0, isCompilerGenerated: true);
+                var opexpr = new Expression(new ExpressionInfo(Context, Type, Location, operatorKind.Value, simpleAssignExpr, 0, false, null));
+                Create(Context, Syntax.Left, opexpr, 0);
                 Create(Context, Syntax.Right, opexpr, 1);
                 opexpr.OperatorCall(trapFile, Syntax);
             }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Cast.cs

@@ -41,7 +41,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 ExprKind.CAST,
                 parent,
                 childIndex,
-                isCompilerGenerated: true,
+                true,
                 ValueAsString(value));
 
             var ret = new Expression(info);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Collections/Spread.cs

@@ -6,7 +6,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
     internal class Spread : Expression
     {
         public Spread(Context cx, SpreadElementSyntax syntax, IExpressionParentEntity parent, int child) :
-            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), ExprKind.SPREAD_ELEMENT, parent, child, isCompilerGenerated: false, null))
+            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), ExprKind.SPREAD_ELEMENT, parent, child, false, null))
         {
             Create(cx, syntax.Expression, this, 0);
         }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Default.cs

@@ -24,7 +24,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 ExprKind.DEFAULT,
                 parent,
                 childIndex,
-                isCompilerGenerated: true,
+                true,
                 value);
 
             return new Expression(info);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Discard.cs

@@ -11,7 +11,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
         }
 
         private Discard(Context cx, CSharpSyntaxNode syntax, IExpressionParentEntity parent, int child) :
-            base(new ExpressionInfo(cx, cx.GetType(syntax), cx.CreateLocation(syntax.GetLocation()), ExprKind.DISCARD, parent, child, isCompilerGenerated: false, null))
+            base(new ExpressionInfo(cx, cx.GetType(syntax), cx.CreateLocation(syntax.GetLocation()), ExprKind.DISCARD, parent, child, false, null))
         {
         }
 

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ElementAccess.cs

@@ -22,7 +22,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
             if (Kind == ExprKind.POINTER_INDIRECTION)
             {
                 var qualifierInfo = new ExpressionNodeInfo(Context, qualifier, this, 0);
-                var add = new Expression(new ExpressionInfo(Context, qualifierInfo.Type, Location, ExprKind.ADD, this, 0, isCompilerGenerated: false, null));
+                var add = new Expression(new ExpressionInfo(Context, qualifierInfo.Type, Location, ExprKind.ADD, this, 0, false, null));
                 qualifierInfo.SetParent(add, 0);
                 CreateFromNode(qualifierInfo);
                 PopulateArguments(trapFile, argumentList, 1);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ImplicitCast.cs

@@ -14,13 +14,13 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
         }
 
         private ImplicitCast(ExpressionNodeInfo info)
-            : base(new ExpressionInfo(info.Context, info.ConvertedType, info.Location, ExprKind.CAST, info.Parent, info.Child, isCompilerGenerated: true, info.ExprValue))
+            : base(new ExpressionInfo(info.Context, info.ConvertedType, info.Location, ExprKind.CAST, info.Parent, info.Child, true, info.ExprValue))
         {
             Expr = Factory.Create(new ExpressionNodeInfo(Context, info.Node, this, 0));
         }
 
         private ImplicitCast(ExpressionNodeInfo info, IMethodSymbol method)
-            : base(new ExpressionInfo(info.Context, info.ConvertedType, info.Location, ExprKind.OPERATOR_INVOCATION, info.Parent, info.Child, isCompilerGenerated: true, info.ExprValue))
+            : base(new ExpressionInfo(info.Context, info.ConvertedType, info.Location, ExprKind.OPERATOR_INVOCATION, info.Parent, info.Child, true, info.ExprValue))
         {
             Expr = Factory.Create(info.SetParent(this, 0));
 
@@ -65,7 +65,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                     kind,
                     parent,
                     childIndex,
-                    isCompilerGenerated: true,
+                    true,
                     v);
 
             var method = GetImplicitConversionMethod(type, value);
@@ -93,7 +93,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                     ExprKind.CAST,
                     parent,
                     childIndex,
-                    isCompilerGenerated: true,
+                    true,
                     ValueAsString(value));
 
             return new Expression(info);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Initializer.cs

@@ -45,7 +45,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 ExprKind.ARRAY_INIT,
                 parent,
                 index,
-                isCompilerGenerated: true,
+                true,
                 null);
 
             return new Expression(info);
@@ -132,7 +132,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 var addMethod = Method.Create(Context, collectionInfo.Symbol as IMethodSymbol);
                 var voidType = AnnotatedTypeSymbol.CreateNotAnnotated(Context.Compilation.GetSpecialType(SpecialType.System_Void));
 
-                var invocation = new Expression(new ExpressionInfo(Context, voidType, Context.CreateLocation(i.GetLocation()), ExprKind.METHOD_INVOCATION, this, child++, isCompilerGenerated: true, null));
+                var invocation = new Expression(new ExpressionInfo(Context, voidType, Context.CreateLocation(i.GetLocation()), ExprKind.METHOD_INVOCATION, this, child++, false, null));
 
                 if (addMethod is not null)
                     trapFile.expr_call(invocation, addMethod);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/InterpolatedString.cs

@@ -25,7 +25,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                     case SyntaxKind.InterpolatedStringText:
                         // Create a string literal
                         var interpolatedText = (InterpolatedStringTextSyntax)c;
-                        new Expression(new ExpressionInfo(Context, Type, Context.CreateLocation(c.GetLocation()), ExprKind.UTF16_STRING_LITERAL, this, child++, isCompilerGenerated: false, interpolatedText.TextToken.ValueText));
+                        new Expression(new ExpressionInfo(Context, Type, Context.CreateLocation(c.GetLocation()), ExprKind.UTF16_STRING_LITERAL, this, child++, false, interpolatedText.TextToken.ValueText));
                         break;
                     default:
                         throw new InternalError(c, $"Unhandled interpolation kind {c.Kind()}");

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Lambda.cs

@@ -63,7 +63,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
         public static Lambda Create(ExpressionNodeInfo info, ParenthesizedLambdaExpressionSyntax node) => new Lambda(info, node);
 
         private Lambda(ExpressionNodeInfo info, SimpleLambdaExpressionSyntax node)
-            : this(info.SetKind(ExprKind.LAMBDA), node.Body, [node.Parameter], null) { }
+            : this(info.SetKind(ExprKind.LAMBDA), node.Body, Enumerators.Singleton(node.Parameter), null) { }
 
         public static Lambda Create(ExpressionNodeInfo info, SimpleLambdaExpressionSyntax node) => new Lambda(info, node);
 

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Literal.cs

@@ -97,7 +97,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 kind,
                 parent,
                 childIndex,
-                isCompilerGenerated: true,
+                true,
                 ValueAsString(value));
 
             return new Expression(info);
@@ -112,7 +112,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 ExprKind.NULL_LITERAL,
                 parent,
                 childIndex,
-                isCompilerGenerated: true,
+                true,
                 ValueAsString(null));
 
             return new Expression(info);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ObjectCreation/AnonymousObjectCreation.cs

@@ -30,7 +30,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 return;
             }
 
-            var objectInitializer = new Expression(new ExpressionInfo(Context, Type, Location, ExprKind.OBJECT_INIT, this, -1, isCompilerGenerated: false, null));
+            var objectInitializer = new Expression(new ExpressionInfo(Context, Type, Location, ExprKind.OBJECT_INIT, this, -1, false, null));
 
             foreach (var init in Syntax.Initializers)
             {
@@ -40,11 +40,11 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 var type = property.GetAnnotatedType();
                 var loc = Context.CreateLocation(init.GetLocation());
 
-                var assignment = new Expression(new ExpressionInfo(Context, type, loc, ExprKind.SIMPLE_ASSIGN, objectInitializer, child++, isCompilerGenerated: false, null));
+                var assignment = new Expression(new ExpressionInfo(Context, type, loc, ExprKind.SIMPLE_ASSIGN, objectInitializer, child++, false, null));
                 Create(Context, init.Expression, assignment, 0);
                 Property.Create(Context, property);
 
-                var access = new Expression(new ExpressionInfo(Context, type, loc, ExprKind.PROPERTY_ACCESS, assignment, 1, isCompilerGenerated: false, null));
+                var access = new Expression(new ExpressionInfo(Context, type, loc, ExprKind.PROPERTY_ACCESS, assignment, 1, false, null));
                 trapFile.expr_access(access, propEntity);
             }
         }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ObjectCreation/DateTimeObjectCreation.cs

@@ -59,7 +59,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 ExprKind.OBJECT_CREATION,
                 parent,
                 childIndex,
-                isCompilerGenerated: true,
+                true,
                 null));
 
             var longTypeSymbol = constructorSymbol.Parameters[0].Type;

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Patterns/BinaryPattern.cs

@@ -8,7 +8,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
     internal class BinaryPattern : Expression
     {
         public BinaryPattern(Context cx, BinaryPatternSyntax syntax, IExpressionParentEntity parent, int child) :
-            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), GetKind(syntax.OperatorToken, syntax), parent, child, isCompilerGenerated: false, null))
+            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), GetKind(syntax.OperatorToken, syntax), parent, child, false, null))
         {
             Pattern.Create(cx, syntax.Left, this, 0);
             Pattern.Create(cx, syntax.Right, this, 1);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Patterns/ListPattern.cs

@@ -7,7 +7,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
     internal class ListPattern : Expression
     {
         internal ListPattern(Context cx, ListPatternSyntax syntax, IExpressionParentEntity parent, int child) :
-            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), ExprKind.LIST_PATTERN, parent, child, isCompilerGenerated: false, null))
+            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), ExprKind.LIST_PATTERN, parent, child, false, null))
         {
             syntax.Patterns.ForEach((p, i) => Pattern.Create(cx, p, this, i));
         }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Patterns/PositionalPattern.cs

@@ -7,7 +7,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
     internal class PositionalPattern : Expression
     {
         internal PositionalPattern(Context cx, PositionalPatternClauseSyntax posPc, IExpressionParentEntity parent, int child) :
-            base(new ExpressionInfo(cx, null, cx.CreateLocation(posPc.GetLocation()), ExprKind.POSITIONAL_PATTERN, parent, child, isCompilerGenerated: false, null))
+            base(new ExpressionInfo(cx, null, cx.CreateLocation(posPc.GetLocation()), ExprKind.POSITIONAL_PATTERN, parent, child, false, null))
         {
             posPc.Subpatterns.ForEach((p, i) => Pattern.Create(cx, p.Pattern, this, i));
         }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Patterns/PropertyPattern.cs

@@ -7,7 +7,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
     internal class PropertyPattern : Expression
     {
         internal PropertyPattern(Context cx, PropertyPatternClauseSyntax pp, IExpressionParentEntity parent, int child) :
-            base(new ExpressionInfo(cx, null, cx.CreateLocation(pp.GetLocation()), ExprKind.PROPERTY_PATTERN, parent, child, isCompilerGenerated: false, null))
+            base(new ExpressionInfo(cx, null, cx.CreateLocation(pp.GetLocation()), ExprKind.PROPERTY_PATTERN, parent, child, false, null))
         {
             child = 0;
             foreach (var sub in pp.Subpatterns)
@@ -56,7 +56,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
             };
 
         private static Expression CreateSyntheticExp(Context cx, Microsoft.CodeAnalysis.Location location, IExpressionParentEntity parent, int child) =>
-            new Expression(new ExpressionInfo(cx, null, cx.CreateLocation(location), ExprKind.PROPERTY_PATTERN, parent, child, isCompilerGenerated: false, null));
+            new Expression(new ExpressionInfo(cx, null, cx.CreateLocation(location), ExprKind.PROPERTY_PATTERN, parent, child, false, null));
 
         private static void MakeExpressions(Context cx, IExpressionParentEntity parent, SubpatternSyntax syntax, int child)
         {

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Patterns/RecursivePattern.cs

@@ -15,7 +15,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
         /// <param name="parent">The parent pattern/expression.</param>
         /// <param name="child">The child index of this pattern.</param>
         public RecursivePattern(Context cx, RecursivePatternSyntax syntax, IExpressionParentEntity parent, int child) :
-            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), ExprKind.RECURSIVE_PATTERN, parent, child, isCompilerGenerated: false, null))
+            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), ExprKind.RECURSIVE_PATTERN, parent, child, false, null))
         {
             // Extract the type access
             if (syntax.Type is TypeSyntax t)

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Patterns/RelationalPattern.cs

@@ -8,7 +8,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
     internal class RelationalPattern : Expression
     {
         public RelationalPattern(Context cx, RelationalPatternSyntax syntax, IExpressionParentEntity parent, int child) :
-             base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), GetKind(syntax.OperatorToken), parent, child, isCompilerGenerated: false, null))
+             base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), GetKind(syntax.OperatorToken), parent, child, false, null))
         {
             Expression.Create(cx, syntax.Expression, this, 0);
         }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Patterns/SlicePattern.cs

@@ -6,7 +6,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
     internal class SlicePattern : Expression
     {
         public SlicePattern(Context cx, SlicePatternSyntax syntax, IExpressionParentEntity parent, int child) :
-            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), ExprKind.SLICE_PATTERN, parent, child, isCompilerGenerated: false, null))
+            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), ExprKind.SLICE_PATTERN, parent, child, false, null))
         {
             if (syntax.Pattern is not null)
             {

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Patterns/UnaryPattern.cs

@@ -6,7 +6,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
     internal class UnaryPattern : Expression
     {
         public UnaryPattern(Context cx, UnaryPatternSyntax syntax, IExpressionParentEntity parent, int child) :
-            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), ExprKind.NOT_PATTERN, parent, child, isCompilerGenerated: false, null))
+            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), ExprKind.NOT_PATTERN, parent, child, false, null))
         {
             Pattern.Create(cx, syntax.Pattern, this, 0);
         }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Query.cs

@@ -23,7 +23,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
             public QueryCall(Context cx, IMethodSymbol? method, SyntaxNode clause, IExpressionParentEntity parent, int child)
                 : base(new ExpressionInfo(cx, method?.GetAnnotatedReturnType(),
                         cx.CreateLocation(clause.GetLocation()),
-                        ExprKind.METHOD_INVOCATION, parent, child, isCompilerGenerated: false, null))
+                        ExprKind.METHOD_INVOCATION, parent, child, false, null))
             {
                 if (method is not null)
                     cx.TrapWriter.Writer.expr_call(this, Method.Create(cx, method));
@@ -97,7 +97,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 Expression.Create(cx, Expr, decl, 0);
 
                 var nameLoc = cx.CreateLocation(name.GetLocation());
-                var access = new Expression(new ExpressionInfo(cx, type, nameLoc, ExprKind.LOCAL_VARIABLE_ACCESS, decl, 1, isCompilerGenerated: false, null));
+                var access = new Expression(new ExpressionInfo(cx, type, nameLoc, ExprKind.LOCAL_VARIABLE_ACCESS, decl, 1, false, null));
                 cx.TrapWriter.Writer.expr_access(access, LocalVariable.Create(cx, variableSymbol));
 
                 return decl;

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Switch.cs

@@ -27,7 +27,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
         internal SwitchCase(Context cx, SwitchExpressionArmSyntax arm, Switch parent, int child) :
             base(new ExpressionInfo(
                 cx, cx.GetType(arm.Expression), cx.CreateLocation(arm.GetLocation()),
-                ExprKind.SWITCH_CASE, parent, child, isCompilerGenerated: false, null))
+                ExprKind.SWITCH_CASE, parent, child, false, null))
         {
             Expressions.Pattern.Create(cx, arm.Pattern, this, 0);
             if (arm.WhenClause is WhenClauseSyntax when)

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/This.cs

@@ -8,7 +8,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
         private This(IExpressionInfo info) : base(info) { }
 
         public static This CreateImplicit(Context cx, ITypeSymbol @class, Extraction.Entities.Location loc, IExpressionParentEntity parent, int child) =>
-            new This(new ExpressionInfo(cx, AnnotatedTypeSymbol.CreateNotAnnotated(@class), loc, Kinds.ExprKind.THIS_ACCESS, parent, child, isCompilerGenerated: true, null));
+            new This(new ExpressionInfo(cx, AnnotatedTypeSymbol.CreateNotAnnotated(@class), loc, Kinds.ExprKind.THIS_ACCESS, parent, child, true, null));
 
         public static This CreateExplicit(ExpressionNodeInfo info) => new This(info.SetKind(ExprKind.THIS_ACCESS));
     }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/TypeAccess.cs

@@ -44,7 +44,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 ExprKind.TYPE_ACCESS,
                 parent,
                 childIndex,
-                isCompilerGenerated: true,
+                true,
                 null);
 
             return new Expression(typeAccessInfo);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/TypeOf.cs

@@ -26,7 +26,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 ExprKind.TYPEOF,
                 parent,
                 childIndex,
-                isCompilerGenerated: true,
+                true,
                 null);
 
             var ret = new Expression(info);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/VariableDeclaration.cs

@@ -15,7 +15,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
 
         public static VariableDeclaration Create(Context cx, ISymbol symbol, AnnotatedTypeSymbol? type, TypeSyntax? optionalSyntax, Extraction.Entities.Location exprLocation, bool isVar, IExpressionParentEntity parent, int child)
         {
-            var ret = new VariableDeclaration(new ExpressionInfo(cx, type, exprLocation, ExprKind.LOCAL_VAR_DECL, parent, child, isCompilerGenerated: false, null));
+            var ret = new VariableDeclaration(new ExpressionInfo(cx, type, exprLocation, ExprKind.LOCAL_VAR_DECL, parent, child, false, null));
             cx.Try(null, null, () =>
             {
                 var l = LocalVariable.Create(cx, symbol);
@@ -52,7 +52,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
         public static Expression CreateParenthesized(Context cx, DeclarationExpressionSyntax node, ParenthesizedVariableDesignationSyntax designation, IExpressionParentEntity parent, int child, INamedTypeSymbol? t)
         {
             var type = t is null ? (AnnotatedTypeSymbol?)null : new AnnotatedTypeSymbol(t, t.NullableAnnotation);
-            var tuple = new Expression(new ExpressionInfo(cx, type, cx.CreateLocation(node.GetLocation()), ExprKind.TUPLE, parent, child, isCompilerGenerated: false, null));
+            var tuple = new Expression(new ExpressionInfo(cx, type, cx.CreateLocation(node.GetLocation()), ExprKind.TUPLE, parent, child, false, null));
 
             cx.Try(null, null, () =>
             {
@@ -68,7 +68,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
         public static Expression CreateParenthesized(Context cx, VarPatternSyntax varPattern, ParenthesizedVariableDesignationSyntax designation, IExpressionParentEntity parent, int child)
         {
             var tuple = new Expression(
-                new ExpressionInfo(cx, null, cx.CreateLocation(varPattern.GetLocation()), ExprKind.TUPLE, parent, child, isCompilerGenerated: false, null),
+                new ExpressionInfo(cx, null, cx.CreateLocation(varPattern.GetLocation()), ExprKind.TUPLE, parent, child, false, null),
                 shouldPopulate: false);
 
             var elementTypes = new List<ITypeSymbol?>();
@@ -148,7 +148,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
             Create(cx, node, node.Designation, parent, child, cx.GetTypeInfo(node).Type.DisambiguateType() as INamedTypeSymbol);
 
         public static VariableDeclaration Create(Context cx, CSharpSyntaxNode c, AnnotatedTypeSymbol? type, IExpressionParentEntity parent, int child) =>
-            new VariableDeclaration(new ExpressionInfo(cx, type, cx.CreateLocation(c.FixedLocation()), ExprKind.LOCAL_VAR_DECL, parent, child, isCompilerGenerated: false, null));
+            new VariableDeclaration(new ExpressionInfo(cx, type, cx.CreateLocation(c.FixedLocation()), ExprKind.LOCAL_VAR_DECL, parent, child, false, null));
 
         public static VariableDeclaration Create(Context cx, CatchDeclarationSyntax d, bool isVar, IExpressionParentEntity parent, int child)
         {
@@ -179,7 +179,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                     Create(cx, d.Initializer.Value, ret, 0);
 
                     // Create an access
-                    var access = new Expression(new ExpressionInfo(cx, type, localVar.Location, ExprKind.LOCAL_VARIABLE_ACCESS, ret, 1, isCompilerGenerated: false, null));
+                    var access = new Expression(new ExpressionInfo(cx, type, localVar.Location, ExprKind.LOCAL_VARIABLE_ACCESS, ret, 1, false, null));
                     cx.TrapWriter.Writer.expr_access(access, localVar);
                 }
 

csharp/extractor/Semmle.Extraction.CSharp/Entities/Field.cs

@@ -110,9 +110,9 @@ namespace Semmle.Extraction.CSharp.Entities
             string? constValue, ref int child)
         {
             var type = Symbol.GetAnnotatedType();
-            var simpleAssignExpr = new Expression(new ExpressionInfo(Context, type, loc, ExprKind.SIMPLE_ASSIGN, this, child++, isCompilerGenerated: true, constValue));
+            var simpleAssignExpr = new Expression(new ExpressionInfo(Context, type, loc, ExprKind.SIMPLE_ASSIGN, this, child++, false, constValue));
             Expression.CreateFromNode(new ExpressionNodeInfo(Context, initializer, simpleAssignExpr, 0));
-            var access = new Expression(new ExpressionInfo(Context, type, Location, ExprKind.FIELD_ACCESS, simpleAssignExpr, 1, isCompilerGenerated: true, constValue));
+            var access = new Expression(new ExpressionInfo(Context, type, Location, ExprKind.FIELD_ACCESS, simpleAssignExpr, 1, false, constValue));
             trapFile.expr_access(access, this);
             return access;
         }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Property.cs

@@ -86,9 +86,9 @@ namespace Semmle.Extraction.CSharp.Entities
                     {
                         var loc = Context.CreateLocation(initializer!.GetLocation());
                         var annotatedType = AnnotatedTypeSymbol.CreateNotAnnotated(Symbol.Type);
-                        var simpleAssignExpr = new Expression(new ExpressionInfo(Context, annotatedType, loc, ExprKind.SIMPLE_ASSIGN, this, child++, isCompilerGenerated: true, null));
+                        var simpleAssignExpr = new Expression(new ExpressionInfo(Context, annotatedType, loc, ExprKind.SIMPLE_ASSIGN, this, child++, false, null));
                         Expression.CreateFromNode(new ExpressionNodeInfo(Context, initializer.Value, simpleAssignExpr, 0));
-                        var access = new Expression(new ExpressionInfo(Context, annotatedType, Location, ExprKind.PROPERTY_ACCESS, simpleAssignExpr, 1, isCompilerGenerated: true, null));
+                        var access = new Expression(new ExpressionInfo(Context, annotatedType, Location, ExprKind.PROPERTY_ACCESS, simpleAssignExpr, 1, false, null));
                         trapFile.expr_access(access, this);
                         if (!Symbol.IsStatic)
                         {

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs

@@ -189,7 +189,7 @@ namespace Semmle.Extraction.CSharp
                 // compilation.Clone() is used to allow symbols to be garbage collected.
                 using var trapWriter = transformedSourcePath.CreateTrapWriter(Logger, options.TrapCompression, discardDuplicates: false);
 
-                upToDate = FileIsUpToDate(sourcePath, trapWriter.TrapFile);
+                upToDate = options.Fast && FileIsUpToDate(sourcePath, trapWriter.TrapFile);
 
                 var currentTaskId = IncrementTaskCount();
                 ReportProgressTaskStarted(currentTaskId, sourcePath);

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Options.cs

@@ -35,7 +35,15 @@ namespace Semmle.Extraction.CSharp
         public static Options CreateWithEnvironment(string[] arguments)
         {
             var options = new Options();
+            var extractionOptions = Environment.GetEnvironmentVariable("LGTM_INDEX_EXTRACTOR");
+
             var argsList = new List<string>(arguments);
+
+            if (!string.IsNullOrEmpty(extractionOptions))
+            {
+                argsList.AddRange(extractionOptions.Split(' '));
+            }
+
             options.ParseArguments(argsList);
             return options;
         }

csharp/extractor/Semmle.Extraction.Tests/Options.cs

@@ -12,6 +12,11 @@ namespace Semmle.Extraction.Tests
         private CSharp.Options? options;
         private CSharp.Standalone.Options? standaloneOptions;
 
+        public OptionsTests()
+        {
+            Environment.SetEnvironmentVariable("LGTM_INDEX_EXTRACTOR", "");
+        }
+
         [Fact]
         public void DefaultOptions()
         {
@@ -23,6 +28,7 @@ namespace Semmle.Extraction.Tests
             Assert.True(options.Threads >= 1);
             Assert.Equal(Verbosity.Info, options.LegacyVerbosity);
             Assert.False(options.Console);
+            Assert.False(options.Fast);
             Assert.Equal(TrapWriter.CompressionMode.Brotli, options.TrapCompression);
         }
 
@@ -159,6 +165,14 @@ namespace Semmle.Extraction.Tests
             Assert.True(standaloneOptions.Help);
         }
 
+        [Fact]
+        public void Fast()
+        {
+            Environment.SetEnvironmentVariable("LGTM_INDEX_EXTRACTOR", "--fast");
+            options = CSharp.Options.CreateWithEnvironment(Array.Empty<string>());
+            Assert.True(options.Fast);
+        }
+
         [Fact]
         public void ArchiveArguments()
         {

csharp/extractor/Semmle.Extraction.Tests/Runtime.cs

@@ -26,8 +26,6 @@ namespace Semmle.Extraction.Tests
         public IList<string> GetListedSdks() => sdks;
 
         public bool Exec(string execArgs) => true;
-
-        public IList<string> GetNugetFeeds(string nugetConfig) => [];
     }
 
     public class RuntimeTests

csharp/extractor/Semmle.Extraction/Options.cs

@@ -60,6 +60,11 @@ namespace Semmle.Extraction
         /// </summary>
         public bool Cache { get; private set; } = true;
 
+        /// <summary>
+        /// Whether "fast extraction mode" has been enabled.
+        /// </summary>
+        public bool Fast { get; private set; } = false;
+
         /// <summary>
         /// Whether extraction is done using `codeql test run`.
         /// </summary>
@@ -110,6 +115,9 @@ namespace Semmle.Extraction
                 case "cache":
                     Cache = value;
                     return true;
+                case "fast":
+                    Fast = value;
+                    return true;
                 case "qltest":
                     QlTest = value;
                     return true;

csharp/extractor/Semmle.Util.Tests/ActionMap.cs

@@ -0,0 +1,54 @@
+using Xunit;
+using Semmle.Util;
+
+using Assert = Xunit.Assert;
+
+namespace SemmleTests.Semmle.Util
+{
+
+    public class ActionMapTests
+    {
+        [Fact]
+        public void TestAddthenOnAdd()
+        {
+            var am = new ActionMap<int, int>();
+            am.Add(1, 2);
+            int value = 0;
+            am.OnAdd(1, x => value = x);
+            Assert.Equal(2, value);
+        }
+
+        [Fact]
+        public void TestOnAddthenAdd()
+        {
+            var am = new ActionMap<int, int>();
+            int value = 0;
+            am.OnAdd(1, x => value = x);
+            am.Add(1, 2);
+            Assert.Equal(2, value);
+        }
+
+        [Fact]
+        public void TestNotAdded()
+        {
+            var am = new ActionMap<int, int>();
+            int value = 0;
+            am.OnAdd(1, x => value = x);
+            am.Add(2, 2);
+            Assert.Equal(0, value);
+        }
+
+        [Fact]
+        public void TestMultipleActions()
+        {
+            var am = new ActionMap<int, int>();
+            int value1 = 0, value2 = 0;
+            am.OnAdd(1, x => value1 = x);
+            am.OnAdd(1, x => value2 = x);
+            am.Add(1, 2);
+            Assert.Equal(2, value1);
+            Assert.Equal(2, value2);
+        }
+
+    }
+}

csharp/extractor/Semmle.Util.Tests/TextTest.cs

@@ -0,0 +1,78 @@
+using Xunit;
+using System;
+using Semmle.Util;
+
+using Assert = Xunit.Assert;
+
+namespace SemmleTests
+{
+    public class TextTest
+    {
+        //#################### PRIVATE VARIABLES ####################
+        #region
+
+        /// <summary>
+        /// A shorter way of writing Environment.NewLine (it gets used repeatedly).
+        /// </summary>
+        private static readonly string NL = Environment.NewLine;
+
+        #endregion
+
+        //#################### TEST METHODS ####################
+        #region
+
+        [Fact]
+        public void GetAllTest()
+        {
+            var input = new string[]
+            {
+                "Said once a young coder from Crewe,",
+                "'I like to write tests, so I do!",
+                "They help me confirm",
+                "That I don't need to squirm -",
+                "My code might look nice, but works too!'"
+            };
+
+            var text = new Text(input);
+
+            Assert.Equal(string.Join(NL, input) + NL, text.GetAll());
+        }
+
+        [Fact]
+        public void GetPortionTest()
+        {
+            var input = new string[]
+            {
+                "There once was a jolly young tester",
+                "Who couldn't leave software to fester -",
+                "He'd prod and he'd poke",
+                "Until something bad broke,",
+                "And then he'd find someone to pester."
+            };
+
+            var text = new Text(input);
+
+            // A single-line range (to test the special case).
+            Assert.Equal("jolly" + NL, text.GetPortion(0, 17, 0, 22));
+
+            // A two-line range.
+            Assert.Equal("prod and he'd poke" + NL + "Until" + NL, text.GetPortion(2, 5, 3, 5));
+
+            // A three-line range (to test that the middle line is included in full).
+            Assert.Equal("poke" + NL + "Until something bad broke," + NL + "And then" + NL, text.GetPortion(2, 19, 4, 8));
+
+            // An invalid but recoverable range (to test that a best effort is made rather than crashing).
+            Assert.Equal(NL + "Who couldn't leave software to fester -" + NL, text.GetPortion(0, int.MaxValue, 1, int.MaxValue));
+
+            // Some quite definitely dodgy ranges (to test that exceptions are thrown).
+            Assert.Throws<Exception>(() => text.GetPortion(-1, 0, 0, 0));
+            Assert.Throws<Exception>(() => text.GetPortion(0, -1, 0, 0));
+            Assert.Throws<Exception>(() => text.GetPortion(0, 0, -1, 0));
+            Assert.Throws<Exception>(() => text.GetPortion(0, 0, 0, -1));
+            Assert.Throws<Exception>(() => text.GetPortion(3, 5, 2, 5));
+            Assert.Throws<Exception>(() => text.GetPortion(2, 5, int.MaxValue, 5));
+        }
+
+        #endregion
+    }
+}

csharp/extractor/Semmle.Util/ActionMap.cs

@@ -0,0 +1,45 @@
+using System;
+using System.Collections.Generic;
+
+namespace Semmle.Util
+{
+    /// <summary>
+    /// A dictionary which performs an action when items are added to the dictionary.
+    /// The order in which keys and actions are added does not matter.
+    /// </summary>
+    /// <typeparam name="TKey"></typeparam>
+    /// <typeparam name="TValue"></typeparam>
+    public class ActionMap<TKey, TValue> where TKey : notnull
+    {
+        public void Add(TKey key, TValue value)
+        {
+
+            if (actions.TryGetValue(key, out var a))
+                a(value);
+            values[key] = value;
+        }
+
+        public void OnAdd(TKey key, Action<TValue> action)
+        {
+            if (actions.TryGetValue(key, out var a))
+            {
+                actions[key] = a + action;
+            }
+            else
+            {
+                actions.Add(key, action);
+            }
+
+            if (values.TryGetValue(key, out var val))
+            {
+                action(val);
+            }
+        }
+
+        // Action associated with each key.
+        private readonly Dictionary<TKey, Action<TValue>> actions = new Dictionary<TKey, Action<TValue>>();
+
+        // Values associated with each key.
+        private readonly Dictionary<TKey, TValue> values = new Dictionary<TKey, TValue>();
+    }
+}

csharp/extractor/Semmle.Util/Enumerators.cs

@@ -0,0 +1,19 @@
+using System.Collections.Generic;
+
+namespace Semmle.Util
+{
+    public static class Enumerators
+    {
+        /// <summary>
+        /// Create an enumerable with a single element.
+        /// </summary>
+        ///
+        /// <typeparam name="T">The type of the enumerable/element.</typeparam>
+        /// <param name="t">The element.</param>
+        /// <returns>An enumerable containing a single element.</returns>
+        public static IEnumerable<T> Singleton<T>(T t)
+        {
+            yield return t;
+        }
+    }
+}

csharp/extractor/Semmle.Util/EnvironmentVariables.cs

@@ -27,12 +27,5 @@ namespace Semmle.Util
             }
             return threads;
         }
-
-        public static bool GetBoolean(string name)
-        {
-            var env = Environment.GetEnvironmentVariable(name);
-            var _ = bool.TryParse(env, out var value);
-            return value;
-        }
     }
 }

csharp/extractor/Semmle.Util/FileRenamer.cs

@@ -0,0 +1,34 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+
+namespace Semmle.Util
+{
+    /// <summary>
+    /// Utility to temporarily rename a set of files.
+    /// </summary>
+    public sealed class FileRenamer : IDisposable
+    {
+        private readonly string[] files;
+        private const string suffix = ".codeqlhidden";
+
+        public FileRenamer(IEnumerable<FileInfo> oldFiles)
+        {
+            files = oldFiles.Select(f => f.FullName).ToArray();
+
+            foreach (var file in files)
+            {
+                File.Move(file, file + suffix);
+            }
+        }
+
+        public void Dispose()
+        {
+            foreach (var file in files)
+            {
+                File.Move(file + suffix, file);
+            }
+        }
+    }
+}

csharp/extractor/Semmle.Util/FileUtils.cs

@@ -102,7 +102,8 @@ namespace Semmle.Util
         private static async Task DownloadFileAsync(string address, string filename)
         {
             using var httpClient = new HttpClient();
-            using var contentStream = await httpClient.GetStreamAsync(address);
+            using var request = new HttpRequestMessage(HttpMethod.Get, address);
+            using var contentStream = await (await httpClient.SendAsync(request)).Content.ReadAsStreamAsync();
             using var stream = new FileStream(filename, FileMode.Create, FileAccess.Write, FileShare.None, 4096, true);
             await contentStream.CopyToAsync(stream);
         }
@@ -111,7 +112,7 @@ namespace Semmle.Util
         /// Downloads the file at <paramref name="address"/> to <paramref name="fileName"/>.
         /// </summary>
         public static void DownloadFile(string address, string fileName) =>
-           DownloadFileAsync(address, fileName).GetAwaiter().GetResult();
+           DownloadFileAsync(address, fileName).Wait();
 
         public static string NestPaths(ILogger logger, string? outerpath, string innerpath)
         {

csharp/extractor/Semmle.Util/FuzzyDictionary.cs

@@ -0,0 +1,165 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace Semmle.Util
+{
+    /// <summary>
+    /// A dictionary from strings to elements of type T.
+    /// </summary>
+    ///
+    /// <remarks>
+    /// This data structure is able to locate items based on an "approximate match"
+    /// of the key. This is used for example when attempting to identify two terms
+    /// in different trap files which are similar but not identical.
+    ///
+    /// The algorithm locates the closest match to a string based on a "distance function".
+    ///
+    /// Whilst many distance functions are possible, a bespoke algorithm is used here,
+    /// for efficiency and suitability for the domain.
+    ///
+    /// The distance is defined as the Hamming Distance of the numbers in the string.
+    /// Each string is split into the base "form" (stripped of numbers) and a vector of
+    /// numbers. (Numbers are non-negative integers in this context).
+    ///
+    /// Strings with a different "form" are considered different and have a distance
+    /// of infinity.
+    ///
+    /// This distance function is reflexive, symmetric and obeys the triangle inequality.
+    ///
+    /// E.g. foo(bar,1,2) has form "foo(bar,,)" and integers {1,2}
+    ///
+    /// distance(foo(bar,1,2), foo(bar,1,2)) = 0
+    /// distance(foo(bar,1,2), foo(bar,1,3)) = 1
+    /// distance(foo(bar,2,1), foo(bar,1,2)) = 2
+    /// distance(foo(bar,1,2), foo(baz,1,2)) = infinity
+    /// </remarks>
+    ///
+    /// <typeparam name="T">The value type.</typeparam>
+    public class FuzzyDictionary<T> where T : class
+    {
+        // All data items indexed by the "base string" (stripped of numbers)
+        private readonly Dictionary<string, List<KeyValuePair<string, T>>> index = new Dictionary<string, List<KeyValuePair<string, T>>>();
+
+        /// <summary>
+        /// Stores a new KeyValuePair in the data structure.
+        /// </summary>
+        /// <param name="k">The key.</param>
+        /// <param name="v">The value.</param>
+        public void Add(string k, T v)
+        {
+            var kv = new KeyValuePair<string, T>(k, v);
+
+            var root = StripDigits(k);
+            index.AddAnother(root, kv);
+        }
+
+        /// <summary>
+        /// Computes the Hamming Distance between two sequences of the same length.
+        /// </summary>
+        /// <param name="v1">Vector 1</param>
+        /// <param name="v2">Vector 2</param>
+        /// <returns>The Hamming Distance.</returns>
+        private static int HammingDistance<TElement>(IEnumerable<TElement> v1, IEnumerable<TElement> v2) where TElement : notnull
+        {
+            return v1.Zip(v2, (x, y) => x.Equals(y) ? 0 : 1).Sum();
+        }
+
+        /// <summary>
+        /// Locates the value with the smallest Hamming Distance from the query.
+        /// </summary>
+        /// <param name="query">The query string.</param>
+        /// <param name="distance">The distance between the query string and the stored string.</param>
+        /// <returns>The best match, or null (default).</returns>
+        public T? FindMatch(string query, out int distance)
+        {
+            var root = StripDigits(query);
+            if (!index.TryGetValue(root, out var list))
+            {
+                distance = 0;
+                return default(T);
+            }
+
+            return BestMatch(query, list, (a, b) => HammingDistance(ExtractIntegers(a), ExtractIntegers(b)), out distance);
+        }
+
+        /// <summary>
+        /// Returns the best match (with the smallest distance) for a query.
+        /// </summary>
+        /// <param name="query">The query string.</param>
+        /// <param name="candidates">The list of candidate matches.</param>
+        /// <param name="distance">The distance function.</param>
+        /// <param name="bestDistance">The distance between the query and the stored string.</param>
+        /// <returns>The stored value.</returns>
+        private static T? BestMatch(string query, IEnumerable<KeyValuePair<string, T>> candidates, Func<string, string, int> distance, out int bestDistance)
+        {
+            var bestMatch = default(T);
+            bestDistance = 0;
+            var first = true;
+
+            foreach (var candidate in candidates)
+            {
+                var d = distance(query, candidate.Key);
+                if (d == 0)
+                    return candidate.Value;
+
+                if (first || d < bestDistance)
+                {
+                    bestDistance = d;
+                    bestMatch = candidate.Value;
+                    first = false;
+                }
+            }
+
+            return bestMatch;
+        }
+
+        /// <summary>
+        /// Removes all digits from a string.
+        /// </summary>
+        /// <param name="input">The input string.</param>
+        /// <returns>String with digits removed.</returns>
+        private static string StripDigits(string input)
+        {
+            var result = new StringBuilder();
+            foreach (var c in input.Where(c => !char.IsDigit(c)))
+                result.Append(c);
+            return result.ToString();
+        }
+
+        /// <summary>
+        /// Extracts and enumerates all non-negative integers in a string.
+        /// </summary>
+        /// <param name="input">The string to enumerate.</param>
+        /// <returns>The sequence of integers.</returns>
+        private static IEnumerable<int> ExtractIntegers(string input)
+        {
+            var inNumber = false;
+            var value = 0;
+            foreach (var c in input)
+            {
+                if (char.IsDigit(c))
+                {
+                    if (inNumber)
+                    {
+                        value = value * 10 + (c - '0');
+                    }
+                    else
+                    {
+                        inNumber = true;
+                        value = c - '0';
+                    }
+                }
+                else
+                {
+                    if (inNumber)
+                    {
+                        yield return value;
+                        inNumber = false;
+                    }
+                }
+            }
+        }
+    }
+}

csharp/extractor/Semmle.Util/SharedReference.cs

@@ -0,0 +1,18 @@
+namespace Semmle.Util
+{
+    /// <summary>
+    /// An instance of this class maintains a shared reference to an object.
+    /// This makes it possible for several different parts of the code to
+    /// share access to an object that can change (that is, they all want
+    /// to refer to the same object, but the object to which they jointly
+    /// refer may vary over time).
+    /// </summary>
+    /// <typeparam name="T">The type of the shared object.</typeparam>
+    public sealed class SharedReference<T> where T : class
+    {
+        /// <summary>
+        /// The shared object to which different parts of the code want to refer.
+        /// </summary>
+        public T? Obj { get; set; }
+    }
+}

csharp/extractor/Semmle.Util/StringExtensions.cs

@@ -0,0 +1,41 @@
+using System.Collections.Generic;
+using System.Linq;
+
+namespace Semmle.Util
+{
+    public static class StringExtensions
+    {
+        public static (string, string) Split(this string self, int index0)
+        {
+            var split = self.Split(new[] { index0 });
+            return (split[0], split[1]);
+        }
+
+        public static (string, string, string) Split(this string self, int index0, int index1)
+        {
+            var split = self.Split(new[] { index0, index1 });
+            return (split[0], split[1], split[2]);
+        }
+
+        public static (string, string, string, string) Split(this string self, int index0, int index1, int index2)
+        {
+            var split = self.Split(new[] { index0, index1, index2 });
+            return (split[0], split[1], split[2], split[3]);
+        }
+
+        private static List<string> Split(this string self, params int[] indices)
+        {
+            var ret = new List<string>();
+            var previousIndex = 0;
+            foreach (var index in indices.OrderBy(i => i))
+            {
+                ret.Add(self.Substring(previousIndex, index - previousIndex));
+                previousIndex = index;
+            }
+
+            ret.Add(self.Substring(previousIndex));
+
+            return ret;
+        }
+    }
+}

csharp/extractor/Semmle.Util/Text.cs

@@ -0,0 +1,105 @@
+using System;
+using System.IO;
+
+namespace Semmle.Util
+{
+    /// <summary>
+    /// An instance of this class represents a piece of text, e.g. the text of a C# source file.
+    /// </summary>
+    public sealed class Text
+    {
+        //#################### PRIVATE VARIABLES ####################
+        #region
+
+        /// <summary>
+        /// The text, stored line-by-line.
+        /// </summary>
+        private readonly string[] lines;
+
+        #endregion
+
+        //#################### CONSTRUCTORS ####################
+        #region
+
+        /// <summary>
+        /// Constructs a text object from an array of lines.
+        /// </summary>
+        /// <param name="lines">The lines of text.</param>
+        public Text(string[] lines)
+        {
+            this.lines = lines;
+        }
+
+        #endregion
+
+        //#################### PUBLIC METHODS ####################
+        #region
+
+        /// <summary>
+        /// Gets the whole text.
+        /// </summary>
+        /// <returns>The whole text.</returns>
+        public string GetAll()
+        {
+            using var sw = new StringWriter();
+            foreach (var s in lines)
+            {
+                sw.WriteLine(s);
+            }
+            return sw.ToString();
+        }
+
+        /// <summary>
+        /// Gets the portion of text that lies in the specified location range.
+        /// </summary>
+        /// <param name="startRow">The row at which the portion starts.</param>
+        /// <param name="startColumn">The column in the start row at which the portion starts.</param>
+        /// <param name="endRow">The row at which the portion ends.</param>
+        /// <param name="endColumn">The column in the end row at which the portion ends.</param>
+        /// <returns>The portion of text that lies in the specified location range.</returns>
+        public string GetPortion(int startRow, int startColumn, int endRow, int endColumn)
+        {
+            // Perform some basic validation on the range bounds.
+            if (startRow < 0 || endRow < 0 || startColumn < 0 || endColumn < 0 || endRow >= lines.Length || startRow > endRow)
+            {
+                throw new Exception
+                (
+                    string.Format("Bad range ({0},{1}):({2},{3}) in a piece of text with {4} lines", startRow, startColumn, endRow, endColumn, lines.Length)
+                );
+            }
+
+            using var sw = new StringWriter();
+            string line;
+
+            for (var i = startRow; i <= endRow; ++i)
+            {
+                if (i == startRow && i == endRow)
+                {
+                    // This is a single-line range, so take the bit between "startColumn" and "endColumn".
+                    line = startColumn <= lines[i].Length ? lines[i].Substring(startColumn, endColumn - startColumn) : "";
+                }
+                else if (i == startRow)
+                {
+                    // This is the first line of a multi-line range, so take the bit from "startColumn" onwards.
+                    line = startColumn <= lines[i].Length ? lines[i].Substring(startColumn) : "";
+                }
+                else if (i == endRow)
+                {
+                    // This is the last line of a multi-line range, so take the bit up to "endColumn".
+                    line = endColumn <= lines[i].Length ? lines[i].Substring(0, endColumn) : lines[i];
+                }
+                else
+                {
+                    // This is a line in the middle of a multi-line range, so take the whole line.
+                    line = lines[i];
+                }
+
+                sw.WriteLine(line);
+            }
+
+            return sw.ToString();
+        }
+
+        #endregion
+    }
+}

csharp/extractor/Semmle.Util/ToolStatusPage.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Globalization;
+using System.IO;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Converters;
 using Newtonsoft.Json.Serialization;
@@ -178,4 +179,64 @@ namespace Semmle.Util
             this.PlaintextMessage = plaintextMessage;
         }
     }
+
+    /// <summary>
+    /// Provides the ability to write diagnostic messages to some output.
+    /// </summary>
+    public interface IDiagnosticsWriter
+    {
+        /// <summary>
+        /// Adds <paramref name="message" /> as a new diagnostics entry.
+        /// </summary>
+        /// <param name="message">The diagnostics entry to add.</param>
+        void AddEntry(DiagnosticMessage message);
+    }
+
+    /// <summary>
+    /// A wrapper around an underlying <see cref="StreamWriter" /> which allows
+    /// <see cref="DiagnosticMessage" /> objects to be serialized to it.
+    /// </summary>
+    public sealed class DiagnosticsStream : IDiagnosticsWriter, IDisposable
+    {
+        private readonly JsonSerializer serializer;
+        private readonly StreamWriter writer;
+
+        /// <summary>
+        /// Initialises a new <see cref="DiagnosticsStream" /> for a file at <paramref name="path" />.
+        /// </summary>
+        /// <param name="path">The path to the file that should be created.</param>
+        public DiagnosticsStream(string path)
+        {
+            this.writer = File.CreateText(path);
+
+            var contractResolver = new DefaultContractResolver
+            {
+                NamingStrategy = new CamelCaseNamingStrategy()
+            };
+
+            serializer = new JsonSerializer
+            {
+                ContractResolver = contractResolver,
+                NullValueHandling = NullValueHandling.Ignore
+            };
+        }
+
+        /// <summary>
+        /// Adds <paramref name="message" /> as a new diagnostics entry.
+        /// </summary>
+        /// <param name="message">The diagnostics entry to add.</param>
+        public void AddEntry(DiagnosticMessage message)
+        {
+            serializer.Serialize(writer, message);
+            writer.Flush();
+        }
+
+        /// <summary>
+        /// Releases all resources used by the <see cref="DiagnosticsStream" /> object.
+        /// </summary>
+        public void Dispose()
+        {
+            writer.Dispose();
+        }
+    }
 }

csharp/extractor/Semmle.Util/ToolStatusPage/DiagnosticsStream.cs

@@ -1,55 +0,0 @@
-using System;
-using System.IO;
-using Newtonsoft.Json;
-using Newtonsoft.Json.Serialization;
-
-namespace Semmle.Util
-{
-    /// <summary>
-    /// A wrapper around an underlying <see cref="StreamWriter" /> which allows
-    /// <see cref="DiagnosticMessage" /> objects to be serialized to it.
-    /// </summary>
-    public sealed class DiagnosticsStream : IDiagnosticsWriter
-    {
-        private readonly JsonSerializer serializer;
-        private readonly StreamWriter writer;
-
-        /// <summary>
-        /// Initialises a new <see cref="DiagnosticsStream" /> for a file at <paramref name="path" />.
-        /// </summary>
-        /// <param name="path">The path to the file that should be created.</param>
-        public DiagnosticsStream(string path)
-        {
-            this.writer = File.CreateText(path);
-
-            var contractResolver = new DefaultContractResolver
-            {
-                NamingStrategy = new CamelCaseNamingStrategy()
-            };
-
-            serializer = new JsonSerializer
-            {
-                ContractResolver = contractResolver,
-                NullValueHandling = NullValueHandling.Ignore
-            };
-        }
-
-        /// <summary>
-        /// Adds <paramref name="message" /> as a new diagnostics entry.
-        /// </summary>
-        /// <param name="message">The diagnostics entry to add.</param>
-        public void AddEntry(DiagnosticMessage message)
-        {
-            serializer.Serialize(writer, message);
-            writer.Flush();
-        }
-
-        /// <summary>
-        /// Releases all resources used by the <see cref="DiagnosticsStream" /> object.
-        /// </summary>
-        public void Dispose()
-        {
-            writer.Dispose();
-        }
-    }
-}

csharp/extractor/Semmle.Util/ToolStatusPage/IDiagnosticsWriter.cs

@@ -1,16 +0,0 @@
-using System;
-
-namespace Semmle.Util
-{
-    /// <summary>
-    /// Provides the ability to write diagnostic messages to some output.
-    /// </summary>
-    public interface IDiagnosticsWriter : IDisposable
-    {
-        /// <summary>
-        /// Adds <paramref name="message" /> as a new diagnostics entry.
-        /// </summary>
-        /// <param name="message">The diagnostics entry to add.</param>
-        void AddEntry(DiagnosticMessage message);
-    }
-}

csharp/extractor/Semmle.Util/Worklist.cs

@@ -0,0 +1,57 @@
+using System.Collections.Generic;
+
+namespace Semmle.Util
+{
+    /// <summary>
+    /// A worklist of items, providing the operations of adding an item, checking
+    /// whether there are new items and iterating a chunk of unprocessed items.
+    /// Any one item will only be accepted into the worklist once.
+    /// </summary>
+    public class Worklist<T>
+    {
+        private readonly HashSet<T> internalSet = new HashSet<T>();
+        private LinkedList<T> internalList = new LinkedList<T>();
+        private bool hasNewElements = false;
+
+        /// <summary>
+        /// Gets a value indicating whether this instance has had any new elements added
+        /// since the last time <c>GetUnprocessedElements()</c> was called.
+        /// </summary>
+        /// <value>
+        /// <c>true</c> if this instance has new elements; otherwise, <c>false</c>.
+        /// </value>
+        public bool HasNewElements => hasNewElements;
+
+        /// <summary>
+        /// Add the specified element to the worklist.
+        /// </summary>
+        /// <param name='element'>
+        /// If set to <c>true</c> element.
+        /// </param>
+        public bool Add(T element)
+        {
+            if (internalSet.Contains(element))
+                return false;
+            internalSet.Add(element);
+            internalList.AddLast(element);
+            hasNewElements = true;
+            return true;
+        }
+
+        /// <summary>
+        /// Gets the unprocessed elements that have been accumulated since the last time
+        /// this method was called. If <c>HasNewElements == true</c>, the resulting list
+        /// will be non-empty.
+        /// </summary>
+        /// <returns>
+        /// The unprocessed elements.
+        /// </returns>
+        public LinkedList<T> GetUnprocessedElements()
+        {
+            var result = internalList;
+            internalList = new LinkedList<T>();
+            hasNewElements = false;
+            return result;
+        }
+    }
+}

csharp/ql/campaigns/Solorigate/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-solorigate-all
-version: 1.7.14-dev
+version: 1.7.13
 groups:
   - csharp
   - solorigate

csharp/ql/campaigns/Solorigate/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-solorigate-queries
-version: 1.7.14-dev
+version: 1.7.13
 groups:
   - csharp
   - solorigate

csharp/ql/integration-tests/all-platforms/diag_recursive_generics/Types.ql

@@ -2,4 +2,4 @@ import csharp
 
 from Class c
 where c.fromSource()
-select c, c.getBaseClass().getFullyQualifiedNameDebug()
+select c, c.getBaseClass().getFullyQualifiedName()

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/Assemblies.expected

@@ -1 +0,0 @@
-| [...]/newtonsoft.json/13.0.3/lib/net6.0/Newtonsoft.Json.dll |

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/Assemblies.ql

@@ -1,11 +0,0 @@
-import csharp
-
-private string getPath(Assembly a) {
-  not a.getCompilation().getOutputAssembly() = a and
-  exists(string s | s = a.getFile().getAbsolutePath() |
-    result = "[...]/" + s.substring(s.indexOf("newtonsoft.json"), s.length())
-  )
-}
-
-from Assembly a
-select getPath(a)

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/CompilationInfo.expected

@@ -1,13 +0,0 @@
-| All Nuget feeds reachable | 0.0 |
-| Fallback nuget restore | 1.0 |
-| Project files on filesystem | 1.0 |
-| Resolved assembly conflicts | 7.0 |
-| Restored .NET framework variants | 0.0 |
-| Solution files on filesystem | 1.0 |
-| Source files generated | 0.0 |
-| Source files on filesystem | 1.0 |
-| Successfully ran fallback nuget restore | 1.0 |
-| Unresolved references | 0.0 |
-| UseWPF set | 0.0 |
-| UseWindowsForms set | 0.0 |
-| WebView extraction enabled | 1.0 |

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/CompilationInfo.ql

@@ -1,15 +0,0 @@
-import csharp
-import semmle.code.csharp.commons.Diagnostics
-
-query predicate compilationInfo(string key, float value) {
-  key != "Resolved references" and
-  not key.matches("Compiler diagnostic count for%") and
-  exists(Compilation c, string infoKey, string infoValue | infoValue = c.getInfo(infoKey) |
-    key = infoKey and
-    value = infoValue.toFloat()
-    or
-    not exists(infoValue.toFloat()) and
-    key = infoKey + ": " + infoValue and
-    value = 1
-  )
-}

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/diagnostics.expected

@@ -1,42 +0,0 @@
-{
-  "markdownMessage": "C# analysis with build-mode 'none' completed.",
-  "severity": "unknown",
-  "source": {
-    "extractorName": "csharp",
-    "id": "csharp/autobuilder/buildless/complete",
-    "name": "C# analysis with build-mode 'none' completed"
-  },
-  "visibility": {
-    "cliSummaryTable": true,
-    "statusPage": false,
-    "telemetry": true
-  }
-}
-{
-  "markdownMessage": "C# with build-mode set to 'none'. This means that all C# source in the working directory will be scanned, with build tools, such as Nuget and Dotnet CLIs, only contributing information about external dependencies.",
-  "severity": "note",
-  "source": {
-    "extractorName": "csharp",
-    "id": "csharp/autobuilder/buildless/mode-active",
-    "name": "C# with build-mode set to 'none'"
-  },
-  "visibility": {
-    "cliSummaryTable": true,
-    "statusPage": true,
-    "telemetry": true
-  }
-}
-{
-  "markdownMessage": "Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.",
-  "severity": "warning",
-  "source": {
-    "extractorName": "csharp",
-    "id": "csharp/autobuilder/buildless/unreachable-feed",
-    "name": "Found unreachable Nuget feed in C# analysis with build-mode 'none'"
-  },
-  "visibility": {
-    "cliSummaryTable": true,
-    "statusPage": true,
-    "telemetry": true
-  }
-}

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/proj/Program.cs

@@ -1,6 +0,0 @@
-class Program
-{
-    static void Main(string[] args)
-    {
-    }
-}