JS: Address review comments

C#: Fully qualified name.

config/identical-files.json

@@ -251,12 +251,6 @@
     "cpp/ql/src/Security/CWE/CWE-020/SafeExternalAPIFunction.qll",
     "cpp/ql/src/Security/CWE/CWE-020/ir/SafeExternalAPIFunction.qll"
   ],
-  "XML": [
-    "cpp/ql/lib/semmle/code/cpp/XML.qll",
-    "csharp/ql/lib/semmle/code/csharp/XML.qll",
-    "java/ql/lib/semmle/code/xml/XML.qll",
-    "python/ql/lib/semmle/python/xml/XML.qll"
-  ],
   "DuplicationProblems.inc.qhelp": [
     "cpp/ql/src/Metrics/Files/DuplicationProblems.inc.qhelp",
     "javascript/ql/src/Metrics/DuplicationProblems.inc.qhelp",

cpp/autobuilder/Semmle.Autobuild.Cpp.Tests/BuildScripts.cs

@@ -203,6 +203,8 @@ namespace Semmle.Autobuild.Cpp.Tests
         public IList<DiagnosticMessage> Diagnostics { get; } = new List<DiagnosticMessage>();
 
         public void AddEntry(DiagnosticMessage message) => this.Diagnostics.Add(message);
+
+        public void Dispose() { }
     }
 
     /// <summary>
@@ -250,12 +252,7 @@ namespace Semmle.Autobuild.Cpp.Tests
             EndCallbackIn.Add(s);
         }
 
-        CppAutobuilder CreateAutoBuilder(bool isWindows,
-            string? buildless = null, string? solution = null, string? buildCommand = null, string? ignoreErrors = null,
-            string? msBuildArguments = null, string? msBuildPlatform = null, string? msBuildConfiguration = null, string? msBuildTarget = null,
-            string? dotnetArguments = null, string? dotnetVersion = null, string? vsToolsVersion = null,
-            string? nugetRestore = null, string? allSolutions = null,
-            string cwd = @"C:\Project")
+        CppAutobuilder CreateAutoBuilder(bool isWindows, string? dotnetVersion = null, string cwd = @"C:\Project")
         {
             string codeqlUpperLanguage = Language.Cpp.UpperCaseName;
             Actions.GetEnvironmentVariable[$"CODEQL_AUTOBUILDER_{codeqlUpperLanguage}_NO_INDEXING"] = "false";
@@ -265,22 +262,7 @@ namespace Semmle.Autobuild.Cpp.Tests
             Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_DIAGNOSTIC_DIR"] = "";
             Actions.GetEnvironmentVariable["CODEQL_JAVA_HOME"] = @"C:\codeql\tools\java";
             Actions.GetEnvironmentVariable["CODEQL_PLATFORM"] = "win64";
-            Actions.GetEnvironmentVariable["SEMMLE_DIST"] = @"C:\odasa";
-            Actions.GetEnvironmentVariable["SEMMLE_JAVA_HOME"] = @"C:\odasa\tools\java";
-            Actions.GetEnvironmentVariable["SEMMLE_PLATFORM_TOOLS"] = @"C:\odasa\tools";
-            Actions.GetEnvironmentVariable["LGTM_INDEX_VSTOOLS_VERSION"] = vsToolsVersion;
-            Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_ARGUMENTS"] = msBuildArguments;
-            Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_PLATFORM"] = msBuildPlatform;
-            Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_CONFIGURATION"] = msBuildConfiguration;
-            Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_TARGET"] = msBuildTarget;
-            Actions.GetEnvironmentVariable["LGTM_INDEX_DOTNET_ARGUMENTS"] = dotnetArguments;
-            Actions.GetEnvironmentVariable["LGTM_INDEX_DOTNET_VERSION"] = dotnetVersion;
-            Actions.GetEnvironmentVariable["LGTM_INDEX_BUILD_COMMAND"] = buildCommand;
-            Actions.GetEnvironmentVariable["LGTM_INDEX_SOLUTION"] = solution;
-            Actions.GetEnvironmentVariable["LGTM_INDEX_IGNORE_ERRORS"] = ignoreErrors;
-            Actions.GetEnvironmentVariable["LGTM_INDEX_BUILDLESS"] = buildless;
-            Actions.GetEnvironmentVariable["LGTM_INDEX_ALL_SOLUTIONS"] = allSolutions;
-            Actions.GetEnvironmentVariable["LGTM_INDEX_NUGET_RESTORE"] = nugetRestore;
+            Actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_OPTION_DOTNET_VERSION"] = dotnetVersion;
             Actions.GetEnvironmentVariable["ProgramFiles(x86)"] = isWindows ? @"C:\Program Files (x86)" : null;
             Actions.GetCurrentDirectory = cwd;
             Actions.IsWindows = isWindows;

cpp/autobuilder/Semmle.Autobuild.Cpp/CppAutobuilder.cs

@@ -26,9 +26,6 @@ namespace Semmle.Autobuild.Cpp
 
         public override BuildScript GetBuildScript()
         {
-            if (Options.BuildCommand != null)
-                return new BuildCommandRule((_, f) => f(null)).Analyse(this, false);
-
             return
                 // First try MSBuild
                 new MsBuildRule().Analyse(this, true) |

cpp/autobuilder/Semmle.Autobuild.Cpp/Program.cs

@@ -17,7 +17,7 @@ namespace Semmle.Autobuild.Cpp
                 try
                 {
                     Console.WriteLine("CodeQL C++ autobuilder");
-                    var builder = new CppAutobuilder(actions, options);
+                    using var builder = new CppAutobuilder(actions, options);
                     return builder.AttemptBuild();
                 }
                 catch (InvalidEnvironmentException ex)

cpp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-all
-version: 0.12.10
+version: 0.12.11-dev
 groups: cpp
 dbscheme: semmlecode.cpp.dbscheme
 extractor: cpp
@@ -11,4 +11,5 @@ dependencies:
   codeql/ssa: ${workspace}
   codeql/tutorial: ${workspace}
   codeql/util: ${workspace}
+  codeql/xml: ${workspace}
 warnOnImplicitThis: true

cpp/ql/lib/semmle/code/cpp/PrintAST.qll

@@ -862,7 +862,7 @@ private predicate namedExprChildPredicates(Expr expr, Element ele, string pred)
     or
     expr.(DeleteOrDeleteArrayExpr).getDestructorCall() = ele and pred = "getDestructorCall()"
     or
-    expr.(DeleteOrDeleteArrayExpr).getExpr() = ele and pred = "getExpr()"
+    expr.(DeleteOrDeleteArrayExpr).getExprWithReuse() = ele and pred = "getExprWithReuse()"
     or
     expr.(DestructorFieldDestruction).getExpr() = ele and pred = "getExpr()"
     or

cpp/ql/lib/semmle/code/cpp/XML.qll

@@ -3,305 +3,67 @@
  */
 
 import semmle.files.FileSystem
+private import codeql.xml.Xml
 
-private class TXmlLocatable =
-  @xmldtd or @xmlelement or @xmlattribute or @xmlnamespace or @xmlcomment or @xmlcharacters;
+private module Input implements InputSig<File, Location> {
+  class XmlLocatableBase = @xmllocatable or @xmlnamespaceable;
 
-/** An XML element that has a location. */
-class XmlLocatable extends @xmllocatable, TXmlLocatable {
-  /** Gets the source location for this element. */
-  Location getLocation() { xmllocations(this, result) }
+  predicate xmllocations_(XmlLocatableBase e, Location loc) { xmllocations(e, loc) }
 
-  /**
-   * Holds if this element is at the specified location.
-   * The location spans column `startcolumn` of line `startline` to
-   * column `endcolumn` of line `endline` in file `filepath`.
-   * For more information, see
-   * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
-   */
-  predicate hasLocationInfo(
-    string filepath, int startline, int startcolumn, int endline, int endcolumn
+  class XmlParentBase = @xmlparent;
+
+  class XmlNamespaceableBase = @xmlnamespaceable;
+
+  class XmlElementBase = @xmlelement;
+
+  class XmlFileBase = File;
+
+  predicate xmlEncoding_(XmlFileBase f, string enc) { xmlEncoding(f, enc) }
+
+  class XmlDtdBase = @xmldtd;
+
+  predicate xmlDTDs_(XmlDtdBase e, string root, string publicId, string systemId, XmlFileBase file) {
+    xmlDTDs(e, root, publicId, systemId, file)
+  }
+
+  predicate xmlElements_(
+    XmlElementBase e, string name, XmlParentBase parent, int idx, XmlFileBase file
   ) {
-    exists(File f, Location l | l = this.getLocation() |
-      locations_default(l, f, startline, startcolumn, endline, endcolumn) and
-      filepath = f.getAbsolutePath()
-    )
+    xmlElements(e, name, parent, idx, file)
   }
 
-  /** Gets a textual representation of this element. */
-  string toString() { none() } // overridden in subclasses
-}
+  class XmlAttributeBase = @xmlattribute;
 
-/**
- * An `XmlParent` is either an `XmlElement` or an `XmlFile`,
- * both of which can contain other elements.
- */
-class XmlParent extends @xmlparent {
-  XmlParent() {
-    // explicitly restrict `this` to be either an `XmlElement` or an `XmlFile`;
-    // the type `@xmlparent` currently also includes non-XML files
-    this instanceof @xmlelement or xmlEncoding(this, _)
+  predicate xmlAttrs_(
+    XmlAttributeBase e, XmlElementBase elementid, string name, string value, int idx,
+    XmlFileBase file
+  ) {
+    xmlAttrs(e, elementid, name, value, idx, file)
   }
 
-  /**
-   * Gets a printable representation of this XML parent.
-   * (Intended to be overridden in subclasses.)
-   */
-  string getName() { none() } // overridden in subclasses
+  class XmlNamespaceBase = @xmlnamespace;
 
-  /** Gets the file to which this XML parent belongs. */
-  XmlFile getFile() { result = this or xmlElements(this, _, _, _, result) }
-
-  /** Gets the child element at a specified index of this XML parent. */
-  XmlElement getChild(int index) { xmlElements(result, _, this, index, _) }
-
-  /** Gets a child element of this XML parent. */
-  XmlElement getAChild() { xmlElements(result, _, this, _, _) }
-
-  /** Gets a child element of this XML parent with the given `name`. */
-  XmlElement getAChild(string name) { xmlElements(result, _, this, _, _) and result.hasName(name) }
-
-  /** Gets a comment that is a child of this XML parent. */
-  XmlComment getAComment() { xmlComments(result, _, this, _) }
-
-  /** Gets a character sequence that is a child of this XML parent. */
-  XmlCharacters getACharactersSet() { xmlChars(result, _, this, _, _, _) }
-
-  /** Gets the depth in the tree. (Overridden in XmlElement.) */
-  int getDepth() { result = 0 }
-
-  /** Gets the number of child XML elements of this XML parent. */
-  int getNumberOfChildren() { result = count(XmlElement e | xmlElements(e, _, this, _, _)) }
-
-  /** Gets the number of places in the body of this XML parent where text occurs. */
-  int getNumberOfCharacterSets() { result = count(int pos | xmlChars(_, _, this, pos, _, _)) }
-
-  /**
-   * Gets the result of appending all the character sequences of this XML parent from
-   * left to right, separated by a space.
-   */
-  string allCharactersString() {
-    result =
-      concat(string chars, int pos | xmlChars(_, chars, this, pos, _, _) | chars, " " order by pos)
+  predicate xmlNs_(XmlNamespaceBase e, string prefixName, string uri, XmlFileBase file) {
+    xmlNs(e, prefixName, uri, file)
   }
 
-  /** Gets the text value contained in this XML parent. */
-  string getTextValue() { result = this.allCharactersString() }
+  predicate xmlHasNs_(XmlNamespaceableBase e, XmlNamespaceBase ns, XmlFileBase file) {
+    xmlHasNs(e, ns, file)
+  }
 
-  /** Gets a printable representation of this XML parent. */
-  string toString() { result = this.getName() }
-}
+  class XmlCommentBase = @xmlcomment;
 
-/** An XML file. */
-class XmlFile extends XmlParent, File {
-  XmlFile() { xmlEncoding(this, _) }
+  predicate xmlComments_(XmlCommentBase e, string text, XmlParentBase parent, XmlFileBase file) {
+    xmlComments(e, text, parent, file)
+  }
 
-  /** Gets a printable representation of this XML file. */
-  override string toString() { result = this.getName() }
+  class XmlCharactersBase = @xmlcharacters;
 
-  /** Gets the name of this XML file. */
-  override string getName() { result = File.super.getAbsolutePath() }
-
-  /** Gets the encoding of this XML file. */
-  string getEncoding() { xmlEncoding(this, result) }
-
-  /** Gets the XML file itself. */
-  override XmlFile getFile() { result = this }
-
-  /** Gets a top-most element in an XML file. */
-  XmlElement getARootElement() { result = this.getAChild() }
-
-  /** Gets a DTD associated with this XML file. */
-  XmlDtd getADtd() { xmlDTDs(result, _, _, _, this) }
-}
-
-/**
- * An XML document type definition (DTD).
- *
- * Example:
- *
- * ```
- * <!ELEMENT person (firstName, lastName?)>
- * <!ELEMENT firstName (#PCDATA)>
- * <!ELEMENT lastName (#PCDATA)>
- * ```
- */
-class XmlDtd extends XmlLocatable, @xmldtd {
-  /** Gets the name of the root element of this DTD. */
-  string getRoot() { xmlDTDs(this, result, _, _, _) }
-
-  /** Gets the public ID of this DTD. */
-  string getPublicId() { xmlDTDs(this, _, result, _, _) }
-
-  /** Gets the system ID of this DTD. */
-  string getSystemId() { xmlDTDs(this, _, _, result, _) }
-
-  /** Holds if this DTD is public. */
-  predicate isPublic() { not xmlDTDs(this, _, "", _, _) }
-
-  /** Gets the parent of this DTD. */
-  XmlParent getParent() { xmlDTDs(this, _, _, _, result) }
-
-  override string toString() {
-    this.isPublic() and
-    result = this.getRoot() + " PUBLIC '" + this.getPublicId() + "' '" + this.getSystemId() + "'"
-    or
-    not this.isPublic() and
-    result = this.getRoot() + " SYSTEM '" + this.getSystemId() + "'"
+  predicate xmlChars_(
+    XmlCharactersBase e, string text, XmlParentBase parent, int idx, int isCDATA, XmlFileBase file
+  ) {
+    xmlChars(e, text, parent, idx, isCDATA, file)
   }
 }
 
-/**
- * An XML element in an XML file.
- *
- * Example:
- *
- * ```
- * <manifest xmlns:android="http://schemas.android.com/apk/res/android"
- *           package="com.example.exampleapp" android:versionCode="1">
- * </manifest>
- * ```
- */
-class XmlElement extends @xmlelement, XmlParent, XmlLocatable {
-  /** Holds if this XML element has the given `name`. */
-  predicate hasName(string name) { name = this.getName() }
-
-  /** Gets the name of this XML element. */
-  override string getName() { xmlElements(this, result, _, _, _) }
-
-  /** Gets the XML file in which this XML element occurs. */
-  override XmlFile getFile() { xmlElements(this, _, _, _, result) }
-
-  /** Gets the parent of this XML element. */
-  XmlParent getParent() { xmlElements(this, _, result, _, _) }
-
-  /** Gets the index of this XML element among its parent's children. */
-  int getIndex() { xmlElements(this, _, _, result, _) }
-
-  /** Holds if this XML element has a namespace. */
-  predicate hasNamespace() { xmlHasNs(this, _, _) }
-
-  /** Gets the namespace of this XML element, if any. */
-  XmlNamespace getNamespace() { xmlHasNs(this, result, _) }
-
-  /** Gets the index of this XML element among its parent's children. */
-  int getElementPositionIndex() { xmlElements(this, _, _, result, _) }
-
-  /** Gets the depth of this element within the XML file tree structure. */
-  override int getDepth() { result = this.getParent().getDepth() + 1 }
-
-  /** Gets an XML attribute of this XML element. */
-  XmlAttribute getAnAttribute() { result.getElement() = this }
-
-  /** Gets the attribute with the specified `name`, if any. */
-  XmlAttribute getAttribute(string name) { result.getElement() = this and result.getName() = name }
-
-  /** Holds if this XML element has an attribute with the specified `name`. */
-  predicate hasAttribute(string name) { exists(this.getAttribute(name)) }
-
-  /** Gets the value of the attribute with the specified `name`, if any. */
-  string getAttributeValue(string name) { result = this.getAttribute(name).getValue() }
-
-  /** Gets a printable representation of this XML element. */
-  override string toString() { result = this.getName() }
-}
-
-/**
- * An attribute that occurs inside an XML element.
- *
- * Examples:
- *
- * ```
- * package="com.example.exampleapp"
- * android:versionCode="1"
- * ```
- */
-class XmlAttribute extends @xmlattribute, XmlLocatable {
-  /** Gets the name of this attribute. */
-  string getName() { xmlAttrs(this, _, result, _, _, _) }
-
-  /** Gets the XML element to which this attribute belongs. */
-  XmlElement getElement() { xmlAttrs(this, result, _, _, _, _) }
-
-  /** Holds if this attribute has a namespace. */
-  predicate hasNamespace() { xmlHasNs(this, _, _) }
-
-  /** Gets the namespace of this attribute, if any. */
-  XmlNamespace getNamespace() { xmlHasNs(this, result, _) }
-
-  /** Gets the value of this attribute. */
-  string getValue() { xmlAttrs(this, _, _, result, _, _) }
-
-  /** Gets a printable representation of this XML attribute. */
-  override string toString() { result = this.getName() + "=" + this.getValue() }
-}
-
-/**
- * A namespace used in an XML file.
- *
- * Example:
- *
- * ```
- * xmlns:android="http://schemas.android.com/apk/res/android"
- * ```
- */
-class XmlNamespace extends XmlLocatable, @xmlnamespace {
-  /** Gets the prefix of this namespace. */
-  string getPrefix() { xmlNs(this, result, _, _) }
-
-  /** Gets the URI of this namespace. */
-  string getUri() { xmlNs(this, _, result, _) }
-
-  /** Holds if this namespace has no prefix. */
-  predicate isDefault() { this.getPrefix() = "" }
-
-  override string toString() {
-    this.isDefault() and result = this.getUri()
-    or
-    not this.isDefault() and result = this.getPrefix() + ":" + this.getUri()
-  }
-}
-
-/**
- * A comment in an XML file.
- *
- * Example:
- *
- * ```
- * <!-- This is a comment. -->
- * ```
- */
-class XmlComment extends @xmlcomment, XmlLocatable {
-  /** Gets the text content of this XML comment. */
-  string getText() { xmlComments(this, result, _, _) }
-
-  /** Gets the parent of this XML comment. */
-  XmlParent getParent() { xmlComments(this, _, result, _) }
-
-  /** Gets a printable representation of this XML comment. */
-  override string toString() { result = this.getText() }
-}
-
-/**
- * A sequence of characters that occurs between opening and
- * closing tags of an XML element, excluding other elements.
- *
- * Example:
- *
- * ```
- * <content>This is a sequence of characters.</content>
- * ```
- */
-class XmlCharacters extends @xmlcharacters, XmlLocatable {
-  /** Gets the content of this character sequence. */
-  string getCharacters() { xmlChars(this, result, _, _, _, _) }
-
-  /** Gets the parent of this character sequence. */
-  XmlParent getParent() { xmlChars(this, _, result, _, _, _) }
-
-  /** Holds if this character sequence is CDATA. */
-  predicate isCDATA() { xmlChars(this, _, _, _, 1, _) }
-
-  /** Gets a printable representation of this XML character sequence. */
-  override string toString() { result = this.getCharacters() }
-}
+import Make<File, Location, Input>

cpp/ql/lib/semmle/code/cpp/exprs/Expr.qll

@@ -1015,8 +1015,33 @@ class DeleteOrDeleteArrayExpr extends Expr, TDeleteOrDeleteArrayExpr {
   Expr getExpr() {
     // If there is a destructor call, the object being deleted is the qualifier
     // otherwise it is the third child.
-    result = this.getChild(3) or result = this.getDestructorCall().getQualifier()
+    exists(Expr exprWithReuse | exprWithReuse = this.getExprWithReuse() |
+      if not exprWithReuse instanceof ReuseExpr
+      then result = exprWithReuse
+      else result = this.getDestructorCall().getQualifier()
+    )
   }
+
+  /**
+   * Gets the object or array being deleted, and gets a `ReuseExpr` when there
+   * is a destructor call and the object is also the qualifier of the call.
+   *
+   * For example, given:
+   * ```
+   * struct HasDestructor { ~HasDestructor(); };
+   * struct PlainOldData { int x, char y; };
+   *
+   * void f(HasDestructor* hasDestructor, PlainOldData* pod) {
+   *   delete hasDestructor;
+   *   delete pod;
+   * }
+   * ```
+   * This predicate yields a `ReuseExpr` for `delete hasDestructor`, as the
+   * the deleted expression has a destructor, and that expression is also
+   * the qualifier of the destructor call. In the case of `delete pod` the
+   * predicate does not yield a `ReuseExpr`, as there is no destructor call.
+   */
+  Expr getExprWithReuse() { result = this.getChild(3) }
 }
 
 /**
@@ -1340,7 +1365,17 @@ class ReuseExpr extends Expr, @reuseexpr {
   /**
    * Gets the expression that is being re-used.
    */
-  Expr getReusedExpr() { expr_reuse(underlyingElement(this), unresolveElement(result), _) }
+  Expr getReusedExpr() {
+    // In the case of a prvalue, the extractor outputs the expression
+    // before conversion, but the converted expression is intended.
+    if this.isPRValueCategory()
+    then result = this.getBaseReusedExpr().getFullyConverted()
+    else result = this.getBaseReusedExpr()
+  }
+
+  private Expr getBaseReusedExpr() {
+    expr_reuse(underlyingElement(this), unresolveElement(result), _)
+  }
 
   override Type getType() { result = this.getReusedExpr().getType() }
 

cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/TranslatedElement.qll

@@ -150,11 +150,6 @@ private predicate ignoreExprOnly(Expr expr) {
   or
   not translateFunction(getEnclosingFunction(expr)) and
   not Raw::varHasIRFunc(getEnclosingVariable(expr))
-  or
-  exists(DeleteOrDeleteArrayExpr deleteExpr |
-    // Ignore the destructor call, because the duplicated qualifier breaks control flow.
-    deleteExpr.getDestructorCall() = expr
-  )
 }
 
 /**

cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/TranslatedExpr.qll

@@ -2245,7 +2245,11 @@ class TranslatedDeleteOrDeleteArrayExpr extends TranslatedNonConstantExpr, Trans
 
   final override Type getCallResultType() { result = expr.getType() }
 
-  final override TranslatedExpr getQualifier() { none() }
+  final override TranslatedExpr getQualifier() {
+    result = getTranslatedExpr(expr.getDestructorCall())
+  }
+
+  final override Instruction getQualifierResult() { none() }
 
   final override predicate hasArguments() {
     // All deallocator calls have at least one argument.
@@ -2260,7 +2264,7 @@ class TranslatedDeleteOrDeleteArrayExpr extends TranslatedNonConstantExpr, Trans
   final override TranslatedExpr getArgument(int index) {
     // The only argument we define is the pointer to be deallocated.
     index = 0 and
-    result = getTranslatedExpr(expr.getExpr().getFullyConverted())
+    result = getTranslatedExpr(expr.getExprWithReuse().getFullyConverted())
   }
 
   final override predicate mayThrowException() {

cpp/ql/lib/semmle/code/cpp/models/Models.qll

@@ -41,3 +41,4 @@ private import implementations.SqLite3
 private import implementations.PostgreSql
 private import implementations.System
 private import implementations.StructuredExceptionHandling
+private import implementations.Fopen

cpp/ql/lib/semmle/code/cpp/models/implementations/Fopen.qll

@@ -0,0 +1,50 @@
+/**
+ * Provides implementation classes modeling `fopen` and various similar
+ * functions. See `semmle.code.cpp.models.Models` for usage information.
+ */
+
+import semmle.code.cpp.models.interfaces.Alias
+import semmle.code.cpp.models.interfaces.SideEffect
+
+/** The function `fopen` and friends. */
+private class Fopen extends Function, AliasFunction, SideEffectFunction {
+  Fopen() {
+    this.hasGlobalOrStdName(["fopen", "fopen_s", "freopen"])
+    or
+    this.hasGlobalName(["_open", "_wfopen", "_fsopen", "_wfsopen", "_wopen"])
+  }
+
+  override predicate hasOnlySpecificWriteSideEffects() { any() }
+
+  override predicate hasOnlySpecificReadSideEffects() { any() }
+
+  override predicate parameterEscapesOnlyViaReturn(int i) { none() }
+
+  override predicate parameterNeverEscapes(int index) {
+    // None of the parameters escape
+    this.getParameter(index).getUnspecifiedType() instanceof PointerType
+  }
+
+  override predicate hasSpecificReadSideEffect(ParameterIndex i, boolean buffer) {
+    (
+      this.hasGlobalOrStdName(["fopen", "fopen_s"])
+      or
+      this.hasGlobalName(["_wfopen", "_fsopen", "_wfsopen"])
+    ) and
+    i = [0, 1] and
+    buffer = true
+    or
+    this.hasGlobalOrStdName("freopen") and
+    (
+      i = [0, 1] and
+      buffer = true
+      or
+      i = 2 and
+      buffer = false
+    )
+    or
+    this.hasGlobalName(["_open", "_wopen"]) and
+    i = 0 and
+    buffer = true
+  }
+}

cpp/ql/src/Critical/DoubleFree.ql

@@ -37,6 +37,5 @@ where
   DoubleFree::flowPath(source, sink) and
   isFree(source.getNode(), _, _, dealloc) and
   isFree(sink.getNode(), e2)
-select sink.getNode(), source, sink,
-  "Memory pointed to by '" + e2.toString() + "' may already have been freed by $@.", dealloc,
-  dealloc.toString()
+select sink.getNode(), source, sink, "Memory pointed to by $@ may already have been freed by $@.",
+  e2, e2.toString(), dealloc, dealloc.toString()

cpp/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-queries
-version: 0.9.9
+version: 0.9.10-dev
 groups:
   - cpp
   - queries

cpp/ql/test/examples/expressions/PrintAST.expected

@@ -426,11 +426,14 @@ DestructorCall.cpp:
 #   12|           getQualifier(): [VariableAccess] c
 #   12|               Type = [PointerType] C *
 #   12|               ValueCategory = prvalue(load)
+#   12|         getExprWithReuse(): [ReuseExpr] reuse of c
+#   12|             Type = [PointerType] C *
+#   12|             ValueCategory = prvalue
 #   13|     getStmt(1): [ExprStmt] ExprStmt
 #   13|       getExpr(): [DeleteExpr] delete
 #   13|           Type = [VoidType] void
 #   13|           ValueCategory = prvalue
-#   13|         getExpr(): [VariableAccess] d
+#   13|         getExprWithReuse(): [VariableAccess] d
 #   13|             Type = [PointerType] D *
 #   13|             ValueCategory = prvalue(load)
 #   14|     getStmt(2): [ReturnStmt] return ...

cpp/ql/test/library-tests/compiler_generated/compilerGenerated.expected

@@ -14,6 +14,7 @@
 | cpp.cpp:10:7:10:7 | operator= | Function |
 | cpp.cpp:10:7:10:7 | ~MyClass | Function |
 | cpp.cpp:15:5:15:12 | call to ~MyClass | Expr |
+| cpp.cpp:15:12:15:12 | reuse of m | Expr |
 | cpp.cpp:16:1:16:1 | return ... | Stmt |
 | file://:0:0:0:0 | operator delete | Function |
 | file://:0:0:0:0 | operator new | Function |

cpp/ql/test/library-tests/ir/ir/ir.cpp

@@ -1933,6 +1933,20 @@ namespace missing_declaration_entries {
         Bar2<int> b;
         b.two_missing_variable_declaration_entries();
     }
+
+    template<typename T> struct Bar3 {
+
+        int two_more_missing_variable_declaration_entries() {
+            extern int g;
+            int z(float);
+            return g;
+        }
+    };
+
+    void test3() {
+        Bar3<int> b;
+        b.two_more_missing_variable_declaration_entries();
+    }
 }
 
 template<typename T> T global_template = 42;
@@ -2417,4 +2431,20 @@ void initialization_with_temp_destructor() {
         y += x;
 }
 
+void param_with_destructor_by_value(ClassWithDestructor c) {
+    // The call to ~ClassWithDestructor::ClassWithDestructor() seems to be missing here.
+}
+
+void param_with_destructor_by_pointer(ClassWithDestructor* c) {
+    // No destructor call should be here
+}
+
+void param_with_destructor_by_ref(ClassWithDestructor& c) {
+    // No destructor call should be here
+}
+
+void param_with_destructor_by_rref(ClassWithDestructor&& c) {
+    // No destructor call should be here
+}
+
 // semmle-extractor-options: -std=c++20 --clang

cpp/ql/test/query-tests/Critical/MemoryFreed/DoubleFree.expected

@@ -41,16 +41,16 @@ nodes
 | test_free.cpp:302:12:302:14 | buf | semmle.label | buf |
 subpaths
 #select
-| test_free.cpp:14:10:14:10 | a | test_free.cpp:11:10:11:10 | pointer to free output argument | test_free.cpp:14:10:14:10 | a | Memory pointed to by 'a' may already have been freed by $@. | test_free.cpp:11:5:11:8 | call to free | call to free |
-| test_free.cpp:31:27:31:27 | a | test_free.cpp:30:10:30:10 | pointer to free output argument | test_free.cpp:31:27:31:27 | a | Memory pointed to by 'a' may already have been freed by $@. | test_free.cpp:30:5:30:8 | call to free | call to free |
-| test_free.cpp:37:27:37:27 | a | test_free.cpp:35:10:35:10 | pointer to free output argument | test_free.cpp:37:27:37:27 | a | Memory pointed to by 'a' may already have been freed by $@. | test_free.cpp:35:5:35:8 | call to free | call to free |
-| test_free.cpp:46:10:46:10 | a | test_free.cpp:42:27:42:27 | pointer to free output argument | test_free.cpp:46:10:46:10 | a | Memory pointed to by 'a' may already have been freed by $@. | test_free.cpp:42:22:42:25 | call to free | call to free |
-| test_free.cpp:46:10:46:10 | a | test_free.cpp:44:27:44:27 | pointer to free output argument | test_free.cpp:46:10:46:10 | a | Memory pointed to by 'a' may already have been freed by $@. | test_free.cpp:44:22:44:25 | call to free | call to free |
-| test_free.cpp:51:10:51:10 | a | test_free.cpp:50:27:50:27 | pointer to free output argument | test_free.cpp:51:10:51:10 | a | Memory pointed to by 'a' may already have been freed by $@. | test_free.cpp:50:22:50:25 | call to free | call to free |
-| test_free.cpp:72:14:72:14 | a | test_free.cpp:69:10:69:10 | pointer to free output argument | test_free.cpp:72:14:72:14 | a | Memory pointed to by 'a' may already have been freed by $@. | test_free.cpp:69:5:69:8 | call to free | call to free |
-| test_free.cpp:85:12:85:12 | a | test_free.cpp:83:12:83:12 | pointer to operator delete output argument | test_free.cpp:85:12:85:12 | a | Memory pointed to by 'a' may already have been freed by $@. | test_free.cpp:83:5:83:13 | delete | delete |
-| test_free.cpp:103:10:103:10 | a | test_free.cpp:101:10:101:10 | pointer to free output argument | test_free.cpp:103:10:103:10 | a | Memory pointed to by 'a' may already have been freed by $@. | test_free.cpp:101:5:101:8 | call to free | call to free |
-| test_free.cpp:129:10:129:11 | * ... | test_free.cpp:128:10:128:11 | pointer to free output argument | test_free.cpp:129:10:129:11 | * ... | Memory pointed to by '* ...' may already have been freed by $@. | test_free.cpp:128:5:128:8 | call to free | call to free |
-| test_free.cpp:154:10:154:10 | a | test_free.cpp:152:27:152:27 | pointer to free output argument | test_free.cpp:154:10:154:10 | a | Memory pointed to by 'a' may already have been freed by $@. | test_free.cpp:152:22:152:25 | call to free | call to free |
-| test_free.cpp:209:10:209:10 | a | test_free.cpp:207:10:207:10 | pointer to free output argument | test_free.cpp:209:10:209:10 | a | Memory pointed to by 'a' may already have been freed by $@. | test_free.cpp:207:5:207:8 | call to free | call to free |
-| test_free.cpp:302:12:302:14 | buf | test_free.cpp:301:12:301:14 | pointer to g_free output argument | test_free.cpp:302:12:302:14 | buf | Memory pointed to by 'buf' may already have been freed by $@. | test_free.cpp:301:5:301:10 | call to g_free | call to g_free |
+| test_free.cpp:14:10:14:10 | a | test_free.cpp:11:10:11:10 | pointer to free output argument | test_free.cpp:14:10:14:10 | a | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:14:10:14:10 | a | a | test_free.cpp:11:5:11:8 | call to free | call to free |
+| test_free.cpp:31:27:31:27 | a | test_free.cpp:30:10:30:10 | pointer to free output argument | test_free.cpp:31:27:31:27 | a | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:31:27:31:27 | a | a | test_free.cpp:30:5:30:8 | call to free | call to free |
+| test_free.cpp:37:27:37:27 | a | test_free.cpp:35:10:35:10 | pointer to free output argument | test_free.cpp:37:27:37:27 | a | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:37:27:37:27 | a | a | test_free.cpp:35:5:35:8 | call to free | call to free |
+| test_free.cpp:46:10:46:10 | a | test_free.cpp:42:27:42:27 | pointer to free output argument | test_free.cpp:46:10:46:10 | a | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:46:10:46:10 | a | a | test_free.cpp:42:22:42:25 | call to free | call to free |
+| test_free.cpp:46:10:46:10 | a | test_free.cpp:44:27:44:27 | pointer to free output argument | test_free.cpp:46:10:46:10 | a | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:46:10:46:10 | a | a | test_free.cpp:44:22:44:25 | call to free | call to free |
+| test_free.cpp:51:10:51:10 | a | test_free.cpp:50:27:50:27 | pointer to free output argument | test_free.cpp:51:10:51:10 | a | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:51:10:51:10 | a | a | test_free.cpp:50:22:50:25 | call to free | call to free |
+| test_free.cpp:72:14:72:14 | a | test_free.cpp:69:10:69:10 | pointer to free output argument | test_free.cpp:72:14:72:14 | a | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:72:14:72:14 | a | a | test_free.cpp:69:5:69:8 | call to free | call to free |
+| test_free.cpp:85:12:85:12 | a | test_free.cpp:83:12:83:12 | pointer to operator delete output argument | test_free.cpp:85:12:85:12 | a | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:85:12:85:12 | a | a | test_free.cpp:83:5:83:13 | delete | delete |
+| test_free.cpp:103:10:103:10 | a | test_free.cpp:101:10:101:10 | pointer to free output argument | test_free.cpp:103:10:103:10 | a | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:103:10:103:10 | a | a | test_free.cpp:101:5:101:8 | call to free | call to free |
+| test_free.cpp:129:10:129:11 | * ... | test_free.cpp:128:10:128:11 | pointer to free output argument | test_free.cpp:129:10:129:11 | * ... | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:129:10:129:11 | * ... | * ... | test_free.cpp:128:5:128:8 | call to free | call to free |
+| test_free.cpp:154:10:154:10 | a | test_free.cpp:152:27:152:27 | pointer to free output argument | test_free.cpp:154:10:154:10 | a | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:154:10:154:10 | a | a | test_free.cpp:152:22:152:25 | call to free | call to free |
+| test_free.cpp:209:10:209:10 | a | test_free.cpp:207:10:207:10 | pointer to free output argument | test_free.cpp:209:10:209:10 | a | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:209:10:209:10 | a | a | test_free.cpp:207:5:207:8 | call to free | call to free |
+| test_free.cpp:302:12:302:14 | buf | test_free.cpp:301:12:301:14 | pointer to g_free output argument | test_free.cpp:302:12:302:14 | buf | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:302:12:302:14 | buf | buf | test_free.cpp:301:5:301:10 | call to g_free | call to g_free |

csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs

@@ -218,6 +218,8 @@ namespace Semmle.Autobuild.CSharp.Tests
         public IList<DiagnosticMessage> Diagnostics { get; } = new List<DiagnosticMessage>();
 
         public void AddEntry(DiagnosticMessage message) => this.Diagnostics.Add(message);
+
+        public void Dispose() { }
     }
 
     /// <summary>
@@ -399,10 +401,8 @@ namespace Semmle.Autobuild.CSharp.Tests
         }
 
         private CSharpAutobuilder CreateAutoBuilder(bool isWindows,
-            string? buildless = null, string? solution = null, string? buildCommand = null, string? ignoreErrors = null,
-            string? msBuildArguments = null, string? msBuildPlatform = null, string? msBuildConfiguration = null, string? msBuildTarget = null,
-            string? dotnetArguments = null, string? dotnetVersion = null, string? vsToolsVersion = null,
-            string? nugetRestore = null, string? allSolutions = null,
+            string? buildless = null,
+            string? dotnetVersion = null,
             string cwd = @"C:\Project")
         {
             var codeqlUpperLanguage = Language.CSharp.UpperCaseName;
@@ -412,20 +412,9 @@ namespace Semmle.Autobuild.CSharp.Tests
             actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_DIAGNOSTIC_DIR"] = "";
             actions.GetEnvironmentVariable["CODEQL_JAVA_HOME"] = @"C:\codeql\tools\java";
             actions.GetEnvironmentVariable["CODEQL_PLATFORM"] = isWindows ? "win64" : "linux64";
-            actions.GetEnvironmentVariable["LGTM_INDEX_VSTOOLS_VERSION"] = vsToolsVersion;
-            actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_ARGUMENTS"] = msBuildArguments;
-            actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_PLATFORM"] = msBuildPlatform;
-            actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_CONFIGURATION"] = msBuildConfiguration;
-            actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_TARGET"] = msBuildTarget;
-            actions.GetEnvironmentVariable["LGTM_INDEX_DOTNET_ARGUMENTS"] = dotnetArguments;
-            actions.GetEnvironmentVariable["LGTM_INDEX_DOTNET_VERSION"] = dotnetVersion;
-            actions.GetEnvironmentVariable["LGTM_INDEX_BUILD_COMMAND"] = buildCommand;
-            actions.GetEnvironmentVariable["LGTM_INDEX_SOLUTION"] = solution;
-            actions.GetEnvironmentVariable["LGTM_INDEX_IGNORE_ERRORS"] = ignoreErrors;
-            actions.GetEnvironmentVariable["LGTM_INDEX_BUILDLESS"] = buildless;
             actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_OPTION_BUILDLESS"] = buildless;
-            actions.GetEnvironmentVariable["LGTM_INDEX_ALL_SOLUTIONS"] = allSolutions;
-            actions.GetEnvironmentVariable["LGTM_INDEX_NUGET_RESTORE"] = nugetRestore;
+            if (dotnetVersion is not null)
+                actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_OPTION_DOTNET_VERSION"] = dotnetVersion;
             actions.GetEnvironmentVariable["ProgramFiles(x86)"] = isWindows ? @"C:\Program Files (x86)" : null;
             actions.GetCurrentDirectory = cwd;
             actions.IsWindows = isWindows;
@@ -600,15 +589,6 @@ namespace Semmle.Autobuild.CSharp.Tests
             TestAutobuilderScript(autobuilder, 0, 1);
         }
 
-        private void SkipVsWhere()
-        {
-            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = false;
-            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"] = false;
-            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat"] = false;
-            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\vcvarsall.bat"] = false;
-            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat"] = false;
-        }
-
         private void TestAutobuilderScript(CSharpAutobuilder autobuilder, int expectedOutput, int commandsRun)
         {
             Assert.Equal(expectedOutput, autobuilder.GetBuildScript().Run(actions, StartCallback, EndCallback));
@@ -628,23 +608,6 @@ namespace Semmle.Autobuild.CSharp.Tests
             }
         }
 
-        [Fact]
-        public void TestLinuxBuildCommand()
-        {
-            actions.RunProcess["./build.sh --skip-tests"] = 0;
-            actions.FileExists["csharp.log"] = true;
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
-            actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln";
-            actions.EnumerateDirectories[@"C:\Project"] = "";
-
-            SkipVsWhere();
-
-            var autobuilder = CreateAutoBuilder(false, buildCommand: "./build.sh --skip-tests");
-            TestAutobuilderScript(autobuilder, 0, 1);
-        }
-
         [Fact]
         public void TestLinuxBuildSh()
         {
@@ -714,177 +677,6 @@ namespace Semmle.Autobuild.CSharp.Tests
             TestAutobuilderScript(autobuilder, 0, 1);
         }
 
-        [Fact]
-        public void TestWindowsBuildBatIgnoreErrors()
-        {
-            actions.EnumerateFiles[@"C:\Project"] = "foo.cs\nbuild.bat";
-            actions.EnumerateDirectories[@"C:\Project"] = "";
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
-            actions.RunProcess[@"cmd.exe /C C:\Project\build.bat"] = 1;
-            actions.RunProcessWorkingDirectory[@"cmd.exe /C C:\Project\build.bat"] = @"C:\Project";
-            actions.RunProcess[@"cmd.exe /C C:\codeql\tools\java\bin\java -jar C:\codeql\csharp\tools\extractor-asp.jar ."] = 0;
-            actions.RunProcess[@"cmd.exe /C C:\codeql\tools\codeql index --xml --extensions config"] = 0;
-            actions.FileExists["csharp.log"] = true;
-
-            var autobuilder = CreateAutoBuilder(true, ignoreErrors: "true");
-            TestAutobuilderScript(autobuilder, 1, 1);
-        }
-
-        [Fact]
-        public void TestWindowsCmdIgnoreErrors()
-        {
-            actions.RunProcess["cmd.exe /C ^\"build.cmd^ --skip-tests^\""] = 3;
-            actions.RunProcess[@"cmd.exe /C C:\codeql\tools\java\bin\java -jar C:\codeql\csharp\tools\extractor-asp.jar ."] = 0;
-            actions.RunProcess[@"cmd.exe /C C:\codeql\tools\codeql index --xml --extensions config"] = 0;
-            actions.FileExists["csharp.log"] = true;
-            SkipVsWhere();
-
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
-            actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln";
-            actions.EnumerateDirectories[@"C:\Project"] = "";
-
-            var autobuilder = CreateAutoBuilder(true, buildCommand: "build.cmd --skip-tests", ignoreErrors: "true");
-            TestAutobuilderScript(autobuilder, 3, 1);
-        }
-
-        [Fact]
-        public void TestWindowCSharpMsBuild()
-        {
-            actions.RunProcess[@"cmd.exe /C C:\Project\.nuget\nuget.exe restore C:\Project\test1.sln -DisableParallelProcessing"] = 0;
-            actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program^ Files^ ^(x86^)\\Microsoft^ Visual^ Studio^ 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test1.sln /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /P:Fu=Bar"] = 0;
-            actions.RunProcess[@"cmd.exe /C C:\Project\.nuget\nuget.exe restore C:\Project\test2.sln -DisableParallelProcessing"] = 0;
-            actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program^ Files^ ^(x86^)\\Microsoft^ Visual^ Studio^ 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test2.sln /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /P:Fu=Bar"] = 0;
-            actions.FileExists["csharp.log"] = true;
-            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = false;
-            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"] = false;
-            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat"] = true;
-            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\vcvarsall.bat"] = false;
-            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat"] = true;
-
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
-            actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest1.cs\ntest2.cs";
-            actions.EnumerateFiles[@"C:\Project\.nuget"] = "nuget.exe";
-            actions.EnumerateDirectories[@"C:\Project"] = @".nuget";
-            actions.EnumerateDirectories[@"C:\Project\.nuget"] = "";
-
-            var autobuilder = CreateAutoBuilder(true, msBuildArguments: "/P:Fu=Bar", msBuildTarget: "Windows", msBuildPlatform: "x86", msBuildConfiguration: "Debug",
-                vsToolsVersion: "12", allSolutions: "true");
-            var testSolution1 = new TestSolution(@"C:\Project\test1.sln");
-            var testSolution2 = new TestSolution(@"C:\Project\test2.sln");
-            autobuilder.ProjectsOrSolutionsToBuild.Add(testSolution1);
-            autobuilder.ProjectsOrSolutionsToBuild.Add(testSolution2);
-
-            TestAutobuilderScript(autobuilder, 0, 4);
-        }
-
-        [Fact]
-        public void TestWindowCSharpMsBuildMultipleSolutions()
-        {
-            actions.RunProcess[@"cmd.exe /C nuget restore C:\Project\test1.csproj -DisableParallelProcessing"] = 0;
-            actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program^ Files^ ^(x86^)\\Microsoft^ Visual^ Studio^ 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test1.csproj /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /P:Fu=Bar"] = 0;
-            actions.RunProcess[@"cmd.exe /C nuget restore C:\Project\test2.csproj -DisableParallelProcessing"] = 0;
-            actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program^ Files^ ^(x86^)\\Microsoft^ Visual^ Studio^ 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test2.csproj /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /P:Fu=Bar"] = 0;
-            actions.FileExists["csharp.log"] = true;
-            actions.FileExists[@"C:\Project\test1.csproj"] = true;
-            actions.FileExists[@"C:\Project\test2.csproj"] = true;
-            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = false;
-            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"] = false;
-            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat"] = true;
-            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\vcvarsall.bat"] = false;
-            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat"] = true;
-
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
-            actions.EnumerateFiles[@"C:\Project"] = "test1.csproj\ntest2.csproj\ntest1.cs\ntest2.cs";
-            actions.EnumerateDirectories[@"C:\Project"] = "";
-
-            var csproj1 = new XmlDocument();
-            csproj1.LoadXml(@"<?xml version=""1.0"" encoding=""utf - 8""?>
-  <Project ToolsVersion=""15.0"" xmlns=""http://schemas.microsoft.com/developer/msbuild/2003"">
-    <ItemGroup>
-      <Compile Include=""test1.cs"" />
-    </ItemGroup>
-  </Project>");
-            actions.LoadXml[@"C:\Project\test1.csproj"] = csproj1;
-
-            var csproj2 = new XmlDocument();
-            csproj2.LoadXml(@"<?xml version=""1.0"" encoding=""utf - 8""?>
-  <Project ToolsVersion=""15.0"" xmlns=""http://schemas.microsoft.com/developer/msbuild/2003"">
-    <ItemGroup>
-      <Compile Include=""test1.cs"" />
-    </ItemGroup>
-  </Project>");
-            actions.LoadXml[@"C:\Project\test2.csproj"] = csproj2;
-
-            var autobuilder = CreateAutoBuilder(true, msBuildArguments: "/P:Fu=Bar", msBuildTarget: "Windows", msBuildPlatform: "x86", msBuildConfiguration: "Debug",
-                vsToolsVersion: "12");
-
-            TestAutobuilderScript(autobuilder, 0, 4);
-        }
-
-        [Fact]
-        public void TestWindowCSharpMsBuildFailed()
-        {
-            actions.RunProcess[@"cmd.exe /C nuget restore C:\Project\test1.sln -DisableParallelProcessing"] = 0;
-            actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program^ Files^ ^(x86^)\\Microsoft^ Visual^ Studio^ 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test1.sln /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /P:Fu=Bar"] = 1;
-            actions.FileExists["csharp.log"] = true;
-            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = false;
-            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"] = false;
-            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat"] = true;
-            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\vcvarsall.bat"] = false;
-            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat"] = true;
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
-            actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest1.cs\ntest2.cs";
-            actions.EnumerateDirectories[@"C:\Project"] = "";
-
-            var autobuilder = CreateAutoBuilder(true, msBuildArguments: "/P:Fu=Bar", msBuildTarget: "Windows", msBuildPlatform: "x86", msBuildConfiguration: "Debug",
-                vsToolsVersion: "12", allSolutions: "true");
-            var testSolution1 = new TestSolution(@"C:\Project\test1.sln");
-            var testSolution2 = new TestSolution(@"C:\Project\test2.sln");
-            autobuilder.ProjectsOrSolutionsToBuild.Add(testSolution1);
-            autobuilder.ProjectsOrSolutionsToBuild.Add(testSolution2);
-
-            TestAutobuilderScript(autobuilder, 1, 2);
-        }
-
-
-        [Fact]
-        public void TestSkipNugetMsBuild()
-        {
-            actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program^ Files^ ^(x86^)\\Microsoft^ Visual^ Studio^ 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test1.sln /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /P:Fu=Bar"] = 0;
-            actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program^ Files^ ^(x86^)\\Microsoft^ Visual^ Studio^ 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test2.sln /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /P:Fu=Bar"] = 0;
-            actions.FileExists["csharp.log"] = true;
-            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = false;
-            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"] = false;
-            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat"] = true;
-            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\vcvarsall.bat"] = false;
-            actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat"] = true;
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
-            actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest1.cs\ntest2.cs";
-            actions.EnumerateDirectories[@"C:\Project"] = "";
-
-            var autobuilder = CreateAutoBuilder(true, msBuildArguments: "/P:Fu=Bar", msBuildTarget: "Windows",
-                msBuildPlatform: "x86", msBuildConfiguration: "Debug", vsToolsVersion: "12",
-                allSolutions: "true", nugetRestore: "false");
-            var testSolution1 = new TestSolution(@"C:\Project\test1.sln");
-            var testSolution2 = new TestSolution(@"C:\Project\test2.sln");
-            autobuilder.ProjectsOrSolutionsToBuild.Add(testSolution1);
-            autobuilder.ProjectsOrSolutionsToBuild.Add(testSolution2);
-
-            TestAutobuilderScript(autobuilder, 0, 2);
-        }
-
         [Fact]
         public void TestSkipNugetBuildless()
         {
@@ -900,35 +692,6 @@ namespace Semmle.Autobuild.CSharp.Tests
             TestAutobuilderScript(autobuilder, 0, 1);
         }
 
-
-        [Fact]
-        public void TestSkipNugetDotnet()
-        {
-            actions.RunProcess["dotnet --info"] = 0;
-            actions.RunProcess[@"dotnet clean C:\Project/test.csproj"] = 0;
-            actions.RunProcess[@"dotnet restore C:\Project/test.csproj"] = 0;
-            actions.RunProcess[@"dotnet build --no-incremental --no-restore C:\Project/test.csproj"] = 0;
-            actions.FileExists["csharp.log"] = true;
-            actions.FileExists[@"C:\Project/test.csproj"] = true;
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
-            actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.cs\ntest.csproj";
-            actions.EnumerateDirectories[@"C:\Project"] = "";
-            var xml = new XmlDocument();
-            xml.LoadXml(@"<Project Sdk=""Microsoft.NET.Sdk"">
-  <PropertyGroup>
-    <OutputType>Exe</OutputType>
-    <TargetFramework>netcoreapp2.1</TargetFramework>
-  </PropertyGroup>
-
-</Project>");
-            actions.LoadXml[@"C:\Project/test.csproj"] = xml;
-
-            var autobuilder = CreateAutoBuilder(false, dotnetArguments: "--no-restore");  // nugetRestore=false does not work for now.
-            TestAutobuilderScript(autobuilder, 0, 4);
-        }
-
         [Fact]
         public void TestDotnetVersionNotInstalled()
         {
@@ -1053,7 +816,7 @@ namespace Semmle.Autobuild.CSharp.Tests
         {
             actions.RunProcess[@"cmd.exe /C nuget restore C:\Project\dirs.proj -DisableParallelProcessing"] = 1;
             actions.RunProcess[@"cmd.exe /C scratch\.nuget\nuget.exe restore C:\Project\dirs.proj -DisableParallelProcessing"] = 0;
-            actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program^ Files^ ^(x86^)\\Microsoft^ Visual^ Studio^ 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\dirs.proj /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /P:Fu=Bar"] = 0;
+            actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program^ Files^ ^(x86^)\\Microsoft^ Visual^ Studio^ 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\dirs.proj /t:rebuild"] = 0;
             actions.FileExists["csharp.log"] = true;
             actions.FileExists[@"C:\Project\a\test.csproj"] = true;
             actions.FileExists[@"C:\Project\dirs.proj"] = true;
@@ -1088,8 +851,7 @@ namespace Semmle.Autobuild.CSharp.Tests
 </Project>");
             actions.LoadXml[@"C:\Project\dirs.proj"] = dirsproj;
 
-            var autobuilder = CreateAutoBuilder(true, msBuildArguments: "/P:Fu=Bar", msBuildTarget: "Windows", msBuildPlatform: "x86", msBuildConfiguration: "Debug",
-                vsToolsVersion: "12", allSolutions: "true");
+            var autobuilder = CreateAutoBuilder(true);
             TestAutobuilderScript(autobuilder, 0, 3);
         }
 

csharp/autobuilder/Semmle.Autobuild.CSharp/CSharpAutobuilder.cs

@@ -25,9 +25,11 @@ namespace Semmle.Autobuild.CSharp
         /// </summary>
         public CSharpAutobuildOptions(IBuildActions actions) : base(actions)
         {
-            Buildless = actions.GetEnvironmentVariable(lgtmPrefix + "BUILDLESS").AsBool("buildless", false) ||
+            Buildless =
                 actions.GetEnvironmentVariable(extractorOptionPrefix + "BUILDLESS").AsBool("buildless", false) ||
                 actions.GetEnvironmentVariable(buildModeEnvironmentVariable)?.ToLower() == "none";
+
+
         }
     }
 
@@ -46,21 +48,12 @@ namespace Semmle.Autobuild.CSharp
             var attempt = BuildScript.Failure;
             switch (GetCSharpBuildStrategy())
             {
-                case CSharpBuildStrategy.CustomBuildCommand:
-                    attempt = new BuildCommandRule(DotNetRule.WithDotNet).Analyse(this, false) & CheckExtractorRun(true);
-                    break;
                 case CSharpBuildStrategy.Buildless:
                     // No need to check that the extractor has been executed in buildless mode
                     attempt = BuildScript.Bind(
                         AddBuildlessStartedDiagnostic() & new StandaloneBuildRule().Analyse(this, false),
                         AddBuildlessEndedDiagnostic);
                     break;
-                case CSharpBuildStrategy.MSBuild:
-                    attempt = new MsBuildRule().Analyse(this, false) & CheckExtractorRun(true);
-                    break;
-                case CSharpBuildStrategy.DotNet:
-                    attempt = new DotNetRule().Analyse(this, false) & CheckExtractorRun(true);
-                    break;
                 case CSharpBuildStrategy.Auto:
                     attempt =
                         // Attempt a few different build strategies to see if one works
@@ -246,32 +239,15 @@ namespace Semmle.Autobuild.CSharp
         /// </summary>
         private CSharpBuildStrategy GetCSharpBuildStrategy()
         {
-            if (Options.BuildCommand is not null)
-                return CSharpBuildStrategy.CustomBuildCommand;
-
             if (Options.Buildless)
                 return CSharpBuildStrategy.Buildless;
 
-            if (Options.MsBuildArguments is not null
-                || Options.MsBuildConfiguration is not null
-                || Options.MsBuildPlatform is not null
-                || Options.MsBuildTarget is not null)
-            {
-                return CSharpBuildStrategy.MSBuild;
-            }
-
-            if (Options.DotNetArguments is not null || Options.DotNetVersion is not null)
-                return CSharpBuildStrategy.DotNet;
-
             return CSharpBuildStrategy.Auto;
         }
 
         private enum CSharpBuildStrategy
         {
-            CustomBuildCommand,
             Buildless,
-            MSBuild,
-            DotNet,
             Auto
         }
     }

csharp/autobuilder/Semmle.Autobuild.CSharp/DotNetRule.cs

@@ -32,7 +32,7 @@ namespace Semmle.Autobuild.CSharp
             if (auto)
             {
                 NotDotNetProjects = builder.ProjectsOrSolutionsToBuild
-                    .SelectMany(p => Enumerators.Singleton(p).Concat(p.IncludedProjects))
+                    .SelectMany(p => new[] { p }.Concat(p.IncludedProjects))
                     .OfType<Project<CSharpAutobuildOptions>>()
                     .Where(p => !p.DotNetProject);
                 var notDotNetProject = NotDotNetProjects.FirstOrDefault();
@@ -150,8 +150,7 @@ namespace Semmle.Autobuild.CSharp
                 Argument("--no-incremental");
 
             return
-                script.Argument(builder.Options.DotNetArguments).
-                    QuoteArgument(projOrSln).
+                script.QuoteArgument(projOrSln).
                     Script;
         }
     }

csharp/autobuilder/Semmle.Autobuild.CSharp/Program.cs

@@ -17,7 +17,7 @@ namespace Semmle.Autobuild.CSharp
                 try
                 {
                     Console.WriteLine("CodeQL C# autobuilder");
-                    var builder = new CSharpAutobuilder(actions, options);
+                    using var builder = new CSharpAutobuilder(actions, options);
                     return builder.AttemptBuild();
                 }
                 catch (InvalidEnvironmentException ex)

csharp/autobuilder/Semmle.Autobuild.Shared/AutobuildOptions.cs

@@ -11,24 +11,9 @@ namespace Semmle.Autobuild.Shared
     /// </summary>
     public abstract class AutobuildOptionsShared
     {
-        protected const string lgtmPrefix = "LGTM_INDEX_";
-
-
         public int SearchDepth { get; } = 3;
         public string RootDirectory { get; }
-        public string? VsToolsVersion { get; }
-        public string? MsBuildArguments { get; }
-        public string? MsBuildPlatform { get; }
-        public string? MsBuildConfiguration { get; }
-        public string? MsBuildTarget { get; }
-        public string? DotNetArguments { get; }
         public string? DotNetVersion { get; }
-        public string? BuildCommand { get; }
-        public IEnumerable<string> Solution { get; }
-        public bool IgnoreErrors { get; }
-
-        public bool AllSolutions { get; }
-        public bool NugetRestore { get; }
         public abstract Language Language { get; }
 
         /// <summary>
@@ -38,19 +23,7 @@ namespace Semmle.Autobuild.Shared
         public AutobuildOptionsShared(IBuildActions actions)
         {
             RootDirectory = actions.GetCurrentDirectory();
-            VsToolsVersion = actions.GetEnvironmentVariable(lgtmPrefix + "VSTOOLS_VERSION");
-            MsBuildArguments = actions.GetEnvironmentVariable(lgtmPrefix + "MSBUILD_ARGUMENTS")?.AsStringWithExpandedEnvVars(actions);
-            MsBuildPlatform = actions.GetEnvironmentVariable(lgtmPrefix + "MSBUILD_PLATFORM");
-            MsBuildConfiguration = actions.GetEnvironmentVariable(lgtmPrefix + "MSBUILD_CONFIGURATION");
-            MsBuildTarget = actions.GetEnvironmentVariable(lgtmPrefix + "MSBUILD_TARGET");
-            DotNetArguments = actions.GetEnvironmentVariable(lgtmPrefix + "DOTNET_ARGUMENTS")?.AsStringWithExpandedEnvVars(actions);
-            DotNetVersion = actions.GetEnvironmentVariable(lgtmPrefix + "DOTNET_VERSION");
-            BuildCommand = actions.GetEnvironmentVariable(lgtmPrefix + "BUILD_COMMAND");
-            Solution = actions.GetEnvironmentVariable(lgtmPrefix + "SOLUTION").AsListWithExpandedEnvVars(actions, Array.Empty<string>());
-
-            IgnoreErrors = actions.GetEnvironmentVariable(lgtmPrefix + "IGNORE_ERRORS").AsBool("ignore_errors", false);
-            AllSolutions = actions.GetEnvironmentVariable(lgtmPrefix + "ALL_SOLUTIONS").AsBool("all_solutions", false);
-            NugetRestore = actions.GetEnvironmentVariable(lgtmPrefix + "NUGET_RESTORE").AsBool("nuget_restore", true);
+            DotNetVersion = actions.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_OPTION_DOTNET_VERSION");
         }
     }
 

csharp/autobuilder/Semmle.Autobuild.Shared/Autobuilder.cs

@@ -92,7 +92,7 @@ namespace Semmle.Autobuild.Shared
     /// The overall design is intended to be extensible so that in theory,
     /// it should be possible to add new build rules without touching this code.
     /// </summary>
-    public abstract class Autobuilder<TAutobuildOptions> : IAutobuilder<TAutobuildOptions> where TAutobuildOptions : AutobuildOptionsShared
+    public abstract class Autobuilder<TAutobuildOptions> : IDisposable, IAutobuilder<TAutobuildOptions> where TAutobuildOptions : AutobuildOptionsShared
     {
         /// <summary>
         /// Full file paths of files found in the project directory, as well as
@@ -161,9 +161,6 @@ namespace Semmle.Autobuild.Shared
             if (matchingFiles.Length == 0)
                 return null;
 
-            if (Options.AllSolutions)
-                return matchingFiles.Select(p => p.ProjectOrSolution);
-
             return matchingFiles
                 .Where(f => f.DistanceFromRoot == matchingFiles[0].DistanceFromRoot)
                 .Select(f => f.ProjectOrSolution);
@@ -185,19 +182,6 @@ namespace Semmle.Autobuild.Shared
             projectsOrSolutionsToBuildLazy = new Lazy<IList<IProjectOrSolution>>(() =>
             {
                 List<IProjectOrSolution>? ret;
-                if (options.Solution.Any())
-                {
-                    ret = new List<IProjectOrSolution>();
-                    foreach (var solution in options.Solution)
-                    {
-                        if (actions.FileExists(solution))
-                            ret.Add(new Solution<TAutobuildOptions>(this, solution, true));
-                        else
-                            logger.LogError($"The specified project or solution file {solution} was not found");
-                    }
-                    return ret;
-                }
-
                 // First look for `.proj` files
                 ret = FindFiles(".proj", f => new Project<TAutobuildOptions>(this, f))?.ToList();
                 if (ret is not null)
@@ -285,9 +269,6 @@ namespace Semmle.Autobuild.Shared
 
             var script = GetBuildScript();
 
-            if (Options.IgnoreErrors)
-                script |= BuildScript.Success;
-
             void startCallback(string s, bool silent)
             {
                 logger.Log(silent ? Severity.Debug : Severity.Info, $"\nRunning {s}");
@@ -370,6 +351,20 @@ namespace Semmle.Autobuild.Shared
             }
         });
 
+        public void Dispose()
+        {
+            Dispose(true);
+            GC.SuppressFinalize(this);
+        }
+
+        protected virtual void Dispose(bool disposing)
+        {
+            if (disposing)
+            {
+                diagnostics.Dispose();
+            }
+        }
+
         /// <summary>
         /// Value of CODEQL_EXTRACTOR_<LANG>_ROOT environment variable.
         /// </summary>

csharp/autobuilder/Semmle.Autobuild.Shared/BuildCommandAutoRule.cs

@@ -82,11 +82,6 @@ namespace Semmle.Autobuild.Shared
             {
                 var command = new CommandBuilder(builder.Actions, dir, environment);
 
-                // A specific Visual Studio version may be required
-                var vsTools = MsBuildRule.GetVcVarsBatFile(builder);
-                if (vsTools is not null)
-                    command.CallBatFile(vsTools.Path);
-
                 command.RunCommand(this.ScriptPath);
                 return command.Script;
             });

csharp/autobuilder/Semmle.Autobuild.Shared/BuildCommandRule.cs

@@ -1,37 +0,0 @@
-using Semmle.Util;
-
-namespace Semmle.Autobuild.Shared
-{
-    /// <summary>
-    /// Execute the build_command rule.
-    /// </summary>
-    public class BuildCommandRule : IBuildRule<AutobuildOptionsShared>
-    {
-        private readonly WithDotNet<AutobuildOptionsShared> withDotNet;
-
-        public BuildCommandRule(WithDotNet<AutobuildOptionsShared> withDotNet)
-        {
-            this.withDotNet = withDotNet;
-        }
-
-        public BuildScript Analyse(IAutobuilder<AutobuildOptionsShared> builder, bool auto)
-        {
-            if (builder.Options.BuildCommand is null)
-                return BuildScript.Failure;
-
-            // Custom build commands may require a specific .NET Core version
-            return withDotNet(builder, environment =>
-                {
-                    var command = new CommandBuilder(builder.Actions, null, environment);
-
-                    // Custom build commands may require a specific Visual Studio version
-                    var vsTools = MsBuildRule.GetVcVarsBatFile(builder);
-                    if (vsTools is not null)
-                        command.CallBatFile(vsTools.Path);
-                    command.RunCommand(builder.Options.BuildCommand);
-
-                    return command.Script;
-                });
-        }
-    }
-}

csharp/autobuilder/Semmle.Autobuild.Shared/MsBuildRule.cs

@@ -42,9 +42,9 @@ namespace Semmle.Autobuild.Shared
             if (auto)
                 builder.Logger.LogInfo("Attempting to build using MSBuild");
 
-            var vsTools = GetVcVarsBatFile(builder);
+            VcVarsBatFile? vsTools = null;
 
-            if (vsTools is null && builder.ProjectsOrSolutionsToBuild.Any())
+            if (builder.ProjectsOrSolutionsToBuild.Any())
             {
                 var firstSolution = builder.ProjectsOrSolutionsToBuild.OfType<ISolution>().FirstOrDefault();
                 vsTools = firstSolution is not null
@@ -67,46 +67,44 @@ namespace Semmle.Autobuild.Shared
 
             foreach (var projectOrSolution in builder.ProjectsOrSolutionsToBuild)
             {
-                if (builder.Options.NugetRestore)
+
+                BuildScript GetNugetRestoreScript() =>
+                    new CommandBuilder(builder.Actions).
+                        RunCommand(nuget).
+                        Argument("restore").
+                        QuoteArgument(projectOrSolution.FullPath).
+                        Argument("-DisableParallelProcessing").
+                        Script;
+                var nugetRestore = GetNugetRestoreScript();
+                var msbuildRestoreCommand = new CommandBuilder(builder.Actions).
+                    MsBuildCommand(builder).
+                    Argument("/t:restore").
+                    QuoteArgument(projectOrSolution.FullPath);
+
+                if (builder.Actions.IsRunningOnAppleSilicon())
                 {
-                    BuildScript GetNugetRestoreScript() =>
-                        new CommandBuilder(builder.Actions).
-                            RunCommand(nuget).
-                            Argument("restore").
-                            QuoteArgument(projectOrSolution.FullPath).
-                            Argument("-DisableParallelProcessing").
-                            Script;
-                    var nugetRestore = GetNugetRestoreScript();
-                    var msbuildRestoreCommand = new CommandBuilder(builder.Actions).
-                        MsBuildCommand(builder).
-                        Argument("/t:restore").
-                        QuoteArgument(projectOrSolution.FullPath);
+                    // On Apple Silicon, only try package restore with `dotnet msbuild /t:restore`
+                    ret &= BuildScript.Try(msbuildRestoreCommand.Script);
+                }
+                else if (nugetDownloaded)
+                {
+                    ret &= BuildScript.Try(nugetRestore | msbuildRestoreCommand.Script);
+                }
+                else
+                {
+                    // If `nuget restore` fails, and we have not already attempted to download `nuget.exe`,
+                    // download it and reattempt `nuget restore`.
+                    var nugetDownloadAndRestore =
+                        BuildScript.Bind(DownloadNugetExe(builder, nugetDownloadPath), exitCode =>
+                        {
+                            nugetDownloaded = true;
+                            if (exitCode != 0)
+                                return BuildScript.Failure;
 
-                    if (builder.Actions.IsRunningOnAppleSilicon())
-                    {
-                        // On Apple Silicon, only try package restore with `dotnet msbuild /t:restore`
-                        ret &= BuildScript.Try(msbuildRestoreCommand.Script);
-                    }
-                    else if (nugetDownloaded)
-                    {
-                        ret &= BuildScript.Try(nugetRestore | msbuildRestoreCommand.Script);
-                    }
-                    else
-                    {
-                        // If `nuget restore` fails, and we have not already attempted to download `nuget.exe`,
-                        // download it and reattempt `nuget restore`.
-                        var nugetDownloadAndRestore =
-                            BuildScript.Bind(DownloadNugetExe(builder, nugetDownloadPath), exitCode =>
-                            {
-                                nugetDownloaded = true;
-                                if (exitCode != 0)
-                                    return BuildScript.Failure;
-
-                                nuget = nugetDownloadPath;
-                                return GetNugetRestoreScript();
-                            });
-                        ret &= BuildScript.Try(nugetRestore | nugetDownloadAndRestore | msbuildRestoreCommand.Script);
-                    }
+                            nuget = nugetDownloadPath;
+                            return GetNugetRestoreScript();
+                        });
+                    ret &= BuildScript.Try(nugetRestore | nugetDownloadAndRestore | msbuildRestoreCommand.Script);
                 }
 
                 var command = new CommandBuilder(builder.Actions);
@@ -124,9 +122,9 @@ namespace Semmle.Autobuild.Shared
                 command.MsBuildCommand(builder);
                 command.QuoteArgument(projectOrSolution.FullPath);
 
-                var target = builder.Options.MsBuildTarget ?? "rebuild";
-                var platform = builder.Options.MsBuildPlatform ?? (projectOrSolution is ISolution s1 ? s1.DefaultPlatformName : null);
-                var configuration = builder.Options.MsBuildConfiguration ?? (projectOrSolution is ISolution s2 ? s2.DefaultConfigurationName : null);
+                var target = "rebuild";
+                var platform = projectOrSolution is ISolution s1 ? s1.DefaultPlatformName : null;
+                var configuration = projectOrSolution is ISolution s2 ? s2.DefaultConfigurationName : null;
 
                 command.Argument("/t:" + target);
                 if (platform is not null)
@@ -134,8 +132,6 @@ namespace Semmle.Autobuild.Shared
                 if (configuration is not null)
                     command.Argument(string.Format("/p:Configuration=\"{0}\"", configuration));
 
-                command.Argument(builder.Options.MsBuildArguments);
-
                 // append the build script which invokes msbuild to the overall build script `ret`;
                 // we insert a check that building the current project or solution was successful:
                 // if it was not successful, we add it to `FailedProjectsOrSolutions`
@@ -148,41 +144,6 @@ namespace Semmle.Autobuild.Shared
             return ret;
         }
 
-        /// <summary>
-        /// Gets the BAT file used to initialize the appropriate Visual Studio
-        /// version/platform, as specified by the `vstools_version` property in
-        /// lgtm.yml.
-        ///
-        /// Returns <code>null</code> when no version is specified.
-        /// </summary>
-        public static VcVarsBatFile? GetVcVarsBatFile<TAutobuildOptions>(IAutobuilder<TAutobuildOptions> builder) where TAutobuildOptions : AutobuildOptionsShared
-        {
-            VcVarsBatFile? vsTools = null;
-
-            if (builder.Options.VsToolsVersion is not null)
-            {
-                if (int.TryParse(builder.Options.VsToolsVersion, out var msToolsVersion))
-                {
-                    foreach (var b in BuildTools.VcVarsAllBatFiles(builder.Actions))
-                    {
-                        builder.Logger.Log(Severity.Info, "Found {0} version {1}", b.Path, b.ToolsVersion);
-                    }
-
-                    vsTools = BuildTools.FindCompatibleVcVars(builder.Actions, msToolsVersion);
-                    if (vsTools is null)
-                        builder.Logger.LogWarning("Could not find build tools matching version {0}", msToolsVersion);
-                    else
-                        builder.Logger.Log(Severity.Info, "Setting Visual Studio tools to {0}", vsTools.Path);
-                }
-                else
-                {
-                    builder.Logger.LogError("The format of vstools_version is incorrect. Please specify an integer.");
-                }
-            }
-
-            return vsTools;
-        }
-
         /// <summary>
         /// Returns a script for downloading `nuget.exe` from nuget.org.
         /// </summary>

csharp/documentation/library-coverage/coverage.csv

@@ -1,44 +1,44 @@
-package,sink,source,summary,sink:code-injection,sink:encryption-decryptor,sink:encryption-encryptor,sink:encryption-keyprop,sink:encryption-symmetrickey,sink:file-content-store,sink:html-injection,sink:js-injection,sink:log-injection,sink:sql-injection,source:commandargs,source:environment,source:file,source:file-write,source:local,source:remote,source:windows-registry,summary:taint,summary:value
-Amazon.Lambda.APIGatewayEvents,,6,,,,,,,,,,,,,,,,,6,,,
-Amazon.Lambda.Core,10,,,,,,,,,,,10,,,,,,,,,,
-Dapper,55,,,,,,,,,,,,55,,,,,,,,,
-ILCompiler,,,81,,,,,,,,,,,,,,,,,,81,
-ILLink.RoslynAnalyzer,,,63,,,,,,,,,,,,,,,,,,63,
-ILLink.Shared,,,32,,,,,,,,,,,,,,,,,,29,3
-ILLink.Tasks,,,5,,,,,,,,,,,,,,,,,,5,
-Internal.IL,,,69,,,,,,,,,,,,,,,,,,67,2
-Internal.Pgo,,,9,,,,,,,,,,,,,,,,,,8,1
-Internal.TypeSystem,,,367,,,,,,,,,,,,,,,,,,331,36
-JsonToItemsTaskFactory,,,7,,,,,,,,,,,,,,,,,,7,
-Microsoft.Android.Build,,,14,,,,,,,,,,,,,,,,,,14,
-Microsoft.Apple.Build,,,7,,,,,,,,,,,,,,,,,,7,
-Microsoft.ApplicationBlocks.Data,28,,,,,,,,,,,,28,,,,,,,,,
-Microsoft.CSharp,,,24,,,,,,,,,,,,,,,,,,24,
-Microsoft.Diagnostics.Tools.Pgo,,,13,,,,,,,,,,,,,,,,,,13,
-Microsoft.EntityFrameworkCore,6,,12,,,,,,,,,,6,,,,,,,,,12
-Microsoft.Extensions.Caching.Distributed,,,15,,,,,,,,,,,,,,,,,,15,
-Microsoft.Extensions.Caching.Memory,,,38,,,,,,,,,,,,,,,,,,37,1
-Microsoft.Extensions.Configuration,,2,89,,,,,,,,,,,,2,,,,,,86,3
-Microsoft.Extensions.DependencyInjection,,,120,,,,,,,,,,,,,,,,,,120,
-Microsoft.Extensions.DependencyModel,,,12,,,,,,,,,,,,,,,,,,12,
-Microsoft.Extensions.Diagnostics.Metrics,,,13,,,,,,,,,,,,,,,,,,13,
-Microsoft.Extensions.FileProviders,,,15,,,,,,,,,,,,,,,,,,15,
-Microsoft.Extensions.FileSystemGlobbing,,,16,,,,,,,,,,,,,,,,,,14,2
-Microsoft.Extensions.Hosting,,,23,,,,,,,,,,,,,,,,,,22,1
-Microsoft.Extensions.Http,,,10,,,,,,,,,,,,,,,,,,10,
-Microsoft.Extensions.Logging,,,60,,,,,,,,,,,,,,,,,,59,1
-Microsoft.Extensions.Options,,,8,,,,,,,,,,,,,,,,,,8,
-Microsoft.Extensions.Primitives,,,64,,,,,,,,,,,,,,,,,,64,
-Microsoft.Interop,,,78,,,,,,,,,,,,,,,,,,78,
-Microsoft.NET.Build.Tasks,,,1,,,,,,,,,,,,,,,,,,1,
-Microsoft.NET.WebAssembly.Webcil,,,7,,,,,,,,,,,,,,,,,,7,
-Microsoft.VisualBasic,,,10,,,,,,,,,,,,,,,,,,5,5
-Microsoft.WebAssembly.Build.Tasks,,,3,,,,,,,,,,,,,,,,,,3,
-Microsoft.Win32,,4,4,,,,,,,,,,,,,,,,,4,4,
-Mono.Linker,,,163,,,,,,,,,,,,,,,,,,163,
-MySql.Data.MySqlClient,48,,,,,,,,,,,,48,,,,,,,,,
-Newtonsoft.Json,,,91,,,,,,,,,,,,,,,,,,73,18
-ServiceStack,194,,7,27,,,,,75,,,,92,,,,,,,,7,
-SourceGenerators,,,4,,,,,,,,,,,,,,,,,,4,
-System,67,30,11864,,8,8,9,,,4,5,,33,2,3,1,17,3,4,,9898,1966
-Windows.Security.Cryptography.Core,1,,,,,,,1,,,,,,,,,,,,,,
+package,sink,source,summary,sink:code-injection,sink:encryption-decryptor,sink:encryption-encryptor,sink:encryption-keyprop,sink:encryption-symmetrickey,sink:file-content-store,sink:html-injection,sink:js-injection,sink:log-injection,sink:sql-injection,source:commandargs,source:database,source:environment,source:file,source:file-write,source:local,source:remote,source:windows-registry,summary:taint,summary:value
+Amazon.Lambda.APIGatewayEvents,,6,,,,,,,,,,,,,,,,,,6,,,
+Amazon.Lambda.Core,10,,,,,,,,,,,10,,,,,,,,,,,
+Dapper,55,42,1,,,,,,,,,,55,,42,,,,,,,,1
+ILCompiler,,,81,,,,,,,,,,,,,,,,,,,81,
+ILLink.RoslynAnalyzer,,,63,,,,,,,,,,,,,,,,,,,63,
+ILLink.Shared,,,32,,,,,,,,,,,,,,,,,,,29,3
+ILLink.Tasks,,,5,,,,,,,,,,,,,,,,,,,5,
+Internal.IL,,,69,,,,,,,,,,,,,,,,,,,67,2
+Internal.Pgo,,,9,,,,,,,,,,,,,,,,,,,8,1
+Internal.TypeSystem,,,367,,,,,,,,,,,,,,,,,,,331,36
+JsonToItemsTaskFactory,,,7,,,,,,,,,,,,,,,,,,,7,
+Microsoft.Android.Build,,,14,,,,,,,,,,,,,,,,,,,14,
+Microsoft.Apple.Build,,,7,,,,,,,,,,,,,,,,,,,7,
+Microsoft.ApplicationBlocks.Data,28,,,,,,,,,,,,28,,,,,,,,,,
+Microsoft.CSharp,,,24,,,,,,,,,,,,,,,,,,,24,
+Microsoft.Diagnostics.Tools.Pgo,,,13,,,,,,,,,,,,,,,,,,,13,
+Microsoft.EntityFrameworkCore,6,,12,,,,,,,,,,6,,,,,,,,,,12
+Microsoft.Extensions.Caching.Distributed,,,15,,,,,,,,,,,,,,,,,,,15,
+Microsoft.Extensions.Caching.Memory,,,38,,,,,,,,,,,,,,,,,,,37,1
+Microsoft.Extensions.Configuration,,2,89,,,,,,,,,,,,,2,,,,,,86,3
+Microsoft.Extensions.DependencyInjection,,,120,,,,,,,,,,,,,,,,,,,120,
+Microsoft.Extensions.DependencyModel,,,12,,,,,,,,,,,,,,,,,,,12,
+Microsoft.Extensions.Diagnostics.Metrics,,,13,,,,,,,,,,,,,,,,,,,13,
+Microsoft.Extensions.FileProviders,,,15,,,,,,,,,,,,,,,,,,,15,
+Microsoft.Extensions.FileSystemGlobbing,,,16,,,,,,,,,,,,,,,,,,,14,2
+Microsoft.Extensions.Hosting,,,23,,,,,,,,,,,,,,,,,,,22,1
+Microsoft.Extensions.Http,,,10,,,,,,,,,,,,,,,,,,,10,
+Microsoft.Extensions.Logging,,,60,,,,,,,,,,,,,,,,,,,59,1
+Microsoft.Extensions.Options,,,8,,,,,,,,,,,,,,,,,,,8,
+Microsoft.Extensions.Primitives,,,64,,,,,,,,,,,,,,,,,,,64,
+Microsoft.Interop,,,78,,,,,,,,,,,,,,,,,,,78,
+Microsoft.NET.Build.Tasks,,,1,,,,,,,,,,,,,,,,,,,1,
+Microsoft.NET.WebAssembly.Webcil,,,7,,,,,,,,,,,,,,,,,,,7,
+Microsoft.VisualBasic,,,10,,,,,,,,,,,,,,,,,,,5,5
+Microsoft.WebAssembly.Build.Tasks,,,3,,,,,,,,,,,,,,,,,,,3,
+Microsoft.Win32,,4,4,,,,,,,,,,,,,,,,,,4,4,
+Mono.Linker,,,163,,,,,,,,,,,,,,,,,,,163,
+MySql.Data.MySqlClient,48,,,,,,,,,,,,48,,,,,,,,,,
+Newtonsoft.Json,,,91,,,,,,,,,,,,,,,,,,,73,18
+ServiceStack,194,,7,27,,,,,75,,,,92,,,,,,,,,7,
+SourceGenerators,,,4,,,,,,,,,,,,,,,,,,,4,
+System,67,44,11872,,8,8,9,,,4,5,,33,2,,3,15,17,3,4,,9906,1966
+Windows.Security.Cryptography.Core,1,,,,,,,1,,,,,,,,,,,,,,,

csharp/documentation/library-coverage/coverage.rst

@@ -8,7 +8,7 @@ C# framework & library support
 
    Framework / library,Package,Flow sources,Taint & value steps,Sinks (total),`CWE-079` :sub:`Cross-site scripting`
    `ServiceStack <https://servicestack.net/>`_,"``ServiceStack.*``, ``ServiceStack``",,7,194,
-   System,"``System.*``, ``System``",30,11864,67,9
-   Others,"``Amazon.Lambda.APIGatewayEvents``, ``Amazon.Lambda.Core``, ``Dapper``, ``ILCompiler``, ``ILLink.RoslynAnalyzer``, ``ILLink.Shared``, ``ILLink.Tasks``, ``Internal.IL``, ``Internal.Pgo``, ``Internal.TypeSystem``, ``JsonToItemsTaskFactory``, ``Microsoft.Android.Build``, ``Microsoft.Apple.Build``, ``Microsoft.ApplicationBlocks.Data``, ``Microsoft.CSharp``, ``Microsoft.Diagnostics.Tools.Pgo``, ``Microsoft.EntityFrameworkCore``, ``Microsoft.Extensions.Caching.Distributed``, ``Microsoft.Extensions.Caching.Memory``, ``Microsoft.Extensions.Configuration``, ``Microsoft.Extensions.DependencyInjection``, ``Microsoft.Extensions.DependencyModel``, ``Microsoft.Extensions.Diagnostics.Metrics``, ``Microsoft.Extensions.FileProviders``, ``Microsoft.Extensions.FileSystemGlobbing``, ``Microsoft.Extensions.Hosting``, ``Microsoft.Extensions.Http``, ``Microsoft.Extensions.Logging``, ``Microsoft.Extensions.Options``, ``Microsoft.Extensions.Primitives``, ``Microsoft.Interop``, ``Microsoft.NET.Build.Tasks``, ``Microsoft.NET.WebAssembly.Webcil``, ``Microsoft.VisualBasic``, ``Microsoft.WebAssembly.Build.Tasks``, ``Microsoft.Win32``, ``Mono.Linker``, ``MySql.Data.MySqlClient``, ``Newtonsoft.Json``, ``SourceGenerators``, ``Windows.Security.Cryptography.Core``",12,1547,148,
-   Totals,,42,13418,409,9
+   System,"``System.*``, ``System``",44,11872,67,9
+   Others,"``Amazon.Lambda.APIGatewayEvents``, ``Amazon.Lambda.Core``, ``Dapper``, ``ILCompiler``, ``ILLink.RoslynAnalyzer``, ``ILLink.Shared``, ``ILLink.Tasks``, ``Internal.IL``, ``Internal.Pgo``, ``Internal.TypeSystem``, ``JsonToItemsTaskFactory``, ``Microsoft.Android.Build``, ``Microsoft.Apple.Build``, ``Microsoft.ApplicationBlocks.Data``, ``Microsoft.CSharp``, ``Microsoft.Diagnostics.Tools.Pgo``, ``Microsoft.EntityFrameworkCore``, ``Microsoft.Extensions.Caching.Distributed``, ``Microsoft.Extensions.Caching.Memory``, ``Microsoft.Extensions.Configuration``, ``Microsoft.Extensions.DependencyInjection``, ``Microsoft.Extensions.DependencyModel``, ``Microsoft.Extensions.Diagnostics.Metrics``, ``Microsoft.Extensions.FileProviders``, ``Microsoft.Extensions.FileSystemGlobbing``, ``Microsoft.Extensions.Hosting``, ``Microsoft.Extensions.Http``, ``Microsoft.Extensions.Logging``, ``Microsoft.Extensions.Options``, ``Microsoft.Extensions.Primitives``, ``Microsoft.Interop``, ``Microsoft.NET.Build.Tasks``, ``Microsoft.NET.WebAssembly.Webcil``, ``Microsoft.VisualBasic``, ``Microsoft.WebAssembly.Build.Tasks``, ``Microsoft.Win32``, ``Mono.Linker``, ``MySql.Data.MySqlClient``, ``Newtonsoft.Json``, ``SourceGenerators``, ``Windows.Security.Cryptography.Core``",54,1548,148,
+   Totals,,98,13427,409,9
 

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs

@@ -0,0 +1,514 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Net.Http;
+using System.Text.RegularExpressions;
+using System.Threading;
+using System.Threading.Tasks;
+using Semmle.Util;
+
+namespace Semmle.Extraction.CSharp.DependencyFetching
+{
+    public sealed partial class DependencyManager
+    {
+        private void RestoreNugetPackages(List<FileInfo> allNonBinaryFiles, IEnumerable<string> allProjects, IEnumerable<string> allSolutions, HashSet<string> dllPaths)
+        {
+            try
+            {
+                var checkNugetFeedResponsiveness = EnvironmentVariables.GetBoolean(EnvironmentVariableNames.CheckNugetFeedResponsiveness);
+                if (checkNugetFeedResponsiveness && !CheckFeeds(allNonBinaryFiles))
+                {
+                    DownloadMissingPackages(allNonBinaryFiles, dllPaths, withNugetConfig: false);
+                    return;
+                }
+
+                using (var nuget = new NugetPackages(sourceDir.FullName, legacyPackageDirectory, logger))
+                {
+                    var count = nuget.InstallPackages();
+
+                    if (nuget.PackageCount > 0)
+                    {
+                        CompilationInfos.Add(("packages.config files", nuget.PackageCount.ToString()));
+                        CompilationInfos.Add(("Successfully restored packages.config files", count.ToString()));
+                    }
+                }
+
+                var nugetPackageDlls = legacyPackageDirectory.DirInfo.GetFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = true });
+                var nugetPackageDllPaths = nugetPackageDlls.Select(f => f.FullName).ToHashSet();
+                var excludedPaths = nugetPackageDllPaths
+                    .Where(path => IsPathInSubfolder(path, legacyPackageDirectory.DirInfo.FullName, "tools"))
+                    .ToList();
+
+                if (nugetPackageDllPaths.Count > 0)
+                {
+                    logger.LogInfo($"Restored {nugetPackageDllPaths.Count} Nuget DLLs.");
+                }
+                if (excludedPaths.Count > 0)
+                {
+                    logger.LogInfo($"Excluding {excludedPaths.Count} Nuget DLLs.");
+                }
+
+                foreach (var excludedPath in excludedPaths)
+                {
+                    logger.LogInfo($"Excluded Nuget DLL: {excludedPath}");
+                }
+
+                nugetPackageDllPaths.ExceptWith(excludedPaths);
+                dllPaths.UnionWith(nugetPackageDllPaths);
+            }
+            catch (Exception exc)
+            {
+                logger.LogError($"Failed to restore Nuget packages with nuget.exe: {exc.Message}");
+            }
+
+            var restoredProjects = RestoreSolutions(allSolutions, out var assets1);
+            var projects = allProjects.Except(restoredProjects);
+            RestoreProjects(projects, out var assets2);
+
+            var dependencies = Assets.GetCompilationDependencies(logger, assets1.Union(assets2));
+
+            var paths = dependencies
+                .Paths
+                .Select(d => Path.Combine(packageDirectory.DirInfo.FullName, d))
+                .ToList();
+            dllPaths.UnionWith(paths);
+
+            LogAllUnusedPackages(dependencies);
+            DownloadMissingPackages(allNonBinaryFiles, dllPaths);
+        }
+
+        /// <summary>
+        /// Executes `dotnet restore` on all solution files in solutions.
+        /// As opposed to RestoreProjects this is not run in parallel using PLINQ
+        /// as `dotnet restore` on a solution already uses multiple threads for restoring
+        /// the projects (this can be disabled with the `--disable-parallel` flag).
+        /// Populates assets with the relative paths to the assets files generated by the restore.
+        /// Returns a list of projects that are up to date with respect to restore.
+        /// </summary>
+        /// <param name="solutions">A list of paths to solution files.</param>
+        private IEnumerable<string> RestoreSolutions(IEnumerable<string> solutions, out IEnumerable<string> assets)
+        {
+            var successCount = 0;
+            var nugetSourceFailures = 0;
+            var assetFiles = new List<string>();
+            var projects = solutions.SelectMany(solution =>
+                {
+                    logger.LogInfo($"Restoring solution {solution}...");
+                    var res = dotnet.Restore(new(solution, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
+                    if (res.Success)
+                    {
+                        successCount++;
+                    }
+                    if (res.HasNugetPackageSourceError)
+                    {
+                        nugetSourceFailures++;
+                    }
+                    assetFiles.AddRange(res.AssetsFilePaths);
+                    return res.RestoredProjects;
+                }).ToList();
+            assets = assetFiles;
+            CompilationInfos.Add(("Successfully restored solution files", successCount.ToString()));
+            CompilationInfos.Add(("Failed solution restore with package source error", nugetSourceFailures.ToString()));
+            CompilationInfos.Add(("Restored projects through solution files", projects.Count.ToString()));
+            return projects;
+        }
+
+        /// <summary>
+        /// Executes `dotnet restore` on all projects in projects.
+        /// This is done in parallel for performance reasons.
+        /// Populates assets with the relative paths to the assets files generated by the restore.
+        /// </summary>
+        /// <param name="projects">A list of paths to project files.</param>
+        private void RestoreProjects(IEnumerable<string> projects, out IEnumerable<string> assets)
+        {
+            var successCount = 0;
+            var nugetSourceFailures = 0;
+            var assetFiles = new List<string>();
+            var sync = new object();
+            Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = threads }, project =>
+            {
+                logger.LogInfo($"Restoring project {project}...");
+                var res = dotnet.Restore(new(project, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
+                lock (sync)
+                {
+                    if (res.Success)
+                    {
+                        successCount++;
+                    }
+                    if (res.HasNugetPackageSourceError)
+                    {
+                        nugetSourceFailures++;
+                    }
+                    assetFiles.AddRange(res.AssetsFilePaths);
+                }
+            });
+            assets = assetFiles;
+            CompilationInfos.Add(("Successfully restored project files", successCount.ToString()));
+            CompilationInfos.Add(("Failed project restore with package source error", nugetSourceFailures.ToString()));
+        }
+
+        private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPaths, bool withNugetConfig = true)
+        {
+            var alreadyDownloadedPackages = GetRestoredPackageDirectoryNames(packageDirectory.DirInfo);
+            var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames();
+
+            var notYetDownloadedPackages = new HashSet<PackageReference>(fileContent.AllPackages);
+            foreach (var alreadyDownloadedPackage in alreadyDownloadedPackages)
+            {
+                notYetDownloadedPackages.Remove(new(alreadyDownloadedPackage, PackageReferenceSource.SdkCsProj));
+            }
+            foreach (var alreadyDownloadedLegacyPackage in alreadyDownloadedLegacyPackages)
+            {
+                notYetDownloadedPackages.Remove(new(alreadyDownloadedLegacyPackage, PackageReferenceSource.PackagesConfig));
+            }
+
+            if (notYetDownloadedPackages.Count == 0)
+            {
+                return;
+            }
+
+            var multipleVersions = notYetDownloadedPackages
+                .GroupBy(p => p.Name)
+                .Where(g => g.Count() > 1)
+                .Select(g => g.Key)
+                .ToList();
+
+            foreach (var package in multipleVersions)
+            {
+                logger.LogWarning($"Found multiple not yet restored packages with name '{package}'.");
+                notYetDownloadedPackages.Remove(new(package, PackageReferenceSource.PackagesConfig));
+            }
+
+            logger.LogInfo($"Found {notYetDownloadedPackages.Count} packages that are not yet restored");
+            var nugetConfig = withNugetConfig
+                ? GetNugetConfig(allFiles)
+                : null;
+
+            CompilationInfos.Add(("Fallback nuget restore", notYetDownloadedPackages.Count.ToString()));
+
+            var successCount = 0;
+            var sync = new object();
+
+            Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = threads }, package =>
+            {
+                var success = TryRestorePackageManually(package.Name, nugetConfig, package.PackageReferenceSource);
+                if (!success)
+                {
+                    return;
+                }
+
+                lock (sync)
+                {
+                    successCount++;
+                }
+            });
+
+            CompilationInfos.Add(("Successfully ran fallback nuget restore", successCount.ToString()));
+
+            dllPaths.Add(missingPackageDirectory.DirInfo.FullName);
+        }
+
+        private string[] GetAllNugetConfigs(List<FileInfo> allFiles) => allFiles.SelectFileNamesByName("nuget.config").ToArray();
+
+        private string? GetNugetConfig(List<FileInfo> allFiles)
+        {
+            var nugetConfigs = GetAllNugetConfigs(allFiles);
+            string? nugetConfig;
+            if (nugetConfigs.Length > 1)
+            {
+                logger.LogInfo($"Found multiple nuget.config files: {string.Join(", ", nugetConfigs)}.");
+                nugetConfig = allFiles
+                    .SelectRootFiles(sourceDir)
+                    .SelectFileNamesByName("nuget.config")
+                    .FirstOrDefault();
+                if (nugetConfig == null)
+                {
+                    logger.LogInfo("Could not find a top-level nuget.config file.");
+                }
+            }
+            else
+            {
+                nugetConfig = nugetConfigs.FirstOrDefault();
+            }
+
+            if (nugetConfig != null)
+            {
+                logger.LogInfo($"Using nuget.config file {nugetConfig}.");
+            }
+
+            return nugetConfig;
+        }
+
+        private void LogAllUnusedPackages(DependencyContainer dependencies)
+        {
+            var allPackageDirectories = GetAllPackageDirectories();
+
+            logger.LogInfo($"Restored {allPackageDirectories.Count} packages");
+            logger.LogInfo($"Found {dependencies.Packages.Count} packages in project.assets.json files");
+
+            allPackageDirectories
+                .Where(package => !dependencies.Packages.Contains(package))
+                .Order()
+                .ForEach(package => logger.LogInfo($"Unused package: {package}"));
+        }
+
+
+        private ICollection<string> GetAllPackageDirectories()
+        {
+            return new DirectoryInfo(packageDirectory.DirInfo.FullName)
+                .EnumerateDirectories("*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
+                .Select(d => d.Name)
+                .ToList();
+        }
+
+        private static bool IsPathInSubfolder(string path, string rootFolder, string subFolder)
+        {
+            return path.IndexOf(
+                $"{Path.DirectorySeparatorChar}{subFolder}{Path.DirectorySeparatorChar}",
+                rootFolder.Length,
+                StringComparison.InvariantCultureIgnoreCase) >= 0;
+        }
+
+        private IEnumerable<string> GetRestoredLegacyPackageNames()
+        {
+            var oldPackageDirectories = GetRestoredPackageDirectoryNames(legacyPackageDirectory.DirInfo);
+            foreach (var oldPackageDirectory in oldPackageDirectories)
+            {
+                // nuget install restores packages to 'packagename.version' folders (dotnet restore to 'packagename/version' folders)
+                // typical folder names look like:
+                //   newtonsoft.json.13.0.3
+                // there are more complex ones too, such as:
+                //   runtime.tizen.4.0.0-armel.Microsoft.NETCore.DotNetHostResolver.2.0.0-preview2-25407-01
+
+                var match = LegacyNugetPackage().Match(oldPackageDirectory);
+                if (!match.Success)
+                {
+                    logger.LogWarning($"Package directory '{oldPackageDirectory}' doesn't match the expected pattern.");
+                    continue;
+                }
+
+                yield return match.Groups[1].Value.ToLowerInvariant();
+            }
+        }
+
+        private static IEnumerable<string> GetRestoredPackageDirectoryNames(DirectoryInfo root)
+        {
+            return Directory.GetDirectories(root.FullName)
+                .Select(d => Path.GetFileName(d).ToLowerInvariant());
+        }
+
+        private bool TryRestorePackageManually(string package, string? nugetConfig, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj)
+        {
+            logger.LogInfo($"Restoring package {package}...");
+            using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package, "missingpackages_workingdir"));
+            var success = dotnet.New(tempDir.DirInfo.FullName);
+            if (!success)
+            {
+                return false;
+            }
+
+            if (packageReferenceSource == PackageReferenceSource.PackagesConfig)
+            {
+                TryChangeTargetFrameworkMoniker(tempDir.DirInfo);
+            }
+
+            success = dotnet.AddPackage(tempDir.DirInfo.FullName, package);
+            if (!success)
+            {
+                return false;
+            }
+
+            var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig));
+            if (!res.Success)
+            {
+                if (res.HasNugetPackageSourceError && nugetConfig is not null)
+                {
+                    // Restore could not be completed because the listed source is unavailable. Try without the nuget.config:
+                    res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: null, ForceReevaluation: true));
+                }
+
+                // TODO: the restore might fail, we could retry with
+                // - a prerelease (*-* instead of *) version of the package,
+                // - a different target framework moniker.
+
+                if (!res.Success)
+                {
+                    logger.LogInfo($"Failed to restore nuget package {package}");
+                    return false;
+                }
+            }
+
+            return true;
+        }
+
+        private void TryChangeTargetFrameworkMoniker(DirectoryInfo tempDir)
+        {
+            try
+            {
+                logger.LogInfo($"Changing the target framework moniker in {tempDir.FullName}...");
+
+                var csprojs = tempDir.GetFiles("*.csproj", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive });
+                if (csprojs.Length != 1)
+                {
+                    logger.LogError($"Could not find the .csproj file in {tempDir.FullName}, count = {csprojs.Length}");
+                    return;
+                }
+
+                var csproj = csprojs[0];
+                var content = File.ReadAllText(csproj.FullName);
+                var matches = TargetFramework().Matches(content);
+                if (matches.Count == 0)
+                {
+                    logger.LogError($"Could not find target framework in {csproj.FullName}");
+                }
+                else
+                {
+                    content = TargetFramework().Replace(content, $"<TargetFramework>{FrameworkPackageNames.LatestNetFrameworkMoniker}</TargetFramework>", 1);
+                    File.WriteAllText(csproj.FullName, content);
+                }
+            }
+            catch (Exception exc)
+            {
+                logger.LogError($"Failed to update target framework in {tempDir.FullName}: {exc}");
+            }
+        }
+
+        private static async Task ExecuteGetRequest(string address, HttpClient httpClient, CancellationToken cancellationToken)
+        {
+            using var stream = await httpClient.GetStreamAsync(address, cancellationToken);
+            var buffer = new byte[1024];
+            int bytesRead;
+            while ((bytesRead = stream.Read(buffer, 0, buffer.Length)) > 0)
+            {
+                // do nothing
+            }
+        }
+
+        private bool IsFeedReachable(string feed)
+        {
+            logger.LogInfo($"Checking if Nuget feed '{feed}' is reachable...");
+            using HttpClient client = new();
+            int timeoutMilliSeconds = int.TryParse(Environment.GetEnvironmentVariable(EnvironmentVariableNames.NugetFeedResponsivenessInitialTimeout), out timeoutMilliSeconds)
+                ? timeoutMilliSeconds
+                : 1000;
+            int tryCount = int.TryParse(Environment.GetEnvironmentVariable(EnvironmentVariableNames.NugetFeedResponsivenessRequestCount), out tryCount)
+                ? tryCount
+                : 4;
+
+            for (var i = 0; i < tryCount; i++)
+            {
+                using var cts = new CancellationTokenSource();
+                cts.CancelAfter(timeoutMilliSeconds);
+                try
+                {
+                    ExecuteGetRequest(feed, client, cts.Token).GetAwaiter().GetResult();
+                    return true;
+                }
+                catch (Exception exc)
+                {
+                    if (exc is TaskCanceledException tce &&
+                        tce.CancellationToken == cts.Token &&
+                        cts.Token.IsCancellationRequested)
+                    {
+                        logger.LogWarning($"Didn't receive answer from Nuget feed '{feed}' in {timeoutMilliSeconds}ms.");
+                        timeoutMilliSeconds *= 2;
+                        continue;
+                    }
+
+                    // We're only interested in timeouts.
+                    logger.LogWarning($"Querying Nuget feed '{feed}' failed: {exc}");
+                    return true;
+                }
+            }
+
+            logger.LogWarning($"Didn't receive answer from Nuget feed '{feed}'. Tried it {tryCount} times.");
+            return false;
+        }
+
+        private bool CheckFeeds(List<FileInfo> allFiles)
+        {
+            logger.LogInfo("Checking Nuget feeds...");
+            var feeds = GetAllFeeds(allFiles);
+
+            var excludedFeeds = Environment.GetEnvironmentVariable(EnvironmentVariableNames.ExcludedNugetFeedsFromResponsivenessCheck)
+                ?.Split(" ", StringSplitOptions.RemoveEmptyEntries)
+                .ToHashSet() ?? [];
+
+            if (excludedFeeds.Count > 0)
+            {
+                logger.LogInfo($"Excluded Nuget feeds from responsiveness check: {string.Join(", ", excludedFeeds.OrderBy(f => f))}");
+            }
+
+            var allFeedsReachable = feeds.All(feed => excludedFeeds.Contains(feed) || IsFeedReachable(feed));
+            if (!allFeedsReachable)
+            {
+                logger.LogWarning("Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.");
+                diagnosticsWriter.AddEntry(new DiagnosticMessage(
+                    Language.CSharp,
+                    "buildless/unreachable-feed",
+                    "Found unreachable Nuget feed in C# analysis with build-mode 'none'",
+                    visibility: new DiagnosticMessage.TspVisibility(statusPage: true, cliSummaryTable: true, telemetry: true),
+                    markdownMessage: "Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.",
+                    severity: DiagnosticMessage.TspSeverity.Warning
+                ));
+            }
+            CompilationInfos.Add(("All Nuget feeds reachable", allFeedsReachable ? "1" : "0"));
+            return allFeedsReachable;
+        }
+
+        private IEnumerable<string> GetFeeds(string nugetConfig)
+        {
+            logger.LogInfo($"Getting Nuget feeds from '{nugetConfig}'...");
+            var results = dotnet.GetNugetFeeds(nugetConfig);
+            var regex = EnabledNugetFeed();
+            foreach (var result in results)
+            {
+                var match = regex.Match(result);
+                if (!match.Success)
+                {
+                    logger.LogError($"Failed to parse feed from '{result}'");
+                    continue;
+                }
+
+                var url = match.Groups[1].Value;
+                if (!url.StartsWith("https://", StringComparison.InvariantCultureIgnoreCase) &&
+                    !url.StartsWith("http://", StringComparison.InvariantCultureIgnoreCase))
+                {
+                    logger.LogInfo($"Skipping feed '{url}' as it is not a valid URL.");
+                    continue;
+                }
+
+                yield return url;
+            }
+        }
+
+        private HashSet<string> GetAllFeeds(List<FileInfo> allFiles)
+        {
+            var nugetConfigs = GetAllNugetConfigs(allFiles);
+            var feeds = nugetConfigs
+                .SelectMany(GetFeeds)
+                .Where(str => !string.IsNullOrWhiteSpace(str))
+                .ToHashSet();
+
+            if (feeds.Count > 0)
+            {
+                logger.LogInfo($"Found {feeds.Count} Nuget feeds in nuget.config files: {string.Join(", ", feeds.OrderBy(f => f))}");
+            }
+            else
+            {
+                logger.LogDebug("No Nuget feeds found in nuget.config files.");
+            }
+            return feeds;
+        }
+
+        [GeneratedRegex(@"<TargetFramework>.*</TargetFramework>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
+        private static partial Regex TargetFramework();
+
+        [GeneratedRegex(@"^(.+)\.(\d+\.\d+\.\d+(-(.+))?)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
+        private static partial Regex LegacyNugetPackage();
+
+        [GeneratedRegex(@"^E\s(.*)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
+        private static partial Regex EnabledNugetFeed();
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -20,6 +20,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
     {
         private readonly AssemblyCache assemblyCache;
         private readonly ILogger logger;
+        private readonly IDiagnosticsWriter diagnosticsWriter;
 
         // Only used as a set, but ConcurrentDictionary is the only concurrent set in .NET.
         private readonly IDictionary<string, bool> usedReferences = new ConcurrentDictionary<string, bool>();
@@ -52,6 +53,25 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             var startTime = DateTime.Now;
 
             this.logger = logger;
+
+            var diagDirEnv = Environment.GetEnvironmentVariable(EnvironmentVariableNames.DiagnosticDir);
+            if (!string.IsNullOrWhiteSpace(diagDirEnv) &&
+                !Directory.Exists(diagDirEnv))
+            {
+                try
+                {
+                    Directory.CreateDirectory(diagDirEnv);
+                }
+                catch (Exception e)
+                {
+                    logger.LogError($"Failed to create diagnostic directory {diagDirEnv}: {e.Message}");
+                    diagDirEnv = null;
+                }
+            }
+
+            this.diagnosticsWriter = new DiagnosticsStream(Path.Combine(
+                diagDirEnv ?? "",
+                $"dependency-manager-{DateTime.UtcNow:yyyyMMddHHmm}-{Environment.ProcessId}.jsonc"));
             this.sourceDir = new DirectoryInfo(srcDir);
 
             packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName, "packages"));
@@ -177,8 +197,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             var frameworkLocations = new HashSet<string>();
 
             var frameworkReferences = Environment.GetEnvironmentVariable(EnvironmentVariableNames.DotnetFrameworkReferences);
-            var frameworkReferencesUseSubfolders = Environment.GetEnvironmentVariable(EnvironmentVariableNames.DotnetFrameworkReferencesUseSubfolders);
-            _ = bool.TryParse(frameworkReferencesUseSubfolders, out var useSubfolders);
+            var useSubfolders = EnvironmentVariables.GetBoolean(EnvironmentVariableNames.DotnetFrameworkReferencesUseSubfolders);
             if (!string.IsNullOrWhiteSpace(frameworkReferences))
             {
                 RemoveFrameworkNugetPackages(dllPaths);
@@ -230,73 +249,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             return frameworkLocations;
         }
 
-        private void RestoreNugetPackages(List<FileInfo> allNonBinaryFiles, IEnumerable<string> allProjects, IEnumerable<string> allSolutions, HashSet<string> dllPaths)
-        {
-            try
-            {
-                using (var nuget = new NugetPackages(sourceDir.FullName, legacyPackageDirectory, logger))
-                {
-                    var count = nuget.InstallPackages();
-
-                    if (nuget.PackageCount > 0)
-                    {
-                        CompilationInfos.Add(("packages.config files", nuget.PackageCount.ToString()));
-                        CompilationInfos.Add(("Successfully restored packages.config files", count.ToString()));
-                    }
-                }
-
-                var nugetPackageDlls = legacyPackageDirectory.DirInfo.GetFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = true });
-                var nugetPackageDllPaths = nugetPackageDlls.Select(f => f.FullName).ToHashSet();
-                var excludedPaths = nugetPackageDllPaths
-                    .Where(path => IsPathInSubfolder(path, legacyPackageDirectory.DirInfo.FullName, "tools"))
-                    .ToList();
-
-                if (nugetPackageDllPaths.Count > 0)
-                {
-                    logger.LogInfo($"Restored {nugetPackageDllPaths.Count} Nuget DLLs.");
-                }
-                if (excludedPaths.Count > 0)
-                {
-                    logger.LogInfo($"Excluding {excludedPaths.Count} Nuget DLLs.");
-                }
-
-                foreach (var excludedPath in excludedPaths)
-                {
-                    logger.LogInfo($"Excluded Nuget DLL: {excludedPath}");
-                }
-
-                nugetPackageDllPaths.ExceptWith(excludedPaths);
-                dllPaths.UnionWith(nugetPackageDllPaths);
-            }
-            catch (Exception exc)
-            {
-                logger.LogError($"Failed to restore Nuget packages with nuget.exe: {exc.Message}");
-            }
-
-            var restoredProjects = RestoreSolutions(allSolutions, out var assets1);
-            var projects = allProjects.Except(restoredProjects);
-            RestoreProjects(projects, out var assets2);
-
-            var dependencies = Assets.GetCompilationDependencies(logger, assets1.Union(assets2));
-
-            var paths = dependencies
-                .Paths
-                .Select(d => Path.Combine(packageDirectory.DirInfo.FullName, d))
-                .ToList();
-            dllPaths.UnionWith(paths);
-
-            LogAllUnusedPackages(dependencies);
-            DownloadMissingPackages(allNonBinaryFiles, dllPaths);
-        }
-
-        private static bool IsPathInSubfolder(string path, string rootFolder, string subFolder)
-        {
-            return path.IndexOf(
-                $"{Path.DirectorySeparatorChar}{subFolder}{Path.DirectorySeparatorChar}",
-                rootFolder.Length,
-                StringComparison.InvariantCultureIgnoreCase) >= 0;
-        }
-
         private void RemoveNugetAnalyzerReferences()
         {
             var packageFolder = packageDirectory.DirInfo.FullName.ToLowerInvariant();
@@ -483,27 +435,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                 .FullName;
         }
 
-        private ICollection<string> GetAllPackageDirectories()
-        {
-            return new DirectoryInfo(packageDirectory.DirInfo.FullName)
-                .EnumerateDirectories("*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
-                .Select(d => d.Name)
-                .ToList();
-        }
-
-        private void LogAllUnusedPackages(DependencyContainer dependencies)
-        {
-            var allPackageDirectories = GetAllPackageDirectories();
-
-            logger.LogInfo($"Restored {allPackageDirectories.Count} packages");
-            logger.LogInfo($"Found {dependencies.Packages.Count} packages in project.assets.json files");
-
-            allPackageDirectories
-                .Where(package => !dependencies.Packages.Contains(package))
-                .Order()
-                .ForEach(package => logger.LogInfo($"Unused package: {package}"));
-        }
-
         private void GenerateSourceFileFromImplicitUsings()
         {
             var usings = new HashSet<string>();
@@ -807,269 +738,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             }
         }
 
-        /// <summary>
-        /// Executes `dotnet restore` on all solution files in solutions.
-        /// As opposed to RestoreProjects this is not run in parallel using PLINQ
-        /// as `dotnet restore` on a solution already uses multiple threads for restoring
-        /// the projects (this can be disabled with the `--disable-parallel` flag).
-        /// Populates assets with the relative paths to the assets files generated by the restore.
-        /// Returns a list of projects that are up to date with respect to restore.
-        /// </summary>
-        /// <param name="solutions">A list of paths to solution files.</param>
-        private IEnumerable<string> RestoreSolutions(IEnumerable<string> solutions, out IEnumerable<string> assets)
-        {
-            var successCount = 0;
-            var nugetSourceFailures = 0;
-            var assetFiles = new List<string>();
-            var projects = solutions.SelectMany(solution =>
-                {
-                    logger.LogInfo($"Restoring solution {solution}...");
-                    var res = dotnet.Restore(new(solution, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
-                    if (res.Success)
-                    {
-                        successCount++;
-                    }
-                    if (res.HasNugetPackageSourceError)
-                    {
-                        nugetSourceFailures++;
-                    }
-                    assetFiles.AddRange(res.AssetsFilePaths);
-                    return res.RestoredProjects;
-                }).ToList();
-            assets = assetFiles;
-            CompilationInfos.Add(("Successfully restored solution files", successCount.ToString()));
-            CompilationInfos.Add(("Failed solution restore with package source error", nugetSourceFailures.ToString()));
-            CompilationInfos.Add(("Restored projects through solution files", projects.Count.ToString()));
-            return projects;
-        }
-
-        /// <summary>
-        /// Executes `dotnet restore` on all projects in projects.
-        /// This is done in parallel for performance reasons.
-        /// Populates assets with the relative paths to the assets files generated by the restore.
-        /// </summary>
-        /// <param name="projects">A list of paths to project files.</param>
-        private void RestoreProjects(IEnumerable<string> projects, out IEnumerable<string> assets)
-        {
-            var successCount = 0;
-            var nugetSourceFailures = 0;
-            var assetFiles = new List<string>();
-            var sync = new object();
-            Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = threads }, project =>
-            {
-                logger.LogInfo($"Restoring project {project}...");
-                var res = dotnet.Restore(new(project, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
-                lock (sync)
-                {
-                    if (res.Success)
-                    {
-                        successCount++;
-                    }
-                    if (res.HasNugetPackageSourceError)
-                    {
-                        nugetSourceFailures++;
-                    }
-                    assetFiles.AddRange(res.AssetsFilePaths);
-                }
-            });
-            assets = assetFiles;
-            CompilationInfos.Add(("Successfully restored project files", successCount.ToString()));
-            CompilationInfos.Add(("Failed project restore with package source error", nugetSourceFailures.ToString()));
-        }
-
-        [GeneratedRegex(@"^(.+)\.(\d+\.\d+\.\d+(-(.+))?)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
-        private static partial Regex LegacyNugetPackage();
-
-
-        private static IEnumerable<string> GetRestoredPackageDirectoryNames(DirectoryInfo root)
-        {
-            return Directory.GetDirectories(root.FullName)
-                .Select(d => Path.GetFileName(d).ToLowerInvariant());
-        }
-
-        private IEnumerable<string> GetRestoredLegacyPackageNames()
-        {
-            var oldPackageDirectories = GetRestoredPackageDirectoryNames(legacyPackageDirectory.DirInfo);
-            foreach (var oldPackageDirectory in oldPackageDirectories)
-            {
-                // nuget install restores packages to 'packagename.version' folders (dotnet restore to 'packagename/version' folders)
-                // typical folder names look like:
-                //   newtonsoft.json.13.0.3
-                // there are more complex ones too, such as:
-                //   runtime.tizen.4.0.0-armel.Microsoft.NETCore.DotNetHostResolver.2.0.0-preview2-25407-01
-
-                var match = LegacyNugetPackage().Match(oldPackageDirectory);
-                if (!match.Success)
-                {
-                    logger.LogWarning($"Package directory '{oldPackageDirectory}' doesn't match the expected pattern.");
-                    continue;
-                }
-
-                yield return match.Groups[1].Value.ToLowerInvariant();
-            }
-        }
-
-        private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPaths)
-        {
-            var alreadyDownloadedPackages = GetRestoredPackageDirectoryNames(packageDirectory.DirInfo);
-            var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames();
-
-            var notYetDownloadedPackages = new HashSet<PackageReference>(fileContent.AllPackages);
-            foreach (var alreadyDownloadedPackage in alreadyDownloadedPackages)
-            {
-                notYetDownloadedPackages.Remove(new(alreadyDownloadedPackage, PackageReferenceSource.SdkCsProj));
-            }
-            foreach (var alreadyDownloadedLegacyPackage in alreadyDownloadedLegacyPackages)
-            {
-                notYetDownloadedPackages.Remove(new(alreadyDownloadedLegacyPackage, PackageReferenceSource.PackagesConfig));
-            }
-
-            if (notYetDownloadedPackages.Count == 0)
-            {
-                return;
-            }
-
-            var multipleVersions = notYetDownloadedPackages
-                .GroupBy(p => p.Name)
-                .Where(g => g.Count() > 1)
-                .Select(g => g.Key)
-                .ToList();
-
-            foreach (var package in multipleVersions)
-            {
-                logger.LogWarning($"Found multiple not yet restored packages with name '{package}'.");
-                notYetDownloadedPackages.Remove(new(package, PackageReferenceSource.PackagesConfig));
-            }
-
-            logger.LogInfo($"Found {notYetDownloadedPackages.Count} packages that are not yet restored");
-
-            var nugetConfigs = allFiles.SelectFileNamesByName("nuget.config").ToArray();
-            string? nugetConfig = null;
-            if (nugetConfigs.Length > 1)
-            {
-                logger.LogInfo($"Found multiple nuget.config files: {string.Join(", ", nugetConfigs)}.");
-                nugetConfig = allFiles
-                    .SelectRootFiles(sourceDir)
-                    .SelectFileNamesByName("nuget.config")
-                    .FirstOrDefault();
-                if (nugetConfig == null)
-                {
-                    logger.LogInfo("Could not find a top-level nuget.config file.");
-                }
-            }
-            else
-            {
-                nugetConfig = nugetConfigs.FirstOrDefault();
-            }
-
-            if (nugetConfig != null)
-            {
-                logger.LogInfo($"Using nuget.config file {nugetConfig}.");
-            }
-
-            CompilationInfos.Add(("Fallback nuget restore", notYetDownloadedPackages.Count.ToString()));
-
-            var successCount = 0;
-            var sync = new object();
-
-            Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = threads }, package =>
-            {
-                var success = TryRestorePackageManually(package.Name, nugetConfig, package.PackageReferenceSource);
-                if (!success)
-                {
-                    return;
-                }
-
-                lock (sync)
-                {
-                    successCount++;
-                }
-            });
-
-            CompilationInfos.Add(("Successfully ran fallback nuget restore", successCount.ToString()));
-
-            dllPaths.Add(missingPackageDirectory.DirInfo.FullName);
-        }
-
-        [GeneratedRegex(@"<TargetFramework>.*</TargetFramework>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
-        private static partial Regex TargetFramework();
-
-        private bool TryRestorePackageManually(string package, string? nugetConfig, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj)
-        {
-            logger.LogInfo($"Restoring package {package}...");
-            using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package, "missingpackages_workingdir"));
-            var success = dotnet.New(tempDir.DirInfo.FullName);
-            if (!success)
-            {
-                return false;
-            }
-
-            if (packageReferenceSource == PackageReferenceSource.PackagesConfig)
-            {
-                TryChangeTargetFrameworkMoniker(tempDir.DirInfo);
-            }
-
-            success = dotnet.AddPackage(tempDir.DirInfo.FullName, package);
-            if (!success)
-            {
-                return false;
-            }
-
-            var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig));
-            if (!res.Success)
-            {
-                if (res.HasNugetPackageSourceError && nugetConfig is not null)
-                {
-                    // Restore could not be completed because the listed source is unavailable. Try without the nuget.config:
-                    res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: null, ForceReevaluation: true));
-                }
-
-                // TODO: the restore might fail, we could retry with
-                // - a prerelease (*-* instead of *) version of the package,
-                // - a different target framework moniker.
-
-                if (!res.Success)
-                {
-                    logger.LogInfo($"Failed to restore nuget package {package}");
-                    return false;
-                }
-            }
-
-            return true;
-        }
-
-        private void TryChangeTargetFrameworkMoniker(DirectoryInfo tempDir)
-        {
-            try
-            {
-                logger.LogInfo($"Changing the target framework moniker in {tempDir.FullName}...");
-
-                var csprojs = tempDir.GetFiles("*.csproj", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive });
-                if (csprojs.Length != 1)
-                {
-                    logger.LogError($"Could not find the .csproj file in {tempDir.FullName}, count = {csprojs.Length}");
-                    return;
-                }
-
-                var csproj = csprojs[0];
-                var content = File.ReadAllText(csproj.FullName);
-                var matches = TargetFramework().Matches(content);
-                if (matches.Count == 0)
-                {
-                    logger.LogError($"Could not find target framework in {csproj.FullName}");
-                }
-                else
-                {
-                    content = TargetFramework().Replace(content, $"<TargetFramework>{FrameworkPackageNames.LatestNetFrameworkMoniker}</TargetFramework>", 1);
-                    File.WriteAllText(csproj.FullName, content);
-                }
-            }
-            catch (Exception exc)
-            {
-                logger.LogError($"Failed to update target framework in {tempDir.FullName}: {exc}");
-            }
-        }
-
         public void Dispose(TemporaryDirectory? dir, string name)
         {
             try
@@ -1091,6 +759,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             {
                 Dispose(tempWorkingDirectory, "temporary working");
             }
+
+            diagnosticsWriter?.Dispose();
         }
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs

@@ -16,12 +16,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
     public partial class DotNet : IDotNet
     {
         private readonly IDotNetCliInvoker dotnetCliInvoker;
+        private readonly ILogger logger;
         private readonly TemporaryDirectory? tempWorkingDirectory;
 
         private DotNet(IDotNetCliInvoker dotnetCliInvoker, ILogger logger, TemporaryDirectory? tempWorkingDirectory = null)
         {
             this.tempWorkingDirectory = tempWorkingDirectory;
             this.dotnetCliInvoker = dotnetCliInvoker;
+            this.logger = logger;
             Info();
         }
 
@@ -89,17 +91,18 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             return dotnetCliInvoker.RunCommand(args);
         }
 
-        public IList<string> GetListedRuntimes() => GetListed("--list-runtimes", "runtime");
+        public IList<string> GetListedRuntimes() => GetResultList("--list-runtimes");
 
-        public IList<string> GetListedSdks() => GetListed("--list-sdks", "SDK");
+        public IList<string> GetListedSdks() => GetResultList("--list-sdks");
 
-        private IList<string> GetListed(string args, string artifact)
+        private IList<string> GetResultList(string args)
         {
-            if (dotnetCliInvoker.RunCommand(args, out var artifacts))
+            if (dotnetCliInvoker.RunCommand(args, out var results))
             {
-                return artifacts;
+                return results;
             }
-            return new List<string>();
+            logger.LogWarning($"Running 'dotnet {args}' failed.");
+            return [];
         }
 
         public bool Exec(string execArgs)
@@ -108,6 +111,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             return dotnetCliInvoker.RunCommand(args);
         }
 
+        public IList<string> GetNugetFeeds(string nugetConfig) => GetResultList($"nuget list source --format Short --configfile \"{nugetConfig}\"");
+
         // The version number should be kept in sync with the version .NET version used for building the application.
         public const string LatestDotNetSdkVersion = "8.0.101";
 

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs

@@ -16,5 +16,30 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         /// Controls whether to use framework dependencies from subfolders.
         /// </summary>
         public const string DotnetFrameworkReferencesUseSubfolders = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_DOTNET_FRAMEWORK_REFERENCES_USE_SUBFOLDERS";
+
+        /// <summary>
+        /// Controls whether to check the responsiveness of NuGet feeds.
+        /// </summary>
+        public const string CheckNugetFeedResponsiveness = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK";
+
+        /// <summary>
+        /// Specifies the NuGet feeds to exclude from the responsiveness check. The value is a space-separated list of feed URLs.
+        /// </summary>
+        public const string ExcludedNugetFeedsFromResponsivenessCheck = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_EXCLUDED";
+
+        /// <summary>
+        /// Specifies the timeout (as an integer) in milliseconds for the initial check of NuGet feeds responsiveness. The value is then doubled for each subsequent check.
+        /// </summary>
+        public const string NugetFeedResponsivenessInitialTimeout = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_TIMEOUT";
+
+        /// <summary>
+        /// Specifies how many requests to make to the NuGet feed to check its responsiveness.
+        /// </summary>
+        public const string NugetFeedResponsivenessRequestCount = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_LIMIT";
+
+        /// <summary>
+        /// Specifies the location of the diagnostic directory.
+        /// </summary>
+        public const string DiagnosticDir = "CODEQL_EXTRACTOR_CSHARP_DIAGNOSTIC_DIR";
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs

@@ -13,6 +13,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         IList<string> GetListedRuntimes();
         IList<string> GetListedSdks();
         bool Exec(string execArgs);
+        IList<string> GetNugetFeeds(string nugetConfig);
     }
 
     public record class RestoreSettings(string File, string PackageDirectory, bool ForceDotnetRefAssemblyFetching, string? PathToNugetConfig = null, bool ForceReevaluation = false);

csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs

@@ -143,7 +143,7 @@ namespace Semmle.Extraction.CSharp.Standalone
             stopwatch.Start();
 
             using var logger = new ConsoleLogger(options.Verbosity, logThreadId: true);
-            logger.Log(Severity.Info, "Extracting C# in buildless mode");
+            logger.Log(Severity.Info, "Extracting C# with build-mode set to 'none'");
             using var dependencyManager = new DependencyManager(options.SrcDir, logger);
 
             if (!dependencyManager.NonGeneratedSourcesFiles.Any())

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expression.cs

@@ -100,8 +100,14 @@ namespace Semmle.Extraction.CSharp.Entities
         /// <param name="child">The child index.</param>
         /// <param name="type">A type hint.</param>
         /// <returns>The new expression.</returns>
-        public static Expression Create(Context cx, ExpressionSyntax node, IExpressionParentEntity parent, int child) =>
-            CreateFromNode(new ExpressionNodeInfo(cx, node, parent, child));
+        public static Expression Create(Context cx, ExpressionSyntax node, IExpressionParentEntity parent, int child, Boolean isCompilerGenerated = false)
+        {
+            var info = new ExpressionNodeInfo(cx, node, parent, child)
+            {
+                IsCompilerGenerated = isCompilerGenerated
+            };
+            return CreateFromNode(info);
+        }
 
         public static Expression CreateFromNode(ExpressionNodeInfo info) => Expressions.ImplicitCast.Create(info);
 

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ArrayCreation.cs

@@ -97,7 +97,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 ExprKind.ARRAY_CREATION,
                 parent,
                 childIndex,
-                true,
+                isCompilerGenerated: true,
                 null);
 
             var arrayCreation = new Expression(info);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Assignment.cs

@@ -26,10 +26,10 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
             if (operatorKind.HasValue)
             {
                 // Convert assignment such as `a += b` into `a = a + b`.
-                var simpleAssignExpr = new Expression(new ExpressionInfo(Context, Type, Location, ExprKind.SIMPLE_ASSIGN, this, 2, false, null));
+                var simpleAssignExpr = new Expression(new ExpressionInfo(Context, Type, Location, ExprKind.SIMPLE_ASSIGN, this, 2, isCompilerGenerated: true, null));
                 Create(Context, Syntax.Left, simpleAssignExpr, 1);
-                var opexpr = new Expression(new ExpressionInfo(Context, Type, Location, operatorKind.Value, simpleAssignExpr, 0, false, null));
-                Create(Context, Syntax.Left, opexpr, 0);
+                var opexpr = new Expression(new ExpressionInfo(Context, Type, Location, operatorKind.Value, simpleAssignExpr, 0, isCompilerGenerated: true, null));
+                Create(Context, Syntax.Left, opexpr, 0, isCompilerGenerated: true);
                 Create(Context, Syntax.Right, opexpr, 1);
                 opexpr.OperatorCall(trapFile, Syntax);
             }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Cast.cs

@@ -41,7 +41,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 ExprKind.CAST,
                 parent,
                 childIndex,
-                true,
+                isCompilerGenerated: true,
                 ValueAsString(value));
 
             var ret = new Expression(info);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Collections/Spread.cs

@@ -6,7 +6,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
     internal class Spread : Expression
     {
         public Spread(Context cx, SpreadElementSyntax syntax, IExpressionParentEntity parent, int child) :
-            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), ExprKind.SPREAD_ELEMENT, parent, child, false, null))
+            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), ExprKind.SPREAD_ELEMENT, parent, child, isCompilerGenerated: false, null))
         {
             Create(cx, syntax.Expression, this, 0);
         }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Default.cs

@@ -24,7 +24,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 ExprKind.DEFAULT,
                 parent,
                 childIndex,
-                true,
+                isCompilerGenerated: true,
                 value);
 
             return new Expression(info);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Discard.cs

@@ -11,7 +11,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
         }
 
         private Discard(Context cx, CSharpSyntaxNode syntax, IExpressionParentEntity parent, int child) :
-            base(new ExpressionInfo(cx, cx.GetType(syntax), cx.CreateLocation(syntax.GetLocation()), ExprKind.DISCARD, parent, child, false, null))
+            base(new ExpressionInfo(cx, cx.GetType(syntax), cx.CreateLocation(syntax.GetLocation()), ExprKind.DISCARD, parent, child, isCompilerGenerated: false, null))
         {
         }
 

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ElementAccess.cs

@@ -22,7 +22,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
             if (Kind == ExprKind.POINTER_INDIRECTION)
             {
                 var qualifierInfo = new ExpressionNodeInfo(Context, qualifier, this, 0);
-                var add = new Expression(new ExpressionInfo(Context, qualifierInfo.Type, Location, ExprKind.ADD, this, 0, false, null));
+                var add = new Expression(new ExpressionInfo(Context, qualifierInfo.Type, Location, ExprKind.ADD, this, 0, isCompilerGenerated: false, null));
                 qualifierInfo.SetParent(add, 0);
                 CreateFromNode(qualifierInfo);
                 PopulateArguments(trapFile, argumentList, 1);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ImplicitCast.cs

@@ -14,13 +14,13 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
         }
 
         private ImplicitCast(ExpressionNodeInfo info)
-            : base(new ExpressionInfo(info.Context, info.ConvertedType, info.Location, ExprKind.CAST, info.Parent, info.Child, true, info.ExprValue))
+            : base(new ExpressionInfo(info.Context, info.ConvertedType, info.Location, ExprKind.CAST, info.Parent, info.Child, isCompilerGenerated: true, info.ExprValue))
         {
             Expr = Factory.Create(new ExpressionNodeInfo(Context, info.Node, this, 0));
         }
 
         private ImplicitCast(ExpressionNodeInfo info, IMethodSymbol method)
-            : base(new ExpressionInfo(info.Context, info.ConvertedType, info.Location, ExprKind.OPERATOR_INVOCATION, info.Parent, info.Child, true, info.ExprValue))
+            : base(new ExpressionInfo(info.Context, info.ConvertedType, info.Location, ExprKind.OPERATOR_INVOCATION, info.Parent, info.Child, isCompilerGenerated: true, info.ExprValue))
         {
             Expr = Factory.Create(info.SetParent(this, 0));
 
@@ -65,7 +65,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                     kind,
                     parent,
                     childIndex,
-                    true,
+                    isCompilerGenerated: true,
                     v);
 
             var method = GetImplicitConversionMethod(type, value);
@@ -93,7 +93,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                     ExprKind.CAST,
                     parent,
                     childIndex,
-                    true,
+                    isCompilerGenerated: true,
                     ValueAsString(value));
 
             return new Expression(info);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Initializer.cs

@@ -45,7 +45,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 ExprKind.ARRAY_INIT,
                 parent,
                 index,
-                true,
+                isCompilerGenerated: true,
                 null);
 
             return new Expression(info);
@@ -132,7 +132,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 var addMethod = Method.Create(Context, collectionInfo.Symbol as IMethodSymbol);
                 var voidType = AnnotatedTypeSymbol.CreateNotAnnotated(Context.Compilation.GetSpecialType(SpecialType.System_Void));
 
-                var invocation = new Expression(new ExpressionInfo(Context, voidType, Context.CreateLocation(i.GetLocation()), ExprKind.METHOD_INVOCATION, this, child++, false, null));
+                var invocation = new Expression(new ExpressionInfo(Context, voidType, Context.CreateLocation(i.GetLocation()), ExprKind.METHOD_INVOCATION, this, child++, isCompilerGenerated: true, null));
 
                 if (addMethod is not null)
                     trapFile.expr_call(invocation, addMethod);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/InterpolatedString.cs

@@ -25,7 +25,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                     case SyntaxKind.InterpolatedStringText:
                         // Create a string literal
                         var interpolatedText = (InterpolatedStringTextSyntax)c;
-                        new Expression(new ExpressionInfo(Context, Type, Context.CreateLocation(c.GetLocation()), ExprKind.UTF16_STRING_LITERAL, this, child++, false, interpolatedText.TextToken.ValueText));
+                        new Expression(new ExpressionInfo(Context, Type, Context.CreateLocation(c.GetLocation()), ExprKind.UTF16_STRING_LITERAL, this, child++, isCompilerGenerated: false, interpolatedText.TextToken.ValueText));
                         break;
                     default:
                         throw new InternalError(c, $"Unhandled interpolation kind {c.Kind()}");

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Lambda.cs

@@ -63,7 +63,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
         public static Lambda Create(ExpressionNodeInfo info, ParenthesizedLambdaExpressionSyntax node) => new Lambda(info, node);
 
         private Lambda(ExpressionNodeInfo info, SimpleLambdaExpressionSyntax node)
-            : this(info.SetKind(ExprKind.LAMBDA), node.Body, Enumerators.Singleton(node.Parameter), null) { }
+            : this(info.SetKind(ExprKind.LAMBDA), node.Body, [node.Parameter], null) { }
 
         public static Lambda Create(ExpressionNodeInfo info, SimpleLambdaExpressionSyntax node) => new Lambda(info, node);
 

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Literal.cs

@@ -97,7 +97,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 kind,
                 parent,
                 childIndex,
-                true,
+                isCompilerGenerated: true,
                 ValueAsString(value));
 
             return new Expression(info);
@@ -112,7 +112,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 ExprKind.NULL_LITERAL,
                 parent,
                 childIndex,
-                true,
+                isCompilerGenerated: true,
                 ValueAsString(null));
 
             return new Expression(info);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ObjectCreation/AnonymousObjectCreation.cs

@@ -30,7 +30,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 return;
             }
 
-            var objectInitializer = new Expression(new ExpressionInfo(Context, Type, Location, ExprKind.OBJECT_INIT, this, -1, false, null));
+            var objectInitializer = new Expression(new ExpressionInfo(Context, Type, Location, ExprKind.OBJECT_INIT, this, -1, isCompilerGenerated: false, null));
 
             foreach (var init in Syntax.Initializers)
             {
@@ -40,11 +40,11 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 var type = property.GetAnnotatedType();
                 var loc = Context.CreateLocation(init.GetLocation());
 
-                var assignment = new Expression(new ExpressionInfo(Context, type, loc, ExprKind.SIMPLE_ASSIGN, objectInitializer, child++, false, null));
+                var assignment = new Expression(new ExpressionInfo(Context, type, loc, ExprKind.SIMPLE_ASSIGN, objectInitializer, child++, isCompilerGenerated: false, null));
                 Create(Context, init.Expression, assignment, 0);
                 Property.Create(Context, property);
 
-                var access = new Expression(new ExpressionInfo(Context, type, loc, ExprKind.PROPERTY_ACCESS, assignment, 1, false, null));
+                var access = new Expression(new ExpressionInfo(Context, type, loc, ExprKind.PROPERTY_ACCESS, assignment, 1, isCompilerGenerated: false, null));
                 trapFile.expr_access(access, propEntity);
             }
         }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ObjectCreation/DateTimeObjectCreation.cs

@@ -59,7 +59,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 ExprKind.OBJECT_CREATION,
                 parent,
                 childIndex,
-                true,
+                isCompilerGenerated: true,
                 null));
 
             var longTypeSymbol = constructorSymbol.Parameters[0].Type;

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Patterns/BinaryPattern.cs

@@ -8,7 +8,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
     internal class BinaryPattern : Expression
     {
         public BinaryPattern(Context cx, BinaryPatternSyntax syntax, IExpressionParentEntity parent, int child) :
-            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), GetKind(syntax.OperatorToken, syntax), parent, child, false, null))
+            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), GetKind(syntax.OperatorToken, syntax), parent, child, isCompilerGenerated: false, null))
         {
             Pattern.Create(cx, syntax.Left, this, 0);
             Pattern.Create(cx, syntax.Right, this, 1);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Patterns/ListPattern.cs

@@ -7,7 +7,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
     internal class ListPattern : Expression
     {
         internal ListPattern(Context cx, ListPatternSyntax syntax, IExpressionParentEntity parent, int child) :
-            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), ExprKind.LIST_PATTERN, parent, child, false, null))
+            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), ExprKind.LIST_PATTERN, parent, child, isCompilerGenerated: false, null))
         {
             syntax.Patterns.ForEach((p, i) => Pattern.Create(cx, p, this, i));
         }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Patterns/PositionalPattern.cs

@@ -7,7 +7,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
     internal class PositionalPattern : Expression
     {
         internal PositionalPattern(Context cx, PositionalPatternClauseSyntax posPc, IExpressionParentEntity parent, int child) :
-            base(new ExpressionInfo(cx, null, cx.CreateLocation(posPc.GetLocation()), ExprKind.POSITIONAL_PATTERN, parent, child, false, null))
+            base(new ExpressionInfo(cx, null, cx.CreateLocation(posPc.GetLocation()), ExprKind.POSITIONAL_PATTERN, parent, child, isCompilerGenerated: false, null))
         {
             posPc.Subpatterns.ForEach((p, i) => Pattern.Create(cx, p.Pattern, this, i));
         }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Patterns/PropertyPattern.cs

@@ -7,7 +7,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
     internal class PropertyPattern : Expression
     {
         internal PropertyPattern(Context cx, PropertyPatternClauseSyntax pp, IExpressionParentEntity parent, int child) :
-            base(new ExpressionInfo(cx, null, cx.CreateLocation(pp.GetLocation()), ExprKind.PROPERTY_PATTERN, parent, child, false, null))
+            base(new ExpressionInfo(cx, null, cx.CreateLocation(pp.GetLocation()), ExprKind.PROPERTY_PATTERN, parent, child, isCompilerGenerated: false, null))
         {
             child = 0;
             foreach (var sub in pp.Subpatterns)
@@ -56,7 +56,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
             };
 
         private static Expression CreateSyntheticExp(Context cx, Microsoft.CodeAnalysis.Location location, IExpressionParentEntity parent, int child) =>
-            new Expression(new ExpressionInfo(cx, null, cx.CreateLocation(location), ExprKind.PROPERTY_PATTERN, parent, child, false, null));
+            new Expression(new ExpressionInfo(cx, null, cx.CreateLocation(location), ExprKind.PROPERTY_PATTERN, parent, child, isCompilerGenerated: false, null));
 
         private static void MakeExpressions(Context cx, IExpressionParentEntity parent, SubpatternSyntax syntax, int child)
         {

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Patterns/RecursivePattern.cs

@@ -15,7 +15,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
         /// <param name="parent">The parent pattern/expression.</param>
         /// <param name="child">The child index of this pattern.</param>
         public RecursivePattern(Context cx, RecursivePatternSyntax syntax, IExpressionParentEntity parent, int child) :
-            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), ExprKind.RECURSIVE_PATTERN, parent, child, false, null))
+            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), ExprKind.RECURSIVE_PATTERN, parent, child, isCompilerGenerated: false, null))
         {
             // Extract the type access
             if (syntax.Type is TypeSyntax t)

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Patterns/RelationalPattern.cs

@@ -8,7 +8,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
     internal class RelationalPattern : Expression
     {
         public RelationalPattern(Context cx, RelationalPatternSyntax syntax, IExpressionParentEntity parent, int child) :
-             base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), GetKind(syntax.OperatorToken), parent, child, false, null))
+             base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), GetKind(syntax.OperatorToken), parent, child, isCompilerGenerated: false, null))
         {
             Expression.Create(cx, syntax.Expression, this, 0);
         }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Patterns/SlicePattern.cs

@@ -6,7 +6,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
     internal class SlicePattern : Expression
     {
         public SlicePattern(Context cx, SlicePatternSyntax syntax, IExpressionParentEntity parent, int child) :
-            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), ExprKind.SLICE_PATTERN, parent, child, false, null))
+            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), ExprKind.SLICE_PATTERN, parent, child, isCompilerGenerated: false, null))
         {
             if (syntax.Pattern is not null)
             {

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Patterns/UnaryPattern.cs

@@ -6,7 +6,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
     internal class UnaryPattern : Expression
     {
         public UnaryPattern(Context cx, UnaryPatternSyntax syntax, IExpressionParentEntity parent, int child) :
-            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), ExprKind.NOT_PATTERN, parent, child, false, null))
+            base(new ExpressionInfo(cx, null, cx.CreateLocation(syntax.GetLocation()), ExprKind.NOT_PATTERN, parent, child, isCompilerGenerated: false, null))
         {
             Pattern.Create(cx, syntax.Pattern, this, 0);
         }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Query.cs

@@ -23,7 +23,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
             public QueryCall(Context cx, IMethodSymbol? method, SyntaxNode clause, IExpressionParentEntity parent, int child)
                 : base(new ExpressionInfo(cx, method?.GetAnnotatedReturnType(),
                         cx.CreateLocation(clause.GetLocation()),
-                        ExprKind.METHOD_INVOCATION, parent, child, false, null))
+                        ExprKind.METHOD_INVOCATION, parent, child, isCompilerGenerated: false, null))
             {
                 if (method is not null)
                     cx.TrapWriter.Writer.expr_call(this, Method.Create(cx, method));
@@ -97,7 +97,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 Expression.Create(cx, Expr, decl, 0);
 
                 var nameLoc = cx.CreateLocation(name.GetLocation());
-                var access = new Expression(new ExpressionInfo(cx, type, nameLoc, ExprKind.LOCAL_VARIABLE_ACCESS, decl, 1, false, null));
+                var access = new Expression(new ExpressionInfo(cx, type, nameLoc, ExprKind.LOCAL_VARIABLE_ACCESS, decl, 1, isCompilerGenerated: false, null));
                 cx.TrapWriter.Writer.expr_access(access, LocalVariable.Create(cx, variableSymbol));
 
                 return decl;

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Switch.cs

@@ -27,7 +27,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
         internal SwitchCase(Context cx, SwitchExpressionArmSyntax arm, Switch parent, int child) :
             base(new ExpressionInfo(
                 cx, cx.GetType(arm.Expression), cx.CreateLocation(arm.GetLocation()),
-                ExprKind.SWITCH_CASE, parent, child, false, null))
+                ExprKind.SWITCH_CASE, parent, child, isCompilerGenerated: false, null))
         {
             Expressions.Pattern.Create(cx, arm.Pattern, this, 0);
             if (arm.WhenClause is WhenClauseSyntax when)

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/This.cs

@@ -8,7 +8,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
         private This(IExpressionInfo info) : base(info) { }
 
         public static This CreateImplicit(Context cx, ITypeSymbol @class, Extraction.Entities.Location loc, IExpressionParentEntity parent, int child) =>
-            new This(new ExpressionInfo(cx, AnnotatedTypeSymbol.CreateNotAnnotated(@class), loc, Kinds.ExprKind.THIS_ACCESS, parent, child, true, null));
+            new This(new ExpressionInfo(cx, AnnotatedTypeSymbol.CreateNotAnnotated(@class), loc, Kinds.ExprKind.THIS_ACCESS, parent, child, isCompilerGenerated: true, null));
 
         public static This CreateExplicit(ExpressionNodeInfo info) => new This(info.SetKind(ExprKind.THIS_ACCESS));
     }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/TypeAccess.cs

@@ -44,7 +44,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 ExprKind.TYPE_ACCESS,
                 parent,
                 childIndex,
-                true,
+                isCompilerGenerated: true,
                 null);
 
             return new Expression(typeAccessInfo);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/TypeOf.cs

@@ -26,7 +26,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 ExprKind.TYPEOF,
                 parent,
                 childIndex,
-                true,
+                isCompilerGenerated: true,
                 null);
 
             var ret = new Expression(info);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/VariableDeclaration.cs

@@ -15,7 +15,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
 
         public static VariableDeclaration Create(Context cx, ISymbol symbol, AnnotatedTypeSymbol? type, TypeSyntax? optionalSyntax, Extraction.Entities.Location exprLocation, bool isVar, IExpressionParentEntity parent, int child)
         {
-            var ret = new VariableDeclaration(new ExpressionInfo(cx, type, exprLocation, ExprKind.LOCAL_VAR_DECL, parent, child, false, null));
+            var ret = new VariableDeclaration(new ExpressionInfo(cx, type, exprLocation, ExprKind.LOCAL_VAR_DECL, parent, child, isCompilerGenerated: false, null));
             cx.Try(null, null, () =>
             {
                 var l = LocalVariable.Create(cx, symbol);
@@ -52,7 +52,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
         public static Expression CreateParenthesized(Context cx, DeclarationExpressionSyntax node, ParenthesizedVariableDesignationSyntax designation, IExpressionParentEntity parent, int child, INamedTypeSymbol? t)
         {
             var type = t is null ? (AnnotatedTypeSymbol?)null : new AnnotatedTypeSymbol(t, t.NullableAnnotation);
-            var tuple = new Expression(new ExpressionInfo(cx, type, cx.CreateLocation(node.GetLocation()), ExprKind.TUPLE, parent, child, false, null));
+            var tuple = new Expression(new ExpressionInfo(cx, type, cx.CreateLocation(node.GetLocation()), ExprKind.TUPLE, parent, child, isCompilerGenerated: false, null));
 
             cx.Try(null, null, () =>
             {
@@ -68,7 +68,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
         public static Expression CreateParenthesized(Context cx, VarPatternSyntax varPattern, ParenthesizedVariableDesignationSyntax designation, IExpressionParentEntity parent, int child)
         {
             var tuple = new Expression(
-                new ExpressionInfo(cx, null, cx.CreateLocation(varPattern.GetLocation()), ExprKind.TUPLE, parent, child, false, null),
+                new ExpressionInfo(cx, null, cx.CreateLocation(varPattern.GetLocation()), ExprKind.TUPLE, parent, child, isCompilerGenerated: false, null),
                 shouldPopulate: false);
 
             var elementTypes = new List<ITypeSymbol?>();
@@ -148,7 +148,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
             Create(cx, node, node.Designation, parent, child, cx.GetTypeInfo(node).Type.DisambiguateType() as INamedTypeSymbol);
 
         public static VariableDeclaration Create(Context cx, CSharpSyntaxNode c, AnnotatedTypeSymbol? type, IExpressionParentEntity parent, int child) =>
-            new VariableDeclaration(new ExpressionInfo(cx, type, cx.CreateLocation(c.FixedLocation()), ExprKind.LOCAL_VAR_DECL, parent, child, false, null));
+            new VariableDeclaration(new ExpressionInfo(cx, type, cx.CreateLocation(c.FixedLocation()), ExprKind.LOCAL_VAR_DECL, parent, child, isCompilerGenerated: false, null));
 
         public static VariableDeclaration Create(Context cx, CatchDeclarationSyntax d, bool isVar, IExpressionParentEntity parent, int child)
         {
@@ -179,7 +179,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                     Create(cx, d.Initializer.Value, ret, 0);
 
                     // Create an access
-                    var access = new Expression(new ExpressionInfo(cx, type, localVar.Location, ExprKind.LOCAL_VARIABLE_ACCESS, ret, 1, false, null));
+                    var access = new Expression(new ExpressionInfo(cx, type, localVar.Location, ExprKind.LOCAL_VARIABLE_ACCESS, ret, 1, isCompilerGenerated: false, null));
                     cx.TrapWriter.Writer.expr_access(access, localVar);
                 }
 

csharp/extractor/Semmle.Extraction.CSharp/Entities/Field.cs

@@ -110,9 +110,9 @@ namespace Semmle.Extraction.CSharp.Entities
             string? constValue, ref int child)
         {
             var type = Symbol.GetAnnotatedType();
-            var simpleAssignExpr = new Expression(new ExpressionInfo(Context, type, loc, ExprKind.SIMPLE_ASSIGN, this, child++, false, constValue));
+            var simpleAssignExpr = new Expression(new ExpressionInfo(Context, type, loc, ExprKind.SIMPLE_ASSIGN, this, child++, isCompilerGenerated: true, constValue));
             Expression.CreateFromNode(new ExpressionNodeInfo(Context, initializer, simpleAssignExpr, 0));
-            var access = new Expression(new ExpressionInfo(Context, type, Location, ExprKind.FIELD_ACCESS, simpleAssignExpr, 1, false, constValue));
+            var access = new Expression(new ExpressionInfo(Context, type, Location, ExprKind.FIELD_ACCESS, simpleAssignExpr, 1, isCompilerGenerated: true, constValue));
             trapFile.expr_access(access, this);
             return access;
         }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Property.cs

@@ -86,9 +86,9 @@ namespace Semmle.Extraction.CSharp.Entities
                     {
                         var loc = Context.CreateLocation(initializer!.GetLocation());
                         var annotatedType = AnnotatedTypeSymbol.CreateNotAnnotated(Symbol.Type);
-                        var simpleAssignExpr = new Expression(new ExpressionInfo(Context, annotatedType, loc, ExprKind.SIMPLE_ASSIGN, this, child++, false, null));
+                        var simpleAssignExpr = new Expression(new ExpressionInfo(Context, annotatedType, loc, ExprKind.SIMPLE_ASSIGN, this, child++, isCompilerGenerated: true, null));
                         Expression.CreateFromNode(new ExpressionNodeInfo(Context, initializer.Value, simpleAssignExpr, 0));
-                        var access = new Expression(new ExpressionInfo(Context, annotatedType, Location, ExprKind.PROPERTY_ACCESS, simpleAssignExpr, 1, false, null));
+                        var access = new Expression(new ExpressionInfo(Context, annotatedType, Location, ExprKind.PROPERTY_ACCESS, simpleAssignExpr, 1, isCompilerGenerated: true, null));
                         trapFile.expr_access(access, this);
                         if (!Symbol.IsStatic)
                         {

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs

@@ -189,7 +189,7 @@ namespace Semmle.Extraction.CSharp
                 // compilation.Clone() is used to allow symbols to be garbage collected.
                 using var trapWriter = transformedSourcePath.CreateTrapWriter(Logger, options.TrapCompression, discardDuplicates: false);
 
-                upToDate = options.Fast && FileIsUpToDate(sourcePath, trapWriter.TrapFile);
+                upToDate = FileIsUpToDate(sourcePath, trapWriter.TrapFile);
 
                 var currentTaskId = IncrementTaskCount();
                 ReportProgressTaskStarted(currentTaskId, sourcePath);

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Options.cs

@@ -35,15 +35,7 @@ namespace Semmle.Extraction.CSharp
         public static Options CreateWithEnvironment(string[] arguments)
         {
             var options = new Options();
-            var extractionOptions = Environment.GetEnvironmentVariable("LGTM_INDEX_EXTRACTOR");
-
             var argsList = new List<string>(arguments);
-
-            if (!string.IsNullOrEmpty(extractionOptions))
-            {
-                argsList.AddRange(extractionOptions.Split(' '));
-            }
-
             options.ParseArguments(argsList);
             return options;
         }

csharp/extractor/Semmle.Extraction.Tests/Options.cs

@@ -12,11 +12,6 @@ namespace Semmle.Extraction.Tests
         private CSharp.Options? options;
         private CSharp.Standalone.Options? standaloneOptions;
 
-        public OptionsTests()
-        {
-            Environment.SetEnvironmentVariable("LGTM_INDEX_EXTRACTOR", "");
-        }
-
         [Fact]
         public void DefaultOptions()
         {
@@ -28,7 +23,6 @@ namespace Semmle.Extraction.Tests
             Assert.True(options.Threads >= 1);
             Assert.Equal(Verbosity.Info, options.LegacyVerbosity);
             Assert.False(options.Console);
-            Assert.False(options.Fast);
             Assert.Equal(TrapWriter.CompressionMode.Brotli, options.TrapCompression);
         }
 
@@ -165,14 +159,6 @@ namespace Semmle.Extraction.Tests
             Assert.True(standaloneOptions.Help);
         }
 
-        [Fact]
-        public void Fast()
-        {
-            Environment.SetEnvironmentVariable("LGTM_INDEX_EXTRACTOR", "--fast");
-            options = CSharp.Options.CreateWithEnvironment(Array.Empty<string>());
-            Assert.True(options.Fast);
-        }
-
         [Fact]
         public void ArchiveArguments()
         {

csharp/extractor/Semmle.Extraction.Tests/Runtime.cs

@@ -26,6 +26,8 @@ namespace Semmle.Extraction.Tests
         public IList<string> GetListedSdks() => sdks;
 
         public bool Exec(string execArgs) => true;
+
+        public IList<string> GetNugetFeeds(string nugetConfig) => [];
     }
 
     public class RuntimeTests

csharp/extractor/Semmle.Extraction/Options.cs

@@ -60,11 +60,6 @@ namespace Semmle.Extraction
         /// </summary>
         public bool Cache { get; private set; } = true;
 
-        /// <summary>
-        /// Whether "fast extraction mode" has been enabled.
-        /// </summary>
-        public bool Fast { get; private set; } = false;
-
         /// <summary>
         /// Whether extraction is done using `codeql test run`.
         /// </summary>
@@ -115,9 +110,6 @@ namespace Semmle.Extraction
                 case "cache":
                     Cache = value;
                     return true;
-                case "fast":
-                    Fast = value;
-                    return true;
                 case "qltest":
                     QlTest = value;
                     return true;

csharp/extractor/Semmle.Util.Tests/ActionMap.cs

@@ -1,54 +0,0 @@
-using Xunit;
-using Semmle.Util;
-
-using Assert = Xunit.Assert;
-
-namespace SemmleTests.Semmle.Util
-{
-
-    public class ActionMapTests
-    {
-        [Fact]
-        public void TestAddthenOnAdd()
-        {
-            var am = new ActionMap<int, int>();
-            am.Add(1, 2);
-            int value = 0;
-            am.OnAdd(1, x => value = x);
-            Assert.Equal(2, value);
-        }
-
-        [Fact]
-        public void TestOnAddthenAdd()
-        {
-            var am = new ActionMap<int, int>();
-            int value = 0;
-            am.OnAdd(1, x => value = x);
-            am.Add(1, 2);
-            Assert.Equal(2, value);
-        }
-
-        [Fact]
-        public void TestNotAdded()
-        {
-            var am = new ActionMap<int, int>();
-            int value = 0;
-            am.OnAdd(1, x => value = x);
-            am.Add(2, 2);
-            Assert.Equal(0, value);
-        }
-
-        [Fact]
-        public void TestMultipleActions()
-        {
-            var am = new ActionMap<int, int>();
-            int value1 = 0, value2 = 0;
-            am.OnAdd(1, x => value1 = x);
-            am.OnAdd(1, x => value2 = x);
-            am.Add(1, 2);
-            Assert.Equal(2, value1);
-            Assert.Equal(2, value2);
-        }
-
-    }
-}

csharp/extractor/Semmle.Util.Tests/TextTest.cs

@@ -1,78 +0,0 @@
-using Xunit;
-using System;
-using Semmle.Util;
-
-using Assert = Xunit.Assert;
-
-namespace SemmleTests
-{
-    public class TextTest
-    {
-        //#################### PRIVATE VARIABLES ####################
-        #region
-
-        /// <summary>
-        /// A shorter way of writing Environment.NewLine (it gets used repeatedly).
-        /// </summary>
-        private static readonly string NL = Environment.NewLine;
-
-        #endregion
-
-        //#################### TEST METHODS ####################
-        #region
-
-        [Fact]
-        public void GetAllTest()
-        {
-            var input = new string[]
-            {
-                "Said once a young coder from Crewe,",
-                "'I like to write tests, so I do!",
-                "They help me confirm",
-                "That I don't need to squirm -",
-                "My code might look nice, but works too!'"
-            };
-
-            var text = new Text(input);
-
-            Assert.Equal(string.Join(NL, input) + NL, text.GetAll());
-        }
-
-        [Fact]
-        public void GetPortionTest()
-        {
-            var input = new string[]
-            {
-                "There once was a jolly young tester",
-                "Who couldn't leave software to fester -",
-                "He'd prod and he'd poke",
-                "Until something bad broke,",
-                "And then he'd find someone to pester."
-            };
-
-            var text = new Text(input);
-
-            // A single-line range (to test the special case).
-            Assert.Equal("jolly" + NL, text.GetPortion(0, 17, 0, 22));
-
-            // A two-line range.
-            Assert.Equal("prod and he'd poke" + NL + "Until" + NL, text.GetPortion(2, 5, 3, 5));
-
-            // A three-line range (to test that the middle line is included in full).
-            Assert.Equal("poke" + NL + "Until something bad broke," + NL + "And then" + NL, text.GetPortion(2, 19, 4, 8));
-
-            // An invalid but recoverable range (to test that a best effort is made rather than crashing).
-            Assert.Equal(NL + "Who couldn't leave software to fester -" + NL, text.GetPortion(0, int.MaxValue, 1, int.MaxValue));
-
-            // Some quite definitely dodgy ranges (to test that exceptions are thrown).
-            Assert.Throws<Exception>(() => text.GetPortion(-1, 0, 0, 0));
-            Assert.Throws<Exception>(() => text.GetPortion(0, -1, 0, 0));
-            Assert.Throws<Exception>(() => text.GetPortion(0, 0, -1, 0));
-            Assert.Throws<Exception>(() => text.GetPortion(0, 0, 0, -1));
-            Assert.Throws<Exception>(() => text.GetPortion(3, 5, 2, 5));
-            Assert.Throws<Exception>(() => text.GetPortion(2, 5, int.MaxValue, 5));
-        }
-
-        #endregion
-    }
-}

csharp/extractor/Semmle.Util/ActionMap.cs

@@ -1,45 +0,0 @@
-using System;
-using System.Collections.Generic;
-
-namespace Semmle.Util
-{
-    /// <summary>
-    /// A dictionary which performs an action when items are added to the dictionary.
-    /// The order in which keys and actions are added does not matter.
-    /// </summary>
-    /// <typeparam name="TKey"></typeparam>
-    /// <typeparam name="TValue"></typeparam>
-    public class ActionMap<TKey, TValue> where TKey : notnull
-    {
-        public void Add(TKey key, TValue value)
-        {
-
-            if (actions.TryGetValue(key, out var a))
-                a(value);
-            values[key] = value;
-        }
-
-        public void OnAdd(TKey key, Action<TValue> action)
-        {
-            if (actions.TryGetValue(key, out var a))
-            {
-                actions[key] = a + action;
-            }
-            else
-            {
-                actions.Add(key, action);
-            }
-
-            if (values.TryGetValue(key, out var val))
-            {
-                action(val);
-            }
-        }
-
-        // Action associated with each key.
-        private readonly Dictionary<TKey, Action<TValue>> actions = new Dictionary<TKey, Action<TValue>>();
-
-        // Values associated with each key.
-        private readonly Dictionary<TKey, TValue> values = new Dictionary<TKey, TValue>();
-    }
-}

csharp/extractor/Semmle.Util/Enumerators.cs

@@ -1,19 +0,0 @@
-using System.Collections.Generic;
-
-namespace Semmle.Util
-{
-    public static class Enumerators
-    {
-        /// <summary>
-        /// Create an enumerable with a single element.
-        /// </summary>
-        ///
-        /// <typeparam name="T">The type of the enumerable/element.</typeparam>
-        /// <param name="t">The element.</param>
-        /// <returns>An enumerable containing a single element.</returns>
-        public static IEnumerable<T> Singleton<T>(T t)
-        {
-            yield return t;
-        }
-    }
-}

csharp/extractor/Semmle.Util/EnvironmentVariables.cs

@@ -27,5 +27,12 @@ namespace Semmle.Util
             }
             return threads;
         }
+
+        public static bool GetBoolean(string name)
+        {
+            var env = Environment.GetEnvironmentVariable(name);
+            var _ = bool.TryParse(env, out var value);
+            return value;
+        }
     }
 }

csharp/extractor/Semmle.Util/FileRenamer.cs

@@ -1,34 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-
-namespace Semmle.Util
-{
-    /// <summary>
-    /// Utility to temporarily rename a set of files.
-    /// </summary>
-    public sealed class FileRenamer : IDisposable
-    {
-        private readonly string[] files;
-        private const string suffix = ".codeqlhidden";
-
-        public FileRenamer(IEnumerable<FileInfo> oldFiles)
-        {
-            files = oldFiles.Select(f => f.FullName).ToArray();
-
-            foreach (var file in files)
-            {
-                File.Move(file, file + suffix);
-            }
-        }
-
-        public void Dispose()
-        {
-            foreach (var file in files)
-            {
-                File.Move(file + suffix, file);
-            }
-        }
-    }
-}

csharp/extractor/Semmle.Util/FileUtils.cs

@@ -102,8 +102,7 @@ namespace Semmle.Util
         private static async Task DownloadFileAsync(string address, string filename)
         {
             using var httpClient = new HttpClient();
-            using var request = new HttpRequestMessage(HttpMethod.Get, address);
-            using var contentStream = await (await httpClient.SendAsync(request)).Content.ReadAsStreamAsync();
+            using var contentStream = await httpClient.GetStreamAsync(address);
             using var stream = new FileStream(filename, FileMode.Create, FileAccess.Write, FileShare.None, 4096, true);
             await contentStream.CopyToAsync(stream);
         }
@@ -112,7 +111,7 @@ namespace Semmle.Util
         /// Downloads the file at <paramref name="address"/> to <paramref name="fileName"/>.
         /// </summary>
         public static void DownloadFile(string address, string fileName) =>
-           DownloadFileAsync(address, fileName).Wait();
+           DownloadFileAsync(address, fileName).GetAwaiter().GetResult();
 
         public static string NestPaths(ILogger logger, string? outerpath, string innerpath)
         {

csharp/extractor/Semmle.Util/FuzzyDictionary.cs

@@ -1,165 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-
-namespace Semmle.Util
-{
-    /// <summary>
-    /// A dictionary from strings to elements of type T.
-    /// </summary>
-    ///
-    /// <remarks>
-    /// This data structure is able to locate items based on an "approximate match"
-    /// of the key. This is used for example when attempting to identify two terms
-    /// in different trap files which are similar but not identical.
-    ///
-    /// The algorithm locates the closest match to a string based on a "distance function".
-    ///
-    /// Whilst many distance functions are possible, a bespoke algorithm is used here,
-    /// for efficiency and suitability for the domain.
-    ///
-    /// The distance is defined as the Hamming Distance of the numbers in the string.
-    /// Each string is split into the base "form" (stripped of numbers) and a vector of
-    /// numbers. (Numbers are non-negative integers in this context).
-    ///
-    /// Strings with a different "form" are considered different and have a distance
-    /// of infinity.
-    ///
-    /// This distance function is reflexive, symmetric and obeys the triangle inequality.
-    ///
-    /// E.g. foo(bar,1,2) has form "foo(bar,,)" and integers {1,2}
-    ///
-    /// distance(foo(bar,1,2), foo(bar,1,2)) = 0
-    /// distance(foo(bar,1,2), foo(bar,1,3)) = 1
-    /// distance(foo(bar,2,1), foo(bar,1,2)) = 2
-    /// distance(foo(bar,1,2), foo(baz,1,2)) = infinity
-    /// </remarks>
-    ///
-    /// <typeparam name="T">The value type.</typeparam>
-    public class FuzzyDictionary<T> where T : class
-    {
-        // All data items indexed by the "base string" (stripped of numbers)
-        private readonly Dictionary<string, List<KeyValuePair<string, T>>> index = new Dictionary<string, List<KeyValuePair<string, T>>>();
-
-        /// <summary>
-        /// Stores a new KeyValuePair in the data structure.
-        /// </summary>
-        /// <param name="k">The key.</param>
-        /// <param name="v">The value.</param>
-        public void Add(string k, T v)
-        {
-            var kv = new KeyValuePair<string, T>(k, v);
-
-            var root = StripDigits(k);
-            index.AddAnother(root, kv);
-        }
-
-        /// <summary>
-        /// Computes the Hamming Distance between two sequences of the same length.
-        /// </summary>
-        /// <param name="v1">Vector 1</param>
-        /// <param name="v2">Vector 2</param>
-        /// <returns>The Hamming Distance.</returns>
-        private static int HammingDistance<TElement>(IEnumerable<TElement> v1, IEnumerable<TElement> v2) where TElement : notnull
-        {
-            return v1.Zip(v2, (x, y) => x.Equals(y) ? 0 : 1).Sum();
-        }
-
-        /// <summary>
-        /// Locates the value with the smallest Hamming Distance from the query.
-        /// </summary>
-        /// <param name="query">The query string.</param>
-        /// <param name="distance">The distance between the query string and the stored string.</param>
-        /// <returns>The best match, or null (default).</returns>
-        public T? FindMatch(string query, out int distance)
-        {
-            var root = StripDigits(query);
-            if (!index.TryGetValue(root, out var list))
-            {
-                distance = 0;
-                return default(T);
-            }
-
-            return BestMatch(query, list, (a, b) => HammingDistance(ExtractIntegers(a), ExtractIntegers(b)), out distance);
-        }
-
-        /// <summary>
-        /// Returns the best match (with the smallest distance) for a query.
-        /// </summary>
-        /// <param name="query">The query string.</param>
-        /// <param name="candidates">The list of candidate matches.</param>
-        /// <param name="distance">The distance function.</param>
-        /// <param name="bestDistance">The distance between the query and the stored string.</param>
-        /// <returns>The stored value.</returns>
-        private static T? BestMatch(string query, IEnumerable<KeyValuePair<string, T>> candidates, Func<string, string, int> distance, out int bestDistance)
-        {
-            var bestMatch = default(T);
-            bestDistance = 0;
-            var first = true;
-
-            foreach (var candidate in candidates)
-            {
-                var d = distance(query, candidate.Key);
-                if (d == 0)
-                    return candidate.Value;
-
-                if (first || d < bestDistance)
-                {
-                    bestDistance = d;
-                    bestMatch = candidate.Value;
-                    first = false;
-                }
-            }
-
-            return bestMatch;
-        }
-
-        /// <summary>
-        /// Removes all digits from a string.
-        /// </summary>
-        /// <param name="input">The input string.</param>
-        /// <returns>String with digits removed.</returns>
-        private static string StripDigits(string input)
-        {
-            var result = new StringBuilder();
-            foreach (var c in input.Where(c => !char.IsDigit(c)))
-                result.Append(c);
-            return result.ToString();
-        }
-
-        /// <summary>
-        /// Extracts and enumerates all non-negative integers in a string.
-        /// </summary>
-        /// <param name="input">The string to enumerate.</param>
-        /// <returns>The sequence of integers.</returns>
-        private static IEnumerable<int> ExtractIntegers(string input)
-        {
-            var inNumber = false;
-            var value = 0;
-            foreach (var c in input)
-            {
-                if (char.IsDigit(c))
-                {
-                    if (inNumber)
-                    {
-                        value = value * 10 + (c - '0');
-                    }
-                    else
-                    {
-                        inNumber = true;
-                        value = c - '0';
-                    }
-                }
-                else
-                {
-                    if (inNumber)
-                    {
-                        yield return value;
-                        inNumber = false;
-                    }
-                }
-            }
-        }
-    }
-}

csharp/extractor/Semmle.Util/SharedReference.cs

@@ -1,18 +0,0 @@
-namespace Semmle.Util
-{
-    /// <summary>
-    /// An instance of this class maintains a shared reference to an object.
-    /// This makes it possible for several different parts of the code to
-    /// share access to an object that can change (that is, they all want
-    /// to refer to the same object, but the object to which they jointly
-    /// refer may vary over time).
-    /// </summary>
-    /// <typeparam name="T">The type of the shared object.</typeparam>
-    public sealed class SharedReference<T> where T : class
-    {
-        /// <summary>
-        /// The shared object to which different parts of the code want to refer.
-        /// </summary>
-        public T? Obj { get; set; }
-    }
-}

csharp/extractor/Semmle.Util/StringExtensions.cs

@@ -1,41 +0,0 @@
-using System.Collections.Generic;
-using System.Linq;
-
-namespace Semmle.Util
-{
-    public static class StringExtensions
-    {
-        public static (string, string) Split(this string self, int index0)
-        {
-            var split = self.Split(new[] { index0 });
-            return (split[0], split[1]);
-        }
-
-        public static (string, string, string) Split(this string self, int index0, int index1)
-        {
-            var split = self.Split(new[] { index0, index1 });
-            return (split[0], split[1], split[2]);
-        }
-
-        public static (string, string, string, string) Split(this string self, int index0, int index1, int index2)
-        {
-            var split = self.Split(new[] { index0, index1, index2 });
-            return (split[0], split[1], split[2], split[3]);
-        }
-
-        private static List<string> Split(this string self, params int[] indices)
-        {
-            var ret = new List<string>();
-            var previousIndex = 0;
-            foreach (var index in indices.OrderBy(i => i))
-            {
-                ret.Add(self.Substring(previousIndex, index - previousIndex));
-                previousIndex = index;
-            }
-
-            ret.Add(self.Substring(previousIndex));
-
-            return ret;
-        }
-    }
-}

csharp/extractor/Semmle.Util/Text.cs

@@ -1,105 +0,0 @@
-using System;
-using System.IO;
-
-namespace Semmle.Util
-{
-    /// <summary>
-    /// An instance of this class represents a piece of text, e.g. the text of a C# source file.
-    /// </summary>
-    public sealed class Text
-    {
-        //#################### PRIVATE VARIABLES ####################
-        #region
-
-        /// <summary>
-        /// The text, stored line-by-line.
-        /// </summary>
-        private readonly string[] lines;
-
-        #endregion
-
-        //#################### CONSTRUCTORS ####################
-        #region
-
-        /// <summary>
-        /// Constructs a text object from an array of lines.
-        /// </summary>
-        /// <param name="lines">The lines of text.</param>
-        public Text(string[] lines)
-        {
-            this.lines = lines;
-        }
-
-        #endregion
-
-        //#################### PUBLIC METHODS ####################
-        #region
-
-        /// <summary>
-        /// Gets the whole text.
-        /// </summary>
-        /// <returns>The whole text.</returns>
-        public string GetAll()
-        {
-            using var sw = new StringWriter();
-            foreach (var s in lines)
-            {
-                sw.WriteLine(s);
-            }
-            return sw.ToString();
-        }
-
-        /// <summary>
-        /// Gets the portion of text that lies in the specified location range.
-        /// </summary>
-        /// <param name="startRow">The row at which the portion starts.</param>
-        /// <param name="startColumn">The column in the start row at which the portion starts.</param>
-        /// <param name="endRow">The row at which the portion ends.</param>
-        /// <param name="endColumn">The column in the end row at which the portion ends.</param>
-        /// <returns>The portion of text that lies in the specified location range.</returns>
-        public string GetPortion(int startRow, int startColumn, int endRow, int endColumn)
-        {
-            // Perform some basic validation on the range bounds.
-            if (startRow < 0 || endRow < 0 || startColumn < 0 || endColumn < 0 || endRow >= lines.Length || startRow > endRow)
-            {
-                throw new Exception
-                (
-                    string.Format("Bad range ({0},{1}):({2},{3}) in a piece of text with {4} lines", startRow, startColumn, endRow, endColumn, lines.Length)
-                );
-            }
-
-            using var sw = new StringWriter();
-            string line;
-
-            for (var i = startRow; i <= endRow; ++i)
-            {
-                if (i == startRow && i == endRow)
-                {
-                    // This is a single-line range, so take the bit between "startColumn" and "endColumn".
-                    line = startColumn <= lines[i].Length ? lines[i].Substring(startColumn, endColumn - startColumn) : "";
-                }
-                else if (i == startRow)
-                {
-                    // This is the first line of a multi-line range, so take the bit from "startColumn" onwards.
-                    line = startColumn <= lines[i].Length ? lines[i].Substring(startColumn) : "";
-                }
-                else if (i == endRow)
-                {
-                    // This is the last line of a multi-line range, so take the bit up to "endColumn".
-                    line = endColumn <= lines[i].Length ? lines[i].Substring(0, endColumn) : lines[i];
-                }
-                else
-                {
-                    // This is a line in the middle of a multi-line range, so take the whole line.
-                    line = lines[i];
-                }
-
-                sw.WriteLine(line);
-            }
-
-            return sw.ToString();
-        }
-
-        #endregion
-    }
-}

csharp/extractor/Semmle.Util/ToolStatusPage/DiagnosticMessage.cs

@@ -1,7 +1,6 @@
 using System;
 using System.Collections.Generic;
 using System.Globalization;
-using System.IO;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Converters;
 using Newtonsoft.Json.Serialization;
@@ -179,64 +178,4 @@ namespace Semmle.Util
             this.PlaintextMessage = plaintextMessage;
         }
     }
-
-    /// <summary>
-    /// Provides the ability to write diagnostic messages to some output.
-    /// </summary>
-    public interface IDiagnosticsWriter
-    {
-        /// <summary>
-        /// Adds <paramref name="message" /> as a new diagnostics entry.
-        /// </summary>
-        /// <param name="message">The diagnostics entry to add.</param>
-        void AddEntry(DiagnosticMessage message);
-    }
-
-    /// <summary>
-    /// A wrapper around an underlying <see cref="StreamWriter" /> which allows
-    /// <see cref="DiagnosticMessage" /> objects to be serialized to it.
-    /// </summary>
-    public sealed class DiagnosticsStream : IDiagnosticsWriter, IDisposable
-    {
-        private readonly JsonSerializer serializer;
-        private readonly StreamWriter writer;
-
-        /// <summary>
-        /// Initialises a new <see cref="DiagnosticsStream" /> for a file at <paramref name="path" />.
-        /// </summary>
-        /// <param name="path">The path to the file that should be created.</param>
-        public DiagnosticsStream(string path)
-        {
-            this.writer = File.CreateText(path);
-
-            var contractResolver = new DefaultContractResolver
-            {
-                NamingStrategy = new CamelCaseNamingStrategy()
-            };
-
-            serializer = new JsonSerializer
-            {
-                ContractResolver = contractResolver,
-                NullValueHandling = NullValueHandling.Ignore
-            };
-        }
-
-        /// <summary>
-        /// Adds <paramref name="message" /> as a new diagnostics entry.
-        /// </summary>
-        /// <param name="message">The diagnostics entry to add.</param>
-        public void AddEntry(DiagnosticMessage message)
-        {
-            serializer.Serialize(writer, message);
-            writer.Flush();
-        }
-
-        /// <summary>
-        /// Releases all resources used by the <see cref="DiagnosticsStream" /> object.
-        /// </summary>
-        public void Dispose()
-        {
-            writer.Dispose();
-        }
-    }
 }

csharp/extractor/Semmle.Util/ToolStatusPage/DiagnosticsStream.cs

@@ -0,0 +1,55 @@
+using System;
+using System.IO;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Serialization;
+
+namespace Semmle.Util
+{
+    /// <summary>
+    /// A wrapper around an underlying <see cref="StreamWriter" /> which allows
+    /// <see cref="DiagnosticMessage" /> objects to be serialized to it.
+    /// </summary>
+    public sealed class DiagnosticsStream : IDiagnosticsWriter
+    {
+        private readonly JsonSerializer serializer;
+        private readonly StreamWriter writer;
+
+        /// <summary>
+        /// Initialises a new <see cref="DiagnosticsStream" /> for a file at <paramref name="path" />.
+        /// </summary>
+        /// <param name="path">The path to the file that should be created.</param>
+        public DiagnosticsStream(string path)
+        {
+            this.writer = File.CreateText(path);
+
+            var contractResolver = new DefaultContractResolver
+            {
+                NamingStrategy = new CamelCaseNamingStrategy()
+            };
+
+            serializer = new JsonSerializer
+            {
+                ContractResolver = contractResolver,
+                NullValueHandling = NullValueHandling.Ignore
+            };
+        }
+
+        /// <summary>
+        /// Adds <paramref name="message" /> as a new diagnostics entry.
+        /// </summary>
+        /// <param name="message">The diagnostics entry to add.</param>
+        public void AddEntry(DiagnosticMessage message)
+        {
+            serializer.Serialize(writer, message);
+            writer.Flush();
+        }
+
+        /// <summary>
+        /// Releases all resources used by the <see cref="DiagnosticsStream" /> object.
+        /// </summary>
+        public void Dispose()
+        {
+            writer.Dispose();
+        }
+    }
+}

csharp/extractor/Semmle.Util/ToolStatusPage/IDiagnosticsWriter.cs

@@ -0,0 +1,16 @@
+using System;
+
+namespace Semmle.Util
+{
+    /// <summary>
+    /// Provides the ability to write diagnostic messages to some output.
+    /// </summary>
+    public interface IDiagnosticsWriter : IDisposable
+    {
+        /// <summary>
+        /// Adds <paramref name="message" /> as a new diagnostics entry.
+        /// </summary>
+        /// <param name="message">The diagnostics entry to add.</param>
+        void AddEntry(DiagnosticMessage message);
+    }
+}

csharp/extractor/Semmle.Util/Worklist.cs

@@ -1,57 +0,0 @@
-using System.Collections.Generic;
-
-namespace Semmle.Util
-{
-    /// <summary>
-    /// A worklist of items, providing the operations of adding an item, checking
-    /// whether there are new items and iterating a chunk of unprocessed items.
-    /// Any one item will only be accepted into the worklist once.
-    /// </summary>
-    public class Worklist<T>
-    {
-        private readonly HashSet<T> internalSet = new HashSet<T>();
-        private LinkedList<T> internalList = new LinkedList<T>();
-        private bool hasNewElements = false;
-
-        /// <summary>
-        /// Gets a value indicating whether this instance has had any new elements added
-        /// since the last time <c>GetUnprocessedElements()</c> was called.
-        /// </summary>
-        /// <value>
-        /// <c>true</c> if this instance has new elements; otherwise, <c>false</c>.
-        /// </value>
-        public bool HasNewElements => hasNewElements;
-
-        /// <summary>
-        /// Add the specified element to the worklist.
-        /// </summary>
-        /// <param name='element'>
-        /// If set to <c>true</c> element.
-        /// </param>
-        public bool Add(T element)
-        {
-            if (internalSet.Contains(element))
-                return false;
-            internalSet.Add(element);
-            internalList.AddLast(element);
-            hasNewElements = true;
-            return true;
-        }
-
-        /// <summary>
-        /// Gets the unprocessed elements that have been accumulated since the last time
-        /// this method was called. If <c>HasNewElements == true</c>, the resulting list
-        /// will be non-empty.
-        /// </summary>
-        /// <returns>
-        /// The unprocessed elements.
-        /// </returns>
-        public LinkedList<T> GetUnprocessedElements()
-        {
-            var result = internalList;
-            internalList = new LinkedList<T>();
-            hasNewElements = false;
-            return result;
-        }
-    }
-}

csharp/ql/campaigns/Solorigate/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-solorigate-all
-version: 1.7.13
+version: 1.7.14-dev
 groups:
   - csharp
   - solorigate

csharp/ql/campaigns/Solorigate/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-solorigate-queries
-version: 1.7.13
+version: 1.7.14-dev
 groups:
   - csharp
   - solorigate

csharp/ql/integration-tests/all-platforms/diag_recursive_generics/Types.ql

@@ -2,4 +2,4 @@ import csharp
 
 from Class c
 where c.fromSource()
-select c, c.getBaseClass().getFullyQualifiedName()
+select c, c.getBaseClass().getFullyQualifiedNameDebug()

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/Assemblies.expected

@@ -0,0 +1 @@
+| [...]/newtonsoft.json/13.0.3/lib/net6.0/Newtonsoft.Json.dll |

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/Assemblies.ql

@@ -0,0 +1,11 @@
+import csharp
+
+private string getPath(Assembly a) {
+  not a.getCompilation().getOutputAssembly() = a and
+  exists(string s | s = a.getFile().getAbsolutePath() |
+    result = "[...]/" + s.substring(s.indexOf("newtonsoft.json"), s.length())
+  )
+}
+
+from Assembly a
+select getPath(a)

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/CompilationInfo.expected

@@ -0,0 +1,13 @@
+| All Nuget feeds reachable | 0.0 |
+| Fallback nuget restore | 1.0 |
+| Project files on filesystem | 1.0 |
+| Resolved assembly conflicts | 7.0 |
+| Restored .NET framework variants | 0.0 |
+| Solution files on filesystem | 1.0 |
+| Source files generated | 0.0 |
+| Source files on filesystem | 1.0 |
+| Successfully ran fallback nuget restore | 1.0 |
+| Unresolved references | 0.0 |
+| UseWPF set | 0.0 |
+| UseWindowsForms set | 0.0 |
+| WebView extraction enabled | 1.0 |

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/CompilationInfo.ql

@@ -0,0 +1,15 @@
+import csharp
+import semmle.code.csharp.commons.Diagnostics
+
+query predicate compilationInfo(string key, float value) {
+  key != "Resolved references" and
+  not key.matches("Compiler diagnostic count for%") and
+  exists(Compilation c, string infoKey, string infoValue | infoValue = c.getInfo(infoKey) |
+    key = infoKey and
+    value = infoValue.toFloat()
+    or
+    not exists(infoValue.toFloat()) and
+    key = infoKey + ": " + infoValue and
+    value = 1
+  )
+}

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/diagnostics.expected

@@ -0,0 +1,42 @@
+{
+  "markdownMessage": "C# analysis with build-mode 'none' completed.",
+  "severity": "unknown",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/buildless/complete",
+    "name": "C# analysis with build-mode 'none' completed"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": false,
+    "telemetry": true
+  }
+}
+{
+  "markdownMessage": "C# with build-mode set to 'none'. This means that all C# source in the working directory will be scanned, with build tools, such as Nuget and Dotnet CLIs, only contributing information about external dependencies.",
+  "severity": "note",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/buildless/mode-active",
+    "name": "C# with build-mode set to 'none'"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}
+{
+  "markdownMessage": "Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.",
+  "severity": "warning",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/buildless/unreachable-feed",
+    "name": "Found unreachable Nuget feed in C# analysis with build-mode 'none'"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/proj/Program.cs

@@ -0,0 +1,6 @@
+class Program
+{
+    static void Main(string[] args)
+    {
+    }
+}