WIP

C#: Adjust compiler argument integration test

.bazelrc

@@ -11,7 +11,8 @@ common --override_module=semmle_code=%workspace%/misc/bazel/semmle_code_stub
 build --repo_env=CC=clang --repo_env=CXX=clang++
 
 build:linux --cxxopt=-std=c++20
-build:macos --cxxopt=-std=c++20 --cpu=darwin_x86_64
+# we currently cannot built the swift extractor for ARM
+build:macos --cxxopt=-std=c++20 --copt=-arch --copt=x86_64 --linkopt=-arch --linkopt=x86_64
 build:windows --cxxopt=/std:c++20 --cxxopt=/Zc:preprocessor
 
 # this requires developer mode, but is required to have pack installer functioning

.vscode/settings.json

@@ -1,5 +1,6 @@
 {
   "omnisharp.autoStart": false,
   "cmake.sourceDirectory": "${workspaceFolder}/swift",
-  "cmake.buildDirectory": "${workspaceFolder}/bazel-cmake-build"
+  "cmake.buildDirectory": "${workspaceFolder}/bazel-cmake-build",
+  "codeQL.githubDatabase.download": "never"
 }

cpp/ql/lib/experimental/buildless/ASTSig.qll

@@ -0,0 +1,88 @@
+import cpp
+
+/*
+    The syntax of a C++ program.
+*/
+signature module BuildlessASTSig
+{
+    class Node;
+    predicate nodeLocation(Node node, Location location);
+
+    // Parent/child relationship between AST nodes
+    predicate edge(Node parent, int index, Node child);
+
+    // Include graph
+    predicate userInclude(Node include, string path);
+    predicate systemInclude(Node include, string path);
+
+    // Namespaces
+    predicate namespace(Node ns);
+    predicate namespaceName(Node ns, string name);
+    predicate namespaceMember(Node ns, Node member);
+
+    // Functions
+    predicate function(Node fn);
+    predicate functionBody(Node fn, Node body);
+    predicate functionReturn(Node fn, Node returnType);
+    predicate functionName(Node fn, string name);
+    predicate functionParameter(Node fn, int i, Node parameterDecl);
+    predicate functionDefinition(Node fn);  // If a definition as opposed to a declaration
+
+    // Statements
+    predicate stmt(Node node);
+    predicate blockStmt(Node stmt);
+    predicate blockMember(Node stmt, int index, Node child);
+    predicate ifStmt(Node stmt, Node condition, Node thenBranch);
+    predicate ifStmt(Node tmt, Node condition, Node thenBranch, Node elseBranch);
+    predicate whileStmt(Node stmt, Node condition, Node body);
+    predicate doWhileStmt(Node stmt, Node condition, Node body);
+    predicate forStmt(Node stmt, Node init, Node condition, Node update, Node body);
+    predicate exprStmt(Node stmt, Node expr);
+    predicate returnStmt(Node stmt, Node expr);
+    predicate returnVoidStmt(Node stmt);
+    // etc
+
+    // Types
+    predicate type(Node type);
+    predicate ptrType(Node type, Node element);
+    predicate refType(Node type, Node element);
+    predicate constType(Node type, Node element);
+    predicate rvalueRefType(Node type, Node element);
+    predicate arrayType(Node type, Node element, Node size);
+    predicate arrayType(Node type, Node element);
+    predicate typename(Node node, string name);  // Any named type, including built-in types
+    predicate templated(Node node);
+    predicate typeDefinition(Node node);  // If a definition as opposed to a declaration
+
+    predicate classOrStructDefinition(Node node);
+    predicate classMember(Node classOrStruct, int child, Node member);
+
+    // Templates
+    predicate templateParameter(Node node, int i, Node parameter);
+    predicate typeParameter(Node templateParameter, Node type, Node parameter);
+    predicate typeParameterDefault(Node templateParameter, Node defaultTypeOrValue);
+
+    // Declarations
+    predicate variableDeclaration(Node decl);
+    predicate variableDeclarationType(Node decl, Node type);
+    predicate variableDeclarationEntry(Node decl, int index, Node entry);
+    predicate variableDeclarationEntryInitializer(Node entry, Node initializer);
+    predicate variableName(Node entry, string name);
+    predicate ptrEntry(Node entry, Node element);
+    predicate refEntry(Node entry, Node element);
+    predicate rvalueRefEntry(Node entry, Node element);
+    predicate arrayEntry(Node entry, Node element);  // ?? Size
+
+    // Expressions
+    predicate expression(Node node);
+    predicate prefixExpr(Node expr, string operator, Node operand);
+    predicate postfixExpr(Node expr, Node operand, string operator);
+    predicate binaryExpr(Node expr, Node lhs, string operator, Node rhs);
+    predicate castExpr(Node expr, Node type, Node operand);
+    predicate callExpr(Node call);
+    predicate callArgument(Node call, int i, Node arg);
+    predicate callReceiver(Node call, Node receiver);
+    predicate accessExpr(Node expr, string name);
+    predicate literal(Node expr, string value);
+    predicate stringLiteral(Node expr, string value);
+}

cpp/ql/lib/experimental/buildless/CompiledAST.qll

@@ -0,0 +1,304 @@
+import cpp
+import ASTSig
+
+module CompiledAST implements BuildlessASTSig {
+  private class SourceLocation extends Location {
+    SourceLocation() { not this.hasLocationInfo(_, 0, 0, 0, 0) }
+  }
+
+  private Type reachableType(Type type) {
+    result = type or
+    result = reachableType(type).stripTopLevelSpecifiers() or
+    result = reachableType(type).(PointerType).getBaseType() or
+    result = reachableType(type).(ReferenceType).getBaseType()
+  }
+
+  private class SourceDeclEntry extends DeclarationEntry {
+    SourceDeclEntry() {
+      not this.isAffectedByMacro() and
+      not this.isFromTemplateInstantiation(_) and
+      not this.isInMacroExpansion() and
+      not this.getDeclaration().isInMacroExpansion() and
+      this.getLocation() instanceof SourceLocation and
+      not this.(FunctionDeclarationEntry).getDeclaration().isCompilerGenerated()
+    }
+  }
+
+  private newtype TNode =
+    // TFunction(SourceLocation loc) { exists(Function f | f.getLocation() = loc) } or
+    TStatement(SourceLocation loc) { exists(Stmt s | s.getLocation() = loc) } or
+    TDeclaration(SourceDeclEntry decl) or
+    TExpression(SourceLocation loc) { exists(Expr e | e.getLocation() = loc) } or
+    TFunctionCallName(SourceLocation loc) { exists(FunctionCall c | c.getLocation() = loc) } or
+    TDeclarationType(SourceDeclEntry decl, Type type) { type = reachableType(decl.getType()) } or
+    TNamespaceDeclaration(NamespaceDeclarationEntry ns) { any() } or
+    TInclude(Include i)
+
+  class Node extends TNode {
+    string toString() { result = "node" }
+
+    SourceLocation getLocation() {
+      this = TStatement(result) or
+      result = this.getDeclaration().getLocation() or
+      this = TExpression(result) or
+      this = TFunctionCallName(result) or
+      result = this.getVariableDeclaration().getLocation() or
+      result = this.getNamespaceDeclaration().getLocation() or
+      result = this.getInclude().getLocation()
+    }
+
+    Include getInclude() { this = TInclude(result) }
+
+    Stmt getStmt() { this = TStatement(result.getLocation()) }
+
+    Function getFunction() {
+      result = this.getDeclaration().getDeclaration() and
+      not result.isFromTemplateInstantiation(_) and
+      not result.isCompilerGenerated()
+    }
+
+    SourceDeclEntry getDeclaration() { this = TDeclaration(result) }
+
+    NamespaceDeclarationEntry getNamespaceDeclaration() { this = TNamespaceDeclaration(result) }
+
+    Type getType() { this = TDeclarationType(_, result) }
+
+    SourceDeclEntry getVariableDeclaration() { this = TDeclarationType(result, _) }
+
+    Expr getExpr() { this = TExpression(result.getLocation()) }
+
+    FunctionCall getFunctionCallName() { this = TFunctionCallName(result.getLocation()) }
+  }
+
+  predicate nodeLocation(Node node, Location location) { location = node.getLocation() }
+
+  // Include graph
+  predicate userInclude(Node include, string path) {
+    exists(string head | head = include.getInclude().getHead() |
+      path = head.substring(1, head.length() - 1) and head.charAt(0) = "\""
+    )
+  }
+
+  predicate systemInclude(Node include, string path) {
+    exists(string head | head = include.getInclude().getHead() |
+      path = head.substring(1, head.length() - 1) and head.charAt(0) = "<"
+    )
+  }
+
+  // Functions
+  predicate function(Node fn) { exists(fn.getFunction()) }
+
+  predicate functionBody(Node fn, Node body) { body.getStmt() = fn.getFunction().getBlock() }
+
+  predicate functionReturn(Node fn, Node returnType) {
+    returnType.getVariableDeclaration() = fn.getDeclaration() and
+    fn.getDeclaration().(FunctionDeclarationEntry).getDeclaration().getType() = returnType.getType()
+  }
+
+  predicate functionName(Node fn, string name) { name = fn.getFunction().getName() }
+
+  predicate functionParameter(Node fn, int i, Node parameterDecl) {
+    functionParameter0(fn, i, parameterDecl) and
+    not exists(Node param2 | functionParameter0(fn, i, param2) |
+      param2.getLocation().getStartLine() < parameterDecl.getLocation().getStartLine()
+    )
+  }
+
+  pragma[nomagic]
+  private predicate functionParameter0(Node fn, int i, Node parameterDecl) {
+    fn.getFunction().getParameter(i).getADeclarationEntry() = parameterDecl.getDeclaration() and
+    fn.getLocation().getFile() = parameterDecl.getLocation().getFile() and
+    fn.getLocation().getStartLine() <= parameterDecl.getLocation().getStartLine()
+  }
+
+  // Statements
+  predicate stmt(Node node) { exists(node.getStmt()) }
+
+  predicate blockStmt(Node stmt) { stmt.getStmt() instanceof BlockStmt }
+
+  predicate blockMember(Node stmt, int index, Node child) {
+    child.getStmt() = stmt.getStmt().(BlockStmt).getChild(index)
+  }
+
+  predicate ifStmt(Node stmt, Node condition, Node thenBranch) { none() }
+
+  predicate ifStmt(Node tmt, Node condition, Node thenBranch, Node elseBranch) { none() }
+
+  predicate whileStmt(Node stmt, Node condition, Node body) { none() }
+
+  predicate doWhileStmt(Node stmt, Node condition, Node body) { none() }
+
+  predicate forStmt(Node stmt, Node init, Node condition, Node update, Node body) { none() }
+
+  predicate exprStmt(Node stmt, Node expr) { none() }
+
+  predicate returnStmt(Node stmt, Node expr) { none() }
+
+  predicate returnVoidStmt(Node stmt) { none() }
+
+  // etc
+  // Types
+  predicate ptrType(Node node, Node element) {
+    exists(PointerType type, SourceDeclEntry e |
+      node = TDeclarationType(e, type) and
+      element = TDeclarationType(e, type.getBaseType())
+    )
+  }
+
+  predicate refType(Node node, Node element) {
+    exists(ReferenceType type, SourceDeclEntry e |
+      node = TDeclarationType(e, type) and
+      element = TDeclarationType(e, type.getBaseType())
+    )
+  }
+
+  predicate rvalueRefType(Node type, Node element) { none() }
+
+  predicate arrayType(Node type, Node element, Node size) { none() }
+
+  predicate arrayType(Node type, Node element) { none() }
+
+  predicate typename(Node node, string name) {
+    exists(Class c | c = node.getDeclaration().getDeclaration() |
+      not c.isAnonymous() and c.getName() = name
+    )
+    or
+    name = node.getType().getName()
+  }
+
+  predicate templated(Node node) { none() }
+
+  predicate classOrStructDefinition(Node node) {
+    node.getDeclaration().getDeclaration() instanceof Class
+  }
+
+  predicate classMember(Node classOrStruct, int child, Node member) {
+    classOrStruct.getDeclaration().getDeclaration().(Class).getAMember() =
+      member.getDeclaration().getDeclaration() and
+    child = 0 and
+    classOrStruct.getLocation().getFile() = member.getLocation().getFile() // TODO: Disambiguate
+    // and not member.getDeclaration().getDeclaration() instanceof FriendDecl
+  }
+
+  // Templates
+  predicate templateParameter(Node node, int i, Node parameter) { none() }
+
+  predicate typeParameter(Node templateParameter, Node type, Node parameter) { none() }
+
+  predicate typeParameterDefault(Node templateParameter, Node defaultTypeOrValue) { none() }
+
+  // Declarations
+  predicate variableDeclaration(Node decl) {
+    decl.getDeclaration() instanceof VariableDeclarationEntry
+  }
+
+  pragma[nomagic]
+  private predicate variableDeclarationType2(Node decl, Node type) {
+    decl.getDeclaration() = type.getVariableDeclaration()
+  }
+
+  predicate variableDeclarationType(Node decl, Node type) {
+    variableDeclarationType2(decl, type) and
+    type.getType() = decl.getDeclaration().getType()
+  }
+
+  predicate variableDeclarationEntry(Node decl, int index, Node entry) { none() }
+
+  predicate variableDeclarationEntryInitializer(Node entry, Node initializer) { none() }
+
+  predicate variableName(Node decl, string name) {
+    decl.getDeclaration().(VariableDeclarationEntry).getName() = name
+  }
+
+  predicate ptrEntry(Node entry, Node element) { none() }
+
+  predicate refEntry(Node entry, Node element) { none() }
+
+  predicate rvalueRefEntry(Node entry, Node element) { none() }
+
+  predicate arrayEntry(Node entry, Node element) { none() }
+
+  // Expressions
+  predicate expression(Node node) { exists(node.getExpr()) or exists(node.getFunctionCallName()) }
+
+  predicate prefixExpr(Node expr, string operator, Node operand) { none() }
+
+  predicate postfixExpr(Node expr, Node operand, string operator) { none() }
+
+  predicate binaryExpr(Node expr, Node lhs, string operator, Node rhs) { none() }
+
+  predicate castExpr(Node expr, Node type, Node operand) { none() }
+
+  predicate callExpr(Node call) { call.getExpr() instanceof Call }
+
+  predicate callArgument(Node call, int i, Node arg) {
+    arg.getExpr() = call.getExpr().(Call).getArgument(i)
+  }
+
+  predicate callReceiver(Node call, Node receiver) {
+    receiver.getFunctionCallName() = call.getExpr()
+  }
+
+  predicate accessExpr(Node expr, string name) {
+    expr.getExpr().(VariableAccess).getTarget().getName() = name or
+    expr.getFunctionCallName().getTarget().getName() = name
+  }
+
+  predicate literal(Node expr, string value) { expr.getExpr().(Literal).toString() = value }
+
+  predicate stringLiteral(Node expr, string value) {
+    expr.getExpr().(StringLiteral).getValue() = value
+  }
+
+  predicate type(Node node) { node = TDeclarationType(_, _) }
+
+  predicate constType(Node node, Node element) {
+    exists(SpecifiedType type, SourceDeclEntry e |
+      node = TDeclarationType(e, type) and
+      element = TDeclarationType(e, type.getBaseType()) and
+      type.isConst()
+    )
+  }
+
+  predicate namespace(Node ns) { exists(ns.getNamespaceDeclaration()) }
+
+  predicate namespaceName(Node ns, string name) {
+    ns.getNamespaceDeclaration().getNamespace().getName() = name
+  }
+
+  pragma[nomagic]
+  private predicate namespaceNamespace(Node ns, Node child) {
+    ns.getNamespaceDeclaration().getNamespace().getAChildNamespace() =
+      child.getNamespaceDeclaration().getNamespace() and
+    ns.getLocation().getFile() = child.getLocation().getFile()
+  }
+
+  pragma[nomagic]
+  private predicate namespaceDecl(Node ns, Node child) {
+    child.getDeclaration().getDeclaration() =
+      ns.getNamespaceDeclaration().getNamespace().getADeclaration() and
+    ns.getLocation().getFile() = child.getLocation().getFile() and
+    ns.getLocation().getStartLine() <= child.getLocation().getStartLine()
+  }
+
+  predicate namespaceMember(Node ns, Node member) {
+    namespaceNamespace(ns, member) or
+    namespaceDecl(ns, member)
+  }
+
+  predicate edge(Node parent, int index, Node child) {
+    namespaceMember(parent, child) and index = 0
+    or
+    classMember(parent, index, child)
+    or
+    blockMember(parent, index, child)
+  }
+
+  predicate typeDefinition(Node node) {
+    node.getDeclaration().(TypeDeclarationEntry).isDefinition()
+  }
+
+  predicate functionDefinition(Node fn) {
+    fn.getDeclaration().(FunctionDeclarationEntry).isDefinition()
+  }
+}

cpp/ql/lib/experimental/buildless/Model.qll

@@ -0,0 +1,282 @@
+import AST
+
+module BuildlessModel<BuildlessASTSig Sig> {
+  module AST = BuildlessAST<Sig>;
+
+  private string getQualifiedName(AST::SourceNamespace ns) {
+    not exists(AST::SourceNamespace p | ns = p.getAChild()) and result = ns.getName()
+    or
+    exists(AST::SourceNamespace p | ns = p.getAChild() and ns != p |
+      result = getQualifiedName(p) + "::" + ns.getName()
+    )
+  }
+
+  private newtype TElement =
+    TNamespace(string fqn) { fqn = getQualifiedName(_) } or
+    TASTNode(AST::SourceElement node) { any() }
+
+    /*
+      Any compile-time concept that can be named.
+    */
+  class Entity extends string
+  {
+    bindingset[this] Entity() { any() }
+  }
+
+  /*
+    An entity that contains named members.
+  */
+  class Scope extends Entity
+  {
+    bindingset[this] Scope() { any() }
+
+    Member getAMember() { result = this.getAMember(_) }
+
+    abstract Member getAMember(string name);
+  }
+
+  /*
+    An entity that is a member of a scope.
+  */
+  class Member extends Entity
+  {
+    bindingset[this] Member() { any() }
+
+    abstract string getName();
+    abstract Scope getParent();
+  }
+
+  class Namespace2 extends Member, Scope
+  {
+    AST::SourceNamespace ns;
+
+    Namespace2() { this = "::" + getQualifiedName(ns) }
+
+    AST::SourceNamespace getAstNode() { result = ns }
+
+    override Namespace2 getParent() { result.getAstNode() = ns.getParent() }
+
+    override string getName() { result = ns.getName() }
+
+    override Member getAMember(string name) {
+      result = this.getMemberNamespace(name)
+      // !! Types
+    }
+
+    Namespace2 getMemberNamespace(string name) {
+      result.getParent() = this and result.getName() = name
+    }
+
+  }
+
+  class Type extends Member, Scope {
+    Type() { exists(SourceTypeDeclaration t | t.getMangledName() = this) }
+
+    AST::SourceType getAstNode() { result = this.getADeclaration().getSourceNode() }
+
+    // string toString() { result = "i am a type" }
+    override string getName() { result = this.getADeclaration().getName() }
+
+    Location getLocation() { result = this.getADefinition().getLocation() }
+
+    SourceTypeDeclaration getADeclaration() { result.getMangledName() = this }
+
+    SourceTypeDefinition getADefinition() { result.getMangledName() = this }
+
+    override Member getAMember(string name)
+    {
+      result = getMemberType(name)
+    }
+
+    Type getMemberType(string name)
+    {
+      none()
+    }
+
+    Namespace2 getParentNamespace() {
+      result.getAstNode() = this.getADeclaration().getParentNamespace()
+    }
+
+    Type getParentType() { result.getADeclaration() = this.getADeclaration().getParentType() }
+
+    override Scope getParent() { result = this.getParentNamespace() or result = this.getParentType() }
+
+    string getFullyQualifiedName() { result = this.getADeclaration().getFullyQualifiedName() }
+
+    Field getAField() { result.getParentType() = this }
+  }
+
+  private Type lookupNameInType(Type type, string name)
+  {
+    // Is the name a member of this type?
+
+    // Is the name in the same scope as the type?
+
+    // Is the name in global scope?
+
+    // TODO!
+    none()
+  }
+
+  class Field extends string {
+    Type containingType;
+    SourceTypeDefinition containingTypeDef;
+    AST::SourceVariableDeclaration fieldDef;
+
+    Field() {
+      containingType.getADefinition() = containingTypeDef and
+      fieldDef = containingTypeDef.getAField() and
+      this = containingTypeDef.getMangledName() + "::" + fieldDef.getName()
+    }
+
+    Location getLocation() { result = fieldDef.getLocation() }
+
+    Type getParentType() { result = containingType }
+
+    string getName() { result = fieldDef.getName() }
+
+    // TODO: The type of the field
+
+  }
+
+  class Element extends TElement {
+    string toString() { result = "element" }
+  }
+
+  class Namespace extends Element, TNamespace {
+    string getFullyQualifiedName() { this = TNamespace(result) }
+
+    override string toString() { result = "namespace " + this.getFullyQualifiedName() }
+
+    NamespaceDeclaration getADeclaration() { result.getNamespace() = this }
+  }
+
+  class SourceElement extends Element, TASTNode {
+    AST::SourceElement node;
+
+    SourceElement() { this = TASTNode(node) }
+
+    Location getLocation() { result = node.getLocation() }
+
+    AST::SourceElement getSourceNode() { result = node }
+  }
+
+  class SourceDeclaration extends SourceElement, TASTNode {
+    abstract SourceDeclaration getParent();
+
+    abstract string getName();
+
+    abstract string getMangledName();
+
+    abstract predicate isDefinition();
+  }
+
+  abstract class SourceDefinition extends SourceDeclaration { }
+
+  class NamespaceDeclaration extends SourceDeclaration {
+    AST::SourceNamespace ns;
+
+    NamespaceDeclaration() { ns = node }
+
+    override string getName() { result = ns.getName() }
+
+    Namespace getNamespace() { result.getFullyQualifiedName() = this.getFullyQualifiedName() }
+
+    override string toString() { result = "namespace " + this.getName() + " { ... }" }
+
+    override NamespaceDeclaration getParent() { result.getSourceNode() = node.getParent() }
+
+    string getFullyQualifiedName() {
+      if exists(this.getParent())
+      then result = this.getParent().getFullyQualifiedName() + "::" + this.getName()
+      else result = this.getName()
+    }
+
+    override string getMangledName() { result = "::" + this.getFullyQualifiedName() }
+
+    override predicate isDefinition() { any() }
+  }
+
+  class SourceTypeDeclaration extends SourceDeclaration {
+    AST::SourceTypeDefinition def;
+
+    SourceTypeDeclaration() { def = node }
+
+    override Location getLocation() { result = def.getLocation() }
+
+    override string toString() { result = "typename " + def.getName() }
+
+    string getFullyQualifiedName() {
+      if exists(this.getParentNamespace())
+      then result = this.getParentNamespace().getFullyQualifiedName() + "::" + this.getName()
+      else
+        if exists(this.getParentType())
+        then result = this.getParentType() + "::" + this.getName()
+        else result = this.getName()
+    }
+
+    NamespaceDeclaration getParentNamespace() { result.getSourceNode() = def.getParent() }
+
+    SourceTypeDeclaration getParentType() { result.getSourceNode() = def.getParent() }
+
+    override SourceDeclaration getParent() {
+      result = this.getParentNamespace() or result = this.getParentType()
+    }
+
+    override string getName() { result = def.getName() }
+
+    // Mangled name
+    override string getMangledName() {
+      if exists(this.getParent())
+      then result = this.getParent().getMangledName() + "::" + this.getName()
+      else result = "::" + this.getName()
+    }
+
+    override predicate isDefinition() { def.isDefinition() }
+
+    AST::SourceVariableDeclaration getAField() {
+      result = def.getAMember()
+    }
+  }
+
+  class SourceTypeDefinition extends SourceTypeDeclaration, SourceDefinition {
+    SourceTypeDefinition() { this.isDefinition() }
+  }
+
+  class SourceFunctionDeclaration extends SourceDeclaration {
+    AST::SourceFunction fn;
+
+    SourceFunctionDeclaration() { fn = node }
+
+    SourceTypeDeclaration getParentType() { result.getSourceNode() = fn.getParent() }
+
+    NamespaceDeclaration getParentNamespace() { result.getSourceNode() = fn.getParent() }
+
+    override SourceDeclaration getParent() {
+      result = this.getParentType() or result = this.getParentNamespace()
+    }
+
+    override string toString() { result = fn.getName() + "()" }
+
+    override Location getLocation() { result = fn.getLocation() }
+
+    override string getName() { result = fn.getName() }
+
+    override string getMangledName() {
+      if exists(this.getParent())
+      then result = this.getParent().getMangledName() + "::" + this.getName() + "()"
+      else result = "::" + this.getName() + "()"
+    }
+
+    override predicate isDefinition() { fn.isDefinition() }
+  }
+
+  class SourceFunctionDefinition extends SourceFunctionDeclaration, SourceDefinition {
+    SourceFunctionDefinition() { this.isDefinition() }
+  }
+
+  predicate invalidParent(SourceDeclaration decl) { decl.getParent+() = decl }
+}
+
+// For debugging in context
+module TestModel = BuildlessModel<CompiledAST>;

cpp/ql/lib/experimental/buildless/TODO.md

@@ -0,0 +1,46 @@
+Next:
+- [x] Namespaces
+- [ ] Unnamed class/struct/union and name mangling
+- [ ] Linker awareness is creating duplicates
+
+
+
+- Resolve the type of a local variable
+
+
+- [ ] Parent scopes - where are things declared
+    Parent class
+    Parent namespace
+    Parent function (for variable declarations and parameters)
+    Parent block (for blocks and statements)
+
+
+
+- [ ] Return type nodes
+- [ ] Construction of types
+- [ ] Construction of functions
+- [ ] Construction of Variables
+
+
+
+
+Names:
+- [x] Identify all function names
+- [x] Identify all parameter names
+- [x] Identify all variable declaration names
+- [x] Identify variable accesses
+
+Types:
+- [x] Locate user type definitions and typedefs
+- [x] Get the type of the parameter
+- [ ] Assign types to variable accesses
+
+Calls:
+- [x] Identify function call expressions
+- [x] Identify the "target" of a call in a trivial case.
+
+Overloads:
+
+Types:
+- [ ] Identify return types
+- [x] Identify parameter types and other declarations

cpp/ql/lib/experimental/buildless/ast.qll

@@ -0,0 +1,196 @@
+import CompiledAST
+import ASTSig
+
+module BuildlessAST<BuildlessASTSig AST> {
+  final class Node = AST::Node;
+
+  // Any node in the abstract syntax tree
+  class SourceElement extends Node {
+    Location getLocation() { AST::nodeLocation(this, result) }
+
+    string toString() { result = "element" }
+
+    SourceElement getParent() { AST::edge(result, _, this) }
+
+    SourceElement getChild(int i) { AST::edge(this, i, result) }
+
+    SourceElement getAChild() { result = this.getChild(_) }
+  }
+
+  bindingset[path]
+  private File getAnIncludeTarget(string path) {
+    exists(string p | p = result.toString() | path = p.suffix(p.length() - path.length()))
+  }
+
+  class Include extends SourceElement {
+    string path;
+
+    Include() { AST::userInclude(this, path) or AST::systemInclude(this, path) }
+
+    File getATarget() {
+      result = getAnIncludeTarget(path)
+      or
+      path.prefix(3) = "../" and result = getAnIncludeTarget(path.suffix(3))
+      or
+      path.prefix(2) = "./" and result = getAnIncludeTarget(path.suffix(2))
+    }
+
+    predicate isSystemInclude() { AST::systemInclude(this, _) }
+
+    predicate isUserInclude() { AST::userInclude(this, _) }
+
+    override string toString() {
+      this.isSystemInclude() and result = "#include <" + path + ">"
+      or
+      this.isUserInclude() and result = "#include \"" + path + "\""
+    }
+  }
+
+  abstract class SourceScope extends SourceElement { }
+
+  class SourceNamespace extends SourceScope, SourceDeclaration {
+    SourceNamespace() { AST::namespace(this) }
+
+    override string getName() { AST::namespaceName(this, result) }
+
+    override string toString() { result = "namespace " + this.getName() }
+  }
+
+  // Any syntax node that is a declaration
+  abstract class SourceDeclaration extends SourceElement {
+    abstract string getName();
+  }
+
+  // A syntax node that declares or defines a function
+  class SourceFunction extends SourceDeclaration {
+    SourceFunction() { AST::function(this) }
+
+    override string getName() { AST::functionName(this, result) }
+
+    override string toString() { result = this.getName() }
+
+    BlockStmt getBody() { AST::functionBody(this, result) }
+
+    SourceParameter getParameter(int i) { AST::functionParameter(this, i, result) }
+
+    SourceType getReturnType() { AST::functionReturn(this, result) }
+
+    predicate isDefinition() { AST::functionDefinition(this) }
+  }
+
+  // A syntax node that declares a variable (including fields and parameters)
+  class SourceVariableDeclaration extends SourceDeclaration {
+    SourceVariableDeclaration() { AST::variableDeclaration(this) }
+
+    override string getName() { AST::variableName(this, result) }
+
+    override string toString() { result = this.getName() }
+
+    SourceType getType() { AST::variableDeclarationType(this, result) }
+  }
+
+  // A syntax node that declares a parameter
+  class SourceParameter extends SourceVariableDeclaration {
+    SourceFunction fn;
+    int index;
+
+    SourceParameter() { AST::functionParameter(fn, index, this) }
+  }
+
+  class Stmt extends SourceElement {
+    Stmt() { AST::stmt(this) }
+
+    override string toString() { result = "stmt" }
+  }
+
+  class BlockStmt extends Stmt {
+    BlockStmt() { AST::blockStmt(this) }
+
+    override string toString() { result = "{ ... }" }
+  }
+
+  class Expr extends SourceElement {
+    Expr() { AST::expression(this) }
+  }
+
+  class AccessExpr extends Expr {
+    string identifier;
+
+    AccessExpr() { AST::accessExpr(this, identifier) }
+
+    string getName() { result = identifier }
+
+    override string toString() { result = this.getName() }
+  }
+
+  class CallExpr extends Expr {
+    CallExpr() { AST::callExpr(this) }
+
+    Expr getReceiver() { AST::callReceiver(this, result) }
+
+    Expr getArgument(int i) { AST::callArgument(this, i, result) }
+
+    override string toString() { result = "...(...)" }
+  }
+
+  class Literal extends Expr {
+    string value;
+
+    Literal() { AST::literal(this, value) }
+
+    override string toString() { result = value }
+
+    string getValue() { result = value }
+  }
+
+  class StringLiteral extends Literal {
+    StringLiteral() { AST::stringLiteral(this, _) }
+  }
+
+  abstract class SourceDefinition extends SourceDeclaration { }
+
+  class SourceTypeDefinition extends SourceDefinition {
+    SourceTypeDefinition() { AST::classOrStructDefinition(this) }
+
+    override string getName() { AST::typename(this, result) }
+
+    override string toString() { result = this.getName() }
+
+    SourceElement getAMember() { AST::classMember(this, _, result) }
+
+    predicate isDefinition() { AST::typeDefinition(this) }
+  }
+
+  // A node that contains a type of some kind
+  class SourceType extends SourceElement {
+    SourceType() { AST::type(this) }
+
+    override string toString() { AST::typename(this, result) }
+  }
+
+  class SourcePointer extends SourceType {
+    SourceType pointee;
+
+    SourcePointer() { AST::ptrType(this, pointee) }
+
+    SourceType getType() { result = pointee }
+  }
+
+  class SourceConst extends SourceType {
+    SourceType type;
+
+    SourceConst() { AST::constType(this, type) }
+
+    SourceType getType() { result = type }
+  }
+
+  class SourceReference extends SourceType {
+    SourceType type;
+
+    SourceReference() { AST::refType(this, type) }
+
+    SourceType getType() { result = type }
+  }
+}
+
+module TestAST = BuildlessAST<CompiledAST>;

cpp/ql/lib/experimental/buildless/checks.ql

@@ -0,0 +1,3 @@
+import Model
+
+select 1

cpp/ql/lib/experimental/buildless/identifiers.qll

@@ -0,0 +1,31 @@
+import ast_sig
+import ast
+import compiled_ast // For debugging in context
+
+module BuildlessIdentifiers<BuildlessASTSig A> {
+  module AST = Buildless<A>;
+
+  string getQualifiedName(AST::SourceNamespace ns) {
+    not exists(AST::SourceNamespace p | ns = p.getAChild()) and result = ns.getName()
+    or
+    exists(AST::SourceNamespace p | ns = p.getAChild() and ns !=p|
+      result = getQualifiedName(p) + "::" + ns.getName()
+    )
+  }
+
+  predicate namespace(string ns) {
+    ns = ["", getQualifiedName(_)]
+  }
+
+  // What are the identifiers in scope at a given point in the program?
+  // Give a potential object that the identifier refers to
+  AST::SourceDeclaration nameLookup(AST::SourceScope scope) {
+    result = scope
+    or
+    result = scope.(AST::SourceNamespace).getAChild()
+    or
+    result = scope.(AST::SourceTypeDefinition).getAMember()
+  }
+}
+
+module TestIdentifiers = BuildlessIdentifiers<CompiledAST>;

cpp/ql/lib/experimental/buildless/test_ast.ql

@@ -0,0 +1,26 @@
+import AST
+import types
+
+query TestAST::SourceFunction lua_copy() { result.getName() = "lua_copy" }
+
+query int lua_copy_count() { result = count(lua_copy()) }
+
+query predicate variables(TestAST::SourceVariableDeclaration decl, TestAST::SourceType sourceType) {
+  sourceType = decl.getType()
+}
+
+query predicate naiveCallTargets(TestAST::CallExpr call, TestAST::SourceFunction target) {
+  call.getReceiver().(TestAST::AccessExpr).getName() = target.getName() and
+  target.getName() = "max"
+}
+
+query predicate fnParents(TestAST::SourceFunction fn, TestAST::SourceNamespace parent) {
+  parent = fn.getParent()
+}
+
+query predicate includes(TestAST::Include include, File target) { target = include.getATarget() }
+
+query predicate unresolvedIncludes(TestAST::Include include) { not exists(include.getATarget()) }
+
+from TestAST::SourceFunction fn
+select fn, fn.getReturnType(), count(fn.getReturnType()), fn.getReturnType().getAQlClass()

cpp/ql/lib/experimental/buildless/test_model.ql

@@ -0,0 +1,13 @@
+import Model
+
+query predicate multipleDefinitions(TestModel::Type type, TestModel::SourceTypeDefinition def1, TestModel::SourceTypeDefinition def2)
+{
+    def1 = type.getADefinition() and
+    def2 = type.getADefinition() and 
+    def1.getLocation() != def2.getLocation() and
+    def1 != def2
+}
+
+from TestModel::SourceDeclaration decl
+where decl.getParent+()=decl
+select decl, "This has an invalid parent $@", decl.getParent()

cpp/ql/lib/experimental/buildless/test_types.ql

@@ -0,0 +1,34 @@
+import types
+
+query predicate constPointers(TestAST::SourceType t, TestAST::SourceConst c, TestAST::SourcePointer p)
+{
+    p.getType() = c and 
+    t = c.getType()
+}
+
+query predicate constRefs(TestAST::SourceType t, TestAST::SourceConst c, TestAST::SourceReference p)
+{
+    p.getType() = c and 
+    t = c.getType()
+}
+
+query predicate nestedNamespaces(TestAST::SourceNamespace parent, TestAST::SourceNamespace child)
+{
+    child = parent.getAChild()
+}
+
+query predicate recursiveNamespace(TestAST::SourceNamespace ns, TestAST::SourceNamespace descendents)
+{
+    ns = ns.getAChild+() and
+    descendents = ns.getAChild+()
+}
+
+query predicate usertypes(TestAST::SourceNamespace ns, TestAST::SourceTypeDefinition td)
+{
+    td = ns.getAChild()
+}
+
+// Let's try to resolve the type of a local variable
+
+from TestTypes::Type t
+select t

cpp/ql/lib/experimental/buildless/types.qll

@@ -0,0 +1,34 @@
+import AST
+
+module BuildlessTypes<BuildlessASTSig Sig> {
+  module AST = BuildlessAST<Sig>;
+
+  private newtype TType =
+    TBuiltinType(string name) { name = ["int", "char"] }
+    or
+    TUserType(string fqn) { exists(AST::SourceTypeDefinition d | d.getName() = fqn) }
+    //or
+    //TPointerType(Type type) { exists(A::SourcePointer p | p.getType() = type) }
+    //or
+    //TConstType(Type type) { exists(A::SourceConst c | c.getType() = type) }
+
+  class Type extends TType {
+    string toString() { result = this.getName() }
+
+    abstract string getName();
+    Location getLocation() { none() }
+  }
+
+  class BuiltinType extends Type, TBuiltinType {
+    override string getName() { this = TBuiltinType(result) }
+  }
+
+  class UserType extends Type, TUserType
+  {
+    override string getName() { this = TUserType(result) }
+
+    override Location getLocation() { exists(AST::SourceTypeDefinition d | this.getName() = d.getName() | result = d.getLocation()) }
+  }
+}
+
+module TestTypes = BuildlessTypes<CompiledAST>;

csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs

@@ -544,51 +544,6 @@ namespace Semmle.Autobuild.CSharp.Tests
             Assert.Equal(2, vcvarsfiles.Length);
         }
 
-        [Fact]
-        public void TestLinuxBuildlessExtractionSuccess()
-        {
-            actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 0;
-            actions.FileExists["csharp.log"] = true;
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
-            actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln";
-            actions.EnumerateDirectories[@"C:\Project"] = "";
-
-            var autobuilder = CreateAutoBuilder(false, buildless: "true");
-            TestAutobuilderScript(autobuilder, 0, 1);
-        }
-
-        [Fact]
-        public void TestLinuxBuildlessExtractionFailed()
-        {
-            actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 10;
-            actions.FileExists["csharp.log"] = true;
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
-            actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln";
-            actions.EnumerateDirectories[@"C:\Project"] = "";
-
-            var autobuilder = CreateAutoBuilder(false, buildless: "true");
-            TestAutobuilderScript(autobuilder, 10, 1);
-        }
-
-        [Fact]
-        public void TestLinuxBuildlessExtractionSolution()
-        {
-            actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 0;
-            actions.FileExists["csharp.log"] = true;
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
-            actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln";
-            actions.EnumerateDirectories[@"C:\Project"] = "";
-
-            var autobuilder = CreateAutoBuilder(false, buildless: "true");
-            TestAutobuilderScript(autobuilder, 0, 1);
-        }
-
         private void TestAutobuilderScript(CSharpAutobuilder autobuilder, int expectedOutput, int commandsRun)
         {
             Assert.Equal(expectedOutput, autobuilder.GetBuildScript().Run(actions, StartCallback, EndCallback));
@@ -677,21 +632,6 @@ namespace Semmle.Autobuild.CSharp.Tests
             TestAutobuilderScript(autobuilder, 0, 1);
         }
 
-        [Fact]
-        public void TestSkipNugetBuildless()
-        {
-            actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 0;
-            actions.FileExists["csharp.log"] = true;
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
-            actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln";
-            actions.EnumerateDirectories[@"C:\Project"] = "";
-
-            var autobuilder = CreateAutoBuilder(false, buildless: "true");
-            TestAutobuilderScript(autobuilder, 0, 1);
-        }
-
         [Fact]
         public void TestDotnetVersionNotInstalled()
         {

csharp/autobuilder/Semmle.Autobuild.CSharp/Semmle.Autobuild.CSharp.csproj

@@ -5,6 +5,7 @@
   <ItemGroup>
     <ProjectReference Include="..\..\extractor\Semmle.Util\Semmle.Util.csproj" />
     <ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp\Semmle.Extraction.CSharp.csproj" />
+    <ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp.Standalone\Semmle.Extraction.CSharp.Standalone.csproj" />
     <ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp.DependencyFetching\Semmle.Extraction.CSharp.DependencyFetching.csproj" />
     <ProjectReference Include="..\Semmle.Autobuild.Shared\Semmle.Autobuild.Shared.csproj" />
   </ItemGroup>

csharp/autobuilder/Semmle.Autobuild.CSharp/StandaloneBuildRule.cs

@@ -10,17 +10,7 @@ namespace Semmle.Autobuild.CSharp
     {
         public BuildScript Analyse(IAutobuilder<CSharpAutobuildOptions> builder, bool auto)
         {
-            if (builder.CodeQLExtractorLangRoot is null
-                || builder.CodeQlPlatform is null)
-            {
-                return BuildScript.Failure;
-            }
-
-            var standalone = builder.Actions.PathCombine(builder.CodeQLExtractorLangRoot, "tools", builder.CodeQlPlatform, "Semmle.Extraction.CSharp.Standalone");
-            var cmd = new CommandBuilder(builder.Actions);
-            cmd.RunCommand(standalone);
-
-            return cmd.Script;
+            return BuildScript.Create(_ => Semmle.Extraction.CSharp.Standalone.Program.Main([]));
         }
     }
 }

csharp/autobuilder/Semmle.Autobuild.Shared/Autobuilder.cs

@@ -73,16 +73,6 @@ namespace Semmle.Autobuild.Shared
         /// A logger.
         /// </summary>
         ILogger Logger { get; }
-
-        /// <summary>
-        /// Value of CODEQL_EXTRACTOR_<LANG>_ROOT environment variable.
-        /// </summary>
-        string? CodeQLExtractorLangRoot { get; }
-
-        /// <summary>
-        /// Value of CODEQL_PLATFORM environment variable.
-        /// </summary>
-        string? CodeQlPlatform { get; }
     }
 
     /// <summary>
@@ -197,9 +187,6 @@ namespace Semmle.Autobuild.Shared
                 return ret ?? new List<IProjectOrSolution>();
             });
 
-            CodeQLExtractorLangRoot = Actions.GetEnvironmentVariable(EnvVars.Root(this.Options.Language));
-            CodeQlPlatform = Actions.GetEnvironmentVariable(EnvVars.Platform);
-
             TrapDir = RequireEnvironmentVariable(EnvVars.TrapDir(this.Options.Language));
             SourceArchiveDir = RequireEnvironmentVariable(EnvVars.SourceArchiveDir(this.Options.Language));
             DiagnosticsDir = RequireEnvironmentVariable(EnvVars.DiagnosticDir(this.Options.Language));
@@ -364,15 +351,5 @@ namespace Semmle.Autobuild.Shared
                 diagnostics.Dispose();
             }
         }
-
-        /// <summary>
-        /// Value of CODEQL_EXTRACTOR_<LANG>_ROOT environment variable.
-        /// </summary>
-        public string? CodeQLExtractorLangRoot { get; }
-
-        /// <summary>
-        /// Value of CODEQL_PLATFORM environment variable.
-        /// </summary>
-        public string? CodeQlPlatform { get; }
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs

@@ -60,6 +60,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         /// </summary>
         public const string FallbackNugetFeeds = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_FALLBACK";
 
+        /// <summary>
+        /// Controls whether to include NuGet feeds from nuget.config files in the fallback restore logic.
+        /// </summary>
+        public const string AddNugetConfigFeedsToFallback = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_FALLBACK_INCLUDE_NUGET_CONFIG_FEEDS";
+
         /// <summary>
         /// Specifies the path to the nuget executable to be used for package restoration.
         /// </summary>

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs

@@ -98,12 +98,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             logger.LogInfo($"Checking NuGet feed responsiveness: {checkNugetFeedResponsiveness}");
             compilationInfoContainer.CompilationInfos.Add(("NuGet feed responsiveness checked", checkNugetFeedResponsiveness ? "1" : "0"));
 
+            HashSet<string>? explicitFeeds = null;
+
             try
             {
-                if (checkNugetFeedResponsiveness && !CheckFeeds())
+                if (checkNugetFeedResponsiveness && !CheckFeeds(out explicitFeeds))
                 {
                     // todo: we could also check the reachability of the inherited nuget feeds, but to use those in the fallback we would need to handle authentication too.
-                    var unresponsiveMissingPackageLocation = DownloadMissingPackagesFromSpecificFeeds();
+                    var unresponsiveMissingPackageLocation = DownloadMissingPackagesFromSpecificFeeds(explicitFeeds);
                     return unresponsiveMissingPackageLocation is null
                         ? []
                         : [unresponsiveMissingPackageLocation];
@@ -163,7 +165,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             LogAllUnusedPackages(dependencies);
 
             var missingPackageLocation = checkNugetFeedResponsiveness
-                ? DownloadMissingPackagesFromSpecificFeeds()
+                ? DownloadMissingPackagesFromSpecificFeeds(explicitFeeds)
                 : DownloadMissingPackages();
 
             if (missingPackageLocation is not null)
@@ -173,13 +175,24 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             return assemblyLookupLocations;
         }
 
-        private List<string> GetReachableFallbackNugetFeeds()
+        private List<string> GetReachableFallbackNugetFeeds(HashSet<string>? feedsFromNugetConfigs)
         {
             var fallbackFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.FallbackNugetFeeds).ToHashSet();
             if (fallbackFeeds.Count == 0)
             {
                 fallbackFeeds.Add(PublicNugetOrgFeed);
-                logger.LogInfo($"No fallback Nuget feeds specified. Using default feed: {PublicNugetOrgFeed}");
+                logger.LogInfo($"No fallback Nuget feeds specified. Adding default feed: {PublicNugetOrgFeed}");
+
+                var shouldAddNugetConfigFeeds = EnvironmentVariables.GetBooleanOptOut(EnvironmentVariableNames.AddNugetConfigFeedsToFallback);
+                logger.LogInfo($"Adding feeds from nuget.config to fallback restore: {shouldAddNugetConfigFeeds}");
+
+                if (shouldAddNugetConfigFeeds && feedsFromNugetConfigs?.Count > 0)
+                {
+                    // There are some feeds in `feedsFromNugetConfigs` that have already been checked for reachability, we could skip those.
+                    // But we might use different responsiveness testing settings when we try them in the fallback logic, so checking them again is safer.
+                    fallbackFeeds.UnionWith(feedsFromNugetConfigs);
+                    logger.LogInfo($"Using Nuget feeds from nuget.config files as fallback feeds: {string.Join(", ", feedsFromNugetConfigs.OrderBy(f => f))}");
+                }
             }
 
             logger.LogInfo($"Checking fallback Nuget feed reachability on feeds: {string.Join(", ", fallbackFeeds.OrderBy(f => f))}");
@@ -194,6 +207,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                 logger.LogInfo($"Reachable fallback Nuget feeds: {string.Join(", ", reachableFallbackFeeds.OrderBy(f => f))}");
             }
 
+            compilationInfoContainer.CompilationInfos.Add(("Reachable fallback Nuget feed count", reachableFallbackFeeds.Count.ToString()));
+
             return reachableFallbackFeeds;
         }
 
@@ -272,9 +287,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             compilationInfoContainer.CompilationInfos.Add(("Failed project restore with package source error", nugetSourceFailures.ToString()));
         }
 
-        private AssemblyLookupLocation? DownloadMissingPackagesFromSpecificFeeds()
+        private AssemblyLookupLocation? DownloadMissingPackagesFromSpecificFeeds(HashSet<string>? feedsFromNugetConfigs)
         {
-            var reachableFallbackFeeds = GetReachableFallbackNugetFeeds();
+            var reachableFallbackFeeds = GetReachableFallbackNugetFeeds(feedsFromNugetConfigs);
             if (reachableFallbackFeeds.Count > 0)
             {
                 return DownloadMissingPackages(fallbackNugetFeeds: reachableFallbackFeeds);
@@ -623,10 +638,10 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             return (timeoutMilliSeconds, tryCount);
         }
 
-        private bool CheckFeeds()
+        private bool CheckFeeds(out HashSet<string> explicitFeeds)
         {
             logger.LogInfo("Checking Nuget feeds...");
-            var (explicitFeeds, allFeeds) = GetAllFeeds();
+            (explicitFeeds, var allFeeds) = GetAllFeeds();
 
             var excludedFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.ExcludedNugetFeedsFromResponsivenessCheck)
                 .ToHashSet() ?? [];

csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/Compilation.cs

@@ -11,26 +11,20 @@ namespace Semmle.Extraction.CSharp.Entities
     {
         internal readonly ConcurrentDictionary<string, int> messageCounts = new();
 
-        private static (string Cwd, string[] Args) settings;
-        private static int hashCode;
-
-        public static (string Cwd, string[] Args) Settings
-        {
-            get { return settings; }
-            set
-            {
-                settings = value;
-                hashCode = settings.Cwd.GetHashCode();
-                for (var i = 0; i < settings.Args.Length; i++)
-                {
-                    hashCode = HashCode.Combine(hashCode, settings.Args[i].GetHashCode());
-                }
-            }
-        }
+        private readonly string cwd;
+        private readonly string[] args;
+        private readonly int hashCode;
 
 #nullable disable warnings
         private Compilation(Context cx) : base(cx, null)
         {
+            cwd = cx.Extractor.Cwd;
+            args = cx.Extractor.Args;
+            hashCode = cwd.GetHashCode();
+            for (var i = 0; i < args.Length; i++)
+            {
+                hashCode = HashCode.Combine(hashCode, args[i].GetHashCode());
+            }
         }
 #nullable restore warnings
 
@@ -38,14 +32,14 @@ namespace Semmle.Extraction.CSharp.Entities
         {
             var assembly = Assembly.CreateOutputAssembly(Context);
 
-            trapFile.compilations(this, FileUtils.ConvertToUnix(Compilation.Settings.Cwd));
+            trapFile.compilations(this, FileUtils.ConvertToUnix(cwd));
             trapFile.compilation_assembly(this, assembly);
 
             // Arguments
             var expandedIndex = 0;
-            for (var i = 0; i < Compilation.Settings.Args.Length; i++)
+            for (var i = 0; i < args.Length; i++)
             {
-                var arg = Compilation.Settings.Args[i];
+                var arg = args[i];
                 trapFile.compilation_args(this, i, arg);
 
                 if (CommandLineExtensions.IsFileArgument(arg))

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs

@@ -97,7 +97,8 @@ namespace Semmle.Extraction.CSharp
             stopwatch.Start();
 
             var options = Options.CreateWithEnvironment(args);
-            Entities.Compilation.Settings = (Directory.GetCurrentDirectory(), options.CompilerArguments.ToArray());
+            var workingDirectory = Directory.GetCurrentDirectory();
+            var compilerArgs = options.CompilerArguments.ToArray();
 
             using var logger = MakeLogger(options.Verbosity, options.Console);
 
@@ -123,7 +124,7 @@ namespace Semmle.Extraction.CSharp
 
                 var compilerArguments = CSharpCommandLineParser.Default.Parse(
                     compilerVersion.ArgsWithResponse,
-                    Entities.Compilation.Settings.Cwd,
+                    workingDirectory,
                     compilerVersion.FrameworkPath,
                     compilerVersion.AdditionalReferenceDirectories
                     );
@@ -131,7 +132,7 @@ namespace Semmle.Extraction.CSharp
                 if (compilerArguments is null)
                 {
                     var sb = new StringBuilder();
-                    sb.Append("  Failed to parse command line: ").AppendList(" ", Entities.Compilation.Settings.Args);
+                    sb.Append("  Failed to parse command line: ").AppendList(" ", compilerArgs);
                     logger.Log(Severity.Error, sb.ToString());
                     ++analyser.CompilationErrors;
                     return ExitCode.Failed;
@@ -143,7 +144,7 @@ namespace Semmle.Extraction.CSharp
                     return ExitCode.Ok;
                 }
 
-                return AnalyseTracing(analyser, compilerArguments, options, canonicalPathCache, stopwatch);
+                return AnalyseTracing(workingDirectory, compilerArgs, analyser, compilerArguments, options, canonicalPathCache, stopwatch);
             }
             catch (Exception ex)  // lgtm[cs/catch-of-all-exceptions]
             {
@@ -376,6 +377,8 @@ namespace Semmle.Extraction.CSharp
         }
 
         private static ExitCode AnalyseTracing(
+            string cwd,
+            string[] args,
             TracingAnalyser analyser,
             CSharpCommandLineArguments compilerArguments,
             Options options,
@@ -420,7 +423,7 @@ namespace Semmle.Extraction.CSharp
                             .WithMetadataImportOptions(MetadataImportOptions.All)
                         );
                 },
-                (compilation, options) => analyser.EndInitialize(compilerArguments, options, compilation),
+                (compilation, options) => analyser.EndInitialize(compilerArguments, options, compilation, cwd, args),
                 () => { });
         }
 

csharp/extractor/Semmle.Extraction.CSharp/Extractor/StandaloneAnalyser.cs

@@ -16,12 +16,10 @@ namespace Semmle.Extraction.CSharp
         public void Initialize(string outputPath, IEnumerable<(string, string)> compilationInfos, CSharpCompilation compilationIn, CommonOptions options)
         {
             compilation = compilationIn;
-            extractor = new StandaloneExtractor(outputPath, compilationInfos, Logger, PathTransformer, options);
+            extractor = new StandaloneExtractor(Directory.GetCurrentDirectory(), outputPath, compilationInfos, Logger, PathTransformer, options);
             this.options = options;
             LogExtractorInfo(Extraction.Extractor.Version);
             SetReferencePaths();
-
-            Entities.Compilation.Settings = (Directory.GetCurrentDirectory(), Array.Empty<string>());
         }
 
 #nullable disable warnings

csharp/extractor/Semmle.Extraction.CSharp/Extractor/TracingAnalyser.cs

@@ -38,13 +38,15 @@ namespace Semmle.Extraction.CSharp
         public void EndInitialize(
            CSharpCommandLineArguments commandLineArguments,
            CommonOptions options,
-           CSharpCompilation compilation)
+           CSharpCompilation compilation,
+           string cwd,
+           string[] args)
         {
             if (!init)
                 throw new InternalError("EndInitialize called without BeginInitialize returning true");
             this.options = options;
             this.compilation = compilation;
-            this.extractor = new TracingExtractor(GetOutputName(compilation, commandLineArguments), Logger, PathTransformer, options);
+            this.extractor = new TracingExtractor(cwd, args, GetOutputName(compilation, commandLineArguments), Logger, PathTransformer, options);
             LogDiagnostics();
 
             SetReferencePaths();

csharp/extractor/Semmle.Extraction/Extractor/Extractor.cs

@@ -10,6 +10,8 @@ namespace Semmle.Extraction
     /// </summary>
     public abstract class Extractor
     {
+        public string Cwd { get; init; }
+        public string[] Args { get; init; }
         public abstract ExtractorMode Mode { get; }
         public string OutputPath { get; }
         public IEnumerable<CompilationInfo> CompilationInfos { get; }
@@ -19,12 +21,14 @@ namespace Semmle.Extraction
         /// </summary>
         /// <param name="logger">The object used for logging.</param>
         /// <param name="pathTransformer">The object used for path transformations.</param>
-        protected Extractor(string outputPath, IEnumerable<CompilationInfo> compilationInfos, ILogger logger, PathTransformer pathTransformer)
+        protected Extractor(string cwd, string[] args, string outputPath, IEnumerable<CompilationInfo> compilationInfos, ILogger logger, PathTransformer pathTransformer)
         {
             OutputPath = outputPath;
             Logger = logger;
             PathTransformer = pathTransformer;
             CompilationInfos = compilationInfos;
+            Cwd = cwd;
+            Args = args;
         }
 
         // Limit the number of error messages in the log file

csharp/extractor/Semmle.Extraction/Extractor/StandaloneExtractor.cs

@@ -12,7 +12,8 @@ namespace Semmle.Extraction
         /// </summary>
         /// <param name="logger">The object used for logging.</param>
         /// <param name="pathTransformer">The object used for path transformations.</param>
-        public StandaloneExtractor(string outputPath, IEnumerable<(string, string)> compilationInfos, ILogger logger, PathTransformer pathTransformer, CommonOptions options) : base(outputPath, compilationInfos, logger, pathTransformer)
+        public StandaloneExtractor(string cwd, string outputPath, IEnumerable<(string, string)> compilationInfos, ILogger logger, PathTransformer pathTransformer, CommonOptions options)
+            : base(cwd, [], outputPath, compilationInfos, logger, pathTransformer)
         {
             Mode = ExtractorMode.Standalone;
             if (options.QlTest)

csharp/extractor/Semmle.Extraction/Extractor/TracingExtractor.cs

@@ -1,4 +1,3 @@
-using System.Linq;
 using Semmle.Util.Logging;
 
 namespace Semmle.Extraction
@@ -13,7 +12,8 @@ namespace Semmle.Extraction
         /// <param name="outputPath">The name of the output DLL/EXE, or null if not specified (standalone extraction).</param>
         /// <param name="logger">The object used for logging.</param>
         /// <param name="pathTransformer">The object used for path transformations.</param>
-        public TracingExtractor(string outputPath, ILogger logger, PathTransformer pathTransformer, CommonOptions options) : base(outputPath, Enumerable.Empty<(string, string)>(), logger, pathTransformer)
+        public TracingExtractor(string cwd, string[] args, string outputPath, ILogger logger, PathTransformer pathTransformer, CommonOptions options)
+            : base(cwd, args, outputPath, [], logger, pathTransformer)
         {
             Mode = ExtractorMode.None;
             if (options.QlTest)

csharp/ql/integration-tests/all-platforms/standalone_resx/CompilationInfo.expected

@@ -3,6 +3,7 @@
 | Failed solution restore with package source error | 0.0 |
 | NuGet feed responsiveness checked | 1.0 |
 | Project files on filesystem | 1.0 |
+| Reachable fallback Nuget feed count | 1.0 |
 | Resource extraction enabled | 1.0 |
 | Restored .NET framework variants | 1.0 |
 | Restored projects through solution files | 0.0 |

csharp/ql/integration-tests/linux-only/compiler_args/CompilerArgs.expected

@@ -1,9 +1,14 @@
 | 0 | /noconfig |
 | 1 | /unsafe- |
 | 2 | /checked- |
+| 3 | /nowarn:1701,1702,1701,1702 |
 | 4 | /fullpaths |
 | 5 | /nostdlib+ |
+| 6 | /errorreport:prompt |
+| 7 | /warn:8 |
+| 8 | /define:TRACE;DEBUG;NET;NET8_0;NETCOREAPP;NET5_0_OR_GREATER;NET6_0_OR_GREATER;NET7_0_OR_GREATER;NET8_0_OR_GREATER;NETCOREAPP1_0_OR_GREATER;NETCOREAPP1_1_OR_GREATER;NETCOREAPP2_0_OR_GREATER;NETCOREAPP2_1_OR_GREATER;NETCOREAPP2_2_OR_GREATER;NETCOREAPP3_0_OR_GREATER;NETCOREAPP3_1_OR_GREATER |
 | 9 | /highentropyva+ |
+| 10 | /nullable:enable |
 | 11 | /reference:[...]/8.0.1/ref/net8.0/Microsoft.CSharp.dll |
 | 12 | /reference:[...]/8.0.1/ref/net8.0/Microsoft.VisualBasic.Core.dll |
 | 13 | /reference:[...]/8.0.1/ref/net8.0/Microsoft.VisualBasic.dll |
@@ -168,10 +173,24 @@
 | 172 | /reference:[...]/8.0.1/ref/net8.0/System.Xml.XPath.XDocument.dll |
 | 173 | /reference:[...]/8.0.1/ref/net8.0/WindowsBase.dll |
 | 174 | /debug+ |
+| 175 | /debug:portable |
+| 176 | /filealign:512 |
+| 177 | /generatedfilesout:obj/Debug/net8.0//generated |
 | 178 | /optimize- |
+| 179 | /out:obj/Debug/net8.0/test.dll |
+| 180 | /refout:obj/Debug/net8.0/refint/test.dll |
+| 181 | /target:exe |
 | 182 | /warnaserror- |
 | 183 | /utf8output |
 | 184 | /deterministic+ |
+| 185 | /sourcelink:obj/Debug/net8.0/test.sourcelink.json |
+| 186 | /langversion:12.0 |
+| 187 | /embed:Program.cs |
+| 188 | /embed:obj/Debug/net8.0/test.GlobalUsings.g.cs |
+| 189 | /embed:"obj/Debug/net8.0/.NETCoreApp,Version=v8.0.AssemblyAttributes.cs" |
+| 190 | /embed:obj/Debug/net8.0/test.AssemblyInfo.cs |
+| 191 | /analyzerconfig:/home/runner/work/semmle-code/semmle-code/.editorconfig |
+| 192 | /analyzerconfig:obj/Debug/net8.0/test.GeneratedMSBuildEditorConfig.editorconfig |
 | 193 | /analyzerconfig:[...]/8.0.101/Sdks/Microsoft.NET.Sdk/analyzers/build/config/analysislevel_8_default.globalconfig |
 | 194 | /analyzer:[...]/8.0.101/Sdks/Microsoft.NET.Sdk/targets/../analyzers/Microsoft.CodeAnalysis.CSharp.NetAnalyzers.dll |
 | 195 | /analyzer:[...]/8.0.101/Sdks/Microsoft.NET.Sdk/targets/../analyzers/Microsoft.CodeAnalysis.NetAnalyzers.dll |
@@ -185,3 +204,4 @@
 | 203 | obj/Debug/net8.0/test.GlobalUsings.g.cs |
 | 204 | obj/Debug/net8.0/.NETCoreApp,Version=v8.0.AssemblyAttributes.cs |
 | 205 | obj/Debug/net8.0/test.AssemblyInfo.cs |
+| 206 | /warnaserror+:NU1605,SYSLIB0011 |

csharp/ql/integration-tests/linux-only/compiler_args/CompilerArgs.ql

@@ -3,7 +3,8 @@ import semmle.code.csharp.commons.Compilation
 
 bindingset[arg]
 private string normalize(string arg) {
-  not exists(arg.indexOf(":")) and result = arg
+  (not exists(arg.indexOf(":")) or not exists(arg.indexOf("/8.0"))) and
+  result = arg
   or
   exists(int i, int j |
     i = arg.indexOf(":") and

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error/CompilationInfo.expected

@@ -4,6 +4,7 @@
 | Fallback nuget restore | 1.0 |
 | NuGet feed responsiveness checked | 1.0 |
 | Project files on filesystem | 1.0 |
+| Reachable fallback Nuget feed count | 1.0 |
 | Resolved assembly conflicts | 7.0 |
 | Resource extraction enabled | 0.0 |
 | Restored .NET framework variants | 0.0 |

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/CompilationInfo.expected

@@ -3,6 +3,7 @@
 | Inherited Nuget feed count | 1.0 |
 | NuGet feed responsiveness checked | 1.0 |
 | Project files on filesystem | 1.0 |
+| Reachable fallback Nuget feed count | 1.0 |
 | Resolved assembly conflicts | 7.0 |
 | Resource extraction enabled | 0.0 |
 | Restored .NET framework variants | 0.0 |

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/Assemblies.expected

@@ -0,0 +1 @@
+| [...]/newtonsoft.json/13.0.3/lib/net6.0/Newtonsoft.Json.dll |

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/Assemblies.ql

@@ -0,0 +1,11 @@
+import csharp
+
+private string getPath(Assembly a) {
+  not a.getCompilation().getOutputAssembly() = a and
+  exists(string s | s = a.getFile().getAbsolutePath() |
+    result = "[...]/" + s.substring(s.indexOf("newtonsoft.json"), s.length())
+  )
+}
+
+from Assembly a
+select getPath(a)

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/CompilationInfo.expected

@@ -0,0 +1,16 @@
+| All Nuget feeds reachable | 0.0 |
+| Fallback nuget restore | 1.0 |
+| NuGet feed responsiveness checked | 1.0 |
+| Project files on filesystem | 1.0 |
+| Reachable fallback Nuget feed count | 2.0 |
+| Resolved assembly conflicts | 7.0 |
+| Resource extraction enabled | 0.0 |
+| Restored .NET framework variants | 0.0 |
+| Solution files on filesystem | 1.0 |
+| Source files generated | 0.0 |
+| Source files on filesystem | 1.0 |
+| Successfully ran fallback nuget restore | 1.0 |
+| Unresolved references | 0.0 |
+| UseWPF set | 0.0 |
+| UseWindowsForms set | 0.0 |
+| WebView extraction enabled | 1.0 |

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/CompilationInfo.ql

@@ -0,0 +1,15 @@
+import csharp
+import semmle.code.csharp.commons.Diagnostics
+
+query predicate compilationInfo(string key, float value) {
+  key != "Resolved references" and
+  not key.matches("Compiler diagnostic count for%") and
+  exists(Compilation c, string infoKey, string infoValue | infoValue = c.getInfo(infoKey) |
+    key = infoKey and
+    value = infoValue.toFloat()
+    or
+    not exists(infoValue.toFloat()) and
+    key = infoKey + ": " + infoValue and
+    value = 1
+  )
+}

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/diagnostics.expected

@@ -0,0 +1,42 @@
+{
+  "markdownMessage": "C# analysis with build-mode 'none' completed.",
+  "severity": "unknown",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/buildless/complete",
+    "name": "C# analysis with build-mode 'none' completed"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": false,
+    "telemetry": true
+  }
+}
+{
+  "markdownMessage": "C# was extracted with build-mode set to 'none'. This means that all C# source in the working directory will be scanned, with build tools, such as Nuget and Dotnet CLIs, only contributing information about external dependencies.",
+  "severity": "note",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/buildless/mode-active",
+    "name": "C# was extracted with build-mode set to 'none'"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}
+{
+  "markdownMessage": "Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.",
+  "severity": "warning",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/buildless/unreachable-feed",
+    "name": "Found unreachable Nuget feed in C# analysis with build-mode 'none'"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/proj/Program.cs

@@ -0,0 +1,6 @@
+class Program
+{
+    static void Main(string[] args)
+    {
+    }
+}

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/proj/nuget.config

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <packageSources>
+    <clear />
+    <add key="x" value="https://www.nuget.org/api/v2/" />
+  </packageSources>
+</configuration>

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/proj/proj.csproj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFrameworks>net8.0</TargetFrameworks>
+  </PropertyGroup>
+
+  <Target Name="DeleteBinObjFolders" BeforeTargets="Clean">
+    <RemoveDir Directories=".\bin" />
+    <RemoveDir Directories=".\obj" />
+  </Target>
+
+  <ItemGroup>
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+  </ItemGroup>
+</Project>

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/standalone.sln

@@ -0,0 +1,19 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.5.002.0
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "proj", "proj\proj.csproj", "{6ED00460-7666-4AE9-A405-4B6C8B02279A}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {4ED55A1C-066C-43DF-B32E-7EAA035985EE}
+	EndGlobalSection
+EndGlobal

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/test.py

@@ -0,0 +1,14 @@
+from create_database_utils import *
+from diagnostics_test_utils import *
+import os
+
+# os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK"] = "true"        # Nuget feed check is enabled by default
+os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_TIMEOUT"] = "1"     # 1ms, the GET request should fail with such short timeout
+os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_LIMIT"] = "1"       # Limit the count of checks to 1
+
+# Making sure the reachability test succeeds when doing a fallback restore:
+os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_FALLBACK_TIMEOUT"] = "1000"
+os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_FALLBACK_LIMIT"] = "5"
+
+run_codeql_database_create([], lang="csharp", extra_args=["--build-mode=none"])
+check_diagnostics()

csharp/ql/src/experimental/Security Features/JsonWebTokenHandler/JsonWebTokenHandlerLib.qll

@@ -125,26 +125,17 @@ class TokenValidationParametersProperty extends Property {
 predicate callableHasAReturnStmtAndAlwaysReturnsTrue(Callable c) {
   c.getReturnType() instanceof BoolType and
   not callableMayThrowException(c) and
-  forall(ReturnStmt rs | rs.getEnclosingCallable() = c |
+  forex(ReturnStmt rs | rs.getEnclosingCallable() = c |
     rs.getNumberOfChildren() = 1 and
     isExpressionAlwaysTrue(rs.getChildExpr(0))
-  ) and
-  exists(ReturnStmt rs | rs.getEnclosingCallable() = c)
+  )
 }
 
 /**
  * Holds if the lambda expression `le` always returns true
  */
 predicate lambdaExprReturnsOnlyLiteralTrue(AnonymousFunctionExpr le) {
-  le.getExpressionBody().(BoolLiteral).getBoolValue() = true
-  or
-  // special scenarios where the expression is not a `BoolLiteral`, but it will evaluatue to `true`
-  exists(Expr e | le.getExpressionBody() = e |
-    not e instanceof Call and
-    not e instanceof Literal and
-    e.getType() instanceof BoolType and
-    e.getValue() = "true"
-  )
+  isExpressionAlwaysTrue(le.getExpressionBody())
 }
 
 class CallableAlwaysReturnsTrue extends Callable {
@@ -152,12 +143,6 @@ class CallableAlwaysReturnsTrue extends Callable {
     callableHasAReturnStmtAndAlwaysReturnsTrue(this)
     or
     lambdaExprReturnsOnlyLiteralTrue(this)
-    or
-    exists(AnonymousFunctionExpr le, Call call, Callable callable | this = le |
-      callable.getACall() = call and
-      call = le.getExpressionBody() and
-      callableHasAReturnStmtAndAlwaysReturnsTrue(callable)
-    )
   }
 }
 
@@ -171,32 +156,6 @@ predicate callableOnlyThrowsArgumentNullException(Callable c) {
   )
 }
 
-/**
- * A specialization of `CallableAlwaysReturnsTrue` that takes into consideration exceptions being thrown for higher precision.
- */
-class CallableAlwaysReturnsTrueHigherPrecision extends CallableAlwaysReturnsTrue {
-  CallableAlwaysReturnsTrueHigherPrecision() {
-    callableOnlyThrowsArgumentNullException(this) and
-    (
-      forall(Call call, Callable callable | call.getEnclosingCallable() = this |
-        callable.getACall() = call and
-        callable instanceof CallableAlwaysReturnsTrueHigherPrecision
-      )
-      or
-      exists(AnonymousFunctionExpr le, Call call, CallableAlwaysReturnsTrueHigherPrecision cat |
-        this = le
-      |
-        le.canReturn(call) and
-        cat.getACall() = call
-      )
-      or
-      exists(LambdaExpr le | le = this |
-        le.getBody() instanceof CallableAlwaysReturnsTrueHigherPrecision
-      )
-    )
-  }
-}
-
 /**
  * A callable that returns a `string` and has a `string` as 1st argument
  */

csharp/ql/src/experimental/Security Features/JsonWebTokenHandler/delegated-security-validations-always-return-true.ql

@@ -17,9 +17,7 @@ import DataFlow
 import JsonWebTokenHandlerLib
 import semmle.code.csharp.commons.QualifiedName
 
-from
-  TokenValidationParametersProperty p, CallableAlwaysReturnsTrueHigherPrecision e, string qualifier,
-  string name
+from TokenValidationParametersProperty p, CallableAlwaysReturnsTrue e, string qualifier, string name
 where e = p.getAnAssignedValue() and p.hasFullyQualifiedName(qualifier, name)
 select e,
   "JsonWebTokenHandler security-sensitive property $@ is being delegated to this callable that always returns \"true\".",

csharp/tools/tracing-config.lua

@@ -179,8 +179,6 @@ function RegisterExtractorPack(id)
     end
 
     local windowsMatchers = {
-        CreatePatternMatcher({ '^semmle%.extraction%.csharp%.standalone%.exe$' },
-            MatchCompilerName, nil, { trace = false }),
         DotnetMatcherBuild,
         MsBuildMatcher,
         CreatePatternMatcher({ '^csc.*%.exe$' }, MatchCompilerName, extractor, {
@@ -222,9 +220,6 @@ function RegisterExtractorPack(id)
         end
     }
     local posixMatchers = {
-        -- The compiler name is case sensitive on Linux and lower cased on MacOS
-        CreatePatternMatcher({ '^semmle%.extraction%.csharp%.standalone$', '^Semmle%.Extraction%.CSharp%.Standalone$' },
-            MatchCompilerName, nil, { trace = false }),
         DotnetMatcherBuild,
         CreatePatternMatcher({ '^mcs%.exe$', '^csc%.exe$' }, MatchCompilerName,
             extractor, {

docs/codeql/ql-language-reference/annotations.rst

@@ -446,7 +446,7 @@ The ``pragma[assume_small_delta]`` annotation has no effect and can be safely re
 Language pragmas
 ================
 
-**Available for**: |classes|, |characteristic predicates|, |member predicates|, |non-member predicates|
+**Available for**: |modules|, |classes|, |characteristic predicates|, |member predicates|, |non-member predicates|
 
 ``language[monotonicAggregates]``
 ---------------------------------

java/documentation/library-coverage/coverage.csv

@@ -87,7 +87,7 @@ java.rmi,,,71,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,71,
 java.security,21,,543,,,11,10,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,539,4
 java.sql,15,1,303,,,,1,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,4,,9,,,,,,,,,1,,,,303,
 java.text,,,134,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,134,
-java.time,,,476,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,388,88
+java.time,,,123,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,35,88
 java.util,47,2,1218,,,,,,,,,1,,,,,,,,,,,34,,,,2,,,,5,2,,1,2,,,,,,,,,,,,,2,,,704,514
 javafx.scene.web,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,
 javax.accessibility,,,31,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,31,

java/documentation/library-coverage/coverage.rst

@@ -18,10 +18,10 @@ Java framework & library support
    `Google Guava <https://guava.dev/>`_,``com.google.common.*``,,730,43,9,,,,,
    JBoss Logging,``org.jboss.logging``,,,324,,,,,,
    `JSON-java <https://github.com/stleary/JSON-java>`_,``org.json``,,236,,,,,,,
-   Java Standard Library,``java.*``,10,4620,240,80,,9,,,26
+   Java Standard Library,``java.*``,10,4267,240,80,,9,,,26
    Java extensions,"``javax.*``, ``jakarta.*``",69,3257,85,5,4,2,1,1,4
    Kotlin Standard Library,``kotlin*``,,1849,16,14,,,,,2
    `Spring <https://spring.io/>`_,``org.springframework.*``,38,481,122,5,,28,14,,35
    Others,"``actions.osgi``, ``antlr``, ``ch.ethz.ssh2``, ``cn.hutool.core.codec``, ``com.alibaba.druid.sql``, ``com.alibaba.fastjson2``, ``com.amazonaws.auth``, ``com.auth0.jwt.algorithms``, ``com.azure.identity``, ``com.esotericsoftware.kryo.io``, ``com.esotericsoftware.kryo5.io``, ``com.fasterxml.jackson.core``, ``com.fasterxml.jackson.databind``, ``com.google.gson``, ``com.hubspot.jinjava``, ``com.jcraft.jsch``, ``com.microsoft.sqlserver.jdbc``, ``com.mitchellbosecke.pebble``, ``com.mongodb``, ``com.opensymphony.xwork2``, ``com.rabbitmq.client``, ``com.sshtools.j2ssh.authentication``, ``com.sun.crypto.provider``, ``com.sun.jndi.ldap``, ``com.sun.net.httpserver``, ``com.sun.net.ssl``, ``com.sun.rowset``, ``com.sun.security.auth.module``, ``com.sun.security.ntlm``, ``com.sun.security.sasl.digest``, ``com.thoughtworks.xstream``, ``com.trilead.ssh2``, ``com.unboundid.ldap.sdk``, ``com.zaxxer.hikari``, ``flexjson``, ``freemarker.cache``, ``freemarker.template``, ``groovy.lang``, ``groovy.text``, ``groovy.util``, ``hudson``, ``io.jsonwebtoken``, ``io.netty.bootstrap``, ``io.netty.buffer``, ``io.netty.channel``, ``io.netty.handler.codec``, ``io.netty.handler.ssl``, ``io.netty.handler.stream``, ``io.netty.resolver``, ``io.netty.util``, ``javafx.scene.web``, ``jenkins``, ``jodd.json``, ``liquibase.database.jvm``, ``liquibase.statement.core``, ``net.schmizz.sshj``, ``net.sf.json``, ``net.sf.saxon.s9api``, ``ognl``, ``okhttp3``, ``org.acegisecurity``, ``org.antlr.runtime``, ``org.apache.commons.codec``, ``org.apache.commons.compress.archivers.tar``, ``org.apache.commons.exec``, ``org.apache.commons.httpclient.util``, ``org.apache.commons.jelly``, ``org.apache.commons.jexl2``, ``org.apache.commons.jexl3``, ``org.apache.commons.lang``, ``org.apache.commons.logging``, ``org.apache.commons.net``, ``org.apache.commons.ognl``, ``org.apache.cxf.catalog``, ``org.apache.cxf.common.classloader``, ``org.apache.cxf.common.jaxb``, ``org.apache.cxf.common.logging``, ``org.apache.cxf.configuration.jsse``, ``org.apache.cxf.helpers``, ``org.apache.cxf.resource``, ``org.apache.cxf.staxutils``, ``org.apache.cxf.tools.corba.utils``, ``org.apache.cxf.tools.util``, ``org.apache.cxf.transform``, ``org.apache.directory.ldap.client.api``, ``org.apache.hadoop.fs``, ``org.apache.hadoop.hive.metastore``, ``org.apache.hadoop.hive.ql.exec``, ``org.apache.hadoop.hive.ql.metadata``, ``org.apache.hc.client5.http.async.methods``, ``org.apache.hc.client5.http.classic.methods``, ``org.apache.hc.client5.http.fluent``, ``org.apache.hive.hcatalog.templeton``, ``org.apache.ibatis.jdbc``, ``org.apache.ibatis.mapping``, ``org.apache.log4j``, ``org.apache.shiro.codec``, ``org.apache.shiro.jndi``, ``org.apache.shiro.mgt``, ``org.apache.sshd.client.session``, ``org.apache.struts.beanvalidation.validation.interceptor``, ``org.apache.struts2``, ``org.apache.tools.ant``, ``org.apache.tools.zip``, ``org.apache.velocity.app``, ``org.apache.velocity.runtime``, ``org.codehaus.cargo.container.installer``, ``org.codehaus.groovy.control``, ``org.dom4j``, ``org.eclipse.jetty.client``, ``org.fusesource.leveldbjni``, ``org.geogebra.web.full.main``, ``org.gradle.api.file``, ``org.hibernate``, ``org.influxdb``, ``org.jdbi.v3.core``, ``org.jenkins.ui.icon``, ``org.jenkins.ui.symbol``, ``org.jooq``, ``org.keycloak.models.map.storage``, ``org.kohsuke.stapler``, ``org.mvel2``, ``org.openjdk.jmh.runner.options``, ``org.owasp.esapi``, ``org.pac4j.jwt.config.encryption``, ``org.pac4j.jwt.config.signature``, ``org.scijava.log``, ``org.slf4j``, ``org.thymeleaf``, ``org.w3c.dom``, ``org.xml.sax``, ``org.xmlpull.v1``, ``org.yaml.snakeyaml``, ``play.libs.ws``, ``play.mvc``, ``ratpack.core.form``, ``ratpack.core.handling``, ``ratpack.core.http``, ``ratpack.exec``, ``ratpack.form``, ``ratpack.func``, ``ratpack.handling``, ``ratpack.http``, ``ratpack.util``, ``retrofit2``, ``sun.awt``, ``sun.jvmstat.perfdata.monitor.protocol.local``, ``sun.jvmstat.perfdata.monitor.protocol.rmi``, ``sun.management.spi``, ``sun.misc``, ``sun.net.ftp``, ``sun.net.www.protocol.http``, ``sun.nio.ch``, ``sun.security.acl``, ``sun.security.jgss.krb5``, ``sun.security.krb5``, ``sun.security.pkcs``, ``sun.security.pkcs11``, ``sun.security.provider``, ``sun.security.ssl``, ``sun.security.x509``, ``sun.tools.jconsole``, ``sun.util.logging.internal``",131,10596,893,125,6,22,18,,208
-   Totals,,310,25483,2569,338,16,128,33,1,409
+   Totals,,310,25130,2569,338,16,128,33,1,409
 

java/ql/lib/change-notes/2024-05-15-static-field-side-effect.md

@@ -0,0 +1,4 @@
+---
+category: majorAnalysis
+---
+* Added support for data flow through side-effects on static fields. For example, when a static field containing an array is updated.

java/ql/lib/semmle/code/java/dataflow/ExternalFlow.qll

@@ -413,25 +413,28 @@ private string paramsStringQualified(Callable c) {
 }
 
 private Element interpretElement0(
-  string package, string type, boolean subtypes, string name, string signature
+  string package, string type, boolean subtypes, string name, string signature, boolean isExact
 ) {
   elementSpec(package, type, subtypes, name, signature, _) and
   (
     exists(Member m |
       (
-        result = m
+        result = m and isExact = true
         or
-        subtypes = true and result.(SrcMethod).overridesOrInstantiates+(m)
+        subtypes = true and result.(SrcMethod).overridesOrInstantiates+(m) and isExact = false
       ) and
       m.hasQualifiedName(package, type, name)
     |
-      signature = "" or
-      paramsStringQualified(m) = signature or
+      signature = ""
+      or
+      paramsStringQualified(m) = signature
+      or
       paramsString(m) = signature
     )
     or
     exists(RefType t |
       t.hasQualifiedName(package, type) and
+      isExact = false and
       (if subtypes = true then result.(SrcRefType).getASourceSupertype*() = t else result = t) and
       name = "" and
       signature = ""
@@ -442,13 +445,16 @@ private Element interpretElement0(
 /** Gets the source/sink/summary/neutral element corresponding to the supplied parameters. */
 cached
 Element interpretElement(
-  string package, string type, boolean subtypes, string name, string signature, string ext
+  string package, string type, boolean subtypes, string name, string signature, string ext,
+  boolean isExact
 ) {
   elementSpec(package, type, subtypes, name, signature, ext) and
-  exists(Element e | e = interpretElement0(package, type, subtypes, name, signature) |
-    ext = "" and result = e
+  exists(Element e, boolean isExact0 |
+    e = interpretElement0(package, type, subtypes, name, signature, isExact0)
+  |
+    ext = "" and result = e and isExact = isExact0
     or
-    ext = "Annotated" and result.(Annotatable).getAnAnnotation().getType() = e
+    ext = "Annotated" and result.(Annotatable).getAnAnnotation().getType() = e and isExact = false
   )
 }
 
@@ -538,13 +544,13 @@ predicate sinkNode(Node node, string kind) { sinkNode(node, kind, _) }
 
 // adapter class for converting Mad summaries to `SummarizedCallable`s
 private class SummarizedCallableAdapter extends SummarizedCallable {
-  SummarizedCallableAdapter() { summaryElement(this, _, _, _, _, _) }
+  SummarizedCallableAdapter() { summaryElement(this, _, _, _, _, _, _) }
 
   private predicate relevantSummaryElementManual(
     string input, string output, string kind, string model
   ) {
     exists(Provenance provenance |
-      summaryElement(this, input, output, kind, provenance, model) and
+      summaryElement(this, input, output, kind, provenance, model, _) and
       provenance.isManual()
     )
   }
@@ -553,11 +559,11 @@ private class SummarizedCallableAdapter extends SummarizedCallable {
     string input, string output, string kind, string model
   ) {
     exists(Provenance provenance |
-      summaryElement(this, input, output, kind, provenance, model) and
+      summaryElement(this, input, output, kind, provenance, model, _) and
       provenance.isGenerated()
     ) and
     not exists(Provenance provenance |
-      neutralElement(this, "summary", provenance) and
+      neutralElement(this, "summary", provenance, _) and
       provenance.isManual()
     )
   }
@@ -576,18 +582,23 @@ private class SummarizedCallableAdapter extends SummarizedCallable {
   }
 
   override predicate hasProvenance(Provenance provenance) {
-    summaryElement(this, _, _, _, provenance, _)
+    summaryElement(this, _, _, _, provenance, _, _)
   }
+
+  override predicate hasExactModel() { summaryElement(this, _, _, _, _, _, true) }
 }
 
 // adapter class for converting Mad neutrals to `NeutralCallable`s
 private class NeutralCallableAdapter extends NeutralCallable {
   string kind;
   string provenance_;
+  boolean exact;
 
-  NeutralCallableAdapter() { neutralElement(this, kind, provenance_) }
+  NeutralCallableAdapter() { neutralElement(this, kind, provenance_, exact) }
 
   override string getKind() { result = kind }
 
   override predicate hasProvenance(Provenance provenance) { provenance = provenance_ }
+
+  override predicate hasExactModel() { exact = true }
 }

java/ql/lib/semmle/code/java/dataflow/FlowSummary.qll

@@ -135,6 +135,8 @@ private class SummarizedSyntheticCallableAdapter extends SummarizedCallable, TSy
       model = sc
     )
   }
+
+  override predicate hasExactModel() { any() }
 }
 
 deprecated class RequiredSummaryComponentStack = Impl::Private::RequiredSummaryComponentStack;

java/ql/lib/semmle/code/java/dataflow/internal/DataFlowDispatch.qll

@@ -19,7 +19,21 @@ private module DispatchImpl {
     )
   }
 
+  private predicate hasExactManualModel(Call c, Callable tgt) {
+    tgt = c.getCallee().getSourceDeclaration() and
+    (
+      exists(Impl::Public::SummarizedCallable sc |
+        sc.getACall() = c and sc.hasExactModel() and sc.hasManualModel()
+      )
+      or
+      exists(Impl::Public::NeutralSummaryCallable nc |
+        nc.getACall() = c and nc.hasExactModel() and nc.hasManualModel()
+      )
+    )
+  }
+
   private Callable sourceDispatch(Call c) {
+    not hasExactManualModel(c, result) and
     result = VirtualDispatch::viableCallable(c) and
     if VirtualDispatch::lowConfidenceDispatchTarget(c, result)
     then not hasHighConfidenceTarget(c)
@@ -122,12 +136,18 @@ private module DispatchImpl {
     mayBenefitFromCallContext(call.asCall(), _, _)
   }
 
+  bindingset[call, tgt]
+  pragma[inline_late]
+  private predicate viableCallableFilter(DataFlowCall call, DataFlowCallable tgt) {
+    tgt = viableCallable(call)
+  }
+
   /**
    * Gets a viable dispatch target of `call` in the context `ctx`. This is
    * restricted to those `call`s for which a context might make a difference.
    */
   DataFlowCallable viableImplInCallContext(DataFlowCall call, DataFlowCall ctx) {
-    result = viableCallable(call) and
+    viableCallableFilter(call, result) and
     exists(int i, Callable c, Method def, RefType t, boolean exact, MethodCall ma |
       ma = call.asCall() and
       mayBenefitFromCallContext(ma, c, i) and

java/ql/lib/semmle/code/java/dataflow/internal/DataFlowPrivate.qll

@@ -40,8 +40,11 @@ private predicate fieldStep(Node node1, Node node2) {
   exists(Field f |
     // Taint fields through assigned values only if they're static
     f.isStatic() and
-    f.getAnAssignedValue() = node1.asExpr() and
     node2.(FieldValueNode).getField() = f
+  |
+    f.getAnAssignedValue() = node1.asExpr()
+    or
+    f.getAnAccess() = node1.(PostUpdateNode).getPreUpdateNode().asExpr()
   )
   or
   exists(Field f, FieldRead fr |

java/ql/lib/semmle/code/java/dataflow/internal/FlowSummaryImpl.qll

@@ -131,7 +131,7 @@ private predicate relatedArgSpec(Callable c, string spec) {
     sourceModel(namespace, type, subtypes, name, signature, ext, spec, _, _, _) or
     sinkModel(namespace, type, subtypes, name, signature, ext, spec, _, _, _)
   |
-    c = interpretElement(namespace, type, subtypes, name, signature, ext)
+    c = interpretElement(namespace, type, subtypes, name, signature, ext, _)
   )
 }
 
@@ -202,7 +202,7 @@ module SourceSinkInterpretationInput implements
       sourceModel(namespace, type, subtypes, name, signature, ext, originalOutput, kind, provenance,
         madId) and
       model = "MaD:" + madId.toString() and
-      baseSource = interpretElement(namespace, type, subtypes, name, signature, ext) and
+      baseSource = interpretElement(namespace, type, subtypes, name, signature, ext, _) and
       (
         e = baseSource and output = originalOutput
         or
@@ -221,7 +221,7 @@ module SourceSinkInterpretationInput implements
       sinkModel(namespace, type, subtypes, name, signature, ext, originalInput, kind, provenance,
         madId) and
       model = "MaD:" + madId.toString() and
-      baseSink = interpretElement(namespace, type, subtypes, name, signature, ext) and
+      baseSink = interpretElement(namespace, type, subtypes, name, signature, ext, _) and
       (
         e = baseSink and originalInput = input
         or
@@ -310,7 +310,7 @@ module Private {
      */
     predicate summaryElement(
       Input::SummarizedCallableBase c, string input, string output, string kind, string provenance,
-      string model
+      string model, boolean isExact
     ) {
       exists(
         string namespace, string type, boolean subtypes, string name, string signature, string ext,
@@ -320,7 +320,7 @@ module Private {
         summaryModel(namespace, type, subtypes, name, signature, ext, originalInput, originalOutput,
           kind, provenance, madId) and
         model = "MaD:" + madId.toString() and
-        baseCallable = interpretElement(namespace, type, subtypes, name, signature, ext) and
+        baseCallable = interpretElement(namespace, type, subtypes, name, signature, ext, isExact) and
         (
           c.asCallable() = baseCallable and input = originalInput and output = originalOutput
           or
@@ -336,10 +336,12 @@ module Private {
      * Holds if a neutral model exists for `c` of kind `kind`
      * and with provenance `provenance`.
      */
-    predicate neutralElement(Input::SummarizedCallableBase c, string kind, string provenance) {
+    predicate neutralElement(
+      Input::SummarizedCallableBase c, string kind, string provenance, boolean isExact
+    ) {
       exists(string namespace, string type, string name, string signature |
         neutralModel(namespace, type, name, signature, kind, provenance) and
-        c.asCallable() = interpretElement(namespace, type, false, name, signature, "")
+        c.asCallable() = interpretElement(namespace, type, false, name, signature, "", isExact)
       )
     }
   }

java/ql/src/Security/CWE/CWE-501/TrustBoundaryViolation.ql

@@ -16,5 +16,5 @@ import TrustBoundaryFlow::PathGraph
 
 from TrustBoundaryFlow::PathNode source, TrustBoundaryFlow::PathNode sink
 where TrustBoundaryFlow::flowPath(source, sink)
-select sink.getNode(), sink, source,
-  "This servlet reads data from a remote source and writes it to a session variable."
+select sink.getNode(), source, sink,
+  "This servlet reads data from a $@ and writes it to a session variable.", source, "remote source"

java/ql/src/Security/CWE/CWE-611/XXE.qhelp

@@ -28,10 +28,9 @@ This improves security but the code will still be at risk of denial of service a
 Protection against denial of service attacks may also be implemented by setting entity expansion limits, which is done
 by default in recent JDK and JRE implementations.
 
-Because there are many different ways to disable external entity retrieval with varying support between different providers,
-in this query we choose to specifically check for the <a href="https://cheatsheetseries.owasp.org/cheatsheets/XML_External_Entity_Prevention_Cheat_Sheet.html#java">OWASP recommended way</a>
-to disable external entity retrieval for a particular parser. There may be other ways of making a particular parser safe
-which deviate from these guidelines, in which case this query will continue to flag the parser as potentially dangerous.
+We recommend visiting OWASP's <a href="https://cheatsheetseries.owasp.org/cheatsheets/XML_External_Entity_Prevention_Cheat_Sheet.html#java">XML Entity Prevention Cheat Sheet</a>,
+finding the specific XML parser, and applying the mitigation listed there. Other mitigations might be sufficient in some cases, but manual verification will be needed,
+as the query will continue to flag the parser as potentially dangerous.
 </p>
 </recommendation>
 

java/ql/src/change-notes/2024-05-23-trusted-boundary-violation.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* The alert message for the query "Trust boundary violation" (`java/trust-boundary-violation`) has been updated to include a link to the remote source.

java/ql/src/utils/modeleditor/ModelEditor.qll

@@ -77,7 +77,7 @@ class Endpoint extends Callable {
   predicate isNeutral() {
     exists(string namespace, string type, string name, string signature |
       neutralModel(namespace, type, name, signature, _, _) and
-      this = interpretElement(namespace, type, false, name, signature, "")
+      this = interpretElement(namespace, type, false, name, signature, "", _)
     )
   }
 

java/ql/src/utils/modelgenerator/internal/CaptureModelsSpecific.qll

@@ -80,10 +80,11 @@ predicate isUninterestingForDataFlowModels(Callable api) {
 predicate isUninterestingForTypeBasedFlowModels(Callable api) { none() }
 
 /**
- * A class of Callables that are relevant for generating summary, source and sinks models for.
+ * A class of callables that are potentially relevant for generating summary, source, sink
+ * and neutral models.
  *
- * In the Standard library and 3rd party libraries it the Callables that can be called
- * from outside the library itself.
+ * In the Standard library and 3rd party libraries it is the callables (or callables that have a
+ * super implementation) that can be called from outside the library itself.
  */
 class TargetApiSpecific extends Callable {
   private Callable lift;
@@ -97,6 +98,11 @@ class TargetApiSpecific extends Callable {
    * Gets the callable that a model will be lifted to.
    */
   Callable lift() { result = lift }
+
+  /**
+   * Holds if this callable is relevant in terms of generating models.
+   */
+  predicate isRelevant() { relevant(this) }
 }
 
 private string isExtensible(Callable c) {
@@ -114,15 +120,13 @@ private string typeAsModel(Callable c) {
   )
 }
 
-private predicate partialLiftedModel(
-  TargetApiSpecific api, string type, string extensible, string name, string parameters
+private predicate partialModel(
+  Callable api, string type, string extensible, string name, string parameters
 ) {
-  exists(Callable c | c = api.lift() |
-    type = typeAsModel(c) and
-    extensible = isExtensible(c) and
-    name = c.getName() and
-    parameters = ExternalFlow::paramsString(c)
-  )
+  type = typeAsModel(api) and
+  extensible = isExtensible(api) and
+  name = api.getName() and
+  parameters = ExternalFlow::paramsString(api)
 }
 
 /**
@@ -130,7 +134,7 @@ private predicate partialLiftedModel(
  */
 string asPartialModel(TargetApiSpecific api) {
   exists(string type, string extensible, string name, string parameters |
-    partialLiftedModel(api, type, extensible, name, parameters) and
+    partialModel(api.lift(), type, extensible, name, parameters) and
     result =
       type + ";" //
         + extensible + ";" //
@@ -145,7 +149,7 @@ string asPartialModel(TargetApiSpecific api) {
  */
 string asPartialNeutralModel(TargetApiSpecific api) {
   exists(string type, string name, string parameters |
-    partialLiftedModel(api, type, _, name, parameters) and
+    partialModel(api, type, _, name, parameters) and
     result =
       type + ";" //
         + name + ";" //

java/ql/src/utils/modelgenerator/internal/CaptureSummaryFlowQuery.qll

@@ -79,5 +79,6 @@ string captureFlow(DataFlowTargetApi api) {
  */
 string captureNoFlow(DataFlowTargetApi api) {
   not exists(DataFlowTargetApi api0 | exists(captureFlow(api0)) and api0.lift() = api.lift()) and
+  api.isRelevant() and
   result = ModelPrinting::asNeutralSummaryModel(api)
 }

java/ql/test/library-tests/dataflow/fields/G.java

@@ -0,0 +1,21 @@
+public class G {
+  static Object[] f;
+
+  void sink(Object o) { }
+
+  void runsink() {
+    sink(f[0]);
+  }
+
+  void test1() {
+    f[0] = new Object();
+  }
+
+  void test2() {
+    addObj(f);
+  }
+
+  void addObj(Object[] xs) {
+    xs[0] = new Object();
+  }
+}

java/ql/test/library-tests/dataflow/fields/flow.expected

@@ -29,3 +29,5 @@
 | F.java:5:14:5:25 | new Object(...) | F.java:20:10:20:17 | f.Field1 |
 | F.java:10:16:10:27 | new Object(...) | F.java:15:10:15:17 | f.Field1 |
 | F.java:24:9:24:20 | new Object(...) | F.java:33:10:33:17 | f.Field1 |
+| G.java:11:12:11:23 | new Object(...) | G.java:7:10:7:13 | ...[...] |
+| G.java:19:13:19:24 | new Object(...) | G.java:7:10:7:13 | ...[...] |

java/ql/test/utils/modelgenerator/dataflow/p/ImplOfExternalSPI.java

@@ -7,7 +7,7 @@ import java.nio.file.Files;
 public class ImplOfExternalSPI extends AbstractImplOfExternalSPI {
 
   // sink=p;AbstractImplOfExternalSPI;true;accept;(File);;Argument[0];path-injection;df-generated
-  // neutral=p;AbstractImplOfExternalSPI;accept;(File);summary;df-generated
+  // neutral=p;ImplOfExternalSPI;accept;(File);summary;df-generated
   @Override
   public boolean accept(File pathname) {
     try {

java/ql/test/utils/modelgenerator/dataflow/p/Inheritance.java

@@ -88,4 +88,28 @@ public class Inheritance {
       return s;
     }
   }
+
+  public interface INeutral {
+    String id(String s);
+  }
+
+  public class F implements INeutral {
+    // neutral=p;Inheritance$F;id;(String);summary;df-generated
+    public String id(String s) {
+      return "";
+    }
+  }
+
+  public class G implements INeutral {
+    // neutral=p;Inheritance$G;id;(String);summary;df-generated
+    public String id(String s) {
+      return "";
+    }
+  }
+
+  private class H implements INeutral {
+    public String id(String s) {
+      return "";
+    }
+  }
 }

java/ql/test/utils/modelgenerator/dataflow/p/PrivateFlowViaPublicInterface.java

@@ -45,7 +45,6 @@ public class PrivateFlowViaPublicInterface {
       return null;
     }
 
-    // neutral=p;PrivateFlowViaPublicInterface$SPI;openStreamNone;();summary;df-generated
     @Override
     public OutputStream openStreamNone() throws IOException {
       return new FileOutputStream(new RandomPojo().someFile);

javascript/ql/src/experimental/semmle/javascript/Execa.qll

@@ -0,0 +1,211 @@
+/**
+ * Models the `execa` library in terms of `FileSystemAccess` and `SystemCommandExecution`.
+ */
+
+import javascript
+
+/**
+ * Provide model for [Execa](https://github.com/sindresorhus/execa) package
+ */
+module Execa {
+  /**
+   * The Execa input file read and output file write
+   */
+  class ExecaFileSystemAccess extends FileSystemReadAccess, DataFlow::Node {
+    API::Node execaArg;
+    boolean isPipedToFile;
+
+    ExecaFileSystemAccess() {
+      (
+        execaArg = API::moduleImport("execa").getMember("$").getParameter(0) and
+        isPipedToFile = false
+        or
+        execaArg =
+          API::moduleImport("execa")
+              .getMember(["execa", "execaCommand", "execaCommandSync", "execaSync"])
+              .getParameter([0, 1, 2]) and
+        isPipedToFile = false
+        or
+        execaArg =
+          API::moduleImport("execa")
+              .getMember(["execa", "execaCommand", "execaCommandSync", "execaSync"])
+              .getReturn()
+              .getMember(["pipeStdout", "pipeAll", "pipeStderr"])
+              .getParameter(0) and
+        isPipedToFile = true
+      ) and
+      this = execaArg.asSink()
+    }
+
+    override DataFlow::Node getADataNode() { none() }
+
+    override DataFlow::Node getAPathArgument() {
+      result = execaArg.getMember("inputFile").asSink() and isPipedToFile = false
+      or
+      result = execaArg.asSink() and isPipedToFile = true
+    }
+  }
+
+  /**
+   * A call to `execa.execa` or `execa.execaSync`
+   */
+  class ExecaCall extends API::CallNode {
+    boolean isSync;
+
+    ExecaCall() {
+      this = API::moduleImport("execa").getMember("execa").getACall() and
+      isSync = false
+      or
+      this = API::moduleImport("execa").getMember("execaSync").getACall() and
+      isSync = true
+    }
+  }
+
+  /**
+   * The system command execution nodes for `execa.execa` or `execa.execaSync` functions
+   */
+  class ExecaExec extends SystemCommandExecution, ExecaCall {
+    ExecaExec() { isSync = [false, true] }
+
+    override DataFlow::Node getACommandArgument() { result = this.getArgument(0) }
+
+    override predicate isShellInterpreted(DataFlow::Node arg) {
+      // if shell: true then first and second args are sinks
+      // options can be third argument
+      arg = [this.getArgument(0), this.getParameter(1).getUnknownMember().asSink()] and
+      isExecaShellEnable(this.getParameter(2))
+      or
+      // options can be second argument
+      arg = this.getArgument(0) and
+      isExecaShellEnable(this.getParameter(1))
+    }
+
+    override DataFlow::Node getArgumentList() {
+      // execa(cmd, [arg]);
+      exists(DataFlow::Node arg | arg = this.getArgument(1) |
+        // if it is a object then it is a option argument not command argument
+        result = arg and not arg.asExpr() instanceof ObjectExpr
+      )
+    }
+
+    override predicate isSync() { isSync = true }
+
+    override DataFlow::Node getOptionsArg() {
+      result = this.getLastArgument() and result.asExpr() instanceof ObjectExpr
+    }
+  }
+
+  /**
+   * A call to `execa.$` or `execa.$.sync` or `execa.$({})` or `execa.$.sync({})` tag functions
+   */
+  private class ExecaScriptCall extends API::CallNode {
+    boolean isSync;
+
+    ExecaScriptCall() {
+      exists(API::Node script |
+        script =
+          [
+            API::moduleImport("execa").getMember("$"),
+            API::moduleImport("execa").getMember("$").getReturn()
+          ]
+      |
+        this = script.getACall() and
+        isSync = false
+        or
+        this = script.getMember("sync").getACall() and
+        isSync = true
+      )
+    }
+  }
+
+  /**
+   * The system command execution nodes for `execa.$` or `execa.$.sync` tag functions
+   */
+  class ExecaScript extends SystemCommandExecution, ExecaScriptCall {
+    ExecaScript() { isSync = [false, true] }
+
+    override DataFlow::Node getACommandArgument() {
+      result = this.getParameter(1).asSink() and
+      not isTaggedTemplateFirstChildAnElement(this.getParameter(1).asSink().asExpr().getParent())
+    }
+
+    override predicate isShellInterpreted(DataFlow::Node arg) {
+      isExecaShellEnable(this.getParameter(0)) and
+      arg = this.getAParameter().asSink()
+    }
+
+    override DataFlow::Node getArgumentList() {
+      result = this.getParameter(any(int i | i >= 1)).asSink() and
+      isTaggedTemplateFirstChildAnElement(this.getParameter(1).asSink().asExpr().getParent())
+      or
+      result = this.getParameter(any(int i | i >= 2)).asSink() and
+      not isTaggedTemplateFirstChildAnElement(this.getParameter(1).asSink().asExpr().getParent())
+    }
+
+    override DataFlow::Node getOptionsArg() { result = this.getParameter(0).asSink() }
+
+    override predicate isSync() { isSync = true }
+  }
+
+  /**
+   * A call to `execa.execaCommandSync` or `execa.execaCommand`
+   */
+  private class ExecaCommandCall extends API::CallNode {
+    boolean isSync;
+
+    ExecaCommandCall() {
+      this = API::moduleImport("execa").getMember("execaCommandSync").getACall() and
+      isSync = true
+      or
+      this = API::moduleImport("execa").getMember("execaCommand").getACall() and
+      isSync = false
+    }
+  }
+
+  /**
+   * The system command execution nodes for `execa.execaCommand` or `execa.execaCommandSync` functions
+   */
+  class ExecaCommandExec extends SystemCommandExecution, ExecaCommandCall {
+    ExecaCommandExec() { isSync = [false, true] }
+
+    override DataFlow::Node getACommandArgument() {
+      result = this.(DataFlow::CallNode).getArgument(0)
+    }
+
+    override DataFlow::Node getArgumentList() {
+      // execaCommand(`${cmd} ${arg}`);
+      result.asExpr() = this.getParameter(0).asSink().asExpr().getAChildExpr() and
+      not result.asExpr() = this.getArgument(0).asExpr().getChildExpr(0)
+    }
+
+    override predicate isShellInterpreted(DataFlow::Node arg) {
+      // execaCommandSync(`${cmd} ${arg}`, {shell: true})
+      arg.asExpr() = this.getArgument(0).asExpr().getAChildExpr+() and
+      isExecaShellEnable(this.getParameter(1))
+      or
+      // there is only one argument that is constructed in previous nodes,
+      // it makes sanitizing really hard to select whether it is vulnerable to argument injection or not
+      arg = this.getParameter(0).asSink() and
+      not exists(this.getArgument(0).asExpr().getChildExpr(1))
+    }
+
+    override predicate isSync() { isSync = true }
+
+    override DataFlow::Node getOptionsArg() {
+      result = this.getLastArgument() and result.asExpr() instanceof ObjectExpr
+    }
+  }
+
+  /** Gets a TemplateLiteral and check if first child is a template element */
+  private predicate isTaggedTemplateFirstChildAnElement(TemplateLiteral templateLit) {
+    exists(templateLit.getChildExpr(0).(TemplateElement))
+  }
+
+  /**
+   * Holds whether Execa has shell enabled options or not, get Parameter responsible for options
+   */
+  pragma[inline]
+  private predicate isExecaShellEnable(API::Node n) {
+    n.getMember("shell").asSink().asExpr().(BooleanLiteral).getValue() = "true"
+  }
+}

javascript/ql/test/experimental/Execa/CommandInjection/tests.expected

@@ -0,0 +1,22 @@
+passingPositiveTests
+| PASSED | CommandInjection | tests.js:11:46:11:70 | // test ... jection |
+| PASSED | CommandInjection | tests.js:12:43:12:67 | // test ... jection |
+| PASSED | CommandInjection | tests.js:13:63:13:87 | // test ... jection |
+| PASSED | CommandInjection | tests.js:14:62:14:86 | // test ... jection |
+| PASSED | CommandInjection | tests.js:15:60:15:84 | // test ... jection |
+| PASSED | CommandInjection | tests.js:17:45:17:69 | // test ... jection |
+| PASSED | CommandInjection | tests.js:18:42:18:66 | // test ... jection |
+| PASSED | CommandInjection | tests.js:19:62:19:86 | // test ... jection |
+| PASSED | CommandInjection | tests.js:20:63:20:87 | // test ... jection |
+| PASSED | CommandInjection | tests.js:21:60:21:84 | // test ... jection |
+| PASSED | CommandInjection | tests.js:23:43:23:67 | // test ... jection |
+| PASSED | CommandInjection | tests.js:24:40:24:64 | // test ... jection |
+| PASSED | CommandInjection | tests.js:25:40:25:64 | // test ... jection |
+| PASSED | CommandInjection | tests.js:26:60:26:84 | // test ... jection |
+| PASSED | CommandInjection | tests.js:28:41:28:65 | // test ... jection |
+| PASSED | CommandInjection | tests.js:29:58:29:82 | // test ... jection |
+| PASSED | CommandInjection | tests.js:31:51:31:75 | // test ... jection |
+| PASSED | CommandInjection | tests.js:32:68:32:92 | // test ... jection |
+| PASSED | CommandInjection | tests.js:34:49:34:73 | // test ... jection |
+| PASSED | CommandInjection | tests.js:35:66:35:90 | // test ... jection |
+failingPositiveTests

javascript/ql/test/experimental/Execa/CommandInjection/tests.js

@@ -0,0 +1,36 @@
+import { execa, execaSync, execaCommand, execaCommandSync, $ } from 'execa';
+import http from 'node:http'
+import url from 'url'
+
+http.createServer(async function (req, res) {
+    let cmd = url.parse(req.url, true).query["cmd"][0];
+    let arg1 = url.parse(req.url, true).query["arg1"];
+    let arg2 = url.parse(req.url, true).query["arg2"];
+    let arg3 = url.parse(req.url, true).query["arg3"];
+
+    await $`${cmd} ${arg1} ${arg2} ${arg3}`; // test: CommandInjection
+    await $`ssh ${arg1} ${arg2} ${arg3}`; // test: CommandInjection
+    $({ shell: false }).sync`${cmd} ${arg1} ${arg2} ${arg3}`; // test: CommandInjection
+    $({ shell: true }).sync`${cmd} ${arg1} ${arg2} ${arg3}`; // test: CommandInjection
+    $({ shell: false }).sync`ssh ${arg1} ${arg2} ${arg3}`; // test: CommandInjection
+
+    $.sync`${cmd} ${arg1} ${arg2} ${arg3}`; // test: CommandInjection
+    $.sync`ssh ${arg1} ${arg2} ${arg3}`; // test: CommandInjection
+    await $({ shell: true })`${cmd} ${arg1} ${arg2} ${arg3}` // test: CommandInjection
+    await $({ shell: false })`${cmd} ${arg1} ${arg2} ${arg3}` // test: CommandInjection
+    await $({ shell: false })`ssh ${arg1} ${arg2} ${arg3}` // test: CommandInjection
+
+    await execa(cmd, [arg1, arg2, arg3]); // test: CommandInjection
+    await execa(cmd, { shell: true }); // test: CommandInjection
+    await execa(cmd, { shell: true }); // test: CommandInjection
+    await execa(cmd, [arg1, arg2, arg3], { shell: true }); // test: CommandInjection
+
+    execaSync(cmd, [arg1, arg2, arg3]); // test: CommandInjection
+    execaSync(cmd, [arg1, arg2, arg3], { shell: true }); // test: CommandInjection
+
+    await execaCommand(cmd + arg1 + arg2 + arg3); // test: CommandInjection
+    await execaCommand(cmd + arg1 + arg2 + arg3, { shell: true }); // test: CommandInjection
+
+    execaCommandSync(cmd + arg1 + arg2 + arg3); // test: CommandInjection
+    execaCommandSync(cmd + arg1 + arg2 + arg3, { shell: true }); // test: CommandInjection
+});

javascript/ql/test/experimental/Execa/CommandInjection/tests.ql

@@ -0,0 +1,38 @@
+import javascript
+
+class InlineTest extends LineComment {
+  string tests;
+
+  InlineTest() { tests = this.getText().regexpCapture("\\s*test:(.*)", 1) }
+
+  string getPositiveTest() {
+    result = tests.trim().splitAt(",").trim() and not result.matches("!%")
+  }
+
+  predicate hasPositiveTest(string test) { test = this.getPositiveTest() }
+
+  predicate inNode(DataFlow::Node n) {
+    this.getLocation().getFile() = n.getFile() and
+    this.getLocation().getStartLine() = n.getStartLine()
+  }
+}
+
+import experimental.semmle.javascript.Execa
+
+query predicate passingPositiveTests(string res, string expectation, InlineTest t) {
+  res = "PASSED" and
+  t.hasPositiveTest(expectation) and
+  expectation = "CommandInjection" and
+  exists(SystemCommandExecution n |
+    t.inNode(n.getArgumentList()) or t.inNode(n.getACommandArgument())
+  )
+}
+
+query predicate failingPositiveTests(string res, string expectation, InlineTest t) {
+  res = "FAILED" and
+  t.hasPositiveTest(expectation) and
+  expectation = "CommandInjection" and
+  not exists(SystemCommandExecution n |
+    t.inNode(n.getArgumentList()) or t.inNode(n.getACommandArgument())
+  )
+}

javascript/ql/test/experimental/Execa/PathInjection/tests.expected

@@ -0,0 +1,6 @@
+passingPositiveTests
+| PASSED | PathInjection | tests.js:9:43:9:64 | // test ... jection |
+| PASSED | PathInjection | tests.js:12:50:12:71 | // test ... jection |
+| PASSED | PathInjection | tests.js:15:61:15:82 | // test ... jection |
+| PASSED | PathInjection | tests.js:18:73:18:94 | // test ... jection |
+failingPositiveTests

javascript/ql/test/experimental/Execa/PathInjection/tests.js

@@ -0,0 +1,19 @@
+import { execa, $ } from 'execa';
+import http from 'node:http'
+import url from 'url'
+
+http.createServer(async function (req, res) {
+    let filePath = url.parse(req.url, true).query["filePath"][0];
+
+    // Piping to stdin from a file
+    await $({ inputFile: filePath })`cat` // test: PathInjection
+
+    // Piping to stdin from a file
+    await execa('cat', { inputFile: filePath }); // test: PathInjection
+
+    // Piping Stdout to file
+    await execa('echo', ['example3']).pipeStdout(filePath); // test: PathInjection
+
+    // Piping all of command output to file
+    await execa('echo', ['example4'], { all: true }).pipeAll(filePath); // test: PathInjection
+});

javascript/ql/test/experimental/Execa/PathInjection/tests.ql

@@ -0,0 +1,34 @@
+import javascript
+
+class InlineTest extends LineComment {
+  string tests;
+
+  InlineTest() { tests = this.getText().regexpCapture("\\s*test:(.*)", 1) }
+
+  string getPositiveTest() {
+    result = tests.trim().splitAt(",").trim() and not result.matches("!%")
+  }
+
+  predicate hasPositiveTest(string test) { test = this.getPositiveTest() }
+
+  predicate inNode(DataFlow::Node n) {
+    this.getLocation().getFile() = n.getFile() and
+    this.getLocation().getStartLine() = n.getStartLine()
+  }
+}
+
+import experimental.semmle.javascript.Execa
+
+query predicate passingPositiveTests(string res, string expectation, InlineTest t) {
+  res = "PASSED" and
+  t.hasPositiveTest(expectation) and
+  expectation = "PathInjection" and
+  exists(FileSystemReadAccess n | t.inNode(n.getAPathArgument()))
+}
+
+query predicate failingPositiveTests(string res, string expectation, InlineTest t) {
+  res = "FAILED" and
+  t.hasPositiveTest(expectation) and
+  expectation = "PathInjection" and
+  not exists(FileSystemReadAccess n | t.inNode(n.getAPathArgument()))
+}

shared/dataflow/codeql/dataflow/internal/FlowSummaryImpl.qll

@@ -253,6 +253,13 @@ module Make<
        * that has provenance `provenance`.
        */
       predicate hasProvenance(Provenance provenance) { provenance = "manual" }
+
+      /**
+       * Holds if there exists a model for which this callable is an exact
+       * match, that is, no overriding was used to identify this callable from
+       * the model.
+       */
+      predicate hasExactModel() { none() }
     }
 
     final private class NeutralCallableFinal = NeutralCallable;
@@ -292,6 +299,13 @@ module Make<
        * Gets the kind of the neutral.
        */
       abstract string getKind();
+
+      /**
+       * Holds if there exists a model for which this callable is an exact
+       * match, that is, no overriding was used to identify this callable from
+       * the model.
+       */
+      predicate hasExactModel() { none() }
     }
   }
 

swift/README.md

@@ -16,7 +16,14 @@ brew install bazelisk
 then from the `ql` directory run
 
 ```bash
-bazel run //swift:create-extractor-pack    # --cpu=darwin_x86_64 # Uncomment on Arm-based Macs
+bazel run //swift:create-extractor-pack
+```
+
+If you are running on macOS and you encounter errors mentioning `XXX is unavailable: introduced in macOS YY.ZZ`,
+you will need to run this from the root of your `codeql` checkout:
+
+```bash
+echo common --macos_sdk_version=$(sw_vers --productVersion) >> local.bazelrc
 ```
 
 which will install `swift/extractor-pack`.