Merge branch 'main' into redsun82/go

.github/workflows/buildifier.yml

@@ -0,0 +1,28 @@
+name: Check bazel formatting
+
+on:
+  pull_request:
+    paths:
+      - "**.bazel"
+      - "**.bzl"
+    branches:
+      - main
+      - "rc/*"
+
+permissions:
+  contents: read
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Check bazel formatting
+        uses: pre-commit/action@646c83fcd040023954eafda54b4db0192ce70507
+        with:
+          extra_args: >
+            buildifier --all-files 2>&1 ||
+            (
+              echo -e "In order to format all bazel files, please run:\n  bazel run //:buildifier"; exit 1
+            )

.pre-commit-config.yaml

@@ -20,13 +20,15 @@ repos:
       - id: autopep8
         files: ^misc/codegen/.*\.py
 
-  - repo: https://github.com/warchant/pre-commit-buildifier
-    rev: 0.0.2
-    hooks:
-      - id: buildifier
-
   - repo: local
     hooks:
+      - id: buildifier
+        name: Format bazel files
+        files: \.(bazel|bzl)
+        language: system
+        entry: bazel run //:buildifier
+        pass_filenames: false
+
       - id: go-gen
         name: Check checked in generated files in go
         files: go/.*

BUILD.bazel

@@ -0,0 +1,9 @@
+load("@buildifier_prebuilt//:rules.bzl", "buildifier")
+
+buildifier(
+    name = "buildifier",
+    exclude_patterns = [
+        "./.git/*",
+    ],
+    lint_mode = "fix",
+)

MODULE.bazel

@@ -25,6 +25,8 @@ bazel_dep(name = "nlohmann_json", version = "3.11.3", repo_name = "json")
 bazel_dep(name = "fmt", version = "10.0.0")
 bazel_dep(name = "gazelle", version = "0.36.0")
 
+bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True)
+
 pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
 pip.parse(
     hub_name = "codegen_deps",

config/identical-files.json

@@ -362,7 +362,7 @@
     "java/ql/lib/semmle/code/java/security/internal/EncryptionKeySizes.qll"
   ],
   "Python model summaries test extension": [
-    "python/ql/test/experimental/dataflow/model-summaries/InlineTaintTest.ext.yml",
-    "python/ql/test/experimental/dataflow/model-summaries/NormalDataflowTest.ext.yml"
+    "python/ql/test/library-tests/dataflow/model-summaries/InlineTaintTest.ext.yml",
+    "python/ql/test/library-tests/dataflow/model-summaries/NormalDataflowTest.ext.yml"
   ]
 }

cpp/ql/lib/semmle/code/cpp/PrintAST.qll

@@ -463,6 +463,25 @@ class StmtNode extends AstNode {
   }
 }
 
+/**
+ * A node representing a child of a `Stmt` that is itself a `Stmt`.
+ */
+class ChildStmtNode extends StmtNode {
+  Stmt childStmt;
+
+  ChildStmtNode() { exists(Stmt parent | parent.getAChild() = childStmt and childStmt = ast) }
+
+  override BaseAstNode getChildInternal(int childIndex) {
+    result = super.getChildInternal(childIndex)
+    or
+    exists(int destructorIndex |
+      result.getAst() = childStmt.getImplicitDestructorCall(destructorIndex) and
+      childIndex =
+        destructorIndex + max(int index | exists(childStmt.getChild(index)) or index = 0) + 1
+    )
+  }
+}
+
 /**
  * A node representing a `DeclStmt`.
  */
@@ -674,6 +693,13 @@ class FunctionNode extends FunctionOrGlobalOrNamespaceVariableNode {
 private string getChildAccessorWithoutConversions(Locatable parent, Element child) {
   shouldPrintDeclaration(getAnEnclosingDeclaration(parent)) and
   (
+    exists(Stmt s, int i | s.getChild(i) = parent |
+      exists(int n |
+        s.getChild(i).(Stmt).getImplicitDestructorCall(n) = child and
+        result = "getImplicitDestructorCall(" + n + ")"
+      )
+    )
+    or
     exists(Stmt s | s = parent |
       namedStmtChildPredicates(s, child, result)
       or

cpp/ql/src/experimental/Security/CWE/CWE-125/DangerousWorksWithMultibyteOrWideCharacters.ql

@@ -24,7 +24,7 @@ predicate exprMayBeString(Expr exp) {
         fctmp.getAnArgument().(VariableAccess).getTarget() = exp.(VariableAccess).getTarget() or
         globalValueNumber(fctmp.getAnArgument()) = globalValueNumber(exp)
       ) and
-      fctmp.getTarget().hasName(["strlen", "strcat", "strncat", "strcpy", "sptintf", "printf"])
+      fctmp.getTarget().hasName(["strlen", "strcat", "strncat", "strcpy", "sprintf", "printf"])
     )
     or
     exists(AssignExpr astmp |

cpp/ql/test/library-tests/ir/ir/PrintAST.expected

@@ -22334,6 +22334,114 @@ ir.cpp:
 # 2480|                   Type = [Struct] B
 # 2480|                   ValueCategory = xvalue
 # 2481|     getStmt(1): [ReturnStmt] return ...
+# 2484| [TopLevelFunction] void destructor_without_block(bool)
+# 2484|   <params>: 
+# 2484|     getParameter(0): [Parameter] b
+# 2484|         Type = [BoolType] bool
+# 2485|   getEntryPoint(): [BlockStmt] { ... }
+# 2486|     getStmt(0): [IfStmt] if (...) ... 
+# 2486|       getCondition(): [VariableAccess] b
+# 2486|           Type = [BoolType] bool
+# 2486|           ValueCategory = prvalue(load)
+# 2487|       getThen(): [DeclStmt] declaration
+# 2487|         getDeclarationEntry(0): [VariableDeclarationEntry] definition of c
+# 2487|             Type = [Class] ClassWithDestructor
+# 2487|           getVariable().getInitializer(): [Initializer] initializer for c
+# 2487|             getExpr(): [ConstructorCall] call to ClassWithDestructor
+# 2487|                 Type = [VoidType] void
+# 2487|                 ValueCategory = prvalue
+#-----|         getImplicitDestructorCall(0): [DestructorCall] call to ~ClassWithDestructor
+#-----|             Type = [VoidType] void
+#-----|             ValueCategory = prvalue
+#-----|           getQualifier(): [VariableAccess] c
+#-----|               Type = [Class] ClassWithDestructor
+#-----|               ValueCategory = lvalue
+# 2489|     getStmt(1): [IfStmt] if (...) ... 
+# 2489|       getCondition(): [VariableAccess] b
+# 2489|           Type = [BoolType] bool
+# 2489|           ValueCategory = prvalue(load)
+# 2490|       getThen(): [DeclStmt] declaration
+# 2490|         getDeclarationEntry(0): [VariableDeclarationEntry] definition of d
+# 2490|             Type = [Class] ClassWithDestructor
+# 2490|           getVariable().getInitializer(): [Initializer] initializer for d
+# 2490|             getExpr(): [ConstructorCall] call to ClassWithDestructor
+# 2490|                 Type = [VoidType] void
+# 2490|                 ValueCategory = prvalue
+#-----|         getImplicitDestructorCall(0): [DestructorCall] call to ~ClassWithDestructor
+#-----|             Type = [VoidType] void
+#-----|             ValueCategory = prvalue
+#-----|           getQualifier(): [VariableAccess] d
+#-----|               Type = [Class] ClassWithDestructor
+#-----|               ValueCategory = lvalue
+# 2492|       getElse(): [DeclStmt] declaration
+# 2492|         getDeclarationEntry(0): [VariableDeclarationEntry] definition of e
+# 2492|             Type = [Class] ClassWithDestructor
+# 2492|           getVariable().getInitializer(): [Initializer] initializer for e
+# 2492|             getExpr(): [ConstructorCall] call to ClassWithDestructor
+# 2492|                 Type = [VoidType] void
+# 2492|                 ValueCategory = prvalue
+#-----|         getImplicitDestructorCall(0): [DestructorCall] call to ~ClassWithDestructor
+#-----|             Type = [VoidType] void
+#-----|             ValueCategory = prvalue
+#-----|           getQualifier(): [VariableAccess] e
+#-----|               Type = [Class] ClassWithDestructor
+#-----|               ValueCategory = lvalue
+# 2494|     getStmt(2): [WhileStmt] while (...) ...
+# 2494|       getCondition(): [VariableAccess] b
+# 2494|           Type = [BoolType] bool
+# 2494|           ValueCategory = prvalue(load)
+# 2495|       getStmt(): [DeclStmt] declaration
+# 2495|         getDeclarationEntry(0): [VariableDeclarationEntry] definition of f
+# 2495|             Type = [Class] ClassWithDestructor
+# 2495|           getVariable().getInitializer(): [Initializer] initializer for f
+# 2495|             getExpr(): [ConstructorCall] call to ClassWithDestructor
+# 2495|                 Type = [VoidType] void
+# 2495|                 ValueCategory = prvalue
+#-----|         getImplicitDestructorCall(0): [DestructorCall] call to ~ClassWithDestructor
+#-----|             Type = [VoidType] void
+#-----|             ValueCategory = prvalue
+#-----|           getQualifier(): [VariableAccess] f
+#-----|               Type = [Class] ClassWithDestructor
+#-----|               ValueCategory = lvalue
+# 2497|     getStmt(3): [ForStmt] for(...;...;...) ...
+# 2497|       getInitialization(): [DeclStmt] declaration
+# 2497|         getDeclarationEntry(0): [VariableDeclarationEntry] definition of i
+# 2497|             Type = [IntType] int
+# 2497|           getVariable().getInitializer(): [Initializer] initializer for i
+# 2497|             getExpr(): [Literal] 0
+# 2497|                 Type = [IntType] int
+# 2497|                 Value = [Literal] 0
+# 2497|                 ValueCategory = prvalue
+# 2497|       getCondition(): [LTExpr] ... < ...
+# 2497|           Type = [BoolType] bool
+# 2497|           ValueCategory = prvalue
+# 2497|         getLesserOperand(): [VariableAccess] i
+# 2497|             Type = [IntType] int
+# 2497|             ValueCategory = prvalue(load)
+# 2497|         getGreaterOperand(): [Literal] 42
+# 2497|             Type = [IntType] int
+# 2497|             Value = [Literal] 42
+# 2497|             ValueCategory = prvalue
+# 2497|       getUpdate(): [PrefixIncrExpr] ++ ...
+# 2497|           Type = [IntType] int
+# 2497|           ValueCategory = lvalue
+# 2497|         getOperand(): [VariableAccess] i
+# 2497|             Type = [IntType] int
+# 2497|             ValueCategory = lvalue
+# 2498|       getStmt(): [DeclStmt] declaration
+# 2498|         getDeclarationEntry(0): [VariableDeclarationEntry] definition of g
+# 2498|             Type = [Class] ClassWithDestructor
+# 2498|           getVariable().getInitializer(): [Initializer] initializer for g
+# 2498|             getExpr(): [ConstructorCall] call to ClassWithDestructor
+# 2498|                 Type = [VoidType] void
+# 2498|                 ValueCategory = prvalue
+#-----|         getImplicitDestructorCall(0): [DestructorCall] call to ~ClassWithDestructor
+#-----|             Type = [VoidType] void
+#-----|             ValueCategory = prvalue
+#-----|           getQualifier(): [VariableAccess] g
+#-----|               Type = [Class] ClassWithDestructor
+#-----|               ValueCategory = lvalue
+# 2499|     getStmt(4): [ReturnStmt] return ...
 perf-regression.cpp:
 #    4| [CopyAssignmentOperator] Big& Big::operator=(Big const&)
 #    4|   <params>: 

cpp/ql/test/library-tests/ir/ir/aliased_ir.expected

@@ -17805,6 +17805,159 @@ ir.cpp:
 # 2478|     v2478_6(void)                 = AliasedUse                      : ~m2480_20
 # 2478|     v2478_7(void)                 = ExitFunction                    : 
 
+# 2484| void destructor_without_block(bool)
+# 2484|   Block 0
+# 2484|     v2484_1(void)        = EnterFunction          : 
+# 2484|     m2484_2(unknown)     = AliasedDefinition      : 
+# 2484|     m2484_3(unknown)     = InitializeNonLocal     : 
+# 2484|     m2484_4(unknown)     = Chi                    : total:m2484_2, partial:m2484_3
+# 2484|     r2484_5(glval<bool>) = VariableAddress[b]     : 
+# 2484|     m2484_6(bool)        = InitializeParameter[b] : &:r2484_5
+# 2486|     r2486_1(glval<bool>) = VariableAddress[b]     : 
+# 2486|     r2486_2(bool)        = Load[b]                : &:r2486_1, m2484_6
+# 2486|     v2486_3(void)        = ConditionalBranch      : r2486_2
+#-----|   False -> Block 2
+#-----|   True -> Block 1
+
+# 2487|   Block 1
+# 2487|     r2487_1(glval<ClassWithDestructor>) = VariableAddress[c]                    : 
+# 2487|     m2487_2(ClassWithDestructor)        = Uninitialized[c]                      : &:r2487_1
+# 2487|     r2487_3(glval<unknown>)             = FunctionAddress[ClassWithDestructor]  : 
+# 2487|     v2487_4(void)                       = Call[ClassWithDestructor]             : func:r2487_3, this:r2487_1
+# 2487|     m2487_5(unknown)                    = ^CallSideEffect                       : ~m2484_4
+# 2487|     m2487_6(unknown)                    = Chi                                   : total:m2484_4, partial:m2487_5
+# 2487|     m2487_7(ClassWithDestructor)        = ^IndirectMayWriteSideEffect[-1]       : &:r2487_1
+# 2487|     m2487_8(ClassWithDestructor)        = Chi                                   : total:m2487_2, partial:m2487_7
+#-----|     r0_1(glval<ClassWithDestructor>)    = VariableAddress[c]                    : 
+#-----|     r0_2(glval<unknown>)                = FunctionAddress[~ClassWithDestructor] : 
+#-----|     v0_3(void)                          = Call[~ClassWithDestructor]            : func:r0_2, this:r0_1
+#-----|     m0_4(unknown)                       = ^CallSideEffect                       : ~m2487_6
+#-----|     m0_5(unknown)                       = Chi                                   : total:m2487_6, partial:m0_4
+#-----|     v0_6(void)                          = ^IndirectReadSideEffect[-1]           : &:r0_1, m2487_8
+#-----|     m0_7(ClassWithDestructor)           = ^IndirectMayWriteSideEffect[-1]       : &:r0_1
+#-----|     m0_8(ClassWithDestructor)           = Chi                                   : total:m2487_8, partial:m0_7
+#-----|   Goto -> Block 2
+
+# 2489|   Block 2
+# 2489|     m2489_1(unknown)     = Phi                : from 0:~m2484_4, from 1:~m0_5
+# 2489|     r2489_2(glval<bool>) = VariableAddress[b] : 
+# 2489|     r2489_3(bool)        = Load[b]            : &:r2489_2, m2484_6
+# 2489|     v2489_4(void)        = ConditionalBranch  : r2489_3
+#-----|   False -> Block 4
+#-----|   True -> Block 3
+
+# 2490|   Block 3
+# 2490|     r2490_1(glval<ClassWithDestructor>) = VariableAddress[d]                    : 
+# 2490|     m2490_2(ClassWithDestructor)        = Uninitialized[d]                      : &:r2490_1
+# 2490|     r2490_3(glval<unknown>)             = FunctionAddress[ClassWithDestructor]  : 
+# 2490|     v2490_4(void)                       = Call[ClassWithDestructor]             : func:r2490_3, this:r2490_1
+# 2490|     m2490_5(unknown)                    = ^CallSideEffect                       : ~m2489_1
+# 2490|     m2490_6(unknown)                    = Chi                                   : total:m2489_1, partial:m2490_5
+# 2490|     m2490_7(ClassWithDestructor)        = ^IndirectMayWriteSideEffect[-1]       : &:r2490_1
+# 2490|     m2490_8(ClassWithDestructor)        = Chi                                   : total:m2490_2, partial:m2490_7
+#-----|     r0_9(glval<ClassWithDestructor>)    = VariableAddress[d]                    : 
+#-----|     r0_10(glval<unknown>)               = FunctionAddress[~ClassWithDestructor] : 
+#-----|     v0_11(void)                         = Call[~ClassWithDestructor]            : func:r0_10, this:r0_9
+#-----|     m0_12(unknown)                      = ^CallSideEffect                       : ~m2490_6
+#-----|     m0_13(unknown)                      = Chi                                   : total:m2490_6, partial:m0_12
+#-----|     v0_14(void)                         = ^IndirectReadSideEffect[-1]           : &:r0_9, m2490_8
+#-----|     m0_15(ClassWithDestructor)          = ^IndirectMayWriteSideEffect[-1]       : &:r0_9
+#-----|     m0_16(ClassWithDestructor)          = Chi                                   : total:m2490_8, partial:m0_15
+#-----|   Goto -> Block 5
+
+# 2492|   Block 4
+# 2492|     r2492_1(glval<ClassWithDestructor>) = VariableAddress[e]                    : 
+# 2492|     m2492_2(ClassWithDestructor)        = Uninitialized[e]                      : &:r2492_1
+# 2492|     r2492_3(glval<unknown>)             = FunctionAddress[ClassWithDestructor]  : 
+# 2492|     v2492_4(void)                       = Call[ClassWithDestructor]             : func:r2492_3, this:r2492_1
+# 2492|     m2492_5(unknown)                    = ^CallSideEffect                       : ~m2489_1
+# 2492|     m2492_6(unknown)                    = Chi                                   : total:m2489_1, partial:m2492_5
+# 2492|     m2492_7(ClassWithDestructor)        = ^IndirectMayWriteSideEffect[-1]       : &:r2492_1
+# 2492|     m2492_8(ClassWithDestructor)        = Chi                                   : total:m2492_2, partial:m2492_7
+#-----|     r0_17(glval<ClassWithDestructor>)   = VariableAddress[e]                    : 
+#-----|     r0_18(glval<unknown>)               = FunctionAddress[~ClassWithDestructor] : 
+#-----|     v0_19(void)                         = Call[~ClassWithDestructor]            : func:r0_18, this:r0_17
+#-----|     m0_20(unknown)                      = ^CallSideEffect                       : ~m2492_6
+#-----|     m0_21(unknown)                      = Chi                                   : total:m2492_6, partial:m0_20
+#-----|     v0_22(void)                         = ^IndirectReadSideEffect[-1]           : &:r0_17, m2492_8
+#-----|     m0_23(ClassWithDestructor)          = ^IndirectMayWriteSideEffect[-1]       : &:r0_17
+#-----|     m0_24(ClassWithDestructor)          = Chi                                   : total:m2492_8, partial:m0_23
+#-----|   Goto -> Block 5
+
+# 2494|   Block 5
+# 2494|     m2494_1(unknown)     = Phi                : from 3:~m0_13, from 4:~m0_21, from 6:~m0_29
+# 2494|     r2494_2(glval<bool>) = VariableAddress[b] : 
+# 2494|     r2494_3(bool)        = Load[b]            : &:r2494_2, m2484_6
+# 2494|     v2494_4(void)        = ConditionalBranch  : r2494_3
+#-----|   False -> Block 7
+#-----|   True -> Block 6
+
+# 2495|   Block 6
+# 2495|     r2495_1(glval<ClassWithDestructor>) = VariableAddress[f]                    : 
+# 2495|     m2495_2(ClassWithDestructor)        = Uninitialized[f]                      : &:r2495_1
+# 2495|     r2495_3(glval<unknown>)             = FunctionAddress[ClassWithDestructor]  : 
+# 2495|     v2495_4(void)                       = Call[ClassWithDestructor]             : func:r2495_3, this:r2495_1
+# 2495|     m2495_5(unknown)                    = ^CallSideEffect                       : ~m2494_1
+# 2495|     m2495_6(unknown)                    = Chi                                   : total:m2494_1, partial:m2495_5
+# 2495|     m2495_7(ClassWithDestructor)        = ^IndirectMayWriteSideEffect[-1]       : &:r2495_1
+# 2495|     m2495_8(ClassWithDestructor)        = Chi                                   : total:m2495_2, partial:m2495_7
+#-----|     r0_25(glval<ClassWithDestructor>)   = VariableAddress[f]                    : 
+#-----|     r0_26(glval<unknown>)               = FunctionAddress[~ClassWithDestructor] : 
+#-----|     v0_27(void)                         = Call[~ClassWithDestructor]            : func:r0_26, this:r0_25
+#-----|     m0_28(unknown)                      = ^CallSideEffect                       : ~m2495_6
+#-----|     m0_29(unknown)                      = Chi                                   : total:m2495_6, partial:m0_28
+#-----|     v0_30(void)                         = ^IndirectReadSideEffect[-1]           : &:r0_25, m2495_8
+#-----|     m0_31(ClassWithDestructor)          = ^IndirectMayWriteSideEffect[-1]       : &:r0_25
+#-----|     m0_32(ClassWithDestructor)          = Chi                                   : total:m2495_8, partial:m0_31
+#-----|   Goto (back edge) -> Block 5
+
+# 2497|   Block 7
+# 2497|     r2497_1(glval<int>) = VariableAddress[i] : 
+# 2497|     r2497_2(int)        = Constant[0]        : 
+# 2497|     m2497_3(int)        = Store[i]           : &:r2497_1, r2497_2
+#-----|   Goto -> Block 8
+
+# 2497|   Block 8
+# 2497|     m2497_4(unknown)    = Phi                : from 7:~m2494_1, from 9:~m0_37
+# 2497|     m2497_5(int)        = Phi                : from 7:m2497_3, from 9:m2497_15
+# 2497|     r2497_6(glval<int>) = VariableAddress[i] : 
+# 2497|     r2497_7(int)        = Load[i]            : &:r2497_6, m2497_5
+# 2497|     r2497_8(int)        = Constant[42]       : 
+# 2497|     r2497_9(bool)       = CompareLT          : r2497_7, r2497_8
+# 2497|     v2497_10(void)      = ConditionalBranch  : r2497_9
+#-----|   False -> Block 10
+#-----|   True -> Block 9
+
+# 2498|   Block 9
+# 2498|     r2498_1(glval<ClassWithDestructor>) = VariableAddress[g]                    : 
+# 2498|     m2498_2(ClassWithDestructor)        = Uninitialized[g]                      : &:r2498_1
+# 2498|     r2498_3(glval<unknown>)             = FunctionAddress[ClassWithDestructor]  : 
+# 2498|     v2498_4(void)                       = Call[ClassWithDestructor]             : func:r2498_3, this:r2498_1
+# 2498|     m2498_5(unknown)                    = ^CallSideEffect                       : ~m2497_4
+# 2498|     m2498_6(unknown)                    = Chi                                   : total:m2497_4, partial:m2498_5
+# 2498|     m2498_7(ClassWithDestructor)        = ^IndirectMayWriteSideEffect[-1]       : &:r2498_1
+# 2498|     m2498_8(ClassWithDestructor)        = Chi                                   : total:m2498_2, partial:m2498_7
+#-----|     r0_33(glval<ClassWithDestructor>)   = VariableAddress[g]                    : 
+#-----|     r0_34(glval<unknown>)               = FunctionAddress[~ClassWithDestructor] : 
+#-----|     v0_35(void)                         = Call[~ClassWithDestructor]            : func:r0_34, this:r0_33
+#-----|     m0_36(unknown)                      = ^CallSideEffect                       : ~m2498_6
+#-----|     m0_37(unknown)                      = Chi                                   : total:m2498_6, partial:m0_36
+#-----|     v0_38(void)                         = ^IndirectReadSideEffect[-1]           : &:r0_33, m2498_8
+#-----|     m0_39(ClassWithDestructor)          = ^IndirectMayWriteSideEffect[-1]       : &:r0_33
+#-----|     m0_40(ClassWithDestructor)          = Chi                                   : total:m2498_8, partial:m0_39
+# 2497|     r2497_11(glval<int>)                = VariableAddress[i]                    : 
+# 2497|     r2497_12(int)                       = Load[i]                               : &:r2497_11, m2497_5
+# 2497|     r2497_13(int)                       = Constant[1]                           : 
+# 2497|     r2497_14(int)                       = Add                                   : r2497_12, r2497_13
+# 2497|     m2497_15(int)                       = Store[i]                              : &:r2497_11, r2497_14
+#-----|   Goto (back edge) -> Block 8
+
+# 2499|   Block 10
+# 2499|     v2499_1(void) = NoOp         : 
+# 2484|     v2484_7(void) = ReturnVoid   : 
+# 2484|     v2484_8(void) = AliasedUse   : ~m2497_4
+# 2484|     v2484_9(void) = ExitFunction : 
+
 perf-regression.cpp:
 #    6| void Big::Big()
 #    6|   Block 0

cpp/ql/test/library-tests/ir/ir/ir.cpp

@@ -2432,7 +2432,7 @@ void initialization_with_temp_destructor() {
 }
 
 void param_with_destructor_by_value(ClassWithDestructor c) {
-    // The call to ~ClassWithDestructor::ClassWithDestructor() seems to be missing here.
+    // The call to ~ClassWithDestructor::ClassWithDestructor() happens on the side of the caller
 }
 
 void param_with_destructor_by_pointer(ClassWithDestructor* c) {
@@ -2481,4 +2481,21 @@ namespace rvalue_conversion_with_destructor {
     }
 }
 
+void destructor_without_block(bool b)
+{
+    if (b)
+      ClassWithDestructor c;
+
+    if (b)
+      ClassWithDestructor d;
+    else
+      ClassWithDestructor e;
+
+    while (b)
+      ClassWithDestructor f;
+
+    for(int i = 0; i < 42; ++i)
+      ClassWithDestructor g;
+}
+
 // semmle-extractor-options: -std=c++20 --clang

cpp/ql/test/library-tests/ir/ir/raw_ir.expected

@@ -16244,6 +16244,134 @@ ir.cpp:
 # 2478|     v2478_5(void)                 = AliasedUse                      : ~m?
 # 2478|     v2478_6(void)                 = ExitFunction                    : 
 
+# 2484| void destructor_without_block(bool)
+# 2484|   Block 0
+# 2484|     v2484_1(void)        = EnterFunction          : 
+# 2484|     mu2484_2(unknown)    = AliasedDefinition      : 
+# 2484|     mu2484_3(unknown)    = InitializeNonLocal     : 
+# 2484|     r2484_4(glval<bool>) = VariableAddress[b]     : 
+# 2484|     mu2484_5(bool)       = InitializeParameter[b] : &:r2484_4
+# 2486|     r2486_1(glval<bool>) = VariableAddress[b]     : 
+# 2486|     r2486_2(bool)        = Load[b]                : &:r2486_1, ~m?
+# 2486|     v2486_3(void)        = ConditionalBranch      : r2486_2
+#-----|   False -> Block 2
+#-----|   True -> Block 1
+
+# 2487|   Block 1
+# 2487|     r2487_1(glval<ClassWithDestructor>) = VariableAddress[c]                    : 
+# 2487|     mu2487_2(ClassWithDestructor)       = Uninitialized[c]                      : &:r2487_1
+# 2487|     r2487_3(glval<unknown>)             = FunctionAddress[ClassWithDestructor]  : 
+# 2487|     v2487_4(void)                       = Call[ClassWithDestructor]             : func:r2487_3, this:r2487_1
+# 2487|     mu2487_5(unknown)                   = ^CallSideEffect                       : ~m?
+# 2487|     mu2487_6(ClassWithDestructor)       = ^IndirectMayWriteSideEffect[-1]       : &:r2487_1
+#-----|     r0_1(glval<ClassWithDestructor>)    = VariableAddress[c]                    : 
+#-----|     r0_2(glval<unknown>)                = FunctionAddress[~ClassWithDestructor] : 
+#-----|     v0_3(void)                          = Call[~ClassWithDestructor]            : func:r0_2, this:r0_1
+#-----|     mu0_4(unknown)                      = ^CallSideEffect                       : ~m?
+#-----|     v0_5(void)                          = ^IndirectReadSideEffect[-1]           : &:r0_1, ~m?
+#-----|     mu0_6(ClassWithDestructor)          = ^IndirectMayWriteSideEffect[-1]       : &:r0_1
+#-----|   Goto -> Block 2
+
+# 2489|   Block 2
+# 2489|     r2489_1(glval<bool>) = VariableAddress[b] : 
+# 2489|     r2489_2(bool)        = Load[b]            : &:r2489_1, ~m?
+# 2489|     v2489_3(void)        = ConditionalBranch  : r2489_2
+#-----|   False -> Block 4
+#-----|   True -> Block 3
+
+# 2490|   Block 3
+# 2490|     r2490_1(glval<ClassWithDestructor>) = VariableAddress[d]                    : 
+# 2490|     mu2490_2(ClassWithDestructor)       = Uninitialized[d]                      : &:r2490_1
+# 2490|     r2490_3(glval<unknown>)             = FunctionAddress[ClassWithDestructor]  : 
+# 2490|     v2490_4(void)                       = Call[ClassWithDestructor]             : func:r2490_3, this:r2490_1
+# 2490|     mu2490_5(unknown)                   = ^CallSideEffect                       : ~m?
+# 2490|     mu2490_6(ClassWithDestructor)       = ^IndirectMayWriteSideEffect[-1]       : &:r2490_1
+#-----|     r0_7(glval<ClassWithDestructor>)    = VariableAddress[d]                    : 
+#-----|     r0_8(glval<unknown>)                = FunctionAddress[~ClassWithDestructor] : 
+#-----|     v0_9(void)                          = Call[~ClassWithDestructor]            : func:r0_8, this:r0_7
+#-----|     mu0_10(unknown)                     = ^CallSideEffect                       : ~m?
+#-----|     v0_11(void)                         = ^IndirectReadSideEffect[-1]           : &:r0_7, ~m?
+#-----|     mu0_12(ClassWithDestructor)         = ^IndirectMayWriteSideEffect[-1]       : &:r0_7
+#-----|   Goto -> Block 5
+
+# 2492|   Block 4
+# 2492|     r2492_1(glval<ClassWithDestructor>) = VariableAddress[e]                    : 
+# 2492|     mu2492_2(ClassWithDestructor)       = Uninitialized[e]                      : &:r2492_1
+# 2492|     r2492_3(glval<unknown>)             = FunctionAddress[ClassWithDestructor]  : 
+# 2492|     v2492_4(void)                       = Call[ClassWithDestructor]             : func:r2492_3, this:r2492_1
+# 2492|     mu2492_5(unknown)                   = ^CallSideEffect                       : ~m?
+# 2492|     mu2492_6(ClassWithDestructor)       = ^IndirectMayWriteSideEffect[-1]       : &:r2492_1
+#-----|     r0_13(glval<ClassWithDestructor>)   = VariableAddress[e]                    : 
+#-----|     r0_14(glval<unknown>)               = FunctionAddress[~ClassWithDestructor] : 
+#-----|     v0_15(void)                         = Call[~ClassWithDestructor]            : func:r0_14, this:r0_13
+#-----|     mu0_16(unknown)                     = ^CallSideEffect                       : ~m?
+#-----|     v0_17(void)                         = ^IndirectReadSideEffect[-1]           : &:r0_13, ~m?
+#-----|     mu0_18(ClassWithDestructor)         = ^IndirectMayWriteSideEffect[-1]       : &:r0_13
+#-----|   Goto -> Block 5
+
+# 2494|   Block 5
+# 2494|     r2494_1(glval<bool>) = VariableAddress[b] : 
+# 2494|     r2494_2(bool)        = Load[b]            : &:r2494_1, ~m?
+# 2494|     v2494_3(void)        = ConditionalBranch  : r2494_2
+#-----|   False -> Block 7
+#-----|   True -> Block 6
+
+# 2495|   Block 6
+# 2495|     r2495_1(glval<ClassWithDestructor>) = VariableAddress[f]                    : 
+# 2495|     mu2495_2(ClassWithDestructor)       = Uninitialized[f]                      : &:r2495_1
+# 2495|     r2495_3(glval<unknown>)             = FunctionAddress[ClassWithDestructor]  : 
+# 2495|     v2495_4(void)                       = Call[ClassWithDestructor]             : func:r2495_3, this:r2495_1
+# 2495|     mu2495_5(unknown)                   = ^CallSideEffect                       : ~m?
+# 2495|     mu2495_6(ClassWithDestructor)       = ^IndirectMayWriteSideEffect[-1]       : &:r2495_1
+#-----|     r0_19(glval<ClassWithDestructor>)   = VariableAddress[f]                    : 
+#-----|     r0_20(glval<unknown>)               = FunctionAddress[~ClassWithDestructor] : 
+#-----|     v0_21(void)                         = Call[~ClassWithDestructor]            : func:r0_20, this:r0_19
+#-----|     mu0_22(unknown)                     = ^CallSideEffect                       : ~m?
+#-----|     v0_23(void)                         = ^IndirectReadSideEffect[-1]           : &:r0_19, ~m?
+#-----|     mu0_24(ClassWithDestructor)         = ^IndirectMayWriteSideEffect[-1]       : &:r0_19
+#-----|   Goto (back edge) -> Block 5
+
+# 2497|   Block 7
+# 2497|     r2497_1(glval<int>) = VariableAddress[i] : 
+# 2497|     r2497_2(int)        = Constant[0]        : 
+# 2497|     mu2497_3(int)       = Store[i]           : &:r2497_1, r2497_2
+#-----|   Goto -> Block 8
+
+# 2497|   Block 8
+# 2497|     r2497_4(glval<int>) = VariableAddress[i] : 
+# 2497|     r2497_5(int)        = Load[i]            : &:r2497_4, ~m?
+# 2497|     r2497_6(int)        = Constant[42]       : 
+# 2497|     r2497_7(bool)       = CompareLT          : r2497_5, r2497_6
+# 2497|     v2497_8(void)       = ConditionalBranch  : r2497_7
+#-----|   False -> Block 10
+#-----|   True -> Block 9
+
+# 2498|   Block 9
+# 2498|     r2498_1(glval<ClassWithDestructor>) = VariableAddress[g]                    : 
+# 2498|     mu2498_2(ClassWithDestructor)       = Uninitialized[g]                      : &:r2498_1
+# 2498|     r2498_3(glval<unknown>)             = FunctionAddress[ClassWithDestructor]  : 
+# 2498|     v2498_4(void)                       = Call[ClassWithDestructor]             : func:r2498_3, this:r2498_1
+# 2498|     mu2498_5(unknown)                   = ^CallSideEffect                       : ~m?
+# 2498|     mu2498_6(ClassWithDestructor)       = ^IndirectMayWriteSideEffect[-1]       : &:r2498_1
+#-----|     r0_25(glval<ClassWithDestructor>)   = VariableAddress[g]                    : 
+#-----|     r0_26(glval<unknown>)               = FunctionAddress[~ClassWithDestructor] : 
+#-----|     v0_27(void)                         = Call[~ClassWithDestructor]            : func:r0_26, this:r0_25
+#-----|     mu0_28(unknown)                     = ^CallSideEffect                       : ~m?
+#-----|     v0_29(void)                         = ^IndirectReadSideEffect[-1]           : &:r0_25, ~m?
+#-----|     mu0_30(ClassWithDestructor)         = ^IndirectMayWriteSideEffect[-1]       : &:r0_25
+# 2497|     r2497_9(glval<int>)                 = VariableAddress[i]                    : 
+# 2497|     r2497_10(int)                       = Load[i]                               : &:r2497_9, ~m?
+# 2497|     r2497_11(int)                       = Constant[1]                           : 
+# 2497|     r2497_12(int)                       = Add                                   : r2497_10, r2497_11
+# 2497|     mu2497_13(int)                      = Store[i]                              : &:r2497_9, r2497_12
+#-----|   Goto (back edge) -> Block 8
+
+# 2499|   Block 10
+# 2499|     v2499_1(void) = NoOp         : 
+# 2484|     v2484_6(void) = ReturnVoid   : 
+# 2484|     v2484_7(void) = AliasedUse   : ~m?
+# 2484|     v2484_8(void) = ExitFunction : 
+
 perf-regression.cpp:
 #    6| void Big::Big()
 #    6|   Block 0

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -191,6 +191,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 
         private HashSet<string> AddFrameworkDlls(HashSet<AssemblyLookupLocation> dllLocations)
         {
+            logger.LogInfo("Adding .NET Framework DLLs");
             var frameworkLocations = new HashSet<string>();
 
             var frameworkReferences = Environment.GetEnvironmentVariable(EnvironmentVariableNames.DotnetFrameworkReferences);

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs

@@ -91,9 +91,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             return dotnetCliInvoker.RunCommand(args);
         }
 
-        public IList<string> GetListedRuntimes() => GetResultList("--list-runtimes", null, false);
+        public IList<string> GetListedRuntimes() => GetResultList("--list-runtimes");
 
-        public IList<string> GetListedSdks() => GetResultList("--list-sdks", null, false);
+        public IList<string> GetListedSdks() => GetResultList("--list-sdks");
 
         private IList<string> GetResultList(string args, string? workingDirectory = null, bool silent = true)
         {
@@ -143,7 +143,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             // See https://docs.microsoft.com/en-us/dotnet/core/tools/global-json
             var versions = new List<string>();
 
-            foreach (var path in files.Where(p => p.EndsWith("global.json", StringComparison.Ordinal)))
+            foreach (var path in files.Where(p => string.Equals(FileUtils.SafeGetFileName(p, logger), "global.json", StringComparison.OrdinalIgnoreCase)))
             {
                 try
                 {

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs

@@ -19,6 +19,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         {
             this.logger = logger;
             this.Exec = exec;
+            logger.LogInfo($"Using .NET CLI executable: '{Exec}'");
         }
 
         private ProcessStartInfo MakeDotnetStartInfo(string args, string? workingDirectory)
@@ -43,7 +44,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         private bool RunCommandAux(string args, string? workingDirectory, out IList<string> output, bool silent)
         {
             var dirLog = string.IsNullOrWhiteSpace(workingDirectory) ? "" : $" in {workingDirectory}";
-            logger.LogInfo($"Running {Exec} {args}{dirLog}");
+            logger.LogInfo($"Running '{Exec} {args}'{dirLog}");
             var pi = MakeDotnetStartInfo(args, workingDirectory);
             var threadId = Environment.CurrentManagedThreadId;
             void onOut(string s) => logger.Log(silent ? Severity.Debug : Severity.Info, s, threadId);
@@ -51,7 +52,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             var exitCode = pi.ReadOutput(out output, onOut, onError);
             if (exitCode != 0)
             {
-                logger.LogError($"Command {Exec} {args}{dirLog} failed with exit code {exitCode}");
+                logger.LogError($"Command '{Exec} {args}'{dirLog} failed with exit code {exitCode}");
                 return false;
             }
             return true;

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs

@@ -184,7 +184,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             {
                 try
                 {
-                    var isPackagesConfig = file.EndsWith("packages.config", StringComparison.OrdinalIgnoreCase);
+                    var isPackagesConfig = string.Equals(FileUtils.SafeGetFileName(file, logger), "packages.config", StringComparison.OrdinalIgnoreCase);
 
                     foreach (ReadOnlySpan<char> line in unsafeFileReader.ReadLines(file))
                     {

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileProvider.cs

@@ -21,6 +21,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         private readonly Lazy<string[]> dlls;
         private readonly Lazy<string[]> nugetConfigs;
         private readonly Lazy<string[]> globalJsons;
+        private readonly Lazy<string[]> packagesConfigs;
         private readonly Lazy<string[]> razorViews;
         private readonly Lazy<string[]> resources;
         private readonly Lazy<string?> rootNugetConfig;
@@ -32,31 +33,38 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 
             all = GetAllFiles();
             allNonBinary = new Lazy<FileInfo[]>(() => all.Where(f => !binaryFileExtensions.Contains(f.Extension.ToLowerInvariant())).ToArray());
-            smallNonBinary = new Lazy<string[]>(() =>
-            {
-                var ret = SelectSmallFiles(allNonBinary.Value).SelectFileNames().ToArray();
-                logger.LogInfo($"Found {ret.Length} small non-binary files in {SourceDir}.");
-                return ret;
-            });
+            smallNonBinary = new Lazy<string[]>(() => ReturnAndLogFiles("small non-binary", SelectSmallFiles(allNonBinary.Value).SelectFileNames().ToArray()));
             sources = new Lazy<string[]>(() => SelectTextFileNamesByExtension("source", ".cs"));
             projects = new Lazy<string[]>(() => SelectTextFileNamesByExtension("project", ".csproj"));
             solutions = new Lazy<string[]>(() => SelectTextFileNamesByExtension("solution", ".sln"));
             dlls = new Lazy<string[]>(() => SelectBinaryFileNamesByExtension("DLL", ".dll"));
-            nugetConfigs = new Lazy<string[]>(() => allNonBinary.Value.SelectFileNamesByName("nuget.config").ToArray());
-            globalJsons = new Lazy<string[]>(() => allNonBinary.Value.SelectFileNamesByName("global.json").ToArray());
+            nugetConfigs = new Lazy<string[]>(() => SelectTextFileNamesByName("nuget.config"));
+            globalJsons = new Lazy<string[]>(() => SelectTextFileNamesByName("global.json"));
+            packagesConfigs = new Lazy<string[]>(() => SelectTextFileNamesByName("packages.config"));
             razorViews = new Lazy<string[]>(() => SelectTextFileNamesByExtension("razor view", ".cshtml", ".razor"));
             resources = new Lazy<string[]>(() => SelectTextFileNamesByExtension("resource", ".resx"));
 
             rootNugetConfig = new Lazy<string?>(() => all.SelectRootFiles(SourceDir).SelectFileNamesByName("nuget.config").FirstOrDefault());
         }
 
-        private string[] SelectTextFileNamesByExtension(string filetype, params string[] extensions)
+        private string[] ReturnAndLogFiles(string filetype, IEnumerable<string> files)
         {
-            var ret = allNonBinary.Value.SelectFileNamesByExtension(extensions).ToArray();
+            var ret = files.ToArray();
             logger.LogInfo($"Found {ret.Length} {filetype} files in {SourceDir}.");
             return ret;
         }
 
+        private string[] SelectTextFileNamesByExtension(string filetype, params string[] extensions)
+            => ReturnAndLogFiles(filetype, allNonBinary.Value.SelectFileNamesByExtension(extensions));
+
+        private string[] SelectTextFileNamesByName(string name)
+        {
+            var ret = allNonBinary.Value.SelectFileNamesByName(name).ToArray();
+            var ending = ret.Length == 0 ? "." : $": {string.Join(", ", ret.OrderBy(s => s))}.";
+            logger.LogInfo($"Found {ret.Length} {name} files in {SourceDir}{ending}");
+            return ret;
+        }
+
         private string[] SelectBinaryFileNamesByExtension(string filetype, params string[] extensions)
         {
             var ret = all.SelectFileNamesByExtension(extensions).ToArray();
@@ -117,6 +125,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         public ICollection<string> NugetConfigs => nugetConfigs.Value;
         public string? RootNugetConfig => rootNugetConfig.Value;
         public IEnumerable<string> GlobalJsons => globalJsons.Value;
+        public ICollection<string> PackagesConfigs => packagesConfigs.Value;
         public ICollection<string> RazorViews => razorViews.Value;
         public ICollection<string> Resources => resources.Value;
     }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs

@@ -20,9 +20,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         /// <summary>
         /// The list of package files.
         /// </summary>
-        private readonly FileInfo[] packageFiles;
+        private readonly ICollection<string> packageFiles;
 
-        public int PackageCount => packageFiles.Length;
+        public int PackageCount => packageFiles.Count;
 
         private readonly string? backupNugetConfig;
         private readonly string? nugetConfigPath;
@@ -37,23 +37,21 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         /// <summary>
         /// Create the package manager for a specified source tree.
         /// </summary>
-        public NugetExeWrapper(string sourceDir, TemporaryDirectory packageDirectory, Util.Logging.ILogger logger)
+        public NugetExeWrapper(FileProvider fileProvider, TemporaryDirectory packageDirectory, Util.Logging.ILogger logger)
         {
             this.packageDirectory = packageDirectory;
             this.logger = logger;
 
-            packageFiles = new DirectoryInfo(sourceDir)
-                .EnumerateFiles("packages.config", SearchOption.AllDirectories)
-                .ToArray();
+            packageFiles = fileProvider.PackagesConfigs;
 
-            if (packageFiles.Length > 0)
+            if (packageFiles.Count > 0)
             {
-                logger.LogInfo($"Found {packageFiles.Length} packages.config files, trying to use nuget.exe for package restore");
-                nugetExe = ResolveNugetExe(sourceDir);
+                logger.LogInfo($"Found packages.config files, trying to use nuget.exe for package restore");
+                nugetExe = ResolveNugetExe(fileProvider.SourceDir.FullName);
                 if (HasNoPackageSource())
                 {
                     // We only modify or add a top level nuget.config file
-                    nugetConfigPath = Path.Combine(sourceDir, "nuget.config");
+                    nugetConfigPath = Path.Combine(fileProvider.SourceDir.FullName, "nuget.config");
                     try
                     {
                         if (File.Exists(nugetConfigPath))
@@ -86,10 +84,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                     }
                 }
             }
-            else
-            {
-                logger.LogInfo("Found no packages.config file");
-            }
         }
 
         /// <summary>
@@ -195,7 +189,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         /// </summary>
         public int InstallPackages()
         {
-            return packageFiles.Count(package => TryRestoreNugetPackage(package.FullName));
+            return packageFiles.Count(package => TryRestoreNugetPackage(package));
         }
 
         private bool HasNoPackageSource()

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs

@@ -105,7 +105,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                         : [unresponsiveMissingPackageLocation];
                 }
 
-                using (var nuget = new NugetExeWrapper(fileProvider.SourceDir.FullName, legacyPackageDirectory, logger))
+                using (var nuget = new NugetExeWrapper(fileProvider, legacyPackageDirectory, logger))
                 {
                     var count = nuget.InstallPackages();
 

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/DotnetSourceGeneratorBase.cs

@@ -55,7 +55,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 
             // group additional files by closes project file:
             var projects = fileProvider.Projects
-                .Select(p => (File: p, Directory: SafeGetDirectoryName(p)))
+                .Select(p => (File: p, Directory: FileUtils.SafeGetDirectoryName(p, logger)))
                 .Where(p => p.Directory.Length > 0);
 
             var groupedFiles = new Dictionary<string, List<string>>();
@@ -93,30 +93,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             }
         }
 
-        private string SafeGetDirectoryName(string fileName)
-        {
-            try
-            {
-                var dir = Path.GetDirectoryName(fileName);
-                if (dir is null)
-                {
-                    return "";
-                }
-
-                if (!dir.EndsWith(Path.DirectorySeparatorChar))
-                {
-                    dir += Path.DirectorySeparatorChar;
-                }
-
-                return dir;
-            }
-            catch (Exception ex)
-            {
-                logger.LogDebug($"Failed to get directory name for {fileName}: {ex.Message}");
-                return "";
-            }
-        }
-
         protected abstract ICollection<string> AdditionalFiles { get; }
 
         protected abstract string FileType { get; }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/ResxGenerator.cs

@@ -19,6 +19,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             TemporaryDirectory tempWorkingDirectory,
             IEnumerable<string> references) : base(fileProvider, fileContent, dotnet, compilationInfoContainer, logger, tempWorkingDirectory, references)
         {
+            if (fileProvider.Resources.Count == 0)
+            {
+                logger.LogDebug("No resources found, skipping resource extraction.");
+                sourceGeneratorFolder = null;
+                return;
+            }
+
             try
             {
                 // The package is downloaded to `missingpackages`, which is okay, we're already after the DLL collection phase.

csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs

@@ -121,17 +121,20 @@ namespace Semmle.Extraction.CSharp.Standalone
 
             public void MissingType(string type)
             {
-                logger.Log(Severity.Debug, "Missing type {0}", type);
+                logger.LogDebug($"Missing type {type}");
             }
 
             public void MissingNamespace(string @namespace)
             {
-                logger.Log(Severity.Info, "Missing namespace {0}", @namespace);
+                logger.LogInfo($"Missing namespace {@namespace}");
             }
 
             public void MissingSummary(int missingTypes, int missingNamespaces)
             {
-                logger.Log(Severity.Info, "Failed to resolve {0} types in {1} namespaces", missingTypes, missingNamespaces);
+                if (missingTypes > 0 || missingNamespaces > 0)
+                {
+                    logger.LogInfo($"Failed to resolve {missingTypes} types in {missingNamespaces} namespaces");
+                }
             }
         }
 

csharp/extractor/Semmle.Util/FileUtils.cs

@@ -185,5 +185,42 @@ namespace Semmle.Util
 
             return new FileInfo(outputPath);
         }
+
+        public static string SafeGetDirectoryName(string path, ILogger logger)
+        {
+            try
+            {
+                var dir = Path.GetDirectoryName(path);
+                if (dir is null)
+                {
+                    return "";
+                }
+
+                if (!dir.EndsWith(Path.DirectorySeparatorChar))
+                {
+                    dir += Path.DirectorySeparatorChar;
+                }
+
+                return dir;
+            }
+            catch (Exception ex)
+            {
+                logger.LogDebug($"Failed to get directory name for {path}: {ex.Message}");
+                return "";
+            }
+        }
+
+        public static string? SafeGetFileName(string path, ILogger logger)
+        {
+            try
+            {
+                return Path.GetFileName(path);
+            }
+            catch (Exception ex)
+            {
+                logger.LogDebug($"Failed to get file name for {path}: {ex.Message}");
+                return null;
+            }
+        }
     }
 }

docs/codeql/codeql-overview/codeql-changelog/codeql-cli-2.17.1.rst

@@ -0,0 +1,106 @@
+.. _codeql-cli-2.17.1:
+
+==========================
+CodeQL 2.17.1 (2024-04-24)
+==========================
+
+.. contents:: Contents
+   :depth: 2
+   :local:
+   :backlinks: none
+
+This is an overview of changes in the CodeQL CLI and relevant CodeQL query and library packs. For additional updates on changes to the CodeQL code scanning experience, check out the `code scanning section on the GitHub blog <https://github.blog/tag/code-scanning/>`__, `relevant GitHub Changelog updates <https://github.blog/changelog/label/code-scanning/>`__, `changes in the CodeQL extension for Visual Studio Code <https://marketplace.visualstudio.com/items/GitHub.vscode-codeql/changelog>`__, and the `CodeQL Action changelog <https://github.com/github/codeql-action/blob/main/CHANGELOG.md>`__.
+
+Security Coverage
+-----------------
+
+CodeQL 2.17.1 runs a total of 412 security queries when configured with the Default suite (covering 160 CWE). The Extended suite enables an additional 130 queries (covering 34 more CWE). 2 security queries have been added with this release.
+
+CodeQL CLI
+----------
+
+Deprecations
+~~~~~~~~~~~~
+
+*   The :code:`--mode` option and :code:`-m` alias to :code:`codeql database create`,
+    :code:`codeql database cleanup`, and :code:`codeql dataset cleanup` has been deprecated. Instead, use the new :code:`--cache-cleanup` option, which has identical behavior.
+
+Improvements
+~~~~~~~~~~~~
+
+*   Improved the diagnostic message produced when no code is processed when creating a database. If a build mode was specified using
+    :code:`--build-mode`, the message is now tailored to your build mode.
+
+Miscellaneous
+~~~~~~~~~~~~~
+
+*   The :code:`scc` tool used by the CodeQL CLI to calculate source code baseline information has been updated to version `3.2.0 <https://github.com/boyter/scc/releases/tag/v3.2.0>`__.
+
+Query Packs
+-----------
+
+Minor Analysis Improvements
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Java
+""""
+
+*   The :code:`java/unknown-javadoc-parameter` now accepts :code:`@param` tags that apply to the parameters of a record.
+
+JavaScript/TypeScript
+"""""""""""""""""""""
+
+*   :code:`API::Node#getInstance()` now includes instances of subclasses, include transitive subclasses.
+    The same changes applies to uses of the :code:`Instance` token in data extensions.
+
+New Queries
+~~~~~~~~~~~
+
+Ruby
+""""
+
+*   Added a new query, :code:`rb/insecure-mass-assignment`, for finding instances of mass assignment operations accepting arbitrary parameters from remote user input.
+*   Added a new query, :code:`rb/csrf-protection-not-enabled`, to detect cases where Cross-Site Request Forgery protection is not enabled in Ruby on Rails controllers.
+
+Language Libraries
+------------------
+
+Minor Analysis Improvements
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+C#
+""
+
+*   Extracting suppress nullable warning expressions did not work when applied directly to a method call (like :code:`System.Console.Readline()!`). This has been fixed.
+
+Golang
+""""""
+
+*   Data flow through variables declared in statements of the form :code:`x := y.(type)` at the beginning of type switches has been fixed, which may result in more alerts.
+*   Added strings.ReplaceAll, http.ParseMultipartForm sanitizers and remove path sanitizer.
+
+Java
+""""
+
+*   About 6,700 summary models and 6,800 neutral summary models for the JDK that were generated using data flow have been added. This may lead to new alerts being reported.
+
+Python
+""""""
+
+*   Improved the type-tracking capabilities (and therefore also API graphs) to allow tracking items in tuples and dictionaries.
+
+Shared Libraries
+----------------
+
+New Features
+~~~~~~~~~~~~
+
+Dataflow Analysis
+"""""""""""""""""
+
+*   The :code:`PathGraph` result of a data flow computation has been augmented with model provenance information for each of the flow steps. Any qltests that include the edges relation in their output (for example, :code:`.qlref`\ s that reference path-problem queries) will need to be have their expected output updated accordingly.
+
+Type-flow Analysis
+""""""""""""""""""
+
+*   Initial release. Adds a library to implement type-flow analysis.

docs/codeql/codeql-overview/codeql-changelog/index.rst

@@ -11,6 +11,7 @@ A list of queries for each suite and language `is available here <https://docs.g
 .. toctree::
    :maxdepth: 1
 
+   codeql-cli-2.17.1
    codeql-cli-2.17.0
    codeql-cli-2.16.6
    codeql-cli-2.16.5

go/BUILD.bazel

@@ -1,7 +1,7 @@
-load("@rules_pkg//pkg:mappings.bzl", "pkg_filegroup", "pkg_files")
-load("@rules_pkg//pkg:install.bzl", "pkg_install")
 load("@bazel_skylib//rules:native_binary.bzl", "native_binary")
 load("@gazelle//:def.bzl", "gazelle")
+load("@rules_pkg//pkg:install.bzl", "pkg_install")
+load("@rules_pkg//pkg:mappings.bzl", "pkg_filegroup", "pkg_files")
 load("//:defs.bzl", "codeql_platform")
 
 gazelle(

go/extractor/project/BUILD.bazel

@@ -20,4 +20,5 @@ go_test(
     name = "project_test",
     srcs = ["project_test.go"],
     embed = [":project"],
+    deps = ["//go/extractor/vendor/golang.org/x/mod/modfile"],
 )

go/extractor/project/project.go

@@ -179,6 +179,13 @@ func findGoModFiles(root string) []string {
 // A regular expression for the Go toolchain version syntax.
 var toolchainVersionRe *regexp.Regexp = regexp.MustCompile(`(?m)^([0-9]+\.[0-9]+\.[0-9]+)$`)
 
+// Returns true if the `go.mod` file specifies a Go language version, that version is `1.21` or greater, and
+// there is no `toolchain` directive, and the Go language version is not a valid toolchain version.
+func hasInvalidToolchainVersion(modFile *modfile.File) bool {
+	return modFile.Toolchain == nil && modFile.Go != nil &&
+		!toolchainVersionRe.Match([]byte(modFile.Go.Version)) && semver.Compare("v"+modFile.Go.Version, "v1.21.0") >= 0
+}
+
 // Given a list of `go.mod` file paths, try to parse them all. The resulting array of `GoModule` objects
 // will be the same length as the input array and the objects will contain at least the `go.mod` path.
 // If parsing the corresponding file is successful, then the parsed contents will also be available.
@@ -196,7 +203,7 @@ func LoadGoModules(emitDiagnostics bool, goModFilePaths []string) []*GoModule {
 			continue
 		}
 
-		modFile, err := modfile.ParseLax(goModFilePath, modFileSrc, nil)
+		modFile, err := modfile.Parse(goModFilePath, modFileSrc, nil)
 
 		if err != nil {
 			log.Printf("Unable to parse %s: %s.\n", goModFilePath, err.Error())
@@ -209,8 +216,7 @@ func LoadGoModules(emitDiagnostics bool, goModFilePaths []string) []*GoModule {
 		// there is no `toolchain` directive, check that it is a valid Go toolchain version. Otherwise,
 		// `go` commands which try to download the right version of the Go toolchain will fail. We detect
 		// this situation and emit a diagnostic.
-		if modFile.Toolchain == nil && modFile.Go != nil &&
-			!toolchainVersionRe.Match([]byte(modFile.Go.Version)) && semver.Compare("v"+modFile.Go.Version, "v1.21.0") >= 0 {
+		if hasInvalidToolchainVersion(modFile) {
 			diagnostics.EmitInvalidToolchainVersion(goModFilePath, modFile.Go.Version)
 		}
 	}

go/extractor/project/project_test.go

@@ -3,6 +3,8 @@ package project
 import (
 	"path/filepath"
 	"testing"
+
+	"golang.org/x/mod/modfile"
 )
 
 func testStartsWithAnyOf(t *testing.T, path string, prefix string, expectation bool) {
@@ -25,3 +27,38 @@ func TestStartsWithAnyOf(t *testing.T) {
 	testStartsWithAnyOf(t, filepath.Join("foo", "bar"), "bar", false)
 	testStartsWithAnyOf(t, filepath.Join("foo", "bar"), filepath.Join("foo", "baz"), false)
 }
+
+func testHasInvalidToolchainVersion(t *testing.T, contents string) bool {
+	modFile, err := modfile.Parse("test.go", []byte(contents), nil)
+
+	if err != nil {
+		t.Errorf("Unable to parse %s: %s.\n", contents, err.Error())
+	}
+
+	return hasInvalidToolchainVersion(modFile)
+}
+
+func TestHasInvalidToolchainVersion(t *testing.T) {
+	invalid := []string{
+		"go 1.21\n",
+		"go 1.22\n",
+	}
+
+	for _, v := range invalid {
+		if !testHasInvalidToolchainVersion(t, v) {
+			t.Errorf("Expected testHasInvalidToolchainVersion(\"%s\") to be true, but got false", v)
+		}
+	}
+
+	valid := []string{
+		"go 1.20\n",
+		"go 1.21.1\n",
+		"go 1.22\n\ntoolchain go1.22.0\n",
+	}
+
+	for _, v := range valid {
+		if testHasInvalidToolchainVersion(t, v) {
+			t.Errorf("Expected testHasInvalidToolchainVersion(\"%s\") to be false, but got true", v)
+		}
+	}
+}

go/ql/src/Security/CWE-020/IncompleteHostnameRegexp.ql

@@ -81,14 +81,12 @@ predicate regexpGuardsError(RegexpPattern regexp) {
 
 module IncompleteHostNameRegexpConfig implements DataFlow::ConfigSig {
   additional predicate isSourceString(DataFlow::Node source, string hostPart) {
-    exists(Expr e |
-      e = source.asExpr() and
-      isIncompleteHostNameRegexpPattern(e.getStringValue(), hostPart)
-    |
-      e instanceof StringLit
-      or
-      e instanceof AddExpr and
-      not isIncompleteHostNameRegexpPattern(e.(AddExpr).getAnOperand().getStringValue(), _)
+    exists(Expr e | e = source.asExpr() |
+      isIncompleteHostNameRegexpPattern(e.getStringValue(), hostPart) and
+      // Exclude constant names to avoid duplicate results, because the string
+      // literals which they are initialised with are also considered as
+      // sources.
+      not e instanceof ConstantName
     )
   }
 
@@ -101,6 +99,10 @@ module IncompleteHostNameRegexpConfig implements DataFlow::ConfigSig {
     ) and
     not regexpGuardsError(sink)
   }
+
+  predicate isAdditionalFlowStep(DataFlow::Node node1, DataFlow::Node node2) {
+    StringOps::Concatenation::taintStep(node1, node2)
+  }
 }
 
 module Flow = DataFlow::Global<IncompleteHostNameRegexpConfig>;

go/ql/src/Security/CWE-020/IncompleteHostnameRegexpGood2.go

@@ -6,7 +6,7 @@ import (
 	"regexp"
 )
 
-func checkRedirectGood(req *http.Request, via []*http.Request) error {
+func checkRedirectGood2(req *http.Request, via []*http.Request) error {
 	// GOOD: the host of `req.URL` must be `example.com`, `www.example.com` or `beta.example.com`
 	re := `^((www|beta)\.)?example\.com/`
 	if matched, _ := regexp.MatchString(re, req.URL.Host); matched {

go/ql/src/change-notes/2024-04-24-incomplete-hostname-regexp.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* The query `go/incomplete-hostname-regexp` now recognizes more sources involving concatenation of string literals and also follows flow through string concatenation. This may lead to more alerts.

go/ql/test/query-tests/Security/CWE-020/IncompleteHostnameRegexp/IncompleteHostnameRegexp.expected

@@ -1,12 +1,22 @@
 edges
 | IncompleteHostnameRegexp.go:11:8:11:36 | "^((www\|beta).)?example.com/" | IncompleteHostnameRegexp.go:12:38:12:39 | re | provenance |  |
+| main.go:49:21:49:45 | `https://www.example.com` | main.go:62:15:62:25 | sourceConst | provenance |  |
+| main.go:62:15:62:25 | sourceConst | main.go:65:15:65:23 | localVar3 | provenance |  |
 nodes
 | IncompleteHostnameRegexp.go:11:8:11:36 | "^((www\|beta).)?example.com/" | semmle.label | "^((www\|beta).)?example.com/" |
 | IncompleteHostnameRegexp.go:12:38:12:39 | re | semmle.label | re |
-| main.go:39:60:39:79 | "^test2.github.com$" | semmle.label | "^test2.github.com$" |
-| main.go:44:15:44:39 | `https://www.example.com` | semmle.label | `https://www.example.com` |
+| main.go:40:60:40:79 | "^test2.github.com$" | semmle.label | "^test2.github.com$" |
+| main.go:45:15:45:39 | `https://www.example.com` | semmle.label | `https://www.example.com` |
+| main.go:49:21:49:45 | `https://www.example.com` | semmle.label | `https://www.example.com` |
+| main.go:56:15:56:34 | ...+... | semmle.label | ...+... |
+| main.go:58:15:58:42 | ...+... | semmle.label | ...+... |
+| main.go:62:15:62:25 | sourceConst | semmle.label | sourceConst |
+| main.go:65:15:65:23 | localVar3 | semmle.label | localVar3 |
 subpaths
 #select
 | IncompleteHostnameRegexp.go:11:8:11:36 | "^((www\|beta).)?example.com/" | IncompleteHostnameRegexp.go:11:8:11:36 | "^((www\|beta).)?example.com/" | IncompleteHostnameRegexp.go:12:38:12:39 | re | This regular expression has an unescaped dot before ')?example.com', so it might match more hosts than expected when $@. | IncompleteHostnameRegexp.go:12:38:12:39 | re | the regular expression is used |
-| main.go:39:60:39:79 | "^test2.github.com$" | main.go:39:60:39:79 | "^test2.github.com$" | main.go:39:60:39:79 | "^test2.github.com$" | This regular expression has an unescaped dot before 'github.com', so it might match more hosts than expected when $@. | main.go:39:60:39:79 | "^test2.github.com$" | the regular expression is used |
-| main.go:44:15:44:39 | `https://www.example.com` | main.go:44:15:44:39 | `https://www.example.com` | main.go:44:15:44:39 | `https://www.example.com` | This regular expression has an unescaped dot before 'example.com', so it might match more hosts than expected when $@. | main.go:44:15:44:39 | `https://www.example.com` | the regular expression is used |
+| main.go:40:60:40:79 | "^test2.github.com$" | main.go:40:60:40:79 | "^test2.github.com$" | main.go:40:60:40:79 | "^test2.github.com$" | This regular expression has an unescaped dot before 'github.com', so it might match more hosts than expected when $@. | main.go:40:60:40:79 | "^test2.github.com$" | the regular expression is used |
+| main.go:45:15:45:39 | `https://www.example.com` | main.go:45:15:45:39 | `https://www.example.com` | main.go:45:15:45:39 | `https://www.example.com` | This regular expression has an unescaped dot before 'example.com', so it might match more hosts than expected when $@. | main.go:45:15:45:39 | `https://www.example.com` | the regular expression is used |
+| main.go:49:21:49:45 | `https://www.example.com` | main.go:49:21:49:45 | `https://www.example.com` | main.go:65:15:65:23 | localVar3 | This regular expression has an unescaped dot before 'example.com', so it might match more hosts than expected when $@. | main.go:65:15:65:23 | localVar3 | the regular expression is used |
+| main.go:56:15:56:34 | ...+... | main.go:56:15:56:34 | ...+... | main.go:56:15:56:34 | ...+... | This regular expression has an unescaped dot before 'example.com', so it might match more hosts than expected when $@. | main.go:56:15:56:34 | ...+... | the regular expression is used |
+| main.go:58:15:58:42 | ...+... | main.go:58:15:58:42 | ...+... | main.go:58:15:58:42 | ...+... | This regular expression has an unescaped dot before 'example.com', so it might match more hosts than expected when $@. | main.go:58:15:58:42 | ...+... | the regular expression is used |

go/ql/test/query-tests/Security/CWE-020/IncompleteHostnameRegexp/IncompleteHostnameRegexpGood2.go

@@ -0,0 +1,16 @@
+package main
+
+import (
+	"errors"
+	"net/http"
+	"regexp"
+)
+
+func checkRedirectGood2(req *http.Request, via []*http.Request) error {
+	// GOOD: the host of `req.URL` must be `example.com`, `www.example.com` or `beta.example.com`
+	re := `^((www|beta)\.)?example\.com/`
+	if matched, _ := regexp.MatchString(re, req.URL.Host); matched {
+		return nil
+	}
+	return errors.New("Invalid redirect")
+}

go/ql/test/query-tests/Security/CWE-020/IncompleteHostnameRegexp/main.go

@@ -3,10 +3,11 @@
 package main
 
 import (
-	"github.com/elazarl/goproxy"
 	"net/http"
 	"regexp"
 	"time"
+
+	"github.com/elazarl/goproxy"
 )
 
 func Match(notARegex string) bool {
@@ -44,3 +45,22 @@ func main() {
 	regexp.Match(`https://www.example.com`, []byte(""))   // NOT OK
 	regexp.Match(`https://www\.example\.com`, []byte("")) // OK
 }
+
+const sourceConst = `https://www.example.com`
+const firstHalfConst = `https://www.example.`
+
+func concatenateStrings() {
+	firstHalf := `https://www.example.`
+	regexp.Match(firstHalf+`com`, []byte("")) // MISSING: NOT OK
+
+	regexp.Match(firstHalfConst+`com`, []byte("")) // NOT OK
+
+	regexp.Match(`https://www.example.`+`com`, []byte("")) // NOT OK
+}
+
+func avoidDuplicateResults() {
+	localVar1 := sourceConst
+	localVar2 := localVar1
+	localVar3 := localVar2
+	regexp.Match(localVar3, []byte("")) // NOT OK
+}

javascript/BUILD.bazel

@@ -1,5 +1,5 @@
-load("@semmle_code//:dist.bzl", "dist")
 load("@rules_pkg//pkg:mappings.bzl", "pkg_files")
+load("@semmle_code//:dist.bzl", "dist")
 load("@semmle_code//buildutils-internal:zipmerge.bzl", "zipmerge")
 
 package(default_visibility = ["//visibility:public"])

javascript/extractor/BUILD.bazel

@@ -1,5 +1,5 @@
-load("@semmle_code//:common.bzl", "codeql_fat_jar", "codeql_java_project")
 load("@rules_pkg//pkg:mappings.bzl", "pkg_files")
+load("@semmle_code//:common.bzl", "codeql_fat_jar", "codeql_java_project")
 
 java_library(
     name = "deps",

javascript/extractor/src/com/semmle/js/extractor/AbstractDetector.java

@@ -0,0 +1,113 @@
+package com.semmle.js.extractor;
+
+import com.semmle.js.ast.AssignmentExpression;
+import com.semmle.js.ast.BlockStatement;
+import com.semmle.js.ast.CallExpression;
+import com.semmle.js.ast.Expression;
+import com.semmle.js.ast.ExpressionStatement;
+import com.semmle.js.ast.IFunction;
+import com.semmle.js.ast.Identifier;
+import com.semmle.js.ast.IfStatement;
+import com.semmle.js.ast.MemberExpression;
+import com.semmle.js.ast.Node;
+import com.semmle.js.ast.ParenthesizedExpression;
+import com.semmle.js.ast.Program;
+import com.semmle.js.ast.Statement;
+import com.semmle.js.ast.TryStatement;
+import com.semmle.js.ast.UnaryExpression;
+import com.semmle.js.ast.VariableDeclaration;
+import com.semmle.js.ast.VariableDeclarator;
+import java.util.List;
+
+/**
+ * A utility base class for running detection logic on statements/expressions by
+ * visiting each node. It performs recursive decent into assignment expressions and
+ * callee expressions for call-expressions (to handle detection of `foo` in `foo()()`)
+ * */
+public abstract class AbstractDetector {
+  protected boolean programDetection(Node ast) {
+    if (!(ast instanceof Program)) return false;
+
+    return visitStatements(((Program) ast).getBody());
+  }
+
+  protected boolean visitStatements(List<Statement> stmts) {
+    for (Statement stmt : stmts) if (visitStatement(stmt)) return true;
+    return false;
+  }
+
+  protected boolean visitStatement(Statement stmt) {
+    if (stmt instanceof ExpressionStatement) {
+      Expression e = stripParens(((ExpressionStatement) stmt).getExpression());
+
+      // check whether `e` is an iife; if so, recursively check its body
+
+      // strip off unary operators to handle `!function(){...}()`
+      if (e instanceof UnaryExpression) e = ((UnaryExpression) e).getArgument();
+
+      if (e instanceof CallExpression && ((CallExpression) e).getArguments().isEmpty()) {
+        Expression callee = stripParens(((CallExpression) e).getCallee());
+        if (callee instanceof IFunction) {
+          Node body = ((IFunction) callee).getBody();
+          if (body instanceof BlockStatement)
+            return visitStatements(((BlockStatement) body).getBody());
+        }
+      }
+
+      if (visitExpression(e)) return true;
+
+    } else if (stmt instanceof VariableDeclaration) {
+      for (VariableDeclarator decl : ((VariableDeclaration) stmt).getDeclarations()) {
+        Expression init = stripParens(decl.getInit());
+        if (visitExpression(init)) return true;
+      }
+
+    } else if (stmt instanceof TryStatement) {
+      return visitStatement(((TryStatement) stmt).getBlock());
+
+    } else if (stmt instanceof BlockStatement) {
+      return visitStatements(((BlockStatement) stmt).getBody());
+
+    } else if (stmt instanceof IfStatement) {
+      IfStatement is = (IfStatement) stmt;
+      return visitStatement(is.getConsequent())
+          || visitStatement(is.getAlternate());
+    }
+
+    return false;
+  }
+
+  private static Expression stripParens(Expression e) {
+    if (e instanceof ParenthesizedExpression)
+      return stripParens(((ParenthesizedExpression) e).getExpression());
+    return e;
+  }
+
+  /**
+   * Recursively check {@code e} if it's a call or an assignment.
+   */
+  protected boolean visitExpression(Expression e) {
+    if (e instanceof CallExpression) {
+      CallExpression call = (CallExpression) e;
+      Expression callee = call.getCallee();
+      // recurse, to handle things like `foo()()`
+      if (visitExpression(callee)) return true;
+      return false;
+    } else if (e instanceof MemberExpression) {
+      return visitExpression(((MemberExpression) e).getObject());
+    } else if (e instanceof AssignmentExpression) {
+      AssignmentExpression assgn = (AssignmentExpression) e;
+
+      // filter out compound assignments
+      if (!"=".equals(assgn.getOperator())) return false;
+
+      return visitExpression(assgn.getRight());
+    }
+    return false;
+  }
+
+  /** Is {@code e} an identifier with name {@code name}? */
+  protected static boolean isIdentifier(Expression e, String name) {
+    return e instanceof Identifier && name.equals(((Identifier) e).getName());
+  }
+}

javascript/extractor/src/com/semmle/js/extractor/ES2015Detector.java

@@ -0,0 +1,34 @@
+package com.semmle.js.extractor;
+
+import com.semmle.js.ast.DynamicImport;
+import com.semmle.js.ast.ExportDeclaration;
+import com.semmle.js.ast.Expression;
+import com.semmle.js.ast.ImportDeclaration;
+import com.semmle.js.ast.Node;
+import com.semmle.js.ast.Statement;
+
+/** A utility class for detecting Node.js code. */
+public class ES2015Detector extends AbstractDetector {
+  /**
+   * Is {@code ast} a program that uses ES2015 import/export code?
+   */
+  public static boolean looksLikeES2015(Node ast) {
+    return new ES2015Detector().programDetection(ast);
+  }
+
+  @Override
+  protected boolean visitStatement(Statement stmt) {
+    if (stmt instanceof ImportDeclaration || stmt instanceof ExportDeclaration) {
+      return true;
+    }
+    return super.visitStatement(stmt);
+  }
+
+  @Override
+  protected boolean visitExpression(Expression e) {
+    if (e instanceof DynamicImport) {
+      return true;
+    }
+    return super.visitExpression(e);
+  }
+}

javascript/extractor/src/com/semmle/js/extractor/JSExtractor.java

@@ -58,6 +58,26 @@ public class JSExtractor {
 
     JSParser.Result parserRes =
         JSParser.parse(config, sourceType, source, textualExtractor.getMetrics());
+
+    // Check if we guessed wrong with the regex in `establishSourceType`, (which could
+    // happen due to a block-comment line starting with '  import').
+    if (config.getSourceType() == SourceType.AUTO && sourceType != SourceType.SCRIPT) {
+      boolean wrongGuess = false;
+
+      if (sourceType == SourceType.MODULE) {
+        // check that we did see an import/export declaration
+        wrongGuess = ES2015Detector.looksLikeES2015(parserRes.getAST()) == false;
+      } else if (sourceType == SourceType.CLOSURE_MODULE ) {
+        // TODO
+      }
+
+      if (wrongGuess) {
+        sourceType = SourceType.SCRIPT;
+        parserRes =
+          JSParser.parse(config, sourceType, source, textualExtractor.getMetrics());
+      }
+    }
+
     return extract(textualExtractor, source, toplevelKind, scopeManager, sourceType, parserRes);
   }
 

javascript/extractor/src/com/semmle/js/extractor/NodeJSDetector.java

@@ -1,132 +1,30 @@
 package com.semmle.js.extractor;
 
 import com.semmle.js.ast.AssignmentExpression;
-import com.semmle.js.ast.BlockStatement;
 import com.semmle.js.ast.CallExpression;
 import com.semmle.js.ast.Expression;
-import com.semmle.js.ast.ExpressionStatement;
-import com.semmle.js.ast.IFunction;
-import com.semmle.js.ast.Identifier;
-import com.semmle.js.ast.IfStatement;
 import com.semmle.js.ast.MemberExpression;
 import com.semmle.js.ast.Node;
-import com.semmle.js.ast.ParenthesizedExpression;
-import com.semmle.js.ast.Program;
-import com.semmle.js.ast.Statement;
-import com.semmle.js.ast.TryStatement;
-import com.semmle.js.ast.UnaryExpression;
-import com.semmle.js.ast.VariableDeclaration;
-import com.semmle.js.ast.VariableDeclarator;
-import java.util.List;
 
 /** A utility class for detecting Node.js code. */
-public class NodeJSDetector {
+public class NodeJSDetector extends AbstractDetector {
   /**
    * Is {@code ast} a program that looks like Node.js code, that is, does it contain a top-level
-   * {@code require} or an export?
+   * {@code require} or an {@code module.exports = ...}/{@code exports = ...}?
    */
   public static boolean looksLikeNodeJS(Node ast) {
-    if (!(ast instanceof Program)) return false;
-
-    return hasToplevelRequireOrExport(((Program) ast).getBody());
+    return new NodeJSDetector().programDetection(ast);
   }
 
-  /**
-   * Does this program contain a statement that looks like a Node.js {@code require} or an export?
-   *
-   * <p>We recursively traverse argument-less immediately invoked function expressions (i.e., no UMD
-   * modules), but not loops or if statements.
-   */
-  private static boolean hasToplevelRequireOrExport(List<Statement> stmts) {
-    for (Statement stmt : stmts) if (hasToplevelRequireOrExport(stmt)) return true;
-    return false;
-  }
-
-  private static boolean hasToplevelRequireOrExport(Statement stmt) {
-    if (stmt instanceof ExpressionStatement) {
-      Expression e = stripParens(((ExpressionStatement) stmt).getExpression());
-
-      // check whether `e` is an iife; if so, recursively check its body
-
-      // strip off unary operators to handle `!function(){...}()`
-      if (e instanceof UnaryExpression) e = ((UnaryExpression) e).getArgument();
-
-      if (e instanceof CallExpression && ((CallExpression) e).getArguments().isEmpty()) {
-        Expression callee = stripParens(((CallExpression) e).getCallee());
-        if (callee instanceof IFunction) {
-          Node body = ((IFunction) callee).getBody();
-          if (body instanceof BlockStatement)
-            return hasToplevelRequireOrExport(((BlockStatement) body).getBody());
-        }
-      }
-
-      if (isRequireCall(e) || isExport(e)) return true;
-
-    } else if (stmt instanceof VariableDeclaration) {
-      for (VariableDeclarator decl : ((VariableDeclaration) stmt).getDeclarations()) {
-        Expression init = stripParens(decl.getInit());
-        if (isRequireCall(init) || isExport(init)) return true;
-      }
-
-    } else if (stmt instanceof TryStatement) {
-      return hasToplevelRequireOrExport(((TryStatement) stmt).getBlock());
-
-    } else if (stmt instanceof BlockStatement) {
-      return hasToplevelRequireOrExport(((BlockStatement) stmt).getBody());
-
-    } else if (stmt instanceof IfStatement) {
-      IfStatement is = (IfStatement) stmt;
-      return hasToplevelRequireOrExport(is.getConsequent())
-          || hasToplevelRequireOrExport(is.getAlternate());
-    }
-
-    return false;
-  }
-
-  private static Expression stripParens(Expression e) {
-    if (e instanceof ParenthesizedExpression)
-      return stripParens(((ParenthesizedExpression) e).getExpression());
-    return e;
-  }
-
-  /**
-   * Is {@code e} a call to a function named {@code require} with one argument, or an assignment
-   * whose right hand side is the result of such a call?
-   */
-  private static boolean isRequireCall(Expression e) {
+  @Override
+  protected boolean visitExpression(Expression e) {
+    // require('...')
     if (e instanceof CallExpression) {
       CallExpression call = (CallExpression) e;
       Expression callee = call.getCallee();
       if (isIdentifier(callee, "require") && call.getArguments().size() == 1) return true;
-      if (isRequireCall(callee)) return true;
-      return false;
-    } else if (e instanceof MemberExpression) {
-      return isRequireCall(((MemberExpression) e).getObject());
-    } else if (e instanceof AssignmentExpression) {
-      AssignmentExpression assgn = (AssignmentExpression) e;
-
-      // filter out compound assignments
-      if (!"=".equals(assgn.getOperator())) return false;
-
-      return isRequireCall(assgn.getRight());
-    }
-    return false;
     }
 
-  /**
-   * Does {@code e} look like a Node.js export?
-   *
-   * <p>Currently, three kinds of exports are recognised:
-   *
-   * <ul>
-   *   <li><code>exports.foo = ...</code>
-   *   <li><code>module.exports = ...</code>
-   *   <li><code>module.exports.foo = ...</code>
-   * </ul>
-   *
-   * Detection is done recursively, so <code>foo = exports.foo = ...</code> is handled correctly.
-   */
-  private static boolean isExport(Expression e) {
     if (e instanceof AssignmentExpression) {
       AssignmentExpression assgn = (AssignmentExpression) e;
 
@@ -149,12 +47,9 @@ public class NodeJSDetector {
           if (isModuleExports(targetBase)) return true;
         }
       }
-
-      // recursively check right hand side
-      return isExport(assgn.getRight());
     }
 
-    return false;
+    return super.visitExpression(e);
   }
 
   /** Is {@code me} a member expression {@code module.exports}? */
@@ -163,9 +58,4 @@ public class NodeJSDetector {
         && isIdentifier(me.getObject(), "module")
         && isIdentifier(me.getProperty(), "exports");
   }
-
-  /** Is {@code e} an identifier with name {@code name}? */
-  private static boolean isIdentifier(Expression e, String name) {
-    return e instanceof Identifier && name.equals(((Identifier) e).getName());
-  }
 }

javascript/extractor/test/com/semmle/js/extractor/test/AllTests.java

@@ -18,6 +18,7 @@ import org.junit.runners.Suite.SuiteClasses;
 @SuiteClasses({
   JSXTests.class,
   NodeJSDetectorTests.class,
+  ES2015DetectorTests.class,
   TrapTests.class,
   ObjectRestSpreadTests.class,
   ClassPropertiesTests.class,

javascript/extractor/test/com/semmle/js/extractor/test/BUILD.bazel

@@ -12,6 +12,9 @@ java_test(
         "TS_WRAPPER_ZIP": "$(rlocationpath //javascript/extractor/lib/typescript)",
     },
     test_class = "com.semmle.js.extractor.test.AllTests",
+    # To use `replaceExpectedOutput` you need to uncomment the following line
+    # (to be allowed to override the .trap files on disk)
+    # tags = ["no-sandbox"],
     deps = [
         "//javascript/extractor",
         "//javascript/extractor:deps",

javascript/extractor/test/com/semmle/js/extractor/test/ES2015DetectorTests.java

@@ -0,0 +1,58 @@
+package com.semmle.js.extractor.test;
+
+import com.semmle.js.ast.Node;
+import com.semmle.js.extractor.ES2015Detector;
+import com.semmle.js.extractor.ExtractionMetrics;
+import com.semmle.js.extractor.ExtractorConfig;
+import com.semmle.js.extractor.ExtractorConfig.SourceType;
+import com.semmle.js.parser.JSParser;
+import com.semmle.js.parser.JSParser.Result;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class ES2015DetectorTests {
+  // using `experimental: true` as we do in real extractor, see `extractSource` method
+  // in `AutoBuild.java`
+  private static final ExtractorConfig CONFIG = new ExtractorConfig(true);
+
+  private void isES2015(String src, boolean expected) {
+    Result res = JSParser.parse(CONFIG, SourceType.MODULE, src, new ExtractionMetrics());
+    Node ast = res.getAST();
+    Assert.assertNotNull(ast);
+    Assert.assertTrue(ES2015Detector.looksLikeES2015(ast) == expected);
+  }
+
+  @Test
+  public void testImport() {
+    isES2015("import * as fs from 'fs';", true);
+  }
+
+  @Test
+  public void testExport() {
+    isES2015("export function foo() { };", true);
+  }
+
+  @Test
+  public void testDynamicImport() {
+    isES2015("import('fs');", true);
+  }
+
+  @Test
+  public void testDynamicImportAssign() {
+    isES2015("var fs = import('fs');", true);
+  }
+
+  @Test
+  public void testDynamicImportThen() {
+    isES2015("import('o').then((o) => {});", true);
+  }
+
+  @Test
+  public void importInBlockComment() {
+    isES2015("/*\n"
+      + "  import * from 'fs';\n"
+      + "*/\n"
+      + "const fs = require('fs');",
+      false);
+  }
+}

javascript/extractor/test/com/semmle/js/extractor/test/NodeJSDetectorTests.java

@@ -11,7 +11,9 @@ import org.junit.Assert;
 import org.junit.Test;
 
 public class NodeJSDetectorTests {
-  private static final ExtractorConfig CONFIG = new ExtractorConfig(false);
+  // using `experimental: true` as we do in real extractor, see `extractSource` method
+  // in `AutoBuild.java`
+  private static final ExtractorConfig CONFIG = new ExtractorConfig(true);
 
   private void isNodeJS(String src, boolean expected) {
     Result res = JSParser.parse(CONFIG, SourceType.SCRIPT, src, new ExtractionMetrics());

javascript/extractor/test/com/semmle/js/extractor/test/TrapTests.java

@@ -161,6 +161,14 @@ public class TrapTests {
                         byte[] actual_utf8_bytes = StringUtil.stringToBytes(sw.toString());
                         String actual = new String(actual_utf8_bytes, Charset.forName("UTF-8"));
                         File trap = new File(outputDir, f.getName() + ".trap");
+                        // NOTE: If you want to replace expected output, you MUST change
+                        // the way this test is run under bazel to escape the bazel
+                        // sandbox. Add `tags = ["no-sandbox"]` to the test rule in
+                        // javascript/extractor/test/com/semmle/js/extractor/test/BUILD.bazel
+                        //
+                        // if you have problems with too much caching, you need to find the right bazel command,
+                        // and run `./build --bazel test ... --cache_test_results=no`
+                        // (at least I had problems getting the "no-cache" tag to work)
                         boolean replaceExpectedOutput = false;
                         if (replaceExpectedOutput) {
                           System.out.println("Replacing expected output for " + trap);

javascript/extractor/tests/flow/output/trap/anonFunctionWithoutParens.js.trap

@@ -72,20 +72,14 @@ locations_default(#20025,#10000,1,40,1,39)
 hasLocation(#20024,#20025)
 toplevels(#20001,0)
 hasLocation(#20001,#20003)
-#20026=@"module;{#10000},1,1"
-scopes(#20026,3)
-scopenodes(#20001,#20026)
-scopenesting(#20026,#20000)
-is_module(#20001)
-is_es2015_module(#20001)
-#20027=*
-entry_cfg_node(#20027,#20001)
-#20028=@"loc,{#10000},1,1,1,0"
-locations_default(#20028,#10000,1,1,1,0)
-hasLocation(#20027,#20028)
-#20029=*
-exit_cfg_node(#20029,#20001)
-hasLocation(#20029,#20025)
-successor(#20027,#20029)
+#20026=*
+entry_cfg_node(#20026,#20001)
+#20027=@"loc,{#10000},1,1,1,0"
+locations_default(#20027,#10000,1,1,1,0)
+hasLocation(#20026,#20027)
+#20028=*
+exit_cfg_node(#20028,#20001)
+hasLocation(#20028,#20025)
+successor(#20026,#20028)
 numlines(#10000,1,1,0)
 filetype(#10000,"javascript")

javascript/extractor/tests/flow/output/trap/anonIndexer.js.trap

@@ -85,20 +85,14 @@ toplevels(#20001,0)
 #20030=@"loc,{#10000},1,1,2,0"
 locations_default(#20030,#10000,1,1,2,0)
 hasLocation(#20001,#20030)
-#20031=@"module;{#10000},1,1"
-scopes(#20031,3)
-scopenodes(#20001,#20031)
-scopenesting(#20031,#20000)
-is_module(#20001)
-is_es2015_module(#20001)
-#20032=*
-entry_cfg_node(#20032,#20001)
-#20033=@"loc,{#10000},1,1,1,0"
-locations_default(#20033,#10000,1,1,1,0)
-hasLocation(#20032,#20033)
-#20034=*
-exit_cfg_node(#20034,#20001)
-hasLocation(#20034,#20029)
-successor(#20032,#20034)
+#20031=*
+entry_cfg_node(#20031,#20001)
+#20032=@"loc,{#10000},1,1,1,0"
+locations_default(#20032,#10000,1,1,1,0)
+hasLocation(#20031,#20032)
+#20033=*
+exit_cfg_node(#20033,#20001)
+hasLocation(#20033,#20029)
+successor(#20031,#20033)
 numlines(#10000,1,1,0)
 filetype(#10000,"javascript")

javascript/extractor/tests/flow/output/trap/declared-module-imports.js.trap

@@ -128,20 +128,14 @@ toplevels(#20001,0)
 #20045=@"loc,{#10000},1,1,5,0"
 locations_default(#20045,#10000,1,1,5,0)
 hasLocation(#20001,#20045)
-#20046=@"module;{#10000},1,1"
-scopes(#20046,3)
-scopenodes(#20001,#20046)
-scopenesting(#20046,#20000)
-is_module(#20001)
-is_es2015_module(#20001)
-#20047=*
-entry_cfg_node(#20047,#20001)
-#20048=@"loc,{#10000},1,1,1,0"
-locations_default(#20048,#10000,1,1,1,0)
-hasLocation(#20047,#20048)
-#20049=*
-exit_cfg_node(#20049,#20001)
-hasLocation(#20049,#20044)
-successor(#20047,#20049)
+#20046=*
+entry_cfg_node(#20046,#20001)
+#20047=@"loc,{#10000},1,1,1,0"
+locations_default(#20047,#10000,1,1,1,0)
+hasLocation(#20046,#20047)
+#20048=*
+exit_cfg_node(#20048,#20001)
+hasLocation(#20048,#20044)
+successor(#20046,#20048)
 numlines(#10000,4,4,0)
 filetype(#10000,"javascript")

javascript/extractor/tests/flow/output/trap/exportOpaqueType.js.trap

@@ -70,20 +70,14 @@ toplevels(#20001,0)
 #20024=@"loc,{#10000},1,1,2,0"
 locations_default(#20024,#10000,1,1,2,0)
 hasLocation(#20001,#20024)
-#20025=@"module;{#10000},1,1"
-scopes(#20025,3)
-scopenodes(#20001,#20025)
-scopenesting(#20025,#20000)
-is_module(#20001)
-is_es2015_module(#20001)
-#20026=*
-entry_cfg_node(#20026,#20001)
-#20027=@"loc,{#10000},1,1,1,0"
-locations_default(#20027,#10000,1,1,1,0)
-hasLocation(#20026,#20027)
-#20028=*
-exit_cfg_node(#20028,#20001)
-hasLocation(#20028,#20023)
-successor(#20026,#20028)
+#20025=*
+entry_cfg_node(#20025,#20001)
+#20026=@"loc,{#10000},1,1,1,0"
+locations_default(#20026,#10000,1,1,1,0)
+hasLocation(#20025,#20026)
+#20027=*
+exit_cfg_node(#20027,#20001)
+hasLocation(#20027,#20023)
+successor(#20025,#20027)
 numlines(#10000,1,1,0)
 filetype(#10000,"javascript")

javascript/extractor/tests/flow/output/trap/importTypeInDeclaredModule.js.trap

@@ -205,20 +205,14 @@ toplevels(#20001,0)
 #20071=@"loc,{#10000},1,1,11,2"
 locations_default(#20071,#10000,1,1,11,2)
 hasLocation(#20001,#20071)
-#20072=@"module;{#10000},1,1"
-scopes(#20072,3)
-scopenodes(#20001,#20072)
-scopenesting(#20072,#20000)
-is_module(#20001)
-is_es2015_module(#20001)
-#20073=*
-entry_cfg_node(#20073,#20001)
-#20074=@"loc,{#10000},1,1,1,0"
-locations_default(#20074,#10000,1,1,1,0)
-hasLocation(#20073,#20074)
-#20075=*
-exit_cfg_node(#20075,#20001)
-hasLocation(#20075,#20070)
-successor(#20073,#20075)
+#20072=*
+entry_cfg_node(#20072,#20001)
+#20073=@"loc,{#10000},1,1,1,0"
+locations_default(#20073,#10000,1,1,1,0)
+hasLocation(#20072,#20073)
+#20074=*
+exit_cfg_node(#20074,#20001)
+hasLocation(#20074,#20070)
+successor(#20072,#20074)
 numlines(#10000,11,10,0)
 filetype(#10000,"javascript")

javascript/extractor/tests/flow/output/trap/variance.js.trap

@@ -160,95 +160,89 @@ toplevels(#20001,0)
 #20057=@"loc,{#10000},1,1,7,1"
 locations_default(#20057,#10000,1,1,7,1)
 hasLocation(#20001,#20057)
-#20058=@"module;{#10000},1,1"
-scopes(#20058,3)
-scopenodes(#20001,#20058)
-scopenesting(#20058,#20000)
-is_module(#20001)
-is_es2015_module(#20001)
-#20059=@"var;{Foo};{#20058}"
-variables(#20059,"Foo",#20058)
-#20060=@"local_type_name;{Foo};{#20058}"
-local_type_names(#20060,"Foo",#20058)
-#20061=*
-stmts(#20061,26,#20001,0,"class F ... : int\n}")
-#20062=@"loc,{#10000},5,1,7,1"
-locations_default(#20062,#10000,5,1,7,1)
-hasLocation(#20061,#20062)
-stmt_containers(#20061,#20001)
+#20058=@"var;{Foo};{#20000}"
+variables(#20058,"Foo",#20000)
+#20059=@"local_type_name;{Foo};{#20000}"
+local_type_names(#20059,"Foo",#20000)
+#20060=*
+stmts(#20060,26,#20001,0,"class F ... : int\n}")
+#20061=@"loc,{#10000},5,1,7,1"
+locations_default(#20061,#10000,5,1,7,1)
+hasLocation(#20060,#20061)
+stmt_containers(#20060,#20001)
+#20062=*
+exprs(#20062,78,#20060,0,"Foo")
+hasLocation(#20062,#20043)
+enclosing_stmt(#20062,#20060)
+expr_containers(#20062,#20001)
+literals("Foo","Foo",#20062)
+decl(#20062,#20058)
+typedecl(#20062,#20059)
 #20063=*
-exprs(#20063,78,#20061,0,"Foo")
-hasLocation(#20063,#20043)
-enclosing_stmt(#20063,#20061)
-expr_containers(#20063,#20001)
-literals("Foo","Foo",#20063)
-decl(#20063,#20059)
-typedecl(#20063,#20060)
+scopes(#20063,10)
+scopenodes(#20060,#20063)
+scopenesting(#20063,#20000)
 #20064=*
-scopes(#20064,10)
-scopenodes(#20061,#20064)
-scopenesting(#20064,#20058)
-#20065=*
-properties(#20065,#20061,2,8,"+x: int")
-#20066=@"loc,{#10000},6,3,6,9"
-locations_default(#20066,#10000,6,3,6,9)
-hasLocation(#20065,#20066)
+properties(#20064,#20060,2,8,"+x: int")
+#20065=@"loc,{#10000},6,3,6,9"
+locations_default(#20065,#10000,6,3,6,9)
+hasLocation(#20064,#20065)
+#20066=*
 #20067=*
+exprs(#20067,0,#20064,0,"x")
+hasLocation(#20067,#20049)
+expr_containers(#20067,#20066)
+literals("x","x",#20067)
 #20068=*
-exprs(#20068,0,#20065,0,"x")
-hasLocation(#20068,#20049)
-expr_containers(#20068,#20067)
-literals("x","x",#20068)
-#20069=*
-properties(#20069,#20061,3,0,"constructor() {}")
-#20070=@"loc,{#10000},5,11,5,10"
-locations_default(#20070,#10000,5,11,5,10)
-hasLocation(#20069,#20070)
+properties(#20068,#20060,3,0,"constructor() {}")
+#20069=@"loc,{#10000},5,11,5,10"
+locations_default(#20069,#10000,5,11,5,10)
+hasLocation(#20068,#20069)
+#20070=*
+exprs(#20070,0,#20068,0,"constructor")
+hasLocation(#20070,#20069)
+enclosing_stmt(#20070,#20060)
+expr_containers(#20070,#20001)
+literals("constructor","constructor",#20070)
+exprs(#20066,9,#20068,1,"() {}")
+hasLocation(#20066,#20069)
+enclosing_stmt(#20066,#20060)
+expr_containers(#20066,#20001)
 #20071=*
-exprs(#20071,0,#20069,0,"constructor")
-hasLocation(#20071,#20070)
-enclosing_stmt(#20071,#20061)
-expr_containers(#20071,#20001)
-literals("constructor","constructor",#20071)
-exprs(#20067,9,#20069,1,"() {}")
-hasLocation(#20067,#20070)
-enclosing_stmt(#20067,#20061)
-expr_containers(#20067,#20001)
-#20072=*
-scopes(#20072,1)
-scopenodes(#20067,#20072)
-scopenesting(#20072,#20064)
-#20073=@"var;{arguments};{#20072}"
-variables(#20073,"arguments",#20072)
-is_arguments_object(#20073)
+scopes(#20071,1)
+scopenodes(#20066,#20071)
+scopenesting(#20071,#20063)
+#20072=@"var;{arguments};{#20071}"
+variables(#20072,"arguments",#20071)
+is_arguments_object(#20072)
+#20073=*
+stmts(#20073,1,#20066,-2,"{}")
+hasLocation(#20073,#20069)
+stmt_containers(#20073,#20066)
+is_method(#20068)
 #20074=*
-stmts(#20074,1,#20067,-2,"{}")
-hasLocation(#20074,#20070)
-stmt_containers(#20074,#20067)
-is_method(#20069)
-#20075=*
-entry_cfg_node(#20075,#20001)
-#20076=@"loc,{#10000},1,1,1,0"
-locations_default(#20076,#10000,1,1,1,0)
-hasLocation(#20075,#20076)
+entry_cfg_node(#20074,#20001)
+#20075=@"loc,{#10000},1,1,1,0"
+locations_default(#20075,#10000,1,1,1,0)
+hasLocation(#20074,#20075)
+#20076=*
+exit_cfg_node(#20076,#20001)
+hasLocation(#20076,#20056)
+successor(#20067,#20064)
+successor(#20066,#20068)
 #20077=*
-exit_cfg_node(#20077,#20001)
-hasLocation(#20077,#20056)
-successor(#20068,#20065)
-successor(#20067,#20069)
+entry_cfg_node(#20077,#20066)
+hasLocation(#20077,#20069)
+successor(#20064,#20073)
 #20078=*
-entry_cfg_node(#20078,#20067)
-hasLocation(#20078,#20070)
-successor(#20065,#20074)
-#20079=*
-exit_cfg_node(#20079,#20067)
-hasLocation(#20079,#20070)
-successor(#20074,#20079)
-successor(#20078,#20068)
-successor(#20071,#20067)
-successor(#20069,#20061)
-successor(#20063,#20071)
-successor(#20061,#20077)
-successor(#20075,#20063)
+exit_cfg_node(#20078,#20066)
+hasLocation(#20078,#20069)
+successor(#20073,#20078)
+successor(#20077,#20067)
+successor(#20070,#20066)
+successor(#20068,#20060)
+successor(#20062,#20070)
+successor(#20060,#20076)
+successor(#20074,#20062)
 numlines(#10000,7,6,0)
 filetype(#10000,"javascript")

javascript/extractor/tests/node/input/detection.js

@@ -0,0 +1,6 @@
+// the comment below (with 'import' on line starting with whitespace) caused the
+// extractor to think it was a es2015 module and not a commonjs module.
+/*
+  import
+*/
+const fs = require('fs');

javascript/extractor/tests/node/output/trap/detection.js.trap

@@ -0,0 +1,203 @@
+#10000=@"/detection.js;sourcefile"
+files(#10000,"/detection.js")
+#10001=@"/;folder"
+folders(#10001,"/")
+containerparent(#10001,#10000)
+#10002=@"loc,{#10000},0,0,0,0"
+locations_default(#10002,#10000,0,0,0,0)
+hasLocation(#10000,#10002)
+#20000=@"global_scope"
+scopes(#20000,0)
+#20001=@"script;{#10000},1,1"
+#20002=*
+comments(#20002,0,#20001," the comment below (with 'import' on line starting with whitespace) caused the","// the  ... sed the")
+#20003=@"loc,{#10000},1,1,1,80"
+locations_default(#20003,#10000,1,1,1,80)
+hasLocation(#20002,#20003)
+#20004=*
+comments(#20004,0,#20001," extractor to think it was a es2015 module and not a commonjs module.","// extr ... module.")
+#20005=@"loc,{#10000},2,1,2,71"
+locations_default(#20005,#10000,2,1,2,71)
+hasLocation(#20004,#20005)
+#20006=*
+comments(#20006,1,#20001,"
+  import
+","/*\n  import\n*/")
+#20007=@"loc,{#10000},3,1,5,2"
+locations_default(#20007,#10000,3,1,5,2)
+hasLocation(#20006,#20007)
+#20008=*
+lines(#20008,#20001,"// the comment below (with 'import' on line starting with whitespace) caused the","
+")
+hasLocation(#20008,#20003)
+#20009=*
+lines(#20009,#20001,"// extractor to think it was a es2015 module and not a commonjs module.","
+")
+hasLocation(#20009,#20005)
+#20010=*
+lines(#20010,#20001,"/*","
+")
+#20011=@"loc,{#10000},3,1,3,2"
+locations_default(#20011,#10000,3,1,3,2)
+hasLocation(#20010,#20011)
+#20012=*
+lines(#20012,#20001,"  import","
+")
+#20013=@"loc,{#10000},4,1,4,8"
+locations_default(#20013,#10000,4,1,4,8)
+hasLocation(#20012,#20013)
+indentation(#10000,4," ",2)
+#20014=*
+lines(#20014,#20001,"*/","
+")
+#20015=@"loc,{#10000},5,1,5,2"
+locations_default(#20015,#10000,5,1,5,2)
+hasLocation(#20014,#20015)
+#20016=*
+lines(#20016,#20001,"const fs = require('fs');","
+")
+#20017=@"loc,{#10000},6,1,6,25"
+locations_default(#20017,#10000,6,1,6,25)
+hasLocation(#20016,#20017)
+numlines(#20001,6,1,5)
+#20018=*
+tokeninfo(#20018,7,#20001,0,"const")
+#20019=@"loc,{#10000},6,1,6,5"
+locations_default(#20019,#10000,6,1,6,5)
+hasLocation(#20018,#20019)
+next_token(#20002,#20018)
+next_token(#20004,#20018)
+next_token(#20006,#20018)
+#20020=*
+tokeninfo(#20020,6,#20001,1,"fs")
+#20021=@"loc,{#10000},6,7,6,8"
+locations_default(#20021,#10000,6,7,6,8)
+hasLocation(#20020,#20021)
+#20022=*
+tokeninfo(#20022,8,#20001,2,"=")
+#20023=@"loc,{#10000},6,10,6,10"
+locations_default(#20023,#10000,6,10,6,10)
+hasLocation(#20022,#20023)
+#20024=*
+tokeninfo(#20024,6,#20001,3,"require")
+#20025=@"loc,{#10000},6,12,6,18"
+locations_default(#20025,#10000,6,12,6,18)
+hasLocation(#20024,#20025)
+#20026=*
+tokeninfo(#20026,8,#20001,4,"(")
+#20027=@"loc,{#10000},6,19,6,19"
+locations_default(#20027,#10000,6,19,6,19)
+hasLocation(#20026,#20027)
+#20028=*
+tokeninfo(#20028,4,#20001,5,"'fs'")
+#20029=@"loc,{#10000},6,20,6,23"
+locations_default(#20029,#10000,6,20,6,23)
+hasLocation(#20028,#20029)
+#20030=*
+tokeninfo(#20030,8,#20001,6,")")
+#20031=@"loc,{#10000},6,24,6,24"
+locations_default(#20031,#10000,6,24,6,24)
+hasLocation(#20030,#20031)
+#20032=*
+tokeninfo(#20032,8,#20001,7,";")
+#20033=@"loc,{#10000},6,25,6,25"
+locations_default(#20033,#10000,6,25,6,25)
+hasLocation(#20032,#20033)
+#20034=*
+tokeninfo(#20034,0,#20001,8,"")
+#20035=@"loc,{#10000},7,1,7,0"
+locations_default(#20035,#10000,7,1,7,0)
+hasLocation(#20034,#20035)
+toplevels(#20001,0)
+#20036=@"loc,{#10000},1,1,7,0"
+locations_default(#20036,#10000,1,1,7,0)
+hasLocation(#20001,#20036)
+#20037=@"var;{global};{#20000}"
+variables(#20037,"global",#20000)
+#20038=@"var;{process};{#20000}"
+variables(#20038,"process",#20000)
+#20039=@"var;{console};{#20000}"
+variables(#20039,"console",#20000)
+#20040=@"var;{Buffer};{#20000}"
+variables(#20040,"Buffer",#20000)
+#20041=@"module;{#10000},1,1"
+scopes(#20041,3)
+scopenodes(#20001,#20041)
+scopenesting(#20041,#20000)
+#20042=@"var;{require};{#20041}"
+variables(#20042,"require",#20041)
+#20043=@"var;{module};{#20041}"
+variables(#20043,"module",#20041)
+#20044=@"var;{exports};{#20041}"
+variables(#20044,"exports",#20041)
+#20045=@"var;{__filename};{#20041}"
+variables(#20045,"__filename",#20041)
+#20046=@"var;{__dirname};{#20041}"
+variables(#20046,"__dirname",#20041)
+#20047=@"var;{arguments};{#20041}"
+variables(#20047,"arguments",#20041)
+is_module(#20001)
+#20048=@"var;{fs};{#20041}"
+variables(#20048,"fs",#20041)
+#20049=*
+stmts(#20049,22,#20001,0,"const f ... ('fs');")
+hasLocation(#20049,#20017)
+stmt_containers(#20049,#20001)
+#20050=*
+exprs(#20050,64,#20049,0,"fs = require('fs')")
+#20051=@"loc,{#10000},6,7,6,24"
+locations_default(#20051,#10000,6,7,6,24)
+hasLocation(#20050,#20051)
+enclosing_stmt(#20050,#20049)
+expr_containers(#20050,#20001)
+#20052=*
+exprs(#20052,78,#20050,0,"fs")
+hasLocation(#20052,#20021)
+enclosing_stmt(#20052,#20049)
+expr_containers(#20052,#20001)
+literals("fs","fs",#20052)
+decl(#20052,#20048)
+#20053=*
+exprs(#20053,13,#20050,1,"require('fs')")
+#20054=@"loc,{#10000},6,12,6,24"
+locations_default(#20054,#10000,6,12,6,24)
+hasLocation(#20053,#20054)
+enclosing_stmt(#20053,#20049)
+expr_containers(#20053,#20001)
+#20055=*
+exprs(#20055,79,#20053,-1,"require")
+hasLocation(#20055,#20025)
+enclosing_stmt(#20055,#20049)
+expr_containers(#20055,#20001)
+literals("require","require",#20055)
+bind(#20055,#20042)
+#20056=*
+exprs(#20056,4,#20053,0,"'fs'")
+hasLocation(#20056,#20029)
+enclosing_stmt(#20056,#20049)
+expr_containers(#20056,#20001)
+literals("fs","'fs'",#20056)
+#20057=*
+regexpterm(#20057,14,#20056,0,"fs")
+#20058=@"loc,{#10000},6,21,6,22"
+locations_default(#20058,#10000,6,21,6,22)
+hasLocation(#20057,#20058)
+regexp_const_value(#20057,"fs")
+#20059=*
+entry_cfg_node(#20059,#20001)
+#20060=@"loc,{#10000},1,1,1,0"
+locations_default(#20060,#10000,1,1,1,0)
+hasLocation(#20059,#20060)
+#20061=*
+exit_cfg_node(#20061,#20001)
+hasLocation(#20061,#20035)
+successor(#20049,#20052)
+successor(#20056,#20053)
+successor(#20055,#20056)
+successor(#20053,#20050)
+successor(#20052,#20055)
+successor(#20050,#20061)
+successor(#20059,#20049)
+is_nodejs(#20001)
+numlines(#10000,6,1,5)
+filetype(#10000,"javascript")

javascript/ql/lib/change-notes/2024-04-02-more-robust-commonjs-detection.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Improved detection of whether a file uses CommonJS module system.

javascript/ql/lib/semmle/javascript/endpoints/EndpointNaming.qll

@@ -432,7 +432,7 @@ string renderName(string package, string name) { result = join("(" + package + "
  * These names are not necessarily part of a package's public API, and so we only used them
  * as a fallback when a publicly-accessible access path cannot be found.
  */
-private module InternalModuleNaming {
+module InternalModuleNaming {
   /** Gets the path to `folder` relative to its enclosing non-private `package.json` file. */
   private string getPackageRelativePathFromFolder(Folder folder) {
     exists(PackageJson json |
@@ -446,7 +446,10 @@ private module InternalModuleNaming {
       getPackageRelativePathFromFolder(folder.getParentContainer()) + "/" + folder.getBaseName()
   }
 
-  private string getPackageRelativePath(Module mod) {
+  /**
+   * Gets the path to `mod` relative to its enclosing package, including the package name.
+   */
+  string getPackageRelativePath(Module mod) {
     exists(PackageJson json, string relativePath |
       not json.isPrivate() and
       json.getExportedModule(relativePath) = mod and

javascript/ql/lib/semmle/javascript/frameworks/data/ModelsAsData.qll

@@ -108,7 +108,11 @@ module ModelExport<ModelExportSig S> {
     }
 
     predicate exposedName(API::Node node, string type, string path) {
-      node = API::moduleExport(type) and path = ""
+      exists(string moduleName |
+        node = API::moduleExport(moduleName) and
+        path = "" and
+        type = "(" + moduleName + ")"
+      )
     }
 
     predicate suggestedName(API::Node node, string type) {

javascript/ql/lib/semmle/javascript/frameworks/data/internal/ApiGraphModelsSpecific.qll

@@ -34,6 +34,11 @@ class Location = JS::Location;
  *
  * Type names have form `package.type` or just `package` if referring to the package export
  * object. If `package` contains a `.` character it must be enclosed in single quotes, such as `'package'.type`.
+ *
+ * A type name of form `(package)` may also be used when refering to the package export object.
+ * We allow this syntax as an alternative to the above, so models generated based on `EndpointNaming` look more consistent.
+ * However, access paths are deliberately not parsed here, as we can not handle aliasing at this stage.
+ * The model generator must explicitly generate the step between `(package)` and `(package).foo`, for example.
  */
 bindingset[rawType]
 predicate parseTypeString(string rawType, string package, string qualifiedName) {
@@ -42,6 +47,9 @@ predicate parseTypeString(string rawType, string package, string qualifiedName)
     package = rawType.regexpCapture(regexp, 1).regexpReplaceAll("^'|'$", "") and
     qualifiedName = rawType.regexpCapture(regexp, 2).regexpReplaceAll("^\\.", "")
   )
+  or
+  package = rawType.regexpCapture("[(]([^)]+)[)]", 1) and
+  qualifiedName = ""
 }
 
 /**

javascript/ql/test/library-tests/ModelGeneration/ModelGeneration.expected

@@ -1,17 +1,17 @@
 typeModel
-| (aliases).Alias1 | aliases | Member[Alias1] |
-| (aliases).Alias1 | aliases | Member[Alias2] |
-| (aliases).Alias1 | aliases | Member[Alias3].Member[x] |
-| (aliases).Alias1 | aliases | Member[Alias4].Member[x].Member[x] |
-| (aliases).Alias1 | aliases | Member[AliasedClass] |
+| (aliases).Alias1 | (aliases) | Member[Alias1] |
+| (aliases).Alias1 | (aliases) | Member[Alias2] |
+| (aliases).Alias1 | (aliases) | Member[Alias3].Member[x] |
+| (aliases).Alias1 | (aliases) | Member[Alias4].Member[x].Member[x] |
+| (aliases).Alias1 | (aliases) | Member[AliasedClass] |
 | (aliases).Alias1.prototype | (aliases).Alias1 | Instance |
 | (aliases).Alias1.prototype | (aliases).Alias1.prototype.foo | ReturnValue |
 | (aliases).Alias1.prototype.foo | (aliases).Alias1.prototype | Member[foo] |
-| (long-access-path).a.shortcut.d | long-access-path | Member[a].Member[b].Member[c].Member[d] |
-| (long-access-path).a.shortcut.d | long-access-path | Member[a].Member[shortcut].Member[d] |
+| (long-access-path).a.shortcut.d | (long-access-path) | Member[a].Member[b].Member[c].Member[d] |
+| (long-access-path).a.shortcut.d | (long-access-path) | Member[a].Member[shortcut].Member[d] |
 | (long-access-path).a.shortcut.d.e | (long-access-path).a.shortcut.d | Member[e] |
-| (reexport).func | reexport | Member[func] |
-| (return-this).FluentInterface | return-this | Member[FluentInterface] |
+| (reexport).func | (reexport) | Member[func] |
+| (return-this).FluentInterface | (return-this) | Member[FluentInterface] |
 | (return-this).FluentInterface.prototype | (return-this).FluentInterface | Instance |
 | (return-this).FluentInterface.prototype | (return-this).FluentInterface.prototype.bar | ReturnValue |
 | (return-this).FluentInterface.prototype | (return-this).FluentInterface.prototype.baz | ReturnValue |
@@ -21,45 +21,45 @@ typeModel
 | (return-this).FluentInterface.prototype.foo | (return-this).FluentInterface.prototype | Member[foo] |
 | (return-this).FluentInterface.prototype.notFluent | (return-this).FluentInterface.prototype | Member[notFluent] |
 | (return-this).FluentInterface.prototype.notFluent2 | (return-this).FluentInterface.prototype | Member[notFluent2] |
-| (root-function).PublicClass | root-function | Member[PublicClass] |
+| (root-function).PublicClass | (root-function) | Member[PublicClass] |
+| (root-function).PublicClass.prototype | (root-function) | ReturnValue |
 | (root-function).PublicClass.prototype | (root-function).PublicClass | Instance |
-| (root-function).PublicClass.prototype | root-function | ReturnValue |
 | (root-function).PublicClass.prototype.method | (root-function).PublicClass.prototype | Member[method] |
-| (semi-internal-class).PublicClass | semi-internal-class | Member[PublicClass] |
+| (semi-internal-class).PublicClass | (semi-internal-class) | Member[PublicClass] |
 | (semi-internal-class).PublicClass.prototype | (semi-internal-class).PublicClass | Instance |
 | (semi-internal-class).PublicClass.prototype | (semi-internal-class).SemiInternalClass.prototype.method | ReturnValue |
 | (semi-internal-class).PublicClass.prototype | (semi-internal-class).getAnonymous~expr2 | ReturnValue |
 | (semi-internal-class).PublicClass.prototype.publicMethod | (semi-internal-class).PublicClass.prototype | Member[publicMethod] |
 | (semi-internal-class).SemiInternalClass.prototype | (semi-internal-class).get | ReturnValue |
 | (semi-internal-class).SemiInternalClass.prototype.method | (semi-internal-class).SemiInternalClass.prototype | Member[method] |
-| (semi-internal-class).get | semi-internal-class | Member[get] |
-| (semi-internal-class).getAnonymous | semi-internal-class | Member[getAnonymous] |
+| (semi-internal-class).get | (semi-internal-class) | Member[get] |
+| (semi-internal-class).getAnonymous | (semi-internal-class) | Member[getAnonymous] |
 | (semi-internal-class).getAnonymous~expr1 | (semi-internal-class).getAnonymous | ReturnValue |
 | (semi-internal-class).getAnonymous~expr2 | (semi-internal-class).getAnonymous~expr1 | Member[method] |
-| (subclass).A | subclass | Member[A] |
+| (subclass).A | (subclass) | Member[A] |
 | (subclass).A.prototype | (subclass).A | Instance |
 | (subclass).A.prototype | (subclass).B.prototype |  |
 | (subclass).A.prototype | (subclass).ExposedMidSubClass.prototype~expr1 |  |
 | (subclass).A.prototype.a | (subclass).A.prototype | Member[a] |
-| (subclass).B | subclass | Member[B] |
+| (subclass).B | (subclass) | Member[B] |
 | (subclass).B.prototype | (subclass).B | Instance |
 | (subclass).B.prototype | (subclass).C.prototype |  |
 | (subclass).B.prototype.b | (subclass).B.prototype | Member[b] |
-| (subclass).C | subclass | Member[C] |
+| (subclass).C | (subclass) | Member[C] |
 | (subclass).C.prototype | (subclass).C | Instance |
 | (subclass).C.prototype.c | (subclass).C.prototype | Member[c] |
-| (subclass).D | subclass | Member[D] |
+| (subclass).D | (subclass) | Member[D] |
 | (subclass).D.prototype | (subclass).D | Instance |
 | (subclass).D.prototype.d | (subclass).D.prototype | Member[d] |
-| (subclass).ExposedMidSubClass | subclass | Member[ExposedMidSubClass] |
+| (subclass).ExposedMidSubClass | (subclass) | Member[ExposedMidSubClass] |
 | (subclass).ExposedMidSubClass.prototype | (subclass).ExposedMidSubClass | Instance |
 | (subclass).ExposedMidSubClass.prototype.m | (subclass).ExposedMidSubClass.prototype | Member[m] |
 | (subclass).ExposedMidSubClass.prototype~expr1 | (subclass).ExposedMidSubClass.prototype |  |
+| upstream-lib | (reexport) | Member[lib] |
 | upstream-lib | (reexport).func | ReturnValue |
-| upstream-lib | reexport | Member[lib] |
 | upstream-lib.Type | (subclass).D.prototype |  |
-| upstream-lib.XYZ | reexport | Member[x].Member[y].Member[z] |
-| upstream-lib.XYZ | reexport | Member[xy].Member[z] |
+| upstream-lib.XYZ | (reexport) | Member[x].Member[y].Member[z] |
+| upstream-lib.XYZ | (reexport) | Member[xy].Member[z] |
 summaryModel
 | (aliases).Alias1.prototype |  |  | Member[foo].ReturnValue | type |
 | (return-this).FluentInterface.prototype |  |  | Member[bar].ReturnValue | type |

javascript/ql/test/library-tests/frameworks/data/test.expected

@@ -78,6 +78,7 @@ taintFlow
 | test.js:265:6:265:39 | new MyS ... ource() | test.js:265:6:265:39 | new MyS ... ource() |
 | test.js:269:10:269:31 | this.ba ... ource() | test.js:269:10:269:31 | this.ba ... ource() |
 | test.js:272:6:272:40 | new MyS ... ource() | test.js:272:6:272:40 | new MyS ... ource() |
+| test.js:274:6:274:39 | testlib ... eName() | test.js:274:6:274:39 | testlib ... eName() |
 isSink
 | test.js:54:18:54:25 | source() | test-sink |
 | test.js:55:22:55:29 | source() | test-sink |

javascript/ql/test/library-tests/frameworks/data/test.ext.yml

@@ -10,6 +10,7 @@ extensions:
       - ['testlib', 'Member[MethodDecorator].DecoratedMember.Parameter[0]', 'test-source']
       - ['testlib', 'Member[ParamDecoratorSource].DecoratedParameter', 'test-source']
       - ['testlib', 'Member[getSource].ReturnValue', 'test-source']
+      - ['(testlib)', 'Member[parenthesizedPackageName].ReturnValue', 'test-source']
 
   - addsTo:
       pack: codeql/javascript-all

javascript/ql/test/library-tests/frameworks/data/test.js

@@ -270,3 +270,5 @@ class MySubclass2 extends MySubclass {
   }
 }
 sink(new MySubclass2().baseclassSource()); // NOT OK
+
+sink(testlib.parenthesizedPackageName()); // NOT OK

misc/codegen/BUILD.bazel

@@ -1,5 +1,3 @@
-load("@codegen_deps//:requirements.bzl", "requirement")
-
 py_binary(
     name = "codegen",
     srcs = ["codegen.py"],

misc/codegen/generators/BUILD.bazel

@@ -1,5 +1,3 @@
-load("@codegen_deps//:requirements.bzl", "requirement")
-
 py_library(
     name = "generators",
     srcs = glob(["*.py"]),

python/extractor/tsg-python/tsp/BUILD.bazel

@@ -1,6 +1,6 @@
+load("@py_deps//:defs.bzl", "aliases", "all_crate_deps")
 load("@rules_rust//cargo:defs.bzl", "cargo_build_script")
 load("@rules_rust//rust:defs.bzl", "rust_library")
-load("@py_deps//:defs.bzl", "aliases", "all_crate_deps")
 
 package(default_visibility = ["//visibility:public"])
 

python/ql/lib/semmle/python/dataflow/new/SensitiveDataSources.qll

@@ -89,7 +89,7 @@ private module SensitiveDataModeling {
    */
   DataFlow::Node sensitiveLookupStringConst(SensitiveDataClassification classification) {
     // Note: If this is implemented with type-tracking, we will get cross-talk as
-    // illustrated in python/ql/test/experimental/dataflow/sensitive-data/test.py
+    // illustrated in python/ql/test/library-tests/dataflow/sensitive-data/test.py
     exists(DataFlow::LocalSourceNode source |
       source.asExpr().(StringLiteral).getText() = sensitiveString(classification) and
       source.flowsTo(result)

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPublic.qll

@@ -638,7 +638,7 @@ newtype TContent =
     //   name = any(AccessPathToken a).getAnArgument("Attribute")
     // instead we use a qltest to alert if we write a new summary in QL that uses an
     // attribute -- see
-    // python/ql/test/experimental/dataflow/summaries-checks/missing-attribute-content.ql
+    // python/ql/test/library-tests/dataflow/summaries-checks/missing-attribute-content.ql
     attr in ["re", "string", "pattern"]
     or
     //

python/ql/test/TestUtilities/dataflow/NormalDataflowTest.qll

@@ -1,6 +1,6 @@
 import python
-import experimental.dataflow.TestUtil.FlowTest
-import experimental.dataflow.testConfig
+import TestUtilities.dataflow.FlowTest
+import TestUtilities.dataflow.testConfig
 private import semmle.python.dataflow.new.internal.PrintNode
 
 module DataFlowTest implements FlowTestSig {

python/ql/test/TestUtilities/dataflow/NormalTaintTrackingTest.qll

@@ -1,6 +1,6 @@
 import python
-import experimental.dataflow.TestUtil.FlowTest
-import experimental.dataflow.testTaintConfig
+import TestUtilities.dataflow.FlowTest
+import TestUtilities.dataflow.testTaintConfig
 private import semmle.python.dataflow.new.internal.PrintNode
 
 module DataFlowTest implements FlowTestSig {

python/ql/test/experimental/dataflow/basic/localFlowStepTest.ql

@@ -1 +0,0 @@
-import experimental.dataflow.TestUtil.LocalFlowStepTest

python/ql/test/experimental/dataflow/basic/maximalFlowTest.ql

@@ -1 +0,0 @@
-import experimental.dataflow.TestUtil.MaximalFlowTest

python/ql/test/experimental/dataflow/coverage/NormalDataflowTest.ql

@@ -1,2 +0,0 @@
-import python
-import experimental.dataflow.TestUtil.NormalDataflowTest

python/ql/test/experimental/dataflow/exceptions/NormalDataflowTest.ql

@@ -1,2 +0,0 @@
-import python
-import experimental.dataflow.TestUtil.NormalDataflowTest

python/ql/test/experimental/dataflow/fieldflow/NormalDataflowTest.ql

@@ -1,2 +0,0 @@
-import python
-import experimental.dataflow.TestUtil.NormalDataflowTest

python/ql/test/experimental/dataflow/match/NormalDataflowTest.ql

@@ -1,2 +0,0 @@
-import python
-import experimental.dataflow.TestUtil.NormalDataflowTest

python/ql/test/experimental/dataflow/model-summaries/NormalDataflowTest.ql

@@ -1,2 +0,0 @@
-import python
-import experimental.dataflow.TestUtil.NormalDataflowTest

python/ql/test/experimental/dataflow/summaries/NormalTaintTrackingTest.ql

@@ -1,3 +0,0 @@
-import python
-private import TestSummaries
-import experimental.dataflow.TestUtil.NormalTaintTrackingTest

python/ql/test/experimental/dataflow/tainttracking/generator-flow/NormalDataflowTest.ql

@@ -1,2 +0,0 @@
-import python
-import experimental.dataflow.TestUtil.NormalDataflowTest

python/ql/test/experimental/meta/debug/dataflowTestPaths.ql

@@ -9,7 +9,7 @@
 // 3. if necessary, look at partial paths by (un)commenting appropriate lines
 import python
 import semmle.python.dataflow.new.DataFlow
-import experimental.dataflow.testConfig
+import TestUtilities.dataflow.testConfig
 
 module Config implements DataFlow::ConfigSig {
   predicate isSource(DataFlow::Node source) { TestConfig::isSource(source) }

python/ql/test/experimental/query-tests/Security/CWE-022-UnsafeUnpacking/DataflowQueryTest.ql

@@ -1,4 +1,4 @@
 import python
-import experimental.dataflow.TestUtil.DataflowQueryTest
+import TestUtilities.dataflow.DataflowQueryTest
 import experimental.Security.UnsafeUnpackQuery
 import FromTaintTrackingConfig<UnsafeUnpackConfig>

python/ql/test/experimental/query-tests/Security/CWE-409/DataflowQueryTest.ql

@@ -1,4 +1,4 @@
 import python
-import experimental.dataflow.TestUtil.DataflowQueryTest
+import TestUtilities.dataflow.DataflowQueryTest
 import experimental.semmle.python.security.DecompressionBomb
 import FromTaintTrackingConfig<BombsConfig>

python/ql/test/library-tests/dataflow/basic/callGraph.ql

@@ -1,4 +1,4 @@
-import experimental.dataflow.callGraphConfig
+import TestUtilities.dataflow.callGraphConfig
 
 from DataFlow::Node source, DataFlow::Node sink
 where

python/ql/test/library-tests/dataflow/basic/callGraphSinks.ql

@@ -1,4 +1,4 @@
-import experimental.dataflow.callGraphConfig
+import TestUtilities.dataflow.callGraphConfig
 
 from DataFlow::Node sink
 where

python/ql/test/library-tests/dataflow/basic/callGraphSources.ql

@@ -1,4 +1,4 @@
-import experimental.dataflow.callGraphConfig
+import TestUtilities.dataflow.callGraphConfig
 
 from DataFlow::Node source
 where