CI: add Windows support to fetch-codeql action

Python: Improve `BarrierGuard`

.github/actions/fetch-codeql/action.yml

@@ -13,6 +13,11 @@ runs:
       shell: bash
       run: echo "GA_CODEQL_CLI_PLATFORM=osx64" >> $GITHUB_ENV
 
+    - name: Select platform - Windows
+      if: runner.os == 'Windows'
+      shell: bash
+      run: echo "GA_CODEQL_CLI_PLATFORM=win64" >> $GITHUB_ENV
+
     - name: Fetch CodeQL
       shell: bash
       run: |

cpp/downgrades/23f7cbb88a4eb29f30c3490363dc201bc054c5ff/exprs.ql

@@ -0,0 +1,17 @@
+class Expr extends @expr {
+  string toString() { none() }
+}
+
+class Location extends @location_expr {
+  string toString() { none() }
+}
+
+predicate isExprWithNewBuiltin(Expr expr) {
+  exists(int kind | exprs(expr, kind, _) | 330 <= kind and kind <= 334)
+}
+
+from Expr expr, int kind, int kind_new, Location location
+where
+  exprs(expr, kind, location) and
+  if isExprWithNewBuiltin(expr) then kind_new = 0 else kind_new = kind
+select expr, kind_new, location

cpp/downgrades/23f7cbb88a4eb29f30c3490363dc201bc054c5ff/upgrade.properties

@@ -0,0 +1,3 @@
+description: Add new builtin operations
+compatibility: partial
+exprs.rel: run exprs.qlo

cpp/ql/lib/change-notes/2022-07-26-additional-builtin-support.md

@@ -0,0 +1,4 @@
+---
+category: feature
+---
+* Added subclasses of `BuiltInOperations` for `__builtin_bit_cast`, `__builtin_shuffle`, `__has_unique_object_representations`, `__is_aggregate`, and `__is_assignable`.

cpp/ql/lib/semmle/code/cpp/exprs/BuiltInOperations.qll

@@ -1,5 +1,5 @@
 /**
- * Provides classes for modeling built-in operations.  Built-in operations are
+ * Provides classes for modeling built-in operations. Built-in operations are
  * typically compiler specific and are used by libraries and generated code.
  */
 
@@ -120,8 +120,8 @@ class BuiltInNoOp extends BuiltInOperation, @noopexpr {
 
 /**
  * A C/C++ `__builtin_offsetof` built-in operation (used by some implementations
- * of `offsetof`).  The operation retains its semantics even in the presence
- * of an overloaded `operator &`).  This is a GNU/Clang extension.
+ * of `offsetof`). The operation retains its semantics even in the presence
+ * of an overloaded `operator &`). This is a gcc/clang extension.
  * ```
  * struct S {
  *   int a, b;
@@ -137,8 +137,8 @@ class BuiltInOperationBuiltInOffsetOf extends BuiltInOperation, @offsetofexpr {
 
 /**
  * A C/C++ `__INTADDR__` built-in operation (used by some implementations
- * of `offsetof`).  The operation retains its semantics even in the presence
- * of an overloaded `operator &`).  This is an EDG extension.
+ * of `offsetof`). The operation retains its semantics even in the presence
+ * of an overloaded `operator &`). This is an EDG extension.
  * ```
  * struct S {
  *   int a, b;
@@ -173,7 +173,7 @@ class BuiltInOperationHasAssign extends BuiltInOperation, @hasassignexpr {
  *
  * Returns `true` if the type has a copy constructor.
  * ```
- * std::integral_constant< bool, __has_copy(_Tp)> hc;
+ * std::integral_constant<bool, __has_copy(_Tp)> hc;
  * ```
  */
 class BuiltInOperationHasCopy extends BuiltInOperation, @hascopyexpr {
@@ -189,7 +189,7 @@ class BuiltInOperationHasCopy extends BuiltInOperation, @hascopyexpr {
  * Returns `true` if a copy assignment operator has an empty exception
  * specification.
  * ```
- * std::integral_constant< bool, __has_nothrow_assign(_Tp)> hnta;
+ * std::integral_constant<bool, __has_nothrow_assign(_Tp)> hnta;
  * ```
  */
 class BuiltInOperationHasNoThrowAssign extends BuiltInOperation, @hasnothrowassign {
@@ -220,7 +220,7 @@ class BuiltInOperationHasNoThrowConstructor extends BuiltInOperation, @hasnothro
  *
  * Returns `true` if the copy constructor has an empty exception specification.
  * ```
- * std::integral_constant< bool, __has_nothrow_copy(MyType) >;
+ * std::integral_constant<bool, __has_nothrow_copy(MyType) >;
  * ```
  */
 class BuiltInOperationHasNoThrowCopy extends BuiltInOperation, @hasnothrowcopy {
@@ -266,7 +266,7 @@ class BuiltInOperationHasTrivialConstructor extends BuiltInOperation, @hastrivia
  *
  * Returns true if the type has a trivial copy constructor.
  * ```
- * std::integral_constant< bool, __has_trivial_copy(MyType) > htc;
+ * std::integral_constant<bool, __has_trivial_copy(MyType)> htc;
  * ```
  */
 class BuiltInOperationHasTrivialCopy extends BuiltInOperation, @hastrivialcopy {
@@ -468,7 +468,7 @@ class BuiltInOperationIsUnion extends BuiltInOperation, @isunionexpr {
  * ```
  * template<typename _Tp1, typename _Tp2>
  *   struct types_compatible
- *   : public integral_constant<bool, __builtin_types_compatible_p(_Tp1, _Tp2) >
+ *   : public integral_constant<bool, __builtin_types_compatible_p(_Tp1, _Tp2)>
  *   { };
  * ```
  */
@@ -479,8 +479,7 @@ class BuiltInOperationBuiltInTypesCompatibleP extends BuiltInOperation, @typesco
 /**
  * A clang `__builtin_shufflevector` expression.
  *
- * It outputs a permutation of elements from one or two input vectors.
- * Please see
+ * It outputs a permutation of elements from one or two input vectors. See
  * https://releases.llvm.org/3.7.0/tools/clang/docs/LanguageExtensions.html#langext-builtin-shufflevector
  * for more information.
  * ```
@@ -494,11 +493,29 @@ class BuiltInOperationBuiltInShuffleVector extends BuiltInOperation, @builtinshu
   override string getAPrimaryQlClass() { result = "BuiltInOperationBuiltInShuffleVector" }
 }
 
+/**
+ * A gcc `__builtin_shuffle` expression.
+ *
+ * It outputs a permutation of elements from one or two input vectors.
+ * See https://gcc.gnu.org/onlinedocs/gcc/Vector-Extensions.html
+ * for more information.
+ * ```
+ * // Concatenate every other element of 4-element vectors V1 and V2.
+ * M = {0, 2, 4, 6};
+ * V3 = __builtin_shuffle(V1, V2, M);
+ * ```
+ */
+class BuiltInOperationBuiltInShuffle extends BuiltInOperation, @builtinshuffle {
+  override string toString() { result = "__builtin_shuffle" }
+
+  override string getAPrimaryQlClass() { result = "BuiltInOperationBuiltInShuffle" }
+}
+
 /**
  * A clang `__builtin_convertvector` expression.
  *
  * Allows for conversion of vectors of equal element count and compatible
- * element types. Please see
+ * element types. See
  * https://releases.llvm.org/3.7.0/tools/clang/docs/LanguageExtensions.html#builtin-convertvector
  * for more information.
  * ```
@@ -547,7 +564,7 @@ class BuiltInOperationBuiltInAddressOf extends UnaryOperation, BuiltInOperation,
  * ```
  * template<typename T, typename... Args>
  *   struct is_trivially_constructible
- *   : public integral_constant<bool, __is_trivially_constructible(T, Args...) >
+ *   : public integral_constant<bool, __is_trivially_constructible(T, Args...)>
  *   { };
  * ```
  */
@@ -612,13 +629,10 @@ class BuiltInOperationIsTriviallyDestructible extends BuiltInOperation, @istrivi
  * The `__is_trivially_assignable` built-in operation (used by some
  * implementations of the `<type_traits>` header).
  *
- * Returns `true` if the assignment operator `C::operator =(const C& c)` is
- * trivial.
+ * Returns `true` if the assignment operator `C::operator =(const D& d)` is
+ * trivial (i.e., it will not call any operation that is non-trivial).
  * ```
- * template<typename T>
- *   struct is_trivially_assignable
- *   : public integral_constant<bool, __is_trivially_assignable(T) >
- *   { };
+ * bool v = __is_trivially_assignable(MyType1, MyType2);
  * ```
  */
 class BuiltInOperationIsTriviallyAssignable extends BuiltInOperation, @istriviallyassignableexpr {
@@ -631,10 +645,10 @@ class BuiltInOperationIsTriviallyAssignable extends BuiltInOperation, @istrivial
  * The `__is_nothrow_assignable` built-in operation (used by some
  * implementations of the `<type_traits>` header).
  *
- * Returns true if there exists a `C::operator =(const C& c) nothrow`
+ * Returns true if there exists a `C::operator =(const D& d) nothrow`
  * assignment operator (i.e, with an empty exception specification).
  * ```
- * bool v = __is_nothrow_assignable(MyType);
+ * bool v = __is_nothrow_assignable(MyType1, MyType2);
  * ```
  */
 class BuiltInOperationIsNothrowAssignable extends BuiltInOperation, @isnothrowassignableexpr {
@@ -643,15 +657,30 @@ class BuiltInOperationIsNothrowAssignable extends BuiltInOperation, @isnothrowas
   override string getAPrimaryQlClass() { result = "BuiltInOperationIsNothrowAssignable" }
 }
 
+/**
+ * The `__is_assignable` built-in operation (used by some implementations
+ * of the `<type_traits>` header).
+ *
+ * Returns true if there exists a `C::operator =(const D& d)` assignment
+ * operator.
+ * ```
+ * bool v = __is_assignable(MyType1, MyType2);
+ * ```
+ */
+class BuiltInOperationIsAssignable extends BuiltInOperation, @isassignable {
+  override string toString() { result = "__is_assignable" }
+
+  override string getAPrimaryQlClass() { result = "BuiltInOperationIsAssignable" }
+}
+
 /**
  * The `__is_standard_layout` built-in operation (used by some implementations
  * of the `<type_traits>` header).
  *
  * Returns `true` if the type is a primitive type, or a `class`, `struct` or
- * `union` WITHOUT (1) virtual functions or base classes, (2) reference member
- * variable or (3) multiple occurrences of base `class` objects, among other
- * restrictions.  Please see
- * https://en.cppreference.com/w/cpp/named_req/StandardLayoutType
+ * `union` without (1) virtual functions or base classes, (2) reference member
+ * variable, or (3) multiple occurrences of base `class` objects, among other
+ * restrictions. See https://en.cppreference.com/w/cpp/named_req/StandardLayoutType
  * for more information.
  * ```
  * bool v = __is_standard_layout(MyType);
@@ -668,7 +697,7 @@ class BuiltInOperationIsStandardLayout extends BuiltInOperation, @isstandardlayo
  * implementations of the `<type_traits>` header).
  *
  * Returns `true` if instances of this type can be copied by trivial
- * means.  The copying is done in a manner similar to the `memcpy`
+ * means. The copying is done in a manner similar to the `memcpy`
  * function.
  */
 class BuiltInOperationIsTriviallyCopyable extends BuiltInOperation, @istriviallycopyableexpr {
@@ -682,13 +711,13 @@ class BuiltInOperationIsTriviallyCopyable extends BuiltInOperation, @istrivially
  * the `<type_traits>` header).
  *
  * Returns `true` if the type is a scalar type, a reference type or an array of
- * literal types, among others. Please see
+ * literal types, among others. See
  * https://en.cppreference.com/w/cpp/named_req/LiteralType
  * for more information.
  *
  * ```
  * template <typename _Tp>
- * std::integral_constant< bool, __is_literal_type(_Tp)> ilt;
+ * std::integral_constant<bool, __is_literal_type(_Tp)> ilt;
  * ```
  */
 class BuiltInOperationIsLiteralType extends BuiltInOperation, @isliteraltypeexpr {
@@ -705,7 +734,7 @@ class BuiltInOperationIsLiteralType extends BuiltInOperation, @isliteraltypeexpr
  * compiler, with semantics of the `memcpy` operation.
  * ```
  * template <typename _Tp>
- * std::integral_constant< bool, __has_trivial_move_constructor(_Tp)> htmc;
+ * std::integral_constant<bool, __has_trivial_move_constructor(_Tp)> htmc;
  * ```
  */
 class BuiltInOperationHasTrivialMoveConstructor extends BuiltInOperation,
@@ -723,7 +752,7 @@ class BuiltInOperationHasTrivialMoveConstructor extends BuiltInOperation,
  * ```
  * template<typename T>
  *   struct has_trivial_move_assign
- *   : public integral_constant<bool, __has_trivial_move_assign(T) >
+ *   : public integral_constant<bool, __has_trivial_move_assign(T)>
  *   { };
  * ```
  */
@@ -758,7 +787,7 @@ class BuiltInOperationHasNothrowMoveAssign extends BuiltInOperation, @hasnothrow
  * ```
  * template<typename T, typename... Args>
  *   struct is_constructible
- *   : public integral_constant<bool, __is_constructible(T, Args...) >
+ *   : public integral_constant<bool, __is_constructible(T, Args...)>
  *   { };
  * ```
  */
@@ -785,7 +814,7 @@ class BuiltInOperationIsNothrowConstructible extends BuiltInOperation, @isnothro
 }
 
 /**
- * The `__has_finalizer` built-in operation.  This is a Microsoft extension.
+ * The `__has_finalizer` built-in operation. This is a Microsoft extension.
  *
  * Returns `true` if the type defines a _finalizer_ `C::!C(void)`, to be called
  * from either the regular destructor or the garbage collector.
@@ -800,10 +829,10 @@ class BuiltInOperationHasFinalizer extends BuiltInOperation, @hasfinalizerexpr {
 }
 
 /**
- * The `__is_delegate` built-in operation.  This is a Microsoft extension.
+ * The `__is_delegate` built-in operation. This is a Microsoft extension.
  *
  * Returns `true` if the function has been declared as a `delegate`, used in
- * message forwarding.  Please see
+ * message forwarding. See
  * https://docs.microsoft.com/en-us/cpp/extensions/delegate-cpp-component-extensions
  * for more information.
  */
@@ -814,9 +843,9 @@ class BuiltInOperationIsDelegate extends BuiltInOperation, @isdelegateexpr {
 }
 
 /**
- * The `__is_interface_class` built-in operation.  This is a Microsoft extension.
+ * The `__is_interface_class` built-in operation. This is a Microsoft extension.
  *
- * Returns `true` if the type has been declared as an `interface`.  Please see
+ * Returns `true` if the type has been declared as an `interface`. See
  * https://docs.microsoft.com/en-us/cpp/extensions/interface-class-cpp-component-extensions
  * for more information.
  */
@@ -827,9 +856,9 @@ class BuiltInOperationIsInterfaceClass extends BuiltInOperation, @isinterfacecla
 }
 
 /**
- * The `__is_ref_array` built-in operation.  This is a Microsoft extension.
+ * The `__is_ref_array` built-in operation. This is a Microsoft extension.
  *
- * Returns `true` if the object passed in is a _platform array_.  Please see
+ * Returns `true` if the object passed in is a _platform array_. See
  * https://docs.microsoft.com/en-us/cpp/extensions/arrays-cpp-component-extensions
  * for more information.
  * ```
@@ -844,9 +873,9 @@ class BuiltInOperationIsRefArray extends BuiltInOperation, @isrefarrayexpr {
 }
 
 /**
- * The `__is_ref_class` built-in operation.  This is a Microsoft extension.
+ * The `__is_ref_class` built-in operation. This is a Microsoft extension.
  *
- * Returns `true` if the type is a _reference class_.  Please see
+ * Returns `true` if the type is a _reference class_. See
  * https://docs.microsoft.com/en-us/cpp/extensions/classes-and-structs-cpp-component-extensions
  * for more information.
  * ```
@@ -861,10 +890,10 @@ class BuiltInOperationIsRefClass extends BuiltInOperation, @isrefclassexpr {
 }
 
 /**
- * The `__is_sealed` built-in operation.  This is a Microsoft extension.
+ * The `__is_sealed` built-in operation. This is a Microsoft extension.
  *
  * Returns `true` if a given class or virtual function is marked as `sealed`,
- * meaning that it cannot be extended or overridden.  The `sealed` keyword
+ * meaning that it cannot be extended or overridden. The `sealed` keyword
  * is similar to the C++11 `final` keyword.
  * ```
  * ref class X sealed {
@@ -879,7 +908,7 @@ class BuiltInOperationIsSealed extends BuiltInOperation, @issealedexpr {
 }
 
 /**
- * The `__is_simple_value_class` built-in operation.  This is a Microsoft extension.
+ * The `__is_simple_value_class` built-in operation. This is a Microsoft extension.
  *
  * Returns `true` if passed a value type that contains no references to the
  * garbage-collected heap.
@@ -898,9 +927,9 @@ class BuiltInOperationIsSimpleValueClass extends BuiltInOperation, @issimplevalu
 }
 
 /**
- * The `__is_value_class` built-in operation.  This is a Microsoft extension.
+ * The `__is_value_class` built-in operation. This is a Microsoft extension.
  *
- * Returns `true` if passed a value type.  Please see
+ * Returns `true` if passed a value type. See
  * https://docs.microsoft.com/en-us/cpp/extensions/classes-and-structs-cpp-component-extensions
  * For more information.
  * ```
@@ -922,7 +951,7 @@ class BuiltInOperationIsValueClass extends BuiltInOperation, @isvalueclassexpr {
  * ```
  * template<typename T>
  *   struct is_final
- *   : public integral_constant<bool, __is_final(T) >
+ *   : public integral_constant<bool, __is_final(T)>
  *   { };
  * ```
  */
@@ -933,7 +962,7 @@ class BuiltInOperationIsFinal extends BuiltInOperation, @isfinalexpr {
 }
 
 /**
- * The `__builtin_choose_expr` expression.  This is a GNU/Clang extension.
+ * The `__builtin_choose_expr` expression. This is a gcc/clang extension.
  *
  * The expression functions similarly to the ternary `?:` operator, except
  * that it is evaluated at compile-time.
@@ -978,3 +1007,50 @@ class BuiltInComplexOperation extends BuiltInOperation, @builtincomplex {
   /** Gets the operand corresponding to the imaginary part of the complex number. */
   Expr getImaginaryOperand() { this.hasChild(result, 1) }
 }
+
+/**
+ * A C++ `__is_aggregate` built-in operation (used by some implementations of the
+ * `<type_traits>` header).
+ *
+ * Returns `true` if the type has is an aggregate type.
+ * ```
+ * std::integral_constant<bool, __is_aggregate(_Tp)> ia;
+ * ```
+ */
+class BuiltInOperationIsAggregate extends BuiltInOperation, @isaggregate {
+  override string toString() { result = "__is_aggregate" }
+
+  override string getAPrimaryQlClass() { result = "BuiltInOperationIsAggregate" }
+}
+
+/**
+ * A C++ `__has_unique_object_representations` built-in operation (used by some
+ * implementations of the `<type_traits>` header).
+ *
+ * Returns `true` if the type is trivially copyable and if the object representation
+ * is unique for two objects with the same value.
+ * ```
+ * bool v = __has_unique_object_representations(MyType);
+ * ```
+ */
+class BuiltInOperationHasUniqueObjectRepresentations extends BuiltInOperation,
+  @hasuniqueobjectrepresentations {
+  override string toString() { result = "__has_unique_object_representations" }
+
+  override string getAPrimaryQlClass() { result = "BuiltInOperationHasUniqueObjectRepresentations" }
+}
+
+/**
+ * A C/C++ `__builtin_bit_cast` built-in operation (used by some implementations
+ * of `std::bit_cast`).
+ *
+ * Performs a bit cast from a value to a type.
+ * ```
+ * __builtin_bit_cast(Type, value);
+ * ```
+ */
+class BuiltInBitCast extends BuiltInOperation, @builtinbitcast {
+  override string toString() { result = "__builtin_bit_cast" }
+
+  override string getAPrimaryQlClass() { result = "BuiltInBitCast" }
+}

cpp/ql/lib/semmle/code/cpp/exprs/Expr.qll

@@ -596,9 +596,12 @@ class ParenthesisExpr extends Conversion, @parexpr {
 }
 
 /**
- * A C/C++ expression that has not been resolved.
+ * A C/C++ expression that could not be resolved, or that can no longer be
+ * represented due to a database upgrade or downgrade.
  *
- * It is assigned `ErroneousType` as its type.
+ * If the expression could not be resolved, it has type `ErroneousType`. In the
+ * case of a database upgrade or downgrade, the original type from before the
+ * upgrade or downgrade is kept if that type can be represented.
  */
 class ErrorExpr extends Expr, @errorexpr {
   override string toString() { result = "<error expr>" }

cpp/ql/lib/semmlecode.cpp.dbscheme

@@ -1650,6 +1650,11 @@ case @expr.kind of
 | 327 = @co_await
 | 328 = @co_yield
 | 329 = @temp_init
+| 330 = @isassignable
+| 331 = @isaggregate
+| 332 = @hasuniqueobjectrepresentations
+| 333 = @builtinbitcast
+| 334 = @builtinshuffle
 ;
 
 @var_args_expr = @vastartexpr
@@ -1711,6 +1716,11 @@ case @expr.kind of
             | @isfinalexpr
             | @builtinchooseexpr
             | @builtincomplex
+            | @isassignable
+            | @isaggregate
+            | @hasuniqueobjectrepresentations
+            | @builtinbitcast
+            | @builtinshuffle
             ;
 
 new_allocated_type(

cpp/ql/lib/upgrades/19e31bf071f588bb7efd1e4d5a185ce4f6fbbd84/upgrade.properties

@@ -0,0 +1,2 @@
+description: Add new builtin operations
+compatibility: backwards

cpp/ql/test/library-tests/builtins/edg/edg.c

@@ -1,4 +1,4 @@
-
+// semmle-extractor-options: --clang
 struct mystruct {
     int f1;
     int f2;
@@ -13,3 +13,6 @@ void f(void) {
     int i2 = edg_offsetof(struct mystruct,f2);
 }
 
+void g(void) {
+    double f = __builtin_bit_cast(double,42l);
+}

cpp/ql/test/library-tests/builtins/edg/expr.expected

@@ -13,3 +13,6 @@
 | edg.c:13:14:13:45 | (size_t)... | 0 | 0 |
 | edg.c:13:14:13:45 | __INTADDR__ | 1 | 1 |
 | edg.c:13:43:13:44 | f2 | 0 | 0 |
+| edg.c:17:16:17:45 | __builtin_bit_cast | 1 | 1 |
+| edg.c:17:16:17:45 | double | 0 | 0 |
+| edg.c:17:42:17:44 | 42 | 1 | 1 |

cpp/ql/test/library-tests/builtins/type_traits/expr.expected

@@ -296,3 +296,20 @@
 | ms.cpp:255:24:255:43 | a_struct |  | <none> |
 | ms.cpp:256:24:256:49 | __is_final | a_final_struct | 1 |
 | ms.cpp:256:24:256:49 | a_final_struct |  | <none> |
+| ms.cpp:258:29:258:62 | __is_assignable | a_struct,a_struct | 1 |
+| ms.cpp:258:29:258:62 | a_struct |  | <none> |
+| ms.cpp:258:29:258:62 | a_struct |  | <none> |
+| ms.cpp:259:29:259:59 | __is_assignable | a_struct,empty | 0 |
+| ms.cpp:259:29:259:59 | a_struct |  | <none> |
+| ms.cpp:259:29:259:59 | empty |  | <none> |
+| ms.cpp:260:29:260:57 | __is_assignable | a_struct,int | 0 |
+| ms.cpp:260:29:260:57 | a_struct |  | <none> |
+| ms.cpp:260:29:260:57 | int |  | <none> |
+| ms.cpp:262:28:262:51 | __is_aggregate | a_struct | 1 |
+| ms.cpp:262:28:262:51 | a_struct |  | <none> |
+| ms.cpp:263:28:263:46 | __is_aggregate | int | 0 |
+| ms.cpp:263:28:263:46 | int |  | <none> |
+| ms.cpp:265:49:265:88 | __has_unique_object_representations | int | 1 |
+| ms.cpp:265:49:265:88 | int |  | <none> |
+| ms.cpp:266:49:266:90 | __has_unique_object_representations | float | 0 |
+| ms.cpp:266:49:266:90 | float |  | <none> |

cpp/ql/test/library-tests/builtins/type_traits/ms.cpp

@@ -254,5 +254,14 @@ void f(void) {
 
     bool b_is_final1 = __is_final(a_struct);
     bool b_is_final2 = __is_final(a_final_struct);
-}
 
+    bool b_is_assignable1 = __is_assignable(a_struct,a_struct);
+    bool b_is_assignable2 = __is_assignable(a_struct,empty);
+    bool b_is_assignable3 = __is_assignable(a_struct,int);
+
+    bool b_is_aggregate1 = __is_aggregate(a_struct);
+    bool b_is_aggregate2 = __is_aggregate(int);
+
+    bool b_has_unique_object_representations1 = __has_unique_object_representations(int);
+    bool b_has_unique_object_representations2 = __has_unique_object_representations(float);
+}

cpp/ql/test/library-tests/vector_types/builtin_ops.expected

@@ -1,2 +1,4 @@
+| vector_types2.cpp:10:15:10:42 | __builtin_shuffle |
+| vector_types2.cpp:11:15:11:45 | __builtin_shuffle |
 | vector_types.cpp:31:13:31:49 | __builtin_shufflevector |
 | vector_types.cpp:58:10:58:52 | __builtin_convertvector |

cpp/ql/test/library-tests/vector_types/variables.expected

@@ -13,6 +13,12 @@
 | file://:0:0:0:0 | gp_offset | gp_offset | file://:0:0:0:0 | unsigned int | 4 |
 | file://:0:0:0:0 | overflow_arg_area | overflow_arg_area | file://:0:0:0:0 | void * | 8 |
 | file://:0:0:0:0 | reg_save_area | reg_save_area | file://:0:0:0:0 | void * | 8 |
+| vector_types2.cpp:5:7:5:7 | a | a | vector_types2.cpp:2:13:2:15 | v4i | 16 |
+| vector_types2.cpp:6:7:6:7 | b | b | vector_types2.cpp:2:13:2:15 | v4i | 16 |
+| vector_types2.cpp:7:7:7:12 | mask_1 | mask_1 | vector_types2.cpp:2:13:2:15 | v4i | 16 |
+| vector_types2.cpp:8:7:8:12 | mask_2 | mask_2 | vector_types2.cpp:2:13:2:15 | v4i | 16 |
+| vector_types2.cpp:10:7:10:11 | res_1 | res_1 | vector_types2.cpp:2:13:2:15 | v4i | 16 |
+| vector_types2.cpp:11:7:11:11 | res_2 | res_2 | vector_types2.cpp:2:13:2:15 | v4i | 16 |
 | vector_types.cpp:9:21:9:21 | x | x | vector_types.cpp:6:15:6:17 | v4f | 16 |
 | vector_types.cpp:14:18:14:20 | lhs | lhs | vector_types.cpp:6:15:6:17 | v4f | 16 |
 | vector_types.cpp:14:27:14:29 | rhs | rhs | vector_types.cpp:6:15:6:17 | v4f | 16 |

cpp/ql/test/library-tests/vector_types/vector_types2.cpp

@@ -0,0 +1,12 @@
+// semmle-extractor-options: --gnu --gnu_version 80000
+typedef int v4i __attribute__((vector_size (16)));
+
+void f() {
+  v4i a = {1,2,3,4};
+  v4i b = {5,6,7,8};
+  v4i mask_1 = {3,0,1,2};
+  v4i mask_2 = {3,5,4,2};
+
+  v4i res_1 = __builtin_shuffle(a, mask_1);
+  v4i res_2 = __builtin_shuffle(a, b, mask_2);
+}

csharp/ql/lib/semmle/code/csharp/controlflow/internal/ControlFlowGraphImplShared.qll

@@ -881,7 +881,12 @@ import Cached
  * graph is restricted to nodes from `RelevantNode`.
  */
 module TestOutput {
-  abstract class RelevantNode extends Node { }
+  abstract class RelevantNode extends Node {
+    /**
+     * Gets a string used to resolve ties in node and edge ordering.
+     */
+    string getOrderDisambuigation() { result = "" }
+  }
 
   query predicate nodes(RelevantNode n, string attr, string val) {
     attr = "semmle.order" and
@@ -894,7 +899,8 @@ module TestOutput {
             p
             order by
               l.getFile().getBaseName(), l.getFile().getAbsolutePath(), l.getStartLine(),
-              l.getStartColumn(), l.getEndLine(), l.getEndColumn(), p.toString()
+              l.getStartColumn(), l.getEndLine(), l.getEndColumn(), p.toString(),
+              p.getOrderDisambuigation()
           )
       ).toString()
   }
@@ -916,7 +922,8 @@ module TestOutput {
             s
             order by
               l.getFile().getBaseName(), l.getFile().getAbsolutePath(), l.getStartLine(),
-              l.getStartColumn(), l.getEndLine(), l.getEndColumn(), t.toString()
+              l.getStartColumn(), l.getEndLine(), l.getEndColumn(), t.toString(), s.toString(),
+              s.getOrderDisambuigation()
           )
       ).toString()
   }

java/documentation/library-coverage/coverage.csv

@@ -36,7 +36,7 @@ java.lang,13,,58,,,,,,,,,,,8,,,,,4,,,1,,,,,,,,,,,,,,,46,12
 java.net,10,3,7,,,,,,,,,,,,,,10,,,,,,,,,,,,,,,,,,,3,7,
 java.nio,15,,6,,13,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,6,
 java.sql,11,,,,,,,,,4,,,,,,,,,,,,,,,,7,,,,,,,,,,,,
-java.util,44,,441,,,,,,,,,,,34,,,,,,5,2,,1,2,,,,,,,,,,,,,24,417
+java.util,44,,458,,,,,,,,,,,34,,,,,,5,2,,1,2,,,,,,,,,,,,,36,422
 javax.faces.context,2,7,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,,7,,
 javax.jms,,9,57,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,9,57,
 javax.json,,,123,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,100,23

java/documentation/library-coverage/coverage.rst

@@ -15,9 +15,9 @@ Java framework & library support
    `Apache HttpComponents <https://hc.apache.org/>`_,"``org.apache.hc.core5.*``, ``org.apache.http``",5,136,28,,,3,,,,25
    `Google Guava <https://guava.dev/>`_,``com.google.common.*``,,728,39,,6,,,,,
    `JSON-java <https://github.com/stleary/JSON-java>`_,``org.json``,,236,,,,,,,,
-   Java Standard Library,``java.*``,3,552,130,28,,,7,,,10
+   Java Standard Library,``java.*``,3,569,130,28,,,7,,,10
    Java extensions,"``javax.*``, ``jakarta.*``",63,609,32,,,4,,1,1,2
    `Spring <https://spring.io/>`_,``org.springframework.*``,29,476,101,,,,19,14,,29
    Others,"``androidx.slice``, ``cn.hutool.core.codec``, ``com.esotericsoftware.kryo.io``, ``com.esotericsoftware.kryo5.io``, ``com.fasterxml.jackson.core``, ``com.fasterxml.jackson.databind``, ``com.opensymphony.xwork2.ognl``, ``com.rabbitmq.client``, ``com.unboundid.ldap.sdk``, ``com.zaxxer.hikari``, ``flexjson``, ``groovy.lang``, ``groovy.util``, ``jodd.json``, ``kotlin.jvm.internal``, ``net.sf.saxon.s9api``, ``ognl``, ``okhttp3``, ``org.apache.commons.codec``, ``org.apache.commons.jexl2``, ``org.apache.commons.jexl3``, ``org.apache.commons.logging``, ``org.apache.commons.ognl``, ``org.apache.directory.ldap.client.api``, ``org.apache.ibatis.jdbc``, ``org.apache.log4j``, ``org.apache.logging.log4j``, ``org.apache.shiro.codec``, ``org.apache.shiro.jndi``, ``org.codehaus.groovy.control``, ``org.dom4j``, ``org.hibernate``, ``org.jboss.logging``, ``org.jdbi.v3.core``, ``org.jooq``, ``org.mvel2``, ``org.scijava.log``, ``org.slf4j``, ``org.xml.sax``, ``org.xmlpull.v1``, ``play.mvc``, ``ratpack.core.form``, ``ratpack.core.handling``, ``ratpack.core.http``, ``ratpack.exec``, ``ratpack.form``, ``ratpack.func``, ``ratpack.handling``, ``ratpack.http``, ``ratpack.util``, ``retrofit2``",65,395,932,,,,14,18,,3
-   Totals,,217,6413,1474,117,6,10,107,33,1,84
+   Totals,,217,6430,1474,117,6,10,107,33,1,84
 

java/kotlin-extractor/src/main/kotlin/KotlinFileExtractor.kt

@@ -888,6 +888,10 @@ open class KotlinFileExtractor(
                     if (shortName.nameInDB != shortName.kotlinName) {
                         tw.writeKtFunctionOriginalNames(methodId, shortName.kotlinName)
                     }
+
+                    if (f.hasInterfaceParent() && f.body != null) {
+                        addModifiers(id, "default") // The actual output class file may or may not have this modifier, depending on the -Xjvm-default setting.
+                    }
                 }
 
                 tw.writeHasLocation(id, locId)
@@ -1386,7 +1390,8 @@ open class KotlinFileExtractor(
         dispatchReceiver: IrExpression?,
         extensionReceiver: IrExpression?,
         typeArguments: List<IrType> = listOf(),
-        extractClassTypeArguments: Boolean = false) {
+        extractClassTypeArguments: Boolean = false,
+        superQualifierSymbol: IrClassSymbol? = null) {
 
         val locId = tw.getLocation(callsite)
 
@@ -1404,7 +1409,8 @@ open class KotlinFileExtractor(
             dispatchReceiver?.let { { callId -> extractExpressionExpr(dispatchReceiver, enclosingCallable, callId, -1, enclosingStmt) } },
             extensionReceiver?.let { { argParent -> extractExpressionExpr(extensionReceiver, enclosingCallable, argParent, 0, enclosingStmt) } },
             typeArguments,
-            extractClassTypeArguments
+            extractClassTypeArguments,
+            superQualifierSymbol
         )
 
     }
@@ -1424,7 +1430,8 @@ open class KotlinFileExtractor(
         extractDispatchReceiver: ((Label<out DbExpr>) -> Unit)?,
         extractExtensionReceiver: ((Label<out DbExpr>) -> Unit)?,
         typeArguments: List<IrType> = listOf(),
-        extractClassTypeArguments: Boolean = false) {
+        extractClassTypeArguments: Boolean = false,
+        superQualifierSymbol: IrClassSymbol? = null) {
 
         val callTarget = syntacticCallTarget.target.realOverrideTarget
         val id = tw.getFreshIdLabel<DbMethodaccess>()
@@ -1483,6 +1490,8 @@ open class KotlinFileExtractor(
 
             if (callTarget.shouldExtractAsStatic) {
                 extractStaticTypeAccessQualifier(callTarget, id, locId, enclosingCallable, enclosingStmt)
+            } else if (superQualifierSymbol != null) {
+                extractSuperAccess(superQualifierSymbol.typeWith(), enclosingCallable, id, -1, enclosingStmt, locId)
             } else if (extractDispatchReceiver != null) {
                 extractDispatchReceiver(id)
             }
@@ -1744,7 +1753,7 @@ open class KotlinFileExtractor(
                     else
                         listOf()
 
-                extractRawMethodAccess(syntacticCallTarget, c, callable, parent, idx, enclosingStmt, (0 until c.valueArgumentsCount).map { c.getValueArgument(it) }, c.dispatchReceiver, c.extensionReceiver, typeArgs, extractClassTypeArguments)
+                extractRawMethodAccess(syntacticCallTarget, c, callable, parent, idx, enclosingStmt, (0 until c.valueArgumentsCount).map { c.getValueArgument(it) }, c.dispatchReceiver, c.extensionReceiver, typeArgs, extractClassTypeArguments, c.superQualifierSymbol)
             }
 
             fun extractSpecialEnumFunction(fnName: String){
@@ -3066,6 +3075,17 @@ open class KotlinFileExtractor(
         }
     }
 
+    private fun extractSuperAccess(irType: IrType, callable: Label<out DbCallable>, parent: Label<out DbExprparent>, idx: Int, enclosingStmt: Label<out DbStmt>, locId: Label<out DbLocation>) =
+        tw.getFreshIdLabel<DbSuperaccess>().also {
+            val type = useType(irType)
+            tw.writeExprs_superaccess(it, type.javaResult.id, parent, idx)
+            tw.writeExprsKotlinType(it, type.kotlinResult.id)
+            tw.writeHasLocation(it, locId)
+            tw.writeCallableEnclosingExpr(it, callable)
+            tw.writeStatementEnclosingExpr(it, enclosingStmt)
+            extractTypeAccessRecursive(irType, locId, it, 0)
+        }
+
     private fun extractThisAccess(e: IrGetValue, exprParent: ExprParent, callable: Label<out DbCallable>) {
         val containingDeclaration = declarationStack.peek()
         val locId = tw.getLocation(e)
@@ -4020,7 +4040,7 @@ open class KotlinFileExtractor(
     /**
      * Extracts a single wildcard type access expression with no enclosing callable and statement.
      */
-    private fun extractWildcardTypeAccess(type: TypeResults, location: Label<DbLocation>, parent: Label<out DbExprparent>, idx: Int): Label<out DbExpr> {
+    private fun extractWildcardTypeAccess(type: TypeResults, location: Label<out DbLocation>, parent: Label<out DbExprparent>, idx: Int): Label<out DbExpr> {
         val id = tw.getFreshIdLabel<DbWildcardtypeaccess>()
         tw.writeExprs_wildcardtypeaccess(id, type.javaResult.id, parent, idx)
         tw.writeExprsKotlinType(id, type.kotlinResult.id)
@@ -4031,7 +4051,7 @@ open class KotlinFileExtractor(
     /**
      * Extracts a single type access expression with no enclosing callable and statement.
      */
-    private fun extractTypeAccess(type: TypeResults, location: Label<DbLocation>, parent: Label<out DbExprparent>, idx: Int): Label<out DbExpr> {
+    private fun extractTypeAccess(type: TypeResults, location: Label<out DbLocation>, parent: Label<out DbExprparent>, idx: Int): Label<out DbExpr> {
         // TODO: elementForLocation allows us to give some sort of
         //   location, but a proper location for the type access will
         //   require upstream changes
@@ -4057,7 +4077,7 @@ open class KotlinFileExtractor(
      * `extractTypeAccessRecursive` if the argument is invariant.
      * No enclosing callable and statement is extracted, this is useful for type access extraction in field declarations.
      */
-    private fun extractWildcardTypeAccessRecursive(t: IrTypeArgument, location: Label<DbLocation>, parent: Label<out DbExprparent>, idx: Int) {
+    private fun extractWildcardTypeAccessRecursive(t: IrTypeArgument, location: Label<out DbLocation>, parent: Label<out DbExprparent>, idx: Int) {
         val typeLabels by lazy { TypeResults(getTypeArgumentLabel(t), TypeResult(fakeKotlinType(), "TODO", "TODO")) }
         when (t) {
             is IrStarProjection -> extractWildcardTypeAccess(typeLabels, location, parent, idx)
@@ -4077,7 +4097,7 @@ open class KotlinFileExtractor(
      * Extracts a type access expression and its child type access expressions in case of a generic type. Nested generics are also handled.
      * No enclosing callable and statement is extracted, this is useful for type access extraction in field declarations.
      */
-    private fun extractTypeAccessRecursive(t: IrType, location: Label<DbLocation>, parent: Label<out DbExprparent>, idx: Int, typeContext: TypeContext = TypeContext.OTHER): Label<out DbExpr> {
+    private fun extractTypeAccessRecursive(t: IrType, location: Label<out DbLocation>, parent: Label<out DbExprparent>, idx: Int, typeContext: TypeContext = TypeContext.OTHER): Label<out DbExpr> {
         val typeAccessId = extractTypeAccess(useType(t, typeContext), location, parent, idx)
         if (t is IrSimpleType) {
             t.arguments.forEachIndexed { argIdx, arg ->

java/ql/consistency-queries/typeParametersInScope.ql

@@ -12,6 +12,8 @@ Type getAMentionedType(RefType type) {
   result = getAMentionedType(type).(InstantiatedType).getATypeArgument()
   or
   result = getAMentionedType(type).(NestedType).getEnclosingType()
+  or
+  result = getAMentionedType(type).(Wildcard).getATypeBound().getType()
 }
 
 Type getATypeUsedInClass(RefType type) {

java/ql/lib/change-notes/2022-05-12-get-underlying-expr.md

@@ -0,0 +1,4 @@
+---
+category: feature
+---
+* The QL predicate `Expr::getUnderlyingExpr` has been added. It can be used to look through casts and not-null expressions and obtain the underlying expression to which they apply.

java/ql/lib/change-notes/2022-07-27-nullness-junit5.md

@@ -0,0 +1,5 @@
+---
+category: minorAnalysis
+---
+* The JUnit5 version of `AssertNotNull` is now recognized, which removes
+  related false positives in the nullness queries.

java/ql/lib/semmle/code/java/Expr.qll

@@ -100,6 +100,18 @@ class Expr extends ExprParent, @expr {
 
   /** Holds if this expression is parenthesized. */
   predicate isParenthesized() { isParenthesized(this, _) }
+
+  /**
+   * Gets the underlying expression looking through casts and not-nulls, if any.
+   * Otherwise just gets this expression.
+   */
+  Expr getUnderlyingExpr() {
+    if this instanceof CastingExpr or this instanceof NotNullExpr
+    then
+      result = this.(CastingExpr).getExpr().getUnderlyingExpr() or
+      result = this.(NotNullExpr).getExpr().getUnderlyingExpr()
+    else result = this
+  }
 }
 
 /**

java/ql/lib/semmle/code/java/frameworks/Assertions.qll

@@ -2,7 +2,8 @@
  * A library providing uniform access to various assertion frameworks.
  *
  * Currently supports `org.junit.Assert`, `junit.framework.*`,
- * `com.google.common.base.Preconditions`, and `java.util.Objects`.
+ * `org.junit.jupiter.api.Assertions`, `com.google.common.base.Preconditions`,
+ * and `java.util.Objects`.
  */
 
 import java
@@ -17,7 +18,11 @@ private newtype AssertKind =
 private predicate assertionMethod(Method m, AssertKind kind) {
   exists(RefType junit |
     m.getDeclaringType() = junit and
-    (junit.hasQualifiedName("org.junit", "Assert") or junit.hasQualifiedName("junit.framework", _))
+    (
+      junit.hasQualifiedName("org.junit", "Assert") or
+      junit.hasQualifiedName("junit.framework", _) or
+      junit.hasQualifiedName("org.junit.jupiter.api", "Assertions")
+    )
   |
     m.hasName("assertNotNull") and kind = AssertKindNotNull()
     or

java/ql/lib/semmle/code/java/security/CleartextStorageSharedPrefsQuery.qll

@@ -51,7 +51,7 @@ private predicate sharedPreferencesInput(DataFlow::Node editor, Expr input) {
   exists(MethodAccess m |
     m.getMethod() instanceof PutSharedPreferenceMethod and
     input = m.getArgument(1) and
-    editor.asExpr() = m.getQualifier()
+    editor.asExpr() = m.getQualifier().getUnderlyingExpr()
   )
 }
 
@@ -61,7 +61,7 @@ private predicate sharedPreferencesInput(DataFlow::Node editor, Expr input) {
  */
 private predicate sharedPreferencesStore(DataFlow::Node editor, MethodAccess m) {
   m.getMethod() instanceof StoreSharedPreferenceMethod and
-  editor.asExpr() = m.getQualifier()
+  editor.asExpr() = m.getQualifier().getUnderlyingExpr()
 }
 
 /** Flow from `SharedPreferences.Editor` to either a setter or a store method. */

java/ql/lib/semmle/code/java/security/UnsafeAndroidAccess.qll

@@ -75,6 +75,8 @@ private predicate webViewLoadUrl(Argument urlArg, WebViewRef webview) {
     loadUrl.getArgument(0) = urlArg and
     loadUrl.getMethod() instanceof WebViewLoadUrlMethod
   |
+    webview.getAnAccess() = DataFlow::exprNode(loadUrl.getQualifier().getUnderlyingExpr())
+    or
     webview.getAnAccess() = DataFlow::getInstanceArgument(loadUrl)
     or
     // `webview` is received as a parameter of an event method in a custom `WebViewClient`,
@@ -82,8 +84,9 @@ private predicate webViewLoadUrl(Argument urlArg, WebViewRef webview) {
     exists(WebViewClientEventMethod eventMethod, MethodAccess setWebClient |
       setWebClient.getMethod() instanceof WebViewSetWebViewClientMethod and
       setWebClient.getArgument(0).getType() = eventMethod.getDeclaringType() and
-      loadUrl.getQualifier() = eventMethod.getWebViewParameter().getAnAccess()
+      loadUrl.getQualifier().getUnderlyingExpr() = eventMethod.getWebViewParameter().getAnAccess()
     |
+      webview.getAnAccess() = DataFlow::exprNode(setWebClient.getQualifier().getUnderlyingExpr()) or
       webview.getAnAccess() = DataFlow::getInstanceArgument(setWebClient)
     )
   )

java/ql/test/TestUtilities/InlineExpectationsTestPrivate.qll

@@ -20,3 +20,15 @@ private class KtExpectationComment extends KtComment, ExpectationComment {
 
   override string getContents() { result = this.getText().suffix(2).trim() }
 }
+
+private class XmlExpectationComment extends ExpectationComment instanceof XMLComment {
+  override string getContents() { result = this.(XMLComment).getText().trim() }
+
+  override Location getLocation() { result = this.(XMLComment).getLocation() }
+
+  override predicate hasLocationInfo(string path, int sl, int sc, int el, int ec) {
+    this.(XMLComment).hasLocationInfo(path, sl, sc, el, ec)
+  }
+
+  override string toString() { result = this.(XMLComment).toString() }
+}

java/ql/test/kotlin/library-tests/generic-instance-methods/test.expected

@@ -14,8 +14,8 @@ calls
 | test.kt:22:15:22:33 | setter(...) | test.kt:12:1:25:1 | user | test.kt:0:0:0:0 | TestKt | file:///!unknown-binary-location/Generic.class:0:0:0:0 | setter | file:///!unknown-binary-location/Generic.class:0:0:0:0 | Generic<? super String> |
 | test.kt:23:15:23:22 | getter(...) | test.kt:12:1:25:1 | user | test.kt:0:0:0:0 | TestKt | file:///!unknown-binary-location/Generic.class:0:0:0:0 | getter | file:///!unknown-binary-location/Generic.class:0:0:0:0 | Generic<? super String> |
 constructors
-| Generic2.class:0:0:0:0 | Generic2<? extends String> | Generic2.class:0:0:0:0 | Generic2<? extends String> | Generic2<? extends String>(java.lang.String) | ? extends String | void | Test.java:1:7:1:14 | Generic2 | Test.java:3:10:3:17 | Generic2 |
-| Generic2.class:0:0:0:0 | Generic2<? super String> | Generic2.class:0:0:0:0 | Generic2<? super String> | Generic2<? super String>(java.lang.Object) | ? super String | void | Test.java:1:7:1:14 | Generic2 | Test.java:3:10:3:17 | Generic2 |
+| Generic2.class:0:0:0:0 | Generic2<? extends String> | Generic2.class:0:0:0:0 | Generic2<? extends String> | Generic2<? extends String>(<nulltype>) | <nulltype> | void | Test.java:1:7:1:14 | Generic2 | Test.java:3:10:3:17 | Generic2 |
+| Generic2.class:0:0:0:0 | Generic2<? super String> | Generic2.class:0:0:0:0 | Generic2<? super String> | Generic2<? super String>(java.lang.String) | String | void | Test.java:1:7:1:14 | Generic2 | Test.java:3:10:3:17 | Generic2 |
 | Generic2.class:0:0:0:0 | Generic2<String> | Generic2.class:0:0:0:0 | Generic2<String> | Generic2<String>(java.lang.String) | String | void | Test.java:1:7:1:14 | Generic2 | Test.java:3:10:3:17 | Generic2 |
 | Test.java:1:7:1:14 | Generic2 | Test.java:3:10:3:17 | Generic2 | Generic2(java.lang.Object) | T | void | Test.java:1:7:1:14 | Generic2 | Test.java:3:10:3:17 | Generic2 |
 | Test.java:14:14:14:17 | Test | Test.java:14:14:14:17 | Test | Test() | No parameters | void | Test.java:14:14:14:17 | Test | Test.java:14:14:14:17 | Test |
@@ -34,14 +34,14 @@ refTypes
 | test.kt:1:1:10:1 | Generic |
 | test.kt:1:15:1:15 | T |
 #select
-| Generic2.class:0:0:0:0 | Generic2<? extends String> | Generic2.class:0:0:0:0 | getter | getter() | No parameters | ? extends String | Test.java:1:7:1:14 | Generic2 | Test.java:9:5:9:10 | getter |
-| Generic2.class:0:0:0:0 | Generic2<? extends String> | Generic2.class:0:0:0:0 | identity | identity(java.lang.String) | ? extends String | ? extends String | Test.java:1:7:1:14 | Generic2 | Test.java:8:5:8:12 | identity |
-| Generic2.class:0:0:0:0 | Generic2<? extends String> | Generic2.class:0:0:0:0 | identity2 | identity2(java.lang.String) | ? extends String | ? extends String | Test.java:1:7:1:14 | Generic2 | Test.java:7:5:7:13 | identity2 |
-| Generic2.class:0:0:0:0 | Generic2<? extends String> | Generic2.class:0:0:0:0 | setter | setter(java.lang.String) | ? extends String | void | Test.java:1:7:1:14 | Generic2 | Test.java:10:8:10:13 | setter |
-| Generic2.class:0:0:0:0 | Generic2<? super String> | Generic2.class:0:0:0:0 | getter | getter() | No parameters | ? super String | Test.java:1:7:1:14 | Generic2 | Test.java:9:5:9:10 | getter |
-| Generic2.class:0:0:0:0 | Generic2<? super String> | Generic2.class:0:0:0:0 | identity | identity(java.lang.Object) | ? super String | ? super String | Test.java:1:7:1:14 | Generic2 | Test.java:8:5:8:12 | identity |
-| Generic2.class:0:0:0:0 | Generic2<? super String> | Generic2.class:0:0:0:0 | identity2 | identity2(java.lang.Object) | ? super String | ? super String | Test.java:1:7:1:14 | Generic2 | Test.java:7:5:7:13 | identity2 |
-| Generic2.class:0:0:0:0 | Generic2<? super String> | Generic2.class:0:0:0:0 | setter | setter(java.lang.Object) | ? super String | void | Test.java:1:7:1:14 | Generic2 | Test.java:10:8:10:13 | setter |
+| Generic2.class:0:0:0:0 | Generic2<? extends String> | Generic2.class:0:0:0:0 | getter | getter() | No parameters | String | Test.java:1:7:1:14 | Generic2 | Test.java:9:5:9:10 | getter |
+| Generic2.class:0:0:0:0 | Generic2<? extends String> | Generic2.class:0:0:0:0 | identity | identity(<nulltype>) | <nulltype> | String | Test.java:1:7:1:14 | Generic2 | Test.java:8:5:8:12 | identity |
+| Generic2.class:0:0:0:0 | Generic2<? extends String> | Generic2.class:0:0:0:0 | identity2 | identity2(<nulltype>) | <nulltype> | String | Test.java:1:7:1:14 | Generic2 | Test.java:7:5:7:13 | identity2 |
+| Generic2.class:0:0:0:0 | Generic2<? extends String> | Generic2.class:0:0:0:0 | setter | setter(<nulltype>) | <nulltype> | void | Test.java:1:7:1:14 | Generic2 | Test.java:10:8:10:13 | setter |
+| Generic2.class:0:0:0:0 | Generic2<? super String> | Generic2.class:0:0:0:0 | getter | getter() | No parameters | Object | Test.java:1:7:1:14 | Generic2 | Test.java:9:5:9:10 | getter |
+| Generic2.class:0:0:0:0 | Generic2<? super String> | Generic2.class:0:0:0:0 | identity | identity(java.lang.String) | String | Object | Test.java:1:7:1:14 | Generic2 | Test.java:8:5:8:12 | identity |
+| Generic2.class:0:0:0:0 | Generic2<? super String> | Generic2.class:0:0:0:0 | identity2 | identity2(java.lang.String) | String | Object | Test.java:1:7:1:14 | Generic2 | Test.java:7:5:7:13 | identity2 |
+| Generic2.class:0:0:0:0 | Generic2<? super String> | Generic2.class:0:0:0:0 | setter | setter(java.lang.String) | String | void | Test.java:1:7:1:14 | Generic2 | Test.java:10:8:10:13 | setter |
 | Generic2.class:0:0:0:0 | Generic2<String> | Generic2.class:0:0:0:0 | getter | getter() | No parameters | String | Test.java:1:7:1:14 | Generic2 | Test.java:9:5:9:10 | getter |
 | Generic2.class:0:0:0:0 | Generic2<String> | Generic2.class:0:0:0:0 | identity | identity(java.lang.String) | String | String | Test.java:1:7:1:14 | Generic2 | Test.java:8:5:8:12 | identity |
 | Generic2.class:0:0:0:0 | Generic2<String> | Generic2.class:0:0:0:0 | identity2 | identity2(java.lang.String) | String | String | Test.java:1:7:1:14 | Generic2 | Test.java:7:5:7:13 | identity2 |

java/ql/test/kotlin/library-tests/super-method-calls/test.expected

@@ -0,0 +1,2 @@
+| test.kt:31:17:31:24 | source(...) | test.kt:31:15:31:25 | f(...) |
+| test.kt:32:17:32:24 | source(...) | test.kt:32:15:32:25 | g(...) |

java/ql/test/kotlin/library-tests/super-method-calls/test.kt

@@ -0,0 +1,36 @@
+open class A {
+
+  open fun f(x: String) = x
+
+}
+
+interface B {
+
+  fun g(x: String) = x
+
+}
+
+interface C {
+
+  fun g(x: String) = x
+
+}
+
+class User : A(), B, C {
+
+  override fun f(x: String) = super.f(x)
+
+  override fun g(x: String) = super<B>.g(x)
+
+  fun source() = "tainted"
+
+  fun sink(s: String) { }
+
+  fun test() {
+
+    sink(this.f(source()))
+    sink(this.g(source()))
+
+  }
+
+}

java/ql/test/kotlin/library-tests/super-method-calls/test.ql

@@ -0,0 +1,18 @@
+import java
+import semmle.code.java.dataflow.DataFlow
+
+class Config extends DataFlow::Configuration {
+  Config() { this = "abc" }
+
+  override predicate isSource(DataFlow::Node n) {
+    n.asExpr().(MethodAccess).getMethod().getName() = "source"
+  }
+
+  override predicate isSink(DataFlow::Node n) {
+    n.asExpr().(Argument).getCall().getCallee().getName() = "sink"
+  }
+}
+
+from Config c, DataFlow::Node n1, DataFlow::Node n2
+where c.hasFlow(n1, n2)
+select n1, n2

java/ql/test/library-tests/wildcard-substitution/Lib.java

@@ -0,0 +1,20 @@
+import java.util.List;
+
+public class Lib<T> {
+
+  public void takesVar(T t) { }
+  public void takesInvar(List<T> lt) { }
+  public void takesUnbound(List<?> lt) { }
+  public void takesExtends(List<? extends T> lt) { }
+  public void takesSuper(List<? super T> lt) { }
+
+  public T returnsVar() { return null; } 
+  public List<T> returnsInvar() { return null; } 
+  public List<?> returnsUnbound() { return null; } 
+  public List<? extends T> returnsExtends() { return null; } 
+  public List<? super T> returnsSuper() { return null; }
+
+  public void takesArray(T[] ts) { }
+  public T[] returnsArray() { return null; }
+
+}

java/ql/test/library-tests/wildcard-substitution/User.java

@@ -0,0 +1,12 @@
+public class User {
+
+  public static void test(Lib<CharSequence> invarLib, Lib<? extends CharSequence> extendsLib, Lib<? super CharSequence> superLib, Lib<?> unboundLib) {
+
+    invarLib.takesVar(null);
+    extendsLib.takesVar(null);
+    superLib.takesVar(null);
+    unboundLib.takesVar(null);
+
+  }
+
+}

java/ql/test/library-tests/wildcard-substitution/test.expected

@@ -0,0 +1,64 @@
+| Lib.class:0:0:0:0 | Lib<? extends CharSequence> | Lib.class:0:0:0:0 | returnsArray | CharSequence[] |
+| Lib.class:0:0:0:0 | Lib<? extends CharSequence> | Lib.class:0:0:0:0 | returnsExtends | List<? extends CharSequence> |
+| Lib.class:0:0:0:0 | Lib<? extends CharSequence> | Lib.class:0:0:0:0 | returnsInvar | List<? extends CharSequence> |
+| Lib.class:0:0:0:0 | Lib<? extends CharSequence> | Lib.class:0:0:0:0 | returnsSuper | List<?> |
+| Lib.class:0:0:0:0 | Lib<? extends CharSequence> | Lib.class:0:0:0:0 | returnsUnbound | List<?> |
+| Lib.class:0:0:0:0 | Lib<? extends CharSequence> | Lib.class:0:0:0:0 | returnsVar | CharSequence |
+| Lib.class:0:0:0:0 | Lib<? extends CharSequence> | Lib.class:0:0:0:0 | takesArray | <nulltype>[] |
+| Lib.class:0:0:0:0 | Lib<? extends CharSequence> | Lib.class:0:0:0:0 | takesExtends | List<? extends CharSequence> |
+| Lib.class:0:0:0:0 | Lib<? extends CharSequence> | Lib.class:0:0:0:0 | takesInvar | List<? extends CharSequence> |
+| Lib.class:0:0:0:0 | Lib<? extends CharSequence> | Lib.class:0:0:0:0 | takesSuper | List<?> |
+| Lib.class:0:0:0:0 | Lib<? extends CharSequence> | Lib.class:0:0:0:0 | takesUnbound | List<?> |
+| Lib.class:0:0:0:0 | Lib<? extends CharSequence> | Lib.class:0:0:0:0 | takesVar | <nulltype> |
+| Lib.class:0:0:0:0 | Lib<? super CharSequence> | Lib.class:0:0:0:0 | returnsArray | Object[] |
+| Lib.class:0:0:0:0 | Lib<? super CharSequence> | Lib.class:0:0:0:0 | returnsExtends | List<?> |
+| Lib.class:0:0:0:0 | Lib<? super CharSequence> | Lib.class:0:0:0:0 | returnsInvar | List<? super CharSequence> |
+| Lib.class:0:0:0:0 | Lib<? super CharSequence> | Lib.class:0:0:0:0 | returnsSuper | List<? super CharSequence> |
+| Lib.class:0:0:0:0 | Lib<? super CharSequence> | Lib.class:0:0:0:0 | returnsUnbound | List<?> |
+| Lib.class:0:0:0:0 | Lib<? super CharSequence> | Lib.class:0:0:0:0 | returnsVar | Object |
+| Lib.class:0:0:0:0 | Lib<? super CharSequence> | Lib.class:0:0:0:0 | takesArray | CharSequence[] |
+| Lib.class:0:0:0:0 | Lib<? super CharSequence> | Lib.class:0:0:0:0 | takesExtends | List<?> |
+| Lib.class:0:0:0:0 | Lib<? super CharSequence> | Lib.class:0:0:0:0 | takesInvar | List<? super CharSequence> |
+| Lib.class:0:0:0:0 | Lib<? super CharSequence> | Lib.class:0:0:0:0 | takesSuper | List<? super CharSequence> |
+| Lib.class:0:0:0:0 | Lib<? super CharSequence> | Lib.class:0:0:0:0 | takesUnbound | List<?> |
+| Lib.class:0:0:0:0 | Lib<? super CharSequence> | Lib.class:0:0:0:0 | takesVar | CharSequence |
+| Lib.class:0:0:0:0 | Lib<?> | Lib.class:0:0:0:0 | returnsArray | Object[] |
+| Lib.class:0:0:0:0 | Lib<?> | Lib.class:0:0:0:0 | returnsExtends | List<? extends Object> |
+| Lib.class:0:0:0:0 | Lib<?> | Lib.class:0:0:0:0 | returnsInvar | List<?> |
+| Lib.class:0:0:0:0 | Lib<?> | Lib.class:0:0:0:0 | returnsSuper | List<?> |
+| Lib.class:0:0:0:0 | Lib<?> | Lib.class:0:0:0:0 | returnsUnbound | List<?> |
+| Lib.class:0:0:0:0 | Lib<?> | Lib.class:0:0:0:0 | returnsVar | Object |
+| Lib.class:0:0:0:0 | Lib<?> | Lib.class:0:0:0:0 | takesArray | <nulltype>[] |
+| Lib.class:0:0:0:0 | Lib<?> | Lib.class:0:0:0:0 | takesExtends | List<? extends Object> |
+| Lib.class:0:0:0:0 | Lib<?> | Lib.class:0:0:0:0 | takesInvar | List<?> |
+| Lib.class:0:0:0:0 | Lib<?> | Lib.class:0:0:0:0 | takesSuper | List<?> |
+| Lib.class:0:0:0:0 | Lib<?> | Lib.class:0:0:0:0 | takesUnbound | List<?> |
+| Lib.class:0:0:0:0 | Lib<?> | Lib.class:0:0:0:0 | takesVar | <nulltype> |
+| Lib.class:0:0:0:0 | Lib<CharSequence> | Lib.class:0:0:0:0 | returnsArray | CharSequence[] |
+| Lib.class:0:0:0:0 | Lib<CharSequence> | Lib.class:0:0:0:0 | returnsExtends | List<? extends CharSequence> |
+| Lib.class:0:0:0:0 | Lib<CharSequence> | Lib.class:0:0:0:0 | returnsInvar | List<CharSequence> |
+| Lib.class:0:0:0:0 | Lib<CharSequence> | Lib.class:0:0:0:0 | returnsSuper | List<? super CharSequence> |
+| Lib.class:0:0:0:0 | Lib<CharSequence> | Lib.class:0:0:0:0 | returnsUnbound | List<?> |
+| Lib.class:0:0:0:0 | Lib<CharSequence> | Lib.class:0:0:0:0 | returnsVar | CharSequence |
+| Lib.class:0:0:0:0 | Lib<CharSequence> | Lib.class:0:0:0:0 | takesArray | CharSequence[] |
+| Lib.class:0:0:0:0 | Lib<CharSequence> | Lib.class:0:0:0:0 | takesExtends | List<? extends CharSequence> |
+| Lib.class:0:0:0:0 | Lib<CharSequence> | Lib.class:0:0:0:0 | takesInvar | List<CharSequence> |
+| Lib.class:0:0:0:0 | Lib<CharSequence> | Lib.class:0:0:0:0 | takesSuper | List<? super CharSequence> |
+| Lib.class:0:0:0:0 | Lib<CharSequence> | Lib.class:0:0:0:0 | takesUnbound | List<?> |
+| Lib.class:0:0:0:0 | Lib<CharSequence> | Lib.class:0:0:0:0 | takesVar | CharSequence |
+| Lib.java:3:14:3:16 | Lib | Lib.java:5:15:5:22 | takesVar | T |
+| Lib.java:3:14:3:16 | Lib | Lib.java:6:15:6:24 | takesInvar | List<T> |
+| Lib.java:3:14:3:16 | Lib | Lib.java:7:15:7:26 | takesUnbound | List<?> |
+| Lib.java:3:14:3:16 | Lib | Lib.java:8:15:8:26 | takesExtends | List<? extends T> |
+| Lib.java:3:14:3:16 | Lib | Lib.java:9:15:9:24 | takesSuper | List<? super T> |
+| Lib.java:3:14:3:16 | Lib | Lib.java:11:12:11:21 | returnsVar | T |
+| Lib.java:3:14:3:16 | Lib | Lib.java:12:18:12:29 | returnsInvar | List<T> |
+| Lib.java:3:14:3:16 | Lib | Lib.java:13:18:13:31 | returnsUnbound | List<?> |
+| Lib.java:3:14:3:16 | Lib | Lib.java:14:28:14:41 | returnsExtends | List<? extends T> |
+| Lib.java:3:14:3:16 | Lib | Lib.java:15:26:15:37 | returnsSuper | List<? super T> |
+| Lib.java:3:14:3:16 | Lib | Lib.java:17:15:17:24 | takesArray | T[] |
+| Lib.java:3:14:3:16 | Lib | Lib.java:18:14:18:25 | returnsArray | T[] |
+| User.java:1:14:1:17 | User | User.java:3:22:3:25 | test | Lib<? extends CharSequence> |
+| User.java:1:14:1:17 | User | User.java:3:22:3:25 | test | Lib<? super CharSequence> |
+| User.java:1:14:1:17 | User | User.java:3:22:3:25 | test | Lib<?> |
+| User.java:1:14:1:17 | User | User.java:3:22:3:25 | test | Lib<CharSequence> |

java/ql/test/library-tests/wildcard-substitution/test.ql

@@ -0,0 +1,7 @@
+import java
+
+Type notVoid(Type t) { result = t and not result instanceof VoidType }
+
+from Callable c
+where c.getSourceDeclaration().fromSource()
+select c.getDeclaringType(), c, notVoid([c.getAParamType(), c.getReturnType()]).toString()

java/ql/test/library-tests/xml/Test.java

@@ -0,0 +1,3 @@
+public class Test {
+
+}

java/ql/test/library-tests/xml/XMLTest.expected

@@ -0,0 +1,2 @@
+| test.xml:4:5:4:32 | attribute=value | Unexpected result: hasXmlResult= |
+| test.xml:5:29:5:52 |  $ hasXmlResult  | Missing result:hasXmlResult= |

java/ql/test/library-tests/xml/XMLTest.ql

@@ -0,0 +1,17 @@
+import semmle.code.xml.XML
+import TestUtilities.InlineExpectationsTest
+
+class XmlTest extends InlineExpectationsTest {
+  XmlTest() { this = "XmlTest" }
+
+  override string getARelevantTag() { result = "hasXmlResult" }
+
+  override predicate hasActualResult(Location location, string element, string tag, string value) {
+    tag = "hasXmlResult" and
+    exists(XMLAttribute a |
+      a.getLocation() = location and
+      element = a.toString() and
+      value = ""
+    )
+  }
+}

java/ql/test/library-tests/xml/test.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<document>
+    <element attribute="value">Text</element> <!-- $ hasXmlResult -->
+    <element attribute="value">Text</element> <!-- Missing -->
+    <element>Text</element> <!-- $ hasXmlResult --> <!-- Spurious -->
+</document>

java/ql/test/query-tests/security/CWE-312/CleartextStorageSharedPrefsTestKt.kt

@@ -0,0 +1,11 @@
+import android.app.Activity
+import android.content.Context
+import android.content.SharedPreferences
+
+class CleartextStorageSharedPrefsTestKt : Activity() {
+    fun testSetSharedPrefs1(context: Context, name: String, password: String) {
+		val sharedPrefs = context.getSharedPreferences("user_prefs", Context.MODE_PRIVATE);
+		sharedPrefs.edit().putString("name", name).apply(); // Safe
+		sharedPrefs.edit().putString("password", password).apply(); // $ hasCleartextStorageSharedPrefs
+	}
+}

java/ql/test/query-tests/security/CWE-312/options

@@ -1 +1,2 @@
 // semmle-extractor-options: --javac-args -cp ${testdir}/../../../stubs/google-android-9.0.0
+// codeql-extractor-kotlin-options: ${testdir}/../../../stubs/google-android-9.0.0

java/ql/test/query-tests/security/CWE-749/AndroidManifest.xml

@@ -44,6 +44,7 @@
 
         <activity android:name=".UnsafeActivity3" android:exported="true" />
         <activity android:name=".UnsafeActivity4" android:exported="true" />
+        <activity android:name=".UnsafeActivityKt" android:exported="true" />
 
         <receiver android:name=".UnsafeAndroidBroadcastReceiver" android:exported="true" />        
     </application>

java/ql/test/query-tests/security/CWE-749/UnsafeActivityKt.kt

@@ -0,0 +1,20 @@
+package com.example.app
+
+import android.app.Activity
+import android.os.Bundle
+import android.webkit.WebSettings
+import android.webkit.WebView
+import android.webkit.WebViewClient
+
+class UnsafeActivityKt : Activity() {
+    override fun onCreate(savedInstanceState : Bundle) {
+
+		val wv = findViewById<WebView>(-1)
+		// Implicit not-nulls happening here
+		wv.settings.setJavaScriptEnabled(true)
+		wv.settings.setAllowFileAccessFromFileURLs(true)
+
+		val thisUrl : String = intent.extras.getString("url")
+		wv.loadUrl(thisUrl) // $ hasUnsafeAndroidAccess
+    }
+}

java/ql/test/query-tests/security/CWE-749/options

@@ -1 +1,2 @@
-//semmle-extractor-options: --javac-args -cp ${testdir}/../../../stubs/android
+//semmle-extractor-options: --javac-args -cp ${testdir}/../../../stubs/google-android-9.0.0
+//codeql-extractor-kotlin-options: ${testdir}/../../../stubs/google-android-9.0.0

javascript/ql/experimental/adaptivethreatmodeling/src/qlpack.yml

@@ -8,4 +8,4 @@ groups:
     - experimental
 dependencies:
     codeql/javascript-experimental-atm-lib: "*"
-    codeql/javascript-experimental-atm-model: 0.2.0
+    codeql/javascript-experimental-atm-model: "0.2.0"

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPublic.qll

@@ -527,16 +527,55 @@ class StarPatternElementNode extends Node, TStarPatternElementNode {
   override Location getLocation() { result = consumer.getLocation() }
 }
 
+/**
+ * Gets a node that controls whether other nodes are evaluated.
+ *
+ * In the base case, this is the last node of `conditionBlock`, and `flipped` is `false`.
+ * This definition accounts for (short circuting) `and`- and `or`-expressions, as the structure
+ * of basic blocks will reflect their semantics.
+ *
+ * However, in the program
+ * ```python
+ * if not is_safe(path):
+ *   return
+ * ```
+ * the last node in the `ConditionBlock` is `not is_safe(path)`.
+ *
+ * We would like to consider also `is_safe(path)` a guard node, albeit with `flipped` being `true`.
+ * Thus we recurse through `not`-expressions.
+ */
+ControlFlowNode guardNode(ConditionBlock conditionBlock, boolean flipped) {
+  // Base case: the last node truly does determine which successor is chosen
+  result = conditionBlock.getLastNode() and
+  flipped = false
+  or
+  // Recursive case: if a guard node is a `not`-expression,
+  // the operand is also a guard node, but with inverted polarity.
+  exists(UnaryExprNode notNode |
+    result = notNode.getOperand() and
+    notNode.getNode().getOp() instanceof Not
+  |
+    notNode = guardNode(conditionBlock, flipped.booleanNot())
+  )
+}
+
 /**
  * A node that controls whether other nodes are evaluated.
+ *
+ * The field `flipped` allows us to match `GuardNode`s underneath
+ * `not`-expressions and still choose the appropriate branch.
  */
 class GuardNode extends ControlFlowNode {
   ConditionBlock conditionBlock;
+  boolean flipped;
 
-  GuardNode() { this = conditionBlock.getLastNode() }
+  GuardNode() { this = guardNode(conditionBlock, flipped) }
 
   /** Holds if this guard controls block `b` upon evaluating to `branch`. */
-  predicate controlsBlock(BasicBlock b, boolean branch) { conditionBlock.controls(b, branch) }
+  predicate controlsBlock(BasicBlock b, boolean branch) {
+    branch in [true, false] and
+    conditionBlock.controls(b, branch.booleanXor(flipped))
+  }
 }
 
 /**

python/ql/test/experimental/dataflow/tainttracking/commonSanitizer/test_string_const_compare.py

@@ -50,7 +50,7 @@ def test_non_eq2():
     if not ts == "safe":
         ensure_tainted(ts) # $ tainted
     else:
-        ensure_not_tainted(ts) # $ SPURIOUS: tainted
+        ensure_not_tainted(ts)
 
 
 def test_in_list():
@@ -157,7 +157,7 @@ def test_not_in2():
     if not ts in ["safe", "also_safe"]:
         ensure_tainted(ts) # $ tainted
     else:
-        ensure_not_tainted(ts) # $ SPURIOUS: tainted
+        ensure_not_tainted(ts)
 
 
 def is_safe(x):

python/ql/test/experimental/dataflow/tainttracking/customSanitizer/InlineTaintTest.expected

@@ -6,12 +6,20 @@ isSanitizer
 | TestTaintTrackingConfiguration | test.py:34:39:34:39 | ControlFlowNode for s |
 | TestTaintTrackingConfiguration | test.py:52:28:52:28 | ControlFlowNode for s |
 | TestTaintTrackingConfiguration | test.py:66:10:66:29 | ControlFlowNode for emulated_escaping() |
-| TestTaintTrackingConfiguration | test_logical.py:30:28:30:28 | ControlFlowNode for s |
-| TestTaintTrackingConfiguration | test_logical.py:45:28:45:28 | ControlFlowNode for s |
-| TestTaintTrackingConfiguration | test_logical.py:50:28:50:28 | ControlFlowNode for s |
-| TestTaintTrackingConfiguration | test_logical.py:89:28:89:28 | ControlFlowNode for s |
-| TestTaintTrackingConfiguration | test_logical.py:100:28:100:28 | ControlFlowNode for s |
-| TestTaintTrackingConfiguration | test_logical.py:145:28:145:28 | ControlFlowNode for s |
+| TestTaintTrackingConfiguration | test_logical.py:33:28:33:28 | ControlFlowNode for s |
+| TestTaintTrackingConfiguration | test_logical.py:40:28:40:28 | ControlFlowNode for s |
+| TestTaintTrackingConfiguration | test_logical.py:48:28:48:28 | ControlFlowNode for s |
+| TestTaintTrackingConfiguration | test_logical.py:53:28:53:28 | ControlFlowNode for s |
+| TestTaintTrackingConfiguration | test_logical.py:92:28:92:28 | ControlFlowNode for s |
+| TestTaintTrackingConfiguration | test_logical.py:103:28:103:28 | ControlFlowNode for s |
+| TestTaintTrackingConfiguration | test_logical.py:111:28:111:28 | ControlFlowNode for s |
+| TestTaintTrackingConfiguration | test_logical.py:130:28:130:28 | ControlFlowNode for s |
+| TestTaintTrackingConfiguration | test_logical.py:137:28:137:28 | ControlFlowNode for s |
 | TestTaintTrackingConfiguration | test_logical.py:148:28:148:28 | ControlFlowNode for s |
-| TestTaintTrackingConfiguration | test_logical.py:155:28:155:28 | ControlFlowNode for s |
+| TestTaintTrackingConfiguration | test_logical.py:151:28:151:28 | ControlFlowNode for s |
+| TestTaintTrackingConfiguration | test_logical.py:158:28:158:28 | ControlFlowNode for s |
+| TestTaintTrackingConfiguration | test_logical.py:167:24:167:24 | ControlFlowNode for s |
+| TestTaintTrackingConfiguration | test_logical.py:176:24:176:24 | ControlFlowNode for s |
+| TestTaintTrackingConfiguration | test_logical.py:185:24:185:24 | ControlFlowNode for s |
+| TestTaintTrackingConfiguration | test_logical.py:193:24:193:24 | ControlFlowNode for s |
 | TestTaintTrackingConfiguration | test_reference.py:31:28:31:28 | ControlFlowNode for s |

python/ql/test/experimental/dataflow/tainttracking/customSanitizer/InlineTaintTest.ql

@@ -6,6 +6,12 @@ predicate isSafeCheck(DataFlow::GuardNode g, ControlFlowNode node, boolean branc
   branch = true
 }
 
+predicate isUnsafeCheck(DataFlow::GuardNode g, ControlFlowNode node, boolean branch) {
+  g.(CallNode).getNode().getFunc().(Name).getId() in ["is_unsafe", "emulated_is_unsafe"] and
+  node = g.(CallNode).getAnArg() and
+  branch = false
+}
+
 class CustomSanitizerOverrides extends TestTaintTrackingConfiguration {
   override predicate isSanitizer(DataFlow::Node node) {
     exists(Call call |
@@ -16,6 +22,8 @@ class CustomSanitizerOverrides extends TestTaintTrackingConfiguration {
     node.asExpr().(Call).getFunc().(Name).getId() = "emulated_escaping"
     or
     node = DataFlow::BarrierGuard<isSafeCheck/3>::getABarrierNode()
+    or
+    node = DataFlow::BarrierGuard<isUnsafeCheck/3>::getABarrierNode()
   }
 }
 

python/ql/test/experimental/dataflow/tainttracking/customSanitizer/test_logical.py

@@ -22,6 +22,9 @@ def random_choice():
 def is_safe(arg):
     return arg == "safe"
 
+def is_unsafe(arg):
+    return arg == TAINTED_STRING
+
 
 def test_basic():
     s = TAINTED_STRING
@@ -34,7 +37,7 @@ def test_basic():
     if not is_safe(s):
         ensure_tainted(s) # $ tainted
     else:
-        ensure_not_tainted(s) # $ SPURIOUS: tainted
+        ensure_not_tainted(s)
 
 
 def test_if_in_depth():
@@ -105,7 +108,7 @@ def test_and():
         ensure_tainted(s) # $ tainted
     else:
         # cannot be tainted
-        ensure_not_tainted(s) # $ SPURIOUS: tainted
+        ensure_not_tainted(s)
 
 
 def test_tricky():
@@ -124,14 +127,14 @@ def test_nesting_not():
     s = TAINTED_STRING
 
     if not(not(is_safe(s))):
-        ensure_not_tainted(s) # $ SPURIOUS: tainted
+        ensure_not_tainted(s)
     else:
         ensure_tainted(s) # $ tainted
 
     if not(not(not(is_safe(s)))):
         ensure_tainted(s) # $ tainted
     else:
-        ensure_not_tainted(s) # $ SPURIOUS: tainted
+        ensure_not_tainted(s)
 
 
 # Adding `and True` makes the sanitizer trigger when it would otherwise not. See output in
@@ -161,7 +164,16 @@ def test_with_return():
     if not is_safe(s):
         return
 
-    ensure_not_tainted(s) # $ SPURIOUS: tainted
+    ensure_not_tainted(s)
+
+
+def test_with_return_neg():
+    s = TAINTED_STRING
+
+    if is_unsafe(s):
+        return
+
+    ensure_not_tainted(s)
 
 
 def test_with_exception():
@@ -170,7 +182,15 @@ def test_with_exception():
     if not is_safe(s):
         raise Exception("unsafe")
 
-    ensure_not_tainted(s) # $ SPURIOUS: tainted
+    ensure_not_tainted(s)
+
+def test_with_exception_neg():
+    s = TAINTED_STRING
+
+    if is_unsafe(s):
+        raise Exception("unsafe")
+
+    ensure_not_tainted(s)
 
 # Make tests runable
 
@@ -182,7 +202,12 @@ test_tricky()
 test_nesting_not()
 test_nesting_not_with_and_true()
 test_with_return()
+test_with_return_neg()
 try:
     test_with_exception()
 except:
     pass
+try:
+    test_with_exception_neg()
+except:
+    pass

ruby/ql/lib/codeql/ruby/controlflow/internal/ControlFlowGraphImplShared.qll

@@ -881,7 +881,12 @@ import Cached
  * graph is restricted to nodes from `RelevantNode`.
  */
 module TestOutput {
-  abstract class RelevantNode extends Node { }
+  abstract class RelevantNode extends Node {
+    /**
+     * Gets a string used to resolve ties in node and edge ordering.
+     */
+    string getOrderDisambuigation() { result = "" }
+  }
 
   query predicate nodes(RelevantNode n, string attr, string val) {
     attr = "semmle.order" and
@@ -894,7 +899,8 @@ module TestOutput {
             p
             order by
               l.getFile().getBaseName(), l.getFile().getAbsolutePath(), l.getStartLine(),
-              l.getStartColumn(), l.getEndLine(), l.getEndColumn(), p.toString()
+              l.getStartColumn(), l.getEndLine(), l.getEndColumn(), p.toString(),
+              p.getOrderDisambuigation()
           )
       ).toString()
   }
@@ -916,7 +922,8 @@ module TestOutput {
             s
             order by
               l.getFile().getBaseName(), l.getFile().getAbsolutePath(), l.getStartLine(),
-              l.getStartColumn(), l.getEndLine(), l.getEndColumn(), t.toString()
+              l.getStartColumn(), l.getEndLine(), l.getEndColumn(), t.toString(), s.toString(),
+              s.getOrderDisambuigation()
           )
       ).toString()
   }

swift/codegen/generators/trapgen.py

@@ -86,11 +86,10 @@ def generate(opts, renderer):
         renderer.render(cpp.TrapList(entries, opts.dbscheme), out / dir / "TrapEntries")
 
     tags = []
-    for index, tag in enumerate(toposort_flatten(tag_graph)):
+    for tag in toposort_flatten(tag_graph):
         tags.append(cpp.Tag(
             name=get_tag_name(tag),
             bases=[get_tag_name(b) for b in sorted(tag_graph[tag])],
-            index=index,
             id=tag,
         ))
     renderer.render(cpp.TagList(tags, opts.dbscheme), out / "TrapTags")

swift/codegen/lib/cpp.py

@@ -83,7 +83,6 @@ class TagBase:
 class Tag:
     name: str
     bases: List[TagBase]
-    index: int
     id: str
 
     def __post_init__(self):

swift/codegen/schema.yml

@@ -112,6 +112,7 @@ LValueType:
 
 ModuleType:
   _extends: Type
+  module: ModuleDecl
 
 PlaceholderType:
   _extends: Type

swift/codegen/templates/trap_tags_h.mustache

@@ -7,7 +7,7 @@ namespace codeql {
 
 // {{id}}
 struct {{name}}Tag {{#has_bases}}: {{#bases}}{{^first}}, {{/first}}{{base}}Tag{{/bases}} {{/has_bases}}{
-  static constexpr const char* prefix = "{{index}}";
+  static constexpr const char* prefix = "{{name}}";
 };
 {{/tags}}
 }

swift/codegen/test/test_cpp.py

@@ -65,7 +65,7 @@ def test_trap_has_first_field_marked():
 def test_tag_has_first_base_marked():
     bases = ["a", "b", "c"]
     expected = [cpp.TagBase("a", first=True), cpp.TagBase("b"), cpp.TagBase("c")]
-    t = cpp.Tag("name", bases, 0, "id")
+    t = cpp.Tag("name", bases, "id")
     assert t.bases == expected
 
 
@@ -75,7 +75,7 @@ def test_tag_has_first_base_marked():
     (["a", "b"], True)
 ])
 def test_tag_has_bases(bases, expected):
-    t = cpp.Tag("name", bases, 0, "id")
+    t = cpp.Tag("name", bases, "id")
     assert t.has_bases is expected
 
 

swift/codegen/test/test_trapgen.py

@@ -162,10 +162,10 @@ def test_one_union_tags(generate_tags):
     assert generate_tags([
         dbscheme.Union(lhs="@left_hand_side", rhs=["@b", "@a", "@c"]),
     ]) == [
-        cpp.Tag(name="LeftHandSide", bases=[], index=0, id="@left_hand_side"),
-        cpp.Tag(name="A", bases=["LeftHandSide"], index=1, id="@a"),
-        cpp.Tag(name="B", bases=["LeftHandSide"], index=2, id="@b"),
-        cpp.Tag(name="C", bases=["LeftHandSide"], index=3, id="@c"),
+        cpp.Tag(name="LeftHandSide", bases=[], id="@left_hand_side"),
+        cpp.Tag(name="A", bases=["LeftHandSide"], id="@a"),
+        cpp.Tag(name="B", bases=["LeftHandSide"], id="@b"),
+        cpp.Tag(name="C", bases=["LeftHandSide"], id="@c"),
     ]
 
 
@@ -175,12 +175,12 @@ def test_multiple_union_tags(generate_tags):
         dbscheme.Union(lhs="@a", rhs=["@b", "@c"]),
         dbscheme.Union(lhs="@e", rhs=["@c", "@f"]),
     ]) == [
-        cpp.Tag(name="D", bases=[], index=0, id="@d"),
-        cpp.Tag(name="E", bases=[], index=1, id="@e"),
-        cpp.Tag(name="A", bases=["D"], index=2, id="@a"),
-        cpp.Tag(name="F", bases=["E"], index=3, id="@f"),
-        cpp.Tag(name="B", bases=["A"], index=4, id="@b"),
-        cpp.Tag(name="C", bases=["A", "E"], index=5, id="@c"),
+        cpp.Tag(name="D", bases=[], id="@d"),
+        cpp.Tag(name="E", bases=[], id="@e"),
+        cpp.Tag(name="A", bases=["D"], id="@a"),
+        cpp.Tag(name="F", bases=["E"], id="@f"),
+        cpp.Tag(name="B", bases=["A"], id="@b"),
+        cpp.Tag(name="C", bases=["A", "E"], id="@c"),
     ]
 
 

swift/extractor/BUILD.bazel

@@ -2,14 +2,10 @@ load("//swift:rules.bzl", "swift_cc_binary")
 
 swift_cc_binary(
     name = "extractor",
-    srcs = [
-        "SwiftOutputRewrite.cpp",
-        "SwiftOutputRewrite.h",
-        "SwiftExtractor.cpp",
-        "SwiftExtractor.h",
-        "SwiftExtractorConfiguration.h",
-        "main.cpp",
-    ],
+    srcs = glob([
+        "*.h",
+        "*.cpp",
+    ]),
     visibility = ["//swift:__pkg__"],
     deps = [
         "//swift/extractor/infra",

swift/extractor/SwiftExtractor.cpp

@@ -14,9 +14,10 @@
 #include "swift/extractor/trap/generated/TrapClasses.h"
 #include "swift/extractor/trap/TrapDomain.h"
 #include "swift/extractor/visitors/SwiftVisitor.h"
-#include "swift/extractor/infra/TargetFile.h"
+#include "swift/extractor/TargetTrapFile.h"
 
 using namespace codeql;
+using namespace std::string_literals;
 
 static void archiveFile(const SwiftExtractorConfiguration& config, swift::SourceFile& file) {
   if (std::error_code ec = llvm::sys::fs::create_directories(config.trapDir)) {
@@ -53,12 +54,18 @@ static std::string getFilename(swift::ModuleDecl& module, swift::SourceFile* pri
   if (primaryFile) {
     return primaryFile->getFilename().str();
   }
-  // Several modules with different name might come from .pcm (clang module) files
-  // In this case we want to differentiate them
-  std::string filename = module.getModuleFilename().str();
-  filename += "-";
-  filename += module.getName().str();
-  return filename;
+  // PCM clang module
+  if (module.isNonSwiftModule()) {
+    // Several modules with different names might come from .pcm (clang module) files
+    // In this case we want to differentiate them
+    // Moreover, pcm files may come from caches located in different directories, but are
+    // unambiguously identified by the base file name, so we can discard the absolute directory
+    std::string filename = "/pcms/"s + llvm::sys::path::filename(module.getModuleFilename()).str();
+    filename += "-";
+    filename += module.getName().str();
+    return filename;
+  }
+  return module.getModuleFilename().str();
 }
 
 static llvm::SmallVector<swift::Decl*> getTopLevelDecls(swift::ModuleDecl& module,
@@ -73,20 +80,6 @@ static llvm::SmallVector<swift::Decl*> getTopLevelDecls(swift::ModuleDecl& modul
   return ret;
 }
 
-static void dumpArgs(TargetFile& out, const SwiftExtractorConfiguration& config) {
-  out << "/* extractor-args:\n";
-  for (const auto& opt : config.frontendOptions) {
-    out << "  " << std::quoted(opt) << " \\\n";
-  }
-  out << "\n*/\n";
-
-  out << "/* swift-frontend-args:\n";
-  for (const auto& opt : config.patchedFrontendOptions) {
-    out << "  " << std::quoted(opt) << " \\\n";
-  }
-  out << "\n*/\n";
-}
-
 static void extractDeclarations(const SwiftExtractorConfiguration& config,
                                 swift::CompilerInstance& compiler,
                                 swift::ModuleDecl& module,
@@ -96,37 +89,26 @@ static void extractDeclarations(const SwiftExtractorConfiguration& config,
   // The extractor can be called several times from different processes with
   // the same input file(s). Using `TargetFile` the first process will win, and the following
   // will just skip the work
-  auto trapTarget = TargetFile::create(filename + ".trap", config.trapDir, config.getTempTrapDir());
+  auto trapTarget = createTargetTrapFile(config, filename);
   if (!trapTarget) {
     // another process arrived first, nothing to do for us
     return;
   }
-  dumpArgs(*trapTarget, config);
   TrapDomain trap{*trapTarget};
 
-  // TODO: move default location emission elsewhere, possibly in a separate global trap file
+  // TODO: remove this and recreate it with IPA when we have that
   // the following cannot conflict with actual files as those have an absolute path starting with /
-  auto unknownFileLabel = trap.createLabel<FileTag>("unknown");
-  auto unknownLocationLabel = trap.createLabel<LocationTag>("unknown");
-  trap.emit(FilesTrap{unknownFileLabel});
-  trap.emit(LocationsTrap{unknownLocationLabel, unknownFileLabel});
+  File unknownFileEntry{trap.createLabel<FileTag>("unknown")};
+  Location unknownLocationEntry{trap.createLabel<LocationTag>("unknown")};
+  unknownLocationEntry.file = unknownFileEntry.id;
+  trap.emit(unknownFileEntry);
+  trap.emit(unknownLocationEntry);
 
   SwiftVisitor visitor(compiler.getSourceMgr(), trap, module, primaryFile);
   auto topLevelDecls = getTopLevelDecls(module, primaryFile);
   for (auto decl : topLevelDecls) {
     visitor.extract(decl);
   }
-  // TODO the following will be moved to the dispatcher when we start caching swift file objects
-  // for the moment, topLevelDecls always contains the current module, which does not have a file
-  // associated with it, so we need a special case when there are no top level declarations
-  if (topLevelDecls.size() == 1) {
-    // In the case of empty files, the dispatcher is not called, but we still want to 'record' the
-    // fact that the file was extracted
-    llvm::SmallString<PATH_MAX> name(filename);
-    llvm::sys::fs::make_absolute(name);
-    auto fileLabel = trap.createLabel<FileTag>(name.str().str());
-    trap.emit(FilesTrap{fileLabel, name.str().str()});
-  }
 }
 
 static std::unordered_set<std::string> collectInputFilenames(swift::CompilerInstance& compiler) {
@@ -176,21 +158,21 @@ void codeql::extractSwiftFiles(const SwiftExtractorConfiguration& config,
   auto modules = collectModules(compiler);
 
   for (auto& module : modules) {
-    // We only extract system and builtin modules here as the other "user" modules can be built
-    // during the build process and then re-used at a later stage. In this case, we extract the
-    // user code twice: once during the module build in a form of a source file, and then as
-    // a pre-built module during building of the dependent source files.
-    if (module->isSystemModule() || module->isBuiltinModule()) {
-      extractDeclarations(config, compiler, *module);
-    } else {
-      for (auto file : module->getFiles()) {
-        auto sourceFile = llvm::dyn_cast<swift::SourceFile>(file);
-        if (!sourceFile || inputFiles.count(sourceFile->getFilename().str()) == 0) {
-          continue;
-        }
-        archiveFile(config, *sourceFile);
-        extractDeclarations(config, compiler, *module, sourceFile);
+    bool isFromSourceFile = false;
+    for (auto file : module->getFiles()) {
+      auto sourceFile = llvm::dyn_cast<swift::SourceFile>(file);
+      if (!sourceFile) {
+        continue;
       }
+      isFromSourceFile = true;
+      if (inputFiles.count(sourceFile->getFilename().str()) == 0) {
+        continue;
+      }
+      archiveFile(config, *sourceFile);
+      extractDeclarations(config, compiler, *module, sourceFile);
+    }
+    if (!isFromSourceFile) {
+      extractDeclarations(config, compiler, *module);
     }
   }
 }

swift/extractor/SwiftExtractorConfiguration.h

@@ -3,6 +3,8 @@
 #include <string>
 #include <vector>
 
+#include "swift/extractor/infra/TargetFile.h"
+
 namespace codeql {
 struct SwiftExtractorConfiguration {
   // The location for storing TRAP files to be imported by CodeQL engine.
@@ -33,4 +35,5 @@ struct SwiftExtractorConfiguration {
   // overall extraction process.
   std::string getTempArtifactDir() const { return scratchDir + "/swift-extraction-artifacts"; }
 };
+
 }  // namespace codeql

swift/extractor/SwiftOutputRewrite.cpp

@@ -1,5 +1,6 @@
 #include "SwiftOutputRewrite.h"
 #include "swift/extractor/SwiftExtractorConfiguration.h"
+#include "swift/extractor/TargetTrapFile.h"
 
 #include <llvm/ADT/SmallString.h>
 #include <llvm/Support/FileSystem.h>
@@ -163,7 +164,7 @@ static std::vector<std::string> computeModuleAliases(llvm::StringRef modulePath,
 namespace codeql {
 
 std::unordered_map<std::string, std::string> rewriteOutputsInPlace(
-    SwiftExtractorConfiguration& config,
+    const SwiftExtractorConfiguration& config,
     std::vector<std::string>& CLIArgs) {
   std::unordered_map<std::string, std::string> remapping;
 
@@ -324,4 +325,17 @@ std::vector<std::string> collectVFSFiles(const SwiftExtractorConfiguration& conf
   return overlays;
 }
 
+void lockOutputSwiftModuleTraps(const SwiftExtractorConfiguration& config,
+                                const std::unordered_map<std::string, std::string>& remapping) {
+  for (const auto& [oldPath, newPath] : remapping) {
+    if (llvm::StringRef(oldPath).endswith(".swiftmodule")) {
+      if (auto target = createTargetTrapFile(config, oldPath)) {
+        *target << "// trap file deliberately empty\n"
+                   "// this swiftmodule was created during the build, so its entities must have"
+                   " been extracted directly from source files";
+      }
+    }
+  }
+}
+
 }  // namespace codeql

swift/extractor/SwiftOutputRewrite.h

@@ -13,7 +13,7 @@ struct SwiftExtractorConfiguration;
 // artifacts produced by the actual Swift compiler.
 // Returns the map containing remapping oldpath -> newPath.
 std::unordered_map<std::string, std::string> rewriteOutputsInPlace(
-    SwiftExtractorConfiguration& config,
+    const SwiftExtractorConfiguration& config,
     std::vector<std::string>& CLIArgs);
 
 // Create directories for all the redirected new paths as the Swift compiler expects them to exist.
@@ -29,4 +29,7 @@ void storeRemappingForVFS(const SwiftExtractorConfiguration& config,
 // This is separate from storeRemappingForVFS as we also collect files produced by other processes.
 std::vector<std::string> collectVFSFiles(const SwiftExtractorConfiguration& config);
 
+// Creates empty trap files for output swiftmodule files
+void lockOutputSwiftModuleTraps(const SwiftExtractorConfiguration& config,
+                                const std::unordered_map<std::string, std::string>& remapping);
 }  // namespace codeql

swift/extractor/TargetTrapFile.cpp

@@ -0,0 +1,23 @@
+#include "swift/extractor/TargetTrapFile.h"
+#include <iomanip>
+namespace codeql {
+std::optional<TargetFile> createTargetTrapFile(const SwiftExtractorConfiguration& configuration,
+                                               std::string_view target) {
+  std::string trap{target};
+  trap += ".trap";
+  auto ret = TargetFile::create(trap, configuration.trapDir, configuration.getTempTrapDir());
+  if (ret) {
+    *ret << "/* extractor-args:\n";
+    for (const auto& opt : configuration.frontendOptions) {
+      *ret << "  " << std::quoted(opt) << " \\\n";
+    }
+    *ret << "\n*/\n"
+            "/* swift-frontend-args:\n";
+    for (const auto& opt : configuration.patchedFrontendOptions) {
+      *ret << "  " << std::quoted(opt) << " \\\n";
+    }
+    *ret << "\n*/\n";
+  }
+  return ret;
+}
+}  // namespace codeql

swift/extractor/TargetTrapFile.h

@@ -0,0 +1,11 @@
+#pragma once
+
+#include "swift/extractor/infra/TargetFile.h"
+#include "swift/extractor/SwiftExtractorConfiguration.h"
+
+namespace codeql {
+
+std::optional<TargetFile> createTargetTrapFile(const SwiftExtractorConfiguration& configuration,
+                                               std::string_view target);
+
+}  // namespace codeql

swift/extractor/infra/FilePath.h

@@ -0,0 +1,24 @@
+#pragma once
+
+#include <string>
+namespace codeql {
+
+// wrapper around `std::string` mainly intended to unambiguously go into an `std::variant`
+// TODO probably not needed once we can use `std::filesystem::path`
+struct FilePath {
+  FilePath() = default;
+  FilePath(const std::string& path) : path{path} {}
+  FilePath(std::string&& path) : path{std::move(path)} {}
+
+  std::string path;
+
+  bool operator==(const FilePath& other) const { return path == other.path; }
+};
+}  // namespace codeql
+
+namespace std {
+template <>
+struct hash<codeql::FilePath> {
+  size_t operator()(const codeql::FilePath& value) { return hash<string>{}(value.path); }
+};
+}  // namespace std

swift/extractor/infra/SwiftDispatcher.h

@@ -8,6 +8,7 @@
 #include "swift/extractor/trap/TrapDomain.h"
 #include "swift/extractor/infra/SwiftTagTraits.h"
 #include "swift/extractor/trap/generated/TrapClasses.h"
+#include "swift/extractor/infra/FilePath.h"
 
 namespace codeql {
 
@@ -17,6 +18,25 @@ namespace codeql {
 // Since SwiftDispatcher sees all the AST nodes, it also attaches a location to every 'locatable'
 // node (AST nodes that are not types: declarations, statements, expressions, etc.).
 class SwiftDispatcher {
+  // types to be supported by assignNewLabel/fetchLabel need to be listed here
+  using Store = TrapLabelStore<const swift::Decl*,
+                               const swift::Stmt*,
+                               const swift::StmtCondition*,
+                               const swift::StmtConditionElement*,
+                               const swift::CaseLabelItem*,
+                               const swift::Expr*,
+                               const swift::Pattern*,
+                               const swift::TypeRepr*,
+                               const swift::TypeBase*,
+                               const swift::IfConfigClause*,
+                               FilePath>;
+
+  template <typename E>
+  static constexpr bool IsStorable = std::is_constructible_v<Store::Handle, const E&>;
+
+  template <typename E>
+  static constexpr bool IsLocatable = std::is_base_of_v<LocatableTag, TrapTagOf<E>>;
+
  public:
   // all references and pointers passed as parameters to this constructor are supposed to outlive
   // the SwiftDispatcher
@@ -27,7 +47,12 @@ class SwiftDispatcher {
       : sourceManager{sourceManager},
         trap{trap},
         currentModule{currentModule},
-        currentPrimarySourceFile{currentPrimarySourceFile} {}
+        currentPrimarySourceFile{currentPrimarySourceFile} {
+    if (currentPrimarySourceFile) {
+      // we make sure the file is in the trap output even if the source is empty
+      fetchLabel(getFilePath(currentPrimarySourceFile->getFilename()));
+    }
+  }
 
   template <typename Entry>
   void emit(const Entry& entry) {
@@ -61,9 +86,11 @@ class SwiftDispatcher {
   // This method gives a TRAP label for already emitted AST node.
   // If the AST node was not emitted yet, then the emission is dispatched to a corresponding
   // visitor (see `visit(T *)` methods below).
-  template <typename E, typename... Args>
-  TrapLabelOf<E> fetchLabel(E* e, Args&&... args) {
-    assert(e && "trying to fetch a label on nullptr, maybe fetchOptionalLabel is to be used?");
+  template <typename E, typename... Args, std::enable_if_t<IsStorable<E>>* = nullptr>
+  TrapLabelOf<E> fetchLabel(const E& e, Args&&... args) {
+    if constexpr (std::is_constructible_v<bool, const E&>) {
+      assert(e && "fetching a label on a null entity, maybe fetchOptionalLabel is to be used?");
+    }
     // this is required so we avoid any recursive loop: a `fetchLabel` during the visit of `e` might
     // end up calling `fetchLabel` on `e` itself, so we want the visit of `e` to call `fetchLabel`
     // only after having called `assignNewLabel` on `e`.
@@ -76,7 +103,7 @@ class SwiftDispatcher {
     visit(e, std::forward<Args>(args)...);
     // TODO when everything is moved to structured C++ classes, this should be moved to createEntry
     if (auto l = store.get(e)) {
-      if constexpr (!std::is_base_of_v<swift::TypeBase, E>) {
+      if constexpr (IsLocatable<E>) {
         attachLocation(e, *l);
       }
       return *l;
@@ -93,19 +120,16 @@ class SwiftDispatcher {
     return fetchLabelFromUnion<AstNodeTag>(node);
   }
 
-  TrapLabel<IfConfigClauseTag> fetchLabel(const swift::IfConfigClause& clause) {
-    return fetchLabel(&clause);
-  }
-
-  TrapLabel<ConditionElementTag> fetchLabel(const swift::StmtConditionElement& element) {
-    return fetchLabel(&element);
+  template <typename E, std::enable_if_t<IsStorable<E*>>* = nullptr>
+  TrapLabelOf<E> fetchLabel(const E& e) {
+    return fetchLabel(&e);
   }
 
   // Due to the lazy emission approach, we must assign a label to a corresponding AST node before
   // it actually gets emitted to handle recursive cases such as recursive calls, or recursive type
   // declarations
-  template <typename E, typename... Args>
-  TrapLabelOf<E> assignNewLabel(E* e, Args&&... args) {
+  template <typename E, typename... Args, std::enable_if_t<IsStorable<E>>* = nullptr>
+  TrapLabelOf<E> assignNewLabel(const E& e, Args&&... args) {
     assert(waitingForNewLabel == Store::Handle{e} && "assignNewLabel called on wrong entity");
     auto label = trap.createLabel<TrapTagOf<E>>(std::forward<Args>(args)...);
     store.insert(e, label);
@@ -113,20 +137,20 @@ class SwiftDispatcher {
     return label;
   }
 
-  template <typename E, typename... Args, std::enable_if_t<!std::is_pointer_v<E>>* = nullptr>
+  template <typename E, typename... Args, std::enable_if_t<IsStorable<E*>>* = nullptr>
   TrapLabelOf<E> assignNewLabel(const E& e, Args&&... args) {
     return assignNewLabel(&e, std::forward<Args>(args)...);
   }
 
   // convenience methods for structured C++ creation
-  template <typename E, typename... Args, std::enable_if_t<!std::is_pointer_v<E>>* = nullptr>
+  template <typename E, typename... Args>
   auto createEntry(const E& e, Args&&... args) {
-    return TrapClassOf<E>{assignNewLabel(&e, std::forward<Args>(args)...)};
+    return TrapClassOf<E>{assignNewLabel(e, std::forward<Args>(args)...)};
   }
 
   // used to create a new entry for entities that should not be cached
   // an example is swift::Argument, that are created on the fly and thus have no stable pointer
-  template <typename E, typename... Args, std::enable_if_t<!std::is_pointer_v<E>>* = nullptr>
+  template <typename E, typename... Args>
   auto createUncachedEntry(const E& e, Args&&... args) {
     auto label = trap.createLabel<TrapTagOf<E>>(std::forward<Args>(args)...);
     attachLocation(&e, label);
@@ -191,7 +215,7 @@ class SwiftDispatcher {
 
   template <typename... Args>
   void emitDebugInfo(const Args&... args) {
-    trap.debug(std::forward<Args>(args)...);
+    trap.debug(args...);
   }
 
   // In order to not emit duplicated entries for declarations, we restrict emission to only
@@ -219,18 +243,6 @@ class SwiftDispatcher {
   }
 
  private:
-  // types to be supported by assignNewLabel/fetchLabel need to be listed here
-  using Store = TrapLabelStore<swift::Decl,
-                               swift::Stmt,
-                               swift::StmtCondition,
-                               swift::StmtConditionElement,
-                               swift::CaseLabelItem,
-                               swift::Expr,
-                               swift::Pattern,
-                               swift::TypeRepr,
-                               swift::TypeBase,
-                               swift::IfConfigClause>;
-
   void attachLocation(swift::SourceLoc start,
                       swift::SourceLoc end,
                       TrapLabel<LocatableTag> locatableLabel) {
@@ -238,16 +250,16 @@ class SwiftDispatcher {
       // invalid locations seem to come from entities synthesized by the compiler
       return;
     }
-    std::string filepath = getFilepath(start);
-    auto fileLabel = trap.createLabel<FileTag>(filepath);
-    // TODO: do not emit duplicate trap entries for Files
-    trap.emit(FilesTrap{fileLabel, filepath});
-    auto [startLine, startColumn] = sourceManager.getLineAndColumnInBuffer(start);
-    auto [endLine, endColumn] = sourceManager.getLineAndColumnInBuffer(end);
-    auto locLabel = trap.createLabel<LocationTag>('{', fileLabel, "}:", startLine, ':', startColumn,
-                                                  ':', endLine, ':', endColumn);
-    trap.emit(LocationsTrap{locLabel, fileLabel, startLine, startColumn, endLine, endColumn});
-    trap.emit(LocatableLocationsTrap{locatableLabel, locLabel});
+    auto file = getFilePath(sourceManager.getDisplayNameForLoc(start));
+    Location entry{{}};
+    entry.file = fetchLabel(file);
+    std::tie(entry.start_line, entry.start_column) = sourceManager.getLineAndColumnInBuffer(start);
+    std::tie(entry.end_line, entry.end_column) = sourceManager.getLineAndColumnInBuffer(end);
+    entry.id = trap.createLabel<LocationTag>('{', entry.file, "}:", entry.start_line, ':',
+                                             entry.start_column, ':', entry.end_line, ':',
+                                             entry.end_column);
+    emit(entry);
+    emit(LocatableLocationsTrap{locatableLabel, entry.id});
   }
 
   template <typename Tag, typename... Ts>
@@ -276,14 +288,13 @@ class SwiftDispatcher {
     return false;
   }
 
-  std::string getFilepath(swift::SourceLoc loc) {
+  static FilePath getFilePath(llvm::StringRef path) {
     // TODO: this needs more testing
     // TODO: check canonicaliztion of names on a case insensitive filesystems
     // TODO: make symlink resolution conditional on CODEQL_PRESERVE_SYMLINKS=true
-    auto displayName = sourceManager.getDisplayNameForLoc(loc);
     llvm::SmallString<PATH_MAX> realPath;
-    if (std::error_code ec = llvm::sys::fs::real_path(displayName, realPath)) {
-      std::cerr << "Cannot get real path: '" << displayName.str() << "': " << ec.message() << "\n";
+    if (std::error_code ec = llvm::sys::fs::real_path(path, realPath)) {
+      std::cerr << "Cannot get real path: '" << path.str() << "': " << ec.message() << "\n";
       return {};
     }
     return realPath.str().str();
@@ -303,6 +314,12 @@ class SwiftDispatcher {
   virtual void visit(swift::TypeRepr* typeRepr, swift::Type type) = 0;
   virtual void visit(swift::TypeBase* type) = 0;
 
+  void visit(const FilePath& file) {
+    auto entry = createEntry(file, file.path);
+    entry.name = file.path;
+    emit(entry);
+  }
+
   const swift::SourceManager& sourceManager;
   TrapDomain& trap;
   Store store;

swift/extractor/infra/SwiftTagTraits.h

@@ -5,6 +5,7 @@
 #include <swift/AST/ASTVisitor.h>
 #include "swift/extractor/trap/TrapTagTraits.h"
 #include "swift/extractor/trap/generated/TrapTags.h"
+#include "swift/extractor/infra/FilePath.h"
 
 namespace codeql {
 
@@ -15,12 +16,12 @@ using SILBoxTypeTag = SilBoxTypeTag;
 using SILFunctionTypeTag = SilFunctionTypeTag;
 using SILTokenTypeTag = SilTokenTypeTag;
 
-#define MAP_TYPE_TO_TAG(TYPE, TAG)           \
-  template <>                                \
-  struct detail::ToTagFunctor<swift::TYPE> { \
-    using type = TAG;                        \
+#define MAP_TYPE_TO_TAG(TYPE, TAG)    \
+  template <>                         \
+  struct detail::ToTagFunctor<TYPE> { \
+    using type = TAG;                 \
   }
-#define MAP_TAG(TYPE) MAP_TYPE_TO_TAG(TYPE, TYPE##Tag)
+#define MAP_TAG(TYPE) MAP_TYPE_TO_TAG(swift::TYPE, TYPE##Tag)
 #define MAP_SUBTAG(TYPE, PARENT)                           \
   MAP_TAG(TYPE);                                           \
   static_assert(std::is_base_of_v<PARENT##Tag, TYPE##Tag>, \
@@ -35,7 +36,7 @@ using SILTokenTypeTag = SilTokenTypeTag;
 
 MAP_TAG(Stmt);
 MAP_TAG(StmtCondition);
-MAP_TYPE_TO_TAG(StmtConditionElement, ConditionElementTag);
+MAP_TYPE_TO_TAG(swift::StmtConditionElement, ConditionElementTag);
 MAP_TAG(CaseLabelItem);
 #define ABSTRACT_STMT(CLASS, PARENT) MAP_SUBTAG(CLASS##Stmt, PARENT)
 #define STMT(CLASS, PARENT) ABSTRACT_STMT(CLASS, PARENT)
@@ -60,7 +61,7 @@ MAP_TAG(Pattern);
 
 MAP_TAG(TypeRepr);
 
-MAP_TYPE_TO_TAG(TypeBase, TypeTag);
+MAP_TYPE_TO_TAG(swift::TypeBase, TypeTag);
 #define ABSTRACT_TYPE(CLASS, PARENT) MAP_SUBTAG(CLASS##Type, PARENT)
 #define TYPE(CLASS, PARENT) ABSTRACT_TYPE(CLASS, PARENT)
 #include <swift/AST/TypeNodes.def>
@@ -68,6 +69,8 @@ MAP_TYPE_TO_TAG(TypeBase, TypeTag);
 OVERRIDE_TAG(FuncDecl, ConcreteFuncDeclTag);
 OVERRIDE_TAG(VarDecl, ConcreteVarDeclTag);
 
+MAP_TYPE_TO_TAG(FilePath, FileTag);
+
 #undef MAP_TAG
 #undef MAP_SUBTAG
 #undef MAP_TYPE_TO_TAG

swift/extractor/main.cpp

@@ -68,6 +68,7 @@ int main(int argc, char** argv) {
       codeql::rewriteOutputsInPlace(configuration, configuration.patchedFrontendOptions);
   codeql::ensureDirectoriesForNewPathsExist(remapping);
   codeql::storeRemappingForVFS(configuration, remapping);
+  codeql::lockOutputSwiftModuleTraps(configuration, remapping);
 
   std::vector<const char*> args;
   for (auto& arg : configuration.patchedFrontendOptions) {

swift/extractor/trap/TrapLabelStore.h

@@ -20,10 +20,10 @@ namespace codeql {
 template <typename... Ts>
 class TrapLabelStore {
  public:
-  using Handle = std::variant<std::monostate, const Ts*...>;
+  using Handle = std::variant<std::monostate, Ts...>;
 
   template <typename T>
-  std::optional<TrapLabelOf<T>> get(const T* e) {
+  std::optional<TrapLabelOf<T>> get(const T& e) {
     if (auto found = store_.find(e); found != store_.end()) {
       return TrapLabelOf<T>::unsafeCreateFromUntyped(found->second);
     }
@@ -31,7 +31,7 @@ class TrapLabelStore {
   }
 
   template <typename T>
-  void insert(const T* e, TrapLabelOf<T> l) {
+  void insert(const T& e, TrapLabelOf<T> l) {
     auto [_, inserted] = store_.emplace(e, l);
     assert(inserted && "already inserted");
   }

swift/extractor/trap/TrapTagTraits.h

@@ -26,7 +26,8 @@ struct ToTrapClassFunctor;
 }  // namespace detail
 
 template <typename T>
-using TrapTagOf = typename detail::ToTagOverride<std::remove_const_t<T>>::type;
+using TrapTagOf =
+    typename detail::ToTagOverride<std::remove_const_t<std::remove_pointer_t<T>>>::type;
 
 template <typename T>
 using TrapLabelOf = TrapLabel<TrapTagOf<T>>;

swift/extractor/visitors/DeclVisitor.cpp

@@ -83,11 +83,13 @@ codeql::PrecedenceGroupDecl DeclVisitor::translatePrecedenceGroupDecl(
   return entry;
 }
 
-codeql::ParamDecl DeclVisitor::translateParamDecl(const swift::ParamDecl& decl) {
-  // TODO: deduplicate
-  ParamDecl entry{dispatcher_.assignNewLabel(decl)};
-  fillVarDecl(decl, entry);
-  entry.is_inout = decl.isInOut();
+std::optional<codeql::ParamDecl> DeclVisitor::translateParamDecl(const swift::ParamDecl& decl) {
+  auto entry = createNamedEntry(decl);
+  if (!entry) {
+    return std::nullopt;
+  }
+  fillVarDecl(decl, *entry);
+  entry->is_inout = decl.isInOut();
   return entry;
 }
 
@@ -111,11 +113,20 @@ codeql::PatternBindingDecl DeclVisitor::translatePatternBindingDecl(
   return entry;
 }
 
-codeql::ConcreteVarDecl DeclVisitor::translateVarDecl(const swift::VarDecl& decl) {
-  // TODO: deduplicate all non-local variables
-  ConcreteVarDecl entry{dispatcher_.assignNewLabel(decl)};
-  entry.introducer_int = static_cast<uint8_t>(decl.getIntroducer());
-  fillVarDecl(decl, entry);
+std::optional<codeql::ConcreteVarDecl> DeclVisitor::translateVarDecl(const swift::VarDecl& decl) {
+  std::optional<codeql::ConcreteVarDecl> entry;
+  // We do not deduplicate variables from non-swift (PCM, clang modules) modules as the mangler
+  // crashes sometimes
+  if (decl.getDeclContext()->isLocalContext() || decl.getModuleContext()->isNonSwiftModule()) {
+    entry.emplace(dispatcher_.assignNewLabel(decl));
+  } else {
+    entry = createNamedEntry(decl);
+    if (!entry) {
+      return std::nullopt;
+    }
+  }
+  entry->introducer_int = static_cast<uint8_t>(decl.getIntroducer());
+  fillVarDecl(decl, *entry);
   return entry;
 }
 
@@ -284,11 +295,30 @@ std::string DeclVisitor::mangledName(const swift::ValueDecl& decl) {
   // ASTMangler::mangleAnyDecl crashes when called on `ModuleDecl`
   // TODO find a more unique string working also when different modules are compiled with the same
   // name
+  std::ostringstream ret;
   if (decl.getKind() == swift::DeclKind::Module) {
-    return static_cast<const swift::ModuleDecl&>(decl).getRealName().str().str();
+    ret << static_cast<const swift::ModuleDecl&>(decl).getRealName().str().str();
+  } else if (decl.getKind() == swift::DeclKind::TypeAlias) {
+    // In cases like this (when coming from PCM)
+    //  typealias CFXMLTree = CFTree
+    //  typealias CFXMLTreeRef = CFXMLTree
+    // mangleAnyDecl mangles both CFXMLTree and CFXMLTreeRef into 'So12CFXMLTreeRefa'
+    // which is not correct and causes inconsistencies. mangleEntity makes these two distinct
+    // prefix adds a couple of special symbols, we don't necessary need them
+    ret << mangler.mangleEntity(&decl);
+  } else {
+    // prefix adds a couple of special symbols, we don't necessary need them
+    ret << mangler.mangleAnyDecl(&decl, /* prefix = */ false);
   }
-  // prefix adds a couple of special symbols, we don't necessary need them
-  return mangler.mangleAnyDecl(&decl, /* prefix = */ false);
+  // there can be separate declarations (`VarDecl` or `AccessorDecl`) which are effectively the same
+  // (with equal mangled name) but come from different clang modules. This is the case for example
+  // for glibc constants like `L_SET` that appear in both `SwiftGlibc` and `CDispatch`.
+  // For the moment, we sidestep the problem by making them separate entities in the DB
+  // TODO find a more solid solution
+  if (decl.getModuleContext()->isNonSwiftModule()) {
+    ret << '_' << decl.getModuleContext()->getRealName().str().str();
+  }
+  return ret.str();
 }
 
 void DeclVisitor::fillAbstractFunctionDecl(const swift::AbstractFunctionDecl& decl,

swift/extractor/visitors/DeclVisitor.h

@@ -30,10 +30,10 @@ class DeclVisitor : public AstVisitorBase<DeclVisitor> {
   codeql::PostfixOperatorDecl translatePostfixOperatorDecl(const swift::PostfixOperatorDecl& decl);
   codeql::InfixOperatorDecl translateInfixOperatorDecl(const swift::InfixOperatorDecl& decl);
   codeql::PrecedenceGroupDecl translatePrecedenceGroupDecl(const swift::PrecedenceGroupDecl& decl);
-  codeql::ParamDecl translateParamDecl(const swift::ParamDecl& decl);
+  std::optional<codeql::ParamDecl> translateParamDecl(const swift::ParamDecl& decl);
   codeql::TopLevelCodeDecl translateTopLevelCodeDecl(const swift::TopLevelCodeDecl& decl);
   codeql::PatternBindingDecl translatePatternBindingDecl(const swift::PatternBindingDecl& decl);
-  codeql::ConcreteVarDecl translateVarDecl(const swift::VarDecl& decl);
+  std::optional<codeql::ConcreteVarDecl> translateVarDecl(const swift::VarDecl& decl);
   std::variant<codeql::StructDecl, codeql::StructDeclsTrap> translateStructDecl(
       const swift::StructDecl& decl);
   std::variant<codeql::ClassDecl, codeql::ClassDeclsTrap> translateClassDecl(

swift/extractor/visitors/SwiftVisitor.h

@@ -14,7 +14,7 @@ class SwiftVisitor : private SwiftDispatcher {
   using SwiftDispatcher::SwiftDispatcher;
 
   template <typename T>
-  void extract(T* entity) {
+  void extract(const T& entity) {
     fetchLabel(entity);
   }
 

swift/extractor/visitors/TypeVisitor.cpp

@@ -381,4 +381,14 @@ codeql::OpenedArchetypeType TypeVisitor::translateOpenedArchetypeType(
   fillArchetypeType(type, entry);
   return entry;
 }
+
+codeql::ModuleType TypeVisitor::translateModuleType(const swift::ModuleType& type) {
+  auto key = type.getModule()->getRealName().str().str();
+  if (type.getModule()->isNonSwiftModule()) {
+    key += "|clang";
+  }
+  auto entry = createTypeEntry(type, key);
+  entry.module = dispatcher_.fetchLabel(type.getModule());
+  return entry;
+}
 }  // namespace codeql

swift/extractor/visitors/TypeVisitor.h

@@ -71,6 +71,7 @@ class TypeVisitor : public TypeVisitorBase<TypeVisitor> {
       const swift::BuiltinUnsafeValueBufferType& type);
   codeql::BuiltinVectorType translateBuiltinVectorType(const swift::BuiltinVectorType& type);
   codeql::OpenedArchetypeType translateOpenedArchetypeType(const swift::OpenedArchetypeType& type);
+  codeql::ModuleType translateModuleType(const swift::ModuleType& type);
 
  private:
   void fillType(const swift::TypeBase& type, codeql::Type& entry);
@@ -83,9 +84,9 @@ class TypeVisitor : public TypeVisitorBase<TypeVisitor> {
   void emitBoundGenericType(swift::BoundGenericType* type, TrapLabel<BoundGenericTypeTag> label);
   void emitAnyGenericType(swift::AnyGenericType* type, TrapLabel<AnyGenericTypeTag> label);
 
-  template <typename T>
-  auto createTypeEntry(const T& type) {
-    auto entry = dispatcher_.createEntry(type);
+  template <typename T, typename... Args>
+  auto createTypeEntry(const T& type, const Args&... args) {
+    auto entry = dispatcher_.createEntry(type, args...);
     fillType(type, entry);
     return entry;
   }

swift/integration-tests/create_database_utils.py

@@ -12,7 +12,7 @@ def run_codeql_database_create(cmds, lang, keep_trap=True):
     codeql_root = pathlib.Path(__file__).parents[2]
     cmd = [
         "codeql", "database", "create",
-        "-s", ".", "-l", "swift", "--internal-use-lua-tracing", f"--search-path={codeql_root}",
+        "-s", ".", "-l", "swift", "--internal-use-lua-tracing", f"--search-path={codeql_root}", "--no-cleanup",
     ]
     if keep_trap:
         cmd.append("--keep-trap")

swift/integration-tests/posix-only/cross-references/VarDecls.expected

@@ -1,5 +1,4 @@
 | Sources/cross-references/lib.swift:10:5:10:5 | X |
-| Sources/cross-references/lib.swift:10:5:10:5 | X |
 | Sources/cross-references/lib.swift:17:16:17:19 | v |
 | Sources/cross-references/lib.swift:22:16:22:19 | v |
 | Sources/cross-references/lib.swift:27:8:27:13 | lhs |

swift/integration-tests/posix-only/partial-modules/Modules.expected

@@ -1,4 +1,5 @@
 | file://:0:0:0:0 | A |
 | file://:0:0:0:0 | B |
+| file://:0:0:0:0 | PackageDescription |
 | file://:0:0:0:0 | main |
 | file://:0:0:0:0 | partial_modules |

swift/integration-tests/runner.py

@@ -62,6 +62,8 @@ def main(opts):
         ]
         if opts.check_databases:
             cmd.append("--check-databases")
+        else:
+            cmd.append("--no-check-databases")
         if opts.learn:
             cmd.append("--learn")
         cmd.extend(str(t.parent) for t in succesful_db_creation)

swift/ql/lib/codeql/swift/controlflow/internal/ControlFlowGraphImplShared.qll

@@ -881,7 +881,12 @@ import Cached
  * graph is restricted to nodes from `RelevantNode`.
  */
 module TestOutput {
-  abstract class RelevantNode extends Node { }
+  abstract class RelevantNode extends Node {
+    /**
+     * Gets a string used to resolve ties in node and edge ordering.
+     */
+    string getOrderDisambuigation() { result = "" }
+  }
 
   query predicate nodes(RelevantNode n, string attr, string val) {
     attr = "semmle.order" and
@@ -894,7 +899,8 @@ module TestOutput {
             p
             order by
               l.getFile().getBaseName(), l.getFile().getAbsolutePath(), l.getStartLine(),
-              l.getStartColumn(), l.getEndLine(), l.getEndColumn(), p.toString()
+              l.getStartColumn(), l.getEndLine(), l.getEndColumn(), p.toString(),
+              p.getOrderDisambuigation()
           )
       ).toString()
   }
@@ -916,7 +922,8 @@ module TestOutput {
             s
             order by
               l.getFile().getBaseName(), l.getFile().getAbsolutePath(), l.getStartLine(),
-              l.getStartColumn(), l.getEndLine(), l.getEndColumn(), t.toString()
+              l.getStartColumn(), l.getEndLine(), l.getEndColumn(), t.toString(), s.toString(),
+              s.getOrderDisambuigation()
           )
       ).toString()
   }

swift/ql/lib/codeql/swift/generated/type/ModuleType.qll

@@ -1,6 +1,14 @@
 // generated by codegen/codegen.py
+import codeql.swift.elements.decl.ModuleDecl
 import codeql.swift.elements.type.Type
 
 class ModuleTypeBase extends @module_type, Type {
   override string getAPrimaryQlClass() { result = "ModuleType" }
+
+  ModuleDecl getModule() {
+    exists(ModuleDecl x |
+      module_types(this, x) and
+      result = x.resolve()
+    )
+  }
 }

swift/ql/lib/swift.dbscheme

@@ -264,7 +264,8 @@ l_value_types(  //dir=type
 );
 
 module_types(  //dir=type
-  unique int id: @module_type
+  unique int id: @module_type,
+  int module: @module_decl ref
 );
 
 placeholder_types(  //dir=type

swift/ql/test/TestUtils.qll

@@ -4,7 +4,11 @@ cached
 predicate toBeTested(Element e) {
   e instanceof File
   or
-  exists(ModuleDecl m | m = e and not m.isBuiltinModule() and not m.isSystemModule())
+  exists(ModuleDecl m |
+    not m.isBuiltinModule() and
+    not m.isSystemModule() and
+    (m = e or m.getInterfaceType() = e)
+  )
   or
   exists(Locatable loc |
     loc.getLocation().getFile().getName().matches("%swift/ql/test%") and

swift/ql/test/extractor-tests/generated/decl/ConcreteVarDecl/ConcreteVarDecl.expected

@@ -0,0 +1,7 @@
+| var_decls.swift:4:7:4:7 | i | getInterfaceType: | Int | getName: | i | getType: | Int | getIntroducerInt: | 1 |
+| var_decls.swift:7:5:7:5 | numbers | getInterfaceType: | [Int] | getName: | numbers | getType: | [Int] | getIntroducerInt: | 1 |
+| var_decls.swift:10:12:10:12 | numbers | getInterfaceType: | [Int] | getName: | numbers | getType: | [Int] | getIntroducerInt: | 0 |
+| var_decls.swift:15:7:15:7 | wrappedValue | getInterfaceType: | T | getName: | wrappedValue | getType: | T | getIntroducerInt: | 1 |
+| var_decls.swift:20:7:20:7 | wrappedValue | getInterfaceType: | Int | getName: | wrappedValue | getType: | Int | getIntroducerInt: | 1 |
+| var_decls.swift:24:15:24:15 | _wrapped | getInterfaceType: | X<Y> | getName: | _wrapped | getType: | X<Y> | getIntroducerInt: | 1 |
+| var_decls.swift:24:15:24:15 | wrapped | getInterfaceType: | Int | getName: | wrapped | getType: | Int | getIntroducerInt: | 1 |

swift/ql/test/extractor-tests/generated/decl/ConcreteVarDecl/ConcreteVarDecl.ql

@@ -0,0 +1,14 @@
+// generated by codegen/codegen.py
+import codeql.swift.elements
+import TestUtils
+
+from ConcreteVarDecl x, Type getInterfaceType, string getName, Type getType, int getIntroducerInt
+where
+  toBeTested(x) and
+  not x.isUnknown() and
+  getInterfaceType = x.getInterfaceType() and
+  getName = x.getName() and
+  getType = x.getType() and
+  getIntroducerInt = x.getIntroducerInt()
+select x, "getInterfaceType:", getInterfaceType, "getName:", getName, "getType:", getType,
+  "getIntroducerInt:", getIntroducerInt

swift/ql/test/extractor-tests/generated/decl/ConcreteVarDecl/ConcreteVarDecl_getAccessorDecl.expected

@@ -0,0 +1,13 @@
+| var_decls.swift:10:12:10:12 | numbers | 0 | var_decls.swift:10:12:10:12 | get |
+| var_decls.swift:15:7:15:7 | wrappedValue | 0 | var_decls.swift:15:7:15:7 | get |
+| var_decls.swift:15:7:15:7 | wrappedValue | 1 | var_decls.swift:15:7:15:7 | set |
+| var_decls.swift:15:7:15:7 | wrappedValue | 2 | var_decls.swift:15:7:15:7 | (unnamed function decl) |
+| var_decls.swift:20:7:20:7 | wrappedValue | 0 | var_decls.swift:20:7:20:7 | get |
+| var_decls.swift:20:7:20:7 | wrappedValue | 1 | var_decls.swift:20:7:20:7 | set |
+| var_decls.swift:20:7:20:7 | wrappedValue | 2 | var_decls.swift:20:7:20:7 | (unnamed function decl) |
+| var_decls.swift:24:15:24:15 | _wrapped | 0 | var_decls.swift:24:15:24:15 | get |
+| var_decls.swift:24:15:24:15 | _wrapped | 1 | var_decls.swift:24:15:24:15 | set |
+| var_decls.swift:24:15:24:15 | _wrapped | 2 | var_decls.swift:24:15:24:15 | (unnamed function decl) |
+| var_decls.swift:24:15:24:15 | wrapped | 0 | var_decls.swift:24:15:24:15 | get |
+| var_decls.swift:24:15:24:15 | wrapped | 1 | var_decls.swift:24:15:24:15 | set |
+| var_decls.swift:24:15:24:15 | wrapped | 2 | var_decls.swift:24:15:24:15 | (unnamed function decl) |

swift/ql/test/extractor-tests/generated/decl/ConcreteVarDecl/ConcreteVarDecl_getAccessorDecl.ql

@@ -0,0 +1,7 @@
+// generated by codegen/codegen.py
+import codeql.swift.elements
+import TestUtils
+
+from ConcreteVarDecl x, int index
+where toBeTested(x) and not x.isUnknown()
+select x, index, x.getAccessorDecl(index)

swift/ql/test/extractor-tests/generated/decl/ConcreteVarDecl/ConcreteVarDecl_getAttachedPropertyWrapperType.expected

@@ -0,0 +1 @@
+| var_decls.swift:24:15:24:15 | wrapped | X<Y> |

swift/ql/test/extractor-tests/generated/decl/ConcreteVarDecl/ConcreteVarDecl_getAttachedPropertyWrapperType.ql

@@ -0,0 +1,7 @@
+// generated by codegen/codegen.py
+import codeql.swift.elements
+import TestUtils
+
+from ConcreteVarDecl x
+where toBeTested(x) and not x.isUnknown()
+select x, x.getAttachedPropertyWrapperType()

swift/ql/test/extractor-tests/generated/decl/ConcreteVarDecl/ConcreteVarDecl_getParentInitializer.expected

@@ -0,0 +1,3 @@
+| var_decls.swift:4:7:4:7 | i | var_decls.swift:4:11:4:11 | 0 |
+| var_decls.swift:7:5:7:5 | numbers | var_decls.swift:7:15:7:18 | [...] |
+| var_decls.swift:10:12:10:12 | numbers | var_decls.swift:10:22:10:35 | [...] |

swift/ql/test/extractor-tests/generated/decl/ConcreteVarDecl/ConcreteVarDecl_getParentInitializer.ql

@@ -0,0 +1,7 @@
+// generated by codegen/codegen.py
+import codeql.swift.elements
+import TestUtils
+
+from ConcreteVarDecl x
+where toBeTested(x) and not x.isUnknown()
+select x, x.getParentInitializer()

swift/ql/test/extractor-tests/generated/decl/ConcreteVarDecl/ConcreteVarDecl_getParentPattern.expected

@@ -0,0 +1,7 @@
+| var_decls.swift:4:7:4:7 | i | var_decls.swift:4:7:4:7 | i |
+| var_decls.swift:7:5:7:5 | numbers | var_decls.swift:7:5:7:5 | numbers |
+| var_decls.swift:10:12:10:12 | numbers | var_decls.swift:10:12:10:12 | numbers |
+| var_decls.swift:15:7:15:7 | wrappedValue | var_decls.swift:15:7:15:21 | ... as ... |
+| var_decls.swift:20:7:20:7 | wrappedValue | var_decls.swift:20:7:20:21 | ... as ... |
+| var_decls.swift:24:15:24:15 | _wrapped | var_decls.swift:24:15:24:15 | ... as ... |
+| var_decls.swift:24:15:24:15 | wrapped | var_decls.swift:24:15:24:25 | ... as ... |

swift/ql/test/extractor-tests/generated/decl/ConcreteVarDecl/ConcreteVarDecl_getParentPattern.ql

@@ -0,0 +1,7 @@
+// generated by codegen/codegen.py
+import codeql.swift.elements
+import TestUtils
+
+from ConcreteVarDecl x
+where toBeTested(x) and not x.isUnknown()
+select x, x.getParentPattern()

swift/ql/test/extractor-tests/generated/decl/ConcreteVarDecl/MISSING_SOURCE.txt

@@ -1,4 +0,0 @@
-// generated by codegen/codegen.py
-
-After a swift source file is added in this directory and codegen/codegen.py is run again, test queries
-will appear and this file will be deleted

swift/ql/test/extractor-tests/generated/decl/ConcreteVarDecl/var_decls.swift

@@ -0,0 +1,25 @@
+func loop() {
+  for i in 1...5 {
+  }
+  var i = 0
+}
+
+var numbers = [42]
+
+struct S {
+static let numbers = [42, 404, 101]
+}
+
+@propertyWrapper
+struct X<T> {
+  var wrappedValue: T
+}
+
+@propertyWrapper
+struct Y {
+  var wrappedValue: Int
+}
+
+struct Wrapped {
+    @X @Y var wrapped : Int
+}