CI: add Windows support to fetch-codeql action

Python: Improve `BarrierGuard`

.github/actions/fetch-codeql/action.yml

@@ -13,6 +13,11 @@ runs:
       shell: bash
       run: echo "GA_CODEQL_CLI_PLATFORM=osx64" >> $GITHUB_ENV
 
+    - name: Select platform - Windows
+      if: runner.os == 'Windows'
+      shell: bash
+      run: echo "GA_CODEQL_CLI_PLATFORM=win64" >> $GITHUB_ENV
+
     - name: Fetch CodeQL
       shell: bash
       run: |

cpp/downgrades/23f7cbb88a4eb29f30c3490363dc201bc054c5ff/exprs.ql

@@ -0,0 +1,17 @@
+class Expr extends @expr {
+  string toString() { none() }
+}
+
+class Location extends @location_expr {
+  string toString() { none() }
+}
+
+predicate isExprWithNewBuiltin(Expr expr) {
+  exists(int kind | exprs(expr, kind, _) | 330 <= kind and kind <= 334)
+}
+
+from Expr expr, int kind, int kind_new, Location location
+where
+  exprs(expr, kind, location) and
+  if isExprWithNewBuiltin(expr) then kind_new = 0 else kind_new = kind
+select expr, kind_new, location

cpp/downgrades/23f7cbb88a4eb29f30c3490363dc201bc054c5ff/upgrade.properties

@@ -0,0 +1,3 @@
+description: Add new builtin operations
+compatibility: partial
+exprs.rel: run exprs.qlo

cpp/ql/lib/change-notes/2022-07-26-additional-builtin-support.md

@@ -0,0 +1,4 @@
+---
+category: feature
+---
+* Added subclasses of `BuiltInOperations` for `__builtin_bit_cast`, `__builtin_shuffle`, `__has_unique_object_representations`, `__is_aggregate`, and `__is_assignable`.

cpp/ql/lib/semmle/code/cpp/exprs/BuiltInOperations.qll

@@ -1,5 +1,5 @@
 /**
- * Provides classes for modeling built-in operations.  Built-in operations are
+ * Provides classes for modeling built-in operations. Built-in operations are
  * typically compiler specific and are used by libraries and generated code.
  */
 
@@ -120,8 +120,8 @@ class BuiltInNoOp extends BuiltInOperation, @noopexpr {
 
 /**
  * A C/C++ `__builtin_offsetof` built-in operation (used by some implementations
- * of `offsetof`).  The operation retains its semantics even in the presence
- * of an overloaded `operator &`).  This is a GNU/Clang extension.
+ * of `offsetof`). The operation retains its semantics even in the presence
+ * of an overloaded `operator &`). This is a gcc/clang extension.
  * ```
  * struct S {
  *   int a, b;
@@ -137,8 +137,8 @@ class BuiltInOperationBuiltInOffsetOf extends BuiltInOperation, @offsetofexpr {
 
 /**
  * A C/C++ `__INTADDR__` built-in operation (used by some implementations
- * of `offsetof`).  The operation retains its semantics even in the presence
- * of an overloaded `operator &`).  This is an EDG extension.
+ * of `offsetof`). The operation retains its semantics even in the presence
+ * of an overloaded `operator &`). This is an EDG extension.
  * ```
  * struct S {
  *   int a, b;
@@ -173,7 +173,7 @@ class BuiltInOperationHasAssign extends BuiltInOperation, @hasassignexpr {
  *
  * Returns `true` if the type has a copy constructor.
  * ```
- * std::integral_constant< bool, __has_copy(_Tp)> hc;
+ * std::integral_constant<bool, __has_copy(_Tp)> hc;
  * ```
  */
 class BuiltInOperationHasCopy extends BuiltInOperation, @hascopyexpr {
@@ -189,7 +189,7 @@ class BuiltInOperationHasCopy extends BuiltInOperation, @hascopyexpr {
  * Returns `true` if a copy assignment operator has an empty exception
  * specification.
  * ```
- * std::integral_constant< bool, __has_nothrow_assign(_Tp)> hnta;
+ * std::integral_constant<bool, __has_nothrow_assign(_Tp)> hnta;
  * ```
  */
 class BuiltInOperationHasNoThrowAssign extends BuiltInOperation, @hasnothrowassign {
@@ -220,7 +220,7 @@ class BuiltInOperationHasNoThrowConstructor extends BuiltInOperation, @hasnothro
  *
  * Returns `true` if the copy constructor has an empty exception specification.
  * ```
- * std::integral_constant< bool, __has_nothrow_copy(MyType) >;
+ * std::integral_constant<bool, __has_nothrow_copy(MyType) >;
  * ```
  */
 class BuiltInOperationHasNoThrowCopy extends BuiltInOperation, @hasnothrowcopy {
@@ -266,7 +266,7 @@ class BuiltInOperationHasTrivialConstructor extends BuiltInOperation, @hastrivia
  *
  * Returns true if the type has a trivial copy constructor.
  * ```
- * std::integral_constant< bool, __has_trivial_copy(MyType) > htc;
+ * std::integral_constant<bool, __has_trivial_copy(MyType)> htc;
  * ```
  */
 class BuiltInOperationHasTrivialCopy extends BuiltInOperation, @hastrivialcopy {
@@ -468,7 +468,7 @@ class BuiltInOperationIsUnion extends BuiltInOperation, @isunionexpr {
  * ```
  * template<typename _Tp1, typename _Tp2>
  *   struct types_compatible
- *   : public integral_constant<bool, __builtin_types_compatible_p(_Tp1, _Tp2) >
+ *   : public integral_constant<bool, __builtin_types_compatible_p(_Tp1, _Tp2)>
  *   { };
  * ```
  */
@@ -479,8 +479,7 @@ class BuiltInOperationBuiltInTypesCompatibleP extends BuiltInOperation, @typesco
 /**
  * A clang `__builtin_shufflevector` expression.
  *
- * It outputs a permutation of elements from one or two input vectors.
- * Please see
+ * It outputs a permutation of elements from one or two input vectors. See
  * https://releases.llvm.org/3.7.0/tools/clang/docs/LanguageExtensions.html#langext-builtin-shufflevector
  * for more information.
  * ```
@@ -494,11 +493,29 @@ class BuiltInOperationBuiltInShuffleVector extends BuiltInOperation, @builtinshu
   override string getAPrimaryQlClass() { result = "BuiltInOperationBuiltInShuffleVector" }
 }
 
+/**
+ * A gcc `__builtin_shuffle` expression.
+ *
+ * It outputs a permutation of elements from one or two input vectors.
+ * See https://gcc.gnu.org/onlinedocs/gcc/Vector-Extensions.html
+ * for more information.
+ * ```
+ * // Concatenate every other element of 4-element vectors V1 and V2.
+ * M = {0, 2, 4, 6};
+ * V3 = __builtin_shuffle(V1, V2, M);
+ * ```
+ */
+class BuiltInOperationBuiltInShuffle extends BuiltInOperation, @builtinshuffle {
+  override string toString() { result = "__builtin_shuffle" }
+
+  override string getAPrimaryQlClass() { result = "BuiltInOperationBuiltInShuffle" }
+}
+
 /**
  * A clang `__builtin_convertvector` expression.
  *
  * Allows for conversion of vectors of equal element count and compatible
- * element types. Please see
+ * element types. See
  * https://releases.llvm.org/3.7.0/tools/clang/docs/LanguageExtensions.html#builtin-convertvector
  * for more information.
  * ```
@@ -547,7 +564,7 @@ class BuiltInOperationBuiltInAddressOf extends UnaryOperation, BuiltInOperation,
  * ```
  * template<typename T, typename... Args>
  *   struct is_trivially_constructible
- *   : public integral_constant<bool, __is_trivially_constructible(T, Args...) >
+ *   : public integral_constant<bool, __is_trivially_constructible(T, Args...)>
  *   { };
  * ```
  */
@@ -612,13 +629,10 @@ class BuiltInOperationIsTriviallyDestructible extends BuiltInOperation, @istrivi
  * The `__is_trivially_assignable` built-in operation (used by some
  * implementations of the `<type_traits>` header).
  *
- * Returns `true` if the assignment operator `C::operator =(const C& c)` is
- * trivial.
+ * Returns `true` if the assignment operator `C::operator =(const D& d)` is
+ * trivial (i.e., it will not call any operation that is non-trivial).
  * ```
- * template<typename T>
- *   struct is_trivially_assignable
- *   : public integral_constant<bool, __is_trivially_assignable(T) >
- *   { };
+ * bool v = __is_trivially_assignable(MyType1, MyType2);
  * ```
  */
 class BuiltInOperationIsTriviallyAssignable extends BuiltInOperation, @istriviallyassignableexpr {
@@ -631,10 +645,10 @@ class BuiltInOperationIsTriviallyAssignable extends BuiltInOperation, @istrivial
  * The `__is_nothrow_assignable` built-in operation (used by some
  * implementations of the `<type_traits>` header).
  *
- * Returns true if there exists a `C::operator =(const C& c) nothrow`
+ * Returns true if there exists a `C::operator =(const D& d) nothrow`
  * assignment operator (i.e, with an empty exception specification).
  * ```
- * bool v = __is_nothrow_assignable(MyType);
+ * bool v = __is_nothrow_assignable(MyType1, MyType2);
  * ```
  */
 class BuiltInOperationIsNothrowAssignable extends BuiltInOperation, @isnothrowassignableexpr {
@@ -643,15 +657,30 @@ class BuiltInOperationIsNothrowAssignable extends BuiltInOperation, @isnothrowas
   override string getAPrimaryQlClass() { result = "BuiltInOperationIsNothrowAssignable" }
 }
 
+/**
+ * The `__is_assignable` built-in operation (used by some implementations
+ * of the `<type_traits>` header).
+ *
+ * Returns true if there exists a `C::operator =(const D& d)` assignment
+ * operator.
+ * ```
+ * bool v = __is_assignable(MyType1, MyType2);
+ * ```
+ */
+class BuiltInOperationIsAssignable extends BuiltInOperation, @isassignable {
+  override string toString() { result = "__is_assignable" }
+
+  override string getAPrimaryQlClass() { result = "BuiltInOperationIsAssignable" }
+}
+
 /**
  * The `__is_standard_layout` built-in operation (used by some implementations
  * of the `<type_traits>` header).
  *
  * Returns `true` if the type is a primitive type, or a `class`, `struct` or
- * `union` WITHOUT (1) virtual functions or base classes, (2) reference member
- * variable or (3) multiple occurrences of base `class` objects, among other
- * restrictions.  Please see
- * https://en.cppreference.com/w/cpp/named_req/StandardLayoutType
+ * `union` without (1) virtual functions or base classes, (2) reference member
+ * variable, or (3) multiple occurrences of base `class` objects, among other
+ * restrictions. See https://en.cppreference.com/w/cpp/named_req/StandardLayoutType
  * for more information.
  * ```
  * bool v = __is_standard_layout(MyType);
@@ -668,7 +697,7 @@ class BuiltInOperationIsStandardLayout extends BuiltInOperation, @isstandardlayo
  * implementations of the `<type_traits>` header).
  *
  * Returns `true` if instances of this type can be copied by trivial
- * means.  The copying is done in a manner similar to the `memcpy`
+ * means. The copying is done in a manner similar to the `memcpy`
  * function.
  */
 class BuiltInOperationIsTriviallyCopyable extends BuiltInOperation, @istriviallycopyableexpr {
@@ -682,13 +711,13 @@ class BuiltInOperationIsTriviallyCopyable extends BuiltInOperation, @istrivially
  * the `<type_traits>` header).
  *
  * Returns `true` if the type is a scalar type, a reference type or an array of
- * literal types, among others. Please see
+ * literal types, among others. See
  * https://en.cppreference.com/w/cpp/named_req/LiteralType
  * for more information.
  *
  * ```
  * template <typename _Tp>
- * std::integral_constant< bool, __is_literal_type(_Tp)> ilt;
+ * std::integral_constant<bool, __is_literal_type(_Tp)> ilt;
  * ```
  */
 class BuiltInOperationIsLiteralType extends BuiltInOperation, @isliteraltypeexpr {
@@ -705,7 +734,7 @@ class BuiltInOperationIsLiteralType extends BuiltInOperation, @isliteraltypeexpr
  * compiler, with semantics of the `memcpy` operation.
  * ```
  * template <typename _Tp>
- * std::integral_constant< bool, __has_trivial_move_constructor(_Tp)> htmc;
+ * std::integral_constant<bool, __has_trivial_move_constructor(_Tp)> htmc;
  * ```
  */
 class BuiltInOperationHasTrivialMoveConstructor extends BuiltInOperation,
@@ -723,7 +752,7 @@ class BuiltInOperationHasTrivialMoveConstructor extends BuiltInOperation,
  * ```
  * template<typename T>
  *   struct has_trivial_move_assign
- *   : public integral_constant<bool, __has_trivial_move_assign(T) >
+ *   : public integral_constant<bool, __has_trivial_move_assign(T)>
  *   { };
  * ```
  */
@@ -758,7 +787,7 @@ class BuiltInOperationHasNothrowMoveAssign extends BuiltInOperation, @hasnothrow
  * ```
  * template<typename T, typename... Args>
  *   struct is_constructible
- *   : public integral_constant<bool, __is_constructible(T, Args...) >
+ *   : public integral_constant<bool, __is_constructible(T, Args...)>
  *   { };
  * ```
  */
@@ -785,7 +814,7 @@ class BuiltInOperationIsNothrowConstructible extends BuiltInOperation, @isnothro
 }
 
 /**
- * The `__has_finalizer` built-in operation.  This is a Microsoft extension.
+ * The `__has_finalizer` built-in operation. This is a Microsoft extension.
  *
  * Returns `true` if the type defines a _finalizer_ `C::!C(void)`, to be called
  * from either the regular destructor or the garbage collector.
@@ -800,10 +829,10 @@ class BuiltInOperationHasFinalizer extends BuiltInOperation, @hasfinalizerexpr {
 }
 
 /**
- * The `__is_delegate` built-in operation.  This is a Microsoft extension.
+ * The `__is_delegate` built-in operation. This is a Microsoft extension.
  *
  * Returns `true` if the function has been declared as a `delegate`, used in
- * message forwarding.  Please see
+ * message forwarding. See
  * https://docs.microsoft.com/en-us/cpp/extensions/delegate-cpp-component-extensions
  * for more information.
  */
@@ -814,9 +843,9 @@ class BuiltInOperationIsDelegate extends BuiltInOperation, @isdelegateexpr {
 }
 
 /**
- * The `__is_interface_class` built-in operation.  This is a Microsoft extension.
+ * The `__is_interface_class` built-in operation. This is a Microsoft extension.
  *
- * Returns `true` if the type has been declared as an `interface`.  Please see
+ * Returns `true` if the type has been declared as an `interface`. See
  * https://docs.microsoft.com/en-us/cpp/extensions/interface-class-cpp-component-extensions
  * for more information.
  */
@@ -827,9 +856,9 @@ class BuiltInOperationIsInterfaceClass extends BuiltInOperation, @isinterfacecla
 }
 
 /**
- * The `__is_ref_array` built-in operation.  This is a Microsoft extension.
+ * The `__is_ref_array` built-in operation. This is a Microsoft extension.
  *
- * Returns `true` if the object passed in is a _platform array_.  Please see
+ * Returns `true` if the object passed in is a _platform array_. See
  * https://docs.microsoft.com/en-us/cpp/extensions/arrays-cpp-component-extensions
  * for more information.
  * ```
@@ -844,9 +873,9 @@ class BuiltInOperationIsRefArray extends BuiltInOperation, @isrefarrayexpr {
 }
 
 /**
- * The `__is_ref_class` built-in operation.  This is a Microsoft extension.
+ * The `__is_ref_class` built-in operation. This is a Microsoft extension.
  *
- * Returns `true` if the type is a _reference class_.  Please see
+ * Returns `true` if the type is a _reference class_. See
  * https://docs.microsoft.com/en-us/cpp/extensions/classes-and-structs-cpp-component-extensions
  * for more information.
  * ```
@@ -861,10 +890,10 @@ class BuiltInOperationIsRefClass extends BuiltInOperation, @isrefclassexpr {
 }
 
 /**
- * The `__is_sealed` built-in operation.  This is a Microsoft extension.
+ * The `__is_sealed` built-in operation. This is a Microsoft extension.
  *
  * Returns `true` if a given class or virtual function is marked as `sealed`,
- * meaning that it cannot be extended or overridden.  The `sealed` keyword
+ * meaning that it cannot be extended or overridden. The `sealed` keyword
  * is similar to the C++11 `final` keyword.
  * ```
  * ref class X sealed {
@@ -879,7 +908,7 @@ class BuiltInOperationIsSealed extends BuiltInOperation, @issealedexpr {
 }
 
 /**
- * The `__is_simple_value_class` built-in operation.  This is a Microsoft extension.
+ * The `__is_simple_value_class` built-in operation. This is a Microsoft extension.
  *
  * Returns `true` if passed a value type that contains no references to the
  * garbage-collected heap.
@@ -898,9 +927,9 @@ class BuiltInOperationIsSimpleValueClass extends BuiltInOperation, @issimplevalu
 }
 
 /**
- * The `__is_value_class` built-in operation.  This is a Microsoft extension.
+ * The `__is_value_class` built-in operation. This is a Microsoft extension.
  *
- * Returns `true` if passed a value type.  Please see
+ * Returns `true` if passed a value type. See
  * https://docs.microsoft.com/en-us/cpp/extensions/classes-and-structs-cpp-component-extensions
  * For more information.
  * ```
@@ -922,7 +951,7 @@ class BuiltInOperationIsValueClass extends BuiltInOperation, @isvalueclassexpr {
  * ```
  * template<typename T>
  *   struct is_final
- *   : public integral_constant<bool, __is_final(T) >
+ *   : public integral_constant<bool, __is_final(T)>
  *   { };
  * ```
  */
@@ -933,7 +962,7 @@ class BuiltInOperationIsFinal extends BuiltInOperation, @isfinalexpr {
 }
 
 /**
- * The `__builtin_choose_expr` expression.  This is a GNU/Clang extension.
+ * The `__builtin_choose_expr` expression. This is a gcc/clang extension.
  *
  * The expression functions similarly to the ternary `?:` operator, except
  * that it is evaluated at compile-time.
@@ -978,3 +1007,50 @@ class BuiltInComplexOperation extends BuiltInOperation, @builtincomplex {
   /** Gets the operand corresponding to the imaginary part of the complex number. */
   Expr getImaginaryOperand() { this.hasChild(result, 1) }
 }
+
+/**
+ * A C++ `__is_aggregate` built-in operation (used by some implementations of the
+ * `<type_traits>` header).
+ *
+ * Returns `true` if the type has is an aggregate type.
+ * ```
+ * std::integral_constant<bool, __is_aggregate(_Tp)> ia;
+ * ```
+ */
+class BuiltInOperationIsAggregate extends BuiltInOperation, @isaggregate {
+  override string toString() { result = "__is_aggregate" }
+
+  override string getAPrimaryQlClass() { result = "BuiltInOperationIsAggregate" }
+}
+
+/**
+ * A C++ `__has_unique_object_representations` built-in operation (used by some
+ * implementations of the `<type_traits>` header).
+ *
+ * Returns `true` if the type is trivially copyable and if the object representation
+ * is unique for two objects with the same value.
+ * ```
+ * bool v = __has_unique_object_representations(MyType);
+ * ```
+ */
+class BuiltInOperationHasUniqueObjectRepresentations extends BuiltInOperation,
+  @hasuniqueobjectrepresentations {
+  override string toString() { result = "__has_unique_object_representations" }
+
+  override string getAPrimaryQlClass() { result = "BuiltInOperationHasUniqueObjectRepresentations" }
+}
+
+/**
+ * A C/C++ `__builtin_bit_cast` built-in operation (used by some implementations
+ * of `std::bit_cast`).
+ *
+ * Performs a bit cast from a value to a type.
+ * ```
+ * __builtin_bit_cast(Type, value);
+ * ```
+ */
+class BuiltInBitCast extends BuiltInOperation, @builtinbitcast {
+  override string toString() { result = "__builtin_bit_cast" }
+
+  override string getAPrimaryQlClass() { result = "BuiltInBitCast" }
+}

cpp/ql/lib/semmle/code/cpp/exprs/Expr.qll

@@ -596,9 +596,12 @@ class ParenthesisExpr extends Conversion, @parexpr {
 }
 
 /**
- * A C/C++ expression that has not been resolved.
+ * A C/C++ expression that could not be resolved, or that can no longer be
+ * represented due to a database upgrade or downgrade.
  *
- * It is assigned `ErroneousType` as its type.
+ * If the expression could not be resolved, it has type `ErroneousType`. In the
+ * case of a database upgrade or downgrade, the original type from before the
+ * upgrade or downgrade is kept if that type can be represented.
  */
 class ErrorExpr extends Expr, @errorexpr {
   override string toString() { result = "<error expr>" }

cpp/ql/lib/semmlecode.cpp.dbscheme

@@ -1650,6 +1650,11 @@ case @expr.kind of
 | 327 = @co_await
 | 328 = @co_yield
 | 329 = @temp_init
+| 330 = @isassignable
+| 331 = @isaggregate
+| 332 = @hasuniqueobjectrepresentations
+| 333 = @builtinbitcast
+| 334 = @builtinshuffle
 ;
 
 @var_args_expr = @vastartexpr
@@ -1711,6 +1716,11 @@ case @expr.kind of
             | @isfinalexpr
             | @builtinchooseexpr
             | @builtincomplex
+            | @isassignable
+            | @isaggregate
+            | @hasuniqueobjectrepresentations
+            | @builtinbitcast
+            | @builtinshuffle
             ;
 
 new_allocated_type(

cpp/ql/lib/upgrades/19e31bf071f588bb7efd1e4d5a185ce4f6fbbd84/upgrade.properties

@@ -0,0 +1,2 @@
+description: Add new builtin operations
+compatibility: backwards

cpp/ql/test/library-tests/builtins/edg/edg.c

@@ -1,4 +1,4 @@
-
+// semmle-extractor-options: --clang
 struct mystruct {
     int f1;
     int f2;
@@ -13,3 +13,6 @@ void f(void) {
     int i2 = edg_offsetof(struct mystruct,f2);
 }
 
+void g(void) {
+    double f = __builtin_bit_cast(double,42l);
+}

cpp/ql/test/library-tests/builtins/edg/expr.expected

@@ -13,3 +13,6 @@
 | edg.c:13:14:13:45 | (size_t)... | 0 | 0 |
 | edg.c:13:14:13:45 | __INTADDR__ | 1 | 1 |
 | edg.c:13:43:13:44 | f2 | 0 | 0 |
+| edg.c:17:16:17:45 | __builtin_bit_cast | 1 | 1 |
+| edg.c:17:16:17:45 | double | 0 | 0 |
+| edg.c:17:42:17:44 | 42 | 1 | 1 |

cpp/ql/test/library-tests/builtins/type_traits/expr.expected

@@ -296,3 +296,20 @@
 | ms.cpp:255:24:255:43 | a_struct |  | <none> |
 | ms.cpp:256:24:256:49 | __is_final | a_final_struct | 1 |
 | ms.cpp:256:24:256:49 | a_final_struct |  | <none> |
+| ms.cpp:258:29:258:62 | __is_assignable | a_struct,a_struct | 1 |
+| ms.cpp:258:29:258:62 | a_struct |  | <none> |
+| ms.cpp:258:29:258:62 | a_struct |  | <none> |
+| ms.cpp:259:29:259:59 | __is_assignable | a_struct,empty | 0 |
+| ms.cpp:259:29:259:59 | a_struct |  | <none> |
+| ms.cpp:259:29:259:59 | empty |  | <none> |
+| ms.cpp:260:29:260:57 | __is_assignable | a_struct,int | 0 |
+| ms.cpp:260:29:260:57 | a_struct |  | <none> |
+| ms.cpp:260:29:260:57 | int |  | <none> |
+| ms.cpp:262:28:262:51 | __is_aggregate | a_struct | 1 |
+| ms.cpp:262:28:262:51 | a_struct |  | <none> |
+| ms.cpp:263:28:263:46 | __is_aggregate | int | 0 |
+| ms.cpp:263:28:263:46 | int |  | <none> |
+| ms.cpp:265:49:265:88 | __has_unique_object_representations | int | 1 |
+| ms.cpp:265:49:265:88 | int |  | <none> |
+| ms.cpp:266:49:266:90 | __has_unique_object_representations | float | 0 |
+| ms.cpp:266:49:266:90 | float |  | <none> |

cpp/ql/test/library-tests/builtins/type_traits/ms.cpp

@@ -254,5 +254,14 @@ void f(void) {
 
     bool b_is_final1 = __is_final(a_struct);
     bool b_is_final2 = __is_final(a_final_struct);
-}
 
+    bool b_is_assignable1 = __is_assignable(a_struct,a_struct);
+    bool b_is_assignable2 = __is_assignable(a_struct,empty);
+    bool b_is_assignable3 = __is_assignable(a_struct,int);
+
+    bool b_is_aggregate1 = __is_aggregate(a_struct);
+    bool b_is_aggregate2 = __is_aggregate(int);
+
+    bool b_has_unique_object_representations1 = __has_unique_object_representations(int);
+    bool b_has_unique_object_representations2 = __has_unique_object_representations(float);
+}

cpp/ql/test/library-tests/vector_types/builtin_ops.expected

@@ -1,2 +1,4 @@
+| vector_types2.cpp:10:15:10:42 | __builtin_shuffle |
+| vector_types2.cpp:11:15:11:45 | __builtin_shuffle |
 | vector_types.cpp:31:13:31:49 | __builtin_shufflevector |
 | vector_types.cpp:58:10:58:52 | __builtin_convertvector |

cpp/ql/test/library-tests/vector_types/variables.expected

@@ -13,6 +13,12 @@
 | file://:0:0:0:0 | gp_offset | gp_offset | file://:0:0:0:0 | unsigned int | 4 |
 | file://:0:0:0:0 | overflow_arg_area | overflow_arg_area | file://:0:0:0:0 | void * | 8 |
 | file://:0:0:0:0 | reg_save_area | reg_save_area | file://:0:0:0:0 | void * | 8 |
+| vector_types2.cpp:5:7:5:7 | a | a | vector_types2.cpp:2:13:2:15 | v4i | 16 |
+| vector_types2.cpp:6:7:6:7 | b | b | vector_types2.cpp:2:13:2:15 | v4i | 16 |
+| vector_types2.cpp:7:7:7:12 | mask_1 | mask_1 | vector_types2.cpp:2:13:2:15 | v4i | 16 |
+| vector_types2.cpp:8:7:8:12 | mask_2 | mask_2 | vector_types2.cpp:2:13:2:15 | v4i | 16 |
+| vector_types2.cpp:10:7:10:11 | res_1 | res_1 | vector_types2.cpp:2:13:2:15 | v4i | 16 |
+| vector_types2.cpp:11:7:11:11 | res_2 | res_2 | vector_types2.cpp:2:13:2:15 | v4i | 16 |
 | vector_types.cpp:9:21:9:21 | x | x | vector_types.cpp:6:15:6:17 | v4f | 16 |
 | vector_types.cpp:14:18:14:20 | lhs | lhs | vector_types.cpp:6:15:6:17 | v4f | 16 |
 | vector_types.cpp:14:27:14:29 | rhs | rhs | vector_types.cpp:6:15:6:17 | v4f | 16 |

cpp/ql/test/library-tests/vector_types/vector_types2.cpp

@@ -0,0 +1,12 @@
+// semmle-extractor-options: --gnu --gnu_version 80000
+typedef int v4i __attribute__((vector_size (16)));
+
+void f() {
+  v4i a = {1,2,3,4};
+  v4i b = {5,6,7,8};
+  v4i mask_1 = {3,0,1,2};
+  v4i mask_2 = {3,5,4,2};
+
+  v4i res_1 = __builtin_shuffle(a, mask_1);
+  v4i res_2 = __builtin_shuffle(a, b, mask_2);
+}

java/ql/consistency-queries/typeParametersInScope.ql

@@ -12,6 +12,8 @@ Type getAMentionedType(RefType type) {
   result = getAMentionedType(type).(InstantiatedType).getATypeArgument()
   or
   result = getAMentionedType(type).(NestedType).getEnclosingType()
+  or
+  result = getAMentionedType(type).(Wildcard).getATypeBound().getType()
 }
 
 Type getATypeUsedInClass(RefType type) {

java/ql/test/TestUtilities/InlineExpectationsTestPrivate.qll

@@ -20,3 +20,15 @@ private class KtExpectationComment extends KtComment, ExpectationComment {
 
   override string getContents() { result = this.getText().suffix(2).trim() }
 }
+
+private class XmlExpectationComment extends ExpectationComment instanceof XMLComment {
+  override string getContents() { result = this.(XMLComment).getText().trim() }
+
+  override Location getLocation() { result = this.(XMLComment).getLocation() }
+
+  override predicate hasLocationInfo(string path, int sl, int sc, int el, int ec) {
+    this.(XMLComment).hasLocationInfo(path, sl, sc, el, ec)
+  }
+
+  override string toString() { result = this.(XMLComment).toString() }
+}

java/ql/test/kotlin/library-tests/generic-instance-methods/test.expected

@@ -14,8 +14,8 @@ calls
 | test.kt:22:15:22:33 | setter(...) | test.kt:12:1:25:1 | user | test.kt:0:0:0:0 | TestKt | file:///!unknown-binary-location/Generic.class:0:0:0:0 | setter | file:///!unknown-binary-location/Generic.class:0:0:0:0 | Generic<? super String> |
 | test.kt:23:15:23:22 | getter(...) | test.kt:12:1:25:1 | user | test.kt:0:0:0:0 | TestKt | file:///!unknown-binary-location/Generic.class:0:0:0:0 | getter | file:///!unknown-binary-location/Generic.class:0:0:0:0 | Generic<? super String> |
 constructors
-| Generic2.class:0:0:0:0 | Generic2<? extends String> | Generic2.class:0:0:0:0 | Generic2<? extends String> | Generic2<? extends String>(java.lang.String) | ? extends String | void | Test.java:1:7:1:14 | Generic2 | Test.java:3:10:3:17 | Generic2 |
-| Generic2.class:0:0:0:0 | Generic2<? super String> | Generic2.class:0:0:0:0 | Generic2<? super String> | Generic2<? super String>(java.lang.Object) | ? super String | void | Test.java:1:7:1:14 | Generic2 | Test.java:3:10:3:17 | Generic2 |
+| Generic2.class:0:0:0:0 | Generic2<? extends String> | Generic2.class:0:0:0:0 | Generic2<? extends String> | Generic2<? extends String>(<nulltype>) | <nulltype> | void | Test.java:1:7:1:14 | Generic2 | Test.java:3:10:3:17 | Generic2 |
+| Generic2.class:0:0:0:0 | Generic2<? super String> | Generic2.class:0:0:0:0 | Generic2<? super String> | Generic2<? super String>(java.lang.String) | String | void | Test.java:1:7:1:14 | Generic2 | Test.java:3:10:3:17 | Generic2 |
 | Generic2.class:0:0:0:0 | Generic2<String> | Generic2.class:0:0:0:0 | Generic2<String> | Generic2<String>(java.lang.String) | String | void | Test.java:1:7:1:14 | Generic2 | Test.java:3:10:3:17 | Generic2 |
 | Test.java:1:7:1:14 | Generic2 | Test.java:3:10:3:17 | Generic2 | Generic2(java.lang.Object) | T | void | Test.java:1:7:1:14 | Generic2 | Test.java:3:10:3:17 | Generic2 |
 | Test.java:14:14:14:17 | Test | Test.java:14:14:14:17 | Test | Test() | No parameters | void | Test.java:14:14:14:17 | Test | Test.java:14:14:14:17 | Test |
@@ -34,14 +34,14 @@ refTypes
 | test.kt:1:1:10:1 | Generic |
 | test.kt:1:15:1:15 | T |
 #select
-| Generic2.class:0:0:0:0 | Generic2<? extends String> | Generic2.class:0:0:0:0 | getter | getter() | No parameters | ? extends String | Test.java:1:7:1:14 | Generic2 | Test.java:9:5:9:10 | getter |
-| Generic2.class:0:0:0:0 | Generic2<? extends String> | Generic2.class:0:0:0:0 | identity | identity(java.lang.String) | ? extends String | ? extends String | Test.java:1:7:1:14 | Generic2 | Test.java:8:5:8:12 | identity |
-| Generic2.class:0:0:0:0 | Generic2<? extends String> | Generic2.class:0:0:0:0 | identity2 | identity2(java.lang.String) | ? extends String | ? extends String | Test.java:1:7:1:14 | Generic2 | Test.java:7:5:7:13 | identity2 |
-| Generic2.class:0:0:0:0 | Generic2<? extends String> | Generic2.class:0:0:0:0 | setter | setter(java.lang.String) | ? extends String | void | Test.java:1:7:1:14 | Generic2 | Test.java:10:8:10:13 | setter |
-| Generic2.class:0:0:0:0 | Generic2<? super String> | Generic2.class:0:0:0:0 | getter | getter() | No parameters | ? super String | Test.java:1:7:1:14 | Generic2 | Test.java:9:5:9:10 | getter |
-| Generic2.class:0:0:0:0 | Generic2<? super String> | Generic2.class:0:0:0:0 | identity | identity(java.lang.Object) | ? super String | ? super String | Test.java:1:7:1:14 | Generic2 | Test.java:8:5:8:12 | identity |
-| Generic2.class:0:0:0:0 | Generic2<? super String> | Generic2.class:0:0:0:0 | identity2 | identity2(java.lang.Object) | ? super String | ? super String | Test.java:1:7:1:14 | Generic2 | Test.java:7:5:7:13 | identity2 |
-| Generic2.class:0:0:0:0 | Generic2<? super String> | Generic2.class:0:0:0:0 | setter | setter(java.lang.Object) | ? super String | void | Test.java:1:7:1:14 | Generic2 | Test.java:10:8:10:13 | setter |
+| Generic2.class:0:0:0:0 | Generic2<? extends String> | Generic2.class:0:0:0:0 | getter | getter() | No parameters | String | Test.java:1:7:1:14 | Generic2 | Test.java:9:5:9:10 | getter |
+| Generic2.class:0:0:0:0 | Generic2<? extends String> | Generic2.class:0:0:0:0 | identity | identity(<nulltype>) | <nulltype> | String | Test.java:1:7:1:14 | Generic2 | Test.java:8:5:8:12 | identity |
+| Generic2.class:0:0:0:0 | Generic2<? extends String> | Generic2.class:0:0:0:0 | identity2 | identity2(<nulltype>) | <nulltype> | String | Test.java:1:7:1:14 | Generic2 | Test.java:7:5:7:13 | identity2 |
+| Generic2.class:0:0:0:0 | Generic2<? extends String> | Generic2.class:0:0:0:0 | setter | setter(<nulltype>) | <nulltype> | void | Test.java:1:7:1:14 | Generic2 | Test.java:10:8:10:13 | setter |
+| Generic2.class:0:0:0:0 | Generic2<? super String> | Generic2.class:0:0:0:0 | getter | getter() | No parameters | Object | Test.java:1:7:1:14 | Generic2 | Test.java:9:5:9:10 | getter |
+| Generic2.class:0:0:0:0 | Generic2<? super String> | Generic2.class:0:0:0:0 | identity | identity(java.lang.String) | String | Object | Test.java:1:7:1:14 | Generic2 | Test.java:8:5:8:12 | identity |
+| Generic2.class:0:0:0:0 | Generic2<? super String> | Generic2.class:0:0:0:0 | identity2 | identity2(java.lang.String) | String | Object | Test.java:1:7:1:14 | Generic2 | Test.java:7:5:7:13 | identity2 |
+| Generic2.class:0:0:0:0 | Generic2<? super String> | Generic2.class:0:0:0:0 | setter | setter(java.lang.String) | String | void | Test.java:1:7:1:14 | Generic2 | Test.java:10:8:10:13 | setter |
 | Generic2.class:0:0:0:0 | Generic2<String> | Generic2.class:0:0:0:0 | getter | getter() | No parameters | String | Test.java:1:7:1:14 | Generic2 | Test.java:9:5:9:10 | getter |
 | Generic2.class:0:0:0:0 | Generic2<String> | Generic2.class:0:0:0:0 | identity | identity(java.lang.String) | String | String | Test.java:1:7:1:14 | Generic2 | Test.java:8:5:8:12 | identity |
 | Generic2.class:0:0:0:0 | Generic2<String> | Generic2.class:0:0:0:0 | identity2 | identity2(java.lang.String) | String | String | Test.java:1:7:1:14 | Generic2 | Test.java:7:5:7:13 | identity2 |

java/ql/test/library-tests/wildcard-substitution/Lib.java

@@ -0,0 +1,20 @@
+import java.util.List;
+
+public class Lib<T> {
+
+  public void takesVar(T t) { }
+  public void takesInvar(List<T> lt) { }
+  public void takesUnbound(List<?> lt) { }
+  public void takesExtends(List<? extends T> lt) { }
+  public void takesSuper(List<? super T> lt) { }
+
+  public T returnsVar() { return null; } 
+  public List<T> returnsInvar() { return null; } 
+  public List<?> returnsUnbound() { return null; } 
+  public List<? extends T> returnsExtends() { return null; } 
+  public List<? super T> returnsSuper() { return null; }
+
+  public void takesArray(T[] ts) { }
+  public T[] returnsArray() { return null; }
+
+}

java/ql/test/library-tests/wildcard-substitution/User.java

@@ -0,0 +1,12 @@
+public class User {
+
+  public static void test(Lib<CharSequence> invarLib, Lib<? extends CharSequence> extendsLib, Lib<? super CharSequence> superLib, Lib<?> unboundLib) {
+
+    invarLib.takesVar(null);
+    extendsLib.takesVar(null);
+    superLib.takesVar(null);
+    unboundLib.takesVar(null);
+
+  }
+
+}

java/ql/test/library-tests/wildcard-substitution/test.expected

@@ -0,0 +1,64 @@
+| Lib.class:0:0:0:0 | Lib<? extends CharSequence> | Lib.class:0:0:0:0 | returnsArray | CharSequence[] |
+| Lib.class:0:0:0:0 | Lib<? extends CharSequence> | Lib.class:0:0:0:0 | returnsExtends | List<? extends CharSequence> |
+| Lib.class:0:0:0:0 | Lib<? extends CharSequence> | Lib.class:0:0:0:0 | returnsInvar | List<? extends CharSequence> |
+| Lib.class:0:0:0:0 | Lib<? extends CharSequence> | Lib.class:0:0:0:0 | returnsSuper | List<?> |
+| Lib.class:0:0:0:0 | Lib<? extends CharSequence> | Lib.class:0:0:0:0 | returnsUnbound | List<?> |
+| Lib.class:0:0:0:0 | Lib<? extends CharSequence> | Lib.class:0:0:0:0 | returnsVar | CharSequence |
+| Lib.class:0:0:0:0 | Lib<? extends CharSequence> | Lib.class:0:0:0:0 | takesArray | <nulltype>[] |
+| Lib.class:0:0:0:0 | Lib<? extends CharSequence> | Lib.class:0:0:0:0 | takesExtends | List<? extends CharSequence> |
+| Lib.class:0:0:0:0 | Lib<? extends CharSequence> | Lib.class:0:0:0:0 | takesInvar | List<? extends CharSequence> |
+| Lib.class:0:0:0:0 | Lib<? extends CharSequence> | Lib.class:0:0:0:0 | takesSuper | List<?> |
+| Lib.class:0:0:0:0 | Lib<? extends CharSequence> | Lib.class:0:0:0:0 | takesUnbound | List<?> |
+| Lib.class:0:0:0:0 | Lib<? extends CharSequence> | Lib.class:0:0:0:0 | takesVar | <nulltype> |
+| Lib.class:0:0:0:0 | Lib<? super CharSequence> | Lib.class:0:0:0:0 | returnsArray | Object[] |
+| Lib.class:0:0:0:0 | Lib<? super CharSequence> | Lib.class:0:0:0:0 | returnsExtends | List<?> |
+| Lib.class:0:0:0:0 | Lib<? super CharSequence> | Lib.class:0:0:0:0 | returnsInvar | List<? super CharSequence> |
+| Lib.class:0:0:0:0 | Lib<? super CharSequence> | Lib.class:0:0:0:0 | returnsSuper | List<? super CharSequence> |
+| Lib.class:0:0:0:0 | Lib<? super CharSequence> | Lib.class:0:0:0:0 | returnsUnbound | List<?> |
+| Lib.class:0:0:0:0 | Lib<? super CharSequence> | Lib.class:0:0:0:0 | returnsVar | Object |
+| Lib.class:0:0:0:0 | Lib<? super CharSequence> | Lib.class:0:0:0:0 | takesArray | CharSequence[] |
+| Lib.class:0:0:0:0 | Lib<? super CharSequence> | Lib.class:0:0:0:0 | takesExtends | List<?> |
+| Lib.class:0:0:0:0 | Lib<? super CharSequence> | Lib.class:0:0:0:0 | takesInvar | List<? super CharSequence> |
+| Lib.class:0:0:0:0 | Lib<? super CharSequence> | Lib.class:0:0:0:0 | takesSuper | List<? super CharSequence> |
+| Lib.class:0:0:0:0 | Lib<? super CharSequence> | Lib.class:0:0:0:0 | takesUnbound | List<?> |
+| Lib.class:0:0:0:0 | Lib<? super CharSequence> | Lib.class:0:0:0:0 | takesVar | CharSequence |
+| Lib.class:0:0:0:0 | Lib<?> | Lib.class:0:0:0:0 | returnsArray | Object[] |
+| Lib.class:0:0:0:0 | Lib<?> | Lib.class:0:0:0:0 | returnsExtends | List<? extends Object> |
+| Lib.class:0:0:0:0 | Lib<?> | Lib.class:0:0:0:0 | returnsInvar | List<?> |
+| Lib.class:0:0:0:0 | Lib<?> | Lib.class:0:0:0:0 | returnsSuper | List<?> |
+| Lib.class:0:0:0:0 | Lib<?> | Lib.class:0:0:0:0 | returnsUnbound | List<?> |
+| Lib.class:0:0:0:0 | Lib<?> | Lib.class:0:0:0:0 | returnsVar | Object |
+| Lib.class:0:0:0:0 | Lib<?> | Lib.class:0:0:0:0 | takesArray | <nulltype>[] |
+| Lib.class:0:0:0:0 | Lib<?> | Lib.class:0:0:0:0 | takesExtends | List<? extends Object> |
+| Lib.class:0:0:0:0 | Lib<?> | Lib.class:0:0:0:0 | takesInvar | List<?> |
+| Lib.class:0:0:0:0 | Lib<?> | Lib.class:0:0:0:0 | takesSuper | List<?> |
+| Lib.class:0:0:0:0 | Lib<?> | Lib.class:0:0:0:0 | takesUnbound | List<?> |
+| Lib.class:0:0:0:0 | Lib<?> | Lib.class:0:0:0:0 | takesVar | <nulltype> |
+| Lib.class:0:0:0:0 | Lib<CharSequence> | Lib.class:0:0:0:0 | returnsArray | CharSequence[] |
+| Lib.class:0:0:0:0 | Lib<CharSequence> | Lib.class:0:0:0:0 | returnsExtends | List<? extends CharSequence> |
+| Lib.class:0:0:0:0 | Lib<CharSequence> | Lib.class:0:0:0:0 | returnsInvar | List<CharSequence> |
+| Lib.class:0:0:0:0 | Lib<CharSequence> | Lib.class:0:0:0:0 | returnsSuper | List<? super CharSequence> |
+| Lib.class:0:0:0:0 | Lib<CharSequence> | Lib.class:0:0:0:0 | returnsUnbound | List<?> |
+| Lib.class:0:0:0:0 | Lib<CharSequence> | Lib.class:0:0:0:0 | returnsVar | CharSequence |
+| Lib.class:0:0:0:0 | Lib<CharSequence> | Lib.class:0:0:0:0 | takesArray | CharSequence[] |
+| Lib.class:0:0:0:0 | Lib<CharSequence> | Lib.class:0:0:0:0 | takesExtends | List<? extends CharSequence> |
+| Lib.class:0:0:0:0 | Lib<CharSequence> | Lib.class:0:0:0:0 | takesInvar | List<CharSequence> |
+| Lib.class:0:0:0:0 | Lib<CharSequence> | Lib.class:0:0:0:0 | takesSuper | List<? super CharSequence> |
+| Lib.class:0:0:0:0 | Lib<CharSequence> | Lib.class:0:0:0:0 | takesUnbound | List<?> |
+| Lib.class:0:0:0:0 | Lib<CharSequence> | Lib.class:0:0:0:0 | takesVar | CharSequence |
+| Lib.java:3:14:3:16 | Lib | Lib.java:5:15:5:22 | takesVar | T |
+| Lib.java:3:14:3:16 | Lib | Lib.java:6:15:6:24 | takesInvar | List<T> |
+| Lib.java:3:14:3:16 | Lib | Lib.java:7:15:7:26 | takesUnbound | List<?> |
+| Lib.java:3:14:3:16 | Lib | Lib.java:8:15:8:26 | takesExtends | List<? extends T> |
+| Lib.java:3:14:3:16 | Lib | Lib.java:9:15:9:24 | takesSuper | List<? super T> |
+| Lib.java:3:14:3:16 | Lib | Lib.java:11:12:11:21 | returnsVar | T |
+| Lib.java:3:14:3:16 | Lib | Lib.java:12:18:12:29 | returnsInvar | List<T> |
+| Lib.java:3:14:3:16 | Lib | Lib.java:13:18:13:31 | returnsUnbound | List<?> |
+| Lib.java:3:14:3:16 | Lib | Lib.java:14:28:14:41 | returnsExtends | List<? extends T> |
+| Lib.java:3:14:3:16 | Lib | Lib.java:15:26:15:37 | returnsSuper | List<? super T> |
+| Lib.java:3:14:3:16 | Lib | Lib.java:17:15:17:24 | takesArray | T[] |
+| Lib.java:3:14:3:16 | Lib | Lib.java:18:14:18:25 | returnsArray | T[] |
+| User.java:1:14:1:17 | User | User.java:3:22:3:25 | test | Lib<? extends CharSequence> |
+| User.java:1:14:1:17 | User | User.java:3:22:3:25 | test | Lib<? super CharSequence> |
+| User.java:1:14:1:17 | User | User.java:3:22:3:25 | test | Lib<?> |
+| User.java:1:14:1:17 | User | User.java:3:22:3:25 | test | Lib<CharSequence> |

java/ql/test/library-tests/wildcard-substitution/test.ql

@@ -0,0 +1,7 @@
+import java
+
+Type notVoid(Type t) { result = t and not result instanceof VoidType }
+
+from Callable c
+where c.getSourceDeclaration().fromSource()
+select c.getDeclaringType(), c, notVoid([c.getAParamType(), c.getReturnType()]).toString()

java/ql/test/library-tests/xml/Test.java

@@ -0,0 +1,3 @@
+public class Test {
+
+}

java/ql/test/library-tests/xml/XMLTest.expected

@@ -0,0 +1,2 @@
+| test.xml:4:5:4:32 | attribute=value | Unexpected result: hasXmlResult= |
+| test.xml:5:29:5:52 |  $ hasXmlResult  | Missing result:hasXmlResult= |

java/ql/test/library-tests/xml/XMLTest.ql

@@ -0,0 +1,17 @@
+import semmle.code.xml.XML
+import TestUtilities.InlineExpectationsTest
+
+class XmlTest extends InlineExpectationsTest {
+  XmlTest() { this = "XmlTest" }
+
+  override string getARelevantTag() { result = "hasXmlResult" }
+
+  override predicate hasActualResult(Location location, string element, string tag, string value) {
+    tag = "hasXmlResult" and
+    exists(XMLAttribute a |
+      a.getLocation() = location and
+      element = a.toString() and
+      value = ""
+    )
+  }
+}

java/ql/test/library-tests/xml/test.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<document>
+    <element attribute="value">Text</element> <!-- $ hasXmlResult -->
+    <element attribute="value">Text</element> <!-- Missing -->
+    <element>Text</element> <!-- $ hasXmlResult --> <!-- Spurious -->
+</document>

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPublic.qll

@@ -527,16 +527,55 @@ class StarPatternElementNode extends Node, TStarPatternElementNode {
   override Location getLocation() { result = consumer.getLocation() }
 }
 
+/**
+ * Gets a node that controls whether other nodes are evaluated.
+ *
+ * In the base case, this is the last node of `conditionBlock`, and `flipped` is `false`.
+ * This definition accounts for (short circuting) `and`- and `or`-expressions, as the structure
+ * of basic blocks will reflect their semantics.
+ *
+ * However, in the program
+ * ```python
+ * if not is_safe(path):
+ *   return
+ * ```
+ * the last node in the `ConditionBlock` is `not is_safe(path)`.
+ *
+ * We would like to consider also `is_safe(path)` a guard node, albeit with `flipped` being `true`.
+ * Thus we recurse through `not`-expressions.
+ */
+ControlFlowNode guardNode(ConditionBlock conditionBlock, boolean flipped) {
+  // Base case: the last node truly does determine which successor is chosen
+  result = conditionBlock.getLastNode() and
+  flipped = false
+  or
+  // Recursive case: if a guard node is a `not`-expression,
+  // the operand is also a guard node, but with inverted polarity.
+  exists(UnaryExprNode notNode |
+    result = notNode.getOperand() and
+    notNode.getNode().getOp() instanceof Not
+  |
+    notNode = guardNode(conditionBlock, flipped.booleanNot())
+  )
+}
+
 /**
  * A node that controls whether other nodes are evaluated.
+ *
+ * The field `flipped` allows us to match `GuardNode`s underneath
+ * `not`-expressions and still choose the appropriate branch.
  */
 class GuardNode extends ControlFlowNode {
   ConditionBlock conditionBlock;
+  boolean flipped;
 
-  GuardNode() { this = conditionBlock.getLastNode() }
+  GuardNode() { this = guardNode(conditionBlock, flipped) }
 
   /** Holds if this guard controls block `b` upon evaluating to `branch`. */
-  predicate controlsBlock(BasicBlock b, boolean branch) { conditionBlock.controls(b, branch) }
+  predicate controlsBlock(BasicBlock b, boolean branch) {
+    branch in [true, false] and
+    conditionBlock.controls(b, branch.booleanXor(flipped))
+  }
 }
 
 /**

python/ql/test/experimental/dataflow/tainttracking/commonSanitizer/test_string_const_compare.py

@@ -50,7 +50,7 @@ def test_non_eq2():
     if not ts == "safe":
         ensure_tainted(ts) # $ tainted
     else:
-        ensure_not_tainted(ts) # $ SPURIOUS: tainted
+        ensure_not_tainted(ts)
 
 
 def test_in_list():
@@ -157,7 +157,7 @@ def test_not_in2():
     if not ts in ["safe", "also_safe"]:
         ensure_tainted(ts) # $ tainted
     else:
-        ensure_not_tainted(ts) # $ SPURIOUS: tainted
+        ensure_not_tainted(ts)
 
 
 def is_safe(x):

python/ql/test/experimental/dataflow/tainttracking/customSanitizer/InlineTaintTest.expected

@@ -6,12 +6,20 @@ isSanitizer
 | TestTaintTrackingConfiguration | test.py:34:39:34:39 | ControlFlowNode for s |
 | TestTaintTrackingConfiguration | test.py:52:28:52:28 | ControlFlowNode for s |
 | TestTaintTrackingConfiguration | test.py:66:10:66:29 | ControlFlowNode for emulated_escaping() |
-| TestTaintTrackingConfiguration | test_logical.py:30:28:30:28 | ControlFlowNode for s |
-| TestTaintTrackingConfiguration | test_logical.py:45:28:45:28 | ControlFlowNode for s |
-| TestTaintTrackingConfiguration | test_logical.py:50:28:50:28 | ControlFlowNode for s |
-| TestTaintTrackingConfiguration | test_logical.py:89:28:89:28 | ControlFlowNode for s |
-| TestTaintTrackingConfiguration | test_logical.py:100:28:100:28 | ControlFlowNode for s |
-| TestTaintTrackingConfiguration | test_logical.py:145:28:145:28 | ControlFlowNode for s |
+| TestTaintTrackingConfiguration | test_logical.py:33:28:33:28 | ControlFlowNode for s |
+| TestTaintTrackingConfiguration | test_logical.py:40:28:40:28 | ControlFlowNode for s |
+| TestTaintTrackingConfiguration | test_logical.py:48:28:48:28 | ControlFlowNode for s |
+| TestTaintTrackingConfiguration | test_logical.py:53:28:53:28 | ControlFlowNode for s |
+| TestTaintTrackingConfiguration | test_logical.py:92:28:92:28 | ControlFlowNode for s |
+| TestTaintTrackingConfiguration | test_logical.py:103:28:103:28 | ControlFlowNode for s |
+| TestTaintTrackingConfiguration | test_logical.py:111:28:111:28 | ControlFlowNode for s |
+| TestTaintTrackingConfiguration | test_logical.py:130:28:130:28 | ControlFlowNode for s |
+| TestTaintTrackingConfiguration | test_logical.py:137:28:137:28 | ControlFlowNode for s |
 | TestTaintTrackingConfiguration | test_logical.py:148:28:148:28 | ControlFlowNode for s |
-| TestTaintTrackingConfiguration | test_logical.py:155:28:155:28 | ControlFlowNode for s |
+| TestTaintTrackingConfiguration | test_logical.py:151:28:151:28 | ControlFlowNode for s |
+| TestTaintTrackingConfiguration | test_logical.py:158:28:158:28 | ControlFlowNode for s |
+| TestTaintTrackingConfiguration | test_logical.py:167:24:167:24 | ControlFlowNode for s |
+| TestTaintTrackingConfiguration | test_logical.py:176:24:176:24 | ControlFlowNode for s |
+| TestTaintTrackingConfiguration | test_logical.py:185:24:185:24 | ControlFlowNode for s |
+| TestTaintTrackingConfiguration | test_logical.py:193:24:193:24 | ControlFlowNode for s |
 | TestTaintTrackingConfiguration | test_reference.py:31:28:31:28 | ControlFlowNode for s |

python/ql/test/experimental/dataflow/tainttracking/customSanitizer/InlineTaintTest.ql

@@ -6,6 +6,12 @@ predicate isSafeCheck(DataFlow::GuardNode g, ControlFlowNode node, boolean branc
   branch = true
 }
 
+predicate isUnsafeCheck(DataFlow::GuardNode g, ControlFlowNode node, boolean branch) {
+  g.(CallNode).getNode().getFunc().(Name).getId() in ["is_unsafe", "emulated_is_unsafe"] and
+  node = g.(CallNode).getAnArg() and
+  branch = false
+}
+
 class CustomSanitizerOverrides extends TestTaintTrackingConfiguration {
   override predicate isSanitizer(DataFlow::Node node) {
     exists(Call call |
@@ -16,6 +22,8 @@ class CustomSanitizerOverrides extends TestTaintTrackingConfiguration {
     node.asExpr().(Call).getFunc().(Name).getId() = "emulated_escaping"
     or
     node = DataFlow::BarrierGuard<isSafeCheck/3>::getABarrierNode()
+    or
+    node = DataFlow::BarrierGuard<isUnsafeCheck/3>::getABarrierNode()
   }
 }
 

python/ql/test/experimental/dataflow/tainttracking/customSanitizer/test_logical.py

@@ -22,6 +22,9 @@ def random_choice():
 def is_safe(arg):
     return arg == "safe"
 
+def is_unsafe(arg):
+    return arg == TAINTED_STRING
+
 
 def test_basic():
     s = TAINTED_STRING
@@ -34,7 +37,7 @@ def test_basic():
     if not is_safe(s):
         ensure_tainted(s) # $ tainted
     else:
-        ensure_not_tainted(s) # $ SPURIOUS: tainted
+        ensure_not_tainted(s)
 
 
 def test_if_in_depth():
@@ -105,7 +108,7 @@ def test_and():
         ensure_tainted(s) # $ tainted
     else:
         # cannot be tainted
-        ensure_not_tainted(s) # $ SPURIOUS: tainted
+        ensure_not_tainted(s)
 
 
 def test_tricky():
@@ -124,14 +127,14 @@ def test_nesting_not():
     s = TAINTED_STRING
 
     if not(not(is_safe(s))):
-        ensure_not_tainted(s) # $ SPURIOUS: tainted
+        ensure_not_tainted(s)
     else:
         ensure_tainted(s) # $ tainted
 
     if not(not(not(is_safe(s)))):
         ensure_tainted(s) # $ tainted
     else:
-        ensure_not_tainted(s) # $ SPURIOUS: tainted
+        ensure_not_tainted(s)
 
 
 # Adding `and True` makes the sanitizer trigger when it would otherwise not. See output in
@@ -161,7 +164,16 @@ def test_with_return():
     if not is_safe(s):
         return
 
-    ensure_not_tainted(s) # $ SPURIOUS: tainted
+    ensure_not_tainted(s)
+
+
+def test_with_return_neg():
+    s = TAINTED_STRING
+
+    if is_unsafe(s):
+        return
+
+    ensure_not_tainted(s)
 
 
 def test_with_exception():
@@ -170,7 +182,15 @@ def test_with_exception():
     if not is_safe(s):
         raise Exception("unsafe")
 
-    ensure_not_tainted(s) # $ SPURIOUS: tainted
+    ensure_not_tainted(s)
+
+def test_with_exception_neg():
+    s = TAINTED_STRING
+
+    if is_unsafe(s):
+        raise Exception("unsafe")
+
+    ensure_not_tainted(s)
 
 # Make tests runable
 
@@ -182,7 +202,12 @@ test_tricky()
 test_nesting_not()
 test_nesting_not_with_and_true()
 test_with_return()
+test_with_return_neg()
 try:
     test_with_exception()
 except:
     pass
+try:
+    test_with_exception_neg()
+except:
+    pass

swift/codegen/generators/trapgen.py

@@ -86,11 +86,10 @@ def generate(opts, renderer):
         renderer.render(cpp.TrapList(entries, opts.dbscheme), out / dir / "TrapEntries")
 
     tags = []
-    for index, tag in enumerate(toposort_flatten(tag_graph)):
+    for tag in toposort_flatten(tag_graph):
         tags.append(cpp.Tag(
             name=get_tag_name(tag),
             bases=[get_tag_name(b) for b in sorted(tag_graph[tag])],
-            index=index,
             id=tag,
         ))
     renderer.render(cpp.TagList(tags, opts.dbscheme), out / "TrapTags")

swift/codegen/lib/cpp.py

@@ -83,7 +83,6 @@ class TagBase:
 class Tag:
     name: str
     bases: List[TagBase]
-    index: int
     id: str
 
     def __post_init__(self):

swift/codegen/schema.yml

@@ -112,6 +112,7 @@ LValueType:
 
 ModuleType:
   _extends: Type
+  module: ModuleDecl
 
 PlaceholderType:
   _extends: Type

swift/codegen/templates/trap_tags_h.mustache

@@ -7,7 +7,7 @@ namespace codeql {
 
 // {{id}}
 struct {{name}}Tag {{#has_bases}}: {{#bases}}{{^first}}, {{/first}}{{base}}Tag{{/bases}} {{/has_bases}}{
-  static constexpr const char* prefix = "{{index}}";
+  static constexpr const char* prefix = "{{name}}";
 };
 {{/tags}}
 }

swift/codegen/test/test_cpp.py

@@ -65,7 +65,7 @@ def test_trap_has_first_field_marked():
 def test_tag_has_first_base_marked():
     bases = ["a", "b", "c"]
     expected = [cpp.TagBase("a", first=True), cpp.TagBase("b"), cpp.TagBase("c")]
-    t = cpp.Tag("name", bases, 0, "id")
+    t = cpp.Tag("name", bases, "id")
     assert t.bases == expected
 
 
@@ -75,7 +75,7 @@ def test_tag_has_first_base_marked():
     (["a", "b"], True)
 ])
 def test_tag_has_bases(bases, expected):
-    t = cpp.Tag("name", bases, 0, "id")
+    t = cpp.Tag("name", bases, "id")
     assert t.has_bases is expected
 
 

swift/codegen/test/test_trapgen.py

@@ -162,10 +162,10 @@ def test_one_union_tags(generate_tags):
     assert generate_tags([
         dbscheme.Union(lhs="@left_hand_side", rhs=["@b", "@a", "@c"]),
     ]) == [
-        cpp.Tag(name="LeftHandSide", bases=[], index=0, id="@left_hand_side"),
-        cpp.Tag(name="A", bases=["LeftHandSide"], index=1, id="@a"),
-        cpp.Tag(name="B", bases=["LeftHandSide"], index=2, id="@b"),
-        cpp.Tag(name="C", bases=["LeftHandSide"], index=3, id="@c"),
+        cpp.Tag(name="LeftHandSide", bases=[], id="@left_hand_side"),
+        cpp.Tag(name="A", bases=["LeftHandSide"], id="@a"),
+        cpp.Tag(name="B", bases=["LeftHandSide"], id="@b"),
+        cpp.Tag(name="C", bases=["LeftHandSide"], id="@c"),
     ]
 
 
@@ -175,12 +175,12 @@ def test_multiple_union_tags(generate_tags):
         dbscheme.Union(lhs="@a", rhs=["@b", "@c"]),
         dbscheme.Union(lhs="@e", rhs=["@c", "@f"]),
     ]) == [
-        cpp.Tag(name="D", bases=[], index=0, id="@d"),
-        cpp.Tag(name="E", bases=[], index=1, id="@e"),
-        cpp.Tag(name="A", bases=["D"], index=2, id="@a"),
-        cpp.Tag(name="F", bases=["E"], index=3, id="@f"),
-        cpp.Tag(name="B", bases=["A"], index=4, id="@b"),
-        cpp.Tag(name="C", bases=["A", "E"], index=5, id="@c"),
+        cpp.Tag(name="D", bases=[], id="@d"),
+        cpp.Tag(name="E", bases=[], id="@e"),
+        cpp.Tag(name="A", bases=["D"], id="@a"),
+        cpp.Tag(name="F", bases=["E"], id="@f"),
+        cpp.Tag(name="B", bases=["A"], id="@b"),
+        cpp.Tag(name="C", bases=["A", "E"], id="@c"),
     ]
 
 

swift/extractor/BUILD.bazel

@@ -2,14 +2,10 @@ load("//swift:rules.bzl", "swift_cc_binary")
 
 swift_cc_binary(
     name = "extractor",
-    srcs = [
-        "SwiftOutputRewrite.cpp",
-        "SwiftOutputRewrite.h",
-        "SwiftExtractor.cpp",
-        "SwiftExtractor.h",
-        "SwiftExtractorConfiguration.h",
-        "main.cpp",
-    ],
+    srcs = glob([
+        "*.h",
+        "*.cpp",
+    ]),
     visibility = ["//swift:__pkg__"],
     deps = [
         "//swift/extractor/infra",

swift/extractor/SwiftExtractor.cpp

@@ -14,9 +14,10 @@
 #include "swift/extractor/trap/generated/TrapClasses.h"
 #include "swift/extractor/trap/TrapDomain.h"
 #include "swift/extractor/visitors/SwiftVisitor.h"
-#include "swift/extractor/infra/TargetFile.h"
+#include "swift/extractor/TargetTrapFile.h"
 
 using namespace codeql;
+using namespace std::string_literals;
 
 static void archiveFile(const SwiftExtractorConfiguration& config, swift::SourceFile& file) {
   if (std::error_code ec = llvm::sys::fs::create_directories(config.trapDir)) {
@@ -53,12 +54,18 @@ static std::string getFilename(swift::ModuleDecl& module, swift::SourceFile* pri
   if (primaryFile) {
     return primaryFile->getFilename().str();
   }
-  // Several modules with different name might come from .pcm (clang module) files
-  // In this case we want to differentiate them
-  std::string filename = module.getModuleFilename().str();
-  filename += "-";
-  filename += module.getName().str();
-  return filename;
+  // PCM clang module
+  if (module.isNonSwiftModule()) {
+    // Several modules with different names might come from .pcm (clang module) files
+    // In this case we want to differentiate them
+    // Moreover, pcm files may come from caches located in different directories, but are
+    // unambiguously identified by the base file name, so we can discard the absolute directory
+    std::string filename = "/pcms/"s + llvm::sys::path::filename(module.getModuleFilename()).str();
+    filename += "-";
+    filename += module.getName().str();
+    return filename;
+  }
+  return module.getModuleFilename().str();
 }
 
 static llvm::SmallVector<swift::Decl*> getTopLevelDecls(swift::ModuleDecl& module,
@@ -73,20 +80,6 @@ static llvm::SmallVector<swift::Decl*> getTopLevelDecls(swift::ModuleDecl& modul
   return ret;
 }
 
-static void dumpArgs(TargetFile& out, const SwiftExtractorConfiguration& config) {
-  out << "/* extractor-args:\n";
-  for (const auto& opt : config.frontendOptions) {
-    out << "  " << std::quoted(opt) << " \\\n";
-  }
-  out << "\n*/\n";
-
-  out << "/* swift-frontend-args:\n";
-  for (const auto& opt : config.patchedFrontendOptions) {
-    out << "  " << std::quoted(opt) << " \\\n";
-  }
-  out << "\n*/\n";
-}
-
 static void extractDeclarations(const SwiftExtractorConfiguration& config,
                                 swift::CompilerInstance& compiler,
                                 swift::ModuleDecl& module,
@@ -96,12 +89,11 @@ static void extractDeclarations(const SwiftExtractorConfiguration& config,
   // The extractor can be called several times from different processes with
   // the same input file(s). Using `TargetFile` the first process will win, and the following
   // will just skip the work
-  auto trapTarget = TargetFile::create(filename + ".trap", config.trapDir, config.getTempTrapDir());
+  auto trapTarget = createTargetTrapFile(config, filename);
   if (!trapTarget) {
     // another process arrived first, nothing to do for us
     return;
   }
-  dumpArgs(*trapTarget, config);
   TrapDomain trap{*trapTarget};
 
   // TODO: remove this and recreate it with IPA when we have that
@@ -166,21 +158,21 @@ void codeql::extractSwiftFiles(const SwiftExtractorConfiguration& config,
   auto modules = collectModules(compiler);
 
   for (auto& module : modules) {
-    // We only extract system and builtin modules here as the other "user" modules can be built
-    // during the build process and then re-used at a later stage. In this case, we extract the
-    // user code twice: once during the module build in a form of a source file, and then as
-    // a pre-built module during building of the dependent source files.
-    if (module->isSystemModule() || module->isBuiltinModule()) {
-      extractDeclarations(config, compiler, *module);
-    } else {
-      for (auto file : module->getFiles()) {
-        auto sourceFile = llvm::dyn_cast<swift::SourceFile>(file);
-        if (!sourceFile || inputFiles.count(sourceFile->getFilename().str()) == 0) {
-          continue;
-        }
-        archiveFile(config, *sourceFile);
-        extractDeclarations(config, compiler, *module, sourceFile);
+    bool isFromSourceFile = false;
+    for (auto file : module->getFiles()) {
+      auto sourceFile = llvm::dyn_cast<swift::SourceFile>(file);
+      if (!sourceFile) {
+        continue;
       }
+      isFromSourceFile = true;
+      if (inputFiles.count(sourceFile->getFilename().str()) == 0) {
+        continue;
+      }
+      archiveFile(config, *sourceFile);
+      extractDeclarations(config, compiler, *module, sourceFile);
+    }
+    if (!isFromSourceFile) {
+      extractDeclarations(config, compiler, *module);
     }
   }
 }

swift/extractor/SwiftExtractorConfiguration.h

@@ -3,6 +3,8 @@
 #include <string>
 #include <vector>
 
+#include "swift/extractor/infra/TargetFile.h"
+
 namespace codeql {
 struct SwiftExtractorConfiguration {
   // The location for storing TRAP files to be imported by CodeQL engine.
@@ -33,4 +35,5 @@ struct SwiftExtractorConfiguration {
   // overall extraction process.
   std::string getTempArtifactDir() const { return scratchDir + "/swift-extraction-artifacts"; }
 };
+
 }  // namespace codeql

swift/extractor/SwiftOutputRewrite.cpp

@@ -1,5 +1,6 @@
 #include "SwiftOutputRewrite.h"
 #include "swift/extractor/SwiftExtractorConfiguration.h"
+#include "swift/extractor/TargetTrapFile.h"
 
 #include <llvm/ADT/SmallString.h>
 #include <llvm/Support/FileSystem.h>
@@ -163,7 +164,7 @@ static std::vector<std::string> computeModuleAliases(llvm::StringRef modulePath,
 namespace codeql {
 
 std::unordered_map<std::string, std::string> rewriteOutputsInPlace(
-    SwiftExtractorConfiguration& config,
+    const SwiftExtractorConfiguration& config,
     std::vector<std::string>& CLIArgs) {
   std::unordered_map<std::string, std::string> remapping;
 
@@ -324,4 +325,17 @@ std::vector<std::string> collectVFSFiles(const SwiftExtractorConfiguration& conf
   return overlays;
 }
 
+void lockOutputSwiftModuleTraps(const SwiftExtractorConfiguration& config,
+                                const std::unordered_map<std::string, std::string>& remapping) {
+  for (const auto& [oldPath, newPath] : remapping) {
+    if (llvm::StringRef(oldPath).endswith(".swiftmodule")) {
+      if (auto target = createTargetTrapFile(config, oldPath)) {
+        *target << "// trap file deliberately empty\n"
+                   "// this swiftmodule was created during the build, so its entities must have"
+                   " been extracted directly from source files";
+      }
+    }
+  }
+}
+
 }  // namespace codeql

swift/extractor/SwiftOutputRewrite.h

@@ -13,7 +13,7 @@ struct SwiftExtractorConfiguration;
 // artifacts produced by the actual Swift compiler.
 // Returns the map containing remapping oldpath -> newPath.
 std::unordered_map<std::string, std::string> rewriteOutputsInPlace(
-    SwiftExtractorConfiguration& config,
+    const SwiftExtractorConfiguration& config,
     std::vector<std::string>& CLIArgs);
 
 // Create directories for all the redirected new paths as the Swift compiler expects them to exist.
@@ -29,4 +29,7 @@ void storeRemappingForVFS(const SwiftExtractorConfiguration& config,
 // This is separate from storeRemappingForVFS as we also collect files produced by other processes.
 std::vector<std::string> collectVFSFiles(const SwiftExtractorConfiguration& config);
 
+// Creates empty trap files for output swiftmodule files
+void lockOutputSwiftModuleTraps(const SwiftExtractorConfiguration& config,
+                                const std::unordered_map<std::string, std::string>& remapping);
 }  // namespace codeql

swift/extractor/TargetTrapFile.cpp

@@ -0,0 +1,23 @@
+#include "swift/extractor/TargetTrapFile.h"
+#include <iomanip>
+namespace codeql {
+std::optional<TargetFile> createTargetTrapFile(const SwiftExtractorConfiguration& configuration,
+                                               std::string_view target) {
+  std::string trap{target};
+  trap += ".trap";
+  auto ret = TargetFile::create(trap, configuration.trapDir, configuration.getTempTrapDir());
+  if (ret) {
+    *ret << "/* extractor-args:\n";
+    for (const auto& opt : configuration.frontendOptions) {
+      *ret << "  " << std::quoted(opt) << " \\\n";
+    }
+    *ret << "\n*/\n"
+            "/* swift-frontend-args:\n";
+    for (const auto& opt : configuration.patchedFrontendOptions) {
+      *ret << "  " << std::quoted(opt) << " \\\n";
+    }
+    *ret << "\n*/\n";
+  }
+  return ret;
+}
+}  // namespace codeql

swift/extractor/TargetTrapFile.h

@@ -0,0 +1,11 @@
+#pragma once
+
+#include "swift/extractor/infra/TargetFile.h"
+#include "swift/extractor/SwiftExtractorConfiguration.h"
+
+namespace codeql {
+
+std::optional<TargetFile> createTargetTrapFile(const SwiftExtractorConfiguration& configuration,
+                                               std::string_view target);
+
+}  // namespace codeql

swift/extractor/infra/SwiftDispatcher.h

@@ -215,7 +215,7 @@ class SwiftDispatcher {
 
   template <typename... Args>
   void emitDebugInfo(const Args&... args) {
-    trap.debug(std::forward<Args>(args)...);
+    trap.debug(args...);
   }
 
   // In order to not emit duplicated entries for declarations, we restrict emission to only
@@ -315,7 +315,7 @@ class SwiftDispatcher {
   virtual void visit(swift::TypeBase* type) = 0;
 
   void visit(const FilePath& file) {
-    auto entry = createEntry(file);
+    auto entry = createEntry(file, file.path);
     entry.name = file.path;
     emit(entry);
   }

swift/extractor/main.cpp

@@ -68,6 +68,7 @@ int main(int argc, char** argv) {
       codeql::rewriteOutputsInPlace(configuration, configuration.patchedFrontendOptions);
   codeql::ensureDirectoriesForNewPathsExist(remapping);
   codeql::storeRemappingForVFS(configuration, remapping);
+  codeql::lockOutputSwiftModuleTraps(configuration, remapping);
 
   std::vector<const char*> args;
   for (auto& arg : configuration.patchedFrontendOptions) {

swift/extractor/visitors/DeclVisitor.cpp

@@ -83,11 +83,13 @@ codeql::PrecedenceGroupDecl DeclVisitor::translatePrecedenceGroupDecl(
   return entry;
 }
 
-codeql::ParamDecl DeclVisitor::translateParamDecl(const swift::ParamDecl& decl) {
-  // TODO: deduplicate
-  ParamDecl entry{dispatcher_.assignNewLabel(decl)};
-  fillVarDecl(decl, entry);
-  entry.is_inout = decl.isInOut();
+std::optional<codeql::ParamDecl> DeclVisitor::translateParamDecl(const swift::ParamDecl& decl) {
+  auto entry = createNamedEntry(decl);
+  if (!entry) {
+    return std::nullopt;
+  }
+  fillVarDecl(decl, *entry);
+  entry->is_inout = decl.isInOut();
   return entry;
 }
 
@@ -111,11 +113,20 @@ codeql::PatternBindingDecl DeclVisitor::translatePatternBindingDecl(
   return entry;
 }
 
-codeql::ConcreteVarDecl DeclVisitor::translateVarDecl(const swift::VarDecl& decl) {
-  // TODO: deduplicate all non-local variables
-  ConcreteVarDecl entry{dispatcher_.assignNewLabel(decl)};
-  entry.introducer_int = static_cast<uint8_t>(decl.getIntroducer());
-  fillVarDecl(decl, entry);
+std::optional<codeql::ConcreteVarDecl> DeclVisitor::translateVarDecl(const swift::VarDecl& decl) {
+  std::optional<codeql::ConcreteVarDecl> entry;
+  // We do not deduplicate variables from non-swift (PCM, clang modules) modules as the mangler
+  // crashes sometimes
+  if (decl.getDeclContext()->isLocalContext() || decl.getModuleContext()->isNonSwiftModule()) {
+    entry.emplace(dispatcher_.assignNewLabel(decl));
+  } else {
+    entry = createNamedEntry(decl);
+    if (!entry) {
+      return std::nullopt;
+    }
+  }
+  entry->introducer_int = static_cast<uint8_t>(decl.getIntroducer());
+  fillVarDecl(decl, *entry);
   return entry;
 }
 
@@ -284,11 +295,30 @@ std::string DeclVisitor::mangledName(const swift::ValueDecl& decl) {
   // ASTMangler::mangleAnyDecl crashes when called on `ModuleDecl`
   // TODO find a more unique string working also when different modules are compiled with the same
   // name
+  std::ostringstream ret;
   if (decl.getKind() == swift::DeclKind::Module) {
-    return static_cast<const swift::ModuleDecl&>(decl).getRealName().str().str();
+    ret << static_cast<const swift::ModuleDecl&>(decl).getRealName().str().str();
+  } else if (decl.getKind() == swift::DeclKind::TypeAlias) {
+    // In cases like this (when coming from PCM)
+    //  typealias CFXMLTree = CFTree
+    //  typealias CFXMLTreeRef = CFXMLTree
+    // mangleAnyDecl mangles both CFXMLTree and CFXMLTreeRef into 'So12CFXMLTreeRefa'
+    // which is not correct and causes inconsistencies. mangleEntity makes these two distinct
+    // prefix adds a couple of special symbols, we don't necessary need them
+    ret << mangler.mangleEntity(&decl);
+  } else {
+    // prefix adds a couple of special symbols, we don't necessary need them
+    ret << mangler.mangleAnyDecl(&decl, /* prefix = */ false);
   }
-  // prefix adds a couple of special symbols, we don't necessary need them
-  return mangler.mangleAnyDecl(&decl, /* prefix = */ false);
+  // there can be separate declarations (`VarDecl` or `AccessorDecl`) which are effectively the same
+  // (with equal mangled name) but come from different clang modules. This is the case for example
+  // for glibc constants like `L_SET` that appear in both `SwiftGlibc` and `CDispatch`.
+  // For the moment, we sidestep the problem by making them separate entities in the DB
+  // TODO find a more solid solution
+  if (decl.getModuleContext()->isNonSwiftModule()) {
+    ret << '_' << decl.getModuleContext()->getRealName().str().str();
+  }
+  return ret.str();
 }
 
 void DeclVisitor::fillAbstractFunctionDecl(const swift::AbstractFunctionDecl& decl,

swift/extractor/visitors/DeclVisitor.h

@@ -30,10 +30,10 @@ class DeclVisitor : public AstVisitorBase<DeclVisitor> {
   codeql::PostfixOperatorDecl translatePostfixOperatorDecl(const swift::PostfixOperatorDecl& decl);
   codeql::InfixOperatorDecl translateInfixOperatorDecl(const swift::InfixOperatorDecl& decl);
   codeql::PrecedenceGroupDecl translatePrecedenceGroupDecl(const swift::PrecedenceGroupDecl& decl);
-  codeql::ParamDecl translateParamDecl(const swift::ParamDecl& decl);
+  std::optional<codeql::ParamDecl> translateParamDecl(const swift::ParamDecl& decl);
   codeql::TopLevelCodeDecl translateTopLevelCodeDecl(const swift::TopLevelCodeDecl& decl);
   codeql::PatternBindingDecl translatePatternBindingDecl(const swift::PatternBindingDecl& decl);
-  codeql::ConcreteVarDecl translateVarDecl(const swift::VarDecl& decl);
+  std::optional<codeql::ConcreteVarDecl> translateVarDecl(const swift::VarDecl& decl);
   std::variant<codeql::StructDecl, codeql::StructDeclsTrap> translateStructDecl(
       const swift::StructDecl& decl);
   std::variant<codeql::ClassDecl, codeql::ClassDeclsTrap> translateClassDecl(

swift/extractor/visitors/TypeVisitor.cpp

@@ -381,4 +381,14 @@ codeql::OpenedArchetypeType TypeVisitor::translateOpenedArchetypeType(
   fillArchetypeType(type, entry);
   return entry;
 }
+
+codeql::ModuleType TypeVisitor::translateModuleType(const swift::ModuleType& type) {
+  auto key = type.getModule()->getRealName().str().str();
+  if (type.getModule()->isNonSwiftModule()) {
+    key += "|clang";
+  }
+  auto entry = createTypeEntry(type, key);
+  entry.module = dispatcher_.fetchLabel(type.getModule());
+  return entry;
+}
 }  // namespace codeql

swift/extractor/visitors/TypeVisitor.h

@@ -71,6 +71,7 @@ class TypeVisitor : public TypeVisitorBase<TypeVisitor> {
       const swift::BuiltinUnsafeValueBufferType& type);
   codeql::BuiltinVectorType translateBuiltinVectorType(const swift::BuiltinVectorType& type);
   codeql::OpenedArchetypeType translateOpenedArchetypeType(const swift::OpenedArchetypeType& type);
+  codeql::ModuleType translateModuleType(const swift::ModuleType& type);
 
  private:
   void fillType(const swift::TypeBase& type, codeql::Type& entry);
@@ -83,9 +84,9 @@ class TypeVisitor : public TypeVisitorBase<TypeVisitor> {
   void emitBoundGenericType(swift::BoundGenericType* type, TrapLabel<BoundGenericTypeTag> label);
   void emitAnyGenericType(swift::AnyGenericType* type, TrapLabel<AnyGenericTypeTag> label);
 
-  template <typename T>
-  auto createTypeEntry(const T& type) {
-    auto entry = dispatcher_.createEntry(type);
+  template <typename T, typename... Args>
+  auto createTypeEntry(const T& type, const Args&... args) {
+    auto entry = dispatcher_.createEntry(type, args...);
     fillType(type, entry);
     return entry;
   }

swift/integration-tests/create_database_utils.py

@@ -12,7 +12,7 @@ def run_codeql_database_create(cmds, lang, keep_trap=True):
     codeql_root = pathlib.Path(__file__).parents[2]
     cmd = [
         "codeql", "database", "create",
-        "-s", ".", "-l", "swift", "--internal-use-lua-tracing", f"--search-path={codeql_root}",
+        "-s", ".", "-l", "swift", "--internal-use-lua-tracing", f"--search-path={codeql_root}", "--no-cleanup",
     ]
     if keep_trap:
         cmd.append("--keep-trap")

swift/integration-tests/posix-only/cross-references/VarDecls.expected

@@ -1,5 +1,4 @@
 | Sources/cross-references/lib.swift:10:5:10:5 | X |
-| Sources/cross-references/lib.swift:10:5:10:5 | X |
 | Sources/cross-references/lib.swift:17:16:17:19 | v |
 | Sources/cross-references/lib.swift:22:16:22:19 | v |
 | Sources/cross-references/lib.swift:27:8:27:13 | lhs |

swift/integration-tests/posix-only/partial-modules/Modules.expected

@@ -1,4 +1,5 @@
 | file://:0:0:0:0 | A |
 | file://:0:0:0:0 | B |
+| file://:0:0:0:0 | PackageDescription |
 | file://:0:0:0:0 | main |
 | file://:0:0:0:0 | partial_modules |

swift/integration-tests/runner.py

@@ -62,6 +62,8 @@ def main(opts):
         ]
         if opts.check_databases:
             cmd.append("--check-databases")
+        else:
+            cmd.append("--no-check-databases")
         if opts.learn:
             cmd.append("--learn")
         cmd.extend(str(t.parent) for t in succesful_db_creation)

swift/ql/lib/codeql/swift/generated/type/ModuleType.qll

@@ -1,6 +1,14 @@
 // generated by codegen/codegen.py
+import codeql.swift.elements.decl.ModuleDecl
 import codeql.swift.elements.type.Type
 
 class ModuleTypeBase extends @module_type, Type {
   override string getAPrimaryQlClass() { result = "ModuleType" }
+
+  ModuleDecl getModule() {
+    exists(ModuleDecl x |
+      module_types(this, x) and
+      result = x.resolve()
+    )
+  }
 }

swift/ql/lib/swift.dbscheme

@@ -264,7 +264,8 @@ l_value_types(  //dir=type
 );
 
 module_types(  //dir=type
-  unique int id: @module_type
+  unique int id: @module_type,
+  int module: @module_decl ref
 );
 
 placeholder_types(  //dir=type

swift/ql/test/TestUtils.qll

@@ -4,7 +4,11 @@ cached
 predicate toBeTested(Element e) {
   e instanceof File
   or
-  exists(ModuleDecl m | m = e and not m.isBuiltinModule() and not m.isSystemModule())
+  exists(ModuleDecl m |
+    not m.isBuiltinModule() and
+    not m.isSystemModule() and
+    (m = e or m.getInterfaceType() = e)
+  )
   or
   exists(Locatable loc |
     loc.getLocation().getFile().getName().matches("%swift/ql/test%") and

swift/ql/test/extractor-tests/generated/decl/ConcreteVarDecl/ConcreteVarDecl.expected

@@ -0,0 +1,7 @@
+| var_decls.swift:4:7:4:7 | i | getInterfaceType: | Int | getName: | i | getType: | Int | getIntroducerInt: | 1 |
+| var_decls.swift:7:5:7:5 | numbers | getInterfaceType: | [Int] | getName: | numbers | getType: | [Int] | getIntroducerInt: | 1 |
+| var_decls.swift:10:12:10:12 | numbers | getInterfaceType: | [Int] | getName: | numbers | getType: | [Int] | getIntroducerInt: | 0 |
+| var_decls.swift:15:7:15:7 | wrappedValue | getInterfaceType: | T | getName: | wrappedValue | getType: | T | getIntroducerInt: | 1 |
+| var_decls.swift:20:7:20:7 | wrappedValue | getInterfaceType: | Int | getName: | wrappedValue | getType: | Int | getIntroducerInt: | 1 |
+| var_decls.swift:24:15:24:15 | _wrapped | getInterfaceType: | X<Y> | getName: | _wrapped | getType: | X<Y> | getIntroducerInt: | 1 |
+| var_decls.swift:24:15:24:15 | wrapped | getInterfaceType: | Int | getName: | wrapped | getType: | Int | getIntroducerInt: | 1 |

swift/ql/test/extractor-tests/generated/decl/ConcreteVarDecl/ConcreteVarDecl.ql

@@ -0,0 +1,14 @@
+// generated by codegen/codegen.py
+import codeql.swift.elements
+import TestUtils
+
+from ConcreteVarDecl x, Type getInterfaceType, string getName, Type getType, int getIntroducerInt
+where
+  toBeTested(x) and
+  not x.isUnknown() and
+  getInterfaceType = x.getInterfaceType() and
+  getName = x.getName() and
+  getType = x.getType() and
+  getIntroducerInt = x.getIntroducerInt()
+select x, "getInterfaceType:", getInterfaceType, "getName:", getName, "getType:", getType,
+  "getIntroducerInt:", getIntroducerInt

swift/ql/test/extractor-tests/generated/decl/ConcreteVarDecl/ConcreteVarDecl_getAccessorDecl.expected

@@ -0,0 +1,13 @@
+| var_decls.swift:10:12:10:12 | numbers | 0 | var_decls.swift:10:12:10:12 | get |
+| var_decls.swift:15:7:15:7 | wrappedValue | 0 | var_decls.swift:15:7:15:7 | get |
+| var_decls.swift:15:7:15:7 | wrappedValue | 1 | var_decls.swift:15:7:15:7 | set |
+| var_decls.swift:15:7:15:7 | wrappedValue | 2 | var_decls.swift:15:7:15:7 | (unnamed function decl) |
+| var_decls.swift:20:7:20:7 | wrappedValue | 0 | var_decls.swift:20:7:20:7 | get |
+| var_decls.swift:20:7:20:7 | wrappedValue | 1 | var_decls.swift:20:7:20:7 | set |
+| var_decls.swift:20:7:20:7 | wrappedValue | 2 | var_decls.swift:20:7:20:7 | (unnamed function decl) |
+| var_decls.swift:24:15:24:15 | _wrapped | 0 | var_decls.swift:24:15:24:15 | get |
+| var_decls.swift:24:15:24:15 | _wrapped | 1 | var_decls.swift:24:15:24:15 | set |
+| var_decls.swift:24:15:24:15 | _wrapped | 2 | var_decls.swift:24:15:24:15 | (unnamed function decl) |
+| var_decls.swift:24:15:24:15 | wrapped | 0 | var_decls.swift:24:15:24:15 | get |
+| var_decls.swift:24:15:24:15 | wrapped | 1 | var_decls.swift:24:15:24:15 | set |
+| var_decls.swift:24:15:24:15 | wrapped | 2 | var_decls.swift:24:15:24:15 | (unnamed function decl) |

swift/ql/test/extractor-tests/generated/decl/ConcreteVarDecl/ConcreteVarDecl_getAccessorDecl.ql

@@ -0,0 +1,7 @@
+// generated by codegen/codegen.py
+import codeql.swift.elements
+import TestUtils
+
+from ConcreteVarDecl x, int index
+where toBeTested(x) and not x.isUnknown()
+select x, index, x.getAccessorDecl(index)

swift/ql/test/extractor-tests/generated/decl/ConcreteVarDecl/ConcreteVarDecl_getAttachedPropertyWrapperType.expected

@@ -0,0 +1 @@
+| var_decls.swift:24:15:24:15 | wrapped | X<Y> |

swift/ql/test/extractor-tests/generated/decl/ConcreteVarDecl/ConcreteVarDecl_getAttachedPropertyWrapperType.ql

@@ -0,0 +1,7 @@
+// generated by codegen/codegen.py
+import codeql.swift.elements
+import TestUtils
+
+from ConcreteVarDecl x
+where toBeTested(x) and not x.isUnknown()
+select x, x.getAttachedPropertyWrapperType()

swift/ql/test/extractor-tests/generated/decl/ConcreteVarDecl/ConcreteVarDecl_getParentInitializer.expected

@@ -0,0 +1,3 @@
+| var_decls.swift:4:7:4:7 | i | var_decls.swift:4:11:4:11 | 0 |
+| var_decls.swift:7:5:7:5 | numbers | var_decls.swift:7:15:7:18 | [...] |
+| var_decls.swift:10:12:10:12 | numbers | var_decls.swift:10:22:10:35 | [...] |

swift/ql/test/extractor-tests/generated/decl/ConcreteVarDecl/ConcreteVarDecl_getParentInitializer.ql

@@ -0,0 +1,7 @@
+// generated by codegen/codegen.py
+import codeql.swift.elements
+import TestUtils
+
+from ConcreteVarDecl x
+where toBeTested(x) and not x.isUnknown()
+select x, x.getParentInitializer()

swift/ql/test/extractor-tests/generated/decl/ConcreteVarDecl/ConcreteVarDecl_getParentPattern.expected

@@ -0,0 +1,7 @@
+| var_decls.swift:4:7:4:7 | i | var_decls.swift:4:7:4:7 | i |
+| var_decls.swift:7:5:7:5 | numbers | var_decls.swift:7:5:7:5 | numbers |
+| var_decls.swift:10:12:10:12 | numbers | var_decls.swift:10:12:10:12 | numbers |
+| var_decls.swift:15:7:15:7 | wrappedValue | var_decls.swift:15:7:15:21 | ... as ... |
+| var_decls.swift:20:7:20:7 | wrappedValue | var_decls.swift:20:7:20:21 | ... as ... |
+| var_decls.swift:24:15:24:15 | _wrapped | var_decls.swift:24:15:24:15 | ... as ... |
+| var_decls.swift:24:15:24:15 | wrapped | var_decls.swift:24:15:24:25 | ... as ... |

swift/ql/test/extractor-tests/generated/decl/ConcreteVarDecl/ConcreteVarDecl_getParentPattern.ql

@@ -0,0 +1,7 @@
+// generated by codegen/codegen.py
+import codeql.swift.elements
+import TestUtils
+
+from ConcreteVarDecl x
+where toBeTested(x) and not x.isUnknown()
+select x, x.getParentPattern()

swift/ql/test/extractor-tests/generated/decl/ConcreteVarDecl/MISSING_SOURCE.txt

@@ -1,4 +0,0 @@
-// generated by codegen/codegen.py
-
-After a swift source file is added in this directory and codegen/codegen.py is run again, test queries
-will appear and this file will be deleted

swift/ql/test/extractor-tests/generated/decl/ConcreteVarDecl/var_decls.swift

@@ -0,0 +1,25 @@
+func loop() {
+  for i in 1...5 {
+  }
+  var i = 0
+}
+
+var numbers = [42]
+
+struct S {
+static let numbers = [42, 404, 101]
+}
+
+@propertyWrapper
+struct X<T> {
+  var wrappedValue: T
+}
+
+@propertyWrapper
+struct Y {
+  var wrappedValue: Int
+}
+
+struct Wrapped {
+    @X @Y var wrapped : Int
+}

swift/ql/test/extractor-tests/generated/decl/ModuleDecl/ModuleDecl.expected

@@ -1 +1,2 @@
 | file://:0:0:0:0 | Foo | getInterfaceType: | module<Foo> | getName: | Foo | isBuiltinModule: | no | isSystemModule: | no |
+| file://:0:0:0:0 | default_module_name | getInterfaceType: | module<default_module_name> | getName: | default_module_name | isBuiltinModule: | no | isSystemModule: | no |

swift/ql/test/extractor-tests/generated/decl/ParamDecl/MISSING_SOURCE.txt

@@ -1,4 +0,0 @@
-// generated by codegen/codegen.py
-
-After a swift source file is added in this directory and codegen/codegen.py is run again, test queries
-will appear and this file will be deleted

swift/ql/test/extractor-tests/generated/decl/ParamDecl/ParamDecl.expected

@@ -0,0 +1,15 @@
+| param_decls.swift:1:10:1:13 | _ | getInterfaceType: | Int | getName: | _ | getType: | Int | isInout: | no |
+| param_decls.swift:1:18:1:29 | y | getInterfaceType: | Double | getName: | y | getType: | Double | isInout: | yes |
+| param_decls.swift:2:10:2:13 | _ | getInterfaceType: | Int | getName: | _ | getType: | Int | isInout: | no |
+| param_decls.swift:2:18:2:29 | y | getInterfaceType: | Double | getName: | y | getType: | Double | isInout: | yes |
+| param_decls.swift:5:5:5:5 | self | getInterfaceType: | S | getName: | self | getType: | S | isInout: | yes |
+| param_decls.swift:5:15:5:15 | x | getInterfaceType: | Int | getName: | x | getType: | Int | isInout: | no |
+| param_decls.swift:5:15:5:15 | x | getInterfaceType: | Int | getName: | x | getType: | Int | isInout: | no |
+| param_decls.swift:5:15:5:18 | x | getInterfaceType: | Int | getName: | x | getType: | Int | isInout: | no |
+| param_decls.swift:5:23:5:23 | y | getInterfaceType: | Int | getName: | y | getType: | Int | isInout: | no |
+| param_decls.swift:5:23:5:23 | y | getInterfaceType: | Int | getName: | y | getType: | Int | isInout: | no |
+| param_decls.swift:5:23:5:26 | y | getInterfaceType: | Int | getName: | y | getType: | Int | isInout: | no |
+| param_decls.swift:7:9:7:9 | newValue | getInterfaceType: | Int? | getName: | newValue | getType: | Int? | isInout: | no |
+| param_decls.swift:12:13:12:22 | s | getInterfaceType: | String | getName: | s | getType: | String | isInout: | yes |
+| param_decls.swift:13:13:13:22 | s | getInterfaceType: | String | getName: | s | getType: | String | isInout: | yes |
+| param_decls.swift:14:26:14:26 | $0 | getInterfaceType: | Int | getName: | $0 | getType: | Int | isInout: | no |

swift/ql/test/extractor-tests/generated/decl/ParamDecl/ParamDecl.ql

@@ -0,0 +1,14 @@
+// generated by codegen/codegen.py
+import codeql.swift.elements
+import TestUtils
+
+from ParamDecl x, Type getInterfaceType, string getName, Type getType, string isInout
+where
+  toBeTested(x) and
+  not x.isUnknown() and
+  getInterfaceType = x.getInterfaceType() and
+  getName = x.getName() and
+  getType = x.getType() and
+  if x.isInout() then isInout = "yes" else isInout = "no"
+select x, "getInterfaceType:", getInterfaceType, "getName:", getName, "getType:", getType,
+  "isInout:", isInout

swift/ql/test/extractor-tests/generated/decl/ParamDecl/ParamDecl_getAccessorDecl.ql

@@ -0,0 +1,7 @@
+// generated by codegen/codegen.py
+import codeql.swift.elements
+import TestUtils
+
+from ParamDecl x, int index
+where toBeTested(x) and not x.isUnknown()
+select x, index, x.getAccessorDecl(index)

swift/ql/test/extractor-tests/generated/decl/ParamDecl/ParamDecl_getAttachedPropertyWrapperType.ql

@@ -0,0 +1,7 @@
+// generated by codegen/codegen.py
+import codeql.swift.elements
+import TestUtils
+
+from ParamDecl x
+where toBeTested(x) and not x.isUnknown()
+select x, x.getAttachedPropertyWrapperType()

swift/ql/test/extractor-tests/generated/decl/ParamDecl/ParamDecl_getParentInitializer.ql

@@ -0,0 +1,7 @@
+// generated by codegen/codegen.py
+import codeql.swift.elements
+import TestUtils
+
+from ParamDecl x
+where toBeTested(x) and not x.isUnknown()
+select x, x.getParentInitializer()

swift/ql/test/extractor-tests/generated/decl/ParamDecl/ParamDecl_getParentPattern.ql

@@ -0,0 +1,7 @@
+// generated by codegen/codegen.py
+import codeql.swift.elements
+import TestUtils
+
+from ParamDecl x
+where toBeTested(x) and not x.isUnknown()
+select x, x.getParentPattern()

swift/ql/test/extractor-tests/generated/decl/ParamDecl/param_decls.swift

@@ -0,0 +1,15 @@
+func foo(_: Int, x y: inout Double) {}
+func bar(_: Int, x y: inout Double) {}
+
+struct S {
+    subscript(x: Int, y: Int) -> Int? {
+        get { nil }
+        set {}
+    }
+}
+
+func closures() {
+  let x = {(s: inout String) -> String in s}
+  let y = {(s: inout String) -> String in ""}
+  let z : (Int) -> Int = { $0 + 1 }
+}

swift/ql/test/extractor-tests/generated/type/ModuleType/MISSING_SOURCE.txt

@@ -1,4 +0,0 @@
-// generated by codegen/codegen.py
-
-After a swift source file is added in this directory and codegen/codegen.py is run again, test queries
-will appear and this file will be deleted

swift/ql/test/extractor-tests/generated/type/ModuleType/ModuleType.expected

@@ -0,0 +1,2 @@
+| module<Foo> | getName: | module<Foo> | getCanonicalType: | module<Foo> | getModule: | file://:0:0:0:0 | Foo |
+| module<default_module_name> | getName: | module<default_module_name> | getCanonicalType: | module<default_module_name> | getModule: | file://:0:0:0:0 | default_module_name |

swift/ql/test/extractor-tests/generated/type/ModuleType/ModuleType.ql

@@ -0,0 +1,12 @@
+// generated by codegen/codegen.py
+import codeql.swift.elements
+import TestUtils
+
+from ModuleType x, string getName, Type getCanonicalType, ModuleDecl getModule
+where
+  toBeTested(x) and
+  not x.isUnknown() and
+  getName = x.getName() and
+  getCanonicalType = x.getCanonicalType() and
+  getModule = x.getModule()
+select x, "getName:", getName, "getCanonicalType:", getCanonicalType, "getModule:", getModule

swift/ql/test/extractor-tests/generated/type/ModuleType/modules.swift

@@ -0,0 +1 @@
+//codeql-extractor-options: -module-name Foo