diff --git a/cpp/downgrades/d77c09d8bdc172c9201dec293de1e14c931d3f05/old.dbscheme b/cpp/downgrades/d77c09d8bdc172c9201dec293de1e14c931d3f05/old.dbscheme new file mode 100644 index 00000000000..d77c09d8bdc --- /dev/null +++ b/cpp/downgrades/d77c09d8bdc172c9201dec293de1e14c931d3f05/old.dbscheme @@ -0,0 +1,2212 @@ + +/** + * An invocation of the compiler. Note that more than one file may be + * compiled per invocation. For example, this command compiles three + * source files: + * + * gcc -c f1.c f2.c f3.c + * + * The `id` simply identifies the invocation, while `cwd` is the working + * directory from which the compiler was invoked. + */ +compilations( + /** + * An invocation of the compiler. Note that more than one file may + * be compiled per invocation. For example, this command compiles + * three source files: + * + * gcc -c f1.c f2.c f3.c + */ + unique int id : @compilation, + string cwd : string ref +); + +/** + * The arguments that were passed to the extractor for a compiler + * invocation. If `id` is for the compiler invocation + * + * gcc -c f1.c f2.c f3.c + * + * then typically there will be rows for + * + * num | arg + * --- | --- + * 0 | *path to extractor* + * 1 | `--mimic` + * 2 | `/usr/bin/gcc` + * 3 | `-c` + * 4 | f1.c + * 5 | f2.c + * 6 | f3.c + */ +#keyset[id, num] +compilation_args( + int id : @compilation ref, + int num : int ref, + string arg : string ref +); + +/** + * The source files that are compiled by a compiler invocation. + * If `id` is for the compiler invocation + * + * gcc -c f1.c f2.c f3.c + * + * then there will be rows for + * + * num | arg + * --- | --- + * 0 | f1.c + * 1 | f2.c + * 2 | f3.c + * + * Note that even if those files `#include` headers, those headers + * do not appear as rows. + */ +#keyset[id, num] +compilation_compiling_files( + int id : @compilation ref, + int num : int ref, + int file : @file ref +); + +/** + * The time taken by the extractor for a compiler invocation. + * + * For each file `num`, there will be rows for + * + * kind | seconds + * ---- | --- + * 1 | CPU seconds used by the extractor frontend + * 2 | Elapsed seconds during the extractor frontend + * 3 | CPU seconds used by the extractor backend + * 4 | Elapsed seconds during the extractor backend + */ +#keyset[id, num, kind] +compilation_time( + int id : @compilation ref, + int num : int ref, + /* kind: + 1 = frontend_cpu_seconds + 2 = frontend_elapsed_seconds + 3 = extractor_cpu_seconds + 4 = extractor_elapsed_seconds + */ + int kind : int ref, + float seconds : float ref +); + +/** + * An error or warning generated by the extractor. + * The diagnostic message `diagnostic` was generated during compiler + * invocation `compilation`, and is the `file_number_diagnostic_number`th + * message generated while extracting the `file_number`th file of that + * invocation. + */ +#keyset[compilation, file_number, file_number_diagnostic_number] +diagnostic_for( + int diagnostic : @diagnostic ref, + int compilation : @compilation ref, + int file_number : int ref, + int file_number_diagnostic_number : int ref +); + +/** + * If extraction was successful, then `cpu_seconds` and + * `elapsed_seconds` are the CPU time and elapsed time (respectively) + * that extraction took for compiler invocation `id`. + */ +compilation_finished( + unique int id : @compilation ref, + float cpu_seconds : float ref, + float elapsed_seconds : float ref +); + + +/** + * External data, loaded from CSV files during snapshot creation. See + * [Tutorial: Incorporating external data](https://help.semmle.com/wiki/display/SD/Tutorial%3A+Incorporating+external+data) + * for more information. + */ +externalData( + int id : @externalDataElement, + string path : string ref, + int column: int ref, + string value : string ref +); + +/** + * The source location of the snapshot. + */ +sourceLocationPrefix(string prefix : string ref); + +/** + * Information about packages that provide code used during compilation. + * The `id` is just a unique identifier. + * The `namespace` is typically the name of the package manager that + * provided the package (e.g. "dpkg" or "yum"). + * The `package_name` is the name of the package, and `version` is its + * version (as a string). + */ +external_packages( + unique int id: @external_package, + string namespace : string ref, + string package_name : string ref, + string version : string ref +); + +/** + * Holds if File `fileid` was provided by package `package`. + */ +header_to_external_package( + int fileid : @file ref, + int package : @external_package ref +); + +/* + * Version history + */ + +svnentries( + unique int id : @svnentry, + string revision : string ref, + string author : string ref, + date revisionDate : date ref, + int changeSize : int ref +) + +svnaffectedfiles( + int id : @svnentry ref, + int file : @file ref, + string action : string ref +) + +svnentrymsg( + unique int id : @svnentry ref, + string message : string ref +) + +svnchurn( + int commit : @svnentry ref, + int file : @file ref, + int addedLines : int ref, + int deletedLines : int ref +) + +/* + * C++ dbscheme + */ + +@location = @location_stmt | @location_expr | @location_default ; + +/** + * The location of an element that is not an expression or a statement. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `file`. + * For more information, see + * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/). + */ +locations_default( + /** The location of an element that is not an expression or a statement. */ + unique int id: @location_default, + int container: @container ref, + int startLine: int ref, + int startColumn: int ref, + int endLine: int ref, + int endColumn: int ref +); + +/** + * The location of a statement. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `file`. + * For more information, see + * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/). + */ +locations_stmt( + /** The location of a statement. */ + unique int id: @location_stmt, + int container: @container ref, + int startLine: int ref, + int startColumn: int ref, + int endLine: int ref, + int endColumn: int ref +); + +/** + * The location of an expression. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `file`. + * For more information, see + * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/). + */ +locations_expr( + /** The location of an expression. */ + unique int id: @location_expr, + int container: @container ref, + int startLine: int ref, + int startColumn: int ref, + int endLine: int ref, + int endColumn: int ref +); + +/** An element for which line-count information is available. */ +@sourceline = @file | @function | @variable | @enumconstant | @xmllocatable; + +numlines( + int element_id: @sourceline ref, + int num_lines: int ref, + int num_code: int ref, + int num_comment: int ref +); + +diagnostics( + unique int id: @diagnostic, + int severity: int ref, + string error_tag: string ref, + string error_message: string ref, + string full_error_message: string ref, + int location: @location_default ref +); + +files( + unique int id: @file, + string name: string ref +); + +folders( + unique int id: @folder, + string name: string ref +); + +@container = @folder | @file + +containerparent( + int parent: @container ref, + unique int child: @container ref +); + +fileannotations( + int id: @file ref, + int kind: int ref, + string name: string ref, + string value: string ref +); + +inmacroexpansion( + int id: @element ref, + int inv: @macroinvocation ref +); + +affectedbymacroexpansion( + int id: @element ref, + int inv: @macroinvocation ref +); + +case @macroinvocation.kind of + 1 = @macro_expansion +| 2 = @other_macro_reference +; + +macroinvocations( + unique int id: @macroinvocation, + int macro_id: @ppd_define ref, + int location: @location_default ref, + int kind: int ref +); + +macroparent( + unique int id: @macroinvocation ref, + int parent_id: @macroinvocation ref +); + +// a macroinvocation may be part of another location +// the way to find a constant expression that uses a macro +// is thus to find a constant expression that has a location +// to which a macro invocation is bound +macrolocationbind( + int id: @macroinvocation ref, + int location: @location ref +); + +#keyset[invocation, argument_index] +macro_argument_unexpanded( + int invocation: @macroinvocation ref, + int argument_index: int ref, + string text: string ref +); + +#keyset[invocation, argument_index] +macro_argument_expanded( + int invocation: @macroinvocation ref, + int argument_index: int ref, + string text: string ref +); + +/* +case @function.kind of + 1 = @normal_function +| 2 = @constructor +| 3 = @destructor +| 4 = @conversion_function +| 5 = @operator +| 6 = @builtin_function // GCC built-in functions, e.g. __builtin___memcpy_chk +; +*/ + +functions( + unique int id: @function, + string name: string ref, + int kind: int ref +); + +function_entry_point( + int id: @function ref, + unique int entry_point: @stmt ref +); + +function_return_type( + int id: @function ref, + int return_type: @type ref +); + +/** + * If `function` is a coroutine, then this gives the `std::experimental::resumable_traits` + * instance associated with it, and the variables representing the `handle` and `promise` + * for it. + */ +coroutine( + unique int function: @function ref, + int traits: @type ref, + int handle: @variable ref, + int promise: @variable ref +); + +/** The `new` function used for allocating the coroutine state, if any. */ +coroutine_new( + unique int function: @function ref, + int new: @function ref +); + +/** The `delete` function used for deallocating the coroutine state, if any. */ +coroutine_delete( + unique int function: @function ref, + int delete: @function ref +); + +purefunctions(unique int id: @function ref); + +function_deleted(unique int id: @function ref); + +function_defaulted(unique int id: @function ref); + +member_function_this_type( + unique int id: @function ref, + int this_type: @type ref +); + +#keyset[id, type_id] +fun_decls( + int id: @fun_decl, + int function: @function ref, + int type_id: @type ref, + string name: string ref, + int location: @location_default ref +); +fun_def(unique int id: @fun_decl ref); +fun_specialized(unique int id: @fun_decl ref); +fun_implicit(unique int id: @fun_decl ref); +fun_decl_specifiers( + int id: @fun_decl ref, + string name: string ref +) +#keyset[fun_decl, index] +fun_decl_throws( + int fun_decl: @fun_decl ref, + int index: int ref, + int type_id: @type ref +); +/* an empty throw specification is different from none */ +fun_decl_empty_throws(unique int fun_decl: @fun_decl ref); +fun_decl_noexcept( + int fun_decl: @fun_decl ref, + int constant: @expr ref +); +fun_decl_empty_noexcept(int fun_decl: @fun_decl ref); +fun_decl_typedef_type( + unique int fun_decl: @fun_decl ref, + int typedeftype_id: @usertype ref +); + +param_decl_bind( + unique int id: @var_decl ref, + int index: int ref, + int fun_decl: @fun_decl ref +); + +#keyset[id, type_id] +var_decls( + int id: @var_decl, + int variable: @variable ref, + int type_id: @type ref, + string name: string ref, + int location: @location_default ref +); +var_def(unique int id: @var_decl ref); +var_decl_specifiers( + int id: @var_decl ref, + string name: string ref +) +is_structured_binding(unique int id: @variable ref); + +type_decls( + unique int id: @type_decl, + int type_id: @type ref, + int location: @location_default ref +); +type_def(unique int id: @type_decl ref); +type_decl_top( + unique int type_decl: @type_decl ref +); + +namespace_decls( + unique int id: @namespace_decl, + int namespace_id: @namespace ref, + int location: @location_default ref, + int bodylocation: @location_default ref +); + +usings( + unique int id: @using, + int element_id: @element ref, + int location: @location_default ref +); + +/** The element which contains the `using` declaration. */ +using_container( + int parent: @element ref, + int child: @using ref +); + +static_asserts( + unique int id: @static_assert, + int condition : @expr ref, + string message : string ref, + int location: @location_default ref, + int enclosing : @element ref +); + +// each function has an ordered list of parameters +#keyset[id, type_id] +#keyset[function, index, type_id] +params( + int id: @parameter, + int function: @functionorblock ref, + int index: int ref, + int type_id: @type ref +); + +overrides( + int new: @function ref, + int old: @function ref +); + +#keyset[id, type_id] +membervariables( + int id: @membervariable, + int type_id: @type ref, + string name: string ref +); + +#keyset[id, type_id] +globalvariables( + int id: @globalvariable, + int type_id: @type ref, + string name: string ref +); + +#keyset[id, type_id] +localvariables( + int id: @localvariable, + int type_id: @type ref, + string name: string ref +); + +autoderivation( + unique int var: @variable ref, + int derivation_type: @type ref +); + +orphaned_variables( + int var: @localvariable ref, + int function: @function ref +) + +enumconstants( + unique int id: @enumconstant, + int parent: @usertype ref, + int index: int ref, + int type_id: @type ref, + string name: string ref, + int location: @location_default ref +); + +@variable = @localscopevariable | @globalvariable | @membervariable; + +@localscopevariable = @localvariable | @parameter; + +/** + * Built-in types are the fundamental types, e.g., integral, floating, and void. + */ +case @builtintype.kind of + 1 = @errortype +| 2 = @unknowntype +| 3 = @void +| 4 = @boolean +| 5 = @char +| 6 = @unsigned_char +| 7 = @signed_char +| 8 = @short +| 9 = @unsigned_short +| 10 = @signed_short +| 11 = @int +| 12 = @unsigned_int +| 13 = @signed_int +| 14 = @long +| 15 = @unsigned_long +| 16 = @signed_long +| 17 = @long_long +| 18 = @unsigned_long_long +| 19 = @signed_long_long +// ... 20 Microsoft-specific __int8 +// ... 21 Microsoft-specific __int16 +// ... 22 Microsoft-specific __int32 +// ... 23 Microsoft-specific __int64 +| 24 = @float +| 25 = @double +| 26 = @long_double +| 27 = @complex_float // C99-specific _Complex float +| 28 = @complex_double // C99-specific _Complex double +| 29 = @complex_long_double // C99-specific _Complex long double +| 30 = @imaginary_float // C99-specific _Imaginary float +| 31 = @imaginary_double // C99-specific _Imaginary double +| 32 = @imaginary_long_double // C99-specific _Imaginary long double +| 33 = @wchar_t // Microsoft-specific +| 34 = @decltype_nullptr // C++11 +| 35 = @int128 // __int128 +| 36 = @unsigned_int128 // unsigned __int128 +| 37 = @signed_int128 // signed __int128 +| 38 = @float128 // __float128 +| 39 = @complex_float128 // _Complex __float128 +| 40 = @decimal32 // _Decimal32 +| 41 = @decimal64 // _Decimal64 +| 42 = @decimal128 // _Decimal128 +| 43 = @char16_t +| 44 = @char32_t +| 45 = @std_float32 // _Float32 +| 46 = @float32x // _Float32x +| 47 = @std_float64 // _Float64 +| 48 = @float64x // _Float64x +| 49 = @std_float128 // _Float128 +// ... 50 _Float128x +| 51 = @char8_t +| 52 = @float16 // _Float16 +| 53 = @complex_float16 // _Complex _Float16 +; + +builtintypes( + unique int id: @builtintype, + string name: string ref, + int kind: int ref, + int size: int ref, + int sign: int ref, + int alignment: int ref +); + +/** + * Derived types are types that are directly derived from existing types and + * point to, refer to, transform type data to return a new type. + */ +case @derivedtype.kind of + 1 = @pointer +| 2 = @reference +| 3 = @type_with_specifiers +| 4 = @array +| 5 = @gnu_vector +| 6 = @routineptr +| 7 = @routinereference +| 8 = @rvalue_reference // C++11 +// ... 9 type_conforming_to_protocols deprecated +| 10 = @block +; + +derivedtypes( + unique int id: @derivedtype, + string name: string ref, + int kind: int ref, + int type_id: @type ref +); + +pointerishsize(unique int id: @derivedtype ref, + int size: int ref, + int alignment: int ref); + +arraysizes( + unique int id: @derivedtype ref, + int num_elements: int ref, + int bytesize: int ref, + int alignment: int ref +); + +typedefbase( + unique int id: @usertype ref, + int type_id: @type ref +); + +/** + * An instance of the C++11 `decltype` operator. For example: + * ``` + * int a; + * decltype(1+a) b; + * ``` + * Here `expr` is `1+a`. + * + * Sometimes an additional pair of parentheses around the expression + * would change the semantics of this decltype, e.g. + * ``` + * struct A { double x; }; + * const A* a = new A(); + * decltype( a->x ); // type is double + * decltype((a->x)); // type is const double& + * ``` + * (Please consult the C++11 standard for more details). + * `parentheses_would_change_meaning` is `true` iff that is the case. + */ +#keyset[id, expr] +decltypes( + int id: @decltype, + int expr: @expr ref, + int base_type: @type ref, + boolean parentheses_would_change_meaning: boolean ref +); + +/* +case @usertype.kind of + 1 = @struct +| 2 = @class +| 3 = @union +| 4 = @enum +| 5 = @typedef // classic C: typedef typedef type name +| 6 = @template +| 7 = @template_parameter +| 8 = @template_template_parameter +| 9 = @proxy_class // a proxy class associated with a template parameter +// ... 10 objc_class deprecated +// ... 11 objc_protocol deprecated +// ... 12 objc_category deprecated +| 13 = @scoped_enum +| 14 = @using_alias // a using name = type style typedef +; +*/ + +usertypes( + unique int id: @usertype, + string name: string ref, + int kind: int ref +); + +usertypesize( + unique int id: @usertype ref, + int size: int ref, + int alignment: int ref +); + +usertype_final(unique int id: @usertype ref); + +usertype_uuid( + unique int id: @usertype ref, + string uuid: string ref +); + +mangled_name( + unique int id: @declaration ref, + int mangled_name : @mangledname +); + +is_pod_class(unique int id: @usertype ref); +is_standard_layout_class(unique int id: @usertype ref); + +is_complete(unique int id: @usertype ref); + +is_class_template(unique int id: @usertype ref); +class_instantiation( + int to: @usertype ref, + int from: @usertype ref +); +class_template_argument( + int type_id: @usertype ref, + int index: int ref, + int arg_type: @type ref +); +class_template_argument_value( + int type_id: @usertype ref, + int index: int ref, + int arg_value: @expr ref +); + +is_proxy_class_for( + unique int id: @usertype ref, + unique int templ_param_id: @usertype ref +); + +type_mentions( + unique int id: @type_mention, + int type_id: @type ref, + int location: @location ref, + // a_symbol_reference_kind from the frontend. + int kind: int ref +); + +is_function_template(unique int id: @function ref); +function_instantiation( + unique int to: @function ref, + int from: @function ref +); +function_template_argument( + int function_id: @function ref, + int index: int ref, + int arg_type: @type ref +); +function_template_argument_value( + int function_id: @function ref, + int index: int ref, + int arg_value: @expr ref +); + +is_variable_template(unique int id: @variable ref); +variable_instantiation( + unique int to: @variable ref, + int from: @variable ref +); +variable_template_argument( + int variable_id: @variable ref, + int index: int ref, + int arg_type: @type ref +); +variable_template_argument_value( + int variable_id: @variable ref, + int index: int ref, + int arg_value: @expr ref +); + +/* + Fixed point types + precision(1) = short, precision(2) = default, precision(3) = long + is_unsigned(1) = unsigned is_unsigned(2) = signed + is_fract_type(1) = declared with _Fract + saturating(1) = declared with _Sat +*/ +/* TODO +fixedpointtypes( + unique int id: @fixedpointtype, + int precision: int ref, + int is_unsigned: int ref, + int is_fract_type: int ref, + int saturating: int ref); +*/ + +routinetypes( + unique int id: @routinetype, + int return_type: @type ref +); + +routinetypeargs( + int routine: @routinetype ref, + int index: int ref, + int type_id: @type ref +); + +ptrtomembers( + unique int id: @ptrtomember, + int type_id: @type ref, + int class_id: @type ref +); + +/* + specifiers for types, functions, and variables + + "public", + "protected", + "private", + + "const", + "volatile", + "static", + + "pure", + "virtual", + "sealed", // Microsoft + "__interface", // Microsoft + "inline", + "explicit", + + "near", // near far extension + "far", // near far extension + "__ptr32", // Microsoft + "__ptr64", // Microsoft + "__sptr", // Microsoft + "__uptr", // Microsoft + "dllimport", // Microsoft + "dllexport", // Microsoft + "thread", // Microsoft + "naked", // Microsoft + "microsoft_inline", // Microsoft + "forceinline", // Microsoft + "selectany", // Microsoft + "nothrow", // Microsoft + "novtable", // Microsoft + "noreturn", // Microsoft + "noinline", // Microsoft + "noalias", // Microsoft + "restrict", // Microsoft +*/ + +specifiers( + unique int id: @specifier, + unique string str: string ref +); + +typespecifiers( + int type_id: @type ref, + int spec_id: @specifier ref +); + +funspecifiers( + int func_id: @function ref, + int spec_id: @specifier ref +); + +varspecifiers( + int var_id: @accessible ref, + int spec_id: @specifier ref +); + +attributes( + unique int id: @attribute, + int kind: int ref, + string name: string ref, + string name_space: string ref, + int location: @location_default ref +); + +case @attribute.kind of + 0 = @gnuattribute +| 1 = @stdattribute +| 2 = @declspec +| 3 = @msattribute +| 4 = @alignas +// ... 5 @objc_propertyattribute deprecated +; + +attribute_args( + unique int id: @attribute_arg, + int kind: int ref, + int attribute: @attribute ref, + int index: int ref, + int location: @location_default ref +); + +case @attribute_arg.kind of + 0 = @attribute_arg_empty +| 1 = @attribute_arg_token +| 2 = @attribute_arg_constant +| 3 = @attribute_arg_type +| 4 = @attribute_arg_constant_expr +; + +attribute_arg_value( + unique int arg: @attribute_arg ref, + string value: string ref +); +attribute_arg_type( + unique int arg: @attribute_arg ref, + int type_id: @type ref +); +attribute_arg_constant( + unique int arg: @attribute_arg ref, + int constant: @expr ref +) +attribute_arg_name( + unique int arg: @attribute_arg ref, + string name: string ref +); + +typeattributes( + int type_id: @type ref, + int spec_id: @attribute ref +); + +funcattributes( + int func_id: @function ref, + int spec_id: @attribute ref +); + +varattributes( + int var_id: @accessible ref, + int spec_id: @attribute ref +); + +stmtattributes( + int stmt_id: @stmt ref, + int spec_id: @attribute ref +); + +@type = @builtintype + | @derivedtype + | @usertype + /* TODO | @fixedpointtype */ + | @routinetype + | @ptrtomember + | @decltype; + +unspecifiedtype( + unique int type_id: @type ref, + int unspecified_type_id: @type ref +); + +member( + int parent: @type ref, + int index: int ref, + int child: @member ref +); + +@enclosingfunction_child = @usertype | @variable | @namespace + +enclosingfunction( + unique int child: @enclosingfunction_child ref, + int parent: @function ref +); + +derivations( + unique int derivation: @derivation, + int sub: @type ref, + int index: int ref, + int super: @type ref, + int location: @location_default ref +); + +derspecifiers( + int der_id: @derivation ref, + int spec_id: @specifier ref +); + +/** + * Contains the byte offset of the base class subobject within the derived + * class. Only holds for non-virtual base classes, but see table + * `virtual_base_offsets` for offsets of virtual base class subobjects. + */ +direct_base_offsets( + unique int der_id: @derivation ref, + int offset: int ref +); + +/** + * Contains the byte offset of the virtual base class subobject for class + * `super` within a most-derived object of class `sub`. `super` can be either a + * direct or indirect base class. + */ +#keyset[sub, super] +virtual_base_offsets( + int sub: @usertype ref, + int super: @usertype ref, + int offset: int ref +); + +frienddecls( + unique int id: @frienddecl, + int type_id: @type ref, + int decl_id: @declaration ref, + int location: @location_default ref +); + +@declaredtype = @usertype ; + +@declaration = @function + | @declaredtype + | @variable + | @enumconstant + | @frienddecl; + +@member = @membervariable + | @function + | @declaredtype + | @enumconstant; + +@locatable = @diagnostic + | @declaration + | @ppd_include + | @ppd_define + | @macroinvocation + /*| @funcall*/ + | @xmllocatable + | @attribute + | @attribute_arg; + +@namedscope = @namespace | @usertype; + +@element = @locatable + | @file + | @folder + | @specifier + | @type + | @expr + | @namespace + | @initialiser + | @stmt + | @derivation + | @comment + | @preprocdirect + | @fun_decl + | @var_decl + | @type_decl + | @namespace_decl + | @using + | @namequalifier + | @specialnamequalifyingelement + | @static_assert + | @type_mention + | @lambdacapture; + +@exprparent = @element; + +comments( + unique int id: @comment, + string contents: string ref, + int location: @location_default ref +); + +commentbinding( + int id: @comment ref, + int element: @element ref +); + +exprconv( + int converted: @expr ref, + unique int conversion: @expr ref +); + +compgenerated(unique int id: @element ref); + +/** + * `destructor_call` destructs the `i`'th entity that should be + * destructed following `element`. Note that entities should be + * destructed in reverse construction order, so for a given `element` + * these should be called from highest to lowest `i`. + */ +#keyset[element, destructor_call] +#keyset[element, i] +synthetic_destructor_call( + int element: @element ref, + int i: int ref, + int destructor_call: @routineexpr ref +); + +namespaces( + unique int id: @namespace, + string name: string ref +); + +namespace_inline( + unique int id: @namespace ref +); + +namespacembrs( + int parentid: @namespace ref, + unique int memberid: @namespacembr ref +); + +@namespacembr = @declaration | @namespace; + +exprparents( + int expr_id: @expr ref, + int child_index: int ref, + int parent_id: @exprparent ref +); + +expr_isload(unique int expr_id: @expr ref); + +@cast = @c_style_cast + | @const_cast + | @dynamic_cast + | @reinterpret_cast + | @static_cast + ; + +/* +case @conversion.kind of + 0 = @simple_conversion // a numeric conversion, qualification conversion, or a reinterpret_cast +| 1 = @bool_conversion // conversion to 'bool' +| 2 = @base_class_conversion // a derived-to-base conversion +| 3 = @derived_class_conversion // a base-to-derived conversion +| 4 = @pm_base_class_conversion // a derived-to-base conversion of a pointer to member +| 5 = @pm_derived_class_conversion // a base-to-derived conversion of a pointer to member +| 6 = @glvalue_adjust // an adjustment of the type of a glvalue +| 7 = @prvalue_adjust // an adjustment of the type of a prvalue +; +*/ +/** + * Describes the semantics represented by a cast expression. This is largely + * independent of the source syntax of the cast, so it is separate from the + * regular expression kind. + */ +conversionkinds( + unique int expr_id: @cast ref, + int kind: int ref +); + +@conversion = @cast + | @array_to_pointer + | @parexpr + | @reference_to + | @ref_indirect + | @temp_init + ; + +/* +case @funbindexpr.kind of + 0 = @normal_call // a normal call +| 1 = @virtual_call // a virtual call +| 2 = @adl_call // a call whose target is only found by ADL +; +*/ +iscall( + unique int caller: @funbindexpr ref, + int kind: int ref +); + +numtemplatearguments( + unique int expr_id: @expr ref, + int num: int ref +); + +specialnamequalifyingelements( + unique int id: @specialnamequalifyingelement, + unique string name: string ref +); + +@namequalifiableelement = @expr | @namequalifier; +@namequalifyingelement = @namespace + | @specialnamequalifyingelement + | @usertype; + +namequalifiers( + unique int id: @namequalifier, + unique int qualifiableelement: @namequalifiableelement ref, + int qualifyingelement: @namequalifyingelement ref, + int location: @location_default ref +); + +varbind( + int expr: @varbindexpr ref, + int var: @accessible ref +); + +funbind( + int expr: @funbindexpr ref, + int fun: @function ref +); + +@any_new_expr = @new_expr + | @new_array_expr; + +@new_or_delete_expr = @any_new_expr + | @delete_expr + | @delete_array_expr; + +@prefix_crement_expr = @preincrexpr | @predecrexpr; + +@postfix_crement_expr = @postincrexpr | @postdecrexpr; + +@increment_expr = @preincrexpr | @postincrexpr; + +@decrement_expr = @predecrexpr | @postdecrexpr; + +@crement_expr = @increment_expr | @decrement_expr; + +@un_arith_op_expr = @arithnegexpr + | @unaryplusexpr + | @conjugation + | @realpartexpr + | @imagpartexpr + | @crement_expr + ; + +@un_bitwise_op_expr = @complementexpr; + +@un_log_op_expr = @notexpr; + +@un_op_expr = @address_of + | @indirect + | @un_arith_op_expr + | @un_bitwise_op_expr + | @builtinaddressof + | @vec_fill + | @un_log_op_expr + | @co_await + | @co_yield + ; + +@bin_log_op_expr = @andlogicalexpr | @orlogicalexpr; + +@cmp_op_expr = @eq_op_expr | @rel_op_expr; + +@eq_op_expr = @eqexpr | @neexpr; + +@rel_op_expr = @gtexpr + | @ltexpr + | @geexpr + | @leexpr + | @spaceshipexpr + ; + +@bin_bitwise_op_expr = @lshiftexpr + | @rshiftexpr + | @andexpr + | @orexpr + | @xorexpr + ; + +@p_arith_op_expr = @paddexpr + | @psubexpr + | @pdiffexpr + ; + +@bin_arith_op_expr = @addexpr + | @subexpr + | @mulexpr + | @divexpr + | @remexpr + | @jmulexpr + | @jdivexpr + | @fjaddexpr + | @jfaddexpr + | @fjsubexpr + | @jfsubexpr + | @minexpr + | @maxexpr + | @p_arith_op_expr + ; + +@bin_op_expr = @bin_arith_op_expr + | @bin_bitwise_op_expr + | @cmp_op_expr + | @bin_log_op_expr + ; + +@op_expr = @un_op_expr + | @bin_op_expr + | @assign_expr + | @conditionalexpr + ; + +@assign_arith_expr = @assignaddexpr + | @assignsubexpr + | @assignmulexpr + | @assigndivexpr + | @assignremexpr + ; + +@assign_bitwise_expr = @assignandexpr + | @assignorexpr + | @assignxorexpr + | @assignlshiftexpr + | @assignrshiftexpr + | @assignpaddexpr + | @assignpsubexpr + ; + +@assign_op_expr = @assign_arith_expr | @assign_bitwise_expr + +@assign_expr = @assignexpr | @assign_op_expr | @blockassignexpr + +/* + case @allocator.form of + 0 = plain + | 1 = alignment + ; +*/ + +/** + * The allocator function associated with a `new` or `new[]` expression. + * The `form` column specified whether the allocation call contains an alignment + * argument. + */ +expr_allocator( + unique int expr: @any_new_expr ref, + int func: @function ref, + int form: int ref +); + +/* + case @deallocator.form of + 0 = plain + | 1 = size + | 2 = alignment + | 3 = size_and_alignment + ; +*/ + +/** + * The deallocator function associated with a `delete`, `delete[]`, `new`, or + * `new[]` expression. For a `new` or `new[]` expression, the deallocator is the + * one used to free memory if the initialization throws an exception. + * The `form` column specifies whether the deallocation call contains a size + * argument, and alignment argument, or both. + */ +expr_deallocator( + unique int expr: @new_or_delete_expr ref, + int func: @function ref, + int form: int ref +); + +/** + * Holds if the `@conditionalexpr` is of the two operand form + * `guard ? : false`. + */ +expr_cond_two_operand( + unique int cond: @conditionalexpr ref +); + +/** + * The guard of `@conditionalexpr` `guard ? true : false` + */ +expr_cond_guard( + unique int cond: @conditionalexpr ref, + int guard: @expr ref +); + +/** + * The expression used when the guard of `@conditionalexpr` + * `guard ? true : false` holds. For the two operand form + * `guard ?: false` consider using `expr_cond_guard` instead. + */ +expr_cond_true( + unique int cond: @conditionalexpr ref, + int true: @expr ref +); + +/** + * The expression used when the guard of `@conditionalexpr` + * `guard ? true : false` does not hold. + */ +expr_cond_false( + unique int cond: @conditionalexpr ref, + int false: @expr ref +); + +/** A string representation of the value. */ +values( + unique int id: @value, + string str: string ref +); + +/** The actual text in the source code for the value, if any. */ +valuetext( + unique int id: @value ref, + string text: string ref +); + +valuebind( + int val: @value ref, + unique int expr: @expr ref +); + +fieldoffsets( + unique int id: @variable ref, + int byteoffset: int ref, + int bitoffset: int ref +); + +bitfield( + unique int id: @variable ref, + int bits: int ref, + int declared_bits: int ref +); + +/* TODO +memberprefix( + int member: @expr ref, + int prefix: @expr ref +); +*/ + +/* + kind(1) = mbrcallexpr + kind(2) = mbrptrcallexpr + kind(3) = mbrptrmbrcallexpr + kind(4) = ptrmbrptrmbrcallexpr + kind(5) = mbrreadexpr // x.y + kind(6) = mbrptrreadexpr // p->y + kind(7) = mbrptrmbrreadexpr // x.*pm + kind(8) = mbrptrmbrptrreadexpr // x->*pm + kind(9) = staticmbrreadexpr // static x.y + kind(10) = staticmbrptrreadexpr // static p->y +*/ +/* TODO +memberaccess( + int member: @expr ref, + int kind: int ref +); +*/ + +initialisers( + unique int init: @initialiser, + int var: @accessible ref, + unique int expr: @expr ref, + int location: @location_expr ref +); + +braced_initialisers( + int init: @initialiser ref +); + +/** + * An ancestor for the expression, for cases in which we cannot + * otherwise find the expression's parent. + */ +expr_ancestor( + int exp: @expr ref, + int ancestor: @element ref +); + +exprs( + unique int id: @expr, + int kind: int ref, + int location: @location_expr ref +); + +/* + case @value.category of + 1 = prval + | 2 = xval + | 3 = lval + ; +*/ +expr_types( + int id: @expr ref, + int typeid: @type ref, + int value_category: int ref +); + +case @expr.kind of + 1 = @errorexpr +| 2 = @address_of // & AddressOfExpr +| 3 = @reference_to // ReferenceToExpr (implicit?) +| 4 = @indirect // * PointerDereferenceExpr +| 5 = @ref_indirect // ReferenceDereferenceExpr (implicit?) +// ... +| 8 = @array_to_pointer // (???) +| 9 = @vacuous_destructor_call // VacuousDestructorCall +// ... +| 11 = @assume // Microsoft +| 12 = @parexpr +| 13 = @arithnegexpr +| 14 = @unaryplusexpr +| 15 = @complementexpr +| 16 = @notexpr +| 17 = @conjugation // GNU ~ operator +| 18 = @realpartexpr // GNU __real +| 19 = @imagpartexpr // GNU __imag +| 20 = @postincrexpr +| 21 = @postdecrexpr +| 22 = @preincrexpr +| 23 = @predecrexpr +| 24 = @conditionalexpr +| 25 = @addexpr +| 26 = @subexpr +| 27 = @mulexpr +| 28 = @divexpr +| 29 = @remexpr +| 30 = @jmulexpr // C99 mul imaginary +| 31 = @jdivexpr // C99 div imaginary +| 32 = @fjaddexpr // C99 add real + imaginary +| 33 = @jfaddexpr // C99 add imaginary + real +| 34 = @fjsubexpr // C99 sub real - imaginary +| 35 = @jfsubexpr // C99 sub imaginary - real +| 36 = @paddexpr // pointer add (pointer + int or int + pointer) +| 37 = @psubexpr // pointer sub (pointer - integer) +| 38 = @pdiffexpr // difference between two pointers +| 39 = @lshiftexpr +| 40 = @rshiftexpr +| 41 = @andexpr +| 42 = @orexpr +| 43 = @xorexpr +| 44 = @eqexpr +| 45 = @neexpr +| 46 = @gtexpr +| 47 = @ltexpr +| 48 = @geexpr +| 49 = @leexpr +| 50 = @minexpr // GNU minimum +| 51 = @maxexpr // GNU maximum +| 52 = @assignexpr +| 53 = @assignaddexpr +| 54 = @assignsubexpr +| 55 = @assignmulexpr +| 56 = @assigndivexpr +| 57 = @assignremexpr +| 58 = @assignlshiftexpr +| 59 = @assignrshiftexpr +| 60 = @assignandexpr +| 61 = @assignorexpr +| 62 = @assignxorexpr +| 63 = @assignpaddexpr // assign pointer add +| 64 = @assignpsubexpr // assign pointer sub +| 65 = @andlogicalexpr +| 66 = @orlogicalexpr +| 67 = @commaexpr +| 68 = @subscriptexpr // access to member of an array, e.g., a[5] +// ... 69 @objc_subscriptexpr deprecated +// ... 70 @cmdaccess deprecated +// ... +| 73 = @virtfunptrexpr +| 74 = @callexpr +// ... 75 @msgexpr_normal deprecated +// ... 76 @msgexpr_super deprecated +// ... 77 @atselectorexpr deprecated +// ... 78 @atprotocolexpr deprecated +| 79 = @vastartexpr +| 80 = @vaargexpr +| 81 = @vaendexpr +| 82 = @vacopyexpr +// ... 83 @atencodeexpr deprecated +| 84 = @varaccess +| 85 = @thisaccess +// ... 86 @objc_box_expr deprecated +| 87 = @new_expr +| 88 = @delete_expr +| 89 = @throw_expr +| 90 = @condition_decl // a variable declared in a condition, e.g., if(int x = y > 2) +| 91 = @braced_init_list +| 92 = @type_id +| 93 = @runtime_sizeof +| 94 = @runtime_alignof +| 95 = @sizeof_pack +| 96 = @expr_stmt // GNU extension +| 97 = @routineexpr +| 98 = @type_operand // used to access a type in certain contexts (haven't found any examples yet....) +| 99 = @offsetofexpr // offsetof ::= type and field +| 100 = @hasassignexpr // __has_assign ::= type +| 101 = @hascopyexpr // __has_copy ::= type +| 102 = @hasnothrowassign // __has_nothrow_assign ::= type +| 103 = @hasnothrowconstr // __has_nothrow_constructor ::= type +| 104 = @hasnothrowcopy // __has_nothrow_copy ::= type +| 105 = @hastrivialassign // __has_trivial_assign ::= type +| 106 = @hastrivialconstr // __has_trivial_constructor ::= type +| 107 = @hastrivialcopy // __has_trivial_copy ::= type +| 108 = @hasuserdestr // __has_user_destructor ::= type +| 109 = @hasvirtualdestr // __has_virtual_destructor ::= type +| 110 = @isabstractexpr // __is_abstract ::= type +| 111 = @isbaseofexpr // __is_base_of ::= type type +| 112 = @isclassexpr // __is_class ::= type +| 113 = @isconvtoexpr // __is_convertible_to ::= type type +| 114 = @isemptyexpr // __is_empty ::= type +| 115 = @isenumexpr // __is_enum ::= type +| 116 = @ispodexpr // __is_pod ::= type +| 117 = @ispolyexpr // __is_polymorphic ::= type +| 118 = @isunionexpr // __is_union ::= type +| 119 = @typescompexpr // GNU __builtin_types_compatible ::= type type +| 120 = @intaddrexpr // frontend internal builtin, used to implement offsetof +// ... +| 122 = @hastrivialdestructor // __has_trivial_destructor ::= type +| 123 = @literal +| 124 = @uuidof +| 127 = @aggregateliteral +| 128 = @delete_array_expr +| 129 = @new_array_expr +// ... 130 @objc_array_literal deprecated +// ... 131 @objc_dictionary_literal deprecated +| 132 = @foldexpr +// ... +| 200 = @ctordirectinit +| 201 = @ctorvirtualinit +| 202 = @ctorfieldinit +| 203 = @ctordelegatinginit +| 204 = @dtordirectdestruct +| 205 = @dtorvirtualdestruct +| 206 = @dtorfielddestruct +// ... +| 210 = @static_cast +| 211 = @reinterpret_cast +| 212 = @const_cast +| 213 = @dynamic_cast +| 214 = @c_style_cast +| 215 = @lambdaexpr +| 216 = @param_ref +| 217 = @noopexpr +// ... +| 294 = @istriviallyconstructibleexpr +| 295 = @isdestructibleexpr +| 296 = @isnothrowdestructibleexpr +| 297 = @istriviallydestructibleexpr +| 298 = @istriviallyassignableexpr +| 299 = @isnothrowassignableexpr +| 300 = @istrivialexpr +| 301 = @isstandardlayoutexpr +| 302 = @istriviallycopyableexpr +| 303 = @isliteraltypeexpr +| 304 = @hastrivialmoveconstructorexpr +| 305 = @hastrivialmoveassignexpr +| 306 = @hasnothrowmoveassignexpr +| 307 = @isconstructibleexpr +| 308 = @isnothrowconstructibleexpr +| 309 = @hasfinalizerexpr +| 310 = @isdelegateexpr +| 311 = @isinterfaceclassexpr +| 312 = @isrefarrayexpr +| 313 = @isrefclassexpr +| 314 = @issealedexpr +| 315 = @issimplevalueclassexpr +| 316 = @isvalueclassexpr +| 317 = @isfinalexpr +| 319 = @noexceptexpr +| 320 = @builtinshufflevector +| 321 = @builtinchooseexpr +| 322 = @builtinaddressof +| 323 = @vec_fill +| 324 = @builtinconvertvector +| 325 = @builtincomplex +| 326 = @spaceshipexpr +| 327 = @co_await +| 328 = @co_yield +| 329 = @temp_init +| 330 = @isassignable +| 331 = @isaggregate +| 332 = @hasuniqueobjectrepresentations +| 333 = @builtinbitcast +| 334 = @builtinshuffle +| 335 = @blockassignexpr +| 336 = @issame +| 337 = @isfunction +| 338 = @islayoutcompatible +| 339 = @ispointerinterconvertiblebaseof +| 340 = @isarray +| 341 = @arrayrank +| 342 = @arrayextent +| 343 = @isarithmetic +| 344 = @iscompletetype +| 345 = @iscompound +| 346 = @isconst +| 347 = @isfloatingpoint +| 348 = @isfundamental +| 349 = @isintegral +| 350 = @islvaluereference +| 351 = @ismemberfunctionpointer +| 352 = @ismemberobjectpointer +| 353 = @ismemberpointer +| 354 = @isobject +| 355 = @ispointer +| 356 = @isreference +| 357 = @isrvaluereference +| 358 = @isscalar +| 359 = @issigned +| 360 = @isunsigned +| 361 = @isvoid +| 362 = @isvolatile +; + +@var_args_expr = @vastartexpr + | @vaendexpr + | @vaargexpr + | @vacopyexpr + ; + +@builtin_op = @var_args_expr + | @noopexpr + | @offsetofexpr + | @intaddrexpr + | @hasassignexpr + | @hascopyexpr + | @hasnothrowassign + | @hasnothrowconstr + | @hasnothrowcopy + | @hastrivialassign + | @hastrivialconstr + | @hastrivialcopy + | @hastrivialdestructor + | @hasuserdestr + | @hasvirtualdestr + | @isabstractexpr + | @isbaseofexpr + | @isclassexpr + | @isconvtoexpr + | @isemptyexpr + | @isenumexpr + | @ispodexpr + | @ispolyexpr + | @isunionexpr + | @typescompexpr + | @builtinshufflevector + | @builtinconvertvector + | @builtinaddressof + | @istriviallyconstructibleexpr + | @isdestructibleexpr + | @isnothrowdestructibleexpr + | @istriviallydestructibleexpr + | @istriviallyassignableexpr + | @isnothrowassignableexpr + | @isstandardlayoutexpr + | @istriviallycopyableexpr + | @isliteraltypeexpr + | @hastrivialmoveconstructorexpr + | @hastrivialmoveassignexpr + | @hasnothrowmoveassignexpr + | @isconstructibleexpr + | @isnothrowconstructibleexpr + | @hasfinalizerexpr + | @isdelegateexpr + | @isinterfaceclassexpr + | @isrefarrayexpr + | @isrefclassexpr + | @issealedexpr + | @issimplevalueclassexpr + | @isvalueclassexpr + | @isfinalexpr + | @builtinchooseexpr + | @builtincomplex + | @isassignable + | @isaggregate + | @hasuniqueobjectrepresentations + | @builtinbitcast + | @builtinshuffle + | @issame + | @isfunction + | @islayoutcompatible + | @ispointerinterconvertiblebaseof + | @isarray + | @arrayrank + | @arrayextent + | @isarithmetic + | @iscompletetype + | @iscompound + | @isconst + | @isfloatingpoint + | @isfundamental + | @isintegral + | @islvaluereference + | @ismemberfunctionpointer + | @ismemberobjectpointer + | @ismemberpointer + | @isobject + | @ispointer + | @isreference + | @isrvaluereference + | @isscalar + | @issigned + | @isunsigned + | @isvoid + | @isvolatile + ; + +new_allocated_type( + unique int expr: @new_expr ref, + int type_id: @type ref +); + +new_array_allocated_type( + unique int expr: @new_array_expr ref, + int type_id: @type ref +); + +/** + * The field being initialized by an initializer expression within an aggregate + * initializer for a class/struct/union. Position is used to sort repeated initializers. + */ +#keyset[aggregate, position] +aggregate_field_init( + int aggregate: @aggregateliteral ref, + int initializer: @expr ref, + int field: @membervariable ref, + int position: int ref +); + +/** + * The index of the element being initialized by an initializer expression + * within an aggregate initializer for an array. Position is used to sort repeated initializers. + */ +#keyset[aggregate, position] +aggregate_array_init( + int aggregate: @aggregateliteral ref, + int initializer: @expr ref, + int element_index: int ref, + int position: int ref +); + +@ctorinit = @ctordirectinit + | @ctorvirtualinit + | @ctorfieldinit + | @ctordelegatinginit; +@dtordestruct = @dtordirectdestruct + | @dtorvirtualdestruct + | @dtorfielddestruct; + + +condition_decl_bind( + unique int expr: @condition_decl ref, + unique int decl: @declaration ref +); + +typeid_bind( + unique int expr: @type_id ref, + int type_id: @type ref +); + +uuidof_bind( + unique int expr: @uuidof ref, + int type_id: @type ref +); + +@runtime_sizeof_or_alignof = @runtime_sizeof | @runtime_alignof; + +sizeof_bind( + unique int expr: @runtime_sizeof_or_alignof ref, + int type_id: @type ref +); + +code_block( + unique int block: @literal ref, + unique int routine: @function ref +); + +lambdas( + unique int expr: @lambdaexpr ref, + string default_capture: string ref, + boolean has_explicit_return_type: boolean ref +); + +lambda_capture( + unique int id: @lambdacapture, + int lambda: @lambdaexpr ref, + int index: int ref, + int field: @membervariable ref, + boolean captured_by_reference: boolean ref, + boolean is_implicit: boolean ref, + int location: @location_default ref +); + +@funbindexpr = @routineexpr + | @new_expr + | @delete_expr + | @delete_array_expr + | @ctordirectinit + | @ctorvirtualinit + | @ctordelegatinginit + | @dtordirectdestruct + | @dtorvirtualdestruct; + +@varbindexpr = @varaccess | @ctorfieldinit | @dtorfielddestruct; +@addressable = @function | @variable ; +@accessible = @addressable | @enumconstant ; + +@access = @varaccess | @routineexpr ; + +fold( + int expr: @foldexpr ref, + string operator: string ref, + boolean is_left_fold: boolean ref +); + +stmts( + unique int id: @stmt, + int kind: int ref, + int location: @location_stmt ref +); + +case @stmt.kind of + 1 = @stmt_expr +| 2 = @stmt_if +| 3 = @stmt_while +| 4 = @stmt_goto +| 5 = @stmt_label +| 6 = @stmt_return +| 7 = @stmt_block +| 8 = @stmt_end_test_while // do { ... } while ( ... ) +| 9 = @stmt_for +| 10 = @stmt_switch_case +| 11 = @stmt_switch +| 13 = @stmt_asm // "asm" statement or the body of an asm function +| 15 = @stmt_try_block +| 16 = @stmt_microsoft_try // Microsoft +| 17 = @stmt_decl +| 18 = @stmt_set_vla_size // C99 +| 19 = @stmt_vla_decl // C99 +| 25 = @stmt_assigned_goto // GNU +| 26 = @stmt_empty +| 27 = @stmt_continue +| 28 = @stmt_break +| 29 = @stmt_range_based_for // C++11 +// ... 30 @stmt_at_autoreleasepool_block deprecated +// ... 31 @stmt_objc_for_in deprecated +// ... 32 @stmt_at_synchronized deprecated +| 33 = @stmt_handler +// ... 34 @stmt_finally_end deprecated +| 35 = @stmt_constexpr_if +| 37 = @stmt_co_return +; + +type_vla( + int type_id: @type ref, + int decl: @stmt_vla_decl ref +); + +variable_vla( + int var: @variable ref, + int decl: @stmt_vla_decl ref +); + +if_initialization( + unique int if_stmt: @stmt_if ref, + int init_id: @stmt ref +); + +if_then( + unique int if_stmt: @stmt_if ref, + int then_id: @stmt ref +); + +if_else( + unique int if_stmt: @stmt_if ref, + int else_id: @stmt ref +); + +constexpr_if_initialization( + unique int constexpr_if_stmt: @stmt_constexpr_if ref, + int init_id: @stmt ref +); + +constexpr_if_then( + unique int constexpr_if_stmt: @stmt_constexpr_if ref, + int then_id: @stmt ref +); + +constexpr_if_else( + unique int constexpr_if_stmt: @stmt_constexpr_if ref, + int else_id: @stmt ref +); + +while_body( + unique int while_stmt: @stmt_while ref, + int body_id: @stmt ref +); + +do_body( + unique int do_stmt: @stmt_end_test_while ref, + int body_id: @stmt ref +); + +switch_initialization( + unique int switch_stmt: @stmt_switch ref, + int init_id: @stmt ref +); + +#keyset[switch_stmt, index] +switch_case( + int switch_stmt: @stmt_switch ref, + int index: int ref, + int case_id: @stmt_switch_case ref +); + +switch_body( + unique int switch_stmt: @stmt_switch ref, + int body_id: @stmt ref +); + +for_initialization( + unique int for_stmt: @stmt_for ref, + int init_id: @stmt ref +); + +for_condition( + unique int for_stmt: @stmt_for ref, + int condition_id: @expr ref +); + +for_update( + unique int for_stmt: @stmt_for ref, + int update_id: @expr ref +); + +for_body( + unique int for_stmt: @stmt_for ref, + int body_id: @stmt ref +); + +@stmtparent = @stmt | @expr_stmt ; +stmtparents( + unique int id: @stmt ref, + int index: int ref, + int parent: @stmtparent ref +); + +ishandler(unique int block: @stmt_block ref); + +@cfgnode = @stmt | @expr | @function | @initialiser ; + +stmt_decl_bind( + int stmt: @stmt_decl ref, + int num: int ref, + int decl: @declaration ref +); + +stmt_decl_entry_bind( + int stmt: @stmt_decl ref, + int num: int ref, + int decl_entry: @element ref +); + +@functionorblock = @function | @stmt_block; + +blockscope( + unique int block: @stmt_block ref, + int enclosing: @functionorblock ref +); + +@jump = @stmt_goto | @stmt_break | @stmt_continue; + +@jumporlabel = @jump | @stmt_label | @literal; + +jumpinfo( + unique int id: @jumporlabel ref, + string str: string ref, + int target: @stmt ref +); + +preprocdirects( + unique int id: @preprocdirect, + int kind: int ref, + int location: @location_default ref +); +case @preprocdirect.kind of + 0 = @ppd_if +| 1 = @ppd_ifdef +| 2 = @ppd_ifndef +| 3 = @ppd_elif +| 4 = @ppd_else +| 5 = @ppd_endif +| 6 = @ppd_plain_include +| 7 = @ppd_define +| 8 = @ppd_undef +| 9 = @ppd_line +| 10 = @ppd_error +| 11 = @ppd_pragma +| 12 = @ppd_objc_import +| 13 = @ppd_include_next +| 18 = @ppd_warning +; + +@ppd_include = @ppd_plain_include | @ppd_objc_import | @ppd_include_next; + +@ppd_branch = @ppd_if | @ppd_ifdef | @ppd_ifndef | @ppd_elif; + +preprocpair( + int begin : @ppd_branch ref, + int elseelifend : @preprocdirect ref +); + +preproctrue(int branch : @ppd_branch ref); +preprocfalse(int branch : @ppd_branch ref); + +preproctext( + unique int id: @preprocdirect ref, + string head: string ref, + string body: string ref +); + +includes( + unique int id: @ppd_include ref, + int included: @file ref +); + +link_targets( + unique int id: @link_target, + int binary: @file ref +); + +link_parent( + int element : @element ref, + int link_target : @link_target ref +); + +/* XML Files */ + +xmlEncoding(unique int id: @file ref, string encoding: string ref); + +xmlDTDs( + unique int id: @xmldtd, + string root: string ref, + string publicId: string ref, + string systemId: string ref, + int fileid: @file ref +); + +xmlElements( + unique int id: @xmlelement, + string name: string ref, + int parentid: @xmlparent ref, + int idx: int ref, + int fileid: @file ref +); + +xmlAttrs( + unique int id: @xmlattribute, + int elementid: @xmlelement ref, + string name: string ref, + string value: string ref, + int idx: int ref, + int fileid: @file ref +); + +xmlNs( + int id: @xmlnamespace, + string prefixName: string ref, + string URI: string ref, + int fileid: @file ref +); + +xmlHasNs( + int elementId: @xmlnamespaceable ref, + int nsId: @xmlnamespace ref, + int fileid: @file ref +); + +xmlComments( + unique int id: @xmlcomment, + string text: string ref, + int parentid: @xmlparent ref, + int fileid: @file ref +); + +xmlChars( + unique int id: @xmlcharacters, + string text: string ref, + int parentid: @xmlparent ref, + int idx: int ref, + int isCDATA: int ref, + int fileid: @file ref +); + +@xmlparent = @file | @xmlelement; +@xmlnamespaceable = @xmlelement | @xmlattribute; + +xmllocations( + int xmlElement: @xmllocatable ref, + int location: @location_default ref +); + +@xmllocatable = @xmlcharacters + | @xmlelement + | @xmlcomment + | @xmlattribute + | @xmldtd + | @file + | @xmlnamespace; diff --git a/cpp/downgrades/d77c09d8bdc172c9201dec293de1e14c931d3f05/semmlecode.cpp.dbscheme b/cpp/downgrades/d77c09d8bdc172c9201dec293de1e14c931d3f05/semmlecode.cpp.dbscheme new file mode 100644 index 00000000000..19887dbd333 --- /dev/null +++ b/cpp/downgrades/d77c09d8bdc172c9201dec293de1e14c931d3f05/semmlecode.cpp.dbscheme @@ -0,0 +1,2212 @@ + +/** + * An invocation of the compiler. Note that more than one file may be + * compiled per invocation. For example, this command compiles three + * source files: + * + * gcc -c f1.c f2.c f3.c + * + * The `id` simply identifies the invocation, while `cwd` is the working + * directory from which the compiler was invoked. + */ +compilations( + /** + * An invocation of the compiler. Note that more than one file may + * be compiled per invocation. For example, this command compiles + * three source files: + * + * gcc -c f1.c f2.c f3.c + */ + unique int id : @compilation, + string cwd : string ref +); + +/** + * The arguments that were passed to the extractor for a compiler + * invocation. If `id` is for the compiler invocation + * + * gcc -c f1.c f2.c f3.c + * + * then typically there will be rows for + * + * num | arg + * --- | --- + * 0 | *path to extractor* + * 1 | `--mimic` + * 2 | `/usr/bin/gcc` + * 3 | `-c` + * 4 | f1.c + * 5 | f2.c + * 6 | f3.c + */ +#keyset[id, num] +compilation_args( + int id : @compilation ref, + int num : int ref, + string arg : string ref +); + +/** + * The source files that are compiled by a compiler invocation. + * If `id` is for the compiler invocation + * + * gcc -c f1.c f2.c f3.c + * + * then there will be rows for + * + * num | arg + * --- | --- + * 0 | f1.c + * 1 | f2.c + * 2 | f3.c + * + * Note that even if those files `#include` headers, those headers + * do not appear as rows. + */ +#keyset[id, num] +compilation_compiling_files( + int id : @compilation ref, + int num : int ref, + int file : @file ref +); + +/** + * The time taken by the extractor for a compiler invocation. + * + * For each file `num`, there will be rows for + * + * kind | seconds + * ---- | --- + * 1 | CPU seconds used by the extractor frontend + * 2 | Elapsed seconds during the extractor frontend + * 3 | CPU seconds used by the extractor backend + * 4 | Elapsed seconds during the extractor backend + */ +#keyset[id, num, kind] +compilation_time( + int id : @compilation ref, + int num : int ref, + /* kind: + 1 = frontend_cpu_seconds + 2 = frontend_elapsed_seconds + 3 = extractor_cpu_seconds + 4 = extractor_elapsed_seconds + */ + int kind : int ref, + float seconds : float ref +); + +/** + * An error or warning generated by the extractor. + * The diagnostic message `diagnostic` was generated during compiler + * invocation `compilation`, and is the `file_number_diagnostic_number`th + * message generated while extracting the `file_number`th file of that + * invocation. + */ +#keyset[compilation, file_number, file_number_diagnostic_number] +diagnostic_for( + int diagnostic : @diagnostic ref, + int compilation : @compilation ref, + int file_number : int ref, + int file_number_diagnostic_number : int ref +); + +/** + * If extraction was successful, then `cpu_seconds` and + * `elapsed_seconds` are the CPU time and elapsed time (respectively) + * that extraction took for compiler invocation `id`. + */ +compilation_finished( + unique int id : @compilation ref, + float cpu_seconds : float ref, + float elapsed_seconds : float ref +); + + +/** + * External data, loaded from CSV files during snapshot creation. See + * [Tutorial: Incorporating external data](https://help.semmle.com/wiki/display/SD/Tutorial%3A+Incorporating+external+data) + * for more information. + */ +externalData( + int id : @externalDataElement, + string path : string ref, + int column: int ref, + string value : string ref +); + +/** + * The source location of the snapshot. + */ +sourceLocationPrefix(string prefix : string ref); + +/** + * Information about packages that provide code used during compilation. + * The `id` is just a unique identifier. + * The `namespace` is typically the name of the package manager that + * provided the package (e.g. "dpkg" or "yum"). + * The `package_name` is the name of the package, and `version` is its + * version (as a string). + */ +external_packages( + unique int id: @external_package, + string namespace : string ref, + string package_name : string ref, + string version : string ref +); + +/** + * Holds if File `fileid` was provided by package `package`. + */ +header_to_external_package( + int fileid : @file ref, + int package : @external_package ref +); + +/* + * Version history + */ + +svnentries( + unique int id : @svnentry, + string revision : string ref, + string author : string ref, + date revisionDate : date ref, + int changeSize : int ref +) + +svnaffectedfiles( + int id : @svnentry ref, + int file : @file ref, + string action : string ref +) + +svnentrymsg( + unique int id : @svnentry ref, + string message : string ref +) + +svnchurn( + int commit : @svnentry ref, + int file : @file ref, + int addedLines : int ref, + int deletedLines : int ref +) + +/* + * C++ dbscheme + */ + +@location = @location_stmt | @location_expr | @location_default ; + +/** + * The location of an element that is not an expression or a statement. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `file`. + * For more information, see + * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/). + */ +locations_default( + /** The location of an element that is not an expression or a statement. */ + unique int id: @location_default, + int container: @container ref, + int startLine: int ref, + int startColumn: int ref, + int endLine: int ref, + int endColumn: int ref +); + +/** + * The location of a statement. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `file`. + * For more information, see + * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/). + */ +locations_stmt( + /** The location of a statement. */ + unique int id: @location_stmt, + int container: @container ref, + int startLine: int ref, + int startColumn: int ref, + int endLine: int ref, + int endColumn: int ref +); + +/** + * The location of an expression. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `file`. + * For more information, see + * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/). + */ +locations_expr( + /** The location of an expression. */ + unique int id: @location_expr, + int container: @container ref, + int startLine: int ref, + int startColumn: int ref, + int endLine: int ref, + int endColumn: int ref +); + +/** An element for which line-count information is available. */ +@sourceline = @file | @function | @variable | @enumconstant | @xmllocatable; + +numlines( + int element_id: @sourceline ref, + int num_lines: int ref, + int num_code: int ref, + int num_comment: int ref +); + +diagnostics( + unique int id: @diagnostic, + int severity: int ref, + string error_tag: string ref, + string error_message: string ref, + string full_error_message: string ref, + int location: @location_default ref +); + +files( + unique int id: @file, + string name: string ref +); + +folders( + unique int id: @folder, + string name: string ref +); + +@container = @folder | @file + +containerparent( + int parent: @container ref, + unique int child: @container ref +); + +fileannotations( + int id: @file ref, + int kind: int ref, + string name: string ref, + string value: string ref +); + +inmacroexpansion( + int id: @element ref, + int inv: @macroinvocation ref +); + +affectedbymacroexpansion( + int id: @element ref, + int inv: @macroinvocation ref +); + +case @macroinvocation.kind of + 1 = @macro_expansion +| 2 = @other_macro_reference +; + +macroinvocations( + unique int id: @macroinvocation, + int macro_id: @ppd_define ref, + int location: @location_default ref, + int kind: int ref +); + +macroparent( + unique int id: @macroinvocation ref, + int parent_id: @macroinvocation ref +); + +// a macroinvocation may be part of another location +// the way to find a constant expression that uses a macro +// is thus to find a constant expression that has a location +// to which a macro invocation is bound +macrolocationbind( + int id: @macroinvocation ref, + int location: @location ref +); + +#keyset[invocation, argument_index] +macro_argument_unexpanded( + int invocation: @macroinvocation ref, + int argument_index: int ref, + string text: string ref +); + +#keyset[invocation, argument_index] +macro_argument_expanded( + int invocation: @macroinvocation ref, + int argument_index: int ref, + string text: string ref +); + +/* +case @function.kind of + 1 = @normal_function +| 2 = @constructor +| 3 = @destructor +| 4 = @conversion_function +| 5 = @operator +| 6 = @builtin_function // GCC built-in functions, e.g. __builtin___memcpy_chk +; +*/ + +functions( + unique int id: @function, + string name: string ref, + int kind: int ref +); + +function_entry_point( + int id: @function ref, + unique int entry_point: @stmt ref +); + +function_return_type( + int id: @function ref, + int return_type: @type ref +); + +/** + * If `function` is a coroutine, then this gives the `std::experimental::resumable_traits` + * instance associated with it, and the variables representing the `handle` and `promise` + * for it. + */ +coroutine( + unique int function: @function ref, + int traits: @type ref, + int handle: @variable ref, + int promise: @variable ref +); + +/** The `new` function used for allocating the coroutine state, if any. */ +coroutine_new( + unique int function: @function ref, + int new: @function ref +); + +/** The `delete` function used for deallocating the coroutine state, if any. */ +coroutine_delete( + unique int function: @function ref, + int delete: @function ref +); + +purefunctions(unique int id: @function ref); + +function_deleted(unique int id: @function ref); + +function_defaulted(unique int id: @function ref); + +member_function_this_type( + unique int id: @function ref, + int this_type: @type ref +); + +#keyset[id, type_id] +fun_decls( + int id: @fun_decl, + int function: @function ref, + int type_id: @type ref, + string name: string ref, + int location: @location_default ref +); +fun_def(unique int id: @fun_decl ref); +fun_specialized(unique int id: @fun_decl ref); +fun_implicit(unique int id: @fun_decl ref); +fun_decl_specifiers( + int id: @fun_decl ref, + string name: string ref +) +#keyset[fun_decl, index] +fun_decl_throws( + int fun_decl: @fun_decl ref, + int index: int ref, + int type_id: @type ref +); +/* an empty throw specification is different from none */ +fun_decl_empty_throws(unique int fun_decl: @fun_decl ref); +fun_decl_noexcept( + int fun_decl: @fun_decl ref, + int constant: @expr ref +); +fun_decl_empty_noexcept(int fun_decl: @fun_decl ref); +fun_decl_typedef_type( + unique int fun_decl: @fun_decl ref, + int typedeftype_id: @usertype ref +); + +param_decl_bind( + unique int id: @var_decl ref, + int index: int ref, + int fun_decl: @fun_decl ref +); + +#keyset[id, type_id] +var_decls( + int id: @var_decl, + int variable: @variable ref, + int type_id: @type ref, + string name: string ref, + int location: @location_default ref +); +var_def(unique int id: @var_decl ref); +var_decl_specifiers( + int id: @var_decl ref, + string name: string ref +) +is_structured_binding(unique int id: @variable ref); + +type_decls( + unique int id: @type_decl, + int type_id: @type ref, + int location: @location_default ref +); +type_def(unique int id: @type_decl ref); +type_decl_top( + unique int type_decl: @type_decl ref +); + +namespace_decls( + unique int id: @namespace_decl, + int namespace_id: @namespace ref, + int location: @location_default ref, + int bodylocation: @location_default ref +); + +usings( + unique int id: @using, + int element_id: @element ref, + int location: @location_default ref +); + +/** The element which contains the `using` declaration. */ +using_container( + int parent: @element ref, + int child: @using ref +); + +static_asserts( + unique int id: @static_assert, + int condition : @expr ref, + string message : string ref, + int location: @location_default ref, + int enclosing : @element ref +); + +// each function has an ordered list of parameters +#keyset[id, type_id] +#keyset[function, index, type_id] +params( + int id: @parameter, + int function: @functionorblock ref, + int index: int ref, + int type_id: @type ref +); + +overrides( + int new: @function ref, + int old: @function ref +); + +#keyset[id, type_id] +membervariables( + int id: @membervariable, + int type_id: @type ref, + string name: string ref +); + +#keyset[id, type_id] +globalvariables( + int id: @globalvariable, + int type_id: @type ref, + string name: string ref +); + +#keyset[id, type_id] +localvariables( + int id: @localvariable, + int type_id: @type ref, + string name: string ref +); + +autoderivation( + unique int var: @variable ref, + int derivation_type: @type ref +); + +orphaned_variables( + int var: @localvariable ref, + int function: @function ref +) + +enumconstants( + unique int id: @enumconstant, + int parent: @usertype ref, + int index: int ref, + int type_id: @type ref, + string name: string ref, + int location: @location_default ref +); + +@variable = @localscopevariable | @globalvariable | @membervariable; + +@localscopevariable = @localvariable | @parameter; + +/** + * Built-in types are the fundamental types, e.g., integral, floating, and void. + */ +case @builtintype.kind of + 1 = @errortype +| 2 = @unknowntype +| 3 = @void +| 4 = @boolean +| 5 = @char +| 6 = @unsigned_char +| 7 = @signed_char +| 8 = @short +| 9 = @unsigned_short +| 10 = @signed_short +| 11 = @int +| 12 = @unsigned_int +| 13 = @signed_int +| 14 = @long +| 15 = @unsigned_long +| 16 = @signed_long +| 17 = @long_long +| 18 = @unsigned_long_long +| 19 = @signed_long_long +// ... 20 Microsoft-specific __int8 +// ... 21 Microsoft-specific __int16 +// ... 22 Microsoft-specific __int32 +// ... 23 Microsoft-specific __int64 +| 24 = @float +| 25 = @double +| 26 = @long_double +| 27 = @complex_float // C99-specific _Complex float +| 28 = @complex_double // C99-specific _Complex double +| 29 = @complex_long_double // C99-specific _Complex long double +| 30 = @imaginary_float // C99-specific _Imaginary float +| 31 = @imaginary_double // C99-specific _Imaginary double +| 32 = @imaginary_long_double // C99-specific _Imaginary long double +| 33 = @wchar_t // Microsoft-specific +| 34 = @decltype_nullptr // C++11 +| 35 = @int128 // __int128 +| 36 = @unsigned_int128 // unsigned __int128 +| 37 = @signed_int128 // signed __int128 +| 38 = @float128 // __float128 +| 39 = @complex_float128 // _Complex __float128 +| 40 = @decimal32 // _Decimal32 +| 41 = @decimal64 // _Decimal64 +| 42 = @decimal128 // _Decimal128 +| 43 = @char16_t +| 44 = @char32_t +| 45 = @std_float32 // _Float32 +| 46 = @float32x // _Float32x +| 47 = @std_float64 // _Float64 +| 48 = @float64x // _Float64x +| 49 = @std_float128 // _Float128 +| 50 = @float128x // _Float128x +| 51 = @char8_t +| 52 = @float16 // _Float16 +| 53 = @complex_float16 // _Complex _Float16 +; + +builtintypes( + unique int id: @builtintype, + string name: string ref, + int kind: int ref, + int size: int ref, + int sign: int ref, + int alignment: int ref +); + +/** + * Derived types are types that are directly derived from existing types and + * point to, refer to, transform type data to return a new type. + */ +case @derivedtype.kind of + 1 = @pointer +| 2 = @reference +| 3 = @type_with_specifiers +| 4 = @array +| 5 = @gnu_vector +| 6 = @routineptr +| 7 = @routinereference +| 8 = @rvalue_reference // C++11 +// ... 9 type_conforming_to_protocols deprecated +| 10 = @block +; + +derivedtypes( + unique int id: @derivedtype, + string name: string ref, + int kind: int ref, + int type_id: @type ref +); + +pointerishsize(unique int id: @derivedtype ref, + int size: int ref, + int alignment: int ref); + +arraysizes( + unique int id: @derivedtype ref, + int num_elements: int ref, + int bytesize: int ref, + int alignment: int ref +); + +typedefbase( + unique int id: @usertype ref, + int type_id: @type ref +); + +/** + * An instance of the C++11 `decltype` operator. For example: + * ``` + * int a; + * decltype(1+a) b; + * ``` + * Here `expr` is `1+a`. + * + * Sometimes an additional pair of parentheses around the expression + * would change the semantics of this decltype, e.g. + * ``` + * struct A { double x; }; + * const A* a = new A(); + * decltype( a->x ); // type is double + * decltype((a->x)); // type is const double& + * ``` + * (Please consult the C++11 standard for more details). + * `parentheses_would_change_meaning` is `true` iff that is the case. + */ +#keyset[id, expr] +decltypes( + int id: @decltype, + int expr: @expr ref, + int base_type: @type ref, + boolean parentheses_would_change_meaning: boolean ref +); + +/* +case @usertype.kind of + 1 = @struct +| 2 = @class +| 3 = @union +| 4 = @enum +| 5 = @typedef // classic C: typedef typedef type name +| 6 = @template +| 7 = @template_parameter +| 8 = @template_template_parameter +| 9 = @proxy_class // a proxy class associated with a template parameter +// ... 10 objc_class deprecated +// ... 11 objc_protocol deprecated +// ... 12 objc_category deprecated +| 13 = @scoped_enum +| 14 = @using_alias // a using name = type style typedef +; +*/ + +usertypes( + unique int id: @usertype, + string name: string ref, + int kind: int ref +); + +usertypesize( + unique int id: @usertype ref, + int size: int ref, + int alignment: int ref +); + +usertype_final(unique int id: @usertype ref); + +usertype_uuid( + unique int id: @usertype ref, + string uuid: string ref +); + +mangled_name( + unique int id: @declaration ref, + int mangled_name : @mangledname +); + +is_pod_class(unique int id: @usertype ref); +is_standard_layout_class(unique int id: @usertype ref); + +is_complete(unique int id: @usertype ref); + +is_class_template(unique int id: @usertype ref); +class_instantiation( + int to: @usertype ref, + int from: @usertype ref +); +class_template_argument( + int type_id: @usertype ref, + int index: int ref, + int arg_type: @type ref +); +class_template_argument_value( + int type_id: @usertype ref, + int index: int ref, + int arg_value: @expr ref +); + +is_proxy_class_for( + unique int id: @usertype ref, + unique int templ_param_id: @usertype ref +); + +type_mentions( + unique int id: @type_mention, + int type_id: @type ref, + int location: @location ref, + // a_symbol_reference_kind from the frontend. + int kind: int ref +); + +is_function_template(unique int id: @function ref); +function_instantiation( + unique int to: @function ref, + int from: @function ref +); +function_template_argument( + int function_id: @function ref, + int index: int ref, + int arg_type: @type ref +); +function_template_argument_value( + int function_id: @function ref, + int index: int ref, + int arg_value: @expr ref +); + +is_variable_template(unique int id: @variable ref); +variable_instantiation( + unique int to: @variable ref, + int from: @variable ref +); +variable_template_argument( + int variable_id: @variable ref, + int index: int ref, + int arg_type: @type ref +); +variable_template_argument_value( + int variable_id: @variable ref, + int index: int ref, + int arg_value: @expr ref +); + +/* + Fixed point types + precision(1) = short, precision(2) = default, precision(3) = long + is_unsigned(1) = unsigned is_unsigned(2) = signed + is_fract_type(1) = declared with _Fract + saturating(1) = declared with _Sat +*/ +/* TODO +fixedpointtypes( + unique int id: @fixedpointtype, + int precision: int ref, + int is_unsigned: int ref, + int is_fract_type: int ref, + int saturating: int ref); +*/ + +routinetypes( + unique int id: @routinetype, + int return_type: @type ref +); + +routinetypeargs( + int routine: @routinetype ref, + int index: int ref, + int type_id: @type ref +); + +ptrtomembers( + unique int id: @ptrtomember, + int type_id: @type ref, + int class_id: @type ref +); + +/* + specifiers for types, functions, and variables + + "public", + "protected", + "private", + + "const", + "volatile", + "static", + + "pure", + "virtual", + "sealed", // Microsoft + "__interface", // Microsoft + "inline", + "explicit", + + "near", // near far extension + "far", // near far extension + "__ptr32", // Microsoft + "__ptr64", // Microsoft + "__sptr", // Microsoft + "__uptr", // Microsoft + "dllimport", // Microsoft + "dllexport", // Microsoft + "thread", // Microsoft + "naked", // Microsoft + "microsoft_inline", // Microsoft + "forceinline", // Microsoft + "selectany", // Microsoft + "nothrow", // Microsoft + "novtable", // Microsoft + "noreturn", // Microsoft + "noinline", // Microsoft + "noalias", // Microsoft + "restrict", // Microsoft +*/ + +specifiers( + unique int id: @specifier, + unique string str: string ref +); + +typespecifiers( + int type_id: @type ref, + int spec_id: @specifier ref +); + +funspecifiers( + int func_id: @function ref, + int spec_id: @specifier ref +); + +varspecifiers( + int var_id: @accessible ref, + int spec_id: @specifier ref +); + +attributes( + unique int id: @attribute, + int kind: int ref, + string name: string ref, + string name_space: string ref, + int location: @location_default ref +); + +case @attribute.kind of + 0 = @gnuattribute +| 1 = @stdattribute +| 2 = @declspec +| 3 = @msattribute +| 4 = @alignas +// ... 5 @objc_propertyattribute deprecated +; + +attribute_args( + unique int id: @attribute_arg, + int kind: int ref, + int attribute: @attribute ref, + int index: int ref, + int location: @location_default ref +); + +case @attribute_arg.kind of + 0 = @attribute_arg_empty +| 1 = @attribute_arg_token +| 2 = @attribute_arg_constant +| 3 = @attribute_arg_type +| 4 = @attribute_arg_constant_expr +; + +attribute_arg_value( + unique int arg: @attribute_arg ref, + string value: string ref +); +attribute_arg_type( + unique int arg: @attribute_arg ref, + int type_id: @type ref +); +attribute_arg_constant( + unique int arg: @attribute_arg ref, + int constant: @expr ref +) +attribute_arg_name( + unique int arg: @attribute_arg ref, + string name: string ref +); + +typeattributes( + int type_id: @type ref, + int spec_id: @attribute ref +); + +funcattributes( + int func_id: @function ref, + int spec_id: @attribute ref +); + +varattributes( + int var_id: @accessible ref, + int spec_id: @attribute ref +); + +stmtattributes( + int stmt_id: @stmt ref, + int spec_id: @attribute ref +); + +@type = @builtintype + | @derivedtype + | @usertype + /* TODO | @fixedpointtype */ + | @routinetype + | @ptrtomember + | @decltype; + +unspecifiedtype( + unique int type_id: @type ref, + int unspecified_type_id: @type ref +); + +member( + int parent: @type ref, + int index: int ref, + int child: @member ref +); + +@enclosingfunction_child = @usertype | @variable | @namespace + +enclosingfunction( + unique int child: @enclosingfunction_child ref, + int parent: @function ref +); + +derivations( + unique int derivation: @derivation, + int sub: @type ref, + int index: int ref, + int super: @type ref, + int location: @location_default ref +); + +derspecifiers( + int der_id: @derivation ref, + int spec_id: @specifier ref +); + +/** + * Contains the byte offset of the base class subobject within the derived + * class. Only holds for non-virtual base classes, but see table + * `virtual_base_offsets` for offsets of virtual base class subobjects. + */ +direct_base_offsets( + unique int der_id: @derivation ref, + int offset: int ref +); + +/** + * Contains the byte offset of the virtual base class subobject for class + * `super` within a most-derived object of class `sub`. `super` can be either a + * direct or indirect base class. + */ +#keyset[sub, super] +virtual_base_offsets( + int sub: @usertype ref, + int super: @usertype ref, + int offset: int ref +); + +frienddecls( + unique int id: @frienddecl, + int type_id: @type ref, + int decl_id: @declaration ref, + int location: @location_default ref +); + +@declaredtype = @usertype ; + +@declaration = @function + | @declaredtype + | @variable + | @enumconstant + | @frienddecl; + +@member = @membervariable + | @function + | @declaredtype + | @enumconstant; + +@locatable = @diagnostic + | @declaration + | @ppd_include + | @ppd_define + | @macroinvocation + /*| @funcall*/ + | @xmllocatable + | @attribute + | @attribute_arg; + +@namedscope = @namespace | @usertype; + +@element = @locatable + | @file + | @folder + | @specifier + | @type + | @expr + | @namespace + | @initialiser + | @stmt + | @derivation + | @comment + | @preprocdirect + | @fun_decl + | @var_decl + | @type_decl + | @namespace_decl + | @using + | @namequalifier + | @specialnamequalifyingelement + | @static_assert + | @type_mention + | @lambdacapture; + +@exprparent = @element; + +comments( + unique int id: @comment, + string contents: string ref, + int location: @location_default ref +); + +commentbinding( + int id: @comment ref, + int element: @element ref +); + +exprconv( + int converted: @expr ref, + unique int conversion: @expr ref +); + +compgenerated(unique int id: @element ref); + +/** + * `destructor_call` destructs the `i`'th entity that should be + * destructed following `element`. Note that entities should be + * destructed in reverse construction order, so for a given `element` + * these should be called from highest to lowest `i`. + */ +#keyset[element, destructor_call] +#keyset[element, i] +synthetic_destructor_call( + int element: @element ref, + int i: int ref, + int destructor_call: @routineexpr ref +); + +namespaces( + unique int id: @namespace, + string name: string ref +); + +namespace_inline( + unique int id: @namespace ref +); + +namespacembrs( + int parentid: @namespace ref, + unique int memberid: @namespacembr ref +); + +@namespacembr = @declaration | @namespace; + +exprparents( + int expr_id: @expr ref, + int child_index: int ref, + int parent_id: @exprparent ref +); + +expr_isload(unique int expr_id: @expr ref); + +@cast = @c_style_cast + | @const_cast + | @dynamic_cast + | @reinterpret_cast + | @static_cast + ; + +/* +case @conversion.kind of + 0 = @simple_conversion // a numeric conversion, qualification conversion, or a reinterpret_cast +| 1 = @bool_conversion // conversion to 'bool' +| 2 = @base_class_conversion // a derived-to-base conversion +| 3 = @derived_class_conversion // a base-to-derived conversion +| 4 = @pm_base_class_conversion // a derived-to-base conversion of a pointer to member +| 5 = @pm_derived_class_conversion // a base-to-derived conversion of a pointer to member +| 6 = @glvalue_adjust // an adjustment of the type of a glvalue +| 7 = @prvalue_adjust // an adjustment of the type of a prvalue +; +*/ +/** + * Describes the semantics represented by a cast expression. This is largely + * independent of the source syntax of the cast, so it is separate from the + * regular expression kind. + */ +conversionkinds( + unique int expr_id: @cast ref, + int kind: int ref +); + +@conversion = @cast + | @array_to_pointer + | @parexpr + | @reference_to + | @ref_indirect + | @temp_init + ; + +/* +case @funbindexpr.kind of + 0 = @normal_call // a normal call +| 1 = @virtual_call // a virtual call +| 2 = @adl_call // a call whose target is only found by ADL +; +*/ +iscall( + unique int caller: @funbindexpr ref, + int kind: int ref +); + +numtemplatearguments( + unique int expr_id: @expr ref, + int num: int ref +); + +specialnamequalifyingelements( + unique int id: @specialnamequalifyingelement, + unique string name: string ref +); + +@namequalifiableelement = @expr | @namequalifier; +@namequalifyingelement = @namespace + | @specialnamequalifyingelement + | @usertype; + +namequalifiers( + unique int id: @namequalifier, + unique int qualifiableelement: @namequalifiableelement ref, + int qualifyingelement: @namequalifyingelement ref, + int location: @location_default ref +); + +varbind( + int expr: @varbindexpr ref, + int var: @accessible ref +); + +funbind( + int expr: @funbindexpr ref, + int fun: @function ref +); + +@any_new_expr = @new_expr + | @new_array_expr; + +@new_or_delete_expr = @any_new_expr + | @delete_expr + | @delete_array_expr; + +@prefix_crement_expr = @preincrexpr | @predecrexpr; + +@postfix_crement_expr = @postincrexpr | @postdecrexpr; + +@increment_expr = @preincrexpr | @postincrexpr; + +@decrement_expr = @predecrexpr | @postdecrexpr; + +@crement_expr = @increment_expr | @decrement_expr; + +@un_arith_op_expr = @arithnegexpr + | @unaryplusexpr + | @conjugation + | @realpartexpr + | @imagpartexpr + | @crement_expr + ; + +@un_bitwise_op_expr = @complementexpr; + +@un_log_op_expr = @notexpr; + +@un_op_expr = @address_of + | @indirect + | @un_arith_op_expr + | @un_bitwise_op_expr + | @builtinaddressof + | @vec_fill + | @un_log_op_expr + | @co_await + | @co_yield + ; + +@bin_log_op_expr = @andlogicalexpr | @orlogicalexpr; + +@cmp_op_expr = @eq_op_expr | @rel_op_expr; + +@eq_op_expr = @eqexpr | @neexpr; + +@rel_op_expr = @gtexpr + | @ltexpr + | @geexpr + | @leexpr + | @spaceshipexpr + ; + +@bin_bitwise_op_expr = @lshiftexpr + | @rshiftexpr + | @andexpr + | @orexpr + | @xorexpr + ; + +@p_arith_op_expr = @paddexpr + | @psubexpr + | @pdiffexpr + ; + +@bin_arith_op_expr = @addexpr + | @subexpr + | @mulexpr + | @divexpr + | @remexpr + | @jmulexpr + | @jdivexpr + | @fjaddexpr + | @jfaddexpr + | @fjsubexpr + | @jfsubexpr + | @minexpr + | @maxexpr + | @p_arith_op_expr + ; + +@bin_op_expr = @bin_arith_op_expr + | @bin_bitwise_op_expr + | @cmp_op_expr + | @bin_log_op_expr + ; + +@op_expr = @un_op_expr + | @bin_op_expr + | @assign_expr + | @conditionalexpr + ; + +@assign_arith_expr = @assignaddexpr + | @assignsubexpr + | @assignmulexpr + | @assigndivexpr + | @assignremexpr + ; + +@assign_bitwise_expr = @assignandexpr + | @assignorexpr + | @assignxorexpr + | @assignlshiftexpr + | @assignrshiftexpr + | @assignpaddexpr + | @assignpsubexpr + ; + +@assign_op_expr = @assign_arith_expr | @assign_bitwise_expr + +@assign_expr = @assignexpr | @assign_op_expr | @blockassignexpr + +/* + case @allocator.form of + 0 = plain + | 1 = alignment + ; +*/ + +/** + * The allocator function associated with a `new` or `new[]` expression. + * The `form` column specified whether the allocation call contains an alignment + * argument. + */ +expr_allocator( + unique int expr: @any_new_expr ref, + int func: @function ref, + int form: int ref +); + +/* + case @deallocator.form of + 0 = plain + | 1 = size + | 2 = alignment + | 3 = size_and_alignment + ; +*/ + +/** + * The deallocator function associated with a `delete`, `delete[]`, `new`, or + * `new[]` expression. For a `new` or `new[]` expression, the deallocator is the + * one used to free memory if the initialization throws an exception. + * The `form` column specifies whether the deallocation call contains a size + * argument, and alignment argument, or both. + */ +expr_deallocator( + unique int expr: @new_or_delete_expr ref, + int func: @function ref, + int form: int ref +); + +/** + * Holds if the `@conditionalexpr` is of the two operand form + * `guard ? : false`. + */ +expr_cond_two_operand( + unique int cond: @conditionalexpr ref +); + +/** + * The guard of `@conditionalexpr` `guard ? true : false` + */ +expr_cond_guard( + unique int cond: @conditionalexpr ref, + int guard: @expr ref +); + +/** + * The expression used when the guard of `@conditionalexpr` + * `guard ? true : false` holds. For the two operand form + * `guard ?: false` consider using `expr_cond_guard` instead. + */ +expr_cond_true( + unique int cond: @conditionalexpr ref, + int true: @expr ref +); + +/** + * The expression used when the guard of `@conditionalexpr` + * `guard ? true : false` does not hold. + */ +expr_cond_false( + unique int cond: @conditionalexpr ref, + int false: @expr ref +); + +/** A string representation of the value. */ +values( + unique int id: @value, + string str: string ref +); + +/** The actual text in the source code for the value, if any. */ +valuetext( + unique int id: @value ref, + string text: string ref +); + +valuebind( + int val: @value ref, + unique int expr: @expr ref +); + +fieldoffsets( + unique int id: @variable ref, + int byteoffset: int ref, + int bitoffset: int ref +); + +bitfield( + unique int id: @variable ref, + int bits: int ref, + int declared_bits: int ref +); + +/* TODO +memberprefix( + int member: @expr ref, + int prefix: @expr ref +); +*/ + +/* + kind(1) = mbrcallexpr + kind(2) = mbrptrcallexpr + kind(3) = mbrptrmbrcallexpr + kind(4) = ptrmbrptrmbrcallexpr + kind(5) = mbrreadexpr // x.y + kind(6) = mbrptrreadexpr // p->y + kind(7) = mbrptrmbrreadexpr // x.*pm + kind(8) = mbrptrmbrptrreadexpr // x->*pm + kind(9) = staticmbrreadexpr // static x.y + kind(10) = staticmbrptrreadexpr // static p->y +*/ +/* TODO +memberaccess( + int member: @expr ref, + int kind: int ref +); +*/ + +initialisers( + unique int init: @initialiser, + int var: @accessible ref, + unique int expr: @expr ref, + int location: @location_expr ref +); + +braced_initialisers( + int init: @initialiser ref +); + +/** + * An ancestor for the expression, for cases in which we cannot + * otherwise find the expression's parent. + */ +expr_ancestor( + int exp: @expr ref, + int ancestor: @element ref +); + +exprs( + unique int id: @expr, + int kind: int ref, + int location: @location_expr ref +); + +/* + case @value.category of + 1 = prval + | 2 = xval + | 3 = lval + ; +*/ +expr_types( + int id: @expr ref, + int typeid: @type ref, + int value_category: int ref +); + +case @expr.kind of + 1 = @errorexpr +| 2 = @address_of // & AddressOfExpr +| 3 = @reference_to // ReferenceToExpr (implicit?) +| 4 = @indirect // * PointerDereferenceExpr +| 5 = @ref_indirect // ReferenceDereferenceExpr (implicit?) +// ... +| 8 = @array_to_pointer // (???) +| 9 = @vacuous_destructor_call // VacuousDestructorCall +// ... +| 11 = @assume // Microsoft +| 12 = @parexpr +| 13 = @arithnegexpr +| 14 = @unaryplusexpr +| 15 = @complementexpr +| 16 = @notexpr +| 17 = @conjugation // GNU ~ operator +| 18 = @realpartexpr // GNU __real +| 19 = @imagpartexpr // GNU __imag +| 20 = @postincrexpr +| 21 = @postdecrexpr +| 22 = @preincrexpr +| 23 = @predecrexpr +| 24 = @conditionalexpr +| 25 = @addexpr +| 26 = @subexpr +| 27 = @mulexpr +| 28 = @divexpr +| 29 = @remexpr +| 30 = @jmulexpr // C99 mul imaginary +| 31 = @jdivexpr // C99 div imaginary +| 32 = @fjaddexpr // C99 add real + imaginary +| 33 = @jfaddexpr // C99 add imaginary + real +| 34 = @fjsubexpr // C99 sub real - imaginary +| 35 = @jfsubexpr // C99 sub imaginary - real +| 36 = @paddexpr // pointer add (pointer + int or int + pointer) +| 37 = @psubexpr // pointer sub (pointer - integer) +| 38 = @pdiffexpr // difference between two pointers +| 39 = @lshiftexpr +| 40 = @rshiftexpr +| 41 = @andexpr +| 42 = @orexpr +| 43 = @xorexpr +| 44 = @eqexpr +| 45 = @neexpr +| 46 = @gtexpr +| 47 = @ltexpr +| 48 = @geexpr +| 49 = @leexpr +| 50 = @minexpr // GNU minimum +| 51 = @maxexpr // GNU maximum +| 52 = @assignexpr +| 53 = @assignaddexpr +| 54 = @assignsubexpr +| 55 = @assignmulexpr +| 56 = @assigndivexpr +| 57 = @assignremexpr +| 58 = @assignlshiftexpr +| 59 = @assignrshiftexpr +| 60 = @assignandexpr +| 61 = @assignorexpr +| 62 = @assignxorexpr +| 63 = @assignpaddexpr // assign pointer add +| 64 = @assignpsubexpr // assign pointer sub +| 65 = @andlogicalexpr +| 66 = @orlogicalexpr +| 67 = @commaexpr +| 68 = @subscriptexpr // access to member of an array, e.g., a[5] +// ... 69 @objc_subscriptexpr deprecated +// ... 70 @cmdaccess deprecated +// ... +| 73 = @virtfunptrexpr +| 74 = @callexpr +// ... 75 @msgexpr_normal deprecated +// ... 76 @msgexpr_super deprecated +// ... 77 @atselectorexpr deprecated +// ... 78 @atprotocolexpr deprecated +| 79 = @vastartexpr +| 80 = @vaargexpr +| 81 = @vaendexpr +| 82 = @vacopyexpr +// ... 83 @atencodeexpr deprecated +| 84 = @varaccess +| 85 = @thisaccess +// ... 86 @objc_box_expr deprecated +| 87 = @new_expr +| 88 = @delete_expr +| 89 = @throw_expr +| 90 = @condition_decl // a variable declared in a condition, e.g., if(int x = y > 2) +| 91 = @braced_init_list +| 92 = @type_id +| 93 = @runtime_sizeof +| 94 = @runtime_alignof +| 95 = @sizeof_pack +| 96 = @expr_stmt // GNU extension +| 97 = @routineexpr +| 98 = @type_operand // used to access a type in certain contexts (haven't found any examples yet....) +| 99 = @offsetofexpr // offsetof ::= type and field +| 100 = @hasassignexpr // __has_assign ::= type +| 101 = @hascopyexpr // __has_copy ::= type +| 102 = @hasnothrowassign // __has_nothrow_assign ::= type +| 103 = @hasnothrowconstr // __has_nothrow_constructor ::= type +| 104 = @hasnothrowcopy // __has_nothrow_copy ::= type +| 105 = @hastrivialassign // __has_trivial_assign ::= type +| 106 = @hastrivialconstr // __has_trivial_constructor ::= type +| 107 = @hastrivialcopy // __has_trivial_copy ::= type +| 108 = @hasuserdestr // __has_user_destructor ::= type +| 109 = @hasvirtualdestr // __has_virtual_destructor ::= type +| 110 = @isabstractexpr // __is_abstract ::= type +| 111 = @isbaseofexpr // __is_base_of ::= type type +| 112 = @isclassexpr // __is_class ::= type +| 113 = @isconvtoexpr // __is_convertible_to ::= type type +| 114 = @isemptyexpr // __is_empty ::= type +| 115 = @isenumexpr // __is_enum ::= type +| 116 = @ispodexpr // __is_pod ::= type +| 117 = @ispolyexpr // __is_polymorphic ::= type +| 118 = @isunionexpr // __is_union ::= type +| 119 = @typescompexpr // GNU __builtin_types_compatible ::= type type +| 120 = @intaddrexpr // frontend internal builtin, used to implement offsetof +// ... +| 122 = @hastrivialdestructor // __has_trivial_destructor ::= type +| 123 = @literal +| 124 = @uuidof +| 127 = @aggregateliteral +| 128 = @delete_array_expr +| 129 = @new_array_expr +// ... 130 @objc_array_literal deprecated +// ... 131 @objc_dictionary_literal deprecated +| 132 = @foldexpr +// ... +| 200 = @ctordirectinit +| 201 = @ctorvirtualinit +| 202 = @ctorfieldinit +| 203 = @ctordelegatinginit +| 204 = @dtordirectdestruct +| 205 = @dtorvirtualdestruct +| 206 = @dtorfielddestruct +// ... +| 210 = @static_cast +| 211 = @reinterpret_cast +| 212 = @const_cast +| 213 = @dynamic_cast +| 214 = @c_style_cast +| 215 = @lambdaexpr +| 216 = @param_ref +| 217 = @noopexpr +// ... +| 294 = @istriviallyconstructibleexpr +| 295 = @isdestructibleexpr +| 296 = @isnothrowdestructibleexpr +| 297 = @istriviallydestructibleexpr +| 298 = @istriviallyassignableexpr +| 299 = @isnothrowassignableexpr +| 300 = @istrivialexpr +| 301 = @isstandardlayoutexpr +| 302 = @istriviallycopyableexpr +| 303 = @isliteraltypeexpr +| 304 = @hastrivialmoveconstructorexpr +| 305 = @hastrivialmoveassignexpr +| 306 = @hasnothrowmoveassignexpr +| 307 = @isconstructibleexpr +| 308 = @isnothrowconstructibleexpr +| 309 = @hasfinalizerexpr +| 310 = @isdelegateexpr +| 311 = @isinterfaceclassexpr +| 312 = @isrefarrayexpr +| 313 = @isrefclassexpr +| 314 = @issealedexpr +| 315 = @issimplevalueclassexpr +| 316 = @isvalueclassexpr +| 317 = @isfinalexpr +| 319 = @noexceptexpr +| 320 = @builtinshufflevector +| 321 = @builtinchooseexpr +| 322 = @builtinaddressof +| 323 = @vec_fill +| 324 = @builtinconvertvector +| 325 = @builtincomplex +| 326 = @spaceshipexpr +| 327 = @co_await +| 328 = @co_yield +| 329 = @temp_init +| 330 = @isassignable +| 331 = @isaggregate +| 332 = @hasuniqueobjectrepresentations +| 333 = @builtinbitcast +| 334 = @builtinshuffle +| 335 = @blockassignexpr +| 336 = @issame +| 337 = @isfunction +| 338 = @islayoutcompatible +| 339 = @ispointerinterconvertiblebaseof +| 340 = @isarray +| 341 = @arrayrank +| 342 = @arrayextent +| 343 = @isarithmetic +| 344 = @iscompletetype +| 345 = @iscompound +| 346 = @isconst +| 347 = @isfloatingpoint +| 348 = @isfundamental +| 349 = @isintegral +| 350 = @islvaluereference +| 351 = @ismemberfunctionpointer +| 352 = @ismemberobjectpointer +| 353 = @ismemberpointer +| 354 = @isobject +| 355 = @ispointer +| 356 = @isreference +| 357 = @isrvaluereference +| 358 = @isscalar +| 359 = @issigned +| 360 = @isunsigned +| 361 = @isvoid +| 362 = @isvolatile +; + +@var_args_expr = @vastartexpr + | @vaendexpr + | @vaargexpr + | @vacopyexpr + ; + +@builtin_op = @var_args_expr + | @noopexpr + | @offsetofexpr + | @intaddrexpr + | @hasassignexpr + | @hascopyexpr + | @hasnothrowassign + | @hasnothrowconstr + | @hasnothrowcopy + | @hastrivialassign + | @hastrivialconstr + | @hastrivialcopy + | @hastrivialdestructor + | @hasuserdestr + | @hasvirtualdestr + | @isabstractexpr + | @isbaseofexpr + | @isclassexpr + | @isconvtoexpr + | @isemptyexpr + | @isenumexpr + | @ispodexpr + | @ispolyexpr + | @isunionexpr + | @typescompexpr + | @builtinshufflevector + | @builtinconvertvector + | @builtinaddressof + | @istriviallyconstructibleexpr + | @isdestructibleexpr + | @isnothrowdestructibleexpr + | @istriviallydestructibleexpr + | @istriviallyassignableexpr + | @isnothrowassignableexpr + | @isstandardlayoutexpr + | @istriviallycopyableexpr + | @isliteraltypeexpr + | @hastrivialmoveconstructorexpr + | @hastrivialmoveassignexpr + | @hasnothrowmoveassignexpr + | @isconstructibleexpr + | @isnothrowconstructibleexpr + | @hasfinalizerexpr + | @isdelegateexpr + | @isinterfaceclassexpr + | @isrefarrayexpr + | @isrefclassexpr + | @issealedexpr + | @issimplevalueclassexpr + | @isvalueclassexpr + | @isfinalexpr + | @builtinchooseexpr + | @builtincomplex + | @isassignable + | @isaggregate + | @hasuniqueobjectrepresentations + | @builtinbitcast + | @builtinshuffle + | @issame + | @isfunction + | @islayoutcompatible + | @ispointerinterconvertiblebaseof + | @isarray + | @arrayrank + | @arrayextent + | @isarithmetic + | @iscompletetype + | @iscompound + | @isconst + | @isfloatingpoint + | @isfundamental + | @isintegral + | @islvaluereference + | @ismemberfunctionpointer + | @ismemberobjectpointer + | @ismemberpointer + | @isobject + | @ispointer + | @isreference + | @isrvaluereference + | @isscalar + | @issigned + | @isunsigned + | @isvoid + | @isvolatile + ; + +new_allocated_type( + unique int expr: @new_expr ref, + int type_id: @type ref +); + +new_array_allocated_type( + unique int expr: @new_array_expr ref, + int type_id: @type ref +); + +/** + * The field being initialized by an initializer expression within an aggregate + * initializer for a class/struct/union. Position is used to sort repeated initializers. + */ +#keyset[aggregate, position] +aggregate_field_init( + int aggregate: @aggregateliteral ref, + int initializer: @expr ref, + int field: @membervariable ref, + int position: int ref +); + +/** + * The index of the element being initialized by an initializer expression + * within an aggregate initializer for an array. Position is used to sort repeated initializers. + */ +#keyset[aggregate, position] +aggregate_array_init( + int aggregate: @aggregateliteral ref, + int initializer: @expr ref, + int element_index: int ref, + int position: int ref +); + +@ctorinit = @ctordirectinit + | @ctorvirtualinit + | @ctorfieldinit + | @ctordelegatinginit; +@dtordestruct = @dtordirectdestruct + | @dtorvirtualdestruct + | @dtorfielddestruct; + + +condition_decl_bind( + unique int expr: @condition_decl ref, + unique int decl: @declaration ref +); + +typeid_bind( + unique int expr: @type_id ref, + int type_id: @type ref +); + +uuidof_bind( + unique int expr: @uuidof ref, + int type_id: @type ref +); + +@runtime_sizeof_or_alignof = @runtime_sizeof | @runtime_alignof; + +sizeof_bind( + unique int expr: @runtime_sizeof_or_alignof ref, + int type_id: @type ref +); + +code_block( + unique int block: @literal ref, + unique int routine: @function ref +); + +lambdas( + unique int expr: @lambdaexpr ref, + string default_capture: string ref, + boolean has_explicit_return_type: boolean ref +); + +lambda_capture( + unique int id: @lambdacapture, + int lambda: @lambdaexpr ref, + int index: int ref, + int field: @membervariable ref, + boolean captured_by_reference: boolean ref, + boolean is_implicit: boolean ref, + int location: @location_default ref +); + +@funbindexpr = @routineexpr + | @new_expr + | @delete_expr + | @delete_array_expr + | @ctordirectinit + | @ctorvirtualinit + | @ctordelegatinginit + | @dtordirectdestruct + | @dtorvirtualdestruct; + +@varbindexpr = @varaccess | @ctorfieldinit | @dtorfielddestruct; +@addressable = @function | @variable ; +@accessible = @addressable | @enumconstant ; + +@access = @varaccess | @routineexpr ; + +fold( + int expr: @foldexpr ref, + string operator: string ref, + boolean is_left_fold: boolean ref +); + +stmts( + unique int id: @stmt, + int kind: int ref, + int location: @location_stmt ref +); + +case @stmt.kind of + 1 = @stmt_expr +| 2 = @stmt_if +| 3 = @stmt_while +| 4 = @stmt_goto +| 5 = @stmt_label +| 6 = @stmt_return +| 7 = @stmt_block +| 8 = @stmt_end_test_while // do { ... } while ( ... ) +| 9 = @stmt_for +| 10 = @stmt_switch_case +| 11 = @stmt_switch +| 13 = @stmt_asm // "asm" statement or the body of an asm function +| 15 = @stmt_try_block +| 16 = @stmt_microsoft_try // Microsoft +| 17 = @stmt_decl +| 18 = @stmt_set_vla_size // C99 +| 19 = @stmt_vla_decl // C99 +| 25 = @stmt_assigned_goto // GNU +| 26 = @stmt_empty +| 27 = @stmt_continue +| 28 = @stmt_break +| 29 = @stmt_range_based_for // C++11 +// ... 30 @stmt_at_autoreleasepool_block deprecated +// ... 31 @stmt_objc_for_in deprecated +// ... 32 @stmt_at_synchronized deprecated +| 33 = @stmt_handler +// ... 34 @stmt_finally_end deprecated +| 35 = @stmt_constexpr_if +| 37 = @stmt_co_return +; + +type_vla( + int type_id: @type ref, + int decl: @stmt_vla_decl ref +); + +variable_vla( + int var: @variable ref, + int decl: @stmt_vla_decl ref +); + +if_initialization( + unique int if_stmt: @stmt_if ref, + int init_id: @stmt ref +); + +if_then( + unique int if_stmt: @stmt_if ref, + int then_id: @stmt ref +); + +if_else( + unique int if_stmt: @stmt_if ref, + int else_id: @stmt ref +); + +constexpr_if_initialization( + unique int constexpr_if_stmt: @stmt_constexpr_if ref, + int init_id: @stmt ref +); + +constexpr_if_then( + unique int constexpr_if_stmt: @stmt_constexpr_if ref, + int then_id: @stmt ref +); + +constexpr_if_else( + unique int constexpr_if_stmt: @stmt_constexpr_if ref, + int else_id: @stmt ref +); + +while_body( + unique int while_stmt: @stmt_while ref, + int body_id: @stmt ref +); + +do_body( + unique int do_stmt: @stmt_end_test_while ref, + int body_id: @stmt ref +); + +switch_initialization( + unique int switch_stmt: @stmt_switch ref, + int init_id: @stmt ref +); + +#keyset[switch_stmt, index] +switch_case( + int switch_stmt: @stmt_switch ref, + int index: int ref, + int case_id: @stmt_switch_case ref +); + +switch_body( + unique int switch_stmt: @stmt_switch ref, + int body_id: @stmt ref +); + +for_initialization( + unique int for_stmt: @stmt_for ref, + int init_id: @stmt ref +); + +for_condition( + unique int for_stmt: @stmt_for ref, + int condition_id: @expr ref +); + +for_update( + unique int for_stmt: @stmt_for ref, + int update_id: @expr ref +); + +for_body( + unique int for_stmt: @stmt_for ref, + int body_id: @stmt ref +); + +@stmtparent = @stmt | @expr_stmt ; +stmtparents( + unique int id: @stmt ref, + int index: int ref, + int parent: @stmtparent ref +); + +ishandler(unique int block: @stmt_block ref); + +@cfgnode = @stmt | @expr | @function | @initialiser ; + +stmt_decl_bind( + int stmt: @stmt_decl ref, + int num: int ref, + int decl: @declaration ref +); + +stmt_decl_entry_bind( + int stmt: @stmt_decl ref, + int num: int ref, + int decl_entry: @element ref +); + +@functionorblock = @function | @stmt_block; + +blockscope( + unique int block: @stmt_block ref, + int enclosing: @functionorblock ref +); + +@jump = @stmt_goto | @stmt_break | @stmt_continue; + +@jumporlabel = @jump | @stmt_label | @literal; + +jumpinfo( + unique int id: @jumporlabel ref, + string str: string ref, + int target: @stmt ref +); + +preprocdirects( + unique int id: @preprocdirect, + int kind: int ref, + int location: @location_default ref +); +case @preprocdirect.kind of + 0 = @ppd_if +| 1 = @ppd_ifdef +| 2 = @ppd_ifndef +| 3 = @ppd_elif +| 4 = @ppd_else +| 5 = @ppd_endif +| 6 = @ppd_plain_include +| 7 = @ppd_define +| 8 = @ppd_undef +| 9 = @ppd_line +| 10 = @ppd_error +| 11 = @ppd_pragma +| 12 = @ppd_objc_import +| 13 = @ppd_include_next +| 18 = @ppd_warning +; + +@ppd_include = @ppd_plain_include | @ppd_objc_import | @ppd_include_next; + +@ppd_branch = @ppd_if | @ppd_ifdef | @ppd_ifndef | @ppd_elif; + +preprocpair( + int begin : @ppd_branch ref, + int elseelifend : @preprocdirect ref +); + +preproctrue(int branch : @ppd_branch ref); +preprocfalse(int branch : @ppd_branch ref); + +preproctext( + unique int id: @preprocdirect ref, + string head: string ref, + string body: string ref +); + +includes( + unique int id: @ppd_include ref, + int included: @file ref +); + +link_targets( + unique int id: @link_target, + int binary: @file ref +); + +link_parent( + int element : @element ref, + int link_target : @link_target ref +); + +/* XML Files */ + +xmlEncoding(unique int id: @file ref, string encoding: string ref); + +xmlDTDs( + unique int id: @xmldtd, + string root: string ref, + string publicId: string ref, + string systemId: string ref, + int fileid: @file ref +); + +xmlElements( + unique int id: @xmlelement, + string name: string ref, + int parentid: @xmlparent ref, + int idx: int ref, + int fileid: @file ref +); + +xmlAttrs( + unique int id: @xmlattribute, + int elementid: @xmlelement ref, + string name: string ref, + string value: string ref, + int idx: int ref, + int fileid: @file ref +); + +xmlNs( + int id: @xmlnamespace, + string prefixName: string ref, + string URI: string ref, + int fileid: @file ref +); + +xmlHasNs( + int elementId: @xmlnamespaceable ref, + int nsId: @xmlnamespace ref, + int fileid: @file ref +); + +xmlComments( + unique int id: @xmlcomment, + string text: string ref, + int parentid: @xmlparent ref, + int fileid: @file ref +); + +xmlChars( + unique int id: @xmlcharacters, + string text: string ref, + int parentid: @xmlparent ref, + int idx: int ref, + int isCDATA: int ref, + int fileid: @file ref +); + +@xmlparent = @file | @xmlelement; +@xmlnamespaceable = @xmlelement | @xmlattribute; + +xmllocations( + int xmlElement: @xmllocatable ref, + int location: @location_default ref +); + +@xmllocatable = @xmlcharacters + | @xmlelement + | @xmlcomment + | @xmlattribute + | @xmldtd + | @file + | @xmlnamespace; diff --git a/cpp/downgrades/d77c09d8bdc172c9201dec293de1e14c931d3f05/upgrade.properties b/cpp/downgrades/d77c09d8bdc172c9201dec293de1e14c931d3f05/upgrade.properties new file mode 100644 index 00000000000..50d352e0cc3 --- /dev/null +++ b/cpp/downgrades/d77c09d8bdc172c9201dec293de1e14c931d3f05/upgrade.properties @@ -0,0 +1,2 @@ +description: Remove _Float128 type +compatibility: full diff --git a/cpp/ql/lib/change-notes/2023-08-07-removal-of-float128x.md b/cpp/ql/lib/change-notes/2023-08-07-removal-of-float128x.md new file mode 100644 index 00000000000..1b91bb6ff89 --- /dev/null +++ b/cpp/ql/lib/change-notes/2023-08-07-removal-of-float128x.md @@ -0,0 +1,4 @@ +--- +category: minorAnalysis +--- +* The `_Float128x` type is no longer exposed as a builtin type. As this type could not occur any code base, this should only affect queries that explicitly looked at the builtin types. diff --git a/cpp/ql/lib/semmle/code/cpp/Type.qll b/cpp/ql/lib/semmle/code/cpp/Type.qll index 91354ef2e08..43757ce0b2d 100644 --- a/cpp/ql/lib/semmle/code/cpp/Type.qll +++ b/cpp/ql/lib/semmle/code/cpp/Type.qll @@ -814,9 +814,6 @@ private predicate floatingPointTypeMapping( // _Float128 kind = 49 and base = 2 and domain = TRealDomain() and realKind = 49 and extended = false or - // _Float128x - kind = 50 and base = 2 and domain = TRealDomain() and realKind = 50 and extended = true - or // _Float16 kind = 52 and base = 2 and domain = TRealDomain() and realKind = 52 and extended = false or diff --git a/cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl.qll b/cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl.qll index 816f835c082..f5c51b43e37 100644 --- a/cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl.qll +++ b/cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl.qll @@ -1,3 +1,3 @@ private import DataFlowImplSpecific -private import codeql.dataflow.DataFlowImpl +private import codeql.dataflow.internal.DataFlowImpl import MakeImpl diff --git a/cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplCommon.qll b/cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplCommon.qll index fca32ad8617..868c3ef6a2b 100644 --- a/cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplCommon.qll +++ b/cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplCommon.qll @@ -1,3 +1,3 @@ private import DataFlowImplSpecific -private import codeql.dataflow.DataFlowImplCommon +private import codeql.dataflow.internal.DataFlowImplCommon import MakeImplCommon diff --git a/cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplSpecific.qll b/cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplSpecific.qll index 317d0b6db29..7fa662bd691 100644 --- a/cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplSpecific.qll +++ b/cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplSpecific.qll @@ -2,7 +2,7 @@ * Provides C++-specific definitions for use in the data flow library. */ -private import codeql.dataflow.DataFlowParameter +private import codeql.dataflow.DataFlow module Private { import DataFlowPrivate @@ -13,7 +13,7 @@ module Public { import DataFlowUtil } -module CppOldDataFlow implements DataFlowParameter { +module CppOldDataFlow implements InputSig { import Private import Public diff --git a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl.qll b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl.qll index 7f60217416c..f3e52187647 100644 --- a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl.qll +++ b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl.qll @@ -1,3 +1,3 @@ private import DataFlowImplSpecific -private import codeql.dataflow.DataFlowImpl +private import codeql.dataflow.internal.DataFlowImpl import MakeImpl diff --git a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplCommon.qll b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplCommon.qll index 65543101184..266693f45f6 100644 --- a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplCommon.qll +++ b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplCommon.qll @@ -1,3 +1,3 @@ private import DataFlowImplSpecific -private import codeql.dataflow.DataFlowImplCommon +private import codeql.dataflow.internal.DataFlowImplCommon import MakeImplCommon diff --git a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplSpecific.qll b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplSpecific.qll index daf36f07415..ed4db065532 100644 --- a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplSpecific.qll +++ b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplSpecific.qll @@ -2,7 +2,7 @@ * Provides IR-specific definitions for use in the data flow library. */ -private import codeql.dataflow.DataFlowParameter +private import codeql.dataflow.DataFlow module Private { import DataFlowPrivate @@ -13,7 +13,7 @@ module Public { import DataFlowUtil } -module CppDataFlow implements DataFlowParameter { +module CppDataFlow implements InputSig { import Private import Public diff --git a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowPrivate.qll b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowPrivate.qll index 6b3f745a641..329164e0fd0 100644 --- a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowPrivate.qll +++ b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowPrivate.qll @@ -1078,7 +1078,7 @@ private IRVariable getIRVariableForParameterNode(ParameterNode p) { /** Holds if `v` is the source variable corresponding to the parameter represented by `p`. */ pragma[nomagic] -private predicate parameterNodeHasSourceVariable(ParameterNode p, Ssa::SourceIRVariable v) { +private predicate parameterNodeHasSourceVariable(ParameterNode p, Ssa::SourceVariable v) { v.getIRVariable() = getIRVariableForParameterNode(p) and exists(Position pos | p.isParameterOf(_, pos) | pos instanceof DirectPosition and diff --git a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowUtil.qll b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowUtil.qll index 209d0246832..f834259dc37 100644 --- a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowUtil.qll +++ b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowUtil.qll @@ -781,26 +781,12 @@ class IndirectArgumentOutNode extends Node, TIndirectArgumentOutNode, PartialDef override Expr getDefinedExpr() { result = operand.getDef().getUnconvertedResultExpression() } } -pragma[nomagic] -predicate indirectReturnOutNodeOperand0(CallInstruction call, Operand operand, int indirectionIndex) { - Ssa::hasRawIndirectInstruction(call, indirectionIndex) and - operandForFullyConvertedCall(operand, call) -} - -pragma[nomagic] -predicate indirectReturnOutNodeInstruction0( - CallInstruction call, Instruction instr, int indirectionIndex -) { - Ssa::hasRawIndirectInstruction(call, indirectionIndex) and - instructionForFullyConvertedCall(instr, call) -} - /** * Holds if `node` is an indirect operand with columns `(operand, indirectionIndex)`, and * `operand` represents a use of the fully converted value of `call`. */ private predicate hasOperand(Node node, CallInstruction call, int indirectionIndex, Operand operand) { - indirectReturnOutNodeOperand0(call, operand, indirectionIndex) and + operandForFullyConvertedCall(operand, call) and hasOperandAndIndex(node, operand, indirectionIndex) } @@ -813,7 +799,7 @@ private predicate hasOperand(Node node, CallInstruction call, int indirectionInd private predicate hasInstruction( Node node, CallInstruction call, int indirectionIndex, Instruction instr ) { - indirectReturnOutNodeInstruction0(call, instr, indirectionIndex) and + instructionForFullyConvertedCall(instr, call) and hasInstructionAndIndex(node, instr, indirectionIndex) } diff --git a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/SsaInternals.qll b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/SsaInternals.qll index 0cd152e2473..4e9a90dc0a1 100644 --- a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/SsaInternals.qll +++ b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/SsaInternals.qll @@ -10,32 +10,35 @@ private import ssa0.SsaInternals as SsaInternals0 import SsaInternalsCommon private module SourceVariables { - int getMaxIndirectionForIRVariable(IRVariable var) { - exists(Type type, boolean isGLValue | - var.getLanguageType().hasType(type, isGLValue) and - if isGLValue = true - then result = 1 + getMaxIndirectionsForType(type) - else result = getMaxIndirectionsForType(type) - ) - } - cached private newtype TSourceVariable = - TSourceIRVariable(BaseIRVariable baseVar, int ind) { - ind = [0 .. getMaxIndirectionForIRVariable(baseVar.getIRVariable())] - } or - TCallVariable(AllocationInstruction call, int ind) { - ind = [0 .. countIndirectionsForCppType(getResultLanguageType(call))] + TMkSourceVariable(SsaInternals0::SourceVariable base, int ind) { + ind = [0 .. countIndirectionsForCppType(base.getLanguageType()) + 1] } - abstract class SourceVariable extends TSourceVariable { + class SourceVariable extends TSourceVariable { + SsaInternals0::SourceVariable base; int ind; - bindingset[ind] - SourceVariable() { any() } + SourceVariable() { this = TMkSourceVariable(base, ind) } + + /** Gets the IR variable associated with this `SourceVariable`, if any. */ + IRVariable getIRVariable() { result = base.(BaseIRVariable).getIRVariable() } + + /** + * Gets the base source variable (i.e., the variable without any + * indirections) of this source variable. + */ + SsaInternals0::SourceVariable getBaseVariable() { result = base } /** Gets a textual representation of this element. */ - abstract string toString(); + string toString() { + ind = 0 and + result = this.getBaseVariable().toString() + or + ind > 0 and + result = this.getBaseVariable().toString() + " indirection" + } /** * Gets the number of loads performed on the base source variable @@ -43,65 +46,19 @@ private module SourceVariables { */ int getIndirection() { result = ind } - /** - * Gets the base source variable (i.e., the variable without any - * indirections) of this source variable. - */ - abstract BaseSourceVariable getBaseVariable(); - /** Holds if this variable is a glvalue. */ - predicate isGLValue() { none() } + predicate isGLValue() { ind = 0 } /** * Gets the type of this source variable. If `isGLValue()` holds, then * the type of this source variable should be thought of as "pointer * to `getType()`". */ - abstract DataFlowType getType(); - } - - class SourceIRVariable extends SourceVariable, TSourceIRVariable { - BaseIRVariable var; - - SourceIRVariable() { this = TSourceIRVariable(var, ind) } - - IRVariable getIRVariable() { result = var.getIRVariable() } - - override BaseIRVariable getBaseVariable() { result.getIRVariable() = this.getIRVariable() } - - override string toString() { - ind = 0 and - result = this.getIRVariable().toString() - or - ind > 0 and - result = this.getIRVariable().toString() + " indirection" + DataFlowType getType() { + if this.isGLValue() + then result = base.getType() + else result = getTypeImpl(base.getType(), ind - 1) } - - override predicate isGLValue() { ind = 0 } - - override DataFlowType getType() { - if ind = 0 then result = var.getType() else result = getTypeImpl(var.getType(), ind - 1) - } - } - - class CallVariable extends SourceVariable, TCallVariable { - AllocationInstruction call; - - CallVariable() { this = TCallVariable(call, ind) } - - AllocationInstruction getCall() { result = call } - - override BaseCallVariable getBaseVariable() { result.getCallInstruction() = call } - - override string toString() { - ind = 0 and - result = "Call" - or - ind > 0 and - result = "Call indirection" - } - - override DataFlowType getType() { result = getTypeImpl(call.getResultType(), ind) } } } @@ -137,8 +94,9 @@ predicate hasRawIndirectInstruction(Instruction instr, int indirectionIndex) { cached private newtype TDefOrUseImpl = - TDefImpl(Operand address, int indirectionIndex) { - exists(Instruction base | isDef(_, _, address, base, _, indirectionIndex) | + TDefImpl(BaseSourceVariableInstruction base, Operand address, int indirectionIndex) { + isDef(_, _, address, base, _, indirectionIndex) and + ( // We only include the definition if the SSA pruning stage // concluded that the definition is live after the write. any(SsaInternals0::Def def).getAddressOperand() = address @@ -148,8 +106,8 @@ private newtype TDefOrUseImpl = base.(VariableAddressInstruction).getAstVariable() instanceof GlobalLikeVariable ) } or - TUseImpl(Operand operand, int indirectionIndex) { - isUse(_, operand, _, _, indirectionIndex) and + TUseImpl(BaseSourceVariableInstruction base, Operand operand, int indirectionIndex) { + isUse(_, operand, base, _, indirectionIndex) and not isDef(_, _, operand, _, _, _) } or TGlobalUse(GlobalLikeVariable v, IRFunction f, int indirectionIndex) { @@ -236,7 +194,7 @@ abstract private class DefOrUseImpl extends TDefOrUseImpl { /** * Gets the instruction that computes the base of this definition or use. - * This is always a `VariableAddressInstruction` or an `AllocationInstruction`. + * This is always a `VariableAddressInstruction` or an `CallInstruction`. */ abstract BaseSourceVariableInstruction getBase(); @@ -308,15 +266,17 @@ abstract class DefImpl extends DefOrUseImpl { } private class DirectDef extends DefImpl, TDefImpl { - DirectDef() { this = TDefImpl(address, ind) } + BaseSourceVariableInstruction base; - override BaseSourceVariableInstruction getBase() { isDef(_, _, address, result, _, _) } + DirectDef() { this = TDefImpl(base, address, ind) } - override int getIndirection() { isDef(_, _, address, _, result, ind) } + override BaseSourceVariableInstruction getBase() { result = base } - override Node0Impl getValue() { isDef(_, result, address, _, _, _) } + override int getIndirection() { isDef(_, _, address, base, result, ind) } - override predicate isCertain() { isDef(true, _, address, _, _, ind) } + override Node0Impl getValue() { isDef(_, result, address, base, _, _) } + + override predicate isCertain() { isDef(true, _, address, base, _, ind) } } private class IteratorDef extends DefImpl, TIteratorDef { @@ -359,6 +319,7 @@ abstract class UseImpl extends DefOrUseImpl { abstract private class OperandBasedUse extends UseImpl { Operand operand; + BaseSourceVariableInstruction base; bindingset[ind] OperandBasedUse() { any() } @@ -366,50 +327,44 @@ abstract private class OperandBasedUse extends UseImpl { final override predicate hasIndexInBlock(IRBlock block, int index) { // See the comment in `ssa0`'s `OperandBasedUse` for an explanation of this // predicate's implementation. - exists(BaseSourceVariableInstruction base | base = this.getBase() | - if base.getAst() = any(Cpp::PostfixCrementOperation c).getOperand() - then - exists(Operand op, int indirectionIndex, int indirection | - indirectionIndex = this.getIndirectionIndex() and - indirection = this.getIndirection() and - op = - min(Operand cand, int i | - isUse(_, cand, base, indirection, indirectionIndex) and - block.getInstruction(i) = cand.getUse() - | - cand order by i - ) and - block.getInstruction(index) = op.getUse() - ) - else operand.getUse() = block.getInstruction(index) - ) + if base.getAst() = any(Cpp::PostfixCrementOperation c).getOperand() + then + exists(Operand op, int indirectionIndex, int indirection | + indirectionIndex = this.getIndirectionIndex() and + indirection = this.getIndirection() and + op = + min(Operand cand, int i | + isUse(_, cand, base, indirection, indirectionIndex) and + block.getInstruction(i) = cand.getUse() + | + cand order by i + ) and + block.getInstruction(index) = op.getUse() + ) + else operand.getUse() = block.getInstruction(index) } + final override BaseSourceVariableInstruction getBase() { result = base } + final Operand getOperand() { result = operand } final override Cpp::Location getLocation() { result = operand.getLocation() } } private class DirectUse extends OperandBasedUse, TUseImpl { - DirectUse() { this = TUseImpl(operand, ind) } + DirectUse() { this = TUseImpl(base, operand, ind) } - override int getIndirection() { isUse(_, operand, _, result, ind) } + override int getIndirection() { isUse(_, operand, base, result, ind) } - override BaseSourceVariableInstruction getBase() { isUse(_, operand, result, _, ind) } - - override predicate isCertain() { isUse(true, operand, _, _, ind) } + override predicate isCertain() { isUse(true, operand, base, _, ind) } override Node getNode() { nodeHasOperand(result, operand, ind) } } private class IteratorUse extends OperandBasedUse, TIteratorUse { - BaseSourceVariableInstruction container; + IteratorUse() { this = TIteratorUse(operand, base, ind) } - IteratorUse() { this = TIteratorUse(operand, container, ind) } - - override int getIndirection() { isIteratorUse(container, operand, result, ind) } - - override BaseSourceVariableInstruction getBase() { result = container } + override int getIndirection() { isIteratorUse(base, operand, result, ind) } override predicate isCertain() { none() } diff --git a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/SsaInternalsCommon.qll b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/SsaInternalsCommon.qll index 33b33113d43..60cb843b61f 100644 --- a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/SsaInternalsCommon.qll +++ b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/SsaInternalsCommon.qll @@ -146,14 +146,6 @@ int countIndirectionsForCppType(LanguageType langType) { ) } -/** - * A `CallInstruction` that calls an allocation function such - * as `malloc` or `operator new`. - */ -class AllocationInstruction extends CallInstruction { - AllocationInstruction() { this.getStaticCallTarget() instanceof Cpp::AllocationFunction } -} - private predicate isIndirectionType(Type t) { t instanceof Indirection } private predicate hasUnspecifiedBaseType(Indirection t, Type base) { @@ -368,17 +360,22 @@ newtype TBaseSourceVariable = // Each IR variable gets its own source variable TBaseIRVariable(IRVariable var) or // Each allocation gets its own source variable - TBaseCallVariable(AllocationInstruction call) + TBaseCallVariable(CallInstruction call) { not call.getResultIRType() instanceof IRVoidType } -abstract class BaseSourceVariable extends TBaseSourceVariable { +abstract private class AbstractBaseSourceVariable extends TBaseSourceVariable { /** Gets a textual representation of this element. */ abstract string toString(); /** Gets the type of this base source variable. */ - abstract DataFlowType getType(); + final DataFlowType getType() { this.getLanguageType().hasUnspecifiedType(result, _) } + + /** Gets the `CppType` of this base source variable. */ + abstract CppType getLanguageType(); } -class BaseIRVariable extends BaseSourceVariable, TBaseIRVariable { +final class BaseSourceVariable = AbstractBaseSourceVariable; + +class BaseIRVariable extends AbstractBaseSourceVariable, TBaseIRVariable { IRVariable var; IRVariable getIRVariable() { result = var } @@ -387,19 +384,19 @@ class BaseIRVariable extends BaseSourceVariable, TBaseIRVariable { override string toString() { result = var.toString() } - override DataFlowType getType() { result = var.getType() } + override CppType getLanguageType() { result = var.getLanguageType() } } -class BaseCallVariable extends BaseSourceVariable, TBaseCallVariable { - AllocationInstruction call; +class BaseCallVariable extends AbstractBaseSourceVariable, TBaseCallVariable { + CallInstruction call; BaseCallVariable() { this = TBaseCallVariable(call) } - AllocationInstruction getCallInstruction() { result = call } + CallInstruction getCallInstruction() { result = call } override string toString() { result = call.toString() } - override DataFlowType getType() { result = call.getResultType() } + override CppType getLanguageType() { result = getResultLanguageType(call) } } /** @@ -499,8 +496,7 @@ private class BaseIRVariableInstruction extends BaseSourceVariableInstruction, override BaseIRVariable getBaseSourceVariable() { result.getIRVariable() = this.getIRVariable() } } -private class BaseAllocationInstruction extends BaseSourceVariableInstruction, AllocationInstruction -{ +private class BaseCallInstruction extends BaseSourceVariableInstruction, CallInstruction { override BaseCallVariable getBaseSourceVariable() { result.getCallInstruction() = this } } diff --git a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/ssa0/SsaInternals.qll b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/ssa0/SsaInternals.qll index 38f9bbeec8e..fd27a4e354f 100644 --- a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/ssa0/SsaInternals.qll +++ b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/ssa0/SsaInternals.qll @@ -15,15 +15,12 @@ private import semmle.code.cpp.ir.dataflow.internal.DataFlowUtil private import semmle.code.cpp.ir.dataflow.internal.SsaInternalsCommon private module SourceVariables { - class SourceVariable instanceof BaseSourceVariable { - string toString() { result = BaseSourceVariable.super.toString() } - + class SourceVariable extends BaseSourceVariable { + /** + * Gets the base source variable of this `SourceVariable`. + */ BaseSourceVariable getBaseVariable() { result = this } } - - class SourceIRVariable = BaseIRVariable; - - class CallVariable = BaseCallVariable; } import SourceVariables diff --git a/cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/SemanticExpr.qll b/cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/SemanticExpr.qll index 46a5c735ca0..72b70ada4ed 100644 --- a/cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/SemanticExpr.qll +++ b/cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/SemanticExpr.qll @@ -307,3 +307,8 @@ class SemConditionalExpr extends SemKnownExpr { branch = false and result = falseResult } } + +/** Holds if `upper = true` and `e <= bound` or `upper = false` and `e >= bound`. */ +predicate semHasConstantBoundConstantSpecific(SemExpr e, float bound, boolean upper) { + Specific::hasConstantBoundConstantSpecific(e, bound, upper) +} diff --git a/cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/SemanticExprSpecific.qll b/cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/SemanticExprSpecific.qll index a92361ece17..2ec277bf8cf 100644 --- a/cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/SemanticExprSpecific.qll +++ b/cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/SemanticExprSpecific.qll @@ -434,6 +434,50 @@ module SemanticExprConfig { /** Gets the expression associated with `instr`. */ SemExpr getSemanticExpr(IR::Instruction instr) { result = Equiv::getEquivalenceClass(instr) } + + private predicate typeBounds(SemType t, float lb, float ub) { + exists(SemIntegerType integralType, float limit | + integralType = t and limit = 2.pow(8 * integralType.getByteSize()) + | + if integralType instanceof SemBooleanType + then lb = 0 and ub = 1 + else + if integralType.isSigned() + then ( + lb = -(limit / 2) and ub = (limit / 2) - 1 + ) else ( + lb = 0 and ub = limit - 1 + ) + ) + or + // This covers all floating point types. The range is (-Inf, +Inf). + t instanceof SemFloatingPointType and lb = -(1.0 / 0.0) and ub = 1.0 / 0.0 + } + + /** + * Holds if `upper = true` and `e <= bound` or `upper = false` and `e >= bound` based + * only on type information. + */ + predicate hasConstantBoundConstantSpecific(Expr e, float bound, boolean upper) { + exists( + SemType converted, SemType unconverted, float unconvertedLb, float convertedLb, + float unconvertedUb, float convertedUb + | + unconverted = getSemanticType(e.getUnconverted().getResultIRType()) and + converted = getSemanticType(e.getConverted().getResultIRType()) and + typeBounds(unconverted, unconvertedLb, unconvertedUb) and + typeBounds(converted, convertedLb, convertedUb) and + ( + upper = true and + unconvertedUb < convertedUb and + bound = unconvertedUb + or + upper = false and + unconvertedLb > convertedLb and + bound = unconvertedLb + ) + ) + } } predicate getSemanticExpr = SemanticExprConfig::getSemanticExpr/1; @@ -457,3 +501,5 @@ IRBound::Bound getCppBound(SemBound bound) { bound = result } SemGuard getSemanticGuard(IRGuards::IRGuardCondition guard) { result = guard } IRGuards::IRGuardCondition getCppGuard(SemGuard guard) { guard = result } + +predicate hasConstantBoundConstantSpecific = SemanticExprConfig::hasConstantBoundConstantSpecific/3; diff --git a/cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/analysis/RangeAnalysisConstantSpecific.qll b/cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/analysis/RangeAnalysisConstantSpecific.qll index d1e53530ef5..54e46bfd00c 100644 --- a/cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/analysis/RangeAnalysisConstantSpecific.qll +++ b/cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/analysis/RangeAnalysisConstantSpecific.qll @@ -74,7 +74,10 @@ module CppLangImplConstant implements LangSig { /** * Holds if `e >= bound` (if `upper = false`) or `e <= bound` (if `upper = true`). */ - predicate hasConstantBound(SemExpr e, float bound, boolean upper) { none() } + predicate hasConstantBound(SemExpr e, float bound, boolean upper, SemReason reason) { + semHasConstantBoundConstantSpecific(e, bound, upper) and + reason instanceof SemTypeReason + } /** * Holds if `e >= bound + delta` (if `upper = false`) or `e <= bound + delta` (if `upper = true`). diff --git a/cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/analysis/RangeAnalysisRelativeSpecific.qll b/cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/analysis/RangeAnalysisRelativeSpecific.qll index 341fab17376..ba66793aae9 100644 --- a/cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/analysis/RangeAnalysisRelativeSpecific.qll +++ b/cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/analysis/RangeAnalysisRelativeSpecific.qll @@ -110,7 +110,7 @@ module CppLangImplRelative implements LangSig { /** * Holds if `e >= bound` (if `upper = false`) or `e <= bound` (if `upper = true`). */ - predicate hasConstantBound(SemExpr e, float bound, boolean upper) { none() } + predicate hasConstantBound(SemExpr e, float bound, boolean upper, SemReason reason) { none() } /** * Holds if `e >= bound + delta` (if `upper = false`) or `e <= bound + delta` (if `upper = true`). diff --git a/cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/analysis/RangeAnalysisStage.qll b/cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/analysis/RangeAnalysisStage.qll index a6cacd8b4a7..3d0af5754d3 100644 --- a/cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/analysis/RangeAnalysisStage.qll +++ b/cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/analysis/RangeAnalysisStage.qll @@ -155,7 +155,7 @@ signature module LangSig { /** * Holds if `e >= bound` (if `upper = false`) or `e <= bound` (if `upper = true`). */ - predicate hasConstantBound(SemExpr e, D::Delta bound, boolean upper); + predicate hasConstantBound(SemExpr e, D::Delta bound, boolean upper, SemReason reason); /** * Holds if `e >= bound + delta` (if `upper = false`) or `e <= bound + delta` (if `upper = true`). @@ -920,14 +920,15 @@ module RangeStage< * Holds if `e` has an upper (for `upper = true`) or lower * (for `upper = false`) bound of `b`. */ - private predicate baseBound(SemExpr e, D::Delta b, boolean upper) { - hasConstantBound(e, b, upper) + private predicate baseBound(SemExpr e, D::Delta b, boolean upper, SemReason reason) { + hasConstantBound(e, b, upper, reason) or upper = false and b = D::fromInt(0) and semPositive(e.(SemBitAndExpr).getAnOperand()) and // REVIEW: We let the language opt out here to preserve original results. - not ignoreZeroLowerBound(e) + not ignoreZeroLowerBound(e) and + reason instanceof SemNoReason } /** @@ -1055,11 +1056,10 @@ module RangeStage< origdelta = delta and reason instanceof SemNoReason or - baseBound(e, delta, upper) and + baseBound(e, delta, upper, reason) and b instanceof SemZeroBound and fromBackEdge = false and - origdelta = delta and - reason instanceof SemNoReason + origdelta = delta or exists(SemSsaVariable v, SemSsaReadPositionBlock bb | boundedSsa(v, bb, b, delta, upper, fromBackEdge, origdelta, reason) and diff --git a/cpp/ql/lib/semmle/code/cpp/security/InvalidPointerDereference/RangeAnalysisUtil.qll b/cpp/ql/lib/semmle/code/cpp/security/InvalidPointerDereference/RangeAnalysisUtil.qll index 12bb50321fa..72e27d7804c 100644 --- a/cpp/ql/lib/semmle/code/cpp/security/InvalidPointerDereference/RangeAnalysisUtil.qll +++ b/cpp/ql/lib/semmle/code/cpp/security/InvalidPointerDereference/RangeAnalysisUtil.qll @@ -20,7 +20,8 @@ private Instruction getABoundIn(SemBound b, IRFunction func) { pragma[inline] private predicate boundedImpl(Instruction i, Instruction b, int delta) { exists(SemBound bound, IRFunction func | - semBounded(getSemanticExpr(i), bound, delta, true, _) and + semBounded(getSemanticExpr(i), bound, delta, true, + any(SemReason reason | not reason instanceof SemTypeReason)) and b = getABoundIn(bound, func) and i.getEnclosingIRFunction() = func ) diff --git a/cpp/ql/lib/semmlecode.cpp.dbscheme b/cpp/ql/lib/semmlecode.cpp.dbscheme index 19887dbd333..d77c09d8bdc 100644 --- a/cpp/ql/lib/semmlecode.cpp.dbscheme +++ b/cpp/ql/lib/semmlecode.cpp.dbscheme @@ -608,7 +608,7 @@ case @builtintype.kind of | 47 = @std_float64 // _Float64 | 48 = @float64x // _Float64x | 49 = @std_float128 // _Float128 -| 50 = @float128x // _Float128x +// ... 50 _Float128x | 51 = @char8_t | 52 = @float16 // _Float16 | 53 = @complex_float16 // _Complex _Float16 diff --git a/cpp/ql/lib/semmlecode.cpp.dbscheme.stats b/cpp/ql/lib/semmlecode.cpp.dbscheme.stats index 24253b0aeb5..23f1331ee6d 100644 --- a/cpp/ql/lib/semmlecode.cpp.dbscheme.stats +++ b/cpp/ql/lib/semmlecode.cpp.dbscheme.stats @@ -16,14 +16,14 @@ @svnentry 575525 - - @location_stmt - 3813678 - @location_default 29769315 + + @location_stmt + 3813678 + @location_expr 13166528 @@ -90,7 +90,7 @@ @localvariable - 581207 + 581199 @enumconstant @@ -276,10 +276,6 @@ @std_float128 466 - - @float128x - 466 - @char8_t 466 @@ -296,10 +292,6 @@ @pointer 567483 - - @reference - 1747669 - @type_with_specifiers 1009066 @@ -313,8 +305,8 @@ 635462 - @rvalue_reference - 613493 + @reference + 1747669 @gnu_vector @@ -324,6 +316,10 @@ @routinereference 242 + + @rvalue_reference + 613493 + @block 10 @@ -364,10 +360,6 @@ @stdattribute 492036 - - @alignas - 9792 - @declspec 243716 @@ -377,12 +369,12 @@ 3 - @attribute_arg_token - 39168 + @alignas + 9792 - @attribute_arg_type - 466 + @attribute_arg_token + 39168 @attribute_arg_constant_expr @@ -396,6 +388,10 @@ @attribute_arg_constant 1 + + @attribute_arg_type + 466 + @derivation 368257 @@ -432,34 +428,18 @@ @lambdacapture 27977 - - @errorexpr - 46892 - @address_of 438815 - - @reference_to - 1592467 - @indirect 292170 - - @ref_indirect - 1938672 - @array_to_pointer 1428626 - - @vacuous_destructor_call - 8150 - @parexpr 3581934 @@ -468,6 +448,10 @@ @arithnegexpr 650875 + + @unaryplusexpr + 2911 + @complementexpr 27792 @@ -610,7 +594,7 @@ @assignorexpr - 23831 + 23830 @assignxorexpr @@ -620,6 +604,10 @@ @assignpaddexpr 13606 + + @assignpsubexpr + 1151 + @andlogicalexpr 249546 @@ -634,7 +622,7 @@ @subscriptexpr - 367604 + 367599 @callexpr @@ -644,6 +632,10 @@ @vastartexpr 3706 + + @vaargexpr + 950 + @vaendexpr 2797 @@ -656,34 +648,6 @@ @varaccess 6019333 - - @thisaccess - 1127035 - - - @new_expr - 47668 - - - @delete_expr - 11749 - - - @throw_expr - 21695 - - - @condition_decl - 42427 - - - @braced_init_list - 1108 - - - @type_id - 36483 - @runtime_sizeof 295357 @@ -693,8 +657,8 @@ 49891 - @sizeof_pack - 5595 + @expr_stmt + 94234 @routineexpr @@ -705,16 +669,12 @@ 1126930 - @isemptyexpr - 1481 + @offsetofexpr + 19960 - @ispodexpr - 634 - - - @hastrivialdestructor - 466 + @typescompexpr + 562865 @literal @@ -724,105 +684,33 @@ @aggregateliteral 913874 - - @delete_array_expr - 1406 - - - @new_array_expr - 5103 - - - @ctordirectinit - 112978 - - - @ctorvirtualinit - 6513 - - - @ctorfieldinit - 201118 - - - @ctordelegatinginit - 3351 - - - @dtordirectdestruct - 41776 - - - @dtorvirtualdestruct - 4128 - - - @dtorfielddestruct - 41705 - - - @static_cast - 210934 - - - @reinterpret_cast - 30752 - - - @const_cast - 35250 - - - @dynamic_cast - 1037 - @c_style_cast 4209495 - - @lambdaexpr - 21449 - - - @param_ref - 245656 - - - @istrivialexpr - 932 - - - @istriviallycopyableexpr - 3730 - - - @isconstructibleexpr - 466 - - - @isfinalexpr - 1693 - - - @noexceptexpr - 25737 - - - @builtinaddressof - 13302 - @temp_init 826674 - @assume - 3203 + @errorexpr + 46892 - @unaryplusexpr - 2911 + @reference_to + 1592467 + + + @ref_indirect + 1938672 + + + @vacuous_destructor_call + 8150 + + + @assume + 3203 @conjugation @@ -868,25 +756,41 @@ @maxexpr 1 - - @assignpsubexpr - 1151 - @virtfunptrexpr 1 - @vaargexpr - 950 + @thisaccess + 1127035 - @expr_stmt - 94234 + @new_expr + 47668 - @offsetofexpr - 19960 + @delete_expr + 11749 + + + @throw_expr + 21695 + + + @condition_decl + 42427 + + + @braced_init_list + 1108 + + + @type_id + 36483 + + + @sizeof_pack + 5595 @hasassignexpr @@ -944,10 +848,18 @@ @isconvtoexpr 104 + + @isemptyexpr + 1481 + @isenumexpr 522 + + @ispodexpr + 634 + @ispolyexpr 3 @@ -956,22 +868,82 @@ @isunionexpr 5 - - @typescompexpr - 562865 - @intaddrexpr 1 + + @hastrivialdestructor + 466 + @uuidof 20120 + + @delete_array_expr + 1406 + + + @new_array_expr + 5103 + @foldexpr 4 + + @ctordirectinit + 112978 + + + @ctorvirtualinit + 6513 + + + @ctorfieldinit + 201118 + + + @ctordelegatinginit + 3351 + + + @dtordirectdestruct + 41776 + + + @dtorvirtualdestruct + 4128 + + + @dtorfielddestruct + 41705 + + + @static_cast + 210934 + + + @reinterpret_cast + 30752 + + + @const_cast + 35250 + + + @dynamic_cast + 1037 + + + @lambdaexpr + 21449 + + + @param_ref + 245656 + @noopexpr 37 @@ -1000,10 +972,18 @@ @isnothrowassignableexpr 4183 + + @istrivialexpr + 932 + @isstandardlayoutexpr 2 + + @istriviallycopyableexpr + 3730 + @isliteraltypeexpr 2 @@ -1020,6 +1000,10 @@ @hasnothrowmoveassignexpr 4 + + @isconstructibleexpr + 466 + @isnothrowconstructibleexpr 14434 @@ -1056,6 +1040,14 @@ @isvalueclassexpr 1 + + @isfinalexpr + 1693 + + + @noexceptexpr + 25737 + @builtinshufflevector 1 @@ -1064,6 +1056,10 @@ @builtinchooseexpr 9050 + + @builtinaddressof + 13302 + @vec_fill 1 @@ -1232,6 +1228,10 @@ @stmt_while 30110 + + @stmt_goto + 110511 + @stmt_label 53056 @@ -1261,8 +1261,8 @@ 20753 - @stmt_try_block - 46921 + @stmt_asm + 109804 @stmt_decl @@ -1270,7 +1270,7 @@ @stmt_empty - 193324 + 193321 @stmt_continue @@ -1281,24 +1281,8 @@ 102331 - @stmt_range_based_for - 8393 - - - @stmt_handler - 65314 - - - @stmt_constexpr_if - 52508 - - - @stmt_goto - 110511 - - - @stmt_asm - 109804 + @stmt_try_block + 46921 @stmt_microsoft_try @@ -1316,6 +1300,18 @@ @stmt_assigned_goto 9061 + + @stmt_range_based_for + 8393 + + + @stmt_handler + 65314 + + + @stmt_constexpr_if + 52508 + @stmt_co_return 2 @@ -1356,10 +1352,6 @@ @ppd_undef 258328 - - @ppd_pragma - 312020 - @ppd_include_next 1865 @@ -1372,6 +1364,10 @@ @ppd_error 46 + + @ppd_pragma + 312020 + @ppd_objc_import 2 @@ -2008,7 +2004,7 @@ seconds - 10051 + 14798 @@ -2089,12 +2085,12 @@ 3 4 - 678 + 239 4 5 - 319 + 757 6 @@ -2109,31 +2105,31 @@ 10 11 - 79 - - - 11 - 12 - 159 - - - 13 - 17 119 - 17 + 11 + 14 + 159 + + + 14 + 17 + 79 + + + 18 21 159 21 - 44 + 45 159 - 55 - 89 + 64 + 130 79 @@ -2202,37 +2198,47 @@ 3 4 - 1236 + 398 4 5 - 518 + 1356 5 6 - 159 + 199 6 7 - 558 + 239 7 - 9 + 8 279 + + 8 + 9 + 39 + 9 - 24 + 10 + 319 + + + 11 + 28 279 - 24 - 91 - 279 + 28 + 96 + 199 @@ -2277,24 +2283,19 @@ 12 - - 3 - 4 - 39 - 4 5 + 79 + + + 182 + 183 39 - 120 - 121 - 39 - - - 129 - 130 + 189 + 190 39 @@ -2311,27 +2312,22 @@ 1 2 - 5823 + 10011 2 3 - 2074 + 3549 3 - 4 - 917 + 5 + 1116 - 4 - 6 - 917 - - - 6 - 40 - 319 + 18 + 47 + 119 @@ -2347,32 +2343,22 @@ 1 2 - 5065 + 9572 2 3 - 1794 + 3470 3 4 - 1236 + 1037 4 - 5 - 757 - - - 5 - 7 - 877 - - - 7 - 67 - 319 + 74 + 717 @@ -2388,12 +2374,12 @@ 1 2 - 9892 + 14518 2 - 3 - 159 + 4 + 279 @@ -2762,11 +2748,11 @@ cpu_seconds - 7838 + 7389 elapsed_seconds - 161 + 138 @@ -2812,17 +2798,17 @@ 1 2 - 6605 + 5948 2 3 - 853 + 979 3 - 12 - 380 + 15 + 461 @@ -2838,12 +2824,12 @@ 1 2 - 7354 + 6778 2 3 - 484 + 610 @@ -2859,56 +2845,51 @@ 1 2 - 23 + 11 2 3 - 23 - - - 4 - 5 - 11 + 34 7 8 - 23 - - - 11 - 12 11 - 25 - 26 + 9 + 10 11 - 52 - 53 + 12 + 13 11 - 114 - 115 + 37 + 38 11 - 145 - 146 + 150 + 151 11 - 244 - 245 + 172 + 173 11 - 248 - 249 + 211 + 212 + 11 + + + 258 + 259 11 @@ -2925,56 +2906,51 @@ 1 2 - 23 + 11 2 3 - 23 - - - 4 - 5 - 11 + 34 7 8 - 23 - - - 11 - 12 11 - 25 - 26 + 9 + 10 11 - 51 - 52 + 12 + 13 11 - 102 - 103 + 36 + 37 11 - 107 - 108 + 125 + 126 11 - 172 - 173 + 126 + 127 11 - 230 - 231 + 145 + 146 + 11 + + + 227 + 228 11 @@ -12311,7 +12287,7 @@ 2 3 - 543265 + 543264 3 @@ -12346,7 +12322,7 @@ 11 337 - 224500 + 224502 339 @@ -18397,11 +18373,11 @@ localvariables - 581207 + 581199 id - 581207 + 581199 type_id @@ -18409,7 +18385,7 @@ name - 91326 + 91325 @@ -18423,7 +18399,7 @@ 1 2 - 581207 + 581199 @@ -18439,7 +18415,7 @@ 1 2 - 581207 + 581199 @@ -18532,7 +18508,7 @@ 1 2 - 57522 + 57521 2 @@ -18568,7 +18544,7 @@ 1 2 - 77150 + 77148 2 @@ -19606,19 +19582,19 @@ builtintypes - 22848 + 22382 id - 22848 + 22382 name - 22848 + 22382 kind - 22848 + 22382 size @@ -19644,7 +19620,7 @@ 1 2 - 22848 + 22382 @@ -19660,7 +19636,7 @@ 1 2 - 22848 + 22382 @@ -19676,7 +19652,7 @@ 1 2 - 22848 + 22382 @@ -19692,7 +19668,7 @@ 1 2 - 22848 + 22382 @@ -19708,7 +19684,7 @@ 1 2 - 22848 + 22382 @@ -19724,7 +19700,7 @@ 1 2 - 22848 + 22382 @@ -19740,7 +19716,7 @@ 1 2 - 22848 + 22382 @@ -19756,7 +19732,7 @@ 1 2 - 22848 + 22382 @@ -19772,7 +19748,7 @@ 1 2 - 22848 + 22382 @@ -19788,7 +19764,7 @@ 1 2 - 22848 + 22382 @@ -19804,7 +19780,7 @@ 1 2 - 22848 + 22382 @@ -19820,7 +19796,7 @@ 1 2 - 22848 + 22382 @@ -19836,7 +19812,7 @@ 1 2 - 22848 + 22382 @@ -19852,7 +19828,7 @@ 1 2 - 22848 + 22382 @@ -19868,7 +19844,7 @@ 1 2 - 22848 + 22382 @@ -19887,8 +19863,8 @@ 466 - 3 - 4 + 2 + 3 466 @@ -19928,8 +19904,8 @@ 466 - 3 - 4 + 2 + 3 466 @@ -19969,8 +19945,8 @@ 466 - 3 - 4 + 2 + 3 466 @@ -20057,8 +20033,8 @@ 466 - 31 - 32 + 30 + 31 466 @@ -20083,8 +20059,8 @@ 466 - 31 - 32 + 30 + 31 466 @@ -20109,8 +20085,8 @@ 466 - 31 - 32 + 30 + 31 466 @@ -20177,8 +20153,8 @@ 466 - 12 - 13 + 11 + 12 466 @@ -20213,8 +20189,8 @@ 466 - 12 - 13 + 11 + 12 466 @@ -20249,8 +20225,8 @@ 466 - 12 - 13 + 11 + 12 466 @@ -21468,7 +21444,7 @@ name - 1348530 + 1348063 kind @@ -21528,7 +21504,7 @@ 3 7 - 104916 + 104450 7 @@ -21559,12 +21535,12 @@ 2 3 - 125900 + 124967 3 7 - 14921 + 15387 @@ -21689,8 +21665,8 @@ 466 - 740 - 741 + 739 + 740 466 @@ -26371,15 +26347,15 @@ unspecifiedtype - 10137757 + 10137290 type_id - 10137757 + 10137290 unspecified_type_id - 6813527 + 6813061 @@ -26393,7 +26369,7 @@ 1 2 - 10137757 + 10137290 @@ -26409,7 +26385,7 @@ 1 2 - 4583230 + 4582764 2 @@ -28996,11 +28972,11 @@ expr_isload - 4981760 + 4981370 expr_id - 4981760 + 4981370 @@ -31127,11 +31103,11 @@ kind - 3387 + 1163 location - 3592604 + 8474327 @@ -31176,68 +31152,73 @@ 1 - 3 - 246 + 10 + 88 - 4 - 14 - 282 + 12 + 18 + 88 - 14 - 38 - 282 + 26 + 100 + 88 - 42 - 83 - 282 + 105 + 305 + 88 - 85 - 142 - 282 + 320 + 417 + 88 - 145 - 339 - 282 + 466 + 784 + 88 - 364 - 564 - 282 + 892 + 1647 + 88 - 653 - 832 - 282 + 1656 + 2402 + 88 - 973 - 1185 - 282 + 3130 + 4067 + 88 - 1329 - 2628 - 282 + 4297 + 4915 + 88 - 3015 - 6253 - 282 + 5184 + 20397 + 88 - 6592 - 78904 - 282 + 25254 + 49060 + 88 - 109469 - 109470 - 35 + 63119 + 136808 + 88 + + + 285063 + 285064 + 17 @@ -31252,68 +31233,73 @@ 1 - 2 - 282 + 9 + 88 - 2 - 3 - 176 + 9 + 15 + 88 - 3 - 6 - 282 + 17 + 96 + 88 - 6 - 13 - 282 + 99 + 222 + 88 - 14 - 26 - 282 + 260 + 383 + 88 - 28 - 62 - 246 + 408 + 577 + 88 - 63 - 83 - 282 + 595 + 749 + 88 - 91 - 183 - 282 + 864 + 1774 + 88 - 206 - 342 - 282 + 1812 + 2545 + 88 - 353 - 448 - 282 + 2601 + 2910 + 88 - 469 - 1020 - 282 + 3419 + 4913 + 88 - 1051 - 14619 - 282 + 5471 + 21108 + 88 - 16978 - 32762 - 141 + 26251 + 76182 + 88 + + + 223904 + 223905 + 17 @@ -31329,32 +31315,22 @@ 1 2 - 1941777 + 7133647 2 3 - 838486 + 661870 3 - 4 - 247163 + 18 + 637071 - 4 - 8 - 284529 - - - 8 - 155 - 269533 - - - 155 - 53477 - 11114 + 18 + 71656 + 41738 @@ -31370,22 +31346,17 @@ 1 2 - 2392776 + 7239544 2 3 - 875922 + 617153 3 - 6 - 306969 - - - 6 - 25 - 16936 + 32 + 617629 @@ -35621,11 +35592,11 @@ stmt_decl_bind - 585138 + 585129 stmt - 545022 + 545014 num @@ -35633,7 +35604,7 @@ decl - 585033 + 585024 @@ -35647,12 +35618,12 @@ 1 2 - 524159 + 524152 2 19 - 20862 + 20861 @@ -35668,12 +35639,12 @@ 1 2 - 524159 + 524152 2 19 - 20862 + 20861 @@ -35871,7 +35842,7 @@ 1 2 - 584995 + 584986 2 @@ -35892,7 +35863,7 @@ 1 2 - 585033 + 585024 @@ -35902,11 +35873,11 @@ stmt_decl_entry_bind - 527594 + 527587 stmt - 487781 + 487773 num @@ -35914,7 +35885,7 @@ decl_entry - 527536 + 527528 @@ -35928,7 +35899,7 @@ 1 2 - 467183 + 467176 2 @@ -35949,7 +35920,7 @@ 1 2 - 467183 + 467176 2 @@ -36152,7 +36123,7 @@ 1 2 - 527515 + 527507 3 @@ -36173,7 +36144,7 @@ 1 2 - 527536 + 527528 diff --git a/cpp/ql/lib/upgrades/19887dbd33327fb07d54251786e0cb2578539775/builtintypes.ql b/cpp/ql/lib/upgrades/19887dbd33327fb07d54251786e0cb2578539775/builtintypes.ql new file mode 100644 index 00000000000..5d1b39088b6 --- /dev/null +++ b/cpp/ql/lib/upgrades/19887dbd33327fb07d54251786e0cb2578539775/builtintypes.ql @@ -0,0 +1,13 @@ +class BuiltinType extends @builtintype { + string toString() { none() } +} + +predicate isFloat128xBuiltinType(BuiltinType type) { + exists(int kind | builtintypes(type, _, kind, _, _, _) | kind = 50) +} + +from BuiltinType type, string name, int kind, int kind_new, int size, int sign, int alignment +where + builtintypes(type, name, kind, size, sign, alignment) and + if isFloat128xBuiltinType(type) then kind_new = 1 else kind_new = kind +select type, name, kind_new, size, sign, alignment diff --git a/cpp/ql/lib/upgrades/19887dbd33327fb07d54251786e0cb2578539775/old.dbscheme b/cpp/ql/lib/upgrades/19887dbd33327fb07d54251786e0cb2578539775/old.dbscheme new file mode 100644 index 00000000000..19887dbd333 --- /dev/null +++ b/cpp/ql/lib/upgrades/19887dbd33327fb07d54251786e0cb2578539775/old.dbscheme @@ -0,0 +1,2212 @@ + +/** + * An invocation of the compiler. Note that more than one file may be + * compiled per invocation. For example, this command compiles three + * source files: + * + * gcc -c f1.c f2.c f3.c + * + * The `id` simply identifies the invocation, while `cwd` is the working + * directory from which the compiler was invoked. + */ +compilations( + /** + * An invocation of the compiler. Note that more than one file may + * be compiled per invocation. For example, this command compiles + * three source files: + * + * gcc -c f1.c f2.c f3.c + */ + unique int id : @compilation, + string cwd : string ref +); + +/** + * The arguments that were passed to the extractor for a compiler + * invocation. If `id` is for the compiler invocation + * + * gcc -c f1.c f2.c f3.c + * + * then typically there will be rows for + * + * num | arg + * --- | --- + * 0 | *path to extractor* + * 1 | `--mimic` + * 2 | `/usr/bin/gcc` + * 3 | `-c` + * 4 | f1.c + * 5 | f2.c + * 6 | f3.c + */ +#keyset[id, num] +compilation_args( + int id : @compilation ref, + int num : int ref, + string arg : string ref +); + +/** + * The source files that are compiled by a compiler invocation. + * If `id` is for the compiler invocation + * + * gcc -c f1.c f2.c f3.c + * + * then there will be rows for + * + * num | arg + * --- | --- + * 0 | f1.c + * 1 | f2.c + * 2 | f3.c + * + * Note that even if those files `#include` headers, those headers + * do not appear as rows. + */ +#keyset[id, num] +compilation_compiling_files( + int id : @compilation ref, + int num : int ref, + int file : @file ref +); + +/** + * The time taken by the extractor for a compiler invocation. + * + * For each file `num`, there will be rows for + * + * kind | seconds + * ---- | --- + * 1 | CPU seconds used by the extractor frontend + * 2 | Elapsed seconds during the extractor frontend + * 3 | CPU seconds used by the extractor backend + * 4 | Elapsed seconds during the extractor backend + */ +#keyset[id, num, kind] +compilation_time( + int id : @compilation ref, + int num : int ref, + /* kind: + 1 = frontend_cpu_seconds + 2 = frontend_elapsed_seconds + 3 = extractor_cpu_seconds + 4 = extractor_elapsed_seconds + */ + int kind : int ref, + float seconds : float ref +); + +/** + * An error or warning generated by the extractor. + * The diagnostic message `diagnostic` was generated during compiler + * invocation `compilation`, and is the `file_number_diagnostic_number`th + * message generated while extracting the `file_number`th file of that + * invocation. + */ +#keyset[compilation, file_number, file_number_diagnostic_number] +diagnostic_for( + int diagnostic : @diagnostic ref, + int compilation : @compilation ref, + int file_number : int ref, + int file_number_diagnostic_number : int ref +); + +/** + * If extraction was successful, then `cpu_seconds` and + * `elapsed_seconds` are the CPU time and elapsed time (respectively) + * that extraction took for compiler invocation `id`. + */ +compilation_finished( + unique int id : @compilation ref, + float cpu_seconds : float ref, + float elapsed_seconds : float ref +); + + +/** + * External data, loaded from CSV files during snapshot creation. See + * [Tutorial: Incorporating external data](https://help.semmle.com/wiki/display/SD/Tutorial%3A+Incorporating+external+data) + * for more information. + */ +externalData( + int id : @externalDataElement, + string path : string ref, + int column: int ref, + string value : string ref +); + +/** + * The source location of the snapshot. + */ +sourceLocationPrefix(string prefix : string ref); + +/** + * Information about packages that provide code used during compilation. + * The `id` is just a unique identifier. + * The `namespace` is typically the name of the package manager that + * provided the package (e.g. "dpkg" or "yum"). + * The `package_name` is the name of the package, and `version` is its + * version (as a string). + */ +external_packages( + unique int id: @external_package, + string namespace : string ref, + string package_name : string ref, + string version : string ref +); + +/** + * Holds if File `fileid` was provided by package `package`. + */ +header_to_external_package( + int fileid : @file ref, + int package : @external_package ref +); + +/* + * Version history + */ + +svnentries( + unique int id : @svnentry, + string revision : string ref, + string author : string ref, + date revisionDate : date ref, + int changeSize : int ref +) + +svnaffectedfiles( + int id : @svnentry ref, + int file : @file ref, + string action : string ref +) + +svnentrymsg( + unique int id : @svnentry ref, + string message : string ref +) + +svnchurn( + int commit : @svnentry ref, + int file : @file ref, + int addedLines : int ref, + int deletedLines : int ref +) + +/* + * C++ dbscheme + */ + +@location = @location_stmt | @location_expr | @location_default ; + +/** + * The location of an element that is not an expression or a statement. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `file`. + * For more information, see + * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/). + */ +locations_default( + /** The location of an element that is not an expression or a statement. */ + unique int id: @location_default, + int container: @container ref, + int startLine: int ref, + int startColumn: int ref, + int endLine: int ref, + int endColumn: int ref +); + +/** + * The location of a statement. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `file`. + * For more information, see + * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/). + */ +locations_stmt( + /** The location of a statement. */ + unique int id: @location_stmt, + int container: @container ref, + int startLine: int ref, + int startColumn: int ref, + int endLine: int ref, + int endColumn: int ref +); + +/** + * The location of an expression. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `file`. + * For more information, see + * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/). + */ +locations_expr( + /** The location of an expression. */ + unique int id: @location_expr, + int container: @container ref, + int startLine: int ref, + int startColumn: int ref, + int endLine: int ref, + int endColumn: int ref +); + +/** An element for which line-count information is available. */ +@sourceline = @file | @function | @variable | @enumconstant | @xmllocatable; + +numlines( + int element_id: @sourceline ref, + int num_lines: int ref, + int num_code: int ref, + int num_comment: int ref +); + +diagnostics( + unique int id: @diagnostic, + int severity: int ref, + string error_tag: string ref, + string error_message: string ref, + string full_error_message: string ref, + int location: @location_default ref +); + +files( + unique int id: @file, + string name: string ref +); + +folders( + unique int id: @folder, + string name: string ref +); + +@container = @folder | @file + +containerparent( + int parent: @container ref, + unique int child: @container ref +); + +fileannotations( + int id: @file ref, + int kind: int ref, + string name: string ref, + string value: string ref +); + +inmacroexpansion( + int id: @element ref, + int inv: @macroinvocation ref +); + +affectedbymacroexpansion( + int id: @element ref, + int inv: @macroinvocation ref +); + +case @macroinvocation.kind of + 1 = @macro_expansion +| 2 = @other_macro_reference +; + +macroinvocations( + unique int id: @macroinvocation, + int macro_id: @ppd_define ref, + int location: @location_default ref, + int kind: int ref +); + +macroparent( + unique int id: @macroinvocation ref, + int parent_id: @macroinvocation ref +); + +// a macroinvocation may be part of another location +// the way to find a constant expression that uses a macro +// is thus to find a constant expression that has a location +// to which a macro invocation is bound +macrolocationbind( + int id: @macroinvocation ref, + int location: @location ref +); + +#keyset[invocation, argument_index] +macro_argument_unexpanded( + int invocation: @macroinvocation ref, + int argument_index: int ref, + string text: string ref +); + +#keyset[invocation, argument_index] +macro_argument_expanded( + int invocation: @macroinvocation ref, + int argument_index: int ref, + string text: string ref +); + +/* +case @function.kind of + 1 = @normal_function +| 2 = @constructor +| 3 = @destructor +| 4 = @conversion_function +| 5 = @operator +| 6 = @builtin_function // GCC built-in functions, e.g. __builtin___memcpy_chk +; +*/ + +functions( + unique int id: @function, + string name: string ref, + int kind: int ref +); + +function_entry_point( + int id: @function ref, + unique int entry_point: @stmt ref +); + +function_return_type( + int id: @function ref, + int return_type: @type ref +); + +/** + * If `function` is a coroutine, then this gives the `std::experimental::resumable_traits` + * instance associated with it, and the variables representing the `handle` and `promise` + * for it. + */ +coroutine( + unique int function: @function ref, + int traits: @type ref, + int handle: @variable ref, + int promise: @variable ref +); + +/** The `new` function used for allocating the coroutine state, if any. */ +coroutine_new( + unique int function: @function ref, + int new: @function ref +); + +/** The `delete` function used for deallocating the coroutine state, if any. */ +coroutine_delete( + unique int function: @function ref, + int delete: @function ref +); + +purefunctions(unique int id: @function ref); + +function_deleted(unique int id: @function ref); + +function_defaulted(unique int id: @function ref); + +member_function_this_type( + unique int id: @function ref, + int this_type: @type ref +); + +#keyset[id, type_id] +fun_decls( + int id: @fun_decl, + int function: @function ref, + int type_id: @type ref, + string name: string ref, + int location: @location_default ref +); +fun_def(unique int id: @fun_decl ref); +fun_specialized(unique int id: @fun_decl ref); +fun_implicit(unique int id: @fun_decl ref); +fun_decl_specifiers( + int id: @fun_decl ref, + string name: string ref +) +#keyset[fun_decl, index] +fun_decl_throws( + int fun_decl: @fun_decl ref, + int index: int ref, + int type_id: @type ref +); +/* an empty throw specification is different from none */ +fun_decl_empty_throws(unique int fun_decl: @fun_decl ref); +fun_decl_noexcept( + int fun_decl: @fun_decl ref, + int constant: @expr ref +); +fun_decl_empty_noexcept(int fun_decl: @fun_decl ref); +fun_decl_typedef_type( + unique int fun_decl: @fun_decl ref, + int typedeftype_id: @usertype ref +); + +param_decl_bind( + unique int id: @var_decl ref, + int index: int ref, + int fun_decl: @fun_decl ref +); + +#keyset[id, type_id] +var_decls( + int id: @var_decl, + int variable: @variable ref, + int type_id: @type ref, + string name: string ref, + int location: @location_default ref +); +var_def(unique int id: @var_decl ref); +var_decl_specifiers( + int id: @var_decl ref, + string name: string ref +) +is_structured_binding(unique int id: @variable ref); + +type_decls( + unique int id: @type_decl, + int type_id: @type ref, + int location: @location_default ref +); +type_def(unique int id: @type_decl ref); +type_decl_top( + unique int type_decl: @type_decl ref +); + +namespace_decls( + unique int id: @namespace_decl, + int namespace_id: @namespace ref, + int location: @location_default ref, + int bodylocation: @location_default ref +); + +usings( + unique int id: @using, + int element_id: @element ref, + int location: @location_default ref +); + +/** The element which contains the `using` declaration. */ +using_container( + int parent: @element ref, + int child: @using ref +); + +static_asserts( + unique int id: @static_assert, + int condition : @expr ref, + string message : string ref, + int location: @location_default ref, + int enclosing : @element ref +); + +// each function has an ordered list of parameters +#keyset[id, type_id] +#keyset[function, index, type_id] +params( + int id: @parameter, + int function: @functionorblock ref, + int index: int ref, + int type_id: @type ref +); + +overrides( + int new: @function ref, + int old: @function ref +); + +#keyset[id, type_id] +membervariables( + int id: @membervariable, + int type_id: @type ref, + string name: string ref +); + +#keyset[id, type_id] +globalvariables( + int id: @globalvariable, + int type_id: @type ref, + string name: string ref +); + +#keyset[id, type_id] +localvariables( + int id: @localvariable, + int type_id: @type ref, + string name: string ref +); + +autoderivation( + unique int var: @variable ref, + int derivation_type: @type ref +); + +orphaned_variables( + int var: @localvariable ref, + int function: @function ref +) + +enumconstants( + unique int id: @enumconstant, + int parent: @usertype ref, + int index: int ref, + int type_id: @type ref, + string name: string ref, + int location: @location_default ref +); + +@variable = @localscopevariable | @globalvariable | @membervariable; + +@localscopevariable = @localvariable | @parameter; + +/** + * Built-in types are the fundamental types, e.g., integral, floating, and void. + */ +case @builtintype.kind of + 1 = @errortype +| 2 = @unknowntype +| 3 = @void +| 4 = @boolean +| 5 = @char +| 6 = @unsigned_char +| 7 = @signed_char +| 8 = @short +| 9 = @unsigned_short +| 10 = @signed_short +| 11 = @int +| 12 = @unsigned_int +| 13 = @signed_int +| 14 = @long +| 15 = @unsigned_long +| 16 = @signed_long +| 17 = @long_long +| 18 = @unsigned_long_long +| 19 = @signed_long_long +// ... 20 Microsoft-specific __int8 +// ... 21 Microsoft-specific __int16 +// ... 22 Microsoft-specific __int32 +// ... 23 Microsoft-specific __int64 +| 24 = @float +| 25 = @double +| 26 = @long_double +| 27 = @complex_float // C99-specific _Complex float +| 28 = @complex_double // C99-specific _Complex double +| 29 = @complex_long_double // C99-specific _Complex long double +| 30 = @imaginary_float // C99-specific _Imaginary float +| 31 = @imaginary_double // C99-specific _Imaginary double +| 32 = @imaginary_long_double // C99-specific _Imaginary long double +| 33 = @wchar_t // Microsoft-specific +| 34 = @decltype_nullptr // C++11 +| 35 = @int128 // __int128 +| 36 = @unsigned_int128 // unsigned __int128 +| 37 = @signed_int128 // signed __int128 +| 38 = @float128 // __float128 +| 39 = @complex_float128 // _Complex __float128 +| 40 = @decimal32 // _Decimal32 +| 41 = @decimal64 // _Decimal64 +| 42 = @decimal128 // _Decimal128 +| 43 = @char16_t +| 44 = @char32_t +| 45 = @std_float32 // _Float32 +| 46 = @float32x // _Float32x +| 47 = @std_float64 // _Float64 +| 48 = @float64x // _Float64x +| 49 = @std_float128 // _Float128 +| 50 = @float128x // _Float128x +| 51 = @char8_t +| 52 = @float16 // _Float16 +| 53 = @complex_float16 // _Complex _Float16 +; + +builtintypes( + unique int id: @builtintype, + string name: string ref, + int kind: int ref, + int size: int ref, + int sign: int ref, + int alignment: int ref +); + +/** + * Derived types are types that are directly derived from existing types and + * point to, refer to, transform type data to return a new type. + */ +case @derivedtype.kind of + 1 = @pointer +| 2 = @reference +| 3 = @type_with_specifiers +| 4 = @array +| 5 = @gnu_vector +| 6 = @routineptr +| 7 = @routinereference +| 8 = @rvalue_reference // C++11 +// ... 9 type_conforming_to_protocols deprecated +| 10 = @block +; + +derivedtypes( + unique int id: @derivedtype, + string name: string ref, + int kind: int ref, + int type_id: @type ref +); + +pointerishsize(unique int id: @derivedtype ref, + int size: int ref, + int alignment: int ref); + +arraysizes( + unique int id: @derivedtype ref, + int num_elements: int ref, + int bytesize: int ref, + int alignment: int ref +); + +typedefbase( + unique int id: @usertype ref, + int type_id: @type ref +); + +/** + * An instance of the C++11 `decltype` operator. For example: + * ``` + * int a; + * decltype(1+a) b; + * ``` + * Here `expr` is `1+a`. + * + * Sometimes an additional pair of parentheses around the expression + * would change the semantics of this decltype, e.g. + * ``` + * struct A { double x; }; + * const A* a = new A(); + * decltype( a->x ); // type is double + * decltype((a->x)); // type is const double& + * ``` + * (Please consult the C++11 standard for more details). + * `parentheses_would_change_meaning` is `true` iff that is the case. + */ +#keyset[id, expr] +decltypes( + int id: @decltype, + int expr: @expr ref, + int base_type: @type ref, + boolean parentheses_would_change_meaning: boolean ref +); + +/* +case @usertype.kind of + 1 = @struct +| 2 = @class +| 3 = @union +| 4 = @enum +| 5 = @typedef // classic C: typedef typedef type name +| 6 = @template +| 7 = @template_parameter +| 8 = @template_template_parameter +| 9 = @proxy_class // a proxy class associated with a template parameter +// ... 10 objc_class deprecated +// ... 11 objc_protocol deprecated +// ... 12 objc_category deprecated +| 13 = @scoped_enum +| 14 = @using_alias // a using name = type style typedef +; +*/ + +usertypes( + unique int id: @usertype, + string name: string ref, + int kind: int ref +); + +usertypesize( + unique int id: @usertype ref, + int size: int ref, + int alignment: int ref +); + +usertype_final(unique int id: @usertype ref); + +usertype_uuid( + unique int id: @usertype ref, + string uuid: string ref +); + +mangled_name( + unique int id: @declaration ref, + int mangled_name : @mangledname +); + +is_pod_class(unique int id: @usertype ref); +is_standard_layout_class(unique int id: @usertype ref); + +is_complete(unique int id: @usertype ref); + +is_class_template(unique int id: @usertype ref); +class_instantiation( + int to: @usertype ref, + int from: @usertype ref +); +class_template_argument( + int type_id: @usertype ref, + int index: int ref, + int arg_type: @type ref +); +class_template_argument_value( + int type_id: @usertype ref, + int index: int ref, + int arg_value: @expr ref +); + +is_proxy_class_for( + unique int id: @usertype ref, + unique int templ_param_id: @usertype ref +); + +type_mentions( + unique int id: @type_mention, + int type_id: @type ref, + int location: @location ref, + // a_symbol_reference_kind from the frontend. + int kind: int ref +); + +is_function_template(unique int id: @function ref); +function_instantiation( + unique int to: @function ref, + int from: @function ref +); +function_template_argument( + int function_id: @function ref, + int index: int ref, + int arg_type: @type ref +); +function_template_argument_value( + int function_id: @function ref, + int index: int ref, + int arg_value: @expr ref +); + +is_variable_template(unique int id: @variable ref); +variable_instantiation( + unique int to: @variable ref, + int from: @variable ref +); +variable_template_argument( + int variable_id: @variable ref, + int index: int ref, + int arg_type: @type ref +); +variable_template_argument_value( + int variable_id: @variable ref, + int index: int ref, + int arg_value: @expr ref +); + +/* + Fixed point types + precision(1) = short, precision(2) = default, precision(3) = long + is_unsigned(1) = unsigned is_unsigned(2) = signed + is_fract_type(1) = declared with _Fract + saturating(1) = declared with _Sat +*/ +/* TODO +fixedpointtypes( + unique int id: @fixedpointtype, + int precision: int ref, + int is_unsigned: int ref, + int is_fract_type: int ref, + int saturating: int ref); +*/ + +routinetypes( + unique int id: @routinetype, + int return_type: @type ref +); + +routinetypeargs( + int routine: @routinetype ref, + int index: int ref, + int type_id: @type ref +); + +ptrtomembers( + unique int id: @ptrtomember, + int type_id: @type ref, + int class_id: @type ref +); + +/* + specifiers for types, functions, and variables + + "public", + "protected", + "private", + + "const", + "volatile", + "static", + + "pure", + "virtual", + "sealed", // Microsoft + "__interface", // Microsoft + "inline", + "explicit", + + "near", // near far extension + "far", // near far extension + "__ptr32", // Microsoft + "__ptr64", // Microsoft + "__sptr", // Microsoft + "__uptr", // Microsoft + "dllimport", // Microsoft + "dllexport", // Microsoft + "thread", // Microsoft + "naked", // Microsoft + "microsoft_inline", // Microsoft + "forceinline", // Microsoft + "selectany", // Microsoft + "nothrow", // Microsoft + "novtable", // Microsoft + "noreturn", // Microsoft + "noinline", // Microsoft + "noalias", // Microsoft + "restrict", // Microsoft +*/ + +specifiers( + unique int id: @specifier, + unique string str: string ref +); + +typespecifiers( + int type_id: @type ref, + int spec_id: @specifier ref +); + +funspecifiers( + int func_id: @function ref, + int spec_id: @specifier ref +); + +varspecifiers( + int var_id: @accessible ref, + int spec_id: @specifier ref +); + +attributes( + unique int id: @attribute, + int kind: int ref, + string name: string ref, + string name_space: string ref, + int location: @location_default ref +); + +case @attribute.kind of + 0 = @gnuattribute +| 1 = @stdattribute +| 2 = @declspec +| 3 = @msattribute +| 4 = @alignas +// ... 5 @objc_propertyattribute deprecated +; + +attribute_args( + unique int id: @attribute_arg, + int kind: int ref, + int attribute: @attribute ref, + int index: int ref, + int location: @location_default ref +); + +case @attribute_arg.kind of + 0 = @attribute_arg_empty +| 1 = @attribute_arg_token +| 2 = @attribute_arg_constant +| 3 = @attribute_arg_type +| 4 = @attribute_arg_constant_expr +; + +attribute_arg_value( + unique int arg: @attribute_arg ref, + string value: string ref +); +attribute_arg_type( + unique int arg: @attribute_arg ref, + int type_id: @type ref +); +attribute_arg_constant( + unique int arg: @attribute_arg ref, + int constant: @expr ref +) +attribute_arg_name( + unique int arg: @attribute_arg ref, + string name: string ref +); + +typeattributes( + int type_id: @type ref, + int spec_id: @attribute ref +); + +funcattributes( + int func_id: @function ref, + int spec_id: @attribute ref +); + +varattributes( + int var_id: @accessible ref, + int spec_id: @attribute ref +); + +stmtattributes( + int stmt_id: @stmt ref, + int spec_id: @attribute ref +); + +@type = @builtintype + | @derivedtype + | @usertype + /* TODO | @fixedpointtype */ + | @routinetype + | @ptrtomember + | @decltype; + +unspecifiedtype( + unique int type_id: @type ref, + int unspecified_type_id: @type ref +); + +member( + int parent: @type ref, + int index: int ref, + int child: @member ref +); + +@enclosingfunction_child = @usertype | @variable | @namespace + +enclosingfunction( + unique int child: @enclosingfunction_child ref, + int parent: @function ref +); + +derivations( + unique int derivation: @derivation, + int sub: @type ref, + int index: int ref, + int super: @type ref, + int location: @location_default ref +); + +derspecifiers( + int der_id: @derivation ref, + int spec_id: @specifier ref +); + +/** + * Contains the byte offset of the base class subobject within the derived + * class. Only holds for non-virtual base classes, but see table + * `virtual_base_offsets` for offsets of virtual base class subobjects. + */ +direct_base_offsets( + unique int der_id: @derivation ref, + int offset: int ref +); + +/** + * Contains the byte offset of the virtual base class subobject for class + * `super` within a most-derived object of class `sub`. `super` can be either a + * direct or indirect base class. + */ +#keyset[sub, super] +virtual_base_offsets( + int sub: @usertype ref, + int super: @usertype ref, + int offset: int ref +); + +frienddecls( + unique int id: @frienddecl, + int type_id: @type ref, + int decl_id: @declaration ref, + int location: @location_default ref +); + +@declaredtype = @usertype ; + +@declaration = @function + | @declaredtype + | @variable + | @enumconstant + | @frienddecl; + +@member = @membervariable + | @function + | @declaredtype + | @enumconstant; + +@locatable = @diagnostic + | @declaration + | @ppd_include + | @ppd_define + | @macroinvocation + /*| @funcall*/ + | @xmllocatable + | @attribute + | @attribute_arg; + +@namedscope = @namespace | @usertype; + +@element = @locatable + | @file + | @folder + | @specifier + | @type + | @expr + | @namespace + | @initialiser + | @stmt + | @derivation + | @comment + | @preprocdirect + | @fun_decl + | @var_decl + | @type_decl + | @namespace_decl + | @using + | @namequalifier + | @specialnamequalifyingelement + | @static_assert + | @type_mention + | @lambdacapture; + +@exprparent = @element; + +comments( + unique int id: @comment, + string contents: string ref, + int location: @location_default ref +); + +commentbinding( + int id: @comment ref, + int element: @element ref +); + +exprconv( + int converted: @expr ref, + unique int conversion: @expr ref +); + +compgenerated(unique int id: @element ref); + +/** + * `destructor_call` destructs the `i`'th entity that should be + * destructed following `element`. Note that entities should be + * destructed in reverse construction order, so for a given `element` + * these should be called from highest to lowest `i`. + */ +#keyset[element, destructor_call] +#keyset[element, i] +synthetic_destructor_call( + int element: @element ref, + int i: int ref, + int destructor_call: @routineexpr ref +); + +namespaces( + unique int id: @namespace, + string name: string ref +); + +namespace_inline( + unique int id: @namespace ref +); + +namespacembrs( + int parentid: @namespace ref, + unique int memberid: @namespacembr ref +); + +@namespacembr = @declaration | @namespace; + +exprparents( + int expr_id: @expr ref, + int child_index: int ref, + int parent_id: @exprparent ref +); + +expr_isload(unique int expr_id: @expr ref); + +@cast = @c_style_cast + | @const_cast + | @dynamic_cast + | @reinterpret_cast + | @static_cast + ; + +/* +case @conversion.kind of + 0 = @simple_conversion // a numeric conversion, qualification conversion, or a reinterpret_cast +| 1 = @bool_conversion // conversion to 'bool' +| 2 = @base_class_conversion // a derived-to-base conversion +| 3 = @derived_class_conversion // a base-to-derived conversion +| 4 = @pm_base_class_conversion // a derived-to-base conversion of a pointer to member +| 5 = @pm_derived_class_conversion // a base-to-derived conversion of a pointer to member +| 6 = @glvalue_adjust // an adjustment of the type of a glvalue +| 7 = @prvalue_adjust // an adjustment of the type of a prvalue +; +*/ +/** + * Describes the semantics represented by a cast expression. This is largely + * independent of the source syntax of the cast, so it is separate from the + * regular expression kind. + */ +conversionkinds( + unique int expr_id: @cast ref, + int kind: int ref +); + +@conversion = @cast + | @array_to_pointer + | @parexpr + | @reference_to + | @ref_indirect + | @temp_init + ; + +/* +case @funbindexpr.kind of + 0 = @normal_call // a normal call +| 1 = @virtual_call // a virtual call +| 2 = @adl_call // a call whose target is only found by ADL +; +*/ +iscall( + unique int caller: @funbindexpr ref, + int kind: int ref +); + +numtemplatearguments( + unique int expr_id: @expr ref, + int num: int ref +); + +specialnamequalifyingelements( + unique int id: @specialnamequalifyingelement, + unique string name: string ref +); + +@namequalifiableelement = @expr | @namequalifier; +@namequalifyingelement = @namespace + | @specialnamequalifyingelement + | @usertype; + +namequalifiers( + unique int id: @namequalifier, + unique int qualifiableelement: @namequalifiableelement ref, + int qualifyingelement: @namequalifyingelement ref, + int location: @location_default ref +); + +varbind( + int expr: @varbindexpr ref, + int var: @accessible ref +); + +funbind( + int expr: @funbindexpr ref, + int fun: @function ref +); + +@any_new_expr = @new_expr + | @new_array_expr; + +@new_or_delete_expr = @any_new_expr + | @delete_expr + | @delete_array_expr; + +@prefix_crement_expr = @preincrexpr | @predecrexpr; + +@postfix_crement_expr = @postincrexpr | @postdecrexpr; + +@increment_expr = @preincrexpr | @postincrexpr; + +@decrement_expr = @predecrexpr | @postdecrexpr; + +@crement_expr = @increment_expr | @decrement_expr; + +@un_arith_op_expr = @arithnegexpr + | @unaryplusexpr + | @conjugation + | @realpartexpr + | @imagpartexpr + | @crement_expr + ; + +@un_bitwise_op_expr = @complementexpr; + +@un_log_op_expr = @notexpr; + +@un_op_expr = @address_of + | @indirect + | @un_arith_op_expr + | @un_bitwise_op_expr + | @builtinaddressof + | @vec_fill + | @un_log_op_expr + | @co_await + | @co_yield + ; + +@bin_log_op_expr = @andlogicalexpr | @orlogicalexpr; + +@cmp_op_expr = @eq_op_expr | @rel_op_expr; + +@eq_op_expr = @eqexpr | @neexpr; + +@rel_op_expr = @gtexpr + | @ltexpr + | @geexpr + | @leexpr + | @spaceshipexpr + ; + +@bin_bitwise_op_expr = @lshiftexpr + | @rshiftexpr + | @andexpr + | @orexpr + | @xorexpr + ; + +@p_arith_op_expr = @paddexpr + | @psubexpr + | @pdiffexpr + ; + +@bin_arith_op_expr = @addexpr + | @subexpr + | @mulexpr + | @divexpr + | @remexpr + | @jmulexpr + | @jdivexpr + | @fjaddexpr + | @jfaddexpr + | @fjsubexpr + | @jfsubexpr + | @minexpr + | @maxexpr + | @p_arith_op_expr + ; + +@bin_op_expr = @bin_arith_op_expr + | @bin_bitwise_op_expr + | @cmp_op_expr + | @bin_log_op_expr + ; + +@op_expr = @un_op_expr + | @bin_op_expr + | @assign_expr + | @conditionalexpr + ; + +@assign_arith_expr = @assignaddexpr + | @assignsubexpr + | @assignmulexpr + | @assigndivexpr + | @assignremexpr + ; + +@assign_bitwise_expr = @assignandexpr + | @assignorexpr + | @assignxorexpr + | @assignlshiftexpr + | @assignrshiftexpr + | @assignpaddexpr + | @assignpsubexpr + ; + +@assign_op_expr = @assign_arith_expr | @assign_bitwise_expr + +@assign_expr = @assignexpr | @assign_op_expr | @blockassignexpr + +/* + case @allocator.form of + 0 = plain + | 1 = alignment + ; +*/ + +/** + * The allocator function associated with a `new` or `new[]` expression. + * The `form` column specified whether the allocation call contains an alignment + * argument. + */ +expr_allocator( + unique int expr: @any_new_expr ref, + int func: @function ref, + int form: int ref +); + +/* + case @deallocator.form of + 0 = plain + | 1 = size + | 2 = alignment + | 3 = size_and_alignment + ; +*/ + +/** + * The deallocator function associated with a `delete`, `delete[]`, `new`, or + * `new[]` expression. For a `new` or `new[]` expression, the deallocator is the + * one used to free memory if the initialization throws an exception. + * The `form` column specifies whether the deallocation call contains a size + * argument, and alignment argument, or both. + */ +expr_deallocator( + unique int expr: @new_or_delete_expr ref, + int func: @function ref, + int form: int ref +); + +/** + * Holds if the `@conditionalexpr` is of the two operand form + * `guard ? : false`. + */ +expr_cond_two_operand( + unique int cond: @conditionalexpr ref +); + +/** + * The guard of `@conditionalexpr` `guard ? true : false` + */ +expr_cond_guard( + unique int cond: @conditionalexpr ref, + int guard: @expr ref +); + +/** + * The expression used when the guard of `@conditionalexpr` + * `guard ? true : false` holds. For the two operand form + * `guard ?: false` consider using `expr_cond_guard` instead. + */ +expr_cond_true( + unique int cond: @conditionalexpr ref, + int true: @expr ref +); + +/** + * The expression used when the guard of `@conditionalexpr` + * `guard ? true : false` does not hold. + */ +expr_cond_false( + unique int cond: @conditionalexpr ref, + int false: @expr ref +); + +/** A string representation of the value. */ +values( + unique int id: @value, + string str: string ref +); + +/** The actual text in the source code for the value, if any. */ +valuetext( + unique int id: @value ref, + string text: string ref +); + +valuebind( + int val: @value ref, + unique int expr: @expr ref +); + +fieldoffsets( + unique int id: @variable ref, + int byteoffset: int ref, + int bitoffset: int ref +); + +bitfield( + unique int id: @variable ref, + int bits: int ref, + int declared_bits: int ref +); + +/* TODO +memberprefix( + int member: @expr ref, + int prefix: @expr ref +); +*/ + +/* + kind(1) = mbrcallexpr + kind(2) = mbrptrcallexpr + kind(3) = mbrptrmbrcallexpr + kind(4) = ptrmbrptrmbrcallexpr + kind(5) = mbrreadexpr // x.y + kind(6) = mbrptrreadexpr // p->y + kind(7) = mbrptrmbrreadexpr // x.*pm + kind(8) = mbrptrmbrptrreadexpr // x->*pm + kind(9) = staticmbrreadexpr // static x.y + kind(10) = staticmbrptrreadexpr // static p->y +*/ +/* TODO +memberaccess( + int member: @expr ref, + int kind: int ref +); +*/ + +initialisers( + unique int init: @initialiser, + int var: @accessible ref, + unique int expr: @expr ref, + int location: @location_expr ref +); + +braced_initialisers( + int init: @initialiser ref +); + +/** + * An ancestor for the expression, for cases in which we cannot + * otherwise find the expression's parent. + */ +expr_ancestor( + int exp: @expr ref, + int ancestor: @element ref +); + +exprs( + unique int id: @expr, + int kind: int ref, + int location: @location_expr ref +); + +/* + case @value.category of + 1 = prval + | 2 = xval + | 3 = lval + ; +*/ +expr_types( + int id: @expr ref, + int typeid: @type ref, + int value_category: int ref +); + +case @expr.kind of + 1 = @errorexpr +| 2 = @address_of // & AddressOfExpr +| 3 = @reference_to // ReferenceToExpr (implicit?) +| 4 = @indirect // * PointerDereferenceExpr +| 5 = @ref_indirect // ReferenceDereferenceExpr (implicit?) +// ... +| 8 = @array_to_pointer // (???) +| 9 = @vacuous_destructor_call // VacuousDestructorCall +// ... +| 11 = @assume // Microsoft +| 12 = @parexpr +| 13 = @arithnegexpr +| 14 = @unaryplusexpr +| 15 = @complementexpr +| 16 = @notexpr +| 17 = @conjugation // GNU ~ operator +| 18 = @realpartexpr // GNU __real +| 19 = @imagpartexpr // GNU __imag +| 20 = @postincrexpr +| 21 = @postdecrexpr +| 22 = @preincrexpr +| 23 = @predecrexpr +| 24 = @conditionalexpr +| 25 = @addexpr +| 26 = @subexpr +| 27 = @mulexpr +| 28 = @divexpr +| 29 = @remexpr +| 30 = @jmulexpr // C99 mul imaginary +| 31 = @jdivexpr // C99 div imaginary +| 32 = @fjaddexpr // C99 add real + imaginary +| 33 = @jfaddexpr // C99 add imaginary + real +| 34 = @fjsubexpr // C99 sub real - imaginary +| 35 = @jfsubexpr // C99 sub imaginary - real +| 36 = @paddexpr // pointer add (pointer + int or int + pointer) +| 37 = @psubexpr // pointer sub (pointer - integer) +| 38 = @pdiffexpr // difference between two pointers +| 39 = @lshiftexpr +| 40 = @rshiftexpr +| 41 = @andexpr +| 42 = @orexpr +| 43 = @xorexpr +| 44 = @eqexpr +| 45 = @neexpr +| 46 = @gtexpr +| 47 = @ltexpr +| 48 = @geexpr +| 49 = @leexpr +| 50 = @minexpr // GNU minimum +| 51 = @maxexpr // GNU maximum +| 52 = @assignexpr +| 53 = @assignaddexpr +| 54 = @assignsubexpr +| 55 = @assignmulexpr +| 56 = @assigndivexpr +| 57 = @assignremexpr +| 58 = @assignlshiftexpr +| 59 = @assignrshiftexpr +| 60 = @assignandexpr +| 61 = @assignorexpr +| 62 = @assignxorexpr +| 63 = @assignpaddexpr // assign pointer add +| 64 = @assignpsubexpr // assign pointer sub +| 65 = @andlogicalexpr +| 66 = @orlogicalexpr +| 67 = @commaexpr +| 68 = @subscriptexpr // access to member of an array, e.g., a[5] +// ... 69 @objc_subscriptexpr deprecated +// ... 70 @cmdaccess deprecated +// ... +| 73 = @virtfunptrexpr +| 74 = @callexpr +// ... 75 @msgexpr_normal deprecated +// ... 76 @msgexpr_super deprecated +// ... 77 @atselectorexpr deprecated +// ... 78 @atprotocolexpr deprecated +| 79 = @vastartexpr +| 80 = @vaargexpr +| 81 = @vaendexpr +| 82 = @vacopyexpr +// ... 83 @atencodeexpr deprecated +| 84 = @varaccess +| 85 = @thisaccess +// ... 86 @objc_box_expr deprecated +| 87 = @new_expr +| 88 = @delete_expr +| 89 = @throw_expr +| 90 = @condition_decl // a variable declared in a condition, e.g., if(int x = y > 2) +| 91 = @braced_init_list +| 92 = @type_id +| 93 = @runtime_sizeof +| 94 = @runtime_alignof +| 95 = @sizeof_pack +| 96 = @expr_stmt // GNU extension +| 97 = @routineexpr +| 98 = @type_operand // used to access a type in certain contexts (haven't found any examples yet....) +| 99 = @offsetofexpr // offsetof ::= type and field +| 100 = @hasassignexpr // __has_assign ::= type +| 101 = @hascopyexpr // __has_copy ::= type +| 102 = @hasnothrowassign // __has_nothrow_assign ::= type +| 103 = @hasnothrowconstr // __has_nothrow_constructor ::= type +| 104 = @hasnothrowcopy // __has_nothrow_copy ::= type +| 105 = @hastrivialassign // __has_trivial_assign ::= type +| 106 = @hastrivialconstr // __has_trivial_constructor ::= type +| 107 = @hastrivialcopy // __has_trivial_copy ::= type +| 108 = @hasuserdestr // __has_user_destructor ::= type +| 109 = @hasvirtualdestr // __has_virtual_destructor ::= type +| 110 = @isabstractexpr // __is_abstract ::= type +| 111 = @isbaseofexpr // __is_base_of ::= type type +| 112 = @isclassexpr // __is_class ::= type +| 113 = @isconvtoexpr // __is_convertible_to ::= type type +| 114 = @isemptyexpr // __is_empty ::= type +| 115 = @isenumexpr // __is_enum ::= type +| 116 = @ispodexpr // __is_pod ::= type +| 117 = @ispolyexpr // __is_polymorphic ::= type +| 118 = @isunionexpr // __is_union ::= type +| 119 = @typescompexpr // GNU __builtin_types_compatible ::= type type +| 120 = @intaddrexpr // frontend internal builtin, used to implement offsetof +// ... +| 122 = @hastrivialdestructor // __has_trivial_destructor ::= type +| 123 = @literal +| 124 = @uuidof +| 127 = @aggregateliteral +| 128 = @delete_array_expr +| 129 = @new_array_expr +// ... 130 @objc_array_literal deprecated +// ... 131 @objc_dictionary_literal deprecated +| 132 = @foldexpr +// ... +| 200 = @ctordirectinit +| 201 = @ctorvirtualinit +| 202 = @ctorfieldinit +| 203 = @ctordelegatinginit +| 204 = @dtordirectdestruct +| 205 = @dtorvirtualdestruct +| 206 = @dtorfielddestruct +// ... +| 210 = @static_cast +| 211 = @reinterpret_cast +| 212 = @const_cast +| 213 = @dynamic_cast +| 214 = @c_style_cast +| 215 = @lambdaexpr +| 216 = @param_ref +| 217 = @noopexpr +// ... +| 294 = @istriviallyconstructibleexpr +| 295 = @isdestructibleexpr +| 296 = @isnothrowdestructibleexpr +| 297 = @istriviallydestructibleexpr +| 298 = @istriviallyassignableexpr +| 299 = @isnothrowassignableexpr +| 300 = @istrivialexpr +| 301 = @isstandardlayoutexpr +| 302 = @istriviallycopyableexpr +| 303 = @isliteraltypeexpr +| 304 = @hastrivialmoveconstructorexpr +| 305 = @hastrivialmoveassignexpr +| 306 = @hasnothrowmoveassignexpr +| 307 = @isconstructibleexpr +| 308 = @isnothrowconstructibleexpr +| 309 = @hasfinalizerexpr +| 310 = @isdelegateexpr +| 311 = @isinterfaceclassexpr +| 312 = @isrefarrayexpr +| 313 = @isrefclassexpr +| 314 = @issealedexpr +| 315 = @issimplevalueclassexpr +| 316 = @isvalueclassexpr +| 317 = @isfinalexpr +| 319 = @noexceptexpr +| 320 = @builtinshufflevector +| 321 = @builtinchooseexpr +| 322 = @builtinaddressof +| 323 = @vec_fill +| 324 = @builtinconvertvector +| 325 = @builtincomplex +| 326 = @spaceshipexpr +| 327 = @co_await +| 328 = @co_yield +| 329 = @temp_init +| 330 = @isassignable +| 331 = @isaggregate +| 332 = @hasuniqueobjectrepresentations +| 333 = @builtinbitcast +| 334 = @builtinshuffle +| 335 = @blockassignexpr +| 336 = @issame +| 337 = @isfunction +| 338 = @islayoutcompatible +| 339 = @ispointerinterconvertiblebaseof +| 340 = @isarray +| 341 = @arrayrank +| 342 = @arrayextent +| 343 = @isarithmetic +| 344 = @iscompletetype +| 345 = @iscompound +| 346 = @isconst +| 347 = @isfloatingpoint +| 348 = @isfundamental +| 349 = @isintegral +| 350 = @islvaluereference +| 351 = @ismemberfunctionpointer +| 352 = @ismemberobjectpointer +| 353 = @ismemberpointer +| 354 = @isobject +| 355 = @ispointer +| 356 = @isreference +| 357 = @isrvaluereference +| 358 = @isscalar +| 359 = @issigned +| 360 = @isunsigned +| 361 = @isvoid +| 362 = @isvolatile +; + +@var_args_expr = @vastartexpr + | @vaendexpr + | @vaargexpr + | @vacopyexpr + ; + +@builtin_op = @var_args_expr + | @noopexpr + | @offsetofexpr + | @intaddrexpr + | @hasassignexpr + | @hascopyexpr + | @hasnothrowassign + | @hasnothrowconstr + | @hasnothrowcopy + | @hastrivialassign + | @hastrivialconstr + | @hastrivialcopy + | @hastrivialdestructor + | @hasuserdestr + | @hasvirtualdestr + | @isabstractexpr + | @isbaseofexpr + | @isclassexpr + | @isconvtoexpr + | @isemptyexpr + | @isenumexpr + | @ispodexpr + | @ispolyexpr + | @isunionexpr + | @typescompexpr + | @builtinshufflevector + | @builtinconvertvector + | @builtinaddressof + | @istriviallyconstructibleexpr + | @isdestructibleexpr + | @isnothrowdestructibleexpr + | @istriviallydestructibleexpr + | @istriviallyassignableexpr + | @isnothrowassignableexpr + | @isstandardlayoutexpr + | @istriviallycopyableexpr + | @isliteraltypeexpr + | @hastrivialmoveconstructorexpr + | @hastrivialmoveassignexpr + | @hasnothrowmoveassignexpr + | @isconstructibleexpr + | @isnothrowconstructibleexpr + | @hasfinalizerexpr + | @isdelegateexpr + | @isinterfaceclassexpr + | @isrefarrayexpr + | @isrefclassexpr + | @issealedexpr + | @issimplevalueclassexpr + | @isvalueclassexpr + | @isfinalexpr + | @builtinchooseexpr + | @builtincomplex + | @isassignable + | @isaggregate + | @hasuniqueobjectrepresentations + | @builtinbitcast + | @builtinshuffle + | @issame + | @isfunction + | @islayoutcompatible + | @ispointerinterconvertiblebaseof + | @isarray + | @arrayrank + | @arrayextent + | @isarithmetic + | @iscompletetype + | @iscompound + | @isconst + | @isfloatingpoint + | @isfundamental + | @isintegral + | @islvaluereference + | @ismemberfunctionpointer + | @ismemberobjectpointer + | @ismemberpointer + | @isobject + | @ispointer + | @isreference + | @isrvaluereference + | @isscalar + | @issigned + | @isunsigned + | @isvoid + | @isvolatile + ; + +new_allocated_type( + unique int expr: @new_expr ref, + int type_id: @type ref +); + +new_array_allocated_type( + unique int expr: @new_array_expr ref, + int type_id: @type ref +); + +/** + * The field being initialized by an initializer expression within an aggregate + * initializer for a class/struct/union. Position is used to sort repeated initializers. + */ +#keyset[aggregate, position] +aggregate_field_init( + int aggregate: @aggregateliteral ref, + int initializer: @expr ref, + int field: @membervariable ref, + int position: int ref +); + +/** + * The index of the element being initialized by an initializer expression + * within an aggregate initializer for an array. Position is used to sort repeated initializers. + */ +#keyset[aggregate, position] +aggregate_array_init( + int aggregate: @aggregateliteral ref, + int initializer: @expr ref, + int element_index: int ref, + int position: int ref +); + +@ctorinit = @ctordirectinit + | @ctorvirtualinit + | @ctorfieldinit + | @ctordelegatinginit; +@dtordestruct = @dtordirectdestruct + | @dtorvirtualdestruct + | @dtorfielddestruct; + + +condition_decl_bind( + unique int expr: @condition_decl ref, + unique int decl: @declaration ref +); + +typeid_bind( + unique int expr: @type_id ref, + int type_id: @type ref +); + +uuidof_bind( + unique int expr: @uuidof ref, + int type_id: @type ref +); + +@runtime_sizeof_or_alignof = @runtime_sizeof | @runtime_alignof; + +sizeof_bind( + unique int expr: @runtime_sizeof_or_alignof ref, + int type_id: @type ref +); + +code_block( + unique int block: @literal ref, + unique int routine: @function ref +); + +lambdas( + unique int expr: @lambdaexpr ref, + string default_capture: string ref, + boolean has_explicit_return_type: boolean ref +); + +lambda_capture( + unique int id: @lambdacapture, + int lambda: @lambdaexpr ref, + int index: int ref, + int field: @membervariable ref, + boolean captured_by_reference: boolean ref, + boolean is_implicit: boolean ref, + int location: @location_default ref +); + +@funbindexpr = @routineexpr + | @new_expr + | @delete_expr + | @delete_array_expr + | @ctordirectinit + | @ctorvirtualinit + | @ctordelegatinginit + | @dtordirectdestruct + | @dtorvirtualdestruct; + +@varbindexpr = @varaccess | @ctorfieldinit | @dtorfielddestruct; +@addressable = @function | @variable ; +@accessible = @addressable | @enumconstant ; + +@access = @varaccess | @routineexpr ; + +fold( + int expr: @foldexpr ref, + string operator: string ref, + boolean is_left_fold: boolean ref +); + +stmts( + unique int id: @stmt, + int kind: int ref, + int location: @location_stmt ref +); + +case @stmt.kind of + 1 = @stmt_expr +| 2 = @stmt_if +| 3 = @stmt_while +| 4 = @stmt_goto +| 5 = @stmt_label +| 6 = @stmt_return +| 7 = @stmt_block +| 8 = @stmt_end_test_while // do { ... } while ( ... ) +| 9 = @stmt_for +| 10 = @stmt_switch_case +| 11 = @stmt_switch +| 13 = @stmt_asm // "asm" statement or the body of an asm function +| 15 = @stmt_try_block +| 16 = @stmt_microsoft_try // Microsoft +| 17 = @stmt_decl +| 18 = @stmt_set_vla_size // C99 +| 19 = @stmt_vla_decl // C99 +| 25 = @stmt_assigned_goto // GNU +| 26 = @stmt_empty +| 27 = @stmt_continue +| 28 = @stmt_break +| 29 = @stmt_range_based_for // C++11 +// ... 30 @stmt_at_autoreleasepool_block deprecated +// ... 31 @stmt_objc_for_in deprecated +// ... 32 @stmt_at_synchronized deprecated +| 33 = @stmt_handler +// ... 34 @stmt_finally_end deprecated +| 35 = @stmt_constexpr_if +| 37 = @stmt_co_return +; + +type_vla( + int type_id: @type ref, + int decl: @stmt_vla_decl ref +); + +variable_vla( + int var: @variable ref, + int decl: @stmt_vla_decl ref +); + +if_initialization( + unique int if_stmt: @stmt_if ref, + int init_id: @stmt ref +); + +if_then( + unique int if_stmt: @stmt_if ref, + int then_id: @stmt ref +); + +if_else( + unique int if_stmt: @stmt_if ref, + int else_id: @stmt ref +); + +constexpr_if_initialization( + unique int constexpr_if_stmt: @stmt_constexpr_if ref, + int init_id: @stmt ref +); + +constexpr_if_then( + unique int constexpr_if_stmt: @stmt_constexpr_if ref, + int then_id: @stmt ref +); + +constexpr_if_else( + unique int constexpr_if_stmt: @stmt_constexpr_if ref, + int else_id: @stmt ref +); + +while_body( + unique int while_stmt: @stmt_while ref, + int body_id: @stmt ref +); + +do_body( + unique int do_stmt: @stmt_end_test_while ref, + int body_id: @stmt ref +); + +switch_initialization( + unique int switch_stmt: @stmt_switch ref, + int init_id: @stmt ref +); + +#keyset[switch_stmt, index] +switch_case( + int switch_stmt: @stmt_switch ref, + int index: int ref, + int case_id: @stmt_switch_case ref +); + +switch_body( + unique int switch_stmt: @stmt_switch ref, + int body_id: @stmt ref +); + +for_initialization( + unique int for_stmt: @stmt_for ref, + int init_id: @stmt ref +); + +for_condition( + unique int for_stmt: @stmt_for ref, + int condition_id: @expr ref +); + +for_update( + unique int for_stmt: @stmt_for ref, + int update_id: @expr ref +); + +for_body( + unique int for_stmt: @stmt_for ref, + int body_id: @stmt ref +); + +@stmtparent = @stmt | @expr_stmt ; +stmtparents( + unique int id: @stmt ref, + int index: int ref, + int parent: @stmtparent ref +); + +ishandler(unique int block: @stmt_block ref); + +@cfgnode = @stmt | @expr | @function | @initialiser ; + +stmt_decl_bind( + int stmt: @stmt_decl ref, + int num: int ref, + int decl: @declaration ref +); + +stmt_decl_entry_bind( + int stmt: @stmt_decl ref, + int num: int ref, + int decl_entry: @element ref +); + +@functionorblock = @function | @stmt_block; + +blockscope( + unique int block: @stmt_block ref, + int enclosing: @functionorblock ref +); + +@jump = @stmt_goto | @stmt_break | @stmt_continue; + +@jumporlabel = @jump | @stmt_label | @literal; + +jumpinfo( + unique int id: @jumporlabel ref, + string str: string ref, + int target: @stmt ref +); + +preprocdirects( + unique int id: @preprocdirect, + int kind: int ref, + int location: @location_default ref +); +case @preprocdirect.kind of + 0 = @ppd_if +| 1 = @ppd_ifdef +| 2 = @ppd_ifndef +| 3 = @ppd_elif +| 4 = @ppd_else +| 5 = @ppd_endif +| 6 = @ppd_plain_include +| 7 = @ppd_define +| 8 = @ppd_undef +| 9 = @ppd_line +| 10 = @ppd_error +| 11 = @ppd_pragma +| 12 = @ppd_objc_import +| 13 = @ppd_include_next +| 18 = @ppd_warning +; + +@ppd_include = @ppd_plain_include | @ppd_objc_import | @ppd_include_next; + +@ppd_branch = @ppd_if | @ppd_ifdef | @ppd_ifndef | @ppd_elif; + +preprocpair( + int begin : @ppd_branch ref, + int elseelifend : @preprocdirect ref +); + +preproctrue(int branch : @ppd_branch ref); +preprocfalse(int branch : @ppd_branch ref); + +preproctext( + unique int id: @preprocdirect ref, + string head: string ref, + string body: string ref +); + +includes( + unique int id: @ppd_include ref, + int included: @file ref +); + +link_targets( + unique int id: @link_target, + int binary: @file ref +); + +link_parent( + int element : @element ref, + int link_target : @link_target ref +); + +/* XML Files */ + +xmlEncoding(unique int id: @file ref, string encoding: string ref); + +xmlDTDs( + unique int id: @xmldtd, + string root: string ref, + string publicId: string ref, + string systemId: string ref, + int fileid: @file ref +); + +xmlElements( + unique int id: @xmlelement, + string name: string ref, + int parentid: @xmlparent ref, + int idx: int ref, + int fileid: @file ref +); + +xmlAttrs( + unique int id: @xmlattribute, + int elementid: @xmlelement ref, + string name: string ref, + string value: string ref, + int idx: int ref, + int fileid: @file ref +); + +xmlNs( + int id: @xmlnamespace, + string prefixName: string ref, + string URI: string ref, + int fileid: @file ref +); + +xmlHasNs( + int elementId: @xmlnamespaceable ref, + int nsId: @xmlnamespace ref, + int fileid: @file ref +); + +xmlComments( + unique int id: @xmlcomment, + string text: string ref, + int parentid: @xmlparent ref, + int fileid: @file ref +); + +xmlChars( + unique int id: @xmlcharacters, + string text: string ref, + int parentid: @xmlparent ref, + int idx: int ref, + int isCDATA: int ref, + int fileid: @file ref +); + +@xmlparent = @file | @xmlelement; +@xmlnamespaceable = @xmlelement | @xmlattribute; + +xmllocations( + int xmlElement: @xmllocatable ref, + int location: @location_default ref +); + +@xmllocatable = @xmlcharacters + | @xmlelement + | @xmlcomment + | @xmlattribute + | @xmldtd + | @file + | @xmlnamespace; diff --git a/cpp/ql/lib/upgrades/19887dbd33327fb07d54251786e0cb2578539775/semmlecode.cpp.dbscheme b/cpp/ql/lib/upgrades/19887dbd33327fb07d54251786e0cb2578539775/semmlecode.cpp.dbscheme new file mode 100644 index 00000000000..d77c09d8bdc --- /dev/null +++ b/cpp/ql/lib/upgrades/19887dbd33327fb07d54251786e0cb2578539775/semmlecode.cpp.dbscheme @@ -0,0 +1,2212 @@ + +/** + * An invocation of the compiler. Note that more than one file may be + * compiled per invocation. For example, this command compiles three + * source files: + * + * gcc -c f1.c f2.c f3.c + * + * The `id` simply identifies the invocation, while `cwd` is the working + * directory from which the compiler was invoked. + */ +compilations( + /** + * An invocation of the compiler. Note that more than one file may + * be compiled per invocation. For example, this command compiles + * three source files: + * + * gcc -c f1.c f2.c f3.c + */ + unique int id : @compilation, + string cwd : string ref +); + +/** + * The arguments that were passed to the extractor for a compiler + * invocation. If `id` is for the compiler invocation + * + * gcc -c f1.c f2.c f3.c + * + * then typically there will be rows for + * + * num | arg + * --- | --- + * 0 | *path to extractor* + * 1 | `--mimic` + * 2 | `/usr/bin/gcc` + * 3 | `-c` + * 4 | f1.c + * 5 | f2.c + * 6 | f3.c + */ +#keyset[id, num] +compilation_args( + int id : @compilation ref, + int num : int ref, + string arg : string ref +); + +/** + * The source files that are compiled by a compiler invocation. + * If `id` is for the compiler invocation + * + * gcc -c f1.c f2.c f3.c + * + * then there will be rows for + * + * num | arg + * --- | --- + * 0 | f1.c + * 1 | f2.c + * 2 | f3.c + * + * Note that even if those files `#include` headers, those headers + * do not appear as rows. + */ +#keyset[id, num] +compilation_compiling_files( + int id : @compilation ref, + int num : int ref, + int file : @file ref +); + +/** + * The time taken by the extractor for a compiler invocation. + * + * For each file `num`, there will be rows for + * + * kind | seconds + * ---- | --- + * 1 | CPU seconds used by the extractor frontend + * 2 | Elapsed seconds during the extractor frontend + * 3 | CPU seconds used by the extractor backend + * 4 | Elapsed seconds during the extractor backend + */ +#keyset[id, num, kind] +compilation_time( + int id : @compilation ref, + int num : int ref, + /* kind: + 1 = frontend_cpu_seconds + 2 = frontend_elapsed_seconds + 3 = extractor_cpu_seconds + 4 = extractor_elapsed_seconds + */ + int kind : int ref, + float seconds : float ref +); + +/** + * An error or warning generated by the extractor. + * The diagnostic message `diagnostic` was generated during compiler + * invocation `compilation`, and is the `file_number_diagnostic_number`th + * message generated while extracting the `file_number`th file of that + * invocation. + */ +#keyset[compilation, file_number, file_number_diagnostic_number] +diagnostic_for( + int diagnostic : @diagnostic ref, + int compilation : @compilation ref, + int file_number : int ref, + int file_number_diagnostic_number : int ref +); + +/** + * If extraction was successful, then `cpu_seconds` and + * `elapsed_seconds` are the CPU time and elapsed time (respectively) + * that extraction took for compiler invocation `id`. + */ +compilation_finished( + unique int id : @compilation ref, + float cpu_seconds : float ref, + float elapsed_seconds : float ref +); + + +/** + * External data, loaded from CSV files during snapshot creation. See + * [Tutorial: Incorporating external data](https://help.semmle.com/wiki/display/SD/Tutorial%3A+Incorporating+external+data) + * for more information. + */ +externalData( + int id : @externalDataElement, + string path : string ref, + int column: int ref, + string value : string ref +); + +/** + * The source location of the snapshot. + */ +sourceLocationPrefix(string prefix : string ref); + +/** + * Information about packages that provide code used during compilation. + * The `id` is just a unique identifier. + * The `namespace` is typically the name of the package manager that + * provided the package (e.g. "dpkg" or "yum"). + * The `package_name` is the name of the package, and `version` is its + * version (as a string). + */ +external_packages( + unique int id: @external_package, + string namespace : string ref, + string package_name : string ref, + string version : string ref +); + +/** + * Holds if File `fileid` was provided by package `package`. + */ +header_to_external_package( + int fileid : @file ref, + int package : @external_package ref +); + +/* + * Version history + */ + +svnentries( + unique int id : @svnentry, + string revision : string ref, + string author : string ref, + date revisionDate : date ref, + int changeSize : int ref +) + +svnaffectedfiles( + int id : @svnentry ref, + int file : @file ref, + string action : string ref +) + +svnentrymsg( + unique int id : @svnentry ref, + string message : string ref +) + +svnchurn( + int commit : @svnentry ref, + int file : @file ref, + int addedLines : int ref, + int deletedLines : int ref +) + +/* + * C++ dbscheme + */ + +@location = @location_stmt | @location_expr | @location_default ; + +/** + * The location of an element that is not an expression or a statement. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `file`. + * For more information, see + * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/). + */ +locations_default( + /** The location of an element that is not an expression or a statement. */ + unique int id: @location_default, + int container: @container ref, + int startLine: int ref, + int startColumn: int ref, + int endLine: int ref, + int endColumn: int ref +); + +/** + * The location of a statement. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `file`. + * For more information, see + * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/). + */ +locations_stmt( + /** The location of a statement. */ + unique int id: @location_stmt, + int container: @container ref, + int startLine: int ref, + int startColumn: int ref, + int endLine: int ref, + int endColumn: int ref +); + +/** + * The location of an expression. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `file`. + * For more information, see + * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/). + */ +locations_expr( + /** The location of an expression. */ + unique int id: @location_expr, + int container: @container ref, + int startLine: int ref, + int startColumn: int ref, + int endLine: int ref, + int endColumn: int ref +); + +/** An element for which line-count information is available. */ +@sourceline = @file | @function | @variable | @enumconstant | @xmllocatable; + +numlines( + int element_id: @sourceline ref, + int num_lines: int ref, + int num_code: int ref, + int num_comment: int ref +); + +diagnostics( + unique int id: @diagnostic, + int severity: int ref, + string error_tag: string ref, + string error_message: string ref, + string full_error_message: string ref, + int location: @location_default ref +); + +files( + unique int id: @file, + string name: string ref +); + +folders( + unique int id: @folder, + string name: string ref +); + +@container = @folder | @file + +containerparent( + int parent: @container ref, + unique int child: @container ref +); + +fileannotations( + int id: @file ref, + int kind: int ref, + string name: string ref, + string value: string ref +); + +inmacroexpansion( + int id: @element ref, + int inv: @macroinvocation ref +); + +affectedbymacroexpansion( + int id: @element ref, + int inv: @macroinvocation ref +); + +case @macroinvocation.kind of + 1 = @macro_expansion +| 2 = @other_macro_reference +; + +macroinvocations( + unique int id: @macroinvocation, + int macro_id: @ppd_define ref, + int location: @location_default ref, + int kind: int ref +); + +macroparent( + unique int id: @macroinvocation ref, + int parent_id: @macroinvocation ref +); + +// a macroinvocation may be part of another location +// the way to find a constant expression that uses a macro +// is thus to find a constant expression that has a location +// to which a macro invocation is bound +macrolocationbind( + int id: @macroinvocation ref, + int location: @location ref +); + +#keyset[invocation, argument_index] +macro_argument_unexpanded( + int invocation: @macroinvocation ref, + int argument_index: int ref, + string text: string ref +); + +#keyset[invocation, argument_index] +macro_argument_expanded( + int invocation: @macroinvocation ref, + int argument_index: int ref, + string text: string ref +); + +/* +case @function.kind of + 1 = @normal_function +| 2 = @constructor +| 3 = @destructor +| 4 = @conversion_function +| 5 = @operator +| 6 = @builtin_function // GCC built-in functions, e.g. __builtin___memcpy_chk +; +*/ + +functions( + unique int id: @function, + string name: string ref, + int kind: int ref +); + +function_entry_point( + int id: @function ref, + unique int entry_point: @stmt ref +); + +function_return_type( + int id: @function ref, + int return_type: @type ref +); + +/** + * If `function` is a coroutine, then this gives the `std::experimental::resumable_traits` + * instance associated with it, and the variables representing the `handle` and `promise` + * for it. + */ +coroutine( + unique int function: @function ref, + int traits: @type ref, + int handle: @variable ref, + int promise: @variable ref +); + +/** The `new` function used for allocating the coroutine state, if any. */ +coroutine_new( + unique int function: @function ref, + int new: @function ref +); + +/** The `delete` function used for deallocating the coroutine state, if any. */ +coroutine_delete( + unique int function: @function ref, + int delete: @function ref +); + +purefunctions(unique int id: @function ref); + +function_deleted(unique int id: @function ref); + +function_defaulted(unique int id: @function ref); + +member_function_this_type( + unique int id: @function ref, + int this_type: @type ref +); + +#keyset[id, type_id] +fun_decls( + int id: @fun_decl, + int function: @function ref, + int type_id: @type ref, + string name: string ref, + int location: @location_default ref +); +fun_def(unique int id: @fun_decl ref); +fun_specialized(unique int id: @fun_decl ref); +fun_implicit(unique int id: @fun_decl ref); +fun_decl_specifiers( + int id: @fun_decl ref, + string name: string ref +) +#keyset[fun_decl, index] +fun_decl_throws( + int fun_decl: @fun_decl ref, + int index: int ref, + int type_id: @type ref +); +/* an empty throw specification is different from none */ +fun_decl_empty_throws(unique int fun_decl: @fun_decl ref); +fun_decl_noexcept( + int fun_decl: @fun_decl ref, + int constant: @expr ref +); +fun_decl_empty_noexcept(int fun_decl: @fun_decl ref); +fun_decl_typedef_type( + unique int fun_decl: @fun_decl ref, + int typedeftype_id: @usertype ref +); + +param_decl_bind( + unique int id: @var_decl ref, + int index: int ref, + int fun_decl: @fun_decl ref +); + +#keyset[id, type_id] +var_decls( + int id: @var_decl, + int variable: @variable ref, + int type_id: @type ref, + string name: string ref, + int location: @location_default ref +); +var_def(unique int id: @var_decl ref); +var_decl_specifiers( + int id: @var_decl ref, + string name: string ref +) +is_structured_binding(unique int id: @variable ref); + +type_decls( + unique int id: @type_decl, + int type_id: @type ref, + int location: @location_default ref +); +type_def(unique int id: @type_decl ref); +type_decl_top( + unique int type_decl: @type_decl ref +); + +namespace_decls( + unique int id: @namespace_decl, + int namespace_id: @namespace ref, + int location: @location_default ref, + int bodylocation: @location_default ref +); + +usings( + unique int id: @using, + int element_id: @element ref, + int location: @location_default ref +); + +/** The element which contains the `using` declaration. */ +using_container( + int parent: @element ref, + int child: @using ref +); + +static_asserts( + unique int id: @static_assert, + int condition : @expr ref, + string message : string ref, + int location: @location_default ref, + int enclosing : @element ref +); + +// each function has an ordered list of parameters +#keyset[id, type_id] +#keyset[function, index, type_id] +params( + int id: @parameter, + int function: @functionorblock ref, + int index: int ref, + int type_id: @type ref +); + +overrides( + int new: @function ref, + int old: @function ref +); + +#keyset[id, type_id] +membervariables( + int id: @membervariable, + int type_id: @type ref, + string name: string ref +); + +#keyset[id, type_id] +globalvariables( + int id: @globalvariable, + int type_id: @type ref, + string name: string ref +); + +#keyset[id, type_id] +localvariables( + int id: @localvariable, + int type_id: @type ref, + string name: string ref +); + +autoderivation( + unique int var: @variable ref, + int derivation_type: @type ref +); + +orphaned_variables( + int var: @localvariable ref, + int function: @function ref +) + +enumconstants( + unique int id: @enumconstant, + int parent: @usertype ref, + int index: int ref, + int type_id: @type ref, + string name: string ref, + int location: @location_default ref +); + +@variable = @localscopevariable | @globalvariable | @membervariable; + +@localscopevariable = @localvariable | @parameter; + +/** + * Built-in types are the fundamental types, e.g., integral, floating, and void. + */ +case @builtintype.kind of + 1 = @errortype +| 2 = @unknowntype +| 3 = @void +| 4 = @boolean +| 5 = @char +| 6 = @unsigned_char +| 7 = @signed_char +| 8 = @short +| 9 = @unsigned_short +| 10 = @signed_short +| 11 = @int +| 12 = @unsigned_int +| 13 = @signed_int +| 14 = @long +| 15 = @unsigned_long +| 16 = @signed_long +| 17 = @long_long +| 18 = @unsigned_long_long +| 19 = @signed_long_long +// ... 20 Microsoft-specific __int8 +// ... 21 Microsoft-specific __int16 +// ... 22 Microsoft-specific __int32 +// ... 23 Microsoft-specific __int64 +| 24 = @float +| 25 = @double +| 26 = @long_double +| 27 = @complex_float // C99-specific _Complex float +| 28 = @complex_double // C99-specific _Complex double +| 29 = @complex_long_double // C99-specific _Complex long double +| 30 = @imaginary_float // C99-specific _Imaginary float +| 31 = @imaginary_double // C99-specific _Imaginary double +| 32 = @imaginary_long_double // C99-specific _Imaginary long double +| 33 = @wchar_t // Microsoft-specific +| 34 = @decltype_nullptr // C++11 +| 35 = @int128 // __int128 +| 36 = @unsigned_int128 // unsigned __int128 +| 37 = @signed_int128 // signed __int128 +| 38 = @float128 // __float128 +| 39 = @complex_float128 // _Complex __float128 +| 40 = @decimal32 // _Decimal32 +| 41 = @decimal64 // _Decimal64 +| 42 = @decimal128 // _Decimal128 +| 43 = @char16_t +| 44 = @char32_t +| 45 = @std_float32 // _Float32 +| 46 = @float32x // _Float32x +| 47 = @std_float64 // _Float64 +| 48 = @float64x // _Float64x +| 49 = @std_float128 // _Float128 +// ... 50 _Float128x +| 51 = @char8_t +| 52 = @float16 // _Float16 +| 53 = @complex_float16 // _Complex _Float16 +; + +builtintypes( + unique int id: @builtintype, + string name: string ref, + int kind: int ref, + int size: int ref, + int sign: int ref, + int alignment: int ref +); + +/** + * Derived types are types that are directly derived from existing types and + * point to, refer to, transform type data to return a new type. + */ +case @derivedtype.kind of + 1 = @pointer +| 2 = @reference +| 3 = @type_with_specifiers +| 4 = @array +| 5 = @gnu_vector +| 6 = @routineptr +| 7 = @routinereference +| 8 = @rvalue_reference // C++11 +// ... 9 type_conforming_to_protocols deprecated +| 10 = @block +; + +derivedtypes( + unique int id: @derivedtype, + string name: string ref, + int kind: int ref, + int type_id: @type ref +); + +pointerishsize(unique int id: @derivedtype ref, + int size: int ref, + int alignment: int ref); + +arraysizes( + unique int id: @derivedtype ref, + int num_elements: int ref, + int bytesize: int ref, + int alignment: int ref +); + +typedefbase( + unique int id: @usertype ref, + int type_id: @type ref +); + +/** + * An instance of the C++11 `decltype` operator. For example: + * ``` + * int a; + * decltype(1+a) b; + * ``` + * Here `expr` is `1+a`. + * + * Sometimes an additional pair of parentheses around the expression + * would change the semantics of this decltype, e.g. + * ``` + * struct A { double x; }; + * const A* a = new A(); + * decltype( a->x ); // type is double + * decltype((a->x)); // type is const double& + * ``` + * (Please consult the C++11 standard for more details). + * `parentheses_would_change_meaning` is `true` iff that is the case. + */ +#keyset[id, expr] +decltypes( + int id: @decltype, + int expr: @expr ref, + int base_type: @type ref, + boolean parentheses_would_change_meaning: boolean ref +); + +/* +case @usertype.kind of + 1 = @struct +| 2 = @class +| 3 = @union +| 4 = @enum +| 5 = @typedef // classic C: typedef typedef type name +| 6 = @template +| 7 = @template_parameter +| 8 = @template_template_parameter +| 9 = @proxy_class // a proxy class associated with a template parameter +// ... 10 objc_class deprecated +// ... 11 objc_protocol deprecated +// ... 12 objc_category deprecated +| 13 = @scoped_enum +| 14 = @using_alias // a using name = type style typedef +; +*/ + +usertypes( + unique int id: @usertype, + string name: string ref, + int kind: int ref +); + +usertypesize( + unique int id: @usertype ref, + int size: int ref, + int alignment: int ref +); + +usertype_final(unique int id: @usertype ref); + +usertype_uuid( + unique int id: @usertype ref, + string uuid: string ref +); + +mangled_name( + unique int id: @declaration ref, + int mangled_name : @mangledname +); + +is_pod_class(unique int id: @usertype ref); +is_standard_layout_class(unique int id: @usertype ref); + +is_complete(unique int id: @usertype ref); + +is_class_template(unique int id: @usertype ref); +class_instantiation( + int to: @usertype ref, + int from: @usertype ref +); +class_template_argument( + int type_id: @usertype ref, + int index: int ref, + int arg_type: @type ref +); +class_template_argument_value( + int type_id: @usertype ref, + int index: int ref, + int arg_value: @expr ref +); + +is_proxy_class_for( + unique int id: @usertype ref, + unique int templ_param_id: @usertype ref +); + +type_mentions( + unique int id: @type_mention, + int type_id: @type ref, + int location: @location ref, + // a_symbol_reference_kind from the frontend. + int kind: int ref +); + +is_function_template(unique int id: @function ref); +function_instantiation( + unique int to: @function ref, + int from: @function ref +); +function_template_argument( + int function_id: @function ref, + int index: int ref, + int arg_type: @type ref +); +function_template_argument_value( + int function_id: @function ref, + int index: int ref, + int arg_value: @expr ref +); + +is_variable_template(unique int id: @variable ref); +variable_instantiation( + unique int to: @variable ref, + int from: @variable ref +); +variable_template_argument( + int variable_id: @variable ref, + int index: int ref, + int arg_type: @type ref +); +variable_template_argument_value( + int variable_id: @variable ref, + int index: int ref, + int arg_value: @expr ref +); + +/* + Fixed point types + precision(1) = short, precision(2) = default, precision(3) = long + is_unsigned(1) = unsigned is_unsigned(2) = signed + is_fract_type(1) = declared with _Fract + saturating(1) = declared with _Sat +*/ +/* TODO +fixedpointtypes( + unique int id: @fixedpointtype, + int precision: int ref, + int is_unsigned: int ref, + int is_fract_type: int ref, + int saturating: int ref); +*/ + +routinetypes( + unique int id: @routinetype, + int return_type: @type ref +); + +routinetypeargs( + int routine: @routinetype ref, + int index: int ref, + int type_id: @type ref +); + +ptrtomembers( + unique int id: @ptrtomember, + int type_id: @type ref, + int class_id: @type ref +); + +/* + specifiers for types, functions, and variables + + "public", + "protected", + "private", + + "const", + "volatile", + "static", + + "pure", + "virtual", + "sealed", // Microsoft + "__interface", // Microsoft + "inline", + "explicit", + + "near", // near far extension + "far", // near far extension + "__ptr32", // Microsoft + "__ptr64", // Microsoft + "__sptr", // Microsoft + "__uptr", // Microsoft + "dllimport", // Microsoft + "dllexport", // Microsoft + "thread", // Microsoft + "naked", // Microsoft + "microsoft_inline", // Microsoft + "forceinline", // Microsoft + "selectany", // Microsoft + "nothrow", // Microsoft + "novtable", // Microsoft + "noreturn", // Microsoft + "noinline", // Microsoft + "noalias", // Microsoft + "restrict", // Microsoft +*/ + +specifiers( + unique int id: @specifier, + unique string str: string ref +); + +typespecifiers( + int type_id: @type ref, + int spec_id: @specifier ref +); + +funspecifiers( + int func_id: @function ref, + int spec_id: @specifier ref +); + +varspecifiers( + int var_id: @accessible ref, + int spec_id: @specifier ref +); + +attributes( + unique int id: @attribute, + int kind: int ref, + string name: string ref, + string name_space: string ref, + int location: @location_default ref +); + +case @attribute.kind of + 0 = @gnuattribute +| 1 = @stdattribute +| 2 = @declspec +| 3 = @msattribute +| 4 = @alignas +// ... 5 @objc_propertyattribute deprecated +; + +attribute_args( + unique int id: @attribute_arg, + int kind: int ref, + int attribute: @attribute ref, + int index: int ref, + int location: @location_default ref +); + +case @attribute_arg.kind of + 0 = @attribute_arg_empty +| 1 = @attribute_arg_token +| 2 = @attribute_arg_constant +| 3 = @attribute_arg_type +| 4 = @attribute_arg_constant_expr +; + +attribute_arg_value( + unique int arg: @attribute_arg ref, + string value: string ref +); +attribute_arg_type( + unique int arg: @attribute_arg ref, + int type_id: @type ref +); +attribute_arg_constant( + unique int arg: @attribute_arg ref, + int constant: @expr ref +) +attribute_arg_name( + unique int arg: @attribute_arg ref, + string name: string ref +); + +typeattributes( + int type_id: @type ref, + int spec_id: @attribute ref +); + +funcattributes( + int func_id: @function ref, + int spec_id: @attribute ref +); + +varattributes( + int var_id: @accessible ref, + int spec_id: @attribute ref +); + +stmtattributes( + int stmt_id: @stmt ref, + int spec_id: @attribute ref +); + +@type = @builtintype + | @derivedtype + | @usertype + /* TODO | @fixedpointtype */ + | @routinetype + | @ptrtomember + | @decltype; + +unspecifiedtype( + unique int type_id: @type ref, + int unspecified_type_id: @type ref +); + +member( + int parent: @type ref, + int index: int ref, + int child: @member ref +); + +@enclosingfunction_child = @usertype | @variable | @namespace + +enclosingfunction( + unique int child: @enclosingfunction_child ref, + int parent: @function ref +); + +derivations( + unique int derivation: @derivation, + int sub: @type ref, + int index: int ref, + int super: @type ref, + int location: @location_default ref +); + +derspecifiers( + int der_id: @derivation ref, + int spec_id: @specifier ref +); + +/** + * Contains the byte offset of the base class subobject within the derived + * class. Only holds for non-virtual base classes, but see table + * `virtual_base_offsets` for offsets of virtual base class subobjects. + */ +direct_base_offsets( + unique int der_id: @derivation ref, + int offset: int ref +); + +/** + * Contains the byte offset of the virtual base class subobject for class + * `super` within a most-derived object of class `sub`. `super` can be either a + * direct or indirect base class. + */ +#keyset[sub, super] +virtual_base_offsets( + int sub: @usertype ref, + int super: @usertype ref, + int offset: int ref +); + +frienddecls( + unique int id: @frienddecl, + int type_id: @type ref, + int decl_id: @declaration ref, + int location: @location_default ref +); + +@declaredtype = @usertype ; + +@declaration = @function + | @declaredtype + | @variable + | @enumconstant + | @frienddecl; + +@member = @membervariable + | @function + | @declaredtype + | @enumconstant; + +@locatable = @diagnostic + | @declaration + | @ppd_include + | @ppd_define + | @macroinvocation + /*| @funcall*/ + | @xmllocatable + | @attribute + | @attribute_arg; + +@namedscope = @namespace | @usertype; + +@element = @locatable + | @file + | @folder + | @specifier + | @type + | @expr + | @namespace + | @initialiser + | @stmt + | @derivation + | @comment + | @preprocdirect + | @fun_decl + | @var_decl + | @type_decl + | @namespace_decl + | @using + | @namequalifier + | @specialnamequalifyingelement + | @static_assert + | @type_mention + | @lambdacapture; + +@exprparent = @element; + +comments( + unique int id: @comment, + string contents: string ref, + int location: @location_default ref +); + +commentbinding( + int id: @comment ref, + int element: @element ref +); + +exprconv( + int converted: @expr ref, + unique int conversion: @expr ref +); + +compgenerated(unique int id: @element ref); + +/** + * `destructor_call` destructs the `i`'th entity that should be + * destructed following `element`. Note that entities should be + * destructed in reverse construction order, so for a given `element` + * these should be called from highest to lowest `i`. + */ +#keyset[element, destructor_call] +#keyset[element, i] +synthetic_destructor_call( + int element: @element ref, + int i: int ref, + int destructor_call: @routineexpr ref +); + +namespaces( + unique int id: @namespace, + string name: string ref +); + +namespace_inline( + unique int id: @namespace ref +); + +namespacembrs( + int parentid: @namespace ref, + unique int memberid: @namespacembr ref +); + +@namespacembr = @declaration | @namespace; + +exprparents( + int expr_id: @expr ref, + int child_index: int ref, + int parent_id: @exprparent ref +); + +expr_isload(unique int expr_id: @expr ref); + +@cast = @c_style_cast + | @const_cast + | @dynamic_cast + | @reinterpret_cast + | @static_cast + ; + +/* +case @conversion.kind of + 0 = @simple_conversion // a numeric conversion, qualification conversion, or a reinterpret_cast +| 1 = @bool_conversion // conversion to 'bool' +| 2 = @base_class_conversion // a derived-to-base conversion +| 3 = @derived_class_conversion // a base-to-derived conversion +| 4 = @pm_base_class_conversion // a derived-to-base conversion of a pointer to member +| 5 = @pm_derived_class_conversion // a base-to-derived conversion of a pointer to member +| 6 = @glvalue_adjust // an adjustment of the type of a glvalue +| 7 = @prvalue_adjust // an adjustment of the type of a prvalue +; +*/ +/** + * Describes the semantics represented by a cast expression. This is largely + * independent of the source syntax of the cast, so it is separate from the + * regular expression kind. + */ +conversionkinds( + unique int expr_id: @cast ref, + int kind: int ref +); + +@conversion = @cast + | @array_to_pointer + | @parexpr + | @reference_to + | @ref_indirect + | @temp_init + ; + +/* +case @funbindexpr.kind of + 0 = @normal_call // a normal call +| 1 = @virtual_call // a virtual call +| 2 = @adl_call // a call whose target is only found by ADL +; +*/ +iscall( + unique int caller: @funbindexpr ref, + int kind: int ref +); + +numtemplatearguments( + unique int expr_id: @expr ref, + int num: int ref +); + +specialnamequalifyingelements( + unique int id: @specialnamequalifyingelement, + unique string name: string ref +); + +@namequalifiableelement = @expr | @namequalifier; +@namequalifyingelement = @namespace + | @specialnamequalifyingelement + | @usertype; + +namequalifiers( + unique int id: @namequalifier, + unique int qualifiableelement: @namequalifiableelement ref, + int qualifyingelement: @namequalifyingelement ref, + int location: @location_default ref +); + +varbind( + int expr: @varbindexpr ref, + int var: @accessible ref +); + +funbind( + int expr: @funbindexpr ref, + int fun: @function ref +); + +@any_new_expr = @new_expr + | @new_array_expr; + +@new_or_delete_expr = @any_new_expr + | @delete_expr + | @delete_array_expr; + +@prefix_crement_expr = @preincrexpr | @predecrexpr; + +@postfix_crement_expr = @postincrexpr | @postdecrexpr; + +@increment_expr = @preincrexpr | @postincrexpr; + +@decrement_expr = @predecrexpr | @postdecrexpr; + +@crement_expr = @increment_expr | @decrement_expr; + +@un_arith_op_expr = @arithnegexpr + | @unaryplusexpr + | @conjugation + | @realpartexpr + | @imagpartexpr + | @crement_expr + ; + +@un_bitwise_op_expr = @complementexpr; + +@un_log_op_expr = @notexpr; + +@un_op_expr = @address_of + | @indirect + | @un_arith_op_expr + | @un_bitwise_op_expr + | @builtinaddressof + | @vec_fill + | @un_log_op_expr + | @co_await + | @co_yield + ; + +@bin_log_op_expr = @andlogicalexpr | @orlogicalexpr; + +@cmp_op_expr = @eq_op_expr | @rel_op_expr; + +@eq_op_expr = @eqexpr | @neexpr; + +@rel_op_expr = @gtexpr + | @ltexpr + | @geexpr + | @leexpr + | @spaceshipexpr + ; + +@bin_bitwise_op_expr = @lshiftexpr + | @rshiftexpr + | @andexpr + | @orexpr + | @xorexpr + ; + +@p_arith_op_expr = @paddexpr + | @psubexpr + | @pdiffexpr + ; + +@bin_arith_op_expr = @addexpr + | @subexpr + | @mulexpr + | @divexpr + | @remexpr + | @jmulexpr + | @jdivexpr + | @fjaddexpr + | @jfaddexpr + | @fjsubexpr + | @jfsubexpr + | @minexpr + | @maxexpr + | @p_arith_op_expr + ; + +@bin_op_expr = @bin_arith_op_expr + | @bin_bitwise_op_expr + | @cmp_op_expr + | @bin_log_op_expr + ; + +@op_expr = @un_op_expr + | @bin_op_expr + | @assign_expr + | @conditionalexpr + ; + +@assign_arith_expr = @assignaddexpr + | @assignsubexpr + | @assignmulexpr + | @assigndivexpr + | @assignremexpr + ; + +@assign_bitwise_expr = @assignandexpr + | @assignorexpr + | @assignxorexpr + | @assignlshiftexpr + | @assignrshiftexpr + | @assignpaddexpr + | @assignpsubexpr + ; + +@assign_op_expr = @assign_arith_expr | @assign_bitwise_expr + +@assign_expr = @assignexpr | @assign_op_expr | @blockassignexpr + +/* + case @allocator.form of + 0 = plain + | 1 = alignment + ; +*/ + +/** + * The allocator function associated with a `new` or `new[]` expression. + * The `form` column specified whether the allocation call contains an alignment + * argument. + */ +expr_allocator( + unique int expr: @any_new_expr ref, + int func: @function ref, + int form: int ref +); + +/* + case @deallocator.form of + 0 = plain + | 1 = size + | 2 = alignment + | 3 = size_and_alignment + ; +*/ + +/** + * The deallocator function associated with a `delete`, `delete[]`, `new`, or + * `new[]` expression. For a `new` or `new[]` expression, the deallocator is the + * one used to free memory if the initialization throws an exception. + * The `form` column specifies whether the deallocation call contains a size + * argument, and alignment argument, or both. + */ +expr_deallocator( + unique int expr: @new_or_delete_expr ref, + int func: @function ref, + int form: int ref +); + +/** + * Holds if the `@conditionalexpr` is of the two operand form + * `guard ? : false`. + */ +expr_cond_two_operand( + unique int cond: @conditionalexpr ref +); + +/** + * The guard of `@conditionalexpr` `guard ? true : false` + */ +expr_cond_guard( + unique int cond: @conditionalexpr ref, + int guard: @expr ref +); + +/** + * The expression used when the guard of `@conditionalexpr` + * `guard ? true : false` holds. For the two operand form + * `guard ?: false` consider using `expr_cond_guard` instead. + */ +expr_cond_true( + unique int cond: @conditionalexpr ref, + int true: @expr ref +); + +/** + * The expression used when the guard of `@conditionalexpr` + * `guard ? true : false` does not hold. + */ +expr_cond_false( + unique int cond: @conditionalexpr ref, + int false: @expr ref +); + +/** A string representation of the value. */ +values( + unique int id: @value, + string str: string ref +); + +/** The actual text in the source code for the value, if any. */ +valuetext( + unique int id: @value ref, + string text: string ref +); + +valuebind( + int val: @value ref, + unique int expr: @expr ref +); + +fieldoffsets( + unique int id: @variable ref, + int byteoffset: int ref, + int bitoffset: int ref +); + +bitfield( + unique int id: @variable ref, + int bits: int ref, + int declared_bits: int ref +); + +/* TODO +memberprefix( + int member: @expr ref, + int prefix: @expr ref +); +*/ + +/* + kind(1) = mbrcallexpr + kind(2) = mbrptrcallexpr + kind(3) = mbrptrmbrcallexpr + kind(4) = ptrmbrptrmbrcallexpr + kind(5) = mbrreadexpr // x.y + kind(6) = mbrptrreadexpr // p->y + kind(7) = mbrptrmbrreadexpr // x.*pm + kind(8) = mbrptrmbrptrreadexpr // x->*pm + kind(9) = staticmbrreadexpr // static x.y + kind(10) = staticmbrptrreadexpr // static p->y +*/ +/* TODO +memberaccess( + int member: @expr ref, + int kind: int ref +); +*/ + +initialisers( + unique int init: @initialiser, + int var: @accessible ref, + unique int expr: @expr ref, + int location: @location_expr ref +); + +braced_initialisers( + int init: @initialiser ref +); + +/** + * An ancestor for the expression, for cases in which we cannot + * otherwise find the expression's parent. + */ +expr_ancestor( + int exp: @expr ref, + int ancestor: @element ref +); + +exprs( + unique int id: @expr, + int kind: int ref, + int location: @location_expr ref +); + +/* + case @value.category of + 1 = prval + | 2 = xval + | 3 = lval + ; +*/ +expr_types( + int id: @expr ref, + int typeid: @type ref, + int value_category: int ref +); + +case @expr.kind of + 1 = @errorexpr +| 2 = @address_of // & AddressOfExpr +| 3 = @reference_to // ReferenceToExpr (implicit?) +| 4 = @indirect // * PointerDereferenceExpr +| 5 = @ref_indirect // ReferenceDereferenceExpr (implicit?) +// ... +| 8 = @array_to_pointer // (???) +| 9 = @vacuous_destructor_call // VacuousDestructorCall +// ... +| 11 = @assume // Microsoft +| 12 = @parexpr +| 13 = @arithnegexpr +| 14 = @unaryplusexpr +| 15 = @complementexpr +| 16 = @notexpr +| 17 = @conjugation // GNU ~ operator +| 18 = @realpartexpr // GNU __real +| 19 = @imagpartexpr // GNU __imag +| 20 = @postincrexpr +| 21 = @postdecrexpr +| 22 = @preincrexpr +| 23 = @predecrexpr +| 24 = @conditionalexpr +| 25 = @addexpr +| 26 = @subexpr +| 27 = @mulexpr +| 28 = @divexpr +| 29 = @remexpr +| 30 = @jmulexpr // C99 mul imaginary +| 31 = @jdivexpr // C99 div imaginary +| 32 = @fjaddexpr // C99 add real + imaginary +| 33 = @jfaddexpr // C99 add imaginary + real +| 34 = @fjsubexpr // C99 sub real - imaginary +| 35 = @jfsubexpr // C99 sub imaginary - real +| 36 = @paddexpr // pointer add (pointer + int or int + pointer) +| 37 = @psubexpr // pointer sub (pointer - integer) +| 38 = @pdiffexpr // difference between two pointers +| 39 = @lshiftexpr +| 40 = @rshiftexpr +| 41 = @andexpr +| 42 = @orexpr +| 43 = @xorexpr +| 44 = @eqexpr +| 45 = @neexpr +| 46 = @gtexpr +| 47 = @ltexpr +| 48 = @geexpr +| 49 = @leexpr +| 50 = @minexpr // GNU minimum +| 51 = @maxexpr // GNU maximum +| 52 = @assignexpr +| 53 = @assignaddexpr +| 54 = @assignsubexpr +| 55 = @assignmulexpr +| 56 = @assigndivexpr +| 57 = @assignremexpr +| 58 = @assignlshiftexpr +| 59 = @assignrshiftexpr +| 60 = @assignandexpr +| 61 = @assignorexpr +| 62 = @assignxorexpr +| 63 = @assignpaddexpr // assign pointer add +| 64 = @assignpsubexpr // assign pointer sub +| 65 = @andlogicalexpr +| 66 = @orlogicalexpr +| 67 = @commaexpr +| 68 = @subscriptexpr // access to member of an array, e.g., a[5] +// ... 69 @objc_subscriptexpr deprecated +// ... 70 @cmdaccess deprecated +// ... +| 73 = @virtfunptrexpr +| 74 = @callexpr +// ... 75 @msgexpr_normal deprecated +// ... 76 @msgexpr_super deprecated +// ... 77 @atselectorexpr deprecated +// ... 78 @atprotocolexpr deprecated +| 79 = @vastartexpr +| 80 = @vaargexpr +| 81 = @vaendexpr +| 82 = @vacopyexpr +// ... 83 @atencodeexpr deprecated +| 84 = @varaccess +| 85 = @thisaccess +// ... 86 @objc_box_expr deprecated +| 87 = @new_expr +| 88 = @delete_expr +| 89 = @throw_expr +| 90 = @condition_decl // a variable declared in a condition, e.g., if(int x = y > 2) +| 91 = @braced_init_list +| 92 = @type_id +| 93 = @runtime_sizeof +| 94 = @runtime_alignof +| 95 = @sizeof_pack +| 96 = @expr_stmt // GNU extension +| 97 = @routineexpr +| 98 = @type_operand // used to access a type in certain contexts (haven't found any examples yet....) +| 99 = @offsetofexpr // offsetof ::= type and field +| 100 = @hasassignexpr // __has_assign ::= type +| 101 = @hascopyexpr // __has_copy ::= type +| 102 = @hasnothrowassign // __has_nothrow_assign ::= type +| 103 = @hasnothrowconstr // __has_nothrow_constructor ::= type +| 104 = @hasnothrowcopy // __has_nothrow_copy ::= type +| 105 = @hastrivialassign // __has_trivial_assign ::= type +| 106 = @hastrivialconstr // __has_trivial_constructor ::= type +| 107 = @hastrivialcopy // __has_trivial_copy ::= type +| 108 = @hasuserdestr // __has_user_destructor ::= type +| 109 = @hasvirtualdestr // __has_virtual_destructor ::= type +| 110 = @isabstractexpr // __is_abstract ::= type +| 111 = @isbaseofexpr // __is_base_of ::= type type +| 112 = @isclassexpr // __is_class ::= type +| 113 = @isconvtoexpr // __is_convertible_to ::= type type +| 114 = @isemptyexpr // __is_empty ::= type +| 115 = @isenumexpr // __is_enum ::= type +| 116 = @ispodexpr // __is_pod ::= type +| 117 = @ispolyexpr // __is_polymorphic ::= type +| 118 = @isunionexpr // __is_union ::= type +| 119 = @typescompexpr // GNU __builtin_types_compatible ::= type type +| 120 = @intaddrexpr // frontend internal builtin, used to implement offsetof +// ... +| 122 = @hastrivialdestructor // __has_trivial_destructor ::= type +| 123 = @literal +| 124 = @uuidof +| 127 = @aggregateliteral +| 128 = @delete_array_expr +| 129 = @new_array_expr +// ... 130 @objc_array_literal deprecated +// ... 131 @objc_dictionary_literal deprecated +| 132 = @foldexpr +// ... +| 200 = @ctordirectinit +| 201 = @ctorvirtualinit +| 202 = @ctorfieldinit +| 203 = @ctordelegatinginit +| 204 = @dtordirectdestruct +| 205 = @dtorvirtualdestruct +| 206 = @dtorfielddestruct +// ... +| 210 = @static_cast +| 211 = @reinterpret_cast +| 212 = @const_cast +| 213 = @dynamic_cast +| 214 = @c_style_cast +| 215 = @lambdaexpr +| 216 = @param_ref +| 217 = @noopexpr +// ... +| 294 = @istriviallyconstructibleexpr +| 295 = @isdestructibleexpr +| 296 = @isnothrowdestructibleexpr +| 297 = @istriviallydestructibleexpr +| 298 = @istriviallyassignableexpr +| 299 = @isnothrowassignableexpr +| 300 = @istrivialexpr +| 301 = @isstandardlayoutexpr +| 302 = @istriviallycopyableexpr +| 303 = @isliteraltypeexpr +| 304 = @hastrivialmoveconstructorexpr +| 305 = @hastrivialmoveassignexpr +| 306 = @hasnothrowmoveassignexpr +| 307 = @isconstructibleexpr +| 308 = @isnothrowconstructibleexpr +| 309 = @hasfinalizerexpr +| 310 = @isdelegateexpr +| 311 = @isinterfaceclassexpr +| 312 = @isrefarrayexpr +| 313 = @isrefclassexpr +| 314 = @issealedexpr +| 315 = @issimplevalueclassexpr +| 316 = @isvalueclassexpr +| 317 = @isfinalexpr +| 319 = @noexceptexpr +| 320 = @builtinshufflevector +| 321 = @builtinchooseexpr +| 322 = @builtinaddressof +| 323 = @vec_fill +| 324 = @builtinconvertvector +| 325 = @builtincomplex +| 326 = @spaceshipexpr +| 327 = @co_await +| 328 = @co_yield +| 329 = @temp_init +| 330 = @isassignable +| 331 = @isaggregate +| 332 = @hasuniqueobjectrepresentations +| 333 = @builtinbitcast +| 334 = @builtinshuffle +| 335 = @blockassignexpr +| 336 = @issame +| 337 = @isfunction +| 338 = @islayoutcompatible +| 339 = @ispointerinterconvertiblebaseof +| 340 = @isarray +| 341 = @arrayrank +| 342 = @arrayextent +| 343 = @isarithmetic +| 344 = @iscompletetype +| 345 = @iscompound +| 346 = @isconst +| 347 = @isfloatingpoint +| 348 = @isfundamental +| 349 = @isintegral +| 350 = @islvaluereference +| 351 = @ismemberfunctionpointer +| 352 = @ismemberobjectpointer +| 353 = @ismemberpointer +| 354 = @isobject +| 355 = @ispointer +| 356 = @isreference +| 357 = @isrvaluereference +| 358 = @isscalar +| 359 = @issigned +| 360 = @isunsigned +| 361 = @isvoid +| 362 = @isvolatile +; + +@var_args_expr = @vastartexpr + | @vaendexpr + | @vaargexpr + | @vacopyexpr + ; + +@builtin_op = @var_args_expr + | @noopexpr + | @offsetofexpr + | @intaddrexpr + | @hasassignexpr + | @hascopyexpr + | @hasnothrowassign + | @hasnothrowconstr + | @hasnothrowcopy + | @hastrivialassign + | @hastrivialconstr + | @hastrivialcopy + | @hastrivialdestructor + | @hasuserdestr + | @hasvirtualdestr + | @isabstractexpr + | @isbaseofexpr + | @isclassexpr + | @isconvtoexpr + | @isemptyexpr + | @isenumexpr + | @ispodexpr + | @ispolyexpr + | @isunionexpr + | @typescompexpr + | @builtinshufflevector + | @builtinconvertvector + | @builtinaddressof + | @istriviallyconstructibleexpr + | @isdestructibleexpr + | @isnothrowdestructibleexpr + | @istriviallydestructibleexpr + | @istriviallyassignableexpr + | @isnothrowassignableexpr + | @isstandardlayoutexpr + | @istriviallycopyableexpr + | @isliteraltypeexpr + | @hastrivialmoveconstructorexpr + | @hastrivialmoveassignexpr + | @hasnothrowmoveassignexpr + | @isconstructibleexpr + | @isnothrowconstructibleexpr + | @hasfinalizerexpr + | @isdelegateexpr + | @isinterfaceclassexpr + | @isrefarrayexpr + | @isrefclassexpr + | @issealedexpr + | @issimplevalueclassexpr + | @isvalueclassexpr + | @isfinalexpr + | @builtinchooseexpr + | @builtincomplex + | @isassignable + | @isaggregate + | @hasuniqueobjectrepresentations + | @builtinbitcast + | @builtinshuffle + | @issame + | @isfunction + | @islayoutcompatible + | @ispointerinterconvertiblebaseof + | @isarray + | @arrayrank + | @arrayextent + | @isarithmetic + | @iscompletetype + | @iscompound + | @isconst + | @isfloatingpoint + | @isfundamental + | @isintegral + | @islvaluereference + | @ismemberfunctionpointer + | @ismemberobjectpointer + | @ismemberpointer + | @isobject + | @ispointer + | @isreference + | @isrvaluereference + | @isscalar + | @issigned + | @isunsigned + | @isvoid + | @isvolatile + ; + +new_allocated_type( + unique int expr: @new_expr ref, + int type_id: @type ref +); + +new_array_allocated_type( + unique int expr: @new_array_expr ref, + int type_id: @type ref +); + +/** + * The field being initialized by an initializer expression within an aggregate + * initializer for a class/struct/union. Position is used to sort repeated initializers. + */ +#keyset[aggregate, position] +aggregate_field_init( + int aggregate: @aggregateliteral ref, + int initializer: @expr ref, + int field: @membervariable ref, + int position: int ref +); + +/** + * The index of the element being initialized by an initializer expression + * within an aggregate initializer for an array. Position is used to sort repeated initializers. + */ +#keyset[aggregate, position] +aggregate_array_init( + int aggregate: @aggregateliteral ref, + int initializer: @expr ref, + int element_index: int ref, + int position: int ref +); + +@ctorinit = @ctordirectinit + | @ctorvirtualinit + | @ctorfieldinit + | @ctordelegatinginit; +@dtordestruct = @dtordirectdestruct + | @dtorvirtualdestruct + | @dtorfielddestruct; + + +condition_decl_bind( + unique int expr: @condition_decl ref, + unique int decl: @declaration ref +); + +typeid_bind( + unique int expr: @type_id ref, + int type_id: @type ref +); + +uuidof_bind( + unique int expr: @uuidof ref, + int type_id: @type ref +); + +@runtime_sizeof_or_alignof = @runtime_sizeof | @runtime_alignof; + +sizeof_bind( + unique int expr: @runtime_sizeof_or_alignof ref, + int type_id: @type ref +); + +code_block( + unique int block: @literal ref, + unique int routine: @function ref +); + +lambdas( + unique int expr: @lambdaexpr ref, + string default_capture: string ref, + boolean has_explicit_return_type: boolean ref +); + +lambda_capture( + unique int id: @lambdacapture, + int lambda: @lambdaexpr ref, + int index: int ref, + int field: @membervariable ref, + boolean captured_by_reference: boolean ref, + boolean is_implicit: boolean ref, + int location: @location_default ref +); + +@funbindexpr = @routineexpr + | @new_expr + | @delete_expr + | @delete_array_expr + | @ctordirectinit + | @ctorvirtualinit + | @ctordelegatinginit + | @dtordirectdestruct + | @dtorvirtualdestruct; + +@varbindexpr = @varaccess | @ctorfieldinit | @dtorfielddestruct; +@addressable = @function | @variable ; +@accessible = @addressable | @enumconstant ; + +@access = @varaccess | @routineexpr ; + +fold( + int expr: @foldexpr ref, + string operator: string ref, + boolean is_left_fold: boolean ref +); + +stmts( + unique int id: @stmt, + int kind: int ref, + int location: @location_stmt ref +); + +case @stmt.kind of + 1 = @stmt_expr +| 2 = @stmt_if +| 3 = @stmt_while +| 4 = @stmt_goto +| 5 = @stmt_label +| 6 = @stmt_return +| 7 = @stmt_block +| 8 = @stmt_end_test_while // do { ... } while ( ... ) +| 9 = @stmt_for +| 10 = @stmt_switch_case +| 11 = @stmt_switch +| 13 = @stmt_asm // "asm" statement or the body of an asm function +| 15 = @stmt_try_block +| 16 = @stmt_microsoft_try // Microsoft +| 17 = @stmt_decl +| 18 = @stmt_set_vla_size // C99 +| 19 = @stmt_vla_decl // C99 +| 25 = @stmt_assigned_goto // GNU +| 26 = @stmt_empty +| 27 = @stmt_continue +| 28 = @stmt_break +| 29 = @stmt_range_based_for // C++11 +// ... 30 @stmt_at_autoreleasepool_block deprecated +// ... 31 @stmt_objc_for_in deprecated +// ... 32 @stmt_at_synchronized deprecated +| 33 = @stmt_handler +// ... 34 @stmt_finally_end deprecated +| 35 = @stmt_constexpr_if +| 37 = @stmt_co_return +; + +type_vla( + int type_id: @type ref, + int decl: @stmt_vla_decl ref +); + +variable_vla( + int var: @variable ref, + int decl: @stmt_vla_decl ref +); + +if_initialization( + unique int if_stmt: @stmt_if ref, + int init_id: @stmt ref +); + +if_then( + unique int if_stmt: @stmt_if ref, + int then_id: @stmt ref +); + +if_else( + unique int if_stmt: @stmt_if ref, + int else_id: @stmt ref +); + +constexpr_if_initialization( + unique int constexpr_if_stmt: @stmt_constexpr_if ref, + int init_id: @stmt ref +); + +constexpr_if_then( + unique int constexpr_if_stmt: @stmt_constexpr_if ref, + int then_id: @stmt ref +); + +constexpr_if_else( + unique int constexpr_if_stmt: @stmt_constexpr_if ref, + int else_id: @stmt ref +); + +while_body( + unique int while_stmt: @stmt_while ref, + int body_id: @stmt ref +); + +do_body( + unique int do_stmt: @stmt_end_test_while ref, + int body_id: @stmt ref +); + +switch_initialization( + unique int switch_stmt: @stmt_switch ref, + int init_id: @stmt ref +); + +#keyset[switch_stmt, index] +switch_case( + int switch_stmt: @stmt_switch ref, + int index: int ref, + int case_id: @stmt_switch_case ref +); + +switch_body( + unique int switch_stmt: @stmt_switch ref, + int body_id: @stmt ref +); + +for_initialization( + unique int for_stmt: @stmt_for ref, + int init_id: @stmt ref +); + +for_condition( + unique int for_stmt: @stmt_for ref, + int condition_id: @expr ref +); + +for_update( + unique int for_stmt: @stmt_for ref, + int update_id: @expr ref +); + +for_body( + unique int for_stmt: @stmt_for ref, + int body_id: @stmt ref +); + +@stmtparent = @stmt | @expr_stmt ; +stmtparents( + unique int id: @stmt ref, + int index: int ref, + int parent: @stmtparent ref +); + +ishandler(unique int block: @stmt_block ref); + +@cfgnode = @stmt | @expr | @function | @initialiser ; + +stmt_decl_bind( + int stmt: @stmt_decl ref, + int num: int ref, + int decl: @declaration ref +); + +stmt_decl_entry_bind( + int stmt: @stmt_decl ref, + int num: int ref, + int decl_entry: @element ref +); + +@functionorblock = @function | @stmt_block; + +blockscope( + unique int block: @stmt_block ref, + int enclosing: @functionorblock ref +); + +@jump = @stmt_goto | @stmt_break | @stmt_continue; + +@jumporlabel = @jump | @stmt_label | @literal; + +jumpinfo( + unique int id: @jumporlabel ref, + string str: string ref, + int target: @stmt ref +); + +preprocdirects( + unique int id: @preprocdirect, + int kind: int ref, + int location: @location_default ref +); +case @preprocdirect.kind of + 0 = @ppd_if +| 1 = @ppd_ifdef +| 2 = @ppd_ifndef +| 3 = @ppd_elif +| 4 = @ppd_else +| 5 = @ppd_endif +| 6 = @ppd_plain_include +| 7 = @ppd_define +| 8 = @ppd_undef +| 9 = @ppd_line +| 10 = @ppd_error +| 11 = @ppd_pragma +| 12 = @ppd_objc_import +| 13 = @ppd_include_next +| 18 = @ppd_warning +; + +@ppd_include = @ppd_plain_include | @ppd_objc_import | @ppd_include_next; + +@ppd_branch = @ppd_if | @ppd_ifdef | @ppd_ifndef | @ppd_elif; + +preprocpair( + int begin : @ppd_branch ref, + int elseelifend : @preprocdirect ref +); + +preproctrue(int branch : @ppd_branch ref); +preprocfalse(int branch : @ppd_branch ref); + +preproctext( + unique int id: @preprocdirect ref, + string head: string ref, + string body: string ref +); + +includes( + unique int id: @ppd_include ref, + int included: @file ref +); + +link_targets( + unique int id: @link_target, + int binary: @file ref +); + +link_parent( + int element : @element ref, + int link_target : @link_target ref +); + +/* XML Files */ + +xmlEncoding(unique int id: @file ref, string encoding: string ref); + +xmlDTDs( + unique int id: @xmldtd, + string root: string ref, + string publicId: string ref, + string systemId: string ref, + int fileid: @file ref +); + +xmlElements( + unique int id: @xmlelement, + string name: string ref, + int parentid: @xmlparent ref, + int idx: int ref, + int fileid: @file ref +); + +xmlAttrs( + unique int id: @xmlattribute, + int elementid: @xmlelement ref, + string name: string ref, + string value: string ref, + int idx: int ref, + int fileid: @file ref +); + +xmlNs( + int id: @xmlnamespace, + string prefixName: string ref, + string URI: string ref, + int fileid: @file ref +); + +xmlHasNs( + int elementId: @xmlnamespaceable ref, + int nsId: @xmlnamespace ref, + int fileid: @file ref +); + +xmlComments( + unique int id: @xmlcomment, + string text: string ref, + int parentid: @xmlparent ref, + int fileid: @file ref +); + +xmlChars( + unique int id: @xmlcharacters, + string text: string ref, + int parentid: @xmlparent ref, + int idx: int ref, + int isCDATA: int ref, + int fileid: @file ref +); + +@xmlparent = @file | @xmlelement; +@xmlnamespaceable = @xmlelement | @xmlattribute; + +xmllocations( + int xmlElement: @xmllocatable ref, + int location: @location_default ref +); + +@xmllocatable = @xmlcharacters + | @xmlelement + | @xmlcomment + | @xmlattribute + | @xmldtd + | @file + | @xmlnamespace; diff --git a/cpp/ql/lib/upgrades/19887dbd33327fb07d54251786e0cb2578539775/upgrade.properties b/cpp/ql/lib/upgrades/19887dbd33327fb07d54251786e0cb2578539775/upgrade.properties new file mode 100644 index 00000000000..58fffa7cbbc --- /dev/null +++ b/cpp/ql/lib/upgrades/19887dbd33327fb07d54251786e0cb2578539775/upgrade.properties @@ -0,0 +1,3 @@ +description: Remove _Float128 type +compatibility: partial +builtintypes.rel: run builtintypes.qlo diff --git a/cpp/ql/src/experimental/Security/CWE/CWE-193/ConstantSizeArrayOffByOne.ql b/cpp/ql/src/experimental/Security/CWE/CWE-193/ConstantSizeArrayOffByOne.ql index c38a012b27b..5b0ab08bec2 100644 --- a/cpp/ql/src/experimental/Security/CWE/CWE-193/ConstantSizeArrayOffByOne.ql +++ b/cpp/ql/src/experimental/Security/CWE/CWE-193/ConstantSizeArrayOffByOne.ql @@ -28,7 +28,8 @@ Instruction getABoundIn(SemBound b, IRFunction func) { pragma[inline] predicate boundedImpl(Instruction i, Instruction b, int delta) { exists(SemBound bound, IRFunction func | - semBounded(getSemanticExpr(i), bound, delta, true, _) and + semBounded(getSemanticExpr(i), bound, delta, true, + any(SemReason reason | not reason instanceof SemTypeReason)) and b = getABoundIn(bound, func) and pragma[only_bind_out](i.getEnclosingIRFunction()) = func ) @@ -93,7 +94,8 @@ predicate arrayTypeHasSizes(ArrayType arr, int baseTypeSize, int size) { bindingset[pai] pragma[inline_late] predicate constantUpperBounded(PointerArithmeticInstruction pai, int delta) { - semBounded(getSemanticExpr(pai.getRight()), any(SemZeroBound b), delta, true, _) + semBounded(getSemanticExpr(pai.getRight()), any(SemZeroBound b), delta, true, + any(SemReason reason | not reason instanceof SemTypeReason)) } bindingset[pai, size] diff --git a/cpp/ql/test/library-tests/ir/range-analysis/SimpleRangeAnalysis_tests.cpp b/cpp/ql/test/library-tests/ir/range-analysis/SimpleRangeAnalysis_tests.cpp index df29578409b..114d164a91a 100644 --- a/cpp/ql/test/library-tests/ir/range-analysis/SimpleRangeAnalysis_tests.cpp +++ b/cpp/ql/test/library-tests/ir/range-analysis/SimpleRangeAnalysis_tests.cpp @@ -195,18 +195,18 @@ int test13(char c, int i) { int z = i+1; // $ overflow=+ range(z); // $ range="==InitializeParameter: i+1" range(c + i + uc + x + y + z); // $ overflow=+- overflow=+ overflow=- MISSING: range=>=1 - range((double)(c + i + uc + x + y + z)); // $ overflow=+ overflow=+- overflow=- MISSING: range=>=1 + range((double)(c + i + uc + x + y + z)); // $ overflow=+ overflow=+- overflow=- range=<=4294967295 MISSING: range=>=1 return (double)(c + i + uc + x + y + z); // $ overflow=+- overflow=+ overflow=- } // Regression test for ODASA-6013. int test14(int x) { int x0 = (int)(char)x; - range(x0); + range(x0); // $ range=<=127 range=>=-128 int x1 = (int)(unsigned char)x; - range(x1); + range(x1); // $ range=<=255 range=>=0 int x2 = (int)(unsigned short)x; - range(x2); + range(x2); // $ range=<=65535 range=>=0 int x3 = (int)(unsigned int)x; range(x3); char c0 = x; @@ -759,9 +759,9 @@ unsigned long mult_overflow() { unsigned long mult_lower_bound(unsigned int ui, unsigned long ul) { if (ui >= 10) { range(ui); // $ range=>=10 - range((unsigned long)ui); // $ range=>=10 - unsigned long result = (unsigned long)ui * ui; // $ overflow=+ - range(result); // $ MISSING: range=>=100 + range((unsigned long)ui); // $ range=>=10 range=<=4294967295 + unsigned long result = (unsigned long)ui * ui; // no overflow + range(result); // $ range=>=100 range=<=18446744065119617024 return result; // BUG: upper bound should be >= 18446744065119617025 } if (ul >= 10) { @@ -888,7 +888,7 @@ void notequal_variations(short n, float f) { } if (n >= 5) { - if (2 * n - 10 == 0) { // $ overflow=+ + if (2 * n - 10 == 0) { // no overflow range(n); // $ range=>=5 MISSING: range===5 return; } @@ -936,7 +936,7 @@ void two_bounds_from_one_test(short ss, unsigned short us) { range(ss); // -32768 .. 32767 } - if (ss + 1 < sizeof(int)) { // $ overflow=+ + if (ss + 1 < sizeof(int)) { // $ overflow=- range(ss); // -1 .. 2 } } diff --git a/cpp/ql/test/library-tests/templates/type_instantiations/types.expected b/cpp/ql/test/library-tests/templates/type_instantiations/types.expected index 713dc44e495..b73feb125e2 100644 --- a/cpp/ql/test/library-tests/templates/type_instantiations/types.expected +++ b/cpp/ql/test/library-tests/templates/type_instantiations/types.expected @@ -13,7 +13,6 @@ | file://:0:0:0:0 | _Float64 | | file://:0:0:0:0 | _Float64x | | file://:0:0:0:0 | _Float128 | -| file://:0:0:0:0 | _Float128x | | file://:0:0:0:0 | _Imaginary double | | file://:0:0:0:0 | _Imaginary float | | file://:0:0:0:0 | _Imaginary long double | diff --git a/cpp/ql/test/library-tests/type_sizes/type_sizes.expected b/cpp/ql/test/library-tests/type_sizes/type_sizes.expected index a92bbe2f69a..69bfebe2195 100644 --- a/cpp/ql/test/library-tests/type_sizes/type_sizes.expected +++ b/cpp/ql/test/library-tests/type_sizes/type_sizes.expected @@ -33,7 +33,6 @@ | file://:0:0:0:0 | _Float64 | 8 | | file://:0:0:0:0 | _Float64x | 16 | | file://:0:0:0:0 | _Float128 | 16 | -| file://:0:0:0:0 | _Float128x | 32 | | file://:0:0:0:0 | _Imaginary double | 8 | | file://:0:0:0:0 | _Imaginary float | 4 | | file://:0:0:0:0 | _Imaginary long double | 16 | diff --git a/cpp/ql/test/library-tests/unspecified_type/types/unspecified_type.expected b/cpp/ql/test/library-tests/unspecified_type/types/unspecified_type.expected index 3e231649b2c..d635ec8b5df 100644 --- a/cpp/ql/test/library-tests/unspecified_type/types/unspecified_type.expected +++ b/cpp/ql/test/library-tests/unspecified_type/types/unspecified_type.expected @@ -15,7 +15,6 @@ | file://:0:0:0:0 | _Float64 | _Float64 | | file://:0:0:0:0 | _Float64x | _Float64x | | file://:0:0:0:0 | _Float128 | _Float128 | -| file://:0:0:0:0 | _Float128x | _Float128x | | file://:0:0:0:0 | _Imaginary double | _Imaginary double | | file://:0:0:0:0 | _Imaginary float | _Imaginary float | | file://:0:0:0:0 | _Imaginary long double | _Imaginary long double | diff --git a/cpp/ql/test/library-tests/variables/variables/types.expected b/cpp/ql/test/library-tests/variables/variables/types.expected index 19e7411005f..f086dec166a 100644 --- a/cpp/ql/test/library-tests/variables/variables/types.expected +++ b/cpp/ql/test/library-tests/variables/variables/types.expected @@ -14,7 +14,6 @@ | _Float64 | BinaryFloatingPointType, RealNumberType | | | | | | _Float64x | BinaryFloatingPointType, RealNumberType | | | | | | _Float128 | BinaryFloatingPointType, RealNumberType | | | | | -| _Float128x | BinaryFloatingPointType, RealNumberType | | | | | | _Imaginary double | BinaryFloatingPointType, ImaginaryNumberType | | | | | | _Imaginary float | BinaryFloatingPointType, ImaginaryNumberType | | | | | | _Imaginary long double | BinaryFloatingPointType, ImaginaryNumberType | | | | | diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Runtime.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Runtime.cs index bb521523ead..c3ea55c7114 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Runtime.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Runtime.cs @@ -22,7 +22,7 @@ namespace Semmle.Extraction.CSharp.Standalone public Runtime(IDotNet dotNet) => this.dotNet = dotNet; - internal sealed class RuntimeVersion : IComparable + internal record RuntimeVersion : IComparable { private readonly string dir; private readonly Version version; @@ -71,11 +71,6 @@ namespace Semmle.Extraction.CSharp.Standalone return c; } - public override bool Equals(object? obj) => - obj is not null && obj is RuntimeVersion other && other.FullPath == FullPath; - - public override int GetHashCode() => FullPath.GetHashCode(); - public override string ToString() => FullPath; } @@ -97,7 +92,7 @@ namespace Semmle.Extraction.CSharp.Standalone var match = RuntimeRegex().Match(r); if (match.Success) { - runtimes.AddOrUpdate(match.Groups[1].Value, new RuntimeVersion(match.Groups[6].Value, match.Groups[2].Value, match.Groups[4].Value, match.Groups[5].Value)); + runtimes.AddOrUpdateToLatest(match.Groups[1].Value, new RuntimeVersion(match.Groups[6].Value, match.Groups[2].Value, match.Groups[4].Value, match.Groups[5].Value)); } }); diff --git a/csharp/extractor/Semmle.Util/DictionaryExtensions.cs b/csharp/extractor/Semmle.Util/DictionaryExtensions.cs index fa932f0cd9c..5ac4f3bfaa9 100644 --- a/csharp/extractor/Semmle.Util/DictionaryExtensions.cs +++ b/csharp/extractor/Semmle.Util/DictionaryExtensions.cs @@ -22,9 +22,9 @@ namespace Semmle.Util /// /// Adds a new value or replaces the existing value (if the new value is greater than the existing) - /// in dictionary for the given key. + /// in this dictionary for the given key. /// - public static void AddOrUpdate(this Dictionary dict, T1 key, T2 value) where T1 : notnull where T2 : IComparable + public static void AddOrUpdateToLatest(this Dictionary dict, T1 key, T2 value) where T1 : notnull where T2 : IComparable { if (!dict.TryGetValue(key, out var existing) || existing.CompareTo(value) < 0) { diff --git a/csharp/ql/lib/change-notes/2023-08-04-netcore-identity-hardcoded-cred.md b/csharp/ql/lib/change-notes/2023-08-04-netcore-identity-hardcoded-cred.md new file mode 100644 index 00000000000..c442579627f --- /dev/null +++ b/csharp/ql/lib/change-notes/2023-08-04-netcore-identity-hardcoded-cred.md @@ -0,0 +1,4 @@ +--- +category: minorAnalysis +--- +* The query library for `cs/hardcoded-credentials` now excludes benign properties such as `UserNameClaimType` and `AllowedUserNameCharacters` from `Microsoft.AspNetCore.Identity` options classes. \ No newline at end of file diff --git a/csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl.qll b/csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl.qll index def6651b44b..4cf39afc812 100644 --- a/csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl.qll +++ b/csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl.qll @@ -1,3 +1,3 @@ private import DataFlowImplSpecific -private import codeql.dataflow.DataFlowImpl +private import codeql.dataflow.internal.DataFlowImpl import MakeImpl diff --git a/csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplCommon.qll b/csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplCommon.qll index 8c557b08220..55829d7d059 100644 --- a/csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplCommon.qll +++ b/csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplCommon.qll @@ -1,3 +1,3 @@ private import DataFlowImplSpecific -private import codeql.dataflow.DataFlowImplCommon +private import codeql.dataflow.internal.DataFlowImplCommon import MakeImplCommon diff --git a/csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplSpecific.qll b/csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplSpecific.qll index fefdff3412a..4c003584d10 100644 --- a/csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplSpecific.qll +++ b/csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplSpecific.qll @@ -2,7 +2,7 @@ * Provides C#-specific definitions for use in the data flow library. */ -private import codeql.dataflow.DataFlowParameter +private import codeql.dataflow.DataFlow module Private { import DataFlowPrivate @@ -13,7 +13,7 @@ module Public { import DataFlowPublic } -module CsharpDataFlow implements DataFlowParameter { +module CsharpDataFlow implements InputSig { import Private import Public diff --git a/csharp/ql/lib/semmle/code/csharp/security/dataflow/HardcodedCredentialsQuery.qll b/csharp/ql/lib/semmle/code/csharp/security/dataflow/HardcodedCredentialsQuery.qll index a5875fe001c..dd6669579e1 100644 --- a/csharp/ql/lib/semmle/code/csharp/security/dataflow/HardcodedCredentialsQuery.qll +++ b/csharp/ql/lib/semmle/code/csharp/security/dataflow/HardcodedCredentialsQuery.qll @@ -169,7 +169,7 @@ private class CredentialVar extends Assignable { exists(string name | name = this.getName() | name.regexpMatch("(?i).*pass(wd|word|code|phrase)(?!.*question).*") or - name.regexpMatch("(?i).*(puid|username|userid).*") + name.regexpMatch("(?i).*(puid|username|userid)(?!.*(characters|claimtype)).*") or name.regexpMatch("(?i).*(cert)(?!.*(format|name)).*") ) diff --git a/csharp/ql/test/query-tests/Security Features/CWE-798/HardcodedConnectionString.expected b/csharp/ql/test/query-tests/Security Features/CWE-798/HardcodedConnectionString.expected index 1191b40ec28..7de4c593e25 100644 --- a/csharp/ql/test/query-tests/Security Features/CWE-798/HardcodedConnectionString.expected +++ b/csharp/ql/test/query-tests/Security Features/CWE-798/HardcodedConnectionString.expected @@ -1,8 +1,8 @@ edges nodes -| HardcodedCredentials.cs:54:48:54:63 | "Password=12345" | semmle.label | "Password=12345" | -| HardcodedCredentials.cs:56:49:56:63 | "User Id=12345" | semmle.label | "User Id=12345" | +| HardcodedCredentials.cs:55:48:55:63 | "Password=12345" | semmle.label | "Password=12345" | +| HardcodedCredentials.cs:57:49:57:63 | "User Id=12345" | semmle.label | "User Id=12345" | subpaths #select -| HardcodedCredentials.cs:54:48:54:63 | "Password=12345" | HardcodedCredentials.cs:54:48:54:63 | "Password=12345" | HardcodedCredentials.cs:54:48:54:63 | "Password=12345" | 'ConnectionString' property includes hard-coded credentials set in $@. | HardcodedCredentials.cs:54:30:54:64 | object creation of type SqlConnection | object creation of type SqlConnection | -| HardcodedCredentials.cs:56:49:56:63 | "User Id=12345" | HardcodedCredentials.cs:56:49:56:63 | "User Id=12345" | HardcodedCredentials.cs:56:49:56:63 | "User Id=12345" | 'ConnectionString' property includes hard-coded credentials set in $@. | HardcodedCredentials.cs:56:31:56:64 | object creation of type SqlConnection | object creation of type SqlConnection | +| HardcodedCredentials.cs:55:48:55:63 | "Password=12345" | HardcodedCredentials.cs:55:48:55:63 | "Password=12345" | HardcodedCredentials.cs:55:48:55:63 | "Password=12345" | 'ConnectionString' property includes hard-coded credentials set in $@. | HardcodedCredentials.cs:55:30:55:64 | object creation of type SqlConnection | object creation of type SqlConnection | +| HardcodedCredentials.cs:57:49:57:63 | "User Id=12345" | HardcodedCredentials.cs:57:49:57:63 | "User Id=12345" | HardcodedCredentials.cs:57:49:57:63 | "User Id=12345" | 'ConnectionString' property includes hard-coded credentials set in $@. | HardcodedCredentials.cs:57:31:57:64 | object creation of type SqlConnection | object creation of type SqlConnection | diff --git a/csharp/ql/test/query-tests/Security Features/CWE-798/HardcodedCredentials.cs b/csharp/ql/test/query-tests/Security Features/CWE-798/HardcodedCredentials.cs index 71fea63ff4a..840d65b1965 100644 --- a/csharp/ql/test/query-tests/Security Features/CWE-798/HardcodedCredentials.cs +++ b/csharp/ql/test/query-tests/Security Features/CWE-798/HardcodedCredentials.cs @@ -3,6 +3,7 @@ using System.Data.SqlClient; using System.Web; using System.Web.Security; using System.Security.Cryptography.X509Certificates; +using Microsoft.AspNetCore.Identity; public class HardcodedHandler : IHttpHandler { @@ -72,6 +73,21 @@ public class HardcodedHandler : IHttpHandler // BAD: Hard-coded user Membership.CreateUser("myusername", "mypassword"); + + var identityOptions = new IdentityOptions + { + User = new UserOptions + { + // GOOD: This is not a credential so hardcoding a string assignment is fine + AllowedUserNameCharacters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._@+" + } + }; + + var claimsIdentityOptions = new ClaimsIdentityOptions + { + // GOOD: This is not a credential so hardcoding a string assignment is fine + UserNameClaimType = "username" + }; } class Foo diff --git a/csharp/ql/test/query-tests/Security Features/CWE-798/HardcodedCredentials.expected b/csharp/ql/test/query-tests/Security Features/CWE-798/HardcodedCredentials.expected index 49214e4426c..4dc81537e20 100644 --- a/csharp/ql/test/query-tests/Security Features/CWE-798/HardcodedCredentials.expected +++ b/csharp/ql/test/query-tests/Security Features/CWE-798/HardcodedCredentials.expected @@ -1,24 +1,24 @@ edges -| HardcodedCredentials.cs:47:30:47:60 | array creation of type Byte[] : Byte[] | HardcodedCredentials.cs:50:13:50:23 | access to local variable rawCertData | +| HardcodedCredentials.cs:48:30:48:60 | array creation of type Byte[] : Byte[] | HardcodedCredentials.cs:51:13:51:23 | access to local variable rawCertData | nodes -| HardcodedCredentials.cs:15:25:15:36 | "myPa55word" | semmle.label | "myPa55word" | -| HardcodedCredentials.cs:31:19:31:28 | "username" | semmle.label | "username" | -| HardcodedCredentials.cs:45:39:45:53 | "myNewPa55word" | semmle.label | "myNewPa55word" | -| HardcodedCredentials.cs:47:30:47:60 | array creation of type Byte[] : Byte[] | semmle.label | array creation of type Byte[] : Byte[] | -| HardcodedCredentials.cs:50:13:50:23 | access to local variable rawCertData | semmle.label | access to local variable rawCertData | -| HardcodedCredentials.cs:51:13:51:24 | "myPa55word" | semmle.label | "myPa55word" | -| HardcodedCredentials.cs:74:31:74:42 | "myusername" | semmle.label | "myusername" | -| HardcodedCredentials.cs:74:45:74:56 | "mypassword" | semmle.label | "mypassword" | +| HardcodedCredentials.cs:16:25:16:36 | "myPa55word" | semmle.label | "myPa55word" | +| HardcodedCredentials.cs:32:19:32:28 | "username" | semmle.label | "username" | +| HardcodedCredentials.cs:46:39:46:53 | "myNewPa55word" | semmle.label | "myNewPa55word" | +| HardcodedCredentials.cs:48:30:48:60 | array creation of type Byte[] : Byte[] | semmle.label | array creation of type Byte[] : Byte[] | +| HardcodedCredentials.cs:51:13:51:23 | access to local variable rawCertData | semmle.label | access to local variable rawCertData | +| HardcodedCredentials.cs:52:13:52:24 | "myPa55word" | semmle.label | "myPa55word" | +| HardcodedCredentials.cs:75:31:75:42 | "myusername" | semmle.label | "myusername" | +| HardcodedCredentials.cs:75:45:75:56 | "mypassword" | semmle.label | "mypassword" | | TestHardcodedCredentials.cs:21:31:21:42 | "myusername" | semmle.label | "myusername" | | TestHardcodedCredentials.cs:21:45:21:56 | "mypassword" | semmle.label | "mypassword" | | TestHardcodedCredentials.cs:26:19:26:28 | "username" | semmle.label | "username" | subpaths #select -| HardcodedCredentials.cs:15:25:15:36 | "myPa55word" | HardcodedCredentials.cs:15:25:15:36 | "myPa55word" | HardcodedCredentials.cs:15:25:15:36 | "myPa55word" | The hard-coded value "myPa55word" flows to $@ which is compared against $@. | HardcodedCredentials.cs:15:25:15:36 | "myPa55word" | "myPa55word" | HardcodedCredentials.cs:15:13:15:20 | access to local variable password | access to local variable password | -| HardcodedCredentials.cs:31:19:31:28 | "username" | HardcodedCredentials.cs:31:19:31:28 | "username" | HardcodedCredentials.cs:31:19:31:28 | "username" | The hard-coded value "username" flows to the $@ parameter in $@. | HardcodedCredentials.cs:31:19:31:28 | "username" | name | HardcodedCredentials.cs:29:31:43:13 | object creation of type MembershipUser | object creation of type MembershipUser | -| HardcodedCredentials.cs:45:39:45:53 | "myNewPa55word" | HardcodedCredentials.cs:45:39:45:53 | "myNewPa55word" | HardcodedCredentials.cs:45:39:45:53 | "myNewPa55word" | The hard-coded value "myNewPa55word" flows to the $@ parameter in $@. | HardcodedCredentials.cs:45:39:45:53 | "myNewPa55word" | newPassword | HardcodedCredentials.cs:45:9:45:54 | call to method ChangePassword | call to method ChangePassword | -| HardcodedCredentials.cs:47:30:47:60 | array creation of type Byte[] | HardcodedCredentials.cs:47:30:47:60 | array creation of type Byte[] : Byte[] | HardcodedCredentials.cs:50:13:50:23 | access to local variable rawCertData | This hard-coded value flows to the $@ parameter in $@. | HardcodedCredentials.cs:50:13:50:23 | access to local variable rawCertData | rawData | HardcodedCredentials.cs:49:33:51:25 | object creation of type X509Certificate2 | object creation of type X509Certificate2 | -| HardcodedCredentials.cs:51:13:51:24 | "myPa55word" | HardcodedCredentials.cs:51:13:51:24 | "myPa55word" | HardcodedCredentials.cs:51:13:51:24 | "myPa55word" | The hard-coded value "myPa55word" flows to the $@ parameter in $@. | HardcodedCredentials.cs:51:13:51:24 | "myPa55word" | password | HardcodedCredentials.cs:49:33:51:25 | object creation of type X509Certificate2 | object creation of type X509Certificate2 | -| HardcodedCredentials.cs:74:31:74:42 | "myusername" | HardcodedCredentials.cs:74:31:74:42 | "myusername" | HardcodedCredentials.cs:74:31:74:42 | "myusername" | The hard-coded value "myusername" flows to the $@ parameter in $@. | HardcodedCredentials.cs:74:31:74:42 | "myusername" | username | HardcodedCredentials.cs:74:9:74:57 | call to method CreateUser | call to method CreateUser | -| HardcodedCredentials.cs:74:45:74:56 | "mypassword" | HardcodedCredentials.cs:74:45:74:56 | "mypassword" | HardcodedCredentials.cs:74:45:74:56 | "mypassword" | The hard-coded value "mypassword" flows to the $@ parameter in $@. | HardcodedCredentials.cs:74:45:74:56 | "mypassword" | password | HardcodedCredentials.cs:74:9:74:57 | call to method CreateUser | call to method CreateUser | +| HardcodedCredentials.cs:16:25:16:36 | "myPa55word" | HardcodedCredentials.cs:16:25:16:36 | "myPa55word" | HardcodedCredentials.cs:16:25:16:36 | "myPa55word" | The hard-coded value "myPa55word" flows to $@ which is compared against $@. | HardcodedCredentials.cs:16:25:16:36 | "myPa55word" | "myPa55word" | HardcodedCredentials.cs:16:13:16:20 | access to local variable password | access to local variable password | +| HardcodedCredentials.cs:32:19:32:28 | "username" | HardcodedCredentials.cs:32:19:32:28 | "username" | HardcodedCredentials.cs:32:19:32:28 | "username" | The hard-coded value "username" flows to the $@ parameter in $@. | HardcodedCredentials.cs:32:19:32:28 | "username" | name | HardcodedCredentials.cs:30:31:44:13 | object creation of type MembershipUser | object creation of type MembershipUser | +| HardcodedCredentials.cs:46:39:46:53 | "myNewPa55word" | HardcodedCredentials.cs:46:39:46:53 | "myNewPa55word" | HardcodedCredentials.cs:46:39:46:53 | "myNewPa55word" | The hard-coded value "myNewPa55word" flows to the $@ parameter in $@. | HardcodedCredentials.cs:46:39:46:53 | "myNewPa55word" | newPassword | HardcodedCredentials.cs:46:9:46:54 | call to method ChangePassword | call to method ChangePassword | +| HardcodedCredentials.cs:48:30:48:60 | array creation of type Byte[] | HardcodedCredentials.cs:48:30:48:60 | array creation of type Byte[] : Byte[] | HardcodedCredentials.cs:51:13:51:23 | access to local variable rawCertData | This hard-coded value flows to the $@ parameter in $@. | HardcodedCredentials.cs:51:13:51:23 | access to local variable rawCertData | rawData | HardcodedCredentials.cs:50:33:52:25 | object creation of type X509Certificate2 | object creation of type X509Certificate2 | +| HardcodedCredentials.cs:52:13:52:24 | "myPa55word" | HardcodedCredentials.cs:52:13:52:24 | "myPa55word" | HardcodedCredentials.cs:52:13:52:24 | "myPa55word" | The hard-coded value "myPa55word" flows to the $@ parameter in $@. | HardcodedCredentials.cs:52:13:52:24 | "myPa55word" | password | HardcodedCredentials.cs:50:33:52:25 | object creation of type X509Certificate2 | object creation of type X509Certificate2 | +| HardcodedCredentials.cs:75:31:75:42 | "myusername" | HardcodedCredentials.cs:75:31:75:42 | "myusername" | HardcodedCredentials.cs:75:31:75:42 | "myusername" | The hard-coded value "myusername" flows to the $@ parameter in $@. | HardcodedCredentials.cs:75:31:75:42 | "myusername" | username | HardcodedCredentials.cs:75:9:75:57 | call to method CreateUser | call to method CreateUser | +| HardcodedCredentials.cs:75:45:75:56 | "mypassword" | HardcodedCredentials.cs:75:45:75:56 | "mypassword" | HardcodedCredentials.cs:75:45:75:56 | "mypassword" | The hard-coded value "mypassword" flows to the $@ parameter in $@. | HardcodedCredentials.cs:75:45:75:56 | "mypassword" | password | HardcodedCredentials.cs:75:9:75:57 | call to method CreateUser | call to method CreateUser | | TestHardcodedCredentials.cs:26:19:26:28 | "username" | TestHardcodedCredentials.cs:26:19:26:28 | "username" | TestHardcodedCredentials.cs:26:19:26:28 | "username" | The hard-coded value "username" flows to the $@ parameter in $@. | TestHardcodedCredentials.cs:26:19:26:28 | "username" | name | TestHardcodedCredentials.cs:24:31:38:13 | object creation of type MembershipUser | object creation of type MembershipUser | diff --git a/csharp/ql/test/query-tests/Security Features/CWE-798/options b/csharp/ql/test/query-tests/Security Features/CWE-798/options index a361e4fde29..cb1a6209e55 100644 --- a/csharp/ql/test/query-tests/Security Features/CWE-798/options +++ b/csharp/ql/test/query-tests/Security Features/CWE-798/options @@ -1,4 +1,5 @@ semmle-extractor-options: /nostdlib /noconfig semmle-extractor-options: --load-sources-from-project:${testdir}/../../../resources/stubs/System.Data.SqlClient/4.8.3/System.Data.SqlClient.csproj semmle-extractor-options: ${testdir}/../../../resources/stubs/System.Web.cs +semmle-extractor-options: --load-sources-from-project:${testdir}/../../../resources/stubs/_frameworks/Microsoft.AspNetCore.App/Microsoft.AspNetCore.App.csproj semmle-extractor-options: ${testdir}/../../../resources/stubs/Microsoft.VisualStudio.TestTools.UnitTesting.cs diff --git a/go/go.mod b/go/go.mod index 516bc228330..5c2de23af33 100644 --- a/go/go.mod +++ b/go/go.mod @@ -3,11 +3,11 @@ module github.com/github/codeql-go go 1.20 require ( - golang.org/x/mod v0.8.0 - golang.org/x/tools v0.6.0 + golang.org/x/mod v0.12.0 + golang.org/x/tools v0.11.1 ) require ( - golang.org/x/sys v0.5.0 // indirect - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + golang.org/x/sys v0.10.0 // indirect + golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect ) diff --git a/go/go.sum b/go/go.sum index d92f42b1d05..062f29a6437 100644 --- a/go/go.sum +++ b/go/go.sum @@ -8,6 +8,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVD golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= @@ -23,6 +25,8 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9w golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -34,7 +38,11 @@ golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.11.1 h1:ojD5zOW8+7dOGzdnNgersm8aPfcDjhMp12UfG93NIMc= +golang.org/x/tools v0.11.1/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= diff --git a/go/ql/lib/semmle/go/dataflow/internal/DataFlowImpl.qll b/go/ql/lib/semmle/go/dataflow/internal/DataFlowImpl.qll index 52dde36a7e8..b95eab3eb01 100644 --- a/go/ql/lib/semmle/go/dataflow/internal/DataFlowImpl.qll +++ b/go/ql/lib/semmle/go/dataflow/internal/DataFlowImpl.qll @@ -1,3 +1,3 @@ private import DataFlowImplSpecific -private import codeql.dataflow.DataFlowImpl +private import codeql.dataflow.internal.DataFlowImpl import MakeImpl diff --git a/go/ql/lib/semmle/go/dataflow/internal/DataFlowImplCommon.qll b/go/ql/lib/semmle/go/dataflow/internal/DataFlowImplCommon.qll index 46d139e8e95..8f8f7b0a36c 100644 --- a/go/ql/lib/semmle/go/dataflow/internal/DataFlowImplCommon.qll +++ b/go/ql/lib/semmle/go/dataflow/internal/DataFlowImplCommon.qll @@ -1,3 +1,3 @@ private import DataFlowImplSpecific -private import codeql.dataflow.DataFlowImplCommon +private import codeql.dataflow.internal.DataFlowImplCommon import MakeImplCommon diff --git a/go/ql/lib/semmle/go/dataflow/internal/DataFlowImplSpecific.qll b/go/ql/lib/semmle/go/dataflow/internal/DataFlowImplSpecific.qll index 5c5ad594521..bc24ff09bfd 100644 --- a/go/ql/lib/semmle/go/dataflow/internal/DataFlowImplSpecific.qll +++ b/go/ql/lib/semmle/go/dataflow/internal/DataFlowImplSpecific.qll @@ -2,7 +2,7 @@ * Provides Go-specific definitions for use in the data flow library. */ -private import codeql.dataflow.DataFlowParameter +private import codeql.dataflow.DataFlow module Private { import DataFlowPrivate @@ -13,7 +13,7 @@ module Public { import DataFlowUtil } -module GoDataFlow implements DataFlowParameter { +module GoDataFlow implements InputSig { import Private import Public diff --git a/go/vendor/golang.org/x/mod/internal/lazyregexp/lazyre.go b/go/vendor/golang.org/x/mod/internal/lazyregexp/lazyre.go index 2681af35af1..150f887e7a4 100644 --- a/go/vendor/golang.org/x/mod/internal/lazyregexp/lazyre.go +++ b/go/vendor/golang.org/x/mod/internal/lazyregexp/lazyre.go @@ -13,7 +13,7 @@ import ( "sync" ) -// Regexp is a wrapper around regexp.Regexp, where the underlying regexp will be +// Regexp is a wrapper around [regexp.Regexp], where the underlying regexp will be // compiled the first time it is needed. type Regexp struct { str string diff --git a/go/vendor/golang.org/x/mod/modfile/print.go b/go/vendor/golang.org/x/mod/modfile/print.go index 524f93022ac..2a0123d4b91 100644 --- a/go/vendor/golang.org/x/mod/modfile/print.go +++ b/go/vendor/golang.org/x/mod/modfile/print.go @@ -16,7 +16,13 @@ import ( func Format(f *FileSyntax) []byte { pr := &printer{} pr.file(f) - return pr.Bytes() + + // remove trailing blank lines + b := pr.Bytes() + for len(b) > 0 && b[len(b)-1] == '\n' && (len(b) == 1 || b[len(b)-2] == '\n') { + b = b[:len(b)-1] + } + return b } // A printer collects the state during printing of a file or expression. @@ -59,7 +65,11 @@ func (p *printer) newline() { } p.trim() - p.printf("\n") + if b := p.Bytes(); len(b) == 0 || (len(b) >= 2 && b[len(b)-1] == '\n' && b[len(b)-2] == '\n') { + // skip the blank line at top of file or after a blank line + } else { + p.printf("\n") + } for i := 0; i < p.margin; i++ { p.printf("\t") } diff --git a/go/vendor/golang.org/x/mod/modfile/read.go b/go/vendor/golang.org/x/mod/modfile/read.go index a503bc2105d..5b5bb5e115b 100644 --- a/go/vendor/golang.org/x/mod/modfile/read.go +++ b/go/vendor/golang.org/x/mod/modfile/read.go @@ -65,7 +65,7 @@ type Comments struct { } // Comment returns the receiver. This isn't useful by itself, but -// a Comments struct is embedded into all the expression +// a [Comments] struct is embedded into all the expression // implementation types, and this gives each of those a Comment // method to satisfy the Expr interface. func (c *Comments) Comment() *Comments { diff --git a/go/vendor/golang.org/x/mod/modfile/rule.go b/go/vendor/golang.org/x/mod/modfile/rule.go index 6bcde8fabe3..930b6c59bc9 100644 --- a/go/vendor/golang.org/x/mod/modfile/rule.go +++ b/go/vendor/golang.org/x/mod/modfile/rule.go @@ -5,17 +5,17 @@ // Package modfile implements a parser and formatter for go.mod files. // // The go.mod syntax is described in -// https://golang.org/cmd/go/#hdr-The_go_mod_file. +// https://pkg.go.dev/cmd/go/#hdr-The_go_mod_file. // -// The Parse and ParseLax functions both parse a go.mod file and return an +// The [Parse] and [ParseLax] functions both parse a go.mod file and return an // abstract syntax tree. ParseLax ignores unknown statements and may be used to // parse go.mod files that may have been developed with newer versions of Go. // -// The File struct returned by Parse and ParseLax represent an abstract -// go.mod file. File has several methods like AddNewRequire and DropReplace -// that can be used to programmatically edit a file. +// The [File] struct returned by Parse and ParseLax represent an abstract +// go.mod file. File has several methods like [File.AddNewRequire] and +// [File.DropReplace] that can be used to programmatically edit a file. // -// The Format function formats a File back to a byte slice which can be +// The [Format] function formats a File back to a byte slice which can be // written to a file. package modfile @@ -35,12 +35,13 @@ import ( // A File is the parsed, interpreted form of a go.mod file. type File struct { - Module *Module - Go *Go - Require []*Require - Exclude []*Exclude - Replace []*Replace - Retract []*Retract + Module *Module + Go *Go + Toolchain *Toolchain + Require []*Require + Exclude []*Exclude + Replace []*Replace + Retract []*Retract Syntax *FileSyntax } @@ -58,6 +59,12 @@ type Go struct { Syntax *Line } +// A Toolchain is the toolchain statement. +type Toolchain struct { + Name string // "go1.21rc1" + Syntax *Line +} + // An Exclude is a single exclude statement. type Exclude struct { Mod module.Version @@ -219,7 +226,7 @@ var dontFixRetract VersionFixer = func(_, vers string) (string, error) { // data is the content of the file. // // fix is an optional function that canonicalizes module versions. -// If fix is nil, all module versions must be canonical (module.CanonicalVersion +// If fix is nil, all module versions must be canonical ([module.CanonicalVersion] // must return the same string). func Parse(file string, data []byte, fix VersionFixer) (*File, error) { return parseToFile(file, data, fix, true) @@ -296,9 +303,13 @@ func parseToFile(file string, data []byte, fix VersionFixer, strict bool) (parse return f, nil } -var GoVersionRE = lazyregexp.New(`^([1-9][0-9]*)\.(0|[1-9][0-9]*)$`) +var GoVersionRE = lazyregexp.New(`^([1-9][0-9]*)\.(0|[1-9][0-9]*)(\.(0|[1-9][0-9]*))?([a-z]+[0-9]+)?$`) var laxGoVersionRE = lazyregexp.New(`^v?(([1-9][0-9]*)\.(0|[1-9][0-9]*))([^0-9].*)$`) +// Toolchains must be named beginning with `go1`, +// like "go1.20.3" or "go1.20.3-gccgo". As a special case, "default" is also permitted. +var ToolchainRE = lazyregexp.New(`^default$|^go1($|\.)`) + func (f *File) add(errs *ErrorList, block *LineBlock, line *Line, verb string, args []string, fix VersionFixer, strict bool) { // If strict is false, this module is a dependency. // We ignore all unknown directives as well as main-module-only @@ -364,6 +375,21 @@ func (f *File) add(errs *ErrorList, block *LineBlock, line *Line, verb string, a f.Go = &Go{Syntax: line} f.Go.Version = args[0] + case "toolchain": + if f.Toolchain != nil { + errorf("repeated toolchain statement") + return + } + if len(args) != 1 { + errorf("toolchain directive expects exactly one argument") + return + } else if strict && !ToolchainRE.MatchString(args[0]) { + errorf("invalid toolchain version '%s': must match format go1.23 or local", args[0]) + return + } + f.Toolchain = &Toolchain{Syntax: line} + f.Toolchain.Name = args[0] + case "module": if f.Module != nil { errorf("repeated module statement") @@ -612,6 +638,22 @@ func (f *WorkFile) add(errs *ErrorList, line *Line, verb string, args []string, f.Go = &Go{Syntax: line} f.Go.Version = args[0] + case "toolchain": + if f.Toolchain != nil { + errorf("repeated toolchain statement") + return + } + if len(args) != 1 { + errorf("toolchain directive expects exactly one argument") + return + } else if !ToolchainRE.MatchString(args[0]) { + errorf("invalid toolchain version '%s': must match format go1.23 or local", args[0]) + return + } + + f.Toolchain = &Toolchain{Syntax: line} + f.Toolchain.Name = args[0] + case "use": if len(args) != 1 { errorf("usage: %s local/dir", verb) @@ -881,7 +923,7 @@ func (f *File) Format() ([]byte, error) { } // Cleanup cleans up the file f after any edit operations. -// To avoid quadratic behavior, modifications like DropRequire +// To avoid quadratic behavior, modifications like [File.DropRequire] // clear the entry but do not remove it from the slice. // Cleanup cleans out all the cleared entries. func (f *File) Cleanup() { @@ -926,7 +968,7 @@ func (f *File) Cleanup() { func (f *File) AddGoStmt(version string) error { if !GoVersionRE.MatchString(version) { - return fmt.Errorf("invalid language version string %q", version) + return fmt.Errorf("invalid language version %q", version) } if f.Go == nil { var hint Expr @@ -944,6 +986,44 @@ func (f *File) AddGoStmt(version string) error { return nil } +// DropGoStmt deletes the go statement from the file. +func (f *File) DropGoStmt() { + if f.Go != nil { + f.Go.Syntax.markRemoved() + f.Go = nil + } +} + +// DropToolchainStmt deletes the toolchain statement from the file. +func (f *File) DropToolchainStmt() { + if f.Toolchain != nil { + f.Toolchain.Syntax.markRemoved() + f.Toolchain = nil + } +} + +func (f *File) AddToolchainStmt(name string) error { + if !ToolchainRE.MatchString(name) { + return fmt.Errorf("invalid toolchain name %q", name) + } + if f.Toolchain == nil { + var hint Expr + if f.Go != nil && f.Go.Syntax != nil { + hint = f.Go.Syntax + } else if f.Module != nil && f.Module.Syntax != nil { + hint = f.Module.Syntax + } + f.Toolchain = &Toolchain{ + Name: name, + Syntax: f.Syntax.addLine(hint, "toolchain", name), + } + } else { + f.Toolchain.Name = name + f.Syntax.updateLine(f.Toolchain.Syntax, "toolchain", name) + } + return nil +} + // AddRequire sets the first require line for path to version vers, // preserving any existing comments for that line and removing all // other lines for path. @@ -995,8 +1075,8 @@ func (f *File) AddNewRequire(path, vers string, indirect bool) { // The requirements in req must specify at most one distinct version for each // module path. // -// If any existing requirements may be removed, the caller should call Cleanup -// after all edits are complete. +// If any existing requirements may be removed, the caller should call +// [File.Cleanup] after all edits are complete. func (f *File) SetRequire(req []*Require) { type elem struct { version string @@ -1387,13 +1467,21 @@ func (f *File) DropRetract(vi VersionInterval) error { func (f *File) SortBlocks() { f.removeDups() // otherwise sorting is unsafe + // semanticSortForExcludeVersionV is the Go version (plus leading "v") at which + // lines in exclude blocks start to use semantic sort instead of lexicographic sort. + // See go.dev/issue/60028. + const semanticSortForExcludeVersionV = "v1.21" + useSemanticSortForExclude := f.Go != nil && semver.Compare("v"+f.Go.Version, semanticSortForExcludeVersionV) >= 0 + for _, stmt := range f.Syntax.Stmt { block, ok := stmt.(*LineBlock) if !ok { continue } less := lineLess - if block.Token[0] == "retract" { + if block.Token[0] == "exclude" && useSemanticSortForExclude { + less = lineExcludeLess + } else if block.Token[0] == "retract" { less = lineRetractLess } sort.SliceStable(block.Line, func(i, j int) bool { @@ -1496,6 +1584,22 @@ func lineLess(li, lj *Line) bool { return len(li.Token) < len(lj.Token) } +// lineExcludeLess reports whether li should be sorted before lj for lines in +// an "exclude" block. +func lineExcludeLess(li, lj *Line) bool { + if len(li.Token) != 2 || len(lj.Token) != 2 { + // Not a known exclude specification. + // Fall back to sorting lexicographically. + return lineLess(li, lj) + } + // An exclude specification has two tokens: ModulePath and Version. + // Compare module path by string order and version by semver rules. + if pi, pj := li.Token[0], lj.Token[0]; pi != pj { + return pi < pj + } + return semver.Compare(li.Token[1], lj.Token[1]) < 0 +} + // lineRetractLess returns whether li should be sorted before lj for lines in // a "retract" block. It treats each line as a version interval. Single versions // are compared as if they were intervals with the same low and high version. diff --git a/go/vendor/golang.org/x/mod/modfile/work.go b/go/vendor/golang.org/x/mod/modfile/work.go index 0c0e521525a..d7b99376ebe 100644 --- a/go/vendor/golang.org/x/mod/modfile/work.go +++ b/go/vendor/golang.org/x/mod/modfile/work.go @@ -12,9 +12,10 @@ import ( // A WorkFile is the parsed, interpreted form of a go.work file. type WorkFile struct { - Go *Go - Use []*Use - Replace []*Replace + Go *Go + Toolchain *Toolchain + Use []*Use + Replace []*Replace Syntax *FileSyntax } @@ -33,7 +34,7 @@ type Use struct { // data is the content of the file. // // fix is an optional function that canonicalizes module versions. -// If fix is nil, all module versions must be canonical (module.CanonicalVersion +// If fix is nil, all module versions must be canonical ([module.CanonicalVersion] // must return the same string). func ParseWork(file string, data []byte, fix VersionFixer) (*WorkFile, error) { fs, err := parse(file, data) @@ -82,7 +83,7 @@ func ParseWork(file string, data []byte, fix VersionFixer) (*WorkFile, error) { } // Cleanup cleans up the file f after any edit operations. -// To avoid quadratic behavior, modifications like DropRequire +// To avoid quadratic behavior, modifications like [WorkFile.DropRequire] // clear the entry but do not remove it from the slice. // Cleanup cleans out all the cleared entries. func (f *WorkFile) Cleanup() { @@ -109,7 +110,7 @@ func (f *WorkFile) Cleanup() { func (f *WorkFile) AddGoStmt(version string) error { if !GoVersionRE.MatchString(version) { - return fmt.Errorf("invalid language version string %q", version) + return fmt.Errorf("invalid language version %q", version) } if f.Go == nil { stmt := &Line{Token: []string{"go", version}} @@ -117,7 +118,7 @@ func (f *WorkFile) AddGoStmt(version string) error { Version: version, Syntax: stmt, } - // Find the first non-comment-only block that's and add + // Find the first non-comment-only block and add // the go statement before it. That will keep file comments at the top. i := 0 for i = 0; i < len(f.Syntax.Stmt); i++ { @@ -133,6 +134,56 @@ func (f *WorkFile) AddGoStmt(version string) error { return nil } +func (f *WorkFile) AddToolchainStmt(name string) error { + if !ToolchainRE.MatchString(name) { + return fmt.Errorf("invalid toolchain name %q", name) + } + if f.Toolchain == nil { + stmt := &Line{Token: []string{"toolchain", name}} + f.Toolchain = &Toolchain{ + Name: name, + Syntax: stmt, + } + // Find the go line and add the toolchain line after it. + // Or else find the first non-comment-only block and add + // the toolchain line before it. That will keep file comments at the top. + i := 0 + for i = 0; i < len(f.Syntax.Stmt); i++ { + if line, ok := f.Syntax.Stmt[i].(*Line); ok && len(line.Token) > 0 && line.Token[0] == "go" { + i++ + goto Found + } + } + for i = 0; i < len(f.Syntax.Stmt); i++ { + if _, ok := f.Syntax.Stmt[i].(*CommentBlock); !ok { + break + } + } + Found: + f.Syntax.Stmt = append(append(f.Syntax.Stmt[:i:i], stmt), f.Syntax.Stmt[i:]...) + } else { + f.Toolchain.Name = name + f.Syntax.updateLine(f.Toolchain.Syntax, "toolchain", name) + } + return nil +} + +// DropGoStmt deletes the go statement from the file. +func (f *WorkFile) DropGoStmt() { + if f.Go != nil { + f.Go.Syntax.markRemoved() + f.Go = nil + } +} + +// DropToolchainStmt deletes the toolchain statement from the file. +func (f *WorkFile) DropToolchainStmt() { + if f.Toolchain != nil { + f.Toolchain.Syntax.markRemoved() + f.Toolchain = nil + } +} + func (f *WorkFile) AddUse(diskPath, modulePath string) error { need := true for _, d := range f.Use { diff --git a/go/vendor/golang.org/x/mod/module/module.go b/go/vendor/golang.org/x/mod/module/module.go index e9dec6e6148..2a364b229b9 100644 --- a/go/vendor/golang.org/x/mod/module/module.go +++ b/go/vendor/golang.org/x/mod/module/module.go @@ -4,7 +4,7 @@ // Package module defines the module.Version type along with support code. // -// The module.Version type is a simple Path, Version pair: +// The [module.Version] type is a simple Path, Version pair: // // type Version struct { // Path string @@ -12,7 +12,7 @@ // } // // There are no restrictions imposed directly by use of this structure, -// but additional checking functions, most notably Check, verify that +// but additional checking functions, most notably [Check], verify that // a particular path, version pair is valid. // // # Escaped Paths @@ -140,7 +140,7 @@ type ModuleError struct { Err error } -// VersionError returns a ModuleError derived from a Version and error, +// VersionError returns a [ModuleError] derived from a [Version] and error, // or err itself if it is already such an error. func VersionError(v Version, err error) error { var mErr *ModuleError @@ -169,7 +169,7 @@ func (e *ModuleError) Unwrap() error { return e.Err } // An InvalidVersionError indicates an error specific to a version, with the // module path unknown or specified externally. // -// A ModuleError may wrap an InvalidVersionError, but an InvalidVersionError +// A [ModuleError] may wrap an InvalidVersionError, but an InvalidVersionError // must not wrap a ModuleError. type InvalidVersionError struct { Version string @@ -193,8 +193,8 @@ func (e *InvalidVersionError) Error() string { func (e *InvalidVersionError) Unwrap() error { return e.Err } // An InvalidPathError indicates a module, import, or file path doesn't -// satisfy all naming constraints. See CheckPath, CheckImportPath, -// and CheckFilePath for specific restrictions. +// satisfy all naming constraints. See [CheckPath], [CheckImportPath], +// and [CheckFilePath] for specific restrictions. type InvalidPathError struct { Kind string // "module", "import", or "file" Path string @@ -294,7 +294,7 @@ func fileNameOK(r rune) bool { } // CheckPath checks that a module path is valid. -// A valid module path is a valid import path, as checked by CheckImportPath, +// A valid module path is a valid import path, as checked by [CheckImportPath], // with three additional constraints. // First, the leading path element (up to the first slash, if any), // by convention a domain name, must contain only lower-case ASCII letters, @@ -380,7 +380,7 @@ const ( // checkPath returns an error describing why the path is not valid. // Because these checks apply to module, import, and file paths, // and because other checks may be applied, the caller is expected to wrap -// this error with InvalidPathError. +// this error with [InvalidPathError]. func checkPath(path string, kind pathKind) error { if !utf8.ValidString(path) { return fmt.Errorf("invalid UTF-8") @@ -532,7 +532,7 @@ var badWindowsNames = []string{ // they require ".vN" instead of "/vN", and for all N, not just N >= 2. // SplitPathVersion returns with ok = false when presented with // a path whose last path element does not satisfy the constraints -// applied by CheckPath, such as "example.com/pkg/v1" or "example.com/pkg/v1.2". +// applied by [CheckPath], such as "example.com/pkg/v1" or "example.com/pkg/v1.2". func SplitPathVersion(path string) (prefix, pathMajor string, ok bool) { if strings.HasPrefix(path, "gopkg.in/") { return splitGopkgIn(path) @@ -582,7 +582,7 @@ func splitGopkgIn(path string) (prefix, pathMajor string, ok bool) { // MatchPathMajor reports whether the semantic version v // matches the path major version pathMajor. // -// MatchPathMajor returns true if and only if CheckPathMajor returns nil. +// MatchPathMajor returns true if and only if [CheckPathMajor] returns nil. func MatchPathMajor(v, pathMajor string) bool { return CheckPathMajor(v, pathMajor) == nil } @@ -622,7 +622,7 @@ func CheckPathMajor(v, pathMajor string) error { // PathMajorPrefix returns the major-version tag prefix implied by pathMajor. // An empty PathMajorPrefix allows either v0 or v1. // -// Note that MatchPathMajor may accept some versions that do not actually begin +// Note that [MatchPathMajor] may accept some versions that do not actually begin // with this prefix: namely, it accepts a 'v0.0.0-' prefix for a '.v1' // pathMajor, even though that pathMajor implies 'v1' tagging. func PathMajorPrefix(pathMajor string) string { @@ -643,7 +643,7 @@ func PathMajorPrefix(pathMajor string) string { } // CanonicalVersion returns the canonical form of the version string v. -// It is the same as semver.Canonical(v) except that it preserves the special build suffix "+incompatible". +// It is the same as [semver.Canonical] except that it preserves the special build suffix "+incompatible". func CanonicalVersion(v string) string { cv := semver.Canonical(v) if semver.Build(v) == "+incompatible" { @@ -652,8 +652,8 @@ func CanonicalVersion(v string) string { return cv } -// Sort sorts the list by Path, breaking ties by comparing Version fields. -// The Version fields are interpreted as semantic versions (using semver.Compare) +// Sort sorts the list by Path, breaking ties by comparing [Version] fields. +// The Version fields are interpreted as semantic versions (using [semver.Compare]) // optionally followed by a tie-breaking suffix introduced by a slash character, // like in "v0.0.1/go.mod". func Sort(list []Version) { @@ -793,7 +793,7 @@ func unescapeString(escaped string) (string, bool) { } // MatchPrefixPatterns reports whether any path prefix of target matches one of -// the glob patterns (as defined by path.Match) in the comma-separated globs +// the glob patterns (as defined by [path.Match]) in the comma-separated globs // list. This implements the algorithm used when matching a module path to the // GOPRIVATE environment variable, as described by 'go help module-private'. // diff --git a/go/vendor/golang.org/x/mod/module/pseudo.go b/go/vendor/golang.org/x/mod/module/pseudo.go index f04ad378869..9cf19d3254e 100644 --- a/go/vendor/golang.org/x/mod/module/pseudo.go +++ b/go/vendor/golang.org/x/mod/module/pseudo.go @@ -125,7 +125,7 @@ func IsPseudoVersion(v string) bool { } // IsZeroPseudoVersion returns whether v is a pseudo-version with a zero base, -// timestamp, and revision, as returned by ZeroPseudoVersion. +// timestamp, and revision, as returned by [ZeroPseudoVersion]. func IsZeroPseudoVersion(v string) bool { return v == ZeroPseudoVersion(semver.Major(v)) } diff --git a/go/vendor/golang.org/x/mod/semver/semver.go b/go/vendor/golang.org/x/mod/semver/semver.go index a30a22bf20f..9a2dfd33a77 100644 --- a/go/vendor/golang.org/x/mod/semver/semver.go +++ b/go/vendor/golang.org/x/mod/semver/semver.go @@ -140,7 +140,7 @@ func Compare(v, w string) int { // Max canonicalizes its arguments and then returns the version string // that compares greater. // -// Deprecated: use Compare instead. In most cases, returning a canonicalized +// Deprecated: use [Compare] instead. In most cases, returning a canonicalized // version is not expected or desired. func Max(v, w string) string { v = Canonical(v) @@ -151,7 +151,7 @@ func Max(v, w string) string { return w } -// ByVersion implements sort.Interface for sorting semantic version strings. +// ByVersion implements [sort.Interface] for sorting semantic version strings. type ByVersion []string func (vs ByVersion) Len() int { return len(vs) } @@ -164,7 +164,7 @@ func (vs ByVersion) Less(i, j int) bool { return vs[i] < vs[j] } -// Sort sorts a list of semantic version strings using ByVersion. +// Sort sorts a list of semantic version strings using [ByVersion]. func Sort(list []string) { sort.Sort(ByVersion(list)) } diff --git a/go/vendor/golang.org/x/sys/execabs/execabs.go b/go/vendor/golang.org/x/sys/execabs/execabs.go index b981cfbb4ae..3bf40fdfecd 100644 --- a/go/vendor/golang.org/x/sys/execabs/execabs.go +++ b/go/vendor/golang.org/x/sys/execabs/execabs.go @@ -63,7 +63,7 @@ func LookPath(file string) (string, error) { } func fixCmd(name string, cmd *exec.Cmd) { - if filepath.Base(name) == name && !filepath.IsAbs(cmd.Path) { + if filepath.Base(name) == name && !filepath.IsAbs(cmd.Path) && !isGo119ErrFieldSet(cmd) { // exec.Command was called with a bare binary name and // exec.LookPath returned a path which is not absolute. // Set cmd.lookPathErr and clear cmd.Path so that it diff --git a/go/vendor/golang.org/x/sys/execabs/execabs_go118.go b/go/vendor/golang.org/x/sys/execabs/execabs_go118.go index 6ab5f50894e..2000064a812 100644 --- a/go/vendor/golang.org/x/sys/execabs/execabs_go118.go +++ b/go/vendor/golang.org/x/sys/execabs/execabs_go118.go @@ -7,6 +7,12 @@ package execabs +import "os/exec" + func isGo119ErrDot(err error) bool { return false } + +func isGo119ErrFieldSet(cmd *exec.Cmd) bool { + return false +} diff --git a/go/vendor/golang.org/x/sys/execabs/execabs_go119.go b/go/vendor/golang.org/x/sys/execabs/execabs_go119.go index 46c5b525e7b..f364b341892 100644 --- a/go/vendor/golang.org/x/sys/execabs/execabs_go119.go +++ b/go/vendor/golang.org/x/sys/execabs/execabs_go119.go @@ -15,3 +15,7 @@ import ( func isGo119ErrDot(err error) bool { return errors.Is(err, exec.ErrDot) } + +func isGo119ErrFieldSet(cmd *exec.Cmd) bool { + return cmd.Err != nil +} diff --git a/go/vendor/golang.org/x/tools/go/gcexportdata/gcexportdata.go b/go/vendor/golang.org/x/tools/go/gcexportdata/gcexportdata.go index 165ede0f8f3..03543bd4bb8 100644 --- a/go/vendor/golang.org/x/tools/go/gcexportdata/gcexportdata.go +++ b/go/vendor/golang.org/x/tools/go/gcexportdata/gcexportdata.go @@ -128,15 +128,14 @@ func Read(in io.Reader, fset *token.FileSet, imports map[string]*types.Package, // (from "version"). Select appropriate importer. if len(data) > 0 { switch data[0] { - case 'i': + case 'v', 'c', 'd': // binary, till go1.10 + return nil, fmt.Errorf("binary (%c) import format is no longer supported", data[0]) + + case 'i': // indexed, till go1.19 _, pkg, err := gcimporter.IImportData(fset, imports, data[1:], path) return pkg, err - case 'v', 'c', 'd': - _, pkg, err := gcimporter.BImportData(fset, imports, data, path) - return pkg, err - - case 'u': + case 'u': // unified, from go1.20 _, pkg, err := gcimporter.UImportData(fset, imports, data[1:], path) return pkg, err diff --git a/go/vendor/golang.org/x/tools/go/packages/golist.go b/go/vendor/golang.org/x/tools/go/packages/golist.go index 6bb7168d2e3..58230038a7c 100644 --- a/go/vendor/golang.org/x/tools/go/packages/golist.go +++ b/go/vendor/golang.org/x/tools/go/packages/golist.go @@ -625,7 +625,12 @@ func (state *golistState) createDriverResponse(words ...string) (*driverResponse } if pkg.PkgPath == "unsafe" { - pkg.GoFiles = nil // ignore fake unsafe.go file + pkg.CompiledGoFiles = nil // ignore fake unsafe.go file (#59929) + } else if len(pkg.CompiledGoFiles) == 0 { + // Work around for pre-go.1.11 versions of go list. + // TODO(matloob): they should be handled by the fallback. + // Can we delete this? + pkg.CompiledGoFiles = pkg.GoFiles } // Assume go list emits only absolute paths for Dir. @@ -663,16 +668,12 @@ func (state *golistState) createDriverResponse(words ...string) (*driverResponse response.Roots = append(response.Roots, pkg.ID) } - // Work around for pre-go.1.11 versions of go list. - // TODO(matloob): they should be handled by the fallback. - // Can we delete this? - if len(pkg.CompiledGoFiles) == 0 { - pkg.CompiledGoFiles = pkg.GoFiles - } - // Temporary work-around for golang/go#39986. Parse filenames out of // error messages. This happens if there are unrecoverable syntax // errors in the source, so we can't match on a specific error message. + // + // TODO(rfindley): remove this heuristic, in favor of considering + // InvalidGoFiles from the list driver. if err := p.Error; err != nil && state.shouldAddFilenameFromError(p) { addFilenameFromPos := func(pos string) bool { split := strings.Split(pos, ":") @@ -891,6 +892,15 @@ func golistargs(cfg *Config, words []string, goVersion int) []string { // probably because you'd just get the TestMain. fmt.Sprintf("-find=%t", !cfg.Tests && cfg.Mode&findFlags == 0 && !usesExportData(cfg)), } + + // golang/go#60456: with go1.21 and later, go list serves pgo variants, which + // can be costly to compute and may result in redundant processing for the + // caller. Disable these variants. If someone wants to add e.g. a NeedPGO + // mode flag, that should be a separate proposal. + if goVersion >= 21 { + fullargs = append(fullargs, "-pgo=off") + } + fullargs = append(fullargs, cfg.BuildFlags...) fullargs = append(fullargs, "--") fullargs = append(fullargs, words...) diff --git a/go/vendor/golang.org/x/tools/go/packages/packages.go b/go/vendor/golang.org/x/tools/go/packages/packages.go index 0f1505b808a..da1a27eea62 100644 --- a/go/vendor/golang.org/x/tools/go/packages/packages.go +++ b/go/vendor/golang.org/x/tools/go/packages/packages.go @@ -308,6 +308,9 @@ type Package struct { TypeErrors []types.Error // GoFiles lists the absolute file paths of the package's Go source files. + // It may include files that should not be compiled, for example because + // they contain non-matching build tags, are documentary pseudo-files such as + // unsafe/unsafe.go or builtin/builtin.go, or are subject to cgo preprocessing. GoFiles []string // CompiledGoFiles lists the absolute file paths of the package's source @@ -627,7 +630,7 @@ func newLoader(cfg *Config) *loader { return ld } -// refine connects the supplied packages into a graph and then adds type and +// refine connects the supplied packages into a graph and then adds type // and syntax information as requested by the LoadMode. func (ld *loader) refine(response *driverResponse) ([]*Package, error) { roots := response.Roots @@ -1040,6 +1043,9 @@ func (ld *loader) loadPackage(lpkg *loaderPackage) { Error: appendError, Sizes: ld.sizes, } + if lpkg.Module != nil && lpkg.Module.GoVersion != "" { + typesinternal.SetGoVersion(tc, "go"+lpkg.Module.GoVersion) + } if (ld.Mode & typecheckCgo) != 0 { if !typesinternal.SetUsesCgo(tc) { appendError(Error{ diff --git a/go/vendor/golang.org/x/tools/go/types/objectpath/objectpath.go b/go/vendor/golang.org/x/tools/go/types/objectpath/objectpath.go new file mode 100644 index 00000000000..c725d839ba1 --- /dev/null +++ b/go/vendor/golang.org/x/tools/go/types/objectpath/objectpath.go @@ -0,0 +1,824 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package objectpath defines a naming scheme for types.Objects +// (that is, named entities in Go programs) relative to their enclosing +// package. +// +// Type-checker objects are canonical, so they are usually identified by +// their address in memory (a pointer), but a pointer has meaning only +// within one address space. By contrast, objectpath names allow the +// identity of an object to be sent from one program to another, +// establishing a correspondence between types.Object variables that are +// distinct but logically equivalent. +// +// A single object may have multiple paths. In this example, +// +// type A struct{ X int } +// type B A +// +// the field X has two paths due to its membership of both A and B. +// The For(obj) function always returns one of these paths, arbitrarily +// but consistently. +package objectpath + +import ( + "fmt" + "go/types" + "sort" + "strconv" + "strings" + _ "unsafe" + + "golang.org/x/tools/internal/typeparams" +) + +// A Path is an opaque name that identifies a types.Object +// relative to its package. Conceptually, the name consists of a +// sequence of destructuring operations applied to the package scope +// to obtain the original object. +// The name does not include the package itself. +type Path string + +// Encoding +// +// An object path is a textual and (with training) human-readable encoding +// of a sequence of destructuring operators, starting from a types.Package. +// The sequences represent a path through the package/object/type graph. +// We classify these operators by their type: +// +// PO package->object Package.Scope.Lookup +// OT object->type Object.Type +// TT type->type Type.{Elem,Key,Params,Results,Underlying} [EKPRU] +// TO type->object Type.{At,Field,Method,Obj} [AFMO] +// +// All valid paths start with a package and end at an object +// and thus may be defined by the regular language: +// +// objectpath = PO (OT TT* TO)* +// +// The concrete encoding follows directly: +// - The only PO operator is Package.Scope.Lookup, which requires an identifier. +// - The only OT operator is Object.Type, +// which we encode as '.' because dot cannot appear in an identifier. +// - The TT operators are encoded as [EKPRUTC]; +// one of these (TypeParam) requires an integer operand, +// which is encoded as a string of decimal digits. +// - The TO operators are encoded as [AFMO]; +// three of these (At,Field,Method) require an integer operand, +// which is encoded as a string of decimal digits. +// These indices are stable across different representations +// of the same package, even source and export data. +// The indices used are implementation specific and may not correspond to +// the argument to the go/types function. +// +// In the example below, +// +// package p +// +// type T interface { +// f() (a string, b struct{ X int }) +// } +// +// field X has the path "T.UM0.RA1.F0", +// representing the following sequence of operations: +// +// p.Lookup("T") T +// .Type().Underlying().Method(0). f +// .Type().Results().At(1) b +// .Type().Field(0) X +// +// The encoding is not maximally compact---every R or P is +// followed by an A, for example---but this simplifies the +// encoder and decoder. +const ( + // object->type operators + opType = '.' // .Type() (Object) + + // type->type operators + opElem = 'E' // .Elem() (Pointer, Slice, Array, Chan, Map) + opKey = 'K' // .Key() (Map) + opParams = 'P' // .Params() (Signature) + opResults = 'R' // .Results() (Signature) + opUnderlying = 'U' // .Underlying() (Named) + opTypeParam = 'T' // .TypeParams.At(i) (Named, Signature) + opConstraint = 'C' // .Constraint() (TypeParam) + + // type->object operators + opAt = 'A' // .At(i) (Tuple) + opField = 'F' // .Field(i) (Struct) + opMethod = 'M' // .Method(i) (Named or Interface; not Struct: "promoted" names are ignored) + opObj = 'O' // .Obj() (Named, TypeParam) +) + +// For is equivalent to new(Encoder).For(obj). +// +// It may be more efficient to reuse a single Encoder across several calls. +func For(obj types.Object) (Path, error) { + return new(Encoder).For(obj) +} + +// An Encoder amortizes the cost of encoding the paths of multiple objects. +// The zero value of an Encoder is ready to use. +type Encoder struct { + scopeMemo map[*types.Scope][]types.Object // memoization of scopeObjects + namedMethodsMemo map[*types.Named][]*types.Func // memoization of namedMethods() + skipMethodSorting bool +} + +// Exposed to gopls via golang.org/x/tools/internal/typesinternal +// TODO(golang/go#61443): eliminate this parameter one way or the other. +// +//go:linkname skipMethodSorting +func skipMethodSorting(enc *Encoder) { + enc.skipMethodSorting = true +} + +// For returns the path to an object relative to its package, +// or an error if the object is not accessible from the package's Scope. +// +// The For function guarantees to return a path only for the following objects: +// - package-level types +// - exported package-level non-types +// - methods +// - parameter and result variables +// - struct fields +// These objects are sufficient to define the API of their package. +// The objects described by a package's export data are drawn from this set. +// +// The set of objects accessible from a package's Scope depends on +// whether the package was produced by type-checking syntax, or +// reading export data; the latter may have a smaller Scope since +// export data trims objects that are not reachable from an exported +// declaration. For example, the For function will return a path for +// an exported method of an unexported type that is not reachable +// from any public declaration; this path will cause the Object +// function to fail if called on a package loaded from export data. +// TODO(adonovan): is this a bug or feature? Should this package +// compute accessibility in the same way? +// +// For does not return a path for predeclared names, imported package +// names, local names, and unexported package-level names (except +// types). +// +// Example: given this definition, +// +// package p +// +// type T interface { +// f() (a string, b struct{ X int }) +// } +// +// For(X) would return a path that denotes the following sequence of operations: +// +// p.Scope().Lookup("T") (TypeName T) +// .Type().Underlying().Method(0). (method Func f) +// .Type().Results().At(1) (field Var b) +// .Type().Field(0) (field Var X) +// +// where p is the package (*types.Package) to which X belongs. +func (enc *Encoder) For(obj types.Object) (Path, error) { + pkg := obj.Pkg() + + // This table lists the cases of interest. + // + // Object Action + // ------ ------ + // nil reject + // builtin reject + // pkgname reject + // label reject + // var + // package-level accept + // func param/result accept + // local reject + // struct field accept + // const + // package-level accept + // local reject + // func + // package-level accept + // init functions reject + // concrete method accept + // interface method accept + // type + // package-level accept + // local reject + // + // The only accessible package-level objects are members of pkg itself. + // + // The cases are handled in four steps: + // + // 1. reject nil and builtin + // 2. accept package-level objects + // 3. reject obviously invalid objects + // 4. search the API for the path to the param/result/field/method. + + // 1. reference to nil or builtin? + if pkg == nil { + return "", fmt.Errorf("predeclared %s has no path", obj) + } + scope := pkg.Scope() + + // 2. package-level object? + if scope.Lookup(obj.Name()) == obj { + // Only exported objects (and non-exported types) have a path. + // Non-exported types may be referenced by other objects. + if _, ok := obj.(*types.TypeName); !ok && !obj.Exported() { + return "", fmt.Errorf("no path for non-exported %v", obj) + } + return Path(obj.Name()), nil + } + + // 3. Not a package-level object. + // Reject obviously non-viable cases. + switch obj := obj.(type) { + case *types.TypeName: + if _, ok := obj.Type().(*typeparams.TypeParam); !ok { + // With the exception of type parameters, only package-level type names + // have a path. + return "", fmt.Errorf("no path for %v", obj) + } + case *types.Const, // Only package-level constants have a path. + *types.Label, // Labels are function-local. + *types.PkgName: // PkgNames are file-local. + return "", fmt.Errorf("no path for %v", obj) + + case *types.Var: + // Could be: + // - a field (obj.IsField()) + // - a func parameter or result + // - a local var. + // Sadly there is no way to distinguish + // a param/result from a local + // so we must proceed to the find. + + case *types.Func: + // A func, if not package-level, must be a method. + if recv := obj.Type().(*types.Signature).Recv(); recv == nil { + return "", fmt.Errorf("func is not a method: %v", obj) + } + + if path, ok := enc.concreteMethod(obj); ok { + // Fast path for concrete methods that avoids looping over scope. + return path, nil + } + + default: + panic(obj) + } + + // 4. Search the API for the path to the var (field/param/result) or method. + + // First inspect package-level named types. + // In the presence of path aliases, these give + // the best paths because non-types may + // refer to types, but not the reverse. + empty := make([]byte, 0, 48) // initial space + objs := enc.scopeObjects(scope) + for _, o := range objs { + tname, ok := o.(*types.TypeName) + if !ok { + continue // handle non-types in second pass + } + + path := append(empty, o.Name()...) + path = append(path, opType) + + T := o.Type() + + if tname.IsAlias() { + // type alias + if r := find(obj, T, path, nil); r != nil { + return Path(r), nil + } + } else { + if named, _ := T.(*types.Named); named != nil { + if r := findTypeParam(obj, typeparams.ForNamed(named), path, nil); r != nil { + // generic named type + return Path(r), nil + } + } + // defined (named) type + if r := find(obj, T.Underlying(), append(path, opUnderlying), nil); r != nil { + return Path(r), nil + } + } + } + + // Then inspect everything else: + // non-types, and declared methods of defined types. + for _, o := range objs { + path := append(empty, o.Name()...) + if _, ok := o.(*types.TypeName); !ok { + if o.Exported() { + // exported non-type (const, var, func) + if r := find(obj, o.Type(), append(path, opType), nil); r != nil { + return Path(r), nil + } + } + continue + } + + // Inspect declared methods of defined types. + if T, ok := o.Type().(*types.Named); ok { + path = append(path, opType) + if !enc.skipMethodSorting { + // Note that method index here is always with respect + // to canonical ordering of methods, regardless of how + // they appear in the underlying type. + for i, m := range enc.namedMethods(T) { + path2 := appendOpArg(path, opMethod, i) + if m == obj { + return Path(path2), nil // found declared method + } + if r := find(obj, m.Type(), append(path2, opType), nil); r != nil { + return Path(r), nil + } + } + } else { + // This branch must match the logic in the branch above, using go/types + // APIs without sorting. + for i := 0; i < T.NumMethods(); i++ { + m := T.Method(i) + path2 := appendOpArg(path, opMethod, i) + if m == obj { + return Path(path2), nil // found declared method + } + if r := find(obj, m.Type(), append(path2, opType), nil); r != nil { + return Path(r), nil + } + } + } + } + } + + return "", fmt.Errorf("can't find path for %v in %s", obj, pkg.Path()) +} + +func appendOpArg(path []byte, op byte, arg int) []byte { + path = append(path, op) + path = strconv.AppendInt(path, int64(arg), 10) + return path +} + +// concreteMethod returns the path for meth, which must have a non-nil receiver. +// The second return value indicates success and may be false if the method is +// an interface method or if it is an instantiated method. +// +// This function is just an optimization that avoids the general scope walking +// approach. You are expected to fall back to the general approach if this +// function fails. +func (enc *Encoder) concreteMethod(meth *types.Func) (Path, bool) { + // Concrete methods can only be declared on package-scoped named types. For + // that reason we can skip the expensive walk over the package scope: the + // path will always be package -> named type -> method. We can trivially get + // the type name from the receiver, and only have to look over the type's + // methods to find the method index. + // + // Methods on generic types require special consideration, however. Consider + // the following package: + // + // L1: type S[T any] struct{} + // L2: func (recv S[A]) Foo() { recv.Bar() } + // L3: func (recv S[B]) Bar() { } + // L4: type Alias = S[int] + // L5: func _[T any]() { var s S[int]; s.Foo() } + // + // The receivers of methods on generic types are instantiations. L2 and L3 + // instantiate S with the type-parameters A and B, which are scoped to the + // respective methods. L4 and L5 each instantiate S with int. Each of these + // instantiations has its own method set, full of methods (and thus objects) + // with receivers whose types are the respective instantiations. In other + // words, we have + // + // S[A].Foo, S[A].Bar + // S[B].Foo, S[B].Bar + // S[int].Foo, S[int].Bar + // + // We may thus be trying to produce object paths for any of these objects. + // + // S[A].Foo and S[B].Bar are the origin methods, and their paths are S.Foo + // and S.Bar, which are the paths that this function naturally produces. + // + // S[A].Bar, S[B].Foo, and both methods on S[int] are instantiations that + // don't correspond to the origin methods. For S[int], this is significant. + // The most precise object path for S[int].Foo, for example, is Alias.Foo, + // not S.Foo. Our function, however, would produce S.Foo, which would + // resolve to a different object. + // + // For S[A].Bar and S[B].Foo it could be argued that S.Bar and S.Foo are + // still the correct paths, since only the origin methods have meaningful + // paths. But this is likely only true for trivial cases and has edge cases. + // Since this function is only an optimization, we err on the side of giving + // up, deferring to the slower but definitely correct algorithm. Most users + // of objectpath will only be giving us origin methods, anyway, as referring + // to instantiated methods is usually not useful. + + if typeparams.OriginMethod(meth) != meth { + return "", false + } + + recvT := meth.Type().(*types.Signature).Recv().Type() + if ptr, ok := recvT.(*types.Pointer); ok { + recvT = ptr.Elem() + } + + named, ok := recvT.(*types.Named) + if !ok { + return "", false + } + + if types.IsInterface(named) { + // Named interfaces don't have to be package-scoped + // + // TODO(dominikh): opt: if scope.Lookup(name) == named, then we can apply this optimization to interface + // methods, too, I think. + return "", false + } + + // Preallocate space for the name, opType, opMethod, and some digits. + name := named.Obj().Name() + path := make([]byte, 0, len(name)+8) + path = append(path, name...) + path = append(path, opType) + + if !enc.skipMethodSorting { + for i, m := range enc.namedMethods(named) { + if m == meth { + path = appendOpArg(path, opMethod, i) + return Path(path), true + } + } + } else { + // This branch must match the logic of the branch above, using go/types + // APIs without sorting. + for i := 0; i < named.NumMethods(); i++ { + m := named.Method(i) + if m == meth { + path = appendOpArg(path, opMethod, i) + return Path(path), true + } + } + } + + // Due to golang/go#59944, go/types fails to associate the receiver with + // certain methods on cgo types. + // + // TODO(rfindley): replace this panic once golang/go#59944 is fixed in all Go + // versions gopls supports. + return "", false + // panic(fmt.Sprintf("couldn't find method %s on type %s; methods: %#v", meth, named, enc.namedMethods(named))) +} + +// find finds obj within type T, returning the path to it, or nil if not found. +// +// The seen map is used to short circuit cycles through type parameters. If +// nil, it will be allocated as necessary. +func find(obj types.Object, T types.Type, path []byte, seen map[*types.TypeName]bool) []byte { + switch T := T.(type) { + case *types.Basic, *types.Named: + // Named types belonging to pkg were handled already, + // so T must belong to another package. No path. + return nil + case *types.Pointer: + return find(obj, T.Elem(), append(path, opElem), seen) + case *types.Slice: + return find(obj, T.Elem(), append(path, opElem), seen) + case *types.Array: + return find(obj, T.Elem(), append(path, opElem), seen) + case *types.Chan: + return find(obj, T.Elem(), append(path, opElem), seen) + case *types.Map: + if r := find(obj, T.Key(), append(path, opKey), seen); r != nil { + return r + } + return find(obj, T.Elem(), append(path, opElem), seen) + case *types.Signature: + if r := findTypeParam(obj, typeparams.ForSignature(T), path, seen); r != nil { + return r + } + if r := find(obj, T.Params(), append(path, opParams), seen); r != nil { + return r + } + return find(obj, T.Results(), append(path, opResults), seen) + case *types.Struct: + for i := 0; i < T.NumFields(); i++ { + fld := T.Field(i) + path2 := appendOpArg(path, opField, i) + if fld == obj { + return path2 // found field var + } + if r := find(obj, fld.Type(), append(path2, opType), seen); r != nil { + return r + } + } + return nil + case *types.Tuple: + for i := 0; i < T.Len(); i++ { + v := T.At(i) + path2 := appendOpArg(path, opAt, i) + if v == obj { + return path2 // found param/result var + } + if r := find(obj, v.Type(), append(path2, opType), seen); r != nil { + return r + } + } + return nil + case *types.Interface: + for i := 0; i < T.NumMethods(); i++ { + m := T.Method(i) + path2 := appendOpArg(path, opMethod, i) + if m == obj { + return path2 // found interface method + } + if r := find(obj, m.Type(), append(path2, opType), seen); r != nil { + return r + } + } + return nil + case *typeparams.TypeParam: + name := T.Obj() + if name == obj { + return append(path, opObj) + } + if seen[name] { + return nil + } + if seen == nil { + seen = make(map[*types.TypeName]bool) + } + seen[name] = true + if r := find(obj, T.Constraint(), append(path, opConstraint), seen); r != nil { + return r + } + return nil + } + panic(T) +} + +func findTypeParam(obj types.Object, list *typeparams.TypeParamList, path []byte, seen map[*types.TypeName]bool) []byte { + for i := 0; i < list.Len(); i++ { + tparam := list.At(i) + path2 := appendOpArg(path, opTypeParam, i) + if r := find(obj, tparam, path2, seen); r != nil { + return r + } + } + return nil +} + +// Object returns the object denoted by path p within the package pkg. +func Object(pkg *types.Package, p Path) (types.Object, error) { + return object(pkg, p, false) +} + +// Note: the skipMethodSorting parameter must match the value of +// Encoder.skipMethodSorting used during encoding. +func object(pkg *types.Package, p Path, skipMethodSorting bool) (types.Object, error) { + if p == "" { + return nil, fmt.Errorf("empty path") + } + + pathstr := string(p) + var pkgobj, suffix string + if dot := strings.IndexByte(pathstr, opType); dot < 0 { + pkgobj = pathstr + } else { + pkgobj = pathstr[:dot] + suffix = pathstr[dot:] // suffix starts with "." + } + + obj := pkg.Scope().Lookup(pkgobj) + if obj == nil { + return nil, fmt.Errorf("package %s does not contain %q", pkg.Path(), pkgobj) + } + + // abstraction of *types.{Pointer,Slice,Array,Chan,Map} + type hasElem interface { + Elem() types.Type + } + // abstraction of *types.{Named,Signature} + type hasTypeParams interface { + TypeParams() *typeparams.TypeParamList + } + // abstraction of *types.{Named,TypeParam} + type hasObj interface { + Obj() *types.TypeName + } + + // The loop state is the pair (t, obj), + // exactly one of which is non-nil, initially obj. + // All suffixes start with '.' (the only object->type operation), + // followed by optional type->type operations, + // then a type->object operation. + // The cycle then repeats. + var t types.Type + for suffix != "" { + code := suffix[0] + suffix = suffix[1:] + + // Codes [AFM] have an integer operand. + var index int + switch code { + case opAt, opField, opMethod, opTypeParam: + rest := strings.TrimLeft(suffix, "0123456789") + numerals := suffix[:len(suffix)-len(rest)] + suffix = rest + i, err := strconv.Atoi(numerals) + if err != nil { + return nil, fmt.Errorf("invalid path: bad numeric operand %q for code %q", numerals, code) + } + index = int(i) + case opObj: + // no operand + default: + // The suffix must end with a type->object operation. + if suffix == "" { + return nil, fmt.Errorf("invalid path: ends with %q, want [AFMO]", code) + } + } + + if code == opType { + if t != nil { + return nil, fmt.Errorf("invalid path: unexpected %q in type context", opType) + } + t = obj.Type() + obj = nil + continue + } + + if t == nil { + return nil, fmt.Errorf("invalid path: code %q in object context", code) + } + + // Inv: t != nil, obj == nil + + switch code { + case opElem: + hasElem, ok := t.(hasElem) // Pointer, Slice, Array, Chan, Map + if !ok { + return nil, fmt.Errorf("cannot apply %q to %s (got %T, want pointer, slice, array, chan or map)", code, t, t) + } + t = hasElem.Elem() + + case opKey: + mapType, ok := t.(*types.Map) + if !ok { + return nil, fmt.Errorf("cannot apply %q to %s (got %T, want map)", code, t, t) + } + t = mapType.Key() + + case opParams: + sig, ok := t.(*types.Signature) + if !ok { + return nil, fmt.Errorf("cannot apply %q to %s (got %T, want signature)", code, t, t) + } + t = sig.Params() + + case opResults: + sig, ok := t.(*types.Signature) + if !ok { + return nil, fmt.Errorf("cannot apply %q to %s (got %T, want signature)", code, t, t) + } + t = sig.Results() + + case opUnderlying: + named, ok := t.(*types.Named) + if !ok { + return nil, fmt.Errorf("cannot apply %q to %s (got %T, want named)", code, t, t) + } + t = named.Underlying() + + case opTypeParam: + hasTypeParams, ok := t.(hasTypeParams) // Named, Signature + if !ok { + return nil, fmt.Errorf("cannot apply %q to %s (got %T, want named or signature)", code, t, t) + } + tparams := hasTypeParams.TypeParams() + if n := tparams.Len(); index >= n { + return nil, fmt.Errorf("tuple index %d out of range [0-%d)", index, n) + } + t = tparams.At(index) + + case opConstraint: + tparam, ok := t.(*typeparams.TypeParam) + if !ok { + return nil, fmt.Errorf("cannot apply %q to %s (got %T, want type parameter)", code, t, t) + } + t = tparam.Constraint() + + case opAt: + tuple, ok := t.(*types.Tuple) + if !ok { + return nil, fmt.Errorf("cannot apply %q to %s (got %T, want tuple)", code, t, t) + } + if n := tuple.Len(); index >= n { + return nil, fmt.Errorf("tuple index %d out of range [0-%d)", index, n) + } + obj = tuple.At(index) + t = nil + + case opField: + structType, ok := t.(*types.Struct) + if !ok { + return nil, fmt.Errorf("cannot apply %q to %s (got %T, want struct)", code, t, t) + } + if n := structType.NumFields(); index >= n { + return nil, fmt.Errorf("field index %d out of range [0-%d)", index, n) + } + obj = structType.Field(index) + t = nil + + case opMethod: + switch t := t.(type) { + case *types.Interface: + if index >= t.NumMethods() { + return nil, fmt.Errorf("method index %d out of range [0-%d)", index, t.NumMethods()) + } + obj = t.Method(index) // Id-ordered + + case *types.Named: + if index >= t.NumMethods() { + return nil, fmt.Errorf("method index %d out of range [0-%d)", index, t.NumMethods()) + } + if skipMethodSorting { + obj = t.Method(index) + } else { + methods := namedMethods(t) // (unmemoized) + obj = methods[index] // Id-ordered + } + + default: + return nil, fmt.Errorf("cannot apply %q to %s (got %T, want interface or named)", code, t, t) + } + t = nil + + case opObj: + hasObj, ok := t.(hasObj) + if !ok { + return nil, fmt.Errorf("cannot apply %q to %s (got %T, want named or type param)", code, t, t) + } + obj = hasObj.Obj() + t = nil + + default: + return nil, fmt.Errorf("invalid path: unknown code %q", code) + } + } + + if obj.Pkg() != pkg { + return nil, fmt.Errorf("path denotes %s, which belongs to a different package", obj) + } + + return obj, nil // success +} + +// namedMethods returns the methods of a Named type in ascending Id order. +func namedMethods(named *types.Named) []*types.Func { + methods := make([]*types.Func, named.NumMethods()) + for i := range methods { + methods[i] = named.Method(i) + } + sort.Slice(methods, func(i, j int) bool { + return methods[i].Id() < methods[j].Id() + }) + return methods +} + +// namedMethods is a memoization of the namedMethods function. Callers must not modify the result. +func (enc *Encoder) namedMethods(named *types.Named) []*types.Func { + m := enc.namedMethodsMemo + if m == nil { + m = make(map[*types.Named][]*types.Func) + enc.namedMethodsMemo = m + } + methods, ok := m[named] + if !ok { + methods = namedMethods(named) // allocates and sorts + m[named] = methods + } + return methods +} + +// scopeObjects is a memoization of scope objects. +// Callers must not modify the result. +func (enc *Encoder) scopeObjects(scope *types.Scope) []types.Object { + m := enc.scopeMemo + if m == nil { + m = make(map[*types.Scope][]types.Object) + enc.scopeMemo = m + } + objs, ok := m[scope] + if !ok { + names := scope.Names() // allocates and sorts + objs = make([]types.Object, len(names)) + for i, name := range names { + objs[i] = scope.Lookup(name) + } + m[scope] = objs + } + return objs +} diff --git a/go/vendor/golang.org/x/tools/internal/event/tag/tag.go b/go/vendor/golang.org/x/tools/internal/event/tag/tag.go new file mode 100644 index 00000000000..581b26c2041 --- /dev/null +++ b/go/vendor/golang.org/x/tools/internal/event/tag/tag.go @@ -0,0 +1,59 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package tag provides the labels used for telemetry throughout gopls. +package tag + +import ( + "golang.org/x/tools/internal/event/keys" +) + +var ( + // create the label keys we use + Method = keys.NewString("method", "") + StatusCode = keys.NewString("status.code", "") + StatusMessage = keys.NewString("status.message", "") + RPCID = keys.NewString("id", "") + RPCDirection = keys.NewString("direction", "") + File = keys.NewString("file", "") + Directory = keys.New("directory", "") + URI = keys.New("URI", "") + Package = keys.NewString("package", "") // sorted comma-separated list of Package IDs + PackagePath = keys.NewString("package_path", "") + Query = keys.New("query", "") + Snapshot = keys.NewUInt64("snapshot", "") + Operation = keys.NewString("operation", "") + + Position = keys.New("position", "") + Category = keys.NewString("category", "") + PackageCount = keys.NewInt("packages", "") + Files = keys.New("files", "") + Port = keys.NewInt("port", "") + Type = keys.New("type", "") + HoverKind = keys.NewString("hoverkind", "") + + NewServer = keys.NewString("new_server", "A new server was added") + EndServer = keys.NewString("end_server", "A server was shut down") + + ServerID = keys.NewString("server", "The server ID an event is related to") + Logfile = keys.NewString("logfile", "") + DebugAddress = keys.NewString("debug_address", "") + GoplsPath = keys.NewString("gopls_path", "") + ClientID = keys.NewString("client_id", "") + + Level = keys.NewInt("level", "The logging level") +) + +var ( + // create the stats we measure + Started = keys.NewInt64("started", "Count of started RPCs.") + ReceivedBytes = keys.NewInt64("received_bytes", "Bytes received.") //, unit.Bytes) + SentBytes = keys.NewInt64("sent_bytes", "Bytes sent.") //, unit.Bytes) + Latency = keys.NewFloat64("latency_ms", "Elapsed time in milliseconds") //, unit.Milliseconds) +) + +const ( + Inbound = "in" + Outbound = "out" +) diff --git a/go/vendor/golang.org/x/tools/internal/gcimporter/bexport.go b/go/vendor/golang.org/x/tools/internal/gcimporter/bexport.go deleted file mode 100644 index 30582ed6d3d..00000000000 --- a/go/vendor/golang.org/x/tools/internal/gcimporter/bexport.go +++ /dev/null @@ -1,852 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Binary package export. -// This file was derived from $GOROOT/src/cmd/compile/internal/gc/bexport.go; -// see that file for specification of the format. - -package gcimporter - -import ( - "bytes" - "encoding/binary" - "fmt" - "go/constant" - "go/token" - "go/types" - "math" - "math/big" - "sort" - "strings" -) - -// If debugFormat is set, each integer and string value is preceded by a marker -// and position information in the encoding. This mechanism permits an importer -// to recognize immediately when it is out of sync. The importer recognizes this -// mode automatically (i.e., it can import export data produced with debugging -// support even if debugFormat is not set at the time of import). This mode will -// lead to massively larger export data (by a factor of 2 to 3) and should only -// be enabled during development and debugging. -// -// NOTE: This flag is the first flag to enable if importing dies because of -// (suspected) format errors, and whenever a change is made to the format. -const debugFormat = false // default: false - -// Current export format version. Increase with each format change. -// -// Note: The latest binary (non-indexed) export format is at version 6. -// This exporter is still at level 4, but it doesn't matter since -// the binary importer can handle older versions just fine. -// -// 6: package height (CL 105038) -- NOT IMPLEMENTED HERE -// 5: improved position encoding efficiency (issue 20080, CL 41619) -- NOT IMPLEMENTED HERE -// 4: type name objects support type aliases, uses aliasTag -// 3: Go1.8 encoding (same as version 2, aliasTag defined but never used) -// 2: removed unused bool in ODCL export (compiler only) -// 1: header format change (more regular), export package for _ struct fields -// 0: Go1.7 encoding -const exportVersion = 4 - -// trackAllTypes enables cycle tracking for all types, not just named -// types. The existing compiler invariants assume that unnamed types -// that are not completely set up are not used, or else there are spurious -// errors. -// If disabled, only named types are tracked, possibly leading to slightly -// less efficient encoding in rare cases. It also prevents the export of -// some corner-case type declarations (but those are not handled correctly -// with with the textual export format either). -// TODO(gri) enable and remove once issues caused by it are fixed -const trackAllTypes = false - -type exporter struct { - fset *token.FileSet - out bytes.Buffer - - // object -> index maps, indexed in order of serialization - strIndex map[string]int - pkgIndex map[*types.Package]int - typIndex map[types.Type]int - - // position encoding - posInfoFormat bool - prevFile string - prevLine int - - // debugging support - written int // bytes written - indent int // for trace -} - -// internalError represents an error generated inside this package. -type internalError string - -func (e internalError) Error() string { return "gcimporter: " + string(e) } - -func internalErrorf(format string, args ...interface{}) error { - return internalError(fmt.Sprintf(format, args...)) -} - -// BExportData returns binary export data for pkg. -// If no file set is provided, position info will be missing. -func BExportData(fset *token.FileSet, pkg *types.Package) (b []byte, err error) { - if !debug { - defer func() { - if e := recover(); e != nil { - if ierr, ok := e.(internalError); ok { - err = ierr - return - } - // Not an internal error; panic again. - panic(e) - } - }() - } - - p := exporter{ - fset: fset, - strIndex: map[string]int{"": 0}, // empty string is mapped to 0 - pkgIndex: make(map[*types.Package]int), - typIndex: make(map[types.Type]int), - posInfoFormat: true, // TODO(gri) might become a flag, eventually - } - - // write version info - // The version string must start with "version %d" where %d is the version - // number. Additional debugging information may follow after a blank; that - // text is ignored by the importer. - p.rawStringln(fmt.Sprintf("version %d", exportVersion)) - var debug string - if debugFormat { - debug = "debug" - } - p.rawStringln(debug) // cannot use p.bool since it's affected by debugFormat; also want to see this clearly - p.bool(trackAllTypes) - p.bool(p.posInfoFormat) - - // --- generic export data --- - - // populate type map with predeclared "known" types - for index, typ := range predeclared() { - p.typIndex[typ] = index - } - if len(p.typIndex) != len(predeclared()) { - return nil, internalError("duplicate entries in type map?") - } - - // write package data - p.pkg(pkg, true) - if trace { - p.tracef("\n") - } - - // write objects - objcount := 0 - scope := pkg.Scope() - for _, name := range scope.Names() { - if !token.IsExported(name) { - continue - } - if trace { - p.tracef("\n") - } - p.obj(scope.Lookup(name)) - objcount++ - } - - // indicate end of list - if trace { - p.tracef("\n") - } - p.tag(endTag) - - // for self-verification only (redundant) - p.int(objcount) - - if trace { - p.tracef("\n") - } - - // --- end of export data --- - - return p.out.Bytes(), nil -} - -func (p *exporter) pkg(pkg *types.Package, emptypath bool) { - if pkg == nil { - panic(internalError("unexpected nil pkg")) - } - - // if we saw the package before, write its index (>= 0) - if i, ok := p.pkgIndex[pkg]; ok { - p.index('P', i) - return - } - - // otherwise, remember the package, write the package tag (< 0) and package data - if trace { - p.tracef("P%d = { ", len(p.pkgIndex)) - defer p.tracef("} ") - } - p.pkgIndex[pkg] = len(p.pkgIndex) - - p.tag(packageTag) - p.string(pkg.Name()) - if emptypath { - p.string("") - } else { - p.string(pkg.Path()) - } -} - -func (p *exporter) obj(obj types.Object) { - switch obj := obj.(type) { - case *types.Const: - p.tag(constTag) - p.pos(obj) - p.qualifiedName(obj) - p.typ(obj.Type()) - p.value(obj.Val()) - - case *types.TypeName: - if obj.IsAlias() { - p.tag(aliasTag) - p.pos(obj) - p.qualifiedName(obj) - } else { - p.tag(typeTag) - } - p.typ(obj.Type()) - - case *types.Var: - p.tag(varTag) - p.pos(obj) - p.qualifiedName(obj) - p.typ(obj.Type()) - - case *types.Func: - p.tag(funcTag) - p.pos(obj) - p.qualifiedName(obj) - sig := obj.Type().(*types.Signature) - p.paramList(sig.Params(), sig.Variadic()) - p.paramList(sig.Results(), false) - - default: - panic(internalErrorf("unexpected object %v (%T)", obj, obj)) - } -} - -func (p *exporter) pos(obj types.Object) { - if !p.posInfoFormat { - return - } - - file, line := p.fileLine(obj) - if file == p.prevFile { - // common case: write line delta - // delta == 0 means different file or no line change - delta := line - p.prevLine - p.int(delta) - if delta == 0 { - p.int(-1) // -1 means no file change - } - } else { - // different file - p.int(0) - // Encode filename as length of common prefix with previous - // filename, followed by (possibly empty) suffix. Filenames - // frequently share path prefixes, so this can save a lot - // of space and make export data size less dependent on file - // path length. The suffix is unlikely to be empty because - // file names tend to end in ".go". - n := commonPrefixLen(p.prevFile, file) - p.int(n) // n >= 0 - p.string(file[n:]) // write suffix only - p.prevFile = file - p.int(line) - } - p.prevLine = line -} - -func (p *exporter) fileLine(obj types.Object) (file string, line int) { - if p.fset != nil { - pos := p.fset.Position(obj.Pos()) - file = pos.Filename - line = pos.Line - } - return -} - -func commonPrefixLen(a, b string) int { - if len(a) > len(b) { - a, b = b, a - } - // len(a) <= len(b) - i := 0 - for i < len(a) && a[i] == b[i] { - i++ - } - return i -} - -func (p *exporter) qualifiedName(obj types.Object) { - p.string(obj.Name()) - p.pkg(obj.Pkg(), false) -} - -func (p *exporter) typ(t types.Type) { - if t == nil { - panic(internalError("nil type")) - } - - // Possible optimization: Anonymous pointer types *T where - // T is a named type are common. We could canonicalize all - // such types *T to a single type PT = *T. This would lead - // to at most one *T entry in typIndex, and all future *T's - // would be encoded as the respective index directly. Would - // save 1 byte (pointerTag) per *T and reduce the typIndex - // size (at the cost of a canonicalization map). We can do - // this later, without encoding format change. - - // if we saw the type before, write its index (>= 0) - if i, ok := p.typIndex[t]; ok { - p.index('T', i) - return - } - - // otherwise, remember the type, write the type tag (< 0) and type data - if trackAllTypes { - if trace { - p.tracef("T%d = {>\n", len(p.typIndex)) - defer p.tracef("<\n} ") - } - p.typIndex[t] = len(p.typIndex) - } - - switch t := t.(type) { - case *types.Named: - if !trackAllTypes { - // if we don't track all types, track named types now - p.typIndex[t] = len(p.typIndex) - } - - p.tag(namedTag) - p.pos(t.Obj()) - p.qualifiedName(t.Obj()) - p.typ(t.Underlying()) - if !types.IsInterface(t) { - p.assocMethods(t) - } - - case *types.Array: - p.tag(arrayTag) - p.int64(t.Len()) - p.typ(t.Elem()) - - case *types.Slice: - p.tag(sliceTag) - p.typ(t.Elem()) - - case *dddSlice: - p.tag(dddTag) - p.typ(t.elem) - - case *types.Struct: - p.tag(structTag) - p.fieldList(t) - - case *types.Pointer: - p.tag(pointerTag) - p.typ(t.Elem()) - - case *types.Signature: - p.tag(signatureTag) - p.paramList(t.Params(), t.Variadic()) - p.paramList(t.Results(), false) - - case *types.Interface: - p.tag(interfaceTag) - p.iface(t) - - case *types.Map: - p.tag(mapTag) - p.typ(t.Key()) - p.typ(t.Elem()) - - case *types.Chan: - p.tag(chanTag) - p.int(int(3 - t.Dir())) // hack - p.typ(t.Elem()) - - default: - panic(internalErrorf("unexpected type %T: %s", t, t)) - } -} - -func (p *exporter) assocMethods(named *types.Named) { - // Sort methods (for determinism). - var methods []*types.Func - for i := 0; i < named.NumMethods(); i++ { - methods = append(methods, named.Method(i)) - } - sort.Sort(methodsByName(methods)) - - p.int(len(methods)) - - if trace && methods != nil { - p.tracef("associated methods {>\n") - } - - for i, m := range methods { - if trace && i > 0 { - p.tracef("\n") - } - - p.pos(m) - name := m.Name() - p.string(name) - if !exported(name) { - p.pkg(m.Pkg(), false) - } - - sig := m.Type().(*types.Signature) - p.paramList(types.NewTuple(sig.Recv()), false) - p.paramList(sig.Params(), sig.Variadic()) - p.paramList(sig.Results(), false) - p.int(0) // dummy value for go:nointerface pragma - ignored by importer - } - - if trace && methods != nil { - p.tracef("<\n} ") - } -} - -type methodsByName []*types.Func - -func (x methodsByName) Len() int { return len(x) } -func (x methodsByName) Swap(i, j int) { x[i], x[j] = x[j], x[i] } -func (x methodsByName) Less(i, j int) bool { return x[i].Name() < x[j].Name() } - -func (p *exporter) fieldList(t *types.Struct) { - if trace && t.NumFields() > 0 { - p.tracef("fields {>\n") - defer p.tracef("<\n} ") - } - - p.int(t.NumFields()) - for i := 0; i < t.NumFields(); i++ { - if trace && i > 0 { - p.tracef("\n") - } - p.field(t.Field(i)) - p.string(t.Tag(i)) - } -} - -func (p *exporter) field(f *types.Var) { - if !f.IsField() { - panic(internalError("field expected")) - } - - p.pos(f) - p.fieldName(f) - p.typ(f.Type()) -} - -func (p *exporter) iface(t *types.Interface) { - // TODO(gri): enable importer to load embedded interfaces, - // then emit Embeddeds and ExplicitMethods separately here. - p.int(0) - - n := t.NumMethods() - if trace && n > 0 { - p.tracef("methods {>\n") - defer p.tracef("<\n} ") - } - p.int(n) - for i := 0; i < n; i++ { - if trace && i > 0 { - p.tracef("\n") - } - p.method(t.Method(i)) - } -} - -func (p *exporter) method(m *types.Func) { - sig := m.Type().(*types.Signature) - if sig.Recv() == nil { - panic(internalError("method expected")) - } - - p.pos(m) - p.string(m.Name()) - if m.Name() != "_" && !token.IsExported(m.Name()) { - p.pkg(m.Pkg(), false) - } - - // interface method; no need to encode receiver. - p.paramList(sig.Params(), sig.Variadic()) - p.paramList(sig.Results(), false) -} - -func (p *exporter) fieldName(f *types.Var) { - name := f.Name() - - if f.Anonymous() { - // anonymous field - we distinguish between 3 cases: - // 1) field name matches base type name and is exported - // 2) field name matches base type name and is not exported - // 3) field name doesn't match base type name (alias name) - bname := basetypeName(f.Type()) - if name == bname { - if token.IsExported(name) { - name = "" // 1) we don't need to know the field name or package - } else { - name = "?" // 2) use unexported name "?" to force package export - } - } else { - // 3) indicate alias and export name as is - // (this requires an extra "@" but this is a rare case) - p.string("@") - } - } - - p.string(name) - if name != "" && !token.IsExported(name) { - p.pkg(f.Pkg(), false) - } -} - -func basetypeName(typ types.Type) string { - switch typ := deref(typ).(type) { - case *types.Basic: - return typ.Name() - case *types.Named: - return typ.Obj().Name() - default: - return "" // unnamed type - } -} - -func (p *exporter) paramList(params *types.Tuple, variadic bool) { - // use negative length to indicate unnamed parameters - // (look at the first parameter only since either all - // names are present or all are absent) - n := params.Len() - if n > 0 && params.At(0).Name() == "" { - n = -n - } - p.int(n) - for i := 0; i < params.Len(); i++ { - q := params.At(i) - t := q.Type() - if variadic && i == params.Len()-1 { - t = &dddSlice{t.(*types.Slice).Elem()} - } - p.typ(t) - if n > 0 { - name := q.Name() - p.string(name) - if name != "_" { - p.pkg(q.Pkg(), false) - } - } - p.string("") // no compiler-specific info - } -} - -func (p *exporter) value(x constant.Value) { - if trace { - p.tracef("= ") - } - - switch x.Kind() { - case constant.Bool: - tag := falseTag - if constant.BoolVal(x) { - tag = trueTag - } - p.tag(tag) - - case constant.Int: - if v, exact := constant.Int64Val(x); exact { - // common case: x fits into an int64 - use compact encoding - p.tag(int64Tag) - p.int64(v) - return - } - // uncommon case: large x - use float encoding - // (powers of 2 will be encoded efficiently with exponent) - p.tag(floatTag) - p.float(constant.ToFloat(x)) - - case constant.Float: - p.tag(floatTag) - p.float(x) - - case constant.Complex: - p.tag(complexTag) - p.float(constant.Real(x)) - p.float(constant.Imag(x)) - - case constant.String: - p.tag(stringTag) - p.string(constant.StringVal(x)) - - case constant.Unknown: - // package contains type errors - p.tag(unknownTag) - - default: - panic(internalErrorf("unexpected value %v (%T)", x, x)) - } -} - -func (p *exporter) float(x constant.Value) { - if x.Kind() != constant.Float { - panic(internalErrorf("unexpected constant %v, want float", x)) - } - // extract sign (there is no -0) - sign := constant.Sign(x) - if sign == 0 { - // x == 0 - p.int(0) - return - } - // x != 0 - - var f big.Float - if v, exact := constant.Float64Val(x); exact { - // float64 - f.SetFloat64(v) - } else if num, denom := constant.Num(x), constant.Denom(x); num.Kind() == constant.Int { - // TODO(gri): add big.Rat accessor to constant.Value. - r := valueToRat(num) - f.SetRat(r.Quo(r, valueToRat(denom))) - } else { - // Value too large to represent as a fraction => inaccessible. - // TODO(gri): add big.Float accessor to constant.Value. - f.SetFloat64(math.MaxFloat64) // FIXME - } - - // extract exponent such that 0.5 <= m < 1.0 - var m big.Float - exp := f.MantExp(&m) - - // extract mantissa as *big.Int - // - set exponent large enough so mant satisfies mant.IsInt() - // - get *big.Int from mant - m.SetMantExp(&m, int(m.MinPrec())) - mant, acc := m.Int(nil) - if acc != big.Exact { - panic(internalError("internal error")) - } - - p.int(sign) - p.int(exp) - p.string(string(mant.Bytes())) -} - -func valueToRat(x constant.Value) *big.Rat { - // Convert little-endian to big-endian. - // I can't believe this is necessary. - bytes := constant.Bytes(x) - for i := 0; i < len(bytes)/2; i++ { - bytes[i], bytes[len(bytes)-1-i] = bytes[len(bytes)-1-i], bytes[i] - } - return new(big.Rat).SetInt(new(big.Int).SetBytes(bytes)) -} - -func (p *exporter) bool(b bool) bool { - if trace { - p.tracef("[") - defer p.tracef("= %v] ", b) - } - - x := 0 - if b { - x = 1 - } - p.int(x) - return b -} - -// ---------------------------------------------------------------------------- -// Low-level encoders - -func (p *exporter) index(marker byte, index int) { - if index < 0 { - panic(internalError("invalid index < 0")) - } - if debugFormat { - p.marker('t') - } - if trace { - p.tracef("%c%d ", marker, index) - } - p.rawInt64(int64(index)) -} - -func (p *exporter) tag(tag int) { - if tag >= 0 { - panic(internalError("invalid tag >= 0")) - } - if debugFormat { - p.marker('t') - } - if trace { - p.tracef("%s ", tagString[-tag]) - } - p.rawInt64(int64(tag)) -} - -func (p *exporter) int(x int) { - p.int64(int64(x)) -} - -func (p *exporter) int64(x int64) { - if debugFormat { - p.marker('i') - } - if trace { - p.tracef("%d ", x) - } - p.rawInt64(x) -} - -func (p *exporter) string(s string) { - if debugFormat { - p.marker('s') - } - if trace { - p.tracef("%q ", s) - } - // if we saw the string before, write its index (>= 0) - // (the empty string is mapped to 0) - if i, ok := p.strIndex[s]; ok { - p.rawInt64(int64(i)) - return - } - // otherwise, remember string and write its negative length and bytes - p.strIndex[s] = len(p.strIndex) - p.rawInt64(-int64(len(s))) - for i := 0; i < len(s); i++ { - p.rawByte(s[i]) - } -} - -// marker emits a marker byte and position information which makes -// it easy for a reader to detect if it is "out of sync". Used for -// debugFormat format only. -func (p *exporter) marker(m byte) { - p.rawByte(m) - // Enable this for help tracking down the location - // of an incorrect marker when running in debugFormat. - if false && trace { - p.tracef("#%d ", p.written) - } - p.rawInt64(int64(p.written)) -} - -// rawInt64 should only be used by low-level encoders. -func (p *exporter) rawInt64(x int64) { - var tmp [binary.MaxVarintLen64]byte - n := binary.PutVarint(tmp[:], x) - for i := 0; i < n; i++ { - p.rawByte(tmp[i]) - } -} - -// rawStringln should only be used to emit the initial version string. -func (p *exporter) rawStringln(s string) { - for i := 0; i < len(s); i++ { - p.rawByte(s[i]) - } - p.rawByte('\n') -} - -// rawByte is the bottleneck interface to write to p.out. -// rawByte escapes b as follows (any encoding does that -// hides '$'): -// -// '$' => '|' 'S' -// '|' => '|' '|' -// -// Necessary so other tools can find the end of the -// export data by searching for "$$". -// rawByte should only be used by low-level encoders. -func (p *exporter) rawByte(b byte) { - switch b { - case '$': - // write '$' as '|' 'S' - b = 'S' - fallthrough - case '|': - // write '|' as '|' '|' - p.out.WriteByte('|') - p.written++ - } - p.out.WriteByte(b) - p.written++ -} - -// tracef is like fmt.Printf but it rewrites the format string -// to take care of indentation. -func (p *exporter) tracef(format string, args ...interface{}) { - if strings.ContainsAny(format, "<>\n") { - var buf bytes.Buffer - for i := 0; i < len(format); i++ { - // no need to deal with runes - ch := format[i] - switch ch { - case '>': - p.indent++ - continue - case '<': - p.indent-- - continue - } - buf.WriteByte(ch) - if ch == '\n' { - for j := p.indent; j > 0; j-- { - buf.WriteString(". ") - } - } - } - format = buf.String() - } - fmt.Printf(format, args...) -} - -// Debugging support. -// (tagString is only used when tracing is enabled) -var tagString = [...]string{ - // Packages - -packageTag: "package", - - // Types - -namedTag: "named type", - -arrayTag: "array", - -sliceTag: "slice", - -dddTag: "ddd", - -structTag: "struct", - -pointerTag: "pointer", - -signatureTag: "signature", - -interfaceTag: "interface", - -mapTag: "map", - -chanTag: "chan", - - // Values - -falseTag: "false", - -trueTag: "true", - -int64Tag: "int64", - -floatTag: "float", - -fractionTag: "fraction", - -complexTag: "complex", - -stringTag: "string", - -unknownTag: "unknown", - - // Type aliases - -aliasTag: "alias", -} diff --git a/go/vendor/golang.org/x/tools/internal/gcimporter/bimport.go b/go/vendor/golang.org/x/tools/internal/gcimporter/bimport.go index b85de014700..d98b0db2a9a 100644 --- a/go/vendor/golang.org/x/tools/internal/gcimporter/bimport.go +++ b/go/vendor/golang.org/x/tools/internal/gcimporter/bimport.go @@ -2,340 +2,24 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// This file is a copy of $GOROOT/src/go/internal/gcimporter/bimport.go. +// This file contains the remaining vestiges of +// $GOROOT/src/go/internal/gcimporter/bimport.go. package gcimporter import ( - "encoding/binary" "fmt" - "go/constant" "go/token" "go/types" - "sort" - "strconv" - "strings" "sync" - "unicode" - "unicode/utf8" ) -type importer struct { - imports map[string]*types.Package - data []byte - importpath string - buf []byte // for reading strings - version int // export format version - - // object lists - strList []string // in order of appearance - pathList []string // in order of appearance - pkgList []*types.Package // in order of appearance - typList []types.Type // in order of appearance - interfaceList []*types.Interface // for delayed completion only - trackAllTypes bool - - // position encoding - posInfoFormat bool - prevFile string - prevLine int - fake fakeFileSet - - // debugging support - debugFormat bool - read int // bytes read -} - -// BImportData imports a package from the serialized package data -// and returns the number of bytes consumed and a reference to the package. -// If the export data version is not recognized or the format is otherwise -// compromised, an error is returned. -func BImportData(fset *token.FileSet, imports map[string]*types.Package, data []byte, path string) (_ int, pkg *types.Package, err error) { - // catch panics and return them as errors - const currentVersion = 6 - version := -1 // unknown version - defer func() { - if e := recover(); e != nil { - // Return a (possibly nil or incomplete) package unchanged (see #16088). - if version > currentVersion { - err = fmt.Errorf("cannot import %q (%v), export data is newer version - update tool", path, e) - } else { - err = fmt.Errorf("cannot import %q (%v), possibly version skew - reinstall package", path, e) - } - } - }() - - p := importer{ - imports: imports, - data: data, - importpath: path, - version: version, - strList: []string{""}, // empty string is mapped to 0 - pathList: []string{""}, // empty string is mapped to 0 - fake: fakeFileSet{ - fset: fset, - files: make(map[string]*fileInfo), - }, - } - defer p.fake.setLines() // set lines for files in fset - - // read version info - var versionstr string - if b := p.rawByte(); b == 'c' || b == 'd' { - // Go1.7 encoding; first byte encodes low-level - // encoding format (compact vs debug). - // For backward-compatibility only (avoid problems with - // old installed packages). Newly compiled packages use - // the extensible format string. - // TODO(gri) Remove this support eventually; after Go1.8. - if b == 'd' { - p.debugFormat = true - } - p.trackAllTypes = p.rawByte() == 'a' - p.posInfoFormat = p.int() != 0 - versionstr = p.string() - if versionstr == "v1" { - version = 0 - } - } else { - // Go1.8 extensible encoding - // read version string and extract version number (ignore anything after the version number) - versionstr = p.rawStringln(b) - if s := strings.SplitN(versionstr, " ", 3); len(s) >= 2 && s[0] == "version" { - if v, err := strconv.Atoi(s[1]); err == nil && v > 0 { - version = v - } - } - } - p.version = version - - // read version specific flags - extend as necessary - switch p.version { - // case currentVersion: - // ... - // fallthrough - case currentVersion, 5, 4, 3, 2, 1: - p.debugFormat = p.rawStringln(p.rawByte()) == "debug" - p.trackAllTypes = p.int() != 0 - p.posInfoFormat = p.int() != 0 - case 0: - // Go1.7 encoding format - nothing to do here - default: - errorf("unknown bexport format version %d (%q)", p.version, versionstr) - } - - // --- generic export data --- - - // populate typList with predeclared "known" types - p.typList = append(p.typList, predeclared()...) - - // read package data - pkg = p.pkg() - - // read objects of phase 1 only (see cmd/compile/internal/gc/bexport.go) - objcount := 0 - for { - tag := p.tagOrIndex() - if tag == endTag { - break - } - p.obj(tag) - objcount++ - } - - // self-verification - if count := p.int(); count != objcount { - errorf("got %d objects; want %d", objcount, count) - } - - // ignore compiler-specific import data - - // complete interfaces - // TODO(gri) re-investigate if we still need to do this in a delayed fashion - for _, typ := range p.interfaceList { - typ.Complete() - } - - // record all referenced packages as imports - list := append(([]*types.Package)(nil), p.pkgList[1:]...) - sort.Sort(byPath(list)) - pkg.SetImports(list) - - // package was imported completely and without errors - pkg.MarkComplete() - - return p.read, pkg, nil -} - func errorf(format string, args ...interface{}) { panic(fmt.Sprintf(format, args...)) } -func (p *importer) pkg() *types.Package { - // if the package was seen before, i is its index (>= 0) - i := p.tagOrIndex() - if i >= 0 { - return p.pkgList[i] - } - - // otherwise, i is the package tag (< 0) - if i != packageTag { - errorf("unexpected package tag %d version %d", i, p.version) - } - - // read package data - name := p.string() - var path string - if p.version >= 5 { - path = p.path() - } else { - path = p.string() - } - if p.version >= 6 { - p.int() // package height; unused by go/types - } - - // we should never see an empty package name - if name == "" { - errorf("empty package name in import") - } - - // an empty path denotes the package we are currently importing; - // it must be the first package we see - if (path == "") != (len(p.pkgList) == 0) { - errorf("package path %q for pkg index %d", path, len(p.pkgList)) - } - - // if the package was imported before, use that one; otherwise create a new one - if path == "" { - path = p.importpath - } - pkg := p.imports[path] - if pkg == nil { - pkg = types.NewPackage(path, name) - p.imports[path] = pkg - } else if pkg.Name() != name { - errorf("conflicting names %s and %s for package %q", pkg.Name(), name, path) - } - p.pkgList = append(p.pkgList, pkg) - - return pkg -} - -// objTag returns the tag value for each object kind. -func objTag(obj types.Object) int { - switch obj.(type) { - case *types.Const: - return constTag - case *types.TypeName: - return typeTag - case *types.Var: - return varTag - case *types.Func: - return funcTag - default: - errorf("unexpected object: %v (%T)", obj, obj) // panics - panic("unreachable") - } -} - -func sameObj(a, b types.Object) bool { - // Because unnamed types are not canonicalized, we cannot simply compare types for - // (pointer) identity. - // Ideally we'd check equality of constant values as well, but this is good enough. - return objTag(a) == objTag(b) && types.Identical(a.Type(), b.Type()) -} - -func (p *importer) declare(obj types.Object) { - pkg := obj.Pkg() - if alt := pkg.Scope().Insert(obj); alt != nil { - // This can only trigger if we import a (non-type) object a second time. - // Excluding type aliases, this cannot happen because 1) we only import a package - // once; and b) we ignore compiler-specific export data which may contain - // functions whose inlined function bodies refer to other functions that - // were already imported. - // However, type aliases require reexporting the original type, so we need - // to allow it (see also the comment in cmd/compile/internal/gc/bimport.go, - // method importer.obj, switch case importing functions). - // TODO(gri) review/update this comment once the gc compiler handles type aliases. - if !sameObj(obj, alt) { - errorf("inconsistent import:\n\t%v\npreviously imported as:\n\t%v\n", obj, alt) - } - } -} - -func (p *importer) obj(tag int) { - switch tag { - case constTag: - pos := p.pos() - pkg, name := p.qualifiedName() - typ := p.typ(nil, nil) - val := p.value() - p.declare(types.NewConst(pos, pkg, name, typ, val)) - - case aliasTag: - // TODO(gri) verify type alias hookup is correct - pos := p.pos() - pkg, name := p.qualifiedName() - typ := p.typ(nil, nil) - p.declare(types.NewTypeName(pos, pkg, name, typ)) - - case typeTag: - p.typ(nil, nil) - - case varTag: - pos := p.pos() - pkg, name := p.qualifiedName() - typ := p.typ(nil, nil) - p.declare(types.NewVar(pos, pkg, name, typ)) - - case funcTag: - pos := p.pos() - pkg, name := p.qualifiedName() - params, isddd := p.paramList() - result, _ := p.paramList() - sig := types.NewSignature(nil, params, result, isddd) - p.declare(types.NewFunc(pos, pkg, name, sig)) - - default: - errorf("unexpected object tag %d", tag) - } -} - const deltaNewFile = -64 // see cmd/compile/internal/gc/bexport.go -func (p *importer) pos() token.Pos { - if !p.posInfoFormat { - return token.NoPos - } - - file := p.prevFile - line := p.prevLine - delta := p.int() - line += delta - if p.version >= 5 { - if delta == deltaNewFile { - if n := p.int(); n >= 0 { - // file changed - file = p.path() - line = n - } - } - } else { - if delta == 0 { - if n := p.int(); n >= 0 { - // file changed - file = p.prevFile[:n] + p.string() - line = p.int() - } - } - } - p.prevFile = file - p.prevLine = line - - return p.fake.pos(file, line, 0) -} - // Synthesize a token.Pos type fakeFileSet struct { fset *token.FileSet @@ -389,205 +73,6 @@ var ( fakeLinesOnce sync.Once ) -func (p *importer) qualifiedName() (pkg *types.Package, name string) { - name = p.string() - pkg = p.pkg() - return -} - -func (p *importer) record(t types.Type) { - p.typList = append(p.typList, t) -} - -// A dddSlice is a types.Type representing ...T parameters. -// It only appears for parameter types and does not escape -// the importer. -type dddSlice struct { - elem types.Type -} - -func (t *dddSlice) Underlying() types.Type { return t } -func (t *dddSlice) String() string { return "..." + t.elem.String() } - -// parent is the package which declared the type; parent == nil means -// the package currently imported. The parent package is needed for -// exported struct fields and interface methods which don't contain -// explicit package information in the export data. -// -// A non-nil tname is used as the "owner" of the result type; i.e., -// the result type is the underlying type of tname. tname is used -// to give interface methods a named receiver type where possible. -func (p *importer) typ(parent *types.Package, tname *types.Named) types.Type { - // if the type was seen before, i is its index (>= 0) - i := p.tagOrIndex() - if i >= 0 { - return p.typList[i] - } - - // otherwise, i is the type tag (< 0) - switch i { - case namedTag: - // read type object - pos := p.pos() - parent, name := p.qualifiedName() - scope := parent.Scope() - obj := scope.Lookup(name) - - // if the object doesn't exist yet, create and insert it - if obj == nil { - obj = types.NewTypeName(pos, parent, name, nil) - scope.Insert(obj) - } - - if _, ok := obj.(*types.TypeName); !ok { - errorf("pkg = %s, name = %s => %s", parent, name, obj) - } - - // associate new named type with obj if it doesn't exist yet - t0 := types.NewNamed(obj.(*types.TypeName), nil, nil) - - // but record the existing type, if any - tname := obj.Type().(*types.Named) // tname is either t0 or the existing type - p.record(tname) - - // read underlying type - t0.SetUnderlying(p.typ(parent, t0)) - - // interfaces don't have associated methods - if types.IsInterface(t0) { - return tname - } - - // read associated methods - for i := p.int(); i > 0; i-- { - // TODO(gri) replace this with something closer to fieldName - pos := p.pos() - name := p.string() - if !exported(name) { - p.pkg() - } - - recv, _ := p.paramList() // TODO(gri) do we need a full param list for the receiver? - params, isddd := p.paramList() - result, _ := p.paramList() - p.int() // go:nointerface pragma - discarded - - sig := types.NewSignature(recv.At(0), params, result, isddd) - t0.AddMethod(types.NewFunc(pos, parent, name, sig)) - } - - return tname - - case arrayTag: - t := new(types.Array) - if p.trackAllTypes { - p.record(t) - } - - n := p.int64() - *t = *types.NewArray(p.typ(parent, nil), n) - return t - - case sliceTag: - t := new(types.Slice) - if p.trackAllTypes { - p.record(t) - } - - *t = *types.NewSlice(p.typ(parent, nil)) - return t - - case dddTag: - t := new(dddSlice) - if p.trackAllTypes { - p.record(t) - } - - t.elem = p.typ(parent, nil) - return t - - case structTag: - t := new(types.Struct) - if p.trackAllTypes { - p.record(t) - } - - *t = *types.NewStruct(p.fieldList(parent)) - return t - - case pointerTag: - t := new(types.Pointer) - if p.trackAllTypes { - p.record(t) - } - - *t = *types.NewPointer(p.typ(parent, nil)) - return t - - case signatureTag: - t := new(types.Signature) - if p.trackAllTypes { - p.record(t) - } - - params, isddd := p.paramList() - result, _ := p.paramList() - *t = *types.NewSignature(nil, params, result, isddd) - return t - - case interfaceTag: - // Create a dummy entry in the type list. This is safe because we - // cannot expect the interface type to appear in a cycle, as any - // such cycle must contain a named type which would have been - // first defined earlier. - // TODO(gri) Is this still true now that we have type aliases? - // See issue #23225. - n := len(p.typList) - if p.trackAllTypes { - p.record(nil) - } - - var embeddeds []types.Type - for n := p.int(); n > 0; n-- { - p.pos() - embeddeds = append(embeddeds, p.typ(parent, nil)) - } - - t := newInterface(p.methodList(parent, tname), embeddeds) - p.interfaceList = append(p.interfaceList, t) - if p.trackAllTypes { - p.typList[n] = t - } - return t - - case mapTag: - t := new(types.Map) - if p.trackAllTypes { - p.record(t) - } - - key := p.typ(parent, nil) - val := p.typ(parent, nil) - *t = *types.NewMap(key, val) - return t - - case chanTag: - t := new(types.Chan) - if p.trackAllTypes { - p.record(t) - } - - dir := chanDir(p.int()) - val := p.typ(parent, nil) - *t = *types.NewChan(dir, val) - return t - - default: - errorf("unexpected type tag %d", i) // panics - panic("unreachable") - } -} - func chanDir(d int) types.ChanDir { // tag values must match the constants in cmd/compile/internal/gc/go.go switch d { @@ -603,394 +88,6 @@ func chanDir(d int) types.ChanDir { } } -func (p *importer) fieldList(parent *types.Package) (fields []*types.Var, tags []string) { - if n := p.int(); n > 0 { - fields = make([]*types.Var, n) - tags = make([]string, n) - for i := range fields { - fields[i], tags[i] = p.field(parent) - } - } - return -} - -func (p *importer) field(parent *types.Package) (*types.Var, string) { - pos := p.pos() - pkg, name, alias := p.fieldName(parent) - typ := p.typ(parent, nil) - tag := p.string() - - anonymous := false - if name == "" { - // anonymous field - typ must be T or *T and T must be a type name - switch typ := deref(typ).(type) { - case *types.Basic: // basic types are named types - pkg = nil // // objects defined in Universe scope have no package - name = typ.Name() - case *types.Named: - name = typ.Obj().Name() - default: - errorf("named base type expected") - } - anonymous = true - } else if alias { - // anonymous field: we have an explicit name because it's an alias - anonymous = true - } - - return types.NewField(pos, pkg, name, typ, anonymous), tag -} - -func (p *importer) methodList(parent *types.Package, baseType *types.Named) (methods []*types.Func) { - if n := p.int(); n > 0 { - methods = make([]*types.Func, n) - for i := range methods { - methods[i] = p.method(parent, baseType) - } - } - return -} - -func (p *importer) method(parent *types.Package, baseType *types.Named) *types.Func { - pos := p.pos() - pkg, name, _ := p.fieldName(parent) - // If we don't have a baseType, use a nil receiver. - // A receiver using the actual interface type (which - // we don't know yet) will be filled in when we call - // types.Interface.Complete. - var recv *types.Var - if baseType != nil { - recv = types.NewVar(token.NoPos, parent, "", baseType) - } - params, isddd := p.paramList() - result, _ := p.paramList() - sig := types.NewSignature(recv, params, result, isddd) - return types.NewFunc(pos, pkg, name, sig) -} - -func (p *importer) fieldName(parent *types.Package) (pkg *types.Package, name string, alias bool) { - name = p.string() - pkg = parent - if pkg == nil { - // use the imported package instead - pkg = p.pkgList[0] - } - if p.version == 0 && name == "_" { - // version 0 didn't export a package for _ fields - return - } - switch name { - case "": - // 1) field name matches base type name and is exported: nothing to do - case "?": - // 2) field name matches base type name and is not exported: need package - name = "" - pkg = p.pkg() - case "@": - // 3) field name doesn't match type name (alias) - name = p.string() - alias = true - fallthrough - default: - if !exported(name) { - pkg = p.pkg() - } - } - return -} - -func (p *importer) paramList() (*types.Tuple, bool) { - n := p.int() - if n == 0 { - return nil, false - } - // negative length indicates unnamed parameters - named := true - if n < 0 { - n = -n - named = false - } - // n > 0 - params := make([]*types.Var, n) - isddd := false - for i := range params { - params[i], isddd = p.param(named) - } - return types.NewTuple(params...), isddd -} - -func (p *importer) param(named bool) (*types.Var, bool) { - t := p.typ(nil, nil) - td, isddd := t.(*dddSlice) - if isddd { - t = types.NewSlice(td.elem) - } - - var pkg *types.Package - var name string - if named { - name = p.string() - if name == "" { - errorf("expected named parameter") - } - if name != "_" { - pkg = p.pkg() - } - if i := strings.Index(name, "·"); i > 0 { - name = name[:i] // cut off gc-specific parameter numbering - } - } - - // read and discard compiler-specific info - p.string() - - return types.NewVar(token.NoPos, pkg, name, t), isddd -} - -func exported(name string) bool { - ch, _ := utf8.DecodeRuneInString(name) - return unicode.IsUpper(ch) -} - -func (p *importer) value() constant.Value { - switch tag := p.tagOrIndex(); tag { - case falseTag: - return constant.MakeBool(false) - case trueTag: - return constant.MakeBool(true) - case int64Tag: - return constant.MakeInt64(p.int64()) - case floatTag: - return p.float() - case complexTag: - re := p.float() - im := p.float() - return constant.BinaryOp(re, token.ADD, constant.MakeImag(im)) - case stringTag: - return constant.MakeString(p.string()) - case unknownTag: - return constant.MakeUnknown() - default: - errorf("unexpected value tag %d", tag) // panics - panic("unreachable") - } -} - -func (p *importer) float() constant.Value { - sign := p.int() - if sign == 0 { - return constant.MakeInt64(0) - } - - exp := p.int() - mant := []byte(p.string()) // big endian - - // remove leading 0's if any - for len(mant) > 0 && mant[0] == 0 { - mant = mant[1:] - } - - // convert to little endian - // TODO(gri) go/constant should have a more direct conversion function - // (e.g., once it supports a big.Float based implementation) - for i, j := 0, len(mant)-1; i < j; i, j = i+1, j-1 { - mant[i], mant[j] = mant[j], mant[i] - } - - // adjust exponent (constant.MakeFromBytes creates an integer value, - // but mant represents the mantissa bits such that 0.5 <= mant < 1.0) - exp -= len(mant) << 3 - if len(mant) > 0 { - for msd := mant[len(mant)-1]; msd&0x80 == 0; msd <<= 1 { - exp++ - } - } - - x := constant.MakeFromBytes(mant) - switch { - case exp < 0: - d := constant.Shift(constant.MakeInt64(1), token.SHL, uint(-exp)) - x = constant.BinaryOp(x, token.QUO, d) - case exp > 0: - x = constant.Shift(x, token.SHL, uint(exp)) - } - - if sign < 0 { - x = constant.UnaryOp(token.SUB, x, 0) - } - return x -} - -// ---------------------------------------------------------------------------- -// Low-level decoders - -func (p *importer) tagOrIndex() int { - if p.debugFormat { - p.marker('t') - } - - return int(p.rawInt64()) -} - -func (p *importer) int() int { - x := p.int64() - if int64(int(x)) != x { - errorf("exported integer too large") - } - return int(x) -} - -func (p *importer) int64() int64 { - if p.debugFormat { - p.marker('i') - } - - return p.rawInt64() -} - -func (p *importer) path() string { - if p.debugFormat { - p.marker('p') - } - // if the path was seen before, i is its index (>= 0) - // (the empty string is at index 0) - i := p.rawInt64() - if i >= 0 { - return p.pathList[i] - } - // otherwise, i is the negative path length (< 0) - a := make([]string, -i) - for n := range a { - a[n] = p.string() - } - s := strings.Join(a, "/") - p.pathList = append(p.pathList, s) - return s -} - -func (p *importer) string() string { - if p.debugFormat { - p.marker('s') - } - // if the string was seen before, i is its index (>= 0) - // (the empty string is at index 0) - i := p.rawInt64() - if i >= 0 { - return p.strList[i] - } - // otherwise, i is the negative string length (< 0) - if n := int(-i); n <= cap(p.buf) { - p.buf = p.buf[:n] - } else { - p.buf = make([]byte, n) - } - for i := range p.buf { - p.buf[i] = p.rawByte() - } - s := string(p.buf) - p.strList = append(p.strList, s) - return s -} - -func (p *importer) marker(want byte) { - if got := p.rawByte(); got != want { - errorf("incorrect marker: got %c; want %c (pos = %d)", got, want, p.read) - } - - pos := p.read - if n := int(p.rawInt64()); n != pos { - errorf("incorrect position: got %d; want %d", n, pos) - } -} - -// rawInt64 should only be used by low-level decoders. -func (p *importer) rawInt64() int64 { - i, err := binary.ReadVarint(p) - if err != nil { - errorf("read error: %v", err) - } - return i -} - -// rawStringln should only be used to read the initial version string. -func (p *importer) rawStringln(b byte) string { - p.buf = p.buf[:0] - for b != '\n' { - p.buf = append(p.buf, b) - b = p.rawByte() - } - return string(p.buf) -} - -// needed for binary.ReadVarint in rawInt64 -func (p *importer) ReadByte() (byte, error) { - return p.rawByte(), nil -} - -// byte is the bottleneck interface for reading p.data. -// It unescapes '|' 'S' to '$' and '|' '|' to '|'. -// rawByte should only be used by low-level decoders. -func (p *importer) rawByte() byte { - b := p.data[0] - r := 1 - if b == '|' { - b = p.data[1] - r = 2 - switch b { - case 'S': - b = '$' - case '|': - // nothing to do - default: - errorf("unexpected escape sequence in export data") - } - } - p.data = p.data[r:] - p.read += r - return b - -} - -// ---------------------------------------------------------------------------- -// Export format - -// Tags. Must be < 0. -const ( - // Objects - packageTag = -(iota + 1) - constTag - typeTag - varTag - funcTag - endTag - - // Types - namedTag - arrayTag - sliceTag - dddTag - structTag - pointerTag - signatureTag - interfaceTag - mapTag - chanTag - - // Values - falseTag - trueTag - int64Tag - floatTag - fractionTag // not used by gc - complexTag - stringTag - nilTag // only used by gc (appears in exported inlined function bodies) - unknownTag // not used by gc (only appears in packages with errors) - - // Type aliases - aliasTag -) - var predeclOnce sync.Once var predecl []types.Type // initialized lazily diff --git a/go/vendor/golang.org/x/tools/internal/gcimporter/gcimporter.go b/go/vendor/golang.org/x/tools/internal/gcimporter/gcimporter.go index 0372fb3a646..b1223713b94 100644 --- a/go/vendor/golang.org/x/tools/internal/gcimporter/gcimporter.go +++ b/go/vendor/golang.org/x/tools/internal/gcimporter/gcimporter.go @@ -7,6 +7,18 @@ // Package gcimporter provides various functions for reading // gc-generated object files that can be used to implement the // Importer interface defined by the Go 1.5 standard library package. +// +// The encoding is deterministic: if the encoder is applied twice to +// the same types.Package data structure, both encodings are equal. +// This property may be important to avoid spurious changes in +// applications such as build systems. +// +// However, the encoder is not necessarily idempotent. Importing an +// exported package may yield a types.Package that, while it +// represents the same set of Go types as the original, may differ in +// the details of its internal representation. Because of these +// differences, re-encoding the imported package may yield a +// different, but equally valid, encoding of the package. package gcimporter // import "golang.org/x/tools/internal/gcimporter" import ( @@ -218,20 +230,17 @@ func Import(packages map[string]*types.Package, path, srcDir string, lookup func // Or, define a new standard go/types/gcexportdata package. fset := token.NewFileSet() - // The indexed export format starts with an 'i'; the older - // binary export format starts with a 'c', 'd', or 'v' - // (from "version"). Select appropriate importer. + // Select appropriate importer. if len(data) > 0 { switch data[0] { - case 'i': + case 'v', 'c', 'd': // binary, till go1.10 + return nil, fmt.Errorf("binary (%c) import format is no longer supported", data[0]) + + case 'i': // indexed, till go1.19 _, pkg, err := IImportData(fset, packages, data[1:], id) return pkg, err - case 'v', 'c', 'd': - _, pkg, err := BImportData(fset, packages, data, id) - return pkg, err - - case 'u': + case 'u': // unified, from go1.20 _, pkg, err := UImportData(fset, packages, data[1:size], id) return pkg, err diff --git a/go/vendor/golang.org/x/tools/internal/gcimporter/iexport.go b/go/vendor/golang.org/x/tools/internal/gcimporter/iexport.go index ba53cdcdd10..3e3fce1731f 100644 --- a/go/vendor/golang.org/x/tools/internal/gcimporter/iexport.go +++ b/go/vendor/golang.org/x/tools/internal/gcimporter/iexport.go @@ -22,17 +22,23 @@ import ( "strconv" "strings" + "golang.org/x/tools/go/types/objectpath" "golang.org/x/tools/internal/tokeninternal" "golang.org/x/tools/internal/typeparams" ) // IExportShallow encodes "shallow" export data for the specified package. // -// No promises are made about the encoding other than that it can be -// decoded by the same version of IIExportShallow. If you plan to save -// export data in the file system, be sure to include a cryptographic -// digest of the executable in the key to avoid version skew. -func IExportShallow(fset *token.FileSet, pkg *types.Package) ([]byte, error) { +// No promises are made about the encoding other than that it can be decoded by +// the same version of IIExportShallow. If you plan to save export data in the +// file system, be sure to include a cryptographic digest of the executable in +// the key to avoid version skew. +// +// If the provided reportf func is non-nil, it will be used for reporting bugs +// encountered during export. +// TODO(rfindley): remove reportf when we are confident enough in the new +// objectpath encoding. +func IExportShallow(fset *token.FileSet, pkg *types.Package, reportf ReportFunc) ([]byte, error) { // In principle this operation can only fail if out.Write fails, // but that's impossible for bytes.Buffer---and as a matter of // fact iexportCommon doesn't even check for I/O errors. @@ -40,26 +46,34 @@ func IExportShallow(fset *token.FileSet, pkg *types.Package) ([]byte, error) { // TODO(adonovan): use byte slices throughout, avoiding copying. const bundle, shallow = false, true var out bytes.Buffer - err := iexportCommon(&out, fset, bundle, shallow, iexportVersion, []*types.Package{pkg}) + err := iexportCommon(&out, fset, bundle, shallow, iexportVersion, []*types.Package{pkg}, reportf) return out.Bytes(), err } -// IImportShallow decodes "shallow" types.Package data encoded by IExportShallow -// in the same executable. This function cannot import data from +// IImportShallow decodes "shallow" types.Package data encoded by +// IExportShallow in the same executable. This function cannot import data from // cmd/compile or gcexportdata.Write. -func IImportShallow(fset *token.FileSet, imports map[string]*types.Package, data []byte, path string, insert InsertType) (*types.Package, error) { +// +// The importer calls getPackages to obtain package symbols for all +// packages mentioned in the export data, including the one being +// decoded. +// +// If the provided reportf func is non-nil, it will be used for reporting bugs +// encountered during import. +// TODO(rfindley): remove reportf when we are confident enough in the new +// objectpath encoding. +func IImportShallow(fset *token.FileSet, getPackages GetPackagesFunc, data []byte, path string, reportf ReportFunc) (*types.Package, error) { const bundle = false - pkgs, err := iimportCommon(fset, imports, data, bundle, path, insert) + const shallow = true + pkgs, err := iimportCommon(fset, getPackages, data, bundle, path, shallow, reportf) if err != nil { return nil, err } return pkgs[0], nil } -// InsertType is the type of a function that creates a types.TypeName -// object for a named type and inserts it into the scope of the -// specified Package. -type InsertType = func(pkg *types.Package, name string) +// ReportFunc is the type of a function used to report formatted bugs. +type ReportFunc = func(string, ...interface{}) // Current bundled export format version. Increase with each format change. // 0: initial implementation @@ -72,16 +86,16 @@ const bundleVersion = 0 // so that calls to IImportData can override with a provided package path. func IExportData(out io.Writer, fset *token.FileSet, pkg *types.Package) error { const bundle, shallow = false, false - return iexportCommon(out, fset, bundle, shallow, iexportVersion, []*types.Package{pkg}) + return iexportCommon(out, fset, bundle, shallow, iexportVersion, []*types.Package{pkg}, nil) } // IExportBundle writes an indexed export bundle for pkgs to out. func IExportBundle(out io.Writer, fset *token.FileSet, pkgs []*types.Package) error { const bundle, shallow = true, false - return iexportCommon(out, fset, bundle, shallow, iexportVersion, pkgs) + return iexportCommon(out, fset, bundle, shallow, iexportVersion, pkgs, nil) } -func iexportCommon(out io.Writer, fset *token.FileSet, bundle, shallow bool, version int, pkgs []*types.Package) (err error) { +func iexportCommon(out io.Writer, fset *token.FileSet, bundle, shallow bool, version int, pkgs []*types.Package, reportf ReportFunc) (err error) { if !debug { defer func() { if e := recover(); e != nil { @@ -99,6 +113,7 @@ func iexportCommon(out io.Writer, fset *token.FileSet, bundle, shallow bool, ver fset: fset, version: version, shallow: shallow, + reportf: reportf, allPkgs: map[*types.Package]bool{}, stringIndex: map[string]uint64{}, declIndex: map[types.Object]uint64{}, @@ -313,8 +328,10 @@ type iexporter struct { out *bytes.Buffer version int - shallow bool // don't put types from other packages in the index - localpkg *types.Package // (nil in bundle mode) + shallow bool // don't put types from other packages in the index + objEncoder *objectpath.Encoder // encodes objects from other packages in shallow mode; lazily allocated + reportf ReportFunc // if non-nil, used to report bugs + localpkg *types.Package // (nil in bundle mode) // allPkgs tracks all packages that have been referenced by // the export data, so we can ensure to include them in the @@ -354,6 +371,17 @@ func (p *iexporter) trace(format string, args ...interface{}) { fmt.Printf(strings.Repeat("..", p.indent)+format+"\n", args...) } +// objectpathEncoder returns the lazily allocated objectpath.Encoder to use +// when encoding objects in other packages during shallow export. +// +// Using a shared Encoder amortizes some of cost of objectpath search. +func (p *iexporter) objectpathEncoder() *objectpath.Encoder { + if p.objEncoder == nil { + p.objEncoder = new(objectpath.Encoder) + } + return p.objEncoder +} + // stringOff returns the offset of s within the string section. // If not already present, it's added to the end. func (p *iexporter) stringOff(s string) uint64 { @@ -413,7 +441,6 @@ type exportWriter struct { p *iexporter data intWriter - currPkg *types.Package prevFile string prevLine int64 prevColumn int64 @@ -436,7 +463,6 @@ func (p *iexporter) doDecl(obj types.Object) { }() } w := p.newWriter() - w.setPkg(obj.Pkg(), false) switch obj := obj.(type) { case *types.Var: @@ -673,6 +699,9 @@ func (w *exportWriter) qualifiedType(obj *types.TypeName) { w.pkg(obj.Pkg()) } +// TODO(rfindley): what does 'pkg' even mean here? It would be better to pass +// it in explicitly into signatures and structs that may use it for +// constructing fields. func (w *exportWriter) typ(t types.Type, pkg *types.Package) { w.data.uint64(w.p.typOff(t, pkg)) } @@ -764,30 +793,53 @@ func (w *exportWriter) doTyp(t types.Type, pkg *types.Package) { case *types.Signature: w.startType(signatureType) - w.setPkg(pkg, true) + w.pkg(pkg) w.signature(t) case *types.Struct: w.startType(structType) n := t.NumFields() + // Even for struct{} we must emit some qualifying package, because that's + // what the compiler does, and thus that's what the importer expects. + fieldPkg := pkg if n > 0 { - w.setPkg(t.Field(0).Pkg(), true) // qualifying package for field objects - } else { - w.setPkg(pkg, true) + fieldPkg = t.Field(0).Pkg() } + if fieldPkg == nil { + // TODO(rfindley): improve this very hacky logic. + // + // The importer expects a package to be set for all struct types, even + // those with no fields. A better encoding might be to set NumFields + // before pkg. setPkg panics with a nil package, which may be possible + // to reach with invalid packages (and perhaps valid packages, too?), so + // (arbitrarily) set the localpkg if available. + // + // Alternatively, we may be able to simply guarantee that pkg != nil, by + // reconsidering the encoding of constant values. + if w.p.shallow { + fieldPkg = w.p.localpkg + } else { + panic(internalErrorf("no package to set for empty struct")) + } + } + w.pkg(fieldPkg) w.uint64(uint64(n)) + for i := 0; i < n; i++ { f := t.Field(i) + if w.p.shallow { + w.objectPath(f) + } w.pos(f.Pos()) w.string(f.Name()) // unexported fields implicitly qualified by prior setPkg - w.typ(f.Type(), pkg) + w.typ(f.Type(), fieldPkg) w.bool(f.Anonymous()) w.string(t.Tag(i)) // note (or tag) } case *types.Interface: w.startType(interfaceType) - w.setPkg(pkg, true) + w.pkg(pkg) n := t.NumEmbeddeds() w.uint64(uint64(n)) @@ -802,10 +854,16 @@ func (w *exportWriter) doTyp(t types.Type, pkg *types.Package) { w.typ(ft, tPkg) } + // See comment for struct fields. In shallow mode we change the encoding + // for interface methods that are promoted from other packages. + n = t.NumExplicitMethods() w.uint64(uint64(n)) for i := 0; i < n; i++ { m := t.ExplicitMethod(i) + if w.p.shallow { + w.objectPath(m) + } w.pos(m.Pos()) w.string(m.Name()) sig, _ := m.Type().(*types.Signature) @@ -827,12 +885,57 @@ func (w *exportWriter) doTyp(t types.Type, pkg *types.Package) { } } -func (w *exportWriter) setPkg(pkg *types.Package, write bool) { - if write { - w.pkg(pkg) +// objectPath writes the package and objectPath to use to look up obj in a +// different package, when encoding in "shallow" mode. +// +// When doing a shallow import, the importer creates only the local package, +// and requests package symbols for dependencies from the client. +// However, certain types defined in the local package may hold objects defined +// (perhaps deeply) within another package. +// +// For example, consider the following: +// +// package a +// func F() chan * map[string] struct { X int } +// +// package b +// import "a" +// var B = a.F() +// +// In this example, the type of b.B holds fields defined in package a. +// In order to have the correct canonical objects for the field defined in the +// type of B, they are encoded as objectPaths and later looked up in the +// importer. The same problem applies to interface methods. +func (w *exportWriter) objectPath(obj types.Object) { + if obj.Pkg() == nil || obj.Pkg() == w.p.localpkg { + // obj.Pkg() may be nil for the builtin error.Error. + // In this case, or if obj is declared in the local package, no need to + // encode. + w.string("") + return } - - w.currPkg = pkg + objectPath, err := w.p.objectpathEncoder().For(obj) + if err != nil { + // Fall back to the empty string, which will cause the importer to create a + // new object. + // + // This is incorrect in shallow mode (golang/go#60819), but matches + // the previous behavior. This code is defensive, as it is hard to + // prove that the objectpath algorithm will succeed in all cases, and + // creating a new object sort of works. + // (we didn't notice the bug during months of gopls@v0.12.0 testing) + // + // However, report a bug so that we can eventually have confidence + // that export/import is producing a correct package. + // + // TODO: remove reportf once we have such confidence. + objectPath = "" + if w.p.reportf != nil { + w.p.reportf("unable to encode object %q in package %q: %v", obj.Name(), obj.Pkg().Path(), err) + } + } + w.string(string(objectPath)) + w.pkg(obj.Pkg()) } func (w *exportWriter) signature(sig *types.Signature) { @@ -913,6 +1016,17 @@ func (w *exportWriter) value(typ types.Type, v constant.Value) { w.int64(int64(v.Kind())) } + if v.Kind() == constant.Unknown { + // golang/go#60605: treat unknown constant values as if they have invalid type + // + // This loses some fidelity over the package type-checked from source, but that + // is acceptable. + // + // TODO(rfindley): we should switch on the recorded constant kind rather + // than the constant type + return + } + switch b := typ.Underlying().(*types.Basic); b.Info() & types.IsConstType { case types.IsBoolean: w.bool(constant.BoolVal(v)) @@ -969,6 +1083,16 @@ func constantToFloat(x constant.Value) *big.Float { return &f } +func valueToRat(x constant.Value) *big.Rat { + // Convert little-endian to big-endian. + // I can't believe this is necessary. + bytes := constant.Bytes(x) + for i := 0; i < len(bytes)/2; i++ { + bytes[i], bytes[len(bytes)-1-i] = bytes[len(bytes)-1-i], bytes[i] + } + return new(big.Rat).SetInt(new(big.Int).SetBytes(bytes)) +} + // mpint exports a multi-precision integer. // // For unsigned types, small values are written out as a single @@ -1178,3 +1302,19 @@ func (q *objQueue) popHead() types.Object { q.head++ return obj } + +// internalError represents an error generated inside this package. +type internalError string + +func (e internalError) Error() string { return "gcimporter: " + string(e) } + +// TODO(adonovan): make this call panic, so that it's symmetric with errorf. +// Otherwise it's easy to forget to do anything with the error. +// +// TODO(adonovan): also, consider switching the names "errorf" and +// "internalErrorf" as the former is used for bugs, whose cause is +// internal inconsistency, whereas the latter is used for ordinary +// situations like bad input, whose cause is external. +func internalErrorf(format string, args ...interface{}) error { + return internalError(fmt.Sprintf(format, args...)) +} diff --git a/go/vendor/golang.org/x/tools/internal/gcimporter/iimport.go b/go/vendor/golang.org/x/tools/internal/gcimporter/iimport.go index 448f903e86a..8e64cf644fc 100644 --- a/go/vendor/golang.org/x/tools/internal/gcimporter/iimport.go +++ b/go/vendor/golang.org/x/tools/internal/gcimporter/iimport.go @@ -21,6 +21,7 @@ import ( "sort" "strings" + "golang.org/x/tools/go/types/objectpath" "golang.org/x/tools/internal/typeparams" ) @@ -85,7 +86,7 @@ const ( // If the export data version is not recognized or the format is otherwise // compromised, an error is returned. func IImportData(fset *token.FileSet, imports map[string]*types.Package, data []byte, path string) (int, *types.Package, error) { - pkgs, err := iimportCommon(fset, imports, data, false, path, nil) + pkgs, err := iimportCommon(fset, GetPackagesFromMap(imports), data, false, path, false, nil) if err != nil { return 0, nil, err } @@ -94,10 +95,49 @@ func IImportData(fset *token.FileSet, imports map[string]*types.Package, data [] // IImportBundle imports a set of packages from the serialized package bundle. func IImportBundle(fset *token.FileSet, imports map[string]*types.Package, data []byte) ([]*types.Package, error) { - return iimportCommon(fset, imports, data, true, "", nil) + return iimportCommon(fset, GetPackagesFromMap(imports), data, true, "", false, nil) } -func iimportCommon(fset *token.FileSet, imports map[string]*types.Package, data []byte, bundle bool, path string, insert InsertType) (pkgs []*types.Package, err error) { +// A GetPackagesFunc function obtains the non-nil symbols for a set of +// packages, creating and recursively importing them as needed. An +// implementation should store each package symbol is in the Pkg +// field of the items array. +// +// Any error causes importing to fail. This can be used to quickly read +// the import manifest of an export data file without fully decoding it. +type GetPackagesFunc = func(items []GetPackagesItem) error + +// A GetPackagesItem is a request from the importer for the package +// symbol of the specified name and path. +type GetPackagesItem struct { + Name, Path string + Pkg *types.Package // to be filled in by GetPackagesFunc call + + // private importer state + pathOffset uint64 + nameIndex map[string]uint64 +} + +// GetPackagesFromMap returns a GetPackagesFunc that retrieves +// packages from the given map of package path to package. +// +// The returned function may mutate m: each requested package that is not +// found is created with types.NewPackage and inserted into m. +func GetPackagesFromMap(m map[string]*types.Package) GetPackagesFunc { + return func(items []GetPackagesItem) error { + for i, item := range items { + pkg, ok := m[item.Path] + if !ok { + pkg = types.NewPackage(item.Path, item.Name) + m[item.Path] = pkg + } + items[i].Pkg = pkg + } + return nil + } +} + +func iimportCommon(fset *token.FileSet, getPackages GetPackagesFunc, data []byte, bundle bool, path string, shallow bool, reportf ReportFunc) (pkgs []*types.Package, err error) { const currentVersion = iexportVersionCurrent version := int64(-1) if !debug { @@ -108,7 +148,7 @@ func iimportCommon(fset *token.FileSet, imports map[string]*types.Package, data } else if version > currentVersion { err = fmt.Errorf("cannot import %q (%v), export data is newer version - update tool", path, e) } else { - err = fmt.Errorf("cannot import %q (%v), possibly version skew - reinstall package", path, e) + err = fmt.Errorf("internal error while importing %q (%v); please report an issue", path, e) } } }() @@ -117,11 +157,8 @@ func iimportCommon(fset *token.FileSet, imports map[string]*types.Package, data r := &intReader{bytes.NewReader(data), path} if bundle { - bundleVersion := r.uint64() - switch bundleVersion { - case bundleVersion: - default: - errorf("unknown bundle format version %d", bundleVersion) + if v := r.uint64(); v != bundleVersion { + errorf("unknown bundle format version %d", v) } } @@ -139,7 +176,7 @@ func iimportCommon(fset *token.FileSet, imports map[string]*types.Package, data sLen := int64(r.uint64()) var fLen int64 var fileOffset []uint64 - if insert != nil { + if shallow { // Shallow mode uses a different position encoding. fLen = int64(r.uint64()) fileOffset = make([]uint64, r.uint64()) @@ -158,7 +195,8 @@ func iimportCommon(fset *token.FileSet, imports map[string]*types.Package, data p := iimporter{ version: int(version), ipath: path, - insert: insert, + shallow: shallow, + reportf: reportf, stringData: stringData, stringCache: make(map[uint64]string), @@ -185,8 +223,9 @@ func iimportCommon(fset *token.FileSet, imports map[string]*types.Package, data p.typCache[uint64(i)] = pt } - pkgList := make([]*types.Package, r.uint64()) - for i := range pkgList { + // Gather the relevant packages from the manifest. + items := make([]GetPackagesItem, r.uint64()) + for i := range items { pkgPathOff := r.uint64() pkgPath := p.stringAt(pkgPathOff) pkgName := p.stringAt(r.uint64()) @@ -195,30 +234,42 @@ func iimportCommon(fset *token.FileSet, imports map[string]*types.Package, data if pkgPath == "" { pkgPath = path } - pkg := imports[pkgPath] - if pkg == nil { - pkg = types.NewPackage(pkgPath, pkgName) - imports[pkgPath] = pkg - } else if pkg.Name() != pkgName { - errorf("conflicting names %s and %s for package %q", pkg.Name(), pkgName, path) - } - if i == 0 && !bundle { - p.localpkg = pkg - } - - p.pkgCache[pkgPathOff] = pkg + items[i].Name = pkgName + items[i].Path = pkgPath + items[i].pathOffset = pkgPathOff // Read index for package. nameIndex := make(map[string]uint64) nSyms := r.uint64() - // In shallow mode we don't expect an index for other packages. - assert(nSyms == 0 || p.localpkg == pkg || p.insert == nil) + // In shallow mode, only the current package (i=0) has an index. + assert(!(shallow && i > 0 && nSyms != 0)) for ; nSyms > 0; nSyms-- { name := p.stringAt(r.uint64()) nameIndex[name] = r.uint64() } - p.pkgIndex[pkg] = nameIndex + items[i].nameIndex = nameIndex + } + + // Request packages all at once from the client, + // enabling a parallel implementation. + if err := getPackages(items); err != nil { + return nil, err // don't wrap this error + } + + // Check the results and complete the index. + pkgList := make([]*types.Package, len(items)) + for i, item := range items { + pkg := item.Pkg + if pkg == nil { + errorf("internal error: getPackages returned nil package for %q", item.Path) + } else if pkg.Path() != item.Path { + errorf("internal error: getPackages returned wrong path %q, want %q", pkg.Path(), item.Path) + } else if pkg.Name() != item.Name { + errorf("internal error: getPackages returned wrong name %s for package %q, want %s", pkg.Name(), item.Path, item.Name) + } + p.pkgCache[item.pathOffset] = pkg + p.pkgIndex[pkg] = item.nameIndex pkgList[i] = pkg } @@ -277,6 +328,13 @@ func iimportCommon(fset *token.FileSet, imports map[string]*types.Package, data typ.Complete() } + // Workaround for golang/go#61561. See the doc for instanceList for details. + for _, typ := range p.instanceList { + if iface, _ := typ.Underlying().(*types.Interface); iface != nil { + iface.Complete() + } + } + return pkgs, nil } @@ -289,8 +347,8 @@ type iimporter struct { version int ipath string - localpkg *types.Package - insert func(pkg *types.Package, name string) // "shallow" mode only + shallow bool + reportf ReportFunc // if non-nil, used to report bugs stringData []byte stringCache map[uint64]string @@ -307,6 +365,12 @@ type iimporter struct { fake fakeFileSet interfaceList []*types.Interface + // Workaround for the go/types bug golang/go#61561: instances produced during + // instantiation may contain incomplete interfaces. Here we only complete the + // underlying type of the instance, which is the most common case but doesn't + // handle parameterized interface literals defined deeper in the type. + instanceList []types.Type // instances for later completion (see golang/go#61561) + // Arguments for calls to SetConstraint that are deferred due to recursive types later []setConstraintArgs @@ -338,13 +402,9 @@ func (p *iimporter) doDecl(pkg *types.Package, name string) { off, ok := p.pkgIndex[pkg][name] if !ok { - // In "shallow" mode, call back to the application to - // find the object and insert it into the package scope. - if p.insert != nil { - assert(pkg != p.localpkg) - p.insert(pkg, name) // "can't fail" - return - } + // In deep mode, the index should be complete. In shallow + // mode, we should have already recursively loaded necessary + // dependencies so the above Lookup succeeds. errorf("%v.%v not in index", pkg, name) } @@ -711,7 +771,8 @@ func (r *importReader) qualifiedIdent() (*types.Package, string) { } func (r *importReader) pos() token.Pos { - if r.p.insert != nil { // shallow mode + if r.p.shallow { + // precise offsets are encoded only in shallow mode return r.posv2() } if r.p.version >= iexportVersionPosCol { @@ -812,13 +873,28 @@ func (r *importReader) doType(base *types.Named) (res types.Type) { fields := make([]*types.Var, r.uint64()) tags := make([]string, len(fields)) for i := range fields { + var field *types.Var + if r.p.shallow { + field, _ = r.objectPathObject().(*types.Var) + } + fpos := r.pos() fname := r.ident() ftyp := r.typ() emb := r.bool() tag := r.string() - fields[i] = types.NewField(fpos, r.currPkg, fname, ftyp, emb) + // Either this is not a shallow import, the field is local, or the + // encoded objectPath failed to produce an object (a bug). + // + // Even in this last, buggy case, fall back on creating a new field. As + // discussed in iexport.go, this is not correct, but mostly works and is + // preferable to failing (for now at least). + if field == nil { + field = types.NewField(fpos, r.currPkg, fname, ftyp, emb) + } + + fields[i] = field tags[i] = tag } return types.NewStruct(fields, tags) @@ -834,6 +910,11 @@ func (r *importReader) doType(base *types.Named) (res types.Type) { methods := make([]*types.Func, r.uint64()) for i := range methods { + var method *types.Func + if r.p.shallow { + method, _ = r.objectPathObject().(*types.Func) + } + mpos := r.pos() mname := r.ident() @@ -843,9 +924,12 @@ func (r *importReader) doType(base *types.Named) (res types.Type) { if base != nil { recv = types.NewVar(token.NoPos, r.currPkg, "", base) } - msig := r.signature(recv, nil, nil) - methods[i] = types.NewFunc(mpos, r.currPkg, mname, msig) + + if method == nil { + method = types.NewFunc(mpos, r.currPkg, mname, msig) + } + methods[i] = method } typ := newInterface(methods, embeddeds) @@ -883,6 +967,9 @@ func (r *importReader) doType(base *types.Named) (res types.Type) { // we must always use the methods of the base (orig) type. // TODO provide a non-nil *Environment t, _ := typeparams.Instantiate(nil, baseType, targs, false) + + // Workaround for golang/go#61561. See the doc for instanceList for details. + r.p.instanceList = append(r.p.instanceList, t) return t case unionType: @@ -901,6 +988,26 @@ func (r *importReader) kind() itag { return itag(r.uint64()) } +// objectPathObject is the inverse of exportWriter.objectPath. +// +// In shallow mode, certain fields and methods may need to be looked up in an +// imported package. See the doc for exportWriter.objectPath for a full +// explanation. +func (r *importReader) objectPathObject() types.Object { + objPath := objectpath.Path(r.string()) + if objPath == "" { + return nil + } + pkg := r.pkg() + obj, err := objectpath.Object(pkg, objPath) + if err != nil { + if r.p.reportf != nil { + r.p.reportf("failed to find object for objectPath %q: %v", objPath, err) + } + } + return obj +} + func (r *importReader) signature(recv *types.Var, rparams []*typeparams.TypeParam, tparams []*typeparams.TypeParam) *types.Signature { params := r.paramList() results := r.paramList() diff --git a/go/vendor/golang.org/x/tools/internal/gcimporter/ureader_yes.go b/go/vendor/golang.org/x/tools/internal/gcimporter/ureader_yes.go index b285a11ce25..b977435f626 100644 --- a/go/vendor/golang.org/x/tools/internal/gcimporter/ureader_yes.go +++ b/go/vendor/golang.org/x/tools/internal/gcimporter/ureader_yes.go @@ -10,8 +10,10 @@ package gcimporter import ( + "fmt" "go/token" "go/types" + "sort" "strings" "golang.org/x/tools/internal/pkgbits" @@ -62,6 +64,14 @@ type typeInfo struct { } func UImportData(fset *token.FileSet, imports map[string]*types.Package, data []byte, path string) (_ int, pkg *types.Package, err error) { + if !debug { + defer func() { + if x := recover(); x != nil { + err = fmt.Errorf("internal error in importing %q (%v); please report an issue", path, x) + } + }() + } + s := string(data) s = s[:strings.LastIndex(s, "\n$$\n")] input := pkgbits.NewPkgDecoder(path, s) @@ -121,6 +131,16 @@ func readUnifiedPackage(fset *token.FileSet, ctxt *types.Context, imports map[st iface.Complete() } + // Imports() of pkg are all of the transitive packages that were loaded. + var imps []*types.Package + for _, imp := range pr.pkgs { + if imp != nil && imp != pkg { + imps = append(imps, imp) + } + } + sort.Sort(byPath(imps)) + pkg.SetImports(imps) + pkg.MarkComplete() return pkg } @@ -260,39 +280,9 @@ func (r *reader) doPkg() *types.Package { pkg := types.NewPackage(path, name) r.p.imports[path] = pkg - imports := make([]*types.Package, r.Len()) - for i := range imports { - imports[i] = r.pkg() - } - pkg.SetImports(flattenImports(imports)) - return pkg } -// flattenImports returns the transitive closure of all imported -// packages rooted from pkgs. -func flattenImports(pkgs []*types.Package) []*types.Package { - var res []*types.Package - seen := make(map[*types.Package]struct{}) - for _, pkg := range pkgs { - if _, ok := seen[pkg]; ok { - continue - } - seen[pkg] = struct{}{} - res = append(res, pkg) - - // pkg.Imports() is already flattened. - for _, pkg := range pkg.Imports() { - if _, ok := seen[pkg]; ok { - continue - } - seen[pkg] = struct{}{} - res = append(res, pkg) - } - } - return res -} - // @@@ Types func (r *reader) typ() types.Type { diff --git a/go/vendor/golang.org/x/tools/internal/gocommand/invoke.go b/go/vendor/golang.org/x/tools/internal/gocommand/invoke.go index d50551693f3..53cf66da019 100644 --- a/go/vendor/golang.org/x/tools/internal/gocommand/invoke.go +++ b/go/vendor/golang.org/x/tools/internal/gocommand/invoke.go @@ -8,10 +8,12 @@ package gocommand import ( "bytes" "context" + "errors" "fmt" "io" "log" "os" + "reflect" "regexp" "runtime" "strconv" @@ -22,6 +24,9 @@ import ( exec "golang.org/x/sys/execabs" "golang.org/x/tools/internal/event" + "golang.org/x/tools/internal/event/keys" + "golang.org/x/tools/internal/event/label" + "golang.org/x/tools/internal/event/tag" ) // An Runner will run go command invocations and serialize @@ -51,9 +56,19 @@ func (runner *Runner) initialize() { // 1.14: go: updating go.mod: existing contents have changed since last read var modConcurrencyError = regexp.MustCompile(`go:.*go.mod.*contents have changed`) +// verb is an event label for the go command verb. +var verb = keys.NewString("verb", "go command verb") + +func invLabels(inv Invocation) []label.Label { + return []label.Label{verb.Of(inv.Verb), tag.Directory.Of(inv.WorkingDir)} +} + // Run is a convenience wrapper around RunRaw. // It returns only stdout and a "friendly" error. func (runner *Runner) Run(ctx context.Context, inv Invocation) (*bytes.Buffer, error) { + ctx, done := event.Start(ctx, "gocommand.Runner.Run", invLabels(inv)...) + defer done() + stdout, _, friendly, _ := runner.RunRaw(ctx, inv) return stdout, friendly } @@ -61,6 +76,9 @@ func (runner *Runner) Run(ctx context.Context, inv Invocation) (*bytes.Buffer, e // RunPiped runs the invocation serially, always waiting for any concurrent // invocations to complete first. func (runner *Runner) RunPiped(ctx context.Context, inv Invocation, stdout, stderr io.Writer) error { + ctx, done := event.Start(ctx, "gocommand.Runner.RunPiped", invLabels(inv)...) + defer done() + _, err := runner.runPiped(ctx, inv, stdout, stderr) return err } @@ -68,6 +86,8 @@ func (runner *Runner) RunPiped(ctx context.Context, inv Invocation, stdout, stde // RunRaw runs the invocation, serializing requests only if they fight over // go.mod changes. func (runner *Runner) RunRaw(ctx context.Context, inv Invocation) (*bytes.Buffer, *bytes.Buffer, error, error) { + ctx, done := event.Start(ctx, "gocommand.Runner.RunRaw", invLabels(inv)...) + defer done() // Make sure the runner is always initialized. runner.initialize() @@ -215,6 +235,18 @@ func (i *Invocation) run(ctx context.Context, stdout, stderr io.Writer) error { cmd := exec.Command("go", goArgs...) cmd.Stdout = stdout cmd.Stderr = stderr + + // cmd.WaitDelay was added only in go1.20 (see #50436). + if waitDelay := reflect.ValueOf(cmd).Elem().FieldByName("WaitDelay"); waitDelay.IsValid() { + // https://go.dev/issue/59541: don't wait forever copying stderr + // after the command has exited. + // After CL 484741 we copy stdout manually, so we we'll stop reading that as + // soon as ctx is done. However, we also don't want to wait around forever + // for stderr. Give a much-longer-than-reasonable delay and then assume that + // something has wedged in the kernel or runtime. + waitDelay.Set(reflect.ValueOf(30 * time.Second)) + } + // On darwin the cwd gets resolved to the real path, which breaks anything that // expects the working directory to keep the original path, including the // go command when dealing with modules. @@ -229,6 +261,7 @@ func (i *Invocation) run(ctx context.Context, stdout, stderr io.Writer) error { cmd.Env = append(cmd.Env, "PWD="+i.WorkingDir) cmd.Dir = i.WorkingDir } + defer func(start time.Time) { log("%s for %v", time.Since(start), cmdDebugStr(cmd)) }(time.Now()) return runCmdContext(ctx, cmd) @@ -242,10 +275,85 @@ var DebugHangingGoCommands = false // runCmdContext is like exec.CommandContext except it sends os.Interrupt // before os.Kill. -func runCmdContext(ctx context.Context, cmd *exec.Cmd) error { - if err := cmd.Start(); err != nil { +func runCmdContext(ctx context.Context, cmd *exec.Cmd) (err error) { + // If cmd.Stdout is not an *os.File, the exec package will create a pipe and + // copy it to the Writer in a goroutine until the process has finished and + // either the pipe reaches EOF or command's WaitDelay expires. + // + // However, the output from 'go list' can be quite large, and we don't want to + // keep reading (and allocating buffers) if we've already decided we don't + // care about the output. We don't want to wait for the process to finish, and + // we don't wait to wait for the WaitDelay to expire either. + // + // Instead, if cmd.Stdout requires a copying goroutine we explicitly replace + // it with a pipe (which is an *os.File), which we can close in order to stop + // copying output as soon as we realize we don't care about it. + var stdoutW *os.File + if cmd.Stdout != nil { + if _, ok := cmd.Stdout.(*os.File); !ok { + var stdoutR *os.File + stdoutR, stdoutW, err = os.Pipe() + if err != nil { + return err + } + prevStdout := cmd.Stdout + cmd.Stdout = stdoutW + + stdoutErr := make(chan error, 1) + go func() { + _, err := io.Copy(prevStdout, stdoutR) + if err != nil { + err = fmt.Errorf("copying stdout: %w", err) + } + stdoutErr <- err + }() + defer func() { + // We started a goroutine to copy a stdout pipe. + // Wait for it to finish, or terminate it if need be. + var err2 error + select { + case err2 = <-stdoutErr: + stdoutR.Close() + case <-ctx.Done(): + stdoutR.Close() + // Per https://pkg.go.dev/os#File.Close, the call to stdoutR.Close + // should cause the Read call in io.Copy to unblock and return + // immediately, but we still need to receive from stdoutErr to confirm + // that it has happened. + <-stdoutErr + err2 = ctx.Err() + } + if err == nil { + err = err2 + } + }() + + // Per https://pkg.go.dev/os/exec#Cmd, “If Stdout and Stderr are the + // same writer, and have a type that can be compared with ==, at most + // one goroutine at a time will call Write.” + // + // Since we're starting a goroutine that writes to cmd.Stdout, we must + // also update cmd.Stderr so that it still holds. + func() { + defer func() { recover() }() + if cmd.Stderr == prevStdout { + cmd.Stderr = cmd.Stdout + } + }() + } + } + + err = cmd.Start() + if stdoutW != nil { + // The child process has inherited the pipe file, + // so close the copy held in this process. + stdoutW.Close() + stdoutW = nil + } + if err != nil { return err } + resChan := make(chan error, 1) go func() { resChan <- cmd.Wait() @@ -253,11 +361,14 @@ func runCmdContext(ctx context.Context, cmd *exec.Cmd) error { // If we're interested in debugging hanging Go commands, stop waiting after a // minute and panic with interesting information. - if DebugHangingGoCommands { + debug := DebugHangingGoCommands + if debug { + timer := time.NewTimer(1 * time.Minute) + defer timer.Stop() select { case err := <-resChan: return err - case <-time.After(1 * time.Minute): + case <-timer.C: HandleHangingGoCommand(cmd.Process) case <-ctx.Done(): } @@ -270,30 +381,25 @@ func runCmdContext(ctx context.Context, cmd *exec.Cmd) error { } // Cancelled. Interrupt and see if it ends voluntarily. - cmd.Process.Signal(os.Interrupt) - select { - case err := <-resChan: - return err - case <-time.After(time.Second): + if err := cmd.Process.Signal(os.Interrupt); err == nil { + // (We used to wait only 1s but this proved + // fragile on loaded builder machines.) + timer := time.NewTimer(5 * time.Second) + defer timer.Stop() + select { + case err := <-resChan: + return err + case <-timer.C: + } } // Didn't shut down in response to interrupt. Kill it hard. // TODO(rfindley): per advice from bcmills@, it may be better to send SIGQUIT // on certain platforms, such as unix. - if err := cmd.Process.Kill(); err != nil && DebugHangingGoCommands { - // Don't panic here as this reliably fails on windows with EINVAL. + if err := cmd.Process.Kill(); err != nil && !errors.Is(err, os.ErrProcessDone) && debug { log.Printf("error killing the Go command: %v", err) } - // See above: don't wait indefinitely if we're debugging hanging Go commands. - if DebugHangingGoCommands { - select { - case err := <-resChan: - return err - case <-time.After(10 * time.Second): // a shorter wait as resChan should return quickly following Kill - HandleHangingGoCommand(cmd.Process) - } - } return <-resChan } diff --git a/go/vendor/golang.org/x/tools/internal/gocommand/version.go b/go/vendor/golang.org/x/tools/internal/gocommand/version.go index 307a76d474a..446c5846a60 100644 --- a/go/vendor/golang.org/x/tools/internal/gocommand/version.go +++ b/go/vendor/golang.org/x/tools/internal/gocommand/version.go @@ -23,21 +23,11 @@ import ( func GoVersion(ctx context.Context, inv Invocation, r *Runner) (int, error) { inv.Verb = "list" inv.Args = []string{"-e", "-f", `{{context.ReleaseTags}}`, `--`, `unsafe`} - inv.Env = append(append([]string{}, inv.Env...), "GO111MODULE=off") - // Unset any unneeded flags, and remove them from BuildFlags, if they're - // present. - inv.ModFile = "" + inv.BuildFlags = nil // This is not a build command. inv.ModFlag = "" - var buildFlags []string - for _, flag := range inv.BuildFlags { - // Flags can be prefixed by one or two dashes. - f := strings.TrimPrefix(strings.TrimPrefix(flag, "-"), "-") - if strings.HasPrefix(f, "mod=") || strings.HasPrefix(f, "modfile=") { - continue - } - buildFlags = append(buildFlags, flag) - } - inv.BuildFlags = buildFlags + inv.ModFile = "" + inv.Env = append(inv.Env[:len(inv.Env):len(inv.Env)], "GO111MODULE=off") + stdoutBytes, err := r.Run(ctx, inv) if err != nil { return 0, err diff --git a/go/vendor/golang.org/x/tools/internal/tokeninternal/tokeninternal.go b/go/vendor/golang.org/x/tools/internal/tokeninternal/tokeninternal.go index a3fb2d4f29d..7e638ec24fc 100644 --- a/go/vendor/golang.org/x/tools/internal/tokeninternal/tokeninternal.go +++ b/go/vendor/golang.org/x/tools/internal/tokeninternal/tokeninternal.go @@ -7,7 +7,9 @@ package tokeninternal import ( + "fmt" "go/token" + "sort" "sync" "unsafe" ) @@ -57,3 +59,93 @@ func GetLines(file *token.File) []int { panic("unexpected token.File size") } } + +// AddExistingFiles adds the specified files to the FileSet if they +// are not already present. It panics if any pair of files in the +// resulting FileSet would overlap. +func AddExistingFiles(fset *token.FileSet, files []*token.File) { + // Punch through the FileSet encapsulation. + type tokenFileSet struct { + // This type remained essentially consistent from go1.16 to go1.21. + mutex sync.RWMutex + base int + files []*token.File + _ *token.File // changed to atomic.Pointer[token.File] in go1.19 + } + + // If the size of token.FileSet changes, this will fail to compile. + const delta = int64(unsafe.Sizeof(tokenFileSet{})) - int64(unsafe.Sizeof(token.FileSet{})) + var _ [-delta * delta]int + + type uP = unsafe.Pointer + var ptr *tokenFileSet + *(*uP)(uP(&ptr)) = uP(fset) + ptr.mutex.Lock() + defer ptr.mutex.Unlock() + + // Merge and sort. + newFiles := append(ptr.files, files...) + sort.Slice(newFiles, func(i, j int) bool { + return newFiles[i].Base() < newFiles[j].Base() + }) + + // Reject overlapping files. + // Discard adjacent identical files. + out := newFiles[:0] + for i, file := range newFiles { + if i > 0 { + prev := newFiles[i-1] + if file == prev { + continue + } + if prev.Base()+prev.Size()+1 > file.Base() { + panic(fmt.Sprintf("file %s (%d-%d) overlaps with file %s (%d-%d)", + prev.Name(), prev.Base(), prev.Base()+prev.Size(), + file.Name(), file.Base(), file.Base()+file.Size())) + } + } + out = append(out, file) + } + newFiles = out + + ptr.files = newFiles + + // Advance FileSet.Base(). + if len(newFiles) > 0 { + last := newFiles[len(newFiles)-1] + newBase := last.Base() + last.Size() + 1 + if ptr.base < newBase { + ptr.base = newBase + } + } +} + +// FileSetFor returns a new FileSet containing a sequence of new Files with +// the same base, size, and line as the input files, for use in APIs that +// require a FileSet. +// +// Precondition: the input files must be non-overlapping, and sorted in order +// of their Base. +func FileSetFor(files ...*token.File) *token.FileSet { + fset := token.NewFileSet() + for _, f := range files { + f2 := fset.AddFile(f.Name(), f.Base(), f.Size()) + lines := GetLines(f) + f2.SetLines(lines) + } + return fset +} + +// CloneFileSet creates a new FileSet holding all files in fset. It does not +// create copies of the token.Files in fset: they are added to the resulting +// FileSet unmodified. +func CloneFileSet(fset *token.FileSet) *token.FileSet { + var files []*token.File + fset.Iterate(func(f *token.File) bool { + files = append(files, f) + return true + }) + newFileSet := token.NewFileSet() + AddExistingFiles(newFileSet, files) + return newFileSet +} diff --git a/go/vendor/golang.org/x/tools/internal/typeparams/common.go b/go/vendor/golang.org/x/tools/internal/typeparams/common.go index 25a1426d30e..d0d0649fe2a 100644 --- a/go/vendor/golang.org/x/tools/internal/typeparams/common.go +++ b/go/vendor/golang.org/x/tools/internal/typeparams/common.go @@ -23,6 +23,7 @@ package typeparams import ( + "fmt" "go/ast" "go/token" "go/types" @@ -87,7 +88,6 @@ func IsTypeParam(t types.Type) bool { func OriginMethod(fn *types.Func) *types.Func { recv := fn.Type().(*types.Signature).Recv() if recv == nil { - return fn } base := recv.Type() @@ -106,6 +106,31 @@ func OriginMethod(fn *types.Func) *types.Func { } orig := NamedTypeOrigin(named) gfn, _, _ := types.LookupFieldOrMethod(orig, true, fn.Pkg(), fn.Name()) + + // This is a fix for a gopls crash (#60628) due to a go/types bug (#60634). In: + // package p + // type T *int + // func (*T) f() {} + // LookupFieldOrMethod(T, true, p, f)=nil, but NewMethodSet(*T)={(*T).f}. + // Here we make them consistent by force. + // (The go/types bug is general, but this workaround is reached only + // for generic T thanks to the early return above.) + if gfn == nil { + mset := types.NewMethodSet(types.NewPointer(orig)) + for i := 0; i < mset.Len(); i++ { + m := mset.At(i) + if m.Obj().Id() == fn.Id() { + gfn = m.Obj() + break + } + } + } + + // In golang/go#61196, we observe another crash, this time inexplicable. + if gfn == nil { + panic(fmt.Sprintf("missing origin method for %s.%s; named == origin: %t, named.NumMethods(): %d, origin.NumMethods(): %d", named, fn, named == orig, named.NumMethods(), orig.NumMethods())) + } + return gfn.(*types.Func) } diff --git a/go/vendor/golang.org/x/tools/internal/typeparams/typeparams_go117.go b/go/vendor/golang.org/x/tools/internal/typeparams/typeparams_go117.go index b4788978ff4..7ed86e1711b 100644 --- a/go/vendor/golang.org/x/tools/internal/typeparams/typeparams_go117.go +++ b/go/vendor/golang.org/x/tools/internal/typeparams/typeparams_go117.go @@ -129,7 +129,7 @@ func NamedTypeArgs(*types.Named) *TypeList { } // NamedTypeOrigin is the identity method at this Go version. -func NamedTypeOrigin(named *types.Named) types.Type { +func NamedTypeOrigin(named *types.Named) *types.Named { return named } diff --git a/go/vendor/golang.org/x/tools/internal/typeparams/typeparams_go118.go b/go/vendor/golang.org/x/tools/internal/typeparams/typeparams_go118.go index 114a36b866b..cf301af1dbe 100644 --- a/go/vendor/golang.org/x/tools/internal/typeparams/typeparams_go118.go +++ b/go/vendor/golang.org/x/tools/internal/typeparams/typeparams_go118.go @@ -103,7 +103,7 @@ func NamedTypeArgs(named *types.Named) *TypeList { } // NamedTypeOrigin returns named.Orig(). -func NamedTypeOrigin(named *types.Named) types.Type { +func NamedTypeOrigin(named *types.Named) *types.Named { return named.Origin() } diff --git a/go/vendor/golang.org/x/tools/internal/typesinternal/types.go b/go/vendor/golang.org/x/tools/internal/typesinternal/types.go index ce7d4351b22..66e8b099bd6 100644 --- a/go/vendor/golang.org/x/tools/internal/typesinternal/types.go +++ b/go/vendor/golang.org/x/tools/internal/typesinternal/types.go @@ -11,6 +11,8 @@ import ( "go/types" "reflect" "unsafe" + + "golang.org/x/tools/go/types/objectpath" ) func SetUsesCgo(conf *types.Config) bool { @@ -50,3 +52,17 @@ func ReadGo116ErrorData(err types.Error) (code ErrorCode, start, end token.Pos, } var SetGoVersion = func(conf *types.Config, version string) bool { return false } + +// SkipEncoderMethodSorting marks the encoder as not requiring sorted methods, +// as an optimization for gopls (which guarantees the order of parsed source files). +// +// TODO(golang/go#61443): eliminate this parameter one way or the other. +// +//go:linkname SkipEncoderMethodSorting golang.org/x/tools/go/types/objectpath.skipMethodSorting +func SkipEncoderMethodSorting(enc *objectpath.Encoder) + +// ObjectpathObject is like objectpath.Object, but allows suppressing method +// sorting (which is not necessary for gopls). +// +//go:linkname ObjectpathObject golang.org/x/tools/go/types/objectpath.object +func ObjectpathObject(pkg *types.Package, p objectpath.Path, skipMethodSorting bool) (types.Object, error) diff --git a/go/vendor/modules.txt b/go/vendor/modules.txt index 596bd8e8ae9..1f1e5a29f25 100644 --- a/go/vendor/modules.txt +++ b/go/vendor/modules.txt @@ -1,21 +1,23 @@ -# golang.org/x/mod v0.8.0 +# golang.org/x/mod v0.12.0 ## explicit; go 1.17 golang.org/x/mod/internal/lazyregexp golang.org/x/mod/modfile golang.org/x/mod/module golang.org/x/mod/semver -# golang.org/x/sys v0.5.0 +# golang.org/x/sys v0.10.0 ## explicit; go 1.17 golang.org/x/sys/execabs -# golang.org/x/tools v0.6.0 +# golang.org/x/tools v0.11.1 ## explicit; go 1.18 golang.org/x/tools/go/gcexportdata golang.org/x/tools/go/internal/packagesdriver golang.org/x/tools/go/packages +golang.org/x/tools/go/types/objectpath golang.org/x/tools/internal/event golang.org/x/tools/internal/event/core golang.org/x/tools/internal/event/keys golang.org/x/tools/internal/event/label +golang.org/x/tools/internal/event/tag golang.org/x/tools/internal/gcimporter golang.org/x/tools/internal/gocommand golang.org/x/tools/internal/packagesinternal @@ -23,5 +25,5 @@ golang.org/x/tools/internal/pkgbits golang.org/x/tools/internal/tokeninternal golang.org/x/tools/internal/typeparams golang.org/x/tools/internal/typesinternal -# golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 -## explicit; go 1.11 +# golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 +## explicit; go 1.17 diff --git a/java/kotlin-extractor/src/main/kotlin/KotlinFileExtractor.kt b/java/kotlin-extractor/src/main/kotlin/KotlinFileExtractor.kt index aa0a137173b..06a5b4bb48c 100644 --- a/java/kotlin-extractor/src/main/kotlin/KotlinFileExtractor.kt +++ b/java/kotlin-extractor/src/main/kotlin/KotlinFileExtractor.kt @@ -552,7 +552,7 @@ open class KotlinFileExtractor( logger.warnElement("Expected annotation property to define a getter", prop) } else { val getterId = useFunction(getter) - val exprId = extractAnnotationValueExpression(v, id, i, "{${getterId}}", getter.returnType, extractEnumTypeAccesses) + val exprId = extractAnnotationValueExpression(v, id, i, "{$getterId}", getter.returnType, extractEnumTypeAccesses) if (exprId != null) { tw.writeAnnotValue(id, getterId, exprId) } @@ -587,7 +587,7 @@ open class KotlinFileExtractor( extractAnnotation(v, parent, idx, extractEnumTypeAccesses, contextLabel) } is IrVararg -> { - tw.getLabelFor("@\"annotationarray;{${parent}};$contextLabel\"").also { arrayId -> + tw.getLabelFor("@\"annotationarray;{$parent};$contextLabel\"").also { arrayId -> // Use the context type (i.e., the type the annotation expects, not the actual type of the array) // because the Java extractor fills in array types using the same technique. These should only // differ for generic annotations. @@ -1193,7 +1193,7 @@ open class KotlinFileExtractor( // n + o'th parameter, where `o` is the parameter offset caused by adding any dispatch receiver to the parameter list. // Note we don't need to add the extension receiver here because `useValueParameter` always assumes an extension receiver // will be prepended if one exists. - val realFunctionId = useFunction(f) + val realFunctionId = useFunction(f, parentId, null) DeclarationStackAdjuster(f, OverriddenFunctionAttributes(id, id, locId, nonSyntheticParams, typeParameters = listOf(), isStatic = true)).use { val realParamsVarId = getValueParameterLabel(id, parameterTypes.size - 2) val intType = pluginContext.irBuiltIns.intType diff --git a/java/kotlin-extractor/src/main/kotlin/KotlinUsesExtractor.kt b/java/kotlin-extractor/src/main/kotlin/KotlinUsesExtractor.kt index fdaebe5f1c8..95b7dd6d28d 100644 --- a/java/kotlin-extractor/src/main/kotlin/KotlinUsesExtractor.kt +++ b/java/kotlin-extractor/src/main/kotlin/KotlinUsesExtractor.kt @@ -612,7 +612,7 @@ open class KotlinUsesExtractor( val componentTypeLabel = recInfo.componentTypeResults.javaResult.id val dimensions = recInfo.dimensions + 1 - val id = tw.getLabelFor("@\"array;$dimensions;{${elementTypeLabel}}\"") { + val id = tw.getLabelFor("@\"array;$dimensions;{$elementTypeLabel}\"") { tw.writeArrays( it, javaShortName, @@ -1141,7 +1141,7 @@ open class KotlinUsesExtractor( // method (and presumably that disambiguation is never needed when the method belongs to a parameterized // instance of a generic class), but as of now I don't know when the raw method would be referred to. val typeArgSuffix = if (functionTypeParameters.isNotEmpty() && classTypeArgsIncludingOuterClasses.isNullOrEmpty()) "<${functionTypeParameters.size}>" else ""; - return "@\"$prefix;{$parentId}.$name($paramTypeIds){$returnTypeId}${typeArgSuffix}\"" + return "@\"$prefix;{$parentId}.$name($paramTypeIds){$returnTypeId}$typeArgSuffix\"" } val javaLangClass by lazy { referenceExternalClass("java.lang.Class") } @@ -1672,7 +1672,7 @@ open class KotlinUsesExtractor( // clashing trap labels. These are always private, so we can just make up a label without // worrying about their names as seen from Java. val extensionPropertyDiscriminator = getExtensionReceiverType(f)?.let { "extension;${useType(it).javaResult.id}" } ?: "" - return "@\"field;{$parentId};${extensionPropertyDiscriminator}${f.name.asString()}\"" + return "@\"field;{$parentId};$extensionPropertyDiscriminator${f.name.asString()}\"" } fun useField(f: IrField): Label = diff --git a/java/ql/lib/change-notes/2023-06-16-initial-version.md b/java/ql/lib/change-notes/2023-06-16-initial-version.md new file mode 100644 index 00000000000..1f9ea943244 --- /dev/null +++ b/java/ql/lib/change-notes/2023-06-16-initial-version.md @@ -0,0 +1,4 @@ +--- +category: majorAnalysis +--- +* Improved support for flow through captured variables that properly adheres to inter-procedural control flow. diff --git a/java/ql/lib/change-notes/2023-08-07-randomdatasource-typo-fix.md b/java/ql/lib/change-notes/2023-08-07-randomdatasource-typo-fix.md new file mode 100644 index 00000000000..199cd8c1238 --- /dev/null +++ b/java/ql/lib/change-notes/2023-08-07-randomdatasource-typo-fix.md @@ -0,0 +1,4 @@ +--- +category: minorAnalysis +--- +* Fixed a typo in the `StdlibRandomSource` class in `RandomDataSource.qll`, which caused the class to improperly model calls to the `nextBytes` method. Queries relying on `StdlibRandomSource` may see an increase in results. diff --git a/java/ql/lib/ext/java.lang.model.yml b/java/ql/lib/ext/java.lang.model.yml index 817683836fb..4b7353b0062 100644 --- a/java/ql/lib/ext/java.lang.model.yml +++ b/java/ql/lib/ext/java.lang.model.yml @@ -176,7 +176,6 @@ extensions: - ["java.lang", "Object", "getClass", "()", "summary", "manual"] - ["java.lang", "Object", "hashCode", "()", "summary", "manual"] - ["java.lang", "Object", "toString", "()", "summary", "manual"] - - ["java.lang", "Runnable", "run", "()", "summary", "manual"] - ["java.lang", "Runtime", "getRuntime", "()", "summary", "manual"] - ["java.lang", "String", "compareTo", "(String)", "summary", "manual"] - ["java.lang", "String", "contains", "(CharSequence)", "summary", "manual"] diff --git a/java/ql/lib/ext/java.nio.file.model.yml b/java/ql/lib/ext/java.nio.file.model.yml index 7b151739a07..53cd7eee4ed 100644 --- a/java/ql/lib/ext/java.nio.file.model.yml +++ b/java/ql/lib/ext/java.nio.file.model.yml @@ -17,11 +17,11 @@ extensions: - ["java.nio.file", "Files", False, "createTempFile", "(Path,String,String,FileAttribute[])", "", "Argument[0]", "path-injection", "manual"] - ["java.nio.file", "Files", False, "delete", "(Path)", "", "Argument[0]", "path-injection", "ai-manual"] - ["java.nio.file", "Files", False, "deleteIfExists", "(Path)", "", "Argument[0]", "path-injection", "ai-manual"] - - ["java.nio.file", "Files", False, "deleteIfExists", "(Path)", "", "Argument[0]", "path-injection", "ai-manual"] - ["java.nio.file", "Files", False, "getFileStore", "(Path)", "", "Argument[0]", "path-injection", "ai-manual"] # the FileStore class is unlikely to be used for later sanitization - ["java.nio.file", "Files", False, "lines", "(Path,Charset)", "", "Argument[0]", "path-injection", "ai-manual"] - ["java.nio.file", "Files", False, "lines", "(Path)", "", "Argument[0]", "path-injection", "ai-manual"] - ["java.nio.file", "Files", False, "move", "", "", "Argument[1]", "path-injection", "manual"] + - ["java.nio.file", "Files", False, "move", "(Path,Path,CopyOption[])", "", "Argument[0]", "path-injection", "ai-manual"] - ["java.nio.file", "Files", False, "newBufferedReader", "(Path,Charset)", "", "Argument[0]", "path-injection", "ai-manual"] - ["java.nio.file", "Files", False, "newBufferedReader", "(Path)", "", "Argument[0]", "path-injection", "ai-manual"] - ["java.nio.file", "Files", False, "newBufferedWriter", "", "", "Argument[0]", "path-injection", "manual"] @@ -37,11 +37,6 @@ extensions: - ["java.nio.file", "Files", False, "write", "", "", "Argument[1]", "file-content-store", "manual"] - ["java.nio.file", "Files", False, "writeString", "", "", "Argument[0]", "path-injection", "manual"] - ["java.nio.file", "Files", False, "writeString", "", "", "Argument[1]", "file-content-store", "manual"] - - ["java.nio.file", "Files", True, "move", "(Path,Path,CopyOption[])", "", "Argument[1]", "path-injection", "ai-manual"] - - ["java.nio.file", "Files", True, "move", "(Path,Path,CopyOption[])", "", "Argument[0]", "path-injection", "ai-manual"] - - ["java.nio.file", "Files", True, "delete", "(Path)", "", "Argument[0]", "path-injection", "ai-manual"] - - ["java.nio.file", "Files", True, "newInputStream", "(Path,OpenOption[])", "", "Argument[0]", "path-injection", "ai-manual"] - - ["java.nio.file", "Files", True, "newOutputStream", "(Path,OpenOption[])", "", "Argument[0]", "path-injection", "ai-manual"] - ["java.nio.file", "FileSystem", False, "getPath", "", "", "Argument[0..1]", "path-injection", "manual"] # old PathCreation - ["java.nio.file", "FileSystems", False, "newFileSystem", "(URI,Map)", "", "Argument[0]", "path-injection", "ai-manual"] - ["java.nio.file", "FileSystems", False, "newFileSystem", "(URI,Map)", "", "Argument[0]", "request-forgery", "ai-manual"] diff --git a/java/ql/lib/ext/threatmodels/supported-threat-models.model.yml b/java/ql/lib/ext/threatmodels/supported-threat-models.model.yml new file mode 100644 index 00000000000..8c6c533228d --- /dev/null +++ b/java/ql/lib/ext/threatmodels/supported-threat-models.model.yml @@ -0,0 +1,7 @@ +extensions: + + - addsTo: + pack: codeql/java-all + extensible: supportedThreatModels + data: + - ["default"] # The "default" threat model is always included. diff --git a/java/ql/lib/ext/threatmodels/threat-model-grouping.model.yml b/java/ql/lib/ext/threatmodels/threat-model-grouping.model.yml new file mode 100644 index 00000000000..2b85b258b57 --- /dev/null +++ b/java/ql/lib/ext/threatmodels/threat-model-grouping.model.yml @@ -0,0 +1,23 @@ +extensions: + + - addsTo: + pack: codeql/java-all + extensible: threatModelGrouping + data: + # Default threat model + - ["remote", "default"] + - ["uri-path", "default"] + + # Android threat models + - ["android-external-storage-dir", "android"] + - ["contentprovider", "android"] + + # Remote threat models + - ["request", "remote"] + - ["response", "remote"] + + # Local threat models + - ["database", "local"] + - ["cli", "local"] + - ["environment", "local"] + - ["file", "local"] diff --git a/java/ql/lib/qlpack.yml b/java/ql/lib/qlpack.yml index a0936da34b3..8b5bd6697c2 100644 --- a/java/ql/lib/qlpack.yml +++ b/java/ql/lib/qlpack.yml @@ -16,4 +16,5 @@ dataExtensions: - ext/*.model.yml - ext/generated/*.model.yml - ext/experimental/*.model.yml + - ext/threatmodels/*.model.yml warnOnImplicitThis: true diff --git a/java/ql/lib/semmle/code/java/dataflow/ExternalFlowConfiguration.qll b/java/ql/lib/semmle/code/java/dataflow/ExternalFlowConfiguration.qll new file mode 100644 index 00000000000..a3bd7d158c2 --- /dev/null +++ b/java/ql/lib/semmle/code/java/dataflow/ExternalFlowConfiguration.qll @@ -0,0 +1,31 @@ +/** + * INTERNAL use only. This is an experimental API subject to change without notice. + * + * This module provides extensible predicates for configuring which kinds of MaD models + * are applicable to generic queries. + */ + +private import ExternalFlowExtensions + +/** + * Holds if the specified kind of source model is supported for the current query. + */ +extensible private predicate supportedThreatModels(string kind); + +/** + * Holds if the specified kind of source model is containted within the specified group. + */ +extensible private predicate threatModelGrouping(string kind, string group); + +/** + * Gets the threat models that are direct descendants of the specified kind/group. + */ +private string getChildThreatModel(string group) { threatModelGrouping(result, group) } + +/** + * Holds if the source model kind `kind` is relevant for generic queries + * under the current threat model configuration. + */ +predicate sourceModelKindConfig(string kind) { + exists(string group | supportedThreatModels(group) and kind = getChildThreatModel*(group)) +} diff --git a/java/ql/lib/semmle/code/java/dataflow/FlowSummary.qll b/java/ql/lib/semmle/code/java/dataflow/FlowSummary.qll index e45ba0be27e..b0e342ab320 100644 --- a/java/ql/lib/semmle/code/java/dataflow/FlowSummary.qll +++ b/java/ql/lib/semmle/code/java/dataflow/FlowSummary.qll @@ -101,6 +101,7 @@ abstract class SyntheticCallable extends string { * A module for importing frameworks that define synthetic callables. */ private module SyntheticCallables { + private import semmle.code.java.dispatch.WrappedInvocation private import semmle.code.java.frameworks.android.Intent private import semmle.code.java.frameworks.Stream } @@ -170,6 +171,8 @@ class SummarizedCallableBase extends TSummarizedCallableBase { } } +class Provenance = Impl::Public::Provenance; + class SummarizedCallable = Impl::Public::SummarizedCallable; class NeutralCallable = Impl::Public::NeutralCallable; diff --git a/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl.qll b/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl.qll index 807e895b261..30746706e31 100644 --- a/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl.qll +++ b/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl.qll @@ -1,3 +1,3 @@ private import DataFlowImplSpecific -private import codeql.dataflow.DataFlowImpl +private import codeql.dataflow.internal.DataFlowImpl import MakeImpl diff --git a/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplCommon.qll b/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplCommon.qll index 44a3d3e4ddf..2118572f779 100644 --- a/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplCommon.qll +++ b/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplCommon.qll @@ -1,3 +1,3 @@ private import DataFlowImplSpecific -private import codeql.dataflow.DataFlowImplCommon +private import codeql.dataflow.internal.DataFlowImplCommon import MakeImplCommon diff --git a/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplSpecific.qll b/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplSpecific.qll index 1665139f999..5ddea224875 100644 --- a/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplSpecific.qll +++ b/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplSpecific.qll @@ -2,7 +2,7 @@ * Provides Java-specific definitions for use in the data flow library. */ -private import codeql.dataflow.DataFlowParameter +private import codeql.dataflow.DataFlow module Private { import DataFlowPrivate @@ -13,7 +13,7 @@ module Public { import DataFlowUtil } -module JavaDataFlow implements DataFlowParameter { +module JavaDataFlow implements InputSig { import Private import Public diff --git a/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowNodes.qll b/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowNodes.qll index 8c44e7df5b0..7ee703808e2 100644 --- a/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowNodes.qll +++ b/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowNodes.qll @@ -55,7 +55,8 @@ private module Cached { ) } or TFlowSummaryNode(FlowSummaryImpl::Private::SummaryNode sn) or - TFieldValueNode(Field f) + TFieldValueNode(Field f) or + TCaptureNode(CaptureFlow::SynthesizedCaptureNode cn) cached newtype TContent = @@ -64,6 +65,7 @@ private module Cached { TCollectionContent() or TMapKeyContent() or TMapValueContent() or + TCapturedVariableContent(CapturedVariable v) or TSyntheticFieldContent(SyntheticField s) cached @@ -73,6 +75,7 @@ private module Cached { TCollectionContentApprox() or TMapKeyContentApprox() or TMapValueContentApprox() or + TCapturedVariableContentApprox(CapturedVariable v) or TSyntheticFieldApproxContent() } @@ -127,6 +130,8 @@ module Public { or result = this.(ImplicitPostUpdateNode).getPreUpdateNode().getType() or + result = this.(CaptureNode).getTypeImpl() + or result = this.(FieldValueNode).getField().getType() } @@ -372,6 +377,7 @@ module Private { result.asCallable() = n.(MallocNode).getClassInstanceExpr().getEnclosingCallable() or result = nodeGetEnclosingCallable(n.(ImplicitPostUpdateNode).getPreUpdateNode()) or result.asSummarizedCallable() = n.(FlowSummaryNode).getSummarizedCallable() or + result.asCallable() = n.(CaptureNode).getSynthesizedCaptureNode().getEnclosingCallable() or result.asFieldScope() = n.(FieldValueNode).getField() } @@ -491,6 +497,28 @@ module Private { c.asSummarizedCallable() = this.getSummarizedCallable() and pos = this.getPosition() } } + + /** + * A synthesized data flow node representing a closure object that tracks + * captured variables. + */ + class CaptureNode extends Node, TCaptureNode { + private CaptureFlow::SynthesizedCaptureNode cn; + + CaptureNode() { this = TCaptureNode(cn) } + + CaptureFlow::SynthesizedCaptureNode getSynthesizedCaptureNode() { result = cn } + + override Location getLocation() { result = cn.getLocation() } + + override string toString() { result = cn.toString() } + + Type getTypeImpl() { + exists(Variable v | cn.isVariableAccess(v) and result = v.getType()) + or + cn.isInstanceAccess() and result = cn.getEnclosingCallable().getDeclaringType() + } + } } private import Private @@ -520,3 +548,14 @@ private class SummaryPostUpdateNode extends FlowSummaryNode, PostUpdateNode { override Node getPreUpdateNode() { result = pre } } + +private class CapturePostUpdateNode extends PostUpdateNode, CaptureNode { + private CaptureNode pre; + + CapturePostUpdateNode() { + CaptureFlow::capturePostUpdateNode(this.getSynthesizedCaptureNode(), + pre.getSynthesizedCaptureNode()) + } + + override Node getPreUpdateNode() { result = pre } +} diff --git a/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowPrivate.qll b/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowPrivate.qll index ed4598c8bb6..212232e077a 100644 --- a/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowPrivate.qll +++ b/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowPrivate.qll @@ -10,6 +10,7 @@ private import semmle.code.java.dataflow.FlowSummary private import FlowSummaryImpl as FlowSummaryImpl private import DataFlowImplConsistency private import DataFlowNodes +private import codeql.dataflow.VariableCapture as VariableCapture import DataFlowNodes::Private private newtype TReturnKind = TNormalReturnKind() @@ -51,26 +52,131 @@ private predicate fieldStep(Node node1, Node node2) { ) } -/** - * Holds if data can flow from `node1` to `node2` through variable capture. - */ -private predicate variableCaptureStep(Node node1, ExprNode node2) { - exists(SsaImplicitInit closure, SsaVariable captured | - closure.captures(captured) and - node2.getExpr() = closure.getAFirstUse() - | - node1.asExpr() = captured.getAUse() - or - not exists(captured.getAUse()) and - exists(SsaVariable capturedDef | capturedDef = captured.getAnUltimateDefinition() | - capturedDef.(SsaImplicitInit).isParameterDefinition(node1.asParameter()) or - capturedDef.(SsaExplicitUpdate).getDefiningExpr().(VariableAssign).getSource() = - node1.asExpr() or - capturedDef.(SsaExplicitUpdate).getDefiningExpr().(AssignOp) = node1.asExpr() - ) +private predicate closureFlowStep(Expr e1, Expr e2) { + simpleAstFlowStep(e1, e2) + or + exists(SsaVariable v | + v.getAUse() = e2 and + v.getAnUltimateDefinition().(SsaExplicitUpdate).getDefiningExpr().(VariableAssign).getSource() = + e1 ) } +private module CaptureInput implements VariableCapture::InputSig { + private import java as J + + class Location = J::Location; + + class BasicBlock instanceof J::BasicBlock { + string toString() { result = super.toString() } + + Callable getEnclosingCallable() { result = super.getEnclosingCallable() } + + Location getLocation() { result = super.getLocation() } + } + + BasicBlock getImmediateBasicBlockDominator(BasicBlock bb) { bbIDominates(result, bb) } + + BasicBlock getABasicBlockSuccessor(BasicBlock bb) { + result = bb.(J::BasicBlock).getABBSuccessor() + } + + //TODO: support capture of `this` in lambdas + class CapturedVariable instanceof LocalScopeVariable { + CapturedVariable() { + 2 <= + strictcount(J::Callable c | + c = this.getCallable() or c = this.getAnAccess().getEnclosingCallable() + ) + } + + string toString() { result = super.toString() } + + Callable getCallable() { result = super.getCallable() } + + Location getLocation() { result = super.getLocation() } + } + + class CapturedParameter extends CapturedVariable instanceof Parameter { } + + class Expr instanceof J::Expr { + string toString() { result = super.toString() } + + Location getLocation() { result = super.getLocation() } + + predicate hasCfgNode(BasicBlock bb, int i) { this = bb.(J::BasicBlock).getNode(i) } + } + + class VariableWrite extends Expr instanceof VariableUpdate { + CapturedVariable v; + + VariableWrite() { super.getDestVar() = v } + + CapturedVariable getVariable() { result = v } + + Expr getSource() { + result = this.(VariableAssign).getSource() or + result = this.(AssignOp) + } + } + + class VariableRead extends Expr instanceof RValue { + CapturedVariable v; + + VariableRead() { super.getVariable() = v } + + CapturedVariable getVariable() { result = v } + } + + class ClosureExpr extends Expr instanceof ClassInstanceExpr { + NestedClass nc; + + ClosureExpr() { + nc.(AnonymousClass).getClassInstanceExpr() = this + or + nc instanceof LocalClass and + super.getConstructedType().getASourceSupertype*().getSourceDeclaration() = nc + } + + predicate hasBody(Callable body) { nc.getACallable() = body } + + predicate hasAliasedAccess(Expr f) { closureFlowStep+(this, f) and not closureFlowStep(f, _) } + } + + class Callable extends J::Callable { + predicate isConstructor() { this instanceof Constructor } + } +} + +class CapturedVariable = CaptureInput::CapturedVariable; + +class CapturedParameter = CaptureInput::CapturedParameter; + +module CaptureFlow = VariableCapture::Flow; + +private CaptureFlow::ClosureNode asClosureNode(Node n) { + result = n.(CaptureNode).getSynthesizedCaptureNode() or + result.(CaptureFlow::ExprNode).getExpr() = n.asExpr() or + result.(CaptureFlow::ExprPostUpdateNode).getExpr() = + n.(PostUpdateNode).getPreUpdateNode().asExpr() or + result.(CaptureFlow::ParameterNode).getParameter() = n.asParameter() or + result.(CaptureFlow::ThisParameterNode).getCallable() = n.(InstanceParameterNode).getCallable() or + exprNode(result.(CaptureFlow::MallocNode).getClosureExpr()).(PostUpdateNode).getPreUpdateNode() = + n +} + +private predicate captureStoreStep(Node node1, CapturedVariableContent c, Node node2) { + CaptureFlow::storeStep(asClosureNode(node1), c.getVariable(), asClosureNode(node2)) +} + +private predicate captureReadStep(Node node1, CapturedVariableContent c, Node node2) { + CaptureFlow::readStep(asClosureNode(node1), c.getVariable(), asClosureNode(node2)) +} + +predicate captureValueStep(Node node1, Node node2) { + CaptureFlow::localFlowStep(asClosureNode(node1), asClosureNode(node2)) +} + /** * Holds if data can flow from `node1` to `node2` through a field or * variable capture. @@ -78,10 +184,6 @@ private predicate variableCaptureStep(Node node1, ExprNode node2) { predicate jumpStep(Node node1, Node node2) { fieldStep(node1, node2) or - variableCaptureStep(node1, node2) - or - variableCaptureStep(node1.(PostUpdateNode).getPreUpdateNode(), node2) - or any(AdditionalValueStep a).step(node1, node2) and node1.getEnclosingCallable() != node2.getEnclosingCallable() or @@ -117,6 +219,8 @@ predicate storeStep(Node node1, ContentSet f, Node node2) { or FlowSummaryImpl::Private::Steps::summaryStoreStep(node1.(FlowSummaryNode).getSummaryNode(), f, node2.(FlowSummaryNode).getSummaryNode()) + or + captureStoreStep(node1, f, node2) } /** @@ -149,6 +253,8 @@ predicate readStep(Node node1, ContentSet f, Node node2) { or FlowSummaryImpl::Private::Steps::summaryReadStep(node1.(FlowSummaryNode).getSummaryNode(), f, node2.(FlowSummaryNode).getSummaryNode()) + or + captureReadStep(node1, f, node2) } /** @@ -231,19 +337,29 @@ private newtype TDataFlowCallable = TSummarizedCallable(SummarizedCallable c) or TFieldScope(Field f) +/** + * A callable or scope enclosing some number of data flow nodes. This can either + * be a source callable, a synthesized callable for which we have a summary + * model, or a synthetic scope for a field value node. + */ class DataFlowCallable extends TDataFlowCallable { + /** Gets the source callable corresponding to this callable, if any. */ Callable asCallable() { this = TSrcCallable(result) } + /** Gets the summary model callable corresponding to this callable, if any. */ SummarizedCallable asSummarizedCallable() { this = TSummarizedCallable(result) } + /** Gets the field corresponding to this callable, if it is a field value scope. */ Field asFieldScope() { this = TFieldScope(result) } + /** Gets a textual representation of this callable. */ string toString() { result = this.asCallable().toString() or result = "Synthetic: " + this.asSummarizedCallable().toString() or result = "Field scope: " + this.asFieldScope().toString() } + /** Gets the location of this callable. */ Location getLocation() { result = this.asCallable().getLocation() or result = this.asSummarizedCallable().getLocation() or @@ -406,6 +522,8 @@ predicate additionalLambdaFlowStep(Node nodeFrom, Node nodeTo, boolean preserves */ predicate allowParameterReturnInSelf(ParameterNode p) { FlowSummaryImpl::Private::summaryAllowParameterReturnInSelf(p) + or + CaptureFlow::heuristicAllowInstanceParameterReturnInSelf(p.(InstanceParameterNode).getCallable()) } /** An approximated `Content`. */ @@ -447,6 +565,10 @@ ContentApprox getContentApprox(Content c) { or c instanceof MapValueContent and result = TMapValueContentApprox() or + exists(CapturedVariable v | + c = TCapturedVariableContent(v) and result = TCapturedVariableContentApprox(v) + ) + or c instanceof SyntheticFieldContent and result = TSyntheticFieldApproxContent() } diff --git a/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowUtil.qll b/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowUtil.qll index 29758d4b972..080724d8f0c 100644 --- a/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowUtil.qll +++ b/java/ql/lib/semmle/code/java/dataflow/internal/DataFlowUtil.qll @@ -135,6 +135,30 @@ private module Cached { import Cached +private predicate capturedVariableRead(Node n) { + n.asExpr().(RValue).getVariable() instanceof CapturedVariable +} + +/** + * Holds if there is a data flow step from `e1` to `e2` that only steps from + * child to parent in the AST. + */ +predicate simpleAstFlowStep(Expr e1, Expr e2) { + e2.(CastingExpr).getExpr() = e1 + or + e2.(ChooseExpr).getAResultExpr() = e1 + or + e2.(AssignExpr).getSource() = e1 + or + e2.(ArrayCreationExpr).getInit() = e1 + or + e2 = any(StmtExpr stmtExpr | e1 = stmtExpr.getResultExpr()) + or + e2 = any(NotNullExpr nne | e1 = nne.getExpr()) + or + e2.(WhenExpr).getBranch(_).getAResult() = e1 +} + private predicate simpleLocalFlowStep0(Node node1, Node node2) { TaintTrackingUtil::forceCachingInSameStage() and // Variable flow steps through adjacent def-use and use-use pairs. @@ -142,39 +166,31 @@ private predicate simpleLocalFlowStep0(Node node1, Node node2) { upd.getDefiningExpr().(VariableAssign).getSource() = node1.asExpr() or upd.getDefiningExpr().(AssignOp) = node1.asExpr() | - node2.asExpr() = upd.getAFirstUse() + node2.asExpr() = upd.getAFirstUse() and + not capturedVariableRead(node2) ) or exists(SsaImplicitInit init | init.isParameterDefinition(node1.asParameter()) and - node2.asExpr() = init.getAFirstUse() + node2.asExpr() = init.getAFirstUse() and + not capturedVariableRead(node2) ) or adjacentUseUse(node1.asExpr(), node2.asExpr()) and not exists(FieldRead fr | hasNonlocalValue(fr) and fr.getField().isStatic() and fr = node1.asExpr() ) and - not FlowSummaryImpl::Private::Steps::prohibitsUseUseFlow(node1, _) + not FlowSummaryImpl::Private::Steps::prohibitsUseUseFlow(node1, _) and + not capturedVariableRead(node2) or ThisFlow::adjacentThisRefs(node1, node2) or - adjacentUseUse(node1.(PostUpdateNode).getPreUpdateNode().asExpr(), node2.asExpr()) + adjacentUseUse(node1.(PostUpdateNode).getPreUpdateNode().asExpr(), node2.asExpr()) and + not capturedVariableRead(node2) or ThisFlow::adjacentThisRefs(node1.(PostUpdateNode).getPreUpdateNode(), node2) or - node2.asExpr().(CastingExpr).getExpr() = node1.asExpr() - or - node2.asExpr().(ChooseExpr).getAResultExpr() = node1.asExpr() - or - node2.asExpr().(AssignExpr).getSource() = node1.asExpr() - or - node2.asExpr().(ArrayCreationExpr).getInit() = node1.asExpr() - or - node2.asExpr() = any(StmtExpr stmtExpr | node1.asExpr() = stmtExpr.getResultExpr()) - or - node2.asExpr() = any(NotNullExpr nne | node1.asExpr() = nne.getExpr()) - or - node2.asExpr().(WhenExpr).getBranch(_).getAResult() = node1.asExpr() + simpleAstFlowStep(node1.asExpr(), node2.asExpr()) or exists(MethodAccess ma, ValuePreservingMethod m, int argNo | ma.getCallee().getSourceDeclaration() = m and m.returnsValue(argNo) @@ -185,6 +201,8 @@ private predicate simpleLocalFlowStep0(Node node1, Node node2) { or FlowSummaryImpl::Private::Steps::summaryLocalStep(node1.(FlowSummaryNode).getSummaryNode(), node2.(FlowSummaryNode).getSummaryNode(), true) + or + captureValueStep(node1, node2) } /** @@ -256,6 +274,19 @@ class MapValueContent extends Content, TMapValueContent { override string toString() { result = "" } } +/** A captured variable. */ +class CapturedVariableContent extends Content, TCapturedVariableContent { + CapturedVariable v; + + CapturedVariableContent() { this = TCapturedVariableContent(v) } + + CapturedVariable getVariable() { result = v } + + override DataFlowType getType() { result = getErasedRepr(v.(Variable).getType()) } + + override string toString() { result = v.toString() } +} + /** A reference through a synthetic instance field. */ class SyntheticFieldContent extends Content, TSyntheticFieldContent { SyntheticField s; diff --git a/java/ql/lib/semmle/code/java/dispatch/WrappedInvocation.qll b/java/ql/lib/semmle/code/java/dispatch/WrappedInvocation.qll index ec1eea5f93c..77f3ff11291 100644 --- a/java/ql/lib/semmle/code/java/dispatch/WrappedInvocation.qll +++ b/java/ql/lib/semmle/code/java/dispatch/WrappedInvocation.qll @@ -58,3 +58,37 @@ Method getRunnerTarget(MethodAccess ma) { result.overridesOrInstantiates*(runmethod) ) } + +import semmle.code.java.dataflow.FlowSummary +import semmle.code.java.dataflow.internal.FlowSummaryImplSpecific as ImplSpecific + +private predicate mayInvokeCallback(SrcMethod m, int n) { + m.getParameterType(n).(RefType).getSourceDeclaration() instanceof FunctionalInterface and + (not m.fromSource() or m.isNative() or m.getFile().getAbsolutePath().matches("%/test/stubs/%")) +} + +private class SummarizedCallableWithCallback extends SummarizedCallable { + private int pos; + + SummarizedCallableWithCallback() { mayInvokeCallback(this.asCallable(), pos) } + + override predicate propagatesFlow( + SummaryComponentStack input, SummaryComponentStack output, boolean preservesValue + ) { + input = SummaryComponentStack::argument(pos) and + output = SummaryComponentStack::push(SummaryComponent::parameter(-1), input) and + preservesValue = true + } + + override predicate hasProvenance(Provenance provenance) { provenance = "hq-generated" } +} + +private class RequiredComponentStackForCallback extends RequiredSummaryComponentStack { + override predicate required(SummaryComponent head, SummaryComponentStack tail) { + exists(int pos | + mayInvokeCallback(_, pos) and + head = SummaryComponent::parameter(-1) and + tail = SummaryComponentStack::argument(pos) + ) + } +} diff --git a/java/ql/lib/semmle/code/java/security/RandomDataSource.qll b/java/ql/lib/semmle/code/java/security/RandomDataSource.qll index e52c6a3b7eb..29c980beb5f 100644 --- a/java/ql/lib/semmle/code/java/security/RandomDataSource.qll +++ b/java/ql/lib/semmle/code/java/security/RandomDataSource.qll @@ -103,7 +103,7 @@ class StdlibRandomSource extends RandomDataSource { } override Expr getOutput() { - if m.hasName("getBytes") then result = this.getArgument(0) else result = this + if m.hasName("nextBytes") then result = this.getArgument(0) else result = this } } diff --git a/java/ql/src/Metrics/Summaries/TopJdkApis.qll b/java/ql/src/Metrics/Summaries/TopJdkApis.qll index 420c231c003..20540f4f619 100644 --- a/java/ql/src/Metrics/Summaries/TopJdkApis.qll +++ b/java/ql/src/Metrics/Summaries/TopJdkApis.qll @@ -307,6 +307,7 @@ class TopJdkApi extends SummarizedCallableBase { predicate hasManualMadModel() { this.hasManualSummary() or this.hasManualNeutral() } /* * Note: the following top JDK APIs are not modeled with MaD: + * `java.lang.Runnable#run()`: specialised lambda flow * `java.lang.String#valueOf(Object)`: a complex case; an alias for `Object.toString`, except the dispatch is hidden * `java.lang.System#getProperty(String)`: needs to be modeled by regular CodeQL matching the get and set keys to reduce FPs * `java.lang.System#setProperty(String,String)`: needs to be modeled by regular CodeQL matching the get and set keys to reduce FPs diff --git a/java/ql/src/Telemetry/AutomodelApplicationModeExtractCandidates.ql b/java/ql/src/Telemetry/AutomodelApplicationModeExtractCandidates.ql index 4940b4a741f..49d594dfdef 100644 --- a/java/ql/src/Telemetry/AutomodelApplicationModeExtractCandidates.ql +++ b/java/ql/src/Telemetry/AutomodelApplicationModeExtractCandidates.ql @@ -66,6 +66,7 @@ where // modeled in a MaD model, then it doesn't belong to any additional sink types, and we don't need to reexamine it. not CharacteristicsImpl::isSink(endpoint, _, _) and meta.hasMetadata(endpoint, package, type, subtypes, name, signature, input) and + includeAutomodelCandidate(package, type, name, signature) and // The message is the concatenation of all sink types for which this endpoint is known neither to be a sink nor to be // a non-sink, and we surface only endpoints that have at least one such sink type. message = diff --git a/java/ql/src/Telemetry/AutomodelCandidateFilter.yml b/java/ql/src/Telemetry/AutomodelCandidateFilter.yml new file mode 100644 index 00000000000..52e64d54446 --- /dev/null +++ b/java/ql/src/Telemetry/AutomodelCandidateFilter.yml @@ -0,0 +1,5 @@ +extensions: + - addsTo: + pack: codeql/java-queries + extensible: automodelCandidateFilter + data: [] diff --git a/java/ql/src/Telemetry/AutomodelFrameworkModeExtractCandidates.ql b/java/ql/src/Telemetry/AutomodelFrameworkModeExtractCandidates.ql index e66af08707c..028a27a9bdc 100644 --- a/java/ql/src/Telemetry/AutomodelFrameworkModeExtractCandidates.ql +++ b/java/ql/src/Telemetry/AutomodelFrameworkModeExtractCandidates.ql @@ -30,6 +30,7 @@ where // modeled in a MaD model, then it doesn't belong to any additional sink types, and we don't need to reexamine it. not CharacteristicsImpl::isSink(endpoint, _, _) and meta.hasMetadata(endpoint, package, type, subtypes, name, signature, input, parameterName) and + includeAutomodelCandidate(package, type, name, signature) and // The message is the concatenation of all sink types for which this endpoint is known neither to be a sink nor to be // a non-sink, and we surface only endpoints that have at least one such sink type. message = diff --git a/java/ql/src/Telemetry/AutomodelJavaUtil.qll b/java/ql/src/Telemetry/AutomodelJavaUtil.qll index 65be12ce1f9..a224fc291a2 100644 --- a/java/ql/src/Telemetry/AutomodelJavaUtil.qll +++ b/java/ql/src/Telemetry/AutomodelJavaUtil.qll @@ -66,3 +66,24 @@ boolean considerSubtypes(Callable callable) { then result = false else result = true } + +/** + * Holds if the given package, type, name and signature is a candidate for automodeling. + * + * This predicate is extensible, so that different endpoints can be selected at runtime. + */ +extensible predicate automodelCandidateFilter( + string package, string type, string name, string signature +); + +/** + * Holds if the given package, type, name and signature is a candidate for automodeling. + * + * This relies on an extensible predicate, and if that is not supplied then + * all endpoints are considered candidates. + */ +bindingset[package, type, name, signature] +predicate includeAutomodelCandidate(string package, string type, string name, string signature) { + not automodelCandidateFilter(_, _, _, _) or + automodelCandidateFilter(package, type, name, signature) +} diff --git a/java/ql/src/qlpack.yml b/java/ql/src/qlpack.yml index 76f4fcc7797..0e7c892f469 100644 --- a/java/ql/src/qlpack.yml +++ b/java/ql/src/qlpack.yml @@ -12,4 +12,5 @@ dependencies: codeql/util: ${workspace} dataExtensions: - Telemetry/ExtractorInformation.yml + - Telemetry/AutomodelCandidateFilter.yml warnOnImplicitThis: true diff --git a/java/ql/test/ext/TopJdkApis/TopJdkApisTest.expected b/java/ql/test/ext/TopJdkApis/TopJdkApisTest.expected index 2e0ace91209..7bdac8e59ce 100644 --- a/java/ql/test/ext/TopJdkApis/TopJdkApisTest.expected +++ b/java/ql/test/ext/TopJdkApis/TopJdkApisTest.expected @@ -1,3 +1,4 @@ +| java.lang.Runnable#run() | no manual model | | java.lang.String#valueOf(Object) | no manual model | | java.lang.System#getProperty(String) | no manual model | | java.lang.System#setProperty(String,String) | no manual model | diff --git a/java/ql/test/library-tests/dataflow/capture/B.java b/java/ql/test/library-tests/dataflow/capture/B.java new file mode 100644 index 00000000000..0d43a0dba17 --- /dev/null +++ b/java/ql/test/library-tests/dataflow/capture/B.java @@ -0,0 +1,251 @@ +import java.util.*; +import java.util.function.*; + +public class B { + static String source(String label) { return null; } + + static void sink(String s) { } + + static void test1() { + List l1 = new ArrayList<>(); + l1.add(source("L")); + List l2 = new ArrayList<>(); + l1.forEach(e -> l2.add(e)); + sink(l2.get(0)); // $ hasValueFlow=L + } + + String bf1; + String bf2; + + void test2() { + B other = new B(); + Consumer f = x -> { this.bf1 = x; bf2 = x; other.bf1 = x; }; + + // no flow + sink(bf1); + sink(this.bf2); + sink(other.bf1); + sink(other.bf2); + + f.accept(source("T")); + + sink(bf1); // $ MISSING: hasValueFlow=T + sink(this.bf2); // $ MISSING: hasValueFlow=T + sink(other.bf1); // $ hasValueFlow=T + sink(other.bf2); + } + + static void convert(Map inp, Map out) { + inp.forEach((key, value) -> { out.put(key, value); }); + } + + void test3() { + HashMap m1 = new HashMap<>(); + HashMap m2 = new HashMap<>(); + m1.put(source("Key"), source("Value")); + convert(m1, m2); + m2.forEach((k, v) -> { + sink(k); // $ hasValueFlow=Key + sink(v); // $ hasValueFlow=Value + }); + } + + String elem; + + void testParamIn1() { + elem = source("pin.This.elem"); + testParamIn2(source("pin.Arg")); + } + + void testParamIn2(String param) { + Runnable r = () -> { + sink(elem); // $ MISSING: hasValueFlow=pin.This.elem + sink(this.elem); // $ MISSING: hasValueFlow=pin.This.elem + sink(param); // $ hasValueFlow=pin.Arg + }; + r.run(); + } + + void testParamOut1() { + B other = new B(); + testParamOut2(other); + sink(elem); // $ MISSING: hasValueFlow=pout.This.elem + sink(this.elem); // $ MISSING: hasValueFlow=pout.This.elem + sink(other.elem); // $ hasValueFlow=pout.param + } + + void testParamOut2(B param) { + Runnable r = () -> { + this.elem = source("pout.This.elem"); + param.elem = source("pout.param"); + }; + r.run(); + } + + void testCrossLambda() { + B b = new B(); + Runnable sink1 = () -> { sink(b.elem); }; + Runnable sink2 = () -> { sink(b.elem); }; // $ hasValueFlow=src + Runnable src = () -> { b.elem = source("src"); }; + doRun(sink1); + doRun(src); + doRun(sink2); + } + + void doRun(Runnable r) { + r.run(); + } + + void testNested() { + List l1 = new ArrayList<>(); + List> l2 = new ArrayList<>(); + l1.add(source("nest.out")); + l2.add(l1); + String s = source("nest.in"); + List out1 = new ArrayList<>(); + List out2 = new ArrayList<>(); + l2.forEach(l -> l.forEach(x -> { + sink(s); // $ hasValueFlow=nest.in + out1.add(x); + out2.add(s); + })); + sink(out1.get(0)); // $ hasValueFlow=nest.out + sink(out2.get(0)); // $ hasValueFlow=nest.in + } + + static interface TwoRuns { + void run1(); + void run2(); + } + + void testAnonymousClass() { + List l1 = new ArrayList<>(); + List l2 = new ArrayList<>(); + TwoRuns r = new TwoRuns() { + @Override + public void run1() { + l1.add(source("run1")); + } + @Override + public void run2() { + l2.add(l1.get(0)); + } + }; + r.run2(); + sink(l2.get(0)); + r.run1(); + r.run2(); + sink(l2.get(0)); // $ hasValueFlow=run1 + } + + void testLocalClass1() { + String s = source("local1"); + class MyLocal { + String f; + MyLocal() { this.f = s; } + String getF() { return this.f; } + } + MyLocal m = new MyLocal(); + sink(m.getF()); // $ hasValueFlow=local1 + } + + void testLocalClass2() { + String s1 = source("s1"); + String s2 = source("s2"); + List l = new ArrayList<>(); + class MyLocal { + String f; + MyLocal() { + this.f = s1; + sink(s2); // $ hasValueFlow=s2 + } + void test() { + sink(f); // $ hasValueFlow=s1 + sink(s2); // $ hasValueFlow=s2 + } + void add(String s) { + l.add(s); + } + String get() { + return l.get(0); + } + } + MyLocal m1 = new MyLocal(); + MyLocal m2 = new MyLocal(); + m1.test(); + sink(m1.get()); + m1.add(source("m1.add")); + sink(m2.get()); // $ hasValueFlow=m1.add + } + + void testComplex() { + String s = source("complex"); + class LocalComplex { + Supplier getBoxSupplier() { + return new Supplier() { + StringBox b = new StringBox(); + @Override + public StringBox get() { return b; } + }; + } + class StringBox { + String get() { + // capture through regular nested class inside local nested class + return s; + } + } + } + LocalComplex lc = new LocalComplex(); + sink(lc.getBoxSupplier().get().get()); // $ MISSING: hasValueFlow=complex + } + + void testCapturedLambda() { + String s = source("double.capture.in"); + List out = new ArrayList<>(); + Runnable r1 = () -> { + sink(s); // $ hasValueFlow=double.capture.in + out.add(source("double.capture.out")); + }; + Runnable r2 = () -> { + r1.run(); + }; + r2.run(); + sink(out.get(0)); // $ MISSING: hasValueFlow=double.capture.out + } + + void testEnhancedForStmtCapture() { + List l = new ArrayList<>(); + l.add(source("list")); + String[] a = new String[] { source("array") }; + for (String x : l) { + Runnable r = () -> sink(x); // $ MISSING: hasValueFlow=list + r.run(); + } + for (String x : a) { + Runnable r = () -> sink(x); // $ MISSING: hasValueFlow=array + r.run(); + } + } + + void testDoubleCall() { + String s = source("src"); + List l = new ArrayList<>(); + List l2 = new ArrayList<>(); + class MyLocal2 { + MyLocal2() { + sink(l.get(0)); // no flow + sink(l2.get(0)); // no flow + l.add(s); + } + void run() { + l2.add(l.get(0)); + } + } + // The ClassInstanceExpr has two calls in the same cfg node: + // First the constructor call for which it is the postupdate, + // and then as instance argument to the run call. + new MyLocal2().run(); + sink(l.get(0)); // $ hasValueFlow=src + sink(l2.get(0)); // $ hasValueFlow=src + } +} diff --git a/java/ql/test/library-tests/dataflow/capture/inlinetest.expected b/java/ql/test/library-tests/dataflow/capture/inlinetest.expected new file mode 100644 index 00000000000..48de9172b36 --- /dev/null +++ b/java/ql/test/library-tests/dataflow/capture/inlinetest.expected @@ -0,0 +1,2 @@ +failures +testFailures diff --git a/java/ql/test/library-tests/dataflow/capture/inlinetest.ql b/java/ql/test/library-tests/dataflow/capture/inlinetest.ql new file mode 100644 index 00000000000..50e3f8d2f7d --- /dev/null +++ b/java/ql/test/library-tests/dataflow/capture/inlinetest.ql @@ -0,0 +1,2 @@ +import TestUtilities.InlineFlowTest +import DefaultFlowTest diff --git a/java/ql/test/library-tests/dataflow/capture/test.expected b/java/ql/test/library-tests/dataflow/capture/test.expected index aca9fe129a9..6a04b82f223 100644 --- a/java/ql/test/library-tests/dataflow/capture/test.expected +++ b/java/ql/test/library-tests/dataflow/capture/test.expected @@ -1,26 +1,93 @@ -| A.java:14:14:14:16 | "A" | A.java:14:14:14:16 | "A" | -| A.java:14:14:14:16 | "A" | A.java:15:16:15:22 | get(...) | -| A.java:14:14:14:16 | "A" | A.java:18:8:18:15 | p | -| A.java:14:14:14:16 | "A" | A.java:32:26:32:26 | p | -| A.java:21:11:21:13 | "B" | A.java:15:16:15:22 | get(...) | -| A.java:21:11:21:13 | "B" | A.java:21:7:21:13 | ...=... | -| A.java:21:11:21:13 | "B" | A.java:21:11:21:13 | "B" | -| A.java:21:11:21:13 | "B" | A.java:33:26:33:26 | s | -| A.java:23:11:23:13 | "C" | A.java:15:16:15:22 | get(...) | -| A.java:23:11:23:13 | "C" | A.java:23:7:23:13 | ...=... | -| A.java:23:11:23:13 | "C" | A.java:23:11:23:13 | "C" | -| A.java:23:11:23:13 | "C" | A.java:33:26:33:26 | s | -| A.java:25:22:25:24 | "D" | A.java:4:9:4:16 | e | -| A.java:25:22:25:24 | "D" | A.java:4:21:4:28 | ...=... | -| A.java:25:22:25:24 | "D" | A.java:4:28:4:28 | e | -| A.java:25:22:25:24 | "D" | A.java:6:31:6:34 | elem | -| A.java:25:22:25:24 | "D" | A.java:15:16:15:22 | get(...) | -| A.java:25:22:25:24 | "D" | A.java:25:22:25:24 | "D" | -| A.java:25:22:25:24 | "D" | A.java:34:26:34:37 | getElem(...) | -| A.java:27:16:27:18 | "E" | A.java:5:18:5:25 | e | -| A.java:27:16:27:18 | "E" | A.java:5:30:5:37 | ...=... | -| A.java:27:16:27:18 | "E" | A.java:5:37:5:37 | e | -| A.java:27:16:27:18 | "E" | A.java:6:31:6:34 | elem | -| A.java:27:16:27:18 | "E" | A.java:15:16:15:22 | get(...) | -| A.java:27:16:27:18 | "E" | A.java:27:16:27:18 | "E" | -| A.java:27:16:27:18 | "E" | A.java:35:26:35:37 | getElem(...) | +| A.java:14:14:14:16 | "A" : String | A.java:14:11:14:20 | f2(...) : new A(...) { ... } [p] | +| A.java:14:14:14:16 | "A" : String | A.java:15:16:15:16 | a : new A(...) { ... } [p] | +| A.java:14:14:14:16 | "A" : String | A.java:15:16:15:22 | get(...) : String | +| A.java:14:14:14:16 | "A" : String | A.java:18:8:18:15 | p : String | +| A.java:14:14:14:16 | "A" : String | A.java:28:11:38:5 | new (...) : new A(...) { ... } [p] | +| A.java:14:14:14:16 | "A" : String | A.java:28:11:38:5 | p : String | +| A.java:14:14:14:16 | "A" : String | A.java:30:14:30:16 | parameter this : new A(...) { ... } [p] | +| A.java:14:14:14:16 | "A" : String | A.java:31:17:31:17 | this : new A(...) { ... } [p] | +| A.java:14:14:14:16 | "A" : String | A.java:32:26:32:26 | p : String | +| A.java:14:14:14:16 | "A" : String | A.java:32:26:32:26 | this : new A(...) { ... } [p] | +| A.java:14:14:14:16 | "A" : String | A.java:33:26:33:26 | this : new A(...) { ... } [p] | +| A.java:14:14:14:16 | "A" : String | A.java:34:26:34:27 | this : new A(...) { ... } [p] | +| A.java:14:14:14:16 | "A" : String | A.java:35:26:35:27 | this : new A(...) { ... } [p] | +| A.java:14:14:14:16 | "A" : String | A.java:39:12:39:12 | a : new A(...) { ... } [p] | +| A.java:14:14:14:16 | "A" : String | A.java:39:12:39:12 | p : String | +| A.java:21:11:21:13 | "B" : String | A.java:14:11:14:20 | f2(...) : new A(...) { ... } [String s] | +| A.java:21:11:21:13 | "B" : String | A.java:15:16:15:16 | a : new A(...) { ... } [String s] | +| A.java:21:11:21:13 | "B" : String | A.java:15:16:15:22 | get(...) : String | +| A.java:21:11:21:13 | "B" : String | A.java:21:7:21:13 | ...=... : String | +| A.java:21:11:21:13 | "B" : String | A.java:25:5:25:26 | phi(String s) : String | +| A.java:21:11:21:13 | "B" : String | A.java:28:11:38:5 | String s : String | +| A.java:21:11:21:13 | "B" : String | A.java:28:11:38:5 | new (...) : new A(...) { ... } [String s] | +| A.java:21:11:21:13 | "B" : String | A.java:30:14:30:16 | parameter this : new A(...) { ... } [String s] | +| A.java:21:11:21:13 | "B" : String | A.java:31:17:31:17 | this : new A(...) { ... } [String s] | +| A.java:21:11:21:13 | "B" : String | A.java:32:26:32:26 | this : new A(...) { ... } [String s] | +| A.java:21:11:21:13 | "B" : String | A.java:33:26:33:26 | s : String | +| A.java:21:11:21:13 | "B" : String | A.java:33:26:33:26 | this : new A(...) { ... } [String s] | +| A.java:21:11:21:13 | "B" : String | A.java:34:26:34:27 | this : new A(...) { ... } [String s] | +| A.java:21:11:21:13 | "B" : String | A.java:35:26:35:27 | this : new A(...) { ... } [String s] | +| A.java:21:11:21:13 | "B" : String | A.java:39:12:39:12 | String s : String | +| A.java:21:11:21:13 | "B" : String | A.java:39:12:39:12 | a : new A(...) { ... } [String s] | +| A.java:23:11:23:13 | "C" : String | A.java:14:11:14:20 | f2(...) : new A(...) { ... } [String s] | +| A.java:23:11:23:13 | "C" : String | A.java:15:16:15:16 | a : new A(...) { ... } [String s] | +| A.java:23:11:23:13 | "C" : String | A.java:15:16:15:22 | get(...) : String | +| A.java:23:11:23:13 | "C" : String | A.java:23:7:23:13 | ...=... : String | +| A.java:23:11:23:13 | "C" : String | A.java:25:5:25:26 | phi(String s) : String | +| A.java:23:11:23:13 | "C" : String | A.java:28:11:38:5 | String s : String | +| A.java:23:11:23:13 | "C" : String | A.java:28:11:38:5 | new (...) : new A(...) { ... } [String s] | +| A.java:23:11:23:13 | "C" : String | A.java:30:14:30:16 | parameter this : new A(...) { ... } [String s] | +| A.java:23:11:23:13 | "C" : String | A.java:31:17:31:17 | this : new A(...) { ... } [String s] | +| A.java:23:11:23:13 | "C" : String | A.java:32:26:32:26 | this : new A(...) { ... } [String s] | +| A.java:23:11:23:13 | "C" : String | A.java:33:26:33:26 | s : String | +| A.java:23:11:23:13 | "C" : String | A.java:33:26:33:26 | this : new A(...) { ... } [String s] | +| A.java:23:11:23:13 | "C" : String | A.java:34:26:34:27 | this : new A(...) { ... } [String s] | +| A.java:23:11:23:13 | "C" : String | A.java:35:26:35:27 | this : new A(...) { ... } [String s] | +| A.java:23:11:23:13 | "C" : String | A.java:39:12:39:12 | String s : String | +| A.java:23:11:23:13 | "C" : String | A.java:39:12:39:12 | a : new A(...) { ... } [String s] | +| A.java:25:22:25:24 | "D" : String | A.java:4:9:4:16 | e : String | +| A.java:25:22:25:24 | "D" : String | A.java:4:21:4:24 | this <.field> [post update] : Box [elem] | +| A.java:25:22:25:24 | "D" : String | A.java:4:21:4:28 | ...=... : String | +| A.java:25:22:25:24 | "D" : String | A.java:4:28:4:28 | e : String | +| A.java:25:22:25:24 | "D" : String | A.java:6:12:6:18 | parameter this : Box [elem] | +| A.java:25:22:25:24 | "D" : String | A.java:6:31:6:34 | elem : String | +| A.java:25:22:25:24 | "D" : String | A.java:6:31:6:34 | this <.field> : Box [elem] | +| A.java:25:22:25:24 | "D" : String | A.java:14:11:14:20 | f2(...) : new A(...) { ... } [Box b1, ... (2)] | +| A.java:25:22:25:24 | "D" : String | A.java:15:16:15:16 | a : new A(...) { ... } [Box b1, ... (2)] | +| A.java:25:22:25:24 | "D" : String | A.java:15:16:15:22 | get(...) : String | +| A.java:25:22:25:24 | "D" : String | A.java:25:14:25:25 | new Box(...) : Box [elem] | +| A.java:25:22:25:24 | "D" : String | A.java:28:11:38:5 | Box b1 : Box [elem] | +| A.java:25:22:25:24 | "D" : String | A.java:28:11:38:5 | new (...) : new A(...) { ... } [Box b1, ... (2)] | +| A.java:25:22:25:24 | "D" : String | A.java:30:14:30:16 | parameter this : new A(...) { ... } [Box b1, ... (2)] | +| A.java:25:22:25:24 | "D" : String | A.java:31:17:31:17 | this : new A(...) { ... } [Box b1, ... (2)] | +| A.java:25:22:25:24 | "D" : String | A.java:32:26:32:26 | this : new A(...) { ... } [Box b1, ... (2)] | +| A.java:25:22:25:24 | "D" : String | A.java:33:26:33:26 | this : new A(...) { ... } [Box b1, ... (2)] | +| A.java:25:22:25:24 | "D" : String | A.java:34:26:34:27 | b1 : Box [elem] | +| A.java:25:22:25:24 | "D" : String | A.java:34:26:34:27 | this : new A(...) { ... } [Box b1, ... (2)] | +| A.java:25:22:25:24 | "D" : String | A.java:34:26:34:37 | getElem(...) : String | +| A.java:25:22:25:24 | "D" : String | A.java:35:26:35:27 | this : new A(...) { ... } [Box b1, ... (2)] | +| A.java:25:22:25:24 | "D" : String | A.java:39:12:39:12 | Box b1 : Box [elem] | +| A.java:25:22:25:24 | "D" : String | A.java:39:12:39:12 | a : new A(...) { ... } [Box b1, ... (2)] | +| A.java:27:16:27:18 | "E" : String | A.java:5:18:5:25 | e : String | +| A.java:27:16:27:18 | "E" : String | A.java:5:30:5:33 | this <.field> [post update] : Box [elem] | +| A.java:27:16:27:18 | "E" : String | A.java:5:30:5:37 | ...=... : String | +| A.java:27:16:27:18 | "E" : String | A.java:5:37:5:37 | e : String | +| A.java:27:16:27:18 | "E" : String | A.java:6:12:6:18 | parameter this : Box [elem] | +| A.java:27:16:27:18 | "E" : String | A.java:6:31:6:34 | elem : String | +| A.java:27:16:27:18 | "E" : String | A.java:6:31:6:34 | this <.field> : Box [elem] | +| A.java:27:16:27:18 | "E" : String | A.java:14:11:14:20 | f2(...) : new A(...) { ... } [Box b2, ... (2)] | +| A.java:27:16:27:18 | "E" : String | A.java:15:16:15:16 | a : new A(...) { ... } [Box b2, ... (2)] | +| A.java:27:16:27:18 | "E" : String | A.java:15:16:15:22 | get(...) : String | +| A.java:27:16:27:18 | "E" : String | A.java:27:5:27:6 | b2 [post update] : Box [elem] | +| A.java:27:16:27:18 | "E" : String | A.java:28:11:38:5 | Box b2 : Box [elem] | +| A.java:27:16:27:18 | "E" : String | A.java:28:11:38:5 | new (...) : new A(...) { ... } [Box b2, ... (2)] | +| A.java:27:16:27:18 | "E" : String | A.java:30:14:30:16 | parameter this : new A(...) { ... } [Box b2, ... (2)] | +| A.java:27:16:27:18 | "E" : String | A.java:31:17:31:17 | this : new A(...) { ... } [Box b2, ... (2)] | +| A.java:27:16:27:18 | "E" : String | A.java:32:26:32:26 | this : new A(...) { ... } [Box b2, ... (2)] | +| A.java:27:16:27:18 | "E" : String | A.java:33:26:33:26 | this : new A(...) { ... } [Box b2, ... (2)] | +| A.java:27:16:27:18 | "E" : String | A.java:34:26:34:27 | this : new A(...) { ... } [Box b2, ... (2)] | +| A.java:27:16:27:18 | "E" : String | A.java:35:26:35:27 | b2 : Box [elem] | +| A.java:27:16:27:18 | "E" : String | A.java:35:26:35:27 | this : new A(...) { ... } [Box b2, ... (2)] | +| A.java:27:16:27:18 | "E" : String | A.java:35:26:35:37 | getElem(...) : String | +| A.java:27:16:27:18 | "E" : String | A.java:39:12:39:12 | Box b2 : Box [elem] | +| A.java:27:16:27:18 | "E" : String | A.java:39:12:39:12 | a : new A(...) { ... } [Box b2, ... (2)] | diff --git a/java/ql/test/library-tests/dataflow/capture/test.ql b/java/ql/test/library-tests/dataflow/capture/test.ql index 448bb030583..da94ff27848 100644 --- a/java/ql/test/library-tests/dataflow/capture/test.ql +++ b/java/ql/test/library-tests/dataflow/capture/test.ql @@ -1,16 +1,23 @@ import java import semmle.code.java.dataflow.DataFlow -StringLiteral src() { result.getCompilationUnit().fromSource() } +StringLiteral src() { + result.getCompilationUnit().fromSource() and + result.getFile().toString() = "A" +} module Config implements DataFlow::ConfigSig { predicate isSource(DataFlow::Node n) { n.asExpr() = src() } - predicate isSink(DataFlow::Node n) { any() } + predicate isSink(DataFlow::Node n) { none() } } module Flow = DataFlow::Global; -from DataFlow::Node src, DataFlow::Node sink -where Flow::flow(src, sink) +int explorationLimit() { result = 100 } + +module PartialFlow = Flow::FlowExploration; + +from PartialFlow::PartialPathNode src, PartialFlow::PartialPathNode sink +where PartialFlow::partialFlow(src, sink, _) select src, sink diff --git a/java/ql/test/library-tests/dataflow/stream-read/A.java b/java/ql/test/library-tests/dataflow/stream-read/A.java index 779f95bcefa..705977757a1 100644 --- a/java/ql/test/library-tests/dataflow/stream-read/A.java +++ b/java/ql/test/library-tests/dataflow/stream-read/A.java @@ -99,7 +99,7 @@ public class A { } public static void testWrapCall() { - sink(wrapStream(null)); // $ SPURIOUS: hasTaintFlow + sink(wrapStream(null)); // no flow sink(wrapStream(source())); // $ hasTaintFlow } diff --git a/java/ql/test/library-tests/dataflow/threat-models/Empty.java b/java/ql/test/library-tests/dataflow/threat-models/Empty.java new file mode 100644 index 00000000000..5be35d3d2db --- /dev/null +++ b/java/ql/test/library-tests/dataflow/threat-models/Empty.java @@ -0,0 +1 @@ +class Empty { } diff --git a/java/ql/test/library-tests/dataflow/threat-models/threat-models1.expected b/java/ql/test/library-tests/dataflow/threat-models/threat-models1.expected new file mode 100644 index 00000000000..d538019ccb7 --- /dev/null +++ b/java/ql/test/library-tests/dataflow/threat-models/threat-models1.expected @@ -0,0 +1,5 @@ +| default | +| remote | +| request | +| response | +| uri-path | diff --git a/java/ql/test/library-tests/dataflow/threat-models/threat-models1.ql b/java/ql/test/library-tests/dataflow/threat-models/threat-models1.ql new file mode 100644 index 00000000000..11371a749dd --- /dev/null +++ b/java/ql/test/library-tests/dataflow/threat-models/threat-models1.ql @@ -0,0 +1,5 @@ +import semmle.code.java.dataflow.ExternalFlowConfiguration as ExternalFlowConfiguration + +query predicate supportedThreatModels(string kind) { + ExternalFlowConfiguration::sourceModelKindConfig(kind) +} diff --git a/java/ql/test/library-tests/dataflow/threat-models/threat-models2.expected b/java/ql/test/library-tests/dataflow/threat-models/threat-models2.expected new file mode 100644 index 00000000000..809a018e98e --- /dev/null +++ b/java/ql/test/library-tests/dataflow/threat-models/threat-models2.expected @@ -0,0 +1,10 @@ +| cli | +| database | +| default | +| environment | +| file | +| local | +| remote | +| request | +| response | +| uri-path | diff --git a/java/ql/test/library-tests/dataflow/threat-models/threat-models2.ext.yml b/java/ql/test/library-tests/dataflow/threat-models/threat-models2.ext.yml new file mode 100644 index 00000000000..1d6ed8c4992 --- /dev/null +++ b/java/ql/test/library-tests/dataflow/threat-models/threat-models2.ext.yml @@ -0,0 +1,7 @@ +extensions: + + - addsTo: + pack: codeql/java-all + extensible: supportedThreatModels + data: + - ["local"] # Add the "local" group threat model. diff --git a/java/ql/test/library-tests/dataflow/threat-models/threat-models2.ql b/java/ql/test/library-tests/dataflow/threat-models/threat-models2.ql new file mode 100644 index 00000000000..11371a749dd --- /dev/null +++ b/java/ql/test/library-tests/dataflow/threat-models/threat-models2.ql @@ -0,0 +1,5 @@ +import semmle.code.java.dataflow.ExternalFlowConfiguration as ExternalFlowConfiguration + +query predicate supportedThreatModels(string kind) { + ExternalFlowConfiguration::sourceModelKindConfig(kind) +} diff --git a/java/ql/test/library-tests/frameworks/ratpack/resources/IntegrationTest.java b/java/ql/test/library-tests/frameworks/ratpack/resources/IntegrationTest.java index 83ca80fc37f..da87794eb56 100644 --- a/java/ql/test/library-tests/frameworks/ratpack/resources/IntegrationTest.java +++ b/java/ql/test/library-tests/frameworks/ratpack/resources/IntegrationTest.java @@ -107,13 +107,13 @@ class IntegrationTest { filterAndMerge_2(pojoForm, mergedParams, name -> false); return mergedParams; }).then(pojoMap -> { - sink(pojoMap.keySet().iterator().next()); //TODO:$hasTaintFlow - sink(pojoMap.get("value")); //TODO:$hasTaintFlow + sink(pojoMap.keySet().iterator().next()); //$hasTaintFlow + sink(pojoMap.get("value")); //$hasTaintFlow pojoMap.forEach((key, value) -> { - sink(key); //TODO:$hasTaintFlow - sink(value); //TODO:$hasTaintFlow + sink(key); //$hasTaintFlow + sink(value); //$hasTaintFlow List values = (List) value; - sink(values.get(0)); //TODO:$hasTaintFlow + sink(values.get(0)); //$hasTaintFlow }); }); } diff --git a/misc/scripts/accept-expected-changes-from-ci.py b/misc/scripts/accept-expected-changes-from-ci.py index 12c1bb21763..47a6bec76ab 100755 --- a/misc/scripts/accept-expected-changes-from-ci.py +++ b/misc/scripts/accept-expected-changes-from-ci.py @@ -336,7 +336,10 @@ def main(pr_number: Optional[int], sha_override: Optional[str] = None, force=Fal check_runs = json.loads(subprocess.check_output(["gh", "api", "--paginate", check_failure_url]).decode("utf-8")) for check_run in check_runs["check_runs"]: if check_run["conclusion"] == "failure": - m = re.fullmatch(r"^https://github\.com/([^/]+/[^/]+)/actions/runs/(\d+)(?:/jobs/(\d+))?$", check_run["details_url"]) + m = re.fullmatch(r"^https://github\.com/([^/]+/[^/]+)/actions/runs/(\d+)(?:/job/(\d+))?$", check_run["details_url"]) + if not m: + LOGGER.error(f"Could not parse details URL for {check_run['name']}: '{check_run['details_url']}'") + continue nwo = m.group(1) run_id = m.group(2) jobs_url = f"https://api.github.com/repos/{nwo}/actions/runs/{run_id}/jobs" diff --git a/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl.qll b/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl.qll index 3f8fbd12817..92f0f17ce82 100644 --- a/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl.qll +++ b/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl.qll @@ -1,3 +1,3 @@ private import DataFlowImplSpecific -private import codeql.dataflow.DataFlowImpl +private import codeql.dataflow.internal.DataFlowImpl import MakeImpl diff --git a/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplCommon.qll b/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplCommon.qll index d0aa8ee7cfc..41c9c4ec1be 100644 --- a/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplCommon.qll +++ b/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplCommon.qll @@ -1,3 +1,3 @@ private import DataFlowImplSpecific -private import codeql.dataflow.DataFlowImplCommon +private import codeql.dataflow.internal.DataFlowImplCommon import MakeImplCommon diff --git a/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplSpecific.qll b/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplSpecific.qll index 5663f5f64d1..cffdefe41ba 100644 --- a/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplSpecific.qll +++ b/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplSpecific.qll @@ -2,7 +2,7 @@ * Provides Python-specific definitions for use in the data flow library. */ -private import codeql.dataflow.DataFlowParameter +private import codeql.dataflow.DataFlow // we need to export `Unit` for the DataFlowImpl* files private import python as Python @@ -15,7 +15,7 @@ module Public { import DataFlowUtil } -module PythonDataFlow implements DataFlowParameter { +module PythonDataFlow implements InputSig { import Private import Public diff --git a/ql/Cargo.lock b/ql/Cargo.lock index 5ce63783ea8..d9ce2ba4767 100644 Binary files a/ql/Cargo.lock and b/ql/Cargo.lock differ diff --git a/ql/buramu/Cargo.toml b/ql/buramu/Cargo.toml index d6b3ea6d7a6..94a3345553b 100644 --- a/ql/buramu/Cargo.toml +++ b/ql/buramu/Cargo.toml @@ -9,4 +9,4 @@ edition = "2018" lazy_static = "1.4.0" chrono = "0.4.26" rayon = "1.7.0" -regex = "1.9.1" +regex = "1.9.3" diff --git a/ql/extractor/Cargo.toml b/ql/extractor/Cargo.toml index 438173b45cf..f082d00f6f0 100644 --- a/ql/extractor/Cargo.toml +++ b/ql/extractor/Cargo.toml @@ -16,5 +16,5 @@ clap = { version = "4.2", features = ["derive"] } tracing = "0.1" tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } rayon = "1.7.0" -regex = "1.9.1" +regex = "1.9.3" codeql-extractor = { path = "../../shared/tree-sitter-extractor" } diff --git a/ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl.qll b/ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl.qll index 598f985ba87..277b639d0ab 100644 --- a/ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl.qll +++ b/ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl.qll @@ -1,3 +1,3 @@ private import DataFlowImplSpecific -private import codeql.dataflow.DataFlowImpl +private import codeql.dataflow.internal.DataFlowImpl import MakeImpl diff --git a/ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplCommon.qll b/ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplCommon.qll index e25906abd0b..05e0bc67b30 100644 --- a/ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplCommon.qll +++ b/ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplCommon.qll @@ -1,3 +1,3 @@ private import DataFlowImplSpecific -private import codeql.dataflow.DataFlowImplCommon +private import codeql.dataflow.internal.DataFlowImplCommon import MakeImplCommon diff --git a/ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplSpecific.qll b/ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplSpecific.qll index 02f0003505f..7ee656da807 100644 --- a/ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplSpecific.qll +++ b/ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplSpecific.qll @@ -2,7 +2,7 @@ * Provides Ruby-specific definitions for use in the data flow library. */ -private import codeql.dataflow.DataFlowParameter +private import codeql.dataflow.DataFlow module Private { import DataFlowPrivate @@ -13,7 +13,7 @@ module Public { import DataFlowPublic } -module RubyDataFlow implements DataFlowParameter { +module RubyDataFlow implements InputSig { import Private import Public diff --git a/shared/dataflow/change-notes/2023-06-16-initial-version.md b/shared/dataflow/change-notes/2023-06-16-initial-version.md new file mode 100644 index 00000000000..3846cb70c1d --- /dev/null +++ b/shared/dataflow/change-notes/2023-06-16-initial-version.md @@ -0,0 +1,4 @@ +--- +category: majorAnalysis +--- +* Initial release. Adds a library to implement flow through captured variables that properly adheres to inter-procedural control flow. diff --git a/shared/dataflow/codeql/dataflow/DataFlow.qll b/shared/dataflow/codeql/dataflow/DataFlow.qll index 1689f812321..ce2c518f478 100644 --- a/shared/dataflow/codeql/dataflow/DataFlow.qll +++ b/shared/dataflow/codeql/dataflow/DataFlow.qll @@ -1,15 +1,234 @@ /** * Provides an implementation of global (interprocedural) data flow. This file - * re-exports the local (intraprocedural) data flow analysis from - * `DataFlowImplSpecific::Public` and adds a global analysis, mainly exposed - * through the `Global` and `GlobalWithState` modules. + * adds a global analysis, mainly exposed through the `Global` and `GlobalWithState` + * modules. */ -import DataFlowParameter +/** Provides language-specific data flow parameters. */ +signature module InputSig { + class Node { + /** Gets a textual representation of this element. */ + string toString(); -module Configs { + /** + * Holds if this element is at the specified location. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `filepath`. + * For more information, see + * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/). + */ + predicate hasLocationInfo( + string filepath, int startline, int startcolumn, int endline, int endcolumn + ); + } + + class ParameterNode extends Node; + + class ArgumentNode extends Node; + + class ReturnNode extends Node { + ReturnKind getKind(); + } + + class OutNode extends Node; + + class PostUpdateNode extends Node { + Node getPreUpdateNode(); + } + + class CastNode extends Node; + + predicate isParameterNode(ParameterNode p, DataFlowCallable c, ParameterPosition pos); + + predicate isArgumentNode(ArgumentNode n, DataFlowCall call, ArgumentPosition pos); + + DataFlowCallable nodeGetEnclosingCallable(Node node); + + DataFlowType getNodeType(Node node); + + predicate nodeIsHidden(Node node); + + class DataFlowExpr; + + /** Gets the node corresponding to `e`. */ + Node exprNode(DataFlowExpr e); + + class DataFlowCall { + /** Gets a textual representation of this element. */ + string toString(); + + DataFlowCallable getEnclosingCallable(); + } + + class DataFlowCallable { + /** Gets a textual representation of this element. */ + string toString(); + } + + class ReturnKind { + /** Gets a textual representation of this element. */ + string toString(); + } + + /** Gets a viable implementation of the target of the given `Call`. */ + DataFlowCallable viableCallable(DataFlowCall c); + + /** + * Holds if the set of viable implementations that can be called by `call` + * might be improved by knowing the call context. + */ + predicate mayBenefitFromCallContext(DataFlowCall call, DataFlowCallable c); + + /** + * Gets a viable dispatch target of `call` in the context `ctx`. This is + * restricted to those `call`s for which a context might make a difference. + */ + DataFlowCallable viableImplInCallContext(DataFlowCall call, DataFlowCall ctx); + + /** + * Gets a node that can read the value returned from `call` with return kind + * `kind`. + */ + OutNode getAnOutNode(DataFlowCall call, ReturnKind kind); + + class DataFlowType { + /** Gets a textual representation of this element. */ + string toString(); + } + + string ppReprType(DataFlowType t); + + bindingset[t1, t2] + predicate compatibleTypes(DataFlowType t1, DataFlowType t2); + + predicate typeStrongerThan(DataFlowType t1, DataFlowType t2); + + class Content { + /** Gets a textual representation of this element. */ + string toString(); + } + + predicate forceHighPrecision(Content c); + + /** + * An entity that represents a set of `Content`s. + * + * The set may be interpreted differently depending on whether it is + * stored into (`getAStoreContent`) or read from (`getAReadContent`). + */ + class ContentSet { + /** Gets a content that may be stored into when storing into this set. */ + Content getAStoreContent(); + + /** Gets a content that may be read from when reading from this set. */ + Content getAReadContent(); + } + + class ContentApprox { + /** Gets a textual representation of this element. */ + string toString(); + } + + ContentApprox getContentApprox(Content c); + + class ParameterPosition { + /** Gets a textual representation of this element. */ + bindingset[this] + string toString(); + } + + class ArgumentPosition { + /** Gets a textual representation of this element. */ + bindingset[this] + string toString(); + } + + predicate parameterMatch(ParameterPosition ppos, ArgumentPosition apos); + + predicate simpleLocalFlowStep(Node node1, Node node2); + + /** + * Holds if data can flow from `node1` to `node2` through a non-local step + * that does not follow a call edge. For example, a step through a global + * variable. + */ + predicate jumpStep(Node node1, Node node2); + + /** + * Holds if data can flow from `node1` to `node2` via a read of `c`. Thus, + * `node1` references an object with a content `c.getAReadContent()` whose + * value ends up in `node2`. + */ + predicate readStep(Node node1, ContentSet c, Node node2); + + /** + * Holds if data can flow from `node1` to `node2` via a store into `c`. Thus, + * `node2` references an object with a content `c.getAStoreContent()` that + * contains the value of `node1`. + */ + predicate storeStep(Node node1, ContentSet c, Node node2); + + /** + * Holds if values stored inside content `c` are cleared at node `n`. For example, + * any value stored inside `f` is cleared at the pre-update node associated with `x` + * in `x.f = newValue`. + */ + predicate clearsContent(Node n, ContentSet c); + + /** + * Holds if the value that is being tracked is expected to be stored inside content `c` + * at node `n`. + */ + predicate expectsContent(Node n, ContentSet c); + + /** + * Holds if the node `n` is unreachable when the call context is `call`. + */ + predicate isUnreachableInCall(Node n, DataFlowCall call); + + default int accessPathLimit() { result = 5 } + + /** + * Holds if flow is allowed to pass from parameter `p` and back to itself as a + * side-effect, resulting in a summary from `p` to itself. + * + * One example would be to allow flow like `p.foo = p.bar;`, which is disallowed + * by default as a heuristic. + */ + predicate allowParameterReturnInSelf(ParameterNode p); + + class LambdaCallKind; + + /** Holds if `creation` is an expression that creates a lambda of kind `kind` for `c`. */ + predicate lambdaCreation(Node creation, LambdaCallKind kind, DataFlowCallable c); + + /** Holds if `call` is a lambda call of kind `kind` where `receiver` is the lambda expression. */ + predicate lambdaCall(DataFlowCall call, LambdaCallKind kind, Node receiver); + + /** Extra data-flow steps needed for lambda flow analysis. */ + predicate additionalLambdaFlowStep(Node nodeFrom, Node nodeTo, boolean preservesValue); + + /** + * Holds if `n` should never be skipped over in the `PathGraph` and in path + * explanations. + */ + default predicate neverSkipInPathGraph(Node n) { none() } + + /** + * Gets an additional term that is added to the `join` and `branch` computations to reflect + * an additional forward or backwards branching factor that is not taken into account + * when calculating the (virtual) dispatch cost. + * + * Argument `arg` is part of a path from a source to a sink, and `p` is the target parameter. + */ + int getAdditionalFlowIntoCallNodeTerm(ArgumentNode arg, ParameterNode p); + + predicate golangSpecificParamArgFilter(DataFlowCall call, ParameterNode p, ArgumentNode arg); +} + +module Configs { private import Lang - private import DataFlowImplCommon::MakeImplCommon + private import internal.DataFlowImplCommon::MakeImplCommon import DataFlowImplCommonPublic /** An input configuration for data flow. */ @@ -211,9 +430,9 @@ module Configs { } } -module DataFlowMake { +module DataFlowMake { private import Lang - private import DataFlowImpl::MakeImpl + private import internal.DataFlowImpl::MakeImpl import Configs /** diff --git a/shared/dataflow/codeql/dataflow/DataFlowParameter.qll b/shared/dataflow/codeql/dataflow/DataFlowParameter.qll deleted file mode 100644 index ecd29b45a28..00000000000 --- a/shared/dataflow/codeql/dataflow/DataFlowParameter.qll +++ /dev/null @@ -1,220 +0,0 @@ -signature module DataFlowParameter { - class Node { - /** Gets a textual representation of this element. */ - string toString(); - - /** - * Holds if this element is at the specified location. - * The location spans column `startcolumn` of line `startline` to - * column `endcolumn` of line `endline` in file `filepath`. - * For more information, see - * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/). - */ - predicate hasLocationInfo( - string filepath, int startline, int startcolumn, int endline, int endcolumn - ); - } - - class ParameterNode extends Node; - - class ArgumentNode extends Node; - - class ReturnNode extends Node { - ReturnKind getKind(); - } - - class OutNode extends Node; - - class PostUpdateNode extends Node { - Node getPreUpdateNode(); - } - - class CastNode extends Node; - - predicate isParameterNode(ParameterNode p, DataFlowCallable c, ParameterPosition pos); - - predicate isArgumentNode(ArgumentNode n, DataFlowCall call, ArgumentPosition pos); - - DataFlowCallable nodeGetEnclosingCallable(Node node); - - DataFlowType getNodeType(Node node); - - predicate nodeIsHidden(Node node); - - class DataFlowExpr; - - /** Gets the node corresponding to `e`. */ - Node exprNode(DataFlowExpr e); - - class DataFlowCall { - /** Gets a textual representation of this element. */ - string toString(); - - DataFlowCallable getEnclosingCallable(); - } - - class DataFlowCallable { - /** Gets a textual representation of this element. */ - string toString(); - } - - class ReturnKind { - /** Gets a textual representation of this element. */ - string toString(); - } - - /** Gets a viable implementation of the target of the given `Call`. */ - DataFlowCallable viableCallable(DataFlowCall c); - - /** - * Holds if the set of viable implementations that can be called by `call` - * might be improved by knowing the call context. - */ - predicate mayBenefitFromCallContext(DataFlowCall call, DataFlowCallable c); - - /** - * Gets a viable dispatch target of `call` in the context `ctx`. This is - * restricted to those `call`s for which a context might make a difference. - */ - DataFlowCallable viableImplInCallContext(DataFlowCall call, DataFlowCall ctx); - - /** - * Gets a node that can read the value returned from `call` with return kind - * `kind`. - */ - OutNode getAnOutNode(DataFlowCall call, ReturnKind kind); - - class DataFlowType { - /** Gets a textual representation of this element. */ - string toString(); - } - - string ppReprType(DataFlowType t); - - bindingset[t1, t2] - predicate compatibleTypes(DataFlowType t1, DataFlowType t2); - - predicate typeStrongerThan(DataFlowType t1, DataFlowType t2); - - class Content { - /** Gets a textual representation of this element. */ - string toString(); - } - - predicate forceHighPrecision(Content c); - - /** - * An entity that represents a set of `Content`s. - * - * The set may be interpreted differently depending on whether it is - * stored into (`getAStoreContent`) or read from (`getAReadContent`). - */ - class ContentSet { - /** Gets a content that may be stored into when storing into this set. */ - Content getAStoreContent(); - - /** Gets a content that may be read from when reading from this set. */ - Content getAReadContent(); - } - - class ContentApprox { - /** Gets a textual representation of this element. */ - string toString(); - } - - ContentApprox getContentApprox(Content c); - - class ParameterPosition { - /** Gets a textual representation of this element. */ - bindingset[this] - string toString(); - } - - class ArgumentPosition { - /** Gets a textual representation of this element. */ - bindingset[this] - string toString(); - } - - predicate parameterMatch(ParameterPosition ppos, ArgumentPosition apos); - - predicate simpleLocalFlowStep(Node node1, Node node2); - - /** - * Holds if data can flow from `node1` to `node2` through a non-local step - * that does not follow a call edge. For example, a step through a global - * variable. - */ - predicate jumpStep(Node node1, Node node2); - - /** - * Holds if data can flow from `node1` to `node2` via a read of `c`. Thus, - * `node1` references an object with a content `c.getAReadContent()` whose - * value ends up in `node2`. - */ - predicate readStep(Node node1, ContentSet c, Node node2); - - /** - * Holds if data can flow from `node1` to `node2` via a store into `c`. Thus, - * `node2` references an object with a content `c.getAStoreContent()` that - * contains the value of `node1`. - */ - predicate storeStep(Node node1, ContentSet c, Node node2); - - /** - * Holds if values stored inside content `c` are cleared at node `n`. For example, - * any value stored inside `f` is cleared at the pre-update node associated with `x` - * in `x.f = newValue`. - */ - predicate clearsContent(Node n, ContentSet c); - - /** - * Holds if the value that is being tracked is expected to be stored inside content `c` - * at node `n`. - */ - predicate expectsContent(Node n, ContentSet c); - - /** - * Holds if the node `n` is unreachable when the call context is `call`. - */ - predicate isUnreachableInCall(Node n, DataFlowCall call); - - default int accessPathLimit() { result = 5 } - - /** - * Holds if flow is allowed to pass from parameter `p` and back to itself as a - * side-effect, resulting in a summary from `p` to itself. - * - * One example would be to allow flow like `p.foo = p.bar;`, which is disallowed - * by default as a heuristic. - */ - predicate allowParameterReturnInSelf(ParameterNode p); - - class LambdaCallKind; - - /** Holds if `creation` is an expression that creates a lambda of kind `kind` for `c`. */ - predicate lambdaCreation(Node creation, LambdaCallKind kind, DataFlowCallable c); - - /** Holds if `call` is a lambda call of kind `kind` where `receiver` is the lambda expression. */ - predicate lambdaCall(DataFlowCall call, LambdaCallKind kind, Node receiver); - - /** Extra data-flow steps needed for lambda flow analysis. */ - predicate additionalLambdaFlowStep(Node nodeFrom, Node nodeTo, boolean preservesValue); - - /** - * Holds if `n` should never be skipped over in the `PathGraph` and in path - * explanations. - */ - default predicate neverSkipInPathGraph(Node n) { none() } - - /** - * Gets an additional term that is added to the `join` and `branch` computations to reflect - * an additional forward or backwards branching factor that is not taken into account - * when calculating the (virtual) dispatch cost. - * - * Argument `arg` is part of a path from a source to a sink, and `p` is the target parameter. - */ - int getAdditionalFlowIntoCallNodeTerm(ArgumentNode arg, ParameterNode p); - - predicate golangSpecificParamArgFilter(DataFlowCall call, ParameterNode p, ArgumentNode arg); -} diff --git a/shared/dataflow/codeql/dataflow/VariableCapture.qll b/shared/dataflow/codeql/dataflow/VariableCapture.qll new file mode 100644 index 00000000000..a1504d44fdc --- /dev/null +++ b/shared/dataflow/codeql/dataflow/VariableCapture.qll @@ -0,0 +1,929 @@ +/** + * Provides a module for synthesizing data-flow nodes and related step relations + * for supporting flow through captured variables. + */ + +private import codeql.util.Boolean +private import codeql.util.Unit +private import codeql.ssa.Ssa as Ssa + +signature module InputSig { + class Location { + predicate hasLocationInfo( + string filepath, int startline, int startcolumn, int endline, int endcolumn + ); + } + + /** + * A basic block, that is, a maximal straight-line sequence of control flow nodes + * without branches or joins. + */ + class BasicBlock { + /** Gets a textual representation of this basic block. */ + string toString(); + + /** Gets the enclosing callable. */ + Callable getEnclosingCallable(); + + /** Gets the location of this basic block. */ + Location getLocation(); + } + + /** + * Gets the basic block that immediately dominates basic block `bb`, if any. + * + * That is, all paths reaching `bb` from some entry point basic block must go + * through the result. + * + * Example: + * + * ```csharp + * int M(string s) { + * if (s == null) + * throw new ArgumentNullException(nameof(s)); + * return s.Length; + * } + * ``` + * + * The basic block starting on line 2 is an immediate dominator of + * the basic block on line 4 (all paths from the entry point of `M` + * to `return s.Length;` must go through the null check. + */ + BasicBlock getImmediateBasicBlockDominator(BasicBlock bb); + + /** Gets an immediate successor of basic block `bb`, if any. */ + BasicBlock getABasicBlockSuccessor(BasicBlock bb); + + /** Holds if `bb` is a control-flow entry point. */ + default predicate entryBlock(BasicBlock bb) { not exists(getImmediateBasicBlockDominator(bb)) } + + /** Holds if `bb` is a control-flow exit point. */ + default predicate exitBlock(BasicBlock bb) { not exists(getABasicBlockSuccessor(bb)) } + + /** A variable that is captured in a closure. */ + class CapturedVariable { + /** Gets a textual representation of this variable. */ + string toString(); + + /** Gets the callable that defines this variable. */ + Callable getCallable(); + + /** Gets the location of this variable. */ + Location getLocation(); + } + + /** A parameter that is captured in a closure. */ + class CapturedParameter extends CapturedVariable; + + /** + * An expression with a value. That is, we expect these expressions to be + * represented in the data flow graph. + */ + class Expr { + /** Gets a textual representation of this expression. */ + string toString(); + + /** Gets the location of this expression. */ + Location getLocation(); + + /** Holds if the `i`th node of basic block `bb` evaluates this expression. */ + predicate hasCfgNode(BasicBlock bb, int i); + } + + /** A write to a captured variable. */ + class VariableWrite { + /** Gets the variable that is the target of this write. */ + CapturedVariable getVariable(); + + /** Gets the expression that is the source of this write. */ + Expr getSource(); + + /** Gets the location of this write. */ + Location getLocation(); + + /** Holds if the `i`th node of basic block `bb` evaluates this expression. */ + predicate hasCfgNode(BasicBlock bb, int i); + } + + /** A read of a captured variable. */ + class VariableRead extends Expr { + /** Gets the variable that this expression reads. */ + CapturedVariable getVariable(); + } + + /** + * An expression constructing a closure that may capture one or more + * variables. This can for example be a lambda or a constructor call of a + * locally defined object. + */ + class ClosureExpr extends Expr { + /** + * Holds if `body` is the callable body of this closure. A lambda expression + * only has one body, but in general a locally defined object may have + * multiple such methods and constructors. + */ + predicate hasBody(Callable body); + + /** + * Holds if `f` is an expression that may hold the value of the closure and + * may occur in a position where the value escapes or where the closure may + * be invoked. + * + * For example, if a lambda is assigned to a variable, then references to + * that variable in return or argument positions should be included. + */ + predicate hasAliasedAccess(Expr f); + } + + class Callable { + /** Gets a textual representation of this callable. */ + string toString(); + + /** Gets the location of this callable. */ + Location getLocation(); + + /** Holds if this callable is a constructor. */ + predicate isConstructor(); + } +} + +signature module OutputSig { + /** + * A data flow node that we need to reference in the step relations for + * captured variables. + * + * Note that only the `SynthesizedCaptureNode` subclass is expected to be + * added as additional nodes in `DataFlow::Node`. The other subclasses are + * expected to already be present and are included here in order to reference + * them in the step relations. + */ + class ClosureNode; + + /** + * A synthesized data flow node representing the storage of a captured + * variable. + */ + class SynthesizedCaptureNode extends ClosureNode { + /** Gets a textual representation of this node. */ + string toString(); + + /** Gets the location of this node. */ + I::Location getLocation(); + + /** Gets the enclosing callable. */ + I::Callable getEnclosingCallable(); + + /** Holds if this node is a synthesized access of `v`. */ + predicate isVariableAccess(I::CapturedVariable v); + + /** Holds if this node is a synthesized instance access. */ + predicate isInstanceAccess(); + } + + /** A data flow node for an expression. */ + class ExprNode extends ClosureNode { + /** Gets the expression corresponding to this node. */ + I::Expr getExpr(); + } + + /** A data flow node for the `PostUpdateNode` of an expression. */ + class ExprPostUpdateNode extends ClosureNode { + /** Gets the expression corresponding to this node. */ + I::Expr getExpr(); + } + + /** A data flow node for a parameter. */ + class ParameterNode extends ClosureNode { + /** Gets the parameter corresponding to this node. */ + I::CapturedParameter getParameter(); + } + + /** A data flow node for an instance parameter. */ + class ThisParameterNode extends ClosureNode { + /** Gets the callable this instance parameter belongs to. */ + I::Callable getCallable(); + } + + /** A data flow node for the instance parameter argument of a constructor call. */ + class MallocNode extends ClosureNode { + /** Gets the closure construction that is the post-update of this node. */ + I::ClosureExpr getClosureExpr(); + } + + /** Holds if `post` is a `PostUpdateNode` for `pre`. */ + predicate capturePostUpdateNode(SynthesizedCaptureNode post, SynthesizedCaptureNode pre); + + /** Holds if there is a local flow step from `node1` to `node2`. */ + predicate localFlowStep(ClosureNode node1, ClosureNode node2); + + /** Holds if there is a store step from `node1` to `node2`. */ + predicate storeStep(ClosureNode node1, I::CapturedVariable v, ClosureNode node2); + + /** Holds if there is a read step from `node1` to `node2`. */ + predicate readStep(ClosureNode node1, I::CapturedVariable v, ClosureNode node2); + + /** Holds if this-to-this summaries are expected for `c`. */ + predicate heuristicAllowInstanceParameterReturnInSelf(I::Callable c); +} + +/** + * Constructs the type `ClosureNode` and associated step relations, which are + * intended to be included in the data-flow node and step relations. + */ +module Flow implements OutputSig { + private import Input + + additional module ConsistencyChecks { + final private class FinalExpr = Expr; + + private class RelevantExpr extends FinalExpr { + RelevantExpr() { + this instanceof VariableRead or + any(VariableWrite vw).getSource() = this or + this instanceof ClosureExpr or + any(ClosureExpr ce).hasAliasedAccess(this) + } + } + + final private class FinalBasicBlock = BasicBlock; + + private class RelevantBasicBlock extends FinalBasicBlock { + RelevantBasicBlock() { + exists(RelevantExpr e | e.hasCfgNode(this, _)) + or + exists(VariableWrite vw | vw.hasCfgNode(this, _)) + } + } + + final private class FinalCallable = Callable; + + private class RelevantCallable extends FinalCallable { + RelevantCallable() { + exists(RelevantBasicBlock bb | bb.getEnclosingCallable() = this) + or + exists(CapturedVariable v | v.getCallable() = this) + or + exists(ClosureExpr ce | ce.hasBody(this)) + } + } + + query predicate uniqueToString(string msg, int n) { + exists(string elem | + n = strictcount(RelevantBasicBlock bb | not exists(bb.toString())) and + elem = "BasicBlock" + or + n = strictcount(CapturedVariable v | not exists(v.toString())) and elem = "CapturedVariable" + or + n = strictcount(RelevantExpr e | not exists(e.toString())) and elem = "Expr" + or + n = strictcount(RelevantCallable c | not exists(c.toString())) and + elem = "Callable" + | + msg = n + " " + elem + "(s) are missing toString" + ) + or + exists(string elem | + n = strictcount(RelevantBasicBlock bb | 2 <= strictcount(bb.toString())) and + elem = "BasicBlock" + or + n = strictcount(CapturedVariable v | 2 <= strictcount(v.toString())) and + elem = "CapturedVariable" + or + n = strictcount(RelevantExpr e | 2 <= strictcount(e.toString())) and + elem = "Expr" + or + n = strictcount(RelevantCallable c | 2 <= strictcount(c.toString())) and + elem = "Callable" + | + msg = n + " " + elem + "(s) have multiple toStrings" + ) + } + + query predicate uniqueEnclosingCallable(RelevantBasicBlock bb, string msg) { + msg = "BasicBlock has no enclosing callable" and not exists(bb.getEnclosingCallable()) + or + msg = "BasicBlock has multiple enclosing callables" and + 2 <= strictcount(bb.getEnclosingCallable()) + } + + query predicate uniqueDominator(RelevantBasicBlock bb, string msg) { + msg = "BasicBlock has multiple immediate dominators" and + 2 <= strictcount(getImmediateBasicBlockDominator(bb)) + } + + query predicate localDominator(RelevantBasicBlock bb, string msg) { + msg = "BasicBlock has non-local dominator" and + bb.getEnclosingCallable() != getImmediateBasicBlockDominator(bb).getEnclosingCallable() + } + + query predicate localSuccessor(RelevantBasicBlock bb, string msg) { + msg = "BasicBlock has non-local successor" and + bb.getEnclosingCallable() != getABasicBlockSuccessor(bb).getEnclosingCallable() + } + + query predicate uniqueDefiningScope(CapturedVariable v, string msg) { + msg = "CapturedVariable has no defining callable" and not exists(v.getCallable()) + or + msg = "CapturedVariable has multiple defining callables" and 2 <= strictcount(v.getCallable()) + } + + query predicate variableIsCaptured(CapturedVariable v, string msg) { + msg = "CapturedVariable is not captured" and + not captureAccess(v, _) + } + + query predicate uniqueLocation(RelevantExpr e, string msg) { + msg = "Expr has no location" and not exists(e.getLocation()) + or + msg = "Expr has multiple locations" and 2 <= strictcount(e.getLocation()) + } + + query predicate uniqueCfgNode(RelevantExpr e, string msg) { + msg = "Expr has no cfg node" and not e.hasCfgNode(_, _) + or + msg = "Expr has multiple cfg nodes" and + 2 <= strictcount(BasicBlock bb, int i | e.hasCfgNode(bb, i)) + } + + private predicate uniqueWriteTarget(VariableWrite vw, string msg) { + msg = "VariableWrite has no target variable" and not exists(vw.getVariable()) + or + msg = "VariableWrite has multiple target variables" and 2 <= strictcount(vw.getVariable()) + } + + query predicate uniqueWriteTarget(string msg) { uniqueWriteTarget(_, msg) } + + private predicate uniqueWriteSource(VariableWrite vw, string msg) { + msg = "VariableWrite has no source expression" and not exists(vw.getSource()) + or + msg = "VariableWrite has multiple source expressions" and 2 <= strictcount(vw.getSource()) + } + + query predicate uniqueWriteSource(string msg) { uniqueWriteSource(_, msg) } + + private predicate uniqueWriteCfgNode(VariableWrite vw, string msg) { + msg = "VariableWrite has no cfg node" and not vw.hasCfgNode(_, _) + or + msg = "VariableWrite has multiple cfg nodes" and + 2 <= strictcount(BasicBlock bb, int i | vw.hasCfgNode(bb, i)) + } + + query predicate uniqueWriteCfgNode(string msg) { uniqueWriteCfgNode(_, msg) } + + private predicate localWriteStep(VariableWrite vw, string msg) { + exists(BasicBlock bb | + vw.hasCfgNode(bb, _) and + bb.getEnclosingCallable() != vw.getVariable().getCallable() and + msg = "VariableWrite is not a local step" + ) + } + + query predicate localWriteStep(string msg) { localWriteStep(_, msg) } + + query predicate uniqueReadVariable(VariableRead vr, string msg) { + msg = "VariableRead has no source variable" and not exists(vr.getVariable()) + or + msg = "VariableRead has multiple source variables" and 2 <= strictcount(vr.getVariable()) + } + + query predicate closureMustHaveBody(ClosureExpr ce, string msg) { + msg = "ClosureExpr has no body" and not ce.hasBody(_) + } + + query predicate closureAliasMustBeLocal(ClosureExpr ce, Expr access, string msg) { + exists(BasicBlock bb1, BasicBlock bb2 | + ce.hasAliasedAccess(access) and + ce.hasCfgNode(bb1, _) and + access.hasCfgNode(bb2, _) and + bb1.getEnclosingCallable() != bb2.getEnclosingCallable() and + msg = "ClosureExpr has non-local alias - these are ignored" + ) + } + + private predicate astClosureParent(Callable closure, Callable parent) { + exists(ClosureExpr ce, BasicBlock bb | + ce.hasBody(closure) and ce.hasCfgNode(bb, _) and parent = bb.getEnclosingCallable() + ) + } + + query predicate variableAccessAstNesting(CapturedVariable v, Callable c, string msg) { + exists(BasicBlock bb, Callable parent | + captureRead(v, bb, _, false, _) or captureWrite(v, bb, _, false, _) + | + bb.getEnclosingCallable() = c and + v.getCallable() = parent and + not astClosureParent+(c, parent) and + msg = "CapturedVariable access is not nested in the defining callable" + ) + } + + query predicate uniqueCallableLocation(RelevantCallable c, string msg) { + msg = "Callable has no location" and not exists(c.getLocation()) + or + msg = "Callable has multiple locations" and 2 <= strictcount(c.getLocation()) + } + + query predicate consistencyOverview(string msg, int n) { + uniqueToString(msg, n) or + n = strictcount(BasicBlock bb | uniqueEnclosingCallable(bb, msg)) or + n = strictcount(BasicBlock bb | uniqueDominator(bb, msg)) or + n = strictcount(BasicBlock bb | localDominator(bb, msg)) or + n = strictcount(BasicBlock bb | localSuccessor(bb, msg)) or + n = strictcount(CapturedVariable v | uniqueDefiningScope(v, msg)) or + n = strictcount(CapturedVariable v | variableIsCaptured(v, msg)) or + n = strictcount(Expr e | uniqueLocation(e, msg)) or + n = strictcount(Expr e | uniqueCfgNode(e, msg)) or + n = strictcount(VariableWrite vw | uniqueWriteTarget(vw, msg)) or + n = strictcount(VariableWrite vw | uniqueWriteSource(vw, msg)) or + n = strictcount(VariableWrite vw | uniqueWriteCfgNode(vw, msg)) or + n = strictcount(VariableWrite vw | localWriteStep(vw, msg)) or + n = strictcount(VariableRead vr | uniqueReadVariable(vr, msg)) or + n = strictcount(ClosureExpr ce | closureMustHaveBody(ce, msg)) or + n = strictcount(ClosureExpr ce, Expr access | closureAliasMustBeLocal(ce, access, msg)) or + n = strictcount(CapturedVariable v, Callable c | variableAccessAstNesting(v, c, msg)) or + n = strictcount(Callable c | uniqueCallableLocation(c, msg)) + } + } + + /* + * Flow through captured variables is handled by making each captured variable + * a field on the closures that capture them. + * + * For each closure creation we add a store step from the captured variable to + * the closure, and inside the closures we access the captured variables with + * a `this.` qualifier. This allows capture flow into closures. + * + * It also means that we get several aliased versions of a captured variable + * so proper care must be taken to be able to observe side-effects or flow out + * of closures. E.g. if two closures `l1` and `l2` capture `x` then we'll have + * three names, `x`, `l1.x`, and `l2.x`, plus any potential aliasing of the + * closures. + * + * To handle this, we select a primary name for a captured variable in each of + * its scopes, keep that name updated, and update the other names from the + * primary name. + * + * In the defining scope of a captured variable, we use the local variable + * itself as the primary storage location, and in the capturing scopes we use + * the synthesized field. For each relevant reference to a closure object we + * then update its field from the primary storage location, and we read the + * field back from the post-update of the closure object reference and back + * into the primary storage location. + * + * If we include references to a closure object that may lead to a call as + * relevant, then this means that we'll be able to observe the side-effects of + * such calls in the primary storage location. + * + * Details: + * For a reference to a closure `f` that captures `x` we synthesize a read of + * `x` at the same control-flow node. We then add a store step from `x` to `f` + * and a read step from `postupdate(f)` to `postupdate(x)`. + * ``` + * SsaRead(x) --store[x]--> f + * postupdate(f) --read[x]--> postupdate(SsaRead(x)) + * ``` + * In a closure scope with a nested closure `g` that also captures `x` the + * steps instead look like this: + * ``` + * SsaRead(this) --read[x]--> this.x --store[x]--> g + * postupdate(g) --read[x]--> postupdate(this.x) + * ``` + * The final store from `postupdate(this.x)` to `postupdate(this)` is + * introduced automatically as a reverse read by the data flow library. + */ + + /** + * Holds if `vr` is a read of `v` in the `i`th node of `bb`. + * `topScope` is true if the read is in the defining callable of `v`. + */ + private predicate captureRead( + CapturedVariable v, BasicBlock bb, int i, boolean topScope, VariableRead vr + ) { + vr.getVariable() = v and + vr.hasCfgNode(bb, i) and + if v.getCallable() != bb.getEnclosingCallable() then topScope = false else topScope = true + } + + /** + * Holds if `vw` is a write of `v` in the `i`th node of `bb`. + * `topScope` is true if the write is in the defining callable of `v`. + */ + private predicate captureWrite( + CapturedVariable v, BasicBlock bb, int i, boolean topScope, VariableWrite vw + ) { + vw.getVariable() = v and + vw.hasCfgNode(bb, i) and + if v.getCallable() != bb.getEnclosingCallable() then topScope = false else topScope = true + } + + /** Gets the enclosing callable of `ce`. */ + private Callable closureExprGetCallable(ClosureExpr ce) { + exists(BasicBlock bb | ce.hasCfgNode(bb, _) and result = bb.getEnclosingCallable()) + } + + /** + * Holds if `v` is available in `c` through capture. This can either be due to + * an explicit variable reference or through the construction of a closure + * that has a nested capture. + */ + private predicate captureAccess(CapturedVariable v, Callable c) { + exists(BasicBlock bb | captureRead(v, bb, _, _, _) or captureWrite(v, bb, _, _, _) | + c = bb.getEnclosingCallable() and + c != v.getCallable() + ) + or + exists(ClosureExpr ce | + c = closureExprGetCallable(ce) and + closureCaptures(ce, v) and + c != v.getCallable() + ) + } + + /** Holds if the closure defined by `ce` captures `v`. */ + private predicate closureCaptures(ClosureExpr ce, CapturedVariable v) { + exists(Callable c | ce.hasBody(c) and captureAccess(v, c)) + } + + predicate heuristicAllowInstanceParameterReturnInSelf(Callable c) { + // If multiple variables are captured, then we should allow flow from one to + // another, which entails a this-to-this summary. + 2 <= strictcount(CapturedVariable v | captureAccess(v, c)) + or + // Constructors that capture a variable may assign it to a field, which also + // entails a this-to-this summary. + captureAccess(_, c) and c.isConstructor() + } + + /** Holds if the constructor, if any, for the closure defined by `ce` captures `v`. */ + private predicate hasConstructorCapture(ClosureExpr ce, CapturedVariable v) { + exists(Callable c | ce.hasBody(c) and c.isConstructor() and captureAccess(v, c)) + } + + /** + * Holds if `access` is a reference to `ce` evaluated in the `i`th node of `bb`. + * The reference is restricted to be in the same callable as `ce` as a + * precaution, even though this is expected to hold for all the given aliased + * accesses. + */ + private predicate localClosureAccess(ClosureExpr ce, Expr access, BasicBlock bb, int i) { + ce.hasAliasedAccess(access) and + access.hasCfgNode(bb, i) and + pragma[only_bind_out](bb.getEnclosingCallable()) = + pragma[only_bind_out](closureExprGetCallable(ce)) + } + + /** + * Holds if we need an additional read of `v` in the `i`th node of `bb` in + * order to synchronize the value stored on `closure`. + * `topScope` is true if the read is in the defining callable of `v`. + * + * Side-effects of potentially calling `closure` at this point will be + * observed in a similarly synthesized post-update node for this read of `v`. + */ + private predicate synthRead( + CapturedVariable v, BasicBlock bb, int i, boolean topScope, Expr closure + ) { + exists(ClosureExpr ce | closureCaptures(ce, v) | + ce.hasCfgNode(bb, i) and ce = closure + or + localClosureAccess(ce, closure, bb, i) + ) and + if v.getCallable() != bb.getEnclosingCallable() then topScope = false else topScope = true + } + + /** + * Holds if there is an access of a captured variable inside a closure in the + * `i`th node of `bb`, such that we need to synthesize a `this.` qualifier. + */ + private predicate synthThisQualifier(BasicBlock bb, int i) { + synthRead(_, bb, i, false, _) or + captureRead(_, bb, i, false, _) or + captureWrite(_, bb, i, false, _) + } + + private newtype TCaptureContainer = + TVariable(CapturedVariable v) or + TThis(Callable c) { captureAccess(_, c) } + + /** + * A storage location for a captured variable in a specific callable. This is + * either the variable itself (in its defining scope) or an instance variable + * `this` (in a capturing scope). + */ + private class CaptureContainer extends TCaptureContainer { + string toString() { + exists(CapturedVariable v | this = TVariable(v) and result = v.toString()) + or + result = "this" and this = TThis(_) + } + } + + /** Holds if `cc` needs a definition at the entry of its callable scope. */ + private predicate entryDef(CaptureContainer cc, BasicBlock bb, int i) { + exists(Callable c | + entryBlock(bb) and + pragma[only_bind_out](bb.getEnclosingCallable()) = c and + i = + min(int j | + j = 1 or + captureRead(_, bb, j, _, _) or + captureWrite(_, bb, j, _, _) or + synthRead(_, bb, j, _, _) + ) - 1 + | + cc = TThis(c) + or + exists(CapturedParameter p | cc = TVariable(p) and p.getCallable() = c) + ) + } + + private module CaptureSsaInput implements Ssa::InputSig { + final class BasicBlock = Input::BasicBlock; + + BasicBlock getImmediateBasicBlockDominator(BasicBlock bb) { + result = Input::getImmediateBasicBlockDominator(bb) + } + + BasicBlock getABasicBlockSuccessor(BasicBlock bb) { + result = Input::getABasicBlockSuccessor(bb) + } + + class ExitBasicBlock extends BasicBlock { + ExitBasicBlock() { exitBlock(this) } + } + + class SourceVariable = CaptureContainer; + + predicate variableWrite(BasicBlock bb, int i, SourceVariable cc, boolean certain) { + ( + exists(CapturedVariable v | cc = TVariable(v) and captureWrite(v, bb, i, true, _)) + or + entryDef(cc, bb, i) + ) and + certain = true + } + + predicate variableRead(BasicBlock bb, int i, SourceVariable cc, boolean certain) { + ( + synthThisQualifier(bb, i) and cc = TThis(bb.getEnclosingCallable()) + or + exists(CapturedVariable v | cc = TVariable(v) | + captureRead(v, bb, i, true, _) or synthRead(v, bb, i, true, _) + ) + ) and + certain = true + } + } + + private module CaptureSsa = Ssa::Make; + + private newtype TClosureNode = + TSynthRead(CapturedVariable v, BasicBlock bb, int i, Boolean isPost) { + synthRead(v, bb, i, _, _) + } or + TSynthThisQualifier(BasicBlock bb, int i, Boolean isPost) { synthThisQualifier(bb, i) } or + TSynthPhi(CaptureSsa::DefinitionExt phi) { + phi instanceof CaptureSsa::PhiNode or phi instanceof CaptureSsa::PhiReadNode + } or + TExprNode(Expr expr, boolean isPost) { + expr instanceof VariableRead and isPost = [false, true] + or + exists(VariableWrite vw | expr = vw.getSource() and isPost = false) + or + synthRead(_, _, _, _, expr) and isPost = [false, true] + } or + TParamNode(CapturedParameter p) or + TThisParamNode(Callable c) { captureAccess(_, c) } or + TMallocNode(ClosureExpr ce) { hasConstructorCapture(ce, _) } + + class ClosureNode extends TClosureNode { + /** Gets a textual representation of this node. */ + string toString() { + exists(CapturedVariable v | this = TSynthRead(v, _, _, _) and result = v.toString()) + or + result = "this" and this = TSynthThisQualifier(_, _, _) + or + exists(CaptureSsa::DefinitionExt phi, CaptureContainer cc | + this = TSynthPhi(phi) and + phi.definesAt(cc, _, _, _) and + result = "phi(" + cc.toString() + ")" + ) + or + exists(Expr expr, boolean isPost | this = TExprNode(expr, isPost) | + isPost = false and result = expr.toString() + or + isPost = true and result = expr.toString() + " [postupdate]" + ) + or + exists(CapturedParameter p | this = TParamNode(p) and result = p.toString()) + or + result = "this" and this = TThisParamNode(_) + or + result = "malloc" and this = TMallocNode(_) + } + + /** Gets the location of this node. */ + Location getLocation() { + exists(CapturedVariable v, BasicBlock bb, int i, Expr closure | + this = TSynthRead(v, bb, i, _) and + synthRead(v, bb, i, _, closure) and + result = closure.getLocation() + ) + or + exists(BasicBlock bb, int i | this = TSynthThisQualifier(bb, i, _) | + synthRead(_, bb, i, false, any(Expr closure | result = closure.getLocation())) or + captureRead(_, bb, i, false, any(VariableRead vr | result = vr.getLocation())) or + captureWrite(_, bb, i, false, any(VariableWrite vw | result = vw.getLocation())) + ) + or + exists(CaptureSsa::DefinitionExt phi, BasicBlock bb | + this = TSynthPhi(phi) and phi.definesAt(_, bb, _, _) and result = bb.getLocation() + ) + or + exists(Expr expr | this = TExprNode(expr, _) and result = expr.getLocation()) + or + exists(CapturedParameter p | this = TParamNode(p) and result = p.getCallable().getLocation()) + or + exists(Callable c | this = TThisParamNode(c) and result = c.getLocation()) + or + exists(ClosureExpr ce | this = TMallocNode(ce) and result = ce.getLocation()) + } + } + + private class TSynthesizedCaptureNode = TSynthRead or TSynthThisQualifier or TSynthPhi; + + class SynthesizedCaptureNode extends ClosureNode, TSynthesizedCaptureNode { + Callable getEnclosingCallable() { + exists(BasicBlock bb | this = TSynthRead(_, bb, _, _) and result = bb.getEnclosingCallable()) + or + exists(BasicBlock bb | + this = TSynthThisQualifier(bb, _, _) and result = bb.getEnclosingCallable() + ) + or + exists(CaptureSsa::DefinitionExt phi, BasicBlock bb | + this = TSynthPhi(phi) and phi.definesAt(_, bb, _, _) and result = bb.getEnclosingCallable() + ) + } + + predicate isVariableAccess(CapturedVariable v) { + this = TSynthRead(v, _, _, _) + or + exists(CaptureSsa::DefinitionExt phi | + this = TSynthPhi(phi) and phi.definesAt(TVariable(v), _, _, _) + ) + } + + predicate isInstanceAccess() { + this instanceof TSynthThisQualifier + or + exists(CaptureSsa::DefinitionExt phi | + this = TSynthPhi(phi) and phi.definesAt(TThis(_), _, _, _) + ) + } + } + + class ExprNode extends ClosureNode, TExprNode { + ExprNode() { this = TExprNode(_, false) } + + Expr getExpr() { this = TExprNode(result, _) } + } + + class ExprPostUpdateNode extends ClosureNode, TExprNode { + ExprPostUpdateNode() { this = TExprNode(_, true) } + + Expr getExpr() { this = TExprNode(result, _) } + } + + class ParameterNode extends ClosureNode, TParamNode { + CapturedParameter getParameter() { this = TParamNode(result) } + } + + class ThisParameterNode extends ClosureNode, TThisParamNode { + Callable getCallable() { this = TThisParamNode(result) } + } + + class MallocNode extends ClosureNode, TMallocNode { + ClosureExpr getClosureExpr() { this = TMallocNode(result) } + } + + predicate capturePostUpdateNode(SynthesizedCaptureNode post, SynthesizedCaptureNode pre) { + exists(CapturedVariable v, BasicBlock bb, int i | + pre = TSynthRead(v, bb, i, false) and post = TSynthRead(v, bb, i, true) + ) + or + exists(BasicBlock bb, int i | + pre = TSynthThisQualifier(bb, i, false) and post = TSynthThisQualifier(bb, i, true) + ) + } + + private predicate step(CaptureContainer cc, BasicBlock bb1, int i1, BasicBlock bb2, int i2) { + CaptureSsa::adjacentDefReadExt(_, cc, bb1, i1, bb2, i2) + } + + private predicate stepToPhi(CaptureContainer cc, BasicBlock bb, int i, TSynthPhi phi) { + exists(CaptureSsa::DefinitionExt next | + CaptureSsa::lastRefRedefExt(_, cc, bb, i, next) and + phi = TSynthPhi(next) + ) + } + + private predicate ssaAccessAt( + ClosureNode n, CaptureContainer cc, boolean isPost, BasicBlock bb, int i + ) { + exists(CapturedVariable v | + synthRead(v, bb, i, true, _) and + n = TSynthRead(v, bb, i, isPost) and + cc = TVariable(v) + ) + or + n = TSynthThisQualifier(bb, i, isPost) and cc = TThis(bb.getEnclosingCallable()) + or + exists(CaptureSsa::DefinitionExt phi | + n = TSynthPhi(phi) and phi.definesAt(cc, bb, i, _) and isPost = false + ) + or + exists(VariableRead vr, CapturedVariable v | + captureRead(v, bb, i, true, vr) and + n = TExprNode(vr, isPost) and + cc = TVariable(v) + ) + or + exists(VariableWrite vw, CapturedVariable v | + captureWrite(v, bb, i, true, vw) and + n = TExprNode(vw.getSource(), false) and + isPost = false and + cc = TVariable(v) + ) + or + exists(CapturedParameter p | + entryDef(cc, bb, i) and + cc = TVariable(p) and + n = TParamNode(p) and + isPost = false + ) + or + exists(Callable c | + entryDef(cc, bb, i) and + cc = TThis(c) and + n = TThisParamNode(c) and + isPost = false + ) + } + + predicate localFlowStep(ClosureNode node1, ClosureNode node2) { + exists(CaptureContainer cc, BasicBlock bb1, int i1, BasicBlock bb2, int i2 | + step(cc, bb1, i1, bb2, i2) and + ssaAccessAt(node1, pragma[only_bind_into](cc), _, bb1, i1) and + ssaAccessAt(node2, pragma[only_bind_into](cc), false, bb2, i2) + ) + or + exists(CaptureContainer cc, BasicBlock bb, int i | + stepToPhi(cc, bb, i, node2) and + ssaAccessAt(node1, cc, _, bb, i) + ) + } + + predicate storeStep(ClosureNode node1, CapturedVariable v, ClosureNode node2) { + // store v in the closure or in the malloc in case of a relevant constructor call + exists(BasicBlock bb, int i, Expr closure | + synthRead(v, bb, i, _, closure) and + node1 = TSynthRead(v, bb, i, false) + | + node2 = TExprNode(closure, false) + or + node2 = TMallocNode(closure) and hasConstructorCapture(closure, v) + ) + or + // write to v inside the closure body + exists(BasicBlock bb, int i, VariableWrite vw | + captureWrite(v, bb, i, false, vw) and + node1 = TExprNode(vw.getSource(), false) and + node2 = TSynthThisQualifier(bb, i, true) + ) + } + + predicate readStep(ClosureNode node1, CapturedVariable v, ClosureNode node2) { + // read v from the closure post-update to observe side-effects + exists(BasicBlock bb, int i, Expr closure, boolean post | + synthRead(v, bb, i, _, closure) and + node1 = TExprNode(closure, post) and + node2 = TSynthRead(v, bb, i, true) + | + post = true + or + // for a constructor call the regular ExprNode is the post-update for the MallocNode + post = false and hasConstructorCapture(closure, v) + ) + or + // read v from the closure inside the closure body + exists(BasicBlock bb, int i | node1 = TSynthThisQualifier(bb, i, false) | + synthRead(v, bb, i, false, _) and + node2 = TSynthRead(v, bb, i, false) + or + exists(VariableRead vr | + captureRead(v, bb, i, false, vr) and + node2 = TExprNode(vr, false) + ) + ) + } +} diff --git a/shared/dataflow/codeql/dataflow/DataFlowImpl.qll b/shared/dataflow/codeql/dataflow/internal/DataFlowImpl.qll similarity index 99% rename from shared/dataflow/codeql/dataflow/DataFlowImpl.qll rename to shared/dataflow/codeql/dataflow/internal/DataFlowImpl.qll index e18393920fa..d6c05ca042a 100644 --- a/shared/dataflow/codeql/dataflow/DataFlowImpl.qll +++ b/shared/dataflow/codeql/dataflow/internal/DataFlowImpl.qll @@ -6,11 +6,11 @@ private import codeql.util.Unit private import codeql.util.Option -import DataFlowParameter +private import codeql.dataflow.DataFlow -module MakeImpl { +module MakeImpl { private import Lang - private import DataFlow::DataFlowMake + private import DataFlowMake private import DataFlowImplCommon::MakeImplCommon private import DataFlowImplCommonPublic diff --git a/shared/dataflow/codeql/dataflow/DataFlowImplCommon.qll b/shared/dataflow/codeql/dataflow/internal/DataFlowImplCommon.qll similarity index 99% rename from shared/dataflow/codeql/dataflow/DataFlowImplCommon.qll rename to shared/dataflow/codeql/dataflow/internal/DataFlowImplCommon.qll index 4e8def5dcd4..27528918b6a 100644 --- a/shared/dataflow/codeql/dataflow/DataFlowImplCommon.qll +++ b/shared/dataflow/codeql/dataflow/internal/DataFlowImplCommon.qll @@ -1,6 +1,6 @@ -import DataFlowParameter +private import codeql.dataflow.DataFlow -module MakeImplCommon { +module MakeImplCommon { private import Lang import Cached diff --git a/swift/extractor/invocation/SwiftDiagnosticsConsumer.cpp b/swift/extractor/invocation/SwiftDiagnosticsConsumer.cpp index cee6f7caf55..9d5aa4654d5 100644 --- a/swift/extractor/invocation/SwiftDiagnosticsConsumer.cpp +++ b/swift/extractor/invocation/SwiftDiagnosticsConsumer.cpp @@ -1,5 +1,5 @@ #include "swift/extractor/invocation/SwiftDiagnosticsConsumer.h" -#include "swift/extractor/trap/generated/TrapClasses.h" +#include "swift/extractor/trap/generated/TrapEntries.h" #include "swift/extractor/trap/TrapDomain.h" #include "swift/extractor/infra/SwiftDiagnosticKind.h" @@ -13,17 +13,13 @@ using namespace codeql; void SwiftDiagnosticsConsumer::handleDiagnostic(swift::SourceManager& sourceManager, const swift::DiagnosticInfo& diagInfo) { - if (diagInfo.IsChildNote) return; - Diagnostics diag{trap.createTypedLabel()}; + auto message = getDiagMessage(sourceManager, diagInfo); + DiagnosticsTrap diag{}; + diag.id = trap.createTypedLabel(); diag.kind = translateDiagnosticsKind(diagInfo.Kind); - diag.text = getDiagMessage(sourceManager, diagInfo); + diag.text = message; trap.emit(diag); locationExtractor.attachLocation(sourceManager, diagInfo, diag.id); - - forwardToLog(sourceManager, diagInfo, diag.text); - for (const auto& child : diagInfo.ChildDiagnosticInfo) { - forwardToLog(sourceManager, *child); - } } std::string SwiftDiagnosticsConsumer::getDiagMessage(swift::SourceManager& sourceManager, @@ -33,29 +29,3 @@ std::string SwiftDiagnosticsConsumer::getDiagMessage(swift::SourceManager& sourc swift::DiagnosticEngine::formatDiagnosticText(out, diagInfo.FormatString, diagInfo.FormatArgs); return text.str().str(); } - -void SwiftDiagnosticsConsumer::forwardToLog(swift::SourceManager& sourceManager, - const swift::DiagnosticInfo& diagInfo, - const std::string& message) { - auto file = sourceManager.getDisplayNameForLoc(diagInfo.Loc); - auto [line, column] = sourceManager.getLineAndColumnInBuffer(diagInfo.Loc); - using Kind = swift::DiagnosticKind; - switch (diagInfo.Kind) { - case Kind::Error: - LOG_ERROR("{}:{}:{} {}", file, line, column, message); - break; - case Kind::Warning: - LOG_WARNING("{}:{}:{} {}", file, line, column, message); - break; - case Kind::Remark: - LOG_INFO("{}:{}:{} {}", file, line, column, message); - break; - case Kind::Note: - LOG_DEBUG("{}:{}:{} {}", file, line, column, message); - break; - default: - LOG_ERROR("unknown diagnostic kind {}, {}:{}:{} {}", diagInfo.Kind, file, line, column, - message); - break; - } -} diff --git a/swift/extractor/invocation/SwiftDiagnosticsConsumer.h b/swift/extractor/invocation/SwiftDiagnosticsConsumer.h index e2817981f76..c85233348fd 100644 --- a/swift/extractor/invocation/SwiftDiagnosticsConsumer.h +++ b/swift/extractor/invocation/SwiftDiagnosticsConsumer.h @@ -2,7 +2,6 @@ #include #include "swift/extractor/infra/SwiftLocationExtractor.h" -#include "swift/logging/SwiftLogging.h" namespace codeql { @@ -18,17 +17,8 @@ class SwiftDiagnosticsConsumer : public swift::DiagnosticConsumer { private: static std::string getDiagMessage(swift::SourceManager& sourceManager, const swift::DiagnosticInfo& diagInfo); - void forwardToLog(swift::SourceManager& sourceManager, - const swift::DiagnosticInfo& diagInfo, - const std::string& message); - - void forwardToLog(swift::SourceManager& sourceManager, const swift::DiagnosticInfo& diagInfo) { - forwardToLog(sourceManager, diagInfo, getDiagMessage(sourceManager, diagInfo)); - } - TrapDomain& trap; SwiftLocationExtractor locationExtractor; - Logger logger{"compiler"}; }; } // namespace codeql diff --git a/swift/extractor/main.cpp b/swift/extractor/main.cpp index 0abb5404035..4f1dd990be9 100644 --- a/swift/extractor/main.cpp +++ b/swift/extractor/main.cpp @@ -91,8 +91,6 @@ class Observer : public swift::FrontendObserver { } void configuredCompiler(swift::CompilerInstance& instance) override { - // remove default consumers to avoid double messaging - instance.getDiags().takeConsumers(); instance.addDiagnosticConsumer(&diagConsumer); } diff --git a/swift/ql/lib/change-notes/2023-08-04-collection-content.md b/swift/ql/lib/change-notes/2023-08-04-collection-content.md new file mode 100644 index 00000000000..28045e63027 --- /dev/null +++ b/swift/ql/lib/change-notes/2023-08-04-collection-content.md @@ -0,0 +1,4 @@ +--- +category: majorAnalysis +--- +* Added `DataFlow::CollectionContent`, which will enable more accurate flow through collections. \ No newline at end of file diff --git a/swift/ql/lib/change-notes/2023-08-07-forced-unwrap copy.md b/swift/ql/lib/change-notes/2023-08-07-forced-unwrap copy.md new file mode 100644 index 00000000000..1eb9bc53012 --- /dev/null +++ b/swift/ql/lib/change-notes/2023-08-07-forced-unwrap copy.md @@ -0,0 +1,5 @@ +--- +category: minorAnalysis +--- + +* Flow through forced optional unwrapping (`!`) on the left side of assignment now works in most cases. diff --git a/swift/ql/lib/codeql/swift/dataflow/ExternalFlow.qll b/swift/ql/lib/codeql/swift/dataflow/ExternalFlow.qll index 5778a760fef..b9223e35ad8 100644 --- a/swift/ql/lib/codeql/swift/dataflow/ExternalFlow.qll +++ b/swift/ql/lib/codeql/swift/dataflow/ExternalFlow.qll @@ -492,6 +492,9 @@ predicate parseContent(AccessPathToken component, Content content) { or component.getName() = "ArrayElement" and content instanceof Content::ArrayContent + or + component.getName() = "CollectionElement" and + content instanceof Content::CollectionContent } cached diff --git a/swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImpl.qll b/swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImpl.qll index 704933f6b5b..532f0def116 100644 --- a/swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImpl.qll +++ b/swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImpl.qll @@ -1,3 +1,3 @@ private import DataFlowImplSpecific -private import codeql.dataflow.DataFlowImpl +private import codeql.dataflow.internal.DataFlowImpl import MakeImpl diff --git a/swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImplCommon.qll b/swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImplCommon.qll index 95e61f3f17d..969275ffa07 100644 --- a/swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImplCommon.qll +++ b/swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImplCommon.qll @@ -1,3 +1,3 @@ private import DataFlowImplSpecific -private import codeql.dataflow.DataFlowImplCommon +private import codeql.dataflow.internal.DataFlowImplCommon import MakeImplCommon diff --git a/swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImplSpecific.qll b/swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImplSpecific.qll index 1496eaf9d79..e6941afd9d9 100644 --- a/swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImplSpecific.qll +++ b/swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImplSpecific.qll @@ -2,7 +2,7 @@ * Provides Swift-specific definitions for use in the data flow library. */ -private import codeql.dataflow.DataFlowParameter +private import codeql.dataflow.DataFlow // we need to export `Unit` for the DataFlowImpl* files private import swift as Swift @@ -15,7 +15,7 @@ module Public { import DataFlowPublic } -module SwiftDataFlow implements DataFlowParameter { +module SwiftDataFlow implements InputSig { import Private import Public diff --git a/swift/ql/lib/codeql/swift/dataflow/internal/DataFlowPrivate.qll b/swift/ql/lib/codeql/swift/dataflow/internal/DataFlowPrivate.qll index 02b32270a88..145ac20573b 100644 --- a/swift/ql/lib/codeql/swift/dataflow/internal/DataFlowPrivate.qll +++ b/swift/ql/lib/codeql/swift/dataflow/internal/DataFlowPrivate.qll @@ -256,7 +256,8 @@ private module Cached { TFieldContent(FieldDecl f) or TTupleContent(int index) { exists(any(TupleExpr te).getElement(index)) } or TEnumContent(ParamDecl f) { exists(EnumElementDecl d | d.getAParam() = f) } or - TArrayContent() + TArrayContent() or + TCollectionContent() } /** @@ -702,6 +703,14 @@ predicate storeStep(Node node1, ContentSet c, Node node2) { init.isFailable() ) or + // assignment to an optional via `!`, e.g. `optional! = ...` + exists(ForceValueExpr fve, AssignExpr assign | + fve = assign.getDest() and + node1.asExpr() = assign.getSource() and + node2.asExpr() = fve.getSubExpr() and + c instanceof OptionalSomeContentSet + ) + or // creation of an array `[v1,v2]` exists(ArrayExpr arr | node1.asExpr() = arr.getAnElement() and @@ -797,6 +806,9 @@ predicate readStep(Node node1, ContentSet c, Node node2) { subscript.getBase().getType() instanceof ArrayType and c.isSingleton(any(Content::ArrayContent ac)) ) + or + FlowSummaryImpl::Private::Steps::summaryReadStep(node1.(FlowSummaryNode).getSummaryNode(), c, + node2.(FlowSummaryNode).getSummaryNode()) } /** @@ -898,7 +910,9 @@ class DataFlowExpr = Expr; * Holds if access paths with `c` at their head always should be tracked at high * precision. This disables adaptive access path precision for such access paths. */ -predicate forceHighPrecision(Content c) { c instanceof Content::ArrayContent } +predicate forceHighPrecision(Content c) { + c instanceof Content::ArrayContent or c instanceof Content::CollectionContent +} /** * Holds if the node `n` is unreachable when the call context is `call`. diff --git a/swift/ql/lib/codeql/swift/dataflow/internal/DataFlowPublic.qll b/swift/ql/lib/codeql/swift/dataflow/internal/DataFlowPublic.qll index 6fccd145928..956e31fce36 100644 --- a/swift/ql/lib/codeql/swift/dataflow/internal/DataFlowPublic.qll +++ b/swift/ql/lib/codeql/swift/dataflow/internal/DataFlowPublic.qll @@ -224,6 +224,11 @@ module Content { class ArrayContent extends Content, TArrayContent { override string toString() { result = "Array element" } } + + /** An element of a collection. */ + class CollectionContent extends Content, TCollectionContent { + override string toString() { result = "Collection element" } + } } /** diff --git a/swift/ql/lib/codeql/swift/dataflow/internal/FlowSummaryImplSpecific.qll b/swift/ql/lib/codeql/swift/dataflow/internal/FlowSummaryImplSpecific.qll index ca1b9a316b5..99ac7888b4f 100644 --- a/swift/ql/lib/codeql/swift/dataflow/internal/FlowSummaryImplSpecific.qll +++ b/swift/ql/lib/codeql/swift/dataflow/internal/FlowSummaryImplSpecific.qll @@ -110,10 +110,20 @@ private string getContentSpecific(ContentSet cs) { result = "Field[" + c.getField().getName() + "]" ) or + exists(Content::EnumContent c | + cs.isSingleton(c) and + result = "EnumElement[" + c.getSignature() + "]" + ) + or exists(Content::ArrayContent c | cs.isSingleton(c) and result = "ArrayElement" ) + or + exists(Content::CollectionContent c | + cs.isSingleton(c) and + result = "CollectionElement" + ) } /** Gets the textual representation of a summary component in the format used for MaD models. */ @@ -125,6 +135,16 @@ string getMadRepresentationSpecific(SummaryComponent sc) { not rk = getReturnValueKind() and result = "ReturnValue" + "[" + rk + "]" ) + or + exists(ContentSet c | + sc = TWithoutContentSummaryComponent(c) and + result = "WithoutContent" + c.toString() + ) + or + exists(ContentSet c | + sc = TWithContentSummaryComponent(c) and + result = "WithContent" + c.toString() + ) } /** Gets the textual representation of a parameter position in the format used for flow summaries. */ diff --git a/swift/ql/lib/codeql/swift/frameworks/StandardLibrary/Collection.qll b/swift/ql/lib/codeql/swift/frameworks/StandardLibrary/Collection.qll index 6022d4b767a..adb78248ccd 100644 --- a/swift/ql/lib/codeql/swift/frameworks/StandardLibrary/Collection.qll +++ b/swift/ql/lib/codeql/swift/frameworks/StandardLibrary/Collection.qll @@ -26,6 +26,8 @@ private class CollectionSummaries extends SummaryModelCsv { ";Collection;true;split(separator:maxSplits:omittingEmptySubsequences:);;;Argument[-1];ReturnValue;taint", ";Collection;true;removeFirst();;;Argument[-1];ReturnValue;taint", ";Collection;true;popFirst();;;Argument[-1];ReturnValue;taint", + ";Collection;true;randomElement();;;Argument[-1].CollectionElement;ReturnValue.OptionalSome;value", + ";Collection;true;randomElement();;;Argument[-1].ArrayElement;ReturnValue.OptionalSome;value", ";RangeReplaceableCollection;true;append(_:);;;Argument[0];Argument[-1];taint", ";RangeReplaceableCollection;true;append(contentsOf:);;;Argument[0];Argument[-1];taint", ";RangeReplaceableCollection;true;remove(at:);;;Argument[-1];ReturnValue;taint", diff --git a/swift/ql/lib/codeql/swift/frameworks/StandardLibrary/CustomUrlSchemes.qll b/swift/ql/lib/codeql/swift/frameworks/StandardLibrary/CustomUrlSchemes.qll index 97f0529159f..3e2e8f76714 100644 --- a/swift/ql/lib/codeql/swift/frameworks/StandardLibrary/CustomUrlSchemes.qll +++ b/swift/ql/lib/codeql/swift/frameworks/StandardLibrary/CustomUrlSchemes.qll @@ -56,9 +56,7 @@ private class ApplicationWithLaunchOptionsFunc extends Function { private class LaunchOptionsUrlVarDecl extends VarDecl { LaunchOptionsUrlVarDecl() { - // ideally this would be the more accurate, but currently less robust: - // this.getEnclosingDecl().asNominalTypeDecl().getFullName() = "UIApplication.LaunchOptionsKey" and - this.getType().(NominalType).getFullName() = "UIApplication.LaunchOptionsKey" and + this.getEnclosingDecl().asNominalTypeDecl().getFullName() = "UIApplication.LaunchOptionsKey" and this.getName() = "url" } } diff --git a/swift/ql/lib/codeql/swift/frameworks/StandardLibrary/Set.qll b/swift/ql/lib/codeql/swift/frameworks/StandardLibrary/Set.qll new file mode 100644 index 00000000000..e5198d043bb --- /dev/null +++ b/swift/ql/lib/codeql/swift/frameworks/StandardLibrary/Set.qll @@ -0,0 +1,20 @@ +/** + * Provides models for `Set` and related Swift classes. + */ + +private import codeql.swift.dataflow.ExternalFlow + +/** + * A model for `Set` and related class members that permit data flow. + */ +private class SetSummaries extends SummaryModelCsv { + override predicate row(string row) { + row = + [ + ";Set;true;insert(_:);;;Argument[-1].CollectionElement;ReturnValue.TupleElement[1];value", + ";Set;true;insert(_:);;;Argument[0];Argument[-1].CollectionElement;value", + ";Set;true;insert(_:);;;Argument[0];ReturnValue.TupleElement[1];value", + ";Set;true;init(_:);;;Argument[0].ArrayElement;ReturnValue.CollectionElement;value" + ] + } +} diff --git a/swift/ql/lib/codeql/swift/frameworks/StandardLibrary/StandardLibrary.qll b/swift/ql/lib/codeql/swift/frameworks/StandardLibrary/StandardLibrary.qll index 14fbb3f6006..93d60dba092 100644 --- a/swift/ql/lib/codeql/swift/frameworks/StandardLibrary/StandardLibrary.qll +++ b/swift/ql/lib/codeql/swift/frameworks/StandardLibrary/StandardLibrary.qll @@ -15,6 +15,7 @@ private import NsObject private import NsString private import NsUrl private import Sequence +private import Set private import String private import Url private import UrlSession diff --git a/swift/ql/test/library-tests/dataflow/dataflow/DataFlow.expected b/swift/ql/test/library-tests/dataflow/dataflow/DataFlow.expected index bab6db77c24..b0404de37ec 100644 --- a/swift/ql/test/library-tests/dataflow/dataflow/DataFlow.expected +++ b/swift/ql/test/library-tests/dataflow/dataflow/DataFlow.expected @@ -1,9 +1,15 @@ edges | file://:0:0:0:0 | self [a, x] | file://:0:0:0:0 | .a [x] | | file://:0:0:0:0 | self [str] | file://:0:0:0:0 | .str | +| file://:0:0:0:0 | self [v2, some:0] | file://:0:0:0:0 | .v2 [some:0] | +| file://:0:0:0:0 | self [v2] | file://:0:0:0:0 | .v2 | +| file://:0:0:0:0 | self [v3] | file://:0:0:0:0 | .v3 | | file://:0:0:0:0 | self [x, some:0] | file://:0:0:0:0 | .x [some:0] | | file://:0:0:0:0 | self [x] | file://:0:0:0:0 | .x | +| file://:0:0:0:0 | value | file://:0:0:0:0 | [post] self [v2] | +| file://:0:0:0:0 | value | file://:0:0:0:0 | [post] self [v3] | | file://:0:0:0:0 | value | file://:0:0:0:0 | [post] self [x] | +| file://:0:0:0:0 | value [some:0] | file://:0:0:0:0 | [post] self [v2, some:0] | | file://:0:0:0:0 | value [some:0] | file://:0:0:0:0 | [post] self [x, some:0] | | test.swift:6:19:6:26 | call to source() | test.swift:7:15:7:15 | t1 | | test.swift:6:19:6:26 | call to source() | test.swift:9:15:9:15 | t1 | @@ -336,18 +342,77 @@ edges | test.swift:693:5:693:5 | [post] arr6 [Array element] | test.swift:694:15:694:15 | arr6 [Array element] | | test.swift:693:17:693:24 | call to source() | test.swift:693:5:693:5 | [post] arr6 [Array element] | | test.swift:694:15:694:15 | arr6 [Array element] | test.swift:694:15:694:21 | ...[...] | +| test.swift:696:16:696:25 | [...] [Array element] | test.swift:697:15:697:15 | arr7 [Array element] | +| test.swift:696:17:696:24 | call to source() | test.swift:696:16:696:25 | [...] [Array element] | +| test.swift:697:15:697:15 | arr7 [Array element] | test.swift:697:15:697:34 | call to randomElement() [some:0] | +| test.swift:697:15:697:34 | call to randomElement() [some:0] | test.swift:697:15:697:35 | ...! | +| test.swift:703:5:703:5 | [post] set1 [Collection element] | test.swift:704:15:704:15 | set1 [Collection element] | +| test.swift:703:17:703:24 | call to source() | test.swift:703:5:703:5 | [post] set1 [Collection element] | +| test.swift:704:15:704:15 | set1 [Collection element] | test.swift:704:15:704:34 | call to randomElement() [some:0] | +| test.swift:704:15:704:34 | call to randomElement() [some:0] | test.swift:704:15:704:35 | ...! | +| test.swift:706:16:706:30 | call to Set.init(_:) [Collection element] | test.swift:707:15:707:15 | set2 [Collection element] | +| test.swift:706:20:706:29 | [...] [Array element] | test.swift:706:16:706:30 | call to Set.init(_:) [Collection element] | +| test.swift:706:21:706:28 | call to source() | test.swift:706:20:706:29 | [...] [Array element] | +| test.swift:707:15:707:15 | set2 [Collection element] | test.swift:707:15:707:34 | call to randomElement() [some:0] | +| test.swift:707:15:707:34 | call to randomElement() [some:0] | test.swift:707:15:707:35 | ...! | +| test.swift:712:9:712:9 | self [v2, some:0] | file://:0:0:0:0 | self [v2, some:0] | +| test.swift:712:9:712:9 | self [v2] | file://:0:0:0:0 | self [v2] | +| test.swift:712:9:712:9 | value | file://:0:0:0:0 | value | +| test.swift:712:9:712:9 | value [some:0] | file://:0:0:0:0 | value [some:0] | +| test.swift:713:9:713:9 | self [v3] | file://:0:0:0:0 | self [v3] | +| test.swift:713:9:713:9 | value | file://:0:0:0:0 | value | +| test.swift:723:5:723:5 | v1 [some:0] | test.swift:733:15:733:15 | v1 [some:0] | +| test.swift:723:11:723:18 | call to source() | test.swift:723:5:723:5 | v1 [some:0] | +| test.swift:724:10:724:17 | call to source() | test.swift:724:10:724:17 | call to source() [some:0] | +| test.swift:724:10:724:17 | call to source() | test.swift:734:15:734:17 | ...! | +| test.swift:724:10:724:17 | call to source() [some:0] | test.swift:734:15:734:15 | v2 [some:0] | +| test.swift:725:10:725:17 | call to source() | test.swift:735:15:735:15 | v3 | +| test.swift:727:5:727:5 | [post] mo1 [v2, some:0] | test.swift:728:5:728:5 | mo1 [v2, some:0] | +| test.swift:727:5:727:5 | [post] mo1 [v2] | test.swift:728:5:728:5 | mo1 [v2] | +| test.swift:727:14:727:21 | call to source() | test.swift:712:9:712:9 | value | +| test.swift:727:14:727:21 | call to source() | test.swift:727:5:727:5 | [post] mo1 [v2] | +| test.swift:727:14:727:21 | call to source() | test.swift:727:14:727:21 | call to source() [some:0] | +| test.swift:727:14:727:21 | call to source() [some:0] | test.swift:712:9:712:9 | value [some:0] | +| test.swift:727:14:727:21 | call to source() [some:0] | test.swift:727:5:727:5 | [post] mo1 [v2, some:0] | +| test.swift:728:5:728:5 | [post] mo1 [v3] | test.swift:738:15:738:15 | mo1 [v3] | +| test.swift:728:5:728:5 | mo1 [v2, some:0] | test.swift:737:15:737:15 | mo1 [v2, some:0] | +| test.swift:728:5:728:5 | mo1 [v2] | test.swift:737:15:737:15 | mo1 [v2] | +| test.swift:728:14:728:21 | call to source() | test.swift:713:9:713:9 | value | +| test.swift:728:14:728:21 | call to source() | test.swift:728:5:728:5 | [post] mo1 [v3] | +| test.swift:733:15:733:15 | v1 [some:0] | test.swift:733:15:733:17 | ...! | +| test.swift:734:15:734:15 | v2 [some:0] | test.swift:734:15:734:17 | ...! | +| test.swift:737:15:737:15 | mo1 [v2, some:0] | test.swift:712:9:712:9 | self [v2, some:0] | +| test.swift:737:15:737:15 | mo1 [v2, some:0] | test.swift:737:15:737:19 | .v2 [some:0] | +| test.swift:737:15:737:15 | mo1 [v2] | test.swift:712:9:712:9 | self [v2] | +| test.swift:737:15:737:15 | mo1 [v2] | test.swift:737:15:737:19 | .v2 | +| test.swift:737:15:737:19 | .v2 | test.swift:737:15:737:21 | ...! | +| test.swift:737:15:737:19 | .v2 [some:0] | test.swift:737:15:737:21 | ...! | +| test.swift:738:15:738:15 | mo1 [v3] | test.swift:713:9:713:9 | self [v3] | +| test.swift:738:15:738:15 | mo1 [v3] | test.swift:738:15:738:19 | .v3 | nodes | file://:0:0:0:0 | .a [x] | semmle.label | .a [x] | | file://:0:0:0:0 | .str | semmle.label | .str | +| file://:0:0:0:0 | .v2 | semmle.label | .v2 | +| file://:0:0:0:0 | .v2 [some:0] | semmle.label | .v2 [some:0] | +| file://:0:0:0:0 | .v3 | semmle.label | .v3 | | file://:0:0:0:0 | .x | semmle.label | .x | | file://:0:0:0:0 | .x [some:0] | semmle.label | .x [some:0] | +| file://:0:0:0:0 | [post] self [v2, some:0] | semmle.label | [post] self [v2, some:0] | +| file://:0:0:0:0 | [post] self [v2] | semmle.label | [post] self [v2] | +| file://:0:0:0:0 | [post] self [v3] | semmle.label | [post] self [v3] | | file://:0:0:0:0 | [post] self [x, some:0] | semmle.label | [post] self [x, some:0] | | file://:0:0:0:0 | [post] self [x] | semmle.label | [post] self [x] | | file://:0:0:0:0 | self [a, x] | semmle.label | self [a, x] | | file://:0:0:0:0 | self [str] | semmle.label | self [str] | +| file://:0:0:0:0 | self [v2, some:0] | semmle.label | self [v2, some:0] | +| file://:0:0:0:0 | self [v2] | semmle.label | self [v2] | +| file://:0:0:0:0 | self [v3] | semmle.label | self [v3] | | file://:0:0:0:0 | self [x, some:0] | semmle.label | self [x, some:0] | | file://:0:0:0:0 | self [x] | semmle.label | self [x] | | file://:0:0:0:0 | value | semmle.label | value | +| file://:0:0:0:0 | value | semmle.label | value | +| file://:0:0:0:0 | value | semmle.label | value | +| file://:0:0:0:0 | value [some:0] | semmle.label | value [some:0] | | file://:0:0:0:0 | value [some:0] | semmle.label | value [some:0] | | test.swift:6:19:6:26 | call to source() | semmle.label | call to source() | | test.swift:7:15:7:15 | t1 | semmle.label | t1 | @@ -706,6 +771,53 @@ nodes | test.swift:693:17:693:24 | call to source() | semmle.label | call to source() | | test.swift:694:15:694:15 | arr6 [Array element] | semmle.label | arr6 [Array element] | | test.swift:694:15:694:21 | ...[...] | semmle.label | ...[...] | +| test.swift:696:16:696:25 | [...] [Array element] | semmle.label | [...] [Array element] | +| test.swift:696:17:696:24 | call to source() | semmle.label | call to source() | +| test.swift:697:15:697:15 | arr7 [Array element] | semmle.label | arr7 [Array element] | +| test.swift:697:15:697:34 | call to randomElement() [some:0] | semmle.label | call to randomElement() [some:0] | +| test.swift:697:15:697:35 | ...! | semmle.label | ...! | +| test.swift:703:5:703:5 | [post] set1 [Collection element] | semmle.label | [post] set1 [Collection element] | +| test.swift:703:17:703:24 | call to source() | semmle.label | call to source() | +| test.swift:704:15:704:15 | set1 [Collection element] | semmle.label | set1 [Collection element] | +| test.swift:704:15:704:34 | call to randomElement() [some:0] | semmle.label | call to randomElement() [some:0] | +| test.swift:704:15:704:35 | ...! | semmle.label | ...! | +| test.swift:706:16:706:30 | call to Set.init(_:) [Collection element] | semmle.label | call to Set.init(_:) [Collection element] | +| test.swift:706:20:706:29 | [...] [Array element] | semmle.label | [...] [Array element] | +| test.swift:706:21:706:28 | call to source() | semmle.label | call to source() | +| test.swift:707:15:707:15 | set2 [Collection element] | semmle.label | set2 [Collection element] | +| test.swift:707:15:707:34 | call to randomElement() [some:0] | semmle.label | call to randomElement() [some:0] | +| test.swift:707:15:707:35 | ...! | semmle.label | ...! | +| test.swift:712:9:712:9 | self [v2, some:0] | semmle.label | self [v2, some:0] | +| test.swift:712:9:712:9 | self [v2] | semmle.label | self [v2] | +| test.swift:712:9:712:9 | value | semmle.label | value | +| test.swift:712:9:712:9 | value [some:0] | semmle.label | value [some:0] | +| test.swift:713:9:713:9 | self [v3] | semmle.label | self [v3] | +| test.swift:713:9:713:9 | value | semmle.label | value | +| test.swift:723:5:723:5 | v1 [some:0] | semmle.label | v1 [some:0] | +| test.swift:723:11:723:18 | call to source() | semmle.label | call to source() | +| test.swift:724:10:724:17 | call to source() | semmle.label | call to source() | +| test.swift:724:10:724:17 | call to source() [some:0] | semmle.label | call to source() [some:0] | +| test.swift:725:10:725:17 | call to source() | semmle.label | call to source() | +| test.swift:727:5:727:5 | [post] mo1 [v2, some:0] | semmle.label | [post] mo1 [v2, some:0] | +| test.swift:727:5:727:5 | [post] mo1 [v2] | semmle.label | [post] mo1 [v2] | +| test.swift:727:14:727:21 | call to source() | semmle.label | call to source() | +| test.swift:727:14:727:21 | call to source() [some:0] | semmle.label | call to source() [some:0] | +| test.swift:728:5:728:5 | [post] mo1 [v3] | semmle.label | [post] mo1 [v3] | +| test.swift:728:5:728:5 | mo1 [v2, some:0] | semmle.label | mo1 [v2, some:0] | +| test.swift:728:5:728:5 | mo1 [v2] | semmle.label | mo1 [v2] | +| test.swift:728:14:728:21 | call to source() | semmle.label | call to source() | +| test.swift:733:15:733:15 | v1 [some:0] | semmle.label | v1 [some:0] | +| test.swift:733:15:733:17 | ...! | semmle.label | ...! | +| test.swift:734:15:734:15 | v2 [some:0] | semmle.label | v2 [some:0] | +| test.swift:734:15:734:17 | ...! | semmle.label | ...! | +| test.swift:735:15:735:15 | v3 | semmle.label | v3 | +| test.swift:737:15:737:15 | mo1 [v2, some:0] | semmle.label | mo1 [v2, some:0] | +| test.swift:737:15:737:15 | mo1 [v2] | semmle.label | mo1 [v2] | +| test.swift:737:15:737:19 | .v2 | semmle.label | .v2 | +| test.swift:737:15:737:19 | .v2 [some:0] | semmle.label | .v2 [some:0] | +| test.swift:737:15:737:21 | ...! | semmle.label | ...! | +| test.swift:738:15:738:15 | mo1 [v3] | semmle.label | mo1 [v3] | +| test.swift:738:15:738:19 | .v3 | semmle.label | .v3 | subpaths | test.swift:75:22:75:22 | x | test.swift:65:16:65:28 | arg1 | test.swift:65:1:70:1 | arg2[return] | test.swift:75:32:75:32 | [post] y | | test.swift:114:19:114:19 | arg | test.swift:109:9:109:14 | arg | test.swift:110:12:110:12 | arg | test.swift:114:12:114:22 | call to ... | @@ -747,6 +859,12 @@ subpaths | test.swift:622:18:622:18 | s [x] | test.swift:615:8:615:11 | s [x] | test.swift:615:3:617:3 | self[return] [s, x] | test.swift:622:12:622:19 | call to S2.init(s:) [s, x] | | test.swift:624:13:624:13 | s2 [s, x] | test.swift:623:11:623:17 | enter #keyPath(...) [s, x] | test.swift:623:11:623:17 | exit #keyPath(...) | test.swift:624:13:624:26 | \\...[...] | | test.swift:630:15:630:15 | array [Array element] | test.swift:629:13:629:22 | enter #keyPath(...) [Array element] | test.swift:629:13:629:22 | exit #keyPath(...) | test.swift:630:15:630:31 | \\...[...] | +| test.swift:727:14:727:21 | call to source() | test.swift:712:9:712:9 | value | file://:0:0:0:0 | [post] self [v2] | test.swift:727:5:727:5 | [post] mo1 [v2] | +| test.swift:727:14:727:21 | call to source() [some:0] | test.swift:712:9:712:9 | value [some:0] | file://:0:0:0:0 | [post] self [v2, some:0] | test.swift:727:5:727:5 | [post] mo1 [v2, some:0] | +| test.swift:728:14:728:21 | call to source() | test.swift:713:9:713:9 | value | file://:0:0:0:0 | [post] self [v3] | test.swift:728:5:728:5 | [post] mo1 [v3] | +| test.swift:737:15:737:15 | mo1 [v2, some:0] | test.swift:712:9:712:9 | self [v2, some:0] | file://:0:0:0:0 | .v2 [some:0] | test.swift:737:15:737:19 | .v2 [some:0] | +| test.swift:737:15:737:15 | mo1 [v2] | test.swift:712:9:712:9 | self [v2] | file://:0:0:0:0 | .v2 | test.swift:737:15:737:19 | .v2 | +| test.swift:738:15:738:15 | mo1 [v3] | test.swift:713:9:713:9 | self [v3] | file://:0:0:0:0 | .v3 | test.swift:738:15:738:19 | .v3 | #select | test.swift:7:15:7:15 | t1 | test.swift:6:19:6:26 | call to source() | test.swift:7:15:7:15 | t1 | result | | test.swift:9:15:9:15 | t1 | test.swift:6:19:6:26 | call to source() | test.swift:9:15:9:15 | t1 | result | @@ -835,3 +953,11 @@ subpaths | test.swift:678:15:678:26 | ...[...] | test.swift:676:20:676:27 | call to source() | test.swift:678:15:678:26 | ...[...] | result | | test.swift:682:15:682:27 | ...[...] | test.swift:681:21:681:28 | call to source() | test.swift:682:15:682:27 | ...[...] | result | | test.swift:694:15:694:21 | ...[...] | test.swift:693:17:693:24 | call to source() | test.swift:694:15:694:21 | ...[...] | result | +| test.swift:697:15:697:35 | ...! | test.swift:696:17:696:24 | call to source() | test.swift:697:15:697:35 | ...! | result | +| test.swift:704:15:704:35 | ...! | test.swift:703:17:703:24 | call to source() | test.swift:704:15:704:35 | ...! | result | +| test.swift:707:15:707:35 | ...! | test.swift:706:21:706:28 | call to source() | test.swift:707:15:707:35 | ...! | result | +| test.swift:733:15:733:17 | ...! | test.swift:723:11:723:18 | call to source() | test.swift:733:15:733:17 | ...! | result | +| test.swift:734:15:734:17 | ...! | test.swift:724:10:724:17 | call to source() | test.swift:734:15:734:17 | ...! | result | +| test.swift:735:15:735:15 | v3 | test.swift:725:10:725:17 | call to source() | test.swift:735:15:735:15 | v3 | result | +| test.swift:737:15:737:21 | ...! | test.swift:727:14:727:21 | call to source() | test.swift:737:15:737:21 | ...! | result | +| test.swift:738:15:738:19 | .v3 | test.swift:728:14:728:21 | call to source() | test.swift:738:15:738:19 | .v3 | result | diff --git a/swift/ql/test/library-tests/dataflow/dataflow/LocalFlow.expected b/swift/ql/test/library-tests/dataflow/dataflow/LocalFlow.expected index 99e9ceb5ba8..6058e6e645b 100644 --- a/swift/ql/test/library-tests/dataflow/dataflow/LocalFlow.expected +++ b/swift/ql/test/library-tests/dataflow/dataflow/LocalFlow.expected @@ -811,3 +811,86 @@ | test.swift:693:5:693:5 | arr6 | test.swift:693:5:693:5 | &... | | test.swift:694:15:694:15 | [post] arr6 | test.swift:694:15:694:15 | &... | | test.swift:694:15:694:15 | arr6 | test.swift:694:15:694:15 | &... | +| test.swift:696:9:696:9 | SSA def(arr7) | test.swift:697:15:697:15 | arr7 | +| test.swift:696:9:696:9 | arr7 | test.swift:696:9:696:9 | SSA def(arr7) | +| test.swift:696:16:696:25 | [...] | test.swift:696:9:696:9 | arr7 | +| test.swift:697:15:697:34 | call to randomElement() | test.swift:697:15:697:35 | ...! | +| test.swift:701:9:701:9 | SSA def(set1) | test.swift:702:15:702:15 | set1 | +| test.swift:701:9:701:9 | set1 | test.swift:701:9:701:9 | SSA def(set1) | +| test.swift:701:9:701:15 | ... as ... | test.swift:701:9:701:9 | set1 | +| test.swift:701:21:701:27 | [...] | test.swift:701:9:701:15 | ... as ... | +| test.swift:702:15:702:15 | [post] set1 | test.swift:703:5:703:5 | set1 | +| test.swift:702:15:702:15 | set1 | test.swift:703:5:703:5 | set1 | +| test.swift:702:15:702:34 | call to randomElement() | test.swift:702:15:702:35 | ...! | +| test.swift:703:5:703:5 | &... | test.swift:704:15:704:15 | set1 | +| test.swift:703:5:703:5 | [post] set1 | test.swift:703:5:703:5 | &... | +| test.swift:703:5:703:5 | set1 | test.swift:703:5:703:5 | &... | +| test.swift:704:15:704:34 | call to randomElement() | test.swift:704:15:704:35 | ...! | +| test.swift:706:9:706:9 | SSA def(set2) | test.swift:707:15:707:15 | set2 | +| test.swift:706:9:706:9 | set2 | test.swift:706:9:706:9 | SSA def(set2) | +| test.swift:706:16:706:30 | call to Set.init(_:) | test.swift:706:9:706:9 | set2 | +| test.swift:707:15:707:34 | call to randomElement() | test.swift:707:15:707:35 | ...! | +| test.swift:710:8:710:8 | SSA def(self) | test.swift:710:8:710:8 | self[return] | +| test.swift:710:8:710:8 | self | test.swift:710:8:710:8 | SSA def(self) | +| test.swift:711:9:711:9 | self | test.swift:711:9:711:9 | SSA def(self) | +| test.swift:711:9:711:9 | self | test.swift:711:9:711:9 | SSA def(self) | +| test.swift:711:9:711:9 | self | test.swift:711:9:711:9 | SSA def(self) | +| test.swift:711:9:711:9 | value | test.swift:711:9:711:9 | SSA def(value) | +| test.swift:712:9:712:9 | self | test.swift:712:9:712:9 | SSA def(self) | +| test.swift:712:9:712:9 | self | test.swift:712:9:712:9 | SSA def(self) | +| test.swift:712:9:712:9 | self | test.swift:712:9:712:9 | SSA def(self) | +| test.swift:712:9:712:9 | value | test.swift:712:9:712:9 | SSA def(value) | +| test.swift:713:9:713:9 | self | test.swift:713:9:713:9 | SSA def(self) | +| test.swift:713:9:713:9 | self | test.swift:713:9:713:9 | SSA def(self) | +| test.swift:713:9:713:9 | self | test.swift:713:9:713:9 | SSA def(self) | +| test.swift:713:9:713:9 | value | test.swift:713:9:713:9 | SSA def(value) | +| test.swift:717:9:717:9 | SSA def(v1) | test.swift:723:5:723:5 | v1 | +| test.swift:717:9:717:9 | v1 | test.swift:717:9:717:9 | SSA def(v1) | +| test.swift:717:9:717:17 | ... as ... | test.swift:717:9:717:9 | v1 | +| test.swift:717:21:717:21 | 0 | test.swift:717:9:717:17 | ... as ... | +| test.swift:718:9:718:17 | ... as ... | test.swift:718:9:718:9 | v2 | +| test.swift:718:21:718:21 | 0 | test.swift:718:9:718:17 | ... as ... | +| test.swift:719:9:719:17 | ... as ... | test.swift:719:9:719:9 | v3 | +| test.swift:719:21:719:21 | 0 | test.swift:719:9:719:17 | ... as ... | +| test.swift:720:9:720:9 | SSA def(mo1) | test.swift:726:5:726:5 | mo1 | +| test.swift:720:9:720:9 | mo1 | test.swift:720:9:720:9 | SSA def(mo1) | +| test.swift:720:15:720:27 | call to MyOptionals.init() | test.swift:720:9:720:9 | mo1 | +| test.swift:721:9:721:9 | SSA def(mo2) | test.swift:729:5:729:5 | mo2 | +| test.swift:721:9:721:9 | mo2 | test.swift:721:9:721:9 | SSA def(mo2) | +| test.swift:721:9:721:26 | ... as ... | test.swift:721:9:721:9 | mo2 | +| test.swift:721:30:721:42 | call to MyOptionals.init() | test.swift:721:9:721:26 | ... as ... | +| test.swift:723:5:723:5 | v1 | test.swift:723:5:723:7 | ...! | +| test.swift:723:5:723:5 | v1 | test.swift:733:15:733:15 | v1 | +| test.swift:724:5:724:17 | SSA def(v2) | test.swift:734:15:734:15 | v2 | +| test.swift:724:10:724:17 | call to source() | test.swift:724:5:724:17 | SSA def(v2) | +| test.swift:725:5:725:17 | SSA def(v3) | test.swift:735:15:735:15 | v3 | +| test.swift:725:10:725:17 | call to source() | test.swift:725:5:725:17 | SSA def(v3) | +| test.swift:726:5:726:5 | [post] mo1 | test.swift:727:5:727:5 | mo1 | +| test.swift:726:5:726:5 | mo1 | test.swift:727:5:727:5 | mo1 | +| test.swift:726:5:726:9 | .v1 | test.swift:726:5:726:11 | ...! | +| test.swift:727:5:727:5 | [post] mo1 | test.swift:728:5:728:5 | mo1 | +| test.swift:727:5:727:5 | mo1 | test.swift:728:5:728:5 | mo1 | +| test.swift:728:5:728:5 | [post] mo1 | test.swift:736:15:736:15 | mo1 | +| test.swift:728:5:728:5 | mo1 | test.swift:736:15:736:15 | mo1 | +| test.swift:729:5:729:5 | mo2 | test.swift:729:5:729:8 | ...! | +| test.swift:729:5:729:5 | mo2 | test.swift:730:5:730:5 | mo2 | +| test.swift:729:5:729:10 | .v1 | test.swift:729:5:729:12 | ...! | +| test.swift:730:5:730:5 | mo2 | test.swift:730:5:730:8 | ...! | +| test.swift:730:5:730:5 | mo2 | test.swift:731:5:731:5 | mo2 | +| test.swift:731:5:731:5 | mo2 | test.swift:731:5:731:8 | ...! | +| test.swift:731:5:731:5 | mo2 | test.swift:739:15:739:15 | mo2 | +| test.swift:733:15:733:15 | v1 | test.swift:733:15:733:17 | ...! | +| test.swift:734:15:734:15 | v2 | test.swift:734:15:734:17 | ...! | +| test.swift:736:15:736:15 | [post] mo1 | test.swift:737:15:737:15 | mo1 | +| test.swift:736:15:736:15 | mo1 | test.swift:737:15:737:15 | mo1 | +| test.swift:736:15:736:19 | .v1 | test.swift:736:15:736:21 | ...! | +| test.swift:737:15:737:15 | [post] mo1 | test.swift:738:15:738:15 | mo1 | +| test.swift:737:15:737:15 | mo1 | test.swift:738:15:738:15 | mo1 | +| test.swift:737:15:737:19 | .v2 | test.swift:737:15:737:21 | ...! | +| test.swift:739:15:739:15 | mo2 | test.swift:739:15:739:18 | ...! | +| test.swift:739:15:739:15 | mo2 | test.swift:740:15:740:15 | mo2 | +| test.swift:739:15:739:20 | .v1 | test.swift:739:15:739:22 | ...! | +| test.swift:740:15:740:15 | mo2 | test.swift:740:15:740:18 | ...! | +| test.swift:740:15:740:15 | mo2 | test.swift:741:15:741:15 | mo2 | +| test.swift:740:15:740:20 | .v2 | test.swift:740:15:740:22 | ...! | +| test.swift:741:15:741:15 | mo2 | test.swift:741:15:741:18 | ...! | diff --git a/swift/ql/test/library-tests/dataflow/dataflow/test.swift b/swift/ql/test/library-tests/dataflow/dataflow/test.swift index e789964a54f..2d738e14c94 100644 --- a/swift/ql/test/library-tests/dataflow/dataflow/test.swift +++ b/swift/ql/test/library-tests/dataflow/dataflow/test.swift @@ -692,4 +692,51 @@ func testArray() { var arr6 = [1,2,3] arr6.insert(source(), at: 2) sink(arg: arr6[0]) // $ flow=693 + + var arr7 = [source()] + sink(arg: arr7.randomElement()!) // $ flow=696 +} + +func testSetCollections() { + var set1: Set = [1,2,3] + sink(arg: set1.randomElement()!) + set1.insert(source()) + sink(arg: set1.randomElement()!) // $flow=703 + + let set2 = Set([source()]) + sink(arg: set2.randomElement()!) // $ flow=706 +} + +struct MyOptionals { + var v1 : Int? = 0 + var v2 : Int? = 0 + var v3 : Int! = 0 +} + +func testWriteOptional() { + var v1 : Int? = 0 + var v2 : Int? = 0 + var v3 : Int! = 0 + var mo1 = MyOptionals() + var mo2 : MyOptionals! = MyOptionals() + + v1! = source() + v2 = source() + v3 = source() + mo1.v1! = source() + mo1.v2 = source() + mo1.v3 = source() + mo2!.v1! = source() + mo2!.v2 = source() + mo2!.v3 = source() + + sink(arg: v1!) // $ flow=723 + sink(arg: v2!) // $ flow=724 + sink(arg: v3) // $ flow=725 + sink(arg: mo1.v1!) // $ MISSING:flow=726 + sink(arg: mo1.v2!) // $ flow=727 + sink(arg: mo1.v3) // $ flow=728 + sink(arg: mo2!.v1!) // $ MISSING:flow=729 + sink(arg: mo2!.v2!) // $ MISSING:flow=730 + sink(arg: mo2!.v3) // $ MISSING:flow=731 }