diff --git a/Cargo.lock b/Cargo.lock index d84c0336ea5..a74037b1320 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -202,9 +202,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.3.3" +version = "2.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" +checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" [[package]] name = "miniz_oxide" @@ -226,9 +226,9 @@ dependencies = [ [[package]] name = "num-integer" -version = "0.1.43" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d59457e662d541ba17869cf51cf177c0b5f0cbf476c66bdc90bf1edac4f875b" +checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" dependencies = [ "autocfg", "num-traits", @@ -236,9 +236,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.12" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac267bcc07f48ee5f8935ab0d24f316fb722d7a1292e2913f0cc196b29ffd611" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" dependencies = [ "autocfg", ] @@ -269,9 +269,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8963b85b8ce3074fecffde43b4b0dded83ce2f367dc8d363afc56679f3ee820b" +checksum = "38cf2c13ed4745de91a5eb834e11c00bcc3709e773173b2ce4c56c9fbde04b9c" dependencies = [ "aho-corasick", "memchr", @@ -291,9 +291,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.20" +version = "0.6.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cab7a364d15cde1e505267766a2d3c4e22a843e1a601f0fa7564c0f82ced11c" +checksum = "3b181ba2dcf07aaccad5448e8ead58db5b742cf85dfe035e2227f137a539a189" [[package]] name = "ruby-extractor" @@ -502,9 +502,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2810660b9d5b18895d140caba6401765749a6a162e5d0736cfc44ea50db9d79d" +checksum = "a1fa8f0c8f4c594e4fc9debc1990deab13238077271ba84dd853d54902ee3401" dependencies = [ "ansi_term 0.12.1", "chrono", @@ -524,9 +524,9 @@ dependencies = [ [[package]] name = "tree-sitter" -version = "0.17.0" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ee7370fec3aecde3862a7d64c571048f70a7298daef1815e8fc68b9de54b5c" +checksum = "d18dcb776d3affaba6db04d11d645946d34a69b3172e588af96ce9fecd20faac" dependencies = [ "cc", "regex", @@ -534,9 +534,9 @@ dependencies = [ [[package]] name = "tree-sitter-ruby" -version = "0.16.2" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50dfb5eb364529287bc6007e1d0f2908b831b1102db3082398ae03da0f35d617" +checksum = "428e841bb6dc0fafd1f4f5732b11e89c3449fce73a207545034d1541aa9163ce" dependencies = [ "cc", "tree-sitter", diff --git a/extractor/Cargo.toml b/extractor/Cargo.toml index 6aabc942542..1d970c95544 100644 --- a/extractor/Cargo.toml +++ b/extractor/Cargo.toml @@ -9,8 +9,8 @@ edition = "2018" [dependencies] flate2 = "1.0" node-types = { path = "../node-types" } -tree-sitter = "0.17.0" -tree-sitter-ruby = "0.16" +tree-sitter = "0.17" +tree-sitter-ruby = "0.17" clap = "2.33" tracing = "0.1" tracing-subscriber = { version = "0.2", features = ["env-filter"] } diff --git a/extractor/src/extractor.rs b/extractor/src/extractor.rs index 597de04b986..12d4509d659 100644 --- a/extractor/src/extractor.rs +++ b/extractor/src/extractor.rs @@ -303,16 +303,12 @@ impl Visitor<'_> { return false; } - if node.is_extra() { - return false; - } - self.stack.push(Vec::new()); return true; } fn leave_node(&mut self, field_name: Option<&'static str>, node: Node) { - if node.is_extra() || node.is_error() || node.is_missing() { + if node.is_error() || node.is_missing() { return; } let child_nodes = self.stack.pop().expect("Vistor: empty stack"); @@ -347,16 +343,20 @@ impl Visitor<'_> { all_args.push(Arg::Label(loc)); self.trap_writer.add_tuple(&table_name, all_args); } - if let Some(parent) = self.stack.last_mut() { - parent.push(( - field_name, - id, - TypeName { - kind: node.kind().to_owned(), - named: node.is_named(), - }, - )) - }; + if !node.is_extra() { + // Extra nodes are independent root nodes and do not belong to the parent node + // Therefore we should not register them in the parent vector + if let Some(parent) = self.stack.last_mut() { + parent.push(( + field_name, + id, + TypeName { + kind: node.kind().to_owned(), + named: node.is_named(), + }, + )) + }; + } } else { error!( "{}:{}: unknown table type: '{}'", @@ -366,6 +366,7 @@ impl Visitor<'_> { ); } } + fn complex_node( &mut self, node: &Node, diff --git a/generator/Cargo.toml b/generator/Cargo.toml index 0615ffeab3d..8f95dbe95dc 100644 --- a/generator/Cargo.toml +++ b/generator/Cargo.toml @@ -10,4 +10,4 @@ edition = "2018" node-types = { path = "../node-types" } tracing = "0.1" tracing-subscriber = { version = "0.2", features = ["env-filter"] } -tree-sitter-ruby = "0.16" +tree-sitter-ruby = "0.17" diff --git a/ql/src/codeql_ruby/ast.qll b/ql/src/codeql_ruby/ast.qll index 69c5e5dd4b4..69e7e566c20 100644 --- a/ql/src/codeql_ruby/ast.qll +++ b/ql/src/codeql_ruby/ast.qll @@ -513,6 +513,18 @@ class HashSplatParameter extends @hash_splat_parameter, Top, BlockParametersChil override Top getAFieldOrChild() { hash_splat_parameter_name(this, result) } } +class HeredocBodyChildType extends @heredoc_body_child_type, Top { } + +class HeredocBody extends @heredoc_body, Top { + override string toString() { result = "HeredocBody" } + + override Location getLocation() { heredoc_body_def(this, result) } + + HeredocBodyChildType getChild(int i) { heredoc_body_child(this, i, result) } + + override Top getAFieldOrChild() { heredoc_body_child(this, _, result) } +} + class IfAlternativeType extends @if_alternative_type, Top { } class If extends @if, Top, UnderscorePrimary { @@ -558,7 +570,7 @@ class In extends @in, Top { } class Interpolation extends @interpolation, Top, BareStringChildType, BareSymbolChildType, - RegexChildType, StringChildType, SubshellChildType, SymbolChildType { + HeredocBodyChildType, RegexChildType, StringChildType, SubshellChildType, SymbolChildType { override string toString() { result = "Interpolation" } override Location getLocation() { interpolation_def(this, _, result) } @@ -1795,6 +1807,14 @@ class ClassVariable extends @class_variable, Top, UnderscoreMethodName, Undersco string getText() { class_variable_def(this, result, _) } } +class Comment extends @comment, Top { + override string toString() { result = "Comment" } + + override Location getLocation() { comment_def(this, _, result) } + + string getText() { comment_def(this, result, _) } +} + class Complex extends @complex, Top, UnderscorePrimary { override string toString() { result = "Complex" } @@ -1869,7 +1889,7 @@ class EnsureUnnamed extends @ensure_unnamed, Top { } class EscapeSequence extends @escape_sequence, Top, BareStringChildType, BareSymbolChildType, - RegexChildType, StringChildType, SubshellChildType, SymbolChildType { + HeredocBodyChildType, RegexChildType, StringChildType, SubshellChildType, SymbolChildType { override string toString() { result = "EscapeSequence" } override Location getLocation() { escape_sequence_def(this, _, result) } @@ -1917,7 +1937,7 @@ class HeredocBeginning extends @heredoc_beginning, Top, UnderscorePrimary { string getText() { heredoc_beginning_def(this, result, _) } } -class HeredocEnd extends @heredoc_end, Top { +class HeredocEnd extends @heredoc_end, Top, HeredocBodyChildType { override string toString() { result = "HeredocEnd" } override Location getLocation() { heredoc_end_def(this, _, result) } diff --git a/ql/src/ruby.dbscheme b/ql/src/ruby.dbscheme index 3456e12d155..64fde42a85b 100644 --- a/ql/src/ruby.dbscheme +++ b/ql/src/ruby.dbscheme @@ -499,6 +499,20 @@ hash_splat_parameter_def( int loc: @location ref ); +@heredoc_body_child_type = @escape_sequence | @heredoc_end | @interpolation + +#keyset[heredoc_body, index] +heredoc_body_child( + int heredoc_body: @heredoc_body ref, + int index: int ref, + unique int heredoc_body_child_type: @heredoc_body_child_type ref +); + +heredoc_body_def( + unique int id: @heredoc_body, + int loc: @location ref +); + @if_alternative_type = @else | @elsif if_alternative( @@ -1542,6 +1556,12 @@ class_variable_def( int loc: @location ref ); +comment_def( + unique int id: @comment, + string text: string ref, + int loc: @location ref +); + complex_def( unique int id: @complex, string text: string ref, @@ -1836,5 +1856,5 @@ tilde_unnamed_def( int loc: @location ref ); -@top = @alias | @argument_list | @array | @assignment | @bare_string | @bare_symbol | @begin | @begin_block | @binary | @block | @block_argument | @block_parameter | @block_parameters | @break | @call | @case__ | @chained_string | @class | @conditional | @destructured_left_assignment | @destructured_parameter | @do | @do_block | @element_reference | @else | @elsif | @empty_statement | @end_block | @ensure | @exception_variable | @exceptions | @for | @hash | @hash_splat_argument | @hash_splat_parameter | @if | @if_modifier | @in | @interpolation | @keyword_parameter | @lambda | @lambda_parameters | @left_assignment_list | @method | @method_call | @method_parameters | @module | @next | @operator | @operator_assignment | @optional_parameter | @pair | @parenthesized_statements | @pattern | @program | @range | @rational | @redo | @regex | @rescue | @rescue_modifier | @rest_assignment | @retry | @return | @right_assignment_list | @scope_resolution | @setter | @singleton_class | @singleton_method | @splat_argument | @splat_parameter | @string__ | @string_array | @subshell | @superclass | @symbol | @symbol_array | @then | @unary | @undef | @unless | @unless_modifier | @until | @until_modifier | @when | @while | @while_modifier | @yield | @bang_unnamed | @bangequal_unnamed | @bangtilde_unnamed | @dquote_unnamed | @hashlbrace_unnamed | @percent_unnamed | @percentequal_unnamed | @percentilparen_unnamed | @percentwlparen_unnamed | @ampersand_unnamed | @ampersandampersand_unnamed | @ampersandampersandequal_unnamed | @ampersanddot_unnamed | @ampersandequal_unnamed | @lparen_unnamed | @rparen_unnamed | @star_unnamed | @starstar_unnamed | @starstarequal_unnamed | @starequal_unnamed | @plus_unnamed | @plusequal_unnamed | @plusat_unnamed | @comma_unnamed | @minus_unnamed | @minusequal_unnamed | @minusrangle_unnamed | @minusat_unnamed | @dot_unnamed | @dotdot_unnamed | @dotdotdot_unnamed | @slash_unnamed | @slashequal_unnamed | @colon_unnamed | @colondquote_unnamed | @coloncolon_unnamed | @semicolon_unnamed | @langle_unnamed | @langlelangle_unnamed | @langlelangleequal_unnamed | @langleequal_unnamed | @langleequalrangle_unnamed | @equal_unnamed | @equalequal_unnamed | @equalequalequal_unnamed | @equalrangle_unnamed | @equaltilde_unnamed | @rangle_unnamed | @rangleequal_unnamed | @ranglerangle_unnamed | @ranglerangleequal_unnamed | @question_unnamed | @b_e_g_i_n__unnamed | @e_n_d__unnamed | @lbracket_unnamed | @lbracketrbracket_unnamed | @lbracketrbracketequal_unnamed | @rbracket_unnamed | @caret_unnamed | @caretequal_unnamed | @underscore__e_n_d____unnamed | @backtick_unnamed | @alias_unnamed | @and_unnamed | @begin_unnamed | @break_unnamed | @case_unnamed | @character | @class_unnamed | @class_variable | @complex | @constant | @def_unnamed | @definedquestion_unnamed | @do_unnamed | @else_unnamed | @elsif_unnamed | @end_unnamed | @ensure_unnamed | @escape_sequence | @false | @float__ | @for_unnamed | @global_variable | @heredoc_beginning | @heredoc_end | @identifier | @if_unnamed | @in_unnamed | @instance_variable | @integer | @module_unnamed | @next_unnamed | @nil | @not_unnamed | @or_unnamed | @r_unnamed | @redo_unnamed | @rescue_unnamed | @retry_unnamed | @return_unnamed | @self | @super | @then_unnamed | @true | @undef_unnamed | @uninterpreted | @unless_unnamed | @until_unnamed | @when_unnamed | @while_unnamed | @yield_unnamed | @lbrace_unnamed | @pipe_unnamed | @pipeequal_unnamed | @pipepipe_unnamed | @pipepipeequal_unnamed | @rbrace_unnamed | @tilde_unnamed +@top = @alias | @argument_list | @array | @assignment | @bare_string | @bare_symbol | @begin | @begin_block | @binary | @block | @block_argument | @block_parameter | @block_parameters | @break | @call | @case__ | @chained_string | @class | @conditional | @destructured_left_assignment | @destructured_parameter | @do | @do_block | @element_reference | @else | @elsif | @empty_statement | @end_block | @ensure | @exception_variable | @exceptions | @for | @hash | @hash_splat_argument | @hash_splat_parameter | @heredoc_body | @if | @if_modifier | @in | @interpolation | @keyword_parameter | @lambda | @lambda_parameters | @left_assignment_list | @method | @method_call | @method_parameters | @module | @next | @operator | @operator_assignment | @optional_parameter | @pair | @parenthesized_statements | @pattern | @program | @range | @rational | @redo | @regex | @rescue | @rescue_modifier | @rest_assignment | @retry | @return | @right_assignment_list | @scope_resolution | @setter | @singleton_class | @singleton_method | @splat_argument | @splat_parameter | @string__ | @string_array | @subshell | @superclass | @symbol | @symbol_array | @then | @unary | @undef | @unless | @unless_modifier | @until | @until_modifier | @when | @while | @while_modifier | @yield | @bang_unnamed | @bangequal_unnamed | @bangtilde_unnamed | @dquote_unnamed | @hashlbrace_unnamed | @percent_unnamed | @percentequal_unnamed | @percentilparen_unnamed | @percentwlparen_unnamed | @ampersand_unnamed | @ampersandampersand_unnamed | @ampersandampersandequal_unnamed | @ampersanddot_unnamed | @ampersandequal_unnamed | @lparen_unnamed | @rparen_unnamed | @star_unnamed | @starstar_unnamed | @starstarequal_unnamed | @starequal_unnamed | @plus_unnamed | @plusequal_unnamed | @plusat_unnamed | @comma_unnamed | @minus_unnamed | @minusequal_unnamed | @minusrangle_unnamed | @minusat_unnamed | @dot_unnamed | @dotdot_unnamed | @dotdotdot_unnamed | @slash_unnamed | @slashequal_unnamed | @colon_unnamed | @colondquote_unnamed | @coloncolon_unnamed | @semicolon_unnamed | @langle_unnamed | @langlelangle_unnamed | @langlelangleequal_unnamed | @langleequal_unnamed | @langleequalrangle_unnamed | @equal_unnamed | @equalequal_unnamed | @equalequalequal_unnamed | @equalrangle_unnamed | @equaltilde_unnamed | @rangle_unnamed | @rangleequal_unnamed | @ranglerangle_unnamed | @ranglerangleequal_unnamed | @question_unnamed | @b_e_g_i_n__unnamed | @e_n_d__unnamed | @lbracket_unnamed | @lbracketrbracket_unnamed | @lbracketrbracketequal_unnamed | @rbracket_unnamed | @caret_unnamed | @caretequal_unnamed | @underscore__e_n_d____unnamed | @backtick_unnamed | @alias_unnamed | @and_unnamed | @begin_unnamed | @break_unnamed | @case_unnamed | @character | @class_unnamed | @class_variable | @comment | @complex | @constant | @def_unnamed | @definedquestion_unnamed | @do_unnamed | @else_unnamed | @elsif_unnamed | @end_unnamed | @ensure_unnamed | @escape_sequence | @false | @float__ | @for_unnamed | @global_variable | @heredoc_beginning | @heredoc_end | @identifier | @if_unnamed | @in_unnamed | @instance_variable | @integer | @module_unnamed | @next_unnamed | @nil | @not_unnamed | @or_unnamed | @r_unnamed | @redo_unnamed | @rescue_unnamed | @retry_unnamed | @return_unnamed | @self | @super | @then_unnamed | @true | @undef_unnamed | @uninterpreted | @unless_unnamed | @until_unnamed | @when_unnamed | @while_unnamed | @yield_unnamed | @lbrace_unnamed | @pipe_unnamed | @pipeequal_unnamed | @pipepipe_unnamed | @pipepipeequal_unnamed | @rbrace_unnamed | @tilde_unnamed