hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-02-19 08:23:45 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #20 from github/aibaars/extract-extra

Browse Source 
Extract 'extra' nodes and their subtrees
This commit is contained in:
Arthur Baars
2020-11-03 13:45:33 +01:00
committed by GitHub
parent d7e9178cda bfc05539ec
commit 65c1f2c359
6 changed files with 79 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
Cargo.lock generated
View File

@@ -202,9 +202,9 @@ dependencies = [
[[package]]
name = "memchr"
version = "2.3.3"
version = "2.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400"
checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
[[package]]
name = "miniz_oxide"
@@ -226,9 +226,9 @@ dependencies = [
[[package]]
name = "num-integer"
version = "0.1.43"
version = "0.1.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d59457e662d541ba17869cf51cf177c0b5f0cbf476c66bdc90bf1edac4f875b"
checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db"
dependencies = [
"autocfg",
"num-traits",
@@ -236,9 +236,9 @@ dependencies = [
[[package]]
name = "num-traits"
version = "0.2.12"
version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac267bcc07f48ee5f8935ab0d24f316fb722d7a1292e2913f0cc196b29ffd611"
checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290"
dependencies = [
"autocfg",
]
@@ -269,9 +269,9 @@ dependencies = [
[[package]]
name = "regex"
version = "1.4.1"
version = "1.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8963b85b8ce3074fecffde43b4b0dded83ce2f367dc8d363afc56679f3ee820b"
checksum = "38cf2c13ed4745de91a5eb834e11c00bcc3709e773173b2ce4c56c9fbde04b9c"
dependencies = [
"aho-corasick",
"memchr",
@@ -291,9 +291,9 @@ dependencies = [
[[package]]
name = "regex-syntax"
version = "0.6.20"
version = "0.6.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8cab7a364d15cde1e505267766a2d3c4e22a843e1a601f0fa7564c0f82ced11c"
checksum = "3b181ba2dcf07aaccad5448e8ead58db5b742cf85dfe035e2227f137a539a189"
[[package]]
name = "ruby-extractor"
@@ -502,9 +502,9 @@ dependencies = [
[[package]]
name = "tracing-subscriber"
version = "0.2.14"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2810660b9d5b18895d140caba6401765749a6a162e5d0736cfc44ea50db9d79d"
checksum = "a1fa8f0c8f4c594e4fc9debc1990deab13238077271ba84dd853d54902ee3401"
dependencies = [
"ansi_term 0.12.1",
"chrono",
@@ -524,9 +524,9 @@ dependencies = [
[[package]]
name = "tree-sitter"
version = "0.17.0"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70ee7370fec3aecde3862a7d64c571048f70a7298daef1815e8fc68b9de54b5c"
checksum = "d18dcb776d3affaba6db04d11d645946d34a69b3172e588af96ce9fecd20faac"
dependencies = [
"cc",
"regex",
@@ -534,9 +534,9 @@ dependencies = [
[[package]]
name = "tree-sitter-ruby"
version = "0.16.2"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50dfb5eb364529287bc6007e1d0f2908b831b1102db3082398ae03da0f35d617"
checksum = "428e841bb6dc0fafd1f4f5732b11e89c3449fce73a207545034d1541aa9163ce"
dependencies = [
"cc",
"tree-sitter",

4
extractor/Cargo.toml
View File

@@ -9,8 +9,8 @@ edition = "2018"
[dependencies]
flate2 = "1.0"
node-types = { path = "../node-types" }
tree-sitter = "0.17.0"
tree-sitter-ruby = "0.16"
tree-sitter = "0.17"
tree-sitter-ruby = "0.17"
clap = "2.33"
tracing = "0.1"
tracing-subscriber = { version = "0.2", features = ["env-filter"] }

31
extractor/src/extractor.rs
View File

@@ -303,16 +303,12 @@ impl Visitor<'_> {
return false;
}
if node.is_extra() {
return false;
}
self.stack.push(Vec::new());
return true;
}
fn leave_node(&mut self, field_name: Option<&'static str>, node: Node) {
if node.is_extra() || node.is_error() || node.is_missing() {
if node.is_error() || node.is_missing() {
return;
}
let child_nodes = self.stack.pop().expect("Vistor: empty stack");
@@ -347,16 +343,20 @@ impl Visitor<'_> {
all_args.push(Arg::Label(loc));
self.trap_writer.add_tuple(&table_name, all_args);
}
if let Some(parent) = self.stack.last_mut() {
parent.push((
field_name,
id,
TypeName {
kind: node.kind().to_owned(),
named: node.is_named(),
},
))
};
if !node.is_extra() {
// Extra nodes are independent root nodes and do not belong to the parent node
// Therefore we should not register them in the parent vector
if let Some(parent) = self.stack.last_mut() {
parent.push((
field_name,
id,
TypeName {
kind: node.kind().to_owned(),
named: node.is_named(),
},
))
};
}
} else {
error!(
"{}:{}: unknown table type: '{}'",
@@ -366,6 +366,7 @@ impl Visitor<'_> {
);
}
}
fn complex_node(
&mut self,
node: &Node,

2
generator/Cargo.toml
View File

@@ -10,4 +10,4 @@ edition = "2018"
node-types = { path = "../node-types" }
tracing = "0.1"
tracing-subscriber = { version = "0.2", features = ["env-filter"] }
tree-sitter-ruby = "0.16"
tree-sitter-ruby = "0.17"

26
ql/src/codeql_ruby/ast.qll
View File

@@ -513,6 +513,18 @@ class HashSplatParameter extends @hash_splat_parameter, Top, BlockParametersChil
override Top getAFieldOrChild() { hash_splat_parameter_name(this, result) }
}
class HeredocBodyChildType extends @heredoc_body_child_type, Top { }
class HeredocBody extends @heredoc_body, Top {
override string toString() { result = "HeredocBody" }
override Location getLocation() { heredoc_body_def(this, result) }
HeredocBodyChildType getChild(int i) { heredoc_body_child(this, i, result) }
override Top getAFieldOrChild() { heredoc_body_child(this, _, result) }
}
class IfAlternativeType extends @if_alternative_type, Top { }
class If extends @if, Top, UnderscorePrimary {
@@ -558,7 +570,7 @@ class In extends @in, Top {
}
class Interpolation extends @interpolation, Top, BareStringChildType, BareSymbolChildType,
RegexChildType, StringChildType, SubshellChildType, SymbolChildType {
HeredocBodyChildType, RegexChildType, StringChildType, SubshellChildType, SymbolChildType {
override string toString() { result = "Interpolation" }
override Location getLocation() { interpolation_def(this, _, result) }
@@ -1795,6 +1807,14 @@ class ClassVariable extends @class_variable, Top, UnderscoreMethodName, Undersco
string getText() { class_variable_def(this, result, _) }
}
class Comment extends @comment, Top {
override string toString() { result = "Comment" }
override Location getLocation() { comment_def(this, _, result) }
string getText() { comment_def(this, result, _) }
}
class Complex extends @complex, Top, UnderscorePrimary {
override string toString() { result = "Complex" }
@@ -1869,7 +1889,7 @@ class EnsureUnnamed extends @ensure_unnamed, Top {
}
class EscapeSequence extends @escape_sequence, Top, BareStringChildType, BareSymbolChildType,
RegexChildType, StringChildType, SubshellChildType, SymbolChildType {
HeredocBodyChildType, RegexChildType, StringChildType, SubshellChildType, SymbolChildType {
override string toString() { result = "EscapeSequence" }
override Location getLocation() { escape_sequence_def(this, _, result) }
@@ -1917,7 +1937,7 @@ class HeredocBeginning extends @heredoc_beginning, Top, UnderscorePrimary {
string getText() { heredoc_beginning_def(this, result, _) }
}
class HeredocEnd extends @heredoc_end, Top {
class HeredocEnd extends @heredoc_end, Top, HeredocBodyChildType {
override string toString() { result = "HeredocEnd" }
override Location getLocation() { heredoc_end_def(this, _, result) }

22
ql/src/ruby.dbscheme
View File

@@ -499,6 +499,20 @@ hash_splat_parameter_def(
int loc: @location ref
);
@heredoc_body_child_type = @escape_sequence | @heredoc_end | @interpolation
#keyset[heredoc_body, index]
heredoc_body_child(
int heredoc_body: @heredoc_body ref,
int index: int ref,
unique int heredoc_body_child_type: @heredoc_body_child_type ref
);
heredoc_body_def(
unique int id: @heredoc_body,
int loc: @location ref
);
@if_alternative_type = @else | @elsif
if_alternative(
@@ -1542,6 +1556,12 @@ class_variable_def(
int loc: @location ref
);
comment_def(
unique int id: @comment,
string text: string ref,
int loc: @location ref
);
complex_def(
unique int id: @complex,
string text: string ref,
@@ -1836,5 +1856,5 @@ tilde_unnamed_def(
int loc: @location ref
);
@top = @alias | @argument_list | @array | @assignment | @bare_string | @bare_symbol | @begin | @begin_block | @binary | @block | @block_argument | @block_parameter | @block_parameters | @break | @call | @case__ | @chained_string | @class | @conditional | @destructured_left_assignment | @destructured_parameter | @do | @do_block | @element_reference | @else | @elsif | @empty_statement | @end_block | @ensure | @exception_variable | @exceptions | @for | @hash | @hash_splat_argument | @hash_splat_parameter | @if | @if_modifier | @in | @interpolation | @keyword_parameter | @lambda | @lambda_parameters | @left_assignment_list | @method | @method_call | @method_parameters | @module | @next | @operator | @operator_assignment | @optional_parameter | @pair | @parenthesized_statements | @pattern | @program | @range | @rational | @redo | @regex | @rescue | @rescue_modifier | @rest_assignment | @retry | @return | @right_assignment_list | @scope_resolution | @setter | @singleton_class | @singleton_method | @splat_argument | @splat_parameter | @string__ | @string_array | @subshell | @superclass | @symbol | @symbol_array | @then | @unary | @undef | @unless | @unless_modifier | @until | @until_modifier | @when | @while | @while_modifier | @yield | @bang_unnamed | @bangequal_unnamed | @bangtilde_unnamed | @dquote_unnamed | @hashlbrace_unnamed | @percent_unnamed | @percentequal_unnamed | @percentilparen_unnamed | @percentwlparen_unnamed | @ampersand_unnamed | @ampersandampersand_unnamed | @ampersandampersandequal_unnamed | @ampersanddot_unnamed | @ampersandequal_unnamed | @lparen_unnamed | @rparen_unnamed | @star_unnamed | @starstar_unnamed | @starstarequal_unnamed | @starequal_unnamed | @plus_unnamed | @plusequal_unnamed | @plusat_unnamed | @comma_unnamed | @minus_unnamed | @minusequal_unnamed | @minusrangle_unnamed | @minusat_unnamed | @dot_unnamed | @dotdot_unnamed | @dotdotdot_unnamed | @slash_unnamed | @slashequal_unnamed | @colon_unnamed | @colondquote_unnamed | @coloncolon_unnamed | @semicolon_unnamed | @langle_unnamed | @langlelangle_unnamed | @langlelangleequal_unnamed | @langleequal_unnamed | @langleequalrangle_unnamed | @equal_unnamed | @equalequal_unnamed | @equalequalequal_unnamed | @equalrangle_unnamed | @equaltilde_unnamed | @rangle_unnamed | @rangleequal_unnamed | @ranglerangle_unnamed | @ranglerangleequal_unnamed | @question_unnamed | @b_e_g_i_n__unnamed | @e_n_d__unnamed | @lbracket_unnamed | @lbracketrbracket_unnamed | @lbracketrbracketequal_unnamed | @rbracket_unnamed | @caret_unnamed | @caretequal_unnamed | @underscore__e_n_d____unnamed | @backtick_unnamed | @alias_unnamed | @and_unnamed | @begin_unnamed | @break_unnamed | @case_unnamed | @character | @class_unnamed | @class_variable | @complex | @constant | @def_unnamed | @definedquestion_unnamed | @do_unnamed | @else_unnamed | @elsif_unnamed | @end_unnamed | @ensure_unnamed | @escape_sequence | @false | @float__ | @for_unnamed | @global_variable | @heredoc_beginning | @heredoc_end | @identifier | @if_unnamed | @in_unnamed | @instance_variable | @integer | @module_unnamed | @next_unnamed | @nil | @not_unnamed | @or_unnamed | @r_unnamed | @redo_unnamed | @rescue_unnamed | @retry_unnamed | @return_unnamed | @self | @super | @then_unnamed | @true | @undef_unnamed | @uninterpreted | @unless_unnamed | @until_unnamed | @when_unnamed | @while_unnamed | @yield_unnamed | @lbrace_unnamed | @pipe_unnamed | @pipeequal_unnamed | @pipepipe_unnamed | @pipepipeequal_unnamed | @rbrace_unnamed | @tilde_unnamed
@top = @alias | @argument_list | @array | @assignment | @bare_string | @bare_symbol | @begin | @begin_block | @binary | @block | @block_argument | @block_parameter | @block_parameters | @break | @call | @case__ | @chained_string | @class | @conditional | @destructured_left_assignment | @destructured_parameter | @do | @do_block | @element_reference | @else | @elsif | @empty_statement | @end_block | @ensure | @exception_variable | @exceptions | @for | @hash | @hash_splat_argument | @hash_splat_parameter | @heredoc_body | @if | @if_modifier | @in | @interpolation | @keyword_parameter | @lambda | @lambda_parameters | @left_assignment_list | @method | @method_call | @method_parameters | @module | @next | @operator | @operator_assignment | @optional_parameter | @pair | @parenthesized_statements | @pattern | @program | @range | @rational | @redo | @regex | @rescue | @rescue_modifier | @rest_assignment | @retry | @return | @right_assignment_list | @scope_resolution | @setter | @singleton_class | @singleton_method | @splat_argument | @splat_parameter | @string__ | @string_array | @subshell | @superclass | @symbol | @symbol_array | @then | @unary | @undef | @unless | @unless_modifier | @until | @until_modifier | @when | @while | @while_modifier | @yield | @bang_unnamed | @bangequal_unnamed | @bangtilde_unnamed | @dquote_unnamed | @hashlbrace_unnamed | @percent_unnamed | @percentequal_unnamed | @percentilparen_unnamed | @percentwlparen_unnamed | @ampersand_unnamed | @ampersandampersand_unnamed | @ampersandampersandequal_unnamed | @ampersanddot_unnamed | @ampersandequal_unnamed | @lparen_unnamed | @rparen_unnamed | @star_unnamed | @starstar_unnamed | @starstarequal_unnamed | @starequal_unnamed | @plus_unnamed | @plusequal_unnamed | @plusat_unnamed | @comma_unnamed | @minus_unnamed | @minusequal_unnamed | @minusrangle_unnamed | @minusat_unnamed | @dot_unnamed | @dotdot_unnamed | @dotdotdot_unnamed | @slash_unnamed | @slashequal_unnamed | @colon_unnamed | @colondquote_unnamed | @coloncolon_unnamed | @semicolon_unnamed | @langle_unnamed | @langlelangle_unnamed | @langlelangleequal_unnamed | @langleequal_unnamed | @langleequalrangle_unnamed | @equal_unnamed | @equalequal_unnamed | @equalequalequal_unnamed | @equalrangle_unnamed | @equaltilde_unnamed | @rangle_unnamed | @rangleequal_unnamed | @ranglerangle_unnamed | @ranglerangleequal_unnamed | @question_unnamed | @b_e_g_i_n__unnamed | @e_n_d__unnamed | @lbracket_unnamed | @lbracketrbracket_unnamed | @lbracketrbracketequal_unnamed | @rbracket_unnamed | @caret_unnamed | @caretequal_unnamed | @underscore__e_n_d____unnamed | @backtick_unnamed | @alias_unnamed | @and_unnamed | @begin_unnamed | @break_unnamed | @case_unnamed | @character | @class_unnamed | @class_variable | @comment | @complex | @constant | @def_unnamed | @definedquestion_unnamed | @do_unnamed | @else_unnamed | @elsif_unnamed | @end_unnamed | @ensure_unnamed | @escape_sequence | @false | @float__ | @for_unnamed | @global_variable | @heredoc_beginning | @heredoc_end | @identifier | @if_unnamed | @in_unnamed | @instance_variable | @integer | @module_unnamed | @next_unnamed | @nil | @not_unnamed | @or_unnamed | @r_unnamed | @redo_unnamed | @rescue_unnamed | @retry_unnamed | @return_unnamed | @self | @super | @then_unnamed | @true | @undef_unnamed | @uninterpreted | @unless_unnamed | @until_unnamed | @when_unnamed | @while_unnamed | @yield_unnamed | @lbrace_unnamed | @pipe_unnamed | @pipeequal_unnamed | @pipepipe_unnamed | @pipepipeequal_unnamed | @rbrace_unnamed | @tilde_unnamed