From 3f5eab04b5627586fb637ac71854f2006896ad24 Mon Sep 17 00:00:00 2001 From: Nick Rolfe Date: Tue, 15 Dec 2020 14:58:13 +0000 Subject: [PATCH 1/5] Create disjoint db types for different operators --- extractor/src/extractor.rs | 102 ++++++--- generator/src/main.rs | 56 +++-- generator/src/ql.rs | 16 ++ generator/src/ql_gen.rs | 170 ++++++++++----- node-types/src/lib.rs | 61 ++++-- .../codeql_ruby/ast/internal/TreeSitter.qll | 109 +++++++++- ql/src/ruby.dbscheme | 202 +++++++++++------- 7 files changed, 523 insertions(+), 193 deletions(-) diff --git a/extractor/src/extractor.rs b/extractor/src/extractor.rs index 13111c95cdf..ffe69820523 100644 --- a/extractor/src/extractor.rs +++ b/extractor/src/extractor.rs @@ -2,6 +2,7 @@ use node_types::{EntryKind, Field, NodeTypeMap, Storage, TypeName}; use std::collections::BTreeMap as Map; use std::collections::BTreeSet as Set; use std::fmt; +use std::ops::Range; use std::path::Path; use tracing::{error, info, span, Level}; use tree_sitter::{Language, Node, Parser, Tree}; @@ -246,6 +247,13 @@ fn full_id_for_folder(path: &Path) -> String { format!("{};folder", normalize_path(path)) } +struct ChildNode { + field_name: Option<&'static str>, + label: Label, + type_name: TypeName, + src_range: Range, +} + struct Visitor<'a> { /// The file path of the source code (as string) path: String, @@ -262,13 +270,13 @@ struct Visitor<'a> { toplevel_child_counter: usize, /// A lookup table from type name to node types schema: &'a NodeTypeMap, - /// A stack for gathering information from child nodes. Whenever a node is entered - /// the parent's [Label], child counter, and an empty list is pushed. All children append their data - /// (field name, label, type) to the the list. When the visitor leaves a node the list - /// containing the child data is popped from the stack and matched against the dbscheme - /// for the node. If the expectations are met the corresponding row definitions are - /// added to the trap_output. - stack: Vec<(Label, usize, Vec<(Option<&'static str>, Label, TypeName)>)>, + /// A stack for gathering information from child nodes. Whenever a node is + /// entered the parent's [Label], child counter, and an empty list is pushed. + /// All children append their data to the the list. When the visitor leaves a + /// node the list containing the child data is popped from the stack and + /// matched against the dbscheme for the node. If the expectations are met + /// the corresponding row definitions are added to the trap_output. + stack: Vec<(Label, usize, Vec)>, } impl Visitor<'_> { @@ -349,7 +357,7 @@ impl Visitor<'_> { fields, name: table_name, } => { - if let Some(args) = self.complex_node(&node, fields, child_nodes, id) { + if let Some(args) = self.complex_node(&node, fields, &child_nodes, id) { let mut all_args = Vec::new(); all_args.push(Arg::Label(id)); all_args.push(Arg::Label(parent_id)); @@ -373,14 +381,15 @@ impl Visitor<'_> { // Extra nodes are independent root nodes and do not belong to the parent node // Therefore we should not register them in the parent vector if let Some(parent) = self.stack.last_mut() { - parent.2.push(( + parent.2.push(ChildNode { field_name, - id, - TypeName { + label: id, + type_name: TypeName { kind: node.kind().to_owned(), named: node.is_named(), }, - )) + src_range: node.byte_range(), + }); }; } } @@ -389,38 +398,65 @@ impl Visitor<'_> { &mut self, node: &Node, fields: &Vec, - child_nodes: Vec<(Option<&str>, Label, TypeName)>, + child_nodes: &Vec, parent_id: Label, ) -> Option> { - let mut map: Map<&Option, (&Field, Vec