hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-04-30 19:26:02 +02:00
Code Issues Packages Projects Releases Wiki Activity

Prefix dbscheme entries with language name

Browse Source
This commit is contained in:
Arthur Baars
2021-07-21 16:16:29 +02:00
parent fe868e4c05
commit 2e10f8f054
11 changed files with 2080 additions and 1833 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Cargo.lock generated
View File

@@ -400,6 +400,7 @@ dependencies = [
"node-types",
"tracing",
"tracing-subscriber",
"tree-sitter-embedded-template",
"tree-sitter-ruby",
]

10
extractor/src/extractor.rs
View File

@@ -152,6 +152,7 @@ impl TrapWriter {
/// Extracts the source file at `path`, which is assumed to be canonicalized.
pub fn extract(
language: Language,
language_prefix: &str,
schema: &NodeTypeMap,
path: &Path,
source: &Vec<u8>,
@@ -183,6 +184,7 @@ pub fn extract(
token_counter: 0,
toplevel_child_counter: 0,
stack: Vec::new(),
language_prefix,
schema,
};
traverse(&tree, &mut visitor);
@@ -293,6 +295,8 @@ struct Visitor<'a> {
token_counter: usize,
/// A counter for top-level child nodes
toplevel_child_counter: usize,
/// Language prefix
language_prefix: &'a str,
/// A lookup table from type name to node types
schema: &'a NodeTypeMap,
/// A stack for gathering information from child nodes. Whenever a node is
@@ -400,7 +404,7 @@ impl Visitor<'_> {
match &table.kind {
EntryKind::Token { kind_id, .. } => {
self.trap_writer.add_tuple(
"ast_node_parent",
&format!("{}_ast_node_parent", self.language_prefix),
vec![
Arg::Label(id),
Arg::Label(parent_id),
@@ -408,7 +412,7 @@ impl Visitor<'_> {
],
);
self.trap_writer.add_tuple(
"tokeninfo",
&format!("{}_tokeninfo", self.language_prefix),
vec![
Arg::Label(id),
Arg::Int(*kind_id),
@@ -426,7 +430,7 @@ impl Visitor<'_> {
} => {
if let Some(args) = self.complex_node(&node, fields, &child_nodes, id) {
self.trap_writer.add_tuple(
"ast_node_parent",
&format!("{}_ast_node_parent", self.language_prefix),
vec![
Arg::Label(id),
Arg::Label(parent_id),

4
extractor/src/main.rs
View File

@@ -124,7 +124,7 @@ fn main() -> std::io::Result<()> {
let language = tree_sitter_ruby::language();
let erb = tree_sitter_embedded_template::language();
let schema = node_types::read_node_types_str(tree_sitter_ruby::NODE_TYPES)?;
let schema = node_types::read_node_types_str("ruby", tree_sitter_ruby::NODE_TYPES)?;
let lines: std::io::Result<Vec<String>> = std::io::BufReader::new(file_list).lines().collect();
let lines = lines?;
lines.par_iter().try_for_each(|line| {
@@ -145,7 +145,7 @@ fn main() -> std::io::Result<()> {
} else {
code_ranges = vec![];
}
let trap = extractor::extract(language, &schema, &path, &source, &code_ranges)?;
let trap = extractor::extract(language, "ruby", &schema, &path, &source, &code_ranges)?;
std::fs::create_dir_all(&src_archive_file.parent().unwrap())?;
std::fs::copy(&path, &src_archive_file)?;
std::fs::create_dir_all(&trap_file.parent().unwrap())?;

1
generator/Cargo.toml
View File

@@ -11,4 +11,5 @@ clap = "2.33"
node-types = { path = "../node-types" }
tracing = "0.1"
tracing-subscriber = { version = "0.2", features = ["env-filter"] }
tree-sitter-embedded-template = { git = "https://github.com/tree-sitter/tree-sitter-embedded-template", rev = "d4aac29c08aa7c596633d00b5ec2dd2d247eafe4" }
tree-sitter-ruby = { git = "https://github.com/tree-sitter/tree-sitter-ruby.git", rev = "32cd5a04adb4accb0c121f037ab59df3c3488228" }

12
generator/src/dbscheme.rs
View File

@@ -117,17 +117,7 @@ impl<'a> fmt::Display for Union<'a> {
}
/// Generates the dbscheme by writing the given dbscheme `entries` to the `file`.
pub fn write<'a>(
language_name: &str,
file: &mut dyn std::io::Write,
entries: &'a [Entry],
) -> std::io::Result<()> {
write!(file, "// CodeQL database schema for {}\n", language_name)?;
write!(
file,
"// Automatically generated from the tree-sitter grammar; do not edit\n\n"
)?;
pub fn write<'a>(file: &mut dyn std::io::Write, entries: &'a [Entry]) -> std::io::Result<()> {
for entry in entries {
match entry {
Entry::Case(case) => write!(file, "{}\n\n", case)?,

214
generator/src/main.rs
View File

@@ -9,8 +9,8 @@ use std::collections::BTreeMap as Map;
use std::collections::BTreeSet as Set;
use std::fs::File;
use std::io::LineWriter;
use std::io::Write;
use std::path::PathBuf;
use tracing::{error, info};
/// Given the name of the parent node, and its field information, returns a pair,
/// the first of which is the field's type. The second is an optional dbscheme
@@ -136,18 +136,10 @@ fn add_field_for_column_storage<'a>(
}
/// Converts the given tree-sitter node types into CodeQL dbscheme entries.
fn convert_nodes<'a>(nodes: &'a node_types::NodeTypeMap) -> Vec<dbscheme::Entry<'a>> {
let mut entries: Vec<dbscheme::Entry> = vec![
create_location_union(),
create_locations_default_table(),
create_sourceline_union(),
create_numlines_table(),
create_files_table(),
create_folders_table(),
create_container_union(),
create_containerparent_table(),
create_source_location_prefix_table(),
];
fn convert_nodes<'a>(
nodes: &'a node_types::NodeTypeMap,
) -> (Vec<dbscheme::Entry<'a>>, Set<&'a str>, Map<&'a str, usize>) {
let mut entries: Vec<dbscheme::Entry> = Vec::new();
let mut ast_node_members: Set<&str> = Set::new();
let token_kinds: Map<&str, usize> = nodes
.iter()
@@ -158,7 +150,6 @@ fn convert_nodes<'a>(nodes: &'a node_types::NodeTypeMap) -> Vec<dbscheme::Entry<
_ => None,
})
.collect();
ast_node_members.insert("token");
for (_, node) in nodes {
match &node.kind {
node_types::EntryKind::Union { members: n_members } => {
@@ -251,48 +242,24 @@ fn convert_nodes<'a>(nodes: &'a node_types::NodeTypeMap) -> Vec<dbscheme::Entry<
}
}
// Add the tokeninfo table
let (token_case, token_table) = create_tokeninfo(token_kinds);
entries.push(dbscheme::Entry::Table(token_table));
entries.push(dbscheme::Entry::Case(token_case));
// Add the diagnostics table
let (diagnostics_case, diagnostics_table) = create_diagnostics();
entries.push(dbscheme::Entry::Table(diagnostics_table));
entries.push(dbscheme::Entry::Case(diagnostics_case));
// Create a union of all database types.
entries.push(dbscheme::Entry::Union(dbscheme::Union {
name: "ast_node",
members: ast_node_members,
}));
// Create the ast_node_parent union.
entries.push(dbscheme::Entry::Union(dbscheme::Union {
name: "ast_node_parent",
members: ["ast_node", "file"].iter().cloned().collect(),
}));
entries.push(dbscheme::Entry::Table(create_ast_node_parent_table()));
entries
(entries, ast_node_members, token_kinds)
}
fn create_ast_node_parent_table<'a>() -> dbscheme::Table<'a> {
fn create_ast_node_parent_table<'a>(name: &'a str, ast_node_name: &'a str) -> dbscheme::Table<'a> {
dbscheme::Table {
name: "ast_node_parent",
name,
columns: vec![
dbscheme::Column {
db_type: dbscheme::DbColumnType::Int,
name: "child",
unique: false,
ql_type: ql::Type::AtType("ast_node"),
ql_type: ql::Type::AtType(ast_node_name),
ql_type_is_ref: true,
},
dbscheme::Column {
db_type: dbscheme::DbColumnType::Int,
name: "parent",
unique: false,
ql_type: ql::Type::AtType("ast_node_parent"),
ql_type: ql::Type::AtType(name),
ql_type_is_ref: true,
},
dbscheme::Column {
@@ -307,18 +274,16 @@ fn create_ast_node_parent_table<'a>() -> dbscheme::Table<'a> {
}
}
fn create_tokeninfo<'a>(
token_kinds: Map<&'a str, usize>,
) -> (dbscheme::Case<'a>, dbscheme::Table<'a>) {
let table = dbscheme::Table {
name: "tokeninfo",
fn create_tokeninfo<'a>(name: &'a str, type_name: &'a str) -> dbscheme::Table<'a> {
dbscheme::Table {
name,
keysets: None,
columns: vec![
dbscheme::Column {
db_type: dbscheme::DbColumnType::Int,
name: "id",
unique: true,
ql_type: ql::Type::AtType("token"),
ql_type: ql::Type::AtType(type_name),
ql_type_is_ref: false,
},
dbscheme::Column {
@@ -357,35 +322,19 @@ fn create_tokeninfo<'a>(
ql_type_is_ref: true,
},
],
};
}
}
fn create_token_case<'a>(name: &'a str, token_kinds: Map<&'a str, usize>) -> dbscheme::Case<'a> {
let branches: Vec<(usize, &str)> = token_kinds
.iter()
.map(|(&name, kind_id)| (*kind_id, name))
.collect();
let case = dbscheme::Case {
name: "token",
dbscheme::Case {
name: name,
column: "kind",
branches: branches,
};
(case, table)
}
fn write_dbscheme(
dbscheme_path: PathBuf,
language: &Language,
entries: &[dbscheme::Entry],
) -> std::io::Result<()> {
info!(
"Writing database schema for {} to '{}'",
&language.name,
match dbscheme_path.to_str() {
None => "<undisplayable>",
Some(p) => p,
}
);
let file = File::create(dbscheme_path)?;
let mut file = LineWriter::new(file);
dbscheme::write(&language.name, &mut file, &entries)
}
}
fn create_location_union<'a>() -> dbscheme::Entry<'a> {
@@ -693,30 +642,107 @@ fn main() -> std::io::Result<()> {
let ql_library_path = matches.value_of("library").expect("missing --library");
let ql_library_path = PathBuf::from(ql_library_path);
let ruby = Language {
name: "Ruby".to_owned(),
node_types: tree_sitter_ruby::NODE_TYPES,
};
match node_types::read_node_types_str(&ruby.node_types) {
Err(e) => {
error!("Failed to read node-types JSON for {}: {}", ruby.name, e);
std::process::exit(1);
}
Ok(nodes) => {
let dbscheme_entries = convert_nodes(&nodes);
let languages = vec![
Language {
name: "Ruby".to_owned(),
node_types: tree_sitter_ruby::NODE_TYPES,
},
Language {
name: "Erb".to_owned(),
node_types: tree_sitter_embedded_template::NODE_TYPES,
},
];
let mut dbscheme_writer = LineWriter::new(File::create(dbscheme_path)?);
write!(
dbscheme_writer,
"// CodeQL database schema for {}\n\
// Automatically generated from the tree-sitter grammar; do not edit\n\n",
languages[0].name
)?;
let (diagnostics_case, diagnostics_table) = create_diagnostics();
dbscheme::write(
&mut dbscheme_writer,
&[
create_location_union(),
create_locations_default_table(),
create_sourceline_union(),
create_numlines_table(),
create_files_table(),
create_folders_table(),
create_container_union(),
create_containerparent_table(),
create_source_location_prefix_table(),
dbscheme::Entry::Table(diagnostics_table),
dbscheme::Entry::Case(diagnostics_case),
],
)?;
if let Err(e) = write_dbscheme(dbscheme_path, &ruby, &dbscheme_entries) {
error!("Failed to write dbscheme: {}", e);
std::process::exit(2);
}
let mut ql_writer = LineWriter::new(File::create(ql_library_path)?);
write!(
ql_writer,
"/*\n\
* CodeQL library for {}
* Automatically generated from the tree-sitter grammar; do not edit\n\
*/\n\n",
languages[0].name
)?;
ql::write(
&mut ql_writer,
&[
ql::TopLevel::Import("codeql.files.FileSystem"),
ql::TopLevel::Import("codeql.Locations"),
],
)?;
let classes = ql_gen::convert_nodes(&nodes);
for language in languages {
let prefix = node_types::to_snake_case(&language.name);
let ast_node_name = format!("{}_ast_node", &prefix);
let ast_node_parent_name = format!("{}_ast_node_parent", &prefix);
let token_name = format!("{}_token", &prefix);
let tokeninfo_name = format!("{}_tokeninfo", &prefix);
let reserved_word_name = format!("{}_reserved_word", &prefix);
let nodes = node_types::read_node_types_str(&prefix, &language.node_types)?;
let (dbscheme_entries, mut ast_node_members, token_kinds) = convert_nodes(&nodes);
ast_node_members.insert(&token_name);
dbscheme::write(&mut dbscheme_writer, &dbscheme_entries)?;
let token_case = create_token_case(&token_name, token_kinds);
dbscheme::write(
&mut dbscheme_writer,
&[
dbscheme::Entry::Table(create_tokeninfo(&tokeninfo_name, &token_name)),
dbscheme::Entry::Case(token_case),
dbscheme::Entry::Union(dbscheme::Union {
name: &ast_node_name,
members: ast_node_members,
}),
dbscheme::Entry::Union(dbscheme::Union {
name: &ast_node_parent_name,
members: [&ast_node_name, "file"].iter().cloned().collect(),
}),
dbscheme::Entry::Table(create_ast_node_parent_table(
&ast_node_parent_name,
&ast_node_name,
)),
],
)?;
if let Err(e) = ql_gen::write(ql_library_path, &ruby, &classes) {
println!("Failed to write QL library: {}", e);
std::process::exit(3);
}
Ok(())
}
let mut body = vec![
ql::TopLevel::Class(ql_gen::create_ast_node_class(
&ast_node_name,
&ast_node_parent_name,
)),
ql::TopLevel::Class(ql_gen::create_token_class(&token_name, &tokeninfo_name)),
ql::TopLevel::Class(ql_gen::create_reserved_word_class(&reserved_word_name)),
];
body.append(&mut ql_gen::convert_nodes(&nodes));
ql::write(
&mut ql_writer,
&[ql::TopLevel::Module(ql::Module {
qldoc: None,
name: &language.name,
body,
})],
)?;
}
Ok(())
}

40
generator/src/ql.rs
View File

@@ -1,9 +1,11 @@
use std::collections::BTreeSet;
use std::fmt;
#[derive(Clone, Eq, PartialEq, Hash)]
pub enum TopLevel<'a> {
Class(Class<'a>),
Import(&'a str),
Module(Module<'a>),
}
impl<'a> fmt::Display for TopLevel<'a> {
@@ -11,6 +13,7 @@ impl<'a> fmt::Display for TopLevel<'a> {
match self {
TopLevel::Import(x) => write!(f, "private import {}", x),
TopLevel::Class(cls) => write!(f, "{}", cls),
TopLevel::Module(m) => write!(f, "{}", m),
}
}
}
@@ -67,6 +70,26 @@ impl<'a> fmt::Display for Class<'a> {
}
}
#[derive(Clone, Eq, PartialEq, Hash)]
pub struct Module<'a> {
pub qldoc: Option<String>,
pub name: &'a str,
pub body: Vec<TopLevel<'a>>,
}
impl<'a> fmt::Display for Module<'a> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
if let Some(qldoc) = &self.qldoc {
write!(f, "/** {} */", qldoc)?;
}
write!(f, "module {} {{ \n", self.name)?;
for decl in &self.body {
write!(f, " {}\n", decl)?;
}
write!(f, "}}")?;
Ok(())
}
}
// The QL type of a column.
#[derive(Clone, Eq, PartialEq, Hash, Ord, PartialOrd)]
pub enum Type<'a> {
@@ -227,24 +250,9 @@ impl<'a> fmt::Display for FormalParameter<'a> {
}
/// Generates a QL library by writing the given `classes` to the `file`.
pub fn write<'a>(
language_name: &str,
file: &mut dyn std::io::Write,
elements: &'a [TopLevel],
) -> std::io::Result<()> {
write!(file, "/*\n")?;
write!(file, " * CodeQL library for {}\n", language_name)?;
write!(
file,
" * Automatically generated from the tree-sitter grammar; do not edit\n"
)?;
write!(file, " */\n\n")?;
write!(file, "module Generated {{\n")?;
pub fn write<'a>(file: &mut dyn std::io::Write, elements: &'a [TopLevel]) -> std::io::Result<()> {
for element in elements {
write!(file, "{}\n\n", &element)?;
}
write!(file, "}}")?;
Ok(())
}

59
generator/src/ql_gen.rs
View File

@@ -1,37 +1,9 @@
use crate::language::Language;
use crate::ql;
use std::collections::BTreeSet;
use std::fs::File;
use std::io::LineWriter;
use std::path::PathBuf;
/// Writes the QL AST library for the given library.
///
/// # Arguments
///
/// `language` - the language for which we're generating a library
/// `classes` - the list of classes to write.
pub fn write(
ql_library_path: PathBuf,
language: &Language,
classes: &[ql::TopLevel]
) -> std::io::Result<()> {
println!(
"Writing QL library for {} to '{}'",
&language.name,
match ql_library_path.to_str() {
None => "<undisplayable>",
Some(p) => p,
}
);
let file = File::create(ql_library_path)?;
let mut file = LineWriter::new(file);
ql::write(&language.name, &mut file, &classes)
}
/// Creates the hard-coded `AstNode` class that acts as a supertype of all
/// classes we generate.
fn create_ast_node_class<'a>() -> ql::Class<'a> {
pub fn create_ast_node_class<'a>(ast_node: &'a str, ast_node_parent: &'a str) -> ql::Class<'a> {
// Default implementation of `toString` calls `this.getAPrimaryQlClass()`
let to_string = ql::Predicate {
qldoc: Some(String::from(
@@ -69,7 +41,7 @@ fn create_ast_node_class<'a>() -> ql::Class<'a> {
return_type: Some(ql::Type::Normal("AstNode")),
formal_parameters: vec![],
body: ql::Expression::Pred(
"ast_node_parent",
ast_node_parent,
vec![
ql::Expression::Var("this"),
ql::Expression::Var("result"),
@@ -86,7 +58,7 @@ fn create_ast_node_class<'a>() -> ql::Class<'a> {
return_type: Some(ql::Type::Int),
formal_parameters: vec![],
body: ql::Expression::Pred(
"ast_node_parent",
ast_node_parent,
vec![
ql::Expression::Var("this"),
ql::Expression::Var("_"),
@@ -111,7 +83,7 @@ fn create_ast_node_class<'a>() -> ql::Class<'a> {
qldoc: Some(String::from("The base class for all AST nodes")),
name: "AstNode",
is_abstract: false,
supertypes: vec![ql::Type::AtType("ast_node")].into_iter().collect(),
supertypes: vec![ql::Type::AtType(ast_node)].into_iter().collect(),
characteristic_predicate: None,
predicates: vec![
to_string,
@@ -124,7 +96,7 @@ fn create_ast_node_class<'a>() -> ql::Class<'a> {
}
}
fn create_token_class<'a>() -> ql::Class<'a> {
pub fn create_token_class<'a>(token_type: &'a str, tokeninfo: &'a str) -> ql::Class<'a> {
let tokeninfo_arity = 6;
let get_value = ql::Predicate {
qldoc: Some(String::from("Gets the value of this token.")),
@@ -132,7 +104,7 @@ fn create_token_class<'a>() -> ql::Class<'a> {
overridden: false,
return_type: Some(ql::Type::String),
formal_parameters: vec![],
body: create_get_field_expr_for_column_storage("result", "tokeninfo", 3, tokeninfo_arity),
body: create_get_field_expr_for_column_storage("result", tokeninfo, 3, tokeninfo_arity),
};
let get_location = ql::Predicate {
qldoc: Some(String::from("Gets the location of this token.")),
@@ -140,7 +112,7 @@ fn create_token_class<'a>() -> ql::Class<'a> {
overridden: true,
return_type: Some(ql::Type::Normal("Location")),
formal_parameters: vec![],
body: create_get_field_expr_for_column_storage("result", "tokeninfo", 4, tokeninfo_arity),
body: create_get_field_expr_for_column_storage("result", tokeninfo, 4, tokeninfo_arity),
};
let to_string = ql::Predicate {
qldoc: Some(String::from(
@@ -159,7 +131,7 @@ fn create_token_class<'a>() -> ql::Class<'a> {
qldoc: Some(String::from("A token.")),
name: "Token",
is_abstract: false,
supertypes: vec![ql::Type::AtType("token"), ql::Type::Normal("AstNode")]
supertypes: vec![ql::Type::AtType(token_type), ql::Type::Normal("AstNode")]
.into_iter()
.collect(),
characteristic_predicate: None,
@@ -173,8 +145,7 @@ fn create_token_class<'a>() -> ql::Class<'a> {
}
// Creates the `ReservedWord` class.
fn create_reserved_word_class<'a>() -> ql::Class<'a> {
let db_name = "reserved_word";
pub fn create_reserved_word_class<'a>(db_name: &'a str) -> ql::Class<'a> {
let class_name = "ReservedWord";
let get_a_primary_ql_class = create_get_a_primary_ql_class(&class_name);
ql::Class {
@@ -457,9 +428,7 @@ fn create_field_getters<'a>(
}
};
let qldoc = match &field.name {
Some(name) => {
format!("Gets the node corresponding to the field `{}`.", name)
}
Some(name) => format!("Gets the node corresponding to the field `{}`.", name),
None => {
if formal_parameters.len() == 0 {
"Gets the child of this node.".to_owned()
@@ -483,13 +452,7 @@ fn create_field_getters<'a>(
/// Converts the given node types into CodeQL classes wrapping the dbscheme.
pub fn convert_nodes<'a>(nodes: &'a node_types::NodeTypeMap) -> Vec<ql::TopLevel<'a>> {
let mut classes: Vec<ql::TopLevel> = vec![
ql::TopLevel::Import("codeql.files.FileSystem"),
ql::TopLevel::Import("codeql.Locations"),
ql::TopLevel::Class(create_ast_node_class()),
ql::TopLevel::Class(create_token_class()),
ql::TopLevel::Class(create_reserved_word_class()),
];
let mut classes: Vec<ql::TopLevel> = Vec::new();
let mut token_kinds = BTreeSet::new();
for (type_name, node) in nodes {
if let node_types::EntryKind::Token { .. } = &node.kind {

60
node-types/src/lib.rs
View File

@@ -81,15 +81,15 @@ pub enum Storage {
},
}
pub fn read_node_types(node_types_path: &Path) -> std::io::Result<NodeTypeMap> {
pub fn read_node_types(prefix: &str, node_types_path: &Path) -> std::io::Result<NodeTypeMap> {
let file = fs::File::open(node_types_path)?;
let node_types = serde_json::from_reader(file)?;
Ok(convert_nodes(&node_types))
Ok(convert_nodes(&prefix, &node_types))
}
pub fn read_node_types_str(node_types_json: &str) -> std::io::Result<NodeTypeMap> {
pub fn read_node_types_str(prefix: &str, node_types_json: &str) -> std::io::Result<NodeTypeMap> {
let node_types = serde_json::from_str(node_types_json)?;
Ok(convert_nodes(&node_types))
Ok(convert_nodes(&prefix, &node_types))
}
fn convert_type(node_type: &NodeType) -> TypeName {
@@ -104,7 +104,7 @@ fn convert_types(node_types: &Vec<NodeType>) -> Set<TypeName> {
std::collections::BTreeSet::from(iter)
}
pub fn convert_nodes(nodes: &Vec<NodeInfo>) -> NodeTypeMap {
pub fn convert_nodes(prefix: &str, nodes: &Vec<NodeInfo>) -> NodeTypeMap {
let mut entries = NodeTypeMap::new();
let mut token_kinds = Set::new();
@@ -125,6 +125,7 @@ pub fn convert_nodes(nodes: &Vec<NodeInfo>) -> NodeTypeMap {
let flattened_name = &node_type_name(&node.kind, node.named);
let dbscheme_name = escape_name(&flattened_name);
let ql_class_name = dbscheme_name_to_class_name(&dbscheme_name);
let dbscheme_name = format!("{}_{}", prefix, &dbscheme_name);
if let Some(subtypes) = &node.subtypes {
// It's a tree-sitter supertype node, for which we create a union
// type.
@@ -150,6 +151,8 @@ pub fn convert_nodes(nodes: &Vec<NodeInfo>) -> NodeTypeMap {
named: node.named,
};
let table_name = escape_name(&(format!("{}_def", &flattened_name)));
let table_name = format!("{}_{}", prefix, &table_name);
let mut fields = Vec::new();
// If the type also has fields or children, then we create either
@@ -157,6 +160,7 @@ pub fn convert_nodes(nodes: &Vec<NodeInfo>) -> NodeTypeMap {
if let Some(node_fields) = &node.fields {
for (field_name, field_info) in node_fields {
add_field(
&prefix,
&type_name,
Some(field_name.to_string()),
field_info,
@@ -167,7 +171,14 @@ pub fn convert_nodes(nodes: &Vec<NodeInfo>) -> NodeTypeMap {
}
if let Some(children) = &node.children {
// Treat children as if they were a field called 'child'.
add_field(&type_name, None, children, &mut fields, &token_kinds);
add_field(
&prefix,
&type_name,
None,
children,
&mut fields,
&token_kinds,
);
}
entries.insert(
type_name,
@@ -188,13 +199,13 @@ pub fn convert_nodes(nodes: &Vec<NodeInfo>) -> NodeTypeMap {
counter += 1;
let unprefixed_name = node_type_name(&type_name.kind, true);
Entry {
dbscheme_name: escape_name(&format!("token_{}", &unprefixed_name)),
dbscheme_name: escape_name(&format!("{}_token_{}", &prefix, &unprefixed_name)),
ql_class_name: dbscheme_name_to_class_name(&escape_name(&unprefixed_name)),
kind: EntryKind::Token { kind_id: counter },
}
} else {
Entry {
dbscheme_name: "reserved_word".to_owned(),
dbscheme_name: format!("{}_reserved_word", &prefix),
ql_class_name: "ReservedWord".to_owned(),
kind: EntryKind::Token { kind_id: 0 },
}
@@ -205,6 +216,7 @@ pub fn convert_nodes(nodes: &Vec<NodeInfo>) -> NodeTypeMap {
}
fn add_field(
prefix: &str,
parent_type_name: &TypeName,
field_name: Option<String>,
field_info: &FieldInfo,
@@ -221,7 +233,8 @@ fn add_field(
// Put the field in an auxiliary table.
let has_index = field_info.multiple;
let field_table_name = escape_name(&format!(
"{}_{}",
"{}_{}_{}",
&prefix,
parent_flattened_name,
&name_for_field_or_child(&field_name)
));
@@ -244,7 +257,7 @@ fn add_field(
let mut field_token_ints: BTreeMap<String, (usize, String)> = BTreeMap::new();
for t in converted_types {
let dbscheme_variant_name =
escape_name(&format!("{}_{}", parent_flattened_name, t.kind));
escape_name(&format!("{}_{}_{}", &prefix, parent_flattened_name, t.kind));
field_token_ints.insert(t.kind.to_owned(), (counter, dbscheme_variant_name));
counter += 1;
}
@@ -256,7 +269,8 @@ fn add_field(
FieldTypeInfo::Multiple {
types: converted_types,
dbscheme_union: format!(
"{}_{}_type",
"{}_{}_{}_type",
&prefix,
&parent_flattened_name,
&name_for_field_or_child(&field_name)
),
@@ -380,6 +394,23 @@ fn escape_name(name: &str) -> String {
result
}
pub fn to_snake_case(word: &str) -> String {
let mut prev_upper = true;
let mut result = String::new();
for c in word.chars() {
if c.is_uppercase() {
if !prev_upper {
result.push('_')
}
prev_upper = true;
result.push(c.to_ascii_lowercase());
} else {
prev_upper = false;
result.push(c);
}
}
result
}
/// Given a valid dbscheme name (i.e. in snake case), produces the equivalent QL
/// name (i.e. in CamelCase). For example, "foo_bar_baz" becomes "FooBarBaz".
fn dbscheme_name_to_class_name(dbscheme_name: &str) -> String {
@@ -402,3 +433,10 @@ fn dbscheme_name_to_class_name(dbscheme_name: &str) -> String {
.collect::<Vec<String>>()
.join("")
}
#[test]
fn to_snake_case_test() {
assert_eq!("ruby", to_snake_case("Ruby"));
assert_eq!("erb", to_snake_case("ERB"));
assert_eq!("embedded_template", to_snake_case("EmbeddedTemplate"));
}

1054
ql/src/codeql_ruby/ast/internal/TreeSitter.qll
View File

File diff suppressed because it is too large Load Diff

2458
ql/src/ruby.dbscheme
View File

File diff suppressed because it is too large Load Diff