hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-02-19 16:33:40 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #11 from github/ql_gen

Browse Source 
Generate QL classes
This commit is contained in:
Nick Rolfe
2020-10-28 12:25:53 +00:00
committed by GitHub
parent 24b4586ddd e03d5da8cd
commit 29899485c7
13 changed files with 3032 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
.github/workflows/build.yml vendored
View File

@@ -28,12 +28,17 @@ jobs:
run: cargo build --release
- name: Generate dbscheme
if: ${{ matrix.os == 'ubuntu-latest' }}
run: target/release/generator
run: target/release/ruby-generator
- uses: actions/upload-artifact@v2
if: ${{ matrix.os == 'ubuntu-latest' }}
with:
name: ruby.dbscheme
path: ruby.dbscheme
- uses: actions/upload-artifact@v2
if: ${{ matrix.os == 'ubuntu-latest' }}
with:
name: ruby_ast.qll
path: ruby_ast.qll
- uses: actions/upload-artifact@v2
with:
name: extractor-${{ matrix.os }}

30
Cargo.lock generated
View File

@@ -96,16 +96,6 @@ dependencies = [
"vec_map",
]
[[package]]
name = "generator"
version = "0.1.0"
dependencies = [
"node-types",
"tracing",
"tracing-subscriber",
"tree-sitter-ruby",
]
[[package]]
name = "generator"
version = "0.6.23"
@@ -142,9 +132,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "libc"
version = "0.2.79"
version = "0.2.80"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2448f6066e80e3bfc792e9c98bf705b4b0fc6e8ef5b43e5889aff0eaa9c58743"
checksum = "4d58d1b70b004888f764dfbf6a26a3b0342a1632d33968e4a179d8011c760614"
[[package]]
name = "log"
@@ -162,7 +152,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a0e8460f2f2121162705187214720353c517b97bdfb3494c0b1e33d83ebe4bed"
dependencies = [
"cfg-if",
"generator 0.6.23",
"generator",
"scoped-tls",
"serde",
"serde_json",
@@ -274,6 +264,16 @@ dependencies = [
"tree-sitter-ruby",
]
[[package]]
name = "ruby-generator"
version = "0.1.0"
dependencies = [
"node-types",
"tracing",
"tracing-subscriber",
"tree-sitter-ruby",
]
[[package]]
name = "rustc_version"
version = "0.2.3"
@@ -365,9 +365,9 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
[[package]]
name = "syn"
version = "1.0.46"
version = "1.0.48"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ad5de3220ea04da322618ded2c42233d02baca219d6f160a3e9c87cda16c942"
checksum = "cc371affeffc477f42a221a1e4297aedcea33d47d19b61455588bd9d8f6b19ac"
dependencies = [
"proc-macro2",
"quote",

21
README.md
View File

@@ -1,3 +1,22 @@
# Ruby analysis support for CodeQL
Under development.
Under development.
## Building the tools from source
[Install Rust](https://www.rust-lang.org/tools/install), then run:
```bash
cargo build --release
```
## Generating the database schema and QL library
The generated `ruby.dbscheme` and `ruby_ast.qll` files are included in the repository, but they can be re-generated as follows:
```bash
# Run the generator
cargo run --release -p ruby-generator
# Then auto-format the QL library
codeql query format -i ruby_ast.qll
```

10
extractor/src/extractor.rs
View File

@@ -225,10 +225,7 @@ impl Visitor<'_> {
"too many values"
},
node.kind(),
match field.name.as_ref() {
Some(x) => x,
None => "child",
}
&field.get_name()
)
}
}
@@ -237,10 +234,7 @@ impl Visitor<'_> {
self.trap_output.push(TrapEntry::ChildOf(
node_type_name(&field.parent.kind, field.parent.named),
parent_id,
match &field.name {
Some(name) => name.to_owned(),
None => "child".to_owned(),
},
field.get_name(),
Index(index),
*child_id,
));

2
generator/Cargo.toml
View File

@@ -1,5 +1,5 @@
[package]
name = "generator"
name = "ruby-generator"
version = "0.1.0"
authors = ["GitHub"]
edition = "2018"

22
generator/src/dbscheme.rs
View File

@@ -1,3 +1,4 @@
use crate::ql;
use std::fmt;
/// Represents a distinct entry in the database schema.
@@ -27,7 +28,7 @@ pub struct Column {
pub db_type: DbColumnType,
pub name: String,
pub unique: bool,
pub ql_type: QlColumnType,
pub ql_type: ql::Type,
pub ql_type_is_ref: bool,
}
@@ -37,18 +38,6 @@ pub enum DbColumnType {
String,
}
// The QL type of a column.
pub enum QlColumnType {
/// Primitive `int` type.
Int,
/// Primitive `string` type.
String,
/// A custom type, defined elsewhere by a table or union.
Custom(String),
}
impl fmt::Display for Table {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
if let Some(keyset) = &self.keysets {
@@ -76,12 +65,7 @@ impl fmt::Display for Table {
DbColumnType::String => "string",
}
)?;
write!(f, "{}: ", column.name)?;
match &column.ql_type {
QlColumnType::Int => write!(f, "int")?,
QlColumnType::String => write!(f, "string")?,
QlColumnType::Custom(name) => write!(f, "@{}", name)?,
}
write!(f, "{}: {}", column.name, column.ql_type)?;
if column.ql_type_is_ref {
write!(f, " ref")?;
}

1
generator/src/language.rs
View File

@@ -4,4 +4,5 @@ pub struct Language {
pub name: String,
pub node_types: &'static str,
pub dbscheme_path: PathBuf,
pub ql_library_path: PathBuf,
}

83
generator/src/main.rs
View File

@@ -1,8 +1,9 @@
mod dbscheme;
mod language;
mod ql;
mod ql_gen;
use language::Language;
use node_types;
use std::collections::BTreeSet as Set;
use std::fs::File;
use std::io::LineWriter;
@@ -27,13 +28,10 @@ fn make_field_type(
// type to represent them.
let field_union_name = format!("{}_{}_type", parent_name, field_name);
let field_union_name = node_types::escape_name(&field_union_name);
let mut members: Vec<String> = Vec::new();
for field_type in types {
members.push(node_types::escape_name(&node_types::node_type_name(
&field_type.kind,
field_type.named,
)));
}
let members: Vec<String> = types
.iter()
.map(|t| node_types::escape_name(&node_types::node_type_name(&t.kind, t.named)))
.collect();
entries.push(dbscheme::Entry::Union(dbscheme::Union {
name: field_union_name.clone(),
members,
@@ -49,10 +47,7 @@ fn add_field(
field: &node_types::Field,
entries: &mut Vec<dbscheme::Entry>,
) {
let field_name = match &field.name {
None => "child".to_owned(),
Some(x) => x.to_owned(),
};
let field_name = field.get_name();
let parent_name = node_types::node_type_name(&field.parent.kind, field.parent.named);
match field.storage {
node_types::Storage::Table { .. } => {
@@ -67,9 +62,7 @@ fn add_field(
unique: false,
db_type: dbscheme::DbColumnType::Int,
name: node_types::escape_name(&parent_name),
ql_type: dbscheme::QlColumnType::Custom(node_types::escape_name(
&parent_name,
)),
ql_type: ql::Type::AtType(node_types::escape_name(&parent_name)),
ql_type_is_ref: true,
},
// Then an index column.
@@ -77,7 +70,7 @@ fn add_field(
unique: false,
db_type: dbscheme::DbColumnType::Int,
name: "index".to_string(),
ql_type: dbscheme::QlColumnType::Int,
ql_type: ql::Type::Int,
ql_type_is_ref: true,
},
// And then the field
@@ -85,7 +78,7 @@ fn add_field(
unique: true,
db_type: dbscheme::DbColumnType::Int,
name: node_types::escape_name(&field_type),
ql_type: dbscheme::QlColumnType::Custom(field_type),
ql_type: ql::Type::AtType(field_type),
ql_type_is_ref: true,
},
],
@@ -106,7 +99,7 @@ fn add_field(
unique: false,
db_type: dbscheme::DbColumnType::Int,
name: node_types::escape_name(&field_name),
ql_type: dbscheme::QlColumnType::Custom(field_type),
ql_type: ql::Type::AtType(field_type),
ql_type_is_ref: true,
});
}
@@ -115,7 +108,10 @@ fn add_field(
/// Converts the given tree-sitter node types into CodeQL dbscheme entries.
fn convert_nodes(nodes: &Vec<node_types::Entry>) -> Vec<dbscheme::Entry> {
let mut entries: Vec<dbscheme::Entry> = Vec::new();
let mut entries: Vec<dbscheme::Entry> = vec![
create_location_table(),
create_source_location_prefix_table(),
];
let mut top_members: Vec<String> = Vec::new();
for node in nodes {
@@ -150,7 +146,7 @@ fn convert_nodes(nodes: &Vec<node_types::Entry>) -> Vec<dbscheme::Entry> {
db_type: dbscheme::DbColumnType::Int,
name: "id".to_string(),
unique: true,
ql_type: dbscheme::QlColumnType::Custom(node_types::escape_name(&name)),
ql_type: ql::Type::AtType(node_types::escape_name(&name)),
ql_type_is_ref: false,
}],
keysets: None,
@@ -170,7 +166,7 @@ fn convert_nodes(nodes: &Vec<node_types::Entry>) -> Vec<dbscheme::Entry> {
unique: false,
db_type: dbscheme::DbColumnType::String,
name: "text".to_string(),
ql_type: dbscheme::QlColumnType::String,
ql_type: ql::Type::String,
ql_type_is_ref: true,
});
}
@@ -180,7 +176,7 @@ fn convert_nodes(nodes: &Vec<node_types::Entry>) -> Vec<dbscheme::Entry> {
unique: false,
db_type: dbscheme::DbColumnType::Int,
name: "loc".to_string(),
ql_type: dbscheme::QlColumnType::Custom("location".to_string()),
ql_type: ql::Type::AtType("location".to_string()),
ql_type_is_ref: true,
});
@@ -200,7 +196,8 @@ fn convert_nodes(nodes: &Vec<node_types::Entry>) -> Vec<dbscheme::Entry> {
fn write_dbscheme(language: &Language, entries: &[dbscheme::Entry]) -> std::io::Result<()> {
info!(
"Writing to '{}'",
"Writing database schema for {} to '{}'",
&language.name,
match language.dbscheme_path.to_str() {
None => "<undisplayable>",
Some(p) => p,
@@ -211,7 +208,7 @@ fn write_dbscheme(language: &Language, entries: &[dbscheme::Entry]) -> std::io::
dbscheme::write(&language.name, &mut file, &entries)
}
fn create_location_entry() -> dbscheme::Entry {
fn create_location_table() -> dbscheme::Entry {
dbscheme::Entry::Table(dbscheme::Table {
name: "location".to_string(),
keysets: None,
@@ -220,49 +217,49 @@ fn create_location_entry() -> dbscheme::Entry {
unique: true,
db_type: dbscheme::DbColumnType::Int,
name: "id".to_string(),
ql_type: dbscheme::QlColumnType::Custom("location".to_string()),
ql_type: ql::Type::AtType("location".to_string()),
ql_type_is_ref: false,
},
dbscheme::Column {
unique: false,
db_type: dbscheme::DbColumnType::String,
name: "file_path".to_string(),
ql_type: dbscheme::QlColumnType::String,
ql_type: ql::Type::String,
ql_type_is_ref: true,
},
dbscheme::Column {
unique: false,
db_type: dbscheme::DbColumnType::Int,
name: "start_line".to_string(),
ql_type: dbscheme::QlColumnType::Int,
ql_type: ql::Type::Int,
ql_type_is_ref: true,
},
dbscheme::Column {
unique: false,
db_type: dbscheme::DbColumnType::Int,
name: "start_column".to_string(),
ql_type: dbscheme::QlColumnType::Int,
ql_type: ql::Type::Int,
ql_type_is_ref: true,
},
dbscheme::Column {
unique: false,
db_type: dbscheme::DbColumnType::Int,
name: "end_line".to_string(),
ql_type: dbscheme::QlColumnType::Int,
ql_type: ql::Type::Int,
ql_type_is_ref: true,
},
dbscheme::Column {
unique: false,
db_type: dbscheme::DbColumnType::Int,
name: "end_column".to_string(),
ql_type: dbscheme::QlColumnType::Int,
ql_type: ql::Type::Int,
ql_type_is_ref: true,
},
],
})
}
fn create_source_location_prefix_entry() -> dbscheme::Entry {
fn create_source_location_prefix_table() -> dbscheme::Entry {
dbscheme::Entry::Table(dbscheme::Table {
name: "sourceLocationPrefix".to_string(),
keysets: None,
@@ -270,7 +267,7 @@ fn create_source_location_prefix_entry() -> dbscheme::Entry {
unique: false,
db_type: dbscheme::DbColumnType::String,
name: "prefix".to_string(),
ql_type: dbscheme::QlColumnType::String,
ql_type: ql::Type::String,
ql_type_is_ref: true,
}],
})
@@ -290,6 +287,7 @@ fn main() {
name: "Ruby".to_string(),
node_types: tree_sitter_ruby::NODE_TYPES,
dbscheme_path: PathBuf::from("ruby.dbscheme"),
ql_library_path: PathBuf::from("ruby_ast.qll"),
};
match node_types::read_node_types_str(&ruby.node_types) {
Err(e) => {
@@ -297,15 +295,18 @@ fn main() {
std::process::exit(1);
}
Ok(nodes) => {
let mut dbscheme_entries = convert_nodes(&nodes);
dbscheme_entries.push(create_location_entry());
dbscheme_entries.push(create_source_location_prefix_entry());
match write_dbscheme(&ruby, &dbscheme_entries) {
Err(e) => {
error!("Failed to write dbscheme: {}", e);
std::process::exit(2);
}
Ok(()) => {}
let dbscheme_entries = convert_nodes(&nodes);
if let Err(e) = write_dbscheme(&ruby, &dbscheme_entries) {
error!("Failed to write dbscheme: {}", e);
std::process::exit(2);
}
let classes = ql_gen::convert_nodes(&nodes);
if let Err(e) = ql_gen::write(&ruby, &classes) {
println!("Failed to write QL library: {}", e);
std::process::exit(3);
}
}
}

180
generator/src/ql.rs Normal file
View File

@@ -0,0 +1,180 @@
use std::fmt;
pub struct Class {
pub name: String,
pub is_abstract: bool,
pub supertypes: Vec<Type>,
pub characteristic_predicate: Option<Expression>,
pub predicates: Vec<Predicate>,
}
impl fmt::Display for Class {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
if self.is_abstract {
write!(f, "abstract ")?;
}
write!(f, "class {} extends ", &self.name)?;
for (index, supertype) in self.supertypes.iter().enumerate() {
if index > 0 {
write!(f, ", ")?;
}
write!(f, "{}", supertype)?;
}
write!(f, " {{ \n")?;
if let Some(charpred) = &self.characteristic_predicate {
write!(
f,
" {}\n",
Predicate {
name: self.name.clone(),
overridden: false,
return_type: None,
formal_parameters: vec![],
body: charpred.clone(),
}
)?;
}
for predicate in &self.predicates {
write!(f, " {}\n", predicate)?;
}
write!(f, "}}")?;
Ok(())
}
}
// The QL type of a column.
#[derive(Clone)]
pub enum Type {
/// Primitive `int` type.
Int,
/// Primitive `string` type.
String,
/// A user-defined type.
Normal(String),
/// A database type that will need to be referred to with an `@` prefix.
AtType(String),
}
impl fmt::Display for Type {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
Type::Int => write!(f, "int"),
Type::String => write!(f, "string"),
Type::Normal(name) => write!(f, "{}", name),
Type::AtType(name) => write!(f, "@{}", name),
}
}
}
#[derive(Clone)]
pub enum Expression {
Var(String),
String(String),
Pred(String, Vec<Expression>),
Or(Vec<Expression>),
Equals(Box<Expression>, Box<Expression>),
}
impl fmt::Display for Expression {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
Expression::Var(x) => write!(f, "{}", x),
Expression::String(s) => write!(f, "\"{}\"", s),
Expression::Pred(n, args) => {
write!(f, "{}(", n)?;
for (index, arg) in args.iter().enumerate() {
if index > 0 {
write!(f, ", ")?;
}
write!(f, "{}", arg)?;
}
write!(f, ")")
}
Expression::Or(disjuncts) => {
if disjuncts.is_empty() {
write!(f, "none()")
} else {
for (index, disjunct) in disjuncts.iter().enumerate() {
if index > 0 {
write!(f, " or ")?;
}
write!(f, "{}", disjunct)?;
}
Ok(())
}
}
Expression::Equals(a, b) => write!(f, "{} = {}", a, b),
}
}
}
#[derive(Clone)]
pub struct Predicate {
pub name: String,
pub overridden: bool,
pub return_type: Option<Type>,
pub formal_parameters: Vec<FormalParameter>,
pub body: Expression,
}
impl fmt::Display for Predicate {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
if self.overridden {
write!(f, "override ")?;
}
match &self.return_type {
None => write!(f, "predicate ")?,
Some(return_type) => write!(f, "{} ", return_type)?,
}
write!(f, "{}(", self.name)?;
for (index, param) in self.formal_parameters.iter().enumerate() {
if index > 0 {
write!(f, ", ")?;
}
write!(f, "{}", param)?;
}
write!(f, ") {{ {} }}", self.body)?;
Ok(())
}
}
#[derive(Clone)]
pub struct FormalParameter {
pub name: String,
pub param_type: Type,
}
impl fmt::Display for FormalParameter {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{} {}", self.param_type, self.name)
}
}
/// Generates a QL library by writing the given `classes` to the `file`.
pub fn write(
language_name: &str,
file: &mut dyn std::io::Write,
classes: &[Class],
) -> std::io::Result<()> {
write!(file, "/*\n")?;
write!(file, " * CodeQL library for {}\n", language_name)?;
write!(
file,
" * Automatically generated from the tree-sitter grammar; do not edit\n"
)?;
write!(file, " */\n\n")?;
for class in classes {
write!(file, "{}\n\n", &class)?;
}
Ok(())
}

533
generator/src/ql_gen.rs Normal file
View File

@@ -0,0 +1,533 @@
use crate::language::Language;
use crate::ql;
use std::collections::{BTreeSet, HashMap};
use std::fs::File;
use std::io::LineWriter;
type SupertypeMap = HashMap<String, BTreeSet<String>>;
/// Writes the QL AST library for the given library.
///
/// # Arguments
///
/// `language` - the language for which we're generating a library
/// `classes` - the list of classes to write.
pub fn write(language: &Language, classes: &[ql::Class]) -> std::io::Result<()> {
println!(
"Writing QL library for {} to '{}'",
&language.name,
match language.ql_library_path.to_str() {
None => "<undisplayable>",
Some(p) => p,
}
);
let file = File::create(&language.ql_library_path)?;
let mut file = LineWriter::new(file);
ql::write(&language.name, &mut file, &classes)
}
/// Creates a map from QL class names to their supertypes.
fn create_supertype_map(nodes: &[node_types::Entry]) -> SupertypeMap {
let mut map = SupertypeMap::new();
for node in nodes {
match &node {
node_types::Entry::Union {
type_name,
members: n_members,
} => {
let supertype_class_name = dbscheme_name_to_class_name(&node_types::escape_name(
&node_types::node_type_name(&type_name.kind, type_name.named),
));
for n_member in n_members {
let subtype_class_name = dbscheme_name_to_class_name(&node_types::escape_name(
&node_types::node_type_name(&n_member.kind, n_member.named),
));
map.entry(subtype_class_name)
.or_insert_with(|| BTreeSet::new())
.insert(supertype_class_name.clone());
}
}
node_types::Entry::Table { type_name, fields } => {
let node_name = node_types::node_type_name(&type_name.kind, type_name.named);
for field in fields {
if field.types.len() != 1 {
// This field can have one of several types. Since we create an ad-hoc
// QL union type to represent them, the class wrapping that union type
// will be a supertype of all its members.
let field_union_name = format!("{}_{}_type", &node_name, &field.get_name());
let field_union_name = node_types::escape_name(&field_union_name);
let supertype_name = dbscheme_name_to_class_name(&field_union_name);
for field_type in &field.types {
let member_name =
node_types::node_type_name(&field_type.kind, field_type.named);
let member_class_name =
dbscheme_name_to_class_name(&node_types::escape_name(&member_name));
map.entry(member_class_name)
.or_insert_with(|| BTreeSet::new())
.insert(supertype_name.clone());
}
}
}
}
}
}
map
}
fn get_base_classes(name: &str, supertype_map: &SupertypeMap) -> Vec<ql::Type> {
let mut base_classes: Vec<ql::Type> = vec![ql::Type::Normal("Top".to_owned())];
if let Some(supertypes) = supertype_map.get(name) {
base_classes.extend(
supertypes
.into_iter()
.map(|st| ql::Type::Normal(st.clone())),
);
}
base_classes
}
/// Creates the hard-coded `Top` class that acts as a supertype of all classes we
/// generate.
fn create_top_class() -> ql::Class {
let to_string = create_none_predicate("toString", false, Some(ql::Type::String), vec![]);
let get_location = create_none_predicate(
"getLocation",
false,
Some(ql::Type::Normal("Location".to_owned())),
vec![],
);
let get_a_field_or_child = create_none_predicate(
"getAFieldOrChild",
false,
Some(ql::Type::Normal("Top".to_owned())),
vec![],
);
ql::Class {
name: "Top".to_owned(),
is_abstract: false,
supertypes: vec![ql::Type::AtType("top".to_owned())],
characteristic_predicate: None,
predicates: vec![to_string, get_location, get_a_field_or_child],
}
}
/// Creates a predicate whose body is `none()`.
fn create_none_predicate(
name: &str,
overridden: bool,
return_type: Option<ql::Type>,
formal_parameters: Vec<ql::FormalParameter>,
) -> ql::Predicate {
ql::Predicate {
name: name.to_owned(),
overridden,
return_type,
formal_parameters,
body: ql::Expression::Pred("none".to_owned(), vec![]),
}
}
/// Creates the special `Location` class to wrap the location table.
fn create_location_class() -> ql::Class {
let to_string = ql::Predicate {
name: "toString".to_owned(),
overridden: false,
return_type: Some(ql::Type::String),
formal_parameters: vec![],
body: ql::Expression::Equals(
Box::new(ql::Expression::Var("result".to_owned())),
Box::new(ql::Expression::String("Location".to_owned())),
),
};
let has_location_info = ql::Predicate {
name: "hasLocationInfo".to_owned(),
overridden: false,
return_type: None,
formal_parameters: vec![
ql::FormalParameter {
name: "filePath".to_owned(),
param_type: ql::Type::String,
},
ql::FormalParameter {
name: "startLine".to_owned(),
param_type: ql::Type::Int,
},
ql::FormalParameter {
name: "startColumn".to_owned(),
param_type: ql::Type::Int,
},
ql::FormalParameter {
name: "endLine".to_owned(),
param_type: ql::Type::Int,
},
ql::FormalParameter {
name: "endColumn".to_owned(),
param_type: ql::Type::Int,
},
],
body: ql::Expression::Pred(
"location".to_owned(),
vec![
ql::Expression::Var("this".to_owned()),
ql::Expression::Var("filePath".to_owned()),
ql::Expression::Var("startLine".to_owned()),
ql::Expression::Var("startColumn".to_owned()),
ql::Expression::Var("endLine".to_owned()),
ql::Expression::Var("endColumn".to_owned()),
],
),
};
ql::Class {
name: "Location".to_owned(),
supertypes: vec![ql::Type::AtType("location".to_owned())],
is_abstract: false,
characteristic_predicate: None,
predicates: vec![to_string, has_location_info],
}
}
/// Given the name of the parent node, and its field information, returns the
/// name of the field's type. This may be an ad-hoc union of all the possible
/// types the field can take, in which case we create a new class and push it to
/// `classes`.
fn create_field_class(
parent_name: &str,
field: &node_types::Field,
classes: &mut Vec<ql::Class>,
supertype_map: &SupertypeMap,
) -> String {
if field.types.len() == 1 {
// This field can only have a single type.
let t = field.types.iter().next().unwrap();
node_types::escape_name(&node_types::node_type_name(&t.kind, t.named))
} else {
// This field can have one of several types. The dbscheme contains a
// union type, so we create a QL class to wrap that.
let field_union_name = format!("{}_{}_type", parent_name, &field.get_name());
let field_union_name = node_types::escape_name(&field_union_name);
let class_name = dbscheme_name_to_class_name(&field_union_name);
classes.push(ql::Class {
name: class_name.clone(),
is_abstract: false,
supertypes: [
vec![ql::Type::AtType(field_union_name.clone())],
get_base_classes(&class_name, &supertype_map),
]
.concat(),
characteristic_predicate: None,
predicates: vec![],
});
field_union_name
}
}
/// Given a valid dbscheme name (i.e. in snake case), produces the equivalent QL
/// name (i.e. in CamelCase). For example, "foo_bar_baz" becomes "FooBarBaz".
fn dbscheme_name_to_class_name(dbscheme_name: &str) -> String {
fn to_title_case(word: &str) -> String {
let mut first = true;
let mut result = String::new();
for c in word.chars() {
if first {
first = false;
result.push(c.to_ascii_uppercase());
} else {
result.push(c);
}
}
result
}
dbscheme_name
.split('_')
.map(|word| to_title_case(word))
.collect::<Vec<String>>()
.join("")
}
/// Creates a `toString` predicate that returns the given text.
fn create_to_string_predicate(text: &str) -> ql::Predicate {
ql::Predicate {
name: "toString".to_owned(),
overridden: true,
return_type: Some(ql::Type::String),
formal_parameters: vec![],
body: ql::Expression::Equals(
Box::new(ql::Expression::Var("result".to_owned())),
Box::new(ql::Expression::String(text.to_owned())),
),
}
}
/// Creates the `getLocation` predicate.
///
/// # Arguments
///
/// `def_table` - the name of the table that defines the entity and its location.
/// `arity` - the total number of columns in the table
fn create_get_location_predicate(def_table: &str, arity: usize) -> ql::Predicate {
ql::Predicate {
name: "getLocation".to_owned(),
overridden: true,
return_type: Some(ql::Type::Normal("Location".to_owned())),
formal_parameters: vec![],
// body of the form: foo_bar_def(_, _, ..., result)
body: ql::Expression::Pred(
def_table.to_owned(),
[
vec![ql::Expression::Var("this".to_owned())],
vec![ql::Expression::Var("_".to_owned()); arity - 2],
vec![ql::Expression::Var("result".to_owned())],
]
.concat(),
),
}
}
/// Creates the `getText` predicate for a leaf node.
///
/// # Arguments
///
/// `def_table` - the name of the table that defines the entity and its text.
fn create_get_text_predicate(def_table: &str) -> ql::Predicate {
ql::Predicate {
name: "getText".to_owned(),
overridden: false,
return_type: Some(ql::Type::String),
formal_parameters: vec![],
body: ql::Expression::Pred(
def_table.to_owned(),
vec![
ql::Expression::Var("this".to_owned()),
ql::Expression::Var("result".to_owned()),
ql::Expression::Var("_".to_owned()),
],
),
}
}
/// Returns an expression to get a field that's defined as a column in the parent's table.
///
/// # Arguments
///
/// * `table_name` - the name of parent's defining table
/// * `column_index` - the index in that table that defines the field
/// * `arity` - the total number of columns in the table
fn create_get_field_expr_for_column_storage(
table_name: &str,
column_index: usize,
arity: usize,
) -> ql::Expression {
let num_underscores_before = column_index - 1;
let num_underscores_after = arity - 2 - num_underscores_before;
ql::Expression::Pred(
table_name.to_owned(),
[
vec![ql::Expression::Var("this".to_owned())],
vec![ql::Expression::Var("_".to_owned()); num_underscores_before],
vec![ql::Expression::Var("result".to_owned())],
vec![ql::Expression::Var("_".to_owned()); num_underscores_after],
]
.concat(),
)
}
/// Returns an expression to get the field with the given index from its
/// auxiliary table. The index name can be "_" so the expression will hold for
/// all indices.
fn create_get_field_expr_for_table_storage(
table_name: &str,
index_var_name: &str,
) -> ql::Expression {
ql::Expression::Pred(
table_name.to_owned(),
vec![
ql::Expression::Var("this".to_owned()),
ql::Expression::Var(index_var_name.to_owned()),
ql::Expression::Var("result".to_owned()),
],
)
}
/// Creates a pair consisting of a predicate to get the given field, and an
/// expression that will get the same field. When the field can occur multiple
/// times, the predicate will take an index argument, while the expression will
/// use the "don't care" expression to hold for all occurrences.
///
/// # Arguments
///
/// `main_table_name` - the name of the defining table for the parent node
/// `main_table_arity` - the number of columns in the main table
/// `main_table_column_index` - a mutable reference to a column index indicating
/// where the field is in the main table. If this is used (i.e. the field has
/// column storage), then the index is incremented.
/// `parent_name` - the name of the parent node
/// `field` - the field whose getters we are creating
/// `field_type` - the db name of the field's type (possibly being a union we created)
fn create_field_getters(
main_table_name: &str,
main_table_arity: usize,
main_table_column_index: &mut usize,
parent_name: &str,
field: &node_types::Field,
field_type: &str,
) -> (ql::Predicate, ql::Expression) {
match &field.storage {
node_types::Storage::Column => {
let result = (
ql::Predicate {
name: format!(
"get{}",
dbscheme_name_to_class_name(&node_types::escape_name(&field.get_name()))
),
overridden: false,
return_type: Some(ql::Type::Normal(dbscheme_name_to_class_name(field_type))),
formal_parameters: vec![],
body: create_get_field_expr_for_column_storage(
&main_table_name,
*main_table_column_index,
main_table_arity,
),
},
create_get_field_expr_for_column_storage(
&main_table_name,
*main_table_column_index,
main_table_arity,
),
);
*main_table_column_index += 1;
result
}
node_types::Storage::Table => {
let field_table_name = format!("{}_{}", parent_name, &field.get_name());
(
ql::Predicate {
name: format!(
"get{}",
dbscheme_name_to_class_name(&node_types::escape_name(&field.get_name()))
),
overridden: false,
return_type: Some(ql::Type::Normal(dbscheme_name_to_class_name(field_type))),
formal_parameters: vec![ql::FormalParameter {
name: "i".to_owned(),
param_type: ql::Type::Int,
}],
body: create_get_field_expr_for_table_storage(&field_table_name, "i"),
},
create_get_field_expr_for_table_storage(&field_table_name, "_"),
)
}
}
}
/// Converts the given node types into CodeQL classes wrapping the dbscheme.
pub fn convert_nodes(nodes: &Vec<node_types::Entry>) -> Vec<ql::Class> {
let supertype_map = create_supertype_map(nodes);
let mut classes: Vec<ql::Class> = vec![create_location_class(), create_top_class()];
for node in nodes {
match &node {
node_types::Entry::Union {
type_name,
members: _,
} => {
// It's a tree-sitter supertype node, so we're wrapping a dbscheme
// union type.
let union_name = node_types::escape_name(&node_types::node_type_name(
&type_name.kind,
type_name.named,
));
let class_name = dbscheme_name_to_class_name(&union_name);
classes.push(ql::Class {
name: class_name.clone(),
is_abstract: false,
supertypes: [
vec![ql::Type::AtType(union_name)],
get_base_classes(&class_name, &supertype_map),
]
.concat(),
characteristic_predicate: None,
predicates: vec![],
});
}
node_types::Entry::Table { type_name, fields } => {
// Count how many columns there will be in the main table.
// There will be:
// - one for the id
// - one for the location
// - one for each field that's stored as a column
// - if there are no fields, one for the text column.
let main_table_arity = 2 + if fields.is_empty() {
1
} else {
fields
.iter()
.filter(|&f| matches!(f.storage, node_types::Storage::Column))
.count()
};
let name = node_types::node_type_name(&type_name.kind, type_name.named);
let dbscheme_name = node_types::escape_name(&name);
let ql_type = ql::Type::AtType(dbscheme_name.clone());
let main_table_name = node_types::escape_name(&(format!("{}_def", name)));
let main_class_name = dbscheme_name_to_class_name(&dbscheme_name);
let mut main_class = ql::Class {
name: main_class_name.clone(),
is_abstract: false,
supertypes: [
vec![ql_type],
get_base_classes(&main_class_name, &supertype_map),
]
.concat(),
characteristic_predicate: None,
predicates: vec![
create_to_string_predicate(&main_class_name),
create_get_location_predicate(&main_table_name, main_table_arity),
],
};
if fields.is_empty() {
main_class
.predicates
.push(create_get_text_predicate(&main_table_name));
} else {
let mut main_table_column_index: usize = 1;
let mut get_child_exprs: Vec<ql::Expression> = Vec::new();
// Iterate through the fields, creating:
// - classes to wrap union types if fields need them,
// - predicates to access the fields,
// - the QL expressions to access the fields that will be part of getAFieldOrChild.
for field in fields {
let field_type =
create_field_class(&name, field, &mut classes, &supertype_map);
let (get_pred, get_child_expr) = create_field_getters(
&main_table_name,
main_table_arity,
&mut main_table_column_index,
&name,
field,
&field_type,
);
main_class.predicates.push(get_pred);
get_child_exprs.push(get_child_expr);
}
main_class.predicates.push(ql::Predicate {
name: "getAFieldOrChild".to_owned(),
overridden: true,
return_type: Some(ql::Type::Normal("Top".to_owned())),
formal_parameters: vec![],
body: ql::Expression::Or(get_child_exprs),
});
}
classes.push(main_class);
}
}
}
classes
}

9
node-types/src/lib.rs
View File

@@ -33,6 +33,15 @@ pub struct Field {
pub storage: Storage,
}
impl Field {
pub fn get_name(&self) -> String {
match &self.name {
Some(name) => name.clone(),
None => "child".to_owned(),
}
}
}
#[derive(Debug)]
pub enum Storage {
/// the field is stored as a column in the parent table

26
ruby.dbscheme
View File

@@ -1,6 +1,19 @@
// CodeQL database schema for Ruby
// Automatically generated from the tree-sitter grammar; do not edit
location(
unique int id: @location,
string file_path: string ref,
int start_line: int ref,
int start_column: int ref,
int end_line: int ref,
int end_column: int ref
);
sourceLocationPrefix(
string prefix: string ref
);
@underscore_arg = @underscore_primary | @assignment | @binary | @conditional | @operator_assignment | @range | @unary
@underscore_lhs = @underscore_variable | @call | @element_reference | @false | @method_call | @nil | @scope_resolution | @true
@@ -1849,16 +1862,3 @@ tilde_unnamed_def(
@top = @alias | @argument_list | @array | @assignment | @bare_string | @bare_symbol | @begin | @begin_block | @binary | @block | @block_argument | @block_parameter | @block_parameters | @break | @call | @case__ | @chained_string | @class | @conditional | @destructured_left_assignment | @destructured_parameter | @do | @do_block | @element_reference | @else | @elsif | @empty_statement | @end_block | @ensure | @exception_variable | @exceptions | @for | @hash | @hash_splat_argument | @hash_splat_parameter | @if | @if_modifier | @in | @interpolation | @keyword_parameter | @lambda | @lambda_parameters | @left_assignment_list | @method | @method_call | @method_parameters | @module | @next | @operator | @operator_assignment | @optional_parameter | @pair | @parenthesized_statements | @pattern | @program | @range | @rational | @redo | @regex | @rescue | @rescue_modifier | @rest_assignment | @retry | @return | @right_assignment_list | @scope_resolution | @setter | @singleton_class | @singleton_method | @splat_argument | @splat_parameter | @string__ | @string_array | @subshell | @superclass | @symbol | @symbol_array | @then | @unary | @undef | @unless | @unless_modifier | @until | @until_modifier | @when | @while | @while_modifier | @yield | @bang_unnamed | @bangequal_unnamed | @bangtilde_unnamed | @dquote_unnamed | @hashlbrace_unnamed | @percent_unnamed | @percentequal_unnamed | @percentilparen_unnamed | @percentwlparen_unnamed | @ampersand_unnamed | @ampersandampersand_unnamed | @ampersandampersandequal_unnamed | @ampersanddot_unnamed | @ampersandequal_unnamed | @lparen_unnamed | @rparen_unnamed | @star_unnamed | @starstar_unnamed | @starstarequal_unnamed | @starequal_unnamed | @plus_unnamed | @plusequal_unnamed | @plusat_unnamed | @comma_unnamed | @minus_unnamed | @minusequal_unnamed | @minusrangle_unnamed | @minusat_unnamed | @dot_unnamed | @dotdot_unnamed | @dotdotdot_unnamed | @slash_unnamed | @slashequal_unnamed | @colon_unnamed | @colondquote_unnamed | @coloncolon_unnamed | @semicolon_unnamed | @langle_unnamed | @langlelangle_unnamed | @langlelangleequal_unnamed | @langleequal_unnamed | @langleequalrangle_unnamed | @equal_unnamed | @equalequal_unnamed | @equalequalequal_unnamed | @equalrangle_unnamed | @equaltilde_unnamed | @rangle_unnamed | @rangleequal_unnamed | @ranglerangle_unnamed | @ranglerangleequal_unnamed | @question_unnamed | @b_e_g_i_n__unnamed | @e_n_d__unnamed | @lbracket_unnamed | @lbracketrbracket_unnamed | @lbracketrbracketequal_unnamed | @rbracket_unnamed | @caret_unnamed | @caretequal_unnamed | @underscore__e_n_d____unnamed | @backtick_unnamed | @alias_unnamed | @and_unnamed | @begin_unnamed | @break_unnamed | @case_unnamed | @character | @class_unnamed | @class_variable | @complex | @constant | @def_unnamed | @definedquestion_unnamed | @do_unnamed | @else_unnamed | @elsif_unnamed | @end_unnamed | @ensure_unnamed | @escape_sequence | @false | @float__ | @for_unnamed | @global_variable | @heredoc_beginning | @heredoc_end | @identifier | @if_unnamed | @in_unnamed | @instance_variable | @integer | @module_unnamed | @next_unnamed | @nil | @not_unnamed | @or_unnamed | @r_unnamed | @redo_unnamed | @rescue_unnamed | @retry_unnamed | @return_unnamed | @self | @super | @then_unnamed | @true | @undef_unnamed | @uninterpreted | @unless_unnamed | @until_unnamed | @when_unnamed | @while_unnamed | @yield_unnamed | @lbrace_unnamed | @pipe_unnamed | @pipeequal_unnamed | @pipepipe_unnamed | @pipepipeequal_unnamed | @rbrace_unnamed | @tilde_unnamed
location(
unique int id: @location,
string file_path: string ref,
int start_line: int ref,
int start_column: int ref,
int end_line: int ref,
int end_column: int ref
);
sourceLocationPrefix(
string prefix: string ref
);

2207
ruby_ast.qll Normal file
View File

File diff suppressed because it is too large Load Diff