From 9109546adbc910fa5464732ecb01f7b3ac621259 Mon Sep 17 00:00:00 2001 From: erik-krogh Date: Sun, 20 Nov 2022 14:19:01 +0100 Subject: [PATCH] sync up the QL extractor with the Ruby one --- ql/Cargo.lock | Bin 14885 -> 15765 bytes ql/extractor/Cargo.toml | 2 +- ql/extractor/src/main.rs | 19 ++++--------------- ql/generator/Cargo.toml | 2 +- ql/generator/src/main.rs | 7 +++---- ql/node-types/Cargo.toml | 2 +- ql/node-types/src/lib.rs | 6 +++--- 7 files changed, 13 insertions(+), 25 deletions(-) diff --git a/ql/Cargo.lock b/ql/Cargo.lock index 68c195d8775275ee4f3618453afff4bd67c45842..bf3a85fdd6c04570891609f97d20cbbdfec8df69 100644 GIT binary patch delta 984 zcmY+CKW|(`6vbtXHn9x^3u0Tgkc=y^&}wGxotZm|f|oA8Vq}Xb`b5aSKi$3EpWWEF!zcOt z=tM5|K3Uz}J3gy&ahlIB2h-VPdVG5MJCn&Nk59&ve@7RyN0WTse_h$xyxP9{P;~kD zZ1?W&o*BCS<^JKwkvGjXpUH7)2(eb58&*`$)Jkpa)Rd15)mQ`QV+<{M7lU`fvhRQF zZ>&y_)W}hJJd^&*$_MlJZayFE2*5g}mSku3G z_iz+JE5Sy5l2AQa$G$dkS*_=w+Y+W`D2X#7oU#+|P4Fq`gkaD@E@<8R=L9ZFI1!l(;5iGil)vUS%S-l#DsXf?gPjvJxu5g(#|P!IaS3R-@L!Of~Hgp{ir&z$Rw>}sqm#k|0BE^{zNvXcDWRjdR-8;j*& zEf#{Ml14jU!3Pj~JFP4%JjBXwzG0Yee&4))`mXu#yM0EERB0VdZhK)t7_9+CR2UFJ zK<1cbN>=FrPJ1hvvn6da>Xf7P4x}?6gF-0Jx?4RzUrrAjUd)Eq%bShMH74jYo<1PNu1OucojqT^W8K|whPE6UsbOXb{7=j1t%j`4)1e) z16p=I3c?g5<;>PdViDO0a;l>A%Y?r4*Rj=5|F^-@)5$FeDP;h3fh|dP6ir>@0}_Ce jsotNfDJNyUR?*dh>bNz`QUH~U?d{d(%^$7uZU56>QRJpp diff --git a/ql/extractor/Cargo.toml b/ql/extractor/Cargo.toml index c5b0e938c0e..bd5ef0539b6 100644 --- a/ql/extractor/Cargo.toml +++ b/ql/extractor/Cargo.toml @@ -13,7 +13,7 @@ tree-sitter = ">= 0.20, < 0.21" tree-sitter-ql = { git = "https://github.com/tree-sitter/tree-sitter-ql.git", rev = "d08db734f8dc52f6bc04db53a966603122bc6985"} tree-sitter-ql-dbscheme = { git = "https://github.com/erik-krogh/tree-sitter-ql-dbscheme.git", rev = "63e1344353f63931e88bfbc2faa2e78e1421b213"} tree-sitter-ql-yaml = {git = "https://github.com/erik-krogh/tree-sitter-ql.git", rev = "cf704bf3671e1ae148e173464fb65a4d2bbf5f99"} -clap = "2.33" +clap = "3.0" tracing = "0.1" tracing-subscriber = { version = "0.3.3", features = ["env-filter"] } rayon = "1.5.0" diff --git a/ql/extractor/src/main.rs b/ql/extractor/src/main.rs index 45cce6acdd1..1b760a50179 100644 --- a/ql/extractor/src/main.rs +++ b/ql/extractor/src/main.rs @@ -3,6 +3,7 @@ mod trap; extern crate num_cpus; +use clap::arg; use rayon::prelude::*; use std::fs; use std::io::BufRead; @@ -44,7 +45,6 @@ fn main() -> std::io::Result<()> { .with_level(true) .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) .init(); - let num_threads = num_codeql_threads(); tracing::info!( "Using {} {}", @@ -64,11 +64,9 @@ fn main() -> std::io::Result<()> { .version("1.0") .author("GitHub") .about("CodeQL QL extractor") - .args_from_usage( - "--source-archive-dir= 'Sets a custom source archive folder' - --output-dir= 'Sets a custom trap folder' - --file-list= 'A text files containing the paths of the files to extract'", - ) + .arg(arg!(--"source-archive-dir" "Sets a custom source archive folder")) + .arg(arg!(--"output-dir" "Sets a custom trap folder")) + .arg(arg!(--"file-list" "A text file containing the paths of the files to extract")) .get_matches(); let src_archive_dir = matches .value_of("source-archive-dir") @@ -97,15 +95,6 @@ fn main() -> std::io::Result<()> { lines .par_iter() .try_for_each(|line| { - // only consider files that end with .ql/.qll/.dbscheme/qlpack.yml - // TODO: This is a bad fix, wait for the post-merge discussion in https://github.com/github/codeql/pull/7444 to be resolved - if !line.ends_with(".ql") - && !line.ends_with(".qll") - && !line.ends_with(".dbscheme") - && !line.ends_with("qlpack.yml") - { - return Ok(()); - } let path = PathBuf::from(line).canonicalize()?; let src_archive_file = path_for(&src_archive_dir, &path, ""); let source = std::fs::read(&path)?; diff --git a/ql/generator/Cargo.toml b/ql/generator/Cargo.toml index c0531a8f1e5..755cda04860 100644 --- a/ql/generator/Cargo.toml +++ b/ql/generator/Cargo.toml @@ -7,7 +7,7 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -clap = "2.33" +clap = "3.0" node-types = { path = "../node-types" } tracing = "0.1" tracing-subscriber = { version = "0.3.3", features = ["env-filter"] } diff --git a/ql/generator/src/main.rs b/ql/generator/src/main.rs index 790cb7f1533..94bdbfecf14 100644 --- a/ql/generator/src/main.rs +++ b/ql/generator/src/main.rs @@ -3,6 +3,7 @@ mod language; mod ql; mod ql_gen; +use clap::arg; use language::Language; use std::collections::BTreeMap as Map; use std::collections::BTreeSet as Set; @@ -553,10 +554,8 @@ fn main() -> std::io::Result<()> { .version("1.0") .author("GitHub") .about("CodeQL QL dbscheme generator") - .args_from_usage( - "--dbscheme= 'Path of the generated dbscheme file' - --library= 'Path of the generated QLL file'", - ) + .arg(arg!(--dbscheme "Path of the generated dbscheme file")) + .arg(arg!(--library "Path of the generated QLL file")) .get_matches(); let dbscheme_path = matches.value_of("dbscheme").expect("missing --dbscheme"); let dbscheme_path = PathBuf::from(dbscheme_path); diff --git a/ql/node-types/Cargo.toml b/ql/node-types/Cargo.toml index 181bd6481e9..c751d7360d6 100644 --- a/ql/node-types/Cargo.toml +++ b/ql/node-types/Cargo.toml @@ -8,4 +8,4 @@ edition = "2018" [dependencies] serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" +serde_json = "1.0" \ No newline at end of file diff --git a/ql/node-types/src/lib.rs b/ql/node-types/src/lib.rs index 0472443a380..a48e9ddbf9a 100644 --- a/ql/node-types/src/lib.rs +++ b/ql/node-types/src/lib.rs @@ -434,7 +434,7 @@ fn dbscheme_name_to_class_name(dbscheme_name: &str) -> String { #[test] fn to_snake_case_test() { - assert_eq!("python", to_snake_case("Python")); - assert_eq!("yaml", to_snake_case("YAML")); - assert_eq!("set_literal", to_snake_case("SetLiteral")); + assert_eq!("ruby", to_snake_case("Ruby")); + assert_eq!("erb", to_snake_case("ERB")); + assert_eq!("embedded_template", to_snake_case("EmbeddedTemplate")); }