sync up the QL extractor with the Ruby one

ql/extractor/Cargo.toml

@@ -13,7 +13,7 @@ tree-sitter = ">= 0.20, < 0.21"
 tree-sitter-ql = { git = "https://github.com/tree-sitter/tree-sitter-ql.git", rev = "d08db734f8dc52f6bc04db53a966603122bc6985"}
 tree-sitter-ql-dbscheme = { git = "https://github.com/erik-krogh/tree-sitter-ql-dbscheme.git", rev = "63e1344353f63931e88bfbc2faa2e78e1421b213"}
 tree-sitter-ql-yaml = {git = "https://github.com/erik-krogh/tree-sitter-ql.git", rev = "cf704bf3671e1ae148e173464fb65a4d2bbf5f99"}
-clap = "2.33"
+clap = "3.0"
 tracing = "0.1"
 tracing-subscriber = { version = "0.3.3", features = ["env-filter"] }
 rayon = "1.5.0"

ql/extractor/src/main.rs

@@ -3,6 +3,7 @@ mod trap;
 
 extern crate num_cpus;
 
+use clap::arg;
 use rayon::prelude::*;
 use std::fs;
 use std::io::BufRead;
@@ -44,7 +45,6 @@ fn main() -> std::io::Result<()> {
         .with_level(true)
         .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
         .init();
-
     let num_threads = num_codeql_threads();
     tracing::info!(
         "Using {} {}",
@@ -64,11 +64,9 @@ fn main() -> std::io::Result<()> {
         .version("1.0")
         .author("GitHub")
         .about("CodeQL QL extractor")
-        .args_from_usage(
-            "--source-archive-dir=<DIR> 'Sets a custom source archive folder'
-                    --output-dir=<DIR>         'Sets a custom trap folder'
-                    --file-list=<FILE_LIST>    'A text files containing the paths of the files to extract'",
-        )
+        .arg(arg!(--"source-archive-dir" <DIR> "Sets a custom source archive folder"))
+        .arg(arg!(--"output-dir" <DIR>         "Sets a custom trap folder"))
+        .arg(arg!(--"file-list" <FILE_LIST>    "A text file containing the paths of the files to extract"))
         .get_matches();
     let src_archive_dir = matches
         .value_of("source-archive-dir")
@@ -97,15 +95,6 @@ fn main() -> std::io::Result<()> {
     lines
         .par_iter()
         .try_for_each(|line| {
-            // only consider files that end with .ql/.qll/.dbscheme/qlpack.yml
-            // TODO: This is a bad fix, wait for the post-merge discussion in https://github.com/github/codeql/pull/7444 to be resolved
-            if !line.ends_with(".ql")
-                && !line.ends_with(".qll")
-                && !line.ends_with(".dbscheme")
-                && !line.ends_with("qlpack.yml")
-            {
-                return Ok(());
-            }
             let path = PathBuf::from(line).canonicalize()?;
             let src_archive_file = path_for(&src_archive_dir, &path, "");
             let source = std::fs::read(&path)?;

ql/generator/Cargo.toml

@@ -7,7 +7,7 @@ edition = "2018"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-clap = "2.33"
+clap = "3.0"
 node-types = { path = "../node-types" }
 tracing = "0.1"
 tracing-subscriber = { version = "0.3.3", features = ["env-filter"] }

ql/generator/src/main.rs

@@ -3,6 +3,7 @@ mod language;
 mod ql;
 mod ql_gen;
 
+use clap::arg;
 use language::Language;
 use std::collections::BTreeMap as Map;
 use std::collections::BTreeSet as Set;
@@ -553,10 +554,8 @@ fn main() -> std::io::Result<()> {
         .version("1.0")
         .author("GitHub")
         .about("CodeQL QL dbscheme generator")
-        .args_from_usage(
-            "--dbscheme=<FILE>                  'Path of the generated dbscheme file'
-             --library=<FILE>                   'Path of the generated QLL file'",
-        )
+        .arg(arg!(--dbscheme <FILE>                  "Path of the generated dbscheme file"))
+        .arg(arg!(--library <FILE>                   "Path of the generated QLL file"))
         .get_matches();
     let dbscheme_path = matches.value_of("dbscheme").expect("missing --dbscheme");
     let dbscheme_path = PathBuf::from(dbscheme_path);

ql/node-types/Cargo.toml

@@ -8,4 +8,4 @@ edition = "2018"
 
 [dependencies]
 serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0"
+serde_json = "1.0"

ql/node-types/src/lib.rs

@@ -434,7 +434,7 @@ fn dbscheme_name_to_class_name(dbscheme_name: &str) -> String {
 
 #[test]
 fn to_snake_case_test() {
-    assert_eq!("python", to_snake_case("Python"));
-    assert_eq!("yaml", to_snake_case("YAML"));
-    assert_eq!("set_literal", to_snake_case("SetLiteral"));
+    assert_eq!("ruby", to_snake_case("Ruby"));
+    assert_eq!("erb", to_snake_case("ERB"));
+    assert_eq!("embedded_template", to_snake_case("EmbeddedTemplate"));
 }