Merge pull request #12786 from hmac/merge-extractor-binaries

Ruby/QL: Merge extractor binaries

.github/workflows/ruby-build.yml

@@ -58,10 +58,8 @@ jobs:
         id: cache-extractor
         with:
           path: |
-            ruby/extractor/target/release/autobuilder
-            ruby/extractor/target/release/autobuilder.exe
-            ruby/extractor/target/release/extractor
-            ruby/extractor/target/release/extractor.exe
+            ruby/extractor/target/release/codeql-extractor-ruby
+            ruby/extractor/target/release/codeql-extractor-ruby.exe
             ruby/extractor/ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
           key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-ruby-extractor-${{ hashFiles('ruby/extractor/rust-toolchain.toml', 'ruby/extractor/Cargo.lock') }}--${{ hashFiles('ruby/extractor/**/*.rs') }}
       - uses: actions/cache@v3
@@ -88,15 +86,13 @@ jobs:
         run: |
           cd extractor
           cross build --release
-          mv target/x86_64-unknown-linux-gnu/release/extractor target/release/
-          mv target/x86_64-unknown-linux-gnu/release/autobuilder target/release/
-          mv target/x86_64-unknown-linux-gnu/release/generator target/release/
+          mv target/x86_64-unknown-linux-gnu/release/codeql-extractor-ruby target/release/
       - name: Release build (windows and macos)
         if: steps.cache-extractor.outputs.cache-hit != 'true' && runner.os != 'Linux'
         run: cd extractor && cargo build --release
       - name: Generate dbscheme
         if: ${{ matrix.os == 'ubuntu-latest' && steps.cache-extractor.outputs.cache-hit != 'true'}}
-        run: extractor/target/release/generator --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
+        run: extractor/target/release/codeql-extractor-ruby generate --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
       - uses: actions/upload-artifact@v3
         if: ${{ matrix.os == 'ubuntu-latest' }}
         with:
@@ -111,10 +107,8 @@ jobs:
         with:
           name: extractor-${{ matrix.os }}
           path: |
-            ruby/extractor/target/release/autobuilder
-            ruby/extractor/target/release/autobuilder.exe
-            ruby/extractor/target/release/extractor
-            ruby/extractor/target/release/extractor.exe
+            ruby/extractor/target/release/codeql-extractor-ruby
+            ruby/extractor/target/release/codeql-extractor-ruby.exe
           retention-days: 1
   compile-queries:
     runs-on: ubuntu-latest-xl
@@ -172,13 +166,10 @@ jobs:
           mkdir -p ruby
           cp -r codeql-extractor.yml tools ql/lib/ruby.dbscheme.stats ruby/
           mkdir -p ruby/tools/{linux64,osx64,win64}
-          cp linux64/autobuilder ruby/tools/linux64/autobuilder
-          cp osx64/autobuilder ruby/tools/osx64/autobuilder
-          cp win64/autobuilder.exe ruby/tools/win64/autobuilder.exe
-          cp linux64/extractor ruby/tools/linux64/extractor
-          cp osx64/extractor ruby/tools/osx64/extractor
-          cp win64/extractor.exe ruby/tools/win64/extractor.exe
-          chmod +x ruby/tools/{linux64,osx64}/{autobuilder,extractor}
+          cp linux64/codeql-extractor-ruby ruby/tools/linux64/extractor
+          cp osx64/codeql-extractor-ruby ruby/tools/osx64/extractor
+          cp win64/codeql-extractor-ruby.exe ruby/tools/win64/extractor.exe
+          chmod +x ruby/tools/{linux64,osx64}/extractor
           zip -rq codeql-ruby.zip ruby
       - uses: actions/upload-artifact@v3
         with:

ql/extractor/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "ql-extractor"
+name = "codeql-extractor-ql"
 version = "0.1.0"
 authors = ["GitHub"]
 edition = "2018"
@@ -13,7 +13,7 @@ tree-sitter-ql-dbscheme = { git = "https://github.com/erik-krogh/tree-sitter-ql-
 tree-sitter-ql-yaml = {git = "https://github.com/erik-krogh/tree-sitter-ql.git", rev = "cf704bf3671e1ae148e173464fb65a4d2bbf5f99"}
 tree-sitter-blame = {path = "../buramu/tree-sitter-blame"}
 tree-sitter-json = {git = "https://github.com/tausbn/tree-sitter-json.git", rev = "745663ee997f1576fe1e7187e6347e0db36ec7a9"}
-clap = "4.2"
+clap = { version = "4.2", features = ["derive"] }
 tracing = "0.1"
 tracing-subscriber = { version = "0.3.16", features = ["env-filter"] }
 rayon = "1.7.0"

ql/extractor/src/autobuilder.rs

@@ -1,8 +1,13 @@
+use clap::Args;
 use std::env;
 use std::path::PathBuf;
 use std::process::Command;
 
-fn main() -> std::io::Result<()> {
+#[derive(Args)]
+// The autobuilder takes no command-line options, but this may change in the future.
+pub struct Options {}
+
+pub fn run(_: Options) -> std::io::Result<()> {
     let dist = env::var("CODEQL_DIST").expect("CODEQL_DIST not set");
     let db = env::var("CODEQL_EXTRACTOR_QL_WIP_DATABASE")
         .expect("CODEQL_EXTRACTOR_QL_WIP_DATABASE not set");

ql/extractor/src/extractor.rs

@@ -1,3 +1,4 @@
+use clap::Args;
 use rayon::prelude::*;
 use std::fs;
 use std::io::BufRead;
@@ -5,7 +6,22 @@ use std::path::{Path, PathBuf};
 
 use codeql_extractor::{diagnostics, extractor, node_types, trap};
 
-fn main() -> std::io::Result<()> {
+#[derive(Args)]
+pub struct Options {
+    /// Sets a custom source achive folder
+    #[arg(long)]
+    source_archive_dir: PathBuf,
+
+    /// Sets a custom trap folder
+    #[arg(long)]
+    output_dir: PathBuf,
+
+    /// A text file containing the paths of the files to extract
+    #[arg(long)]
+    file_list: PathBuf,
+}
+
+pub fn run(options: Options) -> std::io::Result<()> {
     tracing_subscriber::fmt()
         .with_target(false)
         .without_time()
@@ -58,30 +74,9 @@ fn main() -> std::io::Result<()> {
         .build_global()
         .unwrap();
 
-    let matches = clap::Command::new("QL extractor")
-        .version("1.0")
-        .author("GitHub")
-        .about("CodeQL QL extractor")
-        .args(&[
-            clap::arg!(--"source-archive-dir" <DIR> "Sets a custom source archive folder"),
-            clap::arg!(--"output-dir" <DIR> "Sets a custom trap folder"),
-            clap::arg!(--"file-list" <FILE_LIST> "A text file containing the paths of the files to extract"),
-        ])
-        .get_matches();
-    let src_archive_dir = matches
-        .get_one::<String>("source-archive-dir")
-        .expect("missing --source-archive-dir");
-    let src_archive_dir = PathBuf::from(src_archive_dir);
-
-    let trap_dir = matches
-        .get_one::<String>("output-dir")
-        .expect("missing --output-dir");
-    let trap_dir = PathBuf::from(trap_dir);
-
-    let file_list = matches
-        .get_one::<String>("file-list")
-        .expect("missing --file-list");
-    let file_list = fs::File::open(file_list)?;
+    let trap_dir = options.output_dir;
+    let file_list = fs::File::open(options.file_list)?;
+    let source_archive_dir = options.source_archive_dir;
 
     let language = tree_sitter_ql::language();
     let dbscheme = tree_sitter_ql_dbscheme::language();
@@ -114,7 +109,7 @@ fn main() -> std::io::Result<()> {
                 return Ok(());
             }
             let path = PathBuf::from(line).canonicalize()?;
-            let src_archive_file = path_for(&src_archive_dir, &path, "");
+            let src_archive_file = path_for(&source_archive_dir, &path, "");
             let source = std::fs::read(&path)?;
             let code_ranges = vec![];
             let mut trap_writer = trap::Writer::new();

ql/extractor/src/generator.rs

@@ -1,8 +1,20 @@
+use clap::Args;
 use std::path::PathBuf;
 
 use codeql_extractor::generator::{generate, language::Language};
 
-fn main() -> std::io::Result<()> {
+#[derive(Args)]
+pub struct Options {
+    /// Path of the generated dbscheme file
+    #[arg(long)]
+    dbscheme: PathBuf,
+
+    /// Path of the generated QLL file
+    #[arg(long)]
+    library: PathBuf,
+}
+
+pub fn run(options: Options) -> std::io::Result<()> {
     tracing_subscriber::fmt()
         .with_target(false)
         .without_time()
@@ -10,25 +22,6 @@ fn main() -> std::io::Result<()> {
         .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
         .init();
 
-    let matches = clap::Command::new("QL dbscheme generator")
-        .version("1.0")
-        .author("GitHub")
-        .about("CodeQL QL dbscheme generator")
-        .args(&[
-            clap::arg!(--dbscheme <FILE> "Path of the generated dbscheme file"),
-            clap::arg!(--library <FILE> "Path of the generated QLL file"),
-        ])
-        .get_matches();
-    let dbscheme_path = matches
-        .get_one::<String>("dbscheme")
-        .expect("missing --dbscheme");
-    let dbscheme_path = PathBuf::from(dbscheme_path);
-
-    let ql_library_path = matches
-        .get_one::<String>("library")
-        .expect("missing --library");
-    let ql_library_path = PathBuf::from(ql_library_path);
-
     let languages = vec![
         Language {
             name: "QL".to_owned(),
@@ -52,5 +45,5 @@ fn main() -> std::io::Result<()> {
         },
     ];
 
-    generate(languages, dbscheme_path, ql_library_path)
+    generate(languages, options.dbscheme, options.library)
 }

ql/extractor/src/main.rs

@@ -0,0 +1,23 @@
+use clap::Parser;
+
+mod autobuilder;
+mod extractor;
+mod generator;
+
+#[derive(Parser)]
+#[command(author, version, about)]
+enum Cli {
+    Extract(extractor::Options),
+    Generate(generator::Options),
+    Autobuild(autobuilder::Options),
+}
+
+fn main() -> std::io::Result<()> {
+    let cli = Cli::parse();
+
+    match cli {
+        Cli::Extract(options) => extractor::run(options),
+        Cli::Generate(options) => generator::run(options),
+        Cli::Autobuild(options) => autobuilder::run(options),
+    }
+}

ql/scripts/create-extractor-pack.ps1

@@ -1,6 +1,6 @@
 cargo build --release
 
-cargo run --release --bin generator -- --dbscheme ql/src/ql.dbscheme --library ql/src/codeql_ql/ast/internal/TreeSitter.qll
+cargo run --release --bin codeql-extractor-ql -- generate --dbscheme ql/src/ql.dbscheme --library ql/src/codeql_ql/ast/internal/TreeSitter.qll
 codeql query format -i ql\src\codeql_ql\ast\internal\TreeSitter.qll
 
 if (Test-Path -Path extractor-pack) {
@@ -10,5 +10,4 @@ mkdir extractor-pack | Out-Null
 cp codeql-extractor.yml, ql\src\ql.dbscheme, ql\src\ql.dbscheme.stats extractor-pack
 cp -Recurse tools extractor-pack
 mkdir extractor-pack\tools\win64 | Out-Null
-cp target\release\extractor.exe extractor-pack\tools\win64\extractor.exe
-cp target\release\autobuilder.exe extractor-pack\tools\win64\autobuilder.exe
+cp target\release\codeql-extractor-ql.exe extractor-pack\tools\win64\extractor.exe

ql/scripts/create-extractor-pack.sh

@@ -20,12 +20,11 @@ else
 fi
 
 cargo build --release
-cargo run --release --bin generator -- --dbscheme ql/src/ql.dbscheme --library ql/src/codeql_ql/ast/internal/TreeSitter.qll
+cargo run --release --bin codeql-extractor-ql -- generate --dbscheme ql/src/ql.dbscheme --library ql/src/codeql_ql/ast/internal/TreeSitter.qll
 $CODEQL_BINARY query format -i ql/src/codeql_ql/ast/internal/TreeSitter.qll
 
 rm -rf extractor-pack
 mkdir -p extractor-pack
 cp -r codeql-extractor.yml tools ql/src/ql.dbscheme ql/src/ql.dbscheme.stats extractor-pack/
 mkdir -p extractor-pack/tools/${platform}
-cp target/release/extractor extractor-pack/tools/${platform}/extractor
-cp target/release/autobuilder extractor-pack/tools/${platform}/autobuilder
+cp target/release/codeql-extractor-ql extractor-pack/tools/${platform}/extractor

ql/tools/autobuild.cmd

@@ -1,5 +1,5 @@
 @echo off
 
-type NUL && "%CODEQL_EXTRACTOR_QL_ROOT%\tools\%CODEQL_PLATFORM%\autobuilder"
+type NUL && "%CODEQL_EXTRACTOR_QL_ROOT%\tools\%CODEQL_PLATFORM%\extractor" autobuild
 
 exit /b %ERRORLEVEL%

ql/tools/autobuild.sh

@@ -1,4 +1,4 @@
 #!/bin/sh
 
-exec "${CODEQL_EXTRACTOR_QL_ROOT}/tools/${CODEQL_PLATFORM}/autobuilder"
+exec "${CODEQL_EXTRACTOR_QL_ROOT}/tools/${CODEQL_PLATFORM}/extractor" autobuild
 

ql/tools/index-files.cmd

@@ -1,6 +1,7 @@
 @echo off
 
 type NUL && "%CODEQL_EXTRACTOR_QL_ROOT%\tools\win64\extractor.exe" ^
+        extract ^
         --file-list "%1" ^
         --source-archive-dir "%CODEQL_EXTRACTOR_QL_SOURCE_ARCHIVE_DIR%" ^
         --output-dir "%CODEQL_EXTRACTOR_QL_TRAP_DIR%"

ql/tools/index-files.sh

@@ -3,6 +3,7 @@
 set -eu
 
 exec "${CODEQL_EXTRACTOR_QL_ROOT}/tools/${CODEQL_PLATFORM}/extractor" \
+        extract \
         --file-list "$1" \
         --source-archive-dir "$CODEQL_EXTRACTOR_QL_SOURCE_ARCHIVE_DIR" \
         --output-dir "$CODEQL_EXTRACTOR_QL_TRAP_DIR"

ruby/Makefile

@@ -24,7 +24,7 @@ FILES=codeql-extractor.yml\
       ql/lib/ruby.dbscheme.stats\
       ql/lib/ruby.dbscheme
 
-BIN_FILES=target/release/ruby-extractor$(EXE) target/release/ruby-autobuilder$(EXE)
+BIN_FILES=target/release/codeql-extractor-ruby$(EXE)
 
 extractor-common:
 	rm -rf build
@@ -37,15 +37,14 @@ extractor-common:
 tools: $(BIN_FILES)
 	rm -rf tools/bin
 	mkdir tools/bin
-	cp -r target/release/ruby-autobuilder$(EXE) tools/bin/autobuilder$(EXE)
-	cp -r target/release/ruby-extractor$(EXE) tools/bin/extractor$(EXE)
+	cp -r target/release/codeql-extractor-ruby$(EXE) tools/bin/extractor$(EXE)
 
 target/release/%$(EXE):
 	cd extractor && cargo build --release
 
 dbscheme:
 	cd extractor && cargo build --release
-	extractor/target/release/generator --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
+	extractor/target/release/codeql-extractor-ruby generate --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
 	codeql query format -i ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
 
 .PHONY:	extractor
@@ -63,8 +62,7 @@ extractor:	$(FILES) $(BIN_FILES)
 	cp tools/autobuild.cmd extractor-pack/tools/autobuild.cmd
 	cp ql/lib/ruby.dbscheme.stats extractor-pack/ruby.dbscheme.stats
 	cp ql/lib/ruby.dbscheme extractor-pack/ruby.dbscheme
-	cp extractor/target/release/extractor$(EXE) extractor-pack/tools/$(CODEQL_PLATFORM)/extractor$(EXE)
-	cp extractor/target/release/autobuilder$(EXE) extractor-pack/tools/$(CODEQL_PLATFORM)/autobuilder$(EXE)
+	cp extractor/target/release/codeql-extractor-ruby$(EXE) extractor-pack/tools/$(CODEQL_PLATFORM)/extractor$(EXE)
 
 test: extractor dbscheme
 	codeql test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path extractor-pack --consistency-queries ql/consistency-queries ql/test

ruby/extractor/Cargo.toml

@@ -1,5 +1,6 @@
 [package]
-name = "ruby-extractor"
+name = "codeql-extractor-ruby"
+description = "CodeQL Ruby extractor"
 version = "0.1.0"
 authors = ["GitHub"]
 edition = "2018"
@@ -10,7 +11,7 @@ edition = "2018"
 tree-sitter = "0.20"
 tree-sitter-embedded-template = { git = "https://github.com/tree-sitter/tree-sitter-embedded-template.git", rev = "203f7bd3c1bbfbd98fc19add4b8fcb213c059205" }
 tree-sitter-ruby = { git = "https://github.com/tree-sitter/tree-sitter-ruby.git", rev = "206c7077164372c596ffa8eaadb9435c28941364" }
-clap = "4.2"
+clap = { version = "4.2", features = ["derive"] }
 tracing = "0.1"
 tracing-subscriber = { version = "0.3.3", features = ["env-filter"] }
 rayon = "1.5.0"

ruby/extractor/src/autobuilder.rs

@@ -1,8 +1,13 @@
+use clap::Args;
 use std::env;
 use std::path::PathBuf;
 use std::process::Command;
 
-fn main() -> std::io::Result<()> {
+#[derive(Args)]
+// The autobuilder takes no command-line options, but this may change in the future.
+pub struct Options {}
+
+pub fn run(_: Options) -> std::io::Result<()> {
     let dist = env::var("CODEQL_DIST").expect("CODEQL_DIST not set");
     let db = env::var("CODEQL_EXTRACTOR_RUBY_WIP_DATABASE")
         .expect("CODEQL_EXTRACTOR_RUBY_WIP_DATABASE not set");

ruby/extractor/src/bin/generator.rs

@@ -1,43 +0,0 @@
-use clap::arg;
-use std::path::PathBuf;
-
-use codeql_extractor::generator::{generate, language::Language};
-
-fn main() -> std::io::Result<()> {
-    tracing_subscriber::fmt()
-        .with_target(false)
-        .without_time()
-        .with_level(true)
-        .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
-        .init();
-
-    let matches = clap::Command::new("Ruby dbscheme generator")
-        .version("1.0")
-        .author("GitHub")
-        .about("CodeQL Ruby dbscheme generator")
-        .arg(arg!(--dbscheme <FILE>                  "Path of the generated dbscheme file"))
-        .arg(arg!(--library <FILE>                   "Path of the generated QLL file"))
-        .get_matches();
-    let dbscheme_path = matches
-        .get_one::<String>("dbscheme")
-        .expect("missing --dbscheme");
-    let dbscheme_path = PathBuf::from(dbscheme_path);
-
-    let ql_library_path = matches
-        .get_one::<String>("library")
-        .expect("missing --library");
-    let ql_library_path = PathBuf::from(ql_library_path);
-
-    let languages = vec![
-        Language {
-            name: "Ruby".to_owned(),
-            node_types: tree_sitter_ruby::NODE_TYPES,
-        },
-        Language {
-            name: "Erb".to_owned(),
-            node_types: tree_sitter_embedded_template::NODE_TYPES,
-        },
-    ];
-
-    generate(languages, dbscheme_path, ql_library_path)
-}

ruby/extractor/src/extractor.rs

@@ -1,7 +1,5 @@
-#[macro_use]
-extern crate lazy_static;
-
-use clap::arg;
+use clap::Args;
+use lazy_static::lazy_static;
 use rayon::prelude::*;
 use std::borrow::Cow;
 use std::fs;
@@ -11,23 +9,22 @@ use tree_sitter::{Language, Parser, Range};
 
 use codeql_extractor::{diagnostics, extractor, file_paths, node_types, trap};
 
-lazy_static! {
-    static ref CP_NUMBER: regex::Regex = regex::Regex::new("cp([0-9]+)").unwrap();
+#[derive(Args)]
+pub struct Options {
+    /// Sets a custom source achive folder
+    #[arg(long)]
+    source_archive_dir: String,
+
+    /// Sets a custom trap folder
+    #[arg(long)]
+    output_dir: String,
+
+    /// A text file containing the paths of the files to extract
+    #[arg(long)]
+    file_list: String,
 }
 
-/// Returns the `encoding::Encoding` corresponding to the given encoding name, if one exists.
-fn encoding_from_name(encoding_name: &str) -> Option<&(dyn encoding::Encoding + Send + Sync)> {
-    match encoding::label::encoding_from_whatwg_label(encoding_name) {
-        s @ Some(_) => s,
-        None => CP_NUMBER.captures(encoding_name).and_then(|cap| {
-            encoding::label::encoding_from_windows_code_page(
-                str::parse(cap.get(1).unwrap().as_str()).unwrap(),
-            )
-        }),
-    }
-}
-
-fn main() -> std::io::Result<()> {
+pub fn run(options: Options) -> std::io::Result<()> {
     tracing_subscriber::fmt()
         .with_target(false)
         .without_time()
@@ -82,29 +79,11 @@ fn main() -> std::io::Result<()> {
         .build_global()
         .unwrap();
 
-    let matches = clap::Command::new("Ruby extractor")
-        .version("1.0")
-        .author("GitHub")
-        .about("CodeQL Ruby extractor")
-        .arg(arg!(--"source-archive-dir" <DIR> "Sets a custom source archive folder"))
-        .arg(arg!(--"output-dir" <DIR>         "Sets a custom trap folder"))
-        .arg(arg!(--"file-list" <FILE_LIST>    "A text file containing the paths of the files to extract"))
-        .get_matches();
+    let src_archive_dir = file_paths::path_from_string(&options.source_archive_dir);
 
-    let src_archive_dir = matches
-        .get_one::<String>("source-archive-dir")
-        .expect("missing --source-archive-dir");
-    let src_archive_dir = file_paths::path_from_string(src_archive_dir);
+    let trap_dir = file_paths::path_from_string(&options.output_dir);
 
-    let trap_dir = matches
-        .get_one::<String>("output-dir")
-        .expect("missing --output-dir");
-    let trap_dir = file_paths::path_from_string(&trap_dir);
-
-    let file_list = matches
-        .get_one::<String>("file-list")
-        .expect("missing --file-list");
-    let file_list = fs::File::open(file_paths::path_from_string(&file_list))?;
+    let file_list = fs::File::open(file_paths::path_from_string(&options.file_list))?;
 
     let language = tree_sitter_ruby::language();
     let erb = tree_sitter_embedded_template::language();
@@ -242,6 +221,22 @@ fn main() -> std::io::Result<()> {
     write_trap(&trap_dir, path, &trap_writer, trap_compression)
 }
 
+lazy_static! {
+    static ref CP_NUMBER: regex::Regex = regex::Regex::new("cp([0-9]+)").unwrap();
+}
+
+/// Returns the `encoding::Encoding` corresponding to the given encoding name, if one exists.
+fn encoding_from_name(encoding_name: &str) -> Option<&(dyn encoding::Encoding + Send + Sync)> {
+    match encoding::label::encoding_from_whatwg_label(encoding_name) {
+        s @ Some(_) => s,
+        None => CP_NUMBER.captures(encoding_name).and_then(|cap| {
+            encoding::label::encoding_from_windows_code_page(
+                str::parse(cap.get(1).unwrap().as_str()).unwrap(),
+            )
+        }),
+    }
+}
+
 fn write_trap(
     trap_dir: &Path,
     path: PathBuf,

ruby/extractor/src/generator.rs

@@ -0,0 +1,37 @@
+use clap::Args;
+use std::path::PathBuf;
+
+use codeql_extractor::generator::{generate, language::Language};
+
+#[derive(Args)]
+pub struct Options {
+    /// Path of the generated dbscheme file
+    #[arg(long)]
+    dbscheme: PathBuf,
+
+    /// Path of the generated QLL file
+    #[arg(long)]
+    library: PathBuf,
+}
+
+pub fn run(options: Options) -> std::io::Result<()> {
+    tracing_subscriber::fmt()
+        .with_target(false)
+        .without_time()
+        .with_level(true)
+        .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
+        .init();
+
+    let languages = vec![
+        Language {
+            name: "Ruby".to_owned(),
+            node_types: tree_sitter_ruby::NODE_TYPES,
+        },
+        Language {
+            name: "Erb".to_owned(),
+            node_types: tree_sitter_embedded_template::NODE_TYPES,
+        },
+    ];
+
+    generate(languages, options.dbscheme, options.library)
+}

ruby/extractor/src/main.rs

@@ -0,0 +1,23 @@
+use clap::Parser;
+
+mod autobuilder;
+mod extractor;
+mod generator;
+
+#[derive(Parser)]
+#[command(author, version, about)]
+enum Cli {
+    Extract(extractor::Options),
+    Generate(generator::Options),
+    Autobuild(autobuilder::Options),
+}
+
+fn main() -> std::io::Result<()> {
+    let cli = Cli::parse();
+
+    match cli {
+        Cli::Extract(options) => extractor::run(options),
+        Cli::Generate(options) => generator::run(options),
+        Cli::Autobuild(options) => autobuilder::run(options),
+    }
+}

ruby/scripts/create-extractor-pack.ps1

@@ -2,7 +2,7 @@ cd extractor
 cargo build --release
 cd ..
 
-extractor\target\release\generator --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
+extractor\target\release\codeql-extractor-ruby -- generate --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
 
 codeql query format -i ql\lib\codeql/ruby\ast\internal\TreeSitter.qll
 
@@ -12,5 +12,4 @@ cp codeql-extractor.yml, ql\lib\ruby.dbscheme, ql\lib\ruby.dbscheme.stats extrac
 cp -Recurse tools extractor-pack
 cp -Recurse downgrades extractor-pack
 mkdir extractor-pack\tools\win64 | Out-Null
-cp extractor\target\release\extractor.exe extractor-pack\tools\win64\extractor.exe
-cp extractor\target\release\autobuilder.exe extractor-pack\tools\win64\autobuilder.exe
+cp extractor\target\release\codeql-extractor-ruby.exe extractor-pack\tools\win64\extractor.exe

ruby/scripts/create-extractor-pack.sh

@@ -22,7 +22,7 @@ if [[ "$CARGO" == "cross" ]]; then
   BIN_DIR=extractor/target/x86_64-unknown-linux-gnu/release
 fi
 
-"$BIN_DIR/generator" --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
+"$BIN_DIR/codeql-extractor-ruby" generate --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
 
 codeql query format -i ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
 
@@ -30,5 +30,4 @@ rm -rf extractor-pack
 mkdir -p extractor-pack
 cp -r codeql-extractor.yml downgrades tools ql/lib/ruby.dbscheme ql/lib/ruby.dbscheme.stats extractor-pack/
 mkdir -p extractor-pack/tools/${platform}
-cp "$BIN_DIR/extractor" extractor-pack/tools/${platform}/extractor
-cp "$BIN_DIR/autobuilder" extractor-pack/tools/${platform}/autobuilder
+cp "$BIN_DIR/codeql-extractor-ruby" extractor-pack/tools/${platform}/extractor

ruby/tools/autobuild.cmd

@@ -1,5 +1,5 @@
 @echo off
 
-type NUL && "%CODEQL_EXTRACTOR_RUBY_ROOT%\tools\%CODEQL_PLATFORM%\autobuilder"
+type NUL && "%CODEQL_EXTRACTOR_RUBY_ROOT%\tools\%CODEQL_PLATFORM%\extractor" autobuild
 
 exit /b %ERRORLEVEL%

ruby/tools/autobuild.sh

@@ -1,3 +1,3 @@
 #!/bin/sh
 
-exec "${CODEQL_EXTRACTOR_RUBY_ROOT}/tools/${CODEQL_PLATFORM}/autobuilder"
+exec "${CODEQL_EXTRACTOR_RUBY_ROOT}/tools/${CODEQL_PLATFORM}/extractor" autobuild

ruby/tools/index-files.cmd

@@ -1,6 +1,7 @@
 @echo off
 
 type NUL && "%CODEQL_EXTRACTOR_RUBY_ROOT%\tools\win64\extractor.exe" ^
+        extract ^
         --file-list "%1" ^
         --source-archive-dir "%CODEQL_EXTRACTOR_RUBY_SOURCE_ARCHIVE_DIR%" ^
         --output-dir "%CODEQL_EXTRACTOR_RUBY_TRAP_DIR%"

ruby/tools/index-files.sh

@@ -3,6 +3,7 @@
 set -eu
 
 exec "${CODEQL_EXTRACTOR_RUBY_ROOT}/tools/${CODEQL_PLATFORM}/extractor" \
+        extract \
         --file-list "$1" \
         --source-archive-dir "$CODEQL_EXTRACTOR_RUBY_SOURCE_ARCHIVE_DIR" \
         --output-dir "$CODEQL_EXTRACTOR_RUBY_TRAP_DIR"