Merge pull request #12786 from hmac/merge-extractor-binaries

Ruby/QL: Merge extractor binaries

ruby/Makefile

@@ -24,7 +24,7 @@ FILES=codeql-extractor.yml\
       ql/lib/ruby.dbscheme.stats\
       ql/lib/ruby.dbscheme
 
-BIN_FILES=target/release/ruby-extractor$(EXE) target/release/ruby-autobuilder$(EXE)
+BIN_FILES=target/release/codeql-extractor-ruby$(EXE)
 
 extractor-common:
 	rm -rf build
@@ -37,15 +37,14 @@ extractor-common:
 tools: $(BIN_FILES)
 	rm -rf tools/bin
 	mkdir tools/bin
-	cp -r target/release/ruby-autobuilder$(EXE) tools/bin/autobuilder$(EXE)
-	cp -r target/release/ruby-extractor$(EXE) tools/bin/extractor$(EXE)
+	cp -r target/release/codeql-extractor-ruby$(EXE) tools/bin/extractor$(EXE)
 
 target/release/%$(EXE):
 	cd extractor && cargo build --release
 
 dbscheme:
 	cd extractor && cargo build --release
-	extractor/target/release/generator --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
+	extractor/target/release/codeql-extractor-ruby generate --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
 	codeql query format -i ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
 
 .PHONY:	extractor
@@ -63,8 +62,7 @@ extractor:	$(FILES) $(BIN_FILES)
 	cp tools/autobuild.cmd extractor-pack/tools/autobuild.cmd
 	cp ql/lib/ruby.dbscheme.stats extractor-pack/ruby.dbscheme.stats
 	cp ql/lib/ruby.dbscheme extractor-pack/ruby.dbscheme
-	cp extractor/target/release/extractor$(EXE) extractor-pack/tools/$(CODEQL_PLATFORM)/extractor$(EXE)
-	cp extractor/target/release/autobuilder$(EXE) extractor-pack/tools/$(CODEQL_PLATFORM)/autobuilder$(EXE)
+	cp extractor/target/release/codeql-extractor-ruby$(EXE) extractor-pack/tools/$(CODEQL_PLATFORM)/extractor$(EXE)
 
 test: extractor dbscheme
 	codeql test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path extractor-pack --consistency-queries ql/consistency-queries ql/test

ruby/extractor/Cargo.toml

@@ -1,5 +1,6 @@
 [package]
-name = "ruby-extractor"
+name = "codeql-extractor-ruby"
+description = "CodeQL Ruby extractor"
 version = "0.1.0"
 authors = ["GitHub"]
 edition = "2018"
@@ -10,7 +11,7 @@ edition = "2018"
 tree-sitter = "0.20"
 tree-sitter-embedded-template = { git = "https://github.com/tree-sitter/tree-sitter-embedded-template.git", rev = "203f7bd3c1bbfbd98fc19add4b8fcb213c059205" }
 tree-sitter-ruby = { git = "https://github.com/tree-sitter/tree-sitter-ruby.git", rev = "206c7077164372c596ffa8eaadb9435c28941364" }
-clap = "4.2"
+clap = { version = "4.2", features = ["derive"] }
 tracing = "0.1"
 tracing-subscriber = { version = "0.3.3", features = ["env-filter"] }
 rayon = "1.5.0"

ruby/extractor/src/autobuilder.rs

@@ -1,8 +1,13 @@
+use clap::Args;
 use std::env;
 use std::path::PathBuf;
 use std::process::Command;
 
-fn main() -> std::io::Result<()> {
+#[derive(Args)]
+// The autobuilder takes no command-line options, but this may change in the future.
+pub struct Options {}
+
+pub fn run(_: Options) -> std::io::Result<()> {
     let dist = env::var("CODEQL_DIST").expect("CODEQL_DIST not set");
     let db = env::var("CODEQL_EXTRACTOR_RUBY_WIP_DATABASE")
         .expect("CODEQL_EXTRACTOR_RUBY_WIP_DATABASE not set");

ruby/extractor/src/bin/generator.rs

@@ -1,43 +0,0 @@
-use clap::arg;
-use std::path::PathBuf;
-
-use codeql_extractor::generator::{generate, language::Language};
-
-fn main() -> std::io::Result<()> {
-    tracing_subscriber::fmt()
-        .with_target(false)
-        .without_time()
-        .with_level(true)
-        .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
-        .init();
-
-    let matches = clap::Command::new("Ruby dbscheme generator")
-        .version("1.0")
-        .author("GitHub")
-        .about("CodeQL Ruby dbscheme generator")
-        .arg(arg!(--dbscheme <FILE>                  "Path of the generated dbscheme file"))
-        .arg(arg!(--library <FILE>                   "Path of the generated QLL file"))
-        .get_matches();
-    let dbscheme_path = matches
-        .get_one::<String>("dbscheme")
-        .expect("missing --dbscheme");
-    let dbscheme_path = PathBuf::from(dbscheme_path);
-
-    let ql_library_path = matches
-        .get_one::<String>("library")
-        .expect("missing --library");
-    let ql_library_path = PathBuf::from(ql_library_path);
-
-    let languages = vec![
-        Language {
-            name: "Ruby".to_owned(),
-            node_types: tree_sitter_ruby::NODE_TYPES,
-        },
-        Language {
-            name: "Erb".to_owned(),
-            node_types: tree_sitter_embedded_template::NODE_TYPES,
-        },
-    ];
-
-    generate(languages, dbscheme_path, ql_library_path)
-}

ruby/extractor/src/extractor.rs

@@ -1,7 +1,5 @@
-#[macro_use]
-extern crate lazy_static;
-
-use clap::arg;
+use clap::Args;
+use lazy_static::lazy_static;
 use rayon::prelude::*;
 use std::borrow::Cow;
 use std::fs;
@@ -11,23 +9,22 @@ use tree_sitter::{Language, Parser, Range};
 
 use codeql_extractor::{diagnostics, extractor, file_paths, node_types, trap};
 
-lazy_static! {
-    static ref CP_NUMBER: regex::Regex = regex::Regex::new("cp([0-9]+)").unwrap();
+#[derive(Args)]
+pub struct Options {
+    /// Sets a custom source achive folder
+    #[arg(long)]
+    source_archive_dir: String,
+
+    /// Sets a custom trap folder
+    #[arg(long)]
+    output_dir: String,
+
+    /// A text file containing the paths of the files to extract
+    #[arg(long)]
+    file_list: String,
 }
 
-/// Returns the `encoding::Encoding` corresponding to the given encoding name, if one exists.
-fn encoding_from_name(encoding_name: &str) -> Option<&(dyn encoding::Encoding + Send + Sync)> {
-    match encoding::label::encoding_from_whatwg_label(encoding_name) {
-        s @ Some(_) => s,
-        None => CP_NUMBER.captures(encoding_name).and_then(|cap| {
-            encoding::label::encoding_from_windows_code_page(
-                str::parse(cap.get(1).unwrap().as_str()).unwrap(),
-            )
-        }),
-    }
-}
-
-fn main() -> std::io::Result<()> {
+pub fn run(options: Options) -> std::io::Result<()> {
     tracing_subscriber::fmt()
         .with_target(false)
         .without_time()
@@ -82,29 +79,11 @@ fn main() -> std::io::Result<()> {
         .build_global()
         .unwrap();
 
-    let matches = clap::Command::new("Ruby extractor")
-        .version("1.0")
-        .author("GitHub")
-        .about("CodeQL Ruby extractor")
-        .arg(arg!(--"source-archive-dir" <DIR> "Sets a custom source archive folder"))
-        .arg(arg!(--"output-dir" <DIR>         "Sets a custom trap folder"))
-        .arg(arg!(--"file-list" <FILE_LIST>    "A text file containing the paths of the files to extract"))
-        .get_matches();
+    let src_archive_dir = file_paths::path_from_string(&options.source_archive_dir);
 
-    let src_archive_dir = matches
-        .get_one::<String>("source-archive-dir")
-        .expect("missing --source-archive-dir");
-    let src_archive_dir = file_paths::path_from_string(src_archive_dir);
+    let trap_dir = file_paths::path_from_string(&options.output_dir);
 
-    let trap_dir = matches
-        .get_one::<String>("output-dir")
-        .expect("missing --output-dir");
-    let trap_dir = file_paths::path_from_string(&trap_dir);
-
-    let file_list = matches
-        .get_one::<String>("file-list")
-        .expect("missing --file-list");
-    let file_list = fs::File::open(file_paths::path_from_string(&file_list))?;
+    let file_list = fs::File::open(file_paths::path_from_string(&options.file_list))?;
 
     let language = tree_sitter_ruby::language();
     let erb = tree_sitter_embedded_template::language();
@@ -242,6 +221,22 @@ fn main() -> std::io::Result<()> {
     write_trap(&trap_dir, path, &trap_writer, trap_compression)
 }
 
+lazy_static! {
+    static ref CP_NUMBER: regex::Regex = regex::Regex::new("cp([0-9]+)").unwrap();
+}
+
+/// Returns the `encoding::Encoding` corresponding to the given encoding name, if one exists.
+fn encoding_from_name(encoding_name: &str) -> Option<&(dyn encoding::Encoding + Send + Sync)> {
+    match encoding::label::encoding_from_whatwg_label(encoding_name) {
+        s @ Some(_) => s,
+        None => CP_NUMBER.captures(encoding_name).and_then(|cap| {
+            encoding::label::encoding_from_windows_code_page(
+                str::parse(cap.get(1).unwrap().as_str()).unwrap(),
+            )
+        }),
+    }
+}
+
 fn write_trap(
     trap_dir: &Path,
     path: PathBuf,

ruby/extractor/src/generator.rs

@@ -0,0 +1,37 @@
+use clap::Args;
+use std::path::PathBuf;
+
+use codeql_extractor::generator::{generate, language::Language};
+
+#[derive(Args)]
+pub struct Options {
+    /// Path of the generated dbscheme file
+    #[arg(long)]
+    dbscheme: PathBuf,
+
+    /// Path of the generated QLL file
+    #[arg(long)]
+    library: PathBuf,
+}
+
+pub fn run(options: Options) -> std::io::Result<()> {
+    tracing_subscriber::fmt()
+        .with_target(false)
+        .without_time()
+        .with_level(true)
+        .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
+        .init();
+
+    let languages = vec![
+        Language {
+            name: "Ruby".to_owned(),
+            node_types: tree_sitter_ruby::NODE_TYPES,
+        },
+        Language {
+            name: "Erb".to_owned(),
+            node_types: tree_sitter_embedded_template::NODE_TYPES,
+        },
+    ];
+
+    generate(languages, options.dbscheme, options.library)
+}

ruby/extractor/src/main.rs

@@ -0,0 +1,23 @@
+use clap::Parser;
+
+mod autobuilder;
+mod extractor;
+mod generator;
+
+#[derive(Parser)]
+#[command(author, version, about)]
+enum Cli {
+    Extract(extractor::Options),
+    Generate(generator::Options),
+    Autobuild(autobuilder::Options),
+}
+
+fn main() -> std::io::Result<()> {
+    let cli = Cli::parse();
+
+    match cli {
+        Cli::Extract(options) => extractor::run(options),
+        Cli::Generate(options) => generator::run(options),
+        Cli::Autobuild(options) => autobuilder::run(options),
+    }
+}

ruby/scripts/create-extractor-pack.ps1

@@ -2,7 +2,7 @@ cd extractor
 cargo build --release
 cd ..
 
-extractor\target\release\generator --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
+extractor\target\release\codeql-extractor-ruby -- generate --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
 
 codeql query format -i ql\lib\codeql/ruby\ast\internal\TreeSitter.qll
 
@@ -12,5 +12,4 @@ cp codeql-extractor.yml, ql\lib\ruby.dbscheme, ql\lib\ruby.dbscheme.stats extrac
 cp -Recurse tools extractor-pack
 cp -Recurse downgrades extractor-pack
 mkdir extractor-pack\tools\win64 | Out-Null
-cp extractor\target\release\extractor.exe extractor-pack\tools\win64\extractor.exe
-cp extractor\target\release\autobuilder.exe extractor-pack\tools\win64\autobuilder.exe
+cp extractor\target\release\codeql-extractor-ruby.exe extractor-pack\tools\win64\extractor.exe

ruby/scripts/create-extractor-pack.sh

@@ -22,7 +22,7 @@ if [[ "$CARGO" == "cross" ]]; then
   BIN_DIR=extractor/target/x86_64-unknown-linux-gnu/release
 fi
 
-"$BIN_DIR/generator" --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
+"$BIN_DIR/codeql-extractor-ruby" generate --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
 
 codeql query format -i ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
 
@@ -30,5 +30,4 @@ rm -rf extractor-pack
 mkdir -p extractor-pack
 cp -r codeql-extractor.yml downgrades tools ql/lib/ruby.dbscheme ql/lib/ruby.dbscheme.stats extractor-pack/
 mkdir -p extractor-pack/tools/${platform}
-cp "$BIN_DIR/extractor" extractor-pack/tools/${platform}/extractor
-cp "$BIN_DIR/autobuilder" extractor-pack/tools/${platform}/autobuilder
+cp "$BIN_DIR/codeql-extractor-ruby" extractor-pack/tools/${platform}/extractor

ruby/tools/autobuild.cmd

@@ -1,5 +1,5 @@
 @echo off
 
-type NUL && "%CODEQL_EXTRACTOR_RUBY_ROOT%\tools\%CODEQL_PLATFORM%\autobuilder"
+type NUL && "%CODEQL_EXTRACTOR_RUBY_ROOT%\tools\%CODEQL_PLATFORM%\extractor" autobuild
 
 exit /b %ERRORLEVEL%

ruby/tools/autobuild.sh

@@ -1,3 +1,3 @@
 #!/bin/sh
 
-exec "${CODEQL_EXTRACTOR_RUBY_ROOT}/tools/${CODEQL_PLATFORM}/autobuilder"
+exec "${CODEQL_EXTRACTOR_RUBY_ROOT}/tools/${CODEQL_PLATFORM}/extractor" autobuild

ruby/tools/index-files.cmd

@@ -1,6 +1,7 @@
 @echo off
 
 type NUL && "%CODEQL_EXTRACTOR_RUBY_ROOT%\tools\win64\extractor.exe" ^
+        extract ^
         --file-list "%1" ^
         --source-archive-dir "%CODEQL_EXTRACTOR_RUBY_SOURCE_ARCHIVE_DIR%" ^
         --output-dir "%CODEQL_EXTRACTOR_RUBY_TRAP_DIR%"

ruby/tools/index-files.sh

@@ -3,6 +3,7 @@
 set -eu
 
 exec "${CODEQL_EXTRACTOR_RUBY_ROOT}/tools/${CODEQL_PLATFORM}/extractor" \
+        extract \
         --file-list "$1" \
         --source-archive-dir "$CODEQL_EXTRACTOR_RUBY_SOURCE_ARCHIVE_DIR" \
         --output-dir "$CODEQL_EXTRACTOR_RUBY_TRAP_DIR"