Add support for LGTM_INDEX_FILTERS environment variable

* re-implement autobuilder script in Rust
* add additional --include/--exclude flags based on LGTM_INDEX_FILTERS
  environment variable

.github/workflows/build.yml

@@ -62,6 +62,8 @@ jobs:
         with:
           name: extractor-${{ matrix.os }}
           path: |
+            target/release/ruby-autobuilder
+            target/release/ruby-autobuilder.exe
             target/release/ruby-extractor
             target/release/ruby-extractor.exe
           retention-days: 1
@@ -125,10 +127,13 @@ jobs:
           mkdir -p ruby
           cp -r codeql-extractor.yml tools ql/src/ruby.dbscheme.stats ruby/
           mkdir -p ruby/tools/{linux64,osx64,win64}
+          cp linux64/ruby-autobuilder ruby/tools/linux64/autobuilder
+          cp osx64/ruby-autobuilder ruby/tools/osx64/autobuilder
+          cp win64/ruby-autobuilder.exe ruby/tools/win64/autobuilder.exe
           cp linux64/ruby-extractor ruby/tools/linux64/extractor
           cp osx64/ruby-extractor ruby/tools/osx64/extractor
           cp win64/ruby-extractor.exe ruby/tools/win64/extractor.exe
-          chmod +x ruby/tools/{linux64,osx64}/extractor
+          chmod +x ruby/tools/{linux64,osx64}/{autobuilder,extractor}
           zip -rq codeql-ruby.zip ruby
       - uses: actions/upload-artifact@v2
         with:

Cargo.lock

@@ -371,6 +371,10 @@ version = "0.6.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581"
 
+[[package]]
+name = "ruby-autobuilder"
+version = "0.1.0"
+
 [[package]]
 name = "ruby-extractor"
 version = "0.1.0"

Cargo.toml

@@ -1,5 +1,6 @@
 [workspace]
 members = [
+    "autobuilder",
     "extractor",
     "generator",
     "node-types",

autobuilder/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+name = "ruby-autobuilder"
+version = "0.1.0"
+authors = ["GitHub"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]

autobuilder/src/main.rs

@@ -0,0 +1,35 @@
+use std::env;
+use std::process::Command;
+
+fn main() -> std::io::Result<()> {
+    let dist = env::var("CODEQL_DIST").expect("CODEQL_DIST not set");
+    let db = env::var("CODEQL_EXTRACTOR_RUBY_WIP_DATABASE")
+        .expect("CODEQL_EXTRACTOR_RUBY_WIP_DATABASE not set");
+    let ext = if env::consts::OS == "windows" {
+        ".exe"
+    } else {
+        ""
+    };
+    let mut cmd = Command::new(format!("{}/codeql{}", dist, ext));
+    &cmd.arg("database")
+        .arg("index-files")
+        .arg("--include-extension=.rb")
+        .arg("--include-extension=.erb")
+        .arg("--size-limit=5m")
+        .arg("--language=ruby")
+        .arg("--working-dir=.")
+        .arg(db);
+
+    for line in env::var("LGTM_INDEX_FILTERS")
+        .unwrap_or_default()
+        .split("\n")
+    {
+        if line.starts_with("include:") {
+            &cmd.arg("--include").arg(&line[8..]);
+        } else if line.starts_with("exclude:") {
+            &cmd.arg("--exclude").arg(&line[8..]);
+        }
+    }
+    let exit = &cmd.spawn()?.wait()?;
+    std::process::exit(exit.code().unwrap_or(1))
+}

create-extractor-pack.ps1

@@ -9,3 +9,4 @@ cp codeql-extractor.yml, ql\src\ruby.dbscheme, ql\src\ruby.dbscheme.stats extrac
 cp -Recurse tools extractor-pack
 mkdir extractor-pack\tools\win64 | Out-Null
 cp target\release\ruby-extractor.exe extractor-pack\tools\win64\extractor.exe
+cp target\release\ruby-autobuilder.exe extractor-pack\tools\win64\autobuilder.exe

create-extractor-pack.sh

@@ -20,3 +20,4 @@ mkdir -p extractor-pack
 cp -r codeql-extractor.yml tools ql/src/ruby.dbscheme ql/src/ruby.dbscheme.stats extractor-pack/
 mkdir -p extractor-pack/tools/${platform}
 cp target/release/ruby-extractor extractor-pack/tools/${platform}/extractor
+cp target/release/ruby-autobuilder extractor-pack/tools/${platform}/autobuilder

tools/autobuild.cmd

@@ -1,10 +1,5 @@
 @echo off
 
-type NUL && "%CODEQL_DIST%\codeql.exe" database index-files ^
-    --include-extension=.rb ^
-    --include-extension=.erb ^
-    --size-limit=5m ^
-    --language=ruby ^
-    "%CODEQL_EXTRACTOR_RUBY_WIP_DATABASE%"
+type NUL && "%CODEQL_EXTRACTOR_RUBY_ROOT%\tools\%CODEQL_PLATFORM\autobuilder"
 
 exit /b %ERRORLEVEL%

tools/autobuild.sh

@@ -1,11 +1,3 @@
 #!/bin/sh
 
-set -eu
-
-exec "${CODEQL_DIST}/codeql" database index-files \
-    --include-extension=.rb \
-    --include-extension=.erb \
-    --size-limit=5m \
-    --language=ruby \
-    --working-dir=.\
-    "$CODEQL_EXTRACTOR_RUBY_WIP_DATABASE"
+exec "${CODEQL_EXTRACTOR_RUBY_ROOT}/tools/${CODEQL_PLATFORM}/autobuilder"