QL for QL: add autobuilder that respects LGTM_INDEX_FILTERS

.github/workflows/ql-for-ql-build.yml

@@ -77,6 +77,8 @@ jobs:
         with:
           name: extractor-ubuntu-latest
           path: |
+            ql/target/release/ql-autobuilder
+            ql/target/release/ql-autobuilder.exe
             ql/target/release/ql-extractor
             ql/target/release/ql-extractor.exe
           retention-days: 1
@@ -101,6 +103,10 @@ jobs:
           unzip query-pack-zip/*.zip -d pack
           cp -r ql/codeql-extractor.yml ql/tools ql/ql/src/ql.dbscheme.stats pack/
           mkdir -p pack/tools/linux64
+          if [[ -f linux64/ql-autobuilder ]]; then
+            cp linux64/ql-autobuilder pack/tools/linux64/autobuilder
+            chmod +x pack/tools/linux64/autobuilder
+          fi
           if [[ -f linux64/ql-extractor ]]; then
             cp linux64/ql-extractor pack/tools/linux64/extractor
             chmod +x pack/tools/linux64/extractor
@@ -112,13 +118,13 @@ jobs:
           name: codeql-ql-pack
           path: codeql-ql.zip
           retention-days: 1
-  analyze: 
+  analyze:
     runs-on: ubuntu-latest
-    
+
-    needs: 
+    needs:
       - package
-    
+
-    steps: 
+    steps:
       - name: Download pack
         uses: actions/download-artifact@v2
         with:
@@ -149,4 +155,3 @@ jobs:
 
       - name: Perform CodeQL Analysis
         uses: github/codeql-action/analyze@esbena/ql
-

ql/Cargo.toml

@@ -1,5 +1,6 @@
 [workspace]
 members = [
+    "autobuilder",
     "extractor",
     "generator",
     "node-types",

ql/autobuilder/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+name = "ql-autobuilder"
+version = "0.1.0"
+authors = ["GitHub"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]

ql/autobuilder/src/main.rs

@@ -0,0 +1,39 @@
+use std::env;
+use std::path::PathBuf;
+use std::process::Command;
+
+fn main() -> std::io::Result<()> {
+    let dist = env::var("CODEQL_DIST").expect("CODEQL_DIST not set");
+    let db = env::var("CODEQL_EXTRACTOR_QL_WIP_DATABASE")
+        .expect("CODEQL_EXTRACTOR_QL_WIP_DATABASE not set");
+    let codeql = if env::consts::OS == "windows" {
+        "codeql.exe"
+    } else {
+        "codeql"
+    };
+    let codeql: PathBuf = [&dist, codeql].iter().collect();
+    let mut cmd = Command::new(codeql);
+    cmd.arg("database")
+        .arg("index-files")
+        .arg("--include-extension=.ql")
+        .arg("--include-extension=.qll")
+        .arg("--include-extension=.dbscheme")
+        .arg("--include=**/qlpack.yml")
+        .arg("--size-limit=5m")
+        .arg("--language=ql")
+        .arg("--working-dir=.")
+        .arg(db);
+
+    for line in env::var("LGTM_INDEX_FILTERS")
+        .unwrap_or_default()
+        .split('\n')
+    {
+        if let Some(stripped) = line.strip_prefix("include:") {
+            cmd.arg("--include").arg(stripped);
+        } else if let Some(stripped) = line.strip_prefix("exclude:") {
+            cmd.arg("--exclude").arg(stripped);
+        }
+    }
+    let exit = &cmd.spawn()?.wait()?;
+    std::process::exit(exit.code().unwrap_or(1))
+}

ql/create-extractor-pack.ps1

@@ -11,3 +11,4 @@ cp codeql-extractor.yml, ql\src\ql.dbscheme, ql\src\ql.dbscheme.stats extractor-
 cp -Recurse tools extractor-pack
 mkdir extractor-pack\tools\win64 | Out-Null
 cp target\release\ql-extractor.exe extractor-pack\tools\win64\extractor.exe
+cp target\release\ql-autobuilder.exe extractor-pack\tools\win64\autobuilder.exe

ql/create-extractor-pack.sh

@@ -20,3 +20,4 @@ mkdir -p extractor-pack
 cp -r codeql-extractor.yml tools ql/src/ql.dbscheme ql/src/ql.dbscheme.stats extractor-pack/
 mkdir -p extractor-pack/tools/${platform}
 cp target/release/ql-extractor extractor-pack/tools/${platform}/extractor
+cp target/release/ql-autobuilder extractor-pack/tools/${platform}/autobuilder

ql/extractor/Cargo.toml

@@ -13,7 +13,7 @@ tree-sitter = "0.19"
 tree-sitter-ql = { git = "https://github.com/tausbn/tree-sitter-ql.git", rev = "36bdc0eae196f9833182ce3f8932be63534121b3" }
 clap = "2.33"
 tracing = "0.1"
-tracing-subscriber = { version = "0.2", features = ["env-filter"] }
+tracing-subscriber = { version = "0.3.3", features = ["env-filter"] }
 rayon = "1.5.0"
 num_cpus = "1.13.0"
 regex = "1.4.3"

ql/generator/Cargo.toml

@@ -10,5 +10,5 @@ edition = "2018"
 clap = "2.33"
 node-types = { path = "../node-types" }
 tracing = "0.1"
-tracing-subscriber = { version = "0.2", features = ["env-filter"] }
+tracing-subscriber = { version = "0.3.3", features = ["env-filter"] }
 tree-sitter-ql = { git = "https://github.com/tausbn/tree-sitter-ql.git", rev = "36bdc0eae196f9833182ce3f8932be63534121b3" }

ql/node-types/Cargo.toml

@@ -8,4 +8,4 @@ edition = "2018"
 
 [dependencies]
 serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0"
+serde_json = "1.0"

ql/tools/autobuild.cmd

@@ -1,12 +1,5 @@
 @echo off
 
-type NUL && "%CODEQL_DIST%\codeql.exe" database index-files ^
+type NUL && "%CODEQL_EXTRACTOR_QL_ROOT%\tools\%CODEQL_PLATFORM%\autobuilder"
-    --include-extension=.ql ^
-    --include-extension=.qll ^
-    --include-extension=.dbscheme ^
-    --include=**/qlpack.yml ^
-    --size-limit=5m ^
-    --language=ql ^
-    "%CODEQL_EXTRACTOR_QL_WIP_DATABASE%"
 
 exit /b %ERRORLEVEL%

ql/tools/autobuild.sh

@@ -1,13 +1,4 @@
 #!/bin/sh
 
-set -eu
+exec "${CODEQL_EXTRACTOR_QL_ROOT}/tools/${CODEQL_PLATFORM}/autobuilder"
 
-exec "${CODEQL_DIST}/codeql" database index-files \
-    --include-extension=.ql \
-    --include-extension=.qll \
-    --include-extension=.dbscheme \
-    --include=**/qlpack.yml \
-    --size-limit=5m \
-    --language=ql \
-    --working-dir=.\
-    "$CODEQL_EXTRACTOR_QL_WIP_DATABASE"