diff --git a/.github/workflows/ql-for-ql-build.yml b/.github/workflows/ql-for-ql-build.yml index 062ddb2b6d6..b3bc3de7051 100644 --- a/.github/workflows/ql-for-ql-build.yml +++ b/.github/workflows/ql-for-ql-build.yml @@ -77,6 +77,8 @@ jobs: with: name: extractor-ubuntu-latest path: | + ql/target/release/ql-autobuilder + ql/target/release/ql-autobuilder.exe ql/target/release/ql-extractor ql/target/release/ql-extractor.exe retention-days: 1 @@ -101,6 +103,10 @@ jobs: unzip query-pack-zip/*.zip -d pack cp -r ql/codeql-extractor.yml ql/tools ql/ql/src/ql.dbscheme.stats pack/ mkdir -p pack/tools/linux64 + if [[ -f linux64/ql-autobuilder ]]; then + cp linux64/ql-autobuilder pack/tools/linux64/autobuilder + chmod +x pack/tools/linux64/autobuilder + fi if [[ -f linux64/ql-extractor ]]; then cp linux64/ql-extractor pack/tools/linux64/extractor chmod +x pack/tools/linux64/extractor @@ -112,13 +118,13 @@ jobs: name: codeql-ql-pack path: codeql-ql.zip retention-days: 1 - analyze: + analyze: runs-on: ubuntu-latest - - needs: + + needs: - package - - steps: + + steps: - name: Download pack uses: actions/download-artifact@v2 with: @@ -149,4 +155,3 @@ jobs: - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@esbena/ql - diff --git a/ql/Cargo.lock b/ql/Cargo.lock index 16626b5d836..1a49563fcb8 100644 Binary files a/ql/Cargo.lock and b/ql/Cargo.lock differ diff --git a/ql/Cargo.toml b/ql/Cargo.toml index fe2bfbea7ac..7acbd36afa6 100644 --- a/ql/Cargo.toml +++ b/ql/Cargo.toml @@ -1,5 +1,6 @@ [workspace] members = [ + "autobuilder", "extractor", "generator", "node-types", diff --git a/ql/autobuilder/Cargo.toml b/ql/autobuilder/Cargo.toml new file mode 100644 index 00000000000..ac39f025ade --- /dev/null +++ b/ql/autobuilder/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "ql-autobuilder" +version = "0.1.0" +authors = ["GitHub"] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/ql/autobuilder/src/main.rs b/ql/autobuilder/src/main.rs new file mode 100644 index 00000000000..961b2b7506c --- /dev/null +++ b/ql/autobuilder/src/main.rs @@ -0,0 +1,39 @@ +use std::env; +use std::path::PathBuf; +use std::process::Command; + +fn main() -> std::io::Result<()> { + let dist = env::var("CODEQL_DIST").expect("CODEQL_DIST not set"); + let db = env::var("CODEQL_EXTRACTOR_QL_WIP_DATABASE") + .expect("CODEQL_EXTRACTOR_QL_WIP_DATABASE not set"); + let codeql = if env::consts::OS == "windows" { + "codeql.exe" + } else { + "codeql" + }; + let codeql: PathBuf = [&dist, codeql].iter().collect(); + let mut cmd = Command::new(codeql); + cmd.arg("database") + .arg("index-files") + .arg("--include-extension=.ql") + .arg("--include-extension=.qll") + .arg("--include-extension=.dbscheme") + .arg("--include=**/qlpack.yml") + .arg("--size-limit=5m") + .arg("--language=ql") + .arg("--working-dir=.") + .arg(db); + + for line in env::var("LGTM_INDEX_FILTERS") + .unwrap_or_default() + .split('\n') + { + if let Some(stripped) = line.strip_prefix("include:") { + cmd.arg("--include").arg(stripped); + } else if let Some(stripped) = line.strip_prefix("exclude:") { + cmd.arg("--exclude").arg(stripped); + } + } + let exit = &cmd.spawn()?.wait()?; + std::process::exit(exit.code().unwrap_or(1)) +} diff --git a/ql/create-extractor-pack.ps1 b/ql/create-extractor-pack.ps1 index 71e7beec72f..65b4dfc348a 100644 --- a/ql/create-extractor-pack.ps1 +++ b/ql/create-extractor-pack.ps1 @@ -11,3 +11,4 @@ cp codeql-extractor.yml, ql\src\ql.dbscheme, ql\src\ql.dbscheme.stats extractor- cp -Recurse tools extractor-pack mkdir extractor-pack\tools\win64 | Out-Null cp target\release\ql-extractor.exe extractor-pack\tools\win64\extractor.exe +cp target\release\ql-autobuilder.exe extractor-pack\tools\win64\autobuilder.exe diff --git a/ql/create-extractor-pack.sh b/ql/create-extractor-pack.sh index 5e45c41f819..43577e5c448 100755 --- a/ql/create-extractor-pack.sh +++ b/ql/create-extractor-pack.sh @@ -20,3 +20,4 @@ mkdir -p extractor-pack cp -r codeql-extractor.yml tools ql/src/ql.dbscheme ql/src/ql.dbscheme.stats extractor-pack/ mkdir -p extractor-pack/tools/${platform} cp target/release/ql-extractor extractor-pack/tools/${platform}/extractor +cp target/release/ql-autobuilder extractor-pack/tools/${platform}/autobuilder diff --git a/ql/extractor/Cargo.toml b/ql/extractor/Cargo.toml index 3ccf1ce3430..cafaef144a5 100644 --- a/ql/extractor/Cargo.toml +++ b/ql/extractor/Cargo.toml @@ -13,7 +13,7 @@ tree-sitter = "0.19" tree-sitter-ql = { git = "https://github.com/tausbn/tree-sitter-ql.git", rev = "36bdc0eae196f9833182ce3f8932be63534121b3" } clap = "2.33" tracing = "0.1" -tracing-subscriber = { version = "0.2", features = ["env-filter"] } +tracing-subscriber = { version = "0.3.3", features = ["env-filter"] } rayon = "1.5.0" num_cpus = "1.13.0" regex = "1.4.3" diff --git a/ql/generator/Cargo.toml b/ql/generator/Cargo.toml index 38074ab4e91..4ff796cb218 100644 --- a/ql/generator/Cargo.toml +++ b/ql/generator/Cargo.toml @@ -10,5 +10,5 @@ edition = "2018" clap = "2.33" node-types = { path = "../node-types" } tracing = "0.1" -tracing-subscriber = { version = "0.2", features = ["env-filter"] } +tracing-subscriber = { version = "0.3.3", features = ["env-filter"] } tree-sitter-ql = { git = "https://github.com/tausbn/tree-sitter-ql.git", rev = "36bdc0eae196f9833182ce3f8932be63534121b3" } diff --git a/ql/node-types/Cargo.toml b/ql/node-types/Cargo.toml index c751d7360d6..181bd6481e9 100644 --- a/ql/node-types/Cargo.toml +++ b/ql/node-types/Cargo.toml @@ -8,4 +8,4 @@ edition = "2018" [dependencies] serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" \ No newline at end of file +serde_json = "1.0" diff --git a/ql/tools/autobuild.cmd b/ql/tools/autobuild.cmd index b000600fea3..1b8defd6d85 100644 --- a/ql/tools/autobuild.cmd +++ b/ql/tools/autobuild.cmd @@ -1,12 +1,5 @@ @echo off -type NUL && "%CODEQL_DIST%\codeql.exe" database index-files ^ - --include-extension=.ql ^ - --include-extension=.qll ^ - --include-extension=.dbscheme ^ - --include=**/qlpack.yml ^ - --size-limit=5m ^ - --language=ql ^ - "%CODEQL_EXTRACTOR_QL_WIP_DATABASE%" +type NUL && "%CODEQL_EXTRACTOR_QL_ROOT%\tools\%CODEQL_PLATFORM%\autobuilder" exit /b %ERRORLEVEL% diff --git a/ql/tools/autobuild.sh b/ql/tools/autobuild.sh index ef682d0980b..650236bedad 100755 --- a/ql/tools/autobuild.sh +++ b/ql/tools/autobuild.sh @@ -1,13 +1,4 @@ #!/bin/sh -set -eu +exec "${CODEQL_EXTRACTOR_QL_ROOT}/tools/${CODEQL_PLATFORM}/autobuilder" -exec "${CODEQL_DIST}/codeql" database index-files \ - --include-extension=.ql \ - --include-extension=.qll \ - --include-extension=.dbscheme \ - --include=**/qlpack.yml \ - --size-limit=5m \ - --language=ql \ - --working-dir=.\ - "$CODEQL_EXTRACTOR_QL_WIP_DATABASE"