From 8506af068df81cc51fecf2ac23726a4a81ef4de3 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Wed, 29 May 2024 13:47:47 +0200 Subject: [PATCH] Javascript: use `codeql_pack` for javascript extractor --- .gitignore | 3 + codeql-workspace.yml | 4 +- csharp/.gitignore | 1 - javascript/BUILD.bazel | 28 +++--- .../extractor/lib/typescript/BUILD.bazel | 23 ++--- javascript/resources/BUILD.bazel | 15 +++ javascript/resources/codeql-extractor.yml | 94 +++++++++++++++++++ javascript/resources/tools/autobuild.cmd | 30 ++++++ javascript/resources/tools/autobuild.sh | 29 ++++++ .../resources/tools/baseline-config.json | 8 ++ .../resources/tools/configure-baseline.cmd | 2 + .../resources/tools/configure-baseline.sh | 3 + javascript/resources/tools/index-dir.sh | 4 + ql/.gitignore | 1 - ruby/.gitignore | 1 - swift/.gitignore | 3 - 16 files changed, 211 insertions(+), 38 deletions(-) create mode 100644 javascript/resources/BUILD.bazel create mode 100644 javascript/resources/codeql-extractor.yml create mode 100644 javascript/resources/tools/autobuild.cmd create mode 100755 javascript/resources/tools/autobuild.sh create mode 100644 javascript/resources/tools/baseline-config.json create mode 100644 javascript/resources/tools/configure-baseline.cmd create mode 100755 javascript/resources/tools/configure-baseline.sh create mode 100755 javascript/resources/tools/index-dir.sh diff --git a/.gitignore b/.gitignore index 3d2e92e3ca9..e51b5c7db8a 100644 --- a/.gitignore +++ b/.gitignore @@ -62,3 +62,6 @@ node_modules/ # Temporary folders for working with generated models .model-temp + +# intree extractor packs +/*/extractor-pack diff --git a/codeql-workspace.yml b/codeql-workspace.yml index 6e024dae2cf..07548d3b240 100644 --- a/codeql-workspace.yml +++ b/codeql-workspace.yml @@ -16,9 +16,7 @@ provide: - "csharp/ql/campaigns/Solorigate/test/qlpack.yml" - "misc/legacy-support/*/qlpack.yml" - "misc/suite-helpers/qlpack.yml" - - "ruby/extractor-pack/codeql-extractor.yml" - - "swift/extractor-pack/codeql-extractor.yml" - - "ql/extractor-pack/codeql-extractor.yml" + - "*/extractor-pack/codeql-extractor.yml" - ".github/codeql/extensions/**/codeql-pack.yml" versionPolicies: diff --git a/csharp/.gitignore b/csharp/.gitignore index de2e4b63e18..f3ea7470cb9 100644 --- a/csharp/.gitignore +++ b/csharp/.gitignore @@ -14,5 +14,4 @@ csharp.log .vscode/launch.json extractor/Semmle.Extraction.CSharp.Driver/Properties/launchSettings.json -extractor-pack paket-files/ diff --git a/javascript/BUILD.bazel b/javascript/BUILD.bazel index 18315a09813..5081a2bcd7f 100644 --- a/javascript/BUILD.bazel +++ b/javascript/BUILD.bazel @@ -1,6 +1,5 @@ load("@rules_pkg//pkg:mappings.bzl", "pkg_files") -load("@semmle_code//:dist.bzl", "dist") -load("@semmle_code//buildutils-internal:zipmerge.bzl", "zipmerge") +load("//misc/bazel:pkg.bzl", "codeql_pack") package(default_visibility = ["//visibility:public"]) @@ -23,8 +22,10 @@ pkg_files( strip_prefix = None, ) -dist( - name = "javascript-extractor-pack", +# We have to use a zip of the typescript parser wrapper, as it's generated by a genrule +# and we don't know a list of its output files. +codeql_pack( + name = "javascript", srcs = [ ":dbscheme-group", "//javascript/downgrades", @@ -32,17 +33,14 @@ dist( "//javascript/extractor:tools-extractor", "@semmle_code//language-packs/javascript:resources", ], - prefix = "javascript", + visibility = ["//visibility:public"], + zips = {"//javascript/extractor/lib/typescript": "tools"}, ) -# We have to zipmerge in the typescript parser wrapper, as it's generated by a genrule -# and we don't know a list of its output files. Therefore, we sidestep the -# rules_pkg tooling here, and generate the zip for the language pack manually. -zipmerge( - name = "javascript", - srcs = [ - ":javascript-extractor-pack.zip", - "//javascript/extractor/lib/typescript", - ], - out = "javascript.zip", +# TODO copy for internal repository backward compatibility +genrule( + name = "javascript.zip", + srcs = [":javascript-generic-zip"], + outs = ["javascript.zip"], + cmd = "cp $< $@", ) diff --git a/javascript/extractor/lib/typescript/BUILD.bazel b/javascript/extractor/lib/typescript/BUILD.bazel index 904331e4c64..28003162673 100644 --- a/javascript/extractor/lib/typescript/BUILD.bazel +++ b/javascript/extractor/lib/typescript/BUILD.bazel @@ -1,5 +1,3 @@ -load("@semmle_code//:common.bzl", "on_windows") - # Builds a zip file of the compiled typscript-parser-wrapper and its dependencies. genrule( name = "typescript", @@ -33,19 +31,16 @@ genrule( # Install again with only runtime deps "$$NPM install --prod", "mv node_modules build/", - "mkdir -p javascript/tools/typescript-parser-wrapper", - "mv build/* javascript/tools/typescript-parser-wrapper", + "mkdir -p typescript-parser-wrapper", + "mv build/* typescript-parser-wrapper", + "OUT=$$BAZEL_ROOT/$@", + "case $$OSTYPE in", + " cygwin|msys|win32) OUT=$$(cygpath -w $$OUT);;", + "esac", "", - ]) + on_windows( - " && ".join([ - "$$BAZEL_ROOT/$(execpath @bazel_tools//tools/zip:zipper) cC $$(cygpath -w $$BAZEL_ROOT/$@) $$(find javascript -name '*' -print)", - "rm -rf $$TEMP", - ]), - " && ".join([ - "$$BAZEL_ROOT/$(execpath @bazel_tools//tools/zip:zipper) cC $$BAZEL_ROOT/$@ $$(find javascript -name '*' -print)", - "rm -rf $$TEMP", - ]), - ), + "$$BAZEL_ROOT/$(execpath @bazel_tools//tools/zip:zipper) cC $$OUT $$(find typescript-parser-wrapper -name '*' -print)", + "rm -rf $$TEMP", + ]), tools = [ "@bazel_tools//tools/zip:zipper", "@nodejs//:node_bin", diff --git a/javascript/resources/BUILD.bazel b/javascript/resources/BUILD.bazel new file mode 100644 index 00000000000..c9ac9a73489 --- /dev/null +++ b/javascript/resources/BUILD.bazel @@ -0,0 +1,15 @@ +load("//misc/bazel:pkg.bzl", "codeql_pkg_files") + +codeql_pkg_files( + name = "resources", + srcs = glob( + ["**/*"], + exclude = [ + "tools/*.sh", + "BUILD.bazel", + ], + ), + exes = glob(["tools/*.sh"]), + strip_prefix = "", + visibility = ["//javascript:__pkg__"], +) diff --git a/javascript/resources/codeql-extractor.yml b/javascript/resources/codeql-extractor.yml new file mode 100644 index 00000000000..b82b2a96581 --- /dev/null +++ b/javascript/resources/codeql-extractor.yml @@ -0,0 +1,94 @@ +name: "javascript" +aliases: + - javascript-typescript + - typescript +display_name: "JavaScript/TypeScript" +version: 1.22.1 +column_kind: "utf16" +unicode_newlines: true +build_modes: + - none +file_coverage_languages: + - name: javascript + display_name: JavaScript + scc_languages: + - JavaScript + - name: typescript + display_name: TypeScript + scc_languages: + - TypeScript + - TypeScript Typings +github_api_languages: + - JavaScript + - TypeScript +scc_languages: + - JavaScript + - TypeScript + - TypeScript Typings +file_types: + - name: javascript + display_name: JavaScript + extensions: + - .js + - .jsx + - name: ecmascript + display_name: ECMAScript + extensions: + - .es + - .es6 + - .mjs + - name: typescript + display_name: TypeScript + extensions: + - .ts + - .tsx + - name: html + display_name: HTML + extensions: + - .html + - .htm + - .xhtm + - .xhtml + - name: vue + display_name: Vue.js component + extensions: + - .vue + - name: data + display_name: Data or configuration files + extensions: + - .json + - .yml + - .yaml + - .raml +legacy_qltest_extraction: true +options: + trap: + title: TRAP options + description: Options about how the extractor handles TRAP files + type: object + visibility: 3 + properties: + cache: + title: TRAP cache options + description: Options about how the extractor handles its TRAP cache + type: object + properties: + dir: + title: TRAP cache directory + description: The directory of the TRAP cache to use + type: string + bound: + title: TRAP cache bound + description: A soft limit (in MB) on the size of the TRAP cache + type: string + pattern: "[0-9]+" + write: + title: TRAP cache writeable + description: Whether to write to the TRAP cache as well as reading it + type: string + pattern: "(true|TRUE|false|FALSE)" + skip_types: + title: Skip type extraction for TypeScript + description: Whether to skip the extraction of types in a TypeScript application + type: string + pattern: "^(false|true)$" diff --git a/javascript/resources/tools/autobuild.cmd b/javascript/resources/tools/autobuild.cmd new file mode 100644 index 00000000000..c64898bea78 --- /dev/null +++ b/javascript/resources/tools/autobuild.cmd @@ -0,0 +1,30 @@ +@echo off +SETLOCAL EnableDelayedExpansion + +set jvm_args=-Xss16m + +rem If CODEQL_RAM is set, use half for Java and half for TS. +if NOT [%CODEQL_RAM%] == [] ( + set /a "half_ram=CODEQL_RAM/2" + set LGTM_TYPESCRIPT_RAM=%half_ram% + set jvm_args=!jvm_args! -Xmx!half_ram!m +) + +rem If CODEQL_THREADS is set, propagate via LGTM_THREADS. +if NOT [%CODEQL_THREADS%] == [] ( + set LGTM_THREADS=%CODEQL_THREADS% +) + +rem The JS autobuilder expects to find typescript modules under SEMMLE_DIST/tools. +rem They are included in the pack, but we need to set SEMMLE_DIST appropriately. +set SEMMLE_DIST=%CODEQL_EXTRACTOR_JAVASCRIPT_ROOT% + +rem The JS autobuilder expects LGTM_SRC to be set to the source root. +set LGTM_SRC=%CD% + +type NUL && "%CODEQL_JAVA_HOME%\bin\java.exe" %jvm_args% ^ + -cp "%CODEQL_EXTRACTOR_JAVASCRIPT_ROOT%\tools\extractor-javascript.jar" ^ + com.semmle.js.extractor.AutoBuild +exit /b %ERRORLEVEL% + +ENDLOCAL diff --git a/javascript/resources/tools/autobuild.sh b/javascript/resources/tools/autobuild.sh new file mode 100755 index 00000000000..7aaaf287828 --- /dev/null +++ b/javascript/resources/tools/autobuild.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +set -eu + +jvm_args=-Xss16m + +# If CODEQL_RAM is set, use half for Java and half for TS. +if [ -n "${CODEQL_RAM:-}" ] ; then + half_ram="$(( CODEQL_RAM / 2 ))" + LGTM_TYPESCRIPT_RAM="$half_ram" + export LGTM_TYPESCRIPT_RAM + jvm_args="$jvm_args -Xmx${half_ram}m" +fi + +# If CODEQL_THREADS is set, propagate via LGTM_THREADS. +if [ -n "${CODEQL_THREADS:-}" ] ; then + LGTM_THREADS="$CODEQL_THREADS" + export LGTM_THREADS +fi + +# The JS autobuilder expects to find typescript modules under SEMMLE_DIST/tools. +# They are included in the pack, but we need to set SEMMLE_DIST appropriately. +# We want to word-split $jvm_args, so disable the shellcheck warning. +# shellcheck disable=SC2086 +env SEMMLE_DIST="$CODEQL_EXTRACTOR_JAVASCRIPT_ROOT" \ + LGTM_SRC="$(pwd)" \ + "${CODEQL_JAVA_HOME}/bin/java" $jvm_args \ + -cp "$CODEQL_EXTRACTOR_JAVASCRIPT_ROOT/tools/extractor-javascript.jar" \ + com.semmle.js.extractor.AutoBuild diff --git a/javascript/resources/tools/baseline-config.json b/javascript/resources/tools/baseline-config.json new file mode 100644 index 00000000000..e79f2b6f40b --- /dev/null +++ b/javascript/resources/tools/baseline-config.json @@ -0,0 +1,8 @@ +{ + "paths-ignore": [ + "**/node_modules/**", + "**/bower_components/**", + "**/*.min.js", + "**/*-min.js" + ] +} diff --git a/javascript/resources/tools/configure-baseline.cmd b/javascript/resources/tools/configure-baseline.cmd new file mode 100644 index 00000000000..1398391a379 --- /dev/null +++ b/javascript/resources/tools/configure-baseline.cmd @@ -0,0 +1,2 @@ +@echo off +type "%CODEQL_EXTRACTOR_JAVASCRIPT_ROOT%\tools\baseline-config.json" diff --git a/javascript/resources/tools/configure-baseline.sh b/javascript/resources/tools/configure-baseline.sh new file mode 100755 index 00000000000..0743da4453c --- /dev/null +++ b/javascript/resources/tools/configure-baseline.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +cat "$CODEQL_EXTRACTOR_JAVASCRIPT_ROOT/tools/baseline-config.json" diff --git a/javascript/resources/tools/index-dir.sh b/javascript/resources/tools/index-dir.sh new file mode 100755 index 00000000000..c7ee9673b1f --- /dev/null +++ b/javascript/resources/tools/index-dir.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +echo "Not implemented." 1>&2 +exit 1 diff --git a/ql/.gitignore b/ql/.gitignore index 5af0736b59f..b8df693e827 100644 --- a/ql/.gitignore +++ b/ql/.gitignore @@ -1,5 +1,4 @@ target -extractor-pack .vscode/launch.json .cache ql/test/**/*.testproj diff --git a/ruby/.gitignore b/ruby/.gitignore index f34732b9f20..f0b40d44e7a 100644 --- a/ruby/.gitignore +++ b/ruby/.gitignore @@ -1,5 +1,4 @@ extractor/target -extractor-pack .vscode/launch.json .cache ql/test/**/*.testproj diff --git a/swift/.gitignore b/swift/.gitignore index 445d287f34a..c4a93d763d3 100644 --- a/swift/.gitignore +++ b/swift/.gitignore @@ -1,6 +1,3 @@ -# directory created by bazel run //swift:create-extractor-pack -/extractor-pack - # output files created by running tests *.o