Merge branch 'main' into ts53-ts

javascript/extractor/BUILD.bazel

@@ -1,18 +1,63 @@
-load("@//:common.bzl", "codeql_java_project")
+load("@//:common.bzl", "codeql_fat_jar", "codeql_java_project")
+load("@rules_pkg//pkg:mappings.bzl", "pkg_files")
 
-codeql_java_project(
-    name = "extractor",
-    deps = [
-        "@//extractor",
+java_library(
+    name = "deps",
+    visibility = [":__subpackages__"],
+    exports = [
+        "@//extractor:html",
+        "@//extractor:yaml",
         "@//resources/lib/java:commons-compress",
         "@//resources/lib/java:gson",
         "@//resources/lib/java:jericho-html",
         "@//resources/lib/java:slf4j-api",
         "@//resources/lib/java:snakeyaml",
-        "@//resources/lib/java/DO_NOT_DISTRIBUTE:junit",
         "@//third_party:jackson",
         "@//third_party:logback",
         "@//util-java7",
         "@//util-java8",
     ],
 )
+
+codeql_java_project(
+    name = "extractor",
+    deps = [
+        ":deps",
+    ],
+)
+
+pkg_files(
+    name = "javascript-extractor-resources",
+    srcs = glob(["resources/**"]),
+    strip_prefix = "resources",
+)
+
+codeql_fat_jar(
+    name = "extractor-javascript",
+    srcs = [
+        ":extractor",
+        "@//extractor:html",
+        "@//extractor:xml-trap-writer",
+        "@//extractor:yaml",
+        "@//resources/lib/java:commons-compress",
+        "@//resources/lib/java:gson",
+        "@//resources/lib/java:jericho-html",
+        "@//resources/lib/java:slf4j-api",
+        "@//resources/lib/java:snakeyaml",
+        "@//third_party:jackson",
+        "@//third_party:logback",
+        "@//util-java7",
+        "@//util-java8",
+    ],
+    files = [":javascript-extractor-resources"],
+    main_class = "com.semmle.js.extractor.Main",
+)
+
+pkg_files(
+    name = "tools-extractor",
+    srcs = [
+        ":extractor-javascript",
+    ],
+    prefix = "tools",
+    visibility = ["//visibility:public"],
+)

javascript/extractor/lib/typescript/BUILD.bazel

@@ -0,0 +1,55 @@
+load("@//:common.bzl", "on_windows")
+
+# Builds a zip file of the compiled typscript-parser-wrapper and its dependencies.
+genrule(
+    name = "typescript",
+    srcs = [
+        "tsconfig.json",
+        "package.json",
+        "package-lock.json",
+    ] + glob(["src/**"]),
+    outs = ["javascript.zip"],
+    cmd = "\n".join([
+        # the original working directory is not preserved anywhere, but needs to be accessible, as
+        # all paths are relative to this
+        # unfortunately, we need to change the working directory to run npm.
+        "export BAZEL_ROOT=$$(pwd)",
+        # we need a temp dir, and unfortunately, $TMPDIR is not set on Windows
+        "export TEMP=$$(mktemp -d)",
+        # Add node to the path so that npm run can find it - it's calling env node
+        "export PATH=$$BAZEL_ROOT/$$(dirname $(execpath @nodejs//:node_bin)):$$PATH",
+        "export NPM=$$BAZEL_ROOT/$(execpath @nodejs//:npm_bin)",
+        # npm has a global cache which doesn't work on macos, where absolute paths aren't filtered out by the sandbox.
+        # Therefore, set a temporary cache directory.
+        "export NPM_CONFIG_USERCONFIG=$$TEMP/npmrc",
+        "$$NPM config set cache $$TEMP/npm",
+        "$$NPM config set update-notifier false",
+        "rm -rf $(RULEDIR)/inputs",
+        "cp -L -R $$(dirname $(execpath package.json)) $(RULEDIR)/inputs",
+        "cd $(RULEDIR)/inputs",
+        "$$NPM install",
+        "$$NPM run build",
+        "rm -rf node_modules",
+        # Install again with only runtime deps
+        "$$NPM install --prod",
+        "mv node_modules build/",
+        "mkdir -p javascript/tools/typescript-parser-wrapper",
+        "mv build/* javascript/tools/typescript-parser-wrapper",
+        "",
+    ]) + on_windows(
+        " && ".join([
+            "$$BAZEL_ROOT/$(execpath @bazel_tools//tools/zip:zipper) cC $$(cygpath -w $$BAZEL_ROOT/$@) $$(find javascript -name '*' -print)",
+            "rm -rf $$TEMP",
+        ]),
+        " && ".join([
+            "$$BAZEL_ROOT/$(execpath @bazel_tools//tools/zip:zipper) cC $$BAZEL_ROOT/$@ $$(find javascript -name '*' -print)",
+            "rm -rf $$TEMP",
+        ]),
+    ),
+    tools = [
+        "@bazel_tools//tools/zip:zipper",
+        "@nodejs//:node_bin",
+        "@nodejs//:npm_bin",
+    ],
+    visibility = ["//visibility:public"],
+)

javascript/extractor/parser-tests/BUILD.bazel

@@ -0,0 +1,5 @@
+filegroup(
+    name = "parser-tests",
+    srcs = glob(["**/*"]),
+    visibility = ["//visibility:public"],
+)

javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java

@@ -892,10 +892,15 @@ protected DependencyInstallationResult preparePackagesAndDependencies(Set<Path>
           // For named packages, find the main file.
           String name = packageJson.getName();
           if (name != null) {
-            Path entryPoint = guessPackageMainFile(path, packageJson, FileType.TYPESCRIPT.getExtensions());
-            if (entryPoint == null) {
-              // Try a TypeScript-recognized JS extension instead
-              entryPoint = guessPackageMainFile(path, packageJson, Arrays.asList(".js", ".jsx"));
+            Path entryPoint = null; 
+            try {
+              entryPoint = guessPackageMainFile(path, packageJson, FileType.TYPESCRIPT.getExtensions());
+              if (entryPoint == null) {
+                // Try a TypeScript-recognized JS extension instead
+                entryPoint = guessPackageMainFile(path, packageJson, Arrays.asList(".js", ".jsx"));
+              }
+            } catch (InvalidPathException ignore) {
+              // can happen if the `main:` field is invalid. E.g. on Windows a path like `dist/*.js` will crash.
             }
             if (entryPoint != null) {
               System.out.println(relativePath + ": Main file set to " + sourceRoot.relativize(entryPoint));

javascript/extractor/test/com/semmle/js/extractor/test/ASTMatchingTests.java

@@ -12,6 +12,7 @@ import com.semmle.jcorn.SyntaxError;
 import com.semmle.js.ast.AST2JSON;
 import com.semmle.js.ast.Program;
 import com.semmle.util.io.WholeIO;
+import com.semmle.util.tests.TestPaths;
 import java.io.File;
 import java.util.Iterator;
 import java.util.Map.Entry;
@@ -54,7 +55,7 @@ public abstract class ASTMatchingTests {
     }
   }
 
-  private static final File BABYLON_BASE = new File("parser-tests/babylon").getAbsoluteFile();
+  private static final File BABYLON_BASE = TestPaths.get("parser-tests/babylon").toAbsolutePath().toFile();
 
   protected void babylonTest(String dir) {
     babylonTest(dir, new Options().esnext(true));

javascript/extractor/test/com/semmle/js/extractor/test/AutoBuildTests.java

@@ -30,6 +30,7 @@ import com.semmle.js.extractor.FileExtractor;
 import com.semmle.js.extractor.FileExtractor.FileType;
 import com.semmle.js.extractor.VirtualSourceRoot;
 import com.semmle.util.data.StringUtil;
+import com.semmle.util.exception.Exceptions;
 import com.semmle.util.exception.UserError;
 import com.semmle.util.files.FileUtil;
 import com.semmle.util.files.FileUtil8;
@@ -443,8 +444,12 @@ public class AutoBuildTests {
 
   /** Hide {@code p} on using {@link DosFileAttributeView} if available; otherwise do nothing. */
   private void hide(Path p) throws IOException {
+    try {
     DosFileAttributeView attrs = Files.getFileAttributeView(p, DosFileAttributeView.class);
     if (attrs != null) attrs.setHidden(true);
+    } catch (IOException e) {
+      Exceptions.ignore(e, "On Linux within the bazel sandbox, we get a DosFileAttributeView that then throws an exception upon use");
+    }
   }
 
   @Test

javascript/extractor/test/com/semmle/js/extractor/test/BUILD.bazel

@@ -0,0 +1,33 @@
+java_test(
+    name = "test_jar",
+    srcs = glob(["**/*.java"]),
+    test_class = "com.semmle.js.extractor.test.AllTests",
+    deps = [
+        "//javascript/extractor",
+        "//javascript/extractor:deps",
+        "@//resources/lib/java/DO_NOT_DISTRIBUTE:junit",
+    ],
+)
+
+# We need to unzip the typescript wrapper, and provide node on the path.
+# Therefore, we're wrapping the java_test inside a sh_test.
+sh_test(
+    name = "test",
+    size = "small",
+    srcs = ["run_tests.sh"],
+    args = [
+        "$(execpath @nodejs//:node_bin)",
+        "$(JAVABASE)/bin/java",
+        "$(rootpath //javascript/extractor/lib/typescript)",
+        "$(rootpath test_jar_deploy.jar)",
+    ],
+    data = [
+        ":test_jar_deploy.jar",
+        "//javascript/extractor/lib/typescript",
+        "//javascript/extractor/parser-tests",
+        "//javascript/extractor/tests",
+        "@bazel_tools//tools/jdk:current_java_runtime",
+        "@nodejs//:node_bin",
+    ],
+    toolchains = ["@bazel_tools//tools/jdk:current_java_runtime"],
+)

javascript/extractor/test/com/semmle/js/extractor/test/JSXTests.java

@@ -10,6 +10,7 @@ import com.semmle.js.ast.AST2JSON;
 import com.semmle.js.ast.Program;
 import com.semmle.util.files.FileUtil;
 import com.semmle.util.io.WholeIO;
+import com.semmle.util.tests.TestPaths;
 import java.io.File;
 import java.util.ArrayList;
 import java.util.List;
@@ -25,7 +26,7 @@ import org.junit.runners.Parameterized.Parameters;
  */
 @RunWith(Parameterized.class)
 public class JSXTests extends ASTMatchingTests {
-  private static final File BASE = new File("parser-tests/jcorn-jsx").getAbsoluteFile();
+  private static final File BASE = TestPaths.get("parser-tests/jcorn-jsx").toAbsolutePath().toFile();
 
   @Parameters(name = "{0}")
   public static Iterable<Object[]> tests() {

javascript/extractor/test/com/semmle/js/extractor/test/RobustnessTests.java

@@ -3,6 +3,7 @@ package com.semmle.js.extractor.test;
 import com.semmle.jcorn.Options;
 import com.semmle.jcorn.Parser;
 import com.semmle.util.io.WholeIO;
+import com.semmle.util.tests.TestPaths;
 import java.io.File;
 import java.nio.charset.StandardCharsets;
 import org.junit.Test;
@@ -11,7 +12,7 @@ public class RobustnessTests {
 
   @Test
   public void letLookheadTest() {
-    File test = new File("parser-tests/robustness/letLookahead.js");
+    File test = TestPaths.get("parser-tests/robustness/letLookahead.js").toFile();
     String src = new WholeIO(StandardCharsets.UTF_8.name()).strictread(test);
     new Parser(new Options(), src, 0).parse();
   }

javascript/extractor/test/com/semmle/js/extractor/test/TrapTests.java

@@ -13,6 +13,7 @@ import com.semmle.util.extraction.ExtractorOutputConfig;
 import com.semmle.util.io.WholeIO;
 import com.semmle.util.process.Env;
 import com.semmle.util.srcarchive.DummySourceArchive;
+import com.semmle.util.tests.TestPaths;
 import com.semmle.util.trap.ITrapWriterFactory;
 import com.semmle.util.trap.TrapWriter;
 import com.semmle.util.trap.pathtransformers.ProjectLayoutTransformer;
@@ -35,7 +36,7 @@ import org.junit.runners.Parameterized.Parameters;
 
 @RunWith(Parameterized.class)
 public class TrapTests {
-  private static final File BASE = new File("tests").getAbsoluteFile();
+  private static final File BASE = TestPaths.get("tests").toAbsolutePath().toFile();
 
   @Parameters(name = "{0}:{1}")
   public static Iterable<Object[]> tests() {

javascript/extractor/test/com/semmle/js/extractor/test/run_tests.sh

@@ -0,0 +1,31 @@
+NODE=$1
+JAVA=$2
+PARSER_WRAPPER=$3
+TEST_JAR=$4
+
+TEMP=$(mktemp -d)
+
+UNAME=$(uname -s)
+echo $UNAME
+# On Windows, the symlink set up by bazel that points at the test jar is a msys2/linux-style path
+# The JVM can't resolve that, therefore copy the jar to the temp directory, and then set the
+# windows path to it
+if [[ "$UNAME" =~ _NT ]]; then
+    cp $TEST_JAR $TEMP/test.jar
+    TEST_JAR=$(cygpath -w $TEMP/test.jar)
+    echo "On Windows, new test jar: $TEST_JAR"
+fi
+
+# unpack parser wrapper
+unzip -q $PARSER_WRAPPER -d $TEMP/parser_wrapper
+export SEMMLE_TYPESCRIPT_PARSER_WRAPPER=$TEMP/parser_wrapper/javascript/tools/typescript-parser-wrapper/main.js
+
+# setup node on path
+NODE=$(realpath $NODE)
+export PATH="$PATH:$(dirname $NODE)"
+
+$JAVA -Dbazel.test_suite=com.semmle.js.extractor.test.AllTests -jar $TEST_JAR
+EXIT_CODE=$?
+
+rm -rf $TEMP
+exit $EXIT_CODE

javascript/extractor/tests/BUILD.bazel

@@ -0,0 +1,5 @@
+filegroup(
+    name = "tests",
+    srcs = glob(["**/*"]),
+    visibility = ["//visibility:public"],
+)