From ca79b083cf73e7f6be23d2fb0f38f50a78a05be8 Mon Sep 17 00:00:00 2001 From: Asger F Date: Mon, 19 Aug 2019 15:04:02 +0100 Subject: [PATCH 1/3] TS: Add debugging flag and document how to run the debugger --- javascript/extractor/lib/typescript/README.md | 21 +++++++++++++++ .../semmle/js/parser/TypeScriptParser.java | 27 +++++++++++++++++-- 2 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 javascript/extractor/lib/typescript/README.md diff --git a/javascript/extractor/lib/typescript/README.md b/javascript/extractor/lib/typescript/README.md new file mode 100644 index 00000000000..a47d6bed8d9 --- /dev/null +++ b/javascript/extractor/lib/typescript/README.md @@ -0,0 +1,21 @@ +TypeScript parser wrapper +------------------------- + +The TypeScript "parser wrapper" is a Node.js program launched from the JavaScript +extractor in order to extract TypeScript code. + +The two processes communicate via a command/response protocol via stdin/stdout. + +Debugging +--------- + +To debug the parser script: + +1. Launch an extraction process with the environment variable `SEMMLE_TYPESCRIPT_ATTACH_DEBUGGER` + set to `break`. + +2. Open VSCode and choose "Debug: Attach to Node process" from the command palette, and + choose the process that looks like the parser-wrapper. + +If you don't wish to pause on entry, instead set `SEMMLE_TYPESCRIPT_ATTACH_DEBUGGER` to +any non-empty value. diff --git a/javascript/extractor/src/com/semmle/js/parser/TypeScriptParser.java b/javascript/extractor/src/com/semmle/js/parser/TypeScriptParser.java index f798fc297a5..ad073177f99 100644 --- a/javascript/extractor/src/com/semmle/js/parser/TypeScriptParser.java +++ b/javascript/extractor/src/com/semmle/js/parser/TypeScriptParser.java @@ -98,6 +98,16 @@ public class TypeScriptParser { */ public static final String TYPESCRIPT_RAM_RESERVE_SUFFIX = "TYPESCRIPT_RAM_RESERVE"; + /** + * An environment variable which, if set, allows a debugger to be attached to the Node.js + * process. Remote debugging will not be enabled. + *

+ * If set to break the Node.js process will pause on entry waiting for the + * debugger to attach (--inspect-brk). If set to any other non-empty value, + * it will just enable debugging (--inspect). + */ + public static final String TYPESCRIPT_ATTACH_DEBUGGER = "SEMMLE_TYPESCRIPT_ATTACH_DEBUGGER"; + /** The Node.js parser wrapper process, if it has been started already. */ private Process parserWrapperProcess; @@ -203,7 +213,9 @@ public class TypeScriptParser { result.add(nodeJsRuntime); result.addAll(nodeJsRuntimeExtraArgs); for(String arg : args) { - result.add(arg); + if (arg.length() > 0) { + result.add(arg); + } } return result; } @@ -236,9 +248,20 @@ public class TypeScriptParser { int reserveMemoryMb = getMegabyteCountFromPrefixedEnv(TYPESCRIPT_RAM_RESERVE_SUFFIX, 400); File parserWrapper = getParserWrapper(); - + + String debuggerFlag = Env.systemEnv().get(TYPESCRIPT_ATTACH_DEBUGGER); + String inspectArg = ""; + if (debuggerFlag != null && debuggerFlag.length() > 0) { + if (debuggerFlag.equalsIgnoreCase("break")) { + inspectArg = "--inspect-brk"; + } else { + inspectArg = "--inspect"; + } + } + List cmd = getNodeJsRuntimeInvocation( "--max_old_space_size=" + (mainMemoryMb + reserveMemoryMb), + inspectArg, parserWrapper.getAbsolutePath() ); ProcessBuilder pb = new ProcessBuilder(cmd); From 6f217502f9ae0b8bb3cc40461c24d1eab1d07a38 Mon Sep 17 00:00:00 2001 From: Asger F Date: Tue, 20 Aug 2019 09:57:32 +0100 Subject: [PATCH 2/3] TS: Review --- javascript/extractor/lib/typescript/README.md | 7 +-- .../semmle/js/parser/TypeScriptParser.java | 43 +++++++++---------- 2 files changed, 22 insertions(+), 28 deletions(-) diff --git a/javascript/extractor/lib/typescript/README.md b/javascript/extractor/lib/typescript/README.md index a47d6bed8d9..998cec250aa 100644 --- a/javascript/extractor/lib/typescript/README.md +++ b/javascript/extractor/lib/typescript/README.md @@ -11,11 +11,8 @@ Debugging To debug the parser script: -1. Launch an extraction process with the environment variable `SEMMLE_TYPESCRIPT_ATTACH_DEBUGGER` - set to `break`. +1. Launch an extraction process with the environment variable `SEMMLE_TYPESCRIPT_NODE_FLAGS` + set to `--inspect`, or `--inspect-brk` if you wish to pause on entry. 2. Open VSCode and choose "Debug: Attach to Node process" from the command palette, and choose the process that looks like the parser-wrapper. - -If you don't wish to pause on entry, instead set `SEMMLE_TYPESCRIPT_ATTACH_DEBUGGER` to -any non-empty value. diff --git a/javascript/extractor/src/com/semmle/js/parser/TypeScriptParser.java b/javascript/extractor/src/com/semmle/js/parser/TypeScriptParser.java index ad073177f99..22a24a20a11 100644 --- a/javascript/extractor/src/com/semmle/js/parser/TypeScriptParser.java +++ b/javascript/extractor/src/com/semmle/js/parser/TypeScriptParser.java @@ -99,14 +99,19 @@ public class TypeScriptParser { public static final String TYPESCRIPT_RAM_RESERVE_SUFFIX = "TYPESCRIPT_RAM_RESERVE"; /** - * An environment variable which, if set, allows a debugger to be attached to the Node.js - * process. Remote debugging will not be enabled. + * An environment variable with additional VM arguments to pass to the Node process. *

- * If set to break the Node.js process will pause on entry waiting for the - * debugger to attach (--inspect-brk). If set to any other non-empty value, - * it will just enable debugging (--inspect). + * Only --inspect or --inspect-brk may be used at the moment. */ - public static final String TYPESCRIPT_ATTACH_DEBUGGER = "SEMMLE_TYPESCRIPT_ATTACH_DEBUGGER"; + public static final String TYPESCRIPT_NODE_FLAGS = "SEMMLE_TYPESCRIPT_NODE_FLAGS"; + + /** + * Flags that may be passed using {@link #TYPESCRIPT_NODE_FLAGS} + */ + private static final Set allowedDebugNodeFlags = new HashSet(Arrays.fromList( + "--inspect", + "--inspect-brk" + )); /** The Node.js parser wrapper process, if it has been started already. */ private Process parserWrapperProcess; @@ -213,9 +218,7 @@ public class TypeScriptParser { result.add(nodeJsRuntime); result.addAll(nodeJsRuntimeExtraArgs); for(String arg : args) { - if (arg.length() > 0) { - result.add(arg); - } + result.add(arg); } return result; } @@ -249,21 +252,15 @@ public class TypeScriptParser { File parserWrapper = getParserWrapper(); - String debuggerFlag = Env.systemEnv().get(TYPESCRIPT_ATTACH_DEBUGGER); - String inspectArg = ""; - if (debuggerFlag != null && debuggerFlag.length() > 0) { - if (debuggerFlag.equalsIgnoreCase("break")) { - inspectArg = "--inspect-brk"; - } else { - inspectArg = "--inspect"; - } - } + String debugFlagString = Env.systemEnv().getNonEmpty(TYPESCRIPT_NODE_FLAGS); + List debugFlags = debugFlagString == null ? new ArrayList<>() : debugFlagString.split(" "); + debugFlags.retainAll(allowedDebugNodeFlags); + + List cmd = getNodeJsRuntimeInvocation(); + cmd.add("--max_old_space_size=" + (mainMemoryMb + reserveMemoryMb)); + cmd.addAll(debugFlags); + cmd.add(parserWrapper.getAbsolutePath()); - List cmd = getNodeJsRuntimeInvocation( - "--max_old_space_size=" + (mainMemoryMb + reserveMemoryMb), - inspectArg, - parserWrapper.getAbsolutePath() - ); ProcessBuilder pb = new ProcessBuilder(cmd); parserWrapperCommand = StringUtil.glue(" ", cmd); pb.environment().put("SEMMLE_TYPESCRIPT_MEMORY_THRESHOLD", "" + mainMemoryMb); From f18f54fd0dbfc3552beb59b03614af4494ab581c Mon Sep 17 00:00:00 2001 From: Asger F Date: Tue, 20 Aug 2019 10:41:25 +0100 Subject: [PATCH 3/3] TS: Allow changing the port --- .../semmle/js/parser/TypeScriptParser.java | 54 ++++++++++--------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/javascript/extractor/src/com/semmle/js/parser/TypeScriptParser.java b/javascript/extractor/src/com/semmle/js/parser/TypeScriptParser.java index 22a24a20a11..452b2e22eaf 100644 --- a/javascript/extractor/src/com/semmle/js/parser/TypeScriptParser.java +++ b/javascript/extractor/src/com/semmle/js/parser/TypeScriptParser.java @@ -1,6 +1,21 @@ package com.semmle.js.parser; -import ch.qos.logback.classic.Level; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.ByteArrayOutputStream; +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.lang.ProcessBuilder.Redirect; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; @@ -21,21 +36,8 @@ import com.semmle.util.logging.LogbackUtils; import com.semmle.util.process.AbstractProcessBuilder; import com.semmle.util.process.Builder; import com.semmle.util.process.Env; -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.ByteArrayOutputStream; -import java.io.Closeable; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.lang.ProcessBuilder.Redirect; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; + +import ch.qos.logback.classic.Level; /** * The Java half of our wrapper for invoking the TypeScript parser. @@ -105,14 +107,6 @@ public class TypeScriptParser { */ public static final String TYPESCRIPT_NODE_FLAGS = "SEMMLE_TYPESCRIPT_NODE_FLAGS"; - /** - * Flags that may be passed using {@link #TYPESCRIPT_NODE_FLAGS} - */ - private static final Set allowedDebugNodeFlags = new HashSet(Arrays.fromList( - "--inspect", - "--inspect-brk" - )); - /** The Node.js parser wrapper process, if it has been started already. */ private Process parserWrapperProcess; @@ -253,8 +247,16 @@ public class TypeScriptParser { File parserWrapper = getParserWrapper(); String debugFlagString = Env.systemEnv().getNonEmpty(TYPESCRIPT_NODE_FLAGS); - List debugFlags = debugFlagString == null ? new ArrayList<>() : debugFlagString.split(" "); - debugFlags.retainAll(allowedDebugNodeFlags); + List debugFlags = new ArrayList<>(); + if (debugFlagString != null) { + for (String flag : debugFlagString.split(" ")) { + if (!flag.startsWith("--inspect") || flag.contains(":")) { + System.err.println("Ignoring unrecognized Node flag: '" + flag + "'"); + } else { + debugFlags.add(flag); + } + } + } List cmd = getNodeJsRuntimeInvocation(); cmd.add("--max_old_space_size=" + (mainMemoryMb + reserveMemoryMb));