diff --git a/javascript/extractor/src/com/semmle/jcorn/Parser.java b/javascript/extractor/src/com/semmle/jcorn/Parser.java index 6062c86167c..8425287858f 100644 --- a/javascript/extractor/src/com/semmle/jcorn/Parser.java +++ b/javascript/extractor/src/com/semmle/jcorn/Parser.java @@ -721,7 +721,8 @@ public class Parser { return this.finishOp(TokenType.prefix, 1); } - this.raise(this.pos, "Unexpected character '" + codePointToString(code) + "'"); + String msg = String.format("Unexpected character '%s' (U+%04X)", codePointToString(code), code); + this.raise(this.pos, msg); return null; } diff --git a/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java b/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java index db865c97e0c..f599617c14d 100644 --- a/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java +++ b/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java @@ -163,9 +163,9 @@ import java.util.stream.Stream; * following environment variables are available: * * diff --git a/javascript/extractor/src/com/semmle/js/extractor/Main.java b/javascript/extractor/src/com/semmle/js/extractor/Main.java index 1a5f92ca4ae..b89c4d1131b 100644 --- a/javascript/extractor/src/com/semmle/js/extractor/Main.java +++ b/javascript/extractor/src/com/semmle/js/extractor/Main.java @@ -37,7 +37,7 @@ public class Main { * A version identifier that should be updated every time the extractor changes in such a way that * it may produce different tuples for the same file under the same {@link ExtractorConfig}. */ - public static final String EXTRACTOR_VERSION = "2019-03-13"; + public static final String EXTRACTOR_VERSION = "2019-03-21"; public static final Pattern NEWLINE = Pattern.compile("\n"); diff --git a/javascript/extractor/src/com/semmle/js/extractor/test/AutoBuildTests.java b/javascript/extractor/src/com/semmle/js/extractor/test/AutoBuildTests.java index d6934932932..52bc65f2501 100644 --- a/javascript/extractor/src/com/semmle/js/extractor/test/AutoBuildTests.java +++ b/javascript/extractor/src/com/semmle/js/extractor/test/AutoBuildTests.java @@ -84,9 +84,7 @@ public class AutoBuildTests { return f; } - /** - * Add a file with default file type. - */ + /** Add a file with default file type. */ private Path addFile(boolean extracted, Path root, String... components) throws IOException { return addFile(extracted, null, root, components); } diff --git a/javascript/extractor/tests/encoding/input/zwsp.js b/javascript/extractor/tests/encoding/input/zwsp.js new file mode 100644 index 00000000000..67617457103 --- /dev/null +++ b/javascript/extractor/tests/encoding/input/zwsp.js @@ -0,0 +1 @@ +​ diff --git a/javascript/extractor/tests/encoding/output/trap/zwsp.js.trap b/javascript/extractor/tests/encoding/output/trap/zwsp.js.trap new file mode 100644 index 00000000000..60c812c4c89 --- /dev/null +++ b/javascript/extractor/tests/encoding/output/trap/zwsp.js.trap @@ -0,0 +1,26 @@ +#10000=@"/zwsp.js;sourcefile" +files(#10000,"/zwsp.js","zwsp","js",0) +#10001=@"/;folder" +folders(#10001,"/","") +containerparent(#10001,#10000) +#10002=@"loc,{#10000},0,0,0,0" +locations_default(#10002,#10000,0,0,0,0) +hasLocation(#10000,#10002) +#20000=@"global_scope" +scopes(#20000,0) +#20001=@"script;{#10000},1,1" +toplevels(#20001,0) +#20002=@"loc,{#10000},1,1,1,1" +locations_default(#20002,#10000,1,1,1,1) +hasLocation(#20001,#20002) +#20003=* +jsParseErrors(#20003,#20001,"Error: Unexpected character '​' (U+200B)","​ +") +hasLocation(#20003,#20002) +#20004=* +lines(#20004,#20001,"​"," +") +hasLocation(#20004,#20002) +numlines(#20001,1,0,0) +numlines(#10000,1,0,0) +filetype(#10000,"javascript") diff --git a/javascript/extractor/tests/shebang/output/trap/tst.html.trap b/javascript/extractor/tests/shebang/output/trap/tst.html.trap index 525df65c352..3f56959df24 100644 --- a/javascript/extractor/tests/shebang/output/trap/tst.html.trap +++ b/javascript/extractor/tests/shebang/output/trap/tst.html.trap @@ -14,7 +14,7 @@ toplevels(#20001,1) locations_default(#20002,#10000,3,17,3,17) hasLocation(#20001,#20002) #20003=* -jsParseErrors(#20003,#20001,"Error: Unexpected character '#'","#!/usr/bin/node +jsParseErrors(#20003,#20001,"Error: Unexpected character '#' (U+0023)","#!/usr/bin/node ") #20004=@"loc,{#10000},4,1,4,1" locations_default(#20004,#10000,4,1,4,1)