Merge pull request #1340 from xiemaisi/js/es2019

Approved by asger-semmle

change-notes/1.21/extractor-javascript.md

@@ -4,4 +4,6 @@
 
 ## Changes to code extraction
 
+* ECMAScript 2019 support is now enabled by default.
+
 * YAML files are now extracted by default on LGTM. You can specify exclusion filters in your `lgtm.yml` file to override this behavior.

change-notes/support/versions-compilers.csv

@@ -11,6 +11,6 @@ COBOL,ANSI 85 or newer [1]_.,Not applicable,"``.cbl``, ``.CBL``, ``.cpy``, ``.CP
 Java,"Java 6 to 11 [2]_.","javac (OpenJDK and Oracle JDK)
 
 Eclipse compiler for Java (ECJ) batch compiler",``.java``
-JavaScript,ECMAScript 2018 or lower,Not applicable,"``.js``, ``.jsx``, ``.mjs``, ``.es``, ``.es6``, ``.htm``, ``.html``, ``.xhm``, ``.xhtml``, ``.vue``, ``.json`` [3]_."
+JavaScript,ECMAScript 2019 or lower,Not applicable,"``.js``, ``.jsx``, ``.mjs``, ``.es``, ``.es6``, ``.htm``, ``.html``, ``.xhm``, ``.xhtml``, ``.vue``, ``.json``, ``.yaml``, ``.yml``, ``.raml`` [3]_."
 Python,"2.7, 3.5, 3.6, 3.7",Not applicable,``.py``
 TypeScript [4]_.,"2.6-3.4",Standard TypeScript compiler,"``.ts``, ``.tsx``"

javascript/extractor/src/com/semmle/jcorn/ESNextParser.java

@@ -385,7 +385,7 @@ public class ESNextParser extends JSXParser {
       param = this.parseBindingAtom();
       this.checkLVal(param, true, null);
       this.expect(TokenType.parenR);
-    } else if (!options.esnext()) {
+    } else if (options.ecmaVersion() < 10) {
       this.unexpected();
     }
     BlockStatement catchBody = this.parseBlock(false);

javascript/extractor/src/com/semmle/jcorn/Options.java

@@ -51,7 +51,7 @@ public class Options {
   private Function<SyntaxError, ?> onRecoverableError;
 
   public Options() {
-    this.ecmaVersion = 7;
+    this.ecmaVersion = 10;
     this.sourceType = "script";
     this.onInsertedSemicolon = null;
     this.onTrailingComma = null;

javascript/extractor/src/com/semmle/jcorn/Parser.java

@@ -890,6 +890,9 @@ public class Parser {
         out.append(inputSubstring(chunkStart, this.pos));
         out.append(this.readEscapedChar(false));
         chunkStart = this.pos;
+      } else if (options.ecmaVersion() >= 10 && (ch == 0x2028 || ch == 0x2029)) {
+        // ECMAScript 2019 allows Unicode newlines in string literals
+        ++this.pos;
       } else {
         if (Whitespace.isNewLine(ch)) this.raise(this.start, "Unterminated string constant");
         ++this.pos;

javascript/extractor/src/com/semmle/js/extractor/ExtractorConfig.java

@@ -34,7 +34,8 @@ public class ExtractorConfig {
     ECMA2016(2016, 7),
     ECMA2017(2017, 8),
     ECMA2018(2018, 9),
-    ECMA2019(2019, 10);
+    ECMA2019(2019, 10),
+    ECMA2020(2020, 11);
 
     private final int version;
     public final int legacyVersion;
@@ -236,7 +237,7 @@ public class ExtractorConfig {
   private String defaultEncoding;
 
   public ExtractorConfig(boolean experimental) {
-    this.ecmaVersion = experimental ? ECMAVersion.ECMA2019 : ECMAVersion.ECMA2018;
+    this.ecmaVersion = experimental ? ECMAVersion.ECMA2020 : ECMAVersion.ECMA2019;
     this.platform = Platform.AUTO;
     this.jsx = true;
     this.sourceType = SourceType.AUTO;

javascript/extractor/tests/es2019/input/json-superset.js

@@ -0,0 +1,2 @@
+var s1 = "ab
c";
+var s2 = "ab
c";

javascript/extractor/tests/es2019/output/trap/json-superset.js.trap

@@ -0,0 +1,166 @@
+#10000=@"/json-superset.js;sourcefile"
+files(#10000,"/json-superset.js","json-superset","js",0)
+#10001=@"/;folder"
+folders(#10001,"/","")
+containerparent(#10001,#10000)
+#10002=@"loc,{#10000},0,0,0,0"
+locations_default(#10002,#10000,0,0,0,0)
+hasLocation(#10000,#10002)
+#20000=@"global_scope"
+scopes(#20000,0)
+#20001=@"script;{#10000},1,1"
+#20002=*
+lines(#20002,#20001,"var s1 = ""ab","
")
+#20003=@"loc,{#10000},1,1,1,12"
+locations_default(#20003,#10000,1,1,1,12)
+hasLocation(#20002,#20003)
+#20004=*
+lines(#20004,#20001,"c"";","
+")
+#20005=@"loc,{#10000},2,1,2,3"
+locations_default(#20005,#10000,2,1,2,3)
+hasLocation(#20004,#20005)
+#20006=*
+lines(#20006,#20001,"var s2 = ""ab","
")
+#20007=@"loc,{#10000},3,1,3,12"
+locations_default(#20007,#10000,3,1,3,12)
+hasLocation(#20006,#20007)
+#20008=*
+lines(#20008,#20001,"c"";","
+")
+#20009=@"loc,{#10000},4,1,4,3"
+locations_default(#20009,#10000,4,1,4,3)
+hasLocation(#20008,#20009)
+numlines(#20001,4,2,0)
+#20010=*
+tokeninfo(#20010,7,#20001,0,"var")
+#20011=@"loc,{#10000},1,1,1,3"
+locations_default(#20011,#10000,1,1,1,3)
+hasLocation(#20010,#20011)
+#20012=*
+tokeninfo(#20012,6,#20001,1,"s1")
+#20013=@"loc,{#10000},1,5,1,6"
+locations_default(#20013,#10000,1,5,1,6)
+hasLocation(#20012,#20013)
+#20014=*
+tokeninfo(#20014,8,#20001,2,"=")
+#20015=@"loc,{#10000},1,8,1,8"
+locations_default(#20015,#10000,1,8,1,8)
+hasLocation(#20014,#20015)
+#20016=*
+tokeninfo(#20016,4,#20001,3,"""ab
c""")
+#20017=@"loc,{#10000},1,10,1,15"
+locations_default(#20017,#10000,1,10,1,15)
+hasLocation(#20016,#20017)
+#20018=*
+tokeninfo(#20018,8,#20001,4,";")
+#20019=@"loc,{#10000},1,16,1,16"
+locations_default(#20019,#10000,1,16,1,16)
+hasLocation(#20018,#20019)
+#20020=*
+tokeninfo(#20020,7,#20001,5,"var")
+hasLocation(#20020,#20005)
+#20021=*
+tokeninfo(#20021,6,#20001,6,"s2")
+#20022=@"loc,{#10000},2,5,2,6"
+locations_default(#20022,#10000,2,5,2,6)
+hasLocation(#20021,#20022)
+#20023=*
+tokeninfo(#20023,8,#20001,7,"=")
+#20024=@"loc,{#10000},2,8,2,8"
+locations_default(#20024,#10000,2,8,2,8)
+hasLocation(#20023,#20024)
+#20025=*
+tokeninfo(#20025,4,#20001,8,"""ab
c""")
+#20026=@"loc,{#10000},2,10,2,15"
+locations_default(#20026,#10000,2,10,2,15)
+hasLocation(#20025,#20026)
+#20027=*
+tokeninfo(#20027,8,#20001,9,";")
+#20028=@"loc,{#10000},2,16,2,16"
+locations_default(#20028,#10000,2,16,2,16)
+hasLocation(#20027,#20028)
+#20029=*
+tokeninfo(#20029,0,#20001,10,"")
+#20030=@"loc,{#10000},3,1,3,0"
+locations_default(#20030,#10000,3,1,3,0)
+hasLocation(#20029,#20030)
+toplevels(#20001,0)
+#20031=@"loc,{#10000},1,1,3,0"
+locations_default(#20031,#10000,1,1,3,0)
+hasLocation(#20001,#20031)
+#20032=@"var;{s1};{#20000}"
+variables(#20032,"s1",#20000)
+#20033=@"var;{s2};{#20000}"
+variables(#20033,"s2",#20000)
+#20034=*
+stmts(#20034,18,#20001,0,"var s1 = ""ab\u2028c"";")
+#20035=@"loc,{#10000},1,1,1,16"
+locations_default(#20035,#10000,1,1,1,16)
+hasLocation(#20034,#20035)
+stmtContainers(#20034,#20001)
+#20036=*
+exprs(#20036,64,#20034,0,"s1 = ""ab\u2028c""")
+#20037=@"loc,{#10000},1,5,1,15"
+locations_default(#20037,#10000,1,5,1,15)
+hasLocation(#20036,#20037)
+enclosingStmt(#20036,#20034)
+exprContainers(#20036,#20001)
+#20038=*
+exprs(#20038,78,#20036,0,"s1")
+hasLocation(#20038,#20013)
+enclosingStmt(#20038,#20034)
+exprContainers(#20038,#20001)
+literals("s1","s1",#20038)
+decl(#20038,#20032)
+#20039=*
+exprs(#20039,4,#20036,1,"""ab\u2028c""")
+hasLocation(#20039,#20017)
+enclosingStmt(#20039,#20034)
+exprContainers(#20039,#20001)
+literals("ab
c","""ab
c""",#20039)
+#20040=*
+stmts(#20040,18,#20001,1,"var s2 = ""ab\u2029c"";")
+#20041=@"loc,{#10000},2,1,2,16"
+locations_default(#20041,#10000,2,1,2,16)
+hasLocation(#20040,#20041)
+stmtContainers(#20040,#20001)
+#20042=*
+exprs(#20042,64,#20040,0,"s2 = ""ab\u2029c""")
+#20043=@"loc,{#10000},2,5,2,15"
+locations_default(#20043,#10000,2,5,2,15)
+hasLocation(#20042,#20043)
+enclosingStmt(#20042,#20040)
+exprContainers(#20042,#20001)
+#20044=*
+exprs(#20044,78,#20042,0,"s2")
+hasLocation(#20044,#20022)
+enclosingStmt(#20044,#20040)
+exprContainers(#20044,#20001)
+literals("s2","s2",#20044)
+decl(#20044,#20033)
+#20045=*
+exprs(#20045,4,#20042,1,"""ab\u2029c""")
+hasLocation(#20045,#20026)
+enclosingStmt(#20045,#20040)
+exprContainers(#20045,#20001)
+literals("ab
c","""ab
c""",#20045)
+#20046=*
+entry_cfg_node(#20046,#20001)
+#20047=@"loc,{#10000},1,1,1,0"
+locations_default(#20047,#10000,1,1,1,0)
+hasLocation(#20046,#20047)
+#20048=*
+exit_cfg_node(#20048,#20001)
+hasLocation(#20048,#20030)
+successor(#20040,#20044)
+successor(#20045,#20042)
+successor(#20044,#20045)
+successor(#20042,#20048)
+successor(#20034,#20038)
+successor(#20039,#20036)
+successor(#20038,#20039)
+successor(#20036,#20040)
+successor(#20046,#20034)
+numlines(#10000,4,2,0)
+filetype(#10000,"javascript")