Merge pull request #10470 from erik-krogh/flowParse

JS: Try to parse files without using our parser extensions before enabling the extensions

javascript/extractor/src/com/semmle/jcorn/Options.java

@@ -40,7 +40,7 @@ public class Options {
   }
 
   private boolean allowHashBang, allowReturnOutsideFunction, allowImportExportEverywhere, allowGeneratedCodeExprs;
-  private boolean preserveParens, mozExtensions, jscript, esnext, v8Extensions, e4x;
+  private boolean preserveParens, mozExtensions, jscript, esnext, v8Extensions, e4x, allowFlowTypes;
   private int ecmaVersion;
   private AllowReserved allowReserved;
   private String sourceType;
@@ -70,6 +70,7 @@ public class Options {
     this.v8Extensions = false;
     this.e4x = false;
     this.onRecoverableError = null;
+    this.allowFlowTypes = false;
   }
 
   public Options(Options that) {
@@ -92,6 +93,7 @@ public class Options {
     this.onComment = that.onComment;
     this.program = that.program;
     this.onRecoverableError = that.onRecoverableError;
+    this.allowFlowTypes = that.allowFlowTypes;
   }
 
   public boolean allowHashBang() {
@@ -130,6 +132,10 @@ public class Options {
     return v8Extensions;
   }
 
+  public boolean allowFlowTypes() {
+    return allowFlowTypes;
+  }
+
   public boolean e4x() {
     return e4x;
   }
@@ -202,6 +208,10 @@ public class Options {
     this.v8Extensions = v8Extensions;
   }
 
+  public void allowFlowTypes(boolean allowFlowTypes) {
+    this.allowFlowTypes = allowFlowTypes;
+  }
+
   public void e4x(boolean e4x) {
     this.e4x = e4x;
   }

javascript/extractor/src/com/semmle/jcorn/flow/FlowParser.java

@@ -830,7 +830,7 @@ public class FlowParser extends ESNextParser {
 
   /** Should Flow syntax be allowed? */
   private boolean flow() {
-    return options.esnext();
+    return options.allowFlowTypes();
   }
 
   @Override

javascript/extractor/src/com/semmle/js/extractor/Main.java

@@ -41,7 +41,7 @@ public class Main {
    * A version identifier that should be updated every time the extractor changes in such a way that
    * it may produce different tuples for the same file under the same {@link ExtractorConfig}.
    */
-  public static final String EXTRACTOR_VERSION = "2022-08-25";
+  public static final String EXTRACTOR_VERSION = "2022-09-19";
 
   public static final Pattern NEWLINE = Pattern.compile("\n");
 

javascript/extractor/src/com/semmle/js/parser/JcornWrapper.java

@@ -28,20 +28,29 @@ public class JcornWrapper {
             .onToken(tokens)
             .preserveParens(true)
             .allowReturnOutsideFunction(true);
-    if (config.isMozExtensions()) options.mozExtensions(true);
-    if (config.isJscript()) options.jscript(true);
-    if (config.isJsx()) options = new JSXOptions(options);
     if (config.isEsnext()) options.esnext(true);
-    if (config.isV8Extensions()) options.v8Extensions(true);
-    if (config.isE4X()) options.e4x(true);
 
     Program program = null;
     List<ParseError> errors = new ArrayList<>();
+    
     try {
-      if (config.isTolerateParseErrors())
+      try {
-        options.onRecoverableError((err) -> errors.add(mkParseError(err)));
+        // First try to parse as a regular JavaScript program.
-
+        program = sourceType.createParser(options, source, 0).parse();
-      program = sourceType.createParser(options, source, 0).parse();
+      } catch (SyntaxError e) {
+        // If that fails, try to enable all the extensions that we support.
+        if (config.isTolerateParseErrors())
+          options.onRecoverableError((err) -> errors.add(mkParseError(err)));
+        comments.clear();
+        tokens.clear();
+        if (config.isMozExtensions()) options.mozExtensions(true);
+        if (config.isJscript()) options.jscript(true);
+        if (config.isJsx()) options = new JSXOptions(options);
+        if (config.isV8Extensions()) options.v8Extensions(true);
+        if (config.isE4X()) options.e4x(true);
+        if (config.isEsnext()) options.allowFlowTypes(true); // allow the flow-parser to parse types.
+        program = sourceType.createParser(options, source, 0).parse();
+      }
     } catch (SyntaxError e) {
       errors.add(mkParseError(e));
     }

javascript/extractor/tests/mozilla/output/trap/odasa-2593.js.trap

@@ -133,76 +133,71 @@ locations_default(#20044,#10000,1,17,4,1)
 hasLocation(#20043,#20044)
 stmt_containers(#20043,#20039)
 #20045=*
-scopes(#20045,4)
+stmts(#20045,2,#20043,0,"arguments;")
-scopenodes(#20043,#20045)
+#20046=@"loc,{#10000},2,5,2,14"
-scopenesting(#20045,#20041)
+locations_default(#20046,#10000,2,5,2,14)
-#20046=@"var;{arguments};{#20045}"
+hasLocation(#20045,#20046)
-variables(#20046,"arguments",#20045)
+stmt_containers(#20045,#20039)
 #20047=*
-stmts(#20047,2,#20043,0,"arguments;")
+exprs(#20047,79,#20045,0,"arguments")
-#20048=@"loc,{#10000},2,5,2,14"
+hasLocation(#20047,#20021)
-locations_default(#20048,#10000,2,5,2,14)
+enclosing_stmt(#20047,#20045)
-hasLocation(#20047,#20048)
+expr_containers(#20047,#20039)
-stmt_containers(#20047,#20039)
+literals("arguments","arguments",#20047)
-#20049=*
+bind(#20047,#20042)
-exprs(#20049,79,#20047,0,"arguments")
+#20048=*
-hasLocation(#20049,#20021)
+stmts(#20048,2,#20043,1,"let (arguments);")
-enclosing_stmt(#20049,#20047)
+#20049=@"loc,{#10000},3,5,3,20"
-expr_containers(#20049,#20039)
+locations_default(#20049,#10000,3,5,3,20)
-literals("arguments","arguments",#20049)
+hasLocation(#20048,#20049)
-bind(#20049,#20046)
+stmt_containers(#20048,#20039)
 #20050=*
-stmts(#20050,24,#20043,1,"let (arguments);")
+exprs(#20050,13,#20048,0,"let (arguments)")
-#20051=@"loc,{#10000},3,5,3,20"
+#20051=@"loc,{#10000},3,5,3,19"
-locations_default(#20051,#10000,3,5,3,20)
+locations_default(#20051,#10000,3,5,3,19)
 hasLocation(#20050,#20051)
-stmt_containers(#20050,#20039)
+enclosing_stmt(#20050,#20048)
+expr_containers(#20050,#20039)
 #20052=*
-scopes(#20052,4)
+exprs(#20052,79,#20050,-1,"let")
-scopenodes(#20050,#20052)
+hasLocation(#20052,#20025)
-scopenesting(#20052,#20045)
+enclosing_stmt(#20052,#20048)
-#20053=@"var;{arguments};{#20052}"
+expr_containers(#20052,#20039)
-variables(#20053,"arguments",#20052)
+literals("let","let",#20052)
+#20053=@"var;{let};{#20000}"
+variables(#20053,"let",#20000)
+bind(#20052,#20053)
 #20054=*
-exprs(#20054,64,#20050,0,"arguments")
+exprs(#20054,79,#20050,0,"arguments")
 hasLocation(#20054,#20029)
-enclosing_stmt(#20054,#20050)
+enclosing_stmt(#20054,#20048)
 expr_containers(#20054,#20039)
+literals("arguments","arguments",#20054)
+bind(#20054,#20042)
 #20055=*
-exprs(#20055,78,#20054,0,"arguments")
+entry_cfg_node(#20055,#20001)
-hasLocation(#20055,#20029)
+#20056=@"loc,{#10000},1,1,1,0"
-enclosing_stmt(#20055,#20050)
+locations_default(#20056,#10000,1,1,1,0)
-expr_containers(#20055,#20039)
+hasLocation(#20055,#20056)
-literals("arguments","arguments",#20055)
-decl(#20055,#20053)
-#20056=*
-stmts(#20056,0,#20050,-1,";")
-hasLocation(#20056,#20033)
-stmt_containers(#20056,#20039)
 #20057=*
-entry_cfg_node(#20057,#20001)
+exit_cfg_node(#20057,#20001)
-#20058=@"loc,{#10000},1,1,1,0"
+hasLocation(#20057,#20036)
-locations_default(#20058,#10000,1,1,1,0)
+successor(#20039,#20057)
-hasLocation(#20057,#20058)
+#20058=*
+entry_cfg_node(#20058,#20039)
+hasLocation(#20058,#20056)
 #20059=*
-exit_cfg_node(#20059,#20001)
+exit_cfg_node(#20059,#20039)
 hasLocation(#20059,#20036)
-successor(#20039,#20059)
+successor(#20043,#20045)
-#20060=*
+successor(#20048,#20052)
-entry_cfg_node(#20060,#20039)
+successor(#20054,#20050)
-hasLocation(#20060,#20058)
+successor(#20052,#20054)
-#20061=*
+successor(#20050,#20059)
-exit_cfg_node(#20061,#20039)
+successor(#20045,#20047)
-hasLocation(#20061,#20036)
+successor(#20047,#20048)
-successor(#20043,#20047)
+successor(#20058,#20043)
-successor(#20050,#20055)
-successor(#20056,#20061)
-successor(#20055,#20054)
-successor(#20054,#20056)
-successor(#20047,#20049)
-successor(#20049,#20050)
-successor(#20060,#20043)
 successor(#20040,#20039)
-successor(#20057,#20040)
+successor(#20055,#20040)
 numlines(#10000,4,4,0)
 filetype(#10000,"javascript")

javascript/ql/src/change-notes/2022-09-22-plain-parsing.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Improved how the JavaScript parser handles ambiguities between plain JavaScript and dialects such as Flow and E4X that use the same file extension. The parser now prefers plain JavaScript if possible, falling back to dialects only if the source code can not be parsed as plain JavaScript. Previously, there were rare cases where parsing would fail because the parser would erroneously attempt to parse dialect-specific syntax in a regular JavaScript file.

javascript/ql/test/query-tests/LanguageFeatures/SyntaxError/notFlow.js

@@ -0,0 +1 @@
+foo ? () => ({}) : (name) => "foo";