Merge pull request #16061 from RasmusWL/js-extractor-fix

JS: More robust CommonJS/ES2015 detection logic for extractor

javascript/extractor/src/com/semmle/js/extractor/AbstractDetector.java

@@ -0,0 +1,113 @@
+package com.semmle.js.extractor;
+
+import com.semmle.js.ast.AssignmentExpression;
+import com.semmle.js.ast.BlockStatement;
+import com.semmle.js.ast.CallExpression;
+import com.semmle.js.ast.Expression;
+import com.semmle.js.ast.ExpressionStatement;
+import com.semmle.js.ast.IFunction;
+import com.semmle.js.ast.Identifier;
+import com.semmle.js.ast.IfStatement;
+import com.semmle.js.ast.MemberExpression;
+import com.semmle.js.ast.Node;
+import com.semmle.js.ast.ParenthesizedExpression;
+import com.semmle.js.ast.Program;
+import com.semmle.js.ast.Statement;
+import com.semmle.js.ast.TryStatement;
+import com.semmle.js.ast.UnaryExpression;
+import com.semmle.js.ast.VariableDeclaration;
+import com.semmle.js.ast.VariableDeclarator;
+import java.util.List;
+
+/**
+ * A utility base class for running detection logic on statements/expressions by
+ * visiting each node. It performs recursive decent into assignment expressions and
+ * callee expressions for call-expressions (to handle detection of `foo` in `foo()()`)
+ * */
+public abstract class AbstractDetector {
+  protected boolean programDetection(Node ast) {
+    if (!(ast instanceof Program)) return false;
+
+    return visitStatements(((Program) ast).getBody());
+  }
+
+  protected boolean visitStatements(List<Statement> stmts) {
+    for (Statement stmt : stmts) if (visitStatement(stmt)) return true;
+    return false;
+  }
+
+  protected boolean visitStatement(Statement stmt) {
+    if (stmt instanceof ExpressionStatement) {
+      Expression e = stripParens(((ExpressionStatement) stmt).getExpression());
+
+      // check whether `e` is an iife; if so, recursively check its body
+
+      // strip off unary operators to handle `!function(){...}()`
+      if (e instanceof UnaryExpression) e = ((UnaryExpression) e).getArgument();
+
+      if (e instanceof CallExpression && ((CallExpression) e).getArguments().isEmpty()) {
+        Expression callee = stripParens(((CallExpression) e).getCallee());
+        if (callee instanceof IFunction) {
+          Node body = ((IFunction) callee).getBody();
+          if (body instanceof BlockStatement)
+            return visitStatements(((BlockStatement) body).getBody());
+        }
+      }
+
+      if (visitExpression(e)) return true;
+
+    } else if (stmt instanceof VariableDeclaration) {
+      for (VariableDeclarator decl : ((VariableDeclaration) stmt).getDeclarations()) {
+        Expression init = stripParens(decl.getInit());
+        if (visitExpression(init)) return true;
+      }
+
+    } else if (stmt instanceof TryStatement) {
+      return visitStatement(((TryStatement) stmt).getBlock());
+
+    } else if (stmt instanceof BlockStatement) {
+      return visitStatements(((BlockStatement) stmt).getBody());
+
+    } else if (stmt instanceof IfStatement) {
+      IfStatement is = (IfStatement) stmt;
+      return visitStatement(is.getConsequent())
+          || visitStatement(is.getAlternate());
+    }
+
+    return false;
+  }
+
+  private static Expression stripParens(Expression e) {
+    if (e instanceof ParenthesizedExpression)
+      return stripParens(((ParenthesizedExpression) e).getExpression());
+    return e;
+  }
+
+  /**
+   * Recursively check {@code e} if it's a call or an assignment.
+   */
+  protected boolean visitExpression(Expression e) {
+    if (e instanceof CallExpression) {
+      CallExpression call = (CallExpression) e;
+      Expression callee = call.getCallee();
+      // recurse, to handle things like `foo()()`
+      if (visitExpression(callee)) return true;
+      return false;
+    } else if (e instanceof MemberExpression) {
+      return visitExpression(((MemberExpression) e).getObject());
+    } else if (e instanceof AssignmentExpression) {
+      AssignmentExpression assgn = (AssignmentExpression) e;
+
+      // filter out compound assignments
+      if (!"=".equals(assgn.getOperator())) return false;
+
+      return visitExpression(assgn.getRight());
+    }
+    return false;
+  }
+
+  /** Is {@code e} an identifier with name {@code name}? */
+  protected static boolean isIdentifier(Expression e, String name) {
+    return e instanceof Identifier && name.equals(((Identifier) e).getName());
+  }
+}

javascript/extractor/src/com/semmle/js/extractor/ES2015Detector.java

@@ -0,0 +1,34 @@
+package com.semmle.js.extractor;
+
+import com.semmle.js.ast.DynamicImport;
+import com.semmle.js.ast.ExportDeclaration;
+import com.semmle.js.ast.Expression;
+import com.semmle.js.ast.ImportDeclaration;
+import com.semmle.js.ast.Node;
+import com.semmle.js.ast.Statement;
+
+/** A utility class for detecting Node.js code. */
+public class ES2015Detector extends AbstractDetector {
+  /**
+   * Is {@code ast} a program that uses ES2015 import/export code?
+   */
+  public static boolean looksLikeES2015(Node ast) {
+    return new ES2015Detector().programDetection(ast);
+  }
+
+  @Override
+  protected boolean visitStatement(Statement stmt) {
+    if (stmt instanceof ImportDeclaration || stmt instanceof ExportDeclaration) {
+      return true;
+    }
+    return super.visitStatement(stmt);
+  }
+
+  @Override
+  protected boolean visitExpression(Expression e) {
+    if (e instanceof DynamicImport) {
+      return true;
+    }
+    return super.visitExpression(e);
+  }
+}

javascript/extractor/src/com/semmle/js/extractor/JSExtractor.java

@@ -58,6 +58,26 @@ public class JSExtractor {
 
     JSParser.Result parserRes =
         JSParser.parse(config, sourceType, source, textualExtractor.getMetrics());
+
+    // Check if we guessed wrong with the regex in `establishSourceType`, (which could
+    // happen due to a block-comment line starting with '  import').
+    if (config.getSourceType() == SourceType.AUTO && sourceType != SourceType.SCRIPT) {
+      boolean wrongGuess = false;
+
+      if (sourceType == SourceType.MODULE) {
+        // check that we did see an import/export declaration
+        wrongGuess = ES2015Detector.looksLikeES2015(parserRes.getAST()) == false;
+      } else if (sourceType == SourceType.CLOSURE_MODULE ) {
+        // TODO
+      }
+
+      if (wrongGuess) {
+        sourceType = SourceType.SCRIPT;
+        parserRes =
+          JSParser.parse(config, sourceType, source, textualExtractor.getMetrics());
+      }
+    }
+
     return extract(textualExtractor, source, toplevelKind, scopeManager, sourceType, parserRes);
   }
 

javascript/extractor/src/com/semmle/js/extractor/NodeJSDetector.java

@@ -1,132 +1,30 @@
 package com.semmle.js.extractor;
 
 import com.semmle.js.ast.AssignmentExpression;
-import com.semmle.js.ast.BlockStatement;
 import com.semmle.js.ast.CallExpression;
 import com.semmle.js.ast.Expression;
-import com.semmle.js.ast.ExpressionStatement;
-import com.semmle.js.ast.IFunction;
-import com.semmle.js.ast.Identifier;
-import com.semmle.js.ast.IfStatement;
 import com.semmle.js.ast.MemberExpression;
 import com.semmle.js.ast.Node;
-import com.semmle.js.ast.ParenthesizedExpression;
-import com.semmle.js.ast.Program;
-import com.semmle.js.ast.Statement;
-import com.semmle.js.ast.TryStatement;
-import com.semmle.js.ast.UnaryExpression;
-import com.semmle.js.ast.VariableDeclaration;
-import com.semmle.js.ast.VariableDeclarator;
-import java.util.List;
 
 /** A utility class for detecting Node.js code. */
-public class NodeJSDetector {
+public class NodeJSDetector extends AbstractDetector {
   /**
    * Is {@code ast} a program that looks like Node.js code, that is, does it contain a top-level
-   * {@code require} or an export?
+   * {@code require} or an {@code module.exports = ...}/{@code exports = ...}?
    */
   public static boolean looksLikeNodeJS(Node ast) {
-    if (!(ast instanceof Program)) return false;
-
-    return hasToplevelRequireOrExport(((Program) ast).getBody());
+    return new NodeJSDetector().programDetection(ast);
   }
 
-  /**
-   * Does this program contain a statement that looks like a Node.js {@code require} or an export?
-   *
-   * <p>We recursively traverse argument-less immediately invoked function expressions (i.e., no UMD
-   * modules), but not loops or if statements.
-   */
-  private static boolean hasToplevelRequireOrExport(List<Statement> stmts) {
-    for (Statement stmt : stmts) if (hasToplevelRequireOrExport(stmt)) return true;
-    return false;
-  }
-
-  private static boolean hasToplevelRequireOrExport(Statement stmt) {
-    if (stmt instanceof ExpressionStatement) {
-      Expression e = stripParens(((ExpressionStatement) stmt).getExpression());
-
-      // check whether `e` is an iife; if so, recursively check its body
-
-      // strip off unary operators to handle `!function(){...}()`
-      if (e instanceof UnaryExpression) e = ((UnaryExpression) e).getArgument();
-
-      if (e instanceof CallExpression && ((CallExpression) e).getArguments().isEmpty()) {
-        Expression callee = stripParens(((CallExpression) e).getCallee());
-        if (callee instanceof IFunction) {
-          Node body = ((IFunction) callee).getBody();
-          if (body instanceof BlockStatement)
-            return hasToplevelRequireOrExport(((BlockStatement) body).getBody());
-        }
-      }
-
-      if (isRequireCall(e) || isExport(e)) return true;
-
-    } else if (stmt instanceof VariableDeclaration) {
-      for (VariableDeclarator decl : ((VariableDeclaration) stmt).getDeclarations()) {
-        Expression init = stripParens(decl.getInit());
-        if (isRequireCall(init) || isExport(init)) return true;
-      }
-
-    } else if (stmt instanceof TryStatement) {
-      return hasToplevelRequireOrExport(((TryStatement) stmt).getBlock());
-
-    } else if (stmt instanceof BlockStatement) {
-      return hasToplevelRequireOrExport(((BlockStatement) stmt).getBody());
-
-    } else if (stmt instanceof IfStatement) {
-      IfStatement is = (IfStatement) stmt;
-      return hasToplevelRequireOrExport(is.getConsequent())
-          || hasToplevelRequireOrExport(is.getAlternate());
-    }
-
-    return false;
-  }
-
-  private static Expression stripParens(Expression e) {
-    if (e instanceof ParenthesizedExpression)
-      return stripParens(((ParenthesizedExpression) e).getExpression());
-    return e;
-  }
-
-  /**
-   * Is {@code e} a call to a function named {@code require} with one argument, or an assignment
-   * whose right hand side is the result of such a call?
-   */
-  private static boolean isRequireCall(Expression e) {
+  @Override
+  protected boolean visitExpression(Expression e) {
+    // require('...')
     if (e instanceof CallExpression) {
       CallExpression call = (CallExpression) e;
       Expression callee = call.getCallee();
       if (isIdentifier(callee, "require") && call.getArguments().size() == 1) return true;
-      if (isRequireCall(callee)) return true;
-      return false;
-    } else if (e instanceof MemberExpression) {
-      return isRequireCall(((MemberExpression) e).getObject());
-    } else if (e instanceof AssignmentExpression) {
-      AssignmentExpression assgn = (AssignmentExpression) e;
-
-      // filter out compound assignments
-      if (!"=".equals(assgn.getOperator())) return false;
-
-      return isRequireCall(assgn.getRight());
     }
-    return false;
-  }
 
-  /**
-   * Does {@code e} look like a Node.js export?
-   *
-   * <p>Currently, three kinds of exports are recognised:
-   *
-   * <ul>
-   *   <li><code>exports.foo = ...</code>
-   *   <li><code>module.exports = ...</code>
-   *   <li><code>module.exports.foo = ...</code>
-   * </ul>
-   *
-   * Detection is done recursively, so <code>foo = exports.foo = ...</code> is handled correctly.
-   */
-  private static boolean isExport(Expression e) {
     if (e instanceof AssignmentExpression) {
       AssignmentExpression assgn = (AssignmentExpression) e;
 
@@ -149,12 +47,9 @@ public class NodeJSDetector {
           if (isModuleExports(targetBase)) return true;
         }
       }
-
-      // recursively check right hand side
-      return isExport(assgn.getRight());
     }
 
-    return false;
+    return super.visitExpression(e);
   }
 
   /** Is {@code me} a member expression {@code module.exports}? */
@@ -163,9 +58,4 @@ public class NodeJSDetector {
         && isIdentifier(me.getObject(), "module")
         && isIdentifier(me.getProperty(), "exports");
   }
-
-  /** Is {@code e} an identifier with name {@code name}? */
-  private static boolean isIdentifier(Expression e, String name) {
-    return e instanceof Identifier && name.equals(((Identifier) e).getName());
-  }
 }

javascript/extractor/test/com/semmle/js/extractor/test/AllTests.java

@@ -18,6 +18,7 @@ import org.junit.runners.Suite.SuiteClasses;
 @SuiteClasses({
   JSXTests.class,
   NodeJSDetectorTests.class,
+  ES2015DetectorTests.class,
   TrapTests.class,
   ObjectRestSpreadTests.class,
   ClassPropertiesTests.class,

javascript/extractor/test/com/semmle/js/extractor/test/BUILD.bazel

@@ -12,6 +12,9 @@ java_test(
         "TS_WRAPPER_ZIP": "$(rlocationpath //javascript/extractor/lib/typescript)",
     },
     test_class = "com.semmle.js.extractor.test.AllTests",
+    # To use `replaceExpectedOutput` you need to uncomment the following line
+    # (to be allowed to override the .trap files on disk)
+    # tags = ["no-sandbox"],
     deps = [
         "//javascript/extractor",
         "//javascript/extractor:deps",

javascript/extractor/test/com/semmle/js/extractor/test/ES2015DetectorTests.java

@@ -0,0 +1,58 @@
+package com.semmle.js.extractor.test;
+
+import com.semmle.js.ast.Node;
+import com.semmle.js.extractor.ES2015Detector;
+import com.semmle.js.extractor.ExtractionMetrics;
+import com.semmle.js.extractor.ExtractorConfig;
+import com.semmle.js.extractor.ExtractorConfig.SourceType;
+import com.semmle.js.parser.JSParser;
+import com.semmle.js.parser.JSParser.Result;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class ES2015DetectorTests {
+  // using `experimental: true` as we do in real extractor, see `extractSource` method
+  // in `AutoBuild.java`
+  private static final ExtractorConfig CONFIG = new ExtractorConfig(true);
+
+  private void isES2015(String src, boolean expected) {
+    Result res = JSParser.parse(CONFIG, SourceType.MODULE, src, new ExtractionMetrics());
+    Node ast = res.getAST();
+    Assert.assertNotNull(ast);
+    Assert.assertTrue(ES2015Detector.looksLikeES2015(ast) == expected);
+  }
+
+  @Test
+  public void testImport() {
+    isES2015("import * as fs from 'fs';", true);
+  }
+
+  @Test
+  public void testExport() {
+    isES2015("export function foo() { };", true);
+  }
+
+  @Test
+  public void testDynamicImport() {
+    isES2015("import('fs');", true);
+  }
+
+  @Test
+  public void testDynamicImportAssign() {
+    isES2015("var fs = import('fs');", true);
+  }
+
+  @Test
+  public void testDynamicImportThen() {
+    isES2015("import('o').then((o) => {});", true);
+  }
+
+  @Test
+  public void importInBlockComment() {
+    isES2015("/*\n"
+      + "  import * from 'fs';\n"
+      + "*/\n"
+      + "const fs = require('fs');",
+      false);
+  }
+}

javascript/extractor/test/com/semmle/js/extractor/test/NodeJSDetectorTests.java

@@ -11,7 +11,9 @@ import org.junit.Assert;
 import org.junit.Test;
 
 public class NodeJSDetectorTests {
-  private static final ExtractorConfig CONFIG = new ExtractorConfig(false);
+  // using `experimental: true` as we do in real extractor, see `extractSource` method
+  // in `AutoBuild.java`
+  private static final ExtractorConfig CONFIG = new ExtractorConfig(true);
 
   private void isNodeJS(String src, boolean expected) {
     Result res = JSParser.parse(CONFIG, SourceType.SCRIPT, src, new ExtractionMetrics());

javascript/extractor/test/com/semmle/js/extractor/test/TrapTests.java

@@ -161,6 +161,14 @@ public class TrapTests {
                         byte[] actual_utf8_bytes = StringUtil.stringToBytes(sw.toString());
                         String actual = new String(actual_utf8_bytes, Charset.forName("UTF-8"));
                         File trap = new File(outputDir, f.getName() + ".trap");
+                        // NOTE: If you want to replace expected output, you MUST change
+                        // the way this test is run under bazel to escape the bazel
+                        // sandbox. Add `tags = ["no-sandbox"]` to the test rule in
+                        // javascript/extractor/test/com/semmle/js/extractor/test/BUILD.bazel
+                        //
+                        // if you have problems with too much caching, you need to find the right bazel command,
+                        // and run `./build --bazel test ... --cache_test_results=no`
+                        // (at least I had problems getting the "no-cache" tag to work)
                         boolean replaceExpectedOutput = false;
                         if (replaceExpectedOutput) {
                           System.out.println("Replacing expected output for " + trap);

javascript/extractor/tests/flow/output/trap/anonFunctionWithoutParens.js.trap

@@ -72,20 +72,14 @@ locations_default(#20025,#10000,1,40,1,39)
 hasLocation(#20024,#20025)
 toplevels(#20001,0)
 hasLocation(#20001,#20003)
-#20026=@"module;{#10000},1,1"
-scopes(#20026,3)
-scopenodes(#20001,#20026)
-scopenesting(#20026,#20000)
-is_module(#20001)
-is_es2015_module(#20001)
-#20027=*
-entry_cfg_node(#20027,#20001)
-#20028=@"loc,{#10000},1,1,1,0"
-locations_default(#20028,#10000,1,1,1,0)
-hasLocation(#20027,#20028)
-#20029=*
-exit_cfg_node(#20029,#20001)
-hasLocation(#20029,#20025)
-successor(#20027,#20029)
+#20026=*
+entry_cfg_node(#20026,#20001)
+#20027=@"loc,{#10000},1,1,1,0"
+locations_default(#20027,#10000,1,1,1,0)
+hasLocation(#20026,#20027)
+#20028=*
+exit_cfg_node(#20028,#20001)
+hasLocation(#20028,#20025)
+successor(#20026,#20028)
 numlines(#10000,1,1,0)
 filetype(#10000,"javascript")

javascript/extractor/tests/flow/output/trap/anonIndexer.js.trap

@@ -85,20 +85,14 @@ toplevels(#20001,0)
 #20030=@"loc,{#10000},1,1,2,0"
 locations_default(#20030,#10000,1,1,2,0)
 hasLocation(#20001,#20030)
-#20031=@"module;{#10000},1,1"
-scopes(#20031,3)
-scopenodes(#20001,#20031)
-scopenesting(#20031,#20000)
-is_module(#20001)
-is_es2015_module(#20001)
-#20032=*
-entry_cfg_node(#20032,#20001)
-#20033=@"loc,{#10000},1,1,1,0"
-locations_default(#20033,#10000,1,1,1,0)
-hasLocation(#20032,#20033)
-#20034=*
-exit_cfg_node(#20034,#20001)
-hasLocation(#20034,#20029)
-successor(#20032,#20034)
+#20031=*
+entry_cfg_node(#20031,#20001)
+#20032=@"loc,{#10000},1,1,1,0"
+locations_default(#20032,#10000,1,1,1,0)
+hasLocation(#20031,#20032)
+#20033=*
+exit_cfg_node(#20033,#20001)
+hasLocation(#20033,#20029)
+successor(#20031,#20033)
 numlines(#10000,1,1,0)
 filetype(#10000,"javascript")

javascript/extractor/tests/flow/output/trap/declared-module-imports.js.trap

@@ -128,20 +128,14 @@ toplevels(#20001,0)
 #20045=@"loc,{#10000},1,1,5,0"
 locations_default(#20045,#10000,1,1,5,0)
 hasLocation(#20001,#20045)
-#20046=@"module;{#10000},1,1"
-scopes(#20046,3)
-scopenodes(#20001,#20046)
-scopenesting(#20046,#20000)
-is_module(#20001)
-is_es2015_module(#20001)
-#20047=*
-entry_cfg_node(#20047,#20001)
-#20048=@"loc,{#10000},1,1,1,0"
-locations_default(#20048,#10000,1,1,1,0)
-hasLocation(#20047,#20048)
-#20049=*
-exit_cfg_node(#20049,#20001)
-hasLocation(#20049,#20044)
-successor(#20047,#20049)
+#20046=*
+entry_cfg_node(#20046,#20001)
+#20047=@"loc,{#10000},1,1,1,0"
+locations_default(#20047,#10000,1,1,1,0)
+hasLocation(#20046,#20047)
+#20048=*
+exit_cfg_node(#20048,#20001)
+hasLocation(#20048,#20044)
+successor(#20046,#20048)
 numlines(#10000,4,4,0)
 filetype(#10000,"javascript")

javascript/extractor/tests/flow/output/trap/exportOpaqueType.js.trap

@@ -70,20 +70,14 @@ toplevels(#20001,0)
 #20024=@"loc,{#10000},1,1,2,0"
 locations_default(#20024,#10000,1,1,2,0)
 hasLocation(#20001,#20024)
-#20025=@"module;{#10000},1,1"
-scopes(#20025,3)
-scopenodes(#20001,#20025)
-scopenesting(#20025,#20000)
-is_module(#20001)
-is_es2015_module(#20001)
-#20026=*
-entry_cfg_node(#20026,#20001)
-#20027=@"loc,{#10000},1,1,1,0"
-locations_default(#20027,#10000,1,1,1,0)
-hasLocation(#20026,#20027)
-#20028=*
-exit_cfg_node(#20028,#20001)
-hasLocation(#20028,#20023)
-successor(#20026,#20028)
+#20025=*
+entry_cfg_node(#20025,#20001)
+#20026=@"loc,{#10000},1,1,1,0"
+locations_default(#20026,#10000,1,1,1,0)
+hasLocation(#20025,#20026)
+#20027=*
+exit_cfg_node(#20027,#20001)
+hasLocation(#20027,#20023)
+successor(#20025,#20027)
 numlines(#10000,1,1,0)
 filetype(#10000,"javascript")

javascript/extractor/tests/flow/output/trap/importTypeInDeclaredModule.js.trap

@@ -205,20 +205,14 @@ toplevels(#20001,0)
 #20071=@"loc,{#10000},1,1,11,2"
 locations_default(#20071,#10000,1,1,11,2)
 hasLocation(#20001,#20071)
-#20072=@"module;{#10000},1,1"
-scopes(#20072,3)
-scopenodes(#20001,#20072)
-scopenesting(#20072,#20000)
-is_module(#20001)
-is_es2015_module(#20001)
-#20073=*
-entry_cfg_node(#20073,#20001)
-#20074=@"loc,{#10000},1,1,1,0"
-locations_default(#20074,#10000,1,1,1,0)
-hasLocation(#20073,#20074)
-#20075=*
-exit_cfg_node(#20075,#20001)
-hasLocation(#20075,#20070)
-successor(#20073,#20075)
+#20072=*
+entry_cfg_node(#20072,#20001)
+#20073=@"loc,{#10000},1,1,1,0"
+locations_default(#20073,#10000,1,1,1,0)
+hasLocation(#20072,#20073)
+#20074=*
+exit_cfg_node(#20074,#20001)
+hasLocation(#20074,#20070)
+successor(#20072,#20074)
 numlines(#10000,11,10,0)
 filetype(#10000,"javascript")

javascript/extractor/tests/flow/output/trap/variance.js.trap

@@ -160,95 +160,89 @@ toplevels(#20001,0)
 #20057=@"loc,{#10000},1,1,7,1"
 locations_default(#20057,#10000,1,1,7,1)
 hasLocation(#20001,#20057)
-#20058=@"module;{#10000},1,1"
-scopes(#20058,3)
-scopenodes(#20001,#20058)
-scopenesting(#20058,#20000)
-is_module(#20001)
-is_es2015_module(#20001)
-#20059=@"var;{Foo};{#20058}"
-variables(#20059,"Foo",#20058)
-#20060=@"local_type_name;{Foo};{#20058}"
-local_type_names(#20060,"Foo",#20058)
-#20061=*
-stmts(#20061,26,#20001,0,"class F ... : int\n}")
-#20062=@"loc,{#10000},5,1,7,1"
-locations_default(#20062,#10000,5,1,7,1)
-hasLocation(#20061,#20062)
-stmt_containers(#20061,#20001)
+#20058=@"var;{Foo};{#20000}"
+variables(#20058,"Foo",#20000)
+#20059=@"local_type_name;{Foo};{#20000}"
+local_type_names(#20059,"Foo",#20000)
+#20060=*
+stmts(#20060,26,#20001,0,"class F ... : int\n}")
+#20061=@"loc,{#10000},5,1,7,1"
+locations_default(#20061,#10000,5,1,7,1)
+hasLocation(#20060,#20061)
+stmt_containers(#20060,#20001)
+#20062=*
+exprs(#20062,78,#20060,0,"Foo")
+hasLocation(#20062,#20043)
+enclosing_stmt(#20062,#20060)
+expr_containers(#20062,#20001)
+literals("Foo","Foo",#20062)
+decl(#20062,#20058)
+typedecl(#20062,#20059)
 #20063=*
-exprs(#20063,78,#20061,0,"Foo")
-hasLocation(#20063,#20043)
-enclosing_stmt(#20063,#20061)
-expr_containers(#20063,#20001)
-literals("Foo","Foo",#20063)
-decl(#20063,#20059)
-typedecl(#20063,#20060)
+scopes(#20063,10)
+scopenodes(#20060,#20063)
+scopenesting(#20063,#20000)
 #20064=*
-scopes(#20064,10)
-scopenodes(#20061,#20064)
-scopenesting(#20064,#20058)
-#20065=*
-properties(#20065,#20061,2,8,"+x: int")
-#20066=@"loc,{#10000},6,3,6,9"
-locations_default(#20066,#10000,6,3,6,9)
-hasLocation(#20065,#20066)
+properties(#20064,#20060,2,8,"+x: int")
+#20065=@"loc,{#10000},6,3,6,9"
+locations_default(#20065,#10000,6,3,6,9)
+hasLocation(#20064,#20065)
+#20066=*
 #20067=*
+exprs(#20067,0,#20064,0,"x")
+hasLocation(#20067,#20049)
+expr_containers(#20067,#20066)
+literals("x","x",#20067)
 #20068=*
-exprs(#20068,0,#20065,0,"x")
-hasLocation(#20068,#20049)
-expr_containers(#20068,#20067)
-literals("x","x",#20068)
-#20069=*
-properties(#20069,#20061,3,0,"constructor() {}")
-#20070=@"loc,{#10000},5,11,5,10"
-locations_default(#20070,#10000,5,11,5,10)
-hasLocation(#20069,#20070)
+properties(#20068,#20060,3,0,"constructor() {}")
+#20069=@"loc,{#10000},5,11,5,10"
+locations_default(#20069,#10000,5,11,5,10)
+hasLocation(#20068,#20069)
+#20070=*
+exprs(#20070,0,#20068,0,"constructor")
+hasLocation(#20070,#20069)
+enclosing_stmt(#20070,#20060)
+expr_containers(#20070,#20001)
+literals("constructor","constructor",#20070)
+exprs(#20066,9,#20068,1,"() {}")
+hasLocation(#20066,#20069)
+enclosing_stmt(#20066,#20060)
+expr_containers(#20066,#20001)
 #20071=*
-exprs(#20071,0,#20069,0,"constructor")
-hasLocation(#20071,#20070)
-enclosing_stmt(#20071,#20061)
-expr_containers(#20071,#20001)
-literals("constructor","constructor",#20071)
-exprs(#20067,9,#20069,1,"() {}")
-hasLocation(#20067,#20070)
-enclosing_stmt(#20067,#20061)
-expr_containers(#20067,#20001)
-#20072=*
-scopes(#20072,1)
-scopenodes(#20067,#20072)
-scopenesting(#20072,#20064)
-#20073=@"var;{arguments};{#20072}"
-variables(#20073,"arguments",#20072)
-is_arguments_object(#20073)
+scopes(#20071,1)
+scopenodes(#20066,#20071)
+scopenesting(#20071,#20063)
+#20072=@"var;{arguments};{#20071}"
+variables(#20072,"arguments",#20071)
+is_arguments_object(#20072)
+#20073=*
+stmts(#20073,1,#20066,-2,"{}")
+hasLocation(#20073,#20069)
+stmt_containers(#20073,#20066)
+is_method(#20068)
 #20074=*
-stmts(#20074,1,#20067,-2,"{}")
-hasLocation(#20074,#20070)
-stmt_containers(#20074,#20067)
-is_method(#20069)
-#20075=*
-entry_cfg_node(#20075,#20001)
-#20076=@"loc,{#10000},1,1,1,0"
-locations_default(#20076,#10000,1,1,1,0)
-hasLocation(#20075,#20076)
+entry_cfg_node(#20074,#20001)
+#20075=@"loc,{#10000},1,1,1,0"
+locations_default(#20075,#10000,1,1,1,0)
+hasLocation(#20074,#20075)
+#20076=*
+exit_cfg_node(#20076,#20001)
+hasLocation(#20076,#20056)
+successor(#20067,#20064)
+successor(#20066,#20068)
 #20077=*
-exit_cfg_node(#20077,#20001)
-hasLocation(#20077,#20056)
-successor(#20068,#20065)
-successor(#20067,#20069)
+entry_cfg_node(#20077,#20066)
+hasLocation(#20077,#20069)
+successor(#20064,#20073)
 #20078=*
-entry_cfg_node(#20078,#20067)
-hasLocation(#20078,#20070)
-successor(#20065,#20074)
-#20079=*
-exit_cfg_node(#20079,#20067)
-hasLocation(#20079,#20070)
-successor(#20074,#20079)
-successor(#20078,#20068)
-successor(#20071,#20067)
-successor(#20069,#20061)
-successor(#20063,#20071)
-successor(#20061,#20077)
-successor(#20075,#20063)
+exit_cfg_node(#20078,#20066)
+hasLocation(#20078,#20069)
+successor(#20073,#20078)
+successor(#20077,#20067)
+successor(#20070,#20066)
+successor(#20068,#20060)
+successor(#20062,#20070)
+successor(#20060,#20076)
+successor(#20074,#20062)
 numlines(#10000,7,6,0)
 filetype(#10000,"javascript")

javascript/extractor/tests/node/input/detection.js

@@ -0,0 +1,6 @@
+// the comment below (with 'import' on line starting with whitespace) caused the
+// extractor to think it was a es2015 module and not a commonjs module.
+/*
+  import
+*/
+const fs = require('fs');

javascript/extractor/tests/node/output/trap/detection.js.trap

@@ -0,0 +1,203 @@
+#10000=@"/detection.js;sourcefile"
+files(#10000,"/detection.js")
+#10001=@"/;folder"
+folders(#10001,"/")
+containerparent(#10001,#10000)
+#10002=@"loc,{#10000},0,0,0,0"
+locations_default(#10002,#10000,0,0,0,0)
+hasLocation(#10000,#10002)
+#20000=@"global_scope"
+scopes(#20000,0)
+#20001=@"script;{#10000},1,1"
+#20002=*
+comments(#20002,0,#20001," the comment below (with 'import' on line starting with whitespace) caused the","// the  ... sed the")
+#20003=@"loc,{#10000},1,1,1,80"
+locations_default(#20003,#10000,1,1,1,80)
+hasLocation(#20002,#20003)
+#20004=*
+comments(#20004,0,#20001," extractor to think it was a es2015 module and not a commonjs module.","// extr ... module.")
+#20005=@"loc,{#10000},2,1,2,71"
+locations_default(#20005,#10000,2,1,2,71)
+hasLocation(#20004,#20005)
+#20006=*
+comments(#20006,1,#20001,"
+  import
+","/*\n  import\n*/")
+#20007=@"loc,{#10000},3,1,5,2"
+locations_default(#20007,#10000,3,1,5,2)
+hasLocation(#20006,#20007)
+#20008=*
+lines(#20008,#20001,"// the comment below (with 'import' on line starting with whitespace) caused the","
+")
+hasLocation(#20008,#20003)
+#20009=*
+lines(#20009,#20001,"// extractor to think it was a es2015 module and not a commonjs module.","
+")
+hasLocation(#20009,#20005)
+#20010=*
+lines(#20010,#20001,"/*","
+")
+#20011=@"loc,{#10000},3,1,3,2"
+locations_default(#20011,#10000,3,1,3,2)
+hasLocation(#20010,#20011)
+#20012=*
+lines(#20012,#20001,"  import","
+")
+#20013=@"loc,{#10000},4,1,4,8"
+locations_default(#20013,#10000,4,1,4,8)
+hasLocation(#20012,#20013)
+indentation(#10000,4," ",2)
+#20014=*
+lines(#20014,#20001,"*/","
+")
+#20015=@"loc,{#10000},5,1,5,2"
+locations_default(#20015,#10000,5,1,5,2)
+hasLocation(#20014,#20015)
+#20016=*
+lines(#20016,#20001,"const fs = require('fs');","
+")
+#20017=@"loc,{#10000},6,1,6,25"
+locations_default(#20017,#10000,6,1,6,25)
+hasLocation(#20016,#20017)
+numlines(#20001,6,1,5)
+#20018=*
+tokeninfo(#20018,7,#20001,0,"const")
+#20019=@"loc,{#10000},6,1,6,5"
+locations_default(#20019,#10000,6,1,6,5)
+hasLocation(#20018,#20019)
+next_token(#20002,#20018)
+next_token(#20004,#20018)
+next_token(#20006,#20018)
+#20020=*
+tokeninfo(#20020,6,#20001,1,"fs")
+#20021=@"loc,{#10000},6,7,6,8"
+locations_default(#20021,#10000,6,7,6,8)
+hasLocation(#20020,#20021)
+#20022=*
+tokeninfo(#20022,8,#20001,2,"=")
+#20023=@"loc,{#10000},6,10,6,10"
+locations_default(#20023,#10000,6,10,6,10)
+hasLocation(#20022,#20023)
+#20024=*
+tokeninfo(#20024,6,#20001,3,"require")
+#20025=@"loc,{#10000},6,12,6,18"
+locations_default(#20025,#10000,6,12,6,18)
+hasLocation(#20024,#20025)
+#20026=*
+tokeninfo(#20026,8,#20001,4,"(")
+#20027=@"loc,{#10000},6,19,6,19"
+locations_default(#20027,#10000,6,19,6,19)
+hasLocation(#20026,#20027)
+#20028=*
+tokeninfo(#20028,4,#20001,5,"'fs'")
+#20029=@"loc,{#10000},6,20,6,23"
+locations_default(#20029,#10000,6,20,6,23)
+hasLocation(#20028,#20029)
+#20030=*
+tokeninfo(#20030,8,#20001,6,")")
+#20031=@"loc,{#10000},6,24,6,24"
+locations_default(#20031,#10000,6,24,6,24)
+hasLocation(#20030,#20031)
+#20032=*
+tokeninfo(#20032,8,#20001,7,";")
+#20033=@"loc,{#10000},6,25,6,25"
+locations_default(#20033,#10000,6,25,6,25)
+hasLocation(#20032,#20033)
+#20034=*
+tokeninfo(#20034,0,#20001,8,"")
+#20035=@"loc,{#10000},7,1,7,0"
+locations_default(#20035,#10000,7,1,7,0)
+hasLocation(#20034,#20035)
+toplevels(#20001,0)
+#20036=@"loc,{#10000},1,1,7,0"
+locations_default(#20036,#10000,1,1,7,0)
+hasLocation(#20001,#20036)
+#20037=@"var;{global};{#20000}"
+variables(#20037,"global",#20000)
+#20038=@"var;{process};{#20000}"
+variables(#20038,"process",#20000)
+#20039=@"var;{console};{#20000}"
+variables(#20039,"console",#20000)
+#20040=@"var;{Buffer};{#20000}"
+variables(#20040,"Buffer",#20000)
+#20041=@"module;{#10000},1,1"
+scopes(#20041,3)
+scopenodes(#20001,#20041)
+scopenesting(#20041,#20000)
+#20042=@"var;{require};{#20041}"
+variables(#20042,"require",#20041)
+#20043=@"var;{module};{#20041}"
+variables(#20043,"module",#20041)
+#20044=@"var;{exports};{#20041}"
+variables(#20044,"exports",#20041)
+#20045=@"var;{__filename};{#20041}"
+variables(#20045,"__filename",#20041)
+#20046=@"var;{__dirname};{#20041}"
+variables(#20046,"__dirname",#20041)
+#20047=@"var;{arguments};{#20041}"
+variables(#20047,"arguments",#20041)
+is_module(#20001)
+#20048=@"var;{fs};{#20041}"
+variables(#20048,"fs",#20041)
+#20049=*
+stmts(#20049,22,#20001,0,"const f ... ('fs');")
+hasLocation(#20049,#20017)
+stmt_containers(#20049,#20001)
+#20050=*
+exprs(#20050,64,#20049,0,"fs = require('fs')")
+#20051=@"loc,{#10000},6,7,6,24"
+locations_default(#20051,#10000,6,7,6,24)
+hasLocation(#20050,#20051)
+enclosing_stmt(#20050,#20049)
+expr_containers(#20050,#20001)
+#20052=*
+exprs(#20052,78,#20050,0,"fs")
+hasLocation(#20052,#20021)
+enclosing_stmt(#20052,#20049)
+expr_containers(#20052,#20001)
+literals("fs","fs",#20052)
+decl(#20052,#20048)
+#20053=*
+exprs(#20053,13,#20050,1,"require('fs')")
+#20054=@"loc,{#10000},6,12,6,24"
+locations_default(#20054,#10000,6,12,6,24)
+hasLocation(#20053,#20054)
+enclosing_stmt(#20053,#20049)
+expr_containers(#20053,#20001)
+#20055=*
+exprs(#20055,79,#20053,-1,"require")
+hasLocation(#20055,#20025)
+enclosing_stmt(#20055,#20049)
+expr_containers(#20055,#20001)
+literals("require","require",#20055)
+bind(#20055,#20042)
+#20056=*
+exprs(#20056,4,#20053,0,"'fs'")
+hasLocation(#20056,#20029)
+enclosing_stmt(#20056,#20049)
+expr_containers(#20056,#20001)
+literals("fs","'fs'",#20056)
+#20057=*
+regexpterm(#20057,14,#20056,0,"fs")
+#20058=@"loc,{#10000},6,21,6,22"
+locations_default(#20058,#10000,6,21,6,22)
+hasLocation(#20057,#20058)
+regexp_const_value(#20057,"fs")
+#20059=*
+entry_cfg_node(#20059,#20001)
+#20060=@"loc,{#10000},1,1,1,0"
+locations_default(#20060,#10000,1,1,1,0)
+hasLocation(#20059,#20060)
+#20061=*
+exit_cfg_node(#20061,#20001)
+hasLocation(#20061,#20035)
+successor(#20049,#20052)
+successor(#20056,#20053)
+successor(#20055,#20056)
+successor(#20053,#20050)
+successor(#20052,#20055)
+successor(#20050,#20061)
+successor(#20059,#20049)
+is_nodejs(#20001)
+numlines(#10000,6,1,5)
+filetype(#10000,"javascript")

javascript/ql/lib/change-notes/2024-04-02-more-robust-commonjs-detection.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Improved detection of whether a file uses CommonJS module system.