Merge branch 'main' into ts4

javascript/extractor/src/com/semmle/jcorn/jsx/JSXParser.java

@@ -35,6 +35,7 @@ import com.semmle.js.ast.jsx.JSXMemberExpression;
 import com.semmle.js.ast.jsx.JSXNamespacedName;
 import com.semmle.js.ast.jsx.JSXOpeningElement;
 import com.semmle.js.ast.jsx.JSXSpreadAttribute;
+import com.semmle.js.ast.jsx.JSXThisExpr;
 import com.semmle.util.data.Either;
 import java.util.ArrayList;
 import java.util.List;
@@ -225,22 +226,26 @@ public class JSXParser extends Parser {
   }
 
   /** Parse next token as JSX identifier */
-  private JSXIdentifier jsx_parseIdentifier() {
+  private IJSXName jsx_parseIdentifier(boolean expectThisExpr) {
     SourceLocation loc = new SourceLocation(this.startLoc);
     String name = null;
     if (this.type == jsxName) name = String.valueOf(this.value);
     else if (this.type.keyword != null) name = this.type.keyword;
     else this.unexpected();
     this.next();
-    return this.finishNode(new JSXIdentifier(loc, name));
+    if (expectThisExpr && name.equals("this")) {
+      return this.finishNode(new JSXThisExpr(loc));
+    } else {
+      return this.finishNode(new JSXIdentifier(loc, name));
+    }
   }
 
   /** Parse namespaced identifier. */
   private IJSXName jsx_parseNamespacedName() {
     SourceLocation loc = new SourceLocation(this.startLoc);
-    JSXIdentifier namespace = this.jsx_parseIdentifier();
-    if (!((JSXOptions) options).allowNamespaces || !this.eat(colon)) return namespace;
-    return this.finishNode(new JSXNamespacedName(loc, namespace, this.jsx_parseIdentifier()));
+    IJSXName namespace = this.jsx_parseIdentifier(true);
+    if (namespace instanceof JSXThisExpr || (!((JSXOptions) options).allowNamespaces || !this.eat(colon))) return namespace;
+    return this.finishNode(new JSXNamespacedName(loc, (JSXIdentifier)namespace, (JSXIdentifier)this.jsx_parseIdentifier(false)));
   }
 
   /**
@@ -258,7 +263,7 @@ public class JSXParser extends Parser {
     }
     while (this.eat(dot)) {
       SourceLocation loc = new SourceLocation(startPos);
-      node = this.finishNode(new JSXMemberExpression(loc, node, this.jsx_parseIdentifier()));
+      node = this.finishNode(new JSXMemberExpression(loc, node, (JSXIdentifier)this.jsx_parseIdentifier(false)));
     }
     return node;
   }

javascript/extractor/src/com/semmle/js/ast/AST2JSON.java

@@ -15,6 +15,7 @@ import com.semmle.js.ast.jsx.JSXMemberExpression;
 import com.semmle.js.ast.jsx.JSXNamespacedName;
 import com.semmle.js.ast.jsx.JSXOpeningElement;
 import com.semmle.js.ast.jsx.JSXSpreadAttribute;
+import com.semmle.js.ast.jsx.JSXThisExpr;
 import com.semmle.ts.ast.ExportWholeDeclaration;
 import com.semmle.ts.ast.ExternalModuleReference;
 import com.semmle.ts.ast.ImportWholeDeclaration;
@@ -635,6 +636,11 @@ public class AST2JSON extends DefaultVisitor<Void, JsonElement> {
     return result;
   }
 
+  @Override
+  public JsonElement visit(JSXThisExpr nd, Void c) {
+    return this.mkNode(nd, "JSXThisExpr");
+  }
+
   @Override
   public JsonElement visit(JSXMemberExpression nd, Void c) {
     JsonObject result = this.mkNode(nd);

javascript/extractor/src/com/semmle/js/ast/DefaultVisitor.java

@@ -14,6 +14,7 @@ import com.semmle.js.ast.jsx.JSXMemberExpression;
 import com.semmle.js.ast.jsx.JSXNamespacedName;
 import com.semmle.js.ast.jsx.JSXOpeningElement;
 import com.semmle.js.ast.jsx.JSXSpreadAttribute;
+import com.semmle.js.ast.jsx.JSXThisExpr;
 import com.semmle.ts.ast.ArrayTypeExpr;
 import com.semmle.ts.ast.ConditionalTypeExpr;
 import com.semmle.ts.ast.DecoratorList;
@@ -498,6 +499,11 @@ public class DefaultVisitor<C, R> implements Visitor<C, R> {
     return visit((IJSXName) nd, c);
   }
 
+  @Override
+  public R visit(JSXThisExpr nd, C c) {
+    return visit((IJSXName) nd, c);
+  }
+
   @Override
   public R visit(JSXMemberExpression nd, C c) {
     return visit((IJSXName) nd, c);

javascript/extractor/src/com/semmle/js/ast/NodeCopier.java

@@ -10,6 +10,7 @@ import com.semmle.js.ast.jsx.JSXMemberExpression;
 import com.semmle.js.ast.jsx.JSXNamespacedName;
 import com.semmle.js.ast.jsx.JSXOpeningElement;
 import com.semmle.js.ast.jsx.JSXSpreadAttribute;
+import com.semmle.js.ast.jsx.JSXThisExpr;
 import com.semmle.ts.ast.ArrayTypeExpr;
 import com.semmle.ts.ast.ConditionalTypeExpr;
 import com.semmle.ts.ast.DecoratorList;
@@ -567,6 +568,11 @@ public class NodeCopier implements Visitor<Void, INode> {
     return new JSXIdentifier(visit(nd.getLoc()), nd.getName());
   }
 
+  @Override
+  public INode visit(JSXThisExpr nd, Void c) {
+    return new JSXThisExpr(visit(nd.getLoc()));
+  }
+
   @Override
   public INode visit(JSXMemberExpression nd, Void c) {
     return new JSXMemberExpression(visit(nd.getLoc()), copy(nd.getObject()), copy(nd.getName()));

javascript/extractor/src/com/semmle/js/ast/Visitor.java

@@ -10,6 +10,7 @@ import com.semmle.js.ast.jsx.JSXMemberExpression;
 import com.semmle.js.ast.jsx.JSXNamespacedName;
 import com.semmle.js.ast.jsx.JSXOpeningElement;
 import com.semmle.js.ast.jsx.JSXSpreadAttribute;
+import com.semmle.js.ast.jsx.JSXThisExpr;
 import com.semmle.ts.ast.ArrayTypeExpr;
 import com.semmle.ts.ast.ConditionalTypeExpr;
 import com.semmle.ts.ast.DecoratorList;
@@ -200,6 +201,8 @@ public interface Visitor<C, R> {
 
   public R visit(JSXIdentifier nd, C c);
 
+  public R visit(JSXThisExpr nd, C c);
+
   public R visit(JSXMemberExpression nd, C c);
 
   public R visit(JSXNamespacedName nd, C c);

javascript/extractor/src/com/semmle/js/ast/jsx/JSXThisExpr.java

@@ -0,0 +1,21 @@
+package com.semmle.js.ast.jsx;
+
+import com.semmle.js.ast.ThisExpression;
+import com.semmle.js.ast.SourceLocation;
+import com.semmle.js.ast.Visitor;
+
+public class JSXThisExpr extends ThisExpression implements IJSXName {
+  public JSXThisExpr(SourceLocation loc) {
+    super(loc);
+  }
+
+  @Override
+  public <C, R> R accept(Visitor<C, R> v, C c) {
+    return v.visit(this, c);
+  }
+
+  @Override
+  public String getQualifiedName() {
+    return "this";
+  }
+}

javascript/extractor/src/com/semmle/js/extractor/ASTExtractor.java

@@ -82,6 +82,7 @@ import com.semmle.js.ast.SwitchStatement;
 import com.semmle.js.ast.TaggedTemplateExpression;
 import com.semmle.js.ast.TemplateElement;
 import com.semmle.js.ast.TemplateLiteral;
+import com.semmle.js.ast.ThisExpression;
 import com.semmle.js.ast.ThrowStatement;
 import com.semmle.js.ast.TryStatement;
 import com.semmle.js.ast.UnaryExpression;
@@ -105,6 +106,7 @@ import com.semmle.js.ast.jsx.JSXMemberExpression;
 import com.semmle.js.ast.jsx.JSXNamespacedName;
 import com.semmle.js.ast.jsx.JSXOpeningElement;
 import com.semmle.js.ast.jsx.JSXSpreadAttribute;
+import com.semmle.js.ast.jsx.JSXThisExpr;
 import com.semmle.js.extractor.ExtractionMetrics.ExtractionPhase;
 import com.semmle.js.extractor.ExtractorConfig.Platform;
 import com.semmle.js.extractor.ExtractorConfig.SourceType;
@@ -1645,6 +1647,11 @@ public class ASTExtractor {
       return visit((Identifier) nd, c);
     }
 
+    @Override
+    public Label visit(JSXThisExpr nd, Context c) {
+      return visit((ThisExpression) nd, c);
+    }
+
     @Override
     public Label visit(JSXMemberExpression nd, Context c) {
       Label key = super.visit(nd, c);

javascript/extractor/src/com/semmle/js/extractor/ExprKinds.java

@@ -18,6 +18,7 @@ import com.semmle.js.ast.Literal;
 import com.semmle.js.ast.LogicalExpression;
 import com.semmle.js.ast.MemberExpression;
 import com.semmle.js.ast.MetaProperty;
+import com.semmle.js.ast.ThisExpression;
 import com.semmle.js.ast.UnaryExpression;
 import com.semmle.js.ast.UpdateExpression;
 import com.semmle.js.ast.XMLAnyName;
@@ -28,6 +29,7 @@ import com.semmle.js.ast.XMLQualifiedIdentifier;
 import com.semmle.js.ast.jsx.JSXIdentifier;
 import com.semmle.js.ast.jsx.JSXMemberExpression;
 import com.semmle.js.ast.jsx.JSXSpreadAttribute;
+import com.semmle.js.ast.jsx.JSXThisExpr;
 import com.semmle.js.extractor.ASTExtractor.IdContext;
 import com.semmle.ts.ast.DecoratorList;
 import com.semmle.ts.ast.ExpressionWithTypeArguments;
@@ -194,6 +196,11 @@ public class ExprKinds {
                 return visit((Identifier) nd, c);
               }
 
+              @Override
+              public Integer visit(JSXThisExpr nd, Void c) {
+                return visit((ThisExpression) nd, c);
+              }
+
               @Override
               public Integer visit(LogicalExpression nd, Void q) {
                 return binOpKinds.get(nd.getOperator());

javascript/extractor/src/com/semmle/js/extractor/ExtractorState.java

@@ -1,7 +1,10 @@
 package com.semmle.js.extractor;
 
+import java.io.File;
 import java.nio.file.Path;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.Optional;
 
 import com.semmle.js.parser.TypeScriptParser;
 
@@ -23,6 +26,8 @@ public class ExtractorState {
   
   private final ConcurrentHashMap<Path, FileSnippet> snippets = new ConcurrentHashMap<>();
 
+  private static final ConcurrentMap<File, Optional<String>> packageTypeCache = new ConcurrentHashMap<>();
+
   public TypeScriptParser getTypeScriptParser() {
     return typeScriptParser;
   }
@@ -36,6 +41,15 @@ public class ExtractorState {
     return snippets;
   }
 
+  /**
+   * Returns a cache for the "type" field in `package.json` files.
+   *
+   * <p>The map is thread-safe and may be mutated by the caller.
+   */
+  public ConcurrentMap<File, Optional<String>> getPackageTypeCache() {
+    return this.packageTypeCache;
+  }
+
   /**
    * Makes this semantically equivalent to a fresh state, but may internally retain shared resources
    * that are expensive to reacquire.
@@ -43,5 +57,6 @@ public class ExtractorState {
   public void reset() {
     typeScriptParser.reset();
     snippets.clear();
+    packageTypeCache.clear();
   }
 }

javascript/extractor/src/com/semmle/js/extractor/FileExtractor.java

@@ -119,7 +119,7 @@ public class FileExtractor {
     JS(".js", ".jsx", ".mjs", ".cjs", ".es6", ".es") {
       @Override
       public IExtractor mkExtractor(ExtractorConfig config, ExtractorState state) {
-        return new ScriptExtractor(config);
+        return new ScriptExtractor(config, state);
       }
 
       @Override

javascript/extractor/src/com/semmle/js/extractor/Main.java

@@ -43,7 +43,7 @@ public class Main {
    * A version identifier that should be updated every time the extractor changes in such a way that
    * it may produce different tuples for the same file under the same {@link ExtractorConfig}.
    */
-  public static final String EXTRACTOR_VERSION = "2020-08-20-2";
+  public static final String EXTRACTOR_VERSION = "2020-08-24";
 
   public static final Pattern NEWLINE = Pattern.compile("\n");
 

javascript/extractor/src/com/semmle/js/extractor/ScriptExtractor.java

@@ -1,5 +1,16 @@
 package com.semmle.js.extractor;
 
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.concurrent.ConcurrentMap;
+import java.util.Optional;
+
+import com.google.gson.Gson;
+import com.google.gson.JsonSyntaxException;
+
 import com.semmle.js.extractor.ExtractorConfig.Platform;
 import com.semmle.js.extractor.ExtractorConfig.SourceType;
 import com.semmle.js.parser.ParseError;
@@ -9,19 +20,24 @@ import com.semmle.util.trap.TrapWriter.Label;
 /** Extract a stand-alone JavaScript script. */
 public class ScriptExtractor implements IExtractor {
   private ExtractorConfig config;
+  private ConcurrentMap<File, Optional<String>> packageTypeCache;
 
-  public ScriptExtractor(ExtractorConfig config) {
+  public ScriptExtractor(ExtractorConfig config, ExtractorState state) {
     this.config = config;
+    this.packageTypeCache = state.getPackageTypeCache();
   }
 
-  /** True if files with the given extension should always be treated as modules. */
-  private boolean isAlwaysModule(String extension) {
-    return extension.equals(".mjs") || extension.equals(".es6") || extension.equals(".es");
+  /** True if files with the given extension and type (from package.json) should always be treated as ES2015 modules. */
+  private boolean isAlwaysModule(String extension, String packageType) {
+    if (extension.equals(".mjs") || extension.equals(".es6") || extension.equals(".es")) {
+      return true;
+    }
+    return "module".equals(packageType) && extension.equals(".js");
   }
 
-  /** True if files with the given extension should always be treated as CommonJS modules. */
-  private boolean isAlwaysCommonJSModule(String extension) {
-    return extension.equals(".cjs");
+  /** True if files with the given extension and type (from package.json) should always be treated as CommonJS modules. */
+  private boolean isAlwaysCommonJSModule(String extension, String packageType) {
+    return extension.equals(".cjs") || (extension.equals(".js") && "commonjs".equals(packageType));
   }
 
   @Override
@@ -49,13 +65,16 @@ public class ScriptExtractor implements IExtractor {
       locationManager.setStart(2, 1);
     }
 
-    // Some file extensions are interpreted as modules by default.
+    String packageType = getPackageType(locationManager.getSourceFile().getParentFile());
+    String extension = locationManager.getSourceFileExtension();
+
+    // Some files are interpreted as modules by default.
     if (config.getSourceType() == SourceType.AUTO) {
-      if (isAlwaysModule(locationManager.getSourceFileExtension())) {
+      if (isAlwaysModule(extension, packageType)) {
         config = config.withSourceType(SourceType.MODULE);
       }
-      if (isAlwaysCommonJSModule(locationManager.getSourceFileExtension())) {
-        config = config.withSourceType(SourceType.COMMONJS_MODULE);
+      if (isAlwaysCommonJSModule(extension, packageType)) {
+        config = config.withSourceType(SourceType.COMMONJS_MODULE).withPlatform(Platform.NODE);
       }
     }
 
@@ -78,4 +97,40 @@ public class ScriptExtractor implements IExtractor {
 
     return loc;
   }
+
+  /**
+   * A minimal model of `package.json` files that can be used to read the "type" field.
+   */
+  private static class PackageJSON {
+    String type;
+  }
+
+  /**
+   * Returns the "type" field from the nearest `package.json` file (searching up the file hierarchy).
+   */
+  private String getPackageType(File folder) {
+    if (folder == null || !folder.isDirectory()) {
+      return null;
+    }
+    if (packageTypeCache.containsKey(folder)) {
+      return packageTypeCache.get(folder).orElse(null);
+    }
+    File file = new File(folder, "package.json");
+    if (file.isDirectory()) {
+      return null;
+    }
+    if (!file.exists()) {
+      String result = getPackageType(folder.getParentFile());
+      packageTypeCache.put(folder, Optional.ofNullable(result));
+      return result;
+    }
+    try {
+      BufferedReader reader = new BufferedReader(new FileReader(file));
+      String result = new Gson().fromJson(reader, PackageJSON.class).type;
+      packageTypeCache.put(folder, Optional.ofNullable(result));
+      return result;
+    } catch (IOException | JsonSyntaxException e) {
+      return null;
+    }
+  }
 }

javascript/extractor/src/com/semmle/js/parser/TypeScriptASTConverter.java

@@ -111,6 +111,7 @@ import com.semmle.js.ast.jsx.JSXIdentifier;
 import com.semmle.js.ast.jsx.JSXMemberExpression;
 import com.semmle.js.ast.jsx.JSXOpeningElement;
 import com.semmle.js.ast.jsx.JSXSpreadAttribute;
+import com.semmle.js.ast.jsx.JSXThisExpr;
 import com.semmle.js.parser.JSParser.Result;
 import com.semmle.ts.ast.ArrayTypeExpr;
 import com.semmle.ts.ast.ConditionalTypeExpr;
@@ -2377,7 +2378,7 @@ public class TypeScriptASTConverter {
           convertJSXName(me.getObject()),
           (JSXIdentifier) convertJSXName(me.getProperty()));
     }
-    if (e instanceof ThisExpression) return new JSXIdentifier(e.getLoc(), "this");
+    if (e instanceof ThisExpression) return new JSXThisExpr(e.getLoc());
     return (IJSXName) e;
   }