From 3d5c1560e44291f3bb9ee267c1f2b050f7066167 Mon Sep 17 00:00:00 2001
From: Erik Krogh Kristensen <erik-krogh@github.com>
Date: Tue, 18 Aug 2020 14:10:18 +0200
Subject: [PATCH] basic support for `.cjs` files

---
 .../com/semmle/js/extractor/AutoBuild.java    |   2 +-
 .../semmle/js/extractor/FileExtractor.java    |   2 +-
 .../src/com/semmle/js/extractor/Main.java     |   2 +-
 .../semmle/js/extractor/ScriptExtractor.java  |  13 +-
 .../extractor/tests/extensions/input/tst4.cjs |   1 +
 .../extensions/output/trap/tst4.cjs.trap      | 141 ++++++++++++++++++
 .../ql/src/semmle/javascript/NodeJS.qll       |   6 +-
 .../javascript/NodeModuleResolutionImpl.qll   |   8 +-
 .../library-tests/ModuleTypes/commonjs.cjs    |   3 +
 .../library-tests/ModuleTypes/tests.expected  |   1 +
 10 files changed, 168 insertions(+), 11 deletions(-)
 create mode 100644 javascript/extractor/tests/extensions/input/tst4.cjs
 create mode 100644 javascript/extractor/tests/extensions/output/trap/tst4.cjs.trap
 create mode 100644 javascript/ql/test/library-tests/ModuleTypes/commonjs.cjs

diff --git a/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java b/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java
index 9992e912043..28c96f806f7 100644
--- a/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java
+++ b/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java
@@ -144,7 +144,7 @@ import com.semmle.util.trap.TrapWriter;
  *
  * <ul>
  *   <li>All JavaScript files, that is, files with one of the extensions supported by {@link
- *       FileType#JS} (currently ".js", ".jsx", ".mjs", ".es6", ".es").
+ *       FileType#JS} (currently ".js", ".jsx", ".mjs", ".cjs", ".es6", ".es").
  *   <li>All HTML files, that is, files with with one of the extensions supported by {@link
  *       FileType#HTML} (currently ".htm", ".html", ".xhtm", ".xhtml", ".vue").
  *   <li>All YAML files, that is, files with one of the extensions supported by {@link
diff --git a/javascript/extractor/src/com/semmle/js/extractor/FileExtractor.java b/javascript/extractor/src/com/semmle/js/extractor/FileExtractor.java
index 861f7630995..81135dbd04c 100644
--- a/javascript/extractor/src/com/semmle/js/extractor/FileExtractor.java
+++ b/javascript/extractor/src/com/semmle/js/extractor/FileExtractor.java
@@ -58,7 +58,7 @@ public class FileExtractor {
       }
     },
 
-    JS(".js", ".jsx", ".mjs", ".es6", ".es") {
+    JS(".js", ".jsx", ".mjs", ".cjs", ".es6", ".es") {
       @Override
       public IExtractor mkExtractor(ExtractorConfig config, ExtractorState state) {
         return new ScriptExtractor(config);
diff --git a/javascript/extractor/src/com/semmle/js/extractor/Main.java b/javascript/extractor/src/com/semmle/js/extractor/Main.java
index 451baa73a46..bb34dfbe2be 100644
--- a/javascript/extractor/src/com/semmle/js/extractor/Main.java
+++ b/javascript/extractor/src/com/semmle/js/extractor/Main.java
@@ -43,7 +43,7 @@ public class Main {
    * A version identifier that should be updated every time the extractor changes in such a way that
    * it may produce different tuples for the same file under the same {@link ExtractorConfig}.
    */
-  public static final String EXTRACTOR_VERSION = "2020-08-18";
+  public static final String EXTRACTOR_VERSION = "2020-08-19";
 
   public static final Pattern NEWLINE = Pattern.compile("\n");
 
diff --git a/javascript/extractor/src/com/semmle/js/extractor/ScriptExtractor.java b/javascript/extractor/src/com/semmle/js/extractor/ScriptExtractor.java
index 197a3473a65..e3c601aaaad 100644
--- a/javascript/extractor/src/com/semmle/js/extractor/ScriptExtractor.java
+++ b/javascript/extractor/src/com/semmle/js/extractor/ScriptExtractor.java
@@ -19,6 +19,11 @@ public class ScriptExtractor implements IExtractor {
     return extension.equals(".mjs") || extension.equals(".es6") || extension.equals(".es");
   }
 
+  /** True if files with the given extension should always be treated as CommonJS modules. */
+  private boolean isAlwaysCommonJSModule(String extension) {
+    return extension.equals(".cjs");
+  }
+
   @Override
   public LoCInfo extract(TextualExtractor textualExtractor) {
     LocationManager locationManager = textualExtractor.getLocationManager();
@@ -45,9 +50,13 @@ public class ScriptExtractor implements IExtractor {
     }
 
     // Some file extensions are interpreted as modules by default.
-    if (isAlwaysModule(locationManager.getSourceFileExtension())) {
-      if (config.getSourceType() == SourceType.AUTO)
+    if (config.getSourceType() == SourceType.AUTO) {
+      if (isAlwaysModule(locationManager.getSourceFileExtension())) {
         config = config.withSourceType(SourceType.MODULE);
+      }
+      if (isAlwaysCommonJSModule(locationManager.getSourceFileExtension())) {
+        config = config.withSourceType(SourceType.COMMONJS_MODULE);
+      }
     }
 
     ScopeManager scopeManager =
diff --git a/javascript/extractor/tests/extensions/input/tst4.cjs b/javascript/extractor/tests/extensions/input/tst4.cjs
new file mode 100644
index 00000000000..a552cea0167
--- /dev/null
+++ b/javascript/extractor/tests/extensions/input/tst4.cjs
@@ -0,0 +1 @@
+console.log("Hello CommonJS");
diff --git a/javascript/extractor/tests/extensions/output/trap/tst4.cjs.trap b/javascript/extractor/tests/extensions/output/trap/tst4.cjs.trap
new file mode 100644
index 00000000000..5f76c8832f9
--- /dev/null
+++ b/javascript/extractor/tests/extensions/output/trap/tst4.cjs.trap
@@ -0,0 +1,141 @@
+#10000=@"/tst4.cjs;sourcefile"
+files(#10000,"/tst4.cjs","tst4","cjs",0)
+#10001=@"/;folder"
+folders(#10001,"/","")
+containerparent(#10001,#10000)
+#10002=@"loc,{#10000},0,0,0,0"
+locations_default(#10002,#10000,0,0,0,0)
+hasLocation(#10000,#10002)
+#20000=@"global_scope"
+scopes(#20000,0)
+#20001=@"script;{#10000},1,1"
+#20002=*
+lines(#20002,#20001,"console.log(""Hello CommonJS"");","
+")
+#20003=@"loc,{#10000},1,1,1,30"
+locations_default(#20003,#10000,1,1,1,30)
+hasLocation(#20002,#20003)
+numlines(#20001,1,1,0)
+#20004=*
+tokeninfo(#20004,6,#20001,0,"console")
+#20005=@"loc,{#10000},1,1,1,7"
+locations_default(#20005,#10000,1,1,1,7)
+hasLocation(#20004,#20005)
+#20006=*
+tokeninfo(#20006,8,#20001,1,".")
+#20007=@"loc,{#10000},1,8,1,8"
+locations_default(#20007,#10000,1,8,1,8)
+hasLocation(#20006,#20007)
+#20008=*
+tokeninfo(#20008,6,#20001,2,"log")
+#20009=@"loc,{#10000},1,9,1,11"
+locations_default(#20009,#10000,1,9,1,11)
+hasLocation(#20008,#20009)
+#20010=*
+tokeninfo(#20010,8,#20001,3,"(")
+#20011=@"loc,{#10000},1,12,1,12"
+locations_default(#20011,#10000,1,12,1,12)
+hasLocation(#20010,#20011)
+#20012=*
+tokeninfo(#20012,4,#20001,4,"""Hello CommonJS""")
+#20013=@"loc,{#10000},1,13,1,28"
+locations_default(#20013,#10000,1,13,1,28)
+hasLocation(#20012,#20013)
+#20014=*
+tokeninfo(#20014,8,#20001,5,")")
+#20015=@"loc,{#10000},1,29,1,29"
+locations_default(#20015,#10000,1,29,1,29)
+hasLocation(#20014,#20015)
+#20016=*
+tokeninfo(#20016,8,#20001,6,";")
+#20017=@"loc,{#10000},1,30,1,30"
+locations_default(#20017,#10000,1,30,1,30)
+hasLocation(#20016,#20017)
+#20018=*
+tokeninfo(#20018,0,#20001,7,"")
+#20019=@"loc,{#10000},2,1,2,0"
+locations_default(#20019,#10000,2,1,2,0)
+hasLocation(#20018,#20019)
+toplevels(#20001,0)
+#20020=@"loc,{#10000},1,1,2,0"
+locations_default(#20020,#10000,1,1,2,0)
+hasLocation(#20001,#20020)
+#20021=@"module;{#10000},1,1"
+scopes(#20021,3)
+scopenodes(#20001,#20021)
+scopenesting(#20021,#20000)
+#20022=@"var;{require};{#20021}"
+variables(#20022,"require",#20021)
+#20023=@"var;{module};{#20021}"
+variables(#20023,"module",#20021)
+#20024=@"var;{exports};{#20021}"
+variables(#20024,"exports",#20021)
+#20025=@"var;{__filename};{#20021}"
+variables(#20025,"__filename",#20021)
+#20026=@"var;{__dirname};{#20021}"
+variables(#20026,"__dirname",#20021)
+#20027=@"var;{arguments};{#20021}"
+variables(#20027,"arguments",#20021)
+isModule(#20001)
+#20028=*
+stmts(#20028,2,#20001,0,"console ... onJS"");")
+hasLocation(#20028,#20003)
+stmtContainers(#20028,#20001)
+#20029=*
+exprs(#20029,13,#20028,0,"console ... monJS"")")
+#20030=@"loc,{#10000},1,1,1,29"
+locations_default(#20030,#10000,1,1,1,29)
+hasLocation(#20029,#20030)
+enclosingStmt(#20029,#20028)
+exprContainers(#20029,#20001)
+#20031=*
+exprs(#20031,14,#20029,-1,"console.log")
+#20032=@"loc,{#10000},1,1,1,11"
+locations_default(#20032,#10000,1,1,1,11)
+hasLocation(#20031,#20032)
+enclosingStmt(#20031,#20028)
+exprContainers(#20031,#20001)
+#20033=*
+exprs(#20033,79,#20031,0,"console")
+hasLocation(#20033,#20005)
+enclosingStmt(#20033,#20028)
+exprContainers(#20033,#20001)
+literals("console","console",#20033)
+#20034=@"var;{console};{#20000}"
+variables(#20034,"console",#20000)
+bind(#20033,#20034)
+#20035=*
+exprs(#20035,0,#20031,1,"log")
+hasLocation(#20035,#20009)
+enclosingStmt(#20035,#20028)
+exprContainers(#20035,#20001)
+literals("log","log",#20035)
+#20036=*
+exprs(#20036,4,#20029,0,"""Hello CommonJS""")
+hasLocation(#20036,#20013)
+enclosingStmt(#20036,#20028)
+exprContainers(#20036,#20001)
+literals("Hello CommonJS","""Hello CommonJS""",#20036)
+#20037=*
+regexpterm(#20037,14,#20036,0,"Hello CommonJS")
+#20038=@"loc,{#10000},1,14,1,27"
+locations_default(#20038,#10000,1,14,1,27)
+hasLocation(#20037,#20038)
+regexpConstValue(#20037,"Hello CommonJS")
+#20039=*
+entry_cfg_node(#20039,#20001)
+#20040=@"loc,{#10000},1,1,1,0"
+locations_default(#20040,#10000,1,1,1,0)
+hasLocation(#20039,#20040)
+#20041=*
+exit_cfg_node(#20041,#20001)
+hasLocation(#20041,#20019)
+successor(#20028,#20033)
+successor(#20036,#20029)
+successor(#20035,#20031)
+successor(#20033,#20035)
+successor(#20031,#20036)
+successor(#20029,#20041)
+successor(#20039,#20028)
+numlines(#10000,1,1,0)
+filetype(#10000,"javascript")
diff --git a/javascript/ql/src/semmle/javascript/NodeJS.qll b/javascript/ql/src/semmle/javascript/NodeJS.qll
index 6da4a625537..43fbb256998 100644
--- a/javascript/ql/src/semmle/javascript/NodeJS.qll
+++ b/javascript/ql/src/semmle/javascript/NodeJS.qll
@@ -221,7 +221,7 @@ class Require extends CallExpr, Import {
    *
    * <ul>
    * <li> the file `c/p`;
-   * <li> the file `c/p.{tsx,ts,jsx,es6,es,mjs}`;
+   * <li> the file `c/p.{tsx,ts,jsx,es6,es,mjs,cjs}`;
    * <li> the file `c/p.js`;
    * <li> the file `c/p.json`;
    * <li> the file `c/p.node`;
@@ -230,12 +230,12 @@ class Require extends CallExpr, Import {
    *      <li> if `c/p/package.json` exists and specifies a `main` module `m`:
    *        <ul>
    *        <li> the file `c/p/m`;
-   *        <li> the file `c/p/m.{tsx,ts,jsx,es6,es,mjs}`;
+   *        <li> the file `c/p/m.{tsx,ts,jsx,es6,es,mjs,cjs}`;
    *        <li> the file `c/p/m.js`;
    *        <li> the file `c/p/m.json`;
    *        <li> the file `c/p/m.node`;
    *        </ul>
-   *      <li> the file `c/p/index.{tsx,ts,jsx,es6,es,mjs}`;
+   *      <li> the file `c/p/index.{tsx,ts,jsx,es6,es,mjs,cjs}`;
    *      <li> the file `c/p/index.js`;
    *      <li> the file `c/p/index.json`;
    *      <li> the file `c/p/index.node`.
diff --git a/javascript/ql/src/semmle/javascript/NodeModuleResolutionImpl.qll b/javascript/ql/src/semmle/javascript/NodeModuleResolutionImpl.qll
index 8eecf46c617..01b01f1ac04 100644
--- a/javascript/ql/src/semmle/javascript/NodeModuleResolutionImpl.qll
+++ b/javascript/ql/src/semmle/javascript/NodeModuleResolutionImpl.qll
@@ -26,11 +26,13 @@ int getFileExtensionPriority(string ext) {
   or
   ext = "mjs" and result = 5
   or
-  ext = "js" and result = 6
+  ext = "cjs" and result = 6
   or
-  ext = "json" and result = 7
+  ext = "js" and result = 7
   or
-  ext = "node" and result = 8
+  ext = "json" and result = 8
+  or
+  ext = "node" and result = 9
 }
 
 int prioritiesPerCandidate() { result = 3 * (numberOfExtensions() + 1) }
diff --git a/javascript/ql/test/library-tests/ModuleTypes/commonjs.cjs b/javascript/ql/test/library-tests/ModuleTypes/commonjs.cjs
new file mode 100644
index 00000000000..344893b298a
--- /dev/null
+++ b/javascript/ql/test/library-tests/ModuleTypes/commonjs.cjs
@@ -0,0 +1,3 @@
+var fs = require("fs");
+console.log("I'm a .cjs file!");
+console.log(fs);
\ No newline at end of file
diff --git a/javascript/ql/test/library-tests/ModuleTypes/tests.expected b/javascript/ql/test/library-tests/ModuleTypes/tests.expected
index 023ac9d1a61..6aeb75cb04a 100644
--- a/javascript/ql/test/library-tests/ModuleTypes/tests.expected
+++ b/javascript/ql/test/library-tests/ModuleTypes/tests.expected
@@ -1,3 +1,4 @@
+| commonjs.cjs:1:1:3:16 | <toplevel> | node |
 | import.js:1:1:5:2 | <toplevel> | es2015 |
 | mjs.mjs:1:1:1:32 | <toplevel> | es2015 |
 | require.js:1:1:7:1 | <toplevel> | node |