From d4b9beb0104acf439c340f2196ac6b50e6f5f306 Mon Sep 17 00:00:00 2001
From: Max Schaefer <max@semmle.com>
Date: Thu, 24 Oct 2019 15:53:51 +0100
Subject: [PATCH] JavaScript: Teach autobuilder not to extract `node_modules`
 and `bower_components` folders.

---
 change-notes/1.23/extractor-javascript.md     | 11 ++++++
 .../com/semmle/js/extractor/AutoBuild.java    |  4 +++
 .../js/extractor/test/AutoBuildTests.java     | 34 +++++++++++++++++++
 3 files changed, 49 insertions(+)

diff --git a/change-notes/1.23/extractor-javascript.md b/change-notes/1.23/extractor-javascript.md
index 18a67a65900..8b7a35f5b4f 100644
--- a/change-notes/1.23/extractor-javascript.md
+++ b/change-notes/1.23/extractor-javascript.md
@@ -5,6 +5,17 @@
 ## Changes to code extraction
 
 * Asynchronous generator methods are now parsed correctly and no longer cause a spurious syntax error.
+* Files in `node_modules` and `bower_components` folders are no longer extracted by default. If you still want to extract files from these folders, you can add the following filters to your `lgtm.yml` file (or add them to existing filters):
+
+```yaml
+extraction:
+  javascript:
+    index:
+      filters:
+        - include: "**/node_modules"
+        - include: "**/bower_components"
+```
+
 * Recognition of CommonJS modules has improved. As a result, some files that were previously extracted as
   global scripts are now extracted as modules.
 * Top-level `await` is now supported.
diff --git a/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java b/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java
index 5ffa5680cd9..55d3f8632a2 100644
--- a/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java
+++ b/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java
@@ -387,6 +387,10 @@ public class AutoBuild {
     patterns.add("-**/*.min.js");
     patterns.add("-**/*-min.js");
 
+    // exclude `node_modules` and `bower_components`
+    patterns.add("-**/node_modules");
+    patterns.add("-**/bower_components");
+
     String base = LGTM_SRC.toString().replace('\\', '/');
     // process `$LGTM_INDEX_FILTERS`
     for (String pattern : Main.NEWLINE.split(getEnvVar("LGTM_INDEX_FILTERS", ""))) {
diff --git a/javascript/extractor/src/com/semmle/js/extractor/test/AutoBuildTests.java b/javascript/extractor/src/com/semmle/js/extractor/test/AutoBuildTests.java
index b177154510f..28e618f45ea 100644
--- a/javascript/extractor/src/com/semmle/js/extractor/test/AutoBuildTests.java
+++ b/javascript/extractor/src/com/semmle/js/extractor/test/AutoBuildTests.java
@@ -483,6 +483,40 @@ public class AutoBuildTests {
     runTest();
   }
 
+  @Test
+  public void nodeModulesAreExcluded() throws IOException {
+    addFile(true, LGTM_SRC, "index.js");
+    addFile(false, LGTM_SRC, "node_modules", "dep", "main.js");
+    addFile(false, LGTM_SRC, "node_modules", "dep", "node_modules", "leftpad", "index.js");
+    runTest();
+  }
+
+  @Test
+  public void nodeModulesCanBeReincluded() throws IOException {
+    envVars.put("LGTM_INDEX_FILTERS", "include:**/node_modules");
+    addFile(true, LGTM_SRC, "index.js");
+    addFile(true, LGTM_SRC, "node_modules", "dep", "main.js");
+    addFile(true, LGTM_SRC, "node_modules", "dep", "node_modules", "leftpad", "index.js");
+    runTest();
+  }
+
+  @Test
+  public void bowerComponentsAreExcluded() throws IOException {
+    addFile(true, LGTM_SRC, "index.js");
+    addFile(false, LGTM_SRC, "bower_components", "dep", "main.js");
+    addFile(false, LGTM_SRC, "bower_components", "dep", "bower_components", "leftpad", "index.js");
+    runTest();
+  }
+
+  @Test
+  public void bowerComponentsCanBeReincluded() throws IOException {
+    envVars.put("LGTM_INDEX_FILTERS", "include:**/bower_components");
+    addFile(true, LGTM_SRC, "index.js");
+    addFile(true, LGTM_SRC, "bower_components", "dep", "main.js");
+    addFile(true, LGTM_SRC, "bower_components", "dep", "bower_components", "leftpad", "index.js");
+    runTest();
+  }
+
   @Test
   public void customExtensions() throws IOException {
     envVars.put("LGTM_INDEX_FILETYPES", ".jsm:js\n.soy:html");