Merge pull request #2619 from asger-semmle/ts-monorepo-deps

Approved by erik-krogh, max-schaefer

javascript/extractor/lib/typescript/src/common.ts

@@ -1,10 +1,44 @@
 import * as ts from "./typescript";
 import { TypeTable } from "./type_table";
+import * as pathlib from "path";
+import { VirtualSourceRoot } from "./virtual_source_root";
+
+/**
+ * Extracts the package name from the prefix of an import string.
+ */
+const packageNameRex = /^(?:@[\w.-]+[/\\]+)?\w[\w.-]*(?=[/\\]|$)/;
+const extensions = ['.ts', '.tsx', '.d.ts', '.js', '.jsx'];
+
+function getPackageName(importString: string) {
+  let packageNameMatch = packageNameRex.exec(importString);
+  if (packageNameMatch == null) return null;
+  let packageName = packageNameMatch[0];
+  if (packageName.charAt(0) === '@') {
+    packageName = packageName.replace(/[/\\]+/g, '/'); // Normalize slash after the scope.
+  }
+  return packageName;
+}
 
 export class Project {
   public program: ts.Program = null;
+  private host: ts.CompilerHost;
+  private resolutionCache: ts.ModuleResolutionCache;
 
-  constructor(public tsConfig: string, public config: ts.ParsedCommandLine, public typeTable: TypeTable) {}
+  constructor(
+        public tsConfig: string,
+        public config: ts.ParsedCommandLine,
+        public typeTable: TypeTable,
+        public packageEntryPoints: Map<string, string>,
+        public virtualSourceRoot: VirtualSourceRoot) {
+
+    this.resolveModuleNames = this.resolveModuleNames.bind(this);
+
+    this.resolutionCache = ts.createModuleResolutionCache(pathlib.dirname(tsConfig), ts.sys.realpath, config.options);
+    let host = ts.createCompilerHost(config.options, true);
+    host.resolveModuleNames = this.resolveModuleNames;
+    host.trace = undefined; // Disable tracing which would otherwise go to standard out
+    this.host = host;
+  }
 
   public unload(): void {
     this.typeTable.releaseProgram();
@@ -12,9 +46,8 @@ export class Project {
   }
 
   public load(): void {
-    let host = ts.createCompilerHost(this.config.options, true);
-    host.trace = undefined; // Disable tracing which would otherwise go to standard out
-    this.program = ts.createProgram(this.config.fileNames, this.config.options, host);
+    const { config, host } = this;
+    this.program = ts.createProgram(config.fileNames, config.options, host);
     this.typeTable.setProgram(this.program);
   }
 
@@ -27,4 +60,73 @@ export class Project {
     this.unload();
     this.load();
   }
+
+  /**
+   * Override for module resolution in the TypeScript compiler host.
+   */
+  private resolveModuleNames(
+        moduleNames: string[],
+        containingFile: string,
+        reusedNames: string[],
+        redirectedReference: ts.ResolvedProjectReference,
+        options: ts.CompilerOptions) {
+
+    const { host, resolutionCache } = this;
+    return moduleNames.map((moduleName) => {
+      let redirected = this.redirectModuleName(moduleName, containingFile, options);
+      if (redirected != null) return redirected;
+      return ts.resolveModuleName(moduleName, containingFile, options,  host, resolutionCache).resolvedModule;
+    });
+  }
+
+  /**
+   * Returns the path that the given import string should be redirected to, or null if it should
+   * fall back to standard module resolution.
+   */
+  private redirectModuleName(moduleName: string, containingFile: string, options: ts.CompilerOptions): ts.ResolvedModule {
+    // Get a package name from the leading part of the module name, e.g. '@scope/foo' from '@scope/foo/bar'.
+    let packageName = getPackageName(moduleName);
+    if (packageName == null) return null;
+
+    // Get the overridden location of this package, if one exists.
+    let packageEntryPoint = this.packageEntryPoints.get(packageName);
+    if (packageEntryPoint == null) {
+      // The package is not overridden, but we have established that it begins with a valid package name.
+      // Do a lookup in the virtual source root (where dependencies are installed) by changing the 'containing file'.
+      let virtualContainingFile = this.virtualSourceRoot.toVirtualPath(containingFile);
+      if (virtualContainingFile != null) {
+        return ts.resolveModuleName(moduleName, virtualContainingFile, options, this.host, this.resolutionCache).resolvedModule;
+      }
+      return null;
+    }
+
+    // If the requested module name is exactly the overridden package name,
+    // return the entry point file (it is not necessarily called `index.ts`).
+    if (moduleName === packageName) {
+      return { resolvedFileName: packageEntryPoint, isExternalLibraryImport: true };
+    }
+
+    // Get the suffix after the package name, e.g. the '/bar' in '@scope/foo/bar'.
+    let suffix = moduleName.substring(packageName.length);
+
+    // Resolve the suffix relative to the package directory.
+    let packageDir = pathlib.dirname(packageEntryPoint);
+    let joinedPath = pathlib.join(packageDir, suffix);
+
+    // Add implicit '/index'
+    if (ts.sys.directoryExists(joinedPath)) {
+      joinedPath = pathlib.join(joinedPath, 'index');
+    }
+
+    // Try each recognized extension. We must not return a file whose extension is not
+    // recognized by TypeScript.
+    for (let ext of extensions) {
+      let candidate = joinedPath.endsWith(ext) ? joinedPath : (joinedPath + ext);
+      if (ts.sys.fileExists(candidate)) {
+        return { resolvedFileName: candidate, isExternalLibraryImport: true };
+      }
+    }
+
+    return null;
+  }
 }

javascript/extractor/lib/typescript/src/main.ts

@@ -39,6 +39,7 @@ import * as ast_extractor from "./ast_extractor";
 
 import { Project } from "./common";
 import { TypeTable } from "./type_table";
+import { VirtualSourceRoot } from "./virtual_source_root";
 
 interface ParseCommand {
     command: "parse";
@@ -47,6 +48,9 @@ interface ParseCommand {
 interface OpenProjectCommand {
     command: "open-project";
     tsConfig: string;
+    virtualSourceRoot: string | null;
+    packageEntryPoints: [string, string][];
+    packageJsonFiles: [string, string][];
 }
 interface CloseProjectCommand {
     command: "close-project";
@@ -242,26 +246,93 @@ function parseSingleFile(filename: string): {ast: ts.SourceFile, code: string} {
     return {ast, code};
 }
 
+/**
+ * Matches a path segment referencing a package in a node_modules folder, and extracts
+ * two capture groups: the package name, and the relative path in the package.
+ *
+ * For example `lib/node_modules/@foo/bar/src/index.js` extracts the capture groups [`@foo/bar`, `src/index.js`].
+ */
+const nodeModulesRex = /[/\\]node_modules[/\\]((?:@[\w.-]+[/\\])?\w[\w.-]*)[/\\](.*)/;
+
 function handleOpenProjectCommand(command: OpenProjectCommand) {
     Error.stackTraceLimit = Infinity;
     let tsConfigFilename = String(command.tsConfig);
     let tsConfig = ts.readConfigFile(tsConfigFilename, ts.sys.readFile);
     let basePath = pathlib.dirname(tsConfigFilename);
 
+    let packageEntryPoints = new Map(command.packageEntryPoints);
+    let packageJsonFiles = new Map(command.packageJsonFiles);
+    let virtualSourceRoot = new VirtualSourceRoot(process.cwd(), command.virtualSourceRoot);
+
+    /**
+     * Rewrites path segments of form `node_modules/PACK/suffix` to be relative to
+     * the location of package PACK in the source tree, if it exists.
+     */
+    function redirectNodeModulesPath(path: string) {
+        let nodeModulesMatch = nodeModulesRex.exec(path);
+        if (nodeModulesMatch == null) return null;
+        let packageName = nodeModulesMatch[1];
+        let packageJsonFile = packageJsonFiles.get(packageName);
+        if (packageJsonFile == null) return null;
+        let packageDir = pathlib.dirname(packageJsonFile);
+        let suffix = nodeModulesMatch[2];
+        let finalPath = pathlib.join(packageDir, suffix);
+        if (!ts.sys.fileExists(finalPath)) return null;
+        return finalPath;
+    }
+
+    /**
+     * Create the host passed to the tsconfig.json parser.
+     *
+     * We override its file system access in case there is an "extends"
+     * clause pointing into "./node_modules", which must be redirected to
+     * the location of an installed package or a checked-in package.
+     */
     let parseConfigHost: ts.ParseConfigHost = {
         useCaseSensitiveFileNames: true,
-        readDirectory: ts.sys.readDirectory,
-        fileExists: (path: string) => fs.existsSync(path),
-        readFile: ts.sys.readFile,
+        readDirectory: ts.sys.readDirectory, // No need to override traversal/glob matching
+        fileExists: (path: string) => {
+            return ts.sys.fileExists(path)
+                || virtualSourceRoot.toVirtualPathIfFileExists(path) != null
+                || redirectNodeModulesPath(path) != null;
+        },
+        readFile: (path: string) => {
+            if (!ts.sys.fileExists(path)) {
+                let virtualPath = virtualSourceRoot.toVirtualPathIfFileExists(path);
+                if (virtualPath != null) return ts.sys.readFile(virtualPath);
+                virtualPath = redirectNodeModulesPath(path);
+                if (virtualPath != null) return ts.sys.readFile(virtualPath);
+            }
+            return ts.sys.readFile(path);
+        }
     };
     let config = ts.parseJsonConfigFileContent(tsConfig.config, parseConfigHost, basePath);
-    let project = new Project(tsConfigFilename, config, state.typeTable);
+    let project = new Project(tsConfigFilename, config, state.typeTable, packageEntryPoints, virtualSourceRoot);
     project.load();
 
     state.project = project;
     let program = project.program;
     let typeChecker = program.getTypeChecker();
 
+    let diagnostics = program.getSemanticDiagnostics()
+        .filter(d => d.category === ts.DiagnosticCategory.Error);
+    if (diagnostics.length > 0) {
+        console.warn('TypeScript: reported ' + diagnostics.length + ' semantic errors.');
+    }
+    for (let diagnostic of diagnostics) {
+        let text = diagnostic.messageText;
+        if (text && typeof text !== 'string') {
+            text = text.messageText;
+        }
+        let locationStr = '';
+        let { file } = diagnostic;
+        if (file != null) {
+            let { line, character } = file.getLineAndCharacterOfPosition(diagnostic.start);
+            locationStr = `${file.fileName}:${line}:${character}`;
+        }
+        console.warn(`TypeScript: ${locationStr} ${text}`);
+    }
+
     // Associate external module names with the corresponding file symbols.
     // We need these mappings to identify which module a given external type comes from.
     // The TypeScript API lets us resolve a module name to a source file, but there is no
@@ -512,6 +583,9 @@ if (process.argv.length > 2) {
         handleOpenProjectCommand({
             command: "open-project",
             tsConfig: argument,
+            packageEntryPoints: [],
+            packageJsonFiles: [],
+            virtualSourceRoot: null,
         });
         for (let sf of state.project.program.getSourceFiles()) {
             if (pathlib.basename(sf.fileName) === "lib.d.ts") continue;

javascript/extractor/lib/typescript/src/virtual_source_root.ts

@@ -0,0 +1,39 @@
+import * as pathlib from "path";
+import * as ts from "./typescript";
+
+/**
+ * Mapping from the real source root to the virtual source root,
+ * a directory whose folder structure mirrors the real source root, but with `node_modules` installed.
+ */
+export class VirtualSourceRoot {
+  constructor(
+    private sourceRoot: string,
+
+    /**
+     * Directory whose folder structure mirrors the real source root, but with `node_modules` installed,
+     * or undefined if no virtual source root exists.
+     */
+    private virtualSourceRoot: string,
+  ) {}
+
+  /**
+   * Maps a path under the real source root to the corresponding path in the virtual source root.
+   */
+  public toVirtualPath(path: string) {
+    if (!this.virtualSourceRoot) return null;
+    let relative = pathlib.relative(this.sourceRoot, path);
+    if (relative.startsWith('..') || pathlib.isAbsolute(relative)) return null;
+    return pathlib.join(this.virtualSourceRoot, relative);
+  }
+
+  /**
+   * Maps a path under the real source root to the corresponding path in the virtual source root.
+   */
+  public toVirtualPathIfFileExists(path: string) {
+    let virtualPath = this.toVirtualPath(path);
+    if (virtualPath != null && ts.sys.fileExists(virtualPath)) {
+        return virtualPath;
+    }
+    return null;
+  }
+}

javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java

@@ -1,31 +1,11 @@
 package com.semmle.js.extractor;
 
-import com.semmle.js.extractor.ExtractorConfig.SourceType;
-import com.semmle.js.extractor.FileExtractor.FileType;
-import com.semmle.js.extractor.trapcache.DefaultTrapCache;
-import com.semmle.js.extractor.trapcache.DummyTrapCache;
-import com.semmle.js.extractor.trapcache.ITrapCache;
-import com.semmle.js.parser.ParsedProject;
-import com.semmle.js.parser.TypeScriptParser;
-import com.semmle.ts.extractor.TypeExtractor;
-import com.semmle.ts.extractor.TypeTable;
-import com.semmle.util.data.StringUtil;
-import com.semmle.util.exception.CatastrophicError;
-import com.semmle.util.exception.Exceptions;
-import com.semmle.util.exception.ResourceError;
-import com.semmle.util.exception.UserError;
-import com.semmle.util.extraction.ExtractorOutputConfig;
-import com.semmle.util.files.FileUtil;
-import com.semmle.util.io.csv.CSVReader;
-import com.semmle.util.language.LegacyLanguage;
-import com.semmle.util.process.Env;
-import com.semmle.util.projectstructure.ProjectLayout;
-import com.semmle.util.trap.TrapWriter;
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.Reader;
+import java.io.Writer;
 import java.lang.ProcessBuilder.Redirect;
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -50,6 +30,35 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Stream;
 
+import com.google.gson.Gson;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParseException;
+import com.google.gson.JsonParser;
+import com.google.gson.JsonPrimitive;
+import com.semmle.js.extractor.ExtractorConfig.SourceType;
+import com.semmle.js.extractor.FileExtractor.FileType;
+import com.semmle.js.extractor.trapcache.DefaultTrapCache;
+import com.semmle.js.extractor.trapcache.DummyTrapCache;
+import com.semmle.js.extractor.trapcache.ITrapCache;
+import com.semmle.js.parser.ParsedProject;
+import com.semmle.js.parser.TypeScriptParser;
+import com.semmle.ts.extractor.TypeExtractor;
+import com.semmle.ts.extractor.TypeTable;
+import com.semmle.util.data.StringUtil;
+import com.semmle.util.exception.CatastrophicError;
+import com.semmle.util.exception.Exceptions;
+import com.semmle.util.exception.ResourceError;
+import com.semmle.util.exception.UserError;
+import com.semmle.util.extraction.ExtractorOutputConfig;
+import com.semmle.util.files.FileUtil;
+import com.semmle.util.io.WholeIO;
+import com.semmle.util.io.csv.CSVReader;
+import com.semmle.util.language.LegacyLanguage;
+import com.semmle.util.process.Env;
+import com.semmle.util.projectstructure.ProjectLayout;
+import com.semmle.util.trap.TrapWriter;
+
 /**
  * An alternative entry point to the JavaScript extractor.
  *
@@ -388,9 +397,10 @@ public class AutoBuild {
     for (FileType filetype : defaultExtract)
       for (String extension : filetype.getExtensions()) patterns.add("**/*" + extension);
 
-    // include .eslintrc files and package.json files
+    // include .eslintrc files, package.json files, and tsconfig.json files
     patterns.add("**/.eslintrc*");
     patterns.add("**/package.json");
+    patterns.add("**/tsconfig.json");
 
     // include any explicitly specified extensions
     for (String extension : fileTypes.keySet()) patterns.add("**/*" + extension);
@@ -545,12 +555,15 @@ public class AutoBuild {
     List<Path> tsconfigFiles = new ArrayList<>();
     findFilesToExtract(defaultExtractor, filesToExtract, tsconfigFiles);
 
+    DependencyInstallationResult dependencyInstallationResult = DependencyInstallationResult.empty;
     if (!tsconfigFiles.isEmpty() && this.installDependencies) {
-      this.installDependencies(filesToExtract);
+      dependencyInstallationResult = this.installDependencies(filesToExtract);
     }
 
     // extract TypeScript projects and files
-    Set<Path> extractedFiles = extractTypeScript(defaultExtractor, filesToExtract, tsconfigFiles);
+    Set<Path> extractedFiles =
+          extractTypeScript(
+              defaultExtractor, filesToExtract, tsconfigFiles, dependencyInstallationResult);
 
     // extract remaining files
     for (Path f : filesToExtract) {
@@ -587,36 +600,255 @@ public class AutoBuild {
     }
   }
 
-  protected void installDependencies(Set<Path> filesToExtract) {
-    if (!verifyYarnInstallation()) {
-      return;
+  /**
+   * Returns an existing file named <code>dir/stem.ext</code> where <code>.ext</code> is any
+   * of the given extensions, or <code>null</code> if no such file exists.
+   */
+  private static Path tryResolveWithExtensions(Path dir, String stem, Iterable<String> extensions) {
+    for (String ext : extensions) {
+      Path path = dir.resolve(stem + ext);
+      if (Files.exists(dir.resolve(path))) {
+        return path;
+      }
     }
+    return null;
+  }
+  
+  /**
+   * Returns an existing file named <code>dir/stem.ext</code> where <code>ext</code> is any TypeScript or JavaScript extension,
+   * or <code>null</code> if no such file exists.
+   */
+  private static Path tryResolveTypeScriptOrJavaScriptFile(Path dir, String stem) {
+    Path resolved = tryResolveWithExtensions(dir, stem, FileType.TYPESCRIPT.getExtensions());
+    if (resolved != null) return resolved;
+    return tryResolveWithExtensions(dir, stem, FileType.JS.getExtensions());
+  }
+  
+  /**
+   * Gets a child of a JSON object as a string, or <code>null</code>.
+   */
+  private String getChildAsString(JsonObject obj, String name) {
+     JsonElement child = obj.get(name);
+     if (child instanceof JsonPrimitive && ((JsonPrimitive)child).isString()) {
+       return child.getAsString();
+     }
+     return null;
+  }
+
+  /**
+   * Installs dependencies for use by the TypeScript type checker.
+   * <p>
+   * Some packages must be downloaded while others exist within the same repo ("monorepos")
+   * but are not in a location where TypeScript would look for it.
+   * <p>
+   * Downloaded packages are intalled under <tt>SCRATCH_DIR</tt>, in a mirrored directory hierarchy
+   * we call the "virtual source root".
+   * Each <tt>package.json</tt> file is rewritten and copied to the virtual source root,
+   * where <tt>yarn install</tt> is invoked.
+   * <p>
+   * Packages that exists within the repo are stripped from the dependencies
+   * before installation, so they are not downloaded. Since they are part of the main source tree,
+   * these packages are not mirrored under the virtual source root.
+   * Instead, an explicit package location mapping is passed to the TypeScript parser wrapper.
+   * <p>
+   * The TypeScript parser wrapper then overrides module resolution so packages can be found
+   * under the virtual source root and via that package location mapping.
+   */
+  protected DependencyInstallationResult installDependencies(Set<Path> filesToExtract) {
+    if (!verifyYarnInstallation()) {
+      return DependencyInstallationResult.empty;
+    }
+    
+    final Path sourceRoot = Paths.get(".").toAbsolutePath();
+    final Path virtualSourceRoot = Paths.get(EnvironmentVariables.getScratchDir()).toAbsolutePath();
+
+    // Read all package.json files and index them by name.
+    Map<Path, JsonObject> packageJsonFiles = new LinkedHashMap<>();
+    Map<String, Path> packagesInRepo = new LinkedHashMap<>();
+    Map<String, Path> packageMainFile = new LinkedHashMap<>();
     for (Path file : filesToExtract) {
       if (file.getFileName().toString().equals("package.json")) {
-        System.out.println("Installing dependencies from " + file);
-        ProcessBuilder pb =
-            new ProcessBuilder(
-                Arrays.asList(
-                    "yarn",
-                    "install",
-                    "--non-interactive",
-                    "--ignore-scripts",
-                    "--ignore-platform",
-                    "--ignore-engines",
-                    "--ignore-optional",
-                    "--no-default-rc",
-                    "--no-bin-links",
-                    "--pure-lockfile"));
-        pb.directory(file.getParent().toFile());
-        pb.redirectOutput(Redirect.INHERIT);
-        pb.redirectError(Redirect.INHERIT);
         try {
-          pb.start().waitFor(this.installDependenciesTimeout, TimeUnit.MILLISECONDS);
-        } catch (IOException | InterruptedException ex) {
-          throw new ResourceError("Could not install dependencies from " + file, ex);
+          String text = new WholeIO().read(file);
+          JsonElement json = new JsonParser().parse(text);
+          if (!(json instanceof JsonObject)) continue;
+          JsonObject jsonObject = (JsonObject) json;
+          file = file.toAbsolutePath();
+          packageJsonFiles.put(file, jsonObject);
+
+          String name = getChildAsString(jsonObject, "name");
+          if (name != null) {
+            packagesInRepo.put(name, file);
+          }
+        } catch (JsonParseException e) {
+          System.err.println("Could not parse JSON file: " + file);
+          System.err.println(e);
+          // Continue without the malformed package.json file
         }
       }
     }
+
+    // Process all package.json files now that we know the names of all local packages.
+    // - remove dependencies on local packages
+    // - guess the main file for each package
+    // Note that we ignore optional dependencies during installation, so "optionalDependencies"
+    // is ignored here as well.
+    final List<String> dependencyFields =
+        Arrays.asList("dependencies", "devDependencies", "peerDependencies");
+    packageJsonFiles.forEach(
+        (path, packageJson) -> {
+          Path relativePath = sourceRoot.relativize(path);
+          for (String dependencyField : dependencyFields) {
+            JsonElement dependencyElm = packageJson.get(dependencyField);
+            if (!(dependencyElm instanceof JsonObject)) continue;
+            JsonObject dependencyObj = (JsonObject) dependencyElm;
+            List<String> propsToRemove = new ArrayList<>();
+            for (String packageName : dependencyObj.keySet()) {
+              if (packagesInRepo.containsKey(packageName)) {
+                // Remove dependency on local package
+                propsToRemove.add(packageName);
+              } else {
+                // Remove file dependency on a package that doesn't exist in the checkout.
+                String dependency = getChildAsString(dependencyObj, packageName);
+                if (dependency != null && (dependency.startsWith("file:") || dependency.startsWith("./") || dependency.startsWith("../"))) {
+                    if (dependency.startsWith("file:")) {
+                      dependency = dependency.substring("file:".length());
+                    }
+                    Path resolvedPackage = path.getParent().resolve(dependency + "/package.json");
+                    if (!Files.exists(resolvedPackage)) {
+                      propsToRemove.add(packageName);
+                    }
+                }
+              }
+            }
+            for (String prop : propsToRemove) {
+              dependencyObj.remove(prop);
+            }
+          }
+          // For named packages, find the main file.
+          String name = getChildAsString(packageJson, "name");
+          if (name != null) {
+            Path entryPoint = guessPackageMainFile(path, packageJson, FileType.TYPESCRIPT.getExtensions());
+            if (entryPoint == null) {
+              // Try a TypeScript-recognized JS extension instead
+              entryPoint = guessPackageMainFile(path, packageJson, Arrays.asList(".js", ".jsx"));
+            }
+            if (entryPoint != null) {
+              System.out.println(relativePath + ": Main file set to " + sourceRoot.relativize(entryPoint));
+              packageMainFile.put(name, entryPoint);
+            } else {
+              System.out.println(relativePath + ": Main file not found");
+            }
+          }
+        });
+
+    // Write the new package.json files to disk
+    for (Path file : packageJsonFiles.keySet()) {
+      Path relativePath = sourceRoot.relativize(file);
+      Path virtualFile = virtualSourceRoot.resolve(relativePath);
+      
+      try {
+        Files.createDirectories(virtualFile.getParent());
+        try (Writer writer = Files.newBufferedWriter(virtualFile)) {
+          new Gson().toJson(packageJsonFiles.get(file), writer);
+        }
+      } catch (IOException e) {
+        throw new ResourceError("Could not rewrite package.json file: " + virtualFile, e);
+      }
+    }
+
+    // Install dependencies
+    for (Path file : packageJsonFiles.keySet()) {
+      Path virtualFile = virtualSourceRoot.resolve(sourceRoot.relativize(file));
+      System.out.println("Installing dependencies from " + virtualFile);
+      ProcessBuilder pb =
+          new ProcessBuilder(
+              Arrays.asList(
+                  "yarn",
+                  "install",
+                  "--non-interactive",
+                  "--ignore-scripts",
+                  "--ignore-platform",
+                  "--ignore-engines",
+                  "--ignore-optional",
+                  "--no-default-rc",
+                  "--no-bin-links",
+                  "--pure-lockfile"));
+      pb.directory(virtualFile.getParent().toFile());
+      pb.redirectOutput(Redirect.INHERIT);
+      pb.redirectError(Redirect.INHERIT);
+      try {
+        pb.start().waitFor(this.installDependenciesTimeout, TimeUnit.MILLISECONDS);
+      } catch (IOException | InterruptedException ex) {
+        throw new ResourceError("Could not install dependencies from " + file, ex);
+      }
+    }
+
+    return new DependencyInstallationResult(virtualSourceRoot, packageMainFile, packagesInRepo);
+  }
+
+  /**
+   * Attempts to find a TypeScript file that acts as the main entry point to the
+   * given package - that is, the file you get when importing the package by name
+   * without any path suffix.
+   */
+  private Path guessPackageMainFile(Path packageJsonFile, JsonObject packageJson, Iterable<String> extensions) {
+    Path packageDir = packageJsonFile.getParent();
+    
+    // Try <package_dir>/index.ts.
+    Path resolved = tryResolveWithExtensions(packageDir, "index", extensions);
+    if (resolved != null) {
+      return resolved;
+    }
+
+    // Get the "main" property from the package.json
+    // This usually refers to the compiled output, such as `./out/foo.js` but may hint as to
+    // the name of main file ("foo" in this case).
+    String mainStr = getChildAsString(packageJson, "main");
+
+    // Look for source files `./src` if it exists
+    Path sourceDir = packageDir.resolve("src");
+    if (Files.isDirectory(sourceDir)) {
+      // Try `src/index.ts`
+      resolved = tryResolveTypeScriptOrJavaScriptFile(sourceDir, "index");
+      if (resolved != null) {
+        return resolved;
+      }
+
+      // If "main" was defined, try to map it to a file in `src`.
+      // For example `out/dist/foo.bundle.js` might be mapped back to `src/foo.ts`.
+      if (mainStr != null) {
+        Path candidatePath = Paths.get(mainStr);
+
+        // Strip off prefix directories that don't exist under `src/`, such as `out` and `dist`.
+        while (candidatePath.getNameCount() > 1 && !Files.isDirectory(sourceDir.resolve(candidatePath.getParent()))) {
+          candidatePath = candidatePath.subpath(1, candidatePath.getNameCount());
+        }
+
+        // Strip off extensions until a file can be found
+        while (true) {
+          resolved = tryResolveWithExtensions(sourceDir, candidatePath.toString(), extensions);
+          if (resolved != null) {
+            return resolved;
+          }
+          Path withoutExt = candidatePath.resolveSibling(FileUtil.stripExtension(candidatePath.getFileName().toString()));
+          if (withoutExt.equals(candidatePath)) break; // No more extensions to strip
+          candidatePath = withoutExt;
+        }
+      }
+    }
+
+    // Try to resolve main as a sibling of the package.json file, such as "./main.js" -> "./main.ts".
+    if (mainStr != null) {
+      Path mainPath = Paths.get(mainStr);
+      String withoutExt = FileUtil.stripExtension(mainPath.getFileName().toString());
+      resolved = tryResolveWithExtensions(packageDir, withoutExt, extensions);
+      if (resolved != null) {
+        return resolved;
+      }
+    }
+
+    return null;
   }
 
   private ExtractorConfig mkExtractorConfig() {
@@ -628,7 +860,10 @@ public class AutoBuild {
   }
 
   private Set<Path> extractTypeScript(
-      FileExtractor extractor, Set<Path> files, List<Path> tsconfig) {
+      FileExtractor extractor,
+      Set<Path> files,
+      List<Path> tsconfig,
+      DependencyInstallationResult deps) {
     Set<Path> extractedFiles = new LinkedHashSet<>();
 
     if (hasTypeScriptFiles(files) || !tsconfig.isEmpty()) {
@@ -640,7 +875,7 @@ public class AutoBuild {
       for (Path projectPath : tsconfig) {
         File projectFile = projectPath.toFile();
         long start = logBeginProcess("Opening project " + projectFile);
-        ParsedProject project = tsParser.openProject(projectFile);
+        ParsedProject project = tsParser.openProject(projectFile, deps);
         logEndProcess(start, "Done opening project " + projectFile);
         // Extract all files belonging to this project which are also matched
         // by our include/exclude filters.
@@ -729,7 +964,8 @@ public class AutoBuild {
             // extract TypeScript projects from 'tsconfig.json'
             if (typeScriptMode == TypeScriptMode.FULL
                 && file.getFileName().endsWith("tsconfig.json")
-                && !excludes.contains(file)) {
+                && !excludes.contains(file)
+                && isFileIncluded(file)) {
               tsconfigFiles.add(file);
             }
 

javascript/extractor/src/com/semmle/js/extractor/DependencyInstallationResult.java

@@ -0,0 +1,48 @@
+package com.semmle.js.extractor;
+
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.Map;
+
+/** Contains the results of installing dependencies. */
+public class DependencyInstallationResult {
+  private Path virtualSourceRoot;
+  private Map<String, Path> packageEntryPoints;
+  private Map<String, Path> packageJsonFiles;
+
+  public static final DependencyInstallationResult empty =
+      new DependencyInstallationResult(null, Collections.emptyMap(), Collections.emptyMap());
+
+  public DependencyInstallationResult(
+      Path virtualSourceRoot,
+      Map<String, Path> packageEntryPoints,
+      Map<String, Path> packageJsonFiles) {
+    this.packageEntryPoints = packageEntryPoints;
+    this.packageJsonFiles = packageJsonFiles;
+  }
+
+  /**
+   * Returns the virtual source root or <code>null</code> if no virtual source root exists.
+   *
+   * The virtual source root is a directory hierarchy that mirrors the real source
+   * root, where dependencies are installed.
+   */
+  public Path getVirtualSourceRoot() {
+    return virtualSourceRoot;
+  }
+
+  /**
+   * Returns the mapping from package names to the TypeScript file that should
+   * act as its main entry point.
+   */
+  public Map<String, Path> getPackageEntryPoints() {
+    return packageEntryPoints;
+  }
+
+  /**
+   * Returns the mapping from package name to corresponding package.json.
+   */
+  public Map<String, Path> getPackageJsonFiles() {
+    return packageJsonFiles;
+  }
+}

javascript/extractor/src/com/semmle/js/extractor/EnvironmentVariables.java

@@ -7,16 +7,22 @@ import com.semmle.util.process.Env.Var;
 public class EnvironmentVariables {
   public static final String CODEQL_EXTRACTOR_JAVASCRIPT_ROOT_ENV_VAR =
       "CODEQL_EXTRACTOR_JAVASCRIPT_ROOT";
+ 
+  public static final String CODEQL_EXTRACTOR_JAVASCRIPT_SCRATCH_DIR_ENV_VAR =
+      "CODEQL_EXTRACTOR_JAVASCRIPT_SCRATCH_DIR";
+
+  public static final String LGTM_WORKSPACE_ENV_VAR =
+      "LGTM_WORKSPACE";
 
   /**
    * Gets the extractor root based on the <code>CODEQL_EXTRACTOR_JAVASCRIPT_ROOT</code> or <code>
    * SEMMLE_DIST</code> or environment variable, or <code>null</code> if neither is set.
    */
   public static String tryGetExtractorRoot() {
-    String env = Env.systemEnv().get(CODEQL_EXTRACTOR_JAVASCRIPT_ROOT_ENV_VAR);
-    if (env != null && !env.isEmpty()) return env;
-    env = Env.systemEnv().get(Var.SEMMLE_DIST);
-    if (env != null && !env.isEmpty()) return env;
+    String env = Env.systemEnv().getNonEmpty(CODEQL_EXTRACTOR_JAVASCRIPT_ROOT_ENV_VAR);
+    if (env != null) return env;
+    env = Env.systemEnv().getNonEmpty(Var.SEMMLE_DIST);
+    if (env != null) return env;
     return null;
   }
 
@@ -31,4 +37,16 @@ public class EnvironmentVariables {
     }
     return env;
   }
+
+  /**
+   * Gets the scratch directory from the appropriate environment variable.
+   */
+  public static String getScratchDir() {
+    String env = Env.systemEnv().getNonEmpty(CODEQL_EXTRACTOR_JAVASCRIPT_SCRATCH_DIR_ENV_VAR);
+    if (env != null) return env;
+    env = Env.systemEnv().getNonEmpty(LGTM_WORKSPACE_ENV_VAR);
+    if (env != null) return env;
+
+    throw new UserError(CODEQL_EXTRACTOR_JAVASCRIPT_SCRATCH_DIR_ENV_VAR + " or " + LGTM_WORKSPACE_ENV_VAR + " must be set");
+  }
 }

javascript/extractor/src/com/semmle/js/extractor/Main.java

@@ -140,7 +140,7 @@ public class Main {
     for (File projectFile : projectFiles) {
 
       long start = verboseLogStartTimer(ap, "Opening project " + projectFile);
-      ParsedProject project = tsParser.openProject(projectFile);
+      ParsedProject project = tsParser.openProject(projectFile, DependencyInstallationResult.empty);
       verboseLogEndTimer(ap, start);
       // Extract all files belonging to this project which are also matched
       // by our include/exclude filters.

javascript/extractor/src/com/semmle/js/extractor/test/AutoBuildTests.java

@@ -1,14 +1,5 @@
 package com.semmle.js.extractor.test;
 
-import com.semmle.js.extractor.AutoBuild;
-import com.semmle.js.extractor.ExtractorState;
-import com.semmle.js.extractor.FileExtractor;
-import com.semmle.js.extractor.FileExtractor.FileType;
-import com.semmle.util.data.StringUtil;
-import com.semmle.util.exception.UserError;
-import com.semmle.util.files.FileUtil;
-import com.semmle.util.files.FileUtil8;
-import com.semmle.util.process.Env;
 import java.io.File;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
@@ -25,12 +16,24 @@ import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Assume;
 import org.junit.Before;
 import org.junit.Test;
 
+import com.semmle.js.extractor.AutoBuild;
+import com.semmle.js.extractor.DependencyInstallationResult;
+import com.semmle.js.extractor.ExtractorState;
+import com.semmle.js.extractor.FileExtractor;
+import com.semmle.js.extractor.FileExtractor.FileType;
+import com.semmle.util.data.StringUtil;
+import com.semmle.util.exception.UserError;
+import com.semmle.util.files.FileUtil;
+import com.semmle.util.files.FileUtil8;
+import com.semmle.util.process.Env;
+
 public class AutoBuildTests {
   private Path SEMMLE_DIST, LGTM_SRC;
   private Set<String> expected;
@@ -129,8 +132,9 @@ public class AutoBuildTests {
         }
 
         @Override
-        protected void installDependencies(Set<Path> filesToExtract) {
+        protected DependencyInstallationResult installDependencies(Set<Path> filesToExtract) {
           // never install dependencies during testing
+          return DependencyInstallationResult.empty;
         }
 
         @Override

javascript/extractor/src/com/semmle/js/parser/TypeScriptParser.java

@@ -1,12 +1,31 @@
 package com.semmle.js.parser;
 
-import ch.qos.logback.classic.Level;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.lang.ProcessBuilder.Redirect;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
 import com.google.gson.JsonArray;
 import com.google.gson.JsonElement;
+import com.google.gson.JsonNull;
 import com.google.gson.JsonObject;
 import com.google.gson.JsonParseException;
 import com.google.gson.JsonParser;
 import com.google.gson.JsonPrimitive;
+import com.semmle.js.extractor.DependencyInstallationResult;
 import com.semmle.js.extractor.EnvironmentVariables;
 import com.semmle.js.extractor.ExtractionMetrics;
 import com.semmle.js.parser.JSParser.Result;
@@ -23,21 +42,8 @@ import com.semmle.util.logging.LogbackUtils;
 import com.semmle.util.process.AbstractProcessBuilder;
 import com.semmle.util.process.Builder;
 import com.semmle.util.process.Env;
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.ByteArrayOutputStream;
-import java.io.Closeable;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.OutputStream;
-import java.io.OutputStreamWriter;
-import java.lang.ProcessBuilder.Redirect;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
+
+import ch.qos.logback.classic.Level;
 
 /**
  * The Java half of our wrapper for invoking the TypeScript parser.
@@ -401,6 +407,21 @@ public class TypeScriptParser {
     checkResponseType(response, "ok");
   }
 
+  /**
+   * Converts a map to an array of [key, value] pairs.
+   */
+  private JsonArray mapToArray(Map<String, Path> map) {
+    JsonArray result = new JsonArray();
+    map.forEach(
+        (key, path) -> {
+          JsonArray entry = new JsonArray();
+          entry.add(key);
+          entry.add(path.toString());
+          result.add(entry);
+        });
+    return result;
+  }
+
   /**
    * Opens a new project based on a tsconfig.json file. The compiler will analyze all files in the
    * project.
@@ -409,10 +430,15 @@ public class TypeScriptParser {
    *
    * <p>Only one project should be opened at once.
    */
-  public ParsedProject openProject(File tsConfigFile) {
+  public ParsedProject openProject(File tsConfigFile, DependencyInstallationResult deps) {
     JsonObject request = new JsonObject();
     request.add("command", new JsonPrimitive("open-project"));
     request.add("tsConfig", new JsonPrimitive(tsConfigFile.getPath()));
+    request.add("packageEntryPoints", mapToArray(deps.getPackageEntryPoints()));
+    request.add("packageJsonFiles", mapToArray(deps.getPackageJsonFiles()));
+    request.add("virtualSourceRoot", deps.getVirtualSourceRoot() == null
+        ? JsonNull.INSTANCE
+        : new JsonPrimitive(deps.getVirtualSourceRoot().toString()));
     JsonObject response = talkToParserWrapper(request);
     try {
       checkResponseType(response, "project-opened");