From 7e8fb1428e666bfa901663d6c12591239d8fa60f Mon Sep 17 00:00:00 2001 From: Asger Feldthaus Date: Wed, 22 Jan 2020 15:03:03 +0000 Subject: [PATCH] TS: Support tsconfig.json extending from ./node_modules --- .../extractor/lib/typescript/src/common.ts | 21 ++----- .../extractor/lib/typescript/src/main.ts | 59 ++++++++++++++++--- .../lib/typescript/src/virtual_source_root.ts | 34 +++++++++++ .../com/semmle/js/extractor/AutoBuild.java | 2 +- .../DependencyInstallationResult.java | 23 ++++++-- .../semmle/js/parser/TypeScriptParser.java | 29 +++++---- 6 files changed, 127 insertions(+), 41 deletions(-) create mode 100644 javascript/extractor/lib/typescript/src/virtual_source_root.ts diff --git a/javascript/extractor/lib/typescript/src/common.ts b/javascript/extractor/lib/typescript/src/common.ts index a125e7b9bc3..729ebf8dcb4 100644 --- a/javascript/extractor/lib/typescript/src/common.ts +++ b/javascript/extractor/lib/typescript/src/common.ts @@ -1,6 +1,7 @@ import * as ts from "./typescript"; import { TypeTable } from "./type_table"; import * as pathlib from "path"; +import { VirtualSourceRoot } from "./virtual_source_root"; /** * Extracts the package name from the prefix of an import string. @@ -12,11 +13,9 @@ export class Project { public program: ts.Program = null; private host: ts.CompilerHost; private resolutionCache: ts.ModuleResolutionCache; - private sourceRoot: string; - /** Directory whose folder structure mirrors the real source root, but with `node_modules` installed. */ - private virtualSourceRoot: string; - constructor(public tsConfig: string, public config: ts.ParsedCommandLine, public typeTable: TypeTable, public packageLocations: PackageLocationMap) { + constructor(public tsConfig: string, public config: ts.ParsedCommandLine, public typeTable: TypeTable, public packageLocations: PackageLocationMap, + public virtualSourceRoot: VirtualSourceRoot) { this.resolveModuleNames = this.resolveModuleNames.bind(this); this.resolutionCache = ts.createModuleResolutionCache(pathlib.dirname(tsConfig), ts.sys.realpath, config.options); @@ -24,9 +23,6 @@ export class Project { host.resolveModuleNames = this.resolveModuleNames; host.trace = undefined; // Disable tracing which would otherwise go to standard out this.host = host; - - this.sourceRoot = process.cwd(); - this.virtualSourceRoot = process.env["CODEQL_EXTRACTOR_JAVASCRIPT_SCRATCH_DIR"]; } public unload(): void { @@ -83,7 +79,7 @@ export class Project { if (packageEntryPoint == null) { // The package is not overridden, but we have established that it begins with a valid package name. // Do a lookup in the virtual source root (where dependencies are installed) by changing the 'containing file'. - let virtualContainingFile = this.toVirtualPath(containingFile); + let virtualContainingFile = this.virtualSourceRoot.toVirtualPath(containingFile); if (virtualContainingFile != null) { return ts.resolveModuleName(moduleName, virtualContainingFile, options, this.host, this.resolutionCache).resolvedModule; } @@ -119,15 +115,6 @@ export class Project { return null; } - - /** - * Maps a path under the real source root to the corresonding path in the virtual source root. - */ - private toVirtualPath(path: string) { - let relative = pathlib.relative(this.sourceRoot, path); - if (relative.startsWith('..') || pathlib.isAbsolute(relative)) return null; - return pathlib.join(this.virtualSourceRoot, relative); - } } export type PackageLocationMap = Map; diff --git a/javascript/extractor/lib/typescript/src/main.ts b/javascript/extractor/lib/typescript/src/main.ts index 3a488456f55..750d1699f1c 100644 --- a/javascript/extractor/lib/typescript/src/main.ts +++ b/javascript/extractor/lib/typescript/src/main.ts @@ -37,8 +37,9 @@ import * as readline from "readline"; import * as ts from "./typescript"; import * as ast_extractor from "./ast_extractor"; -import { Project } from "./common"; +import { Project, PackageLocationMap } from "./common"; import { TypeTable } from "./type_table"; +import { VirtualSourceRoot } from "./virtual_source_root"; interface ParseCommand { command: "parse"; @@ -47,7 +48,8 @@ interface ParseCommand { interface OpenProjectCommand { command: "open-project"; tsConfig: string; - packageLocations: [string, string][]; + packageEntryPoints: [string, string][]; + packageJsonFiles: [string, string][]; } interface CloseProjectCommand { command: "close-project"; @@ -243,20 +245,62 @@ function parseSingleFile(filename: string): {ast: ts.SourceFile, code: string} { return {ast, code}; } +const nodeModulesRex = /[/\\]node_modules[/\\]((?:@[\w.-]+[/\\])?\w[\w.-]*)[/\\](.*)/; + function handleOpenProjectCommand(command: OpenProjectCommand) { Error.stackTraceLimit = Infinity; let tsConfigFilename = String(command.tsConfig); let tsConfig = ts.readConfigFile(tsConfigFilename, ts.sys.readFile); let basePath = pathlib.dirname(tsConfigFilename); + let packageEntryPoints = new Map(command.packageEntryPoints); + let packageJsonFiles = new Map(command.packageJsonFiles); + let virtualSourceRoot = new VirtualSourceRoot(process.cwd(), process.env["CODEQL_EXTRACTOR_JAVASCRIPT_SCRATCH_DIR"]); + + /** + * Rewrites path segments of form `node_modules/PACK/suffix` to be relative to + * the location of package PACK in the source tree, if it exists. + */ + function redirectNodeModulesPath(path: string) { + let nodeModulesMatch = nodeModulesRex.exec(path); + if (nodeModulesMatch == null) return null; + let packageName = nodeModulesMatch[1]; + let packageJsonFile = packageJsonFiles.get(packageName); + if (packageJsonFile == null) return null; + let packageDir = pathlib.dirname(packageJsonFile); + let suffix = nodeModulesMatch[2]; + let finalPath = pathlib.join(packageDir, suffix); + if (!ts.sys.fileExists(finalPath)) return null; + return finalPath; + } + + /** + * Create the host passed to the tsconfig.json parser. + * + * We override its file system access in case there is an "extends" + * clause pointing into "./node_modules", which must be redirected to + * the location of an installed package or a checked-in package. + */ let parseConfigHost: ts.ParseConfigHost = { useCaseSensitiveFileNames: true, - readDirectory: ts.sys.readDirectory, - fileExists: (path: string) => fs.existsSync(path), - readFile: ts.sys.readFile, + readDirectory: ts.sys.readDirectory, // No need to override traversal/glob matching + fileExists: (path: string) => { + return ts.sys.fileExists(path) + || virtualSourceRoot.toVirtualPathIfFileExists(path) != null + || redirectNodeModulesPath(path) != null; + }, + readFile: (path: string) => { + if (!fs.existsSync(path)) { + let virtualPath = virtualSourceRoot.toVirtualPathIfFileExists(path); + if (virtualPath != null) return ts.sys.readFile(virtualPath); + virtualPath = redirectNodeModulesPath(path); + if (virtualPath != null) return ts.sys.readFile(virtualPath); + } + return ts.sys.readFile(path); + } }; let config = ts.parseJsonConfigFileContent(tsConfig.config, parseConfigHost, basePath); - let project = new Project(tsConfigFilename, config, state.typeTable, new Map(command.packageLocations)); + let project = new Project(tsConfigFilename, config, state.typeTable, packageEntryPoints, virtualSourceRoot); project.load(); state.project = project; @@ -530,7 +574,8 @@ if (process.argv.length > 2) { handleOpenProjectCommand({ command: "open-project", tsConfig: argument, - packageLocations: [], + packageEntryPoints: [], + packageJsonFiles: [], }); for (let sf of state.project.program.getSourceFiles()) { if (pathlib.basename(sf.fileName) === "lib.d.ts") continue; diff --git a/javascript/extractor/lib/typescript/src/virtual_source_root.ts b/javascript/extractor/lib/typescript/src/virtual_source_root.ts new file mode 100644 index 00000000000..ef7e481d666 --- /dev/null +++ b/javascript/extractor/lib/typescript/src/virtual_source_root.ts @@ -0,0 +1,34 @@ +import * as pathlib from "path"; +import * as ts from "./typescript"; + +/** + * Mapping from the source root to the virtual source root. + */ +export class VirtualSourceRoot { + constructor( + private sourceRoot: string, + + /** Directory whose folder structure mirrors the real source root, but with `node_modules` installed. */ + private virtualSourceRoot: string, + ) {} + + /** + * Maps a path under the real source root to the corresonding path in the virtual source root. + */ + public toVirtualPath(path: string) { + let relative = pathlib.relative(this.sourceRoot, path); + if (relative.startsWith('..') || pathlib.isAbsolute(relative)) return null; + return pathlib.join(this.virtualSourceRoot, relative); + } + + /** + * Maps a path under the real source root to the corresonding path in the virtual source root. + */ + public toVirtualPathIfFileExists(path: string) { + let virtualPath = this.toVirtualPath(path); + if (virtualPath != null && ts.sys.fileExists(virtualPath)) { + return virtualPath; + } + return null; + } +} diff --git a/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java b/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java index 417e40a2e7a..1db32a43fcc 100644 --- a/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java +++ b/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java @@ -782,7 +782,7 @@ public class AutoBuild { } } - return new DependencyInstallationResult(packageMainFile); + return new DependencyInstallationResult(packageMainFile, packagesInRepo); } /** diff --git a/javascript/extractor/src/com/semmle/js/extractor/DependencyInstallationResult.java b/javascript/extractor/src/com/semmle/js/extractor/DependencyInstallationResult.java index 27b34369f04..5dd6bd60b6a 100644 --- a/javascript/extractor/src/com/semmle/js/extractor/DependencyInstallationResult.java +++ b/javascript/extractor/src/com/semmle/js/extractor/DependencyInstallationResult.java @@ -6,20 +6,31 @@ import java.util.Map; /** Contains the results of installing dependencies. */ public class DependencyInstallationResult { - private Map packageLocations; + private Map packageEntryPoints; + private Map packageJsonFiles; public static final DependencyInstallationResult empty = - new DependencyInstallationResult(Collections.emptyMap()); + new DependencyInstallationResult(Collections.emptyMap(), Collections.emptyMap()); - public DependencyInstallationResult(Map localPackages) { - this.packageLocations = localPackages; + public DependencyInstallationResult( + Map packageEntryPoints, + Map packageJsonFiles) { + this.packageEntryPoints = packageEntryPoints; + this.packageJsonFiles = packageJsonFiles; } /** * Returns the mapping from package names to the TypeScript file that should * act as its main entry point. */ - public Map getPackageLocations() { - return packageLocations; + public Map getPackageEntryPoints() { + return packageEntryPoints; + } + + /** + * Returns the mapping from package name to corresponding package.json. + */ + public Map getPackageJsonFiles() { + return packageJsonFiles; } } diff --git a/javascript/extractor/src/com/semmle/js/parser/TypeScriptParser.java b/javascript/extractor/src/com/semmle/js/parser/TypeScriptParser.java index c842c6e39ba..27ed0e720fb 100644 --- a/javascript/extractor/src/com/semmle/js/parser/TypeScriptParser.java +++ b/javascript/extractor/src/com/semmle/js/parser/TypeScriptParser.java @@ -11,10 +11,12 @@ import java.io.InputStreamReader; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.lang.ProcessBuilder.Redirect; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; import com.google.gson.JsonArray; import com.google.gson.JsonElement; @@ -404,6 +406,21 @@ public class TypeScriptParser { checkResponseType(response, "ok"); } + /** + * Converts a map to an array of [key, value] pairs. + */ + private JsonArray mapToArray(Map map) { + JsonArray result = new JsonArray(); + map.forEach( + (key, path) -> { + JsonArray entry = new JsonArray(); + entry.add(key); + entry.add(path.toString()); + result.add(entry); + }); + return result; + } + /** * Opens a new project based on a tsconfig.json file. The compiler will analyze all files in the * project. @@ -416,16 +433,8 @@ public class TypeScriptParser { JsonObject request = new JsonObject(); request.add("command", new JsonPrimitive("open-project")); request.add("tsConfig", new JsonPrimitive(tsConfigFile.getPath())); - JsonArray packageLocations = new JsonArray(); - deps.getPackageLocations() - .forEach( - (packageName, packageDir) -> { - JsonArray entry = new JsonArray(); - entry.add(packageName); - entry.add(packageDir.toString()); - packageLocations.add(entry); - }); - request.add("packageLocations", packageLocations); + request.add("packageEntryPoints", mapToArray(deps.getPackageEntryPoints())); + request.add("packageJsonFiles", mapToArray(deps.getPackageJsonFiles())); JsonObject response = talkToParserWrapper(request); try { checkResponseType(response, "project-opened");