hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-05-05 13:45:19 +02:00
Code Issues Packages Projects Releases Wiki Activity

TS: Install deps under scratch dir

Browse Source
This commit is contained in:
Asger Feldthaus
2020-01-16 15:56:31 +00:00
parent 303bac9710
commit a220268ad8
8 changed files with 238 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files

111
javascript/extractor/lib/typescript/src/common.ts
View File

@@ -1,10 +1,33 @@
import * as ts from "./typescript";
import { TypeTable } from "./type_table";
import * as pathlib from "path";
/**
* Extracts the package name from the prefix of an import string.
*/
const packageNameRex = /^(?:@[\w.-]+[/\\])?\w[\w.-]*(?=[/\\]|$)/;
const extensions = ['.ts', '.tsx', '.d.ts'];
export class Project {
public program: ts.Program = null;
private host: ts.CompilerHost;
private resolutionCache: ts.ModuleResolutionCache;
private sourceRoot: string;
/** Directory whose folder structure mirrors the real source root, but with `node_modules` installed. */
private virtualSourceRoot: string;
constructor(public tsConfig: string, public config: ts.ParsedCommandLine, public typeTable: TypeTable) {}
constructor(public tsConfig: string, public config: ts.ParsedCommandLine, public typeTable: TypeTable, public packageLocations: PackageLocationMap) {
this.resolveModuleNames = this.resolveModuleNames.bind(this);
this.resolutionCache = ts.createModuleResolutionCache(pathlib.dirname(tsConfig), ts.sys.realpath, config.options);
let host = ts.createCompilerHost(config.options, true);
host.resolveModuleNames = this.resolveModuleNames;
host.trace = undefined; // Disable tracing which would otherwise go to standard out
this.host = host;
this.sourceRoot = process.cwd();
this.virtualSourceRoot = process.env["CODEQL_EXTRACTOR_JAVASCRIPT_SCRATCH_DIR"];
}
public unload(): void {
this.typeTable.releaseProgram();
@@ -12,9 +35,8 @@ export class Project {
}
public load(): void {
let host = ts.createCompilerHost(this.config.options, true);
host.trace = undefined; // Disable tracing which would otherwise go to standard out
this.program = ts.createProgram(this.config.fileNames, this.config.options, host);
const { config, host } = this;
this.program = ts.createProgram(config.fileNames, config.options, host);
this.typeTable.setProgram(this.program);
}
@@ -27,4 +49,85 @@ export class Project {
this.unload();
this.load();
}
/**
* Override for module resolution in the TypeScript compiler host.
*/
private resolveModuleNames(
moduleNames: string[],
containingFile: string,
reusedNames: string[],
redirectedReference: ts.ResolvedProjectReference,
options: ts.CompilerOptions) {
const { host, resolutionCache } = this;
return moduleNames.map((moduleName) => {
let redirected = this.redirectModuleName(moduleName, containingFile, options);
if (redirected != null) return redirected;
return ts.resolveModuleName(moduleName, containingFile, options, host, resolutionCache).resolvedModule;
});
}
/**
* Returns the path that the given import string should be redirected to, or null if it should
* fall back to standard module resolution.
*/
private redirectModuleName(moduleName: string, containingFile: string, options: ts.CompilerOptions): ts.ResolvedModule {
// Get a package name from the leading part of the module name, e.g. '@scope/foo' from '@scope/foo/bar'.
let packageNameMatch = packageNameRex.exec(moduleName);
if (packageNameMatch == null) return null;
let packageName = packageNameMatch[0];
// Get the overridden location of this package, if one exists.
let packageEntryPoint = this.packageLocations.get(packageName);
if (packageEntryPoint == null) {
// The package is not overridden, but we have established that it begins with a valid package name.
// Do a lookup in the virtual source root (where dependencies are installed) by changing the 'containing file'.
let virtualContainingFile = this.toVirtualPath(containingFile);
if (virtualContainingFile != null) {
return ts.resolveModuleName(moduleName, virtualContainingFile, options, this.host, this.resolutionCache).resolvedModule;
}
return null;
}
// If the requested module name is exactly the overridden package name,
// return the entry point file (it is not necessarily called `index.ts`).
if (moduleName.length === packageName.length) {
return { resolvedFileName: packageEntryPoint, isExternalLibraryImport: true };
}
// Get the suffix after the package name, e.g. the '/bar' in '@scope/foo/bar'.
let suffix = moduleName.substring(packageName.length);
// Resolve the suffix relative to the package directory.
let packageDir = pathlib.dirname(packageEntryPoint);
let joinedPath = pathlib.join(packageDir, suffix);
// Add implicit '/index'
if (ts.sys.directoryExists(joinedPath)) {
joinedPath = pathlib.join(joinedPath, 'index');
}
// Try each recognized extension. We must not return a file whose extension is not
// recognized by TypeScript.
for (let ext of extensions) {
let candidate = joinedPath.endsWith(ext) ? joinedPath : (joinedPath + ext);
if (ts.sys.fileExists(candidate)) {
return { resolvedFileName: candidate, isExternalLibraryImport: true };
}
}
return null;
}
/**
* Maps a path under the real source root to the corresonding path in the virtual source root.
*/
private toVirtualPath(path: string) {
let relative = pathlib.relative(this.sourceRoot, path);
if (relative.startsWith('..') || pathlib.isAbsolute(relative)) return null;
return pathlib.join(this.virtualSourceRoot, relative);
}
}
export type PackageLocationMap = Map<string, string>;

4
javascript/extractor/lib/typescript/src/main.ts
View File

@@ -47,6 +47,7 @@ interface ParseCommand {
interface OpenProjectCommand {
command: "open-project";
tsConfig: string;
packageLocations: [string, string][];
}
interface CloseProjectCommand {
command: "close-project";
@@ -255,7 +256,7 @@ function handleOpenProjectCommand(command: OpenProjectCommand) {
readFile: ts.sys.readFile,
};
let config = ts.parseJsonConfigFileContent(tsConfig.config, parseConfigHost, basePath);
let project = new Project(tsConfigFilename, config, state.typeTable);
let project = new Project(tsConfigFilename, config, state.typeTable, new Map(command.packageLocations));
project.load();
state.project = project;
@@ -529,6 +530,7 @@ if (process.argv.length > 2) {
handleOpenProjectCommand({
command: "open-project",
tsConfig: argument,
packageLocations: [],
});
for (let sf of state.project.program.getSourceFiles()) {
if (pathlib.basename(sf.fileName) === "lib.d.ts") continue;

156
javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java
View File

@@ -17,7 +17,6 @@ import java.nio.file.InvalidPathException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.StandardCopyOption;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Arrays;
@@ -29,7 +28,6 @@ import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
import java.util.stream.Stream;
import com.google.gson.Gson;
@@ -204,6 +202,7 @@ public class AutoBuild {
private final Set<String> xmlExtensions = new LinkedHashSet<>();
private ProjectLayout filters;
private final Path LGTM_SRC, SEMMLE_DIST;
private final Path scratchDir;
private final TypeScriptMode typeScriptMode;
private final String defaultEncoding;
private ExecutorService threadPool;
@@ -217,6 +216,7 @@ public class AutoBuild {
public AutoBuild() {
this.LGTM_SRC = toRealPath(getPathFromEnvVar("LGTM_SRC"));
this.SEMMLE_DIST = Paths.get(EnvironmentVariables.getExtractorRoot());
this.scratchDir = Paths.get(EnvironmentVariables.getScratchDir());
this.outputConfig = new ExtractorOutputConfig(LegacyLanguage.JAVASCRIPT);
this.trapCache = mkTrapCache();
this.typeScriptMode =
@@ -563,12 +563,9 @@ public class AutoBuild {
}
// extract TypeScript projects and files
Set<Path> extractedFiles;
try {
extractedFiles = extractTypeScript(defaultExtractor, filesToExtract, tsconfigFiles);
} finally {
restoreOriginalFiles(dependencyInstallationResult.getOriginalFiles());
}
Set<Path> extractedFiles =
extractTypeScript(
defaultExtractor, filesToExtract, tsconfigFiles, dependencyInstallationResult);
// extract remaining files
for (Path f : filesToExtract) {
@@ -605,19 +602,6 @@ public class AutoBuild {
}
}
protected void restoreOriginalFiles(Map<Path, Path> originalFiles) {
originalFiles.forEach(
(file, original) -> {
try {
Files.move(original, file, StandardCopyOption.REPLACE_EXISTING);
} catch (IOException e) {
throw new ResourceError("Could not restore original file: " + file, e);
}
});
}
private static final Pattern validPackageName = Pattern.compile("(@[\\w.-]+/)?\\w[\\w.-]*");
private static Path tryResolveWithExtensions(Path dir, String stem, Iterable<String> extensions) {
for (String ext : extensions) {
Path path = dir.resolve(stem + ext);
@@ -633,18 +617,27 @@ public class AutoBuild {
if (resolved != null) return resolved;
return tryResolveWithExtensions(dir, stem, FileType.JS.getExtensions());
}
private String getChildAsString(JsonObject obj, String name) {
JsonElement child = obj.get(name);
if (child instanceof JsonPrimitive && ((JsonPrimitive)child).isString()) {
return child.getAsString();
}
return null;
}
protected DependencyInstallationResult installDependencies(Set<Path> filesToExtract) {
if (!verifyYarnInstallation()) {
return DependencyInstallationResult.empty;
}
final Path sourceRoot = Paths.get(".").toAbsolutePath();
final Path virtualSourceRoot = this.scratchDir.toAbsolutePath();
Path rootNodeModules = Paths.get("node_modules");
// Read all package.json files, and install symlinks to them.
// Read all package.json files and index them by name.
Map<Path, JsonObject> packageJsonFiles = new LinkedHashMap<>();
Set<String> packagesInRepo = new LinkedHashSet<>();
Map<String, Path> packagesInRepo = new LinkedHashMap<>();
Map<String, Path> packageMainFile = new LinkedHashMap<>();
for (Path file : filesToExtract) {
if (file.getFileName().toString().equals("package.json")) {
try {
@@ -655,26 +648,9 @@ public class AutoBuild {
file = file.toAbsolutePath();
packageJsonFiles.put(file, jsonObject);
JsonElement nameElm = jsonObject.get("name");
if (nameElm instanceof JsonPrimitive && ((JsonPrimitive) nameElm).isString()) {
String name = nameElm.getAsString();
packagesInRepo.add(name);
if (validPackageName.matcher(name).matches()) {
// Create a symlink to the package: <checkout>/node_modules/foo -> /path/to/foo
try {
Path symlinkPath = rootNodeModules.resolve(name);
if (!Files.exists(symlinkPath)) {
Files.createDirectories(symlinkPath.getParent());
Files.createSymbolicLink(symlinkPath, file.getParent());
} else {
// If node_modules/foo already exists, presumably it contains the right thing,
// so just continue extraction with that in place.
}
} catch (IOException e) {
throw new ResourceError("Could not install symlink to package " + file, e);
}
}
String name = getChildAsString(jsonObject, "name");
if (name != null) {
packagesInRepo.put(name, file);
}
} catch (JsonParseException e) {
System.err.println("Could not parse JSON file: " + file);
@@ -684,59 +660,75 @@ public class AutoBuild {
}
}
// Remove all dependencies on local packages from package.json files so yarn doesn't
// try to download them. Yarn would fail otherwise if these packages were never published.
// These packages will instead be found through the symlink we installed in node_modules above.
// Process all package.json files now that we know the names of all local packages.
// - remove dependencies on local packages
// - guess the main file for each package
// Note that we ignore optional dependencies during installation, so "optionalDependencies"
// is ignored here as well.
final List<String> dependencyFields =
Arrays.asList("dependencies", "devDependencies", "peerDependencies");
final Set<Path> filesToChange = new LinkedHashSet<>();
packageJsonFiles.forEach(
(path, packageJson) -> {
Path relativePath = sourceRoot.relativize(path);
for (String dependencyField : dependencyFields) {
JsonElement dependencyElm = packageJson.get(dependencyField);
if (!(dependencyElm instanceof JsonObject)) continue;
JsonObject dependencyObj = (JsonObject) dependencyElm;
for (String packageName : packagesInRepo) {
if (!dependencyObj.has(packageName)) continue;
dependencyObj.remove(packageName);
filesToChange.add(path);
List<String> propsToRemove = new ArrayList<>();
for (String packageName : dependencyObj.keySet()) {
if (packagesInRepo.containsKey(packageName)) {
// Remove dependency on local package
propsToRemove.add(packageName);
} else {
// Remove file dependency on a package that don't exist in the checkout.
String dependecy = getChildAsString(dependencyObj, packageName);
if (dependecy != null && (dependecy.startsWith("file:") || dependecy.startsWith("./") || dependecy.startsWith("../"))) {
if (dependecy.startsWith("file:")) {
dependecy = dependecy.substring("file:".length());
}
Path resolvedPackage = path.getParent().resolve(dependecy + "/package.json");
if (!Files.exists(resolvedPackage)) {
propsToRemove.add(packageName);
}
}
}
}
for (String prop : propsToRemove) {
dependencyObj.remove(prop);
}
}
// Override "main" to point at the source folder instead of the output directory.
Path entryPoint = guessPackageMainFile(path, packageJson);
if (entryPoint != null) {
System.out.println("Main file for " + path + " set to " + entryPoint);
packageJson.addProperty("main", entryPoint.toString());
packageJson.remove("typings");
filesToChange.add(path);
} else {
System.out.println("No main file found for " + path);
// For named packages, find the main file.
String name = getChildAsString(packageJson, "name");
if (name != null) {
Path entryPoint = guessPackageMainFile(path, packageJson);
if (entryPoint != null) {
System.out.println(relativePath + ": Main file set to " + sourceRoot.relativize(entryPoint));
packageMainFile.put(name, entryPoint);
} else {
System.out.println(relativePath + ": Main file not found");
}
}
});
// Write the new package.json files to disk
Map<Path, Path> originalFiles = new LinkedHashMap<>();
final String backupFilename = "package.json.lgtm.backup";
for (Path file : filesToChange) {
Path backup = file.resolveSibling(backupFilename);
for (Path file : packageJsonFiles.keySet()) {
Path relativePath = sourceRoot.relativize(file);
Path virtualFile = virtualSourceRoot.resolve(relativePath);
try {
originalFiles.put(file, backup);
Files.move(file, backup, StandardCopyOption.REPLACE_EXISTING);
Files.createDirectories(virtualFile.getParent());
try (Writer writer = Files.newBufferedWriter(virtualFile)) {
new Gson().toJson(packageJsonFiles.get(file), writer);
}
} catch (IOException e) {
throw new ResourceError("Could not backup package.json file: " + file, e);
}
try (Writer writer = Files.newBufferedWriter(file)) {
new Gson().toJson(packageJsonFiles.get(file), writer);
} catch (IOException e) {
throw new ResourceError("Could not rewrite package.json file: " + file, e);
throw new ResourceError("Could not rewrite package.json file: " + virtualFile, e);
}
}
// Install dependencies
for (Path file : packageJsonFiles.keySet()) {
System.out.println("Installing dependencies from " + file);
Path virtualFile = virtualSourceRoot.resolve(sourceRoot.relativize(file));
System.out.println("Installing dependencies from " + virtualFile);
ProcessBuilder pb =
new ProcessBuilder(
Arrays.asList(
@@ -750,18 +742,17 @@ public class AutoBuild {
"--no-default-rc",
"--no-bin-links",
"--pure-lockfile"));
pb.directory(file.getParent().toFile());
pb.directory(virtualFile.getParent().toFile());
pb.redirectOutput(Redirect.INHERIT);
pb.redirectError(Redirect.INHERIT);
try {
pb.start().waitFor(this.installDependenciesTimeout, TimeUnit.MILLISECONDS);
} catch (IOException | InterruptedException ex) {
restoreOriginalFiles(originalFiles); // Try to clean up before giving up.
throw new ResourceError("Could not install dependencies from " + file, ex);
}
}
return new DependencyInstallationResult(originalFiles);
return new DependencyInstallationResult(packageMainFile);
}
/**
@@ -838,7 +829,10 @@ public class AutoBuild {
}
private Set<Path> extractTypeScript(
FileExtractor extractor, Set<Path> files, List<Path> tsconfig) {
FileExtractor extractor,
Set<Path> files,
List<Path> tsconfig,
DependencyInstallationResult deps) {
Set<Path> extractedFiles = new LinkedHashSet<>();
if (hasTypeScriptFiles(files) || !tsconfig.isEmpty()) {
@@ -850,7 +844,7 @@ public class AutoBuild {
for (Path projectPath : tsconfig) {
File projectFile = projectPath.toFile();
long start = logBeginProcess("Opening project " + projectFile);
ParsedProject project = tsParser.openProject(projectFile);
ParsedProject project = tsParser.openProject(projectFile, deps);
logEndProcess(start, "Done opening project " + projectFile);
// Extract all files belonging to this project which are also matched
// by our include/exclude filters.

23
javascript/extractor/src/com/semmle/js/extractor/DependencyInstallationResult.java
View File

@@ -4,23 +4,22 @@ import java.nio.file.Path;
import java.util.Collections;
import java.util.Map;
/**
* Contains the results of installing dependencies.
*/
/** Contains the results of installing dependencies. */
public class DependencyInstallationResult {
private Map<Path, Path> originalFiles;
public static final DependencyInstallationResult empty = new DependencyInstallationResult(Collections.emptyMap());
private Map<String, Path> packageLocations;
public DependencyInstallationResult(Map<Path, Path> originalFiles) {
this.originalFiles = originalFiles;
public static final DependencyInstallationResult empty =
new DependencyInstallationResult(Collections.emptyMap());
public DependencyInstallationResult(Map<String, Path> localPackages) {
this.packageLocations = localPackages;
}
/**
* Returns the mapping from files left behind by dependency installation to
* the backups of those files, to be restored after extraction.
* Returns the mapping from package names to the TypeScript file that should
* act as its main entry point.
*/
public Map<Path, Path> getOriginalFiles() {
return originalFiles;
public Map<String, Path> getPackageLocations() {
return packageLocations;
}
}

11
javascript/extractor/src/com/semmle/js/extractor/EnvironmentVariables.java
View File

@@ -7,6 +7,9 @@ import com.semmle.util.process.Env.Var;
public class EnvironmentVariables {
public static final String CODEQL_EXTRACTOR_JAVASCRIPT_ROOT_ENV_VAR =
"CODEQL_EXTRACTOR_JAVASCRIPT_ROOT";
public static final String CODEQL_EXTRACTOR_JAVASCRIPT_SCRATCH_DIR_ENV_VAR =
"CODEQL_EXTRACTOR_JAVASCRIPT_SCRATCH_DIR";
/**
* Gets the extractor root based on the <code>CODEQL_EXTRACTOR_JAVASCRIPT_ROOT</code> or <code>
@@ -31,4 +34,12 @@ public class EnvironmentVariables {
}
return env;
}
public static String getScratchDir() {
String env = Env.systemEnv().get(CODEQL_EXTRACTOR_JAVASCRIPT_SCRATCH_DIR_ENV_VAR);
if (env == null) {
throw new UserError(CODEQL_EXTRACTOR_JAVASCRIPT_SCRATCH_DIR_ENV_VAR + " must be set");
}
return env;
}
}

2
javascript/extractor/src/com/semmle/js/extractor/Main.java
View File

@@ -140,7 +140,7 @@ public class Main {
for (File projectFile : projectFiles) {
long start = verboseLogStartTimer(ap, "Opening project " + projectFile);
ParsedProject project = tsParser.openProject(projectFile);
ParsedProject project = tsParser.openProject(projectFile, DependencyInstallationResult.empty);
verboseLogEndTimer(ap, start);
// Extract all files belonging to this project which are also matched
// by our include/exclude filters.

5
javascript/extractor/src/com/semmle/js/extractor/test/AutoBuildTests.java
View File

@@ -131,11 +131,6 @@ public class AutoBuildTests {
}
}
@Override
protected void restoreOriginalFiles(java.util.Map<Path, Path> originalFiles) {
// Do nothing
}
@Override
protected DependencyInstallationResult installDependencies(Set<Path> filesToExtract) {
// never install dependencies during testing

47
javascript/extractor/src/com/semmle/js/parser/TypeScriptParser.java
View File

@@ -1,12 +1,28 @@
package com.semmle.js.parser;
import ch.qos.logback.classic.Level;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.lang.ProcessBuilder.Redirect;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParseException;
import com.google.gson.JsonParser;
import com.google.gson.JsonPrimitive;
import com.semmle.js.extractor.DependencyInstallationResult;
import com.semmle.js.extractor.EnvironmentVariables;
import com.semmle.js.extractor.ExtractionMetrics;
import com.semmle.js.parser.JSParser.Result;
@@ -23,21 +39,8 @@ import com.semmle.util.logging.LogbackUtils;
import com.semmle.util.process.AbstractProcessBuilder;
import com.semmle.util.process.Builder;
import com.semmle.util.process.Env;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.lang.ProcessBuilder.Redirect;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import ch.qos.logback.classic.Level;
/**
* The Java half of our wrapper for invoking the TypeScript parser.
@@ -409,10 +412,20 @@ public class TypeScriptParser {
*
* <p>Only one project should be opened at once.
*/
public ParsedProject openProject(File tsConfigFile) {
public ParsedProject openProject(File tsConfigFile, DependencyInstallationResult deps) {
JsonObject request = new JsonObject();
request.add("command", new JsonPrimitive("open-project"));
request.add("tsConfig", new JsonPrimitive(tsConfigFile.getPath()));
JsonArray packageLocations = new JsonArray();
deps.getPackageLocations()
.forEach(
(packageName, packageDir) -> {
JsonArray entry = new JsonArray();
entry.add(packageName);
entry.add(packageDir.toString());
packageLocations.add(entry);
});
request.add("packageLocations", packageLocations);
JsonObject response = talkToParserWrapper(request);
try {
checkResponseType(response, "project-opened");