mirror of
https://github.com/github/codeql.git
synced 2026-04-29 02:35:15 +02:00
Merge pull request #3731 from asger-semmle/js/monorepo-bugfixes
Approved by erik-krogh
This commit is contained in:
@@ -41,17 +41,26 @@ import { Project } from "./common";
|
||||
import { TypeTable } from "./type_table";
|
||||
import { VirtualSourceRoot } from "./virtual_source_root";
|
||||
|
||||
// Remove limit on stack trace depth.
|
||||
Error.stackTraceLimit = Infinity;
|
||||
|
||||
interface ParseCommand {
|
||||
command: "parse";
|
||||
filename: string;
|
||||
}
|
||||
interface OpenProjectCommand {
|
||||
command: "open-project";
|
||||
interface LoadCommand {
|
||||
tsConfig: string;
|
||||
sourceRoot: string | null;
|
||||
virtualSourceRoot: string | null;
|
||||
packageEntryPoints: [string, string][];
|
||||
packageJsonFiles: [string, string][];
|
||||
}
|
||||
interface OpenProjectCommand extends LoadCommand {
|
||||
command: "open-project";
|
||||
}
|
||||
interface GetOwnFilesCommand extends LoadCommand {
|
||||
command: "get-own-files";
|
||||
}
|
||||
interface CloseProjectCommand {
|
||||
command: "close-project";
|
||||
tsConfig: string;
|
||||
@@ -72,7 +81,7 @@ interface PrepareFilesCommand {
|
||||
interface GetMetadataCommand {
|
||||
command: "get-metadata";
|
||||
}
|
||||
type Command = ParseCommand | OpenProjectCommand | CloseProjectCommand
|
||||
type Command = ParseCommand | OpenProjectCommand | GetOwnFilesCommand | CloseProjectCommand
|
||||
| GetTypeTableCommand | ResetCommand | QuitCommand | PrepareFilesCommand | GetMetadataCommand;
|
||||
|
||||
/** The state to be shared between commands. */
|
||||
@@ -362,15 +371,22 @@ function parseSingleFile(filename: string): {ast: ts.SourceFile, code: string} {
|
||||
*/
|
||||
const nodeModulesRex = /[/\\]node_modules[/\\]((?:@[\w.-]+[/\\])?\w[\w.-]*)[/\\](.*)/;
|
||||
|
||||
function handleOpenProjectCommand(command: OpenProjectCommand) {
|
||||
Error.stackTraceLimit = Infinity;
|
||||
let tsConfigFilename = String(command.tsConfig);
|
||||
let tsConfig = ts.readConfigFile(tsConfigFilename, ts.sys.readFile);
|
||||
let basePath = pathlib.dirname(tsConfigFilename);
|
||||
interface LoadedConfig {
|
||||
config: ts.ParsedCommandLine;
|
||||
basePath: string;
|
||||
packageEntryPoints: Map<string, string>;
|
||||
packageJsonFiles: Map<string, string>;
|
||||
virtualSourceRoot: VirtualSourceRoot;
|
||||
ownFiles: string[];
|
||||
}
|
||||
|
||||
function loadTsConfig(command: LoadCommand): LoadedConfig {
|
||||
let tsConfig = ts.readConfigFile(command.tsConfig, ts.sys.readFile);
|
||||
let basePath = pathlib.dirname(command.tsConfig);
|
||||
|
||||
let packageEntryPoints = new Map(command.packageEntryPoints);
|
||||
let packageJsonFiles = new Map(command.packageJsonFiles);
|
||||
let virtualSourceRoot = new VirtualSourceRoot(process.cwd(), command.virtualSourceRoot);
|
||||
let virtualSourceRoot = new VirtualSourceRoot(command.sourceRoot, command.virtualSourceRoot);
|
||||
|
||||
/**
|
||||
* Rewrites path segments of form `node_modules/PACK/suffix` to be relative to
|
||||
@@ -415,7 +431,29 @@ function handleOpenProjectCommand(command: OpenProjectCommand) {
|
||||
}
|
||||
};
|
||||
let config = ts.parseJsonConfigFileContent(tsConfig.config, parseConfigHost, basePath);
|
||||
let project = new Project(tsConfigFilename, config, state.typeTable, packageEntryPoints, virtualSourceRoot);
|
||||
|
||||
let ownFiles = config.fileNames.map(file => pathlib.resolve(file));
|
||||
|
||||
return { config, basePath, packageJsonFiles, packageEntryPoints, virtualSourceRoot, ownFiles };
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the list of files included in the given tsconfig.json file's include pattern,
|
||||
* (not including those only references through imports).
|
||||
*/
|
||||
function handleGetFileListCommand(command: GetOwnFilesCommand) {
|
||||
let { config, ownFiles } = loadTsConfig(command);
|
||||
|
||||
console.log(JSON.stringify({
|
||||
type: "file-list",
|
||||
ownFiles,
|
||||
}));
|
||||
}
|
||||
|
||||
function handleOpenProjectCommand(command: OpenProjectCommand) {
|
||||
let { config, packageEntryPoints, virtualSourceRoot, basePath, ownFiles } = loadTsConfig(command);
|
||||
|
||||
let project = new Project(command.tsConfig, config, state.typeTable, packageEntryPoints, virtualSourceRoot);
|
||||
project.load();
|
||||
|
||||
state.project = project;
|
||||
@@ -587,9 +625,14 @@ function handleOpenProjectCommand(command: OpenProjectCommand) {
|
||||
return symbol;
|
||||
}
|
||||
|
||||
// Unlike in the get-own-files command, this command gets all files we can possibly
|
||||
// extract type information for, including files referenced outside the tsconfig's inclusion pattern.
|
||||
let allFiles = program.getSourceFiles().map(sf => pathlib.resolve(sf.fileName));
|
||||
|
||||
console.log(JSON.stringify({
|
||||
type: "project-opened",
|
||||
files: program.getSourceFiles().map(sf => pathlib.resolve(sf.fileName)),
|
||||
ownFiles,
|
||||
allFiles,
|
||||
}));
|
||||
}
|
||||
|
||||
@@ -685,6 +728,9 @@ function runReadLineInterface() {
|
||||
case "open-project":
|
||||
handleOpenProjectCommand(req);
|
||||
break;
|
||||
case "get-own-files":
|
||||
handleGetFileListCommand(req);
|
||||
break;
|
||||
case "close-project":
|
||||
handleCloseProjectCommand(req);
|
||||
break;
|
||||
@@ -720,6 +766,7 @@ if (process.argv.length > 2) {
|
||||
tsConfig: argument,
|
||||
packageEntryPoints: [],
|
||||
packageJsonFiles: [],
|
||||
sourceRoot: null,
|
||||
virtualSourceRoot: null,
|
||||
});
|
||||
for (let sf of state.project.program.getSourceFiles()) {
|
||||
|
||||
@@ -7,20 +7,20 @@ import * as ts from "./typescript";
|
||||
*/
|
||||
export class VirtualSourceRoot {
|
||||
constructor(
|
||||
private sourceRoot: string,
|
||||
private sourceRoot: string | null,
|
||||
|
||||
/**
|
||||
* Directory whose folder structure mirrors the real source root, but with `node_modules` installed,
|
||||
* or undefined if no virtual source root exists.
|
||||
*/
|
||||
private virtualSourceRoot: string,
|
||||
private virtualSourceRoot: string | null,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Maps a path under the real source root to the corresponding path in the virtual source root.
|
||||
*/
|
||||
public toVirtualPath(path: string) {
|
||||
if (!this.virtualSourceRoot) return null;
|
||||
if (!this.virtualSourceRoot || !this.sourceRoot) return null;
|
||||
let relative = pathlib.relative(this.sourceRoot, path);
|
||||
if (relative.startsWith('..') || pathlib.isAbsolute(relative)) return null;
|
||||
return pathlib.join(this.virtualSourceRoot, relative);
|
||||
|
||||
@@ -20,6 +20,7 @@ import java.nio.file.SimpleFileVisitor;
|
||||
import java.nio.file.attribute.BasicFileAttributes;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Comparator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
@@ -28,6 +29,7 @@ import java.util.Set;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import com.google.gson.Gson;
|
||||
@@ -536,6 +538,36 @@ public class AutoBuild {
|
||||
Files.walkFileTree(externs, visitor);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compares files in the order they should be extracted.
|
||||
* <p>
|
||||
* The ordering of tsconfig.json files can affect extraction results. Since we
|
||||
* extract any given source file at most once, and a source file can be included from
|
||||
* multiple tsconfig.json files, we sometimes have to choose arbitrarily which tsconfig.json
|
||||
* to use for a given file (which is based on this ordering).
|
||||
* <p>
|
||||
* We sort them to help ensure reproducible extraction. Additionally, deeply nested files are
|
||||
* preferred over shallow ones to help ensure files are extracted with the most specific
|
||||
* tsconfig.json file.
|
||||
*/
|
||||
public static final Comparator<Path> PATH_ORDERING = new Comparator<Path>() {
|
||||
public int compare(Path f1, Path f2) {
|
||||
if (f1.getNameCount() != f2.getNameCount()) {
|
||||
return f2.getNameCount() - f1.getNameCount();
|
||||
}
|
||||
return f1.compareTo(f2);
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Like {@link #PATH_ORDERING} but for {@link File} objects.
|
||||
*/
|
||||
public static final Comparator<File> FILE_ORDERING = new Comparator<File>() {
|
||||
public int compare(File f1, File f2) {
|
||||
return PATH_ORDERING.compare(f1.toPath(), f2.toPath());
|
||||
}
|
||||
};
|
||||
|
||||
/** Extract all supported candidate files that pass the filters. */
|
||||
private void extractSource() throws IOException {
|
||||
// default extractor
|
||||
@@ -555,9 +587,17 @@ public class AutoBuild {
|
||||
List<Path> tsconfigFiles = new ArrayList<>();
|
||||
findFilesToExtract(defaultExtractor, filesToExtract, tsconfigFiles);
|
||||
|
||||
tsconfigFiles = tsconfigFiles.stream()
|
||||
.sorted(PATH_ORDERING)
|
||||
.collect(Collectors.toList());
|
||||
|
||||
filesToExtract = filesToExtract.stream()
|
||||
.sorted(PATH_ORDERING)
|
||||
.collect(Collectors.toCollection(() -> new LinkedHashSet<>()));
|
||||
|
||||
DependencyInstallationResult dependencyInstallationResult = DependencyInstallationResult.empty;
|
||||
if (!tsconfigFiles.isEmpty() && this.installDependencies) {
|
||||
dependencyInstallationResult = this.installDependencies(filesToExtract);
|
||||
if (!tsconfigFiles.isEmpty()) {
|
||||
dependencyInstallationResult = this.preparePackagesAndDependencies(filesToExtract);
|
||||
}
|
||||
|
||||
// extract TypeScript projects and files
|
||||
@@ -659,7 +699,21 @@ public class AutoBuild {
|
||||
}
|
||||
|
||||
/**
|
||||
* Installs dependencies for use by the TypeScript type checker.
|
||||
* Gets a relative path from <code>from</code> to <code>to</code> provided
|
||||
* the latter is contained in the former. Otherwise returns <code>null</code>.
|
||||
* @return a path or null
|
||||
*/
|
||||
public static Path tryRelativize(Path from, Path to) {
|
||||
Path relative = from.relativize(to);
|
||||
if (relative.startsWith("..") || relative.isAbsolute()) {
|
||||
return null;
|
||||
}
|
||||
return relative;
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepares <tt>package.json</tt> files in a virtual source root, and, if enabled,
|
||||
* installs dependencies for use by the TypeScript type checker.
|
||||
* <p>
|
||||
* Some packages must be downloaded while others exist within the same repo ("monorepos")
|
||||
* but are not in a location where TypeScript would look for it.
|
||||
@@ -677,13 +731,9 @@ public class AutoBuild {
|
||||
* The TypeScript parser wrapper then overrides module resolution so packages can be found
|
||||
* under the virtual source root and via that package location mapping.
|
||||
*/
|
||||
protected DependencyInstallationResult installDependencies(Set<Path> filesToExtract) {
|
||||
if (!verifyYarnInstallation()) {
|
||||
return DependencyInstallationResult.empty;
|
||||
}
|
||||
|
||||
final Path sourceRoot = Paths.get(".").toAbsolutePath();
|
||||
final Path virtualSourceRoot = Paths.get(EnvironmentVariables.getScratchDir()).toAbsolutePath();
|
||||
protected DependencyInstallationResult preparePackagesAndDependencies(Set<Path> filesToExtract) {
|
||||
final Path sourceRoot = LGTM_SRC;
|
||||
final Path virtualSourceRoot = toRealPath(Paths.get(EnvironmentVariables.getScratchDir()));
|
||||
|
||||
// Read all package.json files and index them by name.
|
||||
Map<Path, JsonObject> packageJsonFiles = new LinkedHashMap<>();
|
||||
@@ -697,6 +747,9 @@ public class AutoBuild {
|
||||
if (!(json instanceof JsonObject)) continue;
|
||||
JsonObject jsonObject = (JsonObject) json;
|
||||
file = file.toAbsolutePath();
|
||||
if (tryRelativize(sourceRoot, file) == null) {
|
||||
continue; // Ignore package.json files outside the source root.
|
||||
}
|
||||
packageJsonFiles.put(file, jsonObject);
|
||||
|
||||
String name = getChildAsString(jsonObject, "name");
|
||||
@@ -781,33 +834,35 @@ public class AutoBuild {
|
||||
}
|
||||
|
||||
// Install dependencies
|
||||
for (Path file : packageJsonFiles.keySet()) {
|
||||
Path virtualFile = virtualSourceRoot.resolve(sourceRoot.relativize(file));
|
||||
System.out.println("Installing dependencies from " + virtualFile);
|
||||
ProcessBuilder pb =
|
||||
new ProcessBuilder(
|
||||
Arrays.asList(
|
||||
"yarn",
|
||||
"install",
|
||||
"--non-interactive",
|
||||
"--ignore-scripts",
|
||||
"--ignore-platform",
|
||||
"--ignore-engines",
|
||||
"--ignore-optional",
|
||||
"--no-default-rc",
|
||||
"--no-bin-links",
|
||||
"--pure-lockfile"));
|
||||
pb.directory(virtualFile.getParent().toFile());
|
||||
pb.redirectOutput(Redirect.INHERIT);
|
||||
pb.redirectError(Redirect.INHERIT);
|
||||
try {
|
||||
pb.start().waitFor(this.installDependenciesTimeout, TimeUnit.MILLISECONDS);
|
||||
} catch (IOException | InterruptedException ex) {
|
||||
throw new ResourceError("Could not install dependencies from " + file, ex);
|
||||
if (this.installDependencies && verifyYarnInstallation()) {
|
||||
for (Path file : packageJsonFiles.keySet()) {
|
||||
Path virtualFile = virtualSourceRoot.resolve(sourceRoot.relativize(file));
|
||||
System.out.println("Installing dependencies from " + virtualFile);
|
||||
ProcessBuilder pb =
|
||||
new ProcessBuilder(
|
||||
Arrays.asList(
|
||||
"yarn",
|
||||
"install",
|
||||
"--non-interactive",
|
||||
"--ignore-scripts",
|
||||
"--ignore-platform",
|
||||
"--ignore-engines",
|
||||
"--ignore-optional",
|
||||
"--no-default-rc",
|
||||
"--no-bin-links",
|
||||
"--pure-lockfile"));
|
||||
pb.directory(virtualFile.getParent().toFile());
|
||||
pb.redirectOutput(Redirect.INHERIT);
|
||||
pb.redirectError(Redirect.INHERIT);
|
||||
try {
|
||||
pb.start().waitFor(this.installDependenciesTimeout, TimeUnit.MILLISECONDS);
|
||||
} catch (IOException | InterruptedException ex) {
|
||||
throw new ResourceError("Could not install dependencies from " + file, ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return new DependencyInstallationResult(virtualSourceRoot, packageMainFile, packagesInRepo);
|
||||
return new DependencyInstallationResult(sourceRoot, virtualSourceRoot, packageMainFile, packagesInRepo);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -894,6 +949,16 @@ public class AutoBuild {
|
||||
TypeScriptParser tsParser = extractorState.getTypeScriptParser();
|
||||
verifyTypeScriptInstallation(extractorState);
|
||||
|
||||
// Collect all files included in a tsconfig.json inclusion pattern.
|
||||
// If a given file is referenced by multiple tsconfig files, we prefer to extract it using
|
||||
// one that includes it rather than just references it.
|
||||
Set<File> explicitlyIncludedFiles = new LinkedHashSet<>();
|
||||
if (tsconfig.size() > 1) { // No prioritization needed if there's only one tsconfig.
|
||||
for (Path projectPath : tsconfig) {
|
||||
explicitlyIncludedFiles.addAll(tsParser.getOwnFiles(projectPath.toFile(), deps));
|
||||
}
|
||||
}
|
||||
|
||||
// Extract TypeScript projects
|
||||
for (Path projectPath : tsconfig) {
|
||||
File projectFile = projectPath.toFile();
|
||||
@@ -902,19 +967,21 @@ public class AutoBuild {
|
||||
logEndProcess(start, "Done opening project " + projectFile);
|
||||
// Extract all files belonging to this project which are also matched
|
||||
// by our include/exclude filters.
|
||||
List<File> typeScriptFiles = new ArrayList<File>();
|
||||
for (File sourceFile : project.getSourceFiles()) {
|
||||
List<Path> typeScriptFiles = new ArrayList<Path>();
|
||||
for (File sourceFile : project.getAllFiles()) {
|
||||
Path sourcePath = sourceFile.toPath();
|
||||
if (!files.contains(normalizePath(sourcePath))) continue;
|
||||
if (!project.getOwnFiles().contains(sourceFile) && explicitlyIncludedFiles.contains(sourceFile)) continue;
|
||||
if (!FileType.TYPESCRIPT.getExtensions().contains(FileUtil.extension(sourcePath))) {
|
||||
// For the time being, skip non-TypeScript files, even if the TypeScript
|
||||
// compiler can parse them for us.
|
||||
continue;
|
||||
}
|
||||
if (!extractedFiles.contains(sourcePath)) {
|
||||
typeScriptFiles.add(sourcePath.toFile());
|
||||
typeScriptFiles.add(sourcePath);
|
||||
}
|
||||
}
|
||||
typeScriptFiles.sort(PATH_ORDERING);
|
||||
extractTypeScriptFiles(typeScriptFiles, extractedFiles, extractor, extractorState);
|
||||
tsParser.closeProject(projectFile);
|
||||
}
|
||||
@@ -926,11 +993,11 @@ public class AutoBuild {
|
||||
}
|
||||
|
||||
// Extract remaining TypeScript files.
|
||||
List<File> remainingTypeScriptFiles = new ArrayList<File>();
|
||||
List<Path> remainingTypeScriptFiles = new ArrayList<>();
|
||||
for (Path f : files) {
|
||||
if (!extractedFiles.contains(f)
|
||||
&& FileType.forFileExtension(f.toFile()) == FileType.TYPESCRIPT) {
|
||||
remainingTypeScriptFiles.add(f.toFile());
|
||||
remainingTypeScriptFiles.add(f);
|
||||
}
|
||||
}
|
||||
if (!remainingTypeScriptFiles.isEmpty()) {
|
||||
@@ -1018,15 +1085,18 @@ public class AutoBuild {
|
||||
}
|
||||
|
||||
public void extractTypeScriptFiles(
|
||||
List<File> files,
|
||||
List<Path> files,
|
||||
Set<Path> extractedFiles,
|
||||
FileExtractor extractor,
|
||||
ExtractorState extractorState) {
|
||||
extractorState.getTypeScriptParser().prepareFiles(files);
|
||||
for (File f : files) {
|
||||
Path path = f.toPath();
|
||||
List<File> list = files
|
||||
.stream()
|
||||
.sorted(PATH_ORDERING)
|
||||
.map(p -> p.toFile()).collect(Collectors.toList());
|
||||
extractorState.getTypeScriptParser().prepareFiles(list);
|
||||
for (Path path : files) {
|
||||
extractedFiles.add(path);
|
||||
extract(extractor, f.toPath(), extractorState);
|
||||
extract(extractor, path, extractorState);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -6,24 +6,39 @@ import java.util.Map;
|
||||
|
||||
/** Contains the results of installing dependencies. */
|
||||
public class DependencyInstallationResult {
|
||||
private Path sourceRoot;
|
||||
private Path virtualSourceRoot;
|
||||
private Map<String, Path> packageEntryPoints;
|
||||
private Map<String, Path> packageJsonFiles;
|
||||
|
||||
public static final DependencyInstallationResult empty =
|
||||
new DependencyInstallationResult(null, Collections.emptyMap(), Collections.emptyMap());
|
||||
new DependencyInstallationResult(null, null, Collections.emptyMap(), Collections.emptyMap());
|
||||
|
||||
public DependencyInstallationResult(
|
||||
Path sourceRoot,
|
||||
Path virtualSourceRoot,
|
||||
Map<String, Path> packageEntryPoints,
|
||||
Map<String, Path> packageJsonFiles) {
|
||||
this.sourceRoot = sourceRoot;
|
||||
this.virtualSourceRoot = virtualSourceRoot;
|
||||
this.packageEntryPoints = packageEntryPoints;
|
||||
this.packageJsonFiles = packageJsonFiles;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the source root mirrored by {@link #getVirtualSourceRoot()} or <code>null</code>
|
||||
* if no virtual source root exists.
|
||||
* <p>
|
||||
* When invoked from the AutoBuilder, this corresponds to the source root. When invoked
|
||||
* from ODASA, there is no notion of source root, so this is always <code>null</code> in that context.
|
||||
*/
|
||||
public Path getSourceRoot() {
|
||||
return sourceRoot;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the virtual source root or <code>null</code> if no virtual source root exists.
|
||||
*
|
||||
* <p>
|
||||
* The virtual source root is a directory hierarchy that mirrors the real source
|
||||
* root, where dependencies are installed.
|
||||
*/
|
||||
|
||||
@@ -7,6 +7,7 @@ import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.semmle.js.extractor.ExtractorConfig.HTMLHandling;
|
||||
import com.semmle.js.extractor.ExtractorConfig.Platform;
|
||||
@@ -77,8 +78,8 @@ public class Main {
|
||||
private PathMatcher includeMatcher, excludeMatcher;
|
||||
private FileExtractor fileExtractor;
|
||||
private ExtractorState extractorState;
|
||||
private final Set<File> projectFiles = new LinkedHashSet<>();
|
||||
private final Set<File> files = new LinkedHashSet<>();
|
||||
private Set<File> projectFiles = new LinkedHashSet<>();
|
||||
private Set<File> files = new LinkedHashSet<>();
|
||||
private final Set<File> extractedFiles = new LinkedHashSet<>();
|
||||
|
||||
/* used to detect cyclic directory hierarchies */
|
||||
@@ -138,6 +139,16 @@ public class Main {
|
||||
if (containsTypeScriptFiles()) {
|
||||
tsParser.verifyInstallation(!ap.has(P_QUIET));
|
||||
}
|
||||
|
||||
// Sort files for determinism
|
||||
projectFiles = projectFiles.stream()
|
||||
.sorted(AutoBuild.FILE_ORDERING)
|
||||
.collect(Collectors.toCollection(() -> new LinkedHashSet<>()));
|
||||
|
||||
files = files.stream()
|
||||
.sorted(AutoBuild.FILE_ORDERING)
|
||||
.collect(Collectors.toCollection(() -> new LinkedHashSet<>()));
|
||||
|
||||
for (File projectFile : projectFiles) {
|
||||
|
||||
long start = verboseLogStartTimer(ap, "Opening project " + projectFile);
|
||||
@@ -146,7 +157,7 @@ public class Main {
|
||||
// Extract all files belonging to this project which are also matched
|
||||
// by our include/exclude filters.
|
||||
List<File> filesToExtract = new ArrayList<>();
|
||||
for (File sourceFile : project.getSourceFiles()) {
|
||||
for (File sourceFile : project.getOwnFiles()) {
|
||||
if (files.contains(normalizeFile(sourceFile))
|
||||
&& !extractedFiles.contains(sourceFile.getAbsoluteFile())
|
||||
&& FileType.TYPESCRIPT.getExtensions().contains(FileUtil.extension(sourceFile))) {
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
package com.semmle.js.extractor.test;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.FileVisitResult;
|
||||
@@ -122,18 +121,18 @@ public class AutoBuildTests {
|
||||
|
||||
@Override
|
||||
public void extractTypeScriptFiles(
|
||||
java.util.List<File> files,
|
||||
java.util.List<Path> files,
|
||||
java.util.Set<Path> extractedFiles,
|
||||
FileExtractor extractor,
|
||||
ExtractorState extractorState) {
|
||||
for (File f : files) {
|
||||
for (Path f : files) {
|
||||
actual.add(f.toString());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected DependencyInstallationResult installDependencies(Set<Path> filesToExtract) {
|
||||
// never install dependencies during testing
|
||||
protected DependencyInstallationResult preparePackagesAndDependencies(Set<Path> filesToExtract) {
|
||||
// currently disabled in tests
|
||||
return DependencyInstallationResult.empty;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,15 +1,17 @@
|
||||
package com.semmle.js.parser;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.Set;
|
||||
|
||||
public class ParsedProject {
|
||||
private final File tsConfigFile;
|
||||
private final Set<File> sourceFiles = new LinkedHashSet<>();
|
||||
private final Set<File> ownFiles;
|
||||
private final Set<File> allFiles;
|
||||
|
||||
public ParsedProject(File tsConfigFile) {
|
||||
public ParsedProject(File tsConfigFile, Set<File> ownFiles, Set<File> allFiles) {
|
||||
this.tsConfigFile = tsConfigFile;
|
||||
this.ownFiles = ownFiles;
|
||||
this.allFiles = allFiles;
|
||||
}
|
||||
|
||||
/** Returns the <tt>tsconfig.json</tt> file that defines this project. */
|
||||
@@ -18,11 +20,12 @@ public class ParsedProject {
|
||||
}
|
||||
|
||||
/** Absolute paths to the files included in this project. */
|
||||
public Set<File> getSourceFiles() {
|
||||
return sourceFiles;
|
||||
public Set<File> getOwnFiles() {
|
||||
return allFiles;
|
||||
}
|
||||
|
||||
public void addSourceFile(File file) {
|
||||
sourceFiles.add(file);
|
||||
/** Absolute paths to the files included in or referenced by this project. */
|
||||
public Set<File> getAllFiles() {
|
||||
return allFiles;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,8 +15,10 @@ import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import com.google.gson.JsonArray;
|
||||
@@ -488,6 +490,29 @@ public class TypeScriptParser {
|
||||
return result;
|
||||
}
|
||||
|
||||
private static Set<File> getFilesFromJsonArray(JsonArray array) {
|
||||
Set<File> files = new LinkedHashSet<>();
|
||||
for (JsonElement elm : array) {
|
||||
files.add(new File(elm.getAsString()));
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the set of files included by the inclusion pattern in the given tsconfig.json file.
|
||||
*/
|
||||
public Set<File> getOwnFiles(File tsConfigFile, DependencyInstallationResult deps) {
|
||||
JsonObject request = makeLoadCommand("get-own-files", tsConfigFile, deps);
|
||||
JsonObject response = talkToParserWrapper(request);
|
||||
try {
|
||||
checkResponseType(response, "file-list");
|
||||
return getFilesFromJsonArray(response.get("ownFiles").getAsJsonArray());
|
||||
} catch (IllegalStateException e) {
|
||||
throw new CatastrophicError(
|
||||
"TypeScript parser wrapper sent unexpected response: " + response, e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Opens a new project based on a tsconfig.json file. The compiler will analyze all files in the
|
||||
* project.
|
||||
@@ -497,22 +522,13 @@ public class TypeScriptParser {
|
||||
* <p>Only one project should be opened at once.
|
||||
*/
|
||||
public ParsedProject openProject(File tsConfigFile, DependencyInstallationResult deps) {
|
||||
JsonObject request = new JsonObject();
|
||||
request.add("command", new JsonPrimitive("open-project"));
|
||||
request.add("tsConfig", new JsonPrimitive(tsConfigFile.getPath()));
|
||||
request.add("packageEntryPoints", mapToArray(deps.getPackageEntryPoints()));
|
||||
request.add("packageJsonFiles", mapToArray(deps.getPackageJsonFiles()));
|
||||
request.add("virtualSourceRoot", deps.getVirtualSourceRoot() == null
|
||||
? JsonNull.INSTANCE
|
||||
: new JsonPrimitive(deps.getVirtualSourceRoot().toString()));
|
||||
JsonObject request = makeLoadCommand("open-project", tsConfigFile, deps);
|
||||
JsonObject response = talkToParserWrapper(request);
|
||||
try {
|
||||
checkResponseType(response, "project-opened");
|
||||
ParsedProject project = new ParsedProject(tsConfigFile);
|
||||
JsonArray filesJson = response.get("files").getAsJsonArray();
|
||||
for (JsonElement elm : filesJson) {
|
||||
project.addSourceFile(new File(elm.getAsString()));
|
||||
}
|
||||
ParsedProject project = new ParsedProject(tsConfigFile,
|
||||
getFilesFromJsonArray(response.get("ownFiles").getAsJsonArray()),
|
||||
getFilesFromJsonArray(response.get("allFiles").getAsJsonArray()));
|
||||
return project;
|
||||
} catch (IllegalStateException e) {
|
||||
throw new CatastrophicError(
|
||||
@@ -520,6 +536,21 @@ public class TypeScriptParser {
|
||||
}
|
||||
}
|
||||
|
||||
private JsonObject makeLoadCommand(String command, File tsConfigFile, DependencyInstallationResult deps) {
|
||||
JsonObject request = new JsonObject();
|
||||
request.add("command", new JsonPrimitive(command));
|
||||
request.add("tsConfig", new JsonPrimitive(tsConfigFile.getPath()));
|
||||
request.add("packageEntryPoints", mapToArray(deps.getPackageEntryPoints()));
|
||||
request.add("packageJsonFiles", mapToArray(deps.getPackageJsonFiles()));
|
||||
request.add("sourceRoot", deps.getSourceRoot() == null
|
||||
? JsonNull.INSTANCE
|
||||
: new JsonPrimitive(deps.getSourceRoot().toString()));
|
||||
request.add("virtualSourceRoot", deps.getVirtualSourceRoot() == null
|
||||
? JsonNull.INSTANCE
|
||||
: new JsonPrimitive(deps.getVirtualSourceRoot().toString()));
|
||||
return request;
|
||||
}
|
||||
|
||||
/**
|
||||
* Closes a project previously opened.
|
||||
*
|
||||
|
||||
Reference in New Issue
Block a user