TS: do more work in parallel

javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java

@@ -494,14 +494,19 @@ public class AutoBuild {
 			logEndProcess();
 			// Extract all files belonging to this project which are also matched
 			// by our include/exclude filters.
+			List<File> typeScriptFiles = new ArrayList<File>();
 			for (File sourceFile : project.getSourceFiles()) {
 				Path sourcePath = sourceFile.toPath();
 				if (!filesToExtract.contains(normalizePath(sourcePath)))
 					continue;
 				if (extractedFiles.add(sourcePath)) {
-					extract(extractor, sourcePath);
+					typeScriptFiles.add(sourcePath.toFile());
 				}
 			}
+			tsParser.prepareFiles(typeScriptFiles);
+			for (File file : typeScriptFiles) {
+				extract(extractor, file.toPath());
+			}
 			tsParser.closeProject(projectFile);
 		}
 
@@ -509,12 +514,29 @@ public class AutoBuild {
 			// Extract all the types discovered when extracting the ASTs.
 			TypeTable typeTable = tsParser.getTypeTable();
 			extractTypeTable(tsconfigFiles.iterator().next(), typeTable);
-
-			// The TypeScript compiler instance is no longer needed.
-			tsParser.killProcess();
 		}
 
-		// Extract files that were not part of a project.
+		// Extract remaining TypeScript files.
+		List<File> remainingTypeScriptFiles = new ArrayList<File>();
+		for (Path f : filesToExtract) {
+			if (!extractedFiles.contains(f) && FileType.forFileExtension(f.toFile()) == FileType.TYPESCRIPT) {
+				remainingTypeScriptFiles.add(f.toFile());
+			}
+		}
+		if (!remainingTypeScriptFiles.isEmpty()) {
+			tsParser.prepareFiles(remainingTypeScriptFiles);
+			for (File f : remainingTypeScriptFiles) {
+				Path path = f.toPath();
+				if (extractedFiles.add(path)) {
+					extract(extractor, path);
+				}
+			}
+		}
+
+		// The TypeScript compiler instance is no longer needed.
+		tsParser.killProcess();
+
+		// Extract non-TypeScript files
 		for (Path f : filesToExtract) {
 			if (extractedFiles.add(f)) {
 				extract(extractor, f);

javascript/extractor/src/com/semmle/js/extractor/Main.java

@@ -2,7 +2,9 @@ package com.semmle.js.extractor;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.LinkedHashSet;
+import java.util.List;
 import java.util.Set;
 import java.util.regex.Pattern;
 
@@ -140,16 +142,22 @@ public class Main {
 			tsParser.verifyInstallation(!ap.has(P_QUIET));
 		}
 		for (File projectFile : projectFiles) {
+
 			long start = verboseLogStartTimer(ap, "Opening project " + projectFile);
 			ParsedProject project = tsParser.openProject(projectFile);
 			verboseLogEndTimer(ap, start);
 			// Extract all files belonging to this project which are also matched
 			// by our include/exclude filters.
+			List<File> filesToExtract = new ArrayList<>();
 			for (File sourceFile : project.getSourceFiles()) {
-				if (files.contains(normalizeFile(sourceFile))) {
-					ensureFileIsExtracted(sourceFile, ap);
+				if (files.contains(normalizeFile(sourceFile)) && !extractedFiles.contains(sourceFile.getAbsoluteFile())) {
+					filesToExtract.add(sourceFile);
 				}
 			}
+			tsParser.prepareFiles(filesToExtract);
+			for (int i = 0; i < filesToExtract.size(); ++i) {
+				ensureFileIsExtracted(filesToExtract.get(i), ap);
+			}
 			// Close the project to free memory. This does not need to be in a `finally` as
 			// the project is not a system resource.
 			tsParser.closeProject(projectFile);
@@ -159,14 +167,27 @@ public class Main {
 			// Extract all the types discovered when extracting the ASTs.
 			TypeTable typeTable = tsParser.getTypeTable();
 			extractTypeTable(projectFiles.iterator().next(), typeTable);
+		}
 
-			// The TypeScript compiler instance is no longer needed - free up some memory.
-			if (hasSharedExtractorState) {
-				tsParser.reset(); // This is called from a test runner, so keep the process alive.
-			} else {
-				tsParser.killProcess();
+		List<File> remainingTypescriptFiles = new ArrayList<>();
+		for (File f : files) {
+			if (!extractedFiles.contains(f.getAbsoluteFile()) && FileType.forFileExtension(f) == FileType.TYPESCRIPT) {
+				remainingTypescriptFiles.add(f);
 			}
 		}
+		if (!remainingTypescriptFiles.isEmpty()) {
+			tsParser.prepareFiles(remainingTypescriptFiles);
+			for (File f : remainingTypescriptFiles) {
+				ensureFileIsExtracted(f, ap);
+			}
+		}
+
+		// The TypeScript compiler instance is no longer needed - free up some memory.
+		if (hasSharedExtractorState) {
+			tsParser.reset(); // This is called from a test runner, so keep the process alive.
+		} else {
+			tsParser.killProcess();
+		}
 
 		// Extract files that were not part of a project.
 		for (File f : files) {

javascript/extractor/src/com/semmle/js/parser/TypeScriptParser.java

@@ -292,6 +292,25 @@ public class TypeScriptParser {
 		}
 	}
 
+	/**
+	 * Informs the parser process that the following files are going to be
+	 * requested, in that order.
+	 * <p>
+	 * The parser process uses this list to start work on the next file before it is
+	 * requested.
+	 */
+	public void prepareFiles(List<File> files) {
+		JsonObject request = new JsonObject();
+		request.add("command", new JsonPrimitive("prepare-files"));
+		JsonArray filenames = new JsonArray();
+		for (File file : files) {
+			filenames.add(new JsonPrimitive(file.getAbsolutePath()));
+		}
+		request.add("filenames", filenames);
+		JsonObject response = talkToParserWrapper(request);
+		checkResponseType(response, "ok");
+	}
+
 	/**
 	 * Opens a new project based on a tsconfig.json file. The compiler will analyze
 	 * all files in the project.