From dde0f868b3fb2785e7983fa9243abe60e94dc86f Mon Sep 17 00:00:00 2001 From: Asger Feldthaus Date: Tue, 7 Jan 2020 13:15:47 +0000 Subject: [PATCH] TS: Handle monorepos by rewriting package.json --- .../com/semmle/js/extractor/AutoBuild.java | 176 +++++++++++++++--- .../js/extractor/test/AutoBuildTests.java | 9 +- 2 files changed, 159 insertions(+), 26 deletions(-) diff --git a/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java b/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java index 9f52be05cb6..a93e776a008 100644 --- a/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java +++ b/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java @@ -1,5 +1,11 @@ package com.semmle.js.extractor; +import com.google.gson.Gson; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.JsonParser; +import com.google.gson.JsonPrimitive; import com.semmle.js.extractor.ExtractorConfig.SourceType; import com.semmle.js.extractor.FileExtractor.FileType; import com.semmle.js.extractor.trapcache.DefaultTrapCache; @@ -16,6 +22,7 @@ import com.semmle.util.exception.ResourceError; import com.semmle.util.exception.UserError; import com.semmle.util.extraction.ExtractorOutputConfig; import com.semmle.util.files.FileUtil; +import com.semmle.util.io.WholeIO; import com.semmle.util.io.csv.CSVReader; import com.semmle.util.language.LegacyLanguage; import com.semmle.util.process.Env; @@ -26,6 +33,7 @@ import java.io.File; import java.io.IOException; import java.io.InputStreamReader; import java.io.Reader; +import java.io.Writer; import java.lang.ProcessBuilder.Redirect; import java.net.URI; import java.net.URISyntaxException; @@ -37,9 +45,11 @@ import java.nio.file.InvalidPathException; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.SimpleFileVisitor; +import java.nio.file.StandardCopyOption; import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -48,6 +58,7 @@ import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; import java.util.stream.Stream; /** @@ -391,6 +402,7 @@ public class AutoBuild { // include .eslintrc files and package.json files patterns.add("**/.eslintrc*"); patterns.add("**/package.json"); + patterns.add("**/tsconfig.json"); // include any explicitly specified extensions for (String extension : fileTypes.keySet()) patterns.add("**/*" + extension); @@ -545,12 +557,18 @@ public class AutoBuild { List tsconfigFiles = new ArrayList<>(); findFilesToExtract(defaultExtractor, filesToExtract, tsconfigFiles); + Map originalFiles = Collections.emptyMap(); if (!tsconfigFiles.isEmpty() && this.installDependencies) { - this.installDependencies(filesToExtract); + originalFiles = this.installDependencies(filesToExtract); } // extract TypeScript projects and files - Set extractedFiles = extractTypeScript(defaultExtractor, filesToExtract, tsconfigFiles); + Set extractedFiles; + try { + extractedFiles = extractTypeScript(defaultExtractor, filesToExtract, tsconfigFiles); + } finally { + restoreOriginalFiles(originalFiles); + } // extract remaining files for (Path f : filesToExtract) { @@ -587,36 +605,143 @@ public class AutoBuild { } } - protected void installDependencies(Set filesToExtract) { + protected void restoreOriginalFiles(Map originalFiles) { + originalFiles.forEach( + (file, original) -> { + try { + Files.move(original, file, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + throw new ResourceError("Could not restore original file: " + file, e); + } + }); + } + + private static final Pattern validPackageName = Pattern.compile("(@[\\w.-]+/)?\\w[\\w.-]*"); + + protected Map installDependencies(Set filesToExtract) { if (!verifyYarnInstallation()) { - return; + return Collections.emptyMap(); } + + Path rootNodeModules = Paths.get("node_modules"); + + // Read all package.json files, and install symlinks to them. + Map packageJsonFiles = new LinkedHashMap<>(); + Set packagesInRepo = new LinkedHashSet<>(); for (Path file : filesToExtract) { if (file.getFileName().toString().equals("package.json")) { - System.out.println("Installing dependencies from " + file); - ProcessBuilder pb = - new ProcessBuilder( - Arrays.asList( - "yarn", - "install", - "--non-interactive", - "--ignore-scripts", - "--ignore-platform", - "--ignore-engines", - "--ignore-optional", - "--no-default-rc", - "--no-bin-links", - "--pure-lockfile")); - pb.directory(file.getParent().toFile()); - pb.redirectOutput(Redirect.INHERIT); - pb.redirectError(Redirect.INHERIT); try { - pb.start().waitFor(this.installDependenciesTimeout, TimeUnit.MILLISECONDS); - } catch (IOException | InterruptedException ex) { - throw new ResourceError("Could not install dependencies from " + file, ex); + String text = new WholeIO().read(file); + JsonElement json = new JsonParser().parse(text); + if (!(json instanceof JsonObject)) continue; + JsonObject jsonObject = (JsonObject) json; + file = file.toAbsolutePath(); + packageJsonFiles.put(file, jsonObject); + + JsonElement nameElm = jsonObject.get("name"); + if (nameElm instanceof JsonPrimitive && ((JsonPrimitive) nameElm).isString()) { + String name = nameElm.getAsString(); + packagesInRepo.add(name); + + if (validPackageName.matcher(name).matches()) { + // Create a symlink to the package: /node_modules/foo -> /path/to/foo + try { + Path symlinkPath = rootNodeModules.resolve(name); + if (!Files.exists(symlinkPath)) { + Files.createDirectories(symlinkPath.getParent()); + Files.createSymbolicLink(symlinkPath, file.getParent()); + } else { + // If node_modules/foo already exists, presumably it contains the right thing, + // so just continue extraction with that in place. + } + } catch (IOException e) { + throw new ResourceError("Could not install symlink to package " + file, e); + } + } + } + } catch (JsonParseException e) { + System.err.println("Could not parse JSON file: " + file); + System.err.println(e); + // Continue without the malformed package.json file } } } + + // Remove all dependencies on local packages from package.json files so yarn doesn't + // try to download them. Yarn would fail otherwise if these packages were never published. + // These packages will instead be found through the symlink we installed in node_modules above. + // Note that we ignore optional dependencies during installation, so "optionalDependencies" + // is ignored here as well. + final List dependencyFields = + Arrays.asList("dependencies", "devDependencies", "peerDependencies"); + final Set filesToChange = new LinkedHashSet<>(); + packageJsonFiles.forEach( + (path, packageJson) -> { + for (String dependencyField : dependencyFields) { + JsonElement dependencyElm = packageJson.get(dependencyField); + if (!(dependencyElm instanceof JsonObject)) continue; + JsonObject dependencyObj = (JsonObject) dependencyElm; + for (String packageName : packagesInRepo) { + if (!dependencyObj.has(packageName)) continue; + dependencyObj.remove(packageName); + filesToChange.add(path); + } + } + // Override "main" to point at the source folder instead of the output directory. + // TypeScript uses this field to locate the contents of a package. + // We simply guess that "./src" contains the source code, if it exists. + if (Files.exists(path.getParent().resolve("src"))) { + packageJson.addProperty("main", "./src"); + filesToChange.add(path); + } + }); + + // Write the new package.json files to disk + Map originalFiles = new LinkedHashMap<>(); + final String backupFilename = "package.json.lgtm.backup"; + for (Path file : filesToChange) { + Path backup = file.resolveSibling(backupFilename); + try { + originalFiles.put(file, backup); + Files.move(file, backup, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + throw new ResourceError("Could not backup package.json file: " + file, e); + } + try (Writer writer = Files.newBufferedWriter(file)) { + new Gson().toJson(packageJsonFiles.get(file), writer); + } catch (IOException e) { + throw new ResourceError("Could not rewrite package.json file: " + file, e); + } + } + + // Install dependencies + for (Path file : packageJsonFiles.keySet()) { + System.out.println("Installing dependencies from " + file); + ProcessBuilder pb = + new ProcessBuilder( + Arrays.asList( + "yarn", + "install", + "--non-interactive", + "--ignore-scripts", + "--ignore-platform", + "--ignore-engines", + "--ignore-optional", + "--no-default-rc", + "--no-bin-links", + "--pure-lockfile")); + pb.directory(file.getParent().toFile()); + pb.redirectOutput(Redirect.INHERIT); + pb.redirectError(Redirect.INHERIT); + try { + pb.start().waitFor(this.installDependenciesTimeout, TimeUnit.MILLISECONDS); + } catch (IOException | InterruptedException ex) { + restoreOriginalFiles(originalFiles); // Try to clean up before giving up. + throw new ResourceError("Could not install dependencies from " + file, ex); + } + } + + return originalFiles; } private ExtractorConfig mkExtractorConfig() { @@ -729,7 +854,8 @@ public class AutoBuild { // extract TypeScript projects from 'tsconfig.json' if (typeScriptMode == TypeScriptMode.FULL && file.getFileName().endsWith("tsconfig.json") - && !excludes.contains(file)) { + && !excludes.contains(file) + && isFileIncluded(file)) { tsconfigFiles.add(file); } diff --git a/javascript/extractor/src/com/semmle/js/extractor/test/AutoBuildTests.java b/javascript/extractor/src/com/semmle/js/extractor/test/AutoBuildTests.java index 3e39cfc6706..f78bec0c32d 100644 --- a/javascript/extractor/src/com/semmle/js/extractor/test/AutoBuildTests.java +++ b/javascript/extractor/src/com/semmle/js/extractor/test/AutoBuildTests.java @@ -20,6 +20,7 @@ import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.DosFileAttributeView; import java.util.ArrayList; +import java.util.Collections; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -129,8 +130,14 @@ public class AutoBuildTests { } @Override - protected void installDependencies(Set filesToExtract) { + protected void restoreOriginalFiles(java.util.Map originalFiles) { + // Do nothing + } + + @Override + protected Map installDependencies(Set filesToExtract) { // never install dependencies during testing + return Collections.emptyMap(); } @Override