From 032ed1224235aa97ea2b518d6a41408d69e552b0 Mon Sep 17 00:00:00 2001 From: Max Schaefer Date: Fri, 9 Nov 2018 16:02:22 +0000 Subject: [PATCH 1/4] JavaScript: Use in-dist trap cache when extracting externs. --- .../src/com/semmle/js/extractor/AutoBuild.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java b/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java index 1b4b57812a5..b38e2898900 100644 --- a/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java +++ b/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java @@ -374,6 +374,16 @@ public class AutoBuild { */ private void extractExterns() throws IOException { ExtractorConfig config = new ExtractorConfig(false).withExterns(true); + + // use explicitly specified trap cache, or otherwise $SEMMLE_DIST/.cache/trap-cache/javascript, + // which we pre-populate when building the distribution + ITrapCache trapCache = this.trapCache; + if (trapCache instanceof DummyTrapCache) { + Path trapCachePath = SEMMLE_DIST.resolve(".cache").resolve("trap-cache").resolve("javascript"); + if (Files.isDirectory(trapCachePath)) + trapCache = new DefaultTrapCache(trapCachePath.toString(), null, Main.EXTRACTOR_VERSION); + } + FileExtractor extractor = new FileExtractor(config, outputConfig, trapCache, extractorState); FileVisitor visitor = new SimpleFileVisitor() { @Override From 01b43dff725a475b276bdbcf5ab2e768c51dea0d Mon Sep 17 00:00:00 2001 From: Max Schaefer Date: Mon, 12 Nov 2018 08:33:07 +0000 Subject: [PATCH 2/4] JavaScript: Make in-dist trap cache read-only. --- .../src/com/semmle/js/extractor/AutoBuild.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java b/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java index b38e2898900..efb77fe25d1 100644 --- a/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java +++ b/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java @@ -380,8 +380,16 @@ public class AutoBuild { ITrapCache trapCache = this.trapCache; if (trapCache instanceof DummyTrapCache) { Path trapCachePath = SEMMLE_DIST.resolve(".cache").resolve("trap-cache").resolve("javascript"); - if (Files.isDirectory(trapCachePath)) - trapCache = new DefaultTrapCache(trapCachePath.toString(), null, Main.EXTRACTOR_VERSION); + if (Files.isDirectory(trapCachePath)) { + trapCache = new DefaultTrapCache(trapCachePath.toString(), null, Main.EXTRACTOR_VERSION) { + @Override + public File lookup(String source, ExtractorConfig config, FileType type) { + File f = super.lookup(source, config, type); + // only return `f` if it exists; this has the effect of making the cache read-only + return f.exists() ? f : null; + } + }; + } } FileExtractor extractor = new FileExtractor(config, outputConfig, trapCache, extractorState); From d9d405118420121aaa7a518663832480c7148ea0 Mon Sep 17 00:00:00 2001 From: Max Schaefer Date: Tue, 13 Nov 2018 08:41:38 +0000 Subject: [PATCH 3/4] JavaScript: Extract auxiliary method. --- .../src/com/semmle/js/extractor/AutoBuild.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java b/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java index efb77fe25d1..4e1a2411976 100644 --- a/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java +++ b/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java @@ -285,9 +285,8 @@ public class AutoBuild { excludes.add(toRealPath(folderPath)); } catch (InvalidPathException | URISyntaxException | ResourceError e) { Exceptions.ignore(e, "Ignore path and print warning message instead"); - System.err.println("Ignoring '" + fields[0] + "' classification for " + + warn("Ignoring '" + fields[0] + "' classification for " + folder + ", which is not a valid path."); - System.err.flush(); } } } @@ -354,8 +353,7 @@ public class AutoBuild { patterns.add(realPath); } catch (ResourceError e) { Exceptions.ignore(e, "Ignore exception and print warning instead."); - System.err.println("Skipping path " + path + ", which does not exist."); - System.err.flush(); + warn("Skipping path " + path + ", which does not exist."); } return true; } @@ -557,8 +555,7 @@ public class AutoBuild { protected void extract(FileExtractor extractor, Path file) throws IOException { File f = file.toFile(); if (!f.exists()) { - System.err.println("Skipping " + file + ", which does not exist."); - System.err.flush(); + warn("Skipping " + file + ", which does not exist."); return; } @@ -567,6 +564,11 @@ public class AutoBuild { logEndProcess(); } + private void warn(String msg) { + System.err.println(msg); + System.err.flush(); + } + private void logBeginProcess(String message) { System.out.print(message + "..."); System.out.flush(); From 851e71c7d0deceb546bf71c2bcecc16e7f1666a0 Mon Sep 17 00:00:00 2001 From: Max Schaefer Date: Tue, 13 Nov 2018 08:41:53 +0000 Subject: [PATCH 4/4] JavaScript: Warn about externs trap cache absence/miss. --- .../src/com/semmle/js/extractor/AutoBuild.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java b/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java index 4e1a2411976..6d72d7fab0a 100644 --- a/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java +++ b/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java @@ -380,13 +380,24 @@ public class AutoBuild { Path trapCachePath = SEMMLE_DIST.resolve(".cache").resolve("trap-cache").resolve("javascript"); if (Files.isDirectory(trapCachePath)) { trapCache = new DefaultTrapCache(trapCachePath.toString(), null, Main.EXTRACTOR_VERSION) { + boolean warnedAboutCacheMiss = false; + @Override public File lookup(String source, ExtractorConfig config, FileType type) { File f = super.lookup(source, config, type); // only return `f` if it exists; this has the effect of making the cache read-only - return f.exists() ? f : null; + if (f.exists()) + return f; + // warn on first failed lookup + if (!warnedAboutCacheMiss) { + warn("Trap cache lookup for externs failed."); + warnedAboutCacheMiss = true; + } + return null; } }; + } else { + warn("No externs trap cache found"); } }