hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-04-30 19:26:02 +02:00
Code Issues Packages Projects Releases Wiki Activity

JavaScript: Pull ExtractorState out of FileExtractor.

Browse Source
This commit is contained in:
Max Schaefer
2018-11-14 14:38:47 +00:00
parent 335d43b122
commit 5832f7c3ad
4 changed files with 25 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java
View File

@@ -173,7 +173,6 @@ public class AutoBuild {
private final Path LGTM_SRC, SEMMLE_DIST;
private final TypeScriptMode typeScriptMode;
private final String defaultEncoding;
private ExtractorState extractorState;
public AutoBuild() {
this.LGTM_SRC = toRealPath(getPathFromEnvVar("LGTM_SRC"));
@@ -182,7 +181,6 @@ public class AutoBuild {
this.trapCache = mkTrapCache();
this.typeScriptMode = getEnumFromEnvVar("LGTM_INDEX_TYPESCRIPT", TypeScriptMode.class, TypeScriptMode.BASIC);
this.defaultEncoding = getEnvVar("LGTM_INDEX_DEFAULT_ENCODING");
this.extractorState = new ExtractorState();
setupMatchers();
}
@@ -413,12 +411,12 @@ public class AutoBuild {
}
}
FileExtractor extractor = new FileExtractor(config, outputConfig, trapCache, extractorState);
FileExtractor extractor = new FileExtractor(config, outputConfig, trapCache);
FileVisitor<? super Path> visitor = new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
if (".js".equals(FileUtil.extension(file.toString())))
extract(extractor, file);
extract(extractor, file, null);
return super.visitFile(file, attrs);
}
};
@@ -435,7 +433,7 @@ public class AutoBuild {
config = config.withTypeScriptMode(typeScriptMode);
if (defaultEncoding != null)
config = config.withDefaultEncoding(defaultEncoding);
FileExtractor extractor = new FileExtractor(config, outputConfig, trapCache, extractorState);
FileExtractor extractor = new FileExtractor(config, outputConfig, trapCache);
Path[] currentRoot = new Path[1];
final Set<Path> filesToExtract = new LinkedHashSet<>();
final List<Path> tsconfigFiles = new ArrayList<>();
@@ -482,6 +480,7 @@ public class AutoBuild {
}
// If there are any .ts files, verify that TypeScript is installed.
ExtractorState extractorState = new ExtractorState();
TypeScriptParser tsParser = extractorState.getTypeScriptParser();
boolean hasTypeScriptFiles = false;
for (Path file : filesToExtract) {
@@ -494,7 +493,7 @@ public class AutoBuild {
}
}
if (hasTypeScriptFiles || !tsconfigFiles.isEmpty()) {
verifyTypeScriptInstallation();
verifyTypeScriptInstallation(extractorState);
}
// Extract TypeScript projects
@@ -515,7 +514,7 @@ public class AutoBuild {
typeScriptFiles.add(sourcePath.toFile());
}
}
extractTypeScriptFiles(typeScriptFiles, extractedFiles, extractor);
extractTypeScriptFiles(typeScriptFiles, extractedFiles, extractor, extractorState);
tsParser.closeProject(projectFile);
}
@@ -533,7 +532,7 @@ public class AutoBuild {
}
}
if (!remainingTypeScriptFiles.isEmpty()) {
extractTypeScriptFiles(remainingTypeScriptFiles, extractedFiles, extractor);
extractTypeScriptFiles(remainingTypeScriptFiles, extractedFiles, extractor, extractorState);
}
// The TypeScript compiler instance is no longer needed.
@@ -542,7 +541,7 @@ public class AutoBuild {
// Extract non-TypeScript files
for (Path f : filesToExtract) {
if (extractedFiles.add(f)) {
extract(extractor, f);
extract(extractor, f, null);
}
}
}
@@ -551,16 +550,17 @@ public class AutoBuild {
* Verifies that Node.js and the TypeScript compiler are installed and can be
* found.
*/
public void verifyTypeScriptInstallation() {
public void verifyTypeScriptInstallation(ExtractorState extractorState) {
extractorState.getTypeScriptParser().verifyInstallation(true);
}
public void extractTypeScriptFiles(List<File> files, Set<Path> extractedFiles, FileExtractor extractor) throws IOException {
public void extractTypeScriptFiles(List<File> files, Set<Path> extractedFiles,
FileExtractor extractor, ExtractorState extractorState) throws IOException {
extractorState.getTypeScriptParser().prepareFiles(files);
for (File f : files) {
Path path = f.toPath();
extractedFiles.add(path);
extract(extractor, f.toPath());
extract(extractor, f.toPath(), extractorState);
}
}
@@ -597,7 +597,7 @@ public class AutoBuild {
/**
* Extract a single file.
*/
protected void extract(FileExtractor extractor, Path file) throws IOException {
protected void extract(FileExtractor extractor, Path file, ExtractorState state) throws IOException {
File f = file.toFile();
if (!f.exists()) {
warn("Skipping " + file + ", which does not exist.");
@@ -605,7 +605,7 @@ public class AutoBuild {
}
long start = logBeginProcess("Extracting " + file);
extractor.extract(f);
extractor.extract(f, state);
logEndProcess(start);
}

13
javascript/extractor/src/com/semmle/js/extractor/FileExtractor.java
View File

@@ -395,13 +395,11 @@ public class FileExtractor {
private final ExtractorConfig config;
private final ExtractorOutputConfig outputConfig;
private final ITrapCache trapCache;
private final ExtractorState state;
public FileExtractor(ExtractorConfig config, ExtractorOutputConfig outputConfig, ITrapCache trapCache, ExtractorState state) {
public FileExtractor(ExtractorConfig config, ExtractorOutputConfig outputConfig, ITrapCache trapCache) {
this.config = config;
this.outputConfig = outputConfig;
this.trapCache = trapCache;
this.state = state;
}
public ExtractorConfig getConfig() {
@@ -412,7 +410,7 @@ public class FileExtractor {
return config.hasFileType() || FileType.forFile(f, config) != null;
}
public void extract(File f) throws IOException {
public void extract(File f, ExtractorState state) throws IOException {
// populate source archive
String source = new WholeIO(config.getDefaultEncoding()).strictread(f);
outputConfig.getSourceArchive().add(f, source);
@@ -424,7 +422,7 @@ public class FileExtractor {
locationManager.emitFileLocation(fileLabel, 0, 0, 0, 0);
// now extract the contents
extractContents(f, fileLabel, source, locationManager);
extractContents(f, fileLabel, source, locationManager, state);
}
@@ -440,7 +438,7 @@ public class FileExtractor {
*
* We only cache the content-dependent part, which makes up the bulk of the TRAP
* file anyway. The location-dependent part is emitted from scratch every time
* by the {@link #extract(File)} method above.
* by the {@link #extract(File, ExtractorState)} method above.
*
* In order to keep labels in the main part independent of the file's location,
* we bump the TRAP label counter to a known value (currently 20000) after the
@@ -451,7 +449,8 @@ public class FileExtractor {
* Also note that we support extraction with TRAP writer factories that are not file-backed;
* obviously, no caching is done in that scenario.
*/
private void extractContents(File f, Label fileLabel, String source, LocationManager locationManager) throws IOException {
private void extractContents(File f, Label fileLabel, String source, LocationManager locationManager,
ExtractorState state) throws IOException {
TrapWriter trapwriter = locationManager.getTrapWriter();
FileType fileType = config.hasFileType() ? FileType.valueOf(config.getFileType())
: FileType.forFile(f, config);

4
javascript/extractor/src/com/semmle/js/extractor/Main.java
View File

@@ -125,7 +125,7 @@ public class Main {
ap.error(P_TRAP_CACHE_BOUND + " should only be specified together with " + P_TRAP_CACHE + ".");
trapCache = new DummyTrapCache();
}
fileExtractor = new FileExtractor(extractorConfig, extractorOutputConfig, trapCache, extractorState);
fileExtractor = new FileExtractor(extractorConfig, extractorOutputConfig, trapCache);
setupMatchers(ap);
@@ -211,7 +211,7 @@ public class Main {
}
long start = verboseLogStartTimer(ap, "Extracting " + f);
try {
fileExtractor.extract(f.getAbsoluteFile());
fileExtractor.extract(f.getAbsoluteFile(), extractorState);
verboseLogEndTimer(ap, start);
} catch (IOException e) {
throw new ResourceError("Extraction of " + f + " failed.", e);

5
javascript/extractor/src/com/semmle/js/extractor/test/AutoBuildTests.java
View File

@@ -21,6 +21,7 @@ import org.junit.Before;
import org.junit.Test;
import com.semmle.js.extractor.AutoBuild;
import com.semmle.js.extractor.ExtractorState;
import com.semmle.js.extractor.FileExtractor;
import com.semmle.util.data.StringUtil;
import com.semmle.util.exception.UserError;
@@ -94,12 +95,12 @@ public class AutoBuildTests {
Set<String> actual = new LinkedHashSet<>();
new AutoBuild() {
@Override
protected void extract(FileExtractor extractor, Path file) {
protected void extract(FileExtractor extractor, Path file, ExtractorState state) {
actual.add(file.toString());
}
@Override
public void verifyTypeScriptInstallation() {
public void verifyTypeScriptInstallation(ExtractorState state) {
}
@Override