hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-04-26 17:25:19 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1372 from xiemaisi/js/fail-if-no-code

Browse Source 
Approved by esben-semmle
This commit is contained in:
semmle-qlci
2019-05-29 08:06:44 +01:00
committed by GitHub
parent b0a7f207e4 38a38ab780
commit 5bbbd26787
3 changed files with 42 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
change-notes/1.21/extractor-javascript.md
View File

@@ -5,5 +5,5 @@
## Changes to code extraction
* ECMAScript 2019 support is now enabled by default.
* On LGTM, JavaScript extraction for projects that do not contain any JavaScript or TypeScript code will now fail, even if the project contains other file types (such as HTML or YAML) recognized by the JavaScript extractor.
* YAML files are now extracted by default on LGTM. You can specify exclusion filters in your `lgtm.yml` file to override this behavior.

57
javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java
View File

@@ -1,26 +1,5 @@
package com.semmle.js.extractor;
import com.semmle.js.extractor.ExtractorConfig.SourceType;
import com.semmle.js.extractor.FileExtractor.FileType;
import com.semmle.js.extractor.trapcache.DefaultTrapCache;
import com.semmle.js.extractor.trapcache.DummyTrapCache;
import com.semmle.js.extractor.trapcache.ITrapCache;
import com.semmle.js.parser.ParsedProject;
import com.semmle.js.parser.TypeScriptParser;
import com.semmle.ts.extractor.TypeExtractor;
import com.semmle.ts.extractor.TypeTable;
import com.semmle.util.data.StringUtil;
import com.semmle.util.exception.CatastrophicError;
import com.semmle.util.exception.Exceptions;
import com.semmle.util.exception.ResourceError;
import com.semmle.util.exception.UserError;
import com.semmle.util.extraction.ExtractorOutputConfig;
import com.semmle.util.files.FileUtil;
import com.semmle.util.io.csv.CSVReader;
import com.semmle.util.language.LegacyLanguage;
import com.semmle.util.process.Env;
import com.semmle.util.projectstructure.ProjectLayout;
import com.semmle.util.trap.TrapWriter;
import java.io.File;
import java.io.IOException;
import java.io.Reader;
@@ -48,6 +27,28 @@ import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.stream.Stream;
import com.semmle.js.extractor.ExtractorConfig.SourceType;
import com.semmle.js.extractor.FileExtractor.FileType;
import com.semmle.js.extractor.trapcache.DefaultTrapCache;
import com.semmle.js.extractor.trapcache.DummyTrapCache;
import com.semmle.js.extractor.trapcache.ITrapCache;
import com.semmle.js.parser.ParsedProject;
import com.semmle.js.parser.TypeScriptParser;
import com.semmle.ts.extractor.TypeExtractor;
import com.semmle.ts.extractor.TypeTable;
import com.semmle.util.data.StringUtil;
import com.semmle.util.exception.CatastrophicError;
import com.semmle.util.exception.Exceptions;
import com.semmle.util.exception.ResourceError;
import com.semmle.util.exception.UserError;
import com.semmle.util.extraction.ExtractorOutputConfig;
import com.semmle.util.files.FileUtil;
import com.semmle.util.io.csv.CSVReader;
import com.semmle.util.language.LegacyLanguage;
import com.semmle.util.process.Env;
import com.semmle.util.projectstructure.ProjectLayout;
import com.semmle.util.trap.TrapWriter;
/**
* An alternative entry point to the JavaScript extractor.
*
@@ -194,6 +195,7 @@ public class AutoBuild {
private final TypeScriptMode typeScriptMode;
private final String defaultEncoding;
private ExecutorService threadPool;
private volatile boolean seenCode = false;
public AutoBuild() {
this.LGTM_SRC = toRealPath(getPathFromEnvVar("LGTM_SRC"));
@@ -425,7 +427,7 @@ public class AutoBuild {
}
/** Perform extraction. */
public void run() throws IOException {
public int run() throws IOException {
startThreadPool();
try {
extractSource();
@@ -434,6 +436,11 @@ public class AutoBuild {
} finally {
shutdownThreadPool();
}
if (!seenCode) {
warn("No JavaScript or TypeScript code found.");
return -1;
}
return 0;
}
private void startThreadPool() {
@@ -736,7 +743,9 @@ public class AutoBuild {
try {
long start = logBeginProcess("Extracting " + file);
extractor.extract(f, state);
Integer loc = extractor.extract(f, state);
if (!extractor.getConfig().isExterns() && (loc == null || loc != 0))
seenCode = true;
logEndProcess(start, "Done extracting " + file);
} catch (Throwable t) {
System.err.println("Exception while extracting " + file + ".");
@@ -787,7 +796,7 @@ public class AutoBuild {
public static void main(String[] args) {
try {
new AutoBuild().run();
System.exit(new AutoBuild().run());
} catch (IOException | UserError | CatastrophicError e) {
System.err.println(e.toString());
System.exit(1);

12
javascript/extractor/src/com/semmle/js/extractor/FileExtractor.java
View File

@@ -384,7 +384,10 @@ public class FileExtractor {
return config.hasFileType() || FileType.forFile(f, config) != null;
}
public void extract(File f, ExtractorState state) throws IOException {
/**
* @return the number of lines of code extracted, or {@code null} if the file was cached
*/
public Integer extract(File f, ExtractorState state) throws IOException {
// populate source archive
String source = new WholeIO(config.getDefaultEncoding()).strictread(f);
outputConfig.getSourceArchive().add(f, source);
@@ -396,7 +399,7 @@ public class FileExtractor {
locationManager.emitFileLocation(fileLabel, 0, 0, 0, 0);
// now extract the contents
extractContents(f, fileLabel, source, locationManager, state);
return extractContents(f, fileLabel, source, locationManager, state);
}
/**
@@ -420,7 +423,7 @@ public class FileExtractor {
* <p>Also note that we support extraction with TRAP writer factories that are not file-backed;
* obviously, no caching is done in that scenario.
*/
private void extractContents(
private Integer extractContents(
File f, Label fileLabel, String source, LocationManager locationManager, ExtractorState state)
throws IOException {
TrapWriter trapwriter = locationManager.getTrapWriter();
@@ -440,7 +443,7 @@ public class FileExtractor {
if (cacheFile.exists()) {
FileUtil.append(cacheFile, resultFile);
return;
return null;
}
// not in the cache yet, so use a caching TRAP writer to
@@ -463,6 +466,7 @@ public class FileExtractor {
trapwriter.addTuple("numlines", fileLabel, numLines, linesOfCode, linesOfComments);
trapwriter.addTuple("filetype", fileLabel, fileType.toString());
successful = true;
return linesOfCode;
} finally {
if (!successful && trapwriter instanceof CachingTrapWriter)
((CachingTrapWriter) trapwriter).discard();