hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-04-25 08:45:14 +02:00
Code Issues Packages Projects Releases Wiki Activity

JS: Add a file size limit to extractor

Browse Source
This commit is contained in:
Asger F
2023-08-09 11:35:21 +02:00
parent bc47646a79
commit d8462ad1b3
1 changed files with 11 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java
View File

@@ -222,6 +222,7 @@ public class AutoBuild {
private boolean installDependencies = false;
private final VirtualSourceRoot virtualSourceRoot;
private ExtractorState state;
private final long maximumFileSizeInMegabytes;
/** The default timeout when installing dependencies, in milliseconds. */
public static final int INSTALL_DEPENDENCIES_DEFAULT_TIMEOUT = 10 * 60 * 1000; // 10 minutes
@@ -236,6 +237,7 @@ public class AutoBuild {
this.defaultEncoding = getEnvVar("LGTM_INDEX_DEFAULT_ENCODING");
this.installDependencies = Boolean.valueOf(getEnvVar("LGTM_INDEX_TYPESCRIPT_INSTALL_DEPS"));
this.virtualSourceRoot = makeVirtualSourceRoot();
this.maximumFileSizeInMegabytes = EnvironmentVariables.getMegabyteCountFromPrefixedEnv("MAX_FILE_SIZE", 10);
setupFileTypes();
setupXmlMode();
setupMatchers();
@@ -446,8 +448,8 @@ public class AutoBuild {
}
/**
* Returns whether the autobuilder has seen code.
* This is overridden in tests.
* Returns whether the autobuilder has seen code.
* This is overridden in tests.
*/
protected boolean hasSeenCode() {
return seenCode;
@@ -741,12 +743,12 @@ public class AutoBuild {
dependencyInstallationResult = this.preparePackagesAndDependencies(filesToExtract);
}
Set<Path> extractedFiles = new LinkedHashSet<>();
// Extract HTML files as they may contain TypeScript
CompletableFuture<?> htmlFuture = extractFiles(
filesToExtract, extractedFiles, extractors,
f -> extractors.fileType(f) == FileType.HTML);
htmlFuture.join(); // Wait for HTML extraction to be finished.
// extract TypeScript projects and files
@@ -1229,6 +1231,11 @@ protected DependencyInstallationResult preparePackagesAndDependencies(Set<Path>
warn("Skipping " + file + ", which does not exist.");
return;
}
long fileSize = f.length();
if (fileSize > 1_000_000L * this.maximumFileSizeInMegabytes) {
warn("Skipping " + file + " because it is too large (" + StringUtil.printFloat(fileSize / 1_000_000.0) + " MB). The limit is " + this.maximumFileSizeInMegabytes + " MB.");
return;
}
try {
long start = logBeginProcess("Extracting " + file);