hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-04-30 11:15:13 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into seclab/dotjs

Browse Source
This commit is contained in:
Jorge
2023-11-30 19:42:18 +01:00
committed by GitHub
parent 91bc043f30 30e62d3d28
commit 8abd1d9855
9066 changed files with 655765 additions and 327070 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
javascript/extractor/src/com/semmle/js/ast/DynamicImport.java
View File

@@ -14,7 +14,10 @@ public class DynamicImport extends Expression {
return source;
}
/** Returns the second "argument" provided to the import, such as <code>{ assert: { type: "json" }}</code>. */
/**
* Returns the second "argument" provided to the import, such as <code>{ "with": { type: "json" }}
* </code>.
*/
public Expression getAttributes() {
return attributes;
}

10
javascript/extractor/src/com/semmle/js/ast/ExportAllDeclaration.java
View File

@@ -9,20 +9,20 @@ package com.semmle.js.ast;
*/
public class ExportAllDeclaration extends ExportDeclaration {
private final Literal source;
private final Expression assertion;
private final Expression attributes;
public ExportAllDeclaration(SourceLocation loc, Literal source, Expression assertion) {
public ExportAllDeclaration(SourceLocation loc, Literal source, Expression attributes) {
super("ExportAllDeclaration", loc);
this.source = source;
this.assertion = assertion;
this.attributes = attributes;
}
public Literal getSource() {
return source;
}
public Expression getAssertion() {
return assertion;
public Expression getAttributes() {
return attributes;
}
@Override

29
javascript/extractor/src/com/semmle/js/ast/ExportNamedDeclaration.java
View File

@@ -15,22 +15,30 @@ public class ExportNamedDeclaration extends ExportDeclaration {
private final Statement declaration;
private final List<ExportSpecifier> specifiers;
private final Literal source;
private final Expression assertion;
private final Expression attributes;
private final boolean hasTypeKeyword;
public ExportNamedDeclaration(
SourceLocation loc, Statement declaration, List<ExportSpecifier> specifiers, Literal source, Expression assertion) {
this(loc, declaration, specifiers, source, assertion, false);
SourceLocation loc,
Statement declaration,
List<ExportSpecifier> specifiers,
Literal source,
Expression attributes) {
this(loc, declaration, specifiers, source, attributes, false);
}
public ExportNamedDeclaration(
SourceLocation loc, Statement declaration, List<ExportSpecifier> specifiers, Literal source,
Expression assertion, boolean hasTypeKeyword) {
SourceLocation loc,
Statement declaration,
List<ExportSpecifier> specifiers,
Literal source,
Expression attributes,
boolean hasTypeKeyword) {
super("ExportNamedDeclaration", loc);
this.declaration = declaration;
this.specifiers = specifiers;
this.source = source;
this.assertion = assertion;
this.attributes = attributes;
this.hasTypeKeyword = hasTypeKeyword;
}
@@ -59,9 +67,12 @@ public class ExportNamedDeclaration extends ExportDeclaration {
return v.visit(this, c);
}
/** Returns the expression after the <code>assert</code> keyword, if any, such as <code>{ type: "json" }</code>. */
public Expression getAssertion() {
return assertion;
/**
* Returns the expression after the <code>with</code> keyword, if any, such as <code>
* { type: "json" }</code>.
*/
public Expression getAttributes() {
return attributes;
}
/** Returns true if this is an <code>export type</code> declaration. */

28
javascript/extractor/src/com/semmle/js/ast/ImportDeclaration.java
View File

@@ -1,8 +1,7 @@
package com.semmle.js.ast;
import java.util.List;
import com.semmle.ts.ast.INodeWithSymbol;
import java.util.List;
/**
* An import declaration, which can be of one of the following forms:
@@ -23,21 +22,27 @@ public class ImportDeclaration extends Statement implements INodeWithSymbol {
/** The module from which declarations are imported. */
private final Literal source;
private final Expression assertion;
private final Expression attributes;
private int symbol = -1;
private boolean hasTypeKeyword;
public ImportDeclaration(SourceLocation loc, List<ImportSpecifier> specifiers, Literal source, Expression assertion) {
this(loc, specifiers, source, assertion, false);
public ImportDeclaration(
SourceLocation loc, List<ImportSpecifier> specifiers, Literal source, Expression attributes) {
this(loc, specifiers, source, attributes, false);
}
public ImportDeclaration(SourceLocation loc, List<ImportSpecifier> specifiers, Literal source, Expression assertion, boolean hasTypeKeyword) {
public ImportDeclaration(
SourceLocation loc,
List<ImportSpecifier> specifiers,
Literal source,
Expression attributes,
boolean hasTypeKeyword) {
super("ImportDeclaration", loc);
this.specifiers = specifiers;
this.source = source;
this.assertion = assertion;
this.attributes = attributes;
this.hasTypeKeyword = hasTypeKeyword;
}
@@ -49,9 +54,12 @@ public class ImportDeclaration extends Statement implements INodeWithSymbol {
return specifiers;
}
/** Returns the expression after the <code>assert</code> keyword, if any, such as <code>{ type: "json" }</code>. */
public Expression getAssertion() {
return assertion;
/**
* Returns the expression after the <code>with</code> keyword, if any, such as <code>
* { type: "json" }</code>.
*/
public Expression getAttributes() {
return attributes;
}
@Override

31
javascript/extractor/src/com/semmle/js/ast/NodeCopier.java
View File

@@ -1,8 +1,5 @@
package com.semmle.js.ast;
import java.util.ArrayList;
import java.util.List;
import com.semmle.js.ast.jsx.JSXAttribute;
import com.semmle.js.ast.jsx.JSXClosingElement;
import com.semmle.js.ast.jsx.JSXElement;
@@ -42,16 +39,18 @@ import com.semmle.ts.ast.OptionalTypeExpr;
import com.semmle.ts.ast.ParenthesizedTypeExpr;
import com.semmle.ts.ast.PredicateTypeExpr;
import com.semmle.ts.ast.RestTypeExpr;
import com.semmle.ts.ast.SatisfiesExpr;
import com.semmle.ts.ast.TemplateLiteralTypeExpr;
import com.semmle.ts.ast.TupleTypeExpr;
import com.semmle.ts.ast.TypeAliasDeclaration;
import com.semmle.ts.ast.TypeAssertion;
import com.semmle.ts.ast.SatisfiesExpr;
import com.semmle.ts.ast.TypeParameter;
import com.semmle.ts.ast.TypeofTypeExpr;
import com.semmle.ts.ast.UnaryTypeExpr;
import com.semmle.ts.ast.UnionTypeExpr;
import com.semmle.util.data.IntList;
import java.util.ArrayList;
import java.util.List;
/** Deep cloning of AST nodes. */
public class NodeCopier implements Visitor<Void, INode> {
@@ -429,7 +428,8 @@ public class NodeCopier implements Visitor<Void, INode> {
@Override
public TemplateLiteralTypeExpr visit(TemplateLiteralTypeExpr nd, Void q) {
return new TemplateLiteralTypeExpr(visit(nd.getLoc()), copy(nd.getExpressions()), copy(nd.getQuasis()));
return new TemplateLiteralTypeExpr(
visit(nd.getLoc()), copy(nd.getExpressions()), copy(nd.getQuasis()));
}
@Override
@@ -523,7 +523,8 @@ public class NodeCopier implements Visitor<Void, INode> {
@Override
public ExportAllDeclaration visit(ExportAllDeclaration nd, Void c) {
return new ExportAllDeclaration(visit(nd.getLoc()), copy(nd.getSource()), copy(nd.getAssertion()));
return new ExportAllDeclaration(
visit(nd.getLoc()), copy(nd.getSource()), copy(nd.getAttributes()));
}
@Override
@@ -538,7 +539,7 @@ public class NodeCopier implements Visitor<Void, INode> {
copy(nd.getDeclaration()),
copy(nd.getSpecifiers()),
copy(nd.getSource()),
copy(nd.getAssertion()));
copy(nd.getAttributes()));
}
@Override
@@ -559,7 +560,11 @@ public class NodeCopier implements Visitor<Void, INode> {
@Override
public ImportDeclaration visit(ImportDeclaration nd, Void c) {
return new ImportDeclaration(
visit(nd.getLoc()), copy(nd.getSpecifiers()), copy(nd.getSource()), copy(nd.getAssertion()), nd.hasTypeKeyword());
visit(nd.getLoc()),
copy(nd.getSpecifiers()),
copy(nd.getSource()),
copy(nd.getAttributes()),
nd.hasTypeKeyword());
}
@Override
@@ -725,7 +730,8 @@ public class NodeCopier implements Visitor<Void, INode> {
@Override
public INode visit(TupleTypeExpr nd, Void c) {
return new TupleTypeExpr(visit(nd.getLoc()), copy(nd.getElementTypes()), copy(nd.getElementNames()));
return new TupleTypeExpr(
visit(nd.getLoc()), copy(nd.getElementTypes()), copy(nd.getElementNames()));
}
@Override
@@ -787,9 +793,7 @@ public class NodeCopier implements Visitor<Void, INode> {
@Override
public INode visit(SatisfiesExpr nd, Void c) {
return new SatisfiesExpr(
visit(nd.getLoc()),
copy(nd.getExpression()),
copy(nd.getTypeAnnotation()));
visit(nd.getLoc()), copy(nd.getExpression()), copy(nd.getTypeAnnotation()));
}
@Override
@@ -907,7 +911,8 @@ public class NodeCopier implements Visitor<Void, INode> {
@Override
public INode visit(GeneratedCodeExpr nd, Void c) {
return new GeneratedCodeExpr(visit(nd.getLoc()), nd.getOpeningDelimiter(), nd.getClosingDelimiter(), nd.getBody());
return new GeneratedCodeExpr(
visit(nd.getLoc()), nd.getOpeningDelimiter(), nd.getClosingDelimiter(), nd.getBody());
}
@Override

5
javascript/extractor/src/com/semmle/js/ast/VariableDeclaration.java
View File

@@ -28,8 +28,8 @@ public class VariableDeclaration extends Statement {
}
/**
* The kind of this variable declaration statement; one of <code>"var"</code>, <code>"let"</code>
* or <code>"const"</code>.
* The kind of this variable declaration statement; one of <code>"var"</code>, <code>"let"</code>,
* <code>"const"</code>, or <code>"using"</code>.
*/
public String getKind() {
return kind;
@@ -42,6 +42,7 @@ public class VariableDeclaration extends Statement {
*/
public boolean isBlockScoped(ECMAVersion ecmaVersion) {
return "let".equals(kind)
|| "using".equals(kind)
|| ecmaVersion.compareTo(ECMAVersion.ECMA2015) >= 0 && "const".equals(kind);
}

78
javascript/extractor/src/com/semmle/js/extractor/ASTExtractor.java
View File

@@ -1,15 +1,5 @@
package com.semmle.js.extractor;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.Arrays;
import java.util.List;
import java.util.Set;
import java.util.Stack;
import java.util.regex.Matcher;
import com.semmle.js.ast.AClass;
import com.semmle.js.ast.AFunction;
import com.semmle.js.ast.AFunctionExpression;
@@ -150,11 +140,11 @@ import com.semmle.ts.ast.OptionalTypeExpr;
import com.semmle.ts.ast.ParenthesizedTypeExpr;
import com.semmle.ts.ast.PredicateTypeExpr;
import com.semmle.ts.ast.RestTypeExpr;
import com.semmle.ts.ast.SatisfiesExpr;
import com.semmle.ts.ast.TemplateLiteralTypeExpr;
import com.semmle.ts.ast.TupleTypeExpr;
import com.semmle.ts.ast.TypeAliasDeclaration;
import com.semmle.ts.ast.TypeAssertion;
import com.semmle.ts.ast.SatisfiesExpr;
import com.semmle.ts.ast.TypeExpression;
import com.semmle.ts.ast.TypeParameter;
import com.semmle.ts.ast.TypeofTypeExpr;
@@ -166,6 +156,13 @@ import com.semmle.util.locations.OffsetTranslation;
import com.semmle.util.locations.SourceMap;
import com.semmle.util.trap.TrapWriter;
import com.semmle.util.trap.TrapWriter.Label;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.Stack;
import java.util.regex.Matcher;
/** Extractor for AST-based information; invoked by the {@link JSExtractor}. */
public class ASTExtractor {
@@ -387,7 +384,8 @@ public class ASTExtractor {
return visit(child, parent, childIndex, IdContext.VAR_BIND, binopOperand);
}
private Label visit(INode child, Label parent, int childIndex, IdContext idContext, boolean binopOperand) {
private Label visit(
INode child, Label parent, int childIndex, IdContext idContext, boolean binopOperand) {
if (child == null) return null;
return child.accept(this, new Context(parent, childIndex, idContext, binopOperand));
}
@@ -590,15 +588,28 @@ public class ASTExtractor {
trapwriter.addTuple("literals", valueString, source, key);
Position start = nd.getLoc().getStart();
com.semmle.util.locations.Position startPos = new com.semmle.util.locations.Position(start.getLine(), start.getColumn() + 1 /* Convert from 0-based to 1-based. */, start.getOffset());
com.semmle.util.locations.Position startPos =
new com.semmle.util.locations.Position(
start.getLine(),
start.getColumn() + 1 /* Convert from 0-based to 1-based. */,
start.getOffset());
if (nd.isRegExp()) {
OffsetTranslation offsets = new OffsetTranslation();
offsets.set(0, 1); // skip the initial '/'
SourceMap sourceMap = SourceMap.legacyWithStartPos(SourceMap.fromString(nd.getRaw()).offsetBy(0, offsets), startPos);
SourceMap sourceMap =
SourceMap.legacyWithStartPos(
SourceMap.fromString(nd.getRaw()).offsetBy(0, offsets), startPos);
regexpExtractor.extract(source.substring(1, source.lastIndexOf('/')), sourceMap, nd, false);
} else if (nd.isStringLiteral() && !c.isInsideType() && nd.getRaw().length() < 1000 && !c.isBinopOperand()) {
SourceMap sourceMap = SourceMap.legacyWithStartPos(SourceMap.fromString(nd.getRaw()).offsetBy(0, makeStringLiteralOffsets(nd.getRaw())), startPos);
} else if (nd.isStringLiteral()
&& !c.isInsideType()
&& nd.getRaw().length() < 1000
&& !c.isBinopOperand()) {
SourceMap sourceMap =
SourceMap.legacyWithStartPos(
SourceMap.fromString(nd.getRaw())
.offsetBy(0, makeStringLiteralOffsets(nd.getRaw())),
startPos);
regexpExtractor.extract(valueString, sourceMap, nd, true);
// Scan the string for template tags, if we're in a context where such tags are relevant.
@@ -621,8 +632,8 @@ public class ASTExtractor {
}
/**
* Constant-folds simple string concatenations in `exp` while keeping an offset translation
* that tracks back to the original source.
* Constant-folds simple string concatenations in `exp` while keeping an offset translation that
* tracks back to the original source.
*/
private Pair<String, OffsetTranslation> getStringConcatResult(Expression exp) {
if (exp instanceof BinaryExpression) {
@@ -638,7 +649,9 @@ public class ASTExtractor {
return null;
}
int delta = be.getRight().getLoc().getStart().getOffset() - be.getLeft().getLoc().getStart().getOffset();
int delta =
be.getRight().getLoc().getStart().getOffset()
- be.getLeft().getLoc().getStart().getOffset();
int offset = left.fst().length();
return Pair.make(str, left.snd().append(right.snd(), offset, delta));
}
@@ -748,7 +761,9 @@ public class ASTExtractor {
visit(nd.getProperty(), key, 1, IdContext.TYPE_LABEL);
} else {
IdContext baseIdContext =
(c.idcontext == IdContext.EXPORT || c.idcontext == IdContext.EXPORT_BASE) ? IdContext.EXPORT_BASE : IdContext.VAR_BIND;
(c.idcontext == IdContext.EXPORT || c.idcontext == IdContext.EXPORT_BASE)
? IdContext.EXPORT_BASE
: IdContext.VAR_BIND;
visit(nd.getObject(), key, 0, baseIdContext);
visit(nd.getProperty(), key, 1, nd.isComputed() ? IdContext.VAR_BIND : IdContext.LABEL);
}
@@ -848,8 +863,11 @@ public class ASTExtractor {
@Override
public Label visit(BinaryExpression nd, Context c) {
Label key = super.visit(nd, c);
if (nd.getOperator().equals("in") && nd.getLeft() instanceof Identifier && ((Identifier)nd.getLeft()).getName().startsWith("#")) {
// this happens with Ergonomic brand checks for Private Fields (see https://github.com/tc39/proposal-private-fields-in-in).
if (nd.getOperator().equals("in")
&& nd.getLeft() instanceof Identifier
&& ((Identifier) nd.getLeft()).getName().startsWith("#")) {
// this happens with Ergonomic brand checks for Private Fields (see
// https://github.com/tc39/proposal-private-fields-in-in).
// it's the only case where private field identifiers are used not as a field.
visit(nd.getLeft(), key, 0, IdContext.LABEL, true);
} else {
@@ -875,8 +893,14 @@ public class ASTExtractor {
}
OffsetTranslation offsets = concatResult.snd();
Position start = nd.getLoc().getStart();
com.semmle.util.locations.Position startPos = new com.semmle.util.locations.Position(start.getLine(), start.getColumn() + 1 /* Convert from 0-based to 1-based. */, start.getOffset());
SourceMap sourceMap = SourceMap.legacyWithStartPos(SourceMap.fromString(nd.getLoc().getSource()).offsetBy(0, offsets), startPos);
com.semmle.util.locations.Position startPos =
new com.semmle.util.locations.Position(
start.getLine(),
start.getColumn() + 1 /* Convert from 0-based to 1-based. */,
start.getOffset());
SourceMap sourceMap =
SourceMap.legacyWithStartPos(
SourceMap.fromString(nd.getLoc().getSource()).offsetBy(0, offsets), startPos);
regexpExtractor.extract(foldedString, sourceMap, nd, true);
return;
}
@@ -1759,7 +1783,7 @@ public class ASTExtractor {
public Label visit(ExportAllDeclaration nd, Context c) {
Label lbl = super.visit(nd, c);
visit(nd.getSource(), lbl, 0);
visit(nd.getAssertion(), lbl, -10);
visit(nd.getAttributes(), lbl, -10);
return lbl;
}
@@ -1775,7 +1799,7 @@ public class ASTExtractor {
Label lbl = super.visit(nd, c);
visit(nd.getDeclaration(), lbl, -1);
visit(nd.getSource(), lbl, -2);
visit(nd.getAssertion(), lbl, -10);
visit(nd.getAttributes(), lbl, -10);
IdContext childContext =
nd.hasSource()
? IdContext.LABEL
@@ -1799,7 +1823,7 @@ public class ASTExtractor {
public Label visit(ImportDeclaration nd, Context c) {
Label lbl = super.visit(nd, c);
visit(nd.getSource(), lbl, -1);
visit(nd.getAssertion(), lbl, -10);
visit(nd.getAttributes(), lbl, -10);
IdContext childContext =
nd.hasTypeKeyword()
? IdContext.TYPE_ONLY_IMPORT

30
javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java
View File

@@ -153,7 +153,7 @@ import com.semmle.util.trap.TrapWriter;
* <li>All JavaScript files, that is, files with one of the extensions supported by {@link
* FileType#JS} (currently ".js", ".jsx", ".mjs", ".cjs", ".es6", ".es").
* <li>All HTML files, that is, files with with one of the extensions supported by {@link
* FileType#HTML} (currently ".htm", ".html", ".xhtm", ".xhtml", ".vue", ".html.erb", ".html.dot").
* FileType#HTML} (currently ".htm", ".html", ".xhtm", ".xhtml", ".vue", ".html.erb", ".html.dot", ".jsp").
* <li>All YAML files, that is, files with one of the extensions supported by {@link
* FileType#YAML} (currently ".raml", ".yaml", ".yml").
* <li>Files with base name "package.json" or "tsconfig.json", and files whose base name
@@ -222,6 +222,7 @@ public class AutoBuild {
private boolean installDependencies = false;
private final VirtualSourceRoot virtualSourceRoot;
private ExtractorState state;
private final long maximumFileSizeInMegabytes;
/** The default timeout when installing dependencies, in milliseconds. */
public static final int INSTALL_DEPENDENCIES_DEFAULT_TIMEOUT = 10 * 60 * 1000; // 10 minutes
@@ -236,6 +237,7 @@ public class AutoBuild {
this.defaultEncoding = getEnvVar("LGTM_INDEX_DEFAULT_ENCODING");
this.installDependencies = Boolean.valueOf(getEnvVar("LGTM_INDEX_TYPESCRIPT_INSTALL_DEPS"));
this.virtualSourceRoot = makeVirtualSourceRoot();
this.maximumFileSizeInMegabytes = EnvironmentVariables.getMegabyteCountFromPrefixedEnv("MAX_FILE_SIZE", 10);
setupFileTypes();
setupXmlMode();
setupMatchers();
@@ -446,8 +448,8 @@ public class AutoBuild {
}
/**
* Returns whether the autobuilder has seen code.
* This is overridden in tests.
* Returns whether the autobuilder has seen code.
* This is overridden in tests.
*/
protected boolean hasSeenCode() {
return seenCode;
@@ -741,12 +743,12 @@ public class AutoBuild {
dependencyInstallationResult = this.preparePackagesAndDependencies(filesToExtract);
}
Set<Path> extractedFiles = new LinkedHashSet<>();
// Extract HTML files as they may contain TypeScript
CompletableFuture<?> htmlFuture = extractFiles(
filesToExtract, extractedFiles, extractors,
f -> extractors.fileType(f) == FileType.HTML);
htmlFuture.join(); // Wait for HTML extraction to be finished.
// extract TypeScript projects and files
@@ -890,10 +892,15 @@ protected DependencyInstallationResult preparePackagesAndDependencies(Set<Path>
// For named packages, find the main file.
String name = packageJson.getName();
if (name != null) {
Path entryPoint = guessPackageMainFile(path, packageJson, FileType.TYPESCRIPT.getExtensions());
if (entryPoint == null) {
// Try a TypeScript-recognized JS extension instead
entryPoint = guessPackageMainFile(path, packageJson, Arrays.asList(".js", ".jsx"));
Path entryPoint = null;
try {
entryPoint = guessPackageMainFile(path, packageJson, FileType.TYPESCRIPT.getExtensions());
if (entryPoint == null) {
// Try a TypeScript-recognized JS extension instead
entryPoint = guessPackageMainFile(path, packageJson, Arrays.asList(".js", ".jsx"));
}
} catch (InvalidPathException ignore) {
// can happen if the `main:` field is invalid. E.g. on Windows a path like `dist/*.js` will crash.
}
if (entryPoint != null) {
System.out.println(relativePath + ": Main file set to " + sourceRoot.relativize(entryPoint));
@@ -1229,6 +1236,11 @@ protected DependencyInstallationResult preparePackagesAndDependencies(Set<Path>
warn("Skipping " + file + ", which does not exist.");
return;
}
long fileSize = f.length();
if (fileSize > 1_000_000L * this.maximumFileSizeInMegabytes) {
warn("Skipping " + file + " because it is too large (" + StringUtil.printFloat(fileSize / 1_000_000.0) + " MB). The limit is " + this.maximumFileSizeInMegabytes + " MB.");
return;
}
try {
long start = logBeginProcess("Extracting " + file);

33
javascript/extractor/src/com/semmle/js/extractor/EnvironmentVariables.java
View File

@@ -1,5 +1,6 @@
package com.semmle.js.extractor;
import com.semmle.util.data.UnitParser;
import com.semmle.util.exception.UserError;
import com.semmle.util.process.Env;
import com.semmle.util.process.Env.Var;
@@ -7,7 +8,7 @@ import com.semmle.util.process.Env.Var;
public class EnvironmentVariables {
public static final String CODEQL_EXTRACTOR_JAVASCRIPT_ROOT_ENV_VAR =
"CODEQL_EXTRACTOR_JAVASCRIPT_ROOT";
public static final String CODEQL_EXTRACTOR_JAVASCRIPT_SCRATCH_DIR_ENV_VAR =
"CODEQL_EXTRACTOR_JAVASCRIPT_SCRATCH_DIR";
@@ -19,6 +20,36 @@ public class EnvironmentVariables {
public static final String CODEQL_DIST_ENV_VAR = "CODEQL_DIST";
/**
* Returns a number of megabytes by reading an environment variable with the given suffix,
* or the default value if not set.
* <p>
* The following prefixes are tried:
* <code>CODEQL_EXTRACTOR_JAVASCRIPT_</code>,
* <code>LGTM_</code>,
* <code>SEMMLE_</code>.
*/
public static int getMegabyteCountFromPrefixedEnv(String suffix, int defaultValue) {
String envVar = "CODEQL_EXTRACTOR_JAVASCRIPT_" + suffix;
String value = Env.systemEnv().get(envVar);
if (value == null || value.length() == 0) {
envVar = "LGTM_" + suffix;
value = Env.systemEnv().get(envVar);
}
if (value == null || value.length() == 0) {
envVar = "SEMMLE_" + suffix;
value = Env.systemEnv().get(envVar);
}
if (value == null || value.length() == 0) {
return defaultValue;
}
Integer amount = UnitParser.parseOpt(value, UnitParser.MEGABYTES);
if (amount == null) {
throw new UserError("Invalid value for " + envVar + ": '" + value + "'");
}
return amount;
}
/**
* Gets the extractor root based on the <code>CODEQL_EXTRACTOR_JAVASCRIPT_ROOT</code> or <code>
* SEMMLE_DIST</code> or environment variable, or <code>null</code> if neither is set.

2
javascript/extractor/src/com/semmle/js/extractor/FileExtractor.java
View File

@@ -103,7 +103,7 @@ public class FileExtractor {
/** Information about supported file types. */
public static enum FileType {
HTML(".htm", ".html", ".xhtm", ".xhtml", ".vue", ".hbs", ".ejs", ".njk", ".erb", ".dot") {
HTML(".htm", ".html", ".xhtm", ".xhtml", ".vue", ".hbs", ".ejs", ".njk", ".erb", ".jsp", ".dot") {
@Override
public IExtractor mkExtractor(ExtractorConfig config, ExtractorState state) {
return new HTMLExtractor(config, state);

2
javascript/extractor/src/com/semmle/js/extractor/Main.java
View File

@@ -41,7 +41,7 @@ public class Main {
* A version identifier that should be updated every time the extractor changes in such a way that
* it may produce different tuples for the same file under the same {@link ExtractorConfig}.
*/
public static final String EXTRACTOR_VERSION = "2023-04-19";
public static final String EXTRACTOR_VERSION = "2023-10-13";
public static final Pattern NEWLINE = Pattern.compile("\n");

1
javascript/extractor/src/com/semmle/js/extractor/StmtKinds.java
View File

@@ -58,6 +58,7 @@ public class StmtKinds {
declKinds.put("var", 18);
declKinds.put("const", 22);
declKinds.put("let", 23);
declKinds.put("using", 40);
}
public static int getStmtKind(final Statement stmt) {

6
javascript/extractor/src/com/semmle/js/extractor/TypeExprKinds.java
View File

@@ -7,6 +7,7 @@ import com.semmle.js.ast.Identifier;
import com.semmle.js.ast.Literal;
import com.semmle.js.ast.MemberExpression;
import com.semmle.js.ast.TemplateElement;
import com.semmle.js.ast.ThisExpression;
import com.semmle.js.extractor.ASTExtractor.IdContext;
import com.semmle.ts.ast.ArrayTypeExpr;
import com.semmle.ts.ast.ConditionalTypeExpr;
@@ -100,6 +101,11 @@ public class TypeExprKinds {
return keywordTypeExpr;
}
@Override
public Integer visit(ThisExpression nd, Void c) {
return thisVarTypeAccess;
}
@Override
public Integer visit(ArrayTypeExpr nd, Void c) {
return arrayTypeExpr;

87
javascript/extractor/src/com/semmle/js/extractor/test/ASTMatchingTests.java
View File

@@ -1,87 +0,0 @@
package com.semmle.js.extractor.test;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonNull;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.gson.JsonPrimitive;
import com.semmle.jcorn.ESNextParser;
import com.semmle.jcorn.Options;
import com.semmle.jcorn.SyntaxError;
import com.semmle.js.ast.AST2JSON;
import com.semmle.js.ast.Program;
import com.semmle.util.io.WholeIO;
import java.io.File;
import java.util.Iterator;
import java.util.Map.Entry;
import org.junit.Assert;
/** Base class for tests that use Acorn-like AST templates to specify expected test output. */
public abstract class ASTMatchingTests {
/**
* Assert that the given actual sub-AST matches the expected AST template, where {@code path} is
* the path from the root to the sub-AST.
*/
protected void assertMatch(String path, JsonElement expected, JsonElement actual) {
if (expected == null) Assert.assertNull(path + ": null != " + actual, actual);
if (expected instanceof JsonPrimitive || expected instanceof JsonNull)
Assert.assertEquals(path, expected.toString(), actual.toString());
if (expected instanceof JsonArray) {
Assert.assertTrue(path + ": " + expected + " != " + actual, actual instanceof JsonArray);
Iterator<JsonElement> expectedElements = ((JsonArray) expected).iterator();
Iterator<JsonElement> actualElements = ((JsonArray) actual).iterator();
int elt = 0;
while (expectedElements.hasNext()) {
String newPath = path + "[" + elt++ + "]";
Assert.assertTrue(newPath + ": missing", actualElements.hasNext());
assertMatch(newPath, expectedElements.next(), actualElements.next());
}
}
if (expected instanceof JsonObject) {
Assert.assertTrue(path + ": " + expected + " != " + actual, actual instanceof JsonObject);
JsonObject actualObject = (JsonObject) actual;
for (Entry<String, JsonElement> expectedProp : ((JsonObject) expected).entrySet()) {
String key = expectedProp.getKey();
JsonElement value = expectedProp.getValue();
String newPath = path + "." + key;
Assert.assertTrue(newPath + ": missing", actualObject.has(key));
assertMatch(newPath, value, actualObject.get(key));
}
}
}
private static final File BABYLON_BASE = new File("parser-tests/babylon").getAbsoluteFile();
protected void babylonTest(String dir) {
babylonTest(dir, new Options().esnext(true));
}
protected void babylonTest(String dir, Options options) {
File actual = new File(new File(BABYLON_BASE, dir), "actual.js");
String actualSrc = new WholeIO().strictread(actual);
JsonElement actualJSON;
try {
Program actualAST = new ESNextParser(options, actualSrc, 0).parse();
actualJSON = AST2JSON.convert(actualAST);
} catch (SyntaxError e) {
actualJSON = new JsonPrimitive(e.getMessage());
}
File expected = new File(actual.getParentFile(), "expected.ast");
JsonElement expectedJSON;
if (expected.exists()) {
String expectedSrc = new WholeIO().strictread(expected);
expectedJSON = new JsonParser().parse(expectedSrc).getAsJsonObject();
} else {
File expectedErrFile = new File(actual.getParentFile(), "expected.error");
String expectedErrMsg = new WholeIO().strictread(expectedErrFile).trim();
expectedJSON = new JsonPrimitive(expectedErrMsg);
}
assertMatch("<root>", expectedJSON, actualJSON);
}
}

21
javascript/extractor/src/com/semmle/js/extractor/test/AllTests.java
View File

@@ -1,21 +0,0 @@
package com.semmle.js.extractor.test;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
import org.junit.runners.Suite.SuiteClasses;
@RunWith(Suite.class)
@SuiteClasses({
JSXTests.class,
NodeJSDetectorTests.class,
TrapTests.class,
ObjectRestSpreadTests.class,
ClassPropertiesTests.class,
FunctionSentTests.class,
DecoratorTests.class,
ExportExtensionsTests.class,
AutoBuildTests.class,
RobustnessTests.class,
NumericSeparatorTests.class
})
public class AllTests {}

631
javascript/extractor/src/com/semmle/js/extractor/test/AutoBuildTests.java
View File

@@ -1,631 +0,0 @@
package com.semmle.js.extractor.test;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.nio.file.attribute.DosFileAttributeView;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import org.junit.After;
import org.junit.Assert;
import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
import com.semmle.js.extractor.AutoBuild;
import com.semmle.js.extractor.DependencyInstallationResult;
import com.semmle.js.extractor.ExtractorState;
import com.semmle.js.extractor.FileExtractor;
import com.semmle.js.extractor.FileExtractor.FileType;
import com.semmle.js.extractor.VirtualSourceRoot;
import com.semmle.util.data.StringUtil;
import com.semmle.util.exception.UserError;
import com.semmle.util.files.FileUtil;
import com.semmle.util.files.FileUtil8;
import com.semmle.util.process.Env;
public class AutoBuildTests {
private Path SEMMLE_DIST, LGTM_SRC;
private Set<String> expected;
private Map<String, String> envVars;
/**
* Set up fake distribution and source directory and environment variables pointing to them, and
* add in a few fake externs.
*/
@Before
public void setup() throws IOException {
expected = new LinkedHashSet<>();
envVars = new LinkedHashMap<>();
// set up an empty distribution directory with an empty sub-directory for externs
SEMMLE_DIST = Files.createTempDirectory("autobuild-dist").toRealPath();
Path externs =
Files.createDirectories(Paths.get(SEMMLE_DIST.toString(), "tools", "data", "externs"));
// set up environment variables (the value of TRAP_FOLDER and SOURCE_ARCHIVE doesn't
// really matter, since we won't do any actual extraction)
envVars.put(Env.Var.SEMMLE_DIST.toString(), SEMMLE_DIST.toString());
envVars.put(Env.Var.TRAP_FOLDER.toString(), SEMMLE_DIST.toString());
envVars.put(Env.Var.SOURCE_ARCHIVE.toString(), SEMMLE_DIST.toString());
// set up an empty source directory
LGTM_SRC = Files.createTempDirectory("autobuild-src").toRealPath();
envVars.put("LGTM_SRC", LGTM_SRC.toString());
// add a few fake externs
addFile(true, externs, "a.js");
addFile(false, externs, "a.html");
addFile(true, externs, "sub", "b.js");
addFile(false, externs, "sub", "b.json");
}
/** Clean up distribution and source directory, and reset environment. */
@After
public void teardown() throws IOException {
FileUtil8.recursiveDelete(SEMMLE_DIST);
FileUtil8.recursiveDelete(LGTM_SRC);
}
/**
* Add a file under {@code root} that we either do or don't expect to be extracted, depending on
* the value of {@code extracted}. If the file is expected to be extracted, its path is added to
* {@link #expected}. If non-null, parameter {@code fileType} indicates the file type with which
* we expect the file to be extracted.
*/
private Path addFile(boolean extracted, FileType fileType, Path root, String... components)
throws IOException {
Path f = addFile(root, components);
if (extracted) {
expected.add(f + (fileType == null ? "" : ":" + fileType.toString()));
}
return f;
}
/** Add a file with default file type. */
private Path addFile(boolean extracted, Path root, String... components) throws IOException {
return addFile(extracted, null, root, components);
}
/** Create a file at the specified path under {@code root} and return it. */
private Path addFile(Path root, String... components) throws IOException {
Path p = Paths.get(root.toString(), components);
Files.createDirectories(p.getParent());
return Files.createFile(p);
}
/** Run autobuild and compare the set of actually extracted files against {@link #expected}. */
private void runTest() throws IOException {
Env.systemEnv().pushEnvironmentContext(envVars);
try {
Set<String> actual = new LinkedHashSet<>();
new AutoBuild() {
@Override
protected CompletableFuture<?> extract(FileExtractor extractor, Path file, boolean concurrent) {
String extracted = file.toString();
if (extractor.getConfig().hasFileType())
extracted += ":" + extractor.getFileType(file.toFile());
actual.add(extracted);
return CompletableFuture.completedFuture(null);
}
@Override
protected boolean hasSeenCode() {
return true;
}
@Override
public void verifyTypeScriptInstallation(ExtractorState state) {}
@Override
public void extractTypeScriptFiles(
java.util.List<Path> files,
java.util.Set<Path> extractedFiles,
FileExtractors extractors) {
for (Path f : files) {
actual.add(f.toString());
}
}
@Override
protected DependencyInstallationResult preparePackagesAndDependencies(Set<Path> filesToExtract) {
// currently disabled in tests
return DependencyInstallationResult.empty;
}
@Override
protected VirtualSourceRoot makeVirtualSourceRoot() {
return VirtualSourceRoot.none; // not used in these tests
}
@Override
protected void extractXml() throws IOException {
Files.walkFileTree(
LGTM_SRC,
new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
throws IOException {
String ext = FileUtil.extension(file);
if (!ext.isEmpty() && getXmlExtensions().contains(ext.substring(1)))
actual.add(file.toString());
return FileVisitResult.CONTINUE;
}
});
}
}.run();
String expectedString = StringUtil.glue("\n", expected.stream().sorted().toArray());
String actualString = StringUtil.glue("\n", actual.stream().sorted().toArray());
Assert.assertEquals(expectedString, actualString);
} finally {
Env.systemEnv().popEnvironmentContext();
}
}
@Test
public void basicTest() throws IOException {
addFile(true, LGTM_SRC, "tst.js");
addFile(true, LGTM_SRC, "tst.ts");
addFile(true, LGTM_SRC, "tst.html");
addFile(false, LGTM_SRC, "tst.json");
addFile(true, LGTM_SRC, "package.json");
addFile(true, LGTM_SRC, ".eslintrc.yml");
addFile(true, LGTM_SRC, "vendor", "leftpad", "index.js");
runTest();
}
@Test
public void typescript() throws IOException {
envVars.put("LGTM_INDEX_TYPESCRIPT", "basic");
addFile(true, LGTM_SRC, "tst.ts");
addFile(true, LGTM_SRC, "tst.tsx");
runTest();
}
@Test(expected = UserError.class)
public void typescriptWrongConfig() throws IOException {
envVars.put("LGTM_INDEX_TYPESCRIPT", "true");
addFile(true, LGTM_SRC, "tst.ts");
addFile(true, LGTM_SRC, "tst.tsx");
runTest();
}
@Test
public void includeFile() throws IOException {
envVars.put("LGTM_INDEX_INCLUDE", "tst.js");
addFile(true, LGTM_SRC, "tst.js");
addFile(false, LGTM_SRC, "vendor", "leftpad", "index.js");
runTest();
}
@Test
public void excludeFile() throws IOException {
envVars.put("LGTM_INDEX_EXCLUDE", "vendor/leftpad/index.js");
addFile(true, LGTM_SRC, "tst.js");
addFile(false, LGTM_SRC, "vendor", "leftpad", "index.js");
runTest();
}
@Test
public void excludeFolderByPattern() throws IOException {
envVars.put("LGTM_INDEX_FILTERS", "exclude:**/vendor");
addFile(true, LGTM_SRC, "tst.js");
addFile(false, LGTM_SRC, "vendor", "leftpad", "index.js");
runTest();
}
@Test
public void excludeFolderByPattern2() throws IOException {
envVars.put("LGTM_INDEX_FILTERS", "exclude:*/**/vendor");
addFile(true, LGTM_SRC, "tst.js");
addFile(true, LGTM_SRC, "vendor", "dep", "index.js");
addFile(false, LGTM_SRC, "vendor", "dep", "vendor", "depdep", "index.js");
runTest();
}
@Test
public void excludeFolderByPattern3() throws IOException {
envVars.put("LGTM_INDEX_FILTERS", "exclude:**/vendor\n");
addFile(true, LGTM_SRC, "tst.js");
addFile(false, LGTM_SRC, "vendor", "leftpad", "index.js");
runTest();
}
@Test
public void excludeFolderByPatterns() throws IOException {
envVars.put("LGTM_INDEX_FILTERS", "exclude:foo\nexclude:**/vendor");
addFile(true, LGTM_SRC, "tst.js");
addFile(false, LGTM_SRC, "vendor", "leftpad", "index.js");
runTest();
}
@Test
public void excludeFolderByName() throws IOException {
envVars.put("LGTM_INDEX_EXCLUDE", "vendor");
addFile(true, LGTM_SRC, "tst.js");
addFile(false, LGTM_SRC, "vendor", "leftpad", "index.js");
runTest();
}
@Test
public void excludeFolderByName2() throws IOException {
envVars.put("LGTM_INDEX_EXCLUDE", "vendor\n");
addFile(true, LGTM_SRC, "tst.js");
addFile(false, LGTM_SRC, "vendor", "leftpad", "index.js");
runTest();
}
@Test
public void excludeFolderByName3() throws IOException {
envVars.put("LGTM_INDEX_EXCLUDE", "./vendor\n");
addFile(true, LGTM_SRC, "tst.js");
addFile(false, LGTM_SRC, "vendor", "leftpad", "index.js");
runTest();
}
@Test
public void excludeByExtension() throws IOException {
envVars.put("LGTM_INDEX_FILTERS", "exclude:**/*.js");
addFile(false, LGTM_SRC, "tst.js");
addFile(true, LGTM_SRC, "tst.html");
addFile(false, LGTM_SRC, "vendor", "leftpad", "index.js");
addFile(true, LGTM_SRC, "vendor", "leftpad", "index.html");
runTest();
}
@Test
public void includeByExtension() throws IOException {
envVars.put("LGTM_INDEX_FILTERS", "include:**/*.json");
addFile(true, LGTM_SRC, "tst.js");
addFile(true, LGTM_SRC, "tst.json");
addFile(true, LGTM_SRC, "vendor", "leftpad", "tst.json");
runTest();
}
@Test
public void includeByExtensionInRootOnly() throws IOException {
envVars.put("LGTM_INDEX_FILTERS", "include:*.json");
addFile(true, LGTM_SRC, "tst.js");
addFile(true, LGTM_SRC, "tst.json");
addFile(false, LGTM_SRC, "vendor", "leftpad", "tst.json");
runTest();
}
@Test
public void includeAndExclude() throws IOException {
envVars.put("LGTM_INDEX_FILTERS", "include:**/*.json\n" + "exclude:**/vendor");
addFile(true, LGTM_SRC, "tst.js");
addFile(true, LGTM_SRC, "tst.json");
addFile(false, LGTM_SRC, "vendor", "leftpad", "tst.json");
runTest();
}
@Test
public void excludeByClassification() throws IOException {
Path repositoryFolders = Files.createFile(SEMMLE_DIST.resolve("repositoryFolders.csv"));
List<String> csvLines = new ArrayList<>();
csvLines.add("classification,path");
csvLines.add("thirdparty," + LGTM_SRC.resolve("vendor"));
csvLines.add("external," + LGTM_SRC.resolve("foo").resolve("bar").toUri());
csvLines.add("metadata," + LGTM_SRC.resolve(".git"));
Files.write(repositoryFolders, csvLines, StandardCharsets.UTF_8);
envVars.put("LGTM_REPOSITORY_FOLDERS_CSV", repositoryFolders.toString());
addFile(true, LGTM_SRC, "tst.js");
addFile(false, LGTM_SRC, "foo", "bar", "tst.js");
addFile(false, LGTM_SRC, ".git", "tst.js");
addFile(true, LGTM_SRC, "vendor", "leftpad", "tst.js");
runTest();
}
@Test
public void excludeIncludeNested() throws IOException {
envVars.put("LGTM_INDEX_INCLUDE", ".\ntest/util");
envVars.put("LGTM_INDEX_EXCLUDE", "test\ntest/util/test");
addFile(true, LGTM_SRC, "index.js");
addFile(false, LGTM_SRC, "test", "tst.js");
addFile(false, LGTM_SRC, "test", "subtest", "tst.js");
addFile(true, LGTM_SRC, "test", "util", "util.js");
addFile(false, LGTM_SRC, "test", "util", "test", "utiltst.js");
runTest();
}
@Test
public void includeImplicitlyExcluded() throws IOException {
Path repositoryFolders = Files.createFile(SEMMLE_DIST.resolve("repositoryFolders.csv"));
List<String> csvLines = new ArrayList<>();
csvLines.add("classification,path");
csvLines.add("thirdparty," + LGTM_SRC.resolve("vendor"));
csvLines.add("external," + LGTM_SRC.resolve("foo").resolve("bar"));
csvLines.add("metadata," + LGTM_SRC.resolve(".git"));
Files.write(repositoryFolders, csvLines, StandardCharsets.UTF_8);
envVars.put("LGTM_REPOSITORY_FOLDERS_CSV", repositoryFolders.toString());
envVars.put("LGTM_INDEX_INCLUDE", ".\nfoo/bar");
addFile(true, LGTM_SRC, "tst.js");
addFile(true, LGTM_SRC, "foo", "bar", "tst.js");
addFile(false, LGTM_SRC, ".git", "tst.js");
addFile(true, LGTM_SRC, "vendor", "leftpad", "tst.js");
runTest();
}
/**
* Create a symbolic link from {@code $LGTM_SRC/link} to {@code target} and invoke {@code
* runTest}. Skip running the test if the symbolic link cannot be created.
*/
private void createSymlinkAndRunTest(String link, Path target) throws IOException {
createSymlinkAndRunTest(Paths.get(LGTM_SRC.toString(), link), target);
}
/**
* Create a symbolic link from {@code link} to {@code target} and invoke {@code runTest}. Skip
* running the test if the symbolic link cannot be created.
*/
private void createSymlinkAndRunTest(Path linkPath, Path target) throws IOException {
try {
Files.createSymbolicLink(linkPath, target);
} catch (UnsupportedOperationException | SecurityException | IOException e) {
Assume.assumeNoException("Cannot create symlinks.", e);
}
runTest();
}
@Test
public void symlinkFile() throws IOException {
Path tst_js = addFile(true, LGTM_SRC, "tst.js");
createSymlinkAndRunTest("tst_link.js", tst_js);
}
@Test
public void symlinkDir() throws IOException {
Path testFolder = Files.createTempDirectory("autobuild-test").toAbsolutePath();
try {
addFile(false, testFolder, "tst.js");
createSymlinkAndRunTest("test", testFolder);
} finally {
FileUtil8.recursiveDelete(testFolder);
}
}
@Test
public void deadSymlinkFile() throws IOException {
Path dead = Paths.get("i", "do", "not", "exist", "tst.js");
Assert.assertFalse(Files.exists(dead));
createSymlinkAndRunTest("tst_link.js", dead);
}
@Test
public void deadSymlinkDir() throws IOException {
Path dead = Paths.get("i", "do", "not", "exist");
Assert.assertFalse(Files.exists(dead));
createSymlinkAndRunTest("test", dead);
}
@Test
public void excludeByClassificationSymlink() throws IOException {
// check for robustness in case LGTM_SRC is canonicalised but repositoryFolders.csv is not
Path testFolder = Files.createTempDirectory("autobuild-test").toAbsolutePath();
Files.createDirectories(testFolder);
Path repositoryFolders = Files.createFile(SEMMLE_DIST.resolve("repositoryFolders.csv"));
List<String> csvLines = new ArrayList<>();
csvLines.add("classification,path");
csvLines.add("external," + testFolder.resolve("src").resolve("foo"));
Files.write(repositoryFolders, csvLines, StandardCharsets.UTF_8);
envVars.put("LGTM_REPOSITORY_FOLDERS_CSV", repositoryFolders.toString());
addFile(false, LGTM_SRC, "foo", "tst.js");
createSymlinkAndRunTest(testFolder.resolve("src"), LGTM_SRC);
FileUtil8.recursiveDelete(testFolder);
}
@Test
public void excludeByClassificationBadPath() throws IOException {
// check for robustness in case there are unresolvable paths in repositoryFolders.csv
Path testFolder = Files.createTempDirectory("autobuild-test").toAbsolutePath();
Files.createDirectories(testFolder);
Path repositoryFolders = Files.createFile(SEMMLE_DIST.resolve("repositoryFolders.csv"));
List<String> csvLines = new ArrayList<>();
csvLines.add("classification,path");
csvLines.add("external,no-such-path");
Files.write(repositoryFolders, csvLines, StandardCharsets.UTF_8);
envVars.put("LGTM_REPOSITORY_FOLDERS_CSV", repositoryFolders.toString());
addFile(true, LGTM_SRC, "tst.js");
runTest();
FileUtil8.recursiveDelete(testFolder);
}
/** Hide {@code p} on using {@link DosFileAttributeView} if available; otherwise do nothing. */
private void hide(Path p) throws IOException {
DosFileAttributeView attrs = Files.getFileAttributeView(p, DosFileAttributeView.class);
if (attrs != null) attrs.setHidden(true);
}
@Test
public void hiddenFolders() throws IOException {
Path tst_js = addFile(false, LGTM_SRC, ".DS_Store", "tst.js");
hide(tst_js.getParent());
runTest();
}
@Test
public void hiddenFiles() throws IOException {
Path eslintrc = addFile(true, LGTM_SRC, ".eslintrc.json");
hide(eslintrc);
runTest();
}
@Test
public void noTypescriptExtraction() throws IOException {
envVars.put("LGTM_INDEX_TYPESCRIPT", "none");
addFile(false, LGTM_SRC, "tst.ts");
addFile(false, LGTM_SRC, "sub.js", "tst.ts");
addFile(false, LGTM_SRC, "tst.js.ts");
runTest();
}
@Test
public void includeNonExistentFile() throws IOException {
envVars.put("LGTM_INDEX_INCLUDE", "tst.js");
addFile(false, LGTM_SRC, "vendor", "leftpad", "index.js");
runTest();
}
@Test
public void excludeNonExistentFile() throws IOException {
envVars.put("LGTM_INDEX_EXCLUDE", "vendor/leftpad/index.js");
addFile(true, LGTM_SRC, "tst.js");
runTest();
}
@Test
public void minifiedFilesAreExcluded() throws IOException {
addFile(true, LGTM_SRC, "admin.js");
addFile(false, LGTM_SRC, "jquery.min.js");
addFile(false, LGTM_SRC, "lib", "lodash-min.js");
addFile(true, LGTM_SRC, "compute_min.js");
runTest();
}
@Test
public void minifiedFilesCanBeReIncluded() throws IOException {
envVars.put("LGTM_INDEX_FILTERS", "include:**/*.min.js\ninclude:**/*-min.js");
addFile(true, LGTM_SRC, "admin.js");
addFile(true, LGTM_SRC, "jquery.min.js");
addFile(true, LGTM_SRC, "lib", "lodash-min.js");
addFile(true, LGTM_SRC, "compute_min.js");
runTest();
}
@Test
public void nodeModulesAreExcluded() throws IOException {
addFile(true, LGTM_SRC, "index.js");
addFile(false, LGTM_SRC, "node_modules", "dep", "main.js");
addFile(false, LGTM_SRC, "node_modules", "dep", "node_modules", "leftpad", "index.js");
runTest();
}
@Test
public void nodeModulesCanBeReincluded() throws IOException {
envVars.put("LGTM_INDEX_FILTERS", "include:**/node_modules");
addFile(true, LGTM_SRC, "index.js");
addFile(true, LGTM_SRC, "node_modules", "dep", "main.js");
addFile(true, LGTM_SRC, "node_modules", "dep", "node_modules", "leftpad", "index.js");
runTest();
}
@Test
public void bowerComponentsAreExcluded() throws IOException {
addFile(true, LGTM_SRC, "index.js");
addFile(false, LGTM_SRC, "bower_components", "dep", "main.js");
addFile(false, LGTM_SRC, "bower_components", "dep", "bower_components", "leftpad", "index.js");
runTest();
}
@Test
public void bowerComponentsCanBeReincluded() throws IOException {
envVars.put("LGTM_INDEX_FILTERS", "include:**/bower_components");
addFile(true, LGTM_SRC, "index.js");
addFile(true, LGTM_SRC, "bower_components", "dep", "main.js");
addFile(true, LGTM_SRC, "bower_components", "dep", "bower_components", "leftpad", "index.js");
runTest();
}
@Test
public void customExtensions() throws IOException {
envVars.put("LGTM_INDEX_FILETYPES", ".jsm:js\n.soy:html");
addFile(true, FileType.JS, LGTM_SRC, "tst.jsm");
addFile(false, LGTM_SRC, "tstjsm");
addFile(true, FileType.HTML, LGTM_SRC, "tst.soy");
addFile(true, LGTM_SRC, "tst.html");
addFile(true, LGTM_SRC, "tst.js");
runTest();
}
@Test
public void overrideExtension() throws IOException {
envVars.put("LGTM_INDEX_FILETYPES", ".js:typescript");
addFile(true, FileType.TYPESCRIPT, LGTM_SRC, "tst.js");
runTest();
}
@Test
public void invalidFileType() throws IOException {
envVars.put("LGTM_INDEX_FILETYPES", ".jsm:javascript");
try {
runTest();
Assert.fail("expected UserError");
} catch (UserError ue) {
Assert.assertEquals("Invalid file type 'JAVASCRIPT'.", ue.getMessage());
}
}
@Test
public void includeYaml() throws IOException {
addFile(true, LGTM_SRC, "tst.yaml");
addFile(true, LGTM_SRC, "tst.yml");
addFile(true, LGTM_SRC, "tst.raml");
runTest();
}
@Test
public void dontIncludeXmlByDefault() throws IOException {
addFile(false, LGTM_SRC, "tst.xml");
addFile(false, LGTM_SRC, "tst.qhelp");
runTest();
}
@Test
public void includeXml() throws IOException {
envVars.put("LGTM_INDEX_XML_MODE", "all");
addFile(true, LGTM_SRC, "tst.xml");
addFile(false, LGTM_SRC, "tst.qhelp");
runTest();
}
@Test
public void qhelpAsXml() throws IOException {
envVars.put("LGTM_INDEX_FILETYPES", ".qhelp:xml");
addFile(false, LGTM_SRC, "tst.xml");
addFile(true, LGTM_SRC, "tst.qhelp");
runTest();
}
@Test
public void qhelpAsXmlAndAllXml() throws IOException {
envVars.put("LGTM_INDEX_XML_MODE", "all");
envVars.put("LGTM_INDEX_FILETYPES", ".qhelp:xml");
addFile(true, LGTM_SRC, "tst.xml");
addFile(true, LGTM_SRC, "tst.qhelp");
runTest();
}
@Test
public void skipCodeQLDatabases() throws IOException {
addFile(true, LGTM_SRC, "tst.js");
addFile(false, LGTM_SRC, "db/codeql-database.yml");
addFile(false, LGTM_SRC, "db/foo.js");
runTest();
}
@Test
public void hiddenGitHubFoldersAreIncluded() throws IOException {
Path tst_yml = addFile(true, LGTM_SRC, ".github", "workflows", "test.yml");
hide(tst_yml.getParent().getParent());
runTest();
}
@Test
public void hiddenGitHubFoldersCanBeExcluded() throws IOException {
envVars.put("LGTM_INDEX_FILTERS", "exclude:**/.github");
Path tst_yml = addFile(false, LGTM_SRC, ".github", "workflows", "test.yml");
hide(tst_yml.getParent().getParent());
runTest();
}
}

46
javascript/extractor/src/com/semmle/js/extractor/test/ClassPropertiesTests.java
View File

@@ -1,46 +0,0 @@
package com.semmle.js.extractor.test;
import org.junit.Test;
/**
* Tests for parsing class properties.
*
* <p>Most tests are automatically derived from the Babylon test suite as described in <code>
* parser-tests/babylon/README.md</code>.
*/
public class ClassPropertiesTests extends ASTMatchingTests {
@Test
public void testASIFailureComputed() {
babylonTest("experimental/class-properties/asi-failure-computed");
}
@Test
public void testASIFailureGenerator() {
babylonTest("experimental/class-properties/asi-failure-generator");
}
@Test
public void testASISuccess() {
babylonTest("experimental/class-properties/asi-success");
}
@Test
public void test43() {
babylonTest("experimental/uncategorised/43");
}
@Test
public void test44() {
babylonTest("experimental/uncategorised/44");
}
@Test
public void test45() {
babylonTest("experimental/uncategorised/45");
}
@Test
public void test46() {
babylonTest("experimental/uncategorised/46");
}
}

71
javascript/extractor/src/com/semmle/js/extractor/test/DecoratorTests.java
View File

@@ -1,71 +0,0 @@
package com.semmle.js.extractor.test;
import org.junit.Test;
/**
* Tests for parsing decorators.
*
* <p>Most tests are automatically derived from the Babylon test suite as described in <code>
* parser-tests/babylon/README.md</code>.
*/
public class DecoratorTests extends ASTMatchingTests {
@Test
public void test33() {
babylonTest("experimental/uncategorised/33");
}
@Test
public void test34() {
babylonTest("experimental/uncategorised/34");
}
@Test
public void test35() {
babylonTest("experimental/uncategorised/35");
}
@Test
public void test36() {
babylonTest("experimental/uncategorised/36");
}
@Test
public void test37() {
babylonTest("experimental/uncategorised/37");
}
@Test
public void test38() {
babylonTest("experimental/uncategorised/38");
}
@Test
public void test39() {
babylonTest("experimental/uncategorised/39");
}
@Test
public void test40() {
babylonTest("experimental/uncategorised/40");
}
@Test
public void test41() {
babylonTest("experimental/uncategorised/41");
}
@Test
public void test42() {
babylonTest("experimental/uncategorised/42");
}
@Test
public void test49() {
babylonTest("experimental/uncategorised/49");
}
@Test
public void test62() {
babylonTest("experimental/uncategorised/62");
}
}

42
javascript/extractor/src/com/semmle/js/extractor/test/ExportExtensionsTests.java
View File

@@ -1,42 +0,0 @@
package com.semmle.js.extractor.test;
import com.semmle.jcorn.Options;
import org.junit.Test;
/**
* Tests for parsing export extensions.
*
* <p>Most tests are automatically derived from the Babylon test suite as described in <code>
* parser-tests/babylon/README.md</code>.
*/
public class ExportExtensionsTests extends ASTMatchingTests {
@Override
protected void babylonTest(String dir) {
babylonTest(dir, new Options().esnext(true).sourceType("module"));
}
@Test
public void test50() {
babylonTest("experimental/uncategorised/50");
}
@Test
public void test51() {
babylonTest("experimental/uncategorised/51");
}
@Test
public void test52() {
babylonTest("experimental/uncategorised/52");
}
@Test
public void test53() {
babylonTest("experimental/uncategorised/53");
}
@Test
public void test54() {
babylonTest("experimental/uncategorised/54");
}
}

26
javascript/extractor/src/com/semmle/js/extractor/test/FunctionSentTests.java
View File

@@ -1,26 +0,0 @@
package com.semmle.js.extractor.test;
import org.junit.Test;
/**
* Tests for parsing <code>function.sent</code>.
*
* <p>Most tests are automatically derived from the Babylon test suite as described in <code>
* parser-tests/babylon/README.md</code>.
*/
public class FunctionSentTests extends ASTMatchingTests {
@Test
public void testInsideFunction() {
babylonTest("experimental/function-sent/inside-function");
}
@Test
public void testInsideGenerator() {
babylonTest("experimental/function-sent/inside-generator");
}
@Test
public void testInvalidProperty() {
babylonTest("experimental/function-sent/invalid-property");
}
}

95
javascript/extractor/src/com/semmle/js/extractor/test/JSXTests.java
View File

@@ -1,95 +0,0 @@
package com.semmle.js.extractor.test;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.semmle.jcorn.SyntaxError;
import com.semmle.jcorn.jsx.JSXOptions;
import com.semmle.jcorn.jsx.JSXParser;
import com.semmle.js.ast.AST2JSON;
import com.semmle.js.ast.Program;
import com.semmle.util.files.FileUtil;
import com.semmle.util.io.WholeIO;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
/**
* Tests for the JSX parser, automatically generated from the acorn-jsx test suite as described in
* <code>parser-tests/jcorn-jsx/README.md</code>.
*/
@RunWith(Parameterized.class)
public class JSXTests extends ASTMatchingTests {
private static final File BASE = new File("parser-tests/jcorn-jsx").getAbsoluteFile();
@Parameters(name = "{0}")
public static Iterable<Object[]> tests() {
List<Object[]> testData = new ArrayList<Object[]>();
// iterate over all tests
for (File test : BASE.listFiles(FileUtil.extensionFilter(true, ".js"))) {
String testName = FileUtil.basename(test);
JSXOptions options = new JSXOptions().allowNamespacedObjects(false);
File optionsFile = new File(test.getParentFile(), testName + ".options.json");
if (optionsFile.exists()) {
JsonObject optionsJson =
new JsonParser().parse(new WholeIO().strictread(optionsFile)).getAsJsonObject();
if (optionsJson.has("allowNamespacedObjects"))
options =
options.allowNamespacedObjects(
optionsJson.get("allowNamespacedObjects").getAsBoolean());
if (optionsJson.has("allowNamespaces"))
options = options.allowNamespaces(optionsJson.get("allowNamespaces").getAsBoolean());
}
JsonObject expectedAST;
File expectedASTFile = new File(test.getParentFile(), testName + ".ast");
if (expectedASTFile.exists())
expectedAST =
new JsonParser().parse(new WholeIO().strictread(expectedASTFile)).getAsJsonObject();
else expectedAST = null;
String expectedFailure;
File expectedFailureFile = new File(test.getParentFile(), testName + ".fail");
if (expectedFailureFile.exists())
expectedFailure = new WholeIO().strictread(expectedFailureFile).trim();
else expectedFailure = null;
testData.add(new Object[] {testName, options, expectedAST, expectedFailure});
}
return testData;
}
private final String testName;
private final JSXOptions options;
private final JsonObject expectedAST;
private final String expectedFailure;
public JSXTests(
String testName, JSXOptions options, JsonObject expectedAST, String expectedFailure) {
this.testName = testName;
this.options = options;
this.expectedAST = expectedAST;
this.expectedFailure = expectedFailure;
}
@Test
public void runtest() {
File inputFile = new File(BASE, testName + ".js");
String input = new WholeIO().strictread(inputFile);
try {
Program actualProgram = new JSXParser(options, input, 0).parse();
JsonElement actualAST = AST2JSON.convert(actualProgram);
assertMatch("<root>", expectedAST, actualAST);
} catch (SyntaxError e) {
Assert.assertEquals(expectedFailure, e.getMessage());
}
}
}

214
javascript/extractor/src/com/semmle/js/extractor/test/NodeJSDetectorTests.java
View File

@@ -1,214 +0,0 @@
package com.semmle.js.extractor.test;
import com.semmle.js.ast.Node;
import com.semmle.js.extractor.ExtractionMetrics;
import com.semmle.js.extractor.ExtractorConfig;
import com.semmle.js.extractor.ExtractorConfig.SourceType;
import com.semmle.js.extractor.NodeJSDetector;
import com.semmle.js.parser.JSParser;
import com.semmle.js.parser.JSParser.Result;
import org.junit.Assert;
import org.junit.Test;
public class NodeJSDetectorTests {
private static final ExtractorConfig CONFIG = new ExtractorConfig(false);
private void isNodeJS(String src, boolean expected) {
Result res = JSParser.parse(CONFIG, SourceType.SCRIPT, src, new ExtractionMetrics());
Node ast = res.getAST();
Assert.assertNotNull(ast);
Assert.assertTrue(NodeJSDetector.looksLikeNodeJS(ast) == expected);
}
@Test
public void testBareRequire() {
isNodeJS("require('fs');", true);
}
@Test
public void testRequireInit() {
isNodeJS("var fs = require('fs');", true);
}
@Test
public void testRequireInit2() {
isNodeJS("var foo, fs = require('fs');", true);
}
@Test
public void testDirective() {
isNodeJS("\"use strict\"; require('fs');", true);
}
@Test
public void testComment() {
isNodeJS(
"/** My awesome package.\n"
+ "* (C) me.\n"
+ "*/\n"
+ "\n"
+ "var isArray = require(\"isArray\");",
true);
}
@Test
public void testInitialExport() {
isNodeJS("exports.foo = 0; console.log('Hello, world!');", true);
}
@Test
public void testInitialModuleExport() {
isNodeJS("module.exports.foo = 0; console.log('Hello, world!');", true);
}
@Test
public void testInitialBulkExport() {
isNodeJS("module.exports = {}; console.log('Hello, world!');", true);
}
@Test
public void testTrailingExport() {
isNodeJS("console.log('Hello, world!'); exports.foo = 0;", true);
}
@Test
public void testTrailingModuleExport() {
isNodeJS("console.log('Hello, world!'); module.exports.foo = 0;", true);
}
@Test
public void testTrailingBulkExport() {
isNodeJS("console.log('Hello, world!'); module.exports = {};", true);
}
@Test
public void testInitialNestedExport() {
isNodeJS("mystuff = module.exports = {}; mystuff.foo = 0;", true);
}
@Test
public void testInitialNestedExportInit() {
isNodeJS("var mystuff = module.exports = exports = {}; mystuff.foo = 0;", true);
}
@Test
public void testTrailingRequire() {
isNodeJS("console.log('Hello, world!'); var fs = require('fs');", true);
}
@Test
public void testSandwichedExport() {
isNodeJS("console.log('Hello'); exports.foo = 0; console.log('world!');", true);
}
@Test
public void umd() {
isNodeJS(
"(function(define) {\n"
+ " define(function (require, exports, module) {\n"
+ " var b = require('b');\n"
+ " return function () {};\n"
+ " });\n"
+ "}(\n"
+ " typeof module === 'object' && module.exports && typeof define !== 'function' ?\n"
+ " function (factory) { module.exports = factory(require, exports, module); } :\n"
+ " define\n"
+ "));",
false);
}
@Test
public void testRequirePropAccess() {
isNodeJS("var foo = require('./b').foo;", true);
}
@Test
public void testReExport() {
isNodeJS("module.exports = require('./e');", true);
}
@Test
public void testSeparateVar() {
isNodeJS("var fs; fs = require('fs');", true);
}
@Test
public void testLet() {
isNodeJS("let fs = require('fs');", true);
}
@Test
public void testSeparateLet() {
isNodeJS("let fs; fs = require('fs');", true);
}
@Test
public void testConst() {
isNodeJS("const fs = require('fs');", true);
}
@Test
public void testIife() {
isNodeJS(";(function() { require('fs'); })()", true);
}
@Test
public void testIife2() {
isNodeJS("!function() { require('fs'); }()", true);
}
@Test
public void testUMD() {
isNodeJS("(function(require) { require('fs'); })(myRequire);", false);
}
@Test
public void amdefine() {
isNodeJS(
"if (typeof define !== 'function') define = require('amdefine')(module, require);", true);
}
@Test
public void requireAndReadProp() {
isNodeJS("var readFileSync = require('fs').readFileSync;", true);
}
@Test
public void toplevelAMDRequire() {
isNodeJS("require(['foo'], function(foo) { });", false);
}
@Test
public void requireInTry() {
isNodeJS(
"var fs;"
+ "try {"
+ " fs = require('graceful-fs');"
+ "} catch(e) {"
+ " fs = require('fs');"
+ "}",
true);
}
@Test
public void requireInIf() {
isNodeJS(
"var fs;"
+ "if (useGracefulFs) {"
+ " fs = require('graceful-fs');"
+ "} else {"
+ " fs = require('fs');"
+ "}",
true);
}
@Test
public void requireAndCall() {
isNodeJS("var foo = require('foo')();", true);
}
@Test
public void requireAndCallMethod() {
isNodeJS("var foo = require('foo').bar();", true);
}
}

44
javascript/extractor/src/com/semmle/js/extractor/test/NumericSeparatorTests.java
View File

@@ -1,44 +0,0 @@
package com.semmle.js.extractor.test;
import static org.junit.Assert.*;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import com.semmle.jcorn.ESNextParser;
import com.semmle.jcorn.Options;
import com.semmle.jcorn.SyntaxError;
import com.semmle.js.ast.ExpressionStatement;
import com.semmle.js.ast.Literal;
import com.semmle.js.ast.Program;
import org.junit.Test;
public class NumericSeparatorTests {
private void test(String src, Integer numVal) {
try {
Program p = new ESNextParser(new Options().esnext(true), src, 0).parse();
assertNotNull(numVal);
assertEquals(1, p.getBody().size());
assertTrue(p.getBody().get(0) instanceof ExpressionStatement);
ExpressionStatement exprStmt = (ExpressionStatement) p.getBody().get(0);
assertTrue(exprStmt.getExpression() instanceof Literal);
assertEquals(numVal.longValue(), ((Literal) exprStmt.getExpression()).getValue());
} catch (SyntaxError e) {
assertNull(e.toString(), numVal);
}
}
@Test
public void test() {
test("0b_", null);
test("0b0_1", 0b01);
test("0B0_1", 0b01);
test("0b0_10", 0b010);
test("0B0_10", 0b010);
test("0b01_0", 0b010);
test("0B01_0", 0b010);
test("0b01_00", 0b0100);
test("0B01_00", 0b0100);
test("0b0__0", null);
test("0b0_", null);
}
}

59
javascript/extractor/src/com/semmle/js/extractor/test/ObjectRestSpreadTests.java
View File

@@ -1,59 +0,0 @@
package com.semmle.js.extractor.test;
import com.semmle.jcorn.CustomParser;
import com.semmle.jcorn.Options;
import com.semmle.jcorn.SyntaxError;
import org.junit.Assert;
import org.junit.Test;
/**
* Tests for parsing rest/spread properties.
*
* <p>Most tests are automatically derived from the Babylon test suite as described in <code>
* parser-tests/babylon/README.md</code>.
*/
public class ObjectRestSpreadTests extends ASTMatchingTests {
private void testFail(String input, String msg) {
try {
new CustomParser(new Options().esnext(true), input, 0).parse();
Assert.fail("Expected syntax error, but parsing succeeded.");
} catch (SyntaxError e) {
Assert.assertEquals(msg, e.getMessage());
}
}
@Test
public void test9() {
babylonTest("experimental/uncategorised/9");
}
@Test
public void test10() {
babylonTest("experimental/uncategorised/10");
}
@Test
public void test11() {
babylonTest("experimental/uncategorised/11");
}
@Test
public void test12() {
babylonTest("experimental/uncategorised/12");
}
@Test
public void test13() {
babylonTest("experimental/uncategorised/13");
}
@Test
public void testObjectRestSpread() {
babylonTest("harmony/arrow-functions/object-rest-spread");
}
@Test
public void testFail() {
testFail("({...'hi'} = {})", "Assigning to rvalue (1:5)");
}
}

18
javascript/extractor/src/com/semmle/js/extractor/test/RobustnessTests.java
View File

@@ -1,18 +0,0 @@
package com.semmle.js.extractor.test;
import com.semmle.jcorn.Options;
import com.semmle.jcorn.Parser;
import com.semmle.util.io.WholeIO;
import java.io.File;
import java.nio.charset.StandardCharsets;
import org.junit.Test;
public class RobustnessTests {
@Test
public void letLookheadTest() {
File test = new File("parser-tests/robustness/letLookahead.js");
String src = new WholeIO(StandardCharsets.UTF_8.name()).strictread(test);
new Parser(new Options(), src, 0).parse();
}
}

195
javascript/extractor/src/com/semmle/js/extractor/test/TrapTests.java
View File

@@ -1,195 +0,0 @@
package com.semmle.js.extractor.test;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.gson.JsonPrimitive;
import com.semmle.js.extractor.ExtractorState;
import com.semmle.js.extractor.Main;
import com.semmle.util.data.Pair;
import com.semmle.util.data.StringUtil;
import com.semmle.util.extraction.ExtractorOutputConfig;
import com.semmle.util.io.WholeIO;
import com.semmle.util.process.Env;
import com.semmle.util.srcarchive.DummySourceArchive;
import com.semmle.util.trap.ITrapWriterFactory;
import com.semmle.util.trap.TrapWriter;
import com.semmle.util.trap.pathtransformers.ProjectLayoutTransformer;
import java.io.File;
import java.io.StringWriter;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map.Entry;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
@RunWith(Parameterized.class)
public class TrapTests {
private static final File BASE = new File("tests").getAbsoluteFile();
@Parameters(name = "{0}:{1}")
public static Iterable<Object[]> tests() {
List<Object[]> testData = new ArrayList<Object[]>();
// iterate over all test groups
List<String> testGroups = Arrays.asList(BASE.list());
testGroups.sort(Comparator.naturalOrder());
for (String testgroup : testGroups) {
File root = new File(BASE, testgroup);
if (root.isDirectory()) {
// check for options.json file and process it if it exists
List<String> options = new ArrayList<String>();
File optionsFile = new File(root, "options.json");
if (optionsFile.exists()) {
JsonParser p = new JsonParser();
JsonObject jsonOpts = p.parse(new WholeIO().strictread(optionsFile)).getAsJsonObject();
for (Entry<String, JsonElement> e : jsonOpts.entrySet()) {
JsonElement v = e.getValue();
if (v instanceof JsonPrimitive) {
JsonPrimitive pv = (JsonPrimitive) v;
if (pv.isBoolean() && pv.getAsBoolean()) {
options.add("--" + e.getKey());
} else {
options.add("--" + e.getKey());
options.add(pv.getAsString());
}
} else {
Assert.assertTrue(v instanceof JsonArray);
JsonArray a = (JsonArray) v;
for (JsonElement elt : a) {
options.add("--" + e.getKey());
options.add(elt.getAsString());
}
}
}
}
File inputDir = new File(root, "input");
File jsconfigFile = new File(inputDir, "tsconfig.json");
if (jsconfigFile.exists()) {
// create a single test with all files in the group
testData.add(new Object[] {testgroup, "tsconfig", new ArrayList<String>(options)});
} else {
// create isolated tests for each input file in the group
List<String> tests = Arrays.asList(inputDir.list());
tests.sort(Comparator.naturalOrder());
for (String testfile : tests) {
testData.add(new Object[] {testgroup, testfile, new ArrayList<String>(options)});
}
}
}
}
return testData;
}
private final String testgroup, testname;
private final List<String> options;
private static String userDir;
private static ExtractorState state;
public TrapTests(String testgroup, String testname, List<String> options) {
this.testgroup = testgroup;
this.testname = testname;
this.options = options;
}
@BeforeClass
public static void saveUserDir() {
userDir = System.getProperty("user.dir");
}
@BeforeClass
public static void setupState() {
state = new ExtractorState();
}
@AfterClass
public static void restoreUserDir() {
System.setProperty("user.dir", userDir);
}
@Test
public void runtest() {
state.reset();
File testdir = new File(BASE, testgroup);
File inputDir = new File(testdir, "input");
File inputFile = new File(inputDir, testname);
final File outputDir = new File(testdir, "output/trap");
System.setProperty("user.dir", inputFile.getParent());
options.add("--extract-program-text");
options.add("--quiet");
if (new File(inputDir, "tsconfig.json").exists()) {
// Use full extractor on entire directory.
options.add("--typescript-full");
options.add(inputDir.getAbsolutePath());
} else {
// Use basic extractor on a single file.
options.add("--typescript");
options.add(inputFile.getAbsolutePath());
}
final List<Pair<String, String>> expectedVsActual = new ArrayList<Pair<String, String>>();
Main main =
new Main(
new ExtractorOutputConfig(
new ITrapWriterFactory() {
@Override
public TrapWriter mkTrapWriter(final File f) {
final StringWriter sw = new StringWriter();
ProjectLayoutTransformer transformer =
new ProjectLayoutTransformer(new File(BASE, "project-layout"));
return new TrapWriter(sw, transformer) {
@Override
public void close() {
super.close();
// convert to and from UTF-8 to mimick treatment of unencodable characters
byte[] actual_utf8_bytes = StringUtil.stringToBytes(sw.toString());
String actual = new String(actual_utf8_bytes, Charset.forName("UTF-8"));
File trap = new File(outputDir, f.getName() + ".trap");
boolean replaceExpectedOutput = false;
if (replaceExpectedOutput) {
System.out.println("Replacing expected output for " + trap);
new WholeIO().strictwrite(trap, actual);
}
String expected = new WholeIO().strictreadText(trap);
expectedVsActual.add(Pair.make(expected, actual));
}
@Override
public void addTuple(String tableName, Object... values) {
if ("extraction_data".equals(tableName)
|| "extraction_time".equals(tableName)) {
// ignore non-deterministic tables
return;
}
super.addTuple(tableName, values);
}
};
}
@Override
public File getTrapFileFor(File f) {
return null;
}
},
new DummySourceArchive()),
state);
main.run(options.toArray(new String[options.size()]));
for (Pair<String, String> p : expectedVsActual) Assert.assertEquals(p.fst(), p.snd());
}
}

6
javascript/extractor/src/com/semmle/js/parser/JSONParser.java
View File

@@ -79,7 +79,7 @@ public class JSONParser {
}
private char peek() {
return offset < length ? src.charAt(offset) : (char) -1;
return offset < length ? src.charAt(offset) : Character.MAX_VALUE;
}
private JSONValue readValue() throws ParseError {
@@ -356,7 +356,7 @@ public class JSONParser {
char c;
next();
next();
while ((c = peek()) != '\r' && c != '\n' && c != -1) next();
while ((c = peek()) != '\r' && c != '\n' && c != Character.MAX_VALUE) next();
}
/** Skips the block comment starting at the current position. */
@@ -367,7 +367,7 @@ public class JSONParser {
next();
do {
c = peek();
if (c < 0) raise("Unterminated comment.");
if (c == Character.MAX_VALUE) raise("Unterminated comment");
next();
if (c == '*' && peek() == '/') {
next();