hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-01-25 04:12:58 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1872 from esben-semmle/js/extraction_metrics

Browse Source 
Approved by xiemaisi
This commit is contained in:
semmle-qlci
2019-09-09 10:45:33 +01:00
committed by GitHub
parent 77d7db323d 2a22471975
commit 57afde0240
42 changed files with 3447 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
javascript/extractor/src/com/semmle/js/ast/DefaultVisitor.java
View File

@@ -35,7 +35,6 @@ import com.semmle.ts.ast.InterfaceDeclaration;
import com.semmle.ts.ast.InterfaceTypeExpr;
import com.semmle.ts.ast.IntersectionTypeExpr;
import com.semmle.ts.ast.IsTypeExpr;
import com.semmle.ts.ast.UnaryTypeExpr;
import com.semmle.ts.ast.KeywordTypeExpr;
import com.semmle.ts.ast.MappedTypeExpr;
import com.semmle.ts.ast.NamespaceDeclaration;
@@ -49,6 +48,7 @@ import com.semmle.ts.ast.TypeAssertion;
import com.semmle.ts.ast.TypeExpression;
import com.semmle.ts.ast.TypeParameter;
import com.semmle.ts.ast.TypeofTypeExpr;
import com.semmle.ts.ast.UnaryTypeExpr;
import com.semmle.ts.ast.UnionTypeExpr;
import com.semmle.util.exception.CatastrophicError;

2
javascript/extractor/src/com/semmle/js/ast/Visitor.java
View File

@@ -31,7 +31,6 @@ import com.semmle.ts.ast.InterfaceDeclaration;
import com.semmle.ts.ast.InterfaceTypeExpr;
import com.semmle.ts.ast.IntersectionTypeExpr;
import com.semmle.ts.ast.IsTypeExpr;
import com.semmle.ts.ast.UnaryTypeExpr;
import com.semmle.ts.ast.KeywordTypeExpr;
import com.semmle.ts.ast.MappedTypeExpr;
import com.semmle.ts.ast.NamespaceDeclaration;
@@ -44,6 +43,7 @@ import com.semmle.ts.ast.TypeAliasDeclaration;
import com.semmle.ts.ast.TypeAssertion;
import com.semmle.ts.ast.TypeParameter;
import com.semmle.ts.ast.TypeofTypeExpr;
import com.semmle.ts.ast.UnaryTypeExpr;
import com.semmle.ts.ast.UnionTypeExpr;
/**

18
javascript/extractor/src/com/semmle/js/extractor/ASTExtractor.java
View File

@@ -1,11 +1,5 @@
package com.semmle.js.extractor;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.Stack;
import com.semmle.js.ast.AClass;
import com.semmle.js.ast.AFunction;
import com.semmle.js.ast.AFunctionExpression;
@@ -104,6 +98,7 @@ import com.semmle.js.ast.jsx.JSXMemberExpression;
import com.semmle.js.ast.jsx.JSXNamespacedName;
import com.semmle.js.ast.jsx.JSXOpeningElement;
import com.semmle.js.ast.jsx.JSXSpreadAttribute;
import com.semmle.js.extractor.ExtractionMetrics.ExtractionPhase;
import com.semmle.js.extractor.ExtractorConfig.Platform;
import com.semmle.js.extractor.ExtractorConfig.SourceType;
import com.semmle.js.extractor.ScopeManager.DeclKind;
@@ -149,6 +144,11 @@ import com.semmle.ts.ast.UnionTypeExpr;
import com.semmle.util.collections.CollectionUtil;
import com.semmle.util.trap.TrapWriter;
import com.semmle.util.trap.TrapWriter.Label;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.Stack;
/** Extractor for AST-based information; invoked by the {@link JSExtractor}. */
public class ASTExtractor {
@@ -193,6 +193,10 @@ public class ASTExtractor {
return scopeManager;
}
public ExtractionMetrics getMetrics() {
return lexicalExtractor.getMetrics();
}
/**
* The binding semantics for an identifier.
*
@@ -1946,9 +1950,11 @@ public class ASTExtractor {
}
public void extract(Node root, Platform platform, SourceType sourceType, int toplevelKind) {
lexicalExtractor.getMetrics().startPhase(ExtractionPhase.ASTExtractor_extract);
trapwriter.addTuple("toplevels", toplevelLabel, toplevelKind);
locationManager.emitNodeLocation(root, toplevelLabel);
root.accept(new V(platform, sourceType), null);
lexicalExtractor.getMetrics().stopPhase(ExtractionPhase.ASTExtractor_extract);
}
}

63
javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java
View File

@@ -1,5 +1,26 @@
package com.semmle.js.extractor;
import com.semmle.js.extractor.ExtractorConfig.SourceType;
import com.semmle.js.extractor.FileExtractor.FileType;
import com.semmle.js.extractor.trapcache.DefaultTrapCache;
import com.semmle.js.extractor.trapcache.DummyTrapCache;
import com.semmle.js.extractor.trapcache.ITrapCache;
import com.semmle.js.parser.ParsedProject;
import com.semmle.js.parser.TypeScriptParser;
import com.semmle.ts.extractor.TypeExtractor;
import com.semmle.ts.extractor.TypeTable;
import com.semmle.util.data.StringUtil;
import com.semmle.util.exception.CatastrophicError;
import com.semmle.util.exception.Exceptions;
import com.semmle.util.exception.ResourceError;
import com.semmle.util.exception.UserError;
import com.semmle.util.extraction.ExtractorOutputConfig;
import com.semmle.util.files.FileUtil;
import com.semmle.util.io.csv.CSVReader;
import com.semmle.util.language.LegacyLanguage;
import com.semmle.util.process.Env;
import com.semmle.util.projectstructure.ProjectLayout;
import com.semmle.util.trap.TrapWriter;
import java.io.File;
import java.io.IOException;
import java.io.Reader;
@@ -27,28 +48,6 @@ import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.stream.Stream;
import com.semmle.js.extractor.ExtractorConfig.SourceType;
import com.semmle.js.extractor.FileExtractor.FileType;
import com.semmle.js.extractor.trapcache.DefaultTrapCache;
import com.semmle.js.extractor.trapcache.DummyTrapCache;
import com.semmle.js.extractor.trapcache.ITrapCache;
import com.semmle.js.parser.ParsedProject;
import com.semmle.js.parser.TypeScriptParser;
import com.semmle.ts.extractor.TypeExtractor;
import com.semmle.ts.extractor.TypeTable;
import com.semmle.util.data.StringUtil;
import com.semmle.util.exception.CatastrophicError;
import com.semmle.util.exception.Exceptions;
import com.semmle.util.exception.ResourceError;
import com.semmle.util.exception.UserError;
import com.semmle.util.extraction.ExtractorOutputConfig;
import com.semmle.util.files.FileUtil;
import com.semmle.util.io.csv.CSVReader;
import com.semmle.util.language.LegacyLanguage;
import com.semmle.util.process.Env;
import com.semmle.util.projectstructure.ProjectLayout;
import com.semmle.util.trap.TrapWriter;
/**
* An alternative entry point to the JavaScript extractor.
*
@@ -71,8 +70,8 @@ import com.semmle.util.trap.TrapWriter;
* patterns that can be used to refine the list of files to include and exclude
* <li><code>LGTM_INDEX_TYPESCRIPT</code>: whether to extract TypeScript
* <li><code>LGTM_INDEX_FILETYPES</code>: a newline-separated list of ".extension:filetype" pairs
* specifying which {@link FileType} to use for the given extension; the additional file
* type <code>XML</code> is also supported
* specifying which {@link FileType} to use for the given extension; the additional file type
* <code>XML</code> is also supported
* <li><code>LGTM_INDEX_XML_MODE</code>: whether to extract XML files
* <li><code>LGTM_THREADS</code>: the maximum number of files to extract in parallel
* <li><code>LGTM_TRAP_CACHE</code>: the path of a directory to use for trap caching
@@ -166,8 +165,8 @@ import com.semmle.util.trap.TrapWriter;
* <p>If <code>LGTM_INDEX_XML_MODE</code> is set to <code>ALL</code>, then all files with extension
* <code>.xml</code> under <code>LGTM_SRC</code> are extracted as XML (in addition to any files
* whose file type is specified to be <code>XML</code> via <code>LGTM_INDEX_SOURCE_TYPE</code>).
* Currently XML extraction does not respect inclusion and exclusion filters, but this is a bug,
* not a feature, and hence will change eventually.
* Currently XML extraction does not respect inclusion and exclusion filters, but this is a bug, not
* a feature, and hence will change eventually.
*
* <p>Note that all these customisations only apply to <code>LGTM_SRC</code>. Extraction of externs
* is not customisable.
@@ -288,8 +287,7 @@ public class AutoBuild {
try {
fileType = StringUtil.uc(fileType);
if ("XML".equals(fileType)) {
if (extension.length() < 2)
throw new UserError("Invalid extension '" + extension + "'.");
if (extension.length() < 2) throw new UserError("Invalid extension '" + extension + "'.");
xmlExtensions.add(extension.substring(1));
} else {
fileTypes.put(extension, FileType.valueOf(fileType));
@@ -304,8 +302,7 @@ public class AutoBuild {
private void setupXmlMode() {
String xmlMode = getEnvVar("LGTM_INDEX_XML_MODE", "DISABLED");
xmlMode = StringUtil.uc(xmlMode.trim());
if ("ALL".equals(xmlMode))
xmlExtensions.add("xml");
if ("ALL".equals(xmlMode)) xmlExtensions.add("xml");
else if (!"DISABLED".equals(xmlMode))
throw new UserError("Invalid XML mode '" + xmlMode + "' (should be either ALL or DISABLED).");
}
@@ -744,8 +741,7 @@ public class AutoBuild {
try {
long start = logBeginProcess("Extracting " + file);
Integer loc = extractor.extract(f, state);
if (!extractor.getConfig().isExterns() && (loc == null || loc != 0))
seenCode = true;
if (!extractor.getConfig().isExterns() && (loc == null || loc != 0)) seenCode = true;
logEndProcess(start, "Done extracting " + file);
} catch (Throwable t) {
System.err.println("Exception while extracting " + file + ".");
@@ -776,8 +772,7 @@ public class AutoBuild {
}
protected void extractXml() throws IOException {
if (xmlExtensions.isEmpty())
return;
if (xmlExtensions.isEmpty()) return;
List<String> cmd = new ArrayList<>();
cmd.add("odasa");
cmd.add("index");

5
javascript/extractor/src/com/semmle/js/extractor/CFGExtractor.java
View File

@@ -93,6 +93,7 @@ import com.semmle.js.ast.jsx.JSXMemberExpression;
import com.semmle.js.ast.jsx.JSXNamespacedName;
import com.semmle.js.ast.jsx.JSXOpeningElement;
import com.semmle.js.ast.jsx.JSXSpreadAttribute;
import com.semmle.js.extractor.ExtractionMetrics.ExtractionPhase;
import com.semmle.ts.ast.DecoratorList;
import com.semmle.ts.ast.EnumDeclaration;
import com.semmle.ts.ast.EnumMember;
@@ -171,11 +172,13 @@ public class CFGExtractor {
private final TrapWriter trapwriter;
private final Label toplevelLabel;
private final LocationManager locationManager;
private final ExtractionMetrics metrics;
public CFGExtractor(ASTExtractor astExtractor) {
this.trapwriter = astExtractor.getTrapwriter();
this.toplevelLabel = astExtractor.getToplevelLabel();
this.locationManager = astExtractor.getLocationManager();
this.metrics = astExtractor.getMetrics();
}
@SuppressWarnings("unchecked")
@@ -1955,6 +1958,8 @@ public class CFGExtractor {
}
public void extract(Node nd) {
metrics.startPhase(ExtractionPhase.CFGExtractor_extract);
nd.accept(new V(), new SimpleSuccessorInfo(null));
metrics.stopPhase(ExtractionPhase.CFGExtractor_extract);
}
}

167
javascript/extractor/src/com/semmle/js/extractor/ExtractionMetrics.java Normal file
View File

@@ -0,0 +1,167 @@
package com.semmle.js.extractor;
import com.semmle.util.trap.TrapWriter;
import com.semmle.util.trap.TrapWriter.Label;
import java.io.File;
import java.lang.management.ManagementFactory;
import java.lang.management.ThreadMXBean;
import java.util.Stack;
/** Metrics for the (single-threaded) extraction of a single file. */
public class ExtractionMetrics {
/**
* The phase of the extraction that should be measured time for.
*
* <p>Convention: the enum names have the format <code>{ClassName}_{MethodName}</code>, and should
* identify the methods they correspond to.
*/
public enum ExtractionPhase {
ASTExtractor_extract(0),
CFGExtractor_extract(1),
FileExtractor_extractContents(2),
JSExtractor_extract(3),
JSParser_parse(4),
LexicalExtractor_extractLines(5),
LexicalExtractor_extractTokens(6),
TypeScriptASTConverter_convertAST(7),
TypeScriptParser_talkToParserWrapper(8);
/** The id used in the database for the time spent performing this phase of the extraction. */
final int dbschemeId;
ExtractionPhase(int dbschemeId) {
this.dbschemeId = dbschemeId;
}
}
/** The cache file, if any. */
private File cacheFile;
/** True iff the extraction of this file reuses an existing trap cache file. */
private boolean canReuseCacheFile;
/** The cumulative CPU-time spent in each extraction phase so far. */
private final long[] cpuTimes = new long[ExtractionPhase.values().length];
/** The label for the file that is being extracted. */
private Label fileLabel;
/** The number of UTF16 code units in the file that is being extracted. */
private int length;
/** The previous time a CPU-time measure was performed. */
private long previousCpuTime;
/** The previous time a wallclock-time measure was performed. */
private long previousWallclockTime;
/** The extraction phase stack. */
private final Stack<ExtractionPhase> stack = new Stack<>();
/** The current thread, used for measuring CPU-time. */
private final ThreadMXBean thread = ManagementFactory.getThreadMXBean();
/** The cumulative wallclock-time spent in each extraction phase so far. */
private final long[] wallclockTimes = new long[ExtractionPhase.values().length];
/**
* True iff extraction metrics could not be obtained for this file (due to an unforeseen error
* that should not prevent the ordinary extraction from succeeding).
*/
private boolean timingsFailed;
/**
* Writes the data metrics to a trap file. Note that this makes the resulting trap file content
* non-deterministic.
*/
public void writeDataToTrap(TrapWriter trapwriter) {
trapwriter.addTuple(
"extraction_data",
fileLabel,
cacheFile != null ? cacheFile.getAbsolutePath() : "",
canReuseCacheFile,
length);
}
/**
* Writes the timing metrics to a trap file. Note that this makes the resulting trap file content
* non-deterministic.
*/
public void writeTimingsToTrap(TrapWriter trapwriter) {
if (!stack.isEmpty()) {
failTimings(
String.format(
"Could not properly record extraction times for %s. (stack = %s)%n",
fileLabel, stack.toString()));
}
if (!timingsFailed) {
for (int i = 0; i < ExtractionPhase.values().length; i++) {
trapwriter.addTuple("extraction_time", fileLabel, i, 0, (float) cpuTimes[i]);
trapwriter.addTuple("extraction_time", fileLabel, i, 1, (float) wallclockTimes[i]);
}
}
}
private void failTimings(String msg) {
System.err.println(msg);
System.err.flush();
this.timingsFailed = true;
}
private void incrementCurrentTimer() {
long nowWallclock = System.nanoTime();
long nowCpu = thread.getCurrentThreadCpuTime();
if (!stack.isEmpty()) {
// increment by the time elapsed
wallclockTimes[stack.peek().dbschemeId] += nowWallclock - previousWallclockTime;
cpuTimes[stack.peek().dbschemeId] += nowCpu - previousCpuTime;
}
// update the running clock
previousWallclockTime = nowWallclock;
previousCpuTime = nowCpu;
}
public void setCacheFile(File cacheFile) {
this.cacheFile = cacheFile;
}
public void setCanReuseCacheFile(boolean canReuseCacheFile) {
this.canReuseCacheFile = canReuseCacheFile;
}
public void setFileLabel(Label fileLabel) {
this.fileLabel = fileLabel;
}
public void setLength(int length) {
this.length = length;
}
public void startPhase(ExtractionPhase event) {
incrementCurrentTimer();
stack.push(event);
}
public void stopPhase(
ExtractionPhase
event /* technically not needed, but useful for documentation and sanity checking */) {
if (stack.isEmpty()) {
failTimings(
String.format(
"Inconsistent extraction time recording: trying to stop timer %s, but no timer is running",
event));
return;
}
if (stack.peek() != event) {
failTimings(
String.format(
"Inconsistent extraction time recording: trying to stop timer %s, but current timer is: %s",
event, stack.peek()));
return;
}
incrementCurrentTimer();
stack.pop();
}
}

35
javascript/extractor/src/com/semmle/js/extractor/FileExtractor.java
View File

@@ -1,5 +1,6 @@
package com.semmle.js.extractor;
import com.semmle.js.extractor.ExtractionMetrics.ExtractionPhase;
import com.semmle.js.extractor.trapcache.CachingTrapWriter;
import com.semmle.js.extractor.trapcache.ITrapCache;
import com.semmle.util.data.StringUtil;
@@ -384,10 +385,9 @@ public class FileExtractor {
return config.hasFileType() || FileType.forFile(f, config) != null;
}
/**
* @return the number of lines of code extracted, or {@code null} if the file was cached
*/
/** @return the number of lines of code extracted, or {@code null} if the file was cached */
public Integer extract(File f, ExtractorState state) throws IOException {
// populate source archive
String source = new WholeIO(config.getDefaultEncoding()).strictread(f);
outputConfig.getSourceArchive().add(f, source);
@@ -395,6 +395,7 @@ public class FileExtractor {
// extract language-independent bits
TrapWriter trapwriter = outputConfig.getTrapWriterFactory().mkTrapWriter(f);
Label fileLabel = trapwriter.populateFile(f);
LocationManager locationManager = new LocationManager(f, trapwriter, fileLabel);
locationManager.emitFileLocation(fileLabel, 0, 0, 0, 0);
@@ -426,22 +427,34 @@ public class FileExtractor {
private Integer extractContents(
File f, Label fileLabel, String source, LocationManager locationManager, ExtractorState state)
throws IOException {
ExtractionMetrics metrics = new ExtractionMetrics();
metrics.startPhase(ExtractionPhase.FileExtractor_extractContents);
metrics.setLength(source.length());
metrics.setFileLabel(fileLabel);
TrapWriter trapwriter = locationManager.getTrapWriter();
FileType fileType = getFileType(f);
File cacheFile = null, // the cache file for this extraction
resultFile = null; // the final result TRAP file for this extraction
// check whether we can perform caching
if (bumpIdCounter(trapwriter) && fileType.isTrapCachingAllowed()) {
if (bumpIdCounter(trapwriter)) {
resultFile = outputConfig.getTrapWriterFactory().getTrapFileFor(f);
if (resultFile != null) cacheFile = trapCache.lookup(source, config, fileType);
}
// check whether we can perform caching
if (resultFile != null && fileType.isTrapCachingAllowed()) {
cacheFile = trapCache.lookup(source, config, fileType);
}
if (cacheFile != null) {
boolean canUseCacheFile = cacheFile != null;
boolean canReuseCacheFile = canUseCacheFile && cacheFile.exists();
metrics.setCacheFile(cacheFile);
metrics.setCanReuseCacheFile(canReuseCacheFile);
metrics.writeDataToTrap(trapwriter);
if (canUseCacheFile) {
FileUtil.close(trapwriter);
if (cacheFile.exists()) {
if (canReuseCacheFile) {
FileUtil.append(cacheFile, resultFile);
return null;
}
@@ -459,18 +472,20 @@ public class FileExtractor {
try {
IExtractor extractor = fileType.mkExtractor(config, state);
TextualExtractor textualExtractor =
new TextualExtractor(trapwriter, locationManager, source, config.getExtractLines());
new TextualExtractor(
trapwriter, locationManager, source, config.getExtractLines(), metrics);
LoCInfo loc = extractor.extract(textualExtractor);
int numLines = textualExtractor.getNumLines();
int linesOfCode = loc.getLinesOfCode(), linesOfComments = loc.getLinesOfComments();
trapwriter.addTuple("numlines", fileLabel, numLines, linesOfCode, linesOfComments);
trapwriter.addTuple("filetype", fileLabel, fileType.toString());
metrics.stopPhase(ExtractionPhase.FileExtractor_extractContents);
metrics.writeTimingsToTrap(trapwriter);
successful = true;
return linesOfCode;
} finally {
if (!successful && trapwriter instanceof CachingTrapWriter)
((CachingTrapWriter) trapwriter).discard();
FileUtil.close(trapwriter);
}
}

7
javascript/extractor/src/com/semmle/js/extractor/HTMLExtractor.java
View File

@@ -191,7 +191,12 @@ public class HTMLExtractor implements IExtractor {
JSExtractor extractor = new JSExtractor(config);
try {
TextualExtractor tx =
new TextualExtractor(trapwriter, scriptLocationManager, source, config.getExtractLines());
new TextualExtractor(
trapwriter,
scriptLocationManager,
source,
config.getExtractLines(),
textualExtractor.getMetrics());
return extractor.extract(tx, source, toplevelKind, scopeManager).snd();
} catch (ParseError e) {
e.setPosition(scriptLocationManager.translatePosition(e.getPosition()));

9
javascript/extractor/src/com/semmle/js/extractor/JSExtractor.java
View File

@@ -3,6 +3,7 @@ package com.semmle.js.extractor;
import com.semmle.js.ast.Comment;
import com.semmle.js.ast.Node;
import com.semmle.js.ast.Token;
import com.semmle.js.extractor.ExtractionMetrics.ExtractionPhase;
import com.semmle.js.extractor.ExtractorConfig.ECMAVersion;
import com.semmle.js.extractor.ExtractorConfig.Platform;
import com.semmle.js.extractor.ExtractorConfig.SourceType;
@@ -52,7 +53,8 @@ public class JSExtractor {
SourceType sourceType = establishSourceType(source, true);
JSParser.Result parserRes = JSParser.parse(config, sourceType, source);
JSParser.Result parserRes =
JSParser.parse(config, sourceType, source, textualExtractor.getMetrics());
return extract(textualExtractor, source, toplevelKind, scopeManager, sourceType, parserRes);
}
@@ -87,6 +89,7 @@ public class JSExtractor {
SourceType sourceType,
JSParser.Result parserRes)
throws ParseError {
textualExtractor.getMetrics().startPhase(ExtractionPhase.JSExtractor_extract);
Label toplevelLabel;
TrapWriter trapwriter = textualExtractor.getTrapwriter();
LocationManager locationManager = textualExtractor.getLocationManager();
@@ -104,7 +107,6 @@ public class JSExtractor {
new LexicalExtractor(textualExtractor, parserRes.getTokens(), parserRes.getComments());
ASTExtractor scriptExtractor = new ASTExtractor(lexicalExtractor, scopeManager);
toplevelLabel = scriptExtractor.getToplevelLabel();
lexicalExtractor.extractComments(toplevelLabel);
loc = lexicalExtractor.extractLines(parserRes.getSource(), toplevelLabel);
lexicalExtractor.extractTokens(toplevelLabel);
@@ -126,7 +128,6 @@ public class JSExtractor {
for (ParseError parseError : parserRes.getErrors()) {
if (!config.isTolerateParseErrors()) throw parseError;
Label key = trapwriter.freshLabel();
String errorLine = textualExtractor.getLine(parseError.getPosition().getLine());
trapwriter.addTuple("jsParseErrors", key, toplevelLabel, "Error: " + parseError, errorLine);
@@ -139,6 +140,8 @@ public class JSExtractor {
if (platform == Platform.NODE && sourceType == SourceType.COMMONJS_MODULE)
textualExtractor.getTrapwriter().addTuple("isNodejs", toplevelLabel);
textualExtractor.getMetrics().stopPhase(ExtractionPhase.JSExtractor_extract);
return Pair.make(toplevelLabel, loc);
}

13
javascript/extractor/src/com/semmle/js/extractor/LexicalExtractor.java
View File

@@ -4,6 +4,7 @@ import com.semmle.js.ast.Comment;
import com.semmle.js.ast.Position;
import com.semmle.js.ast.SourceElement;
import com.semmle.js.ast.Token;
import com.semmle.js.extractor.ExtractionMetrics.ExtractionPhase;
import com.semmle.util.trap.TrapWriter;
import com.semmle.util.trap.TrapWriter.Label;
import java.util.List;
@@ -40,9 +41,16 @@ public class LexicalExtractor {
return comments;
}
public ExtractionMetrics getMetrics() {
return textualExtractor.getMetrics();
}
public LoCInfo extractLines(String src, Label toplevelKey) {
textualExtractor.getMetrics().startPhase(ExtractionPhase.LexicalExtractor_extractLines);
Position end = textualExtractor.extractLines(src, toplevelKey);
return emitNumlines(toplevelKey, new Position(1, 0, 0), end);
LoCInfo info = emitNumlines(toplevelKey, new Position(1, 0, 0), end);
textualExtractor.getMetrics().stopPhase(ExtractionPhase.LexicalExtractor_extractLines);
return info;
}
/**
@@ -112,11 +120,11 @@ public class LexicalExtractor {
}
public void extractTokens(Label toplevelKey) {
textualExtractor.getMetrics().startPhase(ExtractionPhase.LexicalExtractor_extractTokens);
int j = 0;
for (int i = 0, n = tokens.size(), idx = 0; i < n; ++i) {
Token token = tokens.get(i);
if (token == null) continue;
Label key = trapwriter.freshLabel();
int kind = -1;
switch (token.getType()) {
@@ -164,6 +172,7 @@ public class LexicalExtractor {
if (token.getLoc().equals(next.getLoc())) tokens.set(i + 1, null);
}
}
textualExtractor.getMetrics().stopPhase(ExtractionPhase.LexicalExtractor_extractTokens);
}
public void extractComments(Label toplevelKey) {

2
javascript/extractor/src/com/semmle/js/extractor/Main.java
View File

@@ -37,7 +37,7 @@ public class Main {
* A version identifier that should be updated every time the extractor changes in such a way that
* it may produce different tuples for the same file under the same {@link ExtractorConfig}.
*/
public static final String EXTRACTOR_VERSION = "2019-09-02";
public static final String EXTRACTOR_VERSION = "2019-09-04";
public static final Pattern NEWLINE = Pattern.compile("\n");

12
javascript/extractor/src/com/semmle/js/extractor/TextualExtractor.java
View File

@@ -20,14 +20,20 @@ public class TextualExtractor {
private final LocationManager locationManager;
private final Label fileLabel;
private final boolean extractLines;
private final ExtractionMetrics metrics;
public TextualExtractor(
TrapWriter trapwriter, LocationManager locationManager, String source, boolean extractLines) {
TrapWriter trapwriter,
LocationManager locationManager,
String source,
boolean extractLines,
ExtractionMetrics metrics) {
this.trapwriter = trapwriter;
this.locationManager = locationManager;
this.source = source;
this.fileLabel = locationManager.getFileLabel();
this.extractLines = extractLines;
this.metrics = metrics;
}
public TrapWriter getTrapwriter() {
@@ -42,6 +48,10 @@ public class TextualExtractor {
return source;
}
public ExtractionMetrics getMetrics() {
return metrics;
}
public String mkToString(SourceElement nd) {
return sanitiseToString(nd.getLoc().getSource());
}

8
javascript/extractor/src/com/semmle/js/extractor/TypeExprKinds.java
View File

@@ -17,7 +17,6 @@ import com.semmle.ts.ast.InferTypeExpr;
import com.semmle.ts.ast.InterfaceTypeExpr;
import com.semmle.ts.ast.IntersectionTypeExpr;
import com.semmle.ts.ast.IsTypeExpr;
import com.semmle.ts.ast.UnaryTypeExpr;
import com.semmle.ts.ast.KeywordTypeExpr;
import com.semmle.ts.ast.MappedTypeExpr;
import com.semmle.ts.ast.OptionalTypeExpr;
@@ -26,6 +25,7 @@ import com.semmle.ts.ast.RestTypeExpr;
import com.semmle.ts.ast.TupleTypeExpr;
import com.semmle.ts.ast.TypeParameter;
import com.semmle.ts.ast.TypeofTypeExpr;
import com.semmle.ts.ast.UnaryTypeExpr;
import com.semmle.ts.ast.UnionTypeExpr;
import com.semmle.util.exception.CatastrophicError;
@@ -129,8 +129,10 @@ public class TypeExprKinds {
@Override
public Integer visit(UnaryTypeExpr nd, Void c) {
switch (nd.getKind()) {
case Keyof: return keyofTypeExpr;
case Readonly: return readonlyTypeExpr;
case Keyof:
return keyofTypeExpr;
case Readonly:
return readonlyTypeExpr;
}
throw new CatastrophicError("Unhandled UnaryTypeExpr kind: " + nd.getKind());
}

2
javascript/extractor/src/com/semmle/js/extractor/TypeScriptExtractor.java
View File

@@ -21,7 +21,7 @@ public class TypeScriptExtractor implements IExtractor {
LocationManager locationManager = textualExtractor.getLocationManager();
String source = textualExtractor.getSource();
File sourceFile = locationManager.getSourceFile();
Result res = parser.parse(sourceFile, source);
Result res = parser.parse(sourceFile, source, textualExtractor.getMetrics());
ScopeManager scopeManager =
new ScopeManager(textualExtractor.getTrapwriter(), ECMAVersion.ECMA2017);
try {

42
javascript/extractor/src/com/semmle/js/extractor/test/AutoBuildTests.java
View File

@@ -1,5 +1,14 @@
package com.semmle.js.extractor.test;
import com.semmle.js.extractor.AutoBuild;
import com.semmle.js.extractor.ExtractorState;
import com.semmle.js.extractor.FileExtractor;
import com.semmle.js.extractor.FileExtractor.FileType;
import com.semmle.util.data.StringUtil;
import com.semmle.util.exception.UserError;
import com.semmle.util.files.FileUtil;
import com.semmle.util.files.FileUtil8;
import com.semmle.util.process.Env;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
@@ -16,23 +25,12 @@ import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.junit.After;
import org.junit.Assert;
import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
import com.semmle.js.extractor.AutoBuild;
import com.semmle.js.extractor.ExtractorState;
import com.semmle.js.extractor.FileExtractor;
import com.semmle.js.extractor.FileExtractor.FileType;
import com.semmle.util.data.StringUtil;
import com.semmle.util.exception.UserError;
import com.semmle.util.files.FileUtil;
import com.semmle.util.files.FileUtil8;
import com.semmle.util.process.Env;
public class AutoBuildTests {
private Path SEMMLE_DIST, LGTM_SRC;
private Set<String> expected;
@@ -132,16 +130,18 @@ public class AutoBuildTests {
@Override
protected void extractXml() throws IOException {
Files.walkFileTree(LGTM_SRC, new SimpleFileVisitor<Path>(){
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
throws IOException {
String ext = FileUtil.extension(file);
if (!ext.isEmpty() && getXmlExtensions().contains(ext.substring(1)))
actual.add(file.toString());
return FileVisitResult.CONTINUE;
}
});
Files.walkFileTree(
LGTM_SRC,
new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
throws IOException {
String ext = FileUtil.extension(file);
if (!ext.isEmpty() && getXmlExtensions().contains(ext.substring(1)))
actual.add(file.toString());
return FileVisitResult.CONTINUE;
}
});
}
}.run();
String expectedString = StringUtil.glue("\n", expected.stream().sorted().toArray());

3
javascript/extractor/src/com/semmle/js/extractor/test/NodeJSDetectorTests.java
View File

@@ -1,6 +1,7 @@
package com.semmle.js.extractor.test;
import com.semmle.js.ast.Node;
import com.semmle.js.extractor.ExtractionMetrics;
import com.semmle.js.extractor.ExtractorConfig;
import com.semmle.js.extractor.ExtractorConfig.SourceType;
import com.semmle.js.extractor.NodeJSDetector;
@@ -13,7 +14,7 @@ public class NodeJSDetectorTests {
private static final ExtractorConfig CONFIG = new ExtractorConfig(false);
private void isNodeJS(String src, boolean expected) {
Result res = JSParser.parse(CONFIG, SourceType.SCRIPT, src);
Result res = JSParser.parse(CONFIG, SourceType.SCRIPT, src, new ExtractionMetrics());
Node ast = res.getAST();
Assert.assertNotNull(ast);
Assert.assertTrue(NodeJSDetector.looksLikeNodeJS(ast) == expected);

10
javascript/extractor/src/com/semmle/js/extractor/test/TrapTests.java
View File

@@ -166,6 +166,16 @@ public class TrapTests {
String expected = new WholeIO().strictreadText(trap);
expectedVsActual.add(Pair.make(expected, actual));
}
@Override
public void addTuple(String tableName, Object... values) {
if ("extraction_data".equals(tableName)
|| "extraction_time".equals(tableName)) {
// ignore non-deterministic tables
return;
}
super.addTuple(tableName, values);
}
};
}

36
javascript/extractor/src/com/semmle/js/parser/JSDocParser.java
View File

@@ -47,9 +47,8 @@ public class JSDocParser {
// This occurs before the start of 'source', so the lineStart is negative.
int firstLineStart = -(startPos.getColumn() + "/**".length() - 1);
this.absoluteOffset = startPos.getOffset();
Pair<String, List<JSDocTagParser.Tag>> r = p.new TagParser(null).parseComment(
startPos.getLine() - 1,
firstLineStart);
Pair<String, List<JSDocTagParser.Tag>> r =
p.new TagParser(null).parseComment(startPos.getLine() - 1, firstLineStart);
List<JSDocTag> tags = new ArrayList<>();
for (JSDocTagParser.Tag tag : r.snd()) {
String title = tag.title;
@@ -269,21 +268,21 @@ public class JSDocParser {
return new SourceLocation(pos());
}
/**
* Returns the absolute position of the start of the current token.
*/
/** Returns the absolute position of the start of the current token. */
private Position pos() {
return new Position(this.lineNumber + 1, startOfCurToken - lineStart, startOfCurToken + absoluteOffset);
return new Position(
this.lineNumber + 1, startOfCurToken - lineStart, startOfCurToken + absoluteOffset);
}
/**
* Returns the absolute position of the end of the previous token.
*
* This can differ from the start of the current token in case the two tokens
* are separated by whitespace.
* <p>This can differ from the start of the current token in case the two tokens are separated
* by whitespace.
*/
private Position endPos() {
return new Position(this.lineNumber + 1, endOfPrevToken - lineStart, endOfPrevToken + absoluteOffset);
return new Position(
this.lineNumber + 1, endOfPrevToken - lineStart, endOfPrevToken + absoluteOffset);
}
private <T extends JSDocTypeExpression> T finishNode(T node) {
@@ -306,7 +305,8 @@ public class JSDocParser {
if (index >= source.length()) return -1;
int ch = source.charAt(index);
++index;
if (isLineTerminator(ch) && !(ch == '\r' && index < endIndex && source.charAt(index) == '\n')) {
if (isLineTerminator(ch)
&& !(ch == '\r' && index < endIndex && source.charAt(index) == '\n')) {
lineNumber += 1;
lineStart = index;
index = skipStars(index, endIndex);
@@ -950,7 +950,10 @@ public class JSDocParser {
consume(Token.COLON);
expr =
finishNode(
new ParameterType(new SourceLocation(loc), ((NameExpression) expr).getName(), parseTypeExpression()));
new ParameterType(
new SourceLocation(loc),
((NameExpression) expr).getName(),
parseTypeExpression()));
}
if (token == Token.EQUAL) {
consume(Token.EQUAL);
@@ -1125,8 +1128,7 @@ public class JSDocParser {
consume(Token.RBRACK, "expected an array-style type declaration (' + value + '[])");
List<JSDocTypeExpression> expressions = new ArrayList<>();
expressions.add(expr);
NameExpression nameExpr =
finishNode(new NameExpression(new SourceLocation(loc), "Array"));
NameExpression nameExpr = finishNode(new NameExpression(new SourceLocation(loc), "Array"));
return finishNode(new TypeApplication(loc, nameExpr, expressions));
}
@@ -1183,7 +1185,8 @@ public class JSDocParser {
return expr;
}
private JSDocTypeExpression parseType(int startIndex, int endIndex, int lineStart, int lineNumber) throws ParseError {
private JSDocTypeExpression parseType(
int startIndex, int endIndex, int lineStart, int lineNumber) throws ParseError {
JSDocTypeExpression expr;
this.lineNumber = lineNumber;
@@ -1202,7 +1205,8 @@ public class JSDocParser {
return expr;
}
private JSDocTypeExpression parseParamType(int startIndex, int endIndex, int lineStart, int lineNumber) throws ParseError {
private JSDocTypeExpression parseParamType(
int startIndex, int endIndex, int lineStart, int lineNumber) throws ParseError {
JSDocTypeExpression expr;
this.lineNumber = lineNumber;

10
javascript/extractor/src/com/semmle/js/parser/JSParser.java
View File

@@ -3,6 +3,8 @@ package com.semmle.js.parser;
import com.semmle.js.ast.Comment;
import com.semmle.js.ast.Node;
import com.semmle.js.ast.Token;
import com.semmle.js.extractor.ExtractionMetrics;
import com.semmle.js.extractor.ExtractionMetrics.ExtractionPhase;
import com.semmle.js.extractor.ExtractorConfig;
import com.semmle.js.extractor.ExtractorConfig.SourceType;
import java.util.List;
@@ -65,7 +67,11 @@ public class JSParser {
}
}
public static Result parse(ExtractorConfig config, SourceType sourceType, String source) {
return JcornWrapper.parse(config, sourceType, source);
public static Result parse(
ExtractorConfig config, SourceType sourceType, String source, ExtractionMetrics metrics) {
metrics.startPhase(ExtractionPhase.JSParser_parse);
Result result = JcornWrapper.parse(config, sourceType, source);
metrics.stopPhase(ExtractionPhase.JSParser_parse);
return result;
}
}

19
javascript/extractor/src/com/semmle/js/parser/TypeScriptASTConverter.java
View File

@@ -1,13 +1,5 @@
package com.semmle.js.parser;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonNull;
@@ -149,6 +141,13 @@ import com.semmle.ts.ast.TypeofTypeExpr;
import com.semmle.ts.ast.UnaryTypeExpr;
import com.semmle.ts.ast.UnionTypeExpr;
import com.semmle.util.collections.CollectionUtil;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Utility class for converting a <a
@@ -2035,7 +2034,9 @@ public class TypeScriptASTConverter {
private Node convertTaggedTemplateExpression(JsonObject node, SourceLocation loc)
throws ParseError {
return new TaggedTemplateExpression(
loc, convertChild(node, "tag"), convertChild(node, "template"),
loc,
convertChild(node, "tag"),
convertChild(node, "template"),
convertChildrenAsTypes(node, "typeArguments"));
}

97
javascript/extractor/src/com/semmle/js/parser/TypeScriptParser.java
View File

@@ -1,5 +1,27 @@
package com.semmle.js.parser;
import ch.qos.logback.classic.Level;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParseException;
import com.google.gson.JsonParser;
import com.google.gson.JsonPrimitive;
import com.semmle.js.extractor.ExtractionMetrics;
import com.semmle.js.parser.JSParser.Result;
import com.semmle.ts.extractor.TypeTable;
import com.semmle.util.data.StringUtil;
import com.semmle.util.data.UnitParser;
import com.semmle.util.exception.CatastrophicError;
import com.semmle.util.exception.Exceptions;
import com.semmle.util.exception.InterruptedError;
import com.semmle.util.exception.ResourceError;
import com.semmle.util.exception.UserError;
import com.semmle.util.io.WholeIO;
import com.semmle.util.logging.LogbackUtils;
import com.semmle.util.process.AbstractProcessBuilder;
import com.semmle.util.process.Builder;
import com.semmle.util.process.Env;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.ByteArrayOutputStream;
@@ -16,40 +38,17 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParseException;
import com.google.gson.JsonParser;
import com.google.gson.JsonPrimitive;
import com.semmle.js.parser.JSParser.Result;
import com.semmle.ts.extractor.TypeTable;
import com.semmle.util.data.StringUtil;
import com.semmle.util.data.UnitParser;
import com.semmle.util.exception.CatastrophicError;
import com.semmle.util.exception.Exceptions;
import com.semmle.util.exception.InterruptedError;
import com.semmle.util.exception.ResourceError;
import com.semmle.util.exception.UserError;
import com.semmle.util.io.WholeIO;
import com.semmle.util.logging.LogbackUtils;
import com.semmle.util.process.AbstractProcessBuilder;
import com.semmle.util.process.Builder;
import com.semmle.util.process.Env;
import ch.qos.logback.classic.Level;
/**
* The Java half of our wrapper for invoking the TypeScript parser.
*
* <p>The Node.js half of the wrapper is expected to live at {@code
* $SEMMLE_DIST/tools/typescript-parser-wrapper/main.js}; non-standard locations can be configured
* using the property {@value #PARSER_WRAPPER_PATH_ENV_VAR}.
*
* <p>The script launches the Node.js wrapper in the Node.js runtime, looking for {@code node}
* on the {@code PATH} by default. Non-standard locations can be configured using the property
* {@value #TYPESCRIPT_NODE_RUNTIME_VAR}, and additional arguments can be configured using the
* property {@value #TYPESCRIPT_NODE_RUNTIME_EXTRA_ARGS_VAR}.
*
* <p>The script launches the Node.js wrapper in the Node.js runtime, looking for {@code node} on
* the {@code PATH} by default. Non-standard locations can be configured using the property {@value
* #TYPESCRIPT_NODE_RUNTIME_VAR}, and additional arguments can be configured using the property
* {@value #TYPESCRIPT_NODE_RUNTIME_EXTRA_ARGS_VAR}.
*
* <p>The script is started upon parsing the first TypeScript file and then is kept running in the
* background, passing it requests for parsing files and getting JSON-encoded ASTs as responses.
@@ -62,8 +61,8 @@ public class TypeScriptParser {
public static final String PARSER_WRAPPER_PATH_ENV_VAR = "SEMMLE_TYPESCRIPT_PARSER_WRAPPER";
/**
* An environment variable that can be set to indicate the location of the Node.js runtime,
* as an alternative to adding Node to the PATH.
* An environment variable that can be set to indicate the location of the Node.js runtime, as an
* alternative to adding Node to the PATH.
*/
public static final String TYPESCRIPT_NODE_RUNTIME_VAR = "SEMMLE_TYPESCRIPT_NODE_RUNTIME";
@@ -71,7 +70,8 @@ public class TypeScriptParser {
* An environment variable that can be set to provide additional arguments to the Node.js runtime
* each time it is invoked. Arguments should be separated by spaces.
*/
public static final String TYPESCRIPT_NODE_RUNTIME_EXTRA_ARGS_VAR = "SEMMLE_TYPESCRIPT_NODE_RUNTIME_EXTRA_ARGS";
public static final String TYPESCRIPT_NODE_RUNTIME_EXTRA_ARGS_VAR =
"SEMMLE_TYPESCRIPT_NODE_RUNTIME_EXTRA_ARGS";
/**
* An environment variable that can be set to specify a timeout to use when verifying the
@@ -102,8 +102,8 @@ public class TypeScriptParser {
/**
* An environment variable with additional VM arguments to pass to the Node process.
* <p>
* Only <code>--inspect</code> or <code>--inspect-brk</code> may be used at the moment.
*
* <p>Only <code>--inspect</code> or <code>--inspect-brk</code> may be used at the moment.
*/
public static final String TYPESCRIPT_NODE_FLAGS = "SEMMLE_TYPESCRIPT_NODE_FLAGS";
@@ -123,8 +123,8 @@ public class TypeScriptParser {
private String nodeJsRuntime;
/**
* Arguments to pass to the Node.js runtime each time it is invoked.
* Initialised by {@link #verifyNodeInstallation}.
* Arguments to pass to the Node.js runtime each time it is invoked. Initialised by {@link
* #verifyNodeInstallation}.
*/
private List<String> nodeJsRuntimeExtraArgs = Collections.emptyList();
@@ -139,7 +139,8 @@ public class TypeScriptParser {
/**
* Verifies that Node.js and TypeScript are installed and throws an exception otherwise.
*
* @param verbose if true, log the Node.js executable path, version strings, and any additional arguments.
* @param verbose if true, log the Node.js executable path, version strings, and any additional
* arguments.
*/
public void verifyInstallation(boolean verbose) {
verifyNodeInstallation();
@@ -158,7 +159,7 @@ public class TypeScriptParser {
ByteArrayOutputStream out = new ByteArrayOutputStream();
ByteArrayOutputStream err = new ByteArrayOutputStream();
// Determine where to find the Node.js runtime.
String explicitNodeJsRuntime = Env.systemEnv().get(TYPESCRIPT_NODE_RUNTIME_VAR);
if (explicitNodeJsRuntime != null) {
@@ -175,7 +176,9 @@ public class TypeScriptParser {
nodeJsRuntimeExtraArgs = Arrays.asList(extraArgs.split("\\s+"));
}
Builder b = new Builder(getNodeJsRuntimeInvocation("--version"), out, err, getParserWrapper().getParentFile());
Builder b =
new Builder(
getNodeJsRuntimeInvocation("--version"), out, err, getParserWrapper().getParentFile());
b.expectFailure(); // We want to do our own logging in case of an error.
int timeout = Env.systemEnv().getInt(TYPESCRIPT_TIMEOUT_VAR, 10000);
@@ -203,15 +206,14 @@ public class TypeScriptParser {
}
/**
* Gets a command line to invoke the Node.js runtime.
* Any arguments in {@link TypeScriptParser#nodeJsRuntimeExtraArgs}
* are passed first, followed by those in {@code args}.
* Gets a command line to invoke the Node.js runtime. Any arguments in {@link
* TypeScriptParser#nodeJsRuntimeExtraArgs} are passed first, followed by those in {@code args}.
*/
private List<String> getNodeJsRuntimeInvocation(String ...args) {
private List<String> getNodeJsRuntimeInvocation(String... args) {
List<String> result = new ArrayList<>();
result.add(nodeJsRuntime);
result.addAll(nodeJsRuntimeExtraArgs);
for(String arg : args) {
for (String arg : args) {
result.add(arg);
}
return result;
@@ -361,17 +363,22 @@ public class TypeScriptParser {
*
* <p>If the file is not part of a project, only syntactic information will be extracted.
*/
public Result parse(File sourceFile, String source) {
public Result parse(File sourceFile, String source, ExtractionMetrics metrics) {
JsonObject request = new JsonObject();
request.add("command", new JsonPrimitive("parse"));
request.add("filename", new JsonPrimitive(sourceFile.getAbsolutePath()));
metrics.startPhase(ExtractionMetrics.ExtractionPhase.TypeScriptParser_talkToParserWrapper);
JsonObject response = talkToParserWrapper(request);
metrics.stopPhase(ExtractionMetrics.ExtractionPhase.TypeScriptParser_talkToParserWrapper);
try {
checkResponseType(response, "ast");
JsonObject nodeFlags = response.get("nodeFlags").getAsJsonObject();
JsonObject syntaxKinds = response.get("syntaxKinds").getAsJsonObject();
JsonObject ast = response.get("ast").getAsJsonObject();
return new TypeScriptASTConverter(nodeFlags, syntaxKinds).convertAST(ast, source);
metrics.startPhase(ExtractionMetrics.ExtractionPhase.TypeScriptASTConverter_convertAST);
Result converted = new TypeScriptASTConverter(nodeFlags, syntaxKinds).convertAST(ast, source);
metrics.stopPhase(ExtractionMetrics.ExtractionPhase.TypeScriptASTConverter_convertAST);
return converted;
} catch (IllegalStateException e) {
throw new CatastrophicError(
"TypeScript parser wrapper sent unexpected response: " + response, e);

2
javascript/extractor/src/com/semmle/ts/ast/UnaryTypeExpr.java
View File

@@ -6,7 +6,7 @@ import com.semmle.js.ast.Visitor;
/**
* A unary operator applied to a type.
*
* This can be <tt>keyof T</tt> or <tt>readonly T</tt>.
* <p>This can be <tt>keyof T</tt> or <tt>readonly T</tt>.
*/
public class UnaryTypeExpr extends TypeExpression {
private final ITypeExpression elementType;

1
javascript/extractor/src/com/semmle/ts/extractor/TypeExtractor.java
View File

@@ -4,7 +4,6 @@ import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import com.semmle.util.trap.TrapWriter;
import com.semmle.util.trap.TrapWriter.Label;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;

25
javascript/ql/src/meta/extraction-metrics/FileData.ql Normal file
View File

@@ -0,0 +1,25 @@
/**
* @name Extraction metrics file data
* @description Extraction metrics and related information for profiling the extraction of individual files.
* @kind table
* @id js/meta/extraction/file-data
* @tags meta
*/
import javascript
import semmle.javascript.meta.ExtractionMetrics::ExtractionMetrics
FileWithExtractionMetrics getACacheMember(string cacheFile) { cacheFile = result.getCacheFile() }
FileWithExtractionMetrics getACacheHit(FileWithExtractionMetrics f) {
result = getACacheMember(f.getCacheFile()) and
result.isFromCache()
}
from FileWithExtractionMetrics file, boolean fromCache
where (if file.isFromCache() then fromCache = true else fromCache = false)
select file.getAbsolutePath() as FILE, file.getCpuTime() as CPU_NANO,
file.getNumberOfLines() as LINES, count(Locatable n | n.getFile() = file) as LOCATABLES,
count(TypeAnnotation n | n.getFile() = file) as TYPES, file.getLength() as LENGTH,
fromCache as FROM_CACHE, count(getACacheMember(file.getCacheFile())) as CACHE_MEMBERS,
count(getACacheHit(file)) as CACHE_HITS, file.getCacheFile() as CACHE_FILE

18
javascript/ql/src/meta/extraction-metrics/MissingMetrics.ql Normal file
View File

@@ -0,0 +1,18 @@
/**
* @name File with missing extraction metrics
* @description A file missing extraction metrics is indicative of a faulty extractor.
* @kind problem
* @problem.severity warning
* @precision low
* @id js/meta/extraction/missing-metrics
* @tags meta
*/
import javascript
import semmle.javascript.meta.ExtractionMetrics::ExtractionMetrics
from File f, string cause
where not extraction_data(f, _, _, _) and cause = "No extraction_data for this file"
or
not extraction_time(f, _,_, _) and cause = "No extraction_time for this file"
select f, cause

12
javascript/ql/src/meta/extraction-metrics/PhaseTimings.ql Normal file
View File

@@ -0,0 +1,12 @@
/**
* @name Extractor phase timings
* @description An overview of how time was spent during extraction
* @kind table
* @id js/meta/extraction/phase-timings
* @tags meta
*/
import semmle.javascript.meta.ExtractionMetrics::ExtractionMetrics
from PhaseName phaseName
select phaseName, Aggregated::getCpuTime(phaseName) as CPU_NANO, Aggregated::getWallclockTime(phaseName) as WALLCLOCK_NANO

130
javascript/ql/src/semmle/javascript/meta/ExtractionMetrics.qll Normal file
View File

@@ -0,0 +1,130 @@
import javascript
/**
* INTERNAL: Do not use in ordinary queries.
*
* Extraction metrics for profiling extraction behaviours.
*/
module ExtractionMetrics {
/**
* A file with extraction metrics.
*/
class FileWithExtractionMetrics extends File {
FileWithExtractionMetrics() {
extraction_data(this, _, _, _) and extraction_time(this, _, _, _)
}
/**
* Gets the CPU time in nanoseconds it took to extract this file.
*/
float getCpuTime() { result = strictsum(getTime(_, 0)) }
/**
* Gets the wall-clock time in nanoseconds it took to extract this file.
*/
float getWallclockTime() { result = strictsum(getTime(_, 1)) }
/**
* Gets the CPU time in nanoseconds it took to process phase `phaseName` during the extraction of this file.
*/
float getCpuTime(PhaseName phaseName) { result = getTime(phaseName, 0) }
/**
* Gets the wall-clock time in nanoseconds it took to process phase `phaseName` during the extraction of this file.
*/
float getWallclockTime(PhaseName phaseName) { result = getTime(phaseName, 1) }
/**
* Holds if this file was extracted from the trap cache.
*/
predicate isFromCache() { extraction_data(this, _, true, _) }
/**
* Gets the path to the cache file used for extraction of this file.
*/
string getCacheFile() { extraction_data(this, result, _, _) }
/**
* Gets the number of UTF16 code units in this file.
*/
int getLength() { extraction_data(this, _, _, result) }
private float getTime(PhaseName phaseName, int timerKind) {
exists(float time |
// note that we use strictsum to make it clear if data is missing because it comes from an upgraded database.
strictsum(int phaseId, float r |
phaseName = getExtractionPhaseName(phaseId) and
extraction_time(this, phaseId, timerKind, r)
|
r
) = time
|
// assume the cache-lookup was for free
if isFromCache() then result = 0 else result = time
)
}
}
/**
* Converts database ids to human-readable names.
*/
private string getExtractionPhaseName(int phaseId) {
// these names ought to match the names used in
// `com.semmle.js.extractor.ExtractionTimer.ExtractionPhase`
"ASTExtractor_extract" = result and 0 = phaseId
or
"CFGExtractor_extract" = result and 1 = phaseId
or
"FileExtractor_extractContents" = result and 2 = phaseId
or
"JSExtractor_extract" = result and 3 = phaseId
or
"JSParser_parse" = result and 4 = phaseId
or
"LexicalExtractor_extractLines" = result and 5 = phaseId
or
"LexicalExtractor_extractTokens" = result and 6 = phaseId
or
"TypeScriptASTConverter_convertAST" = result and 7 = phaseId
or
"TypeScriptParser_talkToParserWrapper" = result and 8 = phaseId
}
/**
* The name of a phase of the extraction.
*/
class PhaseName extends string {
PhaseName() { this = getExtractionPhaseName(_) }
}
/**
* Utilities for aggregating metrics for multiple files.
*/
module Aggregated {
/**
* Gets the total CPU time spent on extraction.
*/
float getCpuTime() { result = strictsum(any(FileWithExtractionMetrics f).getCpuTime()) }
/**
* Gets the total wallclock time spent on extraction.
*/
float getWallclockTime() {
result = strictsum(any(FileWithExtractionMetrics f).getWallclockTime())
}
/**
* Gets the total CPU time spent in phase `phaseName` of the extraction.
*/
float getCpuTime(PhaseName phaseName) {
result = strictsum(any(FileWithExtractionMetrics f).getCpuTime(phaseName))
}
/**
* Gets the total wallclock time spent in phase `phaseName` of the extraction.
*/
float getWallclockTime(PhaseName phaseName) {
result = strictsum(any(FileWithExtractionMetrics f).getWallclockTime(phaseName))
}
}
}

29
javascript/ql/src/semmlecode.javascript.dbscheme
View File

@@ -1136,3 +1136,32 @@ configLocations(
);
@configLocatable = @config | @configName | @configValue;
/**
* The time taken for the extraction of a file.
* This table contains non-deterministic content.
*
* The sum of the `time` column for each (`file`, `timerKind`) pair
* is the total time taken for extraction of `file`. The `extractionPhase`
* column provides a granular view of the extraction time of the file.
*/
extraction_time(
int file : @file ref,
// see `com.semmle.js.extractor.ExtractionMetrics.ExtractionPhase`.
int extractionPhase: int ref,
// 0 for the elapsed CPU time in nanoseconds, 1 for the elapsed wallclock time in nanoseconds
int timerKind: int ref,
float time: float ref
)
/**
* Non-timing related data for the extraction of a single file.
* This table contains non-deterministic content.
*/
extraction_data(
int file : @file ref,
// the absolute path to the cache file
varchar(900) cacheFile: string ref,
boolean fromCache: boolean ref,
int length: int ref
)

452
javascript/ql/src/semmlecode.javascript.dbscheme.stats
View File

@@ -30782,5 +30782,457 @@
</dep>
</dependencies>
</relation>
<relation>
<name>extraction_time</name>
<cardinality>378</cardinality>
<columnsizes>
<e>
<k>file</k>
<v>21</v>
</e>
<e>
<k>extractionPhase</k>
<v>9</v>
</e>
<e>
<k>timerKind</k>
<v>2</v>
</e>
<e>
<k>time</k>
<v>43</v>
</e>
</columnsizes>
<dependencies>
<dep>
<src>file</src>
<trg>extractionPhase</trg>
<val>
<hist>
<budget>12</budget>
<bs>
<b>
<a>9</a>
<b>10</b>
<v>21</v>
</b>
</bs>
</hist>
</val>
</dep>
<dep>
<src>file</src>
<trg>timerKind</trg>
<val>
<hist>
<budget>12</budget>
<bs>
<b>
<a>2</a>
<b>3</b>
<v>21</v>
</b>
</bs>
</hist>
</val>
</dep>
<dep>
<src>file</src>
<trg>time</trg>
<val>
<hist>
<budget>12</budget>
<bs>
<b>
<a>3</a>
<b>4</b>
<v>21</v>
</b>
</bs>
</hist>
</val>
</dep>
<dep>
<src>extractionPhase</src>
<trg>file</trg>
<val>
<hist>
<budget>12</budget>
<bs>
<b>
<a>21</a>
<b>22</b>
<v>9</v>
</b>
</bs>
</hist>
</val>
</dep>
<dep>
<src>extractionPhase</src>
<trg>timerKind</trg>
<val>
<hist>
<budget>12</budget>
<bs>
<b>
<a>2</a>
<b>3</b>
<v>9</v>
</b>
</bs>
</hist>
</val>
</dep>
<dep>
<src>extractionPhase</src>
<trg>time</trg>
<val>
<hist>
<budget>12</budget>
<bs>
<b>
<a>1</a>
<b>2</b>
<v>8</v>
</b>
<b>
<a>42</a>
<b>43</b>
<v>1</v>
</b>
</bs>
</hist>
</val>
</dep>
<dep>
<src>timerKind</src>
<trg>file</trg>
<val>
<hist>
<budget>12</budget>
<bs>
<b>
<a>21</a>
<b>22</b>
<v>2</v>
</b>
</bs>
</hist>
</val>
</dep>
<dep>
<src>timerKind</src>
<trg>extractionPhase</trg>
<val>
<hist>
<budget>12</budget>
<bs>
<b>
<a>9</a>
<b>10</b>
<v>2</v>
</b>
</bs>
</hist>
</val>
</dep>
<dep>
<src>timerKind</src>
<trg>time</trg>
<val>
<hist>
<budget>12</budget>
<bs>
<b>
<a>22</a>
<b>23</b>
<v>2</v>
</b>
</bs>
</hist>
</val>
</dep>
<dep>
<src>time</src>
<trg>file</trg>
<val>
<hist>
<budget>12</budget>
<bs>
<b>
<a>1</a>
<b>2</b>
<v>42</v>
</b>
<b>
<a>21</a>
<b>22</b>
<v>1</v>
</b>
</bs>
</hist>
</val>
</dep>
<dep>
<src>time</src>
<trg>extractionPhase</trg>
<val>
<hist>
<budget>12</budget>
<bs>
<b>
<a>1</a>
<b>2</b>
<v>42</v>
</b>
<b>
<a>8</a>
<b>9</b>
<v>1</v>
</b>
</bs>
</hist>
</val>
</dep>
<dep>
<src>time</src>
<trg>timerKind</trg>
<val>
<hist>
<budget>12</budget>
<bs>
<b>
<a>1</a>
<b>2</b>
<v>42</v>
</b>
<b>
<a>2</a>
<b>3</b>
<v>1</v>
</b>
</bs>
</hist>
</val>
</dep>
</dependencies>
</relation>
<relation>
<name>extraction_data</name>
<cardinality>21</cardinality>
<columnsizes>
<e>
<k>file</k>
<v>21</v>
</e>
<e>
<k>cacheFile</k>
<v>21</v>
</e>
<e>
<k>fromCache</k>
<v>1</v>
</e>
<e>
<k>length</k>
<v>21</v>
</e>
</columnsizes>
<dependencies>
<dep>
<src>file</src>
<trg>cacheFile</trg>
<val>
<hist>
<budget>12</budget>
<bs>
<b>
<a>1</a>
<b>2</b>
<v>21</v>
</b>
</bs>
</hist>
</val>
</dep>
<dep>
<src>file</src>
<trg>fromCache</trg>
<val>
<hist>
<budget>12</budget>
<bs>
<b>
<a>1</a>
<b>2</b>
<v>21</v>
</b>
</bs>
</hist>
</val>
</dep>
<dep>
<src>file</src>
<trg>length</trg>
<val>
<hist>
<budget>12</budget>
<bs>
<b>
<a>1</a>
<b>2</b>
<v>21</v>
</b>
</bs>
</hist>
</val>
</dep>
<dep>
<src>cacheFile</src>
<trg>file</trg>
<val>
<hist>
<budget>12</budget>
<bs>
<b>
<a>1</a>
<b>2</b>
<v>21</v>
</b>
</bs>
</hist>
</val>
</dep>
<dep>
<src>cacheFile</src>
<trg>fromCache</trg>
<val>
<hist>
<budget>12</budget>
<bs>
<b>
<a>1</a>
<b>2</b>
<v>21</v>
</b>
</bs>
</hist>
</val>
</dep>
<dep>
<src>cacheFile</src>
<trg>length</trg>
<val>
<hist>
<budget>12</budget>
<bs>
<b>
<a>1</a>
<b>2</b>
<v>21</v>
</b>
</bs>
</hist>
</val>
</dep>
<dep>
<src>fromCache</src>
<trg>file</trg>
<val>
<hist>
<budget>12</budget>
<bs>
<b>
<a>21</a>
<b>22</b>
<v>1</v>
</b>
</bs>
</hist>
</val>
</dep>
<dep>
<src>fromCache</src>
<trg>cacheFile</trg>
<val>
<hist>
<budget>12</budget>
<bs>
<b>
<a>21</a>
<b>22</b>
<v>1</v>
</b>
</bs>
</hist>
</val>
</dep>
<dep>
<src>fromCache</src>
<trg>length</trg>
<val>
<hist>
<budget>12</budget>
<bs>
<b>
<a>21</a>
<b>22</b>
<v>1</v>
</b>
</bs>
</hist>
</val>
</dep>
<dep>
<src>length</src>
<trg>file</trg>
<val>
<hist>
<budget>12</budget>
<bs>
<b>
<a>1</a>
<b>2</b>
<v>21</v>
</b>
</bs>
</hist>
</val>
</dep>
<dep>
<src>length</src>
<trg>cacheFile</trg>
<val>
<hist>
<budget>12</budget>
<bs>
<b>
<a>1</a>
<b>2</b>
<v>21</v>
</b>
</bs>
</hist>
</val>
</dep>
<dep>
<src>length</src>
<trg>fromCache</trg>
<val>
<hist>
<budget>12</budget>
<bs>
<b>
<a>1</a>
<b>2</b>
<v>21</v>
</b>
</bs>
</hist>
</val>
</dep>
</dependencies>
</relation>
</stats>
</dbstats>

4
javascript/ql/test/library-tests/meta/Extraction/FileWithExtractionMetrics.expected Normal file
View File

@@ -0,0 +1,4 @@
| tst2.js:0:0:0:0 | tst2.js |
| tst.html:0:0:0:0 | tst.html |
| tst.js:0:0:0:0 | tst.js |
| tst.ts:0:0:0:0 | tst.ts |

4
javascript/ql/test/library-tests/meta/Extraction/FileWithExtractionMetrics.ql Normal file
View File

@@ -0,0 +1,4 @@
import semmle.javascript.meta.ExtractionMetrics::ExtractionMetrics
from FileWithExtractionMetrics f
select f

4
javascript/ql/test/library-tests/meta/Extraction/FileWithExtractionMetrics_length.expected Normal file
View File

@@ -0,0 +1,4 @@
| tst2.js:0:0:0:0 | tst2.js | 26 |
| tst.html:0:0:0:0 | tst.html | 127 |
| tst.js:0:0:0:0 | tst.js | 26 |
| tst.ts:0:0:0:0 | tst.ts | 31 |

4
javascript/ql/test/library-tests/meta/Extraction/FileWithExtractionMetrics_length.ql Normal file
View File

@@ -0,0 +1,4 @@
import semmle.javascript.meta.ExtractionMetrics::ExtractionMetrics
from FileWithExtractionMetrics f
select f, f.getLength()

36
javascript/ql/test/library-tests/meta/Extraction/FileWithExtractionMetrics_phases.expected Normal file
View File

@@ -0,0 +1,36 @@
| tst2.js:0:0:0:0 | tst2.js | ASTExtractor_extract |
| tst2.js:0:0:0:0 | tst2.js | CFGExtractor_extract |
| tst2.js:0:0:0:0 | tst2.js | FileExtractor_extractContents |
| tst2.js:0:0:0:0 | tst2.js | JSExtractor_extract |
| tst2.js:0:0:0:0 | tst2.js | JSParser_parse |
| tst2.js:0:0:0:0 | tst2.js | LexicalExtractor_extractLines |
| tst2.js:0:0:0:0 | tst2.js | LexicalExtractor_extractTokens |
| tst2.js:0:0:0:0 | tst2.js | TypeScriptASTConverter_convertAST |
| tst2.js:0:0:0:0 | tst2.js | TypeScriptParser_talkToParserWrapper |
| tst.html:0:0:0:0 | tst.html | ASTExtractor_extract |
| tst.html:0:0:0:0 | tst.html | CFGExtractor_extract |
| tst.html:0:0:0:0 | tst.html | FileExtractor_extractContents |
| tst.html:0:0:0:0 | tst.html | JSExtractor_extract |
| tst.html:0:0:0:0 | tst.html | JSParser_parse |
| tst.html:0:0:0:0 | tst.html | LexicalExtractor_extractLines |
| tst.html:0:0:0:0 | tst.html | LexicalExtractor_extractTokens |
| tst.html:0:0:0:0 | tst.html | TypeScriptASTConverter_convertAST |
| tst.html:0:0:0:0 | tst.html | TypeScriptParser_talkToParserWrapper |
| tst.js:0:0:0:0 | tst.js | ASTExtractor_extract |
| tst.js:0:0:0:0 | tst.js | CFGExtractor_extract |
| tst.js:0:0:0:0 | tst.js | FileExtractor_extractContents |
| tst.js:0:0:0:0 | tst.js | JSExtractor_extract |
| tst.js:0:0:0:0 | tst.js | JSParser_parse |
| tst.js:0:0:0:0 | tst.js | LexicalExtractor_extractLines |
| tst.js:0:0:0:0 | tst.js | LexicalExtractor_extractTokens |
| tst.js:0:0:0:0 | tst.js | TypeScriptASTConverter_convertAST |
| tst.js:0:0:0:0 | tst.js | TypeScriptParser_talkToParserWrapper |
| tst.ts:0:0:0:0 | tst.ts | ASTExtractor_extract |
| tst.ts:0:0:0:0 | tst.ts | CFGExtractor_extract |
| tst.ts:0:0:0:0 | tst.ts | FileExtractor_extractContents |
| tst.ts:0:0:0:0 | tst.ts | JSExtractor_extract |
| tst.ts:0:0:0:0 | tst.ts | JSParser_parse |
| tst.ts:0:0:0:0 | tst.ts | LexicalExtractor_extractLines |
| tst.ts:0:0:0:0 | tst.ts | LexicalExtractor_extractTokens |
| tst.ts:0:0:0:0 | tst.ts | TypeScriptASTConverter_convertAST |
| tst.ts:0:0:0:0 | tst.ts | TypeScriptParser_talkToParserWrapper |

7
javascript/ql/test/library-tests/meta/Extraction/FileWithExtractionMetrics_phases.ql Normal file
View File

@@ -0,0 +1,7 @@
import semmle.javascript.meta.ExtractionMetrics::ExtractionMetrics
from FileWithExtractionMetrics f, PhaseName phase
where
exists(f.getCpuTime(phase)) and
exists(f.getWallclockTime(phase))
select f, phase

7
javascript/ql/test/library-tests/meta/Extraction/tst.html Normal file
View File

@@ -0,0 +1,7 @@
<script>
console.log("extract me")
</script>
<script>
console.log("extract me")
</script>
<script src="./tst.js"></script>

1
javascript/ql/test/library-tests/meta/Extraction/tst.js Normal file
View File

@@ -0,0 +1 @@
console.log("extract me")

1
javascript/ql/test/library-tests/meta/Extraction/tst.ts Normal file
View File

@@ -0,0 +1 @@
console.log("extract me too");

1
javascript/ql/test/library-tests/meta/Extraction/tst2.js Normal file
View File

@@ -0,0 +1 @@
console.log("extract me")

1138
javascript/upgrades/b9202b62ec8a0304316515941cb45646c6dd6fdd/old.dbscheme Normal file
View File

File diff suppressed because it is too large Load Diff

1167
javascript/upgrades/b9202b62ec8a0304316515941cb45646c6dd6fdd/semmlecode.javascript.dbscheme Normal file
View File

File diff suppressed because it is too large Load Diff

2
javascript/upgrades/b9202b62ec8a0304316515941cb45646c6dd6fdd/upgrade.properties Normal file
View File

@@ -0,0 +1,2 @@
description: adds extraction metrics tables
compatibility: backwards