hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-02-23 18:33:42 +01:00
Code Issues Packages Projects Releases Wiki Activity

JavaScript: Extract tokens and comments before AST.

Browse Source 
This allows us to discard token/comment information sooner, thereby reducing heap pressure for very large files.
This commit is contained in:
Max Schaefer
2019-01-29 13:00:17 +00:00
parent aa54d67301
commit 6013b918fc
2 changed files with 12 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
javascript/extractor/src/com/semmle/js/extractor/JSExtractor.java
View File

@@ -94,12 +94,14 @@ public class JSExtractor {
ASTExtractor scriptExtractor = new ASTExtractor(lexicalExtractor, scopeManager);
toplevelLabel = scriptExtractor.getToplevelLabel();
scriptExtractor.extract(ast, platform, sourceType, toplevelKind);
lexicalExtractor.extractComments(toplevelLabel);
loc = lexicalExtractor.extractLines(parserRes.getSource(), toplevelLabel);
lexicalExtractor.extractTokens(toplevelLabel);
new CFGExtractor(scriptExtractor).extract(ast);
new JSDocExtractor(textualExtractor).extract(lexicalExtractor.getComments());
lexicalExtractor.purge();
scriptExtractor.extract(ast, platform, sourceType, toplevelKind);
new CFGExtractor(scriptExtractor).extract(ast);
} else {
lexicalExtractor = new LexicalExtractor(textualExtractor, new ArrayList<Token>(), new ArrayList<Comment>());
ASTExtractor scriptExtractor = new ASTExtractor(lexicalExtractor, null);

8
javascript/extractor/src/com/semmle/js/extractor/LexicalExtractor.java
View File

@@ -173,4 +173,12 @@ public class LexicalExtractor {
public String mkToString(SourceElement nd) {
return textualExtractor.mkToString(nd);
}
/**
* Purge token and comments information to reduce heap pressure.
*/
public void purge() {
this.tokens.clear();
this.comments.clear();
}
}