hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-04-30 11:15:13 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3049 from asger-semmle/js/fix-cyclic-join

Browse Source 
Approved by erik-krogh
This commit is contained in:
semmle-qlci
2020-03-14 16:19:25 +00:00
committed by GitHub
parent 20cae302fd 2bdf26a8f1
commit ff03478ae8
8 changed files with 249 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

164
javascript/extractor/lib/typescript/src/main.ts
View File

@@ -69,8 +69,11 @@ interface PrepareFilesCommand {
command: "prepare-files";
filenames: string[];
}
interface GetMetadataCommand {
command: "get-metadata";
}
type Command = ParseCommand | OpenProjectCommand | CloseProjectCommand
| GetTypeTableCommand | ResetCommand | QuitCommand | PrepareFilesCommand;
| GetTypeTableCommand | ResetCommand | QuitCommand | PrepareFilesCommand | GetMetadataCommand;
/** The state to be shared between commands. */
class State {
@@ -91,7 +94,7 @@ const reloadMemoryThresholdMb = getEnvironmentVariable("SEMMLE_TYPESCRIPT_MEMORY
/**
* Debugging method for finding cycles in the TypeScript AST. Should not be used in production.
*
* If cycles are found, additional properties should be added to `isBlacklistedProperty`.
* If cycles are found, the whitelist in `astProperties` is too permissive.
*/
// tslint:disable-next-line:no-unused-variable
function checkCycle(root: any) {
@@ -104,7 +107,8 @@ function checkCycle(root: any) {
obj.$cycle_visiting = true;
for (let k in obj) {
if (!obj.hasOwnProperty(k)) continue;
if (isBlacklistedProperty(k)) continue;
// Ignore numeric and whitelisted properties.
if (+k !== +k && !astPropertySet.has(k)) continue;
if (k === "$cycle_visiting") continue;
let cycle = visit(obj[k]);
if (cycle) {
@@ -122,30 +126,133 @@ function checkCycle(root: any) {
}
}
/**
* A property that should not be serialized as part of the AST, because they
* lead to cycles or are just not needed.
*
* Because of restrictions on `JSON.stringify`, these properties may also not
* be used as part of a command response.
*/
function isBlacklistedProperty(k: string) {
return k === "parent" || k === "pos" || k === "end"
|| k === "symbol" || k === "localSymbol"
|| k === "flowNode" || k === "returnFlowNode" || k === "endFlowNode" || k === "fallthroughFlowNode"
|| k === "nextContainer" || k === "locals"
|| k === "bindDiagnostics" || k === "bindSuggestionDiagnostics";
}
/** Property names to extract from the TypeScript AST. */
const astProperties: string[] = [
"$declarationKind",
"$declaredSignature",
"$end",
"$lineStarts",
"$overloadIndex",
"$pos",
"$resolvedSignature",
"$symbol",
"$tokens",
"$type",
"argument",
"argumentExpression",
"arguments",
"assertsModifier",
"asteriskToken",
"attributes",
"block",
"body",
"caseBlock",
"catchClause",
"checkType",
"children",
"clauses",
"closingElement",
"closingFragment",
"condition",
"constraint",
"constructor",
"declarationList",
"declarations",
"decorators",
"default",
"delete",
"dotDotDotToken",
"elements",
"elementType",
"elementTypes",
"elseStatement",
"escapedText",
"exclamationToken",
"exportClause",
"expression",
"exprName",
"extendsType",
"falseType",
"finallyBlock",
"flags",
"head",
"heritageClauses",
"importClause",
"incrementor",
"indexType",
"init",
"initializer",
"isExportEquals",
"isTypeOf",
"isTypeOnly",
"keywordToken",
"kind",
"label",
"left",
"literal",
"members",
"messageText",
"modifiers",
"moduleReference",
"moduleSpecifier",
"name",
"namedBindings",
"objectType",
"openingElement",
"openingFragment",
"operand",
"operator",
"operatorToken",
"parameterName",
"parameters",
"parseDiagnostics",
"properties",
"propertyName",
"qualifier",
"questionDotToken",
"questionToken",
"right",
"selfClosing",
"statement",
"statements",
"tag",
"tagName",
"template",
"templateSpans",
"text",
"thenStatement",
"token",
"tokenPos",
"trueType",
"tryBlock",
"type",
"typeArguments",
"typeName",
"typeParameter",
"typeParameters",
"types",
"variableDeclaration",
"whenFalse",
"whenTrue",
];
/** Property names used in a parse command response, in addition to the AST itself. */
const astMetaProperties: string[] = [
"ast",
"type",
];
/** Property names to extract in an AST response. */
const astPropertySet = new Set([...astProperties, ...astMetaProperties]);
/**
* Converts (part of) an AST to a JSON string, ignoring parent pointers.
* Converts (part of) an AST to a JSON string, ignoring properties we're not interested in.
*/
function stringifyAST(obj: any) {
return JSON.stringify(obj, (k, v) => {
if (isBlacklistedProperty(k)) {
return undefined;
}
return v;
// Filter out properties that aren't numeric, empty, or whitelisted.
// Note `k` is the empty string for the root object, which is also covered by +k === +k.
return (+k === +k || astPropertySet.has(k)) ? v : undefined;
});
}
@@ -155,8 +262,6 @@ function extractFile(filename: string): string {
return stringifyAST({
type: "ast",
ast,
nodeFlags: ts.NodeFlags,
syntaxKinds: ts.SyntaxKind
});
}
@@ -522,6 +627,14 @@ function handlePrepareFilesCommand(command: PrepareFilesCommand) {
});
}
function handleGetMetadataCommand(command: GetMetadataCommand) {
console.log(JSON.stringify({
type: "metadata",
syntaxKinds: ts.SyntaxKind,
nodeFlags: ts.NodeFlags,
}));
}
function reset() {
state = new State();
state.typeTable.restrictedExpansion = getEnvironmentVariable("SEMMLE_TYPESCRIPT_NO_EXPANSION", Boolean, true);
@@ -582,6 +695,9 @@ function runReadLineInterface() {
case "reset":
handleResetCommand(req);
break;
case "get-metadata":
handleGetMetadataCommand(req);
break;
case "quit":
rl.close();
break;

51
javascript/extractor/src/com/semmle/js/parser/TypeScriptASTConverter.java
View File

@@ -2,9 +2,7 @@ package com.semmle.js.parser;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -163,10 +161,7 @@ import com.semmle.util.data.IntList;
*/
public class TypeScriptASTConverter {
private String source;
private final JsonObject nodeFlags;
private final JsonObject syntaxKinds;
private final Map<Integer, String> nodeFlagMap = new LinkedHashMap<>();
private final Map<Integer, String> syntaxKindMap = new LinkedHashMap<>();
private final TypeScriptParserMetadata metadata;
private int[] lineStarts;
private int syntaxKindExtends;
@@ -180,22 +175,9 @@ public class TypeScriptASTConverter {
private static final Pattern WHITESPACE_END_PAREN =
Pattern.compile("^" + WHITESPACE_CHAR + "*\\)");
TypeScriptASTConverter(JsonObject nodeFlags, JsonObject syntaxKinds) {
this.nodeFlags = nodeFlags;
this.syntaxKinds = syntaxKinds;
makeEnumIdMap(nodeFlags, nodeFlagMap);
makeEnumIdMap(syntaxKinds, syntaxKindMap);
this.syntaxKindExtends = getSyntaxKind("ExtendsKeyword");
}
/** Builds a mapping from ID to name given a TypeScript enum object. */
private void makeEnumIdMap(JsonObject enumObject, Map<Integer, String> idToName) {
for (Map.Entry<String, JsonElement> entry : enumObject.entrySet()) {
JsonPrimitive prim = entry.getValue().getAsJsonPrimitive();
if (prim.isNumber() && !idToName.containsKey(prim.getAsInt())) {
idToName.put(prim.getAsInt(), entry.getKey());
}
}
TypeScriptASTConverter(TypeScriptParserMetadata metadata) {
this.metadata = metadata;
this.syntaxKindExtends = metadata.getSyntaxKindId("ExtendsKeyword");
}
/**
@@ -1617,7 +1599,7 @@ public class TypeScriptASTConverter {
private Node convertMetaProperty(JsonObject node, SourceLocation loc) throws ParseError {
Position metaStart = loc.getStart();
String keywordKind =
syntaxKinds.get(node.getAsJsonPrimitive("keywordToken").getAsInt() + "").getAsString();
metadata.getSyntaxKindName(node.getAsJsonPrimitive("keywordToken").getAsInt());
String identifier = keywordKind.equals("ImportKeyword") ? "import" : "new";
Position metaEnd =
new Position(
@@ -1995,7 +1977,7 @@ public class TypeScriptASTConverter {
private String getOperator(JsonObject node) throws ParseError {
int operatorId = node.get("operator").getAsInt();
switch (syntaxKindMap.get(operatorId)) {
switch (metadata.getSyntaxKindName(operatorId)) {
case "PlusPlusToken":
return "++";
case "MinusMinusToken":
@@ -2219,7 +2201,7 @@ public class TypeScriptASTConverter {
}
private Node convertTypeOperator(JsonObject node, SourceLocation loc) throws ParseError {
String operator = syntaxKinds.get("" + node.get("operator").getAsInt()).getAsString();
String operator = metadata.getSyntaxKindName(node.get("operator").getAsInt());
if (operator.equals("KeyOfKeyword")) {
return new UnaryTypeExpr(loc, UnaryTypeExpr.Kind.Keyof, convertChildAsType(node, "type"));
}
@@ -2537,12 +2519,7 @@ public class TypeScriptASTConverter {
* <tt>ts.NodeFlags</tt> in enum.
*/
private boolean hasFlag(JsonObject node, String flagName) {
JsonElement flagDescriptor = this.nodeFlags.get(flagName);
if (flagDescriptor == null) {
throw new RuntimeException(
"Incompatible version of TypeScript installed. Missing node flag " + flagName);
}
int flagId = flagDescriptor.getAsInt();
int flagId = metadata.getNodeFlagId(flagName);
JsonElement flags = node.get("flags");
if (flags instanceof JsonPrimitive) {
return (flags.getAsInt() & flagId) != 0;
@@ -2550,16 +2527,6 @@ public class TypeScriptASTConverter {
return false;
}
/** Gets the numeric value of the syntax kind enum with the given name. */
private int getSyntaxKind(String syntaxKind) {
JsonElement descriptor = this.syntaxKinds.get(syntaxKind);
if (descriptor == null) {
throw new RuntimeException(
"Incompatible version of TypeScript installed. Missing syntax kind " + syntaxKind);
}
return descriptor.getAsInt();
}
/** Check whether a node has a child with a given name. */
private boolean hasChild(JsonObject node, String prop) {
if (!node.has(prop)) return false;
@@ -2581,7 +2548,7 @@ public class TypeScriptASTConverter {
if (node instanceof JsonObject) {
JsonElement kind = ((JsonObject) node).get("kind");
if (kind instanceof JsonPrimitive && ((JsonPrimitive) kind).isNumber())
return syntaxKindMap.get(kind.getAsInt());
return metadata.getSyntaxKindName(kind.getAsInt());
}
return null;
}

19
javascript/extractor/src/com/semmle/js/parser/TypeScriptParser.java
View File

@@ -150,6 +150,9 @@ public class TypeScriptParser {
/** If non-zero, we use this instead of relying on the corresponding environment variable. */
private int typescriptRam = 0;
/** Metadata requested immediately after starting the TypeScript parser. */
private TypeScriptParserMetadata metadata;
/** Sets the amount of RAM to allocate to the TypeScript compiler.s */
public void setTypescriptRam(int megabytes) {
this.typescriptRam = megabytes;
@@ -297,6 +300,7 @@ public class TypeScriptParser {
InputStream is = parserWrapperProcess.getInputStream();
InputStreamReader isr = new InputStreamReader(is, "UTF-8");
fromParserWrapper = new BufferedReader(isr);
this.loadMetadata();
} catch (IOException e) {
throw new CatastrophicError(
"Could not start TypeScript parser wrapper " + "(command: ." + parserWrapperCommand + ")",
@@ -385,6 +389,17 @@ public class TypeScriptParser {
return new CatastrophicError("Unexpected response from TypeScript parser wrapper:\n" + response, e);
}
/**
* Requests metadata from the TypeScript process. See {@link TypeScriptParserMetadata}.
*/
private void loadMetadata() {
JsonObject request = new JsonObject();
request.add("command", new JsonPrimitive("get-metadata"));
JsonObject response = talkToParserWrapper(request);
checkResponseType(response, "metadata");
this.metadata = new TypeScriptParserMetadata(response);
}
/**
* Returns the AST for a given source file.
*
@@ -402,11 +417,9 @@ public class TypeScriptParser {
metrics.stopPhase(ExtractionMetrics.ExtractionPhase.TypeScriptParser_talkToParserWrapper);
try {
checkResponseType(response, "ast");
JsonObject nodeFlags = response.get("nodeFlags").getAsJsonObject();
JsonObject syntaxKinds = response.get("syntaxKinds").getAsJsonObject();
JsonObject ast = response.get("ast").getAsJsonObject();
metrics.startPhase(ExtractionMetrics.ExtractionPhase.TypeScriptASTConverter_convertAST);
Result converted = new TypeScriptASTConverter(nodeFlags, syntaxKinds).convertAST(ast, source);
Result converted = new TypeScriptASTConverter(metadata).convertAST(ast, source);
metrics.stopPhase(ExtractionMetrics.ExtractionPhase.TypeScriptASTConverter_convertAST);
return converted;
} catch (IllegalStateException e) {

76
javascript/extractor/src/com/semmle/js/parser/TypeScriptParserMetadata.java Normal file
View File

@@ -0,0 +1,76 @@
package com.semmle.js.parser;
import java.util.LinkedHashMap;
import java.util.Map;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonPrimitive;
/**
* Static data from the TypeScript compiler needed for decoding ASTs.
* <p>
* AST nodes store their kind and flags as integers, but the meaning of this integer changes
* between compiler versions. The metadata contains mappings from integers to logical names
* which are stable across versions.
*/
public class TypeScriptParserMetadata {
private final JsonObject nodeFlags;
private final JsonObject syntaxKinds;
private final Map<Integer, String> syntaxKindMap = new LinkedHashMap<>();
public TypeScriptParserMetadata(JsonObject metadata) {
this.nodeFlags = metadata.get("nodeFlags").getAsJsonObject();
this.syntaxKinds = metadata.get("syntaxKinds").getAsJsonObject();
makeEnumIdMap(syntaxKinds, syntaxKindMap);
}
/** Builds a mapping from ID to name given a TypeScript enum object. */
private void makeEnumIdMap(JsonObject enumObject, Map<Integer, String> idToName) {
for (Map.Entry<String, JsonElement> entry : enumObject.entrySet()) {
JsonPrimitive prim = entry.getValue().getAsJsonPrimitive();
if (prim.isNumber() && !idToName.containsKey(prim.getAsInt())) {
idToName.put(prim.getAsInt(), entry.getKey());
}
}
}
/**
* Returns the logical name associated with syntax kind ID <code>id</code>,
* or throws an exception if it does not exist.
*/
String getSyntaxKindName(int id) {
String name = syntaxKindMap.get(id);
if (name == null) {
throw new RuntimeException(
"Incompatible version of TypeScript installed. Missing syntax kind ID " + id);
}
return name;
}
/**
* Returns the syntax kind ID corresponding to the logical name <code>name</code>,
* or throws an exception if it does not exist.
*/
int getSyntaxKindId(String name) {
JsonElement elm = syntaxKinds.get(name);
if (elm == null) {
throw new RuntimeException(
"Incompatible version of TypeScript installed. Missing syntax kind " + name);
}
return elm.getAsInt();
}
/**
* Returns the NodeFlag ID from the logical name <code>name</code>
* or throws an exception if it does not exist.
*/
int getNodeFlagId(String name) {
JsonElement elm = nodeFlags.get(name);
if (elm == null) {
throw new RuntimeException(
"Incompatible version of TypeScript installed. Missing node flag " + name);
}
return elm.getAsInt();
}
}

1
javascript/ql/test/library-tests/TypeScript/RegressionTests/ArrowReturn/options Normal file
View File

@@ -0,0 +1 @@
semmle-extractor-options: --tolerate-parse-errors

2
javascript/ql/test/library-tests/TypeScript/RegressionTests/ArrowReturn/test.expected Normal file
View File

@@ -0,0 +1,2 @@
| tst.ts:1:1:1:1 | Error: '}' expected. |
| tst.ts:1:25:1:25 | Error: '{' expected. |

4
javascript/ql/test/library-tests/TypeScript/RegressionTests/ArrowReturn/test.ql Normal file
View File

@@ -0,0 +1,4 @@
import javascript
from JSParseError err
select err

1
javascript/ql/test/library-tests/TypeScript/RegressionTests/ArrowReturn/tst.ts Normal file
View File

@@ -0,0 +1 @@
var fn = (x: number) => return x * 2;