hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-16 16:53:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

JS: Remove code path for TypeScript full extraction

Browse Source
This commit is contained in:
Asger F
2025-06-24 17:16:06 +02:00
parent 8efa38be79
commit 74b817b642
9 changed files with 51 additions and 2753 deletions
Download Patch File Download Diff File Expand all files Collapse all files

206
javascript/extractor/lib/typescript/src/ast_extractor.ts
View File

@@ -10,7 +10,6 @@ export interface AugmentedSourceFile extends ts.SourceFile {
/** Internal property that we expose as a workaround. */
redirectInfo?: object | null;
$tokens?: Token[];
$symbol?: number;
$lineStarts?: ReadonlyArray<number>;
}
@@ -18,11 +17,6 @@ export interface AugmentedNode extends ts.Node {
$pos?: any;
$end?: any;
$declarationKind?: string;
$type?: number;
$symbol?: number;
$resolvedSignature?: number;
$overloadIndex?: number;
$declaredSignature?: number;
}
export type AugmentedPos = number;
@@ -73,7 +67,7 @@ function tryGetTypeOfNode(typeChecker: ts.TypeChecker, node: AugmentedNode): ts.
} catch (e) {
let sourceFile = node.getSourceFile();
let { line, character } = sourceFile.getLineAndCharacterOfPosition(node.pos);
console.warn(`Could not compute type of ${ts.SyntaxKind[node.kind]} at ${sourceFile.fileName}:${line+1}:${character+1}`);
console.warn(`Could not compute type of ${ts.SyntaxKind[node.kind]} at ${sourceFile.fileName}:${line + 1}:${character + 1}`);
return null;
}
}
@@ -157,17 +151,6 @@ export function augmentAst(ast: AugmentedSourceFile, code: string, project: Proj
});
}
let typeChecker = project && project.program.getTypeChecker();
let typeTable = project && project.typeTable;
// Associate a symbol with the AST node root, in case it is a module.
if (typeTable != null) {
let symbol = typeChecker.getSymbolAtLocation(ast);
if (symbol != null) {
ast.$symbol = typeTable.getSymbolId(symbol);
}
}
visitAstNode(ast);
function visitAstNode(node: AugmentedNode) {
ts.forEachChild(node, visitAstNode);
@@ -190,192 +173,5 @@ export function augmentAst(ast: AugmentedSourceFile, code: string, project: Proj
node.$declarationKind = "var";
}
}
if (typeChecker != null) {
if (isTypedNode(node) && !typeTable.skipExtractingTypes) {
let contextualType = isContextuallyTypedNode(node)
? typeChecker.getContextualType(node)
: null;
let type = contextualType || tryGetTypeOfNode(typeChecker, node);
if (type != null) {
let parent = node.parent;
let unfoldAlias = ts.isTypeAliasDeclaration(parent) && node === parent.type;
let id = typeTable.buildType(type, unfoldAlias);
if (id != null) {
node.$type = id;
}
}
// Extract the target call signature of a function call.
// In case the callee is overloaded or generic, this is not something we can
// derive from the callee type in QL.
if (ts.isCallOrNewExpression(node)) {
let kind = ts.isCallExpression(node) ? ts.SignatureKind.Call : ts.SignatureKind.Construct;
let resolvedSignature = typeChecker.getResolvedSignature(node);
if (resolvedSignature != null) {
let resolvedId = typeTable.getSignatureId(kind, resolvedSignature);
if (resolvedId != null) {
(node as AugmentedNode).$resolvedSignature = resolvedId;
}
let declaration = resolvedSignature.declaration;
if (declaration != null) {
// Find the generic signature, i.e. without call-site type arguments substituted,
// but with overloading resolved.
let calleeType = typeChecker.getTypeAtLocation(node.expression);
if (calleeType != null && declaration != null) {
let calleeSignatures = typeChecker.getSignaturesOfType(calleeType, kind);
for (let i = 0; i < calleeSignatures.length; ++i) {
if (calleeSignatures[i].declaration === declaration) {
(node as AugmentedNode).$overloadIndex = i;
break;
}
}
}
// Extract the symbol so the declaration can be found from QL.
let name = (declaration as any).name;
let symbol = name && typeChecker.getSymbolAtLocation(name);
if (symbol != null) {
(node as AugmentedNode).$symbol = typeTable.getSymbolId(symbol);
}
}
}
}
}
let symbolNode =
isNamedNodeWithSymbol(node) ? node.name :
ts.isImportDeclaration(node) ? node.moduleSpecifier :
ts.isExternalModuleReference(node) ? node.expression :
null;
if (symbolNode != null) {
let symbol = typeChecker.getSymbolAtLocation(symbolNode);
if (symbol != null) {
node.$symbol = typeTable.getSymbolId(symbol);
}
}
if (ts.isTypeReferenceNode(node)) {
// For type references we inject a symbol on each part of the name.
// We traverse each node in the name here since we know these are part of
// a type annotation. This means we don't have to do it for all identifiers
// and qualified names, which would extract more information than we need.
let namePart: (ts.EntityName & AugmentedNode) = node.typeName;
while (ts.isQualifiedName(namePart)) {
let symbol = typeChecker.getSymbolAtLocation(namePart.right);
if (symbol != null) {
namePart.$symbol = typeTable.getSymbolId(symbol);
}
// Traverse into the prefix.
namePart = namePart.left;
}
let symbol = typeChecker.getSymbolAtLocation(namePart);
if (symbol != null) {
namePart.$symbol = typeTable.getSymbolId(symbol);
}
}
if (ts.isFunctionLike(node)) {
let signature = typeChecker.getSignatureFromDeclaration(node);
if (signature != null) {
let kind = ts.isConstructSignatureDeclaration(node) || ts.isConstructorDeclaration(node)
? ts.SignatureKind.Construct : ts.SignatureKind.Call;
let id = typeTable.getSignatureId(kind, signature);
if (id != null) {
(node as AugmentedNode).$declaredSignature = id;
}
}
}
}
}
}
type NamedNodeWithSymbol = AugmentedNode & (ts.ClassDeclaration | ts.InterfaceDeclaration
| ts.TypeAliasDeclaration | ts.EnumDeclaration | ts.EnumMember | ts.ModuleDeclaration | ts.FunctionDeclaration
| ts.MethodDeclaration | ts.MethodSignature);
/**
* True if the given AST node has a name, and should be associated with a symbol.
*/
function isNamedNodeWithSymbol(node: ts.Node): node is NamedNodeWithSymbol {
switch (node.kind) {
case ts.SyntaxKind.ClassDeclaration:
case ts.SyntaxKind.InterfaceDeclaration:
case ts.SyntaxKind.TypeAliasDeclaration:
case ts.SyntaxKind.EnumDeclaration:
case ts.SyntaxKind.EnumMember:
case ts.SyntaxKind.ModuleDeclaration:
case ts.SyntaxKind.FunctionDeclaration:
case ts.SyntaxKind.MethodDeclaration:
case ts.SyntaxKind.MethodSignature:
return true;
}
return false;
}
/**
* True if the given AST node has a type.
*/
function isTypedNode(node: ts.Node): boolean {
switch (node.kind) {
case ts.SyntaxKind.ArrayLiteralExpression:
case ts.SyntaxKind.ArrowFunction:
case ts.SyntaxKind.AsExpression:
case ts.SyntaxKind.SatisfiesExpression:
case ts.SyntaxKind.AwaitExpression:
case ts.SyntaxKind.BinaryExpression:
case ts.SyntaxKind.CallExpression:
case ts.SyntaxKind.ClassExpression:
case ts.SyntaxKind.ClassDeclaration:
case ts.SyntaxKind.CommaListExpression:
case ts.SyntaxKind.ConditionalExpression:
case ts.SyntaxKind.Constructor:
case ts.SyntaxKind.DeleteExpression:
case ts.SyntaxKind.ElementAccessExpression:
case ts.SyntaxKind.ExpressionStatement:
case ts.SyntaxKind.ExpressionWithTypeArguments:
case ts.SyntaxKind.FalseKeyword:
case ts.SyntaxKind.FunctionDeclaration:
case ts.SyntaxKind.FunctionExpression:
case ts.SyntaxKind.GetAccessor:
case ts.SyntaxKind.Identifier:
case ts.SyntaxKind.IndexSignature:
case ts.SyntaxKind.JsxExpression:
case ts.SyntaxKind.LiteralType:
case ts.SyntaxKind.MethodDeclaration:
case ts.SyntaxKind.MethodSignature:
case ts.SyntaxKind.NewExpression:
case ts.SyntaxKind.NonNullExpression:
case ts.SyntaxKind.NoSubstitutionTemplateLiteral:
case ts.SyntaxKind.NumericLiteral:
case ts.SyntaxKind.ObjectKeyword:
case ts.SyntaxKind.ObjectLiteralExpression:
case ts.SyntaxKind.OmittedExpression:
case ts.SyntaxKind.ParenthesizedExpression:
case ts.SyntaxKind.PartiallyEmittedExpression:
case ts.SyntaxKind.PostfixUnaryExpression:
case ts.SyntaxKind.PrefixUnaryExpression:
case ts.SyntaxKind.PropertyAccessExpression:
case ts.SyntaxKind.RegularExpressionLiteral:
case ts.SyntaxKind.SetAccessor:
case ts.SyntaxKind.StringLiteral:
case ts.SyntaxKind.TaggedTemplateExpression:
case ts.SyntaxKind.TemplateExpression:
case ts.SyntaxKind.TemplateHead:
case ts.SyntaxKind.TemplateMiddle:
case ts.SyntaxKind.TemplateSpan:
case ts.SyntaxKind.TemplateTail:
case ts.SyntaxKind.TrueKeyword:
case ts.SyntaxKind.TypeAssertionExpression:
case ts.SyntaxKind.TypeLiteral:
case ts.SyntaxKind.TypeOfExpression:
case ts.SyntaxKind.VoidExpression:
case ts.SyntaxKind.YieldExpression:
return true;
default:
return ts.isTypeNode(node);
}
}
type ContextuallyTypedNode = (ts.ArrayLiteralExpression | ts.ObjectLiteralExpression) & AugmentedNode;
function isContextuallyTypedNode(node: ts.Node): node is ContextuallyTypedNode {
let kind = node.kind;
return kind === ts.SyntaxKind.ArrayLiteralExpression || kind === ts.SyntaxKind.ObjectLiteralExpression;
}

104
javascript/extractor/lib/typescript/src/common.ts
View File

@@ -1,54 +1,26 @@
import * as ts from "./typescript";
import { TypeTable } from "./type_table";
import * as pathlib from "path";
import { VirtualSourceRoot } from "./virtual_source_root";
/**
* Extracts the package name from the prefix of an import string.
*/
const packageNameRex = /^(?:@[\w.-]+[/\\]+)?\w[\w.-]*(?=[/\\]|$)/;
const extensions = ['.ts', '.tsx', '.d.ts', '.js', '.jsx'];
function getPackageName(importString: string) {
let packageNameMatch = packageNameRex.exec(importString);
if (packageNameMatch == null) return null;
let packageName = packageNameMatch[0];
if (packageName.charAt(0) === '@') {
packageName = packageName.replace(/[/\\]+/g, '/'); // Normalize slash after the scope.
}
return packageName;
}
export class Project {
public program: ts.Program = null;
private host: ts.CompilerHost;
private resolutionCache: ts.ModuleResolutionCache;
constructor(
public tsConfig: string,
public config: ts.ParsedCommandLine,
public typeTable: TypeTable,
public packageEntryPoints: Map<string, string>,
public virtualSourceRoot: VirtualSourceRoot) {
public tsConfig: string,
public config: ts.ParsedCommandLine,
public packageEntryPoints: Map<string, string>) {
this.resolveModuleNames = this.resolveModuleNames.bind(this);
this.resolutionCache = ts.createModuleResolutionCache(pathlib.dirname(tsConfig), ts.sys.realpath, config.options);
let host = ts.createCompilerHost(config.options, true);
host.resolveModuleNames = this.resolveModuleNames;
host.trace = undefined; // Disable tracing which would otherwise go to standard out
this.host = host;
}
public unload(): void {
this.typeTable.releaseProgram();
this.program = null;
}
public load(): void {
const { config, host } = this;
this.program = ts.createProgram(config.fileNames, config.options, host);
this.typeTable.setProgram(this.program, this.virtualSourceRoot);
}
/**
@@ -60,74 +32,4 @@ export class Project {
this.unload();
this.load();
}
/**
* Override for module resolution in the TypeScript compiler host.
*/
private resolveModuleNames(
moduleNames: string[],
containingFile: string,
reusedNames: string[],
redirectedReference: ts.ResolvedProjectReference,
options: ts.CompilerOptions) {
let oppositePath =
this.virtualSourceRoot.toVirtualPath(containingFile) ||
this.virtualSourceRoot.fromVirtualPath(containingFile);
const { host, resolutionCache } = this;
return moduleNames.map((moduleName) => {
let redirected = this.redirectModuleName(moduleName, containingFile, options);
if (redirected != null) return redirected;
if (oppositePath != null) {
// If the containing file is in the virtual source root, try resolving from the real source root, and vice versa.
redirected = ts.resolveModuleName(moduleName, oppositePath, options, host, resolutionCache).resolvedModule;
if (redirected != null) return redirected;
}
return ts.resolveModuleName(moduleName, containingFile, options, host, resolutionCache).resolvedModule;
});
}
/**
* Returns the path that the given import string should be redirected to, or null if it should
* fall back to standard module resolution.
*/
private redirectModuleName(moduleName: string, containingFile: string, options: ts.CompilerOptions): ts.ResolvedModule {
// Get a package name from the leading part of the module name, e.g. '@scope/foo' from '@scope/foo/bar'.
let packageName = getPackageName(moduleName);
if (packageName == null) return null;
// Get the overridden location of this package, if one exists.
let packageEntryPoint = this.packageEntryPoints.get(packageName);
if (packageEntryPoint == null) return null;
// If the requested module name is exactly the overridden package name,
// return the entry point file (it is not necessarily called `index.ts`).
if (moduleName === packageName) {
return { resolvedFileName: packageEntryPoint, isExternalLibraryImport: true };
}
// Get the suffix after the package name, e.g. the '/bar' in '@scope/foo/bar'.
let suffix = moduleName.substring(packageName.length);
// Resolve the suffix relative to the package directory.
let packageDir = pathlib.dirname(packageEntryPoint);
let joinedPath = pathlib.join(packageDir, suffix);
// Add implicit '/index'
if (ts.sys.directoryExists(joinedPath)) {
joinedPath = pathlib.join(joinedPath, 'index');
}
// Try each recognized extension. We must not return a file whose extension is not
// recognized by TypeScript.
for (let ext of extensions) {
let candidate = joinedPath.endsWith(ext) ? joinedPath : (joinedPath + ext);
if (ts.sys.fileExists(candidate)) {
return { resolvedFileName: candidate, isExternalLibraryImport: true };
}
}
return null;
}
}

356
javascript/extractor/lib/typescript/src/main.ts
View File

@@ -31,14 +31,12 @@
"use strict";
import * as fs from "fs";
import * as pathlib from "path";
import * as readline from "readline";
import * as ts from "./typescript";
import * as ast_extractor from "./ast_extractor";
import { Project } from "./common";
import { TypeTable } from "./type_table";
import { VirtualSourceRoot } from "./virtual_source_root";
// Remove limit on stack trace depth.
@@ -55,19 +53,6 @@ interface LoadCommand {
packageEntryPoints: [string, string][];
packageJsonFiles: [string, string][];
}
interface OpenProjectCommand extends LoadCommand {
command: "open-project";
}
interface GetOwnFilesCommand extends LoadCommand {
command: "get-own-files";
}
interface CloseProjectCommand {
command: "close-project";
tsConfig: string;
}
interface GetTypeTableCommand {
command: "get-type-table";
}
interface ResetCommand {
command: "reset";
}
@@ -81,13 +66,11 @@ interface PrepareFilesCommand {
interface GetMetadataCommand {
command: "get-metadata";
}
type Command = ParseCommand | OpenProjectCommand | GetOwnFilesCommand | CloseProjectCommand
| GetTypeTableCommand | ResetCommand | QuitCommand | PrepareFilesCommand | GetMetadataCommand;
type Command = ParseCommand | ResetCommand | QuitCommand | PrepareFilesCommand | GetMetadataCommand;
/** The state to be shared between commands. */
class State {
public project: Project = null;
public typeTable = new TypeTable();
/** List of files that have been requested. */
public pendingFiles: string[] = [];
@@ -205,22 +188,18 @@ function checkCycle(root: any) {
visit(root);
if (path.length > 0) {
path.reverse();
console.log(JSON.stringify({type: "error", message: "Cycle = " + path.join(".")}));
console.log(JSON.stringify({ type: "error", message: "Cycle = " + path.join(".") }));
}
}
/** Property names to extract from the TypeScript AST. */
const astProperties: string[] = [
"$declarationKind",
"$declaredSignature",
"$end",
"$lineStarts",
"$overloadIndex",
"$pos",
"$resolvedSignature",
"$symbol",
"$tokens",
"$type",
"argument",
"argumentExpression",
"arguments",
@@ -392,20 +371,12 @@ function isExtractableSourceFile(ast: ast_extractor.AugmentedSourceFile): boolea
* an already-open project, or by parsing the file.
*/
function getAstForFile(filename: string): ts.SourceFile {
if (state.project != null) {
let ast = state.project.program.getSourceFile(filename);
if (ast != null && isExtractableSourceFile(ast)) {
ast_extractor.augmentAst(ast, ast.text, state.project);
return ast;
}
}
// Fall back to extracting without a project.
let {ast, code} = parseSingleFile(filename);
let { ast, code } = parseSingleFile(filename);
ast_extractor.augmentAst(ast, code, null);
return ast;
}
function parseSingleFile(filename: string): {ast: ts.SourceFile, code: string} {
function parseSingleFile(filename: string): { ast: ts.SourceFile, code: string } {
let code = ts.sys.readFile(filename);
// create a compiler host that only allows access to `filename`
@@ -436,7 +407,7 @@ function parseSingleFile(filename: string): {ast: ts.SourceFile, code: string} {
let ast = program.getSourceFile(filename);
return {ast, code};
return { ast, code };
}
/**
@@ -507,7 +478,7 @@ function loadTsConfig(command: LoadCommand): LoadedConfig {
let virtualExclusions = excludes == null ? [] : [...excludes];
virtualExclusions.push('**/node_modules/**/*');
let virtualResults = ts.sys.readDirectory(virtualDir, extensions, virtualExclusions, includes, depth)
return [ ...originalResults, ...virtualResults ];
return [...originalResults, ...virtualResults];
},
fileExists: (path: string) => {
return ts.sys.fileExists(path)
@@ -531,256 +502,6 @@ function loadTsConfig(command: LoadCommand): LoadedConfig {
return { config, basePath, packageJsonFiles, packageEntryPoints, virtualSourceRoot, ownFiles };
}
/**
* Returns the list of files included in the given tsconfig.json file's include pattern,
* (not including those only references through imports).
*/
function handleGetFileListCommand(command: GetOwnFilesCommand) {
let { config, ownFiles } = loadTsConfig(command);
console.log(JSON.stringify({
type: "file-list",
ownFiles,
}));
}
function handleOpenProjectCommand(command: OpenProjectCommand) {
let { config, packageEntryPoints, virtualSourceRoot, basePath, ownFiles } = loadTsConfig(command);
let project = new Project(command.tsConfig, config, state.typeTable, packageEntryPoints, virtualSourceRoot);
project.load();
state.project = project;
let program = project.program;
let typeChecker = program.getTypeChecker();
let shouldReportDiagnostics = getEnvironmentVariable("SEMMLE_TYPESCRIPT_REPORT_DIAGNOSTICS", v => v.trim().toLowerCase() === "true", false);
let diagnostics = shouldReportDiagnostics
? program.getSemanticDiagnostics().filter(d => d.category === ts.DiagnosticCategory.Error)
: [];
if (diagnostics.length > 0) {
console.warn('TypeScript: reported ' + diagnostics.length + ' semantic errors.');
}
for (let diagnostic of diagnostics) {
let text = diagnostic.messageText;
if (text && typeof text !== 'string') {
text = text.messageText;
}
let locationStr = '';
let { file } = diagnostic;
if (file != null) {
let { line, character } = file.getLineAndCharacterOfPosition(diagnostic.start);
locationStr = `${file.fileName}:${line}:${character}`;
}
console.warn(`TypeScript: ${locationStr} ${text}`);
}
// Associate external module names with the corresponding file symbols.
// We need these mappings to identify which module a given external type comes from.
// The TypeScript API lets us resolve a module name to a source file, but there is no
// inverse mapping, nor a way to enumerate all known module names. So we discover all
// modules on the type roots (usually "node_modules/@types" but this is configurable).
let typeRoots = ts.getEffectiveTypeRoots(config.options, {
getCurrentDirectory: () => basePath,
});
for (let typeRoot of typeRoots || []) {
if (ts.sys.directoryExists(typeRoot)) {
traverseTypeRoot(typeRoot, "");
}
let virtualTypeRoot = virtualSourceRoot.toVirtualPathIfDirectoryExists(typeRoot);
if (virtualTypeRoot != null) {
traverseTypeRoot(virtualTypeRoot, "");
}
}
for (let sourceFile of program.getSourceFiles()) {
addModuleBindingsFromModuleDeclarations(sourceFile);
addModuleBindingsFromFilePath(sourceFile);
}
/** Concatenates two imports paths. These always use `/` as path separator. */
function joinModulePath(prefix: string, suffix: string) {
if (prefix.length === 0) return suffix;
if (suffix.length === 0) return prefix;
return prefix + "/" + suffix;
}
/**
* Traverses a directory that is a type root or contained in a type root, and associates
* module names (i.e. import strings) with files in this directory.
*
* `importPrefix` denotes an import string that resolves to this directory,
* or an empty string if the file itself is a type root.
*
* The `filePath` is a system file path, possibly absolute, whereas `importPrefix`
* is generally short and system-independent, typically just the name of a module.
*/
function traverseTypeRoot(filePath: string, importPrefix: string) {
for (let child of fs.readdirSync(filePath)) {
if (child[0] === ".") continue;
let childPath = pathlib.join(filePath, child);
if (fs.statSync(childPath).isDirectory()) {
traverseTypeRoot(childPath, joinModulePath(importPrefix, child));
continue;
}
let sourceFile = program.getSourceFile(childPath);
if (sourceFile == null) {
continue;
}
let importPath = getImportPathFromFileInFolder(importPrefix, child);
addModuleBindingFromImportPath(sourceFile, importPath);
}
}
function getImportPathFromFileInFolder(folder: string, baseName: string) {
let stem = getStem(baseName);
return (stem === "index")
? folder
: joinModulePath(folder, stem);
}
/**
* Emits module bindings for a module with relative path `folder/baseName`.
*/
function addModuleBindingFromImportPath(sourceFile: ts.SourceFile, importPath: string) {
let symbol = typeChecker.getSymbolAtLocation(sourceFile);
if (symbol == null) return; // Happens if the source file is not a module.
let canonicalSymbol = getEffectiveExportTarget(symbol); // Follow `export = X` declarations.
let symbolId = state.typeTable.getSymbolId(canonicalSymbol);
// Associate the module name with this symbol.
state.typeTable.addModuleMapping(symbolId, importPath);
// Associate global variable names with this module.
// For each `export as X` declaration, the global X refers to this module.
// Note: the `globalExports` map is stored on the original symbol, not the target of `export=`.
if (symbol.globalExports != null) {
symbol.globalExports.forEach((global: ts.Symbol) => {
state.typeTable.addGlobalMapping(symbolId, global.name);
});
}
}
/**
* Returns the basename of `file` without its extension, while treating `.d.ts` as a
* single extension.
*/
function getStem(file: string) {
if (file.endsWith(".d.ts")) {
return pathlib.basename(file, ".d.ts");
}
if (file.endsWith(".d.mts") || file.endsWith(".d.cts")) {
// We don't extract d.mts or d.cts files, but their symbol can coincide with that of a d.ts file,
// which means any module bindings we generate for it will ultimately be visible in QL.
let base = pathlib.basename(file);
return base.substring(0, base.length - '.d.mts'.length);
}
let base = pathlib.basename(file);
let dot = base.lastIndexOf('.');
return dot === -1 || dot === 0 ? base : base.substring(0, dot);
}
/**
* Emits module bindings for a module based on its file path.
*
* This looks for enclosing `node_modules` folders to determine the module name.
* This is needed for modules that ship their own type definitions as opposed to having
* type definitions loaded from a type root (conventionally named `@types/xxx`).
*/
function addModuleBindingsFromFilePath(sourceFile: ts.SourceFile) {
let fullPath = sourceFile.fileName;
let index = fullPath.lastIndexOf('/node_modules/');
if (index === -1) return;
let relativePath = fullPath.substring(index + '/node_modules/'.length);
// Ignore node_modules/@types folders here as they are typically handled as type roots.
if (relativePath.startsWith("@types/")) return;
// If the enclosing package has a "typings" field, only add module bindings for that file.
let packageJsonFile = getEnclosingPackageJson(fullPath);
if (packageJsonFile != null) {
let json = getPackageJson(packageJsonFile);
let typings = getPackageTypings(packageJsonFile);
if (json != null && typings != null) {
let name = json.name;
if (typings === fullPath && typeof name === 'string') {
addModuleBindingFromImportPath(sourceFile, name);
} else if (typings != null) {
return; // Typings field prevents access to other files in package.
}
}
}
// Add module bindings relative to package directory.
let { dir, base } = pathlib.parse(relativePath);
addModuleBindingFromImportPath(sourceFile, getImportPathFromFileInFolder(dir, base));
}
/**
* Emit module name bindings for external module declarations, i.e: `declare module 'X' {..}`
* These can generally occur anywhere; they may or may not be on the type root path.
*/
function addModuleBindingsFromModuleDeclarations(sourceFile: ts.SourceFile) {
for (let stmt of sourceFile.statements) {
if (ts.isModuleDeclaration(stmt) && ts.isStringLiteral(stmt.name)) {
let symbol = (stmt as any).symbol;
if (symbol == null) continue;
symbol = getEffectiveExportTarget(symbol);
let symbolId = state.typeTable.getSymbolId(symbol);
let moduleName = stmt.name.text;
state.typeTable.addModuleMapping(symbolId, moduleName);
}
}
}
/**
* If `symbol` refers to a container with an `export = X` declaration, returns
* the target of `X`, otherwise returns `symbol`.
*/
function getEffectiveExportTarget(symbol: ts.Symbol) {
if (symbol.exports != null && symbol.exports.has(ts.InternalSymbolName.ExportEquals)) {
let exportAlias = symbol.exports.get(ts.InternalSymbolName.ExportEquals);
if (exportAlias.flags & ts.SymbolFlags.Alias) {
return typeChecker.getAliasedSymbol(exportAlias);
}
}
return symbol;
}
// Unlike in the get-own-files command, this command gets all files we can possibly
// extract type information for, including files referenced outside the tsconfig's inclusion pattern.
let allFiles = program.getSourceFiles().map(sf => pathlib.resolve(sf.fileName));
console.log(JSON.stringify({
type: "project-opened",
ownFiles,
allFiles,
}));
}
function handleCloseProjectCommand(command: CloseProjectCommand) {
if (state.project == null) {
console.log(JSON.stringify({
type: "error",
message: "No project is open",
}));
return;
}
state.project.unload();
state.project = null;
console.log(JSON.stringify({type: "project-closed"}));
}
function handleGetTypeTableCommand(command: GetTypeTableCommand) {
console.log(JSON.stringify({
type: "type-table",
typeTable: state.typeTable.getTypeTableJson(),
}));
}
function handleResetCommand(command: ResetCommand) {
reset();
console.log(JSON.stringify({
@@ -807,8 +528,6 @@ function handleGetMetadataCommand(command: GetMetadataCommand) {
function reset() {
state = new State();
state.typeTable.restrictedExpansion = getEnvironmentVariable("SEMMLE_TYPESCRIPT_NO_EXPANSION", v => v.trim().toLowerCase() === "true", true);
state.typeTable.skipExtractingTypes = getEnvironmentVariable("CODEQL_EXTRACTOR_JAVASCRIPT_OPTION_SKIP_TYPES", v => v.trim().toLowerCase() === "true", false);
}
function getEnvironmentVariable<T>(name: string, parse: (x: string) => T, defaultValue: T) {
@@ -848,35 +567,23 @@ function runReadLineInterface() {
rl.on("line", (line: string) => {
let req: Command = JSON.parse(line);
switch (req.command) {
case "parse":
handleParseCommand(req);
break;
case "open-project":
handleOpenProjectCommand(req);
break;
case "get-own-files":
handleGetFileListCommand(req);
break;
case "close-project":
handleCloseProjectCommand(req);
break;
case "get-type-table":
handleGetTypeTableCommand(req);
break;
case "prepare-files":
handlePrepareFilesCommand(req);
break;
case "reset":
handleResetCommand(req);
break;
case "get-metadata":
handleGetMetadataCommand(req);
break;
case "quit":
rl.close();
break;
default:
throw new Error("Unknown command " + (req as any).command + ".");
case "parse":
handleParseCommand(req);
break;
case "prepare-files":
handlePrepareFilesCommand(req);
break;
case "reset":
handleResetCommand(req);
break;
case "get-metadata":
handleGetMetadataCommand(req);
break;
case "quit":
rl.close();
break;
default:
throw new Error("Unknown command " + (req as any).command + ".");
}
});
}
@@ -886,23 +593,6 @@ if (process.argv.length > 2) {
let argument = process.argv[2];
if (argument === "--version") {
console.log("parser-wrapper with TypeScript " + ts.version);
} else if (pathlib.basename(argument) === "tsconfig.json") {
reset();
handleOpenProjectCommand({
command: "open-project",
tsConfig: argument,
packageEntryPoints: [],
packageJsonFiles: [],
sourceRoot: null,
virtualSourceRoot: null,
});
for (let sf of state.project.program.getSourceFiles()) {
if (/lib\..*\.d\.ts/.test(pathlib.basename(sf.fileName)) || pathlib.basename(sf.fileName) === "lib.d.ts") continue;
handleParseCommand({
command: "parse",
filename: sf.fileName,
}, false);
}
} else if (pathlib.extname(argument) === ".ts" || pathlib.extname(argument) === ".tsx") {
handleParseCommand({
command: "parse",

1423
javascript/extractor/lib/typescript/src/type_table.ts
View File

File diff suppressed because it is too large Load Diff

99
javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java
View File

@@ -51,10 +51,8 @@ import com.semmle.js.extractor.trapcache.DummyTrapCache;
import com.semmle.js.extractor.trapcache.ITrapCache;
import com.semmle.js.parser.ParseError;
import com.semmle.js.parser.ParsedProject;
import com.semmle.ts.extractor.TypeExtractor;
import com.semmle.ts.extractor.TypeScriptParser;
import com.semmle.ts.extractor.TypeScriptWrapperOOMError;
import com.semmle.ts.extractor.TypeTable;
import com.semmle.util.data.StringUtil;
import com.semmle.util.diagnostic.DiagnosticLevel;
import com.semmle.util.diagnostic.DiagnosticLocation;
@@ -1065,75 +1063,26 @@ protected DependencyInstallationResult preparePackagesAndDependencies(Set<Path>
FileExtractors extractors,
List<Path> tsconfig,
DependencyInstallationResult deps) {
if (hasTypeScriptFiles(files) || !tsconfig.isEmpty()) {
List<Path> typeScriptFiles = new ArrayList<>();
// Get all TypeScript files.
for (Path f : files) {
if (extractors.fileType(f) == FileType.TYPESCRIPT) {
typeScriptFiles.add(f);
}
}
// Also get TypeScript files from HTML file snippets.
for (Map.Entry<Path, FileSnippet> entry : state.getSnippets().entrySet()) {
if (!extractedFiles.contains(entry.getKey())
&& FileType.forFileExtension(entry.getKey().toFile()) == FileType.TYPESCRIPT) {
typeScriptFiles.add(entry.getKey());
}
}
if (!typeScriptFiles.isEmpty()) {
TypeScriptParser tsParser = state.getTypeScriptParser();
verifyTypeScriptInstallation(state);
// Collect all files included in a tsconfig.json inclusion pattern.
// If a given file is referenced by multiple tsconfig files, we prefer to extract it using
// one that includes it rather than just references it.
Set<File> explicitlyIncludedFiles = new LinkedHashSet<>();
if (tsconfig.size() > 1) { // No prioritization needed if there's only one tsconfig.
for (Path projectPath : tsconfig) {
explicitlyIncludedFiles.addAll(tsParser.getOwnFiles(projectPath.toFile(), deps, virtualSourceRoot));
}
}
// Extract TypeScript projects
for (Path projectPath : tsconfig) {
File projectFile = projectPath.toFile();
long start = logBeginProcess("Opening project " + projectFile);
ParsedProject project = tsParser.openProject(projectFile, deps, virtualSourceRoot);
logEndProcess(start, "Done opening project " + projectFile);
// Extract all files belonging to this project which are also matched
// by our include/exclude filters.
List<Path> typeScriptFiles = new ArrayList<Path>();
for (File sourceFile : project.getAllFiles()) {
Path sourcePath = sourceFile.toPath();
Path normalizedFile = normalizePath(sourcePath);
if (!files.contains(normalizedFile) && !state.getSnippets().containsKey(normalizedFile)) {
continue;
}
if (!project.getOwnFiles().contains(sourceFile) && explicitlyIncludedFiles.contains(sourceFile)) continue;
if (extractors.fileType(sourcePath) != FileType.TYPESCRIPT) {
// For the time being, skip non-TypeScript files, even if the TypeScript
// compiler can parse them for us.
continue;
}
if (extractedFiles.contains(sourcePath)) {
continue;
}
typeScriptFiles.add(sourcePath);
}
typeScriptFiles.sort(PATH_ORDERING);
extractTypeScriptFiles(typeScriptFiles, extractedFiles, extractors);
tsParser.closeProject(projectFile);
}
// Extract all the types discovered when extracting the ASTs.
if (!tsconfig.isEmpty()) {
TypeTable typeTable = tsParser.getTypeTable();
extractTypeTable(tsconfig.iterator().next(), typeTable);
}
// Extract remaining TypeScript files.
List<Path> remainingTypeScriptFiles = new ArrayList<>();
for (Path f : files) {
if (!extractedFiles.contains(f)
&& extractors.fileType(f) == FileType.TYPESCRIPT) {
remainingTypeScriptFiles.add(f);
}
}
for (Map.Entry<Path, FileSnippet> entry : state.getSnippets().entrySet()) {
if (!extractedFiles.contains(entry.getKey())
&& FileType.forFileExtension(entry.getKey().toFile()) == FileType.TYPESCRIPT) {
remainingTypeScriptFiles.add(entry.getKey());
}
}
if (!remainingTypeScriptFiles.isEmpty()) {
extractTypeScriptFiles(remainingTypeScriptFiles, extractedFiles, extractors);
}
extractTypeScriptFiles(typeScriptFiles, extractedFiles, extractors);
// The TypeScript compiler instance is no longer needed.
tsParser.killProcess();
}
@@ -1246,18 +1195,6 @@ protected DependencyInstallationResult preparePackagesAndDependencies(Set<Path>
return path.toAbsolutePath().normalize();
}
private void extractTypeTable(Path fileHandle, TypeTable table) {
TrapWriter trapWriter =
outputConfig
.getTrapWriterFactory()
.mkTrapWriter(new File(fileHandle.toString() + ".codeql-typescript-typetable"));
try {
new TypeExtractor(trapWriter, table).extract();
} finally {
FileUtil.close(trapWriter);
}
}
/**
* Get the source type specified in <code>LGTM_INDEX_SOURCE_TYPE</code>, or the default of {@link
* SourceType#AUTO}.

60
javascript/extractor/src/com/semmle/js/extractor/Main.java
View File

@@ -18,9 +18,7 @@ import com.semmle.js.extractor.ExtractorConfig.SourceType;
import com.semmle.js.extractor.FileExtractor.FileType;
import com.semmle.js.extractor.trapcache.ITrapCache;
import com.semmle.js.parser.ParsedProject;
import com.semmle.ts.extractor.TypeExtractor;
import com.semmle.ts.extractor.TypeScriptParser;
import com.semmle.ts.extractor.TypeTable;
import com.semmle.util.data.StringUtil;
import com.semmle.util.data.UnitParser;
import com.semmle.util.exception.ResourceError;
@@ -142,53 +140,22 @@ public class Main {
tsParser.verifyInstallation(!ap.has(P_QUIET));
}
for (File projectFile : projectFiles) {
long start = verboseLogStartTimer(ap, "Opening project " + projectFile);
ParsedProject project = tsParser.openProject(projectFile, DependencyInstallationResult.empty, extractorConfig.getVirtualSourceRoot());
verboseLogEndTimer(ap, start);
// Extract all files belonging to this project which are also matched
// by our include/exclude filters.
List<File> filesToExtract = new ArrayList<>();
for (File sourceFile : project.getOwnFiles()) {
File normalizedFile = normalizeFile(sourceFile);
if ((files.contains(normalizedFile) || extractorState.getSnippets().containsKey(normalizedFile.toPath()))
&& !extractedFiles.contains(sourceFile.getAbsoluteFile())
&& FileType.TYPESCRIPT.getExtensions().contains(FileUtil.extension(sourceFile))) {
filesToExtract.add(sourceFile);
}
}
tsParser.prepareFiles(filesToExtract);
for (int i = 0; i < filesToExtract.size(); ++i) {
ensureFileIsExtracted(filesToExtract.get(i), ap);
}
// Close the project to free memory. This does not need to be in a `finally` as
// the project is not a system resource.
tsParser.closeProject(projectFile);
}
if (!projectFiles.isEmpty()) {
// Extract all the types discovered when extracting the ASTs.
TypeTable typeTable = tsParser.getTypeTable();
extractTypeTable(projectFiles.iterator().next(), typeTable);
}
List<File> remainingTypescriptFiles = new ArrayList<>();
List<File> typeScriptFiles = new ArrayList<>();
for (File f : files) {
if (!extractedFiles.contains(f.getAbsoluteFile())
&& FileType.forFileExtension(f) == FileType.TYPESCRIPT) {
remainingTypescriptFiles.add(f);
typeScriptFiles.add(f);
}
}
for (Map.Entry<Path, FileSnippet> entry : extractorState.getSnippets().entrySet()) {
if (!extractedFiles.contains(entry.getKey().toFile())
&& FileType.forFileExtension(entry.getKey().toFile()) == FileType.TYPESCRIPT) {
remainingTypescriptFiles.add(entry.getKey().toFile());
typeScriptFiles.add(entry.getKey().toFile());
}
}
if (!remainingTypescriptFiles.isEmpty()) {
tsParser.prepareFiles(remainingTypescriptFiles);
for (File f : remainingTypescriptFiles) {
if (!typeScriptFiles.isEmpty()) {
tsParser.prepareFiles(typeScriptFiles);
for (File f : typeScriptFiles) {
ensureFileIsExtracted(f, ap);
}
}
@@ -225,21 +192,6 @@ public class Main {
return false;
}
private void extractTypeTable(File fileHandle, TypeTable table) {
TrapWriter trapWriter =
extractorOutputConfig
.getTrapWriterFactory()
.mkTrapWriter(
new File(
fileHandle.getParentFile(),
fileHandle.getName() + ".codeql-typescript-typetable"));
try {
new TypeExtractor(trapWriter, table).extract();
} finally {
FileUtil.close(trapWriter);
}
}
private void ensureFileIsExtracted(File f, ArgsParser ap) {
if (!extractedFiles.add(f.getAbsoluteFile())) {
// The file has already been extracted as part of a project.

330
javascript/extractor/src/com/semmle/ts/extractor/TypeExtractor.java
View File

@@ -1,330 +0,0 @@
package com.semmle.ts.extractor;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import com.semmle.util.trap.TrapWriter;
import com.semmle.util.trap.TrapWriter.Label;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
/**
* Extracts type and symbol information into TRAP files.
*
* <p>This is closely coupled with the <code>type_table.ts</code> file in the parser-wrapper. Type
* strings and symbol strings generated in that file are parsed here. See that file for reference
* and documentation.
*/
public class TypeExtractor {
private final TrapWriter trapWriter;
private final TypeTable table;
private static final Map<String, Integer> tagToKind = new LinkedHashMap<String, Integer>();
private static final int referenceKind = 6;
private static final int objectKind = 7;
private static final int typevarKind = 8;
private static final int typeofKind = 9;
private static final int uniqueSymbolKind = 15;
private static final int tupleKind = 18;
private static final int lexicalTypevarKind = 19;
private static final int thisKind = 20;
private static final int numberLiteralTypeKind = 21;
private static final int stringLiteralTypeKind = 22;
private static final int bigintLiteralTypeKind = 25;
static {
tagToKind.put("any", 0);
tagToKind.put("string", 1);
tagToKind.put("number", 2);
tagToKind.put("union", 3);
tagToKind.put("true", 4);
tagToKind.put("false", 5);
tagToKind.put("reference", referenceKind);
tagToKind.put("object", objectKind);
tagToKind.put("typevar", typevarKind);
tagToKind.put("typeof", typeofKind);
tagToKind.put("void", 10);
tagToKind.put("undefined", 11);
tagToKind.put("null", 12);
tagToKind.put("never", 13);
tagToKind.put("plainsymbol", 14);
tagToKind.put("uniquesymbol", uniqueSymbolKind);
tagToKind.put("objectkeyword", 16);
tagToKind.put("intersection", 17);
tagToKind.put("tuple", tupleKind);
tagToKind.put("lextypevar", lexicalTypevarKind);
tagToKind.put("this", thisKind);
tagToKind.put("numlit", numberLiteralTypeKind);
tagToKind.put("strlit", stringLiteralTypeKind);
tagToKind.put("unknown", 23);
tagToKind.put("bigint", 24);
tagToKind.put("bigintlit", bigintLiteralTypeKind);
}
private static final Map<String, Integer> symbolKind = new LinkedHashMap<String, Integer>();
static {
symbolKind.put("root", 0);
symbolKind.put("member", 1);
symbolKind.put("other", 2);
}
public TypeExtractor(TrapWriter trapWriter, TypeTable table) {
this.trapWriter = trapWriter;
this.table = table;
}
public void extract() {
for (int i = 0; i < table.getNumberOfTypes(); ++i) {
extractType(i);
}
extractPropertyLookups(table.getPropertyLookups());
extractTypeAliases(table.getTypeAliases());
for (int i = 0; i < table.getNumberOfSymbols(); ++i) {
extractSymbol(i);
}
extractSymbolNameMapping("symbol_module", table.getModuleMappings());
extractSymbolNameMapping("symbol_global", table.getGlobalMappings());
extractSignatureMappings(table.getSignatureMappings());
for (int i = 0; i < table.getNumberOfSignatures(); ++i) {
extractSignature(i);
}
extractIndexTypeTable(table.getNumberIndexTypes(), "number_index_type");
extractIndexTypeTable(table.getStringIndexTypes(), "string_index_type");
extractBaseTypes(table.getBaseTypes());
extractSelfTypes(table.getSelfTypes());
}
private void extractType(int id) {
Label lbl = trapWriter.globalID("type;" + id);
String contents = table.getTypeString(id);
String[] parts = split(contents);
int kind = tagToKind.get(parts[0]);
trapWriter.addTuple("types", lbl, kind, table.getTypeToStringValue(id));
int firstChild = 1;
switch (kind) {
case referenceKind:
case typevarKind:
case typeofKind:
case uniqueSymbolKind:
{
// The first part of a reference is the symbol for name binding.
Label symbol = trapWriter.globalID("symbol;" + parts[1]);
trapWriter.addTuple("type_symbol", lbl, symbol);
++firstChild;
break;
}
case tupleKind:
{
// The first two parts denote minimum length and index of rest element (or -1 if no rest element).
trapWriter.addTuple("tuple_type_min_length", lbl, Integer.parseInt(parts[1]));
int restIndex = Integer.parseInt(parts[2]);
if (restIndex != -1) {
trapWriter.addTuple("tuple_type_rest_index", lbl, restIndex);
}
firstChild += 2;
break;
}
case objectKind:
case lexicalTypevarKind:
firstChild = parts.length; // No children.
break;
case numberLiteralTypeKind:
case stringLiteralTypeKind:
case bigintLiteralTypeKind:
firstChild = parts.length; // No children.
// The string value may contain `;` so don't use the split().
String value = contents.substring(parts[0].length() + 1);
trapWriter.addTuple("type_literal_value", lbl, value);
break;
}
for (int i = firstChild; i < parts.length; ++i) {
Label childLabel = trapWriter.globalID("type;" + parts[i]);
trapWriter.addTuple("type_child", childLabel, lbl, i - firstChild);
}
}
private void extractPropertyLookups(JsonObject lookups) {
JsonArray baseTypes = lookups.get("baseTypes").getAsJsonArray();
JsonArray names = lookups.get("names").getAsJsonArray();
JsonArray propertyTypes = lookups.get("propertyTypes").getAsJsonArray();
for (int i = 0; i < baseTypes.size(); ++i) {
int baseType = baseTypes.get(i).getAsInt();
String name = names.get(i).getAsString();
int propertyType = propertyTypes.get(i).getAsInt();
trapWriter.addTuple(
"type_property",
trapWriter.globalID("type;" + baseType),
name,
trapWriter.globalID("type;" + propertyType));
}
}
private void extractTypeAliases(JsonObject aliases) {
JsonArray aliasTypes = aliases.get("aliasTypes").getAsJsonArray();
JsonArray underlyingTypes = aliases.get("underlyingTypes").getAsJsonArray();
for (int i = 0; i < aliasTypes.size(); ++i) {
int aliasType = aliasTypes.get(i).getAsInt();
int underlyingType = underlyingTypes.get(i).getAsInt();
trapWriter.addTuple(
"type_alias",
trapWriter.globalID("type;" + aliasType),
trapWriter.globalID("type;" + underlyingType));
}
}
private void extractSymbol(int index) {
// Format is: kind;decl;parent;name
String[] parts = split(table.getSymbolString(index), 4);
int kind = symbolKind.get(parts[0]);
String name = parts[3];
Label label = trapWriter.globalID("symbol;" + index);
trapWriter.addTuple("symbols", label, kind, name);
String parentStr = parts[2];
if (parentStr.length() > 0) {
Label parentLabel = trapWriter.globalID("symbol;" + parentStr);
trapWriter.addTuple("symbol_parent", label, parentLabel);
}
}
private void extractSymbolNameMapping(String relationName, JsonObject mappings) {
JsonArray symbols = mappings.get("symbols").getAsJsonArray();
JsonArray names = mappings.get("names").getAsJsonArray();
for (int i = 0; i < symbols.size(); ++i) {
Label symbol = trapWriter.globalID("symbol;" + symbols.get(i).getAsInt());
String moduleName = names.get(i).getAsString();
trapWriter.addTuple(relationName, symbol, moduleName);
}
}
private void extractSignature(int index) {
// Format is:
// kind;isAbstract;numTypeParams;requiredParams;restParamType;returnType(;paramName;paramType)*
String[] parts = split(table.getSignatureString(index));
Label label = trapWriter.globalID("signature;" + index);
int kind = Integer.parseInt(parts[0]);
boolean isAbstract = parts[1].equals("t");
if (isAbstract) {
trapWriter.addTuple("is_abstract_signature", label);
}
int numberOfTypeParameters = Integer.parseInt(parts[2]);
int requiredParameters = Integer.parseInt(parts[3]);
String restParamTypeTag = parts[4];
if (!restParamTypeTag.isEmpty()) {
trapWriter.addTuple(
"signature_rest_parameter", label, trapWriter.globalID("type;" + restParamTypeTag));
}
Label returnType = trapWriter.globalID("type;" + parts[5]);
trapWriter.addTuple(
"signature_types",
label,
kind,
table.getSignatureToStringValue(index),
numberOfTypeParameters,
requiredParameters);
trapWriter.addTuple("signature_contains_type", returnType, label, -1);
int numberOfParameters = (parts.length - 6) / 2; // includes type parameters
for (int i = 0; i < numberOfParameters; ++i) {
int partIndex = 6 + (2 * i);
String paramName = parts[partIndex];
String paramTypeId = parts[partIndex + 1];
if (paramTypeId.length() > 0) { // Unconstrained type parameters have an empty type ID.
Label paramType = trapWriter.globalID("type;" + parts[partIndex + 1]);
trapWriter.addTuple("signature_contains_type", paramType, label, i);
}
trapWriter.addTuple("signature_parameter_name", label, i, paramName);
}
}
private void extractSignatureMappings(JsonObject mappings) {
JsonArray baseTypes = mappings.get("baseTypes").getAsJsonArray();
JsonArray kinds = mappings.get("kinds").getAsJsonArray();
JsonArray indices = mappings.get("indices").getAsJsonArray();
JsonArray signatures = mappings.get("signatures").getAsJsonArray();
for (int i = 0; i < baseTypes.size(); ++i) {
int baseType = baseTypes.get(i).getAsInt();
int kind = kinds.get(i).getAsInt();
int index = indices.get(i).getAsInt();
int signatureId = signatures.get(i).getAsInt();
trapWriter.addTuple(
"type_contains_signature",
trapWriter.globalID("type;" + baseType),
kind,
index,
trapWriter.globalID("signature;" + signatureId));
}
}
private void extractIndexTypeTable(JsonObject table, String relationName) {
JsonArray baseTypes = table.get("baseTypes").getAsJsonArray();
JsonArray propertyTypes = table.get("propertyTypes").getAsJsonArray();
for (int i = 0; i < baseTypes.size(); ++i) {
int baseType = baseTypes.get(i).getAsInt();
int propertyType = propertyTypes.get(i).getAsInt();
trapWriter.addTuple(
relationName,
trapWriter.globalID("type;" + baseType),
trapWriter.globalID("type;" + propertyType));
}
}
private void extractBaseTypes(JsonObject table) {
JsonArray symbols = table.get("symbols").getAsJsonArray();
JsonArray baseTypeSymbols = table.get("baseTypeSymbols").getAsJsonArray();
for (int i = 0; i < symbols.size(); ++i) {
int symbolId = symbols.get(i).getAsInt();
int baseTypeSymbolId = baseTypeSymbols.get(i).getAsInt();
trapWriter.addTuple(
"base_type_names",
trapWriter.globalID("symbol;" + symbolId),
trapWriter.globalID("symbol;" + baseTypeSymbolId));
}
}
private void extractSelfTypes(JsonObject table) {
JsonArray symbols = table.get("symbols").getAsJsonArray();
JsonArray selfTypes = table.get("selfTypes").getAsJsonArray();
for (int i = 0; i < symbols.size(); ++i) {
int symbolId = symbols.get(i).getAsInt();
int typeId = selfTypes.get(i).getAsInt();
trapWriter.addTuple(
"self_types",
trapWriter.globalID("symbol;" + symbolId),
trapWriter.globalID("type;" + typeId));
}
}
/** Like {@link #split(String)} without a limit. */
private static String[] split(String input) {
return split(input, -1);
}
/**
* Splits the input around the semicolon (<code>;</code>) character, preserving all empty
* substrings.
*
* <p>At most <code>limit</code> substrings will be extracted. If the limit is reached, the last
* substring will extend to the end of the string, possibly itself containing semicolons.
*
* <p>Note that the {@link String#split(String)} method does not preserve empty substrings at the
* end of the string in case the string ends with a semicolon.
*/
private static String[] split(String input, int limit) {
List<String> result = new ArrayList<String>();
int lastPos = 0;
for (int i = 0; i < input.length(); ++i) {
if (input.charAt(i) == ';') {
result.add(input.substring(lastPos, i));
lastPos = i + 1;
if (result.size() == limit - 1) break;
}
}
result.add(input.substring(lastPos));
return result.toArray(EMPTY_STRING_ARRAY);
}
private static final String[] EMPTY_STRING_ARRAY = new String[0];
}

107
javascript/extractor/src/com/semmle/ts/extractor/TypeScriptParser.java
View File

@@ -463,113 +463,6 @@ public class TypeScriptParser {
checkResponseType(response, "ok");
}
/**
* Converts a map to an array of [key, value] pairs.
*/
private JsonArray mapToArray(Map<String, Path> map) {
JsonArray result = new JsonArray();
map.forEach(
(key, path) -> {
JsonArray entry = new JsonArray();
entry.add(key);
entry.add(path.toString());
result.add(entry);
});
return result;
}
private static Set<File> getFilesFromJsonArray(JsonArray array) {
Set<File> files = new LinkedHashSet<>();
for (JsonElement elm : array) {
files.add(new File(elm.getAsString()));
}
return files;
}
/**
* Returns the set of files included by the inclusion pattern in the given tsconfig.json file.
*/
public Set<File> getOwnFiles(File tsConfigFile, DependencyInstallationResult deps, VirtualSourceRoot vroot) {
JsonObject request = makeLoadCommand("get-own-files", tsConfigFile, deps, vroot);
JsonObject response = talkToParserWrapper(request);
try {
checkResponseType(response, "file-list");
return getFilesFromJsonArray(response.get("ownFiles").getAsJsonArray());
} catch (IllegalStateException e) {
throw new CatastrophicError(
"TypeScript parser wrapper sent unexpected response: " + response, e);
}
}
/**
* Opens a new project based on a tsconfig.json file. The compiler will analyze all files in the
* project.
*
* <p>Call {@link #parse} to access individual files in the project.
*
* <p>Only one project should be opened at once.
*/
public ParsedProject openProject(File tsConfigFile, DependencyInstallationResult deps, VirtualSourceRoot vroot) {
JsonObject request = makeLoadCommand("open-project", tsConfigFile, deps, vroot);
JsonObject response = talkToParserWrapper(request);
try {
checkResponseType(response, "project-opened");
ParsedProject project = new ParsedProject(tsConfigFile,
getFilesFromJsonArray(response.get("ownFiles").getAsJsonArray()),
getFilesFromJsonArray(response.get("allFiles").getAsJsonArray()));
return project;
} catch (IllegalStateException e) {
throw new CatastrophicError(
"TypeScript parser wrapper sent unexpected response: " + response, e);
}
}
private JsonObject makeLoadCommand(String command, File tsConfigFile, DependencyInstallationResult deps, VirtualSourceRoot vroot) {
JsonObject request = new JsonObject();
request.add("command", new JsonPrimitive(command));
request.add("tsConfig", new JsonPrimitive(tsConfigFile.getPath()));
request.add("packageEntryPoints", mapToArray(deps.getPackageEntryPoints()));
request.add("packageJsonFiles", mapToArray(deps.getPackageJsonFiles()));
request.add("sourceRoot", vroot.getSourceRoot() == null
? JsonNull.INSTANCE
: new JsonPrimitive(vroot.getSourceRoot().toString()));
request.add("virtualSourceRoot", vroot.getVirtualSourceRoot() == null
? JsonNull.INSTANCE
: new JsonPrimitive(vroot.getVirtualSourceRoot().toString()));
return request;
}
/**
* Closes a project previously opened.
*
* <p>This main purpose is to free heap space in the Node.js process.
*/
public void closeProject(File tsConfigFile) {
JsonObject request = new JsonObject();
request.add("command", new JsonPrimitive("close-project"));
request.add("tsConfig", new JsonPrimitive(tsConfigFile.getPath()));
JsonObject response = talkToParserWrapper(request);
try {
checkResponseType(response, "project-closed");
} catch (IllegalStateException e) {
throw new CatastrophicError(
"TypeScript parser wrapper sent unexpected response: " + response, e);
}
}
public TypeTable getTypeTable() {
JsonObject request = new JsonObject();
request.add("command", new JsonPrimitive("get-type-table"));
JsonObject response = talkToParserWrapper(request);
try {
checkResponseType(response, "type-table");
return new TypeTable(response.get("typeTable").getAsJsonObject());
} catch (IllegalStateException e) {
throw new CatastrophicError(
"TypeScript parser wrapper sent unexpected response: " + response, e);
}
}
/**
* Closes any open project, and in general, brings the TypeScript wrapper to a fresh state as if
* it had just been restarted.

119
javascript/extractor/src/com/semmle/ts/extractor/TypeTable.java
View File

@@ -1,119 +0,0 @@
package com.semmle.ts.extractor;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
/**
* Holds the output of the <code>get-type-table</code> command.
*
* <p>See documentation in <code>parser-wrapper/src/type_table.ts</code>.
*/
public class TypeTable {
private final JsonArray typeStrings;
private final JsonArray typeToStringValues;
private final JsonObject propertyLookups;
private final JsonObject typeAliases;
private final JsonArray symbolStrings;
private final JsonObject moduleMappings;
private final JsonObject globalMappings;
private final JsonArray signatureStrings;
private final JsonObject signatureMappings;
private final JsonArray signatureToStringValues;
private final JsonObject stringIndexTypes;
private final JsonObject numberIndexTypes;
private final JsonObject baseTypes;
private final JsonObject selfTypes;
public TypeTable(JsonObject typeTable) {
this.typeStrings = typeTable.get("typeStrings").getAsJsonArray();
this.typeToStringValues = typeTable.get("typeToStringValues").getAsJsonArray();
this.propertyLookups = typeTable.get("propertyLookups").getAsJsonObject();
this.typeAliases = typeTable.get("typeAliases").getAsJsonObject();
this.symbolStrings = typeTable.get("symbolStrings").getAsJsonArray();
this.moduleMappings = typeTable.get("moduleMappings").getAsJsonObject();
this.globalMappings = typeTable.get("globalMappings").getAsJsonObject();
this.signatureStrings = typeTable.get("signatureStrings").getAsJsonArray();
this.signatureMappings = typeTable.get("signatureMappings").getAsJsonObject();
this.signatureToStringValues = typeTable.get("signatureToStringValues").getAsJsonArray();
this.numberIndexTypes = typeTable.get("numberIndexTypes").getAsJsonObject();
this.stringIndexTypes = typeTable.get("stringIndexTypes").getAsJsonObject();
this.baseTypes = typeTable.get("baseTypes").getAsJsonObject();
this.selfTypes = typeTable.get("selfTypes").getAsJsonObject();
}
public String getTypeString(int index) {
return typeStrings.get(index).getAsString();
}
public String getTypeToStringValue(int index) {
return typeToStringValues.get(index).getAsString();
}
public JsonObject getPropertyLookups() {
return propertyLookups;
}
public JsonObject getTypeAliases() {
return typeAliases;
}
public int getNumberOfTypes() {
return typeStrings.size();
}
public String getSymbolString(int index) {
return symbolStrings.get(index).getAsString();
}
public int getNumberOfSymbols() {
return symbolStrings.size();
}
public JsonObject getModuleMappings() {
return moduleMappings;
}
public JsonObject getGlobalMappings() {
return globalMappings;
}
public JsonArray getSignatureStrings() {
return signatureStrings;
}
public int getNumberOfSignatures() {
return signatureStrings.size();
}
public String getSignatureString(int i) {
return signatureStrings.get(i).getAsString();
}
public JsonObject getSignatureMappings() {
return signatureMappings;
}
public JsonArray getSignatureToStringValues() {
return signatureToStringValues;
}
public String getSignatureToStringValue(int i) {
return signatureToStringValues.get(i).getAsString();
}
public JsonObject getNumberIndexTypes() {
return numberIndexTypes;
}
public JsonObject getStringIndexTypes() {
return stringIndexTypes;
}
public JsonObject getBaseTypes() {
return baseTypes;
}
public JsonObject getSelfTypes() {
return selfTypes;
}
}