From 74b817b6425d8caf2e49958d5cc032313deeb015 Mon Sep 17 00:00:00 2001 From: Asger F Date: Tue, 24 Jun 2025 17:16:06 +0200 Subject: [PATCH] JS: Remove code path for TypeScript full extraction --- .../lib/typescript/src/ast_extractor.ts | 206 +-- .../extractor/lib/typescript/src/common.ts | 104 +- .../extractor/lib/typescript/src/main.ts | 356 +---- .../lib/typescript/src/type_table.ts | 1423 ----------------- .../com/semmle/js/extractor/AutoBuild.java | 99 +- .../src/com/semmle/js/extractor/Main.java | 60 +- .../semmle/ts/extractor/TypeExtractor.java | 330 ---- .../semmle/ts/extractor/TypeScriptParser.java | 107 -- .../com/semmle/ts/extractor/TypeTable.java | 119 -- 9 files changed, 51 insertions(+), 2753 deletions(-) delete mode 100644 javascript/extractor/lib/typescript/src/type_table.ts delete mode 100644 javascript/extractor/src/com/semmle/ts/extractor/TypeExtractor.java delete mode 100644 javascript/extractor/src/com/semmle/ts/extractor/TypeTable.java diff --git a/javascript/extractor/lib/typescript/src/ast_extractor.ts b/javascript/extractor/lib/typescript/src/ast_extractor.ts index 8c34c9997ac..3dbc8a52d64 100644 --- a/javascript/extractor/lib/typescript/src/ast_extractor.ts +++ b/javascript/extractor/lib/typescript/src/ast_extractor.ts @@ -10,7 +10,6 @@ export interface AugmentedSourceFile extends ts.SourceFile { /** Internal property that we expose as a workaround. */ redirectInfo?: object | null; $tokens?: Token[]; - $symbol?: number; $lineStarts?: ReadonlyArray; } @@ -18,11 +17,6 @@ export interface AugmentedNode extends ts.Node { $pos?: any; $end?: any; $declarationKind?: string; - $type?: number; - $symbol?: number; - $resolvedSignature?: number; - $overloadIndex?: number; - $declaredSignature?: number; } export type AugmentedPos = number; @@ -73,7 +67,7 @@ function tryGetTypeOfNode(typeChecker: ts.TypeChecker, node: AugmentedNode): ts. } catch (e) { let sourceFile = node.getSourceFile(); let { line, character } = sourceFile.getLineAndCharacterOfPosition(node.pos); - console.warn(`Could not compute type of ${ts.SyntaxKind[node.kind]} at ${sourceFile.fileName}:${line+1}:${character+1}`); + console.warn(`Could not compute type of ${ts.SyntaxKind[node.kind]} at ${sourceFile.fileName}:${line + 1}:${character + 1}`); return null; } } @@ -157,17 +151,6 @@ export function augmentAst(ast: AugmentedSourceFile, code: string, project: Proj }); } - let typeChecker = project && project.program.getTypeChecker(); - let typeTable = project && project.typeTable; - - // Associate a symbol with the AST node root, in case it is a module. - if (typeTable != null) { - let symbol = typeChecker.getSymbolAtLocation(ast); - if (symbol != null) { - ast.$symbol = typeTable.getSymbolId(symbol); - } - } - visitAstNode(ast); function visitAstNode(node: AugmentedNode) { ts.forEachChild(node, visitAstNode); @@ -190,192 +173,5 @@ export function augmentAst(ast: AugmentedSourceFile, code: string, project: Proj node.$declarationKind = "var"; } } - - if (typeChecker != null) { - if (isTypedNode(node) && !typeTable.skipExtractingTypes) { - let contextualType = isContextuallyTypedNode(node) - ? typeChecker.getContextualType(node) - : null; - let type = contextualType || tryGetTypeOfNode(typeChecker, node); - if (type != null) { - let parent = node.parent; - let unfoldAlias = ts.isTypeAliasDeclaration(parent) && node === parent.type; - let id = typeTable.buildType(type, unfoldAlias); - if (id != null) { - node.$type = id; - } - } - // Extract the target call signature of a function call. - // In case the callee is overloaded or generic, this is not something we can - // derive from the callee type in QL. - if (ts.isCallOrNewExpression(node)) { - let kind = ts.isCallExpression(node) ? ts.SignatureKind.Call : ts.SignatureKind.Construct; - let resolvedSignature = typeChecker.getResolvedSignature(node); - if (resolvedSignature != null) { - let resolvedId = typeTable.getSignatureId(kind, resolvedSignature); - if (resolvedId != null) { - (node as AugmentedNode).$resolvedSignature = resolvedId; - } - let declaration = resolvedSignature.declaration; - if (declaration != null) { - // Find the generic signature, i.e. without call-site type arguments substituted, - // but with overloading resolved. - let calleeType = typeChecker.getTypeAtLocation(node.expression); - if (calleeType != null && declaration != null) { - let calleeSignatures = typeChecker.getSignaturesOfType(calleeType, kind); - for (let i = 0; i < calleeSignatures.length; ++i) { - if (calleeSignatures[i].declaration === declaration) { - (node as AugmentedNode).$overloadIndex = i; - break; - } - } - } - // Extract the symbol so the declaration can be found from QL. - let name = (declaration as any).name; - let symbol = name && typeChecker.getSymbolAtLocation(name); - if (symbol != null) { - (node as AugmentedNode).$symbol = typeTable.getSymbolId(symbol); - } - } - } - } - } - let symbolNode = - isNamedNodeWithSymbol(node) ? node.name : - ts.isImportDeclaration(node) ? node.moduleSpecifier : - ts.isExternalModuleReference(node) ? node.expression : - null; - if (symbolNode != null) { - let symbol = typeChecker.getSymbolAtLocation(symbolNode); - if (symbol != null) { - node.$symbol = typeTable.getSymbolId(symbol); - } - } - if (ts.isTypeReferenceNode(node)) { - // For type references we inject a symbol on each part of the name. - // We traverse each node in the name here since we know these are part of - // a type annotation. This means we don't have to do it for all identifiers - // and qualified names, which would extract more information than we need. - let namePart: (ts.EntityName & AugmentedNode) = node.typeName; - while (ts.isQualifiedName(namePart)) { - let symbol = typeChecker.getSymbolAtLocation(namePart.right); - if (symbol != null) { - namePart.$symbol = typeTable.getSymbolId(symbol); - } - - // Traverse into the prefix. - namePart = namePart.left; - } - let symbol = typeChecker.getSymbolAtLocation(namePart); - if (symbol != null) { - namePart.$symbol = typeTable.getSymbolId(symbol); - } - } - if (ts.isFunctionLike(node)) { - let signature = typeChecker.getSignatureFromDeclaration(node); - if (signature != null) { - let kind = ts.isConstructSignatureDeclaration(node) || ts.isConstructorDeclaration(node) - ? ts.SignatureKind.Construct : ts.SignatureKind.Call; - let id = typeTable.getSignatureId(kind, signature); - if (id != null) { - (node as AugmentedNode).$declaredSignature = id; - } - } - } - } } } - -type NamedNodeWithSymbol = AugmentedNode & (ts.ClassDeclaration | ts.InterfaceDeclaration - | ts.TypeAliasDeclaration | ts.EnumDeclaration | ts.EnumMember | ts.ModuleDeclaration | ts.FunctionDeclaration - | ts.MethodDeclaration | ts.MethodSignature); - -/** - * True if the given AST node has a name, and should be associated with a symbol. - */ -function isNamedNodeWithSymbol(node: ts.Node): node is NamedNodeWithSymbol { - switch (node.kind) { - case ts.SyntaxKind.ClassDeclaration: - case ts.SyntaxKind.InterfaceDeclaration: - case ts.SyntaxKind.TypeAliasDeclaration: - case ts.SyntaxKind.EnumDeclaration: - case ts.SyntaxKind.EnumMember: - case ts.SyntaxKind.ModuleDeclaration: - case ts.SyntaxKind.FunctionDeclaration: - case ts.SyntaxKind.MethodDeclaration: - case ts.SyntaxKind.MethodSignature: - return true; - } - return false; -} - -/** - * True if the given AST node has a type. - */ -function isTypedNode(node: ts.Node): boolean { - switch (node.kind) { - case ts.SyntaxKind.ArrayLiteralExpression: - case ts.SyntaxKind.ArrowFunction: - case ts.SyntaxKind.AsExpression: - case ts.SyntaxKind.SatisfiesExpression: - case ts.SyntaxKind.AwaitExpression: - case ts.SyntaxKind.BinaryExpression: - case ts.SyntaxKind.CallExpression: - case ts.SyntaxKind.ClassExpression: - case ts.SyntaxKind.ClassDeclaration: - case ts.SyntaxKind.CommaListExpression: - case ts.SyntaxKind.ConditionalExpression: - case ts.SyntaxKind.Constructor: - case ts.SyntaxKind.DeleteExpression: - case ts.SyntaxKind.ElementAccessExpression: - case ts.SyntaxKind.ExpressionStatement: - case ts.SyntaxKind.ExpressionWithTypeArguments: - case ts.SyntaxKind.FalseKeyword: - case ts.SyntaxKind.FunctionDeclaration: - case ts.SyntaxKind.FunctionExpression: - case ts.SyntaxKind.GetAccessor: - case ts.SyntaxKind.Identifier: - case ts.SyntaxKind.IndexSignature: - case ts.SyntaxKind.JsxExpression: - case ts.SyntaxKind.LiteralType: - case ts.SyntaxKind.MethodDeclaration: - case ts.SyntaxKind.MethodSignature: - case ts.SyntaxKind.NewExpression: - case ts.SyntaxKind.NonNullExpression: - case ts.SyntaxKind.NoSubstitutionTemplateLiteral: - case ts.SyntaxKind.NumericLiteral: - case ts.SyntaxKind.ObjectKeyword: - case ts.SyntaxKind.ObjectLiteralExpression: - case ts.SyntaxKind.OmittedExpression: - case ts.SyntaxKind.ParenthesizedExpression: - case ts.SyntaxKind.PartiallyEmittedExpression: - case ts.SyntaxKind.PostfixUnaryExpression: - case ts.SyntaxKind.PrefixUnaryExpression: - case ts.SyntaxKind.PropertyAccessExpression: - case ts.SyntaxKind.RegularExpressionLiteral: - case ts.SyntaxKind.SetAccessor: - case ts.SyntaxKind.StringLiteral: - case ts.SyntaxKind.TaggedTemplateExpression: - case ts.SyntaxKind.TemplateExpression: - case ts.SyntaxKind.TemplateHead: - case ts.SyntaxKind.TemplateMiddle: - case ts.SyntaxKind.TemplateSpan: - case ts.SyntaxKind.TemplateTail: - case ts.SyntaxKind.TrueKeyword: - case ts.SyntaxKind.TypeAssertionExpression: - case ts.SyntaxKind.TypeLiteral: - case ts.SyntaxKind.TypeOfExpression: - case ts.SyntaxKind.VoidExpression: - case ts.SyntaxKind.YieldExpression: - return true; - default: - return ts.isTypeNode(node); - } -} - -type ContextuallyTypedNode = (ts.ArrayLiteralExpression | ts.ObjectLiteralExpression) & AugmentedNode; - -function isContextuallyTypedNode(node: ts.Node): node is ContextuallyTypedNode { - let kind = node.kind; - return kind === ts.SyntaxKind.ArrayLiteralExpression || kind === ts.SyntaxKind.ObjectLiteralExpression; -} diff --git a/javascript/extractor/lib/typescript/src/common.ts b/javascript/extractor/lib/typescript/src/common.ts index c660b7bcb5e..9276a567bb0 100644 --- a/javascript/extractor/lib/typescript/src/common.ts +++ b/javascript/extractor/lib/typescript/src/common.ts @@ -1,54 +1,26 @@ import * as ts from "./typescript"; -import { TypeTable } from "./type_table"; -import * as pathlib from "path"; -import { VirtualSourceRoot } from "./virtual_source_root"; - -/** - * Extracts the package name from the prefix of an import string. - */ -const packageNameRex = /^(?:@[\w.-]+[/\\]+)?\w[\w.-]*(?=[/\\]|$)/; -const extensions = ['.ts', '.tsx', '.d.ts', '.js', '.jsx']; - -function getPackageName(importString: string) { - let packageNameMatch = packageNameRex.exec(importString); - if (packageNameMatch == null) return null; - let packageName = packageNameMatch[0]; - if (packageName.charAt(0) === '@') { - packageName = packageName.replace(/[/\\]+/g, '/'); // Normalize slash after the scope. - } - return packageName; -} export class Project { public program: ts.Program = null; private host: ts.CompilerHost; - private resolutionCache: ts.ModuleResolutionCache; constructor( - public tsConfig: string, - public config: ts.ParsedCommandLine, - public typeTable: TypeTable, - public packageEntryPoints: Map, - public virtualSourceRoot: VirtualSourceRoot) { + public tsConfig: string, + public config: ts.ParsedCommandLine, + public packageEntryPoints: Map) { - this.resolveModuleNames = this.resolveModuleNames.bind(this); - - this.resolutionCache = ts.createModuleResolutionCache(pathlib.dirname(tsConfig), ts.sys.realpath, config.options); let host = ts.createCompilerHost(config.options, true); - host.resolveModuleNames = this.resolveModuleNames; host.trace = undefined; // Disable tracing which would otherwise go to standard out this.host = host; } public unload(): void { - this.typeTable.releaseProgram(); this.program = null; } public load(): void { const { config, host } = this; this.program = ts.createProgram(config.fileNames, config.options, host); - this.typeTable.setProgram(this.program, this.virtualSourceRoot); } /** @@ -60,74 +32,4 @@ export class Project { this.unload(); this.load(); } - - /** - * Override for module resolution in the TypeScript compiler host. - */ - private resolveModuleNames( - moduleNames: string[], - containingFile: string, - reusedNames: string[], - redirectedReference: ts.ResolvedProjectReference, - options: ts.CompilerOptions) { - - let oppositePath = - this.virtualSourceRoot.toVirtualPath(containingFile) || - this.virtualSourceRoot.fromVirtualPath(containingFile); - - const { host, resolutionCache } = this; - return moduleNames.map((moduleName) => { - let redirected = this.redirectModuleName(moduleName, containingFile, options); - if (redirected != null) return redirected; - if (oppositePath != null) { - // If the containing file is in the virtual source root, try resolving from the real source root, and vice versa. - redirected = ts.resolveModuleName(moduleName, oppositePath, options, host, resolutionCache).resolvedModule; - if (redirected != null) return redirected; - } - return ts.resolveModuleName(moduleName, containingFile, options, host, resolutionCache).resolvedModule; - }); - } - - /** - * Returns the path that the given import string should be redirected to, or null if it should - * fall back to standard module resolution. - */ - private redirectModuleName(moduleName: string, containingFile: string, options: ts.CompilerOptions): ts.ResolvedModule { - // Get a package name from the leading part of the module name, e.g. '@scope/foo' from '@scope/foo/bar'. - let packageName = getPackageName(moduleName); - if (packageName == null) return null; - - // Get the overridden location of this package, if one exists. - let packageEntryPoint = this.packageEntryPoints.get(packageName); - if (packageEntryPoint == null) return null; - - // If the requested module name is exactly the overridden package name, - // return the entry point file (it is not necessarily called `index.ts`). - if (moduleName === packageName) { - return { resolvedFileName: packageEntryPoint, isExternalLibraryImport: true }; - } - - // Get the suffix after the package name, e.g. the '/bar' in '@scope/foo/bar'. - let suffix = moduleName.substring(packageName.length); - - // Resolve the suffix relative to the package directory. - let packageDir = pathlib.dirname(packageEntryPoint); - let joinedPath = pathlib.join(packageDir, suffix); - - // Add implicit '/index' - if (ts.sys.directoryExists(joinedPath)) { - joinedPath = pathlib.join(joinedPath, 'index'); - } - - // Try each recognized extension. We must not return a file whose extension is not - // recognized by TypeScript. - for (let ext of extensions) { - let candidate = joinedPath.endsWith(ext) ? joinedPath : (joinedPath + ext); - if (ts.sys.fileExists(candidate)) { - return { resolvedFileName: candidate, isExternalLibraryImport: true }; - } - } - - return null; - } } diff --git a/javascript/extractor/lib/typescript/src/main.ts b/javascript/extractor/lib/typescript/src/main.ts index d34f516cf55..eb888a00ea8 100644 --- a/javascript/extractor/lib/typescript/src/main.ts +++ b/javascript/extractor/lib/typescript/src/main.ts @@ -31,14 +31,12 @@ "use strict"; -import * as fs from "fs"; import * as pathlib from "path"; import * as readline from "readline"; import * as ts from "./typescript"; import * as ast_extractor from "./ast_extractor"; import { Project } from "./common"; -import { TypeTable } from "./type_table"; import { VirtualSourceRoot } from "./virtual_source_root"; // Remove limit on stack trace depth. @@ -55,19 +53,6 @@ interface LoadCommand { packageEntryPoints: [string, string][]; packageJsonFiles: [string, string][]; } -interface OpenProjectCommand extends LoadCommand { - command: "open-project"; -} -interface GetOwnFilesCommand extends LoadCommand { - command: "get-own-files"; -} -interface CloseProjectCommand { - command: "close-project"; - tsConfig: string; -} -interface GetTypeTableCommand { - command: "get-type-table"; -} interface ResetCommand { command: "reset"; } @@ -81,13 +66,11 @@ interface PrepareFilesCommand { interface GetMetadataCommand { command: "get-metadata"; } -type Command = ParseCommand | OpenProjectCommand | GetOwnFilesCommand | CloseProjectCommand - | GetTypeTableCommand | ResetCommand | QuitCommand | PrepareFilesCommand | GetMetadataCommand; +type Command = ParseCommand | ResetCommand | QuitCommand | PrepareFilesCommand | GetMetadataCommand; /** The state to be shared between commands. */ class State { public project: Project = null; - public typeTable = new TypeTable(); /** List of files that have been requested. */ public pendingFiles: string[] = []; @@ -205,22 +188,18 @@ function checkCycle(root: any) { visit(root); if (path.length > 0) { path.reverse(); - console.log(JSON.stringify({type: "error", message: "Cycle = " + path.join(".")})); + console.log(JSON.stringify({ type: "error", message: "Cycle = " + path.join(".") })); } } /** Property names to extract from the TypeScript AST. */ const astProperties: string[] = [ "$declarationKind", - "$declaredSignature", "$end", "$lineStarts", "$overloadIndex", "$pos", - "$resolvedSignature", - "$symbol", "$tokens", - "$type", "argument", "argumentExpression", "arguments", @@ -392,20 +371,12 @@ function isExtractableSourceFile(ast: ast_extractor.AugmentedSourceFile): boolea * an already-open project, or by parsing the file. */ function getAstForFile(filename: string): ts.SourceFile { - if (state.project != null) { - let ast = state.project.program.getSourceFile(filename); - if (ast != null && isExtractableSourceFile(ast)) { - ast_extractor.augmentAst(ast, ast.text, state.project); - return ast; - } - } - // Fall back to extracting without a project. - let {ast, code} = parseSingleFile(filename); + let { ast, code } = parseSingleFile(filename); ast_extractor.augmentAst(ast, code, null); return ast; } -function parseSingleFile(filename: string): {ast: ts.SourceFile, code: string} { +function parseSingleFile(filename: string): { ast: ts.SourceFile, code: string } { let code = ts.sys.readFile(filename); // create a compiler host that only allows access to `filename` @@ -436,7 +407,7 @@ function parseSingleFile(filename: string): {ast: ts.SourceFile, code: string} { let ast = program.getSourceFile(filename); - return {ast, code}; + return { ast, code }; } /** @@ -507,7 +478,7 @@ function loadTsConfig(command: LoadCommand): LoadedConfig { let virtualExclusions = excludes == null ? [] : [...excludes]; virtualExclusions.push('**/node_modules/**/*'); let virtualResults = ts.sys.readDirectory(virtualDir, extensions, virtualExclusions, includes, depth) - return [ ...originalResults, ...virtualResults ]; + return [...originalResults, ...virtualResults]; }, fileExists: (path: string) => { return ts.sys.fileExists(path) @@ -531,256 +502,6 @@ function loadTsConfig(command: LoadCommand): LoadedConfig { return { config, basePath, packageJsonFiles, packageEntryPoints, virtualSourceRoot, ownFiles }; } -/** - * Returns the list of files included in the given tsconfig.json file's include pattern, - * (not including those only references through imports). - */ -function handleGetFileListCommand(command: GetOwnFilesCommand) { - let { config, ownFiles } = loadTsConfig(command); - - console.log(JSON.stringify({ - type: "file-list", - ownFiles, - })); -} - -function handleOpenProjectCommand(command: OpenProjectCommand) { - let { config, packageEntryPoints, virtualSourceRoot, basePath, ownFiles } = loadTsConfig(command); - - let project = new Project(command.tsConfig, config, state.typeTable, packageEntryPoints, virtualSourceRoot); - project.load(); - - state.project = project; - let program = project.program; - let typeChecker = program.getTypeChecker(); - - let shouldReportDiagnostics = getEnvironmentVariable("SEMMLE_TYPESCRIPT_REPORT_DIAGNOSTICS", v => v.trim().toLowerCase() === "true", false); - let diagnostics = shouldReportDiagnostics - ? program.getSemanticDiagnostics().filter(d => d.category === ts.DiagnosticCategory.Error) - : []; - if (diagnostics.length > 0) { - console.warn('TypeScript: reported ' + diagnostics.length + ' semantic errors.'); - } - for (let diagnostic of diagnostics) { - let text = diagnostic.messageText; - if (text && typeof text !== 'string') { - text = text.messageText; - } - let locationStr = ''; - let { file } = diagnostic; - if (file != null) { - let { line, character } = file.getLineAndCharacterOfPosition(diagnostic.start); - locationStr = `${file.fileName}:${line}:${character}`; - } - console.warn(`TypeScript: ${locationStr} ${text}`); - } - - // Associate external module names with the corresponding file symbols. - // We need these mappings to identify which module a given external type comes from. - // The TypeScript API lets us resolve a module name to a source file, but there is no - // inverse mapping, nor a way to enumerate all known module names. So we discover all - // modules on the type roots (usually "node_modules/@types" but this is configurable). - let typeRoots = ts.getEffectiveTypeRoots(config.options, { - getCurrentDirectory: () => basePath, - }); - - for (let typeRoot of typeRoots || []) { - if (ts.sys.directoryExists(typeRoot)) { - traverseTypeRoot(typeRoot, ""); - } - let virtualTypeRoot = virtualSourceRoot.toVirtualPathIfDirectoryExists(typeRoot); - if (virtualTypeRoot != null) { - traverseTypeRoot(virtualTypeRoot, ""); - } - } - - for (let sourceFile of program.getSourceFiles()) { - addModuleBindingsFromModuleDeclarations(sourceFile); - addModuleBindingsFromFilePath(sourceFile); - } - - /** Concatenates two imports paths. These always use `/` as path separator. */ - function joinModulePath(prefix: string, suffix: string) { - if (prefix.length === 0) return suffix; - if (suffix.length === 0) return prefix; - return prefix + "/" + suffix; - } - - /** - * Traverses a directory that is a type root or contained in a type root, and associates - * module names (i.e. import strings) with files in this directory. - * - * `importPrefix` denotes an import string that resolves to this directory, - * or an empty string if the file itself is a type root. - * - * The `filePath` is a system file path, possibly absolute, whereas `importPrefix` - * is generally short and system-independent, typically just the name of a module. - */ - function traverseTypeRoot(filePath: string, importPrefix: string) { - for (let child of fs.readdirSync(filePath)) { - if (child[0] === ".") continue; - let childPath = pathlib.join(filePath, child); - if (fs.statSync(childPath).isDirectory()) { - traverseTypeRoot(childPath, joinModulePath(importPrefix, child)); - continue; - } - let sourceFile = program.getSourceFile(childPath); - if (sourceFile == null) { - continue; - } - let importPath = getImportPathFromFileInFolder(importPrefix, child); - addModuleBindingFromImportPath(sourceFile, importPath); - } - } - - function getImportPathFromFileInFolder(folder: string, baseName: string) { - let stem = getStem(baseName); - return (stem === "index") - ? folder - : joinModulePath(folder, stem); - } - - /** - * Emits module bindings for a module with relative path `folder/baseName`. - */ - function addModuleBindingFromImportPath(sourceFile: ts.SourceFile, importPath: string) { - let symbol = typeChecker.getSymbolAtLocation(sourceFile); - if (symbol == null) return; // Happens if the source file is not a module. - - let canonicalSymbol = getEffectiveExportTarget(symbol); // Follow `export = X` declarations. - let symbolId = state.typeTable.getSymbolId(canonicalSymbol); - - // Associate the module name with this symbol. - state.typeTable.addModuleMapping(symbolId, importPath); - - // Associate global variable names with this module. - // For each `export as X` declaration, the global X refers to this module. - // Note: the `globalExports` map is stored on the original symbol, not the target of `export=`. - if (symbol.globalExports != null) { - symbol.globalExports.forEach((global: ts.Symbol) => { - state.typeTable.addGlobalMapping(symbolId, global.name); - }); - } - } - - /** - * Returns the basename of `file` without its extension, while treating `.d.ts` as a - * single extension. - */ - function getStem(file: string) { - if (file.endsWith(".d.ts")) { - return pathlib.basename(file, ".d.ts"); - } - if (file.endsWith(".d.mts") || file.endsWith(".d.cts")) { - // We don't extract d.mts or d.cts files, but their symbol can coincide with that of a d.ts file, - // which means any module bindings we generate for it will ultimately be visible in QL. - let base = pathlib.basename(file); - return base.substring(0, base.length - '.d.mts'.length); - } - let base = pathlib.basename(file); - let dot = base.lastIndexOf('.'); - return dot === -1 || dot === 0 ? base : base.substring(0, dot); - } - - /** - * Emits module bindings for a module based on its file path. - * - * This looks for enclosing `node_modules` folders to determine the module name. - * This is needed for modules that ship their own type definitions as opposed to having - * type definitions loaded from a type root (conventionally named `@types/xxx`). - */ - function addModuleBindingsFromFilePath(sourceFile: ts.SourceFile) { - let fullPath = sourceFile.fileName; - let index = fullPath.lastIndexOf('/node_modules/'); - if (index === -1) return; - - let relativePath = fullPath.substring(index + '/node_modules/'.length); - - // Ignore node_modules/@types folders here as they are typically handled as type roots. - if (relativePath.startsWith("@types/")) return; - - // If the enclosing package has a "typings" field, only add module bindings for that file. - let packageJsonFile = getEnclosingPackageJson(fullPath); - if (packageJsonFile != null) { - let json = getPackageJson(packageJsonFile); - let typings = getPackageTypings(packageJsonFile); - if (json != null && typings != null) { - let name = json.name; - if (typings === fullPath && typeof name === 'string') { - addModuleBindingFromImportPath(sourceFile, name); - } else if (typings != null) { - return; // Typings field prevents access to other files in package. - } - } - } - - // Add module bindings relative to package directory. - let { dir, base } = pathlib.parse(relativePath); - addModuleBindingFromImportPath(sourceFile, getImportPathFromFileInFolder(dir, base)); - } - - /** - * Emit module name bindings for external module declarations, i.e: `declare module 'X' {..}` - * These can generally occur anywhere; they may or may not be on the type root path. - */ - function addModuleBindingsFromModuleDeclarations(sourceFile: ts.SourceFile) { - for (let stmt of sourceFile.statements) { - if (ts.isModuleDeclaration(stmt) && ts.isStringLiteral(stmt.name)) { - let symbol = (stmt as any).symbol; - if (symbol == null) continue; - symbol = getEffectiveExportTarget(symbol); - let symbolId = state.typeTable.getSymbolId(symbol); - let moduleName = stmt.name.text; - state.typeTable.addModuleMapping(symbolId, moduleName); - } - } - } - - /** - * If `symbol` refers to a container with an `export = X` declaration, returns - * the target of `X`, otherwise returns `symbol`. - */ - function getEffectiveExportTarget(symbol: ts.Symbol) { - if (symbol.exports != null && symbol.exports.has(ts.InternalSymbolName.ExportEquals)) { - let exportAlias = symbol.exports.get(ts.InternalSymbolName.ExportEquals); - if (exportAlias.flags & ts.SymbolFlags.Alias) { - return typeChecker.getAliasedSymbol(exportAlias); - } - } - return symbol; - } - - // Unlike in the get-own-files command, this command gets all files we can possibly - // extract type information for, including files referenced outside the tsconfig's inclusion pattern. - let allFiles = program.getSourceFiles().map(sf => pathlib.resolve(sf.fileName)); - - console.log(JSON.stringify({ - type: "project-opened", - ownFiles, - allFiles, - })); -} - -function handleCloseProjectCommand(command: CloseProjectCommand) { - if (state.project == null) { - console.log(JSON.stringify({ - type: "error", - message: "No project is open", - })); - return; - } - state.project.unload(); - state.project = null; - console.log(JSON.stringify({type: "project-closed"})); -} - -function handleGetTypeTableCommand(command: GetTypeTableCommand) { - console.log(JSON.stringify({ - type: "type-table", - typeTable: state.typeTable.getTypeTableJson(), - })); -} - function handleResetCommand(command: ResetCommand) { reset(); console.log(JSON.stringify({ @@ -807,8 +528,6 @@ function handleGetMetadataCommand(command: GetMetadataCommand) { function reset() { state = new State(); - state.typeTable.restrictedExpansion = getEnvironmentVariable("SEMMLE_TYPESCRIPT_NO_EXPANSION", v => v.trim().toLowerCase() === "true", true); - state.typeTable.skipExtractingTypes = getEnvironmentVariable("CODEQL_EXTRACTOR_JAVASCRIPT_OPTION_SKIP_TYPES", v => v.trim().toLowerCase() === "true", false); } function getEnvironmentVariable(name: string, parse: (x: string) => T, defaultValue: T) { @@ -848,35 +567,23 @@ function runReadLineInterface() { rl.on("line", (line: string) => { let req: Command = JSON.parse(line); switch (req.command) { - case "parse": - handleParseCommand(req); - break; - case "open-project": - handleOpenProjectCommand(req); - break; - case "get-own-files": - handleGetFileListCommand(req); - break; - case "close-project": - handleCloseProjectCommand(req); - break; - case "get-type-table": - handleGetTypeTableCommand(req); - break; - case "prepare-files": - handlePrepareFilesCommand(req); - break; - case "reset": - handleResetCommand(req); - break; - case "get-metadata": - handleGetMetadataCommand(req); - break; - case "quit": - rl.close(); - break; - default: - throw new Error("Unknown command " + (req as any).command + "."); + case "parse": + handleParseCommand(req); + break; + case "prepare-files": + handlePrepareFilesCommand(req); + break; + case "reset": + handleResetCommand(req); + break; + case "get-metadata": + handleGetMetadataCommand(req); + break; + case "quit": + rl.close(); + break; + default: + throw new Error("Unknown command " + (req as any).command + "."); } }); } @@ -886,23 +593,6 @@ if (process.argv.length > 2) { let argument = process.argv[2]; if (argument === "--version") { console.log("parser-wrapper with TypeScript " + ts.version); - } else if (pathlib.basename(argument) === "tsconfig.json") { - reset(); - handleOpenProjectCommand({ - command: "open-project", - tsConfig: argument, - packageEntryPoints: [], - packageJsonFiles: [], - sourceRoot: null, - virtualSourceRoot: null, - }); - for (let sf of state.project.program.getSourceFiles()) { - if (/lib\..*\.d\.ts/.test(pathlib.basename(sf.fileName)) || pathlib.basename(sf.fileName) === "lib.d.ts") continue; - handleParseCommand({ - command: "parse", - filename: sf.fileName, - }, false); - } } else if (pathlib.extname(argument) === ".ts" || pathlib.extname(argument) === ".tsx") { handleParseCommand({ command: "parse", diff --git a/javascript/extractor/lib/typescript/src/type_table.ts b/javascript/extractor/lib/typescript/src/type_table.ts deleted file mode 100644 index 4a00dfcc9c7..00000000000 --- a/javascript/extractor/lib/typescript/src/type_table.ts +++ /dev/null @@ -1,1423 +0,0 @@ -import * as ts from "./typescript"; -import { VirtualSourceRoot } from "./virtual_source_root"; - -interface AugmentedSymbol extends ts.Symbol { - parent?: AugmentedSymbol; - - /** Cache of our own symbol ID. */ - $id?: number; -} - -interface AugmentedType extends ts.Type { - /** - * An internal property for predefined types, such as "true", "false", and "object". - */ - intrinsicName?: string; -} - -function isTypeReference(type: ts.Type): type is ts.TypeReference { - return (type.flags & ts.TypeFlags.Object) !== 0 && - ((type as ts.ObjectType).objectFlags & ts.ObjectFlags.Reference) !== 0; -} - -function isTypeVariable(type: ts.Type): type is ts.TypeVariable { - return (type.flags & ts.TypeFlags.TypeVariable) !== 0; -} - -/** - * Returns `true` if the properties of the given type can safely be extracted - * without restricting expansion depth. - * - * This predicate is very approximate, and considers all unions, intersections, - * named types, and mapped types as potentially unsafe. - */ -function isTypeAlwaysSafeToExpand(type: ts.Type): boolean { - let flags = type.flags; - if (flags & ts.TypeFlags.UnionOrIntersection) { - return false; - } - if (flags & ts.TypeFlags.Object) { - let objectType = type as ts.ObjectType; - let objectFlags = objectType.objectFlags; - if (objectFlags & (ts.ObjectFlags.Reference | ts.ObjectFlags.Mapped)) { - return false; - } - } - if (type.aliasSymbol != null) { - return false; - } - return true; -} - -/** - * If `type` is a `this` type, returns the enclosing type. - * Otherwise returns `null`. - */ -function getEnclosingTypeOfThisType(type: ts.TypeVariable): ts.TypeReference { - // A 'this' type is an implicit type parameter to a class or interface. - // The type parameter itself doesn't have any good indicator of being a 'this' type, - // but we can get it like this: - // - the upper bound of the 'this' type parameter is always the enclosing type - // - the enclosing type knows its own 'this' type. - let bound = type.getConstraint(); - if (bound == null) return null; - let target = (bound as ts.TypeReference).target; // undefined if not a TypeReference - if (target == null) return null; - return (target.thisType === type) ? target : null; -} - -const typeDefinitionSymbols = ts.SymbolFlags.Class | ts.SymbolFlags.Interface | - ts.SymbolFlags.TypeAlias | ts.SymbolFlags.EnumMember | ts.SymbolFlags.Enum; - -/** Returns true if the given symbol refers to a type definition. */ -function isTypeDefinitionSymbol(symbol: ts.Symbol) { - return (symbol.flags & typeDefinitionSymbols) !== 0; -} - -/** Gets the nearest enclosing block statement, function body, module body, or top-level. */ -function getEnclosingBlock(node: ts.Node) { - while (true) { - if (node == null) return null; - if (ts.isSourceFile(node) || ts.isFunctionLike(node) || ts.isBlock(node) || ts.isModuleBlock(node)) return node; - node = node.parent; - } -} - -const typeofSymbols = ts.SymbolFlags.Class | ts.SymbolFlags.Namespace | - ts.SymbolFlags.Module | ts.SymbolFlags.Enum | ts.SymbolFlags.EnumMember; - -/** - * Returns true if the given symbol refers to a value that we consider - * a valid target for a `typeof` type. - */ -function isTypeofCandidateSymbol(symbol: ts.Symbol) { - return (symbol.flags & typeofSymbols) !== 0; -} - -const signatureKinds = [ts.SignatureKind.Call, ts.SignatureKind.Construct]; - -/** - * Bitmask of flags set on a signature, but not exposed in the public API. - */ -const enum InternalSignatureFlags { - HasRestParameter = 1 -} - -/** - * Signature interface with some internal properties exposed. - */ -interface AugmentedSignature extends ts.Signature { - flags?: InternalSignatureFlags; -} - -/** - * Encodes property lookup tuples `(baseType, name, property)` as three - * staggered arrays. - */ -interface PropertyLookupTable { - baseTypes: number[]; - names: string[]; - propertyTypes: number[]; -} - -/** - * Encodes `(aliasType, underlyingType)` tuples as two staggered arrays. - * - * Such a tuple denotes that `aliasType` is an alias for `underlyingType`. - */ -interface TypeAliasTable { - aliasTypes: number[]; - underlyingTypes: number[]; -} - -/** - * Encodes type signature tuples `(baseType, kind, index, signature)` as four - * staggered arrays. - */ -interface SignatureTable { - baseTypes: number[]; - kinds: ts.SignatureKind[]; - indices: number[]; - signatures: number[]; -} - -/** - * Enodes `(baseType, propertyType)` tuples as two staggered arrays. - * - * The index key type is not stored in the table - there are separate tables - * for number and string index signatures. - * - * For example, the `(Foo, T)` tuple would be extracted from this sample: - * ``` - * interface Foo { - * [x: string]: T; - * } - * ``` - */ -interface IndexerTable { - baseTypes: number[]; - propertyTypes: number[]; -} - -/** - * Encodes `(symbol, name)` pairs as two staggered arrays. - * - * In general, a table may associate multiple names with a given symbol. - */ -interface SymbolNameTable { - symbols: number[]; - names: string[]; -} - -/** - * Encodes `(symbol, baseTypeSymbol)` pairs as two staggered arrays. - * - * Such a pair associates the canonical name of a type with the canonical name - * of one of its base types. - */ -interface BaseTypeTable { - symbols: number[]; - baseTypeSymbols: number[]; -} - -/** - * Encodes `(symbol, selfType)` pairs as two staggered arrays. - * - * Such a pair associates the canonical name of a type with the self-type of - * that type definition. (e.g `Array` with `Array`). - */ -interface SelfTypeTable { - symbols: number[]; - selfTypes: number[]; -} - -/** - * Denotes whether a type is currently in the worklist ("pending") and whether - * it was discovered in shallow or full context. - * - * Types can be discovered in one of two different contexts: - * - Full context: - * Any type that is the type of an AST node, or is reachable through members of - * such a type, without going through an expansive type. - * - Shallow context: - * Any type that is reachable through the members of an expansive type, - * without following any type references after that. - * - * For example: - * ``` - * interface Expansive { - * expand: Expansive<{x: T}>; - * foo: { bar: T }; - * } - * let instance: Expansive; - * ``` - * The type `Expansive` is discovered in full context, but as it is expansive, - * its members are only discovered in shallow context. - * - * This means `Expansive<{x: number}>` becomes a stub type, a type that has an entity in - * the database, but appears to have no members. - * - * The type `{ bar: number }` is also discovered in shallow context, but because it is - * an "inline type" (not a reference) its members are extracted anyway (in shallow context), - * and will thus appear to have the `bar` property of type `number`. - */ -const enum TypeExtractionState { - /** - * The type is in the worklist and was discovered in shallow context. - */ - PendingShallow, - - /** - * The type has been extracted as a shallow type. - * - * It may later transition to `PendingFull` if it is found that full extraction is warranted. - */ - DoneShallow, - - /** - * The type is in the worklist and is pending full extraction. - */ - PendingFull, - - /** - * The type has been fully extracted. - */ - DoneFull, -} - -/** - * Generates canonical IDs and serialized representations of types. - */ -export class TypeTable { - /** - * Maps type strings to type IDs. The types must be inserted in order, - * so the `n`th type has ID `n`. - * - * A type string is a `;`-separated string consisting of: - * - a tag string such as `union` or `reference`, - * - optionally a symbol ID or kind-specific data (depends on the tag), - * - IDs of child types. - * - * Type strings serve a dual purpose: - * - Canonicalizing types. Two type objects with the same type string are considered identical. - * - Extracting types. The Java-part of the extractor parses type strings to extract data about the type. - */ - private typeIds: Map = new Map(); - private typeToStringValues: string[] = []; - private typeChecker: ts.TypeChecker = null; - - /** - * Needed for TypeChecker.getTypeOfSymbolAtLocation when we don't care about the location. - * There is no way to get the type of a symbol without providing a location, though. - */ - private arbitraryAstNode: ts.Node = null; - - /** - * Maps symbol strings to to symbol IDs. The symbols must be inserted in order, - * so the `n`th symbol has ID `n`. - * - * A symbol string is a `;`-separated string consisting of: - * - a tag string, `root`, `member`, or `other`, - * - an empty string or a `file:pos` string to distinguish this from symbols with other lexical roots, - * - the ID of the parent symbol, or an empty string if this is a root symbol, - * - the unqualified name of the symbol. - * - * Symbol strings serve the same dual purpose as type strings (see `typeIds`). - */ - private symbolIds: Map = new Map(); - - /** - * Maps file names to IDs unique for that file name. - * - * Used to generate short `file:pos` strings in symbol strings. - */ - private fileIds: Map = new Map(); - - /** - * Maps signature strings to signature IDs. The signatures must be inserted in order, - * so the `n`th signature has ID `n`. - * - * A signature string is a `;`-separated string consisting of: - * - a `ts.SignatureKind` value (i.e. the value 0 or 1) - * - number of type parameters - * - number of required parameters - * - ID of the return type - * - interleaved names and bounds (type IDs) of type parameters - * - interleaved names and type IDs of parameters - */ - private signatureIds: Map = new Map(); - - private signatureToStringValues: string[] = []; - - private propertyLookups: PropertyLookupTable = { - baseTypes: [], - names: [], - propertyTypes: [], - }; - - private typeAliases: TypeAliasTable = { - aliasTypes: [], - underlyingTypes: [], - }; - - private signatureMappings: SignatureTable = { - baseTypes: [], - kinds: [], - indices: [], - signatures: [] - }; - - private numberIndexTypes: IndexerTable = { - baseTypes: [], - propertyTypes: [], - }; - - private stringIndexTypes: IndexerTable = { - baseTypes: [], - propertyTypes: [], - }; - - private buildTypeWorklist: [ts.Type, number, boolean][] = []; - - private expansiveTypes: Map = new Map(); - - private moduleMappings: SymbolNameTable = { - symbols: [], - names: [], - }; - private globalMappings: SymbolNameTable = { - symbols: [], - names: [], - }; - - private baseTypes: BaseTypeTable = { - symbols: [], - baseTypeSymbols: [], - }; - - private selfTypes: SelfTypeTable = { - symbols: [], - selfTypes: [], - }; - - /** - * When true, newly discovered types should be extracted as "shallow" types in order - * to prevent expansive types from unfolding into infinitely many types. - * - * @see TypeExtractionState - */ - private isInShallowTypeContext = false; - - /** - * Maps a type ID to the extraction state of that type. - */ - private typeExtractionState: TypeExtractionState[] = []; - - /** - * Number of types we are currently in the process of flattening to a type string. - */ - private typeRecursionDepth = 0; - - /** - * If set to true, all types are considered expansive. - */ - public restrictedExpansion = false; - - /** - * If set to true, skip extracting types. - */ - public skipExtractingTypes = false; - - private virtualSourceRoot: VirtualSourceRoot; - - /** - * Called when a new compiler instance has started. - */ - public setProgram(program: ts.Program, virtualSourceRoot: VirtualSourceRoot) { - this.typeChecker = program.getTypeChecker(); - this.arbitraryAstNode = program.getSourceFiles()[0]; - this.virtualSourceRoot = virtualSourceRoot; - } - - /** - * Called when the compiler instance should be relased from memory. - * - * This can happen because we are done with a project, or because the - * compiler instance needs to be rebooted. - */ - public releaseProgram() { - this.typeChecker = null; - this.arbitraryAstNode = null; - } - - /** - * Gets the canonical ID for the given type, generating a fresh ID if necessary. - */ - public buildType(type: ts.Type, unfoldAlias: boolean): number | null { - this.isInShallowTypeContext = false; - let id = this.getId(type, unfoldAlias); - this.iterateBuildTypeWorklist(); - if (id == null) return null; - return id; - } - - /** - * Caches the result of `getId`: `type -> [id (not unfolded), id (unfolded)]`. - * - * A value of `undefined` means the value is not yet computed, - * and `number | null` corresponds to the return value of `getId`. - */ - private idCache = new WeakMap(); - - /** - * Gets the canonical ID for the given type, generating a fresh ID if necessary. - * - * Returns `null` if we do not support extraction of this type. - */ - public getId(type: ts.Type, unfoldAlias: boolean): number | null { - if (this.skipExtractingTypes) return null; - let cached = this.idCache.get(type) ?? [undefined, undefined]; - let cachedValue = cached[unfoldAlias ? 1 : 0]; - if (cachedValue !== undefined) return cachedValue; - - let result = this.getIdRaw(type, unfoldAlias); - cached[unfoldAlias ? 1 : 0] = result; - return result; - } - - /** - * Gets the canonical ID for the given type, generating a fresh ID if necessary. - * - * Returns `null` if we do not support extraction of this type. - */ - public getIdRaw(type: ts.Type, unfoldAlias: boolean): number | null { - if (this.typeRecursionDepth > 100) { - // Ignore infinitely nested anonymous types, such as `{x: {x: {x: ... }}}`. - // Such a type can't be written directly with TypeScript syntax (as it would need to be named), - // but it can occur rarely as a result of type inference. - - // Caching this value is technically incorrect, as a type might be seen at depth 101 and then we cache the fact that it can't be extracted. - // Then later the type is seen at a lower depth and could be extracted, but then we immediately give up because of the cached failure. - return null; - } - // Replace very long string literal types with `string`. - if ((type.flags & ts.TypeFlags.StringLiteral) && ((type as ts.LiteralType).value as string).length > 30) { - type = this.typeChecker.getBaseTypeOfLiteralType(type); - } - ++this.typeRecursionDepth; - let content = this.getTypeString(type, unfoldAlias); - --this.typeRecursionDepth; - if (content == null) return null; // Type not supported. - let id = this.typeIds.get(content); - if (id == null) { - let stringValue = this.stringifyType(type, unfoldAlias); - if (stringValue == null) { - return null; // Type not supported. - } - id = this.typeIds.size; - this.typeIds.set(content, id); - this.typeToStringValues.push(stringValue); - this.buildTypeWorklist.push([type, id, unfoldAlias]); - this.typeExtractionState.push( - this.isInShallowTypeContext ? TypeExtractionState.PendingShallow : TypeExtractionState.PendingFull); - // If the type is the self-type for a named type (not a generic instantiation of it), - // emit the self-type binding for that type. - if (content.startsWith("reference;") && isTypeSelfReference(type)) { - this.selfTypes.symbols.push(this.getSymbolId(type.aliasSymbol || type.symbol)); - this.selfTypes.selfTypes.push(id); - } - } else if (!this.isInShallowTypeContext) { - // If the type was previously marked as shallow, promote it to full, - // and put it back in the worklist if necessary. - let state = this.typeExtractionState[id]; - if (state === TypeExtractionState.PendingShallow) { - this.typeExtractionState[id] = TypeExtractionState.PendingFull; - } else if (state === TypeExtractionState.DoneShallow) { - this.typeExtractionState[id] = TypeExtractionState.PendingFull; - this.buildTypeWorklist.push([type, id, unfoldAlias]); - } - } - return id; - } - - private stringifyType(type: ts.Type, unfoldAlias: boolean): string { - let formatFlags = unfoldAlias - ? ts.TypeFormatFlags.InTypeAlias - : ts.TypeFormatFlags.UseAliasDefinedOutsideCurrentScope; - let toStringValue: string; - // Some types can't be stringified. Just discard the type if we can't stringify it. - try { - toStringValue = this.typeChecker.typeToString(type, undefined, formatFlags); - } catch (e) { - console.warn("Recovered from a compiler crash while stringifying a type. Discarding the type."); - console.warn(e.stack); - return null; - } - if (toStringValue.length > 50) { - return toStringValue.substring(0, 47) + "..."; - } else { - return toStringValue; - } - } - - private stringifySignature(signature: ts.Signature, kind: ts.SignatureKind) { - let toStringValue: string; - // Some types can't be stringified. Just discard the type if we can't stringify it. - try { - toStringValue = - this.typeChecker.signatureToString( - signature, - signature.declaration, - ts.TypeFormatFlags.UseAliasDefinedOutsideCurrentScope, - kind); - } catch (e) { - console.warn("Recovered from a compiler crash while stringifying a signature. Discarding the signature."); - console.warn(e.stack); - return null; - } - if (toStringValue.length > 70) { - return toStringValue.substring(0, 69) + "..."; - } else { - return toStringValue; - } - } - - /** - * Gets a string representing the kind and contents of the given type. - */ - private getTypeString(type: AugmentedType, unfoldAlias: boolean): string | null { - // Reference to a type alias. - if (!unfoldAlias && type.aliasSymbol != null) { - let tag = "reference;" + this.getSymbolId(type.aliasSymbol); - return type.aliasTypeArguments == null - ? tag - : this.makeTypeStringVector(tag, type.aliasTypeArguments); - } - let flags = type.flags; - let objectFlags = (flags & ts.TypeFlags.Object) && (type as ts.ObjectType).objectFlags; - let symbol: AugmentedSymbol = type.symbol; - // Type that contains a reference to something. - if (symbol != null) { - // Possibly parameterized type. - if (isTypeReference(type)) { - let tag = "reference;" + this.getSymbolId(symbol); - return this.makeTypeStringVectorFromTypeReferenceArguments(tag, type); - } - // Reference to a type variable. - if (flags & ts.TypeFlags.TypeVariable) { - let enclosingType = getEnclosingTypeOfThisType(type); - if (enclosingType != null) { - return "this;" + this.getId(enclosingType, false); - } else if (symbol.parent == null || isFunctionTypeOrTypeAlias(symbol.declarations?.[0])) { - // The type variable is bound on a call signature. Only extract it by name. - return "lextypevar;" + symbol.name; - } else { - return "typevar;" + this.getSymbolId(symbol); - } - } - // Recognize types of form `typeof X` where `X` is a class, namespace, module, or enum. - // The TypeScript API has no explicit tag for `typeof` types. They can be recognized - // as anonymous object types that have a symbol (i.e. a "named anonymous type"). - if ((objectFlags & ts.ObjectFlags.Anonymous) && isTypeofCandidateSymbol(symbol)) { - return "typeof;" + this.getSymbolId(symbol); - } - // Reference to a named type. - // Must occur after the `typeof` case to avoid matching `typeof C` as the type `C`. - if (isTypeDefinitionSymbol(symbol)) { - return "reference;" + this.getSymbolId(type.symbol); - } - } - if (flags === ts.TypeFlags.Any) { - return "any"; - } - if (flags === ts.TypeFlags.String) { - return "string"; - } - if (flags === ts.TypeFlags.Number) { - return "number"; - } - if (flags === ts.TypeFlags.Void) { - return "void"; - } - if (flags === ts.TypeFlags.Never) { - return "never"; - } - if (flags === ts.TypeFlags.BigInt) { - return "bigint"; - } - if (flags & ts.TypeFlags.Null) { - return "null"; - } - if (flags & ts.TypeFlags.Undefined) { - return "undefined"; - } - if (flags === ts.TypeFlags.ESSymbol) { - return "plainsymbol"; - } - if (flags & ts.TypeFlags.Unknown) { - return "unknown"; - } - if (flags === ts.TypeFlags.UniqueESSymbol) { - return "uniquesymbol;" + this.getSymbolId((type as ts.UniqueESSymbolType).symbol); - } - if (flags === ts.TypeFlags.NonPrimitive && type.intrinsicName === "object") { - return "objectkeyword"; - } - // Note that TypeScript represents the `boolean` type as `true|false`. - if (flags === ts.TypeFlags.BooleanLiteral) { - // There is no public API to distinguish true and false. - // We rely on the internal property `intrinsicName`, which - // should be either "true" or "false" here. - return type.intrinsicName; - } - if (flags & ts.TypeFlags.NumberLiteral) { - return "numlit;" + (type as ts.LiteralType).value; - } - if (flags & ts.TypeFlags.StringLiteral) { - return "strlit;" + (type as ts.LiteralType).value; - } - if (flags & ts.TypeFlags.BigIntLiteral) { - let literalType = type as ts.LiteralType; - let value = literalType.value as ts.PseudoBigInt; - return "bigintlit;" + (value.negative ? "-" : "") + value.base10Value; - } - if (flags & ts.TypeFlags.Union) { - let unionType = type as ts.UnionType; - if (unionType.types.length === 0) { - // We ignore malformed types like unions and intersections without any operands. - // These trigger an assertion failure in `typeToString` - presumably because they - // cannot be written using TypeScript syntax - so we ignore them entirely. - return null; - } - return this.makeDeduplicatedTypeStringVector("union", unionType.types); - } - if (flags & ts.TypeFlags.Intersection) { - let intersectionType = type as ts.IntersectionType; - if (intersectionType.types.length === 0) { - return null; // Ignore malformed type. - } - return this.makeDeduplicatedTypeStringVector("intersection", intersectionType.types); - } - if (isTypeReference(type) && (type.target.objectFlags & ts.ObjectFlags.Tuple)) { - // Encode the minimum length and presence of rest element in the first two parts of the type string. - // Handle the absence of `minLength` and `hasRestElement` to be compatible with pre-3.0 compiler versions. - let tupleReference = type as ts.TupleTypeReference; - let tupleType = tupleReference.target; - let minLength = tupleType.minLength != null - ? tupleType.minLength - : this.typeChecker.getTypeArguments(tupleReference).length; - let hasRestElement = tupleType.hasRestElement ? 't' : 'f'; - let restIndex = -1; - for (let i = 0; i < tupleType.elementFlags.length; i++) { - if (tupleType.elementFlags[i] & ts.ElementFlags.Rest) { - restIndex = i; - break; - } - } - let prefix = `tuple;${minLength};${restIndex}`; - return this.makeTypeStringVectorFromTypeReferenceArguments(prefix, type); - } - if (objectFlags & ts.ObjectFlags.Anonymous) { - return this.makeStructuralTypeVector("object;", type as ts.ObjectType); - } - return null; - } - - /** - * Gets the canonical ID for the given symbol. - * - * Note that this may be called with symbols from different compiler instantiations, - * and it should return the same ID for symbols that logically refer to the same thing. - */ - public getSymbolId(symbol: AugmentedSymbol): number { - if (symbol.flags & ts.SymbolFlags.Alias) { - let aliasedSymbol: AugmentedSymbol = this.typeChecker.getAliasedSymbol(symbol); - if (aliasedSymbol.$id !== -1) { // Check if aliased symbol is on-stack - // Follow aliases eagerly, except in cases where this leads to cycles (for things like `import Foo = Foo.Bar`) - symbol = aliasedSymbol; - } - } - // We cache the symbol ID to avoid rebuilding long symbol strings. - let id = symbol.$id; - if (id != null) return id; - symbol.$id = -1; // Mark as on-stack while we are computing the ID - let content = this.getSymbolString(symbol); - id = this.symbolIds.get(content); - if (id != null) { - // The ID was determined in a previous compiler instantiation. - return symbol.$id = id; - } - if (id == null) { - id = this.symbolIds.size; - this.symbolIds.set(content, id); - symbol.$id = id; - - // Associate names with global symbols. - if (this.isGlobalSymbol(symbol)) { - this.addGlobalMapping(id, symbol.name); - } - - // Associate type names with their base type names. - this.extractSymbolBaseTypes(symbol, id); - } - return id; - } - - /** Returns true if the given symbol represents a name in the global scope. */ - private isGlobalSymbol(symbol: AugmentedSymbol): boolean { - let parent = symbol.parent; - if (parent != null) { - if (parent.escapedName === ts.InternalSymbolName.Global) { - return true; // Symbol declared in a global augmentation block. - } - return false; // Symbol is not a root. - } - if (symbol.declarations == null || symbol.declarations.length === 0) return false; - let declaration = symbol.declarations[0]; - let block = getEnclosingBlock(declaration); - if (ts.isSourceFile(block) && !this.isModuleSourceFile(block)) { - return true; // Symbol is declared at the top-level of a non-module file. - } - return false; - } - - /** Returns true if the given source file defines a module. */ - private isModuleSourceFile(file: ts.SourceFile) { - // This is not directly exposed, but a reliable indicator seems to be whether - // the file has a symbol. - return this.typeChecker.getSymbolAtLocation(file) != null; - } - - /** - * Gets a unique string for the given symbol. - */ - private getSymbolString(symbol: AugmentedSymbol): string { - let parent = symbol.parent; - if (parent == null || parent.escapedName === ts.InternalSymbolName.Global) { - return "root;" + this.getSymbolDeclarationString(symbol) + ";;" + this.rewriteSymbolName(symbol); - } else if (parent.exports != null && parent.exports.get(symbol.escapedName) === symbol) { - return "member;;" + this.getSymbolId(parent) + ";" + this.rewriteSymbolName(symbol); - } else { - return "other;" + this.getSymbolDeclarationString(symbol) + ";" + this.getSymbolId(parent) + ";" + this.rewriteSymbolName(symbol); - } - } - - private rewriteSymbolName(symbol: AugmentedSymbol) { - let { virtualSourceRoot, sourceRoot } = this.virtualSourceRoot; - let { name } = symbol; - if (virtualSourceRoot == null || sourceRoot == null) return name; - return name.replace(virtualSourceRoot, sourceRoot); - } - - /** - * Gets a string that distinguishes the given symbol from symbols with different - * lexical roots, or an empty string if the symbol is not a lexical root. - */ - private getSymbolDeclarationString(symbol: AugmentedSymbol): string { - if (symbol.declarations == null || symbol.declarations.length === 0) { - return ""; - } - let decl = symbol.declarations[0]; - if (ts.isSourceFile(decl)) return ""; - return this.getFileId(decl.getSourceFile().fileName) + ":" + decl.pos; - } - - /** - * Gets a number unique for the given filename. - */ - private getFileId(fileName: string): number { - let id = this.fileIds.get(fileName); - if (id == null) { - id = this.fileIds.size; - this.fileIds.set(fileName, id); - } - return id; - } - - /** - * Like `makeTypeStringVector` using the type arguments in the given type reference. - */ - private makeTypeStringVectorFromTypeReferenceArguments(tag: string, type: ts.TypeReference) { - // There can be an extra type argument at the end, denoting an explicit 'this' type argument. - // We discard the extra argument in our model. - let target = type.target; - let typeArguments = this.typeChecker.getTypeArguments(type); - if (typeArguments == null) return tag; - if (target.typeParameters != null) { - return this.makeTypeStringVector(tag, typeArguments, target.typeParameters.length); - } else { - return this.makeTypeStringVector(tag, typeArguments); - } - } - - /** - * Returns the given string with the IDs of the given types appended, - * each separated by `;`. - */ - private makeTypeStringVector(tag: string, types: ReadonlyArray, length = types.length): string | null { - let hash = tag; - for (let i = 0; i < length; ++i) { - let id = this.getId(types[i], false); - if (id == null) return null; - hash += ";" + id; - } - return hash; - } - - /** - * Returns the given string with the IDs of the given types appended, - * ignoring duplicates, and each separated by `;`. - */ - private makeDeduplicatedTypeStringVector(tag: string, types: ReadonlyArray, length = types.length): string | null { - let seenIds = new Set(); - let numberOfSeenIds = 0; - let hash = tag; - for (let i = 0; i < length; ++i) { - let id = this.getId(types[i], false); - if (id == null) return null; - seenIds.add(id); - if (seenIds.size > numberOfSeenIds) { - // This ID was not seen before - ++numberOfSeenIds; - hash += ";" + id; - } - } - return hash; - } - - /** Returns the type of `symbol` or `null` if it could not be computed. */ - private tryGetTypeOfSymbol(symbol: ts.Symbol) { - try { - return this.typeChecker.getTypeOfSymbolAtLocation(symbol, this.arbitraryAstNode) - } catch (e) { - console.warn(`Could not compute type of '${this.typeChecker.symbolToString(symbol)}'`); - return null; - } - } - - /** - * Returns a type string consisting of all the members of the given type. - * - * This must only be called for anonymous object types, as the type string for this - * type could otherwise depend on itself recursively. - */ - private makeStructuralTypeVector(tag: string, type: ts.ObjectType): string | null { - let hash = tag; - for (let property of type.getProperties()) { - let propertyType = this.tryGetTypeOfSymbol(property); - if (propertyType == null) return null; - let propertyTypeId = this.getId(propertyType, false); - if (propertyTypeId == null) return null; - hash += ";p" + this.getSymbolId(property) + ';' + propertyTypeId; - } - for (let kind of signatureKinds) { - for (let signature of this.typeChecker.getSignaturesOfType(type, kind)) { - let id = this.getSignatureId(kind, signature); - if (id == null) return null; - hash += ";c" + id; - } - } - let indexType = type.getStringIndexType(); - if (indexType != null) { - let indexTypeId = this.getId(indexType, false); - if (indexTypeId == null) return null; - hash += ";s" + indexTypeId; - } - indexType = type.getNumberIndexType(); - if (indexType != null) { - let indexTypeId = this.getId(indexType, false); - if (indexTypeId == null) return null; - hash += ";i" + indexTypeId; - } - return hash; - } - - public addModuleMapping(symbolId: number, moduleName: string) { - this.moduleMappings.symbols.push(symbolId); - this.moduleMappings.names.push(moduleName); - } - - public addGlobalMapping(symbolId: number, globalName: string) { - this.globalMappings.symbols.push(symbolId); - this.globalMappings.names.push(globalName); - } - - public getTypeTableJson(): object { - return { - typeStrings: Array.from(this.typeIds.keys()), - typeToStringValues: this.typeToStringValues, - propertyLookups: this.propertyLookups, - typeAliases: this.typeAliases, - symbolStrings: Array.from(this.symbolIds.keys()), - moduleMappings: this.moduleMappings, - globalMappings: this.globalMappings, - signatureStrings: Array.from(this.signatureIds.keys()), - signatureMappings: this.signatureMappings, - signatureToStringValues: this.signatureToStringValues, - numberIndexTypes: this.numberIndexTypes, - stringIndexTypes: this.stringIndexTypes, - baseTypes: this.baseTypes, - selfTypes: this.selfTypes, - }; - } - - /** - * Extracts the deep property and signature graph of recently discovered types. - * - * Types are added to the worklist when they are first assigned an ID, - * which happen transparently during property extraction and expansiveness checks. - */ - private iterateBuildTypeWorklist() { - let worklist = this.buildTypeWorklist; - let typeExtractionState = this.typeExtractionState; - while (worklist.length > 0) { - let [type, id, unfoldAlias] = worklist.pop(); - let isShallowContext = typeExtractionState[id] === TypeExtractionState.PendingShallow; - if (isShallowContext && !isTypeAlwaysSafeToExpand(type)) { - typeExtractionState[id] = TypeExtractionState.DoneShallow; - } else if (type.aliasSymbol != null && !unfoldAlias) { - typeExtractionState[id] = TypeExtractionState.DoneFull; - let underlyingTypeId = this.getId(type, true); - if (underlyingTypeId != null) { - this.typeAliases.aliasTypes.push(id); - this.typeAliases.underlyingTypes.push(underlyingTypeId); - } - } else { - typeExtractionState[id] = TypeExtractionState.DoneFull; - this.isInShallowTypeContext = isShallowContext || this.isExpansiveTypeReference(type); - this.extractProperties(type, id); - this.extractSignatures(type, id); - this.extractIndexers(type, id); - } - } - this.isInShallowTypeContext = false; - } - - /** - * Returns the properties to extract for the given type or `null` if nothing should be extracted. - * - * For performance reasons we only extract properties needed to recognize promise types at the QL - * level. - */ - private getPropertiesToExtract(type: ts.Type) { - if (this.getSelfType(type) === type) { - let thenSymbol = this.typeChecker.getPropertyOfType(type, "then"); - if (thenSymbol != null) { - return [thenSymbol]; - } - } - return null; - } - - private extractProperties(type: ts.Type, id: number) { - let props = this.getPropertiesToExtract(type); - if (props == null) return; - for (let symbol of props) { - let propertyType = this.tryGetTypeOfSymbol(symbol); - if (propertyType == null) continue; - let propertyTypeId = this.getId(propertyType, false); - if (propertyTypeId == null) continue; - this.propertyLookups.baseTypes.push(id); - this.propertyLookups.names.push(symbol.name); - this.propertyLookups.propertyTypes.push(propertyTypeId); - } - } - - /** - * Returns a unique ID for the given call/construct signature. - */ - public getSignatureId(kind: ts.SignatureKind, signature: ts.Signature): number { - let content = this.getSignatureString(kind, signature); - if (content == null) { - return null; - } - let id = this.signatureIds.get(content); - if (id == null) { - let stringValue = this.stringifySignature(signature, kind); - if (stringValue == null) { - return null; // Not supported. - } - id = this.signatureIds.size; - this.signatureIds.set(content, id); - this.signatureToStringValues.push(stringValue); - } - return id; - } - - /** - * Returns a unique string for the given call/constructor signature. - */ - private getSignatureString(kind: ts.SignatureKind, signature: AugmentedSignature): string { - let modifiers = signature.getDeclaration() ? ts.getModifiers(signature.getDeclaration() as ts.MethodSignature) : []; - let isAbstract = modifiers && modifiers.filter(modifier => modifier.kind == ts.SyntaxKind.AbstractKeyword).length > 0 - - let parameters = signature.getParameters(); - let numberOfTypeParameters = signature.typeParameters == null - ? 0 - : signature.typeParameters.length; - // Count the number of required parameters. - let requiredParameters = parameters.length; - for (let i = 0; i < parameters.length; ++i) { - if (parameters[i].flags & ts.SymbolFlags.Optional) { - requiredParameters = i; - break; - } - } - let hasRestParam = (signature.flags & InternalSignatureFlags.HasRestParameter) !== 0; - let restParameterTag = ''; - if (hasRestParam) { - if (requiredParameters === parameters.length) { - // Do not count the rest parameter as a required parameter - requiredParameters = parameters.length - 1; - } - if (parameters.length === 0) return null; - let restParameter = parameters[parameters.length - 1]; - let restParameterType = this.tryGetTypeOfSymbol(restParameter); - if (restParameterType == null) return null; - let restParameterTypeId = this.getId(restParameterType, false); - if (restParameterTypeId == null) return null; - restParameterTag = '' + restParameterTypeId; - } - let returnTypeId = this.getId(signature.getReturnType(), false); - if (returnTypeId == null) { - return null; - } - let tag = `${kind};${isAbstract ? "t" : "f"};${numberOfTypeParameters};${requiredParameters};${restParameterTag};${returnTypeId}`; - for (let typeParameter of signature.typeParameters || []) { - tag += ";" + typeParameter.symbol.name; - let constraint = typeParameter.getConstraint(); - let constraintId: number; - if (constraint == null || (constraintId = this.getId(constraint, false)) == null) { - tag += ";"; - } else { - tag += ";" + constraintId; - } - } - for (let paramIndex = 0; paramIndex < parameters.length; ++paramIndex) { - let parameter = parameters[paramIndex]; - let parameterType = this.tryGetTypeOfSymbol(parameter); - if (parameterType == null) { - return null; - } - let isRestParameter = hasRestParam && (paramIndex === parameters.length - 1); - if (isRestParameter) { - // The type of the rest parameter is the array type, but we wish to extract the non-array type. - if (!isTypeReference(parameterType)) return null; - let typeArguments = parameterType.typeArguments; - if (typeArguments == null || typeArguments.length === 0) return null; - parameterType = typeArguments[0]; - } - let parameterTypeId = this.getId(parameterType, false); - if (parameterTypeId == null) { - return null; - } - tag += ';' + parameter.name + ';' + parameterTypeId; - } - return tag; - } - - private extractSignatures(type: ts.Type, id: number) { - this.extractSignatureList(type, id, ts.SignatureKind.Call, type.getCallSignatures()); - this.extractSignatureList(type, id, ts.SignatureKind.Construct, type.getConstructSignatures()); - } - - private extractSignatureList(type: ts.Type, id: number, kind: ts.SignatureKind, list: ReadonlyArray) { - let index = -1; - for (let signature of list) { - ++index; - let signatureId = this.getSignatureId(kind, signature); - if (signatureId == null) continue; - this.signatureMappings.baseTypes.push(id); - this.signatureMappings.kinds.push(kind); - this.signatureMappings.indices.push(index); - this.signatureMappings.signatures.push(signatureId); - } - } - - private extractIndexers(type: ts.Type, id: number) { - this.extractIndexer(id, type.getStringIndexType(), this.stringIndexTypes); - this.extractIndexer(id, type.getNumberIndexType(), this.numberIndexTypes); - } - - private extractIndexer(baseType: number, indexType: ts.Type, table: IndexerTable) { - if (indexType == null) return; - let indexTypeId = this.getId(indexType, false); - if (indexTypeId == null) return; - table.baseTypes.push(baseType); - table.propertyTypes.push(indexTypeId); - } - - /** - * If the given symbol represents a type name, extracts its base type names. - * - * Base types are only extracted at the level of names, since the type arguments - * of a base type are not generally made available by the TypeScript API. - * - * For example, given these interfaces: - * ``` - * interface Base { x: T } - * interface Sub extends Base {} - * ``` - * a true base type of `Sub` would be `Base`, but all we can - * get from the compiler is just `Base` with no indication of what `S` should be. - */ - private extractSymbolBaseTypes(symbol: ts.Symbol, symbolId: number) { - for (let decl of symbol.declarations || []) { - if (ts.isClassLike(decl) || ts.isInterfaceDeclaration(decl)) { - for (let heritage of decl.heritageClauses || []) { - for (let typeExpr of heritage.types) { - let superType = this.typeChecker.getTypeFromTypeNode(typeExpr); - if (superType == null) continue; - let baseTypeSymbol = superType.symbol; - baseTypeSymbol = (baseTypeSymbol as any)?.type?.symbol ?? baseTypeSymbol; - if (baseTypeSymbol == null) continue; - let baseId = this.getSymbolId(baseTypeSymbol); - // Note: take care not to perform a recursive call between the two `push` calls. - this.baseTypes.symbols.push(symbolId); - this.baseTypes.baseTypeSymbols.push(baseId); - } - } - } - } - } - - /** - * If `type` is a generic instantiation of a type, returns the - * generic self-type for that type, otherwise `null`. - * - * For example, `Promise` maps to `Promise`, where - * `T` is the type parameter declared on the `Promise` interface. - */ - private getSelfType(type: ts.Type): ts.TypeReference { - if (isTypeReference(type) && this.typeChecker.getTypeArguments(type).length > 0) { - return type.target; - } - return null; - } - - /** - * True if the given type is a reference to a type that is part of an expansive cycle, which - * we simply call "expansive types". - * - * Non-expansive types may still lead into an expansive type, as long as it's not part of - * the cycle. - * - * It is guaranteed that any sequence of property reads on a type will loop back to a previously - * seen type or a reach a type that is marked as expansive. That is, this is sufficient to - * guarantee termination of recursive property traversal. - */ - private isExpansiveTypeReference(type: ts.Type): boolean { - if (this.restrictedExpansion) { - return true; - } - let selfType = this.getSelfType(type); - if (selfType != null) { - this.checkExpansiveness(selfType); - let id = this.getId(selfType, false); - return this.expansiveTypes.get(id); - } - return false; - } - - /** - * Checks if the given self-type is an expansive type. The result is stored in `expansiveTypes`. - * - * This follows a variant of Tarjan's SCC algorithm on a graph derived from the properties of types. - * - * The vertices of the graph are generic "self types", that is, types like `Foo` but not `Foo`. - * Types without type arguments are not vertices either, as such types can't be part of an expansive cycle. - * - * A property S.x with type T implies an edge from S to the self-type of every type referenced in T, whose - * type arguments contain a type parameter of S. Moreover, if such a reference contains a deeply nested - * occurence of a type parameter, e.g. `Foo>` it is classified as an "expanding" edge - * - * For example, this interface: - * - * interface Foo { - * x: Bar> - * } - * - * implies the following edges: - * - * Foo ==> Bar (expanding edge) - * Foo --> Baz (neutral edge) - * - * If an SCC contains an expanding edge, all its members are classified as expansive types. - * - * Suppose we extend the example with the interfaces: - * - * interface Bar { - * x: T; - * } - * - * interface Baz { - * x: Foo - * } - * - * The `Bar` interface implies no edges and the `Baz` interface implies the edge: - * - * Baz ==> Foo (expanding edge) - * - * This creates an expanding cycle, Foo --> Baz ==> Foo, so Foo and Baz are considered - * expansive, whereas Bar is not. - */ - private checkExpansiveness(type: ts.TypeReference) { - // `index`, `lowlink` and `stack` are from Tarjan's algorithm. - // Note that the type ID cannot be used as `index` because the index must be - // increasing with the order in which nodes are discovered in the traversal. - let indexTable = new Map(); - let lowlinkTable = new Map(); - let indexCounter = 0; - let stack: number[] = []; // IDs of types on the stack. - - // The expansion depth is the number of expanding edges that were used to - // reach the given node when it was first discovered. It is used to detect - // if the SCC contains an expanding edge. - // We also abuse this to track whether a node is currently on the stack; - // as long as the value is non-null, the node is on the stack. - let expansionDepthTable = new Map(); - - let typeTable = this; - - search(type, 0); - - function search(type: ts.TypeReference, expansionDepth: number): number | null { - let id = typeTable.getId(type, false); - if (id == null) return null; - - let index = indexTable.get(id); - if (index != null) { // Seen this node before? - let initialExpansionDepth = expansionDepthTable.get(id); - if (initialExpansionDepth == null) { - return null; // Not on the stack anymore. Its SCC is already complete. - } - if (expansionDepth > initialExpansionDepth) { - // The type has reached itself using an expansive edge. - // Mark is at expansive. The rest of the SCC will be marked when the SCC is complete. - typeTable.expansiveTypes.set(id, true); - } - return index; - } - - let previousResult = typeTable.expansiveTypes.get(id); - if (previousResult != null) { - // This node was classified by a previous call to checkExpansiveness. - return null; - } - - index = ++indexCounter; - indexTable.set(id, index); - lowlinkTable.set(id, index); - expansionDepthTable.set(id, expansionDepth); - let indexOnStack = stack.length; - stack.push(id); - - /** Indicates if a type contains no type variables, is a type variable, or strictly contains type variables. */ - const enum TypeVarDepth { - noTypeVar = 0, - isTypeVar = 1, - containsTypeVar = 2, - } - - for (let symbol of type.getProperties()) { - let propertyType = typeTable.tryGetTypeOfSymbol(symbol); - if (propertyType == null) continue; - traverseType(propertyType); - } - - if (lowlinkTable.get(id) === index) { - // We have finished an SCC. - // If any type was marked as expansive, propagate this to the entire SCC. - let isExpansive = false; - for (let i = indexOnStack; i < stack.length; ++i) { - let memberId = stack[i]; - if (typeTable.expansiveTypes.get(memberId) === true) { - isExpansive = true; - break; - } - } - for (let i = indexOnStack; i < stack.length; ++i) { - let memberId = stack[i]; - typeTable.expansiveTypes.set(memberId, isExpansive); - expansionDepthTable.set(memberId, null); // Mark as not on stack anymore. - } - stack.length = indexOnStack; // Pop all SCC nodes from stack. - } - - return lowlinkTable.get(id); - - function traverseType(type: ts.Type): TypeVarDepth { - if (isTypeVariable(type)) return TypeVarDepth.isTypeVar; - let depth = TypeVarDepth.noTypeVar; - typeTable.forEachChildType(type, child => { - depth = Math.max(depth, traverseType(child)); - }); - if (depth === TypeVarDepth.noTypeVar) { - // No need to recurse into types that do not reference a type variable. - return TypeVarDepth.noTypeVar; - } - let selfType = typeTable.getSelfType(type); - if (selfType != null) { - // A non-expanding reference such as `Foo` should preserve expansion depth, - // whereas an expanding reference `Foo` should increment it. - visitEdge(selfType, (depth === TypeVarDepth.isTypeVar) ? 0 : 1); - } - return TypeVarDepth.containsTypeVar; - } - - function visitEdge(successor: ts.TypeReference, weight: number) { - let result = search(successor, expansionDepth + weight); - if (result == null) return; - lowlinkTable.set(id, Math.min(lowlinkTable.get(id), result)); - } - } - } - - private forEachChildType(type: ts.Type, callback: (type: ts.Type) => void): void { - // Note: we deliberately do not traverse type aliases here, but the underlying type. - if (isTypeReference(type)) { - // Note that this case also handles tuple types, since a tuple type is represented as - // a reference to a synthetic generic interface. - let typeArguments = this.typeChecker.getTypeArguments(type); - if (typeArguments != null) { - typeArguments.forEach(callback); - } - } else if (type.flags & ts.TypeFlags.UnionOrIntersection) { - (type as ts.UnionOrIntersectionType).types.forEach(callback); - } else if (type.flags & ts.TypeFlags.Object) { - let objectType = type as ts.ObjectType; - let objectFlags = objectType.objectFlags; - if (objectFlags & ts.ObjectFlags.Anonymous) { - // Anonymous interface type like `{ x: number }`. - for (let symbol of type.getProperties()) { - let propertyType = this.tryGetTypeOfSymbol(symbol); - if (propertyType == null) continue; - callback(propertyType); - } - for (let signature of type.getCallSignatures()) { - this.forEachChildTypeOfSignature(signature, callback); - } - for (let signature of type.getConstructSignatures()) { - this.forEachChildTypeOfSignature(signature, callback); - } - let stringIndexType = type.getStringIndexType(); - if (stringIndexType != null) { - callback(stringIndexType); - } - let numberIndexType = type.getNumberIndexType(); - if (numberIndexType != null) { - callback(numberIndexType); - } - } - } - } - - private forEachChildTypeOfSignature(signature: ts.Signature, callback: (type: ts.Type) => void): void { - callback(signature.getReturnType()); - for (let parameter of signature.getParameters()) { - let paramType = this.tryGetTypeOfSymbol(parameter); - if (paramType == null) continue; - callback(paramType); - } - let typeParameters = signature.getTypeParameters(); - if (typeParameters != null) { - for (let typeParameter of typeParameters) { - let constraint = typeParameter.getConstraint(); - if (constraint == null) continue; - callback(constraint); - } - } - } -} - -function isFunctionTypeOrTypeAlias(declaration: ts.Declaration | undefined) { - if (declaration == null) return false; - return declaration.kind === ts.SyntaxKind.FunctionType || declaration.kind === ts.SyntaxKind.TypeAliasDeclaration; -} - -/** - * Given a `type` whose type-string is known to be a `reference`, returns true if this is the self-type for the referenced type. - * - * For example, for `type Foo = ...` this returns true if `type` is `Foo`. - */ -function isTypeSelfReference(type: ts.Type) { - if (type.aliasSymbol != null) { - const { aliasTypeArguments } = type; - if (aliasTypeArguments == null) return true; - let declaration = type.aliasSymbol.declarations?.[0]; - if (declaration == null || declaration.kind !== ts.SyntaxKind.TypeAliasDeclaration) return false; - let alias = declaration as ts.TypeAliasDeclaration; - for (let i = 0; i < aliasTypeArguments.length; ++i) { - if (aliasTypeArguments[i].symbol?.declarations?.[0] !== alias.typeParameters[i]) { - return false; - } - } - return true; - } else if (isTypeReference(type)) { - return type.target === type; - } else { - // Return true because we know we have mapped this type to kind `reference`, and in the cases - // not covered above (i.e. generic types) it is always a self-reference. - return true; - } -} diff --git a/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java b/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java index 341313e15b5..2bdddaf9933 100644 --- a/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java +++ b/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java @@ -51,10 +51,8 @@ import com.semmle.js.extractor.trapcache.DummyTrapCache; import com.semmle.js.extractor.trapcache.ITrapCache; import com.semmle.js.parser.ParseError; import com.semmle.js.parser.ParsedProject; -import com.semmle.ts.extractor.TypeExtractor; import com.semmle.ts.extractor.TypeScriptParser; import com.semmle.ts.extractor.TypeScriptWrapperOOMError; -import com.semmle.ts.extractor.TypeTable; import com.semmle.util.data.StringUtil; import com.semmle.util.diagnostic.DiagnosticLevel; import com.semmle.util.diagnostic.DiagnosticLocation; @@ -1065,75 +1063,26 @@ protected DependencyInstallationResult preparePackagesAndDependencies(Set FileExtractors extractors, List tsconfig, DependencyInstallationResult deps) { - if (hasTypeScriptFiles(files) || !tsconfig.isEmpty()) { + + List typeScriptFiles = new ArrayList<>(); + // Get all TypeScript files. + for (Path f : files) { + if (extractors.fileType(f) == FileType.TYPESCRIPT) { + typeScriptFiles.add(f); + } + } + // Also get TypeScript files from HTML file snippets. + for (Map.Entry entry : state.getSnippets().entrySet()) { + if (!extractedFiles.contains(entry.getKey()) + && FileType.forFileExtension(entry.getKey().toFile()) == FileType.TYPESCRIPT) { + typeScriptFiles.add(entry.getKey()); + } + } + + if (!typeScriptFiles.isEmpty()) { TypeScriptParser tsParser = state.getTypeScriptParser(); verifyTypeScriptInstallation(state); - - // Collect all files included in a tsconfig.json inclusion pattern. - // If a given file is referenced by multiple tsconfig files, we prefer to extract it using - // one that includes it rather than just references it. - Set explicitlyIncludedFiles = new LinkedHashSet<>(); - if (tsconfig.size() > 1) { // No prioritization needed if there's only one tsconfig. - for (Path projectPath : tsconfig) { - explicitlyIncludedFiles.addAll(tsParser.getOwnFiles(projectPath.toFile(), deps, virtualSourceRoot)); - } - } - - // Extract TypeScript projects - for (Path projectPath : tsconfig) { - File projectFile = projectPath.toFile(); - long start = logBeginProcess("Opening project " + projectFile); - ParsedProject project = tsParser.openProject(projectFile, deps, virtualSourceRoot); - logEndProcess(start, "Done opening project " + projectFile); - // Extract all files belonging to this project which are also matched - // by our include/exclude filters. - List typeScriptFiles = new ArrayList(); - for (File sourceFile : project.getAllFiles()) { - Path sourcePath = sourceFile.toPath(); - Path normalizedFile = normalizePath(sourcePath); - if (!files.contains(normalizedFile) && !state.getSnippets().containsKey(normalizedFile)) { - continue; - } - if (!project.getOwnFiles().contains(sourceFile) && explicitlyIncludedFiles.contains(sourceFile)) continue; - if (extractors.fileType(sourcePath) != FileType.TYPESCRIPT) { - // For the time being, skip non-TypeScript files, even if the TypeScript - // compiler can parse them for us. - continue; - } - if (extractedFiles.contains(sourcePath)) { - continue; - } - typeScriptFiles.add(sourcePath); - } - typeScriptFiles.sort(PATH_ORDERING); - extractTypeScriptFiles(typeScriptFiles, extractedFiles, extractors); - tsParser.closeProject(projectFile); - } - - // Extract all the types discovered when extracting the ASTs. - if (!tsconfig.isEmpty()) { - TypeTable typeTable = tsParser.getTypeTable(); - extractTypeTable(tsconfig.iterator().next(), typeTable); - } - - // Extract remaining TypeScript files. - List remainingTypeScriptFiles = new ArrayList<>(); - for (Path f : files) { - if (!extractedFiles.contains(f) - && extractors.fileType(f) == FileType.TYPESCRIPT) { - remainingTypeScriptFiles.add(f); - } - } - for (Map.Entry entry : state.getSnippets().entrySet()) { - if (!extractedFiles.contains(entry.getKey()) - && FileType.forFileExtension(entry.getKey().toFile()) == FileType.TYPESCRIPT) { - remainingTypeScriptFiles.add(entry.getKey()); - } - } - if (!remainingTypeScriptFiles.isEmpty()) { - extractTypeScriptFiles(remainingTypeScriptFiles, extractedFiles, extractors); - } - + extractTypeScriptFiles(typeScriptFiles, extractedFiles, extractors); // The TypeScript compiler instance is no longer needed. tsParser.killProcess(); } @@ -1246,18 +1195,6 @@ protected DependencyInstallationResult preparePackagesAndDependencies(Set return path.toAbsolutePath().normalize(); } - private void extractTypeTable(Path fileHandle, TypeTable table) { - TrapWriter trapWriter = - outputConfig - .getTrapWriterFactory() - .mkTrapWriter(new File(fileHandle.toString() + ".codeql-typescript-typetable")); - try { - new TypeExtractor(trapWriter, table).extract(); - } finally { - FileUtil.close(trapWriter); - } - } - /** * Get the source type specified in LGTM_INDEX_SOURCE_TYPE, or the default of {@link * SourceType#AUTO}. diff --git a/javascript/extractor/src/com/semmle/js/extractor/Main.java b/javascript/extractor/src/com/semmle/js/extractor/Main.java index 59be61388e3..60dd6988116 100644 --- a/javascript/extractor/src/com/semmle/js/extractor/Main.java +++ b/javascript/extractor/src/com/semmle/js/extractor/Main.java @@ -18,9 +18,7 @@ import com.semmle.js.extractor.ExtractorConfig.SourceType; import com.semmle.js.extractor.FileExtractor.FileType; import com.semmle.js.extractor.trapcache.ITrapCache; import com.semmle.js.parser.ParsedProject; -import com.semmle.ts.extractor.TypeExtractor; import com.semmle.ts.extractor.TypeScriptParser; -import com.semmle.ts.extractor.TypeTable; import com.semmle.util.data.StringUtil; import com.semmle.util.data.UnitParser; import com.semmle.util.exception.ResourceError; @@ -142,53 +140,22 @@ public class Main { tsParser.verifyInstallation(!ap.has(P_QUIET)); } - for (File projectFile : projectFiles) { - - long start = verboseLogStartTimer(ap, "Opening project " + projectFile); - ParsedProject project = tsParser.openProject(projectFile, DependencyInstallationResult.empty, extractorConfig.getVirtualSourceRoot()); - verboseLogEndTimer(ap, start); - // Extract all files belonging to this project which are also matched - // by our include/exclude filters. - List filesToExtract = new ArrayList<>(); - for (File sourceFile : project.getOwnFiles()) { - File normalizedFile = normalizeFile(sourceFile); - if ((files.contains(normalizedFile) || extractorState.getSnippets().containsKey(normalizedFile.toPath())) - && !extractedFiles.contains(sourceFile.getAbsoluteFile()) - && FileType.TYPESCRIPT.getExtensions().contains(FileUtil.extension(sourceFile))) { - filesToExtract.add(sourceFile); - } - } - tsParser.prepareFiles(filesToExtract); - for (int i = 0; i < filesToExtract.size(); ++i) { - ensureFileIsExtracted(filesToExtract.get(i), ap); - } - // Close the project to free memory. This does not need to be in a `finally` as - // the project is not a system resource. - tsParser.closeProject(projectFile); - } - - if (!projectFiles.isEmpty()) { - // Extract all the types discovered when extracting the ASTs. - TypeTable typeTable = tsParser.getTypeTable(); - extractTypeTable(projectFiles.iterator().next(), typeTable); - } - - List remainingTypescriptFiles = new ArrayList<>(); + List typeScriptFiles = new ArrayList<>(); for (File f : files) { if (!extractedFiles.contains(f.getAbsoluteFile()) && FileType.forFileExtension(f) == FileType.TYPESCRIPT) { - remainingTypescriptFiles.add(f); + typeScriptFiles.add(f); } } for (Map.Entry entry : extractorState.getSnippets().entrySet()) { if (!extractedFiles.contains(entry.getKey().toFile()) && FileType.forFileExtension(entry.getKey().toFile()) == FileType.TYPESCRIPT) { - remainingTypescriptFiles.add(entry.getKey().toFile()); + typeScriptFiles.add(entry.getKey().toFile()); } } - if (!remainingTypescriptFiles.isEmpty()) { - tsParser.prepareFiles(remainingTypescriptFiles); - for (File f : remainingTypescriptFiles) { + if (!typeScriptFiles.isEmpty()) { + tsParser.prepareFiles(typeScriptFiles); + for (File f : typeScriptFiles) { ensureFileIsExtracted(f, ap); } } @@ -225,21 +192,6 @@ public class Main { return false; } - private void extractTypeTable(File fileHandle, TypeTable table) { - TrapWriter trapWriter = - extractorOutputConfig - .getTrapWriterFactory() - .mkTrapWriter( - new File( - fileHandle.getParentFile(), - fileHandle.getName() + ".codeql-typescript-typetable")); - try { - new TypeExtractor(trapWriter, table).extract(); - } finally { - FileUtil.close(trapWriter); - } - } - private void ensureFileIsExtracted(File f, ArgsParser ap) { if (!extractedFiles.add(f.getAbsoluteFile())) { // The file has already been extracted as part of a project. diff --git a/javascript/extractor/src/com/semmle/ts/extractor/TypeExtractor.java b/javascript/extractor/src/com/semmle/ts/extractor/TypeExtractor.java deleted file mode 100644 index de0b2558c4d..00000000000 --- a/javascript/extractor/src/com/semmle/ts/extractor/TypeExtractor.java +++ /dev/null @@ -1,330 +0,0 @@ -package com.semmle.ts.extractor; - -import com.google.gson.JsonArray; -import com.google.gson.JsonObject; -import com.semmle.util.trap.TrapWriter; -import com.semmle.util.trap.TrapWriter.Label; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -/** - * Extracts type and symbol information into TRAP files. - * - *

This is closely coupled with the type_table.ts file in the parser-wrapper. Type - * strings and symbol strings generated in that file are parsed here. See that file for reference - * and documentation. - */ -public class TypeExtractor { - private final TrapWriter trapWriter; - private final TypeTable table; - - private static final Map tagToKind = new LinkedHashMap(); - - private static final int referenceKind = 6; - private static final int objectKind = 7; - private static final int typevarKind = 8; - private static final int typeofKind = 9; - private static final int uniqueSymbolKind = 15; - private static final int tupleKind = 18; - private static final int lexicalTypevarKind = 19; - private static final int thisKind = 20; - private static final int numberLiteralTypeKind = 21; - private static final int stringLiteralTypeKind = 22; - private static final int bigintLiteralTypeKind = 25; - - static { - tagToKind.put("any", 0); - tagToKind.put("string", 1); - tagToKind.put("number", 2); - tagToKind.put("union", 3); - tagToKind.put("true", 4); - tagToKind.put("false", 5); - tagToKind.put("reference", referenceKind); - tagToKind.put("object", objectKind); - tagToKind.put("typevar", typevarKind); - tagToKind.put("typeof", typeofKind); - tagToKind.put("void", 10); - tagToKind.put("undefined", 11); - tagToKind.put("null", 12); - tagToKind.put("never", 13); - tagToKind.put("plainsymbol", 14); - tagToKind.put("uniquesymbol", uniqueSymbolKind); - tagToKind.put("objectkeyword", 16); - tagToKind.put("intersection", 17); - tagToKind.put("tuple", tupleKind); - tagToKind.put("lextypevar", lexicalTypevarKind); - tagToKind.put("this", thisKind); - tagToKind.put("numlit", numberLiteralTypeKind); - tagToKind.put("strlit", stringLiteralTypeKind); - tagToKind.put("unknown", 23); - tagToKind.put("bigint", 24); - tagToKind.put("bigintlit", bigintLiteralTypeKind); - } - - private static final Map symbolKind = new LinkedHashMap(); - - static { - symbolKind.put("root", 0); - symbolKind.put("member", 1); - symbolKind.put("other", 2); - } - - public TypeExtractor(TrapWriter trapWriter, TypeTable table) { - this.trapWriter = trapWriter; - this.table = table; - } - - public void extract() { - for (int i = 0; i < table.getNumberOfTypes(); ++i) { - extractType(i); - } - extractPropertyLookups(table.getPropertyLookups()); - extractTypeAliases(table.getTypeAliases()); - for (int i = 0; i < table.getNumberOfSymbols(); ++i) { - extractSymbol(i); - } - extractSymbolNameMapping("symbol_module", table.getModuleMappings()); - extractSymbolNameMapping("symbol_global", table.getGlobalMappings()); - extractSignatureMappings(table.getSignatureMappings()); - for (int i = 0; i < table.getNumberOfSignatures(); ++i) { - extractSignature(i); - } - extractIndexTypeTable(table.getNumberIndexTypes(), "number_index_type"); - extractIndexTypeTable(table.getStringIndexTypes(), "string_index_type"); - extractBaseTypes(table.getBaseTypes()); - extractSelfTypes(table.getSelfTypes()); - } - - private void extractType(int id) { - Label lbl = trapWriter.globalID("type;" + id); - String contents = table.getTypeString(id); - String[] parts = split(contents); - int kind = tagToKind.get(parts[0]); - trapWriter.addTuple("types", lbl, kind, table.getTypeToStringValue(id)); - int firstChild = 1; - switch (kind) { - case referenceKind: - case typevarKind: - case typeofKind: - case uniqueSymbolKind: - { - // The first part of a reference is the symbol for name binding. - Label symbol = trapWriter.globalID("symbol;" + parts[1]); - trapWriter.addTuple("type_symbol", lbl, symbol); - ++firstChild; - break; - } - case tupleKind: - { - // The first two parts denote minimum length and index of rest element (or -1 if no rest element). - trapWriter.addTuple("tuple_type_min_length", lbl, Integer.parseInt(parts[1])); - int restIndex = Integer.parseInt(parts[2]); - if (restIndex != -1) { - trapWriter.addTuple("tuple_type_rest_index", lbl, restIndex); - } - firstChild += 2; - break; - } - case objectKind: - case lexicalTypevarKind: - firstChild = parts.length; // No children. - break; - - case numberLiteralTypeKind: - case stringLiteralTypeKind: - case bigintLiteralTypeKind: - firstChild = parts.length; // No children. - // The string value may contain `;` so don't use the split(). - String value = contents.substring(parts[0].length() + 1); - trapWriter.addTuple("type_literal_value", lbl, value); - break; - } - for (int i = firstChild; i < parts.length; ++i) { - Label childLabel = trapWriter.globalID("type;" + parts[i]); - trapWriter.addTuple("type_child", childLabel, lbl, i - firstChild); - } - } - - private void extractPropertyLookups(JsonObject lookups) { - JsonArray baseTypes = lookups.get("baseTypes").getAsJsonArray(); - JsonArray names = lookups.get("names").getAsJsonArray(); - JsonArray propertyTypes = lookups.get("propertyTypes").getAsJsonArray(); - for (int i = 0; i < baseTypes.size(); ++i) { - int baseType = baseTypes.get(i).getAsInt(); - String name = names.get(i).getAsString(); - int propertyType = propertyTypes.get(i).getAsInt(); - trapWriter.addTuple( - "type_property", - trapWriter.globalID("type;" + baseType), - name, - trapWriter.globalID("type;" + propertyType)); - } - } - - private void extractTypeAliases(JsonObject aliases) { - JsonArray aliasTypes = aliases.get("aliasTypes").getAsJsonArray(); - JsonArray underlyingTypes = aliases.get("underlyingTypes").getAsJsonArray(); - for (int i = 0; i < aliasTypes.size(); ++i) { - int aliasType = aliasTypes.get(i).getAsInt(); - int underlyingType = underlyingTypes.get(i).getAsInt(); - trapWriter.addTuple( - "type_alias", - trapWriter.globalID("type;" + aliasType), - trapWriter.globalID("type;" + underlyingType)); - } - } - - private void extractSymbol(int index) { - // Format is: kind;decl;parent;name - String[] parts = split(table.getSymbolString(index), 4); - int kind = symbolKind.get(parts[0]); - String name = parts[3]; - Label label = trapWriter.globalID("symbol;" + index); - trapWriter.addTuple("symbols", label, kind, name); - String parentStr = parts[2]; - if (parentStr.length() > 0) { - Label parentLabel = trapWriter.globalID("symbol;" + parentStr); - trapWriter.addTuple("symbol_parent", label, parentLabel); - } - } - - private void extractSymbolNameMapping(String relationName, JsonObject mappings) { - JsonArray symbols = mappings.get("symbols").getAsJsonArray(); - JsonArray names = mappings.get("names").getAsJsonArray(); - for (int i = 0; i < symbols.size(); ++i) { - Label symbol = trapWriter.globalID("symbol;" + symbols.get(i).getAsInt()); - String moduleName = names.get(i).getAsString(); - trapWriter.addTuple(relationName, symbol, moduleName); - } - } - - private void extractSignature(int index) { - // Format is: - // kind;isAbstract;numTypeParams;requiredParams;restParamType;returnType(;paramName;paramType)* - String[] parts = split(table.getSignatureString(index)); - Label label = trapWriter.globalID("signature;" + index); - int kind = Integer.parseInt(parts[0]); - boolean isAbstract = parts[1].equals("t"); - if (isAbstract) { - trapWriter.addTuple("is_abstract_signature", label); - } - int numberOfTypeParameters = Integer.parseInt(parts[2]); - int requiredParameters = Integer.parseInt(parts[3]); - String restParamTypeTag = parts[4]; - if (!restParamTypeTag.isEmpty()) { - trapWriter.addTuple( - "signature_rest_parameter", label, trapWriter.globalID("type;" + restParamTypeTag)); - } - Label returnType = trapWriter.globalID("type;" + parts[5]); - trapWriter.addTuple( - "signature_types", - label, - kind, - table.getSignatureToStringValue(index), - numberOfTypeParameters, - requiredParameters); - trapWriter.addTuple("signature_contains_type", returnType, label, -1); - int numberOfParameters = (parts.length - 6) / 2; // includes type parameters - for (int i = 0; i < numberOfParameters; ++i) { - int partIndex = 6 + (2 * i); - String paramName = parts[partIndex]; - String paramTypeId = parts[partIndex + 1]; - if (paramTypeId.length() > 0) { // Unconstrained type parameters have an empty type ID. - Label paramType = trapWriter.globalID("type;" + parts[partIndex + 1]); - trapWriter.addTuple("signature_contains_type", paramType, label, i); - } - trapWriter.addTuple("signature_parameter_name", label, i, paramName); - } - } - - private void extractSignatureMappings(JsonObject mappings) { - JsonArray baseTypes = mappings.get("baseTypes").getAsJsonArray(); - JsonArray kinds = mappings.get("kinds").getAsJsonArray(); - JsonArray indices = mappings.get("indices").getAsJsonArray(); - JsonArray signatures = mappings.get("signatures").getAsJsonArray(); - for (int i = 0; i < baseTypes.size(); ++i) { - int baseType = baseTypes.get(i).getAsInt(); - int kind = kinds.get(i).getAsInt(); - int index = indices.get(i).getAsInt(); - int signatureId = signatures.get(i).getAsInt(); - trapWriter.addTuple( - "type_contains_signature", - trapWriter.globalID("type;" + baseType), - kind, - index, - trapWriter.globalID("signature;" + signatureId)); - } - } - - private void extractIndexTypeTable(JsonObject table, String relationName) { - JsonArray baseTypes = table.get("baseTypes").getAsJsonArray(); - JsonArray propertyTypes = table.get("propertyTypes").getAsJsonArray(); - for (int i = 0; i < baseTypes.size(); ++i) { - int baseType = baseTypes.get(i).getAsInt(); - int propertyType = propertyTypes.get(i).getAsInt(); - trapWriter.addTuple( - relationName, - trapWriter.globalID("type;" + baseType), - trapWriter.globalID("type;" + propertyType)); - } - } - - private void extractBaseTypes(JsonObject table) { - JsonArray symbols = table.get("symbols").getAsJsonArray(); - JsonArray baseTypeSymbols = table.get("baseTypeSymbols").getAsJsonArray(); - for (int i = 0; i < symbols.size(); ++i) { - int symbolId = symbols.get(i).getAsInt(); - int baseTypeSymbolId = baseTypeSymbols.get(i).getAsInt(); - trapWriter.addTuple( - "base_type_names", - trapWriter.globalID("symbol;" + symbolId), - trapWriter.globalID("symbol;" + baseTypeSymbolId)); - } - } - - private void extractSelfTypes(JsonObject table) { - JsonArray symbols = table.get("symbols").getAsJsonArray(); - JsonArray selfTypes = table.get("selfTypes").getAsJsonArray(); - for (int i = 0; i < symbols.size(); ++i) { - int symbolId = symbols.get(i).getAsInt(); - int typeId = selfTypes.get(i).getAsInt(); - trapWriter.addTuple( - "self_types", - trapWriter.globalID("symbol;" + symbolId), - trapWriter.globalID("type;" + typeId)); - } - } - - /** Like {@link #split(String)} without a limit. */ - private static String[] split(String input) { - return split(input, -1); - } - - /** - * Splits the input around the semicolon (;) character, preserving all empty - * substrings. - * - *

At most limit substrings will be extracted. If the limit is reached, the last - * substring will extend to the end of the string, possibly itself containing semicolons. - * - *

Note that the {@link String#split(String)} method does not preserve empty substrings at the - * end of the string in case the string ends with a semicolon. - */ - private static String[] split(String input, int limit) { - List result = new ArrayList(); - int lastPos = 0; - for (int i = 0; i < input.length(); ++i) { - if (input.charAt(i) == ';') { - result.add(input.substring(lastPos, i)); - lastPos = i + 1; - if (result.size() == limit - 1) break; - } - } - result.add(input.substring(lastPos)); - return result.toArray(EMPTY_STRING_ARRAY); - } - - private static final String[] EMPTY_STRING_ARRAY = new String[0]; -} diff --git a/javascript/extractor/src/com/semmle/ts/extractor/TypeScriptParser.java b/javascript/extractor/src/com/semmle/ts/extractor/TypeScriptParser.java index d19490286b7..09f05b2d9e0 100644 --- a/javascript/extractor/src/com/semmle/ts/extractor/TypeScriptParser.java +++ b/javascript/extractor/src/com/semmle/ts/extractor/TypeScriptParser.java @@ -463,113 +463,6 @@ public class TypeScriptParser { checkResponseType(response, "ok"); } - /** - * Converts a map to an array of [key, value] pairs. - */ - private JsonArray mapToArray(Map map) { - JsonArray result = new JsonArray(); - map.forEach( - (key, path) -> { - JsonArray entry = new JsonArray(); - entry.add(key); - entry.add(path.toString()); - result.add(entry); - }); - return result; - } - - private static Set getFilesFromJsonArray(JsonArray array) { - Set files = new LinkedHashSet<>(); - for (JsonElement elm : array) { - files.add(new File(elm.getAsString())); - } - return files; - } - - /** - * Returns the set of files included by the inclusion pattern in the given tsconfig.json file. - */ - public Set getOwnFiles(File tsConfigFile, DependencyInstallationResult deps, VirtualSourceRoot vroot) { - JsonObject request = makeLoadCommand("get-own-files", tsConfigFile, deps, vroot); - JsonObject response = talkToParserWrapper(request); - try { - checkResponseType(response, "file-list"); - return getFilesFromJsonArray(response.get("ownFiles").getAsJsonArray()); - } catch (IllegalStateException e) { - throw new CatastrophicError( - "TypeScript parser wrapper sent unexpected response: " + response, e); - } - } - - /** - * Opens a new project based on a tsconfig.json file. The compiler will analyze all files in the - * project. - * - *

Call {@link #parse} to access individual files in the project. - * - *

Only one project should be opened at once. - */ - public ParsedProject openProject(File tsConfigFile, DependencyInstallationResult deps, VirtualSourceRoot vroot) { - JsonObject request = makeLoadCommand("open-project", tsConfigFile, deps, vroot); - JsonObject response = talkToParserWrapper(request); - try { - checkResponseType(response, "project-opened"); - ParsedProject project = new ParsedProject(tsConfigFile, - getFilesFromJsonArray(response.get("ownFiles").getAsJsonArray()), - getFilesFromJsonArray(response.get("allFiles").getAsJsonArray())); - return project; - } catch (IllegalStateException e) { - throw new CatastrophicError( - "TypeScript parser wrapper sent unexpected response: " + response, e); - } - } - - private JsonObject makeLoadCommand(String command, File tsConfigFile, DependencyInstallationResult deps, VirtualSourceRoot vroot) { - JsonObject request = new JsonObject(); - request.add("command", new JsonPrimitive(command)); - request.add("tsConfig", new JsonPrimitive(tsConfigFile.getPath())); - request.add("packageEntryPoints", mapToArray(deps.getPackageEntryPoints())); - request.add("packageJsonFiles", mapToArray(deps.getPackageJsonFiles())); - request.add("sourceRoot", vroot.getSourceRoot() == null - ? JsonNull.INSTANCE - : new JsonPrimitive(vroot.getSourceRoot().toString())); - request.add("virtualSourceRoot", vroot.getVirtualSourceRoot() == null - ? JsonNull.INSTANCE - : new JsonPrimitive(vroot.getVirtualSourceRoot().toString())); - return request; - } - - /** - * Closes a project previously opened. - * - *

This main purpose is to free heap space in the Node.js process. - */ - public void closeProject(File tsConfigFile) { - JsonObject request = new JsonObject(); - request.add("command", new JsonPrimitive("close-project")); - request.add("tsConfig", new JsonPrimitive(tsConfigFile.getPath())); - JsonObject response = talkToParserWrapper(request); - try { - checkResponseType(response, "project-closed"); - } catch (IllegalStateException e) { - throw new CatastrophicError( - "TypeScript parser wrapper sent unexpected response: " + response, e); - } - } - - public TypeTable getTypeTable() { - JsonObject request = new JsonObject(); - request.add("command", new JsonPrimitive("get-type-table")); - JsonObject response = talkToParserWrapper(request); - try { - checkResponseType(response, "type-table"); - return new TypeTable(response.get("typeTable").getAsJsonObject()); - } catch (IllegalStateException e) { - throw new CatastrophicError( - "TypeScript parser wrapper sent unexpected response: " + response, e); - } - } - /** * Closes any open project, and in general, brings the TypeScript wrapper to a fresh state as if * it had just been restarted. diff --git a/javascript/extractor/src/com/semmle/ts/extractor/TypeTable.java b/javascript/extractor/src/com/semmle/ts/extractor/TypeTable.java deleted file mode 100644 index 53d8c437d92..00000000000 --- a/javascript/extractor/src/com/semmle/ts/extractor/TypeTable.java +++ /dev/null @@ -1,119 +0,0 @@ -package com.semmle.ts.extractor; - -import com.google.gson.JsonArray; -import com.google.gson.JsonObject; - -/** - * Holds the output of the get-type-table command. - * - *

See documentation in parser-wrapper/src/type_table.ts. - */ -public class TypeTable { - private final JsonArray typeStrings; - private final JsonArray typeToStringValues; - private final JsonObject propertyLookups; - private final JsonObject typeAliases; - private final JsonArray symbolStrings; - private final JsonObject moduleMappings; - private final JsonObject globalMappings; - private final JsonArray signatureStrings; - private final JsonObject signatureMappings; - private final JsonArray signatureToStringValues; - private final JsonObject stringIndexTypes; - private final JsonObject numberIndexTypes; - private final JsonObject baseTypes; - private final JsonObject selfTypes; - - public TypeTable(JsonObject typeTable) { - this.typeStrings = typeTable.get("typeStrings").getAsJsonArray(); - this.typeToStringValues = typeTable.get("typeToStringValues").getAsJsonArray(); - this.propertyLookups = typeTable.get("propertyLookups").getAsJsonObject(); - this.typeAliases = typeTable.get("typeAliases").getAsJsonObject(); - this.symbolStrings = typeTable.get("symbolStrings").getAsJsonArray(); - this.moduleMappings = typeTable.get("moduleMappings").getAsJsonObject(); - this.globalMappings = typeTable.get("globalMappings").getAsJsonObject(); - this.signatureStrings = typeTable.get("signatureStrings").getAsJsonArray(); - this.signatureMappings = typeTable.get("signatureMappings").getAsJsonObject(); - this.signatureToStringValues = typeTable.get("signatureToStringValues").getAsJsonArray(); - this.numberIndexTypes = typeTable.get("numberIndexTypes").getAsJsonObject(); - this.stringIndexTypes = typeTable.get("stringIndexTypes").getAsJsonObject(); - this.baseTypes = typeTable.get("baseTypes").getAsJsonObject(); - this.selfTypes = typeTable.get("selfTypes").getAsJsonObject(); - } - - public String getTypeString(int index) { - return typeStrings.get(index).getAsString(); - } - - public String getTypeToStringValue(int index) { - return typeToStringValues.get(index).getAsString(); - } - - public JsonObject getPropertyLookups() { - return propertyLookups; - } - - public JsonObject getTypeAliases() { - return typeAliases; - } - - public int getNumberOfTypes() { - return typeStrings.size(); - } - - public String getSymbolString(int index) { - return symbolStrings.get(index).getAsString(); - } - - public int getNumberOfSymbols() { - return symbolStrings.size(); - } - - public JsonObject getModuleMappings() { - return moduleMappings; - } - - public JsonObject getGlobalMappings() { - return globalMappings; - } - - public JsonArray getSignatureStrings() { - return signatureStrings; - } - - public int getNumberOfSignatures() { - return signatureStrings.size(); - } - - public String getSignatureString(int i) { - return signatureStrings.get(i).getAsString(); - } - - public JsonObject getSignatureMappings() { - return signatureMappings; - } - - public JsonArray getSignatureToStringValues() { - return signatureToStringValues; - } - - public String getSignatureToStringValue(int i) { - return signatureToStringValues.get(i).getAsString(); - } - - public JsonObject getNumberIndexTypes() { - return numberIndexTypes; - } - - public JsonObject getStringIndexTypes() { - return stringIndexTypes; - } - - public JsonObject getBaseTypes() { - return baseTypes; - } - - public JsonObject getSelfTypes() { - return selfTypes; - } -}