hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-05-02 20:25:13 +02:00
Code Issues Packages Projects Releases Wiki Activity

JS: Make this work in qltest

Browse Source
This commit is contained in:
Asger Feldthaus
2020-06-29 13:42:55 +01:00
parent 1a16d7339a
commit 3938856e61
11 changed files with 126 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
javascript/extractor/lib/typescript/src/common.ts
View File

@@ -48,7 +48,7 @@ export class Project {
public load(): void {
const { config, host } = this;
this.program = ts.createProgram(config.fileNames, config.options, host);
this.typeTable.setProgram(this.program);
this.typeTable.setProgram(this.program, this.virtualSourceRoot);
}
/**

13
javascript/extractor/lib/typescript/src/main.ts
View File

@@ -416,14 +416,21 @@ function loadTsConfig(command: LoadCommand): LoadedConfig {
useCaseSensitiveFileNames: true,
readDirectory: (rootDir, extensions, excludes?, includes?, depth?) => {
// Perform the glob matching in both real and virtual source roots.
let originalResults = ts.sys.readDirectory(rootDir, extensions, excludes, includes, depth)
let exclusions = excludes == null ? [] : [...excludes];
if (virtualSourceRoot.virtualSourceRoot != null) {
// qltest puts the virtual source root inside the real source root (.testproj).
// Make sure we don't find files inside the virtual source root in this pass.
exclusions.push(virtualSourceRoot.virtualSourceRoot);
}
let originalResults = ts.sys.readDirectory(rootDir, extensions, exclusions, includes, depth)
let virtualDir = virtualSourceRoot.toVirtualPath(rootDir);
if (virtualDir == null) {
return originalResults;
}
// Make sure glob matching does not to discover anything in node_modules.
let virtualExcludes = [ ...(excludes || []), '**/node_modules/**/*' ];
let virtualResults = ts.sys.readDirectory(virtualDir, extensions, virtualExcludes, includes, depth)
let virtualExclusions = excludes == null ? [] : [...excludes];
virtualExclusions.push('**/node_modules/**/*');
let virtualResults = ts.sys.readDirectory(virtualDir, extensions, virtualExclusions, includes, depth)
return [ ...originalResults, ...virtualResults ];
},
fileExists: (path: string) => {

19
javascript/extractor/lib/typescript/src/type_table.ts
View File

@@ -1,4 +1,5 @@
import * as ts from "./typescript";
import { VirtualSourceRoot } from "./virtual_source_root";
interface AugmentedSymbol extends ts.Symbol {
parent?: AugmentedSymbol;
@@ -379,12 +380,15 @@ export class TypeTable {
*/
public restrictedExpansion = false;
private virtualSourceRoot: VirtualSourceRoot;
/**
* Called when a new compiler instance has started.
*/
public setProgram(program: ts.Program) {
public setProgram(program: ts.Program, virtualSourceRoot: VirtualSourceRoot) {
this.typeChecker = program.getTypeChecker();
this.arbitraryAstNode = program.getSourceFiles()[0];
this.virtualSourceRoot = virtualSourceRoot;
}
/**
@@ -703,14 +707,21 @@ export class TypeTable {
private getSymbolString(symbol: AugmentedSymbol): string {
let parent = symbol.parent;
if (parent == null || parent.escapedName === ts.InternalSymbolName.Global) {
return "root;" + this.getSymbolDeclarationString(symbol) + ";;" + symbol.name;
return "root;" + this.getSymbolDeclarationString(symbol) + ";;" + this.rewriteSymbolName(symbol);
} else if (parent.exports != null && parent.exports.get(symbol.escapedName) === symbol) {
return "member;;" + this.getSymbolId(parent) + ";" + symbol.name;
return "member;;" + this.getSymbolId(parent) + ";" + this.rewriteSymbolName(symbol);
} else {
return "other;" + this.getSymbolDeclarationString(symbol) + ";" + this.getSymbolId(parent) + ";" + symbol.name;
return "other;" + this.getSymbolDeclarationString(symbol) + ";" + this.getSymbolId(parent) + ";" + this.rewriteSymbolName(symbol);
}
}
private rewriteSymbolName(symbol: AugmentedSymbol) {
let { virtualSourceRoot, sourceRoot } = this.virtualSourceRoot;
let { name } = symbol;
if (virtualSourceRoot == null || sourceRoot == null) return name;
return name.replace(virtualSourceRoot, sourceRoot);
}
/**
* Gets a string that distinguishes the given symbol from symbols with different
* lexical roots, or an empty string if the symbol is not a lexical root.

14
javascript/extractor/lib/typescript/src/virtual_source_root.ts
View File

@@ -7,13 +7,13 @@ import * as ts from "./typescript";
*/
export class VirtualSourceRoot {
constructor(
private sourceRoot: string | null,
public sourceRoot: string | null,
/**
* Directory whose folder structure mirrors the real source root, but with `node_modules` installed,
* or undefined if no virtual source root exists.
*/
private virtualSourceRoot: string | null,
public virtualSourceRoot: string | null,
) {}
private static translate(oldRoot: string, newRoot: string, path: string) {
@@ -25,9 +25,17 @@ export class VirtualSourceRoot {
/**
* Maps a path under the real source root to the corresponding path in the virtual source root.
*
* Returns `null` for paths already in the virtual source root.
*/
public toVirtualPath(path: string) {
return VirtualSourceRoot.translate(this.sourceRoot, this.virtualSourceRoot, path);
let { virtualSourceRoot } = this;
if (path.startsWith(virtualSourceRoot)) {
// 'qltest' creates a virtual source root inside the real source root.
// Make sure such files don't appear to be inside the real source root.
return null;
}
return VirtualSourceRoot.translate(this.sourceRoot, virtualSourceRoot, path);
}
/**

60
javascript/extractor/src/com/semmle/js/extractor/Main.java
View File

@@ -2,6 +2,8 @@ package com.semmle.js.extractor;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;
@@ -31,6 +33,8 @@ import com.semmle.util.io.WholeIO;
import com.semmle.util.language.LegacyLanguage;
import com.semmle.util.process.ArgsParser;
import com.semmle.util.process.ArgsParser.FileMode;
import com.semmle.util.process.Env;
import com.semmle.util.process.Env.Var;
import com.semmle.util.trap.TrapWriter;
/** The main entry point of the JavaScript extractor. */
@@ -134,12 +138,6 @@ public class Main {
return;
}
TypeScriptParser tsParser = extractorState.getTypeScriptParser();
tsParser.setTypescriptRam(extractorConfig.getTypeScriptRam());
if (containsTypeScriptFiles()) {
tsParser.verifyInstallation(!ap.has(P_QUIET));
}
// Sort files for determinism
projectFiles = projectFiles.stream()
.sorted(AutoBuild.FILE_ORDERING)
@@ -149,16 +147,30 @@ public class Main {
.sorted(AutoBuild.FILE_ORDERING)
.collect(Collectors.toCollection(() -> new LinkedHashSet<>()));
// Extract HTML files first, as they may contain embedded TypeScript code
for (File file : files) {
if (FileType.forFile(file, extractorConfig) == FileType.HTML) {
ensureFileIsExtracted(file, ap);
}
}
TypeScriptParser tsParser = extractorState.getTypeScriptParser();
tsParser.setTypescriptRam(extractorConfig.getTypeScriptRam());
if (containsTypeScriptFiles()) {
tsParser.verifyInstallation(!ap.has(P_QUIET));
}
for (File projectFile : projectFiles) {
long start = verboseLogStartTimer(ap, "Opening project " + projectFile);
ParsedProject project = tsParser.openProject(projectFile, DependencyInstallationResult.empty, VirtualSourceRoot.none);
ParsedProject project = tsParser.openProject(projectFile, DependencyInstallationResult.empty, extractorConfig.getVirtualSourceRoot());
verboseLogEndTimer(ap, start);
// Extract all files belonging to this project which are also matched
// by our include/exclude filters.
List<File> filesToExtract = new ArrayList<>();
for (File sourceFile : project.getOwnFiles()) {
if (files.contains(normalizeFile(sourceFile))
File normalizedFile = normalizeFile(sourceFile);
if ((files.contains(normalizedFile) || extractorState.getSnippets().containsKey(normalizedFile.toPath()))
&& !extractedFiles.contains(sourceFile.getAbsoluteFile())
&& FileType.TYPESCRIPT.getExtensions().contains(FileUtil.extension(sourceFile))) {
filesToExtract.add(sourceFile);
@@ -287,10 +299,14 @@ public class Main {
}
public void collectFiles(ArgsParser ap) {
for (File f : ap.getOneOrMoreFiles("files", FileMode.FILE_OR_DIRECTORY_MUST_EXIST))
for (File f : getFilesArg(ap))
collectFiles(f, true);
}
private List<File> getFilesArg(ArgsParser ap) {
return ap.getOneOrMoreFiles("files", FileMode.FILE_OR_DIRECTORY_MUST_EXIST);
}
public void setupMatchers(ArgsParser ap) {
Set<String> includes = new LinkedHashSet<>();
@@ -444,6 +460,21 @@ public class Main {
if (ap.has(P_TYPESCRIPT)) return TypeScriptMode.BASIC;
return TypeScriptMode.NONE;
}
private Path inferSourceRoot(ArgsParser ap) {
List<File> files = getFilesArg(ap);
Path sourceRoot = files.iterator().next().toPath().toAbsolutePath().getParent();
for (File file : files) {
Path path = file.toPath().toAbsolutePath().getParent();
for (int i = 0; i < sourceRoot.getNameCount(); ++i) {
if (!(i < path.getNameCount() && path.getName(i).equals(sourceRoot.getName(i)))) {
sourceRoot = sourceRoot.subpath(0, i);
break;
}
}
}
return sourceRoot;
}
private ExtractorConfig parseJSOptions(ArgsParser ap) {
ExtractorConfig cfg =
@@ -466,6 +497,17 @@ public class Main {
? UnitParser.parseOpt(ap.getString(P_TYPESCRIPT_RAM), UnitParser.MEGABYTES)
: 0);
if (ap.has(P_DEFAULT_ENCODING)) cfg = cfg.withDefaultEncoding(ap.getString(P_DEFAULT_ENCODING));
// Make a usable virtual source root mapping.
// The concept of source root and scratch directory do not exist in the legacy extractor,
// so we construct these based on what we have.
String odasaDbDir = Env.systemEnv().getNonEmpty(Var.ODASA_DB);
VirtualSourceRoot virtualSourceRoot =
odasaDbDir == null
? VirtualSourceRoot.none
: new VirtualSourceRoot(inferSourceRoot(ap), Paths.get(odasaDbDir, "working"));
cfg = cfg.withVirtualSourceRoot(virtualSourceRoot);
return cfg;
}

5
javascript/extractor/src/com/semmle/js/extractor/VirtualSourceRoot.java
View File

@@ -43,6 +43,11 @@ public class VirtualSourceRoot {
}
public Path toVirtualFile(Path file) {
if (file.startsWith(virtualSourceRoot)) {
// 'qltest' creates a virtual source root inside the real source root.
// Make sure such files don't appear to be inside the real source root.
return null;
}
return translate(sourceRoot, virtualSourceRoot, file);
}