hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-05-13 10:49:26 +02:00
Code Issues Packages Projects Releases Wiki Activity

merge in main

Browse Source
This commit is contained in:
yoff
2022-06-23 09:05:32 +00:00
committed by GitHub
parent 8bf60301da
commit 140dc1a61e
4448 changed files with 340100 additions and 31408 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
javascript/extractor/lib/typescript/package.json
View File

@@ -2,7 +2,7 @@
"name": "typescript-parser-wrapper",
"private": true,
"dependencies": {
"typescript": "4.6.2"
"typescript": "4.7.2"
},
"scripts": {
"build": "tsc --project tsconfig.json",

6
javascript/extractor/lib/typescript/src/main.ts
View File

@@ -670,6 +670,12 @@ function handleOpenProjectCommand(command: OpenProjectCommand) {
if (file.endsWith(".d.ts")) {
return pathlib.basename(file, ".d.ts");
}
if (file.endsWith(".d.mts") || file.endsWith(".d.cts")) {
// We don't extract d.mts or d.cts files, but their symbol can coincide with that of a d.ts file,
// which means any module bindings we generate for it will ultimately be visible in QL.
let base = pathlib.basename(file);
return base.substring(0, base.length - '.d.mts'.length);
}
let base = pathlib.basename(file);
let dot = base.lastIndexOf('.');
return dot === -1 || dot === 0 ? base : base.substring(0, dot);

1
javascript/extractor/lib/typescript/src/type_table.ts
View File

@@ -1068,6 +1068,7 @@ export class TypeTable {
let superType = this.typeChecker.getTypeFromTypeNode(typeExpr);
if (superType == null) continue;
let baseTypeSymbol = superType.symbol;
baseTypeSymbol = (baseTypeSymbol as any)?.type?.symbol ?? baseTypeSymbol;
if (baseTypeSymbol == null) continue;
let baseId = this.getSymbolId(baseTypeSymbol);
// Note: take care not to perform a recursive call between the two `push` calls.

8
javascript/extractor/lib/typescript/yarn.lock
View File

@@ -6,7 +6,7 @@
version "12.7.11"
resolved node-12.7.11.tgz#be879b52031cfb5d295b047f5462d8ef1a716446
typescript@4.6.2:
version "4.6.2"
resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.6.2.tgz#fe12d2727b708f4eef40f51598b3398baa9611d4"
integrity sha512-HM/hFigTBHZhLXshn9sN37H085+hQGeJHJ/X7LpBWLID/fbc2acUMfU+lGD98X81sKP+pFa9f0DZmCwB9GnbAg==
typescript@4.7.2:
version "4.7.2"
resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.7.2.tgz#1f9aa2ceb9af87cca227813b4310fff0b51593c4"
integrity sha512-Mamb1iX2FDUpcTRzltPxgWMKy3fhg0TN378ylbktPGPK/99KbDtMQ4W1hwgsbPAsG3a0xKa1vmw4VKZQbkvz5A==

5
javascript/extractor/src/com/semmle/js/dependencies/Fetcher.java
View File

@@ -141,8 +141,9 @@ public class Fetcher {
entryPath = entryPath.subpath(1, entryPath.getNameCount());
String filename = entryPath.getFileName().toString();
if (!filename.endsWith(".d.ts") && !filename.equals("package.json")) {
continue; // Only extract .d.ts files and package.json
if (!filename.endsWith(".d.ts") && !filename.endsWith(".d.mts") && !filename.endsWith(".d.cts")
&& !filename.equals("package.json")) {
continue; // Only extract .d.ts, .d.mts, .d.cts files, and package.json
}
relativePaths.add(entryPath);
Path outputFile = destDir.resolve(entryPath);

15
javascript/extractor/src/com/semmle/js/extractor/ASTExtractor.java
View File

@@ -590,7 +590,7 @@ public class ASTExtractor {
trapwriter.addTuple("literals", valueString, source, key);
Position start = nd.getLoc().getStart();
com.semmle.util.locations.Position startPos = new com.semmle.util.locations.Position(start.getLine(), start.getColumn() + 1 /* Convert from 0-based to 1-based. */, start.getOffset());
if (nd.isRegExp()) {
OffsetTranslation offsets = new OffsetTranslation();
offsets.set(0, 1); // skip the initial '/'
@@ -622,7 +622,7 @@ public class ASTExtractor {
/**
* Constant-folds simple string concatenations in `exp` while keeping an offset translation
* that tracks back to the original source.
*/
*/
private Pair<String, OffsetTranslation> getStringConcatResult(Expression exp) {
if (exp instanceof BinaryExpression) {
BinaryExpression be = (BinaryExpression) exp;
@@ -636,7 +636,7 @@ public class ASTExtractor {
if (str.length() > 1000) {
return null;
}
int delta = be.getRight().getLoc().getStart().getOffset() - be.getLeft().getLoc().getStart().getOffset();
int offset = left.fst().length();
return Pair.make(str, left.snd().append(right.snd(), offset, delta));
@@ -747,7 +747,7 @@ public class ASTExtractor {
visit(nd.getProperty(), key, 1, IdContext.TYPE_LABEL);
} else {
IdContext baseIdContext =
c.idcontext == IdContext.EXPORT ? IdContext.EXPORT_BASE : IdContext.VAR_BIND;
(c.idcontext == IdContext.EXPORT || c.idcontext == IdContext.EXPORT_BASE) ? IdContext.EXPORT_BASE : IdContext.VAR_BIND;
visit(nd.getObject(), key, 0, baseIdContext);
visit(nd.getProperty(), key, 1, nd.isComputed() ? IdContext.VAR_BIND : IdContext.LABEL);
}
@@ -848,14 +848,14 @@ public class ASTExtractor {
public Label visit(BinaryExpression nd, Context c) {
Label key = super.visit(nd, c);
if (nd.getOperator().equals("in") && nd.getLeft() instanceof Identifier && ((Identifier)nd.getLeft()).getName().startsWith("#")) {
// this happens with Ergonomic brand checks for Private Fields (see https://github.com/tc39/proposal-private-fields-in-in).
// this happens with Ergonomic brand checks for Private Fields (see https://github.com/tc39/proposal-private-fields-in-in).
// it's the only case where private field identifiers are used not as a field.
visit(nd.getLeft(), key, 0, IdContext.LABEL, true);
} else {
visit(nd.getLeft(), key, 0, true);
}
visit(nd.getRight(), key, 1, true);
extractRegxpFromBinop(nd, c);
return key;
}
@@ -1815,7 +1815,7 @@ public class ASTExtractor {
visit(nd.getLocal(), lbl, 1, nd.hasTypeKeyword() ? IdContext.TYPE_ONLY_IMPORT : c.idcontext);
if (nd.hasTypeKeyword()) {
trapwriter.addTuple("has_type_keyword", lbl);
}
}
return lbl;
}
@@ -2191,6 +2191,7 @@ public class ASTExtractor {
visitAll(nd.getBody(), key);
contextManager.leaveContainer();
scopeManager.leaveScope();
emitNodeSymbol(nd, key);
return key;
}

2
javascript/extractor/src/com/semmle/js/extractor/FileExtractor.java
View File

@@ -203,7 +203,7 @@ public class FileExtractor {
}
},
TYPESCRIPT(".ts", ".tsx") {
TYPESCRIPT(".ts", ".tsx", ".mts", ".cts") {
@Override
protected boolean contains(File f, String lcExt, ExtractorConfig config) {
if (config.getTypeScriptMode() == TypeScriptMode.NONE) return false;

5
javascript/extractor/src/com/semmle/js/extractor/HTMLExtractor.java
View File

@@ -40,7 +40,8 @@ public class HTMLExtractor implements IExtractor {
this.textualExtractor = textualExtractor;
this.scopeManager =
new ScopeManager(textualExtractor.getTrapwriter(), config.getEcmaVersion(), true);
new ScopeManager(textualExtractor.getTrapwriter(), config.getEcmaVersion(),
ScopeManager.FileKind.TEMPLATE);
}
/*
@@ -425,7 +426,7 @@ public class HTMLExtractor implements IExtractor {
extractSnippet(
TopLevelKind.ANGULAR_STYLE_TEMPLATE,
config.withSourceType(SourceType.ANGULAR_STYLE_TEMPLATE),
new ScopeManager(textualExtractor.getTrapwriter(), ECMAVersion.ECMA2020, true),
new ScopeManager(textualExtractor.getTrapwriter(), ECMAVersion.ECMA2020, ScopeManager.FileKind.TEMPLATE),
textualExtractor,
m.group(bodyGroup),
m.start(bodyGroup),

6
javascript/extractor/src/com/semmle/js/extractor/Main.java
View File

@@ -43,7 +43,7 @@ public class Main {
* A version identifier that should be updated every time the extractor changes in such a way that
* it may produce different tuples for the same file under the same {@link ExtractorConfig}.
*/
public static final String EXTRACTOR_VERSION = "2022-02-22";
public static final String EXTRACTOR_VERSION = "2022-06-08";
public static final Pattern NEWLINE = Pattern.compile("\n");
@@ -153,7 +153,7 @@ public class Main {
ensureFileIsExtracted(file, ap);
}
}
TypeScriptParser tsParser = extractorState.getTypeScriptParser();
tsParser.setTypescriptRam(extractorConfig.getTypeScriptRam());
if (containsTypeScriptFiles()) {
@@ -460,7 +460,7 @@ public class Main {
if (ap.has(P_TYPESCRIPT)) return TypeScriptMode.BASIC;
return TypeScriptMode.NONE;
}
private Path inferSourceRoot(ArgsParser ap) {
List<File> files = getFilesArg(ap);
Path sourceRoot = files.iterator().next().toPath().toAbsolutePath().getParent();

36
javascript/extractor/src/com/semmle/js/extractor/ScopeManager.java
View File

@@ -97,20 +97,31 @@ public class ScopeManager {
}
}
public static enum FileKind {
/** Any file not specific to one of the other file kinds. */
PLAIN,
/** A file potentially containing template tags. */
TEMPLATE,
/** A d.ts file, containing TypeScript ambient declarations. */
TYPESCRIPT_DECLARATION,
}
private final TrapWriter trapWriter;
private Scope curScope;
private final Scope toplevelScope;
private final ECMAVersion ecmaVersion;
private final Set<String> implicitGlobals = new LinkedHashSet<String>();
private Scope implicitVariableScope;
private final boolean isInTemplateScope;
private final FileKind fileKind;
public ScopeManager(TrapWriter trapWriter, ECMAVersion ecmaVersion, boolean isInTemplateScope) {
public ScopeManager(TrapWriter trapWriter, ECMAVersion ecmaVersion, FileKind fileKind) {
this.trapWriter = trapWriter;
this.toplevelScope = enterScope(ScopeKind.GLOBAL, trapWriter.globalID("global_scope"), null);
this.ecmaVersion = ecmaVersion;
this.implicitVariableScope = toplevelScope;
this.isInTemplateScope = isInTemplateScope;
this.implicitVariableScope = toplevelScope;
this.fileKind = fileKind;
}
/**
@@ -118,7 +129,11 @@ public class ScopeManager {
* relevant template tags.
*/
public boolean isInTemplateFile() {
return isInTemplateScope;
return this.fileKind == FileKind.TEMPLATE;
}
public boolean isInTypeScriptDeclarationFile() {
return this.fileKind == FileKind.TYPESCRIPT_DECLARATION;
}
/**
@@ -221,7 +236,7 @@ public class ScopeManager {
/**
* Get the label for a given variable in the current scope; if it cannot be found, add it to the
* implicit variable scope (usually the global scope).
* implicit variable scope (usually the global scope).
*/
public Label getVarKey(String name) {
Label lbl = curScope.lookupVariable(name);
@@ -411,7 +426,7 @@ public class ScopeManager {
// cases where we turn on the 'declKind' flags
@Override
public Void visit(FunctionDeclaration nd, Void v) {
if (nd.hasDeclareKeyword()) return null;
if (nd.hasDeclareKeyword() && !isInTypeScriptDeclarationFile()) return null;
// strict mode functions are block-scoped, non-strict mode ones aren't
if (blockscope == isStrict) visit(nd.getId(), DeclKind.var);
return null;
@@ -419,7 +434,7 @@ public class ScopeManager {
@Override
public Void visit(ClassDeclaration nd, Void c) {
if (nd.hasDeclareKeyword()) return null;
if (nd.hasDeclareKeyword() && !isInTypeScriptDeclarationFile()) return null;
if (blockscope) visit(nd.getClassDef().getId(), DeclKind.varAndType);
return null;
}
@@ -468,7 +483,7 @@ public class ScopeManager {
@Override
public Void visit(VariableDeclaration nd, Void v) {
if (nd.hasDeclareKeyword()) return null;
if (nd.hasDeclareKeyword() && !isInTypeScriptDeclarationFile()) return null;
// in block scoping mode, only process 'let'; in non-block scoping
// mode, only process non-'let'
if (blockscope == nd.isBlockScoped(ecmaVersion)) visit(nd.getDeclarations());
@@ -503,7 +518,8 @@ public class ScopeManager {
@Override
public Void visit(NamespaceDeclaration nd, Void c) {
if (blockscope) return null;
boolean hasVariable = nd.isInstantiated() && !nd.hasDeclareKeyword();
boolean isAmbientOutsideDtsFile = nd.hasDeclareKeyword() && !isInTypeScriptDeclarationFile();
boolean hasVariable = nd.isInstantiated() && !isAmbientOutsideDtsFile;
visit(nd.getName(), hasVariable ? DeclKind.varAndNamespace : DeclKind.namespace);
return null;
}

2
javascript/extractor/src/com/semmle/js/extractor/ScriptExtractor.java
View File

@@ -77,7 +77,7 @@ public class ScriptExtractor implements IExtractor {
}
ScopeManager scopeManager =
new ScopeManager(textualExtractor.getTrapwriter(), config.getEcmaVersion(), false);
new ScopeManager(textualExtractor.getTrapwriter(), config.getEcmaVersion(), ScopeManager.FileKind.PLAIN);
Label toplevelLabel = null;
LoCInfo loc;
try {

6
javascript/extractor/src/com/semmle/js/extractor/TypeScriptExtractor.java
View File

@@ -22,8 +22,10 @@ public class TypeScriptExtractor implements IExtractor {
String source = textualExtractor.getSource();
File sourceFile = textualExtractor.getExtractedFile();
Result res = state.getTypeScriptParser().parse(sourceFile, source, textualExtractor.getMetrics());
ScopeManager scopeManager =
new ScopeManager(textualExtractor.getTrapwriter(), ECMAVersion.ECMA2017, false);
ScopeManager.FileKind fileKind = sourceFile.getName().endsWith(".d.ts")
? ScopeManager.FileKind.TYPESCRIPT_DECLARATION
: ScopeManager.FileKind.PLAIN;
ScopeManager scopeManager = new ScopeManager(textualExtractor.getTrapwriter(), ECMAVersion.ECMA2017, fileKind);
try {
FileSnippet snippet = state.getSnippets().get(sourceFile.toPath());
SourceType sourceType = snippet != null ? snippet.getSourceType() : jsExtractor.establishSourceType(source, false);

13
javascript/extractor/src/com/semmle/ts/ast/ExternalModuleDeclaration.java
View File

@@ -7,9 +7,10 @@ import com.semmle.js.ast.Visitor;
import java.util.List;
/** A statement of form <code>declare module "X" {...}</code>. */
public class ExternalModuleDeclaration extends Statement {
public class ExternalModuleDeclaration extends Statement implements INodeWithSymbol {
private final Literal name;
private final List<Statement> body;
private int symbol = -1;
public ExternalModuleDeclaration(SourceLocation loc, Literal name, List<Statement> body) {
super("ExternalModuleDeclaration", loc);
@@ -29,4 +30,14 @@ public class ExternalModuleDeclaration extends Statement {
public List<Statement> getBody() {
return body;
}
@Override
public int getSymbol() {
return this.symbol;
}
@Override
public void setSymbol(int symbol) {
this.symbol = symbol;
}
}

6
javascript/extractor/src/com/semmle/ts/extractor/TypeScriptASTConverter.java
View File

@@ -591,7 +591,7 @@ public class TypeScriptASTConverter {
return convertTryStatement(node, loc);
case "TupleType":
return convertTupleType(node, loc);
case "NamedTupleMember":
case "NamedTupleMember":
return convertNamedTupleMember(node, loc);
case "TypeAliasDeclaration":
return convertTypeAliasDeclaration(node, loc);
@@ -1710,7 +1710,9 @@ public class TypeScriptASTConverter {
}
if (nameNode instanceof Literal) {
// Declaration of form: declare module "X" {...}
return new ExternalModuleDeclaration(loc, (Literal) nameNode, body);
ExternalModuleDeclaration decl = new ExternalModuleDeclaration(loc, (Literal) nameNode, body);
attachSymbolInformation(decl, node);
return decl;
}
if (hasFlag(node, "GlobalAugmentation")) {
// Declaration of form: declare global {...}

42
javascript/extractor/tests/ts/output/trap/exportasnamespace.d.ts.trap
View File

@@ -97,28 +97,28 @@ scopenodes(#20001,#20033)
scopenesting(#20033,#20000)
is_module(#20001)
is_es2015_module(#20001)
#20034=*
stmts(#20034,30,#20001,0,"export ... foo();")
hasLocation(#20034,#20003)
stmt_containers(#20034,#20001)
#20034=@"var;{foo};{#20033}"
variables(#20034,"foo",#20033)
#20035=*
stmts(#20035,17,#20034,-1,"declare ... foo();")
#20036=@"loc,{#10000},1,8,1,30"
locations_default(#20036,#10000,1,8,1,30)
hasLocation(#20035,#20036)
stmts(#20035,30,#20001,0,"export ... foo();")
hasLocation(#20035,#20003)
stmt_containers(#20035,#20001)
has_declare_keyword(#20035)
#20037=*
exprs(#20037,78,#20035,-1,"foo")
hasLocation(#20037,#20013)
expr_containers(#20037,#20035)
literals("foo","foo",#20037)
#20038=@"var;{foo};{#20000}"
variables(#20038,"foo",#20000)
decl(#20037,#20038)
#20036=*
stmts(#20036,17,#20035,-1,"declare ... foo();")
#20037=@"loc,{#10000},1,8,1,30"
locations_default(#20037,#10000,1,8,1,30)
hasLocation(#20036,#20037)
stmt_containers(#20036,#20001)
has_declare_keyword(#20036)
#20038=*
exprs(#20038,78,#20036,-1,"foo")
hasLocation(#20038,#20013)
expr_containers(#20038,#20036)
literals("foo","foo",#20038)
decl(#20038,#20034)
#20039=*
scopes(#20039,1)
scopenodes(#20035,#20039)
scopenodes(#20036,#20039)
scopenesting(#20039,#20033)
#20040=@"var;{arguments};{#20039}"
variables(#20040,"arguments",#20039)
@@ -142,8 +142,8 @@ hasLocation(#20043,#20044)
exit_cfg_node(#20045,#20001)
hasLocation(#20045,#20031)
successor(#20041,#20045)
successor(#20034,#20035)
successor(#20035,#20041)
successor(#20043,#20034)
successor(#20035,#20036)
successor(#20036,#20041)
successor(#20043,#20035)
numlines(#10000,2,2,0)
filetype(#10000,"typescript")

3
javascript/extractor/tests/ts/output/trap/namespaces.ts.trap
View File

@@ -305,11 +305,12 @@ hasLocation(#20096,#20097)
enclosing_stmt(#20096,#20092)
expr_containers(#20096,#20001)
#20098=*
exprs(#20098,79,#20096,0,"M")
exprs(#20098,103,#20096,0,"M")
hasLocation(#20098,#20052)
enclosing_stmt(#20098,#20092)
expr_containers(#20098,#20001)
literals("M","M",#20098)
namespacebind(#20098,#20069)
bind(#20098,#20066)
#20099=*
exprs(#20099,0,#20096,1,"N")