Merge pull request #130 from github/erik-krogh/more-types

Better type resolution
This commit is contained in:
Erik Krogh Kristensen
2021-11-16 17:52:15 +01:00
committed by GitHub
15 changed files with 270 additions and 65 deletions

View File

@@ -1,10 +1 @@
import ql
private import codeql_ql.ast.internal.AstNodes as AstNodes
query AstNode nonTotalGetParent() {
exists(AstNodes::toQL(result).getParent()) and
not exists(result.getParent()) and
not result.getLocation().getStartColumn() = 1 and // startcolumn = 1 <=> top level in file <=> fine to have no parent
not result instanceof YAML::YAMLNode and // parents for YAML doens't work
not (result instanceof QLDoc and result.getLocation().getFile().getExtension() = "dbscheme") // qldoc in dbschemes are not hooked up
}
private import codeql_ql.ast.internal.AstNodes::AstConsistency

View File

@@ -819,7 +819,7 @@ class NewType extends TNewType, TypeDeclaration, ModuleDeclaration {
* A branch in a `newtype`.
* E.g. `Bar()` or `Baz()` in `newtype Foo = Bar() or Baz()`.
*/
class NewTypeBranch extends TNewTypeBranch, PredicateOrBuiltin, TypeDeclaration {
class NewTypeBranch extends TNewTypeBranch, Predicate, TypeDeclaration {
QL::DatatypeBranch branch;
NewTypeBranch() { this = TNewTypeBranch(branch) }
@@ -835,7 +835,7 @@ class NewTypeBranch extends TNewTypeBranch, PredicateOrBuiltin, TypeDeclaration
}
/** Gets the body of this branch. */
Formula getBody() { toQL(result) = branch.getChild(_).(QL::Body).getChild() }
override Formula getBody() { toQL(result) = branch.getChild(_).(QL::Body).getChild() }
override NewTypeBranchType getReturnType() { result.getDeclaration() = this }
@@ -1551,7 +1551,7 @@ class ExprAggregate extends TExprAggregate, Aggregate {
override Type getType() {
exists(PrimitiveType prim | prim = result |
kind.regexpMatch("(strict)?count|sum|min|max|rank") and
kind.regexpMatch("(strict)?count") and
result.getName() = "int"
or
kind.regexpMatch("(strict)?concat") and
@@ -1615,16 +1615,16 @@ class FullAggregate extends TFullAggregate, Aggregate {
override Type getType() {
exists(PrimitiveType prim | prim = result |
kind.regexpMatch("(strict)?(count|sum|min|max|rank)") and
kind = ["count", "strictcount"] and
result.getName() = "int"
or
kind.regexpMatch("(strict)?concat") and
result.getName() = "string"
)
or
kind = ["any", "min", "max", "unique"] and
kind = ["any", "min", "max", "unique", "rank", "sum", "strictsum"] and
not exists(this.getExpr(_)) and
result = this.getArgument(0).getTypeExpr().getResolvedType()
result = this.getArgument(0).getType()
or
not kind = ["count", "strictcount"] and
result = this.getExpr(0).getType()
@@ -1761,6 +1761,10 @@ class Super extends TSuper, Expr {
Super() { this = TSuper(_) }
override string getAPrimaryQlClass() { result = "Super" }
override Type getType() {
exists(MemberCall call | call.getBase() = this | result = call.getTarget().getDeclaringType())
}
}
/** An access to `result`. */
@@ -1794,6 +1798,7 @@ class Negation extends TNegation, Formula {
/** An expression, such as `x+4`. */
class Expr extends TExpr, AstNode {
cached
Type getType() { none() }
}
@@ -1874,15 +1879,14 @@ class AddSubExpr extends TAddSubExpr, BinOpExpr {
result = this.getLeftOperand().getType() and
result = this.getRightOperand().getType()
or
// Both operands are subtypes of `int`
result.getName() = "int" and
result = this.getLeftOperand().getType().getASuperType*() and
result = this.getRightOperand().getType().getASuperType*()
// Both operands are subtypes of `int` / `string` / `float`
exprOfPrimitiveAddType(result) = this.getLeftOperand() and
exprOfPrimitiveAddType(result) = this.getRightOperand()
or
// Coercion to from `int` to `float`
exists(PrimitiveType i | result.getName() = "float" and i.getName() = "int" |
this.getAnOperand().getType() = result and
this.getAnOperand().getType().getASuperType*() = i
this.getAnOperand() = exprOfPrimitiveAddType(i)
)
or
// Coercion to `string`
@@ -1899,6 +1903,25 @@ class AddSubExpr extends TAddSubExpr, BinOpExpr {
}
}
pragma[noinline]
private Expr exprOfPrimitiveAddType(PrimitiveType t) {
result.getType() = getASubTypeOfAddPrimitive(t)
}
/**
* Gets a subtype of the given primitive type `prim`.
* This predicate does not consider float to be a supertype of int.
*/
private Type getASubTypeOfAddPrimitive(PrimitiveType prim) {
result = prim and
result.getName() = ["int", "string", "float"]
or
exists(Type superType | superType = getASubTypeOfAddPrimitive(prim) |
result.getASuperType() = superType and
not (result.getName() = "int" and superType.getName() = "float")
)
}
/**
* An addition expression, such as `x + y`.
*/
@@ -1939,15 +1962,18 @@ class MulDivModExpr extends TMulDivModExpr, BinOpExpr {
this.getLeftOperand().getType() = result and
this.getRightOperand().getType() = result
or
// Both operands are subtypes of `int`
result.getName() = "int" and
result = this.getLeftOperand().getType().getASuperType*() and
result = this.getRightOperand().getType().getASuperType*()
// Both operands are subtypes of `int`/`float`
result.getName() = ["int", "float"] and
exprOfPrimitiveAddType(result) = this.getLeftOperand() and
exprOfPrimitiveAddType(result) = this.getRightOperand()
or
// Coercion from `int` to `float`
exists(PrimitiveType i | result.getName() = "float" and i.getName() = "int" |
this.getAnOperand().getType() = result and
this.getAnOperand().getType().getASuperType*() = i
this.getLeftOperand() = exprOfPrimitiveAddType(result) and
this.getRightOperand() = exprOfPrimitiveAddType(i)
or
this.getRightOperand() = exprOfPrimitiveAddType(result) and
this.getLeftOperand() = exprOfPrimitiveAddType(i)
)
}
@@ -2281,6 +2307,8 @@ module YAML {
)
}
YAMLListItem getListItem() { toQL(result).getParent() = yamle }
/** Gets the value of this YAML entry. */
YAMLValue getValue() {
exists(QL::YamlKeyvaluepair pair |
@@ -2393,7 +2421,7 @@ module YAML {
deps.getLocation().getFile() = file and entry.getLocation().getFile() = file
|
deps.isRoot() and
deps.getKey().getQualifiedName() = "dependencies" and
deps.getKey().getQualifiedName() = ["dependencies", "libraryPathDependencies"] and
entry.getLocation().getStartLine() = 1 + deps.getLocation().getStartLine() and
entry.getLocation().getStartColumn() > deps.getLocation().getStartColumn()
)
@@ -2408,8 +2436,11 @@ module YAML {
predicate hasDependency(string name, string version) {
exists(YAMLEntry entry | this.isADependency(entry) |
entry.getKey().getQualifiedName() = name and
entry.getKey().getQualifiedName().trim() = name and
entry.getValue().getValue() = version
or
name = entry.getListItem().getValue().getValue().trim() and
version = "\"*\""
)
}
@@ -2431,7 +2462,7 @@ module YAML {
*/
QLPack getADependency() {
exists(string name | this.hasDependency(name, _) |
result.getName().replaceAll("-", "/") = name
result.getName().replaceAll("-", "/") = name.replaceAll("-", "/")
)
}

View File

@@ -193,7 +193,8 @@ QL::AstNode toQL(AST::AstNode n) {
n = TAnnotationArg(result)
}
class TPredicate = TCharPred or TClasslessPredicate or TClassPredicate or TDBRelation;
class TPredicate =
TCharPred or TClasslessPredicate or TClassPredicate or TDBRelation or TNewTypeBranch;
class TPredOrBuiltin = TPredicate or TNewTypeBranch or TBuiltin;
@@ -208,3 +209,16 @@ class TTypeDeclaration = TClass or TNewType or TNewTypeBranch;
class TModuleDeclaration = TClasslessPredicate or TModule or TClass or TNewType;
class TVarDef = TVarDecl or TAsExpr;
module AstConsistency {
import ql
query predicate nonTotalGetParent(AstNode node) {
exists(toQL(node).getParent()) and
not exists(node.getParent()) and
not node.getLocation().getStartColumn() = 1 and // startcolumn = 1 <=> top level in file <=> fine to have no parent
exists(node.toString()) and // <- there are a few parse errors in "global-data-flow-java-1.ql", this way we filter them out.
not node instanceof YAML::YAMLNode and // parents for YAML doens't work
not (node instanceof QLDoc and node.getLocation().getFile().getExtension() = "dbscheme") // qldoc in dbschemes are not hooked up
}
}

View File

@@ -139,6 +139,7 @@ private predicate resolveQualifiedName(Import imp, ContainerOrModule m, int i) {
}
private predicate resolveSelectionName(Import imp, ContainerOrModule m, int i) {
(m.(File_).getFile().getExtension() = "qll" or not m instanceof File_) and
exists(int last |
resolveQualifiedName(imp, m, last) and
last = count(int j | exists(imp.getQualifiedName(j))) - 1
@@ -281,7 +282,11 @@ module ModConsistency {
query predicate multipleResolve(Import imp, int c, ContainerOrModule m) {
c = strictcount(ContainerOrModule m0 | resolve(imp, m0)) and
c > 1 and
resolve(imp, m)
resolve(imp, m) and
not imp.getLocation()
.getFile()
.getAbsolutePath()
.regexpMatch(".*/(test|examples|ql-training|recorded-call-graph-metrics)/.*")
}
query predicate multipleResolveModuleExpr(ModuleExpr me, int c, ContainerOrModule m) {

View File

@@ -3,44 +3,43 @@ private import Builtins
private import codeql_ql.ast.internal.Module
private import codeql_ql.ast.internal.AstNodes
private class TClasslessPredicateOrNewTypeBranch = TClasslessPredicate or TNewTypeBranch;
private string getPredicateName(TClasslessPredicateOrNewTypeBranch p) {
result = p.(ClasslessPredicate).getName() or
result = p.(NewTypeBranch).getName()
}
private predicate definesPredicate(
FileOrModule m, string name, int arity, TClasslessPredicateOrNewTypeBranch p, boolean public
FileOrModule m, string name, int arity, Predicate p, boolean public
) {
m = getEnclosingModule(p) and
name = getPredicateName(p) and
public = getPublicBool(p) and
arity = [p.(ClasslessPredicate).getArity(), count(p.(NewTypeBranch).getField(_))]
or
// import X
exists(Import imp, FileOrModule m0 |
m = getEnclosingModule(imp) and
m0 = imp.getResolvedModule() and
not exists(imp.importedAs()) and
definesPredicate(m0, name, arity, p, true) and
public = getPublicBool(imp)
)
or
// predicate X = Y
exists(ClasslessPredicate alias |
m = getEnclosingModule(alias) and
name = alias.getName() and
resolvePredicateExpr(alias.getAlias(), p) and
public = getPublicBool(alias) and
arity = alias.getArity()
(
p instanceof NewTypeBranch or
p instanceof ClasslessPredicate
) and
(
m = getEnclosingModule(p) and
name = p.getName() and
public = getPublicBool(p) and
arity = p.getArity()
or
// import X
exists(Import imp, FileOrModule m0 |
m = getEnclosingModule(imp) and
m0 = imp.getResolvedModule() and
not exists(imp.importedAs()) and
definesPredicate(m0, name, arity, p, true) and
public = getPublicBool(imp)
)
or
// predicate X = Y
exists(ClasslessPredicate alias |
m = getEnclosingModule(alias) and
name = alias.getName() and
resolvePredicateExpr(alias.getAlias(), p) and
public = getPublicBool(alias) and
arity = alias.getArity()
)
)
}
cached
private module Cached {
cached
predicate resolvePredicateExpr(PredicateExpr pe, ClasslessPredicate p) {
predicate resolvePredicateExpr(PredicateExpr pe, Predicate p) {
exists(FileOrModule m, boolean public |
not exists(pe.getQualifier()) and
m = getEnclosingModule(pe).getEnclosing*() and
@@ -49,7 +48,7 @@ private module Cached {
m = pe.getQualifier().getResolvedModule() and
public = true
|
definesPredicate(m, pe.getName(), count(p.getParameter(_)), p, public)
definesPredicate(m, pe.getName(), p.getArity(), p, public)
)
}

View File

@@ -340,6 +340,12 @@ module TyConsistency {
resolveTypeExpr(te, t)
}
query predicate multiplePrimitives(TypeExpr te, int c, PrimitiveType t) {
c = strictcount(PrimitiveType t0 | resolveTypeExpr(te, t0)) and
c > 1 and
resolveTypeExpr(te, t)
}
query predicate varDefNoType(VarDef def) {
not exists(def.getType()) and
not def.getLocation()
@@ -360,4 +366,10 @@ module TyConsistency {
.getAbsolutePath()
.regexpMatch(".*/(test|examples|ql-training|recorded-call-graph-metrics)/.*")
}
query predicate multiplePrimitivesExpr(Expr e, int c, PrimitiveType t) {
c = strictcount(PrimitiveType t0 | t0 = e.getType()) and
c > 1 and
t = e.getType()
}
}

View File

@@ -1,5 +1,5 @@
import ql
import codeql_ql.ast.internal.AstNodes
private import codeql_ql.ast.internal.AstNodes
private class TScope =
TClass or TAggregate or TQuantifier or TSelect or TPredicate or TNewTypeBranch;
@@ -89,4 +89,8 @@ module VarConsistency {
decl = f.getDeclaration() and
strictcount(f.getDeclaration()) > 1
}
query predicate noFieldDef(FieldAccess f) { not exists(f.getDeclaration()) }
query predicate noVarDef(VarAccess v) { not exists(v.getDeclaration()) }
}

View File

@@ -8,7 +8,7 @@
* to hold for only the AST nodes you wish to view.
*/
import ast.internal.TreeSitter::Generated
import ast.internal.TreeSitter::QL
private import codeql.Locations
/**

View File

@@ -0,0 +1,45 @@
/**
* @name Consistency predicate that should be empty
* @description This query should have no results on the CodeQL repos.
* It's marked as a warning query to make the results visible.
* @kind problem
* @problem.severity warning
* @precision very-high
* @id ql/diagnostics/empty-consitencies
*/
import ql
import codeql_ql.ast.internal.Predicate::PredConsistency as PredConsistency
import codeql_ql.ast.internal.Type::TyConsistency as TypeConsistency
import codeql_ql.ast.internal.Builtins::BuildinsConsistency as BuildinsConsistency
import codeql_ql.ast.internal.Module::ModConsistency as ModConsistency
import codeql_ql.ast.internal.Variable::VarConsistency as VarConsistency
import codeql_ql.ast.internal.AstNodes::AstConsistency as AstConsistency
from AstNode node, string msg
where
PredConsistency::noResolveCall(node) and msg = "PredConsistency::noResolveCall"
or
PredConsistency::noResolvePredicateExpr(node) and msg = "PredConsistency::noResolvePredicateExpr"
or
TypeConsistency::exprNoType(node) and msg = "TypeConsistency::exprNoType"
or
TypeConsistency::varDefNoType(node) and msg = "TypeConsistency::varDefNoType"
or
TypeConsistency::multiplePrimitives(node, _, _) and msg = "TypeConsistency::multiplePrimitives"
or
TypeConsistency::multiplePrimitivesExpr(node, _, _) and
msg = "TypeConsistency::multiplePrimitivesExpr"
or
AstConsistency::nonTotalGetParent(node) and msg = "AstConsistency::nonTotalGetParent"
or
//or // has 1 result, but the CodeQL compiler also can't figure out that one. I suppoed the file is never imported.
//TypeConsistency::noResolve(node) and msg = "TypeConsistency::noResolve"
//or // has 1 result, but the CodeQL compiler also can't figure out that one. Same file as above.
//ModConsistency::noResolve(node) and msg = "ModConsistency::noResolve"
ModConsistency::noResolveModuleExpr(node) and msg = "ModConsistency::noResolveModuleExpr"
or
VarConsistency::noFieldDef(node) and msg = "VarConsistency::noFieldDef"
or
VarConsistency::noVarDef(node) and msg = "VarConsistency::noVarDef"
select node, msg

View File

@@ -37,3 +37,11 @@ module Buildins {
predicate regexpCapture(string s) { "foo".regexpCapture("\\w", 1) = s }
}
cached
newtype TApiNode = MkRoot()
private predicate edge(TApiNode a, TApiNode b) { a = b }
cached
int distanceFromRoot(TApiNode nd) = shortestDistances(MkRoot/0, edge/2)(_, nd, result)

View File

@@ -22,3 +22,8 @@ getTarget
| packs/src/SrcThing.qll:5:3:5:8 | PredicateCall | packs/src/SrcThing.qll:8:1:8:30 | ClasslessPredicate bar |
dependsOn
| packs/src/qlpack.yml:1:1:1:4 | ql-testing-src-pack | packs/lib/qlpack.yml:1:1:1:4 | ql-testing-lib-pack |
exprPredicate
| Foo.qll:24:22:24:31 | predicate | Foo.qll:22:3:22:32 | ClasslessPredicate myThing0 |
| Foo.qll:26:22:26:31 | predicate | Foo.qll:20:3:20:54 | ClasslessPredicate myThing2 |
| Foo.qll:47:55:47:62 | predicate | Foo.qll:42:20:42:27 | NewTypeBranch MkRoot |
| Foo.qll:47:65:47:70 | predicate | Foo.qll:44:9:44:56 | ClasslessPredicate edge |

View File

@@ -3,3 +3,5 @@ import ql
query AstNode getTarget(Call call) { result = call.getTarget() }
query YAML::QLPack dependsOn(YAML::QLPack pack) { result = pack.getADependency() }
query Predicate exprPredicate(PredicateExpr expr) { result = expr.getResolvedPredicate() }

30
ql/test/type/Test.qll Normal file
View File

@@ -0,0 +1,30 @@
import ql
class Strings extends string {
Strings() { this = ["", "f", "o", "foo", "bar", "b", "a", "r", "ba", "ar"] }
}
class Floats extends float {
Floats() { this = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0] }
}
string conc(Strings a, Strings b) { result = a + b }
float floats(Floats a, Floats b) { result = a + b }
class Base extends string {
Base() { this = "foo" }
int foo() { result = 1 }
}
class Sub extends Base {
Sub() { this = "bar" }
int bar() { result = Base.super.foo() }
int bar2() { result = super.foo() }
}
bindingset[result, a, b]
int integerMul(int a, int b) { result = a * b }

View File

@@ -0,0 +1,56 @@
| Test.qll:4:15:4:18 | this | Test.qll:3:1:5:1 | Strings |
| Test.qll:4:15:4:18 | this | Test.qll:3:1:5:1 | Strings.Strings |
| Test.qll:4:15:4:18 | this | Test.qll:3:1:5:1 | Strings.extends |
| Test.qll:4:22:4:76 | Set | file://:0:0:0:0 | string |
| Test.qll:4:23:4:24 | String | file://:0:0:0:0 | string |
| Test.qll:4:27:4:29 | String | file://:0:0:0:0 | string |
| Test.qll:4:32:4:34 | String | file://:0:0:0:0 | string |
| Test.qll:4:37:4:41 | String | file://:0:0:0:0 | string |
| Test.qll:4:44:4:48 | String | file://:0:0:0:0 | string |
| Test.qll:4:51:4:53 | String | file://:0:0:0:0 | string |
| Test.qll:4:56:4:58 | String | file://:0:0:0:0 | string |
| Test.qll:4:61:4:63 | String | file://:0:0:0:0 | string |
| Test.qll:4:66:4:69 | String | file://:0:0:0:0 | string |
| Test.qll:4:72:4:75 | String | file://:0:0:0:0 | string |
| Test.qll:8:14:8:17 | this | Test.qll:7:1:9:1 | Floats |
| Test.qll:8:14:8:17 | this | Test.qll:7:1:9:1 | Floats.Floats |
| Test.qll:8:14:8:17 | this | Test.qll:7:1:9:1 | Floats.extends |
| Test.qll:8:21:8:70 | Set | file://:0:0:0:0 | float |
| Test.qll:8:22:8:24 | Float | file://:0:0:0:0 | float |
| Test.qll:8:27:8:29 | Float | file://:0:0:0:0 | float |
| Test.qll:8:32:8:34 | Float | file://:0:0:0:0 | float |
| Test.qll:8:37:8:39 | Float | file://:0:0:0:0 | float |
| Test.qll:8:42:8:44 | Float | file://:0:0:0:0 | float |
| Test.qll:8:47:8:49 | Float | file://:0:0:0:0 | float |
| Test.qll:8:52:8:54 | Float | file://:0:0:0:0 | float |
| Test.qll:8:57:8:59 | Float | file://:0:0:0:0 | float |
| Test.qll:8:62:8:64 | Float | file://:0:0:0:0 | float |
| Test.qll:8:67:8:69 | Float | file://:0:0:0:0 | float |
| Test.qll:11:37:11:42 | result | file://:0:0:0:0 | string |
| Test.qll:11:46:11:46 | a | Test.qll:3:1:5:1 | Strings |
| Test.qll:11:46:11:50 | AddExpr | file://:0:0:0:0 | string |
| Test.qll:11:50:11:50 | b | Test.qll:3:1:5:1 | Strings |
| Test.qll:13:36:13:41 | result | file://:0:0:0:0 | float |
| Test.qll:13:45:13:45 | a | Test.qll:7:1:9:1 | Floats |
| Test.qll:13:45:13:49 | AddExpr | file://:0:0:0:0 | float |
| Test.qll:13:49:13:49 | b | Test.qll:7:1:9:1 | Floats |
| Test.qll:16:12:16:15 | this | Test.qll:15:1:19:1 | Base |
| Test.qll:16:12:16:15 | this | Test.qll:15:1:19:1 | Base.Base |
| Test.qll:16:12:16:15 | this | Test.qll:15:1:19:1 | Base.extends |
| Test.qll:16:19:16:23 | String | file://:0:0:0:0 | string |
| Test.qll:18:15:18:20 | result | file://:0:0:0:0 | int |
| Test.qll:18:24:18:24 | Integer | file://:0:0:0:0 | int |
| Test.qll:22:11:22:14 | this | Test.qll:21:1:27:1 | Sub |
| Test.qll:22:11:22:14 | this | Test.qll:21:1:27:1 | Sub.Sub |
| Test.qll:22:11:22:14 | this | Test.qll:21:1:27:1 | Sub.extends |
| Test.qll:22:18:22:22 | String | file://:0:0:0:0 | string |
| Test.qll:24:15:24:20 | result | file://:0:0:0:0 | int |
| Test.qll:24:24:24:33 | Super | Test.qll:15:1:19:1 | Base |
| Test.qll:24:24:24:39 | MemberCall | file://:0:0:0:0 | int |
| Test.qll:26:16:26:21 | result | file://:0:0:0:0 | int |
| Test.qll:26:25:26:29 | Super | Test.qll:15:1:19:1 | Base |
| Test.qll:26:25:26:35 | MemberCall | file://:0:0:0:0 | int |
| Test.qll:30:32:30:37 | result | file://:0:0:0:0 | int |
| Test.qll:30:41:30:41 | a | file://:0:0:0:0 | int |
| Test.qll:30:41:30:45 | MulExpr | file://:0:0:0:0 | int |
| Test.qll:30:45:30:45 | b | file://:0:0:0:0 | int |

3
ql/test/type/type.ql Normal file
View File

@@ -0,0 +1,3 @@
import ql
query Type getType(Expr e) { result = e.getType() }