hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-05-04 21:25:44 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3096 from tausbn/python-autoformat-almost-everything

Browse Source 
Python: Autoformat (almost) all `.qll` files.
This commit is contained in:
Rasmus Wriedt Larsen
2020-03-30 16:19:23 +02:00
committed by GitHub
parent fce04f0bd0 727cde31c9
commit 573494d313
133 changed files with 6905 additions and 11482 deletions
Download Patch File Download Diff File Expand all files Collapse all files

159
python/ql/src/Classes/ClassAttributes.qll
View File

@@ -3,7 +3,6 @@ private import semmle.python.pointsto.PointsTo
/** Helper class for UndefinedClassAttribute.ql and MaybeUndefinedClassAttribute.ql */
class CheckClass extends ClassObject {
private predicate ofInterest() {
not this.unknowableAttributes() and
not this.getPyClass().isProbableMixin() and
@@ -19,7 +18,8 @@ class CheckClass extends ClassObject {
forall(ClassObject sup |
sup = this.getAnImproperSuperType() and
sup.declaresAttribute("__init__") and
not sup = theObjectType() |
not sup = theObjectType()
|
sup.declaredAttribute("__init__") instanceof PyFunctionObject
)
}
@@ -32,108 +32,111 @@ class CheckClass extends ClassObject {
}
predicate sometimesDefines(string name) {
this.alwaysDefines(name) or
exists(SelfAttributeStore sa |
sa.getScope().getScope+() = this.getAnImproperSuperType().getPyClass() |
this.alwaysDefines(name)
or
exists(SelfAttributeStore sa |
sa.getScope().getScope+() = this.getAnImproperSuperType().getPyClass()
|
name = sa.getName()
)
}
private predicate selfDictAssigns() {
exists(Assign a, SelfAttributeRead self_dict, Subscript sub |
exists(Assign a, SelfAttributeRead self_dict, Subscript sub |
self_dict.getName() = "__dict__" and
(
self_dict = sub.getObject()
or
/* Indirect assignment via temporary variable */
exists(SsaVariable v |
v.getAUse() = sub.getObject().getAFlowNode() and
v.getDefinition().(DefinitionNode).getValue() = self_dict.getAFlowNode()
)
(
self_dict = sub.getObject()
or
/* Indirect assignment via temporary variable */
exists(SsaVariable v |
v.getAUse() = sub.getObject().getAFlowNode() and
v.getDefinition().(DefinitionNode).getValue() = self_dict.getAFlowNode()
)
) and
a.getATarget() = sub and
exists(FunctionObject meth | meth = this.lookupAttribute(_) and a.getScope() = meth.getFunction())
exists(FunctionObject meth |
meth = this.lookupAttribute(_) and a.getScope() = meth.getFunction()
)
)
}
pragma [nomagic]
pragma[nomagic]
private predicate monkeyPatched(string name) {
exists(Attribute a |
a.getCtx() instanceof Store and
PointsTo::points_to(a.getObject().getAFlowNode(), _, this, _, _) and a.getName() = name
a.getCtx() instanceof Store and
PointsTo::points_to(a.getObject().getAFlowNode(), _, this, _, _) and
a.getName() = name
)
}
private predicate selfSetattr() {
exists(Call c, Name setattr, Name self, Function method |
( method.getScope() = this.getPyClass() or
method.getScope() = this.getASuperType().getPyClass()
) and
c.getScope() = method and
c.getFunc() = setattr and
setattr.getId() = "setattr" and
c.getArg(0) = self and
self.getId() = "self"
)
exists(Call c, Name setattr, Name self, Function method |
(
method.getScope() = this.getPyClass() or
method.getScope() = this.getASuperType().getPyClass()
) and
c.getScope() = method and
c.getFunc() = setattr and
setattr.getId() = "setattr" and
c.getArg(0) = self and
self.getId() = "self"
)
}
predicate interestingUndefined(SelfAttributeRead a) {
exists(string name | name = a.getName() |
interestingContext(a, name) and
not this.definedInBlock(a.getAFlowNode().getBasicBlock(), name)
)
}
predicate interestingUndefined(SelfAttributeRead a) {
exists(string name | name = a.getName() |
interestingContext(a, name) and
not this.definedInBlock(a.getAFlowNode().getBasicBlock(), name)
)
}
private predicate interestingContext(SelfAttributeRead a, string name) {
name = a.getName() and
this.ofInterest() and
this.getPyClass() = a.getScope().getScope() and
not a.locallyDefined() and
not a.guardedByHasattr() and
a.getScope().isPublic() and
not this.monkeyPatched(name) and
not attribute_assigned_in_method(lookupAttribute("setUp"), name)
}
private predicate interestingContext(SelfAttributeRead a, string name) {
name = a.getName() and
this.ofInterest() and
this.getPyClass() = a.getScope().getScope() and
not a.locallyDefined() and
not a.guardedByHasattr() and
a.getScope().isPublic() and
not this.monkeyPatched(name) and
not attribute_assigned_in_method(lookupAttribute("setUp"), name)
}
private predicate probablyAbstract() {
this.getName().matches("Abstract%")
or
this.isAbstract()
}
private predicate probablyAbstract() {
this.getName().matches("Abstract%")
or
this.isAbstract()
}
private pragma[nomagic] predicate definitionInBlock(BasicBlock b, string name) {
exists(SelfAttributeStore sa |
sa.getAFlowNode().getBasicBlock() = b and sa.getName() = name and sa.getClass() = this.getPyClass()
)
or
exists(FunctionObject method | this.lookupAttribute(_) = method |
attribute_assigned_in_method(method, name) and
b = method.getACall().getBasicBlock()
)
}
private pragma[nomagic] predicate definedInBlock(BasicBlock b, string name) {
// manual specialisation: this is only called from interestingUndefined,
// so we can push the context in from there, which must apply to a
// SelfAttributeRead in the same scope
exists(SelfAttributeRead a |
a.getScope() = b.getScope() and name = a.getName() |
interestingContext(a, name)
)
and
this.definitionInBlock(b, name)
or
exists(BasicBlock prev | this.definedInBlock(prev, name) and prev.getASuccessor() = b)
}
pragma[nomagic]
private predicate definitionInBlock(BasicBlock b, string name) {
exists(SelfAttributeStore sa |
sa.getAFlowNode().getBasicBlock() = b and
sa.getName() = name and
sa.getClass() = this.getPyClass()
)
or
exists(FunctionObject method | this.lookupAttribute(_) = method |
attribute_assigned_in_method(method, name) and
b = method.getACall().getBasicBlock()
)
}
pragma[nomagic]
private predicate definedInBlock(BasicBlock b, string name) {
// manual specialisation: this is only called from interestingUndefined,
// so we can push the context in from there, which must apply to a
// SelfAttributeRead in the same scope
exists(SelfAttributeRead a | a.getScope() = b.getScope() and name = a.getName() |
interestingContext(a, name)
) and
this.definitionInBlock(b, name)
or
exists(BasicBlock prev | this.definedInBlock(prev, name) and prev.getASuccessor() = b)
}
}
private Object object_getattribute() {
result.asBuiltin() = theObjectType().asBuiltin().getMember("__getattribute__")
}
private predicate auto_name(string name) {
name = "__class__" or name = "__dict__"
}
private predicate auto_name(string name) { name = "__class__" or name = "__dict__" }

31
python/ql/src/Classes/Equality.qll
View File

@@ -1,6 +1,5 @@
import python
private Attribute dictAccess(LocalVariable var) {
result.getName() = "__dict__" and
result.getObject() = var.getAnAccess()
@@ -12,18 +11,21 @@ private Call getattr(LocalVariable obj, LocalVariable attr) {
result.getArg(1) = attr.getAnAccess()
}
/** A generic equality method that compares all attributes in its dict,
* or compares attributes using `getattr`. */
/**
* A generic equality method that compares all attributes in its dict,
* or compares attributes using `getattr`.
*/
class GenericEqMethod extends Function {
GenericEqMethod() {
this.getName() = "__eq__" and
exists(LocalVariable self, LocalVariable other |
self.getAnAccess() = this.getArg(0) and self.getId() = "self" and
self.getAnAccess() = this.getArg(0) and
self.getId() = "self" and
other.getAnAccess() = this.getArg(1) and
exists(Compare eq |
exists(Compare eq |
eq.getOp(0) instanceof Eq or
eq.getOp(0) instanceof NotEq |
eq.getOp(0) instanceof NotEq
|
// `self.__dict__ == other.__dict__`
eq.getAChildNode() = dictAccess(self) and
eq.getAChildNode() = dictAccess(other)
@@ -40,11 +42,11 @@ class GenericEqMethod extends Function {
/** An `__eq__` method that just does `self is other` */
class IdentityEqMethod extends Function {
IdentityEqMethod() {
this.getName() = "__eq__" and
exists(LocalVariable self, LocalVariable other |
self.getAnAccess() = this.getArg(0) and self.getId() = "self" and
self.getAnAccess() = this.getArg(0) and
self.getId() = "self" and
other.getAnAccess() = this.getArg(1) and
exists(Compare eq | eq.getOp(0) instanceof Is |
eq.getAChildNode() = self.getAnAccess() and
@@ -52,19 +54,20 @@ class IdentityEqMethod extends Function {
)
)
}
}
/** An (in)equality method that delegates to its complement */
class DelegatingEqualityMethod extends Function {
DelegatingEqualityMethod() {
exists(Return ret, UnaryExpr not_, Compare comp, Cmpop op, Parameter p0, Parameter p1 |
ret.getScope() = this and
ret.getValue() = not_ and
not_.getOp() instanceof Not and not_.getOperand() = comp and
comp.compares(p0.getVariable().getAnAccess(), op, p1.getVariable().getAnAccess()) |
this.getName() = "__eq__" and op instanceof NotEq or
not_.getOp() instanceof Not and
not_.getOperand() = comp and
comp.compares(p0.getVariable().getAnAccess(), op, p1.getVariable().getAnAccess())
|
this.getName() = "__eq__" and op instanceof NotEq
or
this.getName() = "__ne__" and op instanceof Eq
)
}

27
python/ql/src/Classes/MethodCallOrder.qll
View File

@@ -1,17 +1,20 @@
import python
// Helper predicates for multiple call to __init__/__del__ queries.
pragma [noinline]
private predicate multiple_invocation_paths_helper(FunctionInvocation top, FunctionInvocation i1, FunctionInvocation i2, FunctionObject multi) {
pragma[noinline]
private predicate multiple_invocation_paths_helper(
FunctionInvocation top, FunctionInvocation i1, FunctionInvocation i2, FunctionObject multi
) {
i1 != i2 and
i1 = top.getACallee+() and
i2 = top.getACallee+() and
i1.getFunction() = multi
}
pragma [noinline]
private predicate multiple_invocation_paths(FunctionInvocation top, FunctionInvocation i1, FunctionInvocation i2, FunctionObject multi) {
pragma[noinline]
private predicate multiple_invocation_paths(
FunctionInvocation top, FunctionInvocation i1, FunctionInvocation i2, FunctionObject multi
) {
multiple_invocation_paths_helper(top, i1, i2, multi) and
i2.getFunction() = multi
}
@@ -21,7 +24,8 @@ predicate multiple_calls_to_superclass_method(ClassObject self, FunctionObject m
exists(FunctionInvocation top, FunctionInvocation i1, FunctionInvocation i2 |
multiple_invocation_paths(top, i1, i2, multi) and
top.runtime(self.declaredAttribute(name)) and
self.getASuperType().declaredAttribute(name) = multi |
self.getASuperType().declaredAttribute(name) = multi
|
// Only called twice if called from different functions,
// or if one call-site can reach the other.
i1.getCall().getScope() != i2.getCall().getScope()
@@ -53,7 +57,9 @@ private predicate missing_call(FunctionObject meth, string name) {
}
/** Holds if `self.name` does not call `missing`, even though it is expected to. */
predicate missing_call_to_superclass_method(ClassObject self, FunctionObject top, FunctionObject missing, string name) {
predicate missing_call_to_superclass_method(
ClassObject self, FunctionObject top, FunctionObject missing, string name
) {
missing = self.getASuperType().declaredAttribute(name) and
top = self.lookupAttribute(name) and
/* There is no call to missing originating from top */
@@ -63,10 +69,7 @@ predicate missing_call_to_superclass_method(ClassObject self, FunctionObject top
sup = self.getAnImproperSuperType() and
named_attributes_not_method(sup, name)
) and
not self.isAbstract()
and
does_something(missing)
and
not self.isAbstract() and
does_something(missing) and
not missing_call(top, name)
}

1
python/ql/src/Exceptions/NotImplemented.qll
View File

@@ -1,4 +1,3 @@
import python
/** Holds if `notimpl` refers to `NotImplemented` or `NotImplemented()` in the `raise` statement */

10
python/ql/src/Exceptions/Raising.qll
View File

@@ -1,16 +1,12 @@
import python
/** Whether the raise statement 'r' raises 'type' from origin 'orig' */
/** Whether the raise statement 'r' raises 'type' from origin 'orig' */
predicate type_or_typeof(Raise r, ClassValue type, AstNode orig) {
exists(Expr exception |
exception = r.getRaised() |
exists(Expr exception | exception = r.getRaised() |
exception.pointsTo(type, orig)
or
not exists(ClassValue exc_type | exception.pointsTo(exc_type)) and
not type = ClassValue::type() and // First value is an unknown exception type
exists(Value val | exception.pointsTo(val, orig) |
val.getClass() = type
)
exists(Value val | exception.pointsTo(val, orig) | val.getClass() = type)
)
}

179
python/ql/src/Expressions/CallArgs.qll
View File

@@ -1,14 +1,10 @@
import python
import Testing.Mox
private int varargs_length_objectapi(Call call) {
not exists(call.getStarargs()) and result = 0
or
exists(TupleObject t |
call.getStarargs().refersTo(t) |
result = t.getLength()
)
exists(TupleObject t | call.getStarargs().refersTo(t) | result = t.getLength())
or
result = count(call.getStarargs().(List).getAnElt())
}
@@ -16,10 +12,7 @@ private int varargs_length_objectapi(Call call) {
private int varargs_length(Call call) {
not exists(call.getStarargs()) and result = 0
or
exists(TupleValue t |
call.getStarargs().pointsTo(t) |
result = t.length()
)
exists(TupleValue t | call.getStarargs().pointsTo(t) | result = t.length())
or
result = count(call.getStarargs().(List).getAnElt())
}
@@ -38,43 +31,43 @@ private Keyword not_keyword_only_arg(Call call, FunctionValue func) {
not func.getScope().getAKeywordOnlyArg().getId() = result.getArg()
}
/** Gets the count of arguments that are passed as positional parameters even if they
/**
* Gets the count of arguments that are passed as positional parameters even if they
* are named in the call.
* This is the sum of the number of positional arguments, the number of elements in any explicit tuple passed as *arg
* plus the number of keyword arguments that do not match keyword-only arguments (if the function does not take **kwargs).
*/
private int positional_arg_count_for_call_objectapi(Call call, Object callable) {
call = get_a_call_objectapi(callable).getNode() and
exists(int positional_keywords |
exists(FunctionObject func | func = get_function_or_initializer_objectapi(callable) |
not func.getFunction().hasKwArg() and
positional_keywords = count(not_keyword_only_arg_objectapi(call, func))
or
func.getFunction().hasKwArg() and positional_keywords = 0
)
|
result = count(call.getAnArg()) + varargs_length_objectapi(call) + positional_keywords
exists(FunctionObject func | func = get_function_or_initializer_objectapi(callable) |
not func.getFunction().hasKwArg() and
positional_keywords = count(not_keyword_only_arg_objectapi(call, func))
or
func.getFunction().hasKwArg() and positional_keywords = 0
)
|
result = count(call.getAnArg()) + varargs_length_objectapi(call) + positional_keywords
)
}
/** Gets the count of arguments that are passed as positional parameters even if they
/**
* Gets the count of arguments that are passed as positional parameters even if they
* are named in the call.
* This is the sum of the number of positional arguments, the number of elements in any explicit tuple passed as *arg
* plus the number of keyword arguments that do not match keyword-only arguments (if the function does not take **kwargs).
*/
private int positional_arg_count_for_call(Call call, Value callable) {
call = get_a_call(callable).getNode() and
exists(int positional_keywords |
exists(FunctionValue func | func = get_function_or_initializer(callable) |
not func.getScope().hasKwArg() and
positional_keywords = count(not_keyword_only_arg(call, func))
or
func.getScope().hasKwArg() and positional_keywords = 0
)
|
result = count(call.getAnArg()) + varargs_length_objectapi(call) + positional_keywords
exists(FunctionValue func | func = get_function_or_initializer(callable) |
not func.getScope().hasKwArg() and
positional_keywords = count(not_keyword_only_arg(call, func))
or
func.getScope().hasKwArg() and positional_keywords = 0
)
|
result = count(call.getAnArg()) + varargs_length_objectapi(call) + positional_keywords
)
}
@@ -88,33 +81,32 @@ int arg_count(Call call) {
/* Gets a call corresponding to the given class or function*/
private ControlFlowNode get_a_call_objectapi(Object callable) {
result = callable.(ClassObject).getACall()
or
result = callable.(FunctionObject).getACall()
result = callable.(ClassObject).getACall()
or
result = callable.(FunctionObject).getACall()
}
/* Gets a call corresponding to the given class or function*/
private ControlFlowNode get_a_call(Value callable) {
result = callable.(ClassValue).getACall()
or
result = callable.(FunctionValue).getACall()
result = callable.(ClassValue).getACall()
or
result = callable.(FunctionValue).getACall()
}
/* Gets the function object corresponding to the given class or function*/
FunctionObject get_function_or_initializer_objectapi(Object func_or_cls) {
result = func_or_cls.(FunctionObject)
or
result = func_or_cls.(ClassObject).declaredAttribute("__init__")
result = func_or_cls.(FunctionObject)
or
result = func_or_cls.(ClassObject).declaredAttribute("__init__")
}
/* Gets the function object corresponding to the given class or function*/
FunctionValue get_function_or_initializer(Value func_or_cls) {
result = func_or_cls.(FunctionValue)
or
result = func_or_cls.(ClassValue).declaredAttribute("__init__")
result = func_or_cls.(FunctionValue)
or
result = func_or_cls.(ClassValue).declaredAttribute("__init__")
}
/**Whether there is an illegally named parameter called `name` in the `call` to `func` */
predicate illegally_named_parameter_objectapi(Call call, Object func, string name) {
not func.isC() and
@@ -135,19 +127,22 @@ predicate illegally_named_parameter(Call call, Value func, string name) {
predicate too_few_args_objectapi(Call call, Object callable, int limit) {
// Exclude cases where an incorrect name is used as that is covered by 'Wrong name for an argument in a call'
not illegally_named_parameter_objectapi(call, callable, _) and
not exists(call.getStarargs()) and not exists(call.getKwargs()) and
not exists(call.getStarargs()) and
not exists(call.getKwargs()) and
arg_count_objectapi(call) < limit and
exists(FunctionObject func | func = get_function_or_initializer_objectapi(callable) |
call = func.getAFunctionCall().getNode() and limit = func.minParameters() and
// The combination of misuse of `mox.Mox().StubOutWithMock()`
// and a bug in mox's implementation of methods results in having to
// pass 1 too few arguments to the mocked function.
not (useOfMoxInModule(call.getEnclosingModule()) and func.isNormalMethod())
or
call = func.getAMethodCall().getNode() and limit = func.minParameters() - 1
or
callable instanceof ClassObject and
call.getAFlowNode() = get_a_call_objectapi(callable) and limit = func.minParameters() - 1
call = func.getAFunctionCall().getNode() and
limit = func.minParameters() and
// The combination of misuse of `mox.Mox().StubOutWithMock()`
// and a bug in mox's implementation of methods results in having to
// pass 1 too few arguments to the mocked function.
not (useOfMoxInModule(call.getEnclosingModule()) and func.isNormalMethod())
or
call = func.getAMethodCall().getNode() and limit = func.minParameters() - 1
or
callable instanceof ClassObject and
call.getAFlowNode() = get_a_call_objectapi(callable) and
limit = func.minParameters() - 1
)
}
@@ -155,19 +150,22 @@ predicate too_few_args_objectapi(Call call, Object callable, int limit) {
predicate too_few_args(Call call, Value callable, int limit) {
// Exclude cases where an incorrect name is used as that is covered by 'Wrong name for an argument in a call'
not illegally_named_parameter(call, callable, _) and
not exists(call.getStarargs()) and not exists(call.getKwargs()) and
not exists(call.getStarargs()) and
not exists(call.getKwargs()) and
arg_count(call) < limit and
exists(FunctionValue func | func = get_function_or_initializer(callable) |
call = func.getACall().getNode() and limit = func.minParameters() and
// The combination of misuse of `mox.Mox().StubOutWithMock()`
// and a bug in mox's implementation of methods results in having to
// pass 1 too few arguments to the mocked function.
not (useOfMoxInModule(call.getEnclosingModule()) and func.isNormalMethod())
or
call = func.getACall().getNode() and limit = func.minParameters() - 1
or
callable instanceof ClassValue and
call.getAFlowNode() = get_a_call(callable) and limit = func.minParameters() - 1
call = func.getACall().getNode() and
limit = func.minParameters() and
// The combination of misuse of `mox.Mox().StubOutWithMock()`
// and a bug in mox's implementation of methods results in having to
// pass 1 too few arguments to the mocked function.
not (useOfMoxInModule(call.getEnclosingModule()) and func.isNormalMethod())
or
call = func.getACall().getNode() and limit = func.minParameters() - 1
or
callable instanceof ClassValue and
call.getAFlowNode() = get_a_call(callable) and
limit = func.minParameters() - 1
)
}
@@ -175,16 +173,18 @@ predicate too_few_args(Call call, Value callable, int limit) {
predicate too_many_args_objectapi(Call call, Object callable, int limit) {
// Exclude cases where an incorrect name is used as that is covered by 'Wrong name for an argument in a call'
not illegally_named_parameter_objectapi(call, callable, _) and
exists(FunctionObject func |
func = get_function_or_initializer_objectapi(callable) and
not func.getFunction().hasVarArg() and limit >= 0
|
exists(FunctionObject func |
func = get_function_or_initializer_objectapi(callable) and
not func.getFunction().hasVarArg() and
limit >= 0
|
call = func.getAFunctionCall().getNode() and limit = func.maxParameters()
or
or
call = func.getAMethodCall().getNode() and limit = func.maxParameters() - 1
or
or
callable instanceof ClassObject and
call.getAFlowNode() = get_a_call_objectapi(callable) and limit = func.maxParameters() - 1
call.getAFlowNode() = get_a_call_objectapi(callable) and
limit = func.maxParameters() - 1
) and
positional_arg_count_for_call_objectapi(call, callable) > limit
}
@@ -193,16 +193,18 @@ predicate too_many_args_objectapi(Call call, Object callable, int limit) {
predicate too_many_args(Call call, Value callable, int limit) {
// Exclude cases where an incorrect name is used as that is covered by 'Wrong name for an argument in a call'
not illegally_named_parameter(call, callable, _) and
exists(FunctionValue func |
func = get_function_or_initializer(callable) and
not func.getScope().hasVarArg() and limit >= 0
|
exists(FunctionValue func |
func = get_function_or_initializer(callable) and
not func.getScope().hasVarArg() and
limit >= 0
|
call = func.getACall().getNode() and limit = func.maxParameters()
or
or
call = func.getACall().getNode() and limit = func.maxParameters() - 1
or
or
callable instanceof ClassValue and
call.getAFlowNode() = get_a_call(callable) and limit = func.maxParameters() - 1
call.getAFlowNode() = get_a_call(callable) and
limit = func.maxParameters() - 1
) and
positional_arg_count_for_call(call, callable) > limit
}
@@ -221,35 +223,34 @@ predicate wrong_args(Call call, FunctionValue func, int limit, string too) {
too_many_args(call, func, limit) and too = "too many"
}
/** Holds if `call` has correct number of arguments for `func`.
/**
* Holds if `call` has correct number of arguments for `func`.
* Implies nothing about whether `call` could call `func`.
*/
bindingset[call, func]
bindingset[call, func]
predicate correct_args_if_called_as_method_objectapi(Call call, FunctionObject func) {
arg_count_objectapi(call)+1 >= func.minParameters()
and
arg_count_objectapi(call) + 1 >= func.minParameters() and
arg_count_objectapi(call) < func.maxParameters()
}
/** Holds if `call` has correct number of arguments for `func`.
/**
* Holds if `call` has correct number of arguments for `func`.
* Implies nothing about whether `call` could call `func`.
*/
bindingset[call, func]
bindingset[call, func]
predicate correct_args_if_called_as_method(Call call, FunctionValue func) {
arg_count(call)+1 >= func.minParameters()
and
arg_count(call) + 1 >= func.minParameters() and
arg_count(call) < func.maxParameters()
}
/** Holds if `call` is a call to `overriding`, which overrides `func`. */
predicate overridden_call_objectapi(FunctionObject func, FunctionObject overriding, Call call) {
predicate overridden_call_objectapi(FunctionObject func, FunctionObject overriding, Call call) {
overriding.overrides(func) and
overriding.getACall().getNode() = call
}
/** Holds if `call` is a call to `overriding`, which overrides `func`. */
predicate overridden_call(FunctionValue func, FunctionValue overriding, Call call) {
predicate overridden_call(FunctionValue func, FunctionValue overriding, Call call) {
overriding.overrides(func) and
overriding.getACall().getNode() = call
}

92
python/ql/src/Expressions/Formatting/AdvancedFormatting.qll
View File

@@ -1,11 +1,7 @@
import python
library class PossibleAdvancedFormatString extends StrConst {
PossibleAdvancedFormatString() {
this.getText().matches("%{%}%")
}
PossibleAdvancedFormatString() { this.getText().matches("%{%}%") }
private predicate field(int start, int end) {
brace_pair(this, start, end) and
@@ -31,83 +27,77 @@ library class PossibleAdvancedFormatString extends StrConst {
(
result = this.getText().substring(start, end).regexpCapture("\\{([^!:.\\[]+)[!:.\\[].*", 1)
or
result = this.getText().substring(start+1, end-1) and result.regexpMatch("[^!:.\\[]+")
result = this.getText().substring(start + 1, end - 1) and result.regexpMatch("[^!:.\\[]+")
)
}
/** Gets the name of the formatting field at [start, end) */
string getFieldName(int start, int end) {
result = this.fieldId(start, end)
and not exists(this.getFieldNumber(start, end))
result = this.fieldId(start, end) and
not exists(this.getFieldNumber(start, end))
}
private predicate implicitlyNumberedField(int start, int end) {
this.field(start, end) and
exists(string c |
start+1 = this.getText().indexOf(c) |
exists(string c | start + 1 = this.getText().indexOf(c) |
c = "}" or c = ":" or c = "!" or c = "."
)
}
/** Whether this format string has implicitly numbered fields */
predicate isImplicitlyNumbered() {
this.implicitlyNumberedField(_, _)
}
predicate isImplicitlyNumbered() { this.implicitlyNumberedField(_, _) }
/** Whether this format string has explicitly numbered fields */
predicate isExplicitlyNumbered() {
exists(this.fieldId(_, _).toInt())
}
predicate isExplicitlyNumbered() { exists(this.fieldId(_, _).toInt()) }
}
predicate brace_sequence(PossibleAdvancedFormatString fmt, int index, int len) {
exists(string text |
text = fmt.getText() |
text.charAt(index) = "{" and not text.charAt(index-1) = "{" and len = 1
exists(string text | text = fmt.getText() |
text.charAt(index) = "{" and not text.charAt(index - 1) = "{" and len = 1
or
text.charAt(index) = "{" and text.charAt(index-1) = "{" and brace_sequence(fmt, index-1, len-1)
text.charAt(index) = "{" and
text.charAt(index - 1) = "{" and
brace_sequence(fmt, index - 1, len - 1)
)
}
predicate escaped_brace(PossibleAdvancedFormatString fmt, int index) {
exists(int len |
brace_sequence(fmt, index, len) |
len % 2 = 0
)
exists(int len | brace_sequence(fmt, index, len) | len % 2 = 0)
}
predicate escaping_brace(PossibleAdvancedFormatString fmt, int index) {
escaped_brace(fmt, index+1)
escaped_brace(fmt, index + 1)
}
private predicate inner_brace_pair(PossibleAdvancedFormatString fmt, int start, int end) {
not escaping_brace(fmt, start) and
not escaped_brace(fmt, start) and
fmt.getText().charAt(start) = "{" and
exists(string pair | pair = fmt.getText().suffix(start).regexpCapture("(?s)(\\{([^{}]|\\{\\{)*+\\}).*", 1) |
exists(string pair |
pair = fmt.getText().suffix(start).regexpCapture("(?s)(\\{([^{}]|\\{\\{)*+\\}).*", 1)
|
end = start + pair.length()
)
}
private predicate brace_pair(PossibleAdvancedFormatString fmt, int start, int end) {
inner_brace_pair(fmt, start, end)
or
not escaping_brace(fmt, start) and
not escaped_brace(fmt, start) and
exists(string prefix, string postfix, int innerstart, int innerend |
brace_pair(fmt, innerstart, innerend) and
prefix = fmt.getText().regexpFind("\\{([^{}]|\\{\\{)+\\{", _, start) and
innerstart = start+prefix.length()-1 and
postfix = fmt.getText().regexpFind("\\}([^{}]|\\}\\})*\\}", _, innerend-1) and
end = innerend + postfix.length()-1
)
inner_brace_pair(fmt, start, end)
or
not escaping_brace(fmt, start) and
not escaped_brace(fmt, start) and
exists(string prefix, string postfix, int innerstart, int innerend |
brace_pair(fmt, innerstart, innerend) and
prefix = fmt.getText().regexpFind("\\{([^{}]|\\{\\{)+\\{", _, start) and
innerstart = start + prefix.length() - 1 and
postfix = fmt.getText().regexpFind("\\}([^{}]|\\}\\})*\\}", _, innerend - 1) and
end = innerend + postfix.length() - 1
)
}
private predicate advanced_format_call(Call format_expr, PossibleAdvancedFormatString fmt, int args) {
exists(CallNode call |
call = format_expr.getAFlowNode() |
call.getFunction().pointsTo(Value::named("format")) and call.getArg(0).pointsTo(_, fmt.getAFlowNode()) and
exists(CallNode call | call = format_expr.getAFlowNode() |
call.getFunction().pointsTo(Value::named("format")) and
call.getArg(0).pointsTo(_, fmt.getAFlowNode()) and
args = count(format_expr.getAnArg()) - 1
or
call.getFunction().(AttrNode).getObject("format").pointsTo(_, fmt.getAFlowNode()) and
@@ -116,26 +106,14 @@ private predicate advanced_format_call(Call format_expr, PossibleAdvancedFormatS
}
class AdvancedFormatString extends PossibleAdvancedFormatString {
AdvancedFormatString() {
advanced_format_call(_, this, _)
}
AdvancedFormatString() { advanced_format_call(_, this, _) }
}
class AdvancedFormattingCall extends Call {
AdvancedFormattingCall() {
advanced_format_call(this, _, _)
}
AdvancedFormattingCall() { advanced_format_call(this, _, _) }
/** Count of the arguments actually provided */
int providedArgCount() {
advanced_format_call(this, _, result)
}
AdvancedFormatString getAFormat() {
advanced_format_call(this, result, _)
}
int providedArgCount() { advanced_format_call(this, _, result) }
AdvancedFormatString getAFormat() { advanced_format_call(this, result, _) }
}

79
python/ql/src/Expressions/IsComparisons.qll
View File

@@ -1,9 +1,9 @@
import python
predicate comparison_using_is(Compare comp, ControlFlowNode left, Cmpop op, ControlFlowNode right) {
exists(CompareNode fcomp | fcomp = comp.getAFlowNode() |
fcomp.operands(left, op, right) and (op instanceof Is or op instanceof IsNot)
fcomp.operands(left, op, right) and
(op instanceof Is or op instanceof IsNot)
)
}
@@ -12,8 +12,7 @@ predicate overrides_eq_or_cmp(ClassValue c) {
or
c.declaresAttribute("__eq__") and not c = Value::named("object")
or
exists(ClassValue sup |
sup = c.getASuperType() and not sup = Value::named("object") |
exists(ClassValue sup | sup = c.getASuperType() and not sup = Value::named("object") |
sup.declaresAttribute("__eq__")
)
or
@@ -29,58 +28,56 @@ predicate probablySingleton(ClassValue cls) {
predicate invalid_to_use_is_portably(ClassValue c) {
overrides_eq_or_cmp(c) and
// Exclude type/builtin-function/bool as it is legitimate to compare them using 'is' but they implement __eq__
not c = Value::named("type") and not c = ClassValue::builtinFunction() and not c = Value::named("bool") and
not c = Value::named("type") and
not c = ClassValue::builtinFunction() and
not c = Value::named("bool") and
// OK to compare with 'is' if a singleton
not probablySingleton(c)
}
predicate simple_constant(ControlFlowNode f) {
exists(Value val | f.pointsTo(val) | val = Value::named("True") or val = Value::named("False") or val = Value::named("None"))
exists(Value val | f.pointsTo(val) |
val = Value::named("True") or val = Value::named("False") or val = Value::named("None")
)
}
private predicate cpython_interned_value(Expr e) {
exists(string text | text = e.(StrConst).getText() |
text.length() = 0 or
text.length() = 0
or
text.length() = 1 and text.regexpMatch("[U+0000-U+00ff]")
)
or
exists(int i |
i = e.(IntegerLiteral).getN().toInt() |
-5 <= i and i <= 256
)
or
exists(int i | i = e.(IntegerLiteral).getN().toInt() | -5 <= i and i <= 256)
or
exists(Tuple t | t = e and not exists(t.getAnElt()))
}
/** The set of values that can be expected to be interned across
/**
* The set of values that can be expected to be interned across
* the main implementations of Python. PyPy, Jython, etc tend to
* follow CPython, but it varies, so this is a best guess.
*/
private predicate universally_interned_value(Expr e) {
e.(IntegerLiteral).getN().toInt() = 0
or
exists(Tuple t | t = e and not exists(t.getAnElt()))
or
e.(StrConst).getText() = ""
e.(IntegerLiteral).getN().toInt() = 0
or
exists(Tuple t | t = e and not exists(t.getAnElt()))
or
e.(StrConst).getText() = ""
}
predicate cpython_interned_constant(Expr e) {
exists(Expr const |
e.pointsTo(_, const) |
cpython_interned_value(const)
)
exists(Expr const | e.pointsTo(_, const) | cpython_interned_value(const))
}
predicate universally_interned_constant(Expr e) {
exists(Expr const |
e.pointsTo(_, const) |
universally_interned_value(const)
)
exists(Expr const | e.pointsTo(_, const) | universally_interned_value(const))
}
private predicate comparison_both_types(Compare comp, Cmpop op, ClassValue cls1, ClassValue cls2) {
exists(ControlFlowNode op1, ControlFlowNode op2 |
comparison_using_is(comp, op1, op, op2) or comparison_using_is(comp, op2, op, op1) |
comparison_using_is(comp, op1, op, op2) or comparison_using_is(comp, op2, op, op1)
|
op1.inferredValue().getClass() = cls1 and
op2.inferredValue().getClass() = cls2
)
@@ -89,15 +86,17 @@ private predicate comparison_both_types(Compare comp, Cmpop op, ClassValue cls1,
private predicate comparison_one_type(Compare comp, Cmpop op, ClassValue cls) {
not comparison_both_types(comp, _, _, _) and
exists(ControlFlowNode operand |
comparison_using_is(comp, operand, op, _) or comparison_using_is(comp, _, op, operand) |
operand.inferredValue().getClass() = cls
comparison_using_is(comp, operand, op, _) or comparison_using_is(comp, _, op, operand)
|
operand.inferredValue().getClass() = cls
)
}
predicate invalid_portable_is_comparison(Compare comp, Cmpop op, ClassValue cls) {
// OK to use 'is' when defining '__eq__'
not exists(Function eq | eq.getName() = "__eq__" or eq.getName() = "__ne__" | eq = comp.getScope().getScope*())
and
not exists(Function eq | eq.getName() = "__eq__" or eq.getName() = "__ne__" |
eq = comp.getScope().getScope*()
) and
(
comparison_one_type(comp, op, cls) and invalid_to_use_is_portably(cls)
or
@@ -105,32 +104,32 @@ predicate invalid_portable_is_comparison(Compare comp, Cmpop op, ClassValue cls)
invalid_to_use_is_portably(cls) and
invalid_to_use_is_portably(other)
)
)
and
) and
// OK to use 'is' when comparing items from a known set of objects
not exists(Expr left, Expr right, Value val |
comp.compares(left, op, right) and
exists(ImmutableLiteral il | il.getLiteralValue() = val) |
exists(ImmutableLiteral il | il.getLiteralValue() = val)
|
left.pointsTo(val) and right.pointsTo(val)
or
// Simple constant in module, probably some sort of sentinel
exists(AstNode origin |
not left.pointsTo(_) and right.pointsTo(val, origin) and
not left.pointsTo(_) and
right.pointsTo(val, origin) and
origin.getScope().getEnclosingModule() = comp.getScope().getEnclosingModule()
)
)
and
) and
// OK to use 'is' when comparing with a member of an enum
not exists(Expr left, Expr right, AstNode origin |
comp.compares(left, op, right) and
enum_member(origin) |
enum_member(origin)
|
left.pointsTo(_, origin) or right.pointsTo(_, origin)
)
}
private predicate enum_member(AstNode obj) {
exists(ClassValue cls, AssignStmt asgn |
cls.getASuperType().getName() = "Enum" |
exists(ClassValue cls, AssignStmt asgn | cls.getASuperType().getName() = "Enum" |
cls.getScope() = asgn.getScope() and
asgn.getValue() = obj
)

18
python/ql/src/Expressions/RedundantComparison.qll
View File

@@ -1,27 +1,22 @@
import python
class RedundantComparison extends Compare {
RedundantComparison() {
exists(Expr left, Expr right |
this.compares(left, _, right)
and
this.compares(left, _, right) and
same_variable(left, right)
)
}
predicate maybeMissingSelf() {
exists(Name left |
this.compares(left, _, _) and
not this.isConstant() and
exists(Class cls | left.getScope().getScope() = cls |
exists(SelfAttribute sa | sa.getName() = left.getId() |
sa.getClass() = cls
)
exists(SelfAttribute sa | sa.getName() = left.getId() | sa.getClass() = cls)
)
)
}
}
private predicate same_variable(Expr left, Expr right) {
@@ -36,11 +31,14 @@ private predicate name_in_comparison(Compare comp, Name n, Variable v) {
private predicate same_name(Name n1, Name n2) {
n1 != n2 and
exists(Compare comp, Variable v | name_in_comparison(comp, n1, v) and name_in_comparison(comp, n2, v))
exists(Compare comp, Variable v |
name_in_comparison(comp, n1, v) and name_in_comparison(comp, n2, v)
)
}
private predicate same_attribute(Attribute a1, Attribute a2) {
a1 != a2 and
exists(Compare comp | comp.contains(a1) and comp.contains(a2)) and
a1.getName() = a2.getName() and same_name(a1.getObject(), a2.getObject())
a1.getName() = a2.getName() and
same_name(a1.getObject(), a2.getObject())
}

188
python/ql/src/Lexical/CommentedOutCode.qll
View File

@@ -1,6 +1,5 @@
import python
private predicate def_statement(Comment c) {
c.getText().regexpMatch("#(\\S*\\s+)?def\\s.*\\(.*\\).*:\\s*(#.*)?")
}
@@ -12,7 +11,7 @@ private predicate if_statement(Comment c) {
}
private predicate for_statement(Comment c) {
c.getText().regexpMatch("#(\\S*\\s+)?for\\s.*\\sin\\s.*:\\s*(#.*)?")
c.getText().regexpMatch("#(\\S*\\s+)?for\\s.*\\sin\\s.*:\\s*(#.*)?")
}
private predicate with_statement(Comment c) {
@@ -20,11 +19,11 @@ private predicate with_statement(Comment c) {
}
private predicate try_statement(Comment c) {
c.getText().regexpMatch("#(\\S*\\s+)?try:\\s*(#.*)?")
c.getText().regexpMatch("#(\\S*\\s+)?try:\\s*(#.*)?")
or
c.getText().regexpMatch("#(\\S*\\s+)?except\\s*(\\w+\\s*(\\sas\\s+\\w+\\s*)?)?:\\s*(#.*)?")
or
c.getText().regexpMatch("#(\\S*\\s+)?finally:\\s*(#.*)?")
c.getText().regexpMatch("#(\\S*\\s+)?finally:\\s*(#.*)?")
}
private int indentation(Comment c) {
@@ -39,18 +38,16 @@ private predicate class_statement(Comment c) {
c.getText().regexpMatch("#(\\S*\\s+)?class\\s+\\w+.*:\\s*(#.*)?")
}
private predicate triple_quote(Comment c) {
c.getText().regexpMatch("#.*(\"\"\"|''').*")
}
private predicate triple_quote(Comment c) { c.getText().regexpMatch("#.*(\"\"\"|''').*") }
private predicate triple_quoted_string_part(Comment start, Comment end) {
triple_quote(start) and end = start
or
exists(Comment mid |
triple_quote(start) and end = start
or
exists(Comment mid |
triple_quoted_string_part(start, mid) and
end = non_empty_following(mid) and
not triple_quote(end)
)
)
}
private predicate maybe_code(Comment c) {
@@ -59,41 +56,37 @@ private predicate maybe_code(Comment c) {
commented_out_comment(c)
}
private predicate commented_out_comment(Comment c) {
c.getText().regexpMatch("#+\\s+#.*")
}
private predicate commented_out_comment(Comment c) { c.getText().regexpMatch("#+\\s+#.*") }
private int scope_start(Comment start) {
(
def_statement(start) or
class_statement(start)
)
and
result = indentation(start)
and
def_statement(start) or
class_statement(start)
) and
result = indentation(start) and
not non_code(start)
}
private int block_start(Comment start) {
(
if_statement(start) or
for_statement(start) or
try_statement(start) or
with_statement(start)
)
and
result = indentation(start)
and
if_statement(start) or
for_statement(start) or
try_statement(start) or
with_statement(start)
) and
result = indentation(start) and
not non_code(start)
}
private int scope_doc_string_part(Comment start, Comment end) {
result = scope_start(start) and
triple_quote(end) and end = non_empty_following(start)
triple_quote(end) and
end = non_empty_following(start)
or
exists(Comment mid |
exists(Comment mid |
result = scope_doc_string_part(start, mid) and
end = non_empty_following(mid) |
end = non_empty_following(mid)
|
not triple_quote(end)
)
}
@@ -101,15 +94,16 @@ private int scope_doc_string_part(Comment start, Comment end) {
private int scope_part(Comment start, Comment end) {
result = scope_start(start) and end = start
or
exists(Comment mid |
exists(Comment mid |
result = scope_doc_string_part(start, mid) and
end = non_empty_following(mid) and
triple_quote(end)
)
or
exists(Comment mid |
exists(Comment mid |
result = scope_part(start, mid) and
end = non_empty_following(mid) |
end = non_empty_following(mid)
|
indentation(end) > result
)
}
@@ -119,9 +113,10 @@ private int block_part(Comment start, Comment end) {
end = non_empty_following(start) and
indentation(end) > result
or
exists(Comment mid |
exists(Comment mid |
result = block_part(start, mid) and
end = non_empty_following(mid) |
end = non_empty_following(mid)
|
indentation(end) > result
or
result = block_start(end)
@@ -145,13 +140,11 @@ private predicate commented_out_code(Comment c) {
}
private predicate commented_out_code_part(Comment start, Comment end) {
commented_out_code(start) and end = start and
not exists(Comment prev |
non_empty_following(prev) = start |
commented_out_code(prev)
)
commented_out_code(start) and
end = start and
not exists(Comment prev | non_empty_following(prev) = start | commented_out_code(prev))
or
exists(Comment mid |
exists(Comment mid |
commented_out_code_part(start, mid) and
non_empty_following(mid) = end and
commented_out_code(end)
@@ -167,12 +160,7 @@ private predicate commented_out_code_block(Comment start, Comment end) {
/* A single line comment that appears to be commented out code */
class CommentedOutCodeLine extends Comment {
CommentedOutCodeLine () {
exists(CommentedOutCodeBlock b |
b.contains(this)
)
}
CommentedOutCodeLine() { exists(CommentedOutCodeBlock b | b.contains(this)) }
/* Whether this commented-out code line is likely to be example code embedded in a larger comment. */
predicate maybeExampleCode() {
@@ -181,19 +169,13 @@ class CommentedOutCodeLine extends Comment {
block.maybeExampleCode()
)
}
}
/** A block of comments that appears to be commented out code */
class CommentedOutCodeBlock extends @py_comment {
CommentedOutCodeBlock() {
commented_out_code_block(this, _)
}
CommentedOutCodeBlock() { commented_out_code_block(this, _) }
string toString() {
result = "Commented out code"
}
string toString() { result = "Commented out code" }
/** Whether this commented-out code block contains the comment c */
predicate contains(Comment c) {
@@ -207,28 +189,22 @@ class CommentedOutCodeBlock extends @py_comment {
}
/** The length of this comment block (in comments) */
int length() {
result = count(Comment c | this.contains(c))
}
int length() { result = count(Comment c | this.contains(c)) }
predicate hasLocationInfo(string filepath, int bl, int bc, int el, int ec) {
((Comment)this).getLocation().hasLocationInfo(filepath, bl, bc, _, _)
and
exists(Comment end |
commented_out_code_block(this, end) |
this.(Comment).getLocation().hasLocationInfo(filepath, bl, bc, _, _) and
exists(Comment end | commented_out_code_block(this, end) |
end.getLocation().hasLocationInfo(_, _, _, el, ec)
)
}
/** Whether this commented-out code block is likely to be example code embedded in a larger comment. */
predicate maybeExampleCode() {
exists(CommentBlock block |
block.contains((Comment)this) |
exists(CommentBlock block | block.contains(this.(Comment)) |
exists(int all_code |
all_code = sum (CommentedOutCodeBlock code | block.contains((Comment)code) | code.length())
and
all_code = sum(CommentedOutCodeBlock code | block.contains(code.(Comment)) | code.length()) and
/* This ratio may need fine tuning */
block.length() > all_code*2
block.length() > all_code * 2
)
)
}
@@ -239,13 +215,14 @@ private predicate word_pair(Comment c, string s1, string s2) {
exists(int i1, int i2, int o1, int o2 |
s1 = c.getText().regexpFind("\\w+", i1, o1) and
s2 = c.getText().regexpFind("\\w+", i2, o2) and
i2 = i1 + 1 and
i2 = i1 + 1 and
c.getText().prefix(o1).regexpMatch("[^'\"]*") and
c.getText().substring(o1 + s1.length(), o2).regexpMatch("\\s+")
)
}
/** The comment c cannot be code if it contains a word pair "word1 word2" and
/**
* The comment c cannot be code if it contains a word pair "word1 word2" and
* either:
* 1. word1 is not a keyword and word2 is not an operator:
* "x is" could be code, "return y" could be code, but "isnt code" cannot be code.
@@ -257,48 +234,44 @@ private predicate non_code(Comment c) {
exists(string word1, string word2 |
word_pair(c, word1, word2) and
not word2 = operator_keyword()
|
|
not word1 = a_keyword()
or
word1 = keyword_requiring_colon() and not c.getText().matches("%:%")
) and
/* Except comments of the form: # (maybe code) # some comment */
not c.getText().regexpMatch("#\\S+\\s.*#.*")
not c.getText().regexpMatch("#\\S+\\s.*#.*")
or
/* Don't count doctests as code */
c.getText().matches("%>>>%") or c.getText().matches("%...%")
c.getText().matches("%>>>%")
or
c.getText().matches("%...%")
}
private predicate filler(Comment c) {
c.getText().regexpMatch("#+[\\s*#-_=+]*")
}
private predicate filler(Comment c) { c.getText().regexpMatch("#+[\\s*#-_=+]*") }
/** Gets the first non empty comment following c */
/** Gets the first non empty comment following c */
private Comment non_empty_following(Comment c) {
not empty(result) and
(
not empty(result) and
(
result = empty_following(c).getFollowing()
or
not empty(c) and result = c.getFollowing()
)
not empty(c) and result = c.getFollowing()
)
}
/* Helper for non_empty_following() */
private Comment empty_following(Comment c) {
not empty(c) and
empty(result)
and
exists(Comment prev |
result = prev.getFollowing() |
empty(result) and
exists(Comment prev | result = prev.getFollowing() |
prev = c
or
prev = empty_following(c)
)
}
private predicate empty(Comment c) {
c.getText().regexpMatch("#+\\s*")
}
private predicate empty(Comment c) { c.getText().regexpMatch("#+\\s*") }
/* A comment following code on the same line */
private predicate endline_comment(Comment c) {
@@ -315,21 +288,40 @@ private predicate file_or_url(Comment c) {
}
private string operator_keyword() {
result = "import" or result = "and" or result = "is" or result = "or" or result = "in" or result = "not" or result = "as"
result = "import" or
result = "and" or
result = "is" or
result = "or" or
result = "in" or
result = "not" or
result = "as"
}
private string keyword_requiring_colon() {
result = "try" or result = "while" or result = "elif" or result = "else" or result = "if" or
result = "except" or result = "def" or result = "class"
result = "try" or
result = "while" or
result = "elif" or
result = "else" or
result = "if" or
result = "except" or
result = "def" or
result = "class"
}
private string other_keyword() {
result = "del" or result = "lambda" or result = "from" or
result = "global" or result = "with" or result = "assert" or
result = "yield" or result = "finally" or
result = "print" or
result = "exec" or result = "raise" or
result = "return" or result = "for"
result = "del" or
result = "lambda" or
result = "from" or
result = "global" or
result = "with" or
result = "assert" or
result = "yield" or
result = "finally" or
result = "print" or
result = "exec" or
result = "raise" or
result = "return" or
result = "for"
}
private string a_keyword() {

18
python/ql/src/Metrics/Internal/Extents.qll
View File

@@ -15,10 +15,10 @@ import python
* including the body (if any), as opposed to the location of its name only.
*/
class RangeFunction extends Function {
predicate hasLocationInfo(string path, int sl, int sc, int el, int ec) {
super.getLocation().hasLocationInfo(path, sl, sc, _, _)
and this.getBody().getLastItem().getLocation().hasLocationInfo(path, _, _, el, ec)
}
predicate hasLocationInfo(string path, int sl, int sc, int el, int ec) {
super.getLocation().hasLocationInfo(path, sl, sc, _, _) and
this.getBody().getLastItem().getLocation().hasLocationInfo(path, _, _, el, ec)
}
}
/**
@@ -26,8 +26,8 @@ class RangeFunction extends Function {
* including the body (if any), as opposed to the location of its name only.
*/
class RangeClass extends Class {
predicate hasLocationInfo(string path, int sl, int sc, int el, int ec) {
super.getLocation().hasLocationInfo(path, sl, sc, _, _)
and this.getBody().getLastItem().getLocation().hasLocationInfo(path, _, _, el, ec)
}
}
predicate hasLocationInfo(string path, int sl, int sc, int el, int ec) {
super.getLocation().hasLocationInfo(path, sl, sc, _, _) and
this.getBody().getLastItem().getLocation().hasLocationInfo(path, _, _, el, ec)
}
}

7
python/ql/src/Testing/Mox.qll
View File

@@ -2,15 +2,14 @@ import python
/** Whether `mox` or `.StubOutWithMock()` is used in thin module `m`. */
predicate useOfMoxInModule(Module m) {
exists(ModuleObject mox |
mox.getName() = "mox" or mox.getName() = "mox3.mox" |
exists(ControlFlowNode use |
exists(ModuleObject mox | mox.getName() = "mox" or mox.getName() = "mox3.mox" |
exists(ControlFlowNode use |
use.refersTo(mox) and
use.getScope().getEnclosingModule() = m
)
)
or
exists(Call call|
exists(Call call |
call.getFunc().(Attribute).getName() = "StubOutWithMock" and
call.getEnclosingModule() = m
)

55
python/ql/src/analysis/CrossProjectDefinitions.qll
View File

@@ -1,46 +1,42 @@
/**
* Symbols for crosss-project jump-to-definition resolution.
*/
import python
import python
import semmle.python.pointsto.PointsTo
private newtype TSymbol =
TModule(Module m)
or
TModule(Module m) or
TMember(Symbol outer, string part) {
exists(Object o |
outer.resolvesTo() = o |
exists(Object o | outer.resolvesTo() = o |
o.(ModuleObject).hasAttribute(part)
or
o.(ClassObject).hasAttribute(part)
)
}
/** A "symbol" referencing an object in another module
* Symbols are represented by the module name and the dotted name by which the
/**
* A "symbol" referencing an object in another module
* Symbols are represented by the module name and the dotted name by which the
* object would be referred to in that module.
* For example for the code:
* ```
* class C:
* def m(self): pass
* ```
* If the code were in a module `mod`,
* If the code were in a module `mod`,
* then symbol for the method `m` would be "mod/C.m"
*/
class Symbol extends TSymbol {
string toString() {
exists(Module m |
this = TModule(m) and result = m.getName()
)
or
exists(Module m | this = TModule(m) and result = m.getName())
or
exists(TModule outer, string part |
this = TMember(outer, part) and
outer = TModule(_) and
result = outer.(Symbol).toString() + "/" + part
)
or
or
exists(TMember outer, string part |
this = TMember(outer, part) and
outer = TMember(_, _) and
@@ -52,8 +48,7 @@ class Symbol extends TSymbol {
AstNode find() {
this = TModule(result)
or
exists(Symbol s, string name |
this = TMember(s, name) |
exists(Symbol s, string name | this = TMember(s, name) |
exists(ClassObject cls |
s.resolvesTo() = cls and
cls.attributeRefersTo(name, _, result.getAFlowNode())
@@ -66,47 +61,43 @@ class Symbol extends TSymbol {
)
}
/** Find the class or module `Object` that this `Symbol` refers to, if
/**
* Find the class or module `Object` that this `Symbol` refers to, if
* this `Symbol` refers to a class or module.
*/
Object resolvesTo() {
this = TModule(result.(ModuleObject).getModule())
or
exists(Symbol s, string name, Object o |
this = TMember(s, name) and
this = TMember(s, name) and
o = s.resolvesTo() and
result = attribute_in_scope(o, name)
)
}
/** Gets the `Module` for the module part of this `Symbol`.
/**
* Gets the `Module` for the module part of this `Symbol`.
* For example, this would return the `os` module for the `Symbol` "os/environ".
*/
Module getModule() {
this = TModule(result)
or
exists(Symbol outer |
this = TMember(outer, _) and result = outer.getModule()
)
exists(Symbol outer | this = TMember(outer, _) and result = outer.getModule())
}
/** Gets the `Symbol` that is the named member of this `Symbol`. */
Symbol getMember(string name) {
result = TMember(this, name)
}
Symbol getMember(string name) { result = TMember(this, name) }
}
/* Helper for `Symbol`.resolvesTo() */
private Object attribute_in_scope(Object obj, string name) {
exists(ClassObject cls |
cls = obj |
exists(ClassObject cls | cls = obj |
cls.lookupAttribute(name) = result and result.(ControlFlowNode).getScope() = cls.getPyClass()
)
or
exists(ModuleObject mod |
mod = obj |
mod.attr(name) = result and result.(ControlFlowNode).getScope() = mod.getModule()
and not result.(ControlFlowNode).isEntryNode()
exists(ModuleObject mod | mod = obj |
mod.attr(name) = result and
result.(ControlFlowNode).getScope() = mod.getModule() and
not result.(ControlFlowNode).isEntryNode()
)
}

176
python/ql/src/analysis/DefinitionTracking.qll
View File

@@ -1,35 +1,22 @@
/**
* Definition tracking for jump-to-defn query.
*/
import python
import python
import semmle.python.pointsto.PointsTo
private newtype TDefinition =
TLocalDefinition(AstNode a) {
a instanceof Expr or a instanceof Stmt or a instanceof Module
}
TLocalDefinition(AstNode a) { a instanceof Expr or a instanceof Stmt or a instanceof Module }
/** A definition for the purposes of jump-to-definition. */
class Definition extends TLocalDefinition {
string toString() { result = "Definition " + this.getAstNode().getLocation().toString() }
AstNode getAstNode() { this = TLocalDefinition(result) }
string toString() {
result = "Definition " + this.getAstNode().getLocation().toString()
}
AstNode getAstNode() {
this = TLocalDefinition(result)
}
Module getModule() {
result = this.getAstNode().getScope().getEnclosingModule()
}
Location getLocation() {
result = this.getAstNode().getLocation()
}
Module getModule() { result = this.getAstNode().getScope().getEnclosingModule() }
Location getLocation() { result = this.getAstNode().getLocation() }
}
private predicate jump_to_defn(ControlFlowNode use, Definition defn) {
@@ -112,7 +99,7 @@ predicate uni_edged_phi_defn(SingleSuccessorGuard uniphi, Definition defn) {
ssa_variable_defn(uniphi.getInput(), defn)
}
pragma [noinline]
pragma[noinline]
private predicate ssa_node_defn(EssaNodeDefinition def, Definition defn) {
assignment_jump_to_defn(def, defn)
or
@@ -130,7 +117,7 @@ private predicate assignment_jump_to_defn(AssignmentDefinition def, Definition d
defn = TLocalDefinition(def.getValue().getNode())
}
pragma [noinline]
pragma[noinline]
private predicate ssa_node_refinement_defn(EssaNodeRefinement def, Definition defn) {
method_callsite_defn(def, defn)
or
@@ -147,16 +134,13 @@ private predicate ssa_node_refinement_defn(EssaNodeRefinement def, Definition de
uni_edged_phi_defn(def, defn)
}
/* Definition for parameter. `def foo(param): ...` */
private predicate parameter_defn(ParameterDefinition def, Definition defn) {
defn.getAstNode() = def.getDefiningNode().getNode()
}
/* Definition for deletion: `del name` */
private predicate delete_defn(DeletionDefinition def, Definition defn) {
none()
}
private predicate delete_defn(DeletionDefinition def, Definition defn) { none() }
/* Implicit "defn" of the names of submodules at the start of an `__init__.py` file. */
private predicate implicit_submodule_defn(ImplicitSubModuleDefinition def, Definition defn) {
@@ -165,13 +149,16 @@ private predicate implicit_submodule_defn(ImplicitSubModuleDefinition def, Defin
mod = package.submodule(def.getSourceVariable().getName()) and
defn.getAstNode() = mod.getModule()
)
}
/* Helper for scope_entry_value_transfer(...).
* Transfer of values from the callsite to the callee, for enclosing variables, but not arguments/parameters
/*
* Helper for scope_entry_value_transfer(...).
* Transfer of values from the callsite to the callee, for enclosing variables, but not arguments/parameters
*/
private predicate scope_entry_value_transfer_at_callsite(EssaVariable pred_var, ScopeEntryDefinition succ_def) {
private predicate scope_entry_value_transfer_at_callsite(
EssaVariable pred_var, ScopeEntryDefinition succ_def
) {
exists(CallNode callsite, FunctionObject f |
f.getACall() = callsite and
pred_var.getSourceVariable() = succ_def.getSourceVariable() and
@@ -181,8 +168,7 @@ private predicate scope_entry_value_transfer_at_callsite(EssaVariable pred_var,
}
/* Model the transfer of values at scope-entry points. Transfer from `pred_var, pred_context` to `succ_def, succ_context` */
private
predicate scope_entry_value_transfer(EssaVariable pred_var, ScopeEntryDefinition succ_def) {
private predicate scope_entry_value_transfer(EssaVariable pred_var, ScopeEntryDefinition succ_def) {
BaseFlow::scope_entry_value_transfer_from_earlier(pred_var, _, succ_def, _)
or
scope_entry_value_transfer_at_callsite(pred_var, succ_def)
@@ -191,8 +177,7 @@ predicate scope_entry_value_transfer(EssaVariable pred_var, ScopeEntryDefinition
}
/* Helper for scope_entry_value_transfer */
private
predicate class_entry_value_transfer(EssaVariable pred_var, ScopeEntryDefinition succ_def) {
private predicate class_entry_value_transfer(EssaVariable pred_var, ScopeEntryDefinition succ_def) {
exists(ImportTimeScope scope, ControlFlowNode class_def |
class_def = pred_var.getAUse() and
scope.entryEdge(class_def, succ_def.getDefiningNode()) and
@@ -201,7 +186,7 @@ predicate class_entry_value_transfer(EssaVariable pred_var, ScopeEntryDefinition
}
/* Definition for implicit variable declarations at scope-entry. */
pragma [noinline]
pragma[noinline]
private predicate scope_entry_defn(ScopeEntryDefinition def, Definition defn) {
/* Transfer from another scope */
exists(EssaVariable var |
@@ -210,10 +195,12 @@ private predicate scope_entry_defn(ScopeEntryDefinition def, Definition defn) {
)
}
/* Definition for a variable (possibly) redefined by a call:
/*
* Definition for a variable (possibly) redefined by a call:
* Just assume that call does not define variable
*/
pragma [noinline]
pragma[noinline]
private predicate callsite_defn(CallsiteRefinement def, Definition defn) {
ssa_variable_defn(def.getInput(), defn)
}
@@ -225,22 +212,27 @@ private predicate method_callsite_defn(MethodCallsiteRefinement def, Definition
}
/** Helpers for import_star_defn */
pragma [noinline]
private predicate module_and_name_for_import_star(ModuleObject mod, string name, ImportStarRefinement def) {
pragma[noinline]
private predicate module_and_name_for_import_star(
ModuleObject mod, string name, ImportStarRefinement def
) {
exists(ImportStarNode im_star |
module_and_name_for_import_star_helper(mod, name, im_star, def) and
mod.exports(name)
)
}
pragma [noinline]
private predicate module_and_name_for_import_star_helper(ModuleObject mod, string name, ImportStarNode im_star, ImportStarRefinement def) {
pragma[noinline]
private predicate module_and_name_for_import_star_helper(
ModuleObject mod, string name, ImportStarNode im_star, ImportStarRefinement def
) {
im_star = def.getDefiningNode() and
im_star.getModule().refersTo(mod) and
name = def.getSourceVariable().getName()
}
/** Holds if `def` is technically a defn of `var`, but the `from ... import *` does not in fact define `var` */
pragma [noinline]
/** Holds if `def` is technically a defn of `var`, but the `from ... import *` does not in fact define `var` */
pragma[noinline]
private predicate variable_not_redefined_by_import_star(EssaVariable var, ImportStarRefinement def) {
var = def.getInput() and
exists(ModuleObject mod |
@@ -251,8 +243,7 @@ private predicate variable_not_redefined_by_import_star(EssaVariable var, Import
/* Definition for `from ... import *` */
private predicate import_star_defn(ImportStarRefinement def, Definition defn) {
exists(ModuleObject mod, string name |
module_and_name_for_import_star(mod, name, def) |
exists(ModuleObject mod, string name | module_and_name_for_import_star(mod, name, def) |
/* Attribute from imported module */
scope_jump_to_defn_attribute(mod.getModule(), name, defn)
)
@@ -275,18 +266,20 @@ private predicate argument_defn(ArgumentRefinement def, Definition defn) {
}
/** Attribute deletions have no effect as far as value tracking is concerned. */
pragma [noinline]
pragma[noinline]
private predicate attribute_delete_defn(EssaAttributeDeletion def, Definition defn) {
ssa_variable_defn(def.getInput(), defn)
}
/* Definition flow for attributes. These mirror the "normal" defn predicates.
/*
* Definition flow for attributes. These mirror the "normal" defn predicates.
* For each defn predicate `xxx_defn(XXX def, Definition defn)`
* There is an equivalent predicate that tracks the values in attributes:
* `xxx_jump_to_defn_attribute(XXX def, string name, Definition defn)`
* */
*/
/** INTERNAL -- Public for testing only.
/**
* INTERNAL -- Public for testing only.
* Holds if the attribute `name` of the ssa variable `var` refers to (`value`, `cls`, `origin`)
*/
predicate ssa_variable_jump_to_defn_attribute(EssaVariable var, string name, Definition defn) {
@@ -316,7 +309,9 @@ private predicate ssa_phi_jump_to_defn_attribute(PhiFunction phi, string name, D
/** Helper for ssa_defn_jump_to_defn_attribute */
pragma[noinline]
private predicate ssa_node_jump_to_defn_attribute(EssaNodeDefinition def, string name, Definition defn) {
private predicate ssa_node_jump_to_defn_attribute(
EssaNodeDefinition def, string name, Definition defn
) {
assignment_jump_to_defn_attribute(def, name, defn)
or
self_parameter_jump_to_defn_attribute(def, name, defn)
@@ -326,14 +321,18 @@ private predicate ssa_node_jump_to_defn_attribute(EssaNodeDefinition def, string
/** Helper for ssa_defn_jump_to_defn_attribute */
pragma[noinline]
private predicate ssa_node_refinement_jump_to_defn_attribute(EssaNodeRefinement def, string name, Definition defn) {
private predicate ssa_node_refinement_jump_to_defn_attribute(
EssaNodeRefinement def, string name, Definition defn
) {
attribute_assignment_jump_to_defn_attribute(def, name, defn)
or
argument_jump_to_defn_attribute(def, name, defn)
}
pragma[noinline]
private predicate scope_entry_jump_to_defn_attribute(ScopeEntryDefinition def, string name, Definition defn) {
private predicate scope_entry_jump_to_defn_attribute(
ScopeEntryDefinition def, string name, Definition defn
) {
exists(EssaVariable var |
scope_entry_value_transfer(var, def) and
ssa_variable_jump_to_defn_attribute(var, name, defn)
@@ -342,9 +341,10 @@ private predicate scope_entry_jump_to_defn_attribute(ScopeEntryDefinition def, s
private predicate scope_jump_to_defn_attribute(ImportTimeScope s, string name, Definition defn) {
exists(EssaVariable var |
BaseFlow::reaches_exit(var) and var.getScope() = s and
BaseFlow::reaches_exit(var) and
var.getScope() = s and
var.getName() = name
|
|
ssa_variable_defn(var, defn)
)
}
@@ -357,22 +357,23 @@ private predicate jump_to_defn_attribute(ControlFlowNode use, string name, Defin
)
or
/* Instance attributes */
exists(ClassObject cls |
use.refersTo(_, cls, _) |
exists(ClassObject cls | use.refersTo(_, cls, _) |
scope_jump_to_defn_attribute(cls.getPyClass(), name, defn)
)
or
/* Super attributes */
exists(AttrNode f, SuperBoundMethod sbm, Object function |
use = f.getObject(name) and
f.refersTo(sbm) and function = sbm.getFunction(_) and
f.refersTo(sbm) and
function = sbm.getFunction(_) and
function.getOrigin() = defn.getAstNode()
)
or
/* Class or module attribute */
exists(Object obj, Scope scope |
use.refersTo(obj) and
scope_jump_to_defn_attribute(scope, name, defn) |
scope_jump_to_defn_attribute(scope, name, defn)
|
obj.(ClassObject).getPyClass() = scope
or
obj.(PythonModuleObject).getModule() = scope
@@ -382,18 +383,23 @@ private predicate jump_to_defn_attribute(ControlFlowNode use, string name, Defin
}
pragma[noinline]
private predicate assignment_jump_to_defn_attribute(AssignmentDefinition def, string name, Definition defn) {
private predicate assignment_jump_to_defn_attribute(
AssignmentDefinition def, string name, Definition defn
) {
jump_to_defn_attribute(def.getValue(), name, defn)
}
pragma[noinline]
private predicate attribute_assignment_jump_to_defn_attribute(AttributeAssignment def, string name, Definition defn) {
private predicate attribute_assignment_jump_to_defn_attribute(
AttributeAssignment def, string name, Definition defn
) {
defn.getAstNode() = def.getDefiningNode().getNode() and name = def.getName()
or
ssa_variable_jump_to_defn_attribute(def.getInput(), name, defn) and not name = def.getName()
}
/** Holds if `def` defines the attribute `name`
/**
* Holds if `def` defines the attribute `name`
* `def` takes the form `setattr(use, "name")` where `use` is the input to the defn.
*/
private predicate sets_attribute(ArgumentRefinement def, string name) {
@@ -406,31 +412,37 @@ private predicate sets_attribute(ArgumentRefinement def, string name) {
}
pragma[noinline]
private predicate argument_jump_to_defn_attribute(ArgumentRefinement def, string name, Definition defn) {
if sets_attribute(def, name) then
jump_to_defn(def.getDefiningNode().(CallNode).getArg(2), defn)
else
ssa_variable_jump_to_defn_attribute(def.getInput(), name, defn)
private predicate argument_jump_to_defn_attribute(
ArgumentRefinement def, string name, Definition defn
) {
if sets_attribute(def, name)
then jump_to_defn(def.getDefiningNode().(CallNode).getArg(2), defn)
else ssa_variable_jump_to_defn_attribute(def.getInput(), name, defn)
}
/** Gets the (temporally) preceding variable for "self", e.g. `def` is in method foo() and `result` is in `__init__()`. */
/** Gets the (temporally) preceding variable for "self", e.g. `def` is in method foo() and `result` is in `__init__()`. */
private EssaVariable preceding_self_variable(ParameterDefinition def) {
def.isSelf() and
exists(Function preceding, Function method |
method = def.getScope() and
method = def.getScope() and
// Only methods
preceding.isMethod() and preceding.precedes(method) and
BaseFlow::reaches_exit(result) and result.getSourceVariable().(Variable).isSelf() and
preceding.isMethod() and
preceding.precedes(method) and
BaseFlow::reaches_exit(result) and
result.getSourceVariable().(Variable).isSelf() and
result.getScope() = preceding
)
}
pragma [noinline]
private predicate self_parameter_jump_to_defn_attribute(ParameterDefinition def, string name, Definition defn) {
pragma[noinline]
private predicate self_parameter_jump_to_defn_attribute(
ParameterDefinition def, string name, Definition defn
) {
ssa_variable_jump_to_defn_attribute(preceding_self_variable(def), name, defn)
}
/** Gets a definition for 'use'.
/**
* Gets a definition for 'use'.
* This exists primarily for testing use `getPreferredDefinition()` instead.
*/
Definition getADefinition(Expr use) {
@@ -441,7 +453,8 @@ Definition getADefinition(Expr use) {
not result = TLocalDefinition(use)
}
/** Gets the unique definition for 'use', if one can be found.
/**
* Gets the unique definition for 'use', if one can be found.
* Helper for the jump-to-definition query.
*/
Definition getUniqueDefinition(Expr use) {
@@ -452,18 +465,13 @@ Definition getUniqueDefinition(Expr use) {
not result = TLocalDefinition(use)
}
/** Helper class to get suitable locations for attributes */
class NiceLocationExpr extends @py_expr {
string toString() {
result = this.(Expr).toString()
}
string toString() { result = this.(Expr).toString() }
predicate hasLocationInfo(string f, int bl, int bc, int el, int ec) {
/* Attribute location for x.y is that of 'y' so that url does not overlap with that of 'x' */
exists(int abl, int abc |
this.(Attribute).getLocation().hasLocationInfo(f, abl, abc, el, ec) |
/* Attribute location for x.y is that of 'y' so that url does not overlap with that of 'x' */
exists(int abl, int abc | this.(Attribute).getLocation().hasLocationInfo(f, abl, abc, el, ec) |
bl = el and bc = ec - this.(Attribute).getName().length() + 1
)
or
@@ -477,10 +485,8 @@ class NiceLocationExpr extends @py_expr {
exists(string name |
name = this.(ImportMember).getName() and
this.(ImportMember).getLocation().hasLocationInfo(f, _, _, el, ec) and
bl = el and bc = ec-name.length()+1
bl = el and
bc = ec - name.length() + 1
)
}
}

1
python/ql/src/default.qll
View File

@@ -2,4 +2,5 @@
* WARNING: Use of this module is DEPRECATED.
* All new queries should use `import python`.
*/
import python

132
python/ql/src/external/CodeDuplication.qll vendored
View File

@@ -3,10 +3,7 @@
import python
/** Gets the relative path of `file`, with backslashes replaced by forward slashes. */
private
string relativePath(File file) {
result = file.getRelativePath().replaceAll("\\", "/")
}
private string relativePath(File file) { result = file.getRelativePath().replaceAll("\\", "/") }
/**
* Holds if the `index`-th token of block `copy` is in file `file`, spanning
@@ -21,60 +18,42 @@ private predicate tokenLocation(File file, int sl, int sc, int ec, int el, Copy
}
/** A token block used for detection of duplicate and similar code. */
class Copy extends @duplication_or_similarity
{
private
int lastToken() {
result = max(int i | tokens(this, i, _, _, _, _) | i)
}
class Copy extends @duplication_or_similarity {
private int lastToken() { result = max(int i | tokens(this, i, _, _, _, _) | i) }
/** Gets the index of the token in this block starting at the location `loc`, if any. */
int tokenStartingAt(Location loc) {
tokenLocation(loc.getFile(), loc.getStartLine(), loc.getStartColumn(),
_, _, this, result)
tokenLocation(loc.getFile(), loc.getStartLine(), loc.getStartColumn(), _, _, this, result)
}
/** Gets the index of the token in this block ending at the location `loc`, if any. */
int tokenEndingAt(Location loc) {
tokenLocation(loc.getFile(), _, _,
loc.getEndLine(), loc.getEndColumn(), this, result)
tokenLocation(loc.getFile(), _, _, loc.getEndLine(), loc.getEndColumn(), this, result)
}
/** Gets the line on which the first token in this block starts. */
int sourceStartLine() {
tokens(this, 0, result, _, _, _)
}
int sourceStartLine() { tokens(this, 0, result, _, _, _) }
/** Gets the column on which the first token in this block starts. */
int sourceStartColumn() {
tokens(this, 0, _, result, _, _)
}
int sourceStartColumn() { tokens(this, 0, _, result, _, _) }
/** Gets the line on which the last token in this block ends. */
int sourceEndLine() {
tokens(this, this.lastToken(), _, _, result, _)
}
int sourceEndLine() { tokens(this, this.lastToken(), _, _, result, _) }
/** Gets the column on which the last token in this block ends. */
int sourceEndColumn() {
tokens(this, this.lastToken(), _, _, _, result)
}
int sourceEndColumn() { tokens(this, this.lastToken(), _, _, _, result) }
/** Gets the number of lines containing at least (part of) one token in this block. */
int sourceLines() {
result = this.sourceEndLine() + 1 - this.sourceStartLine()
}
int sourceLines() { result = this.sourceEndLine() + 1 - this.sourceStartLine() }
/** Gets an opaque identifier for the equivalence class of this block. */
int getEquivalenceClass() {
duplicateCode(this, _, result) or similarCode(this, _, result)
}
int getEquivalenceClass() { duplicateCode(this, _, result) or similarCode(this, _, result) }
/** Gets the source file in which this block appears. */
File sourceFile() {
exists(string name |
duplicateCode(this, name, _) or similarCode(this, name, _) |
name.replaceAll("\\", "/") = relativePath(result))
exists(string name | duplicateCode(this, name, _) or similarCode(this, name, _) |
name.replaceAll("\\", "/") = relativePath(result)
)
}
/**
@@ -84,7 +63,9 @@ class Copy extends @duplication_or_similarity
* For more information, see
* [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
*/
predicate hasLocationInfo(string filepath, int startline, int startcolumn, int endline, int endcolumn) {
predicate hasLocationInfo(
string filepath, int startline, int startcolumn, int endline, int endcolumn
) {
sourceFile().getName() = filepath and
startline = sourceStartLine() and
startcolumn = sourceStartColumn() and
@@ -102,7 +83,8 @@ class Copy extends @duplication_or_similarity
Copy extendingBlock() {
exists(File file, int sl, int sc, int ec, int el |
tokenLocation(file, sl, sc, ec, el, this, _) and
tokenLocation(file, sl, sc, ec, el, result, 0)) and
tokenLocation(file, sl, sc, ec, el, result, 0)
) and
this != result
}
}
@@ -113,13 +95,17 @@ class Copy extends @duplication_or_similarity
* have the same equivalence class, with `start` being the equivalence class of `start1` and
* `start2`, and `end` the equivalence class of `end1` and `end2`.
*/
predicate similar_extension(SimilarBlock start1, SimilarBlock start2, SimilarBlock ext1, SimilarBlock ext2, int start, int ext) {
predicate similar_extension(
SimilarBlock start1, SimilarBlock start2, SimilarBlock ext1, SimilarBlock ext2, int start, int ext
) {
start1.getEquivalenceClass() = start and
start2.getEquivalenceClass() = start and
ext1.getEquivalenceClass() = ext and
ext2.getEquivalenceClass() = ext and
start1 != start2 and
(ext1 = start1 and ext2 = start2 or
(
ext1 = start1 and ext2 = start2
or
similar_extension(start1.extendingBlock(), start2.extendingBlock(), ext1, ext2, _, ext)
)
}
@@ -130,28 +116,29 @@ predicate similar_extension(SimilarBlock start1, SimilarBlock start2, SimilarBlo
* have the same equivalence class, with `start` being the equivalence class of `start1` and
* `start2`, and `end` the equivalence class of `end1` and `end2`.
*/
predicate duplicate_extension(DuplicateBlock start1, DuplicateBlock start2, DuplicateBlock ext1, DuplicateBlock ext2, int start, int ext) {
predicate duplicate_extension(
DuplicateBlock start1, DuplicateBlock start2, DuplicateBlock ext1, DuplicateBlock ext2, int start,
int ext
) {
start1.getEquivalenceClass() = start and
start2.getEquivalenceClass() = start and
ext1.getEquivalenceClass() = ext and
ext2.getEquivalenceClass() = ext and
start1 != start2 and
(ext1 = start1 and ext2 = start2 or
(
ext1 = start1 and ext2 = start2
or
duplicate_extension(start1.extendingBlock(), start2.extendingBlock(), ext1, ext2, _, ext)
)
}
/** A block of duplicated code. */
class DuplicateBlock extends Copy, @duplication
{
override string toString() {
result = "Duplicate code: " + sourceLines() + " duplicated lines."
}
class DuplicateBlock extends Copy, @duplication {
override string toString() { result = "Duplicate code: " + sourceLines() + " duplicated lines." }
}
/** A block of similar code. */
class SimilarBlock extends Copy, @similarity
{
class SimilarBlock extends Copy, @similarity {
override string toString() {
result = "Similar code: " + sourceLines() + " almost duplicated lines."
}
@@ -167,8 +154,9 @@ predicate duplicateStatement(Scope scope1, Scope scope2, Stmt stmt1, Stmt stmt2)
scope2.contains(stmt2) and
duplicateCoversStatement(equivstart, equivend, first, last, stmt1) and
duplicateCoversStatement(equivstart, equivend, first, last, stmt2) and
stmt1 != stmt2 and scope1 != scope2
)
stmt1 != stmt2 and
scope1 != scope2
)
}
/**
@@ -178,8 +166,9 @@ predicate duplicateStatement(Scope scope1, Scope scope2, Stmt stmt1, Stmt stmt2)
* and `equivstart` and `equivend` are the equivalence classes of the first and the last
* block, respectively.
*/
private
predicate duplicateCoversStatement(int equivstart, int equivend, int first, int last, Stmt stmt) {
private predicate duplicateCoversStatement(
int equivstart, int equivend, int first, int last, Stmt stmt
) {
exists(DuplicateBlock b1, DuplicateBlock b2, Location startloc, Location endloc |
stmt.getLocation() = startloc and
stmt.getLastStatement().getLocation() = endloc and
@@ -204,13 +193,14 @@ predicate duplicateStatements(Scope scope1, Scope scope2, int duplicate, int tot
* Find pairs of scopes that are identical or almost identical
*/
predicate duplicateScopes(Scope s, Scope other, float percent, string message) {
exists(int total, int duplicate |
duplicateStatements(s, other, duplicate, total) |
percent = 100.0 * duplicate / total and percent >= 80.0 and
if duplicate = total then
message = "All " + total + " statements in " + s.getName() + " are identical in $@."
exists(int total, int duplicate | duplicateStatements(s, other, duplicate, total) |
percent = 100.0 * duplicate / total and
percent >= 80.0 and
if duplicate = total
then message = "All " + total + " statements in " + s.getName() + " are identical in $@."
else
message = duplicate + " out of " + total + " statements in " + s.getName() + " are duplicated in $@."
message =
duplicate + " out of " + total + " statements in " + s.getName() + " are duplicated in $@."
)
}
@@ -219,12 +209,13 @@ predicate duplicateScopes(Scope s, Scope other, float percent, string message) {
* respectively, where `scope1` and `scope2` are not the same.
*/
private predicate similarStatement(Scope scope1, Scope scope2, Stmt stmt1, Stmt stmt2) {
exists(int start, int end, int first, int last |
exists(int start, int end, int first, int last |
scope1.contains(stmt1) and
scope2.contains(stmt2) and
similarCoversStatement(start, end, first, last, stmt1) and
similarCoversStatement(start, end, first, last, stmt2) and
stmt1 != stmt2 and scope1 != scope2
stmt1 != stmt2 and
scope1 != scope2
)
}
@@ -235,7 +226,9 @@ private predicate similarStatement(Scope scope1, Scope scope2, Stmt stmt1, Stmt
* and `equivstart` and `equivend` are the equivalence classes of the first and the last
* block, respectively.
*/
private predicate similarCoversStatement(int equivstart, int equivend, int first, int last, Stmt stmt) {
private predicate similarCoversStatement(
int equivstart, int equivend, int first, int last, Stmt stmt
) {
exists(SimilarBlock b1, SimilarBlock b2, Location startloc, Location endloc |
stmt.getLocation() = startloc and
stmt.getLastStatement().getLocation() = endloc and
@@ -260,13 +253,14 @@ private predicate similarStatements(Scope scope1, Scope scope2, int similar, int
* Find pairs of scopes that are similar
*/
predicate similarScopes(Scope s, Scope other, float percent, string message) {
exists(int total, int similar |
similarStatements(s, other, similar, total) |
percent = 100.0 * similar / total and percent >= 80.0 and
if similar = total then
message = "All statements in " + s.getName() + " are similar in $@."
exists(int total, int similar | similarStatements(s, other, similar, total) |
percent = 100.0 * similar / total and
percent >= 80.0 and
if similar = total
then message = "All statements in " + s.getName() + " are similar in $@."
else
message = similar + " out of " + total + " statements in " + s.getName() + " are similar in $@."
message =
similar + " out of " + total + " statements in " + s.getName() + " are similar in $@."
)
}
@@ -275,7 +269,5 @@ predicate similarScopes(Scope s, Scope other, float percent, string message) {
* This is true for blocks of import statements.
*/
predicate whitelistedLineForDuplication(File f, int line) {
exists(ImportingStmt i |
i.getLocation().getFile() = f and i.getLocation().getStartLine() = line
)
exists(ImportingStmt i | i.getLocation().getFile() = f and i.getLocation().getStartLine() = line)
}

73
python/ql/src/external/DefectFilter.qll vendored
View File

@@ -10,58 +10,61 @@ import semmle.python.Files
*
* For more information, see [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
*/
external predicate defectResults(int id, string queryPath, string filepath, int startline,
int startcol, int endline, int endcol, string message);
external predicate defectResults(
int id, string queryPath, string filepath, int startline, int startcol, int endline, int endcol,
string message
);
/**
* A defect query result stored in a dashboard database.
*/
class DefectResult extends int {
DefectResult() { defectResults(this, _, _, _, _, _, _, _) }
DefectResult() { defectResults(this, _, _, _, _, _, _, _) }
/** Gets the path of the query that reported the result. */
string getQueryPath() { defectResults(this, result, _, _, _, _, _, _) }
/** Gets the path of the query that reported the result. */
string getQueryPath() { defectResults(this, result, _, _, _, _, _, _) }
/** Gets the file in which this query result was reported. */
File getFile() {
exists(string path | defectResults(this, _, path, _, _, _, _, _) and result.getName() = path)
}
/** Gets the file in which this query result was reported. */
File getFile() {
exists(string path | defectResults(this, _, path, _, _, _, _, _) and result.getName() = path)
}
/** Gets the file path in which this query result was reported. */
string getFilePath() { defectResults(this, _, result, _, _, _, _, _) }
/** Gets the file path in which this query result was reported. */
string getFilePath() { defectResults(this, _, result, _, _, _, _, _) }
/** Gets the line on which the location of this query result starts. */
int getStartLine() { defectResults(this, _, _, result, _, _, _, _) }
/** Gets the line on which the location of this query result starts. */
int getStartLine() { defectResults(this, _, _, result, _, _, _, _) }
/** Gets the column on which the location of this query result starts. */
int getStartColumn() { defectResults(this, _, _, _, result, _, _, _) }
/** Gets the column on which the location of this query result starts. */
int getStartColumn() { defectResults(this, _, _, _, result, _, _, _) }
/** Gets the line on which the location of this query result ends. */
int getEndLine() { defectResults(this, _, _, _, _, result, _, _) }
/** Gets the line on which the location of this query result ends. */
int getEndLine() { defectResults(this, _, _, _, _, result, _, _) }
/** Gets the column on which the location of this query result ends. */
int getEndColumn() { defectResults(this, _, _, _, _, _, result, _) }
/** Gets the column on which the location of this query result ends. */
int getEndColumn() { defectResults(this, _, _, _, _, _, result, _) }
/** Gets the message associated with this query result. */
string getMessage() { defectResults(this, _, _, _, _, _, _, result) }
/** Gets the message associated with this query result. */
string getMessage() { defectResults(this, _, _, _, _, _, _, result) }
predicate hasLocationInfo(string path, int sl, int sc, int el, int ec) {
defectResults(this, _, path, sl, sc, el, ec, _)
}
/** Gets the URL corresponding to the location of this query result. */
string getURL() {
result = "file://" + getFile().getName() + ":" + getStartLine() + ":" + getStartColumn() + ":" + getEndLine() + ":" + getEndColumn()
}
predicate hasLocationInfo(string path, int sl, int sc, int el, int ec) {
defectResults(this, _, path, sl, sc, el, ec, _)
}
/** Gets the URL corresponding to the location of this query result. */
string getURL() {
result =
"file://" + getFile().getName() + ":" + getStartLine() + ":" + getStartColumn() + ":" +
getEndLine() + ":" + getEndColumn()
}
}
// crude containment by line number only
predicate contains(Location l, DefectResult res) {
exists(string path, int bl1, int el1, int bl2, int el2 |
l.hasLocationInfo(path, bl1, _, el1, _)
and res.hasLocationInfo(path, bl2, _, el2, _)
and bl1 <= bl2 and el1 >= el2
)
exists(string path, int bl1, int el1, int bl2, int el2 |
l.hasLocationInfo(path, bl1, _, el1, _) and
res.hasLocationInfo(path, bl2, _, el2, _) and
bl1 <= bl2 and
el1 >= el2
)
}

87
python/ql/src/external/ExternalArtifact.qll vendored
View File

@@ -1,7 +1,6 @@
import python
class ExternalDefect extends @externalDefect {
string getQueryPath() {
exists(string path |
externalDefects(this, path, _, _, _) and
@@ -9,95 +8,59 @@ class ExternalDefect extends @externalDefect {
)
}
string getMessage() {
externalDefects(this, _, _, result, _)
}
string getMessage() { externalDefects(this, _, _, result, _) }
float getSeverity() {
externalDefects(this, _, _, _, result)
}
float getSeverity() { externalDefects(this, _, _, _, result) }
Location getLocation() {
externalDefects(this,_,result,_,_)
}
Location getLocation() { externalDefects(this, _, result, _, _) }
string toString() {
result = getQueryPath() + ": " + getLocation() + " - " + getMessage()
}
string toString() { result = getQueryPath() + ": " + getLocation() + " - " + getMessage() }
}
class ExternalMetric extends @externalMetric {
string getQueryPath() { externalMetrics(this, result, _, _) }
string getQueryPath() {
externalMetrics(this, result, _, _)
}
float getValue() { externalMetrics(this, _, _, result) }
float getValue() {
externalMetrics(this, _, _, result)
}
Location getLocation() { externalMetrics(this, _, result, _) }
Location getLocation() {
externalMetrics(this,_,result,_)
}
string toString() {
result = getQueryPath() + ": " + getLocation() + " - " + getValue()
}
string toString() { result = getQueryPath() + ": " + getLocation() + " - " + getValue() }
}
class ExternalData extends @externalDataElement {
string getDataPath() { externalData(this, result, _, _) }
string getDataPath() {
externalData(this, result, _, _)
}
string getQueryPath() { result = getDataPath().regexpReplaceAll("\\.[^.]*$", ".ql") }
string getQueryPath() {
result = getDataPath().regexpReplaceAll("\\.[^.]*$", ".ql")
}
int getNumFields() { result = 1 + max(int i | externalData(this, _, i, _) | i) }
int getNumFields() {
result = 1 + max(int i | externalData(this, _, i, _) | i)
}
string getField(int index) { externalData(this, _, index, result) }
string getField(int index) {
externalData(this, _, index, result)
}
int getFieldAsInt(int index) { result = getField(index).toInt() }
int getFieldAsInt(int index) {
result = getField(index).toInt()
}
float getFieldAsFloat(int index) { result = getField(index).toFloat() }
float getFieldAsFloat(int index) {
result = getField(index).toFloat()
}
date getFieldAsDate(int index) { result = getField(index).toDate() }
date getFieldAsDate(int index) {
result = getField(index).toDate()
}
string toString() {
result = getQueryPath() + ": " + buildTupleString(0)
}
string toString() { result = getQueryPath() + ": " + buildTupleString(0) }
private string buildTupleString(int start) {
(start = getNumFields() - 1 and result = getField(start))
start = getNumFields() - 1 and result = getField(start)
or
(start < getNumFields() - 1 and result = getField(start) + "," + buildTupleString(start+1))
start < getNumFields() - 1 and result = getField(start) + "," + buildTupleString(start + 1)
}
}
/**
* External data with a location, and a message, as produced by tools that used to produce QLDs.
*/
class DefectExternalData extends ExternalData {
DefectExternalData() {
this.getField(0).regexpMatch("\\w+://.*:[0-9]+:[0-9]+:[0-9]+:[0-9]+$") and
this.getNumFields() = 2
}
string getURL() { result = getField(0) }
DefectExternalData() {
this.getField(0).regexpMatch("\\w+://.*:[0-9]+:[0-9]+:[0-9]+:[0-9]+$") and
this.getNumFields() = 2
}
string getMessage() { result = getField(1) }
string getURL() { result = getField(0) }
string getMessage() { result = getField(1) }
}

228
python/ql/src/external/Thrift.qll vendored
View File

@@ -3,68 +3,42 @@
* This code is under development and may change without warning.
*/
import external.ExternalArtifact
/** An item in the parse tree of the IDL file */
class ThriftElement extends ExternalData {
string kind;
ThriftElement() {
this.getDataPath() = "thrift-" + kind
}
ThriftElement() { this.getDataPath() = "thrift-" + kind }
string getKind() {
result = kind
}
string getKind() { result = kind }
string getId() {
result = getField(0)
}
string getId() { result = getField(0) }
int getIndex() {
result = getFieldAsInt(1)
}
int getIndex() { result = getFieldAsInt(1) }
ThriftElement getParent() {
result.getId() = this.getField(2)
}
ThriftElement getParent() { result.getId() = this.getField(2) }
string getValue() {
result = this.getField(3)
}
string getValue() { result = this.getField(3) }
ThriftElement getChild(int n) {
result.getIndex() = n and result.getParent() = this
}
ThriftElement getChild(int n) { result.getIndex() = n and result.getParent() = this }
ThriftElement getAChild() {
result = this.getChild(_)
}
ThriftElement getAChild() { result = this.getChild(_) }
override string toString() {
result = this.getKind()
}
override string toString() { result = this.getKind() }
string getPath() {
result = this.getField(4)
}
string getPath() { result = this.getField(4) }
private int line() {
result = this.getFieldAsInt(5)
}
private int line() { result = this.getFieldAsInt(5) }
private int column() {
result = this.getFieldAsInt(6)
}
private int column() { result = this.getFieldAsInt(6) }
predicate hasLocationInfo(string fp, int bl, int bc, int el, int ec) {
fp = this.getPath() and
bl = this.line() and
bc = this.column() and
el = this.line() and
ec = this.column() + this.getValue().length()-1
ec = this.column() + this.getValue().length() - 1
or
exists(ThriftElement first, ThriftElement last |
first = this.getChild(min(int l | exists(this.getChild(l)))) and
@@ -74,19 +48,13 @@ class ThriftElement extends ExternalData {
)
}
File getFile() {
this.hasLocationInfo(result.getAbsolutePath(), _, _, _, _)
}
File getFile() { this.hasLocationInfo(result.getAbsolutePath(), _, _, _, _) }
}
abstract class ThriftNamedElement extends ThriftElement {
abstract ThriftElement getNameElement();
final string getName() {
result = this.getNameElement().getValue()
}
final string getName() { result = this.getNameElement().getValue() }
override string toString() {
result = this.getKind() + " " + this.getName()
@@ -101,14 +69,10 @@ abstract class ThriftNamedElement extends ThriftElement {
this.getNameElement().hasLocationInfo(fp, _, _, el, ec)
)
}
}
class ThriftType extends ThriftNamedElement {
ThriftType() {
kind.matches("%type")
}
ThriftType() { kind.matches("%type") }
override ThriftElement getNameElement() {
result = this.getChild(0)
@@ -116,9 +80,7 @@ class ThriftType extends ThriftNamedElement {
result = this.getChild(0).(ThriftType).getNameElement()
}
override string toString() {
result = "type " + this.getName()
}
override string toString() { result = "type " + this.getName() }
predicate references(ThriftStruct struct) {
this.getName() = struct.getName() and
@@ -127,194 +89,106 @@ class ThriftType extends ThriftNamedElement {
struct.hasLocationInfo(path, _, _, _, _)
)
}
}
/** A thrift typedef */
class ThriftTypeDef extends ThriftNamedElement {
ThriftTypeDef() { kind.matches("typedef") }
ThriftTypeDef() {
kind.matches("typedef")
}
override ThriftElement getNameElement() {
result = this.getChild(2).getChild(0)
}
override ThriftElement getNameElement() { result = this.getChild(2).getChild(0) }
}
/** A thrift enum declaration */
class ThriftEnum extends ThriftNamedElement {
ThriftEnum() { kind.matches("enum") }
ThriftEnum() {
kind.matches("enum")
}
override ThriftElement getNameElement() {
result = this.getChild(0).getChild(0)
}
override ThriftElement getNameElement() { result = this.getChild(0).getChild(0) }
}
/** A thrift enum field */
class ThriftEnumField extends ThriftNamedElement {
ThriftEnumField() { kind.matches("enumfield") }
ThriftEnumField() {
kind.matches("enumfield")
}
override ThriftElement getNameElement() {
result = this.getChild(0).getChild(0)
}
override ThriftElement getNameElement() { result = this.getChild(0).getChild(0) }
}
/** A thrift service declaration */
class ThriftService extends ThriftNamedElement {
ThriftService() { kind.matches("service") }
ThriftService() {
kind.matches("service")
}
override ThriftElement getNameElement() { result = this.getChild(0).getChild(0) }
override ThriftElement getNameElement() {
result = this.getChild(0).getChild(0)
}
ThriftFunction getAFunction() {
result = this.getChild(_)
}
ThriftFunction getAFunction() { result = this.getChild(_) }
ThriftFunction getFunction(string name) {
result.getName() = name and
result = this.getAFunction()
}
}
/** A thrift function declaration */
class ThriftFunction extends ThriftNamedElement {
ThriftFunction() { kind.matches("function") }
ThriftFunction() {
kind.matches("function")
}
override ThriftElement getNameElement() { result = this.getChild(2).getChild(0) }
override ThriftElement getNameElement() {
result = this.getChild(2).getChild(0)
}
ThriftField getArgument(int n) { result = this.getChild(n + 3) }
ThriftField getArgument(int n) {
result = this.getChild(n+3)
}
ThriftField getAnArgument() { result = this.getArgument(_) }
ThriftField getAnArgument() {
result = this.getArgument(_)
}
private ThriftThrows getAllThrows() { result = this.getChild(_) }
private ThriftThrows getAllThrows() {
result = this.getChild(_)
}
ThriftField getAThrows() { result = this.getAllThrows().getAChild() }
ThriftField getAThrows() {
result = this.getAllThrows().getAChild()
}
ThriftType getReturnType() {
result = this.getChild(1).getChild(0)
}
ThriftType getReturnType() { result = this.getChild(1).getChild(0) }
override predicate hasLocationInfo(string fp, int bl, int bc, int el, int ec) {
this.getChild(1).hasLocationInfo(fp, bl, bc, _, _) and
this.getChild(2).hasLocationInfo(fp, _, _, el, ec)
}
ThriftService getService() {
result.getAFunction() = this
}
string getQualifiedName() {
result = this.getService().getName() + "." + this.getName()
}
ThriftService getService() { result.getAFunction() = this }
string getQualifiedName() { result = this.getService().getName() + "." + this.getName() }
}
class ThriftField extends ThriftNamedElement {
ThriftField() { kind.matches("field") }
ThriftField() {
kind.matches("field")
}
override ThriftElement getNameElement() {
result = this.getChild(4)
}
ThriftType getType() {
result = this.getChild(2)
}
override ThriftElement getNameElement() { result = this.getChild(4) }
ThriftType getType() { result = this.getChild(2) }
}
class ThriftStruct extends ThriftNamedElement {
class ThriftStruct extends ThriftNamedElement {
ThriftStruct() { kind.matches("struct") }
ThriftStruct() {
kind.matches("struct")
}
override ThriftElement getNameElement() { result = this.getChild(0).getChild(0) }
override ThriftElement getNameElement() {
result = this.getChild(0).getChild(0)
}
ThriftField getMember(int n) {
result = this.getChild(n+1)
}
ThriftField getAMember() {
result = this.getMember(_)
}
ThriftField getMember(int n) { result = this.getChild(n + 1) }
ThriftField getAMember() { result = this.getMember(_) }
}
class ThriftException extends ThriftNamedElement {
ThriftException() { kind.matches("exception") }
ThriftException() {
kind.matches("exception")
}
override ThriftElement getNameElement() { result = this.getChild(0).getChild(0) }
override ThriftElement getNameElement() {
result = this.getChild(0).getChild(0)
}
ThriftField getMember(int n) {
result = this.getChild(n+1)
}
ThriftField getAMember() {
result = this.getMember(_)
}
ThriftField getMember(int n) { result = this.getChild(n + 1) }
ThriftField getAMember() { result = this.getMember(_) }
}
class ThriftThrows extends ThriftElement {
ThriftThrows() { kind.matches("throws") }
ThriftThrows() {
kind.matches("throws")
}
ThriftField getAThrows() {
result = this.getChild(_)
}
ThriftField getAThrows() { result = this.getChild(_) }
}
/** A parse tree element that holds a primitive value */
class ThriftValue extends ThriftElement {
ThriftValue() { exists(this.getValue()) }
ThriftValue() {
exists(this.getValue())
}
override string toString() {
result = this.getKind() + " " + this.getValue()
}
override string toString() { result = this.getKind() + " " + this.getValue() }
}

120
python/ql/src/external/VCS.qll vendored
View File

@@ -1,92 +1,76 @@
import python
class Commit extends @svnentry {
Commit() {
svnaffectedfiles(this, _, _) and
exists(date svnDate, date snapshotDate |
svnentries(this, _, _, svnDate, _) and
snapshotDate(snapshotDate) and
svnDate <= snapshotDate
)
}
Commit() {
svnaffectedfiles(this, _, _) and
exists(date svnDate, date snapshotDate |
svnentries(this, _, _, svnDate, _) and
snapshotDate(snapshotDate) and
svnDate <= snapshotDate
)
}
string toString() { result = this.getRevisionName() }
string toString() { result = this.getRevisionName() }
string getRevisionName() { svnentries(this, result, _, _, _) }
string getRevisionName() { svnentries(this, result, _, _, _) }
string getAuthor() { svnentries(this, _, result, _, _) }
date getDate() { svnentries(this, _, _, result, _) }
int getChangeSize() { svnentries(this, _, _, _, result) }
string getMessage() { svnentrymsg(this, result) }
string getAnAffectedFilePath(string action) {
exists(File rawFile | svnaffectedfiles(this, rawFile, action) |
result = rawFile.getName()
)
}
string getAnAffectedFilePath() { result = getAnAffectedFilePath(_) }
File getAnAffectedFile(string action) {
svnaffectedfiles(this,result,action)
}
string getAuthor() { svnentries(this, _, result, _, _) }
File getAnAffectedFile() { exists(string action | result = this.getAnAffectedFile(action)) }
date getDate() { svnentries(this, _, _, result, _) }
predicate isRecent() { recentCommit(this) }
int daysToNow() {
exists(date now | snapshotDate(now) |
result = getDate().daysTo(now) and result >= 0
)
}
int getRecentAdditionsForFile(File f) {
svnchurn(this, f, result, _)
}
int getRecentDeletionsForFile(File f) {
svnchurn(this, f, _, result)
}
int getChangeSize() { svnentries(this, _, _, _, result) }
int getRecentChurnForFile(File f) {
result = getRecentAdditionsForFile(f) + getRecentDeletionsForFile(f)
}
string getMessage() { svnentrymsg(this, result) }
string getAnAffectedFilePath(string action) {
exists(File rawFile | svnaffectedfiles(this, rawFile, action) | result = rawFile.getName())
}
string getAnAffectedFilePath() { result = getAnAffectedFilePath(_) }
File getAnAffectedFile(string action) { svnaffectedfiles(this, result, action) }
File getAnAffectedFile() { exists(string action | result = this.getAnAffectedFile(action)) }
predicate isRecent() { recentCommit(this) }
int daysToNow() {
exists(date now | snapshotDate(now) | result = getDate().daysTo(now) and result >= 0)
}
int getRecentAdditionsForFile(File f) { svnchurn(this, f, result, _) }
int getRecentDeletionsForFile(File f) { svnchurn(this, f, _, result) }
int getRecentChurnForFile(File f) {
result = getRecentAdditionsForFile(f) + getRecentDeletionsForFile(f)
}
}
class Author extends string {
Author() { exists(Commit e | this = e.getAuthor()) }
Commit getACommit() { result.getAuthor() = this }
File getAnEditedFile() { result = this.getACommit().getAnAffectedFile() }
Author() { exists(Commit e | this = e.getAuthor()) }
Commit getACommit() { result.getAuthor() = this }
File getAnEditedFile() { result = this.getACommit().getAnAffectedFile() }
}
predicate recentCommit(Commit e) {
exists(date snapshotDate, date commitDate, int days |
snapshotDate(snapshotDate) and
e.getDate() = commitDate and
days = commitDate.daysTo(snapshotDate) and
days >= 0 and days <= 60
)
exists(date snapshotDate, date commitDate, int days |
snapshotDate(snapshotDate) and
e.getDate() = commitDate and
days = commitDate.daysTo(snapshotDate) and
days >= 0 and
days <= 60
)
}
date firstChange(File f) {
result = min(Commit e, date toMin | (f = e.getAnAffectedFile()) and (toMin = e.getDate()) | toMin)
result = min(Commit e, date toMin | f = e.getAnAffectedFile() and toMin = e.getDate() | toMin)
}
predicate firstCommit(Commit e) {
not exists(File f | f = e.getAnAffectedFile() |
firstChange(f) < e.getDate()
)
not exists(File f | f = e.getAnAffectedFile() | firstChange(f) < e.getDate())
}
predicate artificialChange(Commit e) {
firstCommit(e) or e.getChangeSize() >= 50000
}
predicate artificialChange(Commit e) { firstCommit(e) or e.getChangeSize() >= 50000 }

1
python/ql/src/python.qll
View File

@@ -34,5 +34,4 @@ import semmle.python.pointsto.Base
import semmle.python.pointsto.Context
import semmle.python.pointsto.CallGraph
import semmle.python.objects.ObjectAPI
import site

126
python/ql/src/semmle/crypto/Crypto.qll
View File

@@ -2,10 +2,10 @@
* Provides classes for modeling cryptographic libraries.
*/
/* The following information is copied from `/semmlecode-javascript-queries/semmle/javascript/frameworks/CryptoLibraries.qll`
/*
* The following information is copied from `/semmlecode-javascript-queries/semmle/javascript/frameworks/CryptoLibraries.qll`
* which should be considered the definitive version (as of Feb 2018)
*/
/**
* Names of cryptographic algorithms, separated into strong and weak variants.
@@ -13,15 +13,17 @@
* The names are normalized: upper-case, no spaces, dashes or underscores.
*
* The names are inspired by the names used in real world crypto libraries.
*
*/
private module AlgorithmNames {
predicate isStrongHashingAlgorithm(string name) {
name = "DSA" or
name = "ED25519" or
name = "ES256" or name = "ECDSA256" or
name = "ES384" or name = "ECDSA384" or
name = "ES512" or name = "ECDSA512" or
name = "ES256" or
name = "ECDSA256" or
name = "ES384" or
name = "ECDSA384" or
name = "ES512" or
name = "ECDSA512" or
name = "SHA2" or
name = "SHA224" or
name = "SHA256" or
@@ -54,15 +56,21 @@ private module AlgorithmNames {
name = "RSA" or
name = "RABBIT" or
name = "BLOWFISH"
}
predicate isWeakEncryptionAlgorithm(string name) {
name = "DES" or
name = "3DES" or name = "TRIPLEDES" or name = "TDEA" or name = "TRIPLEDEA" or
name = "ARC2" or name = "RC2" or
name = "ARC4" or name = "RC4" or name = "ARCFOUR" or
name = "ARC5" or name = "RC5"
name = "3DES" or
name = "TRIPLEDES" or
name = "TDEA" or
name = "TRIPLEDEA" or
name = "ARC2" or
name = "RC2" or
name = "ARC4" or
name = "RC4" or
name = "ARCFOUR" or
name = "ARC5" or
name = "RC5"
}
predicate isStrongPasswordHashingAlgorithm(string name) {
@@ -72,51 +80,45 @@ private module AlgorithmNames {
name = "SCRYPT"
}
predicate isWeakPasswordHashingAlgorithm(string name) {
none()
}
predicate isWeakPasswordHashingAlgorithm(string name) { none() }
/**
* Normalizes `name`: upper-case, no spaces, dashes or underscores.
*
* All names of this module are in this normalized form.
*/
bindingset[name] string normalizeName(string name) {
result = name.toUpperCase().regexpReplaceAll("[-_ ]", "")
}
bindingset[name]
string normalizeName(string name) { result = name.toUpperCase().regexpReplaceAll("[-_ ]", "") }
}
private import AlgorithmNames
private import AlgorithmNames
/**
* A cryptographic algorithm.
*/
private newtype TCryptographicAlgorithm =
MkHashingAlgorithm(string name, boolean isWeak) {
(isStrongHashingAlgorithm(name) and isWeak = false) or
(isWeakHashingAlgorithm(name) and isWeak = true)
}
or
MkEncryptionAlgorithm(string name, boolean isWeak) {
(isStrongEncryptionAlgorithm(name) and isWeak = false) or
(isWeakEncryptionAlgorithm(name) and isWeak = true)
}
or
MkPasswordHashingAlgorithm(string name, boolean isWeak) {
(isStrongPasswordHashingAlgorithm(name) and isWeak = false) or
(isWeakPasswordHashingAlgorithm(name) and isWeak = true)
}
MkHashingAlgorithm(string name, boolean isWeak) {
isStrongHashingAlgorithm(name) and isWeak = false
or
isWeakHashingAlgorithm(name) and isWeak = true
} or
MkEncryptionAlgorithm(string name, boolean isWeak) {
isStrongEncryptionAlgorithm(name) and isWeak = false
or
isWeakEncryptionAlgorithm(name) and isWeak = true
} or
MkPasswordHashingAlgorithm(string name, boolean isWeak) {
isStrongPasswordHashingAlgorithm(name) and isWeak = false
or
isWeakPasswordHashingAlgorithm(name) and isWeak = true
}
/**
* A cryptographic algorithm.
*/
abstract class CryptographicAlgorithm extends TCryptographicAlgorithm {
/** Gets a textual representation of this element. */
string toString() {
result = getName()
}
string toString() { result = getName() }
/**
* Gets the name of the algorithm.
@@ -127,76 +129,46 @@ abstract class CryptographicAlgorithm extends TCryptographicAlgorithm {
* Holds if this algorithm is weak.
*/
abstract predicate isWeak();
}
/**
* A hashing algorithm such as `MD5` or `SHA512`.
*/
class HashingAlgorithm extends MkHashingAlgorithm, CryptographicAlgorithm {
string name;
boolean isWeak;
HashingAlgorithm() {
this = MkHashingAlgorithm(name, isWeak)
}
HashingAlgorithm() { this = MkHashingAlgorithm(name, isWeak) }
override string getName() {
result = name
}
override predicate isWeak() {
isWeak = true
}
override string getName() { result = name }
override predicate isWeak() { isWeak = true }
}
/**
* An encryption algorithm such as `DES` or `AES512`.
*/
class EncryptionAlgorithm extends MkEncryptionAlgorithm, CryptographicAlgorithm {
string name;
boolean isWeak;
EncryptionAlgorithm() {
this = MkEncryptionAlgorithm(name, isWeak)
}
EncryptionAlgorithm() { this = MkEncryptionAlgorithm(name, isWeak) }
override string getName() {
result = name
}
override predicate isWeak() {
isWeak = true
}
override string getName() { result = name }
override predicate isWeak() { isWeak = true }
}
/**
* A password hashing algorithm such as `PBKDF2` or `SCRYPT`.
*/
class PasswordHashingAlgorithm extends MkPasswordHashingAlgorithm, CryptographicAlgorithm {
class PasswordHashingAlgorithm extends MkPasswordHashingAlgorithm, CryptographicAlgorithm {
string name;
boolean isWeak;
PasswordHashingAlgorithm() {
this = MkPasswordHashingAlgorithm(name, isWeak)
}
PasswordHashingAlgorithm() { this = MkPasswordHashingAlgorithm(name, isWeak) }
override string getName() {
result = name
}
override string getName() { result = name }
override predicate isWeak() {
isWeak = true
}
override predicate isWeak() { isWeak = true }
}

1
python/ql/src/semmle/files/FileSystem.qll
View File

@@ -1,2 +1,3 @@
/** Provides classes for working with files and folders. */
import semmle.python.Files

185
python/ql/src/semmle/python/AstExtended.qll
View File

@@ -2,55 +2,52 @@ import python
/** Syntactic node (Class, Function, Module, Expr, Stmt or Comprehension) corresponding to a flow node */
abstract class AstNode extends AstNode_ {
/* Special comment for documentation generation.
/*
* Special comment for documentation generation.
* All subclasses of `AstNode` that represent concrete syntax should have
* a comment of the form: */
/* syntax: ... */
* a comment of the form:
*/
/* syntax: ... */
/** Gets the scope that this node occurs in */
abstract Scope getScope();
/** Gets a flow node corresponding directly to this node.
/**
* Gets a flow node corresponding directly to this node.
* NOTE: For some statements and other purely syntactic elements,
* there may not be a `ControlFlowNode` */
ControlFlowNode getAFlowNode() {
py_flow_bb_node(result, this, _, _)
}
* there may not be a `ControlFlowNode`
*/
ControlFlowNode getAFlowNode() { py_flow_bb_node(result, this, _, _) }
/** Gets the location for this AST node */
Location getLocation() {
none()
}
Location getLocation() { none() }
/** Whether this syntactic element is artificial, that is it is generated
* by the compiler and is not present in the source */
predicate isArtificial() {
none()
}
/**
* Whether this syntactic element is artificial, that is it is generated
* by the compiler and is not present in the source
*/
predicate isArtificial() { none() }
/** Gets a child node of this node in the AST. This predicate exists to aid exploration of the AST
* and other experiments. The child-parent relation may not be meaningful.
* For a more meaningful relation in terms of dependency use
* Expr.getASubExpression(), Stmt.getASubStatement(), Stmt.getASubExpression() or
* Scope.getAStmt().
*/
/**
* Gets a child node of this node in the AST. This predicate exists to aid exploration of the AST
* and other experiments. The child-parent relation may not be meaningful.
* For a more meaningful relation in terms of dependency use
* Expr.getASubExpression(), Stmt.getASubStatement(), Stmt.getASubExpression() or
* Scope.getAStmt().
*/
abstract AstNode getAChildNode();
/** Gets the parent node of this node in the AST. This predicate exists to aid exploration of the AST
* and other experiments. The child-parent relation may not be meaningful.
* For a more meaningful relation in terms of dependency use
* Expr.getASubExpression(), Stmt.getASubStatement(), Stmt.getASubExpression() or
* Scope.getAStmt() applied to the parent.
*/
AstNode getParentNode() {
result.getAChildNode() = this
}
/**
* Gets the parent node of this node in the AST. This predicate exists to aid exploration of the AST
* and other experiments. The child-parent relation may not be meaningful.
* For a more meaningful relation in terms of dependency use
* Expr.getASubExpression(), Stmt.getASubStatement(), Stmt.getASubExpression() or
* Scope.getAStmt() applied to the parent.
*/
AstNode getParentNode() { result.getAChildNode() = this }
/** Whether this contains `inner` syntactically */
predicate contains(AstNode inner) {
this.getAChildNode+() = inner
}
predicate contains(AstNode inner) { this.getAChildNode+() = inner }
/** Whether this contains `inner` syntactically and `inner` has the same scope as `this` */
predicate containsInScope(AstNode inner) {
@@ -58,133 +55,86 @@ abstract class AstNode extends AstNode_ {
this.getScope() = inner.getScope() and
not inner instanceof Scope
}
}
/* Parents */
/** Internal implementation class */
library class FunctionParent extends FunctionParent_ { }
/** Internal implementation class */
library class FunctionParent extends FunctionParent_ {
}
library class ArgumentsParent extends ArgumentsParent_ { }
/** Internal implementation class */
library class ArgumentsParent extends ArgumentsParent_ {
}
library class ExprListParent extends ExprListParent_ { }
/** Internal implementation class */
library class ExprListParent extends ExprListParent_ {
}
library class ExprContextParent extends ExprContextParent_ { }
/** Internal implementation class */
library class ExprContextParent extends ExprContextParent_ {
}
library class StmtListParent extends StmtListParent_ { }
/** Internal implementation class */
library class StmtListParent extends StmtListParent_ {
}
library class StrListParent extends StrListParent_ { }
/** Internal implementation class */
library class StrListParent extends StrListParent_ {
}
/** Internal implementation class */
library class ExprParent extends ExprParent_ {
}
library class ExprParent extends ExprParent_ { }
library class DictItem extends DictItem_, AstNode {
override string toString() {
result = DictItem_.super.toString()
}
override string toString() { result = DictItem_.super.toString() }
override AstNode getAChildNode() { none() }
override Scope getScope() { none() }
}
/** A comprehension part, the 'for a in seq' part of [ a * a for a in seq ] */
class Comprehension extends Comprehension_, AstNode {
/** Gets the scope of this comprehension */
override Scope getScope() {
/* Comprehensions exists only in Python 2 list comprehensions, so their scope is that of the list comp. */
exists(ListComp l |
this = l.getAGenerator() |
result = l.getScope()
)
exists(ListComp l | this = l.getAGenerator() | result = l.getScope())
}
override string toString() {
result = "Comprehension"
}
override string toString() { result = "Comprehension" }
override Location getLocation() {
result = Comprehension_.super.getLocation()
}
override Location getLocation() { result = Comprehension_.super.getLocation() }
override AstNode getAChildNode() {
result = this.getASubExpression()
}
override AstNode getAChildNode() { result = this.getASubExpression() }
Expr getASubExpression() {
result = this.getIter() or
result = this.getAnIf() or
result = this.getTarget()
}
}
class BytesOrStr extends BytesOrStr_ {
class BytesOrStr extends BytesOrStr_ { }
}
/** Part of a string literal formed by implicit concatenation.
/**
* Part of a string literal formed by implicit concatenation.
* For example the string literal "abc" expressed in the source as `"a" "b" "c"`
* would be composed of three `StringPart`s.
*
*/
class StringPart extends StringPart_, AstNode {
override Scope getScope() {
exists(Bytes b | this = b.getAnImplicitlyConcatenatedPart() | result = b.getScope())
or
exists(Unicode u | this = u.getAnImplicitlyConcatenatedPart() | result = u.getScope())
}
override AstNode getAChildNode() {
none()
}
override AstNode getAChildNode() { none() }
override string toString() {
result = StringPart_.super.toString()
}
override Location getLocation() {
result = StringPart_.super.getLocation()
}
override string toString() { result = StringPart_.super.toString() }
override Location getLocation() { result = StringPart_.super.getLocation() }
}
class StringPartList extends StringPartList_ {
}
class StringPartList extends StringPartList_ { }
/* **** Lists ***/
/** A parameter list */
class ParameterList extends @py_parameter_list {
Function getParent() {
py_parameter_lists(this, result)
}
Function getParent() { py_parameter_lists(this, result) }
/** Gets a parameter */
Parameter getAnItem() {
@@ -198,40 +148,23 @@ class ParameterList extends @py_parameter_list {
py_exprs(result, _, this, index)
}
string toString() {
result = "ParameterList"
}
string toString() { result = "ParameterList" }
}
/** A list of Comprehensions (for generating parts of a set, list or dictionary comprehension) */
class ComprehensionList extends ComprehensionList_ {
}
class ComprehensionList extends ComprehensionList_ { }
/** A list of expressions */
class ExprList extends ExprList_ {
/* syntax: Expr, ... */
}
library class DictItemList extends DictItemList_ { }
library class DictItemList extends DictItemList_ {
}
library class DictItemListParent extends DictItemListParent_ {
}
library class DictItemListParent extends DictItemListParent_ { }
/** A list of strings (the primitive type string not Bytes or Unicode) */
class StringList extends StringList_ {
}
class StringList extends StringList_ { }
/** A list of aliases in an import statement */
class AliasList extends AliasList_ {
}
class AliasList extends AliasList_ { }

2266
python/ql/src/semmle/python/AstGenerated.qll
View File

File diff suppressed because it is too large Load Diff

171
python/ql/src/semmle/python/Class.qll
View File

@@ -1,35 +1,40 @@
import python
/** An (artificial) expression corresponding to a class definition.
/**
* An (artificial) expression corresponding to a class definition.
* It is recommended to use `ClassDef` instead.
*/
class ClassExpr extends ClassExpr_ {
/** Gets the metaclass expression */
Expr getMetaClass() {
if major_version() = 3 then
exists(Keyword metacls | this.getAKeyword() = metacls and metacls.getArg() = "metaclass" and result = metacls.getValue())
Expr getMetaClass() {
if major_version() = 3
then
exists(Keyword metacls |
this.getAKeyword() = metacls and
metacls.getArg() = "metaclass" and
result = metacls.getValue()
)
else
exists(Assign a | a = this.getInnerScope().getAStmt() and ((Name)a.getATarget()).getId() = "__metaclass__" and result = a.getValue())
exists(Assign a |
a = this.getInnerScope().getAStmt() and
a.getATarget().(Name).getId() = "__metaclass__" and
result = a.getValue()
)
}
/** Gets the nth keyword argument of this class definition. */
override DictUnpackingOrKeyword getKeyword(int index) {
result = this.getKeywords().getItem(index)
}
/** Gets a keyword argument of this class definition. */
override DictUnpackingOrKeyword getAKeyword() {
result = this.getKeywords().getAnItem()
}
override DictUnpackingOrKeyword getAKeyword() { result = this.getKeywords().getAnItem() }
override Expr getASubExpression() {
result = this.getABase() or
result = this.getAKeyword().getValue() or
result = this.getKwargs() or
result = this.getStarargs()
result = this.getABase() or
result = this.getAKeyword().getValue() or
result = this.getKwargs() or
result = this.getStarargs()
}
Call getADecoratorCall() {
@@ -38,9 +43,7 @@ class ClassExpr extends ClassExpr_ {
}
/** Gets a decorator of this function expression */
Expr getADecorator() {
result = this.getADecoratorCall().getFunc()
}
Expr getADecorator() { result = this.getADecoratorCall().getFunc() }
override AstNode getAChildNode() {
result = this.getASubExpression()
@@ -49,151 +52,102 @@ class ClassExpr extends ClassExpr_ {
}
/** Gets a tuple (*) argument of this class definition. */
Expr getStarargs() {
result = this.getABase().(Starred).getValue()
}
Expr getStarargs() { result = this.getABase().(Starred).getValue() }
/** Gets a dictionary (**) argument of this class definition. */
Expr getKwargs() {
result = this.getAKeyword().(DictUnpacking).getValue()
}
Expr getKwargs() { result = this.getAKeyword().(DictUnpacking).getValue() }
}
/** A class statement. Note that ClassDef extends Assign as a class definition binds the newly created class */
class ClassDef extends Assign {
/* syntax: class name(...): ... */
ClassDef() {
/* This is an artificial assignment the rhs of which is a (possibly decorated) ClassExpr */
exists(ClassExpr c | this.getValue() = c or this.getValue() = c.getADecoratorCall())
}
override string toString() {
result = "ClassDef"
}
override string toString() { result = "ClassDef" }
/** Gets the class for this statement */
Class getDefinedClass() {
exists(ClassExpr c |
this.getValue() = c or this.getValue() = c.getADecoratorCall() |
exists(ClassExpr c | this.getValue() = c or this.getValue() = c.getADecoratorCall() |
result = c.getInnerScope()
)
}
override Stmt getLastStatement() {
result = this.getDefinedClass().getLastStatement()
}
override Stmt getLastStatement() { result = this.getDefinedClass().getLastStatement() }
}
/** The scope of a class. This is the scope of all the statements within the class definition */
class Class extends Class_, Scope, AstNode {
/** Use getADecorator() instead of getDefinition().getADecorator()
/**
* Use getADecorator() instead of getDefinition().getADecorator()
* Use getMetaClass() instead of getDefinition().getMetaClass()
*/
deprecated ClassExpr getDefinition() {
result = this.getParent()
}
deprecated ClassExpr getDefinition() { result = this.getParent() }
/** Gets a defined init method of this class */
Function getInitMethod() {
result.getScope() = this and result.isInitMethod()
}
Function getInitMethod() { result.getScope() = this and result.isInitMethod() }
/** Gets a method defined in this class */
Function getAMethod() {
result.getScope() = this
}
Function getAMethod() { result.getScope() = this }
override Location getLocation() {
py_scope_location(result, this)
}
override Location getLocation() { py_scope_location(result, this) }
/** Gets the scope (module, class or function) in which this class is defined */
override Scope getEnclosingScope() {
result = this.getParent().getScope()
}
override Scope getEnclosingScope() { result = this.getParent().getScope() }
/** Use getEnclosingScope() instead */
override Scope getScope() {
result = this.getParent().getScope()
}
override Scope getScope() { result = this.getParent().getScope() }
override string toString() {
result = "Class " + this.getName()
}
override string toString() { result = "Class " + this.getName() }
/** Gets the statements forming the body of this class */
override StmtList getBody() {
result = Class_.super.getBody()
}
override StmtList getBody() { result = Class_.super.getBody() }
/** Gets the nth statement in the class */
override Stmt getStmt(int index) {
result = Class_.super.getStmt(index)
}
override Stmt getStmt(int index) { result = Class_.super.getStmt(index) }
/** Gets a statement in the class */
override Stmt getAStmt() {
result = Class_.super.getAStmt()
}
override Stmt getAStmt() { result = Class_.super.getAStmt() }
/** Gets the name used to define this class */
override string getName() {
result = Class_.super.getName()
}
override string getName() { result = Class_.super.getName() }
predicate hasSideEffects() {
any()
}
predicate hasSideEffects() { any() }
/** Whether this is probably a mixin (has 'mixin' or similar in name or docstring) */
predicate isProbableMixin() {
(this.getName().toLowerCase().matches("%mixin%")
or
this.getDocString().getText().toLowerCase().matches("%mixin%")
or
this.getDocString().getText().toLowerCase().matches("%mix-in%")
(
this.getName().toLowerCase().matches("%mixin%")
or
this.getDocString().getText().toLowerCase().matches("%mixin%")
or
this.getDocString().getText().toLowerCase().matches("%mix-in%")
)
}
override AstNode getAChildNode() {
result = this.getAStmt()
}
override AstNode getAChildNode() { result = this.getAStmt() }
Expr getADecorator() {
result = this.getParent().getADecorator()
}
Expr getADecorator() { result = this.getParent().getADecorator() }
/** Gets the metaclass expression */
Expr getMetaClass() {
result = this.getParent().getMetaClass()
}
Expr getMetaClass() { result = this.getParent().getMetaClass() }
/** Gets the ClassObject corresponding to this class */
ClassObject getClassObject() {
result.getOrigin() = this.getParent()
}
ClassObject getClassObject() { result.getOrigin() = this.getParent() }
/** Gets the nth base of this class definition. */
Expr getBase(int index) {
result = this.getParent().getBase(index)
}
Expr getBase(int index) { result = this.getParent().getBase(index) }
/** Gets a base of this class definition. */
Expr getABase() {
result = this.getParent().getABase()
}
Expr getABase() { result = this.getParent().getABase() }
/** Gets the metrics for this class */
ClassMetrics getMetrics() {
result = this
}
ClassMetrics getMetrics() { result = this }
/** Gets the qualified name for this class.
/**
* Gets the qualified name for this class.
* Should return the same name as the `__qualname__` attribute on classes in Python 3.
*/
string getQualifiedName() {
@@ -202,20 +156,13 @@ class Class extends Class_, Scope, AstNode {
exists(string enclosing_name |
enclosing_name = this.getScope().(Function).getQualifiedName()
or
enclosing_name = this.getScope().(Class).getQualifiedName() |
enclosing_name = this.getScope().(Class).getQualifiedName()
|
result = enclosing_name + "." + this.getName()
)
}
override
predicate containsInScope(AstNode inner) {
Scope.super.containsInScope(inner)
}
override
predicate contains(AstNode inner) {
Scope.super.contains(inner)
}
override predicate containsInScope(AstNode inner) { Scope.super.containsInScope(inner) }
override predicate contains(AstNode inner) { Scope.super.contains(inner) }
}

81
python/ql/src/semmle/python/Comment.qll
View File

@@ -2,38 +2,28 @@ import python
/** A source code comment */
class Comment extends @py_comment {
/** Gets the full text of the comment including the leading '#' */
string getText() {
py_comments(this, result, _)
}
string getText() { py_comments(this, result, _) }
/** Gets the contents of the comment excluding the leading '#' */
string getContents() {
result = this.getText().suffix(1)
}
string getContents() { result = this.getText().suffix(1) }
Location getLocation() {
py_comments(this, _, result)
Location getLocation() { py_comments(this, _, result) }
}
string toString() { result = "Comment " + this.getText() }
string toString() {
result = "Comment " + this.getText()
}
/** Gets this immediately following comment.
/**
* Gets this immediately following comment.
* Blanks line are allowed between this comment and the following comment,
* but code or other comments are not.
*/
Comment getFollowing() {
exists(File f, int n |
this.file_line(f, n) |
result.file_line(f, n+1)
exists(File f, int n | this.file_line(f, n) |
result.file_line(f, n + 1)
or
result.file_line(f, n+2) and f.emptyLine(n+1)
result.file_line(f, n + 2) and f.emptyLine(n + 1)
or
result.file_line(f, n+3) and f.emptyLine(n+2) and f.emptyLine(n+1)
result.file_line(f, n + 3) and f.emptyLine(n + 2) and f.emptyLine(n + 1)
)
}
@@ -41,47 +31,34 @@ class Comment extends @py_comment {
this.getLocation().getFile() = f and
this.getLocation().getStartLine() = n
}
}
private predicate comment_block_part(Comment start, Comment part, int i) {
not exists(Comment prev | prev.getFollowing() = part) and
exists(Comment following | part.getFollowing() = following) and
start = part and i = 1
start = part and
i = 1
or
exists(Comment prev |
comment_block_part(start, prev, i-1) and
comment_block_part(start, prev, i - 1) and
part = prev.getFollowing()
)
}
/** A block of consecutive comments */
class CommentBlock extends @py_comment {
CommentBlock() { comment_block_part(this, _, _) }
CommentBlock() {
comment_block_part(this, _, _)
}
private Comment last() { comment_block_part(this, result, this.length()) }
private Comment last() {
comment_block_part(this, result, this.length())
}
string toString() {
result = "Comment block"
}
string toString() { result = "Comment block" }
/** The length of this comment block (in comments) */
int length() {
result = max(int i | comment_block_part(this, _, i))
}
int length() { result = max(int i | comment_block_part(this, _, i)) }
predicate hasLocationInfo(string filepath, int bl, int bc, int el, int ec) {
((Comment)this).getLocation().hasLocationInfo(filepath, bl, bc, _, _)
and
exists(Comment end |
end = this.last() |
end.getLocation().hasLocationInfo(_, _, _, el, ec)
)
this.(Comment).getLocation().hasLocationInfo(filepath, bl, bc, _, _) and
exists(Comment end | end = this.last() | end.getLocation().hasLocationInfo(_, _, _, el, ec))
}
predicate contains(Comment c) {
@@ -91,20 +68,18 @@ class CommentBlock extends @py_comment {
}
string getContents() {
result = concat(Comment c,int i |
comment_block_part(this, c, i) or this = c and i = 0 |
c.getContents() order by i
)
result =
concat(Comment c, int i |
comment_block_part(this, c, i)
or
this = c and i = 0
|
c.getContents() order by i
)
}
}
/** A type-hint comment. Any comment that starts with `# type:` */
class TypeHintComment extends Comment {
TypeHintComment() {
this.getText().regexpMatch("# +type:.*")
}
TypeHintComment() { this.getText().regexpMatch("# +type:.*") }
}

235
python/ql/src/semmle/python/Comparisons.qll
View File

@@ -1,48 +1,64 @@
import python
/** A class representing the six comparison operators, ==, !=, <, <=, > and >=. */
class CompareOp extends int {
CompareOp() {
this in [1..6]
}
CompareOp() { this in [1 .. 6] }
/** Gets the logical inverse operator */
CompareOp invert() {
this = eq() and result = ne() or
this = ne() and result = eq() or
this = lt() and result = ge() or
this = gt() and result = le() or
this = le() and result = gt() or
this = eq() and result = ne()
or
this = ne() and result = eq()
or
this = lt() and result = ge()
or
this = gt() and result = le()
or
this = le() and result = gt()
or
this = ge() and result = lt()
}
/** Gets the reverse operator (swapping the operands) */
/** Gets the reverse operator (swapping the operands) */
CompareOp reverse() {
this = eq() and result = eq() or
this = ne() and result = ne() or
this = lt() and result = gt() or
this = gt() and result = lt() or
this = le() and result = ge() or
this = eq() and result = eq()
or
this = ne() and result = ne()
or
this = lt() and result = gt()
or
this = gt() and result = lt()
or
this = le() and result = ge()
or
this = ge() and result = le()
}
string repr() {
this = eq() and result = "==" or
this = ne() and result = "!=" or
this = lt() and result = "<" or
this = gt() and result = ">" or
this = le() and result = "<=" or
this = eq() and result = "=="
or
this = ne() and result = "!="
or
this = lt() and result = "<"
or
this = gt() and result = ">"
or
this = le() and result = "<="
or
this = ge() and result = ">="
}
predicate forOp(Cmpop op) {
op instanceof Eq and this = eq() or
op instanceof NotEq and this = ne() or
op instanceof Lt and this = lt() or
op instanceof LtE and this = le() or
op instanceof Gt and this = gt() or
op instanceof Eq and this = eq()
or
op instanceof NotEq and this = ne()
or
op instanceof Lt and this = lt()
or
op instanceof LtE and this = le()
or
op instanceof Gt and this = gt()
or
op instanceof GtE and this = ge()
}
@@ -52,31 +68,37 @@ class CompareOp extends int {
or
result = this.invert() and isTrue = false
}
}
CompareOp eq() { result = 1 }
CompareOp ne() { result = 2 }
CompareOp lt() { result = 3 }
CompareOp le() { result = 4 }
CompareOp gt() { result = 5 }
CompareOp ge() { result = 6 }
/* Workaround precision limits in floating point numbers */
bindingset[x] private predicate ok_magnitude(float x) {
x > -9007199254740992.0 // -2**53
and
bindingset[x]
private predicate ok_magnitude(float x) {
x > -9007199254740992.0 and // -2**53
x < 9007199254740992.0 // 2**53
}
bindingset[x,y] private float add(float x, float y) {
bindingset[x, y]
private float add(float x, float y) {
ok_magnitude(x) and
ok_magnitude(y) and
ok_magnitude(result) and
result = x + y
}
bindingset[x,y] private float sub(float x, float y) {
bindingset[x, y]
private float sub(float x, float y) {
ok_magnitude(x) and
ok_magnitude(y) and
ok_magnitude(result) and
@@ -84,7 +106,9 @@ bindingset[x,y] private float sub(float x, float y) {
}
/** Normalise equality cmp into the form `left op right + k`. */
private predicate test(ControlFlowNode cmp, ControlFlowNode left, CompareOp op, ControlFlowNode right, float k) {
private predicate test(
ControlFlowNode cmp, ControlFlowNode left, CompareOp op, ControlFlowNode right, float k
) {
simple_test(cmp, left, op, right) and k = 0
or
add_test(cmp, left, op, right, k)
@@ -98,82 +122,106 @@ private predicate test(ControlFlowNode cmp, ControlFlowNode left, CompareOp op,
/** Various simple tests in left op right + k form. */
private predicate simple_test(CompareNode cmp, ControlFlowNode l, CompareOp cmpop, ControlFlowNode r) {
exists(Cmpop op |
cmp.operands(l, op, r) and cmpop.forOp(op)
)
exists(Cmpop op | cmp.operands(l, op, r) and cmpop.forOp(op))
}
private predicate add_test_left(CompareNode cmp, ControlFlowNode l, CompareOp op, ControlFlowNode r, float k) {
exists(BinaryExprNode lhs, float c, float x, Num n |
private predicate add_test_left(
CompareNode cmp, ControlFlowNode l, CompareOp op, ControlFlowNode r, float k
) {
exists(BinaryExprNode lhs, float c, float x, Num n |
lhs.getNode().getOp() instanceof Add and
test(cmp, lhs, op, r, c) and x = n.getN().toFloat() and k = sub(c, x) |
test(cmp, lhs, op, r, c) and
x = n.getN().toFloat() and
k = sub(c, x)
|
l = lhs.getLeft() and n = lhs.getRight().getNode()
or
l = lhs.getRight() and n = lhs.getLeft().getNode()
)
}
private predicate add_test_right(CompareNode cmp, ControlFlowNode l, CompareOp op, ControlFlowNode r, float k) {
private predicate add_test_right(
CompareNode cmp, ControlFlowNode l, CompareOp op, ControlFlowNode r, float k
) {
exists(BinaryExprNode rhs, float c, float x, Num n |
rhs.getNode().getOp() instanceof Add and
test(cmp, l, op, rhs, c) and x = n.getN().toFloat() and k = add(c, x) |
test(cmp, l, op, rhs, c) and
x = n.getN().toFloat() and
k = add(c, x)
|
r = rhs.getLeft() and n = rhs.getRight().getNode()
or
r = rhs.getRight() and n = rhs.getLeft().getNode()
)
}
/* left + x op right + c => left op right + (c-x)
left op (right + x) + c => left op right + (c+x) */
private predicate add_test(CompareNode cmp, ControlFlowNode l, CompareOp op, ControlFlowNode r, float k) {
/*
* left + x op right + c => left op right + (c-x)
* left op (right + x) + c => left op right + (c+x)
*/
private predicate add_test(
CompareNode cmp, ControlFlowNode l, CompareOp op, ControlFlowNode r, float k
) {
add_test_left(cmp, l, op, r, k)
or
add_test_right(cmp, l, op ,r, k)
add_test_right(cmp, l, op, r, k)
}
private predicate subtract_test_left(CompareNode cmp, ControlFlowNode l, CompareOp op, ControlFlowNode r, float k) {
exists(BinaryExprNode lhs, float c, float x, Num n |
private predicate subtract_test_left(
CompareNode cmp, ControlFlowNode l, CompareOp op, ControlFlowNode r, float k
) {
exists(BinaryExprNode lhs, float c, float x, Num n |
lhs.getNode().getOp() instanceof Sub and
test(cmp, lhs, op, r, c) and
l = lhs.getLeft() and n = lhs.getRight().getNode() and
x = n.getN().toFloat() |
l = lhs.getLeft() and
n = lhs.getRight().getNode() and
x = n.getN().toFloat()
|
k = add(c, x)
)
}
private predicate subtract_test_right(CompareNode cmp, ControlFlowNode l, CompareOp op, ControlFlowNode r, float k) {
private predicate subtract_test_right(
CompareNode cmp, ControlFlowNode l, CompareOp op, ControlFlowNode r, float k
) {
exists(BinaryExprNode rhs, float c, float x, Num n |
rhs.getNode().getOp() instanceof Sub and
test(cmp, l, op, rhs, c) and
r = rhs.getRight() and n = rhs.getLeft().getNode() and
x = n.getN().toFloat() |
r = rhs.getRight() and
n = rhs.getLeft().getNode() and
x = n.getN().toFloat()
|
k = sub(c, x)
)
}
/* left - x op right + c => left op right + (c+x)
left op (right - x) + c => left op right + (c-x) */
private predicate subtract_test(CompareNode cmp, ControlFlowNode l, CompareOp op, ControlFlowNode r, float k) {
/*
* left - x op right + c => left op right + (c+x)
* left op (right - x) + c => left op right + (c-x)
*/
private predicate subtract_test(
CompareNode cmp, ControlFlowNode l, CompareOp op, ControlFlowNode r, float k
) {
subtract_test_left(cmp, l, op, r, k)
or
subtract_test_right(cmp, l, op, r, k)
}
private predicate not_test(UnaryExprNode u, ControlFlowNode l, CompareOp op, ControlFlowNode r, float k) {
u.getNode().getOp() instanceof Not
and
private predicate not_test(
UnaryExprNode u, ControlFlowNode l, CompareOp op, ControlFlowNode r, float k
) {
u.getNode().getOp() instanceof Not and
test(u.getOperand(), l, op.invert(), r, k)
}
/** A comparison which can be simplified to the canonical form `x OP y + k` where `x` and `y` are `ControlFlowNode`s,
/**
* A comparison which can be simplified to the canonical form `x OP y + k` where `x` and `y` are `ControlFlowNode`s,
* `k` is a floating point constant and `OP` is one of `<=`, `>`, `==` or `!=`.
*/
class Comparison extends ControlFlowNode {
Comparison() {
test(this, _, _, _, _)
}
Comparison() { test(this, _, _, _, _) }
/** Whether this condition tests `l op r + k` */
predicate tests(ControlFlowNode l, CompareOp op, ControlFlowNode r, float k) {
@@ -182,14 +230,14 @@ class Comparison extends ControlFlowNode {
/** Whether this condition tests `l op k` */
predicate tests(ControlFlowNode l, CompareOp op, float k) {
exists(ControlFlowNode r, float x, float c |
test(this, l, op, r, c) |
exists(ControlFlowNode r, float x, float c | test(this, l, op, r, c) |
x = r.getNode().(Num).getN().toFloat() and
k = add(c, x)
)
}
/* The following predicates determine whether this test, when its result is `thisIsTrue`,
/*
* The following predicates determine whether this test, when its result is `thisIsTrue`,
* is equivalent to the predicate `v OP k` or `v1 OP v2 + k`.
* For example, the test `x <= y` being false, is equivalent to the predicate `x > y`.
*/
@@ -242,7 +290,8 @@ class Comparison extends ControlFlowNode {
this.tests(v1.getAUse(), ge().conditional(thisIsTrue), v2.getAUse(), k)
}
/** Whether the result of this comparison being `thisIsTrue` implies that the result of `that` is `isThatTrue`.
/**
* Whether the result of this comparison being `thisIsTrue` implies that the result of `that` is `isThatTrue`.
* In other words, does the predicate that is equivalent to the result of `this` being `thisIsTrue`
* imply the predicate that is equivalent to the result of `that` being `thatIsTrue`.
* For example, assume that there are two tests, which when normalised have the form `x < y` and `x > y + 1`.
@@ -315,7 +364,7 @@ class Comparison extends ControlFlowNode {
this.equivalentToGt(thisIsTrue, v, k1) and
that.equivalentToGtEq(thatIsTrue, v, k2) and
ge(k1, k2)
or
or
/* `v >= k1` => `v > k2` iff k1 > k2 */
this.equivalentToGtEq(thisIsTrue, v, k1) and
that.equivalentToGt(thatIsTrue, v, k2) and
@@ -402,7 +451,6 @@ class Comparison extends ControlFlowNode {
ge(k1, k2)
)
}
}
/* Work around differences in floating-point comparisons between Python and QL */
@@ -412,41 +460,27 @@ private predicate is_zero(float x) {
x = -0.0
}
bindingset[x,y] private predicate lt(float x, float y) {
if is_zero(x) then
y > 0
else
x < y
}
bindingset[x, y]
private predicate lt(float x, float y) { if is_zero(x) then y > 0 else x < y }
bindingset[x,y] private predicate eq(float x, float y) {
if is_zero(x) then
is_zero(y)
else
x = y
}
bindingset[x, y]
private predicate eq(float x, float y) { if is_zero(x) then is_zero(y) else x = y }
bindingset[x,y] private predicate gt(float x, float y) {
lt(y, x)
}
bindingset[x, y]
private predicate gt(float x, float y) { lt(y, x) }
bindingset[x,y] private predicate le(float x, float y) {
lt(x, y) or eq(x, y)
}
bindingset[x, y]
private predicate le(float x, float y) { lt(x, y) or eq(x, y) }
bindingset[x,y] private predicate ge(float x, float y) {
lt(y, x) or eq(x, y)
}
bindingset[x, y]
private predicate ge(float x, float y) { lt(y, x) or eq(x, y) }
/** A basic block which terminates in a condition, splitting the subsequent control flow,
/**
* A basic block which terminates in a condition, splitting the subsequent control flow,
* in which the condition is an instance of `Comparison`
*/
class ComparisonControlBlock extends ConditionBlock {
ComparisonControlBlock() {
this.getLastNode() instanceof Comparison
}
ComparisonControlBlock() { this.getLastNode() instanceof Comparison }
/** Whether this conditional guard determines that, in block `b`, `l == r + k` if `eq` is true, or `l != r + k` if `eq` is false, */
predicate controls(ControlFlowNode l, CompareOp op, ControlFlowNode r, float k, BasicBlock b) {
@@ -466,16 +500,13 @@ class ComparisonControlBlock extends ConditionBlock {
)
}
Comparison getTest() {
this.getLastNode() = result
}
Comparison getTest() { this.getLastNode() = result }
/** Whether this conditional guard implies that, in block `b`, the result of `that` is `thatIsTrue` */
predicate impliesThat(BasicBlock b, Comparison that, boolean thatIsTrue) {
exists(boolean controlSense |
this.controls(b, controlSense) and
this.controls(b, controlSense) and
this.getTest().impliesThat(controlSense, that, thatIsTrue)
)
}
}

84
python/ql/src/semmle/python/Comprehensions.qll
View File

@@ -1,8 +1,7 @@
import python
/** Base class for list, set and dictionary comprehensions, and generator expressions. */
abstract class Comp extends Expr {
abstract Function getFunction();
/** Gets the iteration variable for the nth innermost generator of this list comprehension */
@@ -13,35 +12,30 @@ abstract class Comp extends Expr {
private For getNthInnerLoop(int n) {
n = 0 and result = this.getFunction().getStmt(0)
or
result = this.getNthInnerLoop(n-1).getStmt(0)
result = this.getNthInnerLoop(n - 1).getStmt(0)
}
/** Gets the iteration variable for a generator of this list comprehension */
Variable getAnIterationVariable() {
result = this.getIterationVariable(_)
}
Variable getAnIterationVariable() { result = this.getIterationVariable(_) }
/** Gets the scope in which the body of this list comprehension evaluates. */
Scope getEvaluatingScope() {
result = this.getFunction()
}
Scope getEvaluatingScope() { result = this.getFunction() }
/** Gets the expression for elements of this comprehension. */
Expr getElt() {
exists(Yield yield, Stmt body |
result = yield.getValue() and
body = this.getNthInnerLoop(_).getAStmt() |
body = this.getNthInnerLoop(_).getAStmt()
|
yield = body.(ExprStmt).getValue()
or
yield = body.(If).getStmt(0).(ExprStmt).getValue()
)
}
}
/** A list comprehension, such as `[ chr(x) for x in range(ord('A'), ord('Z')+1) ]` */
class ListComp extends ListComp_, Comp {
override Expr getASubExpression() {
result = this.getAGenerator().getASubExpression() or
result = this.getElt() or
@@ -54,9 +48,7 @@ class ListComp extends ListComp_, Comp {
result = this.getFunction()
}
override predicate hasSideEffects() {
any()
}
override predicate hasSideEffects() { any() }
/** Gets the scope in which the body of this list comprehension evaluates. */
override Scope getEvaluatingScope() {
@@ -66,89 +58,53 @@ class ListComp extends ListComp_, Comp {
}
/** Gets the iteration variable for the nth innermost generator of this list comprehension */
override Variable getIterationVariable(int n) {
result = Comp.super.getIterationVariable(n)
}
override Variable getIterationVariable(int n) { result = Comp.super.getIterationVariable(n) }
override Function getFunction() {
result = ListComp_.super.getFunction()
}
override Function getFunction() { result = ListComp_.super.getFunction() }
override string toString() {
result = ListComp_.super.toString()
}
override Expr getElt() {
result = Comp.super.getElt()
}
override string toString() { result = ListComp_.super.toString() }
override Expr getElt() { result = Comp.super.getElt() }
}
/** A set comprehension such as `{ v for v in "0123456789" }` */
class SetComp extends SetComp_, Comp {
override Expr getASubExpression() {
result = this.getIterable()
}
override Expr getASubExpression() { result = this.getIterable() }
override AstNode getAChildNode() {
result = this.getASubExpression() or
result = this.getFunction()
}
override predicate hasSideEffects() {
any()
}
override Function getFunction() {
result = SetComp_.super.getFunction()
}
override predicate hasSideEffects() { any() }
override Function getFunction() { result = SetComp_.super.getFunction() }
}
/** A dictionary comprehension, such as `{ k:v for k, v in enumerate("0123456789") }` */
class DictComp extends DictComp_, Comp {
override Expr getASubExpression() {
result = this.getIterable()
}
override Expr getASubExpression() { result = this.getIterable() }
override AstNode getAChildNode() {
result = this.getASubExpression() or
result = this.getFunction()
}
override predicate hasSideEffects() {
any()
}
override Function getFunction() {
result = DictComp_.super.getFunction()
}
override predicate hasSideEffects() { any() }
override Function getFunction() { result = DictComp_.super.getFunction() }
}
/** A generator expression, such as `(var for var in iterable)` */
class GeneratorExp extends GeneratorExp_, Comp {
override Expr getASubExpression() {
result = this.getIterable()
}
override Expr getASubExpression() { result = this.getIterable() }
override AstNode getAChildNode() {
result = this.getASubExpression() or
result = this.getFunction()
}
override predicate hasSideEffects() {
any()
}
override Function getFunction() {
result = GeneratorExp_.super.getFunction()
}
override predicate hasSideEffects() { any() }
override Function getFunction() { result = GeneratorExp_.super.getFunction() }
}

17
python/ql/src/semmle/python/Constants.qll
View File

@@ -8,30 +8,25 @@ int major_version() {
or
not explicit_major_version(_) and
/* If there is more than one version, prefer 2 for backwards compatibilty */
(
if py_flags_versioned("version.major", "2", "2") then
result = 2
else
result = 3
)
(if py_flags_versioned("version.major", "2", "2") then result = 2 else result = 3)
}
/** the Python minor version number */
int minor_version() {
exists(string v | py_flags_versioned("version.minor", v, major_version().toString()) |
result = v.toInt())
result = v.toInt()
)
}
/** the Python micro version number */
int micro_version() {
exists(string v | py_flags_versioned("version.micro", v, major_version().toString()) |
result = v.toInt())
result = v.toInt()
)
}
private predicate explicit_major_version(int v) {
exists(string version |
py_flags_versioned("language.version", version, _) |
exists(string version | py_flags_versioned("language.version", version, _) |
version.charAt(0) = "2" and v = 2
or
version.charAt(0) = "3" and v = 3

627
python/ql/src/semmle/python/Exprs.qll
View File

File diff suppressed because it is too large Load Diff

244
python/ql/src/semmle/python/Files.qll
View File

@@ -1,22 +1,14 @@
import python
/** A file */
class File extends Container {
File() {
files(this, _, _, _, _)
}
File() { files(this, _, _, _, _) }
/** DEPRECATED: Use `getAbsolutePath` instead. */
deprecated override string getName() {
result = this.getAbsolutePath()
}
deprecated override string getName() { result = this.getAbsolutePath() }
/** DEPRECATED: Use `getAbsolutePath` instead. */
deprecated string getFullName() {
result = this.getAbsolutePath()
}
deprecated string getFullName() { result = this.getAbsolutePath() }
predicate hasLocationInfo(string filepath, int bl, int bc, int el, int ec) {
this.getAbsolutePath() = filepath and bl = 0 and bc = 0 and el = 0 and ec = 0
@@ -30,8 +22,7 @@ class File extends Container {
/** Gets a short name for this file (just the file name) */
string getShortName() {
exists(string simple, string ext | files(this, _, simple, ext, _) |
result = simple + ext)
exists(string simple, string ext | files(this, _, simple, ext, _) | result = simple + ext)
}
private int lastLine() {
@@ -40,27 +31,21 @@ class File extends Container {
/** Whether line n is empty (it contains neither code nor comment). */
predicate emptyLine(int n) {
n in [0..this.lastLine()]
and
n in [0 .. this.lastLine()] and
not occupied_line(this, n)
}
string getSpecifiedEncoding() {
exists(Comment c, Location l |
l = c.getLocation() and l.getFile() = this |
exists(Comment c, Location l | l = c.getLocation() and l.getFile() = this |
l.getStartLine() < 3 and
result = c.getText().regexpCapture(".*coding[:=]\\s*([-\\w.]+).*", 1)
)
}
override string getAbsolutePath() {
files(this, result, _, _, _)
}
override string getAbsolutePath() { files(this, result, _, _, _) }
/** Gets the URL of this file. */
override string getURL() {
result = "file://" + this.getAbsolutePath() + ":0:0:0:0"
}
override string getURL() { result = "file://" + this.getAbsolutePath() + ":0:0:0:0" }
override Container getImportRoot(int n) {
/* File stem must be a legal Python identifier */
@@ -68,56 +53,40 @@ class File extends Container {
result = this.getParent().getImportRoot(n)
}
/** Gets the contents of this file as a string.
/**
* Gets the contents of this file as a string.
* This will only work for those non-python files that
* are specified to be extracted.
*/
string getContents() {
file_contents(this, result)
}
string getContents() { file_contents(this, result) }
}
private predicate occupied_line(File f, int n) {
exists(Location l |
l.getFile() = f |
exists(Location l | l.getFile() = f |
l.getStartLine() = n
or
exists(StrConst s | s.getLocation() = l |
n in [l.getStartLine() .. l.getEndLine()]
)
)
exists(StrConst s | s.getLocation() = l | n in [l.getStartLine() .. l.getEndLine()])
)
}
/** A folder (directory) */
class Folder extends Container {
Folder() {
folders(this, _, _)
}
Folder() { folders(this, _, _) }
/** DEPRECATED: Use `getAbsolutePath` instead. */
deprecated override string getName() {
result = this.getAbsolutePath()
}
deprecated override string getName() { result = this.getAbsolutePath() }
/** DEPRECATED: Use `getBaseName` instead. */
deprecated string getSimple() {
folders(this, _, result)
}
deprecated string getSimple() { folders(this, _, result) }
predicate hasLocationInfo(string filepath, int bl, int bc, int el, int ec) {
this.getAbsolutePath() = filepath and bl = 0 and bc = 0 and el = 0 and ec = 0
}
override string getAbsolutePath() {
folders(this, result, _)
}
override string getAbsolutePath() { folders(this, result, _) }
/** Gets the URL of this folder. */
override string getURL() {
result = "folder://" + this.getAbsolutePath()
}
override string getURL() { result = "folder://" + this.getAbsolutePath() }
override Container getImportRoot(int n) {
this.isImportRoot(n) and result = this
@@ -126,32 +95,24 @@ class Folder extends Container {
this.getBaseName().regexpMatch("[^\\d\\W]\\w*") and
result = this.getParent().getImportRoot(n)
}
}
/**
/**
* A container is an abstract representation of a file system object that can
* hold elements of interest.
* hold elements of interest.
*/
abstract class Container extends @container {
Container getParent() {
containerparent(result, this)
}
Container getParent() { containerparent(result, this) }
/** Gets a child of this container */
deprecated Container getChild() {
containerparent(this, result)
}
deprecated Container getChild() { containerparent(this, result) }
/**
* Gets a textual representation of the path of this container.
*
* This is the absolute path of the container.
*/
string toString() {
result = this.getAbsolutePath()
}
string toString() { result = this.getAbsolutePath() }
/** Gets the name of this container */
abstract string getName();
@@ -165,8 +126,9 @@ abstract class Container extends @container {
* if the root folder is not a reflexive, transitive parent of this container.
*/
string getRelativePath() {
exists (string absPath, string pref |
absPath = this.getAbsolutePath() and sourceLocationPrefix(pref) |
exists(string absPath, string pref |
absPath = this.getAbsolutePath() and sourceLocationPrefix(pref)
|
absPath = pref and result = ""
or
absPath = pref.regexpReplaceAll("/$", "") + "/" + result and
@@ -175,36 +137,28 @@ abstract class Container extends @container {
}
/** Whether this file or folder is part of the standard library */
predicate inStdlib() {
this.inStdlib(_, _)
}
predicate inStdlib() { this.inStdlib(_, _) }
/** Whether this file or folder is part of the standard library
/**
* Whether this file or folder is part of the standard library
* for version `major.minor`
*/
predicate inStdlib(int major, int minor) {
exists(Module m |
m.getPath() = this and
m.getPath() = this and
m.inStdLib(major, minor)
)
}
/* Standard cross-language API */
/** Gets a file or sub-folder in this container. */
Container getAChildContainer() {
containerparent(this, result)
}
Container getAChildContainer() { containerparent(this, result) }
/** Gets a file in this container. */
File getAFile() {
result = this.getAChildContainer()
}
File getAFile() { result = this.getAChildContainer() }
/** Gets a sub-folder in this container. */
Folder getAFolder() {
result = this.getAChildContainer()
}
Folder getAFolder() { result = this.getAChildContainer() }
/**
* Gets the absolute, canonical path of this container, using forward slashes
@@ -276,9 +230,7 @@ abstract class Container extends @container {
* <tr><td>"/tmp/x.tar.gz"</td><td>"gz"</td></tr>
* </table>
*/
string getExtension() {
result = getAbsolutePath().regexpCapture(".*/([^/]*?)(\\.([^.]*))?", 3)
}
string getExtension() { result = getAbsolutePath().regexpCapture(".*/([^/]*?)(\\.([^.]*))?", 3) }
/**
* Gets the stem of this container, that is, the prefix of its base name up to
@@ -297,9 +249,7 @@ abstract class Container extends @container {
* <tr><td>"/tmp/x.tar.gz"</td><td>"x.tar"</td></tr>
* </table>
*/
string getStem() {
result = getAbsolutePath().regexpCapture(".*/([^/]*?)(?:\\.([^.]*))?", 1)
}
string getStem() { result = getAbsolutePath().regexpCapture(".*/([^/]*?)(?:\\.([^.]*))?", 1) }
File getFile(string baseName) {
result = this.getAFile() and
@@ -311,9 +261,7 @@ abstract class Container extends @container {
result.getBaseName() = baseName
}
Container getParentContainer() {
this = result.getAChildContainer()
}
Container getParentContainer() { this = result.getAChildContainer() }
Container getChildContainer(string baseName) {
result = this.getAChildContainer() and
@@ -328,32 +276,31 @@ abstract class Container extends @container {
abstract string getURL();
/** Holds if this folder is on the import path. */
predicate isImportRoot() {
this.isImportRoot(_)
}
predicate isImportRoot() { this.isImportRoot(_) }
/** Holds if this folder is on the import path, at index `n` in the list of
/**
* Holds if this folder is on the import path, at index `n` in the list of
* paths. The list of paths is composed of the paths passed to the extractor and
* `sys.path`. */
predicate isImportRoot(int n) {
this.getName() = import_path_element(n)
}
* `sys.path`.
*/
predicate isImportRoot(int n) { this.getName() = import_path_element(n) }
/** Holds if this folder is the root folder for the standard library. */
predicate isStdLibRoot(int major, int minor) {
major = major_version() and minor = minor_version() and
major = major_version() and
minor = minor_version() and
this.isStdLibRoot()
}
/** Holds if this folder is the root folder for the standard library. */
predicate isStdLibRoot() {
/* Look for a standard lib module and find its import path
/*
* Look for a standard lib module and find its import path
* We use `os` as it is the most likely to be imported and
* `tty` because it is small for testing.
*/
exists(Module m |
m.getName() = "os" or m.getName() = "tty"
|
exists(Module m | m.getName() = "os" or m.getName() = "tty" |
m.getFile().getImportRoot() = this
)
}
@@ -371,7 +318,6 @@ abstract class Container extends @container {
/** Gets the path element from which this container would be loaded, given the index into the list of possible paths `n`. */
abstract Container getImportRoot(int n);
}
private string import_path_element(int n) {
@@ -379,53 +325,46 @@ private string import_path_element(int n) {
path = get_path("extractor.path") and k = 0
or
path = get_path("sys.path") and k = count(get_path("extractor.path").splitAt(pathsep))
|
|
py_flags_versioned("os.pathsep", pathsep, _) and
result = path.splitAt(pathsep, n-k).replaceAll("\\", "/")
result = path.splitAt(pathsep, n - k).replaceAll("\\", "/")
)
}
private string get_path(string name) {
py_flags_versioned(name, result, _)
}
private string get_path(string name) { py_flags_versioned(name, result, _) }
class Location extends @location {
/** Gets the file for this location */
File getFile() {
result = this.getPath()
}
File getFile() { result = this.getPath() }
private Container getPath() {
locations_default(this, result, _, _, _, _)
or
exists(Module m | locations_ast(this, m, _, _, _, _) |
result = m.getPath()
)
exists(Module m | locations_ast(this, m, _, _, _, _) | result = m.getPath())
}
/** Gets the start line of this location */
int getStartLine() {
locations_default(this, _, result, _, _, _)
or locations_ast(this,_,result,_,_,_)
locations_default(this, _, result, _, _, _) or
locations_ast(this, _, result, _, _, _)
}
/** Gets the start column of this location */
int getStartColumn() {
locations_default(this, _, _, result, _, _)
or locations_ast(this, _, _, result, _, _)
locations_default(this, _, _, result, _, _) or
locations_ast(this, _, _, result, _, _)
}
/** Gets the end line of this location */
int getEndLine() {
locations_default(this, _, _, _, result, _)
or locations_ast(this, _, _, _, result, _)
locations_default(this, _, _, _, result, _) or
locations_ast(this, _, _, _, result, _)
}
/** Gets the end column of this location */
int getEndColumn() {
locations_default(this, _, _, _, _, result)
or locations_ast(this, _, _, _, _, result)
locations_default(this, _, _, _, _, result) or
locations_ast(this, _, _, _, _, result)
}
string toString() {
@@ -436,20 +375,20 @@ class Location extends @location {
exists(File f | f.getAbsolutePath() = filepath |
locations_default(this, f, bl, bc, el, ec)
or
exists(Module m | m.getFile() = f |
locations_ast(this, m, bl, bc, el, ec))
)
exists(Module m | m.getFile() = f | locations_ast(this, m, bl, bc, el, ec))
)
}
}
/** A non-empty line in the source code */
class Line extends @py_line {
predicate hasLocationInfo(string filepath, int bl, int bc, int el, int ec) {
exists(Module m | m.getFile().getAbsolutePath() = filepath and
el = bl and bc = 1 and
py_line_lengths(this, m, bl, ec))
exists(Module m |
m.getFile().getAbsolutePath() = filepath and
el = bl and
bc = 1 and
py_line_lengths(this, m, bl, ec)
)
}
string toString() {
@@ -459,58 +398,37 @@ class Line extends @py_line {
}
/** Gets the line number of this line */
int getLineNumber() {
py_line_lengths(this, _, result, _)
}
int getLineNumber() { py_line_lengths(this, _, result, _) }
/** Gets the length of this line */
int getLength() {
py_line_lengths(this, _, _, result)
}
int getLength() { py_line_lengths(this, _, _, result) }
/** Gets the file for this line */
Module getModule() {
py_line_lengths(this, result, _, _)
}
Module getModule() { py_line_lengths(this, result, _, _) }
}
/**
/**
* A syntax error. Note that if there is a syntax error in a module,
* much information about that module will be lost
* much information about that module will be lost
*/
class SyntaxError extends Location {
SyntaxError() { py_syntax_error_versioned(this, _, major_version().toString()) }
SyntaxError() {
py_syntax_error_versioned(this, _, major_version().toString())
}
override string toString() {
result = "Syntax Error"
}
override string toString() { result = "Syntax Error" }
/** Gets the message corresponding to this syntax error */
string getMessage() {
py_syntax_error_versioned(this, result, major_version().toString())
}
string getMessage() { py_syntax_error_versioned(this, result, major_version().toString()) }
}
/**
/**
* An encoding error. Note that if there is an encoding error in a module,
* much information about that module will be lost
* much information about that module will be lost
*/
class EncodingError extends SyntaxError {
EncodingError() {
/* Leave spaces around 'decode' in unlikely event it occurs as a name in a syntax error */
this.getMessage().toLowerCase().matches("% decode %")
}
override string toString() {
result = "Encoding Error"
}
override string toString() { result = "Encoding Error" }
}

747
python/ql/src/semmle/python/Flow.qll
View File

File diff suppressed because it is too large Load Diff

252
python/ql/src/semmle/python/Function.qll
View File

@@ -1,43 +1,37 @@
import python
/**
/**
* A function, independent of defaults and binding.
* It is the syntactic entity that is compiled to a code object.
* It is the syntactic entity that is compiled to a code object.
*/
class Function extends Function_, Scope, AstNode {
/** The expression defining this function */
CallableExpr getDefinition() {
result = this.getParent()
}
CallableExpr getDefinition() { result = this.getParent() }
/** The scope in which this function occurs, will be a class for a method,
* another function for nested functions, generator expressions or comprehensions,
* or a module for a plain function. */
override Scope getEnclosingScope() {
result = this.getParent().(Expr).getScope()
}
/**
* The scope in which this function occurs, will be a class for a method,
* another function for nested functions, generator expressions or comprehensions,
* or a module for a plain function.
*/
override Scope getEnclosingScope() { result = this.getParent().(Expr).getScope() }
override Scope getScope() {
result = this.getEnclosingScope()
}
override Scope getScope() { result = this.getEnclosingScope() }
/** Whether this function is declared in a class */
predicate isMethod() {
exists(Class cls | this.getEnclosingScope() = cls)
}
predicate isMethod() { exists(Class cls | this.getEnclosingScope() = cls) }
/** Whether this is a special method, that is does its name have the form `__xxx__` (except `__init__`) */
predicate isSpecialMethod() {
this.isMethod() and
exists(string name | this.getName() = name |
name.matches("\\_\\_%\\_\\_") and
name != "__init__")
name != "__init__"
)
}
/**
/**
* Whether this function is a generator function,
* that is whether it contains a yield or yield-from expression
* that is whether it contains a yield or yield-from expression
*/
predicate isGenerator() {
exists(Yield y | y.getScope() = this)
@@ -46,88 +40,63 @@ class Function extends Function_, Scope, AstNode {
}
/** Whether this function is declared in a class and is named `__init__` */
predicate isInitMethod() {
this.isMethod() and this.getName() = "__init__"
}
predicate isInitMethod() { this.isMethod() and this.getName() = "__init__" }
/** Gets a decorator of this function */
Expr getADecorator() {
result = ((FunctionExpr)this.getDefinition()).getADecorator()
}
Expr getADecorator() { result = this.getDefinition().(FunctionExpr).getADecorator() }
/** Gets the name of the nth argument (for simple arguments) */
string getArgName(int index) {
result = ((Name)this.getArg(index)).getId()
}
string getArgName(int index) { result = this.getArg(index).(Name).getId() }
Parameter getArgByName(string name) {
result = this.getAnArg() and
result.(Name).getId() = name
}
override Location getLocation() {
py_scope_location(result, this)
}
override Location getLocation() { py_scope_location(result, this) }
override string toString() {
result = "Function " + this.getName()
}
override string toString() { result = "Function " + this.getName() }
/** Gets the statements forming the body of this function */
override StmtList getBody() {
result = Function_.super.getBody()
}
override StmtList getBody() { result = Function_.super.getBody() }
/** Gets the nth statement in the function */
override Stmt getStmt(int index) {
result = Function_.super.getStmt(index)
}
override Stmt getStmt(int index) { result = Function_.super.getStmt(index) }
/** Gets a statement in the function */
override Stmt getAStmt() {
result = Function_.super.getAStmt()
}
override Stmt getAStmt() { result = Function_.super.getAStmt() }
/** Gets the name used to define this function */
override string getName() {
result = Function_.super.getName()
}
override string getName() { result = Function_.super.getName() }
/** Gets the metrics for this function */
FunctionMetrics getMetrics() {
result = this
}
FunctionMetrics getMetrics() { result = this }
/** Gets the FunctionObject corresponding to this function */
FunctionObject getFunctionObject() {
result.getOrigin() = this.getDefinition()
}
FunctionObject getFunctionObject() { result.getOrigin() = this.getDefinition() }
/** Whether this function is a procedure, that is, it has no explicit return statement and always returns None.
* Note that generator and async functions are not procedures as they return generators and coroutines respectively. */
/**
* Whether this function is a procedure, that is, it has no explicit return statement and always returns None.
* Note that generator and async functions are not procedures as they return generators and coroutines respectively.
*/
predicate isProcedure() {
not exists(this.getReturnNode()) and exists(this.getFallthroughNode()) and not this.isGenerator() and not this.isAsync()
not exists(this.getReturnNode()) and
exists(this.getFallthroughNode()) and
not this.isGenerator() and
not this.isAsync()
}
/** Gets the number of positional parameters */
int getPositionalParameterCount() {
result = count(this.getAnArg())
}
int getPositionalParameterCount() { result = count(this.getAnArg()) }
/** Gets the number of keyword-only parameters */
int getKeywordOnlyParameterCount() {
result = count(this.getAKwonlyarg())
}
int getKeywordOnlyParameterCount() { result = count(this.getAKwonlyarg()) }
/** Whether this function accepts a variable number of arguments. That is, whether it has a starred (*arg) parameter. */
predicate hasVarArg() {
exists(this.getVararg())
}
predicate hasVarArg() { exists(this.getVararg()) }
/** Whether this function accepts arbitrary keyword arguments. That is, whether it has a double-starred (**kwarg) parameter. */
predicate hasKwArg() {
exists(this.getKwarg())
}
predicate hasKwArg() { exists(this.getKwarg()) }
override AstNode getAChildNode() {
result = this.getAStmt() or
@@ -136,7 +105,8 @@ class Function extends Function_, Scope, AstNode {
result = this.getKwarg()
}
/** Gets the qualified name for this function.
/**
* Gets the qualified name for this function.
* Should return the same name as the `__qualname__` attribute on functions in Python 3.
*/
string getQualifiedName() {
@@ -145,38 +115,30 @@ class Function extends Function_, Scope, AstNode {
exists(string enclosing_name |
enclosing_name = this.getEnclosingScope().(Function).getQualifiedName()
or
enclosing_name = this.getEnclosingScope().(Class).getQualifiedName() |
enclosing_name = this.getEnclosingScope().(Class).getQualifiedName()
|
result = enclosing_name + "." + this.getName()
)
}
/** Gets the nth keyword-only parameter of this function. */
Name getKeywordOnlyArg(int n) {
result = Function_.super.getKwonlyarg(n)
}
Name getKeywordOnlyArg(int n) { result = Function_.super.getKwonlyarg(n) }
/** Gets a keyword-only parameter of this function. */
Name getAKeywordOnlyArg() {
result = this.getKeywordOnlyArg(_)
}
Name getAKeywordOnlyArg() { result = this.getKeywordOnlyArg(_) }
override Scope getEvaluatingScope() {
major_version() = 2 and exists(Comp comp | comp.getFunction() = this | result = comp.getEvaluatingScope())
major_version() = 2 and
exists(Comp comp | comp.getFunction() = this | result = comp.getEvaluatingScope())
or
not exists(Comp comp | comp.getFunction() = this) and result = this
or
or
major_version() = 3 and result = this
}
override
predicate containsInScope(AstNode inner) {
Scope.super.containsInScope(inner)
}
override predicate containsInScope(AstNode inner) { Scope.super.containsInScope(inner) }
override
predicate contains(AstNode inner) {
Scope.super.contains(inner)
}
override predicate contains(AstNode inner) { Scope.super.contains(inner) }
/** Gets a control flow node for a return value of this function */
ControlFlowNode getAReturnValueFlowNode() {
@@ -185,48 +147,36 @@ class Function extends Function_, Scope, AstNode {
ret.getValue() = result.getNode()
)
}
}
/** A def statement. Note that FunctionDef extends Assign as a function definition binds the newly created function */
class FunctionDef extends Assign {
/* syntax: def name(...): ... */
FunctionDef() {
/* This is an artificial assignment the rhs of which is a (possibly decorated) FunctionExpr */
exists(FunctionExpr f | this.getValue() = f or this.getValue() = f.getADecoratorCall())
}
override string toString() {
result = "FunctionDef"
}
override string toString() { result = "FunctionDef" }
/** Gets the function for this statement */
Function getDefinedFunction() {
exists(FunctionExpr func | this.containsInScope(func) and result = func.getInnerScope())
}
override Stmt getLastStatement() {
result = this.getDefinedFunction().getLastStatement()
}
override Stmt getLastStatement() { result = this.getDefinedFunction().getLastStatement() }
}
class FastLocalsFunction extends Function {
/** A function that uses 'fast' locals, stored in the frame not in a dictionary. */
FastLocalsFunction () {
not exists(ImportStar i | i.getScope() = this)
and
FastLocalsFunction() {
not exists(ImportStar i | i.getScope() = this) and
not exists(Exec e | e.getScope() = this)
}
}
/** A parameter. Either a Tuple or a Name (always a Name for Python 3) */
class Parameter extends Parameter_ {
Parameter() {
/* Parameter_ is just defined as a Name or Tuple, narrow to actual parameters */
exists(ParameterList pl | py_exprs(this, _, pl, _))
@@ -245,36 +195,29 @@ class Parameter extends Parameter_ {
}
/** Gets this parameter if it is a Name (not a Tuple) */
Name asName() {
result = this
}
Name asName() { result = this }
/** Gets this parameter if it is a Tuple (not a Name) */
Tuple asTuple() {
result = this
}
Tuple asTuple() { result = this }
/** Gets the expression for the default value of this parameter */
Expr getDefault() {
exists(Function f, int n, int c, int d, Arguments args |
args = f.getDefinition().getArgs() |
exists(Function f, int n, int c, int d, Arguments args | args = f.getDefinition().getArgs() |
f.getArg(n) = this and
c = count(f.getAnArg()) and
d = count(args.getADefault()) and
result = args.getDefault(d-c+n)
result = args.getDefault(d - c + n)
)
}
/** Gets the annotation expression of this parameter */
Expr getAnnotation() {
exists(Function f, int n, Arguments args |
args = f.getDefinition().getArgs() |
exists(Function f, int n, Arguments args | args = f.getDefinition().getArgs() |
f.getArg(n) = this and
result = args.getAnnotation(n)
)
or
exists(Function f, Arguments args |
args = f.getDefinition().getArgs() |
exists(Function f, Arguments args | args = f.getDefinition().getArgs() |
f.getKwarg() = this and
result = args.getKwargannotation()
or
@@ -283,21 +226,13 @@ class Parameter extends Parameter_ {
)
}
Variable getVariable() {
result.getAnAccess() = this.asName()
}
Variable getVariable() { result.getAnAccess() = this.asName() }
/** Gets the position of this parameter */
int getPosition() {
exists(Function f |
f.getArg(result) = this
)
}
int getPosition() { exists(Function f | f.getArg(result) = this) }
/** Gets the name of this parameter */
string getName() {
result = this.asName().getId()
}
string getName() { result = this.asName().getId() }
/** Holds if this parameter is the first parameter of a method. It is not necessarily called "self" */
predicate isSelf() {
@@ -307,45 +242,39 @@ class Parameter extends Parameter_ {
)
}
/** Holds if this parameter is a 'varargs' parameter.
/**
* Holds if this parameter is a 'varargs' parameter.
* The `varargs` in `f(a, b, *varargs)`.
*/
predicate isVarargs() {
exists(Function func | func.getVararg() = this)
}
predicate isVarargs() { exists(Function func | func.getVararg() = this) }
/** Holds if this parameter is a 'kwargs' parameter.
/**
* Holds if this parameter is a 'kwargs' parameter.
* The `kwargs` in `f(a, b, **kwargs)`.
*/
predicate isKwargs() {
exists(Function func | func.getKwarg() = this)
}
predicate isKwargs() { exists(Function func | func.getKwarg() = this) }
}
/** An expression that generates a callable object, either a function expression or a lambda */
abstract class CallableExpr extends Expr {
/** Gets the parameters of this callable.
* This predicate is called getArgs(), rather than getParameters() for compatibility with Python's AST module. */
/**
* Gets the parameters of this callable.
* This predicate is called getArgs(), rather than getParameters() for compatibility with Python's AST module.
*/
abstract Arguments getArgs();
/** Gets the function scope of this code expression. */
abstract Function getInnerScope();
}
/** An (artificial) expression corresponding to a function definition. */
class FunctionExpr extends FunctionExpr_, CallableExpr {
override Expr getASubExpression() {
result = this.getArgs().getASubExpression() or
result = this.getReturns()
}
override predicate hasSideEffects() {
any()
}
override predicate hasSideEffects() { any() }
Call getADecoratorCall() {
result.getArg(0) = this or
@@ -353,9 +282,7 @@ class FunctionExpr extends FunctionExpr_, CallableExpr {
}
/** Gets a decorator of this function expression */
Expr getADecorator() {
result = this.getADecoratorCall().getFunc()
}
Expr getADecorator() { result = this.getADecoratorCall().getFunc() }
override AstNode getAChildNode() {
result = this.getASubExpression()
@@ -363,47 +290,32 @@ class FunctionExpr extends FunctionExpr_, CallableExpr {
result = this.getInnerScope()
}
override Function getInnerScope() {
result = FunctionExpr_.super.getInnerScope()
}
override Arguments getArgs() {
result = FunctionExpr_.super.getArgs()
}
override Function getInnerScope() { result = FunctionExpr_.super.getInnerScope() }
override Arguments getArgs() { result = FunctionExpr_.super.getArgs() }
}
/** A lambda expression, such as lambda x:x*x */
/** A lambda expression, such as lambda x:x*x */
class Lambda extends Lambda_, CallableExpr {
/** Gets the expression to the right of the colon in this lambda expression */
Expr getExpression() {
exists(Return ret | ret = this.getInnerScope().getStmt(0) |
result = ret.getValue())
exists(Return ret | ret = this.getInnerScope().getStmt(0) | result = ret.getValue())
}
override Expr getASubExpression() {
result = this.getArgs().getASubExpression()
}
override Expr getASubExpression() { result = this.getArgs().getASubExpression() }
override AstNode getAChildNode() {
result = this.getASubExpression() or
result = this.getInnerScope()
}
override Function getInnerScope() {
result = Lambda_.super.getInnerScope()
}
override Arguments getArgs() {
result = Lambda_.super.getArgs()
}
override Function getInnerScope() { result = Lambda_.super.getInnerScope() }
override Arguments getArgs() { result = Lambda_.super.getArgs() }
}
/** The arguments in a function definition */
class Arguments extends Arguments_ {
Expr getASubExpression() {
result = this.getAKwDefault() or
result = this.getAnAnnotation() or
@@ -412,5 +324,3 @@ class Arguments extends Arguments_ {
result = this.getADefault()
}
}

32
python/ql/src/semmle/python/GuardedControlFlow.qll
View File

@@ -2,9 +2,10 @@ import python
/** A basic block which terminates in a condition, splitting the subsequent control flow */
class ConditionBlock extends BasicBlock {
ConditionBlock() {
exists(ControlFlowNode succ | succ = this.getATrueSuccessor() or succ = this.getAFalseSuccessor())
exists(ControlFlowNode succ |
succ = this.getATrueSuccessor() or succ = this.getAFalseSuccessor()
)
}
/** Basic blocks controlled by this condition, i.e. those BBs for which the condition is testIsTrue */
@@ -13,12 +14,12 @@ class ConditionBlock extends BasicBlock {
* For this block to control the block 'controlled' with 'testIsTrue' the following must be true:
* Execution must have passed through the test i.e. 'this' must strictly dominate 'controlled'.
* Execution must have passed through the 'testIsTrue' edge leaving 'this'.
*
*
* Although "passed through the true edge" implies that this.getATrueSuccessor() dominates 'controlled',
* the reverse is not true, as flow may have passed through another edge to get to this.getATrueSuccessor()
* so we need to assert that this.getATrueSuccessor() dominates 'controlled' *and* that
* so we need to assert that this.getATrueSuccessor() dominates 'controlled' *and* that
* all predecessors of this.getATrueSuccessor() are either this or dominated by this.getATrueSuccessor().
*
*
* For example, in the following python snippet:
* <code>
* if x:
@@ -26,7 +27,7 @@ class ConditionBlock extends BasicBlock {
* false_successor
* uncontrolled
* </code>
* false_successor dominates uncontrolled, but not all of its predecessors are this (if x)
* false_successor dominates uncontrolled, but not all of its predecessors are this (if x)
* or dominated by itself. Whereas in the following code:
* <code>
* if x:
@@ -35,22 +36,21 @@ class ConditionBlock extends BasicBlock {
* false_successor
* uncontrolled
* </code>
* the block 'while controlled' is controlled because all of its predecessors are this (if x)
* the block 'while controlled' is controlled because all of its predecessors are this (if x)
* or (in the case of 'also_controlled') dominated by itself.
*
*
* The additional constraint on the predecessors of the test successor implies
* that `this` strictly dominates `controlled` so that isn't necessary to check
* directly.
*/
exists(BasicBlock succ |
exists(BasicBlock succ |
testIsTrue = true and succ = this.getATrueSuccessor()
or
testIsTrue = false and succ = this.getAFalseSuccessor()
|
succ.dominates(controlled) and
forall(BasicBlock pred | pred.getASuccessor() = succ |
pred = this or succ.dominates(pred)
)
|
succ.dominates(controlled) and
forall(BasicBlock pred | pred.getASuccessor() = succ | pred = this or succ.dominates(pred))
)
}
@@ -58,11 +58,11 @@ class ConditionBlock extends BasicBlock {
predicate controlsEdge(BasicBlock pred, BasicBlock succ, boolean testIsTrue) {
this.controls(pred, testIsTrue) and succ = pred.getASuccessor()
or
pred = this and (
pred = this and
(
testIsTrue = true and succ = this.getATrueSuccessor()
or
testIsTrue = false and succ = this.getAFalseSuccessor()
)
}
}

188
python/ql/src/semmle/python/Import.qll
View File

@@ -1,14 +1,12 @@
import python
private import semmle.python.types.Builtins
/** An alias in an import statement, the `mod as name` part of `import mod as name`. May be artificial;
* `import x` is transformed into `import x as x` */
/**
* An alias in an import statement, the `mod as name` part of `import mod as name`. May be artificial;
* `import x` is transformed into `import x as x`
*/
class Alias extends Alias_ {
Location getLocation() {
result = this.getValue().getLocation()
}
Location getLocation() { result = this.getValue().getLocation() }
}
private predicate valid_module_name(string name) {
@@ -19,37 +17,34 @@ private predicate valid_module_name(string name) {
/** An artificial expression representing an import */
class ImportExpr extends ImportExpr_ {
private string basePackageName(int n) {
n = 1 and result = this.getEnclosingModule().getPackageName()
or
exists(string bpnm1 | bpnm1 = this.basePackageName(n-1) and
or
exists(string bpnm1 |
bpnm1 = this.basePackageName(n - 1) and
bpnm1.matches("%.%") and
result = bpnm1.regexpReplaceAll("\\.[^.]*$", "")
)
}
private predicate implicitRelativeImportsAllowed() {
// relative imports are no longer allowed in Python 3
// relative imports are no longer allowed in Python 3
major_version() < 3 and
// and can be explicitly turned off in later versions of Python 2
not getEnclosingModule().hasFromFuture("absolute_import")
}
/** The language specifies level as -1 if relative imports are to be tried first, 0 for absolute imports,
* and level > 0 for explicit relative imports. */
/**
* The language specifies level as -1 if relative imports are to be tried first, 0 for absolute imports,
* and level > 0 for explicit relative imports.
*/
override int getLevel() {
exists(int l | l = super.getLevel() |
l > 0 and result = l
or
/* The extractor may set level to 0 even though relative imports apply */
l = 0 and (
if this.implicitRelativeImportsAllowed() then
result = -1
else
result = 0
)
l = 0 and
(if this.implicitRelativeImportsAllowed() then result = -1 else result = 0)
)
}
@@ -64,49 +59,47 @@ class ImportExpr extends ImportExpr_ {
}
private string qualifiedTopName() {
if (this.getLevel() <= 0) then (
result = this.getTopName()
) else (
if this.getLevel() <= 0
then result = this.getTopName()
else (
result = basePackageName(this.getLevel()) and
valid_module_name(result)
)
}
/** Gets the name by which the lowest level module or package is imported.
* NOTE: This is the name that used to import the module,
* which may not be the name of the module. */
/**
* Gets the name by which the lowest level module or package is imported.
* NOTE: This is the name that used to import the module,
* which may not be the name of the module.
*/
string bottomModuleName() {
result = relativeTopName() + this.remainderOfName()
or
(
not exists(relativeTopName()) and
result = this.qualifiedTopName() + this.remainderOfName()
)
not exists(relativeTopName()) and
result = this.qualifiedTopName() + this.remainderOfName()
}
/** Gets the name of topmost module or package being imported */
string topModuleName() {
result = relativeTopName()
or
(
not exists(relativeTopName()) and
result = this.qualifiedTopName()
)
not exists(relativeTopName()) and
result = this.qualifiedTopName()
}
/** Gets the full name of the module resulting from evaluating this import.
/**
* Gets the full name of the module resulting from evaluating this import.
* NOTE: This is the name that used to import the module,
* which may not be the name of the module. */
* which may not be the name of the module.
*/
string getImportedModuleName() {
exists(string bottomName | bottomName = this.bottomModuleName() |
if this.isTop() then
result = topModuleName()
else
result = bottomName
if this.isTop() then result = topModuleName() else result = bottomName
)
}
/** Gets the names of the modules that may be imported by this import.
/**
* Gets the names of the modules that may be imported by this import.
* For example this predicate would return 'x' and 'x.y' for `import x.y`
*/
string getAnImportedModuleName() {
@@ -115,26 +108,24 @@ class ImportExpr extends ImportExpr_ {
result = this.getAnImportedModuleName().regexpReplaceAll("\\.[^.]*$", "")
}
override Expr getASubExpression() {
none()
}
override Expr getASubExpression() { none() }
override predicate hasSideEffects() {
any()
}
override predicate hasSideEffects() { any() }
private string getTopName() {
result = this.getName().regexpReplaceAll("\\..*", "")
}
private string getTopName() { result = this.getName().regexpReplaceAll("\\..*", "") }
private string remainderOfName() {
not exists(this.getName()) and result = "" or
this.getLevel() <= 0 and result = this.getName().regexpReplaceAll("^[^\\.]*", "") or
not exists(this.getName()) and result = ""
or
this.getLevel() <= 0 and result = this.getName().regexpReplaceAll("^[^\\.]*", "")
or
this.getLevel() > 0 and result = "." + this.getName()
}
/** Whether this import is relative, that is not absolute.
* See https://www.python.org/dev/peps/pep-0328/ */
/**
* Whether this import is relative, that is not absolute.
* See https://www.python.org/dev/peps/pep-0328/
*/
predicate isRelative() {
/* Implicit */
exists(this.relativeTopName())
@@ -142,24 +133,22 @@ class ImportExpr extends ImportExpr_ {
/* Explicit */
this.getLevel() > 0
}
}
/** A `from ... import ...` expression */
class ImportMember extends ImportMember_ {
override Expr getASubExpression() {
result = this.getModule()
}
override Expr getASubExpression() { result = this.getModule() }
override predicate hasSideEffects() {
/* Strictly this only has side-effects if the module is a package */
any()
}
/** Gets the full name of the module resulting from evaluating this import.
* NOTE: This is the name that used to import the module,
* which may not be the name of the module. */
/**
* Gets the full name of the module resulting from evaluating this import.
* NOTE: This is the name that used to import the module,
* which may not be the name of the module.
*/
string getImportedModuleName() {
result = this.getModule().(ImportExpr).getImportedModuleName() + "." + this.getName()
}
@@ -169,26 +158,22 @@ class ImportMember extends ImportMember_ {
/** An import statement */
class Import extends Import_ {
/* syntax: import modname */
private ImportExpr getAModuleExpr() {
result = this.getAName().getValue()
or
result = ((ImportMember)this.getAName().getValue()).getModule()
or
result = this.getAName().getValue().(ImportMember).getModule()
}
/** Use getAnImportedModuleName(),
/**
* Use getAnImportedModuleName(),
* possibly combined with ModuleObject.importedAs()
* Gets a module imported by this import statement */
deprecated Module getAModule() {
result.getName() = this.getAnImportedModuleName()
}
* Gets a module imported by this import statement
*/
deprecated Module getAModule() { result.getName() = this.getAnImportedModuleName() }
/** Whether this a `from ... import ...` statement */
predicate isFromImport() {
this.getAName().getValue() instanceof ImportMember
}
predicate isFromImport() { this.getAName().getValue() instanceof ImportMember }
override Expr getASubExpression() {
result = this.getAModuleExpr() or
@@ -196,73 +181,61 @@ class Import extends Import_ {
result = this.getAName().getValue()
}
override Stmt getASubStatement() {
none()
}
override Stmt getASubStatement() { none() }
/** Gets the name of an imported module.
/**
* Gets the name of an imported module.
* For example, for the import statement `import bar` which
* is a relative import in package "foo", this would return
* "foo.bar".
* The import statment `from foo import bar` would return
* The import statment `from foo import bar` would return
* `foo` and `foo.bar`
* */
*/
string getAnImportedModuleName() {
result = this.getAModuleExpr().getAnImportedModuleName()
or
exists(ImportMember m, string modname |
m = this.getAName().getValue() and
modname = m.getModule().(ImportExpr).getImportedModuleName() |
modname = m.getModule().(ImportExpr).getImportedModuleName()
|
result = modname
or
result = modname + "." + m.getName()
)
}
}
/** An import * statement */
class ImportStar extends ImportStar_ {
/* syntax: from modname import * */
ImportExpr getModuleExpr() {
result = this.getModule()
or
result = ((ImportMember)this.getModule()).getModule()
result = this.getModule().(ImportMember).getModule()
}
override string toString() {
result = "from " + this.getModuleExpr().getName() + " import *"
}
override string toString() { result = "from " + this.getModuleExpr().getName() + " import *" }
/** Use getAnImportedModuleName(),
/**
* Use getAnImportedModuleName(),
* possibly combined with ModuleObject.importedAs()
* Gets the module imported by this import * statement
* Gets the module imported by this import * statement
*/
deprecated Module getTheModule() {
result.getName() = this.getImportedModuleName()
}
deprecated Module getTheModule() { result.getName() = this.getImportedModuleName() }
override Expr getASubExpression() {
result = this.getModule()
}
override Expr getASubExpression() { result = this.getModule() }
override Stmt getASubStatement() {
none()
}
override Stmt getASubStatement() { none() }
/** Gets the name of the imported module. */
string getImportedModuleName() {
result = this.getModuleExpr().getImportedModuleName()
}
string getImportedModuleName() { result = this.getModuleExpr().getImportedModuleName() }
}
/** A statement that imports a module. This can be any statement that includes the `import` keyword,
* such as `import sys`, `from sys import version` or `from sys import *`. */
/**
* A statement that imports a module. This can be any statement that includes the `import` keyword,
* such as `import sys`, `from sys import version` or `from sys import *`.
*/
class ImportingStmt extends Stmt {
ImportingStmt() {
this instanceof Import
or
@@ -271,9 +244,8 @@ class ImportingStmt extends Stmt {
/** Gets the name of an imported module. */
string getAnImportedModuleName() {
result = this.(Import).getAnImportedModuleName()
result = this.(Import).getAnImportedModuleName()
or
result = this.(ImportStar).getImportedModuleName()
}
}

79
python/ql/src/semmle/python/Keywords.qll
View File

@@ -1,105 +1,60 @@
import python
class KeyValuePair extends KeyValuePair_, DictDisplayItem {
/* syntax: Expr : Expr */
override Location getLocation() { result = KeyValuePair_.super.getLocation() }
override Location getLocation() {
result = KeyValuePair_.super.getLocation()
}
override string toString() {
result = KeyValuePair_.super.toString()
}
override string toString() { result = KeyValuePair_.super.toString() }
/** Gets the value of this dictionary unpacking. */
override Expr getValue() {
result = KeyValuePair_.super.getValue()
}
override Expr getValue() { result = KeyValuePair_.super.getValue() }
override Scope getScope() {
result = this.getValue().getScope()
}
override Scope getScope() { result = this.getValue().getScope() }
override AstNode getAChildNode() {
result = this.getKey()
or
result = this.getValue()
}
}
/** A double-starred expression in a call or dict literal. */
class DictUnpacking extends DictUnpacking_, DictUnpackingOrKeyword, DictDisplayItem {
override Location getLocation() { result = DictUnpacking_.super.getLocation() }
override Location getLocation() {
result = DictUnpacking_.super.getLocation()
}
override string toString() {
result = DictUnpacking_.super.toString()
}
override string toString() { result = DictUnpacking_.super.toString() }
/** Gets the value of this dictionary unpacking. */
override Expr getValue() {
result = DictUnpacking_.super.getValue()
}
override Expr getValue() { result = DictUnpacking_.super.getValue() }
override Scope getScope() {
result = this.getValue().getScope()
}
override AstNode getAChildNode() {
result = this.getValue()
}
override Scope getScope() { result = this.getValue().getScope() }
override AstNode getAChildNode() { result = this.getValue() }
}
abstract class DictUnpackingOrKeyword extends DictItem {
abstract Expr getValue();
override string toString() {
result = "DictUnpackingOrKeyword with missing toString"
}
override string toString() { result = "DictUnpackingOrKeyword with missing toString" }
}
abstract class DictDisplayItem extends DictItem {
abstract Expr getValue();
override string toString() {
result = "DictDisplayItem with missing toString"
}
override string toString() { result = "DictDisplayItem with missing toString" }
}
/** A keyword argument in a call. For example `arg=expr` in `foo(0, arg=expr)` */
class Keyword extends Keyword_, DictUnpackingOrKeyword {
class Keyword extends Keyword_, DictUnpackingOrKeyword {
/* syntax: name = Expr */
override Location getLocation() { result = Keyword_.super.getLocation() }
override Location getLocation() {
result = Keyword_.super.getLocation()
}
override string toString() {
result = Keyword_.super.toString()
}
override string toString() { result = Keyword_.super.toString() }
/** Gets the value of this keyword argument. */
override Expr getValue() {
result = Keyword_.super.getValue()
}
override Expr getValue() { result = Keyword_.super.getValue() }
override Scope getScope() {
result = this.getValue().getScope()
}
override AstNode getAChildNode() {
result = this.getValue()
}
override Scope getScope() { result = this.getValue().getScope() }
override AstNode getAChildNode() { result = this.getValue() }
}

374
python/ql/src/semmle/python/Metrics.qll
View File

@@ -2,29 +2,23 @@ import python
/** The metrics for a function */
class FunctionMetrics extends Function {
/** Gets the total number of lines (including blank lines)
* from the definition to the end of the function */
int getNumberOfLines() {
py_alllines(this, result)
}
/**
* Gets the total number of lines (including blank lines)
* from the definition to the end of the function
*/
int getNumberOfLines() { py_alllines(this, result) }
/** Gets the number of lines of code in the function */
int getNumberOfLinesOfCode() {
py_codelines(this, result)
}
int getNumberOfLinesOfCode() { py_codelines(this, result) }
/** Gets the number of lines of comments in the function */
int getNumberOfLinesOfComments() {
py_commentlines(this, result)
}
int getNumberOfLinesOfComments() { py_commentlines(this, result) }
/** Gets the number of lines of docstring in the function */
int getNumberOfLinesOfDocStrings() {
py_docstringlines(this, result)
}
int getNumberOfLinesOfDocStrings() { py_docstringlines(this, result) }
/** Cyclomatic complexity:
/**
* Cyclomatic complexity:
* The number of linearly independent paths through the source code.
* Computed as E - N + 2P,
* where
@@ -34,14 +28,17 @@ class FunctionMetrics extends Function {
*/
int getCyclomaticComplexity() {
exists(int E, int N |
N = count(BasicBlock b | b = this.getABasicBlock() and b.likelyReachable())
and
E = count(BasicBlock b1, BasicBlock b2 |
b1 = this.getABasicBlock() and b1.likelyReachable() and
b2 = this.getABasicBlock() and b2.likelyReachable() and
b2 = b1.getASuccessor() and not b1.unlikelySuccessor(b2)
)
|
N = count(BasicBlock b | b = this.getABasicBlock() and b.likelyReachable()) and
E =
count(BasicBlock b1, BasicBlock b2 |
b1 = this.getABasicBlock() and
b1.likelyReachable() and
b2 = this.getABasicBlock() and
b2.likelyReachable() and
b2 = b1.getASuccessor() and
not b1.unlikelySuccessor(b2)
)
|
result = E - N + 2
)
}
@@ -51,24 +48,23 @@ class FunctionMetrics extends Function {
or
exists(BasicBlock mid | mid = this.getABasicBlock() and result = mid.getASuccessor())
}
/** Dependency of Callables
/**
* Dependency of Callables
* One callable "this" depends on another callable "result"
* if "this" makes some call to a method that may end up being "result".
*/
*/
FunctionMetrics getADependency() {
result != this and
not non_coupling_method(result) and
exists(Call call |
call.getScope() = this |
exists(FunctionObject callee |
callee.getFunction() = result |
exists(Call call | call.getScope() = this |
exists(FunctionObject callee | callee.getFunction() = result |
call.getAFlowNode().getFunction().refersTo(callee)
)
or
exists(Attribute a |
call.getFunc() = a |
unique_root_method(result, a.getName()) or
exists(Attribute a | call.getFunc() = a |
unique_root_method(result, a.getName())
or
exists(Name n | a.getObject() = n and n.getId() = "self" |
result.getScope() = this.getScope() and
result.getName() = a.getName()
@@ -76,145 +72,122 @@ class FunctionMetrics extends Function {
)
)
}
/** Afferent Coupling
/**
* Afferent Coupling
* the number of callables that depend on this method.
* This is sometimes called the "fan-in" of a method.
*/
int getAfferentCoupling() {
result = count(FunctionMetrics m | m.getADependency() = this )
}
/** Efferent Coupling
int getAfferentCoupling() { result = count(FunctionMetrics m | m.getADependency() = this) }
/**
* Efferent Coupling
* the number of methods that this method depends on
* This is sometimes called the "fan-out" of a method.
*/
int getEfferentCoupling() {
result = count(FunctionMetrics m | this.getADependency() = m)
}
int getEfferentCoupling() { result = count(FunctionMetrics m | this.getADependency() = m) }
int getNumberOfParametersWithoutDefault() {
result = this.getPositionalParameterCount() -
count(((FunctionExpr)this.getDefinition()).getArgs().getADefault())
result =
this.getPositionalParameterCount() -
count(this.getDefinition().(FunctionExpr).getArgs().getADefault())
}
int getStatementNestingDepth() {
result = max(Stmt s | s.getScope() = this | getNestingDepth(s))
}
int getNumberOfCalls() {
result = count(Call c | c.getScope() = this)
}
int getStatementNestingDepth() { result = max(Stmt s | s.getScope() = this | getNestingDepth(s)) }
int getNumberOfCalls() { result = count(Call c | c.getScope() = this) }
}
/** The metrics for a class */
class ClassMetrics extends Class {
/** Gets the total number of lines (including blank lines)
* from the definition to the end of the class */
int getNumberOfLines() {
py_alllines(this, result)
}
/**
* Gets the total number of lines (including blank lines)
* from the definition to the end of the class
*/
int getNumberOfLines() { py_alllines(this, result) }
/** Gets the number of lines of code in the class */
int getNumberOfLinesOfCode() {
py_codelines(this, result)
}
int getNumberOfLinesOfCode() { py_codelines(this, result) }
/** Gets the number of lines of comments in the class */
int getNumberOfLinesOfComments() {
py_commentlines(this, result)
}
int getNumberOfLinesOfComments() { py_commentlines(this, result) }
/** Gets the number of lines of docstrings in the class */
int getNumberOfLinesOfDocStrings() {
py_docstringlines(this, result)
}
int getNumberOfLinesOfDocStrings() { py_docstringlines(this, result) }
private predicate dependsOn(Class other) {
other != this and
(
exists(FunctionMetrics f1, FunctionMetrics f2 |
f1.getADependency() = f2 |
exists(FunctionMetrics f1, FunctionMetrics f2 | f1.getADependency() = f2 |
f1.getScope() = this and f2.getScope() = other
)
or
exists(Function f, Call c, ClassObject cls |
c.getScope() = f and f.getScope() = this |
exists(Function f, Call c, ClassObject cls | c.getScope() = f and f.getScope() = this |
c.getFunc().refersTo(cls) and
cls.getPyClass() = other
)
)
}
/** The afferent coupling of a class is the number of classes that
* directly depend on it.
/**
* The afferent coupling of a class is the number of classes that
* directly depend on it.
*/
int getAfferentCoupling() {
result = count(ClassMetrics t | t.dependsOn(this))
}
/** The efferent coupling of a class is the number of classes that
int getAfferentCoupling() { result = count(ClassMetrics t | t.dependsOn(this)) }
/**
* The efferent coupling of a class is the number of classes that
* it directly depends on.
*/
int getEfferentCoupling() {
result = count(ClassMetrics t | this.dependsOn(t))
}
int getEfferentCoupling() { result = count(ClassMetrics t | this.dependsOn(t)) }
int getInheritanceDepth() {
exists(ClassObject cls |
cls.getPyClass() = this |
result = max(classInheritanceDepth(cls))
)
exists(ClassObject cls | cls.getPyClass() = this | result = max(classInheritanceDepth(cls)))
}
/* -------- CHIDAMBER AND KEMERER LACK OF COHESION IN METHODS ------------ */
/* The aim of this metric is to try and determine whether a class
* represents one abstraction (good) or multiple abstractions (bad).
* If a class represents multiple abstractions, it should be split
* up into multiple classes.
*
* In the Chidamber and Kemerer method, this is measured as follows:
* n1 = number of pairs of distinct methods in a class that do *not*
* have at least one commonly accessed field
* n2 = number of pairs of distinct methods in a class that do
* have at least one commonly accessed field
* lcom = ((n1 - n2)/2 max 0)
*
/*
* The aim of this metric is to try and determine whether a class
* represents one abstraction (good) or multiple abstractions (bad).
* If a class represents multiple abstractions, it should be split
* up into multiple classes.
*
* In the Chidamber and Kemerer method, this is measured as follows:
* n1 = number of pairs of distinct methods in a class that do *not*
* have at least one commonly accessed field
* n2 = number of pairs of distinct methods in a class that do
* have at least one commonly accessed field
* lcom = ((n1 - n2)/2 max 0)
*
* We divide by 2 because each pair (m1,m2) is counted twice in n1 and n2.
*/
/** should function f be excluded from the cohesion computation? */
predicate ignoreLackOfCohesion(Function f) {
f.isInitMethod() or f.isSpecialMethod()
}
predicate ignoreLackOfCohesion(Function f) { f.isInitMethod() or f.isSpecialMethod() }
private predicate methodPair(Function m1, Function m2) {
m1.getScope() = this and
m2.getScope() = this and
not this.ignoreLackOfCohesion(m1) and
not this.ignoreLackOfCohesion(m1) and
not this.ignoreLackOfCohesion(m2) and
m1 != m2
}
private predicate one_accesses_other(Function m1, Function m2) {
this.methodPair(m1, m2) and
(
exists(SelfAttributeRead sa |
sa.getName() = m1.getName() and
sa.getScope() = m2
)
or
exists(SelfAttributeRead sa |
sa.getName() = m2.getName() and
sa.getScope() = m1
)
exists(SelfAttributeRead sa |
sa.getName() = m1.getName() and
sa.getScope() = m2
)
or
exists(SelfAttributeRead sa |
sa.getName() = m2.getName() and
sa.getScope() = m1
)
)
}
/** do m1 and m2 access a common field or one calls the other? */
private predicate shareField(Function m1, Function m2) {
this.methodPair(m1, m2) and
@@ -222,142 +195,117 @@ class ClassMetrics extends Class {
exists(SelfAttributeRead sa |
sa.getName() = name and
sa.getScope() = m1
)
and
) and
exists(SelfAttributeRead sa |
sa.getName() = name and
sa.getScope() = m2
)
)
}
private int similarMethodPairs() {
result = count(Function m1, Function m2 |
this.methodPair(m1, m2) and
(this.shareField(m1, m2) or this.one_accesses_other(m1, m2))
) / 2
result =
count(Function m1, Function m2 |
this.methodPair(m1, m2) and
(this.shareField(m1, m2) or this.one_accesses_other(m1, m2))
) / 2
}
private int methodPairs() {
result = count(Function m1, Function m2 | this.methodPair(m1, m2)) / 2
result = count(Function m1, Function m2 | this.methodPair(m1, m2)) / 2
}
/** return Chidamber and Kemerer Lack of Cohesion */
int getLackOfCohesionCK() {
exists(int n |
n = this.methodPairs() - 2 * this.similarMethodPairs()
and
n = this.methodPairs() - 2 * this.similarMethodPairs() and
result = n.maximum(0)
)
}
private predicate similarMethodPairDag(Function m1, Function m2, int line) {
(this.shareField(m1, m2) or this.one_accesses_other(m1, m2)) and
line = m1.getLocation().getStartLine() and
line < m2.getLocation().getStartLine()
}
private predicate subgraph(Function m, int line) {
this.similarMethodPairDag(m, _, line) and not this.similarMethodPairDag(_, m, _)
or
exists(Function other | this.subgraph(other, line) |
this.similarMethodPairDag(other, m, _) or
this.similarMethodPairDag(m, other, _)
)
}
predicate unionSubgraph(Function m, int line) {
line = min(int l | this.subgraph(m, l))
}
)
}
/** return Hitz and Montazeri Lack of Cohesion */
int getLackOfCohesionHM() {
result = count(int line |
this.unionSubgraph(_, line)
)
}
private predicate similarMethodPairDag(Function m1, Function m2, int line) {
(this.shareField(m1, m2) or this.one_accesses_other(m1, m2)) and
line = m1.getLocation().getStartLine() and
line < m2.getLocation().getStartLine()
}
private predicate subgraph(Function m, int line) {
this.similarMethodPairDag(m, _, line) and not this.similarMethodPairDag(_, m, _)
or
exists(Function other | this.subgraph(other, line) |
this.similarMethodPairDag(other, m, _) or
this.similarMethodPairDag(m, other, _)
)
}
predicate unionSubgraph(Function m, int line) { line = min(int l | this.subgraph(m, l)) }
/** return Hitz and Montazeri Lack of Cohesion */
int getLackOfCohesionHM() { result = count(int line | this.unionSubgraph(_, line)) }
}
private int classInheritanceDepth(ClassObject cls) {
/* Prevent run-away recursion in case of circular inheritance */
not cls.getASuperType() = cls
and
not cls.getASuperType() = cls and
(
exists(ClassObject sup |
cls.getABaseType() = sup |
result = classInheritanceDepth(sup) + 1
)
or
not exists(cls.getABaseType()) and (
major_version() = 2 and result = 0
or
major_version() > 2 and result = 1
)
exists(ClassObject sup | cls.getABaseType() = sup | result = classInheritanceDepth(sup) + 1)
or
not exists(cls.getABaseType()) and
(
major_version() = 2 and result = 0
or
major_version() > 2 and result = 1
)
)
}
class ModuleMetrics extends Module {
/** Gets the total number of lines (including blank lines) in the module */
int getNumberOfLines() {
py_alllines(this, result)
}
int getNumberOfLines() { py_alllines(this, result) }
/** Gets the number of lines of code in the module */
int getNumberOfLinesOfCode() {
py_codelines(this, result)
}
/** Gets the number of lines of code in the module */
int getNumberOfLinesOfCode() { py_codelines(this, result) }
/** Gets the number of lines of comments in the module */
int getNumberOfLinesOfComments() {
py_commentlines(this, result)
}
int getNumberOfLinesOfComments() { py_commentlines(this, result) }
/** Gets the number of lines of docstrings in the module */
int getNumberOfLinesOfDocStrings() {
py_docstringlines(this, result)
}
/** The afferent coupling of a class is the number of classes that
* directly depend on it.
*/
int getAfferentCoupling() {
result = count(ModuleMetrics t | t.dependsOn(this))
}
int getNumberOfLinesOfDocStrings() { py_docstringlines(this, result) }
/** The efferent coupling of a class is the number of classes that
/**
* The afferent coupling of a class is the number of classes that
* directly depend on it.
*/
int getAfferentCoupling() { result = count(ModuleMetrics t | t.dependsOn(this)) }
/**
* The efferent coupling of a class is the number of classes that
* it directly depends on.
*/
int getEfferentCoupling() {
result = count(ModuleMetrics t | this.dependsOn(t))
}
int getEfferentCoupling() { result = count(ModuleMetrics t | this.dependsOn(t)) }
private predicate dependsOn(Module other) {
other != this and
(
exists(FunctionMetrics f1, FunctionMetrics f2 |
f1.getADependency() = f2 |
exists(FunctionMetrics f1, FunctionMetrics f2 | f1.getADependency() = f2 |
f1.getEnclosingModule() = this and f2.getEnclosingModule() = other
)
or
exists(Function f, Call c, ClassObject cls |
c.getScope() = f and f.getScope() = this |
exists(Function f, Call c, ClassObject cls | c.getScope() = f and f.getScope() = this |
c.getFunc().refersTo(cls) and
cls.getPyClass().getEnclosingModule() = other
)
)
}
}
/** Helpers for coupling */
predicate unique_root_method(Function func, string name) {
name = func.getName() and
not exists(FunctionObject f, FunctionObject other |
name = func.getName() and
not exists(FunctionObject f, FunctionObject other |
f.getFunction() = func and
other.getName() = name |
other.getName() = name
|
not other.overrides(f)
)
}
@@ -375,13 +323,11 @@ predicate non_coupling_method(Function f) {
private int getNestingDepth(Stmt s) {
not exists(Stmt outer | outer.getASubStatement() = s) and result = 1
or
exists(Stmt outer |
outer.getASubStatement() = s |
if s.(If).isElif() or s instanceof ExceptStmt then
exists(Stmt outer | outer.getASubStatement() = s |
if s.(If).isElif() or s instanceof ExceptStmt
then
/* If statement is an `elif` or `except` then it is not indented relative to its parent */
result = getNestingDepth(outer)
else
result = getNestingDepth(outer) + 1
else result = getNestingDepth(outer) + 1
)
}

148
python/ql/src/semmle/python/Module.qll
View File

@@ -2,47 +2,42 @@ import python
private import semmle.python.objects.ObjectAPI
private import semmle.python.objects.Modules
/** A module. This is the top level element in an AST, corresponding to a source file.
* It is also a Scope; the scope of global variables. */
/**
* A module. This is the top level element in an AST, corresponding to a source file.
* It is also a Scope; the scope of global variables.
*/
class Module extends Module_, Scope, AstNode {
override string toString() {
result = this.getKind() + " " + this.getName()
or
/* No name is defined, which means that this module is not on an import path. So it must be a script */
not exists(this.getName()) and not this.isPackage() and
not exists(this.getName()) and
not this.isPackage() and
result = "Script " + this.getFile().getShortName()
or
/* Package missing name, so just use the path instead */
not exists(this.getName()) and this.isPackage() and
result = "Package at " + this.getPath().getAbsolutePath()
not exists(this.getName()) and
this.isPackage() and
result = "Package at " + this.getPath().getAbsolutePath()
}
/** This method will be deprecated in the next release. Please use `getEnclosingScope()` instead.
* The enclosing scope of this module (always none) */
override Scope getScope() {
none()
}
/**
* This method will be deprecated in the next release. Please use `getEnclosingScope()` instead.
* The enclosing scope of this module (always none)
*/
override Scope getScope() { none() }
/** The enclosing scope of this module (always none) */
override Scope getEnclosingScope() {
none()
}
override Scope getEnclosingScope() { none() }
/** Gets the statements forming the body of this module */
override StmtList getBody() {
result = Module_.super.getBody()
}
override StmtList getBody() { result = Module_.super.getBody() }
/** Gets the nth statement of this module */
override Stmt getStmt(int n) {
result = Module_.super.getStmt(n)
}
override Stmt getStmt(int n) { result = Module_.super.getStmt(n) }
/** Gets a top-level statement in this module */
override Stmt getAStmt() {
result = Module_.super.getAStmt()
}
override Stmt getAStmt() { result = Module_.super.getAStmt() }
/** Gets the name of this module */
override string getName() {
@@ -54,15 +49,13 @@ class Module extends Module_, Scope, AstNode {
/** Gets the short name of the module. For example the short name of module x.y.z is 'z' */
string getShortName() {
result = this.getName().suffix(this.getPackage().getName().length()+1)
result = this.getName().suffix(this.getPackage().getName().length() + 1)
or
result = this.getName() and not exists(this.getPackage())
}
/** Gets this module */
override Module getEnclosingModule() {
result = this
}
override Module getEnclosingModule() { result = this }
/** Gets the __init__ module of this module if the module is a package and it has an __init__ module */
Module getInitModule() {
@@ -70,34 +63,25 @@ class Module extends Module_, Scope, AstNode {
}
/** Whether this module is a package initializer */
predicate isPackageInit() {
this.getName().matches("%\\_\\_init\\_\\_") and not this.isPackage()
}
predicate isPackageInit() { this.getName().matches("%\\_\\_init\\_\\_") and not this.isPackage() }
/** Gets a name exported by this module, that is the names that will be added to a namespace by 'from this-module import *' */
/** Gets a name exported by this module, that is the names that will be added to a namespace by 'from this-module import *' */
string getAnExport() {
py_exports(this, result)
or
exists(ModuleObjectInternal mod |
mod.getSource() = this.getEntryNode() |
exists(ModuleObjectInternal mod | mod.getSource() = this.getEntryNode() |
mod.(ModuleValue).exports(result)
)
}
/** Gets the source file for this module */
File getFile() {
py_module_path(this, result)
}
File getFile() { py_module_path(this, result) }
/** Gets the source file or folder for this module or package */
Container getPath() {
py_module_path(this, result)
}
Container getPath() { py_module_path(this, result) }
/** Whether this is a package */
predicate isPackage() {
this.getPath() instanceof Folder
}
predicate isPackage() { this.getPath() instanceof Folder }
/** Gets the package containing this module (or parent package if this is a package) */
Module getPackage() {
@@ -112,15 +96,13 @@ class Module extends Module_, Scope, AstNode {
}
/** Gets the metrics for this module */
ModuleMetrics getMetrics() {
result = this
}
ModuleMetrics getMetrics() { result = this }
/** Use ModuleObject.getAnImportedModule() instead.
* Gets a module imported by this module */
deprecated Module getAnImportedModule() {
result.getName() = this.getAnImportedModuleName()
}
/**
* Use ModuleObject.getAnImportedModule() instead.
* Gets a module imported by this module
*/
deprecated Module getAnImportedModule() { result.getName() = this.getAnImportedModuleName() }
string getAnImportedModuleName() {
exists(Import i | i.getEnclosingModule() = this | result = i.getAnImportedModuleName())
@@ -142,22 +124,23 @@ class Module extends Module_, Scope, AstNode {
}
/** Whether name is declared in the __all__ list of this module */
predicate declaredInAll(string name)
{
predicate declaredInAll(string name) {
exists(AssignStmt a, GlobalVariable all |
a.defines(all) and a.getScope() = this and
all.getId() = "__all__" and ((List)a.getValue()).getAnElt().(StrConst).getText() = name
a.defines(all) and
a.getScope() = this and
all.getId() = "__all__" and
a.getValue().(List).getAnElt().(StrConst).getText() = name
)
}
override AstNode getAChildNode() {
result = this.getAStmt()
}
override AstNode getAChildNode() { result = this.getAStmt() }
predicate hasFromFuture(string attr) {
exists(Import i, ImportMember im, ImportExpr ie, Alias a, Name name |
im.getModule() = ie and ie.getName() = "__future__" and
a.getAsname() = name and name.getId() = attr and
im.getModule() = ie and
ie.getName() = "__future__" and
a.getAsname() = name and
name.getId() = attr and
i.getASubExpression() = im and
i.getAName() = a and
i.getEnclosingModule() = this
@@ -165,55 +148,40 @@ class Module extends Module_, Scope, AstNode {
}
/** Gets the path element from which this module was loaded. */
Container getLoadPath() {
result = this.getPath().getImportRoot()
}
Container getLoadPath() { result = this.getPath().getImportRoot() }
/** Holds if this module is in the standard library for version `major.minor` */
predicate inStdLib(int major, int minor) {
this.getLoadPath().isStdLibRoot(major, minor)
}
predicate inStdLib(int major, int minor) { this.getLoadPath().isStdLibRoot(major, minor) }
/** Holds if this module is in the standard library */
predicate inStdLib() {
this.getLoadPath().isStdLibRoot()
}
predicate inStdLib() { this.getLoadPath().isStdLibRoot() }
override
predicate containsInScope(AstNode inner) {
Scope.super.containsInScope(inner)
}
override predicate containsInScope(AstNode inner) { Scope.super.containsInScope(inner) }
override
predicate contains(AstNode inner) {
Scope.super.contains(inner)
}
override predicate contains(AstNode inner) { Scope.super.contains(inner) }
/** Gets the kind of this module. */
override string getKind() {
if this.isPackage() then
result = "Package"
if this.isPackage()
then result = "Package"
else (
not exists(Module_.super.getKind()) and result = "Module"
or
result = Module_.super.getKind()
)
}
}
bindingset[name]
private predicate legalDottedName(string name) {
name.regexpMatch("(\\p{L}|_)(\\p{L}|\\d|_)*(\\.(\\p{L}|_)(\\p{L}|\\d|_)*)*")
}
bindingset[name]
private predicate legalShortName(string name) {
name.regexpMatch("(\\p{L}|_)(\\p{L}|\\d|_)*")
}
private predicate legalShortName(string name) { name.regexpMatch("(\\p{L}|_)(\\p{L}|\\d|_)*") }
/** Holds if `f` is potentially a source package.
/**
* Holds if `f` is potentially a source package.
* Does it have an __init__.py file (or --respect-init=False for Python 2) and is it within the source archive?
*/
private predicate isPotentialSourcePackage(Folder f) {
@@ -240,8 +208,12 @@ string moduleNameFromFile(Container file) {
result = moduleNameFromFile(file.getParent()) + "." + basename
)
or
isPotentialSourcePackage(file) and result = file.getStem() and
(not isPotentialSourcePackage(file.getParent()) or not legalShortName(file.getParent().getBaseName()))
isPotentialSourcePackage(file) and
result = file.getStem() and
(
not isPotentialSourcePackage(file.getParent()) or
not legalShortName(file.getParent().getBaseName())
)
or
result = file.getStem() and file.getParent() = file.getImportRoot()
or
@@ -253,8 +225,8 @@ private predicate isStubRoot(Folder f) {
f.getAbsolutePath().matches("%/data/python/stubs")
}
/** Holds if the Container `c` should be the preferred file or folder for
/**
* Holds if the Container `c` should be the preferred file or folder for
* the given name when performing imports.
* Trivially true for any container if it is the only one with its name.
* However, if there are several modules with the same name, then

171
python/ql/src/semmle/python/Operations.qll
View File

@@ -2,123 +2,86 @@ import python
/** Base class for operators */
class Operator extends Operator_ {
/** Gets the name of the special method used to implement this operator */
string getSpecialMethodName() { none() }
}
/* Unary Expression and its operators */
/** A unary expression: (`+x`), (`-x`) or (`~x`) */
class UnaryExpr extends UnaryExpr_ {
override Expr getASubExpression() {
result = this.getOperand()
}
override Expr getASubExpression() { result = this.getOperand() }
}
/** A unary operator: `+`, `-`, `~` or `not` */
class Unaryop extends Unaryop_ {
/** Gets the name of the special method used to implement this operator */
string getSpecialMethodName() { none() }
}
/** An invert (`~`) unary operator */
class Invert extends Invert_ {
override string getSpecialMethodName() { result = "__invert__" }
}
/** A positive (`+`) unary operator */
/** A positive (`+`) unary operator */
class UAdd extends UAdd_ {
override string getSpecialMethodName() { result = "__pos__" }
}
/** A negation (`-`) unary operator */
class USub extends USub_ {
override string getSpecialMethodName() { result = "__neg__" }
}
/** A `not` unary operator */
class Not extends Not_ {
override string getSpecialMethodName() { none() }
}
/* Binary Operation and its operators */
/** A binary expression, such as `x + y` */
class BinaryExpr extends BinaryExpr_ {
override Expr getASubExpression() {
result = this.getLeft() or result = this.getRight()
}
override Expr getASubExpression() { result = this.getLeft() or result = this.getRight() }
}
/** A power (`**`) binary operator */
class Pow extends Pow_ {
override string getSpecialMethodName() { result = "__pow__" }
}
/** A right shift (`>>`) binary operator */
class RShift extends RShift_ {
override string getSpecialMethodName() { result = "__rshift__" }
}
/** A subtract (`-`) binary operator */
class Sub extends Sub_ {
override string getSpecialMethodName() { result = "__sub__" }
}
/** A bitwise and (`&`) binary operator */
class BitAnd extends BitAnd_ {
override string getSpecialMethodName() { result = "__and__" }
}
/** A bitwise or (`|`) binary operator */
class BitOr extends BitOr_ {
override string getSpecialMethodName() { result = "__or__" }
}
/** A bitwise exclusive-or (`^`) binary operator */
class BitXor extends BitXor_ {
override string getSpecialMethodName() { result = "__xor__" }
}
/** An add (`+`) binary operator */
class Add extends Add_ {
override string getSpecialMethodName() { result = "__add__" }
}
/** An (true) divide (`/`) binary operator */
class Div extends Div_ {
override string getSpecialMethodName() {
override string getSpecialMethodName() {
result = "__truediv__"
or
major_version() = 2 and result = "__div__"
@@ -127,185 +90,123 @@ class Div extends Div_ {
/** An floor divide (`//`) binary operator */
class FloorDiv extends FloorDiv_ {
override string getSpecialMethodName() { result = "__floordiv__" }
}
/** A left shift (`<<`) binary operator */
class LShift extends LShift_ {
override string getSpecialMethodName() { result = "__lshift__" }
}
/** A modulo (`%`) binary operator, which includes string formatting */
class Mod extends Mod_ {
override string getSpecialMethodName() { result = "__mod__" }
}
/** A multiplication (`*`) binary operator */
class Mult extends Mult_ {
override string getSpecialMethodName() { result = "__mul__" }
}
/** A matrix multiplication (`@`) binary operator */
class MatMult extends MatMult_ {
override string getSpecialMethodName() { result = "__matmul__" }
}
/* Comparison Operation and its operators */
/** A comparison operation, such as `x<y` */
class Compare extends Compare_ {
override Expr getASubExpression() { result = this.getLeft() or result = this.getAComparator() }
override Expr getASubExpression() {
result = this.getLeft() or result = this.getAComparator()
}
/** Whether as part of this comparison 'left' is compared with 'right' using the operator 'op'.
* For example, the comparison `a<b<c` compares(`a`, `b`, `<`) and compares(`b`, `c`, `<`). */
predicate compares(Expr left, Cmpop op, Expr right)
{
/**
* Whether as part of this comparison 'left' is compared with 'right' using the operator 'op'.
* For example, the comparison `a<b<c` compares(`a`, `b`, `<`) and compares(`b`, `c`, `<`).
*/
predicate compares(Expr left, Cmpop op, Expr right) {
this.getLeft() = left and this.getComparator(0) = right and op = this.getOp(0)
or
exists(int n | this.getComparator(n) = left and this.getComparator(n+1) = right and op = this.getOp(n+1))
exists(int n |
this.getComparator(n) = left and this.getComparator(n + 1) = right and op = this.getOp(n + 1)
)
}
}
/** List of comparison operators in a comparison */
class CmpopList extends CmpopList_ {
}
class CmpopList extends CmpopList_ { }
/** A comparison operator */
abstract class Cmpop extends Cmpop_ {
string getSymbol() {
none()
}
string getSymbol() { none() }
string getSpecialMethodName() { none() }
}
/** A greater than (`>`) comparison operator */
class Gt extends Gt_ {
override string getSymbol() {
result = ">"
}
override string getSymbol() { result = ">" }
override string getSpecialMethodName() { result = "__gt__" }
}
/** A greater than or equals (`>=`) comparison operator */
class GtE extends GtE_ {
override string getSymbol() {
result = ">="
}
override string getSymbol() { result = ">=" }
override string getSpecialMethodName() { result = "__ge__" }
}
/** An `in` comparison operator */
class In extends In_ {
override string getSymbol() {
result = "in"
}
override string getSymbol() { result = "in" }
}
/** An `is` comparison operator */
class Is extends Is_ {
override string getSymbol() {
result = "is"
}
override string getSymbol() { result = "is" }
}
/** An `is not` comparison operator */
class IsNot extends IsNot_ {
override string getSymbol() {
result = "is not"
}
override string getSymbol() { result = "is not" }
}
/** An equals (`==`) comparison operator */
class Eq extends Eq_ {
override string getSymbol() {
result = "=="
}
override string getSymbol() { result = "==" }
override string getSpecialMethodName() { result = "__eq__" }
}
/** A less than (`<`) comparison operator */
class Lt extends Lt_ {
override string getSymbol() {
result = "<"
}
override string getSymbol() { result = "<" }
override string getSpecialMethodName() { result = "__lt__" }
}
/** A less than or equals (`<=`) comparison operator */
class LtE extends LtE_ {
override string getSymbol() {
result = "<="
}
override string getSymbol() { result = "<=" }
override string getSpecialMethodName() { result = "__le__" }
}
/** A not equals (`!=`) comparison operator */
class NotEq extends NotEq_ {
override string getSymbol() {
result = "!="
}
override string getSymbol() { result = "!=" }
override string getSpecialMethodName() { result = "__ne__" }
}
/** An `not in` comparison operator */
class NotIn extends NotIn_ {
override string getSymbol() {
result = "not in"
}
override string getSymbol() { result = "not in" }
}
/* Boolean Operation (and/or) and its operators */
/** A boolean shortcut (and/or) operation */
class BoolExpr extends BoolExpr_ {
override Expr getASubExpression() {
result = this.getAValue()
}
override Expr getASubExpression() { result = this.getAValue() }
string getOperator() {
this.getOp() instanceof And and result = "and"
@@ -315,30 +216,24 @@ class BoolExpr extends BoolExpr_ {
/** Whether part evaluates to partIsTrue if this evaluates to wholeIsTrue */
predicate impliesValue(Expr part, boolean partIsTrue, boolean wholeIsTrue) {
if this.getOp() instanceof And then (
if this.getOp() instanceof And
then (
wholeIsTrue = true and partIsTrue = true and part = this.getAValue()
or
wholeIsTrue = true and ((BoolExpr)this.getAValue()).impliesValue(part, partIsTrue, true)
wholeIsTrue = true and this.getAValue().(BoolExpr).impliesValue(part, partIsTrue, true)
) else (
wholeIsTrue = false and partIsTrue = false and part = this.getAValue()
or
wholeIsTrue = false and ((BoolExpr)this.getAValue()).impliesValue(part, partIsTrue, false)
wholeIsTrue = false and this.getAValue().(BoolExpr).impliesValue(part, partIsTrue, false)
)
}
}
/** A short circuit boolean operator, and/or */
class Boolop extends Boolop_ {
}
class Boolop extends Boolop_ { }
/** An `and` boolean operator */
class And extends And_ {
}
class And extends And_ { }
/** An `or` boolean operator */
class Or extends Or_ {
}
class Or extends Or_ { }

123
python/ql/src/semmle/python/SSA.qll
View File

@@ -2,40 +2,33 @@
import python
/** A single static assignment variable.
/**
* A single static assignment variable.
* An SSA variable is a variable which is only assigned once (statically).
* SSA variables can be defined as normal variables or by a phi node which can occur at joins in the flow graph.
* Definitions without uses do not have a SSA variable.
*/
class SsaVariable extends @py_ssa_var{
SsaVariable() {
py_ssa_var(this, _)
}
class SsaVariable extends @py_ssa_var {
SsaVariable() { py_ssa_var(this, _) }
/** Gets the source variable */
Variable getVariable() {
py_ssa_var(this, result)
}
Variable getVariable() { py_ssa_var(this, result) }
/** Gets a use of this variable */
ControlFlowNode getAUse() {
py_ssa_use(result, this)
}
ControlFlowNode getAUse() { py_ssa_use(result, this) }
/** Gets the definition (which may be a deletion) of this SSA variable */
ControlFlowNode getDefinition() {
py_ssa_defn(this, result)
}
ControlFlowNode getDefinition() { py_ssa_defn(this, result) }
/** Gets an argument of the phi function defining this variable.
/**
* Gets an argument of the phi function defining this variable.
* This predicate uses the raw SSA form produced by the extractor.
* In general, you should use `getAPrunedPhiInput()` instead. */
SsaVariable getAPhiInput() {
py_ssa_phi(this, result)
}
* In general, you should use `getAPrunedPhiInput()` instead.
*/
SsaVariable getAPhiInput() { py_ssa_phi(this, result) }
/** Gets the edge(s) (result->this.getDefinition()) on which the SSA variable 'input' defines this SSA variable.
/**
* Gets the edge(s) (result->this.getDefinition()) on which the SSA variable 'input' defines this SSA variable.
* For each incoming edge `X->B`, where `B` is the basic block containing this phi-node, only one of the input SSA variables
* for this phi-node is live. This predicate returns the predecessor block such that the variable 'input'
* is the live variable on the edge result->B.
@@ -44,23 +37,25 @@ class SsaVariable extends @py_ssa_var{
input = this.getAPhiInput() and
result = this.getAPredecessorBlockForPhi() and
input.getDefinition().getBasicBlock().dominates(result) and
/* Beware the case where an SSA variable that is an input on one edge dominates another edge.
/*
* Beware the case where an SSA variable that is an input on one edge dominates another edge.
* Consider (in SSA form):
* x0 = 0
* if cond:
* x1 = 1
* x2 = phi(x0, x1)
* use(x2)
*
*
* The definition of x0 dominates the exit from the block x1=1, even though it does not reach it.
* Hence we need to check that no other definition dominates the edge and actually reaches it.
* Note that if a dominates c and b dominates c, then either a dominates b or vice-versa.
*/
not exists(SsaVariable other, BasicBlock other_def |
not other = input and
other = this.getAPhiInput() and
other_def = other.getDefinition().getBasicBlock()
|
|
other_def.dominates(result) and
input.getDefinition().getBasicBlock().strictlyDominates(other_def)
)
@@ -81,22 +76,16 @@ class SsaVariable extends @py_ssa_var{
result = this.getAPhiInput().getAnUltimateDefinition()
}
string toString() {
result = "SSA Variable " + this.getId()
}
string toString() { result = "SSA Variable " + this.getId() }
Location getLocation() {
result = this.getDefinition().getLocation()
}
Location getLocation() { result = this.getDefinition().getLocation() }
/** Gets the id (name) of this variable */
string getId() {
result = this.getVariable().getId()
}
string getId() { result = this.getVariable().getId() }
/** Gets the incoming edges for a Phi node. */
private BasicBlock getAPredecessorBlockForPhi() {
exists(getAPhiInput()) and
exists(getAPhiInput()) and
result.getASuccessor() = this.getDefinition().getBasicBlock()
}
@@ -112,11 +101,13 @@ class SsaVariable extends @py_ssa_var{
or
exists(SsaVariable var | var = this.getAPhiInput() | var.reachableWithoutDefinition())
or
/* For phi-nodes, there must be a corresponding phi-input for each control-flow
/*
* For phi-nodes, there must be a corresponding phi-input for each control-flow
* predecessor. Otherwise, the variable will be undefined on that incoming edge.
* WARNING: the same phi-input may cover multiple predecessors, so this check
* cannot be done by counting.
*/
exists(BasicBlock incoming |
incoming = this.getAPredecessorBlockForPhi() and
not this.getAPhiInput().getDefinition().getBasicBlock().dominates(incoming)
@@ -131,11 +122,13 @@ class SsaVariable extends @py_ssa_var{
or
exists(SsaVariable var | var = this.getAPrunedPhiInput() | var.maybeUndefined())
or
/* For phi-nodes, there must be a corresponding phi-input for each control-flow
/*
* For phi-nodes, there must be a corresponding phi-input for each control-flow
* predecessor. Otherwise, the variable will be undefined on that incoming edge.
* WARNING: the same phi-input may cover multiple predecessors, so this check
* cannot be done by counting.
*/
exists(BasicBlock incoming |
reaches_end(incoming) and
incoming = this.getAPrunedPredecessorBlockForPhi() and
@@ -144,14 +137,17 @@ class SsaVariable extends @py_ssa_var{
}
private predicate implicitlyDefined() {
not exists(this.getDefinition()) and not py_ssa_phi(this, _) and
not exists(this.getDefinition()) and
not py_ssa_phi(this, _) and
exists(GlobalVariable var | this.getVariable() = var |
globallyDefinedName(var.getId()) or
var.getId() = "__path__" and ((Module)var.getScope()).isPackageInit()
globallyDefinedName(var.getId())
or
var.getId() = "__path__" and var.getScope().(Module).isPackageInit()
)
}
/** Gets the global variable that is accessed if this local is undefined.
/**
* Gets the global variable that is accessed if this local is undefined.
* Only applies to local variables in class scopes.
*/
GlobalVariable getFallbackGlobal() {
@@ -163,28 +159,26 @@ class SsaVariable extends @py_ssa_var{
)
}
/* Whether this SSA variable is the first parameter of a method
/*
* Whether this SSA variable is the first parameter of a method
* (regardless of whether it is actually called self or not)
*/
predicate isSelf() {
exists(Function func |
func.isMethod()
and
exists(Function func |
func.isMethod() and
this.getDefinition().getNode() = func.getArg(0)
)
}
}
private predicate reaches_end(BasicBlock b) {
not exits_early(b)
and
not exits_early(b) and
(
/* Entry point */
not exists(BasicBlock prev | prev.getASuccessor() = b)
or
exists(BasicBlock prev | prev.getASuccessor() = b |
reaches_end(prev)
)
/* Entry point */
not exists(BasicBlock prev | prev.getASuccessor() = b)
or
exists(BasicBlock prev | prev.getASuccessor() = b | reaches_end(prev))
)
}
@@ -209,32 +203,19 @@ private predicate builtin_constant(string name) {
}
private predicate auto_name(string name) {
name = "__file__" or name = "__builtins__" or name = "__name__"
name = "__file__" or name = "__builtins__" or name = "__name__"
}
/** Whether this name is (almost) always defined, ie. it is a builtin or VM defined name */
predicate globallyDefinedName(string name) {
builtin_constant(name) or auto_name(name)
}
predicate globallyDefinedName(string name) { builtin_constant(name) or auto_name(name) }
/** An SSA variable that is backed by a global variable */
class GlobalSsaVariable extends EssaVariable {
GlobalSsaVariable() { this.getSourceVariable() instanceof GlobalVariable }
GlobalSsaVariable() {
this.getSourceVariable() instanceof GlobalVariable
}
GlobalVariable getVariable() {
result = this.getSourceVariable()
}
string getId() {
result = this.getVariable().getId()
}
override string toString() {
result = "GSSA Variable " + this.getId()
}
GlobalVariable getVariable() { result = this.getSourceVariable() }
string getId() { result = this.getVariable().getId() }
override string toString() { result = "GSSA Variable " + this.getId() }
}

118
python/ql/src/semmle/python/Scope.qll
View File

@@ -1,80 +1,58 @@
import python
/** A Scope. A scope is the lexical extent over which all identifiers with the same name refer to the same variable.
/**
* A Scope. A scope is the lexical extent over which all identifiers with the same name refer to the same variable.
* Modules, Classes and Functions are all Scopes. There are no other scopes.
* The scopes for expressions that create new scopes, lambdas and comprehensions, are handled by creating an anonymous Function. */
* The scopes for expressions that create new scopes, lambdas and comprehensions, are handled by creating an anonymous Function.
*/
class Scope extends Scope_ {
Module getEnclosingModule() { result = this.getEnclosingScope().getEnclosingModule() }
Module getEnclosingModule() {
result = this.getEnclosingScope().getEnclosingModule()
}
/** This method will be deprecated in the next release. Please use `getEnclosingScope()` instead.
/**
* This method will be deprecated in the next release. Please use `getEnclosingScope()` instead.
* The reason for this is to avoid confusion around use of `x.getScope+()` where `x` might be an
* `AstNode` or a `Variable`. Forcing the users to write `x.getScope().getEnclosingScope*()` ensures that
* the apparent semantics and the actual semantics coincide.
* [ Gets the scope enclosing this scope (modules have no enclosing scope) ]
*/
Scope getScope() {
none()
}
Scope getScope() { none() }
/** Gets the scope enclosing this scope (modules have no enclosing scope) */
Scope getEnclosingScope() {
none()
}
Scope getEnclosingScope() { none() }
/** Gets the statements forming the body of this scope */
StmtList getBody() {
none()
}
StmtList getBody() { none() }
/** Gets the nth statement of this scope */
Stmt getStmt(int n) {
none()
}
Stmt getStmt(int n) { none() }
/** Gets a top-level statement in this scope */
Stmt getAStmt() {
none()
}
Stmt getAStmt() { none() }
Location getLocation() {
none()
}
Location getLocation() { none() }
/** Gets the name of this scope */
string getName() {
py_strs(result, this, 0)
}
string getName() { py_strs(result, this, 0) }
/** Gets the docstring for this scope */
StrConst getDocString() {
result = ((ExprStmt)this.getStmt(0)).getValue()
}
StrConst getDocString() { result = this.getStmt(0).(ExprStmt).getValue() }
/** Gets the entry point into this Scope's control flow graph */
ControlFlowNode getEntryNode() {
py_scope_flow(result, this, -1)
}
ControlFlowNode getEntryNode() { py_scope_flow(result, this, -1) }
/** Gets the non-explicit exit from this Scope's control flow graph */
ControlFlowNode getFallthroughNode() {
py_scope_flow(result, this, 0)
}
ControlFlowNode getFallthroughNode() { py_scope_flow(result, this, 0) }
/** Gets the exit of this scope following from a return statement */
ControlFlowNode getReturnNode() {
py_scope_flow(result, this, 2)
}
ControlFlowNode getReturnNode() { py_scope_flow(result, this, 2) }
/** Gets an exit from this Scope's control flow graph */
ControlFlowNode getAnExitNode() {
exists (int i | py_scope_flow(result, this, i) and i >= 0)
}
ControlFlowNode getAnExitNode() { exists(int i | py_scope_flow(result, this, i) and i >= 0) }
/** Gets an exit from this Scope's control flow graph,
* that does not result from an exception */
/**
* Gets an exit from this Scope's control flow graph,
* that does not result from an exception
*/
ControlFlowNode getANormalExit() {
result = this.getFallthroughNode()
or
@@ -82,9 +60,7 @@ class Scope extends Scope_ {
}
/** Holds if this a top-level (non-nested) class or function */
predicate isTopLevel() {
this.getEnclosingModule() = this.getEnclosingScope()
}
predicate isTopLevel() { this.getEnclosingModule() = this.getEnclosingScope() }
/** Holds if this scope is deemed to be public */
predicate isPublic() {
@@ -93,10 +69,9 @@ class Scope extends Scope_ {
/* Not implicitly private */
this.getName().charAt(0) != "_" and
(
this instanceof Module
this instanceof Module
or
exists(Module m |
m = this.getEnclosingScope() and m.isPublic() |
exists(Module m | m = this.getEnclosingScope() and m.isPublic() |
/* If the module has an __all__, is this in it */
not exists(m.getAnExport())
or
@@ -116,66 +91,63 @@ class Scope extends Scope_ {
exists(Scope inner | inner.getEnclosingScope() = this | inner.contains(a))
}
/** Holds if this scope can be expected to execute before `other`.
/**
* Holds if this scope can be expected to execute before `other`.
* Modules precede functions and methods in those modules
* `__init__` precedes other methods. `__enter__` precedes `__exit__`.
* NOTE that this is context-insensitive, so a module "precedes" a function
* NOTE that this is context-insensitive, so a module "precedes" a function
* in that module, even if that function is called from the module scope.
*/
predicate precedes(Scope other) {
exists(Function f, string name |
f = other and name = f.getName() |
if f.isMethod() then (
exists(Function f, string name | f = other and name = f.getName() |
if f.isMethod()
then
// The __init__ method is preceded by the enclosing module
this = f.getEnclosingModule() and name = "__init__"
or
exists(Class c, string pred_name |
// __init__ -> __enter__ -> __exit__
// __init__ -> other-methods
f.getScope() = c and (
f.getScope() = c and
(
pred_name = "__init__" and not name = "__init__" and not name = "__exit__"
or
pred_name = "__enter__" and name = "__exit__"
)
|
|
this.getScope() = c and
pred_name = this.(Function).getName()
or
not exists(Function pre_func |
pre_func.getName() = pred_name and
pre_func.getScope() = c
) and this = other.getEnclosingModule()
) and
this = other.getEnclosingModule()
)
) else (
else
// Normal functions are preceded by the enclosing module
this = f.getEnclosingModule()
)
)
}
/** Gets the evaluation scope for code in this (lexical) scope.
/**
* Gets the evaluation scope for code in this (lexical) scope.
* This is usually the scope itself, but may be an enclosing scope.
* Notably, for list comprehensions in Python 2.
*/
Scope getEvaluatingScope() {
result = this
}
Scope getEvaluatingScope() { result = this }
/** Holds if this scope is in the source archive,
/**
* Holds if this scope is in the source archive,
* that is it is part of the code specified, not library code
*/
predicate inSource() {
exists(this.getEnclosingModule().getFile().getRelativePath())
}
predicate inSource() { exists(this.getEnclosingModule().getFile().getRelativePath()) }
Stmt getLastStatement() {
result = this.getBody().getLastItem().getLastStatement()
}
Stmt getLastStatement() { result = this.getBody().getLastItem().getLastStatement() }
/** Whether this contains `inner` syntactically and `inner` has the same scope as `this` */
predicate containsInScope(AstNode inner) {
this.getBody().contains(inner) and
this = inner.getScope()
}
}

54
python/ql/src/semmle/python/SelfAttribute.qll
View File

@@ -1,23 +1,19 @@
/** Utilities to support queries about instance attribute accesses of
/**
* Utilities to support queries about instance attribute accesses of
* the form `self.attr`.
*/
import python
private import semmle.python.pointsto.Filters
/** An attribute access where the left hand side of the attribute expression
* is `self`.
*/
/**
* An attribute access where the left hand side of the attribute expression
* is `self`.
*/
class SelfAttribute extends Attribute {
SelfAttribute() { self_attribute(this, _) }
SelfAttribute() {
self_attribute(this, _)
}
Class getClass() {
self_attribute(this, result)
}
Class getClass() { self_attribute(this, result) }
}
/** Whether variable 'self' is the self variable in method 'method' */
@@ -29,8 +25,7 @@ private predicate self_variable(Function method, Variable self) {
/** Whether attribute is an access of the form `self.attr` in the body of the class 'cls' */
private predicate self_attribute(Attribute attr, Class cls) {
exists(Function f, Variable self |
self_variable(f, self) |
exists(Function f, Variable self | self_variable(f, self) |
self.getAnAccess() = attr.getObject() and
cls = f.getScope+()
)
@@ -38,14 +33,12 @@ private predicate self_attribute(Attribute attr, Class cls) {
/** Helper class for UndefinedClassAttribute.ql &amp; MaybeUndefinedClassAttribute.ql */
class SelfAttributeRead extends SelfAttribute {
SelfAttributeRead() {
this.getCtx() instanceof Load and
// Be stricter for loads.
// Be stricter for loads.
// We want to generous as to what is defined (i.e. stores),
// but strict as to what needs to be defined (i.e. loads).
exists(ClassObject cls, FunctionObject func |
cls.declaredAttribute(_) = func |
exists(ClassObject cls, FunctionObject func | cls.declaredAttribute(_) = func |
func.getFunction() = this.getScope() and
cls.getPyClass() = this.getClass()
)
@@ -59,35 +52,28 @@ class SelfAttributeRead extends SelfAttribute {
)
}
pragma [noinline] predicate locallyDefined() {
pragma[noinline]
predicate locallyDefined() {
exists(SelfAttributeStore store |
this.getName() = store.getName() and
this.getScope() = store.getScope() |
this.getName() = store.getName() and
this.getScope() = store.getScope()
|
store.getAFlowNode().strictlyDominates(this.getAFlowNode())
)
}
}
class SelfAttributeStore extends SelfAttribute {
SelfAttributeStore() { this.getCtx() instanceof Store }
SelfAttributeStore() {
this.getCtx() instanceof Store
}
Expr getAssignedValue() {
exists(Assign a | a.getATarget() = this |
result = a.getValue()
)
}
Expr getAssignedValue() { exists(Assign a | a.getATarget() = this | result = a.getValue()) }
}
private predicate attr_assigned_in_method_arg_n(FunctionObject method, string name, int n) {
exists(SsaVariable param |
method.getFunction().getArg(n).asName() = param.getDefinition().getNode()
|
exists(AttrNode attr |
|
exists(AttrNode attr |
attr.getObject(name) = param.getAUse() and
attr.isStore()
)

338
python/ql/src/semmle/python/Stmts.qll
View File

@@ -2,34 +2,21 @@ import python
/** A statement */
class Stmt extends Stmt_, AstNode {
/** Gets the scope immediately enclosing this statement */
override Scope getScope() {
py_scopes(this, result)
}
override Scope getScope() { py_scopes(this, result) }
override string toString() {
result = "Stmt"
}
override string toString() { result = "Stmt" }
/** Gets the module enclosing this statement */
Module getEnclosingModule() {
result = this.getScope().getEnclosingModule()
}
Module getEnclosingModule() { result = this.getScope().getEnclosingModule() }
override Location getLocation() {
result = Stmt_.super.getLocation()
}
override Location getLocation() { result = Stmt_.super.getLocation() }
/** Gets an immediate (non-nested) sub-expression of this statement */
Expr getASubExpression() {
none()
}
Expr getASubExpression() { none() }
/** Gets an immediate (non-nested) sub-statement of this statement */
Stmt getASubStatement() {
none()
}
Stmt getASubStatement() { none() }
override AstNode getAChildNode() {
result = this.getASubExpression()
@@ -42,8 +29,8 @@ class Stmt extends Stmt_, AstNode {
this.containsInScope(result.getNode())
}
/** Gets a control flow node for an entry into this statement.
/**
* Gets a control flow node for an entry into this statement.
*/
ControlFlowNode getAnEntryNode() {
result = this.possibleEntryNode() and
@@ -60,131 +47,94 @@ class Stmt extends Stmt_, AstNode {
exists(If ifstmt |
ifstmt.getTest().(ImmutableLiteral).booleanValue() = false and ifstmt.getBody().contains(this)
or
ifstmt.getTest().(ImmutableLiteral).booleanValue() = true and ifstmt.getOrelse().contains(this)
ifstmt.getTest().(ImmutableLiteral).booleanValue() = true and
ifstmt.getOrelse().contains(this)
)
or
exists(While whilestmt |
whilestmt.getTest().(ImmutableLiteral).booleanValue() = false and whilestmt.getBody().contains(this)
whilestmt.getTest().(ImmutableLiteral).booleanValue() = false and
whilestmt.getBody().contains(this)
)
}
/** Gets the final statement in this statement, ordered by location.
/**
* Gets the final statement in this statement, ordered by location.
* Will be this statement if not a compound statement.
*/
Stmt getLastStatement() {
result = this
}
Stmt getLastStatement() { result = this }
}
/** A statement that includes a binding (except imports) */
class Assign extends Assign_ {
/** Use ControlFlowNodes and SsaVariables for data-flow analysis. */
predicate defines(Variable v) {
this.getATarget().defines(v)
}
predicate defines(Variable v) { this.getATarget().defines(v) }
override Expr getASubExpression() {
result = this.getATarget() or
result = this.getValue()
}
override Stmt getASubStatement() {
none()
}
override Stmt getASubStatement() { none() }
}
/** An assignment statement */
class AssignStmt extends Assign {
/* syntax: Expr, ... = Expr */
AssignStmt() { not this instanceof FunctionDef and not this instanceof ClassDef }
AssignStmt() {
not this instanceof FunctionDef and not this instanceof ClassDef
}
override string toString() {
result = "AssignStmt"
}
override string toString() { result = "AssignStmt" }
}
/** An augmented assignment statement, such as `x += y` */
class AugAssign extends AugAssign_ {
/* syntax: Expr += Expr */
override Expr getASubExpression() { result = this.getOperation() }
override Expr getASubExpression() {
result = this.getOperation()
}
Expr getTarget() { result = this.getOperation().(BinaryExpr).getLeft() }
Expr getTarget() {
result = ((BinaryExpr)this.getOperation()).getLeft()
}
Expr getValue() { result = this.getOperation().(BinaryExpr).getRight() }
Expr getValue() {
result = ((BinaryExpr)this.getOperation()).getRight()
}
override Stmt getASubStatement() {
none()
}
override Stmt getASubStatement() { none() }
}
/** An annotated assignment statement, such as `x: int = 0` */
class AnnAssign extends AnnAssign_ {
/* syntax: Expr: Expr = Expr */
override Expr getASubExpression() {
result = this.getAnnotation() or
result = this.getTarget() or
result = this.getValue()
}
override Stmt getASubStatement() {
none()
}
override Stmt getASubStatement() { none() }
/** Holds if the value of the annotation of this assignment is stored at runtime. */
predicate isStored() {
not this.getScope() instanceof Function
and
not this.getScope() instanceof Function and
exists(Name n |
n = this.getTarget()
and
n = this.getTarget() and
not n.isParenthesized()
)
}
}
/** An exec statement */
class Exec extends Exec_ {
/* syntax: exec Expr */
override Expr getASubExpression() {
result = this.getBody() or
result = this.getGlobals() or
result = this.getLocals()
}
override Stmt getASubStatement() {
none()
}
override Stmt getASubStatement() { none() }
}
/** An except statement (part of a `try` statement), such as `except IOError as err:` */
class ExceptStmt extends ExceptStmt_ {
/* syntax: except Expr [ as Expr ]: */
/** Gets the immediately enclosing try statement */
Try getTry() {
result.getAHandler() = this
}
Try getTry() { result.getAHandler() = this }
override Expr getASubExpression() {
result = this.getName()
@@ -192,96 +142,54 @@ class ExceptStmt extends ExceptStmt_ {
result = this.getType()
}
override Stmt getASubStatement() {
result = this.getAStmt()
}
override Stmt getLastStatement() {
result = this.getBody().getLastItem().getLastStatement()
}
override Stmt getASubStatement() { result = this.getAStmt() }
override Stmt getLastStatement() { result = this.getBody().getLastItem().getLastStatement() }
}
/** An assert statement, such as `assert a == b, "A is not equal to b"` */
class Assert extends Assert_ {
/* syntax: assert Expr [, Expr] */
override Expr getASubExpression() { result = this.getMsg() or result = this.getTest() }
override Expr getASubExpression() {
result = this.getMsg() or result = this.getTest()
}
override Stmt getASubStatement() {
none()
}
override Stmt getASubStatement() { none() }
}
/** A break statement */
class Break extends Break_ {
/* syntax: assert Expr [, Expr] */
override Expr getASubExpression() { none() }
override Expr getASubExpression() {
none()
}
override Stmt getASubStatement() {
none()
}
override Stmt getASubStatement() { none() }
}
/** A continue statement */
class Continue extends Continue_ {
/* syntax: continue */
override Expr getASubExpression() { none() }
override Expr getASubExpression() {
none()
}
override Stmt getASubStatement() {
none()
}
override Stmt getASubStatement() { none() }
}
/** A delete statement, such as `del x[-1]` */
class Delete extends Delete_ {
/* syntax: del Expr, ... */
override Expr getASubExpression() { result = this.getATarget() }
override Expr getASubExpression() {
result = this.getATarget()
}
override Stmt getASubStatement() {
none()
}
override Stmt getASubStatement() { none() }
}
/** An expression statement, such as `len(x)` or `yield y` */
class ExprStmt extends ExprStmt_ {
/* syntax: Expr */
override Expr getASubExpression() { result = this.getValue() }
override Expr getASubExpression() {
result = this.getValue()
}
override Stmt getASubStatement() {
none()
}
override Stmt getASubStatement() { none() }
}
/** A for statement, such as `for x in y: print(x)` */
class For extends For_ {
/* syntax: for varname in Expr: ... */
override Stmt getASubStatement() {
result = this.getAStmt() or
result = this.getAnOrelse()
@@ -292,39 +200,26 @@ class For extends For_ {
result = this.getIter()
}
override Stmt getLastStatement() {
result = this.getBody().getLastItem().getLastStatement()
}
override Stmt getLastStatement() { result = this.getBody().getLastItem().getLastStatement() }
}
/** A global statement, such as `global var` */
class Global extends Global_ {
/* syntax: global varname */
override Expr getASubExpression() { none() }
override Expr getASubExpression() {
none()
}
override Stmt getASubStatement() {
none()
}
override Stmt getASubStatement() { none() }
}
/** An if statement, such as `if eggs: print("spam")` */
class If extends If_ {
/* syntax: if Expr: ... */
override Stmt getASubStatement() {
result = this.getAStmt() or
result = this.getAnOrelse()
}
override Expr getASubExpression() {
result = this.getTest()
}
override Expr getASubExpression() { result = this.getTest() }
/** Whether this if statement takes the form `if __name__ == "__main__":` */
predicate isNameEqMain() {
@@ -343,11 +238,13 @@ class If extends If_ {
/** Whether this if statement starts with the keyword `elif` */
predicate isElif() {
/* The Python parser turns all elif chains into nested if-else statements.
/*
* The Python parser turns all elif chains into nested if-else statements.
* An `elif` can be identified as it is the first statement in an `else` block
* and it is not indented relative to its parent `if`.
*/
exists(If i |
exists(If i |
i.getOrelse(0) = this and
this.getLocation().getStartColumn() = i.getLocation().getStartColumn()
)
@@ -365,74 +262,49 @@ class If extends If_ {
not exists(this.getOrelse()) and
result = this.getBody().getLastItem().getLastStatement()
}
}
/** A nonlocal statement, such as `nonlocal var` */
class Nonlocal extends Nonlocal_ {
/* syntax: nonlocal varname */
override Stmt getASubStatement() { none() }
override Stmt getASubStatement() {
none()
}
override Expr getASubExpression() {
none()
}
override Expr getASubExpression() { none() }
Variable getAVariable() {
result.getScope() = this.getScope() and
result.getId() = this.getAName()
}
}
/** A pass statement */
class Pass extends Pass_ {
/* syntax: pass */
override Stmt getASubStatement() { none() }
override Stmt getASubStatement() {
none()
}
override Expr getASubExpression() {
none()
}
override Expr getASubExpression() { none() }
}
/** A print statement (Python 2 only), such as `print 0` */
class Print extends Print_ {
/* syntax: print Expr, ... */
override Stmt getASubStatement() {
none()
}
override Stmt getASubStatement() { none() }
override Expr getASubExpression() {
result = this.getAValue() or
result = this.getDest()
}
}
/** A raise statement, such as `raise CompletelyDifferentException()` */
class Raise extends Raise_ {
/* syntax: raise Expr */
override Stmt getASubStatement() { none() }
override Stmt getASubStatement() {
none()
}
override Expr getASubExpression() { py_exprs(result, _, this, _) }
override Expr getASubExpression() {
py_exprs(result, _, this, _)
}
/** The expression immediately following the `raise`, this is the
/**
* The expression immediately following the `raise`, this is the
* exception raised, but not accounting for tuples in Python 2.
*/
Expr getException() {
@@ -442,12 +314,10 @@ class Raise extends Raise_ {
}
/** The exception raised, accounting for tuples in Python 2. */
Expr getRaised()
{
exists(Expr raw |
raw = this.getException() |
if (not major_version() = 2 or not exists(raw.(Tuple).getAnElt())) then
result = raw
Expr getRaised() {
exists(Expr raw | raw = this.getException() |
if not major_version() = 2 or not exists(raw.(Tuple).getAnElt())
then result = raw
else
/* In Python 2 raising a tuple will result in the first element of the tuple being raised. */
result = raw.(Tuple).getElt(0)
@@ -457,27 +327,16 @@ class Raise extends Raise_ {
/** A return statement, such as return None */
class Return extends Return_ {
/* syntax: return Expr */
override Stmt getASubStatement() { none() }
override Stmt getASubStatement() {
none()
}
override Expr getASubExpression() {
result = this.getValue()
}
override Expr getASubExpression() { result = this.getValue() }
}
/** A try statement */
class Try extends Try_ {
/* syntax: try: ... */
override Expr getASubExpression() {
none()
}
override Expr getASubExpression() { none() }
override Stmt getASubStatement() {
result = this.getAHandler() or
@@ -486,14 +345,10 @@ class Try extends Try_ {
result = this.getAnOrelse()
}
override ExceptStmt getHandler(int i) {
result = Try_.super.getHandler(i)
}
override ExceptStmt getHandler(int i) { result = Try_.super.getHandler(i) }
/** Gets an exception handler of this try statement. */
override ExceptStmt getAHandler() {
result = Try_.super.getAHandler()
}
override ExceptStmt getAHandler() { result = Try_.super.getAHandler() }
override Stmt getLastStatement() {
result = this.getFinalbody().getLastItem().getLastStatement()
@@ -501,23 +356,21 @@ class Try extends Try_ {
not exists(this.getFinalbody()) and
result = this.getOrelse().getLastItem().getLastStatement()
or
not exists(this.getFinalbody()) and not exists(this.getOrelse()) and
not exists(this.getFinalbody()) and
not exists(this.getOrelse()) and
result = this.getHandlers().getLastItem().getLastStatement()
or
not exists(this.getFinalbody()) and not exists(this.getOrelse()) and not exists(this.getHandlers()) and
not exists(this.getFinalbody()) and
not exists(this.getOrelse()) and
not exists(this.getHandlers()) and
result = this.getBody().getLastItem().getLastStatement()
}
}
/** A while statement, such as `while parrot_resting():` */
class While extends While_ {
/* syntax: while Expr: ... */
override Expr getASubExpression() {
result = this.getTest()
}
override Expr getASubExpression() { result = this.getTest() }
override Stmt getASubStatement() {
result = this.getAStmt() or
@@ -530,75 +383,44 @@ class While extends While_ {
not exists(this.getOrelse()) and
result = this.getBody().getLastItem().getLastStatement()
}
}
/** A with statement such as `with f as open("file"): text = f.read()` */
class With extends With_ {
/* syntax: with Expr as varname: ... */
override Expr getASubExpression() {
result = this.getContextExpr() or
result = this.getOptionalVars()
}
override Stmt getASubStatement() {
result = this.getAStmt()
}
override Stmt getLastStatement() {
result = this.getBody().getLastItem().getLastStatement()
}
override Stmt getASubStatement() { result = this.getAStmt() }
override Stmt getLastStatement() { result = this.getBody().getLastItem().getLastStatement() }
}
/** A plain text used in a template is wrapped in a TemplateWrite statement */
class TemplateWrite extends TemplateWrite_ {
override Expr getASubExpression() { result = this.getValue() }
override Expr getASubExpression() {
result = this.getValue()
}
override Stmt getASubStatement() {
none()
}
override Stmt getASubStatement() { none() }
}
class AsyncFor extends For {
/* syntax: async for varname in Expr: ... */
AsyncFor() {
this.isAsync()
}
AsyncFor() { this.isAsync() }
}
class AsyncWith extends With {
/* syntax: async with Expr as varname: ... */
AsyncWith() {
this.isAsync()
}
AsyncWith() { this.isAsync() }
}
/** A list of statements */
class StmtList extends StmtList_ {
/** Whether this list of statements contains s */
predicate contains(AstNode a) {
exists(Stmt item |
item = this.getAnItem() |
item = a or item.contains(a)
)
exists(Stmt item | item = this.getAnItem() | item = a or item.contains(a))
}
Stmt getLastItem() { result = this.getItem(max(int i | exists(this.getItem(i)))) }
}

33
python/ql/src/semmle/python/TestUtils.qll
View File

@@ -1,32 +1,29 @@
/* This file contains test-related utility functions */
import python
/** Removes everything up to the occurrence of `sub` in the string `str` */
bindingset[str,sub]
bindingset[str, sub]
string remove_prefix_before_substring(string str, string sub) {
exists(int index |
index = str.indexOf(sub) and
result = str.suffix(index)
)
or
not exists(str.indexOf(sub)) and
result = str
exists(int index |
index = str.indexOf(sub) and
result = str.suffix(index)
)
or
not exists(str.indexOf(sub)) and
result = str
}
/** Removes the part of the `resources/lib` Python library path that may vary
* from machine to machine. */
/**
* Removes the part of the `resources/lib` Python library path that may vary
* from machine to machine.
*/
string remove_library_prefix(Location loc) {
result = remove_prefix_before_substring(loc.toString(), "resources/lib")
result = remove_prefix_before_substring(loc.toString(), "resources/lib")
}
/** Returns the location of an AST node in compact form: `basename:line:column` */
string compact_location(AstNode a) {
exists(Location l |
l = a.getLocation() |
exists(Location l | l = a.getLocation() |
result = l.getFile().getBaseName() + ":" + l.getStartLine() + ":" + l.getStartColumn()
)
}
}

92
python/ql/src/semmle/python/Variables.qll
View File

@@ -1,24 +1,19 @@
import python
/** A variable, either a global or local variable (including parameters) */
class Variable extends @py_variable {
Variable() {
exists(string name |
variable(this, _, name) and
not name = "*" and not name = "$"
not name = "*" and
not name = "$"
)
}
/** Gets the identifier (name) of this variable */
string getId() {
variable(this, _, result)
}
string getId() { variable(this, _, result) }
string toString() {
result = "Variable " + this.getId()
}
string toString() { result = "Variable " + this.getId() }
/** Gets an access (load or store) of this variable */
Name getAnAccess() {
@@ -28,103 +23,70 @@ class Variable extends @py_variable {
}
/** Gets a load of this variable */
Name getALoad() {
result.uses(this)
}
Name getALoad() { result.uses(this) }
/** Gets a store of this variable */
Name getAStore() {
result.defines(this)
}
Name getAStore() { result.defines(this) }
/** Gets a use of this variable */
NameNode getAUse() {
result.uses(this)
}
NameNode getAUse() { result.uses(this) }
/** Gets the scope of this variable */
Scope getScope() {
variable(this, result, _)
}
Scope getScope() { variable(this, result, _) }
/** Whether there is an access to this variable outside
/**
* Whether there is an access to this variable outside
* of its own scope. Usually occurs in nested functions
* or for global variables.
*/
predicate escapes() {
exists(Name n | n = this.getAnAccess() | n.getScope() != this.getScope())
}
predicate escapes() { exists(Name n | n = this.getAnAccess() | n.getScope() != this.getScope()) }
/** Whether this variable is a parameter */
predicate isParameter() {
none()
}
predicate isSelf() {
none()
}
predicate isParameter() { none() }
predicate isSelf() { none() }
}
/** A local (function or class) variable */
class LocalVariable extends Variable {
LocalVariable() {
exists(Scope s | s = this.getScope() | s instanceof Function or s instanceof Class)
}
override string toString() {
result = "Local Variable " + this.getId()
}
override string toString() { result = "Local Variable " + this.getId() }
/** Whether this variable is a parameter */
override predicate isParameter() {
exists(Parameter p | this.getAnAccess() = p)
}
override predicate isParameter() { exists(Parameter p | this.getAnAccess() = p) }
/** Holds if this variable is the first parameter of a method. It is not necessarily called "self" */
override predicate isSelf() {
exists(Function f, Parameter self |
exists(Function f, Parameter self |
this.getAnAccess() = self and
f.isMethod() and f.getArg(0) = self
f.isMethod() and
f.getArg(0) = self
)
}
}
/** A local variable that uses "load fast" semantics, for lookup:
* If the variable is undefined, then raise an exception.
/**
* A local variable that uses "load fast" semantics, for lookup:
* If the variable is undefined, then raise an exception.
*/
class FastLocalVariable extends LocalVariable {
FastLocalVariable() {
this.getScope() instanceof FastLocalsFunction
}
FastLocalVariable() { this.getScope() instanceof FastLocalsFunction }
}
/** A local variable that uses "load name" semantics, for lookup:
/**
* A local variable that uses "load name" semantics, for lookup:
* If the variable is undefined, then lookup the value in globals().
*/
class NameLocalVariable extends LocalVariable {
NameLocalVariable() {
not this instanceof FastLocalVariable
}
NameLocalVariable() { not this instanceof FastLocalVariable }
}
/** A global (module-level) variable */
class GlobalVariable extends Variable {
GlobalVariable() { exists(Module m | m = this.getScope()) }
GlobalVariable() {
exists(Module m | m = this.getScope())
}
override string toString() {
result = "Global Variable " + this.getId()
}
override string toString() { result = "Global Variable " + this.getId() }
}

21
python/ql/src/semmle/python/dataflow/Implementation.qll
View File

@@ -57,8 +57,8 @@ private newtype TAttributePath =
* It might make sense to add another level, attribute of attribute.
* But some experimentation would be needed.
*/
TAttribute(string name) { exists(Attribute a | a.getName() = name) }
TAttribute(string name) { exists(Attribute a | a.getName() = name) }
/**
* The attribute of the tracked value holding the taint.
@@ -445,9 +445,8 @@ class TaintTrackingImplementation extends string {
context = TNoParam() and
src = TTaintTrackingNode_(retval, TNoParam(), path, kind, this) and
node.asCfgNode() = call and
retval.asCfgNode() = any(Return ret | ret.getScope() = pyfunc.getScope())
.getValue()
.getAFlowNode()
retval.asCfgNode() =
any(Return ret | ret.getScope() = pyfunc.getScope()).getValue().getAFlowNode()
) and
edgeLabel = "return"
}
@@ -469,9 +468,8 @@ class TaintTrackingImplementation extends string {
this.callContexts(call, src, pyfunc, context, callee) and
retnode = TTaintTrackingNode_(retval, callee, path, kind, this) and
node.asCfgNode() = call and
retval.asCfgNode() = any(Return ret | ret.getScope() = pyfunc.getScope())
.getValue()
.getAFlowNode()
retval.asCfgNode() =
any(Return ret | ret.getScope() = pyfunc.getScope()).getValue().getAFlowNode()
) and
edgeLabel = "call"
}
@@ -965,11 +963,12 @@ private TaintKind taint_at_depth(SequenceKind parent_kind, int depth) {
or
// recursive case
depth > 1 and
result = taint_at_depth(parent_kind.getMember(), depth-1)
result = taint_at_depth(parent_kind.getMember(), depth - 1)
)
}
/** Helper predicate for taintedMultiAssignment
/**
* Helper predicate for taintedMultiAssignment
*
* Returns the `depth` the elements that are assigned to `left_defn` with iterable unpacking has,
* compared to `left_parent`. Special care is taken for `StarredNode` that is assigned a sequence of items.
@@ -984,9 +983,7 @@ int iterable_unpacking_descent(SequenceNode left_parent, ControlFlowNode left_de
exists(Assign a | a.getATarget().getASubExpression*().getAFlowNode() = left_parent) and
left_parent.getAnElement() = left_defn and
// Handle `a, *b = some_iterable`
if left_defn instanceof StarredNode
then result = 0
else result = 1
if left_defn instanceof StarredNode then result = 0 else result = 1
or
result = 1 + iterable_unpacking_descent(left_parent.getAnElement(), left_defn)
}

78
python/ql/src/semmle/python/dependencies/Dependencies.qll
View File

@@ -1,13 +1,14 @@
import python
import semmle.python.dependencies.DependencyKind
private predicate importDependency(Object target, AstNode source) {
source.getScope() != target.getOrigin() and /* Imports of own module are ignored */
source.getScope() != target.getOrigin() and
/* Imports of own module are ignored */
(
exists(ModuleObject importee, ImportingStmt imp_stmt |
source = imp_stmt and
importee = target |
importee = target
|
exists(ImportMember im | imp_stmt.contains(im) |
importee.importedAs(im.getImportedModuleName())
)
@@ -23,11 +24,9 @@ private predicate importDependency(Object target, AstNode source) {
)
or
/* from m import name, where m.name is not a submodule */
exists(PythonModuleObject importee, ImportingStmt imp_stmt |
source = imp_stmt |
exists(PythonModuleObject importee, ImportingStmt imp_stmt | source = imp_stmt |
exists(ImportMember im | imp_stmt.contains(im) |
importee.importedAs(im.getModule().(ImportExpr).getImportedModuleName())
and
importee.importedAs(im.getModule().(ImportExpr).getImportedModuleName()) and
defn_of_module_attribute(target, importee.getModule(), im.getName())
)
)
@@ -35,16 +34,12 @@ private predicate importDependency(Object target, AstNode source) {
}
class PythonImport extends DependencyKind {
PythonImport() {
this = "import"
}
PythonImport() { this = "import" }
override predicate isADependency(AstNode source, Object target) {
this = this and
importDependency(target, source)
}
}
private predicate interesting(Object target) {
@@ -58,10 +53,7 @@ private predicate interesting(Object target) {
}
class PythonUse extends DependencyKind {
PythonUse() {
this = "use"
}
PythonUse() { this = "use" }
override predicate isADependency(AstNode source, Object target) {
interesting(target) and
@@ -71,22 +63,20 @@ class PythonUse extends DependencyKind {
use.getNode() = source and
use.refersTo(obj) and
use.isLoad()
|
|
interesting(obj) and target = obj
)
and
) and
not has_more_specific_dependency_source(source)
}
}
/** Whether there is a more specific dependency source than this one.
/**
* Whether there is a more specific dependency source than this one.
* E.g. if the expression pack.mod.func is a dependency on the function 'func' in 'pack.mod'
* don't make pack.mod depend on the module 'pack.mod'
*/
private predicate has_more_specific_dependency_source(Expr e) {
exists(Attribute member |
member.getObject() = e |
exists(Attribute member | member.getObject() = e |
attribute_access_dependency(_, member)
or
has_more_specific_dependency_source(member)
@@ -94,35 +84,25 @@ private predicate has_more_specific_dependency_source(Expr e) {
}
class PythonInheritance extends DependencyKind {
PythonInheritance() {
this = "inheritance"
}
PythonInheritance() { this = "inheritance" }
override predicate isADependency(AstNode source, Object target) {
this = this and
exists(ClassObject cls |
source = cls.getOrigin()
|
exists(ClassObject cls | source = cls.getOrigin() |
target = cls.getASuperType()
or
target = cls.getAnInferredType()
)
}
}
class PythonAttribute extends DependencyKind {
PythonAttribute() {
this = "attribute"
}
PythonAttribute() { this = "attribute" }
override predicate isADependency(AstNode source, Object target) {
this = this and
attribute_access_dependency(target, source)
}
}
private predicate attribute_access_dependency(Object target, AstNode source) {
@@ -133,30 +113,23 @@ private predicate attribute_access_dependency(Object target, AstNode source) {
}
private predicate use_of_attribute(Attribute attr, Scope s, string name) {
exists(AttrNode cfg |
cfg.isLoad() and cfg.getNode() = attr
|
exists(Object obj |
cfg.getObject(name).refersTo(obj) |
exists(AttrNode cfg | cfg.isLoad() and cfg.getNode() = attr |
exists(Object obj | cfg.getObject(name).refersTo(obj) |
s = obj.(PythonModuleObject).getModule() or
s = obj.(ClassObject).getPyClass()
) or
exists(ClassObject cls |
cfg.getObject(name).refersTo(_, cls, _) |
s = cls.getPyClass()
)
or
exists(ClassObject cls | cfg.getObject(name).refersTo(_, cls, _) | s = cls.getPyClass())
)
or
exists(SelfAttributeRead sar |
sar = attr |
exists(SelfAttributeRead sar | sar = attr |
sar.getClass() = s and
sar.getName() = name
)
}
private predicate defn_of_attribute(Object target, Scope s, string name) {
exists(Assign asgn |
target.(ControlFlowNode).getNode() = asgn |
exists(Assign asgn | target.(ControlFlowNode).getNode() = asgn |
defn_of_instance_attribute(asgn, s, name)
or
defn_of_class_attribute(asgn, s, name)
@@ -165,13 +138,14 @@ private predicate defn_of_attribute(Object target, Scope s, string name) {
defn_of_module_attribute(target, s, name)
}
/* Whether asgn defines an instance attribute, that is does
/*
* Whether asgn defines an instance attribute, that is does
* asgn take the form self.name = ... where self is an instance
* of class c and asgn is not a redefinition.
*/
private predicate defn_of_instance_attribute(Assign asgn, Class c, string name) {
exists(SelfAttributeStore sas |
asgn.getATarget() = sas |
exists(SelfAttributeStore sas | asgn.getATarget() = sas |
sas.getClass() = c and
sas.getName() = name and
not exists(SelfAttributeStore in_init |

20
python/ql/src/semmle/python/dependencies/DependencyKind.qll
View File

@@ -3,7 +3,6 @@ import semmle.python.dependencies.Dependencies
/**
* A library describing an abstract mechanism for representing dependency categories.
*/
/*
* A DependencyCategory is a unique string key used by Architect to identify different categories
* of dependencies that might be viewed independently.
@@ -12,20 +11,17 @@ import semmle.python.dependencies.Dependencies
* accepted by Architect.
* </p>
*/
abstract class DependencyKind extends string {
abstract class DependencyKind extends string {
bindingset[this]
DependencyKind() {
this = this
}
DependencyKind() { this = this }
/* Tech inventory interface */
/**
* Identify dependencies associated with this category.
* <p>
* The source element is the source of the dependency.
* </p>
*/
* Identify dependencies associated with this category.
* <p>
* The source element is the source of the dependency.
* </p>
*/
abstract predicate isADependency(AstNode source, Object target);
}
}

70
python/ql/src/semmle/python/dependencies/TechInventory.qll
View File

@@ -2,49 +2,41 @@ import python
import semmle.python.dependencies.Dependencies
import semmle.python.dependencies.DependencyKind
/** Combine the source-file and package into a single string:
/**
* Combine the source-file and package into a single string:
* /path/to/file.py<|>package-name-and-version
*/
string munge(File sourceFile, ExternalPackage package) {
result = "/" + sourceFile.getRelativePath() + "<|>" + package.getName() + "<|>" + package.getVersion() or
not exists(package.getVersion()) and result = "/" + sourceFile.getRelativePath() + "<|>" + package.getName() + "<|>unknown"
result =
"/" + sourceFile.getRelativePath() + "<|>" + package.getName() + "<|>" + package.getVersion()
or
not exists(package.getVersion()) and
result = "/" + sourceFile.getRelativePath() + "<|>" + package.getName() + "<|>unknown"
}
abstract class ExternalPackage extends Object {
ExternalPackage() {
this instanceof ModuleObject
}
ExternalPackage() { this instanceof ModuleObject }
abstract string getName();
abstract string getVersion();
Object getAttribute(string name) {
result = this.(ModuleObject).attr(name)
}
PackageObject getPackage() {
result = this.(ModuleObject).getPackage()
}
Object getAttribute(string name) { result = this.(ModuleObject).attr(name) }
PackageObject getPackage() { result = this.(ModuleObject).getPackage() }
}
bindingset[text]
private predicate is_version(string text) {
text.regexpMatch("\\d+\\.\\d+(\\.\\d+)?([ab]\\d+)?")
}
private predicate is_version(string text) { text.regexpMatch("\\d+\\.\\d+(\\.\\d+)?([ab]\\d+)?") }
bindingset[v]
private string version_format(float v) {
exists(int i, int f |
i = (v+0.05).floor() and f = ((v+0.05-i)*10).floor() |
exists(int i, int f | i = (v + 0.05).floor() and f = ((v + 0.05 - i) * 10).floor() |
result = i + "." + f
)
}
class DistPackage extends ExternalPackage {
DistPackage() {
exists(Folder parent |
parent = this.(ModuleObject).getPath().getParent() and
@@ -56,12 +48,13 @@ class DistPackage extends ExternalPackage {
)
}
/* We don't extract the meta-data for dependencies (yet), so make a best guess from the source
* https://www.python.org/dev/peps/pep-0396/
/*
* We don't extract the meta-data for dependencies (yet), so make a best guess from the source
* https://www.python.org/dev/peps/pep-0396/
*/
private predicate possibleVersion(string version, int priority) {
exists(Object v |
v = this.getAttribute("__version__") and priority = 3 |
exists(Object v | v = this.getAttribute("__version__") and priority = 3 |
version = v.(StringObject).getText() and is_version(version)
or
version = version_format(v.(NumericObject).floatValue())
@@ -71,33 +64,36 @@ class DistPackage extends ExternalPackage {
or
exists(SequenceObject tuple, NumericObject major, NumericObject minor, string base_version |
this.getAttribute("version_info") = tuple and
major = tuple.getInferredElement(0) and minor = tuple.getInferredElement(1) and
base_version = major.intValue() + "." + minor.intValue() |
major = tuple.getInferredElement(0) and
minor = tuple.getInferredElement(1) and
base_version = major.intValue() + "." + minor.intValue()
|
version = base_version + "." + tuple.getBuiltinElement(2).(NumericObject).intValue()
or
not exists(tuple.getBuiltinElement(2)) and version = base_version
) and priority = 2
) and
priority = 2
or
exists(string v |
v.toLowerCase() = "version" |
exists(string v | v.toLowerCase() = "version" |
is_version(version) and
version = this.getAttribute(v).(StringObject).getText()
) and priority = 1
) and
priority = 1
}
override string getVersion() {
this.possibleVersion(result, max(int priority | this.possibleVersion(_, priority)))
}
override string getName() {
result = this.(ModuleObject).getShortName()
}
override string getName() { result = this.(ModuleObject).getShortName() }
predicate fromSource(Object src) {
exists(ModuleObject m |
m.getModule() = src.(ControlFlowNode).getEnclosingModule() or
src = m |
m = this or
src = m
|
m = this
or
m.getPackage+() = this and
not exists(DistPackage inter |
m.getPackage*() = inter and
@@ -105,12 +101,10 @@ class DistPackage extends ExternalPackage {
)
)
}
}
predicate dependency(AstNode src, DistPackage package) {
exists(DependencyKind cat, Object target |
cat.isADependency(src, target) |
exists(DependencyKind cat, Object target | cat.isADependency(src, target) |
package.fromSource(target)
)
}

134
python/ql/src/semmle/python/essa/Definitions.qll
View File

@@ -1,6 +1,7 @@
import python
/* Classification of variables. These should be non-overlapping and complete.
/*
* Classification of variables. These should be non-overlapping and complete.
*
* Function local variables - Non escaping variables in a function, except 'self'
* Self variables - The 'self' variable for a method.
@@ -11,31 +12,24 @@ import python
* Escaping globals -- Global variables that have definitions and at least one of those definitions is in another scope.
*/
/** A source language variable, to be converted into a set of SSA variables. */
/** A source language variable, to be converted into a set of SSA variables. */
abstract class SsaSourceVariable extends @py_variable {
SsaSourceVariable() {
/* Exclude `True`, `False` and `None` */
not this.(Variable).getALoad() instanceof NameConstant
}
/** Gets the name of this variable */
string getName() {
variable(this, _, result)
}
string getName() { variable(this, _, result) }
Scope getScope() {
variable(this, result, _)
}
Scope getScope() { variable(this, result, _) }
/** Gets an implicit use of this variable */
abstract ControlFlowNode getAnImplicitUse();
abstract ControlFlowNode getScopeEntryDefinition();
string toString() {
result = "SsaSourceVariable " + this.getName()
}
string toString() { result = "SsaSourceVariable " + this.getName() }
/** Gets a use of this variable, either explicit or implicit. */
ControlFlowNode getAUse() {
@@ -43,14 +37,18 @@ abstract class SsaSourceVariable extends @py_variable {
or
result = this.getAnImplicitUse()
or
/* `import *` is a definition of *all* variables, so must be a use as well, for pass-through
/*
* `import *` is a definition of *all* variables, so must be a use as well, for pass-through
* once we have established that a variable is not redefined.
*/
SsaSource::import_star_refinement(this, result, _)
or
/* Add a use at the end of scope for all variables to keep them live
/*
* Add a use at the end of scope for all variables to keep them live
* This is necessary for taint-tracking.
*/
result = this.getScope().getANormalExit()
}
@@ -73,15 +71,18 @@ abstract class SsaSourceVariable extends @py_variable {
SsaSource::with_definition(this, def)
}
/** Holds if `def` defines an ESSA variable for this variable in such a way
/**
* Holds if `def` defines an ESSA variable for this variable in such a way
* that the new variable is a refinement in some way of the variable used at `use`.
*/
predicate hasRefinement(ControlFlowNode use, ControlFlowNode def) {
this.hasDefiningNode(_) and /* Can't have a refinement unless there is a definition */
this.hasDefiningNode(_) and
/* Can't have a refinement unless there is a definition */
refinement(this, use, def)
}
/** Holds if the edge `pred`->`succ` defines an ESSA variable for this variable in such a way
/**
* Holds if the edge `pred`->`succ` defines an ESSA variable for this variable in such a way
* that the new variable is a refinement in some way of the variable used at `use`.
*/
predicate hasRefinementEdge(ControlFlowNode use, BasicBlock pred, BasicBlock succ) {
@@ -100,7 +101,6 @@ abstract class SsaSourceVariable extends @py_variable {
}
abstract CallNode redefinedAtCallSite();
}
private predicate refinement(SsaSourceVariable v, ControlFlowNode use, ControlFlowNode def) {
@@ -119,9 +119,7 @@ private predicate refinement(SsaSourceVariable v, ControlFlowNode use, ControlFl
def = v.redefinedAtCallSite() and def = use
}
class FunctionLocalVariable extends SsaSourceVariable {
FunctionLocalVariable() {
this.(LocalVariable).getScope() instanceof Function and
not this instanceof NonLocalVariable
@@ -132,8 +130,7 @@ class FunctionLocalVariable extends SsaSourceVariable {
}
override ControlFlowNode getScopeEntryDefinition() {
exists(Scope s |
s.getEntryNode() = result |
exists(Scope s | s.getEntryNode() = result |
s = this.(LocalVariable).getScope() and
not this.(LocalVariable).isParameter()
or
@@ -143,11 +140,9 @@ class FunctionLocalVariable extends SsaSourceVariable {
}
override CallNode redefinedAtCallSite() { none() }
}
class NonLocalVariable extends SsaSourceVariable {
NonLocalVariable() {
exists(Function f |
this.(LocalVariable).getScope() = f and
@@ -169,37 +164,27 @@ class NonLocalVariable extends SsaSourceVariable {
this.(LocalVariable).getScope().getEntryNode() = result
}
pragma [noinline]
Scope scope_as_local_variable() {
result = this.(LocalVariable).getScope()
}
pragma[noinline]
Scope scope_as_local_variable() { result = this.(LocalVariable).getScope() }
override CallNode redefinedAtCallSite() {
result.getScope().getScope*() = this.scope_as_local_variable()
}
}
class ClassLocalVariable extends SsaSourceVariable {
ClassLocalVariable() { this.(LocalVariable).getScope() instanceof Class }
ClassLocalVariable() {
this.(LocalVariable).getScope() instanceof Class
}
override ControlFlowNode getAnImplicitUse() {
none()
}
override ControlFlowNode getAnImplicitUse() { none() }
override ControlFlowNode getScopeEntryDefinition() {
result = this.(LocalVariable).getScope().getEntryNode()
}
override CallNode redefinedAtCallSite() { none() }
}
class BuiltinVariable extends SsaSourceVariable {
BuiltinVariable() {
this instanceof GlobalVariable and
not exists(this.(Variable).getAStore()) and
@@ -208,20 +193,14 @@ class BuiltinVariable extends SsaSourceVariable {
not exists(ImportStar is | is.getScope() = this.(Variable).getScope())
}
override ControlFlowNode getAnImplicitUse() {
none()
}
override ControlFlowNode getAnImplicitUse() { none() }
override ControlFlowNode getScopeEntryDefinition() {
none()
}
override ControlFlowNode getScopeEntryDefinition() { none() }
override CallNode redefinedAtCallSite() { none() }
}
class ModuleVariable extends SsaSourceVariable {
ModuleVariable() {
this instanceof GlobalVariable and
(
@@ -235,10 +214,8 @@ class ModuleVariable extends SsaSourceVariable {
)
}
pragma [noinline]
CallNode global_variable_callnode() {
result.getScope() = this.(GlobalVariable).getScope()
}
pragma[noinline]
CallNode global_variable_callnode() { result.getScope() = this.(GlobalVariable).getScope() }
pragma[noinline]
ImportMemberNode global_variable_import() {
@@ -251,8 +228,7 @@ class ModuleVariable extends SsaSourceVariable {
or
result = global_variable_import()
or
exists(ImportTimeScope scope |
scope.entryEdge(result, _) |
exists(ImportTimeScope scope | scope.entryEdge(result, _) |
this = scope.getOuterVariable(_) or
this.(Variable).getAUse().getScope() = scope
)
@@ -264,14 +240,14 @@ class ModuleVariable extends SsaSourceVariable {
)
or
exists(ImportTimeScope s |
result = s.getANormalExit() and this.(Variable).getScope() = s and
result = s.getANormalExit() and
this.(Variable).getScope() = s and
implicit_definition(this)
)
}
override ControlFlowNode getScopeEntryDefinition() {
exists(Scope s |
s.getEntryNode() = result |
exists(Scope s | s.getEntryNode() = result |
/* Module entry point */
this.(GlobalVariable).getScope() = s
or
@@ -282,31 +258,28 @@ class ModuleVariable extends SsaSourceVariable {
this.(GlobalVariable).getAUse().getScope() = s
)
or
exists(ImportTimeScope scope |
scope.entryEdge(_, result) |
exists(ImportTimeScope scope | scope.entryEdge(_, result) |
this = scope.getOuterVariable(_) or
this.(Variable).getAUse().getScope() = scope
)
}
override CallNode redefinedAtCallSite() { none() }
}
class NonEscapingGlobalVariable extends ModuleVariable {
NonEscapingGlobalVariable() {
this instanceof GlobalVariable and
exists(this.(Variable).getAStore()) and
not variable_or_attribute_defined_out_of_scope(this)
}
}
class EscapingGlobalVariable extends ModuleVariable {
EscapingGlobalVariable() {
this instanceof GlobalVariable and exists(this.(Variable).getAStore()) and variable_or_attribute_defined_out_of_scope(this)
this instanceof GlobalVariable and
exists(this.(Variable).getAStore()) and
variable_or_attribute_defined_out_of_scope(this)
}
override ControlFlowNode getAnImplicitUse() {
@@ -328,36 +301,25 @@ class EscapingGlobalVariable extends ModuleVariable {
result = this.innerScope().getEntryNode()
}
pragma [noinline]
Scope scope_as_global_variable() {
result = this.(GlobalVariable).getScope()
}
pragma[noinline]
Scope scope_as_global_variable() { result = this.(GlobalVariable).getScope() }
override CallNode redefinedAtCallSite() {
result.(CallNode).getScope().getScope*() = this.scope_as_global_variable()
}
}
class EscapingAssignmentGlobalVariable extends EscapingGlobalVariable {
EscapingAssignmentGlobalVariable() {
exists(NameNode n | n.defines(this) and not n.getScope() = this.getScope())
}
}
class SpecialSsaSourceVariable extends SsaSourceVariable {
SpecialSsaSourceVariable() {
variable(this, _, "*") or variable(this, _, "$")
}
SpecialSsaSourceVariable() { variable(this, _, "*") or variable(this, _, "$") }
override ControlFlowNode getAnImplicitUse() {
exists(ImportTimeScope s |
result = s.getANormalExit() and this.getScope() = s
)
exists(ImportTimeScope s | result = s.getANormalExit() and this.getScope() = s)
}
override ControlFlowNode getScopeEntryDefinition() {
@@ -365,31 +327,31 @@ class SpecialSsaSourceVariable extends SsaSourceVariable {
this.getScope().getEntryNode() = result
}
pragma [noinline]
Scope scope_as_global_variable() {
result = this.(GlobalVariable).getScope()
}
pragma[noinline]
Scope scope_as_global_variable() { result = this.(GlobalVariable).getScope() }
override CallNode redefinedAtCallSite() {
result.(CallNode).getScope().getScope*() = this.scope_as_global_variable()
}
}
/** Holds if this variable is implicitly defined */
private predicate implicit_definition(Variable v) {
v.getId() = "*" or v.getId() = "$"
or
v.getId() = "*" or
v.getId() = "$" or
exists(ImportStar is | is.getScope() = v.getScope())
}
private predicate variable_or_attribute_defined_out_of_scope(Variable v) {
exists(NameNode n | n.defines(v) and not n.getScope() = v.getScope())
or
exists(AttrNode a | a.isStore() and a.getObject() = v.getAUse() and not a.getScope() = v.getScope())
exists(AttrNode a |
a.isStore() and a.getObject() = v.getAUse() and not a.getScope() = v.getScope()
)
}
private predicate class_with_global_metaclass(Class cls, GlobalVariable metaclass) {
metaclass.getId() = "__metaclass__" and major_version() = 2 and
metaclass.getId() = "__metaclass__" and
major_version() = 2 and
cls.getEnclosingModule() = metaclass.getScope()
}

405
python/ql/src/semmle/python/essa/Essa.qll
View File

@@ -6,46 +6,35 @@ import python
private import SsaCompute
import semmle.python.essa.Definitions
/** An (enhanced) SSA variable derived from `SsaSourceVariable`. */
class EssaVariable extends TEssaDefinition {
/** Gets the (unique) definition of this variable. */
EssaDefinition getDefinition() {
this = result
}
EssaDefinition getDefinition() { this = result }
/** Gets a use of this variable, where a "use" is defined by
/**
* Gets a use of this variable, where a "use" is defined by
* `SsaSourceVariable.getAUse()`.
* Note that this differs from `EssaVariable.getASourceUse()`.
*/
ControlFlowNode getAUse() {
result = this.getDefinition().getAUse()
}
ControlFlowNode getAUse() { result = this.getDefinition().getAUse() }
/** Gets the source variable from which this variable is derived. */
SsaSourceVariable getSourceVariable() {
result = this.getDefinition().getSourceVariable()
}
SsaSourceVariable getSourceVariable() { result = this.getDefinition().getSourceVariable() }
/** Gets the name of this variable. */
string getName() {
result = this.getSourceVariable().getName()
}
string getName() { result = this.getSourceVariable().getName() }
string toString() {
result = "SSA variable " + this.getName()
}
string toString() { result = "SSA variable " + this.getName() }
/** Gets a string representation of this variable.
/**
* Gets a string representation of this variable.
* WARNING: The format of this may change and it may be very inefficient to compute.
* To used for debugging and testing only.
*/
string getRepresentation() {
result = this.getSourceVariable().getName() + "_" + var_rank(this)
}
string getRepresentation() { result = this.getSourceVariable().getName() + "_" + var_rank(this) }
/** Gets a use of this variable, where a "use" is defined by
/**
* Gets a use of this variable, where a "use" is defined by
* `SsaSourceVariable.getASourceUse()`.
* Note that this differs from `EssaVariable.getAUse()`.
*/
@@ -63,23 +52,21 @@ class EssaVariable extends TEssaDefinition {
}
/** Gets the scope of this variable. */
Scope getScope() {
result = this.getDefinition().getScope()
}
Scope getScope() { result = this.getDefinition().getScope() }
/** Holds if this the meta-variable for a scope.
/**
* Holds if this the meta-variable for a scope.
* This is used to attach attributes for undeclared variables implicitly
* defined by `from ... import *` and the like.
*/
predicate isMetaVariable() {
this.getName() = "$"
}
predicate isMetaVariable() { this.getName() = "$" }
}
/* Helper for location_string
/*
* Helper for location_string
* NOTE: This is Python specific, to make `getRepresentation()` portable will require further work.
*/
private int exception_handling(BasicBlock b) {
b.reachesExit() and result = 0
or
@@ -91,14 +78,15 @@ pragma[noinline]
private string location_string(EssaVariable v) {
exists(EssaDefinition def, BasicBlock b, int index, int line, int col |
def = v.getDefinition() and
(if b.getNode(0).isNormalExit() then
line = 100000 and col = 0
else
b.hasLocationInfo(_, line, col, _, _)
(
if b.getNode(0).isNormalExit()
then line = 100000 and col = 0
else b.hasLocationInfo(_, line, col, _, _)
) and
/* Add large numbers to values to prevent 1000 sorting before 99 */
result = (line + 100000) + ":" + (col*2 + 10000 + exception_handling(b)) + ":" + (index + 100003)
|
result =
(line + 100000) + ":" + (col * 2 + 10000 + exception_handling(b)) + ":" + (index + 100003)
|
def = TEssaNodeDefinition(_, b, index)
or
def = TEssaNodeRefinement(_, b, index)
@@ -119,37 +107,31 @@ private int var_rank(EssaVariable v) {
exists(int r, SsaSourceVariable var |
var = v.getSourceVariable() and
var_index(v) = rank[r](EssaVariable x | x.getSourceVariable() = var | var_index(x)) and
result = r-1
result = r - 1
)
}
/** Underlying IPA type for EssaDefinition and EssaVariable. */
private cached newtype TEssaDefinition =
cached
private newtype TEssaDefinition =
TEssaNodeDefinition(SsaSourceVariable v, BasicBlock b, int i) {
EssaDefinitions::variableDefinition(v, _, b, _, i)
}
or
} or
TEssaNodeRefinement(SsaSourceVariable v, BasicBlock b, int i) {
EssaDefinitions::variableRefinement(v, _, b, _, i)
}
or
} or
TEssaEdgeDefinition(SsaSourceVariable v, BasicBlock pred, BasicBlock succ) {
EssaDefinitions::piNode(v, pred, succ)
}
or
TPhiFunction(SsaSourceVariable v, BasicBlock b) {
EssaDefinitions::phiNode(v, b)
}
} or
TPhiFunction(SsaSourceVariable v, BasicBlock b) { EssaDefinitions::phiNode(v, b) }
/** Definition of an extended-SSA (ESSA) variable.
/**
* Definition of an extended-SSA (ESSA) variable.
* There is exactly one definition for each variable,
* and exactly one variable for each definition.
*/
abstract class EssaDefinition extends TEssaDefinition {
string toString() {
result = "EssaDefinition"
}
string toString() { result = "EssaDefinition" }
/** Gets the source variable for which this a definition, either explicit or implicit. */
abstract SsaSourceVariable getSourceVariable();
@@ -160,7 +142,8 @@ abstract class EssaDefinition extends TEssaDefinition {
/** Holds if this definition reaches the end of `b`. */
abstract predicate reachesEndOfBlock(BasicBlock b);
/** Gets the location of a control flow node that is indicative of this definition.
/**
* Gets the location of a control flow node that is indicative of this definition.
* Since definitions may occur on edges of the control flow graph, the given location may
* be imprecise.
* Distinct `EssaDefinitions` may return the same ControlFlowNode even for
@@ -168,29 +151,26 @@ abstract class EssaDefinition extends TEssaDefinition {
*/
abstract Location getLocation();
/** Gets a representation of this SSA definition for debugging purposes.
* Since this is primarily for debugging and testing, performance may be poor. */
/**
* Gets a representation of this SSA definition for debugging purposes.
* Since this is primarily for debugging and testing, performance may be poor.
*/
abstract string getRepresentation();
abstract Scope getScope();
EssaVariable getVariable() {
result.getDefinition() = this
}
EssaVariable getVariable() { result.getDefinition() = this }
abstract BasicBlock getBasicBlock();
}
/** An ESSA definition corresponding to an edge refinement of the underlying variable.
/**
* An ESSA definition corresponding to an edge refinement of the underlying variable.
* For example, the edges leaving a test on a variable both represent refinements of that
* variable. On one edge the test is true, on the other it is false.
*/
class EssaEdgeRefinement extends EssaDefinition, TEssaEdgeDefinition {
override string toString() {
result = "SSA filter definition"
}
override string toString() { result = "SSA filter definition" }
boolean getSense() {
this.getPredecessor().getATrueSuccessor() = this.getSuccessor() and result = true
@@ -198,19 +178,13 @@ class EssaEdgeRefinement extends EssaDefinition, TEssaEdgeDefinition {
this.getPredecessor().getAFalseSuccessor() = this.getSuccessor() and result = false
}
override SsaSourceVariable getSourceVariable() {
this = TEssaEdgeDefinition(result, _, _)
}
override SsaSourceVariable getSourceVariable() { this = TEssaEdgeDefinition(result, _, _) }
/** Gets the basic block preceding the edge on which this refinement occurs. */
BasicBlock getPredecessor() {
this = TEssaEdgeDefinition(_, result, _)
}
BasicBlock getPredecessor() { this = TEssaEdgeDefinition(_, result, _) }
/** Gets the basic block succeeding the edge on which this refinement occurs. */
BasicBlock getSuccessor() {
this = TEssaEdgeDefinition(_, _, result)
}
BasicBlock getSuccessor() { this = TEssaEdgeDefinition(_, _, result) }
override ControlFlowNode getAUse() {
SsaDefinitions::reachesUse(this.getSourceVariable(), this.getSuccessor(), piIndex(), result)
@@ -220,13 +194,11 @@ class EssaEdgeRefinement extends EssaDefinition, TEssaEdgeDefinition {
SsaDefinitions::reachesEndOfBlock(this.getSourceVariable(), this.getSuccessor(), piIndex(), b)
}
override Location getLocation() {
result = this.getSuccessor().getNode(0).getLocation()
}
override Location getLocation() { result = this.getSuccessor().getNode(0).getLocation() }
/** Gets the SSA variable to which this refinement applies. */
EssaVariable getInput() {
exists(SsaSourceVariable var , EssaDefinition def |
exists(SsaSourceVariable var, EssaDefinition def |
var = this.getSourceVariable() and
var = def.getSourceVariable() and
def.reachesEndOfBlock(this.getPredecessor()) and
@@ -239,19 +211,13 @@ class EssaEdgeRefinement extends EssaDefinition, TEssaEdgeDefinition {
}
/** Gets the scope of the variable defined by this definition. */
override Scope getScope() {
result = this.getPredecessor().getScope()
}
override BasicBlock getBasicBlock(){
result = this.getSuccessor()
}
override Scope getScope() { result = this.getPredecessor().getScope() }
override BasicBlock getBasicBlock() { result = this.getSuccessor() }
}
/** A Phi-function as specified in classic SSA form. */
class PhiFunction extends EssaDefinition, TPhiFunction {
override ControlFlowNode getAUse() {
SsaDefinitions::reachesUse(this.getSourceVariable(), this.getBasicBlock(), phiIndex(), result)
}
@@ -260,9 +226,7 @@ class PhiFunction extends EssaDefinition, TPhiFunction {
SsaDefinitions::reachesEndOfBlock(this.getSourceVariable(), this.getBasicBlock(), phiIndex(), b)
}
override SsaSourceVariable getSourceVariable() {
this = TPhiFunction(result, _)
}
override SsaSourceVariable getSourceVariable() { this = TPhiFunction(result, _) }
/** Gets an input refinement that exists on one of the incoming edges to this phi node. */
private EssaEdgeRefinement inputEdgeRefinement(BasicBlock pred) {
@@ -290,37 +254,29 @@ class PhiFunction extends EssaDefinition, TPhiFunction {
}
/** Gets the input variable for this phi node on the edge `pred` -> `this.getBasicBlock()`, if any. */
cached EssaVariable getInput(BasicBlock pred) {
cached
EssaVariable getInput(BasicBlock pred) {
result.getDefinition() = this.reachingDefinition(pred)
or
result.getDefinition() = this.inputEdgeRefinement(pred)
}
/** Gets an input variable for this phi node. */
EssaVariable getAnInput() {
result = this.getInput(_)
}
EssaVariable getAnInput() { result = this.getInput(_) }
/** Holds if forall incoming edges in the flow graph, there is an input variable */
predicate isComplete() {
forall(BasicBlock pred |
pred = this.getBasicBlock().getAPredecessor() |
forall(BasicBlock pred | pred = this.getBasicBlock().getAPredecessor() |
exists(this.getInput(pred))
)
}
override string toString() {
result = "SSA Phi Function"
}
override string toString() { result = "SSA Phi Function" }
/** Gets the basic block that succeeds this phi node. */
override BasicBlock getBasicBlock() {
this = TPhiFunction(_, result)
}
override BasicBlock getBasicBlock() { this = TPhiFunction(_, result) }
override Location getLocation() {
result = this.getBasicBlock().getNode(0).getLocation()
}
override Location getLocation() { result = this.getBasicBlock().getNode(0).getLocation() }
/** Helper for `argList(n)`. */
private int rankInput(EssaVariable input) {
@@ -330,12 +286,10 @@ class PhiFunction extends EssaDefinition, TPhiFunction {
/** Helper for `argList()`. */
private string argList(int n) {
exists(EssaVariable input |
n = this.rankInput(input)
|
exists(EssaVariable input | n = this.rankInput(input) |
n = 1 and result = input.getRepresentation()
or
n > 1 and result = this.argList(n-1) + ", " + input.getRepresentation()
n > 1 and result = this.argList(n - 1) + ", " + input.getRepresentation()
)
}
@@ -352,13 +306,12 @@ class PhiFunction extends EssaDefinition, TPhiFunction {
or
result = "phi(" + this.argList() + ")"
or
exists(this.getAnInput()) and not exists(this.argList()) and
exists(this.getAnInput()) and
not exists(this.argList()) and
result = "phi(" + this.getSourceVariable().getName() + "??)"
}
override Scope getScope() {
result = this.getBasicBlock().getScope()
}
override Scope getScope() { result = this.getBasicBlock().getScope() }
private EssaEdgeRefinement piInputDefinition(EssaVariable input) {
input = this.getAnInput() and
@@ -367,7 +320,8 @@ class PhiFunction extends EssaDefinition, TPhiFunction {
input = this.getAnInput() and result = input.getDefinition().(PhiFunction).piInputDefinition(_)
}
/** Gets the variable which is the common and complete input to all pi-nodes that are themselves
/**
* Gets the variable which is the common and complete input to all pi-nodes that are themselves
* inputs to this phi-node.
* For example:
* ```
@@ -387,19 +341,15 @@ class PhiFunction extends EssaDefinition, TPhiFunction {
* meaning that we cannot track `x` from `x0` to `x3`.
* By using `getShortCircuitInput()` we can do so, since the short-circuit input of `x3` is `x0`.
*/
pragma [noinline]
pragma[noinline]
EssaVariable getShortCircuitInput() {
exists(BasicBlock common |
forall(EssaVariable input |
input = this.getAnInput() |
forall(EssaVariable input | input = this.getAnInput() |
common = this.piInputDefinition(input).getPredecessor()
)
and
forall(BasicBlock succ |
succ = common.getASuccessor() |
) and
forall(BasicBlock succ | succ = common.getASuccessor() |
succ = this.piInputDefinition(_).getSuccessor()
)
and
) and
exists(EssaEdgeRefinement ref |
ref = this.piInputDefinition(_) and
ref.getPredecessor() = common and
@@ -409,14 +359,12 @@ class PhiFunction extends EssaDefinition, TPhiFunction {
}
}
/** A definition of an ESSA variable that is not directly linked to
/**
* A definition of an ESSA variable that is not directly linked to
* another ESSA variable.
*/
class EssaNodeDefinition extends EssaDefinition, TEssaNodeDefinition {
override string toString() {
result = "Essa node definition"
}
override string toString() { result = "Essa node definition" }
override ControlFlowNode getAUse() {
exists(SsaSourceVariable v, BasicBlock b, int i |
@@ -432,22 +380,14 @@ class EssaNodeDefinition extends EssaDefinition, TEssaNodeDefinition {
)
}
override SsaSourceVariable getSourceVariable() {
this = TEssaNodeDefinition(result, _, _)
}
override SsaSourceVariable getSourceVariable() { this = TEssaNodeDefinition(result, _, _) }
/** Gets the ControlFlowNode corresponding to this definition */
ControlFlowNode getDefiningNode() {
this.definedBy(_, result)
}
ControlFlowNode getDefiningNode() { this.definedBy(_, result) }
override Location getLocation() {
result = this.getDefiningNode().getLocation()
}
override Location getLocation() { result = this.getDefiningNode().getLocation() }
override string getRepresentation() {
result = this.getAQlClass()
}
override string getRepresentation() { result = this.getAQlClass() }
override Scope getScope() {
exists(BasicBlock defb |
@@ -457,26 +397,19 @@ class EssaNodeDefinition extends EssaDefinition, TEssaNodeDefinition {
}
predicate definedBy(SsaSourceVariable v, ControlFlowNode def) {
exists(BasicBlock b, int i |
def = b.getNode(i) |
this = TEssaNodeDefinition(v, b, i+i)
exists(BasicBlock b, int i | def = b.getNode(i) |
this = TEssaNodeDefinition(v, b, i + i)
or
this = TEssaNodeDefinition(v, b, i+i+1)
this = TEssaNodeDefinition(v, b, i + i + 1)
)
}
override BasicBlock getBasicBlock(){
result = this.getDefiningNode().getBasicBlock()
}
override BasicBlock getBasicBlock() { result = this.getDefiningNode().getBasicBlock() }
}
/** A definition of an ESSA variable that takes another ESSA variable as an input. */
class EssaNodeRefinement extends EssaDefinition, TEssaNodeRefinement {
override string toString() {
result = "SSA filter definition"
}
override string toString() { result = "SSA filter definition" }
/** Gets the SSA variable to which this refinement applies. */
EssaVariable getInput() {
@@ -498,18 +431,12 @@ class EssaNodeRefinement extends EssaDefinition, TEssaNodeRefinement {
)
}
override SsaSourceVariable getSourceVariable() {
this = TEssaNodeRefinement(result, _, _)
}
override SsaSourceVariable getSourceVariable() { this = TEssaNodeRefinement(result, _, _) }
/** Gets the ControlFlowNode corresponding to this definition */
ControlFlowNode getDefiningNode() {
this.definedBy(_, result)
}
ControlFlowNode getDefiningNode() { this.definedBy(_, result) }
override Location getLocation() {
result = this.getDefiningNode().getLocation()
}
override Location getLocation() { result = this.getDefiningNode().getLocation() }
override string getRepresentation() {
result = this.getAQlClass() + "(" + this.getInput().getRepresentation() + ")"
@@ -526,23 +453,19 @@ class EssaNodeRefinement extends EssaDefinition, TEssaNodeRefinement {
}
predicate definedBy(SsaSourceVariable v, ControlFlowNode def) {
exists(BasicBlock b, int i |
def = b.getNode(i) |
this = TEssaNodeRefinement(v, b, i+i)
exists(BasicBlock b, int i | def = b.getNode(i) |
this = TEssaNodeRefinement(v, b, i + i)
or
this = TEssaNodeRefinement(v, b, i+i+1)
this = TEssaNodeRefinement(v, b, i + i + 1)
)
}
override BasicBlock getBasicBlock(){
result = this.getDefiningNode().getBasicBlock()
}
override BasicBlock getBasicBlock() { result = this.getDefiningNode().getBasicBlock() }
}
pragma[noopt]
private EssaVariable potential_input(EssaNodeRefinement ref) {
exists(ControlFlowNode use, SsaSourceVariable var, ControlFlowNode def |
exists(ControlFlowNode use, SsaSourceVariable var, ControlFlowNode def |
var.hasRefinement(use, def) and
use = result.getAUse() and
var = result.getSourceVariable() and
@@ -559,7 +482,6 @@ deprecated class PyNodeRefinement = EssaNodeRefinement;
/** An assignment to a variable `v = val` */
class AssignmentDefinition extends EssaNodeDefinition {
AssignmentDefinition() {
SsaSource::assignment_definition(this.getSourceVariable(), this.getDefiningNode(), _)
}
@@ -568,15 +490,11 @@ class AssignmentDefinition extends EssaNodeDefinition {
SsaSource::assignment_definition(this.getSourceVariable(), this.getDefiningNode(), result)
}
override string getRepresentation() {
result = this.getValue().getNode().toString()
}
override string getRepresentation() { result = this.getValue().getNode().toString() }
}
/** Capture of a raised exception `except ExceptionType ex:` */
class ExceptionCapture extends EssaNodeDefinition {
class ExceptionCapture extends EssaNodeDefinition {
ExceptionCapture() {
SsaSource::exception_capture(this.getSourceVariable(), this.getDefiningNode())
}
@@ -588,15 +506,11 @@ class ExceptionCapture extends EssaNodeDefinition {
)
}
override string getRepresentation() {
result = "except " + this.getSourceVariable().getName()
}
override string getRepresentation() { result = "except " + this.getSourceVariable().getName() }
}
/** An assignment to a variable as part of a multiple assignment `..., v, ... = val` */
class MultiAssignmentDefinition extends EssaNodeDefinition {
MultiAssignmentDefinition() {
SsaSource::multi_assignment_definition(this.getSourceVariable(), this.getDefiningNode(), _, _)
}
@@ -610,130 +524,99 @@ class MultiAssignmentDefinition extends EssaNodeDefinition {
/** Holds if `this` has (zero-based) index `index` in `lhs`. */
predicate indexOf(int index, SequenceNode lhs) {
SsaSource::multi_assignment_definition(this.getSourceVariable(), this.getDefiningNode(), index, lhs)
SsaSource::multi_assignment_definition(this.getSourceVariable(), this.getDefiningNode(), index,
lhs)
}
}
/** A definition of a variable in a `with` statement */
class WithDefinition extends EssaNodeDefinition {
WithDefinition() { SsaSource::with_definition(this.getSourceVariable(), this.getDefiningNode()) }
WithDefinition () {
SsaSource::with_definition(this.getSourceVariable(), this.getDefiningNode())
}
override string getRepresentation() {
result = "with"
}
override string getRepresentation() { result = "with" }
}
/** A definition of a variable by declaring it as a parameter */
class ParameterDefinition extends EssaNodeDefinition {
ParameterDefinition() {
SsaSource::parameter_definition(this.getSourceVariable(), this.getDefiningNode())
}
predicate isSelf() {
this.getDefiningNode().getNode().(Parameter).isSelf()
}
predicate isSelf() { this.getDefiningNode().getNode().(Parameter).isSelf() }
/** Gets the control flow node for the default value of this parameter */
ControlFlowNode getDefault() {
result.getNode() = this.getParameter().getDefault()
}
ControlFlowNode getDefault() { result.getNode() = this.getParameter().getDefault() }
/** Gets the annotation control flow node of this parameter */
ControlFlowNode getAnnotation() {
result.getNode() = this.getParameter().getAnnotation()
}
ControlFlowNode getAnnotation() { result.getNode() = this.getParameter().getAnnotation() }
/** Gets the name of this parameter definition */
string getName() {
result = this.getParameter().asName().getId()
}
string getName() { result = this.getParameter().asName().getId() }
predicate isVarargs() {
exists(Function func | func.getVararg() = this.getDefiningNode().getNode())
}
/** Holds if this parameter is a 'kwargs' parameter.
/**
* Holds if this parameter is a 'kwargs' parameter.
* The `kwargs` in `f(a, b, **kwargs)`.
*/
predicate isKwargs() {
exists(Function func | func.getKwarg() = this.getDefiningNode().getNode())
}
Parameter getParameter() {
result = this.getDefiningNode().getNode()
}
Parameter getParameter() { result = this.getDefiningNode().getNode() }
}
/** A deletion of a variable `del v` */
class DeletionDefinition extends EssaNodeDefinition {
DeletionDefinition() {
SsaSource::deletion_definition(this.getSourceVariable(), this.getDefiningNode())
}
}
/** Definition of variable at the entry of a scope. Usually this represents the transfer of
/**
* Definition of variable at the entry of a scope. Usually this represents the transfer of
* a global or non-local variable from one scope to another.
*/
class ScopeEntryDefinition extends EssaNodeDefinition {
ScopeEntryDefinition() {
this.getDefiningNode() = this.getSourceVariable().getScopeEntryDefinition() and
not this instanceof ImplicitSubModuleDefinition
}
override Scope getScope() {
result.getEntryNode() = this.getDefiningNode()
}
override Scope getScope() { result.getEntryNode() = this.getDefiningNode() }
}
/** Possible redefinition of variable via `from ... import *` */
class ImportStarRefinement extends EssaNodeRefinement {
ImportStarRefinement() {
SsaSource::import_star_refinement(this.getSourceVariable(), _, this.getDefiningNode())
}
}
/** Assignment of an attribute `obj.attr = val` */
class AttributeAssignment extends EssaNodeRefinement {
AttributeAssignment() {
SsaSource::attribute_assignment_refinement(this.getSourceVariable(), _, this.getDefiningNode())
}
string getName() {
result = this.getDefiningNode().(AttrNode).getName()
}
string getName() { result = this.getDefiningNode().(AttrNode).getName() }
ControlFlowNode getValue() {
result = this.getDefiningNode().(DefinitionNode).getValue()
}
ControlFlowNode getValue() { result = this.getDefiningNode().(DefinitionNode).getValue() }
override string getRepresentation() {
result = this.getAQlClass() + " '" + this.getName() + "'(" + this.getInput().getRepresentation() + ")"
result =
this.getAQlClass() + " '" + this.getName() + "'(" + this.getInput().getRepresentation() + ")"
or
not exists(this.getInput()) and
result = this.getAQlClass() + " '" + this.getName() + "'(" + this.getSourceVariable().getName() + "??)"
result =
this.getAQlClass() + " '" + this.getName() + "'(" + this.getSourceVariable().getName() + "??)"
}
}
/** A use of a variable as an argument, `foo(v)`, which might modify the object referred to. */
class ArgumentRefinement extends EssaNodeRefinement {
ControlFlowNode argument;
ArgumentRefinement() {
@@ -747,20 +630,15 @@ class ArgumentRefinement extends EssaNodeRefinement {
/** Deletion of an attribute `del obj.attr`. */
class EssaAttributeDeletion extends EssaNodeRefinement {
EssaAttributeDeletion() {
SsaSource::attribute_deletion_refinement(this.getSourceVariable(), _, this.getDefiningNode())
}
string getName() {
result = this.getDefiningNode().(AttrNode).getName()
}
string getName() { result = this.getDefiningNode().(AttrNode).getName() }
}
/** A pi-node (guard) with only one successor. */
class SingleSuccessorGuard extends EssaNodeRefinement {
SingleSuccessorGuard() {
SsaSource::test_refinement(this.getSourceVariable(), _, this.getDefiningNode())
}
@@ -778,35 +656,28 @@ class SingleSuccessorGuard extends EssaNodeRefinement {
result = EssaNodeRefinement.super.getRepresentation() + " [??]"
}
ControlFlowNode getTest() {
result = this.getDefiningNode()
}
ControlFlowNode getTest() { result = this.getDefiningNode() }
predicate useAndTest(ControlFlowNode use, ControlFlowNode test) {
test = this.getDefiningNode() and
SsaSource::test_refinement(this.getSourceVariable(), use, test)
}
}
/** Implicit definition of the names of sub-modules in a package.
/**
* Implicit definition of the names of sub-modules in a package.
* Although the interpreter does not pre-define these names, merely populating them
* as they are imported, this is a good approximation for static analysis.
*/
class ImplicitSubModuleDefinition extends EssaNodeDefinition {
ImplicitSubModuleDefinition() {
SsaSource::init_module_submodule_defn(this.getSourceVariable(), this.getDefiningNode())
}
}
/** An implicit (possible) definition of an escaping variable at a call-site */
class CallsiteRefinement extends EssaNodeRefinement {
override string toString() {
result = "CallsiteRefinement"
}
override string toString() { result = "CallsiteRefinement" }
CallsiteRefinement() {
exists(SsaSourceVariable var, ControlFlowNode defn |
@@ -818,49 +689,37 @@ class CallsiteRefinement extends EssaNodeRefinement {
)
}
CallNode getCall() {
this.getDefiningNode() = result
}
CallNode getCall() { this.getDefiningNode() = result }
}
/** An implicit (possible) modification of the object referred at a method call */
class MethodCallsiteRefinement extends EssaNodeRefinement {
MethodCallsiteRefinement() {
SsaSource::method_call_refinement(this.getSourceVariable(), _, this.getDefiningNode())
and not this instanceof SingleSuccessorGuard
}
CallNode getCall() {
this.getDefiningNode() = result
SsaSource::method_call_refinement(this.getSourceVariable(), _, this.getDefiningNode()) and
not this instanceof SingleSuccessorGuard
}
CallNode getCall() { this.getDefiningNode() = result }
}
/** An implicit (possible) modification of `self` at a method call */
class SelfCallsiteRefinement extends MethodCallsiteRefinement {
SelfCallsiteRefinement() {
this.getSourceVariable().(Variable).isSelf()
}
SelfCallsiteRefinement() { this.getSourceVariable().(Variable).isSelf() }
}
/** Python specific sub-class of generic EssaEdgeRefinement */
class PyEdgeRefinement extends EssaEdgeRefinement {
override string getRepresentation() {
/* This is for testing so use capital 'P' to make it sort before 'phi' and
* be more visually distinctive. */
/*
* This is for testing so use capital 'P' to make it sort before 'phi' and
* be more visually distinctive.
*/
result = "Pi(" + this.getInput().getRepresentation() + ") [" + this.getSense() + "]"
or
not exists(this.getInput()) and
result = "Pi(" + this.getSourceVariable().getName() + "??) [" + this.getSense() + "]"
}
ControlFlowNode getTest() {
result = this.getPredecessor().getLastNode()
}
ControlFlowNode getTest() { result = this.getPredecessor().getLastNode() }
}

225
python/ql/src/semmle/python/essa/SsaCompute.qll
View File

@@ -1,28 +1,29 @@
/** Provides predicates for computing Enhanced SSA form
/**
* Provides predicates for computing Enhanced SSA form
* Computation of ESSA form is identical to plain SSA form,
* but what counts as a use of definition differs.
*
*
* ## Language independent data-flow graph construction
*
*
* Construction of the data-flow graph is based on the principles behind SSA variables.
*
*
* The definition of an SSA variable is that (statically):
*
*
* * Each variable has exactly one definition
* * A variable's definition dominates all its uses.
*
* SSA form was originally designed for compiler use and thus a "definition" of an SSA variable is
* the same as a definition of the underlying source-code variable. For register allocation this is
*
* SSA form was originally designed for compiler use and thus a "definition" of an SSA variable is
* the same as a definition of the underlying source-code variable. For register allocation this is
* sufficient to treat the variable as equivalent to the value held in the variable.
*
*
* However, this doesn't always work the way we want it for data-flow analysis.
*
* When we start to consider attribute assignment, tests on the value referred to be a variable,
*
* When we start to consider attribute assignment, tests on the value referred to be a variable,
* escaping variables, implicit definitions, etc., we need something finer grained.
*
*
* A data-flow variable has the same properties as a normal SSA variable, but it also has the property that
* *anything* that may change the way we view an object referred to by a variable should be treated as a definition of that variable.
*
*
* For example, tests are treated as definitions, so for the following Python code:
* ```python
* x = None
@@ -39,7 +40,7 @@
* from which is it possible to infer that `x3` may not be None.
* [ Phi functions are standard SSA, a Pi function is a filter or guard on the possible values that a variable
* may hold]
*
*
* Attribute assignments are also treated as definitions, so for the following Python code:
* ```python
* x = C()
@@ -55,13 +56,13 @@
* y1 = attr-assign(y0, .b = 1)
* ```
* From which we can infer that `x1.a` is `1` but we know nothing about `y0.a` despite it being the same type.
*
*
* We can also insert "definitions" for transfers of values (say in global variables) where we do not yet know the call-graph. For example,
* ```python
* def foo():
* global g
* g = 1
*
*
* def bar():
* foo()
* g
@@ -72,7 +73,7 @@
* def foo():
* g0 = scope-entry(g)
* g1 = 1
*
*
* def bar():
* g2 = scope-entry(g)
* foo()
@@ -86,57 +87,63 @@
* g3 = g1
* ```
* and thus it falls out that `g3` must be `1`.
*
*/
import python
private cached module SsaComputeImpl {
cached module EssaDefinitionsImpl {
cached
private module SsaComputeImpl {
cached
module EssaDefinitionsImpl {
/** Whether `n` is a live update that is a definition of the variable `v`. */
cached predicate variableDefinition(SsaSourceVariable v, ControlFlowNode n, BasicBlock b, int rankix, int i) {
cached
predicate variableDefinition(
SsaSourceVariable v, ControlFlowNode n, BasicBlock b, int rankix, int i
) {
SsaComputeImpl::variableDefine(v, n, b, i) and
SsaComputeImpl::defUseRank(v, b, rankix, i) and
(
SsaComputeImpl::defUseRank(v, b, rankix+1, _) and not SsaComputeImpl::defRank(v, b, rankix+1, _)
(
SsaComputeImpl::defUseRank(v, b, rankix + 1, _) and
not SsaComputeImpl::defRank(v, b, rankix + 1, _)
or
not SsaComputeImpl::defUseRank(v, b, rankix+1, _) and Liveness::liveAtExit(v, b)
not SsaComputeImpl::defUseRank(v, b, rankix + 1, _) and Liveness::liveAtExit(v, b)
)
}
/** Whether `n` is a live update that is a definition of the variable `v`. */
cached predicate variableRefinement(SsaSourceVariable v, ControlFlowNode n, BasicBlock b, int rankix, int i) {
cached
predicate variableRefinement(
SsaSourceVariable v, ControlFlowNode n, BasicBlock b, int rankix, int i
) {
SsaComputeImpl::variableRefine(v, n, b, i) and
SsaComputeImpl::defUseRank(v, b, rankix, i) and
(
SsaComputeImpl::defUseRank(v, b, rankix+1, _) and not SsaComputeImpl::defRank(v, b, rankix+1, _)
(
SsaComputeImpl::defUseRank(v, b, rankix + 1, _) and
not SsaComputeImpl::defRank(v, b, rankix + 1, _)
or
not SsaComputeImpl::defUseRank(v, b, rankix+1, _) and Liveness::liveAtExit(v, b)
not SsaComputeImpl::defUseRank(v, b, rankix + 1, _) and Liveness::liveAtExit(v, b)
)
}
cached predicate variableUpdate(SsaSourceVariable v, ControlFlowNode n, BasicBlock b, int rankix, int i) {
cached
predicate variableUpdate(SsaSourceVariable v, ControlFlowNode n, BasicBlock b, int rankix, int i) {
variableDefinition(v, n, b, rankix, i)
or
variableRefinement(v, n, b, rankix, i)
}
/** Holds if `def` is a pi-node for `v` on the edge `pred` -> `succ` */
cached predicate piNode(SsaSourceVariable v, BasicBlock pred, BasicBlock succ) {
cached
predicate piNode(SsaSourceVariable v, BasicBlock pred, BasicBlock succ) {
v.hasRefinementEdge(_, pred, succ) and
Liveness::liveAtEntry(v, succ)
}
/** A phi node for `v` at the beginning of basic block `b`. */
cached predicate phiNode(SsaSourceVariable v, BasicBlock b) {
/** A phi node for `v` at the beginning of basic block `b`. */
cached
predicate phiNode(SsaSourceVariable v, BasicBlock b) {
(
exists(BasicBlock def | def.dominanceFrontier(b) |
SsaComputeImpl::ssaDef(v, def)
)
exists(BasicBlock def | def.dominanceFrontier(b) | SsaComputeImpl::ssaDef(v, def))
or
piNode(v, _, b) and strictcount(b.getAPredecessor()) > 1
) and
@@ -144,25 +151,26 @@ private cached module SsaComputeImpl {
}
}
cached predicate variableDefine(SsaSourceVariable v, ControlFlowNode n, BasicBlock b, int i) {
v.hasDefiningNode(n)
and
cached
predicate variableDefine(SsaSourceVariable v, ControlFlowNode n, BasicBlock b, int i) {
v.hasDefiningNode(n) and
exists(int j |
n = b.getNode(j) and
i = j*2 + 1
i = j * 2 + 1
)
}
cached predicate variableRefine(SsaSourceVariable v, ControlFlowNode n, BasicBlock b, int i) {
v.hasRefinement(_, n)
and
cached
predicate variableRefine(SsaSourceVariable v, ControlFlowNode n, BasicBlock b, int i) {
v.hasRefinement(_, n) and
exists(int j |
n = b.getNode(j) and
i = j*2 + 1
i = j * 2 + 1
)
}
cached predicate variableDef(SsaSourceVariable v, ControlFlowNode n, BasicBlock b, int i) {
cached
predicate variableDef(SsaSourceVariable v, ControlFlowNode n, BasicBlock b, int i) {
variableDefine(v, n, b, i) or variableRefine(v, n, b, i)
}
@@ -174,22 +182,25 @@ private cached module SsaComputeImpl {
* irrelevant indices at which there is no definition or use when traversing
* basic blocks.
*/
cached predicate defUseRank(SsaSourceVariable v, BasicBlock b, int rankix, int i) {
i = rank[rankix](int j | variableDef(v, _, b, j) or variableUse(v, _, b, j))
cached
predicate defUseRank(SsaSourceVariable v, BasicBlock b, int rankix, int i) {
i = rank[rankix](int j | variableDef(v, _, b, j) or variableUse(v, _, b, j))
}
/** A definition of a variable occurring at the specified rank index in basic block `b`. */
cached predicate defRank(SsaSourceVariable v, BasicBlock b, int rankix, int i) {
variableDef(v, _, b, i) and
defUseRank(v, b, rankix, i)
cached
predicate defRank(SsaSourceVariable v, BasicBlock b, int rankix, int i) {
variableDef(v, _, b, i) and
defUseRank(v, b, rankix, i)
}
/** A `VarAccess` `use` of `v` in `b` at index `i`. */
cached predicate variableUse(SsaSourceVariable v, ControlFlowNode use, BasicBlock b, int i) {
cached
predicate variableUse(SsaSourceVariable v, ControlFlowNode use, BasicBlock b, int i) {
(v.getAUse() = use or v.hasRefinement(use, _)) and
exists(int j |
b.getNode(j) = use and
i = 2*j
b.getNode(j) = use and
i = 2 * j
)
}
@@ -197,7 +208,8 @@ private cached module SsaComputeImpl {
* A definition of an SSA variable occurring at the specified position.
* This is either a phi node, a `VariableUpdate`, or a parameter.
*/
cached predicate ssaDef(SsaSourceVariable v, BasicBlock b) {
cached
predicate ssaDef(SsaSourceVariable v, BasicBlock b) {
EssaDefinitions::phiNode(v, b)
or
EssaDefinitions::variableUpdate(v, _, b, _, _)
@@ -216,10 +228,13 @@ private cached module SsaComputeImpl {
*/
/** The maximum rank index for the given variable and basic block. */
cached int lastRank(SsaSourceVariable v, BasicBlock b) {
cached
int lastRank(SsaSourceVariable v, BasicBlock b) {
result = max(int rankix | defUseRank(v, b, rankix, _))
or
not defUseRank(v, b, _, _) and (EssaDefinitions::phiNode(v, b) or EssaDefinitions::piNode(v, _, b)) and result = 0
not defUseRank(v, b, _, _) and
(EssaDefinitions::phiNode(v, b) or EssaDefinitions::piNode(v, _, b)) and
result = 0
}
private predicate ssaDefRank(SsaSourceVariable v, BasicBlock b, int rankix, int i) {
@@ -227,22 +242,35 @@ private cached module SsaComputeImpl {
or
EssaDefinitions::phiNode(v, b) and rankix = 0 and i = phiIndex()
or
EssaDefinitions::piNode(v, _, b) and EssaDefinitions::phiNode(v, b) and rankix = -1 and i = piIndex()
EssaDefinitions::piNode(v, _, b) and
EssaDefinitions::phiNode(v, b) and
rankix = -1 and
i = piIndex()
or
EssaDefinitions::piNode(v, _, b) and not EssaDefinitions::phiNode(v, b) and rankix = 0 and i = piIndex()
EssaDefinitions::piNode(v, _, b) and
not EssaDefinitions::phiNode(v, b) and
rankix = 0 and
i = piIndex()
}
/** The SSA definition reaches the rank index `rankix` in its own basic block `b`. */
cached predicate ssaDefReachesRank(SsaSourceVariable v, BasicBlock b, int i, int rankix) {
ssaDefRank(v, b, rankix, i) or
ssaDefReachesRank(v, b, i, rankix-1) and rankix <= lastRank(v, b) and not ssaDefRank(v, b, rankix, _)
cached
predicate ssaDefReachesRank(SsaSourceVariable v, BasicBlock b, int i, int rankix) {
ssaDefRank(v, b, rankix, i)
or
ssaDefReachesRank(v, b, i, rankix - 1) and
rankix <= lastRank(v, b) and
not ssaDefRank(v, b, rankix, _)
}
/**
* The SSA definition of `v` at `def` reaches `use` in the same basic block
* without crossing another SSA definition of `v`.
*/
cached predicate ssaDefReachesUseWithinBlock(SsaSourceVariable v, BasicBlock b, int i, ControlFlowNode use) {
cached
predicate ssaDefReachesUseWithinBlock(
SsaSourceVariable v, BasicBlock b, int i, ControlFlowNode use
) {
exists(int rankix, int useix |
ssaDefReachesRank(v, b, i, rankix) and
defUseRank(v, b, rankix, useix) and
@@ -250,41 +278,44 @@ private cached module SsaComputeImpl {
)
}
cached module LivenessImpl {
cached
module LivenessImpl {
cached
predicate liveAtExit(SsaSourceVariable v, BasicBlock b) { liveAtEntry(v, b.getASuccessor()) }
cached predicate liveAtExit(SsaSourceVariable v, BasicBlock b) {
liveAtEntry(v, b.getASuccessor())
}
cached predicate liveAtEntry(SsaSourceVariable v, BasicBlock b) {
cached
predicate liveAtEntry(SsaSourceVariable v, BasicBlock b) {
SsaComputeImpl::defUseRank(v, b, 1, _) and not SsaComputeImpl::defRank(v, b, 1, _)
or
not SsaComputeImpl::defUseRank(v, b, _, _) and liveAtExit(v, b)
}
}
cached module SsaDefinitionsImpl {
pragma [noinline]
private predicate reachesEndOfBlockRec(SsaSourceVariable v, BasicBlock defbb, int defindex, BasicBlock b) {
cached
module SsaDefinitionsImpl {
pragma[noinline]
private predicate reachesEndOfBlockRec(
SsaSourceVariable v, BasicBlock defbb, int defindex, BasicBlock b
) {
exists(BasicBlock idom | reachesEndOfBlock(v, defbb, defindex, idom) |
idom = b.getImmediateDominator()
)
}
/**
* The SSA definition of `v` at `def` reaches the end of a basic block `b`, at
* which point it is still live, without crossing another SSA definition of `v`.
*/
cached
predicate reachesEndOfBlock(SsaSourceVariable v, BasicBlock defbb, int defindex, BasicBlock b) {
Liveness::liveAtExit(v, b) and
(
defbb = b and SsaComputeImpl::ssaDefReachesRank(v, defbb, defindex, SsaComputeImpl::lastRank(v, b))
or
// It is sufficient to traverse the dominator graph, cf. discussion above.
reachesEndOfBlockRec(v, defbb, defindex, b) and
not SsaComputeImpl::ssaDef(v, b)
Liveness::liveAtExit(v, b) and
(
defbb = b and
SsaComputeImpl::ssaDefReachesRank(v, defbb, defindex, SsaComputeImpl::lastRank(v, b))
or
// It is sufficient to traverse the dominator graph, cf. discussion above.
reachesEndOfBlockRec(v, defbb, defindex, b) and
not SsaComputeImpl::ssaDef(v, b)
)
}
@@ -294,15 +325,16 @@ private cached module SsaComputeImpl {
*/
cached
predicate reachesUse(SsaSourceVariable v, BasicBlock defbb, int defindex, ControlFlowNode use) {
SsaComputeImpl::ssaDefReachesUseWithinBlock(v, defbb, defindex, use) or
exists(BasicBlock b |
SsaComputeImpl::variableUse(v, use, b, _) and
reachesEndOfBlock(v, defbb, defindex, b.getAPredecessor()) and
not SsaComputeImpl::ssaDefReachesUseWithinBlock(v, b, _, use)
)
SsaComputeImpl::ssaDefReachesUseWithinBlock(v, defbb, defindex, use)
or
exists(BasicBlock b |
SsaComputeImpl::variableUse(v, use, b, _) and
reachesEndOfBlock(v, defbb, defindex, b.getAPredecessor()) and
not SsaComputeImpl::ssaDefReachesUseWithinBlock(v, b, _, use)
)
}
/***
/**
* Holds if `(defbb, defindex)` is an SSA definition of `v` that reaches an exit without crossing another
* SSA definition of `v`.
*/
@@ -315,9 +347,7 @@ private cached module SsaComputeImpl {
SsaComputeImpl::variableUse(v, use, last, index)
)
}
}
}
import SsaComputeImpl::SsaDefinitionsImpl as SsaDefinitions
@@ -325,9 +355,8 @@ import SsaComputeImpl::EssaDefinitionsImpl as EssaDefinitions
import SsaComputeImpl::LivenessImpl as Liveness
/* This is exported primarily for testing */
/* A note on numbering
/*
* A note on numbering
* In order to create an SSA graph, we need an order of definitions and uses within a basic block.
* To do this we index definitions and uses as follows:
* Phi-functions have an index of -1, so precede all normal uses and definitions in a block.
@@ -337,10 +366,8 @@ import SsaComputeImpl::LivenessImpl as Liveness
* * a definition at the `j`th node of a block is given the index `2*j + 1`.
*/
pragma [inline]
pragma[inline]
int phiIndex() { result = -1 }
pragma [inline]
pragma[inline]
int piIndex() { result = -2 }

72
python/ql/src/semmle/python/essa/SsaDefinitions.qll
View File

@@ -1,44 +1,48 @@
/** Provides classes and predicates for determining the uses and definitions of
/**
* Provides classes and predicates for determining the uses and definitions of
* variables for ESSA form.
*/
import python
private import semmle.python.pointsto.Base
cached module SsaSource {
cached
module SsaSource {
/** Holds if `v` is used as the receiver in a method call. */
cached predicate method_call_refinement(Variable v, ControlFlowNode use, CallNode call) {
cached
predicate method_call_refinement(Variable v, ControlFlowNode use, CallNode call) {
use = v.getAUse() and
call.getFunction().(AttrNode).getObject() = use and
not test_contains(_, call)
}
/** Holds if `v` is defined by assignment at `defn` and given `value`. */
cached predicate assignment_definition(Variable v, ControlFlowNode defn, ControlFlowNode value) {
cached
predicate assignment_definition(Variable v, ControlFlowNode defn, ControlFlowNode value) {
defn.(NameNode).defines(v) and defn.(DefinitionNode).getValue() = value
}
/** Holds if `v` is defined by assignment of the captured exception. */
cached predicate exception_capture(Variable v, NameNode defn) {
cached
predicate exception_capture(Variable v, NameNode defn) {
defn.defines(v) and
exists(ExceptFlowNode ex | ex.getName() = defn)
}
/** Holds if `v` is defined by a with statement. */
cached predicate with_definition(Variable v, ControlFlowNode defn) {
cached
predicate with_definition(Variable v, ControlFlowNode defn) {
exists(With with, Name var |
with.getOptionalVars() = var and
var.getAFlowNode() = defn |
var.getAFlowNode() = defn
|
var = v.getAStore()
)
}
/** Holds if `v` is defined by multiple assignment at `defn`. */
cached predicate multi_assignment_definition(Variable v, ControlFlowNode defn, int n, SequenceNode lhs) {
cached
predicate multi_assignment_definition(Variable v, ControlFlowNode defn, int n, SequenceNode lhs) {
(
defn.(NameNode).defines(v)
or
@@ -50,44 +54,52 @@ cached module SsaSource {
}
/** Holds if `v` is defined by a `for` statement, the definition being `defn` */
cached predicate iteration_defined_variable(Variable v, ControlFlowNode defn, ControlFlowNode sequence) {
cached
predicate iteration_defined_variable(Variable v, ControlFlowNode defn, ControlFlowNode sequence) {
exists(ForNode for | for.iterates(defn, sequence)) and
defn.(NameNode).defines(v)
}
/** Holds if `v` is a parameter variable and `defn` is the CFG node for that parameter. */
cached predicate parameter_definition(Variable v, ControlFlowNode defn) {
cached
predicate parameter_definition(Variable v, ControlFlowNode defn) {
exists(Function f, Name param |
f.getAnArg() = param or
f.getVararg() = param or
f.getKwarg() = param or
f.getKeywordOnlyArg(_) = param |
f.getKeywordOnlyArg(_) = param
|
defn.getNode() = param and
param.getVariable() = v
)
}
/** Holds if `v` is deleted at `del`. */
cached predicate deletion_definition(Variable v, DeletionNode del) {
cached
predicate deletion_definition(Variable v, DeletionNode del) {
del.getTarget().(NameNode).deletes(v)
}
/** Holds if the name of `var` refers to a submodule of a package and `f` is the entry point
/**
* Holds if the name of `var` refers to a submodule of a package and `f` is the entry point
* to the __init__ module of that package.
*/
cached predicate init_module_submodule_defn(SsaSourceVariable var, ControlFlowNode f) {
cached
predicate init_module_submodule_defn(SsaSourceVariable var, ControlFlowNode f) {
var instanceof GlobalVariable and
exists(Module init |
init.isPackageInit() and exists(init.getPackage().getSubModule(var.getName())) and
init.isPackageInit() and
exists(init.getPackage().getSubModule(var.getName())) and
init.getEntryNode() = f and
var.getScope() = init
)
}
/** Holds if the `v` is in scope at a `from import ... *` and may thus be redefined by that statement */
cached predicate import_star_refinement(SsaSourceVariable v, ControlFlowNode use, ControlFlowNode def) {
use = def and def instanceof ImportStarNode
and
cached
predicate import_star_refinement(SsaSourceVariable v, ControlFlowNode use, ControlFlowNode def) {
use = def and
def instanceof ImportStarNode and
(
v.getScope() = def.getScope()
or
@@ -99,13 +111,16 @@ cached module SsaSource {
}
/** Holds if an attribute is assigned at `def` and `use` is the use of `v` for that assignment */
cached predicate attribute_assignment_refinement(Variable v, ControlFlowNode use, ControlFlowNode def) {
cached
predicate attribute_assignment_refinement(Variable v, ControlFlowNode use, ControlFlowNode def) {
use.(NameNode).uses(v) and
def.isStore() and def.(AttrNode).getObject() = use
def.isStore() and
def.(AttrNode).getObject() = use
}
/** Holds if a `v` is used as an argument to `call`, which *may* modify the object referred to by `v` */
cached predicate argument_refinement(Variable v, ControlFlowNode use, CallNode call) {
cached
predicate argument_refinement(Variable v, ControlFlowNode use, CallNode call) {
use.(NameNode).uses(v) and
call.getArg(0) = use and
not method_call_refinement(v, _, call) and
@@ -113,13 +128,15 @@ cached module SsaSource {
}
/** Holds if an attribute is deleted at `def` and `use` is the use of `v` for that deletion */
cached predicate attribute_deletion_refinement(Variable v, NameNode use, DeletionNode def) {
cached
predicate attribute_deletion_refinement(Variable v, NameNode use, DeletionNode def) {
use.uses(v) and
def.getTarget().(AttrNode).getObject() = use
}
/** Holds if the set of possible values for `v` is refined by `test` and `use` is the use of `v` in that test. */
cached predicate test_refinement(Variable v, ControlFlowNode use, ControlFlowNode test) {
cached
predicate test_refinement(Variable v, ControlFlowNode use, ControlFlowNode test) {
use.(NameNode).uses(v) and
test.getAChild*() = use and
test.isBranch() and
@@ -129,5 +146,4 @@ cached module SsaSource {
not block.getLastNode() = test
)
}
}

127
python/ql/src/semmle/python/filters/GeneratedCode.qll
View File

@@ -5,105 +5,118 @@ import semmle.python.templates.Templates
* A file that is detected as being generated.
*/
abstract class GeneratedFile extends File {
abstract string getTool();
}
/* We distinguish between a "lax" match which just includes "generated by" or similar versus a "strict" match which includes "this file is generated by" or similar
/*
* We distinguish between a "lax" match which just includes "generated by" or similar versus a "strict" match which includes "this file is generated by" or similar
* "lax" matches are taken to indicate generated file if they occur at the top of a file. "strict" matches can occur anywhere.
* There is no formal reason for the above, it just seems to work well in practice.
*/
library class GenericGeneratedFile extends GeneratedFile {
GenericGeneratedFile() {
not this instanceof SpecificGeneratedFile
and
not this instanceof SpecificGeneratedFile and
(
(lax_generated_by(this, _) or lax_generated_from(this, _)) and dont_modify(this)
(lax_generated_by(this, _) or lax_generated_from(this, _)) and
dont_modify(this)
or
strict_generated_by(this, _) or strict_generated_from(this, _)
strict_generated_by(this, _)
or
strict_generated_from(this, _)
or
auto_generated(this)
)
}
override string getTool() {
lax_generated_by(this, result) or strict_generated_by(this, result)
}
override string getTool() { lax_generated_by(this, result) or strict_generated_by(this, result) }
}
private string comment_or_docstring(File f, boolean before_code) {
exists(Comment c |
exists(Comment c |
c.getLocation().getFile() = f and
result = c.getText() |
if exists(Stmt s | s.getEnclosingModule().getFile() = f and s.getLocation().getStartLine() < c.getLocation().getStartLine()) then
before_code = false
else
before_code = true
result = c.getText()
|
if
exists(Stmt s |
s.getEnclosingModule().getFile() = f and
s.getLocation().getStartLine() < c.getLocation().getStartLine()
)
then before_code = false
else before_code = true
)
or
exists(Module m | m.getFile() = f |
result = m.getDocString().getText() and
before_code = true
)
}
private predicate lax_generated_by(File f, string tool) {
exists(string comment | comment = comment_or_docstring(f, _) |
tool = comment.regexpCapture("(?is).*\\b(?:(?:auto[ -]?)?generated|created automatically) by (?:the )?([-/\\w.]+[-/\\w]).*", 1)
tool =
comment
.regexpCapture("(?is).*\\b(?:(?:auto[ -]?)?generated|created automatically) by (?:the )?([-/\\w.]+[-/\\w]).*",
1)
)
}
private predicate lax_generated_from(File f, string src) {
exists(string comment | comment = comment_or_docstring(f, _) |
src = comment.regexpCapture("(?is).*\\b((?:auto[ -]?)?generated|created automatically) from ([-/\\w.]+[-/\\w]).*", 1)
)
exists(string comment | comment = comment_or_docstring(f, _) |
src =
comment
.regexpCapture("(?is).*\\b((?:auto[ -]?)?generated|created automatically) from ([-/\\w.]+[-/\\w]).*",
1)
)
}
private predicate strict_generated_by(File f, string tool) {
exists(string comment | comment = comment_or_docstring(f, true) |
tool = comment.regexpCapture("(?is)# *(?:this +)?(?:(?:code|file) +)?(?:is +)?(?:(?:auto(?:matically)?[ -]?)?generated|created automatically) by (?:the )?([-/\\w.]+[-/\\w]).*", 1)
tool =
comment
.regexpCapture("(?is)# *(?:this +)?(?:(?:code|file) +)?(?:is +)?(?:(?:auto(?:matically)?[ -]?)?generated|created automatically) by (?:the )?([-/\\w.]+[-/\\w]).*",
1)
)
}
private predicate strict_generated_from(File f, string src) {
exists(string comment | comment = comment_or_docstring(f, true) |
src = comment.regexpCapture("(?is)# *(?:this +)?(?:(?:code|file) +)?(?:is +)?(?:(?:auto(?:matically)?[ -]?)?generated|created automatically) from ([-/\\w.]+[-/\\w]).*", 1)
src =
comment
.regexpCapture("(?is)# *(?:this +)?(?:(?:code|file) +)?(?:is +)?(?:(?:auto(?:matically)?[ -]?)?generated|created automatically) from ([-/\\w.]+[-/\\w]).*",
1)
)
}
private predicate dont_modify(File f) {
comment_or_docstring(f, _).regexpMatch("(?is).*\\b(Do not|Don't) (edit|modify|make changes)\\b.*")
comment_or_docstring(f, _).regexpMatch("(?is).*\\b(Do not|Don't) (edit|modify|make changes)\\b.*")
}
private predicate auto_generated(File f) {
exists(Comment c |
c.getLocation().getFile() = f and
c.getText().regexpMatch("(?is)# *this +(code|file) +is +(auto(matically)?[ -]?generated|created automatically).*")
c
.getText()
.regexpMatch("(?is)# *this +(code|file) +is +(auto(matically)?[ -]?generated|created automatically).*")
)
}
/**
* A file generated by a template engine
* A file generated by a template engine
*/
abstract library class SpecificGeneratedFile extends GeneratedFile {
/* Currently cover Spitfire, Pyxl and Mako.
/*
* Currently cover Spitfire, Pyxl and Mako.
* Django templates are not compiled to Python.
* Jinja2 templates are compiled direct to bytecode via the ast.
*/
}
}
/** File generated by the spitfire templating engine */
class SpitfireGeneratedFile extends SpecificGeneratedFile {
SpitfireGeneratedFile() {
exists(Module m |
m.getFile() = this and not m instanceof SpitfireTemplate |
exists(Module m | m.getFile() = this and not m instanceof SpitfireTemplate |
exists(ImportMember template_method, ImportExpr spitfire_runtime_template |
spitfire_runtime_template.getName() = "spitfire.runtime.template" and
template_method.getModule() = spitfire_runtime_template and
@@ -112,28 +125,18 @@ class SpitfireGeneratedFile extends SpecificGeneratedFile {
)
}
override string getTool() {
result = "spitfire"
}
override string getTool() { result = "spitfire" }
}
/** File generated by the pyxl templating engine */
class PyxlGeneratedFile extends SpecificGeneratedFile {
PyxlGeneratedFile() { this.getSpecifiedEncoding() = "pyxl" }
PyxlGeneratedFile() {
this.getSpecifiedEncoding() = "pyxl"
}
override string getTool() {
result = "pyxl"
}
override string getTool() { result = "pyxl" }
}
/** File generated by the mako templating engine */
class MakoGeneratedFile extends SpecificGeneratedFile {
MakoGeneratedFile() {
exists(Module m | m.getFile() = this |
from_mako_import(m) = "runtime" and
@@ -147,38 +150,31 @@ class MakoGeneratedFile extends SpecificGeneratedFile {
) and
exists(Assign a, Name n |
a.getScope() = m and a.getATarget() = n and n.getId() = "_magic_number"
)
)
)
}
override string getTool() {
result = "mako"
}
override string getTool() { result = "mako" }
}
string from_mako_import(Module m) {
exists(ImportMember member, ImportExpr mako |
member.getScope() = m and
member.getModule() = mako and
mako.getName() = "mako" |
result = member.getName()
mako.getName() = "mako"
|
result = member.getName()
)
}
/** File generated by Google's protobuf tool. */
class ProtobufGeneratedFile extends SpecificGeneratedFile {
ProtobufGeneratedFile() {
this.getName().regexpMatch(".*_pb2?.py")
and
exists(Module m |
m.getFile() = this |
exists(ImportExpr imp |
imp.getEnclosingModule() = m |
this.getName().regexpMatch(".*_pb2?.py") and
exists(Module m | m.getFile() = this |
exists(ImportExpr imp | imp.getEnclosingModule() = m |
imp.getImportedModuleName() = "google.net.proto2.python.public"
)
and
) and
exists(AssignStmt a, Name n |
a.getEnclosingModule() = m and
a.getATarget() = n and
@@ -187,8 +183,5 @@ class ProtobufGeneratedFile extends SpecificGeneratedFile {
)
}
override string getTool() {
result = "protobuf"
}
override string getTool() { result = "protobuf" }
}

19
python/ql/src/semmle/python/filters/Tests.qll
View File

@@ -1,12 +1,11 @@
import python
abstract class TestScope extends Scope {}
abstract class TestScope extends Scope { }
// don't extend Class directly to avoid ambiguous method warnings
class UnitTestClass extends TestScope {
UnitTestClass() {
exists(ClassObject c |
this = c.getPyClass() |
exists(ClassObject c | this = c.getPyClass() |
c.getASuperType() = theUnitTestPackage().attr(_)
or
c.getASuperType().getName().toLowerCase() = "testcase"
@@ -14,35 +13,27 @@ class UnitTestClass extends TestScope {
}
}
PackageObject theUnitTestPackage() {
result.getName() = "unittest"
}
PackageObject theUnitTestPackage() { result.getName() = "unittest" }
abstract class Test extends TestScope {}
abstract class Test extends TestScope { }
class UnitTestFunction extends Test {
UnitTestFunction() {
this.getScope+() instanceof UnitTestClass
and
this.getScope+() instanceof UnitTestClass and
this.(Function).getName().matches("test%")
}
}
class PyTestFunction extends Test {
PyTestFunction() {
exists(Module pytest | pytest.getName() = "pytest") and
this.(Function).getName().matches("test%")
}
}
class NoseTestFunction extends Test {
NoseTestFunction() {
exists(Module nose | nose.getName() = "nose") and
this.(Function).getName().matches("test%")
}
}

27
python/ql/src/semmle/python/libraries/Zope.qll
View File

@@ -1,12 +1,10 @@
/** Utilities for handling the zope libraries */
import python
private import semmle.python.pointsto.PointsTo
/** A method that to a sub-class of `zope.interface.Interface` */
deprecated class ZopeInterfaceMethod extends PyFunctionObject {
/** Holds if this method belongs to a class that sub-classes `zope.interface.Interface` */
ZopeInterfaceMethod() {
exists(Object interface, ClassObject owner |
@@ -16,22 +14,17 @@ deprecated class ZopeInterfaceMethod extends PyFunctionObject {
)
}
override int minParameters() {
result = super.minParameters() + 1
}
override int minParameters() { result = super.minParameters() + 1 }
override int maxParameters() {
if exists(this.getFunction().getVararg()) then
result = super.maxParameters()
else
result = super.maxParameters() + 1
if exists(this.getFunction().getVararg())
then result = super.maxParameters()
else result = super.maxParameters() + 1
}
}
/** A method that belongs to a sub-class of `zope.interface.Interface` */
class ZopeInterfaceMethodValue extends PythonFunctionValue {
/** Holds if this method belongs to a class that sub-classes `zope.interface.Interface` */
ZopeInterfaceMethodValue() {
exists(Value interface, ClassValue owner |
@@ -44,15 +37,11 @@ class ZopeInterfaceMethodValue extends PythonFunctionValue {
)
}
override int minParameters() {
result = super.minParameters() + 1
}
override int minParameters() { result = super.minParameters() + 1 }
override int maxParameters() {
if exists(this.getScope().getVararg()) then
result = super.maxParameters()
else
result = super.maxParameters() + 1
if exists(this.getScope().getVararg())
then result = super.maxParameters()
else result = super.maxParameters() + 1
}
}

268
python/ql/src/semmle/python/objects/Callables.qll
View File

@@ -1,6 +1,4 @@
import python
private import semmle.python.objects.TObject
private import semmle.python.objects.ObjectInternal
private import semmle.python.pointsto.PointsTo
@@ -8,9 +6,7 @@ private import semmle.python.pointsto.PointsToContext
private import semmle.python.pointsto.MRO
private import semmle.python.types.Builtins
abstract class CallableObjectInternal extends ObjectInternal {
/** Gets the scope of this callable if it has one */
abstract Function getScope();
@@ -26,7 +22,10 @@ abstract class CallableObjectInternal extends ObjectInternal {
override ClassDecl getClassDeclaration() { none() }
pragma [noinline] override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() }
pragma[noinline]
override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) {
none()
}
abstract NameNode getParameter(int n);
@@ -36,11 +35,11 @@ abstract class CallableObjectInternal extends ObjectInternal {
override int length() { none() }
pragma [noinline] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) {
none()
}
pragma[noinline]
override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { none() }
pragma [noinline] override predicate attributesUnknown() { none() }
pragma[noinline]
override predicate attributesUnknown() { none() }
override predicate subscriptUnknown() { none() }
@@ -50,12 +49,10 @@ abstract class CallableObjectInternal extends ObjectInternal {
/* Callables aren't iterable */
override ObjectInternal getIterNext() { none() }
}
/** Class representing Python functions */
class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFunctionObject {
override Function getScope() {
exists(CallableExpr expr |
this = TPythonFunctionObject(expr.getAFlowNode()) and
@@ -63,9 +60,7 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti
)
}
override string toString() {
result = "Function " + this.getScope().getQualifiedName()
}
override string toString() { result = "Function " + this.getScope().getQualifiedName() }
override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
this = TPythonFunctionObject(node) and context.appliesTo(node)
@@ -77,19 +72,16 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti
override predicate notTestableForEquality() { none() }
override Builtin getBuiltin() {
none()
}
override Builtin getBuiltin() { none() }
override ControlFlowNode getOrigin() {
this = TPythonFunctionObject(result)
}
override ControlFlowNode getOrigin() { this = TPythonFunctionObject(result) }
pragma [nomagic]
pragma[nomagic]
override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) {
exists(Function func, ControlFlowNode rval, ControlFlowNode forigin |
func = this.getScope() and
callee.appliesToScope(func) |
callee.appliesToScope(func)
|
rval = func.getAReturnValueFlowNode() and
PointsToInternal::pointsTo(rval, callee, obj, forigin) and
origin = CfgOrigin::fromCfgNode(forigin)
@@ -98,17 +90,20 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti
procedureReturnsNone(callee, obj, origin)
}
private predicate procedureReturnsNone(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) {
private predicate procedureReturnsNone(
PointsToContext callee, ObjectInternal obj, CfgOrigin origin
) {
exists(Function func |
func = this.getScope() and
callee.appliesToScope(func) |
callee.appliesToScope(func)
|
PointsToInternal::reachableBlock(blockReturningNone(func), callee) and
obj = ObjectInternal::none_() and
origin = CfgOrigin::unknown()
)
}
pragma [noinline]
pragma[noinline]
override predicate callResult(ObjectInternal obj, CfgOrigin origin) {
this.getScope().isProcedure() and
obj = ObjectInternal::none_() and
@@ -119,41 +114,37 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti
scope = this.getScope() and paramOffset = 0
}
override string getName() {
result = this.getScope().getName()
}
override string getName() { result = this.getScope().getName() }
override boolean isDescriptor() { result = true }
pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) {
pragma[noinline]
override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) {
any(ObjectInternal obj).binds(cls, _, this) and
value = this and origin = CfgOrigin::fromCfgNode(this.getOrigin())
value = this and
origin = CfgOrigin::fromCfgNode(this.getOrigin())
}
pragma [noinline] override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) {
pragma[noinline]
override predicate descriptorGetInstance(
ObjectInternal instance, ObjectInternal value, CfgOrigin origin
) {
value = TBoundMethod(instance, this) and origin = CfgOrigin::unknown()
}
override CallNode getACall(PointsToContext ctx) {
PointsTo::pointsTo(result.getFunction(), ctx, this, _)
or
exists(BoundMethodObjectInternal bm |
bm.getACall(ctx) = result and this = bm.getFunction()
)
exists(BoundMethodObjectInternal bm | bm.getACall(ctx) = result and this = bm.getFunction())
}
override NameNode getParameter(int n) {
result.getNode() = this.getScope().getArg(n)
}
override NameNode getParameter(int n) { result.getNode() = this.getScope().getArg(n) }
override NameNode getParameterByName(string name) {
result.getNode() = this.getScope().getArgByName(name)
}
override predicate neverReturns() {
InterProceduralPointsTo::neverReturns(this.getScope())
}
override predicate neverReturns() { InterProceduralPointsTo::neverReturns(this.getScope()) }
override predicate functionAndOffset(CallableObjectInternal function, int offset) {
function = this and offset = 0
@@ -164,10 +155,8 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti
override predicate useOriginAsLegacyObject() { none() }
override predicate isNotSubscriptedType() { any() }
}
private BasicBlock blockReturningNone(Function func) {
exists(Return ret |
not exists(ret.getValue()) and
@@ -176,38 +165,31 @@ private BasicBlock blockReturningNone(Function func) {
)
}
/** Class representing built-in functions such as `len` or `print`. */
class BuiltinFunctionObjectInternal extends CallableObjectInternal, TBuiltinFunctionObject {
override Builtin getBuiltin() { this = TBuiltinFunctionObject(result) }
override Builtin getBuiltin() {
this = TBuiltinFunctionObject(result)
}
override string toString() { result = "Builtin-function " + this.getBuiltin().getName() }
override string toString() {
result = "Builtin-function " + this.getBuiltin().getName()
}
override predicate introducedAt(ControlFlowNode node, PointsToContext context) { none() }
override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
none()
}
override ObjectInternal getClass() {
result = TBuiltinClassObject(this.getBuiltin().getClass())
}
override ObjectInternal getClass() { result = TBuiltinClassObject(this.getBuiltin().getClass()) }
override predicate notTestableForEquality() { none() }
override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() }
override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) {
none()
}
pragma [noinline]
pragma[noinline]
override predicate callResult(ObjectInternal obj, CfgOrigin origin) {
exists(Builtin func, BuiltinClassObjectInternal cls |
func = this.getBuiltin() and
func != Builtin::builtin("isinstance") and
func != Builtin::builtin("issubclass") and
func != Builtin::builtin("callable") and
cls = ObjectInternal::fromBuiltin(this.getReturnType()) |
cls = ObjectInternal::fromBuiltin(this.getReturnType())
|
obj = TUnknownInstance(cls)
or
cls = ObjectInternal::noneType() and obj = ObjectInternal::none_()
@@ -221,17 +203,11 @@ class BuiltinFunctionObjectInternal extends CallableObjectInternal, TBuiltinFunc
origin = CfgOrigin::unknown()
}
override ControlFlowNode getOrigin() {
none()
}
override ControlFlowNode getOrigin() { none() }
override predicate calleeAndOffset(Function scope, int paramOffset) {
none()
}
override predicate calleeAndOffset(Function scope, int paramOffset) { none() }
override string getName() {
result = this.getBuiltin().getName()
}
override string getName() { result = this.getBuiltin().getName() }
Builtin getReturnType() {
exists(Builtin func |
@@ -251,21 +227,25 @@ class BuiltinFunctionObjectInternal extends CallableObjectInternal, TBuiltinFunc
override boolean isDescriptor() { result = false }
pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() }
pragma[noinline]
override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) {
none()
}
pragma [noinline] override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() }
pragma[noinline]
override predicate descriptorGetInstance(
ObjectInternal instance, ObjectInternal value, CfgOrigin origin
) {
none()
}
override CallNode getACall(PointsToContext ctx) {
PointsTo::pointsTo(result.getFunction(), ctx, this, _)
}
override NameNode getParameter(int n) {
none()
}
override NameNode getParameter(int n) { none() }
override NameNode getParameterByName(string name) {
none()
}
override NameNode getParameterByName(string name) { none() }
override predicate neverReturns() {
exists(ModuleObjectInternal sys |
@@ -283,7 +263,6 @@ class BuiltinFunctionObjectInternal extends CallableObjectInternal, TBuiltinFunc
override predicate useOriginAsLegacyObject() { none() }
override predicate isNotSubscriptedType() { any() }
}
private Builtin getBuiltinFunctionReturnType(Builtin func) {
@@ -297,7 +276,8 @@ private Builtin getBuiltinFunctionReturnType(Builtin func) {
func = Builtin::builtin("__import__") and result = Builtin::special("ModuleType")
or
/* Fix a few minor inaccuracies in the CPython analysis */
ext_rettype(func, result) and not (
ext_rettype(func, result) and
not (
func = Builtin::builtin("__import__")
or
func = Builtin::builtin("compile") and result = Builtin::special("NoneType")
@@ -309,34 +289,27 @@ private Builtin getBuiltinFunctionReturnType(Builtin func) {
}
/** Class representing methods of built-in classes (otherwise known as method-descriptors) such as `list.append`. */
class BuiltinMethodObjectInternal extends CallableObjectInternal, TBuiltinMethodObject {
override Builtin getBuiltin() { this = TBuiltinMethodObject(result) }
override Builtin getBuiltin() {
this = TBuiltinMethodObject(result)
}
override string toString() { result = "builtin method " + this.getBuiltin().getName() }
override string toString() {
result = "builtin method " + this.getBuiltin().getName()
}
override ObjectInternal getClass() { result = TBuiltinClassObject(this.getBuiltin().getClass()) }
override ObjectInternal getClass() {
result = TBuiltinClassObject(this.getBuiltin().getClass())
}
override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
none()
}
override predicate introducedAt(ControlFlowNode node, PointsToContext context) { none() }
override predicate notTestableForEquality() { none() }
override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() }
override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) {
none()
}
pragma [noinline]
pragma[noinline]
override predicate callResult(ObjectInternal obj, CfgOrigin origin) {
exists(Builtin func, BuiltinClassObjectInternal cls |
func = this.getBuiltin() and
cls = ObjectInternal::fromBuiltin(this.getReturnType()) |
cls = ObjectInternal::fromBuiltin(this.getReturnType())
|
obj = TUnknownInstance(cls)
or
cls = ObjectInternal::noneType() and obj = ObjectInternal::none_()
@@ -352,41 +325,34 @@ class BuiltinMethodObjectInternal extends CallableObjectInternal, TBuiltinMethod
Builtin getReturnType() {
/* If we have a record of the return type in our stubs, use that. */
exists(Builtin func |
func = this.getBuiltin() |
ext_rettype(func, result)
)
exists(Builtin func | func = this.getBuiltin() | ext_rettype(func, result))
}
private predicate returnTypeUnknown() {
exists(Builtin func |
func = this.getBuiltin() |
not ext_rettype(func, _)
)
exists(Builtin func | func = this.getBuiltin() | not ext_rettype(func, _))
}
override ControlFlowNode getOrigin() {
none()
}
override ControlFlowNode getOrigin() { none() }
override predicate calleeAndOffset(Function scope, int paramOffset) {
none()
}
override predicate calleeAndOffset(Function scope, int paramOffset) { none() }
override string getName() {
result = this.getBuiltin().getName()
}
override string getName() { result = this.getBuiltin().getName() }
override Function getScope() { none() }
override boolean isDescriptor() { result = true }
pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) {
pragma[noinline]
override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) {
any(ObjectInternal obj).binds(cls, _, this) and
value = this and origin = CfgOrigin::unknown()
value = this and
origin = CfgOrigin::unknown()
}
pragma [noinline] override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) {
pragma[noinline]
override predicate descriptorGetInstance(
ObjectInternal instance, ObjectInternal value, CfgOrigin origin
) {
value = TBoundMethod(instance, this) and origin = CfgOrigin::unknown()
}
@@ -394,13 +360,9 @@ class BuiltinMethodObjectInternal extends CallableObjectInternal, TBuiltinMethod
PointsTo::pointsTo(result.getFunction(), ctx, this, _)
}
override NameNode getParameter(int n) {
none()
}
override NameNode getParameter(int n) { none() }
override NameNode getParameterByName(string name) {
none()
}
override NameNode getParameterByName(string name) { none() }
override predicate neverReturns() { none() }
@@ -413,27 +375,20 @@ class BuiltinMethodObjectInternal extends CallableObjectInternal, TBuiltinMethod
override predicate useOriginAsLegacyObject() { none() }
override predicate isNotSubscriptedType() { any() }
}
/** Class representing bound-methods.
/**
* Class representing bound-methods.
* Note that built-in methods, such as `[].append` are also represented as bound-methods.
* Although built-in methods and bound-methods are distinct classes in CPython, their behaviour
* is the same and we treat them identically.
*/
class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod {
override Builtin getBuiltin() { none() }
override Builtin getBuiltin() {
none()
}
CallableObjectInternal getFunction() { this = TBoundMethod(_, result) }
CallableObjectInternal getFunction() {
this = TBoundMethod(_, result)
}
ObjectInternal getSelf() {
this = TBoundMethod(result, _)
}
ObjectInternal getSelf() { this = TBoundMethod(result, _) }
override string toString() {
result = "Method(" + this.getFunction() + ", " + this.getSelf() + ")"
@@ -443,9 +398,7 @@ class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod {
result = TBuiltinClassObject(Builtin::special("MethodType"))
}
override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
none()
}
override predicate introducedAt(ControlFlowNode node, PointsToContext context) { none() }
override predicate notTestableForEquality() { any() }
@@ -457,43 +410,41 @@ class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod {
this.getFunction().callResult(obj, origin)
}
override ControlFlowNode getOrigin() {
none()
}
override ControlFlowNode getOrigin() { none() }
override predicate calleeAndOffset(Function scope, int paramOffset) {
this.getFunction().calleeAndOffset(scope, paramOffset-1)
this.getFunction().calleeAndOffset(scope, paramOffset - 1)
}
override string getName() {
result = this.getFunction().getName()
}
override string getName() { result = this.getFunction().getName() }
override Function getScope() {
result = this.getFunction().getScope()
}
override Function getScope() { result = this.getFunction().getScope() }
override boolean isDescriptor() { result = false }
pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() }
pragma[noinline]
override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) {
none()
}
pragma [noinline] override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() }
pragma[noinline]
override predicate descriptorGetInstance(
ObjectInternal instance, ObjectInternal value, CfgOrigin origin
) {
none()
}
override CallNode getACall(PointsToContext ctx) {
PointsTo::pointsTo(result.getFunction(), ctx, this, _)
}
override NameNode getParameter(int n) {
result = this.getFunction().getParameter(n+1)
}
override NameNode getParameter(int n) { result = this.getFunction().getParameter(n + 1) }
override NameNode getParameterByName(string name) {
result = this.getFunction().getParameterByName(name)
}
override predicate neverReturns() {
this.getFunction().neverReturns()
}
override predicate neverReturns() { this.getFunction().neverReturns() }
override predicate functionAndOffset(CallableObjectInternal function, int offset) {
function = this.getFunction() and offset = 1
@@ -501,10 +452,7 @@ class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod {
override predicate useOriginAsLegacyObject() { any() }
override predicate contextSensitiveCallee() {
this.getFunction().contextSensitiveCallee()
}
override predicate contextSensitiveCallee() { this.getFunction().contextSensitiveCallee() }
override predicate isNotSubscriptedType() { any() }
}
}

259
python/ql/src/semmle/python/objects/Classes.qll
View File

@@ -1,6 +1,4 @@
import python
private import semmle.python.objects.TObject
private import semmle.python.objects.ObjectInternal
private import semmle.python.pointsto.PointsTo
@@ -10,19 +8,16 @@ private import semmle.python.types.Builtins
/** Class representing classes */
abstract class ClassObjectInternal extends ObjectInternal {
override string getName() { result = this.getClassDeclaration().getName() }
override string getName() {
result = this.getClassDeclaration().getName()
}
/** Holds if this is a class whose instances we treat specially, rather than as a generic instance.
/**
* Holds if this is a class whose instances we treat specially, rather than as a generic instance.
* For example, `type` or `int`.
*/
boolean isSpecial() {
result = Types::getMro(this).containsSpecial()
}
boolean isSpecial() { result = Types::getMro(this).containsSpecial() }
/** Looks up the attribute `name` on this class.
/**
* Looks up the attribute `name` on this class.
* Note that this may be different from `this.attr(name)`.
* For example given the class:
* ```class C:
@@ -51,11 +46,20 @@ abstract class ClassObjectInternal extends ObjectInternal {
override boolean isDescriptor() { result = false }
pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() }
pragma[noinline]
override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) {
none()
}
pragma [noinline] override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() }
pragma[noinline]
override predicate descriptorGetInstance(
ObjectInternal instance, ObjectInternal value, CfgOrigin origin
) {
none()
}
pragma [noinline] override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) {
pragma[noinline]
override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) {
instance = this and
PointsToInternal::attributeRequired(this, name) and
this.lookup(name, descriptor, _) and
@@ -63,11 +67,14 @@ abstract class ClassObjectInternal extends ObjectInternal {
}
/** Approximation to descriptor protocol, skipping meta-descriptor protocol */
pragma [noinline] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) {
pragma[noinline]
override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) {
exists(ObjectInternal descriptor, CfgOrigin desc_origin |
this.lookup(name, descriptor, desc_origin) |
this.lookup(name, descriptor, desc_origin)
|
descriptor.isDescriptor() = false and
value = descriptor and origin = desc_origin
value = descriptor and
origin = desc_origin
or
descriptor.isDescriptor() = true and
descriptor.descriptorGetClass(this, value, origin)
@@ -80,13 +87,9 @@ abstract class ClassObjectInternal extends ObjectInternal {
override boolean isClass() { result = true }
override int intValue() {
none()
}
override int intValue() { none() }
override string strValue() {
none()
}
override string strValue() { none() }
override predicate subscriptUnknown() { none() }
@@ -104,12 +107,10 @@ abstract class ClassObjectInternal extends ObjectInternal {
}
override predicate isNotSubscriptedType() { any() }
}
/** Class representing Python source classes */
class PythonClassObjectInternal extends ClassObjectInternal, TPythonClassObject {
/** Gets the scope for this Python class */
Class getScope() {
exists(ClassExpr expr |
@@ -118,34 +119,24 @@ class PythonClassObjectInternal extends ClassObjectInternal, TPythonClassObject
)
}
override string toString() {
result = "class " + this.getScope().getName()
}
override string toString() { result = "class " + this.getScope().getName() }
override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
this = TPythonClassObject(node) and context.appliesTo(node)
}
override ClassDecl getClassDeclaration() {
this = TPythonClassObject(result)
}
override ClassDecl getClassDeclaration() { this = TPythonClassObject(result) }
override ObjectInternal getClass() {
result = Types::getMetaClass(this)
}
override ObjectInternal getClass() { result = Types::getMetaClass(this) }
override Builtin getBuiltin() {
none()
}
override Builtin getBuiltin() { none() }
override ControlFlowNode getOrigin() {
this = TPythonClassObject(result)
}
override ControlFlowNode getOrigin() { this = TPythonClassObject(result) }
override predicate calleeAndOffset(Function scope, int paramOffset) {
exists(PythonFunctionObjectInternal init |
this.lookup("__init__", init, _) and
init.calleeAndOffset(scope, paramOffset-1)
init.calleeAndOffset(scope, paramOffset - 1)
)
}
@@ -153,7 +144,8 @@ class PythonClassObjectInternal extends ClassObjectInternal, TPythonClassObject
Types::getMro(this).lookup(name, value, origin)
}
pragma [noinline] override predicate attributesUnknown() { none() }
pragma[noinline]
override predicate attributesUnknown() { none() }
override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) {
none()
@@ -169,27 +161,17 @@ class PythonClassObjectInternal extends ClassObjectInternal, TPythonClassObject
override predicate functionAndOffset(CallableObjectInternal function, int offset) {
this.lookup("__init__", function, _) and offset = 1
}
}
/** Class representing built-in classes, except `type` */
class BuiltinClassObjectInternal extends ClassObjectInternal, TBuiltinClassObject {
override Builtin getBuiltin() { this = TBuiltinClassObject(result) }
override Builtin getBuiltin() {
this = TBuiltinClassObject(result)
}
override string toString() { result = "builtin-class " + this.getBuiltin().getName() }
override string toString() {
result = "builtin-class " + this.getBuiltin().getName()
}
override predicate introducedAt(ControlFlowNode node, PointsToContext context) { none() }
override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
none()
}
override ClassDecl getClassDeclaration() {
this = TBuiltinClassObject(result)
}
override ClassDecl getClassDeclaration() { this = TBuiltinClassObject(result) }
override ObjectInternal getClass() {
result = TBuiltinClassObject(this.getBuiltin().getClass())
@@ -198,19 +180,16 @@ class BuiltinClassObjectInternal extends ClassObjectInternal, TBuiltinClassObjec
result = TType()
}
override ControlFlowNode getOrigin() {
none()
}
override ControlFlowNode getOrigin() { none() }
override predicate calleeAndOffset(Function scope, int paramOffset) {
none()
}
override predicate calleeAndOffset(Function scope, int paramOffset) { none() }
override predicate lookup(string name, ObjectInternal value, CfgOrigin origin) {
Types::getMro(this).lookup(name, value, origin)
}
pragma [noinline] override predicate attributesUnknown() { none() }
pragma[noinline]
override predicate attributesUnknown() { none() }
override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) {
none()
@@ -222,33 +201,21 @@ class BuiltinClassObjectInternal extends ClassObjectInternal, TBuiltinClassObjec
}
override predicate notTestableForEquality() { none() }
}
/** A class representing an unknown class */
class UnknownClassInternal extends ClassObjectInternal, TUnknownClass {
override string toString() { result = "Unknown class" }
override string toString() {
result = "Unknown class"
}
override ClassDecl getClassDeclaration() { result = Builtin::unknownType() }
override ClassDecl getClassDeclaration() {
result = Builtin::unknownType()
}
override ObjectInternal getClass() { result = this }
override ObjectInternal getClass() {
result = this
}
override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
none()
}
override predicate introducedAt(ControlFlowNode node, PointsToContext context) { none() }
override predicate notTestableForEquality() { any() }
override Builtin getBuiltin() {
result = Builtin::unknownType()
}
override Builtin getBuiltin() { result = Builtin::unknownType() }
override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) {
none()
@@ -258,134 +225,94 @@ class UnknownClassInternal extends ClassObjectInternal, TUnknownClass {
obj = ObjectInternal::unknown() and origin = CfgOrigin::unknown()
}
override ControlFlowNode getOrigin() {
none()
}
override ControlFlowNode getOrigin() { none() }
override predicate calleeAndOffset(Function scope, int paramOffset) {
none()
}
override predicate calleeAndOffset(Function scope, int paramOffset) { none() }
override predicate lookup(string name, ObjectInternal value, CfgOrigin origin) {
none()
}
pragma [noinline] override predicate attributesUnknown() { any() }
override predicate lookup(string name, ObjectInternal value, CfgOrigin origin) { none() }
pragma[noinline]
override predicate attributesUnknown() { any() }
}
/** A class representing the built-in class `type`. */
class TypeInternal extends ClassObjectInternal, TType {
override string toString() { result = "builtin-class type" }
override string toString() {
result = "builtin-class type"
}
override ClassDecl getClassDeclaration() { result = Builtin::special("type") }
override ClassDecl getClassDeclaration() {
result = Builtin::special("type")
}
override ObjectInternal getClass() { result = this }
override ObjectInternal getClass() {
result = this
}
override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
none()
}
override predicate introducedAt(ControlFlowNode node, PointsToContext context) { none() }
override predicate notTestableForEquality() { none() }
override Builtin getBuiltin() {
result = Builtin::special("type")
}
override Builtin getBuiltin() { result = Builtin::special("type") }
override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) {
none()
}
override predicate callResult(ObjectInternal obj, CfgOrigin origin) {
none()
}
override predicate callResult(ObjectInternal obj, CfgOrigin origin) { none() }
override ControlFlowNode getOrigin() {
none()
}
override ControlFlowNode getOrigin() { none() }
override predicate calleeAndOffset(Function scope, int paramOffset) {
none()
}
override predicate calleeAndOffset(Function scope, int paramOffset) { none() }
override predicate lookup(string name, ObjectInternal value, CfgOrigin origin) {
Types::getMro(this).lookup(name, value, origin)
}
pragma [noinline] override predicate attributesUnknown() { any() }
pragma[noinline]
override predicate attributesUnknown() { any() }
}
/** A class representing a dynamically created class `type(name, *args, **kwargs)`. */
class DynamicallyCreatedClass extends ClassObjectInternal, TDynamicClass {
override string toString() { result = this.getOrigin().getNode().toString() }
override string toString() {
result = this.getOrigin().getNode().toString()
}
override ObjectInternal getClass() {
this = TDynamicClass(_, result, _)
}
override ObjectInternal getClass() { this = TDynamicClass(_, result, _) }
override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) {
none()
}
override predicate callResult(ObjectInternal obj, CfgOrigin origin) {
none()
}
override predicate callResult(ObjectInternal obj, CfgOrigin origin) { none() }
override predicate lookup(string name, ObjectInternal value, CfgOrigin origin) {
exists(ClassObjectInternal decl |
decl = Types::getMro(this).findDeclaringClass(name) |
exists(ClassObjectInternal decl | decl = Types::getMro(this).findDeclaringClass(name) |
Types::declaredAttribute(decl, name, value, origin)
)
}
override Builtin getBuiltin() {
none()
}
override Builtin getBuiltin() { none() }
override ControlFlowNode getOrigin() {
this = TDynamicClass(result, _, _)
}
override ControlFlowNode getOrigin() { this = TDynamicClass(result, _, _) }
pragma [noinline] override predicate attributesUnknown() { any() }
pragma[noinline]
override predicate attributesUnknown() { any() }
override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
this = TDynamicClass(node, _, context)
}
override predicate calleeAndOffset(Function scope, int paramOffset) {
none()
}
override predicate calleeAndOffset(Function scope, int paramOffset) { none() }
override predicate notTestableForEquality() { none() }
override ClassDecl getClassDeclaration() { none() }
}
class SubscriptedTypeInternal extends ObjectInternal, TSubscriptedType {
ObjectInternal getGeneric() { this = TSubscriptedType(result, _) }
ObjectInternal getGeneric() {
this = TSubscriptedType(result, _)
}
ObjectInternal getSpecializer() {
this = TSubscriptedType(_, result)
}
ObjectInternal getSpecializer() { this = TSubscriptedType(_, result) }
override string getName() { result = this.getGeneric().getName() }
override string toString() { result = this.getGeneric().toString() + "[" + this.getSpecializer().toString() + "]" }
override string toString() {
result = this.getGeneric().toString() + "[" + this.getSpecializer().toString() + "]"
}
override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
exists(ObjectInternal generic, ObjectInternal index |
@@ -395,16 +322,12 @@ class SubscriptedTypeInternal extends ObjectInternal, TSubscriptedType {
}
/** Gets the class declaration for this object, if it is a class with a declaration. */
override ClassDecl getClassDeclaration() {
result = this.getGeneric().getClassDeclaration()
}
override ClassDecl getClassDeclaration() { result = this.getGeneric().getClassDeclaration() }
/** True if this "object" is a class. That is, its class inherits from `type` */
override boolean isClass() { result = true }
override ObjectInternal getClass() {
result = this.getGeneric().getClass()
}
override ObjectInternal getClass() { result = this.getGeneric().getClass() }
override predicate notTestableForEquality() { none() }
@@ -414,9 +337,11 @@ class SubscriptedTypeInternal extends ObjectInternal, TSubscriptedType {
override predicate callResult(ObjectInternal obj, CfgOrigin origin) { none() }
override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() }
override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) {
none()
}
override predicate calleeAndOffset(Function scope, int paramOffset){ none() }
override predicate calleeAndOffset(Function scope, int paramOffset) { none() }
override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { none() }
@@ -424,17 +349,25 @@ class SubscriptedTypeInternal extends ObjectInternal, TSubscriptedType {
override boolean isDescriptor() { result = false }
override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() }
override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) {
none()
}
override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() }
override predicate descriptorGetInstance(
ObjectInternal instance, ObjectInternal value, CfgOrigin origin
) {
none()
}
override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() }
override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) {
none()
}
override int length() { none() }
override boolean booleanValue() { result = true }
override int intValue() { none()}
override int intValue() { none() }
override string strValue() { none() }
@@ -448,8 +381,4 @@ class SubscriptedTypeInternal extends ObjectInternal, TSubscriptedType {
override ObjectInternal getIterNext() { result = ObjectInternal::unknown() }
override predicate isNotSubscriptedType() { none() }
}

245
python/ql/src/semmle/python/objects/Constants.qll
View File

@@ -1,5 +1,4 @@
import python
private import semmle.python.objects.TObject
private import semmle.python.objects.ObjectInternal
private import semmle.python.pointsto.PointsTo
@@ -7,16 +6,13 @@ private import semmle.python.pointsto.MRO
private import semmle.python.pointsto.PointsToContext
private import semmle.python.types.Builtins
/** Class representing constants.
/**
* Class representing constants.
* Includes `None`, `True` and `False` as
* well as strings and integers.
*/
abstract class ConstantObjectInternal extends ObjectInternal {
override ClassDecl getClassDeclaration() {
none()
}
override ClassDecl getClassDeclaration() { none() }
override boolean isClass() { result = false }
@@ -32,33 +28,41 @@ abstract class ConstantObjectInternal extends ObjectInternal {
none()
}
override ControlFlowNode getOrigin() {
none()
}
override ControlFlowNode getOrigin() { none() }
override predicate calleeAndOffset(Function scope, int paramOffset) {
none()
}
override predicate calleeAndOffset(Function scope, int paramOffset) { none() }
pragma [noinline] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) {
pragma[noinline]
override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) {
PointsToInternal::attributeRequired(this, name) and
exists(ObjectInternal cls_attr, CfgOrigin attr_orig |
this.getClass().(ClassObjectInternal).lookup(name, cls_attr, attr_orig) and
cls_attr.isDescriptor() = true and cls_attr.descriptorGetInstance(this, value, origin)
cls_attr.isDescriptor() = true and
cls_attr.descriptorGetInstance(this, value, origin)
)
}
pragma [noinline] override predicate attributesUnknown() { none() }
pragma[noinline]
override predicate attributesUnknown() { none() }
override predicate subscriptUnknown() { none() }
override boolean isDescriptor() { result = false }
pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() }
pragma[noinline]
override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) {
none()
}
pragma [noinline] override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() }
pragma[noinline]
override predicate descriptorGetInstance(
ObjectInternal instance, ObjectInternal value, CfgOrigin origin
) {
none()
}
pragma [noinline] override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) {
pragma[noinline]
override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) {
exists(ClassObjectInternal cls |
receiver_type(_, name, this, cls) and
cls.lookup(name, descriptor, _) and
@@ -77,29 +81,23 @@ abstract class ConstantObjectInternal extends ObjectInternal {
abstract ImmutableLiteral getLiteral();
override predicate isNotSubscriptedType() { any() }
}
pragma[nomagic]
private boolean callToBool(CallNode call, PointsToContext context) {
PointsToInternal::pointsTo(call.getFunction(), context, ClassValue::bool(), _) and
exists(ObjectInternal arg |
PointsToInternal::pointsTo(call.getArg(0),context, arg, _) and
PointsToInternal::pointsTo(call.getArg(0), context, arg, _) and
arg.booleanValue() = result
)
}
private abstract class BooleanObjectInternal extends ConstantObjectInternal {
override ObjectInternal getClass() {
result = ClassValue::bool()
}
abstract private class BooleanObjectInternal extends ConstantObjectInternal {
override ObjectInternal getClass() { result = ClassValue::bool() }
override int length() { none() }
override string strValue() {
none()
}
override string strValue() { none() }
/* Booleans aren't iterable */
override ObjectInternal getIterNext() { none() }
@@ -107,18 +105,12 @@ private abstract class BooleanObjectInternal extends ConstantObjectInternal {
override ImmutableLiteral getLiteral() {
result.(BooleanLiteral).booleanValue() = this.booleanValue()
}
}
private class TrueObjectInternal extends BooleanObjectInternal, TTrue {
override string toString() { result = "bool True" }
override string toString() {
result = "bool True"
}
override boolean booleanValue() {
result = true
}
override boolean booleanValue() { result = true }
override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
node.(NameNode).getId() = "True" and context.appliesTo(node)
@@ -126,25 +118,15 @@ private class TrueObjectInternal extends BooleanObjectInternal, TTrue {
callToBool(node, context) = true
}
override int intValue() {
result = 1
}
override Builtin getBuiltin() {
result = Builtin::special("True")
}
override int intValue() { result = 1 }
override Builtin getBuiltin() { result = Builtin::special("True") }
}
private class FalseObjectInternal extends BooleanObjectInternal, TFalse {
override string toString() { result = "bool False" }
override string toString() {
result = "bool False"
}
override boolean booleanValue() {
result = false
}
override boolean booleanValue() { result = false }
override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
node.(NameNode).getId() = "False" and context.appliesTo(node)
@@ -152,84 +134,51 @@ private class FalseObjectInternal extends BooleanObjectInternal, TFalse {
callToBool(node, context) = false
}
override int intValue() {
result = 0
}
override Builtin getBuiltin() {
result = Builtin::special("False")
}
override int intValue() { result = 0 }
override Builtin getBuiltin() { result = Builtin::special("False") }
}
private class NoneObjectInternal extends ConstantObjectInternal, TNone {
override string toString() { result = "None" }
override string toString() {
result = "None"
}
override boolean booleanValue() { result = false }
override boolean booleanValue() {
result = false
}
override ObjectInternal getClass() {
result = TBuiltinClassObject(Builtin::special("NoneType"))
}
override ObjectInternal getClass() { result = TBuiltinClassObject(Builtin::special("NoneType")) }
override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
node.(NameNode).getId() = "None" and context.appliesTo(node)
}
override Builtin getBuiltin() {
result = Builtin::special("None")
}
override Builtin getBuiltin() { result = Builtin::special("None") }
override int intValue() {
none()
}
override int intValue() { none() }
override string strValue() {
none()
}
override string strValue() { none() }
override int length() { none() }
/* None isn't iterable */
override ObjectInternal getIterNext() { none() }
override ImmutableLiteral getLiteral() {
result instanceof None
}
override ImmutableLiteral getLiteral() { result instanceof None }
}
class IntObjectInternal extends ConstantObjectInternal, TInt {
override string toString() {
result = "int " + this.intValue().toString()
}
override string toString() { result = "int " + this.intValue().toString() }
override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
context.appliesTo(node) and
node.getNode().(IntegerLiteral).getValue() = this.intValue()
}
override ObjectInternal getClass() {
result = TBuiltinClassObject(Builtin::special("int"))
}
override ObjectInternal getClass() { result = TBuiltinClassObject(Builtin::special("int")) }
override Builtin getBuiltin() {
result.intValue() = this.intValue()
}
override Builtin getBuiltin() { result.intValue() = this.intValue() }
override int intValue() {
this = TInt(result)
}
override int intValue() { this = TInt(result) }
override string strValue() {
none()
}
override string strValue() { none() }
override boolean booleanValue() {
this.intValue() = 0 and result = false
@@ -247,17 +196,13 @@ class IntObjectInternal extends ConstantObjectInternal, TInt {
or
result.(NegativeIntegerLiteral).getOperand().(IntegerLiteral).getValue() = -this.intValue()
}
}
class FloatObjectInternal extends ConstantObjectInternal, TFloat {
override string toString() {
if this.floatValue() = this.floatValue().floor() then (
result = "float " + this.floatValue().floor().toString() + ".0"
) else (
result = "float " + this.floatValue().toString()
)
if this.floatValue() = this.floatValue().floor()
then result = "float " + this.floatValue().floor().toString() + ".0"
else result = "float " + this.floatValue().toString()
}
override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
@@ -265,25 +210,15 @@ class FloatObjectInternal extends ConstantObjectInternal, TFloat {
node.getNode().(FloatLiteral).getValue() = this.floatValue()
}
override ObjectInternal getClass() {
result = TBuiltinClassObject(Builtin::special("float"))
}
override ObjectInternal getClass() { result = TBuiltinClassObject(Builtin::special("float")) }
override Builtin getBuiltin() {
result.floatValue() = this.floatValue()
}
override Builtin getBuiltin() { result.floatValue() = this.floatValue() }
float floatValue() {
this = TFloat(result)
}
float floatValue() { this = TFloat(result) }
override int intValue() {
this = TFloat(result)
}
override int intValue() { this = TFloat(result) }
override string strValue() {
none()
}
override string strValue() { none() }
override boolean booleanValue() {
this.floatValue() = 0.0 and result = false
@@ -296,18 +231,11 @@ class FloatObjectInternal extends ConstantObjectInternal, TFloat {
/* floats aren't iterable */
override ObjectInternal getIterNext() { none() }
override ImmutableLiteral getLiteral() {
result.(FloatLiteral).getValue() = this.floatValue()
}
override ImmutableLiteral getLiteral() { result.(FloatLiteral).getValue() = this.floatValue() }
}
class UnicodeObjectInternal extends ConstantObjectInternal, TUnicode {
override string toString() {
result = "'" + this.strValue() + "'"
}
override string toString() { result = "'" + this.strValue() + "'" }
override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
context.appliesTo(node) and
@@ -315,22 +243,16 @@ class UnicodeObjectInternal extends ConstantObjectInternal, TUnicode {
node.getNode().(StrConst).isUnicode()
}
override ObjectInternal getClass() {
result = TBuiltinClassObject(Builtin::special("unicode"))
}
override ObjectInternal getClass() { result = TBuiltinClassObject(Builtin::special("unicode")) }
override Builtin getBuiltin() {
result.(Builtin).strValue() = this.strValue() and
result.getClass() = Builtin::special("unicode")
}
override int intValue() {
none()
}
override int intValue() { none() }
override string strValue() {
this = TUnicode(result)
}
override string strValue() { this = TUnicode(result) }
override boolean booleanValue() {
this.strValue() = "" and result = false
@@ -338,25 +260,15 @@ class UnicodeObjectInternal extends ConstantObjectInternal, TUnicode {
this.strValue() != "" and result = true
}
override int length() {
result = this.strValue().length()
}
override int length() { result = this.strValue().length() }
override ObjectInternal getIterNext() {
result = TUnknownInstance(this.getClass())
}
override ImmutableLiteral getLiteral() {
result.(Unicode).getText() = this.strValue()
}
override ObjectInternal getIterNext() { result = TUnknownInstance(this.getClass()) }
override ImmutableLiteral getLiteral() { result.(Unicode).getText() = this.strValue() }
}
class BytesObjectInternal extends ConstantObjectInternal, TBytes {
override string toString() {
result = "'" + this.strValue() + "'"
}
override string toString() { result = "'" + this.strValue() + "'" }
override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
context.appliesTo(node) and
@@ -364,22 +276,16 @@ class BytesObjectInternal extends ConstantObjectInternal, TBytes {
not node.getNode().(StrConst).isUnicode()
}
override ObjectInternal getClass() {
result = TBuiltinClassObject(Builtin::special("bytes"))
}
override ObjectInternal getClass() { result = TBuiltinClassObject(Builtin::special("bytes")) }
override Builtin getBuiltin() {
result.(Builtin).strValue() = this.strValue() and
result.getClass() = Builtin::special("bytes")
}
override int intValue() {
none()
}
override int intValue() { none() }
override string strValue() {
this = TBytes(result)
}
override string strValue() { this = TBytes(result) }
override boolean booleanValue() {
this.strValue() = "" and result = false
@@ -387,20 +293,9 @@ class BytesObjectInternal extends ConstantObjectInternal, TBytes {
this.strValue() != "" and result = true
}
override int length() {
result = this.strValue().length()
}
override int length() { result = this.strValue().length() }
override ObjectInternal getIterNext() {
result = TUnknownInstance(this.getClass())
}
override ImmutableLiteral getLiteral() {
result.(Bytes).getText() = this.strValue()
}
override ObjectInternal getIterNext() { result = TUnknownInstance(this.getClass()) }
override ImmutableLiteral getLiteral() { result.(Bytes).getText() = this.strValue() }
}

151
python/ql/src/semmle/python/objects/Descriptors.qll
View File

@@ -1,5 +1,4 @@
import python
private import semmle.python.objects.TObject
private import semmle.python.objects.ObjectInternal
private import semmle.python.pointsto.PointsTo
@@ -9,16 +8,11 @@ private import semmle.python.types.Builtins
/** Class representing property objects in Python */
class PropertyInternal extends ObjectInternal, TProperty {
/** Gets the name of this property */
override string getName() {
result = this.getGetter().getName()
}
override string getName() { result = this.getGetter().getName() }
/** Gets the getter function of this property */
CallableObjectInternal getGetter() {
this = TProperty(_, _, result)
}
CallableObjectInternal getGetter() { this = TProperty(_, _, result) }
private CallNode getCallNode() { this = TProperty(result, _, _) }
@@ -55,11 +49,9 @@ class PropertyInternal extends ObjectInternal, TProperty {
)
}
private Context getContext() { this = TProperty(_,result, _) }
private Context getContext() { this = TProperty(_, result, _) }
override string toString() {
result = "property " + this.getName()
}
override string toString() { result = "property " + this.getName() }
override boolean booleanValue() { result = true }
@@ -79,7 +71,9 @@ class PropertyInternal extends ObjectInternal, TProperty {
override ControlFlowNode getOrigin() { this = TProperty(result, _, _) }
override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() }
override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) {
none()
}
override predicate callResult(ObjectInternal obj, CfgOrigin origin) { none() }
@@ -89,7 +83,8 @@ class PropertyInternal extends ObjectInternal, TProperty {
override predicate calleeAndOffset(Function scope, int paramOffset) { none() }
pragma [noinline] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) {
pragma[noinline]
override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) {
value = TPropertySetterOrDeleter(this, name) and origin = CfgOrigin::unknown()
}
@@ -103,20 +98,28 @@ class PropertyInternal extends ObjectInternal, TProperty {
override predicate binds(ObjectInternal cls, string name, ObjectInternal descriptor) { none() }
pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) {
pragma[noinline]
override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) {
any(ObjectInternal obj).binds(cls, _, this) and
value = this and origin = CfgOrigin::fromCfgNode(this.getOrigin())
value = this and
origin = CfgOrigin::fromCfgNode(this.getOrigin())
}
pragma [noinline] override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) {
/* Just give an unknown value for now. We could improve this, but it would mean
pragma[noinline]
override predicate descriptorGetInstance(
ObjectInternal instance, ObjectInternal value, CfgOrigin origin
) {
/*
* Just give an unknown value for now. We could improve this, but it would mean
* changing Contexts to account for property accesses.
*/
exists(ClassObjectInternal cls, string name |
name = this.getName() and
receiver_type(_, name, instance, cls) and
cls.lookup(name, this, _) and
origin = CfgOrigin::unknown() and value = ObjectInternal::unknown()
origin = CfgOrigin::unknown() and
value = ObjectInternal::unknown()
)
}
@@ -128,40 +131,31 @@ class PropertyInternal extends ObjectInternal, TProperty {
override ObjectInternal getIterNext() { none() }
override predicate isNotSubscriptedType() { any() }
}
private class PropertySetterOrDeleter extends ObjectInternal, TPropertySetterOrDeleter {
override string toString() { result = this.getProperty().toString() + "." + this.getName() }
override string toString() {
result = this.getProperty().toString() + "." + this.getName()
}
override string getName() { this = TPropertySetterOrDeleter(_, result) }
override string getName() {
this = TPropertySetterOrDeleter(_, result)
}
PropertyInternal getProperty() {
this = TPropertySetterOrDeleter(result, _)
}
PropertyInternal getProperty() { this = TPropertySetterOrDeleter(result, _) }
override predicate callResult(ObjectInternal obj, CfgOrigin origin) {
exists(ControlFlowNode call |
obj = this.getProperty() and obj = TProperty(call, _, _) and
obj = this.getProperty() and
obj = TProperty(call, _, _) and
origin = CfgOrigin::fromCfgNode(call)
)
}
override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
none()
}
override predicate introducedAt(ControlFlowNode node, PointsToContext context) { none() }
override ClassDecl getClassDeclaration() { none() }
override boolean isClass() { result = false }
override ObjectInternal getClass() {
result = TBuiltinClassObject(Builtin::special("MethodType"))
result = TBuiltinClassObject(Builtin::special("MethodType"))
}
override predicate notTestableForEquality() { none() }
@@ -170,7 +164,9 @@ private class PropertySetterOrDeleter extends ObjectInternal, TPropertySetterOrD
override ControlFlowNode getOrigin() { none() }
override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() }
override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) {
none()
}
override int intValue() { none() }
@@ -196,23 +192,24 @@ private class PropertySetterOrDeleter extends ObjectInternal, TPropertySetterOrD
override ObjectInternal getIterNext() { none() }
override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() }
override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) {
none()
}
override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() }
override predicate descriptorGetInstance(
ObjectInternal instance, ObjectInternal value, CfgOrigin origin
) {
none()
}
override predicate useOriginAsLegacyObject() { none() }
override predicate isNotSubscriptedType() { any() }
}
/** A class representing classmethods in Python */
class ClassMethodObjectInternal extends ObjectInternal, TClassMethod {
override string toString() {
result = "classmethod(" + this.getFunction() + ")"
}
override string toString() { result = "classmethod(" + this.getFunction() + ")" }
override boolean booleanValue() { result = true }
@@ -224,9 +221,7 @@ class ClassMethodObjectInternal extends ObjectInternal, TClassMethod {
}
/** Gets the function wrapped by this classmethod object */
CallableObjectInternal getFunction() {
this = TClassMethod(_, result)
}
CallableObjectInternal getFunction() { this = TClassMethod(_, result) }
override ClassDecl getClassDeclaration() { none() }
@@ -240,7 +235,9 @@ class ClassMethodObjectInternal extends ObjectInternal, TClassMethod {
override ControlFlowNode getOrigin() { this = TClassMethod(result, _) }
override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() }
override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) {
none()
}
override predicate callResult(ObjectInternal obj, CfgOrigin origin) { none() }
@@ -258,26 +255,31 @@ class ClassMethodObjectInternal extends ObjectInternal, TClassMethod {
override boolean isDescriptor() { result = true }
pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) {
pragma[noinline]
override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) {
value = TBoundMethod(cls, this.getFunction()) and
origin = CfgOrigin::unknown()
}
pragma [noinline] override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) {
pragma[noinline]
override predicate descriptorGetInstance(
ObjectInternal instance, ObjectInternal value, CfgOrigin origin
) {
any(ObjectInternal obj).binds(instance, _, this) and
value = TBoundMethod(instance.getClass(), this.getFunction()) and
origin = CfgOrigin::unknown()
}
/** Holds if attribute lookup on this object may "bind" `cls` to `descriptor`.
/**
* Holds if attribute lookup on this object may "bind" `cls` to `descriptor`.
* `cls` will always be a class as this is a classmethod.
* Here "bind" means that `instance` is passed to the `classmethod.__get__()` method
* at runtime. The term "bind" is used as this most likely results in a bound-method.
*/
pragma [noinline] override predicate binds(ObjectInternal cls, string name, ObjectInternal descriptor) {
pragma[noinline]
override predicate binds(ObjectInternal cls, string name, ObjectInternal descriptor) {
descriptor = this.getFunction() and
exists(ObjectInternal instance |
any(ObjectInternal obj).binds(instance, name, this) |
exists(ObjectInternal instance | any(ObjectInternal obj).binds(instance, name, this) |
instance.isClass() = false and cls = instance.getClass()
or
instance.isClass() = true and cls = instance
@@ -286,9 +288,7 @@ class ClassMethodObjectInternal extends ObjectInternal, TClassMethod {
override int length() { none() }
override string getName() {
result = this.getFunction().getName()
}
override string getName() { result = this.getFunction().getName() }
override predicate contextSensitiveCallee() { none() }
@@ -298,14 +298,10 @@ class ClassMethodObjectInternal extends ObjectInternal, TClassMethod {
override ObjectInternal getIterNext() { none() }
override predicate isNotSubscriptedType() { any() }
}
class StaticMethodObjectInternal extends ObjectInternal, TStaticMethod {
override string toString() {
result = "staticmethod()"
}
override string toString() { result = "staticmethod()" }
override boolean booleanValue() { result = true }
@@ -316,9 +312,7 @@ class StaticMethodObjectInternal extends ObjectInternal, TStaticMethod {
)
}
CallableObjectInternal getFunction() {
this = TStaticMethod(_, result)
}
CallableObjectInternal getFunction() { this = TStaticMethod(_, result) }
override ClassDecl getClassDeclaration() { none() }
@@ -332,7 +326,9 @@ class StaticMethodObjectInternal extends ObjectInternal, TStaticMethod {
override ControlFlowNode getOrigin() { this = TStaticMethod(result, _) }
override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() }
override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) {
none()
}
override predicate callResult(ObjectInternal obj, CfgOrigin origin) { none() }
@@ -352,23 +348,29 @@ class StaticMethodObjectInternal extends ObjectInternal, TStaticMethod {
override boolean isDescriptor() { result = true }
pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) {
pragma[noinline]
override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) {
any(ObjectInternal obj).binds(cls, _, this) and
value = this.getFunction() and origin = CfgOrigin::unknown()
value = this.getFunction() and
origin = CfgOrigin::unknown()
}
pragma [noinline] override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) {
pragma[noinline]
override predicate descriptorGetInstance(
ObjectInternal instance, ObjectInternal value, CfgOrigin origin
) {
any(ObjectInternal obj).binds(instance, _, this) and
value = this.getFunction() and origin = CfgOrigin::unknown()
value = this.getFunction() and
origin = CfgOrigin::unknown()
}
override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() }
override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) {
none()
}
override int length() { none() }
override string getName() {
result = this.getFunction().getName()
}
override string getName() { result = this.getFunction().getName() }
override predicate contextSensitiveCallee() { none() }
@@ -378,5 +380,4 @@ class StaticMethodObjectInternal extends ObjectInternal, TStaticMethod {
override ObjectInternal getIterNext() { none() }
override predicate isNotSubscriptedType() { any() }
}

297
python/ql/src/semmle/python/objects/Instances.qll
View File

@@ -1,6 +1,4 @@
import python
private import semmle.python.objects.TObject
private import semmle.python.objects.ObjectInternal
private import semmle.python.pointsto.PointsTo
@@ -10,18 +8,19 @@ private import semmle.python.types.Builtins
/** A class representing instances */
abstract class InstanceObject extends ObjectInternal {
pragma [nomagic]
pragma[nomagic]
override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) {
exists(ObjectInternal cls_attr |
this.classAttribute(name, cls_attr)
|
/* If class attribute is not a descriptor, that usually means it is some sort of
exists(ObjectInternal cls_attr | this.classAttribute(name, cls_attr) |
/*
* If class attribute is not a descriptor, that usually means it is some sort of
* default value and likely overridden by an instance attribute. In that case
* use `unknown` to signal that an attribute exists but to avoid false positives
* f using the default value.
*/
cls_attr.isDescriptor() = false and value = ObjectInternal::unknown() and origin = CfgOrigin::unknown()
cls_attr.isDescriptor() = false and
value = ObjectInternal::unknown() and
origin = CfgOrigin::unknown()
or
cls_attr.isDescriptor() = true and cls_attr.descriptorGetInstance(this, value, origin)
)
@@ -29,13 +28,13 @@ abstract class InstanceObject extends ObjectInternal {
this.selfAttribute(name, value, origin)
}
pragma [noinline]
pragma[noinline]
private predicate classAttribute(string name, ObjectInternal cls_attr) {
PointsToInternal::attributeRequired(this, name) and
this.getClass().(ClassObjectInternal).lookup(name, cls_attr, _)
}
pragma [noinline]
pragma[noinline]
private predicate selfAttribute(string name, ObjectInternal value, CfgOrigin origin) {
PointsToInternal::attributeRequired(this, name) and
exists(EssaVariable self, PythonFunctionObjectInternal init, Context callee |
@@ -56,7 +55,6 @@ abstract class InstanceObject extends ObjectInternal {
override ObjectInternal getIterNext() { result = ObjectInternal::unknown() }
override predicate isNotSubscriptedType() { any() }
}
private predicate self_variable_reaching_init_exit(EssaVariable self) {
@@ -65,14 +63,12 @@ private predicate self_variable_reaching_init_exit(EssaVariable self) {
self.getScope().getName() = "__init__"
}
/** A class representing instances instantiated at a specific point in the program (statically)
/**
* A class representing instances instantiated at a specific point in the program (statically)
* For example the code `C()` would be a specific instance of `C`.
*/
class SpecificInstanceInternal extends TSpecificInstance, InstanceObject {
override string toString() {
result = this.getOrigin().getNode().toString()
}
override string toString() { result = this.getOrigin().getNode().toString() }
override boolean booleanValue() {
//result = this.getClass().instancesBooleanValue()
@@ -84,9 +80,7 @@ class SpecificInstanceInternal extends TSpecificInstance, InstanceObject {
}
/** Gets the class declaration for this object, if it is a declared class. */
override ClassDecl getClassDeclaration() {
none()
}
override ClassDecl getClassDeclaration() { none() }
override boolean isClass() { result = false }
@@ -95,64 +89,65 @@ class SpecificInstanceInternal extends TSpecificInstance, InstanceObject {
override ObjectInternal getClass() {
exists(ClassObjectInternal cls, ClassDecl decl |
this = TSpecificInstance(_, cls, _) and
decl = cls.getClassDeclaration() |
if decl.callReturnsInstance() then
result = cls
else
result = TUnknownClass()
decl = cls.getClassDeclaration()
|
if decl.callReturnsInstance() then result = cls else result = TUnknownClass()
)
}
/** Gets the `Builtin` for this object, if any.
* All objects (except unknown and undefined values) should return
/**
* Gets the `Builtin` for this object, if any.
* All objects (except unknown and undefined values) should return
* exactly one result for either this method or `getOrigin()`.
*/
override Builtin getBuiltin() {
none()
}
override Builtin getBuiltin() { none() }
/** Gets a control flow node that represents the source origin of this
/**
* Gets a control flow node that represents the source origin of this
* objects.
* All objects (except unknown and undefined values) should return
* All objects (except unknown and undefined values) should return
* exactly one result for either this method or `getBuiltin()`.
*/
override ControlFlowNode getOrigin() {
this = TSpecificInstance(result, _, _)
}
override ControlFlowNode getOrigin() { this = TSpecificInstance(result, _, _) }
override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) {
none()
}
override predicate callResult(ObjectInternal obj, CfgOrigin origin) {
override predicate callResult(ObjectInternal obj, CfgOrigin origin) {
// TO DO -- Handle cases where class overrides __call__ in more detail, like normal calls.
this.getClass().(ClassObjectInternal).lookup("__call__", _, _) and
obj = ObjectInternal::unknown() and origin = CfgOrigin::unknown()
obj = ObjectInternal::unknown() and
origin = CfgOrigin::unknown()
}
override int intValue() {
none()
}
override int intValue() { none() }
override string strValue() {
none()
}
override string strValue() { none() }
override predicate calleeAndOffset(Function scope, int paramOffset) {
none()
}
override predicate calleeAndOffset(Function scope, int paramOffset) { none() }
pragma [noinline] override predicate attributesUnknown() { any() }
pragma[noinline]
override predicate attributesUnknown() { any() }
override predicate subscriptUnknown() { any() }
override boolean isDescriptor() { result = false }
pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() }
pragma[noinline]
override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) {
none()
}
pragma [noinline] override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() }
pragma[noinline]
override predicate descriptorGetInstance(
ObjectInternal instance, ObjectInternal value, CfgOrigin origin
) {
none()
}
pragma [noinline] override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) {
pragma[noinline]
override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) {
exists(ClassObjectInternal cls |
receiver_type(_, name, this, cls) and
cls.lookup(name, descriptor, _) and
@@ -161,9 +156,7 @@ class SpecificInstanceInternal extends TSpecificInstance, InstanceObject {
this = instance
}
override int length() {
result = lengthFromClass(this.getClass())
}
override int length() { result = lengthFromClass(this.getClass()) }
override predicate initializer(PythonFunctionObjectInternal init, Context callee) {
exists(CallNode call, Context caller, ClassObjectInternal cls |
@@ -174,13 +167,12 @@ class SpecificInstanceInternal extends TSpecificInstance, InstanceObject {
}
override predicate useOriginAsLegacyObject() { none() }
}
/** A class representing context-free instances represented by `self` in the source code
/**
* A class representing context-free instances represented by `self` in the source code
*/
class SelfInstanceInternal extends TSelfInstance, InstanceObject {
override string toString() {
result = "self instance of " + this.getClass().(ClassObjectInternal).getName()
}
@@ -191,34 +183,24 @@ class SelfInstanceInternal extends TSelfInstance, InstanceObject {
result = maybe()
}
override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
none()
}
override predicate introducedAt(ControlFlowNode node, PointsToContext context) { none() }
predicate parameterAndContext(ParameterDefinition def, PointsToContext context) {
this = TSelfInstance(def, context, _)
}
/** Gets the class declaration for this object, if it is a declared class. */
override ClassDecl getClassDeclaration() {
none()
}
override ClassDecl getClassDeclaration() { none() }
override boolean isClass() { result = false }
override predicate notTestableForEquality() { any() }
override ObjectInternal getClass() {
this = TSelfInstance(_, _, result)
}
override ObjectInternal getClass() { this = TSelfInstance(_, _, result) }
ParameterDefinition getParameter() {
this = TSelfInstance(result, _, _)
}
ParameterDefinition getParameter() { this = TSelfInstance(result, _, _) }
override Builtin getBuiltin() {
none()
}
override Builtin getBuiltin() { none() }
override ControlFlowNode getOrigin() {
exists(ParameterDefinition def |
@@ -231,35 +213,39 @@ class SelfInstanceInternal extends TSelfInstance, InstanceObject {
none()
}
override predicate callResult(ObjectInternal obj, CfgOrigin origin) {
override predicate callResult(ObjectInternal obj, CfgOrigin origin) {
// In general instances aren't callable, but some are...
// TO DO -- Handle cases where class overrides __call__
none()
}
override int intValue() {
none()
}
override int intValue() { none() }
override string strValue() {
none()
}
override string strValue() { none() }
override predicate calleeAndOffset(Function scope, int paramOffset) {
none()
}
override predicate calleeAndOffset(Function scope, int paramOffset) { none() }
pragma [noinline] override predicate attributesUnknown() { any() }
pragma[noinline]
override predicate attributesUnknown() { any() }
override predicate subscriptUnknown() { any() }
override boolean isDescriptor() { result = false }
pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() }
pragma[noinline]
override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) {
none()
}
pragma [noinline] override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() }
pragma[noinline]
override predicate descriptorGetInstance(
ObjectInternal instance, ObjectInternal value, CfgOrigin origin
) {
none()
}
pragma [noinline] override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) {
pragma[noinline]
override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) {
exists(AttrNode attr, ClassObjectInternal cls |
receiver_type(attr, name, this, cls) and
cls_descriptor(cls, name, descriptor)
@@ -267,9 +253,7 @@ class SelfInstanceInternal extends TSelfInstance, InstanceObject {
instance = this
}
override int length() {
result = lengthFromClass(this.getClass())
}
override int length() { result = lengthFromClass(this.getClass()) }
override predicate initializer(PythonFunctionObjectInternal init, Context callee) {
callee.isRuntime() and
@@ -278,53 +262,41 @@ class SelfInstanceInternal extends TSelfInstance, InstanceObject {
}
override predicate useOriginAsLegacyObject() { none() }
}
/** A class representing a value that has a known class, but no other information */
class UnknownInstanceInternal extends TUnknownInstance, ObjectInternal {
override string toString() {
result = "instance of " + this.getClass().(ClassObjectInternal).getName()
}
override boolean booleanValue() {
result = maybe()
}
override boolean booleanValue() { result = maybe() }
override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
none()
}
override predicate introducedAt(ControlFlowNode node, PointsToContext context) { none() }
/** Gets the class declaration for this object, if it is a declared class. */
override ClassDecl getClassDeclaration() {
none()
}
override ClassDecl getClassDeclaration() { none() }
override boolean isClass() { result = false }
override predicate notTestableForEquality() { any() }
override ObjectInternal getClass() {
this = TUnknownInstance(result)
}
override ObjectInternal getClass() { this = TUnknownInstance(result) }
/** Gets the `Builtin` for this object, if any.
* All objects (except unknown and undefined values) should return
/**
* Gets the `Builtin` for this object, if any.
* All objects (except unknown and undefined values) should return
* exactly one result for either this method or `getOrigin()`.
*/
override Builtin getBuiltin() {
none()
}
override Builtin getBuiltin() { none() }
/** Gets a control flow node that represents the source origin of this
/**
* Gets a control flow node that represents the source origin of this
* objects.
* All objects (except unknown and undefined values) should return
* All objects (except unknown and undefined values) should return
* exactly one result for either this method or `getBuiltin()`.
*/
override ControlFlowNode getOrigin() {
none()
}
override ControlFlowNode getOrigin() { none() }
override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) {
none()
@@ -336,40 +308,45 @@ class UnknownInstanceInternal extends TUnknownInstance, ObjectInternal {
none()
}
override int intValue() {
none()
}
override int intValue() { none() }
override string strValue() {
none()
}
override string strValue() { none() }
override predicate calleeAndOffset(Function scope, int paramOffset) {
none()
}
override predicate calleeAndOffset(Function scope, int paramOffset) { none() }
pragma [noinline] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) {
pragma[noinline]
override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) {
PointsToInternal::attributeRequired(this, name) and
exists(ObjectInternal cls_attr, CfgOrigin attr_orig |
this.getClass().(ClassObjectInternal).lookup(name, cls_attr, attr_orig)
|
|
cls_attr.isDescriptor() = false and value = cls_attr and origin = attr_orig
or
cls_attr.isDescriptor() = true and cls_attr.descriptorGetInstance(this, value, origin)
)
}
pragma [noinline] override predicate attributesUnknown() { any() }
pragma[noinline]
override predicate attributesUnknown() { any() }
override predicate subscriptUnknown() { any() }
override boolean isDescriptor() { result = false }
pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() }
pragma[noinline]
override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) {
none()
}
pragma [noinline] override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() }
pragma[noinline]
override predicate descriptorGetInstance(
ObjectInternal instance, ObjectInternal value, CfgOrigin origin
) {
none()
}
pragma [noinline] override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) {
pragma[noinline]
override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) {
exists(AttrNode attr, ClassObjectInternal cls |
receiver_type(attr, name, this, cls) and
cls_descriptor(cls, name, descriptor)
@@ -377,9 +354,7 @@ class UnknownInstanceInternal extends TUnknownInstance, ObjectInternal {
instance = this
}
override int length() {
result = lengthFromClass(this.getClass())
}
override int length() { result = lengthFromClass(this.getClass()) }
override string getName() { none() }
@@ -390,7 +365,6 @@ class UnknownInstanceInternal extends TUnknownInstance, ObjectInternal {
override ObjectInternal getIterNext() { result = ObjectInternal::unknown() }
override predicate isNotSubscriptedType() { any() }
}
private int lengthFromClass(ClassObjectInternal cls) {
@@ -404,7 +378,6 @@ private predicate cls_descriptor(ClassObjectInternal cls, string name, ObjectInt
/** A class representing an instance of the `super` class */
class SuperInstance extends TSuperInstance, ObjectInternal {
override string toString() {
result = "super(" + this.getStartClass().toString() + ", " + this.getSelf().toString() + ")"
}
@@ -419,34 +392,28 @@ class SuperInstance extends TSuperInstance, ObjectInternal {
}
/** Gets the class declared as the starting point for MRO lookup. */
ClassObjectInternal getStartClass() {
this = TSuperInstance(_, result)
}
ClassObjectInternal getStartClass() { this = TSuperInstance(_, result) }
/** Gets 'self' object */
ObjectInternal getSelf() {
this = TSuperInstance(result, _)
}
ObjectInternal getSelf() { this = TSuperInstance(result, _) }
override ClassDecl getClassDeclaration() { none() }
override boolean isClass() { result = false }
override ObjectInternal getClass() {
result = ObjectInternal::superType()
}
override ObjectInternal getClass() { result = ObjectInternal::superType() }
override predicate notTestableForEquality() { any() }
override Builtin getBuiltin() { none() }
override ControlFlowNode getOrigin() {
none()
}
override ControlFlowNode getOrigin() { none() }
override predicate callResult(ObjectInternal obj, CfgOrigin origin) { none() }
override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() }
override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) {
none()
}
override int intValue() { none() }
@@ -454,46 +421,60 @@ class SuperInstance extends TSuperInstance, ObjectInternal {
override predicate calleeAndOffset(Function scope, int paramOffset) { none() }
pragma [noinline] override predicate attributesUnknown() { none() }
pragma[noinline]
override predicate attributesUnknown() { none() }
override predicate subscriptUnknown() { any() }
override boolean isDescriptor() { result = false }
pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() }
pragma[noinline]
override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) {
none()
}
pragma [noinline] override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() }
pragma[noinline]
override predicate descriptorGetInstance(
ObjectInternal instance, ObjectInternal value, CfgOrigin origin
) {
none()
}
pragma [noinline] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) {
pragma[noinline]
override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) {
exists(ObjectInternal cls_attr, CfgOrigin attr_orig |
this.attribute_descriptor(name, cls_attr, attr_orig)
|
|
cls_attr.isDescriptor() = false and value = cls_attr and origin = attr_orig
or
cls_attr.isDescriptor() = true and cls_attr.descriptorGetInstance(this.getSelf(), value, origin)
cls_attr.isDescriptor() = true and
cls_attr.descriptorGetInstance(this.getSelf(), value, origin)
)
}
/* Helper for `attribute` */
pragma [noinline] private predicate attribute_descriptor(string name, ObjectInternal cls_attr, CfgOrigin attr_orig) {
pragma[noinline]
private predicate attribute_descriptor(string name, ObjectInternal cls_attr, CfgOrigin attr_orig) {
PointsToInternal::attributeRequired(this, name) and
this.lookup(name, cls_attr, attr_orig)
}
private predicate lookup(string name, ObjectInternal value, CfgOrigin origin) {
Types::getMro(this.getSelf().getClass()).startingAt(this.getStartClass()).getTail().lookup(name, value, origin)
Types::getMro(this.getSelf().getClass())
.startingAt(this.getStartClass())
.getTail()
.lookup(name, value, origin)
}
pragma [noinline] override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) {
pragma[noinline]
override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) {
descriptor.isDescriptor() = true and
this.lookup(name, descriptor, _) and
instance = this.getSelf() and
receiver_type(_, name, this, _)
}
override int length() {
none()
}
override int length() { none() }
override string getName() { none() }
@@ -504,6 +485,4 @@ class SuperInstance extends TSuperInstance, ObjectInternal {
override ObjectInternal getIterNext() { result = ObjectInternal::unknown() }
override predicate isNotSubscriptedType() { any() }
}

319
python/ql/src/semmle/python/objects/Modules.qll
View File

@@ -1,5 +1,4 @@
import python
private import semmle.python.objects.TObject
private import semmle.python.objects.ObjectInternal
private import semmle.python.pointsto.PointsTo
@@ -9,7 +8,6 @@ private import semmle.python.types.Builtins
/** A class representing modules */
abstract class ModuleObjectInternal extends ObjectInternal {
/** Gets the source scope of this module, if it has one. */
abstract Module getSourceModule();
@@ -27,30 +25,35 @@ abstract class ModuleObjectInternal extends ObjectInternal {
override predicate notTestableForEquality() { none() }
override boolean booleanValue() {
result = true
}
override boolean booleanValue() { result = true }
override ObjectInternal getClass() {
result = ObjectInternal::moduleType()
}
override ObjectInternal getClass() { result = ObjectInternal::moduleType() }
override boolean isDescriptor() { result = false }
pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() }
pragma[noinline]
override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) {
none()
}
pragma [noinline] override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() }
pragma[noinline]
override predicate descriptorGetInstance(
ObjectInternal instance, ObjectInternal value, CfgOrigin origin
) {
none()
}
pragma [noinline] override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() }
pragma[noinline]
override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) {
none()
}
override int length() { none() }
override predicate subscriptUnknown() { any() }
/** Holds if this module is a `__init__.py` module. */
predicate isInitModule() {
any(PackageObjectInternal package).getInitModule() = this
}
predicate isInitModule() { any(PackageObjectInternal package).getInitModule() = this }
override predicate contextSensitiveCallee() { none() }
@@ -59,14 +62,16 @@ abstract class ModuleObjectInternal extends ObjectInternal {
/* Modules aren't iterable */
override ObjectInternal getIterNext() { none() }
/** Holds if this module "exports" name.
/**
* Holds if this module "exports" name.
* That is, does it define `name` in `__all__` or is
* `__all__` not defined and `name` a global variable that does not start with "_"
* This is the set of names imported by `from ... import *`.
*/
predicate exports(string name) {
not this.(ModuleObjectInternal).attribute("__all__", _, _) and this.hasAttribute(name)
and not name.charAt(0) = "_"
not this.(ModuleObjectInternal).attribute("__all__", _, _) and
this.hasAttribute(name) and
not name.charAt(0) = "_"
or
py_exports(this.getSourceModule(), name)
}
@@ -75,101 +80,61 @@ abstract class ModuleObjectInternal extends ObjectInternal {
abstract predicate hasCompleteExportInfo();
override predicate isNotSubscriptedType() { any() }
}
/** A class representing built-in modules */
class BuiltinModuleObjectInternal extends ModuleObjectInternal, TBuiltinModuleObject {
override Builtin getBuiltin() { this = TBuiltinModuleObject(result) }
override Builtin getBuiltin() {
this = TBuiltinModuleObject(result)
}
override string toString() { result = "Module " + this.getBuiltin().getName() }
override string toString() {
result = "Module " + this.getBuiltin().getName()
}
override string getName() { result = this.getBuiltin().getName() }
override string getName() {
result = this.getBuiltin().getName()
}
override predicate introducedAt(ControlFlowNode node, PointsToContext context) { none() }
override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
none()
}
override ClassDecl getClassDeclaration() { none() }
override ClassDecl getClassDeclaration() {
none()
}
override Module getSourceModule() { none() }
override Module getSourceModule() {
none()
}
override int intValue() { none() }
override int intValue() {
none()
}
override string strValue() { none() }
override string strValue() {
none()
}
override predicate calleeAndOffset(Function scope, int paramOffset) { none() }
override predicate calleeAndOffset(Function scope, int paramOffset) {
none()
}
pragma [noinline] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) {
pragma[noinline]
override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) {
value = ObjectInternal::fromBuiltin(this.getBuiltin().getMember(name)) and
origin = CfgOrigin::unknown()
}
pragma [noinline] override predicate attributesUnknown() { none() }
pragma[noinline]
override predicate attributesUnknown() { none() }
override ControlFlowNode getOrigin() {
none()
}
override predicate hasCompleteExportInfo() {
any()
}
override ControlFlowNode getOrigin() { none() }
override predicate hasCompleteExportInfo() { any() }
}
/** A class representing packages */
class PackageObjectInternal extends ModuleObjectInternal, TPackageObject {
override Builtin getBuiltin() { none() }
override Builtin getBuiltin() {
none()
}
override string toString() {
result = "Package " + this.getName()
}
override string toString() { result = "Package " + this.getName() }
/** Gets the folder for this package */
Folder getFolder() {
this = TPackageObject(result)
}
Folder getFolder() { this = TPackageObject(result) }
override string getName() {
result = moduleNameFromFile(this.getFolder())
}
override string getName() { result = moduleNameFromFile(this.getFolder()) }
override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
none()
}
override predicate introducedAt(ControlFlowNode node, PointsToContext context) { none() }
override ClassDecl getClassDeclaration() {
none()
}
override ClassDecl getClassDeclaration() { none() }
override Module getSourceModule() {
result.getFile() = this.getFolder().getFile("__init__.py")
}
override Module getSourceModule() { result.getFile() = this.getFolder().getFile("__init__.py") }
/** Gets the init module of this package */
PythonModuleObjectInternal getInitModule() {
result = TPythonModule(this.getSourceModule())
}
PythonModuleObjectInternal getInitModule() { result = TPythonModule(this.getSourceModule()) }
predicate hasNoInitModule() {
exists(Folder f |
@@ -188,19 +153,14 @@ class PackageObjectInternal extends ModuleObjectInternal, TPackageObject {
)
}
override int intValue() {
none()
}
override int intValue() { none() }
override string strValue() {
none()
}
override string strValue() { none() }
override predicate calleeAndOffset(Function scope, int paramOffset) {
none()
}
override predicate calleeAndOffset(Function scope, int paramOffset) { none() }
pragma [noinline] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) {
pragma[noinline]
override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) {
this.getInitModule().attribute(name, value, origin)
or
exists(Module init |
@@ -215,12 +175,14 @@ class PackageObjectInternal extends ModuleObjectInternal, TPackageObject {
this.hasNoInitModule() and
exists(ModuleObjectInternal mod |
mod = this.submodule(name) and
value = mod |
value = mod
|
origin = CfgOrigin::fromObject(mod)
)
}
pragma [noinline] override predicate attributesUnknown() { none() }
pragma[noinline]
override predicate attributesUnknown() { none() }
override ControlFlowNode getOrigin() {
exists(Module package |
@@ -238,7 +200,6 @@ class PackageObjectInternal extends ModuleObjectInternal, TPackageObject {
}
override predicate hasCompleteExportInfo() {
not exists(this.getInitModule())
or
this.getInitModule().hasCompleteExportInfo()
@@ -247,53 +208,34 @@ class PackageObjectInternal extends ModuleObjectInternal, TPackageObject {
/** A class representing Python modules */
class PythonModuleObjectInternal extends ModuleObjectInternal, TPythonModule {
override Builtin getBuiltin() { none() }
override Builtin getBuiltin() {
none()
}
override string toString() { result = this.getSourceModule().toString() }
override string toString() {
result = this.getSourceModule().toString()
}
override string getName() { result = this.getSourceModule().getName() }
override string getName() {
result = this.getSourceModule().getName()
}
override predicate introducedAt(ControlFlowNode node, PointsToContext context) { none() }
override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
none()
}
override ClassDecl getClassDeclaration() { none() }
override ClassDecl getClassDeclaration() {
none()
}
override Module getSourceModule() { this = TPythonModule(result) }
override Module getSourceModule() {
this = TPythonModule(result)
}
override int intValue() { none() }
override int intValue() {
none()
}
override string strValue() { none() }
override string strValue() {
none()
}
override predicate calleeAndOffset(Function scope, int paramOffset) { none() }
override predicate calleeAndOffset(Function scope, int paramOffset) {
none()
}
pragma [noinline] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) {
pragma[noinline]
override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) {
value != ObjectInternal::undefined() and
ModuleAttributes::pointsToAtExit(this.getSourceModule(), name, value, origin)
}
pragma [noinline] override predicate attributesUnknown() { none() }
pragma[noinline]
override predicate attributesUnknown() { none() }
override ControlFlowNode getOrigin() {
result = this.getSourceModule().getEntryNode()
}
override ControlFlowNode getOrigin() { result = this.getSourceModule().getEntryNode() }
/** Holds if this value has the attribute `name` */
override predicate hasAttribute(string name) {
@@ -317,76 +259,56 @@ class PythonModuleObjectInternal extends ModuleObjectInternal, TPythonModule {
not exists(Call modify, Attribute attr, GlobalVariable all |
modify.getScope() = this.getSourceModule() and
modify.getFunc() = attr and
all.getId() = "__all__" |
all.getId() = "__all__"
|
attr.getObject().(Name).uses(all)
)
}
}
/** A class representing a module that is missing from the DB, but inferred to exists from imports. */
class AbsentModuleObjectInternal extends ModuleObjectInternal, TAbsentModule {
override Builtin getBuiltin() {
none()
}
override Builtin getBuiltin() { none() }
override string toString() {
if exists(Module m, SyntaxError se | se.getFile() = m.getFile() and m.getName() = this.getName()) then
result = "Unparsable module " + this.getName()
else
result = "Missing module " + this.getName()
if
exists(Module m, SyntaxError se | se.getFile() = m.getFile() and m.getName() = this.getName())
then result = "Unparsable module " + this.getName()
else result = "Missing module " + this.getName()
}
override string getName() {
this = TAbsentModule(result)
}
override string getName() { this = TAbsentModule(result) }
override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
missing_imported_module(node, context, this.getName())
}
override ClassDecl getClassDeclaration() {
none()
}
override ClassDecl getClassDeclaration() { none() }
override Module getSourceModule() {
none()
}
override Module getSourceModule() { none() }
override int intValue() {
none()
}
override int intValue() { none() }
override string strValue() {
none()
}
override string strValue() { none() }
override predicate calleeAndOffset(Function scope, int paramOffset) {
none()
}
override predicate calleeAndOffset(Function scope, int paramOffset) { none() }
pragma [noinline] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) {
pragma[noinline]
override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) {
value = TAbsentModuleAttribute(this, name) and origin = CfgOrigin::unknown()
}
pragma [noinline] override predicate attributesUnknown() { none() }
pragma[noinline]
override predicate attributesUnknown() { none() }
override ControlFlowNode getOrigin() {
none()
}
override ControlFlowNode getOrigin() { none() }
override predicate hasCompleteExportInfo() {
none()
}
override predicate hasCompleteExportInfo() { none() }
}
/** A class representing an attribute of a missing module. */
class AbsentModuleAttributeObjectInternal extends ObjectInternal, TAbsentModuleAttribute {
override Builtin getBuiltin() {
none()
}
override Builtin getBuiltin() { none() }
override string toString() {
exists(ModuleObjectInternal mod, string name |
@@ -396,39 +318,28 @@ class AbsentModuleAttributeObjectInternal extends ObjectInternal, TAbsentModuleA
}
override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
exists(ModuleObjectInternal mod, string name |
this = TAbsentModuleAttribute(mod, name) |
exists(ModuleObjectInternal mod, string name | this = TAbsentModuleAttribute(mod, name) |
PointsToInternal::pointsTo(node.(AttrNode).getObject(name), context, mod, _)
or
PointsToInternal::pointsTo(node.(ImportMemberNode).getModule(name), context, mod, _)
)
}
override ClassDecl getClassDeclaration() {
none()
}
override ClassDecl getClassDeclaration() { none() }
override int intValue() {
none()
}
override int intValue() { none() }
override string strValue() {
none()
}
override string strValue() { none() }
override predicate calleeAndOffset(Function scope, int paramOffset) {
none()
}
override predicate calleeAndOffset(Function scope, int paramOffset) { none() }
pragma [noinline] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) {
none()
}
pragma[noinline]
override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { none() }
pragma [noinline] override predicate attributesUnknown() { any() }
pragma[noinline]
override predicate attributesUnknown() { any() }
override ControlFlowNode getOrigin() {
none()
}
override ControlFlowNode getOrigin() { none() }
override predicate callResult(ObjectInternal obj, CfgOrigin origin) {
obj = ObjectInternal::unknown() and origin = CfgOrigin::unknown()
@@ -442,31 +353,36 @@ class AbsentModuleAttributeObjectInternal extends ObjectInternal, TAbsentModuleA
override predicate notTestableForEquality() { any() }
override boolean booleanValue() {
result = maybe()
}
override boolean booleanValue() { result = maybe() }
override ObjectInternal getClass() {
result = ObjectInternal::unknownClass()
}
override ObjectInternal getClass() { result = ObjectInternal::unknownClass() }
override boolean isDescriptor() { result = false }
override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() }
override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) {
none()
}
override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() }
override predicate descriptorGetInstance(
ObjectInternal instance, ObjectInternal value, CfgOrigin origin
) {
none()
}
override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() }
override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) {
none()
}
override int length() { none() }
override predicate subscriptUnknown() { any() }
/* We know what this is called, but not its innate name.
* However, if we are looking for things by name, this is a reasonable approximation */
override string getName() {
this = TAbsentModuleAttribute(_, result)
}
/*
* We know what this is called, but not its innate name.
* However, if we are looking for things by name, this is a reasonable approximation
*/
override string getName() { this = TAbsentModuleAttribute(_, result) }
override predicate contextSensitiveCallee() { none() }
@@ -476,5 +392,4 @@ class AbsentModuleAttributeObjectInternal extends ObjectInternal, TAbsentModuleA
override ObjectInternal getIterNext() { none() }
override predicate isNotSubscriptedType() { any() }
}

682
python/ql/src/semmle/python/objects/ObjectAPI.qll
View File

File diff suppressed because it is too large Load Diff

436
python/ql/src/semmle/python/objects/ObjectInternal.qll
View File

@@ -16,16 +16,17 @@ import semmle.python.objects.Constants
import semmle.python.objects.Sequences
import semmle.python.objects.Descriptors
class ObjectInternal extends TObject {
abstract string toString();
/** The boolean value of this object, this may be both
* true and false if the "object" represents a set of possible objects. */
/**
* The boolean value of this object, this may be both
* true and false if the "object" represents a set of possible objects.
*/
abstract boolean booleanValue();
/** Holds if this object is introduced into the code base at `node` given the `context`
/**
* Holds if this object is introduced into the code base at `node` given the `context`
* This means that `node`, in `context`, points-to this object, but the object has not flowed
* there from anywhere else.
* Examples:
@@ -41,22 +42,25 @@ class ObjectInternal extends TObject {
/** True if this "object" is a class. That is, its class inherits from `type` */
abstract boolean isClass();
/** Gets the class of this object. */
/** Gets the class of this object. */
abstract ObjectInternal getClass();
/** True if this "object" can be meaningfully analysed to determine the boolean value of
/**
* True if this "object" can be meaningfully analysed to determine the boolean value of
* equality tests on it.
* For example, `None` or `int` can be, but `int()` or an unknown string cannot.
*/
abstract predicate notTestableForEquality();
/** Gets the `Builtin` for this object, if any.
/**
* Gets the `Builtin` for this object, if any.
* Objects (except unknown and undefined values) should attempt to return
* exactly one result for either this method or `getOrigin()`.
*/
abstract Builtin getBuiltin();
/** Gets a control flow node that represents the source origin of this
/**
* Gets a control flow node that represents the source origin of this
* object, if it has a meaningful location in the source code.
* This method exists primarily for providing backwards compatibility and
* locations for source objects.
@@ -64,7 +68,8 @@ class ObjectInternal extends TObject {
*/
abstract ControlFlowNode getOrigin();
/** Holds if `obj` is the result of calling `this` and `origin` is
/**
* Holds if `obj` is the result of calling `this` and `origin` is
* the origin of `obj`.
*
* This is the context-insensitive version.
@@ -72,7 +77,8 @@ class ObjectInternal extends TObject {
*/
abstract predicate callResult(ObjectInternal obj, CfgOrigin origin);
/** Holds if `obj` is the result of calling `this` and `origin` is
/**
* Holds if `obj` is the result of calling `this` and `origin` is
* the origin of `obj` with callee context `callee`.
*
* This is the context-sensitive version.
@@ -80,34 +86,36 @@ class ObjectInternal extends TObject {
*/
abstract predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin);
/** The integer value of things that have integer values and whose integer value is
/**
* The integer value of things that have integer values and whose integer value is
* tracked.
* That is, some ints, mainly small numbers, and bools.
*/
abstract int intValue();
/** The string value of things that have string values.
/**
* The string value of things that have string values.
* That is, strings.
*/
abstract string strValue();
/** Holds if the function `scope` is called when this object is called and `paramOffset`
/**
* Holds if the function `scope` is called when this object is called and `paramOffset`
* is the difference from the parameter position and the argument position.
* For a normal function `paramOffset` is 0. For classes and bound-methods it is 1.
* Used by points-to to help determine flow from arguments to parameters.
*/
abstract predicate calleeAndOffset(Function scope, int paramOffset);
final predicate isBuiltin() {
exists(this.getBuiltin())
}
final predicate isBuiltin() { exists(this.getBuiltin()) }
/** Holds if the result of getting the attribute `name` is `value` and that `value` comes
/**
* Holds if the result of getting the attribute `name` is `value` and that `value` comes
* from `origin`. Note this is *not* the same as class lookup. For example
* for an object `x` the attribute `name` (`x.name`) may refer to a bound-method, an attribute of the
* for an object `x` the attribute `name` (`x.name`) may refer to a bound-method, an attribute of the
* instance, or an attribute of the class.
*/
pragma [nomagic]
pragma[nomagic]
abstract predicate attribute(string name, ObjectInternal value, CfgOrigin origin);
/** Holds if the attributes of this object are wholly or partly unknowable */
@@ -116,59 +124,71 @@ class ObjectInternal extends TObject {
/** Holds if the result of subscripting this object are wholly or partly unknowable */
abstract predicate subscriptUnknown();
/** For backwards compatibility shim -- Not all objects have a "source".
/**
* For backwards compatibility shim -- Not all objects have a "source".
* Objects (except unknown and undefined values) should attempt to return
* exactly one result for this method.
* */
*/
@py_object getSource() {
result = this.getOrigin()
or
result = this.getBuiltin()
}
/** Holds if this object is a descriptor.
/**
* Holds if this object is a descriptor.
* Holds, for example, for functions and properties and not for integers.
*/
abstract boolean isDescriptor();
/** Holds if the result of attribute access on the class holding this descriptor is `value`, originating at `origin`
/**
* Holds if the result of attribute access on the class holding this descriptor is `value`, originating at `origin`
* For example, although `T.__dict__['name'] = classmethod(f)`, `T.name` is a bound-method, binding `f` and `T`
*/
pragma[nomagic]
abstract predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin);
/** Holds if the result of attribute access on an instance of a class holding this descriptor is `value`, originating at `origin`
/**
* Holds if the result of attribute access on an instance of a class holding this descriptor is `value`, originating at `origin`
* For example, with `T.__dict__['name'] = classmethod(f)`, `T().name` is a bound-method, binding `f` and `T`
*/
pragma[nomagic]
abstract predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin);
abstract predicate descriptorGetInstance(
ObjectInternal instance, ObjectInternal value, CfgOrigin origin
);
/** Holds if attribute lookup on this object may "bind" `instance` to `descriptor`.
/**
* Holds if attribute lookup on this object may "bind" `instance` to `descriptor`.
* Here "bind" means that `instance` is passed to the `descriptor.__get__()` method
* at runtime. The term "bind" is used as this most likely results in a bound-method.
*/
abstract predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor);
/** Gets the length of the sequence that this "object" represents.
/**
* Gets the length of the sequence that this "object" represents.
* Always returns a value for a sequence, will be -1 if the object has no fixed length.
*/
abstract int length();
/** Holds if the object `function` is called when this object is called and `paramOffset`
/**
* Holds if the object `function` is called when this object is called and `paramOffset`
* is the difference from the parameter position and the argument position.
* For a normal function `paramOffset` is 0. For classes and bound-methods it is 1.
* This is used to implement the `CallableValue` public API.
*/
predicate functionAndOffset(CallableObjectInternal function, int offset) { none() }
/** Holds if this 'object' represents an entity that should be exposed to the legacy points_to API
/**
* Holds if this 'object' represents an entity that should be exposed to the legacy points_to API
* This should hold for almost all objects that do not have an underlying DB object representing their source,
* for example `super` objects and bound-method. This should not hold for objects that are inferred to exists by
* an import statements or the like, but which aren't in the database. */
/* This predicate can be removed when the legacy points_to API is removed. */
* an import statements or the like, but which aren't in the database.
*/
/* This predicate can be removed when the legacy points_to API is removed. */
abstract predicate useOriginAsLegacyObject();
/** Gets the name of this of this object if it has a meaningful name.
/**
* Gets the name of this of this object if it has a meaningful name.
* Note that the name of an object is not necessarily the name by which it is called
* For example the function named `posixpath.join` will be called `os.path.join`.
*/
@@ -176,50 +196,36 @@ class ObjectInternal extends TObject {
abstract predicate contextSensitiveCallee();
/** Gets the 'object' resulting from iterating over this object.
/**
* Gets the 'object' resulting from iterating over this object.
* Used in the context `for i in this:`. The result is the 'object'
* assigned to `i`.
*/
abstract ObjectInternal getIterNext();
/** Holds if this value has the attribute `name` */
predicate hasAttribute(string name) {
this.(ObjectInternal).attribute(name, _, _)
}
predicate hasAttribute(string name) { this.(ObjectInternal).attribute(name, _, _) }
abstract predicate isNotSubscriptedType();
}
class BuiltinOpaqueObjectInternal extends ObjectInternal, TBuiltinOpaqueObject {
override Builtin getBuiltin() { this = TBuiltinOpaqueObject(result) }
override Builtin getBuiltin() {
this = TBuiltinOpaqueObject(result)
}
override string toString() {
result = this.getBuiltin().getClass().getName() + " object"
}
override string toString() { result = this.getBuiltin().getClass().getName() + " object" }
override boolean booleanValue() {
// TO DO ... Depends on class. `result = this.getClass().instancesBooleanValue()`
result = maybe()
}
override ClassDecl getClassDeclaration() {
none()
}
override ClassDecl getClassDeclaration() { none() }
override boolean isClass() { result = false }
override ObjectInternal getClass() {
result = TBuiltinClassObject(this.getBuiltin().getClass())
}
override ObjectInternal getClass() { result = TBuiltinClassObject(this.getBuiltin().getClass()) }
override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
none()
}
override predicate introducedAt(ControlFlowNode node, PointsToContext context) { none() }
override predicate notTestableForEquality() { any() }
@@ -231,46 +237,47 @@ class BuiltinOpaqueObjectInternal extends ObjectInternal, TBuiltinOpaqueObject {
obj = ObjectInternal::unknown() and origin = CfgOrigin::unknown()
}
override ControlFlowNode getOrigin() {
none()
}
override ControlFlowNode getOrigin() { none() }
override int intValue() {
none()
}
override int intValue() { none() }
override string strValue() {
none()
}
override string strValue() { none() }
override predicate calleeAndOffset(Function scope, int paramOffset) {
none()
}
override predicate calleeAndOffset(Function scope, int paramOffset) { none() }
pragma [noinline] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) {
pragma[noinline]
override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) {
value = ObjectInternal::fromBuiltin(this.getBuiltin().getMember(name)) and
origin = CfgOrigin::unknown()
}
pragma [noinline] override predicate attributesUnknown() { none() }
pragma[noinline]
override predicate attributesUnknown() { none() }
override predicate subscriptUnknown() {
exists(this.getBuiltin().getItem(_))
}
override predicate subscriptUnknown() { exists(this.getBuiltin().getItem(_)) }
override boolean isDescriptor() { result = false }
pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() }
pragma[noinline]
override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) {
none()
}
pragma [noinline] override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() }
pragma[noinline]
override predicate descriptorGetInstance(
ObjectInternal instance, ObjectInternal value, CfgOrigin origin
) {
none()
}
pragma [noinline] override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() }
pragma[noinline]
override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) {
none()
}
override int length() { none() }
override string getName() {
result = this.getBuiltin().getName()
}
override string getName() { result = this.getBuiltin().getName() }
override predicate contextSensitiveCallee() { none() }
@@ -279,39 +286,24 @@ class BuiltinOpaqueObjectInternal extends ObjectInternal, TBuiltinOpaqueObject {
override ObjectInternal getIterNext() { result = ObjectInternal::unknown() }
override predicate isNotSubscriptedType() { any() }
}
class UnknownInternal extends ObjectInternal, TUnknown {
override string toString() { result = "Unknown value" }
override string toString() {
result = "Unknown value"
}
override boolean booleanValue() { result = maybe() }
override boolean booleanValue() {
result = maybe()
}
override ClassDecl getClassDeclaration() {
none()
}
override ClassDecl getClassDeclaration() { none() }
override boolean isClass() { result = false }
override ObjectInternal getClass() {
result = TUnknownClass()
}
override ObjectInternal getClass() { result = TUnknownClass() }
override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
none()
}
override predicate introducedAt(ControlFlowNode node, PointsToContext context) { none() }
override predicate notTestableForEquality() { any() }
override Builtin getBuiltin() {
result = Builtin::unknown()
}
override Builtin getBuiltin() { result = Builtin::unknown() }
override predicate callResult(ObjectInternal obj, CfgOrigin origin) {
obj = ObjectInternal::unknown() and origin = CfgOrigin::unknown()
@@ -321,37 +313,40 @@ class UnknownInternal extends ObjectInternal, TUnknown {
none()
}
override ControlFlowNode getOrigin() {
none()
}
override ControlFlowNode getOrigin() { none() }
override int intValue() {
none()
}
override int intValue() { none() }
override string strValue() {
none()
}
override string strValue() { none() }
override predicate calleeAndOffset(Function scope, int paramOffset) {
none()
}
override predicate calleeAndOffset(Function scope, int paramOffset) { none() }
pragma [noinline] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) {
none()
}
pragma[noinline]
override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { none() }
pragma [noinline] override predicate attributesUnknown() { any() }
pragma[noinline]
override predicate attributesUnknown() { any() }
override predicate subscriptUnknown() { any() }
override boolean isDescriptor() { result = false }
pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() }
pragma[noinline]
override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) {
none()
}
pragma [noinline] override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() }
pragma[noinline]
override predicate descriptorGetInstance(
ObjectInternal instance, ObjectInternal value, CfgOrigin origin
) {
none()
}
pragma [noinline] override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() }
pragma[noinline]
override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) {
none()
}
override int length() { result = -1 }
@@ -364,38 +359,24 @@ class UnknownInternal extends ObjectInternal, TUnknown {
override ObjectInternal getIterNext() { result = ObjectInternal::unknown() }
override predicate isNotSubscriptedType() { any() }
}
class UndefinedInternal extends ObjectInternal, TUndefined {
override string toString() { result = "Undefined variable" }
override string toString() {
result = "Undefined variable"
}
override boolean booleanValue() { none() }
override boolean booleanValue() {
none()
}
override ClassDecl getClassDeclaration() {
none()
}
override ClassDecl getClassDeclaration() { none() }
override boolean isClass() { result = false }
override predicate notTestableForEquality() { any() }
override ObjectInternal getClass() {
none()
}
override ObjectInternal getClass() { none() }
override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
none()
}
override predicate introducedAt(ControlFlowNode node, PointsToContext context) { none() }
override Builtin getBuiltin() {
none()
}
override Builtin getBuiltin() { none() }
override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) {
none()
@@ -407,37 +388,40 @@ class UndefinedInternal extends ObjectInternal, TUndefined {
obj = ObjectInternal::unknown() and origin = CfgOrigin::unknown()
}
override ControlFlowNode getOrigin() {
none()
}
override ControlFlowNode getOrigin() { none() }
override int intValue() {
none()
}
override int intValue() { none() }
override string strValue() {
none()
}
override string strValue() { none() }
override predicate calleeAndOffset(Function scope, int paramOffset) {
none()
}
override predicate calleeAndOffset(Function scope, int paramOffset) { none() }
pragma [noinline] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) {
none()
}
pragma[noinline]
override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { none() }
pragma [noinline] override predicate attributesUnknown() { none() }
pragma[noinline]
override predicate attributesUnknown() { none() }
override predicate subscriptUnknown() { none() }
override boolean isDescriptor() { none() }
pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() }
pragma[noinline]
override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) {
none()
}
pragma [noinline] override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() }
pragma[noinline]
override predicate descriptorGetInstance(
ObjectInternal instance, ObjectInternal value, CfgOrigin origin
) {
none()
}
pragma [noinline] override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() }
pragma[noinline]
override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) {
none()
}
override int length() { none() }
@@ -445,39 +429,31 @@ class UndefinedInternal extends ObjectInternal, TUndefined {
override predicate useOriginAsLegacyObject() { none() }
/** Holds if this object requires context to determine the object resulting from a call to it.
* True for most callables. */
/**
* Holds if this object requires context to determine the object resulting from a call to it.
* True for most callables.
*/
override predicate contextSensitiveCallee() { none() }
override ObjectInternal getIterNext() { none() }
override predicate isNotSubscriptedType() { any() }
}
module ObjectInternal {
ObjectInternal bool(boolean b) {
b = true and result = TTrue()
or
b = false and result = TFalse()
}
ObjectInternal none_() {
result = TNone()
}
ObjectInternal none_() { result = TNone() }
ObjectInternal unknown() {
result = TUnknown()
}
ObjectInternal unknown() { result = TUnknown() }
ClassObjectInternal unknownClass() {
result = TUnknownClass()
}
ClassObjectInternal unknownClass() { result = TUnknownClass() }
ObjectInternal undefined() {
result = TUndefined()
}
ObjectInternal undefined() { result = TUndefined() }
ObjectInternal builtin(string name) {
result = TBuiltinClassObject(Builtin::builtin(name))
@@ -490,12 +466,10 @@ module ObjectInternal {
}
ObjectInternal sysModules() {
result = TBuiltinOpaqueObject(Builtin::special("sys").getMember("modules"))
result = TBuiltinOpaqueObject(Builtin::special("sys").getMember("modules"))
}
ObjectInternal fromInt(int n) {
result = TInt(n)
}
ObjectInternal fromInt(int n) { result = TInt(n) }
ObjectInternal fromBuiltin(Builtin b) {
b = result.getBuiltin() and
@@ -506,67 +480,38 @@ module ObjectInternal {
b = Builtin::special("sys").getMember("version_info") and result = TSysVersionInfo()
}
ObjectInternal classMethod() {
result = TBuiltinClassObject(Builtin::special("ClassMethod"))
}
ObjectInternal classMethod() { result = TBuiltinClassObject(Builtin::special("ClassMethod")) }
ObjectInternal staticMethod() {
result = TBuiltinClassObject(Builtin::special("StaticMethod"))
}
ObjectInternal staticMethod() { result = TBuiltinClassObject(Builtin::special("StaticMethod")) }
ObjectInternal boundMethod() {
result = TBuiltinClassObject(Builtin::special("MethodType"))
}
ObjectInternal boundMethod() { result = TBuiltinClassObject(Builtin::special("MethodType")) }
ObjectInternal moduleType() {
result = TBuiltinClassObject(Builtin::special("ModuleType"))
}
ObjectInternal moduleType() { result = TBuiltinClassObject(Builtin::special("ModuleType")) }
ObjectInternal noneType() {
result = TBuiltinClassObject(Builtin::special("NoneType"))
}
ObjectInternal noneType() { result = TBuiltinClassObject(Builtin::special("NoneType")) }
ObjectInternal type() {
result = TType()
}
ObjectInternal type() { result = TType() }
ObjectInternal property() {
result = TBuiltinClassObject(Builtin::special("property"))
}
ObjectInternal property() { result = TBuiltinClassObject(Builtin::special("property")) }
ObjectInternal superType() {
result = TBuiltinClassObject(Builtin::special("super"))
}
ObjectInternal superType() { result = TBuiltinClassObject(Builtin::special("super")) }
/** The old-style class type (Python 2 only) */
ObjectInternal classType() {
result = TBuiltinClassObject(Builtin::special("ClassType"))
}
ObjectInternal emptyTuple() {
result.(BuiltinTupleObjectInternal).length() = 0
}
ObjectInternal classType() { result = TBuiltinClassObject(Builtin::special("ClassType")) }
ObjectInternal emptyTuple() { result.(BuiltinTupleObjectInternal).length() = 0 }
}
class DecoratedFunction extends ObjectInternal, TDecoratedFunction {
CallNode getDecoratorCall() { this = TDecoratedFunction(result) }
CallNode getDecoratorCall() {
this = TDecoratedFunction(result)
}
override Builtin getBuiltin() {
none()
}
override Builtin getBuiltin() { none() }
private ObjectInternal decoratedObject() {
PointsTo::pointsTo(this.getDecoratorCall().getArg(0), _, result, _)
}
override string getName() {
result = this.decoratedObject().getName()
}
override string getName() { result = this.decoratedObject().getName() }
override string toString() {
result = "Decorated " + this.decoratedObject().toString()
@@ -576,17 +521,13 @@ class DecoratedFunction extends ObjectInternal, TDecoratedFunction {
override boolean booleanValue() { result = true }
override ClassDecl getClassDeclaration() {
none()
}
override ClassDecl getClassDeclaration() { none() }
override boolean isClass() { result = false }
override ObjectInternal getClass() { result = TUnknownClass() }
override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
none()
}
override predicate introducedAt(ControlFlowNode node, PointsToContext context) { none() }
override predicate notTestableForEquality() { none() }
@@ -598,25 +539,15 @@ class DecoratedFunction extends ObjectInternal, TDecoratedFunction {
obj = ObjectInternal::unknown() and origin = CfgOrigin::unknown()
}
override ControlFlowNode getOrigin() {
result = this.getDecoratorCall()
}
override ControlFlowNode getOrigin() { result = this.getDecoratorCall() }
override int intValue() {
none()
}
override int intValue() { none() }
override string strValue() {
none()
}
override string strValue() { none() }
override predicate calleeAndOffset(Function scope, int paramOffset) {
none()
}
override predicate calleeAndOffset(Function scope, int paramOffset) { none() }
override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) {
none()
}
override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { none() }
override predicate attributesUnknown() { none() }
@@ -624,11 +555,22 @@ class DecoratedFunction extends ObjectInternal, TDecoratedFunction {
override boolean isDescriptor() { result = false }
pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() }
pragma[noinline]
override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) {
none()
}
pragma [noinline] override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() }
pragma[noinline]
override predicate descriptorGetInstance(
ObjectInternal instance, ObjectInternal value, CfgOrigin origin
) {
none()
}
pragma [noinline] override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() }
pragma[noinline]
override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) {
none()
}
override int length() { none() }
@@ -639,17 +581,13 @@ class DecoratedFunction extends ObjectInternal, TDecoratedFunction {
override predicate useOriginAsLegacyObject() { none() }
override predicate isNotSubscriptedType() { any() }
}
/** Helper for boolean predicates returning both `true` and `false` */
boolean maybe() {
result = true or result = false
}
boolean maybe() { result = true or result = false }
/** Helper for attributes */
pragma [nomagic]
pragma[nomagic]
predicate receiver_type(AttrNode attr, string name, ObjectInternal value, ClassObjectInternal cls) {
PointsToInternal::pointsTo(attr.getObject(name), _, value, _) and value.getClass() = cls
}

158
python/ql/src/semmle/python/objects/Sequences.qll
View File

@@ -1,9 +1,4 @@
import python
private import semmle.python.objects.TObject
private import semmle.python.objects.ObjectInternal
private import semmle.python.pointsto.PointsTo
@@ -12,7 +7,6 @@ private import semmle.python.pointsto.MRO
private import semmle.python.types.Builtins
abstract class SequenceObjectInternal extends ObjectInternal {
/** Gets the `n`th item of this sequence, if one exists. */
abstract ObjectInternal getItem(int n);
@@ -24,45 +18,50 @@ abstract class SequenceObjectInternal extends ObjectInternal {
override boolean isDescriptor() { result = false }
pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() }
pragma[noinline]
override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) {
none()
}
pragma [noinline] override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() }
pragma[noinline]
override predicate descriptorGetInstance(
ObjectInternal instance, ObjectInternal value, CfgOrigin origin
) {
none()
}
pragma [noinline] override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() }
pragma[noinline]
override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) {
none()
}
override string getName() { none() }
override predicate contextSensitiveCallee() { none() }
override ObjectInternal getIterNext() { result = this.getItem(_) }
}
abstract class TupleObjectInternal extends SequenceObjectInternal {
override string toString() {
result = "(" + this.contents(0) + ")"
}
override string toString() { result = "(" + this.contents(0) + ")" }
private string contents(int n) {
n < 4 and n = this.length() and result = ""
or
n = 3 and this.length() > 3 and result = (this.length()-3).toString() + " more..."
n = 3 and this.length() > 3 and result = (this.length() - 3).toString() + " more..."
or
result = this.item(n) + ", " + this.contents(n+1)
result = this.item(n) + ", " + this.contents(n + 1)
}
private string item(int n) {
exists(ObjectInternal item | item = this.getItem(n) |
// To avoid infinite recursion, nested tuples are replaced with the string "...".
if item instanceof TupleObjectInternal then
result = "(...)"
else
result = item.toString()
if item instanceof TupleObjectInternal then result = "(...)" else result = item.toString()
)
or
n in [0..this.length()-1] and
not exists(this.getItem(n)) and result = "?"
n in [0 .. this.length() - 1] and
not exists(this.getItem(n)) and
result = "?"
}
/** Gets the class declaration for this object, if it is a declared class. */
@@ -73,60 +72,59 @@ abstract class TupleObjectInternal extends SequenceObjectInternal {
override ObjectInternal getClass() { result = ObjectInternal::builtin("tuple") }
/** True if this "object" can be meaningfully analysed for
/**
* True if this "object" can be meaningfully analysed for
* truth or false in comparisons. For example, `None` or `int` can be, but `int()`
* or an unknown string cannot.
*/
override predicate notTestableForEquality() { none() }
/** Holds if `obj` is the result of calling `this` and `origin` is
/**
* Holds if `obj` is the result of calling `this` and `origin` is
* the origin of `obj`.
*/
override predicate callResult(ObjectInternal obj, CfgOrigin origin) { none() }
/** Holds if `obj` is the result of calling `this` and `origin` is
/**
* Holds if `obj` is the result of calling `this` and `origin` is
* the origin of `obj` with callee context `callee`.
*/
override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() }
override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) {
none()
}
/** The integer value of things that have integer values.
/**
* The integer value of things that have integer values.
* That is, ints and bools.
*/
override int intValue() { none() }
/** The integer value of things that have integer values.
/**
* The integer value of things that have integer values.
* That is, strings.
*/
override string strValue() { none() }
override predicate calleeAndOffset(Function scope, int paramOffset) { none() }
pragma [noinline] override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { none() }
pragma[noinline]
override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { none() }
pragma [noinline] override predicate attributesUnknown() { none() }
pragma[noinline]
override predicate attributesUnknown() { none() }
override predicate subscriptUnknown() { none() }
}
/** A tuple built-in to the interpreter, including the empty tuple. */
class BuiltinTupleObjectInternal extends TBuiltinTuple, TupleObjectInternal {
override predicate introducedAt(ControlFlowNode node, PointsToContext context) { none() }
override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
none()
}
override Builtin getBuiltin() { this = TBuiltinTuple(result) }
override Builtin getBuiltin() {
this = TBuiltinTuple(result)
}
override ControlFlowNode getOrigin() { none() }
override ControlFlowNode getOrigin() {
none()
}
override ObjectInternal getItem(int n) {
result.getBuiltin() = this.getBuiltin().getItem(n)
}
override ObjectInternal getItem(int n) { result.getBuiltin() = this.getBuiltin().getItem(n) }
override int length() {
exists(Builtin b |
@@ -138,23 +136,17 @@ class BuiltinTupleObjectInternal extends TBuiltinTuple, TupleObjectInternal {
override predicate useOriginAsLegacyObject() { none() }
override predicate isNotSubscriptedType() { any() }
}
/** A tuple declared by a tuple expression in the Python source code */
class PythonTupleObjectInternal extends TPythonTuple, TupleObjectInternal {
override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
this = TPythonTuple(node, context)
}
override Builtin getBuiltin() {
none()
}
override Builtin getBuiltin() { none() }
override ControlFlowNode getOrigin() {
this = TPythonTuple(result, _)
}
override ControlFlowNode getOrigin() { this = TPythonTuple(result, _) }
override ObjectInternal getItem(int n) {
exists(TupleNode t, PointsToContext context |
@@ -173,51 +165,37 @@ class PythonTupleObjectInternal extends TPythonTuple, TupleObjectInternal {
override predicate useOriginAsLegacyObject() { none() }
override predicate isNotSubscriptedType() { any() }
}
/** A tuple created by a `*` parameter */
class VarargsTupleObjectInternal extends TVarargsTuple, TupleObjectInternal {
class VarargsTupleObjectInternal extends TVarargsTuple, TupleObjectInternal {
override predicate introducedAt(ControlFlowNode node, PointsToContext context) { none() }
override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
none()
}
override Builtin getBuiltin() { none() }
override Builtin getBuiltin() {
none()
}
override ControlFlowNode getOrigin() {
none()
}
override ControlFlowNode getOrigin() { none() }
override ObjectInternal getItem(int n) {
exists(CallNode call, PointsToContext context, int offset, int length |
this = TVarargsTuple(call, context, offset, length) and
n < length and
InterProceduralPointsTo::positional_argument_points_to(call, offset+n, context, result, _)
InterProceduralPointsTo::positional_argument_points_to(call, offset + n, context, result, _)
)
}
override int length() {
this = TVarargsTuple(_, _, _, result)
}
override int length() { this = TVarargsTuple(_, _, _, result) }
override predicate useOriginAsLegacyObject() { any() }
override predicate isNotSubscriptedType() { any() }
}
/** The `sys.version_info` object. We treat this specially to prevent premature pruning and
/**
* The `sys.version_info` object. We treat this specially to prevent premature pruning and
* false positives when we are unsure of the actual version of Python that the code is expecting.
*/
class SysVersionInfoObjectInternal extends TSysVersionInfo, SequenceObjectInternal {
override string toString() {
result = "sys.version_info"
}
override string toString() { result = "sys.version_info" }
override ObjectInternal getItem(int n) {
n = 0 and result = TInt(major_version())
@@ -235,39 +213,45 @@ class SysVersionInfoObjectInternal extends TSysVersionInfo, SequenceObjectIntern
/** True if this "object" is a class. */
override boolean isClass() { result = false }
override ObjectInternal getClass() {
result.getBuiltin() = this.getClassDeclaration()
}
override ObjectInternal getClass() { result.getBuiltin() = this.getClassDeclaration() }
override predicate notTestableForEquality() { none() }
/** Gets the `Builtin` for this object, if any.
/**
* Gets the `Builtin` for this object, if any.
* Objects (except unknown and undefined values) should attempt to return
* exactly one result for either this method or `getOrigin()`.
*/
override Builtin getBuiltin() { none() }
/** Gets a control flow node that represents the source origin of this
/**
* Gets a control flow node that represents the source origin of this
* objects.
*/
override ControlFlowNode getOrigin() { none() }
/** Holds if `obj` is the result of calling `this` and `origin` is
/**
* Holds if `obj` is the result of calling `this` and `origin` is
* the origin of `obj`.
*/
override predicate callResult(ObjectInternal obj, CfgOrigin origin) { none() }
/** Holds if `obj` is the result of calling `this` and `origin` is
/**
* Holds if `obj` is the result of calling `this` and `origin` is
* the origin of `obj` with callee context `callee`.
*/
override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() }
override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) {
none()
}
/** The integer value of things that have integer values.
/**
* The integer value of things that have integer values.
* That is, ints and bools.
*/
override int intValue() { none() }
/** The integer value of things that have integer values.
/**
* The integer value of things that have integer values.
* That is, strings.
*/
override string strValue() { none() }
@@ -280,7 +264,8 @@ class SysVersionInfoObjectInternal extends TSysVersionInfo, SequenceObjectIntern
override predicate subscriptUnknown() { none() }
/** Gets the length of the sequence that this "object" represents.
/**
* Gets the length of the sequence that this "object" represents.
* Always returns a value for a sequence, will be -1 if object has no fixed length.
*/
override int length() { result = 5 }
@@ -290,5 +275,4 @@ class SysVersionInfoObjectInternal extends TSysVersionInfo, SequenceObjectIntern
override predicate useOriginAsLegacyObject() { any() }
override predicate isNotSubscriptedType() { any() }
}

305
python/ql/src/semmle/python/objects/TObject.qll
View File

@@ -4,239 +4,193 @@ private import semmle.python.objects.ObjectInternal
private import semmle.python.pointsto.PointsTo
private import semmle.python.pointsto.PointsToContext
/** Internal type backing `ObjectInternal` and `Value`
/**
* Internal type backing `ObjectInternal` and `Value`
* See `ObjectInternal.qll` for an explanation of the API.
*/
cached newtype TObject =
cached
newtype TObject =
/* Builtin class objects */
TBuiltinClassObject(Builtin bltn) {
bltn.isClass() and
not bltn = Builtin::unknownType() and
not bltn = Builtin::special("type")
}
or
} or
/* Builtin function objects (module members) */
TBuiltinFunctionObject(Builtin bltn) { bltn.isFunction() }
or
TBuiltinFunctionObject(Builtin bltn) { bltn.isFunction() } or
/* Builtin method objects (class members) */
TBuiltinMethodObject(Builtin bltn) { bltn.isMethod() }
or
TBuiltinMethodObject(Builtin bltn) { bltn.isMethod() } or
/* Builtin module objects */
TBuiltinModuleObject(Builtin bltn) { bltn.isModule() }
or
TBuiltinModuleObject(Builtin bltn) { bltn.isModule() } or
/* Other builtin objects from the interpreter */
TBuiltinOpaqueObject(Builtin bltn) {
not bltn.isClass() and not bltn.isFunction() and
not bltn.isMethod() and not bltn.isModule() and
not bltn.isClass() and
not bltn.isFunction() and
not bltn.isMethod() and
not bltn.isModule() and
not bltn.getClass() = Builtin::special("tuple") and
not exists(bltn.intValue()) and
not exists(bltn.floatValue()) and
not exists(bltn.strValue()) and
not py_special_objects(bltn, _)
}
or
} or
/* Python function objects (including lambdas) */
TPythonFunctionObject(ControlFlowNode callable) {
callable.getNode() instanceof CallableExpr
}
or
TPythonFunctionObject(ControlFlowNode callable) { callable.getNode() instanceof CallableExpr } or
/* Python class objects */
TPythonClassObject(ControlFlowNode classexpr) {
classexpr.getNode() instanceof ClassExpr
}
or
TPythonClassObject(ControlFlowNode classexpr) { classexpr.getNode() instanceof ClassExpr } or
/* Package objects */
TPackageObject(Folder f) {
isPreferredModuleForName(f, _)
}
or
TPackageObject(Folder f) { isPreferredModuleForName(f, _) } or
/* Python module objects */
TPythonModule(Module m) {
not m.isPackage() and isPreferredModuleForName(m.getFile(), _) and
not m.isPackage() and
isPreferredModuleForName(m.getFile(), _) and
not exists(SyntaxError se | se.getFile() = m.getFile())
}
or
} or
/* `True` */
TTrue()
or
TTrue() or
/* `False` */
TFalse()
or
TFalse() or
/* `None` */
TNone()
or
TNone() or
/* Represents any value about which nothing useful is known */
TUnknown()
or
TUnknown() or
/* Represents any value known to be a class, but not known to be any specific class */
TUnknownClass()
or
TUnknownClass() or
/* Represents the absence of a value. Used by points-to for tracking undefined variables */
TUndefined()
or
TUndefined() or
/* The integer `n` */
TInt(int n) {
// Powers of 2 are used for flags
is_power_2(n) or
is_power_2(n)
or
// And all combinations of flags up to 2^8
n in [0..511] or
n in [0 .. 511]
or
// Any number explicitly mentioned in the source code.
exists(IntegerLiteral num |
n = num.getValue() or
exists(UnaryExpr neg | neg.getOp() instanceof USub and neg.getOperand() = num)
and n = -num.getN().toInt()
n = num.getValue()
or
exists(UnaryExpr neg | neg.getOp() instanceof USub and neg.getOperand() = num) and
n = -num.getN().toInt()
)
or
n = any(Builtin b).intValue()
}
or
} or
/* The float `f` */
TFloat(float f) {
f = any(FloatLiteral num).getValue()
}
or
TFloat(float f) { f = any(FloatLiteral num).getValue() } or
/* The unicode string `s` */
TUnicode(string s) {
// Any string explicitly mentioned in the source code.
exists(StrConst str |
exists(StrConst str |
s = str.getText() and
str.isUnicode()
)
or
// Any string from the library put in the DB by the extractor.
exists(Builtin b |
s = b.strValue() and
s = b.strValue() and
b.getClass() = Builtin::special("unicode")
)
or
s = "__main__"
}
or
} or
/* The byte string `s` */
TBytes(string s) {
// Any string explicitly mentioned in the source code.
exists(StrConst str |
exists(StrConst str |
s = str.getText() and
not str.isUnicode()
)
or
// Any string from the library put in the DB by the extractor.
exists(Builtin b |
s = b.strValue() and
s = b.strValue() and
b.getClass() = Builtin::special("bytes")
)
or
s = "__main__"
}
or
} or
/* An instance of `cls`, instantiated at `instantiation` given the `context`. */
TSpecificInstance(ControlFlowNode instantiation, ClassObjectInternal cls, PointsToContext context) {
PointsToInternal::pointsTo(instantiation.(CallNode).getFunction(), context, cls, _) and
cls.isSpecial() = false
or
literal_instantiation(instantiation, cls, context)
}
or
} or
/* A non-specific instance `cls` which enters the scope at `def` given the callee `context`. */
TSelfInstance(ParameterDefinition def, PointsToContext context, PythonClassObjectInternal cls) {
self_parameter(def, context, cls)
}
or
} or
/* A bound method */
TBoundMethod(ObjectInternal self, CallableObjectInternal function) {
any(ObjectInternal obj).binds(self, _, function) and
function.isDescriptor() = true
}
or
} or
/* Represents any value whose class is known, but nothing else */
TUnknownInstance(BuiltinClassObjectInternal cls) {
cls != ObjectInternal::superType() and
cls != ObjectInternal::builtin("bool") and
cls != ObjectInternal::noneType()
}
or
} or
/* Represents an instance of `super` */
TSuperInstance(ObjectInternal self, ClassObjectInternal startclass) {
super_instantiation(_, self, startclass, _)
}
or
} or
/* Represents an instance of `classmethod` */
TClassMethod(CallNode instantiation, CallableObjectInternal function) {
class_method(instantiation, function, _)
}
or
} or
/* Represents an instance of `staticmethod` */
TStaticMethod(CallNode instantiation, CallableObjectInternal function) {
static_method(instantiation, function, _)
}
or
} or
/* Represents a builtin tuple */
TBuiltinTuple(Builtin bltn) {
bltn.getClass() = Builtin::special("tuple")
}
or
TBuiltinTuple(Builtin bltn) { bltn.getClass() = Builtin::special("tuple") } or
/* Represents a tuple in the Python source */
TPythonTuple(TupleNode origin, PointsToContext context) {
origin.isLoad() and
context.appliesTo(origin)
}
or
} or
/* Varargs tuple */
TVarargsTuple(CallNode call, PointsToContext context, int offset, int length) {
InterProceduralPointsTo::varargs_tuple(call, context, _, _, offset, length)
}
or
} or
/* `type` */
TType()
or
TType() or
/* Represents an instance of `property` */
TProperty(CallNode call, Context ctx, CallableObjectInternal getter) {
PointsToInternal::pointsTo(call.getFunction(), ctx, ObjectInternal::property(), _) and
PointsToInternal::pointsTo(call.getArg(0), ctx, getter, _)
}
or
} or
/* Represents the `setter` or `deleter` method of a property object. */
TPropertySetterOrDeleter(PropertyInternal property, string method) {
exists(AttrNode attr |
PointsToInternal::pointsTo(attr.getObject(method), _, property, _)
) and
( method = "setter" or method = "deleter" )
}
or
exists(AttrNode attr | PointsToInternal::pointsTo(attr.getObject(method), _, property, _)) and
(method = "setter" or method = "deleter")
} or
/* Represents a dynamically created class */
TDynamicClass(CallNode instantiation, ClassObjectInternal metacls, PointsToContext context) {
PointsToInternal::pointsTo(instantiation.getFunction(), context, metacls, _) and
not count(instantiation.getAnArg()) = 1 and
Types::getMro(metacls).contains(TType())
}
or
} or
/* Represents `sys.version_info`. Acts like a tuple with a range of values depending on the version being analysed. */
TSysVersionInfo()
or
TSysVersionInfo() or
/* Represents a module that is inferred to perhaps exist, but is not present in the database. */
TAbsentModule(string name) {
missing_imported_module(_, _, name)
}
or
TAbsentModule(string name) { missing_imported_module(_, _, name) } or
/* Represents an attribute of a module that is inferred to perhaps exist, but is not present in the database. */
TAbsentModuleAttribute(AbsentModuleObjectInternal mod, string attrname) {
(
PointsToInternal::pointsTo(any(AttrNode attr).getObject(attrname), _, mod, _)
or
PointsToInternal::pointsTo(any(ImportMemberNode imp).getModule(attrname), _, mod, _)
)
and
) and
exists(string modname |
modname = mod.getName() and
not common_module_name(modname + "." + attrname)
)
}
or
} or
/* Opaque object representing the result of calling a decorator on a function that we don't understand */
TDecoratedFunction(CallNode call) {
call.isFunctionDecoratorCall()
}
or
TDecoratedFunction(CallNode call) { call.isFunctionDecoratorCall() } or
/* Represents a subscript operation applied to a type. For type-hint analysis */
TSubscriptedType(ObjectInternal generic, ObjectInternal index) {
isType(generic) and
@@ -252,16 +206,22 @@ predicate isType(ObjectInternal t) {
}
private predicate is_power_2(int n) {
n = 1 or
exists(int half | is_power_2(half) and n = half*2)
n = 1
or
exists(int half | is_power_2(half) and n = half * 2)
}
predicate static_method(CallNode instantiation, CallableObjectInternal function, PointsToContext context) {
PointsToInternal::pointsTo(instantiation.getFunction(), context, ObjectInternal::builtin("staticmethod"), _) and
predicate static_method(
CallNode instantiation, CallableObjectInternal function, PointsToContext context
) {
PointsToInternal::pointsTo(instantiation.getFunction(), context,
ObjectInternal::builtin("staticmethod"), _) and
PointsToInternal::pointsTo(instantiation.getArg(0), context, function, _)
}
predicate class_method(CallNode instantiation, CallableObjectInternal function, PointsToContext context) {
predicate class_method(
CallNode instantiation, CallableObjectInternal function, PointsToContext context
) {
PointsToInternal::pointsTo(instantiation.getFunction(), context, ObjectInternal::classMethod(), _) and
PointsToInternal::pointsTo(instantiation.getArg(0), context, function, _)
}
@@ -285,14 +245,20 @@ predicate literal_instantiation(ControlFlowNode n, ClassObjectInternal cls, Poin
)
}
predicate super_instantiation(CallNode instantiation, ObjectInternal self, ClassObjectInternal startclass, PointsToContext context) {
predicate super_instantiation(
CallNode instantiation, ObjectInternal self, ClassObjectInternal startclass,
PointsToContext context
) {
super_2args(instantiation, self, startclass, context)
or
super_noargs(instantiation, self, startclass, context)
}
pragma [noinline]
private predicate super_2args(CallNode instantiation, ObjectInternal self, ClassObjectInternal startclass, PointsToContext context) {
pragma[noinline]
private predicate super_2args(
CallNode instantiation, ObjectInternal self, ClassObjectInternal startclass,
PointsToContext context
) {
exists(ControlFlowNode arg0, ControlFlowNode arg1 |
super_call2(instantiation, arg0, arg1, context) and
PointsToInternal::pointsTo(arg0, context, startclass, _) and
@@ -300,17 +266,23 @@ private predicate super_2args(CallNode instantiation, ObjectInternal self, Class
)
}
pragma [noinline]
private predicate super_call2(CallNode call, ControlFlowNode arg0, ControlFlowNode arg1, PointsToContext context) {
pragma[noinline]
private predicate super_call2(
CallNode call, ControlFlowNode arg0, ControlFlowNode arg1, PointsToContext context
) {
exists(ControlFlowNode func |
call2(call, func, arg0, arg1) and
PointsToInternal::pointsTo(func, context, ObjectInternal::superType(), _)
)
}
pragma [noinline]
private predicate super_noargs(CallNode instantiation, ObjectInternal self, ClassObjectInternal startclass, PointsToContext context) {
PointsToInternal::pointsTo(instantiation.getFunction(), context, ObjectInternal::builtin("super"), _) and
pragma[noinline]
private predicate super_noargs(
CallNode instantiation, ObjectInternal self, ClassObjectInternal startclass,
PointsToContext context
) {
PointsToInternal::pointsTo(instantiation.getFunction(), context, ObjectInternal::builtin("super"),
_) and
not exists(instantiation.getArg(0)) and
exists(Function func |
instantiation.getScope() = func and
@@ -328,7 +300,10 @@ predicate call2(CallNode call, ControlFlowNode func, ControlFlowNode arg0, Contr
arg1 = call.getArg(1)
}
predicate call3(CallNode call, ControlFlowNode func, ControlFlowNode arg0, ControlFlowNode arg1, ControlFlowNode arg2) {
predicate call3(
CallNode call, ControlFlowNode func, ControlFlowNode arg0, ControlFlowNode arg1,
ControlFlowNode arg2
) {
not exists(call.getArg(3)) and
func = call.getFunction() and
arg0 = call.getArg(0) and
@@ -337,9 +312,11 @@ predicate call3(CallNode call, ControlFlowNode func, ControlFlowNode arg0, Contr
}
bindingset[self, function]
predicate method_binding(AttrNode instantiation, ObjectInternal self, CallableObjectInternal function, PointsToContext context) {
exists(ObjectInternal obj, string name |
receiver(instantiation, context, obj, name) |
predicate method_binding(
AttrNode instantiation, ObjectInternal self, CallableObjectInternal function,
PointsToContext context
) {
exists(ObjectInternal obj, string name | receiver(instantiation, context, obj, name) |
exists(ObjectInternal cls |
cls = obj.getClass() and
cls != ObjectInternal::superType() and
@@ -356,41 +333,45 @@ predicate method_binding(AttrNode instantiation, ObjectInternal self, CallableOb
)
}
/** Helper for method_binding */
pragma [noinline]
pragma[noinline]
predicate receiver(AttrNode instantiation, PointsToContext context, ObjectInternal obj, string name) {
PointsToInternal::pointsTo(instantiation.getObject(name), context, obj, _)
}
/** Helper self parameters: `def meth(self, ...): ...`. */
pragma [noinline]
private predicate self_parameter(ParameterDefinition def, PointsToContext context, PythonClassObjectInternal cls) {
pragma[noinline]
private predicate self_parameter(
ParameterDefinition def, PointsToContext context, PythonClassObjectInternal cls
) {
def.isSelf() and
/* Exclude the special parameter name `.0` which is used for unfolded comprehensions. */
def.getName() != ".0" and
exists(Function scope |
def.getScope() = scope and
context.isRuntime() and context.appliesToScope(scope) and
context.isRuntime() and
context.appliesToScope(scope) and
scope.getScope() = cls.getScope() and
concrete_class(cls) and
/* We want to allow decorated functions, otherwise we lose a lot of useful information.
/*
* We want to allow decorated functions, otherwise we lose a lot of useful information.
* However, we want to exclude any function whose arguments are permuted by the decorator.
* In general we can't do that, but we can special case the most common ones.
*/
neither_class_nor_static_method(scope)
)
}
private cached predicate concrete_class(PythonClassObjectInternal cls) {
cached
private predicate concrete_class(PythonClassObjectInternal cls) {
cls.getClass() != abcMetaClassObject()
or
exists(Class c |
c = cls.getScope() and
not exists(c.getMetaClass())
|
forall(Function f |
f.getScope() = c |
|
forall(Function f | f.getScope() = c |
not exists(Raise r, Name ex |
r.getScope() = f and
(r.getException() = ex or r.getException().(Call).getFunc() = ex) and
@@ -402,8 +383,7 @@ private cached predicate concrete_class(PythonClassObjectInternal cls) {
private PythonClassObjectInternal abcMetaClassObject() {
/* Avoid using points-to and thus negative recursion */
exists(Class abcmeta |
result.getScope() = abcmeta |
exists(Class abcmeta | result.getScope() = abcmeta |
abcmeta.getName() = "ABCMeta" and
abcmeta.getScope().getName() = "abc"
)
@@ -412,19 +392,19 @@ private PythonClassObjectInternal abcMetaClassObject() {
private predicate neither_class_nor_static_method(Function f) {
not exists(f.getADecorator())
or
exists(ControlFlowNode deco |
deco = f.getADecorator().getAFlowNode() |
exists(ObjectInternal o |
PointsToInternal::pointsTo(deco, _, o, _) |
exists(ControlFlowNode deco | deco = f.getADecorator().getAFlowNode() |
exists(ObjectInternal o | PointsToInternal::pointsTo(deco, _, o, _) |
o != ObjectInternal::staticMethod() and
o != ObjectInternal::classMethod()
)
or not deco instanceof NameNode
or
not deco instanceof NameNode
)
}
predicate missing_imported_module(ControlFlowNode imp, Context ctx, string name) {
ctx.isImport() and imp.(ImportExprNode).getNode().getAnImportedModuleName() = name and
ctx.isImport() and
imp.(ImportExprNode).getNode().getAnImportedModuleName() = name and
(
not exists(Module m | m.getName() = name) and
not exists(Builtin b | b.isModule() and b.getName() = name)
@@ -441,41 +421,41 @@ predicate missing_imported_module(ControlFlowNode imp, Context ctx, string name)
)
}
/* Helper for missing modules to determine if name `x.y` is a module `x.y` or
/*
* Helper for missing modules to determine if name `x.y` is a module `x.y` or
* an attribute `y` of module `x`. This list should be added to as required.
*/
predicate common_module_name(string name) {
name = "zope.interface"
or
name = "six.moves"
}
/** A declaration of a class, either a built-in class or a source definition
/**
* A declaration of a class, either a built-in class or a source definition
* This acts as a helper for ClassObjectInternal allowing some lookup without
* recursion.
*/
library class ClassDecl extends @py_object {
ClassDecl() {
this.(Builtin).isClass() and not this = Builtin::unknownType()
or
this.(ControlFlowNode).getNode() instanceof ClassExpr
}
string toString() {
result = "ClassDecl"
}
string toString() { result = "ClassDecl" }
/** Gets the class scope for Python class declarations */
Class getClass() {
result = this.(ControlFlowNode).getNode().(ClassExpr).getInnerScope()
}
Class getClass() { result = this.(ControlFlowNode).getNode().(ClassExpr).getInnerScope() }
/** Holds if this class declares the attribute `name` */
predicate declaresAttribute(string name) {
exists(this.(Builtin).getMember(name))
or
exists(SsaVariable var | name = var.getId() and var.getAUse() = this.getClass().getANormalExit())
exists(SsaVariable var |
name = var.getId() and var.getAUse() = this.getClass().getANormalExit()
)
}
/** Gets the name of this class */
@@ -485,11 +465,11 @@ library class ClassDecl extends @py_object {
result = this.getClass().getName()
}
/** Whether this is a class whose instances must be treated specially, rather than as generic instances.
/**
* Whether this is a class whose instances must be treated specially, rather than as generic instances.
*/
predicate isSpecial() {
exists(string name |
this = Builtin::special(name) |
exists(string name | this = Builtin::special(name) |
name = "type" or
name = "super" or
name = "bool" or
@@ -505,16 +485,15 @@ library class ClassDecl extends @py_object {
/** Holds if for class `C`, `C()` returns an instance of `C` */
predicate callReturnsInstance() {
exists(Class pycls |
pycls = this.getClass() |
exists(Class pycls | pycls = this.getClass() |
/* Django does this, so we need to account for it */
not exists(Function init, LocalVariable self |
/* `self.__class__ = ...` in the `__init__` method */
pycls.getInitMethod() = init and
self.isSelf() and self.getScope() = init and
self.isSelf() and
self.getScope() = init and
exists(AttrNode a | a.isStore() and a.getObject("__class__") = self.getAUse())
)
and
) and
not exists(Function new | new.getName() = "__new__" and new.getScope() = pycls)
)
or
@@ -527,11 +506,9 @@ library class ClassDecl extends @py_object {
m.getName() = "_abcoll"
or
m.getName() = "_collections_abc"
|
|
this.getClass().getScope() = m and
this.getName() = name
)
}
}

158
python/ql/src/semmle/python/pointsto/Base.qll
View File

@@ -4,22 +4,24 @@
* to the possible objects it points-to the inferred types of those objects and the 'origin'
* of those objects. The 'origin' is the point in source code that the object can be traced
* back to.
*
*
* This file contains non-layered parts of the points-to analysis.
*/
import python
import semmle.python.essa.SsaDefinitions
private import semmle.python.types.Builtins
module BasePointsTo {
/** INTERNAL -- Use n.refersTo(value, _, origin) instead */
pragma [noinline]
pragma[noinline]
predicate points_to(ControlFlowNode f, Object value, ControlFlowNode origin) {
(
(
f.isLiteral() and value = f and not f.getNode() instanceof ImmutableLiteral
or
f.isFunction() and value = f
) and origin = f
) and
origin = f
}
}
@@ -35,12 +37,13 @@ predicate varargs_points_to(ControlFlowNode f, ClassObject cls) {
cls = theTupleType()
}
/** Gets the class of the object for simple cases, namely constants, functions,
/**
* Gets the class of the object for simple cases, namely constants, functions,
* comprehensions and built-in objects.
*
* This exists primarily for internal use. Use getAnInferredType() instead.
*/
pragma [noinline]
pragma[noinline]
ClassObject simple_types(Object obj) {
result = comprehension(obj.getOrigin())
or
@@ -79,37 +82,42 @@ private int tuple_index_value(Object t, int i) {
result = t.(TupleNode).getElement(i).getNode().(Num).getN().toInt()
or
exists(Object item |
py_citems(t, i, item) and
py_citems(t, i, item) and
result = item.(NumericObject).intValue()
)
}
pragma [noinline]
pragma[noinline]
int version_tuple_value(Object t) {
not exists(tuple_index_value(t, 1)) and result = tuple_index_value(t, 0)*10
not exists(tuple_index_value(t, 1)) and result = tuple_index_value(t, 0) * 10
or
not exists(tuple_index_value(t, 2)) and result = tuple_index_value(t, 0)*10 + tuple_index_value(t, 1)
not exists(tuple_index_value(t, 2)) and
result = tuple_index_value(t, 0) * 10 + tuple_index_value(t, 1)
or
tuple_index_value(t, 2) = 0 and result = tuple_index_value(t, 0)*10 + tuple_index_value(t, 1)
tuple_index_value(t, 2) = 0 and result = tuple_index_value(t, 0) * 10 + tuple_index_value(t, 1)
or
tuple_index_value(t, 2) > 0 and result = tuple_index_value(t, 0)*10 + tuple_index_value(t, 1) + 1
tuple_index_value(t, 2) > 0 and
result = tuple_index_value(t, 0) * 10 + tuple_index_value(t, 1) + 1
}
/** Choose a version numbers that represent the extreme of supported versions. */
private int major_minor() {
if major_version() = 3 then
(result = 33 or result = 37) // 3.3 to 3.7
else
(result = 25 or result = 27) // 2.5 to 2.7
if major_version() = 3
then (
result = 33 or result = 37
) else (
// 3.3 to 3.7
result = 25 or result = 27
) // 2.5 to 2.7
}
/** Compares the given tuple object to both the maximum and minimum possible sys.version_info values */
int version_tuple_compare(Object t) {
version_tuple_value(t) < major_minor() and result = -1
version_tuple_value(t) < major_minor() and result = -1
or
version_tuple_value(t) = major_minor() and result = 0
version_tuple_value(t) = major_minor() and result = 0
or
version_tuple_value(t) > major_minor() and result = 1
version_tuple_value(t) > major_minor() and result = 1
}
/* Holds if `cls` is a new-style class if it were to have no explicit base classes */
@@ -121,14 +129,15 @@ predicate baseless_is_new_style(ClassObject cls) {
exists(cls.declaredMetaClass())
}
/* The following predicates exist in order to provide
/*
* The following predicates exist in order to provide
* more precise type information than the underlying
* database relations. This help to optimise the points-to
* analysis.
*/
/** Holds if this class (not on a super-class) declares name */
pragma [noinline]
pragma[noinline]
predicate class_declares_attribute(ClassObject cls, string name) {
exists(Class defn |
defn = cls.getPyClass() and
@@ -136,7 +145,7 @@ predicate class_declares_attribute(ClassObject cls, string name) {
)
or
exists(Builtin o |
o = cls.asBuiltin().getMember(name) and
o = cls.asBuiltin().getMember(name) and
not exists(Builtin sup |
sup = cls.asBuiltin().getBaseClass() and
o = sup.getMember(name)
@@ -151,9 +160,9 @@ private predicate class_defines_name(Class cls, string name) {
/** Gets a return value CFG node, provided that is safe to track across returns */
ControlFlowNode safe_return_node(PyFunctionObject func) {
result = func.getAReturnedNode()
result = func.getAReturnedNode() and
// Not a parameter
and not exists(Parameter p, SsaVariable pvar |
not exists(Parameter p, SsaVariable pvar |
p.asName().getAFlowNode() = pvar.getDefinition() and
result = pvar.getAUse()
) and
@@ -163,9 +172,11 @@ ControlFlowNode safe_return_node(PyFunctionObject func) {
/** Holds if it can be determined from the control flow graph alone that this function can never return */
predicate function_can_never_return(FunctionObject func) {
/* A Python function never returns if it has no normal exits that are not dominated by a
/*
* A Python function never returns if it has no normal exits that are not dominated by a
* call to a function which itself never returns.
*/
exists(Function f |
f = func.getFunction() and
not exists(f.getAnExitNode())
@@ -174,27 +185,21 @@ predicate function_can_never_return(FunctionObject func) {
func = ModuleObject::named("sys").attr("exit")
}
private newtype TIterationDefinition =
private newtype TIterationDefinition =
TIterationDefinition_(SsaSourceVariable var, ControlFlowNode def, ControlFlowNode sequence) {
SsaSource::iteration_defined_variable(var, def, sequence)
}
/** DEPRECATED. For backwards compatibility only.
* A definition of a variable in a for loop `for v in ...:` */
/**
* DEPRECATED. For backwards compatibility only.
* A definition of a variable in a for loop `for v in ...:`
*/
deprecated class IterationDefinition extends TIterationDefinition {
string toString() { result = "IterationDefinition" }
string toString() {
result = "IterationDefinition"
}
ControlFlowNode getSequence() {
this = TIterationDefinition_(_, _, result)
}
ControlFlowNode getSequence() { this = TIterationDefinition_(_, _, result) }
}
/** Hold if outer contains inner, both are contained within a test and inner is a use is a plain use or an attribute lookup */
pragma[noinline]
predicate contains_interesting_expression_within_test(ControlFlowNode outer, ControlFlowNode inner) {
@@ -202,7 +207,8 @@ predicate contains_interesting_expression_within_test(ControlFlowNode outer, Con
exists(ControlFlowNode test |
outer.getAChild*() = inner and
test_contains(test, outer) and
test_contains(test, inner) |
test_contains(test, inner)
|
inner instanceof NameNode or
inner instanceof AttrNode
)
@@ -216,10 +222,14 @@ predicate test_contains(ControlFlowNode expr, ControlFlowNode use) {
}
/** Holds if `test` is a test (a branch), `use` is within that test and `def` is an edge from that test with `sense` */
predicate refinement_test(ControlFlowNode test, ControlFlowNode use, boolean sense, PyEdgeRefinement def) {
/* Because calls such as `len` may create a new variable, we need to go via the source variable
predicate refinement_test(
ControlFlowNode test, ControlFlowNode use, boolean sense, PyEdgeRefinement def
) {
/*
* Because calls such as `len` may create a new variable, we need to go via the source variable
* That is perfectly safe as we are only dealing with calls that do not mutate their arguments.
*/
use = def.getInput().getSourceVariable().(Variable).getAUse() and
test = def.getPredecessor().getLastNode() and
test_contains(test, use) and
@@ -227,11 +237,12 @@ predicate refinement_test(ControlFlowNode test, ControlFlowNode use, boolean sen
}
/** Holds if `f` is an import of the form `from .[...] import name` and the enclosing scope is an __init__ module */
pragma [noinline]
pragma[noinline]
predicate live_import_from_dot_in_init(ImportMemberNode f, EssaVariable var) {
exists(string name |
import_from_dot_in_init(f.getModule(name)) and
var.getSourceVariable().getName() = name and var.getAUse() = f
var.getSourceVariable().getName() = name and
var.getAUse() = f
)
}
@@ -247,23 +258,23 @@ predicate import_from_dot_in_init(ImportExprNode f) {
}
/** Gets the pseudo-object representing the value referred to by an undefined variable */
Object undefinedVariable() {
py_special_objects(result, "_semmle_undefined_value")
}
Object undefinedVariable() { py_special_objects(result, "_semmle_undefined_value") }
/** Gets the pseudo-object representing an unknown value */
Object unknownValue() {
result.asBuiltin() = Builtin::unknown()
}
Object unknownValue() { result.asBuiltin() = Builtin::unknown() }
BuiltinCallable theTypeNewMethod() {
result.asBuiltin() = theTypeType().asBuiltin().getMember("__new__")
}
/** Gets the `value, cls, origin` that `f` would refer to if it has not been assigned some other value */
pragma [noinline]
predicate potential_builtin_points_to(NameNode f, Object value, ClassObject cls, ControlFlowNode origin) {
f.isGlobal() and f.isLoad() and origin = f and
pragma[noinline]
predicate potential_builtin_points_to(
NameNode f, Object value, ClassObject cls, ControlFlowNode origin
) {
f.isGlobal() and
f.isLoad() and
origin = f and
(
builtin_name_points_to(f.getId(), value, cls)
or
@@ -271,36 +282,41 @@ predicate potential_builtin_points_to(NameNode f, Object value, ClassObject cls,
)
}
pragma [noinline]
pragma[noinline]
predicate builtin_name_points_to(string name, Object value, ClassObject cls) {
value = Object::builtin(name) and cls.asBuiltin() = value.asBuiltin().getClass()
}
module BaseFlow {
predicate reaches_exit(EssaVariable var) {
var.getAUse() = var.getScope().getANormalExit()
}
predicate reaches_exit(EssaVariable var) { var.getAUse() = var.getScope().getANormalExit() }
/* Helper for this_scope_entry_value_transfer(...). Transfer of values from earlier scope to later on */
cached predicate scope_entry_value_transfer_from_earlier(EssaVariable pred_var, Scope pred_scope, ScopeEntryDefinition succ_def, Scope succ_scope) {
cached
predicate scope_entry_value_transfer_from_earlier(
EssaVariable pred_var, Scope pred_scope, ScopeEntryDefinition succ_def, Scope succ_scope
) {
exists(SsaSourceVariable var |
reaches_exit(pred_var) and
pred_var.getScope() = pred_scope and
var = pred_var.getSourceVariable() and
var = succ_def.getSourceVariable() and
succ_def.getScope() = succ_scope
|
|
pred_scope.precedes(succ_scope)
or
/* If an `__init__` method does not modify the global variable, then
/*
* If an `__init__` method does not modify the global variable, then
* we can skip it and take the value directly from the module.
*/
exists(Scope init |
init.getName() = "__init__" and init.precedes(succ_scope) and pred_scope.precedes(init) and
not var.(Variable).getAStore().getScope() = init and var instanceof GlobalVariable
init.getName() = "__init__" and
init.precedes(succ_scope) and
pred_scope.precedes(init) and
not var.(Variable).getAStore().getScope() = init and
var instanceof GlobalVariable
)
)
)
}
}
@@ -312,15 +328,17 @@ predicate simple_points_to(ControlFlowNode f, Object value, ClassObject cls, Con
or
BasePointsTo::points_to(f, value, origin) and cls = simple_types(value)
or
value = f.getNode().(ImmutableLiteral).getLiteralObject() and cls = simple_types(value) and origin = f
value = f.getNode().(ImmutableLiteral).getLiteralObject() and
cls = simple_types(value) and
origin = f
}
/** Holds if `bit` is a binary expression node with a bitwise operator.
/**
* Holds if `bit` is a binary expression node with a bitwise operator.
* Helper for `this_binary_expr_points_to`.
*/
predicate bitwise_expression_node(BinaryExprNode bit, ControlFlowNode left, ControlFlowNode right) {
exists(Operator op |
op = bit.getNode().getOp() |
exists(Operator op | op = bit.getNode().getOp() |
op instanceof BitAnd or
op instanceof BitOr or
op instanceof BitXor
@@ -329,16 +347,14 @@ predicate bitwise_expression_node(BinaryExprNode bit, ControlFlowNode left, Cont
right = bit.getRight()
}
private
Module theCollectionsAbcModule() {
private Module theCollectionsAbcModule() {
result.getName() = "_abcoll"
or
result.getName() = "_collections_abc"
}
ClassObject collectionsAbcClass(string name) {
exists(Class cls |
exists(Class cls |
result.getPyClass() = cls and
cls.getName() = name and
cls.getScope() = theCollectionsAbcModule()

51
python/ql/src/semmle/python/pointsto/CallGraph.qll
View File

@@ -1,42 +1,47 @@
/**
* Context-sensitive call-graph.
*
*
* NOTE: Since an "invocation" contains callsite information
* and a path back to its ancestor calls, the "invocation" call-graph must be a tree.
* This has two important consequences:
* 1. The graph is incomplete; it has quite limited depth in order to keep the graph to a sensible size.
* 2. The graph is precise. Since different invocations are distinct, there can be no "cross-talk" between
* 2. The graph is precise. Since different invocations are distinct, there can be no "cross-talk" between
* different calls to the same function.
*/
import python
private import semmle.python.pointsto.PointsToContext
private newtype TTInvocation = TInvocation(FunctionObject f, Context c) {
exists(Context outer, CallNode call |
call = f.getACall(outer) and
c.fromCall(call, outer)
)
or
c.appliesToScope(f.getFunction())
}
private newtype TTInvocation =
TInvocation(FunctionObject f, Context c) {
exists(Context outer, CallNode call |
call = f.getACall(outer) and
c.fromCall(call, outer)
)
or
c.appliesToScope(f.getFunction())
}
/** This class represents a static approximation to the
* dynamic call-graph. A `FunctionInvocation` represents
/**
* This class represents a static approximation to the
* dynamic call-graph. A `FunctionInvocation` represents
* all calls made to a function for a given context.
*/
class FunctionInvocation extends TTInvocation {
string toString() { result = "Invocation" }
FunctionObject getFunction() { this = TInvocation(result, _) }
Context getContext() { this = TInvocation(_, result) }
/** Gets the callee invocation for the given callsite.
/**
* Gets the callee invocation for the given callsite.
* The callsite must be within the function of this invocation.
*/
FunctionInvocation getCallee(CallNode call) {
exists(FunctionObject callee, Context callee_context, FunctionObject caller, Context caller_context |
exists(
FunctionObject callee, Context callee_context, FunctionObject caller, Context caller_context
|
this = TInvocation(caller, caller_context) and
result = TInvocation(callee, callee_context) and
call = callee.getACall(caller_context) and
@@ -45,12 +50,11 @@ class FunctionInvocation extends TTInvocation {
)
}
/** Gets a callee invocation.
/**
* Gets a callee invocation.
* That is any invocation made from within this invocation.
*/
FunctionInvocation getACallee() {
result = this.getCallee(_)
}
FunctionInvocation getACallee() { result = this.getCallee(_) }
/** Holds if this is an invocation `f` in the "runtime" context. */
predicate runtime(FunctionObject f) {
@@ -61,13 +65,8 @@ class FunctionInvocation extends TTInvocation {
}
/** Gets the call from which this invocation was made. */
CallNode getCall() {
this.getContext().fromCall(result, _)
}
CallNode getCall() { this.getContext().fromCall(result, _) }
/** Gets the caller invocation of this invocation, if any. */
FunctionInvocation getCaller() {
this = result.getCallee(_)
}
FunctionInvocation getCaller() { this = result.getCallee(_) }
}

24
python/ql/src/semmle/python/pointsto/Filters.qll
View File

@@ -1,8 +1,8 @@
/** Helper predicates for standard tests in Python commonly
/**
* Helper predicates for standard tests in Python commonly
* used to filter objects by value or by type.
*/
import python
/** Holds if `c` is a call to `hasattr(obj, attr)`. */
@@ -21,13 +21,15 @@ predicate is_callable(CallNode c, ControlFlowNode obj) {
/** Holds if `c` is a call to `isinstance(use, cls)`. */
predicate isinstance(CallNode fc, ControlFlowNode cls, ControlFlowNode use) {
fc.getFunction().(NameNode).getId() = "isinstance" and
cls = fc.getArg(1) and fc.getArg(0) = use
cls = fc.getArg(1) and
fc.getArg(0) = use
}
/** Holds if `c` is a call to `issubclass(use, cls)`. */
predicate issubclass(CallNode fc, ControlFlowNode cls, ControlFlowNode use) {
fc.getFunction().(NameNode).getId() = "issubclass" and
fc.getArg(0) = use and cls = fc.getArg(1)
fc.getArg(0) = use and
cls = fc.getArg(1)
}
/** Holds if `c` is a test comparing `x` and `y`. `is` is true if the operator is `is` or `==`, it is false if the operator is `is not` or `!=`. */
@@ -35,11 +37,15 @@ predicate equality_test(CompareNode c, ControlFlowNode x, boolean is, ControlFlo
exists(Cmpop op |
c.operands(x, op, y) or
c.operands(y, op, x)
|
(is = true and op instanceof Is or
is = false and op instanceof IsNot or
is = true and op instanceof Eq or
is = false and op instanceof NotEq
|
(
is = true and op instanceof Is
or
is = false and op instanceof IsNot
or
is = true and op instanceof Eq
or
is = false and op instanceof NotEq
)
)
}

3
python/ql/src/semmle/python/pointsto/Final.qll
View File

@@ -1,6 +1,5 @@
/* For backwards compatibility */
import PointsTo::PointsTo as P
/** DEPRECATED: Use `PointsTo` instead */
deprecated module FinalPointsTo = P;
deprecated module FinalPointsTo = P;

211
python/ql/src/semmle/python/pointsto/MRO.qll
View File

@@ -1,4 +1,5 @@
/** Classes and predicates for computing the Method Resolution Order (MRO) of classes.
/**
* Classes and predicates for computing the Method Resolution Order (MRO) of classes.
* Supports both old-style (diamond) inheritance and new-style (C3 linearization) inheritance.
*/
@@ -17,19 +18,16 @@
*/
import python
private import semmle.python.objects.TObject
private import semmle.python.objects.ObjectInternal
private import semmle.python.pointsto.PointsTo
private import semmle.python.pointsto.PointsToContext
private import semmle.python.types.Builtins
cached newtype TClassList = Empty()
or
Cons(ClassObjectInternal head, TClassList tail) {
required_cons(head, tail)
}
cached
newtype TClassList =
Empty() or
Cons(ClassObjectInternal head, TClassList tail) { required_cons(head, tail) }
/* Keep ClassList finite and as small as possible */
private predicate required_cons(ClassObjectInternal head, ClassList tail) {
@@ -38,7 +36,7 @@ private predicate required_cons(ClassObjectInternal head, ClassList tail) {
tail = merge_of_linearization_of_bases(head)
or
exists(ClassObjectInternal cls, int n |
head = Types::getBase(cls, n) and tail = bases(cls, n+1)
head = Types::getBase(cls, n) and tail = bases(cls, n + 1)
)
or
head = ObjectInternal::builtin("object") and tail = Empty()
@@ -52,12 +50,12 @@ private predicate required_cons(ClassObjectInternal head, ClassList tail) {
or
exists(ClassList list, int n |
n = list.firstIndex(head) and
tail = list.deduplicate(n+1)
tail = list.deduplicate(n + 1)
)
or
exists(ClassListList list, int n |
head = list.getHead().getItem(n) and
tail = flatten_list(list, n+1)
tail = flatten_list(list, n + 1)
)
or
tail = list_old_style_base_mros(head).flatten()
@@ -70,16 +68,12 @@ private ClassObjectInternal sole_base(ClassObjectInternal cls) {
/** A list of classes, used to represent the MRO of a class */
class ClassList extends TClassList {
string toString() {
result = "[" + this.contents() + "]"
}
string toString() { result = "[" + this.contents() + "]" }
string contents() {
this = Empty() and result = ""
or
exists(ClassObjectInternal head |
head = this.getHead() |
exists(ClassObjectInternal head | head = this.getHead() |
this.getTail() = Empty() and result = className(head)
or
this.getTail() != Empty() and result = className(head) + ", " + this.getTail().contents()
@@ -98,25 +92,19 @@ class ClassList extends TClassList {
result = this.getTail().length() + 1
}
ClassObjectInternal getHead() {
this = Cons(result, _)
}
ClassObjectInternal getHead() { this = Cons(result, _) }
ClassList getTail() {
this = Cons(_, result)
}
ClassList getTail() { this = Cons(_, result) }
ClassObjectInternal getItem(int n) {
n = 0 and result = this.getHead()
n = 0 and result = this.getHead()
or
result = this.getTail().getItem(n-1)
result = this.getTail().getItem(n - 1)
}
ClassObjectInternal getAnItem() {
result = this.getItem(_)
}
ClassObjectInternal getAnItem() { result = this.getItem(_) }
pragma [inline]
pragma[inline]
ClassList removeHead(ClassObjectInternal cls) {
this.getHead() = cls and result = this.getTail()
or
@@ -159,18 +147,15 @@ class ClassList extends TClassList {
}
ClassObjectInternal findDeclaringClass(string name) {
exists(ClassDecl head |
head = this.getHead().getClassDeclaration() |
if head.declaresAttribute(name) then
result = this.getHead()
else
result = this.getTail().findDeclaringClass(name)
exists(ClassDecl head | head = this.getHead().getClassDeclaration() |
if head.declaresAttribute(name)
then result = this.getHead()
else result = this.getTail().findDeclaringClass(name)
)
}
predicate lookup(string name, ObjectInternal value, CfgOrigin origin) {
exists(ClassObjectInternal decl |
decl = this.findDeclaringClass(name) |
exists(ClassObjectInternal decl | decl = this.findDeclaringClass(name) |
Types::declaredAttribute(decl, name, value, origin)
)
}
@@ -182,83 +167,63 @@ class ClassList extends TClassList {
}
ClassList startingAt(ClassObjectInternal cls) {
exists(ClassObjectInternal head |
head = this.getHead() |
if head = cls then
result = this
else
result = this.getTail().startingAt(cls)
exists(ClassObjectInternal head | head = this.getHead() |
if head = cls then result = this else result = this.getTail().startingAt(cls)
)
}
ClassList deduplicate() {
result = this.deduplicate(0)
}
ClassList deduplicate() { result = this.deduplicate(0) }
/* Helpers for `deduplicate()` */
int firstIndex(ClassObjectInternal cls) {
result = this.firstIndex(cls, 0)
}
int firstIndex(ClassObjectInternal cls) { result = this.firstIndex(cls, 0) }
/* Helper for firstIndex(cls), getting the first index of `cls` where result >= n */
private int firstIndex(ClassObjectInternal cls, int n) {
this.getItem(n) = cls and result = n
or
this.getItem(n) != cls and result = this.firstIndex(cls, n+1)
this.getItem(n) != cls and result = this.firstIndex(cls, n + 1)
}
/** Holds if the class at `n` is a duplicate of an earlier position. */
private predicate duplicate(int n) {
exists(ClassObjectInternal cls |
cls = this.getItem(n) and this.firstIndex(cls) < n
)
exists(ClassObjectInternal cls | cls = this.getItem(n) and this.firstIndex(cls) < n)
}
/** Gets a class list which is the de-duplicated form of the list containing elements of
/**
* Gets a class list which is the de-duplicated form of the list containing elements of
* this list from `n` onwards.
*/
ClassList deduplicate(int n) {
n = this.length() and result = Empty()
or
this.duplicate(n) and result = this.deduplicate(n+1)
this.duplicate(n) and result = this.deduplicate(n + 1)
or
exists(ClassObjectInternal cls |
n = this.firstIndex(cls) and
result = Cons(cls, this.deduplicate(n+1))
result = Cons(cls, this.deduplicate(n + 1))
)
}
predicate isEmpty() {
this = Empty()
}
predicate isEmpty() { this = Empty() }
ClassList reverse() {
reverse_step(this, Empty(), result)
}
ClassList reverse() { reverse_step(this, Empty(), result) }
/** Holds if this MRO contains a class whose instances we treat specially, rather than as a generic instance.
/**
* Holds if this MRO contains a class whose instances we treat specially, rather than as a generic instance.
* For example, `type` or `int`.
*/
boolean containsSpecial() {
this = Empty() and result = false
or
exists(ClassDecl decl |
decl = this.getHead().getClassDeclaration() |
if decl.isSpecial() then
result = true
else
result = this.getTail().containsSpecial()
exists(ClassDecl decl | decl = this.getHead().getClassDeclaration() |
if decl.isSpecial() then result = true else result = this.getTail().containsSpecial()
)
}
}
private newtype TClassListList =
EmptyList() or
ConsList(TClassList head, TClassListList tail) {
required_list(head, tail)
}
ConsList(TClassList head, TClassListList tail) { required_list(head, tail) }
/* Keep ClassListList finite and as small as possible */
private predicate required_list(ClassList head, ClassListList tail) {
@@ -268,26 +233,22 @@ private predicate required_list(ClassList head, ClassListList tail) {
or
exists(ClassObjectInternal cls, int n |
head = Mro::newStyleMro(Types::getBase(cls, n)) and
tail = list_of_linearization_of_bases_plus_bases(cls, n+1)
tail = list_of_linearization_of_bases_plus_bases(cls, n + 1)
)
or
exists(ClassObjectInternal cls, int n |
head = Mro::oldStyleMro(Types::getBase(cls, n)) and
tail = list_old_style_base_mros(cls, n+1)
tail = list_old_style_base_mros(cls, n + 1)
)
}
private class ClassListList extends TClassListList {
string toString() {
result = "[" + this.contents() + "]"
}
string toString() { result = "[" + this.contents() + "]" }
string contents() {
this = EmptyList() and result = ""
or
exists(ClassList head |
head = this.getHead() |
exists(ClassList head | head = this.getHead() |
this.getTail() = EmptyList() and result = head.toString()
or
this.getTail() != EmptyList() and result = head.toString() + ", " + this.getTail().contents()
@@ -300,18 +261,14 @@ private class ClassListList extends TClassListList {
result = this.getTail().length() + 1
}
ClassList getHead() {
this = ConsList(result, _)
}
ClassList getHead() { this = ConsList(result, _) }
ClassListList getTail() {
this = ConsList(_, result)
}
ClassListList getTail() { this = ConsList(_, result) }
ClassList getItem(int n) {
n = 0 and result = this.getHead()
or
result = this.getTail().getItem(n-1)
result = this.getTail().getItem(n - 1)
}
private ClassObjectInternal getAHead() {
@@ -320,7 +277,7 @@ private class ClassListList extends TClassListList {
result = this.getTail().getAHead()
}
pragma [nomagic]
pragma[nomagic]
ClassList merge() {
exists(ClassList reversed |
merge_step(reversed, EmptyList(), this) and
@@ -331,13 +288,17 @@ private class ClassListList extends TClassListList {
}
/* Join ordering helper */
pragma [noinline]
predicate removedClassParts(ClassObjectInternal cls, ClassList removed_head, ClassListList removed_tail, int n) {
cls = this.bestMergeCandidate() and n = this.length()-1 and
removed_head = this.getItem(n).removeHead(cls) and removed_tail = EmptyList()
pragma[noinline]
predicate removedClassParts(
ClassObjectInternal cls, ClassList removed_head, ClassListList removed_tail, int n
) {
cls = this.bestMergeCandidate() and
n = this.length() - 1 and
removed_head = this.getItem(n).removeHead(cls) and
removed_tail = EmptyList()
or
exists(ClassList prev_head, ClassListList prev_tail |
this.removedClassParts(cls, prev_head, prev_tail, n+1) and
this.removedClassParts(cls, prev_head, prev_tail, n + 1) and
removed_head = this.getItem(n).removeHead(cls) and
removed_tail = ConsList(prev_head, prev_tail)
)
@@ -356,12 +317,10 @@ private class ClassListList extends TClassListList {
cls = this.getAHead() and n = this.length()
or
this.getItem(n).legalMergeHead(cls) and
this.legalMergeCandidate(cls, n+1)
this.legalMergeCandidate(cls, n + 1)
}
predicate legalMergeCandidate(ClassObjectInternal cls) {
this.legalMergeCandidate(cls, 0)
}
predicate legalMergeCandidate(ClassObjectInternal cls) { this.legalMergeCandidate(cls, 0) }
predicate illegalMergeCandidate(ClassObjectInternal cls) {
cls = this.getAHead() and
@@ -369,20 +328,17 @@ private class ClassListList extends TClassListList {
}
ClassObjectInternal bestMergeCandidate(int n) {
exists(ClassObjectInternal head |
head = this.getItem(n).getHead()
|
exists(ClassObjectInternal head | head = this.getItem(n).getHead() |
legalMergeCandidate(head) and result = head
or
illegalMergeCandidate(head) and result = this.bestMergeCandidate(n+1)
illegalMergeCandidate(head) and result = this.bestMergeCandidate(n + 1)
)
}
ClassObjectInternal bestMergeCandidate() {
result = this.bestMergeCandidate(0)
}
ClassObjectInternal bestMergeCandidate() { result = this.bestMergeCandidate(0) }
/** Gets a ClassList representing the this list of list flattened into a single list.
/**
* Gets a ClassList representing the this list of list flattened into a single list.
* Used for old-style MRO computation.
*/
ClassList flatten() {
@@ -390,17 +346,14 @@ private class ClassListList extends TClassListList {
or
result = flatten_list(this, 0)
}
}
private ClassList flatten_list(ClassListList list, int n) {
need_flattening(list) and
exists(ClassList head, ClassListList tail |
list = ConsList(head, tail)
|
exists(ClassList head, ClassListList tail | list = ConsList(head, tail) |
n = head.length() and result = tail.flatten()
or
result = Cons(head.getItem(n), flatten_list(list, n+1))
result = Cons(head.getItem(n), flatten_list(list, n + 1))
)
}
@@ -414,12 +367,10 @@ private predicate need_flattening(ClassListList list) {
)
}
private ClassList bases(ClassObjectInternal cls) {
result = bases(cls, 0)
}
private ClassList bases(ClassObjectInternal cls) { result = bases(cls, 0) }
private ClassList bases(ClassObjectInternal cls, int n) {
result = Cons(Types::getBase(cls, n), bases(cls, n+1))
result = Cons(Types::getBase(cls, n), bases(cls, n + 1))
or
result = Empty() and n = Types::base_count(cls)
}
@@ -432,7 +383,7 @@ private ClassListList list_of_linearization_of_bases_plus_bases(ClassObjectInter
result = ConsList(bases(cls), EmptyList()) and n = Types::base_count(cls) and n > 1
or
exists(ClassListList partial |
partial = list_of_linearization_of_bases_plus_bases(cls, n+1) and
partial = list_of_linearization_of_bases_plus_bases(cls, n + 1) and
result = ConsList(Mro::newStyleMro(Types::getBase(cls, n)), partial)
)
}
@@ -445,18 +396,23 @@ private ClassListList list_old_style_base_mros(ClassObjectInternal cls) {
result = list_old_style_base_mros(cls, 0)
}
pragma [nomagic]
pragma[nomagic]
private ClassListList list_old_style_base_mros(ClassObjectInternal cls, int n) {
n = Types::base_count(cls) and result = EmptyList()
or
result = ConsList(Mro::oldStyleMro(Types::getBase(cls, n)), list_old_style_base_mros(cls, n+1))
result = ConsList(Mro::oldStyleMro(Types::getBase(cls, n)), list_old_style_base_mros(cls, n + 1))
}
/** Holds if the pair `reversed_mro`, `remaining_list` represents a step in the C3 merge operation
/**
* Holds if the pair `reversed_mro`, `remaining_list` represents a step in the C3 merge operation
* of computing the C3 linearization of `original`.
*/
private predicate merge_step(ClassList reversed_mro, ClassListList remaining_list, ClassListList original) {
remaining_list = list_of_linearization_of_bases_plus_bases(_) and reversed_mro = Empty() and remaining_list = original
private predicate merge_step(
ClassList reversed_mro, ClassListList remaining_list, ClassListList original
) {
remaining_list = list_of_linearization_of_bases_plus_bases(_) and
reversed_mro = Empty() and
remaining_list = original
or
/* Removes the best merge candidate from `remaining_list` and prepends it to `reversed_mro` */
exists(ClassObjectInternal head, ClassList prev_reverse_mro, ClassListList prev_list |
@@ -470,7 +426,6 @@ private predicate merge_step(ClassList reversed_mro, ClassListList remaining_lis
}
/* Helpers for `ClassList.reverse()` */
private predicate needs_reversing(ClassList lst) {
merge_step(lst, EmptyList(), _)
or
@@ -487,8 +442,8 @@ private predicate reverse_step(ClassList lst, ClassList remainder, ClassList rev
}
module Mro {
cached ClassList newStyleMro(ClassObjectInternal cls) {
cached
ClassList newStyleMro(ClassObjectInternal cls) {
cls = ObjectInternal::builtin("object") and result = Cons(cls, Empty())
or
result = Cons(cls, merge_of_linearization_of_bases(cls))
@@ -496,9 +451,9 @@ module Mro {
result = Cons(cls, newStyleMro(sole_base(cls)))
}
cached ClassList oldStyleMro(ClassObjectInternal cls) {
cached
ClassList oldStyleMro(ClassObjectInternal cls) {
Types::isOldStyle(cls) and
result = Cons(cls, list_old_style_base_mros(cls).flatten()).(ClassList).deduplicate()
}
}
}

86
python/ql/src/semmle/python/pointsto/Overview.qll
View File

@@ -1,45 +1,45 @@
/*
*
* ## Points-to analysis for Python
*
*
*
*
* The purpose of points-to analysis is to determine what values a variable might hold at runtime.
* This allows us to write useful queries to check for the misuse of those values.
* In the academic and technical literature, points-to analysis (AKA pointer analysis) attempts to determine which variables can refer to which heap allocated objects.
* From the point of view of Python we can treat all Python objects as "heap allocated objects".
*
*
*
*
* The output of the points-to analysis consists of a large set of relations which provide not only points-to information, but call-graph, pruned flow-graph and exception-raising information.
*
* These relations are computed by a large set of mutually recursive predicates which infer the flow of values through the program.
*
* These relations are computed by a large set of mutually recursive predicates which infer the flow of values through the program.
* Our analysis is inter-procedural use contexts to maintain the precision of an intra-procedural analysis.
*
*
* ### Precision
*
*
* In conventional points-to, the computed points-to set should be a super-set of the real points-to set (were it possible to determine such a thing).
* However for our purposes we want the points-to set to be a sub-set of the real points-to set.
* This is simply because conventional points-to is used to determine compiler optimisations, so the points-to set needs to be a conservative over-estimate of what is possible.
* However for our purposes we want the points-to set to be a sub-set of the real points-to set.
* This is simply because conventional points-to is used to determine compiler optimisations, so the points-to set needs to be a conservative over-estimate of what is possible.
* We have the opposite concern; we want to eliminate false positives where possible.
*
* This should be born in mind when reading the literature about points-to analysis. In conventional points-to, a precise analysis produces as small a points-to set as possible.
*
* This should be born in mind when reading the literature about points-to analysis. In conventional points-to, a precise analysis produces as small a points-to set as possible.
* Our analysis is precise (or very close to it). Instead of seeking to maximise precision, we seek to maximise *recall* and produce as large a points-to set as possible (whilst remaining precise).
*
*
* When it comes to designing the inference, we always choose precision over recall.
* We want to minimise false positives so it is important to avoid making incorrect inferences, even if it means losing a lot of potential information.
* If a potential new points-to fact would increase the number of values we are able to infer, but decrease precision, then we omit it.
*
*
* ###Objects
*
* In convention points-to an 'object' is generally considered to be any static instantiation. E.g. in Java this is simply anything looking like `new X(..)`.
* However, in Python as there is no `new` expression we cannot known what is a class merely from the syntax.
*
* In convention points-to an 'object' is generally considered to be any static instantiation. E.g. in Java this is simply anything looking like `new X(..)`.
* However, in Python as there is no `new` expression we cannot known what is a class merely from the syntax.
* Consequently, we must start with only with the simplest objects and extend to instance creation as we can infer classes.
*
*
* To perform points-to analysis we start with the set of built-in objects, all literal constants, and class and function definitions.
* From there we can propagate those values. Whenever we see a call `x()` we add a new object if `x` refers to some class.
*
*
* In the `PointsTo::points_to` relation, the second argument, `Object value` is the "value" referred to by the ControlFlowNode (which will correspond to an rvalue in the source code).
* The set of "values" used will change as the library continues to improve, but currently include the following:
*
*
* * Classes (both in the source and builtin)
* * Functions (both in the source and builtin)
* * Literal constants defined in the source (string and numbers)
@@ -49,56 +49,56 @@
* * Bound methods, static- and class-methods, and properties.
* * Instances of `super`.
* * Missing modules, where no concrete module is found for an import.
*
*
* A number of constructs that might create a new object, such as binary operations, are omitted if there is no useful information to can be attached to them and they would just increase the size of the database.
*
*
* ###Contexts
*
* In order to better handle value tracking in functions, we introduce context to the points-to relation.
*
* In order to better handle value tracking in functions, we introduce context to the points-to relation.
* There is one `default` context, equivalent to having no context, a `main` context for scripts and any number of call-site contexts.
*
* Adding context to a conventional points-to analysis can significantly improve its precision. Whereas, for our points-to analysis adding context significantly improves the recall of our analysis.
* The consensus in the academic literature is that "object sensitivity" is superior to "call-site sensitivity".
*
* Adding context to a conventional points-to analysis can significantly improve its precision. Whereas, for our points-to analysis adding context significantly improves the recall of our analysis.
* The consensus in the academic literature is that "object sensitivity" is superior to "call-site sensitivity".
* However, since we are seeking to maximise not minimise our points-to set, it is entirely possible that the reverse is true for us.
* We use "call-site sensitivity" at the moment, although the exact set of contexts used will change.
*
*
* ### Points-to analysis over the ESSA dataflow graph
*
*
* In order to perform points-to analysis on the dataflow graph, we
* need to understand the many implicit "definitions" that occur within Python code.
*
*
* These are:
*
*
* 1. Implicit definition as "undefined" for any local or global variable at the start of its scope.
* Many of these will be dead and will be eliminated during construction of the dataflow graph.
* 2. Implicit definition of `__name__`, `__package__` and `__module__` at the start of the relevant scopes.
* 3. Implicit definition of all submodules as global variables at the start of an `__init__` module
*
*
* In addition, there are the "artificial", data-flow definitions:
*
*
* 1. Phi functions
* 2. Pi (guard, or filter) functions.
* 3. "Refinements" of a variable. These are not definitions of the variable, but may modify the object referred to by the variable,
* possibly changing some inferred facts about the object.
* 4. Definition of any variable that escapes the scope, at entry, exit and at all call-sites.
*
*
* As an example, consider:
* ```python
* if a:
* float = "global"
* #float can now be either the class 'float' or the string "global"
*
*
* class C2:
* if b:
* float = "local"
* float
*
*
* float #Cannot be "local"
* ```
*
*
* Ignoring `__name__` and `__package__`, the data-flow graph looks something like this, noting that there are two variables named "float"
* in the scope `C2`, the local and the global.
*
*
* ```
* a_0 = undefined
* b_0 = undefined
@@ -112,14 +112,14 @@
* float_6 = phi(float_4, float_5) |
* float_7 = float_3 (transfer values in global 'float', but not local, back to module scope).
* ```
*
*
* ### Implementation
*
*
* <b>This section is for information purposes only. Any or all details may change without notice.</b>
*
*
* QL, being based on Datalog, has fixed-point semantics which makes it impossible to make negative statements that are recursive.
* To work around this we need to define many predicates over boolean variables. Suppose we have a predicate with determines whether a test can be true or false at runtime.
* We might naively implement this as `predicate test_is_true(ControlFlowNode test, Context ctx)` but this would lead to negative recursion if we want to know when the test can be false.
* Instead we implement it as `boolean test_result(ControlFlowNode test, Context ctx)` where the absence of a value indicates merely that we do (yet) know what value the test may have.
*
*/

1362
python/ql/src/semmle/python/pointsto/PointsTo.qll
View File

File diff suppressed because it is too large Load Diff

151
python/ql/src/semmle/python/pointsto/PointsToContext.qll
View File

@@ -1,6 +1,7 @@
import python
private import semmle.python.pointsto.PointsTo
private import semmle.python.objects.ObjectInternal
/*
* A note on 'cost'. Cost doesn't represent the cost to compute,
* but (a vague estimate of) the cost to compute per value gained.
@@ -14,7 +15,7 @@ private int given_cost() {
)
}
pragma [noinline]
pragma[noinline]
private int max_context_cost() {
not py_flags_versioned("context.cost", _, _) and result = 7
or
@@ -22,13 +23,13 @@ private int max_context_cost() {
}
private int syntactic_call_count(Scope s) {
exists(Function f |
f = s and f.getName() != "__init__" |
result = count(CallNode call |
call.getFunction().(NameNode).getId() = f.getName()
or
call.getFunction().(AttrNode).getName() = f.getName()
)
exists(Function f | f = s and f.getName() != "__init__" |
result =
count(CallNode call |
call.getFunction().(NameNode).getId() = f.getName()
or
call.getFunction().(AttrNode).getName() = f.getName()
)
)
or
s.getName() = "__init__" and result = 1
@@ -37,11 +38,13 @@ private int syntactic_call_count(Scope s) {
}
private int incoming_call_cost(Scope s) {
/* Syntactic call count will often be a considerable overestimate
/*
* Syntactic call count will often be a considerable overestimate
* of the actual number of calls, so we use the square root.
* Cost = log(sqrt(call-count))
*/
result = ((syntactic_call_count(s)+1).log(2)*0.5).floor()
result = ((syntactic_call_count(s) + 1).log(2) * 0.5).floor()
}
private int context_cost(TPointsToContext ctx) {
@@ -55,15 +58,10 @@ private int context_cost(TPointsToContext ctx) {
}
private int call_cost(CallNode call) {
if call.getScope().inSource() then
result = 2
else
result = 3
if call.getScope().inSource() then result = 2 else result = 3
}
private int outgoing_calls(Scope s) {
result = strictcount(CallNode call | call.getScope() = s)
}
private int outgoing_calls(Scope s) { result = strictcount(CallNode call | call.getScope() = s) }
predicate super_method_call(CallNode call) {
call.getFunction().(AttrNode).getObject().(CallNode).getFunction().(NameNode).getId() = "super"
@@ -74,74 +72,65 @@ private int outgoing_call_cost(CallNode c) {
result = outgoing_calls(c.getScope()).log(2).floor()
}
/** Cost of contexts for a call, the more callers the
/**
* Cost of contexts for a call, the more callers the
* callee of call has the more expensive it is to add contexts for it.
* This seems to be an effective heuristics for preventing an explosion
* in the number of contexts while retaining good results.
*/
private int splay_cost(CallNode c) {
if super_method_call(c) then
result = 0
else
result = outgoing_call_cost(c) + incoming_call_cost(c.getScope())
if super_method_call(c)
then result = 0
else result = outgoing_call_cost(c) + incoming_call_cost(c.getScope())
}
private predicate call_to_init_or_del(CallNode call) {
exists(string mname |
mname = "__init__" or mname = "__del__" |
exists(string mname | mname = "__init__" or mname = "__del__" |
mname = call.getFunction().(AttrNode).getName()
)
}
/** Total cost estimate */
private int total_call_cost(CallNode call) {
/* We want to always follow __init__ and __del__ calls as they tell us about object construction,
/*
* We want to always follow __init__ and __del__ calls as they tell us about object construction,
* but we need to be aware of cycles, so they must have a non-zero cost.
*/
if call_to_init_or_del(call) then
result = 1
else
result = call_cost(call) + splay_cost(call)
if call_to_init_or_del(call) then result = 1 else result = call_cost(call) + splay_cost(call)
}
pragma [noinline]
pragma[noinline]
private int total_cost(CallNode call, PointsToContext ctx) {
ctx.appliesTo(call) and
result = total_call_cost(call) + context_cost(ctx)
}
private cached newtype TPointsToContext =
TMainContext()
or
TRuntimeContext()
or
TImportContext()
or
cached
private newtype TPointsToContext =
TMainContext() or
TRuntimeContext() or
TImportContext() or
TCallContext(ControlFlowNode call, PointsToContext outerContext, int cost) {
total_cost(call, outerContext) = cost and
cost <= max_context_cost()
}
or
} or
TObjectContext(SelfInstanceInternal object)
module Context {
PointsToContext forObject(ObjectInternal object) {
result = TObjectContext(object)
}
PointsToContext forObject(ObjectInternal object) { result = TObjectContext(object) }
}
/** Points-to context. Context can be one of:
/**
* Points-to context. Context can be one of:
* * "main": Used for scripts.
* * "import": Use for non-script modules.
* * "default": Use for functions and methods without caller context.
* * All other contexts are call contexts and consist of a pair of call-site and caller context.
*/
class PointsToContext extends TPointsToContext {
cached string toString() {
cached
string toString() {
this = TMainContext() and result = "main"
or
this = TRuntimeContext() and result = "runtime"
@@ -166,10 +155,8 @@ class PointsToContext extends TPointsToContext {
this = TCallContext(call, caller, _)
}
/** Gets the caller context for this callee context. */
PointsToContext getOuter() {
this = TCallContext(_, result, _)
}
/** Gets the caller context for this callee context. */
PointsToContext getOuter() { this = TCallContext(_, result, _) }
/** Holds if this context is relevant to the given scope. */
predicate appliesToScope(Scope s) {
@@ -182,9 +169,12 @@ class PointsToContext extends TPointsToContext {
this = TRuntimeContext() and executes_in_runtime_context(s)
or
/* Called functions, regardless of their name */
exists(PythonFunctionObjectInternal callable, ControlFlowNode call, TPointsToContext outerContext |
exists(
PythonFunctionObjectInternal callable, ControlFlowNode call, TPointsToContext outerContext
|
call = callable.getACall(outerContext) and
this = TCallContext(call, outerContext, _) |
this = TCallContext(call, outerContext, _)
|
s = callable.getScope()
)
or
@@ -192,30 +182,20 @@ class PointsToContext extends TPointsToContext {
}
/** Holds if this context can apply to the CFG node `n`. */
pragma [inline]
predicate appliesTo(ControlFlowNode n) {
this.appliesToScope(n.getScope())
}
pragma[inline]
predicate appliesTo(ControlFlowNode n) { this.appliesToScope(n.getScope()) }
/** Holds if this context is a call context. */
predicate isCall() {
this = TCallContext(_, _, _)
}
predicate isCall() { this = TCallContext(_, _, _) }
/** Holds if this is the "main" context. */
predicate isMain() {
this = TMainContext()
}
predicate isMain() { this = TMainContext() }
/** Holds if this is the "import" context. */
predicate isImport() {
this = TImportContext()
}
predicate isImport() { this = TImportContext() }
/** Holds if this is the "default" context. */
predicate isRuntime() {
this = TRuntimeContext()
}
predicate isRuntime() { this = TRuntimeContext() }
/** Holds if this context or one of its caller contexts is the default context. */
predicate fromRuntime() {
@@ -231,19 +211,13 @@ class PointsToContext extends TPointsToContext {
result = this.getOuter().getDepth() + 1
}
int getCost() {
result = context_cost(this)
}
int getCost() { result = context_cost(this) }
CallNode getCall() {
this = TCallContext(result, _, _)
}
CallNode getCall() { this = TCallContext(result, _, _) }
/** Holds if a call would be too expensive to create a new context for */
pragma [nomagic]
predicate untrackableCall(CallNode call) {
total_cost(call, this) > max_context_cost()
}
pragma[nomagic]
predicate untrackableCall(CallNode call) { total_cost(call, this) > max_context_cost() }
CallNode getRootCall() {
this = TCallContext(result, TImportContext(), _)
@@ -252,35 +226,30 @@ class PointsToContext extends TPointsToContext {
}
/** Gets a version of Python that this context includes */
pragma [inline]
pragma[inline]
Version getAVersion() {
/* Currently contexts do not include any version information, but may do in the future */
result = major_version()
}
}
private predicate in_source(Scope s) {
exists(s.getEnclosingModule().getFile().getRelativePath())
}
private predicate in_source(Scope s) { exists(s.getEnclosingModule().getFile().getRelativePath()) }
/** Holds if this scope can be executed in the default context.
/**
* Holds if this scope can be executed in the default context.
* All modules and classes executed at import time and
* all "public" functions and methods, including those invoked by the VM.
*/
predicate executes_in_runtime_context(Function f) {
/* "Public" scope, i.e. functions whose name starts not with an underscore, or special methods */
(f.getName().charAt(0) != "_" or f.isSpecialMethod() or f.isInitMethod())
and
(f.getName().charAt(0) != "_" or f.isSpecialMethod() or f.isInitMethod()) and
in_source(f)
}
private predicate maybe_main(Module m) {
exists(If i, Compare cmp, Name name, StrConst main |
m.getAStmt() = i and i.getTest() = cmp |
exists(If i, Compare cmp, Name name, StrConst main | m.getAStmt() = i and i.getTest() = cmp |
cmp.compares(name, any(Eq eq), main) and
name.getId() = "__name__" and
main.getText() = "__main__"
)
}

24
python/ql/src/semmle/python/protocols.qll
View File

@@ -1,31 +1,19 @@
import python
/** Retained for backwards compatibility use ClassObject.isIterator() instead. */
predicate is_iterator(ClassObject c) {
c.isIterator()
}
predicate is_iterator(ClassObject c) { c.isIterator() }
/** Retained for backwards compatibility use ClassObject.isIterable() instead. */
predicate is_iterable(ClassObject c) {
c.isIterable()
}
predicate is_iterable(ClassObject c) { c.isIterable() }
/** Retained for backwards compatibility use ClassObject.isCollection() instead. */
predicate is_collection(ClassObject c) {
c.isCollection()
}
predicate is_collection(ClassObject c) { c.isCollection() }
/** Retained for backwards compatibility use ClassObject.isMapping() instead. */
predicate is_mapping(ClassObject c) {
c.isMapping()
}
predicate is_mapping(ClassObject c) { c.isMapping() }
/** Retained for backwards compatibility use ClassObject.isSequence() instead. */
predicate is_sequence(ClassObject c) {
c.isSequence()
}
predicate is_sequence(ClassObject c) { c.isSequence() }
/** Retained for backwards compatibility use ClassObject.isContextManager() instead. */
predicate is_context_manager(ClassObject c) {
c.isContextManager()
}
predicate is_context_manager(ClassObject c) { c.isContextManager() }

416
python/ql/src/semmle/python/regex.qll
View File

@@ -2,13 +2,20 @@ import python
import semmle.python.objects.ObjectInternal
private predicate re_module_function(string name, int flags) {
name = "compile" and flags = 1 or
name = "search" and flags = 2 or
name = "match" and flags = 2 or
name = "split" and flags = 3 or
name = "findall" and flags = 2 or
name = "finditer" and flags = 2 or
name = "sub" and flags = 4 or
name = "compile" and flags = 1
or
name = "search" and flags = 2
or
name = "match" and flags = 2
or
name = "split" and flags = 3
or
name = "findall" and flags = 2
or
name = "finditer" and flags = 2
or
name = "sub" and flags = 4
or
name = "subn" and flags = 4
}
@@ -17,16 +24,15 @@ private predicate re_module_function(string name, int flags) {
* If regex mode is not known, `mode` will be `"None"`.
*/
predicate used_as_regex(Expr s, string mode) {
(s instanceof Bytes or s instanceof Unicode)
and
(s instanceof Bytes or s instanceof Unicode) and
/* Call to re.xxx(regex, ... [mode]) */
exists(CallNode call, string name |
call.getArg(0).refersTo(_, _, s.getAFlowNode()) and
call.getFunction().pointsTo(Module::named("re").attr(name)) |
call.getFunction().pointsTo(Module::named("re").attr(name))
|
mode = "None"
or
exists(Value obj |
mode = mode_from_mode_object(obj) |
exists(Value obj | mode = mode_from_mode_object(obj) |
exists(int flags_arg |
re_module_function(name, flags_arg) and
call.getArg(flags_arg).pointsTo(obj)
@@ -39,30 +45,30 @@ predicate used_as_regex(Expr s, string mode) {
string mode_from_mode_object(Value obj) {
(
result = "DEBUG" or result = "IGNORECASE" or result = "LOCALE" or
result = "MULTILINE" or result = "DOTALL" or result = "UNICODE" or
result = "DEBUG" or
result = "IGNORECASE" or
result = "LOCALE" or
result = "MULTILINE" or
result = "DOTALL" or
result = "UNICODE" or
result = "VERBOSE"
) and
exists(int flag |
flag = Value::named("sre_constants.SRE_FLAG_" + result).(ObjectInternal).intValue()
and
flag = Value::named("sre_constants.SRE_FLAG_" + result).(ObjectInternal).intValue() and
obj.(ObjectInternal).intValue().bitAnd(flag) = flag
)
}
/** A StrConst used as a regular expression */
abstract class RegexString extends Expr {
RegexString() {
(this instanceof Bytes or this instanceof Unicode)
}
RegexString() { (this instanceof Bytes or this instanceof Unicode) }
predicate char_set_start(int start, int end) {
this.nonEscapedCharAt(start) = "[" and
(
this.getChar(start+1) = "^" and end = start + 2
or
not this.getChar(start+1) = "^" and end = start + 1
this.getChar(start + 1) = "^" and end = start + 2
or
not this.getChar(start + 1) = "^" and end = start + 1
)
}
@@ -70,58 +76,47 @@ abstract class RegexString extends Expr {
predicate charSet(int start, int end) {
exists(int inner_start, int inner_end |
this.char_set_start(start, inner_start) and
not this.char_set_start(_, start) |
end = inner_end + 1 and inner_end > inner_start and
not this.char_set_start(_, start)
|
end = inner_end + 1 and
inner_end > inner_start and
this.nonEscapedCharAt(inner_end) = "]" and
not exists(int mid | this.nonEscapedCharAt(mid) = "]" |
mid > inner_start and mid < inner_end
)
not exists(int mid | this.nonEscapedCharAt(mid) = "]" | mid > inner_start and mid < inner_end)
)
}
predicate escapingChar(int pos) {
this.escaping(pos) = true
}
predicate escapingChar(int pos) { this.escaping(pos) = true }
private boolean escaping(int pos) {
pos = -1 and result = false
or
this.getChar(pos) = "\\" and result = this.escaping(pos-1).booleanNot()
this.getChar(pos) = "\\" and result = this.escaping(pos - 1).booleanNot()
or
this.getChar(pos) != "\\" and result = false
}
/** Gets the text of this regex */
string getText() {
result = ((Unicode)this).getS()
result = this.(Unicode).getS()
or
result = ((Bytes)this).getS()
result = this.(Bytes).getS()
}
string getChar(int i) {
result = this.getText().charAt(i)
}
string getChar(int i) { result = this.getText().charAt(i) }
string nonEscapedCharAt(int i) {
result = this.getText().charAt(i) and
not this.escapingChar(i-1)
not this.escapingChar(i - 1)
}
private predicate isOptionDivider(int i) {
this.nonEscapedCharAt(i) = "|"
}
private predicate isOptionDivider(int i) { this.nonEscapedCharAt(i) = "|" }
private predicate isGroupEnd(int i) {
this.nonEscapedCharAt(i) = ")"
}
private predicate isGroupEnd(int i) { this.nonEscapedCharAt(i) = ")" }
private predicate isGroupStart(int i) {
this.nonEscapedCharAt(i) = "("
}
private predicate isGroupStart(int i) { this.nonEscapedCharAt(i) = "(" }
predicate failedToParse(int i) {
exists(this.getChar(i))
and
exists(this.getChar(i)) and
not exists(int start, int end |
this.top_level(start, end) and
start <= i and
@@ -130,42 +125,47 @@ abstract class RegexString extends Expr {
}
private predicate escapedCharacter(int start, int end) {
this.escapingChar(start) and not exists(this.getText().substring(start+1, end+1).toInt()) and
this.escapingChar(start) and
not exists(this.getText().substring(start + 1, end + 1).toInt()) and
(
this.getChar(start+1) = "x" and end = start + 4
this.getChar(start + 1) = "x" and end = start + 4
or
end in [start+2..start+4] and
exists(this.getText().substring(start+1, end).toInt())
end in [start + 2 .. start + 4] and
exists(this.getText().substring(start + 1, end).toInt())
or
this.getChar(start+1) != "x" and end = start + 2
this.getChar(start + 1) != "x" and end = start + 2
)
}
private predicate inCharSet(int index) {
exists(int x, int y | this.charSet(x, y) and index in [x+1 .. y-2])
exists(int x, int y | this.charSet(x, y) and index in [x + 1 .. y - 2])
}
/* 'simple' characters are any that don't alter the parsing of the regex.
/*
* 'simple' characters are any that don't alter the parsing of the regex.
*/
private predicate simpleCharacter(int start, int end) {
end = start+1 and
end = start + 1 and
not this.charSet(start, _) and
not this.charSet(_, start+1) and
exists(string c |
c = this.getChar(start) |
not this.charSet(_, start + 1) and
exists(string c | c = this.getChar(start) |
exists(int x, int y, int z |
this.charSet(x, z) and
this.char_set_start(x, y) |
this.charSet(x, z) and
this.char_set_start(x, y)
|
start = y
or
start = z-2
start = z - 2
or
start > y and start < z-2 and not c = "-"
start > y and start < z - 2 and not c = "-"
)
or
not this.inCharSet(start) and
not c = "(" and not c = "[" and
not c = ")" and not c = "|" and
not c = "(" and
not c = "[" and
not c = ")" and
not c = "|" and
not this.qualifier(start, _, _)
)
}
@@ -176,28 +176,20 @@ abstract class RegexString extends Expr {
not exists(int x, int y | this.escapedCharacter(x, y) and x <= start and y >= end)
or
this.escapedCharacter(start, end)
)
and
not exists(int x, int y |
this.group_start(x, y) and x <= start and y >= end
)
) and
not exists(int x, int y | this.group_start(x, y) and x <= start and y >= end)
}
predicate normalCharacter(int start, int end) {
this.character(start, end)
and
this.character(start, end) and
not this.specialCharacter(start, end, _)
}
predicate specialCharacter(int start, int end, string char) {
this.character(start, end)
and
end = start+1
and
char = this.getChar(start)
and
(char = "$" or char = "^" or char = ".")
and
this.character(start, end) and
end = start + 1 and
char = this.getChar(start) and
(char = "$" or char = "^" or char = ".") and
not this.inCharSet(start)
}
@@ -210,17 +202,17 @@ abstract class RegexString extends Expr {
/** Gets the number of the group in start,end */
int getGroupNumber(int start, int end) {
this.group(start, end) and
result = count(int i | this.group(i, _) and i < start and not this.non_capturing_group_start(i, _)) + 1
this.group(start, end) and
result =
count(int i | this.group(i, _) and i < start and not this.non_capturing_group_start(i, _)) + 1
}
/** Gets the name, if it has one, of the group in start,end */
string getGroupName(int start, int end) {
this.group(start, end)
and
this.group(start, end) and
exists(int name_end |
this.named_group_start(start, name_end) and
result = this.getText().substring(start+4, name_end-1)
result = this.getText().substring(start + 4, name_end - 1)
)
}
@@ -236,8 +228,7 @@ abstract class RegexString extends Expr {
}
private predicate emptyGroup(int start, int end) {
exists(int endm1 |
end = endm1+1 |
exists(int endm1 | end = endm1 + 1 |
this.group_start(start, endm1) and
this.isGroupEnd(endm1)
)
@@ -263,21 +254,20 @@ abstract class RegexString extends Expr {
exists(int in_start |
this.negative_lookahead_assertion_start(start, in_start)
or
this.negative_lookbehind_assertion_start(start, in_start) |
this.negative_lookbehind_assertion_start(start, in_start)
|
this.groupContents(start, end, in_start, _)
)
}
private predicate positiveLookaheadAssertionGroup(int start, int end) {
exists(int in_start |
this.lookahead_assertion_start(start, in_start) |
exists(int in_start | this.lookahead_assertion_start(start, in_start) |
this.groupContents(start, end, in_start, _)
)
}
private predicate positiveLookbehindAssertionGroup(int start, int end) {
exists(int in_start |
this.lookbehind_assertion_start(start, in_start) |
exists(int in_start | this.lookbehind_assertion_start(start, in_start) |
this.groupContents(start, end, in_start, _)
)
}
@@ -306,42 +296,43 @@ abstract class RegexString extends Expr {
private predicate non_capturing_group_start(int start, int end) {
this.isGroupStart(start) and
this.getChar(start+1) = "?" and
this.getChar(start+2) = ":" and
end = start+3
this.getChar(start + 1) = "?" and
this.getChar(start + 2) = ":" and
end = start + 3
}
private predicate simple_group_start(int start, int end) {
this.isGroupStart(start) and
this.getChar(start+1) != "?" and end = start+1
this.getChar(start + 1) != "?" and
end = start + 1
}
private predicate named_group_start(int start, int end) {
this.isGroupStart(start) and
this.getChar(start+1) = "?" and
this.getChar(start+2) = "P" and
this.getChar(start+3) = "<" and
not this.getChar(start+4) = "=" and
not this.getChar(start+4) = "!" and
this.getChar(start + 1) = "?" and
this.getChar(start + 2) = "P" and
this.getChar(start + 3) = "<" and
not this.getChar(start + 4) = "=" and
not this.getChar(start + 4) = "!" and
exists(int name_end |
name_end = min(int i | i > start+4 and this.getChar(i) = ">") and
name_end = min(int i | i > start + 4 and this.getChar(i) = ">") and
end = name_end + 1
)
}
private predicate named_backreference_start(int start, int end) {
this.isGroupStart(start) and
this.getChar(start+1) = "?" and
this.getChar(start+2) = "P" and
this.getChar(start+3) = "=" and
end = min(int i | i > start+4 and this.getChar(i) = "?")
this.getChar(start + 1) = "?" and
this.getChar(start + 2) = "P" and
this.getChar(start + 3) = "=" and
end = min(int i | i > start + 4 and this.getChar(i) = "?")
}
private predicate flag_group_start(int start, int end, string c) {
this.isGroupStart(start) and
this.getChar(start+1) = "?" and
end = start+3 and
c = this.getChar(start+2) and
this.getChar(start + 1) = "?" and
end = start + 3 and
c = this.getChar(start + 2) and
(
c = "i" or
c = "L" or
@@ -352,12 +343,12 @@ abstract class RegexString extends Expr {
)
}
/** Gets the mode of this regular expression string if
/**
* Gets the mode of this regular expression string if
* it is defined by a prefix.
*/
string getModeFromPrefix() {
exists(string c |
this.flag_group_start(_, _, c) |
exists(string c | this.flag_group_start(_, _, c) |
c = "i" and result = "IGNORECASE"
or
c = "L" and result = "LOCALE"
@@ -366,7 +357,7 @@ abstract class RegexString extends Expr {
or
c = "s" and result = "DOTALL"
or
c = "u" and result = "UNICODE"
c = "u" and result = "UNICODE"
or
c = "x" and result = "VERBOSE"
)
@@ -374,39 +365,39 @@ abstract class RegexString extends Expr {
private predicate lookahead_assertion_start(int start, int end) {
this.isGroupStart(start) and
this.getChar(start+1) = "?" and
this.getChar(start+2) = "=" and
end = start+3
this.getChar(start + 1) = "?" and
this.getChar(start + 2) = "=" and
end = start + 3
}
private predicate negative_lookahead_assertion_start(int start, int end) {
this.isGroupStart(start) and
this.getChar(start+1) = "?" and
this.getChar(start+2) = "!" and
end = start+3
this.getChar(start + 1) = "?" and
this.getChar(start + 2) = "!" and
end = start + 3
}
private predicate lookbehind_assertion_start(int start, int end) {
this.isGroupStart(start) and
this.getChar(start+1) = "?" and
this.getChar(start+2) = "<" and
this.getChar(start+3) = "=" and
end = start+4
this.getChar(start + 1) = "?" and
this.getChar(start + 2) = "<" and
this.getChar(start + 3) = "=" and
end = start + 4
}
private predicate negative_lookbehind_assertion_start(int start, int end) {
this.isGroupStart(start) and
this.getChar(start+1) = "?" and
this.getChar(start+2) = "<" and
this.getChar(start+3) = "!" and
end = start+4
this.getChar(start + 1) = "?" and
this.getChar(start + 2) = "<" and
this.getChar(start + 3) = "!" and
end = start + 4
}
private predicate comment_group_start(int start, int end) {
this.isGroupStart(start) and
this.getChar(start+1) = "?" and
this.getChar(start+2) = "#" and
end = start+3
this.getChar(start + 1) = "?" and
this.getChar(start + 2) = "#" and
end = start + 3
}
predicate groupContents(int start, int end, int in_start, int in_end) {
@@ -417,20 +408,21 @@ abstract class RegexString extends Expr {
}
private predicate named_backreference(int start, int end, string name) {
this.named_backreference_start(start, start+4) and
end = min(int i | i > start+4 and this.getChar(i) = ")") + 1 and
name = this.getText().substring(start+4, end-2)
this.named_backreference_start(start, start + 4) and
end = min(int i | i > start + 4 and this.getChar(i) = ")") + 1 and
name = this.getText().substring(start + 4, end - 2)
}
private predicate numbered_backreference(int start, int end, int value) {
this.escapingChar(start)
and
this.escapingChar(start) and
exists(string text, string svalue, int len |
end = start + len and
text = this.getText() and len in [2..3] |
svalue = text.substring(start+1, start+len) and
text = this.getText() and
len in [2 .. 3]
|
svalue = text.substring(start + 1, start + len) and
value = svalue.toInt() and
not exists(text.substring(start+1, start+len+1).toInt()) and
not exists(text.substring(start + 1, start + len + 1).toInt()) and
value != 0
)
}
@@ -443,17 +435,14 @@ abstract class RegexString extends Expr {
}
/** Gets the number of the back reference in start,end */
int getBackrefNumber(int start, int end) {
this.numbered_backreference(start, end, result)
}
int getBackrefNumber(int start, int end) { this.numbered_backreference(start, end, result) }
/** Gets the name, if it has one, of the back reference in start,end */
string getBackrefName(int start, int end) {
this.named_backreference(start, end, result)
}
string getBackrefName(int start, int end) { this.named_backreference(start, end, result) }
private predicate baseItem(int start, int end) {
this.character(start, end) and not exists(int x, int y | this.charSet(x, y) and x <= start and y >= end)
this.character(start, end) and
not exists(int x, int y | this.charSet(x, y) and x <= start and y >= end)
or
this.group(start, end)
or
@@ -463,12 +452,8 @@ abstract class RegexString extends Expr {
private predicate qualifier(int start, int end, boolean maybe_empty) {
this.short_qualifier(start, end, maybe_empty) and not this.getChar(end) = "?"
or
exists(int short_end |
this.short_qualifier(start, short_end, maybe_empty) |
if this.getChar(short_end) = "?" then
end = short_end+1
else
end = short_end
exists(int short_end | this.short_qualifier(start, short_end, maybe_empty) |
if this.getChar(short_end) = "?" then end = short_end + 1 else end = short_end
)
}
@@ -479,26 +464,28 @@ abstract class RegexString extends Expr {
this.getChar(start) = "*" and maybe_empty = true
or
this.getChar(start) = "?" and maybe_empty = true
) and end = start + 1
) and
end = start + 1
or
exists(int endin | end = endin + 1 |
this.getChar(start) = "{" and this.getChar(endin) = "}" and
this.getChar(start) = "{" and
this.getChar(endin) = "}" and
end > start and
exists(string multiples |
multiples = this.getText().substring(start+1, endin) |
exists(string multiples | multiples = this.getText().substring(start + 1, endin) |
multiples.regexpMatch("0*,[0-9]*") and maybe_empty = true
or
multiples.regexpMatch("0*[1-9][0-9]*,[0-9]*") and maybe_empty = false
)
and
) and
not exists(int mid |
this.getChar(mid) = "}" and
mid > start and mid < endin
mid > start and
mid < endin
)
)
}
/** Whether the text in the range start,end is a qualified item, where item is a character,
/**
* Whether the text in the range start,end is a qualified item, where item is a character,
* a character set or a group.
*/
predicate qualifiedItem(int start, int end, boolean maybe_empty) {
@@ -520,19 +507,18 @@ abstract class RegexString extends Expr {
(
start = 0 or
this.group_start(_, start) or
this.isOptionDivider(start-1)
)
and
this.item(start, end) or
(
exists(int mid |
this.subsequence(start, mid) and
this.item(mid, end)
)
this.isOptionDivider(start - 1)
) and
this.item(start, end)
or
exists(int mid |
this.subsequence(start, mid) and
this.item(mid, end)
)
}
/** Whether the text in the range start,end is a sequence of 1 or more items, where an item is a character,
/**
* Whether the text in the range start,end is a sequence of 1 or more items, where an item is a character,
* a character set or a group.
*/
predicate sequence(int start, int end) {
@@ -552,9 +538,12 @@ abstract class RegexString extends Expr {
}
private predicate item_end(int end) {
this.character(_, end) or
exists(int endm1 | this.isGroupEnd(endm1) and end = endm1 + 1) or
this.charSet(_, end) or
this.character(_, end)
or
exists(int endm1 | this.isGroupEnd(endm1) and end = endm1 + 1)
or
this.charSet(_, end)
or
this.qualifier(_, end, _)
}
@@ -564,30 +553,36 @@ abstract class RegexString extends Expr {
}
private predicate subalternation(int start, int end, int item_start) {
this.sequenceOrQualified(start, end) and not this.isOptionDivider(start-1) and
this.sequenceOrQualified(start, end) and
not this.isOptionDivider(start - 1) and
item_start = start
or
start = end and not this.item_end(start) and this.isOptionDivider(end) and
start = end and
not this.item_end(start) and
this.isOptionDivider(end) and
item_start = start
or
exists(int mid |
this.subalternation(start, mid, _) and
this.isOptionDivider(mid) and
item_start = mid+1 |
item_start = mid + 1
|
this.sequenceOrQualified(item_start, end)
or
not this.item_start(end) and end = item_start
)
}
/** Whether the text in the range start,end is an alternation
/**
* Whether the text in the range start,end is an alternation
*/
predicate alternation(int start, int end) {
this.top_level(start, end) and
exists(int less | this.subalternation(start, less, _) and less < end)
}
/** Whether the text in the range start,end is an alternation and the text in part_start, part_end is one of the
/**
* Whether the text in the range start,end is an alternation and the text in part_start, part_end is one of the
* options in that alternation.
*/
predicate alternationOption(int start, int end, int part_start, int part_end) {
@@ -599,22 +594,19 @@ abstract class RegexString extends Expr {
private predicate firstPart(int start, int end) {
start = 0 and end = this.getText().length()
or
exists(int x |
this.firstPart(x, end) |
exists(int x | this.firstPart(x, end) |
this.emptyMatchAtStartGroup(x, start) or
this.qualifiedItem(x, start, true) or
this.specialCharacter(x, start, "^")
)
or
exists(int y |
this.firstPart(start, y) |
exists(int y | this.firstPart(start, y) |
this.item(start, end)
or
this.qualifiedPart(start, end, y, _)
)
or
exists(int x, int y |
this.firstPart(x, y) |
exists(int x, int y | this.firstPart(x, y) |
this.groupContents(x, y, start, end)
or
this.alternationOption(x, y, start, end)
@@ -625,8 +617,7 @@ abstract class RegexString extends Expr {
private predicate lastPart(int start, int end) {
start = 0 and end = this.getText().length()
or
exists(int y |
this.lastPart(start, y) |
exists(int y | this.lastPart(start, y) |
this.emptyMatchAtEndGroup(end, y) or
this.qualifiedItem(end, y, true) or
this.specialCharacter(end, y, "$")
@@ -637,60 +628,52 @@ abstract class RegexString extends Expr {
this.item(start, end)
)
or
exists(int y |
this.lastPart(start, y) |
this.qualifiedPart(start, end, y, _)
)
exists(int y | this.lastPart(start, y) | this.qualifiedPart(start, end, y, _))
or
exists(int x, int y |
this.lastPart(x, y) |
exists(int x, int y | this.lastPart(x, y) |
this.groupContents(x, y, start, end)
or
this.alternationOption(x, y, start, end)
)
}
/** Whether the item at [start, end) is one of the first items
/**
* Whether the item at [start, end) is one of the first items
* to be matched.
*/
predicate firstItem(int start, int end) {
(
this.character(start, end)
or
this.qualifiedItem(start, end, _)
or
this.charSet(start, end)
)
and
this.character(start, end)
or
this.qualifiedItem(start, end, _)
or
this.charSet(start, end)
) and
this.firstPart(start, end)
}
/** Whether the item at [start, end) is one of the last items
/**
* Whether the item at [start, end) is one of the last items
* to be matched.
*/
predicate lastItem(int start, int end) {
(
this.character(start, end)
or
this.qualifiedItem(start, end, _)
or
this.charSet(start, end)
)
and
this.character(start, end)
or
this.qualifiedItem(start, end, _)
or
this.charSet(start, end)
) and
this.lastPart(start, end)
}
}
/** A StrConst used as a regular expression */
class Regex extends RegexString {
Regex() { used_as_regex(this, _) }
Regex() {
used_as_regex(this, _)
}
/** Gets a mode (if any) of this regular expression. Can be any of:
/**
* Gets a mode (if any) of this regular expression. Can be any of:
* DEBUG
* IGNORECASE
* LOCALE
@@ -705,5 +688,4 @@ class Regex extends RegexString {
or
result = this.getModeFromPrefix()
}
}
}

22
python/ql/src/semmle/python/security/ClearText.qll
View File

@@ -5,17 +5,12 @@ import semmle.python.dataflow.Files
import semmle.python.web.Http
module ClearTextStorage {
abstract class Sink extends TaintSink {
override predicate sinks(TaintKind kind) {
kind instanceof SensitiveData
}
override predicate sinks(TaintKind kind) { kind instanceof SensitiveData }
}
class CookieStorageSink extends Sink {
CookieStorageSink() {
any(CookieSet cookie).getValue() = this
}
CookieStorageSink() { any(CookieSet cookie).getValue() = this }
}
class FileStorageSink extends Sink {
@@ -23,20 +18,17 @@ module ClearTextStorage {
exists(CallNode call, AttrNode meth, string name |
any(OpenFile fd).taints(meth.getObject(name)) and
call.getFunction() = meth and
call.getAnArg() = this |
call.getAnArg() = this
|
name = "write"
)
}
}
}
module ClearTextLogging {
abstract class Sink extends TaintSink {
override predicate sinks(TaintKind kind) {
kind instanceof SensitiveData
}
override predicate sinks(TaintKind kind) { kind instanceof SensitiveData }
}
class PrintSink extends Sink {
@@ -53,7 +45,8 @@ module ClearTextLogging {
exists(CallNode call, AttrNode meth, string name |
call.getFunction() = meth and
meth.getObject(name).(NameNode).getId().matches("logg%") and
call.getAnArg() = this |
call.getAnArg() = this
|
name = "error" or
name = "warn" or
name = "warning" or
@@ -62,5 +55,4 @@ module ClearTextLogging {
)
}
}
}

104
python/ql/src/semmle/python/security/Crypto.qll
View File

@@ -1,49 +1,32 @@
import python
import semmle.python.security.TaintTracking
private import semmle.python.security.SensitiveData
private import semmle.crypto.Crypto as CryptoLib
abstract class WeakCryptoSink extends TaintSink {
override predicate sinks(TaintKind taint) {
taint instanceof SensitiveData
}
override predicate sinks(TaintKind taint) { taint instanceof SensitiveData }
}
/** Modeling the 'pycrypto' package https://github.com/dlitz/pycrypto (latest release 2013) */
module Pycrypto {
ModuleValue cipher(string name) {
result = Module::named("Crypto.Cipher").attr(name)
}
ModuleValue cipher(string name) { result = Module::named("Crypto.Cipher").attr(name) }
class CipherInstance extends TaintKind {
string name;
CipherInstance() {
this = "Crypto.Cipher." + name and
this = "Crypto.Cipher." + name and
exists(cipher(name))
}
string getName() {
result = name
}
string getName() { result = name }
CryptoLib::CryptographicAlgorithm getAlgorithm() {
result.getName() = name
}
predicate isWeak() {
this.getAlgorithm().isWeak()
}
CryptoLib::CryptographicAlgorithm getAlgorithm() { result.getName() = name }
predicate isWeak() { this.getAlgorithm().isWeak() }
}
class CipherInstanceSource extends TaintSource {
CipherInstance instance;
CipherInstanceSource() {
@@ -53,18 +36,12 @@ module Pycrypto {
)
}
override string toString() {
result = "Source of " + instance
}
override predicate isSourceOf(TaintKind kind) {
kind = instance
}
override string toString() { result = "Source of " + instance }
override predicate isSourceOf(TaintKind kind) { kind = instance }
}
class PycryptoWeakCryptoSink extends WeakCryptoSink {
string name;
PycryptoWeakCryptoSink() {
@@ -77,36 +54,24 @@ module Pycrypto {
)
}
override string toString() {
result = "Use of weak crypto algorithm " + name
}
override string toString() { result = "Use of weak crypto algorithm " + name }
}
}
module Cryptography {
ModuleValue ciphers() {
result = Module::named("cryptography.hazmat.primitives.ciphers") and
result.isPackage()
}
class CipherClass extends ClassValue {
CipherClass() {
ciphers().attr("Cipher") = this
}
CipherClass() { ciphers().attr("Cipher") = this }
}
class AlgorithmClass extends ClassValue {
AlgorithmClass() { ciphers().attr("algorithms").attr(_) = this }
AlgorithmClass() {
ciphers().attr("algorithms").attr(_) = this
}
string getAlgorithmName() {
result = this.declaredAttribute("name").(StringValue).getText()
}
string getAlgorithmName() { result = this.declaredAttribute("name").(StringValue).getText() }
predicate isWeak() {
exists(CryptoLib::CryptographicAlgorithm algo |
@@ -117,61 +82,39 @@ module Cryptography {
}
class CipherInstance extends TaintKind {
AlgorithmClass cls;
CipherInstance() {
this = "cryptography.Cipher." + cls.getAlgorithmName()
}
CipherInstance() { this = "cryptography.Cipher." + cls.getAlgorithmName() }
AlgorithmClass getAlgorithm() {
result = cls
}
AlgorithmClass getAlgorithm() { result = cls }
predicate isWeak() {
cls.isWeak()
}
predicate isWeak() { cls.isWeak() }
override TaintKind getTaintOfMethodResult(string name) {
name = "encryptor" and
result.(Encryptor).getAlgorithm() = this.getAlgorithm()
}
}
class CipherSource extends TaintSource {
CipherSource() {
this.(CallNode).getFunction().pointsTo(any(CipherClass cls))
}
CipherSource() { this.(CallNode).getFunction().pointsTo(any(CipherClass cls)) }
override predicate isSourceOf(TaintKind kind) {
this.(CallNode).getArg(0).pointsTo().getClass() = kind.(CipherInstance).getAlgorithm()
}
override string toString() {
result = "cryptography.Cipher.source"
}
override string toString() { result = "cryptography.Cipher.source" }
}
class Encryptor extends TaintKind {
AlgorithmClass cls;
Encryptor() {
this = "cryptography.encryptor." + cls.getAlgorithmName()
}
AlgorithmClass getAlgorithm() {
result = cls
}
Encryptor() { this = "cryptography.encryptor." + cls.getAlgorithmName() }
AlgorithmClass getAlgorithm() { result = cls }
}
class CryptographyWeakCryptoSink extends WeakCryptoSink {
CryptographyWeakCryptoSink() {
exists(CallNode call, AttrNode method, Encryptor encryptor |
call.getAnArg() = this and
@@ -181,17 +124,11 @@ module Cryptography {
)
}
override string toString() {
result = "Use of weak crypto algorithm"
}
override string toString() { result = "Use of weak crypto algorithm" }
}
}
private class CipherConfig extends TaintTracking::Configuration {
CipherConfig() { this = "Crypto cipher config" }
override predicate isSource(TaintTracking::Source source) {
@@ -199,5 +136,4 @@ private class CipherConfig extends TaintTracking::Configuration {
or
source instanceof Cryptography::CipherSource
}
}

64
python/ql/src/semmle/python/security/Exceptions.qll
View File

@@ -7,43 +7,31 @@ import python
import semmle.python.security.TaintTracking
import semmle.python.security.strings.Basic
private Value traceback_function(string name) {
result = Module::named("traceback").attr(name)
}
private Value traceback_function(string name) { result = Module::named("traceback").attr(name) }
/**
* This represents information relating to an exception, for instance the
* message, arguments or parts of the exception traceback.
*/
class ExceptionInfo extends StringKind {
ExceptionInfo() { this = "exception.info" }
ExceptionInfo() {
this = "exception.info"
}
override string repr() {
result = "exception info"
}
override string repr() { result = "exception info" }
}
/** A class representing sources of information about
/**
* A class representing sources of information about
* execution state exposed in tracebacks and the like.
*/
abstract class ErrorInfoSource extends TaintSource {}
abstract class ErrorInfoSource extends TaintSource { }
/**
* This kind represents exceptions themselves.
*/
class ExceptionKind extends TaintKind {
ExceptionKind() { this = "exception.kind" }
ExceptionKind() {
this = "exception.kind"
}
override string repr() {
result = "exception"
}
override string repr() { result = "exception" }
override TaintKind getTaintOfAttribute(string name) {
name = "args" and result instanceof ExceptionInfoSequence
@@ -57,7 +45,6 @@ class ExceptionKind extends TaintKind {
* `except` statement.
*/
class ExceptionSource extends ErrorInfoSource {
ExceptionSource() {
exists(ClassValue cls |
cls.getASuperType() = ClassValue::baseException() and
@@ -67,13 +54,9 @@ class ExceptionSource extends ErrorInfoSource {
this = any(ExceptStmt s).getName().getAFlowNode()
}
override string toString() {
result = "exception.source"
}
override string toString() { result = "exception.source" }
override predicate isSourceOf(TaintKind kind) {
kind instanceof ExceptionKind
}
override predicate isSourceOf(TaintKind kind) { kind instanceof ExceptionKind }
}
/**
@@ -81,18 +64,14 @@ class ExceptionSource extends ErrorInfoSource {
* for instance the contents of the `args` attribute, or the stack trace.
*/
class ExceptionInfoSequence extends SequenceKind {
ExceptionInfoSequence() {
this.getItem() instanceof ExceptionInfo
}
ExceptionInfoSequence() { this.getItem() instanceof ExceptionInfo }
}
/**
* Represents calls to functions in the `traceback` module that return
* sequences of exception information.
*/
class CallToTracebackFunction extends ErrorInfoSource {
CallToTracebackFunction() {
exists(string name |
name = "extract_tb" or
@@ -107,13 +86,9 @@ class CallToTracebackFunction extends ErrorInfoSource {
)
}
override string toString() {
result = "exception.info.sequence.source"
}
override string toString() { result = "exception.info.sequence.source" }
override predicate isSourceOf(TaintKind kind) {
kind instanceof ExceptionInfoSequence
}
override predicate isSourceOf(TaintKind kind) { kind instanceof ExceptionInfoSequence }
}
/**
@@ -121,16 +96,9 @@ class CallToTracebackFunction extends ErrorInfoSource {
* string of information about an exception.
*/
class FormattedTracebackSource extends ErrorInfoSource {
FormattedTracebackSource() { this = traceback_function("format_exc").getACall() }
FormattedTracebackSource() {
this = traceback_function("format_exc").getACall()
}
override string toString() { result = "exception.info.source" }
override string toString() {
result = "exception.info.source"
}
override predicate isSourceOf(TaintKind kind) {
kind instanceof ExceptionInfo
}
override predicate isSourceOf(TaintKind kind) { kind instanceof ExceptionInfo }
}

4
python/ql/src/semmle/python/security/Paths.qll
View File

@@ -1,8 +1,6 @@
import semmle.python.dataflow.Implementation
module TaintTrackingPaths {
predicate edge(TaintTrackingNode src, TaintTrackingNode dest, string label) {
exists(TaintTrackingNode source, TaintTrackingNode sink |
source.getConfiguration().hasFlowPath(source, sink) and
@@ -11,10 +9,8 @@ module TaintTrackingPaths {
dest.getASuccessor*() = sink
)
}
}
query predicate edges(TaintTrackingNode fromnode, TaintTrackingNode tonode) {
TaintTrackingPaths::edge(fromnode, tonode, _)
}

2
python/ql/src/semmle/python/security/SQL.qll
View File

@@ -1,4 +1,4 @@
import python
import semmle.python.security.TaintTracking
abstract class SqlInjectionSink extends TaintSink {}
abstract class SqlInjectionSink extends TaintSink { }

57
python/ql/src/semmle/python/security/SensitiveData.qll
View File

@@ -20,7 +20,6 @@ import semmle.python.web.HttpRequest
* This is copied from the javascript library, but should be language independent.
*/
private module HeuristicNames {
/**
* Gets a regular expression that identifies strings that may indicate the presence of secret
* or trusted data.
@@ -32,8 +31,8 @@ private module HeuristicNames {
* user names or other account information.
*/
string maybeAccountInfo() {
result = "(?is).*acc(ou)?nt.*" or
result = "(?is).*(puid|username|userid).*"
result = "(?is).*acc(ou)?nt.*" or
result = "(?is).*(puid|username|userid).*"
}
/**
@@ -41,8 +40,8 @@ private module HeuristicNames {
* a password or an authorization key.
*/
string maybePassword() {
result = "(?is).*pass(wd|word|code|phrase)(?!.*question).*" or
result = "(?is).*(auth(entication|ori[sz]ation)?)key.*"
result = "(?is).*pass(wd|word|code|phrase)(?!.*question).*" or
result = "(?is).*(auth(entication|ori[sz]ation)?)key.*"
}
/**
@@ -51,7 +50,7 @@ private module HeuristicNames {
*/
string maybeCertificate() { result = "(?is).*(cert)(?!.*(format|name)).*" }
/**
/**
* Gets a regular expression that identifies strings that may indicate the presence
* of sensitive data, with `classification` describing the kind of sensitive data involved.
*/
@@ -78,35 +77,35 @@ private module HeuristicNames {
name.regexpMatch(HeuristicNames::maybeSensitive(result)) and
not name.regexpMatch(HeuristicNames::notSensitive())
}
}
abstract class SensitiveData extends TaintKind {
bindingset[this]
SensitiveData() { this = this }
}
module SensitiveData {
class Secret extends SensitiveData {
Secret() { this = "sensitive.data.secret" }
override string repr() { result = "a secret" }
}
class Id extends SensitiveData {
Id() { this = "sensitive.data.id" }
override string repr() { result = "an ID" }
}
class Password extends SensitiveData {
Password() { this = "sensitive.data.password" }
override string repr() { result = "a password" }
}
class Certificate extends SensitiveData {
Certificate() { this = "sensitive.data.certificate" }
override string repr() { result = "a certificate or key" }
}
@@ -115,53 +114,35 @@ module SensitiveData {
}
abstract class Source extends TaintSource {
abstract string repr();
}
private class SensitiveCallSource extends Source {
SensitiveData data;
SensitiveCallSource() {
exists(Value callee |
callee.getACall() = this |
data = fromFunction(callee)
)
exists(Value callee | callee.getACall() = this | data = fromFunction(callee))
}
override predicate isSourceOf(TaintKind kind) {
kind = data
}
override string repr() {
result = "a call returning " + data.repr()
}
override predicate isSourceOf(TaintKind kind) { kind = data }
override string repr() { result = "a call returning " + data.repr() }
}
/** An access to a variable or property that might contain sensitive data. */
private class SensitiveVariableAccess extends SensitiveData::Source {
SensitiveData data;
SensitiveVariableAccess() {
data = HeuristicNames::getSensitiveDataForName(this.(AttrNode).getName())
}
override predicate isSourceOf(TaintKind kind) {
kind = data
}
override string repr() {
result = "an attribute or property containing " + data.repr()
}
override predicate isSourceOf(TaintKind kind) { kind = data }
override string repr() { result = "an attribute or property containing " + data.repr() }
}
private class SensitiveRequestParameter extends SensitiveData::Source {
SensitiveData data;
SensitiveRequestParameter() {
@@ -172,16 +153,10 @@ module SensitiveData {
)
}
override predicate isSourceOf(TaintKind kind) {
kind = data
}
override string repr() {
result = "a request parameter containing " + data.repr()
}
override predicate isSourceOf(TaintKind kind) { kind = data }
override string repr() { result = "a request parameter containing " + data.repr() }
}
}
//Backwards compatibility

1
python/ql/src/semmle/python/security/TaintTracking.qll
View File

@@ -1,2 +1,3 @@
/** For backwards compatibility */
import semmle.python.dataflow.TaintTracking

10
python/ql/src/semmle/python/security/flow/AnyCall.qll
View File

@@ -3,13 +3,7 @@ import semmle.python.security.strings.Basic
/** Assume that taint flows from argument to result for *any* call */
class AnyCallStringFlow extends DataFlowExtension::DataFlowNode {
AnyCallStringFlow() { any(CallNode call).getAnArg() = this }
AnyCallStringFlow() {
any(CallNode call).getAnArg() = this
}
override ControlFlowNode getASuccessorNode() {
result.(CallNode).getAnArg() = this
}
override ControlFlowNode getASuccessorNode() { result.(CallNode).getAnArg() = this }
}

8
python/ql/src/semmle/python/security/injection/Deserialization.qll
View File

@@ -1,14 +1,8 @@
import python
import semmle.python.security.TaintTracking
/** `pickle.loads(untrusted)` vulnerability. */
abstract class DeserializationSink extends TaintSink {
bindingset[this]
DeserializationSink() {
this = this
}
DeserializationSink() { this = this }
}

20
python/ql/src/semmle/python/security/injection/Exec.qll
View File

@@ -1,33 +1,30 @@
/** Provides class and predicates to track external data that
/**
* Provides class and predicates to track external data that
* may represent malicious Python code.
*
* This module is intended to be imported into a taint-tracking query
* to extend `TaintKind` and `TaintSink`.
*
*/
import python
import python
import semmle.python.security.TaintTracking
import semmle.python.security.strings.Untrusted
private FunctionObject exec_or_eval() {
result = Object::builtin("exec")
or
result = Object::builtin("eval")
}
/** A taint sink that represents an argument to exec or eval that is vulnerable to malicious input.
/**
* A taint sink that represents an argument to exec or eval that is vulnerable to malicious input.
* The `vuln` in `exec(vuln)` or similar.
*/
class StringEvaluationNode extends TaintSink {
override string toString() { result = "exec or eval" }
StringEvaluationNode() {
exists(Exec exec |
exec.getASubExpression().getAFlowNode() = this
)
exists(Exec exec | exec.getASubExpression().getAFlowNode() = this)
or
exists(CallNode call |
exec_or_eval().getACall() = call and
@@ -35,8 +32,5 @@ class StringEvaluationNode extends TaintSink {
)
}
override predicate sinks(TaintKind kind) {
kind instanceof ExternalStringKind
}
override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind }
}

24
python/ql/src/semmle/python/security/injection/Marshal.qll
View File

@@ -1,26 +1,23 @@
/** Provides class and predicates to track external data that
/**
* Provides class and predicates to track external data that
* may represent malicious marshals.
*
* This module is intended to be imported into a taint-tracking query
* to extend `TaintKind` and `TaintSink`.
*
*/
import python
import python
import semmle.python.security.TaintTracking
import semmle.python.security.strings.Untrusted
import semmle.python.security.injection.Deserialization
private FunctionObject marshalLoads() { result = ModuleObject::named("marshal").attr("loads") }
private FunctionObject marshalLoads() {
result = ModuleObject::named("marshal").attr("loads")
}
/** A taint sink that is potentially vulnerable to malicious marshaled objects.
* The `vuln` in `marshal.loads(vuln)`. */
/**
* A taint sink that is potentially vulnerable to malicious marshaled objects.
* The `vuln` in `marshal.loads(vuln)`.
*/
class UnmarshalingNode extends DeserializationSink {
override string toString() { result = "unmarshaling vulnerability" }
UnmarshalingNode() {
@@ -30,8 +27,5 @@ class UnmarshalingNode extends DeserializationSink {
)
}
override predicate sinks(TaintKind kind) {
kind instanceof ExternalStringKind
}
override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind }
}

54
python/ql/src/semmle/python/security/injection/Path.qll
View File

@@ -1,28 +1,22 @@
import python
import semmle.python.security.TaintTracking
import semmle.python.security.strings.Untrusted
/** Prevents taint flowing through ntpath.normpath()
/**
* Prevents taint flowing through ntpath.normpath()
* NormalizedPath below handles that case.
*/
class PathSanitizer extends Sanitizer {
PathSanitizer() {
this = "path.sanitizer"
}
PathSanitizer() { this = "path.sanitizer" }
override predicate sanitizingNode(TaintKind taint, ControlFlowNode node) {
taint instanceof ExternalStringKind and
abspath_call(node, _)
}
}
private FunctionObject abspath() {
exists(ModuleObject os_path |
ModuleObject::named("os").attr("path") = os_path
|
exists(ModuleObject os_path | ModuleObject::named("os").attr("path") = os_path |
os_path.attr("abspath") = result
or
os_path.attr("normpath") = result
@@ -31,15 +25,9 @@ private FunctionObject abspath() {
/** A path that has been normalized, but not verified to be safe */
class NormalizedPath extends TaintKind {
NormalizedPath() { this = "normalized.path.injection" }
NormalizedPath() {
this = "normalized.path.injection"
}
override string repr() {
result = "normalized path"
}
override string repr() { result = "normalized path" }
}
private predicate abspath_call(CallNode call, ControlFlowNode arg) {
@@ -47,39 +35,31 @@ private predicate abspath_call(CallNode call, ControlFlowNode arg) {
arg = call.getArg(0)
}
class AbsPath extends DataFlowExtension::DataFlowNode {
AbsPath() { abspath_call(_, this) }
AbsPath() {
abspath_call(_, this)
override ControlFlowNode getASuccessorNode(TaintKind fromkind, TaintKind tokind) {
abspath_call(result, this) and
tokind instanceof NormalizedPath and
fromkind instanceof ExternalStringKind
}
override
ControlFlowNode getASuccessorNode(TaintKind fromkind, TaintKind tokind) {
abspath_call(result, this) and tokind instanceof NormalizedPath and fromkind instanceof ExternalStringKind
}
}
class NormalizedPathSanitizer extends Sanitizer {
NormalizedPathSanitizer() {
this = "normalized.path.sanitizer"
}
NormalizedPathSanitizer() { this = "normalized.path.sanitizer" }
override predicate sanitizingEdge(TaintKind taint, PyEdgeRefinement test) {
taint instanceof NormalizedPath and
test.getTest().(CallNode).getFunction().(AttrNode).getName() = "startswith" and
test.getSense() = true
}
}
/** A taint sink that is vulnerable to malicious paths.
/**
* A taint sink that is vulnerable to malicious paths.
* The `vuln` in `open(vuln)` and similar.
*/
class OpenNode extends TaintSink {
override string toString() { result = "argument to open()" }
OpenNode() {
@@ -94,10 +74,4 @@ class OpenNode extends TaintSink {
or
kind instanceof NormalizedPath
}
}

17
python/ql/src/semmle/python/security/injection/Pickle.qll
View File

@@ -1,17 +1,16 @@
/** Provides class and predicates to track external data that
/**
* Provides class and predicates to track external data that
* may represent malicious pickles.
*
* This module is intended to be imported into a taint-tracking query
* to extend `TaintKind` and `TaintSink`.
*
*/
import python
import python
import semmle.python.security.TaintTracking
import semmle.python.security.strings.Untrusted
import semmle.python.security.injection.Deserialization
private ModuleObject pickleModule() {
result.getName() = "pickle"
or
@@ -20,13 +19,10 @@ private ModuleObject pickleModule() {
result.getName() = "dill"
}
private FunctionObject pickleLoads() {
result = pickleModule().attr("loads")
}
private FunctionObject pickleLoads() { result = pickleModule().attr("loads") }
/** `pickle.loads(untrusted)` vulnerability. */
class UnpicklingNode extends DeserializationSink {
override string toString() { result = "unpickling untrusted data" }
UnpicklingNode() {
@@ -36,8 +32,5 @@ class UnpicklingNode extends DeserializationSink {
)
}
override predicate sinks(TaintKind kind) {
kind instanceof ExternalStringKind
}
override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind }
}

49
python/ql/src/semmle/python/security/injection/Sql.qll
View File

@@ -1,29 +1,27 @@
/** Provides class and predicates to track external data that
/**
* Provides class and predicates to track external data that
* may represent malicious SQL queries or parts of queries.
*
* This module is intended to be imported into a taint-tracking query
* to extend `TaintKind` and `TaintSink`.
*
*/
import python
import python
import semmle.python.security.TaintTracking
import semmle.python.security.strings.Untrusted
import semmle.python.security.SQL
private StringObject first_part(ControlFlowNode command) {
command.(BinaryExprNode).getOp() instanceof Add and
command.(BinaryExprNode).getLeft().refersTo(result)
or
exists(CallNode call, SequenceObject seq |
call = command |
exists(CallNode call, SequenceObject seq | call = command |
call = theStrType().lookupAttribute("join") and
call.getArg(0).refersTo(seq) and
seq.getInferredElement(0) = result
)
or
command.(BinaryExprNode).getOp() instanceof Mod and
command.(BinaryExprNode).getOp() instanceof Mod and
command.getNode().(StrConst).getLiteralObject() = result
}
@@ -32,52 +30,45 @@ predicate probable_sql_command(ControlFlowNode command, ControlFlowNode inject)
exists(string prefix |
inject = command.getAChild*() and
first_part(command).getText().regexpMatch(" *" + prefix + ".*")
|
|
prefix = "CREATE" or prefix = "SELECT"
)
}
/** A taint kind representing a DB cursor.
/**
* A taint kind representing a DB cursor.
* This will be overridden to provide specific kinds of DB cursor.
*/
abstract class DbCursor extends TaintKind {
bindingset[this]
DbCursor() { any() }
string getExecuteMethodName() { result = "execute" }
}
/** A part of a string that appears to be a SQL command and is thus
/**
* A part of a string that appears to be a SQL command and is thus
* vulnerable to malicious input.
*/
class SimpleSqlStringInjection extends SqlInjectionSink {
override string toString() { result = "simple SQL string injection" }
SimpleSqlStringInjection() {
probable_sql_command(_, this)
}
override predicate sinks(TaintKind kind) {
kind instanceof ExternalStringKind
}
SimpleSqlStringInjection() { probable_sql_command(_, this) }
override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind }
}
/** A taint source representing sources of DB connections.
/**
* A taint source representing sources of DB connections.
* This will be overridden to provide specific kinds of DB connection sources.
*/
abstract class DbConnectionSource extends TaintSource {
abstract class DbConnectionSource extends TaintSource { }
}
/** A taint sink that is vulnerable to malicious SQL queries.
/**
* A taint sink that is vulnerable to malicious SQL queries.
* The `vuln` in `db.connection.execute(vuln)` and similar.
*/
class DbConnectionExecuteArgument extends SqlInjectionSink {
override string toString() { result = "db.connection.execute" }
DbConnectionExecuteArgument() {
@@ -88,9 +79,5 @@ class DbConnectionExecuteArgument extends SqlInjectionSink {
)
}
override predicate sinks(TaintKind kind) {
kind instanceof ExternalStringKind
}
override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind }
}

55
python/ql/src/semmle/python/security/injection/Xml.qll
View File

@@ -1,36 +1,26 @@
/** Provides class and predicates to track external data that
/**
* Provides class and predicates to track external data that
* may represent malicious XML objects.
*
* This module is intended to be imported into a taint-tracking query
* to extend `TaintKind` and `TaintSink`.
*/
import python
import python
import semmle.python.security.TaintTracking
import semmle.python.security.strings.Untrusted
import semmle.python.security.injection.Deserialization
private ModuleObject xmlElementTreeModule() { result.getName() = "xml.etree.ElementTree" }
private ModuleObject xmlElementTreeModule() {
result.getName() = "xml.etree.ElementTree"
}
private ModuleObject xmlMiniDomModule() { result.getName() = "xml.dom.minidom" }
private ModuleObject xmlMiniDomModule() {
result.getName() = "xml.dom.minidom"
}
private ModuleObject xmlPullDomModule() { result.getName() = "xml.dom.pulldom" }
private ModuleObject xmlPullDomModule() {
result.getName() = "xml.dom.pulldom"
}
private ModuleObject xmlSaxModule() {
result.getName() = "xml.sax"
}
private ModuleObject xmlSaxModule() { result.getName() = "xml.sax" }
private class ExpatParser extends TaintKind {
ExpatParser() { this = "expat.parser" }
}
private FunctionObject expatCreateParseFunction() {
@@ -38,18 +28,11 @@ private FunctionObject expatCreateParseFunction() {
}
private class ExpatCreateParser extends TaintSource {
ExpatCreateParser() { expatCreateParseFunction().getACall() = this }
ExpatCreateParser() {
expatCreateParseFunction().getACall() = this
}
override predicate isSourceOf(TaintKind kind) { kind instanceof ExpatParser }
override predicate isSourceOf(TaintKind kind) {
kind instanceof ExpatParser
}
string toString() {
result = "expat.create.parser"
}
string toString() { result = "expat.create.parser" }
}
private FunctionObject xmlFromString() {
@@ -64,30 +47,22 @@ private FunctionObject xmlFromString() {
/** A (potentially) malicious XML string. */
class ExternalXmlString extends ExternalStringKind {
ExternalXmlString() {
this = "external xml encoded object"
}
ExternalXmlString() { this = "external xml encoded object" }
}
/** A call to an XML library function that is potentially vulnerable to a
/**
* A call to an XML library function that is potentially vulnerable to a
* specially crafted XML string.
*/
class XmlLoadNode extends DeserializationSink {
override string toString() { result = "xml.load vulnerability" }
XmlLoadNode() {
exists(CallNode call |
call.getAnArg() = this |
exists(CallNode call | call.getAnArg() = this |
xmlFromString().getACall() = call or
any(ExpatParser parser).taints(call.getFunction().(AttrNode).getObject("Parse"))
)
}
override predicate sinks(TaintKind kind) {
kind instanceof ExternalXmlString
}
override predicate sinks(TaintKind kind) { kind instanceof ExternalXmlString }
}

16
python/ql/src/semmle/python/security/injection/Yaml.qll
View File

@@ -1,25 +1,20 @@
/** Provides class and predicates to track external data that
/**
* Provides class and predicates to track external data that
* may represent malicious yaml-encoded objects.
*
* This module is intended to be imported into a taint-tracking query
* to extend `TaintKind` and `TaintSink`.
*
*/
import python
import semmle.python.security.TaintTracking
import semmle.python.security.strings.Untrusted
import semmle.python.security.injection.Deserialization
private FunctionObject yamlLoad() {
result = ModuleObject::named("yaml").attr("load")
}
private FunctionObject yamlLoad() { result = ModuleObject::named("yaml").attr("load") }
/** `yaml.load(untrusted)` vulnerability. */
class YamlLoadNode extends DeserializationSink {
override string toString() { result = "yaml.load vulnerability" }
YamlLoadNode() {
@@ -29,8 +24,5 @@ class YamlLoadNode extends DeserializationSink {
)
}
override predicate sinks(TaintKind kind) {
kind instanceof ExternalStringKind
}
override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind }
}

54
python/ql/src/semmle/python/security/strings/Basic.qll
View File

@@ -1,15 +1,11 @@
import python
private import Common
import semmle.python.security.TaintTracking
/** An extensible kind of taint representing any kind of string. */
abstract class StringKind extends TaintKind {
bindingset[this]
StringKind() {
this = this
}
StringKind() { this = this }
override TaintKind getTaintForFlowStep(ControlFlowNode fromnode, ControlFlowNode tonode) {
result = this and
@@ -27,43 +23,44 @@ abstract class StringKind extends TaintKind {
}
override ClassValue getType() {
result = Value::named("bytes") or result = Value::named("str") or result = Value::named("unicode")
result = Value::named("bytes") or
result = Value::named("str") or
result = Value::named("unicode")
}
}
private class StringEqualitySanitizer extends Sanitizer {
StringEqualitySanitizer() { this = "string equality sanitizer" }
/** The test `if untrusted == "KNOWN_VALUE":` sanitizes `untrusted` on its `true` edge. */
override predicate sanitizingEdge(TaintKind taint, PyEdgeRefinement test) {
taint instanceof StringKind and
exists(ControlFlowNode const, Cmpop op |
const.getNode() instanceof StrConst |
exists(ControlFlowNode const, Cmpop op | const.getNode() instanceof StrConst |
(
test.getTest().(CompareNode).operands(const, op, _)
or
test.getTest().(CompareNode).operands(_, op, const)
) and (
) and
(
op instanceof Eq and test.getSense() = true
or
op instanceof NotEq and test.getSense() = false
)
)
}
}
/* tonode = fromnode.xxx() where the call to xxx returns an identical or similar string */
private predicate str_method_call(ControlFlowNode fromnode, CallNode tonode) {
exists(string method_name |
tonode.getFunction().(AttrNode).getObject(method_name) = fromnode
|
method_name = "strip" or method_name = "format" or
method_name = "lstrip" or method_name = "rstrip" or
method_name = "ljust" or method_name = "rjust" or
method_name = "title" or method_name = "capitalize"
exists(string method_name | tonode.getFunction().(AttrNode).getObject(method_name) = fromnode |
method_name = "strip" or
method_name = "format" or
method_name = "lstrip" or
method_name = "rstrip" or
method_name = "ljust" or
method_name = "rjust" or
method_name = "title" or
method_name = "capitalize"
)
}
@@ -79,8 +76,10 @@ private predicate encode_decode(ControlFlowNode fromnode, CallNode tonode) {
not func.getFunction().isMethod() and
func.getACall() = tonode and
tonode.getAnArg() = fromnode and
func.getName() = name |
name = "encode" or name = "decode" or
func.getName() = name
|
name = "encode" or
name = "decode" or
name = "decodestring"
)
}
@@ -99,22 +98,19 @@ private predicate to_str(ControlFlowNode fromnode, CallNode tonode) {
private predicate slice(ControlFlowNode fromnode, SubscriptNode tonode) {
exists(Slice all |
all = tonode.getIndex().getNode() and
not exists(all.getStart()) and not exists(all.getStop()) and
not exists(all.getStart()) and
not exists(all.getStop()) and
tonode.getObject() = fromnode
)
}
/* tonode = os.path.join(..., fromnode, ...) */
private predicate os_path_join(ControlFlowNode fromnode, CallNode tonode) {
tonode = Value::named("os.path.join").getACall()
and tonode.getAnArg() = fromnode
tonode = Value::named("os.path.join").getACall() and
tonode.getAnArg() = fromnode
}
/** A kind of "taint", representing a dictionary mapping str->"taint" */
class StringDictKind extends DictKind {
StringDictKind() {
this.getValue() instanceof StringKind
}
StringDictKind() { this.getValue() instanceof StringKind }
}

4
python/ql/src/semmle/python/security/strings/Common.qll
View File

@@ -1,12 +1,10 @@
import python
/* A call that returns a copy (or similar) of the argument */
predicate copy_call(ControlFlowNode fromnode, CallNode tonode) {
tonode.getFunction().(AttrNode).getObject("copy") = fromnode
or
exists(ModuleValue copy, string name |
name = "copy" or name = "deepcopy" |
exists(ModuleValue copy, string name | name = "copy" or name = "deepcopy" |
copy.attr(name).(FunctionValue).getACall() = tonode and
tonode.getArg(0) = fromnode
)

9
python/ql/src/semmle/python/security/strings/External.qll
View File

@@ -206,7 +206,6 @@ class ExternalFileObject extends TaintKind {
*/
class UrlsplitUrlparseTempSanitizer extends Sanitizer {
// TODO: remove this once we have better support for named tuples
UrlsplitUrlparseTempSanitizer() { this = "UrlsplitUrlparseTempSanitizer" }
override predicate sanitizingEdge(TaintKind taint, PyEdgeRefinement test) {
@@ -238,9 +237,7 @@ class UrlsplitUrlparseTempSanitizer extends Sanitizer {
/** holds for `== "KNOWN_VALUE"` on `true` edge, and `!= "KNOWN_VALUE"` on `false` edge */
private predicate test_equality_with_const(CompareNode cmp, ControlFlowNode tainted, boolean sense) {
exists(ControlFlowNode const, Cmpop op |
const.getNode() instanceof StrConst
|
exists(ControlFlowNode const, Cmpop op | const.getNode() instanceof StrConst |
(
cmp.operands(const, op, tainted)
or
@@ -257,7 +254,9 @@ class UrlsplitUrlparseTempSanitizer extends Sanitizer {
/** holds for `in ["KNOWN_VALUE", ...]` on `true` edge, and `not in ["KNOWN_VALUE", ...]` on `false` edge */
private predicate test_in_const_seq(CompareNode cmp, ControlFlowNode tainted, boolean sense) {
exists(SequenceNode const_seq, Cmpop op |
forall(ControlFlowNode elem | elem = const_seq.getAnElement() | elem.getNode() instanceof StrConst)
forall(ControlFlowNode elem | elem = const_seq.getAnElement() |
elem.getNode() instanceof StrConst
)
|
cmp.operands(tainted, op, const_seq) and
(

10
python/ql/src/semmle/python/security/strings/Untrusted.qll
View File

@@ -1,14 +1,10 @@
import python
import External
/** A kind of taint representing an externally controlled string.
/**
* A kind of taint representing an externally controlled string.
* This class is a simple sub-class of `ExternalStringKind`.
*/
class UntrustedStringKind extends ExternalStringKind {
UntrustedStringKind() {
this = "externally controlled string"
}
UntrustedStringKind() { this = "externally controlled string" }
}

42
python/ql/src/semmle/python/strings.qll
View File

@@ -9,23 +9,22 @@ predicate mapping_format(StrConst e) {
}
/*
MAPPING_KEY = "(\\([^)]+\\))?"
CONVERSION_FLAGS = "[#0\\- +]?"
MINIMUM_FIELD_WIDTH = "(\\*|[0-9]*)"
PRECISION = "(\\.(\\*|[0-9]*))?"
LENGTH_MODIFIER = "[hlL]?"
TYPE = "[bdiouxXeEfFgGcrs%]"
*/
* MAPPING_KEY = "(\\([^)]+\\))?"
* CONVERSION_FLAGS = "[#0\\- +]?"
* MINIMUM_FIELD_WIDTH = "(\\*|[0-9]*)"
* PRECISION = "(\\.(\\*|[0-9]*))?"
* LENGTH_MODIFIER = "[hlL]?"
* TYPE = "[bdiouxXeEfFgGcrs%]"
*/
private
string conversion_specifier_string(StrConst e, int number, int position) {
private string conversion_specifier_string(StrConst e, int number, int position) {
exists(string s, string REGEX | s = e.getText() |
REGEX = "%(\\([^)]*\\))?[#0\\- +]*(\\*|[0-9]*)(\\.(\\*|[0-9]*))?(h|H|l|L)?[badiouxXeEfFgGcrs%]" and
result = s.regexpFind(REGEX, number, position))
result = s.regexpFind(REGEX, number, position)
)
}
private
string conversion_specifier(StrConst e, int number) {
private string conversion_specifier(StrConst e, int number) {
result = conversion_specifier_string(e, number, _) and result != "%%"
}
@@ -39,22 +38,17 @@ int illegal_conversion_specifier(StrConst e) {
/** Gets the number of format items in a format string */
int format_items(StrConst e) {
result = count(int i | | conversion_specifier(e, i)) +
// a conversion specifier uses an extra item for each *
count(int i, int j | conversion_specifier(e, i).charAt(j) = "*")
result =
count(int i | | conversion_specifier(e, i)) +
// a conversion specifier uses an extra item for each *
count(int i, int j | conversion_specifier(e, i).charAt(j) = "*")
}
private string str(Expr e) {
result = ((Num)e).getN()
result = e.(Num).getN()
or
result = "'" + ((StrConst)e).getText() + "'"
result = "'" + e.(StrConst).getText() + "'"
}
/** Gets a string representation of an expression more suited for embedding in message strings than .toString() */
string repr(Expr e) {
if exists(str(e)) then
result = str(e)
else
result = e.toString()
}
string repr(Expr e) { if exists(str(e)) then result = str(e) else result = e.toString() }

70
python/ql/src/semmle/python/templates/PyxlTags.qll
View File

@@ -1,32 +1,22 @@
import python
/** A Tag in Pyxl (which gets converted to a call in Python).
*
/**
* A Tag in Pyxl (which gets converted to a call in Python).
*/
class PyxlTag extends Call {
PyxlTag() { pyxl_tag(this, _) }
PyxlTag() {
pyxl_tag(this, _)
}
string getPyxlTagName() {
pyxl_tag(this, result)
}
string getPyxlTagName() { pyxl_tag(this, result) }
/** Gets the pyxl or Python node that is enclosed by this one in the pyxl source */
Expr getEnclosedNode() {
none()
}
Expr getEnclosedNode() { none() }
/** Gets the Python code (if any) that is contained in this pyxl node */
Expr getEnclosedPythonCode() {
result = this.getEnclosedNode() and not result instanceof PyxlTag
or
result = ((PyxlTag)this.getEnclosedNode()).getEnclosedPythonCode()
result = this.getEnclosedNode().(PyxlTag).getEnclosedPythonCode()
}
}
private predicate pyxl_tag(Call c, string name) {
@@ -39,53 +29,33 @@ private predicate pyxl_tag(Call c, string name) {
}
class PyxlHtmlTag extends PyxlTag {
PyxlHtmlTag() { this.getPyxlTagName().prefix(2) = "x_" }
PyxlHtmlTag() {
this.getPyxlTagName().prefix(2) = "x_"
}
string getTagName() {
result = this.getPyxlTagName().suffix(2)
}
string getTagName() { result = this.getPyxlTagName().suffix(2) }
/** Html tags get transformed into a call. This node is the callee function and the enclosed node is an argument. */
override Expr getEnclosedNode() {
exists(Call c |
exists(Call c |
c.getFunc() = this and
result = c.getAnArg()
)
}
}
class PyxlIfTag extends PyxlTag {
PyxlIfTag() { this.getPyxlTagName() = "_push_condition" }
PyxlIfTag() {
this.getPyxlTagName() = "_push_condition"
}
override Expr getEnclosedNode() {
result = this.getAnArg()
}
override Expr getEnclosedNode() { result = this.getAnArg() }
}
class PyxlEndIfTag extends PyxlTag {
PyxlEndIfTag() { this.getPyxlTagName() = "_leave_if" }
PyxlEndIfTag() {
this.getPyxlTagName() = "_leave_if"
}
override Expr getEnclosedNode() {
result = this.getAnArg()
}
override Expr getEnclosedNode() { result = this.getAnArg() }
}
class PyxlRawHtml extends PyxlTag{
PyxlRawHtml() {
this.getPyxlTagName() = "rawhtml"
}
class PyxlRawHtml extends PyxlTag {
PyxlRawHtml() { this.getPyxlTagName() = "rawhtml" }
/** The text for this raw html, if it is simple text. */
string getText() {
@@ -95,13 +65,7 @@ class PyxlRawHtml extends PyxlTag{
)
}
Expr getValue() {
result = this.getArg(0)
}
override Expr getEnclosedNode() {
result = this.getAnArg()
}
Expr getValue() { result = this.getArg(0) }
override Expr getEnclosedNode() { result = this.getAnArg() }
}

17
python/ql/src/semmle/python/templates/Templates.qll
View File

@@ -1,24 +1,15 @@
import python
abstract class Template extends Module {
}
abstract class Template extends Module { }
class SpitfireTemplate extends Template {
SpitfireTemplate() {
this.getKind() = "Spitfire template"
}
SpitfireTemplate() { this.getKind() = "Spitfire template" }
}
class PyxlModule extends Template {
PyxlModule() {
PyxlModule() {
exists(Comment c | c.getLocation().getFile() = this.getFile() |
c.getText().regexpMatch("# *coding.*pyxl.*")
)
}
}
}

Some files were not shown because too many files have changed in this diff Show More