hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-04-30 11:15:13 +02:00
Code Issues Packages Projects Releases Wiki Activity

Python: Autoformat query-local libs.

Browse Source
This commit is contained in:
Taus Brock-Nannestad
2020-03-20 16:42:46 +01:00
parent 9044ff6959
commit 6904898a8b
14 changed files with 504 additions and 536 deletions
Download Patch File Download Diff File Expand all files Collapse all files

159
python/ql/src/Classes/ClassAttributes.qll
View File

@@ -3,7 +3,6 @@ private import semmle.python.pointsto.PointsTo
/** Helper class for UndefinedClassAttribute.ql and MaybeUndefinedClassAttribute.ql */
class CheckClass extends ClassObject {
private predicate ofInterest() {
not this.unknowableAttributes() and
not this.getPyClass().isProbableMixin() and
@@ -19,7 +18,8 @@ class CheckClass extends ClassObject {
forall(ClassObject sup |
sup = this.getAnImproperSuperType() and
sup.declaresAttribute("__init__") and
not sup = theObjectType() |
not sup = theObjectType()
|
sup.declaredAttribute("__init__") instanceof PyFunctionObject
)
}
@@ -32,108 +32,111 @@ class CheckClass extends ClassObject {
}
predicate sometimesDefines(string name) {
this.alwaysDefines(name) or
exists(SelfAttributeStore sa |
sa.getScope().getScope+() = this.getAnImproperSuperType().getPyClass() |
this.alwaysDefines(name)
or
exists(SelfAttributeStore sa |
sa.getScope().getScope+() = this.getAnImproperSuperType().getPyClass()
|
name = sa.getName()
)
}
private predicate selfDictAssigns() {
exists(Assign a, SelfAttributeRead self_dict, Subscript sub |
exists(Assign a, SelfAttributeRead self_dict, Subscript sub |
self_dict.getName() = "__dict__" and
(
self_dict = sub.getObject()
or
/* Indirect assignment via temporary variable */
exists(SsaVariable v |
v.getAUse() = sub.getObject().getAFlowNode() and
v.getDefinition().(DefinitionNode).getValue() = self_dict.getAFlowNode()
)
(
self_dict = sub.getObject()
or
/* Indirect assignment via temporary variable */
exists(SsaVariable v |
v.getAUse() = sub.getObject().getAFlowNode() and
v.getDefinition().(DefinitionNode).getValue() = self_dict.getAFlowNode()
)
) and
a.getATarget() = sub and
exists(FunctionObject meth | meth = this.lookupAttribute(_) and a.getScope() = meth.getFunction())
exists(FunctionObject meth |
meth = this.lookupAttribute(_) and a.getScope() = meth.getFunction()
)
)
}
pragma [nomagic]
pragma[nomagic]
private predicate monkeyPatched(string name) {
exists(Attribute a |
a.getCtx() instanceof Store and
PointsTo::points_to(a.getObject().getAFlowNode(), _, this, _, _) and a.getName() = name
a.getCtx() instanceof Store and
PointsTo::points_to(a.getObject().getAFlowNode(), _, this, _, _) and
a.getName() = name
)
}
private predicate selfSetattr() {
exists(Call c, Name setattr, Name self, Function method |
( method.getScope() = this.getPyClass() or
method.getScope() = this.getASuperType().getPyClass()
) and
c.getScope() = method and
c.getFunc() = setattr and
setattr.getId() = "setattr" and
c.getArg(0) = self and
self.getId() = "self"
)
exists(Call c, Name setattr, Name self, Function method |
(
method.getScope() = this.getPyClass() or
method.getScope() = this.getASuperType().getPyClass()
) and
c.getScope() = method and
c.getFunc() = setattr and
setattr.getId() = "setattr" and
c.getArg(0) = self and
self.getId() = "self"
)
}
predicate interestingUndefined(SelfAttributeRead a) {
exists(string name | name = a.getName() |
interestingContext(a, name) and
not this.definedInBlock(a.getAFlowNode().getBasicBlock(), name)
)
}
predicate interestingUndefined(SelfAttributeRead a) {
exists(string name | name = a.getName() |
interestingContext(a, name) and
not this.definedInBlock(a.getAFlowNode().getBasicBlock(), name)
)
}
private predicate interestingContext(SelfAttributeRead a, string name) {
name = a.getName() and
this.ofInterest() and
this.getPyClass() = a.getScope().getScope() and
not a.locallyDefined() and
not a.guardedByHasattr() and
a.getScope().isPublic() and
not this.monkeyPatched(name) and
not attribute_assigned_in_method(lookupAttribute("setUp"), name)
}
private predicate interestingContext(SelfAttributeRead a, string name) {
name = a.getName() and
this.ofInterest() and
this.getPyClass() = a.getScope().getScope() and
not a.locallyDefined() and
not a.guardedByHasattr() and
a.getScope().isPublic() and
not this.monkeyPatched(name) and
not attribute_assigned_in_method(lookupAttribute("setUp"), name)
}
private predicate probablyAbstract() {
this.getName().matches("Abstract%")
or
this.isAbstract()
}
private predicate probablyAbstract() {
this.getName().matches("Abstract%")
or
this.isAbstract()
}
private pragma[nomagic] predicate definitionInBlock(BasicBlock b, string name) {
exists(SelfAttributeStore sa |
sa.getAFlowNode().getBasicBlock() = b and sa.getName() = name and sa.getClass() = this.getPyClass()
)
or
exists(FunctionObject method | this.lookupAttribute(_) = method |
attribute_assigned_in_method(method, name) and
b = method.getACall().getBasicBlock()
)
}
private pragma[nomagic] predicate definedInBlock(BasicBlock b, string name) {
// manual specialisation: this is only called from interestingUndefined,
// so we can push the context in from there, which must apply to a
// SelfAttributeRead in the same scope
exists(SelfAttributeRead a |
a.getScope() = b.getScope() and name = a.getName() |
interestingContext(a, name)
)
and
this.definitionInBlock(b, name)
or
exists(BasicBlock prev | this.definedInBlock(prev, name) and prev.getASuccessor() = b)
}
pragma[nomagic]
private predicate definitionInBlock(BasicBlock b, string name) {
exists(SelfAttributeStore sa |
sa.getAFlowNode().getBasicBlock() = b and
sa.getName() = name and
sa.getClass() = this.getPyClass()
)
or
exists(FunctionObject method | this.lookupAttribute(_) = method |
attribute_assigned_in_method(method, name) and
b = method.getACall().getBasicBlock()
)
}
pragma[nomagic]
private predicate definedInBlock(BasicBlock b, string name) {
// manual specialisation: this is only called from interestingUndefined,
// so we can push the context in from there, which must apply to a
// SelfAttributeRead in the same scope
exists(SelfAttributeRead a | a.getScope() = b.getScope() and name = a.getName() |
interestingContext(a, name)
) and
this.definitionInBlock(b, name)
or
exists(BasicBlock prev | this.definedInBlock(prev, name) and prev.getASuccessor() = b)
}
}
private Object object_getattribute() {
result.asBuiltin() = theObjectType().asBuiltin().getMember("__getattribute__")
}
private predicate auto_name(string name) {
name = "__class__" or name = "__dict__"
}
private predicate auto_name(string name) { name = "__class__" or name = "__dict__" }

31
python/ql/src/Classes/Equality.qll
View File

@@ -1,6 +1,5 @@
import python
private Attribute dictAccess(LocalVariable var) {
result.getName() = "__dict__" and
result.getObject() = var.getAnAccess()
@@ -12,18 +11,21 @@ private Call getattr(LocalVariable obj, LocalVariable attr) {
result.getArg(1) = attr.getAnAccess()
}
/** A generic equality method that compares all attributes in its dict,
* or compares attributes using `getattr`. */
/**
* A generic equality method that compares all attributes in its dict,
* or compares attributes using `getattr`.
*/
class GenericEqMethod extends Function {
GenericEqMethod() {
this.getName() = "__eq__" and
exists(LocalVariable self, LocalVariable other |
self.getAnAccess() = this.getArg(0) and self.getId() = "self" and
self.getAnAccess() = this.getArg(0) and
self.getId() = "self" and
other.getAnAccess() = this.getArg(1) and
exists(Compare eq |
exists(Compare eq |
eq.getOp(0) instanceof Eq or
eq.getOp(0) instanceof NotEq |
eq.getOp(0) instanceof NotEq
|
// `self.__dict__ == other.__dict__`
eq.getAChildNode() = dictAccess(self) and
eq.getAChildNode() = dictAccess(other)
@@ -40,11 +42,11 @@ class GenericEqMethod extends Function {
/** An `__eq__` method that just does `self is other` */
class IdentityEqMethod extends Function {
IdentityEqMethod() {
this.getName() = "__eq__" and
exists(LocalVariable self, LocalVariable other |
self.getAnAccess() = this.getArg(0) and self.getId() = "self" and
self.getAnAccess() = this.getArg(0) and
self.getId() = "self" and
other.getAnAccess() = this.getArg(1) and
exists(Compare eq | eq.getOp(0) instanceof Is |
eq.getAChildNode() = self.getAnAccess() and
@@ -52,19 +54,20 @@ class IdentityEqMethod extends Function {
)
)
}
}
/** An (in)equality method that delegates to its complement */
class DelegatingEqualityMethod extends Function {
DelegatingEqualityMethod() {
exists(Return ret, UnaryExpr not_, Compare comp, Cmpop op, Parameter p0, Parameter p1 |
ret.getScope() = this and
ret.getValue() = not_ and
not_.getOp() instanceof Not and not_.getOperand() = comp and
comp.compares(p0.getVariable().getAnAccess(), op, p1.getVariable().getAnAccess()) |
this.getName() = "__eq__" and op instanceof NotEq or
not_.getOp() instanceof Not and
not_.getOperand() = comp and
comp.compares(p0.getVariable().getAnAccess(), op, p1.getVariable().getAnAccess())
|
this.getName() = "__eq__" and op instanceof NotEq
or
this.getName() = "__ne__" and op instanceof Eq
)
}

27
python/ql/src/Classes/MethodCallOrder.qll
View File

@@ -1,17 +1,20 @@
import python
// Helper predicates for multiple call to __init__/__del__ queries.
pragma [noinline]
private predicate multiple_invocation_paths_helper(FunctionInvocation top, FunctionInvocation i1, FunctionInvocation i2, FunctionObject multi) {
pragma[noinline]
private predicate multiple_invocation_paths_helper(
FunctionInvocation top, FunctionInvocation i1, FunctionInvocation i2, FunctionObject multi
) {
i1 != i2 and
i1 = top.getACallee+() and
i2 = top.getACallee+() and
i1.getFunction() = multi
}
pragma [noinline]
private predicate multiple_invocation_paths(FunctionInvocation top, FunctionInvocation i1, FunctionInvocation i2, FunctionObject multi) {
pragma[noinline]
private predicate multiple_invocation_paths(
FunctionInvocation top, FunctionInvocation i1, FunctionInvocation i2, FunctionObject multi
) {
multiple_invocation_paths_helper(top, i1, i2, multi) and
i2.getFunction() = multi
}
@@ -21,7 +24,8 @@ predicate multiple_calls_to_superclass_method(ClassObject self, FunctionObject m
exists(FunctionInvocation top, FunctionInvocation i1, FunctionInvocation i2 |
multiple_invocation_paths(top, i1, i2, multi) and
top.runtime(self.declaredAttribute(name)) and
self.getASuperType().declaredAttribute(name) = multi |
self.getASuperType().declaredAttribute(name) = multi
|
// Only called twice if called from different functions,
// or if one call-site can reach the other.
i1.getCall().getScope() != i2.getCall().getScope()
@@ -53,7 +57,9 @@ private predicate missing_call(FunctionObject meth, string name) {
}
/** Holds if `self.name` does not call `missing`, even though it is expected to. */
predicate missing_call_to_superclass_method(ClassObject self, FunctionObject top, FunctionObject missing, string name) {
predicate missing_call_to_superclass_method(
ClassObject self, FunctionObject top, FunctionObject missing, string name
) {
missing = self.getASuperType().declaredAttribute(name) and
top = self.lookupAttribute(name) and
/* There is no call to missing originating from top */
@@ -63,10 +69,7 @@ predicate missing_call_to_superclass_method(ClassObject self, FunctionObject top
sup = self.getAnImproperSuperType() and
named_attributes_not_method(sup, name)
) and
not self.isAbstract()
and
does_something(missing)
and
not self.isAbstract() and
does_something(missing) and
not missing_call(top, name)
}

1
python/ql/src/Exceptions/NotImplemented.qll
View File

@@ -1,4 +1,3 @@
import python
/** Holds if `notimpl` refers to `NotImplemented` or `NotImplemented()` in the `raise` statement */

10
python/ql/src/Exceptions/Raising.qll
View File

@@ -1,16 +1,12 @@
import python
/** Whether the raise statement 'r' raises 'type' from origin 'orig' */
/** Whether the raise statement 'r' raises 'type' from origin 'orig' */
predicate type_or_typeof(Raise r, ClassValue type, AstNode orig) {
exists(Expr exception |
exception = r.getRaised() |
exists(Expr exception | exception = r.getRaised() |
exception.pointsTo(type, orig)
or
not exists(ClassValue exc_type | exception.pointsTo(exc_type)) and
not type = ClassValue::type() and // First value is an unknown exception type
exists(Value val | exception.pointsTo(val, orig) |
val.getClass() = type
)
exists(Value val | exception.pointsTo(val, orig) | val.getClass() = type)
)
}

179
python/ql/src/Expressions/CallArgs.qll
View File

@@ -1,14 +1,10 @@
import python
import Testing.Mox
private int varargs_length_objectapi(Call call) {
not exists(call.getStarargs()) and result = 0
or
exists(TupleObject t |
call.getStarargs().refersTo(t) |
result = t.getLength()
)
exists(TupleObject t | call.getStarargs().refersTo(t) | result = t.getLength())
or
result = count(call.getStarargs().(List).getAnElt())
}
@@ -16,10 +12,7 @@ private int varargs_length_objectapi(Call call) {
private int varargs_length(Call call) {
not exists(call.getStarargs()) and result = 0
or
exists(TupleValue t |
call.getStarargs().pointsTo(t) |
result = t.length()
)
exists(TupleValue t | call.getStarargs().pointsTo(t) | result = t.length())
or
result = count(call.getStarargs().(List).getAnElt())
}
@@ -38,43 +31,43 @@ private Keyword not_keyword_only_arg(Call call, FunctionValue func) {
not func.getScope().getAKeywordOnlyArg().getId() = result.getArg()
}
/** Gets the count of arguments that are passed as positional parameters even if they
/**
* Gets the count of arguments that are passed as positional parameters even if they
* are named in the call.
* This is the sum of the number of positional arguments, the number of elements in any explicit tuple passed as *arg
* plus the number of keyword arguments that do not match keyword-only arguments (if the function does not take **kwargs).
*/
private int positional_arg_count_for_call_objectapi(Call call, Object callable) {
call = get_a_call_objectapi(callable).getNode() and
exists(int positional_keywords |
exists(FunctionObject func | func = get_function_or_initializer_objectapi(callable) |
not func.getFunction().hasKwArg() and
positional_keywords = count(not_keyword_only_arg_objectapi(call, func))
or
func.getFunction().hasKwArg() and positional_keywords = 0
)
|
result = count(call.getAnArg()) + varargs_length_objectapi(call) + positional_keywords
exists(FunctionObject func | func = get_function_or_initializer_objectapi(callable) |
not func.getFunction().hasKwArg() and
positional_keywords = count(not_keyword_only_arg_objectapi(call, func))
or
func.getFunction().hasKwArg() and positional_keywords = 0
)
|
result = count(call.getAnArg()) + varargs_length_objectapi(call) + positional_keywords
)
}
/** Gets the count of arguments that are passed as positional parameters even if they
/**
* Gets the count of arguments that are passed as positional parameters even if they
* are named in the call.
* This is the sum of the number of positional arguments, the number of elements in any explicit tuple passed as *arg
* plus the number of keyword arguments that do not match keyword-only arguments (if the function does not take **kwargs).
*/
private int positional_arg_count_for_call(Call call, Value callable) {
call = get_a_call(callable).getNode() and
exists(int positional_keywords |
exists(FunctionValue func | func = get_function_or_initializer(callable) |
not func.getScope().hasKwArg() and
positional_keywords = count(not_keyword_only_arg(call, func))
or
func.getScope().hasKwArg() and positional_keywords = 0
)
|
result = count(call.getAnArg()) + varargs_length_objectapi(call) + positional_keywords
exists(FunctionValue func | func = get_function_or_initializer(callable) |
not func.getScope().hasKwArg() and
positional_keywords = count(not_keyword_only_arg(call, func))
or
func.getScope().hasKwArg() and positional_keywords = 0
)
|
result = count(call.getAnArg()) + varargs_length_objectapi(call) + positional_keywords
)
}
@@ -88,33 +81,32 @@ int arg_count(Call call) {
/* Gets a call corresponding to the given class or function*/
private ControlFlowNode get_a_call_objectapi(Object callable) {
result = callable.(ClassObject).getACall()
or
result = callable.(FunctionObject).getACall()
result = callable.(ClassObject).getACall()
or
result = callable.(FunctionObject).getACall()
}
/* Gets a call corresponding to the given class or function*/
private ControlFlowNode get_a_call(Value callable) {
result = callable.(ClassValue).getACall()
or
result = callable.(FunctionValue).getACall()
result = callable.(ClassValue).getACall()
or
result = callable.(FunctionValue).getACall()
}
/* Gets the function object corresponding to the given class or function*/
FunctionObject get_function_or_initializer_objectapi(Object func_or_cls) {
result = func_or_cls.(FunctionObject)
or
result = func_or_cls.(ClassObject).declaredAttribute("__init__")
result = func_or_cls.(FunctionObject)
or
result = func_or_cls.(ClassObject).declaredAttribute("__init__")
}
/* Gets the function object corresponding to the given class or function*/
FunctionValue get_function_or_initializer(Value func_or_cls) {
result = func_or_cls.(FunctionValue)
or
result = func_or_cls.(ClassValue).declaredAttribute("__init__")
result = func_or_cls.(FunctionValue)
or
result = func_or_cls.(ClassValue).declaredAttribute("__init__")
}
/**Whether there is an illegally named parameter called `name` in the `call` to `func` */
predicate illegally_named_parameter_objectapi(Call call, Object func, string name) {
not func.isC() and
@@ -135,19 +127,22 @@ predicate illegally_named_parameter(Call call, Value func, string name) {
predicate too_few_args_objectapi(Call call, Object callable, int limit) {
// Exclude cases where an incorrect name is used as that is covered by 'Wrong name for an argument in a call'
not illegally_named_parameter_objectapi(call, callable, _) and
not exists(call.getStarargs()) and not exists(call.getKwargs()) and
not exists(call.getStarargs()) and
not exists(call.getKwargs()) and
arg_count_objectapi(call) < limit and
exists(FunctionObject func | func = get_function_or_initializer_objectapi(callable) |
call = func.getAFunctionCall().getNode() and limit = func.minParameters() and
// The combination of misuse of `mox.Mox().StubOutWithMock()`
// and a bug in mox's implementation of methods results in having to
// pass 1 too few arguments to the mocked function.
not (useOfMoxInModule(call.getEnclosingModule()) and func.isNormalMethod())
or
call = func.getAMethodCall().getNode() and limit = func.minParameters() - 1
or
callable instanceof ClassObject and
call.getAFlowNode() = get_a_call_objectapi(callable) and limit = func.minParameters() - 1
call = func.getAFunctionCall().getNode() and
limit = func.minParameters() and
// The combination of misuse of `mox.Mox().StubOutWithMock()`
// and a bug in mox's implementation of methods results in having to
// pass 1 too few arguments to the mocked function.
not (useOfMoxInModule(call.getEnclosingModule()) and func.isNormalMethod())
or
call = func.getAMethodCall().getNode() and limit = func.minParameters() - 1
or
callable instanceof ClassObject and
call.getAFlowNode() = get_a_call_objectapi(callable) and
limit = func.minParameters() - 1
)
}
@@ -155,19 +150,22 @@ predicate too_few_args_objectapi(Call call, Object callable, int limit) {
predicate too_few_args(Call call, Value callable, int limit) {
// Exclude cases where an incorrect name is used as that is covered by 'Wrong name for an argument in a call'
not illegally_named_parameter(call, callable, _) and
not exists(call.getStarargs()) and not exists(call.getKwargs()) and
not exists(call.getStarargs()) and
not exists(call.getKwargs()) and
arg_count(call) < limit and
exists(FunctionValue func | func = get_function_or_initializer(callable) |
call = func.getACall().getNode() and limit = func.minParameters() and
// The combination of misuse of `mox.Mox().StubOutWithMock()`
// and a bug in mox's implementation of methods results in having to
// pass 1 too few arguments to the mocked function.
not (useOfMoxInModule(call.getEnclosingModule()) and func.isNormalMethod())
or
call = func.getACall().getNode() and limit = func.minParameters() - 1
or
callable instanceof ClassValue and
call.getAFlowNode() = get_a_call(callable) and limit = func.minParameters() - 1
call = func.getACall().getNode() and
limit = func.minParameters() and
// The combination of misuse of `mox.Mox().StubOutWithMock()`
// and a bug in mox's implementation of methods results in having to
// pass 1 too few arguments to the mocked function.
not (useOfMoxInModule(call.getEnclosingModule()) and func.isNormalMethod())
or
call = func.getACall().getNode() and limit = func.minParameters() - 1
or
callable instanceof ClassValue and
call.getAFlowNode() = get_a_call(callable) and
limit = func.minParameters() - 1
)
}
@@ -175,16 +173,18 @@ predicate too_few_args(Call call, Value callable, int limit) {
predicate too_many_args_objectapi(Call call, Object callable, int limit) {
// Exclude cases where an incorrect name is used as that is covered by 'Wrong name for an argument in a call'
not illegally_named_parameter_objectapi(call, callable, _) and
exists(FunctionObject func |
func = get_function_or_initializer_objectapi(callable) and
not func.getFunction().hasVarArg() and limit >= 0
|
exists(FunctionObject func |
func = get_function_or_initializer_objectapi(callable) and
not func.getFunction().hasVarArg() and
limit >= 0
|
call = func.getAFunctionCall().getNode() and limit = func.maxParameters()
or
or
call = func.getAMethodCall().getNode() and limit = func.maxParameters() - 1
or
or
callable instanceof ClassObject and
call.getAFlowNode() = get_a_call_objectapi(callable) and limit = func.maxParameters() - 1
call.getAFlowNode() = get_a_call_objectapi(callable) and
limit = func.maxParameters() - 1
) and
positional_arg_count_for_call_objectapi(call, callable) > limit
}
@@ -193,16 +193,18 @@ predicate too_many_args_objectapi(Call call, Object callable, int limit) {
predicate too_many_args(Call call, Value callable, int limit) {
// Exclude cases where an incorrect name is used as that is covered by 'Wrong name for an argument in a call'
not illegally_named_parameter(call, callable, _) and
exists(FunctionValue func |
func = get_function_or_initializer(callable) and
not func.getScope().hasVarArg() and limit >= 0
|
exists(FunctionValue func |
func = get_function_or_initializer(callable) and
not func.getScope().hasVarArg() and
limit >= 0
|
call = func.getACall().getNode() and limit = func.maxParameters()
or
or
call = func.getACall().getNode() and limit = func.maxParameters() - 1
or
or
callable instanceof ClassValue and
call.getAFlowNode() = get_a_call(callable) and limit = func.maxParameters() - 1
call.getAFlowNode() = get_a_call(callable) and
limit = func.maxParameters() - 1
) and
positional_arg_count_for_call(call, callable) > limit
}
@@ -221,35 +223,34 @@ predicate wrong_args(Call call, FunctionValue func, int limit, string too) {
too_many_args(call, func, limit) and too = "too many"
}
/** Holds if `call` has correct number of arguments for `func`.
/**
* Holds if `call` has correct number of arguments for `func`.
* Implies nothing about whether `call` could call `func`.
*/
bindingset[call, func]
bindingset[call, func]
predicate correct_args_if_called_as_method_objectapi(Call call, FunctionObject func) {
arg_count_objectapi(call)+1 >= func.minParameters()
and
arg_count_objectapi(call) + 1 >= func.minParameters() and
arg_count_objectapi(call) < func.maxParameters()
}
/** Holds if `call` has correct number of arguments for `func`.
/**
* Holds if `call` has correct number of arguments for `func`.
* Implies nothing about whether `call` could call `func`.
*/
bindingset[call, func]
bindingset[call, func]
predicate correct_args_if_called_as_method(Call call, FunctionValue func) {
arg_count(call)+1 >= func.minParameters()
and
arg_count(call) + 1 >= func.minParameters() and
arg_count(call) < func.maxParameters()
}
/** Holds if `call` is a call to `overriding`, which overrides `func`. */
predicate overridden_call_objectapi(FunctionObject func, FunctionObject overriding, Call call) {
predicate overridden_call_objectapi(FunctionObject func, FunctionObject overriding, Call call) {
overriding.overrides(func) and
overriding.getACall().getNode() = call
}
/** Holds if `call` is a call to `overriding`, which overrides `func`. */
predicate overridden_call(FunctionValue func, FunctionValue overriding, Call call) {
predicate overridden_call(FunctionValue func, FunctionValue overriding, Call call) {
overriding.overrides(func) and
overriding.getACall().getNode() = call
}

92
python/ql/src/Expressions/Formatting/AdvancedFormatting.qll
View File

@@ -1,11 +1,7 @@
import python
library class PossibleAdvancedFormatString extends StrConst {
PossibleAdvancedFormatString() {
this.getText().matches("%{%}%")
}
PossibleAdvancedFormatString() { this.getText().matches("%{%}%") }
private predicate field(int start, int end) {
brace_pair(this, start, end) and
@@ -31,83 +27,77 @@ library class PossibleAdvancedFormatString extends StrConst {
(
result = this.getText().substring(start, end).regexpCapture("\\{([^!:.\\[]+)[!:.\\[].*", 1)
or
result = this.getText().substring(start+1, end-1) and result.regexpMatch("[^!:.\\[]+")
result = this.getText().substring(start + 1, end - 1) and result.regexpMatch("[^!:.\\[]+")
)
}
/** Gets the name of the formatting field at [start, end) */
string getFieldName(int start, int end) {
result = this.fieldId(start, end)
and not exists(this.getFieldNumber(start, end))
result = this.fieldId(start, end) and
not exists(this.getFieldNumber(start, end))
}
private predicate implicitlyNumberedField(int start, int end) {
this.field(start, end) and
exists(string c |
start+1 = this.getText().indexOf(c) |
exists(string c | start + 1 = this.getText().indexOf(c) |
c = "}" or c = ":" or c = "!" or c = "."
)
}
/** Whether this format string has implicitly numbered fields */
predicate isImplicitlyNumbered() {
this.implicitlyNumberedField(_, _)
}
predicate isImplicitlyNumbered() { this.implicitlyNumberedField(_, _) }
/** Whether this format string has explicitly numbered fields */
predicate isExplicitlyNumbered() {
exists(this.fieldId(_, _).toInt())
}
predicate isExplicitlyNumbered() { exists(this.fieldId(_, _).toInt()) }
}
predicate brace_sequence(PossibleAdvancedFormatString fmt, int index, int len) {
exists(string text |
text = fmt.getText() |
text.charAt(index) = "{" and not text.charAt(index-1) = "{" and len = 1
exists(string text | text = fmt.getText() |
text.charAt(index) = "{" and not text.charAt(index - 1) = "{" and len = 1
or
text.charAt(index) = "{" and text.charAt(index-1) = "{" and brace_sequence(fmt, index-1, len-1)
text.charAt(index) = "{" and
text.charAt(index - 1) = "{" and
brace_sequence(fmt, index - 1, len - 1)
)
}
predicate escaped_brace(PossibleAdvancedFormatString fmt, int index) {
exists(int len |
brace_sequence(fmt, index, len) |
len % 2 = 0
)
exists(int len | brace_sequence(fmt, index, len) | len % 2 = 0)
}
predicate escaping_brace(PossibleAdvancedFormatString fmt, int index) {
escaped_brace(fmt, index+1)
escaped_brace(fmt, index + 1)
}
private predicate inner_brace_pair(PossibleAdvancedFormatString fmt, int start, int end) {
not escaping_brace(fmt, start) and
not escaped_brace(fmt, start) and
fmt.getText().charAt(start) = "{" and
exists(string pair | pair = fmt.getText().suffix(start).regexpCapture("(?s)(\\{([^{}]|\\{\\{)*+\\}).*", 1) |
exists(string pair |
pair = fmt.getText().suffix(start).regexpCapture("(?s)(\\{([^{}]|\\{\\{)*+\\}).*", 1)
|
end = start + pair.length()
)
}
private predicate brace_pair(PossibleAdvancedFormatString fmt, int start, int end) {
inner_brace_pair(fmt, start, end)
or
not escaping_brace(fmt, start) and
not escaped_brace(fmt, start) and
exists(string prefix, string postfix, int innerstart, int innerend |
brace_pair(fmt, innerstart, innerend) and
prefix = fmt.getText().regexpFind("\\{([^{}]|\\{\\{)+\\{", _, start) and
innerstart = start+prefix.length()-1 and
postfix = fmt.getText().regexpFind("\\}([^{}]|\\}\\})*\\}", _, innerend-1) and
end = innerend + postfix.length()-1
)
inner_brace_pair(fmt, start, end)
or
not escaping_brace(fmt, start) and
not escaped_brace(fmt, start) and
exists(string prefix, string postfix, int innerstart, int innerend |
brace_pair(fmt, innerstart, innerend) and
prefix = fmt.getText().regexpFind("\\{([^{}]|\\{\\{)+\\{", _, start) and
innerstart = start + prefix.length() - 1 and
postfix = fmt.getText().regexpFind("\\}([^{}]|\\}\\})*\\}", _, innerend - 1) and
end = innerend + postfix.length() - 1
)
}
private predicate advanced_format_call(Call format_expr, PossibleAdvancedFormatString fmt, int args) {
exists(CallNode call |
call = format_expr.getAFlowNode() |
call.getFunction().pointsTo(Value::named("format")) and call.getArg(0).pointsTo(_, fmt.getAFlowNode()) and
exists(CallNode call | call = format_expr.getAFlowNode() |
call.getFunction().pointsTo(Value::named("format")) and
call.getArg(0).pointsTo(_, fmt.getAFlowNode()) and
args = count(format_expr.getAnArg()) - 1
or
call.getFunction().(AttrNode).getObject("format").pointsTo(_, fmt.getAFlowNode()) and
@@ -116,26 +106,14 @@ private predicate advanced_format_call(Call format_expr, PossibleAdvancedFormatS
}
class AdvancedFormatString extends PossibleAdvancedFormatString {
AdvancedFormatString() {
advanced_format_call(_, this, _)
}
AdvancedFormatString() { advanced_format_call(_, this, _) }
}
class AdvancedFormattingCall extends Call {
AdvancedFormattingCall() {
advanced_format_call(this, _, _)
}
AdvancedFormattingCall() { advanced_format_call(this, _, _) }
/** Count of the arguments actually provided */
int providedArgCount() {
advanced_format_call(this, _, result)
}
AdvancedFormatString getAFormat() {
advanced_format_call(this, result, _)
}
int providedArgCount() { advanced_format_call(this, _, result) }
AdvancedFormatString getAFormat() { advanced_format_call(this, result, _) }
}

79
python/ql/src/Expressions/IsComparisons.qll
View File

@@ -1,9 +1,9 @@
import python
predicate comparison_using_is(Compare comp, ControlFlowNode left, Cmpop op, ControlFlowNode right) {
exists(CompareNode fcomp | fcomp = comp.getAFlowNode() |
fcomp.operands(left, op, right) and (op instanceof Is or op instanceof IsNot)
fcomp.operands(left, op, right) and
(op instanceof Is or op instanceof IsNot)
)
}
@@ -12,8 +12,7 @@ predicate overrides_eq_or_cmp(ClassValue c) {
or
c.declaresAttribute("__eq__") and not c = Value::named("object")
or
exists(ClassValue sup |
sup = c.getASuperType() and not sup = Value::named("object") |
exists(ClassValue sup | sup = c.getASuperType() and not sup = Value::named("object") |
sup.declaresAttribute("__eq__")
)
or
@@ -29,58 +28,56 @@ predicate probablySingleton(ClassValue cls) {
predicate invalid_to_use_is_portably(ClassValue c) {
overrides_eq_or_cmp(c) and
// Exclude type/builtin-function/bool as it is legitimate to compare them using 'is' but they implement __eq__
not c = Value::named("type") and not c = ClassValue::builtinFunction() and not c = Value::named("bool") and
not c = Value::named("type") and
not c = ClassValue::builtinFunction() and
not c = Value::named("bool") and
// OK to compare with 'is' if a singleton
not probablySingleton(c)
}
predicate simple_constant(ControlFlowNode f) {
exists(Value val | f.pointsTo(val) | val = Value::named("True") or val = Value::named("False") or val = Value::named("None"))
exists(Value val | f.pointsTo(val) |
val = Value::named("True") or val = Value::named("False") or val = Value::named("None")
)
}
private predicate cpython_interned_value(Expr e) {
exists(string text | text = e.(StrConst).getText() |
text.length() = 0 or
text.length() = 0
or
text.length() = 1 and text.regexpMatch("[U+0000-U+00ff]")
)
or
exists(int i |
i = e.(IntegerLiteral).getN().toInt() |
-5 <= i and i <= 256
)
or
exists(int i | i = e.(IntegerLiteral).getN().toInt() | -5 <= i and i <= 256)
or
exists(Tuple t | t = e and not exists(t.getAnElt()))
}
/** The set of values that can be expected to be interned across
/**
* The set of values that can be expected to be interned across
* the main implementations of Python. PyPy, Jython, etc tend to
* follow CPython, but it varies, so this is a best guess.
*/
private predicate universally_interned_value(Expr e) {
e.(IntegerLiteral).getN().toInt() = 0
or
exists(Tuple t | t = e and not exists(t.getAnElt()))
or
e.(StrConst).getText() = ""
e.(IntegerLiteral).getN().toInt() = 0
or
exists(Tuple t | t = e and not exists(t.getAnElt()))
or
e.(StrConst).getText() = ""
}
predicate cpython_interned_constant(Expr e) {
exists(Expr const |
e.pointsTo(_, const) |
cpython_interned_value(const)
)
exists(Expr const | e.pointsTo(_, const) | cpython_interned_value(const))
}
predicate universally_interned_constant(Expr e) {
exists(Expr const |
e.pointsTo(_, const) |
universally_interned_value(const)
)
exists(Expr const | e.pointsTo(_, const) | universally_interned_value(const))
}
private predicate comparison_both_types(Compare comp, Cmpop op, ClassValue cls1, ClassValue cls2) {
exists(ControlFlowNode op1, ControlFlowNode op2 |
comparison_using_is(comp, op1, op, op2) or comparison_using_is(comp, op2, op, op1) |
comparison_using_is(comp, op1, op, op2) or comparison_using_is(comp, op2, op, op1)
|
op1.inferredValue().getClass() = cls1 and
op2.inferredValue().getClass() = cls2
)
@@ -89,15 +86,17 @@ private predicate comparison_both_types(Compare comp, Cmpop op, ClassValue cls1,
private predicate comparison_one_type(Compare comp, Cmpop op, ClassValue cls) {
not comparison_both_types(comp, _, _, _) and
exists(ControlFlowNode operand |
comparison_using_is(comp, operand, op, _) or comparison_using_is(comp, _, op, operand) |
operand.inferredValue().getClass() = cls
comparison_using_is(comp, operand, op, _) or comparison_using_is(comp, _, op, operand)
|
operand.inferredValue().getClass() = cls
)
}
predicate invalid_portable_is_comparison(Compare comp, Cmpop op, ClassValue cls) {
// OK to use 'is' when defining '__eq__'
not exists(Function eq | eq.getName() = "__eq__" or eq.getName() = "__ne__" | eq = comp.getScope().getScope*())
and
not exists(Function eq | eq.getName() = "__eq__" or eq.getName() = "__ne__" |
eq = comp.getScope().getScope*()
) and
(
comparison_one_type(comp, op, cls) and invalid_to_use_is_portably(cls)
or
@@ -105,32 +104,32 @@ predicate invalid_portable_is_comparison(Compare comp, Cmpop op, ClassValue cls)
invalid_to_use_is_portably(cls) and
invalid_to_use_is_portably(other)
)
)
and
) and
// OK to use 'is' when comparing items from a known set of objects
not exists(Expr left, Expr right, Value val |
comp.compares(left, op, right) and
exists(ImmutableLiteral il | il.getLiteralValue() = val) |
exists(ImmutableLiteral il | il.getLiteralValue() = val)
|
left.pointsTo(val) and right.pointsTo(val)
or
// Simple constant in module, probably some sort of sentinel
exists(AstNode origin |
not left.pointsTo(_) and right.pointsTo(val, origin) and
not left.pointsTo(_) and
right.pointsTo(val, origin) and
origin.getScope().getEnclosingModule() = comp.getScope().getEnclosingModule()
)
)
and
) and
// OK to use 'is' when comparing with a member of an enum
not exists(Expr left, Expr right, AstNode origin |
comp.compares(left, op, right) and
enum_member(origin) |
enum_member(origin)
|
left.pointsTo(_, origin) or right.pointsTo(_, origin)
)
}
private predicate enum_member(AstNode obj) {
exists(ClassValue cls, AssignStmt asgn |
cls.getASuperType().getName() = "Enum" |
exists(ClassValue cls, AssignStmt asgn | cls.getASuperType().getName() = "Enum" |
cls.getScope() = asgn.getScope() and
asgn.getValue() = obj
)

18
python/ql/src/Expressions/RedundantComparison.qll
View File

@@ -1,27 +1,22 @@
import python
class RedundantComparison extends Compare {
RedundantComparison() {
exists(Expr left, Expr right |
this.compares(left, _, right)
and
this.compares(left, _, right) and
same_variable(left, right)
)
}
predicate maybeMissingSelf() {
exists(Name left |
this.compares(left, _, _) and
not this.isConstant() and
exists(Class cls | left.getScope().getScope() = cls |
exists(SelfAttribute sa | sa.getName() = left.getId() |
sa.getClass() = cls
)
exists(SelfAttribute sa | sa.getName() = left.getId() | sa.getClass() = cls)
)
)
}
}
private predicate same_variable(Expr left, Expr right) {
@@ -36,11 +31,14 @@ private predicate name_in_comparison(Compare comp, Name n, Variable v) {
private predicate same_name(Name n1, Name n2) {
n1 != n2 and
exists(Compare comp, Variable v | name_in_comparison(comp, n1, v) and name_in_comparison(comp, n2, v))
exists(Compare comp, Variable v |
name_in_comparison(comp, n1, v) and name_in_comparison(comp, n2, v)
)
}
private predicate same_attribute(Attribute a1, Attribute a2) {
a1 != a2 and
exists(Compare comp | comp.contains(a1) and comp.contains(a2)) and
a1.getName() = a2.getName() and same_name(a1.getObject(), a2.getObject())
a1.getName() = a2.getName() and
same_name(a1.getObject(), a2.getObject())
}

188
python/ql/src/Lexical/CommentedOutCode.qll
View File

@@ -1,6 +1,5 @@
import python
private predicate def_statement(Comment c) {
c.getText().regexpMatch("#(\\S*\\s+)?def\\s.*\\(.*\\).*:\\s*(#.*)?")
}
@@ -12,7 +11,7 @@ private predicate if_statement(Comment c) {
}
private predicate for_statement(Comment c) {
c.getText().regexpMatch("#(\\S*\\s+)?for\\s.*\\sin\\s.*:\\s*(#.*)?")
c.getText().regexpMatch("#(\\S*\\s+)?for\\s.*\\sin\\s.*:\\s*(#.*)?")
}
private predicate with_statement(Comment c) {
@@ -20,11 +19,11 @@ private predicate with_statement(Comment c) {
}
private predicate try_statement(Comment c) {
c.getText().regexpMatch("#(\\S*\\s+)?try:\\s*(#.*)?")
c.getText().regexpMatch("#(\\S*\\s+)?try:\\s*(#.*)?")
or
c.getText().regexpMatch("#(\\S*\\s+)?except\\s*(\\w+\\s*(\\sas\\s+\\w+\\s*)?)?:\\s*(#.*)?")
or
c.getText().regexpMatch("#(\\S*\\s+)?finally:\\s*(#.*)?")
c.getText().regexpMatch("#(\\S*\\s+)?finally:\\s*(#.*)?")
}
private int indentation(Comment c) {
@@ -39,18 +38,16 @@ private predicate class_statement(Comment c) {
c.getText().regexpMatch("#(\\S*\\s+)?class\\s+\\w+.*:\\s*(#.*)?")
}
private predicate triple_quote(Comment c) {
c.getText().regexpMatch("#.*(\"\"\"|''').*")
}
private predicate triple_quote(Comment c) { c.getText().regexpMatch("#.*(\"\"\"|''').*") }
private predicate triple_quoted_string_part(Comment start, Comment end) {
triple_quote(start) and end = start
or
exists(Comment mid |
triple_quote(start) and end = start
or
exists(Comment mid |
triple_quoted_string_part(start, mid) and
end = non_empty_following(mid) and
not triple_quote(end)
)
)
}
private predicate maybe_code(Comment c) {
@@ -59,41 +56,37 @@ private predicate maybe_code(Comment c) {
commented_out_comment(c)
}
private predicate commented_out_comment(Comment c) {
c.getText().regexpMatch("#+\\s+#.*")
}
private predicate commented_out_comment(Comment c) { c.getText().regexpMatch("#+\\s+#.*") }
private int scope_start(Comment start) {
(
def_statement(start) or
class_statement(start)
)
and
result = indentation(start)
and
def_statement(start) or
class_statement(start)
) and
result = indentation(start) and
not non_code(start)
}
private int block_start(Comment start) {
(
if_statement(start) or
for_statement(start) or
try_statement(start) or
with_statement(start)
)
and
result = indentation(start)
and
if_statement(start) or
for_statement(start) or
try_statement(start) or
with_statement(start)
) and
result = indentation(start) and
not non_code(start)
}
private int scope_doc_string_part(Comment start, Comment end) {
result = scope_start(start) and
triple_quote(end) and end = non_empty_following(start)
triple_quote(end) and
end = non_empty_following(start)
or
exists(Comment mid |
exists(Comment mid |
result = scope_doc_string_part(start, mid) and
end = non_empty_following(mid) |
end = non_empty_following(mid)
|
not triple_quote(end)
)
}
@@ -101,15 +94,16 @@ private int scope_doc_string_part(Comment start, Comment end) {
private int scope_part(Comment start, Comment end) {
result = scope_start(start) and end = start
or
exists(Comment mid |
exists(Comment mid |
result = scope_doc_string_part(start, mid) and
end = non_empty_following(mid) and
triple_quote(end)
)
or
exists(Comment mid |
exists(Comment mid |
result = scope_part(start, mid) and
end = non_empty_following(mid) |
end = non_empty_following(mid)
|
indentation(end) > result
)
}
@@ -119,9 +113,10 @@ private int block_part(Comment start, Comment end) {
end = non_empty_following(start) and
indentation(end) > result
or
exists(Comment mid |
exists(Comment mid |
result = block_part(start, mid) and
end = non_empty_following(mid) |
end = non_empty_following(mid)
|
indentation(end) > result
or
result = block_start(end)
@@ -145,13 +140,11 @@ private predicate commented_out_code(Comment c) {
}
private predicate commented_out_code_part(Comment start, Comment end) {
commented_out_code(start) and end = start and
not exists(Comment prev |
non_empty_following(prev) = start |
commented_out_code(prev)
)
commented_out_code(start) and
end = start and
not exists(Comment prev | non_empty_following(prev) = start | commented_out_code(prev))
or
exists(Comment mid |
exists(Comment mid |
commented_out_code_part(start, mid) and
non_empty_following(mid) = end and
commented_out_code(end)
@@ -167,12 +160,7 @@ private predicate commented_out_code_block(Comment start, Comment end) {
/* A single line comment that appears to be commented out code */
class CommentedOutCodeLine extends Comment {
CommentedOutCodeLine () {
exists(CommentedOutCodeBlock b |
b.contains(this)
)
}
CommentedOutCodeLine() { exists(CommentedOutCodeBlock b | b.contains(this)) }
/* Whether this commented-out code line is likely to be example code embedded in a larger comment. */
predicate maybeExampleCode() {
@@ -181,19 +169,13 @@ class CommentedOutCodeLine extends Comment {
block.maybeExampleCode()
)
}
}
/** A block of comments that appears to be commented out code */
class CommentedOutCodeBlock extends @py_comment {
CommentedOutCodeBlock() {
commented_out_code_block(this, _)
}
CommentedOutCodeBlock() { commented_out_code_block(this, _) }
string toString() {
result = "Commented out code"
}
string toString() { result = "Commented out code" }
/** Whether this commented-out code block contains the comment c */
predicate contains(Comment c) {
@@ -207,28 +189,22 @@ class CommentedOutCodeBlock extends @py_comment {
}
/** The length of this comment block (in comments) */
int length() {
result = count(Comment c | this.contains(c))
}
int length() { result = count(Comment c | this.contains(c)) }
predicate hasLocationInfo(string filepath, int bl, int bc, int el, int ec) {
((Comment)this).getLocation().hasLocationInfo(filepath, bl, bc, _, _)
and
exists(Comment end |
commented_out_code_block(this, end) |
this.(Comment).getLocation().hasLocationInfo(filepath, bl, bc, _, _) and
exists(Comment end | commented_out_code_block(this, end) |
end.getLocation().hasLocationInfo(_, _, _, el, ec)
)
}
/** Whether this commented-out code block is likely to be example code embedded in a larger comment. */
predicate maybeExampleCode() {
exists(CommentBlock block |
block.contains((Comment)this) |
exists(CommentBlock block | block.contains(this.(Comment)) |
exists(int all_code |
all_code = sum (CommentedOutCodeBlock code | block.contains((Comment)code) | code.length())
and
all_code = sum(CommentedOutCodeBlock code | block.contains(code.(Comment)) | code.length()) and
/* This ratio may need fine tuning */
block.length() > all_code*2
block.length() > all_code * 2
)
)
}
@@ -239,13 +215,14 @@ private predicate word_pair(Comment c, string s1, string s2) {
exists(int i1, int i2, int o1, int o2 |
s1 = c.getText().regexpFind("\\w+", i1, o1) and
s2 = c.getText().regexpFind("\\w+", i2, o2) and
i2 = i1 + 1 and
i2 = i1 + 1 and
c.getText().prefix(o1).regexpMatch("[^'\"]*") and
c.getText().substring(o1 + s1.length(), o2).regexpMatch("\\s+")
)
}
/** The comment c cannot be code if it contains a word pair "word1 word2" and
/**
* The comment c cannot be code if it contains a word pair "word1 word2" and
* either:
* 1. word1 is not a keyword and word2 is not an operator:
* "x is" could be code, "return y" could be code, but "isnt code" cannot be code.
@@ -257,48 +234,44 @@ private predicate non_code(Comment c) {
exists(string word1, string word2 |
word_pair(c, word1, word2) and
not word2 = operator_keyword()
|
|
not word1 = a_keyword()
or
word1 = keyword_requiring_colon() and not c.getText().matches("%:%")
) and
/* Except comments of the form: # (maybe code) # some comment */
not c.getText().regexpMatch("#\\S+\\s.*#.*")
not c.getText().regexpMatch("#\\S+\\s.*#.*")
or
/* Don't count doctests as code */
c.getText().matches("%>>>%") or c.getText().matches("%...%")
c.getText().matches("%>>>%")
or
c.getText().matches("%...%")
}
private predicate filler(Comment c) {
c.getText().regexpMatch("#+[\\s*#-_=+]*")
}
private predicate filler(Comment c) { c.getText().regexpMatch("#+[\\s*#-_=+]*") }
/** Gets the first non empty comment following c */
/** Gets the first non empty comment following c */
private Comment non_empty_following(Comment c) {
not empty(result) and
(
not empty(result) and
(
result = empty_following(c).getFollowing()
or
not empty(c) and result = c.getFollowing()
)
not empty(c) and result = c.getFollowing()
)
}
/* Helper for non_empty_following() */
private Comment empty_following(Comment c) {
not empty(c) and
empty(result)
and
exists(Comment prev |
result = prev.getFollowing() |
empty(result) and
exists(Comment prev | result = prev.getFollowing() |
prev = c
or
prev = empty_following(c)
)
}
private predicate empty(Comment c) {
c.getText().regexpMatch("#+\\s*")
}
private predicate empty(Comment c) { c.getText().regexpMatch("#+\\s*") }
/* A comment following code on the same line */
private predicate endline_comment(Comment c) {
@@ -315,21 +288,40 @@ private predicate file_or_url(Comment c) {
}
private string operator_keyword() {
result = "import" or result = "and" or result = "is" or result = "or" or result = "in" or result = "not" or result = "as"
result = "import" or
result = "and" or
result = "is" or
result = "or" or
result = "in" or
result = "not" or
result = "as"
}
private string keyword_requiring_colon() {
result = "try" or result = "while" or result = "elif" or result = "else" or result = "if" or
result = "except" or result = "def" or result = "class"
result = "try" or
result = "while" or
result = "elif" or
result = "else" or
result = "if" or
result = "except" or
result = "def" or
result = "class"
}
private string other_keyword() {
result = "del" or result = "lambda" or result = "from" or
result = "global" or result = "with" or result = "assert" or
result = "yield" or result = "finally" or
result = "print" or
result = "exec" or result = "raise" or
result = "return" or result = "for"
result = "del" or
result = "lambda" or
result = "from" or
result = "global" or
result = "with" or
result = "assert" or
result = "yield" or
result = "finally" or
result = "print" or
result = "exec" or
result = "raise" or
result = "return" or
result = "for"
}
private string a_keyword() {

18
python/ql/src/Metrics/Internal/Extents.qll
View File

@@ -15,10 +15,10 @@ import python
* including the body (if any), as opposed to the location of its name only.
*/
class RangeFunction extends Function {
predicate hasLocationInfo(string path, int sl, int sc, int el, int ec) {
super.getLocation().hasLocationInfo(path, sl, sc, _, _)
and this.getBody().getLastItem().getLocation().hasLocationInfo(path, _, _, el, ec)
}
predicate hasLocationInfo(string path, int sl, int sc, int el, int ec) {
super.getLocation().hasLocationInfo(path, sl, sc, _, _) and
this.getBody().getLastItem().getLocation().hasLocationInfo(path, _, _, el, ec)
}
}
/**
@@ -26,8 +26,8 @@ class RangeFunction extends Function {
* including the body (if any), as opposed to the location of its name only.
*/
class RangeClass extends Class {
predicate hasLocationInfo(string path, int sl, int sc, int el, int ec) {
super.getLocation().hasLocationInfo(path, sl, sc, _, _)
and this.getBody().getLastItem().getLocation().hasLocationInfo(path, _, _, el, ec)
}
}
predicate hasLocationInfo(string path, int sl, int sc, int el, int ec) {
super.getLocation().hasLocationInfo(path, sl, sc, _, _) and
this.getBody().getLastItem().getLocation().hasLocationInfo(path, _, _, el, ec)
}
}

7
python/ql/src/Testing/Mox.qll
View File

@@ -2,15 +2,14 @@ import python
/** Whether `mox` or `.StubOutWithMock()` is used in thin module `m`. */
predicate useOfMoxInModule(Module m) {
exists(ModuleObject mox |
mox.getName() = "mox" or mox.getName() = "mox3.mox" |
exists(ControlFlowNode use |
exists(ModuleObject mox | mox.getName() = "mox" or mox.getName() = "mox3.mox" |
exists(ControlFlowNode use |
use.refersTo(mox) and
use.getScope().getEnclosingModule() = m
)
)
or
exists(Call call|
exists(Call call |
call.getFunc().(Attribute).getName() = "StubOutWithMock" and
call.getEnclosingModule() = m
)

55
python/ql/src/analysis/CrossProjectDefinitions.qll
View File

@@ -1,46 +1,42 @@
/**
* Symbols for crosss-project jump-to-definition resolution.
*/
import python
import python
import semmle.python.pointsto.PointsTo
private newtype TSymbol =
TModule(Module m)
or
TModule(Module m) or
TMember(Symbol outer, string part) {
exists(Object o |
outer.resolvesTo() = o |
exists(Object o | outer.resolvesTo() = o |
o.(ModuleObject).hasAttribute(part)
or
o.(ClassObject).hasAttribute(part)
)
}
/** A "symbol" referencing an object in another module
* Symbols are represented by the module name and the dotted name by which the
/**
* A "symbol" referencing an object in another module
* Symbols are represented by the module name and the dotted name by which the
* object would be referred to in that module.
* For example for the code:
* ```
* class C:
* def m(self): pass
* ```
* If the code were in a module `mod`,
* If the code were in a module `mod`,
* then symbol for the method `m` would be "mod/C.m"
*/
class Symbol extends TSymbol {
string toString() {
exists(Module m |
this = TModule(m) and result = m.getName()
)
or
exists(Module m | this = TModule(m) and result = m.getName())
or
exists(TModule outer, string part |
this = TMember(outer, part) and
outer = TModule(_) and
result = outer.(Symbol).toString() + "/" + part
)
or
or
exists(TMember outer, string part |
this = TMember(outer, part) and
outer = TMember(_, _) and
@@ -52,8 +48,7 @@ class Symbol extends TSymbol {
AstNode find() {
this = TModule(result)
or
exists(Symbol s, string name |
this = TMember(s, name) |
exists(Symbol s, string name | this = TMember(s, name) |
exists(ClassObject cls |
s.resolvesTo() = cls and
cls.attributeRefersTo(name, _, result.getAFlowNode())
@@ -66,47 +61,43 @@ class Symbol extends TSymbol {
)
}
/** Find the class or module `Object` that this `Symbol` refers to, if
/**
* Find the class or module `Object` that this `Symbol` refers to, if
* this `Symbol` refers to a class or module.
*/
Object resolvesTo() {
this = TModule(result.(ModuleObject).getModule())
or
exists(Symbol s, string name, Object o |
this = TMember(s, name) and
this = TMember(s, name) and
o = s.resolvesTo() and
result = attribute_in_scope(o, name)
)
}
/** Gets the `Module` for the module part of this `Symbol`.
/**
* Gets the `Module` for the module part of this `Symbol`.
* For example, this would return the `os` module for the `Symbol` "os/environ".
*/
Module getModule() {
this = TModule(result)
or
exists(Symbol outer |
this = TMember(outer, _) and result = outer.getModule()
)
exists(Symbol outer | this = TMember(outer, _) and result = outer.getModule())
}
/** Gets the `Symbol` that is the named member of this `Symbol`. */
Symbol getMember(string name) {
result = TMember(this, name)
}
Symbol getMember(string name) { result = TMember(this, name) }
}
/* Helper for `Symbol`.resolvesTo() */
private Object attribute_in_scope(Object obj, string name) {
exists(ClassObject cls |
cls = obj |
exists(ClassObject cls | cls = obj |
cls.lookupAttribute(name) = result and result.(ControlFlowNode).getScope() = cls.getPyClass()
)
or
exists(ModuleObject mod |
mod = obj |
mod.attr(name) = result and result.(ControlFlowNode).getScope() = mod.getModule()
and not result.(ControlFlowNode).isEntryNode()
exists(ModuleObject mod | mod = obj |
mod.attr(name) = result and
result.(ControlFlowNode).getScope() = mod.getModule() and
not result.(ControlFlowNode).isEntryNode()
)
}

176
python/ql/src/analysis/DefinitionTracking.qll
View File

@@ -1,35 +1,22 @@
/**
* Definition tracking for jump-to-defn query.
*/
import python
import python
import semmle.python.pointsto.PointsTo
private newtype TDefinition =
TLocalDefinition(AstNode a) {
a instanceof Expr or a instanceof Stmt or a instanceof Module
}
TLocalDefinition(AstNode a) { a instanceof Expr or a instanceof Stmt or a instanceof Module }
/** A definition for the purposes of jump-to-definition. */
class Definition extends TLocalDefinition {
string toString() { result = "Definition " + this.getAstNode().getLocation().toString() }
AstNode getAstNode() { this = TLocalDefinition(result) }
string toString() {
result = "Definition " + this.getAstNode().getLocation().toString()
}
AstNode getAstNode() {
this = TLocalDefinition(result)
}
Module getModule() {
result = this.getAstNode().getScope().getEnclosingModule()
}
Location getLocation() {
result = this.getAstNode().getLocation()
}
Module getModule() { result = this.getAstNode().getScope().getEnclosingModule() }
Location getLocation() { result = this.getAstNode().getLocation() }
}
private predicate jump_to_defn(ControlFlowNode use, Definition defn) {
@@ -112,7 +99,7 @@ predicate uni_edged_phi_defn(SingleSuccessorGuard uniphi, Definition defn) {
ssa_variable_defn(uniphi.getInput(), defn)
}
pragma [noinline]
pragma[noinline]
private predicate ssa_node_defn(EssaNodeDefinition def, Definition defn) {
assignment_jump_to_defn(def, defn)
or
@@ -130,7 +117,7 @@ private predicate assignment_jump_to_defn(AssignmentDefinition def, Definition d
defn = TLocalDefinition(def.getValue().getNode())
}
pragma [noinline]
pragma[noinline]
private predicate ssa_node_refinement_defn(EssaNodeRefinement def, Definition defn) {
method_callsite_defn(def, defn)
or
@@ -147,16 +134,13 @@ private predicate ssa_node_refinement_defn(EssaNodeRefinement def, Definition de
uni_edged_phi_defn(def, defn)
}
/* Definition for parameter. `def foo(param): ...` */
private predicate parameter_defn(ParameterDefinition def, Definition defn) {
defn.getAstNode() = def.getDefiningNode().getNode()
}
/* Definition for deletion: `del name` */
private predicate delete_defn(DeletionDefinition def, Definition defn) {
none()
}
private predicate delete_defn(DeletionDefinition def, Definition defn) { none() }
/* Implicit "defn" of the names of submodules at the start of an `__init__.py` file. */
private predicate implicit_submodule_defn(ImplicitSubModuleDefinition def, Definition defn) {
@@ -165,13 +149,16 @@ private predicate implicit_submodule_defn(ImplicitSubModuleDefinition def, Defin
mod = package.submodule(def.getSourceVariable().getName()) and
defn.getAstNode() = mod.getModule()
)
}
/* Helper for scope_entry_value_transfer(...).
* Transfer of values from the callsite to the callee, for enclosing variables, but not arguments/parameters
/*
* Helper for scope_entry_value_transfer(...).
* Transfer of values from the callsite to the callee, for enclosing variables, but not arguments/parameters
*/
private predicate scope_entry_value_transfer_at_callsite(EssaVariable pred_var, ScopeEntryDefinition succ_def) {
private predicate scope_entry_value_transfer_at_callsite(
EssaVariable pred_var, ScopeEntryDefinition succ_def
) {
exists(CallNode callsite, FunctionObject f |
f.getACall() = callsite and
pred_var.getSourceVariable() = succ_def.getSourceVariable() and
@@ -181,8 +168,7 @@ private predicate scope_entry_value_transfer_at_callsite(EssaVariable pred_var,
}
/* Model the transfer of values at scope-entry points. Transfer from `pred_var, pred_context` to `succ_def, succ_context` */
private
predicate scope_entry_value_transfer(EssaVariable pred_var, ScopeEntryDefinition succ_def) {
private predicate scope_entry_value_transfer(EssaVariable pred_var, ScopeEntryDefinition succ_def) {
BaseFlow::scope_entry_value_transfer_from_earlier(pred_var, _, succ_def, _)
or
scope_entry_value_transfer_at_callsite(pred_var, succ_def)
@@ -191,8 +177,7 @@ predicate scope_entry_value_transfer(EssaVariable pred_var, ScopeEntryDefinition
}
/* Helper for scope_entry_value_transfer */
private
predicate class_entry_value_transfer(EssaVariable pred_var, ScopeEntryDefinition succ_def) {
private predicate class_entry_value_transfer(EssaVariable pred_var, ScopeEntryDefinition succ_def) {
exists(ImportTimeScope scope, ControlFlowNode class_def |
class_def = pred_var.getAUse() and
scope.entryEdge(class_def, succ_def.getDefiningNode()) and
@@ -201,7 +186,7 @@ predicate class_entry_value_transfer(EssaVariable pred_var, ScopeEntryDefinition
}
/* Definition for implicit variable declarations at scope-entry. */
pragma [noinline]
pragma[noinline]
private predicate scope_entry_defn(ScopeEntryDefinition def, Definition defn) {
/* Transfer from another scope */
exists(EssaVariable var |
@@ -210,10 +195,12 @@ private predicate scope_entry_defn(ScopeEntryDefinition def, Definition defn) {
)
}
/* Definition for a variable (possibly) redefined by a call:
/*
* Definition for a variable (possibly) redefined by a call:
* Just assume that call does not define variable
*/
pragma [noinline]
pragma[noinline]
private predicate callsite_defn(CallsiteRefinement def, Definition defn) {
ssa_variable_defn(def.getInput(), defn)
}
@@ -225,22 +212,27 @@ private predicate method_callsite_defn(MethodCallsiteRefinement def, Definition
}
/** Helpers for import_star_defn */
pragma [noinline]
private predicate module_and_name_for_import_star(ModuleObject mod, string name, ImportStarRefinement def) {
pragma[noinline]
private predicate module_and_name_for_import_star(
ModuleObject mod, string name, ImportStarRefinement def
) {
exists(ImportStarNode im_star |
module_and_name_for_import_star_helper(mod, name, im_star, def) and
mod.exports(name)
)
}
pragma [noinline]
private predicate module_and_name_for_import_star_helper(ModuleObject mod, string name, ImportStarNode im_star, ImportStarRefinement def) {
pragma[noinline]
private predicate module_and_name_for_import_star_helper(
ModuleObject mod, string name, ImportStarNode im_star, ImportStarRefinement def
) {
im_star = def.getDefiningNode() and
im_star.getModule().refersTo(mod) and
name = def.getSourceVariable().getName()
}
/** Holds if `def` is technically a defn of `var`, but the `from ... import *` does not in fact define `var` */
pragma [noinline]
/** Holds if `def` is technically a defn of `var`, but the `from ... import *` does not in fact define `var` */
pragma[noinline]
private predicate variable_not_redefined_by_import_star(EssaVariable var, ImportStarRefinement def) {
var = def.getInput() and
exists(ModuleObject mod |
@@ -251,8 +243,7 @@ private predicate variable_not_redefined_by_import_star(EssaVariable var, Import
/* Definition for `from ... import *` */
private predicate import_star_defn(ImportStarRefinement def, Definition defn) {
exists(ModuleObject mod, string name |
module_and_name_for_import_star(mod, name, def) |
exists(ModuleObject mod, string name | module_and_name_for_import_star(mod, name, def) |
/* Attribute from imported module */
scope_jump_to_defn_attribute(mod.getModule(), name, defn)
)
@@ -275,18 +266,20 @@ private predicate argument_defn(ArgumentRefinement def, Definition defn) {
}
/** Attribute deletions have no effect as far as value tracking is concerned. */
pragma [noinline]
pragma[noinline]
private predicate attribute_delete_defn(EssaAttributeDeletion def, Definition defn) {
ssa_variable_defn(def.getInput(), defn)
}
/* Definition flow for attributes. These mirror the "normal" defn predicates.
/*
* Definition flow for attributes. These mirror the "normal" defn predicates.
* For each defn predicate `xxx_defn(XXX def, Definition defn)`
* There is an equivalent predicate that tracks the values in attributes:
* `xxx_jump_to_defn_attribute(XXX def, string name, Definition defn)`
* */
*/
/** INTERNAL -- Public for testing only.
/**
* INTERNAL -- Public for testing only.
* Holds if the attribute `name` of the ssa variable `var` refers to (`value`, `cls`, `origin`)
*/
predicate ssa_variable_jump_to_defn_attribute(EssaVariable var, string name, Definition defn) {
@@ -316,7 +309,9 @@ private predicate ssa_phi_jump_to_defn_attribute(PhiFunction phi, string name, D
/** Helper for ssa_defn_jump_to_defn_attribute */
pragma[noinline]
private predicate ssa_node_jump_to_defn_attribute(EssaNodeDefinition def, string name, Definition defn) {
private predicate ssa_node_jump_to_defn_attribute(
EssaNodeDefinition def, string name, Definition defn
) {
assignment_jump_to_defn_attribute(def, name, defn)
or
self_parameter_jump_to_defn_attribute(def, name, defn)
@@ -326,14 +321,18 @@ private predicate ssa_node_jump_to_defn_attribute(EssaNodeDefinition def, string
/** Helper for ssa_defn_jump_to_defn_attribute */
pragma[noinline]
private predicate ssa_node_refinement_jump_to_defn_attribute(EssaNodeRefinement def, string name, Definition defn) {
private predicate ssa_node_refinement_jump_to_defn_attribute(
EssaNodeRefinement def, string name, Definition defn
) {
attribute_assignment_jump_to_defn_attribute(def, name, defn)
or
argument_jump_to_defn_attribute(def, name, defn)
}
pragma[noinline]
private predicate scope_entry_jump_to_defn_attribute(ScopeEntryDefinition def, string name, Definition defn) {
private predicate scope_entry_jump_to_defn_attribute(
ScopeEntryDefinition def, string name, Definition defn
) {
exists(EssaVariable var |
scope_entry_value_transfer(var, def) and
ssa_variable_jump_to_defn_attribute(var, name, defn)
@@ -342,9 +341,10 @@ private predicate scope_entry_jump_to_defn_attribute(ScopeEntryDefinition def, s
private predicate scope_jump_to_defn_attribute(ImportTimeScope s, string name, Definition defn) {
exists(EssaVariable var |
BaseFlow::reaches_exit(var) and var.getScope() = s and
BaseFlow::reaches_exit(var) and
var.getScope() = s and
var.getName() = name
|
|
ssa_variable_defn(var, defn)
)
}
@@ -357,22 +357,23 @@ private predicate jump_to_defn_attribute(ControlFlowNode use, string name, Defin
)
or
/* Instance attributes */
exists(ClassObject cls |
use.refersTo(_, cls, _) |
exists(ClassObject cls | use.refersTo(_, cls, _) |
scope_jump_to_defn_attribute(cls.getPyClass(), name, defn)
)
or
/* Super attributes */
exists(AttrNode f, SuperBoundMethod sbm, Object function |
use = f.getObject(name) and
f.refersTo(sbm) and function = sbm.getFunction(_) and
f.refersTo(sbm) and
function = sbm.getFunction(_) and
function.getOrigin() = defn.getAstNode()
)
or
/* Class or module attribute */
exists(Object obj, Scope scope |
use.refersTo(obj) and
scope_jump_to_defn_attribute(scope, name, defn) |
scope_jump_to_defn_attribute(scope, name, defn)
|
obj.(ClassObject).getPyClass() = scope
or
obj.(PythonModuleObject).getModule() = scope
@@ -382,18 +383,23 @@ private predicate jump_to_defn_attribute(ControlFlowNode use, string name, Defin
}
pragma[noinline]
private predicate assignment_jump_to_defn_attribute(AssignmentDefinition def, string name, Definition defn) {
private predicate assignment_jump_to_defn_attribute(
AssignmentDefinition def, string name, Definition defn
) {
jump_to_defn_attribute(def.getValue(), name, defn)
}
pragma[noinline]
private predicate attribute_assignment_jump_to_defn_attribute(AttributeAssignment def, string name, Definition defn) {
private predicate attribute_assignment_jump_to_defn_attribute(
AttributeAssignment def, string name, Definition defn
) {
defn.getAstNode() = def.getDefiningNode().getNode() and name = def.getName()
or
ssa_variable_jump_to_defn_attribute(def.getInput(), name, defn) and not name = def.getName()
}
/** Holds if `def` defines the attribute `name`
/**
* Holds if `def` defines the attribute `name`
* `def` takes the form `setattr(use, "name")` where `use` is the input to the defn.
*/
private predicate sets_attribute(ArgumentRefinement def, string name) {
@@ -406,31 +412,37 @@ private predicate sets_attribute(ArgumentRefinement def, string name) {
}
pragma[noinline]
private predicate argument_jump_to_defn_attribute(ArgumentRefinement def, string name, Definition defn) {
if sets_attribute(def, name) then
jump_to_defn(def.getDefiningNode().(CallNode).getArg(2), defn)
else
ssa_variable_jump_to_defn_attribute(def.getInput(), name, defn)
private predicate argument_jump_to_defn_attribute(
ArgumentRefinement def, string name, Definition defn
) {
if sets_attribute(def, name)
then jump_to_defn(def.getDefiningNode().(CallNode).getArg(2), defn)
else ssa_variable_jump_to_defn_attribute(def.getInput(), name, defn)
}
/** Gets the (temporally) preceding variable for "self", e.g. `def` is in method foo() and `result` is in `__init__()`. */
/** Gets the (temporally) preceding variable for "self", e.g. `def` is in method foo() and `result` is in `__init__()`. */
private EssaVariable preceding_self_variable(ParameterDefinition def) {
def.isSelf() and
exists(Function preceding, Function method |
method = def.getScope() and
method = def.getScope() and
// Only methods
preceding.isMethod() and preceding.precedes(method) and
BaseFlow::reaches_exit(result) and result.getSourceVariable().(Variable).isSelf() and
preceding.isMethod() and
preceding.precedes(method) and
BaseFlow::reaches_exit(result) and
result.getSourceVariable().(Variable).isSelf() and
result.getScope() = preceding
)
}
pragma [noinline]
private predicate self_parameter_jump_to_defn_attribute(ParameterDefinition def, string name, Definition defn) {
pragma[noinline]
private predicate self_parameter_jump_to_defn_attribute(
ParameterDefinition def, string name, Definition defn
) {
ssa_variable_jump_to_defn_attribute(preceding_self_variable(def), name, defn)
}
/** Gets a definition for 'use'.
/**
* Gets a definition for 'use'.
* This exists primarily for testing use `getPreferredDefinition()` instead.
*/
Definition getADefinition(Expr use) {
@@ -441,7 +453,8 @@ Definition getADefinition(Expr use) {
not result = TLocalDefinition(use)
}
/** Gets the unique definition for 'use', if one can be found.
/**
* Gets the unique definition for 'use', if one can be found.
* Helper for the jump-to-definition query.
*/
Definition getUniqueDefinition(Expr use) {
@@ -452,18 +465,13 @@ Definition getUniqueDefinition(Expr use) {
not result = TLocalDefinition(use)
}
/** Helper class to get suitable locations for attributes */
class NiceLocationExpr extends @py_expr {
string toString() {
result = this.(Expr).toString()
}
string toString() { result = this.(Expr).toString() }
predicate hasLocationInfo(string f, int bl, int bc, int el, int ec) {
/* Attribute location for x.y is that of 'y' so that url does not overlap with that of 'x' */
exists(int abl, int abc |
this.(Attribute).getLocation().hasLocationInfo(f, abl, abc, el, ec) |
/* Attribute location for x.y is that of 'y' so that url does not overlap with that of 'x' */
exists(int abl, int abc | this.(Attribute).getLocation().hasLocationInfo(f, abl, abc, el, ec) |
bl = el and bc = ec - this.(Attribute).getName().length() + 1
)
or
@@ -477,10 +485,8 @@ class NiceLocationExpr extends @py_expr {
exists(string name |
name = this.(ImportMember).getName() and
this.(ImportMember).getLocation().hasLocationInfo(f, _, _, el, ec) and
bl = el and bc = ec-name.length()+1
bl = el and
bc = ec - name.length() + 1
)
}
}