Python: Add better class support, including inheritance.

This commit is contained in:
Mark Shannon
2019-03-20 14:35:41 +00:00
parent 5a46df2132
commit bf692f4aad
11 changed files with 570 additions and 55 deletions

View File

@@ -5,6 +5,7 @@ private import semmle.python.objects.TObject
private import semmle.python.objects.ObjectInternal
private import semmle.python.pointsto.PointsTo2
private import semmle.python.pointsto.PointsToContext2
private import semmle.python.pointsto.MRO2
private import semmle.python.types.Builtins
@@ -20,40 +21,37 @@ abstract class ClassObjectInternal extends ObjectInternal {
override predicate notClass() { none() }
override predicate isComparable() {
any()
}
override predicate notComparable() {
override int intValue() {
none()
}
override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) {
// TO DO .. Result should (in most cases) be an instance
override string strValue() {
none()
}
string getName() {
result = this.getClassDeclaration().getName()
}
abstract predicate attribute(string name, ObjectInternal value, CfgOrigin origin);
}
class PythonClassObjectInternal extends ClassObjectInternal, TPythonClassObject {
Class getScope() {
exists(ClassDef def |
this = TPythonClassObject(def.getAFlowNode()) and
result = def.getDefinedClass()
exists(ClassExpr expr |
this = TPythonClassObject(expr.getAFlowNode()) and
result = expr.getInnerScope()
)
}
override string toString() {
result = this.getScope().toString()
result = "class " + this.getScope().getName()
}
override predicate introduced(ControlFlowNode node, PointsToContext2 context) {
exists(DefinitionNode def |
this = TPythonClassObject(def) and
node = def.getValue() and
context.appliesTo(node)
)
this = TPythonClassObject(node) and context.appliesTo(node)
}
override ClassDecl getClassDeclaration() {
@@ -61,7 +59,7 @@ class PythonClassObjectInternal extends ClassObjectInternal, TPythonClassObject
}
override ObjectInternal getClass() {
result = TBuiltinClassObject(Builtin::special("FunctionType"))
result = Types::getMetaClass(this)
}
override Builtin getBuiltin() {
@@ -80,6 +78,26 @@ class PythonClassObjectInternal extends ClassObjectInternal, TPythonClassObject
)
}
override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) {
exists(ClassObjectInternal decl |
decl = Types::getMro(this).findDeclaringClass(name) |
Types::declaredAttribute(decl, name, value, origin)
)
}
override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) {
// TO DO .. Result should (in most cases) be an instance
none()
}
override predicate isComparable() {
any()
}
override predicate notComparable() {
none()
}
}
class BuiltinClassObjectInternal extends ClassObjectInternal, TBuiltinClassObject {
@@ -108,4 +126,25 @@ class BuiltinClassObjectInternal extends ClassObjectInternal, TBuiltinClassObjec
none()
}
override predicate calleeAndOffset(Function scope, int paramOffset) {
none()
}
override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) {
value.getBuiltin() = this.getBuiltin().getMember(name) and origin = CfgOrigin::unknown()
}
override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) {
// TO DO .. Result should (in most cases) be an instance
none()
}
override predicate isComparable() {
any()
}
override predicate notComparable() {
none()
}
}

View File

@@ -48,6 +48,10 @@ abstract class BooleanObjectInternal extends ObjectInternal {
none()
}
override predicate calleeAndOffset(Function scope, int paramOffset) {
none()
}
}
class TrueObjectInternal extends BooleanObjectInternal, TTrue {
@@ -64,6 +68,14 @@ class TrueObjectInternal extends BooleanObjectInternal, TTrue {
node.(NameNode).getId() = "True" and context.appliesTo(node)
}
override int intValue() {
result = 1
}
override string strValue() {
none()
}
}
class FalseObjectInternal extends BooleanObjectInternal, TFalse {
@@ -80,6 +92,14 @@ class FalseObjectInternal extends BooleanObjectInternal, TFalse {
node.(NameNode).getId() = "False" and context.appliesTo(node)
}
override int intValue() {
result = 0
}
override string strValue() {
none()
}
}
class NoneObjectInternal extends ObjectInternal, TNone {
@@ -131,6 +151,18 @@ class NoneObjectInternal extends ObjectInternal, TNone {
none()
}
override int intValue() {
none()
}
override string strValue() {
none()
}
override predicate calleeAndOffset(Function scope, int paramOffset) {
none()
}
}
@@ -184,12 +216,20 @@ class IntObjectInternal extends ObjectInternal, TInt {
this = TInt(result)
}
override string strValue() {
none()
}
override boolean booleanValue() {
this.intValue() = 0 and result = false
or
this.intValue() != 0 and result = true
}
override predicate calleeAndOffset(Function scope, int paramOffset) {
none()
}
}
@@ -239,6 +279,10 @@ class StringObjectInternal extends ObjectInternal, TString {
none()
}
override int intValue() {
none()
}
override string strValue() {
this = TString(result)
}
@@ -249,6 +293,10 @@ class StringObjectInternal extends ObjectInternal, TString {
this.strValue() != "" and result = true
}
override predicate calleeAndOffset(Function scope, int paramOffset) {
none()
}
}

View File

@@ -0,0 +1,100 @@
import python
private import semmle.python.objects.TObject
private import semmle.python.objects.ObjectInternal
private import semmle.python.pointsto.PointsTo2
private import semmle.python.pointsto.PointsToContext2
private import semmle.python.types.Builtins
class InstanceInternal extends TInstance, ObjectInternal {
override string toString() {
result = "instance of " + this.getClass().(ClassObjectInternal).getClassDeclaration().getName()
}
/** The boolean value of this object, if it has one */
override boolean booleanValue() {
//this.getClass().instancesAlways(result)
none()
}
/** Holds if this object may be true or false when evaluated as a bool */
override predicate maybe() {
// this.getClass().instancesMaybe()
any()
}
override predicate introduced(ControlFlowNode node, PointsToContext2 context) {
this = TInstance(node, _, context)
}
/** Gets the class declaration for this object, if it is a declared class. */
override ClassDecl getClassDeclaration() {
none()
}
override predicate isClass() {
none()
}
override predicate notClass() {
any()
}
override ObjectInternal getClass() {
this = TInstance(_, result, _)
}
/** Holds if whatever this "object" represents can be meaningfully analysed for
* truth or false in comparisons. For example, `None` or `int` can be, but `int()`
* or an unknown string cannot.
*/
override predicate isComparable() {
none()
}
/** The negation of `isComparable()` */
override predicate notComparable() {
any()
}
/** Gets the `Builtin` for this object, if any.
* All objects (except unknown and undefined values) should return
* exactly one result for either this method or `getOrigin()`.
*/
override Builtin getBuiltin() {
none()
}
/** Gets a control flow node that represents the source origin of this
* objects.
* All objects (except unknown and undefined values) should return
* exactly one result for either this method or `getBuiltin()`.
*/
override ControlFlowNode getOrigin() {
this = TInstance(result, _, _)
}
/** Holds if `obj` is the result of calling `this` and `origin` is
* the origin of `obj`.
*/
override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) {
// In general instances aren't callable, but some are...
// TO DO -- Handle cases where class overrides __call__
none()
}
override int intValue() {
none()
}
override string strValue() {
none()
}
override predicate calleeAndOffset(Function scope, int paramOffset) {
none()
}
}

View File

@@ -12,8 +12,6 @@ abstract class ModuleObjectInternal extends ObjectInternal {
abstract Module getSourceModule();
abstract predicate isBuiltin();
override predicate callResult(PointsToContext2 callee, ObjectInternal obj, CfgOrigin origin) {
// Modules aren't callable
none()
@@ -73,8 +71,16 @@ class BuiltinModuleObjectInternal extends ModuleObjectInternal, TBuiltinModuleOb
none()
}
override predicate isBuiltin() {
any()
override int intValue() {
none()
}
override string strValue() {
none()
}
override predicate calleeAndOffset(Function scope, int paramOffset) {
none()
}
}
@@ -142,10 +148,6 @@ class PackageObjectInternal extends ModuleObjectInternal, TPackageObject {
)
}
override predicate isBuiltin() {
none()
}
ModuleObjectInternal submodule(string name) {
result.getName() = this.getName() + "." + name
}
@@ -157,6 +159,18 @@ class PackageObjectInternal extends ModuleObjectInternal, TPackageObject {
bl = 0 and bc = 0 and el = 0 and ec = 0
}
override int intValue() {
none()
}
override string strValue() {
none()
}
override predicate calleeAndOffset(Function scope, int paramOffset) {
none()
}
}
class PythonModuleObjectInternal extends ModuleObjectInternal, TPythonModule {
@@ -211,7 +225,15 @@ class PythonModuleObjectInternal extends ModuleObjectInternal, TPythonModule {
result = TPythonModule(this.getSourceModule())
}
override predicate isBuiltin() {
override int intValue() {
none()
}
override string strValue() {
none()
}
override predicate calleeAndOffset(Function scope, int paramOffset) {
none()
}

View File

@@ -8,7 +8,7 @@ import semmle.python.objects.Modules
import semmle.python.objects.Classes
import semmle.python.objects.Constants
abstract class ObjectInternal extends TObject {
class ObjectInternal extends TObject {
abstract string toString();
@@ -63,18 +63,18 @@ abstract class ObjectInternal extends TObject {
/** The integer value of things that have integer values.
* That is, ints and bools.
*/
int intValue() {
none()
}
abstract int intValue();
/** The integer value of things that have integer values.
* That is, strings.
*/
string strValue() {
none()
}
abstract string strValue();
predicate calleeAndOffset(Function scope, int paramOffset) { none() }
abstract predicate calleeAndOffset(Function scope, int paramOffset);
final predicate isBuiltin() {
exists(this.getBuiltin())
}
}
@@ -144,6 +144,14 @@ class PythonFunctionObjectInternal extends ObjectInternal, TPythonFunctionObject
scope = this.getScope() and paramOffset = 0
}
override int intValue() {
none()
}
override string strValue() {
none()
}
}
/// BOUND METHODS
@@ -200,6 +208,18 @@ class BuiltinFunctionObjectInternal extends ObjectInternal, TBuiltinFunctionObje
none()
}
override int intValue() {
none()
}
override string strValue() {
none()
}
override predicate calleeAndOffset(Function scope, int paramOffset) {
none()
}
}
@@ -252,6 +272,18 @@ class BuiltinMethodObjectInternal extends ObjectInternal, TBuiltinMethodObject {
none()
}
override int intValue() {
none()
}
override string strValue() {
none()
}
override predicate calleeAndOffset(Function scope, int paramOffset) {
none()
}
}
@@ -308,6 +340,18 @@ class BuiltinOpaqueObjectInternal extends ObjectInternal, TBuiltinOpaqueObject {
none()
}
override int intValue() {
none()
}
override string strValue() {
none()
}
override predicate calleeAndOffset(Function scope, int paramOffset) {
none()
}
}
private predicate callee_for_object(PointsToContext2 callee, ObjectInternal obj) {
@@ -318,18 +362,12 @@ private predicate callee_for_object(PointsToContext2 callee, ObjectInternal obj)
}
class UnknownClassInternal extends ObjectInternal, TUnknownClass {
class UnknownClassInternal extends ClassObjectInternal, TUnknownClass {
override string toString() {
none()
}
override boolean booleanValue() {
none()
}
override predicate maybe() { any() }
override ClassDecl getClassDeclaration() {
result = Builtin::unknownType()
}
@@ -367,6 +405,14 @@ class UnknownClassInternal extends ObjectInternal, TUnknownClass {
none()
}
override predicate calleeAndOffset(Function scope, int paramOffset) {
none()
}
override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) {
none()
}
}
class UnknownInternal extends ObjectInternal, TUnknown {
@@ -418,6 +464,18 @@ class UnknownInternal extends ObjectInternal, TUnknown {
none()
}
override int intValue() {
none()
}
override string strValue() {
none()
}
override predicate calleeAndOffset(Function scope, int paramOffset) {
none()
}
}
class UndefinedInternal extends ObjectInternal, TUndefined {
@@ -471,6 +529,18 @@ class UndefinedInternal extends ObjectInternal, TUndefined {
none()
}
override int intValue() {
none()
}
override string strValue() {
none()
}
override predicate calleeAndOffset(Function scope, int paramOffset) {
none()
}
}
module ObjectInternal {
@@ -490,7 +560,7 @@ module ObjectInternal {
result = TUnknown()
}
ObjectInternal unknownClass() {
ClassObjectInternal unknownClass() {
result = TUnknownClass()
}

View File

@@ -1,5 +1,8 @@
import python
private import semmle.python.types.Builtins
private import semmle.python.objects.ObjectInternal
private import semmle.python.pointsto.PointsTo2
private import semmle.python.pointsto.PointsToContext2
newtype TObject =
TBuiltinClassObject(Builtin bltn) {
@@ -22,8 +25,8 @@ newtype TObject =
callable.getNode() instanceof CallableExpr
}
or
TPythonClassObject(ControlFlowNode classdef) {
classdef.getNode() instanceof ClassDef
TPythonClassObject(ControlFlowNode classexpr) {
classexpr.getNode() instanceof ClassExpr
}
or
TPackageObject(Folder f)
@@ -67,31 +70,52 @@ newtype TObject =
s = quoted_string.regexpCapture("[bu]'([\\s\\S]*)'", 1)
)
}
or
TInstance(CallNode instantiation, ClassObjectInternal cls, PointsToContext2 context) {
PointsTo2::points_to(instantiation.getFunction(), context, cls, _) and
normal_class(cls)
}
private predicate is_power_2(int n) {
n = 1 or
exists(int half | is_power_2(half) and n = half*2)
}
private predicate normal_class(ClassObjectInternal cls) {
exists(Builtin bltn |
bltn = cls.getBuiltin() |
not bltn = Builtin::special(_)
)
//or
//cls.getMro().inheritsFromType(false)
}
library class ClassDecl extends @py_object {
ClassDecl() {
this.(Builtin).isClass() and not this = Builtin::unknownType()
or
this.(ControlFlowNode).getNode() instanceof ClassDef
this.(ControlFlowNode).getNode() instanceof ClassExpr
}
string toString() {
result = "ClassDecl"
}
private Class getClass() {
result = this.(ControlFlowNode).getNode().(ClassExpr).getInnerScope()
}
predicate declaresAttribute(string name) {
exists(this.(Builtin).getMember(name))
or
exists(Class cls |
cls = this.(ControlFlowNode).getNode().(ClassDef).getDefinedClass() and
exists(SsaVariable var | name = var.getId() and var.getAUse() = cls.getANormalExit())
)
exists(SsaVariable var | name = var.getId() and var.getAUse() = this.getClass().getANormalExit())
}
string getName() {
result = this.(Builtin).getName()
or
result = this.getClass().getName()
}
}

View File

@@ -1,8 +1,11 @@
import python
private import semmle.python.objects.TObject
private import semmle.python.objects.ObjectInternal
private import semmle.python.pointsto.Filters
private import semmle.python.pointsto.PointsToContext2
private import semmle.python.pointsto.MRO2
private import semmle.python.types.Builtins
/* Use this version for speed */
library class CfgOrigin extends @py_object {
@@ -818,3 +821,194 @@ module Conditionals {
}
}
module Types {
int base_count(ClassObjectInternal cls) {
cls = ObjectInternal::builtin("object") and result = 0
or
exists(cls.getBuiltin()) and cls != ObjectInternal::builtin("object") and result = 1
or
exists(Class pycls |
pycls = cls.(PythonClassObjectInternal).getScope() |
result = strictcount(pycls.getABase())
or
isNewStyle(cls) and not exists(pycls.getABase()) and result = 1
or
isOldStyle(cls) and not exists(pycls.getABase()) and result = 0
)
}
ClassObjectInternal getBase(ClassObjectInternal cls, int n) {
result.getBuiltin() = cls.getBuiltin().getBaseClass() and n = 0
or
exists(Class pycls |
pycls = cls.(PythonClassObjectInternal).getScope() |
PointsTo2::points_to(pycls.getBase(n).getAFlowNode(), _, result, _)
or
not exists(pycls.getABase()) and n = 0 and
isNewStyle(cls) and result = ObjectInternal::builtin("object")
)
or
cls = ObjectInternal::unknownClass() and n = 0 and
result = ObjectInternal::builtin("object")
}
predicate isOldStyle(ClassObjectInternal cls) {
//To do...
none()
}
predicate isNewStyle(ClassObjectInternal cls) {
//To do...
any()
}
ClassList getMro(ClassObjectInternal cls) {
isNewStyle(cls) and
result = Mro::newStyleMro(cls)
or
// To do, old-style
none()
}
predicate declaredAttribute(ClassObjectInternal cls, string name, ObjectInternal value, CfgOrigin origin) {
value.getBuiltin() = cls.getBuiltin().getMember(name) and origin = CfgOrigin::unknown()
or
value != ObjectInternal::undefined() and
exists(EssaVariable var |
name = var.getName() and
var.getAUse() = cls.(PythonClassObjectInternal).getScope().getANormalExit() and
PointsTo2::ssa_variable_points_to(var, _, value, origin)
)
}
ClassObjectInternal getMetaClass(PythonClassObjectInternal cls) {
result = declaredMetaClass(cls)
or
hasDeclaredMetaclass(cls) = false and result = getInheritedMetaclass(cls)
}
private ClassObjectInternal declaredMetaClass(PythonClassObjectInternal cls) {
exists(ObjectInternal obj |
PointsTo2::ssa_variable_points_to(metaclass_var(cls.getScope()), _, obj, _) |
result = obj
or
obj = ObjectInternal::unknown() and result = ObjectInternal::unknownClass()
)
or
exists(Builtin meta |
result.getBuiltin() = meta and
meta = cls.getBuiltin().getClass() and
meta.inheritsFromType()
)
or
exists(ControlFlowNode meta |
six_add_metaclass(_, cls, meta) and
PointsTo2::points_to(meta, _, result, _)
)
}
private boolean hasDeclaredMetaclass(PythonClassObjectInternal cls) {
result = has_six_add_metaclass(cls).booleanOr(has_metaclass_var_metaclass(cls))
}
private boolean has_six_add_metaclass(PythonClassObjectInternal cls) {
// TO DO...
none()
}
private boolean has_metaclass_var_metaclass(PythonClassObjectInternal cls) {
exists(ObjectInternal obj |
PointsTo2::ssa_variable_points_to(metaclass_var(cls.getScope()), _, obj, _) |
obj = ObjectInternal::undefined() and result = false
or
obj != ObjectInternal::undefined() and result = true
)
or
exists(Class pycls |
pycls = cls.getScope() and
not exists(metaclass_var(pycls)) and result = false
)
}
private EssaVariable metaclass_var(Class cls) {
result.getASourceUse() = cls.getMetaClass().getAFlowNode()
or
major_version() = 2 and not exists(cls.getMetaClass()) and
result.getName() = "__metaclass__" and
cls.(ImportTimeScope).entryEdge(result.getAUse(), _)
}
/** INTERNAL -- Do not use */
cached predicate six_add_metaclass(CallNode decorator_call, ClassObjectInternal decorated, ControlFlowNode metaclass) {
//TO DO...
none()
//exists(CallNode decorator |
// decorator_call.getArg(0) = decorated and
// decorator = decorator_call.getFunction() and
// decorator.getArg(0) = metaclass |
// PointsTo2::points_to(decorator.getFunction(), _, six_add_metaclass_function(), _)
// or
// exists(ModuleObjectInternal six |
// six.getName() = "six" and
// PointsTo2::points_to(decorator.getFunction().(AttrNode).getObject("add_metaclass"), _, six, _)
// )
//)
}
private ObjectInternal six_add_metaclass_function() {
exists(Module six, FunctionExpr add_metaclass |
add_metaclass.getInnerScope().getName() = "add_metaclass" and
add_metaclass.getScope() = six and
result.getOrigin() = add_metaclass.getAFlowNode()
)
}
private ClassObjectInternal getInheritedMetaclass(ClassObjectInternal cls) {
result = getInheritedMetaclass(cls, 0)
or
// Best guess if base is not a known class
exists(ObjectInternal base |
base = getBase(cls, _) and
result = ObjectInternal::unknownClass() |
base.notClass()
or
base = ObjectInternal::unknownClass()
)
}
private ClassObjectInternal getInheritedMetaclass(ClassObjectInternal cls, int n) {
exists(Class c |
c = cls.(PythonClassObjectInternal).getScope() and
n = count(c.getABase())
|
result = ObjectInternal::builtin("type")
)
or
exists(ClassObjectInternal meta1, ClassObjectInternal meta2 |
meta1 = getMetaClass(getBase(cls, n)) and
meta2 = getInheritedMetaclass(cls, n+1)
|
/* Choose sub-class */
improperSuperType(meta1) = meta2 and result = meta1
or
improperSuperType(meta2) = meta1 and result = meta2
or
/* Make sure we have a metaclass, even if base is unknown */
meta1 = ObjectInternal::unknownClass() and result = ObjectInternal::builtin("type")
or
meta2 = ObjectInternal::unknownClass() and result = meta1
)
}
private ClassObjectInternal improperSuperType(ClassObjectInternal cls) {
result = cls
or
result = improperSuperType(getBase(cls, _))
}
}

View File

@@ -1,10 +1,11 @@
import python
/** Make unknown type visible */
class UnknownType extends ClassObject {
private import semmle.python.objects.ObjectInternal
private import semmle.python.pointsto.PointsTo2
UnknownType() { this = theUnknownType() }
/** Make unknown type visible */
class UnknownType extends UnknownClassInternal {
override string toString() { result = "*UNKNOWN TYPE" }
@@ -12,6 +13,6 @@ class UnknownType extends ClassObject {
}
from ClassObject c
from ClassObjectInternal c
where not c.isBuiltin()
select c.toString(), c.getMro()
select c.toString(), Types::getMro(c)

View File

@@ -0,0 +1 @@
fail

View File

@@ -0,0 +1,8 @@
import python
private import semmle.python.objects.ObjectInternal
from ClassObjectInternal cls, ControlFlowNode f
where cls.introduced(f, _)
select cls.getName(), f

View File

@@ -0,0 +1,8 @@
class Foo(object):
pass
class Bar(object):
pass