Python points-to: Support descriptor protocols, particularly functions.

python/ql/src/semmle/python/objects/Callables.qll

@@ -38,12 +38,16 @@ abstract class CallableObjectInternal extends ObjectInternal {
 
     override predicate attributesUnknown() { none() }
 
+    abstract Function getScope();
+
+    override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() }
+
 }
 
 
 class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFunctionObject {
 
-    Function getScope() {
+    override Function getScope() {
         exists(CallableExpr expr |
             this = TPythonFunctionObject(expr.getAFlowNode()) and
             result = expr.getInnerScope()
@@ -51,7 +55,7 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti
     }
 
     override string toString() {
-        result = this.getScope().toString()
+        result = "Function " + this.getScope().getQualifiedName()
     }
 
     override predicate introduced(ControlFlowNode node, PointsToContext context) {
@@ -103,6 +107,12 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti
         result = this.getScope().getName()
     }
 
+    override boolean isDescriptor() { result = true }
+
+    override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) {
+        value = TBoundMethod(instance, this) and origin = CfgOrigin::unknown()
+    }
+
 }
 
 class BuiltinFunctionObjectInternal extends CallableObjectInternal, TBuiltinFunctionObject {
@@ -176,6 +186,13 @@ class BuiltinFunctionObjectInternal extends CallableObjectInternal, TBuiltinFunc
         )
     }
 
+    override Function getScope() { none() }
+
+    override boolean isDescriptor() { result = false }
+
+    override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() }
+
+
 }
 
 
@@ -218,6 +235,19 @@ class BuiltinMethodObjectInternal extends CallableObjectInternal, TBuiltinMethod
         result = this.getBuiltin().getName()
     }
 
+    override Function getScope() { none() }
+
+    override boolean isDescriptor() { result = true }
+
+    override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) {
+        instance.isClass() = false and
+        value = TBoundMethod(instance, this) and origin = CfgOrigin::unknown()
+        or
+        any(ObjectInternal obj).binds(instance, _, this) and
+        instance.isClass() = true and 
+        value = this and origin = CfgOrigin::unknown()
+    }
+
 }
 
 class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod {
@@ -227,15 +257,15 @@ class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod {
     }
 
     CallableObjectInternal getFunction() {
-        this = TBoundMethod(_, _, result, _)
+        this = TBoundMethod(_, result)
     }
 
     ObjectInternal getSelf() {
-        this = TBoundMethod(_, result, _, _)
+        this = TBoundMethod(result, _)
     }
 
     override string toString() {
-        result = "bound method '" + this.getFunction().getName() + "' of " + this.getSelf().toString()
+        result = "Method(" + this.getFunction() + ", " + this.getSelf() + ")"
     }
 
     override ObjectInternal getClass() {
@@ -243,7 +273,7 @@ class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod {
     }
 
     override predicate introduced(ControlFlowNode node, PointsToContext context) {
-        this = TBoundMethod(node, _, _, context)
+        none()
     }
 
     override boolean isComparable() { result = false }
@@ -253,7 +283,7 @@ class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod {
     }
 
     override predicate callResult(ObjectInternal obj, CfgOrigin origin) {
-        none()
+        this.getFunction().callResult(obj, origin)
     }
 
     override ControlFlowNode getOrigin() {
@@ -268,6 +298,15 @@ class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod {
         result = this.getFunction().getName()
     }
 
+
+    override Function getScope() { 
+        result = this.getFunction().getScope()
+    }
+
+    override boolean isDescriptor() { result = false }
+
+    override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() }
+
 }
 
 class ClassMethodObjectInternal extends ObjectInternal, TClassMethod {
@@ -317,6 +356,22 @@ class ClassMethodObjectInternal extends ObjectInternal, TClassMethod {
 
     override predicate attributesUnknown() { none() }
 
+    override boolean isDescriptor() { result = true }
+
+    override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) {
+        any(ObjectInternal obj).binds(instance, _, this) and
+        (
+            instance.isClass() = false and
+            value = TBoundMethod(instance.getClass(), this.getFunction())
+            or
+            instance.isClass() = true and
+            value = TBoundMethod(instance, this.getFunction())
+        ) and
+        origin = CfgOrigin::unknown()
+    }
+
+    override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() }
+
 }
 
 class StaticMethodObjectInternal extends ObjectInternal, TStaticMethod {
@@ -366,6 +421,15 @@ class StaticMethodObjectInternal extends ObjectInternal, TStaticMethod {
 
     override predicate attributesUnknown() { none() }
 
+    override boolean isDescriptor() { result = true }
+
+    override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) {
+        any(ObjectInternal obj).binds(instance, _, this) and
+        value = this.getFunction() and origin = CfgOrigin::unknown()
+    }
+
+    override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() }
+
 }
 
 

python/ql/src/semmle/python/objects/Classes.qll

@@ -33,6 +33,17 @@ abstract class ClassObjectInternal extends ObjectInternal {
         result = Types::getMro(this).isSpecial()
     }
 
+    override boolean isDescriptor() { result = false }
+
+    override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() }
+
+    override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) {
+        instance = this and
+        PointsTo2::attributeRequired(this, name) and
+        this.attribute(name, descriptor, _) and
+        descriptor.isDescriptor() = true
+    }
+
 }
 
 class PythonClassObjectInternal extends ClassObjectInternal, TPythonClassObject {

python/ql/src/semmle/python/objects/Constants.qll

@@ -3,6 +3,7 @@ import python
 private import semmle.python.objects.TObject
 private import semmle.python.objects.ObjectInternal
 private import semmle.python.pointsto.PointsTo2
+private import semmle.python.pointsto.MRO2
 private import semmle.python.pointsto.PointsToContext
 private import semmle.python.types.Builtins
 
@@ -54,6 +55,12 @@ abstract class BooleanObjectInternal extends ObjectInternal {
 
     override predicate attributesUnknown() { none() }
 
+    override boolean isDescriptor() { result = false }
+
+    override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() }
+
+    override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() }
+
 }
 
 class TrueObjectInternal extends BooleanObjectInternal, TTrue {
@@ -179,6 +186,12 @@ class NoneObjectInternal extends ObjectInternal, TNone {
         result = Builtin::special("None")
     }
 
+    override boolean isDescriptor() { result = false }
+
+    override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() }
+
+    override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() }
+
 }
 
 
@@ -252,6 +265,12 @@ class IntObjectInternal extends ObjectInternal, TInt {
         result.(Builtin).intValue() = this.intValue()
     }
 
+    override boolean isDescriptor() { result = false }
+
+    override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() }
+
+    override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() }
+
 }
 
 
@@ -324,6 +343,12 @@ class StringObjectInternal extends ObjectInternal, TString {
         result.(Builtin).strValue() = this.strValue()
     }
 
+    override boolean isDescriptor() { result = false }
+
+    override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() }
+
+    override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() }
+
 }
 
 

python/ql/src/semmle/python/objects/Instances.qll

@@ -4,6 +4,7 @@ import python
 private import semmle.python.objects.TObject
 private import semmle.python.objects.ObjectInternal
 private import semmle.python.pointsto.PointsTo2
+private import semmle.python.pointsto.MRO2
 private import semmle.python.pointsto.PointsToContext
 private import semmle.python.types.Builtins
 
@@ -76,17 +77,43 @@ class SpecificInstanceInternal extends TSpecificInstance, ObjectInternal {
     }
 
     override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) {
-        none()
+        PointsTo2::attributeRequired(this, name) and
+        instance_getattr(this, Types::getMro(this.getClass()), name, value, origin)
     }
 
     override predicate attributesUnknown() { any() }
 
+    override boolean isDescriptor() { result = false }
+
+    override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() }
+
+    override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) {
+        this = instance and descriptor.isDescriptor() = true and
+        exists(AttrNode attr |
+            PointsTo2::points_to(attr.getObject(name), _, instance, _) and
+            this.getClass().attribute(name, descriptor, _)
+        )
+    }
+
 }
 
+bindingset[instance, mro, name]
+predicate instance_getattr(ObjectInternal instance, ClassList mro, string name, ObjectInternal value, CfgOrigin origin) {
+    exists(ObjectInternal descriptor, CfgOrigin desc_origin |
+        Types::declaredAttribute(mro.findDeclaringClass(name), name, descriptor, desc_origin) |
+        descriptor.isDescriptor() = false and
+        value = descriptor and origin = desc_origin
+        or
+        descriptor.isDescriptor() = true and
+        descriptor.descriptorGet(instance, value, origin)
+    )
+}
+
+
 class SelfInstanceInternal extends TSelfInstance, ObjectInternal {
 
     override string toString() {
-        result = "self"
+        result = "self instance of " + this.getClass().(ClassObjectInternal).getName()
     }
 
     /** The boolean value of this object, if it has one */
@@ -159,11 +186,25 @@ class SelfInstanceInternal extends TSelfInstance, ObjectInternal {
     }
 
     override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) {
-        none()
+        PointsTo2::attributeRequired(this, name) and
+        instance_getattr(this, Types::getMro(this.getClass()), name, value, origin)
     }
 
     override predicate attributesUnknown() { any() }
 
+    override boolean isDescriptor() { result = false }
+
+    override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() }
+
+    override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) {
+        descriptor.isDescriptor() = true and
+        this = instance and
+        exists(AttrNode attr |
+            PointsTo2::points_to(attr.getObject(name), _, this, _) and
+            this.getClass().attribute(name, descriptor, _)
+        )
+    }
+
 }
 
 /** Represents a value that has a known class, but no other information */
@@ -238,18 +279,32 @@ class UnknownInstanceInternal extends TUnknownInstance, ObjectInternal {
     }
 
     override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) {
-        none()
+        PointsTo2::attributeRequired(this, name) and
+        instance_getattr(this, Types::getMro(this.getClass()), name, value, origin)
     }
 
     override predicate attributesUnknown() { any() }
 
+    override boolean isDescriptor() { result = false }
+
+    override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() }
+
+    override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) {
+        descriptor.isDescriptor() = true and
+        this = instance and
+        exists(AttrNode attr |
+            PointsTo2::points_to(attr.getObject(name), _, this, _) and
+            this.getClass().attribute(name, descriptor, _)
+        )
+    }
+
 }
 
 
 class SuperInstance extends TSuperInstance, ObjectInternal {
 
     override string toString() {
-        result = "super()"
+        result = "super(" + this.getStartClass().toString() + ", " + this.getSelf().toString() + ")"
     }
 
     override boolean booleanValue() { result = true }
@@ -295,12 +350,29 @@ class SuperInstance extends TSuperInstance, ObjectInternal {
 
     override predicate calleeAndOffset(Function scope, int paramOffset) { none() }
 
-    override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { none() }
-
     override predicate attributesUnknown() { none() }
 
+    override boolean isDescriptor() { result = false }
+
+    override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() }
+
+    override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) {
+        PointsTo2::attributeRequired(this, name) and
+        instance_getattr(this.getSelf(), this.getMro(), name, value, origin)
+    }
+
+    private ClassList getMro() {
+        result = Types::getMro(this.getSelf().getClass()).startingAt(this.getStartClass()).getTail()
+    }
+
+    override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) {
+        descriptor.isDescriptor() = true and
+        exists(AttrNode attr |
+            PointsTo2::points_to(attr.getObject(name), _, this, _) and
+            instance = this.getSelf() and
+            Types::declaredAttribute(this.getMro().findDeclaringClass(name), name, descriptor, _)
+        )
+    }
+
 }
 
-
-
-

python/ql/src/semmle/python/objects/Modules.qll

@@ -3,6 +3,7 @@ import python
 private import semmle.python.objects.TObject
 private import semmle.python.objects.ObjectInternal
 private import semmle.python.pointsto.PointsTo2
+private import semmle.python.pointsto.MRO2
 private import semmle.python.pointsto.PointsToContext
 private import semmle.python.types.Builtins
 
@@ -38,6 +39,12 @@ abstract class ModuleObjectInternal extends ObjectInternal {
         result = ObjectInternal::moduleType()
     }
 
+    override boolean isDescriptor() { result = false }
+
+
+    override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() }
+
+    override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() }
 }
 
 class BuiltinModuleObjectInternal extends ModuleObjectInternal, TBuiltinModuleObject {

python/ql/src/semmle/python/objects/ObjectInternal.qll

@@ -2,6 +2,7 @@ import python
 
 private import semmle.python.objects.TObject
 private import semmle.python.pointsto.PointsTo2
+private import semmle.python.pointsto.MRO2
 private import semmle.python.pointsto.PointsToContext
 private import semmle.python.types.Builtins
 import semmle.python.objects.Modules
@@ -86,6 +87,16 @@ class ObjectInternal extends TObject {
         result = this.getBuiltin()
     }
 
+    abstract boolean isDescriptor();
+
+    abstract predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin);
+
+    /** Holds if attribute lookup on this object may "bind" `instance` to `descriptor`.
+     * Here "bind" means that `instance` is passed to the `descriptor.__get__()` method
+     * at runtime. The term "bind" is used as this most likely results in a bound-method.
+     */
+    abstract predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor);
+
 }
 
 
@@ -151,6 +162,12 @@ class BuiltinOpaqueObjectInternal extends ObjectInternal, TBuiltinOpaqueObject {
 
     override predicate attributesUnknown() { none() }
 
+    override boolean isDescriptor() { result = false }
+
+    override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() }
+
+    override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() }
+
 }
 
 
@@ -214,6 +231,12 @@ class UnknownInternal extends ObjectInternal, TUnknown {
 
     override predicate attributesUnknown() { any() }
 
+    override boolean isDescriptor() { result = false }
+
+    override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() }
+
+    override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() }
+
 }
 
 class UndefinedInternal extends ObjectInternal, TUndefined {
@@ -278,6 +301,12 @@ class UndefinedInternal extends ObjectInternal, TUndefined {
 
     override predicate attributesUnknown() { none() }
 
+    override boolean isDescriptor() { none() }
+
+    override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() }
+
+    override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() }
+
 }
 
 module ObjectInternal {

python/ql/src/semmle/python/objects/Sequences.qll

@@ -25,6 +25,12 @@ abstract class SequenceObjectInternal extends ObjectInternal {
         this.length() != 0 and result = true
     }
 
+    override boolean isDescriptor() { result = false }
+
+    override predicate descriptorGet(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() }
+
+    override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() }
+
 }
 
 abstract class TupleObjectInternal extends SequenceObjectInternal {

python/ql/src/semmle/python/objects/TObject.qll

@@ -88,11 +88,12 @@ newtype TObject =
         self_parameter(def, context, cls)
     }
     or
-    TBoundMethod(AttrNode instantiation, ObjectInternal self, CallableObjectInternal function, PointsToContext context) {
-        method_binding(instantiation, self, function, context)
+    TBoundMethod(ObjectInternal self, CallableObjectInternal function) {
+        any(ObjectInternal obj).binds(self, _, function) and
+        function.isDescriptor() = true
     }
     or
-    TUnknownInstance(BuiltinClassObjectInternal cls) { any() }
+    TUnknownInstance(BuiltinClassObjectInternal cls) { cls != ObjectInternal::builtin("super") }
     or
     TSuperInstance(ObjectInternal self, ClassObjectInternal startclass) {
         super_instantiation(_, self, startclass, _)
@@ -162,7 +163,10 @@ predicate super_instantiation(CallNode instantiation, ObjectInternal self, Class
     )
 }
 
+
+bindingset[self, function]
 predicate method_binding(AttrNode instantiation, ObjectInternal self, CallableObjectInternal function, PointsToContext context) {
+    
     exists(ObjectInternal obj, string name |
         receiver(instantiation, context, obj, name) |
         exists(ObjectInternal cls |
@@ -172,9 +176,10 @@ predicate method_binding(AttrNode instantiation, ObjectInternal self, CallableOb
             self = obj
         )
         or
-        exists(SuperInstance sup |
+        exists(SuperInstance sup, ClassObjectInternal decl |
             sup = obj and
-            sup.getStartClass().attribute(name, function, _) and
+            decl = Types::getMro(self.getClass()).startingAt(sup.getStartClass()).findDeclaringClass(name) and
+            Types::declaredAttribute(decl, name, function, _) and
             self = sup.getSelf()
         )
     )
@@ -283,6 +288,7 @@ library class ClassDecl extends @py_object {
         exists(string name |
             this = Builtin::special(name) |
             name = "type" or
+            name = "super" or
             name = "bool" or
             name = "NoneType" or
             name = "int" or

python/ql/src/semmle/python/pointsto/PointsTo2.qll

@@ -129,6 +129,18 @@ cached module PointsTo2 {
         // f.(CustomPointsToFact).pointsTo(context, value, origin)
     }
 
+    /** Holds if the attribute `name` is required for `obj` */
+    cached predicate attributeRequired(ObjectInternal obj, string name) {
+        points_to(any(AttrNode a).getObject(name), _, obj, _)
+        or
+        exists(CallNode call, PointsToContext ctx, StringObjectInternal nameobj |
+            points_to(call.getFunction(), ctx, ObjectInternal::builtin("getattr"), _) and
+            points_to(call.getArg(0), ctx, obj, _) and
+            points_to(call.getArg(1), ctx, nameobj, _) and
+            nameobj.strValue() = name
+        )
+    }
+
     cached CallNode get_a_call(ObjectInternal func, PointsToContext context) {
         points_to(result.getFunction(), context, func, _)
     }
@@ -261,14 +273,13 @@ cached module PointsTo2 {
     /** Holds if `f` is an attribute `x.attr` and points to `(value, cls, origin)`. */
     pragma [noinline]
     private predicate attribute_load_points_to(AttrNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) {
+        f.isLoad() and
         exists(ObjectInternal object, string name, CfgOrigin orig |
             points_to(f.getObject(name), context, object, _) |
             object.attribute(name, value, orig) and
             origin = orig.fix(f)
-        )
-        or
-        exists(ObjectInternal object |
-            points_to(f.getObject(), context, object, _) and
+            or
+            object.attributesUnknown() and
             origin = f and value = ObjectInternal::unknown()
         )
         // TO DO -- Support CustomPointsToAttribute

python/ql/src/semmle/python/pointsto/PointsToContext.qll

@@ -169,10 +169,10 @@ class PointsToContext extends TPointsToContext {
         this = TRuntimeContext() and executes_in_runtime_context(s)
         or
         /* Called functions, regardless of their name */
-        exists(PythonFunctionObjectInternal func, ControlFlowNode call, TPointsToContext outerContext |
-            call = PointsTo2::get_a_call(func, outerContext) and
-            this = TCallContext(call, outerContext, _) and
-            s = func.getScope()
+        exists(CallableObjectInternal callable, ControlFlowNode call, TPointsToContext outerContext |
+            call = PointsTo2::get_a_call(callable, outerContext) and
+            this = TCallContext(call, outerContext, _) |
+            s = callable.getScope()
         )
         or
         InterProceduralPointsTo::callsite_calls_function(_, _, s, this, _)

python/ql/src/semmle/python/types/FunctionObject.qll

@@ -59,7 +59,7 @@ abstract class FunctionObject extends Object {
 
     /** Gets a call-site from where this function is called */
     ControlFlowNode getACall() {
-        result = PointsTo2::get_a_call(theCallable(), _)
+        result = theCallable().getACall()
     }
 
     /** Gets a call-site from where this function is called, given the `context` */