Move missing/multiple calls to init/del queries to folder

python/ql/src/Classes/CallsToInitDel/MethodCallOrder.qll

@@ -0,0 +1,77 @@
+deprecated module;
+
+import python
+
+// Helper predicates for multiple call to __init__/__del__ queries.
+pragma[noinline]
+private predicate multiple_invocation_paths_helper(
+  FunctionInvocation top, FunctionInvocation i1, FunctionInvocation i2, FunctionObject multi
+) {
+  i1 != i2 and
+  i1 = top.getACallee+() and
+  i2 = top.getACallee+() and
+  i1.getFunction() = multi
+}
+
+pragma[noinline]
+private predicate multiple_invocation_paths(
+  FunctionInvocation top, FunctionInvocation i1, FunctionInvocation i2, FunctionObject multi
+) {
+  multiple_invocation_paths_helper(top, i1, i2, multi) and
+  i2.getFunction() = multi
+}
+
+/** Holds if `self.name` calls `multi` by multiple paths, and thus calls it more than once. */
+predicate multiple_calls_to_superclass_method(ClassObject self, FunctionObject multi, string name) {
+  exists(FunctionInvocation top, FunctionInvocation i1, FunctionInvocation i2 |
+    multiple_invocation_paths(top, i1, i2, multi) and
+    top.runtime(self.declaredAttribute(name)) and
+    self.getASuperType().declaredAttribute(name) = multi
+  |
+    // Only called twice if called from different functions,
+    // or if one call-site can reach the other.
+    i1.getCall().getScope() != i2.getCall().getScope()
+    or
+    i1.getCall().strictlyReaches(i2.getCall())
+  )
+}
+
+/** Holds if all attributes called `name` can be inferred to be methods. */
+private predicate named_attributes_not_method(ClassObject cls, string name) {
+  cls.declaresAttribute(name) and not cls.declaredAttribute(name) instanceof FunctionObject
+}
+
+/** Holds if `f` actually does something. */
+private predicate does_something(FunctionObject f) {
+  f.isBuiltin() and not f = theObjectType().lookupAttribute("__init__")
+  or
+  exists(Stmt s | s = f.getFunction().getAStmt() and not s instanceof Pass)
+}
+
+/** Holds if `meth` looks like it should have a call to `name`, but does not */
+private predicate missing_call(FunctionObject meth, string name) {
+  exists(CallNode call, AttrNode attr |
+    call.getScope() = meth.getFunction() and
+    call.getFunction() = attr and
+    attr.getName() = name and
+    not exists(FunctionObject f | f.getACall() = call)
+  )
+}
+
+/** Holds if `self.name` does not call `missing`, even though it is expected to. */
+predicate missing_call_to_superclass_method(
+  ClassObject self, FunctionObject top, FunctionObject missing, string name
+) {
+  missing = self.getASuperType().declaredAttribute(name) and
+  top = self.lookupAttribute(name) and
+  /* There is no call to missing originating from top */
+  not top.getACallee*() = missing and
+  /* Make sure that all named 'methods' are objects that we can understand. */
+  not exists(ClassObject sup |
+    sup = self.getAnImproperSuperType() and
+    named_attributes_not_method(sup, name)
+  ) and
+  not self.isAbstract() and
+  does_something(missing) and
+  not missing_call(top, name)
+}

python/ql/src/Classes/CallsToInitDel/MissingCallToDel.py

@@ -0,0 +1,26 @@
+
+class Vehicle(object):
+    
+    def __del__(self):
+        recycle(self.base_parts)
+        
+class Car(Vehicle):
+    
+    def __del__(self):
+        recycle(self.car_parts)
+        Vehicle.__del__(self)
+        
+#Car.__del__ is missed out.
+class SportsCar(Car, Vehicle):
+    
+    def __del__(self):
+        recycle(self.sports_car_parts)
+        Vehicle.__del__(self)
+        
+#Fix SportsCar by calling Car.__del__
+class FixedSportsCar(Car, Vehicle):
+    
+    def __del__(self):
+        recycle(self.sports_car_parts)
+        Car.__del__(self)
+        

python/ql/src/Classes/CallsToInitDel/MissingCallToDel.qhelp

@@ -0,0 +1,50 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+
+<overview>
+<p>Python, unlike statically typed languages such as Java, allows complete freedom when calling methods during object destruction.
+However, standard object-oriented principles apply to Python classes using deep inheritance hierarchies.
+Therefore the developer has responsibility for ensuring that objects are properly cleaned up when
+there are multiple <code>__del__</code> methods that need to be called.
+</p>
+<p>
+If the  <code>__del__</code> method of a superclass is not called during object destruction it is likely that
+that resources may be leaked.
+</p>
+
+<p>A call to the <code>__del__</code> method of a superclass during object destruction may be omitted:
+</p>
+<ul>
+        <li>When a subclass calls the <code>__del__</code> method of the wrong class.</li>
+        <li>When a call to the <code>__del__</code> method of one its base classes is omitted.</li>
+</ul>
+
+
+</overview>
+<recommendation>
+<p>Either be careful to explicitly call the <code>__del__</code> of the correct base class, or
+use <code>super()</code> throughout the inheritance hierarchy.</p>
+
+<p>Alternatively refactor one or more of the classes to use composition rather than inheritance.</p>
+
+
+</recommendation>
+<example>
+<p>In this example, explicit calls to <code>__del__</code> are used, but <code>SportsCar</code> erroneously calls
+<code>Vehicle.__del__</code>. This is fixed in <code>FixedSportsCar</code> by calling <code>Car.__del__</code>.
+</p>
+
+<sample src="MissingCallToDel.py" />
+
+</example>
+<references>
+
+        <li>Python Tutorial: <a href="https://docs.python.org/2/tutorial/classes.html">Classes</a>.</li>
+        <li>Python Standard Library: <a href="https://docs.python.org/2/library/functions.html#super">super</a>.</li>
+        <li>Artima Developer: <a href="http://www.artima.com/weblogs/viewpost.jsp?thread=236275">Things to Know About Python Super</a>.</li>
+        <li>Wikipedia: <a href="http://en.wikipedia.org/wiki/Composition_over_inheritance">Composition over inheritance</a>.</li>
+
+</references>
+</qhelp>

python/ql/src/Classes/CallsToInitDel/MissingCallToDel.ql

@@ -0,0 +1,26 @@
+/**
+ * @name Missing call to `__del__` during object destruction
+ * @description An omitted call to a super-class `__del__` method may lead to class instances not being cleaned up properly.
+ * @kind problem
+ * @tags quality
+ *       reliability
+ *       correctness
+ *       performance
+ * @problem.severity error
+ * @sub-severity low
+ * @precision high
+ * @id py/missing-call-to-delete
+ */
+
+import python
+import MethodCallOrder
+
+from ClassObject self, FunctionObject missing
+where
+  missing_call_to_superclass_method(self, _, missing, "__del__") and
+  not missing.neverReturns() and
+  not self.failedInference() and
+  not missing.isBuiltin()
+select self,
+  "Class " + self.getName() + " may not be cleaned up properly as $@ is not called during deletion.",
+  missing, missing.descriptiveString()

python/ql/src/Classes/CallsToInitDel/MissingCallToInit.py

@@ -0,0 +1,26 @@
+
+class Vehicle(object):
+    
+    def __init__(self):
+        self.mobile = True
+        
+class Car(Vehicle):
+    
+    def __init__(self):
+        Vehicle.__init__(self)
+        self.car_init()
+        
+#Car.__init__ is missed out.
+class SportsCar(Car, Vehicle):
+    
+    def __init__(self):
+        Vehicle.__init__(self)
+        self.sports_car_init()
+        
+#Fix SportsCar by calling Car.__init__
+class FixedSportsCar(Car, Vehicle):
+    
+    def __init__(self):
+        Car.__init__(self)
+        self.sports_car_init()
+        

python/ql/src/Classes/CallsToInitDel/MissingCallToInit.qhelp

@@ -0,0 +1,52 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+
+<overview>
+<p>Python, unlike statically typed languages such as Java, allows complete freedom when calling methods during object initialization.
+However, standard object-oriented principles apply to Python classes using deep inheritance hierarchies.
+Therefore the developer has responsibility for ensuring that objects are properly initialized when
+there are multiple <code>__init__</code> methods that need to be called.
+</p>
+<p>
+If the  <code>__init__</code> method of a superclass is not called during object initialization it is likely that
+that object will end up in an incorrect state.
+</p>
+
+<p>A call to the <code>__init__</code> method of a superclass during object initialization may be omitted:
+</p>
+<ul>
+        <li>When a subclass calls the <code>__init__</code> method of the wrong class.</li>
+        <li>When a call to the <code>__init__</code> method of one its base classes is omitted.</li>
+        <li>When multiple inheritance is used and a class inherits from several base classes,
+        and at least one of those does not use <code>super()</code> in its own <code>__init__</code> method.</li>
+</ul>
+
+
+</overview>
+<recommendation>
+<p>Either be careful to explicitly call the <code>__init__</code> of the correct base class, or
+use <code>super()</code> throughout the inheritance hierarchy.</p>
+
+<p>Alternatively refactor one or more of the classes to use composition rather than inheritance.</p>
+
+
+</recommendation>
+<example>
+<p>In this example, explicit calls to <code>__init__</code> are used, but <code>SportsCar</code> erroneously calls
+<code>Vehicle.__init__</code>. This is fixed in <code>FixedSportsCar</code> by calling <code>Car.__init__</code>.
+</p>
+
+<sample src="MissingCallToInit.py" />
+
+</example>
+<references>
+
+        <li>Python Tutorial: <a href="https://docs.python.org/2/tutorial/classes.html">Classes</a>.</li>
+        <li>Python Standard Library: <a href="https://docs.python.org/2/library/functions.html#super">super</a>.</li>
+        <li>Artima Developer: <a href="http://www.artima.com/weblogs/viewpost.jsp?thread=236275">Things to Know About Python Super</a>.</li>
+        <li>Wikipedia: <a href="http://en.wikipedia.org/wiki/Composition_over_inheritance">Composition over inheritance</a>.</li>
+
+</references>
+</qhelp>

python/ql/src/Classes/CallsToInitDel/MissingCallToInit.ql

@@ -0,0 +1,29 @@
+/**
+ * @name Missing call to `__init__` during object initialization
+ * @description An omitted call to a super-class `__init__` method may lead to objects of this class not being fully initialized.
+ * @kind problem
+ * @tags quality
+ *       reliability
+ *       correctness
+ * @problem.severity error
+ * @sub-severity low
+ * @precision high
+ * @id py/missing-call-to-init
+ */
+
+import python
+import MethodCallOrder
+
+from ClassObject self, FunctionObject initializer, FunctionObject missing
+where
+  self.lookupAttribute("__init__") = initializer and
+  missing_call_to_superclass_method(self, initializer, missing, "__init__") and
+  // If a superclass is incorrect, don't flag this class as well.
+  not missing_call_to_superclass_method(self.getASuperType(), _, missing, "__init__") and
+  not missing.neverReturns() and
+  not self.failedInference() and
+  not missing.isBuiltin() and
+  not self.isAbstract()
+select self,
+  "Class " + self.getName() + " may not be initialized properly as $@ is not called from its $@.",
+  missing, missing.descriptiveString(), initializer, "__init__ method"

python/ql/src/Classes/CallsToInitDel/SuperclassDelCalledMultipleTimes.py

@@ -0,0 +1,29 @@
+#Calling a method multiple times by using explicit calls when a base inherits from other base
+class Vehicle(object):
+    
+    def __del__(self):
+        recycle(self.base_parts)
+        
+        
+class Car(Vehicle):
+    
+    def __del__(self):
+        recycle(self.car_parts)
+        Vehicle.__del__(self)
+    
+    
+class SportsCar(Car, Vehicle):
+    
+    # Vehicle.__del__ will get called twice
+    def __del__(self):
+        recycle(self.sports_car_parts)
+        Car.__del__(self)
+        Vehicle.__del__(self)
+        
+        
+#Fix SportsCar by only calling Car.__del__
+class FixedSportsCar(Car, Vehicle):
+    
+    def __del__(self):
+        recycle(self.sports_car_parts)
+        Car.__del__(self)

python/ql/src/Classes/CallsToInitDel/SuperclassDelCalledMultipleTimes.qhelp

@@ -0,0 +1,58 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+
+<overview>
+<p>Python, unlike statically typed languages such as Java, allows complete freedom when calling methods during object destruction.
+However, standard object-oriented principles apply to Python classes using deep inheritance hierarchies.
+Therefore the developer has responsibility for ensuring that objects are properly cleaned up when
+there are multiple <code>__del__</code> methods that need to be called.
+</p>
+
+<p>
+Calling a <code>__del__</code> method more than once during object destruction risks resources being released multiple
+times. The relevant <code>__del__</code> method may not be designed to be called more than once.
+</p>
+
+<p>There are a number of ways that a <code>__del__</code> method may be be called more than once.</p>
+<ul>
+    <li>There may be more than one explicit call to the method in the hierarchy of <code>__del__</code> methods.</li>
+    <li>A class using multiple inheritance directly calls the <code>__del__</code> methods of its base types.
+    One or more of those base types uses <code>super()</code> to pass down the inheritance chain.</li>
+</ul>
+
+
+</overview>
+<recommendation>
+<p>Either be careful not to explicitly call a <code>__del__</code> method more than once, or
+use <code>super()</code> throughout the inheritance hierarchy.</p>
+
+<p>Alternatively refactor one or more of the classes to use composition rather than inheritance.</p>
+
+</recommendation>
+<example>
+<p>In the first example, explicit calls to <code>__del__</code> are used, but <code>SportsCar</code> erroneously calls
+both <code>Vehicle.__del__</code> and <code>Car.__del__</code>.
+This can be fixed by removing the call to <code>Vehicle.__del__</code>, as shown in <code>FixedSportsCar</code>.
+</p>
+
+<sample src="SuperclassDelCalledMultipleTimes.py" />
+
+<p>In the second example, there is a mixture of explicit calls to <code>__del__</code> and calls using <code>super()</code>.
+To fix this example, <code>super()</code> should be used throughout.
+</p>
+
+<sample src="SuperclassInitCalledMultipleTimes2.py" />
+
+</example>
+<references>
+
+        <li>Python Tutorial: <a href="https://docs.python.org/2/tutorial/classes.html">Classes</a>.</li>
+        <li>Python Standard Library: <a href="https://docs.python.org/2/library/functions.html#super">super</a>.</li>
+        <li>Artima Developer: <a href="http://www.artima.com/weblogs/viewpost.jsp?thread=236275">Things to Know About Python Super</a>.</li>
+        <li>Wikipedia: <a href="http://en.wikipedia.org/wiki/Composition_over_inheritance">Composition over inheritance</a>.</li>
+
+
+</references>
+</qhelp>

python/ql/src/Classes/CallsToInitDel/SuperclassDelCalledMultipleTimes.ql

@@ -0,0 +1,29 @@
+/**
+ * @name Multiple calls to `__del__` during object destruction
+ * @description A duplicated call to a super-class `__del__` method may lead to class instances not be cleaned up properly.
+ * @kind problem
+ * @tags quality
+ *       reliability
+ *       correctness
+ * @problem.severity warning
+ * @sub-severity high
+ * @precision very-high
+ * @id py/multiple-calls-to-delete
+ */
+
+import python
+import MethodCallOrder
+
+from ClassObject self, FunctionObject multi
+where
+  multiple_calls_to_superclass_method(self, multi, "__del__") and
+  not multiple_calls_to_superclass_method(self.getABaseType(), multi, "__del__") and
+  not exists(FunctionObject better |
+    multiple_calls_to_superclass_method(self, better, "__del__") and
+    better.overrides(multi)
+  ) and
+  not self.failedInference()
+select self,
+  "Class " + self.getName() +
+    " may not be cleaned up properly as $@ may be called multiple times during destruction.", multi,
+  multi.descriptiveString()

python/ql/src/Classes/CallsToInitDel/SuperclassDelCalledMultipleTimes2.py

@@ -0,0 +1,32 @@
+
+#Calling a method multiple times by using explicit calls when a base uses super()
+class Vehicle(object):
+     
+    def __del__(self):
+        recycle(self.base_parts)
+        super(Vehicle, self).__del__()
+        
+class Car(Vehicle):
+    
+    def __del__(self):
+        recycle(self.car_parts)
+        super(Car, self).__del__()
+        
+        
+class SportsCar(Car, Vehicle):
+    
+    # Vehicle.__del__ will get called twice
+    def __del__(self):
+        recycle(self.sports_car_parts)
+        Car.__del__(self)
+        Vehicle.__del__(self)
+        
+        
+#Fix SportsCar by using super()
+class FixedSportsCar(Car, Vehicle):
+    
+    def __del__(self):
+        recycle(self.sports_car_parts)
+        super(SportsCar, self).__del__()
+     
+

python/ql/src/Classes/CallsToInitDel/SuperclassInitCalledMultipleTimes.py

@@ -0,0 +1,36 @@
+#Calling a method multiple times by using explicit calls when a base inherits from other base
+class Vehicle(object):
+    
+    def __init__(self):
+        self.mobile = True
+        
+class Car(Vehicle):
+    
+    def __init__(self):
+        Vehicle.__init__(self)
+        self.car_init()
+        
+    def car_init(self):
+        pass
+    
+class SportsCar(Car, Vehicle):
+    
+    # Vehicle.__init__ will get called twice
+    def __init__(self):
+        Vehicle.__init__(self)
+        Car.__init__(self)
+        self.sports_car_init()
+        
+    def sports_car_init(self):
+        pass
+        
+#Fix SportsCar by only calling Car.__init__
+class FixedSportsCar(Car, Vehicle):
+    
+    def __init__(self):
+        Car.__init__(self)
+        self.sports_car_init()
+        
+    def sports_car_init(self):
+        pass
+ 

python/ql/src/Classes/CallsToInitDel/SuperclassInitCalledMultipleTimes.qhelp

@@ -0,0 +1,58 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+
+<overview>
+<p>Python, unlike statically typed languages such as Java, allows complete freedom when calling methods during object initialization.
+However, standard object-oriented principles apply to Python classes using deep inheritance hierarchies.
+Therefore the developer has responsibility for ensuring that objects are properly initialized when
+there are multiple <code>__init__</code> methods that need to be called.
+</p>
+
+<p>
+Calling an <code>__init__</code> method more than once during object initialization risks the object being incorrectly initialized.
+It is unlikely that the relevant <code>__init__</code> method is designed to be called more than once.
+</p>
+
+<p>There are a number of ways that an <code>__init__</code> method may be be called more than once.</p>
+<ul>
+    <li>There may be more than one explicit call to the method in the hierarchy of <code>__init__</code> methods.</li>
+    <li>A class using multiple inheritance directly calls the <code>__init__</code> methods of its base types.
+    One or more of those base types uses <code>super()</code> to pass down the inheritance chain.</li>
+</ul>
+
+
+</overview>
+<recommendation>
+<p>Either be careful not to explicitly call an <code>__init__</code> method more than once, or
+use <code>super()</code> throughout the inheritance hierarchy.</p>
+
+<p>Alternatively refactor one or more of the classes to use composition rather than inheritance.</p>
+
+</recommendation>
+<example>
+<p>In the first example, explicit calls to <code>__init__</code> are used, but <code>SportsCar</code> erroneously calls
+both <code>Vehicle.__init__</code> and <code>Car.__init__</code>.
+This can be fixed by removing the call to <code>Vehicle.__init__</code>, as shown in <code>FixedSportsCar</code>.
+</p>
+
+<sample src="SuperclassInitCalledMultipleTimes.py" />
+
+<p>In the second example, there is a mixture of explicit calls to <code>__init__</code> and calls using <code>super()</code>.
+To fix this example, <code>super()</code> should be used throughout.
+</p>
+
+<sample src="SuperclassInitCalledMultipleTimes2.py" />
+
+</example>
+<references>
+
+        <li>Python Tutorial: <a href="https://docs.python.org/2/tutorial/classes.html">Classes</a>.</li>
+        <li>Python Standard Library: <a href="https://docs.python.org/2/library/functions.html#super">super</a>.</li>
+        <li>Artima Developer: <a href="http://www.artima.com/weblogs/viewpost.jsp?thread=236275">Things to Know About Python Super</a>.</li>
+        <li>Wikipedia: <a href="http://en.wikipedia.org/wiki/Composition_over_inheritance">Composition over inheritance</a>.</li>
+
+
+</references>
+</qhelp>

python/ql/src/Classes/CallsToInitDel/SuperclassInitCalledMultipleTimes.ql

@@ -0,0 +1,30 @@
+/**
+ * @name Multiple calls to `__init__` during object initialization
+ * @description A duplicated call to a super-class `__init__` method may lead to objects of this class not being properly initialized.
+ * @kind problem
+ * @tags quality
+ *       reliability
+ *       correctness
+ * @problem.severity warning
+ * @sub-severity high
+ * @precision very-high
+ * @id py/multiple-calls-to-init
+ */
+
+import python
+import MethodCallOrder
+
+from ClassObject self, FunctionObject multi
+where
+  multi != theObjectType().lookupAttribute("__init__") and
+  multiple_calls_to_superclass_method(self, multi, "__init__") and
+  not multiple_calls_to_superclass_method(self.getABaseType(), multi, "__init__") and
+  not exists(FunctionObject better |
+    multiple_calls_to_superclass_method(self, better, "__init__") and
+    better.overrides(multi)
+  ) and
+  not self.failedInference()
+select self,
+  "Class " + self.getName() +
+    " may not be initialized properly as $@ may be called multiple times during initialization.",
+  multi, multi.descriptiveString()

python/ql/src/Classes/CallsToInitDel/SuperclassInitCalledMultipleTimes2.py

@@ -0,0 +1,38 @@
+
+#Calling a method multiple times by using explicit calls when a base uses super()
+class Vehicle(object):
+     
+    def __init__(self):
+        super(Vehicle, self).__init__()
+        self.mobile = True
+        
+class Car(Vehicle):
+    
+    def __init__(self):
+        super(Car, self).__init__()
+        self.car_init()
+        
+    def car_init(self):
+        pass
+        
+class SportsCar(Car, Vehicle):
+    
+    # Vehicle.__init__ will get called twice
+    def __init__(self):
+        Vehicle.__init__(self)
+        Car.__init__(self)
+        self.sports_car_init()
+        
+    def sports_car_init(self):
+        pass
+        
+#Fix SportsCar by using super()
+class FixedSportsCar(Car, Vehicle):
+    
+    def __init__(self):
+        super(SportsCar, self).__init__()
+        self.sports_car_init()
+        
+    def sports_car_init(self):
+        pass
+