Merge pull request #914 from markshannon/python-add-2-3-query-tests

Python: Add 2/3 specific query tests.

python/ql/test/2/query-tests/Classes/equals-hash/EqualsOrHash.expected

@@ -0,0 +1,2 @@
+| equals_hash.py:8:5:8:28 | Function __eq__ | Class $@ implements __eq__ but does not define __hash__. | equals_hash.py:3:1:3:17 | class Eq | Eq |
+| equals_hash.py:24:5:24:23 | Function __hash__ | Class $@ implements __hash__ but does not define __eq__ or __cmp__. | equals_hash.py:19:1:19:19 | class Hash | Hash |

python/ql/test/2/query-tests/Classes/equals-hash/EqualsOrHash.qlref

@@ -0,0 +1 @@
+Classes/EqualsOrHash.ql

python/ql/test/2/query-tests/Classes/equals-hash/EqualsOrNotEquals.expected

@@ -0,0 +1,2 @@
+| equals_hash.py:8:5:8:28 | Function __eq__ | Class $@ implements __eq__ but does not implement __ne__. | equals_hash.py:3:1:3:17 | class Eq | Eq |
+| equals_hash.py:16:5:16:28 | Function __ne__ | Class $@ implements __ne__ but does not implement __eq__. | equals_hash.py:11:1:11:17 | class Ne | Ne |

python/ql/test/2/query-tests/Classes/equals-hash/EqualsOrNotEquals.qlref

@@ -0,0 +1 @@
+Classes/EqualsOrNotEquals.ql

python/ql/test/2/query-tests/Classes/equals-hash/equals_hash.py

@@ -0,0 +1,63 @@
+#Equals and hash
+
+class Eq(object):
+
+    def __init__(self, data):
+        self.data = data
+
+    def __eq__(self, other):
+        return self.data == other.data
+
+class Ne(object):
+
+    def __init__(self, data):
+        self.data = data
+
+    def __ne__(self, other):
+        return self.data != other.data
+
+class Hash(object):
+
+    def __init__(self, data):
+        self.data = data
+
+    def __hash__(self):
+        return hash(self.data)
+    
+class Unhashable1(object):
+    
+    __hash__ = None
+    
+
+class EqOK1(Unhashable1):
+    
+    def __eq__(self, other):
+        return False
+    
+    def __ne__(self, other):
+        return True
+    
+class Unhashable2(object):
+    
+    #Not the idiomatic way of doing it, but not uncommon either
+    def __hash__(self):
+        raise TypeError("unhashable object")
+
+
+class EqOK2(Unhashable2):
+
+    def __eq__(self, other):
+        return False
+
+    def __ne__(self, other):
+        return True
+
+class ReflectiveNotEquals(object):
+
+    def __ne__(self, other):
+        return not self == other
+
+class EqOK3(ReflectiveNotEquals, Unhashable1):
+
+    def __eq__(self, other):
+        return self.data == other.data

python/ql/test/2/query-tests/Classes/inconsistent-mro/InconsistentMRO.expected

@@ -0,0 +1 @@
+| inconsistent_mro.py:9:1:9:14 | class Z | Construction of class Z can fail due to invalid method resolution order(MRO) for bases $@ and $@. | inconsistent_mro.py:3:1:3:16 | class X | X | inconsistent_mro.py:6:1:6:11 | class Y | Y |

python/ql/test/2/query-tests/Classes/inconsistent-mro/InconsistentMRO.qlref

@@ -0,0 +1 @@
+Classes/InconsistentMRO.ql

python/ql/test/2/query-tests/Classes/inconsistent-mro/inconsistent_mro.py

@@ -0,0 +1,17 @@
+#Inconsistent MRO
+
+class X(object):
+    pass
+
+class Y(X):
+    pass
+
+class Z(X, Y):
+    pass
+    
+class O:
+    pass
+    
+#This is OK in Python 2
+class N(object, O):
+    pass

python/ql/test/2/query-tests/Classes/new-style/PropertyInOldStyleClass.expected

@@ -0,0 +1 @@
+| property_old_style.py:8:6:8:13 | Property piosc | Property piosc will not work properly, as class OldStyle is an old-style class. |

python/ql/test/2/query-tests/Classes/new-style/PropertyInOldStyleClass.qlref

@@ -0,0 +1 @@
+Classes/PropertyInOldStyleClass.ql

python/ql/test/2/query-tests/Classes/new-style/SlotsInOldStyleClass.expected

@@ -0,0 +1 @@
+| newstyle_test.py:4:1:4:16 | class OldStyle1 | Using __slots__ in an old style class just creates a class attribute called '__slots__' |

python/ql/test/2/query-tests/Classes/new-style/SlotsInOldStyleClass.qlref

@@ -0,0 +1 @@
+Classes/SlotsInOldStyleClass.ql

python/ql/test/2/query-tests/Classes/new-style/SuperInOldStyleClass.expected

@@ -0,0 +1 @@
+| newstyle_test.py:15:9:15:15 | super() | super() will not work in old-style classes |

python/ql/test/2/query-tests/Classes/new-style/SuperInOldStyleClass.qlref

@@ -0,0 +1 @@
+Classes/SuperInOldStyleClass.ql

python/ql/test/2/query-tests/Classes/new-style/global_meta.py

@@ -0,0 +1,13 @@
+
+
+__metaclass__ = type
+
+class NewStyleEvenForPython2:
+
+    __slots__ = [ "ok" ]
+
+class OkToUseSuper(NewStyleEvenForPython2):
+
+    def __init__(self):
+        super(OkToUseSuper, self).__init__()
+

python/ql/test/2/query-tests/Classes/new-style/newstyle_test.py

@@ -0,0 +1,20 @@
+
+#Only works for Python2
+
+class OldStyle1:
+
+    __slots__ = [ 'a', 'b' ]
+
+    def __init__(self, a, b):
+        self.a = a
+        self.b = b
+
+class OldStyle2:
+
+    def __init__(self, x):
+        super().__init__(x)
+
+class NewStyle1(object):
+
+    def __init__(self, y):
+        super().__init__(y)

python/ql/test/2/query-tests/Classes/new-style/property_old_style.py

@@ -0,0 +1,10 @@
+#Property in old-style class
+
+class OldStyle:
+
+    def __init__(self, x):
+        self._x = x
+
+    @property
+    def piosc(self):
+        return self._x

python/ql/test/2/query-tests/Classes/undefined-attribute/MaybeUndefinedClassAttribute.qlref

@@ -0,0 +1 @@
+Classes/MaybeUndefinedClassAttribute.ql

python/ql/test/2/query-tests/Classes/undefined-attribute/UndefinedClassAttribute.qlref

@@ -0,0 +1 @@
+Classes/UndefinedClassAttribute.ql

python/ql/test/2/query-tests/Classes/undefined-attribute/undefined_attribute.py

@@ -0,0 +1,15 @@
+
+#Defined in metaclass
+class Meta(type):
+
+    def __init__(cls, name, bases, dct):
+        type.__init__(cls, name, bases, dct)
+        cls.defined_in_meta = 1
+
+class HasMeta(object):
+
+    __metaclass__ = Meta
+
+    def ok5(self):
+        return self.defined_in_meta
+

python/ql/test/2/query-tests/Exceptions/general/CatchingBaseException.qlref

@@ -0,0 +1 @@
+Exceptions/CatchingBaseException.ql

python/ql/test/2/query-tests/Exceptions/general/EmptyExcept.qlref

@@ -0,0 +1 @@
+Exceptions/EmptyExcept.ql

python/ql/test/2/query-tests/Exceptions/general/IllegalExceptionHandlerType.qlref

@@ -0,0 +1 @@
+Exceptions/IllegalExceptionHandlerType.ql

python/ql/test/2/query-tests/Exceptions/general/IllegalRaise.expected

@@ -0,0 +1 @@
+| exceptions_test.py:17:9:17:40 | Raise | Illegal class 'int' raised; will result in a TypeError being raised instead. |

python/ql/test/2/query-tests/Exceptions/general/IllegalRaise.qlref

@@ -0,0 +1 @@
+Exceptions/IllegalRaise.ql

python/ql/test/2/query-tests/Exceptions/general/IncorrectExceptOrder.qlref

@@ -0,0 +1 @@
+Exceptions/IncorrectExceptOrder.ql

python/ql/test/2/query-tests/Exceptions/general/exceptions_test.py

@@ -0,0 +1,17 @@
+
+
+def fail_batch_and_raise(exception):
+    if exception is None:
+        raise Exception()
+    elif exc_info is not None:
+        #This failed due to incorrect inference of the class of type(exception) as type.
+        raise type(exception), exception, exc_info[2]
+
+#Weird Python2 nonsense.
+def raise_tuple(cond):
+    if cond:
+        # This is OK. Honestly. Thankfully it's fixed in Python 3.
+        raise (Exception, "bananas", 17)
+    else:
+        #This is an error
+        raise (17, "bananas", Exception)

python/ql/test/2/query-tests/Exceptions/raising/RaisingTuple.expected

@@ -0,0 +1,3 @@
+| test.py:8:5:8:12 | Raise | Raising $@ will result in the first element (recursively) being raised and all other elements being discarded. | test.py:7:10:7:29 | Tuple | a tuple |
+| test.py:11:5:11:32 | Raise | Raising $@ will result in the first element (recursively) being raised and all other elements being discarded. | test.py:11:12:11:31 | Tuple | a tuple |
+| test.py:15:5:15:23 | Raise | Raising $@ will result in the first element (recursively) being raised and all other elements being discarded. | test.py:14:10:14:19 | Tuple | a tuple |

python/ql/test/2/query-tests/Exceptions/raising/RaisingTuple.qlref

@@ -0,0 +1 @@
+Exceptions/RaisingTuple.ql

python/ql/test/2/query-tests/Exceptions/raising/test.py

@@ -0,0 +1,15 @@
+
+
+def ok():
+    raise Exception, "message"
+
+def bad1():
+    ex = Exception, "message"
+    raise ex
+
+def bad2():
+    raise (Exception, "message")
+
+def bad3():
+    ex = Exception, 
+    raise ex, "message"

python/ql/test/2/query-tests/Expressions/TruncatedDivision.expected

@@ -0,0 +1,2 @@
+| TruncatedDivision_test.py:8:12:8:16 | BinaryExpr | Result of division may be truncated as its $@ and $@ arguments may both be integers. | TruncatedDivision_test.py:8:12:8:12 | TruncatedDivision_test.py:8 | left | TruncatedDivision_test.py:8:16:8:16 | TruncatedDivision_test.py:8 | right |
+| TruncatedDivision_test.py:11:12:11:40 | BinaryExpr | Result of division may be truncated as its $@ and $@ arguments may both be integers. | TruncatedDivision_test.py:2:12:2:12 | TruncatedDivision_test.py:2 | left | TruncatedDivision_test.py:5:12:5:12 | TruncatedDivision_test.py:5 | right |

python/ql/test/2/query-tests/Expressions/TruncatedDivision.qlref

@@ -0,0 +1 @@
+Expressions/TruncatedDivision.ql

python/ql/test/2/query-tests/Expressions/TruncatedDivisionCorrect_test.py

@@ -0,0 +1,27 @@
+from __future__ import division
+
+def return_three():
+    return 3
+
+def return_two():
+    return 2
+
+def f1():
+    return 3 / 2
+
+def f2():
+    return return_three() / return_two()
+
+def f3(x):
+    if isinstance(x, float):
+        return x / 2
+    else:
+        return (1.0 * x) / 2
+
+def f4():
+    do_stuff(f3(1))
+    do_stuff(f3(1.0))
+
+def f5():
+    return int(return_three() / return_two())
+

python/ql/test/2/query-tests/Expressions/TruncatedDivision_test.py

@@ -0,0 +1,24 @@
+def return_three():
+    return 3
+
+def return_two():
+    return 2
+
+def f1():
+    return 3 / 2
+
+def f2():
+    return return_three() / return_two()
+
+def f3(x):
+    if isinstance(x, float):
+        return x / 2
+    else:
+        return (1.0 * x) / 2
+
+def f4():
+    do_stuff(f3(1))
+    do_stuff(f3(1.0))
+
+def f5():
+    return int(return_three() / return_two())

python/ql/test/2/query-tests/Expressions/UseofApply.expected

@@ -0,0 +1 @@
+| expressions_test.py:3:5:3:21 | ControlFlowNode for apply() | Call to the obsolete builtin function 'apply'. |

python/ql/test/2/query-tests/Expressions/UseofApply.qlref

@@ -0,0 +1 @@
+Expressions/UseofApply.ql

python/ql/test/2/query-tests/Expressions/UseofInput.expected

@@ -0,0 +1 @@
+| expressions_test.py:6:12:6:18 | ControlFlowNode for input() | The unsafe built-in function 'input' is used. |

python/ql/test/2/query-tests/Expressions/UseofInput.qlref

@@ -0,0 +1 @@
+Expressions/UseofInput.ql

python/ql/test/2/query-tests/Expressions/expressions_test.py

@@ -0,0 +1,7 @@
+
+def use_of_apply(func, args):
+    apply(func, args)
+
+def use_of_input():
+    return input()
+

python/ql/test/2/query-tests/Functions/DeprecatedSliceMethod.qlref

@@ -0,0 +1 @@
+Functions/DeprecatedSliceMethod.ql

python/ql/test/2/query-tests/Functions/functions_test.py

@@ -0,0 +1,13 @@
+
+    
+class DeprecatedButNecessarySliceMethods(list):
+    
+    def __getslice__(self, start, stop):
+        pass
+    
+    def __setslice__(self, start, stop, value):
+        pass
+    
+    def __delslice__(self, start, stop):
+        pass
+ 

python/ql/test/2/query-tests/Imports/encoding_error/EncodingError.expected

@@ -0,0 +1 @@
+| EncodingError.py:5:8:5:8 | Encoding Error | 'ascii' codec can't decode byte 0xc3 in position 47: ordinal not in range(128) |

python/ql/test/2/query-tests/Imports/encoding_error/EncodingError.py

@@ -0,0 +1,7 @@
+123456789
+223456789
+323456789
+423456789
+5234567ø9
+623456789
+723456789

python/ql/test/2/query-tests/Imports/encoding_error/EncodingError.qlref

@@ -0,0 +1 @@
+Imports/EncodingError.ql

python/ql/test/2/query-tests/Imports/syntax_error/EncodingError.expected

@@ -0,0 +1 @@
+| bad_encoding.py:11:19:11:19 | Encoding Error | 'ascii' codec can't decode byte 0x82 in position 82: ordinal not in range(128) |

python/ql/test/2/query-tests/Imports/syntax_error/EncodingError.qlref

@@ -0,0 +1 @@
+Imports/EncodingError.ql

python/ql/test/2/query-tests/Imports/syntax_error/SyntaxError.expected

@@ -0,0 +1 @@
+| nonsense.py:1:1:1:1 | Syntax Error | Syntax Error (in Python 2.7). |

python/ql/test/2/query-tests/Imports/syntax_error/SyntaxError.qlref

@@ -0,0 +1 @@
+Imports/SyntaxError.ql

python/ql/test/2/query-tests/Imports/syntax_error/bad_encoding.py

@@ -0,0 +1,12 @@
+"""Multi-line docstring
+
+
+
+
+"""
+
+# encoding:shift-jis
+
+def f():
+    print "Python <20>̊J<CC8A><4A><EFBFBD>́A1990 <20>N<EFBFBD><4E><EFBFBD>납<EFBFBD><EB82A9><EFBFBD>J<EFBFBD>n<EFBFBD><6E><EFBFBD><EFBFBD><EFBFBD>Ă<EFBFBD><C482>܂<EFBFBD>"
+"""

python/ql/test/2/query-tests/Imports/syntax_error/nonsense.py

@@ -0,0 +1,37 @@
+ `Twas brillig, and the slithy toves
+  Did gyre and gimble in the wabe:
+All mimsy were the borogoves,
+  And the mome raths outgrabe.
+
+
+"Beware the Jabberwock, my son!
+  The jaws that bite, the claws that catch!
+Beware the Jubjub bird, and shun
+  The frumious Bandersnatch!"
+
+He took his vorpal sword in hand:
+  Long time the manxome foe he sought --
+So rested he by the Tumtum tree,
+  And stood awhile in thought.
+
+And, as in uffish thought he stood,
+  The Jabberwock, with eyes of flame,
+Came whiffling through the tulgey wood,
+  And burbled as it came!
+
+One, two! One, two! And through and through
+  The vorpal blade went snicker-snack!
+He left it dead, and with its head
+  He went galumphing back.
+
+"And, has thou slain the Jabberwock?
+  Come to my arms, my beamish boy!
+O frabjous day! Callooh! Callay!'
+  He chortled in his joy.
+
+
+`Twas brillig, and the slithy toves
+  Did gyre and gimble in the wabe;
+All mimsy were the borogoves,
+  And the mome raths outgrabe.
+

python/ql/test/2/query-tests/Imports/syntax_error/test.py

@@ -0,0 +1 @@
+print("Hello World", file=sys.stdout)

python/ql/test/2/query-tests/Lexical/OldOctalLiteral.expected

@@ -0,0 +1 @@
+| lexical_test.py:3:1:3:3 | IntegerLiteral | Confusing octal literal, use 0o17 instead. |

python/ql/test/2/query-tests/Lexical/OldOctalLiteral.qlref

@@ -0,0 +1 @@
+Lexical/OldOctalLiteral.ql

python/ql/test/2/query-tests/Lexical/lexical_test.py

@@ -0,0 +1,13 @@
+
+#Bad Octal literal
+017
+#Good Octal literal
+0o17
+#Special case file permissions
+permission = 0777
+
+#Other literals beginning with 0
+0
+0L
+0b1
+0x1

python/ql/test/2/query-tests/Statements/ExecUsed.expected

@@ -0,0 +1 @@
+| statements_test.py:5:5:5:14 | exec() | The 'exec' statement is used. |

python/ql/test/2/query-tests/Statements/ExecUsed.qlref

@@ -0,0 +1 @@
+Statements/ExecUsed.ql

python/ql/test/2/query-tests/Statements/IterableStringOrSequence.expected

@@ -0,0 +1 @@
+| statements_test.py:21:5:21:19 | For | Iteration over $@, of class list, may also iterate over $@. | statements_test.py:20:13:20:33 | ControlFlowNode for List | sequence | statements_test.py:18:13:18:26 | ControlFlowNode for Str | string |

python/ql/test/2/query-tests/Statements/IterableStringOrSequence.qlref

@@ -0,0 +1 @@
+Statements/IterableStringOrSequence.ql

python/ql/test/2/query-tests/Statements/TopLevelPrint.expected

@@ -0,0 +1 @@
+| module.py:2:1:2:31 | ExprStmt | Print statement may execute during import. |

python/ql/test/2/query-tests/Statements/TopLevelPrint.qlref

@@ -0,0 +1 @@
+Statements/TopLevelPrint.ql

python/ql/test/2/query-tests/Statements/fake_six.py

@@ -0,0 +1,3 @@
+
+
+string_types = str,

python/ql/test/2/query-tests/Statements/module.py

@@ -0,0 +1,2 @@
+#Top level prints in modules are bad
+print ("Side effect on import")

python/ql/test/2/query-tests/Statements/statements_test.py

@@ -0,0 +1,31 @@
+
+
+
+def exec_used(val):
+    exec (val)
+
+#Top level print
+import module
+
+#This is OK
+if __name__ == "__main__":
+    for i in range(10):
+        print ("Hello World")
+
+#Iteration over string or list
+def f(x):
+    if x:
+        s = u"Hello World"
+    else:
+        s = [ u'Hello', u'World']
+    for thing in s:
+        print (thing)
+
+import fake_six
+
+#ODASA 3737
+def g(arg = ''):
+    if isinstance(arg, fake_six.string_types):
+        arg = [ arg ]
+    for msg in arg:
+        pass

python/ql/test/2/query-tests/Variables/LeakyComp/LeakyComp.expected

@@ -0,0 +1,2 @@
+| test.py:5:11:5:11 | x | x may have a different value in Python 3, as the $@ will not be in scope. | test.py:4:12:4:12 | x | list comprehension variable |
+| test.py:10:11:10:11 | y | y may have a different value in Python 3, as the $@ will not be in scope. | test.py:9:12:9:12 | y | list comprehension variable |

python/ql/test/2/query-tests/Variables/LeakyComp/LeakyComp.qlref

@@ -0,0 +1 @@
+Variables/LeakingListComprehension.ql

python/ql/test/2/query-tests/Variables/LeakyComp/test.py

@@ -0,0 +1,49 @@
+from __future__ import print_function
+
+def undefined_in_3():
+    [x for x in range(3)]
+    print(x)
+
+def different_in_3():
+    y = 10
+    [y for y in range(3)]
+    print(y)
+
+def ok():
+    [z for z in range(4)]
+
+#FP observed in sembuild
+def use_in_loop(seq):
+    [x for x in range(3)]
+    for x in seq:
+        use(x) #x redefined -- fine in 2 and 3.
+
+def test_6395(dev):
+    ret = {}
+
+    res = foo(dev)
+    if not res:
+        return False
+
+    for line in res.splitlines():  # pylint: disable=no-member
+        line = line.strip()
+        if not line:
+            continue
+
+        key, val = [val.strip() for val in bar(line)]
+        if not (key and val):
+            continue
+
+        mval = None
+        if ' ' in val:
+            rval, mval = [val.strip() for val in bar(val)]
+            mval = mval[1:-1]
+        else:
+            rval = val
+
+    return ret
+
+def test_6441(seq):
+    for var in seq:
+        foo(var)
+        [var for var in bar()]

python/ql/test/2/query-tests/Variables/undefined/UndefinedExport.expected

@@ -0,0 +1,4 @@
+| UndefinedExport.py:3:18:3:20 | Str | The name 'y' is exported by __all__ but is not defined. |
+| UndefinedExport.py:3:23:3:25 | Str | The name 'z' is exported by __all__ but is not defined. |
+| UndefinedExport.py:3:28:3:35 | Str | The name 'module' is exported by __all__ but is not defined. |
+| package/__init__.py:1:23:1:34 | Str | The name 'not_exists' is exported by __all__ but is not defined. |

python/ql/test/2/query-tests/Variables/undefined/UndefinedExport.py

@@ -0,0 +1,9 @@
+
+
+__all__ = [ "x", "y", "z", "module" ]
+
+x = 1
+if 0:
+    y = 2
+
+import package.module

python/ql/test/2/query-tests/Variables/undefined/UndefinedExport.qlref

@@ -0,0 +1 @@
+Variables/UndefinedExport.ql

python/ql/test/2/query-tests/Variables/undefined/UndefinedGlobal.qlref

@@ -0,0 +1 @@
+Variables/UndefinedGlobal.ql

python/ql/test/2/query-tests/Variables/undefined/UninitializedLocal.py

@@ -0,0 +1,12 @@
+
+def const_xrange():
+    for i in xrange(4):
+        x = i
+    return x
+
+def rec1(a):
+    if a > 0:
+        rec1(a - 1)
+
+def rec2(b):
+    [rec2(b - 1) for c in range(b)]

python/ql/test/2/query-tests/Variables/undefined/UninitializedLocal.qlref

@@ -0,0 +1 @@
+Variables/UninitializedLocal.ql

python/ql/test/2/query-tests/Variables/undefined/options

@@ -0,0 +1 @@
+semmle-extractor-options: --max-import-depth=2

python/ql/test/2/query-tests/Variables/undefined/package/__init__.py

@@ -0,0 +1 @@
+__all__ = [ "module", "not_exists" ]

python/ql/test/2/query-tests/Variables/undefined/package/module.py

@@ -0,0 +1 @@
+

python/ql/test/2/query-tests/Variables/undefined/uses_execfile.py

@@ -0,0 +1,8 @@
+from other import setup
+
+execfile('x/_version.py')
+
+setup(
+    name='x',
+    version=__version__
+    )

python/ql/test/2/query-tests/options

@@ -0,0 +1,2 @@
+automatic_locations: true
+semmle-extractor-options: --max-import-depth=1