Initial commit of Python queries and QL libraries.

python/ql/src/Variables/Definition.qll

@@ -0,0 +1,160 @@
+import python
+
+
+/**
+ * A control-flow node that defines a variable
+ */
+class Definition extends NameNode, DefinitionNode {
+
+    /**
+     * The variable defined by this control-flow node.
+     */
+    Variable getVariable() {
+        this.defines(result)
+    }
+
+    /**
+     * The SSA variable corresponding to the current definition. Since SSA variables
+     * are only generated for definitions with at least one use, not all definitions
+     * will have an SSA variable.
+     */
+    SsaVariable getSsaVariable() {
+        result.getDefinition() = this
+    }
+
+    /**
+     * The index of this definition in its basic block.
+     */
+    private int indexInBB(BasicBlock bb, Variable v) {
+        v = this.getVariable() and
+        this = bb.getNode(result)
+    }
+
+    /**
+     * The rank of this definition among other definitions of the same variable
+     * in its basic block. The first definition will have rank 1, and subsequent
+     * definitions will have sequentially increasing ranks.
+     */
+    private int rankInBB(BasicBlock bb, Variable v) {
+        exists(int defIdx | defIdx = this.indexInBB(bb, v) |
+            defIdx = rank[result](int idx, Definition def | idx = def.indexInBB(bb, v) | idx)
+        )
+    }
+
+    /** Is this definition the first in its basic block for its variable? */
+    predicate isFirst() {
+        this.rankInBB(_, _) = 1
+    }
+
+    /** Is this definition the last in its basic block for its variable? */
+    predicate isLast() {
+        exists(BasicBlock b, Variable v |
+            this.rankInBB(b, v) = max(Definition other | any() | other.rankInBB(b, v))
+        )
+    }
+
+    /**
+     * Is this definition unused? A definition is unused if the value it provides
+     * is not read anywhere.
+     */
+    predicate isUnused() {
+        // SSA variables only exist for definitions that have at least one use.
+        not exists(this.getSsaVariable()) and
+        // If a variable is used in a foreign scope, all bets are off.
+        not this.getVariable().escapes() and
+        // Global variables don't have SSA variables unless the scope is global.
+        this.getVariable().getScope() = this.getScope() and
+        // A call to locals() or vars() in the variable scope counts as a use
+        not exists(Function f, Call c, string locals_or_vars |
+            c.getScope() = f and this.getScope() = f and
+            c.getFunc().(Name).getId() = locals_or_vars |
+            locals_or_vars = "locals" or locals_or_vars = "vars"
+        )
+    }
+
+    /**
+     * An immediate re-definition of this definition's variable.
+     */
+    Definition getARedef() {
+        result != this and
+        exists(Variable var | var = this.getVariable() and var = result.getVariable() |
+            // Definitions in different basic blocks.
+            this.isLast() and
+            reaches_without_redef(var, this.getBasicBlock(), result.getBasicBlock()) and
+            result.isFirst()
+        )
+        or
+        // Definitions in the same basic block.
+        exists(BasicBlock common, Variable var |
+            this.rankInBB(common, var) + 1 = result.rankInBB(common, var)
+        )
+    }
+
+    /**
+     * We only consider assignments as potential alert targets, not parameters
+     * and imports and other name-defining constructs.
+     * We also ignore anything named "_", "empty", "unused" or "dummy"
+     */
+    predicate isRelevant() {
+        exists(AstNode p |
+            p = this.getNode().getParentNode() |
+            p instanceof Assign or p instanceof AugAssign or p instanceof Tuple
+        )
+        and
+        not name_acceptable_for_unused_variable(this.getVariable())
+        and
+        /* Decorated classes and functions are used */
+        not exists(this.getNode().getParentNode().(FunctionDef).getDefinedFunction().getADecorator())
+        and
+        not exists(this.getNode().getParentNode().(ClassDef).getDefinedClass().getADecorator())
+    }
+
+}
+
+/**
+ * Check whether basic block `a` reaches basic block `b` without an intervening
+ * definition of variable `v`. The relation is not transitive by default, so any
+ * observed transitivity will be caused by loops in the control-flow graph.
+ */
+private
+predicate reaches_without_redef(Variable v, BasicBlock a, BasicBlock b) {
+    exists(Definition def | a.getASuccessor() = b |
+        def.getBasicBlock() = a and def.getVariable() = v and maybe_redefined(v)
+    ) or
+    exists(BasicBlock mid | reaches_without_redef(v, a, mid) |
+        not exists(NameNode cfn | cfn.defines(v) |
+            cfn.getBasicBlock() = mid
+        ) and
+        mid.getASuccessor() = b
+    )
+}
+
+private predicate maybe_redefined(Variable v) {
+    strictcount(Definition d | d.defines(v)) > 1
+}
+
+predicate name_acceptable_for_unused_variable(Variable var) {
+    exists(string name |
+        var.getId() = name |
+        name.regexpMatch("_+") or name = "empty" or
+        name.matches("%unused%") or name = "dummy" or
+        name.regexpMatch("__.*")
+    )
+}
+
+
+class ListComprehensionDeclaration extends ListComp {
+
+    Name getALeakedVariableUse() {
+        major_version() = 2 and
+        this.getIterationVariable(_).getId() = result.getId() and
+        result.getScope() = this.getScope() and
+        this.getAFlowNode().strictlyReaches(result.getAFlowNode()) and
+        result.isUse()
+    }
+
+    Name getDefinition() {
+        result = this.getIterationVariable(0).getAStore()
+    }
+
+}

python/ql/src/Variables/Global.qhelp

@@ -0,0 +1,20 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+<overview>
+<p>The use of the <code>global</code> keyword enables functions to modify variables outside of their scope.
+These functions may then include side effects that may not be apparent to users
+of that function, making the code harder to understand.</p>
+</overview>
+
+<recommendation>
+<p>Remove the <code>global</code> statement, if possible.</p>
+</recommendation>
+
+<references>
+
+  <li>Python Language Reference: <a href="http://docs.python.org/reference/simple_stmts.html#the-global-statement">The global statement</a>.</li>
+
+</references>
+</qhelp>

python/ql/src/Variables/Global.ql

@@ -0,0 +1,18 @@
+/**
+ * @name Use of the 'global' statement.
+ * @description Use of the 'global' statement may indicate poor modularity.
+ * @kind problem
+ * @problem.severity recommendation
+ * @sub-severity low
+ * @deprecated
+ * @precision very-high
+ * @id py/use-of-global
+ */
+
+import python
+
+from Global g
+where not g.getScope() instanceof Module
+select g, "Updating global variables except at module initialization is discouraged"
+
+

python/ql/src/Variables/GlobalAtModuleLevel.qhelp

@@ -0,0 +1,20 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+<overview>
+<p>The <code>global</code> statement is used to specify that assignments to that name are assignments to the
+variable in the global (module) scope, rather than in the local scope.
+At the module level, this statement is redundant because the local scope and global scope are the same.</p>
+
+</overview>
+<recommendation>
+<p>Remove the <code>global</code> statement.</p>
+
+</recommendation>
+<references>
+
+  <li>Python Language Reference: <a href="http://docs.python.org/reference/simple_stmts.html#the-global-statement">The global statement</a>.</li>
+
+</references>
+</qhelp>

python/ql/src/Variables/GlobalAtModuleLevel.ql

@@ -0,0 +1,17 @@
+/**
+ * @name Use of 'global' at module level
+ * @description Use of the 'global' statement at module level
+ * @kind problem
+ * @tags maintainability
+ *       useless-code
+ * @problem.severity warning
+ * @sub-severity low
+ * @precision very-high
+ * @id py/redundant-global-declaration
+ */
+
+import python
+
+from Global g
+where g.getScope() instanceof Module
+select g, "Declaring '" + g.getAName() + "' as global at module-level is redundant."

python/ql/src/Variables/LeakingListComprehension.py

@@ -0,0 +1,7 @@
+
+def two_or_three():
+    x = 3
+    [0 for x in range(3)]
+    return x # Will return 2 in Python 2 and 3 in Python 3.
+
+print(two_or_three())

python/ql/src/Variables/LeakingListComprehension.qhelp

@@ -0,0 +1,41 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+
+
+<overview>
+<p> In Python 2 list comprehensions are evaluated in the enclosing scope, which means that the iteration variable of a list comprehension is visible
+outside of the list comprehension. In Python 3 the iteration variable is no longer visible in the enclosing scope.
+</p>
+
+<p>
+Code that uses the value of a list comprehension iteration variable after the list comprehension has finished will
+behave differently under Python 2 and Python 3.
+</p>
+
+</overview>
+<recommendation>
+<p>Explicitly set the variable in the outer scope to the value that it would have held when run under Python 2.
+Then rename the list comprehension variable for additional clarity.
+</p>
+
+</recommendation>
+<example>
+<p>In this example, <code>x</code> is initially assigned the value of 3.
+In Python 3, <code>x</code> will be unchanged as the list comprehension is evaluated in its own scope.
+In Python 2, evaluation of the list comprehension occurs in the scope of <code>two_or_three</code>, setting <code>x</code> to 2.</p>
+<sample src="LeakingListComprehension.py" />
+
+<p> The following example is the same code as above, but the list comprehension variable is renamed to ensure it does not overwrite <code>x</code>.</p>
+<sample src="LeakingListComprehensionFixed.py" /> 
+
+</example>
+<references>
+    
+    <li>Python Tutorial: <a href="https://docs.python.org/2/tutorial/datastructures.html#list-comprehensions">List Comprehensions</a>.</li>
+    <li>The History of Python: <a href="http://python-history.blogspot.co.uk/2010/06/from-list-comprehensions-to-generator.html">From List Comprehensions to Generator Expressions</a>.</li>
+    <li>Python Language Reference: <a href="https://docs.python.org/2/reference/expressions.html#list-displays">List displays</a>.</li>
+
+</references>
+</qhelp>

python/ql/src/Variables/LeakingListComprehension.ql

@@ -0,0 +1,30 @@
+/**
+ * @name List comprehension variable used in enclosing scope
+ * @description Using the iteration variable of a list comprehension in the enclosing scope will result in different behavior between Python 2 and 3 and is confusing.
+ * @kind problem
+ * @tags portability
+ *       correctness
+ * @problem.severity warning
+ * @sub-severity high
+ * @precision very-high
+ * @id py/leaking-list-comprehension
+ */
+
+import python
+import Definition
+
+from ListComprehensionDeclaration l, Name use, Name defn
+where
+  use = l.getALeakedVariableUse() and
+  defn = l.getDefinition() and
+  l.getAFlowNode().strictlyReaches(use.getAFlowNode()) and
+  /* Make sure we aren't in a loop, as the variable may be redefined */
+  not use.getAFlowNode().strictlyReaches(l.getAFlowNode()) and
+  not l.contains(use) and
+  not use.deletes(_) and
+  not exists(SsaVariable v |
+      v.getAUse() = use.getAFlowNode() and
+      not v.getDefinition().strictlyDominates(l.getAFlowNode())
+  )
+
+select use, use.getId() + " may have a different value in Python 3, as the $@ will not be in scope.", defn, "list comprehension variable" 

python/ql/src/Variables/LeakingListComprehensionFixed.py

@@ -0,0 +1,7 @@
+
+def just_three():
+    x = 3
+    [0 for y in range(3)]
+    return x # Will return always return 3.
+
+print(just_three())

python/ql/src/Variables/Loop.qll

@@ -0,0 +1,38 @@
+import python
+
+
+private predicate empty_sequence(Expr e) {
+    exists(SsaVariable var | var.getAUse().getNode() = e | empty_sequence(var.getDefinition().getNode())) or
+    e instanceof List and not exists(e.(List).getAnElt()) or
+    e instanceof Tuple and not exists(e.(Tuple).getAnElt()) or
+    e.(StrConst).getText().length() = 0
+}
+
+/* This has the potential for refinement, but we err on the side of fewer false positives for now. */
+private predicate probably_non_empty_sequence(Expr e) {
+    not empty_sequence(e)
+}
+
+/** A loop which probably defines v */
+private Stmt loop_probably_defines(Variable v) {
+    exists(Name defn | defn.defines(v) and result.contains(defn) |
+        probably_non_empty_sequence(result.(For).getIter())
+        or
+        probably_non_empty_sequence(result.(While).getTest())
+    )
+}
+
+/** Holds if the variable used by `use` is probably defined in a loop */
+predicate probably_defined_in_loop(Name use) {
+    exists(Stmt loop |
+        loop = loop_probably_defines(use.getVariable()) |
+        loop.getAFlowNode().strictlyReaches(use.getAFlowNode())
+    )
+}
+
+/** Holds if `s` is a loop that probably executes at least once */
+predicate loop_probably_executes_at_least_once(Stmt s) {
+    probably_non_empty_sequence(s.(For).getIter())
+    or
+    probably_non_empty_sequence(s.(While).getTest())
+}

python/ql/src/Variables/LoopVariableCapture.py

@@ -0,0 +1,18 @@
+
+#Make a list of functions to increment their arguments by 0 to 9.
+def make_incrementers():
+    result = []
+    for i in range(10):
+        def incrementer(x):
+            return x + i
+        result.append(incrementer)
+    return result
+
+#This will fail
+def test():
+    incs = make_incrementers()
+    for x in range(10):
+        for y in range(10):
+            assert incs[x](y) == x+y
+
+test()

python/ql/src/Variables/LoopVariableCapture.qhelp

@@ -0,0 +1,60 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+
+<overview>
+<p> 
+Nested functions are a useful feature of Python as it allows a function to access the variables of its enclosing function.
+However, the programmer needs to be aware that when an inner function accesses a variable in an outer scope, 
+it is the variable that is captured, not the value of that variable.
+</p>
+<p>
+Therefore, care must be taken when the captured variable is a loop variable, since it is the loop <em>variable</em> and
+<em>not</em> the <em>value</em> of that variable that is captured.
+This will mean that by the time that the inner function executes, 
+the loop variable will have its final value, not the value when the inner function was created.
+</p>
+
+</overview>
+<recommendation>
+<p>
+The simplest way to fix this problem is to add a local variable of the same name as the outer variable and initialize that 
+using the outer variable as a default.
+<code>
+for var in seq:
+    ...
+    def inner_func(arg):
+        ...
+        use(var)
+</code>
+becomes
+<code>
+for var in seq:
+    ...
+    def inner_func(arg, var=var):
+        ...
+        use(var)
+</code>
+</p>
+
+</recommendation>
+<example>
+<p>
+In this example, a list of functions is created which should each increment its argument by its index in the list.
+However, since <code>i</code> will be 9 when the functions execute, they will each increment their argument by 9.
+ </p>
+<sample src="LoopVariableCapture.py" />
+<p>
+This can be fixed by adding the default value as shown below. The default value is computed when the function is created, so the desired effect is achieved.
+</p>
+
+<sample src="LoopVariableCapture2.py" />
+
+</example>
+<references>
+<li>The Hitchhiker’s Guide to Python: <a href="http://docs.python-guide.org/en/latest/writing/gotchas/#late-binding-closures">Late Binding Closures</a></li>
+<li>Python Language Reference: <a href="https://docs.python.org/reference/executionmodel.html#naming-and-binding">Naming and binding</a></li>
+
+</references>
+</qhelp>

python/ql/src/Variables/LoopVariableCapture.ql

@@ -0,0 +1,47 @@
+/**
+ * @name Loop variable capture
+ * @description Capture of a loop variable is not the same as capturing the value of a loop variable, and may be erroneous.
+ * @kind problem
+ * @tags correctness
+ * @problem.severity error
+ * @sub-severity low
+ * @precision high
+ * @id py/loop-variable-capture
+ */
+
+import python
+
+// Gets the scope of the iteration variable of the looping scope
+Scope iteration_variable_scope(AstNode loop) {
+    result = loop.(For).getScope()
+    or
+    result = loop.(Comp).getFunction()
+}
+
+predicate capturing_looping_construct(CallableExpr capturing, AstNode loop, Variable var) {
+    var.getScope() = iteration_variable_scope(loop) and
+    var.getAnAccess().getScope() = capturing.getInnerScope() and
+    capturing.getParentNode+() = loop and
+    (
+        loop.(For).getTarget() = var.getAnAccess()
+        or
+        var = loop.(Comp).getAnIterationVariable()
+    )
+}
+
+predicate escaping_capturing_looping_construct(CallableExpr capturing, AstNode loop, Variable var) {
+    capturing_looping_construct(capturing, loop, var) 
+    and
+    // Escapes if used out side of for loop or is a lambda in a comprehension
+    (
+        exists(Expr e, For forloop | forloop = loop and e.refersTo(_, _, capturing) | not forloop.contains(e))
+        or
+        loop.(Comp).getElt() = capturing
+        or
+        loop.(Comp).getElt().(Tuple).getAnElt() = capturing
+    )
+}
+
+from CallableExpr capturing, AstNode loop, Variable var
+where escaping_capturing_looping_construct(capturing, loop, var)
+select capturing, "Capture of loop variable '$@'", loop, var.getId()

python/ql/src/Variables/LoopVariableCapture2.py

@@ -0,0 +1,18 @@
+
+#Make a list of functions to increment their arguments by 0 to 9.
+def make_incrementers():
+    result = []
+    for i in range(10):
+        def incrementer(x, i=i):
+            return x + i
+        result.append(incrementer)
+    return result
+
+#This will pass
+def test():
+    incs = make_incrementers()
+    for x in range(10):
+        for y in range(10):
+            assert incs[x](y) == x+y
+
+test()

python/ql/src/Variables/MonkeyPatched.qll

@@ -0,0 +1,25 @@
+import python
+
+
+predicate monkey_patched_builtin(string name) {
+    exists(AttrNode attr, SubscriptNode subscr, StrConst s | 
+        subscr.isStore() and
+        subscr.getIndex().getNode() = s and
+        s.getText() = name and
+        subscr.getValue() = attr and
+        attr.getObject("__dict__").refersTo(theBuiltinModuleObject())
+    )
+    or
+    exists(CallNode call, ControlFlowNode bltn, StrConst s | 
+        call.getArg(0) = bltn and
+        bltn.refersTo(theBuiltinModuleObject()) and
+        call.getArg(1).getNode() = s and
+        s.getText() = name and
+        call.getFunction().refersTo(builtin_object("setattr"))
+    )
+    or
+    exists(AttrNode attr | 
+        attr.isStore() and
+        attr.getObject(name).refersTo(theBuiltinModuleObject())
+    )
+}

python/ql/src/Variables/MultiplyDefined.py

@@ -0,0 +1,3 @@
+x = 42
+x = 12
+print x

python/ql/src/Variables/MultiplyDefined.qhelp

@@ -0,0 +1,29 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+
+
+<overview>
+<p> Multiple assignments to a single variable without an intervening usage makes the first assignment redundant.
+Its value is lost.
+</p>
+
+</overview>
+<recommendation>
+<p>Ensure that the second assignment is in fact correct.
+Then delete the first assignment (taking care not to delete right hand side if it has side effects).</p>
+
+</recommendation>
+<example>
+<p>In this example, <code>x</code> is assigned the value of 42 but then the value is changed to 12
+before <code>x</code> is used. This makes the first assignment useless.</p>
+<sample src="MultiplyDefined.py" />
+
+</example>
+<references>
+
+    <li>Python: <a href="http://docs.python.org/reference/simple_stmts.html#assignment-statements">Assignment statements</a>.</li>
+
+</references>
+</qhelp>

python/ql/src/Variables/MultiplyDefined.ql

@@ -0,0 +1,61 @@
+/**
+ * @name Variable defined multiple times
+ * @description Assignment to a variable occurs multiple times without any intermediate use of that variable
+ * @kind problem
+ * @tags maintainability
+ *       useless-code
+ *       external/cwe/cwe-563
+ * @problem.severity warning
+ * @sub-severity low
+ * @precision very-high
+ * @id py/multiple-definition
+ */
+
+import python
+import Definition
+
+predicate multiply_defined(AstNode asgn1, AstNode asgn2, Variable v) {
+    /* Must be redefined on all possible paths in the CFG corresponding to the original source.
+     * For example, splitting may create a path where `def` is unconditionally redefined, even though
+     * it is not in the original source. */
+    forex(Definition def, Definition redef |
+        def.getVariable() = v and
+        def = asgn1.getAFlowNode() and
+        redef = asgn2.getAFlowNode() |
+        def.isUnused() and
+        def.getARedef() = redef and
+        def.isRelevant()
+    )
+}
+
+predicate simple_literal(Expr e) {
+    e.(Num).getN() = "0" or
+    e instanceof NameConstant or
+    e instanceof List and not exists(e.(List).getAnElt()) or
+    e instanceof Tuple and not exists(e.(Tuple).getAnElt()) or
+    e instanceof Dict and not exists(e.(Dict).getAKey()) or
+    e.(StrConst).getText() = ""
+}
+
+/** A multiple definition is 'uninteresting' if it sets a variable to a 
+ * simple literal before reassigning it.
+ * x = None
+ * if cond:
+ *     x = value1
+ * else:
+ *     x = value2
+ */
+predicate uninteresting_definition(AstNode asgn1) {
+    exists(AssignStmt a | 
+        a.getATarget() = asgn1 |
+        simple_literal(a.getValue())
+    )
+}
+
+
+from AstNode asgn1, AstNode asgn2, Variable v
+where
+    multiply_defined(asgn1, asgn2, v) and
+    forall(Name el | el = asgn1.getParentNode().(Tuple).getAnElt() | multiply_defined(el, _, _)) and
+    not uninteresting_definition(asgn1)
+select asgn1, "This assignment to '" + v.getId() + "' is unnecessary as it is redefined $@ before this value is used.", asgn2 as t, "here"

python/ql/src/Variables/ShadowBuiltin.py

@@ -0,0 +1,8 @@
+def test():
+    int = 1           # Variable should be renamed to avoid
+    def print_int():  # shadowing the int() built-in function
+        print int
+    print_int()
+    print int 
+
+test()

python/ql/src/Variables/ShadowBuiltin.qhelp

@@ -0,0 +1,30 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+
+<overview>
+<p>When a local variable is defined with the same name as a built-in type or function, the local
+variable "shadows" or "hides" the built-in object. This can lead to
+confusion as a reader of the code may expect the variable to refer to a built-in object.
+</p>
+
+</overview>
+<recommendation>
+
+<p>Change the name of the local variable so that it no longer matches the name of a built-in object.
+</p>
+
+</recommendation>
+<example>
+<sample src="ShadowBuiltin.py" />
+
+
+</example>
+<references>
+
+<li>Python Standard Library: <a href="http://docs.python.org/2/library/functions.html">Built-in Functions</a>,
+                            <a href="http://docs.python.org/2/library/stdtypes.html">Built-in Types</a>.</li>
+
+</references>
+</qhelp>

python/ql/src/Variables/ShadowBuiltin.ql

@@ -0,0 +1,64 @@
+/**
+ * @name Builtin shadowed by local variable
+ * @description Defining a local variable with the same name as a built-in object
+ *              makes the built-in object unusable within the current scope and makes the code
+ *              more difficult to read.
+ * @kind problem
+ * @tags maintainability
+ *       readability
+ * @problem.severity recommendation
+ * @sub-severity low
+ * @precision medium
+ * @id py/local-shadows-builtin
+ */
+ 
+import python
+import Shadowing
+ 
+predicate white_list(string name) {
+    /* These are rarely used and thus unlikely to be confusing */
+    name = "iter" or
+    name = "next" or
+    name = "input" or
+    name = "file" or
+    name = "apply" or
+    name = "slice" or
+    name = "buffer" or
+    name = "coerce" or
+    name = "intern" or
+    name = "exit" or
+    name = "quit" or
+    name = "license" or
+    /* These are short and/or hard to avoid */
+    name = "dir" or
+    name = "id" or
+    name = "max" or
+    name = "min" or
+    name = "sum" or
+    name = "cmp" or
+    name = "chr" or
+    name = "ord" or
+    name = "bytes" or
+    name = "_"
+}
+
+predicate shadows(Name d, string name, Scope scope, int line) {
+    exists(LocalVariable l | d.defines(l) and scope instanceof Function and
+           l.getId() = name and
+           exists(builtin_object(l.getId()))
+    ) and
+    d.getScope() = scope and
+    d.getLocation().getStartLine() = line and
+    not white_list(name) and
+    not optimizing_parameter(d)
+}
+
+predicate first_shadowing_definition(Name d, string name) {
+    exists(int first, Scope scope |
+        shadows(d, name, scope, first) and
+        first = min(int line | shadows(_, name, scope, line)))
+}
+
+from Name d, string name
+where first_shadowing_definition(d, name)
+select d, "Local variable " + name + " shadows a builtin variable."

python/ql/src/Variables/ShadowGlobal.py

@@ -0,0 +1,10 @@
+var = 2              # Global variable
+
+def test2():
+    def print_var():
+        var = 3 
+        print var    # Local variable which "shadows" the global variable
+    print_var()      # making it more difficult to determine which "var"
+    print var        # is referenced
+
+test2()

python/ql/src/Variables/ShadowGlobal.qhelp

@@ -0,0 +1,36 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+
+<overview>
+<p>Python statements can access variables in both the local namespace and in the global namespace.
+When a local and a global variable have the same name, the local variable "shadows" or "hides" the
+global variable. When the variable is referenced, the variable with local scope is used unless you
+explicitly use the <code>global</code> statement to reference the global variable. This can lead to
+confusion as a reader of the code may expect the variable to refer to a global.
+</p>
+
+</overview>
+<recommendation>
+
+<p>Avoid using the same name for variables in local and global namespaces.</p>
+
+</recommendation>
+<example>
+<p>The following simple example shows how a local variable can "shadow" a global variable. The local
+variable should be renamed to make the code easier to interpret.</p>
+<sample src="ShadowGlobal.py" />
+
+
+</example>
+<references>
+
+<li>J. Lusth, <i>The Art and Craft of Programming - Python Edition</i>, Section: Scope. University of Alabama, 2012. (<a href="http://troll.cs.ua.edu/ACP-PY/index_13.html">Published online</a>).</li>
+<li>New Mexico Tech Computer Center: <a href="http://infohost.nmt.edu/tcc/help/pubs/python/web/global-statement.html">The global
+statement: Declare access to a global name</a>.</li>
+
+
+
+</references>
+</qhelp>

python/ql/src/Variables/ShadowGlobal.ql

@@ -0,0 +1,66 @@
+/**
+ * @name Global shadowed by local variable
+ * @description Defining a local variable with the same name as a global variable
+ *              makes the global variable unusable within the current scope and makes the code
+ *              more difficult to read.
+ * @kind problem
+ * @tags maintainability
+ *       readability
+ * @problem.severity recommendation
+ * @sub-severity low
+ * @precision medium
+ * @id py/local-shadows-global
+ */
+
+import python
+import Shadowing
+
+predicate shadows(Name d, GlobalVariable g, Scope scope, int line) {
+    exists(LocalVariable l | d.defines(l) and l.getId() = g.getId() and
+        scope instanceof Function and g.getScope() = scope.getScope() and
+        not exists(Import il, Import ig, Name gd | il.contains(d) and gd.defines(g) and ig.contains(gd)) and
+        not exists(Assign a | a.getATarget() = d and a.getValue() = g.getAnAccess())
+    ) and
+    not exists(builtin_object(g.getId())) and
+    d.getScope() = scope and
+    d.getLocation().getStartLine() = line and
+    exists(Name defn | defn.defines(g) |
+        not exists(If i | i.isNameEqMain() | 
+            i.contains(defn)
+        )
+    ) and
+    not optimizing_parameter(d)
+}
+
+/* pytest dynamically populates its namespace so, we cannot look directly for the pytest.fixture function */
+AttrNode pytest_fixture_attr() {
+    exists(ModuleObject pytest |
+        result.getObject("fixture").refersTo(pytest)
+    )
+}
+
+Object pytest_fixture() {
+    exists(CallNode call |
+        call.getFunction() = pytest_fixture_attr()
+        or
+        call.getFunction().(CallNode).getFunction() = pytest_fixture_attr()
+        |
+        call.refersTo(result)
+    )
+}
+
+/* pytest fixtures require that the parameter name is also a global */
+predicate assigned_pytest_fixture(GlobalVariable v) {
+    exists(NameNode def | def.defines(v) and def.(DefinitionNode).getValue().refersTo(pytest_fixture()))
+}
+
+predicate first_shadowing_definition(Name d, GlobalVariable g) {
+    exists(int first, Scope scope |
+        shadows(d, g, scope, first) and
+        first = min(int line | shadows(_, g, scope, line)))
+}
+
+from Name d, GlobalVariable g, Name def
+where first_shadowing_definition(d, g) and not exists(Name n | n.deletes(g)) and 
+      def.defines(g) and not assigned_pytest_fixture(g) and not g.getId() = "_"
+select d, "Local variable '" + g.getId() + "' shadows a global variable defined $@.", def, "here"

python/ql/src/Variables/Shadowing.qll

@@ -0,0 +1,13 @@
+import python
+
+/* Parameters with defaults that are used as an optimization. 
+ * E.g. def f(x, len=len): ...
+ * (In general, this kind of optimization is not recommended.)
+ */
+predicate optimizing_parameter(Parameter p) {
+    exists(string name, Name glob |
+        p.getDefault() = glob |
+        glob.getId() = name and 
+        p.asName().getId() = name
+    )
+}

python/ql/src/Variables/SuspiciousUnusedLoopIterationVariable.py

@@ -0,0 +1,6 @@
+
+#
+def test():
+    for t in [TypeA, TypeB]:
+        x = TypeA()
+        run_test(x)

python/ql/src/Variables/SuspiciousUnusedLoopIterationVariable.qhelp

@@ -0,0 +1,35 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+
+<overview>
+<p>A for loop iteration variable is not used in the body of the loop, and the loop does not count the number of items in the sequence.
+This is suspicious as there is rarely any reason to iterate over a sequence and not use the contents. 
+Not using the loop variable can often indicate a logical error or typo.
+</p>
+
+</overview>
+<recommendation>
+<p>Carefully check that the loop variable should not be used. 
+If the variable is genuinely not being used and the code is correct, then rename the variable to <code>_</code> 
+or <code>unused</code> to indicate to readers of the code that it is intentionally unused.
+</p>
+
+</recommendation>
+<example>
+<p>In this example, the <code>for</code> loop iteration variable <code>x</code> is never used. It appears that the
+original <code>test</code> function was used to test <code>TypeA</code> and was subsequently modified to test <code>TypeB</code> as well.
+</p>
+<sample src="SuspiciousUnusedLoopIterationVariable.py" />
+<p>
+It is likely that the change from <code>x = TypeA()</code> to <code>x = t()</code> was forgotten. The fixed version is shown below.
+</p>
+<sample src="SuspiciousUnusedLoopIterationVariableFixed.py" />
+
+</example>
+<references>
+    <li>Python Language Reference: <a href="https://docs.python.org/reference/compound_stmts.html#the-for-statement">The for statement</a>.</li>
+    <li>Python Tutorial: <a href="https://docs.python.org/tutorial/controlflow.html#for-statements">For statements</a>.</li>
+</references>
+</qhelp>

python/ql/src/Variables/SuspiciousUnusedLoopIterationVariable.ql

@@ -0,0 +1,126 @@
+/**
+ * @name Suspicious unused loop iteration variable
+ * @description A loop iteration variable is unused, which suggests an error.
+ * @kind problem
+ * @tags maintainability
+ *       correctness
+ * @problem.severity error
+ * @sub-severity low
+ * @precision high
+ * @id py/unused-loop-variable
+ */
+
+import python
+import Definition
+
+predicate is_increment(Stmt s) {
+    /* x += n */
+    s.(AugAssign).getValue() instanceof IntegerLiteral  
+    or
+    /* x = x + n */
+    exists(Name t, BinaryExpr add |
+        t = s.(AssignStmt).getTarget(0) and
+        add = s.(AssignStmt).getValue() and
+        add.getLeft().(Name).getVariable() = t.getVariable() and
+        add.getRight() instanceof IntegerLiteral
+    )
+}
+
+predicate counting_loop(For f) {
+    is_increment(f.getAStmt())
+}
+
+predicate empty_loop(For f) {
+    not exists(f.getStmt(1)) and f.getStmt(0) instanceof Pass
+}
+
+predicate one_item_only(For f) {
+    not exists(Continue c | f.contains(c)) and
+    exists(Stmt s | 
+        s = f.getBody().getLastItem() |
+        s instanceof Return
+        or
+        s instanceof Break
+    )
+}
+
+predicate points_to_call_to_range(ControlFlowNode f) {
+    /* (x)range is a function in Py2 and a class in Py3, so we must treat it as a plain object */ 
+    exists(Object range, Object call |
+        range = builtin_object("range") or
+        range = builtin_object("xrange")
+    |
+        f.refersTo(call) and
+        call.(CallNode).getFunction().refersTo(range)
+    )
+    or
+    /* In case points-to fails due to 'from six.moves import range' or similar. */
+    exists(string range |
+        f.getNode().(Call).getFunc().(Name).getId() = range |
+        range = "range" or range = "xrange"
+    )
+    or
+    /* If range is wrapped in a list it is still a range */
+    exists(CallNode call |
+        f.refersTo(call) and
+        call = theListType().getACall() and
+        points_to_call_to_range(call.getArg(0))
+    )
+}
+
+/** Whether n is a use of a variable that is a not effectively a constant. */
+predicate use_of_non_constant(Name n) {
+    exists(Variable var |
+        n.uses(var) and
+        /* use is local */
+        not n.getScope() instanceof Module and
+        /* variable is not global */
+        not var.getScope() instanceof Module |
+        /* Defined more than once (dynamically) */
+        strictcount(Name def | def.defines(var)) > 1 or
+        exists(For f, Name def | f.contains(def) and def.defines(var)) or
+        exists(While w, Name def | w.contains(def) and def.defines(var))
+    )
+}
+
+/** Whether loop body is implicitly repeating something N times.
+ * E.g. queue.add(None)
+ */
+predicate implicit_repeat(For f) {
+    not exists(f.getStmt(1)) and
+    exists(ImmutableLiteral imm |
+        f.getStmt(0).contains(imm)
+    ) and
+    not exists(Name n | f.getBody().contains(n) and use_of_non_constant(n))
+}
+
+/** Get the CFG node for the iterable relating to the for-statement `f` in a comprehension.
+ * The for-statement `f` is the artificial for-statement in a comprehension
+ * and the result is the iterable in that comprehension.
+ * E.g. gets `x` from `{ y for y in x }`.
+ */
+ControlFlowNode get_comp_iterable(For f) {
+    exists(Comp c | 
+        c.getFunction().getStmt(0) = f |
+        c.getAFlowNode().getAPredecessor() = result
+    )
+}
+
+from For f, Variable v, string msg
+
+where f.getTarget() = v.getAnAccess() and
+      not f.getAStmt().contains(v.getAnAccess()) and
+      not points_to_call_to_range(f.getIter().getAFlowNode()) and
+      not points_to_call_to_range(get_comp_iterable(f)) and
+      not name_acceptable_for_unused_variable(v) and
+      not f.getScope().getName() = "genexpr" and
+      not empty_loop(f) and
+      not one_item_only(f) and
+      not counting_loop(f) and
+      not implicit_repeat(f) and
+      if exists(Name del | del.deletes(v) and f.getAStmt().contains(del)) then
+          msg = "' is deleted, but not used, in the loop body."
+      else
+          msg = "' is not used in the loop body."
+
+select f, "For loop variable '" + v.getId() + msg

python/ql/src/Variables/SuspiciousUnusedLoopIterationVariableFixed.py

@@ -0,0 +1,6 @@
+
+#
+def test():
+    for t in [TypeA, TypeB]:
+        x = t
+        run_test(x)

python/ql/src/Variables/Undefined.qll

@@ -0,0 +1,138 @@
+import python
+import Loop
+import semmle.python.security.TaintTracking
+
+/** Marker for "uninitialized". */
+class Uninitialized extends TaintKind {
+
+    Uninitialized() { this = "undefined" }
+
+}
+
+/** A source of an uninitialized variable.
+ * Either the start of the scope or a deletion. 
+ */
+class UninitializedSource extends TaintedDefinition {
+
+    UninitializedSource() {
+        exists(FastLocalVariable var |
+            this.getSourceVariable() = var and
+            not var.escapes() |
+            this instanceof ScopeEntryDefinition
+            or
+            this instanceof DeletionDefinition
+        )
+    }
+
+    override predicate isSourceOf(TaintKind kind) {
+        kind instanceof Uninitialized
+    }
+
+}
+
+/** A loop where we are guaranteed (or is at least likely) to execute the body at least once. 
+ */
+class AtLeastOnceLoop extends DataFlowExtension::DataFlowVariable {
+
+    AtLeastOnceLoop() {
+        loop_entry_variables(this, _)
+    }
+
+    /* If we are guaranteed to iterate over a loop at least once, then we can prune any edges that
+     * don't pass through the body.
+     */
+    override predicate prunedSuccessor(EssaVariable succ) {
+        loop_entry_variables(this, succ)
+    }
+
+}
+
+private predicate loop_entry_variables(EssaVariable pred, EssaVariable succ) {
+    exists(PhiFunction phi, BasicBlock pb |
+        loop_entry_edge(pb, phi.getBasicBlock()) and
+        succ = phi.getVariable() and
+        pred = phi.getInput(pb)
+    )
+}
+
+private predicate loop_entry_edge(BasicBlock pred, BasicBlock loop) {
+    pred = loop.getAPredecessor() and
+    pred = loop.getImmediateDominator() and
+    exists(Stmt s |
+        loop_probably_executes_at_least_once(s) and
+        s.getAFlowNode().getBasicBlock() = loop
+    )
+}
+
+class UnitializedSanitizer extends Sanitizer {
+
+    UnitializedSanitizer() { this = "use of variable" }
+
+    override
+    predicate sanitizingDefinition(TaintKind taint, EssaDefinition def) {
+        // An assignment cannot leave a variable uninitialized
+        taint instanceof Uninitialized and
+        (
+            def instanceof AssignmentDefinition
+            or
+            def instanceof ExceptionCapture
+            or
+            def instanceof ParameterDefinition
+            or
+            /* A use is a "sanitizer" of "uninitialized", as any use of an undefined
+             * variable will raise, making the subsequent code unreacahable.
+             */
+            exists(def.(EssaNodeRefinement).getInput().getASourceUse())
+            or
+            exists(def.(PhiFunction).getAnInput().getASourceUse())
+            or
+            exists(def.(EssaEdgeRefinement).getInput().getASourceUse())
+        )
+    }
+
+    override
+    predicate sanitizingNode(TaintKind taint, ControlFlowNode node) {
+        taint instanceof Uninitialized and
+        exists(EssaVariable v |
+            v.getASourceUse() = node and
+            not first_use(node, v)
+        )
+    }
+
+}
+
+/** Since any use of a local will raise if it is uninitialized, then
+ * any use dominated by another use of the same variable must be defined, or is unreachable.
+ */
+private predicate first_use(NameNode u, EssaVariable v) {
+    v.getASourceUse() = u and
+    not exists(NameNode other |
+        v.getASourceUse() = other and
+        other.strictlyDominates(u)
+    )
+}
+
+/* Holds if `call` is a call of the form obj.method_name(...) and 
+ * there is a function called `method_name` that can exit the program.
+ */
+private predicate maybe_call_to_exiting_function(CallNode call) {
+    exists(FunctionObject exits, string name |
+        exits.neverReturns() and exits.getName() = name
+        |
+        call.getFunction().(NameNode).getId() = name or
+        call.getFunction().(AttrNode).getName() = name
+    )
+}
+
+/** Prune edges where the predecessor block looks like it might contain a call to an exit function. */
+class ExitFunctionGuardedEdge extends DataFlowExtension::DataFlowVariable {
+
+    predicate prunedSuccessor(EssaVariable succ) {
+        exists(CallNode exit_call |
+            succ.(PhiFunction).getInput(exit_call.getBasicBlock()) = this and
+            maybe_call_to_exiting_function(exit_call)
+        )
+    }
+
+}
+

python/ql/src/Variables/UndefinedExport.py

@@ -0,0 +1,5 @@
+__all__ = ['spamm', 'troll', 'paywall']
+
+def spam(): return 'Spam'
+def troll(): return 'Troll'
+def paywall(): return 'Pay wall'

python/ql/src/Variables/UndefinedExport.qhelp

@@ -0,0 +1,37 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+
+<overview>
+<p>When a module is imported using <code>import *</code>, all attributes listed in
+<code>__all__</code> are imported. If <code>__all__</code> includes attributes that
+are not defined in the module then an exception is triggered. This usually indicates
+a typographic error in the attributes in  <code>__all__</code> or in the name of the
+object.</p>
+
+</overview>
+<recommendation>
+
+<p>Correct any typographic errors, either in the name of the object or in the string in
+<code>__all__</code>. If there are no typographic errors, either delete the name from
+<code>__all__</code> or add the object to the module.</p>
+
+</recommendation>
+<example>
+<p>
+In the example, the function name <code>spam</code> has been misspelled in the <code>__all__</code> list.
+This will result in <code>spamm</code> being highlighted as an undefined export.
+Correcting the spelling will fix the defect.
+</p>
+<sample src="UndefinedExport.py" />
+
+</example>
+<references>
+
+  <li>Python Language Reference: <a href="http://docs.python.org/2/reference/simple_stmts.html#import">The import statement</a>.</li>
+  <li>Python Tutorial: <a href="http://docs.python.org/2/tutorial/modules.html#importing-from-a-package">Importing * from a Package</a>.</li>
+
+
+</references>
+</qhelp>

python/ql/src/Variables/UndefinedExport.ql

@@ -0,0 +1,52 @@
+/**
+ * @name Explicit export is not defined
+ * @description Including an undefined attribute in __all__ causes an exception when
+ *              the module is imported using '*'
+ * @kind problem
+ * @tags reliability
+ *       maintainability
+ * @problem.severity error
+ * @sub-severity low
+ * @precision high
+ * @id py/undefined-export
+ */
+
+import python
+
+/** Whether name is declared in the __all__ list of this module */
+predicate declaredInAll(Module m, StrConst name)
+{
+    exists(Assign a, GlobalVariable all | 
+        a.defines(all) and a.getScope() = m and
+        all.getId() = "__all__" and ((List)a.getValue()).getAnElt() = name
+    )
+}
+
+predicate mutates_globals(PythonModuleObject m) {
+    exists(CallNode globals |
+        globals = theGlobalsFunction().(FunctionObject).getACall() and
+        globals.getScope() = m.getModule() |
+        exists(AttrNode attr | attr.getObject() = globals)
+        or
+        exists(SubscriptNode sub | sub.getValue() = globals and sub.isStore())
+    )
+    or
+    exists(Object enum_convert |
+        enum_convert.hasLongName("enum.Enum._convert") and
+        exists(CallNode call |
+            call.getScope() = m.getModule()
+            |
+            enum_convert.(FunctionObject).getACall() = call or
+            call.getFunction().refersTo(enum_convert)
+        )
+    )
+}
+
+from PythonModuleObject m, StrConst name, string exported_name
+where declaredInAll(m.getModule(), name) and
+exported_name = name.strValue() and
+not m.hasAttribute(exported_name) and
+not (m.getShortName() = "__init__" and exists(m.getPackage().getModule().getSubModule(exported_name))) and
+not exists(ImportStarNode imp | imp.getEnclosingModule() = m.getModule() | not imp.getModule().refersTo(_)) and
+not mutates_globals(m)
+select name, "The name '" + exported_name  + "' is exported by __all__ but is not defined."

python/ql/src/Variables/UndefinedGlobal.py

@@ -0,0 +1,12 @@
+import math
+
+angle = 0.01
+
+sin(angle)       # NameError: name 'sin' is not defined (function imported from 'math')
+
+math.sin(angle)  # 'sin' function now correctly defined
+
+math.tan(angel)  # NameError: name 'angel' not defined (typographic error)
+
+math.tan(angle)  # Global variable now correctly defined
+

python/ql/src/Variables/UndefinedGlobal.qhelp

@@ -0,0 +1,36 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+
+<overview>
+<p>This global variable may not be defined.
+If this code is executed and the variable is undefined then a <code>NameError</code> will occur.
+</p>
+
+</overview>
+<recommendation>
+
+<p>Check that the name of the global variable is not a typographic error. If the name is correct
+then define the variable or import the module that defines the function or method.</p>
+
+<p>If it is expected this variable will be initialized from another module before it is used, then the <code>NameError</code> may not occur.
+Nonetheless, the code will be more robust and clearer if the variable is set to a default value in its own module.
+</p>
+
+</recommendation>
+<example>
+<p>The following examples show two different examples of undefined "global variables".</p>
+
+<sample src="UndefinedGlobal.py" />
+
+
+</example>
+<references>
+
+<li>Python Standard Library: <a href="https://docs.python.org/library/exceptions.html#exceptions.NameError">NameError</a>.</li>
+<li>The Python Tutorial: <a href="http://docs.python.org/2/tutorial/modules.html">Modules</a>.</li>
+
+
+</references>
+</qhelp>

python/ql/src/Variables/UndefinedGlobal.ql

@@ -0,0 +1,132 @@
+/**
+ * @name Use of an undefined global variable
+ * @description Using a global variable before it is initialized causes an exception.
+ * @kind problem
+ * @tags reliability
+ *       correctness
+ * @problem.severity error
+ * @sub-severity low
+ * @precision low
+ * @id py/undefined-global-variable
+ */
+
+import python
+import Variables.MonkeyPatched
+import Loop
+import semmle.python.pointsto.PointsTo
+
+predicate guarded_against_name_error(Name u) {
+    exists(Try t | t.getBody().getAnItem().contains(u) |
+                   ((Name)((ExceptStmt)t.getAHandler()).getType()).getId() = "NameError"
+          )
+    or
+    exists(ConditionBlock guard, BasicBlock controlled, Call globals |
+        guard.getLastNode().getNode().contains(globals) or
+        guard.getLastNode().getNode() = globals |
+        globals.getFunc().(Name).getId() = "globals" and
+        guard.controls(controlled, _) and
+        controlled.contains(u.getAFlowNode())
+    )
+}
+
+predicate contains_unknown_import_star(Module m) {
+     exists(ImportStar imp | imp.getScope() = m |
+        not exists(ModuleObject imported | imported.importedAs(imp.getImportedModuleName()))
+        or
+        exists(ModuleObject imported | 
+            imported.importedAs(imp.getImportedModuleName()) |
+            not imported.exportsComplete()
+        )
+    )
+}
+
+predicate undefined_use_in_function(Name u) {
+    exists(Function f | u.getScope().getScope*() = f and
+        /* Either function is a method or inner function or it is live at the end of the module scope */
+        (not f.getScope() = u.getEnclosingModule() or ((ImportTimeScope)u.getEnclosingModule()).definesName(f.getName()))
+        and
+        /* There is a use, but not a definition of this global variable in the function or enclosing scope */
+        exists(GlobalVariable v | u.uses(v) |
+            not exists(Assign a, Scope defnScope | 
+                a.getATarget() = v.getAnAccess() and a.getScope() = defnScope |
+                defnScope = f or 
+                /* Exclude modules as that case is handled more precisely below. */
+                (defnScope = f.getScope().getScope*() and not defnScope instanceof Module)
+            )
+        )
+    )
+    and
+    not ((ImportTimeScope)u.getEnclosingModule()).definesName(u.getId())
+    and
+    not exists(ModuleObject m | m.getModule() = u.getEnclosingModule() | m.hasAttribute(u.getId()))
+    and
+    not globallyDefinedName(u.getId())
+    and
+    not exists(SsaVariable var | var.getAUse().getNode() = u and not var.maybeUndefined())
+    and
+    not guarded_against_name_error(u)
+    and
+    not (u.getEnclosingModule().isPackageInit() and u.getId() = "__path__")
+}
+
+predicate undefined_use_in_class_or_module(Name u) {
+    exists(GlobalVariable v | u.uses(v))
+    and
+    not exists(Function f | u.getScope().getScope*() = f)
+    and
+    exists(SsaVariable var | var.getAUse().getNode() = u | var.maybeUndefined())
+    and
+    not guarded_against_name_error(u)
+    and
+    not exists(ModuleObject m | m.getModule() = u.getEnclosingModule() | m.hasAttribute(u.getId()))
+    and
+    not (u.getEnclosingModule().isPackageInit() and u.getId() = "__path__")
+    and
+    not globallyDefinedName(u.getId())
+}
+
+predicate use_of_exec(Module m) {
+    exists(Exec exec | exec.getScope() = m)
+    or
+    exists(CallNode call, FunctionObject exec |
+        exec.getACall() = call and call.getScope() = m |
+        exec = builtin_object("exec") or
+        exec = builtin_object("execfile")
+    )
+}
+
+predicate undefined_use(Name u) {
+    (
+        undefined_use_in_class_or_module(u)
+        or
+        undefined_use_in_function(u)
+    ) and
+    not monkey_patched_builtin(u.getId()) and
+    not contains_unknown_import_star(u.getEnclosingModule()) and
+    not use_of_exec(u.getEnclosingModule()) and
+    not exists(u.getVariable().getAStore()) and
+    not u.refersTo(_) and
+    not  probably_defined_in_loop(u)
+}
+
+private predicate first_use_in_a_block(Name use) {
+    exists(GlobalVariable v, BasicBlock b, int i |
+        i = min(int j | b.getNode(j).getNode() = v.getALoad()) and b.getNode(i) = use.getAFlowNode()
+    )
+}
+
+predicate first_undefined_use(Name use) {
+    undefined_use(use) and
+    exists(GlobalVariable v |
+        v.getALoad() = use |
+        first_use_in_a_block(use) and
+        not exists(ControlFlowNode other | 
+            other.getNode() = v.getALoad() and
+            other.getBasicBlock().strictlyDominates(use.getAFlowNode().getBasicBlock())
+        )
+    )
+}
+
+from Name u
+where first_undefined_use(u)
+select u, "This use of global variable '" + u.getId()  + "' may be undefined."

python/ql/src/Variables/UndefinedPlaceHolder.qhelp

@@ -0,0 +1,29 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+
+<overview>
+<p>This place-holder variable may not be defined.
+If this code is executed and the variable is undefined then a <code>NameError</code> will occur.
+</p>
+
+</overview>
+<recommendation>
+
+<p>Check that the name of the place-holder variable is not a typographic error.
+If the name is correct, either define a value for the variable, or import the module that defines the function or method that sets the value.
+</p>
+
+<p>If another module initializes this variable before it is used, then the <code>NameError</code> may not occur.
+However, you can make the code more robust and clearer by setting the variable to a default value in its own module.
+</p>
+
+</recommendation>
+
+<references>
+
+<li>Python Standard Library: <a href="https://docs.python.org/library/exceptions.html#exceptions.NameError">NameError</a>.</li>
+
+</references>
+</qhelp>

python/ql/src/Variables/UndefinedPlaceHolder.ql

@@ -0,0 +1,54 @@
+/**
+ * @name Use of an undefined placeholder variable
+ * @description Using a variable before it is initialized causes an exception.
+ * @kind problem
+ * @problem.severity error
+ * @sub-severity low
+ * @precision medium
+ * @id py/undefined-placeholder-variable
+ */
+
+import python
+import Variables.MonkeyPatched
+
+/* Local variable part */
+
+predicate initialized_as_local(PlaceHolder use) {
+    exists(SsaVariable l, Function f | f = use.getScope() and l.getAUse() = use.getAFlowNode() |
+        l.getVariable() instanceof LocalVariable and
+        not l.maybeUndefined()
+    )
+}
+
+/* Not a template member */
+
+Class enclosing_class(PlaceHolder use) {
+    result.getAMethod() = use.getScope()
+}
+
+predicate template_attribute(PlaceHolder use) {
+    exists(ImportTimeScope cls |
+        cls = enclosing_class(use) |
+        cls.definesName(use.getId())
+    )
+}
+
+/* Global Stuff */
+
+predicate not_a_global(PlaceHolder use) {
+    not exists(PythonModuleObject mo | mo.hasAttribute(use.getId()) and mo.getModule() = use.getEnclosingModule())
+    and
+    not globallyDefinedName(use.getId()) and
+    not monkey_patched_builtin(use.getId()) and
+    not globallyDefinedName(use.getId())
+}
+
+from PlaceHolder p
+where 
+not initialized_as_local(p) and
+not template_attribute(p) and
+not_a_global(p)
+select p, "This use of place-holder variable '" + p.getId()  + "' may be undefined"
+
+
+

python/ql/src/Variables/UninitializedLocal.py

@@ -0,0 +1,33 @@
+def test():
+    var = 1 
+    def print_var():
+        print var      # Use variable from outer scope
+    print_var()
+    print var 
+
+
+def test1():
+    var = 2 
+    def print_var():
+        print var       # Attempt to use variable from local scope. 
+        var = 3         # Since this is not initialized yet, this results
+    print_var()         # in an UnboundLocalError
+    print var 
+
+
+def test2():
+    var = 2 
+    def print_var():
+        var = 3         # Initialize local version of the variable
+        print var       # Use variable from local scope.
+    print_var()         # Note that this local variable "shadows" the variable from
+    print var           # outer scope which makes code more difficult to interpret.
+
+
+def test3():
+    var = 4
+    def print_var():
+        nonlocal var    # Use non-local variable from outer scope.
+        print var
+    print_var()
+    print var

python/ql/src/Variables/UninitializedLocal.qhelp

@@ -0,0 +1,41 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+
+
+<overview>
+<p> This local variable may be used before it is defined. If a variable is assigned to in a function
+and not explicitly declared <code>global</code> or <code>nonlocal</code> then it is assumed to be a
+local variable.
+If it is used before it is defined then an <code>UnboundLocalError</code> will be raised.
+</p>
+
+</overview>
+<recommendation>
+
+<p>Review the code and consider the intended scope of the variable. Determine whether the variable
+should be global or local in scope. If a global variable is required then add a <code>global</code>
+statement, or in Python 3 you can use a <code>nonlocal</code> statement if the variable occurs in an
+enclosing function. Otherwise, ensure that the variable is defined before it is used.</p>
+
+</recommendation>
+<example>
+<p>The following code includes different functions that use variables. <code>test1()</code>
+fails with an <code>UnboundLocalError</code> because the local variable <code>var</code> is used
+before it is initialized.</p>
+
+<sample src="UninitializedLocal.py" />
+
+
+</example>
+<references>
+
+  <li>Python Standard Library: <a href="http://docs.python.org/library/exceptions.html#exceptions.UnboundLocalError">Built-in Exceptions: UnboundLocalError</a>.</li>
+  <li>Python Frequently Asked Questions: <a href="http://docs.python.org/2/faq/programming.html#why-am-i-getting-an-unboundlocalerror-when-the-variable-has-a-value">Why am I getting an UnboundLocalError when the variable has a value?</a>.</li>
+  <li>Python Course: <a href="http://www.python-course.eu/global_vs_local_variables.php">Global and Local Variables</a>.</li>
+  <li>Python Language Reference: <a href="http://docs.python.org/2.7/reference/simple_stmts.html#index-54">The global statement</a>,
+    <a href="http://docs.python.org/3.3/reference/simple_stmts.html#index-43">The nonlocal statement</a>.</li>
+
+</references>
+</qhelp>

python/ql/src/Variables/UninitializedLocal.ql

@@ -0,0 +1,42 @@
+/**
+ * @name Potentially uninitialized local variable
+ * @description Using a local variable before it is initialized causes an UnboundLocalError.
+ * @kind problem
+ * @tags reliability
+ *       correctness
+ * @problem.severity error
+ * @sub-severity low
+ * @precision medium
+ * @id py/uninitialized-local-variable
+ */
+
+import python
+import Undefined
+
+
+predicate uninitialized_local(NameNode use) {
+    exists(FastLocalVariable local |
+        use.uses(local) or use.deletes(local) |
+        not local.escapes()
+    )
+    and
+    (
+        any(Uninitialized uninit).taints(use)
+        or
+        not exists(EssaVariable var | var.getASourceUse() = use)
+    )
+}
+
+predicate explicitly_guarded(NameNode u) {
+    exists(Try t |
+        t.getBody().contains(u.getNode()) and
+        t.getAHandler().getType().refersTo(theNameErrorType())
+    )
+}
+
+
+from NameNode u
+where uninitialized_local(u) and not explicitly_guarded(u)
+select u.getNode(), "Local variable '" + u.getId() + "' may be used before it is initialized."
+
+

python/ql/src/Variables/UnusedLocalVariable.py

@@ -0,0 +1,11 @@
+import random
+
+def write_random_to_file():
+    no = random.randint(1, 10)
+    with open("random.txt", "w") as file:
+        file.write(str(no))
+    return no
+
+def write_random():
+    random_no = write_random_to_file()
+    print "A random number was written to random.txt"

python/ql/src/Variables/UnusedLocalVariable.qhelp

@@ -0,0 +1,31 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+
+<overview>
+<p> A local variable is defined (by an assignment) but never used.
+</p>
+
+<include src="UnusedVariableNaming.qhelp" />
+<include src="UnusedTuple.qhelp" />
+
+</overview>
+<recommendation>
+<p>If the variable is included for documentation purposes or is otherwise intentionally unused, then change its name to indicate that it is unused,
+otherwise delete the assignment (taking care not to delete right hand side if it has side effects).</p>
+
+</recommendation>
+<example>
+<p>In this example, the <code>random_no</code> variable is never read but its assignment 
+has a side effect. Because of this it is important to remove only the left hand side of the 
+assignment in line 10.</p>
+<sample src="UnusedLocalVariable.py" />
+
+</example>
+<references>
+
+  <li>Python: <a href="http://docs.python.org/2/reference/simple_stmts.html#assignment-statements">Assignment statements</a>.</li>
+
+</references>
+</qhelp>

python/ql/src/Variables/UnusedLocalVariable.ql

@@ -0,0 +1,34 @@
+/**
+ * @name Unused local variable
+ * @description Local variable is defined but not used
+ * @kind problem
+ * @tags maintainability
+ *       useless-code
+ *       external/cwe/cwe-563
+ * @problem.severity recommendation
+ * @sub-severity high
+ * @precision very-high
+ * @id py/unused-local-variable
+ */
+
+import python
+import Definition
+
+predicate unused_local(Name unused, LocalVariable v) {
+    forex(Definition def |
+        def.getNode() = unused |
+        def.getVariable() = v and
+        def.isUnused() and
+        not exists(def.getARedef()) and
+        def.isRelevant() and
+        not exists(def.getNode().getParentNode().(FunctionDef).getDefinedFunction().getADecorator()) and
+        not exists(def.getNode().getParentNode().(ClassDef).getDefinedClass().getADecorator())
+    )
+}
+
+
+from Name unused, LocalVariable v
+where unused_local(unused, v) and 
+// If unused is part of a tuple, count it as unused if all elements of that tuple are unused.
+forall(Name el | el = unused.getParentNode().(Tuple).getAnElt() | unused_local(el, _))
+select unused, "The value assigned to local variable '" + v.getId() + "' is never used."

python/ql/src/Variables/UnusedModuleVariable.py

@@ -0,0 +1,9 @@
+import random
+
+def write_random_to_file():
+    no = random.randint(1, 10)
+    with open("random.txt", "w") as file:
+        file.write(str(no))
+    return no
+
+random_no = write_random_to_file()

python/ql/src/Variables/UnusedModuleVariable.qhelp

@@ -0,0 +1,34 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+
+<overview>
+<p> A global (module-level) variable is defined (by an assignment) but never used
+and is not explicitly made public by inclusion in the <code>__all__</code> list.
+</p>
+
+<include src="UnusedVariableNaming.qhelp" />
+<include src="UnusedTuple.qhelp" />
+
+</overview>
+<recommendation>
+<p>If the variable is included for documentation purposes or is otherwise intentionally unused, then change its name to indicate that it is unused,
+otherwise delete the assignment (taking care not to delete right hand side if it has side effects).</p>
+
+</recommendation>
+<example>
+<p>In this example, the <code>random_no</code> variable is never read but its assignment
+has a side effect. Because of this it is important to only remove the left hand side of the
+assignment in line 9.</p>
+<sample src="UnusedModuleVariable.py" />
+
+</example>
+<references>
+
+  <li>Python: <a href="http://docs.python.org/reference/simple_stmts.html#assignment-statements">Assignment statements</a>,
+  <a href="http://docs.python.org/reference/simple_stmts.html#the-import-statement">The import statement</a>.</li>
+  <li>Python Tutorial: <a href="http://docs.python.org/2/tutorial/modules.html#importing-from-a-package">Importing * from a package</a>.</li>
+
+</references>
+</qhelp>

python/ql/src/Variables/UnusedModuleVariable.ql

@@ -0,0 +1,62 @@
+/**
+ * @name Unused global variable
+ * @description Global variable is defined but not used
+ * @kind problem
+ * @tags efficiency
+ *       useless-code
+ *       external/cwe/cwe-563
+ * @problem.severity recommendation
+ * @sub-severity low
+ * @precision high
+ * @id py/unused-global-variable
+ */
+
+import python
+import Definition
+
+/** Whether the module contains an __all__ definition, 
+ * but it is more complex than a simple list of strings */
+predicate complex_all(Module m) {
+    exists(Assign a, GlobalVariable all | 
+        a.defines(all) and a.getScope() = m and all.getId() = "__all__" |
+        not a.getValue() instanceof List or
+        exists(Expr e |
+            e = a.getValue().(List).getAnElt() |
+            not e instanceof StrConst
+        )
+    )
+    or
+    exists(Call c, GlobalVariable all |
+        c.getFunc().(Attribute).getObject() = all.getALoad() and
+        c.getScope() = m and all.getId() = "__all__"
+    )
+}
+
+predicate unused_global(Name unused, GlobalVariable v) {
+    not exists(ImportingStmt is | is.contains(unused)) and
+    forex(DefinitionNode defn |
+        defn.getNode() = unused |
+        not defn.getValue().getNode() instanceof FunctionExpr and
+        not defn.getValue().getNode() instanceof ClassExpr and
+        not exists(Name u | 
+            // A use of the variable
+            u.uses(v) |
+            // That is reachable from this definition, directly
+            defn.strictlyReaches(u.getAFlowNode())
+            or // indirectly
+            defn.getBasicBlock().reachesExit() and u.getScope() != unused.getScope()
+        ) and
+        not unused.getEnclosingModule().getAnExport() = v.getId() and
+        not exists(unused.getParentNode().(ClassDef).getDefinedClass().getADecorator()) and
+        not exists(unused.getParentNode().(FunctionDef).getDefinedFunction().getADecorator()) and
+        unused.defines(v) and
+        not name_acceptable_for_unused_variable(v) and
+        not complex_all(unused.getEnclosingModule())
+    )
+}
+
+from Name unused, GlobalVariable v
+where unused_global(unused, v) and
+// If unused is part of a tuple, count it as unused if all elements of that tuple are unused.
+forall(Name el | el = unused.getParentNode().(Tuple).getAnElt() | unused_global(el, _))
+select unused, "The global variable '" + v.getId() + "' is not used."

python/ql/src/Variables/UnusedParameter.py

@@ -0,0 +1,5 @@
+import random
+
+def write_to_file(text, filename):
+    with open("log.txt", "w") as file:
+        file.write(text)

python/ql/src/Variables/UnusedParameter.qhelp

@@ -0,0 +1,30 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+
+
+
+<overview>
+<p> A parameter is never used.
+</p>
+
+<include src="UnusedVariableNaming.qhelp" />
+
+</overview>
+<recommendation>
+
+<p>Delete the parameter from the relevant function or method.
+If that is not possible (due to overriding or similar) rename the parameter
+ as described above.
+
+</p></recommendation>
+<example>
+
+<p>In this example the parameter <code>filename</code> is ignored which is misleading.
+
+</p><sample src="UnusedParameter.py" />
+
+
+</example>
+</qhelp>

python/ql/src/Variables/UnusedParameter.ql

@@ -0,0 +1,31 @@
+/**
+ * @name Unused parameter
+ * @description Parameter is defined but not used
+ * @kind problem
+ * @tags maintainability
+ * @problem.severity recommendation
+ * @sub-severity high
+ * @precision medium
+ * @id py/unused-parameter
+ */
+
+import python
+import Definition
+
+
+predicate unused_parameter(FunctionObject f, LocalVariable v) {
+  v.isParameter() and
+  v.getScope() = f.getFunction() and
+  not name_acceptable_for_unused_variable(v) and
+  not exists(NameNode u | u.uses(v)) and
+  not exists(Name inner, LocalVariable iv | inner.uses(iv) and iv.getId() = v.getId() and inner.getScope().getScope() = v.getScope())
+}
+
+predicate is_abstract(FunctionObject func) {
+    ((Name)func.getFunction().getADecorator()).getId().matches("%abstract%")
+}
+
+from PyFunctionObject f, LocalVariable v
+where v.getId() != "self" and unused_parameter(f, v) and not f.isOverridingMethod() and not f.isOverriddenMethod() and
+not is_abstract(f)
+select f, "The parameter '" + v.getId() + "' is never used."

python/ql/src/Variables/UnusedTuple.qhelp

@@ -0,0 +1,12 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+<fragment>
+
+<p> Variables that are defined in a group, for example <code>x, y = func()</code> are handled collectively.
+If they are all unused, then this is reported. Otherwise they are all treated as used.
+</p>
+
+</fragment>
+</qhelp>

python/ql/src/Variables/UnusedVariableNaming.qhelp

@@ -0,0 +1,24 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+<fragment>
+
+<p> It is sometimes necessary to have a variable which is not used.
+These unused variables should have distinctive names, to make it clear to readers of the code that they are deliberately not used.
+The most common conventions for indicating this are to name the variable <code>_</code> or to start the name of the
+variable with <code>unused</code> or <code>_unused</code>.
+</p>
+
+<p>
+The query accepts the following names for variables that are intended to be unused:
+</p>
+<ul>
+<li>Any name consisting entirely of underscores.</li>
+<li>Any name containing <code>unused</code>.</li>
+<li>The names <code>dummy</code> or <code>empty</code>.</li>
+<li>Any "special" name of the form <code>__xxx__</code>.</li>
+</ul>
+
+</fragment>
+</qhelp>