Merge pull request #850 from markshannon/python-improve-import-analysis

Python improve import analysis

python/ql/src/semmle/python/dataflow/SsaDefinitions.qll

@@ -337,7 +337,7 @@ private predicate class_with_global_metaclass(Class cls, GlobalVariable metaclas
 
 /** Holds if this variable is implicitly defined */
 private predicate implicit_definition(Variable v) {
-    v.getId() = "*"
+    v.getId() = "*" or v.getId() = "$"
     or
     exists(ImportStar is | is.getScope() = v.getScope())
 }

python/ql/src/semmle/python/pointsto/PointsTo.qll

@@ -114,19 +114,23 @@ module PointsTo {
         /** INTERNAL -- Do not use.
          *
          * Holds if `package.name` points to `(value, cls, origin)`, where `package` is a package object. */
-        cached predicate package_attribute_points_to(PackageObject package, string name, Object value, ClassObject cls, ControlFlowNode origin) {
+        cached predicate package_attribute_points_to(PackageObject package, string name, Object value, ClassObject cls, ObjectOrCfg origin) {
             py_module_attributes(package.getInitModule().getModule(), name, value, cls, origin)
             or
             exists(Module init |
-                init = package.getInitModule().getModule() |
-                not exists(PythonSsaSourceVariable v | v.getScope() = init | v.getName() = name or v.getName() = "*")
-                or
-                exists(EssaVariable v, PointsToContext imp |
-                    v.getScope() = init and v.getName() = "*" and v.getAUse() = init.getANormalExit() |
-                    SSA::ssa_variable_named_attribute_points_to(v, imp, name, undefinedVariable(), _, _) and
-                    imp.isImport()
+                init = package.getInitModule().getModule() and
+                not exists(EssaVariable var | var.getAUse() = init.getANormalExit() and var.getSourceVariable().getName() = name) and
+                exists(EssaVariable var, Context context |
+                    isModuleStateVariable(var) and var.getAUse() = init.getANormalExit() and
+                    context.isImport() and
+                    SSA::ssa_variable_named_attribute_points_to(var, context, name, undefinedVariable(), _, _) and
+                    origin = value and
+                    value = package.submodule(name) and
+                    cls = theModuleType()
                 )
-            ) and explicitly_imported(value) and
+            )
+            or
+            package.hasNoInitModule() and
             value = package.submodule(name) and cls = theModuleType() and origin = value
         }
 
@@ -145,7 +149,7 @@ module PointsTo {
             or
             not exists(EssaVariable var | var.getAUse() = m.getANormalExit() and var.getSourceVariable().getName() = name) and
             exists(EssaVariable var, PointsToContext imp |
-                var.getAUse() = m.getANormalExit() and var.getName() = "*" |
+                var.getAUse() = m.getANormalExit() and isModuleStateVariable(var) |
                 SSA::ssa_variable_named_attribute_points_to(var, imp, name, obj, cls, origin) and
                 imp.isImport() and obj != undefinedVariable()
             )
@@ -661,19 +665,25 @@ module PointsTo {
     /** Holds if `f` is a "from import" expression, `from mod import x` and points to `(value, cls, origin)`. */
     pragma [nomagic]
     private predicate from_import_points_to(ImportMemberNode f, PointsToContext context, Object value, ClassObject cls, ControlFlowNode origin) {
-        exists(EssaVariable var, ObjectOrCfg orig |
-            live_import_from_dot_in_init(f, var) and
-            ssa_variable_points_to(var, context, value, cls, orig) and
+        exists(string name, ModuleObject mod, ObjectOrCfg orig |
+            points_to(f.getModule(name), context, mod, _, _) and
             origin = origin_from_object_or_here(orig, f)
-        )
-        or
-        not live_import_from_dot_in_init(f, _) and
-        exists(string name, ModuleObject mod |
-            points_to(f.getModule(name), context, mod, _, _) |
-            exists(ObjectOrCfg orig |
-                Layer::module_attribute_points_to(mod, name, value, cls, orig) and
-                origin = origin_from_object_or_here(orig, f)
+            |
+            mod.getSourceModule() = f.getEnclosingModule() and
+            exists(EssaVariable var |
+                var.getSourceVariable().getName() = name and var.getAUse() = f and
+                ssa_variable_points_to(var, context, value, cls, orig)
             )
+            or
+            mod.getSourceModule() = f.getEnclosingModule() and
+            not exists(EssaVariable var | var.getSourceVariable().getName() = name and var.getAUse() = f) and
+            exists(EssaVariable dollar |
+                isModuleStateVariable(dollar) and dollar.getAUse() = f and
+                SSA::ssa_variable_named_attribute_points_to(dollar, context, name, value, cls, orig)
+            )
+            or
+            not mod.getSourceModule() = f.getEnclosingModule() and
+            Layer::module_attribute_points_to(mod, name, value, cls, orig)
         )
     }
 
@@ -1679,7 +1689,7 @@ module PointsTo {
             /* Undefined variable */
             exists(Scope scope |
                 not def.getVariable().getName() = "__name__" and
-                not def.getVariable().getName() = "*" and
+                not def.getVariable().getName() = "$" and
                 def.getScope() = scope and context.appliesToScope(scope) |
                 def.getSourceVariable() instanceof GlobalVariable and scope instanceof Module
                 or
@@ -1852,7 +1862,7 @@ module PointsTo {
             )
             or
             origin = def.getDefiningNode() and
-            def.getSourceVariable().getName() = "*" and
+            isModuleStateVariable(def.getVariable()) and
             context.isImport() and
             exists(PackageObject package |
                 package.getInitModule().getModule() = def.getScope() |
@@ -1963,7 +1973,7 @@ module PointsTo {
         /* Helper for import_star_named_attribute_points_to */
         pragma [noinline]
         private predicate star_variable_import_star_module(ImportStarRefinement def, ImportStarNode imp, PointsToContext context, ModuleObject mod) {
-            def.getSourceVariable().getName() = "*" and
+            isModuleStateVariable(def.getVariable()) and
             exists(ControlFlowNode fmod |
                 fmod = imp.getModule() and
                 imp = def.getDefiningNode() and
@@ -1983,7 +1993,7 @@ module PointsTo {
         /* Helper for ssa_star_variable_input_points_to */
         pragma [noinline]
         private predicate ssa_star_import_star_input(ImportStarRefinement def, EssaVariable var) {
-            def.getSourceVariable().getName() = "*" and var = def.getInput()
+            isModuleStateVariable(def.getVariable()) and var = def.getInput()
         }
 
         pragma [noinline]
@@ -2777,6 +2787,15 @@ module PointsTo {
 
     }
 
+    /** Get the ESSA pseudo-variable used to retain module state
+     * during module initialization. Module attributes are handled 
+     * as attributes of this variable, allowing the SSA form to track 
+     * mutations of the module during its creation.
+     */
+    private predicate isModuleStateVariable(EssaVariable var) {
+        var.getName() = "$" and var.getScope() instanceof Module
+    }
+
     /** INTERNAL -- Public for testing only */
     module Test {
 

python/ql/src/semmle/python/types/ModuleObject.qll

@@ -16,6 +16,11 @@ abstract class ModuleObject extends Object {
         none()
     }
 
+    /** Gets the source scope corresponding to this module, if this is a Python module */
+    Module getSourceModule() {
+        none()
+    }
+
     Container getPath() {
         none()
     }
@@ -136,6 +141,10 @@ class PythonModuleObject extends ModuleObject {
         result = this.getOrigin()
     }
 
+    override Module getSourceModule() {
+        result = this.getOrigin()
+    }
+
     override Container getPath() {
         result = this.getModule().getFile()
     }
@@ -206,6 +215,10 @@ class PackageObject extends ModuleObject {
         result = this.getOrigin()
     }
 
+    override Module getSourceModule() {
+        result = this.getModule().getInitModule()
+    }
+
     override Container getPath() {
         exists(ModuleObject m | 
             m.getPackage() = this |