Merge pull request #10176 from RasmusWL/import-problem

Python: Add testcase for import problem

python/ql/test/experimental/dataflow/typetracking_imports/README.md

@@ -0,0 +1,5 @@
+A testcase observed in real code, where mixing `from .this import that` with `from .other import *` (in that order) causes import resolution to not work properly.
+
+This needs to be in a separate folder, since using relative imports requires a valid top-level package. We emulate real extractor behavior using `-R` extractor option.
+
+From this directory, you can run the code with `python -m pkg.use`.

python/ql/test/experimental/dataflow/typetracking_imports/highlight_problem.expected

@@ -0,0 +1,9 @@
+| pkg/alias_only_direct.py:0:0:0:0 | Module pkg.alias_only_direct | pkg/alias_only_direct.py:1:22:1:24 | GSSA Variable foo | use to normal exit |
+| pkg/alias_problem.py:0:0:0:0 | Module pkg.alias_problem | pkg/alias_problem.py:1:22:1:24 | GSSA Variable foo | no use to normal exit |
+| pkg/alias_problem.py:0:0:0:0 | Module pkg.alias_problem | pkg/alias_problem.py:2:1:2:20 | GSSA Variable foo | use to normal exit |
+| pkg/alias_problem_fixed.py:0:0:0:0 | Module pkg.alias_problem_fixed | pkg/alias_problem_fixed.py:0:0:0:0 | GSSA Variable foo | no use to normal exit |
+| pkg/alias_problem_fixed.py:0:0:0:0 | Module pkg.alias_problem_fixed | pkg/alias_problem_fixed.py:3:22:3:24 | GSSA Variable foo | use to normal exit |
+| pkg/problem_absolute_import.py:0:0:0:0 | Module pkg.problem_absolute_import | pkg/problem_absolute_import.py:1:25:1:27 | GSSA Variable foo | no use to normal exit |
+| pkg/problem_absolute_import.py:0:0:0:0 | Module pkg.problem_absolute_import | pkg/problem_absolute_import.py:2:1:2:23 | GSSA Variable foo | use to normal exit |
+| pkg/works_absolute_import.py:0:0:0:0 | Module pkg.works_absolute_import | pkg/works_absolute_import.py:0:0:0:0 | GSSA Variable foo | no use to normal exit |
+| pkg/works_absolute_import.py:0:0:0:0 | Module pkg.works_absolute_import | pkg/works_absolute_import.py:2:25:2:27 | GSSA Variable foo | use to normal exit |

python/ql/test/experimental/dataflow/typetracking_imports/highlight_problem.ql

@@ -0,0 +1,15 @@
+import python
+
+// looking at `module_export` predicate in DataFlowPrivate, the core of the problem is
+// that in alias_problem.py, the direct import of `foo` does not flow to a normal exit of
+// the module. Instead there is a second variable foo coming from `from .other import*` that
+// goes to the normal exit of the module.
+from Module m, EssaVariable v, string useToNormalExit
+where
+  m = v.getScope().getEnclosingModule() and
+  not m.getName() in ["pkg.use", "pkg.foo_def"] and
+  v.getName() = "foo" and
+  if v.getAUse() = m.getANormalExit()
+  then useToNormalExit = "use to normal exit"
+  else useToNormalExit = "no use to normal exit"
+select m, v, useToNormalExit

python/ql/test/experimental/dataflow/typetracking_imports/options

@@ -0,0 +1 @@
+semmle-extractor-options: --max-import-depth=1 -R ./pkg/

python/ql/test/experimental/dataflow/typetracking_imports/pkg/alias_only_direct.py

@@ -0,0 +1 @@
+from .foo_def import foo # $ tracked

python/ql/test/experimental/dataflow/typetracking_imports/pkg/alias_problem.py

@@ -0,0 +1,2 @@
+from .foo_def import foo # $ tracked
+from .other import *

python/ql/test/experimental/dataflow/typetracking_imports/pkg/alias_problem_fixed.py

@@ -0,0 +1,3 @@
+# this ordering makes the problem go away
+from .other import *
+from .foo_def import foo # $ tracked

python/ql/test/experimental/dataflow/typetracking_imports/pkg/alias_star.py

@@ -0,0 +1,2 @@
+from .foo_def import *
+from .other import *

python/ql/test/experimental/dataflow/typetracking_imports/pkg/foo_def.py

@@ -0,0 +1,5 @@
+# apparently adding the assignment makes type-tracker unhappy, so we add this eval so
+# it's possible to run the example and see that everything works
+exec("tracked = 'tracked'")
+foo = tracked # $ tracked
+print(foo) # $ tracked

python/ql/test/experimental/dataflow/typetracking_imports/pkg/other.py

@@ -0,0 +1 @@
+bar = "bar-text"

python/ql/test/experimental/dataflow/typetracking_imports/pkg/problem_absolute_import.py

@@ -0,0 +1,2 @@
+from pkg.foo_def import foo # $ tracked
+from pkg.other import *

python/ql/test/experimental/dataflow/typetracking_imports/pkg/use.py

@@ -0,0 +1,46 @@
+def test_direct_import():
+    from .foo_def import foo # $ tracked
+    print(foo) # $ tracked
+
+test_direct_import()
+
+
+def test_alias_problem():
+    from .alias_problem import foo # $ MISSING: tracked
+    print(foo) # $ MISSING: tracked
+
+test_alias_problem()
+
+
+def test_alias_problem_fixed():
+    from .alias_problem_fixed import foo # $ tracked
+    print(foo) # $ tracked
+
+test_alias_problem_fixed()
+
+
+def test_alias_star():
+    from .alias_star import foo # $ tracked
+    print(foo) # $ tracked
+
+test_alias_star()
+
+
+def test_alias_only_direct():
+    from .alias_only_direct import foo # $ tracked
+    print(foo) # $ tracked
+
+test_alias_only_direct()
+
+
+def test_problem_absolute_import():
+    from pkg.problem_absolute_import import foo # $ MISSING: tracked
+    print(foo) # $ MISSING: tracked
+
+test_problem_absolute_import()
+
+def test_works_absolute_import():
+    from pkg.works_absolute_import import foo # $ tracked
+    print(foo) # $ tracked
+
+test_works_absolute_import()

python/ql/test/experimental/dataflow/typetracking_imports/pkg/works_absolute_import.py

@@ -0,0 +1,2 @@
+from pkg.other import *
+from pkg.foo_def import foo # $ tracked

python/ql/test/experimental/dataflow/typetracking_imports/tracked.qlref

@@ -0,0 +1 @@
+../typetracking/tracked.ql