Merge pull request #5069 from tausbn/python-api-graphs

Python: Add support for API graphs

python/ql/test/experimental/dataflow/ApiGraphs/mypkg/__init__.py

@@ -0,0 +1 @@
+foo = 42

python/ql/test/experimental/dataflow/ApiGraphs/mypkg/bar.py

@@ -0,0 +1 @@
+pass

python/ql/test/experimental/dataflow/ApiGraphs/mypkg/foo.py

@@ -0,0 +1 @@
+pass

python/ql/test/experimental/dataflow/ApiGraphs/options

@@ -0,0 +1 @@
+semmle-extractor-options: --lang=3

python/ql/test/experimental/dataflow/ApiGraphs/test.py

@@ -0,0 +1,81 @@
+import a1 #$ use=moduleImport("a1")
+
+x = a1.blah1 #$ use=moduleImport("a1").getMember("blah1")
+
+import a2 as m2 #$ use=moduleImport("a2")
+
+x2 = m2.blah2 #$ use=moduleImport("a2").getMember("blah2")
+
+import a3.b3 as m3 #$ use=moduleImport("a3").getMember("b3")
+
+x3 = m3.blah3 #$  use=moduleImport("a3").getMember("b3").getMember("blah3")
+
+from a4.b4 import c4 as m4 #$ use=moduleImport("a4").getMember("b4").getMember("c4")
+
+x4 = m4.blah4 #$ use=moduleImport("a4").getMember("b4").getMember("c4").getMember("blah4")
+
+import a.b.c.d #$ use=moduleImport("a")
+
+ab = a.b #$ use=moduleImport("a").getMember("b")
+
+abc = ab.c #$ use=moduleImport("a").getMember("b").getMember("c")
+
+abcd = abc.d #$ use=moduleImport("a").getMember("b").getMember("c").getMember("d")
+
+x5 = abcd.method() #$ use=moduleImport("a").getMember("b").getMember("c").getMember("d").getMember("method").getReturn()
+
+from a6 import m6 #$ use=moduleImport("a6").getMember("m6")
+
+x6 = m6().foo().bar() #$ use=moduleImport("a6").getMember("m6").getReturn().getMember("foo").getReturn().getMember("bar").getReturn()
+
+import foo.baz.baz as fbb #$ use=moduleImport("foo").getMember("baz").getMember("baz")
+from foo.bar.baz import quux as fbbq #$ use=moduleImport("foo").getMember("bar").getMember("baz").getMember("quux")
+from ham.bar.eggs import spam as hbes #$ use=moduleImport("ham").getMember("bar").getMember("eggs").getMember("spam")
+fbb.quux #$ use=moduleImport("foo").getMember("baz").getMember("baz").getMember("quux")
+fbbq #$ use=moduleImport("foo").getMember("bar").getMember("baz").getMember("quux")
+hbes #$ use=moduleImport("ham").getMember("bar").getMember("eggs").getMember("spam")
+
+import foo.bar.baz #$ use=moduleImport("foo")
+
+# Relative imports. These are ignored
+
+from .foo import bar
+
+from ..foobar import baz
+
+
+# Use of imports across scopes
+
+def use_m4():
+    x = m4.blah4 #$ use=moduleImport("a4").getMember("b4").getMember("c4").getMember("blah4")
+
+def local_import_use():
+    from foo import bar #$ use=moduleImport("foo").getMember("bar")
+
+    x = bar() #$ use=moduleImport("foo").getMember("bar").getReturn()
+
+from eggs import ham as spam #$ use=moduleImport("eggs").getMember("ham")
+
+def bbb():
+    f = spam #$ use=moduleImport("eggs").getMember("ham")
+
+from danger import SOURCE #$ use=moduleImport("danger").getMember("SOURCE")
+
+foo = SOURCE #$ use=moduleImport("danger").getMember("SOURCE")
+
+def change_foo():
+    global foo
+    foo = SOURCE #$ use=moduleImport("danger").getMember("SOURCE")
+
+def f():
+    global foo
+    sink(foo) #$ use=moduleImport("danger").getMember("SOURCE")
+    foo = NONSOURCE
+    change_foo()
+    sink(foo) #$ use=moduleImport("danger").getMember("SOURCE")
+
+# Star imports
+
+from unknown import * #$ use=moduleImport("unknown")
+
+hello() #$ MISSING: use=moduleImport("unknown").getMember("hello").getReturn()

python/ql/test/experimental/dataflow/ApiGraphs/test1.py

@@ -0,0 +1,6 @@
+import mypkg #$ use=moduleImport("mypkg")
+print(mypkg.foo) #$ use=moduleImport("mypkg").getMember("foo") // 42
+try:
+    print(mypkg.bar) #$ use=moduleImport("mypkg").getMember("bar")
+except AttributeError as e:
+    print(e)  # module 'mypkg' has no attribute 'bar'

python/ql/test/experimental/dataflow/ApiGraphs/test2.py

@@ -0,0 +1,4 @@
+from mypkg import foo #$ use=moduleImport("mypkg").getMember("foo")
+from mypkg import bar #$ use=moduleImport("mypkg").getMember("bar")
+print(foo) #$ use=moduleImport("mypkg").getMember("foo")
+print(bar) #$ use=moduleImport("mypkg").getMember("bar")

python/ql/test/experimental/dataflow/ApiGraphs/test3.py

@@ -0,0 +1,4 @@
+import mypkg.foo #$ use=moduleImport("mypkg")
+import mypkg.bar #$ use=moduleImport("mypkg")
+print(mypkg.foo)  #$ use=moduleImport("mypkg").getMember("foo") // <module 'mypkg.foo' ...
+print(mypkg.bar)  #$ use=moduleImport("mypkg").getMember("bar") // <module 'mypkg.bar' ...

python/ql/test/experimental/dataflow/ApiGraphs/test4.py

@@ -0,0 +1,4 @@
+import mypkg.foo as _foo #$ use=moduleImport("mypkg").getMember("foo")
+import mypkg.bar as _bar #$ use=moduleImport("mypkg").getMember("bar")
+print(_foo) #$ use=moduleImport("mypkg").getMember("foo") // <module 'mypkg.foo' ...
+print(_bar) #$ use=moduleImport("mypkg").getMember("bar") // <module 'mypkg.bar' ...

python/ql/test/experimental/dataflow/ApiGraphs/test5.py

@@ -0,0 +1,10 @@
+import mypkg #$ use=moduleImport("mypkg")
+
+print(mypkg.foo) #$ use=moduleImport("mypkg").getMember("foo") // 42
+try:
+    print(mypkg.bar) #$ use=moduleImport("mypkg").getMember("bar")
+except AttributeError as e:
+    print(e)  # module 'mypkg' has no attribute 'bar'
+
+from mypkg import bar as _bar #$ use=moduleImport("mypkg").getMember("bar")
+print(mypkg.bar) #$ use=moduleImport("mypkg").getMember("bar") // <module 'mypkg.bar' ...

python/ql/test/experimental/dataflow/ApiGraphs/test6.py

@@ -0,0 +1,6 @@
+import mypkg #$ use=moduleImport("mypkg")
+
+print(mypkg.foo) #$ use=moduleImport("mypkg").getMember("foo") // 42
+
+import mypkg.foo #$ use=moduleImport("mypkg")
+print(mypkg.foo) #$ use=moduleImport("mypkg").getMember("foo") // <module 'mypkg.foo' ...

python/ql/test/experimental/dataflow/ApiGraphs/test7.py

@@ -0,0 +1,10 @@
+from mypkg import foo #$ use=moduleImport("mypkg").getMember("foo")
+
+print(foo) #$ use=moduleImport("mypkg").getMember("foo") // 42
+
+import mypkg.foo #$ use=moduleImport("mypkg")
+print(foo) #$ use=moduleImport("mypkg").getMember("foo") // 42
+print(mypkg.foo) #$ use=moduleImport("mypkg").getMember("foo") // <module 'mypkg.foo' ...
+
+from mypkg import foo #$ use=moduleImport("mypkg").getMember("foo")
+print(foo) #$ use=moduleImport("mypkg").getMember("foo") // <module 'mypkg.foo' ...

python/ql/test/experimental/dataflow/ApiGraphs/test_deep.py

@@ -0,0 +1,4 @@
+from start.middle.end import foo #$ use=moduleImport("start").getMember("middle").getMember("end").getMember("foo")
+from start.middle.end import bar #$ use=moduleImport("start").getMember("middle").getMember("end").getMember("bar")
+print(foo) #$ use=moduleImport("start").getMember("middle").getMember("end").getMember("foo")
+print(bar) #$ use=moduleImport("start").getMember("middle").getMember("end").getMember("bar")

python/ql/test/experimental/dataflow/ApiGraphs/use.ql

@@ -0,0 +1,30 @@
+import python
+import semmle.python.dataflow.new.DataFlow
+import TestUtilities.InlineExpectationsTest
+import semmle.python.ApiGraphs
+
+class ApiUseTest extends InlineExpectationsTest {
+  ApiUseTest() { this = "ApiUseTest" }
+
+  override string getARelevantTag() { result = "use" }
+
+  private predicate relevant_node(API::Node a, DataFlow::Node n, Location l) {
+    n = a.getAUse() and l = n.getLocation()
+  }
+
+  override predicate hasActualResult(Location location, string element, string tag, string value) {
+    exists(API::Node a, DataFlow::Node n | relevant_node(a, n, location) |
+      tag = "use" and
+      // Only report the longest path on this line:
+      value =
+        max(API::Node a2, Location l2 |
+          relevant_node(a2, _, l2) and
+          l2.getFile() = location.getFile() and
+          l2.getStartLine() = location.getStartLine()
+        |
+          a2.getPath()
+        ) and
+      element = n.toString()
+    )
+  }
+}