Merge pull request #14791 from RasmusWL/python-3.12

Python: Update `.expected` to support Python 3.12

python/ql/src/Summary/options

@@ -0,0 +1 @@
+semmle-extractor-options: --lang=3

python/ql/test/2/library-tests/PointsTo/imports2/options

@@ -0,0 +1 @@
+semmle-extractor-options: --lang=2 --max-import-depth=2 -r package

python/ql/test/2/library-tests/modules/usage/options

@@ -0,0 +1 @@
+semmle-extractor-options: --lang=2 -F script

python/ql/test/3/library-tests/PointsTo/imports/Runtime.expected

@@ -0,0 +1,57 @@
+| __init__.py | 1 | ControlFlowNode for ImportExpr | Module package.module | ControlFlowNode for ImportExpr |
+| __init__.py | 2 | ControlFlowNode for ImportMember | Function module | ControlFlowNode for FunctionExpr |
+| __init__.py | 2 | ControlFlowNode for module | Function module | ControlFlowNode for FunctionExpr |
+| __init__.py | 4 | ControlFlowNode for ImportExpr | Module package | ControlFlowNode for ImportExpr |
+| __init__.py | 4 | ControlFlowNode for ImportMember | Module package.module2 | Entry node for Module package.module2 |
+| __init__.py | 4 | ControlFlowNode for module3 | Module package.module2 | Entry node for Module package.module2 |
+| __init__.py | 5 | ControlFlowNode for IntegerLiteral | int 7 | ControlFlowNode for IntegerLiteral |
+| __init__.py | 5 | ControlFlowNode for module2 | int 7 | ControlFlowNode for IntegerLiteral |
+| __init__.py | 6 | ControlFlowNode for ImportExpr | Module package | ControlFlowNode for ImportExpr |
+| __init__.py | 6 | ControlFlowNode for ImportMember | int 7 | ControlFlowNode for IntegerLiteral |
+| __init__.py | 6 | ControlFlowNode for module4 | int 7 | ControlFlowNode for IntegerLiteral |
+| __init__.py | 7 | ControlFlowNode for ImportExpr | Module package | ControlFlowNode for ImportExpr |
+| __init__.py | 7 | ControlFlowNode for ImportMember | Module package.module2 | Entry node for Module package.module2 |
+| __init__.py | 7 | ControlFlowNode for module5 | Module package.module2 | Entry node for Module package.module2 |
+| __init__.py | 8 | ControlFlowNode for ImportExpr | Module package | ControlFlowNode for ImportExpr |
+| __init__.py | 8 | ControlFlowNode for ImportMember | Module package.moduleX | Entry node for Module package.moduleX |
+| __init__.py | 8 | ControlFlowNode for moduleX | Module package.moduleX | Entry node for Module package.moduleX |
+| module2.py | 1 | ControlFlowNode for IntegerLiteral | int 0 | ControlFlowNode for IntegerLiteral |
+| module2.py | 1 | ControlFlowNode for x | int 0 | ControlFlowNode for IntegerLiteral |
+| module.py | 2 | ControlFlowNode for FunctionExpr | Function module | ControlFlowNode for FunctionExpr |
+| module.py | 2 | ControlFlowNode for module | Function module | ControlFlowNode for FunctionExpr |
+| moduleX.py | 1 | ControlFlowNode for ClassExpr | class Y | ControlFlowNode for ClassExpr |
+| moduleX.py | 1 | ControlFlowNode for Y | class Y | ControlFlowNode for ClassExpr |
+| moduleX.py | 1 | ControlFlowNode for object | builtin-class object | ControlFlowNode for object |
+| test.py | 1 | ControlFlowNode for ImportExpr | Module package | ControlFlowNode for ImportExpr |
+| test.py | 2 | ControlFlowNode for ImportMember | Function module | ControlFlowNode for FunctionExpr |
+| test.py | 2 | ControlFlowNode for module | Function module | ControlFlowNode for FunctionExpr |
+| test.py | 4 | ControlFlowNode for ImportExpr | Module package | ControlFlowNode for ImportExpr |
+| test.py | 5 | ControlFlowNode for ImportMember | Module package.x | Entry node for Module package.x |
+| test.py | 5 | ControlFlowNode for x | Module package.x | Entry node for Module package.x |
+| test.py | 8 | ControlFlowNode for C | class C | ControlFlowNode for ClassExpr |
+| test.py | 8 | ControlFlowNode for ClassExpr | class C | ControlFlowNode for ClassExpr |
+| test.py | 8 | ControlFlowNode for object | builtin-class object | ControlFlowNode for object |
+| test.py | 10 | ControlFlowNode for ImportExpr | Module package | ControlFlowNode for ImportExpr |
+| test.py | 10 | ControlFlowNode for ImportMember | int 7 | ControlFlowNode for IntegerLiteral |
+| test.py | 10 | ControlFlowNode for module2 | int 7 | ControlFlowNode for IntegerLiteral |
+| test.py | 12 | ControlFlowNode for FunctionExpr | Function f | ControlFlowNode for FunctionExpr |
+| test.py | 12 | ControlFlowNode for f | Function f | ControlFlowNode for FunctionExpr |
+| test.py | 13 | ControlFlowNode for ImportExpr | Module package | ControlFlowNode for ImportExpr |
+| test.py | 13 | ControlFlowNode for ImportMember | Module package.x | Entry node for Module package.x |
+| test.py | 13 | ControlFlowNode for x | Module package.x | Entry node for Module package.x |
+| test.py | 15 | ControlFlowNode for ImportExpr | Module package | ControlFlowNode for ImportExpr |
+| test.py | 15 | ControlFlowNode for ImportMember | Module package.moduleX | Entry node for Module package.moduleX |
+| test.py | 15 | ControlFlowNode for moduleX | Module package.moduleX | Entry node for Module package.moduleX |
+| test.py | 16 | ControlFlowNode for Attribute | class Y | ControlFlowNode for ClassExpr |
+| test.py | 16 | ControlFlowNode for moduleX | Module package.moduleX | Entry node for Module package.moduleX |
+| test.py | 19 | ControlFlowNode for ImportExpr | Module tty | ControlFlowNode for ImportExpr |
+| test.py | 19 | ControlFlowNode for tty | Module tty | ControlFlowNode for ImportExpr |
+| test.py | 22 | ControlFlowNode for Attribute | Builtin-function exc_info | ControlFlowNode for from sys import * |
+| test.py | 22 | ControlFlowNode for x | Module package.x | Entry node for Module package.x |
+| test.py | 24 | ControlFlowNode for IntegerLiteral | int 0 | ControlFlowNode for IntegerLiteral |
+| test.py | 24 | ControlFlowNode for argv | int 0 | ControlFlowNode for IntegerLiteral |
+| test.py | 27 | ControlFlowNode for ImportExpr | Module sys | ControlFlowNode for ImportExpr |
+| test.py | 31 | ControlFlowNode for argv | list object | ControlFlowNode for from sys import * |
+| test.py | 33 | ControlFlowNode for ImportExpr | Module socket | ControlFlowNode for ImportExpr |
+| test.py | 34 | ControlFlowNode for timeout | builtin-class TimeoutError | ControlFlowNode for from _socket import * |
+| x.py | 2 | ControlFlowNode for ImportExpr | Module sys | ControlFlowNode for ImportExpr |

python/ql/test/3/library-tests/PointsTo/imports/Runtime.ql

@@ -0,0 +1,9 @@
+import python
+
+from int line, ControlFlowNode f, Object o, ControlFlowNode orig
+where
+  not f.getLocation().getFile().inStdlib() and
+  f.refersTo(o, orig) and
+  line = f.getLocation().getStartLine() and
+  line != 0
+select f.getLocation().getFile().getShortName(), line, f.toString(), o.toString(), orig.toString()

python/ql/test/3/library-tests/PointsTo/imports/RuntimeWithType.expected

@@ -0,0 +1,57 @@
+| __init__.py | 1 | ControlFlowNode for ImportExpr | Module package.module | builtin-class module | ControlFlowNode for ImportExpr |
+| __init__.py | 2 | ControlFlowNode for ImportMember | Function module | builtin-class function | ControlFlowNode for FunctionExpr |
+| __init__.py | 2 | ControlFlowNode for module | Function module | builtin-class function | ControlFlowNode for FunctionExpr |
+| __init__.py | 4 | ControlFlowNode for ImportExpr | Module package | builtin-class module | ControlFlowNode for ImportExpr |
+| __init__.py | 4 | ControlFlowNode for ImportMember | Module package.module2 | builtin-class module | Entry node for Module package.module2 |
+| __init__.py | 4 | ControlFlowNode for module3 | Module package.module2 | builtin-class module | Entry node for Module package.module2 |
+| __init__.py | 5 | ControlFlowNode for IntegerLiteral | int 7 | builtin-class int | ControlFlowNode for IntegerLiteral |
+| __init__.py | 5 | ControlFlowNode for module2 | int 7 | builtin-class int | ControlFlowNode for IntegerLiteral |
+| __init__.py | 6 | ControlFlowNode for ImportExpr | Module package | builtin-class module | ControlFlowNode for ImportExpr |
+| __init__.py | 6 | ControlFlowNode for ImportMember | int 7 | builtin-class int | ControlFlowNode for IntegerLiteral |
+| __init__.py | 6 | ControlFlowNode for module4 | int 7 | builtin-class int | ControlFlowNode for IntegerLiteral |
+| __init__.py | 7 | ControlFlowNode for ImportExpr | Module package | builtin-class module | ControlFlowNode for ImportExpr |
+| __init__.py | 7 | ControlFlowNode for ImportMember | Module package.module2 | builtin-class module | Entry node for Module package.module2 |
+| __init__.py | 7 | ControlFlowNode for module5 | Module package.module2 | builtin-class module | Entry node for Module package.module2 |
+| __init__.py | 8 | ControlFlowNode for ImportExpr | Module package | builtin-class module | ControlFlowNode for ImportExpr |
+| __init__.py | 8 | ControlFlowNode for ImportMember | Module package.moduleX | builtin-class module | Entry node for Module package.moduleX |
+| __init__.py | 8 | ControlFlowNode for moduleX | Module package.moduleX | builtin-class module | Entry node for Module package.moduleX |
+| module2.py | 1 | ControlFlowNode for IntegerLiteral | int 0 | builtin-class int | ControlFlowNode for IntegerLiteral |
+| module2.py | 1 | ControlFlowNode for x | int 0 | builtin-class int | ControlFlowNode for IntegerLiteral |
+| module.py | 2 | ControlFlowNode for FunctionExpr | Function module | builtin-class function | ControlFlowNode for FunctionExpr |
+| module.py | 2 | ControlFlowNode for module | Function module | builtin-class function | ControlFlowNode for FunctionExpr |
+| moduleX.py | 1 | ControlFlowNode for ClassExpr | class Y | builtin-class type | ControlFlowNode for ClassExpr |
+| moduleX.py | 1 | ControlFlowNode for Y | class Y | builtin-class type | ControlFlowNode for ClassExpr |
+| moduleX.py | 1 | ControlFlowNode for object | builtin-class object | builtin-class type | ControlFlowNode for object |
+| test.py | 1 | ControlFlowNode for ImportExpr | Module package | builtin-class module | ControlFlowNode for ImportExpr |
+| test.py | 2 | ControlFlowNode for ImportMember | Function module | builtin-class function | ControlFlowNode for FunctionExpr |
+| test.py | 2 | ControlFlowNode for module | Function module | builtin-class function | ControlFlowNode for FunctionExpr |
+| test.py | 4 | ControlFlowNode for ImportExpr | Module package | builtin-class module | ControlFlowNode for ImportExpr |
+| test.py | 5 | ControlFlowNode for ImportMember | Module package.x | builtin-class module | Entry node for Module package.x |
+| test.py | 5 | ControlFlowNode for x | Module package.x | builtin-class module | Entry node for Module package.x |
+| test.py | 8 | ControlFlowNode for C | class C | builtin-class type | ControlFlowNode for ClassExpr |
+| test.py | 8 | ControlFlowNode for ClassExpr | class C | builtin-class type | ControlFlowNode for ClassExpr |
+| test.py | 8 | ControlFlowNode for object | builtin-class object | builtin-class type | ControlFlowNode for object |
+| test.py | 10 | ControlFlowNode for ImportExpr | Module package | builtin-class module | ControlFlowNode for ImportExpr |
+| test.py | 10 | ControlFlowNode for ImportMember | int 7 | builtin-class int | ControlFlowNode for IntegerLiteral |
+| test.py | 10 | ControlFlowNode for module2 | int 7 | builtin-class int | ControlFlowNode for IntegerLiteral |
+| test.py | 12 | ControlFlowNode for FunctionExpr | Function f | builtin-class function | ControlFlowNode for FunctionExpr |
+| test.py | 12 | ControlFlowNode for f | Function f | builtin-class function | ControlFlowNode for FunctionExpr |
+| test.py | 13 | ControlFlowNode for ImportExpr | Module package | builtin-class module | ControlFlowNode for ImportExpr |
+| test.py | 13 | ControlFlowNode for ImportMember | Module package.x | builtin-class module | Entry node for Module package.x |
+| test.py | 13 | ControlFlowNode for x | Module package.x | builtin-class module | Entry node for Module package.x |
+| test.py | 15 | ControlFlowNode for ImportExpr | Module package | builtin-class module | ControlFlowNode for ImportExpr |
+| test.py | 15 | ControlFlowNode for ImportMember | Module package.moduleX | builtin-class module | Entry node for Module package.moduleX |
+| test.py | 15 | ControlFlowNode for moduleX | Module package.moduleX | builtin-class module | Entry node for Module package.moduleX |
+| test.py | 16 | ControlFlowNode for Attribute | class Y | builtin-class type | ControlFlowNode for ClassExpr |
+| test.py | 16 | ControlFlowNode for moduleX | Module package.moduleX | builtin-class module | Entry node for Module package.moduleX |
+| test.py | 19 | ControlFlowNode for ImportExpr | Module tty | builtin-class module | ControlFlowNode for ImportExpr |
+| test.py | 19 | ControlFlowNode for tty | Module tty | builtin-class module | ControlFlowNode for ImportExpr |
+| test.py | 22 | ControlFlowNode for Attribute | Builtin-function exc_info | builtin-class builtin_function_or_method | ControlFlowNode for from sys import * |
+| test.py | 22 | ControlFlowNode for x | Module package.x | builtin-class module | Entry node for Module package.x |
+| test.py | 24 | ControlFlowNode for IntegerLiteral | int 0 | builtin-class int | ControlFlowNode for IntegerLiteral |
+| test.py | 24 | ControlFlowNode for argv | int 0 | builtin-class int | ControlFlowNode for IntegerLiteral |
+| test.py | 27 | ControlFlowNode for ImportExpr | Module sys | builtin-class module | ControlFlowNode for ImportExpr |
+| test.py | 31 | ControlFlowNode for argv | list object | builtin-class list | ControlFlowNode for from sys import * |
+| test.py | 33 | ControlFlowNode for ImportExpr | Module socket | builtin-class module | ControlFlowNode for ImportExpr |
+| test.py | 34 | ControlFlowNode for timeout | builtin-class TimeoutError | builtin-class type | ControlFlowNode for from _socket import * |
+| x.py | 2 | ControlFlowNode for ImportExpr | Module sys | builtin-class module | ControlFlowNode for ImportExpr |

python/ql/test/3/library-tests/PointsTo/imports/RuntimeWithType.ql

@@ -0,0 +1,10 @@
+import python
+
+from int line, ControlFlowNode f, Object o, ClassObject cls, ControlFlowNode orig
+where
+  not f.getLocation().getFile().inStdlib() and
+  f.refersTo(o, cls, orig) and
+  line = f.getLocation().getStartLine() and
+  line != 0
+select f.getLocation().getFile().getShortName(), line, f.toString(), o.toString(), cls.toString(),
+  orig.toString()

python/ql/test/3/library-tests/PointsTo/imports/options

@@ -0,0 +1 @@
+semmle-extractor-options: --lang=3 --max-import-depth=2 -r package

python/ql/test/3/library-tests/PointsTo/imports/package/__init__.py

@@ -0,0 +1,15 @@
+from .module \
+import module
+
+from . import module2 as module3
+module2 = 7
+from . import module2 as module4
+from . import module3 as module5
+from package import moduleX
+
+#We should now have:
+#module2 = 7
+#module3 = package.module2
+#module4 = 7
+#module5 = package.module2
+#moduleX = package.moduleX

python/ql/test/3/library-tests/PointsTo/imports/package/module.py

@@ -0,0 +1,3 @@
+
+def module(args):
+    pass

python/ql/test/3/library-tests/PointsTo/imports/package/module2.py

@@ -0,0 +1 @@
+x = 0

python/ql/test/3/library-tests/PointsTo/imports/package/moduleX.py

@@ -0,0 +1,2 @@
+class Y(object):
+    pass

python/ql/test/3/library-tests/PointsTo/imports/package/x.py

@@ -0,0 +1,2 @@
+
+from sys import *

python/ql/test/3/library-tests/PointsTo/imports/test.py

@@ -0,0 +1,34 @@
+from package \
+import module
+
+from package \
+import x
+#Should work correctly in nested scopes as well.
+
+class C(object):
+
+    from package import module2
+
+    def f(self):
+        from package import x
+
+from package import moduleX
+moduleX.Y
+
+#A small stdlib module to test version handling.
+import tty
+
+#Check imports of builtin-objects using import * with no corresponding variable.
+x.exc_info
+
+argv = 0
+
+try:
+    from sys import *
+except:
+    pass
+
+argv
+
+from socket import *
+timeout

python/ql/test/3/library-tests/modules/usage/ModuleUsage.expected

@@ -0,0 +1,6 @@
+| file://:0:0:0:0 | Module sys | isUsedAsModule |
+| file://:0:0:0:0 | Module sys.monitoring | isUsedAsModule |
+| imported.py:0:0:0:0 | Module imported | isUsedAsModule |
+| main.py:0:0:0:0 | Module main | isUsedAsScript |
+| myscript.py:0:0:0:0 | Script myscript | isUsedAsScript |
+| script:0:0:0:0 | Script script | isUsedAsScript |

python/ql/test/3/library-tests/modules/usage/ModuleUsage.ql

@@ -0,0 +1,16 @@
+import python
+
+from ModuleValue mv, string usage
+where
+  // builtin module has different name in Python 2 and 3
+  not mv = Module::builtinModule() and
+  (
+    mv.isUsedAsModule() and usage = "isUsedAsModule"
+    or
+    mv.isUsedAsScript() and usage = "isUsedAsScript"
+    or
+    not mv.isUsedAsModule() and
+    not mv.isUsedAsScript() and
+    usage = "<UNKNOWN>"
+  )
+select mv, usage

python/ql/test/3/library-tests/modules/usage/imported.py

@@ -0,0 +1,6 @@
+def func():
+    pass
+
+if __name__ == "__main__":
+    print("I could have done something interesting...")
+    print("but I didn't")

python/ql/test/3/library-tests/modules/usage/main.py

@@ -0,0 +1,5 @@
+import imported
+
+if __name__ == "__main__":
+    imported.func()
+    print('Done')

python/ql/test/3/library-tests/modules/usage/myscript.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+
+print("I'm actually a script you see ;)")

python/ql/test/3/library-tests/modules/usage/options

@@ -0,0 +1 @@
+semmle-extractor-options: --lang=3 -F script

python/ql/test/3/library-tests/modules/usage/script

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+
+print('Under construction :)')

python/ql/test/3/library-tests/types/properties/BuiltinProperties.expected

@@ -1,9 +1,12 @@
 | builtin-class object | __class__ | Property __class__ | method-wrapper __get__ | method-wrapper __set__ | method-wrapper __delete__ |
 | builtin-class type | __abstractmethods__ | Property __abstractmethods__ | method-wrapper __get__ | method-wrapper __set__ | method-wrapper __delete__ |
+| builtin-class type | __annotations__ | Property __annotations__ | method-wrapper __get__ | method-wrapper __set__ | method-wrapper __delete__ |
 | builtin-class type | __bases__ | Property __bases__ | method-wrapper __get__ | method-wrapper __set__ | method-wrapper __delete__ |
 | builtin-class type | __dict__ | Property __dict__ | method-wrapper __get__ | method-wrapper __set__ | method-wrapper __delete__ |
 | builtin-class type | __doc__ | Property __doc__ | method-wrapper __get__ | method-wrapper __set__ | method-wrapper __delete__ |
 | builtin-class type | __module__ | Property __module__ | method-wrapper __get__ | method-wrapper __set__ | method-wrapper __delete__ |
+| builtin-class type | __mro__ | Property __mro__ | method-wrapper __get__ | method-wrapper __set__ | method-wrapper __delete__ |
 | builtin-class type | __name__ | Property __name__ | method-wrapper __get__ | method-wrapper __set__ | method-wrapper __delete__ |
 | builtin-class type | __qualname__ | Property __qualname__ | method-wrapper __get__ | method-wrapper __set__ | method-wrapper __delete__ |
-| builtin-class type | __text_signature__ | Property __text_signature__ | method-wrapper __get__ | method-wrapper __set__ | method-wrapper __delete__ |
+| builtin-class type | __text_signature__ | Property __text_signature__ | method-wrapper __get__ | method-wrapper __set__ | method-wrapper __delete__ |
+| builtin-class type | __type_params__ | Property __type_params__ | method-wrapper __get__ | method-wrapper __set__ | method-wrapper __delete__ |

python/ql/test/3/query-tests/Summary/LinesOfCode.expected

@@ -0,0 +1 @@
+| 51 |

python/ql/test/3/query-tests/Summary/LinesOfCode.qlref

@@ -0,0 +1 @@
+Summary/LinesOfCode.ql

python/ql/test/3/query-tests/Summary/LinesOfUserCode.expected

@@ -0,0 +1 @@
+| 11 |

python/ql/test/3/query-tests/Summary/LinesOfUserCode.qlref

@@ -0,0 +1 @@
+Summary/LinesOfUserCode.ql

python/ql/test/3/query-tests/Summary/also_python_code

@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+
+# although this is actually Python code, it is not included by the extractor by default.
+
+print("this is also code")
+
+print("but just dummy code")

python/ql/test/3/query-tests/Summary/my_file.py

@@ -0,0 +1,26 @@
+"""
+module level docstring
+
+is not included
+"""
+# this line is not code
+
+# `tty` was chosen for stability over python versions (so we don't get diffrent results
+# on different computers, that has different versions of Python).
+#
+# According to https://github.com/python/cpython/tree/master/Lib (at 2021-04-23) `tty`
+# was last changed in 2001, so chances of this being changed in the future are slim.
+import tty
+
+s = """
+all these lines are code
+"""
+
+print(s)
+
+def func():
+    """
+    this string is a doc-string. Although the module-level docstring is not considered
+    code, this one apparently is ¯\_(ツ)_/¯
+    """
+    pass

python/ql/test/3/query-tests/Summary/not_python

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# Although this is valid python code, it should not be counted as such.
+
+print("foo")

python/ql/test/experimental/dataflow/coverage/datamodel.py

@@ -186,14 +186,20 @@ SINK(asyncio.run(c.coro(SOURCE))) # $ MISSING: flow
 class A:
 
   def __await__(self):
-    # yield SOURCE  -- see https://groups.google.com/g/dev-python/c/_lrrc-vp9TI?pli=1
-    return (yield from asyncio.coroutine(lambda: SOURCE)())
+    fut = asyncio.Future()
+    fut.set_result(SOURCE)
+    yield from fut
 
-async def agen(x):
+async def atest_custom_await_impl():
   a = A()
-  return await a
+  x = await a
+  # TODO: Figure out how to actually return something from our custom __await__
+  # implementation. The problem is we have to play nicely with the asyncio framework,
+  # which have their own expectations on what a return value from __await__ should look
+  # like.
+  assert x is None
+  SINK_F(x)
 
-SINK(asyncio.run(agen(SOURCE))) # $ MISSING: flow
 
 # Asynchronous generator functions
 # A function or method which is defined using async def and which uses the yield statement is called a asynchronous generator function. Such a function, when called, returns an asynchronous iterator object which can be used in an async for statement to execute the body of the function.

python/ql/test/library-tests/PointsTo/imports/options

@@ -1 +0,0 @@
-semmle-extractor-options: --max-import-depth=2 -r package

python/ql/test/library-tests/modules/usage/options

@@ -1 +0,0 @@
-semmle-extractor-options: -F script