Merge pull request #16212 from github/tausbn/python-add-copy-of-internal-tests

Python: Add copy of internal tests

python/ql/test/2/extractor-tests/ellipsis/Test.expected

@@ -0,0 +1 @@
+| 2 |

python/ql/test/2/extractor-tests/ellipsis/Test.ql

@@ -0,0 +1,3 @@
+import python
+
+select count(Ellipsis e)

python/ql/test/2/extractor-tests/ellipsis/test.py

@@ -0,0 +1,6 @@
+a = {}
+a[...] = 1
+b = a[
+  ...]
+
+print(b)

python/ql/test/2/extractor-tests/exec/successors.expected

@@ -0,0 +1,23 @@
+| test.py | 0 | Entry node for Module test | 1 | ControlFlowNode for FunctionExpr | normal |
+| test.py | 1 | ControlFlowNode for FunctionExpr | 1 | ControlFlowNode for f | normal |
+| test.py | 1 | ControlFlowNode for f | 0 | Exit node for Module test | normal |
+| test.py | 1 | ControlFlowNode for x | 2 | ControlFlowNode for exec | normal |
+| test.py | 1 | Entry node for Function f | 1 | ControlFlowNode for x | normal |
+| test.py | 2 | ControlFlowNode for Str | 2 | ControlFlowNode for exec() | normal |
+| test.py | 2 | ControlFlowNode for exec | 2 | ControlFlowNode for Str | normal |
+| test.py | 2 | ControlFlowNode for exec() | 3 | ControlFlowNode for x | normal |
+| test.py | 3 | ControlFlowNode for Return | 1 | Exit node for Function f | normal |
+| test.py | 3 | ControlFlowNode for x | 3 | ControlFlowNode for Return | normal |
+| unicode.py | 0 | Entry node for Module unicode | 1 | ControlFlowNode for ImportExpr | normal |
+| unicode.py | 1 | ControlFlowNode for ImportExpr | 1 | ControlFlowNode for ImportMember | normal |
+| unicode.py | 1 | ControlFlowNode for ImportMember | 1 | ControlFlowNode for unicode_literals | normal |
+| unicode.py | 1 | ControlFlowNode for unicode_literals | 3 | ControlFlowNode for FunctionExpr | normal |
+| unicode.py | 3 | ControlFlowNode for FunctionExpr | 3 | ControlFlowNode for f | normal |
+| unicode.py | 3 | ControlFlowNode for f | 0 | Exit node for Module unicode | normal |
+| unicode.py | 3 | ControlFlowNode for x | 4 | ControlFlowNode for exec | normal |
+| unicode.py | 3 | Entry node for Function f | 3 | ControlFlowNode for x | normal |
+| unicode.py | 4 | ControlFlowNode for Str | 4 | ControlFlowNode for exec() | normal |
+| unicode.py | 4 | ControlFlowNode for exec | 4 | ControlFlowNode for Str | normal |
+| unicode.py | 4 | ControlFlowNode for exec() | 5 | ControlFlowNode for x | normal |
+| unicode.py | 5 | ControlFlowNode for Return | 3 | Exit node for Function f | normal |
+| unicode.py | 5 | ControlFlowNode for x | 5 | ControlFlowNode for Return | normal |

python/ql/test/2/extractor-tests/exec/successors.ql

@@ -0,0 +1,14 @@
+import python
+
+from ControlFlowNode p, ControlFlowNode s, string kind, string filename
+where
+  p.getASuccessor() = s and
+  (
+    p.getAnExceptionalSuccessor() = s and kind = "exception"
+    or
+    not p.getAnExceptionalSuccessor() = s and kind = "normal"
+  ) and
+  filename = p.getLocation().getFile().getShortName() and
+  not filename = "__future__.py"
+select filename, p.getLocation().getStartLine(), p.toString(), s.getLocation().getStartLine(),
+  s.toString(), kind

python/ql/test/2/extractor-tests/exec/test.py

@@ -0,0 +1,3 @@
+def f(x):
+    exec("raise thing")
+    return x

python/ql/test/2/extractor-tests/exec/unicode.py

@@ -0,0 +1,5 @@
+from __future__ import unicode_literals
+
+def f(x):
+    exec("raise thing")
+    return x

python/ql/test/2/extractor-tests/hidden/options

@@ -0,0 +1,2 @@
+semmle-extractor-options: -R . -p . --filter exclude:**/src_archive/**
+

python/ql/test/2/extractor-tests/hidden/test.expected

@@ -0,0 +1,4 @@
+| folder/module.py |
+| package |
+| package/__init__.py |
+| package/module.py |

python/ql/test/2/extractor-tests/hidden/test.ql

@@ -0,0 +1,5 @@
+import python
+
+from Container f
+where exists(Module m | m.getPath() = f)
+select f.toString()

python/ql/test/2/extractor-tests/import_depth/options

@@ -0,0 +1 @@
+semmle-extractor-options: --lang=2 --max-import-depth=1

python/ql/test/2/extractor-tests/import_depth/test.expected

@@ -0,0 +1,4 @@
+| Module package |
+| Module package.__init__ |
+| Module sys |
+| Module test |

python/ql/test/2/extractor-tests/import_depth/test.py

@@ -0,0 +1 @@
+import package

python/ql/test/2/extractor-tests/import_depth/test.ql

@@ -0,0 +1,6 @@
+import python
+
+from ModuleObject m
+/* Exclude the builtins module as it has a different name under 2 and 3. */
+where not m = theBuiltinModuleObject()
+select m.toString()

python/ql/test/2/extractor-tests/multibyte/Test.expected

@@ -0,0 +1,6 @@
+| test.py:3:1:3:11 | Str | \u0111\u0142e\u00b6\u014b\u00b6\u0142\u014b |
+| test.py:3:15:3:19 | Str | hi |
+| test.py:3:23:3:27 | Str | \n |
+| test.py:4:1:4:11 | Str | \u0111\u0142e\u00b6\u014b\u00b6\u0142\u014b |
+| test.py:4:15:4:19 | Str | hi |
+| test.py:4:23:4:27 | Str | \n |

python/ql/test/2/extractor-tests/multibyte/Test.ql

@@ -0,0 +1,4 @@
+import python
+
+from StrConst s
+select s, s.getText()

python/ql/test/2/extractor-tests/multibyte/options

@@ -0,0 +1 @@
+automatic_locations: true

python/ql/test/2/extractor-tests/multibyte/test.py

@@ -0,0 +1,4 @@
+#coding=utf8
+
+b"đłe¶ŋ¶łŋ" + b"hi" + b"\n"
+u"đłe¶ŋ¶łŋ" + u"hi" + u"\n"

python/ql/test/2/extractor-tests/normalise/Numbers.expected

@@ -0,0 +1,7 @@
+| int 0 | 0 |
+| int 1 | 1 |
+| int 2 | 2 |
+| int 123745 | 123745 |
+| int 268435455 | 268435455 |
+| int 17216961135462248174 | 17216961135462248174 |
+| int 100000000000000000000000000000000000000000000 | 100000000000000000000000000000000000000000000 |

python/ql/test/2/extractor-tests/normalise/Numbers.ql

@@ -0,0 +1,16 @@
+/**
+ * Much of the QL library handling integral values assumes that
+ * equivalence of the DB entities implies equivalence of the
+ * Python object and vice-versa.
+ * In Python 2, 1L == 1, which can cause problems, so we
+ * normalise all longs to ints.
+ */
+
+import python
+
+from NumericObject n
+where
+  exists(IntegerLiteral i | i.getLiteralObject() = n |
+    i.getEnclosingModule().getFile().getShortName() = "test.py"
+  )
+select n.toString(), n.repr()

python/ql/test/2/extractor-tests/normalise/test.py

@@ -0,0 +1,23 @@
+
+#ints
+0
+0L
+1
+1L
+2
+2L
+123745
+123745L
+
+0xfffffff
+0xfffffffL
+
+#Avoid values between 32 and 64 bits, as long is 32 bits on windows and 64 bits on 64 bit linux machines.
+
+
+
+#longs
+0xeeeeeeeeeeeeeeee
+0xeeeeeeeeeeeeeeeeL
+100000000000000000000000000000000000000000000
+100000000000000000000000000000000000000000000L

python/ql/test/2/extractor-tests/object_hash/Success.expected

@@ -0,0 +1 @@
+| 1 |

python/ql/test/2/extractor-tests/object_hash/Success.ql

@@ -0,0 +1,4 @@
+import python
+
+where exists(theSysModuleObject())
+select 1

python/ql/test/2/extractor-tests/object_hash/test.py

@@ -0,0 +1 @@
+import cStringIO

python/ql/test/2/extractor-tests/old_style_disequality/test.expected

@@ -0,0 +1 @@
+| test.py:1:1:1:6 | Compare |

python/ql/test/2/extractor-tests/old_style_disequality/test.py

@@ -0,0 +1 @@
+1 <> 2

python/ql/test/2/extractor-tests/old_style_disequality/test.ql

@@ -0,0 +1,3 @@
+import python
+
+select any(Compare o)

python/ql/test/2/extractor-tests/options

@@ -0,0 +1 @@
+automatic_locations: true

python/ql/test/2/extractor-tests/syntax_error/Modules.expected

@@ -0,0 +1 @@
+| Module test | test |

python/ql/test/2/extractor-tests/syntax_error/Modules.ql

@@ -0,0 +1,4 @@
+import python
+
+from Module m
+select m.toString(), m.getName()

python/ql/test/2/extractor-tests/syntax_error/Test.expected

@@ -0,0 +1 @@
+| Syntax Error |

python/ql/test/2/extractor-tests/syntax_error/Test.ql

@@ -0,0 +1,4 @@
+import python
+
+from SyntaxError s
+select s.toString()

python/ql/test/2/extractor-tests/syntax_error/test.py

@@ -0,0 +1,2 @@
+
+Not python at all :(

python/ql/test/2/library-tests/locations2.7plus/Locations.expected

@@ -0,0 +1,35 @@
+| Call | 2 | 17 | 2 | 28 |
+| DictComp | 2 | 1 | 2 | 29 |
+| ExprStmt | 2 | 1 | 2 | 29 |
+| ExprStmt | 2 | 2 | 2 | 4 |
+| ExprStmt | 3 | 1 | 3 | 14 |
+| ExprStmt | 4 | 1 | 4 | 16 |
+| ExprStmt | 4 | 3 | 4 | 3 |
+| FastLocalsFunction | 2 | 1 | 2 | 29 |
+| FastLocalsFunction | 4 | 1 | 4 | 16 |
+| For | 2 | 1 | 2 | 29 |
+| For | 4 | 1 | 4 | 16 |
+| FunctionMetrics | 2 | 1 | 2 | 29 |
+| FunctionMetrics | 4 | 1 | 4 | 16 |
+| IntegerLiteral | 3 | 3 | 3 | 3 |
+| IntegerLiteral | 3 | 6 | 3 | 6 |
+| IntegerLiteral | 3 | 9 | 3 | 9 |
+| IntegerLiteral | 3 | 12 | 3 | 12 |
+| ModuleMetrics | 0 | 0 | 0 | 0 |
+| Name | 2 | 1 | 2 | 29 |
+| Name | 2 | 2 | 2 | 2 |
+| Name | 2 | 4 | 2 | 4 |
+| Name | 2 | 10 | 2 | 10 |
+| Name | 2 | 12 | 2 | 12 |
+| Name | 2 | 17 | 2 | 25 |
+| Name | 2 | 27 | 2 | 27 |
+| Name | 4 | 1 | 4 | 16 |
+| Name | 4 | 3 | 4 | 3 |
+| Name | 4 | 9 | 4 | 9 |
+| Name | 4 | 14 | 4 | 14 |
+| Set | 3 | 1 | 3 | 14 |
+| SetComp | 4 | 1 | 4 | 16 |
+| Tuple | 2 | 2 | 2 | 4 |
+| Tuple | 2 | 10 | 2 | 12 |
+| Yield | 2 | 2 | 2 | 4 |
+| Yield | 4 | 3 | 4 | 3 |

python/ql/test/2/library-tests/locations2.7plus/Locations.ql

@@ -0,0 +1,5 @@
+import python
+
+from AstNode ast, Location l
+where ast.getLocation() = l
+select ast.getAQlClass(), l.getStartLine(), l.getStartColumn(), l.getEndLine(), l.getEndColumn()

python/ql/test/2/library-tests/locations2.7plus/test.py

@@ -0,0 +1,5 @@
+
+{k:v for k,v in enumerate(y)}
+{ 1, 2, 3, 4 }
+{ i for i in y }
+

python/ql/test/3/extractor-tests/Kannada/Test.expected

@@ -0,0 +1 @@
+| Script \u0ca8\u0ca8\u0ccd\u0ca8_\u0cb8\u0ccd\u0c95\u0ccd\u0cb0\u0cbf\u0caa\u0ccd\u0c9f\u0ccd.py |

python/ql/test/3/extractor-tests/Kannada/Test.ql

@@ -0,0 +1,4 @@
+import python
+
+from Module m
+select m.toString()

python/ql/test/3/extractor-tests/Kannada/ನನ್ನ_ಸ್ಕ್ರಿಪ್ಟ್.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+import sys
+from dbgimporter import import_and_enable_debugger
+import_and_enable_debugger()
+def ಏನಾದರೂ_ಮಾಡು():
+    print('ಏನೋ ಮಾಡಿದೆ'.encode(sys.stdout.encoding, errors='replace'))
+
+
+ಏನಾದರೂ_ಮಾಡು()

python/ql/test/3/extractor-tests/annotations/Assign.expected

@@ -0,0 +1,8 @@
+| 1 | AnnAssign | x | int | IntegerLiteral |
+| 2 | AnnAssign | y | Thing | something() |
+| 3 | AnnAssign | z | Any | ---- |
+| 4 | AnnAssign | Attribute | complex | ---- |
+| 5 | AnnAssign | Attribute | not_simple | None |
+| 8 | AnnAssign | a | int | IntegerLiteral |
+| 9 | AnnAssign | b | Thing | something() |
+| 10 | AnnAssign | c | Any | ---- |

python/ql/test/3/extractor-tests/annotations/Assign.ql

@@ -0,0 +1,9 @@
+import python
+
+from AnnAssign a, string value
+where
+  value = a.getValue().toString()
+  or
+  not exists(a.getValue()) and value = "----"
+select a.getLocation().getStartLine(), a.toString(), a.getTarget().toString(),
+  a.getAnnotation().toString(), value

python/ql/test/3/extractor-tests/annotations/Locations.expected

@@ -0,0 +1,38 @@
+| 0 | 0 | 0 | 0 | Module test |
+| 1 | 1 | 1 | 1 | x |
+| 1 | 1 | 1 | 10 | AnnAssign |
+| 1 | 4 | 1 | 6 | int |
+| 1 | 10 | 1 | 10 | IntegerLiteral |
+| 2 | 1 | 2 | 1 | y |
+| 2 | 1 | 2 | 22 | AnnAssign |
+| 2 | 4 | 2 | 8 | Thing |
+| 2 | 12 | 2 | 20 | something |
+| 2 | 12 | 2 | 22 | something() |
+| 3 | 1 | 3 | 1 | z |
+| 3 | 1 | 3 | 6 | AnnAssign |
+| 3 | 4 | 3 | 6 | Any |
+| 4 | 1 | 4 | 1 | a |
+| 4 | 1 | 4 | 3 | Attribute |
+| 4 | 1 | 4 | 13 | AnnAssign |
+| 4 | 7 | 4 | 13 | complex |
+| 5 | 1 | 5 | 1 | c |
+| 5 | 1 | 5 | 3 | Attribute |
+| 5 | 1 | 5 | 23 | AnnAssign |
+| 5 | 7 | 5 | 16 | not_simple |
+| 5 | 20 | 5 | 23 | None |
+| 7 | 1 | 7 | 8 | Function f |
+| 7 | 1 | 7 | 8 | FunctionDef |
+| 7 | 1 | 7 | 8 | FunctionExpr |
+| 7 | 5 | 7 | 5 | f |
+| 8 | 5 | 8 | 5 | a |
+| 8 | 5 | 8 | 14 | AnnAssign |
+| 8 | 8 | 8 | 10 | int |
+| 8 | 14 | 8 | 14 | IntegerLiteral |
+| 9 | 5 | 9 | 5 | b |
+| 9 | 5 | 9 | 26 | AnnAssign |
+| 9 | 8 | 9 | 12 | Thing |
+| 9 | 16 | 9 | 24 | something |
+| 9 | 16 | 9 | 26 | something() |
+| 10 | 5 | 10 | 5 | c |
+| 10 | 5 | 10 | 10 | AnnAssign |
+| 10 | 8 | 10 | 10 | Any |

python/ql/test/3/extractor-tests/annotations/Locations.ql

@@ -0,0 +1,5 @@
+import python
+
+from AstNode a, Location l
+where l = a.getLocation()
+select l.getStartLine(), l.getStartColumn(), l.getEndLine(), l.getEndColumn(), a.toString()

python/ql/test/3/extractor-tests/annotations/Successors.expected

@@ -0,0 +1,34 @@
+| 0 | Module test | 1 | IntegerLiteral |
+| 1 | AnnAssign | 2 | something |
+| 1 | IntegerLiteral | 1 | x |
+| 1 | int | 1 | AnnAssign |
+| 1 | x | 1 | int |
+| 2 | AnnAssign | 3 | z |
+| 2 | Thing | 2 | AnnAssign |
+| 2 | something | 2 | something() |
+| 2 | something() | 2 | y |
+| 2 | y | 2 | Thing |
+| 3 | AnnAssign | 4 | a |
+| 3 | Any | 3 | AnnAssign |
+| 3 | z | 3 | Any |
+| 4 | AnnAssign | 5 | None |
+| 4 | Attribute | 4 | complex |
+| 4 | a | 4 | Attribute |
+| 4 | complex | 4 | AnnAssign |
+| 5 | AnnAssign | 7 | FunctionExpr |
+| 5 | Attribute | 5 | not_simple |
+| 5 | None | 5 | c |
+| 5 | c | 5 | Attribute |
+| 5 | not_simple | 5 | AnnAssign |
+| 7 | Function f | 8 | IntegerLiteral |
+| 7 | FunctionExpr | 7 | f |
+| 7 | f | 0 | Module test |
+| 8 | AnnAssign | 9 | something |
+| 8 | IntegerLiteral | 8 | a |
+| 8 | a | 8 | AnnAssign |
+| 9 | AnnAssign | 10 | c |
+| 9 | b | 9 | AnnAssign |
+| 9 | something | 9 | something() |
+| 9 | something() | 9 | b |
+| 10 | AnnAssign | 7 | Function f |
+| 10 | c | 10 | AnnAssign |

python/ql/test/3/extractor-tests/annotations/Successors.ql

@@ -0,0 +1,6 @@
+import python
+
+from ControlFlowNode p, ControlFlowNode s
+where p.getASuccessor() = s
+select p.getLocation().getStartLine(), p.getNode().toString(), s.getLocation().getStartLine(),
+  s.getNode().toString()

python/ql/test/3/extractor-tests/annotations/test.py

@@ -0,0 +1,10 @@
+x: int = 0
+y: Thing = something()
+z: Any
+a.x : complex
+c.y : not_simple = None
+
+def f():
+    a: int = 0
+    b: Thing = something()
+    c: Any

python/ql/test/3/extractor-tests/async3.5/AsyncFor.expected

@@ -0,0 +1 @@
+| 3 | For |

python/ql/test/3/extractor-tests/async3.5/AsyncFor.ql

@@ -0,0 +1,4 @@
+import python
+
+from AsyncFor a
+select a.getLocation().getStartLine(), a.toString()

python/ql/test/3/extractor-tests/async3.5/AsyncWith.expected

@@ -0,0 +1 @@
+| 4 | With |

python/ql/test/3/extractor-tests/async3.5/AsyncWith.ql

@@ -0,0 +1,4 @@
+import python
+
+from AsyncWith aw
+select aw.getLocation().getStartLine(), aw.toString()

python/ql/test/3/extractor-tests/async3.5/Await.expected

@@ -0,0 +1 @@
+| 6 | Await |

python/ql/test/3/extractor-tests/async3.5/Await.ql

@@ -0,0 +1,4 @@
+import python
+
+from Await a
+select a.getLocation().getStartLine(), a.toString()

python/ql/test/3/extractor-tests/async3.5/test.py

@@ -0,0 +1,8 @@
+
+async def foo():
+    async for x in y:
+        async with a as b:
+            pass
+    await z
+
+

python/ql/test/3/extractor-tests/async3.6/AsyncFor.expected

@@ -0,0 +1,9 @@
+| 7 | For | normal |
+| 14 | For | async |
+| 15 | For | normal |
+| 16 | For | async |
+| 17 | For | normal |
+| 22 | For | normal |
+| 23 | For | normal |
+| 24 | For | normal |
+| 25 | For | normal |

python/ql/test/3/extractor-tests/async3.6/AsyncFor.ql

@@ -0,0 +1,8 @@
+import python
+
+from For f, string kind
+where
+  f instanceof AsyncFor and kind = "async"
+  or
+  not f instanceof AsyncFor and kind = "normal"
+select f.getLocation().getStartLine(), f.toString(), kind

python/ql/test/3/extractor-tests/async3.6/Await.expected

@@ -0,0 +1,3 @@
+| 9 | Await |
+| 15 | Await |
+| 17 | Await |

python/ql/test/3/extractor-tests/async3.6/Await.ql

@@ -0,0 +1,4 @@
+import python
+
+from Await a
+select a.getLocation().getStartLine(), a.toString()

python/ql/test/3/extractor-tests/async3.6/Locations.expected

@@ -0,0 +1,159 @@
+| 0 | 0 | 0 | 0 | Module test |
+| 5 | 7 | 5 | 28 | Function ticker |
+| 5 | 7 | 5 | 28 | FunctionDef |
+| 5 | 7 | 5 | 28 | FunctionExpr |
+| 5 | 11 | 5 | 16 | ticker |
+| 5 | 18 | 5 | 22 | delay |
+| 5 | 25 | 5 | 26 | to |
+| 6 | 5 | 6 | 61 | ExprStmt |
+| 6 | 5 | 6 | 61 | Str |
+| 7 | 5 | 7 | 23 | For |
+| 7 | 9 | 7 | 9 | i |
+| 7 | 14 | 7 | 18 | range |
+| 7 | 14 | 7 | 22 | range() |
+| 7 | 20 | 7 | 21 | to |
+| 8 | 9 | 8 | 15 | ExprStmt |
+| 8 | 9 | 8 | 15 | Yield |
+| 8 | 15 | 8 | 15 | i |
+| 9 | 9 | 9 | 34 | Await |
+| 9 | 9 | 9 | 34 | ExprStmt |
+| 9 | 15 | 9 | 21 | asyncio |
+| 9 | 15 | 9 | 27 | Attribute |
+| 9 | 15 | 9 | 34 | Attribute() |
+| 9 | 29 | 9 | 33 | delay |
+| 13 | 7 | 13 | 14 | Function f |
+| 13 | 7 | 13 | 14 | FunctionDef |
+| 13 | 7 | 13 | 14 | FunctionExpr |
+| 13 | 11 | 13 | 11 | f |
+| 14 | 5 | 14 | 5 | a |
+| 14 | 5 | 14 | 43 | AssignStmt |
+| 14 | 9 | 14 | 43 | .0 |
+| 14 | 9 | 14 | 43 | For |
+| 14 | 9 | 14 | 43 | Function listcomp |
+| 14 | 9 | 14 | 43 | ListComp |
+| 14 | 10 | 14 | 10 | ExprStmt |
+| 14 | 10 | 14 | 10 | Yield |
+| 14 | 10 | 14 | 10 | i |
+| 14 | 22 | 14 | 22 | i |
+| 14 | 27 | 14 | 31 | aiter |
+| 14 | 27 | 14 | 33 | aiter() |
+| 14 | 38 | 14 | 38 | i |
+| 14 | 38 | 14 | 42 | BinaryExpr |
+| 14 | 38 | 14 | 42 | If |
+| 14 | 42 | 14 | 42 | IntegerLiteral |
+| 15 | 5 | 15 | 5 | b |
+| 15 | 5 | 15 | 59 | AssignStmt |
+| 15 | 9 | 15 | 59 | .0 |
+| 15 | 9 | 15 | 59 | For |
+| 15 | 9 | 15 | 59 | Function listcomp |
+| 15 | 9 | 15 | 59 | ListComp |
+| 15 | 10 | 15 | 20 | Await |
+| 15 | 10 | 15 | 20 | ExprStmt |
+| 15 | 10 | 15 | 20 | Yield |
+| 15 | 16 | 15 | 18 | fun |
+| 15 | 16 | 15 | 20 | fun() |
+| 15 | 26 | 15 | 28 | fun |
+| 15 | 33 | 15 | 37 | funcs |
+| 15 | 42 | 15 | 58 | Await |
+| 15 | 42 | 15 | 58 | If |
+| 15 | 48 | 15 | 56 | condition |
+| 15 | 48 | 15 | 58 | condition() |
+| 16 | 5 | 16 | 5 | c |
+| 16 | 5 | 16 | 43 | AssignStmt |
+| 16 | 9 | 16 | 43 | .0 |
+| 16 | 9 | 16 | 43 | For |
+| 16 | 9 | 16 | 43 | Function setcomp |
+| 16 | 9 | 16 | 43 | SetComp |
+| 16 | 10 | 16 | 10 | ExprStmt |
+| 16 | 10 | 16 | 10 | Yield |
+| 16 | 10 | 16 | 10 | i |
+| 16 | 22 | 16 | 22 | i |
+| 16 | 27 | 16 | 31 | aiter |
+| 16 | 27 | 16 | 33 | aiter() |
+| 16 | 38 | 16 | 38 | i |
+| 16 | 38 | 16 | 42 | BinaryExpr |
+| 16 | 38 | 16 | 42 | If |
+| 16 | 42 | 16 | 42 | IntegerLiteral |
+| 17 | 5 | 17 | 5 | d |
+| 17 | 5 | 17 | 59 | AssignStmt |
+| 17 | 9 | 17 | 59 | .0 |
+| 17 | 9 | 17 | 59 | For |
+| 17 | 9 | 17 | 59 | Function setcomp |
+| 17 | 9 | 17 | 59 | SetComp |
+| 17 | 10 | 17 | 20 | Await |
+| 17 | 10 | 17 | 20 | ExprStmt |
+| 17 | 10 | 17 | 20 | Yield |
+| 17 | 16 | 17 | 18 | fun |
+| 17 | 16 | 17 | 20 | fun() |
+| 17 | 26 | 17 | 28 | fun |
+| 17 | 33 | 17 | 37 | funcs |
+| 17 | 42 | 17 | 58 | Await |
+| 17 | 42 | 17 | 58 | If |
+| 17 | 48 | 17 | 56 | condition |
+| 17 | 48 | 17 | 58 | condition() |
+| 21 | 1 | 21 | 8 | Function g |
+| 21 | 1 | 21 | 8 | FunctionDef |
+| 21 | 1 | 21 | 8 | FunctionExpr |
+| 21 | 5 | 21 | 5 | g |
+| 22 | 5 | 22 | 5 | a |
+| 22 | 5 | 22 | 36 | AssignStmt |
+| 22 | 9 | 22 | 36 | .0 |
+| 22 | 9 | 22 | 36 | For |
+| 22 | 9 | 22 | 36 | Function listcomp |
+| 22 | 9 | 22 | 36 | ListComp |
+| 22 | 10 | 22 | 10 | ExprStmt |
+| 22 | 10 | 22 | 10 | Yield |
+| 22 | 10 | 22 | 10 | i |
+| 22 | 16 | 22 | 16 | i |
+| 22 | 21 | 22 | 24 | iter |
+| 22 | 21 | 22 | 26 | iter() |
+| 22 | 31 | 22 | 31 | i |
+| 22 | 31 | 22 | 35 | BinaryExpr |
+| 22 | 31 | 22 | 35 | If |
+| 22 | 35 | 22 | 35 | IntegerLiteral |
+| 23 | 5 | 23 | 5 | b |
+| 23 | 5 | 23 | 47 | AssignStmt |
+| 23 | 9 | 23 | 47 | .0 |
+| 23 | 9 | 23 | 47 | For |
+| 23 | 9 | 23 | 47 | Function listcomp |
+| 23 | 9 | 23 | 47 | ListComp |
+| 23 | 10 | 23 | 12 | fun |
+| 23 | 10 | 23 | 14 | ExprStmt |
+| 23 | 10 | 23 | 14 | Yield |
+| 23 | 10 | 23 | 14 | fun() |
+| 23 | 20 | 23 | 22 | fun |
+| 23 | 27 | 23 | 31 | funcs |
+| 23 | 36 | 23 | 44 | condition |
+| 23 | 36 | 23 | 46 | If |
+| 23 | 36 | 23 | 46 | condition() |
+| 24 | 5 | 24 | 5 | c |
+| 24 | 5 | 24 | 36 | AssignStmt |
+| 24 | 9 | 24 | 36 | .0 |
+| 24 | 9 | 24 | 36 | For |
+| 24 | 9 | 24 | 36 | Function setcomp |
+| 24 | 9 | 24 | 36 | SetComp |
+| 24 | 10 | 24 | 10 | ExprStmt |
+| 24 | 10 | 24 | 10 | Yield |
+| 24 | 10 | 24 | 10 | i |
+| 24 | 16 | 24 | 16 | i |
+| 24 | 21 | 24 | 24 | iter |
+| 24 | 21 | 24 | 26 | iter() |
+| 24 | 31 | 24 | 31 | i |
+| 24 | 31 | 24 | 35 | BinaryExpr |
+| 24 | 31 | 24 | 35 | If |
+| 24 | 35 | 24 | 35 | IntegerLiteral |
+| 25 | 5 | 25 | 5 | d |
+| 25 | 5 | 25 | 47 | AssignStmt |
+| 25 | 9 | 25 | 47 | .0 |
+| 25 | 9 | 25 | 47 | For |
+| 25 | 9 | 25 | 47 | Function setcomp |
+| 25 | 9 | 25 | 47 | SetComp |
+| 25 | 10 | 25 | 12 | fun |
+| 25 | 10 | 25 | 14 | ExprStmt |
+| 25 | 10 | 25 | 14 | Yield |
+| 25 | 10 | 25 | 14 | fun() |
+| 25 | 20 | 25 | 22 | fun |
+| 25 | 27 | 25 | 31 | funcs |
+| 25 | 36 | 25 | 44 | condition |
+| 25 | 36 | 25 | 46 | If |
+| 25 | 36 | 25 | 46 | condition() |

python/ql/test/3/extractor-tests/async3.6/Locations.ql

@@ -0,0 +1,5 @@
+import python
+
+from AstNode a, Location l
+where l = a.getLocation()
+select l.getStartLine(), l.getStartColumn(), l.getEndLine(), l.getEndColumn(), a.toString()

python/ql/test/3/extractor-tests/async3.6/Successors.expected

@@ -0,0 +1,32 @@
+| 5 | Function ticker | 5 | delay |
+| 5 | delay | 5 | to |
+| 5 | to | 6 | Str |
+| 6 | Str | 7 | range |
+| 7 | For | 5 | Function ticker |
+| 7 | For | 7 | i |
+| 7 | i | 8 | i |
+| 7 | range | 7 | to |
+| 7 | range() | 7 | For |
+| 7 | to | 7 | range() |
+| 8 | Yield | 9 | asyncio |
+| 8 | i | 8 | Yield |
+| 9 | Attribute | 9 | delay |
+| 9 | Attribute() | 9 | Await |
+| 9 | Await | 7 | For |
+| 9 | asyncio | 9 | Attribute |
+| 9 | delay | 9 | Attribute() |
+| 13 | Function f | 14 | aiter |
+| 14 | ListComp | 14 | a |
+| 14 | a | 15 | funcs |
+| 14 | aiter | 14 | aiter() |
+| 14 | aiter() | 14 | ListComp |
+| 15 | ListComp | 15 | b |
+| 15 | b | 16 | aiter |
+| 15 | funcs | 15 | ListComp |
+| 16 | SetComp | 16 | c |
+| 16 | aiter | 16 | aiter() |
+| 16 | aiter() | 16 | SetComp |
+| 16 | c | 17 | funcs |
+| 17 | SetComp | 17 | d |
+| 17 | d | 13 | Function f |
+| 17 | funcs | 17 | SetComp |

python/ql/test/3/extractor-tests/async3.6/Successors.ql

@@ -0,0 +1,6 @@
+import python
+
+from ControlFlowNode p, ControlFlowNode s
+where p.getASuccessor() = s and p.getScope().(Function).isAsync()
+select p.getLocation().getStartLine(), p.getNode().toString(), s.getLocation().getStartLine(),
+  s.getNode().toString()

python/ql/test/3/extractor-tests/async3.6/test.py

@@ -0,0 +1,26 @@
+
+
+#PEP 525
+
+async def ticker(delay, to):
+    """Yield numbers from 0 to *to* every *delay* seconds."""
+    for i in range(to):
+        yield i
+        await asyncio.sleep(delay)
+
+#PEP 530
+
+async def f():
+    a = [i async for i in aiter() if i % 2]
+    b = [await fun() for fun in funcs if await condition()]
+    c = {i async for i in aiter() if i % 2}
+    d = {await fun() for fun in funcs if await condition()}
+
+# Not async versions
+
+def g():
+    a = [i for i in iter() if i % 2]
+    b = [fun() for fun in funcs if condition()]
+    c = {i for i in iter() if i % 2}
+    d = {fun() for fun in funcs if condition()}
+

python/ql/test/3/extractor-tests/decorators/test.py

@@ -0,0 +1,3 @@
+@foo[0].bar
+def baz():
+    pass

python/ql/test/3/extractor-tests/decorators/test.ql

@@ -0,0 +1,3 @@
+import python
+
+select any(SyntaxError s)

python/ql/test/3/extractor-tests/fstrings3.6/FString.expected

@@ -0,0 +1,75 @@
+| 2 | Fstring | 0 | Str |
+| 2 | Fstring | 1 | world |
+| 2 | Fstring | 2 | Str |
+| 3 | Fstring | 0 | Str |
+| 3 | Fstring | 1 | one |
+| 3 | Fstring | 2 | Str |
+| 3 | Fstring | 3 | two |
+| 3 | Fstring | 4 | Str |
+| 3 | Fstring | 5 | three |
+| 3 | Fstring | 6 | Str |
+| 4 | Fstring | 0 | Str |
+| 4 | Fstring | 1 | cruel |
+| 4 | Fstring | 2 | Str |
+| 5 | Fstring | 0 | Str |
+| 5 | Fstring | 1 | thing |
+| 5 | Fstring | 2 | Str |
+| 6 | Fstring | 0 | Str |
+| 6 | Fstring | 1 | x |
+| 6 | Fstring | 2 | Str |
+| 8 | Fstring | 0 | Str |
+| 8 | Fstring | 1 | name |
+| 8 | Fstring | 2 | Str |
+| 8 | Fstring | 3 | BinaryExpr |
+| 8 | Fstring | 4 | Str |
+| 8 | Fstring | 5 | anniversary |
+| 8 | Fstring | 6 | Str |
+| 13 | Fstring | 0 | Str |
+| 13 | Fstring | 1 | world |
+| 13 | Fstring | 2 | Str |
+| 16 | Fstring | 0 | Str |
+| 16 | Fstring | 1 | Tuple |
+| 16 | Fstring | 2 | Str |
+| 16 | Fstring | 3 | Tuple |
+| 16 | Fstring | 4 | Str |
+| 19 | Fstring | 0 | Str |
+| 19 | Fstring | 1 | Tuple |
+| 19 | Fstring | 2 | Str |
+| 19 | Fstring | 3 | Tuple |
+| 19 | Fstring | 4 | Str |
+| 32 | Fstring | 0 | Str |
+| 32 | Fstring | 1 | IntegerLiteral |
+| 32 | Fstring | 2 | Str |
+| 34 | Fstring | 0 | Str |
+| 34 | Fstring | 1 | IntegerLiteral |
+| 34 | Fstring | 2 | Str |
+| 36 | Fstring | 0 | Str |
+| 36 | Fstring | 1 | IntegerLiteral |
+| 36 | Fstring | 2 | Str |
+| 38 | Fstring | 0 | Str |
+| 38 | Fstring | 1 | IntegerLiteral |
+| 38 | Fstring | 2 | Str |
+| 53 | Fstring | 0 | Str |
+| 53 | Fstring | 1 | degrees |
+| 53 | Fstring | 2 | Str |
+| 56 | Fstring | 0 | Str |
+| 56 | Fstring | 1 | IntegerLiteral |
+| 56 | Fstring | 2 | Str |
+| 59 | Fstring | 0 | Str |
+| 59 | Fstring | 1 | IntegerLiteral |
+| 59 | Fstring | 2 | Str |
+| 62 | Fstring | 0 | Str |
+| 62 | Fstring | 1 | IntegerLiteral |
+| 62 | Fstring | 2 | Str |
+| 67 | Fstring | 0 | Str |
+| 67 | Fstring | 1 | IntegerLiteral |
+| 67 | Fstring | 2 | Str |
+| 70 | Fstring | 0 | Str |
+| 70 | Fstring | 1 | IntegerLiteral |
+| 70 | Fstring | 2 | Str |
+| 75 | Fstring | 0 | Str |
+| 75 | Fstring | 1 | IntegerLiteral |
+| 75 | Fstring | 2 | Str |
+| 78 | Fstring | 0 | Str |
+| 78 | Fstring | 1 | IntegerLiteral |
+| 78 | Fstring | 2 | Str |

python/ql/test/3/extractor-tests/fstrings3.6/FString.ql

@@ -0,0 +1,7 @@
+import python
+
+from Fstring str, int n, Expr e
+where
+  e = str.getValue(n) and
+  not exists(FormattedValue v | v.getFormatSpec() = str)
+select str.getLocation().getStartLine(), str.toString(), n, e.toString()

python/ql/test/3/extractor-tests/fstrings3.6/Formatted.oldexpected

@@ -0,0 +1,11 @@
+| 2 | FormattedValue |  | world | s |
+| 3 | FormattedValue | #06x | one |   |
+| 3 | FormattedValue | ++++ | three | s |
+| 3 | FormattedValue | format | two |   |
+| 4 | FormattedValue |  | cruel |   |
+| 5 | FormattedValue |  | thing | a |
+| 6 | FormattedValue |  | x | r |
+| 8 | FormattedValue |  | BinaryExpr |   |
+| 8 | FormattedValue |  | name |   |
+| 8 | FormattedValue | %A, %B %d, %Y | anniversary |   |
+| 13 | FormattedValue |  | world | s |

python/ql/test/3/extractor-tests/fstrings3.6/Formatted.ql

@@ -0,0 +1,15 @@
+import python
+
+from FormattedValue val, string format, string typeconv
+where
+  (
+    typeconv = val.getConversion()
+    or
+    not exists(val.getConversion()) and typeconv = " "
+  ) and
+  (
+    format = val.getFormatSpec().getValue(0).(StrConst).getText()
+    or
+    not exists(val.getFormatSpec()) and format = ""
+  )
+select val.getLocation().getStartLine(), val, format, val.getValue(), typeconv

python/ql/test/3/extractor-tests/fstrings3.6/Locations.expected

@@ -0,0 +1,116 @@
+| 2 | 1 | 2 | 9 | Str |
+| 2 | 1 | 2 | 18 | Fstring |
+| 2 | 10 | 2 | 14 | world |
+| 2 | 17 | 2 | 18 | Str |
+| 3 | 1 | 3 | 4 | Str |
+| 3 | 1 | 3 | 42 | Fstring |
+| 3 | 5 | 3 | 7 | one |
+| 3 | 13 | 3 | 15 | Str |
+| 3 | 16 | 3 | 18 | two |
+| 3 | 26 | 3 | 28 | Str |
+| 3 | 29 | 3 | 33 | three |
+| 3 | 41 | 3 | 42 | Str |
+| 4 | 1 | 4 | 11 | Str |
+| 4 | 1 | 4 | 24 | Fstring |
+| 4 | 12 | 4 | 16 | cruel |
+| 4 | 17 | 4 | 24 | Str |
+| 5 | 1 | 5 | 8 | Str |
+| 5 | 1 | 5 | 17 | Fstring |
+| 5 | 9 | 5 | 13 | thing |
+| 5 | 16 | 5 | 17 | Str |
+| 6 | 1 | 6 | 3 | Str |
+| 6 | 1 | 6 | 8 | Fstring |
+| 6 | 4 | 6 | 4 | x |
+| 6 | 7 | 6 | 8 | Str |
+| 8 | 1 | 8 | 16 | Str |
+| 8 | 1 | 10 | 31 | Fstring |
+| 8 | 17 | 8 | 20 | name |
+| 8 | 21 | 9 | 1 | Str |
+| 9 | 2 | 9 | 4 | age |
+| 9 | 2 | 9 | 6 | BinaryExpr |
+| 9 | 6 | 9 | 6 | IntegerLiteral |
+| 9 | 7 | 10 | 1 | Str |
+| 10 | 2 | 10 | 12 | anniversary |
+| 10 | 27 | 10 | 31 | Str |
+| 13 | 1 | 13 | 12 | Str |
+| 13 | 1 | 13 | 21 | Fstring |
+| 13 | 13 | 13 | 17 | world |
+| 13 | 20 | 13 | 21 | Str |
+| 16 | 1 | 16 | 3 | Str |
+| 16 | 1 | 16 | 13 | Fstring |
+| 16 | 4 | 16 | 4 | IntegerLiteral |
+| 16 | 4 | 16 | 6 | Tuple |
+| 16 | 6 | 16 | 6 | IntegerLiteral |
+| 16 | 7 | 16 | 8 | Str |
+| 16 | 9 | 16 | 9 | IntegerLiteral |
+| 16 | 9 | 16 | 11 | Tuple |
+| 16 | 11 | 16 | 11 | IntegerLiteral |
+| 16 | 12 | 16 | 13 | Str |
+| 19 | 1 | 19 | 3 | Str |
+| 19 | 1 | 19 | 11 | Fstring |
+| 19 | 4 | 19 | 4 | IntegerLiteral |
+| 19 | 4 | 19 | 5 | Tuple |
+| 19 | 6 | 19 | 7 | Str |
+| 19 | 8 | 19 | 8 | IntegerLiteral |
+| 19 | 8 | 19 | 9 | Tuple |
+| 19 | 10 | 19 | 11 | Str |
+| 24 | 2 | 24 | 9 | Str |
+| 26 | 2 | 26 | 5 | Str |
+| 28 | 2 | 28 | 9 | Str |
+| 30 | 2 | 30 | 5 | Str |
+| 32 | 2 | 32 | 6 | Str |
+| 32 | 2 | 32 | 11 | Fstring |
+| 32 | 7 | 32 | 7 | IntegerLiteral |
+| 32 | 8 | 32 | 11 | Str |
+| 34 | 2 | 34 | 4 | Str |
+| 34 | 2 | 34 | 7 | Fstring |
+| 34 | 5 | 34 | 5 | IntegerLiteral |
+| 34 | 6 | 34 | 7 | Str |
+| 36 | 2 | 36 | 6 | Str |
+| 36 | 2 | 36 | 11 | Fstring |
+| 36 | 7 | 36 | 7 | IntegerLiteral |
+| 36 | 8 | 36 | 11 | Str |
+| 38 | 2 | 38 | 4 | Str |
+| 38 | 2 | 38 | 7 | Fstring |
+| 38 | 5 | 38 | 5 | IntegerLiteral |
+| 38 | 6 | 38 | 7 | Str |
+| 40 | 2 | 40 | 8 | Str |
+| 42 | 2 | 42 | 4 | Str |
+| 44 | 2 | 44 | 8 | Str |
+| 46 | 2 | 46 | 4 | Str |
+| 53 | 1 | 53 | 18 | Str |
+| 53 | 1 | 53 | 27 | Fstring |
+| 53 | 19 | 53 | 25 | degrees |
+| 53 | 26 | 53 | 27 | Str |
+| 56 | 1 | 56 | 3 | Str |
+| 56 | 1 | 56 | 21 | Fstring |
+| 56 | 4 | 56 | 4 | IntegerLiteral |
+| 56 | 5 | 56 | 21 | Str |
+| 59 | 1 | 59 | 7 | Str |
+| 59 | 1 | 59 | 11 | Fstring |
+| 59 | 8 | 59 | 8 | IntegerLiteral |
+| 59 | 9 | 59 | 11 | Str |
+| 62 | 1 | 62 | 4 | Str |
+| 62 | 1 | 62 | 10 | Fstring |
+| 62 | 5 | 62 | 5 | IntegerLiteral |
+| 62 | 6 | 62 | 10 | Str |
+| 67 | 1 | 67 | 6 | Str |
+| 67 | 1 | 67 | 43 | Fstring |
+| 67 | 7 | 67 | 7 | IntegerLiteral |
+| 67 | 38 | 67 | 43 | Str |
+| 70 | 1 | 70 | 6 | Str |
+| 70 | 1 | 70 | 39 | Fstring |
+| 70 | 7 | 70 | 7 | IntegerLiteral |
+| 70 | 34 | 70 | 39 | Str |
+| 75 | 1 | 75 | 7 | Str |
+| 75 | 1 | 75 | 11 | Fstring |
+| 75 | 8 | 75 | 8 | IntegerLiteral |
+| 75 | 9 | 75 | 11 | Str |
+| 78 | 1 | 78 | 4 | Str |
+| 78 | 1 | 78 | 10 | Fstring |
+| 78 | 5 | 78 | 5 | IntegerLiteral |
+| 78 | 6 | 78 | 10 | Str |
+| 81 | 1 | 81 | 3 | Str |
+| 82 | 1 | 82 | 3 | Str |
+| 83 | 1 | 83 | 7 | Str |
+| 84 | 1 | 84 | 7 | Str |

python/ql/test/3/extractor-tests/fstrings3.6/Locations.ql

@@ -0,0 +1,5 @@
+import python
+
+from Expr e, Location l
+where l = e.getLocation()
+select l.getStartLine(), l.getStartColumn(), l.getEndLine(), l.getEndColumn(), e.toString()

python/ql/test/3/extractor-tests/fstrings3.6/Successors.expected

@@ -0,0 +1,117 @@
+| 0 | 0 | Module test |
+| 1 | 2 | "hello " |
+| 2 | 2 | world |
+| 3 | 2 | "" |
+| 4 | 2 | Fstring |
+| 5 | 3 | "1" |
+| 6 | 3 | one |
+| 7 | 3 | "2" |
+| 8 | 3 | two |
+| 9 | 3 | "3" |
+| 10 | 3 | three |
+| 11 | 3 | "" |
+| 12 | 3 | Fstring |
+| 13 | 4 | "goodbye " |
+| 14 | 4 | cruel |
+| 15 | 4 | " world" |
+| 16 | 4 | Fstring |
+| 17 | 5 | "ascii" |
+| 18 | 5 | thing |
+| 19 | 5 | "" |
+| 20 | 5 | Fstring |
+| 21 | 6 | "" |
+| 22 | 6 | x |
+| 23 | 6 | "" |
+| 24 | 6 | Fstring |
+| 25 | 8 | "My name is " |
+| 26 | 8 | name |
+| 27 | 8 | ", my age next year is \n" |
+| 28 | 9 | age |
+| 29 | 9 | IntegerLiteral |
+| 30 | 9 | BinaryExpr |
+| 31 | 9 | ", my anniversary is \n" |
+| 32 | 10 | anniversary |
+| 33 | 10 | "." |
+| 34 | 8 | Fstring |
+| 35 | 13 | "hello " |
+| 36 | 13 | world |
+| 37 | 13 | "" |
+| 38 | 13 | Fstring |
+| 39 | 16 | "" |
+| 40 | 16 | IntegerLiteral |
+| 41 | 16 | IntegerLiteral |
+| 42 | 16 | Tuple |
+| 43 | 16 | "" |
+| 44 | 16 | IntegerLiteral |
+| 45 | 16 | IntegerLiteral |
+| 46 | 16 | Tuple |
+| 47 | 16 | "" |
+| 48 | 16 | Fstring |
+| 49 | 19 | "" |
+| 50 | 19 | IntegerLiteral |
+| 51 | 19 | Tuple |
+| 52 | 19 | "" |
+| 53 | 19 | IntegerLiteral |
+| 54 | 19 | Tuple |
+| 55 | 19 | "" |
+| 56 | 19 | Fstring |
+| 57 | 24 | "0" |
+| 58 | 26 | "0" |
+| 59 | 28 | "0" |
+| 60 | 30 | "0" |
+| 61 | 32 | "" |
+| 62 | 32 | IntegerLiteral |
+| 63 | 32 | "" |
+| 64 | 32 | Fstring |
+| 65 | 34 | "" |
+| 66 | 34 | IntegerLiteral |
+| 67 | 34 | "" |
+| 68 | 34 | Fstring |
+| 69 | 36 | "" |
+| 70 | 36 | IntegerLiteral |
+| 71 | 36 | "" |
+| 72 | 36 | Fstring |
+| 73 | 38 | "" |
+| 74 | 38 | IntegerLiteral |
+| 75 | 38 | "" |
+| 76 | 38 | Fstring |
+| 77 | 40 | "0" |
+| 78 | 42 | "0" |
+| 79 | 44 | "0" |
+| 80 | 46 | "0" |
+| 81 | 53 | "\u00b0" |
+| 82 | 53 | degrees |
+| 83 | 53 | "" |
+| 84 | 53 | Fstring |
+| 85 | 56 | "" |
+| 86 | 56 | IntegerLiteral |
+| 87 | 56 | "\u00b0" |
+| 88 | 56 | Fstring |
+| 89 | 59 | "{{ a" |
+| 90 | 59 | IntegerLiteral |
+| 91 | 59 | "c" |
+| 92 | 59 | Fstring |
+| 93 | 62 | "a" |
+| 94 | 62 | IntegerLiteral |
+| 95 | 62 | "{{c" |
+| 96 | 62 | Fstring |
+| 97 | 67 | "pre" |
+| 98 | 67 | IntegerLiteral |
+| 99 | 67 | "post" |
+| 100 | 67 | Fstring |
+| 101 | 70 | "pre" |
+| 102 | 70 | IntegerLiteral |
+| 103 | 70 | "post" |
+| 104 | 70 | Fstring |
+| 105 | 75 | "}} a" |
+| 106 | 75 | IntegerLiteral |
+| 107 | 75 | "c" |
+| 108 | 75 | Fstring |
+| 109 | 78 | "a" |
+| 110 | 78 | IntegerLiteral |
+| 111 | 78 | "}}c" |
+| 112 | 78 | Fstring |
+| 113 | 81 | "" |
+| 114 | 82 | "" |
+| 115 | 83 | "" |
+| 116 | 84 | "" |

python/ql/test/3/extractor-tests/fstrings3.6/Successors.ql

@@ -0,0 +1,11 @@
+import python
+
+string repr(AstNode a) {
+  not a instanceof StrConst and result = a.toString()
+  or
+  result = "\"" + a.(StrConst).getText() + "\""
+}
+
+from ControlFlowNode p, ControlFlowNode s, BasicBlock b, int n
+where p.getASuccessor() = s and p = b.getNode(n)
+select n, p.getLocation().getStartLine(), repr(p.getNode())

python/ql/test/3/extractor-tests/fstrings3.6/test.py

@@ -0,0 +1,84 @@
+
+f'hello {world!s}'
+f'1{one:#06x}2{two:format}3{three!s:++++}'
+f'goodbye {cruel} world'
+f'ascii{thing!a}'
+f'{x!r}'
+
+f'''My name is {name}, my age next year is 
+{age+1}, my anniversary is 
+{anniversary:%A, %B %d, %Y}.'''
+
+#Implicit concatenation
+"hello" f' {world!s}'
+
+# Simplified version of FP reported in issue #1990
+f"{1,1}{1,1}"
+
+# Trailing comma for 1-element tuple:
+f"{1,}{1,}"
+
+# Parenthesized with newline after string.
+# Simplified version of FP reported in issue #2453
+
+(f"""0"""
+)
+(f"0"
+)
+(f'''0'''
+)
+(f'0'
+)
+(f"""{0}"""
+)
+(f"{0}"
+)
+(f'''{0}'''
+)
+(f'{0}'
+)
+("""0"""
+)
+("0"
+)
+('''0'''
+)
+('0'
+)
+
+
+
+
+# Unicode literals inside fstrings in initial position
+f'\N{DEGREE SIGN}{degrees}'
+
+# Unicode literals inside fstrings in non-initial position
+f'{1}\N{DEGREE SIGN}'
+
+# {{ in initial position
+f'{{ a{1}c'
+
+# {{ in non-initial position
+f'a{1}{{c'
+
+
+
+# Unicode literals inside fspecs in initial position
+f'pre{1:\N{LATIN SMALL LETTER B}>0.0f}post'
+
+# Unicode literals inside fspecs in initial position
+f'pre{1:0\N{LATIN SMALL LETTER B}}post'
+
+
+
+# }} in initial position
+f'}} a{1}c'
+
+# }} in non-initial position
+f'a{1}}}c'
+
+# Empty f-strings
+f''
+f""
+f''''''
+f""""""

python/ql/test/3/extractor-tests/fstrings3.8/FString.expected

@@ -0,0 +1,17 @@
+| 4 | Fstring | 0 | Str |
+| 4 | Fstring | 1 | foo |
+| 4 | Fstring | 2 | Str |
+| 5 | Fstring | 0 | Str |
+| 5 | Fstring | 1 | foo |
+| 5 | Fstring | 2 | Str |
+| 6 | Fstring | 0 | Str |
+| 6 | Fstring | 1 | Attribute() |
+| 6 | Fstring | 2 | Str |
+| 7 | Fstring | 0 | Str |
+| 7 | Fstring | 1 | foo |
+| 7 | Fstring | 2 | Str |
+| 8 | Fstring | 0 | Str |
+| 8 | Fstring | 1 | foo |
+| 8 | Fstring | 2 | Str |
+| 8 | Fstring | 3 | Attribute() |
+| 8 | Fstring | 4 | Str |

python/ql/test/3/extractor-tests/fstrings3.8/FString.ql

@@ -0,0 +1,7 @@
+import python
+
+from Fstring str, int n, Expr e
+where
+  e = str.getValue(n) and
+  not exists(FormattedValue v | v.getFormatSpec() = str)
+select str.getLocation().getStartLine(), str.toString(), n, e.toString()

python/ql/test/3/extractor-tests/fstrings3.8/Locations.expected

@@ -0,0 +1,40 @@
+| 1 | 1 | 1 | 3 | foo |
+| 1 | 7 | 1 | 7 | IntegerLiteral |
+| 3 | 1 | 3 | 3 | bar |
+| 3 | 7 | 3 | 11 | Str |
+| 4 | 1 | 4 | 5 | print |
+| 4 | 1 | 4 | 16 | print() |
+| 4 | 7 | 4 | 9 | Str |
+| 4 | 7 | 4 | 15 | Fstring |
+| 4 | 10 | 4 | 12 | foo |
+| 4 | 14 | 4 | 15 | Str |
+| 5 | 1 | 5 | 5 | print |
+| 5 | 1 | 5 | 18 | print() |
+| 5 | 7 | 5 | 9 | Str |
+| 5 | 7 | 5 | 17 | Fstring |
+| 5 | 10 | 5 | 12 | foo |
+| 5 | 16 | 5 | 17 | Str |
+| 6 | 1 | 6 | 5 | print |
+| 6 | 1 | 6 | 24 | print() |
+| 6 | 7 | 6 | 9 | Str |
+| 6 | 7 | 6 | 23 | Fstring |
+| 6 | 10 | 6 | 12 | bar |
+| 6 | 10 | 6 | 18 | Attribute |
+| 6 | 10 | 6 | 20 | Attribute() |
+| 6 | 22 | 6 | 23 | Str |
+| 7 | 1 | 7 | 5 | print |
+| 7 | 1 | 7 | 23 | print() |
+| 7 | 7 | 7 | 10 | Str |
+| 7 | 7 | 7 | 22 | Fstring |
+| 7 | 11 | 7 | 13 | foo |
+| 7 | 20 | 7 | 22 | Str |
+| 8 | 1 | 8 | 5 | print |
+| 8 | 1 | 8 | 37 | print() |
+| 8 | 7 | 8 | 10 | Str |
+| 8 | 7 | 8 | 36 | Fstring |
+| 8 | 11 | 8 | 13 | foo |
+| 8 | 20 | 8 | 21 | Str |
+| 8 | 22 | 8 | 24 | bar |
+| 8 | 22 | 8 | 30 | Attribute |
+| 8 | 22 | 8 | 32 | Attribute() |
+| 8 | 34 | 8 | 36 | Str |

python/ql/test/3/extractor-tests/fstrings3.8/Locations.ql

@@ -0,0 +1,5 @@
+import python
+
+from Expr e, Location l
+where l = e.getLocation()
+select l.getStartLine(), l.getStartColumn(), l.getEndLine(), l.getEndColumn(), e.toString()

python/ql/test/3/extractor-tests/fstrings3.8/Successors.expected

@@ -0,0 +1,41 @@
+| 0 | 0 | Module test |
+| 1 | 1 | IntegerLiteral |
+| 2 | 1 | foo |
+| 3 | 3 | "bar" |
+| 4 | 3 | bar |
+| 5 | 4 | print |
+| 6 | 4 | "" |
+| 7 | 4 | foo |
+| 8 | 4 | "" |
+| 9 | 4 | Fstring |
+| 10 | 4 | print() |
+| 11 | 5 | print |
+| 12 | 5 | "" |
+| 13 | 5 | foo |
+| 14 | 5 | "" |
+| 15 | 5 | Fstring |
+| 16 | 5 | print() |
+| 17 | 6 | print |
+| 18 | 6 | "" |
+| 19 | 6 | bar |
+| 20 | 6 | Attribute |
+| 21 | 6 | Attribute() |
+| 22 | 6 | "" |
+| 23 | 6 | Fstring |
+| 24 | 6 | print() |
+| 25 | 7 | print |
+| 26 | 7 | "*" |
+| 27 | 7 | foo |
+| 28 | 7 | "*" |
+| 29 | 7 | Fstring |
+| 30 | 7 | print() |
+| 31 | 8 | print |
+| 32 | 8 | "*" |
+| 33 | 8 | foo |
+| 34 | 8 | "" |
+| 35 | 8 | bar |
+| 36 | 8 | Attribute |
+| 37 | 8 | Attribute() |
+| 38 | 8 | "*" |
+| 39 | 8 | Fstring |
+| 40 | 8 | print() |

python/ql/test/3/extractor-tests/fstrings3.8/Successors.ql

@@ -0,0 +1,11 @@
+import python
+
+string repr(AstNode a) {
+  not a instanceof StrConst and result = a.toString()
+  or
+  result = "\"" + a.(StrConst).getText() + "\""
+}
+
+from ControlFlowNode p, ControlFlowNode s, BasicBlock b, int n
+where p.getASuccessor() = s and p = b.getNode(n)
+select n, p.getLocation().getStartLine(), repr(p.getNode())

python/ql/test/3/extractor-tests/fstrings3.8/test.py

@@ -0,0 +1,8 @@
+foo = 5
+
+bar = "bar"
+print(f'{foo=}')
+print(f'{foo = }')
+print(f'{bar.upper()=}')
+print(f'*{foo=:+<20}*')
+print(f'*{foo=:+<20}{bar.upper()=}*')

python/ql/test/3/extractor-tests/import_depth/options

@@ -0,0 +1 @@
+semmle-extractor-options: --lang=3 --max-import-depth=1

python/ql/test/3/extractor-tests/import_depth/test.expected

@@ -0,0 +1,5 @@
+| Module package |
+| Module package.__init__ |
+| Module sys |
+| Module sys.monitoring |
+| Module test |

python/ql/test/3/extractor-tests/import_depth/test.py

@@ -0,0 +1 @@
+import package

python/ql/test/3/extractor-tests/import_depth/test.ql

@@ -0,0 +1,6 @@
+import python
+
+from ModuleObject m
+/* Exclude the builtins module as it has a different name under 2 and 3. */
+where not m = theBuiltinModuleObject()
+select m.toString()

python/ql/test/3/extractor-tests/matmult/Test.expected

@@ -0,0 +1 @@
+| 3 | BinaryExpr |

python/ql/test/3/extractor-tests/matmult/Test.ql

@@ -0,0 +1,5 @@
+import python
+
+from BinaryExpr b
+where b.getOp() instanceof MatMult
+select b.getLocation().getStartLine(), b.toString()

python/ql/test/3/extractor-tests/matmult/matmul.py

@@ -0,0 +1,3 @@
+def matmul(a,b):
+    a * b
+    return a @ b

python/ql/test/3/extractor-tests/multibyte/Test.expected

@@ -0,0 +1,3 @@
+| test.py:4:1:4:11 | Str | \u0111\u0142e\u00b6\u014b\u00b6\u0142\u014b |
+| test.py:4:15:4:19 | Str | hi |
+| test.py:4:23:4:27 | Str | \n |

python/ql/test/3/extractor-tests/multibyte/Test.ql

@@ -0,0 +1,4 @@
+import python
+
+from StrConst s
+select s, s.getText()