Files
codeql/python/extractor/tests/tokenizer/basic.tokens

473 lines
19 KiB
Plaintext

2,0-2,52: COMMENT '#AST nodes: Classes, Functions, Modules, expr, stmts'
4,0-4,5: NAME 'class'
4,6-4,7: NAME 'C'
4,7-4,8: COLON ':'
4,8-4,9: NEWLINE '\n'
6,0-6,4: INDENT ' '
6,4-6,7: NAME 'def'
6,8-6,13: NAME 'stmts'
6,13-6,14: LPAR '('
6,14-6,16: NAME 'p0'
6,16-6,17: COMMA ','
6,18-6,20: NAME 'p1'
6,20-6,21: RPAR ')'
6,21-6,22: COLON ':'
6,22-6,23: NEWLINE '\n'
7,0-7,8: INDENT ' '
7,8-7,14: NAME 'global'
7,15-7,16: NAME 'x'
7,16-7,17: NEWLINE '\n'
8,8-8,14: NAME 'assert'
8,15-8,16: NAME 'x'
8,17-8,19: OP '=='
8,20-8,21: NUMBER '2'
8,21-8,22: NEWLINE '\n'
9,8-9,9: NAME 'y'
9,10-9,11: OP '='
9,12-9,13: NUMBER '3'
9,13-9,14: NEWLINE '\n'
10,8-10,9: NAME 'y'
10,10-10,12: OP '+='
10,13-10,14: NUMBER '4'
10,14-10,15: NEWLINE '\n'
11,8-11,13: NAME 'while'
11,14-11,18: NAME 'True'
11,18-11,19: COLON ':'
11,19-11,20: NEWLINE '\n'
12,0-12,12: INDENT ' '
12,12-12,17: NAME 'break'
12,17-12,18: NEWLINE '\n'
13,8-13,8: DEDENT ''
13,8-13,13: NAME 'while'
13,14-13,15: NAME 'x'
13,16-13,17: OP '>'
13,18-13,19: NUMBER '0'
13,19-13,20: COLON ':'
13,20-13,21: NEWLINE '\n'
14,0-14,12: INDENT ' '
14,12-14,13: NAME 'x'
14,14-14,16: OP '-='
14,17-14,18: NUMBER '1'
14,18-14,19: NEWLINE '\n'
15,12-15,20: NAME 'continue'
15,20-15,21: NEWLINE '\n'
17,8-17,8: DEDENT ''
17,8-17,9: NAME 'f'
17,9-17,10: LPAR '('
17,10-17,11: RPAR ')'
17,11-17,12: NEWLINE '\n'
18,8-18,11: NAME 'for'
18,12-18,13: NAME 'x'
18,14-18,16: NAME 'in'
18,17-18,18: NAME 'y'
18,18-18,19: COLON ':'
18,19-18,20: NEWLINE '\n'
19,0-19,12: INDENT ' '
19,12-19,16: NAME 'pass'
19,16-19,17: NEWLINE '\n'
20,8-20,8: DEDENT ''
20,8-20,10: NAME 'if'
20,11-20,12: NAME 'x'
20,12-20,13: COLON ':'
20,13-20,14: NEWLINE '\n'
21,0-21,12: INDENT ' '
21,12-21,17: NAME 'print'
21,17-21,18: LPAR '('
21,18-21,19: NAME 'y'
21,19-21,20: RPAR ')'
21,20-21,21: NEWLINE '\n'
22,8-22,8: DEDENT ''
22,8-22,14: NAME 'import'
22,15-22,16: NAME 'a'
22,16-22,17: NEWLINE '\n'
23,8-23,14: NAME 'import'
23,15-23,16: NAME 'a'
23,16-23,17: DOT '.'
23,17-23,18: NAME 'b'
23,19-23,21: NAME 'as'
23,22-23,23: NAME 'c'
23,23-23,24: NEWLINE '\n'
24,8-24,14: NAME 'import'
24,15-24,16: NAME 'a'
24,17-24,19: NAME 'as'
24,20-24,21: NAME 'b'
24,21-24,22: NEWLINE '\n'
25,8-25,12: NAME 'from'
25,13-25,14: NAME 'a'
25,14-25,15: DOT '.'
25,15-25,16: NAME 'b'
25,17-25,23: NAME 'import'
25,24-25,25: NAME 'c'
25,25-25,26: NEWLINE '\n'
28,8-28,12: NAME 'with'
28,13-28,17: NAME 'open'
28,17-28,18: LPAR '('
28,18-28,24: STRING '"file"'
28,24-28,25: RPAR ')'
28,26-28,28: NAME 'as'
28,29-28,30: NAME 'f'
28,30-28,31: COLON ':'
28,31-28,32: NEWLINE '\n'
29,0-29,12: INDENT ' '
29,12-29,16: NAME 'pass'
29,16-29,17: NEWLINE '\n'
30,8-30,8: DEDENT ''
30,8-30,11: NAME 'try'
30,11-30,12: COLON ':'
30,12-30,13: NEWLINE '\n'
31,0-31,12: INDENT ' '
31,12-31,13: NUMBER '1'
31,13-31,14: OP '/'
31,14-31,15: NUMBER '0'
31,15-31,16: NEWLINE '\n'
32,8-32,8: DEDENT ''
32,8-32,14: NAME 'except'
32,15-32,24: NAME 'Exception'
32,25-32,27: NAME 'as'
32,28-32,30: NAME 'ex'
32,30-32,31: COLON ':'
32,31-32,32: NEWLINE '\n'
33,0-33,12: INDENT ' '
33,12-33,15: NAME 'del'
33,16-33,17: NAME 'y'
33,17-33,18: NEWLINE '\n'
34,8-34,8: DEDENT ''
34,8-34,15: NAME 'finally'
34,15-34,16: COLON ':'
34,16-34,17: NEWLINE '\n'
35,0-35,12: INDENT ' '
35,12-35,15: NAME 'del'
35,16-35,17: NAME 'x'
35,17-35,18: NEWLINE '\n'
36,8-36,8: DEDENT ''
36,8-36,10: NAME 'if'
36,11-36,12: NAME 'x'
36,12-36,13: COLON ':'
36,13-36,14: NEWLINE '\n'
37,0-37,12: INDENT ' '
37,12-37,17: NAME 'raise'
37,18-37,27: NAME 'Exception'
37,27-37,28: LPAR '('
37,28-37,29: RPAR ')'
37,29-37,30: NEWLINE '\n'
38,8-38,8: DEDENT ''
38,8-38,12: NAME 'else'
38,12-38,13: COLON ':'
38,13-38,14: NEWLINE '\n'
39,0-39,12: INDENT ' '
39,12-39,18: NAME 'return'
39,18-39,19: NEWLINE '\n'
41,4-41,4: DEDENT ''
41,4-41,4: DEDENT ''
41,4-41,7: NAME 'def'
41,8-41,13: NAME 'exprs'
41,13-41,14: LPAR '('
41,14-41,16: NAME 'p2'
41,16-41,17: COMMA ','
41,18-41,20: NAME 'p3'
41,20-41,21: RPAR ')'
41,21-41,22: COLON ':'
41,22-41,23: NEWLINE '\n'
42,0-42,8: INDENT ' '
42,8-42,10: NAME 'p2'
42,10-42,11: DOT '.'
42,11-42,12: NAME 'x'
42,13-42,14: OP '='
42,15-42,16: NUMBER '2'
42,16-42,17: NEWLINE '\n'
43,8-43,9: NAME 'a'
43,10-43,11: OP '='
43,12-43,14: NAME 'p3'
43,14-43,15: DOT '.'
43,15-43,16: NAME 'y'
43,16-43,17: NEWLINE '\n'
44,8-44,9: NAME 'x'
44,10-44,11: OP '='
44,12-44,13: NUMBER '1'
44,14-44,15: OP '+'
44,16-44,17: NUMBER '2'
44,17-44,18: NEWLINE '\n'
45,8-45,9: NAME 'y'
45,10-45,11: OP '='
45,12-45,23: STRING 'b\'h4tpvhsa\''
45,23-45,24: NEWLINE '\n'
46,8-46,12: NAME 'call'
46,12-46,13: LPAR '('
46,13-46,17: NAME 'arg0'
46,17-46,18: COMMA ','
46,19-46,23: NAME 'arg1'
46,23-46,24: COMMA ','
46,25-46,30: NAME 'name0'
46,30-46,31: OP '='
46,31-46,35: STRING '"Hi"'
46,35-46,36: COMMA ','
46,37-46,42: NAME 'name1'
46,42-46,43: OP '='
46,43-46,44: NAME 'y'
46,44-46,45: COMMA ','
46,46-46,47: OP '*'
46,47-46,48: LPAR '('
46,48-46,49: RPAR ')'
46,49-46,50: COMMA ','
46,51-46,53: OP '**'
46,53-46,54: LBRACE '{'
46,54-46,55: RBRACE '}'
46,55-46,56: RPAR ')'
46,56-46,57: NEWLINE '\n'
47,8-47,9: NAME 'x'
47,10-47,11: OP '<'
47,12-47,13: NAME 'y'
47,13-47,14: NEWLINE '\n'
48,8-48,9: LBRACE '{'
48,9-48,10: NUMBER '1'
48,10-48,11: COLON ':'
48,11-48,12: NUMBER '1'
48,12-48,13: COMMA ','
48,14-48,15: NUMBER '2'
48,15-48,16: COLON ':'
48,17-48,18: NUMBER '2'
48,18-48,19: RBRACE '}'
48,19-48,20: NEWLINE '\n'
50,8-50,9: NAME 'x'
50,9-50,10: LSQB '['
50,10-50,11: NAME 'a'
50,11-50,12: COMMA ','
50,13-50,14: NUMBER '7'
50,14-50,15: RSQB ']'
50,15-50,16: NEWLINE '\n'
51,8-51,9: LPAR '('
51,9-51,10: NAME 'x'
51,11-51,14: NAME 'for'
51,15-51,16: NAME 'x'
51,17-51,19: NAME 'in'
51,20-51,21: NAME 'y'
51,21-51,22: RPAR ')'
51,22-51,23: NEWLINE '\n'
52,8-52,10: NUMBER '17'
52,11-52,13: NAME 'if'
52,14-52,15: NAME 'x'
52,16-52,17: OP '<'
52,18-52,19: NAME 'y'
52,20-52,24: NAME 'else'
52,25-52,27: NUMBER '16'
52,27-52,28: NEWLINE '\n'
53,8-53,14: NAME 'lambda'
53,15-53,16: NAME 'x'
53,17-53,18: COLON ':'
53,19-53,20: NAME 'x'
53,21-53,22: OP '*'
53,23-53,24: NAME 'y'
53,24-53,25: NEWLINE '\n'
54,8-54,9: LSQB '['
54,10-54,11: NUMBER '1'
54,11-54,12: COMMA ','
54,13-54,14: NUMBER '2'
54,14-54,15: COMMA ','
54,16-54,17: NAME 'a'
54,17-54,18: COMMA ','
54,19-54,20: NAME 'x'
54,20-54,21: DOT '.'
54,21-54,22: NAME 'b'
54,22-54,23: COMMA ','
54,24-54,26: NAME 'p1'
54,26-54,27: DOT '.'
54,27-54,28: NAME 'c'
54,29-54,30: RSQB ']'
54,30-54,31: NEWLINE '\n'
55,8-55,9: LSQB '['
55,10-55,11: NAME 'a'
55,12-55,13: OP '+'
55,14-55,18: STRING '"Hi"'
55,19-55,22: NAME 'for'
55,23-55,24: NAME 'a'
55,25-55,27: NAME 'in'
55,28-55,31: NAME 'str'
55,31-55,32: LPAR '('
55,32-55,33: NAME 'y'
55,33-55,34: RPAR ')'
55,35-55,36: RSQB ']'
55,36-55,37: NEWLINE '\n'
59,8-59,18: COMMENT '#a, *b = y'
60,8-60,13: STRING 'u"Hi"'
60,13-60,14: NEWLINE '\n'
61,8-61,9: NAME 'x'
61,9-61,10: LSQB '['
61,10-61,11: NUMBER '0'
61,11-61,12: RSQB ']'
61,12-61,13: NEWLINE '\n'
62,8-62,9: NAME 'x'
62,9-62,10: LSQB '['
62,10-62,11: NAME 'y'
62,11-62,12: LSQB '['
62,12-62,13: NUMBER '0'
62,13-62,14: RSQB ']'
62,14-62,15: RSQB ']'
62,15-62,16: NEWLINE '\n'
63,8-63,9: LPAR '('
63,9-63,11: NAME 'p2'
63,11-63,12: COMMA ','
63,13-63,15: NAME 'p3'
63,15-63,16: COMMA ','
63,17-63,18: NUMBER '7'
63,18-63,19: RPAR ')'
63,19-63,20: NEWLINE '\n'
65,0-65,23: COMMENT '#Some multiline strings'
66,0-66,0: DEDENT ''
66,0-66,0: DEDENT ''
66,0-67,23: STRING '\'\'\'\nSingle quotes string\'\'\''
67,23-67,24: NEWLINE '\n'
69,0-71,9: STRING '"""\nDouble-quotes\nstring"""'
71,9-71,10: NEWLINE '\n'
73,0-75,3: STRING 'r\'\'\'\nBytes\n\'\'\''
75,3-75,4: NEWLINE '\n'
77,0-80,3: STRING 'U"""\nRaw\nUnicode\n"""'
80,3-80,4: NEWLINE '\n'
82,0-82,19: COMMENT '#Decorated function'
83,0-83,1: AT '@'
83,1-83,5: NAME 'deco'
83,5-83,6: NEWLINE '\n'
84,0-84,3: NAME 'def'
84,4-84,5: NAME 'f'
84,5-84,6: LPAR '('
84,6-84,7: RPAR ')'
84,7-84,8: COLON ':'
84,8-84,9: NEWLINE '\n'
85,0-85,4: INDENT ' '
85,4-85,8: NAME 'pass'
85,8-85,9: NEWLINE '\n'
87,0-87,32: COMMENT '#Inner function (see ODASA-1774)'
88,0-88,0: DEDENT ''
88,0-88,3: NAME 'def'
88,4-88,9: NAME 'outer'
88,9-88,10: LPAR '('
88,10-88,11: RPAR ')'
88,11-88,12: COLON ':'
88,12-88,13: NEWLINE '\n'
89,0-89,4: INDENT ' '
89,4-89,7: NAME 'def'
89,8-89,13: NAME 'inner'
89,13-89,14: LPAR '('
89,14-89,15: RPAR ')'
89,15-89,16: COLON ':'
89,16-89,17: NEWLINE '\n'
90,0-90,8: INDENT ' '
90,8-90,12: NAME 'pass'
90,12-90,13: NEWLINE '\n'
92,0-92,29: COMMENT '#Oddly laid out comprehension'
93,0-93,0: DEDENT ''
93,0-93,0: DEDENT ''
93,0-93,1: LSQB '['
93,1-93,2: LSQB '['
94,2-94,3: NAME 'x'
94,4-94,7: NAME 'for'
94,8-94,9: NAME 'x'
94,10-94,12: NAME 'in'
94,13-94,14: NAME 'y'
95,2-95,3: RSQB ']'
97,2-97,5: NAME 'for'
97,6-97,7: NAME 'a'
97,8-97,10: NAME 'in'
97,11-97,12: NAME 'b'
98,0-98,1: RSQB ']'
98,1-98,2: NEWLINE '\n'
100,0-100,25: COMMENT '#Nested binary operations'
101,0-101,7: STRING '"Hello"'
101,8-101,9: OP '+'
101,10-101,13: STRING '" "'
101,14-101,15: OP '+'
101,16-101,23: STRING '"world"'
101,23-101,24: NEWLINE '\n'
102,0-102,1: NUMBER '1'
102,1-102,2: OP '+'
102,2-102,3: NUMBER '2'
102,3-102,4: OP '+'
102,4-102,5: NAME 'f'
102,5-102,6: LPAR '('
102,6-102,7: RPAR ')'
102,7-102,8: NEWLINE '\n'
103,0-103,1: NUMBER '1'
103,1-103,2: OP '+'
103,2-103,3: LPAR '('
103,3-103,4: NUMBER '2'
103,4-103,5: OP '+'
103,5-103,6: NUMBER '3'
103,6-103,7: RPAR ')'
103,7-103,8: NEWLINE '\n'
105,0-105,12: COMMENT '# operations'
106,0-106,1: NAME 'a'
106,1-106,2: OP '|'
106,2-106,3: NAME 'b'
106,3-106,4: OP '&'
106,4-106,5: NAME 'c'
106,5-106,6: OP '+'
106,6-106,7: NAME 'd'
106,7-106,8: OP '-'
106,8-106,9: NAME 'e'
106,9-106,10: NEWLINE '\n'
107,0-107,1: NAME 'x'
107,1-107,2: OP '*'
107,2-107,3: NAME 'f'
107,3-107,4: OP '%'
107,4-107,5: NAME 'g'
107,5-107,6: OP '^'
107,6-107,7: NAME 'h'
107,7-107,8: AT '@'
107,8-107,9: NAME 'j'
107,9-107,11: OP '**'
107,11-107,12: NAME 'k'
107,12-107,13: NEWLINE '\n'
109,0-109,18: COMMENT '#Augmented assigns'
110,0-110,1: NAME 'a'
110,2-110,4: OP '@='
110,5-110,6: NAME 'b'
110,6-110,7: NEWLINE '\n'
111,0-111,1: NAME 'a'
111,2-111,4: OP '|='
111,5-111,6: NAME 'b'
111,6-111,7: NEWLINE '\n'
112,0-112,1: NAME 'a'
112,2-112,4: OP '*='
112,5-112,6: NAME 'b'
112,6-112,7: NEWLINE '\n'
114,0-114,1: OP '~'
114,1-114,2: NAME 'a'
114,2-114,3: NEWLINE '\n'
116,0-116,12: COMMENT '#Comparisons'
117,0-117,1: OP '<'
117,1-117,2: NEWLINE '\n'
118,0-118,1: OP '>'
118,1-118,2: NEWLINE '\n'
119,0-119,2: OP '<='
119,2-119,3: NEWLINE '\n'
120,0-120,2: OP '>='
120,2-120,3: NEWLINE '\n'
121,0-121,2: OP '!='
121,2-121,3: NEWLINE '\n'
122,0-122,2: OP '=='
122,2-122,3: NEWLINE '\n'
123,0-123,2: NAME 'is'
123,2-123,3: NEWLINE '\n'
124,0-124,2: NAME 'is'
124,3-124,6: NAME 'not'
124,6-124,7: NEWLINE '\n'
126,0-126,1: LPAR '('
126,1-127,3: STRING '"""\n"""'
127,3-127,4: RPAR ')'
127,4-127,5: NEWLINE '\n'
128,0-128,3: NAME 'del'
128,4-128,5: NAME 'x'
128,5-128,6: NEWLINE '\n'
130,0-130,1: BACKQUOTE '`'
130,1-130,10: NAME 'backticks'
130,10-130,11: BACKQUOTE '`'
130,11-130,12: NEWLINE '\n'
132,0-132,1: NAME 'x'
132,3-132,4: COLONEQUAL ':='
132,5-132,6: NAME 'y'
132,6-132,7: NEWLINE '\n'
134,0-134,1: NUMBER '1'
134,2-134,4: OP '<>'
134,5-134,6: NUMBER '2'
134,6-134,7: NEWLINE '\n'
135,0-135,0: ENDMARKER ''