Files
codeql/python/extractor/tests/tokenizer/gen_tokens.tokens

276 lines
11 KiB
Plaintext

1,0-1,6: NAME 'import'
1,7-1,10: NAME 'sys'
1,10-1,11: NEWLINE '\n'
2,0-2,6: NAME 'import'
2,7-2,15: NAME 'tokenize'
2,15-2,16: NEWLINE '\n'
3,0-3,6: NAME 'import'
3,7-3,12: NAME 'token'
3,12-3,13: NEWLINE '\n'
5,0-5,3: NAME 'def'
5,4-5,14: NAME 'printtoken'
5,14-5,15: LPAR '('
5,15-5,19: NAME 'type'
5,19-5,20: COMMA ','
5,21-5,26: NAME 'token'
5,26-5,27: COMMA ','
5,28-5,33: NAME 'start'
5,33-5,34: COMMA ','
5,35-5,38: NAME 'end'
5,38-5,39: COMMA ','
5,40-5,41: NAME '_'
5,41-5,42: RPAR ')'
5,42-5,43: COLON ':'
5,44-5,45: NEWLINE '\n'
6,4-6,63: COMMENT '# Use Python 3 tokenize style output, regardless of version'
7,0-7,4: INDENT ' '
7,4-7,6: NAME 'if'
7,7-7,15: NAME 'tokenize'
7,15-7,16: DOT '.'
7,16-7,24: NAME 'tok_name'
7,24-7,25: LSQB '['
7,25-7,29: NAME 'type'
7,29-7,30: RSQB ']'
7,31-7,34: NAME 'not'
7,35-7,37: NAME 'in'
7,38-7,39: LPAR '('
7,39-7,49: STRING '"ENCODING"'
7,49-7,50: COMMA ','
7,51-7,55: STRING '"NL"'
7,55-7,56: RPAR ')'
7,56-7,57: COLON ':'
7,57-7,58: NEWLINE '\n'
8,0-8,8: INDENT ' '
8,8-8,19: NAME 'token_range'
8,20-8,21: OP '='
8,22-8,36: STRING '"%d,%d-%d,%d:"'
8,37-8,38: OP '%'
8,39-8,40: LPAR '('
8,40-8,45: NAME 'start'
8,46-8,47: OP '+'
8,48-8,51: NAME 'end'
8,51-8,52: RPAR ')'
8,52-8,53: NEWLINE '\n'
9,8-9,13: NAME 'print'
9,13-9,14: LPAR '('
9,14-9,28: STRING '"%-20s%-15s%r"'
9,29-9,30: OP '%'
10,12-10,13: LPAR '('
10,13-10,24: NAME 'token_range'
10,24-10,25: COMMA ','
10,26-10,34: NAME 'tokenize'
10,34-10,35: DOT '.'
10,35-10,43: NAME 'tok_name'
10,43-10,44: LSQB '['
10,44-10,48: NAME 'type'
10,48-10,49: RSQB ']'
10,49-10,50: COMMA ','
10,51-10,56: NAME 'token'
10,56-10,57: RPAR ')'
11,8-11,9: RPAR ')'
11,9-11,10: NEWLINE '\n'
13,0-13,0: DEDENT ''
13,0-13,0: DEDENT ''
13,0-13,8: NAME 'OP_TYPES'
13,9-13,10: OP '='
13,11-13,12: LBRACE '{'
14,4-14,7: STRING '"("'
14,8-14,9: COLON ':'
14,10-14,15: NAME 'token'
14,15-14,16: DOT '.'
14,16-14,20: NAME 'LPAR'
14,20-14,21: COMMA ','
15,4-15,7: STRING '")"'
15,8-15,9: COLON ':'
15,10-15,15: NAME 'token'
15,15-15,16: DOT '.'
15,16-15,20: NAME 'RPAR'
15,20-15,21: COMMA ','
16,4-16,7: STRING '"["'
16,8-16,9: COLON ':'
16,10-16,15: NAME 'token'
16,15-16,16: DOT '.'
16,16-16,20: NAME 'LSQB'
16,20-16,21: COMMA ','
17,4-17,7: STRING '"]"'
17,8-17,9: COLON ':'
17,10-17,15: NAME 'token'
17,15-17,16: DOT '.'
17,16-17,20: NAME 'RSQB'
17,20-17,21: COMMA ','
18,4-18,7: STRING '"{"'
18,8-18,9: COLON ':'
18,10-18,15: NAME 'token'
18,15-18,16: DOT '.'
18,16-18,22: NAME 'LBRACE'
18,22-18,23: COMMA ','
19,4-19,7: STRING '"}"'
19,8-19,9: COLON ':'
19,10-19,15: NAME 'token'
19,15-19,16: DOT '.'
19,16-19,22: NAME 'RBRACE'
19,22-19,23: COMMA ','
20,4-20,7: STRING '":"'
20,8-20,9: COLON ':'
20,10-20,15: NAME 'token'
20,15-20,16: DOT '.'
20,16-20,21: NAME 'COLON'
20,21-20,22: COMMA ','
21,4-21,7: STRING '","'
21,8-21,9: COLON ':'
21,10-21,15: NAME 'token'
21,15-21,16: DOT '.'
21,16-21,21: NAME 'COMMA'
21,21-21,22: COMMA ','
22,4-22,7: STRING '"."'
22,8-22,9: COLON ':'
22,10-22,15: NAME 'token'
22,15-22,16: DOT '.'
22,16-22,19: NAME 'DOT'
22,19-22,20: COMMA ','
23,4-23,7: STRING '"@"'
23,8-23,9: COLON ':'
23,10-23,15: NAME 'token'
23,15-23,16: DOT '.'
23,16-23,18: NAME 'AT'
23,18-23,19: COMMA ','
24,4-24,5: RBRACE '}'
24,5-24,6: NEWLINE '\n'
26,0-26,3: NAME 'def'
26,4-26,8: NAME 'main'
26,8-26,9: LPAR '('
26,9-26,10: RPAR ')'
26,10-26,11: COLON ':'
26,11-26,12: NEWLINE '\n'
27,0-27,4: INDENT ' '
27,4-27,12: NAME 'readline'
27,13-27,14: OP '='
27,15-27,19: NAME 'open'
27,19-27,20: LPAR '('
27,20-27,23: NAME 'sys'
27,23-27,24: DOT '.'
27,24-27,28: NAME 'argv'
27,28-27,29: LSQB '['
27,29-27,30: NUMBER '1'
27,30-27,31: RSQB ']'
27,31-27,32: COMMA ','
27,33-27,37: STRING '"rb"'
27,37-27,38: RPAR ')'
27,38-27,39: DOT '.'
27,39-27,47: NAME 'readline'
27,47-27,48: NEWLINE '\n'
28,4-28,6: NAME 'if'
28,7-28,10: NAME 'sys'
28,10-28,11: DOT '.'
28,11-28,18: NAME 'version'
28,19-28,20: OP '<'
28,21-28,24: STRING '"3"'
28,24-28,25: COLON ':'
28,25-28,26: NEWLINE '\n'
29,0-29,8: INDENT ' '
29,8-29,16: NAME 'tokenize'
29,16-29,17: DOT '.'
29,17-29,25: NAME 'tokenize'
29,25-29,26: LPAR '('
29,26-29,34: NAME 'readline'
29,34-29,35: COMMA ','
29,36-29,46: NAME 'printtoken'
29,46-29,47: RPAR ')'
29,47-29,48: NEWLINE '\n'
30,4-30,4: DEDENT ''
30,4-30,8: NAME 'else'
30,8-30,9: COLON ':'
30,9-30,10: NEWLINE '\n'
31,0-31,8: INDENT ' '
31,8-31,11: NAME 'for'
31,12-31,16: NAME 'type'
31,16-31,17: COMMA ','
31,18-31,23: NAME 'token'
31,23-31,24: COMMA ','
31,25-31,30: NAME 'start'
31,30-31,31: COMMA ','
31,32-31,35: NAME 'end'
31,35-31,36: COMMA ','
31,37-31,38: NAME '_'
31,39-31,41: NAME 'in'
31,42-31,50: NAME 'tokenize'
31,50-31,51: DOT '.'
31,51-31,59: NAME 'tokenize'
31,59-31,60: LPAR '('
31,60-31,68: NAME 'readline'
31,68-31,69: RPAR ')'
31,69-31,70: COLON ':'
31,70-31,71: NEWLINE '\n'
32,0-32,12: INDENT ' '
32,12-32,14: NAME 'if'
32,15-32,23: NAME 'tokenize'
32,23-32,24: DOT '.'
32,24-32,32: NAME 'tok_name'
32,32-32,33: LSQB '['
32,33-32,37: NAME 'type'
32,37-32,38: RSQB ']'
32,39-32,41: OP '=='
32,42-32,46: STRING '"OP"'
32,46-32,47: COLON ':'
32,47-32,48: NEWLINE '\n'
33,0-33,16: INDENT ' '
33,16-33,20: NAME 'type'
33,21-33,22: OP '='
33,23-33,31: NAME 'OP_TYPES'
33,31-33,32: DOT '.'
33,32-33,35: NAME 'get'
33,35-33,36: LPAR '('
33,36-33,41: NAME 'token'
33,41-33,42: COMMA ','
33,43-33,47: NAME 'type'
33,47-33,48: RPAR ')'
33,48-33,49: NEWLINE '\n'
34,12-34,12: DEDENT ''
34,12-34,14: NAME 'if'
34,15-34,23: NAME 'tokenize'
34,23-34,24: DOT '.'
34,24-34,32: NAME 'tok_name'
34,32-34,33: LSQB '['
34,33-34,37: NAME 'type'
34,37-34,38: RSQB ']'
34,39-34,42: NAME 'not'
34,43-34,45: NAME 'in'
34,46-34,47: LPAR '('
34,47-34,57: STRING '"ENCODING"'
34,57-34,58: COMMA ','
34,59-34,63: STRING '"NL"'
34,63-34,64: RPAR ')'
34,64-34,65: COLON ':'
34,65-34,66: NEWLINE '\n'
35,0-35,16: INDENT ' '
35,16-35,26: NAME 'printtoken'
35,26-35,27: LPAR '('
35,27-35,31: NAME 'type'
35,31-35,32: COMMA ','
35,33-35,38: NAME 'token'
35,38-35,39: COMMA ','
35,40-35,45: NAME 'start'
35,45-35,46: COMMA ','
35,47-35,50: NAME 'end'
35,50-35,51: COMMA ','
35,52-35,53: NAME '_'
35,53-35,54: RPAR ')'
35,54-35,55: NEWLINE '\n'
37,0-37,0: DEDENT ''
37,0-37,0: DEDENT ''
37,0-37,0: DEDENT ''
37,0-37,0: DEDENT ''
37,0-37,2: NAME 'if'
37,3-37,11: NAME '__name__'
37,12-37,14: OP '=='
37,15-37,25: STRING '"__main__"'
37,25-37,26: COLON ':'
37,26-37,27: NEWLINE '\n'
38,0-38,4: INDENT ' '
38,4-38,8: NAME 'main'
38,8-38,9: LPAR '('
38,9-38,10: RPAR ')'
38,10-38,11: NEWLINE '\n'
39,0-39,0: DEDENT ''
39,0-39,0: ENDMARKER ''