hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-17 01:03:14 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #17688 from github/tausbn/python-3.13-default-type-parser-support

Browse Source 
Python: Add support for type parameter defaults
This commit is contained in:
Taus
2024-10-15 18:01:51 +02:00
committed by GitHub
parent d4e0cb2ffa 417e60a466
commit 28f8874243
24 changed files with 55159 additions and 51484 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1236
python/downgrades/5af903da088e3746aa283700a43a779302453523/old.dbscheme Normal file
View File

File diff suppressed because it is too large Load Diff

42
python/downgrades/5af903da088e3746aa283700a43a779302453523/py_exprs.ql Normal file
View File

@@ -0,0 +1,42 @@
// We must wrap the DB types, as these cannot appear in argument lists
class TypeParameter_ extends @py_type_parameter {
string toString() { result = "TypeParameter" }
}
class Expr_ extends @py_expr {
string toString() { result = "Expr" }
}
class ExprParent_ extends @py_expr_parent {
string toString() { result = "ExprParent" }
}
class TypeVar_ extends @py_TypeVar, TypeParameter_ {
override string toString() { result = "TypeVar" }
}
class TypeVarTuple_ extends @py_TypeVarTuple, TypeParameter_ {
override string toString() { result = "TypeVarTuple" }
}
class ParamSpec_ extends @py_ParamSpec, TypeParameter_ {
override string toString() { result = "ParamSpec" }
}
// From the dbscheme:
// py_exprs(unique int id : @py_expr,
// int kind: int ref,
// int parent : @py_expr_parent ref,
// int idx : int ref);
query predicate py_exprs_without_type_parameter_defaults(
Expr_ id, int kind, ExprParent_ parent, int idx
) {
py_exprs(id, kind, parent, idx) and
// From the dbscheme
// /* <Field> ParamSpec.default = 2, expr */
// /* <Field> TypeVar.default = 3, expr */
// /* <Field> TypeVarTuple.default = 2, expr */
(parent instanceof ParamSpec_ implies idx != 2) and
(parent instanceof TypeVar_ implies idx != 3) and
(parent instanceof TypeVarTuple_ implies idx != 2)
}

1233
python/downgrades/5af903da088e3746aa283700a43a779302453523/semmlecode.python.dbscheme Normal file
View File

File diff suppressed because it is too large Load Diff

3
python/downgrades/5af903da088e3746aa283700a43a779302453523/upgrade.properties Normal file
View File

@@ -0,0 +1,3 @@
description: Remove support for type parameter defaults.
compatibility: backwards
py_exprs.rel: run py_exprs.qlo py_exprs_without_type_parameter_defaults

6
python/extractor/Makefile
View File

@@ -21,19 +21,19 @@ $(TOKENIZER_FILE): $(TOKENIZER_DEPS)
MASTER_FILE = semmle/python/master.py
DBSCHEME_FILE = $(GIT_ROOT)/ql/python/ql/lib/semmlecode.python.dbscheme
DBSCHEME_FILE = $(GIT_ROOT)/python/ql/lib/semmlecode.python.dbscheme
.PHONY: dbscheme
dbscheme: $(MASTER_FILE)
python3 -m semmle.dbscheme_gen $(DBSCHEME_FILE)
AST_GENERATED_DIR = $(GIT_ROOT)/ql/python/ql/lib/semmle/python/
AST_GENERATED_DIR = $(GIT_ROOT)/python/ql/lib/semmle/python/
AST_GENERATED_FILE = $(AST_GENERATED_DIR)AstGenerated.qll
.PHONY: ast
ast: $(MASTER_FILE)
python3 -m semmle.query_gen $(AST_GENERATED_DIR)
$(GIT_ROOT)/target/intree/codeql/codeql query format --in-place $(AST_GENERATED_FILE)
codeql query format --in-place $(AST_GENERATED_FILE)
################################################################################
# Tests

15
python/extractor/semmle/python/ast.py
View File

@@ -500,10 +500,11 @@ class Num(expr):
self.text = text
class ParamSpec(type_parameter):
__slots__ = "name",
__slots__ = "name", "default",
def __init__(self, name):
def __init__(self, name, default):
self.name = name
self.default = default
@@ -607,17 +608,19 @@ class TypeAlias(stmt):
self.value = value
class TypeVar(type_parameter):
__slots__ = "name", "bound",
__slots__ = "name", "bound", "default"
def __init__(self, name, bound):
def __init__(self, name, bound, default):
self.name = name
self.bound = bound
self.default = default
class TypeVarTuple(type_parameter):
__slots__ = "name",
__slots__ = "name", "default",
def __init__(self, name):
def __init__(self, name, default):
self.name = name
self.default = default
class UnaryOp(expr):
__slots__ = "op", "operand",

3
python/extractor/semmle/python/master.py
View File

@@ -397,6 +397,7 @@ Num.field('n', number, 'value')
Num.field('text', number)
ParamSpec.field('name', expr)
ParamSpec.field('default', expr)
Print.field('dest', expr, 'destination')
Print.field('values', expr_list)
@@ -448,8 +449,10 @@ TypeAlias.field('value', expr)
TypeVar.field('name', expr)
TypeVar.field('bound', expr)
TypeVar.field('default', expr)
TypeVarTuple.field('name', expr)
TypeVarTuple.field('default', expr)
UnaryOp.field('op', unaryop, 'operator')
UnaryOp.field('operand', expr)

2
python/extractor/semmle/util.py
View File

@@ -10,7 +10,7 @@ from io import BytesIO
#Semantic version of extractor.
#Update this if any changes are made
VERSION = "7.0.0"
VERSION = "7.1.0"
PY_EXTENSIONS = ".py", ".pyw"

294
python/extractor/tests/parser/types_new.expected
View File

@@ -1,4 +1,4 @@
Module: [1, 0] - [6, 0]
Module: [1, 0] - [23, 0]
body: [
TypeAlias: [1, 0] - [1, 34]
name:
@@ -12,6 +12,7 @@ Module: [1, 0] - [6, 0]
variable: Variable('T1', None)
ctx: Store
bound: None
default: None
TypeVar: [1, 11] - [1, 17]
name:
Name: [1, 11] - [1, 13]
@@ -21,16 +22,19 @@ Module: [1, 0] - [6, 0]
Name: [1, 15] - [1, 17]
variable: Variable('E1', None)
ctx: Load
default: None
TypeVarTuple: [1, 19] - [1, 22]
name:
Name: [1, 20] - [1, 22]
variable: Variable('T3', None)
ctx: Store
default: None
ParamSpec: [1, 24] - [1, 28]
name:
Name: [1, 26] - [1, 28]
variable: Variable('T4', None)
ctx: Store
default: None
]
value:
Name: [1, 32] - [1, 34]
@@ -64,6 +68,7 @@ Module: [1, 0] - [6, 0]
variable: Variable('T6', None)
ctx: Store
bound: None
default: None
TypeVar: [3, 10] - [3, 16]
name:
Name: [3, 10] - [3, 12]
@@ -73,16 +78,19 @@ Module: [1, 0] - [6, 0]
Name: [3, 14] - [3, 16]
variable: Variable('E2', None)
ctx: Load
default: None
TypeVarTuple: [3, 18] - [3, 21]
name:
Name: [3, 19] - [3, 21]
variable: Variable('T8', None)
ctx: Store
default: None
ParamSpec: [3, 23] - [3, 27]
name:
Name: [3, 25] - [3, 27]
variable: Variable('T9', None)
ctx: Store
default: None
]
args: []
vararg: None
@@ -109,6 +117,7 @@ Module: [1, 0] - [6, 0]
variable: Variable('T10', None)
ctx: Store
bound: None
default: None
TypeVar: [5, 13] - [5, 20]
name:
Name: [5, 13] - [5, 16]
@@ -118,16 +127,19 @@ Module: [1, 0] - [6, 0]
Name: [5, 18] - [5, 20]
variable: Variable('E3', None)
ctx: Load
default: None
TypeVarTuple: [5, 22] - [5, 26]
name:
Name: [5, 23] - [5, 26]
variable: Variable('T12', None)
ctx: Store
default: None
ParamSpec: [5, 28] - [5, 33]
name:
Name: [5, 30] - [5, 33]
variable: Variable('T13', None)
ctx: Store
default: None
]
bases: []
keywords: []
@@ -139,4 +151,284 @@ Module: [1, 0] - [6, 0]
value:
Ellipsis: [5, 36] - [5, 39]
]
Assign: [10, 0] - [10, 22]
targets: [
Name: [10, 6] - [10, 10]
variable: Variable('Foo1', None)
ctx: Store
]
value:
ClassExpr: [10, 0] - [10, 22]
name: 'Foo1'
type_parameters: [
TypeVar: [10, 11] - [10, 20]
name:
Name: [10, 11] - [10, 14]
variable: Variable('T14', None)
ctx: Store
bound: None
default:
Name: [10, 17] - [10, 20]
variable: Variable('str', None)
ctx: Load
]
bases: []
keywords: []
inner_scope:
Class: [10, 0] - [10, 22]
name: 'Foo1'
body: [
Expr: [10, 23] - [10, 26]
value:
Ellipsis: [10, 23] - [10, 26]
]
Assign: [13, 0] - [13, 30]
targets: [
Name: [13, 6] - [13, 10]
variable: Variable('Baz1', None)
ctx: Store
]
value:
ClassExpr: [13, 0] - [13, 30]
name: 'Baz1'
type_parameters: [
ParamSpec: [13, 11] - [13, 28]
name:
Name: [13, 13] - [13, 15]
variable: Variable('P1', None)
ctx: Store
default:
List: [13, 18] - [13, 28]
elts: [
Name: [13, 19] - [13, 22]
variable: Variable('int', None)
ctx: Load
Name: [13, 24] - [13, 27]
variable: Variable('str', None)
ctx: Load
]
ctx: Load
]
bases: []
keywords: []
inner_scope:
Class: [13, 0] - [13, 30]
name: 'Baz1'
body: [
Expr: [13, 31] - [13, 34]
value:
Ellipsis: [13, 31] - [13, 34]
]
Assign: [16, 0] - [16, 37]
targets: [
Name: [16, 6] - [16, 10]
variable: Variable('Qux1', None)
ctx: Store
]
value:
ClassExpr: [16, 0] - [16, 37]
name: 'Qux1'
type_parameters: [
TypeVarTuple: [16, 11] - [16, 35]
name:
Name: [16, 12] - [16, 15]
variable: Variable('Ts1', None)
ctx: Store
default:
Starred: [16, 18] - [16, 35]
value:
Subscript: [16, 19] - [16, 35]
value:
Name: [16, 19] - [16, 24]
variable: Variable('tuple', None)
ctx: Load
index:
Tuple: [16, 25] - [16, 34]
elts: [
Name: [16, 25] - [16, 28]
variable: Variable('int', None)
ctx: Load
Name: [16, 30] - [16, 34]
variable: Variable('bool', None)
ctx: Load
]
ctx: Load
ctx: Load
ctx: Load
]
bases: []
keywords: []
inner_scope:
Class: [16, 0] - [16, 37]
name: 'Qux1'
body: [
Expr: [16, 38] - [16, 41]
value:
Ellipsis: [16, 38] - [16, 41]
]
TypeAlias: [19, 0] - [19, 40]
name:
Name: [19, 5] - [19, 9]
variable: Variable('Foo2', None)
ctx: Store
type_parameters: [
TypeVar: [19, 10] - [19, 13]
name:
Name: [19, 10] - [19, 13]
variable: Variable('T15', None)
ctx: Store
bound: None
default: None
TypeVar: [19, 15] - [19, 23]
name:
Name: [19, 15] - [19, 17]
variable: Variable('U1', None)
ctx: Store
bound: None
default:
Name: [19, 20] - [19, 23]
variable: Variable('str', None)
ctx: Load
]
value:
Subscript: [19, 27] - [19, 40]
value:
Name: [19, 27] - [19, 31]
variable: Variable('Bar1', None)
ctx: Load
index:
Tuple: [19, 32] - [19, 39]
elts: [
Name: [19, 32] - [19, 35]
variable: Variable('T15', None)
ctx: Load
Name: [19, 37] - [19, 39]
variable: Variable('U1', None)
ctx: Load
]
ctx: Load
ctx: Load
TypeAlias: [20, 0] - [20, 41]
name:
Name: [20, 5] - [20, 9]
variable: Variable('Baz2', None)
ctx: Store
type_parameters: [
ParamSpec: [20, 10] - [20, 27]
name:
Name: [20, 12] - [20, 14]
variable: Variable('P2', None)
ctx: Store
default:
List: [20, 17] - [20, 27]
elts: [
Name: [20, 18] - [20, 21]
variable: Variable('int', None)
ctx: Load
Name: [20, 23] - [20, 26]
variable: Variable('str', None)
ctx: Load
]
ctx: Load
]
value:
Subscript: [20, 31] - [20, 41]
value:
Name: [20, 31] - [20, 35]
variable: Variable('Spam', None)
ctx: Load
index:
BinOp: [20, 36] - [20, 40]
left:
Name: [20, 36] - [20, 36]
variable: Variable('', None)
ctx: Load
op: Pow
right:
Name: [20, 38] - [20, 40]
variable: Variable('P2', None)
ctx: Load
ctx: Load
TypeAlias: [21, 0] - [21, 41]
name:
Name: [21, 5] - [21, 9]
variable: Variable('Qux2', None)
ctx: Store
type_parameters: [
TypeVarTuple: [21, 10] - [21, 28]
name:
Name: [21, 11] - [21, 14]
variable: Variable('Ts2', None)
ctx: Store
default:
Starred: [21, 17] - [21, 28]
value:
Subscript: [21, 18] - [21, 28]
value:
Name: [21, 18] - [21, 23]
variable: Variable('tuple', None)
ctx: Load
index:
Name: [21, 24] - [21, 27]
variable: Variable('str', None)
ctx: Load
ctx: Load
ctx: Load
]
value:
Subscript: [21, 32] - [21, 41]
value:
Name: [21, 32] - [21, 35]
variable: Variable('Ham', None)
ctx: Load
index:
Starred: [21, 36] - [21, 40]
value:
Name: [21, 37] - [21, 40]
variable: Variable('Ts2', None)
ctx: Load
ctx: Load
ctx: Load
TypeAlias: [22, 0] - [22, 39]
name:
Name: [22, 5] - [22, 8]
variable: Variable('Rab', None)
ctx: Store
type_parameters: [
TypeVar: [22, 9] - [22, 11]
name:
Name: [22, 9] - [22, 11]
variable: Variable('U2', None)
ctx: Store
bound: None
default: None
TypeVar: [22, 13] - [22, 22]
name:
Name: [22, 13] - [22, 16]
variable: Variable('T15', None)
ctx: Store
bound: None
default:
Name: [22, 19] - [22, 22]
variable: Variable('str', None)
ctx: Load
]
value:
Subscript: [22, 26] - [22, 39]
value:
Name: [22, 26] - [22, 30]
variable: Variable('Bar2', None)
ctx: Load
index:
Tuple: [22, 31] - [22, 38]
elts: [
Name: [22, 31] - [22, 34]
variable: Variable('T15', None)
ctx: Load
Name: [22, 36] - [22, 38]
variable: Variable('U2', None)
ctx: Load
]
ctx: Load
ctx: Load
]

17
python/extractor/tests/parser/types_new.py
View File

@@ -3,3 +3,20 @@ type T[T1, T2: E1, *T3, **T4] = T5
def f[T6, T7: E2, *T8, **T9](): ...
class C[T10, T11: E3, *T12, **T13]: ...
# From PEP-696 (https://peps.python.org/pep-0696/#grammar-changes):
# TypeVars
class Foo1[T14 = str]: ...
# ParamSpecs
class Baz1[**P1 = [int, str]]: ...
# TypeVarTuples
class Qux1[*Ts1 = *tuple[int, bool]]: ...
# TypeAliases
type Foo2[T15, U1 = str] = Bar1[T15, U1]
type Baz2[**P2 = [int, str]] = Spam[**P2]
type Qux2[*Ts2 = *tuple[str]] = Ham[*Ts2]
type Rab[U2, T15 = str] = Bar2[T15, U2]

15
python/extractor/tsg-python/python.tsg
View File

@@ -3388,6 +3388,7 @@
(typevar_parameter
name: (_) @name
bound: (_)? @bound
default: (_)? @default
) @typevar
{
attr (@name.node) ctx = "store"
@@ -3396,22 +3397,36 @@
attr (@bound.node) ctx = "load"
attr (@typevar.node) bound = @bound.node
}
if some @default {
attr (@default.node) ctx = "load"
attr (@typevar.node) default = @default.node
}
}
(typevartuple_parameter
name: (_) @name
default: (_)? @default
) @typevartuple
{
attr (@name.node) ctx = "store"
attr (@typevartuple.node) name = @name.node
if some @default {
attr (@default.node) ctx = "load"
attr (@typevartuple.node) default = @default.node
}
}
(paramspec_parameter
name: (_) @name
default: (_)? @default
) @paramspec
{
attr (@name.node) ctx = "store"
attr (@paramspec.node) name = @name.node
if some @default {
attr (@default.node) ctx = "load"
attr (@paramspec.node) default = @default.node
}
}
;;;;;; End of Type parameters (`T: ..., *T, **T`)

12
python/extractor/tsg-python/tsp/grammar.js
View File

@@ -589,17 +589,20 @@ module.exports = grammar({
typevar_parameter: $ => seq(
field('name', $.identifier),
optional($._type_bound)
optional($._type_bound),
optional($._type_param_default)
),
typevartuple_parameter: $ => seq(
'*',
field('name', $.identifier),
optional($._type_param_default)
),
paramspec_parameter: $ => seq(
'**',
field('name', $.identifier),
optional($._type_param_default),
),
_type_parameter: $ => choice(
@@ -608,6 +611,11 @@ module.exports = grammar({
$.paramspec_parameter,
),
_type_param_default: $ => seq(
'=',
field('default', choice($.list_splat, $.expression))
),
parenthesized_list_splat: $ => prec(PREC.parenthesized_list_splat, seq(
'(',
choice(
@@ -923,7 +931,7 @@ module.exports = grammar({
subscript: $ => prec(PREC.call, seq(
field('value', $.primary_expression),
'[',
commaSep1(field('subscript', choice($.expression, $.slice))),
commaSep1(field('subscript', choice($.list_splat, $.expression, $.slice))),
optional(','),
']'
)),

72
python/extractor/tsg-python/tsp/src/grammar.json
View File

@@ -1,4 +1,5 @@
{
"$schema": "https://tree-sitter.github.io/tree-sitter/assets/schemas/grammar.schema.json",
"name": "python",
"word": "identifier",
"rules": {
@@ -2970,6 +2971,18 @@
"type": "BLANK"
}
]
},
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_type_param_default"
},
{
"type": "BLANK"
}
]
}
]
},
@@ -2987,6 +3000,18 @@
"type": "SYMBOL",
"name": "identifier"
}
},
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_type_param_default"
},
{
"type": "BLANK"
}
]
}
]
},
@@ -3004,6 +3029,18 @@
"type": "SYMBOL",
"name": "identifier"
}
},
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_type_param_default"
},
{
"type": "BLANK"
}
]
}
]
},
@@ -3024,6 +3061,32 @@
}
]
},
"_type_param_default": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "="
},
{
"type": "FIELD",
"name": "default",
"content": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "list_splat"
},
{
"type": "SYMBOL",
"name": "expression"
}
]
}
}
]
},
"parenthesized_list_splat": {
"type": "PREC",
"value": 1,
@@ -5006,6 +5069,10 @@
"content": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "list_splat"
},
{
"type": "SYMBOL",
"name": "expression"
@@ -5032,6 +5099,10 @@
"content": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "list_splat"
},
{
"type": "SYMBOL",
"name": "expression"
@@ -6612,4 +6683,3 @@
"parameter"
]
}

55
python/extractor/tsg-python/tsp/src/node-types.json
View File

@@ -2691,6 +2691,7 @@
{
"type": "module",
"named": true,
"root": true,
"fields": {},
"children": {
"multiple": true,
@@ -2809,6 +2810,20 @@
"type": "paramspec_parameter",
"named": true,
"fields": {
"default": {
"multiple": false,
"required": false,
"types": [
{
"type": "expression",
"named": true
},
{
"type": "list_splat",
"named": true
}
]
},
"name": {
"multiple": false,
"required": true,
@@ -3193,6 +3208,10 @@
"type": "expression",
"named": true
},
{
"type": "list_splat",
"named": true
},
{
"type": "slice",
"named": true
@@ -3476,6 +3495,20 @@
}
]
},
"default": {
"multiple": false,
"required": false,
"types": [
{
"type": "expression",
"named": true
},
{
"type": "list_splat",
"named": true
}
]
},
"name": {
"multiple": false,
"required": true,
@@ -3492,6 +3525,20 @@
"type": "typevartuple_parameter",
"named": true,
"fields": {
"default": {
"multiple": false,
"required": false,
"types": [
{
"type": "expression",
"named": true
},
{
"type": "list_splat",
"named": true
}
]
},
"name": {
"multiple": false,
"required": true,
@@ -3765,6 +3812,10 @@
"type": ":=",
"named": false
},
{
"type": ";",
"named": false
},
{
"type": "<",
"named": false
@@ -3821,6 +3872,10 @@
"type": "[",
"named": false
},
{
"type": "\\",
"named": false
},
{
"type": "]",
"named": false

100738
python/extractor/tsg-python/tsp/src/parser.c
View File

File diff suppressed because it is too large Load Diff

54
python/extractor/tsg-python/tsp/src/tree_sitter/alloc.h Normal file
View File

@@ -0,0 +1,54 @@
#ifndef TREE_SITTER_ALLOC_H_
#define TREE_SITTER_ALLOC_H_
#ifdef __cplusplus
extern "C" {
#endif
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
// Allow clients to override allocation functions
#ifdef TREE_SITTER_REUSE_ALLOCATOR
extern void *(*ts_current_malloc)(size_t size);
extern void *(*ts_current_calloc)(size_t count, size_t size);
extern void *(*ts_current_realloc)(void *ptr, size_t size);
extern void (*ts_current_free)(void *ptr);
#ifndef ts_malloc
#define ts_malloc ts_current_malloc
#endif
#ifndef ts_calloc
#define ts_calloc ts_current_calloc
#endif
#ifndef ts_realloc
#define ts_realloc ts_current_realloc
#endif
#ifndef ts_free
#define ts_free ts_current_free
#endif
#else
#ifndef ts_malloc
#define ts_malloc malloc
#endif
#ifndef ts_calloc
#define ts_calloc calloc
#endif
#ifndef ts_realloc
#define ts_realloc realloc
#endif
#ifndef ts_free
#define ts_free free
#endif
#endif
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_ALLOC_H_

290
python/extractor/tsg-python/tsp/src/tree_sitter/array.h Normal file
View File

@@ -0,0 +1,290 @@
#ifndef TREE_SITTER_ARRAY_H_
#define TREE_SITTER_ARRAY_H_
#ifdef __cplusplus
extern "C" {
#endif
#include "./alloc.h"
#include <assert.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#ifdef _MSC_VER
#pragma warning(disable : 4101)
#elif defined(__GNUC__) || defined(__clang__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-variable"
#endif
#define Array(T) \
struct { \
T *contents; \
uint32_t size; \
uint32_t capacity; \
}
/// Initialize an array.
#define array_init(self) \
((self)->size = 0, (self)->capacity = 0, (self)->contents = NULL)
/// Create an empty array.
#define array_new() \
{ NULL, 0, 0 }
/// Get a pointer to the element at a given `index` in the array.
#define array_get(self, _index) \
(assert((uint32_t)(_index) < (self)->size), &(self)->contents[_index])
/// Get a pointer to the first element in the array.
#define array_front(self) array_get(self, 0)
/// Get a pointer to the last element in the array.
#define array_back(self) array_get(self, (self)->size - 1)
/// Clear the array, setting its size to zero. Note that this does not free any
/// memory allocated for the array's contents.
#define array_clear(self) ((self)->size = 0)
/// Reserve `new_capacity` elements of space in the array. If `new_capacity` is
/// less than the array's current capacity, this function has no effect.
#define array_reserve(self, new_capacity) \
_array__reserve((Array *)(self), array_elem_size(self), new_capacity)
/// Free any memory allocated for this array. Note that this does not free any
/// memory allocated for the array's contents.
#define array_delete(self) _array__delete((Array *)(self))
/// Push a new `element` onto the end of the array.
#define array_push(self, element) \
(_array__grow((Array *)(self), 1, array_elem_size(self)), \
(self)->contents[(self)->size++] = (element))
/// Increase the array's size by `count` elements.
/// New elements are zero-initialized.
#define array_grow_by(self, count) \
do { \
if ((count) == 0) break; \
_array__grow((Array *)(self), count, array_elem_size(self)); \
memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \
(self)->size += (count); \
} while (0)
/// Append all elements from one array to the end of another.
#define array_push_all(self, other) \
array_extend((self), (other)->size, (other)->contents)
/// Append `count` elements to the end of the array, reading their values from the
/// `contents` pointer.
#define array_extend(self, count, contents) \
_array__splice( \
(Array *)(self), array_elem_size(self), (self)->size, \
0, count, contents \
)
/// Remove `old_count` elements from the array starting at the given `index`. At
/// the same index, insert `new_count` new elements, reading their values from the
/// `new_contents` pointer.
#define array_splice(self, _index, old_count, new_count, new_contents) \
_array__splice( \
(Array *)(self), array_elem_size(self), _index, \
old_count, new_count, new_contents \
)
/// Insert one `element` into the array at the given `index`.
#define array_insert(self, _index, element) \
_array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, &(element))
/// Remove one element from the array at the given `index`.
#define array_erase(self, _index) \
_array__erase((Array *)(self), array_elem_size(self), _index)
/// Pop the last element off the array, returning the element by value.
#define array_pop(self) ((self)->contents[--(self)->size])
/// Assign the contents of one array to another, reallocating if necessary.
#define array_assign(self, other) \
_array__assign((Array *)(self), (const Array *)(other), array_elem_size(self))
/// Swap one array with another
#define array_swap(self, other) \
_array__swap((Array *)(self), (Array *)(other))
/// Get the size of the array contents
#define array_elem_size(self) (sizeof *(self)->contents)
/// Search a sorted array for a given `needle` value, using the given `compare`
/// callback to determine the order.
///
/// If an existing element is found to be equal to `needle`, then the `index`
/// out-parameter is set to the existing value's index, and the `exists`
/// out-parameter is set to true. Otherwise, `index` is set to an index where
/// `needle` should be inserted in order to preserve the sorting, and `exists`
/// is set to false.
#define array_search_sorted_with(self, compare, needle, _index, _exists) \
_array__search_sorted(self, 0, compare, , needle, _index, _exists)
/// Search a sorted array for a given `needle` value, using integer comparisons
/// of a given struct field (specified with a leading dot) to determine the order.
///
/// See also `array_search_sorted_with`.
#define array_search_sorted_by(self, field, needle, _index, _exists) \
_array__search_sorted(self, 0, _compare_int, field, needle, _index, _exists)
/// Insert a given `value` into a sorted array, using the given `compare`
/// callback to determine the order.
#define array_insert_sorted_with(self, compare, value) \
do { \
unsigned _index, _exists; \
array_search_sorted_with(self, compare, &(value), &_index, &_exists); \
if (!_exists) array_insert(self, _index, value); \
} while (0)
/// Insert a given `value` into a sorted array, using integer comparisons of
/// a given struct field (specified with a leading dot) to determine the order.
///
/// See also `array_search_sorted_by`.
#define array_insert_sorted_by(self, field, value) \
do { \
unsigned _index, _exists; \
array_search_sorted_by(self, field, (value) field, &_index, &_exists); \
if (!_exists) array_insert(self, _index, value); \
} while (0)
// Private
typedef Array(void) Array;
/// This is not what you're looking for, see `array_delete`.
static inline void _array__delete(Array *self) {
if (self->contents) {
ts_free(self->contents);
self->contents = NULL;
self->size = 0;
self->capacity = 0;
}
}
/// This is not what you're looking for, see `array_erase`.
static inline void _array__erase(Array *self, size_t element_size,
uint32_t index) {
assert(index < self->size);
char *contents = (char *)self->contents;
memmove(contents + index * element_size, contents + (index + 1) * element_size,
(self->size - index - 1) * element_size);
self->size--;
}
/// This is not what you're looking for, see `array_reserve`.
static inline void _array__reserve(Array *self, size_t element_size, uint32_t new_capacity) {
if (new_capacity > self->capacity) {
if (self->contents) {
self->contents = ts_realloc(self->contents, new_capacity * element_size);
} else {
self->contents = ts_malloc(new_capacity * element_size);
}
self->capacity = new_capacity;
}
}
/// This is not what you're looking for, see `array_assign`.
static inline void _array__assign(Array *self, const Array *other, size_t element_size) {
_array__reserve(self, element_size, other->size);
self->size = other->size;
memcpy(self->contents, other->contents, self->size * element_size);
}
/// This is not what you're looking for, see `array_swap`.
static inline void _array__swap(Array *self, Array *other) {
Array swap = *other;
*other = *self;
*self = swap;
}
/// This is not what you're looking for, see `array_push` or `array_grow_by`.
static inline void _array__grow(Array *self, uint32_t count, size_t element_size) {
uint32_t new_size = self->size + count;
if (new_size > self->capacity) {
uint32_t new_capacity = self->capacity * 2;
if (new_capacity < 8) new_capacity = 8;
if (new_capacity < new_size) new_capacity = new_size;
_array__reserve(self, element_size, new_capacity);
}
}
/// This is not what you're looking for, see `array_splice`.
static inline void _array__splice(Array *self, size_t element_size,
uint32_t index, uint32_t old_count,
uint32_t new_count, const void *elements) {
uint32_t new_size = self->size + new_count - old_count;
uint32_t old_end = index + old_count;
uint32_t new_end = index + new_count;
assert(old_end <= self->size);
_array__reserve(self, element_size, new_size);
char *contents = (char *)self->contents;
if (self->size > old_end) {
memmove(
contents + new_end * element_size,
contents + old_end * element_size,
(self->size - old_end) * element_size
);
}
if (new_count > 0) {
if (elements) {
memcpy(
(contents + index * element_size),
elements,
new_count * element_size
);
} else {
memset(
(contents + index * element_size),
0,
new_count * element_size
);
}
}
self->size += new_count - old_count;
}
/// A binary search routine, based on Rust's `std::slice::binary_search_by`.
/// This is not what you're looking for, see `array_search_sorted_with` or `array_search_sorted_by`.
#define _array__search_sorted(self, start, compare, suffix, needle, _index, _exists) \
do { \
*(_index) = start; \
*(_exists) = false; \
uint32_t size = (self)->size - *(_index); \
if (size == 0) break; \
int comparison; \
while (size > 1) { \
uint32_t half_size = size / 2; \
uint32_t mid_index = *(_index) + half_size; \
comparison = compare(&((self)->contents[mid_index] suffix), (needle)); \
if (comparison <= 0) *(_index) = mid_index; \
size -= half_size; \
} \
comparison = compare(&((self)->contents[*(_index)] suffix), (needle)); \
if (comparison == 0) *(_exists) = true; \
else if (comparison < 0) *(_index) += 1; \
} while (0)
/// Helper macro for the `_sorted_by` routines below. This takes the left (existing)
/// parameter by reference in order to work with the generic sorting function above.
#define _compare_int(a, b) ((int)*(a) - (int)(b))
#ifdef _MSC_VER
#pragma warning(default : 4101)
#elif defined(__GNUC__) || defined(__clang__)
#pragma GCC diagnostic pop
#endif
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_ARRAY_H_

68
python/extractor/tsg-python/tsp/src/tree_sitter/parser.h
View File

@@ -13,9 +13,8 @@ extern "C" {
#define ts_builtin_sym_end 0
#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
typedef uint16_t TSStateId;
#ifndef TREE_SITTER_API_H_
typedef uint16_t TSStateId;
typedef uint16_t TSSymbol;
typedef uint16_t TSFieldId;
typedef struct TSLanguage TSLanguage;
@@ -48,6 +47,7 @@ struct TSLexer {
uint32_t (*get_column)(TSLexer *);
bool (*is_at_included_range_start)(const TSLexer *);
bool (*eof)(const TSLexer *);
void (*log)(const TSLexer *, const char *, ...);
};
typedef enum {
@@ -87,6 +87,11 @@ typedef union {
} entry;
} TSParseActionEntry;
typedef struct {
int32_t start;
int32_t end;
} TSCharacterRange;
struct TSLanguage {
uint32_t version;
uint32_t symbol_count;
@@ -126,13 +131,38 @@ struct TSLanguage {
const TSStateId *primary_state_ids;
};
static inline bool set_contains(TSCharacterRange *ranges, uint32_t len, int32_t lookahead) {
uint32_t index = 0;
uint32_t size = len - index;
while (size > 1) {
uint32_t half_size = size / 2;
uint32_t mid_index = index + half_size;
TSCharacterRange *range = &ranges[mid_index];
if (lookahead >= range->start && lookahead <= range->end) {
return true;
} else if (lookahead > range->end) {
index = mid_index;
}
size -= half_size;
}
TSCharacterRange *range = &ranges[index];
return (lookahead >= range->start && lookahead <= range->end);
}
/*
* Lexer Macros
*/
#ifdef _MSC_VER
#define UNUSED __pragma(warning(suppress : 4101))
#else
#define UNUSED __attribute__((unused))
#endif
#define START_LEXER() \
bool result = false; \
bool skip = false; \
UNUSED \
bool eof = false; \
int32_t lookahead; \
goto start; \
@@ -148,6 +178,17 @@ struct TSLanguage {
goto next_state; \
}
#define ADVANCE_MAP(...) \
{ \
static const uint16_t map[] = { __VA_ARGS__ }; \
for (uint32_t i = 0; i < sizeof(map) / sizeof(map[0]); i += 2) { \
if (map[i] == lookahead) { \
state = map[i + 1]; \
goto next_state; \
} \
} \
}
#define SKIP(state_value) \
{ \
skip = true; \
@@ -166,7 +207,7 @@ struct TSLanguage {
* Parse Table Macros
*/
#define SMALL_STATE(id) id - LARGE_STATE_COUNT
#define SMALL_STATE(id) ((id) - LARGE_STATE_COUNT)
#define STATE(id) id
@@ -176,7 +217,7 @@ struct TSLanguage {
{{ \
.shift = { \
.type = TSParseActionTypeShift, \
.state = state_value \
.state = (state_value) \
} \
}}
@@ -184,7 +225,7 @@ struct TSLanguage {
{{ \
.shift = { \
.type = TSParseActionTypeShift, \
.state = state_value, \
.state = (state_value), \
.repetition = true \
} \
}}
@@ -197,14 +238,15 @@ struct TSLanguage {
} \
}}
#define REDUCE(symbol_val, child_count_val, ...) \
{{ \
.reduce = { \
.type = TSParseActionTypeReduce, \
.symbol = symbol_val, \
.child_count = child_count_val, \
__VA_ARGS__ \
}, \
#define REDUCE(symbol_name, children, precedence, prod_id) \
{{ \
.reduce = { \
.type = TSParseActionTypeReduce, \
.symbol = symbol_name, \
.child_count = children, \
.dynamic_precedence = precedence, \
.production_id = prod_id \
}, \
}}
#define RECOVER() \

5
python/ql/lib/change-notes/2024-10-09-type-parameter-defaults.md Normal file
View File

@@ -0,0 +1,5 @@
---
category: minorAnalysis
---
- Added support for type parameter defaults, as specified in [PEP-696](https://peps.python.org/pep-0696/).

9
python/ql/lib/semmle/python/AstGenerated.qll
View File

@@ -1126,6 +1126,9 @@ class ParamSpec_ extends @py_ParamSpec, TypeParameter {
/** Gets the name of this parameter spec. */
Expr getName() { py_exprs(result, _, this, 1) }
/** Gets the default of this parameter spec. */
Expr getDefault() { py_exprs(result, _, this, 2) }
override string toString() { result = "ParamSpec" }
}
@@ -1466,6 +1469,9 @@ class TypeVar_ extends @py_TypeVar, TypeParameter {
/** Gets the bound of this type variable. */
Expr getBound() { py_exprs(result, _, this, 2) }
/** Gets the default of this type variable. */
Expr getDefault() { py_exprs(result, _, this, 3) }
override string toString() { result = "TypeVar" }
}
@@ -1474,6 +1480,9 @@ class TypeVarTuple_ extends @py_TypeVarTuple, TypeParameter {
/** Gets the name of this type variable tuple. */
Expr getName() { py_exprs(result, _, this, 1) }
/** Gets the default of this type variable tuple. */
Expr getDefault() { py_exprs(result, _, this, 2) }
override string toString() { result = "TypeVarTuple" }
}

3
python/ql/lib/semmlecode.python.dbscheme
View File

@@ -617,6 +617,7 @@ py_extracted_version(int module : @py_Module ref,
/* <Field> ParamSpec.location = 0, location */
/* <Field> ParamSpec.name = 1, expr */
/* <Field> ParamSpec.default = 2, expr */
/* <Field> Pass.location = 0, location */
@@ -715,9 +716,11 @@ py_extracted_version(int module : @py_Module ref,
/* <Field> TypeVar.location = 0, location */
/* <Field> TypeVar.name = 1, expr */
/* <Field> TypeVar.bound = 2, expr */
/* <Field> TypeVar.default = 3, expr */
/* <Field> TypeVarTuple.location = 0, location */
/* <Field> TypeVarTuple.name = 1, expr */
/* <Field> TypeVarTuple.default = 2, expr */
/* <Field> UnaryExpr.location = 0, location */
/* <Field> UnaryExpr.parenthesised = 1, bool */

1233
python/ql/lib/upgrades/728c6d65e61d808ae276013ebc15abc3a97aaef1/old.dbscheme Normal file
View File

File diff suppressed because it is too large Load Diff

1236
python/ql/lib/upgrades/728c6d65e61d808ae276013ebc15abc3a97aaef1/semmlecode.python.dbscheme Normal file
View File

File diff suppressed because it is too large Load Diff

2
python/ql/lib/upgrades/728c6d65e61d808ae276013ebc15abc3a97aaef1/upgrade.properties Normal file
View File

@@ -0,0 +1,2 @@
description: Add support for type parameter defaults.
compatibility: backwards