hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-16 16:53:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

Python: Fix generator expression locations

Browse Source 
Our logic for detecting the first and last item in a generator
expression was faulty, sometimes matching comments as well. Because
attributes (like `_location_start`) can only be written once, this
caused `tree-sitter-graph` to get unhappy.

To fix this, we now require the first item to be an `expression`, and
the last one to be either a `for_in_clause` or an `if_clause`.
Crucially, `comment` is neither of these, and this prevents the
unfortunate overlap.
This commit is contained in:
Taus
2024-10-28 14:53:09 +00:00
parent ef60b730ea
commit 5d6600e61f
2 changed files with 11 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
python/extractor/tests/parser/comprehensions.py
View File

@@ -65,3 +65,10 @@ t = tuple(x for y in z)
d for e in f if g # comment
# comment
] # comment
# Generator expression with comments
(# comment
alpha # comment
for beta in gamma # comment
# comment
)

8
python/extractor/tsg-python/python.tsg
View File

@@ -404,7 +404,7 @@
;;; GeneratorExp
(generator_expression . "(" . (comment)* . (_) @start (_) @end . (comment)* . ")" .) @generatorexp
(generator_expression . "(" . (comment)* . (expression) @start [(for_in_clause) (if_clause)] @end . (comment)* . ")" .) @generatorexp
{
attr (@generatorexp.node) _location_start = (location-start @start)
attr (@generatorexp.node) _location_end = (location-end @end)
@@ -416,13 +416,13 @@
attr (@if.node) _location_end = (location-end @expr)
}
(generator_expression . "(" . (comment)* . (_) @start (for_in_clause) @child (_) @end . (comment)* . ")" .) @genexpr
(generator_expression . "(" . (comment)* . (expression) @start (for_in_clause) @child [(for_in_clause) (if_clause)] @end . (comment)* . ")" .) @genexpr
{
attr (@child.node) _location_start = (location-start @start)
attr (@child.node) _location_end = (location-end @end)
}
(generator_expression . "(" . (comment)* . (_) @start (for_in_clause) @end . (comment)* . ")" .) @genexpr
(generator_expression . "(" . (comment)* . (expression) @start (for_in_clause) @end . (comment)* . ")" .) @genexpr
{
attr (@end.node) _location_start = (location-start @start)
attr (@end.node) _location_end = (location-end @end)
@@ -863,7 +863,7 @@
; information for the entire generator expression (yes, it is a wide parameter!) and so we must recreate the logic for
; setting this location information correctly.
(generator_expression . "(" . (comment)* . (_) @start (_) @end . (comment)* . ")" .) @genexpr
(generator_expression . "(" . (comment)* . (expression) @start [(for_in_clause) (if_clause)] @end . (comment)* . ")" .) @genexpr
{
; Synthesize the `genexpr` function
let @genexpr.fun = (ast-node @genexpr "Function")