Merge pull request #21694 from github/tausbn/python-add-support-for-pep-810

Python: Add support for PEP 810

python/downgrades/eb5fc917c79bb23ce2de4a022f3e566d57a91be9/py_bools.ql

@@ -0,0 +1,12 @@
+class BoolParent extends @py_bool_parent {
+  string toString() { result = "BoolParent" }
+}
+
+// Drop py_bools rows for Import and ImportStar parents,
+// since the old schema does not include them in @py_bool_parent.
+from BoolParent parent, int idx
+where
+  py_bools(parent, idx) and
+  not parent instanceof @py_Import and
+  not parent instanceof @py_ImportStar
+select parent, idx

python/downgrades/eb5fc917c79bb23ce2de4a022f3e566d57a91be9/upgrade.properties

@@ -0,0 +1,3 @@
+description: Remove is_lazy field from Import and ImportStar (downgrade PEP 810 lazy imports)
+compatibility: backwards
+py_bools.rel: run py_bools.qlo

python/extractor/semmle/python/ast.py

@@ -845,17 +845,19 @@ class If(stmt):
 
 
 class Import(stmt):
-    __slots__ = "names",
+    __slots__ = "is_lazy", "names",
 
-    def __init__(self, names):
+    def __init__(self, names, is_lazy=False):
         self.names = names
+        self.is_lazy = is_lazy
 
 
 class ImportFrom(stmt):
-    __slots__ = "module",
+    __slots__ = "is_lazy", "module",
 
-    def __init__(self, module):
+    def __init__(self, module, is_lazy=False):
         self.module = module
+        self.is_lazy = is_lazy
 
 
 class Nonlocal(stmt):

python/extractor/semmle/python/master.py

@@ -334,9 +334,11 @@ IfExp.field('body', expr, 'if-true expression')
 IfExp.field('orelse', expr, 'if-false expression')
 
 Import.field('names', alias_list, 'alias list')
+Import.field('is_lazy', bool_, 'lazy')
 
 ImportFrom.set_name('ImportStar')
 ImportFrom.field('module', expr)
+ImportFrom.field('is_lazy', bool_, 'lazy')
 
 ImportMember.field('module', expr)
 ImportMember.field('name', string)

python/extractor/semmle/python/parser/dump_ast.py

@@ -72,8 +72,8 @@ class AstDumper(object):
             # just not print it in that case.
             if field == "parenthesised" and value is None:
                 continue
-            # Likewise, the default value for `is_async` is `False`, so we don't need to print it.
-            if field == "is_async" and value is False:
+            # Likewise, the default value for `is_async` and `is_lazy` is `False`, so we don't need to print it.
+            if field in ("is_async", "is_lazy") and value is False:
                 continue
             output.write("{}  {}:".format(indent,field))
             if isinstance(value, list):

python/extractor/semmle/python/parser/tsg_parser.py

@@ -291,7 +291,7 @@ def create_placeholder_args(cls):
     if cls in (ast.Raise, ast.Ellipsis):
         return {}
     fields = ast_fields[cls]
-    args = {field: None for field in fields if field != "is_async"}
+    args = {field: None for field in fields if field not in ("is_async", "is_lazy")}
     for field in list_fields.get(cls, ()):
         args[field] = []
     if cls in (ast.GeneratorExp, ast.ListComp, ast.SetComp, ast.DictComp):

python/extractor/semmle/query_gen.py

@@ -211,6 +211,8 @@ HEADER = '''/**
  *      WARNING: Any modifications to this file will be lost.
  *      Relations can be changed by modifying master.py.
  */
+overlay[local]
+module;
 '''
 
 def main():

python/extractor/tests/parser/lazy_imports_new.expected

@@ -0,0 +1,284 @@
+Module: [2, 0] - [35, 0]
+  body: [
+    Import: [2, 0] - [2, 13]
+      is_lazy: True
+      names: [
+        alias: [2, 12] - [2, 13]
+          value:
+            ImportExpr: [2, 12] - [2, 13]
+              level: 0
+              name: 'a'
+              top: True
+          asname:
+            Name: [2, 12] - [2, 13]
+              variable: Variable('a', None)
+              ctx: Store
+      ]
+    Import: [4, 0] - [4, 18]
+      is_lazy: True
+      names: [
+        alias: [4, 12] - [4, 14]
+          value:
+            ImportExpr: [4, 12] - [4, 14]
+              level: 0
+              name: 'b1'
+              top: True
+          asname:
+            Name: [4, 12] - [4, 14]
+              variable: Variable('b1', None)
+              ctx: Store
+        alias: [4, 16] - [4, 18]
+          value:
+            ImportExpr: [4, 16] - [4, 18]
+              level: 0
+              name: 'b2'
+              top: True
+          asname:
+            Name: [4, 16] - [4, 18]
+              variable: Variable('b2', None)
+              ctx: Store
+      ]
+    Import: [6, 0] - [6, 20]
+      is_lazy: True
+      names: [
+        alias: [6, 12] - [6, 20]
+          value:
+            ImportExpr: [6, 12] - [6, 20]
+              level: 0
+              name: 'c1.c2.c3'
+              top: True
+          asname:
+            Name: [6, 12] - [6, 20]
+              variable: Variable('c1', None)
+              ctx: Store
+      ]
+    Import: [8, 0] - [8, 23]
+      is_lazy: True
+      names: [
+        alias: [8, 12] - [8, 23]
+          value:
+            ImportExpr: [8, 12] - [8, 17]
+              level: 0
+              name: 'd1.d2'
+              top: False
+          asname:
+            Name: [8, 21] - [8, 23]
+              variable: Variable('d3', None)
+              ctx: Store
+      ]
+    Import: [10, 0] - [10, 20]
+      is_lazy: True
+      names: [
+        alias: [10, 19] - [10, 20]
+          value:
+            ImportMember: [10, 19] - [10, 20]
+              module:
+                ImportExpr: [10, 10] - [10, 11]
+                  level: 0
+                  name: 'e'
+                  top: False
+              name: 'f'
+          asname:
+            Name: [10, 19] - [10, 20]
+              variable: Variable('f', None)
+              ctx: Store
+      ]
+    Import: [12, 0] - [12, 29]
+      is_lazy: True
+      names: [
+        alias: [12, 23] - [12, 25]
+          value:
+            ImportMember: [12, 23] - [12, 25]
+              module:
+                ImportExpr: [12, 10] - [12, 15]
+                  level: 0
+                  name: 'g1.g2'
+                  top: False
+              name: 'h1'
+          asname:
+            Name: [12, 23] - [12, 25]
+              variable: Variable('h1', None)
+              ctx: Store
+        alias: [12, 27] - [12, 29]
+          value:
+            ImportMember: [12, 27] - [12, 29]
+              module:
+                ImportExpr: [12, 10] - [12, 15]
+                  level: 0
+                  name: 'g1.g2'
+                  top: False
+              name: 'h2'
+          asname:
+            Name: [12, 27] - [12, 29]
+              variable: Variable('h2', None)
+              ctx: Store
+      ]
+    Import: [14, 0] - [14, 32]
+      is_lazy: True
+      names: [
+        alias: [14, 20] - [14, 28]
+          value:
+            ImportMember: [14, 20] - [14, 28]
+              module:
+                ImportExpr: [14, 10] - [14, 12]
+                  level: 0
+                  name: 'i1'
+                  top: False
+              name: 'j1'
+          asname:
+            Name: [14, 26] - [14, 28]
+              variable: Variable('j2', None)
+              ctx: Store
+        alias: [14, 30] - [14, 32]
+          value:
+            ImportMember: [14, 30] - [14, 32]
+              module:
+                ImportExpr: [14, 10] - [14, 12]
+                  level: 0
+                  name: 'i1'
+                  top: False
+              name: 'j3'
+          asname:
+            Name: [14, 30] - [14, 32]
+              variable: Variable('j3', None)
+              ctx: Store
+      ]
+    Import: [16, 0] - [16, 37]
+      is_lazy: True
+      names: [
+        alias: [16, 25] - [16, 33]
+          value:
+            ImportMember: [16, 25] - [16, 33]
+              module:
+                ImportExpr: [16, 10] - [16, 17]
+                  level: 2
+                  name: 'k1.k2'
+                  top: False
+              name: 'l1'
+          asname:
+            Name: [16, 31] - [16, 33]
+              variable: Variable('l2', None)
+              ctx: Store
+        alias: [16, 35] - [16, 37]
+          value:
+            ImportMember: [16, 35] - [16, 37]
+              module:
+                ImportExpr: [16, 10] - [16, 17]
+                  level: 2
+                  name: 'k1.k2'
+                  top: False
+              name: 'l3'
+          asname:
+            Name: [16, 35] - [16, 37]
+              variable: Variable('l3', None)
+              ctx: Store
+      ]
+    Import: [18, 0] - [18, 20]
+      is_lazy: True
+      names: [
+        alias: [18, 19] - [18, 20]
+          value:
+            ImportMember: [18, 19] - [18, 20]
+              module:
+                ImportExpr: [18, 10] - [18, 11]
+                  level: 1
+                  name: None
+                  top: False
+              name: 'm'
+          asname:
+            Name: [18, 19] - [18, 20]
+              variable: Variable('m', None)
+              ctx: Store
+      ]
+    Import: [20, 0] - [20, 22]
+      is_lazy: True
+      names: [
+        alias: [20, 21] - [20, 22]
+          value:
+            ImportMember: [20, 21] - [20, 22]
+              module:
+                ImportExpr: [20, 10] - [20, 13]
+                  level: 3
+                  name: None
+                  top: False
+              name: 'n'
+          asname:
+            Name: [20, 21] - [20, 22]
+              variable: Variable('n', None)
+              ctx: Store
+      ]
+    ImportFrom: [22, 0] - [22, 20]
+      is_lazy: True
+      module:
+        ImportExpr: [22, 10] - [22, 11]
+          level: 0
+          name: 'o'
+          top: False
+    Assign: [26, 0] - [26, 8]
+      targets: [
+        Name: [26, 0] - [26, 4]
+          variable: Variable('lazy', None)
+          ctx: Store
+      ]
+      value:
+        Num: [26, 7] - [26, 8]
+          n: 1
+          text: '1'
+    Assign: [28, 0] - [28, 11]
+      targets: [
+        Subscript: [28, 0] - [28, 7]
+          value:
+            Name: [28, 0] - [28, 4]
+              variable: Variable('lazy', None)
+              ctx: Load
+          index:
+            Num: [28, 5] - [28, 6]
+              n: 2
+              text: '2'
+          ctx: Store
+      ]
+      value:
+        Num: [28, 10] - [28, 11]
+          n: 3
+          text: '3'
+    Assign: [30, 0] - [30, 12]
+      targets: [
+        Attribute: [30, 0] - [30, 8]
+          value:
+            Name: [30, 0] - [30, 4]
+              variable: Variable('lazy', None)
+              ctx: Load
+          attr: 'foo'
+          ctx: Store
+      ]
+      value:
+        Num: [30, 11] - [30, 12]
+          n: 4
+          text: '4'
+    Expr: [32, 0] - [32, 6]
+      value:
+        Call: [32, 0] - [32, 6]
+          func:
+            Name: [32, 0] - [32, 4]
+              variable: Variable('lazy', None)
+              ctx: Load
+          positional_args: []
+          named_args: []
+    AnnAssign: [34, 0] - [34, 14]
+      value: None
+      annotation:
+        Name: [34, 10] - [34, 14]
+          variable: Variable('case', None)
+          ctx: Load
+      target:
+        Subscript: [34, 0] - [34, 7]
+          value:
+            Name: [34, 0] - [34, 4]
+              variable: Variable('lazy', None)
+              ctx: Load
+          index:
+            Num: [34, 5] - [34, 6]
+              n: 5
+              text: '5'
+          ctx: Store
+  ]

python/extractor/tests/parser/lazy_imports_new.py

@@ -0,0 +1,34 @@
+# Basic lazy imports (PEP 810)
+lazy import a
+
+lazy import b1, b2
+
+lazy import c1.c2.c3
+
+lazy import d1.d2 as d3
+
+lazy from e import f
+
+lazy from g1.g2 import h1, h2
+
+lazy from i1 import j1 as j2, j3
+
+lazy from ..k1.k2 import l1 as l2, l3
+
+lazy from . import m
+
+lazy from ... import n
+
+lazy from o import *
+
+
+# `lazy` used as a regular identifier (soft keyword behavior)
+lazy = 1
+
+lazy[2] = 3
+
+lazy.foo = 4
+
+lazy()
+
+lazy[5] : case

python/extractor/tsg-python/python.tsg

@@ -1777,6 +1777,13 @@
 
     attr (@importfrom.importexpr) level = level
 }
+; Set is_lazy for lazy import statements (PEP 810)
+[
+    (import_statement is_lazy: _)
+    (import_from_statement is_lazy: _)
+] @lazy_import
+{ attr (@lazy_import.node) is_lazy = #true }
+
 ;;;;;; End of Import (`from ... import ...`)
 
 ;;;;;; Raise (`raise ...`)

python/extractor/tsg-python/tsp/grammar.js

@@ -109,6 +109,7 @@ module.exports = grammar({
     ),
 
     import_statement: $ => seq(
+      optional(field('is_lazy', 'lazy')),
       'import',
       $._import_list
     ),
@@ -131,6 +132,7 @@ module.exports = grammar({
     ),
 
     import_from_statement: $ => seq(
+      optional(field('is_lazy', 'lazy')),
       'from',
       field('module_name', choice(
         $.relative_import,
@@ -1228,6 +1230,7 @@ module.exports = grammar({
         'await',
         'match',
         'type',
+        'lazy',
       ),
       $.identifier
     )),

python/extractor/tsg-python/tsp/src/grammar.json

@@ -144,6 +144,22 @@
     "import_statement": {
       "type": "SEQ",
       "members": [
+        {
+          "type": "CHOICE",
+          "members": [
+            {
+              "type": "FIELD",
+              "name": "is_lazy",
+              "content": {
+                "type": "STRING",
+                "value": "lazy"
+              }
+            },
+            {
+              "type": "BLANK"
+            }
+          ]
+        },
         {
           "type": "STRING",
           "value": "import"
@@ -232,6 +248,22 @@
     "import_from_statement": {
       "type": "SEQ",
       "members": [
+        {
+          "type": "CHOICE",
+          "members": [
+            {
+              "type": "FIELD",
+              "name": "is_lazy",
+              "content": {
+                "type": "STRING",
+                "value": "lazy"
+              }
+            },
+            {
+              "type": "BLANK"
+            }
+          ]
+        },
         {
           "type": "STRING",
           "value": "from"
@@ -6721,6 +6753,10 @@
             {
               "type": "STRING",
               "value": "type"
+            },
+            {
+              "type": "STRING",
+              "value": "lazy"
             }
           ]
         },

python/extractor/tsg-python/tsp/src/node-types.json

@@ -1811,6 +1811,16 @@
     "type": "import_from_statement",
     "named": true,
     "fields": {
+      "is_lazy": {
+        "multiple": false,
+        "required": false,
+        "types": [
+          {
+            "type": "lazy",
+            "named": false
+          }
+        ]
+      },
       "module_name": {
         "multiple": false,
         "required": true,
@@ -1860,6 +1870,16 @@
     "type": "import_statement",
     "named": true,
     "fields": {
+      "is_lazy": {
+        "multiple": false,
+        "required": false,
+        "types": [
+          {
+            "type": "lazy",
+            "named": false
+          }
+        ]
+      },
       "name": {
         "multiple": true,
         "required": true,
@@ -4154,6 +4174,10 @@
     "type": "lambda",
     "named": false
   },
+  {
+    "type": "lazy",
+    "named": false
+  },
   {
     "type": "match",
     "named": false

python/extractor/tsg-python/tsp/src/tree_sitter/array.h

@@ -60,7 +60,13 @@ extern "C" {
 
 /// Free any memory allocated for this array. Note that this does not free any
 /// memory allocated for the array's contents.
-#define array_delete(self) _array__delete((self), (void *)(self)->contents, sizeof(*self))
+#define array_delete(self)                           \
+  do {                                               \
+    if ((self)->contents) ts_free((self)->contents); \
+    (self)->contents = NULL;                         \
+    (self)->size = 0;                                \
+    (self)->capacity = 0;                            \
+  } while (0)
 
 /// Push a new `element` onto the end of the array.
 #define array_push(self, element)                                 \
@@ -130,12 +136,11 @@ extern "C" {
 /// Swap one array with another
 #define array_swap(self, other)                                     \
   do {                                                              \
-    struct Swap swapped_contents = _array__swap(                    \
-      (void *)(self)->contents, &(self)->size, &(self)->capacity,   \
-      (void *)(other)->contents, &(other)->size, &(other)->capacity \
-    );                                                              \
-    (self)->contents = swapped_contents.self_contents;              \
-    (other)->contents = swapped_contents.other_contents;            \
+    void *_array_swap_tmp = (void *)(self)->contents;               \
+    (self)->contents = (other)->contents;                           \
+    (other)->contents = _array_swap_tmp;                            \
+    _array__swap(&(self)->size, &(self)->capacity,                  \
+                 &(other)->size, &(other)->capacity);               \
   } while (0)
 
 /// Get the size of the array contents
@@ -188,12 +193,6 @@ extern "C" {
 // The `Array` type itself was not altered as a solution in order to avoid breakage
 // with existing consumers (in particular, parsers with external scanners).
 
-/// This is not what you're looking for, see `array_delete`.
-static inline void _array__delete(void *self, void *contents, size_t self_size) {
-  if (contents) ts_free(contents);
-  if (self) memset(self, 0, self_size);
-}
-
 /// This is not what you're looking for, see `array_erase`.
 static inline void _array__erase(void* self_contents, uint32_t *size,
                                 size_t element_size, uint32_t index) {
@@ -228,31 +227,15 @@ static inline void *_array__assign(void* self_contents, uint32_t *self_size, uin
   return new_contents;
 }
 
-struct Swap {
-  void *self_contents;
-  void *other_contents;
-};
-
 /// This is not what you're looking for, see `array_swap`.
-// static inline void _array__swap(Array *self, Array *other) {
-static inline struct Swap _array__swap(void *self_contents, uint32_t *self_size, uint32_t *self_capacity,
-                               void *other_contents, uint32_t *other_size, uint32_t *other_capacity) {
-  void *new_self_contents = other_contents;
-  uint32_t new_self_size = *other_size;
-  uint32_t new_self_capacity = *other_capacity;
-
-  void *new_other_contents = self_contents;
-  *other_size = *self_size;
-  *other_capacity = *self_capacity;
-
-  *self_size = new_self_size;
-  *self_capacity = new_self_capacity;
-
-  struct Swap out = {
-    .self_contents = new_self_contents,
-    .other_contents = new_other_contents,
-  };
-  return out;
+static inline void _array__swap(uint32_t *self_size, uint32_t *self_capacity,
+                               uint32_t *other_size, uint32_t *other_capacity) {
+  uint32_t tmp_size = *self_size;
+  uint32_t tmp_capacity = *self_capacity;
+  *self_size = *other_size;
+  *self_capacity = *other_capacity;
+  *other_size = tmp_size;
+  *other_capacity = tmp_capacity;
 }
 
 /// This is not what you're looking for, see `array_push` or `array_grow_by`.

python/ql/lib/change-notes/2026-04-10-support-lazy-keyword.md

@@ -0,0 +1,5 @@
+---
+category: minorAnalysis
+---
+
+- The Python extractor now supports the new `lazy import ...` and `lazy from ... import ...` (as defined in [PEP-810](https://peps.python.org/pep-0810/)) that will be part of Python 3.15.

python/ql/lib/semmle/python/AstGenerated.qll

@@ -698,6 +698,9 @@ class Import_ extends @py_Import, Stmt {
   /** Gets an alias of this import statement. */
   Alias getAName() { result = this.getNames().getAnItem() }
 
+  /** Whether the lazy property of this import statement is true. */
+  predicate isLazy() { py_bools(this, 2) }
+
   override string toString() { result = "Import" }
 }
 
@@ -720,6 +723,9 @@ class ImportStar_ extends @py_ImportStar, Stmt {
   /** Gets the module of this import * statement. */
   Expr getModule() { py_exprs(result, _, this, 1) }
 
+  /** Whether the lazy property of this import * statement is true. */
+  predicate isLazy() { py_bools(this, 2) }
+
   override string toString() { result = "ImportStar" }
 }
 

python/ql/lib/semmlecode.python.dbscheme

@@ -517,6 +517,7 @@ py_extracted_version(int module : @py_Module ref,
 
 /* <Field> Import.location = 0, location */
 /* <Field> Import.names = 1, alias_list */
+/* <Field> Import.is_lazy = 2, bool */
 
 /* <Field> ImportExpr.location = 0, location */
 /* <Field> ImportExpr.parenthesised = 1, bool */
@@ -526,6 +527,7 @@ py_extracted_version(int module : @py_Module ref,
 
 /* <Field> ImportStar.location = 0, location */
 /* <Field> ImportStar.module = 1, expr */
+/* <Field> ImportStar.is_lazy = 2, bool */
 
 /* <Field> ImportMember.location = 0, location */
 /* <Field> ImportMember.parenthesised = 1, bool */
@@ -1127,7 +1129,7 @@ case @py_unaryop.kind of
 
 @py_ast_node = @py_Class | @py_Function | @py_Module | @py_StringPart | @py_comprehension | @py_dict_item | @py_expr | @py_pattern | @py_stmt | @py_type_parameter;
 
-@py_bool_parent = @py_For | @py_Function | @py_Print | @py_With | @py_expr | @py_pattern;
+@py_bool_parent = @py_For | @py_Function | @py_Import | @py_ImportStar | @py_Print | @py_With | @py_expr | @py_pattern;
 
 @py_dict_item_list_parent = @py_Call | @py_ClassExpr | @py_Dict;
 

python/ql/lib/upgrades/279cbb08d387ecd57ac177e87c94cfd5ca62f792/upgrade.properties

@@ -0,0 +1,2 @@
+description: Add is_lazy field to Import and ImportStar for PEP 810 lazy imports
+compatibility: backwards

python/ql/test/3/extractor-tests/lazy-imports/test.expected

@@ -0,0 +1,9 @@
+| 2 | Import | lazy |
+| 3 | Import | lazy |
+| 4 | Import | lazy |
+| 5 | Import | lazy |
+| 6 | Import | lazy |
+| 7 | from l import * | lazy |
+| 10 | Import | normal |
+| 11 | Import | normal |
+| 12 | from w import * | normal |

python/ql/test/3/extractor-tests/lazy-imports/test.py

@@ -0,0 +1,12 @@
+# Lazy imports (PEP 810)
+lazy import a
+lazy from b import c
+lazy from d import e as f
+lazy import g.h as i
+lazy from ..j import k
+lazy from l import *
+
+# Non-lazy imports
+import x
+from y import z
+from w import *

python/ql/test/3/extractor-tests/lazy-imports/test.ql

@@ -0,0 +1,11 @@
+import python
+
+string lazy(Stmt s) {
+  if s.(Import).isLazy() or s.(ImportStar).isLazy() then result = "lazy" else result = "normal"
+}
+
+from Stmt s
+where
+  s.getLocation().getFile().getShortName() = "test.py" and
+  (s instanceof Import or s instanceof ImportStar)
+select s.getLocation().getStartLine(), s.toString(), lazy(s)