Merge branch 'main' into azure_python_sdk_url_summary_upstream

python/codeql-extractor.yml

@@ -2,6 +2,7 @@ name: "python"
 display_name: "Python"
 version: 1.22.1
 column_kind: utf32
+overlay_support_version: 20250626
 build_modes:
   - none
 default_queries:

python/downgrades/279cbb08d387ecd57ac177e87c94cfd5ca62f792/upgrade.properties

@@ -0,0 +1,2 @@
+description: Sections for databaseMetadata and overlayChangedFiles
+compatibility: full

python/downgrades/6a1f497168da2f43828161d3c86db7d4c94c2b53/upgrade.properties

@@ -0,0 +1,4 @@
+description: Add databaseMetadata and overlayChangedFiles relations
+compatibility: full
+databaseMetadata.rel: delete
+overlayChangedFiles.rel: delete

python/downgrades/8d257a4a9bc78e39856d6cd33499389fc5148d4f/py_exprs.ql

@@ -0,0 +1,21 @@
+// We must wrap the DB types, as these cannot appear in argument lists
+class Expr_ extends @py_expr {
+  string toString() { result = "Expr" }
+}
+
+class ExprParent_ extends @py_expr_parent {
+  string toString() { result = "ExprList" }
+}
+
+query predicate py_exprs_without_template_strings(Expr_ id, int kind, ExprParent_ parent, int idx) {
+  py_exprs(id, kind, parent, idx) and
+  // From the dbscheme:
+  //
+  // case @py_expr.kind of
+  // ...
+  // |   39 = @py_SpecialOperation
+  // |   40 = @py_TemplateString
+  // |   41 = @py_JoinedTemplateString
+  // |   42 = @py_TemplateStringPart;
+  not kind in [40, 41, 42]
+}

python/downgrades/8d257a4a9bc78e39856d6cd33499389fc5148d4f/upgrade.properties

@@ -0,0 +1,4 @@
+description: Remove support for template string literals
+compatibility: backwards
+py_TemplateString_lists.rel: delete
+py_exprs.rel: run py_exprs.qlo py_exprs_without_template_strings

python/downgrades/acf8d3b08ae3cfac8833d16efbfa5a10fef86819/upgrade.properties

@@ -0,0 +1,2 @@
+description: Add @top type
+compatibility: full

python/extractor/imp.py

@@ -17,9 +17,16 @@ except ImportError:
     # Platform doesn't support dynamic loading.
     create_dynamic = None
 
-from importlib._bootstrap import _ERR_MSG, _exec, _load, _builtin_from_name
+from importlib._bootstrap import _exec, _load, _builtin_from_name
 from importlib._bootstrap_external import SourcelessFileLoader
 
+# In Python 3.14, `_ERR_MSG` was removed in favor of `_ERR_MSG_PREFIX`.
+try:
+    from importlib._bootstrap import _ERR_MSG
+except ImportError:
+    from importlib._bootstrap import _ERR_MSG_PREFIX
+    _ERR_MSG = _ERR_MSG_PREFIX + '{!r}'
+
 from importlib import machinery
 from importlib import util
 import importlib

python/extractor/semmle/dbscheme.template

@@ -40,6 +40,28 @@ externalData(
   string value : string ref
 );
 
+/*- Database metadata -*/
+
+/**
+ * The CLI will automatically emit applicable tuples for this table,
+ * such as `databaseMetadata("isOverlay", "true")` when building an
+ * overlay database.
+ */
+databaseMetadata(
+  string metadataKey: string ref,
+  string value: string ref
+);
+
+/*- Overlay support -*/
+
+/**
+ * The CLI will automatically emit tuples for each new/modified/deleted file
+ * when building an overlay database.
+ */
+overlayChangedFiles(
+  string path: string ref
+);
+
 /*- DEPRECATED: Snapshot date -*/
 
 snapshotDate(unique date snapshotDate : date ref);
@@ -420,3 +442,12 @@ py_decorated_object(int object : @py_object ref,
 @py_object = @py_cobject | @py_flow_node;
 
 @py_source_element = @py_ast_node | @container;
+
+/** The union of all Python database entities */
+@top =
+  @py_source_element | @py_object | @py_base_var | @location | @py_line | @py_comment |
+      @py_expr_parent | @py_expr_context |
+      @py_operator | @py_boolop | @py_cmpop | @py_unaryop |
+      @py_cmpop_list | @py_alias_list | @py_StringPart_list | @py_comprehension_list | @py_dict_item_list | @py_pattern_list | @py_stmt_list | @py_str_list | @py_type_parameter_list |
+      @externalDefect | @externalMetric | @externalDataElement | @duplication_or_similarity | @svnentry |
+      @xmllocatable | @yaml_locatable;

python/extractor/semmle/dbscheme_gen.py

@@ -62,6 +62,7 @@ def write(nodes, out):
 
 HEADER = '''/*
  *      This dbscheme is auto-generated by '%s'.
+ *      Run "make dbscheme" in python/extractor/ to regenerate.
  *      WARNING: Any modifications to this file will be lost.
  *      Relations can be changed by modifying master.py or
  *      by adding rules to dbscheme.template

python/extractor/semmle/path_rename.py

@@ -25,7 +25,7 @@ def renamer_from_options_and_env(options, logger):
         except (AttributeError, ImportError):
             raise SemmleError("Cannot get renamer from module " + options.renamer)
     else:
-        path_transformer = os.environ.get("SEMMLE_PATH_TRANSFORMER", None)
+        path_transformer = os.environ.get("CODEQL_PATH_TRANSFORMER", None) or os.environ.get("SEMMLE_PATH_TRANSFORMER", None)
         if path_transformer:
             logger.info("Using path transformer '%s'", path_transformer)
             rename = projectlayout.get_renamer(path_transformer)

python/extractor/semmle/projectlayout.py

@@ -12,12 +12,16 @@ import collections
 import re
 from functools import total_ordering
 import sys
+from pathlib import PureWindowsPath
+import os
 
 def get_renamer(filename):
     layout = load(filename)
     def rename(path):
         renamed = layout.artificial_path(path)
         return path if renamed is None else renamed
+    if os.name == "nt":
+        return lambda path: rename(PureWindowsPath(path).as_posix())
     return rename
 
 def load(filename):
@@ -257,7 +261,7 @@ class _Rewrite(object):
             exclude = path
             self._line = line;
             self._original = u'-' + exclude;
-            if not exclude.startswith(u"/"):
+            if os.name != 'nt' and not exclude.startswith(u"/"):
                 exclude = u'/' + exclude
             if exclude.find(u"//") != -1:
                 raise _error(u"Illegal '//' in exclude path", line)
@@ -274,14 +278,14 @@ class _Rewrite(object):
             include = path
             self._line = line;
             self._original = include;
-            if not include.startswith(u"/"):
+            if os.name != 'nt' and not include.startswith(u"/"):
                 include = u'/' + include
             doubleslash = include.find(u"//")
             if doubleslash != include.find(u"//"):
                 raise _error(u"More than one '//' in include path (project-layout)", line)
             if self._verify_stars.match(include):
                 raise _error(u"Illegal use of '**' in include path (project-layout)", line)
-            if not virtual.startswith(u"/"):
+            if os.name != 'nt' and not virtual.startswith(u"/"):
                 virtual = u"/" + virtual
             if virtual.endswith(u"/"):
                 virtual = virtual[0 : -1]

python/extractor/semmle/python/ast.py

@@ -56,6 +56,15 @@ class StringPart(AstBase):
         self.text = text
         self.s = s
 
+class TemplateStringPart(AstBase):
+    '''A string constituent of a template string literal'''
+
+    __slots__ = "text", "s",
+
+    def __init__(self, text, s):
+        self.text = text
+        self.s = s
+
 class alias(AstBase):
     __slots__ = "value", "asname",
 
@@ -356,6 +365,19 @@ class JoinedStr(expr):
     def __init__(self, values):
         self.values = values
 
+class TemplateString(expr):
+    __slots__ = "prefix", "values",
+
+    def __init__(self, prefix, values):
+        self.prefix = prefix
+        self.values = values
+
+class JoinedTemplateString(expr):
+    __slots__ = "strings",
+
+    def __init__(self, strings):
+        self.strings = strings
+
 
 class Lambda(expr):
     __slots__ = "args", "inner_scope",

python/extractor/semmle/python/master.py

@@ -186,12 +186,20 @@ FormattedStringLiteral.set_name("Fstring")
 
 FormattedValue = ClassNode("FormattedValue", expr, descriptive_name='formatted value')
 
+
 AnnAssign = ClassNode("AnnAssign", stmt, descriptive_name='annotated assignment')
 
 AssignExpr = ClassNode('AssignExpr', expr, "assignment expression")
 
 SpecialOperation = ClassNode('SpecialOperation', expr, "special operation")
 
+TemplateString = ClassNode('TemplateString', expr, 'template string literal')
+
+template_string_list = ListNode(TemplateString)
+
+JoinedTemplateString = ClassNode("JoinedTemplateString", expr, descriptive_name='joined template string')
+TemplateStringPart = ClassNode('TemplateStringPart', expr, "string part of a template string")
+
 type_parameter = ClassNode('type_parameter', descriptive_name='type parameter')
 type_parameter.field('location', location)
 type_parameter_list = ListNode(type_parameter)
@@ -435,6 +443,9 @@ Subscript.field('value', expr)
 Subscript.field('index', expr)
 Subscript.field('ctx', expr_context, 'context')
 
+TemplateString.field('prefix', string, 'prefix')
+TemplateString.field('values', expr_list, 'values')
+
 Try.field('body', stmt_list)
 Try.field('orelse', stmt_list, 'else block')
 Try.field('handlers', stmt_list, 'exception handlers')
@@ -484,10 +495,15 @@ PlaceHolder.field('ctx', expr_context, 'context')
 StringPart.field('text', string)
 StringPart.field('location', location)
 
+TemplateStringPart.field('text', string)
+
+
 Await.field('value', expr, 'expression waited upon')
 
 FormattedStringLiteral.field('values', expr_list)
 
+JoinedTemplateString.field('strings', template_string_list)
+
 FormattedValue.field('value', expr, "expression to be formatted")
 FormattedValue.field('conversion', string, 'type conversion')
 FormattedValue.field('format_spec', FormattedStringLiteral, 'format specifier')

python/extractor/semmle/python/parser/tsg_parser.py

@@ -273,6 +273,8 @@ list_fields = {
     ast.Print: ("values",),
     ast.Set: ("elts",),
     ast.Str: ("implicitly_concatenated_parts",),
+    ast.TemplateString: ("values",),
+    ast.JoinedTemplateString: ("strings",),
     ast.TypeAlias: ("type_parameters",),
     ast.Try: ("body", "handlers", "orelse", "finalbody"),
     ast.Tuple: ("elts",),

python/extractor/semmle/util.py

@@ -10,7 +10,7 @@ from io import BytesIO
 
 #Semantic version of extractor.
 #Update this if any changes are made
-VERSION = "7.1.4"
+VERSION = "7.1.7"
 
 PY_EXTENSIONS = ".py", ".pyw"
 

python/extractor/semmle/worker.py

@@ -11,6 +11,7 @@ from semmle.extractors import SuperExtractor, ModulePrinter, SkippedBuiltin
 from semmle.profiling import get_profiler
 from semmle.path_rename import renamer_from_options_and_env
 from semmle.logging import WARN, recursion_error_message, internal_error_message, Logger
+from semmle.util import FileExtractable, FolderExtractable
 
 class ExtractorFailure(Exception):
     'Generic exception representing the failure of an extractor.'
@@ -19,17 +20,32 @@ class ExtractorFailure(Exception):
 
 class ModuleImportGraph(object):
 
-    def __init__(self, max_depth):
+    def __init__(self, max_depth, logger: Logger):
         self.modules = {}
         self.succ = defaultdict(set)
         self.todo = set()
         self.done = set()
         self.max_depth = max_depth
+        self.logger = logger
+
+        # During overlay extraction, only traverse the files that were changed.
+        self.overlay_changes = None
+        if 'CODEQL_EXTRACTOR_PYTHON_OVERLAY_CHANGES' in os.environ:
+            overlay_changes_file = os.environ['CODEQL_EXTRACTOR_PYTHON_OVERLAY_CHANGES']
+            logger.info("Overlay extraction mode: only extracting files changed according to '%s'", overlay_changes_file)
+            try:
+                with open(overlay_changes_file, 'r', encoding='utf-8') as f:
+                    data = json.load(f)
+                    changed_paths = data.get('changes', [])
+                    self.overlay_changes = { os.path.abspath(p) for p in changed_paths }
+            except (IOError, ValueError) as e:
+                logger.warn("Failed to read overlay changes from '%s' (falling back to full extraction): %s", overlay_changes_file, e)
+                self.overlay_changes = None
 
     def add_root(self, mod):
         self.modules[mod] = 0
         if mod not in self.done:
-            self.todo.add(mod)
+            self.add_todo(mod)
 
     def add_import(self, mod, imported):
         assert mod in self.modules
@@ -39,7 +55,7 @@ class ModuleImportGraph(object):
                 self._reduce_depth(imported, self.modules[mod] + 1)
         else:
             if self.modules[mod] < self.max_depth and imported not in self.done:
-                self.todo.add(imported)
+                self.add_todo(imported)
             self.modules[imported] = self.modules[mod] + 1
 
     def _reduce_depth(self, mod, depth):
@@ -48,7 +64,7 @@ class ModuleImportGraph(object):
         if depth > self.max_depth:
             return
         if mod not in self.done:
-            self.todo.add(mod)
+            self.add_todo(mod)
         self.modules[mod] = depth
         for imp in self.succ[mod]:
             self._reduce_depth(imp, depth+1)
@@ -61,11 +77,25 @@ class ModuleImportGraph(object):
 
     def push_back(self, mod):
         self.done.remove(mod)
-        self.todo.add(mod)
+        self.add_todo(mod)
 
     def empty(self):
         return not self.todo
 
+    def add_todo(self, mod):
+        if not self._module_in_overlay_changes(mod):
+            self.logger.debug("Skipping module '%s' as it was not changed in overlay extraction.", mod)
+            return
+        self.todo.add(mod)
+
+    def _module_in_overlay_changes(self, mod):
+        if self.overlay_changes is not None:
+            if isinstance(mod, FileExtractable):
+                return mod.path in self.overlay_changes
+            if isinstance(mod, FolderExtractable):
+                return mod.path + '/__init__.py' in self.overlay_changes
+        return True
+
 class ExtractorPool(object):
     '''Pool of worker processes running extractors'''
 
@@ -90,7 +120,7 @@ class ExtractorPool(object):
         self.enqueued = set()
         self.done = set()
         self.requirements = {}
-        self.import_graph = ModuleImportGraph(options.max_import_depth)
+        self.import_graph = ModuleImportGraph(options.max_import_depth, logger)
         logger.debug("Source archive: %s", archive)
         self.logger = logger
         DiagnosticsWriter.create_output_dir()
@@ -162,6 +192,10 @@ class ExtractorPool(object):
             self.module_queue.put(None)
         for p in self.procs:
             p.join()
+        if 'CODEQL_EXTRACTOR_PYTHON_OVERLAY_BASE_METADATA_OUT' in os.environ:
+            with open(os.environ['CODEQL_EXTRACTOR_PYTHON_OVERLAY_BASE_METADATA_OUT'], 'w', encoding='utf-8') as f:
+                metadata = {}
+                json.dump(metadata, f)
         self.logger.info("Processed %d modules in %0.2fs", len(self.import_graph.done), time.time() - self.start_time)
 
     def stop(self, timeout=2.0):

python/extractor/tests/parser/exception_groups_new.expected

@@ -1,4 +1,4 @@
-Module: [1, 0] - [27, 0]
+Module: [1, 0] - [32, 0]
   body: [
     Try: [1, 0] - [1, 4]
       body: [
@@ -153,4 +153,28 @@ Module: [1, 0] - [27, 0]
           ]
       ]
       finalbody: []
+    Try: [28, 0] - [28, 4]
+      body: [
+        Pass: [29, 4] - [29, 8]
+      ]
+      orelse: []
+      handlers: [
+        ExceptGroupStmt: [30, 0] - [31, 8]
+          type:
+            Tuple: [30, 8] - [30, 12]
+              elts: [
+                Name: [30, 8] - [30, 9]
+                  variable: Variable('x', None)
+                  ctx: Load
+                Name: [30, 11] - [30, 12]
+                  variable: Variable('y', None)
+                  ctx: Load
+              ]
+              ctx: Load
+          name: None
+          body: [
+            Pass: [31, 4] - [31, 8]
+          ]
+      ]
+      finalbody: []
   ]

python/extractor/tests/parser/exception_groups_new.py

@@ -24,3 +24,8 @@ try:
     pass
 except *foo as e:
     pass
+
+try:
+    pass
+except* x, y:
+    pass

python/extractor/tests/parser/exceptions_new.expected

@@ -0,0 +1,64 @@
+Module: [1, 0] - [9, 0]
+  body: [
+    Try: [1, 0] - [1, 4]
+      body: [
+        Pass: [2, 4] - [2, 8]
+      ]
+      orelse: []
+      handlers: [
+        ExceptStmt: [3, 0] - [3, 12]
+          type:
+            Tuple: [3, 7] - [3, 11]
+              elts: [
+                Name: [3, 7] - [3, 8]
+                  variable: Variable('a', None)
+                  ctx: Load
+                Name: [3, 10] - [3, 11]
+                  variable: Variable('b', None)
+                  ctx: Load
+              ]
+              ctx: Load
+          name: None
+          body: [
+            Pass: [4, 4] - [4, 8]
+          ]
+        ExceptStmt: [5, 0] - [5, 14]
+          type:
+            Tuple: [5, 8] - [5, 12]
+              elts: [
+                Name: [5, 8] - [5, 9]
+                  variable: Variable('c', None)
+                  ctx: Load
+                Name: [5, 11] - [5, 12]
+                  variable: Variable('d', None)
+                  ctx: Load
+              ]
+              ctx: Load
+              parenthesised: True
+          name: None
+          body: [
+            Pass: [6, 4] - [6, 8]
+          ]
+        ExceptStmt: [7, 0] - [7, 19]
+          type:
+            Tuple: [7, 8] - [7, 12]
+              elts: [
+                Name: [7, 8] - [7, 9]
+                  variable: Variable('e', None)
+                  ctx: Load
+                Name: [7, 11] - [7, 12]
+                  variable: Variable('f', None)
+                  ctx: Load
+              ]
+              ctx: Load
+              parenthesised: True
+          name:
+            Name: [7, 17] - [7, 18]
+              variable: Variable('g', None)
+              ctx: Store
+          body: [
+            Pass: [8, 4] - [8, 8]
+          ]
+      ]
+      finalbody: []
+  ]

python/extractor/tests/parser/exceptions_new.py

@@ -0,0 +1,8 @@
+try:
+    pass
+except a, b: # new, relaxed syntax
+    pass
+except (c, d): # old syntax
+    pass
+except (e, f) as g: # old syntax
+    pass

python/extractor/tests/parser/template_strings_new.expected

@@ -0,0 +1,194 @@
+Module: [1, 0] - [18, 0]
+  body: [
+    Assign: [1, 0] - [1, 14]
+      targets: [
+        Name: [1, 0] - [1, 4]
+          variable: Variable('name', None)
+          ctx: Store
+      ]
+      value:
+        Str: [1, 7] - [1, 14]
+          s: 'World'
+          prefix: '"'
+          implicitly_concatenated_parts: None
+    Assign: [2, 0] - [2, 15]
+      targets: [
+        Name: [2, 0] - [2, 5]
+          variable: Variable('value', None)
+          ctx: Store
+      ]
+      value:
+        Num: [2, 8] - [2, 15]
+          n: 42.5678
+          text: '42.5678'
+    Assign: [3, 0] - [3, 15]
+      targets: [
+        Name: [3, 0] - [3, 5]
+          variable: Variable('first', None)
+          ctx: Store
+      ]
+      value:
+        Str: [3, 8] - [3, 15]
+          s: 'first'
+          prefix: '"'
+          implicitly_concatenated_parts: None
+    Assign: [4, 0] - [4, 17]
+      targets: [
+        Name: [4, 0] - [4, 6]
+          variable: Variable('second', None)
+          ctx: Store
+      ]
+      value:
+        Str: [4, 9] - [4, 17]
+          s: 'second'
+          prefix: '"'
+          implicitly_concatenated_parts: None
+    If: [6, 0] - [6, 5]
+      test:
+        Num: [6, 3] - [6, 4]
+          n: 1
+          text: '1'
+      body: [
+        Expr: [7, 4] - [7, 7]
+          value:
+            TemplateString: [7, 4] - [7, 7]
+              prefix: 't"'
+              values: []
+      ]
+      orelse: None
+    If: [8, 0] - [8, 5]
+      test:
+        Num: [8, 3] - [8, 4]
+          n: 2
+          text: '2'
+      body: [
+        Expr: [9, 4] - [9, 21]
+          value:
+            TemplateString: [9, 4] - [9, 21]
+              prefix: 't"'
+              values: [
+                TemplateStringPart: [9, 6] - [9, 13]
+                  text: '"Hello, "'
+                  s: 'Hello, '
+                Name: [9, 14] - [9, 18]
+                  variable: Variable('name', None)
+                  ctx: Load
+                TemplateStringPart: [9, 19] - [9, 20]
+                  text: '"!"'
+                  s: '!'
+              ]
+      ]
+      orelse: None
+    If: [10, 0] - [10, 5]
+      test:
+        Num: [10, 3] - [10, 4]
+          n: 3
+          text: '3'
+      body: [
+        Expr: [11, 4] - [11, 42]
+          value:
+            TemplateString: [11, 4] - [11, 42]
+              prefix: 't"'
+              values: [
+                TemplateStringPart: [11, 6] - [11, 13]
+                  text: '"Value: "'
+                  s: 'Value: '
+                Name: [11, 14] - [11, 19]
+                  variable: Variable('value', None)
+                  ctx: Load
+                TemplateStringPart: [11, 24] - [11, 31]
+                  text: '", Hex: "'
+                  s: ', Hex: '
+                Name: [11, 32] - [11, 37]
+                  variable: Variable('value', None)
+                  ctx: Load
+              ]
+      ]
+      orelse: None
+    If: [12, 0] - [12, 5]
+      test:
+        Num: [12, 3] - [12, 4]
+          n: 4
+          text: '4'
+      body: [
+        Expr: [13, 4] - [13, 29]
+          value:
+            TemplateString: [13, 4] - [13, 29]
+              prefix: 't"'
+              values: [
+                TemplateStringPart: [13, 6] - [13, 28]
+                  text: '"Just a regular string."'
+                  s: 'Just a regular string.'
+              ]
+      ]
+      orelse: None
+    If: [14, 0] - [14, 5]
+      test:
+        Num: [14, 3] - [14, 4]
+          n: 5
+          text: '5'
+      body: [
+        Expr: [15, 4] - [15, 50]
+          value:
+            TemplateString: [15, 4] - [15, 50]
+              prefix: 't"'
+              values: [
+                TemplateStringPart: [15, 6] - [15, 15]
+                  text: '"Multiple "'
+                  s: 'Multiple '
+                Name: [15, 16] - [15, 21]
+                  variable: Variable('first', None)
+                  ctx: Load
+                TemplateStringPart: [15, 22] - [15, 27]
+                  text: '" and "'
+                  s: ' and '
+                Name: [15, 28] - [15, 34]
+                  variable: Variable('second', None)
+                  ctx: Load
+                TemplateStringPart: [15, 35] - [15, 49]
+                  text: '" placeholders."'
+                  s: ' placeholders.'
+              ]
+      ]
+      orelse: None
+    If: [16, 0] - [16, 5]
+      test:
+        Num: [16, 3] - [16, 4]
+          n: 6
+          text: '6'
+      body: [
+        Expr: [17, 4] - [17, 66]
+          value:
+            JoinedTemplateString: [17, 4] - [17, 66]
+              strings: [
+                TemplateString: [17, 4] - [17, 31]
+                  prefix: 't"'
+                  values: [
+                    TemplateStringPart: [17, 6] - [17, 30]
+                      text: '"Implicit concatenation: "'
+                      s: 'Implicit concatenation: '
+                  ]
+                TemplateString: [17, 32] - [17, 49]
+                  prefix: 't"'
+                  values: [
+                    TemplateStringPart: [17, 34] - [17, 41]
+                      text: '"Hello, "'
+                      s: 'Hello, '
+                    Name: [17, 42] - [17, 46]
+                      variable: Variable('name', None)
+                      ctx: Load
+                    TemplateStringPart: [17, 47] - [17, 48]
+                      text: '"!"'
+                      s: '!'
+                  ]
+                TemplateString: [17, 50] - [17, 66]
+                  prefix: 't"'
+                  values: [
+                    TemplateStringPart: [17, 52] - [17, 65]
+                      text: '" How are you?"'
+                      s: ' How are you?'
+                  ]
+              ]
+      ]
+      orelse: None
+  ]

python/extractor/tests/parser/template_strings_new.py

@@ -0,0 +1,17 @@
+name = "World"
+value = 42.5678
+first = "first"
+second = "second"
+
+if 1:
+    t""
+if 2:
+    t"Hello, {name}!"
+if 3:
+    t"Value: {value:.2f}, Hex: {value:#x}"
+if 4:
+    t"Just a regular string."
+if 5:
+    t"Multiple {first} and {second} placeholders."
+if 6:
+    t"Implicit concatenation: " t"Hello, {name}!" t" How are you?"

python/extractor/tests/test_use_projectlayout.py

@@ -19,7 +19,7 @@ class ProjectLayoutUseTest(ExtractorTest):
 
     def test_invalid_layout(self):
         try:
-            with environment("SEMMLE_PATH_TRANSFORMER", "nonsuch/project-layout"):
+            with environment("CODEQL_PATH_TRANSFORMER", "nonsuch/project-layout"):
                 self.run_extractor("-R", self.src_path)
         except subprocess.CalledProcessError as ex:
             self.assertEqual(ex.returncode, 2)

python/extractor/tsg-python/python.tsg

@@ -12,7 +12,7 @@
 (assignment !type) @assign
 { let @assign.node = (ast-node @assign "Assign") }
 
-[ (expression_list) (tuple) (tuple_pattern) (pattern_list) (index_expression_list) ] @tuple
+[ (expression_list) (tuple) (tuple_pattern) (pattern_list) (index_expression_list) (exception_list)] @tuple
 { let @tuple.node = (ast-node @tuple "Tuple") }
 
 (list_pattern) @list
@@ -117,6 +117,9 @@
 (string string_content: (_) @part)
 { let @part.node = (ast-node @part "StringPart") }
 
+(template_string string_content: (_) @part)
+{ let @part.node = (ast-node @part "TemplateStringPart") }
+
 ; A string concatenation that contains no interpolated expressions is just a `Str` (and its children
 ; will be `StringPart`s). A string concatenation that contains interpolated expressions is a
 ; `JoinedStr`, however.
@@ -142,6 +145,12 @@
     }
 }
 
+(template_string) @tstring
+{ let @tstring.node = (ast-node @tstring "TemplateString") }
+
+(concatenated_template_string) @tstrings
+{ let @tstrings.node = (ast-node @tstrings "JoinedTemplateString") }
+
 (pair) @kvpair
 { let @kvpair.node = (ast-node @kvpair "KeyValuePair") }
 
@@ -2052,6 +2061,44 @@
 
 ;;;;;; End of JoinedStr (`f"foo"`)
 
+;;;;;; JoinedTemplateString / TemplateString (`t"foo"`)
+
+; Record the prefix of the template string.
+(template_string) @tstring
+{
+    attr (@tstring.node) prefix = (string-prefix @tstring)
+}
+
+; Attach raw children (string parts and interpolations) to the template string node.
+(template_string (string_content) @part) @tmpl_any
+{
+    edge @tmpl_any.node -> @part.node
+    attr (@tmpl_any.node -> @part.node) values = (named-child-index @part)
+    attr (@part.node) ctx = "load"
+    let safe_string = (concatenate-strings (string-safe-prefix @tmpl_any) (source-text @part) (string-quotes @tmpl_any))
+    attr (@part.node) s = safe_string
+    attr (@part.node) text = safe_string
+}
+
+(template_string (interpolation expression: (_) @part) @interp) @tmpl_any
+{
+    edge @tmpl_any.node -> @part.node
+    attr (@tmpl_any.node -> @part.node) values = (named-child-index @interp)
+    attr (@part.node) ctx = "load"
+}
+
+
+; Concatenated template strings simply have a list-like field containing the template strings that
+; are concatenated together.
+(concatenated_template_string (template_string) @tstring) @tmpl_concat
+{
+    edge @tmpl_concat.node -> @tstring.node
+    attr (@tmpl_concat.node -> @tstring.node) strings = (named-child-index @tstring)
+    attr (@tstring.node) ctx = "load"
+}
+
+;;;;;; End of JoinedTemplateString / TemplateString (`t"foo"`)
+
 
 
 ;;;;;; List (`[...]`)
@@ -3398,6 +3445,9 @@
     (tuple element: (_) @elt) @parent
 
     (tuple_pattern element: (_) @elt) @parent
+
+    ; An exception list, as in `except A, B, C: ...`
+    (exception_list element: (_) @elt) @parent
 ]
 {
     edge @parent.node -> @elt.node
@@ -3433,6 +3483,7 @@
     (parenthesized_expression inner: (_) @elt)
     (set element: (_) @elt)
     (match_sequence_pattern (_) @elt)
+    (exception_list element: (_) @elt)
 ] @seq
 {
     attr (@elt.node) _inherited_ctx = @seq.node

python/extractor/tsg-python/src/main.rs

@@ -140,15 +140,22 @@ pub mod extra_functions {
         }
 
         fn safe(&self) -> Prefix {
+            // Remove format (f/F) and template (t/T) flags when generating a safe prefix.
             Prefix {
-                flags: self.flags.clone().replace("f", "").replace("F", ""),
+                flags: self
+                    .flags
+                    .clone()
+                    .replace("f", "")
+                    .replace("F", "")
+                    .replace("t", "")
+                    .replace("T", ""),
                 quotes: self.quotes.clone(),
             }
         }
     }
 
     fn get_prefix(s: &str) -> Prefix {
-        let flags_matcher = regex::Regex::new("^[bfurBFUR]{0,2}").unwrap();
+        let flags_matcher = regex::Regex::new("^[bfurtBFURT]{0,2}").unwrap();
         let mut end = 0;
         let flags = match flags_matcher.find(s) {
             Some(m) => {
@@ -170,7 +177,7 @@ pub mod extra_functions {
             quotes = "}";
         }
         Prefix {
-            flags: flags.to_lowercase().to_owned(),
+            flags: flags.to_owned(),
             quotes: quotes.to_owned(),
         }
     }
@@ -198,6 +205,12 @@ pub mod extra_functions {
         let p = get_prefix("\"\"\"\"\"\"");
         assert_eq!(p.flags, "");
         assert_eq!(p.quotes, "\"\"\"");
+        let p = get_prefix("t\"hello\"");
+        assert_eq!(p.flags, "t");
+        assert_eq!(p.quotes, "\"");
+        let p = get_prefix("Tr'world'");
+        assert_eq!(p.flags, "Tr");
+        assert_eq!(p.quotes, "'");
     }
 
     fn get_string_contents(s: String) -> String {
@@ -227,6 +240,10 @@ pub mod extra_functions {
         assert_eq!(get_string_contents(s.to_owned()), "");
         let s = "''''''";
         assert_eq!(get_string_contents(s.to_owned()), "");
+        let s = "t\"tmpl\"";
+        assert_eq!(get_string_contents(s.to_owned()), "tmpl");
+        let s = "Tr'world'";
+        assert_eq!(get_string_contents(s.to_owned()), "world");
     }
 
     pub struct StringPrefix;
@@ -291,7 +308,11 @@ pub mod extra_functions {
             let node = graph[parameters.param()?.into_syntax_node_ref()?];
             parameters.finish()?;
             let prefix = get_prefix(&source[node.byte_range()]).full();
-            let prefix = prefix.replace("f", "").replace("F", "");
+            let prefix = prefix
+                .replace("f", "")
+                .replace("F", "")
+                .replace("t", "")
+                .replace("T", "");
             Ok(Value::String(prefix))
         }
     }

python/extractor/tsg-python/tsp/.gitattributes

@@ -0,0 +1,4 @@
+# Mark tree-sitter generated files
+src/grammar.json linguist-generated
+src/node-types.json linguist-generated
+src/parser.c linguist-generated

python/extractor/tsg-python/tsp/grammar.js

@@ -55,6 +55,7 @@ module.exports = grammar({
     $._string_start,
     $._string_content,
     $._string_end,
+    $._template_string_start,
   ],
 
   inline: $ => [
@@ -296,12 +297,21 @@ module.exports = grammar({
       )
     ),
 
+    exception_list: $ => seq(
+      field('element', $.expression),
+      repeat1(
+        seq(
+          ',',
+          field('element', $.expression))
+        )
+      ),
+
     except_clause: $ => seq(
       'except',
       optional(seq(
-        field('type', $.expression),
+        field('type', choice($.expression, $.exception_list)),
         optional(seq(
-          choice('as', ','),
+          'as',
           field('alias', $.expression)
         ))
       )),
@@ -313,7 +323,7 @@ module.exports = grammar({
       'except',
       '*',
       seq(
-        field('type', $.expression),
+        field('type', choice($.expression, $.exception_list)),
         optional(seq(
           'as',
           field('alias', $.expression)
@@ -423,6 +433,8 @@ module.exports = grammar({
       ),
       $.string,
       $.concatenated_string,
+      $.template_string,
+      $.concatenated_template_string,
       $.none,
       $.true,
       $.false
@@ -765,6 +777,8 @@ module.exports = grammar({
       $.keyword_identifier,
       $.string,
       $.concatenated_string,
+      $.template_string,
+      $.concatenated_template_string,
       $.integer,
       $.float,
       $.true,
@@ -1099,6 +1113,20 @@ module.exports = grammar({
       field('suffix', alias($._string_end, '"'))
     ),
 
+    concatenated_template_string: $ => seq(
+      $.template_string,
+      repeat1($.template_string)
+    ),
+
+    template_string: $ => seq(
+      field('prefix', alias($._template_string_start, '"')),
+      repeat(choice(
+        field('interpolation', $.interpolation),
+        field('string_content', $.string_content)
+      )),
+      field('suffix', alias($._string_end, '"'))
+    ),
+
     string_content: $ => prec.right(0, repeat1(
       choice(
         $._escape_interpolation,

python/extractor/tsg-python/tsp/src/grammar.json

@@ -1087,6 +1087,39 @@
         }
       ]
     },
+    "exception_list": {
+      "type": "SEQ",
+      "members": [
+        {
+          "type": "FIELD",
+          "name": "element",
+          "content": {
+            "type": "SYMBOL",
+            "name": "expression"
+          }
+        },
+        {
+          "type": "REPEAT1",
+          "content": {
+            "type": "SEQ",
+            "members": [
+              {
+                "type": "STRING",
+                "value": ","
+              },
+              {
+                "type": "FIELD",
+                "name": "element",
+                "content": {
+                  "type": "SYMBOL",
+                  "name": "expression"
+                }
+              }
+            ]
+          }
+        }
+      ]
+    },
     "except_clause": {
       "type": "SEQ",
       "members": [
@@ -1104,8 +1137,17 @@
                   "type": "FIELD",
                   "name": "type",
                   "content": {
-                    "type": "SYMBOL",
-                    "name": "expression"
+                    "type": "CHOICE",
+                    "members": [
+                      {
+                        "type": "SYMBOL",
+                        "name": "expression"
+                      },
+                      {
+                        "type": "SYMBOL",
+                        "name": "exception_list"
+                      }
+                    ]
                   }
                 },
                 {
@@ -1115,17 +1157,8 @@
                       "type": "SEQ",
                       "members": [
                         {
-                          "type": "CHOICE",
-                          "members": [
-                            {
-                              "type": "STRING",
-                              "value": "as"
-                            },
-                            {
-                              "type": "STRING",
-                              "value": ","
-                            }
-                          ]
+                          "type": "STRING",
+                          "value": "as"
                         },
                         {
                           "type": "FIELD",
@@ -1181,8 +1214,17 @@
               "type": "FIELD",
               "name": "type",
               "content": {
-                "type": "SYMBOL",
-                "name": "expression"
+                "type": "CHOICE",
+                "members": [
+                  {
+                    "type": "SYMBOL",
+                    "name": "expression"
+                  },
+                  {
+                    "type": "SYMBOL",
+                    "name": "exception_list"
+                  }
+                ]
               }
             },
             {
@@ -1800,6 +1842,14 @@
           "type": "SYMBOL",
           "name": "concatenated_string"
         },
+        {
+          "type": "SYMBOL",
+          "name": "template_string"
+        },
+        {
+          "type": "SYMBOL",
+          "name": "concatenated_template_string"
+        },
         {
           "type": "SYMBOL",
           "name": "none"
@@ -3891,6 +3941,14 @@
           "type": "SYMBOL",
           "name": "concatenated_string"
         },
+        {
+          "type": "SYMBOL",
+          "name": "template_string"
+        },
+        {
+          "type": "SYMBOL",
+          "name": "concatenated_template_string"
+        },
         {
           "type": "SYMBOL",
           "name": "integer"
@@ -5982,6 +6040,77 @@
         }
       ]
     },
+    "concatenated_template_string": {
+      "type": "SEQ",
+      "members": [
+        {
+          "type": "SYMBOL",
+          "name": "template_string"
+        },
+        {
+          "type": "REPEAT1",
+          "content": {
+            "type": "SYMBOL",
+            "name": "template_string"
+          }
+        }
+      ]
+    },
+    "template_string": {
+      "type": "SEQ",
+      "members": [
+        {
+          "type": "FIELD",
+          "name": "prefix",
+          "content": {
+            "type": "ALIAS",
+            "content": {
+              "type": "SYMBOL",
+              "name": "_template_string_start"
+            },
+            "named": false,
+            "value": "\""
+          }
+        },
+        {
+          "type": "REPEAT",
+          "content": {
+            "type": "CHOICE",
+            "members": [
+              {
+                "type": "FIELD",
+                "name": "interpolation",
+                "content": {
+                  "type": "SYMBOL",
+                  "name": "interpolation"
+                }
+              },
+              {
+                "type": "FIELD",
+                "name": "string_content",
+                "content": {
+                  "type": "SYMBOL",
+                  "name": "string_content"
+                }
+              }
+            ]
+          }
+        },
+        {
+          "type": "FIELD",
+          "name": "suffix",
+          "content": {
+            "type": "ALIAS",
+            "content": {
+              "type": "SYMBOL",
+              "name": "_string_end"
+            },
+            "named": false,
+            "value": "\""
+          }
+        }
+      ]
+    },
     "string_content": {
       "type": "PREC_RIGHT",
       "value": 0,
@@ -6710,6 +6839,10 @@
     {
       "type": "SYMBOL",
       "name": "_string_end"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "_template_string_start"
     }
   ],
   "inline": [

python/extractor/tsg-python/tsp/src/node-types.json

@@ -241,6 +241,10 @@
         "type": "concatenated_string",
         "named": true
       },
+      {
+        "type": "concatenated_template_string",
+        "named": true
+      },
       {
         "type": "dictionary",
         "named": true
@@ -305,6 +309,10 @@
         "type": "subscript",
         "named": true
       },
+      {
+        "type": "template_string",
+        "named": true
+      },
       {
         "type": "true",
         "named": true
@@ -1000,6 +1008,21 @@
       ]
     }
   },
+  {
+    "type": "concatenated_template_string",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": true,
+      "required": true,
+      "types": [
+        {
+          "type": "template_string",
+          "named": true
+        }
+      ]
+    }
+  },
   {
     "type": "conditional_expression",
     "named": true,
@@ -1286,6 +1309,10 @@
         "multiple": false,
         "required": false,
         "types": [
+          {
+            "type": "exception_list",
+            "named": true
+          },
           {
             "type": "expression",
             "named": true
@@ -1321,6 +1348,26 @@
       "type": {
         "multiple": false,
         "required": true,
+        "types": [
+          {
+            "type": "exception_list",
+            "named": true
+          },
+          {
+            "type": "expression",
+            "named": true
+          }
+        ]
+      }
+    }
+  },
+  {
+    "type": "exception_list",
+    "named": true,
+    "fields": {
+      "element": {
+        "multiple": true,
+        "required": true,
         "types": [
           {
             "type": "expression",
@@ -2460,6 +2507,10 @@
           "type": "concatenated_string",
           "named": true
         },
+        {
+          "type": "concatenated_template_string",
+          "named": true
+        },
         {
           "type": "false",
           "named": true
@@ -2472,6 +2523,10 @@
           "type": "string",
           "named": true
         },
+        {
+          "type": "template_string",
+          "named": true
+        },
         {
           "type": "true",
           "named": true
@@ -3257,6 +3312,52 @@
       }
     }
   },
+  {
+    "type": "template_string",
+    "named": true,
+    "fields": {
+      "interpolation": {
+        "multiple": true,
+        "required": false,
+        "types": [
+          {
+            "type": "interpolation",
+            "named": true
+          }
+        ]
+      },
+      "prefix": {
+        "multiple": false,
+        "required": true,
+        "types": [
+          {
+            "type": "\"",
+            "named": false
+          }
+        ]
+      },
+      "string_content": {
+        "multiple": true,
+        "required": false,
+        "types": [
+          {
+            "type": "string_content",
+            "named": true
+          }
+        ]
+      },
+      "suffix": {
+        "multiple": false,
+        "required": true,
+        "types": [
+          {
+            "type": "\"",
+            "named": false
+          }
+        ]
+      }
+    }
+  },
   {
     "type": "try_statement",
     "named": true,

python/extractor/tsg-python/tsp/src/scanner.cc

@@ -17,6 +17,7 @@ enum TokenType {
   STRING_START,
   STRING_CONTENT,
   STRING_END,
+  TEMPLATE_STRING_START,
 };
 
 struct Delimiter {
@@ -28,6 +29,7 @@ struct Delimiter {
     Format = 1 << 4,
     Triple = 1 << 5,
     Bytes = 1 << 6,
+    Template = 1 << 7,
   };
 
   Delimiter() : flags(0) {}
@@ -36,6 +38,14 @@ struct Delimiter {
     return flags & Format;
   }
 
+  bool is_template() const {
+    return flags & Template;
+  }
+
+  bool can_interpolate() const {
+    return is_format() || is_template();
+  }
+
   bool is_raw() const {
     return flags & Raw;
   }
@@ -59,6 +69,10 @@ struct Delimiter {
     flags |= Format;
   }
 
+  void set_template() {
+    flags |= Template;
+  }
+
   void set_raw() {
     flags |= Raw;
   }
@@ -154,7 +168,7 @@ struct Scanner {
       int32_t end_character = delimiter.end_character();
       bool has_content = false;
       while (lexer->lookahead) {
-        if ((lexer->lookahead == '{' || lexer->lookahead == '}') && delimiter.is_format()) {
+        if ((lexer->lookahead == '{' || lexer->lookahead == '}') && delimiter.can_interpolate()) {
           lexer->mark_end(lexer);
           lexer->result_symbol = STRING_CONTENT;
           return has_content;
@@ -322,13 +336,17 @@ struct Scanner {
       }
     }
 
-    if (first_comment_indent_length == -1 && valid_symbols[STRING_START]) {
+    bool expects_string_start = valid_symbols[STRING_START] || valid_symbols[TEMPLATE_STRING_START];
+
+    if (first_comment_indent_length == -1 && expects_string_start) {
       Delimiter delimiter;
 
       bool has_flags = false;
       while (lexer->lookahead) {
         if (lexer->lookahead == 'f' || lexer->lookahead == 'F') {
           delimiter.set_format();
+        } else if (lexer->lookahead == 't' || lexer->lookahead == 'T') {
+          delimiter.set_template();
         } else if (lexer->lookahead == 'r' || lexer->lookahead == 'R') {
           delimiter.set_raw();
         } else if (lexer->lookahead == 'b' || lexer->lookahead == 'B') {
@@ -372,7 +390,7 @@ struct Scanner {
 
       if (delimiter.end_character()) {
         delimiter_stack.push_back(delimiter);
-        lexer->result_symbol = STRING_START;
+        lexer->result_symbol = delimiter.is_template() ? TEMPLATE_STRING_START : STRING_START;
         return true;
       } else if (has_flags) {
         return false;

python/extractor/tsg-python/tsp/src/tree_sitter/parser.h

@@ -123,7 +123,6 @@ struct TSLanguage {
     unsigned (*serialize)(void *, char *);
     void (*deserialize)(void *, const char *, unsigned);
   } external_scanner;
-  const TSStateId *primary_state_ids;
 };
 
 /*

python/ql/consistency-queries/OverlayDiscardConsistency.ql

@@ -0,0 +1 @@
+import semmle.python.internal.OverlayDiscardConsistencyQuery

python/ql/examples/snippets/builtin_object.ql

@@ -8,7 +8,8 @@
  */
 
 import python
+private import LegacyPointsTo
 
-from Expr e, string name
+from ExprWithPointsTo e, string name
 where e.pointsTo(Value::named(name)) and not name.charAt(_) = "."
 select e

python/ql/examples/snippets/call.ql

@@ -7,6 +7,7 @@
  */
 
 import python
+private import LegacyPointsTo
 
 from Value len, CallNode call
 where len.getName() = "len" and len.getACall() = call

python/ql/examples/snippets/catch_exception.ql

@@ -8,9 +8,10 @@
  */
 
 import python
+private import LegacyPointsTo
 
 from ExceptStmt ex, ClassValue cls
 where
   cls.getName() = "MyExceptionClass" and
-  ex.getType().pointsTo(cls)
+  ex.getType().(ExprWithPointsTo).pointsTo(cls)
 select ex

python/ql/examples/snippets/conditional_expression.ql

@@ -9,10 +9,11 @@
  */
 
 import python
+private import LegacyPointsTo
 
 from IfExp e, ClassObject cls1, ClassObject cls2
 where
-  e.getBody().refersTo(_, cls1, _) and
-  e.getOrelse().refersTo(_, cls2, _) and
+  e.getBody().(ExprWithPointsTo).refersTo(_, cls1, _) and
+  e.getOrelse().(ExprWithPointsTo).refersTo(_, cls2, _) and
   cls1 != cls2
 select e

python/ql/examples/snippets/extend_class.ql

@@ -11,6 +11,7 @@
  */
 
 import python
+private import LegacyPointsTo
 
 from ClassObject sub, ClassObject base
 where

python/ql/examples/snippets/method_call.ql

@@ -7,6 +7,7 @@
  */
 
 import python
+private import LegacyPointsTo
 
 from AstNode call, PythonFunctionValue method
 where

python/ql/examples/snippets/mutualrecursion.ql

@@ -7,6 +7,7 @@
  */
 
 import python
+private import LegacyPointsTo
 
 from FunctionObject m, FunctionObject n
 where m != n and m.getACallee() = n and n.getACallee() = m

python/ql/examples/snippets/new_instance.ql

@@ -8,9 +8,10 @@
  */
 
 import python
+private import LegacyPointsTo
 
 from Call new, ClassValue cls
 where
   cls.getName() = "MyClass" and
-  new.getFunc().pointsTo(cls)
+  new.getFunc().(ExprWithPointsTo).pointsTo(cls)
 select new

python/ql/examples/snippets/override_method.ql

@@ -7,6 +7,7 @@
  */
 
 import python
+private import LegacyPointsTo
 
 from FunctionObject override, FunctionObject base
 where

python/ql/examples/snippets/print.ql

@@ -6,6 +6,7 @@
  */
 
 import python
+private import LegacyPointsTo
 
 from AstNode print
 where
@@ -13,5 +14,5 @@ where
   print instanceof Print
   or
   /* Python 3 or with `from __future__ import print_function` */
-  print.(Call).getFunc().pointsTo(Value::named("print"))
+  print.(Call).getFunc().(ExprWithPointsTo).pointsTo(Value::named("print"))
 select print

python/ql/examples/snippets/raise_exception.ql

@@ -8,9 +8,10 @@
  */
 
 import python
+private import LegacyPointsTo
 
 from Raise raise, ClassValue ex
 where
   ex.getName() = "AnException" and
-  raise.getException().pointsTo(ex.getASuperType())
+  raise.getException().(ExprWithPointsTo).pointsTo(ex.getASuperType())
 select raise, "Don't raise instances of 'AnException'"

python/ql/examples/snippets/recursion.ql

@@ -7,6 +7,7 @@
  */
 
 import python
+private import LegacyPointsTo
 
 from PythonFunctionValue f
 where f.getACall().getScope() = f.getScope()

python/ql/examples/snippets/store_none.ql

@@ -10,9 +10,10 @@
  */
 
 import python
+private import LegacyPointsTo
 
 from SubscriptNode store
 where
   store.isStore() and
-  store.getIndex().pointsTo(Value::named("None"))
+  store.getIndex().(ControlFlowNodeWithPointsTo).pointsTo(Value::named("None"))
 select store

python/ql/integration-tests/query-suite/not_included_in_qls.expected

@@ -87,6 +87,7 @@ ql/python/ql/src/experimental/Security/CWE-079/EmailXss.ql
 ql/python/ql/src/experimental/Security/CWE-091/XsltInjection.ql
 ql/python/ql/src/experimental/Security/CWE-094/Js2Py.ql
 ql/python/ql/src/experimental/Security/CWE-1236/CsvInjection.ql
+ql/python/ql/src/experimental/Security/CWE-1427/PromptInjection.ql
 ql/python/ql/src/experimental/Security/CWE-176/UnicodeBypassValidation.ql
 ql/python/ql/src/experimental/Security/CWE-208/TimingAttackAgainstHash/PossibleTimingAttackAgainstHash.ql
 ql/python/ql/src/experimental/Security/CWE-208/TimingAttackAgainstHash/TimingAttackAgainstHash.ql

python/ql/integration-tests/query-suite/python-code-scanning.qls.expected

@@ -13,8 +13,10 @@ ql/python/ql/src/Security/CWE-079/ReflectedXss.ql
 ql/python/ql/src/Security/CWE-089/SqlInjection.ql
 ql/python/ql/src/Security/CWE-090/LdapInjection.ql
 ql/python/ql/src/Security/CWE-094/CodeInjection.ql
+ql/python/ql/src/Security/CWE-1004/NonHttpOnlyCookie.ql
 ql/python/ql/src/Security/CWE-113/HeaderInjection.ql
 ql/python/ql/src/Security/CWE-116/BadTagFilter.ql
+ql/python/ql/src/Security/CWE-1275/SameSiteNoneCookie.ql
 ql/python/ql/src/Security/CWE-209/StackTraceExposure.ql
 ql/python/ql/src/Security/CWE-215/FlaskDebug.ql
 ql/python/ql/src/Security/CWE-285/PamAuthorization.ql

python/ql/integration-tests/query-suite/python-security-and-quality.qls.expected

@@ -106,9 +106,11 @@ ql/python/ql/src/Security/CWE-079/ReflectedXss.ql
 ql/python/ql/src/Security/CWE-089/SqlInjection.ql
 ql/python/ql/src/Security/CWE-090/LdapInjection.ql
 ql/python/ql/src/Security/CWE-094/CodeInjection.ql
+ql/python/ql/src/Security/CWE-1004/NonHttpOnlyCookie.ql
 ql/python/ql/src/Security/CWE-113/HeaderInjection.ql
 ql/python/ql/src/Security/CWE-116/BadTagFilter.ql
 ql/python/ql/src/Security/CWE-117/LogInjection.ql
+ql/python/ql/src/Security/CWE-1275/SameSiteNoneCookie.ql
 ql/python/ql/src/Security/CWE-209/StackTraceExposure.ql
 ql/python/ql/src/Security/CWE-215/FlaskDebug.ql
 ql/python/ql/src/Security/CWE-285/PamAuthorization.ql

python/ql/integration-tests/query-suite/python-security-extended.qls.expected

@@ -16,9 +16,11 @@ ql/python/ql/src/Security/CWE-079/ReflectedXss.ql
 ql/python/ql/src/Security/CWE-089/SqlInjection.ql
 ql/python/ql/src/Security/CWE-090/LdapInjection.ql
 ql/python/ql/src/Security/CWE-094/CodeInjection.ql
+ql/python/ql/src/Security/CWE-1004/NonHttpOnlyCookie.ql
 ql/python/ql/src/Security/CWE-113/HeaderInjection.ql
 ql/python/ql/src/Security/CWE-116/BadTagFilter.ql
 ql/python/ql/src/Security/CWE-117/LogInjection.ql
+ql/python/ql/src/Security/CWE-1275/SameSiteNoneCookie.ql
 ql/python/ql/src/Security/CWE-209/StackTraceExposure.ql
 ql/python/ql/src/Security/CWE-215/FlaskDebug.ql
 ql/python/ql/src/Security/CWE-285/PamAuthorization.ql

python/ql/lib/CHANGELOG.md

@@ -1,3 +1,60 @@
+## 6.0.0
+
+### Breaking Changes
+
+* All modules that depend on the points-to analysis have now been removed from the top level `python.qll` module. To access the points-to functionality, import the new `LegacyPointsTo` module. This also means that some predicates have been removed from various classes, for instance `Function.getFunctionObject()`. To access these predicates, import the `LegacyPointsTo` module and use the `FunctionWithPointsTo` class instead. Most cases follow this pattern, but there are a few exceptions:
+  * The `getLiteralObject` method on `ImmutableLiteral` subclasses has been replaced with a predicate `getLiteralObject(ImmutableLiteral l)` in the `LegacyPointsTo` module.
+  * The `getMetrics` method on `Function`, `Class`, and `Module` has been removed. To access metrics, import `LegacyPointsTo` and use the classes `FunctionMetrics`, etc. instead.
+
+### New Features
+
+* The extractor now supports the new, relaxed syntax `except A, B, C: ...` (which would previously have to be written as `except (A, B, C): ...`) as defined in [PEP-758](https://peps.python.org/pep-0758/). This may cause changes in results for code that uses Python 2-style exception binding (`except Foo, e: ...`). The more modern format, `except Foo as e: ...` (available since Python 2.6) is unaffected.
+* The Python extractor now supports template strings as defined in [PEP-750](https://peps.python.org/pep-0750/), through the classes `TemplateString` and `JoinedTemplateString`.
+
+### Minor Analysis Improvements
+
+* When a code-scanning configuration specifies the `paths:` and/or `paths-ignore:` settings, these are now taken into account by the Python extractor's search for YAML files.
+* The `compression.zstd` library (added in Python 3.14) is now supported by the `py/decompression-bomb` query.
+* Added taint flow model and type model for `urllib.parse`.
+* Remote flow sources for the `python-socketio` package have been modeled.
+* Additional models for remote flow sources for `tornado.websocket.WebSocketHandler` have been added.
+
+## 5.0.4
+
+No user-facing changes.
+
+## 5.0.3
+
+No user-facing changes.
+
+## 5.0.2
+
+No user-facing changes.
+
+## 5.0.1
+
+### Bug Fixes
+
+- Fixed a bug in the Python extractor's import handling where failing to find an import in `find_module` would cause a `KeyError` to be raised. (Contributed by @akoeplinger.)
+
+## 5.0.0
+
+### Breaking Changes
+
+- The classes `ControlFlowNode`, `Expr`, and `Module` no longer expose predicates that invoke the points-to analysis. To access these predicates, import the module `LegacyPointsTo` and follow the instructions given therein.
+
+## 4.1.0
+
+### New Features
+
+* Initial support for incremental Python databases via `codeql database create --overlay-base`/`--overlay-changes`.
+
+## 4.0.17
+
+### Bug Fixes
+
+* The Python extractor no longer crashes with an `ImportError` when run using Python 3.14.
+
 ## 4.0.16
 
 ### Minor Analysis Improvements

python/ql/lib/LegacyPointsTo.qll

@@ -0,0 +1,432 @@
+/**
+ * DEPRECATED: Using the methods in this module may lead to a degradation of performance. Use at
+ * your own peril.
+ *
+ * This module contains legacy points-to predicates and methods for various classes in the
+ * points-to analysis.
+ *
+ * Existing code that depends on, say, points-to predicates on `ControlFlowNode` should be modified
+ * to use `ControlFlowNodeWithPointsTo` instead. In particular, if inside a method call chain such
+ * as
+ *
+ * `someCallNode.getFunction().pointsTo(...)`
+ *
+ *  an explicit cast should be added as follows
+ *
+ * `someCallNode.getFunction().(ControlFlowNodeWithPointsTo).pointsTo(...)`
+ *
+ * Similarly, if a bound variable has type `ControlFlowNode`, and a points-to method is called on
+ * it, the type should be changed to `ControlFlowNodeWithPointsTo`.
+ */
+
+private import python
+import semmle.python.pointsto.Base
+import semmle.python.pointsto.Context
+import semmle.python.pointsto.PointsTo
+import semmle.python.pointsto.PointsToContext
+import semmle.python.objects.ObjectAPI
+import semmle.python.objects.ObjectInternal
+import semmle.python.types.Object
+import semmle.python.types.ClassObject
+import semmle.python.types.FunctionObject
+import semmle.python.types.ModuleObject
+import semmle.python.types.Exceptions
+import semmle.python.types.Properties
+import semmle.python.types.Descriptors
+import semmle.python.SelfAttribute
+import semmle.python.Metrics
+
+/**
+ * An extension of `ControlFlowNode` that provides points-to predicates.
+ */
+class ControlFlowNodeWithPointsTo extends ControlFlowNode {
+  /** Gets the value that this ControlFlowNode points-to. */
+  predicate pointsTo(Value value) { this.pointsTo(_, value, _) }
+
+  /** Gets the value that this ControlFlowNode points-to. */
+  Value pointsTo() { this.pointsTo(_, result, _) }
+
+  /** Gets a value that this ControlFlowNode may points-to. */
+  Value inferredValue() { this.pointsTo(_, result, _) }
+
+  /** Gets the value and origin that this ControlFlowNode points-to. */
+  predicate pointsTo(Value value, ControlFlowNode origin) { this.pointsTo(_, value, origin) }
+
+  /** Gets the value and origin that this ControlFlowNode points-to, given the context. */
+  predicate pointsTo(Context context, Value value, ControlFlowNode origin) {
+    PointsTo::pointsTo(this, context, value, origin)
+  }
+
+  /**
+   * Gets what this flow node might "refer-to". Performs a combination of localized (intra-procedural) points-to
+   * analysis and global module-level analysis. This points-to analysis favours precision over recall. It is highly
+   * precise, but may not provide information for a significant number of flow-nodes.
+   * If the class is unimportant then use `refersTo(value)` or `refersTo(value, origin)` instead.
+   */
+  pragma[nomagic]
+  predicate refersTo(Object obj, ClassObject cls, ControlFlowNode origin) {
+    this.refersTo(_, obj, cls, origin)
+  }
+
+  /** Gets what this expression might "refer-to" in the given `context`. */
+  pragma[nomagic]
+  predicate refersTo(Context context, Object obj, ClassObject cls, ControlFlowNode origin) {
+    not obj = unknownValue() and
+    not cls = theUnknownType() and
+    PointsTo::points_to(this, context, obj, cls, origin)
+  }
+
+  /**
+   * Whether this flow node might "refer-to" to `value` which is from `origin`
+   * Unlike `this.refersTo(value, _, origin)` this predicate includes results
+   * where the class cannot be inferred.
+   */
+  pragma[nomagic]
+  predicate refersTo(Object obj, ControlFlowNode origin) {
+    not obj = unknownValue() and
+    PointsTo::points_to(this, _, obj, _, origin)
+  }
+
+  /** Equivalent to `this.refersTo(value, _)` */
+  predicate refersTo(Object obj) { this.refersTo(obj, _) }
+
+  /**
+   * Check whether this control-flow node has complete points-to information.
+   * This would mean that the analysis managed to infer an over approximation
+   * of possible values at runtime.
+   */
+  predicate hasCompletePointsToSet() {
+    // If the tracking failed, then `this` will be its own "origin". In that
+    // case, we want to exclude nodes for which there is also a different
+    // origin, as that would indicate that some paths failed and some did not.
+    this.refersTo(_, _, this) and
+    not exists(ControlFlowNode other | other != this and this.refersTo(_, _, other))
+    or
+    // If `this` is a use of a variable, then we must have complete points-to
+    // for that variable.
+    exists(SsaVariable v | v.getAUse() = this | varHasCompletePointsToSet(v))
+  }
+
+  /** Whether it is unlikely that this ControlFlowNode can be reached */
+  predicate unlikelyReachable() {
+    not start_bb_likely_reachable(this.getBasicBlock())
+    or
+    exists(BasicBlock b |
+      start_bb_likely_reachable(b) and
+      not end_bb_likely_reachable(b) and
+      // If there is an unlikely successor edge earlier in the BB
+      // than this node, then this node must be unreachable.
+      exists(ControlFlowNode p, int i, int j |
+        p.(RaisingNode).unlikelySuccessor(_) and
+        p = b.getNode(i) and
+        this = b.getNode(j) and
+        i < j
+      )
+    )
+  }
+}
+
+/**
+ * Check whether a SSA variable has complete points-to information.
+ * This would mean that the analysis managed to infer an overapproximation
+ * of possible values at runtime.
+ */
+private predicate varHasCompletePointsToSet(SsaVariable var) {
+  // Global variables may be modified non-locally or concurrently.
+  not var.getVariable() instanceof GlobalVariable and
+  (
+    // If we have complete points-to information on the definition of
+    // this variable, then the variable has complete information.
+    var.getDefinition()
+        .(DefinitionNode)
+        .getValue()
+        .(ControlFlowNodeWithPointsTo)
+        .hasCompletePointsToSet()
+    or
+    // If this variable is a phi output, then we have complete
+    // points-to information about it if all phi inputs had complete
+    // information.
+    forex(SsaVariable phiInput | phiInput = var.getAPhiInput() |
+      varHasCompletePointsToSet(phiInput)
+    )
+  )
+}
+
+private predicate start_bb_likely_reachable(BasicBlock b) {
+  exists(Scope s | s.getEntryNode() = b.getNode(_))
+  or
+  exists(BasicBlock pred |
+    pred = b.getAPredecessor() and
+    end_bb_likely_reachable(pred) and
+    not pred.getLastNode().(RaisingNode).unlikelySuccessor(b)
+  )
+}
+
+private predicate end_bb_likely_reachable(BasicBlock b) {
+  start_bb_likely_reachable(b) and
+  not exists(ControlFlowNode p, ControlFlowNode s |
+    p.(RaisingNode).unlikelySuccessor(s) and
+    p = b.getNode(_) and
+    s = b.getNode(_) and
+    not p = b.getLastNode()
+  )
+}
+
+/**
+ * An extension of `BasicBlock` that provides points-to related methods.
+ */
+class BasicBlockWithPointsTo extends BasicBlock {
+  /**
+   * Whether (as inferred by type inference) it is highly unlikely (or impossible) for control to flow from this to succ.
+   */
+  predicate unlikelySuccessor(BasicBlockWithPointsTo succ) {
+    this.getLastNode().(RaisingNode).unlikelySuccessor(succ.firstNode())
+    or
+    not end_bb_likely_reachable(this) and succ = this.getASuccessor()
+  }
+
+  /**
+   * Whether (as inferred by type inference) this basic block is likely to be reachable.
+   */
+  predicate likelyReachable() { start_bb_likely_reachable(this) }
+}
+
+/**
+ * An extension of `Expr` that provides points-to predicates.
+ */
+class ExprWithPointsTo extends Expr {
+  /**
+   * NOTE: `refersTo` will be deprecated in 2019. Use `pointsTo` instead.
+   * Gets what this expression might "refer-to". Performs a combination of localized (intra-procedural) points-to
+   *  analysis and global module-level analysis. This points-to analysis favours precision over recall. It is highly
+   *  precise, but may not provide information for a significant number of flow-nodes.
+   *  If the class is unimportant then use `refersTo(value)` or `refersTo(value, origin)` instead.
+   * NOTE: For complex dataflow, involving multiple stages of points-to analysis, it may be more precise to use
+   * `ControlFlowNode.refersTo(...)` instead.
+   */
+  predicate refersTo(Object obj, ClassObject cls, AstNode origin) {
+    this.refersTo(_, obj, cls, origin)
+  }
+
+  /**
+   * NOTE: `refersTo` will be deprecated in 2019. Use `pointsTo` instead.
+   * Gets what this expression might "refer-to" in the given `context`.
+   */
+  predicate refersTo(Context context, Object obj, ClassObject cls, AstNode origin) {
+    this.getAFlowNode()
+        .(ControlFlowNodeWithPointsTo)
+        .refersTo(context, obj, cls, origin.getAFlowNode())
+  }
+
+  /**
+   * NOTE: `refersTo` will be deprecated in 2019. Use `pointsTo` instead.
+   * Holds if this expression might "refer-to" to `value` which is from `origin`
+   * Unlike `this.refersTo(value, _, origin)`, this predicate includes results
+   * where the class cannot be inferred.
+   */
+  pragma[nomagic]
+  predicate refersTo(Object obj, AstNode origin) {
+    this.getAFlowNode().(ControlFlowNodeWithPointsTo).refersTo(obj, origin.getAFlowNode())
+  }
+
+  /**
+   * NOTE: `refersTo` will be deprecated in 2019. Use `pointsTo` instead.
+   * Equivalent to `this.refersTo(value, _)`
+   */
+  predicate refersTo(Object obj) { this.refersTo(obj, _) }
+
+  /**
+   * Holds if this expression might "point-to" to `value` which is from `origin`
+   * in the given `context`.
+   */
+  predicate pointsTo(Context context, Value value, AstNode origin) {
+    this.getAFlowNode()
+        .(ControlFlowNodeWithPointsTo)
+        .pointsTo(context, value, origin.getAFlowNode())
+  }
+
+  /**
+   * Holds if this expression might "point-to" to `value` which is from `origin`.
+   */
+  predicate pointsTo(Value value, AstNode origin) {
+    this.getAFlowNode().(ControlFlowNodeWithPointsTo).pointsTo(value, origin.getAFlowNode())
+  }
+
+  /**
+   * Holds if this expression might "point-to" to `value`.
+   */
+  predicate pointsTo(Value value) { this.pointsTo(value, _) }
+
+  /** Gets a value that this expression might "point-to". */
+  Value pointsTo() { this.pointsTo(result) }
+
+  override string getAQlClass() { none() }
+}
+
+/**
+ * An extension of `Module` that provides points-to related methods.
+ */
+class ModuleWithPointsTo extends Module {
+  /** Gets a name exported by this module, that is the names that will be added to a namespace by 'from this-module import *' */
+  string getAnExport() {
+    py_exports(this, result)
+    or
+    exists(ModuleObjectInternal mod | mod.getSource() = this.getEntryNode() |
+      mod.(ModuleValue).exports(result)
+    )
+  }
+
+  override string getAQlClass() { none() }
+}
+
+/**
+ * An extension of `Function` that provides points-to related methods.
+ */
+class FunctionWithPointsTo extends Function {
+  /** Gets the FunctionObject corresponding to this function */
+  FunctionObject getFunctionObject() { result.getOrigin() = this.getDefinition() }
+
+  override string getAQlClass() { none() }
+}
+
+/**
+ * An extension of `Class` that provides points-to related methods.
+ */
+class ClassWithPointsTo extends Class {
+  /** Gets the ClassObject corresponding to this class */
+  ClassObject getClassObject() { result.getOrigin() = this.getParent() }
+
+  override string getAQlClass() { none() }
+}
+
+/** Gets the `Object` corresponding to the immutable literal `l`. */
+Object getLiteralObject(ImmutableLiteral l) {
+  l instanceof IntegerLiteral and
+  (
+    py_cobjecttypes(result, theIntType()) and py_cobjectnames(result, l.(Num).getN())
+    or
+    py_cobjecttypes(result, theLongType()) and py_cobjectnames(result, l.(Num).getN())
+  )
+  or
+  l instanceof FloatLiteral and
+  py_cobjecttypes(result, theFloatType()) and
+  py_cobjectnames(result, l.(Num).getN())
+  or
+  l instanceof ImaginaryLiteral and
+  py_cobjecttypes(result, theComplexType()) and
+  py_cobjectnames(result, l.(Num).getN())
+  or
+  l instanceof NegativeIntegerLiteral and
+  (
+    (py_cobjecttypes(result, theIntType()) or py_cobjecttypes(result, theLongType())) and
+    py_cobjectnames(result, "-" + l.(UnaryExpr).getOperand().(IntegerLiteral).getN())
+  )
+  or
+  l instanceof Bytes and
+  py_cobjecttypes(result, theBytesType()) and
+  py_cobjectnames(result, l.(Bytes).quotedString())
+  or
+  l instanceof Unicode and
+  py_cobjecttypes(result, theUnicodeType()) and
+  py_cobjectnames(result, l.(Unicode).quotedString())
+  or
+  l instanceof True and
+  name_consts(l, "True") and
+  result = theTrueObject()
+  or
+  l instanceof False and
+  name_consts(l, "False") and
+  result = theFalseObject()
+  or
+  l instanceof None and
+  name_consts(l, "None") and
+  result = theNoneObject()
+}
+
+private predicate gettext_installed() {
+  // Good enough (and fast) approximation
+  exists(Module m | m.getName() = "gettext")
+}
+
+private predicate builtin_constant(string name) {
+  exists(Object::builtin(name))
+  or
+  name = "WindowsError"
+  or
+  name = "_" and gettext_installed()
+}
+
+/** Whether this name is (almost) always defined, ie. it is a builtin or VM defined name */
+predicate globallyDefinedName(string name) { builtin_constant(name) or auto_name(name) }
+
+private predicate auto_name(string name) {
+  name = "__file__" or name = "__builtins__" or name = "__name__"
+}
+
+/** An extension of `SsaVariable` that provides points-to related methods. */
+class SsaVariableWithPointsTo extends SsaVariable {
+  /** Gets an argument of the phi function defining this variable, pruned of unlikely edges. */
+  SsaVariable getAPrunedPhiInput() {
+    result = this.getAPhiInput() and
+    exists(BasicBlock incoming | incoming = this.getPredecessorBlockForPhiArgument(result) |
+      not incoming.getLastNode().(RaisingNode).unlikelySuccessor(this.getDefinition())
+    )
+  }
+
+  /** Gets the incoming edges for a Phi node, pruned of unlikely edges. */
+  private BasicBlockWithPointsTo getAPrunedPredecessorBlockForPhi() {
+    result = this.getAPredecessorBlockForPhi() and
+    not result.unlikelySuccessor(this.getDefinition().getBasicBlock())
+  }
+
+  private predicate implicitlyDefined() {
+    not exists(this.getDefinition()) and
+    not py_ssa_phi(this, _) and
+    exists(GlobalVariable var | this.getVariable() = var |
+      globallyDefinedName(var.getId())
+      or
+      var.getId() = "__path__" and var.getScope().(Module).isPackageInit()
+    )
+  }
+
+  /** Whether this variable may be undefined */
+  predicate maybeUndefined() {
+    not exists(this.getDefinition()) and not py_ssa_phi(this, _) and not this.implicitlyDefined()
+    or
+    this.getDefinition().isDelete()
+    or
+    exists(SsaVariableWithPointsTo var | var = this.getAPrunedPhiInput() | var.maybeUndefined())
+    or
+    /*
+     * For phi-nodes, there must be a corresponding phi-input for each control-flow
+     * predecessor. Otherwise, the variable will be undefined on that incoming edge.
+     * WARNING: the same phi-input may cover multiple predecessors, so this check
+     *          cannot be done by counting.
+     */
+
+    exists(BasicBlock incoming |
+      reaches_end(incoming) and
+      incoming = this.getAPrunedPredecessorBlockForPhi() and
+      not this.getAPhiInput().getDefinition().getBasicBlock().dominates(incoming)
+    )
+  }
+
+  override string getAQlClass() { none() }
+}
+
+private predicate reaches_end(BasicBlock b) {
+  not exits_early(b) and
+  (
+    /* Entry point */
+    not exists(BasicBlock prev | prev.getASuccessor() = b)
+    or
+    exists(BasicBlock prev | prev.getASuccessor() = b | reaches_end(prev))
+  )
+}
+
+private predicate exits_early(BasicBlock b) {
+  exists(FunctionObject f |
+    f.neverReturns() and
+    f.getACall().getBasicBlock() = b
+  )
+}

python/ql/lib/analysis/DefinitionTracking.qll

@@ -3,7 +3,8 @@
  */
 
 import python
-import semmle.python.pointsto.PointsTo
+private import LegacyPointsTo
+private import semmle.python.types.ImportTime
 import IDEContextual
 
 private newtype TDefinition =
@@ -36,22 +37,22 @@ private predicate jump_to_defn(ControlFlowNode use, Definition defn) {
   )
   or
   exists(PythonModuleObject mod |
-    use.(ImportExprNode).refersTo(mod) and
+    use.(ImportExprNode).(ControlFlowNodeWithPointsTo).refersTo(mod) and
     defn.getAstNode() = mod.getModule()
   )
   or
   exists(PythonModuleObject mod, string name |
-    use.(ImportMemberNode).getModule(name).refersTo(mod) and
+    use.(ImportMemberNode).getModule(name).(ControlFlowNodeWithPointsTo).refersTo(mod) and
     scope_jump_to_defn_attribute(mod.getModule(), name, defn)
   )
   or
   exists(PackageObject package |
-    use.(ImportExprNode).refersTo(package) and
+    use.(ImportExprNode).(ControlFlowNodeWithPointsTo).refersTo(package) and
     defn.getAstNode() = package.getInitModule().getModule()
   )
   or
   exists(PackageObject package, string name |
-    use.(ImportMemberNode).getModule(name).refersTo(package) and
+    use.(ImportMemberNode).getModule(name).(ControlFlowNodeWithPointsTo).refersTo(package) and
     scope_jump_to_defn_attribute(package.getInitModule().getModule(), name, defn)
   )
   or
@@ -230,7 +231,7 @@ private predicate module_and_name_for_import_star_helper(
   ModuleObject mod, string name, ImportStarNode im_star, ImportStarRefinement def
 ) {
   im_star = def.getDefiningNode() and
-  im_star.getModule().refersTo(mod) and
+  im_star.getModule().(ControlFlowNodeWithPointsTo).refersTo(mod) and
   name = def.getSourceVariable().getName()
 }
 
@@ -239,7 +240,7 @@ pragma[noinline]
 private predicate variable_not_redefined_by_import_star(EssaVariable var, ImportStarRefinement def) {
   var = def.getInput() and
   exists(ModuleObject mod |
-    def.getDefiningNode().(ImportStarNode).getModule().refersTo(mod) and
+    def.getDefiningNode().(ImportStarNode).getModule().(ControlFlowNodeWithPointsTo).refersTo(mod) and
     not mod.exports(var.getSourceVariable().getName())
   )
 }
@@ -352,7 +353,9 @@ private predicate scope_jump_to_defn_attribute(ImportTimeScope s, string name, D
   )
 }
 
-private predicate jump_to_defn_attribute(ControlFlowNode use, string name, Definition defn) {
+private predicate jump_to_defn_attribute(
+  ControlFlowNodeWithPointsTo use, string name, Definition defn
+) {
   /* Local attribute */
   exists(EssaVariable var |
     use = var.getASourceUse() and
@@ -367,7 +370,7 @@ private predicate jump_to_defn_attribute(ControlFlowNode use, string name, Defin
   /* Super attributes */
   exists(AttrNode f, SuperBoundMethod sbm, Object function |
     use = f.getObject(name) and
-    f.refersTo(sbm) and
+    f.(ControlFlowNodeWithPointsTo).refersTo(sbm) and
     function = sbm.getFunction(_) and
     function.getOrigin() = defn.getAstNode()
   )
@@ -408,7 +411,7 @@ private predicate attribute_assignment_jump_to_defn_attribute(
 private predicate sets_attribute(ArgumentRefinement def, string name) {
   exists(CallNode call |
     call = def.getDefiningNode() and
-    call.getFunction().refersTo(Object::builtin("setattr")) and
+    call.getFunction().(ControlFlowNodeWithPointsTo).refersTo(Object::builtin("setattr")) and
     def.getInput().getAUse() = call.getArg(0) and
     call.getArg(1).getNode().(StringLiteral).getText() = name
   )

python/ql/lib/change-notes/2025-12-01-websockets.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Remote flow sources for the `websockets` package have been modeled.

python/ql/lib/change-notes/2026-01-02-prompt-injection.md

@@ -0,0 +1,5 @@
+---
+category: minorAnalysis
+---
+* Added experimental query `py/prompt-injection` to detect potential prompt injection vulnerabilities in code using LLMs.
+* Added taint flow model and type model for `agents` and `openai` modules.

python/ql/lib/change-notes/2026-01-16-summarized-callable.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* The predicate `SummarizedCallable.propagatesFlow` has been extended with the columns `Provenance p` and `boolean isExact`, and as a consequence the predicates `SummarizedCallable.hasProvenance` and `SummarizedCallable.hasExactModel` have been removed.

python/ql/lib/change-notes/2026-01-20-support-ListElement-in-python-MaD.md

@@ -0,0 +1,4 @@
+---
+category: feature
+---
+* It is now possible to refer to list elements in the Python models-as-data language, via the `ListElement` path.

python/ql/lib/change-notes/released/4.0.17.md

@@ -0,0 +1,5 @@
+## 4.0.17
+
+### Bug Fixes
+
+* The Python extractor no longer crashes with an `ImportError` when run using Python 3.14.

python/ql/lib/change-notes/released/4.1.0.md

@@ -0,0 +1,5 @@
+## 4.1.0
+
+### New Features
+
+* Initial support for incremental Python databases via `codeql database create --overlay-base`/`--overlay-changes`.

python/ql/lib/change-notes/released/5.0.0.md

@@ -0,0 +1,5 @@
+## 5.0.0
+
+### Breaking Changes
+
+- The classes `ControlFlowNode`, `Expr`, and `Module` no longer expose predicates that invoke the points-to analysis. To access these predicates, import the module `LegacyPointsTo` and follow the instructions given therein.

python/ql/lib/change-notes/released/5.0.1.md

@@ -0,0 +1,5 @@
+## 5.0.1
+
+### Bug Fixes
+
+- Fixed a bug in the Python extractor's import handling where failing to find an import in `find_module` would cause a `KeyError` to be raised. (Contributed by @akoeplinger.)

python/ql/lib/change-notes/released/5.0.2.md

@@ -0,0 +1,3 @@
+## 5.0.2
+
+No user-facing changes.

python/ql/lib/change-notes/released/5.0.3.md

@@ -0,0 +1,3 @@
+## 5.0.3
+
+No user-facing changes.

python/ql/lib/change-notes/released/5.0.4.md

@@ -0,0 +1,3 @@
+## 5.0.4
+
+No user-facing changes.

python/ql/lib/change-notes/released/6.0.0.md

@@ -0,0 +1,20 @@
+## 6.0.0
+
+### Breaking Changes
+
+* All modules that depend on the points-to analysis have now been removed from the top level `python.qll` module. To access the points-to functionality, import the new `LegacyPointsTo` module. This also means that some predicates have been removed from various classes, for instance `Function.getFunctionObject()`. To access these predicates, import the `LegacyPointsTo` module and use the `FunctionWithPointsTo` class instead. Most cases follow this pattern, but there are a few exceptions:
+  * The `getLiteralObject` method on `ImmutableLiteral` subclasses has been replaced with a predicate `getLiteralObject(ImmutableLiteral l)` in the `LegacyPointsTo` module.
+  * The `getMetrics` method on `Function`, `Class`, and `Module` has been removed. To access metrics, import `LegacyPointsTo` and use the classes `FunctionMetrics`, etc. instead.
+
+### New Features
+
+* The extractor now supports the new, relaxed syntax `except A, B, C: ...` (which would previously have to be written as `except (A, B, C): ...`) as defined in [PEP-758](https://peps.python.org/pep-0758/). This may cause changes in results for code that uses Python 2-style exception binding (`except Foo, e: ...`). The more modern format, `except Foo as e: ...` (available since Python 2.6) is unaffected.
+* The Python extractor now supports template strings as defined in [PEP-750](https://peps.python.org/pep-0750/), through the classes `TemplateString` and `JoinedTemplateString`.
+
+### Minor Analysis Improvements
+
+* When a code-scanning configuration specifies the `paths:` and/or `paths-ignore:` settings, these are now taken into account by the Python extractor's search for YAML files.
+* The `compression.zstd` library (added in Python 3.14) is now supported by the `py/decompression-bomb` query.
+* Added taint flow model and type model for `urllib.parse`.
+* Remote flow sources for the `python-socketio` package have been modeled.
+* Additional models for remote flow sources for `tornado.websocket.WebSocketHandler` have been added.

python/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 4.0.16
+lastReleaseVersion: 6.0.0

python/ql/lib/python.qll

@@ -14,29 +14,30 @@ import semmle.python.Patterns
 import semmle.python.Keywords
 import semmle.python.Comprehensions
 import semmle.python.Flow
-import semmle.python.Metrics
+private import semmle.python.Metrics
 import semmle.python.Constants
 import semmle.python.Scope
 import semmle.python.Comment
 import semmle.python.GuardedControlFlow
-import semmle.python.types.ImportTime
-import semmle.python.types.Object
-import semmle.python.types.ClassObject
-import semmle.python.types.FunctionObject
-import semmle.python.types.ModuleObject
-import semmle.python.types.Version
-import semmle.python.types.Descriptors
+private import semmle.python.types.ImportTime
+private import semmle.python.types.Object
+private import semmle.python.types.ClassObject
+private import semmle.python.types.FunctionObject
+private import semmle.python.types.ModuleObject
+private import semmle.python.types.Version
+private import semmle.python.types.Descriptors
 import semmle.python.SSA
-import semmle.python.SelfAttribute
-import semmle.python.types.Properties
+private import semmle.python.SelfAttribute
+private import semmle.python.types.Properties
 import semmle.python.xml.XML
 import semmle.python.essa.Essa
-import semmle.python.pointsto.Base
-import semmle.python.pointsto.Context
-import semmle.python.pointsto.CallGraph
-import semmle.python.objects.ObjectAPI
+private import semmle.python.pointsto.Base
+private import semmle.python.pointsto.Context
+private import semmle.python.pointsto.CallGraph
+private import semmle.python.objects.ObjectAPI
 import semmle.python.Unit
 import site
+private import semmle.python.Overlay
 // Removing this import perturbs the compilation process enough that the points-to analysis gets
 // compiled -- and cached -- differently depending on whether the data flow library is imported. By
 // importing it privately here, we ensure that the points-to analysis is compiled the same way.

python/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/python-all
-version: 4.0.17-dev
+version: 6.0.1-dev
 groups: python
 dbscheme: semmlecode.python.dbscheme
 extractor: python
@@ -19,3 +19,4 @@ dataExtensions:
   - semmle/python/frameworks/**/*.model.yml
   - ext/*.model.yml
 warnOnImplicitThis: true
+compileForOverlayEval: true

python/ql/lib/semmle/python/AstExtended.qll

@@ -218,6 +218,9 @@ class DictItemListParent extends DictItemListParent_ { }
 /** A list of strings (the primitive type string not Bytes or Unicode) */
 class StringList extends StringList_ { }
 
+/** A list of template strings. */
+class TemplateStringList extends TemplateStringList_ { }
+
 /** A list of aliases in an import statement */
 class AliasList extends AliasList_ { }
 
@@ -273,3 +276,9 @@ class ParamSpec extends ParamSpec_, TypeParameter {
 
   override Expr getAChildNode() { result = this.getName() }
 }
+
+/** A template string literal. */
+class TemplateString extends TemplateString_, Expr { }
+
+/** An (implicitly) concatenated list of template strings. */
+class JoinedTemplateString extends JoinedTemplateString_, Expr { }

python/ql/lib/semmle/python/AstGenerated.qll

@@ -768,6 +768,20 @@ class Fstring_ extends @py_Fstring, Expr {
   override string toString() { result = "Fstring" }
 }
 
+/** INTERNAL: See the class `JoinedTemplateString` for further information. */
+class JoinedTemplateString_ extends @py_JoinedTemplateString, Expr {
+  /** Gets the strings of this joined template string. */
+  TemplateStringList getStrings() { py_TemplateString_lists(result, this) }
+
+  /** Gets the nth string of this joined template string. */
+  TemplateString getString(int index) { result = this.getStrings().getItem(index) }
+
+  /** Gets a string of this joined template string. */
+  TemplateString getAString() { result = this.getStrings().getAnItem() }
+
+  override string toString() { result = "JoinedTemplateString" }
+}
+
 /** INTERNAL: See the class `KeyValuePair` for further information. */
 class KeyValuePair_ extends @py_KeyValuePair, DictItem {
   /** Gets the location of this key-value pair. */
@@ -1373,6 +1387,48 @@ class TemplateDottedNotation_ extends @py_TemplateDottedNotation, Expr {
   override string toString() { result = "TemplateDottedNotation" }
 }
 
+/** INTERNAL: See the class `TemplateString` for further information. */
+class TemplateString_ extends @py_TemplateString, Expr {
+  /** Gets the prefix of this template string literal. */
+  string getPrefix() { py_strs(result, this, 2) }
+
+  /** Gets the values of this template string literal. */
+  ExprList getValues() { py_expr_lists(result, this, 3) }
+
+  /** Gets the nth value of this template string literal. */
+  Expr getValue(int index) { result = this.getValues().getItem(index) }
+
+  /** Gets a value of this template string literal. */
+  Expr getAValue() { result = this.getValues().getAnItem() }
+
+  override ExprParent getParent() { py_exprs(this, _, result, _) }
+
+  override string toString() { result = "TemplateString" }
+}
+
+/** INTERNAL: See the class `TemplateStringPart` for further information. */
+class TemplateStringPart_ extends @py_TemplateStringPart, Expr {
+  /** Gets the text of this string part of a template string. */
+  string getText() { py_strs(result, this, 2) }
+
+  override string toString() { result = "TemplateStringPart" }
+}
+
+/** INTERNAL: See the class `TemplateStringList` for further information. */
+class TemplateStringList_ extends @py_TemplateString_list {
+  /** Gets a parent of this template string literal list */
+  JoinedTemplateString getParent() { py_TemplateString_lists(this, result) }
+
+  /** Gets an item of this template string literal list */
+  Expr getAnItem() { py_exprs(result, _, this, _) }
+
+  /** Gets the nth item of this template string literal list */
+  Expr getItem(int index) { py_exprs(result, _, this, index) }
+
+  /** Gets a textual representation of this element. */
+  string toString() { result = "TemplateStringList" }
+}
+
 /** INTERNAL: See the class `TemplateWrite` for further information. */
 class TemplateWrite_ extends @py_TemplateWrite, Stmt {
   /** Gets the value of this template write statement. */

python/ql/lib/semmle/python/Class.qll

@@ -141,18 +141,12 @@ class Class extends Class_, Scope, AstNode {
   /** Gets the metaclass expression */
   Expr getMetaClass() { result = this.getParent().getMetaClass() }
 
-  /** Gets the ClassObject corresponding to this class */
-  ClassObject getClassObject() { result.getOrigin() = this.getParent() }
-
   /** Gets the nth base of this class definition. */
   Expr getBase(int index) { result = this.getParent().getBase(index) }
 
   /** Gets a base of this class definition. */
   Expr getABase() { result = this.getParent().getABase() }
 
-  /** Gets the metrics for this class */
-  ClassMetrics getMetrics() { result = this }
-
   /**
    * Gets the qualified name for this class.
    * Should return the same name as the `__qualname__` attribute on classes in Python 3.

python/ql/lib/semmle/python/Concepts.qll

@@ -12,6 +12,7 @@ private import semmle.python.dataflow.new.TaintTracking
 private import semmle.python.Files
 private import semmle.python.Frameworks
 private import semmle.python.security.internal.EncryptionKeySizes
+private import semmle.python.dataflow.new.SensitiveDataSources
 private import codeql.threatmodels.ThreatModels
 private import codeql.concepts.ConceptsShared
 
@@ -115,6 +116,16 @@ module SystemCommandExecution {
 class FileSystemAccess extends DataFlow::Node instanceof FileSystemAccess::Range {
   /** Gets an argument to this file system access that is interpreted as a path. */
   DataFlow::Node getAPathArgument() { result = super.getAPathArgument() }
+
+  /**
+   * Gets an argument to this file system access that is interpreted as a path
+   * which is vulnerable to path injection.
+   *
+   * By default all path arguments are considered vulnerable, but this can be overridden to
+   * exclude certain arguments that are known to be safe, for example because they are
+   * restricted to a specific directory.
+   */
+  DataFlow::Node getAVulnerablePathArgument() { result = super.getAVulnerablePathArgument() }
 }
 
 /** Provides a class for modeling new file system access APIs. */
@@ -129,6 +140,16 @@ module FileSystemAccess {
   abstract class Range extends DataFlow::Node {
     /** Gets an argument to this file system access that is interpreted as a path. */
     abstract DataFlow::Node getAPathArgument();
+
+    /**
+     * Gets an argument to this file system access that is interpreted as a path
+     * which is vulnerable to path injection.
+     *
+     * By default all path arguments are considered vulnerable, but this can be overridden to
+     * exclude certain arguments that are known to be safe, for example because they are
+     * restricted to a specific directory.
+     */
+    DataFlow::Node getAVulnerablePathArgument() { result = this.getAPathArgument() }
   }
 }
 
@@ -1290,6 +1311,18 @@ module Http {
        */
       DataFlow::Node getValueArg() { result = super.getValueArg() }
 
+      /** Holds if the name of this cookie indicates it may contain sensitive information. */
+      predicate isSensitive() {
+        exists(DataFlow::Node name |
+          name = [this.getNameArg(), this.getHeaderArg()] and
+          (
+            DataFlow::localFlow(any(SensitiveDataSource src), name)
+            or
+            name = sensitiveLookupStringConst(_)
+          )
+        )
+      }
+
       /**
        * Holds if the `Secure` flag of the cookie is known to have a value of `b`.
        */

python/ql/lib/semmle/python/Exprs.qll

@@ -1,6 +1,4 @@
-import python
-private import semmle.python.pointsto.PointsTo
-private import semmle.python.objects.ObjectInternal
+private import python
 private import semmle.python.internal.CachedStages
 
 /** An expression */
@@ -52,67 +50,6 @@ class Expr extends Expr_, AstNode {
   Expr getASubExpression() { none() }
 
   override AstNode getAChildNode() { result = this.getASubExpression() }
-
-  /**
-   * NOTE: `refersTo` will be deprecated in 2019. Use `pointsTo` instead.
-   * Gets what this expression might "refer-to". Performs a combination of localized (intra-procedural) points-to
-   *  analysis and global module-level analysis. This points-to analysis favours precision over recall. It is highly
-   *  precise, but may not provide information for a significant number of flow-nodes.
-   *  If the class is unimportant then use `refersTo(value)` or `refersTo(value, origin)` instead.
-   * NOTE: For complex dataflow, involving multiple stages of points-to analysis, it may be more precise to use
-   * `ControlFlowNode.refersTo(...)` instead.
-   */
-  predicate refersTo(Object obj, ClassObject cls, AstNode origin) {
-    this.refersTo(_, obj, cls, origin)
-  }
-
-  /**
-   * NOTE: `refersTo` will be deprecated in 2019. Use `pointsTo` instead.
-   * Gets what this expression might "refer-to" in the given `context`.
-   */
-  predicate refersTo(Context context, Object obj, ClassObject cls, AstNode origin) {
-    this.getAFlowNode().refersTo(context, obj, cls, origin.getAFlowNode())
-  }
-
-  /**
-   * NOTE: `refersTo` will be deprecated in 2019. Use `pointsTo` instead.
-   * Holds if this expression might "refer-to" to `value` which is from `origin`
-   * Unlike `this.refersTo(value, _, origin)`, this predicate includes results
-   * where the class cannot be inferred.
-   */
-  pragma[nomagic]
-  predicate refersTo(Object obj, AstNode origin) {
-    this.getAFlowNode().refersTo(obj, origin.getAFlowNode())
-  }
-
-  /**
-   * NOTE: `refersTo` will be deprecated in 2019. Use `pointsTo` instead.
-   * Equivalent to `this.refersTo(value, _)`
-   */
-  predicate refersTo(Object obj) { this.refersTo(obj, _) }
-
-  /**
-   * Holds if this expression might "point-to" to `value` which is from `origin`
-   * in the given `context`.
-   */
-  predicate pointsTo(Context context, Value value, AstNode origin) {
-    this.getAFlowNode().pointsTo(context, value, origin.getAFlowNode())
-  }
-
-  /**
-   * Holds if this expression might "point-to" to `value` which is from `origin`.
-   */
-  predicate pointsTo(Value value, AstNode origin) {
-    this.getAFlowNode().pointsTo(value, origin.getAFlowNode())
-  }
-
-  /**
-   * Holds if this expression might "point-to" to `value`.
-   */
-  predicate pointsTo(Value value) { this.pointsTo(value, _) }
-
-  /** Gets a value that this expression might "point-to". */
-  Value pointsTo() { this.pointsTo(result) }
 }
 
 /** An assignment expression, such as `x := y` */
@@ -303,17 +240,12 @@ class Bytes extends StringLiteral {
   /* syntax: b"hello" */
   Bytes() { not this.isUnicode() }
 
-  override Object getLiteralObject() {
-    py_cobjecttypes(result, theBytesType()) and
-    py_cobjectnames(result, this.quotedString())
-  }
-
   /**
    * The extractor puts quotes into the name of each string (to prevent "0" clashing with 0).
    * The following predicate help us match up a string/byte literals in the source
    * which the equivalent object.
    */
-  private string quotedString() {
+  string quotedString() {
     exists(string b_unquoted | b_unquoted = this.getS() | result = "b'" + b_unquoted + "'")
   }
 }
@@ -329,11 +261,7 @@ class Ellipsis extends Ellipsis_ {
  * Consists of string (both unicode and byte) literals and numeric literals.
  */
 abstract class ImmutableLiteral extends Expr {
-  abstract Object getLiteralObject();
-
   abstract boolean booleanValue();
-
-  final Value getLiteralValue() { result.(ConstantObjectInternal).getLiteral() = this }
 }
 
 /** A numerical constant expression, such as `7` or `4.2` */
@@ -357,12 +285,6 @@ class IntegerLiteral extends Num {
 
   override string toString() { result = "IntegerLiteral" }
 
-  override Object getLiteralObject() {
-    py_cobjecttypes(result, theIntType()) and py_cobjectnames(result, this.getN())
-    or
-    py_cobjecttypes(result, theLongType()) and py_cobjectnames(result, this.getN())
-  }
-
   override boolean booleanValue() {
     this.getValue() = 0 and result = false
     or
@@ -382,10 +304,6 @@ class FloatLiteral extends Num {
 
   override string toString() { result = "FloatLiteral" }
 
-  override Object getLiteralObject() {
-    py_cobjecttypes(result, theFloatType()) and py_cobjectnames(result, this.getN())
-  }
-
   override boolean booleanValue() {
     this.getValue() = 0.0 and result = false
     or
@@ -408,10 +326,6 @@ class ImaginaryLiteral extends Num {
 
   override string toString() { result = "ImaginaryLiteral" }
 
-  override Object getLiteralObject() {
-    py_cobjecttypes(result, theComplexType()) and py_cobjectnames(result, this.getN())
-  }
-
   override boolean booleanValue() {
     this.getValue() = 0.0 and result = false
     or
@@ -430,11 +344,6 @@ class NegativeIntegerLiteral extends ImmutableLiteral, UnaryExpr {
 
   override boolean booleanValue() { result = this.getOperand().(IntegerLiteral).booleanValue() }
 
-  override Object getLiteralObject() {
-    (py_cobjecttypes(result, theIntType()) or py_cobjecttypes(result, theLongType())) and
-    py_cobjectnames(result, "-" + this.getOperand().(IntegerLiteral).getN())
-  }
-
   /**
    * Gets the (integer) value of this constant. Will not return a result if the value does not fit into
    * a 32 bit signed value
@@ -450,11 +359,6 @@ class Unicode extends StringLiteral {
   /* syntax: "hello" */
   Unicode() { this.isUnicode() }
 
-  override Object getLiteralObject() {
-    py_cobjecttypes(result, theUnicodeType()) and
-    py_cobjectnames(result, this.quotedString())
-  }
-
   /**
    * Gets the quoted representation fo this string.
    *
@@ -658,12 +562,11 @@ class StringLiteral extends Str_, ImmutableLiteral {
     this.getText() != "" and result = true
   }
 
-  override Object getLiteralObject() { none() }
-
   override string toString() { result = "StringLiteral" }
 }
 
-private predicate name_consts(Name_ n, string id) {
+/** Holds if `n` is a named constant (`True`, `False`, or `None`) with name `id`. */
+predicate name_consts(Name_ n, string id) {
   exists(Variable v | py_variables(v, n) and id = v.getId() |
     id = "True" or id = "False" or id = "None"
   )
@@ -692,8 +595,6 @@ class True extends BooleanLiteral {
   /* syntax: True */
   True() { name_consts(this, "True") }
 
-  override Object getLiteralObject() { name_consts(this, "True") and result = theTrueObject() }
-
   override boolean booleanValue() { result = true }
 }
 
@@ -702,8 +603,6 @@ class False extends BooleanLiteral {
   /* syntax: False */
   False() { name_consts(this, "False") }
 
-  override Object getLiteralObject() { name_consts(this, "False") and result = theFalseObject() }
-
   override boolean booleanValue() { result = false }
 }
 
@@ -712,8 +611,6 @@ class None extends NameConstant {
   /* syntax: None */
   None() { name_consts(this, "None") }
 
-  override Object getLiteralObject() { name_consts(this, "None") and result = theNoneObject() }
-
   override boolean booleanValue() { result = false }
 }
 

python/ql/lib/semmle/python/Flow.qll

@@ -1,5 +1,4 @@
 import python
-private import semmle.python.pointsto.PointsTo
 private import semmle.python.internal.CachedStages
 private import codeql.controlflow.BasicBlock as BB
 
@@ -144,56 +143,6 @@ class ControlFlowNode extends @py_flow_node {
   /** Whether this flow node is the first in its scope */
   predicate isEntryNode() { py_scope_flow(this, _, -1) }
 
-  /** Gets the value that this ControlFlowNode points-to. */
-  predicate pointsTo(Value value) { this.pointsTo(_, value, _) }
-
-  /** Gets the value that this ControlFlowNode points-to. */
-  Value pointsTo() { this.pointsTo(_, result, _) }
-
-  /** Gets a value that this ControlFlowNode may points-to. */
-  Value inferredValue() { this.pointsTo(_, result, _) }
-
-  /** Gets the value and origin that this ControlFlowNode points-to. */
-  predicate pointsTo(Value value, ControlFlowNode origin) { this.pointsTo(_, value, origin) }
-
-  /** Gets the value and origin that this ControlFlowNode points-to, given the context. */
-  predicate pointsTo(Context context, Value value, ControlFlowNode origin) {
-    PointsTo::pointsTo(this, context, value, origin)
-  }
-
-  /**
-   * Gets what this flow node might "refer-to". Performs a combination of localized (intra-procedural) points-to
-   * analysis and global module-level analysis. This points-to analysis favours precision over recall. It is highly
-   * precise, but may not provide information for a significant number of flow-nodes.
-   * If the class is unimportant then use `refersTo(value)` or `refersTo(value, origin)` instead.
-   */
-  pragma[nomagic]
-  predicate refersTo(Object obj, ClassObject cls, ControlFlowNode origin) {
-    this.refersTo(_, obj, cls, origin)
-  }
-
-  /** Gets what this expression might "refer-to" in the given `context`. */
-  pragma[nomagic]
-  predicate refersTo(Context context, Object obj, ClassObject cls, ControlFlowNode origin) {
-    not obj = unknownValue() and
-    not cls = theUnknownType() and
-    PointsTo::points_to(this, context, obj, cls, origin)
-  }
-
-  /**
-   * Whether this flow node might "refer-to" to `value` which is from `origin`
-   * Unlike `this.refersTo(value, _, origin)` this predicate includes results
-   * where the class cannot be inferred.
-   */
-  pragma[nomagic]
-  predicate refersTo(Object obj, ControlFlowNode origin) {
-    not obj = unknownValue() and
-    PointsTo::points_to(this, _, obj, _, origin)
-  }
-
-  /** Equivalent to `this.refersTo(value, _)` */
-  predicate refersTo(Object obj) { this.refersTo(obj, _) }
-
   /** Gets the basic block containing this flow node */
   BasicBlock getBasicBlock() { result.contains(this) }
 
@@ -241,41 +190,6 @@ class ControlFlowNode extends @py_flow_node {
   /** Whether this node is a normal (non-exceptional) exit */
   predicate isNormalExit() { py_scope_flow(this, _, 0) or py_scope_flow(this, _, 2) }
 
-  /** Whether it is unlikely that this ControlFlowNode can be reached */
-  predicate unlikelyReachable() {
-    not start_bb_likely_reachable(this.getBasicBlock())
-    or
-    exists(BasicBlock b |
-      start_bb_likely_reachable(b) and
-      not end_bb_likely_reachable(b) and
-      // If there is an unlikely successor edge earlier in the BB
-      // than this node, then this node must be unreachable.
-      exists(ControlFlowNode p, int i, int j |
-        p.(RaisingNode).unlikelySuccessor(_) and
-        p = b.getNode(i) and
-        this = b.getNode(j) and
-        i < j
-      )
-    )
-  }
-
-  /**
-   * Check whether this control-flow node has complete points-to information.
-   * This would mean that the analysis managed to infer an over approximation
-   * of possible values at runtime.
-   */
-  predicate hasCompletePointsToSet() {
-    // If the tracking failed, then `this` will be its own "origin". In that
-    // case, we want to exclude nodes for which there is also a different
-    // origin, as that would indicate that some paths failed and some did not.
-    this.refersTo(_, _, this) and
-    not exists(ControlFlowNode other | other != this and this.refersTo(_, _, other))
-    or
-    // If `this` is a use of a variable, then we must have complete points-to
-    // for that variable.
-    exists(SsaVariable v | v.getAUse() = this | varHasCompletePointsToSet(v))
-  }
-
   /** Whether this strictly dominates other. */
   pragma[inline]
   predicate strictlyDominates(ControlFlowNode other) {
@@ -332,28 +246,6 @@ private class AnyNode extends ControlFlowNode {
   override AstNode getNode() { result = super.getNode() }
 }
 
-/**
- * Check whether a SSA variable has complete points-to information.
- * This would mean that the analysis managed to infer an overapproximation
- * of possible values at runtime.
- */
-private predicate varHasCompletePointsToSet(SsaVariable var) {
-  // Global variables may be modified non-locally or concurrently.
-  not var.getVariable() instanceof GlobalVariable and
-  (
-    // If we have complete points-to information on the definition of
-    // this variable, then the variable has complete information.
-    var.getDefinition().(DefinitionNode).getValue().hasCompletePointsToSet()
-    or
-    // If this variable is a phi output, then we have complete
-    // points-to information about it if all phi inputs had complete
-    // information.
-    forex(SsaVariable phiInput | phiInput = var.getAPhiInput() |
-      varHasCompletePointsToSet(phiInput)
-    )
-  )
-}
-
 /** A control flow node corresponding to a call expression, such as `func(...)` */
 class CallNode extends ControlFlowNode {
   CallNode() { toAst(this) instanceof Call }
@@ -991,6 +883,58 @@ class StarredNode extends ControlFlowNode {
   ControlFlowNode getValue() { toAst(result) = toAst(this).(Starred).getValue() }
 }
 
+/** The ControlFlowNode for an 'except' statement. */
+class ExceptFlowNode extends ControlFlowNode {
+  ExceptFlowNode() { this.getNode() instanceof ExceptStmt }
+
+  /**
+   * Gets the type handled by this exception handler.
+   * `ExceptionType` in `except ExceptionType as e:`
+   */
+  ControlFlowNode getType() {
+    exists(ExceptStmt ex |
+      this.getBasicBlock().dominates(result.getBasicBlock()) and
+      ex = this.getNode() and
+      result = ex.getType().getAFlowNode()
+    )
+  }
+
+  /**
+   * Gets the name assigned to the handled exception, if any.
+   * `e` in `except ExceptionType as e:`
+   */
+  ControlFlowNode getName() {
+    exists(ExceptStmt ex |
+      this.getBasicBlock().dominates(result.getBasicBlock()) and
+      ex = this.getNode() and
+      result = ex.getName().getAFlowNode()
+    )
+  }
+}
+
+/** The ControlFlowNode for an 'except*' statement. */
+class ExceptGroupFlowNode extends ControlFlowNode {
+  ExceptGroupFlowNode() { this.getNode() instanceof ExceptGroupStmt }
+
+  /**
+   * Gets the type handled by this exception handler.
+   * `ExceptionType` in `except* ExceptionType as e:`
+   */
+  ControlFlowNode getType() {
+    this.getBasicBlock().dominates(result.getBasicBlock()) and
+    result = this.getNode().(ExceptGroupStmt).getType().getAFlowNode()
+  }
+
+  /**
+   * Gets the name assigned to the handled exception, if any.
+   * `e` in `except* ExceptionType as e:`
+   */
+  ControlFlowNode getName() {
+    this.getBasicBlock().dominates(result.getBasicBlock()) and
+    result = this.getNode().(ExceptGroupStmt).getName().getAFlowNode()
+  }
+}
+
 private module Scopes {
   private predicate fast_local(NameNode n) {
     exists(FastLocalVariable v |
@@ -1094,7 +1038,8 @@ class BasicBlock extends @py_flow_node {
     )
   }
 
-  private ControlFlowNode firstNode() { result = this }
+  /** Gets the first node in this basic block */
+  ControlFlowNode firstNode() { result = this }
 
   /** Gets the last node in this basic block */
   ControlFlowNode getLastNode() {
@@ -1183,15 +1128,6 @@ class BasicBlock extends @py_flow_node {
     )
   }
 
-  /**
-   * Whether (as inferred by type inference) it is highly unlikely (or impossible) for control to flow from this to succ.
-   */
-  predicate unlikelySuccessor(BasicBlock succ) {
-    this.getLastNode().(RaisingNode).unlikelySuccessor(succ.firstNode())
-    or
-    not end_bb_likely_reachable(this) and succ = this.getASuccessor()
-  }
-
   /** Holds if this basic block strictly reaches the other. Is the start of other reachable from the end of this. */
   cached
   predicate strictlyReaches(BasicBlock other) {
@@ -1202,11 +1138,6 @@ class BasicBlock extends @py_flow_node {
   /** Holds if this basic block reaches the other. Is the start of other reachable from the end of this. */
   predicate reaches(BasicBlock other) { this = other or this.strictlyReaches(other) }
 
-  /**
-   * Whether (as inferred by type inference) this basic block is likely to be reachable.
-   */
-  predicate likelyReachable() { start_bb_likely_reachable(this) }
-
   /**
    * Gets the `ConditionBlock`, if any, that controls this block and
    * does not control any other `ConditionBlock`s that control this block.
@@ -1234,26 +1165,6 @@ class BasicBlock extends @py_flow_node {
   }
 }
 
-private predicate start_bb_likely_reachable(BasicBlock b) {
-  exists(Scope s | s.getEntryNode() = b.getNode(_))
-  or
-  exists(BasicBlock pred |
-    pred = b.getAPredecessor() and
-    end_bb_likely_reachable(pred) and
-    not pred.getLastNode().(RaisingNode).unlikelySuccessor(b)
-  )
-}
-
-private predicate end_bb_likely_reachable(BasicBlock b) {
-  start_bb_likely_reachable(b) and
-  not exists(ControlFlowNode p, ControlFlowNode s |
-    p.(RaisingNode).unlikelySuccessor(s) and
-    p = b.getNode(_) and
-    s = b.getNode(_) and
-    not p = b.getLastNode()
-  )
-}
-
 private class ControlFlowNodeAlias = ControlFlowNode;
 
 final private class FinalBasicBlock = BasicBlock;

python/ql/lib/semmle/python/Frameworks.qll

@@ -78,6 +78,7 @@ private import semmle.python.frameworks.Sanic
 private import semmle.python.frameworks.ServerLess
 private import semmle.python.frameworks.Setuptools
 private import semmle.python.frameworks.Simplejson
+private import semmle.python.frameworks.Socketio
 private import semmle.python.frameworks.SqlAlchemy
 private import semmle.python.frameworks.SSRFSink
 private import semmle.python.frameworks.Starlette
@@ -90,6 +91,7 @@ private import semmle.python.frameworks.TRender
 private import semmle.python.frameworks.Twisted
 private import semmle.python.frameworks.Ujson
 private import semmle.python.frameworks.Urllib3
+private import semmle.python.frameworks.Websockets
 private import semmle.python.frameworks.Xmltodict
 private import semmle.python.frameworks.Yaml
 private import semmle.python.frameworks.Yarl

python/ql/lib/semmle/python/Function.qll

@@ -84,12 +84,6 @@ class Function extends Function_, Scope, AstNode {
   /** Gets the name used to define this function */
   override string getName() { result = Function_.super.getName() }
 
-  /** Gets the metrics for this function */
-  FunctionMetrics getMetrics() { result = this }
-
-  /** Gets the FunctionObject corresponding to this function */
-  FunctionObject getFunctionObject() { result.getOrigin() = this.getDefinition() }
-
   /**
    * Whether this function is a procedure, that is, it has no explicit return statement and always returns None.
    * Note that generator and async functions are not procedures as they return generators and coroutines respectively.

python/ql/lib/semmle/python/Metrics.qll

@@ -1,4 +1,5 @@
 import python
+private import LegacyPointsTo
 
 /** The metrics for a function */
 class FunctionMetrics extends Function {
@@ -28,9 +29,9 @@ class FunctionMetrics extends Function {
    */
   int getCyclomaticComplexity() {
     exists(int e, int n |
-      n = count(BasicBlock b | b = this.getABasicBlock() and b.likelyReachable()) and
+      n = count(BasicBlockWithPointsTo b | b = this.getABasicBlock() and b.likelyReachable()) and
       e =
-        count(BasicBlock b1, BasicBlock b2 |
+        count(BasicBlockWithPointsTo b1, BasicBlockWithPointsTo b2 |
           b1 = this.getABasicBlock() and
           b1.likelyReachable() and
           b2 = this.getABasicBlock() and
@@ -59,7 +60,7 @@ class FunctionMetrics extends Function {
     not non_coupling_method(result) and
     exists(Call call | call.getScope() = this |
       exists(FunctionObject callee | callee.getFunction() = result |
-        call.getAFlowNode().getFunction().refersTo(callee)
+        call.getAFlowNode().getFunction().(ControlFlowNodeWithPointsTo).refersTo(callee)
       )
       or
       exists(Attribute a | call.getFunc() = a |
@@ -123,7 +124,7 @@ class ClassMetrics extends Class {
       )
       or
       exists(Function f, Call c, ClassObject cls | c.getScope() = f and f.getScope() = this |
-        c.getFunc().refersTo(cls) and
+        c.getFunc().(ExprWithPointsTo).refersTo(cls) and
         cls.getPyClass() = other
       )
     )
@@ -292,7 +293,7 @@ class ModuleMetrics extends Module {
       )
       or
       exists(Function f, Call c, ClassObject cls | c.getScope() = f and f.getScope() = this |
-        c.getFunc().refersTo(cls) and
+        c.getFunc().(ExprWithPointsTo).refersTo(cls) and
         cls.getPyClass().getEnclosingModule() = other
       )
     )

python/ql/lib/semmle/python/Module.qll

@@ -1,5 +1,4 @@
 import python
-private import semmle.python.objects.Modules
 private import semmle.python.internal.CachedStages
 
 /**
@@ -66,15 +65,6 @@ class Module extends Module_, Scope, AstNode {
   /** Whether this module is a package initializer */
   predicate isPackageInit() { this.getName().matches("%\\_\\_init\\_\\_") and not this.isPackage() }
 
-  /** Gets a name exported by this module, that is the names that will be added to a namespace by 'from this-module import *' */
-  string getAnExport() {
-    py_exports(this, result)
-    or
-    exists(ModuleObjectInternal mod | mod.getSource() = this.getEntryNode() |
-      mod.(ModuleValue).exports(result)
-    )
-  }
-
   /** Gets the source file for this module */
   File getFile() { py_module_path(this, result) }
 
@@ -96,9 +86,6 @@ class Module extends Module_, Scope, AstNode {
     result = this.getName().regexpReplaceAll("\\.[^.]*$", "")
   }
 
-  /** Gets the metrics for this module */
-  ModuleMetrics getMetrics() { result = this }
-
   string getAnImportedModuleName() {
     exists(Import i | i.getEnclosingModule() = this | result = i.getAnImportedModuleName())
     or

python/ql/lib/semmle/python/Overlay.qll

@@ -0,0 +1,330 @@
+/**
+ * Defines entity discard predicates for Python overlay analysis.
+ */
+
+private import internal.OverlayXml
+
+/*- Predicates -*/
+/**
+ * Holds always for the overlay variant and never for the base variant.
+ * This local predicate is used to define local predicates that behave
+ * differently for the base and overlay variant.
+ */
+overlay[local]
+predicate isOverlay() { databaseMetadata("isOverlay", "true") }
+
+overlay[local]
+private string getPathForLocation(@location loc) {
+  exists(@file file | locations_default(loc, file, _, _, _, _) | files(file, result))
+  or
+  exists(@py_Module mod | locations_ast(loc, mod, _, _, _, _) | result = getPathForModule(mod))
+}
+
+overlay[local]
+private string getPathForModule(@py_Module mod) {
+  exists(@container fileOrFolder | py_module_path(mod, fileOrFolder) |
+    result = getPathForContainer(fileOrFolder)
+  )
+}
+
+overlay[local]
+private string getPathForContainer(@container fileOrFolder) {
+  files(fileOrFolder, result) or folders(fileOrFolder, result)
+}
+
+/*- Discardable entities and their discard predicates -*/
+/** Python database entities that use named TRAP IDs; the rest use *-ids. */
+overlay[local]
+private class NamedEntity = @py_Module or @container or @py_cobject;
+
+overlay[discard_entity]
+private predicate discardNamedEntity(@top el) {
+  el instanceof NamedEntity and
+  // Entities with named IDs can exist both in base, overlay, or both.
+  exists(Discardable d | d = el |
+    overlayChangedFiles(d.getPath()) and
+    not d.existsInOverlay()
+  )
+}
+
+overlay[discard_entity]
+private predicate discardStarEntity(@top el) {
+  not el instanceof NamedEntity and
+  // Entities with *-ids can exist either in base or overlay, but not both.
+  exists(Discardable d | d = el |
+    overlayChangedFiles(d.getPath()) and
+    d.existsInBase()
+  )
+}
+
+/**
+ * An abstract base class for all elements that can be discarded from the base.
+ */
+overlay[local]
+abstract class Discardable extends @top {
+  /** Gets the path to the file in which this element occurs. */
+  abstract string getPath();
+
+  /** Holds if this element exists in the base variant. */
+  predicate existsInBase() { not isOverlay() and exists(this) }
+
+  /** Holds if this element exists in the overlay variant. */
+  predicate existsInOverlay() { isOverlay() and exists(this) }
+
+  /** Gets a textual representation of this discardable element. */
+  string toString() { none() }
+}
+
+/**
+ * Discardable locatable AST nodes (`@py_location_parent`).
+ */
+overlay[local]
+final private class DiscardableLocatable extends Discardable instanceof @py_location_parent {
+  override string getPath() {
+    exists(@location loc | py_locations(loc, this) | result = getPathForLocation(loc))
+  }
+}
+
+/**
+ * Discardable scopes (classes, functions, modules).
+ */
+overlay[local]
+final private class DiscardableScope extends Discardable instanceof @py_scope {
+  override string getPath() {
+    exists(@location loc | py_scope_location(loc, this) | result = getPathForLocation(loc))
+    or
+    result = getPathForModule(this)
+  }
+}
+
+/**
+ * Discardable files and folders.
+ */
+overlay[local]
+final private class DiscardableContainer extends Discardable instanceof @container {
+  override string getPath() { result = getPathForContainer(this) }
+}
+
+/** Discardable control flow nodes */
+overlay[local]
+final private class DiscardableCfgNode extends Discardable instanceof @py_flow_node {
+  override string getPath() {
+    exists(Discardable d | result = d.getPath() | py_flow_bb_node(this, d.(@py_ast_node), _, _))
+  }
+}
+
+/** Discardable Python variables. */
+overlay[local]
+final private class DiscardableVar extends Discardable instanceof @py_variable {
+  override string getPath() {
+    exists(Discardable parent | result = parent.getPath() | variable(this, parent.(@py_scope), _))
+  }
+}
+
+/** Discardable SSA variables. */
+overlay[local]
+final private class DiscardableSsaVar extends Discardable instanceof @py_ssa_var {
+  override string getPath() {
+    exists(DiscardableVar other | result = other.getPath() | py_ssa_var(this, other))
+  }
+}
+
+/** Discardable locations. */
+overlay[local]
+final private class DiscardableLocation extends Discardable instanceof @location {
+  override string getPath() { result = getPathForLocation(this) }
+}
+
+/** Discardable lines. */
+overlay[local]
+final private class DiscardableLine extends Discardable instanceof @py_line {
+  override string getPath() {
+    exists(Discardable d | result = d.getPath() | py_line_lengths(this, d.(@py_Module), _, _))
+  }
+}
+
+/** Discardable string part lists. */
+overlay[local]
+final private class DiscardableStringPartList extends Discardable instanceof @py_StringPart_list {
+  override string getPath() {
+    exists(Discardable d | result = d.getPath() | py_StringPart_lists(this, d.(@py_Bytes_or_Str)))
+  }
+}
+
+/** Discardable alias */
+overlay[local]
+final private class DiscardableAlias extends Discardable instanceof @py_alias {
+  override string getPath() {
+    exists(DiscardableAliasList d | result = d.getPath() | py_aliases(this, d, _))
+  }
+}
+
+/** Discardable alias list */
+overlay[local]
+final private class DiscardableAliasList extends Discardable instanceof @py_alias_list {
+  override string getPath() {
+    exists(Discardable d | result = d.getPath() | py_alias_lists(this, d.(@py_Import)))
+  }
+}
+
+/** Discardable arguments */
+overlay[local]
+final private class DiscardableArguments extends Discardable instanceof @py_arguments {
+  override string getPath() {
+    exists(Discardable d | result = d.getPath() | py_arguments(this, d.(@py_arguments_parent)))
+  }
+}
+
+/** Discardable boolop */
+overlay[local]
+final private class DiscardableBoolOp extends Discardable instanceof @py_boolop {
+  override string getPath() {
+    exists(Discardable d | result = d.getPath() | py_boolops(this, _, d.(@py_BoolExpr)))
+  }
+}
+
+/** Discardable cmpop */
+overlay[local]
+final private class DiscardableCmpOp extends Discardable instanceof @py_cmpop {
+  override string getPath() {
+    exists(DiscardableCmpOpList d | result = d.getPath() | py_cmpops(this, _, d, _))
+  }
+}
+
+/** Discardable cmpop list */
+overlay[local]
+final private class DiscardableCmpOpList extends Discardable instanceof @py_cmpop_list {
+  override string getPath() {
+    exists(Discardable d | result = d.getPath() | py_cmpop_lists(this, d.(@py_Compare)))
+  }
+}
+
+/** Discardable comprehension list */
+overlay[local]
+final private class DiscardableComprehensionList extends Discardable instanceof @py_comprehension_list
+{
+  override string getPath() {
+    exists(Discardable d | result = d.getPath() | py_comprehension_lists(this, d.(@py_ListComp)))
+  }
+}
+
+/** Discardable dict item list */
+overlay[local]
+final private class DiscardableDictItemList extends Discardable instanceof @py_dict_item_list {
+  override string getPath() {
+    exists(Discardable d | result = d.getPath() |
+      py_dict_item_lists(this, d.(@py_dict_item_list_parent))
+    )
+  }
+}
+
+/** Discardable expr context */
+overlay[local]
+final private class DiscardableExprContext extends Discardable instanceof @py_expr_context {
+  override string getPath() {
+    exists(Discardable d | result = d.getPath() |
+      py_expr_contexts(this, _, d.(@py_expr_context_parent))
+    )
+  }
+}
+
+/** Discardable expr list */
+overlay[local]
+final private class DiscardableExprList extends Discardable instanceof @py_expr_list {
+  override string getPath() {
+    exists(Discardable d | result = d.getPath() | py_expr_lists(this, d.(@py_expr_list_parent), _))
+  }
+}
+
+/** Discardable operator */
+overlay[local]
+final private class DiscardableOperator extends Discardable instanceof @py_operator {
+  override string getPath() {
+    exists(Discardable d | result = d.getPath() | py_operators(this, _, d.(@py_BinaryExpr)))
+  }
+}
+
+/** Discardable parameter list */
+overlay[local]
+final private class DiscardableParameterList extends Discardable instanceof @py_parameter_list {
+  override string getPath() {
+    exists(Discardable d | result = d.getPath() | py_parameter_lists(this, d.(@py_Function)))
+  }
+}
+
+/** Discardable pattern list */
+overlay[local]
+final private class DiscardablePatternList extends Discardable instanceof @py_pattern_list {
+  override string getPath() {
+    exists(Discardable d | result = d.getPath() |
+      py_pattern_lists(this, d.(@py_pattern_list_parent), _)
+    )
+  }
+}
+
+/** Discardable stmt list */
+overlay[local]
+final private class DiscardableStmtList extends Discardable instanceof @py_stmt_list {
+  override string getPath() {
+    exists(Discardable d | result = d.getPath() | py_stmt_lists(this, d.(@py_stmt_list_parent), _))
+  }
+}
+
+/** Discardable str list */
+overlay[local]
+final private class DiscardableStrList extends Discardable instanceof @py_str_list {
+  override string getPath() {
+    exists(Discardable d | result = d.getPath() | py_str_lists(this, d.(@py_str_list_parent)))
+  }
+}
+
+/** Discardable type parameter list */
+overlay[local]
+final private class DiscardableTypeParameterList extends Discardable instanceof @py_type_parameter_list
+{
+  override string getPath() {
+    exists(Discardable d | result = d.getPath() |
+      py_type_parameter_lists(this, d.(@py_type_parameter_list_parent))
+    )
+  }
+}
+
+/** Discardable unaryop */
+overlay[local]
+final private class DiscardableUnaryOp extends Discardable instanceof @py_unaryop {
+  override string getPath() {
+    exists(Discardable d | result = d.getPath() | py_unaryops(this, _, d.(@py_UnaryExpr)))
+  }
+}
+
+/** Discardable comment */
+overlay[local]
+final private class DiscardableComment extends Discardable instanceof @py_comment {
+  override string getPath() {
+    exists(DiscardableLocation d | result = d.getPath() | py_comments(this, _, d))
+  }
+}
+
+/*- YAML -*/
+overlay[local]
+final private class DiscardableYamlLocatable extends Discardable instanceof @yaml_locatable {
+  override string getPath() {
+    exists(@location loc | yaml_locations(this, loc) | result = getPathForLocation(loc))
+  }
+}
+
+overlay[local]
+private predicate overlayYamlExtracted(string path) {
+  exists(DiscardableYamlLocatable l | l.existsInOverlay() | path = l.getPath())
+}
+
+overlay[discard_entity]
+private predicate discardBaseYamlLocatable(@yaml_locatable el) {
+  exists(DiscardableYamlLocatable d | d = el |
+    // The Yaml extractor is currently not incremental and may extract more
+    // Yaml files than those included in `overlayChangedFiles`, so this discard predicate
+    // handles those files alongside the normal `discardStarEntity` logic.
+    overlayYamlExtracted(d.getPath()) and
+    d.existsInBase()
+  )
+}

python/ql/lib/semmle/python/SSA.qll

@@ -61,14 +61,6 @@ class SsaVariable extends @py_ssa_var {
     )
   }
 
-  /** Gets an argument of the phi function defining this variable, pruned of unlikely edges. */
-  SsaVariable getAPrunedPhiInput() {
-    result = this.getAPhiInput() and
-    exists(BasicBlock incoming | incoming = this.getPredecessorBlockForPhiArgument(result) |
-      not incoming.getLastNode().(RaisingNode).unlikelySuccessor(this.getDefinition())
-    )
-  }
-
   /** Gets a variable that ultimately defines this variable and is not itself defined by another variable */
   SsaVariable getAnUltimateDefinition() {
     result = this and not exists(this.getAPhiInput())
@@ -85,17 +77,11 @@ class SsaVariable extends @py_ssa_var {
   string getId() { result = this.getVariable().getId() }
 
   /** Gets the incoming edges for a Phi node. */
-  private BasicBlock getAPredecessorBlockForPhi() {
+  BasicBlock getAPredecessorBlockForPhi() {
     exists(this.getAPhiInput()) and
     result.getASuccessor() = this.getDefinition().getBasicBlock()
   }
 
-  /** Gets the incoming edges for a Phi node, pruned of unlikely edges. */
-  private BasicBlock getAPrunedPredecessorBlockForPhi() {
-    result = this.getAPredecessorBlockForPhi() and
-    not result.unlikelySuccessor(this.getDefinition().getBasicBlock())
-  }
-
   /** Whether it is possible to reach a use of this variable without passing a definition */
   predicate reachableWithoutDefinition() {
     not exists(this.getDefinition()) and not py_ssa_phi(this, _)
@@ -115,38 +101,6 @@ class SsaVariable extends @py_ssa_var {
     )
   }
 
-  /** Whether this variable may be undefined */
-  predicate maybeUndefined() {
-    not exists(this.getDefinition()) and not py_ssa_phi(this, _) and not this.implicitlyDefined()
-    or
-    this.getDefinition().isDelete()
-    or
-    exists(SsaVariable var | var = this.getAPrunedPhiInput() | var.maybeUndefined())
-    or
-    /*
-     * For phi-nodes, there must be a corresponding phi-input for each control-flow
-     * predecessor. Otherwise, the variable will be undefined on that incoming edge.
-     * WARNING: the same phi-input may cover multiple predecessors, so this check
-     *          cannot be done by counting.
-     */
-
-    exists(BasicBlock incoming |
-      reaches_end(incoming) and
-      incoming = this.getAPrunedPredecessorBlockForPhi() and
-      not this.getAPhiInput().getDefinition().getBasicBlock().dominates(incoming)
-    )
-  }
-
-  private predicate implicitlyDefined() {
-    not exists(this.getDefinition()) and
-    not py_ssa_phi(this, _) and
-    exists(GlobalVariable var | this.getVariable() = var |
-      globallyDefinedName(var.getId())
-      or
-      var.getId() = "__path__" and var.getScope().(Module).isPackageInit()
-    )
-  }
-
   /**
    * Gets the global variable that is accessed if this local is undefined.
    *  Only applies to local variables in class scopes.
@@ -173,43 +127,6 @@ class SsaVariable extends @py_ssa_var {
   }
 }
 
-private predicate reaches_end(BasicBlock b) {
-  not exits_early(b) and
-  (
-    /* Entry point */
-    not exists(BasicBlock prev | prev.getASuccessor() = b)
-    or
-    exists(BasicBlock prev | prev.getASuccessor() = b | reaches_end(prev))
-  )
-}
-
-private predicate exits_early(BasicBlock b) {
-  exists(FunctionObject f |
-    f.neverReturns() and
-    f.getACall().getBasicBlock() = b
-  )
-}
-
-private predicate gettext_installed() {
-  // Good enough (and fast) approximation
-  exists(Module m | m.getName() = "gettext")
-}
-
-private predicate builtin_constant(string name) {
-  exists(Object::builtin(name))
-  or
-  name = "WindowsError"
-  or
-  name = "_" and gettext_installed()
-}
-
-private predicate auto_name(string name) {
-  name = "__file__" or name = "__builtins__" or name = "__name__"
-}
-
-/** Whether this name is (almost) always defined, ie. it is a builtin or VM defined name */
-predicate globallyDefinedName(string name) { builtin_constant(name) or auto_name(name) }
-
 /** An SSA variable that is backed by a global variable */
 class GlobalSsaVariable extends EssaVariable {
   GlobalSsaVariable() { this.getSourceVariable() instanceof GlobalVariable }

python/ql/lib/semmle/python/SelfAttribute.qll

@@ -5,6 +5,7 @@
 
 import python
 private import semmle.python.pointsto.Filters
+private import LegacyPointsTo
 
 /**
  * An attribute access where the left hand side of the attribute expression

python/ql/lib/semmle/python/SpecialMethods.qll

@@ -9,6 +9,7 @@
  */
 
 private import python
+private import LegacyPointsTo
 
 /** A control flow node which might correspond to a special method call. */
 class PotentialSpecialMethodCallNode extends ControlFlowNode instanceof SpecialMethod::Potential { }
@@ -106,7 +107,11 @@ class SpecialMethodCallNode extends PotentialSpecialMethodCallNode {
   SpecialMethodCallNode() {
     exists(SpecialMethod::Potential pot |
       this = pot and
-      pot.getSelf().pointsTo().getClass().lookup(pot.getSpecialMethodName()) = resolvedSpecialMethod
+      pot.getSelf()
+          .(ControlFlowNodeWithPointsTo)
+          .pointsTo()
+          .getClass()
+          .lookup(pot.getSpecialMethodName()) = resolvedSpecialMethod
     )
   }
 

python/ql/lib/semmle/python/dataflow/new/FlowSummary.qll

@@ -22,30 +22,39 @@ deprecated class SummaryComponentStack = Impl::Private::SummaryComponentStack;
 
 deprecated module SummaryComponentStack = Impl::Private::SummaryComponentStack;
 
-/** A callable with a flow summary, identified by a unique string. */
-abstract class SummarizedCallable extends LibraryCallable, Impl::Public::SummarizedCallable {
-  bindingset[this]
-  SummarizedCallable() { any() }
+class Provenance = Impl::Public::Provenance;
 
-  /**
-   * DEPRECATED: Use `propagatesFlow` instead.
-   */
-  deprecated predicate propagatesFlowExt(string input, string output, boolean preservesValue) {
-    this.propagatesFlow(input, output, preservesValue, _)
+/** Provides the `Range` class used to define the extent of `SummarizedCallable`. */
+module SummarizedCallable {
+  /** A callable with a flow summary, identified by a unique string. */
+  abstract class Range extends LibraryCallable, Impl::Public::SummarizedCallable {
+    bindingset[this]
+    Range() { any() }
+
+    override predicate propagatesFlow(
+      string input, string output, boolean preservesValue, Provenance p, boolean isExact,
+      string model
+    ) {
+      this.propagatesFlow(input, output, preservesValue) and
+      p = "manual" and
+      isExact = true and
+      model = this
+    }
+
+    /**
+     * Holds if data may flow from `input` to `output` through this callable.
+     *
+     * `preservesValue` indicates whether this is a value-preserving step or a taint-step.
+     */
+    predicate propagatesFlow(string input, string output, boolean preservesValue) { none() }
   }
-
-  override predicate propagatesFlow(
-    string input, string output, boolean preservesValue, string model
-  ) {
-    this.propagatesFlow(input, output, preservesValue) and model = this
-  }
-
-  /**
-   * Holds if data may flow from `input` to `output` through this callable.
-   *
-   * `preservesValue` indicates whether this is a value-preserving step or a taint-step.
-   */
-  predicate propagatesFlow(string input, string output, boolean preservesValue) { none() }
 }
 
+final private class SummarizedCallableFinal = SummarizedCallable::Range;
+
+/** A callable with a flow summary, identified by a unique string. */
+final class SummarizedCallable extends SummarizedCallableFinal,
+  Impl::Public::RelevantSummarizedCallable
+{ }
+
 deprecated class RequiredSummaryComponentStack = Impl::Private::RequiredSummaryComponentStack;

python/ql/lib/semmle/python/dataflow/new/SensitiveDataSources.qll

@@ -334,3 +334,5 @@ private module SensitiveDataModeling {
 }
 
 predicate sensitiveDataExtraStepForCalls = SensitiveDataModeling::extraStepForCalls/2;
+
+predicate sensitiveLookupStringConst = SensitiveDataModeling::sensitiveLookupStringConst/1;

python/ql/lib/semmle/python/dataflow/new/internal/Attributes.qll

@@ -1,9 +1,10 @@
 /** This module provides an API for attribute reads and writes. */
 
+private import python
 import DataFlowUtil
 import DataFlowPublic
 private import DataFlowPrivate
-private import semmle.python.types.Builtins
+private import semmle.python.dataflow.new.internal.Builtins
 
 /**
  * A data flow node that reads or writes an attribute of an object.
@@ -134,8 +135,12 @@ private class BuiltInCallNode extends CallNode {
 
   BuiltInCallNode() {
     // TODO disallow instances where the name of the built-in may refer to an in-scope variable of that name.
-    exists(NameNode id | this.getFunction() = id and id.getId() = name and id.isGlobal()) and
-    name = any(Builtin b).getName()
+    exists(NameNode id |
+      name = Builtins::getBuiltinName() and
+      this.getFunction() = id and
+      id.getId() = name and
+      id.isGlobal()
+    )
   }
 
   /** Gets the name of the built-in function that is called at this `CallNode` */

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPublic.qll

@@ -584,10 +584,6 @@ class GuardNode extends ControlFlowNode {
 
 /**
  * Holds if the guard `g` validates `node` upon evaluating to `branch`.
- *
- * The expression `e` is expected to be a syntactic part of the guard `g`.
- * For example, the guard `g` might be a call `isSafe(x)` and the expression `e`
- * the argument `x`.
  */
 signature predicate guardChecksSig(GuardNode g, ControlFlowNode node, boolean branch);
 
@@ -600,15 +596,72 @@ signature predicate guardChecksSig(GuardNode g, ControlFlowNode node, boolean br
 module BarrierGuard<guardChecksSig/3 guardChecks> {
   /** Gets a node that is safely guarded by the given guard check. */
   ExprNode getABarrierNode() {
+    result = ParameterizedBarrierGuard<Unit, extendedGuardChecks/4>::getABarrierNode(_)
+  }
+
+  private predicate extendedGuardChecks(GuardNode g, ControlFlowNode node, boolean branch, Unit u) {
+    guardChecks(g, node, branch) and
+    u = u
+  }
+}
+
+bindingset[this]
+private signature class ParamSig;
+
+private module WithParam<ParamSig P> {
+  signature predicate guardChecksSig(GuardNode g, ControlFlowNode node, boolean branch, P param);
+}
+
+/**
+ * Provides a set of barrier nodes for a guard that validates a node.
+ *
+ * This is expected to be used in `isBarrier`/`isSanitizer` definitions
+ * in data flow and taint tracking.
+ */
+module ParameterizedBarrierGuard<ParamSig P, WithParam<P>::guardChecksSig/4 guardChecks> {
+  /** Gets a node that is safely guarded by the given guard check with parameter `param`. */
+  ExprNode getABarrierNode(P param) {
     exists(GuardNode g, EssaDefinition def, ControlFlowNode node, boolean branch |
       AdjacentUses::useOfDef(def, node) and
-      guardChecks(g, node, branch) and
+      guardChecks(g, node, branch, param) and
       AdjacentUses::useOfDef(def, result.asCfgNode()) and
       g.controlsBlock(result.asCfgNode().getBasicBlock(), branch)
     )
   }
 }
 
+/**
+ * Provides a set of barrier nodes for a guard that validates a node as described by an external predicate.
+ *
+ * This is expected to be used in `isBarrier`/`isSanitizer` definitions
+ * in data flow and taint tracking.
+ */
+module ExternalBarrierGuard {
+  private import semmle.python.ApiGraphs
+
+  private predicate guardCheck(GuardNode g, ControlFlowNode node, boolean branch, string kind) {
+    exists(API::CallNode call, API::Node parameter |
+      parameter = call.getAParameter() and
+      parameter = ModelOutput::getABarrierGuardNode(kind, branch)
+    |
+      g = call.asCfgNode() and
+      node = parameter.asSink().asCfgNode()
+    )
+  }
+
+  /**
+   * Gets a node that is an external barrier of the given kind.
+   *
+   * This only provides external barrier nodes defined as guards. To get all externally defined barrer nodes,
+   * use `ModelOutput::barrierNode(node, kind)`.
+   *
+   * INTERNAL: Do not use.
+   */
+  ExprNode getAnExternalBarrierNode(string kind) {
+    result = ParameterizedBarrierGuard<string, guardCheck/4>::getABarrierNode(kind)
+  }
+}
+
 /**
  * Algebraic datatype for tracking data content associated with values.
  * Content can be collection elements or object attributes.

python/ql/lib/semmle/python/dataflow/new/internal/FlowSummaryImpl.qll

@@ -18,6 +18,8 @@ module Input implements InputSig<Location, DataFlowImplSpecific::PythonDataFlow>
 
   class SinkBase = Void;
 
+  predicate callableFromSource(SummarizedCallableBase c) { none() }
+
   ArgumentPosition callbackSelfParameterPosition() { result.isLambdaSelf() }
 
   ReturnKind getStandardReturnValueKind() { any() }

python/ql/lib/semmle/python/dataflow/new/internal/ImportResolution.qll

@@ -9,6 +9,7 @@ private import semmle.python.dataflow.new.DataFlow
 private import semmle.python.dataflow.new.internal.ImportStar
 private import semmle.python.dataflow.new.TypeTracking
 private import semmle.python.dataflow.new.internal.DataFlowPrivate
+private import semmle.python.essa.SsaDefinitions
 
 /**
  * Python modules and the way imports are resolved are... complicated. Here's a crash course in how

python/ql/lib/semmle/python/dataflow/new/internal/TypeTrackingImpl.qll

@@ -30,7 +30,7 @@ private module SummaryTypeTrackerInput implements SummaryTypeTracker::Input {
     predicate propagatesFlow(
       SummaryComponentStack input, SummaryComponentStack output, boolean preservesValue
     ) {
-      super.propagatesFlow(input, output, preservesValue, _)
+      super.propagatesFlow(input, output, preservesValue, _, _, _)
     }
   }
 

python/ql/lib/semmle/python/dataflow/old/Configuration.qll

@@ -1,6 +1,6 @@
 import python
 import semmle.python.dataflow.TaintTracking
-private import semmle.python.objects.ObjectInternal
+private import LegacyPointsTo
 private import semmle.python.dataflow.Implementation
 
 module TaintTracking {

python/ql/lib/semmle/python/dataflow/old/Files.qll

@@ -1,4 +1,5 @@
 import python
+private import LegacyPointsTo
 import semmle.python.dataflow.TaintTracking
 
 class OpenFile extends TaintKind {

python/ql/lib/semmle/python/dataflow/old/Implementation.qll

@@ -1,6 +1,6 @@
 import python
+private import LegacyPointsTo
 import semmle.python.dataflow.TaintTracking
-private import semmle.python.objects.ObjectInternal
 private import semmle.python.pointsto.Filters as Filters
 import semmle.python.dataflow.Legacy
 
@@ -256,7 +256,7 @@ class TaintTrackingImplementation extends string instanceof TaintTracking::Confi
     TaintKind kind, string edgeLabel
   ) {
     this.unprunedStep(src, node, context, path, kind, edgeLabel) and
-    node.getBasicBlock().likelyReachable() and
+    node.getBasicBlock().(BasicBlockWithPointsTo).likelyReachable() and
     not super.isBarrier(node) and
     (
       not path = TNoAttribute()
@@ -374,7 +374,7 @@ class TaintTrackingImplementation extends string instanceof TaintTracking::Confi
     exists(ModuleValue m, string name |
       src = TTaintTrackingNode_(_, context, path, kind, this) and
       this.moduleAttributeTainted(m, name, src) and
-      node.asCfgNode().(ImportMemberNode).getModule(name).pointsTo(m)
+      node.asCfgNode().(ImportMemberNode).getModule(name).(ControlFlowNodeWithPointsTo).pointsTo(m)
     )
   }
 
@@ -408,7 +408,9 @@ class TaintTrackingImplementation extends string instanceof TaintTracking::Confi
       src = TTaintTrackingNode_(srcnode, context, srcpath, srckind, this) and
       exists(CallNode call, ControlFlowNode arg |
         call = node.asCfgNode() and
-        call.getFunction().pointsTo(ObjectInternal::builtin("getattr")) and
+        call.getFunction()
+            .(ControlFlowNodeWithPointsTo)
+            .pointsTo(ObjectInternal::builtin("getattr")) and
         arg = call.getArg(0) and
         attrname = call.getArg(1).getNode().(StringLiteral).getText() and
         arg = srcnode.asCfgNode()
@@ -515,7 +517,7 @@ class TaintTrackingImplementation extends string instanceof TaintTracking::Confi
     TaintTrackingContext caller, TaintTrackingContext callee
   ) {
     exists(ClassValue cls |
-      call.getFunction().pointsTo(cls) and
+      call.getFunction().(ControlFlowNodeWithPointsTo).pointsTo(cls) and
       cls.lookup("__init__") = init
     |
       exists(int arg, TaintKind callerKind, AttributePath callerPath, DataFlow::Node argument |
@@ -682,7 +684,9 @@ private class EssaTaintTracking extends string instanceof TaintTracking::Configu
     TaintTrackingNode src, PhiFunction defn, TaintTrackingContext context, AttributePath path,
     TaintKind kind
   ) {
-    exists(DataFlow::Node srcnode, BasicBlock pred, EssaVariable predvar, DataFlow::Node phi |
+    exists(
+      DataFlow::Node srcnode, BasicBlockWithPointsTo pred, EssaVariable predvar, DataFlow::Node phi
+    |
       src = TTaintTrackingNode_(srcnode, context, path, kind, this) and
       defn = phi.asVariable().getDefinition() and
       predvar = defn.getInput(pred) and
@@ -878,7 +882,7 @@ private class EssaTaintTracking extends string instanceof TaintTracking::Configu
         const.getNode() instanceof ImmutableLiteral
       )
       or
-      exists(ControlFlowNode c, ClassValue cls |
+      exists(ControlFlowNodeWithPointsTo c, ClassValue cls |
         Filters::isinstance(test, c, use) and
         c.pointsTo(cls)
       |
@@ -978,7 +982,7 @@ module Implementation {
       tonode.getArg(0) = fromnode
     )
     or
-    tonode.getFunction().pointsTo(ObjectInternal::builtin("reversed")) and
+    tonode.getFunction().(ControlFlowNodeWithPointsTo).pointsTo(ObjectInternal::builtin("reversed")) and
     tonode.getArg(0) = fromnode
   }
 }