Python: Support template strings in rest of extractor

Adds three new AST nodes to the mix:

- `TemplateString` represents a t-string in Python 3.14
- `TemplateStringPart` represents one of the string constituents of a
t-string. (The interpolated expressions are represented as `Expr` nodes,
just like f-strings.)
- `JoinedTemplateString` represents an implicit concatenation of
template strings.

Importantly, we _completely avoid_ the complicated construction we
currently do for format strings (as well as the confusing nomenclature).
No extra injection of empty strings (so that a template string is a
strict alternation of strings and expressions). A `JoinedTemplateString`
simply has a list of template string children, and a `TemplateString`
has a list of "values" which may be either `Expr` or
`TemplateStringPart` nodes.

If we ever find that we actually want the more complicated interface for
these strings, then I would much rather we reconstruct this inside of QL
rather than in the parser.

python/extractor/semmle/python/ast.py

@@ -56,6 +56,15 @@ class StringPart(AstBase):
         self.text = text
         self.s = s
 
+class TemplateStringPart(AstBase):
+    '''A string constituent of a template string literal'''
+
+    __slots__ = "text", "s",
+
+    def __init__(self, text, s):
+        self.text = text
+        self.s = s
+
 class alias(AstBase):
     __slots__ = "value", "asname",
 
@@ -356,6 +365,19 @@ class JoinedStr(expr):
     def __init__(self, values):
         self.values = values
 
+class TemplateString(expr):
+    __slots__ = "prefix", "values",
+
+    def __init__(self, prefix, values):
+        self.prefix = prefix
+        self.values = values
+
+class JoinedTemplateString(expr):
+    __slots__ = "strings",
+
+    def __init__(self, strings):
+        self.strings = strings
+
 
 class Lambda(expr):
     __slots__ = "args", "inner_scope",

python/extractor/semmle/python/master.py

@@ -186,12 +186,20 @@ FormattedStringLiteral.set_name("Fstring")
 
 FormattedValue = ClassNode("FormattedValue", expr, descriptive_name='formatted value')
 
+
 AnnAssign = ClassNode("AnnAssign", stmt, descriptive_name='annotated assignment')
 
 AssignExpr = ClassNode('AssignExpr', expr, "assignment expression")
 
 SpecialOperation = ClassNode('SpecialOperation', expr, "special operation")
 
+TemplateString = ClassNode('TemplateString', expr, 'template string literal')
+
+template_string_list = ListNode(TemplateString)
+
+JoinedTemplateString = ClassNode("JoinedTemplateString", expr, descriptive_name='joined template string')
+TemplateStringPart = ClassNode('TemplateStringPart', expr, "string part of a template string")
+
 type_parameter = ClassNode('type_parameter', descriptive_name='type parameter')
 type_parameter.field('location', location)
 type_parameter_list = ListNode(type_parameter)
@@ -435,6 +443,9 @@ Subscript.field('value', expr)
 Subscript.field('index', expr)
 Subscript.field('ctx', expr_context, 'context')
 
+TemplateString.field('prefix', string, 'prefix')
+TemplateString.field('values', expr_list, 'values')
+
 Try.field('body', stmt_list)
 Try.field('orelse', stmt_list, 'else block')
 Try.field('handlers', stmt_list, 'exception handlers')
@@ -484,10 +495,15 @@ PlaceHolder.field('ctx', expr_context, 'context')
 StringPart.field('text', string)
 StringPart.field('location', location)
 
+TemplateStringPart.field('text', string)
+
+
 Await.field('value', expr, 'expression waited upon')
 
 FormattedStringLiteral.field('values', expr_list)
 
+JoinedTemplateString.field('strings', template_string_list)
+
 FormattedValue.field('value', expr, "expression to be formatted")
 FormattedValue.field('conversion', string, 'type conversion')
 FormattedValue.field('format_spec', FormattedStringLiteral, 'format specifier')

python/extractor/semmle/python/parser/tsg_parser.py

@@ -273,6 +273,8 @@ list_fields = {
     ast.Print: ("values",),
     ast.Set: ("elts",),
     ast.Str: ("implicitly_concatenated_parts",),
+    ast.TemplateString: ("values",),
+    ast.JoinedTemplateString: ("strings",),
     ast.TypeAlias: ("type_parameters",),
     ast.Try: ("body", "handlers", "orelse", "finalbody"),
     ast.Tuple: ("elts",),