unified: Add the swift-syntax parser and unresolved operator sequence (dormant)

Adds the plumbing for the swift-syntax front-end, without yet switching
the runtime over to it (the Swift front-end still parses with
tree-sitter):

- `languages/swift/parse.rs` shells out to the separate
  `swift-syntax-parse` binary and adapts its JSON into a `yeast::Ast`
  (plus side-channel `extra` tokens) via `swift_adapter`. Running the
  parser out-of-process keeps the Swift toolchain out of the extractor's
  own build. It is wired in as a module but left `allow(dead_code)`
  until the runtime uses it.

- `swift_node_types.yml` is the authoritative swift-syntax input schema
  (generated from swift-syntax by a one-off tool). The adapter seeds
  every parse with it, pre-registering every input kind and field so
  that rule matching never references a name absent from a given file's
  tree. The adapter now emits `ExtraToken`s directly during its single
  traversal, so `parse.rs` hands the parsed tree straight through with
  no second pass.

- `ast_types.yml` gains an `unresolved_operator_sequence` type (with an
  `expr_or_operator` union) for flat operator chains the parser can't
  resolve — e.g. a chain using an operator imported from another module,
  whose precedence is unknown. Nothing produces it yet; the mapping
  rules that do are added when the rules are ported. The dbscheme and QL
  library are regenerated to match.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

unified/extractor/ast_types.yml

@@ -31,6 +31,7 @@ supertypes:
     - throw_expr
     - try_expr
     - switch_expr
+    - unresolved_operator_sequence
     - unsupported_node
   expr_or_pattern:
     - expr
@@ -38,6 +39,11 @@ supertypes:
   expr_or_type:
     - expr
     - type_expr
+  # An element of an `unresolved_operator_sequence`: either an operand (`expr`)
+  # or one of the infix operators separating the operands.
+  expr_or_operator:
+    - expr
+    - infix_operator
   pattern:
     - name_pattern
     - tuple_pattern
@@ -137,6 +143,19 @@ named:
     operand: expr
     operator: operator
 
+  # A flat, unresolved operator sequence such as `a <+> b <+> c`.
+  #
+  # Swift's grammar doesn't encode operator precedence, so an operator chain is
+  # first parsed as a flat list of operands and operators. The parser front-end
+  # resolves this into structured `binary_expr` trees when it knows the
+  # operators' precedence (standard-library operators, and operators declared in
+  # the same file). When it encounters an operator whose precedence it can't
+  # determine (e.g. one imported from another module), it leaves that chain
+  # unresolved and emits it here rather than guessing a (possibly wrong)
+  # structure. The `element`s alternate operands (`expr`) and infix operators.
+  unresolved_operator_sequence:
+    element*: expr_or_operator
+
   # Plain assignment
   assign_expr:
     target: expr_or_pattern

unified/extractor/src/languages/mod.rs

@@ -12,6 +12,17 @@ mod swift;
 #[allow(dead_code)]
 pub mod swift_adapter;
 
+/// Swift front-end parser: shells out to `swift-syntax-parse` and adapts its
+/// JSON output via [`swift_adapter`].
+///
+/// Dormant for now: the runtime Swift front-end is still tree-sitter, so
+/// nothing in the binary calls this yet. `allow(dead_code)` for the same
+/// binary-crate reason as [`swift_adapter`]; both allows are removed once the
+/// runtime switches the Swift front-end to swift-syntax.
+#[path = "swift/parse.rs"]
+#[allow(dead_code)]
+pub mod swift_parse;
+
 /// Shared YEAST output AST schema for all languages.
 pub(crate) const OUTPUT_AST_SCHEMA: &str = include_str!("../../ast_types.yml");
 

unified/extractor/src/languages/swift/adapter.rs

@@ -2,9 +2,8 @@
 //! in-memory format the CodeQL desugaring rules operate on.
 //!
 //! The JSON tree is produced by the `swift-syntax-rs` crate's Swift FFI shim
-//! (`parse_to_json`). This module is pure Rust (only `yeast` + `serde_json`),
-//! so the extractor consumes swift-syntax output without pulling in the Swift
-//! toolchain (the JSON is produced out-of-process).
+//! (`parse_to_json`). This module needs no Swift toolchain, so the extractor
+//! consumes swift-syntax output out-of-process.
 //!
 //! The mapping mirrors tree-sitter's node model, which is what yeast (and the
 //! extractor's rewrite rules) expect:
@@ -24,31 +23,19 @@
 
 use std::collections::BTreeMap;
 
+use codeql_extractor::extractor::ExtraToken;
 use serde_json::Value;
-use yeast::schema::Schema;
 use yeast::{Ast, Id, NodeContent, Point, Range};
 
-/// A comment (or `unexpectedText`) recovered from the syntax tree's trivia.
-///
-/// These are collected into a side channel rather than embedded in the
-/// [`yeast::Ast`], mirroring how the extractor treats tree-sitter `extra`
-/// nodes: they carry a location and text but are not attached to a parent.
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct TriviaToken {
-    /// The trivia kind (e.g. `lineComment`, `blockComment`, `docLineComment`,
-    /// `docBlockComment`, `unexpectedText`).
-    pub kind: String,
-    /// The verbatim source text of the piece (e.g. `// comment`).
-    pub text: String,
-    /// The source range the piece occupies.
-    pub range: Range,
-}
-
 /// The result of adapting a swift-syntax JSON tree: the [`yeast::Ast`] plus the
-/// comment/`unexpectedText` trivia harvested from it (in source order).
+/// comment/`unexpectedText` [`ExtraToken`]s harvested from it (in source order).
+///
+/// The extra tokens are collected into a side channel rather than embedded in
+/// the [`yeast::Ast`], mirroring how the extractor treats tree-sitter `extra`
+/// nodes: they carry a location and text but are not attached to a parent.
 pub struct AdaptedTree {
     pub ast: Ast,
-    pub trivia: Vec<TriviaToken>,
+    pub extras: Vec<ExtraToken>,
 }
 
 /// swift-syntax `TokenKind` cases whose text is *not* determined by the kind
@@ -170,17 +157,18 @@ fn children_of(value: &Value) -> Vec<&Value> {
 /// in the schema on the fly, immediately before the node is created. Children
 /// are built first so a parent's field lists reference existing ids. Any
 /// comment/`unexpectedText` trivia carried by a token is harvested into
-/// `trivia` during the same pass rather than embedded in the tree.
-fn build(node: &Value, ast: &mut Ast, trivia: &mut Vec<TriviaToken>) -> Result<Id, String> {
+/// `extras` (as [`ExtraToken`]s) during the same pass rather than embedded in
+/// the tree.
+fn build(node: &Value, ast: &mut Ast, extras: &mut Vec<ExtraToken>) -> Result<Id, String> {
     let info = classify(node)?;
-    collect_trivia(node, trivia);
+    collect_extras(node, extras);
 
     let mut fields: BTreeMap<u16, Vec<Id>> = BTreeMap::new();
     for (field, value) in field_entries(node) {
         let field_id = ast.register_field(field);
         let mut ids = Vec::new();
         for child in children_of(value) {
-            ids.push(build(child, ast, trivia)?);
+            ids.push(build(child, ast, extras)?);
         }
         fields.insert(field_id, ids);
     }
@@ -201,9 +189,10 @@ fn build(node: &Value, ast: &mut Ast, trivia: &mut Vec<TriviaToken>) -> Result<I
 }
 
 /// Harvest a token's `leadingTrivia`/`trailingTrivia` pieces (each already
-/// filtered to comments/`unexpectedText` upstream) into `out`. Non-token nodes
-/// have no trivia keys, so this is a no-op for them.
-fn collect_trivia(node: &Value, out: &mut Vec<TriviaToken>) {
+/// filtered to comments/`unexpectedText` upstream) into `out` as
+/// [`ExtraToken`]s. Non-token nodes have no trivia keys, so this is a no-op for
+/// them.
+fn collect_extras(node: &Value, out: &mut Vec<ExtraToken>) {
     for key in ["leadingTrivia", "trailingTrivia"] {
         let Some(Value::Array(pieces)) = node.get(key) else {
             continue;
@@ -220,8 +209,8 @@ fn collect_trivia(node: &Value, out: &mut Vec<TriviaToken>) {
                 .and_then(Value::as_str)
                 .unwrap_or("")
                 .to_string();
-            out.push(TriviaToken {
-                kind: kind.to_string(),
+            out.push(ExtraToken {
+                kind: trivia_kind_id(kind),
                 text,
                 range,
             });
@@ -229,6 +218,21 @@ fn collect_trivia(node: &Value, out: &mut Vec<TriviaToken>) {
     }
 }
 
+/// Map a swift-syntax trivia kind name to the stable integer id stored in an
+/// [`ExtraToken`]'s `kind` (and written to the `unified_trivia_tokeninfo`
+/// table). The value is opaque to the QL library (which reads only the text),
+/// but is kept stable and meaningful.
+fn trivia_kind_id(kind: &str) -> usize {
+    match kind {
+        "lineComment" => 1,
+        "blockComment" => 2,
+        "docLineComment" => 3,
+        "docBlockComment" => 4,
+        "unexpectedText" => 5,
+        _ => 0,
+    }
+}
+
 /// Parse a node's `range` into a [`yeast::Range`].
 ///
 /// The JSON carries, for `start` and `end`, a 0-based UTF-8 file byte `offset`,
@@ -258,21 +262,29 @@ fn parse_range(node: &Value) -> Option<Range> {
     })
 }
 
+/// The authoritative swift-syntax input node-types schema, generated from
+/// swift-syntax (see the schemagen tool). [`json_to_ast`] seeds every parse
+/// with the schema built from this, pre-registering every input kind and field
+/// so rule matching never references a name absent from a given file's tree.
+const SWIFT_NODE_TYPES: &str = include_str!("../../../swift_node_types.yml");
+
 /// Convert a swift-syntax JSON tree (as produced by [`crate::parse_to_json`])
 /// into a [`yeast::Ast`] plus the comment/`unexpectedText` trivia harvested
-/// from it. Both are produced in a single traversal.
+/// from it. Both are produced in a single traversal. The AST is seeded with the
+/// authoritative swift-syntax schema ([`SWIFT_NODE_TYPES`]); the adapter only
+/// ever consumes swift-syntax input, so the schema is not a parameter.
 pub fn json_to_ast(json: &str) -> Result<AdaptedTree, String> {
     let root: Value = serde_json::from_str(json).map_err(|e| format!("invalid JSON: {e}"))?;
 
-    let mut ast = Ast::with_schema(Schema::new());
-    let mut trivia = Vec::new();
-    let root_id = build(&root, &mut ast, &mut trivia)?;
+    let mut ast = Ast::with_schema(yeast::node_types_yaml::schema_from_yaml(SWIFT_NODE_TYPES)?);
+    let mut extras = Vec::new();
+    let root_id = build(&root, &mut ast, &mut extras)?;
     ast.set_root(root_id);
 
-    // Emit trivia in source order (the traversal visits nodes bottom-up).
-    trivia.sort_by_key(|t| t.range.start_byte);
+    // Emit extras in source order (the traversal visits nodes bottom-up).
+    extras.sort_by_key(|t| t.range.start_byte);
 
-    Ok(AdaptedTree { ast, trivia })
+    Ok(AdaptedTree { ast, extras })
 }
 
 #[cfg(test)]
@@ -373,7 +385,7 @@ mod tests {
     }
 
     #[test]
-    fn collects_trivia_into_side_channel() {
+    fn collects_extras_into_side_channel() {
         // A token carrying a trailing line comment in its trivia.
         let json = r#"{
             "kind": "sourceFile",
@@ -395,9 +407,10 @@ mod tests {
         let adapted = json_to_ast(json).expect("adapter should succeed");
 
         // The comment is in the side channel, with its text and location.
-        assert_eq!(adapted.trivia.len(), 1);
-        let comment = &adapted.trivia[0];
-        assert_eq!(comment.kind, "lineComment");
+        assert_eq!(adapted.extras.len(), 1);
+        let comment = &adapted.extras[0];
+        // `lineComment` maps to extra kind id 1.
+        assert_eq!(comment.kind, 1);
         assert_eq!(comment.text, "// c");
         assert_eq!(comment.range.start_byte, 2);
         assert_eq!(comment.range.end_byte, 6);

unified/extractor/src/languages/swift/parse.rs

@@ -0,0 +1,73 @@
+//! Swift front-end parser: shells out to the separate `swift-syntax-parse`
+//! binary (which links swift-syntax) to obtain a JSON syntax tree, then adapts
+//! that JSON into a `yeast::Ast` via the pure-Rust [`swift_adapter`] module.
+//!
+//! Running the parser in a separate process keeps the Swift toolchain out of
+//! the extractor's own build: the extractor never links Swift, so working on
+//! other (e.g. tree-sitter based) languages needs no Swift toolchain. Each call
+//! spawns the parser afresh; a longer-lived parser process could be swapped in
+//! behind this same seam later without touching the extraction pipeline.
+
+use std::io::Write;
+use std::process::{Command, Stdio};
+
+use codeql_extractor::extractor::ParsedTree;
+
+use super::swift_adapter;
+
+/// Environment variable naming the `swift-syntax-parse` executable. When unset,
+/// `swift-syntax-parse` is looked up on `PATH`.
+const PARSE_BIN_ENV: &str = "CODEQL_EXTRACTOR_UNIFIED_SWIFT_SYNTAX_PARSE";
+
+/// Parse Swift `source` into a [`ParsedTree`] (a raw `yeast::Ast` plus
+/// side-channel `extra` tokens), ready to be desugared via `run_from_ast`.
+pub fn parse(source: &[u8]) -> Result<ParsedTree, String> {
+    let source =
+        std::str::from_utf8(source).map_err(|e| format!("Swift source is not valid UTF-8: {e}"))?;
+    let json = run_parser(source)?;
+    let mut adapted = swift_adapter::json_to_ast(&json)?;
+    adapted.ast.set_source(source.as_bytes().to_vec());
+    Ok(ParsedTree {
+        ast: adapted.ast,
+        extras: adapted.extras,
+    })
+}
+
+/// The `swift-syntax-parse` executable to invoke.
+fn parse_bin() -> String {
+    std::env::var(PARSE_BIN_ENV).unwrap_or_else(|_| "swift-syntax-parse".to_string())
+}
+
+/// Run the external parser, feeding `source` on stdin and returning its JSON
+/// stdout.
+fn run_parser(source: &str) -> Result<String, String> {
+    let bin = parse_bin();
+    let mut child = Command::new(&bin)
+        .stdin(Stdio::piped())
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .spawn()
+        .map_err(|e| format!("failed to spawn Swift parser `{bin}`: {e}"))?;
+
+    // The parser reads all of stdin before writing any stdout, so writing the
+    // whole source and then closing stdin (by dropping it) cannot deadlock.
+    child
+        .stdin
+        .take()
+        .expect("child stdin was piped")
+        .write_all(source.as_bytes())
+        .map_err(|e| format!("failed to write source to Swift parser `{bin}`: {e}"))?;
+
+    let output = child
+        .wait_with_output()
+        .map_err(|e| format!("failed to run Swift parser `{bin}`: {e}"))?;
+    if !output.status.success() {
+        return Err(format!(
+            "Swift parser `{bin}` failed ({}): {}",
+            output.status,
+            String::from_utf8_lossy(&output.stderr).trim()
+        ));
+    }
+    String::from_utf8(output.stdout)
+        .map_err(|e| format!("Swift parser produced non-UTF-8 output: {e}"))
+}

unified/ql/lib/codeql/unified/Ast.qll

@@ -567,6 +567,8 @@ module Unified {
     final override AstNode getAFieldOrChild() { unified_expr_equality_pattern_def(this, result) }
   }
 
+  final class ExprOrOperator extends @unified_expr_or_operator, AstNodeImpl { }
+
   final class ExprOrPattern extends @unified_expr_or_pattern, AstNodeImpl { }
 
   final class ExprOrType extends @unified_expr_or_type, AstNodeImpl { }
@@ -1407,6 +1409,22 @@ module Unified {
     }
   }
 
+  /** A class representing `unresolved_operator_sequence` nodes. */
+  final class UnresolvedOperatorSequence extends @unified_unresolved_operator_sequence, AstNodeImpl {
+    /** Gets the name of the primary QL class for this element. */
+    final override string getAPrimaryQlClass() { result = "UnresolvedOperatorSequence" }
+
+    /** Gets the node corresponding to the field `element`. */
+    final ExprOrOperator getElement(int i) {
+      unified_unresolved_operator_sequence_element(this, i, result)
+    }
+
+    /** Gets a field or child node of this node. */
+    final override AstNode getAFieldOrChild() {
+      unified_unresolved_operator_sequence_element(this, _, result)
+    }
+  }
+
   /** A class representing `unsupported_node` tokens. */
   final class UnsupportedNode extends @unified_token_unsupported_node, TokenImpl {
     /** Gets the name of the primary QL class for this element. */
@@ -1773,6 +1791,8 @@ module Unified {
       or
       result = node.(UnaryExpr).getOperator() and i = -1 and name = "getOperator"
       or
+      result = node.(UnresolvedOperatorSequence).getElement(i) and name = "getElement"
+      or
       result = node.(VariableDeclaration).getModifier(i) and name = "getModifier"
       or
       result = node.(VariableDeclaration).getPattern() and i = -1 and name = "getPattern"

unified/ql/lib/unified.dbscheme

@@ -452,13 +452,15 @@ unified_equality_type_constraint_def(
   int right: @unified_type_expr ref
 );
 
-@unified_expr = @unified_array_literal | @unified_assign_expr | @unified_binary_expr | @unified_block | @unified_break_expr | @unified_call_expr | @unified_compound_assign_expr | @unified_continue_expr | @unified_function_expr | @unified_if_expr | @unified_key_value_pair | @unified_map_literal | @unified_member_access_expr | @unified_name_expr | @unified_pattern_guard_expr | @unified_return_expr | @unified_switch_expr | @unified_throw_expr | @unified_token_boolean_literal | @unified_token_builtin_expr | @unified_token_empty_expr | @unified_token_float_literal | @unified_token_int_literal | @unified_token_regex_literal | @unified_token_string_literal | @unified_token_super_expr | @unified_token_unsupported_node | @unified_try_expr | @unified_tuple_expr | @unified_type_cast_expr | @unified_type_test_expr | @unified_unary_expr
+@unified_expr = @unified_array_literal | @unified_assign_expr | @unified_binary_expr | @unified_block | @unified_break_expr | @unified_call_expr | @unified_compound_assign_expr | @unified_continue_expr | @unified_function_expr | @unified_if_expr | @unified_key_value_pair | @unified_map_literal | @unified_member_access_expr | @unified_name_expr | @unified_pattern_guard_expr | @unified_return_expr | @unified_switch_expr | @unified_throw_expr | @unified_token_boolean_literal | @unified_token_builtin_expr | @unified_token_empty_expr | @unified_token_float_literal | @unified_token_int_literal | @unified_token_regex_literal | @unified_token_string_literal | @unified_token_super_expr | @unified_token_unsupported_node | @unified_try_expr | @unified_tuple_expr | @unified_type_cast_expr | @unified_type_test_expr | @unified_unary_expr | @unified_unresolved_operator_sequence
 
 unified_expr_equality_pattern_def(
   unique int id: @unified_expr_equality_pattern,
   int expr: @unified_expr ref
 );
 
+@unified_expr_or_operator = @unified_expr | @unified_token_infix_operator
+
 @unified_expr_or_pattern = @unified_expr | @unified_pattern
 
 @unified_expr_or_type = @unified_expr | @unified_type_expr
@@ -999,6 +1001,17 @@ unified_unary_expr_def(
   int operator: @unified_operator ref
 );
 
+#keyset[unified_unresolved_operator_sequence, index]
+unified_unresolved_operator_sequence_element(
+  int unified_unresolved_operator_sequence: @unified_unresolved_operator_sequence ref,
+  int index: int ref,
+  unique int element: @unified_expr_or_operator ref
+);
+
+unified_unresolved_operator_sequence_def(
+  unique int id: @unified_unresolved_operator_sequence
+);
+
 #keyset[unified_variable_declaration, index]
 unified_variable_declaration_modifier(
   int unified_variable_declaration: @unified_variable_declaration ref,
@@ -1072,7 +1085,7 @@ unified_trivia_tokeninfo(
   string value: string ref
 );
 
-@unified_ast_node = @unified_accessor_declaration | @unified_argument | @unified_array_literal | @unified_assign_expr | @unified_associated_type_declaration | @unified_base_type | @unified_binary_expr | @unified_block | @unified_bound_type_constraint | @unified_break_expr | @unified_bulk_importing_pattern | @unified_call_expr | @unified_catch_clause | @unified_class_like_declaration | @unified_compound_assign_expr | @unified_constructor_declaration | @unified_constructor_pattern | @unified_continue_expr | @unified_destructor_declaration | @unified_do_while_stmt | @unified_equality_type_constraint | @unified_expr_equality_pattern | @unified_for_each_stmt | @unified_function_declaration | @unified_function_expr | @unified_function_type_expr | @unified_generic_type_expr | @unified_guard_if_stmt | @unified_if_expr | @unified_import_declaration | @unified_initializer_declaration | @unified_key_value_pair | @unified_labeled_stmt | @unified_map_literal | @unified_member_access_expr | @unified_name_expr | @unified_name_pattern | @unified_named_type_expr | @unified_operator_syntax_declaration | @unified_or_pattern | @unified_parameter | @unified_pattern_element | @unified_pattern_guard_expr | @unified_return_expr | @unified_switch_case | @unified_switch_expr | @unified_throw_expr | @unified_token | @unified_top_level | @unified_trivia_token | @unified_try_expr | @unified_tuple_expr | @unified_tuple_pattern | @unified_tuple_type_element | @unified_tuple_type_expr | @unified_type_alias_declaration | @unified_type_cast_expr | @unified_type_parameter | @unified_type_test_expr | @unified_type_test_pattern | @unified_unary_expr | @unified_variable_declaration | @unified_while_stmt
+@unified_ast_node = @unified_accessor_declaration | @unified_argument | @unified_array_literal | @unified_assign_expr | @unified_associated_type_declaration | @unified_base_type | @unified_binary_expr | @unified_block | @unified_bound_type_constraint | @unified_break_expr | @unified_bulk_importing_pattern | @unified_call_expr | @unified_catch_clause | @unified_class_like_declaration | @unified_compound_assign_expr | @unified_constructor_declaration | @unified_constructor_pattern | @unified_continue_expr | @unified_destructor_declaration | @unified_do_while_stmt | @unified_equality_type_constraint | @unified_expr_equality_pattern | @unified_for_each_stmt | @unified_function_declaration | @unified_function_expr | @unified_function_type_expr | @unified_generic_type_expr | @unified_guard_if_stmt | @unified_if_expr | @unified_import_declaration | @unified_initializer_declaration | @unified_key_value_pair | @unified_labeled_stmt | @unified_map_literal | @unified_member_access_expr | @unified_name_expr | @unified_name_pattern | @unified_named_type_expr | @unified_operator_syntax_declaration | @unified_or_pattern | @unified_parameter | @unified_pattern_element | @unified_pattern_guard_expr | @unified_return_expr | @unified_switch_case | @unified_switch_expr | @unified_throw_expr | @unified_token | @unified_top_level | @unified_trivia_token | @unified_try_expr | @unified_tuple_expr | @unified_tuple_pattern | @unified_tuple_type_element | @unified_tuple_type_expr | @unified_type_alias_declaration | @unified_type_cast_expr | @unified_type_parameter | @unified_type_test_expr | @unified_type_test_pattern | @unified_unary_expr | @unified_unresolved_operator_sequence | @unified_variable_declaration | @unified_while_stmt
 
 unified_ast_node_location(
   unique int node: @unified_ast_node ref,