Unified: Add some more AST nodes and rules

unified/extractor/ast_types.yml

@@ -1,8 +1,10 @@
 supertypes:
   expr:
+    - name_expr
+    - int_literal
+    - string_literal
     - binary_expr
     - unary_expr
-    - name_expr
     - lambda_expr
     - unsupported_node
   stmt:
@@ -23,7 +25,17 @@ supertypes:
 named:
   # Top-level is the root node, currently containing a list of expressions
   top_level:
-    body*: expr
+    body*: [expr, stmt]
+
+  # An identifier used in the context of an expression
+  name_expr:
+    identifier: identifier
+
+  # An integer literal
+  int_literal:
+
+  # A string literal
+  string_literal:
 
   # Application of a binary operator, such as `a + b`
   binary_expr:
@@ -36,10 +48,6 @@ named:
     operand: expr
     operator: operator
 
-  # An identifier used in the context of an expression
-  name_expr:
-    identifier: identifier
-
   lambda_expr:
     parameter*: parameter
     body: [expr, stmt]
@@ -50,6 +58,12 @@ named:
 
   empty_stmt:
 
+  block_stmt:
+    body*: stmt
+
+  expr_stmt:
+    expr: expr
+
   if_stmt:
     condition: condition
     then?: stmt

unified/extractor/src/languages/swift/swift.rs

@@ -10,6 +10,10 @@ fn translation_rules() -> Vec<yeast::Rule> {
                 body: {..children}
             )
         ),
+        // ---- Binary expressions ----
+        // Swift's parser produces a different node kind for each operator
+        // family, but the field shape (`lhs` / `op` / `rhs`) is uniform, so
+        // each maps onto `binary_expr`.
         rule!(
             (additive_expression
                 lhs: (_) @left
@@ -33,10 +37,134 @@ fn translation_rules() -> Vec<yeast::Rule> {
                 right: {right})
         ),
         rule!(
-            (simple_identifier)
+            (comparison_expression
+                lhs: (_) @left
+                op: _ @operator
+                rhs: (_) @right)
             =>
-            name_expr
+            (binary_expr
+                left: {left}
+                operator: (operator #{operator})
+                right: {right})
         ),
+        rule!(
+            (equality_expression
+                lhs: (_) @left
+                op: _ @operator
+                rhs: (_) @right)
+            =>
+            (binary_expr
+                left: {left}
+                operator: (operator #{operator})
+                right: {right})
+        ),
+        rule!(
+            (conjunction_expression
+                lhs: (_) @left
+                op: _ @operator
+                rhs: (_) @right)
+            =>
+            (binary_expr
+                left: {left}
+                operator: (operator #{operator})
+                right: {right})
+        ),
+        rule!(
+            (disjunction_expression
+                lhs: (_) @left
+                op: _ @operator
+                rhs: (_) @right)
+            =>
+            (binary_expr
+                left: {left}
+                operator: (operator #{operator})
+                right: {right})
+        ),
+        rule!(
+            (nil_coalescing_expression
+                lhs: (_) @left
+                op: _ @operator
+                rhs: (_) @right)
+            =>
+            (binary_expr
+                left: {left}
+                operator: (operator #{operator})
+                right: {right})
+        ),
+        rule!(
+            (range_expression
+                start: (_) @left
+                op: _ @operator
+                end: (_) @right)
+            =>
+            (binary_expr
+                left: {left}
+                operator: (operator #{operator})
+                right: {right})
+        ),
+        // ---- Unary expressions ----
+        rule!(
+            (prefix_expression
+                operation: _ @operator
+                target: (_) @operand)
+            =>
+            (unary_expr
+                operand: {operand}
+                operator: (operator #{operator}))
+        ),
+        // ---- Identifiers / name expressions ----
+        rule!(
+            (simple_identifier) @name
+            =>
+            (name_expr
+                identifier: (identifier #{name}))
+        ),
+        // ---- Literals ----
+        rule!(
+            (integer_literal) @lit
+            =>
+            (int_literal #{lit})
+        ),
+        // String literals: render the *raw* source text, including the
+        // surrounding quotes. Interpolations (e.g. `"hi \(x)"`) are not
+        // yet broken out into structured pieces \u2014 they show up as part
+        // of the literal's source text.
+        rule!(
+            (line_string_literal) @lit
+            =>
+            (string_literal #{lit})
+        ),
+        // ---- Patterns ----
+        // The Swift parser uses a `pattern` node with a `bound_identifier`
+        // field for simple bindings such as `let x = ...`.
+        rule!(
+            (pattern bound_identifier: (simple_identifier) @id)
+            =>
+            (var_pattern
+                identifier: (identifier #{id}))
+        ),
+        // ---- Variable declarations ----
+        // `let x = e` / `var x = e` (with initializer).
+        rule!(
+            (property_declaration
+                name: (_) @pat
+                value: (_) @value)
+            =>
+            (variable_declaration_stmt
+                variable_declarator: (variable_declarator
+                    pattern: {pat}
+                    value: {value}))
+        ),
+        // `var x: T` (no initializer).
+        rule!(
+            (property_declaration
+                name: (_) @pat)
+            =>
+            (variable_declaration_stmt
+                variable_declarator: (variable_declarator
+                    pattern: {pat}))
+        ),
+        // ---- Fallbacks ----
         rule!(
             (_)
             =>

unified/extractor/tests/corpus/swift/closures.txt

@@ -2,11 +2,13 @@
 Closure with explicit parameters
 ===
 
+// TODO: map lambda_literal -> lambda_expr
 let f = { (x: Int) -> Int in x * 2 }
 
 ---
 
 source_file
+  comment "// TODO: map lambda_literal -> lambda_expr"
   property_declaration
     name:
       pattern
@@ -35,7 +37,15 @@ source_file
 ---
 
 top_level
-  body: unsupported_node "let f = { (x: Int) -> Int in x * 2 }"
+  body:
+    unsupported_node "// TODO: map lambda_literal -> lambda_expr"
+    variable_declaration_stmt
+      variable_declarator:
+        variable_declarator
+          value: unsupported_node "{ (x: Int) -> Int in x * 2 }"
+          pattern:
+            var_pattern
+              identifier: identifier "f"
 
 ===
 Closure with shorthand parameters
@@ -63,7 +73,14 @@ source_file
 ---
 
 top_level
-  body: unsupported_node "let f = { $0 + $1 }"
+  body:
+    variable_declaration_stmt
+      variable_declarator:
+        variable_declarator
+          value: unsupported_node "{ $0 + $1 }"
+          pattern:
+            var_pattern
+              identifier: identifier "f"
 
 ===
 Trailing closure
@@ -130,7 +147,14 @@ source_file
 ---
 
 top_level
-  body: unsupported_node "let f = { [weak self] in self?.doThing() }"
+  body:
+    variable_declaration_stmt
+      variable_declarator:
+        variable_declarator
+          value: unsupported_node "{ [weak self] in self?.doThing() }"
+          pattern:
+            var_pattern
+              identifier: identifier "f"
 
 ===
 Multi-statement closure
@@ -185,4 +209,11 @@ source_file
 ---
 
 top_level
-  body: unsupported_node "let f = { (x: Int) -> Int in\n  let y = x + 1\n  return y * 2\n}"
+  body:
+    variable_declaration_stmt
+      variable_declarator:
+        variable_declarator
+          value: unsupported_node "{ (x: Int) -> Int in\n  let y = x + 1\n  return y * 2\n}"
+          pattern:
+            var_pattern
+              identifier: identifier "f"

unified/extractor/tests/corpus/swift/collections.txt

@@ -23,7 +23,14 @@ source_file
 ---
 
 top_level
-  body: unsupported_node "let xs = [1, 2, 3]"
+  body:
+    variable_declaration_stmt
+      variable_declarator:
+        variable_declarator
+          value: unsupported_node "[1, 2, 3]"
+          pattern:
+            var_pattern
+              identifier: identifier "xs"
 
 ===
 Empty array literal with type
@@ -52,7 +59,14 @@ source_file
 ---
 
 top_level
-  body: unsupported_node "let xs: [Int] = []"
+  body:
+    variable_declaration_stmt
+      variable_declarator:
+        variable_declarator
+          value: unsupported_node "[]"
+          pattern:
+            var_pattern
+              identifier: identifier "xs"
 
 ===
 Dictionary literal
@@ -83,7 +97,14 @@ source_file
 ---
 
 top_level
-  body: unsupported_node "let d = [\"a\": 1, \"b\": 2]"
+  body:
+    variable_declaration_stmt
+      variable_declarator:
+        variable_declarator
+          value: unsupported_node "[\"a\": 1, \"b\": 2]"
+          pattern:
+            var_pattern
+              identifier: identifier "d"
 
 ===
 Set literal
@@ -118,7 +139,14 @@ source_file
 ---
 
 top_level
-  body: unsupported_node "let s: Set<Int> = [1, 2, 3]"
+  body:
+    variable_declaration_stmt
+      variable_declarator:
+        variable_declarator
+          value: unsupported_node "[1, 2, 3]"
+          pattern:
+            var_pattern
+              identifier: identifier "s"
 
 ===
 Tuple literal
@@ -146,7 +174,14 @@ source_file
 ---
 
 top_level
-  body: unsupported_node "let t = (1, \"two\", 3.0)"
+  body:
+    variable_declaration_stmt
+      variable_declarator:
+        variable_declarator
+          value: unsupported_node "(1, \"two\", 3.0)"
+          pattern:
+            var_pattern
+              identifier: identifier "t"
 
 ===
 Subscript access
@@ -174,7 +209,14 @@ source_file
 ---
 
 top_level
-  body: unsupported_node "let first = xs[0]"
+  body:
+    variable_declaration_stmt
+      variable_declarator:
+        variable_declarator
+          value: unsupported_node "xs[0]"
+          pattern:
+            var_pattern
+              identifier: identifier "first"
 
 ===
 Dictionary subscript
@@ -204,7 +246,14 @@ source_file
 ---
 
 top_level
-  body: unsupported_node "let v = d[\"key\"]"
+  body:
+    variable_declaration_stmt
+      variable_declarator:
+        variable_declarator
+          value: unsupported_node "d[\"key\"]"
+          pattern:
+            var_pattern
+              identifier: identifier "v"
 
 ===
 Tuple member access
@@ -231,4 +280,11 @@ source_file
 ---
 
 top_level
-  body: unsupported_node "let n = t.0"
+  body:
+    variable_declaration_stmt
+      variable_declarator:
+        variable_declarator
+          value: unsupported_node "t.0"
+          pattern:
+            var_pattern
+              identifier: identifier "n"

unified/extractor/tests/corpus/swift/control-flow.txt

@@ -2,6 +2,7 @@
 If statement
 ===
 
+// TODO: map if_statement -> if_stmt (needs a block-body stmt in ast_types.yml)
 if x > 0 {
   print(x)
 }
@@ -9,6 +10,7 @@ if x > 0 {
 ---
 
 source_file
+  comment "// TODO: map if_statement -> if_stmt (needs a block-body stmt in ast_types.yml)"
   if_statement
     condition:
       comparison_expression
@@ -26,7 +28,9 @@ source_file
 ---
 
 top_level
-  body: unsupported_node "if x > 0 {\n  print(x)\n}"
+  body:
+    unsupported_node "// TODO: map if_statement -> if_stmt (needs a block-body stmt in ast_types.yml)"
+    unsupported_node "if x > 0 {\n  print(x)\n}"
 
 ===
 If-else
@@ -131,6 +135,7 @@ top_level
 If-let optional binding
 ===
 
+// TODO: map if-let -> if_stmt with let_pattern_condition
 if let value = optional {
   print(value)
 }
@@ -138,6 +143,7 @@ if let value = optional {
 ---
 
 source_file
+  comment "// TODO: map if-let -> if_stmt with let_pattern_condition"
   if_statement
     bound_identifier: simple_identifier "value"
     condition:
@@ -156,17 +162,21 @@ source_file
 ---
 
 top_level
-  body: unsupported_node "if let value = optional {\n  print(value)\n}"
+  body:
+    unsupported_node "// TODO: map if-let -> if_stmt with let_pattern_condition"
+    unsupported_node "if let value = optional {\n  print(value)\n}"
 
 ===
 Guard let
 ===
 
+// TODO: map guard_statement -> guard_if_stmt (with let_pattern_condition)
 guard let value = optional else { return }
 
 ---
 
 source_file
+  comment "// TODO: map guard_statement -> guard_if_stmt (with let_pattern_condition)"
   guard_statement
     bound_identifier: simple_identifier "value"
     condition:
@@ -181,7 +191,9 @@ source_file
 ---
 
 top_level
-  body: unsupported_node "guard let value = optional else { return }"
+  body:
+    unsupported_node "// TODO: map guard_statement -> guard_if_stmt (with let_pattern_condition)"
+    unsupported_node "guard let value = optional else { return }"
 
 ===
 Ternary expression
@@ -214,7 +226,14 @@ source_file
 ---
 
 top_level
-  body: unsupported_node "let y = x > 0 ? 1 : -1"
+  body:
+    variable_declaration_stmt
+      variable_declarator:
+        variable_declarator
+          value: unsupported_node "x > 0 ? 1 : -1"
+          pattern:
+            var_pattern
+              identifier: identifier "y"
 
 ===
 Switch statement

unified/extractor/tests/corpus/swift/desugar.txt

@@ -18,8 +18,8 @@ top_level
   body:
     binary_expr
       operator: operator "+"
-      left: unsupported_node "1"
-      right: unsupported_node "2"
+      left: int_literal "1"
+      right: int_literal "2"
 
 ===
 Another additive expression is desugared
@@ -41,5 +41,9 @@ top_level
   body:
     binary_expr
       operator: operator "+"
-      left: name_expr "foo"
-      right: name_expr "bar"
+      left:
+        name_expr
+          identifier: identifier "foo"
+      right:
+        name_expr
+          identifier: identifier "bar"

unified/extractor/tests/corpus/swift/literals.txt

@@ -12,7 +12,7 @@ source_file
 ---
 
 top_level
-  body: unsupported_node "42"
+  body: int_literal "42"
 
 ===
 Negative integer literal
@@ -30,7 +30,10 @@ source_file
 ---
 
 top_level
-  body: unsupported_node "-7"
+  body:
+    unary_expr
+      operator: operator "-"
+      operand: int_literal "7"
 
 ===
 Floating-point literal
@@ -98,7 +101,7 @@ source_file
 ---
 
 top_level
-  body: unsupported_node "\"hello\""
+  body: string_literal "\"hello\""
 
 ===
 String with interpolation
@@ -118,4 +121,4 @@ source_file
 ---
 
 top_level
-  body: unsupported_node "\"hello \\(name)\""
+  body: string_literal "\"hello \\(name)\""

unified/extractor/tests/corpus/swift/operators.txt

@@ -18,8 +18,12 @@ top_level
   body:
     binary_expr
       operator: operator "+"
-      left: name_expr "a"
-      right: name_expr "b"
+      left:
+        name_expr
+          identifier: identifier "a"
+      right:
+        name_expr
+          identifier: identifier "b"
 
 ===
 Subtraction
@@ -41,8 +45,12 @@ top_level
   body:
     binary_expr
       operator: operator "-"
-      left: name_expr "a"
-      right: name_expr "b"
+      left:
+        name_expr
+          identifier: identifier "a"
+      right:
+        name_expr
+          identifier: identifier "b"
 
 ===
 Multiplication
@@ -64,8 +72,12 @@ top_level
   body:
     binary_expr
       operator: operator "*"
-      left: name_expr "a"
-      right: name_expr "b"
+      left:
+        name_expr
+          identifier: identifier "a"
+      right:
+        name_expr
+          identifier: identifier "b"
 
 ===
 Division
@@ -87,8 +99,12 @@ top_level
   body:
     binary_expr
       operator: operator "/"
-      left: name_expr "a"
-      right: name_expr "b"
+      left:
+        name_expr
+          identifier: identifier "a"
+      right:
+        name_expr
+          identifier: identifier "b"
 
 ===
 Operator precedence: addition and multiplication
@@ -114,12 +130,18 @@ top_level
   body:
     binary_expr
       operator: operator "+"
-      left: name_expr "a"
+      left:
+        name_expr
+          identifier: identifier "a"
       right:
         binary_expr
           operator: operator "*"
-          left: name_expr "b"
-          right: name_expr "c"
+          left:
+            name_expr
+              identifier: identifier "b"
+          right:
+            name_expr
+              identifier: identifier "c"
 
 ===
 Parenthesised expression
@@ -148,7 +170,9 @@ top_level
     binary_expr
       operator: operator "*"
       left: unsupported_node "(a + b)"
-      right: name_expr "c"
+      right:
+        name_expr
+          identifier: identifier "c"
 
 ===
 Comparison
@@ -167,7 +191,15 @@ source_file
 ---
 
 top_level
-  body: unsupported_node "a < b"
+  body:
+    binary_expr
+      operator: operator "<"
+      left:
+        name_expr
+          identifier: identifier "a"
+      right:
+        name_expr
+          identifier: identifier "b"
 
 ===
 Equality
@@ -186,7 +218,15 @@ source_file
 ---
 
 top_level
-  body: unsupported_node "a == b"
+  body:
+    binary_expr
+      operator: operator "=="
+      left:
+        name_expr
+          identifier: identifier "a"
+      right:
+        name_expr
+          identifier: identifier "b"
 
 ===
 Logical and
@@ -205,7 +245,15 @@ source_file
 ---
 
 top_level
-  body: unsupported_node "a && b"
+  body:
+    binary_expr
+      operator: operator "&&"
+      left:
+        name_expr
+          identifier: identifier "a"
+      right:
+        name_expr
+          identifier: identifier "b"
 
 ===
 Logical or
@@ -224,7 +272,15 @@ source_file
 ---
 
 top_level
-  body: unsupported_node "a || b"
+  body:
+    binary_expr
+      operator: operator "||"
+      left:
+        name_expr
+          identifier: identifier "a"
+      right:
+        name_expr
+          identifier: identifier "b"
 
 ===
 Logical not
@@ -242,7 +298,12 @@ source_file
 ---
 
 top_level
-  body: unsupported_node "!a"
+  body:
+    unary_expr
+      operator: operator "!"
+      operand:
+        name_expr
+          identifier: identifier "a"
 
 ===
 Range operator
@@ -261,4 +322,8 @@ source_file
 ---
 
 top_level
-  body: unsupported_node "1...10"
+  body:
+    binary_expr
+      operator: operator "..."
+      left: int_literal "1"
+      right: int_literal "10"

unified/extractor/tests/corpus/swift/optionals-and-errors.txt

@@ -24,7 +24,13 @@ source_file
 ---
 
 top_level
-  body: unsupported_node "let x: Int? = nil"
+  body:
+    variable_declaration_stmt
+      variable_declarator:
+        variable_declarator
+          pattern:
+            var_pattern
+              identifier: identifier "x"
 
 ===
 Optional chaining
@@ -59,7 +65,14 @@ source_file
 ---
 
 top_level
-  body: unsupported_node "let n = obj?.foo?.bar"
+  body:
+    variable_declaration_stmt
+      variable_declarator:
+        variable_declarator
+          value: unsupported_node "obj?.foo?.bar"
+          pattern:
+            var_pattern
+              identifier: identifier "n"
 
 ===
 Force unwrap
@@ -84,7 +97,14 @@ source_file
 ---
 
 top_level
-  body: unsupported_node "let n = opt!"
+  body:
+    variable_declaration_stmt
+      variable_declarator:
+        variable_declarator
+          value: unsupported_node "opt!"
+          pattern:
+            var_pattern
+              identifier: identifier "n"
 
 ===
 Nil-coalescing
@@ -109,7 +129,14 @@ source_file
 ---
 
 top_level
-  body: unsupported_node "let n = opt ?? 0"
+  body:
+    variable_declaration_stmt
+      variable_declarator:
+        variable_declarator
+          value: unsupported_node "opt ?? 0"
+          pattern:
+            var_pattern
+              identifier: identifier "n"
 
 ===
 Throwing function
@@ -204,7 +231,14 @@ source_file
 ---
 
 top_level
-  body: unsupported_node "let result = try? foo()"
+  body:
+    variable_declaration_stmt
+      variable_declarator:
+        variable_declarator
+          value: unsupported_node "try? foo()"
+          pattern:
+            var_pattern
+              identifier: identifier "result"
 
 ===
 Try! expression
@@ -233,4 +267,11 @@ source_file
 ---
 
 top_level
-  body: unsupported_node "let result = try! foo()"
+  body:
+    variable_declaration_stmt
+      variable_declarator:
+        variable_declarator
+          value: unsupported_node "try! foo()"
+          pattern:
+            var_pattern
+              identifier: identifier "result"

unified/extractor/tests/corpus/swift/variables.txt

@@ -18,7 +18,14 @@ source_file
 ---
 
 top_level
-  body: unsupported_node "let x = 1"
+  body:
+    variable_declaration_stmt
+      variable_declarator:
+        variable_declarator
+          value: int_literal "1"
+          pattern:
+            var_pattern
+              identifier: identifier "x"
 
 ===
 Var binding
@@ -40,7 +47,14 @@ source_file
 ---
 
 top_level
-  body: unsupported_node "var x = 1"
+  body:
+    variable_declaration_stmt
+      variable_declarator:
+        variable_declarator
+          value: int_literal "1"
+          pattern:
+            var_pattern
+              identifier: identifier "x"
 
 ===
 Let with type annotation
@@ -66,7 +80,14 @@ source_file
 ---
 
 top_level
-  body: unsupported_node "let x: Int = 1"
+  body:
+    variable_declaration_stmt
+      variable_declarator:
+        variable_declarator
+          value: int_literal "1"
+          pattern:
+            var_pattern
+              identifier: identifier "x"
 
 ===
 Var without initialiser
@@ -91,17 +112,25 @@ source_file
 ---
 
 top_level
-  body: unsupported_node "var x: Int"
+  body:
+    variable_declaration_stmt
+      variable_declarator:
+        variable_declarator
+          pattern:
+            var_pattern
+              identifier: identifier "x"
 
 ===
 Tuple destructuring binding
 ===
 
+// TODO: map tuple destructuring pattern -> apply_pattern
 let (a, b) = pair
 
 ---
 
 source_file
+  comment "// TODO: map tuple destructuring pattern -> apply_pattern"
   property_declaration
     name:
       pattern
@@ -116,17 +145,27 @@ source_file
 ---
 
 top_level
-  body: unsupported_node "let (a, b) = pair"
+  body:
+    unsupported_node "// TODO: map tuple destructuring pattern -> apply_pattern"
+    variable_declaration_stmt
+      variable_declarator:
+        variable_declarator
+          value:
+            name_expr
+              identifier: identifier "pair"
+          pattern: unsupported_node "(a, b)"
 
 ===
 Multiple bindings on one line
 ===
 
+// TODO: emit multiple variable_declarators in one variable_declaration_stmt
 let x = 1, y = 2
 
 ---
 
 source_file
+  comment "// TODO: emit multiple variable_declarators in one variable_declaration_stmt"
   property_declaration
     name:
       pattern
@@ -142,7 +181,15 @@ source_file
 ---
 
 top_level
-  body: unsupported_node "let x = 1, y = 2"
+  body:
+    unsupported_node "// TODO: emit multiple variable_declarators in one variable_declaration_stmt"
+    variable_declaration_stmt
+      variable_declarator:
+        variable_declarator
+          value: int_literal "1"
+          pattern:
+            var_pattern
+              identifier: identifier "x"
 
 ===
 Assignment