diff --git a/shared/yeast-macros/src/parse.rs b/shared/yeast-macros/src/parse.rs index 01070c74bbc..986a9bac641 100644 --- a/shared/yeast-macros/src/parse.rs +++ b/shared/yeast-macros/src/parse.rs @@ -617,6 +617,76 @@ fn extract_captures_inner( } } +/// A rule's return-type annotation, when the body is a Rust block. Written +/// between `=>` and the block body using the schema's own vocabulary: +/// +/// ```text +/// => kind { … } // single node of that kind +/// => kind? { … } // Option (0 or 1) +/// => kind* { … } // Vec (0+) +/// ``` +/// +/// Template bodies (`=> (kind …)`) never carry an annotation — the +/// output kind is the template root. The shorthand `=> kind` (no +/// body) also carries no annotation. See `parse_rule_top` for dispatch. +#[derive(Clone, Debug)] +struct ReturnAnnotation { + kind: Ident, + multiplicity: AnnotationMultiplicity, +} + +#[derive(Clone, Copy, Debug, PartialEq)] +enum AnnotationMultiplicity { + Single, + Optional, + Repeated, +} + +/// Peek at the token stream to decide whether the transform following +/// `=>` is a **new** annotation form (`kind [? | *] { … }`). If so, +/// consume the annotation and return it, leaving the `{ … }` body in +/// the stream for the caller to parse. Otherwise leave the stream +/// untouched and return `None`. +/// +/// The lookahead distinguishes: +/// `kind {` → annotation (single) +/// `kind? {` → annotation (optional) +/// `kind* {` → annotation (repeated) +/// `kind` → shorthand form (no `{` follows) — NOT an annotation +/// anything else → template or bare block — NOT an annotation +fn try_consume_return_annotation(tokens: &mut Tokens) -> Result> { + // Must start with an identifier (the kind name). + let mut lookahead = tokens.clone(); + let Some(TokenTree::Ident(_)) = lookahead.next() else { + return Ok(None); + }; + // Then optionally `?` or `*`, then a `{` group. + let after_suffix = match lookahead.peek() { + Some(TokenTree::Punct(p)) if p.as_char() == '?' || p.as_char() == '*' => { + lookahead.next(); + lookahead.peek() + } + other => other, + }; + if !matches!(after_suffix, Some(TokenTree::Group(g)) if g.delimiter() == Delimiter::Brace) { + return Ok(None); + } + // Commit: consume the ident + suffix from the real stream. + let kind = expect_ident(tokens, "expected output-kind name in annotation")?; + let multiplicity = match tokens.peek() { + Some(TokenTree::Punct(p)) if p.as_char() == '?' => { + tokens.next(); + AnnotationMultiplicity::Optional + } + Some(TokenTree::Punct(p)) if p.as_char() == '*' => { + tokens.next(); + AnnotationMultiplicity::Repeated + } + _ => AnnotationMultiplicity::Single, + }; + Ok(Some(ReturnAnnotation { kind, multiplicity })) +} + /// Parse `rule!( query => transform )`. pub fn parse_rule_top(input: TokenStream) -> Result { let mut tokens = input.into_iter().peekable(); @@ -688,8 +758,52 @@ pub fn parse_rule_top(input: TokenStream) -> Result { }) .collect(); - // Parse transform: either shorthand `=> kind_name` or full `=> (template ...)` - let transform_body = if peek_is_field(&mut tokens) && { + // Parse transform: the token(s) after `=>` fall into one of three + // shapes, dispatched in order: + // + // 1. `kind [? | *] { rust_body }` — annotated Rust body (NEW). + // Static-analysis-ready: the annotation declares the output + // kind and multiplicity in the schema's own vocabulary. + // 2. `kind` alone — shorthand: emit `(kind field: {@cap})…` from + // the query's captures. + // 3. anything else — full template form (`(kind …)` or bare + // `{ … }` splice via `parse_direct_list`). + let annotation = try_consume_return_annotation(&mut tokens)?; + + let transform_body = if let Some(annotation) = annotation { + // Annotation form: `=> kind [? | *] { rust_body }`. + let body_group = expect_group(&mut tokens, Delimiter::Brace)?; + if let Some(tok) = tokens.next() { + return Err(syn::Error::new_spanned( + tok, + "unexpected token after annotated rule body", + )); + } + let body = body_group.stream(); + // The annotation is not yet consumed by codegen — it will drive + // typed handles once the schema-driven codegen lands. For now, + // emit a self-documenting reference to the annotated kind and + // preserve today's `Vec` closure return so behavior + // is unchanged. + let kind_str = annotation.kind.to_string(); + let mult_str = match annotation.multiplicity { + AnnotationMultiplicity::Single => "single", + AnnotationMultiplicity::Optional => "optional", + AnnotationMultiplicity::Repeated => "repeated", + }; + let _ = (kind_str, mult_str); // silence unused warnings until wired + + // For now, adapt the user's typed return value to the framework's + // `Vec` closure result. This uses `IntoFieldIds`, which + // already accepts a bare `Id`, an iterable of ids, or `Option` + // — matching the three annotation multiplicities. + quote! { + let __value = { #body }; + let mut __ids: Vec = Vec::new(); + yeast::IntoFieldIds::extend_into(__value, &mut __ids); + __ids + } + } else if peek_is_field(&mut tokens) && { // Shorthand form: bare identifier = output node kind. // Auto-generate template from captures. let mut lookahead = tokens.clone(); @@ -749,6 +863,26 @@ pub fn parse_rule_top(input: TokenStream) -> Result { vec![__id] } } else { + // Reject bare `{ ... }` transforms — they used to be accepted + // as either a Rust body producing a `Vec` or a template + // consisting of a single `{cap}` splice. Both patterns lost + // static-analysis information (no visible output kind), so we + // now require rules with block bodies to use the annotation + // form `=> kind [? | *] { ... }`. Templates must start with a + // parenthesized node (e.g. `(if_expr ...)`). + if let Some(TokenTree::Group(g)) = tokens.peek() { + if g.delimiter() == Delimiter::Brace { + let span = g.span(); + return Err(syn::Error::new( + span, + "bare `{...}` rule bodies are no longer accepted; \ + use the annotation form `=> kind [? | *] { ... }` \ + (where the kind names the output node's schema kind, \ + optionally suffixed with `?` or `*` for multiplicity)", + )); + } + } + // Full template form let transform_items = parse_direct_list(&mut tokens, &ctx_ident)?; @@ -1026,10 +1160,7 @@ struct NamedString { } fn parse_named_string_arg(tokens: &mut Tokens, expected_name: &str) -> Result { - let name = expect_ident( - tokens, - &format!("expected `{expected_name}:` argument"), - )?; + let name = expect_ident(tokens, &format!("expected `{expected_name}:` argument"))?; if name != expected_name { return Err(syn::Error::new_spanned( name, diff --git a/shared/yeast/doc/yeast.md b/shared/yeast/doc/yeast.md index 3427597be2e..90edb510c1a 100644 --- a/shared/yeast/doc/yeast.md +++ b/shared/yeast/doc/yeast.md @@ -312,13 +312,15 @@ already conforms to the output schema. For rules that need the raw (input-schema) capture — typically to read its source text or to translate it explicitly with mutable context state between calls — use `@@name` instead. The body sees the original -input-schema `Id`: +input-schema `Id`. Because these rules always have a Rust block body, +they use the annotation form (see [the `rule!` macro +section](#the-rule-macro) for the full grammar): ```rust yeast::rule!( (assignment left: (_) @@raw_lhs right: (_) @rhs) => - { + call { // raw_lhs is untranslated: read its original source text. let text = ctx.ast.source_text(raw_lhs); // rhs is already translated by the auto-translate prefix. @@ -372,26 +374,79 @@ automatically: single captures bind as `Id`, repeated captures (after ## The `rule!` macro -`rule!` combines a query and a transform into a single declaration: +`rule!` combines a query and a transform into a single declaration. +There are three transform forms, each suited to a different level of +rule complexity: ```rust -// Full template form +// 1. Template form — a tree literal describing the output. yeast::rule!( (query_pattern field: (_) @capture) => (output_template field: {capture}) ) -// Shorthand form — captures become fields on the output node +// 2. Shorthand form — captures become fields on a bare output kind. yeast::rule!( (query_pattern field: (_) @capture) => output_kind ) + +// 3. Annotation form — a Rust block body preceded by the output kind. +yeast::rule!( + (query_pattern child: (_)+ @@children) + => + output_kind* { + // arbitrary Rust; must evaluate to a value compatible with the + // declared multiplicity (see below). + let mut result = Vec::new(); + for child in children { + result.extend(ctx.translate(child)?); + } + result + } +) ``` The shorthand `=> kind` form auto-generates the template, mapping each capture name to a field of the same name on the output node. +### Annotation form + +Rules that need imperative logic — mutating [`BuildCtx`] state per +iteration, computing intermediate values, or looping over captures — +use the annotation form. It has three shapes distinguished by a suffix +on the output-kind identifier: + +| Syntax | Body must evaluate to | Meaning | +|---------------------|-------------------------------------|--------------------------------| +| `=> kind { ... }` | a single node id of `kind` | Emit exactly one node. | +| `=> kind? { ... }` | an `Option` of a node id of `kind` | Emit 0 or 1 nodes (`None`/`Some`). | +| `=> kind* { ... }` | an iterable of node ids of `kind` | Emit 0+ nodes; flattens into the enclosing splice slot. | + +The suffix mirrors the `?` / `*` markers used elsewhere in the schema +DSL (see [`ast_types.yml`](../../../unified/extractor/ast_types.yml)): +bare identifier = required single, `?` = optional single, `*` = +repeated. + +The annotation names the schema kind of the output, giving the macro +enough information for future static analysis (e.g. computing the +static output type of translated captures at their consumer sites). + +**Bare `=> { ... }` block bodies are rejected** — every Rust-block body +must carry an annotation, so the output kind is always visible without +having to inspect the block's expression. + +### Choosing between the forms + +Prefer the simplest form that fits: + +- If the whole transform is a tree literal, use the **template form**. +- If the transform is a template whose root matches a query capture + 1:1, use the **shorthand form**. +- If the transform needs Rust logic (loops, `let` bindings, calls to + `ctx.translate`, etc.), use the **annotation form**. + ## Integration with the extractor A YEAST desugaring pass is configured with a [`DesugaringConfig`], which diff --git a/shared/yeast/tests/test.rs b/shared/yeast/tests/test.rs index 4a2fb26f4fe..3a24709dd9f 100644 --- a/shared/yeast/tests/test.rs +++ b/shared/yeast/tests/test.rs @@ -989,7 +989,7 @@ fn test_one_shot_recurses_into_returned_capture() { yeast::rule!( (assignment left: (_) @left right: (_) @right) => - {left} + identifier { left } ), yeast::rule!((identifier) => (identifier "ID")), yeast::rule!((integer) => (integer "INT")), @@ -1084,7 +1084,7 @@ fn test_raw_capture_marker() { yeast::rule!( (assignment left: (_) @@raw_lhs right: (_) @rhs) => - { + call { let text = ctx.ast.source_text(raw_lhs); tree!((call method: (identifier #{text.as_str()}) @@ -1139,7 +1139,7 @@ fn test_raw_capture_marker_explicit_translate() { yeast::rule!( (assignment left: (_) @@raw_lhs right: (_) @rhs) => - { + call { let translated_lhs = ctx.translate(raw_lhs)?; tree!((call method: {translated_lhs} @@ -1442,3 +1442,113 @@ fn test_rules_macro_mixes_bare_and_explicit_forms() { }; assert_eq!(rules.len(), 2); } + +// ---- Rule-body return-type annotation tests ---- +// +// The annotation form `=> kind [? | *] { rust_body }` is the future +// interface for Rust-bodied rules: the schema-vocabulary annotation +// declares the rule's output kind for static analysis. Today's codegen +// does NOT yet consume the annotation (it just adapts the returned +// value to `Vec` via `IntoFieldIds`); these tests only exercise +// the parser + the runtime-equivalence property. + +/// Annotation form with `*` (repeated): the rule body returns a +/// `Vec` and the annotation says the outputs are `assignment`s. +#[test] +fn test_rule_annotation_repeated() { + // Behaviourally equivalent to a two-node splice template. + let r: Rule = rule!( + (assignment left: (_) @l right: (_) @r) + => + assignment* { + let a1 = tree!((assignment left: {l} right: {r})); + let a2 = tree!((assignment left: {r} right: {l})); + vec![a1, a2] + } + ); + let ast = run_and_ast("x = 1", vec![r]); + // Just verify the run completes without a schema error; two + // top-level `assignment` nodes should appear as siblings. + let mut count = 0usize; + for id in ast.reachable_node_ids() { + if let Some(n) = ast.get_node(id) { + if n.kind_name() == "assignment" { + count += 1; + } + } + } + assert!( + count >= 2, + "expected at least two assignment nodes, got {count}" + ); +} + +/// Annotation form with `?` (optional): the rule body returns +/// `Option`. This uses `None` so the rule effectively deletes the +/// node. +#[test] +fn test_rule_annotation_optional_none() { + // Delete every `integer` (returning None yields no output nodes). + let r: Rule = rule!( + (integer) @lit + => + integer? { + let _ = lit; + None:: + } + ); + let ast = run_and_ast("42", vec![r]); + // No integer node should survive. + for id in ast.reachable_node_ids() { + if let Some(n) = ast.get_node(id) { + assert_ne!(n.kind_name(), "integer", "integer should have been deleted"); + } + } +} + +/// Annotation form (single): the rule body returns a bare `Id`. +#[test] +fn test_rule_annotation_single() { + // Identity on assignment nodes, expressed with the annotation form. + let r: Rule = rule!( + (assignment left: (_) @l right: (_) @r) + => + assignment { + tree!((assignment left: {l} right: {r})) + } + ); + let ast = run_and_ast("x = 1", vec![r]); + let mut has_assignment = false; + for id in ast.reachable_node_ids() { + if let Some(n) = ast.get_node(id) { + if n.kind_name() == "assignment" { + has_assignment = true; + } + } + } + assert!(has_assignment, "expected an assignment node"); +} + +/// The shorthand `=> kind` form (no body, no annotation) must still be +/// distinguished from the annotation form and continue to work. +#[test] +fn test_shorthand_still_works_alongside_annotation_syntax() { + let r: Rule = rule!( + (assignment left: (_) @method right: (_) @receiver) + => + call + ); + let ast = run_and_ast("x = 1", vec![r]); + let mut has_call = false; + for id in ast.reachable_node_ids() { + if let Some(n) = ast.get_node(id) { + if n.kind_name() == "call" { + has_call = true; + } + } + } + assert!( + has_call, + "shorthand form should still produce a `call` node" + ); +} diff --git a/unified/extractor/src/languages/swift/swift.rs b/unified/extractor/src/languages/swift/swift.rs index 5689d930bff..908a7bd437f 100644 --- a/unified/extractor/src/languages/swift/swift.rs +++ b/unified/extractor/src/languages/swift/swift.rs @@ -104,8 +104,8 @@ fn translation_rules() -> Vec> { ) ), // Declarations may be wrapped in local/global wrapper nodes. - rule!((global_declaration _ @inner) => {inner}), - rule!((local_declaration _ @inner) => {inner}), + rule!((global_declaration _ @inner) => stmt { inner }), + rule!((local_declaration _ @inner) => stmt { inner }), // ---- Literals ---- rule!((integer_literal) => (int_literal)), rule!((hex_literal) => (int_literal)), @@ -198,7 +198,7 @@ fn translation_rules() -> Vec> { type: _? @ty computed_value: (computed_property accessor: _+ @@accessors)) => - {{ + accessor_declaration* { ctx.property_name = Some(tree!((identifier #{pattern}))); ctx.property_type = ty; @@ -210,7 +210,7 @@ fn translation_rules() -> Vec> { result.extend(ctx.translate(acc)?); } result - }} + } ), // Computed property: shorthand getter (no explicit get/set, just // statements) → a single accessor_declaration with kind "get". @@ -249,7 +249,7 @@ fn translation_rules() -> Vec> { value: _? @val observers: (willset_didset_block willset: _? @@ws didset: _? @@ds)) => - {{ + member* { let var_decl = tree!( (variable_declaration modifier: {ctx.binding_modifier} @@ -271,7 +271,7 @@ fn translation_rules() -> Vec> { result.extend(ctx.translate(obs)?); } result - }} + } ), // property_binding with any pattern name (identifier or // destructuring). Reads outer modifiers / chained tag from `ctx`. @@ -305,7 +305,7 @@ fn translation_rules() -> Vec> { declarator: _* @@decls (modifiers)* @mods) => - {{ + member* { let binding_text = ctx.ast.source_text(binding_kind); ctx.binding_modifier = Some(ctx.literal("modifier", &binding_text)); ctx.outer_modifiers = mods; @@ -316,7 +316,7 @@ fn translation_rules() -> Vec> { result.extend(ctx.translate(decl)?); } result - }} + } ), // ---- Enums ---- // enum_type_parameter → parameter (with optional name as pattern). @@ -376,7 +376,7 @@ fn translation_rules() -> Vec> { rule!( (enum_entry case: _+ @@cases (modifiers)* @mods) => - {{ + member* { ctx.outer_modifiers = mods; let mut result = Vec::new(); @@ -385,7 +385,7 @@ fn translation_rules() -> Vec> { result.extend(ctx.translate(case)?); } result - }} + } ), // Plain assignment: `x = expr` rule!( @@ -400,9 +400,9 @@ fn translation_rules() -> Vec> { (compound_assign_expr target: {target} operator: (infix_operator #{op}) value: {value}) ), // Unwrap `type` wrapper node - rule!((type name: @inner) => {inner}), + rule!((type name: @inner) => type_expr { inner }), // `directly_assignable_expression` is just a wrapper; unwrap it - rule!((directly_assignable_expression expr: @inner) => {inner}), + rule!((directly_assignable_expression expr: @inner) => expr { inner }), // Pattern with bound_identifier → name_pattern rule!((pattern bound_identifier: @name) => (name_pattern identifier: (identifier #{name}))), // Pattern with 'let' or 'var' binding: extract the inner pattern @@ -410,7 +410,7 @@ fn translation_rules() -> Vec> { rule!( (pattern kind: (binding_pattern binding: _? pattern: @pattern)) => - {pattern} + pattern { pattern } ), // case T.foo(x,y) pattern rule!( @@ -463,10 +463,10 @@ fn translation_rules() -> Vec> { rule!( (function_parameter parameter: @@p default_value: _? @def) => - {{ + parameter* { ctx.default_value = def; ctx.translate(p)? - }} + } ), // Parameter with external name and type rule!( @@ -689,7 +689,7 @@ fn translation_rules() -> Vec> { element: (pattern_element pattern: (name_pattern identifier: (identifier #{name}))))) ), // If-condition — unwrap (pass through the inner expression/pattern) - rule!((if_condition kind: @inner) => {inner}), + rule!((if_condition kind: @inner) => expr_or_pattern { inner }), // ---- Loops ---- // For-in loop with optional where-clause guard. rule!( @@ -722,7 +722,7 @@ fn translation_rules() -> Vec> { body: (block stmt: {body})) ), // Labeled statement (e.g. `outer: for ...`). Strip the trailing ':' from the label token. - rule!((labeled_statement label: (statement_label) @lbl statement: @stmt) => { + rule!((labeled_statement label: (statement_label) @lbl statement: @stmt) => labeled_stmt { let text = ctx.ast.source_text(lbl); let name = &text[..text.len() - 1]; tree!((labeled_stmt label: (identifier #{name}) stmt: {stmt})) @@ -744,7 +744,7 @@ fn translation_rules() -> Vec> { rule!((dictionary_literal_item key: @k value: @v) => (key_value_pair key: {k} value: {v})), // ---- Optionals and errors ---- // Optional chaining — unwrap the marker - rule!((optional_chain_marker expr: @inner) => {inner}), + rule!((optional_chain_marker expr: @inner) => expr { inner }), // try/try?/try! expr → unary_expr with operator "try", "try?" or "try!" rule!((try_expression (try_operator) @op expr: @inner) => (unary_expr operator: (prefix_operator #{op}) operand: {inner})), rule!((try_expression operator: (try_operator) @op expr: @inner) => (unary_expr operator: (prefix_operator #{op}) operand: {inner})), @@ -800,7 +800,7 @@ fn translation_rules() -> Vec> { rule!( (identifier part: _+ @parts) => - {member_chain(&mut ctx, parts)} + expr { member_chain(&mut ctx, parts) } ), // Scoped import declaration (for example `import struct Foo.Bar`): // flatten the identifier parts into a member_access_expr and bind the @@ -831,7 +831,7 @@ fn translation_rules() -> Vec> { // Super expression → super_expr rule!((super_expression) => (super_expr)), // Modifiers — unwrap to individual modifier children - rule!((modifiers _* @mods) => {mods}), + rule!((modifiers _* @mods) => modifier* { mods }), rule!((attribute) @m => (modifier #{m})), rule!((visibility_modifier) @m => (modifier #{m})), rule!((function_modifier) @m => (modifier #{m})), @@ -843,7 +843,7 @@ fn translation_rules() -> Vec> { rule!((inheritance_modifier) @m => (modifier #{m})), rule!((property_behavior_modifier) @m => (modifier #{m})), // Type annotations — unwrap - rule!((type_annotation type: @inner) => {inner}), + rule!((type_annotation type: @inner) => type_expr { inner }), // user_type is split into simple_user_type parts. // Keep a conservative textual fallback to avoid dropping type information. rule!((user_type) @ty => (named_type_expr name: (identifier #{ty}))), @@ -1018,7 +1018,7 @@ fn translation_rules() -> Vec> { type: _? @ty (modifiers)* @mods) => - {{ + accessor_declaration* { ctx.property_name = Some(tree!((identifier #{name}))); ctx.property_type = ty; ctx.outer_modifiers = mods; @@ -1029,7 +1029,7 @@ fn translation_rules() -> Vec> { result.extend(ctx.translate(acc)?); } result - }} + } ), // getter_specifier / setter_specifier → bodyless accessor_declaration // getter_specifier / setter_specifier → bodyless @@ -1056,7 +1056,7 @@ fn translation_rules() -> Vec> { modifier: {chained_modifier(&mut ctx)}) ), // protocol_property_requirements wrapper — should be consumed by above; fallback - rule!((protocol_property_requirements accessor: _* @accs) => {accs}), + rule!((protocol_property_requirements accessor: _* @accs) => accessor_declaration* { accs }), // Computed getter → accessor_declaration (body optional). // Reads property name/type from the outer property_binding rule // and binding/outer modifiers + chained tag from the outer @@ -1116,7 +1116,7 @@ fn translation_rules() -> Vec> { // willset/didset block — spread to children (only reachable as a // fallback; the outer property_binding manual rule normally // captures the willset/didset clauses directly). - rule!((willset_didset_block _* @clauses) => {clauses}), + rule!((willset_didset_block _* @clauses) => accessor_declaration* { clauses }), // willset clause → accessor_declaration (body optional). Reads // `ctx.property_name` set by the outer property_binding rule and // binding/outer modifiers + chained tag from the outer @@ -1152,11 +1152,6 @@ fn translation_rules() -> Vec> { => (unsupported_node) ), - rule!( - _ @node - => - {node} - ), ] }